2020-08-24 19:12:15,596 INFO [Thread-925] org.apache.hadoop.hdfs.DataStreamer: Exception in createBlockOutputStream blk_1090328244_16602678
java.io.IOException: Broken pipe
at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47)
at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
at sun.nio.ch.IOUtil.write(IOUtil.java:65)
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:471)
at org.apache.hadoop.net.SocketOutputStream
W
r
i
t
e
r
.
p
e
r
f
o
r
m
I
O
(
S
o
c
k
e
t
O
u
t
p
u
t
S
t
r
e
a
m
.
j
a
v
a
:
63
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
n
e
t
.
S
o
c
k
e
t
I
O
W
i
t
h
T
i
m
e
o
u
t
.
d
o
I
O
(
S
o
c
k
e
t
I
O
W
i
t
h
T
i
m
e
o
u
t
.
j
a
v
a
:
142
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
n
e
t
.
S
o
c
k
e
t
O
u
t
p
u
t
S
t
r
e
a
m
.
w
r
i
t
e
(
S
o
c
k
e
t
O
u
t
p
u
t
S
t
r
e
a
m
.
j
a
v
a
:
159
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
n
e
t
.
S
o
c
k
e
t
O
u
t
p
u
t
S
t
r
e
a
m
.
w
r
i
t
e
(
S
o
c
k
e
t
O
u
t
p
u
t
S
t
r
e
a
m
.
j
a
v
a
:
117
)
a
t
j
a
v
a
.
i
o
.
B
u
f
f
e
r
e
d
O
u
t
p
u
t
S
t
r
e
a
m
.
w
r
i
t
e
(
B
u
f
f
e
r
e
d
O
u
t
p
u
t
S
t
r
e
a
m
.
j
a
v
a
:
122
)
a
t
j
a
v
a
.
i
o
.
D
a
t
a
O
u
t
p
u
t
S
t
r
e
a
m
.
w
r
i
t
e
(
D
a
t
a
O
u
t
p
u
t
S
t
r
e
a
m
.
j
a
v
a
:
107
)
a
t
c
o
m
.
g
o
o
g
l
e
.
p
r
o
t
o
b
u
f
.
C
o
d
e
d
O
u
t
p
u
t
S
t
r
e
a
m
.
r
e
f
r
e
s
h
B
u
f
f
e
r
(
C
o
d
e
d
O
u
t
p
u
t
S
t
r
e
a
m
.
j
a
v
a
:
833
)
a
t
c
o
m
.
g
o
o
g
l
e
.
p
r
o
t
o
b
u
f
.
C
o
d
e
d
O
u
t
p
u
t
S
t
r
e
a
m
.
f
l
u
s
h
(
C
o
d
e
d
O
u
t
p
u
t
S
t
r
e
a
m
.
j
a
v
a
:
843
)
a
t
c
o
m
.
g
o
o
g
l
e
.
p
r
o
t
o
b
u
f
.
A
b
s
t
r
a
c
t
M
e
s
s
a
g
e
L
i
t
e
.
w
r
i
t
e
D
e
l
i
m
i
t
e
d
T
o
(
A
b
s
t
r
a
c
t
M
e
s
s
a
g
e
L
i
t
e
.
j
a
v
a
:
91
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
p
r
o
t
o
c
o
l
.
d
a
t
a
t
r
a
n
s
f
e
r
.
S
e
n
d
e
r
.
s
e
n
d
(
S
e
n
d
e
r
.
j
a
v
a
:
82
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
p
r
o
t
o
c
o
l
.
d
a
t
a
t
r
a
n
s
f
e
r
.
S
e
n
d
e
r
.
w
r
i
t
e
B
l
o
c
k
(
S
e
n
d
e
r
.
j
a
v
a
:
170
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
.
c
r
e
a
t
e
B
l
o
c
k
O
u
t
p
u
t
S
t
r
e
a
m
(
D
a
t
a
S
t
r
e
a
m
e
r
.
j
a
v
a
:
1753
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
.
n
e
x
t
B
l
o
c
k
O
u
t
p
u
t
S
t
r
e
a
m
(
D
a
t
a
S
t
r
e
a
m
e
r
.
j
a
v
a
:
1679
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
.
r
u
n
(
D
a
t
a
S
t
r
e
a
m
e
r
.
j
a
v
a
:
716
)
2020
−
08
−
2419
:
12
:
15
,
604
W
A
R
N
[
T
h
r
e
a
d
−
925
]
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
:
A
b
a
n
d
o
n
i
n
g
B
P
−
396010141
−
192.168.1.50
−
1542955669571
:
b
l
k
1
09032824
4
1
66026782020
−
08
−
2419
:
12
:
15
,
621
W
A
R
N
[
T
h
r
e
a
d
−
925
]
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
:
E
x
c
l
u
d
i
n
g
d
a
t
a
n
o
d
e
D
a
t
a
n
o
d
e
I
n
f
o
W
i
t
h
S
t
o
r
a
g
e
[
192.168.1.59
:
9866
,
D
S
−
9022681
a
−
a
071
−
40
b
8
−
9395
−
0
d
a
664
a
3
f
313
,
D
I
S
K
]
2020
−
08
−
2419
:
12
:
15
,
641
I
N
F
O
[
T
h
r
e
a
d
−
925
]
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
:
E
x
c
e
p
t
i
o
n
i
n
c
r
e
a
t
e
B
l
o
c
k
O
u
t
p
u
t
S
t
r
e
a
m
b
l
k
1
09032824
5
1
6602679
j
a
v
a
.
i
o
.
E
O
F
E
x
c
e
p
t
i
o
n
:
U
n
e
x
p
e
c
t
e
d
E
O
F
w
h
i
l
e
t
r
y
i
n
g
t
o
r
e
a
d
r
e
s
p
o
n
s
e
f
r
o
m
s
e
r
v
e
r
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
p
r
o
t
o
c
o
l
P
B
.
P
B
H
e
l
p
e
r
C
l
i
e
n
t
.
v
i
n
t
P
r
e
f
i
x
e
d
(
P
B
H
e
l
p
e
r
C
l
i
e
n
t
.
j
a
v
a
:
549
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
.
c
r
e
a
t
e
B
l
o
c
k
O
u
t
p
u
t
S
t
r
e
a
m
(
D
a
t
a
S
t
r
e
a
m
e
r
.
j
a
v
a
:
1762
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
.
n
e
x
t
B
l
o
c
k
O
u
t
p
u
t
S
t
r
e
a
m
(
D
a
t
a
S
t
r
e
a
m
e
r
.
j
a
v
a
:
1679
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
.
r
u
n
(
D
a
t
a
S
t
r
e
a
m
e
r
.
j
a
v
a
:
716
)
2020
−
08
−
2419
:
12
:
15
,
642
W
A
R
N
[
T
h
r
e
a
d
−
925
]
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
:
A
b
a
n
d
o
n
i
n
g
B
P
−
396010141
−
192.168.1.50
−
1542955669571
:
b
l
k
1
09032824
5
1
66026792020
−
08
−
2419
:
12
:
15
,
649
W
A
R
N
[
T
h
r
e
a
d
−
925
]
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
:
E
x
c
l
u
d
i
n
g
d
a
t
a
n
o
d
e
D
a
t
a
n
o
d
e
I
n
f
o
W
i
t
h
S
t
o
r
a
g
e
[
192.168.1.56
:
9866
,
D
S
−
64514098
−
c
4
c
f
−
4
a
9
c
−
9
c
83
−
8
a
49342
f
a
b
73
,
D
I
S
K
]
2020
−
08
−
2419
:
12
:
15
,
671
I
N
F
O
[
T
h
r
e
a
d
−
925
]
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
:
E
x
c
e
p
t
i
o
n
i
n
c
r
e
a
t
e
B
l
o
c
k
O
u
t
p
u
t
S
t
r
e
a
m
b
l
k
1
09032824
6
1
6602680
j
a
v
a
.
i
o
.
I
O
E
x
c
e
p
t
i
o
n
:
G
o
t
e
r
r
o
r
,
s
t
a
t
u
s
=
E
R
R
O
R
,
s
t
a
t
u
s
m
e
s
s
a
g
e
,
a
c
k
w
i
t
h
f
i
r
s
t
B
a
d
L
i
n
k
a
s
192.168.1.61
:
9866
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
p
r
o
t
o
c
o
l
.
d
a
t
a
t
r
a
n
s
f
e
r
.
D
a
t
a
T
r
a
n
s
f
e
r
P
r
o
t
o
U
t
i
l
.
c
h
e
c
k
B
l
o
c
k
O
p
S
t
a
t
u
s
(
D
a
t
a
T
r
a
n
s
f
e
r
P
r
o
t
o
U
t
i
l
.
j
a
v
a
:
134
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
p
r
o
t
o
c
o
l
.
d
a
t
a
t
r
a
n
s
f
e
r
.
D
a
t
a
T
r
a
n
s
f
e
r
P
r
o
t
o
U
t
i
l
.
c
h
e
c
k
B
l
o
c
k
O
p
S
t
a
t
u
s
(
D
a
t
a
T
r
a
n
s
f
e
r
P
r
o
t
o
U
t
i
l
.
j
a
v
a
:
110
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
.
c
r
e
a
t
e
B
l
o
c
k
O
u
t
p
u
t
S
t
r
e
a
m
(
D
a
t
a
S
t
r
e
a
m
e
r
.
j
a
v
a
:
1778
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
.
n
e
x
t
B
l
o
c
k
O
u
t
p
u
t
S
t
r
e
a
m
(
D
a
t
a
S
t
r
e
a
m
e
r
.
j
a
v
a
:
1679
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
.
r
u
n
(
D
a
t
a
S
t
r
e
a
m
e
r
.
j
a
v
a
:
716
)
2020
−
08
−
2419
:
12
:
15
,
672
W
A
R
N
[
T
h
r
e
a
d
−
925
]
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
:
A
b
a
n
d
o
n
i
n
g
B
P
−
396010141
−
192.168.1.50
−
1542955669571
:
b
l
k
1
09032824
6
1
66026802020
−
08
−
2419
:
12
:
15
,
679
W
A
R
N
[
T
h
r
e
a
d
−
925
]
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
:
E
x
c
l
u
d
i
n
g
d
a
t
a
n
o
d
e
D
a
t
a
n
o
d
e
I
n
f
o
W
i
t
h
S
t
o
r
a
g
e
[
192.168.1.61
:
9866
,
D
S
−
d
940267
b
−
b
889
−
4610
−
a
170
−
2660
c
1882
e
b
0
,
D
I
S
K
]
2020
−
08
−
2419
:
12
:
15
,
691
I
N
F
O
[
T
h
r
e
a
d
−
925
]
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
:
E
x
c
e
p
t
i
o
n
i
n
c
r
e
a
t
e
B
l
o
c
k
O
u
t
p
u
t
S
t
r
e
a
m
b
l
k
1
09032824
7
1
6602681
j
a
v
a
.
i
o
.
E
O
F
E
x
c
e
p
t
i
o
n
:
U
n
e
x
p
e
c
t
e
d
E
O
F
w
h
i
l
e
t
r
y
i
n
g
t
o
r
e
a
d
r
e
s
p
o
n
s
e
f
r
o
m
s
e
r
v
e
r
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
p
r
o
t
o
c
o
l
P
B
.
P
B
H
e
l
p
e
r
C
l
i
e
n
t
.
v
i
n
t
P
r
e
f
i
x
e
d
(
P
B
H
e
l
p
e
r
C
l
i
e
n
t
.
j
a
v
a
:
549
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
.
c
r
e
a
t
e
B
l
o
c
k
O
u
t
p
u
t
S
t
r
e
a
m
(
D
a
t
a
S
t
r
e
a
m
e
r
.
j
a
v
a
:
1762
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
.
n
e
x
t
B
l
o
c
k
O
u
t
p
u
t
S
t
r
e
a
m
(
D
a
t
a
S
t
r
e
a
m
e
r
.
j
a
v
a
:
1679
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
.
r
u
n
(
D
a
t
a
S
t
r
e
a
m
e
r
.
j
a
v
a
:
716
)
2020
−
08
−
2419
:
12
:
15
,
691
W
A
R
N
[
T
h
r
e
a
d
−
925
]
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
:
A
b
a
n
d
o
n
i
n
g
B
P
−
396010141
−
192.168.1.50
−
1542955669571
:
b
l
k
1
09032824
7
1
66026812020
−
08
−
2419
:
12
:
15
,
697
W
A
R
N
[
T
h
r
e
a
d
−
925
]
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
:
E
x
c
l
u
d
i
n
g
d
a
t
a
n
o
d
e
D
a
t
a
n
o
d
e
I
n
f
o
W
i
t
h
S
t
o
r
a
g
e
[
192.168.1.54
:
9866
,
D
S
−
4
f
e
b
1760
−
4119
−
41
a
3
−
8
d
b
d
−
e
a
5
f
d
40787
a
f
,
D
I
S
K
]
2020
−
08
−
2419
:
12
:
15
,
706
W
A
R
N
[
T
h
r
e
a
d
−
925
]
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
:
D
a
t
a
S
t
r
e
a
m
e
r
E
x
c
e
p
t
i
o
n
j
a
v
a
.
i
o
.
I
O
E
x
c
e
p
t
i
o
n
:
U
n
a
b
l
e
t
o
c
r
e
a
t
e
n
e
w
b
l
o
c
k
.
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
.
n
e
x
t
B
l
o
c
k
O
u
t
p
u
t
S
t
r
e
a
m
(
D
a
t
a
S
t
r
e
a
m
e
r
.
j
a
v
a
:
1694
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
.
r
u
n
(
D
a
t
a
S
t
r
e
a
m
e
r
.
j
a
v
a
:
716
)
2020
−
08
−
2419
:
12
:
15
,
725
W
A
R
N
[
T
h
r
e
a
d
−
925
]
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
:
C
o
u
l
d
n
o
t
g
e
t
b
l
o
c
k
l
o
c
a
t
i
o
n
s
.
S
o
u
r
c
e
f
i
l
e
"
X
X
X
"
−
A
b
o
r
t
i
n
g
.
.
.
b
l
o
c
k
=
=
n
u
l
l
2020
−
08
−
2419
:
12
:
17
,
732
I
N
F
O
[
m
a
i
n
]
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
m
a
p
r
e
d
.
M
a
p
T
a
s
k
:
I
g
n
o
r
i
n
g
e
x
c
e
p
t
i
o
n
d
u
r
i
n
g
c
l
o
s
e
f
o
r
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
m
a
p
r
e
d
.
M
a
p
T
a
s
k
Writer.performIO(SocketOutputStream.java:63) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142) at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:159) at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:117) at java.io.BufferedOutputStream.write(BufferedOutputStream.java:122) at java.io.DataOutputStream.write(DataOutputStream.java:107) at com.google.protobuf.CodedOutputStream.refreshBuffer(CodedOutputStream.java:833) at com.google.protobuf.CodedOutputStream.flush(CodedOutputStream.java:843) at com.google.protobuf.AbstractMessageLite.writeDelimitedTo(AbstractMessageLite.java:91) at org.apache.hadoop.hdfs.protocol.datatransfer.Sender.send(Sender.java:82) at org.apache.hadoop.hdfs.protocol.datatransfer.Sender.writeBlock(Sender.java:170) at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1753) at org.apache.hadoop.hdfs.DataStreamer.nextBlockOutputStream(DataStreamer.java:1679) at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:716) 2020-08-24 19:12:15,604 WARN [Thread-925] org.apache.hadoop.hdfs.DataStreamer: Abandoning BP-396010141-192.168.1.50-1542955669571:blk_1090328244_16602678 2020-08-24 19:12:15,621 WARN [Thread-925] org.apache.hadoop.hdfs.DataStreamer: Excluding datanode DatanodeInfoWithStorage[192.168.1.59:9866,DS-9022681a-a071-40b8-9395-0da664a3f313,DISK] 2020-08-24 19:12:15,641 INFO [Thread-925] org.apache.hadoop.hdfs.DataStreamer: Exception in createBlockOutputStream blk_1090328245_16602679 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:549) at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1762) at org.apache.hadoop.hdfs.DataStreamer.nextBlockOutputStream(DataStreamer.java:1679) at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:716) 2020-08-24 19:12:15,642 WARN [Thread-925] org.apache.hadoop.hdfs.DataStreamer: Abandoning BP-396010141-192.168.1.50-1542955669571:blk_1090328245_16602679 2020-08-24 19:12:15,649 WARN [Thread-925] org.apache.hadoop.hdfs.DataStreamer: Excluding datanode DatanodeInfoWithStorage[192.168.1.56:9866,DS-64514098-c4cf-4a9c-9c83-8a49342fab73,DISK] 2020-08-24 19:12:15,671 INFO [Thread-925] org.apache.hadoop.hdfs.DataStreamer: Exception in createBlockOutputStream blk_1090328246_16602680 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 192.168.1.61:9866 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:134) at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:110) at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1778) at org.apache.hadoop.hdfs.DataStreamer.nextBlockOutputStream(DataStreamer.java:1679) at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:716) 2020-08-24 19:12:15,672 WARN [Thread-925] org.apache.hadoop.hdfs.DataStreamer: Abandoning BP-396010141-192.168.1.50-1542955669571:blk_1090328246_16602680 2020-08-24 19:12:15,679 WARN [Thread-925] org.apache.hadoop.hdfs.DataStreamer: Excluding datanode DatanodeInfoWithStorage[192.168.1.61:9866,DS-d940267b-b889-4610-a170-2660c1882eb0,DISK] 2020-08-24 19:12:15,691 INFO [Thread-925] org.apache.hadoop.hdfs.DataStreamer: Exception in createBlockOutputStream blk_1090328247_16602681 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:549) at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1762) at org.apache.hadoop.hdfs.DataStreamer.nextBlockOutputStream(DataStreamer.java:1679) at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:716) 2020-08-24 19:12:15,691 WARN [Thread-925] org.apache.hadoop.hdfs.DataStreamer: Abandoning BP-396010141-192.168.1.50-1542955669571:blk_1090328247_16602681 2020-08-24 19:12:15,697 WARN [Thread-925] org.apache.hadoop.hdfs.DataStreamer: Excluding datanode DatanodeInfoWithStorage[192.168.1.54:9866,DS-4feb1760-4119-41a3-8dbd-ea5fd40787af,DISK] 2020-08-24 19:12:15,706 WARN [Thread-925] org.apache.hadoop.hdfs.DataStreamer: DataStreamer Exception java.io.IOException: Unable to create new block. at org.apache.hadoop.hdfs.DataStreamer.nextBlockOutputStream(DataStreamer.java:1694) at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:716) 2020-08-24 19:12:15,725 WARN [Thread-925] org.apache.hadoop.hdfs.DataStreamer: Could not get block locations. Source file "XXX" - Aborting...block==null 2020-08-24 19:12:17,732 INFO [main] org.apache.hadoop.mapred.MapTask: Ignoring exception during close for org.apache.hadoop.mapred.MapTask
Writer.performIO(SocketOutputStream.java:63)atorg.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142)atorg.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:159)atorg.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:117)atjava.io.BufferedOutputStream.write(BufferedOutputStream.java:122)atjava.io.DataOutputStream.write(DataOutputStream.java:107)atcom.google.protobuf.CodedOutputStream.refreshBuffer(CodedOutputStream.java:833)atcom.google.protobuf.CodedOutputStream.flush(CodedOutputStream.java:843)atcom.google.protobuf.AbstractMessageLite.writeDelimitedTo(AbstractMessageLite.java:91)atorg.apache.hadoop.hdfs.protocol.datatransfer.Sender.send(Sender.java:82)atorg.apache.hadoop.hdfs.protocol.datatransfer.Sender.writeBlock(Sender.java:170)atorg.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1753)atorg.apache.hadoop.hdfs.DataStreamer.nextBlockOutputStream(DataStreamer.java:1679)atorg.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:716)2020−08−2419:12:15,604WARN[Thread−925]org.apache.hadoop.hdfs.DataStreamer:AbandoningBP−396010141−192.168.1.50−1542955669571:blk1090328244166026782020−08−2419:12:15,621WARN[Thread−925]org.apache.hadoop.hdfs.DataStreamer:ExcludingdatanodeDatanodeInfoWithStorage[192.168.1.59:9866,DS−9022681a−a071−40b8−9395−0da664a3f313,DISK]2020−08−2419:12:15,641INFO[Thread−925]org.apache.hadoop.hdfs.DataStreamer:ExceptionincreateBlockOutputStreamblk109032824516602679java.io.EOFException:UnexpectedEOFwhiletryingtoreadresponsefromserveratorg.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:549)atorg.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1762)atorg.apache.hadoop.hdfs.DataStreamer.nextBlockOutputStream(DataStreamer.java:1679)atorg.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:716)2020−08−2419:12:15,642WARN[Thread−925]org.apache.hadoop.hdfs.DataStreamer:AbandoningBP−396010141−192.168.1.50−1542955669571:blk1090328245166026792020−08−2419:12:15,649WARN[Thread−925]org.apache.hadoop.hdfs.DataStreamer:ExcludingdatanodeDatanodeInfoWithStorage[192.168.1.56:9866,DS−64514098−c4cf−4a9c−9c83−8a49342fab73,DISK]2020−08−2419:12:15,671INFO[Thread−925]org.apache.hadoop.hdfs.DataStreamer:ExceptionincreateBlockOutputStreamblk109032824616602680java.io.IOException:Goterror,status=ERROR,statusmessage,ackwithfirstBadLinkas192.168.1.61:9866atorg.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:134)atorg.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:110)atorg.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1778)atorg.apache.hadoop.hdfs.DataStreamer.nextBlockOutputStream(DataStreamer.java:1679)atorg.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:716)2020−08−2419:12:15,672WARN[Thread−925]org.apache.hadoop.hdfs.DataStreamer:AbandoningBP−396010141−192.168.1.50−1542955669571:blk1090328246166026802020−08−2419:12:15,679WARN[Thread−925]org.apache.hadoop.hdfs.DataStreamer:ExcludingdatanodeDatanodeInfoWithStorage[192.168.1.61:9866,DS−d940267b−b889−4610−a170−2660c1882eb0,DISK]2020−08−2419:12:15,691INFO[Thread−925]org.apache.hadoop.hdfs.DataStreamer:ExceptionincreateBlockOutputStreamblk109032824716602681java.io.EOFException:UnexpectedEOFwhiletryingtoreadresponsefromserveratorg.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:549)atorg.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1762)atorg.apache.hadoop.hdfs.DataStreamer.nextBlockOutputStream(DataStreamer.java:1679)atorg.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:716)2020−08−2419:12:15,691WARN[Thread−925]org.apache.hadoop.hdfs.DataStreamer:AbandoningBP−396010141−192.168.1.50−1542955669571:blk1090328247166026812020−08−2419:12:15,697WARN[Thread−925]org.apache.hadoop.hdfs.DataStreamer:ExcludingdatanodeDatanodeInfoWithStorage[192.168.1.54:9866,DS−4feb1760−4119−41a3−8dbd−ea5fd40787af,DISK]2020−08−2419:12:15,706WARN[Thread−925]org.apache.hadoop.hdfs.DataStreamer:DataStreamerExceptionjava.io.IOException:Unabletocreatenewblock.atorg.apache.hadoop.hdfs.DataStreamer.nextBlockOutputStream(DataStreamer.java:1694)atorg.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:716)2020−08−2419:12:15,725WARN[Thread−925]org.apache.hadoop.hdfs.DataStreamer:Couldnotgetblocklocations.Sourcefile"XXX"−Aborting...block==null2020−08−2419:12:17,732INFO[main]org.apache.hadoop.mapred.MapTask:Ignoringexceptionduringclosefororg.apache.hadoop.mapred.MapTaskNewDirectOutputCollector@21aa5fdf
java.lang.IllegalArgumentException: Self-suppression not permitted
at java.lang.Throwable.addSuppressed(Throwable.java:1043)
at java.io.FilterOutputStream.close(FilterOutputStream.java:159)
at com.io.LineRecordWriter.close(LineRecordWriter.java:80)
at com.io.MultipleOutputFormat
M
u
l
t
i
R
e
c
o
r
d
W
r
i
t
e
r
.
c
l
o
s
e
(
M
u
l
t
i
p
l
e
O
u
t
p
u
t
F
o
r
m
a
t
.
j
a
v
a
:
111
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
m
a
p
r
e
d
.
M
a
p
T
a
s
k
MultiRecordWriter.close(MultipleOutputFormat.java:111) at org.apache.hadoop.mapred.MapTask
MultiRecordWriter.close(MultipleOutputFormat.java:111)atorg.apache.hadoop.mapred.MapTaskNewDirectOutputCollector.close(MapTask.java:682)
at org.apache.hadoop.mapred.MapTask.closeQuietly(MapTask.java:2076)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:809)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:347)
at org.apache.hadoop.mapred.YarnChild
2.
r
u
n
(
Y
a
r
n
C
h
i
l
d
.
j
a
v
a
:
174
)
a
t
j
a
v
a
.
s
e
c
u
r
i
t
y
.
A
c
c
e
s
s
C
o
n
t
r
o
l
l
e
r
.
d
o
P
r
i
v
i
l
e
g
e
d
(
N
a
t
i
v
e
M
e
t
h
o
d
)
a
t
j
a
v
a
x
.
s
e
c
u
r
i
t
y
.
a
u
t
h
.
S
u
b
j
e
c
t
.
d
o
A
s
(
S
u
b
j
e
c
t
.
j
a
v
a
:
422
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
s
e
c
u
r
i
t
y
.
U
s
e
r
G
r
o
u
p
I
n
f
o
r
m
a
t
i
o
n
.
d
o
A
s
(
U
s
e
r
G
r
o
u
p
I
n
f
o
r
m
a
t
i
o
n
.
j
a
v
a
:
1729
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
m
a
p
r
e
d
.
Y
a
r
n
C
h
i
l
d
.
m
a
i
n
(
Y
a
r
n
C
h
i
l
d
.
j
a
v
a
:
168
)
C
a
u
s
e
d
b
y
:
j
a
v
a
.
i
o
.
I
O
E
x
c
e
p
t
i
o
n
:
C
o
u
l
d
n
o
t
g
e
t
b
l
o
c
k
l
o
c
a
t
i
o
n
s
.
S
o
u
r
c
e
f
i
l
e
"
X
X
X
X
"
−
A
b
o
r
t
i
n
g
.
.
.
b
l
o
c
k
=
=
n
u
l
l
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
.
s
e
t
u
p
P
i
p
e
l
i
n
e
F
o
r
A
p
p
e
n
d
O
r
R
e
c
o
v
e
r
y
(
D
a
t
a
S
t
r
e
a
m
e
r
.
j
a
v
a
:
1477
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
.
p
r
o
c
e
s
s
D
a
t
a
n
o
d
e
O
r
E
x
t
e
r
n
a
l
E
r
r
o
r
(
D
a
t
a
S
t
r
e
a
m
e
r
.
j
a
v
a
:
1256
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
a
t
a
S
t
r
e
a
m
e
r
.
r
u
n
(
D
a
t
a
S
t
r
e
a
m
e
r
.
j
a
v
a
:
667
)
2020
−
08
−
2419
:
12
:
17
,
733
W
A
R
N
[
m
a
i
n
]
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
m
a
p
r
e
d
.
Y
a
r
n
C
h
i
l
d
:
E
x
c
e
p
t
i
o
n
r
u
n
n
i
n
g
c
h
i
l
d
:
j
a
v
a
.
i
o
.
I
O
E
x
c
e
p
t
i
o
n
:
w
r
i
t
e
b
e
y
o
n
d
e
n
d
o
f
s
t
r
e
a
m
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
i
o
.
c
o
m
p
r
e
s
s
.
B
l
o
c
k
C
o
m
p
r
e
s
s
o
r
S
t
r
e
a
m
.
w
r
i
t
e
(
B
l
o
c
k
C
o
m
p
r
e
s
s
o
r
S
t
r
e
a
m
.
j
a
v
a
:
85
)
a
t
j
a
v
a
.
i
o
.
D
a
t
a
O
u
t
p
u
t
S
t
r
e
a
m
.
w
r
i
t
e
(
D
a
t
a
O
u
t
p
u
t
S
t
r
e
a
m
.
j
a
v
a
:
107
)
a
t
c
o
m
.
i
o
.
L
i
n
e
R
e
c
o
r
d
W
r
i
t
e
r
.
w
r
i
t
e
O
b
j
e
c
t
(
L
i
n
e
R
e
c
o
r
d
W
r
i
t
e
r
.
j
a
v
a
:
52
)
a
t
c
o
m
.
i
o
.
L
i
n
e
R
e
c
o
r
d
W
r
i
t
e
r
.
w
r
i
t
e
(
L
i
n
e
R
e
c
o
r
d
W
r
i
t
e
r
.
j
a
v
a
:
74
)
a
t
c
o
m
.
i
o
.
M
u
l
t
i
p
l
e
O
u
t
p
u
t
F
o
r
m
a
t
2.run(YarnChild.java:174) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1729) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:168) Caused by: java.io.IOException: Could not get block locations. Source file "XXXX" - Aborting...block==null at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1477) at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1256) at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:667) 2020-08-24 19:12:17,733 WARN [main] org.apache.hadoop.mapred.YarnChild: Exception running child : java.io.IOException: write beyond end of stream at org.apache.hadoop.io.compress.BlockCompressorStream.write(BlockCompressorStream.java:85) at java.io.DataOutputStream.write(DataOutputStream.java:107) at com.io.LineRecordWriter.writeObject(LineRecordWriter.java:52) at com.io.LineRecordWriter.write(LineRecordWriter.java:74) at com.io.MultipleOutputFormat
2.run(YarnChild.java:174)atjava.security.AccessController.doPrivileged(NativeMethod)atjavax.security.auth.Subject.doAs(Subject.java:422)atorg.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1729)atorg.apache.hadoop.mapred.YarnChild.main(YarnChild.java:168)Causedby:java.io.IOException:Couldnotgetblocklocations.Sourcefile"XXXX"−Aborting...block==nullatorg.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1477)atorg.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1256)atorg.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:667)2020−08−2419:12:17,733WARN[main]org.apache.hadoop.mapred.YarnChild:Exceptionrunningchild:java.io.IOException:writebeyondendofstreamatorg.apache.hadoop.io.compress.BlockCompressorStream.write(BlockCompressorStream.java:85)atjava.io.DataOutputStream.write(DataOutputStream.java:107)atcom.io.LineRecordWriter.writeObject(LineRecordWriter.java:52)atcom.io.LineRecordWriter.write(LineRecordWriter.java:74)atcom.io.MultipleOutputFormatMultiRecordWriter.write(MultipleOutputFormat.java:148)
at com.fio.MultipleOutputFormat
M
u
l
t
i
R
e
c
o
r
d
W
r
i
t
e
r
.
w
r
i
t
e
(
M
u
l
t
i
p
l
e
O
u
t
p
u
t
F
o
r
m
a
t
.
j
a
v
a
:
1
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
m
a
p
r
e
d
.
M
a
p
T
a
s
k
MultiRecordWriter.write(MultipleOutputFormat.java:1) at org.apache.hadoop.mapred.MapTask
MultiRecordWriter.write(MultipleOutputFormat.java:1)atorg.apache.hadoop.mapred.MapTaskNewDirectOutputCollector.write(MapTask.java:670)
at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89)
at org.apache.hadoop.mapreduce.lib.map.WrappedMapper$Context.write(WrappedMapper.java:112)
at com…map(SplitMapper.java:262)
at com…map(SplitMapper.java:1)
at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:146)
at org.apache.hadoop.mapreduce.lib.input.DelegatingMapper.run(DelegatingMapper.java:55)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:799)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:347)
at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:174)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1729)
at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:168)
主要是每台机器map的task数调的太多了，导致IO到瓶颈，卡主了。
把任务的内存调大点，map数就自然减小了。
本人鄙视网上资料的一言不合就是namenode format。。。。
扫一扫
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
