int h = PipelineAck.combineHeader(datanode.getECN(), Status.SUCCESS); int h1 = PipelineAck.combineHeader(datanode.getECN(), Status.ERROR); replies = new int[] {h, h1}; } else { short ackLen = type == PacketResponderType.LAST_IN_PIPELINE ? 0 : ack .getNumOfReplies(); replies = new int[ackLen + 1]; replies[0] = myHeader; for (int i = 0; i < ackLen; ++i) { replies[i + 1] = ack.getHeaderFlag(i); if (ackLen > 0 && PipelineAck.getStatusFromHeader(replies[1]) == Status.ERROR_CHECKSUM) { throw new IOException("Shutting down writer and responder " PipelineAck replyAck = new PipelineAck(seqno, replies, totalAckTimeNanos); if (replyAck.isSuccess() && offsetInBlock > replicaInfo.getBytesAcked()) { replicaInfo.setBytesAcked(offsetInBlock); replyAck.write(upstreamOut); upstreamOut.flush(); long duration = Time.monotonicNow() - begin; Status myStatus = PipelineAck.getStatusFromHeader(myHeader); if (myStatus == Status.ERROR_CHECKSUM) { throw new IOException("Shutting down writer and responder "
Packet pkt = null; long expected = -2; PipelineAck ack = new PipelineAck(); long seqno = PipelineAck.UNKOWN_SEQNO; long ackRecvNanoTime = 0; ack.readFields(downstreamIn); ackRecvNanoTime = System.nanoTime(); if (LOG.isDebugEnabled()) { Status oobStatus = ack.getOOBStatus(); if (oobStatus != null) { LOG.info("Relaying an out of band ack of type " + oobStatus); sendAckUpstream(ack, PipelineAck.UNKOWN_SEQNO, 0L, 0L, PipelineAck.combineHeader(datanode.getECN(), Status.SUCCESS)); continue; seqno = ack.getSeqno(); - ack.getDownstreamAckTimeNanos(); if (ackTimeNanos < 0) { if (LOG.isDebugEnabled()) { sendAckUpstream(ack, expected, totalAckTimeNanos, (pkt != null ? pkt.offsetInBlock : 0), PipelineAck.combineHeader(datanode.getECN(), myStatus)); if (pkt != null) {
@Override protected void channelRead0(ChannelHandlerContext ctx, PipelineAckProto ack) throws Exception { Status reply = getStatus(ack); if (reply != Status.SUCCESS) { failed(ctx.channel(), () -> new IOException("Bad response " + reply + " for block " + block + " from datanode " + ctx.channel().remoteAddress())); return; } if (PipelineAck.isRestartOOBStatus(reply)) { failed(ctx.channel(), () -> new IOException("Restart response " + reply + " for block " + block + " from datanode " + ctx.channel().remoteAddress())); return; } if (ack.getSeqno() == HEART_BEAT_SEQNO) { return; } completed(ctx.channel()); }
PipelineAck ack = new PipelineAck(); ack.readFields(blockReplyStream); if (ack.getSeqno() != DFSPacket.HEART_BEAT_SEQNO) { Long begin = packetSendTime.get(ack.getSeqno()); if (begin != null) { long duration = Time.monotonicNow() - begin; long seqno = ack.getSeqno(); for (int i = ack.getNumOfReplies()-1; i >=0 && dfsClient.clientRunning; i--) { final Status reply = PipelineAck.getStatusFromHeader(ack .getHeaderFlag(i)); if (PipelineAck.getECNFromHeader(ack.getHeaderFlag(i)) == PipelineAck.ECN.CONGESTED) { congestedNodesFromAck.add(targets[i]); if (PipelineAck.isRestartOOBStatus(reply)) { final String message = "Datanode " + i + " is restarting: " + targets[i];
@Test public void TestPipeLineAckCompatibility() throws IOException { DataTransferProtos.PipelineAckProto proto = DataTransferProtos .PipelineAckProto.newBuilder() .setSeqno(0) .addReply(Status.CHECKSUM_OK) .build(); DataTransferProtos.PipelineAckProto newProto = DataTransferProtos .PipelineAckProto.newBuilder().mergeFrom(proto) .addFlag(PipelineAck.combineHeader(PipelineAck.ECN.SUPPORTED, Status.CHECKSUM_OK)) .build(); ByteArrayOutputStream oldAckBytes = new ByteArrayOutputStream(); proto.writeDelimitedTo(oldAckBytes); PipelineAck oldAck = new PipelineAck(); oldAck.readFields(new ByteArrayInputStream(oldAckBytes.toByteArray())); assertEquals( PipelineAck.combineHeader(PipelineAck.ECN.DISABLED, Status.CHECKSUM_OK), oldAck.getHeaderFlag(0)); PipelineAck newAck = new PipelineAck(); ByteArrayOutputStream newAckBytes = new ByteArrayOutputStream(); newProto.writeDelimitedTo(newAckBytes); newAck.readFields(new ByteArrayInputStream(newAckBytes.toByteArray())); assertEquals(PipelineAck.combineHeader(PipelineAck.ECN.SUPPORTED, Status.CHECKSUM_OK), newAck.getHeaderFlag(0)); }
private void writeZeroLengthPacket(ExtendedBlock block, String description) throws IOException { PacketHeader hdr = new PacketHeader( 8, // size of packet block.getNumBytes(), // OffsetInBlock 100, // sequencenumber true, // lastPacketInBlock 0, // chunk length false); // sync block hdr.write(sendOut); sendOut.writeInt(0); // zero checksum //ok finally write a block with 0 len sendResponse(Status.SUCCESS, "", null, recvOut); new PipelineAck(100, new int[] {PipelineAck.combineHeader (PipelineAck.ECN.DISABLED, Status.SUCCESS)}).write (recvOut); sendRecvData(description, false); }
try { sendAckUpstreamUnprotected(null, PipelineAck.UNKOWN_SEQNO, 0L, 0L, PipelineAck.combineHeader(datanode.getECN(), ackStatus)); } finally {
wait(PipelineAck.getOOBTimeout(ackStatus)); try { sendAckUpstreamUnprotected(null, PipelineAck.UNKOWN_SEQNO, 0L, 0L, PipelineAck.combineHeader(datanode.getECN(), ackStatus)); } finally {
/** * Returns the OOB status if this ack contains one. * @return null if it is not an OOB ack. */ public Status getOOBStatus() { // Normal data transfer acks will have a valid sequence number, so // this will return right away in most cases. if (getSeqno() != UNKOWN_SEQNO) { return null; } for (Status s : proto.getReplyList()) { // The following check is valid because protobuf guarantees to // preserve the ordering of enum elements. if (s.getNumber() >= OOB_START && s.getNumber() <= OOB_END) { return s; } } return null; }
PipelineAck ack = new PipelineAck(); ack.readFields(blockReplyStream); long duration = Time.monotonicNow() - begin; if (duration > dfsclientSlowLogThresholdMs && ack.getSeqno() != DFSPacket.HEART_BEAT_SEQNO) { DFSClient.LOG .warn("Slow ReadProcessor read fields took " + duration long seqno = ack.getSeqno(); for (int i = ack.getNumOfReplies()-1; i >=0 && dfsClient.clientRunning; i--) { final Status reply = PipelineAck.getStatusFromHeader(ack .getHeaderFlag(i)); if (PipelineAck.isRestartOOBStatus(reply)) { if (shouldWaitForRestart(i)) { restartDeadline = dfsClient.getConf().datanodeRestartTimeout
new PipelineAck(100, new int[] {PipelineAck.combineHeader (PipelineAck.ECN.DISABLED, Status.ERROR)}).write(recvOut); sendRecvData("negative DATA_CHUNK len while writing block " + newBlockId, true); new PipelineAck(100, new int[] {PipelineAck.combineHeader (PipelineAck.ECN.DISABLED, Status.SUCCESS)}).write(recvOut); sendRecvData("Writing a zero len block blockid " + newBlockId, false);
/** * get the header flag of ith reply */ public int getHeaderFlag(int i) { if (proto.getFlagCount() > 0) { return proto.getFlag(i); } else { return combineHeader(ECN.DISABLED, proto.getReply(i)); } }
wait(PipelineAck.getOOBTimeout(ackStatus)); try { sendAckUpstreamUnprotected(null, PipelineAck.UNKOWN_SEQNO, 0L, 0L, PipelineAck.combineHeader(datanode.getECN(), ackStatus)); } finally {
/** * Returns the OOB status if this ack contains one. * @return null if it is not an OOB ack. */ public Status getOOBStatus() { // Normal data transfer acks will have a valid sequence number, so // this will return right away in most cases. if (getSeqno() != UNKOWN_SEQNO) { return null; } for (Status s : proto.getReplyList()) { // The following check is valid because protobuf guarantees to // preserve the ordering of enum elements. if (s.getNumber() >= OOB_START && s.getNumber() <= OOB_END) { return s; } } return null; }
PipelineAck ack = new PipelineAck(); ack.readFields(blockReplyStream); long duration = Time.monotonicNow() - begin; if (duration > dfsclientSlowLogThresholdMs && ack.getSeqno() != DFSPacket.HEART_BEAT_SEQNO) { DFSClient.LOG .warn("Slow ReadProcessor read fields took " + duration long seqno = ack.getSeqno(); for (int i = ack.getNumOfReplies()-1; i >=0 && dfsClient.clientRunning; i--) { final Status reply = PipelineAck.getStatusFromHeader(ack .getHeaderFlag(i)); if (PipelineAck.isRestartOOBStatus(reply)) { if (shouldWaitForRestart(i)) { restartDeadline = dfsClient.getConf().datanodeRestartTimeout
int h = PipelineAck.combineHeader(datanode.getECN(), Status.SUCCESS); int h1 = PipelineAck.combineHeader(datanode.getECN(), Status.ERROR); replies = new int[] {h, h1}; } else { short ackLen = type == PacketResponderType.LAST_IN_PIPELINE ? 0 : ack .getNumOfReplies(); replies = new int[ackLen + 1]; replies[0] = myHeader; for (int i = 0; i < ackLen; ++i) { replies[i + 1] = ack.getHeaderFlag(i); if (ackLen > 0 && PipelineAck.getStatusFromHeader(replies[1]) == Status.ERROR_CHECKSUM) { throw new IOException("Shutting down writer and responder " PipelineAck replyAck = new PipelineAck(seqno, replies, totalAckTimeNanos); if (replyAck.isSuccess() && offsetInBlock > replicaInfo.getBytesAcked()) { replicaInfo.setBytesAcked(offsetInBlock); replyAck.write(upstreamOut); upstreamOut.flush(); long duration = Time.monotonicNow() - begin; Status myStatus = PipelineAck.getStatusFromHeader(myHeader); if (myStatus == Status.ERROR_CHECKSUM) { throw new IOException("Shutting down writer and responder "
Packet pkt = null; long expected = -2; PipelineAck ack = new PipelineAck(); long seqno = PipelineAck.UNKOWN_SEQNO; long ackRecvNanoTime = 0; ack.readFields(downstreamIn); ackRecvNanoTime = System.nanoTime(); if (LOG.isDebugEnabled()) { Status oobStatus = ack.getOOBStatus(); if (oobStatus != null) { LOG.info("Relaying an out of band ack of type " + oobStatus); sendAckUpstream(ack, PipelineAck.UNKOWN_SEQNO, 0L, 0L, PipelineAck.combineHeader(datanode.getECN(), Status.SUCCESS)); continue; seqno = ack.getSeqno(); - ack.getDownstreamAckTimeNanos(); if (ackTimeNanos < 0) { if (LOG.isDebugEnabled()) { sendAckUpstream(ack, expected, totalAckTimeNanos, (pkt != null ? pkt.offsetInBlock : 0), PipelineAck.combineHeader(datanode.getECN(), myStatus)); if (pkt != null) {
/** * get the header flag of ith reply */ public int getHeaderFlag(int i) { if (proto.getFlagCount() > 0) { return proto.getFlag(i); } else { return combineHeader(ECN.DISABLED, proto.getReply(i)); } }
throws Exception { Status pipelineStatus = resp.getStatus(); if (PipelineAck.isRestartOOBStatus(pipelineStatus)) { throw new IOException("datanode " + dnInfo + " is restarting");
/** * Returns the OOB status if this ack contains one. * @return null if it is not an OOB ack. */ public Status getOOBStatus() { // Normal data transfer acks will have a valid sequence number, so // this will return right away in most cases. if (getSeqno() != UNKOWN_SEQNO) { return null; } for (Status s : proto.getReplyList()) { // The following check is valid because protobuf guarantees to // preserve the ordering of enum elements. if (s.getNumber() >= OOB_START && s.getNumber() <= OOB_END) { return s; } } return null; }