/** * Validate that the response corresponds to the request we expect or else explode */ private static void correlate(RequestHeader requestHeader, ResponseHeader responseHeader) { if (requestHeader.correlationId() != responseHeader.correlationId()) throw new IllegalStateException("Correlation id for response (" + responseHeader.correlationId() + ") does not match request (" + requestHeader.correlationId() + "), request header: " + requestHeader); }
@Override public void handleCompletedMetadataResponse(RequestHeader requestHeader, long now, MetadataResponse response) { this.metadataFetchInProgress = false; // If any partition has leader with missing listeners, log a few for diagnosing broker configuration // issues. This could be a transient issue if listeners were added dynamically to brokers. List<TopicPartition> missingListenerPartitions = response.topicMetadata().stream().flatMap(topicMetadata -> topicMetadata.partitionMetadata().stream() .filter(partitionMetadata -> partitionMetadata.error() == Errors.LISTENER_NOT_FOUND) .map(partitionMetadata -> new TopicPartition(topicMetadata.topic(), partitionMetadata.partition()))) .collect(Collectors.toList()); if (!missingListenerPartitions.isEmpty()) { int count = missingListenerPartitions.size(); log.warn("{} partitions have leader brokers without a matching listener, including {}", count, missingListenerPartitions.subList(0, Math.min(10, count))); } // check if any topics metadata failed to get updated Map<String, Errors> errors = response.errors(); if (!errors.isEmpty()) log.warn("Error while fetching metadata with correlation id {} : {}", requestHeader.correlationId(), errors); // don't update the cluster if there are no valid nodes...the topic we want may still be in the process of being // created which means we will get errors and no nodes until it exists if (response.brokers().isEmpty()) { log.trace("Ignoring empty metadata response with correlation id {}.", requestHeader.correlationId()); this.metadata.failedUpdate(now, null); } else { this.metadata.update(response, now); } }
private void handleApiVersionsResponse(List<ClientResponse> responses, InFlightRequest req, long now, ApiVersionsResponse apiVersionsResponse) { final String node = req.destination; if (apiVersionsResponse.error() != Errors.NONE) { if (req.request.version() == 0 || apiVersionsResponse.error() != Errors.UNSUPPORTED_VERSION) { log.warn("Received error {} from node {} when making an ApiVersionsRequest with correlation id {}. Disconnecting.", apiVersionsResponse.error(), node, req.header.correlationId()); this.selector.close(node); processDisconnection(responses, node, now, ChannelState.LOCAL_CLOSE); } else { nodesNeedingApiVersionsFetch.put(node, new ApiVersionsRequest.Builder((short) 0)); } return; } NodeApiVersions nodeVersionInfo = new NodeApiVersions(apiVersionsResponse.apiVersions()); apiVersions.update(node, nodeVersionInfo); this.connectionStates.ready(node); log.debug("Recorded API versions for node {}: {}", node, nodeVersionInfo); }
@Test public void testClearAll() { int correlationId1 = addRequest(dest); int correlationId2 = addRequest(dest); List<NetworkClient.InFlightRequest> clearedRequests = TestUtils.toList(this.inFlightRequests.clearAll(dest)); assertEquals(0, inFlightRequests.count()); assertEquals(2, clearedRequests.size()); assertEquals(correlationId1, clearedRequests.get(0).header.correlationId()); assertEquals(correlationId2, clearedRequests.get(1).header.correlationId()); }
RequestHeader requestHeader = response.requestHeader(); long receivedTimeMs = response.receivedTimeMs(); int correlationId = requestHeader.correlationId(); if (response.wasDisconnected()) { log.trace("Cancelled request with header {} due to node {} being disconnected",
request.header.apiKey(), request.request, request.header.correlationId(), nodeId); if (!request.isInternalRequest) responses.add(request.disconnected(now, disconnectState.exception()));
List<ClientResponse> responses = client.poll(Long.MAX_VALUE, time.milliseconds()); for (ClientResponse response : responses) { if (response.requestHeader().correlationId() == request.correlationId()) { if (response.wasDisconnected()) { throw new IOException("Connection to " + response.destination() + " was disconnected before the response was read");
int correlationId = response.requestHeader().correlationId();
/** * Handle any completed receives and update the response list with the responses received. * * @param responses The list of responses to update * @param now The current time */ private void handleCompletedReceives(List<ClientResponse> responses, long now) { for (NetworkReceive receive : this.selector.completedReceives()) { String source = receive.source(); InFlightRequest req = inFlightRequests.completeNext(source); Struct responseStruct = parseStructMaybeUpdateThrottleTimeMetrics(receive.payload(), req.header, throttleTimeSensor, now); if (log.isTraceEnabled()) { log.trace("Completed receive from node {} for {} with correlation id {}, received {}", req.destination, req.header.apiKey(), req.header.correlationId(), responseStruct); } // If the received response includes a throttle delay, throttle the connection. AbstractResponse body = AbstractResponse. parseResponse(req.header.apiKey(), responseStruct, req.header.apiVersion()); maybeThrottle(body, req.header.apiVersion(), req.destination, now); if (req.isInternalRequest && body instanceof MetadataResponse) metadataUpdater.handleCompletedMetadataResponse(req.header, now, (MetadataResponse) body); else if (req.isInternalRequest && body instanceof ApiVersionsResponse) handleApiVersionsResponse(responses, req, now, (ApiVersionsResponse) body); else responses.add(req.completed(body, now)); } }
@Override public void onComplete(ClientResponse response) { if (response.requestHeader().correlationId() != inFlightRequestCorrelationId) { fatalError(new RuntimeException("Detected more than one in-flight transactional request.")); } else { clearInFlightTransactionalRequestCorrelationId(); if (response.wasDisconnected()) { log.debug("Disconnected from {}. Will retry.", response.destination()); if (this.needsCoordinator()) lookupCoordinator(this.coordinatorType(), this.coordinatorKey()); reenqueue(); } else if (response.versionMismatch() != null) { fatalError(response.versionMismatch()); } else if (response.hasResponse()) { log.trace("Received transactional response {} for request {}", response.responseBody(), requestBuilder()); synchronized (TransactionManager.this) { handleResponse(response.responseBody()); } } else { fatalError(new KafkaException("Could not execute transactional request for unknown reasons")); } } }
@Test public void testSerdeControlledShutdownV0() { // Verify that version 0 of controlled shutdown does not include the clientId field int correlationId = 2342; ByteBuffer rawBuffer = ByteBuffer.allocate(32); rawBuffer.putShort(ApiKeys.CONTROLLED_SHUTDOWN.id); rawBuffer.putShort((short) 0); rawBuffer.putInt(correlationId); rawBuffer.flip(); RequestHeader deserialized = RequestHeader.parse(rawBuffer); assertEquals(ApiKeys.CONTROLLED_SHUTDOWN, deserialized.apiKey()); assertEquals(0, deserialized.apiVersion()); assertEquals(correlationId, deserialized.correlationId()); assertEquals("", deserialized.clientId()); Struct serialized = deserialized.toStruct(); ByteBuffer serializedBuffer = toBuffer(serialized); assertEquals(ApiKeys.CONTROLLED_SHUTDOWN.id, serializedBuffer.getShort(0)); assertEquals(0, serializedBuffer.getShort(2)); assertEquals(correlationId, serializedBuffer.getInt(4)); assertEquals(8, serializedBuffer.limit()); }
@Test public void testCompleteNext() { int correlationId1 = addRequest(dest); int correlationId2 = addRequest(dest); assertEquals(2, inFlightRequests.count()); assertEquals(correlationId1, inFlightRequests.completeNext(dest).header.correlationId()); assertEquals(1, inFlightRequests.count()); assertEquals(correlationId2, inFlightRequests.completeNext(dest).header.correlationId()); assertEquals(0, inFlightRequests.count()); }
@Test public void testCompleteLastSent() { int correlationId1 = addRequest(dest); int correlationId2 = addRequest(dest); assertEquals(2, inFlightRequests.count()); assertEquals(correlationId2, inFlightRequests.completeLastSent(dest).header.correlationId()); assertEquals(1, inFlightRequests.count()); assertEquals(correlationId1, inFlightRequests.completeLastSent(dest).header.correlationId()); assertEquals(0, inFlightRequests.count()); }
@Test public void testRequestHeaderWithNullClientId() { RequestHeader header = new RequestHeader(ApiKeys.FIND_COORDINATOR, (short) 1, null, 10); Struct headerStruct = header.toStruct(); ByteBuffer buffer = toBuffer(headerStruct); RequestHeader deserialized = RequestHeader.parse(buffer); assertEquals(header.apiKey(), deserialized.apiKey()); assertEquals(header.apiVersion(), deserialized.apiVersion()); assertEquals(header.correlationId(), deserialized.correlationId()); assertEquals("", deserialized.clientId()); // null defaults to "" }
private void checkSimpleRequestResponse(NetworkClient networkClient) { awaitReady(networkClient, node); // has to be before creating any request, as it may send ApiVersionsRequest and its response is mocked with correlation id 0 ProduceRequest.Builder builder = ProduceRequest.Builder.forCurrentMagic((short) 1, 1000, Collections.emptyMap()); TestCallbackHandler handler = new TestCallbackHandler(); ClientRequest request = networkClient.newClientRequest( node.idString(), builder, time.milliseconds(), true, defaultRequestTimeoutMs, handler); networkClient.send(request, time.milliseconds()); networkClient.poll(1, time.milliseconds()); assertEquals(1, networkClient.inFlightRequestCount()); ResponseHeader respHeader = new ResponseHeader(request.correlationId()); Struct resp = new Struct(ApiKeys.PRODUCE.responseSchema(ApiKeys.PRODUCE.latestVersion())); resp.set("responses", new Object[0]); Struct responseHeaderStruct = respHeader.toStruct(); int size = responseHeaderStruct.sizeOf() + resp.sizeOf(); ByteBuffer buffer = ByteBuffer.allocate(size); responseHeaderStruct.writeTo(buffer); resp.writeTo(buffer); buffer.flip(); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); List<ClientResponse> responses = networkClient.poll(1, time.milliseconds()); assertEquals(1, responses.size()); assertTrue("The handler should have executed.", handler.executed); assertTrue("Should have a response body.", handler.response.hasResponse()); assertEquals("Should be correlated to the original request", request.correlationId(), handler.response.requestHeader().correlationId()); }
assertEquals(request1.correlationId(), response1.requestHeader().correlationId()); assertEquals(request2.correlationId(), response2.requestHeader().correlationId());
/** * Validate that the response corresponds to the request we expect or else explode */ private void correlate(RequestHeader requestHeader, ResponseHeader responseHeader) { if (requestHeader.correlationId() != responseHeader.correlationId()) throw new IllegalStateException("Correlation id for response (" + responseHeader.correlationId() + ") does not match request (" + requestHeader.correlationId() + ")"); }
private void handleMetadataResponse(RequestHeader header, Struct body, long now) { this.metadataFetchInProgress = false; MetadataResponse response = new MetadataResponse(body); Cluster cluster = response.cluster(); // don't update the cluster if there are no valid nodes...the topic we want may still be in the process of being // created which means we will get errors and no nodes until it exists if (cluster.nodes().size() > 0) { this.metadata.update(cluster, now); } else { log.trace("Ignoring empty metadata response with correlation id {}.", header.correlationId()); this.metadata.failedUpdate(now); } }
private void handleDisconnect(ClientResponse response, long now) { log.trace("Cancelled request {} due to node {} being disconnected", response, response.request().request().destination()); int correlation = response.request().request().header().correlationId(); @SuppressWarnings("unchecked") Map<TopicPartition, RecordBatch> responseBatches = (Map<TopicPartition, RecordBatch>) response.request().attachment(); for (RecordBatch batch : responseBatches.values()) completeBatch(batch, Errors.NETWORK_EXCEPTION, -1L, correlation, now); }
private void handleLeaveGroupRequest(ChannelHandlerContext ctx, Request request) { LeaveGroupRequest leaveGroupRequest = (LeaveGroupRequest) request.getBody(); ResponseHeader responseHeader = new ResponseHeader(request.getHeader().correlationId()); // let the coordinator to handle leave-group coordinator.handleLeaveGroup( leaveGroupRequest.groupId(), leaveGroupRequest.memberId(), (errorCode) -> { LeaveGroupResponse response = new LeaveGroupResponse(errorCode); sendResponse(ctx, new Response(responseHeader, response)); }); }