@Test public void testV6AndBelowCannotUseZStdCompression() { ByteBuffer buffer = ByteBuffer.allocate(256); MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.ZSTD, TimestampType.CREATE_TIME, 0L); builder.append(10L, null, "a".getBytes()); Map<TopicPartition, MemoryRecords> produceData = new HashMap<>(); produceData.put(new TopicPartition("test", 0), builder.build()); // Can't create ProduceRequest instance with version within [3, 7) for (short version = 3; version < 7; version++) { ProduceRequest.Builder requestBuilder = new ProduceRequest.Builder(version, version, (short) 1, 5000, produceData, null); assertThrowsInvalidRecordExceptionForAllVersions(requestBuilder); } // Works fine with current version (>= 7) ProduceRequest.Builder.forCurrentMagic((short) 1, 5000, produceData); }
@Test public void testMixedTransactionalData() { final long producerId = 15L; final short producerEpoch = 5; final int sequence = 10; final String transactionalId = "txnlId"; final MemoryRecords nonTxnRecords = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("foo".getBytes())); final MemoryRecords txnRecords = MemoryRecords.withTransactionalRecords(CompressionType.NONE, producerId, producerEpoch, sequence, new SimpleRecord("bar".getBytes())); final Map<TopicPartition, MemoryRecords> recordsByPartition = new LinkedHashMap<>(); recordsByPartition.put(new TopicPartition("foo", 0), txnRecords); recordsByPartition.put(new TopicPartition("foo", 1), nonTxnRecords); final ProduceRequest.Builder builder = ProduceRequest.Builder.forMagic(RecordVersion.current().value, (short) -1, 5000, recordsByPartition, transactionalId); final ProduceRequest request = builder.build(); assertTrue(request.hasTransactionalRecords()); assertTrue(request.hasIdempotentRecords()); }
@Test public void testBuildWithCurrentMessageFormat() { ByteBuffer buffer = ByteBuffer.allocate(256); MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE, TimestampType.CREATE_TIME, 0L); builder.append(10L, null, "a".getBytes()); Map<TopicPartition, MemoryRecords> produceData = new HashMap<>(); produceData.put(new TopicPartition("test", 0), builder.build()); ProduceRequest.Builder requestBuilder = ProduceRequest.Builder.forMagic(RecordBatch.CURRENT_MAGIC_VALUE, (short) 1, 5000, produceData, null); assertEquals(3, requestBuilder.oldestAllowedVersion()); assertEquals(ApiKeys.PRODUCE.latestVersion(), requestBuilder.latestAllowedVersion()); }
transactionalId = transactionManager.transactionalId(); ProduceRequest.Builder requestBuilder = ProduceRequest.Builder.forMagic(minUsedMagic, acks, timeout, produceRecordsByPartition, transactionalId); RequestCompletionHandler callback = new RequestCompletionHandler() {
ProduceRequest.Builder builder = ProduceRequest.Builder.forCurrentMagic((short) 1, 1000, Collections.emptyMap()); ClientRequest request = client.newClientRequest(node.idString(), builder, time.milliseconds(), true);
ProduceRequest.Builder builder = ProduceRequest.Builder.forCurrentMagic((short) 1, 1000, Collections.emptyMap()); TestCallbackHandler handler = new TestCallbackHandler();
selector.clear(); ProduceRequest.Builder builder = ProduceRequest.Builder.forCurrentMagic((short) 1, 1000, Collections.emptyMap()); TestCallbackHandler handler = new TestCallbackHandler();
private void checkSimpleRequestResponse(NetworkClient networkClient) { awaitReady(networkClient, node); // has to be before creating any request, as it may send ApiVersionsRequest and its response is mocked with correlation id 0 ProduceRequest.Builder builder = ProduceRequest.Builder.forCurrentMagic((short) 1, 1000, Collections.emptyMap()); TestCallbackHandler handler = new TestCallbackHandler(); ClientRequest request = networkClient.newClientRequest( node.idString(), builder, time.milliseconds(), true, defaultRequestTimeoutMs, handler); networkClient.send(request, time.milliseconds()); networkClient.poll(1, time.milliseconds()); assertEquals(1, networkClient.inFlightRequestCount()); ResponseHeader respHeader = new ResponseHeader(request.correlationId()); Struct resp = new Struct(ApiKeys.PRODUCE.responseSchema(ApiKeys.PRODUCE.latestVersion())); resp.set("responses", new Object[0]); Struct responseHeaderStruct = respHeader.toStruct(); int size = responseHeaderStruct.sizeOf() + resp.sizeOf(); ByteBuffer buffer = ByteBuffer.allocate(size); responseHeaderStruct.writeTo(buffer); resp.writeTo(buffer); buffer.flip(); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); List<ClientResponse> responses = networkClient.poll(1, time.milliseconds()); assertEquals(1, responses.size()); assertTrue("The handler should have executed.", handler.executed); assertTrue("Should have a response body.", handler.response.hasResponse()); assertEquals("Should be correlated to the original request", request.correlationId(), handler.response.requestHeader().correlationId()); }
@Test public void testClose() { client.ready(node, time.milliseconds()); awaitReady(client, node); client.poll(1, time.milliseconds()); assertTrue("The client should be ready", client.isReady(node, time.milliseconds())); ProduceRequest.Builder builder = ProduceRequest.Builder.forCurrentMagic((short) 1, 1000, Collections.<TopicPartition, MemoryRecords>emptyMap()); ClientRequest request = client.newClientRequest(node.idString(), builder, time.milliseconds(), true); client.send(request, time.milliseconds()); assertEquals("There should be 1 in-flight request after send", 1, client.inFlightRequestCount(node.idString())); assertTrue(client.hasInFlightRequests(node.idString())); assertTrue(client.hasInFlightRequests()); client.close(node.idString()); assertEquals("There should be no in-flight request after close", 0, client.inFlightRequestCount(node.idString())); assertFalse(client.hasInFlightRequests(node.idString())); assertFalse(client.hasInFlightRequests()); assertFalse("Connection should not be ready after close", client.isReady(node, 0)); }
public static Builder forMagic(byte magic, short acks, int timeout, Map<TopicPartition, MemoryRecords> partitionRecords, String transactionalId) { // Message format upgrades correspond with a bump in the produce request version. Older // message format versions are generally not supported by the produce request versions // following the bump. final short minVersion; final short maxVersion; if (magic < RecordBatch.MAGIC_VALUE_V2) { minVersion = 2; maxVersion = 2; } else { minVersion = 3; maxVersion = ApiKeys.PRODUCE.latestVersion(); } return new Builder(minVersion, maxVersion, acks, timeout, partitionRecords, transactionalId); }
@Test public void testMixedIdempotentData() { final long producerId = 15L; final short producerEpoch = 5; final int sequence = 10; final MemoryRecords nonTxnRecords = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("foo".getBytes())); final MemoryRecords txnRecords = MemoryRecords.withIdempotentRecords(CompressionType.NONE, producerId, producerEpoch, sequence, new SimpleRecord("bar".getBytes())); final Map<TopicPartition, MemoryRecords> recordsByPartition = new LinkedHashMap<>(); recordsByPartition.put(new TopicPartition("foo", 0), txnRecords); recordsByPartition.put(new TopicPartition("foo", 1), nonTxnRecords); final ProduceRequest.Builder builder = ProduceRequest.Builder.forMagic(RecordVersion.current().value, (short) -1, 5000, recordsByPartition, null); final ProduceRequest request = builder.build(); assertFalse(request.hasTransactionalRecords()); assertTrue(request.hasIdempotentRecords()); }
@Test public void testV3AndAboveShouldContainOnlyOneRecordBatch() { ByteBuffer buffer = ByteBuffer.allocate(256); MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, CompressionType.NONE, TimestampType.CREATE_TIME, 0L); builder.append(10L, null, "a".getBytes()); builder.close(); builder = MemoryRecords.builder(buffer, CompressionType.NONE, TimestampType.CREATE_TIME, 1L); builder.append(11L, "1".getBytes(), "b".getBytes()); builder.append(12L, null, "c".getBytes()); builder.close(); buffer.flip(); Map<TopicPartition, MemoryRecords> produceData = new HashMap<>(); produceData.put(new TopicPartition("test", 0), MemoryRecords.readableRecords(buffer)); ProduceRequest.Builder requestBuilder = ProduceRequest.Builder.forCurrentMagic((short) 1, 5000, produceData); assertThrowsInvalidRecordExceptionForAllVersions(requestBuilder); }
public ProduceRequest buildUnsafe(short version) { return build(version, false); }
@Override public ProduceRequest build(short version) { return build(version, true); }
@Test public void testRequestTimeout() { awaitReady(client, node); // has to be before creating any request, as it may send ApiVersionsRequest and its response is mocked with correlation id 0 ProduceRequest.Builder builder = ProduceRequest.Builder.forCurrentMagic((short) 1, 1000, Collections.emptyMap()); TestCallbackHandler handler = new TestCallbackHandler(); int requestTimeoutMs = defaultRequestTimeoutMs + 5000; ClientRequest request = client.newClientRequest(node.idString(), builder, time.milliseconds(), true, requestTimeoutMs, handler); assertEquals(requestTimeoutMs, request.requestTimeoutMs()); testRequestTimeout(request); }
private void assertThrowsInvalidRecordException(ProduceRequest.Builder builder, short version) { try { builder.build(version).toStruct(); fail("Builder did not raise " + InvalidRecordException.class.getName() + " as expected"); } catch (RuntimeException e) { assertTrue("Unexpected exception type " + e.getClass().getName(), InvalidRecordException.class.isAssignableFrom(e.getClass())); } }
@Test public void testV3AndAboveCannotHaveNoRecordBatches() { Map<TopicPartition, MemoryRecords> produceData = new HashMap<>(); produceData.put(new TopicPartition("test", 0), MemoryRecords.EMPTY); ProduceRequest.Builder requestBuilder = ProduceRequest.Builder.forCurrentMagic((short) 1, 5000, produceData); assertThrowsInvalidRecordExceptionForAllVersions(requestBuilder); }
@Test public void testBuildWithOldMessageFormat() { ByteBuffer buffer = ByteBuffer.allocate(256); MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V1, CompressionType.NONE, TimestampType.CREATE_TIME, 0L); builder.append(10L, null, "a".getBytes()); Map<TopicPartition, MemoryRecords> produceData = new HashMap<>(); produceData.put(new TopicPartition("test", 0), builder.build()); ProduceRequest.Builder requestBuilder = ProduceRequest.Builder.forMagic(RecordBatch.MAGIC_VALUE_V1, (short) 1, 5000, produceData, null); assertEquals(2, requestBuilder.oldestAllowedVersion()); assertEquals(2, requestBuilder.latestAllowedVersion()); }
@Test public void testDefaultRequestTimeout() { awaitReady(client, node); // has to be before creating any request, as it may send ApiVersionsRequest and its response is mocked with correlation id 0 ProduceRequest.Builder builder = ProduceRequest.Builder.forCurrentMagic((short) 1, 1000, Collections.emptyMap()); ClientRequest request = client.newClientRequest(node.idString(), builder, time.milliseconds(), true); assertEquals(defaultRequestTimeoutMs, request.requestTimeoutMs()); testRequestTimeout(request); }
@Test public void testV3AndAboveCannotUseMagicV0() { ByteBuffer buffer = ByteBuffer.allocate(256); MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V0, CompressionType.NONE, TimestampType.NO_TIMESTAMP_TYPE, 0L); builder.append(10L, null, "a".getBytes()); Map<TopicPartition, MemoryRecords> produceData = new HashMap<>(); produceData.put(new TopicPartition("test", 0), builder.build()); ProduceRequest.Builder requestBuilder = ProduceRequest.Builder.forCurrentMagic((short) 1, 5000, produceData); assertThrowsInvalidRecordExceptionForAllVersions(requestBuilder); }