/** * Creates a record with a specified timestamp to be sent to a specified topic and partition * * @param topic The topic the record will be appended to * @param partition The partition to which the record should be sent * @param timestamp The timestamp of the record, in milliseconds since epoch. If null, the producer will assign * the timestamp using System.currentTimeMillis(). * @param key The key that will be included in the record * @param value The record contents * @param headers the headers that will be included in the record */ public ProducerRecord(String topic, Integer partition, Long timestamp, K key, V value, Iterable<Header> headers) { if (topic == null) throw new IllegalArgumentException("Topic cannot be null."); if (timestamp != null && timestamp < 0) throw new IllegalArgumentException( String.format("Invalid timestamp: %d. Timestamp should always be non-negative or null.", timestamp)); if (partition != null && partition < 0) throw new IllegalArgumentException( String.format("Invalid partition: %d. Partition number should always be non-negative or null.", partition)); this.topic = topic; this.partition = partition; this.key = key; this.value = value; this.timestamp = timestamp; this.headers = new RecordHeaders(headers); }
@Test public void testReadOnly() throws IOException { RecordHeaders headers = new RecordHeaders(); headers.add(new RecordHeader("key", "value".getBytes())); Iterator<Header> headerIteratorBeforeClose = headers.iterator(); headers.setReadOnly(); try { headers.add(new RecordHeader("key", "value".getBytes())); fail("IllegalStateException expected as headers are closed"); } catch (IllegalStateException ise) { headers.remove("key"); fail("IllegalStateException expected as headers are closed"); } catch (IllegalStateException ise) { Iterator<Header> headerIterator = headers.iterator(); headerIterator.next(); headerIterator.remove();
@Test public void testHasRoomForMethodWithHeaders() { if (magic >= RecordBatch.MAGIC_VALUE_V2) { MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(100), magic, compression, TimestampType.CREATE_TIME, 0L); RecordHeaders headers = new RecordHeaders(); headers.add("hello", "world.world".getBytes()); headers.add("hello", "world.world".getBytes()); headers.add("hello", "world.world".getBytes()); headers.add("hello", "world.world".getBytes()); headers.add("hello", "world.world".getBytes()); builder.append(logAppendTime, "key".getBytes(), "value".getBytes()); // Make sure that hasRoomFor accounts for header sizes by letting a record without headers pass, but stopping // a record with a large number of headers. assertTrue(builder.hasRoomFor(logAppendTime, "key".getBytes(), "value".getBytes(), Record.EMPTY_HEADERS)); assertFalse(builder.hasRoomFor(logAppendTime, "key".getBytes(), "value".getBytes(), headers.toArray())); } }
@Test public void testNew() throws IOException { RecordHeaders headers = new RecordHeaders(); headers.add(new RecordHeader("key", "value".getBytes())); headers.setReadOnly(); RecordHeaders newHeaders = new RecordHeaders(headers); newHeaders.add(new RecordHeader("key", "value2".getBytes())); //Ensure existing headers are not modified assertHeader("key", "value", headers.lastHeader("key")); assertEquals(1, getCount(headers)); //Ensure new headers are modified assertHeader("key", "value2", newHeaders.lastHeader("key")); assertEquals(2, getCount(newHeaders)); }
@Test public void testKafkaRecordSerializableWithHeaders() throws IOException { RecordHeaders headers = new RecordHeaders(); headers.add("headerKey", "headerVal".getBytes(StandardCharsets.UTF_8)); verifySerialization(headers); }
@Test public void testReserializedNonTrusted() { DefaultKafkaHeaderMapper mapper = new DefaultKafkaHeaderMapper(); Message<String> message = MessageBuilder.withPayload("foo") .setHeader("fix", new Foo()) .build(); RecordHeaders recordHeaders = new RecordHeaders(); mapper.fromHeaders(message.getHeaders(), recordHeaders); assertThat(recordHeaders.toArray().length).isEqualTo(2); // 1 + json_types Map<String, Object> headers = new HashMap<>(); mapper.toHeaders(recordHeaders, headers); assertThat(headers.get("fix")).isInstanceOf(NonTrustedHeaderType.class); NonTrustedHeaderType ntht = (NonTrustedHeaderType) headers.get("fix"); assertThat(ntht.getHeaderValue()).isNotNull(); assertThat(ntht.getUntrustedType()).isEqualTo(Foo.class.getName()); assertThat(headers).hasSize(1); recordHeaders = new RecordHeaders(); mapper.fromHeaders(new MessageHeaders(headers), recordHeaders); headers = new HashMap<>(); mapper.toHeaders(recordHeaders, headers); assertThat(headers.get("fix")).isInstanceOf(NonTrustedHeaderType.class); ntht = (NonTrustedHeaderType) headers.get("fix"); assertThat(ntht.getHeaderValue()).isNotNull(); assertThat(ntht.getUntrustedType()).isEqualTo(Foo.class.getName()); mapper.addTrustedPackages(getClass().getPackage().getName()); headers = new HashMap<>(); mapper.toHeaders(recordHeaders, headers); assertThat(headers.get("fix")).isInstanceOf(Foo.class); }
@Test public void testHeaders() throws IOException { RecordHeaders headers = new RecordHeaders(); headers.add(new RecordHeader("key", "value".getBytes())); headers.add(new RecordHeader("key1", "key1value".getBytes())); headers.add(new RecordHeader("key", "value2".getBytes())); headers.add(new RecordHeader("key2", "key2value".getBytes())); Iterator<Header> keyHeaders = headers.headers("key").iterator(); assertHeader("key", "value", keyHeaders.next()); assertHeader("key", "value2", keyHeaders.next()); assertFalse(keyHeaders.hasNext()); keyHeaders = headers.headers("key1").iterator(); assertHeader("key1", "key1value", keyHeaders.next()); assertFalse(keyHeaders.hasNext()); keyHeaders = headers.headers("key2").iterator(); assertHeader("key2", "key2value", keyHeaders.next()); assertFalse(keyHeaders.hasNext()); }
@Override public Headers add(String key, byte[] value) throws IllegalStateException { return add(new RecordHeader(key, value)); }
private void setReadOnly(Headers headers) { if (headers instanceof RecordHeaders) { ((RecordHeaders) headers).setReadOnly(); } }
private RecordHeaders convertHeaderFor(SourceRecord record) { Headers headers = record.headers(); RecordHeaders result = new RecordHeaders(); if (headers != null) { String topic = record.topic(); for (Header header : headers) { String key = header.key(); byte[] rawHeader = headerConverter.fromConnectHeader(topic, key, header.schema(), header.value()); result.add(key, rawHeader); } } return result; }
.setHeader("customToString", new Bar("fiz")) .build(); RecordHeaders recordHeaders = new RecordHeaders(); mapper.fromHeaders(message.getHeaders(), recordHeaders); assertThat(recordHeaders.toArray().length).isEqualTo(9); // 8 + json_types Map<String, Object> headers = new HashMap<>(); mapper.toHeaders(recordHeaders, headers);
private void enhanceHeaders(RecordHeaders kafkaHeaders, ConsumerRecord<?, ?> record, Exception exception) { kafkaHeaders.add( new RecordHeader(KafkaHeaders.DLT_ORIGINAL_TOPIC, record.topic().getBytes(StandardCharsets.UTF_8))); kafkaHeaders.add(new RecordHeader(KafkaHeaders.DLT_ORIGINAL_PARTITION, ByteBuffer.allocate(Integer.BYTES).putInt(record.partition()).array())); kafkaHeaders.add(new RecordHeader(KafkaHeaders.DLT_ORIGINAL_OFFSET, ByteBuffer.allocate(Long.BYTES).putLong(record.offset()).array())); kafkaHeaders.add(new RecordHeader(KafkaHeaders.DLT_ORIGINAL_TIMESTAMP, ByteBuffer.allocate(Long.BYTES).putLong(record.timestamp()).array())); kafkaHeaders.add(new RecordHeader(KafkaHeaders.DLT_ORIGINAL_TIMESTAMP_TYPE, record.timestampType().toString().getBytes(StandardCharsets.UTF_8))); kafkaHeaders.add(new RecordHeader(KafkaHeaders.DLT_EXCEPTION_FQCN, exception.getClass().getName().getBytes(StandardCharsets.UTF_8))); kafkaHeaders.add(new RecordHeader(KafkaHeaders.DLT_EXCEPTION_MESSAGE, exception.getMessage().getBytes(StandardCharsets.UTF_8))); kafkaHeaders.add(new RecordHeader(KafkaHeaders.DLT_EXCEPTION_STACKTRACE, getStackTraceAsString(exception).getBytes(StandardCharsets.UTF_8))); }
V value) { this(topic, partition, offset, timestamp, timestampType, checksum, serializedKeySize, serializedValueSize, key, value, new RecordHeaders());
private ConsumerRecords<byte[], byte[]> createTestRecordsWithHeaders() { RecordHeader header = new RecordHeader("testHeader", new byte[0]); RecordHeaders headers = new RecordHeaders(); headers.add(header); TimestampType timestampType = TimestampType.NO_TIMESTAMP_TYPE; byte testByte = 0; byte[] testKey = { testByte }; byte[] testValue = { testByte }; ConnectHeaders destinationHeaders = new ConnectHeaders(); destinationHeaders.add(header.key(), header.value(), Schema.OPTIONAL_BYTES_SCHEMA); ConsumerRecord<byte[], byte[]> testConsumerRecord = new ConsumerRecord<byte[], byte[]>(FIRST_TOPIC, FIRST_PARTITION, FIRST_OFFSET, System.currentTimeMillis(), timestampType, 0L, 0, 0, testKey, testValue, headers); TopicPartition topicPartition = new TopicPartition(FIRST_TOPIC, FIRST_PARTITION); List<ConsumerRecord<byte[], byte[]>> consumerRecords = new ArrayList<>(); consumerRecords.add(testConsumerRecord); Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> consumerRecordMap = new HashMap<>(1); consumerRecordMap.put(topicPartition, consumerRecords); ConsumerRecords<byte[], byte[]> testRecords = new ConsumerRecords<>(consumerRecordMap); return testRecords; }
@Test @SuppressWarnings("deprecation") public void testNullChecksumInConstructor() { String key = "key"; String value = "value"; long timestamp = 242341324L; ConsumerRecord<String, String> record = new ConsumerRecord<>("topic", 0, 23L, timestamp, TimestampType.CREATE_TIME, null, key.length(), value.length(), key, value, new RecordHeaders()); assertEquals(DefaultRecord.computePartialChecksum(timestamp, key.length(), value.length()), record.checksum()); }
@Test public void testRemove() { Headers headers = new RecordHeaders(); headers.add(new RecordHeader("key", "value".getBytes())); assertTrue(headers.iterator().hasNext()); headers.remove("key"); assertFalse(headers.iterator().hasNext()); }
@Test public void testLastHeader() { Headers headers = new RecordHeaders(); headers.add(new RecordHeader("key", "value".getBytes())); headers.add(new RecordHeader("key", "value2".getBytes())); headers.add(new RecordHeader("key", "value3".getBytes())); assertHeader("key", "value3", headers.lastHeader("key")); assertEquals(3, getCount(headers)); }
@Test public void testAdd() { Headers headers = new RecordHeaders(); headers.add(new RecordHeader("key", "value".getBytes())); Header header = headers.iterator().next(); assertHeader("key", "value", header); headers.add(new RecordHeader("key2", "value2".getBytes())); assertHeader("key2", "value2", headers.lastHeader("key2")); assertEquals(2, getCount(headers)); }
/** * Subclasses can populate additional headers before they are mapped. * @param message the message. * @return the headers * @since 2.1 */ protected Headers initialRecordHeaders(Message<?> message) { return new RecordHeaders(); }