/** * Intended to be overridden for tests. Make the producer with the given props */ protected Producer<K, V> mkProducer(Properties props) { return new KafkaProducer<>(props); }
public FlinkKafkaInternalProducer(Properties properties) { transactionalId = properties.getProperty(ProducerConfig.TRANSACTIONAL_ID_CONFIG); kafkaProducer = new KafkaProducer<>(properties); }
public FlinkKafkaProducer(Properties properties) { transactionalId = properties.getProperty(ProducerConfig.TRANSACTIONAL_ID_CONFIG); kafkaProducer = new KafkaProducer<>(properties); }
/** * Prepare this State. * @param options The KafkaProducer config. */ public void prepare(Properties options) { Objects.requireNonNull(mapper, "mapper can not be null"); Objects.requireNonNull(topicSelector, "topicSelector can not be null"); producer = new KafkaProducer<>(options); }
/** * Used for testing only. */ @VisibleForTesting protected <K, V> KafkaProducer<K, V> getKafkaProducer(Properties props) { return new KafkaProducer<>(props); }
KafkaCollector(String brokerAddress, String topic, int partition) { this.topic = checkNotNull(topic); this.partition = partition; this.serializer = new EventDeSerializer(); // create Kafka producer Properties properties = new Properties(); properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerAddress); properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getCanonicalName()); properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getCanonicalName()); this.producer = new KafkaProducer<>(properties); }
private void createProducer() { Properties producerProps = new Properties(); producerProps.setProperty(BOOTSTRAP_SERVERS_CONFIG, KAFKA_HOST + ":" + KAFKA_PORT); producerProps.setProperty(KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); producerProps.setProperty(VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); producer = new KafkaProducer<>(producerProps); }
@Test(expected = ConfigException.class) public void testNoSerializerProvided() { Properties producerProps = new Properties(); producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9000"); new KafkaProducer(producerProps); }
@Test public void testProducerWithInvalidCredentials() { Map<String, Object> props = new HashMap<>(saslClientConfigs); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:" + server.port()); StringSerializer serializer = new StringSerializer(); try (KafkaProducer<String, String> producer = new KafkaProducer<>(props, serializer, serializer)) { ProducerRecord<String, String> record = new ProducerRecord<>(topic, "message"); producer.send(record).get(); fail("Expected an authentication error!"); } catch (Exception e) { assertTrue("Expected SaslAuthenticationException, got " + e.getCause().getClass(), e.getCause() instanceof SaslAuthenticationException); } }
@Test(expected = KafkaException.class) public void testInvalidSocketSendBufferSize() { Map<String, Object> config = new HashMap<>(); config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); config.put(ProducerConfig.SEND_BUFFER_CONFIG, -2); new KafkaProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); }
@Test public void testTransactionalProducerWithInvalidCredentials() { Map<String, Object> props = new HashMap<>(saslClientConfigs); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:" + server.port()); props.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, "txclient-1"); props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "true"); StringSerializer serializer = new StringSerializer(); try (KafkaProducer<String, String> producer = new KafkaProducer<>(props, serializer, serializer)) { producer.initTransactions(); fail("Expected an authentication error!"); } catch (SaslAuthenticationException e) { // expected exception } }
@Test(expected = KafkaException.class) public void testInvalidSocketReceiveBufferSize() { Map<String, Object> config = new HashMap<>(); config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, -2); new KafkaProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()); }
@Test public void testOsDefaultSocketBufferSizes() { Map<String, Object> config = new HashMap<>(); config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); config.put(ProducerConfig.SEND_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); config.put(ProducerConfig.RECEIVE_BUFFER_CONFIG, Selectable.USE_DEFAULT_BUFFER_SIZE); new KafkaProducer<>(config, new ByteArraySerializer(), new ByteArraySerializer()).close(); }
@Test public void testConstructorFailureCloseResource() { Properties props = new Properties(); props.setProperty(ProducerConfig.CLIENT_ID_CONFIG, "testConstructorClose"); props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "some.invalid.hostname.foo.bar.local:9999"); props.setProperty(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName()); final int oldInitCount = MockMetricsReporter.INIT_COUNT.get(); final int oldCloseCount = MockMetricsReporter.CLOSE_COUNT.get(); try (KafkaProducer<byte[], byte[]> ignored = new KafkaProducer<>(props, new ByteArraySerializer(), new ByteArraySerializer())) { fail("should have caught an exception and returned"); } catch (KafkaException e) { assertEquals(oldInitCount + 1, MockMetricsReporter.INIT_COUNT.get()); assertEquals(oldCloseCount + 1, MockMetricsReporter.CLOSE_COUNT.get()); assertEquals("Failed to construct kafka producer", e.getMessage()); } }
@Test public void testMetricsReporterAutoGeneratedClientId() { Properties props = new Properties(); props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); props.setProperty(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName()); KafkaProducer<String, String> producer = new KafkaProducer<>( props, new StringSerializer(), new StringSerializer()); MockMetricsReporter mockMetricsReporter = (MockMetricsReporter) producer.metrics.reporters().get(0); Assert.assertEquals(producer.getClientId(), mockMetricsReporter.clientId); producer.close(); }
@Test public void testConstructorWithSerializers() { Properties producerProps = new Properties(); producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9000"); new KafkaProducer<>(producerProps, new ByteArraySerializer(), new ByteArraySerializer()).close(); }
@Test public void testConstructorWithNotStringKey() { Properties props = new Properties(); props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); props.put(1, "not string key"); try (KafkaProducer<?, ?> ff = new KafkaProducer<>(props, new StringSerializer(), new StringSerializer())) { fail("Constructor should throw exception"); } catch (ConfigException e) { assertTrue("Unexpected exception message: " + e.getMessage(), e.getMessage().contains("not string key")); } }
@Test public void closeShouldBeIdempotent() { Properties producerProps = new Properties(); producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9000"); Producer producer = new KafkaProducer<>(producerProps, new ByteArraySerializer(), new ByteArraySerializer()); producer.close(); producer.close(); }
@Test public void testPartitionsForWithNullTopic() { Properties props = new Properties(); props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9000"); try (KafkaProducer<byte[], byte[]> producer = new KafkaProducer<>(props, new ByteArraySerializer(), new ByteArraySerializer())) { producer.partitionsFor(null); fail("Expected NullPointerException to be raised"); } catch (NullPointerException e) { // expected } }
@Test public void testMetricConfigRecordingLevel() { Properties props = new Properties(); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9000"); try (KafkaProducer producer = new KafkaProducer<>(props, new ByteArraySerializer(), new ByteArraySerializer())) { assertEquals(Sensor.RecordingLevel.INFO, producer.metrics.config().recordLevel()); } props.put(ProducerConfig.METRICS_RECORDING_LEVEL_CONFIG, "DEBUG"); try (KafkaProducer producer = new KafkaProducer<>(props, new ByteArraySerializer(), new ByteArraySerializer())) { assertEquals(Sensor.RecordingLevel.DEBUG, producer.metrics.config().recordLevel()); } }