@Override public List<PartitionInfo> partitionsFor(String topic) { return kafkaProducer.partitionsFor(topic); }
@Override public List<PartitionInfo> partitionsFor(String topic) { return kafkaProducer.partitionsFor(topic); }
@Override public List<PartitionInfo> partitionsFor(String topic) { return kafkaProducer.partitionsFor(topic); }
protected static int[] getPartitionsByTopic(String topic, KafkaProducer<byte[], byte[]> producer) { // the fetched list is immutable, so we're creating a mutable copy in order to sort it List<PartitionInfo> partitionsList = new ArrayList<>(producer.partitionsFor(topic)); // sort the partitions by partition id to make sure the fetched partition list is the same across subtasks Collections.sort(partitionsList, new Comparator<PartitionInfo>() { @Override public int compare(PartitionInfo o1, PartitionInfo o2) { return Integer.compare(o1.partition(), o2.partition()); } }); int[] partitions = new int[partitionsList.size()]; for (int i = 0; i < partitions.length; i++) { partitions[i] = partitionsList.get(i).partition(); } return partitions; }
private Integer getNumPartitions(String topic) { try { return this.kafka.partitionsFor(topic).size(); //returns 1 for new topics } catch (KafkaException e) { LOGGER.error("Topic '" + topic + "' name does not exist. Exception: " + e.getLocalizedMessage()); throw e; } }
@Test public void testPartitionsForWithNullTopic() { Properties props = new Properties(); props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9000"); try (KafkaProducer<byte[], byte[]> producer = new KafkaProducer<>(props, new ByteArraySerializer(), new ByteArraySerializer())) { producer.partitionsFor(null); fail("Expected NullPointerException to be raised"); } catch (NullPointerException e) { // expected } }
public void populateJsonMsgWithTimestamps(String topic, int numMsg) { KafkaProducer<String, String> producer = null; Random rand = new Random(); try { producer = new KafkaProducer<String, String>(producerProperties); int halfCount = numMsg / 2; for(PartitionInfo tpInfo : producer.partitionsFor(topic)) { for (int i = 1; i <= numMsg; ++i) { JsonObject object = new JsonObject(); object.addProperty("stringKey", UUID.randomUUID().toString()); object.addProperty("intKey", numMsg - i); object.addProperty("boolKey", i % 2 == 0); long timestamp = i < halfCount ? (halfCount - i) : i; ProducerRecord<String, String> message = new ProducerRecord<String, String>(tpInfo.topic(), tpInfo.partition(), timestamp, "key"+i, object.toString()); logger.info("Publishing message : {}", message); Future<RecordMetadata> future = producer.send(message); logger.info("Committed offset of the message : {}", future.get().offset()); } } } catch (Throwable th) { logger.error(th.getMessage(), th); throw new DrillRuntimeException(th.getMessage(), th); } finally { if (producer != null) { producer.close(); } } }
/** * Tests that partitions list is determinate and correctly provided to custom partitioner. */ @SuppressWarnings("unchecked") @Test public void testPartitionerInvokedWithDeterminatePartitionList() throws Exception { FlinkKafkaPartitioner<String> mockPartitioner = mock(FlinkKafkaPartitioner.class); RuntimeContext mockRuntimeContext = mock(StreamingRuntimeContext.class); when(mockRuntimeContext.getIndexOfThisSubtask()).thenReturn(0); when(mockRuntimeContext.getNumberOfParallelSubtasks()).thenReturn(1); // out-of-order list of 4 partitions List<PartitionInfo> mockPartitionsList = new ArrayList<>(4); mockPartitionsList.add(new PartitionInfo(DummyFlinkKafkaProducer.DUMMY_TOPIC, 3, null, null, null)); mockPartitionsList.add(new PartitionInfo(DummyFlinkKafkaProducer.DUMMY_TOPIC, 1, null, null, null)); mockPartitionsList.add(new PartitionInfo(DummyFlinkKafkaProducer.DUMMY_TOPIC, 0, null, null, null)); mockPartitionsList.add(new PartitionInfo(DummyFlinkKafkaProducer.DUMMY_TOPIC, 2, null, null, null)); final DummyFlinkKafkaProducer<String> producer = new DummyFlinkKafkaProducer<>( FakeStandardProducerConfig.get(), new KeyedSerializationSchemaWrapper<>(new SimpleStringSchema()), mockPartitioner); producer.setRuntimeContext(mockRuntimeContext); final KafkaProducer mockProducer = producer.getMockKafkaProducer(); when(mockProducer.partitionsFor(anyString())).thenReturn(mockPartitionsList); when(mockProducer.metrics()).thenReturn(null); producer.open(new Configuration()); verify(mockPartitioner, times(1)).open(0, 1); producer.invoke("foobar", SinkContextUtil.forTimestamp(0)); verify(mockPartitioner, times(1)).partition( "foobar", null, "foobar".getBytes(), DummyFlinkKafkaProducer.DUMMY_TOPIC, new int[] {0, 1, 2, 3}); }
t.start(); try { producer.partitionsFor(topic); fail("Expect TimeoutException"); } catch (TimeoutException e) {
@Override public List<PartitionInfo> partitionsFor(String topic) { return kafkaProducer.partitionsFor(topic); }
@Override public List<PartitionInfo> partitionsFor(String topic) { return kafkaProducer.partitionsFor(topic); }
@Override public List<PartitionInfo> partitionsFor(String topic) { return kafkaProducer.partitionsFor(topic); }
@Override public List<PartitionInfo> partitionsFor(String topic) { return producer.partitionsFor(topic); }
@Override public List<PartitionInfo> partitionsFor(String topic) { return kafkaProducer.partitionsFor(topic); }
@Override public int getTopicPartitionSize() { Properties props = new Properties(properties); props.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); props.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); KafkaProducer producer = createKafkaProducer(props); try { int size = 0; for (String t : topic) { size += producer.partitionsFor(t).size(); } return size; } finally { producer.close(); } }
@Override public int getTopicPartitionSize() { Properties props = new Properties(properties); props.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); props.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); KafkaProducer producer = createKafkaProducer(props); try { int size = 0; for (String t : topic) { size += producer.partitionsFor(t).size(); } return size; } finally { producer.close(); } }
/** Ensures there are no problems reading metadata about the topic. */ @Override public CheckResult check() { try { get().partitionsFor(topic); // make sure we can query the metadata return CheckResult.OK; } catch (RuntimeException e) { return CheckResult.failed(e); } }
/** Ensures there are no problems reading metadata about the topic. */ @Override public CheckResult check() { try { get().partitionsFor(topic()); // make sure we can query the metadata return CheckResult.OK; } catch (RuntimeException e) { return CheckResult.failed(e); } }
/** Ensures there are no problems reading metadata about the topic. */ @Override public CheckResult check() { try { get().partitionsFor(topic()); // make sure we can query the metadata return CheckResult.OK; } catch (RuntimeException e) { return CheckResult.failed(e); } }
@SuppressWarnings("unchecked") public KafkaTopicRepositoryTest() { kafkaProducer = mock(KafkaProducer.class); when(kafkaProducer.partitionsFor(anyString())).then( invocation -> partitionsOfTopic((String) invocation.getArguments()[0]) ); kafkaFactory = createKafkaFactory(); kafkaTopicRepository = createKafkaRepository(kafkaFactory); }