/** * @param zkServers Zookeeper server string: host1:port1[,host2:port2,...] * @param topic topic to create (if not already existing) * @param partitions number of topic partitions */ public static void maybeCreateTopic(String zkServers, String topic, int partitions) { maybeCreateTopic(zkServers, topic, partitions, new Properties()); }
protected final void startMessaging() throws IOException, InterruptedException { log.info("Starting local test Zookeeper server"); localZKServer = new LocalZKServer(localZKPort); localZKServer.start(); log.info("Starting local Kafka broker"); localKafkaBroker = new LocalKafkaBroker(localKafkaBrokerPort, localZKPort); localKafkaBroker.start(); String zkHostPort = "localhost:" + localZKPort; KafkaUtils.deleteTopic(zkHostPort, INPUT_TOPIC); KafkaUtils.deleteTopic(zkHostPort, UPDATE_TOPIC); KafkaUtils.maybeCreateTopic(zkHostPort, INPUT_TOPIC, 1); KafkaUtils.maybeCreateTopic(zkHostPort, UPDATE_TOPIC, 1); // Give topics a little time to elect leader try { Thread.sleep(5000); } catch (InterruptedException ie) { // continue } }
int zkPort = IOUtils.chooseFreePort(); int kafkaBrokerPort = IOUtils.chooseFreePort(); try (LocalZKServer localZKServer = new LocalZKServer(zkPort); LocalKafkaBroker localKafkaBroker = new LocalKafkaBroker(kafkaBrokerPort, zkPort)) { localZKServer.start(); localKafkaBroker.start(); KafkaUtils.deleteTopic(zkHostPort, TOPIC); KafkaUtils.maybeCreateTopic(zkHostPort, TOPIC, 4); ProduceData produce = new ProduceData(new DefaultCSVDatumGenerator(), localKafkaBroker.getPort(), TOPIC, NUM_DATA, try (CloseableIterator<KeyMessage<String,String>> data = new ConsumeData(TOPIC, kafkaBrokerPort).iterator()) { ConsumeTopicRunnable consumeTopic = new ConsumeTopicRunnable(data, NUM_DATA); new Thread(LoggingCallable.log(consumeTopic).asRunnable(), "ConsumeTopicThread").start(); consumeTopic.awaitRun(); produce.start(); consumeTopic.awaitMessages(); keys = consumeTopic.getKeys(); } finally { KafkaUtils.deleteTopic(zkHostPort, TOPIC);
int zkPort = IOUtils.chooseFreePort(); int kafkaBrokerPort = IOUtils.chooseFreePort(); try (LocalZKServer localZKServer = new LocalZKServer(zkPort); LocalKafkaBroker localKafkaBroker = new LocalKafkaBroker(kafkaBrokerPort, zkPort)) { localZKServer.start(); localKafkaBroker.start(); KafkaUtils.deleteTopic(zkHostPort, TOPIC); KafkaUtils.maybeCreateTopic(zkHostPort, TOPIC, 1, ConfigUtils.keyValueToProperties( "max.message.bytes", maxMessageSize )); ProduceData produce = new ProduceData(new BigDatumGenerator(largeMessageSize), localKafkaBroker.getPort(), TOPIC, NUM_DATA, new ConsumeData(TOPIC, maxMessageSize, kafkaBrokerPort).iterator()) { ConsumeTopicRunnable consumeTopic = new ConsumeTopicRunnable(data, NUM_DATA); new Thread(LoggingCallable.log(consumeTopic).asRunnable(), "ConsumeTopicThread").start(); consumeTopic.awaitRun(); produce.start(); consumeTopic.awaitMessages(); keyMessages = consumeTopic.getKeyMessages();
int intervalMsec) throws InterruptedException { ProduceData produce = new ProduceData(datumGenerator, getKafkaBrokerPort(), INPUT_TOPIC, new ConsumeData(UPDATE_TOPIC, getKafkaBrokerPort()).iterator(); BatchLayer<?,?,?> batchLayer = new BatchLayer<>(config)) { ConsumeTopicRunnable consumeInput = new ConsumeTopicRunnable(data); new Thread(LoggingCallable.log(consumeInput).asRunnable(), "ConsumeInputThread").start(); consumeInput.awaitRun(); produce.start(); sleepSeconds(genIntervalSec); keyMessages = consumeInput.getKeyMessages();
new ConsumeData(INPUT_TOPIC, getKafkaBrokerPort()).iterator()) { ConsumeTopicRunnable consumeInput = new ConsumeTopicRunnable(data, inputs.length); new Thread(LoggingCallable.log(consumeInput).asRunnable(), "ConsumeInputThread").start(); consumeInput.awaitRun(); consumeInput.awaitMessages(); keyMessages = consumeInput.getKeyMessages();
protected final void startUpdateTopics(DatumGenerator<String,String> updateGenerator, int howManyUpdate) throws InterruptedException { log.info("Producing updates"); new ProduceData(updateGenerator, getKafkaBrokerPort(), UPDATE_TOPIC, howManyUpdate, 0).start(); }
public static void main(String[] args) throws Exception { int port = args.length > 0 ? Integer.parseInt(args[0]) : IOUtils.chooseFreePort(); try (LocalZKServer zkServer = new LocalZKServer(port)) { JVMUtils.closeAtShutdown(zkServer); zkServer.start(); zkServer.await(); } }
public static void main(String[] args) throws Exception { int port = args.length > 0 ? Integer.parseInt(args[0]) : IOUtils.chooseFreePort(); int zkPort = args.length > 1 ? Integer.parseInt(args[1]) : IOUtils.chooseFreePort(); try (final LocalKafkaBroker kafkaBroker = new LocalKafkaBroker(port, zkPort)) { JVMUtils.closeAtShutdown(kafkaBroker); kafkaBroker.start(); kafkaBroker.await(); } }
Preconditions.checkArgument(KafkaUtils.topicExists(inputTopicLockMaster, inputTopic), "Topic %s does not exist; did you create it?", inputTopic); Preconditions.checkArgument(KafkaUtils.topicExists(updateTopicLockMaster, updateTopic), "Topic %s does not exist; did you create it?", updateTopic); inputProducer = new TopicProducerImpl<>(inputTopicBroker, inputTopic); )); consumer.subscribe(Collections.singletonList(updateTopic)); consumerIterator = new ConsumeDataIterator<>(consumer);
@Override public CloseableIterator<KeyMessage<String,String>> iterator() { KafkaConsumer<String,String> consumer = new KafkaConsumer<>( ConfigUtils.keyValueToProperties( "group.id", "OryxGroup-ConsumeData", "bootstrap.servers", "localhost:" + kafkaPort, "key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer", "value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer", "max.partition.fetch.bytes", maxMessageSize, "auto.offset.reset", "earliest" // For tests, always start at the beginning )); consumer.subscribe(Collections.singletonList(topic)); return new ConsumeDataIterator<>(consumer); }
protected List<KeyMessage<String,String>> startServerProduceConsumeTopics( Config config, int howMany, int intervalMsec) throws InterruptedException { return startServerProduceConsumeTopics(config, new DefaultCSVDatumGenerator(), howMany, intervalMsec); }
protected final JavaInputDStream<ConsumerRecord<K,M>> buildInputDStream( JavaStreamingContext streamingContext) { Preconditions.checkArgument( KafkaUtils.topicExists(inputTopicLockMaster, inputTopic), "Topic %s does not exist; did you create it?", inputTopic); if (updateTopic != null && updateTopicLockMaster != null) { Preconditions.checkArgument( KafkaUtils.topicExists(updateTopicLockMaster, updateTopic), "Topic %s does not exist; did you create it?", updateTopic); } String groupID = getGroupID(); Map<String,Object> kafkaParams = new HashMap<>(); kafkaParams.put("group.id", groupID); // Don't re-consume old messages from input by default kafkaParams.put("auto.offset.reset", "latest"); // Ignored by Kafka 0.10 Spark integration kafkaParams.put("bootstrap.servers", inputBroker); kafkaParams.put("key.deserializer", keyDecoderClass.getName()); kafkaParams.put("value.deserializer", messageDecoderClass.getName()); LocationStrategy locationStrategy = LocationStrategies.PreferConsistent(); ConsumerStrategy<K,M> consumerStrategy = ConsumerStrategies.Subscribe( Collections.singleton(inputTopic), kafkaParams, Collections.emptyMap()); return org.apache.spark.streaming.kafka010.KafkaUtils.createDirectStream( streamingContext, locationStrategy, consumerStrategy); }
@After public final void tearDownKafkaZK() { if (localKafkaBroker != null) { log.info("Deleting topics"); String zkHostPort = "localhost:" + localZKPort; KafkaUtils.deleteTopic(zkHostPort, INPUT_TOPIC); KafkaUtils.deleteTopic(zkHostPort, UPDATE_TOPIC); log.info("Stopping Kafka"); IOUtils.closeQuietly(localKafkaBroker); localKafkaBroker = null; } if (localZKServer != null) { log.info("Stopping Zookeeper"); IOUtils.closeQuietly(localZKServer); localZKServer = null; } }
@Override protected KeyMessage<K,V> computeNext() { if (iterator == null || !iterator.hasNext()) { try { long timeout = MIN_POLL_MS; ConsumerRecords<K, V> records; while ((records = consumer.poll(timeout)).isEmpty()) { timeout = Math.min(MAX_POLL_MS, timeout * 2); } iterator = records.iterator(); } catch (Exception e) { consumer.close(); return endOfData(); } } ConsumerRecord<K,V> mm = iterator.next(); return new KeyMessageImpl<>(mm.key(), mm.value()); }
/** * @param javaRDD RDD whose underlying RDD must be an instance of {@code HasOffsetRanges}, * such as {@code KafkaRDD} */ @Override public void call(JavaRDD<T> javaRDD) { OffsetRange[] ranges = ((HasOffsetRanges) javaRDD.rdd()).offsetRanges(); Map<Pair<String,Integer>,Long> newOffsets = new HashMap<>(ranges.length); for (OffsetRange range : ranges) { newOffsets.put(new Pair<>(range.topic(), range.partition()), range.untilOffset()); } log.info("Updating offsets: {}", newOffsets); KafkaUtils.setOffsets(inputTopicLockMaster, group, newOffsets); }
public void start() throws InterruptedException { RandomGenerator random = RandomManager.getRandom(); Properties props = ConfigUtils.keyValueToProperties( "bootstrap.servers", "localhost:" + kafkaPort, "key.serializer", "org.apache.kafka.common.serialization.StringSerializer", "value.serializer", "org.apache.kafka.common.serialization.StringSerializer", "compression.type", "gzip", "linger.ms", 0, "batch.size", 0, "acks", 1, "max.request.size", 1 << 26 // TODO ); try (Producer<String,String> producer = new KafkaProducer<>(props)) { for (int i = 0; i < howMany; i++) { Pair<String,String> datum = datumGenerator.generate(i, random); ProducerRecord<String,String> record = new ProducerRecord<>(topic, datum.getFirst(), datum.getSecond()); producer.send(record); log.debug("Sent datum {} = {}", record.key(), record.value()); if (intervalMsec > 0) { Thread.sleep(intervalMsec); } } } }
ProduceData inputProducer = new ProduceData(inputGenerator, kafkaPort, INPUT_TOPIC, howManyInput, 10); ProduceData updateProducer = new ProduceData(updateGenerator, kafkaPort, UPDATE_TOPIC, new ConsumeData(UPDATE_TOPIC, kafkaPort).iterator(); SpeedLayer<?,?,?> speedLayer = new SpeedLayer<>(config)) { ConsumeTopicRunnable consumeUpdate = new ConsumeTopicRunnable(data); new Thread(LoggingCallable.log(consumeUpdate).asRunnable(), "ConsumeUpdateThread").start(); consumeUpdate.awaitRun(); updateProducer.start(); inputProducer.start(); keyMessages = consumeUpdate.getKeyMessages();
)); consumer.subscribe(Collections.singletonList(updateTopic)); consumerIterator = new ConsumeDataIterator<>(consumer);
final List<KeyMessage<String,String>> startServerProduceConsumeTopics( Config config, int howMany, int howManyUpdate) throws InterruptedException { return startServerProduceConsumeTopics(config, new DefaultCSVDatumGenerator(), new MockModelGenerator(), howMany, howManyUpdate); }