@Override public void connect(KafkaTransportProtocol protocol, InternalEventProcessor<byte[]> eventProcessor) throws SpRuntimeException { LOG.info("Kafka consumer: Connecting to " + protocol.getTopicDefinition().getActualTopicName()); if (protocol.getTopicDefinition() instanceof WildcardTopicDefinition) { this.patternTopic = true; } this.eventProcessor = eventProcessor; this.kafkaUrl = protocol.getBrokerHostname() + ":" + protocol.getKafkaPort(); this.topic = protocol.getTopicDefinition().getActualTopicName(); this.groupId = UUID.randomUUID().toString(); this.isRunning = true; Thread thread = new Thread(this); thread.start(); }
@Override public void connect(KafkaTransportProtocol protocol, InternalEventProcessor<byte[]> eventProcessor) throws SpRuntimeException { LOG.info("Kafka consumer: Connecting to " + protocol.getTopicDefinition().getActualTopicName()); if (protocol.getTopicDefinition() instanceof WildcardTopicDefinition) { this.patternTopic = true; } this.eventProcessor = eventProcessor; this.kafkaUrl = protocol.getBrokerHostname() + ":" + protocol.getKafkaPort(); this.topic = protocol.getTopicDefinition().getActualTopicName(); this.groupId = UUID.randomUUID().toString(); this.isRunning = true; Thread thread = new Thread(this); thread.start(); }
@Override public void connect(KafkaTransportProtocol protocolSettings) { LOG.info("Kafka producer: Connecting to " +protocolSettings.getTopicDefinition().getActualTopicName()); this.brokerUrl = protocolSettings.getBrokerHostname() +":" +protocolSettings.getKafkaPort(); this.topic = protocolSettings.getTopicDefinition().getActualTopicName(); this.producer = new KafkaProducer<>(getProperties()); this.connected = true; }
@Override public void connect(KafkaTransportProtocol protocolSettings) { LOG.info("Kafka producer: Connecting to " +protocolSettings.getTopicDefinition().getActualTopicName()); this.brokerUrl = protocolSettings.getBrokerHostname() +":" +protocolSettings.getKafkaPort(); this.topic = protocolSettings.getTopicDefinition().getActualTopicName(); this.producer = new KafkaProducer<>(getProperties()); this.connected = true; }
private SourceFunction<String> getKafkaConsumer(KafkaTransportProtocol protocol) { if (protocol.getTopicDefinition() instanceof SimpleTopicDefinition) { return new FlinkKafkaConsumer010<>(protocol .getTopicDefinition() .getActualTopicName(), new SimpleStringSchema (), getProperties(protocol)); } else { String patternTopic = replaceWildcardWithPatternFormat(protocol.getTopicDefinition().getActualTopicName()); return new FlinkKafkaConsumer010<>(Pattern.compile(patternTopic), new SimpleStringSchema (), getProperties(protocol)); } }
/** * This method takes the i's input stream and creates a source for the Spark streaming job * Currently just kafka is supported as a protocol * TODO Add also jms support * * @param i * @param streamingContext * @return */ private JavaInputDStream<ConsumerRecord<String, String>> getStreamSource(int i, JavaStreamingContext streamingContext) { if (graph.getInputStreams().size() - 1 >= i) { SpDataStream stream = graph.getInputStreams().get(i); if (stream != null) { KafkaTransportProtocol protocol = (KafkaTransportProtocol) stream.getEventGrounding().getTransportProtocol(); //System.out.println("Listening on Kafka topic '" + protocol.getTopicName() + "'"); return KafkaUtils.createDirectStream(streamingContext, LocationStrategies.PreferConsistent(), ConsumerStrategies.<String, String>Subscribe(Arrays.asList(protocol.getTopicDefinition().getActualTopicName()), kafkaParams)); } else { return null; } } else { return null; } } }