/** * Return the next record when available. Will never time out since this is a streaming source. */ @Override public RecordEnvelope<D> readRecordEnvelopeImpl() throws DataRecordException, IOException { if (!_isStarted.get()) { throw new IOException("Streaming extractor has not been started."); } while ((_records == null) || (!_records.hasNext())) { synchronized (_consumer) { if (_close.get()) { throw new ClosedChannelException(); } _records = _consumer.poll(this.fetchTimeOut).iterator(); } } ConsumerRecord<S, D> record = _records.next(); _rowCount.getAndIncrement(); return new RecordEnvelope<D>(record.value(), new KafkaWatermark(_partition, new LongWatermark(record.offset()))); }
@Override public void start(WatermarkStorage watermarkStorage) throws IOException { Preconditions.checkArgument(watermarkStorage != null, "Watermark Storage should not be null"); Map<String, CheckpointableWatermark> watermarkMap = watermarkStorage.getCommittedWatermarks(KafkaWatermark.class, Collections.singletonList(_partition.toString())); KafkaWatermark watermark = (KafkaWatermark) watermarkMap.get(_partition.toString()); if (watermark == null) { LOG.info("Offset is null - seeking to beginning of topic and partition for {} ", _partition.toString()); _consumer.seekToBeginning(_partition); } else { // seek needs to go one past the last committed offset LOG.info("Offset found in consumer for partition {}. Seeking to one past what we found : {}", _partition.toString(), watermark.getLwm().getValue() + 1); _consumer.seek(_partition, watermark.getLwm().getValue() + 1); } _isStarted.set(true); }
new KafkaSimpleStreamingExtractor.KafkaWatermark(tP, new LongWatermark(0)); byte [] reuse = new byte[1]; RecordEnvelope<byte[]> oldRecord = new RecordEnvelope<>(reuse, kwm);
TopicPartition tP = new TopicPartition(topic, 0); KafkaSimpleStreamingExtractor.KafkaWatermark kwm = new KafkaSimpleStreamingExtractor.KafkaWatermark(tP, new LongWatermark(0)); byte[] reuse = new byte[1]; RecordEnvelope<byte[]> oldRecord = new RecordEnvelope<>(reuse, kwm);
public void run () { TopicPartition tP = new TopicPartition(topic, 0); KafkaSimpleStreamingExtractor.KafkaWatermark kwm = new KafkaSimpleStreamingExtractor.KafkaWatermark(tP, new LongWatermark(0)); byte[] reuse = new byte[1]; RecordEnvelope<byte[]> oldRecord = new RecordEnvelope<>(reuse, kwm); try { RecordEnvelope<byte[]> record = kSSE.readRecordEnvelope(); } catch (Exception e) { Assert.assertTrue((e instanceof WakeupException) || (e instanceof ClosedChannelException)); } } };
@Override public boolean equals(Object obj) { if (obj == null) { return false; } if (!(obj instanceof KafkaWatermark)) { return false; } return this.compareTo((CheckpointableWatermark) obj) == 0; }