@Override public void ack() { _watermark.ack(); }
/** * Create a derived record envelope from this one. * Derived envelopes share the same watermark. * The original envelope must be acknowledged separately. */ public AcknowledgableRecordEnvelope derivedEnvelope(D record) { _watermark.incrementAck(); return new AcknowledgableRecordEnvelope(record, _watermark); }
public Map<String, CheckpointableWatermark> getUnacknowledgedWatermarks() { Map<String, CheckpointableWatermark> unackedWatermarks = new HashMap<>(_watermarksMap.size()); for (Map.Entry<String, Deque<AcknowledgableWatermark>> entry: _watermarksMap.entrySet()) { String source = entry.getKey(); Iterable<AcknowledgableWatermark> watermarks = entry.getValue(); AcknowledgableWatermark lowestUnacked = null; for (AcknowledgableWatermark watermark: watermarks) { if (!watermark.isAcked()) { lowestUnacked = watermark; break; } } if (lowestUnacked != null) { unackedWatermarks.put(source, lowestUnacked.getCheckpointableWatermark()); } } return unackedWatermarks; }
AcknowledgableWatermark ackableWatermark = new AcknowledgableWatermark(recordEnvelope.getWatermark()); if (watermarkTracker.isPresent()) { watermarkTracker.get().track(ackableWatermark); ackableWatermark.incrementAck()); ackableWatermark.ack();
AcknowledgableWatermark ackable = new AcknowledgableWatermark(checkpointableWatermark); acknowledgableWatermarks[i] = ackable; tracker.track(ackable); acknowledgableWatermarks[i].ack();
watermark.ack(); if (isStreamingTask()) { ((RecordEnvelope) recordForFork).addCallBack(watermark.incrementAck()); watermark.ack();
private void writeEnvelope(ConsoleWriter consoleWriter, String content, String source, long value) throws IOException { CheckpointableWatermark watermark = new DefaultCheckpointableWatermark(source, new LongWatermark(value)); AcknowledgableWatermark ackable = new AcknowledgableWatermark(watermark); RecordEnvelope<String> mockEnvelope = (RecordEnvelope<String>) new RecordEnvelope<>(content).addCallBack(ackable); consoleWriter.writeEnvelope(mockEnvelope); Assert.assertTrue(ackable.isAcked()); }
public void testWatermarkComputation(Long committed, Long unacknowledged, Long expected) throws IOException { State state = new State(); state.setProp(ConfigurationKeys.WRITER_PARTITIONER_CLASS, TestPartitioner.class.getCanonicalName()); String defaultSource = "default"; WatermarkAwareWriter mockDataWriter = mock(WatermarkAwareWriter.class); when(mockDataWriter.isWatermarkCapable()).thenReturn(true); when(mockDataWriter.getCommittableWatermark()).thenReturn(Collections.singletonMap(defaultSource, new DefaultCheckpointableWatermark(defaultSource, new LongWatermark(committed)))); when(mockDataWriter.getUnacknowledgedWatermark()).thenReturn(Collections.singletonMap(defaultSource, new DefaultCheckpointableWatermark(defaultSource, new LongWatermark(unacknowledged)))); PartitionAwareDataWriterBuilder builder = mock(PartitionAwareDataWriterBuilder.class); when(builder.validatePartitionSchema(any(Schema.class))).thenReturn(true); when(builder.forPartition(any(GenericRecord.class))).thenReturn(builder); when(builder.withWriterId(any(String.class))).thenReturn(builder); when(builder.build()).thenReturn(mockDataWriter); PartitionedDataWriter writer = new PartitionedDataWriter<String, String>(builder, state); RecordEnvelope<String> recordEnvelope = new RecordEnvelope<String>("0"); recordEnvelope.addCallBack( new AcknowledgableWatermark(new DefaultCheckpointableWatermark(defaultSource, new LongWatermark(0)))); writer.writeEnvelope(recordEnvelope); Map<String, CheckpointableWatermark> watermark = writer.getCommittableWatermark(); System.out.println(watermark.toString()); if (expected == null) { Assert.assertTrue(watermark.isEmpty(), "Expected watermark to be absent"); } else { Assert.assertTrue(watermark.size() == 1); Assert.assertEquals((long) expected, ((LongWatermark) watermark.values().iterator().next().getWatermark()).getValue()); } }
/** * Get the original watermark that was attached to this record, * typically by a {@link org.apache.gobblin.source.extractor.StreamingExtractor} */ public CheckpointableWatermark getWatermark() { return _watermark.getCheckpointableWatermark(); }
AcknowledgableWatermark ackableWatermark = new AcknowledgableWatermark(recordEnvelope.getWatermark()); if (watermarkTracker.isPresent()) { watermarkTracker.get().track(ackableWatermark); ackableWatermark.incrementAck()); ackableWatermark.ack();
AcknowledgableWatermark ackable = new AcknowledgableWatermark(checkpointableWatermark); acknowledgableWatermarks[i] = ackable; tracker.track(ackable); acknowledgableWatermarks[i].ack();
watermark.ack(); if (isStreamingTask()) { ((RecordEnvelope) recordForFork).addCallBack(watermark.incrementAck()); watermark.ack();
final AcknowledgableWatermark ackable = new AcknowledgableWatermark(checkpointableWatermark); tracker.track(ackable); acknowledgableWatermarks[i] = ackable;
/** * Track a watermark. * Assumptions: Track is called sequentially from the same thread for watermarks that are * progressively increasing. */ public void track(AcknowledgableWatermark acknowledgableWatermark) { if (!_started.get() && _autoStart) { start(); } maybeAbort(); String source = acknowledgableWatermark.getCheckpointableWatermark().getSource(); Deque<AcknowledgableWatermark> sourceWatermarks = _watermarksMap.get(source); if (sourceWatermarks == null) { sourceWatermarks = new ConcurrentLinkedDeque<>(); _watermarksMap.put(source, sourceWatermarks); } sourceWatermarks.add(acknowledgableWatermark); _watermarksInserted.mark(); }
public Map<String, CheckpointableWatermark> getCommittableWatermarks() { Map<String, CheckpointableWatermark> commitableWatermarks = new HashMap<String, CheckpointableWatermark>(_watermarksMap.size()); for (Map.Entry<String, Deque<AcknowledgableWatermark>> entry: _watermarksMap.entrySet()) { String source = entry.getKey(); Iterable<AcknowledgableWatermark> watermarks = entry.getValue(); AcknowledgableWatermark highestWatermark = null; for (AcknowledgableWatermark watermark: watermarks) { if (watermark.isAcked()) { highestWatermark = watermark; } else { // hopefully we've already found the highest contiguous acked watermark break; } } if (highestWatermark != null) { commitableWatermarks.put(source, highestWatermark.getCheckpointableWatermark()); } } return commitableWatermarks; }
@Override public Object call() throws Exception { ackable.ack(); numAcks.decrementAndGet(); return null; } }, sleepTime, TimeUnit.MILLISECONDS);
AcknowledgableWatermark ackableWatermark = new AcknowledgableWatermark(r.getWatermark()); if (watermarkTracker.isPresent()) { watermarkTracker.get().track(ackableWatermark);
/** * Create a derived record envelope from this one. * Derived envelopes share the same watermark. * The original envelope must be acknowledged separately. */ public AcknowledgableRecordEnvelope derivedEnvelope(D record) { _watermark.incrementAck(); return new AcknowledgableRecordEnvelope(record, _watermark); }