public final Map<String, CheckpointableWatermark> getCommittableWatermark() { return watermarkAwareWriter.get().getCommittableWatermark(); }
@Override public Map<String, CheckpointableWatermark> getCommittableWatermark() { // The committable watermark from a collection of commitable and unacknowledged watermarks is the highest // committable watermark that is less than the lowest unacknowledged watermark WatermarkTracker watermarkTracker = new MultiWriterWatermarkTracker(); for (Map.Entry<GenericRecord, DataWriter<D>> entry : this.partitionWriters.asMap().entrySet()) { if (entry.getValue() instanceof WatermarkAwareWriter) { Map<String, CheckpointableWatermark> commitableWatermarks = ((WatermarkAwareWriter) entry.getValue()).getCommittableWatermark(); if (!commitableWatermarks.isEmpty()) { watermarkTracker.committedWatermarks(commitableWatermarks); } Map<String, CheckpointableWatermark> unacknowledgedWatermark = ((WatermarkAwareWriter) entry.getValue()).getUnacknowledgedWatermark(); if (!unacknowledgedWatermark.isEmpty()) { watermarkTracker.unacknowledgedWatermarks(unacknowledgedWatermark); } } } return watermarkTracker.getAllCommitableWatermarks(); //TODO: Change this to use List of committables instead }
private boolean isDataWriterWatermarkCapable(DataWriter<D> dataWriter) { return (dataWriter instanceof WatermarkAwareWriter) && (((WatermarkAwareWriter) dataWriter).isWatermarkCapable()); }
public void testWatermarkComputation(Long committed, Long unacknowledged, Long expected) throws IOException { State state = new State(); state.setProp(ConfigurationKeys.WRITER_PARTITIONER_CLASS, TestPartitioner.class.getCanonicalName()); String defaultSource = "default"; WatermarkAwareWriter mockDataWriter = mock(WatermarkAwareWriter.class); when(mockDataWriter.isWatermarkCapable()).thenReturn(true); when(mockDataWriter.getCommittableWatermark()).thenReturn(Collections.singletonMap(defaultSource, new DefaultCheckpointableWatermark(defaultSource, new LongWatermark(committed)))); when(mockDataWriter.getUnacknowledgedWatermark()).thenReturn(Collections.singletonMap(defaultSource, new DefaultCheckpointableWatermark(defaultSource, new LongWatermark(unacknowledged)))); PartitionAwareDataWriterBuilder builder = mock(PartitionAwareDataWriterBuilder.class); when(builder.validatePartitionSchema(any(Schema.class))).thenReturn(true); when(builder.forPartition(any(GenericRecord.class))).thenReturn(builder); when(builder.withWriterId(any(String.class))).thenReturn(builder); when(builder.build()).thenReturn(mockDataWriter); PartitionedDataWriter writer = new PartitionedDataWriter<String, String>(builder, state); RecordEnvelope<String> recordEnvelope = new RecordEnvelope<String>("0"); recordEnvelope.addCallBack( new AcknowledgableWatermark(new DefaultCheckpointableWatermark(defaultSource, new LongWatermark(0)))); writer.writeEnvelope(recordEnvelope); Map<String, CheckpointableWatermark> watermark = writer.getCommittableWatermark(); System.out.println(watermark.toString()); if (expected == null) { Assert.assertTrue(watermark.isEmpty(), "Expected watermark to be absent"); } else { Assert.assertTrue(watermark.size() == 1); Assert.assertEquals((long) expected, ((LongWatermark) watermark.values().iterator().next().getWatermark()).getValue()); } }
public Map<String, CheckpointableWatermark> getUnacknowledgedWatermark() { return watermarkAwareWriter.get().getUnacknowledgedWatermark(); }
public void writeEnvelope(final RecordEnvelope<D> recordEnvelope) throws IOException { watermarkAwareWriter.get().writeEnvelope(recordEnvelope); }
@Override public Map<String, CheckpointableWatermark> getUnacknowledgedWatermark() { Preconditions.checkState(isWatermarkCapable()); return watermarkAwareWriter.get().getUnacknowledgedWatermark(); }
if (this.rowLevelPolicyChecker.executePolicies(convertedRecord, this.rowLevelPolicyCheckingResult)) { ((WatermarkAwareWriter) this.writer.get()).writeEnvelope( recordEnvelope.withRecord(convertedRecord));
@Override public Map<String, CheckpointableWatermark> getCommittableWatermark() { Preconditions.checkState(isWatermarkCapable()); return watermarkAwareWriter.get().getCommittableWatermark(); }
@Override public Map<String, CheckpointableWatermark> getCommittableWatermark() { // The committable watermark from a collection of commitable and unacknowledged watermarks is the highest // committable watermark that is less than the lowest unacknowledged watermark WatermarkTracker watermarkTracker = new MultiWriterWatermarkTracker(); for (Map.Entry<GenericRecord, DataWriter<D>> entry : this.partitionWriters.asMap().entrySet()) { if (entry.getValue() instanceof WatermarkAwareWriter) { Map<String, CheckpointableWatermark> commitableWatermarks = ((WatermarkAwareWriter) entry.getValue()).getCommittableWatermark(); if (!commitableWatermarks.isEmpty()) { watermarkTracker.committedWatermarks(commitableWatermarks); } Map<String, CheckpointableWatermark> unacknowledgedWatermark = ((WatermarkAwareWriter) entry.getValue()).getUnacknowledgedWatermark(); if (!unacknowledgedWatermark.isEmpty()) { watermarkTracker.unacknowledgedWatermarks(unacknowledgedWatermark); } } } return watermarkTracker.getAllCommitableWatermarks(); //TODO: Change this to use List of committables instead }
@Override public Map<String, CheckpointableWatermark> getUnacknowledgedWatermark() { WatermarkTracker watermarkTracker = new MultiWriterWatermarkTracker(); for (Map.Entry<GenericRecord, DataWriter<D>> entry : this.partitionWriters.asMap().entrySet()) { Map<String, CheckpointableWatermark> unacknowledgedWatermark = ((WatermarkAwareWriter) entry.getValue()).getUnacknowledgedWatermark(); if (!unacknowledgedWatermark.isEmpty()) { watermarkTracker.unacknowledgedWatermarks(unacknowledgedWatermark); } } return watermarkTracker.getAllUnacknowledgedWatermarks(); }
public final boolean isWatermarkCapable() { return watermarkAwareWriter.get().isWatermarkCapable(); }
public void writeEnvelope(final RecordEnvelope<D> recordEnvelope) throws IOException { watermarkAwareWriter.get().writeEnvelope(recordEnvelope); }
WatermarkTracker watermarkTracker = new MultiWriterWatermarkTracker(); for (WatermarkAwareWriter writer : _watermarkAwareWriters) { Map<String, CheckpointableWatermark> writerWatermarks = writer.getCommittableWatermark(); _logger.debug("Retrieved from writer {} : watermark {} ", writer.getClass().getName(), writerWatermarks); watermarkTracker.committedWatermarks(writerWatermarks);
public Map<String, CheckpointableWatermark> getUnacknowledgedWatermark() { return watermarkAwareWriter.get().getUnacknowledgedWatermark(); }
@Override public boolean isWatermarkCapable() { return watermarkAwareWriter.isPresent() && watermarkAwareWriter.get().isWatermarkCapable(); }
if (this.rowLevelPolicyChecker.executePolicies(convertedRecord, this.rowLevelPolicyCheckingResult)) { ((WatermarkAwareWriter) this.writer.get()).writeEnvelope( recordEnvelope.withRecord(convertedRecord));
/** * Test that when we have commits failing to watermark storage, the manager continues to try * at every interval and keeps track of the exception it is seeing. */ @Test public void testFailingWatermarkStorage() throws IOException, InterruptedException { WatermarkStorage reallyBadWatermarkStorage = mock(WatermarkStorage.class); IOException exceptionToThrow = new IOException("Failed to write coz the programmer told me to"); doThrow(exceptionToThrow).when(reallyBadWatermarkStorage).commitWatermarks(any(Iterable.class)); long commitInterval = 1000; MultiWriterWatermarkManager watermarkManager = new MultiWriterWatermarkManager(reallyBadWatermarkStorage, commitInterval, Optional.<Logger>absent()); WatermarkAwareWriter mockWriter = mock(WatermarkAwareWriter.class); CheckpointableWatermark watermark = new DefaultCheckpointableWatermark("default", new LongWatermark(0)); when(mockWriter.getCommittableWatermark()).thenReturn(Collections.singletonMap("default", watermark)); watermarkManager.registerWriter(mockWriter); try { watermarkManager.start(); } catch (Exception e) { Assert.fail("Should not throw exception", e); } Thread.sleep(commitInterval * 2 + (commitInterval/2)); // sleep for 2.5 iterations watermarkManager.close(); int expectedCalls = 3; // 2 calls from iterations, 1 additional attempt due to close verify(reallyBadWatermarkStorage, atLeast(expectedCalls)).commitWatermarks(any(Iterable.class)); Assert.assertEquals(watermarkManager.getCommitStatus().getLastCommitException(), exceptionToThrow, "Testing tracking of failed exceptions"); }
@Override public Map<String, CheckpointableWatermark> getUnacknowledgedWatermark() { Preconditions.checkState(isWatermarkCapable()); return watermarkAwareWriter.get().getUnacknowledgedWatermark(); }
private boolean isDataWriterWatermarkCapable(DataWriter<D> dataWriter) { return (dataWriter instanceof WatermarkAwareWriter) && (((WatermarkAwareWriter) dataWriter).isWatermarkCapable()); }