@Override public void start(WatermarkStorage watermarkStorage) throws IOException { this.watermarkStorage = Optional.of(watermarkStorage); Map<String, CheckpointableWatermark> lastCommitted; try { lastCommitted = this.watermarkStorage.get() .getCommittedWatermarks(DefaultCheckpointableWatermark.class, ImmutableList.of("" + extractor.partition)); } catch (IOException e) { // failed to get watermarks ... log a warning message log.warn("Failed to get watermarks... will start from the beginning", e); lastCommitted = Collections.EMPTY_MAP; } for (Map.Entry entry: lastCommitted.entrySet()) { log.info("{}: Found these committed watermarks: key: {}, value: {}", this, entry.getKey(), entry.getValue()); } LongWatermark currentWatermark; if (!lastCommitted.isEmpty() && lastCommitted.containsKey(""+extractor.partition)) { currentWatermark = (LongWatermark) (lastCommitted.get("" + extractor.partition)).getWatermark(); } else { // first record starts from 0 currentWatermark = new LongWatermark(-1); } extractor.setCurrentWatermark(currentWatermark); log.info("{}: Set current watermark to : {}", this, currentWatermark); } };
_logger.info("Will commit watermark {}", watermarksToCommit.toString()); _watermarkStorage.commitWatermarks(watermarksToCommit.values()); _commitStatus.onSuccess(watermarksToCommit); } catch (Exception e) {
@Override public void start(WatermarkStorage watermarkStorage) throws IOException { Preconditions.checkArgument(watermarkStorage != null, "Watermark Storage should not be null"); Map<String, CheckpointableWatermark> watermarkMap = watermarkStorage.getCommittedWatermarks(KafkaWatermark.class, Collections.singletonList(_partition.toString())); KafkaWatermark watermark = (KafkaWatermark) watermarkMap.get(_partition.toString()); if (watermark == null) { LOG.info("Offset is null - seeking to beginning of topic and partition for {} ", _partition.toString()); _consumer.seekToBeginning(_partition); } else { // seek needs to go one past the last committed offset LOG.info("Offset found in consumer for partition {}. Seeking to one past what we found : {}", _partition.toString(), watermark.getLwm().getValue() + 1); _consumer.seek(_partition, watermark.getLwm().getValue() + 1); } _isStarted.set(true); }
_logger.info("Will commit watermark {}", watermarksToCommit.toString()); _watermarkStorage.commitWatermarks(watermarksToCommit.values()); _commitStatus.onSuccess(watermarksToCommit); } catch (Exception e) {
currentIteration++; externalWatermarkStorage = mockWatermarkStorage.getCommittedWatermarks(CheckpointableWatermark.class, ImmutableList .of("default")); if (!externalWatermarkStorage.isEmpty()) { mockWatermarkStorage.getCommittedWatermarks(CheckpointableWatermark.class, ImmutableList .of("0"));
/** * Test that when no sources are registered, no side effects are observed */ @Test public void testNoRegisteredSource() throws IOException, InterruptedException { WatermarkStorage mockWatermarkStorage = mock(WatermarkStorage.class); MultiWriterWatermarkManager watermarkManager = new MultiWriterWatermarkManager(mockWatermarkStorage, 1000, Optional.<Logger>absent()); try { watermarkManager.start(); } catch (Exception e) { Assert.fail("Should not throw exception", e); } Thread.sleep(2000); watermarkManager.close(); verify(mockWatermarkStorage, times(0)).commitWatermarks(any(Iterable.class)); MultiWriterWatermarkManager.CommitStatus watermarkMgrStatus = watermarkManager.getCommitStatus(); Assert.assertTrue(watermarkMgrStatus.getLastCommittedWatermarks().isEmpty(), "Last committed watermarks should be empty"); Assert.assertEquals(watermarkMgrStatus.getLastWatermarkCommitSuccessTimestampMillis(), 0 , "Last committed watermark timestamp should be 0"); }
currentIteration++; Map<String, CheckpointableWatermark> externalWatermarkStorage = mockTaskContext.getWatermarkStorage() .getCommittedWatermarks(CheckpointableWatermark.class, ImmutableList.of("default")); if (!externalWatermarkStorage.isEmpty()) { for (CheckpointableWatermark watermark : externalWatermarkStorage.values()) { .getCommittedWatermarks(CheckpointableWatermark.class, ImmutableList.of("default"))));
/** * Test that when we have commits failing to watermark storage, the manager continues to try * at every interval and keeps track of the exception it is seeing. */ @Test public void testFailingWatermarkStorage() throws IOException, InterruptedException { WatermarkStorage reallyBadWatermarkStorage = mock(WatermarkStorage.class); IOException exceptionToThrow = new IOException("Failed to write coz the programmer told me to"); doThrow(exceptionToThrow).when(reallyBadWatermarkStorage).commitWatermarks(any(Iterable.class)); long commitInterval = 1000; MultiWriterWatermarkManager watermarkManager = new MultiWriterWatermarkManager(reallyBadWatermarkStorage, commitInterval, Optional.<Logger>absent()); WatermarkAwareWriter mockWriter = mock(WatermarkAwareWriter.class); CheckpointableWatermark watermark = new DefaultCheckpointableWatermark("default", new LongWatermark(0)); when(mockWriter.getCommittableWatermark()).thenReturn(Collections.singletonMap("default", watermark)); watermarkManager.registerWriter(mockWriter); try { watermarkManager.start(); } catch (Exception e) { Assert.fail("Should not throw exception", e); } Thread.sleep(commitInterval * 2 + (commitInterval/2)); // sleep for 2.5 iterations watermarkManager.close(); int expectedCalls = 3; // 2 calls from iterations, 1 additional attempt due to close verify(reallyBadWatermarkStorage, atLeast(expectedCalls)).commitWatermarks(any(Iterable.class)); Assert.assertEquals(watermarkManager.getCommitStatus().getLastCommitException(), exceptionToThrow, "Testing tracking of failed exceptions"); }
when(mockWatermarkStorage.getCommittedWatermarks(any(Class.class), any(Iterable.class))) .thenReturn(committedWatermarks);
_logger.info("Will commit watermark {}", watermarksToCommit.toString()); _watermarkStorage.commitWatermarks(watermarksToCommit.values()); _commitStatus.onSuccess(watermarksToCommit); } catch (Exception e) {
@Override public void start(WatermarkStorage watermarkStorage) throws IOException { this.watermarkStorage = Optional.of(watermarkStorage); Map<String, CheckpointableWatermark> lastCommitted; try { lastCommitted = this.watermarkStorage.get() .getCommittedWatermarks(DefaultCheckpointableWatermark.class, ImmutableList.of("" + extractor.partition)); } catch (IOException e) { // failed to get watermarks ... log a warning message log.warn("Failed to get watermarks... will start from the beginning", e); lastCommitted = Collections.EMPTY_MAP; } for (Map.Entry entry: lastCommitted.entrySet()) { log.info("{}: Found these committed watermarks: key: {}, value: {}", this, entry.getKey(), entry.getValue()); } LongWatermark currentWatermark; if (!lastCommitted.isEmpty() && lastCommitted.containsKey(""+extractor.partition)) { currentWatermark = (LongWatermark) (lastCommitted.get("" + extractor.partition)).getWatermark(); } else { // first record starts from 0 currentWatermark = new LongWatermark(-1); } extractor.setCurrentWatermark(currentWatermark); log.info("{}: Set current watermark to : {}", this, currentWatermark); } };
_logger.info("Will commit watermark {}", watermarksToCommit.toString()); _watermarkStorage.commitWatermarks(watermarksToCommit.values()); _commitStatus.onSuccess(watermarksToCommit); } catch (Exception e) {