/** * Returns the HWM of the workUnit * {@inheritDoc} * @see org.apache.gobblin.source.extractor.filebased.FileBasedExtractor#getHighWatermark() */ @Override public long getHighWatermark() { return this.workUnit.getHighWaterMark(); } }
@SuppressWarnings("deprecation") protected static WatermarkInterval getWatermarkIntervalFromMultiWorkUnit(MultiWorkUnit multiWorkUnit) { List<Long> lowWatermarkValues = Lists.newArrayList(); List<Long> expectedHighWatermarkValues = Lists.newArrayList(); for (WorkUnit workUnit : multiWorkUnit.getWorkUnits()) { lowWatermarkValues.add(workUnit.getLowWaterMark()); expectedHighWatermarkValues.add(workUnit.getHighWaterMark()); } return new WatermarkInterval(new MultiLongWatermark(lowWatermarkValues), new MultiLongWatermark(expectedHighWatermarkValues)); }
@SuppressWarnings("deprecation") protected static WatermarkInterval getWatermarkIntervalFromWorkUnit(WorkUnit workUnit) { if (workUnit instanceof MultiWorkUnit) { return getWatermarkIntervalFromMultiWorkUnit((MultiWorkUnit) workUnit); } List<Long> lowWatermarkValues = Lists.newArrayList(workUnit.getLowWaterMark()); List<Long> expectedHighWatermarkValues = Lists.newArrayList(workUnit.getHighWaterMark()); return new WatermarkInterval(new MultiLongWatermark(lowWatermarkValues), new MultiLongWatermark(expectedHighWatermarkValues)); }
Assert.assertEquals(workUnitList.size(), 2); Assert.assertEquals(workUnitList.get(0).getHighWaterMark(), 1000); Assert.assertEquals(workUnitList.get(0).getLowWaterMark(), 0); Assert.assertEquals(workUnitList.get(0).getProp("k1"), "v1"); Assert.assertEquals(workUnitList.get(1).getHighWaterMark(), 2000); Assert.assertEquals(workUnitList.get(1).getLowWaterMark(), 1001); Assert.assertEquals(workUnitList.get(1).getProp("k2"), "v2");
/** * Gets the LWM for this job runs. The new LWM is the HWM of the previous run + 1 unit (day,hour,minute..etc). * If there was no previous execution then it is set to the given lowWaterMark + 1 unit. */ private long getLowWaterMark(Iterable<WorkUnitState> previousStates, String lowWaterMark) { long lowWaterMarkValue = retriever.getWatermarkFromString(lowWaterMark); // Find the max HWM from the previous states, this is the new current LWM for (WorkUnitState previousState : previousStates) { if (previousState.getWorkingState().equals(WorkUnitState.WorkingState.COMMITTED)) { long previousHighWaterMark = previousState.getWorkunit().getHighWaterMark(); if (previousHighWaterMark > lowWaterMarkValue) { lowWaterMarkValue = previousHighWaterMark; } } } return lowWaterMarkValue + getRetriever().getWatermarkIncrementMs(); }
/** * Returns the HWM of the workUnit * {@inheritDoc} * @see org.apache.gobblin.source.extractor.filebased.FileBasedExtractor#getHighWatermark() */ @Override public long getHighWatermark() { return this.workUnit.getHighWaterMark(); } }
@SuppressWarnings("deprecation") protected static WatermarkInterval getWatermarkIntervalFromMultiWorkUnit(MultiWorkUnit multiWorkUnit) { List<Long> lowWatermarkValues = Lists.newArrayList(); List<Long> expectedHighWatermarkValues = Lists.newArrayList(); for (WorkUnit workUnit : multiWorkUnit.getWorkUnits()) { lowWatermarkValues.add(workUnit.getLowWaterMark()); expectedHighWatermarkValues.add(workUnit.getHighWaterMark()); } return new WatermarkInterval(new MultiLongWatermark(lowWatermarkValues), new MultiLongWatermark(expectedHighWatermarkValues)); }
@SuppressWarnings("deprecation") protected static WatermarkInterval getWatermarkIntervalFromWorkUnit(WorkUnit workUnit) { if (workUnit instanceof MultiWorkUnit) { return getWatermarkIntervalFromMultiWorkUnit((MultiWorkUnit) workUnit); } List<Long> lowWatermarkValues = Lists.newArrayList(workUnit.getLowWaterMark()); List<Long> expectedHighWatermarkValues = Lists.newArrayList(workUnit.getHighWaterMark()); return new WatermarkInterval(new MultiLongWatermark(lowWatermarkValues), new MultiLongWatermark(expectedHighWatermarkValues)); }
/** * Gets the LWM for this job runs. The new LWM is the HWM of the previous run + 1 unit (day,hour,minute..etc). * If there was no previous execution then it is set to the given lowWaterMark + 1 unit. */ private long getLowWaterMark(Iterable<WorkUnitState> previousStates, String lowWaterMark) { long lowWaterMarkValue = retriever.getWatermarkFromString(lowWaterMark); // Find the max HWM from the previous states, this is the new current LWM for (WorkUnitState previousState : previousStates) { if (previousState.getWorkingState().equals(WorkUnitState.WorkingState.COMMITTED)) { long previousHighWaterMark = previousState.getWorkunit().getHighWaterMark(); if (previousHighWaterMark > lowWaterMarkValue) { lowWaterMarkValue = previousHighWaterMark; } } } return lowWaterMarkValue + getRetriever().getWatermarkIncrementMs(); }