@Override public void close() throws Exception { readable.close(); if (watermarkTriggerService != null) { watermarkTriggerService.shutdown(); } }
/** * @param readables collection of readables * @return Set of source locations from source tasks in {@code taskDAG} * @throws Exception for any exception raised during querying source locations for a readable */ private static Set<String> getSourceDataLocations(final Collection<Readable> readables) throws Exception { final List<String> sourceLocations = new ArrayList<>(); for (final Readable readable : readables) { sourceLocations.addAll(readable.getLocations()); } return new HashSet<>(sourceLocations); }
private Object retrieveElement() throws NoSuchElementException, IOException { // Emit watermark if (!bounded && isWatermarkTriggerTime()) { return new Watermark(readable.readWatermark()); } // Data final Object element = readable.readCurrent(); return element; } }
/** * This is non-blocking operation. * @return current data * @throws NoSuchElementException if the current data is not available */ @Override Object fetchDataElement() throws NoSuchElementException, IOException { if (readable.isFinished()) { return Finishmark.getInstance(); } else { final long start = System.currentTimeMillis(); final Object element = retrieveElement(); boundedSourceReadTime += System.currentTimeMillis() - start; return element; } }
SourceVertexDataFetcher(final SourceVertex dataSource, final Readable readable, final OutputCollector outputCollector) { super(dataSource, outputCollector); this.readable = readable; this.readable.prepare(); this.bounded = dataSource.isBounded(); if (!bounded) { this.watermarkTriggerService = Executors.newScheduledThreadPool(1); this.watermarkTriggerService.scheduleAtFixedRate(() -> { watermarkTriggered = true; }, WATERMARK_PERIOD, WATERMARK_PERIOD, TimeUnit.MILLISECONDS); } else { this.watermarkTriggerService = null; } }
private Object retrieveElement() throws NoSuchElementException, IOException { // Emit watermark if (!bounded && isWatermarkTriggerTime()) { return new Watermark(readable.readWatermark()); } // Data final Object element = readable.readCurrent(); return element; } }
/** * This is non-blocking operation. * @return current data * @throws NoSuchElementException if the current data is not available */ @Override Object fetchDataElement() throws NoSuchElementException, IOException { if (readable.isFinished()) { return Finishmark.getInstance(); } else { final long start = System.currentTimeMillis(); final Object element = retrieveElement(); boundedSourceReadTime += System.currentTimeMillis() - start; return element; } }
SourceVertexDataFetcher(final SourceVertex dataSource, final Readable readable, final OutputCollector outputCollector) { super(dataSource, outputCollector); this.readable = readable; this.readable.prepare(); this.bounded = dataSource.isBounded(); if (!bounded) { this.watermarkTriggerService = Executors.newScheduledThreadPool(1); this.watermarkTriggerService.scheduleAtFixedRate(() -> { watermarkTriggered = true; }, WATERMARK_PERIOD, WATERMARK_PERIOD, TimeUnit.MILLISECONDS); } else { this.watermarkTriggerService = null; } }
@Override public void close() throws Exception { readable.close(); if (watermarkTriggerService != null) { watermarkTriggerService.shutdown(); } }
/** * @param readables collection of readables * @return Set of source locations from source tasks in {@code taskDAG} * @throws Exception for any exception raised during querying source locations for a readable */ private static Set<String> getSourceDataLocations(final Collection<Readable> readables) throws Exception { final List<String> sourceLocations = new ArrayList<>(); for (final Readable readable : readables) { sourceLocations.addAll(readable.getLocations()); } return new HashSet<>(sourceLocations); }