@Override public WorkUnitStream getWorkunitStream(SourceState state) { try { return new BasicWorkUnitStream.Builder(createWorkUnitStream(state).iterator()).build(); } catch (IOException ioe) { throw new RuntimeException(ioe); } }
@Override public WorkUnitStream getWorkunitStream(SourceState state) { try { if (this.source instanceof WorkUnitStreamSource) { return ((WorkUnitStreamSource) this.source).getWorkunitStream(state); } List<WorkUnit> workUnits = this.source.getWorkunits(state); if (workUnits == null) { // Return an empty list if no work units are returned by the source workUnits = Collections.emptyList(); } return new BasicWorkUnitStream.Builder(workUnits).build(); } catch (Throwable t) { this.logger.error("Failed to get work units for job " + this.jobId, t); // Return null in case of errors return null; } }
previousPartitionUrnWatermark = maxWorkUnit.get().getProp(PARTITION_URN); return new BasicWorkUnitStream.Builder(getWorkUnitIterator(datasetStream.iterator(), previousDatasetUrnWatermark, previousPartitionUrnWatermark, maximumWorkUnits)).setFiniteStream(true).build();
workUnitStream = ((WorkUnitStreamSource) source).getWorkunitStream(jobState); } else { workUnitStream = new BasicWorkUnitStream.Builder(source.getWorkunits(jobState)).build();
previousPartitionUrnWatermark = maxWorkUnit.get().getProp(PARTITION_URN); return new BasicWorkUnitStream.Builder(getWorkUnitIterator(datasetStream.iterator(), previousDatasetUrnWatermark, previousPartitionUrnWatermark, maximumWorkUnits)).setFiniteStream(true).build();
@Override public WorkUnitStream getWorkunitStream(SourceState state) { try { fs = getSourceFileSystem(state); state.setProp(COMPACTION_INIT_TIME, DateTimeUtils.currentTimeMillis()); suite = CompactionSuiteUtils.getCompactionSuiteFactory(state).createSuite(state); initRequestAllocator(state); initJobDir(state); copyJarDependencies(state); DatasetsFinder finder = DatasetUtils.instantiateDatasetFinder(state.getProperties(), getSourceFileSystem(state), DefaultFileSystemGlobFinder.class.getName()); List<Dataset> datasets = finder.findDatasets(); CompactionWorkUnitIterator workUnitIterator = new CompactionWorkUnitIterator (); // Spawn a single thread to create work units new Thread(new SingleWorkUnitGeneratorService (state, prioritize(datasets, state), workUnitIterator), "SingleWorkUnitGeneratorService").start(); return new BasicWorkUnitStream.Builder (workUnitIterator).build(); } catch (IOException e) { throw new RuntimeException(e); } }
@Override public WorkUnitStream getWorkunitStream(SourceState state) { try { if (this.source instanceof WorkUnitStreamSource) { return ((WorkUnitStreamSource) this.source).getWorkunitStream(state); } List<WorkUnit> workUnits = this.source.getWorkunits(state); if (workUnits == null) { // Return an empty list if no work units are returned by the source workUnits = Collections.emptyList(); } return new BasicWorkUnitStream.Builder(workUnits).build(); } catch (Throwable t) { this.logger.error("Failed to get work units for job " + this.jobId, t); // Return null in case of errors return null; } }
@Override public WorkUnitStream getWorkunitStream(SourceState state) { try { return new BasicWorkUnitStream.Builder(createWorkUnitStream(state).iterator()).build(); } catch (IOException ioe) { throw new RuntimeException(ioe); } }
workUnitStream = ((WorkUnitStreamSource) source).getWorkunitStream(jobState); } else { workUnitStream = new BasicWorkUnitStream.Builder(source.getWorkunits(jobState)).build();
@Override public WorkUnitStream getWorkunitStream(SourceState state) { try { fs = getSourceFileSystem(state); state.setProp(COMPACTION_INIT_TIME, DateTimeUtils.currentTimeMillis()); suite = CompactionSuiteUtils.getCompactionSuiteFactory(state).createSuite(state); initRequestAllocator(state); initJobDir(state); copyJarDependencies(state); DatasetsFinder finder = DatasetUtils.instantiateDatasetFinder(state.getProperties(), getSourceFileSystem(state), DefaultFileSystemGlobFinder.class.getName()); List<Dataset> datasets = finder.findDatasets(); CompactionWorkUnitIterator workUnitIterator = new CompactionWorkUnitIterator (); // Spawn a single thread to create work units new Thread(new SingleWorkUnitGeneratorService (state, prioritize(datasets, state), workUnitIterator), "SingleWorkUnitGeneratorService").start(); return new BasicWorkUnitStream.Builder (workUnitIterator).build(); } catch (IOException e) { throw new RuntimeException(e); } }
@Override public WorkUnitStream getWorkunitStream(SourceState state) { int numTasks = state.getPropAsInt(NUM_TASKS_KEY); String eventBusId = state.getProp(EventBusPublishingTaskFactory.EVENTBUS_ID_KEY); EventBus eventBus = TestingEventBuses.getEventBus(eventBusId); return new BasicWorkUnitStream.Builder(new WorkUnitIterator(eventBus, eventBusId, numTasks)).build(); }