/** * Get a post-fork {@link TaskPublisher} for publishing data in the given branch. * * @param taskState {@link TaskState} of a {@link Task} * @param results Task-level policy checking results * @return a {@link TaskPublisher} */ public TaskPublisher getTaskPublisher(TaskState taskState, TaskLevelPolicyCheckResults results) throws Exception { return TaskPublisherBuilderFactory.newTaskPublisherBuilder(taskState, results).build(); }
/** * Is the publisher cacheable in the SharedResourcesBroker? * @param publisher * @return true if cacheable, else false */ public static boolean isPublisherCacheable(DataPublisher publisher) { // only threadsafe publishers are cacheable. non-threadsafe publishers are marked immediately for invalidation return publisher.supportsCapability(Capability.THREADSAFE, Collections.EMPTY_MAP); }
@Override public DataPublisher createDataPublisher(JobState.DatasetState datasetState) { return new NoopPublisher(datasetState); } }
public static <S extends ScopeType<S>> DataPublisher get(String publisherClassName, State state, SharedResourcesBroker<S> broker) throws IOException { try { return broker.getSharedResource(new DataPublisherFactory<S>(), new DataPublisherKey(publisherClassName, state)); } catch (NotConfiguredException nce) { throw new IOException(nce); } }
/** * This method publishes task output data for the given {@link WorkUnitState}, but if there are output data of * other tasks in the same folder, it may also publish those data. */ private void publishMultiTaskData(WorkUnitState state, int branchId, Set<Path> writerOutputPathsMoved) throws IOException { publishData(state, branchId, false, writerOutputPathsMoved); addLineageInfo(state, branchId); }
/** * First publish the metadata via {@link DataPublisher#publishMetadata(WorkUnitState)}, and then publish the output data * via the {@link DataPublisher#publishData(WorkUnitState)} method. * * @param state is a {@link WorkUnitState}. * @throws IOException if there is a problem with publishing the metadata or the data. */ public void publish(WorkUnitState state) throws IOException { publishMetadata(state); publishData(state); }
public HiveRegTaskStateCollectorServiceHandlerImpl(JobState jobState) { hiveRegHandler = new HiveRegistrationPublisher(jobState); }
@Override public void publish(Collection<? extends WorkUnitState> states) throws IOException { super.publish(states); // PUBLISHER_DIRS key must be updated for HiveRegistrationPublisher for (Path path : this.publisherOutputDirs) { this.state.appendToSetProp(ConfigurationKeys.PUBLISHER_DIRS, path.toString()); } this.hivePublisher.publish(states); } }
public static TaskPublisherBuilder newTaskPublisherBuilder(WorkUnitState workUnitState, TaskLevelPolicyCheckResults results) { return TaskPublisherBuilder.newBuilder(workUnitState, results); } }
@Override public void close() throws IOException { hiveRegHandler.close(); } }
public TaskPublisher build() throws Exception { return new TaskPublisher(this.workUnitState, this.results); } }
@Override public void handle(Collection<? extends WorkUnitState> taskStates) throws IOException { this.hiveRegHandler.publishData(taskStates); }
public static TaskPublisherBuilder newBuilder(WorkUnitState taskState, TaskLevelPolicyCheckResults results) { return new TaskPublisherBuilder(taskState, results); }
private FsWriterMetrics buildWriterMetrics(String fileName, String partitionKey, int writerId, int numRecords) { return buildWriterMetrics(fileName, partitionKey, writerId, 0, numRecords); }
@Override public DataPublisher createDataPublisher(JobState.DatasetState datasetState) { return new NoopPublisher(datasetState); } }
@Override public DataPublisher createDataPublisher(JobState.DatasetState datasetState) { return new NoopPublisher(datasetState); } }
@Override public DataPublisher createDataPublisher(JobState.DatasetState datasetState) { return new NoopPublisher(datasetState); } }
@Override public DataPublisher createDataPublisher(JobState.DatasetState datasetState) { return new NoopPublisher(datasetState); } }
@Override public DataPublisher createDataPublisher(JobState.DatasetState datasetState) { return new NoopPublisher(datasetState); } }