/** * Set the specified key, value pair in this {@link MultiWorkUnit} only, but do not propagate it to all the inner * {@link WorkUnit}s. * * @param key property key * @param value property value */ public void setPropExcludeInnerWorkUnits(String key, Object value) { super.setProp(key, value); }
/** * Factory method. * * @param other a {@link WorkUnit} instance * @return A copy of the given {@link WorkUnit} instance */ public static WorkUnit copyOf(WorkUnit other) { return new WorkUnit(other); }
/** * Set SLA event metadata in the workunit. The publisher will use this metadta to publish sla events */ public static void setTableSlaEventMetadata(WorkUnit state, Table table, long updateTime, long lowWatermark, long beginGetWorkunitsTime) { state.setProp(SlaEventKeys.DATASET_URN_KEY, state.getProp(ConfigurationKeys.DATASET_URN_KEY)); state.setProp(SlaEventKeys.PARTITION_KEY, table.getCompleteName()); state.setProp(SlaEventKeys.UPSTREAM_TS_IN_MILLI_SECS_KEY, String.valueOf(updateTime)); // Time when the workunit was created state.setProp(SlaEventKeys.ORIGIN_TS_IN_MILLI_SECS_KEY, System.currentTimeMillis()); state.setProp(EventConstants.WORK_UNIT_CREATE_TIME, state.getProp(SlaEventKeys.ORIGIN_TS_IN_MILLI_SECS_KEY)); state.setProp(EventConstants.BEGIN_GET_WORKUNITS_TIME, beginGetWorkunitsTime); state.setProp(SlaEventKeys.PREVIOUS_PUBLISH_TS_IN_MILLI_SECS_KEY, lowWatermark); }
public ImmutableWorkUnit(WorkUnit workUnit) { super(workUnit.getExtract()); // Only copy the specProperties from the given workUnit. Properties specificPropertiesCopy = new Properties(); specificPropertiesCopy.putAll(workUnit.getSpecProperties()); super.setProps(workUnit.getCommonProperties(), specificPropertiesCopy); }
@Override public void readFields(DataInput in) throws IOException { int numWorkUnits = in.readInt(); for (int i = 0; i < numWorkUnits; i++) { WorkUnit workUnit = WorkUnit.createEmpty(); workUnit.readFields(in); this.workUnits.add(workUnit); } super.readFields(in); }
CopyEntity.DatasetAndPartition datasetAndPartition = copyEntity.getDatasetAndPartition(metadata); WorkUnit workUnit = new WorkUnit(extract); workUnit.addAll(this.state); serializeCopyEntity(workUnit, copyEntity); serializeCopyableDataset(workUnit, metadata); GobblinMetrics.addCustomTagToState(workUnit, new Tag<>(CopyEventSubmitterHelper.DATASET_ROOT_METADATA_NAME, this.copyableDataset.datasetURN())); workUnit.setProp(ConfigurationKeys.DATASET_URN_KEY, datasetAndPartition.toString()); workUnit.setProp(SlaEventKeys.DATASET_URN_KEY, this.copyableDataset.datasetURN()); workUnit.setProp(SlaEventKeys.PARTITION_KEY, copyEntity.getFileSet()); setWorkUnitWeight(workUnit, copyEntity, minWorkUnitWeight); setWorkUnitWatermark(workUnit, watermarkGenerator, copyEntity);
@Override public List<WorkUnit> getWorkunits(SourceState state) { Config rootCfg = ConfigUtils.propertiesToConfig(state.getProperties()); Config cfg = rootCfg.hasPath(CONFIG_NAMESPACE) ? rootCfg.getConfig(CONFIG_NAMESPACE) : ConfigFactory.empty(); int numHellos = cfg.hasPath(NUM_HELLOS_KEY) ? cfg.getInt(NUM_HELLOS_KEY) : DEFAULT_NUM_HELLOS; Extract extract = new Extract(TableType.APPEND_ONLY, HelloWorldSource.class.getPackage().getName(), HelloWorldSource.class.getSimpleName()); List<WorkUnit> wus = new ArrayList<>(numHellos); for (int i = 1; i <= numHellos; ++i) { WorkUnit wu = new WorkUnit(extract); wu.setProp(HELLO_ID_FULL_KEY, i); wus.add(wu); } return wus; }
/** * Get the expected high {@link Watermark} as a {@link JsonElement}. * * @return a {@link JsonElement} representing the expected high {@link Watermark}. */ public JsonElement getExpectedHighWatermark() { return JSON_PARSER.parse(getProp(ConfigurationKeys.WATERMARK_INTERVAL_VALUE_KEY)).getAsJsonObject() .get(WatermarkInterval.EXPECTED_HIGH_WATERMARK_TO_JSON_KEY); }
WorkUnit watermarkWorkunit = WorkUnit.createEmpty(); watermarkWorkunit.setProp(IS_WATERMARK_WORKUNIT_KEY, true); watermarkWorkunit.setProp(ConfigurationKeys.DATASET_URN_KEY, tableKey); watermarkWorkunit.setWatermarkInterval(new WatermarkInterval(new MultiKeyValueLongWatermark( this.previousWatermarks.get(tableKey)), new MultiKeyValueLongWatermark(expectedPartitionWatermarks)));
/** * Get the low {@link Watermark} as a {@link JsonElement}. * * @return a {@link JsonElement} representing the low {@link Watermark} or * {@code null} if the low {@link Watermark} is not set. */ public JsonElement getLowWatermark() { if (!contains(ConfigurationKeys.WATERMARK_INTERVAL_VALUE_KEY)) { return null; } return JSON_PARSER.parse(getProp(ConfigurationKeys.WATERMARK_INTERVAL_VALUE_KEY)).getAsJsonObject() .get(WatermarkInterval.LOW_WATERMARK_TO_JSON_KEY); }
@Override protected void forWorkUnit(WorkUnit workUnit) { workUnit.setProp(ConfigurationKeys.JOB_ID_KEY, this.jobId); String taskId = JobLauncherUtils.newTaskId(this.jobId, this.taskIdSequence++); workUnit.setId(taskId); workUnit.setProp(ConfigurationKeys.TASK_ID_KEY, taskId); workUnit.setProp(ConfigurationKeys.TASK_KEY_KEY, Long.toString(Id.Task.parse(taskId).getSequence())); } }
@Override public List<WorkUnit> getWorkunits(SourceState state) { List<WorkUnit> workUnits = Lists.newArrayList(); if (!state.contains(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL)) { return workUnits; } // Create a single snapshot-type extract for all files Extract extract = new Extract(Extract.TableType.SNAPSHOT_ONLY, state.getProp(ConfigurationKeys.EXTRACT_NAMESPACE_NAME_KEY, "ExampleNamespace"), "ExampleTable"); String filesToPull = state.getProp(ConfigurationKeys.SOURCE_FILEBASED_FILES_TO_PULL); for (String file : Splitter.on(',').omitEmptyStrings().split(filesToPull)) { // Create one work unit for each file to pull WorkUnit workUnit = WorkUnit.create(extract); workUnit.setProp(SOURCE_FILE_KEY, file); workUnits.add(workUnit); } return workUnits; }
/** * Get the low {@link Watermark}. A default {@link Gson} object will be used to deserialize the watermark. * * @param watermarkClass the watermark class for this {@code WorkUnit}. * @return the low watermark in this {@code WorkUnit}. */ public <T extends Watermark> T getLowWatermark(Class<T> watermarkClass) { return getLowWatermark(watermarkClass, GSON); }
@Override public boolean apply(WorkUnit workUnit) { if (workUnit instanceof MultiWorkUnit) { Preconditions.checkArgument(!workUnit.contains(ConfigurationKeys.WORK_UNIT_SKIP_KEY), "Error: MultiWorkUnit cannot be skipped"); for (WorkUnit wu : ((MultiWorkUnit) workUnit).getWorkUnits()) { Preconditions.checkArgument(!wu.contains(ConfigurationKeys.WORK_UNIT_SKIP_KEY), "Error: MultiWorkUnit cannot contain skipped WorkUnit"); } } if (workUnit.getPropAsBoolean(ConfigurationKeys.WORK_UNIT_SKIP_KEY, false)) { WorkUnitState workUnitState = new WorkUnitState(workUnit, this.jobState); workUnitState.setWorkingState(WorkUnitState.WorkingState.SKIPPED); this.jobState.addSkippedTaskState(new TaskState(workUnitState)); return false; } return true; } }
/** * Default constructor used for deserialization. */ public WorkUnitState() { this.workUnit = WorkUnit.createEmpty(); this.jobState = new State(); // Not available on deserialization this.taskBroker = null; }
/** * Get the {@link gobblin.source.workunit.Extract} associated with the {@link WorkUnit}. * * @return {@link gobblin.source.workunit.Extract} associated with the {@link WorkUnit} */ public Extract getExtract() { return new Extract(this.workUnit.getExtract()); }
/** * Get the expected high {@link Watermark}. A default {@link Gson} object will be used to deserialize the watermark. * * @param watermarkClass the watermark class for this {@code WorkUnit}. * @return the expected high watermark in this {@code WorkUnit}. */ public <T extends Watermark> T getExpectedHighWatermark(Class<T> watermarkClass) { return getExpectedHighWatermark(watermarkClass, GSON); }
@Override public boolean contains(String key) { return super.contains(key) || this.workUnit.contains(key) || this.jobState.contains(key); }
@Override public long weight(WorkUnit workUnit) { return workUnit.getPropAsLong(this.field); } }
@Override public Properties getProperties() { Properties props = new Properties(); props.putAll(this.jobState.getProperties()); props.putAll(this.workUnit.getProperties()); props.putAll(super.getProperties()); return props; }