/** * Get the expected high {@link Watermark} as a {@link JsonElement}. * * @return a {@link JsonElement} representing the expected high {@link Watermark}. */ public JsonElement getExpectedHighWatermark() { return JSON_PARSER.parse(getProp(ConfigurationKeys.WATERMARK_INTERVAL_VALUE_KEY)).getAsJsonObject() .get(WatermarkInterval.EXPECTED_HIGH_WATERMARK_TO_JSON_KEY); }
/** * Get the low {@link Watermark} as a {@link JsonElement}. * * @return a {@link JsonElement} representing the low {@link Watermark} or * {@code null} if the low {@link Watermark} is not set. */ public JsonElement getLowWatermark() { if (!contains(ConfigurationKeys.WATERMARK_INTERVAL_VALUE_KEY)) { return null; } return JSON_PARSER.parse(getProp(ConfigurationKeys.WATERMARK_INTERVAL_VALUE_KEY)).getAsJsonObject() .get(WatermarkInterval.LOW_WATERMARK_TO_JSON_KEY); }
@Override public String getProp(String key) { String value = super.getProp(key); if (value == null) { value = this.workUnit.getProp(key); } if (value == null) { value = this.jobState.getProp(key); } return value; }
@Override public String getProp(String key, String def) { String value = super.getProp(key); if (value == null) { value = this.workUnit.getProp(key); } if (value == null) { value = this.jobState.getProp(key, def); } return value; }
private String readProp(String key, WorkUnitState workUnitState) { String value = workUnitState.getWorkunit().getProp(key); if (StringUtils.isBlank(value)) { value = workUnitState.getProp(key); } if (StringUtils.isBlank(value)) { value = workUnitState.getJobState().getProp(key); } return value; }
/** * Set SLA event metadata in the workunit. The publisher will use this metadta to publish sla events */ public static void setTableSlaEventMetadata(WorkUnit state, Table table, long updateTime, long lowWatermark, long beginGetWorkunitsTime) { state.setProp(SlaEventKeys.DATASET_URN_KEY, state.getProp(ConfigurationKeys.DATASET_URN_KEY)); state.setProp(SlaEventKeys.PARTITION_KEY, table.getCompleteName()); state.setProp(SlaEventKeys.UPSTREAM_TS_IN_MILLI_SECS_KEY, String.valueOf(updateTime)); // Time when the workunit was created state.setProp(SlaEventKeys.ORIGIN_TS_IN_MILLI_SECS_KEY, System.currentTimeMillis()); state.setProp(EventConstants.WORK_UNIT_CREATE_TIME, state.getProp(SlaEventKeys.ORIGIN_TS_IN_MILLI_SECS_KEY)); state.setProp(EventConstants.BEGIN_GET_WORKUNITS_TIME, beginGetWorkunitsTime); state.setProp(SlaEventKeys.PREVIOUS_PUBLISH_TS_IN_MILLI_SECS_KEY, lowWatermark); }
/** * Prepare the job input. * @throws IOException */ private void prepareJobInput(List<WorkUnit> workUnits) throws IOException { Closer closer = Closer.create(); try { ParallelRunner parallelRunner = closer.register(new ParallelRunner(this.parallelRunnerThreads, this.fs)); int multiTaskIdSequence = 0; // Serialize each work unit into a file named after the task ID for (WorkUnit workUnit : workUnits) { String workUnitFileName; if (workUnit instanceof MultiWorkUnit) { workUnitFileName = JobLauncherUtils.newMultiTaskId(this.jobContext.getJobId(), multiTaskIdSequence++) + MULTI_WORK_UNIT_FILE_EXTENSION; } else { workUnitFileName = workUnit.getProp(ConfigurationKeys.TASK_ID_KEY) + WORK_UNIT_FILE_EXTENSION; } Path workUnitFile = new Path(this.jobInputPath, workUnitFileName); LOG.debug("Writing work unit file " + workUnitFileName); parallelRunner.serializeToFile(workUnit, workUnitFile); // Append the work unit file path to the job input file } } catch (Throwable t) { throw closer.rethrow(t); } finally { closer.close(); } }
/** * Set SLA event metadata in the workunit. The publisher will use this metadta to publish sla events */ public static void setPartitionSlaEventMetadata(WorkUnit state, Table table, Partition partition, long updateTime, long lowWatermark, long beginGetWorkunitsTime) { state.setProp(SlaEventKeys.DATASET_URN_KEY, state.getProp(ConfigurationKeys.DATASET_URN_KEY)); state.setProp(SlaEventKeys.PARTITION_KEY, partition.getName()); state.setProp(SlaEventKeys.UPSTREAM_TS_IN_MILLI_SECS_KEY, String.valueOf(updateTime)); // Time when the workunit was created state.setProp(SlaEventKeys.ORIGIN_TS_IN_MILLI_SECS_KEY, System.currentTimeMillis()); state.setProp(EventConstants.WORK_UNIT_CREATE_TIME, state.getProp(SlaEventKeys.ORIGIN_TS_IN_MILLI_SECS_KEY)); state.setProp(SlaEventKeys.PREVIOUS_PUBLISH_TS_IN_MILLI_SECS_KEY, lowWatermark); state.setProp(EventConstants.BEGIN_GET_WORKUNITS_TIME, beginGetWorkunitsTime); state.setProp(EventConstants.SOURCE_DATA_LOCATION, partition.getDataLocation()); }
private static void computeAndSetWorkUnitGuid(WorkUnit workUnit) throws IOException { Guid guid = Guid.fromStrings(workUnit.contains(ConfigurationKeys.CONVERTER_CLASSES_KEY) ? workUnit .getProp(ConfigurationKeys.CONVERTER_CLASSES_KEY) : ""); setWorkUnitGuid(workUnit, guid.append(deserializeCopyEntity(workUnit))); }
WorkUnit workUnit = this.workUnits.next(); countDownLatch.countUp(); String taskId = workUnit.getProp(ConfigurationKeys.TASK_ID_KEY); SubscopedBrokerBuilder<GobblinScopeTypes, ?> taskBrokerBuilder = this.jobBroker.newSubscopedBuilder(new TaskScopeInstance(taskId));
workUnit.getWorkunit().getProp(SlaEventKeys.ORIGIN_TS_IN_MILLI_SECS_KEY)));