@Override public JobDataFragment range(int offset, int limit) { return delegate.range(offset, limit); }
@Override public JobDataFragment range(int offset, int limit) { return new JobDataFragmentWrapper(offset, delegate.range(offset, limit)); }
public JobData(Job job, int offset, int limit) { this.delegate = job.getData().range(offset, limit); this.job = job; }
public static List<String> getRefreshPath(final JobId jobId, final JobData jobData, final Path accelerationBasePath) { // extract written path from writer's metadata JobDataFragment data = jobData.range(0, 1); Text text = (Text) Preconditions.checkNotNull(data.extractValue(RecordWriter.PATH_COLUMN, 0), "Empty write path for job %s", jobId.getId()); // relative path to the acceleration base path final String path = PathUtils.relativePath(new Path(text.toString()), accelerationBasePath); // extract first 2 components of the path "<reflection-id>."<modified-materialization-id>" List<String> components = PathUtils.toPathComponents(path); Preconditions.checkState(components.size() >= 2, "Refresh path %s is incomplete", path); return ImmutableList.of(ACCELERATOR_STORAGEPLUGIN_NAME, components.get(0), components.get(1)); }
/** * @return next updateId */ private static long getUpdateId(final JobId jobId, final JobData jobData) { final int fetchLimit = 1000; long maxValue = Long.MIN_VALUE; int offset = 0; JobDataFragment data = jobData.range(offset, fetchLimit); while (data.getReturnedRowCount() > 0) { for (int i = 0; i < data.getReturnedRowCount(); i++) { byte[] b = (byte[]) data.extractValue(RecordWriter.METADATA_COLUMN, i); if(b == null) { throw new IllegalStateException("Didn't find metadata output for job " + jobId.getId()); } long val = Long.parseLong(new String(b)); maxValue = Math.max(maxValue, val); } offset += data.getReturnedRowCount(); data = jobData.range(offset, fetchLimit); } return maxValue; }