public static void addResource(Configuration hiveConf, Configuration conf) { if (Hadoop.Configuration.addResource.isNoop()) { for (Map.Entry<String, String> entry : conf) { hiveConf.set(entry.getKey(), entry.getValue()); } } else { Hadoop.Configuration.addResource.invoke(hiveConf, conf); } } }
public static void addResource(Configuration hiveConf, Configuration conf) { if (Hadoop.Configuration.addResource.isNoop()) { for (Map.Entry<String, String> entry : conf) { hiveConf.set(entry.getKey(), entry.getValue()); } } else { Hadoop.Configuration.addResource.invoke(hiveConf, conf); } } }
private static String getJobDatasetName(JobContext jobContext) { return Hadoop.JobContext.getJobID.invoke(jobContext).toString(); }
/** * Configures the {@code Job} to use the {@code DatasetKeyInputFormat} and * returns a helper to add further configuration. * * @param job the {@code Job} to configure * * @since 0.15.0 */ public static ConfigBuilder configure(Job job) { job.setInputFormatClass(DatasetKeyInputFormat.class); Configuration conf = Hadoop.JobContext.getConfiguration.invoke(job); return new ConfigBuilder(conf); }
public <R> R invoke(Object... args) { return method.invoke(receiver, args); } }
@SuppressWarnings("deprecation") private static <E> View<E> load(JobContext jobContext) { Configuration conf = Hadoop.JobContext.getConfiguration.invoke(jobContext); Class<E> type = getType(jobContext); String outputUri = conf.get(KITE_OUTPUT_URI); return Datasets.<E, View<E>>load(outputUri, type); }
private static DatasetRepository getDatasetRepository(JobContext jobContext) { Configuration conf = Hadoop.JobContext.getConfiguration.invoke(jobContext); DatasetRepository repo = DatasetRepositories.repositoryFor(conf.get(KITE_OUTPUT_URI)); if (repo instanceof TemporaryDatasetRepositoryAccessor) { Dataset<Object> dataset = load(jobContext).getDataset(); String namespace = dataset.getNamespace(); repo = ((TemporaryDatasetRepositoryAccessor) repo) .getTemporaryRepository(namespace, getJobDatasetName(jobContext)); } return repo; }
@Override public List<InputSplit> getSplits(JobContext jobContext) throws IOException { Configuration conf = Hadoop.JobContext.getConfiguration.invoke(jobContext); return getDelegate(conf).getSplits(jobContext); }
@BeforeClass public static void setupFS() throws IOException { final Configuration conf = new Configuration(); cluster = new MiniDFSCluster(new Configuration(), 1, true, null); // Builder is not compatible with hadoop1 //cluster = new MiniDFSCluster.Builder(conf).build(); dfs = getFS.invoke(cluster); lfs = FileSystem.getLocal(conf); }
@Override public List<InputSplit> getSplits(JobContext jobContext) throws IOException { Configuration conf = Hadoop.JobContext.getConfiguration.invoke(jobContext); return getDelegate(conf).getSplits(jobContext); }
@Override public void flush() throws IOException { writer.flush(); Hadoop.FSDataOutputStream.hflush.invoke(outgoing); }
@SuppressWarnings("deprecation") private static <E> View<E> load(JobContext jobContext) { Configuration conf = Hadoop.JobContext.getConfiguration.invoke(jobContext); Class<E> type = getType(jobContext); String outputUri = conf.get(KITE_OUTPUT_URI); return Datasets.<E, View<E>>load(outputUri, type); }
private static DatasetRepository getDatasetRepository(JobContext jobContext) { Configuration conf = Hadoop.JobContext.getConfiguration.invoke(jobContext); DatasetRepository repo = DatasetRepositories.repositoryFor(conf.get(KITE_OUTPUT_URI)); if (repo instanceof TemporaryDatasetRepositoryAccessor) { Dataset<Object> dataset = load(jobContext).getDataset(); String namespace = dataset.getNamespace(); repo = ((TemporaryDatasetRepositoryAccessor) repo) .getTemporaryRepository(namespace, getJobDatasetName(jobContext)); } return repo; }
@Override public OutputCommitter getOutputCommitter(TaskAttemptContext taskAttemptContext) { Configuration conf = Hadoop.TaskAttemptContext.getConfiguration.invoke(taskAttemptContext); DefaultConfiguration.init(conf); View<E> view = load(taskAttemptContext); return usePerTaskAttemptDatasets(view, conf) ? new MergeOutputCommitter<E>() : new NullOutputCommitter(); }
@Override public OutputCommitter getOutputCommitter(TaskAttemptContext taskAttemptContext) { Configuration conf = Hadoop.TaskAttemptContext.getConfiguration.invoke(taskAttemptContext); DefaultConfiguration.init(conf); View<E> view = load(taskAttemptContext); return usePerTaskAttemptDatasets(view) ? new MergeOutputCommitter<E>() : new NullOutputCommitter(); }
@Override public void sync() throws IOException { flush(); Hadoop.FSDataOutputStream.hsync.invoke(outgoing); }
@Override public void sync() throws IOException { flush(); Hadoop.FSDataOutputStream.hsync.invoke(out); }
@Override public void flush() throws IOException { // Avro sync forces the end of the current block so the data is recoverable dataFileWriter.flush(); Hadoop.FSDataOutputStream.hflush.invoke(out); }
@BeforeClass public static void setupFS() throws IOException { final Configuration conf = new Configuration(); cluster = new MiniDFSCluster(new Configuration(), 1, true, null); // Builder is not compatible with hadoop1 //cluster = new MiniDFSCluster.Builder(conf).build(); dfs = getFS.invoke(cluster); }
@Override @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR", justification="Delegate set by setConf") public RecordReader<E, Void> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { Configuration conf = Hadoop.TaskAttemptContext.getConfiguration.invoke(taskAttemptContext); DefaultConfiguration.init(conf); return delegate.createRecordReader(inputSplit, taskAttemptContext); }