private static <E> Dataset<E> loadJobDataset(JobContext jobContext) { DatasetRepository repo = getDatasetRepository(jobContext); return repo.load(TEMP_NAMESPACE, getJobDatasetName(jobContext)); }
private static void deleteJobDataset(JobContext jobContext) { DatasetRepository repo = getDatasetRepository(jobContext); repo.delete(TEMP_NAMESPACE, getJobDatasetName(jobContext)); }
private static <E> Dataset<E> loadJobDataset(JobContext jobContext) { DatasetRepository repo = getDatasetRepository(jobContext); return repo.load(TEMP_NAMESPACE, getJobDatasetName(jobContext)); }
private static void deleteJobDataset(JobContext jobContext) { DatasetRepository repo = getDatasetRepository(jobContext); repo.delete(TEMP_NAMESPACE, getJobDatasetName(jobContext)); }
private static DatasetRepository getDatasetRepository(JobContext jobContext) { Configuration conf = Hadoop.JobContext.getConfiguration.invoke(jobContext); DatasetRepository repo = DatasetRepositories.repositoryFor(conf.get(KITE_OUTPUT_URI)); if (repo instanceof TemporaryDatasetRepositoryAccessor) { Dataset<Object> dataset = load(jobContext).getDataset(); String namespace = dataset.getNamespace(); repo = ((TemporaryDatasetRepositoryAccessor) repo) .getTemporaryRepository(namespace, getJobDatasetName(jobContext)); } return repo; }
private static DatasetRepository getDatasetRepository(JobContext jobContext) { Configuration conf = Hadoop.JobContext.getConfiguration.invoke(jobContext); DatasetRepository repo = DatasetRepositories.repositoryFor(conf.get(KITE_OUTPUT_URI)); if (repo instanceof TemporaryDatasetRepositoryAccessor) { Dataset<Object> dataset = load(jobContext).getDataset(); String namespace = dataset.getNamespace(); repo = ((TemporaryDatasetRepositoryAccessor) repo) .getTemporaryRepository(namespace, getJobDatasetName(jobContext)); } return repo; }
/** * The job dataset may already exist if the ApplicationMaster was restarted */ @SuppressWarnings("unchecked") private static <E> Dataset<E> loadOrCreateJobDataset(JobContext jobContext) { Dataset<Object> dataset = load(jobContext).getDataset(); String jobDatasetName = getJobDatasetName(jobContext); DatasetRepository repo = getDatasetRepository(jobContext); if (repo.exists(TEMP_NAMESPACE, jobDatasetName)) { Dataset<E> tempDataset = repo.load(TEMP_NAMESPACE, jobDatasetName, DatasetKeyOutputFormat.<E>getType(jobContext)); try { Compatibility.checkCompatible(dataset.getDescriptor(), tempDataset.getDescriptor()); return tempDataset; } catch (RuntimeException ex) { // swallow } } return repo.create(TEMP_NAMESPACE, jobDatasetName, copy(dataset.getDescriptor()), DatasetKeyOutputFormat.<E>getType(jobContext)); }
/** * The job dataset may already exist if the ApplicationMaster was restarted */ @SuppressWarnings("unchecked") private static <E> Dataset<E> loadOrCreateJobDataset(JobContext jobContext) { Dataset<Object> dataset = load(jobContext).getDataset(); String jobDatasetName = getJobDatasetName(jobContext); DatasetRepository repo = getDatasetRepository(jobContext); if (repo.exists(TEMP_NAMESPACE, jobDatasetName)) { Dataset<E> tempDataset = repo.load(TEMP_NAMESPACE, jobDatasetName, DatasetKeyOutputFormat.<E>getType(jobContext)); try { Compatibility.checkCompatible(dataset.getDescriptor(), tempDataset.getDescriptor()); return tempDataset; } catch (RuntimeException ex) { // swallow } } return repo.create(TEMP_NAMESPACE, jobDatasetName, copy(dataset.getDescriptor()), DatasetKeyOutputFormat.<E>getType(jobContext)); }