@Override public <T extends Dataset> T getDataset(String namespace, String name, Map<String, String> arguments) throws DatasetInstantiationException { return delegate.getDataset(namespace, name, arguments); }
@Override public <T extends Dataset> T getDataset(String namespace, String name, Map<String, String> arguments) throws DatasetInstantiationException { return delegate.getDataset(namespace, name, arguments); }
@Override public <T extends Dataset> T getDataset(String name, Map<String, String> arguments) throws DatasetInstantiationException { return delegate.getDataset(name, arguments); }
@Override public <T extends Dataset> T getDataset(String name) throws DatasetInstantiationException { return delegate.getDataset(name); }
@Override public <T extends Dataset> T getDataset(String name, Map<String, String> arguments) throws DatasetInstantiationException { return delegate.getDataset(name, arguments); }
@Override public <T extends Dataset> T getDataset(String namespace, String name) throws DatasetInstantiationException { return delegate.getDataset(namespace, name); }
@Override public <T extends Dataset> T getDataset(String name) throws DatasetInstantiationException { return delegate.getDataset(name); }
@Override public <T extends Dataset> T getDataset(String namespace, String name) throws DatasetInstantiationException { return delegate.getDataset(namespace, name); }
/** * Returns a {@link CloseableBatchWritable} that writes data to the given dataset. */ <K, V> CloseableBatchWritable<K, V> getBatchWritable(String namespace, String datasetName, Map<String, String> datasetArgs) { Dataset dataset = getDataset(namespace, datasetName, datasetArgs, AccessType.WRITE); // Must be BatchWritable. Preconditions.checkArgument(dataset instanceof BatchWritable, "Dataset '%s:%s' is not a BatchWritable.", namespace, datasetName); @SuppressWarnings("unchecked") final BatchWritable<K, V> delegate = (BatchWritable<K, V>) dataset; return new CloseableBatchWritable<K, V>() { @Override public void write(K k, V v) { delegate.write(k, v); } @Override public void close() throws IOException { try { flushOperations(); } catch (Exception e) { Throwables.propagateIfInstanceOf(e, IOException.class); throw new IOException(e); } } }; } }
/** * Returns a {@link CloseableBatchWritable} that writes data to the given dataset. */ <K, V> CloseableBatchWritable<K, V> getBatchWritable(String namespace, String datasetName, Map<String, String> datasetArgs) { Dataset dataset = getDataset(namespace, datasetName, datasetArgs, AccessType.WRITE); // Must be BatchWritable. Preconditions.checkArgument(dataset instanceof BatchWritable, "Dataset '%s:%s' is not a BatchWritable.", namespace, datasetName); @SuppressWarnings("unchecked") final BatchWritable<K, V> delegate = (BatchWritable<K, V>) dataset; return new CloseableBatchWritable<K, V>() { @Override public void write(K k, V v) { delegate.write(k, v); } @Override public void close() throws IOException { try { flushOperations(); } catch (Exception e) { Throwables.propagateIfInstanceOf(e, IOException.class); throw new IOException(e); } } }; } }
Dataset dataset; if (datasetNamespace == null) { dataset = getDataset(datasetName, datasetArgs, AccessType.READ); } else { dataset = getDataset(datasetNamespace, datasetName, datasetArgs, AccessType.READ);
Dataset dataset; if (datasetNamespace == null) { dataset = getDataset(datasetName, datasetArgs, AccessType.READ); } else { dataset = getDataset(datasetNamespace, datasetName, datasetArgs, AccessType.READ);
DynamicPartitionerWriterWrapper(TaskAttemptContext job) { this.job = job; Configuration configuration = job.getConfiguration(); Class<? extends DynamicPartitioner> partitionerClass = configuration .getClass(PartitionedFileSetArguments.DYNAMIC_PARTITIONER_CLASS_NAME, null, DynamicPartitioner.class); this.dynamicPartitioner = new InstantiatorFactory(false).get(TypeToken.of(partitionerClass)).create(); this.partitionWriteOption = DynamicPartitioner.PartitionWriteOption.valueOf( configuration.get(PartitionedFileSetArguments.DYNAMIC_PARTITIONER_WRITE_OPTION)); MapReduceClassLoader classLoader = MapReduceClassLoader.getFromConfiguration(configuration); this.taskContext = classLoader.getTaskContextProvider().get(job); // name the output file 'part-<RunId>-m-00000' instead of 'part-m-00000' String outputName = DynamicPartitioningOutputFormat.getOutputName(job); if (partitionWriteOption == DynamicPartitioner.PartitionWriteOption.CREATE_OR_APPEND) { outputName = outputName + "-" + taskContext.getProgramRunId().getRun(); } this.outputName = outputName; String outputDatasetName = configuration.get(Constants.Dataset.Partitioned.HCONF_ATTR_OUTPUT_DATASET); this.outputDataset = taskContext.getDataset(outputDatasetName); this.partitioning = outputDataset.getPartitioning(); this.dynamicPartitioner.initialize(taskContext); this.fileOutputFormatName = job.getConfiguration() .getClass(Constants.Dataset.Partitioned.HCONF_ATTR_OUTPUT_FORMAT_CLASS_NAME, null, FileOutputFormat.class) .getName(); }
DynamicPartitionerWriterWrapper(TaskAttemptContext job) { this.job = job; Configuration configuration = job.getConfiguration(); Class<? extends DynamicPartitioner> partitionerClass = configuration .getClass(PartitionedFileSetArguments.DYNAMIC_PARTITIONER_CLASS_NAME, null, DynamicPartitioner.class); this.dynamicPartitioner = new InstantiatorFactory(false).get(TypeToken.of(partitionerClass)).create(); this.partitionWriteOption = DynamicPartitioner.PartitionWriteOption.valueOf( configuration.get(PartitionedFileSetArguments.DYNAMIC_PARTITIONER_WRITE_OPTION)); MapReduceClassLoader classLoader = MapReduceClassLoader.getFromConfiguration(configuration); this.taskContext = classLoader.getTaskContextProvider().get(job); // name the output file 'part-<RunId>-m-00000' instead of 'part-m-00000' String outputName = DynamicPartitioningOutputFormat.getOutputName(job); if (partitionWriteOption == DynamicPartitioner.PartitionWriteOption.CREATE_OR_APPEND) { outputName = outputName + "-" + taskContext.getProgramRunId().getRun(); } this.outputName = outputName; String outputDatasetName = configuration.get(Constants.Dataset.Partitioned.HCONF_ATTR_OUTPUT_DATASET); this.outputDataset = taskContext.getDataset(outputDatasetName); this.partitioning = outputDataset.getPartitioning(); this.dynamicPartitioner.initialize(taskContext); this.fileOutputFormatName = job.getConfiguration() .getClass(Constants.Dataset.Partitioned.HCONF_ATTR_OUTPUT_FORMAT_CLASS_NAME, null, FileOutputFormat.class) .getName(); }
outputDataset = taskContext.getDataset(outputDatasetName); DynamicPartitioner.PartitionWriteOption partitionWriteOption = DynamicPartitioner.PartitionWriteOption.valueOf( configuration.get(PartitionedFileSetArguments.DYNAMIC_PARTITIONER_WRITE_OPTION));
outputDataset = taskContext.getDataset(outputDatasetName); DynamicPartitioner.PartitionWriteOption partitionWriteOption = DynamicPartitioner.PartitionWriteOption.valueOf( configuration.get(PartitionedFileSetArguments.DYNAMIC_PARTITIONER_WRITE_OPTION));