@Override public String getOutputFormatClassName() { checkNotExternal(); PartitionKey outputKey = PartitionedFileSetArguments.getOutputPartitionKey(runtimeArguments, getPartitioning()); if (outputKey == null) { return "co.cask.cdap.internal.app.runtime.batch.dataset.partitioned.DynamicPartitioningOutputFormat"; } return files.getOutputFormatClassName(); }
@Override public String getOutputFormatClassName() { checkNotExternal(); PartitionKey outputKey = PartitionedFileSetArguments.getOutputPartitionKey(runtimeArguments, getPartitioning()); if (outputKey == null) { return "co.cask.cdap.internal.app.runtime.batch.dataset.partitioned.DynamicPartitioningOutputFormat"; } return files.getOutputFormatClassName(); }
protected static Map<String, String> updateArgumentsIfNeeded(Map<String, String> arguments, Partitioning partitioning) { if (FileSetArguments.getOutputPath(arguments) == null) { PartitionKey key = PartitionedFileSetArguments.getOutputPartitionKey(arguments, partitioning); // we need to copy the map, to avoid modifying the passed-in map arguments = Maps.newHashMap(arguments); if (key != null) { FileSetArguments.setOutputPath(arguments, PartitionedFileSetDataset.getOutputPath(key, partitioning)); } else if (PartitionedFileSetArguments.getDynamicPartitioner(arguments) != null) { // when using DynamicPartitioner, use the baseLocation of the fileSet as the output location FileSetArguments.setBaseOutputPath(arguments); } } return arguments; }
protected static Map<String, String> updateArgumentsIfNeeded(Map<String, String> arguments, Partitioning partitioning) { if (FileSetArguments.getOutputPath(arguments) == null) { PartitionKey key = PartitionedFileSetArguments.getOutputPartitionKey(arguments, partitioning); // we need to copy the map, to avoid modifying the passed-in map arguments = Maps.newHashMap(arguments); if (key != null) { FileSetArguments.setOutputPath(arguments, PartitionedFileSetDataset.getOutputPath(key, partitioning)); } else if (PartitionedFileSetArguments.getDynamicPartitioner(arguments) != null) { // when using DynamicPartitioner, use the baseLocation of the fileSet as the output location FileSetArguments.setBaseOutputPath(arguments); } } return arguments; }
@Override public Map<String, String> getOutputFormatConfiguration() { checkNotExternal(); // copy the output properties of the embedded file set to the output arguments Map<String, String> outputArgs = new HashMap<>(files.getOutputFormatConfiguration()); // we set the file set's output path in the definition's getDataset(), so there is no need to configure it again. // here we just want to validate that an output partition key or dynamic partitioner was specified in the arguments. PartitionKey outputKey = PartitionedFileSetArguments.getOutputPartitionKey(runtimeArguments, getPartitioning()); if (outputKey == null) { String dynamicPartitionerClassName = PartitionedFileSetArguments.getDynamicPartitioner(runtimeArguments); if (dynamicPartitionerClassName == null) { throw new DataSetException( "Either a Partition key or a DynamicPartitioner class must be given as a runtime argument."); } copyDynamicPartitionerArguments(runtimeArguments, outputArgs); outputArgs.put(Constants.Dataset.Partitioned.HCONF_ATTR_OUTPUT_FORMAT_CLASS_NAME, files.getOutputFormatClassName()); outputArgs.put(Constants.Dataset.Partitioned.HCONF_ATTR_OUTPUT_DATASET, getName()); } else { assertNotExists(outputKey, true); } return ImmutableMap.copyOf(outputArgs); }
@Override public Map<String, String> getOutputFormatConfiguration() { checkNotExternal(); // copy the output properties of the embedded file set to the output arguments Map<String, String> outputArgs = new HashMap<>(files.getOutputFormatConfiguration()); // we set the file set's output path in the definition's getDataset(), so there is no need to configure it again. // here we just want to validate that an output partition key or dynamic partitioner was specified in the arguments. PartitionKey outputKey = PartitionedFileSetArguments.getOutputPartitionKey(runtimeArguments, getPartitioning()); if (outputKey == null) { String dynamicPartitionerClassName = PartitionedFileSetArguments.getDynamicPartitioner(runtimeArguments); if (dynamicPartitionerClassName == null) { throw new DataSetException( "Either a Partition key or a DynamicPartitioner class must be given as a runtime argument."); } copyDynamicPartitionerArguments(runtimeArguments, outputArgs); outputArgs.put(Constants.Dataset.Partitioned.HCONF_ATTR_OUTPUT_FORMAT_CLASS_NAME, files.getOutputFormatClassName()); outputArgs.put(Constants.Dataset.Partitioned.HCONF_ATTR_OUTPUT_DATASET, getName()); } else { assertNotExists(outputKey, true); } return ImmutableMap.copyOf(outputArgs); }
PartitionKey partitionKey; try { partitionKey = PartitionedFileSetArguments.getOutputPartitionKey(properties, partitioning); } catch (Exception e) { responder.sendString(HttpResponseStatus.BAD_REQUEST, "invalid partition key: " + e.getMessage());
@Override public void onSuccess() throws DataSetException { String outputPath = FileSetArguments.getOutputPath(runtimeArguments); // If there is no output path, it is either using DynamicPartitioner or the job would have failed. // Either way, we can't do much here. if (outputPath == null) { return; } // its possible that there is no output key, if using the DynamicPartitioner, in which case // DynamicPartitioningOutputFormat is responsible for registering the partitions and the metadata PartitionKey outputKey = PartitionedFileSetArguments.getOutputPartitionKey(runtimeArguments, getPartitioning()); if (outputKey != null) { Map<String, String> metadata = PartitionedFileSetArguments.getOutputPartitionMetadata(runtimeArguments); addPartition(outputKey, outputPath, metadata, true, false); } // currently, FileSetDataset#onSuccess is a no-op, but call it, in case it does something in the future ((FileSetDataset) files).onSuccess(); }
PartitionKey partitionKey; try { partitionKey = PartitionedFileSetArguments.getOutputPartitionKey(properties, partitioning); } catch (Exception e) { responder.sendString(HttpResponseStatus.BAD_REQUEST, "invalid partition key: " + e.getMessage());
@Override public void onSuccess() throws DataSetException { String outputPath = FileSetArguments.getOutputPath(runtimeArguments); // If there is no output path, it is either using DynamicPartitioner or the job would have failed. // Either way, we can't do much here. if (outputPath == null) { return; } // its possible that there is no output key, if using the DynamicPartitioner, in which case // DynamicPartitioningOutputFormat is responsible for registering the partitions and the metadata PartitionKey outputKey = PartitionedFileSetArguments.getOutputPartitionKey(runtimeArguments, getPartitioning()); if (outputKey != null) { Map<String, String> metadata = PartitionedFileSetArguments.getOutputPartitionMetadata(runtimeArguments); addPartition(outputKey, outputPath, metadata, true, false); } // currently, FileSetDataset#onSuccess is a no-op, but call it, in case it does something in the future ((FileSetDataset) files).onSuccess(); }
@Test public void testSetGetOutputPartitionKey() throws Exception { Map<String, String> arguments = new HashMap<>(); PartitionKey key = PartitionKey.builder() .addIntField("i", 42) .addLongField("l", 17L) .addStringField("s", "x") .build(); PartitionedFileSetArguments.setOutputPartitionKey(arguments, key); Assert.assertEquals(key, PartitionedFileSetArguments.getOutputPartitionKey(arguments, PARTITIONING)); }