@Test public void testSetGetOutputPartitionMetadata() throws Exception { Map<String, String> arguments = new HashMap<>(); Map<String, String> metadata = ImmutableMap.of("metakey1", "value1", "metaKey2", "value3"); PartitionedFileSetArguments.setOutputPartitionMetadata(arguments, metadata); Assert.assertEquals(metadata, PartitionedFileSetArguments.getOutputPartitionMetadata(arguments)); // test also with empty metadata arguments.clear(); PartitionedFileSetArguments.setOutputPartitionMetadata(arguments, Collections.<String, String>emptyMap()); Assert.assertEquals(Collections.<String, String>emptyMap(), PartitionedFileSetArguments.getOutputPartitionMetadata(arguments)); }
private void copyDynamicPartitionerArguments(Map<String, String> fromMap, Map<String, String> toMap) { String dynamicPartitionerClassName = PartitionedFileSetArguments.getDynamicPartitioner(fromMap); DynamicPartitioner.PartitionWriteOption partitionWriteOption = PartitionedFileSetArguments.getDynamicPartitionerWriteOption(fromMap); PartitionedFileSetArguments.setDynamicPartitioner(toMap, dynamicPartitionerClassName, partitionWriteOption); PartitionedFileSetArguments.setDynamicPartitionerConcurrency( toMap, PartitionedFileSetArguments.isDynamicPartitionerConcurrencyAllowed(fromMap)); // propagate output metadata into OutputFormatConfiguration so DynamicPartitionerOutputCommitter can assign // the metadata when it creates the partitions Map<String, String> metadata = PartitionedFileSetArguments.getOutputPartitionMetadata(fromMap); PartitionedFileSetArguments.setOutputPartitionMetadata(toMap, metadata); }
private void copyDynamicPartitionerArguments(Map<String, String> fromMap, Map<String, String> toMap) { String dynamicPartitionerClassName = PartitionedFileSetArguments.getDynamicPartitioner(fromMap); DynamicPartitioner.PartitionWriteOption partitionWriteOption = PartitionedFileSetArguments.getDynamicPartitionerWriteOption(fromMap); PartitionedFileSetArguments.setDynamicPartitioner(toMap, dynamicPartitionerClassName, partitionWriteOption); PartitionedFileSetArguments.setDynamicPartitionerConcurrency( toMap, PartitionedFileSetArguments.isDynamicPartitionerConcurrencyAllowed(fromMap)); // propagate output metadata into OutputFormatConfiguration so DynamicPartitionerOutputCommitter can assign // the metadata when it creates the partitions Map<String, String> metadata = PartitionedFileSetArguments.getOutputPartitionMetadata(fromMap); PartitionedFileSetArguments.setOutputPartitionMetadata(toMap, metadata); }
@Override public void onSuccess() throws DataSetException { String outputPath = FileSetArguments.getOutputPath(runtimeArguments); // If there is no output path, it is either using DynamicPartitioner or the job would have failed. // Either way, we can't do much here. if (outputPath == null) { return; } // its possible that there is no output key, if using the DynamicPartitioner, in which case // DynamicPartitioningOutputFormat is responsible for registering the partitions and the metadata PartitionKey outputKey = PartitionedFileSetArguments.getOutputPartitionKey(runtimeArguments, getPartitioning()); if (outputKey != null) { Map<String, String> metadata = PartitionedFileSetArguments.getOutputPartitionMetadata(runtimeArguments); addPartition(outputKey, outputPath, metadata, true, false); } // currently, FileSetDataset#onSuccess is a no-op, but call it, in case it does something in the future ((FileSetDataset) files).onSuccess(); }
@Override public void onSuccess() throws DataSetException { String outputPath = FileSetArguments.getOutputPath(runtimeArguments); // If there is no output path, it is either using DynamicPartitioner or the job would have failed. // Either way, we can't do much here. if (outputPath == null) { return; } // its possible that there is no output key, if using the DynamicPartitioner, in which case // DynamicPartitioningOutputFormat is responsible for registering the partitions and the metadata PartitionKey outputKey = PartitionedFileSetArguments.getOutputPartitionKey(runtimeArguments, getPartitioning()); if (outputKey != null) { Map<String, String> metadata = PartitionedFileSetArguments.getOutputPartitionMetadata(runtimeArguments); addPartition(outputKey, outputPath, metadata, true, false); } // currently, FileSetDataset#onSuccess is a no-op, but call it, in case it does something in the future ((FileSetDataset) files).onSuccess(); }