@Test public void testSetGetOutputPartitionMetadata() throws Exception { Map<String, String> arguments = new HashMap<>(); Map<String, String> metadata = ImmutableMap.of("metakey1", "value1", "metaKey2", "value3"); PartitionedFileSetArguments.setOutputPartitionMetadata(arguments, metadata); Assert.assertEquals(metadata, PartitionedFileSetArguments.getOutputPartitionMetadata(arguments)); // test also with empty metadata arguments.clear(); PartitionedFileSetArguments.setOutputPartitionMetadata(arguments, Collections.<String, String>emptyMap()); Assert.assertEquals(Collections.<String, String>emptyMap(), PartitionedFileSetArguments.getOutputPartitionMetadata(arguments)); }
private void copyDynamicPartitionerArguments(Map<String, String> fromMap, Map<String, String> toMap) { String dynamicPartitionerClassName = PartitionedFileSetArguments.getDynamicPartitioner(fromMap); DynamicPartitioner.PartitionWriteOption partitionWriteOption = PartitionedFileSetArguments.getDynamicPartitionerWriteOption(fromMap); PartitionedFileSetArguments.setDynamicPartitioner(toMap, dynamicPartitionerClassName, partitionWriteOption); PartitionedFileSetArguments.setDynamicPartitionerConcurrency( toMap, PartitionedFileSetArguments.isDynamicPartitionerConcurrencyAllowed(fromMap)); // propagate output metadata into OutputFormatConfiguration so DynamicPartitionerOutputCommitter can assign // the metadata when it creates the partitions Map<String, String> metadata = PartitionedFileSetArguments.getOutputPartitionMetadata(fromMap); PartitionedFileSetArguments.setOutputPartitionMetadata(toMap, metadata); }
private void copyDynamicPartitionerArguments(Map<String, String> fromMap, Map<String, String> toMap) { String dynamicPartitionerClassName = PartitionedFileSetArguments.getDynamicPartitioner(fromMap); DynamicPartitioner.PartitionWriteOption partitionWriteOption = PartitionedFileSetArguments.getDynamicPartitionerWriteOption(fromMap); PartitionedFileSetArguments.setDynamicPartitioner(toMap, dynamicPartitionerClassName, partitionWriteOption); PartitionedFileSetArguments.setDynamicPartitionerConcurrency( toMap, PartitionedFileSetArguments.isDynamicPartitionerConcurrencyAllowed(fromMap)); // propagate output metadata into OutputFormatConfiguration so DynamicPartitionerOutputCommitter can assign // the metadata when it creates the partitions Map<String, String> metadata = PartitionedFileSetArguments.getOutputPartitionMetadata(fromMap); PartitionedFileSetArguments.setOutputPartitionMetadata(toMap, metadata); }