private void copyDynamicPartitionerArguments(Map<String, String> fromMap, Map<String, String> toMap) { String dynamicPartitionerClassName = PartitionedFileSetArguments.getDynamicPartitioner(fromMap); DynamicPartitioner.PartitionWriteOption partitionWriteOption = PartitionedFileSetArguments.getDynamicPartitionerWriteOption(fromMap); PartitionedFileSetArguments.setDynamicPartitioner(toMap, dynamicPartitionerClassName, partitionWriteOption); PartitionedFileSetArguments.setDynamicPartitionerConcurrency( toMap, PartitionedFileSetArguments.isDynamicPartitionerConcurrencyAllowed(fromMap)); // propagate output metadata into OutputFormatConfiguration so DynamicPartitionerOutputCommitter can assign // the metadata when it creates the partitions Map<String, String> metadata = PartitionedFileSetArguments.getOutputPartitionMetadata(fromMap); PartitionedFileSetArguments.setOutputPartitionMetadata(toMap, metadata); }
private void copyDynamicPartitionerArguments(Map<String, String> fromMap, Map<String, String> toMap) { String dynamicPartitionerClassName = PartitionedFileSetArguments.getDynamicPartitioner(fromMap); DynamicPartitioner.PartitionWriteOption partitionWriteOption = PartitionedFileSetArguments.getDynamicPartitionerWriteOption(fromMap); PartitionedFileSetArguments.setDynamicPartitioner(toMap, dynamicPartitionerClassName, partitionWriteOption); PartitionedFileSetArguments.setDynamicPartitionerConcurrency( toMap, PartitionedFileSetArguments.isDynamicPartitionerConcurrencyAllowed(fromMap)); // propagate output metadata into OutputFormatConfiguration so DynamicPartitionerOutputCommitter can assign // the metadata when it creates the partitions Map<String, String> metadata = PartitionedFileSetArguments.getOutputPartitionMetadata(fromMap); PartitionedFileSetArguments.setOutputPartitionMetadata(toMap, metadata); }
@Test public void testDynamicPartitionerWriterConcurrency() { Map<String, String> arguments = new HashMap<>(); // should not be able to get or set the concurrency setting, without a dynamic partitioner set on the arguments try { PartitionedFileSetArguments.isDynamicPartitionerConcurrencyAllowed(arguments); Assert.fail(); } catch (IllegalArgumentException expected) { } try { PartitionedFileSetArguments.setDynamicPartitionerConcurrency(arguments, false); Assert.fail(); } catch (IllegalArgumentException expected) { } // set a DynamicPartitioner PartitionedFileSetArguments.setDynamicPartitioner(arguments, TestDynamicPartitioner.class.getName()); // default value should be true Assert.assertTrue(PartitionedFileSetArguments.isDynamicPartitionerConcurrencyAllowed(arguments)); // try set+get PartitionedFileSetArguments.setDynamicPartitionerConcurrency(arguments, false); Assert.assertFalse(PartitionedFileSetArguments.isDynamicPartitionerConcurrencyAllowed(arguments)); PartitionedFileSetArguments.setDynamicPartitionerConcurrency(arguments, true); Assert.assertTrue(PartitionedFileSetArguments.isDynamicPartitionerConcurrencyAllowed(arguments)); }