/** * Determines whether concurrent writers are allowed in the case of DynamicPartitioner. * * @param arguments arguments the runtime arguments to get the class from * @return whether concurrent writers are allowed in the case of DynamicPartitioner. */ public static boolean isDynamicPartitionerConcurrencyAllowed(Map<String, String> arguments) { ensureDynamicPartitionerConfigured(arguments); String concurrencyAllowed = arguments.get(DYNAMIC_PARTITIONER_ALLOW_CONCURRENCY); if (concurrencyAllowed == null) { // default to true return true; } return Boolean.valueOf(concurrencyAllowed); }
/** * Sets whether concurrent writers are allowed when using DynamicPartitioner. * When concurrency is not allowed, the writer for a particular {@link PartitionKey} is closed when a new partition * key is encountered. If any partition key is encountered for which a corresponding writer has already been closed, * the job's Outputformat will throw an IllegalArgumentException. * Therefore, this should only be used if the data encountered in a task is grouped by the partition key returned * by the DynamicPartitioner. * * @param arguments arguments the runtime arguments to get the class from * @param allowConcurrency whether to allow multiple partition writers concurrently */ public static void setDynamicPartitionerConcurrency(Map<String, String> arguments, boolean allowConcurrency) { ensureDynamicPartitionerConfigured(arguments); arguments.put(DYNAMIC_PARTITIONER_ALLOW_CONCURRENCY, Boolean.toString(allowConcurrency)); }
/** * Return the DynamicPartitioner.PartitionWriteOption class that was previously assigned onto runtime arguments. * * @param args the runtime arguments to get the class from * @return name of the DynamicPartitioner.PartitionWriteOption class */ public static DynamicPartitioner.PartitionWriteOption getDynamicPartitionerWriteOption(Map<String, String> args) { ensureDynamicPartitionerConfigured(args); // if DynamicPartitioner is configured, we always also configure the following option, so we can safely call // Enum#valueOf(String) return DynamicPartitioner.PartitionWriteOption.valueOf(args.get(DYNAMIC_PARTITIONER_WRITE_OPTION)); }