PartitioningContext context) if (context.getParallelPartitionCount() == 0) { return partitions; if (context.getParallelPartitionCount() == partitions.size()) { LOG.debug("no change is partition count: " + partitions.size()); return partitions; LOG.debug("block writer parallel partition count {}", context.getParallelPartitionCount()); int morePartitionsToCreate = context.getParallelPartitionCount() - partitions.size(); if (morePartitionsToCreate < 0) {
List<InputPort<?>> inputPortList = context.getInputPorts(); if (inputPortList != null && !inputPortList.isEmpty()) { DefaultPartition.assignPartitionKeys(newPartitions, inputPortList.iterator().next()); if (context.getParallelPartitionCount() != 0) { newPartitions = repartitionParallel(partitions, context); } else if (partition.getPartitionKeys().isEmpty()) {
newPartitions.addAll(partitions); int morePartitionsToCreate = context.getParallelPartitionCount() - newPartitions.size(); if (morePartitionsToCreate < 0) {
/*** * Returns the number of partitions. The configuration is used as the source of truth for * number of partitions. The config is then overridden if the code in the client driver code explicitly sets * the number of partitions. * @param context The context as provided by the launcher * @return The number of partitions that are planned for this input operator. At a minimum it is 1. */ public int getNumberOfPartitions(PartitioningContext context) { int proposedPartitionCount = context.getParallelPartitionCount(); if ( prototypeKuduInputOperator.getNumberOfPartitions() != -1 ) { // -1 is the default // There is a manual override of partitions from code. Use this proposedPartitionCount = prototypeKuduInputOperator.getNumberOfPartitions(); LOG.info(" Set the partition count based on the code as opposed to configuration "); } if ( proposedPartitionCount <= 0) { // Parallel partitions not enabled. But the design is to use one to many mapping. Hence defaulting to one LOG.info(" Defaulting to one partition as parallel partitioning is not enabled"); proposedPartitionCount = 1; } LOG.info(" Planning to use " + proposedPartitionCount + " partitions"); return proposedPartitionCount; }
public static int getRequiredPartitionCount(PartitioningContext context, int count) { return context.getParallelPartitionCount() == 0 ? count : context.getParallelPartitionCount(); }