/** * @param limit limit to be applied while consuming partitions * @param predicate predicate to be applied while consuming partitions * @return a list of {@link Partition}s of the underlying {@link PartitionedFileSet} created since the last call * to this method. This excludes partitions created in in-progress transactions including the one in which the * call to this method is made. */ public List<PartitionDetail> consumePartitions(int limit, Predicate<PartitionDetail> predicate) { PartitionConsumerResult partitionConsumerResult = partitionedFileSet.consumePartitions(partitionConsumerState, limit, predicate); partitionConsumerState = partitionConsumerResult.getPartitionConsumerState(); return partitionConsumerResult.getPartitions(); } }
/** * @param limit limit to be applied while consuming partitions * @param predicate predicate to be applied while consuming partitions * @return a list of {@link Partition}s of the underlying {@link PartitionedFileSet} created since the last call * to this method. This excludes partitions created in in-progress transactions including the one in which the * call to this method is made. */ public List<PartitionDetail> consumePartitions(int limit, Predicate<PartitionDetail> predicate) { PartitionConsumerResult partitionConsumerResult = partitionedFileSet.consumePartitions(partitionConsumerState, limit, predicate); partitionConsumerState = partitionConsumerResult.getPartitionConsumerState(); return partitionConsumerResult.getPartitions(); } }
/** * Populates the ConsumerWorkingSet by fetching partitions from the given PartitionedFileSet. * * @param partitionedFileSet the PartitionedFileSet to fetch partitions from * @param configuration the ConsumerConfiguration which defines parameters for consuming */ public void populate(PartitionedFileSet partitionedFileSet, ConsumerConfiguration configuration) { int numToPopulate = configuration.getMaxWorkingSetSize() - partitions.size(); Predicate<PartitionDetail> predicate = configuration.getPartitionPredicate(); co.cask.cdap.api.dataset.lib.PartitionConsumerResult result = partitionedFileSet.consumePartitions(partitionConsumerState, numToPopulate, predicate); List<PartitionDetail> partitions = result.getPartitions(); for (PartitionDetail partition : partitions) { addPartition(partition.getPartitionKey()); } partitionConsumerState = result.getPartitionConsumerState(); }