@Override public int compareTo(KuduScanToken other) { if (!message.getTableName().equals(other.message.getTableName())) { throw new IllegalArgumentException("Scan tokens from different tables may not be compared"); } return tablet.getPartition().compareTo(other.getTablet().getPartition()); }
@Override public KuduInputSplit[] createInputSplits(int minNumSplits) throws IOException { startTableContext(); Preconditions.checkNotNull(tableContext,"tableContext should not be null"); List<KuduScanToken> tokens = tableContext.scanTokens(tableFilters, tableProjections, rowsLimit); KuduInputSplit[] splits = new KuduInputSplit[tokens.size()]; for (int i = 0; i < tokens.size(); i++) { KuduScanToken token = tokens.get(i); List<String> locations = new ArrayList<>(token.getTablet().getReplicas().size()); for (LocatedTablet.Replica replica : token.getTablet().getReplicas()) { locations.add(getLocation(replica.getRpcHost(), replica.getRpcPort())); } KuduInputSplit split = new KuduInputSplit( token.serialize(), i, locations.toArray(new String[locations.size()]) ); splits[i] = split; } if (splits.length < minNumSplits) { LOG.warn(" The minimum desired number of splits with your configured parallelism level " + "is {}. Current kudu splits = {}. {} instances will remain idle.", minNumSplits, splits.length, (minNumSplits - splits.length) ); } return splits; }
/** * Retrieves the partitions of a table. * * @param table the table * @return the partitions of the table */ private List<Partition> getTablePartitions(KuduTable table) { List<Partition> partitions = new ArrayList<>(); for (KuduScanToken token : client.newScanTokenBuilder(table).build()) { partitions.add(token.getTablet().getPartition()); } return partitions; }
LOG.debug(" Predicates scheduled for this query are " + predicateList.size()); for ( int i = 0; i < allPossibleScanTokens.size(); i++) { LOG.debug("A tablet scheduled for all operators scanning is " + allPossibleScanTokens.get(i).getTablet()); returnOfAssignments.add(aMetaForThisQuery); LOG.debug("Added query scan for this operator " + aMetaForThisQuery + " with scan tablet as " + allPossibleScanTokens.get(aPartofThePie.getOrdinal()).getTablet());
/** * Retrieves this table's range partitions. The range partitions will be returned * in sorted order by value, and will contain no duplicates. * * @param timeout the timeout of the operation * @return a list of the formatted range partitions */ @InterfaceAudience.Private @InterfaceStability.Unstable public List<Partition> getRangePartitions(long timeout) throws Exception { // TODO: This could be moved into the RangeSchemaPB returned from server // to avoid an extra call to get the range partitions. List<Partition> rangePartitions = new ArrayList<>(); List<KuduScanToken> scanTokens = new KuduScanToken.KuduScanTokenBuilder(client, this) .setTimeout(timeout) .build(); for (KuduScanToken token : scanTokens) { Partition partition = token.getTablet().getPartition(); // Filter duplicate range partitions by taking only the tablets whose hash // partitions are all 0s. if (!Iterators.all(partition.getHashBuckets().iterator(), Predicates.equalTo(0))) { continue; } rangePartitions.add(partition); } return rangePartitions; } }