@Override public DiskPartitionAssignment getDataDirectoryPerPartition(DataDirectoriesConfigurator configurator, Collection<Integer> partitionNumbers) { return new DiskPartitionAssignment(Collections.<Integer, String>emptyMap()); }
public static DiskPartitionAssignment getDataDirectoryAssignments(DataDirectoriesConfigurator configurator, Collection<Integer> partitionNumbers) { ArrayList<String> sortedDataDirectories = new ArrayList<String>(configurator.getDataDirectories()); Collections.sort(sortedDataDirectories); LinkedList<Integer> sortedPartitions = new LinkedList<>(partitionNumbers); Collections.sort(sortedPartitions); // TODO we can make this dynamic based on disk size, but not urgent double numPartitionsPerDisk = (double)partitionNumbers.size() / sortedDataDirectories.size(); Multimap<String, Integer> partitionsPerDisk = HashMultimap.create(); for (String dataDirectory : sortedDataDirectories) { int numToAssign = (int)Math.ceil(numPartitionsPerDisk * (partitionsPerDisk.keySet().size() + 1)) - partitionsPerDisk.values().size(); for (int i = 0; i < numToAssign && !sortedPartitions.isEmpty(); i++) { partitionsPerDisk.put(dataDirectory, sortedPartitions.pop()); } } Map<Integer, String> inverse = Maps.newHashMap(); for (Map.Entry<String, Integer> entry : partitionsPerDisk.entries()) { inverse.put(entry.getValue(), entry.getKey()); } return new DiskPartitionAssignment(inverse); }
@Override public void map(Text domainName, IntWritable partitionNumber, OutputCollector<KeyAndPartitionWritable, ValueWritable> outputCollector, Reporter reporter) throws IOException { LOG.info("Compacting Domain " + domainName.toString() + " Version " + domainVersionToCompact.getVersionNumber() + " Partition " + partitionNumber.get() + " in " + localTmpOutput.getAbsolutePath()); // Get compacting updater DiskPartitionAssignment assignment = new DiskPartitionAssignment(Collections.singletonMap( partitionNumber.get(), localTmpOutput.getAbsolutePath() )); Compactor compactor = storageEngine.getCompactor(assignment, partitionNumber.get()); if (compactor == null) { throw new RuntimeException("Failed to load compacting updater for domain " + domainName + " with storage engine: " + storageEngine); } // Close coordinator when possible compactor.closeCoordinatorOpportunistically(coordinator); // Perform compaction compactor.compact(domainVersionToCompact, new OutputCollectorWriter(reporter, partitionNumber, outputCollector)); }