@Override protected void fetchVersion(DomainVersion domainVersion, String fetchRoot) throws IOException { String fileToFetch = Cueball.getName(domainVersion); LOG.info("Fetching from " + partitionRemoteFileOps + " file " + fileToFetch + " to " + fetchRoot); partitionRemoteFileOps.copyToLocalRoot(fileToFetch, fetchRoot); }
@Override public Set<String> getFiles(DiskPartitionAssignment assignment, int domainVersionNumber, int partitionNumber) throws IOException { Set<String> result = new HashSet<String>(); result.add(getTargetDirectory(assignment, partitionNumber) + "/" + getName(domainVersionNumber, true)); return result; }
private void fetchCueballVersion(DomainVersion version, String fetchRoot) throws IOException { String cueballFileToFetch = Cueball.getName(version); LOG.info("Fetching from " + partitionRemoteFileOps + " for file " + cueballFileToFetch + " to " + fetchRoot); partitionRemoteFileOps.copyToLocalRoot(cueballFileToFetch, fetchRoot); }
@Override public List<String> getRemotePartitionFilePaths(IncrementalUpdatePlan updatePlan, PartitionRemoteFileOps partitionRemoteFileOps) throws IOException { List<String> result = new ArrayList<String>(); for (DomainVersion domainVersion : updatePlan.getAllVersions()) { result.add(partitionRemoteFileOps.getRemoteAbsolutePath(Cueball.getName(domainVersion))); } return result; } }
public static CueballFilePath getCueballFilePathForVersion(DomainVersion version, DomainVersion currentVersion, String localPartitionRoot, String localPartitionRootCache, boolean isBase) { if (currentVersion != null && currentVersion.equals(version)) { // If version is current version, data is in root return new CueballFilePath(localPartitionRoot + "/" + Cueball.getName(version.getVersionNumber(), isBase)); } else { // Otherwise, version must be in cache return new CueballFilePath(localPartitionRootCache + "/" + Cueball.getName(version.getVersionNumber(), isBase)); } }
public static boolean isEmptyVersion(PartitionRemoteFileOps partitionRemoteFileOps, DomainVersion domainVersion) throws IOException { return !partitionRemoteFileOps.exists(Cueball.getName(domainVersion.getVersionNumber(), true)) && !partitionRemoteFileOps.exists(Cueball.getName(domainVersion.getVersionNumber(), false)); }
@Override public void deleteVersion(int versionNumber) throws IOException { for (int partition = 0; partition < domain.getNumParts(); ++partition) { PartitionRemoteFileOps fileOps = fileOpsFactory.getPartitionRemoteFileOps(remoteDomainRoot, partition); fileOps.attemptDelete(Cueball.getName(versionNumber, true)); fileOps.attemptDelete(Cueball.getName(versionNumber, false)); } } }
public static String getName(DomainVersion domainVersion) throws IOException { return getName(domainVersion.getVersionNumber(), IncrementalDomainVersionProperties.isBase(domainVersion)); }
@Override public List<String> getRemotePartitionFilePaths(IncrementalUpdatePlan updatePlan, PartitionRemoteFileOps partitionRemoteFileOps) throws IOException { List<String> result = new ArrayList<String>(); for (DomainVersion domainVersion : updatePlan.getAllVersions()) { result.add(partitionRemoteFileOps.getRemoteAbsolutePath(Curly.getName(domainVersion))); result.add(partitionRemoteFileOps.getRemoteAbsolutePath(Cueball.getName(domainVersion))); } return result; } }
private static void doTestPerformanceCueballMerger(String localTmpDir, DomainVersion baseVersion, DomainVersion deltaVersion, DomainVersion newBaseVersion) throws IOException { HankTimer timer = new HankTimer(); new CueballMerger().merge( new CueballFilePath(localTmpDir + "/0/" + Cueball.getName(baseVersion)), Collections.singletonList(new CueballFilePath(localTmpDir + "/0/" + Cueball.getName(deltaVersion))), localTmpDir + "/" + Cueball.getName(newBaseVersion), KEY_HASH_SIZE, VALUE_SIZE, null, HASH_INDEX_BITS, new NoCueballCompressionCodec()); double elapsedS = timer.getDurationMs() / 1000.0; System.out.println("Merge done in " + elapsedS + " seconds"); }
@Override public void deleteVersion(int versionNumber) throws IOException { for (int partition = 0; partition < domain.getNumParts(); ++partition) { PartitionRemoteFileOps fileOps = fileOpsFactory.getPartitionRemoteFileOps(remoteDomainRoot, partition); fileOps.attemptDelete(Cueball.getName(versionNumber, true)); fileOps.attemptDelete(Cueball.getName(versionNumber, false)); fileOps.attemptDelete(Curly.getName(versionNumber, true)); fileOps.attemptDelete(Curly.getName(versionNumber, false)); } } }
@Override public Writer getWriter(DomainVersion domainVersion, PartitionRemoteFileOps partitionRemoteFileOps, int partitionNumber) throws IOException { IncrementalDomainVersionProperties domainVersionProperties = getDomainVersionProperties(domainVersion); return new CueballWriter(partitionRemoteFileOps.getOutputStream(getName(domainVersion.getVersionNumber(), domainVersionProperties.isBase())), keyHashSize, hasher, valueSize, getCompressionCodec(), hashIndexBits ); }
+ Cueball.getName(updatingToVersion.getVersionNumber(), true);
@Override public Writer getCompactorWriter(DomainVersion domainVersion, PartitionRemoteFileOps partitionRemoteFileOps, int partitionNumber) throws IOException { IncrementalDomainVersionProperties domainVersionProperties = getDomainVersionProperties(domainVersion); // Note: We use the identity hasher since keys coming in are already hashed keys return new CueballWriter(partitionRemoteFileOps.getOutputStream(getName(domainVersion.getVersionNumber(), domainVersionProperties.isBase())), keyHashSize, new IdentityHasher(), valueSize, getCompressionCodec(), hashIndexBits ); }