public static SortedSet<CueballFilePath> getBases(String... dirs) throws IOException { SortedSet<CueballFilePath> result = new TreeSet<CueballFilePath>(); Set<String> paths = FsUtils.getMatchingPaths(BASE_REGEX, dirs); for (String path : paths) { result.add(new CueballFilePath(path)); } return result; }
public CueballStreamBufferMergeSort(CueballFilePath cueballBase, List<CueballFilePath> cueballDeltas, int keyHashSize, int valueSize, int hashIndexBits, CueballCompressionCodec compressionCodec, ValueTransformer transformer) throws IOException { this.keyHashSize = keyHashSize; this.valueSize = valueSize; this.transformer = transformer; cueballStreamBuffers = new CueballStreamBuffer[cueballDeltas.size() + 1]; // Open the base CueballStreamBuffer cueballBaseStreamBuffer = new CueballStreamBuffer(cueballBase.getPath(), 0, keyHashSize, valueSize, hashIndexBits, compressionCodec); cueballStreamBuffers[0] = cueballBaseStreamBuffer; // Open all the deltas int i = 1; for (CueballFilePath delta : cueballDeltas) { CueballStreamBuffer cueballStreamBuffer = new CueballStreamBuffer(delta.getPath(), i, keyHashSize, valueSize, hashIndexBits, compressionCodec); cueballStreamBuffers[i++] = cueballStreamBuffer; } }
@Override protected Integer detectCurrentVersionNumber() throws IOException { SortedSet<CueballFilePath> localBases = Cueball.getBases(localPartitionRoot); if (localBases.size() > 0) { return localBases.last().getVersion(); } else { return null; } }
this.fullRecordSize = valueSize + keyHashSize; this.prefixer = new HashPrefixCalculator(hashIndexBits); this.versionNumber = latestBase.getVersion(); channel = new FileInputStream(latestBase.getPath()).getChannel(); Footer footer = new Footer(channel, hashIndexBits); hashIndex = footer.getHashIndex();
public static SortedSet<CueballFilePath> getDeltas(String... dirs) throws IOException { SortedSet<CueballFilePath> result = new TreeSet<CueballFilePath>(); Set<String> paths = FsUtils.getMatchingPaths(DELTA_REGEX, dirs); for (String path : paths) { result.add(new CueballFilePath(path)); } return result; }
private Set<DomainVersion> detectCachedVersions(SortedSet<CueballFilePath> cachedFiles) throws IOException { Set<DomainVersion> cachedVersions = new HashSet<DomainVersion>(); for (CueballFilePath file : cachedFiles) { DomainVersion version = domain.getVersion(file.getVersion()); if (version != null) { cachedVersions.add(version); } } return cachedVersions; }
public static CueballFilePath getCueballFilePathForVersion(DomainVersion version, DomainVersion currentVersion, String localPartitionRoot, String localPartitionRootCache, boolean isBase) { if (currentVersion != null && currentVersion.equals(version)) { // If version is current version, data is in root return new CueballFilePath(localPartitionRoot + "/" + Cueball.getName(version.getVersionNumber(), isBase)); } else { // Otherwise, version must be in cache return new CueballFilePath(localPartitionRootCache + "/" + Cueball.getName(version.getVersionNumber(), isBase)); } }
@Override protected Integer detectCurrentVersionNumber() throws IOException { SortedSet<CueballFilePath> localCueballBases = Cueball.getBases(localPartitionRoot); SortedSet<CurlyFilePath> localCurlyBases = Curly.getBases(localPartitionRoot); if (localCueballBases.size() > 0 && localCurlyBases.size() > 0) { if (localCueballBases.last().getVersion() == localCurlyBases.last().getVersion()) { return localCurlyBases.last().getVersion(); } else { return null; } } else { return null; } }
private static void doTestPerformanceCueballMerger(String localTmpDir, DomainVersion baseVersion, DomainVersion deltaVersion, DomainVersion newBaseVersion) throws IOException { HankTimer timer = new HankTimer(); new CueballMerger().merge( new CueballFilePath(localTmpDir + "/0/" + Cueball.getName(baseVersion)), Collections.singletonList(new CueballFilePath(localTmpDir + "/0/" + Cueball.getName(deltaVersion))), localTmpDir + "/" + Cueball.getName(newBaseVersion), KEY_HASH_SIZE, VALUE_SIZE, null, HASH_INDEX_BITS, new NoCueballCompressionCodec()); double elapsedS = timer.getDurationMs() / 1000.0; System.out.println("Merge done in " + elapsedS + " seconds"); }
private Set<DomainVersion> detectCachedVersions(SortedSet<CueballFilePath> cueballCachedFiles, SortedSet<CurlyFilePath> curlyCachedFiles) throws IOException { // Record in a set all cached Cueball versions HashSet<Integer> cachedCueballVersions = new HashSet<Integer>(); for (CueballFilePath cueballCachedFile : cueballCachedFiles) { cachedCueballVersions.add(cueballCachedFile.getVersion()); } // Compute cached Curly versions Set<DomainVersion> cachedVersions = new HashSet<DomainVersion>(); for (CurlyFilePath curlyCachedFile : curlyCachedFiles) { // Check that the corresponding Cueball version is also cached if (cachedCueballVersions.contains(curlyCachedFile.getVersion())) { DomainVersion version = domain.getVersion(curlyCachedFile.getVersion()); if (version != null) { cachedVersions.add(version); } } } return cachedVersions; }