private void fetchCueballVersion(DomainVersion version, String fetchRoot) throws IOException { String cueballFileToFetch = Cueball.getName(version); LOG.info("Fetching from " + partitionRemoteFileOps + " for file " + cueballFileToFetch + " to " + fetchRoot); partitionRemoteFileOps.copyToLocalRoot(cueballFileToFetch, fetchRoot); }
@Override public Writer getWriter(DomainVersion domainVersion, PartitionRemoteFileOps partitionRemoteFileOps, int partitionNumber) throws IOException { Writer cueballWriter = cueballStorageEngine.getWriter(domainVersion, partitionRemoteFileOps, partitionNumber); return getWriter(domainVersion, partitionRemoteFileOps, partitionNumber, cueballWriter); }
@Override protected Integer detectCurrentVersionNumber() throws IOException { SortedSet<CueballFilePath> localBases = Cueball.getBases(localPartitionRoot); if (localBases.size() > 0) { return localBases.last().getVersion(); } else { return null; } }
@Override public Set<String> getFiles(DiskPartitionAssignment assignment, int domainVersionNumber, int partitionNumber) throws IOException { Set<String> result = new HashSet<String>(); result.add(getTargetDirectory(assignment, partitionNumber) + "/" + getName(domainVersionNumber, true)); return result; }
@Override public PartitionUpdater getUpdater(DiskPartitionAssignment assignment, int partitionNumber) throws IOException { String localDir = getTargetDirectory(assignment, partitionNumber); return new CueballPartitionUpdater(domain, getPartitionRemoteFileOps(RemoteLocation.PARTITION_SERVER, partitionNumber), new CueballMerger(), keyHashSize, valueSize, hashIndexBits, getCompressionCodec(), localDir); }
@Override public Writer getWriter(DomainVersion domainVersion, PartitionRemoteFileOps partitionRemoteFileOps, int partitionNumber) throws IOException { IncrementalDomainVersionProperties domainVersionProperties = getDomainVersionProperties(domainVersion); return new CueballWriter(partitionRemoteFileOps.getOutputStream(getName(domainVersion.getVersionNumber(), domainVersionProperties.isBase())), keyHashSize, hasher, valueSize, getCompressionCodec(), hashIndexBits ); }
final Cueball cueball = new Cueball(hashLength, h, valueLength, indexBits, "", "", null, codecClass, null, 0); final CueballWriter writer = (CueballWriter)cueball.getWriter(new MockDomainVersion(0, 0L), new LocalPartitionRemoteFileOps(outputPath, part.getKey()), part.getKey()); for (int i = 0; i < part.getValue().size(); i++) {
this.cueballStorageEngine = new Cueball(keyHashSize, hasher, cueballValueNumBytes,
@Override public Reader getReader(ReaderConfigurator configurator, int partitionNumber, DiskPartitionAssignment assignment) throws IOException { return new CueballReader( getTargetDirectory(assignment, partitionNumber), keyHashSize, hasher, valueSize, hashIndexBits, getCompressionCodec(), configurator.getCacheNumBytesCapacity(), (int)configurator.getCacheNumItemsCapacity()); }
@Override protected Set<DomainVersion> detectCachedDeltasCore() throws IOException { return detectCachedVersions(Cueball.getDeltas(localPartitionRootCache)); }
@Override public DiskPartitionAssignment getDataDirectoryPerPartition(DataDirectoriesConfigurator configurator, Collection<Integer> partitionNumbers) { return getDataDirectoryAssignments(configurator, partitionNumbers); }
@Override public ByteBuffer getComparableKey(ByteBuffer key) { return cueballStorageEngine.getComparableKey(key); }
@Override public Writer getCompactorWriter(DomainVersion domainVersion, PartitionRemoteFileOps fileOps, int partitionNumber) throws IOException { Writer cueballWriter = cueballStorageEngine.getCompactorWriter(domainVersion, fileOps, partitionNumber); return getWriter(domainVersion, fileOps, partitionNumber, cueballWriter); }
@Override public Writer getCompactorWriter(DomainVersion domainVersion, PartitionRemoteFileOps partitionRemoteFileOps, int partitionNumber) throws IOException { IncrementalDomainVersionProperties domainVersionProperties = getDomainVersionProperties(domainVersion); // Note: We use the identity hasher since keys coming in are already hashed keys return new CueballWriter(partitionRemoteFileOps.getOutputStream(getName(domainVersion.getVersionNumber(), domainVersionProperties.isBase())), keyHashSize, new IdentityHasher(), valueSize, getCompressionCodec(), hashIndexBits ); }
return new Cueball((Integer)options.get(KEY_HASH_SIZE_KEY), hasher, (Integer)options.get(VALUE_SIZE_KEY),
@Override protected Set<DomainVersion> detectCachedDeltasCore() throws IOException { return detectCachedVersions(Cueball.getDeltas(localPartitionRootCache), Curly.getDeltas(localPartitionRootCache)); }
@Override public DiskPartitionAssignment getDataDirectoryPerPartition(DataDirectoriesConfigurator configurator, Collection<Integer> partitionNumbers) { return Cueball.getDataDirectoryAssignments(configurator, partitionNumbers); }
@Override protected void fetchVersion(DomainVersion domainVersion, String fetchRoot) throws IOException { String fileToFetch = Cueball.getName(domainVersion); LOG.info("Fetching from " + partitionRemoteFileOps + " file " + fileToFetch + " to " + fetchRoot); partitionRemoteFileOps.copyToLocalRoot(fileToFetch, fetchRoot); }
public static void testPerformanceCueballWriter(String localTmpDir) throws IOException { // Fill in all indexable blocks long numRecords = getNumTotalRecords(); Writer writer = getCueball(localTmpDir).getWriter( new MockDomainVersion(0, 0L, new IncrementalDomainVersionProperties.Base()), new LocalPartitionRemoteFileOps(localTmpDir, 0), 0); HankTimer timer = new HankTimer(); for (long i = 0; i < numRecords; ++i) { writer.write(key(i, KEY_SIZE), value(i, VALUE_SIZE)); } writer.close(); double elapsedMs = timer.getDurationMs(); double elapsedSecs = elapsedMs / 1000.0; long totalBytes = numRecords * (KEY_HASH_SIZE + VALUE_SIZE); System.out.println("Test took " + elapsedMs + "ms, wrote " + numRecords + " records totalling " + FormatUtils.formatNumBytes(totalBytes)); System.out.println(String.format("Throughput: %.2f writes/sec", numRecords / elapsedSecs)); System.out.println("Throughput: " + FormatUtils.formatDataThroughput(totalBytes / elapsedSecs)); createZeroFile(localTmpDir + "/zero_file0", numRecords, IOStreamUtils.DEFAULT_BUFFER_SIZE); createZeroFile(localTmpDir + "/zero_file1", numRecords, 10 << 20); }
private static Cueball getCueball(String localTmpDir) { String root = localTmpDir + "/remote_domain_root"; return new Cueball( KEY_HASH_SIZE, new KeyHasher(HASH_INDEX_BITS), VALUE_SIZE, HASH_INDEX_BITS, root, root, new LocalPartitionRemoteFileOps.Factory(), NoCueballCompressionCodec.class, null, 0); }