@Override public PartitionRemoteFileOps getPartitionRemoteFileOps(String remoteRoot, int partitionNumber) throws IOException { return new LocalPartitionRemoteFileOps(remoteRoot, partitionNumber); } }
for (Map.Entry<Integer, List<byte[]>> part : partitionedKeys.entrySet()) { final CurlyWriter writer = (CurlyWriter)curly.getWriter(new MockDomainVersion(0, 0L), new LocalPartitionRemoteFileOps(outputPath, part.getKey()), part.getKey()); for (int i = 0; i < part.getValue().size(); i++) { final byte[] keyHash = part.getValue().get(i);
new LocalPartitionRemoteFileOps(outputPath, part.getKey()), part.getKey()); for (int i = 0; i < part.getValue().size(); i++) { r.nextBytes(valueBytes);
public static void testPerformanceCueballWriter(String localTmpDir) throws IOException { // Fill in all indexable blocks long numRecords = getNumTotalRecords(); Writer writer = getCueball(localTmpDir).getWriter( new MockDomainVersion(0, 0L, new IncrementalDomainVersionProperties.Base()), new LocalPartitionRemoteFileOps(localTmpDir, 0), 0); HankTimer timer = new HankTimer(); for (long i = 0; i < numRecords; ++i) { writer.write(key(i, KEY_SIZE), value(i, VALUE_SIZE)); } writer.close(); double elapsedMs = timer.getDurationMs(); double elapsedSecs = elapsedMs / 1000.0; long totalBytes = numRecords * (KEY_HASH_SIZE + VALUE_SIZE); System.out.println("Test took " + elapsedMs + "ms, wrote " + numRecords + " records totalling " + FormatUtils.formatNumBytes(totalBytes)); System.out.println(String.format("Throughput: %.2f writes/sec", numRecords / elapsedSecs)); System.out.println("Throughput: " + FormatUtils.formatDataThroughput(totalBytes / elapsedSecs)); createZeroFile(localTmpDir + "/zero_file0", numRecords, IOStreamUtils.DEFAULT_BUFFER_SIZE); createZeroFile(localTmpDir + "/zero_file1", numRecords, 10 << 20); }
private static void prepareTestPerformanceCueballMerger(String localTmpDir, DomainVersion baseVersion, DomainVersion deltaVersion) throws IOException { PartitionRemoteFileOps partitionRemoteFileOps = new LocalPartitionRemoteFileOps(localTmpDir, 0); long numRecords = getNumTotalRecords(); int deltaFrequency = 1000000; // num records in delta = numRecords / deltaFrequency Cueball cueball = getCueball(localTmpDir); // Create base Writer baseWriter = cueball.getWriter( baseVersion, partitionRemoteFileOps, 0); for (long i = 0; i < numRecords; ++i) { if (i % deltaFrequency != 0) { baseWriter.write(key(i, KEY_SIZE), value(i, VALUE_SIZE)); } } baseWriter.close(); // Create delta Writer deltaWriter = cueball.getWriter( deltaVersion, partitionRemoteFileOps, 0); for (long i = 0; i < numRecords; ++i) { if (i % deltaFrequency == 0) { deltaWriter.write(key(i, KEY_SIZE), value(i, VALUE_SIZE)); } } deltaWriter.close(); }