@Override public void close() throws IOException { if (blockCompressionCodec != null) { flushCompressedBlock(); } recordFileStream.flush(); recordFileStream.close(); keyfileWriter.close(); if (hashedValueToEncodedRecordOffsetCache != null) { hashedValueToEncodedRecordOffsetCache.clear(); } }
@Override public void run(Coordinator coordinator) throws IOException { DomainVersion domainVersion = DomainBuilderProperties.getDomainVersion(coordinator, domainName, domainVersionNumber); domainVersion.addPartitionProperties(writerPartition, writer.getNumBytesWritten(), writer.getNumRecordsWritten()); } });
private static void prepareTestPerformanceCueballMerger(String localTmpDir, DomainVersion baseVersion, DomainVersion deltaVersion) throws IOException { PartitionRemoteFileOps partitionRemoteFileOps = new LocalPartitionRemoteFileOps(localTmpDir, 0); long numRecords = getNumTotalRecords(); int deltaFrequency = 1000000; // num records in delta = numRecords / deltaFrequency Cueball cueball = getCueball(localTmpDir); // Create base Writer baseWriter = cueball.getWriter( baseVersion, partitionRemoteFileOps, 0); for (long i = 0; i < numRecords; ++i) { if (i % deltaFrequency != 0) { baseWriter.write(key(i, KEY_SIZE), value(i, VALUE_SIZE)); } } baseWriter.close(); // Create delta Writer deltaWriter = cueball.getWriter( deltaVersion, partitionRemoteFileOps, 0); for (long i = 0; i < numRecords; ++i) { if (i % deltaFrequency == 0) { deltaWriter.write(key(i, KEY_SIZE), value(i, VALUE_SIZE)); } } deltaWriter.close(); }
keyfileWriter.write(key, cachedValueRecordEncodedOffset); numFoldedValues += 1; numFoldedBytesApproximate += value.remaining(); keyfileWriter.write(key, valueOffsetBuffer); EncodingHelper.encodeLittleEndianFixedWidthLong(offsetInDecompressedBlock, valueOffsetBuffer.array(), offsetNumBytes, offsetInBlockNumBytes); keyfileWriter.write(key, valueOffsetBuffer);
@Override public long getNumRecordsWritten() { return keyfileWriter.getNumRecordsWritten(); }
@Override public long getNumBytesWritten() { return keyfileWriter.getNumBytesWritten() + currentRecordOffset; }
public static void testPerformanceCueballWriter(String localTmpDir) throws IOException { // Fill in all indexable blocks long numRecords = getNumTotalRecords(); Writer writer = getCueball(localTmpDir).getWriter( new MockDomainVersion(0, 0L, new IncrementalDomainVersionProperties.Base()), new LocalPartitionRemoteFileOps(localTmpDir, 0), 0); HankTimer timer = new HankTimer(); for (long i = 0; i < numRecords; ++i) { writer.write(key(i, KEY_SIZE), value(i, VALUE_SIZE)); } writer.close(); double elapsedMs = timer.getDurationMs(); double elapsedSecs = elapsedMs / 1000.0; long totalBytes = numRecords * (KEY_HASH_SIZE + VALUE_SIZE); System.out.println("Test took " + elapsedMs + "ms, wrote " + numRecords + " records totalling " + FormatUtils.formatNumBytes(totalBytes)); System.out.println(String.format("Throughput: %.2f writes/sec", numRecords / elapsedSecs)); System.out.println("Throughput: " + FormatUtils.formatDataThroughput(totalBytes / elapsedSecs)); createZeroFile(localTmpDir + "/zero_file0", numRecords, IOStreamUtils.DEFAULT_BUFFER_SIZE); createZeroFile(localTmpDir + "/zero_file1", numRecords, 10 << 20); }
@Override public final void write(KeyAndPartitionWritable key, ValueWritable value) throws IOException { int partition = key.getPartition(); // If writing a new partition, get a new writer if (writerPartition == null || writerPartition != partition) { // Set up new writer setNewPartitionWriter(partition); } if (key.getKey() == null && value.getAsByteBuffer() == null) { // Probably a marker tuple, skip it LOG.info("Skipping empty tuple: key=" + key.toString() + ", value=" + value.toString()); } else { // Write record writer.write(key.getKey(), value.getAsByteBuffer()); ++numRecordsWritten; } }
private void closeCurrentWriterIfNeeded() throws IOException { if (writer != null) { LOG.info("Closing current partition writer: " + writer.toString()); writer.close(); FileSystem fs = writerOutputPath.getFileSystem(jobConf); if (numRecordsWritten > 0) { // Move non empty partition data moveContentsAndDelete(writerOutputPath, new Path(outputPath), fs, LOG); // Record metatada only if it's not an empty partition RunWithCoordinator.run(configurator, new RunnableWithCoordinator() { @Override public void run(Coordinator coordinator) throws IOException { DomainVersion domainVersion = DomainBuilderProperties.getDomainVersion(coordinator, domainName, domainVersionNumber); domainVersion.addPartitionProperties(writerPartition, writer.getNumBytesWritten(), writer.getNumRecordsWritten()); } }); } else { // Move empty partition data moveContentsAndDelete(writerOutputPath, new Path(outputPath, EMPTY_PARTITIONS_DIR), fs, LOG); } } } }