new CueballWriter(newCueballBaseOutputStream, keyHashSize, null, valueSize, compressionCodec, hashIndexBits); newCueballBaseWriter.writeHash(keyValuePair.keyHash, keyValuePair.value); newCueballBaseWriter.close();
@Override public String toString() { return "CurlyWriter [" + "numRecordsWritten=" + getNumRecordsWritten() + ", numBytesWritten=" + getNumBytesWritten() + "]"; } }
@Override public Writer getWriter(DomainVersion domainVersion, PartitionRemoteFileOps partitionRemoteFileOps, int partitionNumber) throws IOException { IncrementalDomainVersionProperties domainVersionProperties = getDomainVersionProperties(domainVersion); return new CueballWriter(partitionRemoteFileOps.getOutputStream(getName(domainVersion.getVersionNumber(), domainVersionProperties.isBase())), keyHashSize, hasher, valueSize, getCompressionCodec(), hashIndexBits ); }
writeHash(ByteBuffer.wrap(keyHashBytes), value); numRecordsWritten++;
@Override public Writer getCompactorWriter(DomainVersion domainVersion, PartitionRemoteFileOps partitionRemoteFileOps, int partitionNumber) throws IOException { IncrementalDomainVersionProperties domainVersionProperties = getDomainVersionProperties(domainVersion); // Note: We use the identity hasher since keys coming in are already hashed keys return new CueballWriter(partitionRemoteFileOps.getOutputStream(getName(domainVersion.getVersionNumber(), domainVersionProperties.isBase())), keyHashSize, new IdentityHasher(), valueSize, getCompressionCodec(), hashIndexBits ); }
@Override public void close() throws IOException { // clear the last block, if there is one if (uncompressedOffset > 0) { clearUncompressed(); } // serialize the footer byte[] footer = new byte[8 * hashIndex.length + 4 + 4]; for (int i = 0; i < hashIndex.length; i++) { EncodingHelper.encodeLittleEndianFixedWidthLong(hashIndex[i], footer, i * 8, 8); } // write the buffer size hints EncodingHelper.encodeLittleEndianFixedWidthLong(maxUncompressedBlockSize, footer, footer.length - 8, 4); EncodingHelper.encodeLittleEndianFixedWidthLong(maxCompressedBlockSize, footer, footer.length - 4, 4); stream.write(footer); numBytesWritten += footer.length; // flush everything and close stream.flush(); stream.close(); }