private void initialize() { byteArrayOutputCache = new ExtendedByteArrayDataOutput(); }
private void initialize() { byteArrayOutputCache = new ExtendedByteArrayDataOutput(); }
private synchronized void decompress() { if (uncompressed) { return; } ByteBuffer dest = ByteBuffer.allocate(uncompressedSize); byteArrayOutputCache = new ExtendedByteArrayDataOutput(dest.array(), uncompressedSize); uncompressed = true; }
/** * Create an extended data output (can be subclassed) * * @return ExtendedDataOutput object */ public ExtendedDataOutput createExtendedDataOutput() { if (useUnsafeSerialization) { return new UnsafeByteArrayOutputStream(); } else { return new ExtendedByteArrayDataOutput(); } }
/** * Default constructor */ public CountingOutputStream() { dataOutput = new ExtendedByteArrayDataOutput(); reset(); }
/** * Construct LongsDiffWriter * @param useUnsafeWriter use unsafe writer */ public LongsDiffWriter(boolean useUnsafeWriter) { if (useUnsafeWriter) { out = new UnsafeByteArrayOutputStream(); } else { out = new ExtendedByteArrayDataOutput(); } }
/** * Create an extended data output (can be subclassed) * * @param expectedSize Expected size * @return ExtendedDataOutput object */ public ExtendedDataOutput createExtendedDataOutput(int expectedSize) { if (useUnsafeSerialization) { return new UnsafeByteArrayOutputStream(expectedSize); } else { return new ExtendedByteArrayDataOutput(expectedSize); } }
private void initialize() { byteArrayOutputCache = new ExtendedByteArrayDataOutput(); reset(); }
/** * Create an extended data output (can be subclassed) * * @param buf Buffer to use for the output (reuse perhaps) * @param pos How much of the buffer is already used * @return ExtendedDataOutput object */ public ExtendedDataOutput createExtendedDataOutput(byte[] buf, int pos) { if (useUnsafeSerialization) { return new UnsafeByteArrayOutputStream(buf, pos); } else { return new ExtendedByteArrayDataOutput(buf, pos); } }
/** * Serialize given writable object, and return it's size. * * @param w Writable object * @return it's size after serialization */ public static int size(Writable w) { try { ExtendedByteArrayDataOutput out = new ExtendedByteArrayDataOutput(); w.write(out); return out.getPos(); } catch (IOException e) { throw new RuntimeException(e); } }
/** * Serialize given writable to byte array, * using new instance of ExtendedByteArrayDataOutput. * * @param w Writable object * @return array of bytes * @param <T> Type of the object */ public static <T extends Writable> byte[] toByteArray(T w) { try { ExtendedByteArrayDataOutput out = new ExtendedByteArrayDataOutput(); w.write(out); return out.toByteArray(); } catch (IOException e) { throw new RuntimeException(e); } }
private synchronized void decompress() { if (uncompressed) { return; } //LOG.info("Big no no"); //long start = System.currentTimeMillis(); LZ4FastDecompressor decompressor = lz4factory.fastDecompressor(); ByteBuffer dest = ByteBuffer.allocate(uncompressedSize); ByteBuffer src = ByteBuffer.wrap(byteArrayOutputCache.getByteArray(), 0, byteArrayOutputCache.getPos()); decompressor.decompress(src, 0, dest, 0, uncompressedSize); byteArrayOutputCache = new ExtendedByteArrayDataOutput(dest.array(), uncompressedSize); uncompressed = true; }
private void compress() { if (!uncompressed) { return; } uncompressedSize = byteArrayOutputCache.getPos(); LZ4Compressor lz4Compressor = lz4factory.fastCompressor(); int maxCompressedLength = lz4Compressor.maxCompressedLength(uncompressedSize); ByteBuffer compressed = ByteBuffer.wrap(new byte[maxCompressedLength]); int compressedLength = lz4Compressor.compress(ByteBuffer.wrap(byteArrayOutputCache.getByteArray()), 0, uncompressedSize, compressed, 0, maxCompressedLength); byteArrayOutputCache = new ExtendedByteArrayDataOutput(compressed.array(), compressedLength); uncompressed = false; }
private void compress() { if (!uncompressed) { return; } uncompressedSize = byteArrayOutputCache.getPos(); LZ4Compressor lz4Compressor = lz4factory.fastCompressor(); int maxCompressedLength = lz4Compressor.maxCompressedLength(uncompressedSize); ByteBuffer compressed = ByteBuffer.wrap(new byte[maxCompressedLength]); int compressedLength = lz4Compressor.compress(ByteBuffer.wrap(byteArrayOutputCache.getByteArray()), 0, uncompressedSize, compressed, 0, maxCompressedLength); byteArrayOutputCache = new ExtendedByteArrayDataOutput(compressed.array(), compressedLength); uncompressed = false; }
public void writeEzip(ODAG ezip, int partId) throws IOException { reset(); if (byteArrayOutputCache == null) { byteArrayOutputCache = new ExtendedByteArrayDataOutput(); } this.partId = partId; pattern = ezip.getPattern(); ezip.write(byteArrayOutputCache); }
private void decompressDataInput() { if (!Configuration.get().isUseCompressedCaches() || uncompressed) { return; } //LOG.info("Big no no"); LZ4FastDecompressor decompressor = lz4factory.fastDecompressor(); ByteBuffer dest = ByteBuffer.allocate(uncompressedSize); ByteBuffer src = ByteBuffer.wrap(byteArrayOutputCache.getByteArray(), 0, byteArrayOutputCache.getPos()); decompressor.decompress(src, 0, dest, 0, uncompressedSize); byteArrayOutputCache = new ExtendedByteArrayDataOutput(dest.array(), uncompressedSize); uncompressed = true; }
private void compressDataOutput() { if (!Configuration.get().isUseCompressedCaches() || !uncompressed) { return; } uncompressedSize = byteArrayOutputCache.getPos(); LZ4Compressor lz4Compressor = lz4factory.fastCompressor(); int maxCompressedLength = lz4Compressor.maxCompressedLength(uncompressedSize); ByteBuffer compressed = ByteBuffer.wrap(new byte[maxCompressedLength]); int compressedLength = lz4Compressor.compress(ByteBuffer.wrap(byteArrayOutputCache.getByteArray()), 0, uncompressedSize, compressed, 0, maxCompressedLength); byteArrayOutputCache = new ExtendedByteArrayDataOutput(compressed.array(), compressedLength); uncompressed = false; }
/** * Write object to a byte array with the first 4 bytes as the size of the * entire buffer (including the size). * * @param writableObject Object to write from. * @param buffer Use this buffer instead * @param unsafe Use unsafe serialization? * @return Byte array with serialized object. */ public static byte[] writeToByteArrayWithSize(Writable writableObject, byte[] buffer, boolean unsafe) { ExtendedDataOutput extendedDataOutput; if (unsafe) { extendedDataOutput = new UnsafeByteArrayOutputStream(buffer); } else { extendedDataOutput = new ExtendedByteArrayDataOutput(buffer); } try { extendedDataOutput.writeInt(-1); writableObject.write(extendedDataOutput); extendedDataOutput.writeInt(0, extendedDataOutput.getPos()); } catch (IOException e) { throw new IllegalStateException("writeToByteArrayWithSize: " + "IOException", e); } return extendedDataOutput.getByteArray(); }
extendedDataOutput = new UnsafeByteArrayOutputStream(buffer); } else { extendedDataOutput = new ExtendedByteArrayDataOutput(buffer);
@Override public void initialize(int phase) { super.initialize(phase); nextEmbeddingStash = new ODAGStash(); splitEzips = Configuration.get().getBoolean("splitEzips", true); WorkerContext workerContext = getWorkerContext(); numberEzipAggregators = Configuration.get().getODAGNumAggregators(); if (numberEzipAggregators == -1) { // TODO: Figure out a better default? numberEzipAggregators = workerContext.getWorkerCount(); } workerContext.setupLocalCoordination(coordinationObjectFactory); int numPartitions = workerContext.getNumberPartitions(); numBlocks = Configuration.get().getInteger("numBlocks", numPartitions * numPartitions); maxBlockSize = Configuration.get().getInteger("maxBlockSize", 10000); reusableDestinationId = new IntWritable(); reusablePartWrapper = new ODAGPartLZ4Wrapper(); reusableMessageWrapper = new MessageWrapper(reusablePartWrapper); dataOutputs = new ExtendedByteArrayDataOutput[numberEzipAggregators]; for (int i = 0; i < dataOutputs.length; ++i) { dataOutputs[i] = new ExtendedByteArrayDataOutput(); } hasContent = new boolean[numberEzipAggregators]; }