/** * copies the input into a new byte array * @param bytesInput * @return * @throws IOException */ public static BytesInput copy(BytesInput bytesInput) throws IOException { return from(bytesInput.toByteArray()); }
@SuppressWarnings("unused") @Override public void writeAllTo(OutputStream out) throws IOException { for (BytesInput input : inputs) { if (DEBUG) LOG.debug("write " + input.size() + " bytes to out"); if (DEBUG && input instanceof SequenceBytesIn) LOG.debug("{"); input.writeAllTo(out); if (DEBUG && input instanceof SequenceBytesIn) LOG.debug("}"); } }
@Override public void writePage(BytesInput bytesInput, int valueCount, Encoding encoding) throws IOException { if (valueCount == 0) { throw new ParquetEncodingException("illegal page of 0 values"); } memSize += bytesInput.size(); pages.add(new Page(BytesInput.copy(bytesInput), valueCount, (int)bytesInput.size(), encoding)); totalValueCount += valueCount; if (DEBUG) LOG.debug("page written for " + bytesInput.size() + " bytes and " + valueCount + " records"); }
public BytesInput toBytesInput() { return BytesInput.concat( BytesInput.fromUnsignedVarInt(blockSizeInValues), BytesInput.fromUnsignedVarInt(miniBlockNumInABlock)); } }
/** * @param size the size of the page * @return the page * @throws IOException */ public BytesInput readAsBytesInput(int size) throws IOException { final BytesInput r = BytesInput.from(this.buf, this.pos, size); this.pos += size; return r; }
public void collect(BytesInput bytesInput) throws IOException { byte[] bytes = bytesInput.toByteArray(); slabs.add(bytes); size += bytes.length; }
Encoding dataEncoding, BytesInput data, Statistics<?> statistics) throws IOException { int rlByteLength = toIntWithCheck(repetitionLevels.size()); int dlByteLength = toIntWithCheck(definitionLevels.size()); int uncompressedSize = toIntWithCheck( data.size() + repetitionLevels.size() + definitionLevels.size() ); compressedData.size() + repetitionLevels.size() + definitionLevels.size() ); tempOutputStream.reset(); BytesInput.concat( BytesInput.from(tempOutputStream), repetitionLevels, definitionLevels,
public BytesInput compress(BytesInput bytes) throws IOException { final BytesInput compressedBytes; if (codec == null) { compressedBytes = bytes; } else { compressedOutBuffer.reset(); if (compressor != null) { // null compressor for non-native gzip compressor.reset(); } CompressionOutputStream cos = codec.createOutputStream(compressedOutBuffer, compressor); bytes.writeAllTo(cos); cos.finish(); cos.close(); compressedBytes = BytesInput.from(compressedOutBuffer); } return compressedBytes; }
/** * creates an uncompressed page * @param bytes the content of the page * @param dictionarySize the value count in the dictionary * @param encoding the encoding used */ public DictionaryPage(BytesInput bytes, int dictionarySize, Encoding encoding) { this(bytes, (int)bytes.size(), dictionarySize, encoding); // TODO: fix sizes long or int }
/** * @return the bytes representing the packed values * @throws IOException */ public BytesInput toBytes() throws IOException { int packedByteLength = packedPosition + BytesUtils.paddedByteCountFromBits(inputSize * bitWidth); if (DEBUG) LOG.debug("writing " + (slabs.size() * slabSize + packedByteLength) + " bytes"); if (inputSize > 0) { for (int i = inputSize; i < input.length; i++) { input[i] = 0; } pack(); } return concat(concat(slabs), BytesInput.from(packed, 0, packedByteLength)); }
public BytesInput readAsBytesInput(int size) throws IOException { if (pos + size > count) { // this is to workaround a bug where the compressedLength // of the chunk is missing the size of the header of the dictionary // to allow reading older files (using dictionary) we need this. // usually 13 to 19 bytes are missing int l1 = count - pos; int l2 = size - l1; LOG.info("completed the column chunk with " + l2 + " bytes"); return BytesInput.concat(super.readAsBytesInput(l1), BytesInput.copy(BytesInput.from(f, l2))); } return super.readAsBytesInput(size); }
@Override public BytesInput getBytes() { serializeCurrentValue(); BytesInput buf = bitWriter.finish(); if (Log.DEBUG) LOG.debug("writing a buffer of size " + buf.size() + " + 4 bytes"); // We serialize the length so that on deserialization we can // deserialize as we go, instead of having to load everything // into memory return concat(BytesInput.fromInt((int)buf.size()), buf); }
/** * getBytes will trigger flushing block buffer, DO NOT write after getBytes() is called without calling reset() * * @return */ @Override public BytesInput getBytes() { //The Page Header should include: blockSizeInValues, numberOfMiniBlocks, totalValueCount if (deltaValuesToFlush != 0) { flushBlockBuffer(); } return BytesInput.concat( config.toBytesInput(), BytesInput.fromUnsignedVarInt(totalValueCount), BytesInput.fromZigZagVarInt(firstValue), BytesInput.from(baos)); }
private void readPageV2(DataPageV2 page) { this.repetitionLevelColumn = newRLEIterator(path.getMaxRepetitionLevel(), page.getRepetitionLevels()); this.definitionLevelColumn = newRLEIterator(path.getMaxDefinitionLevel(), page.getDefinitionLevels()); try { if (DEBUG) LOG.debug("page data size " + page.getData().size() + " bytes and " + pageValueCount + " records"); initDataReader(page.getDataEncoding(), page.getData().toByteArray(), 0, page.getValueCount()); } catch (IOException e) { throw new ParquetDecodingException("could not read page " + page + " in col " + path, e); } }
@Override public BytesInput getBytes() { serializeCurrentValue(); byte[] buf = bitWriter.finish(); if (Log.DEBUG) LOG.debug("writing a buffer of size " + buf.length + " + 4 bytes"); // We serialize the length so that on deserialization we can // deserialize as we go, instead of having to load everything // into memory return BytesInput.fromSequence(BytesInput.fromInt(buf.length), BytesInput.from(buf)); }
bicw.getBytes().writeAllTo(tmp); bicw.reset();
@Override public BytesInput getBytes() { return BytesInput.empty(); }
public DictionaryPage copy() throws IOException { return new DictionaryPage(BytesInput.copy(bytes), getUncompressedSize(), dictionarySize, encoding); }
@Override public BytesInput getBytes() { return BytesInput.concat(prefixLengthWriter.getBytes(), suffixWriter.getBytes()); }
public BytesInput getBytesInput(int size) throws IOException { BytesInput bytesInput = BytesInput.from(buf, pos, size); pos += size; return bytesInput; }