private static void writeLengthPrefixedString(SliceOutput output, String string) { byte[] bytes = string.getBytes(UTF_8); output.writeInt(bytes.length); output.writeBytes(bytes); } }
@Override public void encodeValueInto(Block block, int position, SliceOutput output) { output.writeInt(Integer.reverseBytes((int) type.getLong(block, position))); }
@Override public BlockBuilder writeInt(int value) { checkCapacity(); sliceOutput.writeInt(value); currentEntrySize += SIZE_OF_INT; return this; }
private static void writeLengthPrefixedString(SliceOutput output, String value) { byte[] bytes = value.getBytes(UTF_8); output.writeInt(bytes.length); output.writeBytes(bytes); } }
@Override public BlockBuilder writeInt(int value) { if (!initialized) { initializeCapacity(); } sliceOutput.writeInt(value); currentEntrySize += SIZE_OF_INT; return this; }
public static void writeSerializedPage(SliceOutput output, SerializedPage page) { output.writeInt(page.getPositionCount()); output.writeByte(page.getCompression().getMarker()); output.writeInt(page.getUncompressedSizeInBytes()); output.writeInt(page.getSizeInBytes()); output.writeBytes(page.getSlice()); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) block; // write the run length sliceOutput.writeInt(rleBlock.getPositionCount()); // write the value blockEncodingSerde.writeBlock(sliceOutput, rleBlock.getValue()); }
@Override public void encodeColumn(Block block, SliceOutput output, EncodeOutput encodeOutput) { for (int position = 0; position < block.getPositionCount(); position++) { if (!block.isNull(position)) { output.writeInt(Integer.reverseBytes((int) type.getLong(block, position))); } encodeOutput.closeEntry(); } }
private static void writeMultiPoint(MultiPoint geometry, SliceOutput output) { output.writeByte(GeometrySerializationType.MULTI_POINT.code()); output.writeInt(EsriShapeType.MULTI_POINT.code); writeEnvelope(geometry, output); output.writeInt(geometry.getNumPoints()); for (Coordinate coordinate : geometry.getCoordinates()) { writeCoordinate(coordinate, output); } }
private static void writeLengthPrefixedString(SliceOutput output, String value) { byte[] bytes = value.getBytes(UTF_8); output.writeInt(bytes.length); output.writeBytes(bytes); } }
@Override public final void encodeValueInto(Block block, int position, SliceOutput output) { buffer.reset(); encodeValue(block, position, buffer); // structural types nested in structural types are length prefixed Slice slice = buffer.slice(); output.writeInt(Integer.reverseBytes(slice.length())); output.writeBytes(slice); }
private static void writePolyline(Geometry geometry, SliceOutput output, boolean multitype) { int numParts; int numPoints = geometry.getNumPoints(); if (multitype) { numParts = geometry.getNumGeometries(); output.writeByte(GeometrySerializationType.MULTI_LINE_STRING.code()); } else { numParts = numPoints > 0 ? 1 : 0; output.writeByte(GeometrySerializationType.LINE_STRING.code()); } output.writeInt(EsriShapeType.POLYLINE.code); writeEnvelope(geometry, output); output.writeInt(numParts); output.writeInt(numPoints); int partIndex = 0; for (int i = 0; i < numParts; i++) { output.writeInt(partIndex); partIndex += geometry.getGeometryN(i).getNumPoints(); } writeCoordinates(geometry.getCoordinates(), output); }
static void writeRawPage(Page page, SliceOutput output, BlockEncodingSerde serde) { output.writeInt(page.getChannelCount()); for (int channel = 0; channel < page.getChannelCount(); channel++) { writeBlock(serde, output, page.getBlock(channel)); } }
@Test public void test() throws IOException { List<List<Slice>> groups = new ArrayList<>(); for (int groupIndex = 0; groupIndex < 3; groupIndex++) { List<Slice> group = new ArrayList<>(); for (int i = 0; i < 1000; i++) { Slice value = Slices.allocate(8); SliceOutput output = value.getOutput(); output.writeInt(groupIndex); output.writeInt(i); group.add(value); } groups.add(group); } testWriteValue(groups); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { int positionCount = block.getPositionCount(); sliceOutput.appendInt(positionCount); encodeNullsAsBits(sliceOutput, block); for (int position = 0; position < positionCount; position++) { if (!block.isNull(position)) { sliceOutput.writeInt(block.getInt(position, 0)); } } }
private static void writeSync(SliceOutput output) { output.writeInt(-1); output.writeLong(syncFirst); output.writeLong(syncSecond); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractArrayBlock arrayBlock = (AbstractArrayBlock) block; int positionCount = arrayBlock.getPositionCount(); int offsetBase = arrayBlock.getOffsetBase(); int[] offsets = arrayBlock.getOffsets(); int valuesStartOffset = offsets[offsetBase]; int valuesEndOffset = offsets[offsetBase + positionCount]; Block values = arrayBlock.getRawElementBlock().getRegion(valuesStartOffset, valuesEndOffset - valuesStartOffset); blockEncodingSerde.writeBlock(sliceOutput, values); sliceOutput.appendInt(positionCount); for (int position = 0; position < positionCount + 1; position++) { sliceOutput.writeInt(offsets[offsetBase + position] - valuesStartOffset); } encodeNullsAsBits(sliceOutput, block); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractRowBlock rowBlock = (AbstractRowBlock) block; int numFields = rowBlock.numFields; int positionCount = rowBlock.getPositionCount(); int offsetBase = rowBlock.getOffsetBase(); int[] fieldBlockOffsets = rowBlock.getFieldBlockOffsets(); int startFieldBlockOffset = fieldBlockOffsets[offsetBase]; int endFieldBlockOffset = fieldBlockOffsets[offsetBase + positionCount]; sliceOutput.appendInt(numFields); for (int i = 0; i < numFields; i++) { blockEncodingSerde.writeBlock(sliceOutput, rowBlock.getRawFieldBlocks()[i].getRegion(startFieldBlockOffset, endFieldBlockOffset - startFieldBlockOffset)); } sliceOutput.appendInt(positionCount); for (int position = 0; position < positionCount + 1; position++) { sliceOutput.writeInt(fieldBlockOffsets[offsetBase + position] - startFieldBlockOffset); } EncoderUtil.encodeNullsAsBits(sliceOutput, block); }
private static void writeFileHeader(SliceOutput output) { // write header output.writeBytes(RCFILE_MAGIC); output.writeByte(CURRENT_VERSION); // write codec information output.writeBoolean(false); // write metadata (which contains just the column count) output.writeInt(Integer.reverseBytes(1)); output.writeByte(COLUMN_COUNT_METADATA_KEY.length()); output.writeBytes(COLUMN_COUNT_METADATA_KEY); output.writeByte(1); output.writeByte('1'); // write sync sequence output.writeLong(syncFirst); output.writeLong(syncSecond); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { AbstractMapBlock mapBlock = (AbstractMapBlock) block; int positionCount = mapBlock.getPositionCount(); int offsetBase = mapBlock.getOffsetBase(); int[] offsets = mapBlock.getOffsets(); int[] hashTable = mapBlock.getHashTables(); int entriesStartOffset = offsets[offsetBase]; int entriesEndOffset = offsets[offsetBase + positionCount]; TypeSerde.writeType(sliceOutput, mapBlock.keyType); blockEncodingSerde.writeBlock(sliceOutput, mapBlock.getRawKeyBlock().getRegion(entriesStartOffset, entriesEndOffset - entriesStartOffset)); blockEncodingSerde.writeBlock(sliceOutput, mapBlock.getRawValueBlock().getRegion(entriesStartOffset, entriesEndOffset - entriesStartOffset)); sliceOutput.appendInt((entriesEndOffset - entriesStartOffset) * HASH_MULTIPLIER); sliceOutput.writeBytes(wrappedIntArray(hashTable, entriesStartOffset * HASH_MULTIPLIER, (entriesEndOffset - entriesStartOffset) * HASH_MULTIPLIER)); sliceOutput.appendInt(positionCount); for (int position = 0; position < positionCount + 1; position++) { sliceOutput.writeInt(offsets[offsetBase + position] - entriesStartOffset); } EncoderUtil.encodeNullsAsBits(sliceOutput, block); }