public SerializedField getMetadata() { int bufferSize = structVector.getBufferSize(); SerializedField.Builder b = SerializedField.newBuilder() .setNamePart(NamePart.newBuilder().setName(structVector.getField().getName())) .setMajorType(Types.optional(MinorType.STRUCT)) .setBufferLength(bufferSize) .setValueCount(structVector.valueCount); b.addChild(buildValidityMetadata()); int expectedBufferSize = BaseValueVectorHelper.getValidityBufferSizeFromCount(structVector.valueCount); for(ValueVector v : structVector.getChildren()) { SerializedField metadata = TypeHelper.getMetadata(v); expectedBufferSize += metadata.getBufferLength(); b.addChild(metadata); } Preconditions.checkState(expectedBufferSize == bufferSize, "Invalid buffer count: %s != %s", expectedBufferSize, bufferSize); return b.build(); }
builder.addChild(input.mergeObject(com.dremio.exec.proto.UserBitShared.SerializedField.newBuilder(), com.dremio.exec.proto.SchemaUserBitShared.SerializedField.MERGE));
private SerializedField buildOffsetAndDataMetadata() { SerializedField offsetField = SerializedField.newBuilder() .setNamePart(NamePart.newBuilder().setName("$offsets$").build()) .setValueCount((vector.valueCount == 0) ? 0 : vector.valueCount + 1) .setBufferLength((vector.valueCount == 0) ? 0 : (vector.valueCount + 1) * 4) .setMajorType(com.dremio.common.types.Types.required(com.dremio.common.types.TypeProtos.MinorType.UINT4)) .build(); SerializedField.Builder dataBuilder = SerializedField.newBuilder() .setNamePart(NamePart.newBuilder().setName("$values$").build()) .setValueCount(vector.valueCount) .setBufferLength(vector.getBufferSize() - getValidityBufferSizeFromCount(vector.valueCount)) .addChild(offsetField) .setMajorType(com.dremio.common.types.Types.required(com.dremio.common.types.TypeProtos.MinorType.VARCHAR)); return dataBuilder.build(); }
private SerializedField buildOffsetAndDataMetadata() { SerializedField offsetField = SerializedField.newBuilder() .setNamePart(NamePart.newBuilder().setName("$offsets$").build()) .setValueCount((vector.valueCount == 0) ? 0 : vector.valueCount + 1) .setBufferLength((vector.valueCount == 0) ? 0 : (vector.valueCount + 1) * 4) .setMajorType(com.dremio.common.types.Types.required(com.dremio.common.types.TypeProtos.MinorType.UINT4)) .build(); SerializedField.Builder dataBuilder = SerializedField.newBuilder() .setNamePart(NamePart.newBuilder().setName("$values$").build()) .setValueCount(vector.valueCount) .setBufferLength(vector.getBufferSize() - getValidityBufferSizeFromCount(vector.valueCount)) .addChild(offsetField) .setMajorType(com.dremio.common.types.Types.required(com.dremio.common.types.TypeProtos.MinorType.VARBINARY)); return dataBuilder.build(); }
private void allocateAccumulatorForNewBatch(FieldVector vector, ArrowBuf buf) { final int validitySize = getValidityBufferSizeFromCount(maxValuesPerBatch); final int totalBufferSize = getTotalBufferSize(); UserBitShared.SerializedField field = TypeHelper.getMetadata(output); UserBitShared.SerializedField.Builder builder = field.toBuilder(); builder.setValueCount(maxValuesPerBatch); builder.setBufferLength(totalBufferSize); builder.clearChild(); builder.addChild(field.getChild(0).toBuilder().setValueCount(maxValuesPerBatch).setBufferLength(validitySize)); builder.addChild(field.getChild(1).toBuilder().setValueCount(maxValuesPerBatch).setBufferLength(totalBufferSize - validitySize)); UserBitShared.SerializedField newField = builder.build(); TypeHelper.load(vector, newField, buf); }
public static SerializedField getSerializedField(Field field) { SerializedField.Builder serializedFieldBuilder = getAsBuilder(getMajorTypeForField(field)); if (field.getChildren() != null) { for (Field childField : field.getChildren()) { serializedFieldBuilder.addChild(getSerializedField(childField)); } } serializedFieldBuilder.setNamePart(NamePart.newBuilder().setName(field.getName())); return serializedFieldBuilder.build(); }
public SerializedField getMetadata() { SerializedField.Builder b = SerializedField.newBuilder() .setNamePart(NamePart.newBuilder().setName(unionVector.getField().getName())) .setMajorType(Types.optional(MinorType.UNION)) .setBufferLength(unionVector.getBufferSize()) .setValueCount(unionVector.valueCount); b.addChild(buildTypeField()); b.addChild(TypeHelper.getMetadata(unionVector.internalStruct)); return b.build(); }
public SerializedField getMetadata() { SerializedField.Builder b = SerializedField.newBuilder() .setNamePart(NamePart.newBuilder().setName(structVector.getField().getName())) .setMajorType(Types.optional(MinorType.STRUCT)) .setBufferLength(structVector.getBufferSize()) .setValueCount(structVector.valueCount); for(ValueVector v : structVector.getChildren()) { b.addChild(TypeHelper.getMetadata(v)); } return b.build(); } }
public SerializedField.Builder getMetadataBuilder() { return SerializedField.newBuilder() .setMajorType(MajorType.newBuilder().setMinorType(MinorType.LIST).setMode(DataMode.OPTIONAL).build()) .setNamePart(NamePart.newBuilder().setName(listVector.getField().getName())) .setValueCount(listVector.getValueCount()) .setBufferLength(listVector.getBufferSize()) .addChild(buildOffsetMetadata()) .addChild(buildValidityMetadata()) .addChild(TypeHelper.getMetadata(listVector.vector)); }