private void writeList(ListVector arrowVector, ListColumnVector hiveVector, ListTypeInfo typeInfo, int size, VectorizedRowBatch vectorizedRowBatch, boolean isNative) { final int OFFSET_WIDTH = 4; final TypeInfo elementTypeInfo = typeInfo.getListElementTypeInfo(); final ColumnVector hiveElementVector = hiveVector.child; final FieldVector arrowElementVector = (FieldVector) arrowVector.addOrGetVector(toFieldType(elementTypeInfo)).getVector(); arrowElementVector.setInitialCapacity(hiveVector.childCount); arrowElementVector.allocateNew(); write(arrowElementVector, hiveElementVector, elementTypeInfo, hiveVector.childCount, vectorizedRowBatch, isNative); final ArrowBuf offsetBuffer = arrowVector.getOffsetBuffer(); int nextOffset = 0; for (int rowIndex = 0; rowIndex < size; rowIndex++) { if (hiveVector.isNull[rowIndex]) { offsetBuffer.setInt(rowIndex * OFFSET_WIDTH, nextOffset); } else { offsetBuffer.setInt(rowIndex * OFFSET_WIDTH, nextOffset); nextOffset += (int) hiveVector.lengths[rowIndex]; arrowVector.setNotNull(rowIndex); } } offsetBuffer.setInt(size * OFFSET_WIDTH, nextOffset); }
@Override public void initializeChildrenFromFields(List<Field> children) { if (children.size() != 1) { throw new IllegalArgumentException("Lists have only one child. Found: " + children); } Field field = children.get(0); AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(field.getFieldType()); if (!addOrGetVector.isCreated()) { throw new IllegalArgumentException("Child vector already existed: " + addOrGetVector.getVector()); } addOrGetVector.getVector().initializeChildrenFromFields(field.getChildren()); }
public void materialize(Field field) { if (field.getChildren().size() == 0) { return; } Field innerField = field.getChildren().get(0); ValueVector innerVector = listVector.addOrGetVector(innerField.getFieldType()).getVector(); ComplexTypeHelper.materialize(innerVector, innerField); }
public TransferImpl(ListVector to) { this.to = to; to.addOrGetVector(vector.getField().getFieldType()); if (to.getDataVector() instanceof ZeroVector) { to.addOrGetVector(vector.getField().getFieldType()); } dataTransferPair = getDataVector().makeTransferPair(to.getDataVector()); }
private ValueVector addChild(ValueVector vector, MinorType child) { return ((ListVector)vector).addOrGetVector(nullable(child.getType())).getVector(); }
protected FieldWriter getWriter(MinorType type, ArrowType arrowType) { if (state == State.UNION) { ((UnionWriter) writer).getWriter(type); } else if (state == State.UNTYPED) { if (type == null) { // ??? return null; } if (arrowType == null) { arrowType = type.getType(); } ValueVector v = listVector.addOrGetVector(FieldType.nullable(arrowType)).getVector(); v.allocateNew(); setWriter(v, arrowType); writer.setPosition(position); } else if (type != this.type) { promoteToUnion(); ((UnionWriter) writer).getWriter(type); } return writer; }
public CustomGenerator(int numRows, BufferAllocator allocator) { Preconditions.checkState(numRows > 0); values = listOfStrings(numRows); rowIds = randomListOfInts(numRows); listValues = listOfLists(numRows); BatchSchema schema = BatchSchema.newBuilder() .addField(ID) .addField(VALUE) .addField(MYLIST) .build(); container = VectorContainer.create(allocator, schema); id = container.addOrGet(ID); value = container.addOrGet(VALUE); list = container.addOrGet(MYLIST); Types.MinorType type = Types.MinorType.BIGINT; list.addOrGetVector(FieldType.nullable(type.getType())); }
public void load(SerializedField metadata, ArrowBuf buffer) { /* release the current buffers (if any) */ listVector.clear(); /* load inner offset buffer */ final SerializedField offsetMetadata = metadata.getChild(0); final int offsetLength = offsetMetadata.getBufferLength(); loadOffsetBuffer(offsetMetadata, buffer); /* load inner validity buffer */ final SerializedField bitMetadata = metadata.getChild(1); final int bitLength = bitMetadata.getBufferLength(); loadValidityBuffer(bitMetadata, buffer.slice(offsetLength, bitLength)); /* load inner data vector */ final SerializedField vectorMetadata = metadata.getChild(2); if (listVector.getDataVector() == BaseRepeatedValueVector.DEFAULT_DATA_VECTOR) { listVector.addOrGetVector(FieldType.nullable(getArrowMinorType(vectorMetadata.getMajorType().getMinorType()).getType())); } final int vectorLength = vectorMetadata.getBufferLength(); TypeHelper.load(listVector.vector, vectorMetadata, buffer.slice(offsetLength + bitLength, vectorLength)); listVector.setLastSet(metadata.getValueCount()); listVector.valueCount = metadata.getValueCount(); }