private void readStruct(FieldVector arrowVector, StructColumnVector hiveVector, StructTypeInfo typeInfo) { final int size = arrowVector.getValueCount(); final List<TypeInfo> fieldTypeInfos = typeInfo.getAllStructFieldTypeInfos(); final int fieldSize = arrowVector.getChildrenFromFields().size(); for (int i = 0; i < fieldSize; i++) { read(arrowVector.getChildrenFromFields().get(i), hiveVector.fields[i], fieldTypeInfos.get(i)); } for (int i = 0; i < size; i++) { if (arrowVector.isNull(i)) { VectorizedBatchUtil.setNullColIsNullValue(hiveVector, i); } else { hiveVector.isNull[i] = false; } } }
private void readList(FieldVector arrowVector, ListColumnVector hiveVector, ListTypeInfo typeInfo) { final int size = arrowVector.getValueCount(); final ArrowBuf offsets = arrowVector.getOffsetBuffer(); final int OFFSET_WIDTH = 4; read(arrowVector.getChildrenFromFields().get(0), hiveVector.child, typeInfo.getListElementTypeInfo()); for (int i = 0; i < size; i++) { if (arrowVector.isNull(i)) { VectorizedBatchUtil.setNullColIsNullValue(hiveVector, i); } else { hiveVector.isNull[i] = false; final int offset = offsets.getInt(i * OFFSET_WIDTH); hiveVector.offsets[i] = offset; hiveVector.lengths[i] = offsets.getInt((i + 1) * OFFSET_WIDTH) - offset; } } }
private static void loadBuffers(FieldVector vector, Field field, Iterator<ArrowBuf> buffers, Iterator<ArrowFieldNode> nodes) { checkArgument(nodes.hasNext(), "no more field nodes for for field " + field + " and vector " + vector); ArrowFieldNode fieldNode = nodes.next(); List<BufferLayout> bufferLayouts = TypeLayout.getTypeLayout(field.getType()).getBufferLayouts(); List<ArrowBuf> ownBuffers = new ArrayList<>(bufferLayouts.size()); for (int j = 0; j < bufferLayouts.size(); j++) { ownBuffers.add(buffers.next()); } try { vector.loadFieldBuffers(fieldNode, ownBuffers); } catch (RuntimeException e) { throw new IllegalArgumentException("Could not load buffers for field " + field + ". error message: " + e.getMessage(), e); } List<Field> children = field.getChildren(); if (children.size() > 0) { List<FieldVector> childrenFromFields = vector.getChildrenFromFields(); checkArgument(children.size() == childrenFromFields.size(), "should have as many children as in the schema: found " + childrenFromFields.size() + " expected " + children.size()); for (int i = 0; i < childrenFromFields.size(); i++) { Field child = children.get(i); FieldVector fieldVector = childrenFromFields.get(i); loadBuffers(fieldVector, child, buffers, nodes); } } }
List<FieldVector> childrenFromFields = vector.getChildrenFromFields(); checkArgument(children.size() == childrenFromFields.size(), "should have as many children as in the schema: " + "found " + childrenFromFields.size() + " expected " + children.size());
private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) { nodes.add(new ArrowFieldNode(vector.getValueCount(), includeNullCount ? vector.getNullCount() : -1)); List<ArrowBuf> fieldBuffers = vector.getFieldBuffers(); List<BufferType> expectedBuffers = TypeLayout.getTypeLayout(vector.getField().getType()).getBufferTypes(); if (fieldBuffers.size() != expectedBuffers.size()) { throw new IllegalArgumentException(String.format( "wrong number of buffers for field %s in vector %s. found: %s", vector.getField(), vector.getClass().getSimpleName(), fieldBuffers)); } buffers.addAll(fieldBuffers); for (FieldVector child : vector.getChildrenFromFields()) { appendNodes(child, nodes, buffers); } }
public VectorSchemaRoot(FieldVector parent) { this(parent.getField().getChildren(), parent.getChildrenFromFields(), parent.getValueCount()); }
List<FieldVector> children = vector.getChildrenFromFields(); if (fields.size() != children.size()) { throw new IllegalArgumentException("fields and children are not the same size: " + fields.size() + " != " +
List<FieldVector> vectorChildren = vector.getChildrenFromFields(); if (fields.size() != vectorChildren.size()) { throw new IllegalArgumentException(