private static int countFields(Field f, int count){ if(f.getChildren().isEmpty()){ count++; }else { for(Field child : f.getChildren()){ count = countFields(child, count); } } return count; }
@Override public String visit(List type) { StringBuilder sb = new StringBuilder(); if(includeName){ sb.append(field.getName()); sb.append("::"); } sb.append("list<"); sb.append(describe(field.getChildren().get(0), false)); sb.append(">"); return sb.toString(); }
private static FieldWithOrdinal getChildField(Field f, String name) { Map<String, FieldWithOrdinal> children = new HashMap<>(); int i = 0; for(Field child : f.getChildren()){ children.put(child.getName().toLowerCase(), new FieldWithOrdinal(child, i)); i++; } return children.get(name.toLowerCase()); }
public void materialize(Field field) { if (field.getChildren().size() == 0) { return; } Field innerField = field.getChildren().get(0); ValueVector innerVector = listVector.addOrGetVector(innerField.getFieldType()).getVector(); ComplexTypeHelper.materialize(innerVector, innerField); }
/** * Validates the list type as expected. * @param arrowSchema current arrow schema * @return child schema */ private List<Field> handleRepeatedField(List<Field> arrowSchema, GroupType groupType) { // validating that the list type is as expected if (arrowSchema.size() != 1 && !arrowSchema.get(0).getName().equals("$data$")) { UserException.dataReadError() .message("invalid children. Expected a single child named $data$, was actually %s for repeated type %s. ", arrowSchema, groupType); } // in the case of list, we skip over the inner type (struct = list(nameForChild).struct() bellow) return arrowSchema.get(0).getChildren(); }
public static void toString(Field field, int depth, StringBuilder b) { b.append("\n"); for (int i = 0; i < depth; i++) { b.append(" "); } b.append(field.getName()); b.append(";"); b.append(field.isNullable()); b.append(";"); b.append(Describer.describe(field.getType())); for (Field child : field.getChildren()) { toString(child, depth + 1, b); } }
public static CompleteType fromField(Field field){ // IGNORE this until the NullableMapVector.getField() returns a nullable type. // Preconditions.checkArgument(field.isNullable(), "Dremio only supports nullable types."); return new CompleteType(field.getType(), field.getChildren()); }
@Override public Field visit(ArrowType.List type) { if(field.getName().equals(column.getAsUnescapedPath())){ Field child = field.getChildren().get(0); return new Field(field.getName(), child.isNullable(), child.getType(), child.getChildren()); } return field; }
@Override public FieldConverter getNewListConverter(int fieldId, String fieldName, FieldReader reader) { if (reader.getField().getChildren().get(0).getFieldType().getType().equals(Null.INSTANCE)) { return null; } return new ListParquetConverter(fieldId, fieldName, reader); }
@Override public void initializeChildrenFromFields(List<Field> children) { if (children.size() != 1) { throw new IllegalArgumentException("Lists have only one child. Found: " + children); } Field field = children.get(0); AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(field.getFieldType()); if (!addOrGetVector.isCreated()) { throw new IllegalArgumentException("Child vector already existed: " + addOrGetVector.getVector()); } addOrGetVector.getVector().initializeChildrenFromFields(field.getChildren()); }
@Override public void initializeChildrenFromFields(List<Field> children) { if (children.size() != 1) { throw new IllegalArgumentException("Lists have only one child. Found: " + children); } Field field = children.get(0); AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(field.getFieldType()); if (!addOrGetVector.isCreated()) { throw new IllegalArgumentException("Child vector already existed: " + addOrGetVector.getVector()); } addOrGetVector.getVector().initializeChildrenFromFields(field.getChildren()); }
public static MajorType getMajorTypeForField(Field field) { if (field.getType() instanceof ObjectType) { return Types.required(TypeProtos.MinorType.GENERIC_OBJECT); } return getMajorTypeForArrowType(field.getType(), field.getChildren()); }
public void materialize(Field field) { for (Field child : field.getChildren()) { FieldVector v = TypeHelper.getNewVector(child, unionVector.getAllocator()); ComplexTypeHelper.materialize(v, child); unionVector.addVector(v); } }
public void materialize(Field field) { List<Field> children = field.getChildren(); for (Field child : children) { FieldVector v = TypeHelper.getNewVector(child, structVector.allocator, structVector.callBack); ComplexTypeHelper.materialize(v, child); structVector.putChild(child.getName(), v); } }
public void initializeChildrenFromFields(List<Field> children) { for (Field field : children) { FieldVector vector = (FieldVector) this.add(field.getName(), field.getFieldType()); vector.initializeChildrenFromFields(field.getChildren()); } }
public void materialize(Field field) { List<Field> children = field.getChildren(); for (Field child : children) { FieldVector v = TypeHelper.getNewVector(child, structVector.allocator, structVector.callBack); ComplexTypeHelper.materialize(v, child); structVector.putChild(child.getName(), v); } }
public static SerializedField getSerializedField(Field field) { SerializedField.Builder serializedFieldBuilder = getAsBuilder(getMajorTypeForField(field)); if (field.getChildren() != null) { for (Field childField : field.getChildren()) { serializedFieldBuilder.addChild(getSerializedField(childField)); } } serializedFieldBuilder.setNamePart(NamePart.newBuilder().setName(field.getName())); return serializedFieldBuilder.build(); }
Converter groupConverterFromArrowSchema(String nameForChild, String fieldName, GroupType groupType, Collection<SchemaPath> c) { final Field arrowField = Schema.findField(arrowSchema, fieldName); final ArrowTypeID arrowTypeType = arrowField.getType().getTypeID(); final List<Field> arrowChildren = arrowField.getChildren(); if (arrowTypeType == ArrowTypeID.Union) { // if it's a union we will add the children directly to the parent return new UnionGroupConverter(mutator, getWriterProvider(), groupType, c, options, arrowChildren, nameForChild, schemaHelper); } else if (arrowTypeType == ArrowTypeID.List) { // make sure the parquet schema matches the arrow schema and delegate handling the logical list to defaultGroupConverter() Preconditions.checkState(groupType.getOriginalType() == OriginalType.LIST, "parquet schema doesn't match the arrow schema for LIST " + nameForChild); } return defaultGroupConverter(mutator, groupType, nameForChild, c, arrowChildren); }
@Override protected BatchSchema constructSchema(FunctionLookupContext context) { final BatchSchema childSchema = child.getSchema(context); SchemaBuilder b = BatchSchema.newBuilder(); for (Field field : childSchema.getFields()) { // Revert back to original type if (dictionaryEncodedFields.containsKey(field.getName())) { b.addField(new Field(field.getName(), field.isNullable(), dictionaryEncodedFields.get(field.getName()).getArrowType(), field.getChildren())); } else { b.addField(field); } } b.setSelectionVectorMode(childSchema.getSelectionVectorMode()); return b.build(); }
@Override public Field getField() { Field f = super.getField(); FieldType type = new FieldType(true, f.getType(), f.getFieldType().getDictionary(), f.getFieldType().getMetadata()); return new Field(f.getName(), type, f.getChildren()); }