static void toJson(Types.StructType struct, JsonGenerator generator) throws IOException { generator.writeStartObject(); generator.writeStringField(TYPE, STRUCT); generator.writeArrayFieldStart(FIELDS); for (Types.NestedField field : struct.fields()) { generator.writeStartObject(); generator.writeNumberField(ID, field.fieldId()); generator.writeStringField(NAME, field.name()); generator.writeBooleanField(REQUIRED, field.isRequired()); generator.writeFieldName(TYPE); toJson(field.type(), generator); generator.writeEndObject(); } generator.writeEndArray(); generator.writeEndObject(); }
@Override public Type struct(Types.StructType struct, Iterable<Type> fieldTypes) { Preconditions.checkNotNull(sourceType, "Evaluation must start with a schema."); Preconditions.checkArgument(sourceType.isStructType(), "Not a struct: " + sourceType); Types.StructType sourceStruct = sourceType.asStructType(); List<Types.NestedField> fields = struct.fields(); int length = fields.size(); List<Type> types = Lists.newArrayList(fieldTypes); List<Types.NestedField> newFields = Lists.newArrayListWithExpectedSize(length); for (int i = 0; i < length; i += 1) { Types.NestedField field = fields.get(i); int sourceFieldId = sourceStruct.field(field.name()).fieldId(); if (field.isRequired()) { newFields.add(Types.NestedField.required(sourceFieldId, field.name(), types.get(i))); } else { newFields.add(Types.NestedField.optional(sourceFieldId, field.name(), types.get(i))); } } return Types.StructType.of(newFields); }
@Override public Type struct(Types.StructType struct, Iterable<Type> fieldTypes) { Preconditions.checkArgument(sourceType.isStructType(), "Not a struct: " + sourceType); List<Types.NestedField> fields = struct.fields(); int length = fields.size(); List<Type> types = Lists.newArrayList(fieldTypes); List<Types.NestedField> newFields = Lists.newArrayListWithExpectedSize(length); boolean hasChange = false; for (int i = 0; i < length; i += 1) { Types.NestedField field = fields.get(i); Type resultType = types.get(i); if (field.type() == resultType) { newFields.add(field); } else if (field.isRequired()) { hasChange = true; newFields.add(Types.NestedField.required(field.fieldId(), field.name(), resultType)); } else { hasChange = true; newFields.add(Types.NestedField.optional(field.fieldId(), field.name(), resultType)); } } if (hasChange) { return Types.StructType.of(newFields); } return struct; }
switch (op()) { case IS_NULL: if (field.isRequired()) { return Expressions.alwaysFalse(); if (field.isRequired()) { return Expressions.alwaysTrue();
if (readField.isRequired()) { return ImmutableList.of(readField.name() + " is required, but is missing"); if (readField.isRequired() && field.isOptional()) { errors.add(readField.name() + " should be required, but is optional");
@Override public Type field(Types.NestedField field, Supplier<Type> fieldResult) { Preconditions.checkArgument(current instanceof StructType, "Not a struct: %s", current); StructType struct = (StructType) current; // fields are resolved by name because Spark only sees the current table schema. if (struct.getFieldIndex(field.name()).isEmpty()) { // make sure that filter fields are projected even if they aren't in the requested schema. if (filterRefs.contains(field.fieldId())) { return field.type(); } return null; } int fieldIndex = struct.fieldIndex(field.name()); StructField f = struct.fields()[fieldIndex]; Preconditions.checkArgument(f.nullable() || field.isRequired(), "Cannot project an optional field as non-null: %s", field.name()); this.current = f.dataType(); try { return fieldResult.get(); } catch (IllegalArgumentException e) { throw new IllegalArgumentException( "Invalid projection for field " + field.name() + ": " + e.getMessage(), e); } finally { this.current = struct; } }
@Override public Type field(Types.NestedField field, Supplier<Type> fieldResult) { Preconditions.checkArgument(current instanceof StructType, "Not a struct: %s", current); StructType struct = (StructType) current; // fields are resolved by name because Spark only sees the current table schema. if (struct.getFieldIndex(field.name()).isEmpty()) { // make sure that filter fields are projected even if they aren't in the requested schema. if (filterRefs.contains(field.fieldId())) { return field.type(); } return null; } int fieldIndex = struct.fieldIndex(field.name()); StructField f = struct.fields()[fieldIndex]; Preconditions.checkArgument(f.nullable() || field.isRequired(), "Cannot project an optional field as non-null: %s", field.name()); this.current = f.dataType(); try { return fieldResult.get(); } catch (IllegalArgumentException e) { throw new IllegalArgumentException( "Invalid projection for field " + field.name() + ": " + e.getMessage(), e); } finally { this.current = struct; } }