/** * Returns true if this field is optional. Optional fields are represented * as a type union containing the null type. * * @param field * @return */ /* package private */boolean isOptional(Field field) { if(field.schema().getType() == Type.UNION) { for(Schema nestedType: field.schema().getTypes()) { if(nestedType.getType() == Type.NULL) { return true; } } } return false; }
public Record(Record other, boolean deepCopy) { schema = other.schema; values = new Object[schema.getFields().size()]; if (deepCopy) { for (int ii = 0; ii < values.length; ii++) { values[ii] = INSTANCE.deepCopy( schema.getFields().get(ii).schema(), other.values[ii]); } } else { System.arraycopy(other.values, 0, values, 0, other.values.length); } } @Override public Schema getSchema() { return schema; }
/** * Gets the array schema. * * @param delta the delta * @param field the field * @return the array schema */ private static Schema getArraySchema(GenericRecord delta, String field) { List<Schema> fieldTypes = delta.getSchema().getField(field).schema().getTypes(); for (Schema type : fieldTypes) { if (type.getType() == Schema.Type.ARRAY) { return type; } } return null; }
/** * Helper method to extract the supported Avro schema from the given Avro field schema. * <p>Currently we support INT/LONG/FLOAT/DOUBLE/BOOLEAN/STRING/ENUM */ private static org.apache.avro.Schema extractSupportedSchema(org.apache.avro.Schema fieldSchema) { org.apache.avro.Schema.Type fieldType = fieldSchema.getType(); if (fieldType == org.apache.avro.Schema.Type.UNION) { org.apache.avro.Schema nonNullSchema = null; for (org.apache.avro.Schema childFieldSchema : fieldSchema.getTypes()) { if (childFieldSchema.getType() != org.apache.avro.Schema.Type.NULL) { if (nonNullSchema == null) { nonNullSchema = childFieldSchema; } else { throw new IllegalStateException("More than one non-null schema in UNION schema"); } } } if (nonNullSchema != null) { return extractSupportedSchema(nonNullSchema); } else { throw new IllegalStateException("Cannot find non-null schema in UNION schema"); } } else if (fieldType == org.apache.avro.Schema.Type.RECORD) { List<Field> recordFields = fieldSchema.getFields(); Preconditions.checkState(recordFields.size() == 1, "Not one field in the RECORD schema"); return extractSupportedSchema(recordFields.get(0).schema()); } else { return fieldSchema; } }
/** * Check is record need deal locator. * * @param schema the input schema * @return boolean 'true' if record need deal locator */ public static boolean isRecordNeedDeallocator(Schema schema) { if (schema.getType() == Type.RECORD) { for (Field f : schema.getFields()) { Type type = f.schema().getType(); if (type == Type.ARRAY || type == Type.BYTES || type == Type.STRING || type == Type.FIXED || type == Type.RECORD || type == Type.UNION) { return true; } } } return false; }
private void checkConvertedAvroData(Schema schema, GenericRecord record) { Assert.assertEquals(schema.getNamespace(), "example.avro"); Assert.assertEquals(schema.getType(), Schema.Type.RECORD); Assert.assertEquals(schema.getName(), "User"); Assert.assertEquals(schema.getFields().size(), 3); Schema.Field nameField = schema.getField("name"); Assert.assertEquals(nameField.name(), "name"); Assert.assertEquals(nameField.schema().getType(), Schema.Type.STRING); Schema.Field favNumberField = schema.getField("favorite_number"); Assert.assertEquals(favNumberField.name(), "favorite_number"); Assert.assertEquals(favNumberField.schema().getType(), Schema.Type.INT); Schema.Field favColorField = schema.getField("favorite_color"); Assert.assertEquals(favColorField.name(), "favorite_color"); Assert.assertEquals(favColorField.schema().getType(), Schema.Type.STRING); Assert.assertEquals(record.get("name"), "Alyssa"); Assert.assertEquals(record.get("favorite_number"), 256d); Assert.assertEquals(record.get("favorite_color"), "yellow"); }
/** * Creates a Builder by copying an existing Builder. * @param other The existing Builder to copy. */ private Builder(org.apache.flink.streaming.tests.avro.InnerPayLoadAvro.Builder other) { super(other); if (isValidValue(fields()[0], other.sequenceNumber)) { this.sequenceNumber = data().deepCopy(fields()[0].schema(), other.sequenceNumber); fieldSetFlags()[0] = true; } }
private static Schema getDeltaSchemaByFullName(Schema deltaSchema, String fullName) { Schema deltaT = deltaSchema.getElementType(); Schema deltaUnion = deltaT.getField("delta").schema(); List<Schema> deltas = deltaUnion.getTypes(); for (Schema delta : deltas) { if (delta.getFullName().equals(fullName)) { return delta; } } return null; }
/** * Return whether the Avro field is a single-value field. */ public static boolean isSingleValueField(Field field) { try { org.apache.avro.Schema fieldSchema = extractSupportedSchema(field.schema()); return fieldSchema.getType() != org.apache.avro.Schema.Type.ARRAY; } catch (Exception e) { throw new RuntimeException("Caught exception while extracting non-null schema from field: " + field.name(), e); } }
/** * Convert to the output schema of a field */ protected Field convertFieldSchema(Schema inputSchema, Field field, WorkUnitState workUnit) throws SchemaConversionException { if (field.name().equals(payloadField)) { // Create a payload field with latest schema return createLatestPayloadField(field); } // Make a copy of the field to the output schema return new Field(field.name(), field.schema(), field.doc(), field.defaultValue(), field.order()); }
private static void _copyFields( List<Schema.Field> fields, List<Schema.Field> copiedFieldList) { for (Schema.Field field : fields) { Schema.Field newField = new Schema.Field( field.name(), field.schema(), field.doc(), field.defaultVal()); _copyFieldProperties(field.getObjectProps(), newField); copiedFieldList.add(newField); } }
/** Called to read a single field of a record. May be overridden for more * efficient or alternate implementations.*/ protected void readField(Object r, Field f, Object oldDatum, ResolvingDecoder in, Object state) throws IOException { data.setField(r, f.name(), f.pos(), read(oldDatum, f.schema(), in), state); }
/** Called to write a single field of a record. May be overridden for more * efficient or alternate implementations.*/ protected void writeField(Object datum, Field f, Encoder out, Object state) throws IOException { Object value = data.getField(datum, f.name(), f.pos(), state); try { write(f.schema(), value, out); } catch (NullPointerException e) { throw npe(e, " in field " + f.name()); } }
/** Called to write a single field of a record. May be overridden for more * efficient or alternate implementations.*/ protected void writeField(Object datum, Field f, Encoder out, Object state) throws IOException { Object value = data.getField(datum, f.name(), f.pos(), state); try { write(f.schema(), value, out); } catch (NullPointerException e) { throw npe(e, " in field " + f.name()); } }
/** Called to read a single field of a record. May be overridden for more * efficient or alternate implementations.*/ protected void readField(Object r, Field f, Object oldDatum, ResolvingDecoder in, Object state) throws IOException { data.setField(r, f.name(), f.pos(), read(oldDatum, f.schema(), in), state); }
/** * Gets the schema by full name. * * @param delta the delta * @param field the field * @param fullName the full name * @return the schema by full name */ private static Schema getSchemaByFullName(GenericRecord delta, String field, String fullName) { Schema fieldSchema = delta.getSchema().getField(field).schema(); if (fieldSchema.getType() == Schema.Type.UNION) { List<Schema> fieldTypes = fieldSchema.getTypes(); return getSchemaByFullName(fieldTypes, fullName); } else { return fieldSchema.getFullName().equals(fullName) ? fieldSchema : null; } }