/** * Returns the number of parameter units required by fields for the * AllArgsConstructor. * * @param record a Record schema */ protected int calcAllArgConstructorParameterUnits(Schema record) { if (record.getType() != Schema.Type.RECORD) throw new RuntimeException("This method must only be called for record schemas."); return record.getFields().size(); }
private static TypeInfo generateRecordTypeInfo(Schema schema, Set<Schema> seenSchemas) throws AvroSerdeException { assert schema.getType().equals(Schema.Type.RECORD); if (seenSchemas == null) { seenSchemas = Collections.newSetFromMap(new IdentityHashMap<Schema, Boolean>()); } else if (seenSchemas.contains(schema)) { throw new AvroSerdeException( "Recursive schemas are not supported. Recursive schema was " + schema .getFullName()); } seenSchemas.add(schema); List<Schema.Field> fields = schema.getFields(); List<String> fieldNames = new ArrayList<String>(fields.size()); List<TypeInfo> typeInfos = new ArrayList<TypeInfo>(fields.size()); for(int i = 0; i < fields.size(); i++) { fieldNames.add(i, fields.get(i).name()); typeInfos.add(i, generateTypeInfo(fields.get(i).schema(), seenSchemas)); } return TypeInfoFactory.getStructTypeInfo(fieldNames, typeInfos); }
public Record(Record other, boolean deepCopy) { schema = other.schema; values = new Object[schema.getFields().size()]; if (deepCopy) { for (int ii = 0; ii < values.length; ii++) { values[ii] = INSTANCE.deepCopy( schema.getFields().get(ii).schema(), other.values[ii]); } } else { System.arraycopy(other.values, 0, values, 0, other.values.length); } } @Override public Schema getSchema() { return schema; }
static GenericRecord[] generateGenericNested(Schema schema, int count) { Random r = newRandom(); GenericRecord[] sourceData = new GenericRecord[count]; Schema doubleSchema = schema.getFields().get(0).schema(); for (int i = 0; i < sourceData.length; i++) { GenericRecord rec = new GenericData.Record(schema); GenericRecord inner; inner = new GenericData.Record(doubleSchema); inner.put(0, r.nextDouble()); rec.put(0, inner); inner = new GenericData.Record(doubleSchema); inner.put(0, r.nextDouble()); rec.put(1, inner); inner = new GenericData.Record(doubleSchema); inner.put(0, r.nextDouble()); rec.put(2, inner); rec.put(3, r.nextInt()); rec.put(4, r.nextInt()); rec.put(5, r.nextInt()); sourceData[i] = rec; } return sourceData; }
private static JsonNode injectUuidsFromJsonNodes(JsonNode json, Schema schema) { if (json == null) { return json; } switch (schema.getType()) { case RECORD: schema.getFields().stream() .filter(f -> !f.name().equals(UUID_FIELD)) .forEach(f -> injectUuidsFromJsonNodes(json.get(f.name()), f.schema())); boolean addressable = schema.getFields().stream().filter(f -> f.name().equals( UUID_FIELD)).findFirst().isPresent(); if (addressable) { ((ObjectNode) json).put(UUID_FIELD, (Integer) null); } break; case UNION: schema.getTypes() .forEach(s -> injectUuidsFromJsonNodes(json.get(s.getName()), s)); break; case ARRAY: json.getElements().forEachRemaining((elem) -> injectUuids(elem, schema.getElementType())); break; default: return json; } return json; }
/** * Check is record need deal locator. * * @param schema the input schema * @return boolean 'true' if record need deal locator */ public static boolean isRecordNeedDeallocator(Schema schema) { if (schema.getType() == Type.RECORD) { for (Field f : schema.getFields()) { Type type = f.schema().getType(); if (type == Type.ARRAY || type == Type.BYTES || type == Type.STRING || type == Type.FIXED || type == Type.RECORD || type == Type.UNION) { return true; } } } return false; }
/** * Helper method to extract the supported Avro schema from the given Avro field schema. * <p>Currently we support INT/LONG/FLOAT/DOUBLE/BOOLEAN/STRING/ENUM */ private static org.apache.avro.Schema extractSupportedSchema(org.apache.avro.Schema fieldSchema) { org.apache.avro.Schema.Type fieldType = fieldSchema.getType(); if (fieldType == org.apache.avro.Schema.Type.UNION) { org.apache.avro.Schema nonNullSchema = null; for (org.apache.avro.Schema childFieldSchema : fieldSchema.getTypes()) { if (childFieldSchema.getType() != org.apache.avro.Schema.Type.NULL) { if (nonNullSchema == null) { nonNullSchema = childFieldSchema; } else { throw new IllegalStateException("More than one non-null schema in UNION schema"); } } } if (nonNullSchema != null) { return extractSupportedSchema(nonNullSchema); } else { throw new IllegalStateException("Cannot find non-null schema in UNION schema"); } } else if (fieldType == org.apache.avro.Schema.Type.RECORD) { List<Field> recordFields = fieldSchema.getFields(); Preconditions.checkState(recordFields.size() == 1, "Not one field in the RECORD schema"); return extractSupportedSchema(recordFields.get(0).schema()); } else { return fieldSchema; } }
public Record(Record other, boolean deepCopy) { schema = other.schema; values = new Object[schema.getFields().size()]; if (deepCopy) { for (int ii = 0; ii < values.length; ii++) { values[ii] = INSTANCE.deepCopy( schema.getFields().get(ii).schema(), other.values[ii]); } } else { System.arraycopy(other.values, 0, values, 0, other.values.length); } } @Override public Schema getSchema() { return schema; }
@Test public void testEquals() { Schema s = recordSchema(); GenericRecord r0 = new GenericData.Record(s); GenericRecord r1 = new GenericData.Record(s); GenericRecord r2 = new GenericData.Record(s); Collection<CharSequence> l0 = new ArrayDeque<>(); List<CharSequence> l1 = new ArrayList<>(); GenericArray<CharSequence> l2 = new GenericData.Array<>(1, s.getFields().get(0).schema()); String foo = "foo"; l0.add(new StringBuffer(foo)); l1.add(foo); l2.add(new Utf8(foo)); r0.put(0, l0); r1.put(0, l1); r2.put(0, l2); assertEquals(r0, r1); assertEquals(r0, r2); assertEquals(r1, r2); }
Schema recordSchema = baseRecord.getSchema(); for (Schema.Field fieldSchema : recordSchema.getFields()) { if (baseRecord.get(fieldSchema.name()) != null) { Object field = baseRecord.get(fieldSchema.name()); if (field instanceof GenericArray) { if (fieldSchema.schema().getType() == Schema.Type.UNION) { for (Schema unoinedSchema : fieldSchema.schema().getTypes()) { if (unoinedSchema.getType() == Schema.Type.ARRAY) { COMPARATOR.setSchema(unoinedSchema.getElementType()); break; COMPARATOR.setSchema(fieldSchema.schema().getElementType()); GenericArray arrayField = (GenericArray) baseRecord.get(fieldSchema.name()); canonizeArray(arrayField); Collections.sort(arrayField, COMPARATOR);
private List<Schema.Field> getFields(Schema.Field schemaField) { List<Schema.Field> fields = new ArrayList<Schema.Field>(); JsonNode nullDefault = JsonNodeFactory.instance.nullNode(); if (schemaField.schema().getType() == Schema.Type.RECORD) { for (Schema.Field field : schemaField.schema().getFields()) { fields.add(new Schema.Field(field.name(), field.schema(), field.doc(), nullDefault)); } } else { fields.add(new Schema.Field(schemaField.name(), schemaField.schema(), schemaField.doc(), nullDefault)); } return fields; }
/** * Initialize converters per each schema field * * @param schema design schema * @review */ protected void initConverters(Schema schema) { schemaFields = schema.getFields(); avroConverters = new AvroConverter[schemaFields.size()]; for (int i = 0; i < schemaFields.size(); i++) { Schema.Field field = schemaFields.get(i); Schema fieldSchema = AvroUtils.unwrapIfNullable(field.schema()); if (LogicalTypeUtils.isLogicalTimestampMillis(fieldSchema)) { String datePattern = field.getProp( SchemaConstants.TALEND_COLUMN_PATTERN); avroConverters[i] = new StringTimestampConverter(datePattern); } else { Schema.Type type = fieldSchema.getType(); avroConverters[i] = _converterRegistry.get(type); } } }
private GenericRecord convertRowToAvroRecord(Schema schema, Row row) { final List<Schema.Field> fields = schema.getFields(); final int length = fields.size(); final GenericRecord record = new GenericData.Record(schema); for (int i = 0; i < length; i++) { final Schema.Field field = fields.get(i); record.put(i, convertFlinkType(field.schema(), row.getField(i))); } return record; }
private FieldAccessor[] createAccessorsFor(Schema schema) { List<Schema.Field> avroFields = schema.getFields(); FieldAccessor[] result = new FieldAccessor[avroFields.size()]; for(Schema.Field avroField : schema.getFields()) { result[avroField.pos()] = byName.get(avroField.name()); } return result; }