/** * Gets the schema by full name. * * @param arraySchema the array schema * @param fullName the full name * @return the schema by full name */ private static Schema getSchemaByFullName(Schema arraySchema, String fullName) { if (arraySchema.getElementType().getType() == Schema.Type.UNION) { List<Schema> itemTypes = arraySchema.getElementType().getTypes(); return getSchemaByFullName(itemTypes, fullName); } else { return arraySchema.getElementType().getFullName().equals( fullName) ? arraySchema.getElementType() : null; } }
public static boolean isPrimitiveArray(Schema schema) { return schema.getType().equals(Schema.Type.ARRAY) && isPrimitive(schema.getElementType()); }
/** * Gets the delta schema by full name. * * @param fullName the full name * @return the delta schema by full name */ private Schema getDeltaSchemaByFullName(String fullName) { Schema deltaT = deltaSchema.getElementType(); Schema deltaUnion = deltaT.getField(DELTA).schema(); List<Schema> deltas = deltaUnion.getTypes(); for (Schema delta : deltas) { if (delta.getFullName().equals(fullName)) { return delta; } } return null; }
switch (schema.getType()) { case RECORD: queue.add(schema); for (Schema.Field field : schema.getFields()) enqueue(field.schema()); break; case MAP: enqueue(schema.getValueType()); break; case ARRAY: enqueue(schema.getElementType()); break; case UNION: for (Schema s : schema.getTypes()) enqueue(s); break;
/** * Check if schemas are equals to each other. * * @param s1 the schema one * @param s2 the schema two * @return boolean 'true' if the input schemas are equals */ public static boolean isEqualSchemas(Schema s1, Schema s2) { if (!(s1.getType().equals(s2.getType()) && s1.getFullName().equals(s2.getFullName()))) { return false; } switch (s1.getType()) { case RECORD: return isEqualRecords(s1, s2); case UNION: return isEqualUnions(s1, s2); case ARRAY: return isEqualSchemas(s1.getElementType(), s2.getElementType()); case MAP: return isEqualSchemas(s1.getValueType(), s2.getValueType()); case ENUM: return isEqualEnums(s1, s2); case FIXED: return s1.getFixedSize() == s2.getFixedSize(); default: return true; } }
public RuntimeMonitorClient(HttpRequestConfig requestConfig, KeyStore keyStore, KeyStore trustStore, Supplier<InetSocketAddress> serverAddressSupplier, Proxy proxy) { this.requestConfig = requestConfig; this.httpsEnabler = new HttpsEnabler().setKeyStore(keyStore, ""::toCharArray).setTrustStore(trustStore); this.responseDatumReader = new GenericDatumReader<>( MonitorSchemas.V1.MonitorResponse.SCHEMA.getValueType().getElementType()); this.serverAddressSupplier = serverAddressSupplier; this.proxy = proxy; }
@Test public void testReadUUIDList() throws IOException { Schema uuidListSchema = SchemaBuilder.record(RecordWithUUIDList.class.getName()) .fields() .name("uuids").type().array().items().stringType().noDefault() .endRecord(); uuidListSchema.getField("uuids").schema().addProp( SpecificData.CLASS_PROP, List.class.getName()); LogicalTypes.uuid().addToSchema( uuidListSchema.getField("uuids").schema().getElementType()); UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); GenericRecord r = new GenericData.Record(uuidListSchema); r.put("uuids", Arrays.asList(u1.toString(), u2.toString())); RecordWithUUIDList expected = new RecordWithUUIDList(); expected.uuids = Arrays.asList(u1, u2); File test = write(uuidListSchema, r); Assert.assertEquals("Should convert Strings to UUIDs", expected, read(REFLECT.createDatumReader(uuidListSchema), test).get(0)); }
aliases.put(alias, namedSchema.name); switch (schema.getType()) { case RECORD: if (seen.containsKey(schema)) return; // break loops break; case ARRAY: getAliases(schema.getElementType(), seen, aliases, fieldAliases); break; case MAP: getAliases(schema.getValueType(), seen, aliases, fieldAliases); break; case UNION: for (Schema s : schema.getTypes()) getAliases(s, seen, aliases, fieldAliases); break;
private static JsonNode injectUuidsFromJsonNodes(JsonNode json, Schema schema) { if (json == null) { return json; } switch (schema.getType()) { case RECORD: schema.getFields().stream() .filter(f -> !f.name().equals(UUID_FIELD)) .forEach(f -> injectUuidsFromJsonNodes(json.get(f.name()), f.schema())); boolean addressable = schema.getFields().stream().filter(f -> f.name().equals( UUID_FIELD)).findFirst().isPresent(); if (addressable) { ((ObjectNode) json).put(UUID_FIELD, (Integer) null); } break; case UNION: schema.getTypes() .forEach(s -> injectUuidsFromJsonNodes(json.get(s.getName()), s)); break; case ARRAY: json.getElements().forEachRemaining((elem) -> injectUuids(elem, schema.getElementType())); break; default: return json; } return json; }
/* package private */void compareArrayTypes(Schema oldSchema, Schema newSchema, List<Message> messages, String name) { if(oldSchema == null || newSchema == null || oldSchema.getType() != Schema.Type.ARRAY) { throw new IllegalArgumentException("Old schema must be ARRAY type. Name=" + name + ". Type=" + oldSchema); } if(newSchema.getType() != Schema.Type.ARRAY) { messages.add(new Message(Level.ERROR, "Illegal type change from " + oldSchema.getType() + " to " + newSchema.getType() + " for field " + name)); return; } // Compare the array element types compareTypes(oldSchema.getElementType(), newSchema.getElementType(), messages, name + ".<array element>"); }
switch (avroSchema.getType()) { case INT: { final LogicalType logicalType = avroSchema.getLogicalType(); for (final Field field : recordFields) { final Object avroFieldValue = record.get(field.name()); final Object fieldValue = normalizeValue(avroFieldValue, field.schema(), fieldName + "/" + field.name()); values.put(field.name(), fieldValue); final Object[] valueArray = new Object[list.size()]; for (int i = 0; i < list.size(); i++) { final Schema elementSchema = avroSchema.getElementType(); valueArray[i] = normalizeValue(list.get(i), elementSchema, fieldName + "[" + i + "]"); final Object[] valueArray = new Object[array.size()]; for (int i = 0; i < array.size(); i++) { final Schema elementSchema = avroSchema.getElementType(); valueArray[i] = normalizeValue(array.get(i), elementSchema, fieldName + "[" + i + "]"); obj = normalizeValue(obj, avroSchema.getValueType(), fieldName + "[" + key + "]");
private static Schema getDeltaSchemaByFullName(Schema deltaSchema, String fullName) { Schema deltaT = deltaSchema.getElementType(); Schema deltaUnion = deltaT.getField("delta").schema(); List<Schema> deltas = deltaUnion.getTypes(); for (Schema delta : deltas) { if (delta.getFullName().equals(fullName)) { return delta; } } return null; }
public RuntimeMonitorClient(HttpRequestConfig requestConfig, KeyStore keyStore, KeyStore trustStore, Supplier<InetSocketAddress> serverAddressSupplier, Proxy proxy) { this.requestConfig = requestConfig; this.httpsEnabler = new HttpsEnabler().setKeyStore(keyStore, ""::toCharArray).setTrustStore(trustStore); this.responseDatumReader = new GenericDatumReader<>( MonitorSchemas.V1.MonitorResponse.SCHEMA.getValueType().getElementType()); this.serverAddressSupplier = serverAddressSupplier; this.proxy = proxy; }
@Test public void testReadUUIDArray() throws IOException { Schema uuidArraySchema = SchemaBuilder.record(RecordWithUUIDArray.class.getName()) .fields() .name("uuids").type().array().items().stringType().noDefault() .endRecord(); LogicalTypes.uuid().addToSchema( uuidArraySchema.getField("uuids").schema().getElementType()); UUID u1 = UUID.randomUUID(); UUID u2 = UUID.randomUUID(); GenericRecord r = new GenericData.Record(uuidArraySchema); r.put("uuids", Arrays.asList(u1.toString(), u2.toString())); RecordWithUUIDArray expected = new RecordWithUUIDArray(); expected.uuids = new UUID[] {u1, u2}; File test = write(uuidArraySchema, r); Assert.assertEquals("Should convert Strings to UUIDs", expected, read(REFLECT.createDatumReader(uuidArraySchema), test).get(0)); }
private boolean isCustomCodable(Schema schema, Set<Schema> seen) { if (! seen.add(schema)) return true; if (schema.getLogicalType() != null) return false; boolean result = true; switch (schema.getType()) { case RECORD: for (Schema.Field f : schema.getFields()) result &= isCustomCodable(f.schema(), seen); break; case MAP: result = isCustomCodable(schema.getValueType(), seen); break; case ARRAY: result = isCustomCodable(schema.getElementType(), seen); break; case UNION: List<Schema> types = schema.getTypes(); // Only know how to handle "nulling" unions for now if (types.size() != 2 || ! types.contains(NULL_SCHEMA)) return false; for (Schema s : types) result &= isCustomCodable(s, seen); break; default: } return result; }
Object field = baseRecord.get(fieldSchema.name()); if (field instanceof GenericArray) { if (fieldSchema.schema().getType() == Schema.Type.UNION) { for (Schema unoinedSchema : fieldSchema.schema().getTypes()) { if (unoinedSchema.getType() == Schema.Type.ARRAY) { COMPARATOR.setSchema(unoinedSchema.getElementType()); break; COMPARATOR.setSchema(fieldSchema.schema().getElementType());
public ArrayConverter(String fieldName, boolean nullable, String sourceType, Schema schemaNode, WorkUnitState state, List<String> ignoreFields) throws UnsupportedDateTypeException { super(fieldName, nullable, sourceType); super.setElementConverter( getConvertor(fieldName, schemaNode.getElementType().getType().getName(), schemaNode.getElementType(), state, isNullable(), ignoreFields)); }
switch (sc.getType()) { case NULL: case BOOLEAN: case ARRAY: return Symbol.seq(Symbol.repeat(Symbol.ARRAY_END, Symbol.ITEM_END, generate(sc.getElementType(), seen)), Symbol.ARRAY_START); case MAP: return Symbol.seq(Symbol.repeat(Symbol.MAP_END, Symbol.ITEM_END, generate(sc.getValueType(), seen), Symbol.MAP_KEY_MARKER, Symbol.STRING), Symbol.MAP_START); for (Field f : sc.getFields()) { production[--i] = Symbol.fieldAdjustAction(n, f.name()); production[--i] = generate(f.schema(), seen); production[--i] = Symbol.FIELD_END; n++;
public static Schema getDeltaSchemaByFullName(Schema deltaSchema, String fullName) { Schema deltaT = deltaSchema.getElementType(); Schema deltaUnion = deltaT.getField("delta").schema(); List<Schema> deltas = deltaUnion.getTypes(); for (Schema delta : deltas) { if (delta.getFullName().equals(fullName)) { return delta; } } return null; }
@Test public void testReflectedSchema() { Schema expected = SchemaBuilder.record(RecordWithUUIDList.class.getName()) .fields() .name("uuids").type().array().items().stringType().noDefault() .endRecord(); expected.getField("uuids").schema().addProp( SpecificData.CLASS_PROP, List.class.getName()); LogicalTypes.uuid().addToSchema( expected.getField("uuids").schema().getElementType()); Schema actual = REFLECT.getSchema(RecordWithUUIDList.class); Assert.assertEquals("Should use the UUID logical type", expected, actual); }