Refine search
arrayValue.size(), schema); for (Object obj : arrayValue) { arrayCopy.add(deepCopy(schema.getElementType(), obj)); return value; // immutable case ENUM: return createEnum(value.toString(), schema); case FIXED: return createFixed(null, ((GenericFixed) value).bytes(), schema); case FLOAT: return value; // immutable new HashMap<>(mapValue.size()); for (Map.Entry<CharSequence, Object> entry : mapValue.entrySet()) { mapCopy.put((CharSequence)(deepCopy(STRINGS, entry.getKey())), deepCopy(schema.getValueType(), entry.getValue())); return null; case RECORD: Object oldState = getRecordState(value, schema); Object newRecord = newRecord(null, schema); Object newState = getRecordState(newRecord, schema); for (Field f : schema.getFields()) { int pos = f.pos(); String name = f.name(); Object newValue = deepCopy(f.schema(), getField(value, name, pos, oldState)); setField(newRecord, name, pos, newValue, newState);
for (Field field : target.getSchema().getFields()) { if (field.schema().getType() == Schema.Type.UNION) { Object fieldData = source.get(field.name()); Schema actualFieldSchema = GenericData.get().induce(fieldData); if (actualFieldSchema.getType() == Schema.Type.RECORD) { for (Schema candidateType : field.schema().getTypes()) { if (candidateType.getFullName().equals(actualFieldSchema.getFullName())) { GenericRecord record = new GenericData.Record(candidateType);
switch (schema.getType()) { case RECORD: if (!isRecord(datum)) return false; for (Field f : schema.getFields()) { if (!validate(f.schema(), getField(datum, f.name(), f.pos()))) return false; if (!isEnum(datum)) return false; return schema.getEnumSymbols().contains(datum.toString()); case ARRAY: if (!(isArray(datum))) return false; for (Object element : getArrayAsCollection(datum)) if (!validate(schema.getElementType(), element)) return false; return true; case MAP: if (!(isMap(datum))) return false; @SuppressWarnings(value="unchecked") Map<Object,Object> map = (Map<Object,Object>)datum; for (Map.Entry<Object,Object> entry : map.entrySet()) if (!validate(schema.getValueType(), entry.getValue())) return false; return true; case UNION: try { int i = resolveUnion(schema, datum); return validate(schema.getTypes().get(i), datum); } catch (UnresolvedUnionException e) { return false;
@Override public boolean equals(Object o) { if (o == this) return true; // identical object if (!(o instanceof Record)) return false; // not a record Record that = (Record)o; if (!this.schema.equals(that.schema)) return false; // not the same schema return GenericData.get().compare(this, that, schema, true) == 0; } @Override public int hashCode() {
private Object deserializeUnion(Object datum, Schema fileSchema, Schema recordSchema, UnionTypeInfo columnType) throws AvroSerdeException { // Calculate tags individually since the schema can evolve and can have different tags. In worst case, both schemas are same // and we would end up doing calculations twice to get the same tag int fsTag = GenericData.get().resolveUnion(fileSchema, datum); // Determine index of value from fileSchema int rsTag = GenericData.get().resolveUnion(recordSchema, datum); // Determine index of value from recordSchema Object desered = worker(datum, fileSchema == null ? null : fileSchema.getTypes().get(fsTag), recordSchema.getTypes().get(rsTag), columnType.getAllUnionObjectTypeInfos().get(rsTag)); return new StandardUnionObjectInspector.StandardUnion((byte)rsTag, desered); }
@Test(dataProvider = "google-referrer") public void testReferrer(Map<String, Object> props, EventMapper.RequestParams headers) throws Exception { ReferrerEventMapper mapper = new ReferrerEventMapper(); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(builder.build().dependentFields.get("_referrer").stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_referrer", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, headers, InetAddress.getLocalHost(), null); assertEquals("Google", event.getAttribute("_referrer_source")); assertEquals("test", event.getAttribute("_referrer_term")); assertEquals("search", event.getAttribute("_referrer_medium")); assertEquals("google.com", event.getAttribute("_referrer_domain")); assertEquals("/?q=test", event.getAttribute("_referrer_path")); assertNull(resp); GenericData.get().validate(properties.getSchema(), properties); }
switch (s.getType()) { case MAP: int size = values[column].nextLength(); map.put(key, read(s.getValueType())); // value Object record = model.newRecord(null, s); Map<String,Object> rDefaults = defaults.get(s.getFullName()); for (Field f : s.getFields()) { Object value = ((rDefaults != null) && rDefaults.containsKey(f.name())) ? model.deepCopy(f.schema(), rDefaults.get(f.name())) : read(f.schema()); model.setField(record, f.name(), f.pos(), value);
@SuppressWarnings(value="unchecked") private Object generate(Schema schema, Random random, int d) { switch (schema.getType()) { case RECORD: GenericRecord record = new GenericData.Record(schema); for (Schema.Field field : schema.getFields()) { Object value = (field.getObjectProp(USE_DEFAULT) == null) ? generate(field.schema(), random, d+1) : GenericData.get().getDefaultValue(field); record.put(field.name(), value); List<String> symbols = schema.getEnumSymbols(); return new GenericData.EnumSymbol (schema, symbols.get(random.nextInt(symbols.size())));
if (o == null) return 0; // incomplete datum int hashCode = 1; switch (s.getType()) { case RECORD: for (Field f : s.getFields()) { if (f.order() == Field.Order.IGNORE) continue; hashCode = hashCodeAdd(hashCode, getField(o, f.name(), f.pos()), f.schema()); Schema elementType = s.getElementType(); for (Object e : a) hashCode = hashCodeAdd(hashCode, e, elementType); return hashCode; case UNION: return hashCode(o, s.getTypes().get(resolveUnion(s, o))); case ENUM: return s.getEnumOrdinal(o.toString());
protected int compare(Object o1, Object o2, Schema s, boolean equals) { if (o1 == o2) return 0; switch (s.getType()) { case RECORD: for (Field f : s.getFields()) { if (f.order() == Field.Order.IGNORE) continue; // ignore this field int pos = f.pos(); String name = f.name(); int compare = compare(getField(o1, name, pos), getField(o2, name, pos), f.schema(), equals); if (compare != 0) // not equal Schema elementType = s.getElementType(); while(e1.hasNext() && e2.hasNext()) { int compare = compare(e1.next(), e2.next(), elementType, equals); if (compare != 0) return compare; throw new AvroRuntimeException("Can't compare maps!"); case UNION: int i1 = resolveUnion(s, o1); int i2 = resolveUnion(s, o2); return (i1 == i2) ? compare(o1, o2, s.getTypes().get(i1), equals) : i1 - i2; case NULL:
return column+1; switch (s.getType()) { case MAP: Map<?,?> map = (Map)o; writer.writeValue(null, column); writer.writeValue(e.getKey(), column+1); int c = write(e.getValue(), s.getValueType(), column+2); assert(c == column+arrayWidths[column]); for (Field f : s.getFields()) column = write(model.getField(o,f.name(),f.pos()), f.schema(), column); return column; case ARRAY: int b = model.resolveUnion(s, o); int i = 0; for (Schema branch : s.getTypes()) {
@Override public T copy(T from) { if (CONCURRENT_ACCESS_CHECK) { enterExclusiveThread(); } try { checkAvroInitialized(); return avroData.deepCopy(runtimeSchema, from); } finally { if (CONCURRENT_ACCESS_CHECK) { exitExclusiveThread(); } } }
for (Field field : Interop.SCHEMA$.getFields()) { if (interop.get(field.pos()) instanceof ByteBuffer) { assertTrue(Arrays.equals(((ByteBuffer)interop.get(field.pos())).array(), ((ByteBuffer)GenericData.get().deepCopy(field.schema(), interop.get(field.pos()))).array())); if ((field.schema().getType() != Type.ENUM) && (field.schema().getType() != Type.NULL) && (field.schema().getType() != Type.BOOLEAN) && (field.schema().getType() != Type.INT) assertFalse("Field " + field.name() + " is same instance in deep copy", interop.get(field.pos()) == GenericData.get().deepCopy( field.schema(), interop.get(field.pos())));
@Test public void testByteBufferDeepCopy() { // Test that a deep copy of a byte buffer respects the byte buffer // limits and capacity. byte[] buffer_value = {0, 1, 2, 3, 0, 0, 0}; ByteBuffer buffer = ByteBuffer.wrap(buffer_value, 1, 4); Schema schema = Schema.createRecord("my_record", "doc", "mytest", false); Field byte_field = new Field("bytes", Schema.create(Type.BYTES), null, null); schema.setFields(Arrays.asList(byte_field)); GenericRecord record = new GenericData.Record(schema); record.put(byte_field.name(), buffer); GenericRecord copy = GenericData.get().deepCopy(schema, record); ByteBuffer buffer_copy = (ByteBuffer) copy.get(byte_field.name()); assertEquals(buffer, buffer_copy); }
/** * Creates a GenericRecordBuilder by copying an existing record instance. * @param other the record instance to copy. */ public GenericRecordBuilder(Record other) { super(other.getSchema(), GenericData.get()); record = new GenericData.Record(other, /* deepCopy = */ true); // Set all fields in the RecordBuilder that are set in the record for (Field f : schema().getFields()) { Object value = other.get(f.pos()); // Only set the value if it is not null, if the schema type is null, // or if the schema type is a union that accepts nulls. if (isValidValue(f, value)) { set(f, data().deepCopy(f.schema(), value)); } } }
public Writable serialize(Object o, ObjectInspector objectInspector, List<String> columnNames, List<TypeInfo> columnTypes, Schema schema) throws AvroSerdeException { StructObjectInspector soi = (StructObjectInspector) objectInspector; GenericData.Record record = new GenericData.Record(schema); List<? extends StructField> outputFieldRefs = soi.getAllStructFieldRefs(); if(outputFieldRefs.size() != columnNames.size()) { throw new AvroSerdeException("Number of input columns was different than output columns (in = " + columnNames.size() + " vs out = " + outputFieldRefs.size()); } int size = schema.getFields().size(); if(outputFieldRefs.size() != size) { throw new AvroSerdeException("Hive passed in a different number of fields than the schema expected: (Hive wanted " + outputFieldRefs.size() +", Avro expected " + schema.getFields().size()); } List<? extends StructField> allStructFieldRefs = soi.getAllStructFieldRefs(); List<Object> structFieldsDataAsList = soi.getStructFieldsDataAsList(o); for(int i = 0; i < size; i++) { Field field = schema.getFields().get(i); TypeInfo typeInfo = columnTypes.get(i); StructField structFieldRef = allStructFieldRefs.get(i); Object structFieldData = structFieldsDataAsList.get(i); ObjectInspector fieldOI = structFieldRef.getFieldObjectInspector(); Object val = serialize(typeInfo, fieldOI, structFieldData, field.schema()); record.put(field.name(), val); } if(!GenericData.get().validate(schema, record)) { throw new SerializeToAvroException(schema, record); } cache.setRecord(record); return cache; }
public Record(Record other, boolean deepCopy) { schema = other.schema; values = new Object[schema.getFields().size()]; if (deepCopy) { for (int ii = 0; ii < values.length; ii++) { values[ii] = INSTANCE.deepCopy( schema.getFields().get(ii).schema(), other.values[ii]); } } else { System.arraycopy(other.values, 0, values, 0, other.values.length); } } @Override public Schema getSchema() { return schema; }
Schema.Field changedField = toInsert.getSchema().getField(subField); GenericData genericData = GenericData.get(); .validate(changedField.schema(), genericData.getField(toInsert, changedField.name(), changedField.pos())); if (!valid) { toInsert.put(subField, oldValue); throw new IncorrectTypeException( "Incorrect type - can't insert a " + value.getClass().getCanonicalName() + " into an Avro record of type " + changedField.schema().getType().toString());
private void testCopy(Schema schema, Object value, GenericData model) { // test direct copy of instance checkCopy(value, model.deepCopy(schema, value), false); // test nested in a record Schema recordSchema = Schema.createRecord("X", "", "test", false); List<Schema.Field> fields = new ArrayList<>(); fields.add(new Schema.Field("x", schema, "", null)); recordSchema.setFields(fields); GenericRecordBuilder builder = new GenericRecordBuilder(recordSchema); builder.set("x", value); GenericData.Record record = builder.build(); checkCopy(record, model.deepCopy(recordSchema, record), true); // test nested in array Schema arraySchema = Schema.createArray(schema); ArrayList array = new ArrayList(Arrays.asList(value)); checkCopy(array, model.deepCopy(arraySchema, array), true); // test record nested in array Schema recordArraySchema = Schema.createArray(recordSchema); ArrayList recordArray = new ArrayList(Arrays.asList(record)); checkCopy(recordArray, model.deepCopy(recordArraySchema, recordArray), true); }