Refine search
if (!sourceArray.isEmpty() && destinationChild instanceof GenericArray) { GenericArray destArray = (GenericArray) destinationChild; if (!destArray.isEmpty() && destArray.get(0).getClass() == sourceArray.get(0).getClass()) { boolean resolveStrategy = false; if (destArray.get(0) instanceof GenericContainer) { GenericContainer destFirst = (GenericContainer) destArray.get(0); GenericContainer sourceFirst = (GenericContainer) sourceArray.get(0); if (sourceArray.getSchema().getElementType().getType() == Schema.Type.RECORD) { GenericArray destArray = new GenericData.Array<>(sourceArray.size(), sourceArray.getSchema()); for (Object item : sourceArray) { GenericRecord recordItem = (GenericRecord) item; GenericRecord destRecord = new GenericData.Record( destArray.add(destRecord); case APPEND: GenericArray destArray = (GenericArray) destinationChild; if (sourceArray.getSchema().getElementType().getType() == Schema.Type.RECORD) { for (Object item : sourceArray) { GenericRecord recordItem = (GenericRecord) item; GenericRecord destRecord = new GenericData.Record( destArray.add(destRecord); destArray.addAll(sourceArray);
@Test /** check that GenericArray.reset() retains reusable elements and that GenericArray.prune() cleans * them up properly. */ public void testGenericArrayPeek() { Schema elementSchema = SchemaBuilder.record("element").fields().requiredString("value").endRecord(); Schema arraySchema = Schema.createArray(elementSchema); GenericRecord record = new GenericData.Record( elementSchema ); record.put("value", "string"); GenericArray<GenericRecord> list = new GenericData.Array<GenericRecord>(1, arraySchema); list.add(record); list.reset(); assertTrue( record == list.peek() ); list.prune(); assertNull( list.peek() ); }
/** * Adds the complex item to array. * * @param container the record * @param array the array * @throws DeltaCalculatorException the delta calculator exception */ private void addComplexItemToArray(GenericContainer container, GenericArray array) throws DeltaCalculatorException { Schema itemSchema = getSchemaByFullName(array.getSchema(), getFullName(container)); if (itemSchema.getType() == Type.RECORD) { GenericRecord subDelta = new GenericData.Record(itemSchema); fillDeltaWithoutMerge(subDelta, (GenericRecord) container); array.add(subDelta); } else { array.add(container); } }
public int compare(Object o1, Object o2, Schema s) { if (o1 == o2) return 0; switch (s.getType()) { case RECORD: if (!(o1 instanceof IndexedRecord)) IndexedRecord r1 = (IndexedRecord)o1; IndexedRecord r2 = (IndexedRecord)o2; for (Field f : s.getFields()) { if (f.order() == Field.Order.IGNORE) continue; // ignore this field int pos = f.pos(); int compare = compare(r1.get(pos), r2.get(pos), f.schema()); if (compare != 0) // not equal return f.order() == Field.Order.DESCENDING ? -compare : compare; return s.getEnumOrdinal(o1.toString()) - s.getEnumOrdinal(o2.toString()); case ARRAY: GenericArray a1 = (GenericArray)o1; GenericArray a2 = (GenericArray)o2; Iterator e1 = a1.iterator(); Iterator e2 = a2.iterator(); Schema elementType = a1.getSchema().getElementType(); while(e1.hasNext() && e2.hasNext()) { int compare = compare(e1.next(), e2.next(), elementType);
@SuppressWarnings(value="unchecked") private Object generate(Schema schema, Random random, int d) { switch (schema.getType()) { case RECORD: GenericRecord record = new GenericData.Record(schema); for (Schema.Field field : schema.getFields()) { Object value = (field.getObjectProp(USE_DEFAULT) == null) ? generate(field.schema(), random, d+1) : GenericData.get().getDefaultValue(field); record.put(field.name(), value); List<String> symbols = schema.getEnumSymbols(); return new GenericData.EnumSymbol (schema, symbols.get(random.nextInt(symbols.size()))); new GenericData.Array(length<=0?0:length, schema); for (int i = 0; i < length; i++) array.add(generate(schema.getElementType(), random, d+1)); return array; case MAP:
return; final Schema.Type type = schema.getType(); for (final Schema.Field field : schema.getFields()) { if (field.schema().getType() == Schema.Type.RECORD || (field.schema().getType() == Schema.Type.UNION && field.schema().getTypes().get(0).getType() == Schema.Type.NULL && field.schema().getTypes().get(1).getType() == Schema.Type.RECORD)) { _writer = (MapOrListWriterImpl) writer.map(field.name()); process(((GenericRecord) value).get(field.name()), field.schema(), field.name(), _writer, fieldSelection.getChild(field.name())); assert fieldName != null; final GenericArray<?> array = (GenericArray<?>) value; Schema elementSchema = array.getSchema().getElementType(); Type elementType = elementSchema.getType(); if (elementType == Schema.Type.RECORD || elementType == Schema.Type.MAP){
delta.put("testField1", testField1); GenericRecord itemRecord1 = new GenericData.Record(getSchemaByFullName( testField1.getSchema().getElementType().getTypes(), "org.kaa.config.testRecordItemT")); itemRecord1.put("testField2", 1); byte[] rawItemUuid1 = new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}; GenericFixed itemUuid1 = new GenericData.Fixed(itemRecord1 .getSchema().getField("__uuid").schema(), rawItemUuid1); itemRecord1.put("__uuid", itemUuid1); GenericRecord itemRecord2 = new GenericData.Record(getSchemaByFullName( testField1.getSchema().getElementType().getTypes(), "org.kaa.config.testRecordItemT")); itemRecord2.put("testField2", 2); byte[] rawItemUuid2 = new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}; GenericFixed itemUuid2 = new GenericData.Fixed(itemRecord2 .getSchema().getField("__uuid").schema(), rawItemUuid2); itemRecord2.put("__uuid", itemUuid2); GenericRecord itemRecord3 = new GenericData.Record(getSchemaByFullName( testField1.getSchema().getElementType().getTypes(), "org.kaa.config.testRecordItemT")); itemRecord3.put("testField2", 3); testField1.add(itemRecord1); testField1.add(itemRecord2); testField1.add(itemRecord3);
@Override public boolean delete(Collection<K> keys) throws Exception { if(keys == null) { throw new NullPointerException("keys"); } Schema schema = _protocol.getMessages().get(ProtocolConstants.MSG_MDEL).getRequest(); GenericRecord req = new GenericData.Record(schema); GenericArray<ByteBuffer> array = new GenericData.Array<ByteBuffer>(keys.size(), schema.getField("keys").schema()); for(K key : keys) { if(key != null) { array.add(serializeKey(key)); } } if(array.size() == 0) { return false; } req.put("src", _sourceUtf8); req.put("keys", array); return (Boolean)send(ProtocolConstants.MSG_MDEL, req); }
Schema avroSchema = schemaParser.parse(configuraionSchema.getRawSchema()); GenericRecord recordNew1 = new GenericData.Record(schemaParser.getTypes().get("org.kaaproject.recordT")); recordNew1.put("intField", 4); GenericFixed uuidNew1 = AvroUtils.generateUuidObject(); recordNew1.put(CommonConstants.UUID_FIELD, uuidNew1); GenericRecord rootNew = new GenericData.Record(avroSchema); GenericArray arrayNew = new GenericData.Array<>(2, rootNew.getSchema().getField("complexArrayField").schema()); arrayNew.add(recordNew1); arrayNew.add(recordNew2); rootNew.put("complexArrayField", arrayNew); rootNew.put(CommonConstants.UUID_FIELD, uuidNew2); GenericRecord rootOld = new GenericData.Record(avroSchema); GenericArray arrayOld = new GenericData.Array<>(2, rootOld.getSchema().getField("complexArrayField").schema()); arrayOld.add(recordOld1); arrayOld.add(recordOld2); rootOld.put("complexArrayField", arrayOld); rootOld.put(CommonConstants.UUID_FIELD, uuidNew2); GenericArray processedArray = (GenericArray) processedConfiguration.get("complexArrayField"); GenericRecord record1 = (GenericRecord) processedArray.get(0); GenericRecord record2 = (GenericRecord) processedArray.get(1); Assert.assertEquals(uuidNew1, record1.get(CommonConstants.UUID_FIELD)); Assert.assertNotEquals(uuidNew2, record2.get(CommonConstants.UUID_FIELD));
GenericRecord datum = new GenericData.Record(schema); datum.put("threadName", event.getThreadName()); datum.put("level", event.getLevel() == null ? Level.ERROR_INT : event.getLevel().toInt()); datum.put("message", event.getMessage()); GenericArray<String> argArray = new GenericData.Array<>(arguments.length, schema.getField("argumentArray").schema().getTypes().get(1)); for (Object argument : arguments) { argArray.add(argument == null ? null : argument.toString()); datum.put("argumentArray", argArray); datum.put("formattedMessage", event.getFormattedMessage()); datum.put("loggerName", event.getLoggerName()); datum.put("loggerContextVO", LoggerContextSerializer.encode(schema.getField("loggerContextVO").schema(), event.getLoggerContextVO())); datum.put("throwableProxy", ThrowableProxySerializer.encode(schema.getField("throwableProxy").schema(), event.getThrowableProxy())); if (event.hasCallerData()) {
switch (schema.getType()) { case STRING: return new Text(value.toString()); Writable[] values1 = new Writable[schema.getFields().size()]; int index1 = 0; for (Schema.Field field : schema.getFields()) { values1[index1++] = avroToArrayWritable(record.get(field.name()), field.schema()); case ARRAY: GenericArray arrayValue = (GenericArray) value; Writable[] values2 = new Writable[arrayValue.size()]; int index2 = 0; for (Object obj : arrayValue) {
Schema rootSchema = root.getSchema(); for (Field field : rootSchema.getFields()) { Object value = root.get(field.name()); if (value instanceof List) { List<Object> values = (List<Object>) value; Schema arraySchema = getArraySchema(delta, field.name()); GenericArray deltaArray = new GenericData.Array(values.size(), arraySchema); for (Object item : values) { addComplexItemToArray(record, deltaArray); } else { deltaArray.add(item); delta.put(field.name(), deltaArray); } else if (value instanceof GenericContainer) { processComplexField(delta, field.name(), (GenericContainer) value, null, null);
private static Object getAvroObject(Object javaObject, Schema schema) { if (javaObject == null) { return null; } Object avroObject; if (schema.getType() == STRING) { avroObject = new Utf8((String) javaObject); } else if (schema.getType() == ARRAY) { avroObject = new GenericData.Array(((Collection<Object>) javaObject).size(), schema); for (Object arrayElement : (Collection<Object>) javaObject) { ((GenericArray) avroObject).add(getAvroObject(arrayElement, schema.getElementType())); } } else if (schema.getType() == MAP) { avroObject = new LinkedHashMap<>(); for (Map.Entry<Object, Object> entry : ((Map<Object, Object>) javaObject).entrySet()) { ((Map<Object, Object>) avroObject).put(getAvroObject(entry.getKey(), Schema.create(STRING)), getAvroObject(entry.getValue(), schema.getValueType())); } } else if (schema.getType() == RECORD) { avroObject = new GenericData.Record(schema); for (int j = 0; j < schema.getFields().size(); j++) { ((GenericRecord) avroObject).put(j, getAvroObject(((Object[]) javaObject)[j], schema.getFields().get(j).schema())); } } else { avroObject = javaObject; } return avroObject; }
LOG.trace("Processing new record: {}, old record: {}", currentRecord, previousRecord); if (isRecordHaveUuid(currentRecord)) { GenericData.Fixed uuidValue = (GenericData.Fixed) currentRecord.get(UUID_FIELD); if (uuidValue == null) { if (previousRecord != null) { GenericRecord validatingRecord = findRecordByUuid(rootPreviousRecord, uuidValue); if (validatingRecord == null || !validatingRecord.getSchema().getFullName().equals( currentRecord.getSchema().getFullName())) { LOG.trace("Unknown UUID {}. Generating a new one", uuidValue); generateUuidForRecord(currentRecord); List<Schema.Field> fields = currentRecord.getSchema().getFields(); if (fields != null && !fields.isEmpty()) { for (Schema.Field field : fields) { int position = field.pos(); Object currentValue = currentRecord.get(position); Object previousValue = null; GenericArray array = (GenericArray) currentValue; if (array != null) { int size = array.size(); for (int i = 0; i < size; i++) { Object item = array.get(i); if (item instanceof GenericRecord) { GenericRecord itemRecord = (GenericRecord) item; array.set(i, validateRecord(itemRecord, null, rootPreviousRecord));
String newStr = new String(baosNew.toByteArray(), "UTF-8"); BaseData oldData = new BaseData(new BaseSchema(schema.toString()), oldStr); BaseData newData = new BaseData(new BaseSchema(schema.toString()), newStr); RawBinaryDelta deltaResult = calculator.calculate(oldData, newData); GenericRecord delta1 = new GenericData.Record(getDeltaSchemaByFullName(protocolSchema, "org.kaa.config.testT")); GenericEnumSymbol reset = new GenericData.EnumSymbol(getSchemaByFullName(delta1.getSchema().getField("testField1").schema().getTypes(), "org.kaaproject.configuration.resetT"), "reset"); byte[] rawUuid = new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; GenericFixed uuid = new GenericData.Fixed(delta1.getSchema().getField("__uuid").schema(), rawUuid); delta1.put("testField1", reset); delta1.put("__uuid", uuid); GenericRecord delta2 = new GenericData.Record(getDeltaSchemaByFullName(protocolSchema, "org.kaa.config.testT")); delta2.put("__uuid", uuid); GenericArray<Integer> testField1 = new GenericData.Array<Integer>(3, getArraySchema(delta2, "testField1")); testField1.add(321); testField1.add(456); testField1.add(654); delta2.put("testField1", testField1);
Schema rootRecordSchema = Schema.createRecord(Arrays.asList(arrayAField, recordAField)); GenericRecord mixedRecord = new GenericData.Record(rootRecordSchema); arrayA.add(9); arrayA.add(8); arrayA.add(2); arrayA.add(1); mixedRecord.put("arrayA", arrayA); GenericRecord recordA = new GenericData.Record(recordASchema); GenericArray<String> arrayB = new GenericData.Array<String>(5, arrayBSchema); arrayB.add("abc"); arrayB.add("jkl"); arrayB.add("ghi"); arrayB.add("def"); recordA.put("arrayB", arrayB); mixedRecord.put("recordA", recordA); GenericArray<Integer> newArrayA = (GenericArray<Integer>) mixedRecord.get("arrayA"); for (int i = 0; i < newArrayA.size() - 1; ++i) { Assert.assertTrue("Items are in bad order", newArrayA.get(i) < newArrayA.get(i + 1)); GenericArray<String> newArrayB = (GenericArray<String>) ((GenericRecord) mixedRecord.get("recordA")).get("arrayB"); for (int i = 0; i < newArrayB.size() - 1; ++i) { Assert.assertTrue("Items are in bad order", newArrayB.get(i).compareTo(newArrayB.get(i + 1)) < 0);
protected void addValueToArray(PredicateObjectMap pom, GenericRecord subject, Object object, String shortHandPredicateURI) { Object currentObj = subject.get(shortHandPredicateURI); GenericArray<GenericRecord> array = null; GenericArray<String> strings = null; array.add((GenericRecord) object); array = new GenericData.Array<>(subject.getSchema().getField(shortHandPredicateURI).schema().getTypes().get(0), new LinkedList<GenericRecord>()); array.add((GenericRecord)object); array.add((GenericRecord)currentObj); array = new GenericData.Array<>(objectToAdd.getSchema(), new LinkedList<GenericRecord>()); array.add(objectToAdd); strings.add((String) object); strings.add((String)object); strings.add((String)currentObj); strings.add(objectToAdd);
GenericRecord recordWithUuid = new GenericData.Record(recordSchema); byte[] uuid_value = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; GenericFixed uuid = (GenericFixed) GenericData.get().createFixed(null, uuid_value, uuidSchema); recordWithUuid.put("__uuid", uuid); Assert.assertNull("Uuid is still present after removal in recordWithoutUuid", recordWithUuid.get("__uuid")); Schema intFieldSchema = Schema.create(Type.INT); Schema inArrayRecordSchema = Schema.createRecord(Arrays.asList(uuidInArrayField, intField)); GenericRecord inArrayRecord1 = new GenericData.Record(inArrayRecordSchema); inArrayRecord1.put("__uuid", GenericData.get().createFixed(null, uuid_value, uuidSchema)); inArrayRecord1.put("int_value", new Integer(100)); GenericRecord inArrayRecord2 = new GenericData.Record(inArrayRecordSchema); inArrayRecord2.put("__uuid", GenericData.get().createFixed(null, uuid_value, uuidSchema)); inArrayRecord2.put("int_value", new Integer(200)); Schema.Field arrayField = new Schema.Field("array1", arraySchema, "", null); GenericArray<GenericRecord> array1 = new GenericData.Array<GenericRecord>(2, arraySchema); array1.add(inArrayRecord1); array1.add(inArrayRecord2); Schema.Field uuidOfComplexRecordField = new Schema.Field("__uuid", uuidSchema, "", null); Schema complexRecordSchemaWithUuid = Schema.createRecord(Arrays.asList(innerRecordField, arrayField, uuidOfComplexRecordField, otherFixed));
protected void collapseSameType(GenericRecord obj) { for (Field f : obj.getSchema().getFields()) { Object value = obj.get(f.name()); if(value == null) Set<Object> valuesHash = new HashSet<>(); boolean unmodified = true; for (int i = 0; i < array.size(); i++) { Object o = array.get(i); if (o instanceof GenericRecord) collapseSameType((GenericRecord) o); GenericArray<Object> newValues = new GenericData.Array<>(array.getSchema(), valuesHash); obj.put(f.name(), newValues);