Refine search
@Override public Schema load(String url) throws IOException { Schema.Parser parser = new Schema.Parser(); InputStream is = null; try { FileSystem fs = FileSystem.get(URI.create(url), conf); if (url.toLowerCase(Locale.ENGLISH).startsWith("hdfs:/")) { is = fs.open(new Path(url)); } else { is = new URL(url).openStream(); } return parser.parse(is); } finally { if (is != null) { is.close(); } } } });
@Override public Schema copy() throws CopyNotSupportedException { return new Schema.Parser().setValidate(false).parse(this.schema.toString()); } }
final public Schema ImportSchema() throws ParseException { String importFile; jj_consume_token(SCHEMA); importFile = JsonString(); jj_consume_token(SEMICOLON); try { Parser parser = new Schema.Parser(); parser.addTypes(names); // inherit names InputStream stream = findFile(importFile).openStream(); try { Schema value = parser.parse(stream); names = parser.getTypes(); // update names {if (true) return value;} } finally { stream.close(); } } catch (IOException e) { {if (true) throw error("Error importing "+importFile+": "+e, token);} } throw new Error("Missing return statement in function"); }
private void updateRecordFromTestResource(String resourceName, String avroFileName) throws IOException { if (avroFileName == null) { avroFileName = resourceName + ".avro"; } recordSchema = new Schema.Parser().parse( getClass().getClassLoader().getResourceAsStream(resourceName + ".avsc") ); DatumReader<GenericRecord> reader = new GenericDatumReader<>(recordSchema); DataFileReader<GenericRecord> dataFileReader = new DataFileReader<GenericRecord>( new File(getClass().getClassLoader().getResource(avroFileName).getPath()), reader); Assert.assertTrue(dataFileReader.hasNext()); record = dataFileReader.next(record); accessor = new AvroGenericRecordAccessor(record); }
@Test public void testValidationOfComplexWithoutOldConfiguration() throws Exception { // Read Configuration Schema Path schemaPath = Paths.get(Thread.currentThread().getContextClassLoader().getResource("generation/simpleComplexSchema.json").toURI()); BaseSchema configuraionSchema = new BaseSchema(new String(Files.readAllBytes(schemaPath))); Schema.Parser schemaParser = new Schema.Parser(); Schema avroSchema = schemaParser.parse(configuraionSchema.getRawSchema()); GenericRecord recordNew1 = new GenericData.Record(schemaParser.getTypes().get("org.kaaproject.recordT")); recordNew1.put("intField", 4); GenericFixed uuidNew1 = AvroUtils.generateUuidObject(); recordNew1.put(CommonConstants.UUID_FIELD, uuidNew1); GenericRecord rootNew = new GenericData.Record(avroSchema); rootNew.put("recordField", recordNew1); GenericAvroConverter<GenericRecord> converter = new GenericAvroConverter<>(avroSchema); String configurationBodyNew = converter.encodeToJson(rootNew); DefaultUuidValidator uuidGenerator = new DefaultUuidValidator(configuraionSchema, new BaseDataFactory()); KaaData processedConfigurationBody = uuidGenerator.validateUuidFields(rootNew, null); GenericRecord processedConfiguration = converter.decodeJson(processedConfigurationBody.getRawData()); GenericRecord processedRecord = (GenericRecord) processedConfiguration.get("recordField"); Assert.assertNotEquals(uuidNew1, processedRecord.get(CommonConstants.UUID_FIELD)); Assert.assertNotNull(processedConfiguration.get(CommonConstants.UUID_FIELD)); }
@Test public void testValidationOfIdenticalUuids() throws Exception { Path schemaPath = Paths.get(Thread.currentThread().getContextClassLoader().getResource("generation/arraySchema.json").toURI()); BaseSchema configuraionSchema = new BaseSchema(new String(Files.readAllBytes(schemaPath))); Schema.Parser schemaParser = new Schema.Parser(); Schema avroSchema = schemaParser.parse(configuraionSchema.getRawSchema()); GenericRecord recordNew1 = new GenericData.Record(schemaParser.getTypes().get("org.kaaproject.recordT")); recordNew1.put("intField", 4); GenericFixed uuidNew1 = AvroUtils.generateUuidObject(); recordNew1.put(CommonConstants.UUID_FIELD, uuidNew1); GenericRecord recordNew2 = new GenericData.Record(schemaParser.getTypes().get("org.kaaproject.recordT")); recordNew2.put("intField", 5); recordNew2.put(CommonConstants.UUID_FIELD, uuidNew1); GenericRecord recordOld1 = new GenericData.Record(schemaParser.getTypes().get("org.kaaproject.recordT")); recordOld1.put("intField", 6); recordOld1.put(CommonConstants.UUID_FIELD, uuidNew1); GenericRecord recordOld2 = new GenericData.Record(schemaParser.getTypes().get("org.kaaproject.recordT")); recordOld2.put("intField", 7); GenericFixed uuidOld2 = AvroUtils.generateUuidObject(); GenericRecord rootOld = new GenericData.Record(avroSchema); GenericArray arrayOld = new GenericData.Array<>(2, rootOld.getSchema().getField("complexArrayField").schema()); arrayOld.add(recordOld1); arrayOld.add(recordOld2);
@Test public void testPrimitiveFieldsDelta() throws IOException, DeltaCalculatorException { URL protocolSchemaUrl = Thread.currentThread().getContextClassLoader().getResource("delta/primitiveFieldsDeltaProtocolSchema.json"); Schema protocolSchema = new Schema.Parser().parse(new File(protocolSchemaUrl.getPath())); URL schemaUrl = Thread.currentThread().getContextClassLoader().getResource("delta/primitiveFieldsDeltaSchema.json"); Schema schema = new Schema.Parser().parse(new File(schemaUrl.getPath())); DeltaCalculationAlgorithm calculator = new DefaultDeltaCalculationAlgorithm(protocolSchema, schema); ByteArrayOutputStream baosOld = new ByteArrayOutputStream(); URL oldConfigUrl = Thread.currentThread().getContextClassLoader().getResource("delta/primitiveFieldsDeltaCurrent.json"); IOUtils.copy(new FileInputStream(oldConfigUrl.getPath()), baosOld, 1024); String oldStr = new String(baosOld.toByteArray(), "UTF-8"); ByteArrayOutputStream baosNew = new ByteArrayOutputStream(); URL newConfigUrl = Thread.currentThread().getContextClassLoader().getResource("delta/primitiveFieldsDeltaNew.json"); IOUtils.copy(new FileInputStream(newConfigUrl.getPath()), baosNew, 1024); String newStr = new String(baosNew.toByteArray(), "UTF-8"); BaseData oldData = new BaseData(new BaseSchema(schema.toString()), oldStr); BaseData newData = new BaseData(new BaseSchema(schema.toString()), newStr); RawBinaryDelta deltaResult = calculator.calculate(oldData, newData); GenericRecord delta = new GenericData.Record(getDeltaSchemaByFullName(protocolSchema, "org.kaa.config.testT")); GenericEnumSymbol unchanged = new GenericData.EnumSymbol(getSchemaByFullName(delta.getSchema().getField("testField1").schema().getTypes(), "org.kaaproject.configuration.unchangedT"), "unchanged"); byte[] rawUuid = new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; GenericFixed uuid = new GenericData.Fixed(delta.getSchema().getField("__uuid").schema(), rawUuid); delta.put("testField1", unchanged); delta.put("testField2", 456); delta.put("__uuid", uuid); AvroBinaryDelta deltaExpected = new AvroBinaryDelta(protocolSchema); deltaExpected.addDelta(delta); assertArrayEquals(deltaExpected.getData(), deltaResult.getData()); }
@Test public void testIdenticalConfigurations() throws IOException, URISyntaxException, DeltaCalculatorException { URL protocolSchemaUrl = Thread.currentThread().getContextClassLoader().getResource(COMPLEX_PROTOCOL_SCHEMA); URL schemaUrl = Thread.currentThread().getContextClassLoader().getResource(COMPLEX_SCHEMA); Schema protocolSchema = new Schema.Parser().parse(new File(protocolSchemaUrl.getPath())); Schema schema = new Schema.Parser().parse(new File(schemaUrl.getPath())); DeltaCalculationAlgorithm calculator = new DefaultDeltaCalculationAlgorithm(protocolSchema, schema); ByteArrayOutputStream baosOld = new ByteArrayOutputStream(); URL oldConfigUrl = Thread.currentThread().getContextClassLoader().getResource(OLD_COMPLEX_CONFIG); IOUtils.copy(new FileInputStream(oldConfigUrl.getPath()), baosOld, 1024); String oldStr = new String(baosOld.toByteArray(), "UTF-8"); ByteArrayOutputStream baosNew = new ByteArrayOutputStream(); URL newConfigUrl = Thread.currentThread().getContextClassLoader().getResource(OLD_COMPLEX_CONFIG); IOUtils.copy(new FileInputStream(newConfigUrl.getPath()), baosNew, 1024); String newStr = new String(baosNew.toByteArray(), "UTF-8"); BaseData oldData = new BaseData(new BaseSchema(schema.toString()), oldStr); BaseData newData = new BaseData(new BaseSchema(schema.toString()), newStr); RawBinaryDelta deltaResult = calculator.calculate(oldData, newData); assertFalse(deltaResult.hasChanges()); assertNull(deltaResult.getData()); assertFalse(deltaResult.hasChanges()); } }
@Test(expected = OverrideException.class) public void testResolveFailsWhenSchemaForParentIsNotFound() throws Exception { Path schemaUrl = Paths.get(Thread.currentThread().getContextClassLoader().getResource("override/schema.json").toURI()); DataSchema configurationSchema = new DataSchema(new String(Files.readAllBytes(schemaUrl))); SchemaGenerationAlgorithmFactory factory = new SchemaGenerationAlgorithmFactoryImpl(); SchemaGenerationAlgorithm generator = factory.createSchemaGenerator(configurationSchema); KaaSchema baseSchemaString = generator.getBaseSchema(); Schema.Parser baseParser = new Schema.Parser(); baseParser.parse(baseSchemaString.getRawSchema()); ArrayOverrideStrategyResolver arrayMergeStrategyResolver = new ArrayOverrideStrategyResolver(baseParser.getTypes()); arrayMergeStrategyResolver.resolve("wrong_parent_name", "org.kaa.config", "child_name"); }
@Test public void testMapWithNonStringKeyToStringIsJson() throws Exception { Schema intMapSchema = new Schema.Parser().parse("{\"type\": \"map\", \"values\": \"string\", \"java-key-class\" : \"java.lang.Integer\"}"); Field intMapField = new Field("intMap", Schema.createMap(intMapSchema), null, null); Schema decMapSchema = new Schema.Parser().parse("{\"type\": \"map\", \"values\": \"string\", \"java-key-class\" : \"java.math.BigDecimal\"}"); Field decMapField = new Field("decMap", Schema.createMap(decMapSchema), null, null); Schema boolMapSchema = new Schema.Parser().parse("{\"type\": \"map\", \"values\": \"string\", \"java-key-class\" : \"java.lang.Boolean\"}"); Field boolMapField = new Field("boolMap", Schema.createMap(boolMapSchema), null, null); Schema fileMapSchema = new Schema.Parser().parse("{\"type\": \"map\", \"values\": \"string\", \"java-key-class\" : \"java.io.File\"}"); Field fileMapField = new Field("fileMap", Schema.createMap(fileMapSchema), null, null); Schema schema = Schema.createRecord("my_record", "doc", "mytest", false); java.io.File f = new java.io.File( getClass().getResource("/SchemaBuilder.avsc").toURI() ); filePair.put(f, "File"); GenericRecord r = new GenericData.Record(schema); r.put(intMapField.name(), intPair); r.put(decMapField.name(), decPair); r.put(boolMapField.name(), boolPair); r.put(fileMapField.name(), filePair);
@Test public void testFieldsPickWithNestedRecord() throws Exception { Schema inputSchema = new Schema.Parser().parse(getClass().getResourceAsStream("/converter/pickfields_nested_with_union.avsc")); WorkUnitState workUnitState = new WorkUnitState(); workUnitState.setProp(ConfigurationKeys.CONVERTER_AVRO_FIELD_PICK_FIELDS, "name,favorite_number,nested1.nested1_string,nested1.nested2_union.nested2_string"); try (AvroFieldsPickConverter converter = new AvroFieldsPickConverter()) { Schema convertedSchema = converter.convertSchema(inputSchema, workUnitState); Schema expectedSchema = new Schema.Parser().parse(getClass().getResourceAsStream("/converter/converted_pickfields_nested_with_union.avsc")); JSONAssert.assertEquals(expectedSchema.toString(), convertedSchema.toString(), false); try (DataFileReader<GenericRecord> srcDataFileReader = new DataFileReader<GenericRecord>( new File(getClass().getResource("/converter/pickfields_nested_with_union.avro").toURI()), new GenericDatumReader<GenericRecord>(inputSchema)); DataFileReader<GenericRecord> expectedDataFileReader = new DataFileReader<GenericRecord>( new File(getClass().getResource("/converter/converted_pickfields_nested_with_union.avro").toURI()), new GenericDatumReader<GenericRecord>(expectedSchema));) { while (expectedDataFileReader.hasNext()) { GenericRecord expected = expectedDataFileReader.next(); GenericRecord actual = converter.convertRecord(convertedSchema, srcDataFileReader.next(), workUnitState).iterator().next(); Assert.assertEquals(actual, expected); } Assert.assertTrue(!srcDataFileReader.hasNext()); } } } }
/** * {@inheritDoc} */ @Override public RecordReader<NullWritable, GenericRecord> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException { Schema readerSchema = AvroJob.getInputValueSchema(context.getConfiguration()); if (readerSchema == null) { String schemaFilePath = context.getConfiguration().get(CONF_INPUT_VALUE_SCHEMA_PATH); if (StringUtils.isNotBlank(schemaFilePath)) { log.info("Using file: %s as reader schema.", schemaFilePath); try (FSDataInputStream inputStream = FileSystem.get(context.getConfiguration()).open(new Path(schemaFilePath))) { readerSchema = new Schema.Parser().parse(inputStream); } } } if (null == readerSchema) { log.warn("Reader schema was not set. Use AvroJob.setInputKeySchema() if desired."); log.info("Using a reader schema equal to the writer schema."); } return new AvroValueRecordReader(readerSchema); } }
public static Schema getAvroSchemaFromFileSystem(String schemaURL, Configuration conf) throws IOException { Path schemaPath = new Path(schemaURL); FileSystem fs = schemaPath.getFileSystem(conf); FSDataInputStream inputStream = fs.open(schemaPath); try { return new Schema.Parser().parse(inputStream); } finally { IOUtils.closeStream(inputStream); } } }
@Test public void testSerializeAsPath() throws Exception { Schema schema = new Schema.Parser().parse("{\"type\":\"record\", \"name\":\"test\", " + "\"fields\":[" + "{\"name\": \"name\", \"type\": \"string\"}, " + "{\"name\": \"title\", \"type\": \"string\"}" + "]}"); GenericRecord partition = new GenericData.Record(schema); partition.put("name", "a/b:c\\d e"); partition.put("title", "title"); Assert.assertEquals(AvroUtils.serializeAsPath(partition, true, true), new Path("name=a_b_c_d_e/title=title")); Assert.assertEquals(AvroUtils.serializeAsPath(partition, false, true), new Path("a_b_c_d_e/title")); Assert.assertEquals(AvroUtils.serializeAsPath(partition, false, false), new Path("a/b_c_d_e/title")); }
@Test public void testConverter() throws IOException, DataConversionException, SchemaRegistryException { Schema inputSchema = new Schema.Parser().parse(getClass().getResourceAsStream("/converter/envelope.avsc")); GenericDatumReader<GenericRecord> datumReader = new GenericDatumReader<>(inputSchema); GenericRecord inputRecord = dataFileReader.next(); Schema latestPayloadSchema = new Schema.Parser().parse(getClass().getResourceAsStream("/converter/record.avsc")); when(mockRegistry.getLatestSchemaByTopic(any())).thenReturn(latestPayloadSchema); when(mockRegistry.getSchemaByKey(any())).thenReturn(inputSchema.getField("nestedRecord").schema()); GenericRecord payload = (GenericRecord) outputRecord.get("payload"); GenericRecord expectedPayload = (GenericRecord) outputRecord.get("nestedRecord"); Schema payloadSchema = payload.getSchema(); Schema expectedPayloadSchema = expectedPayload.getSchema(); Assert.assertTrue(expectedPayloadSchema.getName().equals(payloadSchema.getName())); Assert.assertTrue(expectedPayloadSchema.getNamespace().equals(payloadSchema.getNamespace())); Assert.assertTrue(expectedPayloadSchema.getFields().size() == payloadSchema.getFields().size());
@Test public void testSelfReferenceInUnion(){ Schema schema = new Schema.Parser().parse(SIMPLE_BINARY_TREE); Field leftField = schema.getField("left"); assertEquals(JsonProperties.NULL_VALUE,leftField.defaultVal()); final Schema leftFieldSchema = leftField.schema(); assertEquals(Type.UNION,leftFieldSchema.getType()); assertEquals("null",leftFieldSchema.getTypes().get(0).getName()); assertEquals("Node",leftFieldSchema.getTypes().get(1).getName()); Field rightField = schema.getField("right"); assertEquals(JsonProperties.NULL_VALUE,rightField.defaultVal()); final Schema rightFieldSchema = rightField.schema(); assertEquals(Type.UNION,rightFieldSchema.getType()); assertEquals("null",rightFieldSchema.getTypes().get(0).getName()); assertEquals("Node",rightFieldSchema.getTypes().get(1).getName()); }
@Test(dependsOnMethods = "testPrepareTask") public void testRun() throws IOException { TaskResult taskResult = this.gobblinHelixTask.run(); System.out.println(taskResult.getInfo()); Assert.assertEquals(taskResult.getStatus(), TaskResult.Status.COMPLETED); File outputAvroFile = new File(this.taskOutputDir.toString(), TestHelper.REL_WRITER_FILE_PATH + File.separator + TestHelper.WRITER_FILE_NAME); Assert.assertTrue(outputAvroFile.exists()); Schema schema = new Schema.Parser().parse(TestHelper.SOURCE_SCHEMA); TestHelper.assertGenericRecords(outputAvroFile, schema); }
"\"logicalType\":\"decimal\",\"precision\":4,\"scale\":2}},{\"name\":\"type_decimal_fixed\",\"type\":{\"type\":\"fixed\"," + "\"name\":\"Fixed2\",\"size\":2,\"logicalType\":\"decimal\",\"precision\":4,\"scale\":2}}]}"; final Schema schema = new Schema.Parser().parse(schemaString); GenericRecord addr = new GenericData.Record(schema.getField("type_nested").schema().getTypes().get(1)); addr.put("num", 42); addr.put("street", "Main Street 42"); addr.put("city", "Test City"); addr.put("state", "Test State"); addr.put("zip", "12345"); rowAddr.setField(4, "12345"); final GenericRecord user = new GenericData.Record(schema); user.put("name", "Charlie"); user.put("favorite_number", null); user.put("type_array_boolean", Arrays.asList(true, true, false)); user.put("type_nullable_array", null); user.put("type_enum", new GenericData.EnumSymbol(schema.getField("type_enum").schema(), "RED")); user.put("type_map", Collections.singletonMap("test", 12L)); user.put("type_fixed", new Fixed16(new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}));
@Test public void testCopy() throws CopyNotSupportedException { GenericRecord record = new GenericData.Record(new Schema.Parser().parse(AVRO_SCHEMA)); record.put("name", "foo"); record.put("favorite_number", 68); record.put("favorite_colors", Arrays.asList("blue", "black", "red")); CopyableGenericRecord copyableGenericRecord = new CopyableGenericRecord(record); GenericRecord copy = copyableGenericRecord.copy(); Assert.assertEquals(record, copy); copy.put("name", "bar"); Assert.assertNotEquals(record, copy); } }
@Test public void testSerialization() throws DataConversionException, IOException, SchemaConversionException { Schema inputSchema = new Schema.Parser() .parse(getClass().getClassLoader().getResourceAsStream("converter/bytes_to_avro/test_record_schema.avsc")); GenericRecord testRecord = new GenericData.Record(inputSchema); testRecord.put("testStr", "testing12" + ((i == 0) ? "3": "4")); testRecord.put("testInt", -2); Assert.assertEquals(outputSchema, inputSchema.toString()); Assert.assertEquals(record, expectedRecord);