Refine search
private static Schema parseAvroSchema(String previousSchemaDefinition) { Schema.Parser parser = new Schema.Parser(); return parser.parse(previousSchemaDefinition); }
private void readObject(ObjectInputStream ois) throws ClassNotFoundException, IOException { this.schema = new Schema.Parser().parse(ois.readUTF()); } }
public GobblinMetricsPinotFlattenerConverter() throws IOException { try (InputStream is = GobblinMetricsPinotFlattenerConverter.class.getClassLoader().getResourceAsStream("FlatGobblinMetric.avsc")) { this.schema = new Schema.Parser().parse(is); } }
public static Schema readSchemaFromJsonFile(String directory, String filename) throws IOException { return new Schema.Parser() .parse(ConversionHiveTestUtils.class.getClassLoader() .getResourceAsStream(StringUtils.removeEnd(directory, Path.SEPARATOR) + Path.SEPARATOR + filename)); }
@Override public Schema load(String literal) { Preconditions.checkNotNull(literal, "Schema literal cannot be null without a Schema URL"); return new Schema.Parser().parse(literal); } });
@Test public void testGenericRecord() throws IOException { final Path outputPath = new Path(File.createTempFile("avro-output-file", "generic.avro").getAbsolutePath()); final AvroOutputFormat<GenericRecord> outputFormat = new AvroOutputFormat<>(outputPath, GenericRecord.class); Schema schema = new Schema.Parser().parse("{\"type\":\"record\", \"name\":\"user\", \"fields\": [{\"name\":\"user_name\", \"type\":\"string\"}, {\"name\":\"favorite_number\", \"type\":\"int\"}, {\"name\":\"favorite_color\", \"type\":\"string\"}]}"); outputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE); outputFormat.setSchema(schema); output(outputFormat, schema); GenericDatumReader<GenericRecord> reader = new GenericDatumReader<>(schema); DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(new File(outputPath.getPath()), reader); while (dataFileReader.hasNext()) { GenericRecord record = dataFileReader.next(); assertEquals(record.get("user_name").toString(), "testUser"); assertEquals(record.get("favorite_number"), 1); assertEquals(record.get("favorite_color").toString(), "blue"); } //cleanup FileSystem fs = FileSystem.getLocalFileSystem(); fs.delete(outputPath, false); }
@SuppressWarnings("SameParameterValue") private static Schema parseAvroSchema(final String avroSchema) { final Schema.Parser parser = new Schema.Parser(); return parser.parse(avroSchema); }
@JsonCreator public InlineSchemasAvroBytesDecoder( @JacksonInject @Json ObjectMapper mapper, @JsonProperty("schemas") Map<String, Map<String, Object>> schemas ) throws Exception { Preconditions.checkArgument( schemas != null && schemas.size() > 0, "at least one schema must be provided in schemas attribute" ); this.schemas = schemas; schemaObjs = new HashMap<>(schemas.size()); for (Map.Entry<String, Map<String, Object>> e : schemas.entrySet()) { int id = Integer.parseInt(e.getKey()); Map<String, Object> schema = e.getValue(); String schemaStr = mapper.writeValueAsString(schema); LOGGER.debug("Schema string [%s] = [%s]", id, schemaStr); schemaObjs.put(id, new Schema.Parser().parse(schemaStr)); } }
public static org.apache.avro.Schema getAvroSchema(Schema schema) { ObjectNode avroSchema = JsonUtils.newObjectNode(); avroSchema.put("name", "data_gen_record"); avroSchema.put("type", "record"); ArrayNode fields = JsonUtils.newArrayNode(); for (FieldSpec fieldSpec : schema.getAllFieldSpecs()) { JsonNode jsonObject = fieldSpec.toAvroSchemaJsonObject(); fields.add(jsonObject); } avroSchema.set("fields", fields); return new org.apache.avro.Schema.Parser().parse(avroSchema.toString()); }
@Test public void testSchemaEvolutionAddingColumn() throws Exception { DecoderTestColumnHandle originalColumn = new DecoderTestColumnHandle(0, "row0", VARCHAR, "string_field", null, null, false, false, false); DecoderTestColumnHandle newlyAddedColumn = new DecoderTestColumnHandle(1, "row1", VARCHAR, "string_field_added", null, null, false, false, false); // the decoded avro data file does not have string_field_added byte[] originalData = buildAvroData(new Schema.Parser().parse( getAvroSchema("string_field", "\"string\"")), "string_field", "string_field_value"); String addedColumnSchema = getAvroSchema(ImmutableMap.of( "string_field", "\"string\"", "string_field_added", "[\"null\", \"string\"]")); Map<DecoderColumnHandle, FieldValueProvider> decodedRow = decodeRow( originalData, ImmutableSet.of(originalColumn, newlyAddedColumn), ImmutableMap.of(DATA_SCHEMA, addedColumnSchema)); assertEquals(decodedRow.size(), 2); checkValue(decodedRow, originalColumn, "string_field_value"); checkIsNull(decodedRow, newlyAddedColumn); }
public static Schema getSchemaFor(File file) { Schema.Parser parser = new Schema.Parser(); Schema schema; try { schema = parser.parse(file); } catch (IOException e) { throw new RuntimeException("Failed to parse Avro schema from " + file.getName(), e); } return schema; }
public static Schema getSchemaFor(InputStream stream) { Schema.Parser parser = new Schema.Parser(); Schema schema; try { schema = parser.parse(stream); } catch (IOException e) { throw new RuntimeException("Failed to parse Avro schema", e); } return schema; }
public static Schema getSchemaFor(String str) { Schema.Parser parser = new Schema.Parser(); Schema schema = parser.parse(str); return schema; }
/** * Creates an Avro serialization schema for the given Avro schema string. * * @param avroSchemaString Avro schema string used to serialize Flink's row to Avro's record */ public AvroRowSerializationSchema(String avroSchemaString) { Preconditions.checkNotNull(avroSchemaString, "Avro schema must not be null."); this.recordClazz = null; this.schemaString = avroSchemaString; try { this.schema = new Schema.Parser().parse(avroSchemaString); } catch (SchemaParseException e) { throw new IllegalArgumentException("Could not parse Avro schema string.", e); } this.datumWriter = new GenericDatumWriter<>(schema); this.arrayOutputStream = new ByteArrayOutputStream(); this.encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); }
/** * Creates a Avro deserialization schema for the given Avro schema string. * * @param avroSchemaString Avro schema string to deserialize Avro's record to Flink's row */ public AvroRowDeserializationSchema(String avroSchemaString) { Preconditions.checkNotNull(avroSchemaString, "Avro schema must not be null."); recordClazz = null; final TypeInformation<?> typeInfo = AvroSchemaConverter.convertToTypeInfo(avroSchemaString); Preconditions.checkArgument(typeInfo instanceof RowTypeInfo, "Row type information expected."); this.typeInfo = (RowTypeInfo) typeInfo; schemaString = avroSchemaString; schema = new Schema.Parser().parse(avroSchemaString); record = new GenericData.Record(schema); datumReader = new GenericDatumReader<>(schema); inputStream = new MutableByteArrayInputStream(); decoder = DecoderFactory.get().binaryDecoder(inputStream, null); }
@Test public void testSchemaEvolutionRemovingColumn() throws Exception { byte[] originalData = buildAvroData(new Schema.Parser().parse( getAvroSchema(ImmutableMap.of( "string_field", "\"string\"", "string_field_to_be_removed", "[\"null\", \"string\"]"))), ImmutableMap.of( "string_field", "string_field_value", "string_field_to_be_removed", "removed_field_value")); DecoderTestColumnHandle evolvedColumn = new DecoderTestColumnHandle(0, "row0", VARCHAR, "string_field", null, null, false, false, false); String removedColumnSchema = getAvroSchema("string_field", "\"string\""); Map<DecoderColumnHandle, FieldValueProvider> decodedEvolvedRow = decodeRow( originalData, ImmutableSet.of(evolvedColumn), ImmutableMap.of(DATA_SCHEMA, removedColumnSchema)); assertEquals(decodedEvolvedRow.size(), 1); checkValue(decodedEvolvedRow, evolvedColumn, "string_field_value"); }
private static Requirement createAvroTable(String schemaPath, String tableName, String topicName, ImmutableMap<String, Object> record) { try { Schema schema = new Schema.Parser().parse(new File(schemaPath)); byte[] avroData = convertRecordToAvro(schema, record); return immutableTable(new KafkaTableDefinition( tableName, topicName, new ListKafkaDataSource(ImmutableList.of( new KafkaMessage( contentsBuilder() .appendBytes(avroData) .build()))), 1, 1)); } catch (IOException e) { throw new UncheckedIOException(e); } }
/** * Converts an Avro schema string into a nested row structure with deterministic field order and data * types that are compatible with Flink's Table & SQL API. * * @param avroSchemaString Avro schema definition string * @return type information matching the schema */ @SuppressWarnings("unchecked") public static <T> TypeInformation<T> convertToTypeInfo(String avroSchemaString) { Preconditions.checkNotNull(avroSchemaString, "Avro schema must not be null."); final Schema schema; try { schema = new Schema.Parser().parse(avroSchemaString); } catch (SchemaParseException e) { throw new IllegalArgumentException("Could not parse Avro schema string.", e); } return (TypeInformation<T>) convertToTypeInfo(schema); }
protected GenericSchema(SchemaInfo schemaInfo) { this.schemaInfo = schemaInfo; this.schema = new org.apache.avro.Schema.Parser().parse( new String(schemaInfo.getSchema(), UTF_8) ); this.fields = schema.getFields() .stream() .map(f -> new Field(f.name(), f.pos())) .collect(Collectors.toList()); }