private void verifySchema(Map<String, String> properties) throws IOException { String schemaKey = StreamDataFileConstants.Property.Key.SCHEMA; String schemaStr = properties.get(schemaKey); if (schemaStr == null) { throw new IOException("Missing '" + schemaKey + "' property."); } try { Schema schema = new SchemaTypeAdapter().read(new JsonReader(new StringReader(schemaStr))); if (!StreamEventDataCodec.STREAM_DATA_SCHEMA.equals(schema)) { throw new IOException("Unsupported schema " + schemaStr); } } catch (JsonSyntaxException e) { throw new IOException("Invalid schema.", e); } }
/** * Parse the given JSON representation, as returned by {@link #toString()} into a Schema object. * * @param schemaJson the json representation of the schema * @return the json representation parsed into a schema object * @throws IOException if there was an exception parsing the schema */ public static Schema parseJson(String schemaJson) throws IOException { return SCHEMA_TYPE_ADAPTER.fromJson(schemaJson); }
@Override public void write(JsonWriter writer, Schema schema) throws IOException { if (schema == null) { writer.nullValue(); return; } Set<String> knownRecords = new HashSet<>(); write(writer, schema, knownRecords); }
public static GsonBuilder addTypeAdapters(GsonBuilder builder) { return builder .registerTypeAdapter(Schema.class, new SchemaTypeAdapter()) .registerTypeAdapter(ApplicationSpecification.class, new ApplicationSpecificationCodec()) .registerTypeAdapter(MapReduceSpecification.class, new MapReduceSpecificationCodec()) .registerTypeAdapter(SparkSpecification.class, new SparkSpecificationCodec()) .registerTypeAdapter(WorkflowSpecification.class, new WorkflowSpecificationCodec()) .registerTypeAdapter(WorkflowNode.class, new WorkflowNodeCodec()) .registerTypeAdapter(CustomActionSpecification.class, new CustomActionSpecificationCodec()) .registerTypeAdapter(ConditionSpecification.class, new ConditionSpecificationCodec()) .registerTypeAdapter(ServiceSpecification.class, new ServiceSpecificationCodec()) .registerTypeAdapter(WorkerSpecification.class, new WorkerSpecificationCodec()) .registerTypeAdapter(BasicThrowable.class, new BasicThrowableCodec()) .registerTypeAdapter(Trigger.class, new TriggerCodec()) .registerTypeAdapter(SatisfiableTrigger.class, new TriggerCodec()) .registerTypeAdapter(Constraint.class, new ConstraintCodec()) .registerTypeAdapterFactory(new AppSpecTypeAdapterFactory()); }
enumValues = readEnum(reader); break; case ITEMS: items = read(reader, knownRecords); break; case KEYS: keys = read(reader, knownRecords); break; case VALUES: values = read(reader, knownRecords); break; case FIELDS: fields = getFields(name, reader, knownRecords); knownRecords.put(elementName, Schema.recordOf(elementName, fields)); break;
@Override public Schema read(JsonReader reader) throws IOException { return read(reader, new HashMap<String, Schema>()); }
return readUnion(reader, knownRecords); case BEGIN_OBJECT: return readObject(reader, knownRecords);
public static GsonBuilder addTypeAdapters(GsonBuilder builder) { return builder .registerTypeAdapter(Schema.class, new SchemaTypeAdapter()) .registerTypeAdapter(ApplicationSpecification.class, new ApplicationSpecificationCodec()) .registerTypeAdapter(FlowSpecification.class, new FlowSpecificationCodec()) .registerTypeAdapter(FlowletSpecification.class, new FlowletSpecificationCodec()) .registerTypeAdapter(MapReduceSpecification.class, new MapReduceSpecificationCodec()) .registerTypeAdapter(SparkSpecification.class, new SparkSpecificationCodec()) .registerTypeAdapter(WorkflowSpecification.class, new WorkflowSpecificationCodec()) .registerTypeAdapter(WorkflowNode.class, new WorkflowNodeCodec()) .registerTypeAdapter(CustomActionSpecification.class, new CustomActionSpecificationCodec()) .registerTypeAdapter(ConditionSpecification.class, new ConditionSpecificationCodec()) .registerTypeAdapter(ServiceSpecification.class, new ServiceSpecificationCodec()) .registerTypeAdapter(WorkerSpecification.class, new WorkerSpecificationCodec()) .registerTypeAdapter(BasicThrowable.class, new BasicThrowableCodec()) .registerTypeAdapter(Trigger.class, new TriggerCodec()) .registerTypeAdapter(SatisfiableTrigger.class, new TriggerCodec()) .registerTypeAdapter(Constraint.class, new ConstraintCodec()) .registerTypeAdapterFactory(new AppSpecTypeAdapterFactory()); }
enumValues = readEnum(reader); break; case ITEMS: items = read(reader, knownRecords); break; case KEYS: keys = read(reader, knownRecords); break; case VALUES: values = read(reader, knownRecords); break; case FIELDS: fields = getFields(name, reader, knownRecords); knownRecords.put(elementName, Schema.recordOf(elementName, fields)); break;
@Override public Schema read(JsonReader reader) throws IOException { return read(reader, new HashMap<String, Schema>()); }
return readUnion(reader, knownRecords); case BEGIN_OBJECT: return readObject(reader, knownRecords);
@Override public Map<String, List<JsonElement>> get(ApplicationId applicationId, String tracerName) { // PreviewStore is a singleton and we have to create gson for each operation since gson is not thread safe. Gson gson = new GsonBuilder().registerTypeAdapter(Schema.class, new SchemaTypeAdapter()).create(); byte[] startRowKey = new MDSKey.Builder().add(applicationId.getNamespace()) .add(applicationId.getApplication()).add(tracerName).build().getKey(); byte[] stopRowKey = new MDSKey(Bytes.stopKeyForPrefix(startRowKey)).getKey(); Map<String, List<JsonElement>> result = new HashMap<>(); try (Scanner scanner = table.scan(startRowKey, stopRowKey, null, null, null)) { Row indexRow; while ((indexRow = scanner.next()) != null) { Map<byte[], byte[]> columns = indexRow.getColumns(); String propertyName = Bytes.toString(columns.get(PROPERTY)); JsonElement value = gson.fromJson(Bytes.toString(columns.get(VALUE)), JsonElement.class); List<JsonElement> values = result.get(propertyName); if (values == null) { values = new ArrayList<>(); result.put(propertyName, values); } values.add(value); } } catch (IOException e) { String message = String.format("Error while reading preview data for application '%s' and tracer '%s'.", applicationId, tracerName); throw new RuntimeException(message, e); } return result; }
/** * Constructs {@link Schema.Type#UNION UNION} type schema from the json input. * * @param reader The {@link JsonReader} for streaming json input tokens. * @param knownRecords Map of record names and associated schema already encountered during the reading * @return A {@link Schema} of type {@link Schema.Type#UNION UNION}. * @throws IOException When fails to construct a valid schema from the input. */ private Schema readUnion(JsonReader reader, Map<String, Schema> knownRecords) throws IOException { List<Schema> unionSchemas = new ArrayList<>(); reader.beginArray(); while (reader.peek() != JsonToken.END_ARRAY) { unionSchemas.add(read(reader, knownRecords)); } reader.endArray(); return Schema.unionOf(unionSchemas); }
@Override public void write(JsonWriter writer, Schema schema) throws IOException { if (schema == null) { writer.nullValue(); return; } Set<String> knownRecords = new HashSet<>(); write(writer, schema, knownRecords); }
/** * Parse the given json representation of a schema object contained in a reader into a schema object. * * @param reader the reader for reading the json representation of a schema * @return the parsed schema object * @throws IOException if there was an exception parsing the schema */ public static Schema parseJson(Reader reader) throws IOException { return SCHEMA_TYPE_ADAPTER.fromJson(reader); }
@Override public Map<String, List<JsonElement>> get(ApplicationId applicationId, String tracerName) { // PreviewStore is a singleton and we have to create gson for each operation since gson is not thread safe. Gson gson = new GsonBuilder().registerTypeAdapter(Schema.class, new SchemaTypeAdapter()).create(); byte[] startRowKey = new MDSKey.Builder().add(applicationId.getNamespace()) .add(applicationId.getApplication()).add(tracerName).build().getKey(); byte[] stopRowKey = new MDSKey(Bytes.stopKeyForPrefix(startRowKey)).getKey(); Map<String, List<JsonElement>> result = new HashMap<>(); try (Scanner scanner = table.scan(startRowKey, stopRowKey, null, null, null)) { Row indexRow; while ((indexRow = scanner.next()) != null) { Map<byte[], byte[]> columns = indexRow.getColumns(); String propertyName = Bytes.toString(columns.get(PROPERTY)); JsonElement value = gson.fromJson(Bytes.toString(columns.get(VALUE)), JsonElement.class); List<JsonElement> values = result.get(propertyName); if (values == null) { values = new ArrayList<>(); result.put(propertyName, values); } values.add(value); } } catch (IOException e) { String message = String.format("Error while reading preview data for application '%s' and tracer '%s'.", applicationId, tracerName); throw new RuntimeException(message, e); } return result; }
/** * Constructs {@link Schema.Type#UNION UNION} type schema from the json input. * * @param reader The {@link JsonReader} for streaming json input tokens. * @param knownRecords Map of record names and associated schema already encountered during the reading * @return A {@link Schema} of type {@link Schema.Type#UNION UNION}. * @throws IOException When fails to construct a valid schema from the input. */ private Schema readUnion(JsonReader reader, Map<String, Schema> knownRecords) throws IOException { List<Schema> unionSchemas = new ArrayList<>(); reader.beginArray(); while (reader.peek() != JsonToken.END_ARRAY) { unionSchemas.add(read(reader, knownRecords)); } reader.endArray(); return Schema.unionOf(unionSchemas); }
/** * Helper method to encode this schema into json string. * * @return A json string representing this schema. */ private String buildString() { // Return type only if this type does not represent logical type, otherwise use jsonwriter to convert logical type // to correct schema. if (type.isSimpleType() && logicalType == null) { return '"' + type.name().toLowerCase() + '"'; } StringWriter writer = new StringWriter(); try (JsonWriter jsonWriter = new JsonWriter(writer)) { SCHEMA_TYPE_ADAPTER.write(jsonWriter, this); } catch (IOException e) { // It should never throw IOException on the StringWriter, if it does, something is very wrong. throw new RuntimeException(e); } return writer.toString(); }
/** * Parse the given JSON representation, as returned by {@link #toString()} into a Schema object. * * @param schemaJson the json representation of the schema * @return the json representation parsed into a schema object * @throws IOException if there was an exception parsing the schema */ public static Schema parseJson(String schemaJson) throws IOException { return SCHEMA_TYPE_ADAPTER.fromJson(schemaJson); }
@Test public void testGenerateSchema() throws UnsupportedTypeException { Schema schema = (new ReflectionSchemaGenerator()).generate((new TypeToken<Child<Node>>() { }).getType()); Gson gson = new GsonBuilder() .registerTypeAdapter(Schema.class, new SchemaTypeAdapter()) .create(); Assert.assertEquals(schema, gson.fromJson(gson.toJson(schema), Schema.class)); }