private Type buildSchema() { JsonArray inputSchema = this.jsonSchema.getDataTypeValues(); List<Type> parquetTypes = new ArrayList<>(); for (JsonElement element : inputSchema) { JsonObject map = (JsonObject) element; JsonSchema elementSchema = new JsonSchema(map); String columnName = elementSchema.getColumnName(); JsonElementConverter converter = JsonElementConversionFactory.getConverter(elementSchema, false); Type schemaType = converter.schema(); this.converters.put(columnName, converter); parquetTypes.add(schemaType); } String docName = this.jsonSchema.getColumnName(); switch (recordType) { case ROOT: return new MessageType(docName, parquetTypes); case CHILD: return new GroupType(this.jsonSchema.optionalOrRequired(), docName, parquetTypes); default: throw new RuntimeException("Unsupported Record type"); } }
@Override JsonSchema getElementSchema() { JsonSchema jsonSchema = JsonSchema.buildBaseSchema(this.elementType); jsonSchema.setColumnName(ARRAY_KEY); return jsonSchema; } }
public JsonSchema(JsonObject jsonobject) { setJsonSchemaProperties(jsonobject); this.type = InputType.valueOf(getDataType().get(TYPE_KEY).getAsString().toUpperCase()); }
@Override JsonSchema getElementSchema() { JsonSchema jsonSchema = JsonSchema.buildBaseSchema(STRING); jsonSchema.setColumnName(this.jsonSchema.getColumnName()); return jsonSchema; } }
/** * Set properties for {@link JsonSchema} from a {@link JsonObject}. * @param jsonObject */ private void setJsonSchemaProperties(JsonObject jsonObject) { setColumnName(jsonObject.get(COLUMN_NAME_KEY).getAsString()); setDataType(jsonObject.get(DATA_TYPE_KEY).getAsJsonObject()); setNullable(jsonObject.has(IS_NULLABLE_KEY) && jsonObject.get(IS_NULLABLE_KEY).getAsBoolean()); setComment(getOptionalProperty(jsonObject, COMMENT_KEY)); setDefaultValue(getOptionalProperty(jsonObject, DEFAULT_VALUE_KEY)); }
protected Type buildSchema() { return new PrimitiveType(this.repeated ? REPEATED : this.jsonSchema.optionalOrRequired(), this.outputType, this.jsonSchema.getColumnName()); }
@Override public MessageType convertSchema(JsonArray inputSchema, WorkUnitState workUnit) throws SchemaConversionException { String fieldName = workUnit.getExtract().getTable(); JsonSchema jsonSchema = new JsonSchema(inputSchema); jsonSchema.setColumnName(fieldName); recordConverter = new RecordConverter(jsonSchema, ROOT); return (MessageType) recordConverter.schema(); }
/** * Convert value to a parquet type and perform null check. * @param value * @return Parquet safe type */ public Object convert(JsonElement value) { if (value.isJsonNull()) { if (this.jsonSchema.isNullable()) { return null; } throw new RuntimeException( "Field: " + this.jsonSchema.getColumnName() + " is not nullable and contains a null value"); } return convertField(value); }
/** * Get symbols for a {@link InputType#ENUM} type. * @return */ public JsonArray getSymbols() { if (this.type.equals(ENUM)) { return getDataType().get(ENUM_SYMBOLS_KEY).getAsJsonArray(); } return new JsonArray(); }
/** * Parquet {@link Repetition} for this {@link JsonSchema}. * @return */ public Repetition optionalOrRequired() { return this.isNullable() ? OPTIONAL : REQUIRED; }
public MapConverter(JsonSchema mapSchema) { super(mapSchema, mapSchema.getElementTypeUsingKey(MAP_ITEMS_KEY), false); }
public EnumConverter(JsonSchema enumSchema) { super(enumSchema, STRING, false); JsonArray symbolsArray = enumSchema.getSymbols(); symbolsArray.forEach(e -> symbols.add(e.getAsString())); }
/** * Builds a {@link JsonSchema} object for a given {@link InputType} object. * @param type * @return */ public static JsonSchema buildBaseSchema(InputType type) { JsonObject jsonObject = new JsonObject(); JsonObject dataType = new JsonObject(); jsonObject.addProperty(COLUMN_NAME_KEY, DEFAULT_RECORD_COLUMN_NAME); dataType.addProperty(TYPE_KEY, type.toString()); jsonObject.add(DATA_TYPE_KEY, dataType); return new JsonSchema(jsonObject); }
InputType fieldType = schema.getInputType(); switch (fieldType) { case INT:
/** * Set properties for {@link JsonSchema} from a {@link JsonObject}. * @param jsonObject */ private void setJsonSchemaProperties(JsonObject jsonObject) { setColumnName(jsonObject.get(COLUMN_NAME_KEY).getAsString()); setDataType(jsonObject.get(DATA_TYPE_KEY).getAsJsonObject()); setNullable(jsonObject.has(IS_NULLABLE_KEY) && jsonObject.get(IS_NULLABLE_KEY).getAsBoolean()); setComment(getOptionalProperty(jsonObject, COMMENT_KEY)); setDefaultValue(getOptionalProperty(jsonObject, DEFAULT_VALUE_KEY)); }
@Override protected Type buildSchema() { List<Type> fields = new ArrayList<>(); fields.add(0, this.elementConverter.schema()); return new GroupType(this.jsonSchema.optionalOrRequired(), this.jsonSchema.getColumnName(), fields); }
@Override JsonSchema getElementSchema() { JsonSchema jsonSchema = JsonSchema.buildBaseSchema(STRING); jsonSchema.setColumnName(this.jsonSchema.getColumnName()); return jsonSchema; } }
@Override public MessageType convertSchema(JsonArray inputSchema, WorkUnitState workUnit) throws SchemaConversionException { String fieldName = workUnit.getExtract().getTable(); JsonSchema jsonSchema = new JsonSchema(inputSchema); jsonSchema.setColumnName(fieldName); recordConverter = new RecordConverter(jsonSchema, ROOT); return (MessageType) recordConverter.schema(); }
/** * Convert value to a parquet type and perform null check. * @param value * @return Parquet safe type */ public Object convert(JsonElement value) { if (value.isJsonNull()) { if (this.jsonSchema.isNullable()) { return null; } throw new RuntimeException( "Field: " + this.jsonSchema.getColumnName() + " is not nullable and contains a null value"); } return convertField(value); }
/** * {@link InputType} of the elements composed within complex type. * @param itemKey * @return */ public InputType getElementTypeUsingKey(String itemKey) { String type = this.getDataType().get(itemKey).getAsString().toUpperCase(); return InputType.valueOf(type); }