@Override Object convertField(JsonElement value) { ParquetGroup r1 = new ParquetGroup((GroupType) schema()); JsonObject inputRecord = value.getAsJsonObject(); for (Map.Entry<String, JsonElement> entry : inputRecord.entrySet()) { String key = entry.getKey(); JsonElementConverter converter = this.converters.get(key); Object convertedValue = converter.convert(entry.getValue()); boolean valueIsNull = convertedValue == null; Type.Repetition repetition = converter.jsonSchema.optionalOrRequired(); if (valueIsNull && repetition.equals(OPTIONAL)) { continue; } r1.add(key, convertedValue); } return r1; }
/** * Convert value to a parquet type and perform null check. * @param value * @return Parquet safe type */ public Object convert(JsonElement value) { if (value.isJsonNull()) { if (this.jsonSchema.isNullable()) { return null; } throw new RuntimeException( "Field: " + this.jsonSchema.getColumnName() + " is not nullable and contains a null value"); } return convertField(value); }
private Type buildSchema() { JsonArray inputSchema = this.jsonSchema.getDataTypeValues(); List<Type> parquetTypes = new ArrayList<>(); for (JsonElement element : inputSchema) { JsonObject map = (JsonObject) element; JsonSchema elementSchema = new JsonSchema(map); String columnName = elementSchema.getColumnName(); JsonElementConverter converter = JsonElementConversionFactory.getConverter(elementSchema, false); Type schemaType = converter.schema(); this.converters.put(columnName, converter); parquetTypes.add(schemaType); } String docName = this.jsonSchema.getColumnName(); switch (recordType) { case ROOT: return new MessageType(docName, parquetTypes); case CHILD: return new GroupType(this.jsonSchema.optionalOrRequired(), docName, parquetTypes); default: throw new RuntimeException("Unsupported Record type"); } }
private Type buildSchema() { JsonArray inputSchema = this.jsonSchema.getDataTypeValues(); List<Type> parquetTypes = new ArrayList<>(); for (JsonElement element : inputSchema) { JsonObject map = (JsonObject) element; JsonSchema elementSchema = new JsonSchema(map); String columnName = elementSchema.getColumnName(); JsonElementConverter converter = JsonElementConversionFactory.getConverter(elementSchema, false); Type schemaType = converter.schema(); this.converters.put(columnName, converter); parquetTypes.add(schemaType); } String docName = this.jsonSchema.getColumnName(); switch (recordType) { case ROOT: return new MessageType(docName, parquetTypes); case CHILD: return new GroupType(this.jsonSchema.optionalOrRequired(), docName, parquetTypes); default: throw new RuntimeException("Unsupported Record type"); } }
@Override protected Type buildSchema() { JsonElementConverter elementConverter = this.elementConverter; JsonElementConverter keyConverter = getKeyConverter(); GroupType mapGroup = Types.repeatedGroup().addFields(keyConverter.schema(), elementConverter.schema()).named(MAP_KEY) .asGroupType(); String columnName = this.jsonSchema.getColumnName(); switch (this.jsonSchema.optionalOrRequired()) { case OPTIONAL: return Types.optionalGroup().addFields(mapGroup).named(columnName).asGroupType(); case REQUIRED: return Types.requiredGroup().addFields(mapGroup).named(columnName).asGroupType(); default: return null; } }
@Override Object convertField(JsonElement value) { ParquetGroup r1 = new ParquetGroup((GroupType) schema()); JsonObject inputRecord = value.getAsJsonObject(); for (Map.Entry<String, JsonElement> entry : inputRecord.entrySet()) { String key = entry.getKey(); JsonElementConverter converter = this.converters.get(key); Object convertedValue = converter.convert(entry.getValue()); boolean valueIsNull = convertedValue == null; Type.Repetition repetition = converter.jsonSchema.optionalOrRequired(); if (valueIsNull && repetition.equals(OPTIONAL)) { continue; } r1.add(key, convertedValue); } return r1; }
/** * Convert value to a parquet type and perform null check. * @param value * @return Parquet safe type */ public Object convert(JsonElement value) { if (value.isJsonNull()) { if (this.jsonSchema.isNullable()) { return null; } throw new RuntimeException( "Field: " + this.jsonSchema.getColumnName() + " is not nullable and contains a null value"); } return convertField(value); }
@Override protected Type buildSchema() { return this.elementConverter.schema(); }
@Override Object convertField(JsonElement value) { if (symbols.contains(value.getAsString()) || this.jsonSchema.isNullable()) { return this.elementConverter.convert(value); } throw new RuntimeException("Symbol " + value.getAsString() + " does not belong to set " + symbols.toString()); }
@Override Object convertField(JsonElement value) { ParquetGroup array = new ParquetGroup((GroupType) schema()); JsonElementConverter converter = this.elementConverter; for (JsonElement elem : (JsonArray) value) { array.add(ARRAY_KEY, converter.convert(elem)); } return array; }
@Override Object convertField(JsonElement value) { ParquetGroup mapGroup = new ParquetGroup((GroupType) schema()); JsonElementConverter converter = this.elementConverter; JsonObject map = (JsonObject) value; for (Map.Entry<String, JsonElement> entry : map.entrySet()) { ParquetGroup entrySet = (ParquetGroup) mapGroup.addGroup(MAP_KEY); entrySet.add(MAP_KEY_COLUMN_NAME, entry.getKey()); entrySet.add(MAP_VALUE_COLUMN_NAME, converter.convert(entry.getValue())); } return mapGroup; }
@Override protected Type buildSchema() { List<Type> fields = new ArrayList<>(); fields.add(0, this.elementConverter.schema()); return new GroupType(this.jsonSchema.optionalOrRequired(), this.jsonSchema.getColumnName(), fields); }
@Override protected Type buildSchema() { JsonElementConverter elementConverter = this.elementConverter; JsonElementConverter keyConverter = getKeyConverter(); GroupType mapGroup = Types.repeatedGroup().addFields(keyConverter.schema(), elementConverter.schema()).named(MAP_KEY) .asGroupType(); String columnName = this.jsonSchema.getColumnName(); switch (this.jsonSchema.optionalOrRequired()) { case OPTIONAL: return Types.optionalGroup().addFields(mapGroup).named(columnName).asGroupType(); case REQUIRED: return Types.requiredGroup().addFields(mapGroup).named(columnName).asGroupType(); default: return null; } }
@Override protected Type buildSchema() { return this.elementConverter.schema(); }
@Override Object convertField(JsonElement value) { ParquetGroup mapGroup = new ParquetGroup((GroupType) schema()); JsonElementConverter converter = this.elementConverter; JsonObject map = (JsonObject) value; for (Map.Entry<String, JsonElement> entry : map.entrySet()) { ParquetGroup entrySet = (ParquetGroup) mapGroup.addGroup(MAP_KEY); entrySet.add(MAP_KEY_COLUMN_NAME, entry.getKey()); entrySet.add(MAP_VALUE_COLUMN_NAME, converter.convert(entry.getValue())); } return mapGroup; }
@Override protected Type buildSchema() { List<Type> fields = new ArrayList<>(); fields.add(0, this.elementConverter.schema()); return new GroupType(this.jsonSchema.optionalOrRequired(), this.jsonSchema.getColumnName(), fields); }
@Override Object convertField(JsonElement value) { ParquetGroup array = new ParquetGroup((GroupType) schema()); JsonElementConverter converter = this.elementConverter; for (JsonElement elem : (JsonArray) value) { array.add(ARRAY_KEY, converter.convert(elem)); } return array; }
@Override Object convertField(JsonElement value) { if (symbols.contains(value.getAsString()) || this.jsonSchema.isNullable()) { return this.elementConverter.convert(value); } throw new RuntimeException("Symbol " + value.getAsString() + " does not belong to set " + symbols.toString()); }