private static TypeDescription getTypeDescriptionFromTableProperties(Properties tableProperties) { TypeDescription schema = null; if (tableProperties != null) { final String columnNameProperty = tableProperties.getProperty(IOConstants.COLUMNS); final String columnTypeProperty = tableProperties.getProperty(IOConstants.COLUMNS_TYPES); if (!Strings.isNullOrEmpty(columnNameProperty) && !Strings.isNullOrEmpty(columnTypeProperty)) { List<String> columnNames = columnNameProperty.length() == 0 ? new ArrayList<String>() : Arrays.asList(columnNameProperty.split(",")); List<TypeInfo> columnTypes = columnTypeProperty.length() == 0 ? new ArrayList<TypeInfo>() : TypeInfoUtils .getTypeInfosFromTypeString(columnTypeProperty); schema = TypeDescription.createStruct(); for (int i = 0; i < columnNames.size(); i++) { schema.addField(columnNames.get(i), OrcInputFormat.convertTypeInfo(columnTypes.get(i))); } } } if (LOG.isDebugEnabled()) { LOG.debug("ORC schema = " + schema); } return schema; }
TypeDescription schema = TypeDescription.createStruct(); for (int i = 0; i < columnNames.size(); ++i) { schema.addField(columnNames.get(i),
TypeDescription schema = TypeDescription.createStruct(); for (int i = 0; i < columnNames.size(); ++i) { schema.addField(columnNames.get(i),
TypeDescription result = TypeDescription.createStruct(); for (int i = 0; i < schemaEvolutionTypeDescrs.size(); i++) { result.addField(schemaEvolutionColumnNames.get(i), schemaEvolutionTypeDescrs.get(i));
TypeDescription result = TypeDescription.createStruct(); for (int i = 0; i < schemaEvolutionTypeDescrs.size(); i++) { result.addField(schemaEvolutionColumnNames.get(i), schemaEvolutionTypeDescrs.get(i));
TypeDescription result = TypeDescription.createStruct(); for(String fieldName: sinfo.getAllStructFieldNames()) { result.addField(fieldName,
TypeDescription result = TypeDescription.createStruct(); for(String fieldName: sinfo.getAllStructFieldNames()) { result.addField(fieldName,
public TypeDescription buildTypeDescription( List<? extends IOrcOutputField> fields ) { TypeDescription typeDescription = TypeDescription.createStruct(); fields.forEach( field -> addStructField( typeDescription, field ) ); return typeDescription; }
public static TypeDescription convert(final DataType[] fieldTypes, final String[] fieldNames) { final TypeDescription td = TypeDescription.createStruct(); for (int i = 0; i < fieldTypes.length; i++) { td.addField(fieldNames[i], convertType(fieldTypes[i])); } return td; }
public static TypeDescription convertSchema(Schema schema) { TypeDescription description = TypeDescription.createStruct(); for (Column eachColumn : schema.getRootColumns()) { description.addField(eachColumn.getQualifiedName(), convertTypeInfo(eachColumn.getTypeDesc())); } return description; }
public static TypeDescription convertSchema(Schema schema) { TypeDescription description = TypeDescription.createStruct(); for (Column eachColumn : schema.getRootColumns()) { description.addField(eachColumn.getQualifiedName(), convertTypeInfo(eachColumn.getTypeDesc())); } return description; }
private static TypeDescription getSchema(Schema schema) { TypeDescription result = TypeDescription.createStruct(); for (String col : schema.getColumnNames()) { ColumnDescription cd = schema.getDescription(col); TypeDescription current; switch (cd.kind) { case None: default: throw new RuntimeException("Unexpected data type " + cd.kind); case String: case Json: current = TypeDescription.createString(); break; case Date: current = TypeDescription.createTimestamp(); break; case Integer: current = TypeDescription.createInt(); break; case Duration: case Double: current = TypeDescription.createDouble(); break; } result.addField(col, current); } return result; }
/** * @param typeDescr * @return ORC types for the ACID event based on the row's type description */ public static TypeDescription createEventSchema(TypeDescription typeDescr) { TypeDescription result = TypeDescription.createStruct() .addField("operation", TypeDescription.createInt()) .addField("originalTransaction", TypeDescription.createLong()) .addField("bucket", TypeDescription.createInt()) .addField("rowId", TypeDescription.createLong()) .addField("currentTransaction", TypeDescription.createLong()) .addField("row", typeDescr.clone()); return result; }
public static void convert(InputStream input, String outputOrc) throws IOException { TypeDescription schema = createStruct() .addField("id", createLong()) .addField("type", createString()) .addField("lon", createDecimal().withScale(7).withPrecision(10)) .addField("nds", createList( createStruct() .addField("ref", createLong()) )) .addField("members", createList( createStruct() .addField("type", createString()) .addField("ref", createLong())
.withMaxLength(desc.getDataType().getLength()); case RECORD: { TypeDescription result = TypeDescription.createStruct(); for (Column eachColumn : desc.getNestedSchema().getRootColumns()) { result.addField(eachColumn.getQualifiedName(),
.withMaxLength(desc.getDataType().getLength()); case RECORD: { TypeDescription result = TypeDescription.createStruct(); for (Column eachColumn : desc.getNestedSchema().getRootColumns()) { result.addField(eachColumn.getQualifiedName(),
convertTypeFromProtobuf(types, type.getSubtypes(1))); case STRUCT: { TypeDescription result = TypeDescription.createStruct(); for(int f=0; f < type.getSubtypesCount(); ++f) { result.addField(type.getFieldNames(f),
result = TypeDescription.createStruct(); for(Types.NestedField field: type.asStructType().fields()) { result.addField(field.name(), toOrc(field.fieldId(), field.type(), columnIds));