StructField arrayField = prepareSubFields(fieldName, ((ArrayType) dataType).getElementType()); arrayFields.add(arrayField); return new StructField(fieldName, DataTypes.createArrayType(arrayField.getDataType()), arrayFields); } else if (DataTypes.isStructType(dataType)) { List<StructField> fields = ((StructType) dataType).getFields(); for (StructField field : fields) { structFields.add(prepareSubFields(field.getFieldName(), field.getDataType())); return new StructField(fieldName, DataTypes.createStructType(structFields), structFields); } else if (DataTypes.isMapType(dataType)) { MapType mapType = (MapType) dataType; StructField keyField = new StructField(fieldName + ".key", mapType.getKeyType()); StructField valueField = prepareSubFields(fieldName + ".value", mapType.getValueType()); mapFields.add(keyField); mapFields.add(valueField); StructField field = new StructField(fieldName + ".val", DataTypes.createStructType(mapFields)); MapType mapDataType = DataTypes.createMapType(keyField.getDataType(), field.getDataType()); List<StructField> mapStructField = new ArrayList<>(); mapStructField.add(field); return new StructField(fieldName, mapDataType, mapStructField); } else { return new StructField(fieldName, dataType);
/** * Throw exception if {@param field} name is repeated */ private void checkRepeatColumnName(StructField field, String parentName) { checkRepeatColumnName( new StructField(parentName + "." + field.getFieldName(), field.getDataType(), field.getChildren())); }
switch (type) { case BOOLEAN: return new StructField(fieldName, DataTypes.BOOLEAN); case INT: if (logicalType instanceof LogicalTypes.Date) { return new StructField(fieldName, DataTypes.DATE); } else { return new StructField(fieldName, DataTypes.INT); return new StructField(fieldName, DataTypes.TIMESTAMP); } else { return new StructField(fieldName, DataTypes.LONG); return new StructField(fieldName, DataTypes.DOUBLE); case ENUM: case STRING: return new StructField(fieldName, DataTypes.STRING); case FLOAT: return new StructField(fieldName, DataTypes.FLOAT); case MAP: StructField keyField = new StructField(fieldName + ".key", DataTypes.STRING); StructField valueField = prepareSubFields(fieldName + ".value", childSchema.getValueType()); if (null != valueField) { keyValueFields.add(valueField); StructField mapKeyValueField =
private void initComplexTypeChildren() { if (getDataType().isComplexType()) { StructField subFields = prepareSubFields(getFieldName(), getDataType()); if (DataTypes.isArrayType(getDataType()) || DataTypes.isMapType(getDataType())) { children = subFields.getChildren(); } else if (DataTypes.isStructType(getDataType())) { children = ((StructType) subFields.getDataType()).getFields(); } } }
private void checkRepeatColumnName(StructField field) { for (ColumnSchema column : sortColumns) { if (column.getColumnName().equalsIgnoreCase(field.getFieldName())) { throw new IllegalArgumentException("column name already exists"); } } for (ColumnSchema column : dimension) { if (column.getColumnName().equalsIgnoreCase(field.getFieldName())) { throw new IllegalArgumentException("column name already exists"); } } for (ColumnSchema column : complex) { if (column.getColumnName().equalsIgnoreCase(field.getFieldName())) { throw new IllegalArgumentException("column name already exists"); } } for (ColumnSchema column : measures) { if (column.getColumnName().equalsIgnoreCase(field.getFieldName())) { throw new IllegalArgumentException("column name already exists"); } } }
this.type = DataTypes.DOUBLE; } else if (type.equalsIgnoreCase("array")) { this.type = DataTypes.createArrayType(fields.get(0).getDataType()); } else if (type.equalsIgnoreCase("struct")) { this.type = DataTypes.createStructType(fields);
for (ProjectionDimension dim : queryDimension) { fields[dim.getOrdinal()] = new StructField(dim.getColumnName(), dim.getDimension().getDataType()); || dataType == DataTypes.FLOAT || dataType == DataTypes.BYTE) { fields[msr.getOrdinal()] = new StructField(msr.getColumnName(), msr.getMeasure().getDataType()); } else if (DataTypes.isDecimal(dataType)) { fields[msr.getOrdinal()] = new StructField(msr.getColumnName(), DataTypes.createDecimalType(msr.getMeasure().getPrecision(), msr.getMeasure().getScale())); } else { fields[msr.getOrdinal()] = new StructField(msr.getColumnName(), DataTypes.DOUBLE); vectors[i] = new CarbonColumnVectorImpl( CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT, fields[i].getDataType());
if (null != mapField) { StructField keyValueField = mapField.getChildren().get(0); ((StructType) keyValueField.getDataType()).getFields().get(1).getDataType(); MapType mapType = DataTypes.createMapType(DataTypes.STRING, valueType); mapSubFields.add(keyValueField);
private static void setFileHeader(Configuration configuration, CarbonLoadModel model) throws IOException { StructType inputSchema = getInputSchema(configuration); if (inputSchema == null || inputSchema.getFields().size() == 0) { throw new UnsupportedOperationException("Input schema must be set"); } List<StructField> fields = inputSchema.getFields(); StringBuilder builder = new StringBuilder(); String[] columns = new String[fields.size()]; int i = 0; for (StructField field : fields) { builder.append(field.getFieldName()); builder.append(","); columns[i++] = field.getFieldName(); } String header = builder.toString(); model.setCsvHeader(header.substring(0, header.length() - 1)); model.setCsvHeaderColumns(columns); }
return mapField.getDataType();
newColumn.setColumnName(parentName + "." + field.getFieldName()); } else { newColumn.setColumnName(field.getFieldName()); newColumn.setDataType(field.getDataType()); if (isSortColumn || field.getDataType() == DataTypes.STRING || field.getDataType() == DataTypes.VARCHAR || field.getDataType() == DataTypes.DATE || field.getDataType() == DataTypes.TIMESTAMP || field.getDataType().isComplexType() || (isComplexChild)) { newColumn.setDimensionColumn(true); newColumn.setColumnUniqueId(field.getFieldName()); newColumn.setColumnReferenceId(newColumn.getColumnUniqueId()); newColumn .setEncodingList(createEncoding(field.getDataType(), isInvertedIdxColumn, isComplexChild)); if (field.getDataType().isComplexType()) { if (DataTypes.isArrayType(field.getDataType()) || DataTypes.isMapType(field.getDataType())) { newColumn.setNumberOfChild(1); } else { newColumn.setNumberOfChild(((StructType) field.getDataType()).getFields().size()); if (DataTypes.isDecimal(field.getDataType())) { DecimalType decimalType = (DecimalType) field.getDataType(); newColumn.setPrecision(decimalType.getPrecision()); newColumn.setScale(decimalType.getScale());
DataTypes.createArrayType(field.getChildren().get(0).getDataType()); tableSchemaBuilder .addColumn(new StructField(field.getFieldName(), complexType), valIndex, false, isInvertedIdxColumn > -1); } else if (field.getDataType().getName().equalsIgnoreCase("STRUCT")) { for (StructField childFld : field.getChildren()) { structFieldsArray .add(new StructField(childFld.getFieldName(), childFld.getDataType())); .addColumn(new StructField(field.getFieldName(), complexType), valIndex, false, isInvertedIdxColumn > -1); } else if (field.getDataType().getName().equalsIgnoreCase("MAP")) { field.getChildren().get(0).getDataType()); tableSchemaBuilder .addColumn(new StructField(field.getFieldName(), mapType), valIndex, false, isInvertedIdxColumn > -1); .addColumn(new StructField(field.getFieldName(), field.getDataType()), valIndex, isSortColumn > -1, isInvertedIdxColumn > -1); if (isSortColumn > -1) {