public static void setInputSchema(Configuration configuration, StructType inputSchema) throws IOException { if (inputSchema != null && inputSchema.getFields().size() > 0) { configuration.set(INPUT_SCHEMA, ObjectSerializationUtil.convertObjectToString(inputSchema)); } else { throw new UnsupportedOperationException("Input schema must be set"); } }
/** * create struct type with specified fields */ public static StructType createStructType(List<StructField> fields) { return new StructType(fields); }
private static void setFileHeader(Configuration configuration, CarbonLoadModel model) throws IOException { StructType inputSchema = getInputSchema(configuration); if (inputSchema == null || inputSchema.getFields().size() == 0) { throw new UnsupportedOperationException("Input schema must be set"); } List<StructField> fields = inputSchema.getFields(); StringBuilder builder = new StringBuilder(); String[] columns = new String[fields.size()]; int i = 0; for (StructField field : fields) { builder.append(field.getFieldName()); builder.append(","); columns[i++] = field.getFieldName(); } String header = builder.toString(); model.setCsvHeader(header.substring(0, header.length() - 1)); model.setCsvHeaderColumns(columns); }
@Override public boolean equals(Object obj) { if (this == obj) { return true; } if (obj == null) { return false; } if (getClass() != obj.getClass()) { return false; } StructType other = (StructType) obj; if (!this.getName().equalsIgnoreCase(other.getName())) { return false; } if (!this.getFields().equals(other.getFields())) { return false; } return true; }
/** * create a struct type object with no field */ public static StructType createDefaultStructType() { return new StructType(new ArrayList<StructField>()); }
((StructType) keyValueField.getDataType()).getFields().get(1).getDataType(); MapType mapType = DataTypes.createMapType(DataTypes.STRING, valueType); mapSubFields.add(keyValueField);
} else if (DataTypes.isStructType(dataType)) { List<StructField> structFields = new ArrayList<>(); List<StructField> fields = ((StructType) dataType).getFields(); for (StructField field : fields) { structFields.add(prepareSubFields(field.getFieldName(), field.getDataType()));
private void initComplexTypeChildren() { if (getDataType().isComplexType()) { StructField subFields = prepareSubFields(getFieldName(), getDataType()); if (DataTypes.isArrayType(getDataType()) || DataTypes.isMapType(getDataType())) { children = subFields.getChildren(); } else if (DataTypes.isStructType(getDataType())) { children = ((StructType) subFields.getDataType()).getFields(); } } }
newColumn.setNumberOfChild(1); } else { newColumn.setNumberOfChild(((StructType) field.getDataType()).getFields().size()); field.getFieldName(), valIndex, false, true, isInvertedIdxColumn); } else if (DataTypes.isStructType(field.getDataType()) && ((StructType) field.getDataType()).getFields().size() > 0) { List<StructField> fields = ((StructType) field.getDataType()).getFields(); for (int i = 0; i < fields.size(); i++) { addColumn(fields.get(i), parentFieldName, valIndex, false, true, isInvertedIdxColumn);