public ParquetGroup(GroupType schema) { this.schema = schema; this.data = new List[schema.getFields().size()]; for (int i = 0; i < schema.getFieldCount(); ++i) { this.data[i] = new ArrayList(); } }
public String toString(String indent) { StringBuilder result = new StringBuilder(); int i = 0; for (Type field : this.schema.getFields()) { String name = field.getName(); List<Object> values = this.data[i]; for (Object value : values) { result.append(indent).append(name); if (value == null) { result.append(": NULL\n"); } else if (value instanceof Group) { result.append("\n").append(((ParquetGroup) value).toString(indent + " ")); } else { result.append(": ").append(value.toString()).append("\n"); } } i++; } return result.toString(); }
@SuppressWarnings("unchecked") public SimpleGroup(GroupType schema) { this.schema = schema; this.data = new List[schema.getFields().size()]; for (int i = 0; i < schema.getFieldCount(); i++) { this.data[i] = new ArrayList<Object>(); } }
/** * {@inheritDoc} */ @Override public int hashCode() { return super.hashCode() * 31 + getFields().hashCode(); }
@SuppressWarnings("unchecked") public SimpleGroup(GroupType schema) { this.schema = schema; this.data = new List[schema.getFields().size()]; for (int i = 0; i < schema.getFieldCount(); i++) { this.data[i] = new ArrayList<Object>(); } }
public ParquetGroup(GroupType schema) { this.schema = schema; this.data = new List[schema.getFields().size()]; for (int i = 0; i < schema.getFieldCount(); ++i) { this.data[i] = new ArrayList(); } }
private void visitChildren(GroupColumnIO newIO, GroupType groupType) { GroupColumnIO oldIO = current; current = newIO; for (Type type : groupType.getFields()) { type.accept(this); } current = oldIO; }
/** * Searchs for a fieldName into a parquet GroupType by ignoring string case. * GroupType#getType(String fieldName) is case sensitive, so we use this method. * * @param groupType Group of field types where to search for fieldName * @param fieldName The field what we are searching * @return The Type object of the field found; null otherwise. */ private static Type getFieldTypeIgnoreCase(GroupType groupType, String fieldName) { for (Type type : groupType.getFields()) { if (type.getName().equalsIgnoreCase(fieldName)) { return type; } } return null; }
private void visitChildren(final List<SchemaElement> result, GroupType groupType, SchemaElement element) { element.setNum_children(groupType.getFieldCount()); result.add(element); for (parquet.schema.Type field : groupType.getFields()) { addToList(result, field); } } });
public MessageType getConvertedMessageType() { // the root should be a GroupType if (currentType == null) return new MessageType(currentName, new ArrayList<Type>()); GroupType rootType = currentType.asGroupType(); return new MessageType(currentName, rootType.getFields()); }
/** * Create an Array Hive schema from equivalent parquet list type */ private static String createHiveArray(Type elementType, String elementName) { StringBuilder array = new StringBuilder(); array.append("ARRAY< "); if (elementType.isPrimitive()) { array.append(convertField(elementType)); } else { final GroupType groupType = elementType.asGroupType(); final List<Type> groupFields = groupType.getFields(); if (groupFields.size() > 1 || (groupFields.size() == 1 && ( elementType.getName().equals("array") || elementType.getName() .equals(elementName + "_tuple")))) { array.append(convertField(elementType)); } else { array.append(convertField(groupType.getFields().get(0))); } } array.append(">"); return array.toString(); }
public SimpleRecordConverter(GroupType schema, String name, SimpleRecordConverter parent) { this.converters = new Converter[schema.getFieldCount()]; this.parent = parent; this.name = name; int i = 0; for (Type field: schema.getFields()) { converters[i++] = createConverter(field); } }
/** * {@inheritDoc} */ @Override protected int typeHashCode() { int c = 17; c += 31 * getRepetition().hashCode(); c += 31 * getName().hashCode(); c += 31 * getFields().hashCode(); return c; }
private static void showDetails(PrettyPrintWriter out, GroupType type, int depth, MessageType container, List<String> cpath) { String name = Strings.repeat(".", depth) + type.getName(); Repetition rep = type.getRepetition(); int fcount = type.getFieldCount(); out.format("%s: %s F:%d%n", name, rep, fcount); cpath.add(type.getName()); for (Type ftype : type.getFields()) { showDetails(out, ftype, depth + 1, container, cpath); } cpath.remove(cpath.size() - 1); }
/** * {@inheritDoc} */ @Override protected boolean typeEquals(Type other) { Type otherType = (Type) other; if (otherType.isPrimitive()) { return false; } else { GroupType groupType = otherType.asGroupType(); return getRepetition() == groupType.getRepetition() && getName().equals(groupType.getName()) && getFields().equals(groupType.getFields()); } }
public ParquetMapConverter(Type type, String columnName, GroupType mapType, int fieldIndex) { checkArgument( mapType.getFieldCount() == 1, "Expected MAP column '%s' to only have one field, but has %s fields", mapType.getName(), mapType.getFieldCount()); this.mapType = type; this.fieldIndex = fieldIndex; parquet.schema.Type entryType = mapType.getFields().get(0); entryConverter = new ParquetMapEntryConverter(type, columnName + ".entry", entryType.asGroupType()); }
void checkGroupContains(Type subType) { if (subType.isPrimitive()) { throw new InvalidRecordException(subType + " found: expected " + this); } List<Type> fields = subType.asGroupType().getFields(); for (Type otherType : fields) { Type thisType = this.getType(otherType.getName()); thisType.checkContains(otherType); } }
@Override void checkContains(Type subType) { super.checkContains(subType); if (subType.isPrimitive()) { throw new InvalidRecordException(subType + " found: expected " + this); } List<Type> fields = subType.asGroupType().getFields(); for (Type otherType : fields) { Type thisType = this.getType(otherType.getName()); thisType.checkContains(otherType); } }
private void visitChildren(GroupColumnIO newIO, GroupType groupType, GroupType requestedGroupType) { GroupColumnIO oldIO = current; current = newIO; for (Type type : groupType.getFields()) { // if the file schema does not contain the field it will just stay null if (requestedGroupType.containsField(type.getName())) { currentRequestedIndex = requestedGroupType.getFieldIndex(type.getName()); currentRequestedType = requestedGroupType.getType(currentRequestedIndex); if (currentRequestedType.getRepetition().isMoreRestrictiveThan(type.getRepetition())) { incompatibleSchema(type, currentRequestedType); } type.accept(this); } } current = oldIO; }