int fieldD = type.getMaxDefinitionLevel(path(fieldType.getName())) - 1; int id = fieldType.getId().intValue(); readersById.put(id, option(fieldType, fieldD, fieldReaders.get(i))); typesById.put(id, fieldType); types.add(typesById.get(id)); } else { reorderedFields.add(ParquetValueReaders.nulls()); types.add(null);
@Override public ParquetValueReader<?> struct(Types.StructType ignored, GroupType struct, List<ParquetValueReader<?>> fieldReaders) { // the expected struct is ignored because nested fields are never found when the List<ParquetValueReader<?>> newFields = Lists.newArrayListWithExpectedSize( fieldReaders.size()); List<Type> types = Lists.newArrayListWithExpectedSize(fieldReaders.size()); List<Type> fields = struct.getFields(); for (int i = 0; i < fields.size(); i += 1) { Type fieldType = fields.get(i); int fieldD = type.getMaxDefinitionLevel(path(fieldType.getName())) - 1; newFields.add(option(fieldType, fieldD, fieldReaders.get(i))); types.add(fieldType); } return new TupleReader(types, newFields, partitionValues); } }
@Override public ParquetValueReader<?> struct(Types.StructType ignored, GroupType struct, List<ParquetValueReader<?>> fieldReaders) { // the expected struct is ignored because nested fields are never found when the List<ParquetValueReader<?>> newFields = Lists.newArrayListWithExpectedSize( fieldReaders.size()); List<Type> types = Lists.newArrayListWithExpectedSize(fieldReaders.size()); List<Type> fields = struct.getFields(); for (int i = 0; i < fields.size(); i += 1) { Type fieldType = fields.get(i); int fieldD = type.getMaxDefinitionLevel(path(fieldType.getName()))-1; newFields.add(option(fieldType, fieldD, fieldReaders.get(i))); types.add(fieldType); } return new InternalRowReader(types, newFields); } }
int fieldD = type.getMaxDefinitionLevel(path(fieldType.getName()))-1; int id = fieldType.getId().intValue(); readersById.put(id, option(fieldType, fieldD, fieldReaders.get(i))); typesById.put(id, fieldType); types.add(typesById.get(id)); } else { reorderedFields.add(ParquetValueReaders.nulls()); types.add(null);
@Override public ParquetValueReader<?> struct(StructType expected, GroupType struct, List<ParquetValueReader<?>> fieldReaders) { // the expected struct is ignored because nested fields are never found when the List<ParquetValueReader<?>> newFields = Lists.newArrayListWithExpectedSize( fieldReaders.size()); List<Type> types = Lists.newArrayListWithExpectedSize(fieldReaders.size()); List<Type> fields = struct.getFields(); for (int i = 0; i < fields.size(); i += 1) { Type fieldType = fields.get(i); int fieldD = type.getMaxDefinitionLevel(path(fieldType.getName()))-1; newFields.add(option(fieldType, fieldD, fieldReaders.get(i))); types.add(fieldType); } return new RecordReader(types, newFields, expected); } }
int fieldD = type.getMaxDefinitionLevel(path(fieldType.getName()))-1; int id = fieldType.getId().intValue(); readersById.put(id, option(fieldType, fieldD, fieldReaders.get(i))); typesById.put(id, fieldType); types.add(typesById.get(id)); } else { reorderedFields.add(ParquetValueReaders.nulls()); types.add(null);
@Override public ParquetValueReader<?> map(Types.MapType expectedMap, GroupType map, ParquetValueReader<?> keyReader, ParquetValueReader<?> valueReader) { GroupType repeatedKeyValue = map.getFields().get(0).asGroupType(); String[] repeatedPath = currentPath(); int repeatedD = type.getMaxDefinitionLevel(repeatedPath) - 1; int repeatedR = type.getMaxRepetitionLevel(repeatedPath) - 1; Type keyType = repeatedKeyValue.getType(0); int keyD = type.getMaxDefinitionLevel(path(keyType.getName())) - 1; Type valueType = repeatedKeyValue.getType(1); int valueD = type.getMaxDefinitionLevel(path(valueType.getName())) - 1; return new MapReader<>(repeatedD, repeatedR, option(keyType, keyD, keyReader), option(valueType, valueD, valueReader)); }
int fieldD = type.getMaxDefinitionLevel(path(fieldType.getName()))-1; int id = fieldType.getId().intValue(); readersById.put(id, option(fieldType, fieldD, fieldReaders.get(i))); typesById.put(id, fieldType); types.add(typesById.get(id)); } else { reorderedFields.add(ParquetValueReaders.nulls()); types.add(null);
@Override public ParquetValueReader<?> map(Types.MapType expectedMap, GroupType map, ParquetValueReader<?> keyReader, ParquetValueReader<?> valueReader) { GroupType repeatedKeyValue = map.getFields().get(0).asGroupType(); String[] repeatedPath = currentPath(); int repeatedD = type.getMaxDefinitionLevel(repeatedPath)-1; int repeatedR = type.getMaxRepetitionLevel(repeatedPath)-1; Type keyType = repeatedKeyValue.getType(0); int keyD = type.getMaxDefinitionLevel(path(keyType.getName()))-1; Type valueType = repeatedKeyValue.getType(1); int valueD = type.getMaxDefinitionLevel(path(valueType.getName()))-1; return new MapReader<>(repeatedD, repeatedR, option(keyType, keyD, keyReader), option(valueType, valueD, valueReader)); }
@Override public ParquetValueReader<?> map(Types.MapType expectedMap, GroupType map, ParquetValueReader<?> keyReader, ParquetValueReader<?> valueReader) { GroupType repeatedKeyValue = map.getFields().get(0).asGroupType(); String[] repeatedPath = currentPath(); int repeatedD = type.getMaxDefinitionLevel(repeatedPath)-1; int repeatedR = type.getMaxRepetitionLevel(repeatedPath)-1; Type keyType = repeatedKeyValue.getType(0); int keyD = type.getMaxDefinitionLevel(path(keyType.getName()))-1; Type valueType = repeatedKeyValue.getType(1); int valueD = type.getMaxDefinitionLevel(path(valueType.getName()))-1; return new MapReader<>(repeatedD, repeatedR, option(keyType, keyD, keyReader), option(valueType, valueD, valueReader)); }
@Override public ParquetValueReader<?> map(Types.MapType expectedMap, GroupType map, ParquetValueReader<?> keyReader, ParquetValueReader<?> valueReader) { GroupType repeatedKeyValue = map.getFields().get(0).asGroupType(); String[] repeatedPath = currentPath(); int repeatedD = type.getMaxDefinitionLevel(repeatedPath)-1; int repeatedR = type.getMaxRepetitionLevel(repeatedPath)-1; Type keyType = repeatedKeyValue.getType(0); int keyD = type.getMaxDefinitionLevel(path(keyType.getName()))-1; Type valueType = repeatedKeyValue.getType(1); int valueD = type.getMaxDefinitionLevel(path(valueType.getName()))-1; return new MapReader<>(repeatedD, repeatedR, option(keyType, keyD, keyReader), option(valueType, valueD, valueReader)); }
@Override public ParquetValueReader<?> list(Types.ListType expectedList, GroupType array, ParquetValueReader<?> elementReader) { GroupType repeated = array.getFields().get(0).asGroupType(); String[] repeatedPath = currentPath(); int repeatedD = type.getMaxDefinitionLevel(repeatedPath)-1; int repeatedR = type.getMaxRepetitionLevel(repeatedPath)-1; Type elementType = repeated.getType(0); int elementD = type.getMaxDefinitionLevel(path(elementType.getName()))-1; return new ListReader<>(repeatedD, repeatedR, option(elementType, elementD, elementReader)); }
@Override public ParquetValueReader<?> list(Types.ListType expectedList, GroupType array, ParquetValueReader<?> elementReader) { GroupType repeated = array.getFields().get(0).asGroupType(); String[] repeatedPath = currentPath(); int repeatedD = type.getMaxDefinitionLevel(repeatedPath) - 1; int repeatedR = type.getMaxRepetitionLevel(repeatedPath) - 1; Type elementType = repeated.getType(0); int elementD = type.getMaxDefinitionLevel(path(elementType.getName())) - 1; return new ArrayReader<>(repeatedD, repeatedR, option(elementType, elementD, elementReader)); }
@Override public ParquetValueReader<?> list(Types.ListType expectedList, GroupType array, ParquetValueReader<?> elementReader) { GroupType repeated = array.getFields().get(0).asGroupType(); String[] repeatedPath = currentPath(); int repeatedD = type.getMaxDefinitionLevel(repeatedPath)-1; int repeatedR = type.getMaxRepetitionLevel(repeatedPath)-1; Type elementType = repeated.getType(0); int elementD = type.getMaxDefinitionLevel(path(elementType.getName()))-1; return new ArrayReader<>(repeatedD, repeatedR, option(elementType, elementD, elementReader)); }
@Override public ParquetValueReader<?> list(Types.ListType expectedList, GroupType array, ParquetValueReader<?> elementReader) { GroupType repeated = array.getFields().get(0).asGroupType(); String[] repeatedPath = currentPath(); int repeatedD = type.getMaxDefinitionLevel(repeatedPath)-1; int repeatedR = type.getMaxRepetitionLevel(repeatedPath)-1; Type elementType = repeated.getType(0); int elementD = type.getMaxDefinitionLevel(path(elementType.getName()))-1; return new ListReader<>(repeatedD, repeatedR, option(elementType, elementD, elementReader)); }