public static Schema selectNot(Schema schema, Set<Integer> fieldIds) { Set<Integer> projectedIds = getProjectedIds(schema); projectedIds.removeAll(fieldIds); return select(schema, projectedIds); }
Schema partitionSchema = TypeUtil.select(requiredSchema, idColumns); PartitionRowConverter convertToRow = new PartitionRowConverter(partitionSchema, spec); JoinedRow joined = new JoinedRow();
/** * Creates a projection schema for a subset of columns, selected by name. * <p> * Names that identify nested fields will select part or all of the field's top-level column. * * @param names a List of String names for selected columns * @return a projection schema from this schema, by name */ public Schema select(Collection<String> names) { if (names.contains(ALL_COLUMNS)) { return this; } Set<Integer> selected = Sets.newHashSet(); for (String name : names) { Integer id = lazyNameToId().get(name); if (id != null) { selected.add(id); } } return TypeUtil.select(this, selected); }
@Override public TableScan select(Collection<String> columns) { Set<Integer> requiredFieldIds = Sets.newHashSet(); // all of the filter columns are required requiredFieldIds.addAll( Binder.boundReferences(table.schema().asStruct(), Collections.singletonList(rowFilter))); // all of the projection columns are required requiredFieldIds.addAll(TypeUtil.getProjectedIds(table.schema().select(columns))); Schema projection = TypeUtil.select(table.schema(), requiredFieldIds); return new BaseTableScan(ops, table, snapshotId, projection, rowFilter); }
@Test public void testDeleteFields() { // use schema projection to test column deletes Set<Integer> ALL_IDS = ImmutableSet.copyOf(TypeUtil.getProjectedIds(SCHEMA)); List<String> columns = Lists.newArrayList("id", "data", "preferences", "preferences.feature1", "preferences.feature2", "locations", "locations.lat", "locations.long", "points", "points.x", "points.y", "doubles", "properties"); for (String name : columns) { Set<Integer> selected = Sets.newHashSet(ALL_IDS); // remove the id and any nested fields from the projection Types.NestedField nested = SCHEMA.findField(name); selected.remove(nested.fieldId()); selected.removeAll(TypeUtil.getProjectedIds(nested.type())); Schema del = new SchemaUpdate(SCHEMA, 19).deleteColumn(name).apply(); Assert.assertEquals("Should match projection with '" + name + "' removed", TypeUtil.select(SCHEMA, selected).asStruct(), del.asStruct()); } }