private ManifestReader(PartitionSpec spec, Iterable<ManifestEntry> entries) { this.file = null; this.metadata = ImmutableMap.of(); this.spec = spec; this.schema = spec.schema(); this.entries = entries; }
private InclusiveMetricsEvaluator metricsEvaluator() { if (lazyMetricsEvaluator == null) { this.lazyMetricsEvaluator = new InclusiveMetricsEvaluator(reader.spec().schema(), rowFilter); } return lazyMetricsEvaluator; } }
@Override public <T> Expression predicate(UnboundPredicate<T> pred) { Expression bound = pred.bind(spec.schema().asStruct()); if (bound instanceof BoundPredicate) { return predicate((BoundPredicate<?>) bound); } return bound; } }
@Override public <T> Expression predicate(UnboundPredicate<T> pred) { Expression bound = pred.bind(spec.schema().asStruct()); if (bound instanceof BoundPredicate) { Expression boundResidual = predicate((BoundPredicate<?>) bound); if (boundResidual instanceof Predicate) { return pred; // replace inclusive original unbound predicate } return boundResidual; // use the non-predicate residual (e.g. alwaysTrue) } // if binding didn't result in a Predicate, return the expression return bound; } }
@SuppressWarnings("unchecked") PartitionKey(PartitionSpec spec) { this.spec = spec; List<PartitionField> fields = spec.fields(); this.size = fields.size(); this.partitionTuple = new Object[size]; this.transforms = new Transform[size]; this.accessors = (Accessor<InternalRow>[]) Array.newInstance(Accessor.class, size); Schema schema = spec.schema(); Map<Integer, Accessor<InternalRow>> accessors = buildAccessors(schema); for (int i = 0; i < size; i += 1) { PartitionField field = fields.get(i); Accessor<InternalRow> accessor = accessors.get(field.sourceId()); if (accessor == null) { throw new RuntimeException( "Cannot build accessor for field: " + schema.findField(field.sourceId())); } this.accessors[i] = accessor; this.transforms[i] = field.transform(); } }
private static PartitionSpec freshSpec(int specId, Schema schema, PartitionSpec partitionSpec) { PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(schema) .withSpecId(specId); for (PartitionField field : partitionSpec.fields()) { // look up the name of the source field in the old schema to get the new schema's id String sourceName = partitionSpec.schema().findColumnName(field.sourceId()); specBuilder.add( schema.findField(sourceName).fieldId(), field.name(), field.transform().toString()); } return specBuilder.build(); }
ManifestReader reader = ManifestReader.read(ops.io().newInputFile(manifest.path())); toClose.add(reader); String schemaString = SchemaParser.toJson(reader.spec().schema()); String specString = PartitionSpecParser.toJson(reader.spec()); ResidualEvaluator residuals = new ResidualEvaluator(reader.spec(), rowFilter);
private static <D> FileAppender<D> newAppender(FileFormat format, PartitionSpec spec, OutputFile file) { Schema manifestSchema = ManifestEntry.getSchema(spec.partitionType()); try { switch (format) { case AVRO: return Avro.write(file) .schema(manifestSchema) .named("manifest_entry") .meta("schema", SchemaParser.toJson(spec.schema())) .meta("partition-spec", PartitionSpecParser.toJsonFields(spec)) .meta("partition-spec-id", String.valueOf(spec.specId())) .build(); default: throw new IllegalArgumentException("Unsupported format: " + format); } } catch (IOException e) { throw new RuntimeIOException(e, "Failed to create manifest writer for path: " + file); } } }
public FileAppender<InternalRow> newAppender(OutputFile file, FileFormat format) { Schema schema = spec.schema(); try { switch (format) {
BoundPredicate<?> bound = assertAndUnwrap(predicate.bind(spec.schema().asStruct()));
BoundPredicate<?> bound = assertAndUnwrap(predicate.bind(spec.schema().asStruct()));