@Override public Coder<Row> getAccumulatorCoder(CoderRegistry registry, Coder<Row> inputCoder) throws CannotProvideCoderException { return SchemaCoder.of(EMPTY_SCHEMA); }
@Override public Coder<Row> getDefaultOutputCoder(CoderRegistry registry, Coder<Row> inputCoder) { return SchemaCoder.of(EMPTY_SCHEMA); } }
/** * Returns a {@link Create.TimestampedValues} PTransform like this one that uses the given * {@code Schema} to represent objects. */ @Experimental(Kind.SCHEMAS) public TimestampedValues<T> withSchema( Schema schema, SerializableFunction<T, Row> toRowFunction, SerializableFunction<Row, T> fromRowFunction) { return withCoder(SchemaCoder.of(schema, toRowFunction, fromRowFunction)); }
public static <T> Builder<T> create( Schema schema, SerializableFunction<T, Row> toRowFunction, SerializableFunction<Row, T> fromRowFunction) { return create(SchemaCoder.of(schema, toRowFunction, fromRowFunction)); }
/** * Returns a {@link Create.Values} PTransform like this one that uses the given {@code Schema} * to represent objects. */ @Experimental(Kind.SCHEMAS) public Values<T> withSchema( Schema schema, SerializableFunction<T, Row> toRowFunction, SerializableFunction<Row, T> fromRowFunction) { return withCoder(SchemaCoder.of(schema, toRowFunction, fromRowFunction)); }
/** Sets a {@link Schema} on this {@link PCollection}. */ @Experimental(Kind.SCHEMAS) public PCollection<T> setSchema( Schema schema, SerializableFunction<T, Row> toRowFunction, SerializableFunction<Row, T> fromRowFunction) { return setCoder(SchemaCoder.of(schema, toRowFunction, fromRowFunction)); }
private static <T> PCollection<KV<Row, Row>> extractKey( PCollection<T> pCollection, Schema schema, Schema keySchema, FieldAccessDescriptor keyFields, String tag) { return pCollection .apply( "extractKey" + tag, ParDo.of( new DoFn<T, KV<Row, Row>>() { @ProcessElement public void process(@Element Row row, OutputReceiver<KV<Row, Row>> o) { o.output(KV.of(Select.selectRow(row, keyFields, schema, keySchema), row)); } })) .setCoder(KvCoder.of(SchemaCoder.of(keySchema), SchemaCoder.of(schema))); } }
public Coder<Row> rowCoder() { return SchemaCoder.of( tableWithRows.table.getSchema(), SerializableFunctions.identity(), SerializableFunctions.identity()); }
@Override public Coder<Row> getDefaultOutputCoder(CoderRegistry registry, Coder<T> inputCoder) { return SchemaCoder.of( getOutputSchema(), SerializableFunctions.identity(), SerializableFunctions.identity()); }
/** * Returns a {@link Create.Values} PTransform like this one that uses the given {@code Schema} * to represent objects. */ @Experimental(Kind.SCHEMAS) public Values<T> withRowSchema(Schema schema) { return withCoder( SchemaCoder.of( schema, (SerializableFunction<T, Row>) SerializableFunctions.<Row>identity(), (SerializableFunction<Row, T>) SerializableFunctions.<Row>identity())); }
try { Coder<T> coder = SchemaCoder.of( schemaRegistry.getSchema(typeDescriptor), schemaRegistry.getToRowFunction(typeDescriptor),
public PCollection<Row> sideInputJoin( PCollection<KV<Row, Row>> extractedLeftRows, PCollection<KV<Row, Row>> extractedRightRows, Schema leftSchema, Schema rightSchema) { // we always make the Unbounded table on the left to do the sideInput join // (will convert the result accordingly before return) boolean swapped = (extractedLeftRows.isBounded() == PCollection.IsBounded.BOUNDED); JoinRelType realJoinType = (swapped && joinType != JoinRelType.INNER) ? JoinRelType.LEFT : joinType; PCollection<KV<Row, Row>> realLeftRows = swapped ? extractedRightRows : extractedLeftRows; PCollection<KV<Row, Row>> realRightRows = swapped ? extractedLeftRows : extractedRightRows; Row realRightNullRow; if (swapped) { Schema leftNullSchema = buildNullSchema(leftSchema); realRightRows = setValueCoder(realRightRows, SchemaCoder.of(leftNullSchema)); realRightNullRow = Row.nullRow(leftNullSchema); } else { Schema rightNullSchema = buildNullSchema(rightSchema); realRightRows = setValueCoder(realRightRows, SchemaCoder.of(rightNullSchema)); realRightNullRow = Row.nullRow(rightNullSchema); } // swapped still need to pass down because, we need to swap the result back. return sideInputJoinHelper( realJoinType, realLeftRows, realRightRows, realRightNullRow, swapped); }
try { SchemaCoder<T> schemaCoder = SchemaCoder.of( schemaRegistry.getSchema(token), schemaRegistry.getToRowFunction(token),
Row rightNullRow = Row.nullRow(rigthNullSchema); extractedRightRows = setValueCoder(extractedRightRows, SchemaCoder.of(rigthNullSchema)); Row leftNullRow = Row.nullRow(leftNullSchema); extractedLeftRows = setValueCoder(extractedLeftRows, SchemaCoder.of(leftNullSchema)); Row rightNullRow = Row.nullRow(rightNullSchema); extractedLeftRows = setValueCoder(extractedLeftRows, SchemaCoder.of(leftNullSchema)); extractedRightRows = setValueCoder(extractedRightRows, SchemaCoder.of(rightNullSchema));
return SchemaCoder.of( schemaRegistry.getSchema(o.getClass()), (SerializableFunction) schemaRegistry.getToRowFunction(o.getClass()),
@Override public PCollection<KV<Row, Iterable<InputT>>> expand(PCollection<InputT> input) { Schema schema = input.getSchema(); FieldAccessDescriptor resolved = fieldAccessDescriptor.resolve(schema); keySchema = Select.getOutputSchema(schema, resolved); return input .apply( "Group by fields", ParDo.of( new DoFn<InputT, KV<Row, InputT>>() { @ProcessElement public void process( @Element InputT element, @Element Row row, OutputReceiver<KV<Row, InputT>> o) { o.output(KV.of(Select.selectRow(row, resolved, schema, keySchema), element)); } })) .setCoder(KvCoder.of(SchemaCoder.of(keySchema), input.getCoder())) .apply(GroupByKey.create()); } }
try { coder = SchemaCoder.of( schemaRegistry.getSchema(typeDescriptor.get()), schemaRegistry.getToRowFunction(typeDescriptor.get()),
try { coder = SchemaCoder.of( schemaRegistry.getSchema(typeDescriptor.get()), schemaRegistry.getToRowFunction(typeDescriptor.get()),
.build(); Coder<Row> coder = SchemaCoder.of( beamSchema, SerializableFunctions.identity(), SerializableFunctions.identity()); CoderProperties.coderDecodeEncodeEqual(coder, row);
"waitForSuccess", resultSignal.signalSuccessWhen( SchemaCoder.of( PAYLOAD_SCHEMA, SerializableFunctions.identity(), SerializableFunctions.identity()), observedRows ->