/** * Call this constructor to initialize the fields for which this base class provides boilerplate * accessors, with an auto-generated tag. */ private SimplePCollectionView( PCollection<ElemT> pCollection, ViewFn<PrimitiveViewT, ViewT> viewFn, WindowMappingFn<W> windowMappingFn, WindowingStrategy<?, W> windowingStrategy) { this(pCollection, new TupleTag<>(), viewFn, windowMappingFn, windowingStrategy); }
private static String toProto(TupleTag<?> tag) { return tag.getId(); }
public static TupleTagList getAdditionalOutputTags(AppliedPTransform<?, ?, ?> application) throws IOException { PTransform<?, ?> transform = application.getTransform(); if (transform instanceof ParDo.MultiOutput) { return ((ParDo.MultiOutput<?, ?>) transform).getAdditionalOutputTags(); } RunnerApi.PTransform protoTransform = PTransformTranslation.toProto( application, SdkComponents.create(application.getPipeline().getOptions())); ParDoPayload payload = ParDoPayload.parseFrom(protoTransform.getSpec().getPayload()); TupleTag<?> mainOutputTag = getMainOutputTag(payload); Set<String> outputTags = Sets.difference( protoTransform.getOutputsMap().keySet(), Collections.singleton(mainOutputTag.getId())); ArrayList<TupleTag<?>> additionalOutputTags = new ArrayList<>(); for (String outputTag : outputTags) { additionalOutputTags.add(new TupleTag<>(outputTag)); } return TupleTagList.of(additionalOutputTags); }
@Override public boolean equals(Object other) { if (!(other instanceof PCollectionView)) { return false; } @SuppressWarnings("unchecked") PCollectionView<?> otherView = (PCollectionView<?>) other; return tag.equals(otherView.getTagInternal()); }
@Test public void testNonstaticTupleTag() { assertNotEquals(new TupleTag<>().getId(), new TupleTag<>().getId()); assertNotEquals(createNonstaticTupleTag(), createNonstaticTupleTag()); TupleTag<Object> tag = createNonstaticTupleTag(); // Check that the name is derived from the method it is created in. assertThat( Iterables.get(Splitter.on('#').split(tag.getId()), 0), startsWith("org.apache.beam.sdk.values.TupleTagTest.createNonstaticTupleTag")); // Check that after the name there is a ':' followed by a line number, and just make // sure the line number is big enough to be reasonable, so superficial changes don't break // the test. assertThat( Integer.parseInt( Iterables.get( Splitter.on(':').split(Iterables.get(Splitter.on('#').split(tag.getId()), 0)), 1)), greaterThan(15)); } }
@Override public Boolean call(Tuple2<TupleTag<V>, WindowedValue<?>> input) { return tag.equals(input._1()); } }
private PCollection(Pipeline p, WindowingStrategy<?, ?> windowingStrategy, IsBounded isBounded) { super(p); this.windowingStrategy = windowingStrategy; this.isBounded = isBounded; this.tag = new TupleTag<>(); }
public boolean isOutputTag(TupleTag<FeatureRowExtended> tag) { return tagMap.containsKey(tag.getId()); } }
@Override public boolean equals(Object o) { if (o == this) { return true; } if (o instanceof BigQueryConverters.FailsafeJsonToTableRow) { BigQueryConverters.FailsafeJsonToTableRow<?> that = (BigQueryConverters.FailsafeJsonToTableRow<?>) o; return (this.successTag.equals(that.successTag())) && (this.failureTag.equals(that.failureTag())); } return false; }
private CoGbkResultSchema schemaFromCloudObject(CloudObject cloudObject) { List<TupleTag<?>> tags = new ArrayList<>(); List<Map<String, Object>> serializedTags = Structs.getListOfMaps(cloudObject, PropertyNames.TUPLE_TAGS, Collections.emptyList()); for (Map<String, Object> serializedTag : serializedTags) { TupleTag<?> tag = new TupleTag<>(Structs.getString(serializedTag, PropertyNames.VALUE)); tags.add(tag); } return CoGbkResultSchema.of(tags); }
@Override public <T> void output(TupleTag<T> outputTag, WindowedValue<T> output) { if (outputTags.contains(outputTag)) { outputs.add(new RawUnionValue(outputTag.getId(), output)); } }
@Override public boolean equals(Object other) { if (!(other instanceof PCollectionView)) { return false; } @SuppressWarnings("unchecked") PCollectionView<?> otherView = (PCollectionView<?>) other; return tag.equals(otherView.getTagInternal()); }
@Override public int hashCode() { return tag.hashCode(); }
private CoGbkResultSchema createSchema(int size) { List<TupleTag<?>> tags = new ArrayList<>(); for (int i = 0; i < size; i++) { tags.add(new TupleTag<Integer>("tag" + i)); } return new CoGbkResultSchema(TupleTagList.of(tags)); }
private static void translateOutputs( Map<TupleTag<?>, PValue> outputs, StepTranslationContext stepContext) { for (Map.Entry<TupleTag<?>, PValue> taggedOutput : outputs.entrySet()) { TupleTag<?> tag = taggedOutput.getKey(); checkArgument( taggedOutput.getValue() instanceof PCollection, "Non %s returned from Multi-output %s", PCollection.class.getSimpleName(), stepContext); stepContext.addOutput(tag.getId(), (PCollection<?>) taggedOutput.getValue()); } }
@Override public boolean equals(Object o) { if (o == this) { return true; } if (o instanceof DatastoreConverters.CheckSameKey) { DatastoreConverters.CheckSameKey that = (DatastoreConverters.CheckSameKey) o; return (this.goodTag.equals(that.goodTag())) && (this.errorTag.equals(that.errorTag())); } return false; }