public boolean isOutputTag(TupleTag<FeatureRowExtended> tag) { return tagMap.containsKey(tag.getId()); } }
@Override public <T> void output(TupleTag<T> outputTag, WindowedValue<T> output) { if (outputTags.contains(outputTag)) { outputs.add(new RawUnionValue(outputTag.getId(), output)); } }
private static void translateOutputs( Map<TupleTag<?>, PValue> outputs, StepTranslationContext stepContext) { for (Map.Entry<TupleTag<?>, PValue> taggedOutput : outputs.entrySet()) { TupleTag<?> tag = taggedOutput.getKey(); checkArgument( taggedOutput.getValue() instanceof PCollection, "Non %s returned from Multi-output %s", PCollection.class.getSimpleName(), stepContext); stepContext.addOutput(tag.getId(), (PCollection<?>) taggedOutput.getValue()); } }
public static Map<String, SideInput> translateSideInputs( List<PCollectionView<?>> views, SdkComponents components) { Map<String, SideInput> sideInputs = new HashMap<>(); for (PCollectionView<?> sideInput : views) { sideInputs.put( sideInput.getTagInternal().getId(), ParDoTranslation.translateView(sideInput, components)); } return sideInputs; }
@Override public Map<String, SideInput> translateSideInputs(SdkComponents components) { Map<String, SideInput> sideInputs = new HashMap<>(); for (PCollectionView<?> sideInput : parDo.getSideInputs()) { sideInputs.put( sideInput.getTagInternal().getId(), translateView(sideInput, components)); } return sideInputs; }
private static Map<String, TupleTag<?>> createOutputMap( TupleTag mainOutput, List<TupleTag<?>> additionalOutputs) { Map<String, TupleTag<?>> outputMap = new HashMap<>(additionalOutputs.size() + 1); if (mainOutput != null) { outputMap.put(mainOutput.getId(), mainOutput); } for (TupleTag<?> additionalTag : additionalOutputs) { outputMap.put(additionalTag.getId(), additionalTag); } return outputMap; } }
private static Map<String, TupleTag<?>> createOutputMap( TupleTag mainOutput, List<TupleTag<?>> additionalOutputs) { Map<String, TupleTag<?>> outputMap = new HashMap<>(additionalOutputs.size() + 1); if (mainOutput != null) { outputMap.put(mainOutput.getId(), mainOutput); } for (TupleTag<?> additionalTag : additionalOutputs) { outputMap.put(additionalTag.getId(), additionalTag); } return outputMap; } }
private static void translateSideInputs( StepTranslationContext stepContext, List<PCollectionView<?>> sideInputs, TranslationContext context) { Map<String, Object> nonParInputs = new HashMap<>(); for (PCollectionView<?> view : sideInputs) { nonParInputs.put( view.getTagInternal().getId(), context.asOutputReference(view, context.getProducer(view))); } stepContext.addInput(PropertyNames.NON_PARALLEL_INPUTS, nonParInputs); }
private static void transformSideInputs( List<PCollectionView<?>> sideInputs, SingleInputUdfOperator<?, ?, ?> outputDataSet, FlinkBatchTranslationContext context) { // get corresponding Flink broadcast DataSets for (PCollectionView<?> input : sideInputs) { DataSet<?> broadcastSet = context.getSideInputDataSet(input); outputDataSet.withBroadcastSet(broadcastSet, input.getTagInternal().getId()); } }
@Override public Map<String, SideInput> translateSideInputs(SdkComponents components) { Map<String, SideInput> sideInputs = new HashMap<>(); for (PCollectionView<?> view : transform.getSink().getDynamicDestinations().getSideInputs()) { sideInputs.put( view.getTagInternal().getId(), ParDoTranslation.translateView(view, components)); } return sideInputs; }
private static void transformSideInputs( List<PCollectionView<?>> sideInputs, SingleInputUdfOperator<?, ?, ?> outputDataSet, FlinkBatchTranslationContext context) { // get corresponding Flink broadcast DataSets for (PCollectionView<?> input : sideInputs) { DataSet<?> broadcastSet = context.getSideInputDataSet(input); outputDataSet.withBroadcastSet(broadcastSet, input.getTagInternal().getId()); } }
private static void transformSideInputs( List<PCollectionView<?>> sideInputs, SingleInputUdfOperator<?, ?, ?> outputDataSet, FlinkBatchTranslationContext context) { // get corresponding Flink broadcast DataSets for (PCollectionView<?> input : sideInputs) { DataSet<?> broadcastSet = context.getSideInputDataSet(input); outputDataSet.withBroadcastSet(broadcastSet, input.getTagInternal().getId()); } }
@Test public void testStaticTupleTag() { assertEquals("org.apache.beam.sdk.values.TupleTagTest#0", staticTag.getId()); assertEquals("org.apache.beam.sdk.values.TupleTagTest#3", staticBlockTag.getId()); assertEquals("org.apache.beam.sdk.values.TupleTagTest#1", staticMethodTag.getId()); assertEquals("org.apache.beam.sdk.values.TupleTagTest#2", instanceMethodTag.getId()); assertEquals( "org.apache.beam.sdk.values.TupleTagTest$AnotherClass#0", AnotherClass.anotherTag.getId()); }
private CloudObject toCloudObject(CoGbkResultSchema schema) { CloudObject result = CloudObject.forClass(CoGbkResultSchema.class); List<CloudObject> tags = new ArrayList<>(schema.getTupleTagList().size()); for (TupleTag<?> tag : schema.getTupleTagList().getAll()) { CloudObject tagCloudObject = CloudObject.forClass(TupleTag.class); Structs.addString(tagCloudObject, PropertyNames.VALUE, tag.getId()); tags.add(tagCloudObject); } Structs.addList(result, PropertyNames.TUPLE_TAGS, tags); return result; }
private void outputToAllWithSideInputs(ProcessContext c, String value) { if (!sideInputViews.isEmpty()) { List<Integer> sideInputValues = new ArrayList<>(); for (PCollectionView<Integer> sideInputView : sideInputViews) { sideInputValues.add(c.sideInput(sideInputView)); } value += ": " + sideInputValues; } c.output(value); for (TupleTag<String> additionalOutputTupleTag : additionalOutputTupleTags) { c.output(additionalOutputTupleTag, additionalOutputTupleTag.getId() + ": " + value); } } }
public SideInputInitializer(PCollectionView<ViewT> view) { checkArgument( Materializations.MULTIMAP_MATERIALIZATION_URN.equals( view.getViewFn().getMaterialization().getUrn()), "This handler is only capable of dealing with %s materializations " + "but was asked to handle %s for PCollectionView with tag %s.", Materializations.MULTIMAP_MATERIALIZATION_URN, view.getViewFn().getMaterialization().getUrn(), view.getTagInternal().getId()); this.view = view; }
public SideInputInitializer(PCollectionView<ViewT> view) { checkArgument( Materializations.MULTIMAP_MATERIALIZATION_URN.equals( view.getViewFn().getMaterialization().getUrn()), "This handler is only capable of dealing with %s materializations " + "but was asked to handle %s for PCollectionView with tag %s.", Materializations.MULTIMAP_MATERIALIZATION_URN, view.getViewFn().getMaterialization().getUrn(), view.getTagInternal().getId()); this.view = view; }
@Test public void testConstructor() throws Exception { TupleTag tag = InstanceBuilder.ofType(TupleTag.class).withArg(String.class, "hello world!").build(); Assert.assertEquals("hello world!", tag.getId()); }
@Test public void testFullNameLookup() throws Exception { TupleTag tag = InstanceBuilder.ofType(TupleTag.class) .fromClassName(InstanceBuilderTest.class.getName()) .fromFactoryMethod("createTag") .withArg(String.class, "hello world!") .build(); Assert.assertEquals("hello world!", tag.getId()); }
@Test public void testToProto() throws Exception { SdkComponents components = SdkComponents.create(); components.registerEnvironment(Environments.createDockerEnvironment("java")); ParDoPayload payload = ParDoTranslation.translateParDo(parDo, p, components); assertThat(ParDoTranslation.getDoFn(payload), equalTo(parDo.getFn())); assertThat(ParDoTranslation.getMainOutputTag(payload), equalTo(parDo.getMainOutputTag())); for (PCollectionView<?> view : parDo.getSideInputs()) { payload.getSideInputsOrThrow(view.getTagInternal().getId()); } }