@Override public <T> boolean contains(PCollectionView<T> view) { return sideInputs.containsKey(view.getTagInternal()); }
@Override public <T> boolean contains(PCollectionView<T> view) { return sideInputs.containsKey(view.getTagInternal()); }
@Override public <T> boolean contains(PCollectionView<T> view) { return sideInputs.containsKey(view.getTagInternal()); }
@Override public <T> boolean contains(PCollectionView<T> view) { return sideInputs.containsKey(view.getTagInternal()); }
public FlinkSideInputReader(Map<PCollectionView<?>, WindowingStrategy<?, ?>> indexByView, RuntimeContext runtimeContext) { sideInputs = new HashMap<>(); for (Map.Entry<PCollectionView<?>, WindowingStrategy<?, ?>> entry : indexByView.entrySet()) { sideInputs.put(entry.getKey().getTagInternal(), entry.getValue()); } this.runtimeContext = runtimeContext; }
@Override public boolean equals(Object other) { if (!(other instanceof PCollectionView)) { return false; } @SuppressWarnings("unchecked") PCollectionView<?> otherView = (PCollectionView<?>) other; return tag.equals(otherView.getTagInternal()); }
@Override public boolean equals(Object other) { if (!(other instanceof PCollectionView)) { return false; } @SuppressWarnings("unchecked") PCollectionView<?> otherView = (PCollectionView<?>) other; return tag.equals(otherView.getTagInternal()); }
public static Map<String, SideInput> translateSideInputs( List<PCollectionView<?>> views, SdkComponents components) { Map<String, SideInput> sideInputs = new HashMap<>(); for (PCollectionView<?> sideInput : views) { sideInputs.put( sideInput.getTagInternal().getId(), ParDoTranslation.translateView(sideInput, components)); } return sideInputs; }
@Override public Map<String, SideInput> translateSideInputs(SdkComponents components) { Map<String, SideInput> sideInputs = new HashMap<>(); for (PCollectionView<?> sideInput : parDo.getSideInputs()) { sideInputs.put( sideInput.getTagInternal().getId(), translateView(sideInput, components)); } return sideInputs; }
private static void translateSideInputs( StepTranslationContext stepContext, List<PCollectionView<?>> sideInputs, TranslationContext context) { Map<String, Object> nonParInputs = new HashMap<>(); for (PCollectionView<?> view : sideInputs) { nonParInputs.put( view.getTagInternal().getId(), context.asOutputReference(view, context.getProducer(view))); } stepContext.addInput(PropertyNames.NON_PARALLEL_INPUTS, nonParInputs); }
private static void transformSideInputs( List<PCollectionView<?>> sideInputs, SingleInputUdfOperator<?, ?, ?> outputDataSet, FlinkBatchTranslationContext context) { // get corresponding Flink broadcast DataSets for (PCollectionView<?> input : sideInputs) { DataSet<?> broadcastSet = context.getSideInputDataSet(input); outputDataSet.withBroadcastSet(broadcastSet, input.getTagInternal().getId()); } }
@Override public Map<String, SideInput> translateSideInputs(SdkComponents components) { Map<String, SideInput> sideInputs = new HashMap<>(); for (PCollectionView<?> view : transform.getSink().getDynamicDestinations().getSideInputs()) { sideInputs.put( view.getTagInternal().getId(), ParDoTranslation.translateView(view, components)); } return sideInputs; }
private static void transformSideInputs( List<PCollectionView<?>> sideInputs, SingleInputUdfOperator<?, ?, ?> outputDataSet, FlinkBatchTranslationContext context) { // get corresponding Flink broadcast DataSets for (PCollectionView<?> input : sideInputs) { DataSet<?> broadcastSet = context.getSideInputDataSet(input); outputDataSet.withBroadcastSet(broadcastSet, input.getTagInternal().getId()); } }
private static void transformSideInputs( List<PCollectionView<?>> sideInputs, SingleInputUdfOperator<?, ?, ?> outputDataSet, FlinkBatchTranslationContext context) { // get corresponding Flink broadcast DataSets for (PCollectionView<?> input : sideInputs) { DataSet<?> broadcastSet = context.getSideInputDataSet(input); outputDataSet.withBroadcastSet(broadcastSet, input.getTagInternal().getId()); } }
/** * Expands a list of {@link PCollectionView} into the form needed for {@link * PTransform#getAdditionalInputs()}. */ public static Map<TupleTag<?>, PValue> toAdditionalInputs(Iterable<PCollectionView<?>> views) { ImmutableMap.Builder<TupleTag<?>, PValue> additionalInputs = ImmutableMap.builder(); for (PCollectionView<?> view : views) { additionalInputs.put(view.getTagInternal(), view.getPCollection()); } return additionalInputs.build(); }
public SideInputInitializer(PCollectionView<ViewT> view) { checkArgument( Materializations.MULTIMAP_MATERIALIZATION_URN.equals( view.getViewFn().getMaterialization().getUrn()), "This handler is only capable of dealing with %s materializations " + "but was asked to handle %s for PCollectionView with tag %s.", Materializations.MULTIMAP_MATERIALIZATION_URN, view.getViewFn().getMaterialization().getUrn(), view.getTagInternal().getId()); this.view = view; }
public SideInputInitializer(PCollectionView<ViewT> view) { checkArgument( Materializations.MULTIMAP_MATERIALIZATION_URN.equals( view.getViewFn().getMaterialization().getUrn()), "This handler is only capable of dealing with %s materializations " + "but was asked to handle %s for PCollectionView with tag %s.", Materializations.MULTIMAP_MATERIALIZATION_URN, view.getViewFn().getMaterialization().getUrn(), view.getTagInternal().getId()); this.view = view; }
/** Create a new {@link SideInputContainer} with the provided views and the provided context. */ public static SideInputContainer create( final EvaluationContext context, Collection<PCollectionView<?>> containedViews) { for (PCollectionView<?> pCollectionView : containedViews) { checkArgument( Materializations.MULTIMAP_MATERIALIZATION_URN.equals( pCollectionView.getViewFn().getMaterialization().getUrn()), "This handler is only capable of dealing with %s materializations " + "but was asked to handle %s for PCollectionView with tag %s.", Materializations.MULTIMAP_MATERIALIZATION_URN, pCollectionView.getViewFn().getMaterialization().getUrn(), pCollectionView.getTagInternal().getId()); } LoadingCache<PCollectionViewWindow<?>, AtomicReference<Iterable<? extends WindowedValue<?>>>> viewByWindows = CacheBuilder.newBuilder().build(new CallbackSchedulingLoader(context)); return new SideInputContainer(containedViews, viewByWindows); }
@Test public void testToProto() throws Exception { SdkComponents components = SdkComponents.create(); components.registerEnvironment(Environments.createDockerEnvironment("java")); ParDoPayload payload = ParDoTranslation.translateParDo(parDo, p, components); assertThat(ParDoTranslation.getDoFn(payload), equalTo(parDo.getFn())); assertThat(ParDoTranslation.getMainOutputTag(payload), equalTo(parDo.getMainOutputTag())); for (PCollectionView<?> view : parDo.getSideInputs()) { payload.getSideInputsOrThrow(view.getTagInternal().getId()); } }
@Test public void getMainInputSingleOutputSideInputs() { AppliedPTransform<PCollection<Long>, ?, ?> application = AppliedPTransform.of( "application", ImmutableMap.<TupleTag<?>, PValue>builder() .put(new TupleTag<Long>(), mainInput) .put(sideInput.getTagInternal(), sideInput.getPCollection()) .build(), Collections.singletonMap(new TupleTag<Long>(), output), ParDo.of(new TestDoFn()).withSideInputs(sideInput), pipeline); PCollection<Long> input = PTransformReplacements.getSingletonMainInput(application); assertThat(input, equalTo(mainInput)); }