@Nullable @Override public <T> T get(final PCollectionView<T> view, final BoundedWindow window) { // This gets called after isReady() final T sideInputData = (T) inMemorySideInputs.get(Pair.of(view, window)); return sideInputData == null // The upstream gave us an empty sideInput ? ((ViewFn<Object, T>) view.getViewFn()).apply(new CreateViewTransform.MultiView<T>(Collections.emptyList())) // The upstream gave us a concrete sideInput : sideInputData; }
public static PTransformMatcher createViewWithViewFn(final Class<? extends ViewFn> viewFnType) { return application -> { if (!(application.getTransform() instanceof CreatePCollectionView)) { return false; } CreatePCollectionView<?, ?> createView = (CreatePCollectionView<?, ?>) application.getTransform(); ViewFn<?, ?> viewFn = createView.getView().getViewFn(); return viewFn.getClass().equals(viewFnType); }; }
(List<WindowedValue<?>>) (List<?>) elements.getValue(); resultMap.put(elements.getKey(), view.getViewFn().apply(elementsIterable));
partitionedElements.entrySet()) { ViewFn<MultimapView, ViewT> viewFn = (ViewFn<MultimapView, ViewT>) view.getViewFn(); Coder keyCoder = ((KvCoder<?, ?>) view.getCoderInternal()).getKeyCoder(); resultMap.put(
public FlinkSideInputReader( Map<PCollectionView<?>, WindowingStrategy<?, ?>> indexByView, RuntimeContext runtimeContext) { for (PCollectionView<?> view : indexByView.keySet()) { checkArgument( Materializations.MULTIMAP_MATERIALIZATION_URN.equals( view.getViewFn().getMaterialization().getUrn()), "This handler is only capable of dealing with %s materializations " + "but was asked to handle %s for PCollectionView with tag %s.", Materializations.MULTIMAP_MATERIALIZATION_URN, view.getViewFn().getMaterialization().getUrn(), view.getTagInternal().getId()); } sideInputs = new HashMap<>(); for (Map.Entry<PCollectionView<?>, WindowingStrategy<?, ?>> entry : indexByView.entrySet()) { sideInputs.put(entry.getKey().getTagInternal(), entry.getValue()); } this.runtimeContext = runtimeContext; }
public FlinkSideInputReader( Map<PCollectionView<?>, WindowingStrategy<?, ?>> indexByView, RuntimeContext runtimeContext) { for (PCollectionView<?> view : indexByView.keySet()) { checkArgument( Materializations.MULTIMAP_MATERIALIZATION_URN.equals( view.getViewFn().getMaterialization().getUrn()), "This handler is only capable of dealing with %s materializations " + "but was asked to handle %s for PCollectionView with tag %s.", Materializations.MULTIMAP_MATERIALIZATION_URN, view.getViewFn().getMaterialization().getUrn(), view.getTagInternal().getId()); } sideInputs = new HashMap<>(); for (Map.Entry<PCollectionView<?>, WindowingStrategy<?, ?>> entry : indexByView.entrySet()) { sideInputs.put(entry.getKey().getTagInternal(), entry.getValue()); } this.runtimeContext = runtimeContext; }
public SideInputInitializer(PCollectionView<ViewT> view) { checkArgument( Materializations.MULTIMAP_MATERIALIZATION_URN.equals( view.getViewFn().getMaterialization().getUrn()), "This handler is only capable of dealing with %s materializations " + "but was asked to handle %s for PCollectionView with tag %s.", Materializations.MULTIMAP_MATERIALIZATION_URN, view.getViewFn().getMaterialization().getUrn(), view.getTagInternal().getId()); this.view = view; }
public SideInputInitializer(PCollectionView<ViewT> view) { checkArgument( Materializations.MULTIMAP_MATERIALIZATION_URN.equals( view.getViewFn().getMaterialization().getUrn()), "This handler is only capable of dealing with %s materializations " + "but was asked to handle %s for PCollectionView with tag %s.", Materializations.MULTIMAP_MATERIALIZATION_URN, view.getViewFn().getMaterialization().getUrn(), view.getTagInternal().getId()); this.view = view; }
@Nullable @Override public <T> T get(PCollectionView<T> view, BoundedWindow window) { Iterable<?> elements = getIterable(view, window); // TODO: Add support for choosing which representation is contained based upon the // side input materialization. We currently can assume that we always have a multimap // materialization as that is the only supported type within the Java SDK. ViewFn<MultimapView, T> viewFn = (ViewFn<MultimapView, T>) view.getViewFn(); Coder<?> keyCoder = ((KvCoder<?, ?>) view.getCoderInternal()).getKeyCoder(); return (T) viewFn.apply(InMemoryMultimapSideInputView.fromIterable(keyCoder, (Iterable) elements)); }
@Nullable @Override public <T> T get(PCollectionView<T> view, BoundedWindow window) { checkNotNull(view, "View passed to sideInput cannot be null"); TupleTag<Iterable<WindowedValue<?>>> tag = view.getTagInternal(); checkNotNull( sideInputs.get(tag), "Side input for " + view + " not available."); Map<BoundedWindow, T> sideInputs = runtimeContext.getBroadcastVariableWithInitializer( tag.getId(), new SideInputInitializer<>(view)); T result = sideInputs.get(window); if (result == null) { result = view.getViewFn().apply(Collections.<WindowedValue<?>>emptyList()); } return result; }
@Nullable @Override public <T> T get(PCollectionView<T> view, BoundedWindow window) { checkNotNull(view, "View passed to sideInput cannot be null"); TupleTag<?> tag = view.getTagInternal(); checkNotNull(sideInputs.get(tag), "Side input for " + view + " not available."); Map<BoundedWindow, T> sideInputs = runtimeContext.getBroadcastVariableWithInitializer( tag.getId(), new SideInputInitializer<>(view)); T result = sideInputs.get(window); if (result == null) { ViewFn<MultimapView, T> viewFn = (ViewFn<MultimapView, T>) view.getViewFn(); result = viewFn.apply(EMPTY_MULTMAP_VIEW); } return result; }
@Nullable @Override public <T> T get(PCollectionView<T> view, BoundedWindow window) { checkNotNull(view, "View passed to sideInput cannot be null"); TupleTag<?> tag = view.getTagInternal(); checkNotNull(sideInputs.get(tag), "Side input for " + view + " not available."); Map<BoundedWindow, T> sideInputs = runtimeContext.getBroadcastVariableWithInitializer( tag.getId(), new SideInputInitializer<>(view)); T result = sideInputs.get(window); if (result == null) { ViewFn<MultimapView, T> viewFn = (ViewFn<MultimapView, T>) view.getViewFn(); result = viewFn.apply(EMPTY_MULTMAP_VIEW); } return result; }
.orElseThrow(() -> new RuntimeException(String.format("No incoming PCollection to %s", src))) .getCoder(); final ViewFn viewFn = view.getViewFn(); if (viewFn instanceof PCollectionViews.IterableViewFn) { return IterableCoder.of(inputKVCoder.getValueCoder());
/** Create a new {@link SideInputContainer} with the provided views and the provided context. */ public static SideInputContainer create( final EvaluationContext context, Collection<PCollectionView<?>> containedViews) { for (PCollectionView<?> pCollectionView : containedViews) { checkArgument( Materializations.MULTIMAP_MATERIALIZATION_URN.equals( pCollectionView.getViewFn().getMaterialization().getUrn()), "This handler is only capable of dealing with %s materializations " + "but was asked to handle %s for PCollectionView with tag %s.", Materializations.MULTIMAP_MATERIALIZATION_URN, pCollectionView.getViewFn().getMaterialization().getUrn(), pCollectionView.getTagInternal().getId()); } LoadingCache<PCollectionViewWindow<?>, AtomicReference<Iterable<? extends WindowedValue<?>>>> viewByWindows = CacheBuilder.newBuilder().build(new CallbackSchedulingLoader(context)); return new SideInputContainer(containedViews, viewByWindows); }
/** * @param ctx provides translation context * @param beamNode the beam node to be translated * @param transform transform which can be obtained from {@code beamNode} */ @PrimitiveTransformTranslator(View.CreatePCollectionView.class) private static void createPCollectionViewTranslator(final PipelineTranslationContext ctx, final TransformHierarchy.Node beamNode, final View.CreatePCollectionView<?, ?> transform) { final IRVertex vertex = new OperatorVertex(new CreateViewTransform(transform.getView().getViewFn())); ctx.addVertex(vertex); beamNode.getInputs().values().forEach(input -> ctx.addEdgeTo(vertex, input)); ctx.registerMainOutputFrom(beamNode, vertex, transform.getView()); beamNode.getOutputs().values().forEach(output -> ctx.registerMainOutputFrom(beamNode, vertex, output)); }
public static SideInput translateView(PCollectionView<?> view, SdkComponents components) { Builder builder = SideInput.newBuilder(); builder.setAccessPattern( FunctionSpec.newBuilder().setUrn(view.getViewFn().getMaterialization().getUrn()).build()); builder.setViewFn(translateViewFn(view.getViewFn(), components)); builder.setWindowMappingFn(translateWindowMappingFn(view.getWindowMappingFn(), components)); return builder.build(); }
@Override @Nullable public <T> T get(final PCollectionView<T> view, final BoundedWindow window) { checkArgument( readerViews.contains(view), "call to get(PCollectionView) with unknown view: %s", view); checkArgument( isReady(view, window), "calling get() on PCollectionView %s that is not ready in window %s", view, window); // Safe covariant cast since we know that the view only contains KVs. @SuppressWarnings("unchecked") Iterable<KV<?, ?>> elements = Iterables.transform( (Iterable<WindowedValue<KV<?, ?>>>) viewContents.getUnchecked(PCollectionViewWindow.of(view, window)).get(), WindowedValue::getValue); ViewFn<MultimapView, T> viewFn = (ViewFn<MultimapView, T>) view.getViewFn(); Coder<?> keyCoder = ((KvCoder<?, ?>) view.getCoderInternal()).getKeyCoder(); return (T) viewFn.apply(InMemoryMultimapSideInputView.fromIterable(keyCoder, (Iterable) elements)); }
@Override public void visitPrimitiveTransform(Node node) { if (node.getTransform() instanceof WriteView) { assertThat( "There should only be one WriteView primitive in the graph", writeViewVisited.getAndSet(true), is(false)); PCollectionView<?> replacementView = ((WriteView) node.getTransform()).getView(); // replacementView.getPCollection() is null, but that is not a requirement // so not asserted one way or the other assertThat( replacementView.getTagInternal(), equalTo((TupleTag) view.getTagInternal())); assertThat(replacementView.getViewFn(), equalTo(view.getViewFn())); assertThat(replacementView.getWindowMappingFn(), equalTo(view.getWindowMappingFn())); assertThat(node.getInputs().entrySet(), hasSize(1)); } } });
@Test public void createViewWithViewFnNotCreatePCollectionView() { PCollection<Integer> input = p.apply(Create.of(1)); PCollectionView<Iterable<Integer>> view = input.apply(View.asIterable()); PTransformMatcher matcher = PTransformMatchers.createViewWithViewFn(view.getViewFn().getClass()); assertThat(matcher.matches(getAppliedTransform(View.asIterable())), is(false)); }
@Test public void createViewWithViewFn() { PCollection<Integer> input = p.apply(Create.of(1)); PCollectionView<Iterable<Integer>> view = input.apply(View.asIterable()); ViewFn<?, ?> viewFn = view.getViewFn(); CreatePCollectionView<?, ?> createView = CreatePCollectionView.of(view); PTransformMatcher matcher = PTransformMatchers.createViewWithViewFn(viewFn.getClass()); assertThat(matcher.matches(getAppliedTransform(createView)), is(true)); }