@Override public OperatorVertex getClone() { return new OperatorVertex(this); }
/** * Retrieve communication pattern of the edge. * * @param src source vertex. * @param dst destination vertex. * @return the communication pattern. */ public static CommunicationPatternProperty.Value getEdgeCommunicationPattern(final IRVertex src, final IRVertex dst) { if (dst instanceof OperatorVertex && (((OperatorVertex) dst).getTransform() instanceof ReduceByKeyTransform || ((OperatorVertex) dst).getTransform() instanceof GroupByKeyTransform)) { return CommunicationPatternProperty.Value.Shuffle; } else { return CommunicationPatternProperty.Value.OneToOne; } }
@Override public ObjectNode getPropertiesAsJsonNode() { final ObjectNode node = getIRVertexPropertiesAsJsonNode(); node.put("transform", transform.toString()); return node; } }
builder.addVertex(abv); abv.setPropertyPermanently(ResourceSlotProperty.of(false)); dstVtxIdToABV.put(dstId, Pair.of(abv, metricCollectionId)); } else {
@Override public OperatorVertex getClone() { return new OperatorVertex(this); }
@Override public void emitWatermark(final Watermark watermark) { operatorVertex.getTransform().onWatermark(watermark); }
@Override public ObjectNode getPropertiesAsJsonNode() { final ObjectNode node = getIRVertexPropertiesAsJsonNode(); node.put("transform", transform.toString()); return node; } }
/** * @return the generated vertex. */ private OperatorVertex generateMetricAggregationVertex() { // Define a custom data aggregator for skew handling. // Here, the aggregator gathers key frequency data used in shuffle data repartitioning. final BiFunction<Object, Map<Object, Long>, Map<Object, Long>> dynOptDataAggregator = (BiFunction<Object, Map<Object, Long>, Map<Object, Long>> & Serializable) (element, aggregatedDynOptData) -> { final Object key = ((Pair<Object, Long>) element).left(); final Long count = ((Pair<Object, Long>) element).right(); final Map<Object, Long> aggregatedDynOptDataMap = (Map<Object, Long>) aggregatedDynOptData; if (aggregatedDynOptDataMap.containsKey(key)) { aggregatedDynOptDataMap.compute(key, (existingKey, accumulatedCount) -> accumulatedCount + count); } else { aggregatedDynOptDataMap.put(key, count); } return aggregatedDynOptData; }; final AggregateMetricTransform abt = new AggregateMetricTransform<Pair<Object, Long>, Map<Object, Long>>(new HashMap<>(), dynOptDataAggregator); return new OperatorVertex(abt); }
@Override public void emit(final O output) { nextOperatorVertex.getTransform().onData(output); }
/** * @return the generated vertex. */ private OperatorVertex generateMetricAggregationVertex() { // Define a custom data aggregator for skew handling. // Here, the aggregator gathers key frequency data used in shuffle data repartitioning. final BiFunction<Object, Map<Object, Long>, Map<Object, Long>> dynOptDataAggregator = (BiFunction<Object, Map<Object, Long>, Map<Object, Long>> & Serializable) (element, aggregatedDynOptData) -> { final Object key = ((Pair<Object, Long>) element).left(); final Long count = ((Pair<Object, Long>) element).right(); final Map<Object, Long> aggregatedDynOptDataMap = (Map<Object, Long>) aggregatedDynOptData; if (aggregatedDynOptDataMap.containsKey(key)) { aggregatedDynOptDataMap.compute(key, (existingKey, accumulatedCount) -> accumulatedCount + count); } else { aggregatedDynOptDataMap.put(key, count); } return aggregatedDynOptData; }; final AggregateMetricTransform abt = new AggregateMetricTransform<Pair<Object, Long>, Map<Object, Long>>(new HashMap<>(), dynOptDataAggregator); return new OperatorVertex(abt); }
@Override public void emitWatermark(final Watermark watermark) { operatorVertex.getTransform().onWatermark(watermark); }
/** * @param ctx provides translation context * @param beamNode the beam node to be translated * @param transform transform which can be obtained from {@code beamNode} */ @PrimitiveTransformTranslator(GroupByKey.class) private static void groupByKeyTranslator(final PipelineTranslationContext ctx, final TransformHierarchy.Node beamNode, final GroupByKey<?, ?> transform) { final IRVertex vertex = new OperatorVertex(createGBKTransform(ctx, beamNode)); ctx.addVertex(vertex); beamNode.getInputs().values().forEach(input -> ctx.addEdgeTo(vertex, input)); beamNode.getOutputs().values().forEach(output -> ctx.registerMainOutputFrom(beamNode, vertex, output)); }
@Override public void emit(final O output) { nextOperatorVertex.getTransform().onData(output); }
return new OperatorVertex(mct);
private void emit(final OperatorVertex vertex, final O output) { vertex.getTransform().onData(output); }
return new OperatorVertex(mct);
private void emit(final OperatorVertex vertex, final O output) { vertex.getTransform().onData(output); }
/** * @param ctx provides translation context * @param beamNode the beam node to be translated * @param transform transform which can be obtained from {@code beamNode} */ @PrimitiveTransformTranslator(Flatten.PCollections.class) private static void flattenTranslator(final PipelineTranslationContext ctx, final TransformHierarchy.Node beamNode, final Flatten.PCollections<?> transform) { final IRVertex vertex = new OperatorVertex(new FlattenTransform()); ctx.addVertex(vertex); beamNode.getInputs().values().forEach(input -> ctx.addEdgeTo(vertex, input)); beamNode.getOutputs().values().forEach(output -> ctx.registerMainOutputFrom(beamNode, vertex, output)); }
/** * @param src source IR vertex. * @param dst destination IR vertex. * @return the communication pattern property value. */ private CommunicationPatternProperty.Value getCommPattern(final IRVertex src, final IRVertex dst) { final Class<?> constructUnionTableFn; try { constructUnionTableFn = Class.forName("org.apache.beam.sdk.transforms.join.CoGroupByKey$ConstructUnionTableFn"); } catch (final ClassNotFoundException e) { throw new RuntimeException(e); } final Transform srcTransform = src instanceof OperatorVertex ? ((OperatorVertex) src).getTransform() : null; final Transform dstTransform = dst instanceof OperatorVertex ? ((OperatorVertex) dst).getTransform() : null; final DoFn srcDoFn = srcTransform instanceof DoFnTransform ? ((DoFnTransform) srcTransform).getDoFn() : null; if (srcDoFn != null && srcDoFn.getClass().equals(constructUnionTableFn)) { return CommunicationPatternProperty.Value.Shuffle; } if (srcTransform instanceof FlattenTransform) { return CommunicationPatternProperty.Value.OneToOne; } if (dstTransform instanceof GroupByKeyAndWindowDoFnTransform || dstTransform instanceof GroupByKeyTransform) { return CommunicationPatternProperty.Value.Shuffle; } if (dstTransform instanceof CreateViewTransform) { return CommunicationPatternProperty.Value.BroadCast; } return CommunicationPatternProperty.Value.OneToOne; }
/** * @param ctx provides translation context * @param beamNode the beam node to be translated * @param transform transform which can be obtained from {@code beamNode} */ @PrimitiveTransformTranslator(ParDo.SingleOutput.class) private static void parDoSingleOutputTranslator(final PipelineTranslationContext ctx, final TransformHierarchy.Node beamNode, final ParDo.SingleOutput<?, ?> transform) { final Map<Integer, PCollectionView<?>> sideInputMap = getSideInputMap(transform.getSideInputs()); final AbstractDoFnTransform doFnTransform = createDoFnTransform(ctx, beamNode, sideInputMap); final IRVertex vertex = new OperatorVertex(doFnTransform); ctx.addVertex(vertex); beamNode.getInputs().values().stream() .filter(input -> !transform.getAdditionalInputs().values().contains(input)) .forEach(input -> ctx.addEdgeTo(vertex, input)); ctx.addSideInputEdges(vertex, sideInputMap); beamNode.getOutputs().values().forEach(output -> ctx.registerMainOutputFrom(beamNode, vertex, output)); }