private static boolean isCustomPartitioned(List<StreamEdge> edges) { boolean result = true; for (StreamEdge edge: edges) { if (!(edge.getPartitioner() instanceof CustomPartitionerWrapper)) { result = false; } } return result; }
private static boolean isPartitioned(List<StreamEdge> edges) { boolean result = true; for (StreamEdge edge: edges) { if (!(edge.getPartitioner() instanceof KeyGroupStreamPartitioner)) { result = false; } } return result; }
for (StreamEdge output : getNonChainedOutputs(cl)) { int outputname = output.getTargetId(); builder.append("\n\t").append(outputname).append(": ").append(output.getPartitioner());
private void connect(Integer headOfChain, StreamEdge edge) { physicalEdgesInOrder.add(edge); Integer downStreamvertexID = edge.getTargetId(); JobVertex headVertex = jobVertices.get(headOfChain); JobVertex downStreamVertex = jobVertices.get(downStreamvertexID); StreamConfig downStreamConfig = new StreamConfig(downStreamVertex.getConfiguration()); downStreamConfig.setNumberOfInputs(downStreamConfig.getNumberOfInputs() + 1); StreamPartitioner<?> partitioner = edge.getPartitioner(); JobEdge jobEdge; if (partitioner instanceof ForwardPartitioner || partitioner instanceof RescalePartitioner) { jobEdge = downStreamVertex.connectNewDataSetAsInput( headVertex, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED_BOUNDED); } else { jobEdge = downStreamVertex.connectNewDataSetAsInput( headVertex, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED_BOUNDED); } // set strategy name so that web interface can show it. jobEdge.setShipStrategyName(partitioner.toString()); if (LOG.isDebugEnabled()) { LOG.debug("CONNECTED: {} - {} -> {}", partitioner.getClass().getSimpleName(), headOfChain, downStreamvertexID); } }
private void decorateEdge(ArrayNode inputArray, StreamEdge inEdge, int mappedInputID) { ObjectNode input = mapper.createObjectNode(); inputArray.add(input); input.put(ID, mappedInputID); input.put(SHIP_STRATEGY, inEdge.getPartitioner().toString()); input.put(SIDE, (inputArray.size() == 0) ? "first" : "second"); }
private static <OUT> RecordWriter<SerializationDelegate<StreamRecord<OUT>>> createRecordWriter( StreamEdge edge, int outputIndex, Environment environment, String taskName, long bufferTimeout) { @SuppressWarnings("unchecked") StreamPartitioner<OUT> outputPartitioner = (StreamPartitioner<OUT>) edge.getPartitioner(); LOG.debug("Using partitioner {} for output {} of task {}", outputPartitioner, outputIndex, taskName); ResultPartitionWriter bufferWriter = environment.getWriter(outputIndex); // we initialize the partitioner here with the number of key groups (aka max. parallelism) if (outputPartitioner instanceof ConfigurableStreamPartitioner) { int numKeyGroups = bufferWriter.getNumTargetKeyGroups(); if (0 < numKeyGroups) { ((ConfigurableStreamPartitioner) outputPartitioner).configure(numKeyGroups); } } RecordWriter<SerializationDelegate<StreamRecord<OUT>>> output = RecordWriter.createRecordWriter(bufferWriter, outputPartitioner, bufferTimeout, taskName); output.setMetricGroup(environment.getMetricGroup().getIOMetricGroup()); return output; } }
public static boolean isChainable(StreamEdge edge, StreamGraph streamGraph) { StreamNode upStreamVertex = streamGraph.getSourceVertex(edge); StreamNode downStreamVertex = streamGraph.getTargetVertex(edge); StreamOperator<?> headOperator = upStreamVertex.getOperator(); StreamOperator<?> outOperator = downStreamVertex.getOperator(); return downStreamVertex.getInEdges().size() == 1 && outOperator != null && headOperator != null && upStreamVertex.isSameSlotSharingGroup(downStreamVertex) && outOperator.getChainingStrategy() == ChainingStrategy.ALWAYS && (headOperator.getChainingStrategy() == ChainingStrategy.HEAD || headOperator.getChainingStrategy() == ChainingStrategy.ALWAYS) && (edge.getPartitioner() instanceof ForwardPartitioner) && upStreamVertex.getParallelism() == downStreamVertex.getParallelism() && streamGraph.isChainingEnabled(); }
private boolean isChainable(StreamEdge edge, boolean isChainingEnabled, StreamGraph streamGraph) { StreamNode upStreamVertex = streamGraph.getSourceVertex(edge); StreamNode downStreamVertex = streamGraph.getTargetVertex(edge); StreamOperator<?> headOperator = upStreamVertex.getOperator(); StreamOperator<?> outOperator = downStreamVertex.getOperator(); return downStreamVertex.getInEdges().size() == 1 && outOperator != null && headOperator != null && upStreamVertex.isSameSlotSharingGroup(downStreamVertex) && outOperator.getChainingStrategy() == ChainingStrategy.ALWAYS && (headOperator.getChainingStrategy() == ChainingStrategy.HEAD || headOperator.getChainingStrategy() == ChainingStrategy.ALWAYS) && (edge.getPartitioner() instanceof ForwardPartitioner) && upStreamVertex.getParallelism() == downStreamVertex.getParallelism() && isChainingEnabled; } }
/** * Tests that the max parallelism is properly set for connected * streams. */ @Test public void testMaxParallelismWithConnectedKeyedStream() { int maxParallelism = 42; StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Integer> input1 = env.fromElements(1, 2, 3, 4).setMaxParallelism(128); DataStream<Integer> input2 = env.fromElements(1, 2, 3, 4).setMaxParallelism(129); env.getConfig().setMaxParallelism(maxParallelism); DataStream<Integer> keyedResult = input1 .connect(input2) .keyBy(value -> value, value -> value) .map(new NoOpIntCoMap()); keyedResult.addSink(new DiscardingSink<>()); StreamGraph graph = env.getStreamGraph(); StreamNode keyedResultNode = graph.getStreamNode(keyedResult.getId()); StreamPartitioner<?> streamPartitioner1 = keyedResultNode.getInEdges().get(0).getPartitioner(); StreamPartitioner<?> streamPartitioner2 = keyedResultNode.getInEdges().get(1).getPartitioner(); }
StreamPartitioner<?> broadcastPartitioner = env.getStreamGraph().getStreamEdges(src.getId(), broadcastSink.getTransformation().getId()).get(0).getPartitioner(); assertTrue(broadcastPartitioner instanceof BroadcastPartitioner); StreamPartitioner<?> shufflePartitioner = env.getStreamGraph().getStreamEdges(src.getId(), shuffleSink.getTransformation().getId()).get(0).getPartitioner(); assertTrue(shufflePartitioner instanceof ShufflePartitioner); StreamPartitioner<?> forwardPartitioner = env.getStreamGraph().getStreamEdges(src.getId(), forwardSink.getTransformation().getId()).get(0).getPartitioner(); assertTrue(forwardPartitioner instanceof ForwardPartitioner); StreamPartitioner<?> rebalancePartitioner = env.getStreamGraph().getStreamEdges(src.getId(), rebalanceSink.getTransformation().getId()).get(0).getPartitioner(); assertTrue(rebalancePartitioner instanceof RebalancePartitioner); StreamPartitioner<?> globalPartitioner = env.getStreamGraph().getStreamEdges(src.getId(), globalSink.getTransformation().getId()).get(0).getPartitioner(); assertTrue(globalPartitioner instanceof GlobalPartitioner);
assertTrue(graph.getStreamNode(map1.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner); assertTrue(graph.getStreamNode(map1.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo")); assertTrue(graph.getStreamNode(map1.getId()).getOutputSelectors().contains(selector)); assertTrue(graph.getStreamNode(map2.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner); assertTrue(graph.getStreamNode(map2.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo")); assertTrue(graph.getStreamNode(map2.getId()).getOutputSelectors().contains(selector)); assertTrue(graph.getStreamNode(map3.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner); assertTrue(graph.getStreamNode(map3.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo")); assertTrue(graph.getStreamNode(map3.getId()).getOutputSelectors().contains(selector));
/** * Tests that the KeyGroupStreamPartitioner are properly set up with the correct value of * maximum parallelism. */ @Test public void testSetupOfKeyGroupPartitioner() { int maxParallelism = 42; StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().setMaxParallelism(maxParallelism); DataStream<Integer> source = env.fromElements(1, 2, 3); DataStream<Integer> keyedResult = source.keyBy(value -> value).map(new NoOpIntMap()); keyedResult.addSink(new DiscardingSink<>()); StreamGraph graph = env.getStreamGraph(); StreamNode keyedResultNode = graph.getStreamNode(keyedResult.getId()); StreamPartitioner<?> streamPartitioner = keyedResultNode.getInEdges().get(0).getPartitioner(); }
assertTrue(edge.getPartitioner() instanceof RebalancePartitioner); } else if (graph.getTargetVertex(edge).getOperatorName().equals("IterForwardMap")) { assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); assertTrue(edge.getPartitioner() instanceof ShufflePartitioner); assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); assertTrue(edge.getSelectedNames().contains("even"));
assertTrue(graph.getStreamNode(rebalanceMap.getId()).getInEdges().get(0).getPartitioner() instanceof RebalancePartitioner); assertTrue(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0).getPartitioner() instanceof BroadcastPartitioner); assertEquals(rebalanceMap.getId(), graph.getSourceVertex(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0)).getId()); assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner); assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even")); assertTrue(graph.getStreamNode(map1Operator.getId()).getOutputSelectors().contains(selector1)); assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof GlobalPartitioner); assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("odd")); assertTrue(graph.getStreamNode(map2Operator.getId()).getOutputSelectors().contains(selector2)); assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof ShufflePartitioner); assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even")); assertTrue(graph.getStreamNode(map3Operator.getId()).getOutputSelectors().contains(selector3));
assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); assertEquals(4, graph.getTargetVertex(edge).getParallelism()); } else if (graph.getTargetVertex(edge).getOperatorName().equals("shuffle")) { assertTrue(edge.getPartitioner() instanceof RebalancePartitioner); assertEquals(2, graph.getTargetVertex(edge).getParallelism()); String tailName = graph.getSourceVertex(edge).getOperatorName(); if (tailName.equals("split")) { assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); assertTrue(edge.getSelectedNames().contains("even")); } else if (tailName.equals("bc")) { assertTrue(edge.getPartitioner() instanceof BroadcastPartitioner); } else if (tailName.equals("shuffle")) { assertTrue(edge.getPartitioner() instanceof ShufflePartitioner);
assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); boolean hasBroadcast = false; for (StreamEdge edge: streamGraph.getStreamNode(selfUnionDifferentPartition.getId()).getInEdges()) { if (edge.getPartitioner() instanceof ForwardPartitioner) { hasForward = true; if (edge.getPartitioner() instanceof BroadcastPartitioner) { hasBroadcast = true; for (StreamEdge edge: streamGraph.getStreamNode(unionDifferingParallelism.getId()).getInEdges()) { if (edge.getSourceId() == input2.getId()) { assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); } else if (edge.getSourceId() == input3.getId()) { assertTrue(edge.getPartitioner() instanceof RebalancePartitioner); } else { fail("Wrong input edge."); for (StreamEdge edge: streamGraph.getStreamNode(unionDifferingPartitioning.getId()).getInEdges()) { if (edge.getSourceId() == input4.getId()) { assertTrue(edge.getPartitioner() instanceof BroadcastPartitioner); } else if (edge.getSourceId() == input5.getId()) { assertTrue(edge.getPartitioner() instanceof ForwardPartitioner); } else { fail("Wrong input edge.");
assertTrue(env.getStreamGraph().getStreamNode(sink.getTransformation().getId()).getInEdges().get(0).getPartitioner() instanceof ForwardPartitioner); assertNotNull(env.getStreamGraph().getStreamNode(sink2.getTransformation().getId()).getStateKeySerializer()); assertEquals(key1, env.getStreamGraph().getStreamNode(sink2.getTransformation().getId()).getStatePartitioner1()); assertTrue(env.getStreamGraph().getStreamNode(sink2.getTransformation().getId()).getInEdges().get(0).getPartitioner() instanceof KeyGroupStreamPartitioner); assertTrue(env.getStreamGraph().getStreamNode(sink3.getTransformation().getId()).getInEdges().get(0).getPartitioner() instanceof KeyGroupStreamPartitioner);
private void decorateEdge(ArrayNode inputArray, StreamEdge inEdge, int mappedInputID) { ObjectNode input = mapper.createObjectNode(); inputArray.add(input); input.put(ID, mappedInputID); input.put(SHIP_STRATEGY, inEdge.getPartitioner().toString()); input.put(SIDE, (inputArray.size() == 0) ? "first" : "second"); }
private void decorateEdge(ArrayNode inputArray, StreamEdge inEdge, int mappedInputID) { ObjectNode input = mapper.createObjectNode(); inputArray.add(input); input.put(ID, mappedInputID); input.put(SHIP_STRATEGY, inEdge.getPartitioner().toString()); input.put(SIDE, (inputArray.size() == 0) ? "first" : "second"); }
private void decorateEdge(JSONArray inputArray, StreamEdge inEdge, int mappedInputID) throws JSONException { JSONObject input = new JSONObject(); inputArray.put(input); input.put(ID, mappedInputID); input.put(SHIP_STRATEGY, inEdge.getPartitioner()); input.put(SIDE, (inputArray.length() == 0) ? "first" : "second"); }