@Test public void testConfigurationOfParallelism() { Operator operator = new MockOperator(); // verify explicit change in parallelism int parallelism = 36; operator.setParallelism(parallelism); assertEquals(parallelism, operator.getParallelism()); // verify that parallelism is reset to default flag value parallelism = ExecutionConfig.PARALLELISM_DEFAULT; operator.setParallelism(parallelism); assertEquals(parallelism, operator.getParallelism()); }
private DataSet<Tuple> translateMerge(List<DataSet<Tuple>> inputs, FlowNode node) { DataSet<Tuple> unioned = null; TypeInformation<Tuple> type = null; int maxDop = -1; for(DataSet<Tuple> input : inputs) { maxDop = Math.max(maxDop, ((Operator)input).getParallelism()); if(unioned == null) { unioned = input; type = input.getType(); } else { unioned = unioned.union(input); } } return unioned.map(new IdMapper()) .returns(type) .setParallelism(maxDop); }
private DataSet<Tuple2<Tuple, Tuple[]>> prepareInnerCrossInput(List<DataSet<Tuple>> inputs, FlowNode node, Fields[] inputFields, int dop) { int numJoinInputs = inputs.size(); TypeInformation<Tuple2<Tuple, Tuple[]>> tupleJoinListsTypeInfo = new org.apache.flink.api.java.typeutils.TupleTypeInfo<>( new TupleTypeInfo(Fields.UNKNOWN), new TupleArrayTypeInfo(numJoinInputs, Arrays.copyOf(inputFields, 1)) ); int mapDop = ((Operator)inputs.get(0)).getParallelism(); // prepare tuple list for join DataSet<Tuple2<Tuple, Tuple[]>> tupleJoinLists = inputs.get(0) .map(new JoinPrepareMapper(numJoinInputs, null, null)) .returns(tupleJoinListsTypeInfo) .setParallelism(mapDop) .name("coGroup-" + node.getID()); for (int i = 1; i < inputs.size(); i++) { tupleJoinListsTypeInfo = new org.apache.flink.api.java.typeutils.TupleTypeInfo<>( new TupleTypeInfo(Fields.UNKNOWN), new TupleArrayTypeInfo(numJoinInputs, Arrays.copyOf(inputFields, i+1)) ); tupleJoinLists = tupleJoinLists.crossWithTiny(inputs.get(i)) .with(new TupleAppendCrosser(i)) .returns(tupleJoinListsTypeInfo) .setParallelism(dop) .name("coGroup-" + node.getID()); } return tupleJoinLists; }
private DataSet<Tuple> translateMap(DataSet<Tuple> input, FlowNode node) { Fields outFields = getOutScope(node).getOutValuesFields(); registerKryoTypes(outFields); int dop = ((Operator)input).getParallelism(); return input .mapPartition(new EachMapper(node)) .returns(new TupleTypeInfo(outFields)) .withParameters(this.getFlinkNodeConfig(node)) .setParallelism(dop) .name("map-" + node.getID()); }
); int inputDop = ((Operator)input).getParallelism();
int probeSideDOP = ((Operator)inputs.get(0)).getParallelism(); ); int mapDop = ((Operator) inputs.get(0)).getParallelism();
int probeSideDOP = ((Operator)inputs.get(0)).getParallelism();
int probeSideDOP = ((Operator)inputs.get(0)).getParallelism();
int inputDop = ((Operator)input).getParallelism(); int dop;