@Test public void testConfigurationOfParallelism() { Operator operator = new MockOperator(); // verify explicit change in parallelism int parallelism = 36; operator.setParallelism(parallelism); assertEquals(parallelism, operator.getParallelism()); // verify that parallelism is reset to default flag value parallelism = ExecutionConfig.PARALLELISM_DEFAULT; operator.setParallelism(parallelism); assertEquals(parallelism, operator.getParallelism()); }
/** * Returns the type of the result of this operator. * * @return The result type of the operator. */ public TypeInformation<OUT> getResultType() { return getType(); }
@Test public void testConfigurationOfResource() throws Exception{ Operator operator = new MockOperator(); Method opMethod = Operator.class.getDeclaredMethod("setResources", ResourceSpec.class, ResourceSpec.class); opMethod.setAccessible(true); // verify explicit change in resources ResourceSpec minResources = ResourceSpec.newBuilder().setCpuCores(1.0).setHeapMemoryInMB(100).build(); ResourceSpec preferredResources = ResourceSpec.newBuilder().setCpuCores(2.0).setHeapMemoryInMB(200).build(); opMethod.invoke(operator, minResources, preferredResources); assertEquals(minResources, operator.getMinResources()); assertEquals(preferredResources, operator.getPreferredResources()); }
private DataSet<Tuple> translateMerge(List<DataSet<Tuple>> inputs, FlowNode node) { DataSet<Tuple> unioned = null; TypeInformation<Tuple> type = null; int maxDop = -1; for(DataSet<Tuple> input : inputs) { maxDop = Math.max(maxDop, ((Operator)input).getParallelism()); if(unioned == null) { unioned = input; type = input.getType(); } else { unioned = unioned.union(input); } } return unioned.map(new IdMapper()) .returns(type) .setParallelism(maxDop); }
.returns(BatchElement.class) .name(operator.getName() + "::map-input") .setParallelism(inputParallelism); context.getAccumulatorFactory(), context.getSettings())) .setParallelism(operator.getParallelism()) .name(operator.getName() + "::reduce");
.returns(new TypeHint<BatchElement<Window, Pair>>() {}) .name(operator.getName() + "::extract-left-key") .setParallelism(operator.getParallelism()); .returns(new TypeHint<BatchElement<Window, Pair>>() {}) .name(operator.getName() + "::extract-right-key") .setParallelism(operator.getParallelism());
@SuppressWarnings({ "rawtypes", "unchecked" }) @Test public void testSortPartitionByFieldExpression() throws Exception { /* * Test sort partition on field expression */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(4); DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env); List<Tuple1<Boolean>> result = ds .map(new IdMapper()).setParallelism(4) // parallelize input .sortPartition("f1", Order.DESCENDING) .mapPartition(new OrderCheckMapper<>(new Tuple3Checker())) .distinct().collect(); String expected = "(true)\n"; compareResultAsText(result, expected); }
/** * Tests compiler fail for join program with replicated data source behind map and changing parallelism. */ @Test(expected = CompilerException.class) public void checkJoinWithReplicatedSourceInputBehindMapChangingparallelism() { ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(); env.setParallelism(DEFAULT_PARALLELISM); TupleTypeInfo<Tuple1<String>> typeInfo = TupleTypeInfo.getBasicTupleTypeInfo(String.class); ReplicatingInputFormat<Tuple1<String>, FileInputSplit> rif = new ReplicatingInputFormat<Tuple1<String>, FileInputSplit>(new TupleCsvInputFormat<Tuple1<String>>(new Path("/some/path"), typeInfo)); DataSet<Tuple1<String>> source1 = env.createInput(rif, new TupleTypeInfo<Tuple1<String>>(BasicTypeInfo.STRING_TYPE_INFO)); DataSet<Tuple1<String>> source2 = env.readCsvFile("/some/otherpath").types(String.class); DataSink<Tuple2<Tuple1<String>, Tuple1<String>>> out = source1 .map(new IdMap()).setParallelism(DEFAULT_PARALLELISM+1) .join(source2).where("*").equalTo("*") .writeAsText("/some/newpath"); Plan plan = env.createProgramPlan(); // submit the plan to the compiler OptimizedPlan oPlan = compileNoStats(plan); }
@Override @SuppressWarnings("unchecked") public DataSet<?> translate(FlinkOperator<FlatMap> operator, BatchExecutorContext context) { Settings settings = context.getSettings(); FlinkAccumulatorFactory accumulatorFactory = context.getAccumulatorFactory(); DataSet<?> input = context.getSingleInputStream(operator); UnaryFunctor mapper = operator.getOriginalOperator().getFunctor(); ExtractEventTime timeAssigner = operator.getOriginalOperator().getEventTimeExtractor(); if (timeAssigner != null) { input = input.map(i -> { BatchElement wel = (BatchElement) i; wel.setTimestamp(timeAssigner.extractTimestamp(wel.getElement())); return wel; }) .returns((Class) BatchElement.class); } return input .flatMap(new BatchUnaryFunctorWrapper(mapper, accumulatorFactory, settings)) .returns((Class) BatchElement.class) .setParallelism(operator.getParallelism()) .name(operator.getName()); } }
private DataSet<Tuple2<Tuple, Tuple[]>> prepareInnerCrossInput(List<DataSet<Tuple>> inputs, FlowNode node, Fields[] inputFields, int dop) { int numJoinInputs = inputs.size(); TypeInformation<Tuple2<Tuple, Tuple[]>> tupleJoinListsTypeInfo = new org.apache.flink.api.java.typeutils.TupleTypeInfo<>( new TupleTypeInfo(Fields.UNKNOWN), new TupleArrayTypeInfo(numJoinInputs, Arrays.copyOf(inputFields, 1)) ); int mapDop = ((Operator)inputs.get(0)).getParallelism(); // prepare tuple list for join DataSet<Tuple2<Tuple, Tuple[]>> tupleJoinLists = inputs.get(0) .map(new JoinPrepareMapper(numJoinInputs, null, null)) .returns(tupleJoinListsTypeInfo) .setParallelism(mapDop) .name("coGroup-" + node.getID()); for (int i = 1; i < inputs.size(); i++) { tupleJoinListsTypeInfo = new org.apache.flink.api.java.typeutils.TupleTypeInfo<>( new TupleTypeInfo(Fields.UNKNOWN), new TupleArrayTypeInfo(numJoinInputs, Arrays.copyOf(inputFields, i+1)) ); tupleJoinLists = tupleJoinLists.crossWithTiny(inputs.get(i)) .with(new TupleAppendCrosser(i)) .returns(tupleJoinListsTypeInfo) .setParallelism(dop) .name("coGroup-" + node.getID()); } return tupleJoinLists; }
private DataSet<Tuple> translateMap(DataSet<Tuple> input, FlowNode node) { Fields outFields = getOutScope(node).getOutValuesFields(); registerKryoTypes(outFields); int dop = ((Operator)input).getParallelism(); return input .mapPartition(new EachMapper(node)) .returns(new TupleTypeInfo(outFields)) .withParameters(this.getFlinkNodeConfig(node)) .setParallelism(dop) .name("map-" + node.getID()); }
/** * Returns the type of the result of this operator. * * @return The result type of the operator. */ public TypeInformation<OUT> getResultType() { return getType(); }
); int inputDop = ((Operator)input).getParallelism();
/** * Returns the type of the result of this operator. * * @return The result type of the operator. */ public TypeInformation<OUT> getResultType() { return getType(); }
int probeSideDOP = ((Operator)inputs.get(0)).getParallelism(); ); int mapDop = ((Operator) inputs.get(0)).getParallelism();
int probeSideDOP = ((Operator)inputs.get(0)).getParallelism();
int probeSideDOP = ((Operator)inputs.get(0)).getParallelism();
int inputDop = ((Operator)input).getParallelism(); int dop;