@Test(expected = IllegalArgumentException.class) public void testThatExceptionIsThrownForOuterJoinTypeNull() throws Exception { final List<String> leftInput = Arrays.asList("foo", "bar", "foobar"); final List<String> rightInput = Arrays.asList("bar", "foobar", "foo"); baseOperator.setOuterJoinType(null); ExecutionConfig executionConfig = new ExecutionConfig(); executionConfig.disableObjectReuse(); baseOperator.executeOnCollections(leftInput, rightInput, runtimeContext, executionConfig); }
private void testOuterJoin(List<String> leftInput, List<String> rightInput, List<String> expected) throws Exception { executionConfig.disableObjectReuse(); List<String> resultSafe = baseOperator.executeOnCollections(leftInput, rightInput, runtimeContext, executionConfig); executionConfig.enableObjectReuse(); List<String> resultRegular = baseOperator.executeOnCollections(leftInput, rightInput, runtimeContext, executionConfig); assertEquals(expected, resultSafe); assertEquals(expected, resultRegular); assertTrue(joiner.opened.get()); assertTrue(joiner.closed.get()); }
@Test public void testMultiChainingWithoutObjectReuse() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().disableObjectReuse(); testMultiChaining(env); }
@Test public void testMultiChainingWithSplitWithoutObjectReuse() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().disableObjectReuse(); testMultiChainingWithSplit(env); }
private void testExecuteOnCollection(FlatMapFunction<String, String> udf, List<String> input, boolean mutableSafe) throws Exception { ExecutionConfig executionConfig = new ExecutionConfig(); if (mutableSafe) { executionConfig.disableObjectReuse(); } else { executionConfig.enableObjectReuse(); } final TaskInfo taskInfo = new TaskInfo("Test UDF", 4, 0, 4, 0); // run on collections final List<String> result = getTestFlatMapOperator(udf) .executeOnCollections(input, new RuntimeUDFContext( taskInfo, null, executionConfig, new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>(), new UnregisteredMetricsGroup()), executionConfig); Assert.assertEquals(input.size(), result.size()); Assert.assertEquals(input, result); }
executionConfig.disableObjectReuse(); List<Integer> resultSafe = base.executeOnCollections(inputData1, inputData2, null, executionConfig); executionConfig.enableObjectReuse();
@Test public void testMapPlain() { try { final MapFunction<String, Integer> parser = new MapFunction<String, Integer>() { @Override public Integer map(String value) { return Integer.parseInt(value); } }; MapOperatorBase<String, Integer, MapFunction<String, Integer>> op = new MapOperatorBase<String, Integer, MapFunction<String,Integer>>( parser, new UnaryOperatorInformation<String, Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO), "TestMapper"); List<String> input = new ArrayList<String>(asList("1", "2", "3", "4", "5", "6")); ExecutionConfig executionConfig = new ExecutionConfig(); executionConfig.disableObjectReuse(); List<Integer> resultMutableSafe = op.executeOnCollections(input, null, executionConfig); executionConfig.enableObjectReuse(); List<Integer> resultRegular = op.executeOnCollections(input, null, executionConfig); assertEquals(asList(1, 2, 3, 4, 5, 6), resultMutableSafe); assertEquals(asList(1, 2, 3, 4, 5, 6), resultRegular); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
public TestEnvironment( JobExecutor jobExecutor, int parallelism, boolean isObjectReuseEnabled, Collection<Path> jarFiles, Collection<URL> classPaths) { this.jobExecutor = Preconditions.checkNotNull(jobExecutor); this.jarFiles = Preconditions.checkNotNull(jarFiles); this.classPaths = Preconditions.checkNotNull(classPaths); setParallelism(parallelism); // disabled to improve build time getConfig().setCodeAnalysisMode(CodeAnalysisMode.DISABLE); if (isObjectReuseEnabled) { getConfig().enableObjectReuse(); } else { getConfig().disableObjectReuse(); } lastEnv = null; }
@Test public void testAdvanced() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.getConfig().disableObjectReuse();
@Before public void setupEnvironment() { TestEnvironment testEnvironment; switch(mode){ case CLUSTER: // This only works because of the quirks we built in the TestEnvironment. // We should refactor this in the future!!! testEnvironment = miniClusterResource.getTestEnvironment(); testEnvironment.getConfig().disableObjectReuse(); testEnvironment.setAsContext(); break; case CLUSTER_OBJECT_REUSE: // This only works because of the quirks we built in the TestEnvironment. // We should refactor this in the future!!! testEnvironment = miniClusterResource.getTestEnvironment(); testEnvironment.getConfig().enableObjectReuse(); testEnvironment.setAsContext(); break; case COLLECTION: new CollectionTestEnvironment().setAsContext(); break; } }
@Test public void testDataSourcePlain() { try { TestNonRichInputFormat in = new TestNonRichInputFormat(); GenericDataSourceBase<String, TestNonRichInputFormat> source = new GenericDataSourceBase<String, TestNonRichInputFormat>( in, new OperatorInformation<String>(BasicTypeInfo.STRING_TYPE_INFO), "testSource"); ExecutionConfig executionConfig = new ExecutionConfig(); executionConfig.disableObjectReuse(); List<String> resultMutableSafe = source.executeOnCollections(null, executionConfig); in.reset(); executionConfig.enableObjectReuse(); List<String> resultRegular = source.executeOnCollections(null, executionConfig); assertEquals(asList(TestIOData.NAMES), resultMutableSafe); assertEquals(asList(TestIOData.NAMES), resultRegular); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
public void testGroupedReduce(ExecutionEnvironment env) throws Exception { /* * Test ReduceCombineDriver and ReduceDriver */ LOG.info("Testing grouped reduce"); env.getConfig().enableObjectReuse(); List<Tuple2<IntValue, IntValue>> enabledResult = getDataSet(env) .groupBy(0) .reduce(new OverwriteObjectsReduce(true)) .collect(); Collections.sort(enabledResult, comparator); env.getConfig().disableObjectReuse(); List<Tuple2<IntValue, IntValue>> disabledResult = getDataSet(env) .groupBy(0) .reduce(new OverwriteObjectsReduce(true)) .collect(); Collections.sort(disabledResult, comparator); Assert.assertThat(disabledResult, is(enabledResult)); }
@Test public void testDataSourcePlain() { try { TestNonRichOutputFormat out = new TestNonRichOutputFormat(); GenericDataSinkBase<String> sink = new GenericDataSinkBase<String>( out, new UnaryOperatorInformation<String, Nothing>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.getInfoFor(Nothing.class)), "test_sink"); sink.setInput(source); ExecutionConfig executionConfig = new ExecutionConfig(); executionConfig.disableObjectReuse(); in.reset(); sink.executeOnCollections(asList(TestIOData.NAMES), null, executionConfig); assertEquals(out.output, asList(TestIOData.NAMES)); executionConfig.enableObjectReuse(); out.clear(); in.reset(); sink.executeOnCollections(asList(TestIOData.NAMES), null, executionConfig); assertEquals(out.output, asList(TestIOData.NAMES)); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
final HashMap<String, Future<Path>> cpTasks = new HashMap<>(); final TaskInfo taskInfo = new TaskInfo("test_sink", 1, 0, 1, 0); executionConfig.disableObjectReuse(); in.reset();
public void testReduce(ExecutionEnvironment env) throws Exception { /* * Test ChainedAllReduceDriver */ LOG.info("Testing reduce"); env.getConfig().enableObjectReuse(); Tuple2<IntValue, IntValue> enabledResult = getDataSet(env) .reduce(new OverwriteObjectsReduce(false)) .collect() .get(0); env.getConfig().disableObjectReuse(); Tuple2<IntValue, IntValue> disabledResult = getDataSet(env) .reduce(new OverwriteObjectsReduce(false)) .collect() .get(0); Assert.assertEquals(NUMBER_OF_ELEMENTS, enabledResult.f1.getValue()); Assert.assertEquals(NUMBER_OF_ELEMENTS, disabledResult.f1.getValue()); Assert.assertEquals(disabledResult, enabledResult); }
@Test public void testKeyedReduce() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); if (objectReuse) { env.getConfig().enableObjectReuse(); } else { env.getConfig().disableObjectReuse(); } DataSet<Tuple2<String, Integer>> input = env.fromCollection(REDUCE_DATA); DataSet<Tuple2<String, Integer>> result = input .groupBy(0) .reduce(new ReduceFunction<Tuple2<String, Integer>>() { @Override public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) { value2.f1 += value1.f1; return value2; } }); Tuple2<String, Integer> res = result.collect().get(0); assertEquals(new Tuple2<>("a", 60), res); }
@Test public void testGlobalReduce() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); if (objectReuse) { env.getConfig().enableObjectReuse(); } else { env.getConfig().disableObjectReuse(); } DataSet<Tuple2<String, Integer>> input = env.fromCollection(REDUCE_DATA); DataSet<Tuple2<String, Integer>> result = input.reduce( new ReduceFunction<Tuple2<String, Integer>>() { @Override public Tuple2<String, Integer> reduce( Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) { if (value1.f1 % 3 == 0) { value1.f1 += value2.f1; return value1; } else { value2.f1 += value1.f1; return value2; } } }); Tuple2<String, Integer> res = result.collect().get(0); assertEquals(new Tuple2<>("a", 60), res); }
env.getConfig().enableObjectReuse(); } else { env.getConfig().disableObjectReuse();
env.getConfig().enableObjectReuse(); } else { env.getConfig().disableObjectReuse();
@Test public void testNestedPojoFieldAccessor() throws Exception { StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment(); see.getConfig().disableObjectReuse(); see.setParallelism(4); DataStream<Data> dataStream = see.fromCollection(elements); DataStream<Data> summedStream = dataStream .keyBy("aaa") .sum("stats.count") .keyBy("aaa") .flatMap(new FlatMapFunction<Data, Data>() { Data[] first = new Data[3]; @Override public void flatMap(Data value, Collector<Data> out) throws Exception { if (first[value.aaa] == null) { first[value.aaa] = value; if (value.stats.count != 123) { throw new RuntimeException("Expected stats.count to be 123"); } } else { if (value.stats.count != 2 * 123) { throw new RuntimeException("Expected stats.count to be 2 * 123"); } } } }); summedStream.print(); see.execute(); }