public MockOperator() { super(ExecutionEnvironment.createCollectionsEnvironment(), ValueTypeInfo.NULL_VALUE_TYPE_INFO); } }
@Test public void testBulkIteration() { try { ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment(); IterativeDataSet<Integer> iteration = env.fromElements(1).iterate(10); DataSet<Integer> result = iteration.closeWith(iteration.map(new AddSuperstepNumberMapper())); List<Integer> collected = new ArrayList<Integer>(); result.output(new LocalCollectionOutputFormat<Integer>(collected)); env.execute(); assertEquals(1, collected.size()); assertEquals(56, collected.get(0).intValue()); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
public static void main(String[] args) throws Exception { // initialize a new Collection-based execution environment final ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment(); // create objects for users and emails User[] usersArray = { new User(1, "Peter"), new User(2, "John"), new User(3, "Bill") }; EMail[] emailsArray = {new EMail(1, "Re: Meeting", "How about 1pm?"), new EMail(1, "Re: Meeting", "Sorry, I'm not availble"), new EMail(3, "Re: Re: Project proposal", "Give me a few more days to think about it.")}; // convert objects into a DataSet DataSet<User> users = env.fromElements(usersArray); DataSet<EMail> emails = env.fromElements(emailsArray); // join the two DataSets DataSet<Tuple2<User, EMail>> joined = users.join(emails).where("userIdentifier").equalTo("userId"); // retrieve the resulting Tuple2 elements into a ArrayList. List<Tuple2<User, EMail>> result = joined.collect(); // Do some work with the resulting ArrayList (=Collection). for (Tuple2<User, EMail> t : result) { System.err.println("Result = " + t); } } }
@Test public void testUnaryOp() { try { ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment(); DataSet<String> bcData = env.fromElements(SUFFIX); List<String> result = new ArrayList<String>(); env.fromElements(TEST_DATA) .map(new SuffixAppender()).withBroadcastSet(bcData, BC_VAR_NAME) .output(new LocalCollectionOutputFormat<String>(result)); env.execute(); assertEquals(TEST_DATA.length, result.size()); for (String s : result) { assertTrue(s.indexOf(SUFFIX) > 0); } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
@Test public void testBulkIterationWithTerminationCriterion() { try { ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment(); IterativeDataSet<Integer> iteration = env.fromElements(1).iterate(100); DataSet<Integer> iterationResult = iteration.map(new AddSuperstepNumberMapper()); DataSet<Integer> terminationCriterion = iterationResult.filter(new FilterFunction<Integer>() { public boolean filter(Integer value) { return value < 50; } }); List<Integer> collected = new ArrayList<Integer>(); iteration.closeWith(iterationResult, terminationCriterion) .output(new LocalCollectionOutputFormat<Integer>(collected)); env.execute(); assertEquals(1, collected.size()); assertEquals(56, collected.get(0).intValue()); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
@Test public void testBinaryOp() { try { ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment(); DataSet<String> bcData = env.fromElements(SUFFIX); DataSet<String> inData = env.fromElements(TEST_DATA); List<String> result = new ArrayList<String>(); inData.cross(inData).with(new SuffixCross()).withBroadcastSet(bcData, BC_VAR_NAME) .output(new LocalCollectionOutputFormat<String>(result)); env.execute(); assertEquals(TEST_DATA.length * TEST_DATA.length, result.size()); for (String s : result) { assertTrue(s.indexOf(SUFFIX) == 2); } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
public static void main(String[] args) throws Exception { ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment(); BatchTableEnvironment tEnv = TableEnvironment.getTableEnvironment(env); DataSet<WC> input = env.fromElements( new WC("Hello", 1), new WC("Ciao", 1), new WC("Hello", 1)); Table table = tEnv.fromDataSet(input); Table filtered = table .groupBy("word") .select("word, frequency.sum as frequency") .filter("frequency = 2"); DataSet<WC> result = tEnv.toDataSet(filtered, WC.class); result.print(); }
@Test public void testAccumulator() { try { final int numElements = 100; ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment(); env.generateSequence(1, numElements) .map(new CountingMapper()) .output(new DiscardingOutputFormat<Long>()); JobExecutionResult result = env.execute(); assertTrue(result.getNetRuntime() >= 0); assertEquals(numElements, (int) result.getAccumulatorResult(ACCUMULATOR_NAME)); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
@Test public void testDeltaIteration() { try { ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment();
public static void main(String[] args) throws Exception { // initialize a new Collection-based execution environment final ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment(); // create objects for users and emails User[] usersArray = { new User(1, "Peter"), new User(2, "John"), new User(3, "Bill") }; EMail[] emailsArray = {new EMail(1, "Re: Meeting", "How about 1pm?"), new EMail(1, "Re: Meeting", "Sorry, I'm not availble"), new EMail(3, "Re: Re: Project proposal", "Give me a few more days to think about it.")}; // convert objects into a DataSet DataSet<User> users = env.fromElements(usersArray); DataSet<EMail> emails = env.fromElements(emailsArray); // join the two DataSets DataSet<Tuple2<User,EMail>> joined = users.join(emails).where("userIdentifier").equalTo("userId"); // retrieve the resulting Tuple2 elements into a ArrayList. List<Tuple2<User,EMail>> result = joined.collect(); // Do some work with the resulting ArrayList (=Collection). for(Tuple2<User, EMail> t : result) { System.err.println("Result = " + t); } } }
public static void main(String[] args) throws Exception { // initialize a new Collection-based execution environment final ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment(); // create objects for users and emails User[] usersArray = { new User(1, "Peter"), new User(2, "John"), new User(3, "Bill") }; EMail[] emailsArray = {new EMail(1, "Re: Meeting", "How about 1pm?"), new EMail(1, "Re: Meeting", "Sorry, I'm not availble"), new EMail(3, "Re: Re: Project proposal", "Give me a few more days to think about it.")}; // convert objects into a DataSet DataSet<User> users = env.fromElements(usersArray); DataSet<EMail> emails = env.fromElements(emailsArray); // join the two DataSets DataSet<Tuple2<User, EMail>> joined = users.join(emails).where("userIdentifier").equalTo("userId"); // retrieve the resulting Tuple2 elements into a ArrayList. List<Tuple2<User, EMail>> result = joined.collect(); // Do some work with the resulting ArrayList (=Collection). for (Tuple2<User, EMail> t : result) { System.err.println("Result = " + t); } } }
public static void main(String[] args) throws Exception { // initialize a new Collection-based execution environment final ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment(); // create objects for users and emails User[] usersArray = { new User(1, "Peter"), new User(2, "John"), new User(3, "Bill") }; EMail[] emailsArray = {new EMail(1, "Re: Meeting", "How about 1pm?"), new EMail(1, "Re: Meeting", "Sorry, I'm not availble"), new EMail(3, "Re: Re: Project proposal", "Give me a few more days to think about it.")}; // convert objects into a DataSet DataSet<User> users = env.fromElements(usersArray); DataSet<EMail> emails = env.fromElements(emailsArray); // join the two DataSets DataSet<Tuple2<User,EMail>> joined = users.join(emails).where("userIdentifier").equalTo("userId"); // retrieve the resulting Tuple2 elements into a ArrayList. List<Tuple2<User,EMail>> result = joined.collect(); // Do some work with the resulting ArrayList (=Collection). for (Tuple2<User, EMail> t : result) { System.err.println("Result = " + t); } } }
public static void main(String[] args) throws Exception { // initialize a new Collection-based execution environment final ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment(); // create objects for users and emails User[] usersArray = { new User(1, "Peter"), new User(2, "John"), new User(3, "Bill") }; EMail[] emailsArray = {new EMail(1, "Re: Meeting", "How about 1pm?"), new EMail(1, "Re: Meeting", "Sorry, I'm not availble"), new EMail(3, "Re: Re: Project proposal", "Give me a few more days to think about it.")}; // convert objects into a DataSet DataSet<User> users = env.fromElements(usersArray); DataSet<EMail> emails = env.fromElements(emailsArray); // join the two DataSets DataSet<Tuple2<User, EMail>> joined = users.join(emails).where("userIdentifier").equalTo("userId"); // retrieve the resulting Tuple2 elements into a ArrayList. List<Tuple2<User, EMail>> result = joined.collect(); // Do some work with the resulting ArrayList (=Collection). for (Tuple2<User, EMail> t : result) { System.err.println("Result = " + t); } } }