public static void main(String[] args) throws Exception { if(!parseParameters(args)) { return; } // set up the execution environment final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); // get input data DataSet<String> text = getTextDataSet(env); DataSet<Tuple2<String, Integer>> counts = // split up the lines in pairs (2-tuples) containing: (word,1) text.flatMap(new Tokenizer()) // group by the tuple field "0" and sum up tuple field "1" .groupBy(0) .aggregate(Aggregations.SUM, 1); // emit result if(fileOutput) { counts.writeAsCsv(outputPath, "\n", " "); } else { counts.print(); } // execute program env.execute("WordCount Example"); }
/** * Groups a {@link Tuple} {@link DataSet} using field position keys.<br/> * <b>Note: Field position keys only be specified for Tuple DataSets.</b></br> * The field position keys specify the fields of Tuples on which the DataSet is grouped. * This method returns an {@link UnsortedGrouping} on which one of the following grouping transformation * can be applied. * <ul> * <li>{@link UnsortedGrouping#sortGroup(int, eu.stratosphere.api.common.operators.Order)} to get a {@link SortedGrouping}. * <li>{@link Grouping#aggregate(Aggregations, int)} to apply an Aggregate transformation. * <li>{@link Grouping#reduce(ReduceFunction)} to apply a Reduce transformation. * <li>{@link Grouping#reduceGroup(GroupReduceFunction)} to apply a GroupReduce transformation. * </ul> * * @param fields One or more field positions on which the DataSet will be grouped. * @return A Grouping on which a transformation needs to be applied to obtain a transformed DataSet. * * @see Tuple * @see Grouping * @see UnsortedGrouping * @see SortedGrouping * @see AggregateOperator * @see ReduceOperator * @see GroupReduceOperator * @see DataSet */ public UnsortedGrouping<T> groupBy(int... fields) { return new UnsortedGrouping<T>(this, new Keys.FieldPositionKeys<T>(fields, getType(), false)); }
linksInput.groupBy(0).reduceGroup(new BuildOutgoingEdgeList()); .groupBy(0).aggregate(SUM, 1)
groupBy(1).reduceGroup(new Tuple3GroupReduce()); groupBy(4,0).reduceGroup(new Tuple5GroupReduce()); groupBy(1).sortGroup(2,Order.ASCENDING).reduceGroup(new Tuple3SortedGroupReduce()); return in.f1; }).reduceGroup(new Tuple3GroupReduce()); return in.myInt; }).reduceGroup(new CustomTypeGroupReduce()); groupBy(1).reduceGroup(new BCTuple3GroupReduce()).withBroadcastSet(intDs, "ints"); groupBy(1).reduceGroup(new InputReturningTuple3GroupReduce()); return in.myInt; }).reduceGroup(new CustomTypeGroupReduceWithCombine()); groupBy(1).reduceGroup(new Tuple3GroupReduceWithCombine()); groupBy(1).sortGroup(2,Order.DESCENDING).reduceGroup(new Tuple3SortedGroupReduce());
.groupBy(0).reduce(new CentroidAccumulator())
.groupBy(Edge.V1).sortGroup(Edge.V2, Order.ASCENDING).reduceGroup(new TriadBuilder())
.groupBy(new KeySelector<Long, Long>() { public Long getKey(Long id) { return id; } }).reduceGroup(new RemoveDuplicatesReduce()); candidatesDependencies.join(iteration.getSolutionSet()).where(0).equalTo(0) .with(new NeighborWithComponentIDJoin()) .groupBy(0).reduceGroup(new MinimumReduce());
linksInput.groupBy(0).reduceGroup(new BuildOutgoingEdgeList()); .groupBy(0).aggregate(SUM, 1)
.groupBy(0).reduce(new CentroidAccumulator())
.groupBy(Edge.V1).sortGroup(Edge.V2, Order.ASCENDING).reduceGroup(new TriadBuilder())
public static void main(String[] args) throws Exception { if(!parseParameters(args)) { return; } // set up the execution environment final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); // get input data DataSet<String> text = getTextDataSet(env); DataSet<Tuple2<String, Integer>> counts = // split up the lines in pairs (2-tuples) containing: (word,1) text.flatMap(new Tokenizer()) // group by the tuple field "0" and sum up tuple field "1" .groupBy(0) .aggregate(Aggregations.SUM, 1); // emit result if(fileOutput) { counts.writeAsCsv(outputPath, "\n", " "); } else { counts.print(); } // execute program env.execute("WordCount Example"); }
groupBy(1).reduce(new Tuple3Reduce("B-)")); groupBy(4,0).reduce(new Tuple5Reduce()); return in.f1; }).reduce(new Tuple3Reduce("B-)")); return in.myInt; }).reduce(new CustomTypeReduce()); groupBy(1).reduce(new BCTuple3Reduce()).withBroadcastSet(intDs, "ints"); groupBy(1).reduce(new InputReturningTuple3Reduce());
/** * Groups a {@link DataSet} using a {@link KeySelector} function. * The KeySelector function is called for each element of the DataSet and extracts a single * key value on which the DataSet is grouped. </br> * This method returns an {@link UnsortedGrouping} on which one of the following grouping transformation * can be applied. * <ul> * <li>{@link UnsortedGrouping#sortGroup(int, eu.stratosphere.api.common.operators.Order)} to get a {@link SortedGrouping}. * <li>{@link Grouping#aggregate(Aggregations, int)} to apply an Aggregate transformation. * <li>{@link Grouping#reduce(ReduceFunction)} to apply a Reduce transformation. * <li>{@link Grouping#reduceGroup(GroupReduceFunction)} to apply a GroupReduce transformation. * </ul> * * @param keyExtractor The KeySelector function which extracts the key values from the DataSet on which it is grouped. * @return An UnsortedGrouping on which a transformation needs to be applied to obtain a transformed DataSet. * * @see KeySelector * @see Grouping * @see UnsortedGrouping * @see SortedGrouping * @see AggregateOperator * @see ReduceOperator * @see GroupReduceOperator * @see DataSet */ public <K extends Comparable<K>> UnsortedGrouping<T> groupBy(KeySelector<T, K> keyExtractor) { return new UnsortedGrouping<T>(this, new Keys.SelectorFunctionKeys<T, K>(keyExtractor, getType())); }
@Override protected void testProgram() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<String> text = env.fromElements(WordCountData.TEXT); DataSet<Tuple2<String, Integer>> words = text.flatMap(new WordCount.Tokenizer()); DataSet<Tuple2<String, Integer>> result = words.groupBy(0).aggregate(Aggregations.SUM, 1); result.output(new LocalCollectionOutputFormat<Tuple2<String, Integer>>(resultsCollected)); env.execute("Word Count Collection"); } }
.groupBy(0).reduce(new CentroidAccumulator())
public static DataSet<Tuple2<Long, Double>> constructPlan(DataSet<Tuple2<Long, Double>> initialData, int numIterations) { DeltaIteration<Tuple2<Long, Double>, Tuple2<Long, Double>> iteration = initialData.iterateDelta(initialData, numIterations, 0); DataSet<Tuple2<Long, Double>> delta = iteration.getSolutionSet() .join(iteration.getWorkset().flatMap(new Duplicator())).where(0).equalTo(0).with(new SummingJoin()).name(JOIN_1) .groupBy(0).aggregate(Aggregations.MIN, 1).map(new Expander()) .join(iteration.getSolutionSet()).where(0).equalTo(0).with(new SummingJoinProject()).name(JOIN_2); DataSet<Tuple2<Long, Double>> changes = delta.groupBy(0).aggregate(Aggregations.SUM, 1); DataSet<Tuple2<Long, Double>> result = iteration.closeWith(delta, changes); return result; }
lineitemsOfOrders.groupBy(0,1).aggregate(Aggregations.SUM, 2);
lineitemsOfOrders.groupBy(0,1).aggregate(Aggregations.SUM, 2);