/** * Helper method which sets up an iteration with the given vertex value. * * @param iteration */ private void setUpIteration(DeltaIteration<?, ?> iteration) { // set up the iteration operator if (this.configuration != null) { iteration.name(this.configuration.getName("Vertex-centric iteration (" + computeFunction + ")")); iteration.parallelism(this.configuration.getParallelism()); iteration.setSolutionSetUnManaged(this.configuration.isSolutionSetUnmanagedMemory()); // register all aggregators for (Map.Entry<String, Aggregator<?>> entry : this.configuration.getAggregators().entrySet()) { iteration.registerAggregator(entry.getKey(), entry.getValue()); } } else { // no configuration provided; set default name iteration.name("Vertex-centric iteration (" + computeFunction + ")"); } }
/** * Helper method which sets up an iteration with the given vertex value(either simple or with degrees). * * @param iteration */ private void setUpIteration(DeltaIteration<?, ?> iteration) { // set up the iteration operator if (this.configuration != null) { iteration.name(this.configuration.getName("Scatter-gather iteration (" + gatherFunction + " | " + scatterFunction + ")")); iteration.parallelism(this.configuration.getParallelism()); iteration.setSolutionSetUnManaged(this.configuration.isSolutionSetUnmanagedMemory()); // register all aggregators for (Map.Entry<String, Aggregator<?>> entry : this.configuration.getAggregators().entrySet()) { iteration.registerAggregator(entry.getKey(), entry.getValue()); } } else { // no configuration provided; set default name iteration.name("Scatter-gather iteration (" + gatherFunction + " | " + scatterFunction + ")"); } }
"Gather-sum-apply iteration (" + gather + " | " + sum + " | " + apply + ")")); iteration.parallelism(this.configuration.getParallelism()); iteration.setSolutionSetUnManaged(this.configuration.isSolutionSetUnmanagedMemory());
@Override protected void testProgram() throws Exception { // set up execution environment ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); // read vertex and edge data DataSet<Tuple1<Long>> vertices = env.readCsvFile(verticesPath).types(Long.class); DataSet<Tuple2<Long, Long>> edges = env.readCsvFile(edgesPath).fieldDelimiter(" ").types(Long.class, Long.class) .flatMap(new UndirectEdge()); // assign the initial components (equal to the vertex id) DataSet<Tuple2<Long, Long>> verticesWithInitialId = vertices.map(new ConnectedComponentsITCase.DuplicateValue<Long>()); // open a delta iteration DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration = verticesWithInitialId.iterateDelta(verticesWithInitialId, 100, 0); iteration.setSolutionSetUnManaged(true); // apply the step logic: join with the edges, select the minimum neighbor, update if the component of the candidate is smaller DataSet<Tuple2<Long, Long>> changes = iteration.getWorkset().join(edges).where(0).equalTo(0).with(new NeighborWithComponentIDJoin()) .groupBy(0).aggregate(Aggregations.MIN, 1) .join(iteration.getSolutionSet()).where(0).equalTo(0) .with(new ComponentIdFilter()); // close the delta iteration (delta and new workset are identical) DataSet<Tuple2<Long, Long>> result = iteration.closeWith(changes, changes); result.writeAsCsv(resultPath, "\n", " "); // execute program env.execute("Connected Components Example"); }
/** * Helper method which sets up an iteration with the given vertex value(either simple or with degrees) * * @param iteration */ private void setUpIteration(DeltaIteration<?, ?> iteration) { // set up the iteration operator if (this.configuration != null) { iteration.name(this.configuration.getName("Scatter-gather iteration (" + gatherFunction + " | " + scatterFunction + ")")); iteration.parallelism(this.configuration.getParallelism()); iteration.setSolutionSetUnManaged(this.configuration.isSolutionSetUnmanagedMemory()); // register all aggregators for (Map.Entry<String, Aggregator<?>> entry : this.configuration.getAggregators().entrySet()) { iteration.registerAggregator(entry.getKey(), entry.getValue()); } } else { // no configuration provided; set default name iteration.name("Scatter-gather iteration (" + gatherFunction + " | " + scatterFunction + ")"); } }
/** * Helper method which sets up an iteration with the given vertex value. * * @param iteration */ private void setUpIteration(DeltaIteration<?, ?> iteration) { // set up the iteration operator if (this.configuration != null) { iteration.name(this.configuration.getName("Vertex-centric iteration (" + computeFunction + ")")); iteration.parallelism(this.configuration.getParallelism()); iteration.setSolutionSetUnManaged(this.configuration.isSolutionSetUnmanagedMemory()); // register all aggregators for (Map.Entry<String, Aggregator<?>> entry : this.configuration.getAggregators().entrySet()) { iteration.registerAggregator(entry.getKey(), entry.getValue()); } } else { // no configuration provided; set default name iteration.name("Vertex-centric iteration (" + computeFunction + ")"); } }
/** * Helper method which sets up an iteration with the given vertex value. * * @param iteration */ private void setUpIteration(DeltaIteration<?, ?> iteration) { // set up the iteration operator if (this.configuration != null) { iteration.name(this.configuration.getName("Vertex-centric iteration (" + computeFunction + ")")); iteration.parallelism(this.configuration.getParallelism()); iteration.setSolutionSetUnManaged(this.configuration.isSolutionSetUnmanagedMemory()); // register all aggregators for (Map.Entry<String, Aggregator<?>> entry : this.configuration.getAggregators().entrySet()) { iteration.registerAggregator(entry.getKey(), entry.getValue()); } } else { // no configuration provided; set default name iteration.name("Vertex-centric iteration (" + computeFunction + ")"); } }
/** * Helper method which sets up an iteration with the given vertex value(either simple or with degrees). * * @param iteration */ private void setUpIteration(DeltaIteration<?, ?> iteration) { // set up the iteration operator if (this.configuration != null) { iteration.name(this.configuration.getName("Scatter-gather iteration (" + gatherFunction + " | " + scatterFunction + ")")); iteration.parallelism(this.configuration.getParallelism()); iteration.setSolutionSetUnManaged(this.configuration.isSolutionSetUnmanagedMemory()); // register all aggregators for (Map.Entry<String, Aggregator<?>> entry : this.configuration.getAggregators().entrySet()) { iteration.registerAggregator(entry.getKey(), entry.getValue()); } } else { // no configuration provided; set default name iteration.name("Scatter-gather iteration (" + gatherFunction + " | " + scatterFunction + ")"); } }
/** * Helper method which sets up an iteration with the given vertex value * * @param iteration */ private void setUpIteration(DeltaIteration<?, ?> iteration) { // set up the iteration operator if (this.configuration != null) { iteration.name(this.configuration.getName("Vertex-centric iteration (" + computeFunction + ")")); iteration.parallelism(this.configuration.getParallelism()); iteration.setSolutionSetUnManaged(this.configuration.isSolutionSetUnmanagedMemory()); // register all aggregators for (Map.Entry<String, Aggregator<?>> entry : this.configuration.getAggregators().entrySet()) { iteration.registerAggregator(entry.getKey(), entry.getValue()); } } else { // no configuration provided; set default name iteration.name("Vertex-centric iteration (" + computeFunction + ")"); } }
/** * Helper method which sets up an iteration with the given vertex value(either simple or with degrees). * * @param iteration */ private void setUpIteration(DeltaIteration<?, ?> iteration) { // set up the iteration operator if (this.configuration != null) { iteration.name(this.configuration.getName("Scatter-gather iteration (" + gatherFunction + " | " + scatterFunction + ")")); iteration.parallelism(this.configuration.getParallelism()); iteration.setSolutionSetUnManaged(this.configuration.isSolutionSetUnmanagedMemory()); // register all aggregators for (Map.Entry<String, Aggregator<?>> entry : this.configuration.getAggregators().entrySet()) { iteration.registerAggregator(entry.getKey(), entry.getValue()); } } else { // no configuration provided; set default name iteration.name("Scatter-gather iteration (" + gatherFunction + " | " + scatterFunction + ")"); } }
"Gather-sum-apply iteration (" + gather + " | " + sum + " | " + apply + ")")); iteration.parallelism(this.configuration.getParallelism()); iteration.setSolutionSetUnManaged(this.configuration.isSolutionSetUnmanagedMemory());
"Gather-sum-apply iteration (" + gather + " | " + sum + " | " + apply + ")")); iteration.parallelism(this.configuration.getParallelism()); iteration.setSolutionSetUnManaged(this.configuration.isSolutionSetUnmanagedMemory());
"Gather-sum-apply iteration (" + gather + " | " + sum + " | " + apply + ")")); iteration.parallelism(this.configuration.getParallelism()); iteration.setSolutionSetUnManaged(this.configuration.isSolutionSetUnmanagedMemory());