@Override public Graph<K, Long, Double> run(Graph<K, Long, Double> graph) { DataSet<Vertex<K, Tuple2<Long, Double>>> initializedVertices = graph.getVertices() .map(new AddScoreToVertexValuesMapper<>()); Graph<K, Tuple2<Long, Double>, Double> graphWithScoredVertices = Graph.fromDataSet(initializedVertices, graph.getEdges(), graph.getContext()).getUndirected(); return graphWithScoredVertices.runScatterGatherIteration(new LabelMessenger<>(), new VertexLabelUpdater<>(delta), maxIterations) .mapVertices(new RemoveScoreFromVertexValuesMapper<>()); }
/** * Translate {@link Vertex} values using the given {@link MapFunction}. * * @param translator implements conversion from {@code VV} to {@code NEW} * @param <NEW> new vertex value type * @return graph with translated vertex values * @throws Exception */ public <NEW> Graph<K, NEW, EV> translateVertexValues(TranslateFunction<VV, NEW> translator) throws Exception { return run(new TranslateVertexValues<>(translator)); }
/** * Creates a Graph from CSV input with edge values, but without vertex values. * @param vertexKey the type of the vertex IDs * @param edgeValue the type of the edge values * @return a Graph where the edges are read from an edges CSV file (with values). */ public <K, EV> Graph<K, NullValue, EV> edgeTypes(Class<K> vertexKey, Class<EV> edgeValue) { if (edgeReader == null) { throw new RuntimeException("The edge input file cannot be null!"); } DataSet<Tuple3<K, K, EV>> edges = edgeReader .types(vertexKey, vertexKey, edgeValue) .name(GraphCsvReader.class.getName()); return Graph.fromTupleDataSet(edges, executionContext); }
/** * Performs Difference on the vertex and edge sets of the input graphs * removes common vertices and edges. If a source/target vertex is removed, * its corresponding edge will also be removed * * @param graph the graph to perform difference with * @return a new graph where the common vertices and edges have been removed */ public Graph<K, VV, EV> difference(Graph<K, VV, EV> graph) { DataSet<Vertex<K, VV>> removeVerticesData = graph.getVertices(); return this.removeVertices(removeVerticesData); }
@Override public Graph<K, VV, NEW> runInternal(Graph<K, VV, OLD> input) throws Exception { DataSet<Edge<K, NEW>> translatedEdges = translateEdgeValues(input.getEdges(), translator, parallelism); return Graph.fromDataSet(input.getVertices(), translatedEdges, input.getContext()); } }
@Override public DataSet<Vertex<K, VV>> run(Graph<K, VV, EV> graph) throws Exception { // get type information for vertex value TypeInformation<VV> valueTypeInfo = ((TupleTypeInfo<?>) graph.getVertices().getType()).getTypeAt(1); Graph<K, VV, NullValue> undirectedGraph = graph .mapEdges(new MapTo<>(NullValue.getInstance())) .getUndirected(); return undirectedGraph.runScatterGatherIteration( new CCMessenger<>(valueTypeInfo), new CCUpdater<>(), maxIterations).getVertices(); }
@Override public DataSet<Vertex<K, VV>> run(Graph<K, VV, EV> graph) throws Exception { // get type information for vertex value TypeInformation<VV> valueTypeInfo = ((TupleTypeInfo<?>) graph.getVertices().getType()).getTypeAt(1); Graph<K, VV, NullValue> undirectedGraph = graph .mapEdges(new MapTo<>(NullValue.getInstance())) .getUndirected(); return undirectedGraph.runGatherSumApplyIteration( new GatherNeighborIds<>(valueTypeInfo), new SelectMinId<>(valueTypeInfo), new UpdateComponentId<>(valueTypeInfo), maxIterations).getVertices(); }
Graph<String, NullValue, Integer> userSongGraph = Graph.fromTupleDataSet(validTriplets, env); .groupReduceOnEdges(new GetTopSongPerUser(), EdgeDirection.OUT) .filter(new FilterSongNodes()); .getEdges() .reduceGroup(new CreateSimilarUserEdges()).distinct(); Graph<String, Long, NullValue> similarUsersGraph = Graph.fromDataSet(similarUsers, new MapFunction<String, Long>() { public Long map(String value) { return 1L; }, env).getUndirected(); .zipWithUniqueId(similarUsersGraph.getVertexIds()) .map(new MapFunction<Tuple2<Long, String>, Tuple2<String, Long>>() { @Override .joinWithVertices(idsWithInitialLabels, new VertexJoinFunction<Long, Long>() { public Long vertexJoin(Long vertexValue, Long inputValue) { return inputValue; }).run(new LabelPropagation<>(maxIterations));
@Override public DataSet<Vertex<K, Double>> run(Graph<K, VV, Double> input) { return input.mapVertices(new InitVerticesMapper<>(srcVertexId)) .runScatterGatherIteration(new MinDistanceMessenger<>(), new VertexDistanceUpdater<>(), maxIterations).getVertices(); }
@Override public DataSet<Vertex<K, Double>> run(Graph<K, VV, Double> input) { return input.mapVertices(new InitVerticesMapper<>(srcVertexId)) .runGatherSumApplyIteration(new CalculateDistances(), new ChooseMinDistance(), new UpdateDistance<>(), maxIterations) .getVertices(); }
transGraph.getEdges().map(new ToGellyEdgeWithNullValue()); Graph<GradoopId, GradoopId, NullValue> gellyTransGraph = Graph.fromDataSet( transVertices.map(new VertexToGellyVertexWithGradoopId()), transEdges, .getUndirected() .runScatterGatherIteration(new BtgMessenger(), new BtgUpdater(), 100); .getVerticesAsTuple2() .map(new SwitchPair<>()) .groupBy(0)
@Override public DataSet<Vertex<K, VV>> run(Graph<K, VV, EV> input) { TypeInformation<VV> valueType = ((TupleTypeInfo<?>) input.getVertices().getType()).getTypeAt(1); // iteratively adopt the most frequent label among the neighbors of each vertex return input .mapEdges(new MapTo<>(NullValue.getInstance())) .runScatterGatherIteration( new SendNewLabelToNeighbors<>(valueType), new UpdateVertexLabel<>(), maxIterations) .getVertices(); }
/** * Return the degree of all vertices in the graph. * * @return A DataSet of {@code Tuple2<vertexId, degree>} */ public DataSet<Tuple2<K, LongValue>> getDegrees() { return outDegrees() .union(inDegrees()).name("In- and out-degree") .groupBy(0).sum(1).name("Sum"); }
@Override public DataSet<Vertex<K, Double>> run(Graph<K, Double, Double> network) throws Exception { DataSet<Tuple2<K, LongValue>> vertexOutDegrees = network.outDegrees(); Graph<K, Double, Double> networkWithWeights = network .joinWithEdgesOnSource(vertexOutDegrees, new InitWeights()); ScatterGatherConfiguration parameters = new ScatterGatherConfiguration(); parameters.setOptNumVertices(true); return networkWithWeights.runScatterGatherIteration(new RankMessenger<>(), new VertexRankUpdater<>(beta), maxIterations, parameters) .getVertices(); }
@Override public DataSet<Edge<K, Tuple2<EV, Degrees>>> runInternal(Graph<K, VV, EV> input) throws Exception { // t, d(t) DataSet<Vertex<K, Degrees>> vertexDegrees = input .run(new VertexDegrees<K, VV, EV>() .setParallelism(parallelism)); // s, t, d(t) return input.getEdges() .join(vertexDegrees, JoinHint.REPARTITION_HASH_SECOND) .where(1) .equalTo(0) .with(new JoinEdgeWithVertexDegree<>()) .setParallelism(parallelism) .name("Edge target degrees"); } }
/** * Creates a graph from a Collection of edges. * Vertices are created automatically and their values are set to * NullValue. * * @param edges a Collection of edges. * @param context the flink execution environment. * @return the newly created graph. */ public static <K, EV> Graph<K, NullValue, EV> fromCollection(Collection<Edge<K, EV>> edges, ExecutionEnvironment context) { return fromDataSet(context.fromCollection(edges), context); }
@Override public DataSet<Vertex<K, Double>> run(Graph<K, Double, Double> network) throws Exception { DataSet<Tuple2<K, LongValue>> vertexOutDegrees = network.outDegrees(); Graph<K, Double, Double> networkWithWeights = network .joinWithEdgesOnSource(vertexOutDegrees, new InitWeights()); GSAConfiguration parameters = new GSAConfiguration(); parameters.setOptNumVertices(true); return networkWithWeights.runGatherSumApplyIteration(new GatherRanks(), new SumRanks(), new UpdateRanks<>(beta), maxIterations, parameters) .getVertices(); }
public static void main(String[] args) throws Exception { if (!parseParameters(args)) { return; } ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Edge<Long, Double>> edges = getEdgesDataSet(env); Graph<Long, Double, Double> graph = Graph.fromDataSet(edges, new InitVertices(srcVertexId), env); // Execute the scatter-gather iteration Graph<Long, Double, Double> result = graph.runScatterGatherIteration( new MinDistanceMessenger(), new VertexDistanceUpdater(), maxIterations); // Extract the vertices as the result DataSet<Vertex<Long, Double>> singleSourceShortestPaths = result.getVertices(); // emit result if (fileOutput) { singleSourceShortestPaths.writeAsCsv(outputPath, "\n", ","); // since file sinks are lazy, we trigger the execution explicitly env.execute("Single Source Shortest Paths Example"); } else { singleSourceShortestPaths.print(); } }
graph.removeEdge(edgeToBeRemoved); if (isInSSSP(edgeToBeRemoved, ssspGraph.getEdges())) { Graph<Long, Double, Double> result = ssspGraph.runScatterGatherIteration(new InvalidateMessenger(edgeToBeRemoved), new VertexDistanceUpdater(), maxIterations, parameters); DataSet<Vertex<Long, Double>> resultedVertices = result.getVertices(); graph.getVertices().writeAsCsv(outputPath, "\n", ","); env.execute("Incremental SSSP Example"); } else { graph.getVertices().print();
@Override public ChecksumHashCode<K, VV, EV> run(Graph<K, VV, EV> input) throws Exception { super.run(input); vertexChecksum = new org.apache.flink.graph.asm.dataset.ChecksumHashCode<>(); vertexChecksum.run(input.getVertices()); edgeChecksum = new org.apache.flink.graph.asm.dataset.ChecksumHashCode<>(); edgeChecksum.run(input.getEdges()); return this; }