@Override public GraphCollection execute( GraphCollection firstCollection, GraphCollection secondCollection) { // do some init stuff for the actual operator this.firstCollection = firstCollection; this.secondCollection = secondCollection; final DataSet<GraphHead> newGraphHeads = computeNewGraphHeads(); final DataSet<Vertex> newVertices = computeNewVertices(newGraphHeads); final DataSet<Edge> newEdges = computeNewEdges(newVertices); return firstCollection.getConfig().getGraphCollectionFactory() .fromDataSets(newGraphHeads, newVertices, newEdges); }
/** * {@inheritDoc} */ @Override public GraphCollection execute( GraphCollection firstCollection, GraphCollection secondCollection) { // do some init stuff for the actual operator this.firstCollection = firstCollection; this.secondCollection = secondCollection; final DataSet<GraphHead> newGraphHeads = computeNewGraphHeads(); final DataSet<Vertex> newVertices = computeNewVertices(newGraphHeads); final DataSet<Edge> newEdges = computeNewEdges(newVertices); return firstCollection.getConfig().getGraphCollectionFactory() .fromDataSets(newGraphHeads, newVertices, newEdges); }
@Override public GraphCollection execute(GraphCollection collection) { return collection.getConfig().getGraphCollectionFactory().fromDataSets( collection.getGraphHeads().distinct(new Id<>()), collection.getVertices(), collection.getEdges()); }
@Override public GraphCollection execute(GraphCollection collection) { return collection.getConfig().getGraphCollectionFactory().fromDataSets( collection.getGraphHeads().distinct(new Id<>()), collection.getVertices(), collection.getEdges()); } }
/** * Fusing the already-combined sources * * @param searchGraph Logical Graph defining the data lake * @param graphPatterns Collection of elements representing which vertices will be merged into * a vertex * @return A single merged graph */ @Override public LogicalGraph execute(LogicalGraph searchGraph, LogicalGraph graphPatterns) { return execute(searchGraph, graphPatterns.getConfig().getGraphCollectionFactory() .fromDataSets( graphPatterns.getGraphHead(), graphPatterns.getVertices(), graphPatterns.getEdges())); }
/** * Fusing the already-combined sources * * @param searchGraph Logical Graph defining the data lake * @param graphPatterns Collection of elements representing which vertices will be merged into * a vertex * @return A single merged graph */ @Override public LogicalGraph execute(LogicalGraph searchGraph, LogicalGraph graphPatterns) { return execute(searchGraph, graphPatterns.getConfig().getGraphCollectionFactory() .fromDataSets( graphPatterns.getGraphHead(), graphPatterns.getVertices(), graphPatterns.getEdges())); }
@Override public GraphCollection execute(LogicalGraph graph) { LogicalGraph graphWithWccIds = graph.callForGraph(new AnnotateWeaklyConnectedComponents( propertyKey, maxIterations)); GraphCollection split = graphWithWccIds.splitBy(propertyKey); return graph.getConfig().getGraphCollectionFactory().fromDataSets( split.getGraphHeads(), split.getVertices().map(new PropertyRemover<>(propertyKey)), split.getEdges().map(new PropertyRemover<>(propertyKey))); }
@Override public GraphCollection execute(LogicalGraph graph) { LogicalGraph graphWithWccIds = graph.callForGraph(new AnnotateWeaklyConnectedComponents( propertyKey, maxIterations)); GraphCollection split = graphWithWccIds.splitBy(propertyKey); return graph.getConfig().getGraphCollectionFactory().fromDataSets( split.getGraphHeads(), split.getVertices().map(new PropertyRemover<>(propertyKey)), split.getEdges().map(new PropertyRemover<>(propertyKey))); } }
/** * Returns a collection of all logical graph contained in the database. * * @return collection of all logical graphs */ public GraphCollection getGraphCollection() { ExecutionEnvironment env = config.getExecutionEnvironment(); DataSet<Vertex> newVertices = env.fromCollection(getVertices()) .filter(vertex -> vertex.getGraphCount() > 0); DataSet<Edge> newEdges = env.fromCollection(getEdges()) .filter(edge -> edge.getGraphCount() > 0); return config.getGraphCollectionFactory() .fromDataSets(env.fromCollection(getGraphHeads()), newVertices, newEdges); }
/** * Extracts a {@link GraphCollection} from a set of {@link Element}. * * @param elements EPGM elements * @param config Gradoop Flink config * @param mayOverlap elements may be contained in multiple graphs * @return Graph collection */ @SuppressWarnings("unchecked") public static GraphCollection extractGraphCollection( DataSet<Element> elements, GradoopFlinkConfig config, boolean mayOverlap) { Class<GraphHead> graphHeadType = config.getGraphHeadFactory().getType(); Class<Vertex> vertexType = config.getVertexFactory().getType(); Class<Edge> edgeType = config.getEdgeFactory().getType(); return config.getGraphCollectionFactory().fromDataSets( extractGraphHeads(elements, graphHeadType), extractVertices(elements, vertexType, mayOverlap), extractEdges(elements, edgeType, mayOverlap) ); }
/** * Extracts a {@link GraphCollection} from a set of {@link Element}. * * @param elements EPGM elements * @param config Gradoop Flink config * @param mayOverlap elements may be contained in multiple graphs * @return Graph collection */ @SuppressWarnings("unchecked") public static GraphCollection extractGraphCollection( DataSet<Element> elements, GradoopFlinkConfig config, boolean mayOverlap) { Class<GraphHead> graphHeadType = config.getGraphHeadFactory().getType(); Class<Vertex> vertexType = config.getVertexFactory().getType(); Class<Edge> edgeType = config.getEdgeFactory().getType(); return config.getGraphCollectionFactory().fromDataSets( extractGraphHeads(elements, graphHeadType), extractVertices(elements, vertexType, mayOverlap), extractEdges(elements, edgeType, mayOverlap) ); }
/** * Returns a collection of all logical graph contained in the database. * * @return collection of all logical graphs */ public GraphCollection getGraphCollection() { ExecutionEnvironment env = config.getExecutionEnvironment(); DataSet<Vertex> newVertices = env.fromCollection(getVertices()) .filter(vertex -> vertex.getGraphCount() > 0); DataSet<Edge> newEdges = env.fromCollection(getEdges()) .filter(edge -> edge.getGraphCount() > 0); return config.getGraphCollectionFactory() .fromDataSets(env.fromCollection(getGraphHeads()), newVertices, newEdges); }
@Override public GraphCollection execute(GraphCollection collection) { DataSet<GraphHead> graphHeads = collection.getGraphHeads().first(limit); DataSet<GradoopId> firstIds = graphHeads.map(new Id<>()); DataSet<Vertex> filteredVertices = collection.getVertices() .filter(new InAnyGraphBroadcast<>()) .withBroadcastSet(firstIds, GraphsContainmentFilterBroadcast.GRAPH_IDS); DataSet<Edge> filteredEdges = collection.getEdges() .filter(new InAnyGraphBroadcast<>()) .withBroadcastSet(firstIds, GraphsContainmentFilterBroadcast.GRAPH_IDS); return collection.getConfig().getGraphCollectionFactory() .fromDataSets(graphHeads, filteredVertices, filteredEdges); }
@Override public GraphCollection execute(GraphCollection collection) { DataSet<GraphHead> graphHeads = collection.getGraphHeads().first(limit); DataSet<GradoopId> firstIds = graphHeads.map(new Id<>()); DataSet<Vertex> filteredVertices = collection.getVertices() .filter(new InAnyGraphBroadcast<>()) .withBroadcastSet(firstIds, GraphsContainmentFilterBroadcast.GRAPH_IDS); DataSet<Edge> filteredEdges = collection.getEdges() .filter(new InAnyGraphBroadcast<>()) .withBroadcastSet(firstIds, GraphsContainmentFilterBroadcast.GRAPH_IDS); return collection.getConfig().getGraphCollectionFactory() .fromDataSets(graphHeads, filteredVertices, filteredEdges); } }
@Override protected GraphCollection executeForVertex(LogicalGraph graph) { GradoopFlinkConfig config = graph.getConfig(); EPGMGraphHeadFactory<GraphHead> graphHeadFactory = config.getGraphHeadFactory(); EPGMVertexFactory<Vertex> vertexFactory = config.getVertexFactory(); String variable = getQueryHandler().getVertices().iterator().next().getVariable(); DataSet<Vertex> matchingVertices = graph.getVertices() .filter(new MatchingVertices<>(getQuery())); if (!doAttachData()) { matchingVertices = matchingVertices .map(new Id<>()) .map(new ObjectTo1<>()) .map(new VertexFromId(vertexFactory)); } DataSet<Tuple2<Vertex, GraphHead>> pairs = matchingVertices .map(new AddGraphElementToNewGraph<>(graphHeadFactory, variable)) .returns(new TupleTypeInfo<>( TypeExtractor.getForClass(vertexFactory.getType()), TypeExtractor.getForClass(graphHeadFactory.getType()))); return config.getGraphCollectionFactory().fromDataSets( pairs.map(new Value1Of2<>()), pairs.map(new Value0Of2<>())); }
@Override public GraphCollection getGraphCollection() { GraphCollectionFactory factory = getFlinkConfig().getGraphCollectionFactory(); ExecutionEnvironment env = getFlinkConfig().getExecutionEnvironment(); return factory.fromDataSets( /*graph head format*/ env.createInput(new GraphHeadInputFormat( getStore().getConfig().getAccumuloProperties(), graphHeadQuery)), /*vertex input format*/ env.createInput(new VertexInputFormat(getStore().getConfig().getAccumuloProperties(), vertexQuery)), /*edge input format*/ env.createInput(new EdgeInputFormat(getStore().getConfig().getAccumuloProperties(), edgeQuery))); }
@Override public GraphCollection executeForGVELayout(GraphCollection collection) { DataSet<GraphHead> graphHeads = collection.getGraphHeads(); DataSet<GradoopId> graphIds = graphHeads.map(new Id<>()); DataSet<Tuple2<GradoopId, Map<String, PropertyValue>>> aggregate = aggregateVertices(collection.getVertices(), graphIds) .union(aggregateEdges(collection.getEdges(), graphIds)) .groupBy(0) .reduceGroup(new CombinePartitionApplyAggregates(aggregateFunctions)); graphHeads = graphHeads .coGroup(aggregate) .where(new Id<>()).equalTo(0) .with(new SetAggregateProperties(aggregateFunctions)); return collection.getConfig().getGraphCollectionFactory() .fromDataSets(graphHeads, collection.getVertices(), collection.getEdges()); }
@Override public GraphCollection executeForGVELayout(GraphCollection collection) { DataSet<GraphHead> graphHeads = collection.getGraphHeads(); DataSet<GradoopId> graphIds = graphHeads.map(new Id<>()); DataSet<Tuple2<GradoopId, Map<String, PropertyValue>>> aggregate = aggregateVertices(collection.getVertices(), graphIds) .union(aggregateEdges(collection.getEdges(), graphIds)) .groupBy(0) .reduceGroup(new CombinePartitionApplyAggregates(aggregateFunctions)); graphHeads = graphHeads .coGroup(aggregate) .where(new Id<>()).equalTo(0) .with(new SetAggregateProperties(aggregateFunctions)); return collection.getConfig().getGraphCollectionFactory() .fromDataSets(graphHeads, collection.getVertices(), collection.getEdges()); }
@Override public GraphCollection getGraphCollection() { DataSet<Tuple3<String, String, String>> metaData = new CSVMetaDataSource().readDistributed(getMetaDataPath(), getConfig()); DataSet<GraphHead> graphHeads = getConfig().getExecutionEnvironment() .readTextFile(getGraphHeadCSVPath()) .map(new CSVLineToGraphHead(getConfig().getGraphHeadFactory())) .withBroadcastSet(metaData, BC_METADATA); DataSet<Vertex> vertices = getConfig().getExecutionEnvironment() .readTextFile(getVertexCSVPath()) .map(new CSVLineToVertex(getConfig().getVertexFactory())) .withBroadcastSet(metaData, BC_METADATA); DataSet<Edge> edges = getConfig().getExecutionEnvironment() .readTextFile(getEdgeCSVPath()) .map(new CSVLineToEdge(getConfig().getEdgeFactory())) .withBroadcastSet(metaData, BC_METADATA); return getConfig().getGraphCollectionFactory().fromDataSets(graphHeads, vertices, edges); } }
@Override public GraphCollection getGraphCollection() { DataSet<Tuple3<String, String, String>> metaData = new CSVMetaDataSource().readDistributed(getMetaDataPath(), getConfig()); DataSet<GraphHead> graphHeads = getConfig().getExecutionEnvironment() .readTextFile(getGraphHeadCSVPath()) .map(new CSVLineToGraphHead(getConfig().getGraphHeadFactory())) .withBroadcastSet(metaData, BC_METADATA); DataSet<Vertex> vertices = getConfig().getExecutionEnvironment() .readTextFile(getVertexCSVPath()) .map(new CSVLineToVertex(getConfig().getVertexFactory())) .withBroadcastSet(metaData, BC_METADATA); DataSet<Edge> edges = getConfig().getExecutionEnvironment() .readTextFile(getEdgeCSVPath()) .map(new CSVLineToEdge(getConfig().getEdgeFactory())) .withBroadcastSet(metaData, BC_METADATA); return getConfig().getGraphCollectionFactory().fromDataSets(graphHeads, vertices, edges); } }