/** * Returns example integrated instance graph from GDL input. * @return integrated instance graph * @throws IOException */ public static LogicalGraph getIntegratedInstanceGraph() throws IOException { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); GradoopFlinkConfig gradoopConf = GradoopFlinkConfig.createConfig(env); FlinkAsciiGraphLoader loader = new FlinkAsciiGraphLoader(gradoopConf); String gdl = IOUtils.toString(CategoryCharacteristicPatterns.class .getResourceAsStream("/data/gdl/itbda.gdl")); gdl = gdl .replaceAll("SOURCEID_KEY", SOURCEID_KEY) .replaceAll("SUPERTYPE_KEY", SUPERTYPE_KEY) .replaceAll("SUPERCLASS_VALUE_MASTER", SUPERCLASS_VALUE_MASTER) .replaceAll("SUPERCLASS_VALUE_TRANSACTIONAL", SUPERCLASS_VALUE_TRANSACTIONAL); loader.initDatabaseFromString(gdl); return loader.getLogicalGraphByVariable("iig"); }
/** * Returns a collection of all logical graph contained in the database. * * @return collection of all logical graphs */ public GraphCollection getGraphCollection() { ExecutionEnvironment env = config.getExecutionEnvironment(); DataSet<Vertex> newVertices = env.fromCollection(getVertices()) .filter(vertex -> vertex.getGraphCount() > 0); DataSet<Edge> newEdges = env.fromCollection(getEdges()) .filter(edge -> edge.getGraphCount() > 0); return config.getGraphCollectionFactory() .fromDataSets(env.fromCollection(getGraphHeads()), newVertices, newEdges); }
/** * Returns a logical graph containing the complete vertex and edge space of * the database. * This is equivalent to {@link #getLogicalGraph(boolean) getLogicalGraph(true)}. * * @return logical graph of vertex and edge space */ public LogicalGraph getLogicalGraph() { return getLogicalGraph(true); }
/** * Builds a {@link LogicalGraph} from the graph referenced by the given * graph variable. * * @param variable graph variable used in GDL script * @return LogicalGraph */ public LogicalGraph getLogicalGraphByVariable(String variable) { GraphHead graphHead = getGraphHeadByVariable(variable); Collection<Vertex> vertices = getVerticesByGraphVariables(variable); Collection<Edge> edges = getEdgesByGraphVariables(variable); return config.getLogicalGraphFactory().fromCollections(graphHead, vertices, edges); }
/** * Builds a {@link GraphCollection} from the graph referenced by the given * graph variables. * * @param variables graph variables used in GDL script * @return GraphCollection */ public GraphCollection getGraphCollectionByVariables(String... variables) { Collection<GraphHead> graphHeads = getGraphHeadsByVariables(variables); Collection<Vertex> vertices = getVerticesByGraphVariables(variables); Collection<Edge> edges = getEdgesByGraphVariables(variables); return config.getGraphCollectionFactory().fromCollections(graphHeads, vertices, edges); }
@Test public void write() throws Exception { AccumuloEPGMStore accumuloStore = new AccumuloEPGMStore(AccumuloTestSuite.getAcConfig(TEST_02)); FlinkAsciiGraphLoader loader = new FlinkAsciiGraphLoader( GradoopFlinkConfig.createConfig(getExecutionEnvironment())); InputStream inputStream = getClass().getResourceAsStream( GradoopTestUtils.SOCIAL_NETWORK_GDL_FILE); loader.initDatabaseFromStream(inputStream); GradoopFlinkConfig flinkConfig = GradoopFlinkConfig.createConfig(getExecutionEnvironment()); new AccumuloDataSink(accumuloStore, flinkConfig) .write(flinkConfig.getGraphCollectionFactory() .fromCollections( loader.getGraphHeads(), loader.getVertices(), loader.getEdges())); getExecutionEnvironment().execute(); accumuloStore.flush(); validateEPGMElementCollections(loader.getGraphHeads(), accumuloStore.getGraphSpace().readRemainsAndClose()); validateEPGMElementCollections(loader.getVertices(), accumuloStore.getVertexSpace().readRemainsAndClose()); validateEPGMGraphElementCollections(loader.getVertices(), accumuloStore.getVertexSpace().readRemainsAndClose()); validateEPGMElementCollections(loader.getEdges(), accumuloStore.getEdgeSpace().readRemainsAndClose()); validateEPGMGraphElementCollections(loader.getEdges(), accumuloStore.getEdgeSpace().readRemainsAndClose()); accumuloStore.close(); }
/** * Returns a logical graph containing the complete vertex and edge space of * the database. * * @param withGraphContainment true, if vertices and edges shall be updated to * be contained in the logical graph representing * the database * @return logical graph of vertex and edge space */ public LogicalGraph getLogicalGraph(boolean withGraphContainment) { if (withGraphContainment) { return config.getLogicalGraphFactory().fromCollections(getVertices(), getEdges()) .transformGraphHead(new RenameLabel<>(GradoopConstants.DEFAULT_GRAPH_LABEL, GradoopConstants.DB_GRAPH_LABEL)); } else { GraphHead graphHead = config.getGraphHeadFactory() .createGraphHead(GradoopConstants.DB_GRAPH_LABEL); return config.getLogicalGraphFactory().fromCollections(graphHead, getVertices(), getEdges()); } }
FlinkAsciiGraphLoader loader = new FlinkAsciiGraphLoader(cfg); loader.initDatabaseFromString(graph); GraphCollection inputGraphs = loader.getGraphCollectionByVariables("g1", "g2"); DataSink inputGraphSink = new DOTDataSink("out/input.dot", true); inputGraphs.writeTo(inputGraphSink, true); LogicalGraph graph1 = loader.getLogicalGraphByVariable("g1"); LogicalGraph graph2 = loader.getLogicalGraphByVariable("g2"); LogicalGraph overlap = graph2.overlap(graph1); DataSink overlapSink = new DOTDataSink("out/overlap.dot", true);
@Test public void testExecute() throws Exception { FlinkAsciiGraphLoader loader = new FlinkAsciiGraphLoader(getConfig()); loader.initDatabaseFromFile(getFilePath("/data/gdl/iig_btgs.gdl")); LogicalGraph iig = loader.getLogicalGraphByVariable("iig"); GraphCollection expectation = loader .getGraphCollectionByVariables("btg1", "btg2", "btg3", "btg4"); GraphCollection result = iig .callForCollection(new BusinessTransactionGraphs()); collectAndAssertTrue(expectation.equalsByGraphElementData(result)); } }
FlinkAsciiGraphLoader loader = new FlinkAsciiGraphLoader(GradoopFlinkConfig.createConfig(env)); loader.initDatabaseFromFile( URLDecoder.decode(EXAMPLE_DATA_FILE, StandardCharsets.UTF_8.name())); LogicalGraph callGraph = loader.getLogicalGraph();
/** * Executes a rollUp on edges using a single grouping key and checks if the result * is correct. * * @throws Exception If the execution fails. */ @Test public void testEdgeRollUpWithSingleGroupingKey() throws Exception { FlinkAsciiGraphLoader loader = getSocialNetworkLoader(); LogicalGraph input = loader.getGraphCollectionByVariables("g0", "g1", "g2", "g3") .reduce(new ReduceCombination()); //expected loader.initDatabaseFromString("g0 {edgeRollUpGroupingKeys:\":label\"}[" + "(v0)" + "(v0)-[e_0:knows{count:10L}]->(v0)" + "(v0)-[e_1:hasModerator{count:1L}]->(v0)" + "(v0)-[e_2:hasMember{count:2L}]->(v0)" + "]"); GraphCollection expected = loader.getGraphCollection(); List<String> vertexGK = Collections.emptyList(); List<AggregateFunction> vertexAGG = Collections.emptyList(); List<String> edgeGK = Collections.singletonList(Grouping.LABEL_SYMBOL); List<AggregateFunction> edgeAGG = Collections.singletonList(new Count("count")); GraphCollection output = input.groupEdgesByRollUp(vertexGK, vertexAGG, edgeGK, edgeAGG); collectAndAssertTrue(output.equalsByGraphData(expected)); }
@Test public void testSameGraph() throws Exception { FlinkAsciiGraphLoader loader = getSocialNetworkLoader(); loader.appendToDatabaseFromString("expected[]"); LogicalGraph g0 = loader.getLogicalGraphByVariable("g0"); LogicalGraph expected = loader.getLogicalGraphByVariable("expected"); assertTrue("exclusion of same graph failed", expected.equalsByElementIds(g0.exclude(g0)).collect().get(0)); }
@Test public void testEdgeInducedSubgraphProjectFirst() throws Exception { FlinkAsciiGraphLoader loader = getSocialNetworkLoader(); loader.appendToDatabaseFromString("expected[" + "(databases)<-[ghtd]-(gdbs)-[ghtg1]->(graphs)" + "(graphs)<-[ghtg2]-(gps)-[ghth]->(hadoop)" + "]"); LogicalGraph input = loader.getLogicalGraph(); LogicalGraph expected = loader.getLogicalGraphByVariable("expected"); LogicalGraph output = input.subgraph(null, e -> e.getLabel().equals("hasTag"), Subgraph.Strategy.EDGE_INDUCED_PROJECT_FIRST); collectAndAssertTrue(output.equalsByElementData(expected)); }
@Test public void testFromGraphTransactions() throws Exception { FlinkAsciiGraphLoader loader = getSocialNetworkLoader(); GraphTransaction g0 = new GraphTransaction(loader.getGraphHeadByVariable("g0"), Sets.newHashSet(loader.getVerticesByGraphVariables("g0")), Sets.newHashSet(loader.getEdgesByGraphVariables("g0"))); GraphTransaction g1 = new GraphTransaction(loader.getGraphHeadByVariable("g1"), Sets.newHashSet(loader.getVerticesByGraphVariables("g1")), Sets.newHashSet(loader.getEdgesByGraphVariables("g1"))); GraphTransaction g2 = new GraphTransaction(loader.getGraphHeadByVariable("g2"), Sets.newHashSet(loader.getVerticesByGraphVariables("g2")), Sets.newHashSet(loader.getEdgesByGraphVariables("g2"))); DataSet<GraphTransaction> transactions = getExecutionEnvironment().fromElements(g0, g1, g2); GraphCollectionLayout<GraphHead, Vertex, Edge> collectionLayout = getFactory().fromTransactions(transactions); Collection<GraphHead> loadedGraphHeads = Lists.newArrayList(); Collection<Vertex> loadedVertices = Lists.newArrayList(); Collection<Edge> loadedEdges = Lists.newArrayList(); collectionLayout.getGraphHeads().output(new LocalCollectionOutputFormat<>(loadedGraphHeads)); collectionLayout.getVertices().output(new LocalCollectionOutputFormat<>(loadedVertices)); collectionLayout.getEdges().output(new LocalCollectionOutputFormat<>(loadedEdges)); getExecutionEnvironment().execute(); validateEPGMElementCollections(loader.getGraphHeadsByVariables("g0", "g1", "g2"), loadedGraphHeads); validateEPGMElementCollections(loader.getVerticesByGraphVariables("g0", "g1", "g2"), loadedVertices); validateEPGMElementCollections(loader.getEdgesByGraphVariables("g0", "g1", "g2"), loadedEdges); validateEPGMGraphElementCollections(loader.getVerticesByGraphVariables("g0", "g1", "g2"), loadedVertices); validateEPGMGraphElementCollections(loader.getEdgesByGraphVariables("g0", "g1", "g2"), loadedEdges); }
@Test public void testDirected() throws Exception { FlinkAsciiGraphLoader loader = new FlinkAsciiGraphLoader(getConfig()); loader.initDatabaseFromFile(getFilePath("/data/gdl/cam_test.gdl")); GraphCollection g = loader.getGraphCollection(); CanonicalAdjacencyMatrixBuilder cam = new CanonicalAdjacencyMatrixBuilder( new GraphHeadToDataString(), new VertexToDataString(), new EdgeToDataString(), true); String result = cam.execute(g).collect().get(0); String expectation = FileUtils.readFileToString( FileUtils.getFile(getFilePath("/data/expected/cam_test_directed"))); assertEquals(expectation, result); }
@Test public void testGetEdges() throws Exception { FlinkAsciiGraphLoader loader = getSocialNetworkLoader(); Collection<Edge> inputEdges = loader.getEdges(); List<Edge> outputEdges = loader.getLogicalGraph(false).getEdges().collect(); validateEPGMElementCollections(inputEdges, outputEdges); validateEPGMGraphElementCollections(inputEdges, outputEdges); } }
@Test public void testGetVertices() throws Exception { FlinkAsciiGraphLoader loader = getSocialNetworkLoader(); Collection<Vertex> inputVertices = loader.getVertices(); List<Vertex> outputVertices = loader.getLogicalGraph(false).getVertices().collect(); validateEPGMElementCollections(inputVertices, outputVertices); validateEPGMGraphElementCollections(inputVertices, outputVertices); }
@Test public void testVertexOnlyOverlappingGraphs() throws Exception { FlinkAsciiGraphLoader loader = getLoaderFromString( "g1[(a)-[e1]->(b)]" + "g2[(a)-[e2]->(b)]" + "expected[(a)(b)]"); LogicalGraph g1 = loader.getLogicalGraphByVariable("g1"); LogicalGraph g2 = loader.getLogicalGraphByVariable("g2"); LogicalGraph expected = loader.getLogicalGraphByVariable("expected"); collectAndAssertTrue(g1.overlap(g2).equalsByElementIds(expected)); }
@Test public void testFromGraphsMethod() throws Exception { LogicalGraph graph1 = loader.getLogicalGraphByVariable("g1"); LogicalGraph graph2 = loader.getLogicalGraphByVariable("g2"); GraphCollection expected = loader.getGraphCollectionByVariables("g1", "g2"); GraphCollection result = factory.fromGraphs(graph1, graph2); collectAndAssertTrue(result.equalsByGraphElementData(expected)); } }
@Test public void testOverlappingCollections() throws Exception { FlinkAsciiGraphLoader loader = getSocialNetworkLoader(); GraphCollection col02 = loader.getGraphCollectionByVariables("g0", "g2"); GraphCollection col12 = loader.getGraphCollectionByVariables("g1", "g2"); GraphCollection result = col02.union(col12); GraphCollection expectation = loader .getGraphCollectionByVariables("g0", "g1", "g2"); checkAssertions(expectation, result, ""); }