@Test public void testFromIndexedDataSets() throws Exception { FlinkAsciiGraphLoader loader = getSocialNetworkLoader(); Map<String, DataSet<GraphHead>> indexedGraphHeads = loader.getGraphHeads().stream() .collect(Collectors.groupingBy(GraphHead::getLabel)).entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> getExecutionEnvironment().fromCollection(e.getValue()))); Map<String, DataSet<Vertex>> indexedVertices = loader.getVertices().stream() .collect(Collectors.groupingBy(Vertex::getLabel)).entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> getExecutionEnvironment().fromCollection(e.getValue()))); Map<String, DataSet<Edge>> indexedEdges = loader.getEdges().stream() .collect(Collectors.groupingBy(Edge::getLabel)).entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> getExecutionEnvironment().fromCollection(e.getValue()))); GraphCollectionLayout<GraphHead, Vertex, Edge> collectionLayout = getFactory() .fromIndexedDataSets(indexedGraphHeads, indexedVertices, indexedEdges); Collection<GraphHead> loadedGraphHeads = Lists.newArrayList(); Collection<Vertex> loadedVertices = Lists.newArrayList(); Collection<Edge> loadedEdges = Lists.newArrayList(); collectionLayout.getGraphHeads().output(new LocalCollectionOutputFormat<>( loadedGraphHeads)); collectionLayout.getVertices().output(new LocalCollectionOutputFormat<>( loadedVertices)); collectionLayout.getEdges().output(new LocalCollectionOutputFormat<>( loadedEdges)); getExecutionEnvironment().execute(); validateEPGMElementCollections(loader.getGraphHeads(), loadedGraphHeads); validateEPGMElementCollections(loader.getVertices(), loadedVertices); validateEPGMElementCollections(loader.getEdges(), loadedEdges); validateEPGMGraphElementCollections(loader.getVertices(), loadedVertices); validateEPGMGraphElementCollections(loader.getEdges(), loadedEdges); }
/** * Returns a collection of all logical graph contained in the database. * * @return collection of all logical graphs */ public GraphCollection getGraphCollection() { ExecutionEnvironment env = config.getExecutionEnvironment(); DataSet<Vertex> newVertices = env.fromCollection(getVertices()) .filter(vertex -> vertex.getGraphCount() > 0); DataSet<Edge> newEdges = env.fromCollection(getEdges()) .filter(edge -> edge.getGraphCount() > 0); return config.getGraphCollectionFactory() .fromDataSets(env.fromCollection(getGraphHeads()), newVertices, newEdges); }
/** * Returns a collection of all logical graph contained in the database. * * @return collection of all logical graphs */ public GraphCollection getGraphCollection() { ExecutionEnvironment env = config.getExecutionEnvironment(); DataSet<Vertex> newVertices = env.fromCollection(getVertices()) .filter(vertex -> vertex.getGraphCount() > 0); DataSet<Edge> newEdges = env.fromCollection(getEdges()) .filter(edge -> edge.getGraphCount() > 0); return config.getGraphCollectionFactory() .fromDataSets(env.fromCollection(getGraphHeads()), newVertices, newEdges); }
@Test public void testFromCollections() throws Exception { FlinkAsciiGraphLoader loader = getSocialNetworkLoader(); GraphCollectionLayout<GraphHead, Vertex, Edge> collectionLayout = getFactory() .fromCollections(loader.getGraphHeads(), loader.getVertices(), loader.getEdges()); Collection<GraphHead> loadedGraphHeads = Lists.newArrayList(); Collection<Vertex> loadedVertices = Lists.newArrayList(); Collection<Edge> loadedEdges = Lists.newArrayList(); collectionLayout.getGraphHeads().output(new LocalCollectionOutputFormat<>(loadedGraphHeads)); collectionLayout.getVertices().output(new LocalCollectionOutputFormat<>(loadedVertices)); collectionLayout.getEdges().output(new LocalCollectionOutputFormat<>(loadedEdges)); getExecutionEnvironment().execute(); validateEPGMElementCollections(loader.getGraphHeads(), loadedGraphHeads); validateEPGMElementCollections(loader.getVertices(), loadedVertices); validateEPGMElementCollections(loader.getEdges(), loadedEdges); validateEPGMGraphElementCollections(loader.getVertices(), loadedVertices); validateEPGMGraphElementCollections(loader.getEdges(), loadedEdges); }
@Test public void testFromDataSetsWithoutGraphHead() throws Exception { FlinkAsciiGraphLoader loader = getLoaderFromString("()-[]->(),[()]"); GraphCollectionLayout<GraphHead, Vertex, Edge> collectionLayout = getFactory() .fromDataSets( getExecutionEnvironment().fromCollection(loader.getGraphHeads()), getExecutionEnvironment().fromCollection(loader.getVertices()), getExecutionEnvironment().fromCollection(loader.getEdges())); Collection<GraphHead> loadedGraphHeads = Lists.newArrayList(); Collection<Vertex> loadedVertices = Lists.newArrayList(); Collection<Edge> loadedEdges = Lists.newArrayList(); collectionLayout.getGraphHeads().output(new LocalCollectionOutputFormat<>(loadedGraphHeads)); collectionLayout.getVertices().output(new LocalCollectionOutputFormat<>(loadedVertices)); collectionLayout.getEdges().output(new LocalCollectionOutputFormat<>(loadedEdges)); getExecutionEnvironment().execute(); validateEPGMElementCollections(loader.getGraphHeads(), loadedGraphHeads); validateEPGMElementCollections(loader.getVertices(), loadedVertices); validateEPGMElementCollections(loader.getEdges(), loadedEdges); validateEPGMGraphElementCollections(loader.getVertices(), loadedVertices); validateEPGMGraphElementCollections(loader.getEdges(), loadedEdges); }
@Test public void testFromDataSets() throws Exception { FlinkAsciiGraphLoader loader = getSocialNetworkLoader(); Collection<GraphHead> graphHeads = loader.getGraphHeads(); Collection<Vertex> vertices = loader.getVertices(); Collection<Edge> edges = loader.getEdges(); DataSet<GraphHead> graphHeadDataSet = getExecutionEnvironment() .fromCollection(graphHeads); DataSet<Vertex> vertexDataSet = getExecutionEnvironment() .fromCollection(vertices); DataSet<Edge> edgeDataSet = getExecutionEnvironment() .fromCollection(edges); GraphCollectionLayout<GraphHead, Vertex, Edge> collectionLayout = getFactory() .fromDataSets(graphHeadDataSet, vertexDataSet, edgeDataSet); Collection<GraphHead> loadedGraphHeads = Lists.newArrayList(); Collection<Vertex> loadedVertices = Lists.newArrayList(); Collection<Edge> loadedEdges = Lists.newArrayList(); collectionLayout.getGraphHeads().output(new LocalCollectionOutputFormat<>(loadedGraphHeads)); collectionLayout.getVertices().output(new LocalCollectionOutputFormat<>(loadedVertices)); collectionLayout.getEdges().output(new LocalCollectionOutputFormat<>(loadedEdges)); getExecutionEnvironment().execute(); validateEPGMElementCollections(graphHeads, loadedGraphHeads); validateEPGMElementCollections(vertices, loadedVertices); validateEPGMElementCollections(edges, loadedEdges); validateEPGMGraphElementCollections(vertices, loadedVertices); validateEPGMGraphElementCollections(edges, loadedEdges); }
@Test public void write() throws Exception { AccumuloEPGMStore accumuloStore = new AccumuloEPGMStore(AccumuloTestSuite.getAcConfig(TEST_02)); FlinkAsciiGraphLoader loader = new FlinkAsciiGraphLoader( GradoopFlinkConfig.createConfig(getExecutionEnvironment())); InputStream inputStream = getClass().getResourceAsStream( GradoopTestUtils.SOCIAL_NETWORK_GDL_FILE); loader.initDatabaseFromStream(inputStream); GradoopFlinkConfig flinkConfig = GradoopFlinkConfig.createConfig(getExecutionEnvironment()); new AccumuloDataSink(accumuloStore, flinkConfig) .write(flinkConfig.getGraphCollectionFactory() .fromCollections( loader.getGraphHeads(), loader.getVertices(), loader.getEdges())); getExecutionEnvironment().execute(); accumuloStore.flush(); validateEPGMElementCollections(loader.getGraphHeads(), accumuloStore.getGraphSpace().readRemainsAndClose()); validateEPGMElementCollections(loader.getVertices(), accumuloStore.getVertexSpace().readRemainsAndClose()); validateEPGMGraphElementCollections(loader.getVertices(), accumuloStore.getVertexSpace().readRemainsAndClose()); validateEPGMElementCollections(loader.getEdges(), accumuloStore.getEdgeSpace().readRemainsAndClose()); validateEPGMGraphElementCollections(loader.getEdges(), accumuloStore.getEdgeSpace().readRemainsAndClose()); accumuloStore.close(); }
loader.getGraphHeads(), loader.getVertices(), loader.getEdges())); loader.getGraphHeads(), newStore.getGraphSpace().readRemainsAndClose() );