/** * Create an instance of {@link GraphType} using the specified * {@link IdMapper} to generate unique object identifiers. * @param g the associated Graph * @param ids the id mapper */ public GraphType(Graph g, IdMapper<String> ids) { if (ids == null) { ids = new GraphType.Mapper(); } ArrayList<VertexType<?,?>> vertices = new ArrayList<VertexType<?,?>>(); ArrayList<EdgeType> edges = new ArrayList<EdgeType>(); for (Vertex<? extends Oplet<?,?>, ?, ?> v : g.getVertices()) { @SuppressWarnings({ "rawtypes", "unchecked" }) VertexType<?,?> vertex = new VertexType(v, ids); vertices.add(vertex); } for (Edge e : g.getEdges()) { edges.add(new EdgeType(e, ids)); } this.vertices = vertices; this.edges = edges; }
@Override public <U> TStream<U> fanin(FanIn<T,U> fanin, List<TStream<T>> others) { if (others.isEmpty() || others.size() == 1 && others.contains(this)) throw new IllegalArgumentException("others"); // use pipe() if (new HashSet<>(others).size() != others.size()) throw new IllegalArgumentException("others has dups"); for (TStream<T> other : others) verify(other); others = new ArrayList<>(others); others.add(0, this); Vertex<Oplet<T,U>, T, U> fanInVertex = graph().insert(fanin, others.size(), 1); int inputPort = 0; for (TStream<T> other : others) { @SuppressWarnings("unchecked") ConnectorStream<G,T> cs = (ConnectorStream<G, T>) other; cs.connector.connect(fanInVertex, inputPort++); } return derived(fanInVertex.getConnectors().get(0)); }
protected <N extends Pipe<T, U>, U> TStream<U> connectPipe(N pipeOp) { return derived(graph().pipe(connector, pipeOp)); }
Vertex<TestOp<String, Integer>, String, Integer> v = g.insert(op, 1, 1); assertNotNull(v); assertSame(op, v.getInstance()); Collection<Vertex<? extends Oplet<?, ?>, ?, ?>> c = getGraph().getVertices(); assertNotNull(c); assertTrue(getGraph().getEdges().isEmpty()); Vertex<TestOp<Integer, Void>, Integer, Void> v2 = g.insert(op2, 1, 0); c = getGraph().getVertices(); assertNotNull(c); assertSame(v2, c.toArray()[1]); assertTrue(getGraph().getEdges().isEmpty()); Collection<Edge> edges = getGraph().getEdges(); assertFalse(edges.isEmpty()); assertEquals(1, edges.size());
@Test public void testEmptyGraph() { assertTrue(getGraph().getVertices().isEmpty()); }
@Test public void testGraphToJson2() { Graph g = getGraph(); TestOp<String, Integer> op1 = new TestOp<>(); Vertex<TestOp<String, Integer>, String, Integer> v = g.insert(op1, 1, 1); TestOp<Integer, Integer> op2 = new TestOp<>(); /*Connector<Integer> out2 = */g.pipe(v.getConnectors().get(0), op2); Gson gson = new GsonBuilder().setPrettyPrinting().create(); String json = gson.toJson(new GraphType(g)); GraphType gt2 = new Gson().fromJson(json, GraphType.class); assertEquals(2, gt2.getVertices().size()); assertEquals(1, gt2.getEdges().size()); }
/** * Add counter metrics to all the topology's streams. * <p> * {@link CounterOp} oplets are inserted between every two graph * vertices with the following exceptions: * <ul> * <li>Oplets are only inserted upstream from a FanOut oplet.</li> * <li>If a chain of Peek oplets exists between oplets A and B, a Metric * oplet is inserted after the last Peek, right upstream from oplet B.</li> * <li>If a chain a Peek oplets is followed by a FanOut, a metric oplet is * inserted between the last Peek and the FanOut oplet.</li> * <li>Oplets are not inserted immediately downstream from another * {@code CounterOp} oplet (but they are inserted upstream from one.)</li> * </ul> * The implementation is not idempotent: Calling the method twice * will insert a new set of metric oplets into the graph. * @param t * The topology * @see org.apache.edgent.graph.Graph#peekAll(org.apache.edgent.function.Supplier, org.apache.edgent.function.Predicate) Graph.peekAll() */ public static void counter(Topology t) { // peekAll() embodies the above exclusion semantics t.graph().peekAll( () -> new CounterOp<>(), v -> !(v.getInstance() instanceof CounterOp) ); } }
protected <N extends Source<T>, T> TStream<T> sourceStream(N sourceOp) { return new ConnectorStream<GraphTopology<X>, T>(this, graph().source(sourceOp)); }
@Test public void testMetricsEverywhere() throws Exception { Topology t = newTopology(); TStream<String> s = t.strings("a", "b", "c"); // Condition inserts a sink Condition<Long> tc = t.getTester().tupleCount(s, 3); Graph g = t.graph(); Collection<Vertex<? extends Oplet<?, ?>, ?, ?>> vertices = g.getVertices(); // Two vertices before submission assertEquals(2, vertices.size()); complete(t, tc); // At least three vertices after submission // (provide may have added other oplets as well) Collection<Vertex<? extends Oplet<?, ?>, ?, ?>> verticesAfterSubmit = g.getVertices(); assertTrue("size="+verticesAfterSubmit.size(), verticesAfterSubmit.size() >= 3); // There is exactly one vertex for a metric oplet int numOplets = 0; for (Vertex<? extends Oplet<?, ?>, ?, ?> v : verticesAfterSubmit) { Oplet<?,?> oplet = v.getInstance(); if (oplet instanceof CounterOp) { numOplets++; } } assertEquals(1, numOplets); }
@Test public void testGraphToJson4() { Graph g = getGraph(); /* /-- V2 * V0(Integer)-- V1(Double)-- FanOut * \-- V3 */ Vertex<TestOp<String, Integer>, String, Integer> v0 = g.insert(new TestOp<>(), 1, 1); Connector<Integer> out0 = v0.getConnectors().get(0); Connector<Double> out1 = g.pipe(out0, new TestOp<Integer, Double>()); Vertex<TestOp<Double, String>, Double, String> v2 = g.insert(new TestOp<Double, String>(), 1, 1); Vertex<TestOp<Double, String>, Double, String> v3 = g.insert(new TestOp<Double, String>(), 1, 1); out1.connect(v2, 0); out1.connect(v3, 0); Gson gson = new GsonBuilder().setPrettyPrinting().create(); String json = gson.toJson(new GraphType(g)); GraphType gt2 = new Gson().fromJson(json, GraphType.class); assertEquals(5, gt2.getVertices().size()); assertEquals(4, gt2.getEdges().size()); }
t.graph().peekAll( () -> { return new org.apache.edgent.streamscope.oplets.StreamScope<>(new StreamScope<>()); }, Functions.alwaysTrue());
protected <N extends Source<T>, T> TStream<T> sourceStream(N sourceOp) { return new ConnectorStream<GraphTopology<X>, T>(this, graph().source(sourceOp)); }
/** * Create an instance of {@link GraphType} using the specified * {@link IdMapper} to generate unique object identifiers. * @param g the associated Graph * @param ids the id mapper */ public GraphType(Graph g, IdMapper<String> ids) { if (ids == null) { ids = new GraphType.Mapper(); } ArrayList<VertexType<?,?>> vertices = new ArrayList<VertexType<?,?>>(); ArrayList<EdgeType> edges = new ArrayList<EdgeType>(); for (Vertex<? extends Oplet<?,?>, ?, ?> v : g.getVertices()) { @SuppressWarnings({ "rawtypes", "unchecked" }) VertexType<?,?> vertex = new VertexType(v, ids); vertices.add(vertex); } for (Edge e : g.getEdges()) { edges.add(new EdgeType(e, ids)); } this.vertices = vertices; this.edges = edges; }
Collection<Vertex<? extends Oplet<?, ?>, ?, ?>> vertices = g.getVertices(); Collection<Vertex<? extends Oplet<?, ?>, ?, ?>> verticesAfterSubmit = g.getVertices(); assertTrue("size="+verticesAfterSubmit.size(), verticesAfterSubmit.size() >= 7);
@Override public <U> TStream<U> fanin(FanIn<T,U> fanin, List<TStream<T>> others) { if (others.isEmpty() || others.size() == 1 && others.contains(this)) throw new IllegalArgumentException("others"); // use pipe() if (new HashSet<>(others).size() != others.size()) throw new IllegalArgumentException("others has dups"); for (TStream<T> other : others) verify(other); others = new ArrayList<>(others); others.add(0, this); Vertex<Oplet<T,U>, T, U> fanInVertex = graph().insert(fanin, others.size(), 1); int inputPort = 0; for (TStream<T> other : others) { @SuppressWarnings("unchecked") ConnectorStream<G,T> cs = (ConnectorStream<G, T>) other; cs.connector.connect(fanInVertex, inputPort++); } return derived(fanInVertex.getConnectors().get(0)); }
protected <N extends Pipe<T, U>, U> TStream<U> connectPipe(N pipeOp) { return derived(graph().pipe(connector, pipeOp)); }
/** * Test Peek. This will only work with an embedded setup. * * @throws Exception on failure */ @Test public void metricsEverywherePeek() throws Exception { Topology t = newTopology(); Graph g = t.graph(); TStream<String> s = t.strings("a", "b", "c"); List<String> peekedValues = new ArrayList<>(); TStream<String> speek = s.peek(tuple -> peekedValues.add(tuple)); speek.sink(tuple -> System.out.print(".")); Collection<Vertex<? extends Oplet<?, ?>, ?, ?>> vertices = g.getVertices(); assertEquals(3, vertices.size()); Collection<Edge> edges = g.getEdges(); assertEquals(2, edges.size()); Metrics.counter(t); printGraph(g); // One single counter inserted after the peek vertices = g.getVertices(); assertEquals(4, vertices.size()); edges = g.getEdges(); assertEquals(3, edges.size()); }
Collection<Vertex<? extends Oplet<?, ?>, ?, ?>> vertices = t.graph().getVertices(); PeriodicSource<?> src = null; for (Vertex<? extends Oplet<?, ?>, ?, ?> v : vertices) {
@Override public List<TStream<T>> split(int n, ToIntFunction<T> splitter) { if (n <= 0) throw new IllegalArgumentException("n <= 0"); Split<T> splitOp = new Split<T>(splitter); Vertex<Split<T>, T, T> splitVertex = graph().insert(splitOp, 1, n); connector.connect(splitVertex, 0); List<TStream<T>> outputs = new ArrayList<>(n); for (int i = 0; i < n; i++) { outputs.add(derived(splitVertex.getConnectors().get(i))); } return outputs; }
private void _testFanoutWithPeek(boolean after) throws Exception { Topology t = newTopology(); Graph g = t.graph(); /* -- Filter -- Sink(.) * / * Source -- Peek -- FanOut ---- Modify -- Sink(@) * */ TStream<Integer> d = integers(t, 1, 2, 3); List<Integer> peekedValues = new ArrayList<>(); if (!after) d.peek(tuple -> peekedValues.add(tuple)); TStream<Integer> df = d.filter(tuple -> tuple.intValue() > 0); TStream<Integer> dm = d.modify(tuple -> new Integer(tuple.intValue() + 1)); if (after) d.peek(tuple -> peekedValues.add(tuple)); df.sink(tuple -> System.out.print(".")); dm.sink(tuple -> System.out.print("@")); assertEquals(7, g.getVertices().size()); assertEquals(6, g.getEdges().size()); // Insert counter metrics into all the streams Metrics.counter(t); printGraph(g); assertEquals(10, g.getVertices().size()); assertEquals(9, g.getEdges().size()); }