@Override public Condition<Long> atLeastTupleCount(TStream<?> stream, long expectedCount) { AtomicLong count = new AtomicLong(); stream.sink(t -> { count.incrementAndGet(); }); return new Condition<Long>() { @Override public boolean valid() { return count.get() >= expectedCount; } @Override public Long getResult() { return count.get(); } }; }
List<TStream<T>> channels = stream.split(width, splitter); for (int ch = 0; ch < width; ch++) channels.set(ch, channels.get(ch).tag("parallel.split-ch"+ch)); channels.set(ch, isolate(channels.get(ch), chBufferSize).tag("parallel.isolated-ch"+ch)); final int finalCh = ch; results.add(pipeline.apply(channels.get(ch), ch) .tag("parallel-ch"+ch) .peek(tuple -> splitter.channelDone(finalCh))); TStream<R> result = results.get(0).union(new HashSet<>(results)).tag("parallel.union");
List<TStream<T>> channels = stream.split(width, splitter); for (int ch = 0; ch < width; ch++) channels.set(ch, channels.get(ch).tag("parallel.split-ch"+ch)); channels.set(ch, isolate(channels.get(ch), chBufferSize).tag("parallel.isolated-ch"+ch)); results.add(pipeline.apply(channels.get(ch), ch).tag("parallel-ch"+ch)); TStream<R> result = results.get(0).union(new HashSet<>(results)).tag("parallel.union");
gaussian = gaussian.peek(g -> System.out.println("R1:" + g)); gaussian = gaussian.filter(g -> g < 10000000); gaussian.print(); gaussian = gaussian.modify(g-> g*3 + 1); TStream<String> s1 = gaussian.map(g -> "g1: " + g.toString()).tag("s1", "gaussian"); s1.sink(tuple -> {}); List<TStream<Double>> splits1 = gaussian.split(11, tuple -> { switch (tuple.toString().charAt(0)) { case '-': //negative numbers TStream<Double> sp0 = splits1.get(0).tag("split","sp0"); sp0 = Metrics.counter(sp0); Metrics.rateMeter(sp0); sp0.print(); TStream<Double> sp0_1 = sp0.tag("split","sp0_1"); sp0_1.print(); TStream<Double> sp1 =splits1.get(1).tag("split","sp1"); sp1 = Metrics.counter(sp1); sp1.print(); TStream<Double> sp2 =splits1.get(2).tag("split","sp2"); sp2 = Metrics.counter(sp2); sp2.print();
public void completeAndValidate(boolean ordered, String msg, Topology t, TStream<String> s, MsgGenerator mgen, int secTimeout, String... expected) throws Exception { s = s.filter(tuple -> tuple.matches(mgen.pattern())); s.sink(tuple -> System.out.println( String.format("[%s][%s] rcvd: %s", t.getName(), simpleTS(), tuple))); super.completeAndValidate(ordered, msg, t, s, secTimeout, expected); }
private void _testFanoutWithPeek(boolean after) throws Exception { Topology t = newTopology(); Graph g = t.graph(); /* -- Filter -- Sink(.) * / * Source -- Peek -- FanOut ---- Modify -- Sink(@) * */ TStream<Integer> d = integers(t, 1, 2, 3); List<Integer> peekedValues = new ArrayList<>(); if (!after) d.peek(tuple -> peekedValues.add(tuple)); TStream<Integer> df = d.filter(tuple -> tuple.intValue() > 0); TStream<Integer> dm = d.modify(tuple -> new Integer(tuple.intValue() + 1)); if (after) d.peek(tuple -> peekedValues.add(tuple)); df.sink(tuple -> System.out.print(".")); dm.sink(tuple -> System.out.print("@")); assertEquals(7, g.getVertices().size()); assertEquals(6, g.getEdges().size()); // Insert counter metrics into all the streams Metrics.counter(t); printGraph(g); assertEquals(10, g.getVertices().size()); assertEquals(9, g.getEdges().size()); }
@Test public void testBlockingDelay() throws Exception { // Timing variances on shared machines can cause this test to fail assumeTrue(!Boolean.getBoolean("edgent.build.ci")); Topology topology = newTopology(); TStream<String> strings = topology.strings("a", "b", "c", "d"); TStream<Long> starts = strings.map(v -> System.currentTimeMillis()); // delay stream starts = PlumbingStreams.blockingDelay(starts, 300, TimeUnit.MILLISECONDS); // calculate delay starts = starts.modify(v -> System.currentTimeMillis() - v); starts = starts.filter(v -> v >= 300); Condition<Long> tc = topology.getTester().tupleCount(starts, 4); complete(topology, tc); assertTrue("valid:" + tc.getResult(), tc.valid()); }
/** * Publishes events derived from {@code stream} using the topic * {@link IotDevicePubSub#EVENTS} as a JsonObject containing eventId, event, * and qos keys. */ @Override public TSink<JsonObject> events(TStream<JsonObject> stream, String eventId, int qos) { stream = stream.map(event -> { JsonObject publishedEvent = new JsonObject(); publishedEvent.addProperty("eventId", eventId); publishedEvent.add("event", event); publishedEvent.addProperty("qos", qos); return publishedEvent; }); return PublishSubscribe.publish(stream, EVENTS_TOPIC, JsonObject.class); }
fanouts.add(isolate(stream, 1).tag("concurrent.isolated-ch"+i)); int ch = 0; for (Function<TStream<T>,TStream<U>> pipeline : pipelines) { results.add(pipeline.apply(fanouts.get(ch)).tag("concurrent-ch"+ch)); ch++; TStream<List<U>> barrier = barrier(results, barrierQueueCapacity).tag("concurrent.barrier"); return barrier.map(combiner);
@Test public void testBasicRead() throws Exception { Topology t = this.newTopology("testBasicRead"); populatePersonsTable(getPersonList()); List<String> expected = expectedPersons(person->true, getPersonList()); JdbcStreams db = new JdbcStreams(t, () -> getDataSource(DB_NAME), dataSource -> connect(dataSource)); // Create a stream of Person from a stream of ids TStream<Person> rcvdPerson = readPersonsTable(t, db, getPersonIdList(), 0/*msec*/); TStream<String> rcvd = rcvdPerson.map(person -> person.toString()); rcvd.sink(tuple -> System.out.println( String.format("%s rcvd: %s", t.getName(), tuple))); completeAndValidate("", t, rcvd, SEC_TIMEOUT, expected.toArray(new String[0])); }
TStream<byte[]> s = t.strings(expected).map(tup -> tup.getBytes(StandardCharsets.UTF_8)); s = PlumbingStreams.blockingOneShotDelay(s, 2, TimeUnit.SECONDS); int restartAfterTupleCnt = 2; CountDownLatch latch = new CountDownLatch(restartAfterTupleCnt); s = s.filter(tuple -> { if (numSent.getAndIncrement() != restartAfterTupleCnt ) return true; .peek(tuple -> latch.countDown()) .map(tup -> new String(tup, StandardCharsets.UTF_8));
@Test public void testValveInitiallyOpen() throws Exception { Topology top = newTopology("testValve"); TStream<Integer> values = top.of(1, 2, 3, 4, 5, 6, 7, 8, 9, 10); Valve<Integer> valve = new Valve<>(); AtomicInteger cnt = new AtomicInteger(); TStream<Integer> filtered = values .peek(tuple -> { // reject 4,5,6 int curCnt = cnt.incrementAndGet(); if (curCnt > 6) valve.setOpen(true); else if (curCnt > 3) valve.setOpen(false); }) .filter(valve); Condition<Long> count = top.getTester().tupleCount(filtered, 7); Condition<List<Integer>> contents = top.getTester().streamContents(filtered, 1,2,3,7,8,9,10 ); complete(top, count); assertTrue(contents.getResult().toString(), contents.valid()); }
@Test public void metricsEverywhereMultiplePeek() throws Exception { Topology t = newTopology(); Graph g = t.graph(); TStream<String> s = t.strings("a", "b", "c"); List<String> peekedValues = new ArrayList<>(); TStream<String> speek = s.peek(tuple -> peekedValues.add(tuple + "1st")); TStream<String> speek2 = speek.peek(tuple -> peekedValues.add(tuple + "2nd")); TStream<String> speek3 = speek2.peek(tuple -> peekedValues.add(tuple + "3rd")); speek3.sink(tuple -> System.out.print(".")); Collection<Vertex<? extends Oplet<?, ?>, ?, ?>> vertices = g.getVertices(); assertEquals(5, vertices.size()); Collection<Edge> edges = g.getEdges(); assertEquals(4, edges.size()); Metrics.counter(t); printGraph(g); // One single counter inserted after the 3rd peek vertices = g.getVertices(); assertEquals(6, vertices.size()); edges = g.getEdges(); assertEquals(5, edges.size()); }
@Test public void testTag() throws Exception { Topology t = newTopology(); List<String> tags = new ArrayList<>(Arrays.asList("tag1", "tag2")); TStream<String> s = t.strings("a", "b"); assertEquals(0, s.getTags().size()); TStream<String> s2 = s.tag("tag1", "tag2"); assertSame(s, s2); assertTrue("s.tags="+s.getTags(), s.getTags().containsAll(tags)); tags.add("tag3"); s.tag("tag3"); assertTrue("s.tags="+s.getTags(), s.getTags().containsAll(tags)); s.tag("tag3", "tag2", "tag1"); // ok to redundantly add assertTrue("s.tags="+s.getTags(), s.getTags().containsAll(tags)); // test access at runtime s2 = s.peek(tuple -> { assertTrue("s.tags="+s.getTags(), s.getTags().containsAll(tags)); }).filter(tuple -> true); // just verify that tag presence doesn't otherwise muck up things Condition<Long> tc = t.getTester().tupleCount(s2, 2); Condition<List<String>> contents = t.getTester().streamContents(s2, "a", "b"); complete(t, tc); assertTrue("contents "+contents.getResult(), contents.valid()); }
/** * Subscribes to commands. */ @Override public TStream<JsonObject> commands(String... commandIdentifiers) { TStream<JsonObject> commandsStream = PublishSubscribe.subscribe(this, COMMANDS_TOPIC, JsonObject.class); if (commandIdentifiers.length > 0) { Set<String> cmdIds = new HashSet<>(Arrays.asList(commandIdentifiers)); commandsStream = commandsStream.filter( cmd -> cmdIds.contains(cmd.get(CMD_ID).getAsString())); } return commandsStream; }
if (numBefore > 0) { for (int i = 0; i < numBefore; i++) s.peek(tuple -> tuple.peekedCnt++); TStream<Peeked> sf = s.filter(tuple -> tuple.value > 0); if (numMiddle > 0) { for (int i = 0; i < numMiddle; i++) s.peek(tuple -> tuple.peekedCnt++); TStream<Peeked> sm = s.modify(tuple -> new Peeked(tuple.value + 37, tuple.peekedCnt)); s.peek(tuple -> tuple.peekedCnt++);
@Test public void testAlias() throws Exception { Topology t = newTopology(); TStream<String> s = t.strings("a", "b"); assertEquals(null, s.getAlias()); TStream<String> s2 = s.alias("sAlias"); assertSame(s, s2); assertEquals("sAlias", s.getAlias()); try { s.alias("another"); // expect ISE - alias already set assertTrue(false); } catch (IllegalStateException e) { ; // expected } // test access at runtime s2 = s.peek(tuple -> { assertEquals("sAlias", s.getAlias()); }).filter(tuple -> true); // just verify that alias presence doesn't otherwise muck up things Condition<Long> tc = t.getTester().tupleCount(s2, 2); Condition<List<String>> contents = t.getTester().streamContents(s2, "a", "b"); complete(t, tc); assertTrue("contents "+contents.getResult(), contents.valid()); }
/** * Increment a counter metric when peeking at each tuple. * * @param <T> * TStream tuple type * @param stream to stream to instrument * @return a {@link TStream} containing the input tuples */ public static <T> TStream<T> counter(TStream<T> stream) { return stream.pipe(new CounterOp<T>()); }