channels.set(ch, channels.get(ch).tag("parallel.split-ch"+ch)); channels.set(ch, isolate(channels.get(ch), chBufferSize).tag("parallel.isolated-ch"+ch)); results.add(pipeline.apply(channels.get(ch), ch).tag("parallel-ch"+ch)); TStream<R> result = results.get(0).union(new HashSet<>(results)).tag("parallel.union");
channels.set(ch, channels.get(ch).tag("parallel.split-ch"+ch)); channels.set(ch, isolate(channels.get(ch), chBufferSize).tag("parallel.isolated-ch"+ch)); results.add(pipeline.apply(channels.get(ch), ch).tag("parallel-ch"+ch)); TStream<R> result = results.get(0).union(new HashSet<>(results)).tag("parallel.union");
private TStream<JsonObject> allCommands() { if (commandStream == null) { String topicFilter = commandTopic(null); commandStream = connector.subscribe(topicFilter, commandQoS, (topic, payload) -> { JsonObject jo = new JsonObject(); jo.addProperty(CMD_DEVICE, deviceId); jo.addProperty(CMD_ID, extractCmd(topic)); jo.addProperty(CMD_TS, System.currentTimeMillis()); String fmt = extractCmdFmt(topic); jo.addProperty(CMD_FORMAT, fmt); if ("json".equals(fmt)) { jo.add(CMD_PAYLOAD, JsonFunctions.fromBytes().apply(payload)); } else { jo.addProperty(CMD_PAYLOAD, new String(payload, StandardCharsets.UTF_8)); } return jo; }) .tag("allDeviceCmds"); } return commandStream; }
fanouts.add(isolate(stream, 1).tag("concurrent.isolated-ch"+i)); int ch = 0; for (Function<TStream<T>,TStream<U>> pipeline : pipelines) { results.add(pipeline.apply(fanouts.get(ch)).tag("concurrent-ch"+ch)); ch++; TStream<List<U>> barrier = barrier(results, barrierQueueCapacity).tag("concurrent.barrier");
fanouts.add(isolate(stream, 1).tag("concurrent.isolated-ch"+i)); int ch = 0; for (Function<TStream<T>,TStream<U>> pipeline : pipelines) { results.add(pipeline.apply(fanouts.get(ch)).tag("concurrent-ch"+ch)); ch++; TStream<List<U>> barrier = barrier(results, barrierQueueCapacity).tag("concurrent.barrier");
channels.set(ch, channels.get(ch).tag("parallel.split-ch"+ch)); channels.set(ch, isolate(channels.get(ch), chBufferSize).tag("parallel.isolated-ch"+ch)); final int finalCh = ch; results.add(pipeline.apply(channels.get(ch), ch) .tag("parallel-ch"+ch) .peek(tuple -> splitter.channelDone(finalCh))); TStream<R> result = results.get(0).union(new HashSet<>(results)).tag("parallel.union");
channels.set(ch, channels.get(ch).tag("parallel.split-ch"+ch)); channels.set(ch, isolate(channels.get(ch), chBufferSize).tag("parallel.isolated-ch"+ch)); final int finalCh = ch; results.add(pipeline.apply(channels.get(ch), ch) .tag("parallel-ch"+ch) .peek(tuple -> splitter.channelDone(finalCh))); TStream<R> result = results.get(0).union(new HashSet<>(results)).tag("parallel.union");
@SuppressWarnings("unused") private BiFunction<TStream<JsonObject>,Integer,TStream<JsonObject>> fakeParallelPipelineTiming(long period, TimeUnit unit) { return (stream,channel) -> stream .map(jo -> { jo.addProperty("startPipelineMsec", System.currentTimeMillis()); return jo; }) .map(fakeJsonAnalytic(channel, period, unit)) .filter(t->true) .map(jo -> { jo.addProperty("endPipelineMsec", System.currentTimeMillis()); return jo; }) .tag("pipeline-ch"+channel); }
private Function<TStream<Integer>,TStream<JsonObject>> fakePipeline(int channel, long period, TimeUnit unit) { return stream -> stream.map(fakeAnalytic(channel, period, unit)).filter(t->true).tag("pipeline-ch"+channel); }
TStream<String> s1 = gaussian.map(g -> "g1: " + g.toString()).tag("s1", "gaussian"); s1.sink(tuple -> {}); TStream<Double> sp0 = splits1.get(0).tag("split","sp0"); sp0 = Metrics.counter(sp0); Metrics.rateMeter(sp0); TStream<Double> sp0_1 = sp0.tag("split","sp0_1"); sp0_1.print(); TStream<Double> sp1 =splits1.get(1).tag("split","sp1"); sp1 = Metrics.counter(sp1); sp1.print(); TStream<Double> sp2 =splits1.get(2).tag("split","sp2"); sp2 = Metrics.counter(sp2); sp2.print(); TStream<Double> sp3 =splits1.get(3).tag("split","sp3"); sp3 = Metrics.counter(sp3); sp3.print(); TStream<Double> sp4 =splits1.get(4).tag("split","sp4"); sp4 = Metrics.counter(sp4); sp4.print(); TStream<Double> sp5 =splits1.get(5).tag("split","sp5"); sp5 = Metrics.counter(sp1); sp5.print();
private BiFunction<TStream<Integer>,Integer,TStream<JsonObject>> fakeParallelPipeline(long period, TimeUnit unit) { return (stream,channel) -> stream .map(value -> fakeParallelAnalytic(period, unit).apply(value,channel)) .filter(t->true) .tag("pipeline-ch"+channel); }
}; TStream<Integer> result = PlumbingStreams.concurrent(values, pipelines, combiner).tag("result");
TStream<JsonObject> result = PlumbingStreams.parallelBalanced(values, width, pipeline).tag("result"); TStream<Integer> result2 = result.map(jo -> { int r = jo.get("result").getAsInt();
TStream<Integer> values = top.of(resultTuples); TStream<JsonObject> result = PlumbingStreams.parallel(values, width, splitter, pipeline).tag("result"); TStream<Integer> result2 = result.map(jo -> { int r = jo.get("result").getAsInt();
TStream<Integer> values = top.of(resultTuples); TStream<JsonObject> result = PlumbingStreams.parallelMap(values, width, splitter, mapper).tag("result"); TStream<Integer> result2 = result.map(jo -> { int r = jo.get("result").getAsInt();
@Test public void testTag() throws Exception { Topology t = newTopology(); List<String> tags = new ArrayList<>(Arrays.asList("tag1", "tag2")); TStream<String> s = t.strings("a", "b"); assertEquals(0, s.getTags().size()); TStream<String> s2 = s.tag("tag1", "tag2"); assertSame(s, s2); assertTrue("s.tags="+s.getTags(), s.getTags().containsAll(tags)); tags.add("tag3"); s.tag("tag3"); assertTrue("s.tags="+s.getTags(), s.getTags().containsAll(tags)); s.tag("tag3", "tag2", "tag1"); // ok to redundantly add assertTrue("s.tags="+s.getTags(), s.getTags().containsAll(tags)); // test access at runtime s2 = s.peek(tuple -> { assertTrue("s.tags="+s.getTags(), s.getTags().containsAll(tags)); }).filter(tuple -> true); // just verify that tag presence doesn't otherwise muck up things Condition<Long> tc = t.getTester().tupleCount(s2, 2); Condition<List<String>> contents = t.getTester().streamContents(s2, "a", "b"); complete(t, tc); assertTrue("contents "+contents.getResult(), contents.valid()); }