@Override public void restoreState(Pair<Long, Long> state) { currentCount.set(state.getFirst()); processed = state.getSecond(); }
@Override public void initState(KeyValueState<String, Pair<Long, Long>> state) { this.state = state; globalAvg = state.get(STATE_KEY, Pair.of(0L, 0L)); LOG.info("initState with global avg [" + (double) globalAvg.getFirst() / globalAvg.getSecond() + "]"); }
@Override public void execute(Pair<K, V> input) { K key = input.getFirst(); V val = input.getSecond(); V agg = state.get(key); final V res = (agg == null) ? val : reducer.apply(agg, val); state.put(key, res); mayBeForwardAggUpdate(() -> Pair.of(key, res)); }
private <T1, T2> List<Tuple3<K, T1, T2>> join(Multimap<K, T1> tab, List<Pair<K, T2>> rows, JoinType leftType, JoinType rightType) { List<Tuple3<K, T1, T2>> res = new ArrayList<>(); for (Pair<K, T2> row : rows) { K key = row.getFirst(); Collection<T1> values = tab.removeAll(key); if (values.isEmpty()) { if (rightType == JoinType.OUTER) { res.add(new Tuple3<>(row.getFirst(), null, row.getSecond())); } } else { for (T1 mapValue : values) { res.add(new Tuple3<>(row.getFirst(), mapValue, row.getSecond())); } } } // whatever remains in the tab are non matching left rows. if (leftType == JoinType.OUTER) { for (Map.Entry<K, T1> row : tab.entries()) { res.add(new Tuple3<>(row.getKey(), row.getValue(), null)); } } return res; }
@Override public void execute(Pair<K, A> input) { K key = input.getFirst(); A val = input.getSecond(); A accumulator = state.get(key); if (accumulator == null) { accumulator = aggregator.init(); } state.put(key, aggregator.merge(accumulator, val)); mayBeForwardAggUpdate(() -> Pair.of(key, aggregator.result(state.get(key)))); }
private <T> Multimap<K, T> getJoinTable(List<Pair<K, T>> rows) { Multimap<K, T> m = ArrayListMultimap.create(); for (Pair<K, T> v : rows) { m.put(v.getFirst(), v.getSecond()); } return m; }
@Override public void execute(Pair<K, ?> input, String sourceStream) { K key = input.getFirst(); if (sourceStream.equals(leftStream)) { V1 val = (V1) input.getSecond(); Pair<K, V1> pair = Pair.of(key, val); leftRows.add(pair); if (!context.isWindowed()) { joinAndForward(Collections.singletonList(pair), rightRows); } } else if (sourceStream.equals(rightStream)) { V2 val = (V2) input.getSecond(); Pair<K, V2> pair = Pair.of(key, val); rightRows.add(pair); if (!context.isWindowed()) { joinAndForward(leftRows, Collections.singletonList(pair)); } } }
@Override public void execute(Pair<K, V> input) { K key = input.getFirst(); V val = input.getSecond(); A accumulator = state.get(key); if (accumulator == null) { accumulator = aggregator.init(); } state.put(key, aggregator.apply(accumulator, val)); if (emitAggregate) { mayBeForwardAggUpdate(() -> Pair.of(key, state.get(key))); } else { mayBeForwardAggUpdate(() -> Pair.of(key, aggregator.result(state.get(key)))); } }
@Override public void execute(Pair<K, ?> input, String sourceStream) { K key = input.getFirst(); if (sourceStream.equals(firstStream)) { V1 val = (V1) input.getSecond(); firstMap.put(key, val); } else if (sourceStream.equals(secondStream)) { V2 val = (V2) input.getSecond(); secondMap.put(key, val); } if (!context.isWindowed()) { forwardValues(); } }
@Override public void execute(Pair<K, V> input) { context.forward(Pair.of(input.getFirst(), function.apply(input.getSecond()))); } }
@Override protected void execute(Pair<K, V> input) { for (R res : function.apply(input.getSecond())) { context.forward(Pair.of(input.getFirst(), res)); } } }
@Override public void execute(TupleWindow inputWindow) { LOG.trace("Window triggered at {}, inputWindow {}", new Date(), inputWindow); if (delegate.isEventTimestamp()) { delegate.setEventTimestamp(inputWindow.getEndTimestamp()); } for (Tuple tuple : inputWindow.get()) { Pair<Object, String> valueAndStream = delegate.getValueAndStream(tuple); if (!StreamUtil.isPunctuation(valueAndStream.getFirst())) { delegate.process(valueAndStream.getFirst(), valueAndStream.getSecond()); } } for (String stream : delegate.getInitialStreams()) { delegate.process(PUNCTUATION, stream); } }
@Override public void execute(TupleWindow window) { int sum = 0; int count = 0; // iterate over tuples in the current window Iterator<Tuple> it = window.getIter(); while (it.hasNext()) { Tuple tuple = it.next(); sum += tuple.getInteger(0); ++count; } LOG.debug("Count : {}", count); globalAvg = Pair.of(globalAvg.getFirst() + sum, globalAvg.getSecond() + count); // update the value in state state.put(STATE_KEY, globalAvg); // emit the averages downstream collector.emit(new Values(new Averages((double) globalAvg.getFirst() / globalAvg.getSecond(), (double) sum / count))); }
@Test public void testExternalDataSource() throws Exception { List<String> stmt = new ArrayList<>(); stmt.add("CREATE EXTERNAL TABLE FOO (ID INT PRIMARY KEY) LOCATION 'mock:///foo'"); stmt.add("CREATE EXTERNAL TABLE BAR (ID INT PRIMARY KEY) LOCATION 'mock:///foo'"); stmt.add("INSERT INTO BAR SELECT STREAM ID + 1 FROM FOO WHERE ID > 2"); StormSqlLocalClusterImpl impl = new StormSqlLocalClusterImpl(); List<Pair<Object, Values>> values = TestUtils.MockInsertBolt.getCollectedValues(); impl.runLocal(cluster, stmt, (__) -> values.size() >= 2, WAIT_TIMEOUT_MS); Assert.assertEquals(2, values.size()); Assert.assertEquals(4, values.get(0).getFirst()); Assert.assertEquals(5, values.get(1).getFirst()); }
@Test public void testExternalUdf() throws Exception { List<String> stmt = new ArrayList<>(); stmt.add("CREATE EXTERNAL TABLE FOO (ID INT PRIMARY KEY) LOCATION 'mock:///foo'"); stmt.add("CREATE EXTERNAL TABLE BAR (ID INT PRIMARY KEY) LOCATION 'mock:///foo'"); stmt.add("CREATE FUNCTION MYPLUS AS 'org.apache.storm.sql.TestUtils$MyPlus'"); stmt.add("INSERT INTO BAR SELECT STREAM MYPLUS(ID, 1) FROM FOO WHERE ID > 2"); StormSqlLocalClusterImpl impl = new StormSqlLocalClusterImpl(); List<Pair<Object, Values>> values = TestUtils.MockInsertBolt.getCollectedValues(); impl.runLocal(cluster, stmt, (__) -> values.size() >= 2, WAIT_TIMEOUT_MS); Assert.assertEquals(2, values.size()); Assert.assertEquals(4, values.get(0).getFirst()); Assert.assertEquals(5, values.get(1).getFirst()); }
@Override public <T> void forward(T input) { if (PUNCTUATION.equals(input)) { emit(punctuation, punctuationStreamId); maybeAck(); } else if (processorNode.emitsPair()) { Pair<?, ?> value = (Pair<?, ?>) input; emit(new Values(value.getFirst(), value.getSecond()), outputStreamId); } else { emit(new Values(input), outputStreamId); } }
@Test public void testExternalDataSourceNested() throws Exception { List<String> stmt = new ArrayList<>(); stmt.add("CREATE EXTERNAL TABLE FOO (ID INT PRIMARY KEY, MAPFIELD ANY, NESTEDMAPFIELD ANY, ARRAYFIELD ANY) LOCATION 'mocknested:///foo'"); stmt.add("CREATE EXTERNAL TABLE BAR (ID INT PRIMARY KEY, MAPFIELD ANY, NESTEDMAPFIELD ANY, ARRAYFIELD ANY) LOCATION 'mocknested:///foo'"); stmt.add("INSERT INTO BAR SELECT STREAM ID, MAPFIELD['c'], NESTEDMAPFIELD, ARRAYFIELD " + "FROM FOO " + "WHERE CAST(MAPFIELD['b'] AS INTEGER) = 2 AND CAST(ARRAYFIELD[2] AS INTEGER) = 200"); StormSqlLocalClusterImpl impl = new StormSqlLocalClusterImpl(); List<Pair<Object, Values>> values = TestUtils.MockInsertBolt.getCollectedValues();; impl.runLocal(cluster, stmt, (__) -> values.size() >= 1, WAIT_TIMEOUT_MS); Map<String, Integer> map = ImmutableMap.of("b", 2, "c", 4); Map<String, Map<String, Integer>> nestedMap = ImmutableMap.of("a", map); Assert.assertEquals(2, values.get(0).getFirst()); Assert.assertEquals(new Values(2, 4, nestedMap, Arrays.asList(100, 200, 300)), values.get(0).getSecond()); }
@Override protected void execute(Pair<K, V> input) { K key = input.getFirst(); V val = input.getSecond(); R agg = keyValueState.get(key); if (agg == null) { agg = stateUpdater.init(); } R newAgg = stateUpdater.apply(agg, val); keyValueState.put(key, newAgg); context.forward(Pair.of(key, newAgg)); } }
void processAndAck(Tuple input) { RefCountedTuple refCountedTuple = new RefCountedTuple(input); setAnchor(refCountedTuple); if (isEventTimestamp()) { setEventTimestamp(input.getLongByField(getTimestampField())); } Pair<Object, String> valueAndStream = getValueAndStream(input); process(valueAndStream.getFirst(), valueAndStream.getSecond()); ack(refCountedTuple); }