/** * Constructs a new pair of values. * * @param first the first value * @param second the second value * @param <T1> the type of the first value * @param <T2> the type of the second value * @return a new pair of values */ public static <T1, T2> Pair<T1, T2> of(T1 first, T2 second) { return new Pair<>(first, second); }
/** * Joins two values and produces a {@link Pair} of the values as the result. * * @param value1 the first value * @param value2 the second value * @return a pair of the first and second value */ @Override public Pair<V1, V2> apply(V1 value1, V2 value2) { return Pair.of(value1, value2); } }
@Test public void testExternalDataSource() throws Exception { List<String> stmt = new ArrayList<>(); stmt.add("CREATE EXTERNAL TABLE FOO (ID INT PRIMARY KEY) LOCATION 'mock:///foo'"); stmt.add("CREATE EXTERNAL TABLE BAR (ID INT PRIMARY KEY) LOCATION 'mock:///foo'"); stmt.add("INSERT INTO BAR SELECT STREAM ID + 1 FROM FOO WHERE ID > 2"); StormSqlLocalClusterImpl impl = new StormSqlLocalClusterImpl(); List<Pair<Object, Values>> values = TestUtils.MockInsertBolt.getCollectedValues(); impl.runLocal(cluster, stmt, (__) -> values.size() >= 2, WAIT_TIMEOUT_MS); Assert.assertEquals(2, values.size()); Assert.assertEquals(4, values.get(0).getFirst()); Assert.assertEquals(5, values.get(1).getFirst()); }
@Override public Pair<Integer, Integer> init() { return Pair.of(0, 0); }
@Test public void testExternalUdf() throws Exception { List<String> stmt = new ArrayList<>(); stmt.add("CREATE EXTERNAL TABLE FOO (ID INT PRIMARY KEY) LOCATION 'mock:///foo'"); stmt.add("CREATE EXTERNAL TABLE BAR (ID INT PRIMARY KEY) LOCATION 'mock:///foo'"); stmt.add("CREATE FUNCTION MYPLUS AS 'org.apache.storm.sql.TestUtils$MyPlus'"); stmt.add("INSERT INTO BAR SELECT STREAM MYPLUS(ID, 1) FROM FOO WHERE ID > 2"); StormSqlLocalClusterImpl impl = new StormSqlLocalClusterImpl(); List<Pair<Object, Values>> values = TestUtils.MockInsertBolt.getCollectedValues(); impl.runLocal(cluster, stmt, (__) -> values.size() >= 2, WAIT_TIMEOUT_MS); Assert.assertEquals(2, values.size()); Assert.assertEquals(4, values.get(0).getFirst()); Assert.assertEquals(5, values.get(1).getFirst()); }
@Override public Pair<Integer, Integer> apply(Pair<Integer, Integer> sumAndCount, Integer value) { return Pair.of(sumAndCount._1 + value, sumAndCount._2 + 1); }
@Override public void execute(Pair<K, ?> input, String sourceStream) { K key = input.getFirst(); if (sourceStream.equals(firstStream)) { V1 val = (V1) input.getSecond(); firstMap.put(key, val); } else if (sourceStream.equals(secondStream)) { V2 val = (V2) input.getSecond(); secondMap.put(key, val); } if (!context.isWindowed()) { forwardValues(); } }
@Override public Pair<Long, Long> getState() { return Pair.of(currentCount.get(), processed); }
private <T1, T2> List<Tuple3<K, T1, T2>> join(Multimap<K, T1> tab, List<Pair<K, T2>> rows, JoinType leftType, JoinType rightType) { List<Tuple3<K, T1, T2>> res = new ArrayList<>(); for (Pair<K, T2> row : rows) { K key = row.getFirst(); Collection<T1> values = tab.removeAll(key); if (values.isEmpty()) { if (rightType == JoinType.OUTER) { res.add(new Tuple3<>(row.getFirst(), null, row.getSecond())); } } else { for (T1 mapValue : values) { res.add(new Tuple3<>(row.getFirst(), mapValue, row.getSecond())); } } } // whatever remains in the tab are non matching left rows. if (leftType == JoinType.OUTER) { for (Map.Entry<K, T1> row : tab.entries()) { res.add(new Tuple3<>(row.getKey(), row.getValue(), null)); } } return res; }
@Override public Pair<Integer, Integer> merge(Pair<Integer, Integer> sumAndCount1, Pair<Integer, Integer> sumAndCount2) { System.out.println("Merge " + sumAndCount1 + " and " + sumAndCount2); return Pair.of( sumAndCount1._1 + sumAndCount2._1, sumAndCount1._2 + sumAndCount2._2 ); }
@Override public <T> void forward(T input) { if (PUNCTUATION.equals(input)) { emit(punctuation, punctuationStreamId); maybeAck(); } else if (processorNode.emitsPair()) { Pair<?, ?> value = (Pair<?, ?>) input; emit(new Values(value.getFirst(), value.getSecond()), outputStreamId); } else { emit(new Values(input), outputStreamId); } }
@Override public void execute(Pair<K, ?> input, String sourceStream) { K key = input.getFirst(); if (sourceStream.equals(leftStream)) { V1 val = (V1) input.getSecond(); Pair<K, V1> pair = Pair.of(key, val); leftRows.add(pair); if (!context.isWindowed()) { joinAndForward(Collections.singletonList(pair), rightRows); } } else if (sourceStream.equals(rightStream)) { V2 val = (V2) input.getSecond(); Pair<K, V2> pair = Pair.of(key, val); rightRows.add(pair); if (!context.isWindowed()) { joinAndForward(leftRows, Collections.singletonList(pair)); } } }
@Override public Pair<Object, Values> apply(Values input) { return Pair.of(input.get(primaryKeyIndex), input); } }