/** * Gets the values with which the specified {@code keys} are associated. * * @param keys the keys with which the associated values are to be fetched. * @return a map of the keys that were found and their respective values. * @throws NullPointerException if the specified {@code keys} list, or any of the keys, is {@code null}. */ default Map<K, V> getAll(List<K> keys) { Map<K, V> map = new HashMap<>(keys.size()); for (K key : keys) { V value = get(key); if (value != null) { map.put(key, value); } } return map; }
@Override public void deleteAll(List<K> keys) { instrument(metrics.numDeleteAlls, metrics.deleteAllNs, () -> kvStore.deleteAll(keys)); }
@Override public void put(K key, V val, long timestamp) { // For append mode, values are differentiated by an unique sequence number. For overwrite mode, the sequence // number is always zero. This ensures that only the most recent value is retained. if (appendMode) { seqNum.getAndIncrement(); } TimeSeriesKey<K> timeSeriesKey = new TimeSeriesKey<>(key, timestamp, seqNum.get()); LOG.trace("Inserting {} -> {} into the store", timeSeriesKey, val); kvStore.put(timeSeriesKey, val); }
@Override public Collection<JM> handleMessage(M message, MessageCollector collector, TaskCoordinator coordinator) { try { KeyValueStore<K, TimestampedValue<M>> thisState = thisPartialJoinFn.getState(); KeyValueStore<K, TimestampedValue<OM>> otherState = otherPartialJoinFn.getState(); K key = thisPartialJoinFn.getKey(message); thisState.put(key, new TimestampedValue<>(message, clock.currentTimeMillis())); TimestampedValue<OM> otherMessage = otherState.get(key); long now = clock.currentTimeMillis(); if (otherMessage != null && otherMessage.getTimestamp() > now - ttlMs) { JM joinResult = thisPartialJoinFn.apply(message, otherMessage.getValue()); return Collections.singletonList(joinResult); } } catch (Exception e) { throw new SamzaException("Error handling message in PartialJoinOperatorImpl " + getOpImplId(), e); } return Collections.emptyList(); }
@Override public void remove(K key, long startTimestamp, long endTimeStamp) { validateRange(startTimestamp, endTimeStamp); TimeSeriesKey<K> fromKey = new TimeSeriesKey(key, startTimestamp, 0); TimeSeriesKey<K> toKey = new TimeSeriesKey(key, endTimeStamp, 0); List<TimeSeriesKey<K>> keysToDelete = new LinkedList<>(); KeyValueIterator<TimeSeriesKey<K>, V> range = kvStore.range(fromKey, toKey); try { while (range.hasNext()) { keysToDelete.add(range.next().getKey()); } } finally { range.close(); } kvStore.deleteAll(keysToDelete); }
@Override public void putAll(List<Entry<K, V>> entries) { writeMetrics.numPutAlls.inc(); long startNs = System.nanoTime(); kvStore.putAll(entries); writeMetrics.putAllNs.update(System.nanoTime() - startNs); }
@Override public void flush() { kvStore.flush(); }
@Before public void setUp() { keys = Arrays.asList("k1", "k2", "k3"); values = new HashMap<>(); values.put("k1", "v1"); values.put("k2", "v2"); values.put("k3", null); kvStore = mock(KeyValueStore.class); when(kvStore.get("k1")).thenReturn("v1"); when(kvStore.get("k2")).thenReturn("v2"); when(kvStore.getAll(keys)).thenReturn(values); getNs = new Timer(""); getAllNs = new Timer(""); numGets = new Counter(""); numGetAlls = new Counter(""); getCallbackNs = new Timer(""); numMissedLookups = new Counter(""); metricsRegistry = mock(MetricsRegistry.class); String groupName = LocalTable.class.getSimpleName(); when(metricsRegistry.newCounter(groupName, TABLE_ID + "-num-gets")).thenReturn(numGets); when(metricsRegistry.newCounter(groupName, TABLE_ID + "-num-getAlls")).thenReturn(numGetAlls); when(metricsRegistry.newCounter(groupName, TABLE_ID + "-num-missed-lookups")).thenReturn(numMissedLookups); when(metricsRegistry.newTimer(groupName, TABLE_ID + "-get-ns")).thenReturn(getNs); when(metricsRegistry.newTimer(groupName, TABLE_ID + "-getAll-ns")).thenReturn(getAllNs); when(metricsRegistry.newTimer(groupName, TABLE_ID + "-get-callback-ns")).thenReturn(getCallbackNs); }
@Test public void testPutAll() throws Exception { ReadWriteTable table = createTable(false); List<Entry> entries = Arrays.asList(new Entry("k1", "v1"), new Entry("k2", null)); table.putAll(entries); table.putAllAsync(entries).get(); verify(kvStore, times(2)).putAll(any()); verify(kvStore, times(2)).deleteAll(any()); Assert.assertEquals(2, numPutAlls.getCount()); Assert.assertEquals(2, numDeleteAlls.getCount()); Assert.assertTrue(putAllNs.getSnapshot().getAverage() > 0); Assert.assertTrue(deleteAllNs.getSnapshot().getAverage() > 0); Assert.assertEquals(0, putNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, deleteNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, flushNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, numPuts.getCount()); Assert.assertEquals(0, numDeletes.getCount()); Assert.assertEquals(0, numFlushes.getCount()); Assert.assertEquals(0, putCallbackNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, deleteCallbackNs.getSnapshot().getAverage(), 0.001); }
@Test public void testPut() throws Exception { ReadWriteTable table = createTable(false); table.put("k1", "v1"); table.putAsync("k2", "v2").get(); table.putAsync("k3", null).get(); verify(kvStore, times(2)).put(any(), any()); verify(kvStore, times(1)).delete(any()); Assert.assertEquals(2, numPuts.getCount()); Assert.assertEquals(1, numDeletes.getCount()); Assert.assertTrue(putNs.getSnapshot().getAverage() > 0); Assert.assertTrue(deleteNs.getSnapshot().getAverage() > 0); Assert.assertEquals(0, putAllNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, deleteAllNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, flushNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, numPutAlls.getCount()); Assert.assertEquals(0, numDeleteAlls.getCount()); Assert.assertEquals(0, numFlushes.getCount()); Assert.assertEquals(0, putCallbackNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, deleteCallbackNs.getSnapshot().getAverage(), 0.001); }
/** * Deletes the mappings for the specified {@code keys} from this key-value store (if such mappings exist). * * @param keys the keys for which the mappings are to be deleted. * @throws NullPointerException if the specified {@code keys} list, or any of the keys, is {@code null}. */ default void deleteAll(List<K> keys) { for (K key : keys) { delete(key); } }
@Override public ClosableIterator<TimestampedValue<V>> get(K key, long startTimestamp, long endTimestamp) { validateRange(startTimestamp, endTimestamp); TimeSeriesKey<K> fromKey = new TimeSeriesKey(key, startTimestamp, 0); TimeSeriesKey<K> toKey = new TimeSeriesKey(key, endTimestamp, 0); KeyValueIterator<TimeSeriesKey<K>, V> range = kvStore.range(fromKey, toKey); LOG.trace("Getting entries in the store for {} from {} to {}", new Object[] {key, startTimestamp, endTimestamp}); return new TimeSeriesStoreIterator<>(range); }
@Override public Map<K, V> getAll(List<K> keys) { Map<K, V> result = instrument(metrics.numGetAlls, metrics.getAllNs, () -> kvStore.getAll(keys)); result.values().stream().filter(Objects::isNull).forEach(v -> incCounter(metrics.numMissedLookups)); return result; }
@Override public Collection<JM> handleMessage(M message, MessageCollector collector, TaskCoordinator coordinator) { try { KeyValueStore<K, TimestampedValue<M>> thisState = thisPartialJoinFn.getState(); KeyValueStore<K, TimestampedValue<OM>> otherState = otherPartialJoinFn.getState(); K key = thisPartialJoinFn.getKey(message); thisState.put(key, new TimestampedValue<>(message, clock.currentTimeMillis())); TimestampedValue<OM> otherMessage = otherState.get(key); long now = clock.currentTimeMillis(); if (otherMessage != null && otherMessage.getTimestamp() > now - ttlMs) { JM joinResult = thisPartialJoinFn.apply(message, otherMessage.getValue()); return Collections.singletonList(joinResult); } } catch (Exception e) { throw new SamzaException("Error handling message in PartialJoinOperatorImpl " + getOpImplId(), e); } return Collections.emptyList(); }
@Override public void remove(K key, long startTimestamp, long endTimeStamp) { validateRange(startTimestamp, endTimeStamp); TimeSeriesKey<K> fromKey = new TimeSeriesKey(key, startTimestamp, 0); TimeSeriesKey<K> toKey = new TimeSeriesKey(key, endTimeStamp, 0); List<TimeSeriesKey<K>> keysToDelete = new LinkedList<>(); KeyValueIterator<TimeSeriesKey<K>, V> range = kvStore.range(fromKey, toKey); try { while (range.hasNext()) { keysToDelete.add(range.next().getKey()); } } finally { range.close(); } kvStore.deleteAll(keysToDelete); }
@Override public void putAll(List<Entry<K, V>> entries) { writeMetrics.numPutAlls.inc(); long startNs = System.nanoTime(); kvStore.putAll(entries); writeMetrics.putAllNs.update(System.nanoTime() - startNs); }
@Override public void flush() { kvStore.flush(); }
/** * Deletes the mappings for the specified {@code keys} from this key-value store (if such mappings exist). * * @param keys the keys for which the mappings are to be deleted. * @throws NullPointerException if the specified {@code keys} list, or any of the keys, is {@code null}. */ default void deleteAll(List<K> keys) { for (K key : keys) { delete(key); } }
@Override public ClosableIterator<TimestampedValue<V>> get(K key, long startTimestamp, long endTimestamp) { validateRange(startTimestamp, endTimestamp); TimeSeriesKey<K> fromKey = new TimeSeriesKey(key, startTimestamp, 0); TimeSeriesKey<K> toKey = new TimeSeriesKey(key, endTimestamp, 0); KeyValueIterator<TimeSeriesKey<K>, V> range = kvStore.range(fromKey, toKey); LOG.trace("Getting entries in the store for {} from {} to {}", new Object[] {key, startTimestamp, endTimestamp}); return new TimeSeriesStoreIterator<>(range); }
@Override public Map<K, V> getAll(List<K> keys) { readMetrics.numGetAlls.inc(); long startNs = System.nanoTime(); Map<K, V> result = kvStore.getAll(keys); readMetrics.getAllNs.update(System.nanoTime() - startNs); return result; }