@Override public void putAll(List<Entry<K, V>> entries) { writeMetrics.numPutAlls.inc(); long startNs = System.nanoTime(); kvStore.putAll(entries); writeMetrics.putAllNs.update(System.nanoTime() - startNs); }
@Override public void putAll(List<Entry<K, V>> entries) { writeMetrics.numPutAlls.inc(); long startNs = System.nanoTime(); kvStore.putAll(entries); writeMetrics.putAllNs.update(System.nanoTime() - startNs); }
/** * Processes the incoming side input message envelope and updates the last processed offset for its SSP. * * @param message incoming message to be processed */ public void process(IncomingMessageEnvelope message) { SystemStreamPartition ssp = message.getSystemStreamPartition(); Set<String> storeNames = sspsToStores.get(ssp); for (String storeName : storeNames) { SideInputsProcessor sideInputsProcessor = storeToProcessor.get(storeName); KeyValueStore keyValueStore = (KeyValueStore) stores.get(storeName); Collection<Entry<?, ?>> entriesToBeWritten = sideInputsProcessor.process(message, keyValueStore); keyValueStore.putAll(ImmutableList.copyOf(entriesToBeWritten)); } // update the last processed offset lastProcessedOffsets.put(ssp, message.getOffset()); }
/** * Processes the incoming side input message envelope and updates the last processed offset for its SSP. * * @param message incoming message to be processed */ public void process(IncomingMessageEnvelope message) { SystemStreamPartition ssp = message.getSystemStreamPartition(); Set<String> storeNames = sspsToStores.get(ssp); for (String storeName : storeNames) { SideInputsProcessor sideInputsProcessor = storeToProcessor.get(storeName); KeyValueStore keyValueStore = (KeyValueStore) stores.get(storeName); Collection<Entry<?, ?>> entriesToBeWritten = sideInputsProcessor.process(message, keyValueStore); keyValueStore.putAll(ImmutableList.copyOf(entriesToBeWritten)); } // update the last processed offset lastProcessedOffsets.put(ssp, message.getOffset()); }
/** * Processes the incoming side input message envelope and updates the last processed offset for its SSP. * * @param message incoming message to be processed */ public void process(IncomingMessageEnvelope message) { SystemStreamPartition ssp = message.getSystemStreamPartition(); Set<String> storeNames = sspsToStores.get(ssp); for (String storeName : storeNames) { SideInputsProcessor sideInputsProcessor = storeToProcessor.get(storeName); KeyValueStore keyValueStore = (KeyValueStore) stores.get(storeName); Collection<Entry<?, ?>> entriesToBeWritten = sideInputsProcessor.process(message, keyValueStore); keyValueStore.putAll(ImmutableList.copyOf(entriesToBeWritten)); } // update the last processed offset lastProcessedOffsets.put(ssp, message.getOffset()); }
/** * Processes the incoming side input message envelope and updates the last processed offset for its SSP. * * @param message incoming message to be processed */ public void process(IncomingMessageEnvelope message) { SystemStreamPartition ssp = message.getSystemStreamPartition(); Set<String> storeNames = sspsToStores.get(ssp); for (String storeName : storeNames) { SideInputsProcessor sideInputsProcessor = storeToProcessor.get(storeName); KeyValueStore keyValueStore = (KeyValueStore) stores.get(storeName); Collection<Entry<?, ?>> entriesToBeWritten = sideInputsProcessor.process(message, keyValueStore); keyValueStore.putAll(ImmutableList.copyOf(entriesToBeWritten)); } // update the last processed offset lastProcessedOffsets.put(ssp, message.getOffset()); }
/** * Processes the incoming side input message envelope and updates the last processed offset for its SSP. * * @param message incoming message to be processed */ public void process(IncomingMessageEnvelope message) { SystemStreamPartition ssp = message.getSystemStreamPartition(); Set<String> storeNames = sspsToStores.get(ssp); for (String storeName : storeNames) { SideInputsProcessor sideInputsProcessor = storeToProcessor.get(storeName); KeyValueStore keyValueStore = (KeyValueStore) stores.get(storeName); Collection<Entry<?, ?>> entriesToBeWritten = sideInputsProcessor.process(message, keyValueStore); keyValueStore.putAll(ImmutableList.copyOf(entriesToBeWritten)); } // update the last processed offset lastProcessedOffsets.put(ssp, message.getOffset()); }
@Override public void putAll(List<Entry<K, V>> entries) { List<Entry<K, V>> toPut = new LinkedList<>(); List<K> toDelete = new LinkedList<>(); entries.forEach(e -> { if (e.getValue() != null) { toPut.add(e); } else { toDelete.add(e.getKey()); } }); if (!toPut.isEmpty()) { instrument(metrics.numPutAlls, metrics.putAllNs, () -> kvStore.putAll(toPut)); } if (!toDelete.isEmpty()) { deleteAll(toDelete); } }
@Test public void testPutAll() throws Exception { ReadWriteTable table = createTable(false); List<Entry> entries = Arrays.asList(new Entry("k1", "v1"), new Entry("k2", null)); table.putAll(entries); table.putAllAsync(entries).get(); verify(kvStore, times(2)).putAll(any()); verify(kvStore, times(2)).deleteAll(any()); Assert.assertEquals(2, numPutAlls.getCount()); Assert.assertEquals(2, numDeleteAlls.getCount()); Assert.assertTrue(putAllNs.getSnapshot().getAverage() > 0); Assert.assertTrue(deleteAllNs.getSnapshot().getAverage() > 0); Assert.assertEquals(0, putNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, deleteNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, flushNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, numPuts.getCount()); Assert.assertEquals(0, numDeletes.getCount()); Assert.assertEquals(0, numFlushes.getCount()); Assert.assertEquals(0, putCallbackNs.getSnapshot().getAverage(), 0.001); Assert.assertEquals(0, deleteCallbackNs.getSnapshot().getAverage(), 0.001); }