/** * Get the Collection of values currently in the list. * @return the collection of values */ public Collection<T> getValues() { this.evict(); return Collections.unmodifiableList(this.elements.stream().map(x -> x.getValue()).collect(Collectors.toList())); }
@Override public byte[] toBytes(TimestampedValue<V> tv) { byte[] vBytes = vSerde.toBytes(tv.getValue()); int vBytesLength = vBytes != null ? vBytes.length : 0; ByteBuffer bb = ByteBuffer.allocate(vBytesLength + TIMESTAMP_BYTES); if (vBytes != null) { bb.put(vBytes); } bb.putLong(tv.getTimestamp()); return bb.array(); } }
/** * Get the Collection of values currently in the list. * @return the collection of values */ public Collection<T> getValues() { this.evict(); return Collections.unmodifiableList(this.elements.stream().map(x -> x.getValue()).collect(Collectors.toList())); }
@Override public byte[] toBytes(TimestampedValue<V> tv) { byte[] vBytes = vSerde.toBytes(tv.getValue()); int vBytesLength = vBytes != null ? vBytes.length : 0; ByteBuffer bb = ByteBuffer.allocate(vBytesLength + TIMESTAMP_BYTES); if (vBytes != null) { bb.put(vBytes); } bb.putLong(tv.getTimestamp()); return bb.array(); } }
@Override public byte[] toBytes(TimestampedValue<V> tv) { byte[] vBytes = vSerde.toBytes(tv.getValue()); int vBytesLength = vBytes != null ? vBytes.length : 0; ByteBuffer bb = ByteBuffer.allocate(vBytesLength + TIMESTAMP_BYTES); if (vBytes != null) { bb.put(vBytes); } bb.putLong(tv.getTimestamp()); return bb.array(); } }
@Override public byte[] toBytes(TimestampedValue<V> tv) { byte[] vBytes = vSerde.toBytes(tv.getValue()); int vBytesLength = vBytes != null ? vBytes.length : 0; ByteBuffer bb = ByteBuffer.allocate(vBytesLength + TIMESTAMP_BYTES); if (vBytes != null) { bb.put(vBytes); } bb.putLong(tv.getTimestamp()); return bb.array(); } }
@Override public byte[] toBytes(TimestampedValue<V> tv) { byte[] vBytes = vSerde.toBytes(tv.getValue()); int vBytesLength = vBytes != null ? vBytes.length : 0; ByteBuffer bb = ByteBuffer.allocate(vBytesLength + TIMESTAMP_BYTES); if (vBytes != null) { bb.put(vBytes); } bb.putLong(tv.getTimestamp()); return bb.array(); } }
/** * Return a list of values in the store for the provided key and timestamp * * @param key the key to look up in the store * @param timestamp the timestamp to look up in the store * @return the list of values for the provided key */ private List<Object> getValues(K key, long timestamp) { ClosableIterator<TimestampedValue<Object>> iterator = timeSeriesStore.get(key, timestamp); List<TimestampedValue<Object>> timestampedValues = toList(iterator); List<Object> values = timestampedValues.stream().map(element -> element.getValue()).collect(Collectors.toList()); LOG.trace("Returning {} for key {} and timestamp {}", new Object[] {values, key, timestamp}); return values; }
/** * Return a list of values in the store for the provided key and timestamp * * @param key the key to look up in the store * @param timestamp the timestamp to look up in the store * @return the list of values for the provided key */ private List<Object> getValues(K key, long timestamp) { ClosableIterator<TimestampedValue<Object>> iterator = timeSeriesStore.get(key, timestamp); List<TimestampedValue<Object>> timestampedValues = toList(iterator); List<Object> values = timestampedValues.stream().map(element -> element.getValue()).collect(Collectors.toList()); LOG.trace("Returning {} for key {} and timestamp {}", new Object[] {values, key, timestamp}); return values; }
/** * Return a list of values in the store for the provided key and timestamp * * @param key the key to look up in the store * @param timestamp the timestamp to look up in the store * @return the list of values for the provided key */ private List<Object> getValues(K key, long timestamp) { ClosableIterator<TimestampedValue<Object>> iterator = timeSeriesStore.get(key, timestamp); List<TimestampedValue<Object>> timestampedValues = toList(iterator); List<Object> values = timestampedValues.stream().map(element -> element.getValue()).collect(Collectors.toList()); LOG.trace("Returning {} for key {} and timestamp {}", new Object[] {values, key, timestamp}); return values; }
/** * Return a list of values in the store for the provided key and timestamp * * @param key the key to look up in the store * @param timestamp the timestamp to look up in the store * @return the list of values for the provided key */ private List<Object> getValues(K key, long timestamp) { ClosableIterator<TimestampedValue<Object>> iterator = timeSeriesStore.get(key, timestamp); List<TimestampedValue<Object>> timestampedValues = toList(iterator); List<Object> values = timestampedValues.stream().map(element -> element.getValue()).collect(Collectors.toList()); LOG.trace("Returning {} for key {} and timestamp {}", new Object[] {values, key, timestamp}); return values; }
/** * Return a list of values in the store for the provided key and timestamp * * @param key the key to look up in the store * @param timestamp the timestamp to look up in the store * @return the list of values for the provided key */ private List<Object> getValues(K key, long timestamp) { ClosableIterator<TimestampedValue<Object>> iterator = timeSeriesStore.get(key, timestamp); List<TimestampedValue<Object>> timestampedValues = toList(iterator); List<Object> values = timestampedValues.stream().map(element -> element.getValue()).collect(Collectors.toList()); LOG.trace("Returning {} for key {} and timestamp {}", new Object[] {values, key, timestamp}); return values; }
@Test public void testEmptyValueDeserialization() { byte[] bytesWithNoValue = new byte[8]; ByteBuffer.wrap(bytesWithNoValue).putLong(1234L); TimestampedValueSerde<byte[]> timestampedValueSerde = new TimestampedValueSerde<>(new ByteSerde()); TimestampedValue<byte[]> timestampedValue = timestampedValueSerde.fromBytes(bytesWithNoValue); assertEquals(1234L, timestampedValue.getTimestamp()); assertEquals(0, timestampedValue.getValue().length); }
@Override public Collection<JM> handleMessage(M message, MessageCollector collector, TaskCoordinator coordinator) { try { KeyValueStore<K, TimestampedValue<M>> thisState = thisPartialJoinFn.getState(); KeyValueStore<K, TimestampedValue<OM>> otherState = otherPartialJoinFn.getState(); K key = thisPartialJoinFn.getKey(message); thisState.put(key, new TimestampedValue<>(message, clock.currentTimeMillis())); TimestampedValue<OM> otherMessage = otherState.get(key); long now = clock.currentTimeMillis(); if (otherMessage != null && otherMessage.getTimestamp() > now - ttlMs) { JM joinResult = thisPartialJoinFn.apply(message, otherMessage.getValue()); return Collections.singletonList(joinResult); } } catch (Exception e) { throw new SamzaException("Error handling message in PartialJoinOperatorImpl " + getOpImplId(), e); } return Collections.emptyList(); }
@Override public Collection<JM> handleMessage(M message, MessageCollector collector, TaskCoordinator coordinator) { try { KeyValueStore<K, TimestampedValue<M>> thisState = thisPartialJoinFn.getState(); KeyValueStore<K, TimestampedValue<OM>> otherState = otherPartialJoinFn.getState(); K key = thisPartialJoinFn.getKey(message); thisState.put(key, new TimestampedValue<>(message, clock.currentTimeMillis())); TimestampedValue<OM> otherMessage = otherState.get(key); long now = clock.currentTimeMillis(); if (otherMessage != null && otherMessage.getTimestamp() > now - ttlMs) { JM joinResult = thisPartialJoinFn.apply(message, otherMessage.getValue()); return Collections.singletonList(joinResult); } } catch (Exception e) { throw new SamzaException("Error handling message in PartialJoinOperatorImpl " + getOpImplId(), e); } return Collections.emptyList(); }
@Override public Collection<JM> handleMessage(M message, MessageCollector collector, TaskCoordinator coordinator) { try { KeyValueStore<K, TimestampedValue<M>> thisState = thisPartialJoinFn.getState(); KeyValueStore<K, TimestampedValue<OM>> otherState = otherPartialJoinFn.getState(); K key = thisPartialJoinFn.getKey(message); thisState.put(key, new TimestampedValue<>(message, clock.currentTimeMillis())); TimestampedValue<OM> otherMessage = otherState.get(key); long now = clock.currentTimeMillis(); if (otherMessage != null && otherMessage.getTimestamp() > now - ttlMs) { JM joinResult = thisPartialJoinFn.apply(message, otherMessage.getValue()); return Collections.singletonList(joinResult); } } catch (Exception e) { throw new SamzaException("Error handling message in PartialJoinOperatorImpl " + getOpImplId(), e); } return Collections.emptyList(); }
@Override public Collection<JM> handleMessage(M message, MessageCollector collector, TaskCoordinator coordinator) { try { KeyValueStore<K, TimestampedValue<M>> thisState = thisPartialJoinFn.getState(); KeyValueStore<K, TimestampedValue<OM>> otherState = otherPartialJoinFn.getState(); K key = thisPartialJoinFn.getKey(message); thisState.put(key, new TimestampedValue<>(message, clock.currentTimeMillis())); TimestampedValue<OM> otherMessage = otherState.get(key); long now = clock.currentTimeMillis(); if (otherMessage != null && otherMessage.getTimestamp() > now - ttlMs) { JM joinResult = thisPartialJoinFn.apply(message, otherMessage.getValue()); return Collections.singletonList(joinResult); } } catch (Exception e) { throw new SamzaException("Error handling message in PartialJoinOperatorImpl " + getOpImplId(), e); } return Collections.emptyList(); }
@Override public Collection<JM> handleMessage(M message, MessageCollector collector, TaskCoordinator coordinator) { try { KeyValueStore<K, TimestampedValue<M>> thisState = thisPartialJoinFn.getState(); KeyValueStore<K, TimestampedValue<OM>> otherState = otherPartialJoinFn.getState(); K key = thisPartialJoinFn.getKey(message); thisState.put(key, new TimestampedValue<>(message, clock.currentTimeMillis())); TimestampedValue<OM> otherMessage = otherState.get(key); long now = clock.currentTimeMillis(); if (otherMessage != null && otherMessage.getTimestamp() > now - ttlMs) { JM joinResult = thisPartialJoinFn.apply(message, otherMessage.getValue()); return Collections.singletonList(joinResult); } } catch (Exception e) { throw new SamzaException("Error handling message in PartialJoinOperatorImpl " + getOpImplId(), e); } return Collections.emptyList(); }
@Test public void testPutWithMultipleEntries() { TimeSeriesStore<String, byte[]> timeSeriesStore = newTimeSeriesStore(new StringSerde("UTF-8"), true); // insert 100 entries at timestamps "1" and "2" for (int i = 0; i < 100; i++) { timeSeriesStore.put("hello", "world-1".getBytes(), 1L); timeSeriesStore.put("hello", "world-2".getBytes(), 2L); } // read from time-range [0,2) should return 100 entries List<TimestampedValue<byte[]>> values = readStore(timeSeriesStore, "hello", 0L, 2L); Assert.assertEquals(100, values.size()); values.forEach(timeSeriesValue -> { Assert.assertEquals("world-1", new String(timeSeriesValue.getValue())); }); // read from time-range [2,4) should return 100 entries values = readStore(timeSeriesStore, "hello", 2L, 4L); Assert.assertEquals(100, values.size()); values.forEach(timeSeriesValue -> { Assert.assertEquals("world-2", new String(timeSeriesValue.getValue())); }); // read all entries in the store values = readStore(timeSeriesStore, "hello", 0L, Integer.MAX_VALUE); Assert.assertEquals(200, values.size()); }
@Test public void testGetOnTimestampBoundaries() { TimeSeriesStore<String, byte[]> timeSeriesStore = newTimeSeriesStore(new StringSerde("UTF-8"), true); // insert an entry with key "hello" at timestamps "1" and "2" timeSeriesStore.put("hello", "world-1".getBytes(), 1L); timeSeriesStore.put("hello", "world-1".getBytes(), 2L); timeSeriesStore.put("hello", "world-2".getBytes(), 2L); // read from time-range List<TimestampedValue<byte[]>> values = readStore(timeSeriesStore, "hello", 0L, 1L); Assert.assertEquals(0, values.size()); // read from time-range [1,2) should return one entry values = readStore(timeSeriesStore, "hello", 1L, 2L); Assert.assertEquals(1, values.size()); Assert.assertEquals("world-1", new String(values.get(0).getValue())); // read from time-range [2,3) should return two entries values = readStore(timeSeriesStore, "hello", 2L, 3L); Assert.assertEquals(2, values.size()); Assert.assertEquals("world-1", new String(values.get(0).getValue())); Assert.assertEquals(2L, values.get(0).getTimestamp()); // read from time-range [0,3) should return three entries values = readStore(timeSeriesStore, "hello", 0L, 3L); Assert.assertEquals(3, values.size()); // read from time-range [2,999999) should return two entries values = readStore(timeSeriesStore, "hello", 2L, 999999L); Assert.assertEquals(2, values.size()); // read from time-range [3,4) should return no entries values = readStore(timeSeriesStore, "hello", 3L, 4L); Assert.assertEquals(0, values.size()); }