/** * Removes entries from elements to ensure no element has a timestamp more than maxStaleness before current timestamp. */ private void evictBasedOnTimestamp() { Instant currentTimestamp = Instant.now(); TimestampedValue<T> valueInfo = this.elements.peek(); // continue remove-head if currenttimestamp - head-element's timestamp > durationThreshold while (valueInfo != null && currentTimestamp.toEpochMilli() - valueInfo.getTimestamp() > this.maxStaleness.toMillis()) { this.elements.poll(); valueInfo = this.elements.peek(); } } }
/** * Removes entries from elements to ensure no element has a timestamp more than maxStaleness before current timestamp. */ private void evictBasedOnTimestamp() { Instant currentTimestamp = Instant.now(); TimestampedValue<T> valueInfo = this.elements.peek(); // continue remove-head if currenttimestamp - head-element's timestamp > durationThreshold while (valueInfo != null && currentTimestamp.toEpochMilli() - valueInfo.getTimestamp() > this.maxStaleness.toMillis()) { this.elements.poll(); valueInfo = this.elements.peek(); } } }
@Override public byte[] toBytes(TimestampedValue<V> tv) { byte[] vBytes = vSerde.toBytes(tv.getValue()); int vBytesLength = vBytes != null ? vBytes.length : 0; ByteBuffer bb = ByteBuffer.allocate(vBytesLength + TIMESTAMP_BYTES); if (vBytes != null) { bb.put(vBytes); } bb.putLong(tv.getTimestamp()); return bb.array(); } }
@Override public byte[] toBytes(TimestampedValue<V> tv) { byte[] vBytes = vSerde.toBytes(tv.getValue()); int vBytesLength = vBytes != null ? vBytes.length : 0; ByteBuffer bb = ByteBuffer.allocate(vBytesLength + TIMESTAMP_BYTES); if (vBytes != null) { bb.put(vBytes); } bb.putLong(tv.getTimestamp()); return bb.array(); } }
@Override public byte[] toBytes(TimestampedValue<V> tv) { byte[] vBytes = vSerde.toBytes(tv.getValue()); int vBytesLength = vBytes != null ? vBytes.length : 0; ByteBuffer bb = ByteBuffer.allocate(vBytesLength + TIMESTAMP_BYTES); if (vBytes != null) { bb.put(vBytes); } bb.putLong(tv.getTimestamp()); return bb.array(); } }
@Override public byte[] toBytes(TimestampedValue<V> tv) { byte[] vBytes = vSerde.toBytes(tv.getValue()); int vBytesLength = vBytes != null ? vBytes.length : 0; ByteBuffer bb = ByteBuffer.allocate(vBytesLength + TIMESTAMP_BYTES); if (vBytes != null) { bb.put(vBytes); } bb.putLong(tv.getTimestamp()); return bb.array(); } }
@Override public byte[] toBytes(TimestampedValue<V> tv) { byte[] vBytes = vSerde.toBytes(tv.getValue()); int vBytesLength = vBytes != null ? vBytes.length : 0; ByteBuffer bb = ByteBuffer.allocate(vBytesLength + TIMESTAMP_BYTES); if (vBytes != null) { bb.put(vBytes); } bb.putLong(tv.getTimestamp()); return bb.array(); } }
long timestamp = (timestampedValues.isEmpty())? clock.currentTimeMillis() : timestampedValues.get(0).getTimestamp();
long timestamp = (timestampedValues.isEmpty())? clock.currentTimeMillis() : timestampedValues.get(0).getTimestamp();
long timestamp = (timestampedValues.isEmpty())? clock.currentTimeMillis() : timestampedValues.get(0).getTimestamp();
@Test public void testEmptyValueDeserialization() { byte[] bytesWithNoValue = new byte[8]; ByteBuffer.wrap(bytesWithNoValue).putLong(1234L); TimestampedValueSerde<byte[]> timestampedValueSerde = new TimestampedValueSerde<>(new ByteSerde()); TimestampedValue<byte[]> timestampedValue = timestampedValueSerde.fromBytes(bytesWithNoValue); assertEquals(1234L, timestampedValue.getTimestamp()); assertEquals(0, timestampedValue.getValue().length); }
long timestamp = (timestampedValues.isEmpty())? clock.currentTimeMillis() : timestampedValues.get(0).getTimestamp();
long timestamp = (timestampedValues.isEmpty())? clock.currentTimeMillis() : timestampedValues.get(0).getTimestamp();
@Override public Collection<JM> handleMessage(M message, MessageCollector collector, TaskCoordinator coordinator) { try { KeyValueStore<K, TimestampedValue<M>> thisState = thisPartialJoinFn.getState(); KeyValueStore<K, TimestampedValue<OM>> otherState = otherPartialJoinFn.getState(); K key = thisPartialJoinFn.getKey(message); thisState.put(key, new TimestampedValue<>(message, clock.currentTimeMillis())); TimestampedValue<OM> otherMessage = otherState.get(key); long now = clock.currentTimeMillis(); if (otherMessage != null && otherMessage.getTimestamp() > now - ttlMs) { JM joinResult = thisPartialJoinFn.apply(message, otherMessage.getValue()); return Collections.singletonList(joinResult); } } catch (Exception e) { throw new SamzaException("Error handling message in PartialJoinOperatorImpl " + getOpImplId(), e); } return Collections.emptyList(); }
@Override public Collection<JM> handleMessage(M message, MessageCollector collector, TaskCoordinator coordinator) { try { KeyValueStore<K, TimestampedValue<M>> thisState = thisPartialJoinFn.getState(); KeyValueStore<K, TimestampedValue<OM>> otherState = otherPartialJoinFn.getState(); K key = thisPartialJoinFn.getKey(message); thisState.put(key, new TimestampedValue<>(message, clock.currentTimeMillis())); TimestampedValue<OM> otherMessage = otherState.get(key); long now = clock.currentTimeMillis(); if (otherMessage != null && otherMessage.getTimestamp() > now - ttlMs) { JM joinResult = thisPartialJoinFn.apply(message, otherMessage.getValue()); return Collections.singletonList(joinResult); } } catch (Exception e) { throw new SamzaException("Error handling message in PartialJoinOperatorImpl " + getOpImplId(), e); } return Collections.emptyList(); }
@Override public Collection<JM> handleMessage(M message, MessageCollector collector, TaskCoordinator coordinator) { try { KeyValueStore<K, TimestampedValue<M>> thisState = thisPartialJoinFn.getState(); KeyValueStore<K, TimestampedValue<OM>> otherState = otherPartialJoinFn.getState(); K key = thisPartialJoinFn.getKey(message); thisState.put(key, new TimestampedValue<>(message, clock.currentTimeMillis())); TimestampedValue<OM> otherMessage = otherState.get(key); long now = clock.currentTimeMillis(); if (otherMessage != null && otherMessage.getTimestamp() > now - ttlMs) { JM joinResult = thisPartialJoinFn.apply(message, otherMessage.getValue()); return Collections.singletonList(joinResult); } } catch (Exception e) { throw new SamzaException("Error handling message in PartialJoinOperatorImpl " + getOpImplId(), e); } return Collections.emptyList(); }
@Override public Collection<JM> handleMessage(M message, MessageCollector collector, TaskCoordinator coordinator) { try { KeyValueStore<K, TimestampedValue<M>> thisState = thisPartialJoinFn.getState(); KeyValueStore<K, TimestampedValue<OM>> otherState = otherPartialJoinFn.getState(); K key = thisPartialJoinFn.getKey(message); thisState.put(key, new TimestampedValue<>(message, clock.currentTimeMillis())); TimestampedValue<OM> otherMessage = otherState.get(key); long now = clock.currentTimeMillis(); if (otherMessage != null && otherMessage.getTimestamp() > now - ttlMs) { JM joinResult = thisPartialJoinFn.apply(message, otherMessage.getValue()); return Collections.singletonList(joinResult); } } catch (Exception e) { throw new SamzaException("Error handling message in PartialJoinOperatorImpl " + getOpImplId(), e); } return Collections.emptyList(); }
@Override public Collection<JM> handleMessage(M message, MessageCollector collector, TaskCoordinator coordinator) { try { KeyValueStore<K, TimestampedValue<M>> thisState = thisPartialJoinFn.getState(); KeyValueStore<K, TimestampedValue<OM>> otherState = otherPartialJoinFn.getState(); K key = thisPartialJoinFn.getKey(message); thisState.put(key, new TimestampedValue<>(message, clock.currentTimeMillis())); TimestampedValue<OM> otherMessage = otherState.get(key); long now = clock.currentTimeMillis(); if (otherMessage != null && otherMessage.getTimestamp() > now - ttlMs) { JM joinResult = thisPartialJoinFn.apply(message, otherMessage.getValue()); return Collections.singletonList(joinResult); } } catch (Exception e) { throw new SamzaException("Error handling message in PartialJoinOperatorImpl " + getOpImplId(), e); } return Collections.emptyList(); }
@Test public void testGetOnTimestampBoundaries() { TimeSeriesStore<String, byte[]> timeSeriesStore = newTimeSeriesStore(new StringSerde("UTF-8"), true); // insert an entry with key "hello" at timestamps "1" and "2" timeSeriesStore.put("hello", "world-1".getBytes(), 1L); timeSeriesStore.put("hello", "world-1".getBytes(), 2L); timeSeriesStore.put("hello", "world-2".getBytes(), 2L); // read from time-range List<TimestampedValue<byte[]>> values = readStore(timeSeriesStore, "hello", 0L, 1L); Assert.assertEquals(0, values.size()); // read from time-range [1,2) should return one entry values = readStore(timeSeriesStore, "hello", 1L, 2L); Assert.assertEquals(1, values.size()); Assert.assertEquals("world-1", new String(values.get(0).getValue())); // read from time-range [2,3) should return two entries values = readStore(timeSeriesStore, "hello", 2L, 3L); Assert.assertEquals(2, values.size()); Assert.assertEquals("world-1", new String(values.get(0).getValue())); Assert.assertEquals(2L, values.get(0).getTimestamp()); // read from time-range [0,3) should return three entries values = readStore(timeSeriesStore, "hello", 0L, 3L); Assert.assertEquals(3, values.size()); // read from time-range [2,999999) should return two entries values = readStore(timeSeriesStore, "hello", 2L, 999999L); Assert.assertEquals(2, values.size()); // read from time-range [3,4) should return no entries values = readStore(timeSeriesStore, "hello", 3L, 4L); Assert.assertEquals(0, values.size()); }
@Test public void testGetOnTimestampBoundariesWithOverwriteMode() { // instantiate a store in overwrite mode TimeSeriesStore<String, byte[]> timeSeriesStore = newTimeSeriesStore(new StringSerde("UTF-8"), false); // insert an entry with key "hello" at timestamps "1" and "2" timeSeriesStore.put("hello", "world-1".getBytes(), 1L); timeSeriesStore.put("hello", "world-1".getBytes(), 2L); timeSeriesStore.put("hello", "world-2".getBytes(), 2L); // read from time-range List<TimestampedValue<byte[]>> values = readStore(timeSeriesStore, "hello", 0L, 1L); Assert.assertEquals(0, values.size()); // read from time-range [1,2) should return one entry values = readStore(timeSeriesStore, "hello", 1L, 2L); Assert.assertEquals(1, values.size()); Assert.assertEquals("world-1", new String(values.get(0).getValue())); // read from time-range [2,3) should return the most recent entry values = readStore(timeSeriesStore, "hello", 2L, 3L); Assert.assertEquals(1, values.size()); Assert.assertEquals("world-2", new String(values.get(0).getValue())); Assert.assertEquals(2L, values.get(0).getTimestamp()); // read from time-range [0,3) should return two entries values = readStore(timeSeriesStore, "hello", 0L, 3L); Assert.assertEquals(2, values.size()); // read from time-range [2,999999) should return one entry values = readStore(timeSeriesStore, "hello", 2L, 999999L); Assert.assertEquals(1, values.size()); // read from time-range [3,4) should return no entries values = readStore(timeSeriesStore, "hello", 3L, 4L); Assert.assertEquals(0, values.size()); }