/** * Add a value to the list. * (Timestamp assigned to this value is the current timestamp.) * @param value The Gauge value to be added */ public void add(T value) { this.elements.add(new TimestampedValue<T>(value, Instant.now().toEpochMilli())); // perform any evictions that may be needed. this.evict(); }
@Override public byte[] toBytes(TimestampedValue<V> tv) { byte[] vBytes = vSerde.toBytes(tv.getValue()); int vBytesLength = vBytes != null ? vBytes.length : 0; ByteBuffer bb = ByteBuffer.allocate(vBytesLength + TIMESTAMP_BYTES); if (vBytes != null) { bb.put(vBytes); } bb.putLong(tv.getTimestamp()); return bb.array(); } }
@Override public Collection<JM> handleMessage(M message, MessageCollector collector, TaskCoordinator coordinator) { try { KeyValueStore<K, TimestampedValue<M>> thisState = thisPartialJoinFn.getState(); KeyValueStore<K, TimestampedValue<OM>> otherState = otherPartialJoinFn.getState(); K key = thisPartialJoinFn.getKey(message); thisState.put(key, new TimestampedValue<>(message, clock.currentTimeMillis())); TimestampedValue<OM> otherMessage = otherState.get(key); long now = clock.currentTimeMillis(); if (otherMessage != null && otherMessage.getTimestamp() > now - ttlMs) { JM joinResult = thisPartialJoinFn.apply(message, otherMessage.getValue()); return Collections.singletonList(joinResult); } } catch (Exception e) { throw new SamzaException("Error handling message in PartialJoinOperatorImpl " + getOpImplId(), e); } return Collections.emptyList(); }
/** * Get the Collection of values currently in the list. * @return the collection of values */ public Collection<T> getValues() { this.evict(); return Collections.unmodifiableList(this.elements.stream().map(x -> x.getValue()).collect(Collectors.toList())); }
/** * Removes entries from elements to ensure no element has a timestamp more than maxStaleness before current timestamp. */ private void evictBasedOnTimestamp() { Instant currentTimestamp = Instant.now(); TimestampedValue<T> valueInfo = this.elements.peek(); // continue remove-head if currenttimestamp - head-element's timestamp > durationThreshold while (valueInfo != null && currentTimestamp.toEpochMilli() - valueInfo.getTimestamp() > this.maxStaleness.toMillis()) { this.elements.poll(); valueInfo = this.elements.peek(); } } }
@Override public Collection<JM> handleMessage(M message, MessageCollector collector, TaskCoordinator coordinator) { try { KeyValueStore<K, TimestampedValue<M>> thisState = thisPartialJoinFn.getState(); KeyValueStore<K, TimestampedValue<OM>> otherState = otherPartialJoinFn.getState(); K key = thisPartialJoinFn.getKey(message); thisState.put(key, new TimestampedValue<>(message, clock.currentTimeMillis())); TimestampedValue<OM> otherMessage = otherState.get(key); long now = clock.currentTimeMillis(); if (otherMessage != null && otherMessage.getTimestamp() > now - ttlMs) { JM joinResult = thisPartialJoinFn.apply(message, otherMessage.getValue()); return Collections.singletonList(joinResult); } } catch (Exception e) { throw new SamzaException("Error handling message in PartialJoinOperatorImpl " + getOpImplId(), e); } return Collections.emptyList(); }
/** * Get the Collection of values currently in the list. * @return the collection of values */ public Collection<T> getValues() { this.evict(); return Collections.unmodifiableList(this.elements.stream().map(x -> x.getValue()).collect(Collectors.toList())); }
/** * Removes entries from elements to ensure no element has a timestamp more than maxStaleness before current timestamp. */ private void evictBasedOnTimestamp() { Instant currentTimestamp = Instant.now(); TimestampedValue<T> valueInfo = this.elements.peek(); // continue remove-head if currenttimestamp - head-element's timestamp > durationThreshold while (valueInfo != null && currentTimestamp.toEpochMilli() - valueInfo.getTimestamp() > this.maxStaleness.toMillis()) { this.elements.poll(); valueInfo = this.elements.peek(); } } }
@Override public Collection<JM> handleMessage(M message, MessageCollector collector, TaskCoordinator coordinator) { try { KeyValueStore<K, TimestampedValue<M>> thisState = thisPartialJoinFn.getState(); KeyValueStore<K, TimestampedValue<OM>> otherState = otherPartialJoinFn.getState(); K key = thisPartialJoinFn.getKey(message); thisState.put(key, new TimestampedValue<>(message, clock.currentTimeMillis())); TimestampedValue<OM> otherMessage = otherState.get(key); long now = clock.currentTimeMillis(); if (otherMessage != null && otherMessage.getTimestamp() > now - ttlMs) { JM joinResult = thisPartialJoinFn.apply(message, otherMessage.getValue()); return Collections.singletonList(joinResult); } } catch (Exception e) { throw new SamzaException("Error handling message in PartialJoinOperatorImpl " + getOpImplId(), e); } return Collections.emptyList(); }
@Override public byte[] toBytes(TimestampedValue<V> tv) { byte[] vBytes = vSerde.toBytes(tv.getValue()); int vBytesLength = vBytes != null ? vBytes.length : 0; ByteBuffer bb = ByteBuffer.allocate(vBytesLength + TIMESTAMP_BYTES); if (vBytes != null) { bb.put(vBytes); } bb.putLong(tv.getTimestamp()); return bb.array(); } }
@Override public TimestampedValue<V> fromBytes(byte[] bytes) { ByteBuffer bb = ByteBuffer.wrap(bytes); byte[] vBytes = new byte[bytes.length - TIMESTAMP_BYTES]; bb.get(vBytes, 0, vBytes.length); V v = vSerde.fromBytes(vBytes); long ts = bb.getLong(); return new TimestampedValue<>(v, ts); }
/** * Return a list of values in the store for the provided key and timestamp * * @param key the key to look up in the store * @param timestamp the timestamp to look up in the store * @return the list of values for the provided key */ private List<Object> getValues(K key, long timestamp) { ClosableIterator<TimestampedValue<Object>> iterator = timeSeriesStore.get(key, timestamp); List<TimestampedValue<Object>> timestampedValues = toList(iterator); List<Object> values = timestampedValues.stream().map(element -> element.getValue()).collect(Collectors.toList()); LOG.trace("Returning {} for key {} and timestamp {}", new Object[] {values, key, timestamp}); return values; }
long timestamp = (timestampedValues.isEmpty())? clock.currentTimeMillis() : timestampedValues.get(0).getTimestamp();
@Override public Collection<JM> handleMessage(M message, MessageCollector collector, TaskCoordinator coordinator) { try { KeyValueStore<K, TimestampedValue<M>> thisState = thisPartialJoinFn.getState(); KeyValueStore<K, TimestampedValue<OM>> otherState = otherPartialJoinFn.getState(); K key = thisPartialJoinFn.getKey(message); thisState.put(key, new TimestampedValue<>(message, clock.currentTimeMillis())); TimestampedValue<OM> otherMessage = otherState.get(key); long now = clock.currentTimeMillis(); if (otherMessage != null && otherMessage.getTimestamp() > now - ttlMs) { JM joinResult = thisPartialJoinFn.apply(message, otherMessage.getValue()); return Collections.singletonList(joinResult); } } catch (Exception e) { throw new SamzaException("Error handling message in PartialJoinOperatorImpl " + getOpImplId(), e); } return Collections.emptyList(); }
@Override public byte[] toBytes(TimestampedValue<V> tv) { byte[] vBytes = vSerde.toBytes(tv.getValue()); int vBytesLength = vBytes != null ? vBytes.length : 0; ByteBuffer bb = ByteBuffer.allocate(vBytesLength + TIMESTAMP_BYTES); if (vBytes != null) { bb.put(vBytes); } bb.putLong(tv.getTimestamp()); return bb.array(); } }
@Override public TimestampedValue<V> fromBytes(byte[] bytes) { ByteBuffer bb = ByteBuffer.wrap(bytes); byte[] vBytes = new byte[bytes.length - TIMESTAMP_BYTES]; bb.get(vBytes, 0, vBytes.length); V v = vSerde.fromBytes(vBytes); long ts = bb.getLong(); return new TimestampedValue<>(v, ts); }
/** * Return a list of values in the store for the provided key and timestamp * * @param key the key to look up in the store * @param timestamp the timestamp to look up in the store * @return the list of values for the provided key */ private List<Object> getValues(K key, long timestamp) { ClosableIterator<TimestampedValue<Object>> iterator = timeSeriesStore.get(key, timestamp); List<TimestampedValue<Object>> timestampedValues = toList(iterator); List<Object> values = timestampedValues.stream().map(element -> element.getValue()).collect(Collectors.toList()); LOG.trace("Returning {} for key {} and timestamp {}", new Object[] {values, key, timestamp}); return values; }
long timestamp = (timestampedValues.isEmpty())? clock.currentTimeMillis() : timestampedValues.get(0).getTimestamp();
@Override public Collection<JM> handleMessage(M message, MessageCollector collector, TaskCoordinator coordinator) { try { KeyValueStore<K, TimestampedValue<M>> thisState = thisPartialJoinFn.getState(); KeyValueStore<K, TimestampedValue<OM>> otherState = otherPartialJoinFn.getState(); K key = thisPartialJoinFn.getKey(message); thisState.put(key, new TimestampedValue<>(message, clock.currentTimeMillis())); TimestampedValue<OM> otherMessage = otherState.get(key); long now = clock.currentTimeMillis(); if (otherMessage != null && otherMessage.getTimestamp() > now - ttlMs) { JM joinResult = thisPartialJoinFn.apply(message, otherMessage.getValue()); return Collections.singletonList(joinResult); } } catch (Exception e) { throw new SamzaException("Error handling message in PartialJoinOperatorImpl " + getOpImplId(), e); } return Collections.emptyList(); }
@Override public byte[] toBytes(TimestampedValue<V> tv) { byte[] vBytes = vSerde.toBytes(tv.getValue()); int vBytesLength = vBytes != null ? vBytes.length : 0; ByteBuffer bb = ByteBuffer.allocate(vBytesLength + TIMESTAMP_BYTES); if (vBytes != null) { bb.put(vBytes); } bb.putLong(tv.getTimestamp()); return bb.array(); } }