@Test public void testSharedBufferCountersClearing() throws Exception { SharedBuffer<Event> sharedBuffer = TestSharedBuffer.createTestBuffer(Event.createTypeSerializer()); int numberEvents = 4; Event[] events = new Event[numberEvents]; for (int i = 0; i < numberEvents; i++) { events[i] = new Event(i + 1, "e" + (i + 1), i); sharedBuffer.registerEvent(events[i], i); } sharedBuffer.advanceTime(3); Iterator<Map.Entry<Long, Integer>> counters = sharedBuffer.getEventCounters(); Map.Entry<Long, Integer> entry = counters.next(); assertEquals(3, entry.getKey().longValue()); assertEquals(1, entry.getValue().intValue()); assertFalse(counters.hasNext()); }
/** * Advances the time for the given NFA to the given timestamp. This means that no more events with timestamp * <b>lower</b> than the given timestamp should be passed to the nfa, This can lead to pruning and timeouts. */ private void advanceTime(NFAState nfaState, long timestamp) throws Exception { try (SharedBufferAccessor<IN> sharedBufferAccessor = partialMatches.getAccessor()) { Collection<Tuple2<Map<String, List<IN>>, Long>> timedOut = nfa.advanceTime(sharedBufferAccessor, nfaState, timestamp); if (!timedOut.isEmpty()) { processTimedOutSequences(timedOut); } } }
@VisibleForTesting boolean hasNonEmptySharedBuffer(KEY key) throws Exception { setCurrentKey(key); return !partialMatches.isEmpty(); }
/** * Decreases the reference counter for the given event so that it can be * removed once the reference counter reaches 0. * * @param eventId id of the event * @throws Exception Thrown if the system cannot access the state. */ public void releaseEvent(EventId eventId) throws Exception { Lockable<V> eventWrapper = sharedBuffer.getEvent(eventId); if (eventWrapper != null) { if (eventWrapper.release()) { sharedBuffer.removeEvent(eventId); } else { sharedBuffer.upsertEvent(eventId, eventWrapper); } } }
/** * Increases the reference counter for the given entry so that it is not * accidentally removed. * * @param node id of the entry */ public void lockNode(final NodeId node) { Lockable<SharedBufferNode> sharedBufferNode = sharedBuffer.getEntry(node); if (sharedBufferNode != null) { sharedBufferNode.lock(); sharedBuffer.upsertEntry(node, sharedBufferNode); } }
/** * Increases the reference counter for the given event so that it is not * accidentally removed. * * @param eventId id of the entry */ private void lockEvent(EventId eventId) { Lockable<V> eventWrapper = sharedBuffer.getEvent(eventId); checkState( eventWrapper != null, "Referring to non existent event with id %s", eventId); eventWrapper.lock(); sharedBuffer.upsertEvent(eventId, eventWrapper); }
eventIds[i] = sharedBuffer.registerEvent(events[i], timestamp); expectedPattern3.get("b").add(events[7]); try (SharedBufferAccessor<Event> sharedBufferAccessor = sharedBuffer.getAccessor()) { NodeId a10 = sharedBufferAccessor.put("a1", eventIds[0], null, DeweyNumber.fromString("1")); NodeId aLoop0 = sharedBufferAccessor.put("a[]", eventIds[1], a10, DeweyNumber.fromString("1.0")); assertTrue(sharedBuffer.isEmpty());
eventIds[i] = sharedBuffer.registerEvent(events[i], timestamp); expectedResult.get("c").add(events[4]); try (SharedBufferAccessor<Event> sharedBufferAccessor = sharedBuffer.getAccessor()) { NodeId a = sharedBufferAccessor.put("a", eventIds[0], null, DeweyNumber.fromString("1")); NodeId b = sharedBufferAccessor.put("b", eventIds[1], a, DeweyNumber.fromString("1.0"));
/** * Notifies shared buffer that there will be no events with timestamp <&eq; the given value. It allows to clear * internal counters for number of events seen so far per timestamp. * * @param timestamp watermark, no earlier events will arrive * @throws Exception Thrown if the system cannot access the state. */ public void advanceTime(long timestamp) throws Exception { sharedBuffer.advanceTime(timestamp); }
/** * Adds another unique event to the shared buffer and assigns a unique id for it. It automatically creates a * lock on this event, so it won't be removed during processing of that event. Therefore the lock should be removed * after processing all {@link org.apache.flink.cep.nfa.ComputationState}s * * <p><b>NOTE:</b>Should be called only once for each unique event! * * @param value event to be registered * @return unique id of that event that should be used when putting entries to the buffer. * @throws Exception Thrown if the system cannot access the state. */ public EventId registerEvent(V value, long timestamp) throws Exception { return sharedBuffer.registerEvent(value, timestamp); }
@Override public void process(Object key, ValueState<MigratedNFA<IN>> state) throws Exception { MigratedNFA<IN> oldState = state.value(); computationStates.update(new NFAState(oldState.getComputationStates())); org.apache.flink.cep.nfa.SharedBuffer<IN> sharedBuffer = oldState.getSharedBuffer(); partialMatches.init(sharedBuffer.getEventsBuffer(), sharedBuffer.getPages()); state.clear(); } }
/** * Persists the entry in the cache to the underlay state. * * @throws Exception Thrown if the system cannot access the state. */ public void close() throws Exception { sharedBuffer.flushCache(); }
@Override public void initializeState(StateInitializationContext context) throws Exception { super.initializeState(context); // initializeState through the provided context computationStates = context.getKeyedStateStore().getState( new ValueStateDescriptor<>( NFA_STATE_NAME, new NFAStateSerializer())); partialMatches = new SharedBuffer<>(context.getKeyedStateStore(), inputSerializer); elementQueueState = context.getKeyedStateStore().getMapState( new MapStateDescriptor<>( EVENT_QUEUE_STATE_NAME, LongSerializer.INSTANCE, new ListSerializer<>(inputSerializer))); migrateOldState(); }
/** * Removes the {@code SharedBufferNode}, when the ref is decreased to zero, and also * decrease the ref of the edge on this node. * * @param node id of the entry * @param sharedBufferNode the node body to be removed * @throws Exception Thrown if the system cannot access the state. */ private void removeNode(NodeId node, SharedBufferNode sharedBufferNode) throws Exception { sharedBuffer.removeEntry(node); EventId eventId = node.getEventId(); releaseEvent(eventId); for (SharedBufferEdge sharedBufferEdge : sharedBufferNode.getEdges()) { releaseNode(sharedBufferEdge.getTarget()); } }
/** * Extracts the real event from the sharedBuffer with pre-extracted eventId. * * @param match the matched event's eventId. * @return the event associated with the eventId. */ public Map<String, List<V>> materializeMatch(Map<String, List<EventId>> match) { Map<String, List<V>> materializedMatch = new LinkedHashMap<>(match.size()); for (Map.Entry<String, List<EventId>> pattern : match.entrySet()) { List<V> events = new ArrayList<>(pattern.getValue().size()); for (EventId eventId : pattern.getValue()) { try { V event = sharedBuffer.getEvent(eventId).getElement(); events.add(event); } catch (Exception ex) { throw new WrappingRuntimeException(ex); } } materializedMatch.put(pattern.getKey(), events); } return materializedMatch; }
@Test public void testClearingSharedBufferWithMultipleEdgesBetweenEntries() throws Exception { SharedBuffer<Event> sharedBuffer = TestSharedBuffer.createTestBuffer(Event.createTypeSerializer()); int numberEvents = 8; Event[] events = new Event[numberEvents]; EventId[] eventIds = new EventId[numberEvents]; final long timestamp = 1L; for (int i = 0; i < numberEvents; i++) { events[i] = new Event(i + 1, "e" + (i + 1), i); eventIds[i] = sharedBuffer.registerEvent(events[i], timestamp); } try (SharedBufferAccessor<Event> sharedBufferAccessor = sharedBuffer.getAccessor()) { NodeId start = sharedBufferAccessor.put("start", eventIds[1], null, DeweyNumber.fromString("1")); NodeId b0 = sharedBufferAccessor.put("branching", eventIds[2], start, DeweyNumber.fromString("1.0")); NodeId b1 = sharedBufferAccessor.put("branching", eventIds[3], start, DeweyNumber.fromString("1.1")); NodeId b00 = sharedBufferAccessor.put("branching", eventIds[3], b0, DeweyNumber.fromString("1.0.0")); sharedBufferAccessor.put("branching", eventIds[4], b00, DeweyNumber.fromString("1.0.0.0")); NodeId b10 = sharedBufferAccessor.put("branching", eventIds[4], b1, DeweyNumber.fromString("1.1.0")); //simulate IGNORE (next event can point to events[2]) sharedBufferAccessor.lockNode(b0); sharedBufferAccessor.releaseNode(b10); for (EventId eventId : eventIds) { sharedBufferAccessor.releaseEvent(eventId); } } //There should be still events[1] and events[2] in the buffer assertFalse(sharedBuffer.isEmpty()); }
/** * Decreases the reference counter for the given entry so that it can be * removed once the reference counter reaches 0. * * @param node id of the entry * @throws Exception Thrown if the system cannot access the state. */ public void releaseNode(final NodeId node) throws Exception { Lockable<SharedBufferNode> sharedBufferNode = sharedBuffer.getEntry(node); if (sharedBufferNode != null) { if (sharedBufferNode.release()) { removeNode(node, sharedBufferNode.getElement()); } else { sharedBuffer.upsertEntry(node, sharedBufferNode); } } }
/** * Decreases the reference counter for the given event so that it can be * removed once the reference counter reaches 0. * * @param eventId id of the event * @throws Exception Thrown if the system cannot access the state. */ public void releaseEvent(EventId eventId) throws Exception { Lockable<V> eventWrapper = sharedBuffer.getEvent(eventId); if (eventWrapper != null) { if (eventWrapper.release()) { sharedBuffer.removeEvent(eventId); } else { sharedBuffer.upsertEvent(eventId, eventWrapper); } } }
/** * Increases the reference counter for the given event so that it is not * accidentally removed. * * @param eventId id of the entry */ private void lockEvent(EventId eventId) { Lockable<V> eventWrapper = sharedBuffer.getEvent(eventId); checkState( eventWrapper != null, "Referring to non existent event with id %s", eventId); eventWrapper.lock(); sharedBuffer.upsertEvent(eventId, eventWrapper); }