/** * This method is deprecated in favor of WatermarkManager.buildEndOfStreamEnvelope(SystemStreamPartition ssp). * * @param ssp The SSP that is at end-of-stream. * @return an IncomingMessageEnvelope corresponding to end-of-stream for that SSP. */ public static IncomingMessageEnvelope buildEndOfStreamEnvelope(SystemStreamPartition ssp) { return new IncomingMessageEnvelope(ssp, END_OF_STREAM_OFFSET, null, new EndOfStreamMessage(null)); }
/** * Update the state upon receiving an end-of-stream message. * @param eos message of {@link EndOfStreamMessage} * @param ssp system stream partition */ void update(EndOfStreamMessage eos, SystemStreamPartition ssp) { EndOfStreamState state = eosStates.get(ssp); state.update(eos.getTaskName()); }
/** * Aggregate {@link EndOfStreamMessage} from each ssp of the stream. * Invoke onEndOfStream() if the stream reaches the end. * @param eos {@link EndOfStreamMessage} object * @param ssp system stream partition * @param collector message collector * @param coordinator task coordinator */ public final void aggregateEndOfStream(EndOfStreamMessage eos, SystemStreamPartition ssp, MessageCollector collector, TaskCoordinator coordinator) { LOG.info("Received end-of-stream message from task {} in {}", eos.getTaskName(), ssp); eosStates.update(eos, ssp); SystemStream stream = ssp.getSystemStream(); if (eosStates.isEndOfStream(stream)) { LOG.info("Input {} reaches the end for task {}", stream.toString(), taskName.getTaskName()); if (eos.getTaskName() != null) { // This is the aggregation task, which already received all the eos messages from upstream // broadcast the end-of-stream to all the peer partitions controlMessageSender.broadcastToOtherPartitions(new EndOfStreamMessage(), ssp, collector); } // populate the end-of-stream through the dag onEndOfStream(collector, coordinator); if (eosStates.allEndOfStream()) { // all inputs have been end-of-stream, shut down the task LOG.info("All input streams have reached the end for task {}", taskName.getTaskName()); coordinator.commit(TaskCoordinator.RequestScope.CURRENT_TASK); coordinator.shutdown(TaskCoordinator.RequestScope.CURRENT_TASK); } } }
@Test public void testEndOfStreamMessageSerde() { IntermediateMessageSerde imserde = new IntermediateMessageSerde(new ObjectSerde()); String streamId = "test-stream"; String taskName = "task-1"; EndOfStreamMessage eos = new EndOfStreamMessage(taskName); byte[] bytes = imserde.toBytes(eos); EndOfStreamMessage de = (EndOfStreamMessage) imserde.fromBytes(bytes); assertEquals(MessageType.of(de), MessageType.END_OF_STREAM); assertEquals(de.getTaskName(), taskName); assertEquals(de.getVersion(), 1); } }
/** * Aggregate {@link EndOfStreamMessage} from each ssp of the stream. * Invoke onEndOfStream() if the stream reaches the end. * @param eos {@link EndOfStreamMessage} object * @param ssp system stream partition * @param collector message collector * @param coordinator task coordinator */ public final void aggregateEndOfStream(EndOfStreamMessage eos, SystemStreamPartition ssp, MessageCollector collector, TaskCoordinator coordinator) { LOG.info("Received end-of-stream message from task {} in {}", eos.getTaskName(), ssp); eosStates.update(eos, ssp); SystemStream stream = ssp.getSystemStream(); if (eosStates.isEndOfStream(stream)) { LOG.info("Input {} reaches the end for task {}", stream.toString(), taskName.getTaskName()); if (eos.getTaskName() != null) { // This is the aggregation task, which already received all the eos messages from upstream // broadcast the end-of-stream to all the peer partitions controlMessageSender.broadcastToOtherPartitions(new EndOfStreamMessage(), ssp, collector); } // populate the end-of-stream through the dag onEndOfStream(collector, coordinator); if (eosStates.allEndOfStream()) { // all inputs have been end-of-stream, shut down the task LOG.info("All input streams have reached the end for task {}", taskName.getTaskName()); coordinator.commit(TaskCoordinator.RequestScope.CURRENT_TASK); coordinator.shutdown(TaskCoordinator.RequestScope.CURRENT_TASK); } } }
/** * Aggregate {@link EndOfStreamMessage} from each ssp of the stream. * Invoke onEndOfStream() if the stream reaches the end. * @param eos {@link EndOfStreamMessage} object * @param ssp system stream partition * @param collector message collector * @param coordinator task coordinator */ public final void aggregateEndOfStream(EndOfStreamMessage eos, SystemStreamPartition ssp, MessageCollector collector, TaskCoordinator coordinator) { LOG.info("Received end-of-stream message from task {} in {}", eos.getTaskName(), ssp); eosStates.update(eos, ssp); SystemStream stream = ssp.getSystemStream(); if (eosStates.isEndOfStream(stream)) { LOG.info("Input {} reaches the end for task {}", stream.toString(), taskName.getTaskName()); if (eos.getTaskName() != null) { // This is the aggregation task, which already received all the eos messages from upstream // broadcast the end-of-stream to all the peer partitions controlMessageSender.broadcastToOtherPartitions(new EndOfStreamMessage(), ssp, collector); } // populate the end-of-stream through the dag onEndOfStream(collector, coordinator); if (eosStates.allEndOfStream()) { // all inputs have been end-of-stream, shut down the task LOG.info("All input streams have reached the end for task {}", taskName.getTaskName()); coordinator.commit(TaskCoordinator.RequestScope.CURRENT_TASK); coordinator.shutdown(TaskCoordinator.RequestScope.CURRENT_TASK); } } }
/** * This method is deprecated in favor of WatermarkManager.buildEndOfStreamEnvelope(SystemStreamPartition ssp). * * @param ssp The SSP that is at end-of-stream. * @return an IncomingMessageEnvelope corresponding to end-of-stream for that SSP. */ public static IncomingMessageEnvelope buildEndOfStreamEnvelope(SystemStreamPartition ssp) { return new IncomingMessageEnvelope(ssp, END_OF_STREAM_OFFSET, null, new EndOfStreamMessage(null)); }
/** * Update the state upon receiving an end-of-stream message. * @param eos message of {@link EndOfStreamMessage} * @param ssp system stream partition */ void update(EndOfStreamMessage eos, SystemStreamPartition ssp) { EndOfStreamState state = eosStates.get(ssp); state.update(eos.getTaskName()); }
/** * Aggregate {@link EndOfStreamMessage} from each ssp of the stream. * Invoke onEndOfStream() if the stream reaches the end. * @param eos {@link EndOfStreamMessage} object * @param ssp system stream partition * @param collector message collector * @param coordinator task coordinator */ public final void aggregateEndOfStream(EndOfStreamMessage eos, SystemStreamPartition ssp, MessageCollector collector, TaskCoordinator coordinator) { LOG.info("Received end-of-stream message from task {} in {}", eos.getTaskName(), ssp); eosStates.update(eos, ssp); SystemStream stream = ssp.getSystemStream(); if (eosStates.isEndOfStream(stream)) { LOG.info("Input {} reaches the end for task {}", stream.toString(), taskName.getTaskName()); if (eos.getTaskName() != null) { // This is the aggregation task, which already received all the eos messages from upstream // broadcast the end-of-stream to all the peer partitions controlMessageSender.broadcastToOtherPartitions(new EndOfStreamMessage(), ssp, collector); } // populate the end-of-stream through the dag onEndOfStream(collector, coordinator); if (eosStates.allEndOfStream()) { // all inputs have been end-of-stream, shut down the task LOG.info("All input streams have reached the end for task {}", taskName.getTaskName()); coordinator.commit(TaskCoordinator.RequestScope.CURRENT_TASK); coordinator.shutdown(TaskCoordinator.RequestScope.CURRENT_TASK); } } }
@Override protected Collection<Void> handleEndOfStream(MessageCollector collector, TaskCoordinator coordinator) { sendControlMessage(new EndOfStreamMessage(taskName), collector); return Collections.emptyList(); }
/** * Update the state upon receiving an end-of-stream message. * @param eos message of {@link EndOfStreamMessage} * @param ssp system stream partition */ void update(EndOfStreamMessage eos, SystemStreamPartition ssp) { EndOfStreamState state = eosStates.get(ssp); state.update(eos.getTaskName()); }
/** * Aggregate {@link EndOfStreamMessage} from each ssp of the stream. * Invoke onEndOfStream() if the stream reaches the end. * @param eos {@link EndOfStreamMessage} object * @param ssp system stream partition * @param collector message collector * @param coordinator task coordinator */ public final void aggregateEndOfStream(EndOfStreamMessage eos, SystemStreamPartition ssp, MessageCollector collector, TaskCoordinator coordinator) { LOG.info("Received end-of-stream message from task {} in {}", eos.getTaskName(), ssp); eosStates.update(eos, ssp); SystemStream stream = ssp.getSystemStream(); if (eosStates.isEndOfStream(stream)) { LOG.info("Input {} reaches the end for task {}", stream.toString(), taskName.getTaskName()); if (eos.getTaskName() != null) { // This is the aggregation task, which already received all the eos messages from upstream // broadcast the end-of-stream to all the peer partitions controlMessageSender.broadcastToOtherPartitions(new EndOfStreamMessage(), ssp, collector); } // populate the end-of-stream through the dag onEndOfStream(collector, coordinator); if (eosStates.allEndOfStream()) { // all inputs have been end-of-stream, shut down the task LOG.info("All input streams have reached the end for task {}", taskName.getTaskName()); coordinator.commit(TaskCoordinator.RequestScope.CURRENT_TASK); coordinator.shutdown(TaskCoordinator.RequestScope.CURRENT_TASK); } } }
@Override protected Collection<Void> handleEndOfStream(MessageCollector collector, TaskCoordinator coordinator) { sendControlMessage(new EndOfStreamMessage(taskName), collector); return Collections.emptyList(); }
/** * Update the state upon receiving an end-of-stream message. * @param eos message of {@link EndOfStreamMessage} * @param ssp system stream partition */ void update(EndOfStreamMessage eos, SystemStreamPartition ssp) { EndOfStreamState state = eosStates.get(ssp); state.update(eos.getTaskName()); }
@Override protected Collection<Void> handleEndOfStream(MessageCollector collector, TaskCoordinator coordinator) { sendControlMessage(new EndOfStreamMessage(taskName), collector); return Collections.emptyList(); }
/** * Update the state upon receiving an end-of-stream message. * @param eos message of {@link EndOfStreamMessage} * @param ssp system stream partition */ void update(EndOfStreamMessage eos, SystemStreamPartition ssp) { EndOfStreamState state = eosStates.get(ssp); state.update(eos.getTaskName()); }
@Override protected Collection<Void> handleEndOfStream(MessageCollector collector, TaskCoordinator coordinator) { sendControlMessage(new EndOfStreamMessage(taskName), collector); return Collections.emptyList(); }
@Override protected Collection<Void> handleEndOfStream(MessageCollector collector, TaskCoordinator coordinator) { sendControlMessage(new EndOfStreamMessage(taskName), collector); return Collections.emptyList(); }
@Override protected Collection<Void> handleEndOfStream(MessageCollector collector, TaskCoordinator coordinator) { sendControlMessage(new EndOfStreamMessage(taskName), collector); return Collections.emptyList(); }
@Override protected Collection<Void> handleEndOfStream(MessageCollector collector, TaskCoordinator coordinator) { sendControlMessage(new EndOfStreamMessage(taskName), collector); return Collections.emptyList(); }