/** * Update the state upon receiving a watermark message. * @param watermarkMessage message of {@link WatermarkMessage} * @param ssp system stream partition * @return true iff the stream has a new watermark */ void update(WatermarkMessage watermarkMessage, SystemStreamPartition ssp) { WatermarkState state = watermarkStates.get(ssp); if (state != null) { state.update(watermarkMessage.getTimestamp(), watermarkMessage.getTaskName()); } else { LOG.error("SSP {} doesn't have watermark states", ssp); } }
/** * Update the state upon receiving a watermark message. * @param watermarkMessage message of {@link WatermarkMessage} * @param ssp system stream partition * @return true iff the stream has a new watermark */ void update(WatermarkMessage watermarkMessage, SystemStreamPartition ssp) { WatermarkState state = watermarkStates.get(ssp); if (state != null) { state.update(watermarkMessage.getTimestamp(), watermarkMessage.getTaskName()); } else { LOG.error("SSP {} doesn't have watermark states", ssp); } }
/** * Update the state upon receiving a watermark message. * @param watermarkMessage message of {@link WatermarkMessage} * @param ssp system stream partition * @return true iff the stream has a new watermark */ void update(WatermarkMessage watermarkMessage, SystemStreamPartition ssp) { WatermarkState state = watermarkStates.get(ssp); if (state != null) { state.update(watermarkMessage.getTimestamp(), watermarkMessage.getTaskName()); } else { LOG.error("SSP {} doesn't have watermark states", ssp); } }
/** * Update the state upon receiving a watermark message. * @param watermarkMessage message of {@link WatermarkMessage} * @param ssp system stream partition * @return true iff the stream has a new watermark */ void update(WatermarkMessage watermarkMessage, SystemStreamPartition ssp) { WatermarkState state = watermarkStates.get(ssp); if (state != null) { state.update(watermarkMessage.getTimestamp(), watermarkMessage.getTaskName()); } else { LOG.error("SSP {} doesn't have watermark states", ssp); } }
/** * Update the state upon receiving a watermark message. * @param watermarkMessage message of {@link WatermarkMessage} * @param ssp system stream partition * @return true iff the stream has a new watermark */ void update(WatermarkMessage watermarkMessage, SystemStreamPartition ssp) { WatermarkState state = watermarkStates.get(ssp); if (state != null) { state.update(watermarkMessage.getTimestamp(), watermarkMessage.getTaskName()); } else { LOG.error("SSP {} doesn't have watermark states", ssp); } }
/** * Aggregate the {@link WatermarkMessage} from each ssp into a watermark. Then call onWatermark() if * a new watermark exits. * @param watermarkMessage a {@link WatermarkMessage} object * @param ssp {@link SystemStreamPartition} that the message is coming from. * @param collector message collector * @param coordinator task coordinator */ public final void aggregateWatermark(WatermarkMessage watermarkMessage, SystemStreamPartition ssp, MessageCollector collector, TaskCoordinator coordinator) { LOG.debug("Received watermark {} from {}", watermarkMessage.getTimestamp(), ssp); watermarkStates.update(watermarkMessage, ssp); long watermark = watermarkStates.getWatermark(ssp.getSystemStream()); if (currentWatermark < watermark) { LOG.debug("Got watermark {} from stream {}", watermark, ssp.getSystemStream()); if (watermarkMessage.getTaskName() != null) { // This is the aggregation task, which already received all the watermark messages from upstream // broadcast the watermark to all the peer partitions controlMessageSender.broadcastToOtherPartitions(new WatermarkMessage(watermark), ssp, collector); } // populate the watermark through the dag onWatermark(watermark, collector, coordinator); // update metrics watermarkStates.updateAggregateMetric(ssp, watermark); } }
/** * Aggregate the {@link WatermarkMessage} from each ssp into a watermark. Then call onWatermark() if * a new watermark exits. * @param watermarkMessage a {@link WatermarkMessage} object * @param ssp {@link SystemStreamPartition} that the message is coming from. * @param collector message collector * @param coordinator task coordinator */ public final void aggregateWatermark(WatermarkMessage watermarkMessage, SystemStreamPartition ssp, MessageCollector collector, TaskCoordinator coordinator) { LOG.debug("Received watermark {} from {}", watermarkMessage.getTimestamp(), ssp); watermarkStates.update(watermarkMessage, ssp); long watermark = watermarkStates.getWatermark(ssp.getSystemStream()); if (currentWatermark < watermark) { LOG.debug("Got watermark {} from stream {}", watermark, ssp.getSystemStream()); if (watermarkMessage.getTaskName() != null) { // This is the aggregation task, which already received all the watermark messages from upstream // broadcast the watermark to all the peer partitions controlMessageSender.broadcastToOtherPartitions(new WatermarkMessage(watermark), ssp, collector); } // populate the watermark through the dag onWatermark(watermark, collector, coordinator); // update metrics watermarkStates.updateAggregateMetric(ssp, watermark); } }
/** * Aggregate the {@link WatermarkMessage} from each ssp into a watermark. Then call onWatermark() if * a new watermark exits. * @param watermarkMessage a {@link WatermarkMessage} object * @param ssp {@link SystemStreamPartition} that the message is coming from. * @param collector message collector * @param coordinator task coordinator */ public final void aggregateWatermark(WatermarkMessage watermarkMessage, SystemStreamPartition ssp, MessageCollector collector, TaskCoordinator coordinator) { LOG.debug("Received watermark {} from {}", watermarkMessage.getTimestamp(), ssp); watermarkStates.update(watermarkMessage, ssp); long watermark = watermarkStates.getWatermark(ssp.getSystemStream()); if (currentWatermark < watermark) { LOG.debug("Got watermark {} from stream {}", watermark, ssp.getSystemStream()); if (watermarkMessage.getTaskName() != null) { // This is the aggregation task, which already received all the watermark messages from upstream // broadcast the watermark to all the peer partitions controlMessageSender.broadcastToOtherPartitions(new WatermarkMessage(watermark), ssp, collector); } // populate the watermark through the dag onWatermark(watermark, collector, coordinator); // update metrics watermarkStates.updateAggregateMetric(ssp, watermark); } }
/** * Aggregate the {@link WatermarkMessage} from each ssp into a watermark. Then call onWatermark() if * a new watermark exits. * @param watermarkMessage a {@link WatermarkMessage} object * @param ssp {@link SystemStreamPartition} that the message is coming from. * @param collector message collector * @param coordinator task coordinator */ public final void aggregateWatermark(WatermarkMessage watermarkMessage, SystemStreamPartition ssp, MessageCollector collector, TaskCoordinator coordinator) { LOG.debug("Received watermark {} from {}", watermarkMessage.getTimestamp(), ssp); watermarkStates.update(watermarkMessage, ssp); long watermark = watermarkStates.getWatermark(ssp.getSystemStream()); if (currentWatermark < watermark) { LOG.debug("Got watermark {} from stream {}", watermark, ssp.getSystemStream()); if (watermarkMessage.getTaskName() != null) { // This is the aggregation task, which already received all the watermark messages from upstream // broadcast the watermark to all the peer partitions controlMessageSender.broadcastToOtherPartitions(new WatermarkMessage(watermark), ssp, collector); } // populate the watermark through the dag onWatermark(watermark, collector, coordinator); // update metrics watermarkStates.updateAggregateMetric(ssp, watermark); } }
/** * Aggregate the {@link WatermarkMessage} from each ssp into a watermark. Then call onWatermark() if * a new watermark exits. * @param watermarkMessage a {@link WatermarkMessage} object * @param ssp {@link SystemStreamPartition} that the message is coming from. * @param collector message collector * @param coordinator task coordinator */ public final void aggregateWatermark(WatermarkMessage watermarkMessage, SystemStreamPartition ssp, MessageCollector collector, TaskCoordinator coordinator) { LOG.debug("Received watermark {} from {}", watermarkMessage.getTimestamp(), ssp); watermarkStates.update(watermarkMessage, ssp); long watermark = watermarkStates.getWatermark(ssp.getSystemStream()); if (currentWatermark < watermark) { LOG.debug("Got watermark {} from stream {}", watermark, ssp.getSystemStream()); if (watermarkMessage.getTaskName() != null) { // This is the aggregation task, which already received all the watermark messages from upstream // broadcast the watermark to all the peer partitions controlMessageSender.broadcastToOtherPartitions(new WatermarkMessage(watermark), ssp, collector); } // populate the watermark through the dag onWatermark(watermark, collector, coordinator); // update metrics watermarkStates.updateAggregateMetric(ssp, watermark); } }
@Test public void testWatermarkMessageSerde() { IntermediateMessageSerde imserde = new IntermediateMessageSerde(new ObjectSerde()); String taskName = "task-1"; WatermarkMessage watermark = new WatermarkMessage(System.currentTimeMillis(), taskName); byte[] bytes = imserde.toBytes(watermark); WatermarkMessage de = (WatermarkMessage) imserde.fromBytes(bytes); assertEquals(MessageType.of(de), MessageType.WATERMARK); assertEquals(de.getTaskName(), taskName); assertTrue(de.getTimestamp() > 0); }