public static IncomingMessageEnvelope buildWatermarkEnvelope(SystemStreamPartition ssp, long watermark) { return new IncomingMessageEnvelope(ssp, null, null, new WatermarkMessage(watermark, null)); }
public static IncomingMessageEnvelope buildWatermarkEnvelope(SystemStreamPartition ssp, long watermark) { return new IncomingMessageEnvelope(ssp, null, null, new WatermarkMessage(watermark, null)); }
@Override protected Collection<Void> handleWatermark(long watermark, MessageCollector collector, TaskCoordinator coordinator) { sendControlMessage(new WatermarkMessage(watermark, taskName), collector); return Collections.emptyList(); }
@Override protected Collection<Void> handleWatermark(long watermark, MessageCollector collector, TaskCoordinator coordinator) { sendControlMessage(new WatermarkMessage(watermark, taskName), collector); return Collections.emptyList(); }
@Override protected Collection<Void> handleWatermark(long watermark, MessageCollector collector, TaskCoordinator coordinator) { sendControlMessage(new WatermarkMessage(watermark, taskName), collector); return Collections.emptyList(); }
@Override protected Collection<Void> handleWatermark(long watermark, MessageCollector collector, TaskCoordinator coordinator) { sendControlMessage(new WatermarkMessage(watermark, taskName), collector); return Collections.emptyList(); }
@Override protected Collection<Void> handleWatermark(long watermark, MessageCollector collector, TaskCoordinator coordinator) { sendControlMessage(new WatermarkMessage(watermark, taskName), collector); return Collections.emptyList(); }
@Override protected Collection<Void> handleWatermark(long watermark, MessageCollector collector, TaskCoordinator coordinator) { sendControlMessage(new WatermarkMessage(watermark, taskName), collector); return Collections.emptyList(); }
@Override protected Collection<Void> handleWatermark(long watermark, MessageCollector collector, TaskCoordinator coordinator) { sendControlMessage(new WatermarkMessage(watermark, taskName), collector); return Collections.emptyList(); }
@Override protected Collection<Void> handleWatermark(long watermark, MessageCollector collector, TaskCoordinator coordinator) { sendControlMessage(new WatermarkMessage(watermark, taskName), collector); return Collections.emptyList(); }
@Override protected Collection<Void> handleWatermark(long watermark, MessageCollector collector, TaskCoordinator coordinator) { sendControlMessage(new WatermarkMessage(watermark, taskName), collector); return Collections.emptyList(); }
@Override protected Collection<Void> handleWatermark(long watermark, MessageCollector collector, TaskCoordinator coordinator) { sendControlMessage(new WatermarkMessage(watermark, taskName), collector); return Collections.emptyList(); }
/** * Aggregate the {@link WatermarkMessage} from each ssp into a watermark. Then call onWatermark() if * a new watermark exits. * @param watermarkMessage a {@link WatermarkMessage} object * @param ssp {@link SystemStreamPartition} that the message is coming from. * @param collector message collector * @param coordinator task coordinator */ public final void aggregateWatermark(WatermarkMessage watermarkMessage, SystemStreamPartition ssp, MessageCollector collector, TaskCoordinator coordinator) { LOG.debug("Received watermark {} from {}", watermarkMessage.getTimestamp(), ssp); watermarkStates.update(watermarkMessage, ssp); long watermark = watermarkStates.getWatermark(ssp.getSystemStream()); if (currentWatermark < watermark) { LOG.debug("Got watermark {} from stream {}", watermark, ssp.getSystemStream()); if (watermarkMessage.getTaskName() != null) { // This is the aggregation task, which already received all the watermark messages from upstream // broadcast the watermark to all the peer partitions controlMessageSender.broadcastToOtherPartitions(new WatermarkMessage(watermark), ssp, collector); } // populate the watermark through the dag onWatermark(watermark, collector, coordinator); // update metrics watermarkStates.updateAggregateMetric(ssp, watermark); } }
/** * Aggregate the {@link WatermarkMessage} from each ssp into a watermark. Then call onWatermark() if * a new watermark exits. * @param watermarkMessage a {@link WatermarkMessage} object * @param ssp {@link SystemStreamPartition} that the message is coming from. * @param collector message collector * @param coordinator task coordinator */ public final void aggregateWatermark(WatermarkMessage watermarkMessage, SystemStreamPartition ssp, MessageCollector collector, TaskCoordinator coordinator) { LOG.debug("Received watermark {} from {}", watermarkMessage.getTimestamp(), ssp); watermarkStates.update(watermarkMessage, ssp); long watermark = watermarkStates.getWatermark(ssp.getSystemStream()); if (currentWatermark < watermark) { LOG.debug("Got watermark {} from stream {}", watermark, ssp.getSystemStream()); if (watermarkMessage.getTaskName() != null) { // This is the aggregation task, which already received all the watermark messages from upstream // broadcast the watermark to all the peer partitions controlMessageSender.broadcastToOtherPartitions(new WatermarkMessage(watermark), ssp, collector); } // populate the watermark through the dag onWatermark(watermark, collector, coordinator); // update metrics watermarkStates.updateAggregateMetric(ssp, watermark); } }
/** * Aggregate the {@link WatermarkMessage} from each ssp into a watermark. Then call onWatermark() if * a new watermark exits. * @param watermarkMessage a {@link WatermarkMessage} object * @param ssp {@link SystemStreamPartition} that the message is coming from. * @param collector message collector * @param coordinator task coordinator */ public final void aggregateWatermark(WatermarkMessage watermarkMessage, SystemStreamPartition ssp, MessageCollector collector, TaskCoordinator coordinator) { LOG.debug("Received watermark {} from {}", watermarkMessage.getTimestamp(), ssp); watermarkStates.update(watermarkMessage, ssp); long watermark = watermarkStates.getWatermark(ssp.getSystemStream()); if (currentWatermark < watermark) { LOG.debug("Got watermark {} from stream {}", watermark, ssp.getSystemStream()); if (watermarkMessage.getTaskName() != null) { // This is the aggregation task, which already received all the watermark messages from upstream // broadcast the watermark to all the peer partitions controlMessageSender.broadcastToOtherPartitions(new WatermarkMessage(watermark), ssp, collector); } // populate the watermark through the dag onWatermark(watermark, collector, coordinator); // update metrics watermarkStates.updateAggregateMetric(ssp, watermark); } }
/** * Aggregate the {@link WatermarkMessage} from each ssp into a watermark. Then call onWatermark() if * a new watermark exits. * @param watermarkMessage a {@link WatermarkMessage} object * @param ssp {@link SystemStreamPartition} that the message is coming from. * @param collector message collector * @param coordinator task coordinator */ public final void aggregateWatermark(WatermarkMessage watermarkMessage, SystemStreamPartition ssp, MessageCollector collector, TaskCoordinator coordinator) { LOG.debug("Received watermark {} from {}", watermarkMessage.getTimestamp(), ssp); watermarkStates.update(watermarkMessage, ssp); long watermark = watermarkStates.getWatermark(ssp.getSystemStream()); if (currentWatermark < watermark) { LOG.debug("Got watermark {} from stream {}", watermark, ssp.getSystemStream()); if (watermarkMessage.getTaskName() != null) { // This is the aggregation task, which already received all the watermark messages from upstream // broadcast the watermark to all the peer partitions controlMessageSender.broadcastToOtherPartitions(new WatermarkMessage(watermark), ssp, collector); } // populate the watermark through the dag onWatermark(watermark, collector, coordinator); // update metrics watermarkStates.updateAggregateMetric(ssp, watermark); } }
/** * Aggregate the {@link WatermarkMessage} from each ssp into a watermark. Then call onWatermark() if * a new watermark exits. * @param watermarkMessage a {@link WatermarkMessage} object * @param ssp {@link SystemStreamPartition} that the message is coming from. * @param collector message collector * @param coordinator task coordinator */ public final void aggregateWatermark(WatermarkMessage watermarkMessage, SystemStreamPartition ssp, MessageCollector collector, TaskCoordinator coordinator) { LOG.debug("Received watermark {} from {}", watermarkMessage.getTimestamp(), ssp); watermarkStates.update(watermarkMessage, ssp); long watermark = watermarkStates.getWatermark(ssp.getSystemStream()); if (currentWatermark < watermark) { LOG.debug("Got watermark {} from stream {}", watermark, ssp.getSystemStream()); if (watermarkMessage.getTaskName() != null) { // This is the aggregation task, which already received all the watermark messages from upstream // broadcast the watermark to all the peer partitions controlMessageSender.broadcastToOtherPartitions(new WatermarkMessage(watermark), ssp, collector); } // populate the watermark through the dag onWatermark(watermark, collector, coordinator); // update metrics watermarkStates.updateAggregateMetric(ssp, watermark); } }
@Test public void testSend() { SystemStreamMetadata metadata = mock(SystemStreamMetadata.class); Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadata = new HashMap<>(); partitionMetadata.put(new Partition(0), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class)); partitionMetadata.put(new Partition(1), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class)); partitionMetadata.put(new Partition(2), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class)); partitionMetadata.put(new Partition(3), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class)); when(metadata.getSystemStreamPartitionMetadata()).thenReturn(partitionMetadata); StreamMetadataCache metadataCache = mock(StreamMetadataCache.class); when(metadataCache.getSystemStreamMetadata(anyObject(), anyBoolean())).thenReturn(metadata); SystemStream systemStream = new SystemStream("test-system", "test-stream"); Set<Integer> partitions = new HashSet<>(); MessageCollector collector = mock(MessageCollector.class); doAnswer(invocation -> { OutgoingMessageEnvelope envelope = (OutgoingMessageEnvelope) invocation.getArguments()[0]; partitions.add((Integer) envelope.getPartitionKey()); assertEquals(envelope.getSystemStream(), systemStream); return null; }).when(collector).send(any()); ControlMessageSender sender = new ControlMessageSender(metadataCache); WatermarkMessage watermark = new WatermarkMessage(System.currentTimeMillis(), "task 0"); sender.send(watermark, systemStream, collector); assertEquals(partitions.size(), 1); }
@Test public void testBroadcast() { SystemStreamMetadata metadata = mock(SystemStreamMetadata.class); Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadata = new HashMap<>(); partitionMetadata.put(new Partition(0), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class)); partitionMetadata.put(new Partition(1), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class)); partitionMetadata.put(new Partition(2), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class)); partitionMetadata.put(new Partition(3), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class)); when(metadata.getSystemStreamPartitionMetadata()).thenReturn(partitionMetadata); StreamMetadataCache metadataCache = mock(StreamMetadataCache.class); when(metadataCache.getSystemStreamMetadata(anyObject(), anyBoolean())).thenReturn(metadata); SystemStream systemStream = new SystemStream("test-system", "test-stream"); Set<Integer> partitions = new HashSet<>(); MessageCollector collector = mock(MessageCollector.class); doAnswer(invocation -> { OutgoingMessageEnvelope envelope = (OutgoingMessageEnvelope) invocation.getArguments()[0]; partitions.add((Integer) envelope.getPartitionKey()); assertEquals(envelope.getSystemStream(), systemStream); return null; }).when(collector).send(any()); ControlMessageSender sender = new ControlMessageSender(metadataCache); WatermarkMessage watermark = new WatermarkMessage(System.currentTimeMillis(), "task 0"); SystemStreamPartition ssp = new SystemStreamPartition(systemStream, new Partition(0)); sender.broadcastToOtherPartitions(watermark, ssp, collector); assertEquals(partitions.size(), 3); } }
@Test public void testWatermarkMessageSerde() { IntermediateMessageSerde imserde = new IntermediateMessageSerde(new ObjectSerde()); String taskName = "task-1"; WatermarkMessage watermark = new WatermarkMessage(System.currentTimeMillis(), taskName); byte[] bytes = imserde.toBytes(watermark); WatermarkMessage de = (WatermarkMessage) imserde.fromBytes(bytes); assertEquals(MessageType.of(de), MessageType.WATERMARK); assertEquals(de.getTaskName(), taskName); assertTrue(de.getTimestamp() > 0); }