/** * Sends a specified message envelope from a specified Samza source. * @param source String representing the source of the message. * @param envelope Aggregate object representing the serialized message to send from the source. */ @Override public void send(String source, OutgoingMessageEnvelope envelope) { Object key = envelope.getKey(); Object message = envelope.getMessage(); Object partitionKey; // We use the partition key from message if available, if not fallback to message key or use message as partition // key as the final resort. if (envelope.getPartitionKey() != null) { partitionKey = envelope.getPartitionKey(); } else if (key != null) { partitionKey = key; } else { partitionKey = message; } Preconditions.checkNotNull(partitionKey, "Failed to compute partition key for the message: " + envelope); int partition = Math.abs(partitionKey.hashCode()) % memoryManager.getPartitionCountForSystemStream(envelope.getSystemStream()); SystemStreamPartition ssp = new SystemStreamPartition(envelope.getSystemStream(), new Partition(partition)); memoryManager.put(ssp, key, message); }
@Override protected Collection<Void> handleMessage(M message, MessageCollector collector, TaskCoordinator coordinator) { collector.send(new OutgoingMessageEnvelope(systemStream, 0, null, message)); return Collections.emptyList(); }
protected Object getEnvelopePartitionId(OutgoingMessageEnvelope envelope) { return envelope.getPartitionKey() == null ? envelope.getKey() : envelope.getPartitionKey(); }
@Override public void send(String source, OutgoingMessageEnvelope envelope) { LOG.info(String.format(String.format("Message %d :", messageCounter.incrementAndGet()))); String msg = String.format("OutputStream:%s Key:%s Value:%s", envelope.getSystemStream(), envelope.getKey(), new String((byte[]) envelope.getMessage())); LOG.info(msg); SamzaExecutor.saveOutputMessage(envelope); }
protected EventData createEventData(String streamId, OutgoingMessageEnvelope envelope) { Optional<Interceptor> interceptor = Optional.ofNullable(interceptors.getOrDefault(streamId, null)); byte[] eventValue = (byte[]) envelope.getMessage(); if (interceptor.isPresent()) { eventValue = interceptor.get().intercept(eventValue); } EventData eventData = new EventDataImpl(eventValue); eventData.getProperties().put(PRODUCE_TIMESTAMP, Long.toString(System.currentTimeMillis())); if (config.getSendKeyInEventProperties(systemName)) { String keyValue = ""; if (envelope.getKey() != null) { keyValue = (envelope.getKey() instanceof byte[]) ? new String((byte[]) envelope.getKey()) : envelope.getKey().toString(); } eventData.getProperties().put(KEY, keyValue); } return eventData; }
private String getCompressedFormat(OutgoingMessageEnvelope envelope) { return new String((byte[]) envelope.getMessage()); }
@Test public void testSend() { SystemStreamMetadata metadata = mock(SystemStreamMetadata.class); Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadata = new HashMap<>(); partitionMetadata.put(new Partition(0), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class)); partitionMetadata.put(new Partition(1), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class)); partitionMetadata.put(new Partition(2), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class)); partitionMetadata.put(new Partition(3), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class)); when(metadata.getSystemStreamPartitionMetadata()).thenReturn(partitionMetadata); StreamMetadataCache metadataCache = mock(StreamMetadataCache.class); when(metadataCache.getSystemStreamMetadata(anyObject(), anyBoolean())).thenReturn(metadata); SystemStream systemStream = new SystemStream("test-system", "test-stream"); Set<Integer> partitions = new HashSet<>(); MessageCollector collector = mock(MessageCollector.class); doAnswer(invocation -> { OutgoingMessageEnvelope envelope = (OutgoingMessageEnvelope) invocation.getArguments()[0]; partitions.add((Integer) envelope.getPartitionKey()); assertEquals(envelope.getSystemStream(), systemStream); return null; }).when(collector).send(any()); ControlMessageSender sender = new ControlMessageSender(metadataCache); WatermarkMessage watermark = new WatermarkMessage(System.currentTimeMillis(), "task 0"); sender.send(watermark, systemStream, collector); assertEquals(partitions.size(), 1); }
protected IndexRequest getRequest(OutgoingMessageEnvelope envelope) { String[] parts = envelope.getSystemStream().getStream().split("/"); if (parts.length != 2) { throw new SamzaException("Elasticsearch stream name must match pattern {index}/{type}"); } String index = parts[0]; String type = parts[1]; return Requests.indexRequest(index).type(type); }
protected Optional<String> getId(OutgoingMessageEnvelope envelope) { Object id = envelope.getKey(); if (id == null) { return Optional.absent(); } return Optional.of(id.toString()); }
protected Optional<String> getRoutingKey(OutgoingMessageEnvelope envelope) { Object partitionKey = envelope.getPartitionKey(); if (partitionKey == null) { return Optional.absent(); } return Optional.of(partitionKey.toString()); }
private String getPrettyFormat(OutgoingMessageEnvelope envelope) { lastErrorMsg = ""; String value = new String((byte[]) envelope.getMessage()); ObjectMapper mapper = new ObjectMapper(); String formattedValue; try { Object json = mapper.readValue(value, Object.class); formattedValue = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(json); } catch (IOException ex) { formattedValue = value; String msg = "getPrettyFormat failed with exception while formatting json "; lastErrorMsg = msg + ex.toString(); LOG.error(msg, ex); } return formattedValue; }
@Override public void send(String source, OutgoingMessageEnvelope envelope) { String msg = String.format("OutputStream:%s Key:%s Value:%s", envelope.getSystemStream(), envelope.getKey(), envelope.getMessage()); LOG.info(msg); System.out.println(msg); }
public static CoordinatorStreamMessage deserializeCoordinatorStreamMessage(OutgoingMessageEnvelope msg) { JsonSerde<List<?>> keySerde = new JsonSerde<>(); Object[] keyArray = keySerde.fromBytes((byte[]) msg.getKey()).toArray(); JsonSerde<Map<String, Object>> msgSerde = new JsonSerde<>(); Map<String, Object> valueMap = msgSerde.fromBytes((byte[]) msg.getMessage()); return new CoordinatorStreamMessage(keyArray, valueMap); }
@Test public void testBroadcast() { SystemStreamMetadata metadata = mock(SystemStreamMetadata.class); Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadata = new HashMap<>(); partitionMetadata.put(new Partition(0), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class)); partitionMetadata.put(new Partition(1), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class)); partitionMetadata.put(new Partition(2), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class)); partitionMetadata.put(new Partition(3), mock(SystemStreamMetadata.SystemStreamPartitionMetadata.class)); when(metadata.getSystemStreamPartitionMetadata()).thenReturn(partitionMetadata); StreamMetadataCache metadataCache = mock(StreamMetadataCache.class); when(metadataCache.getSystemStreamMetadata(anyObject(), anyBoolean())).thenReturn(metadata); SystemStream systemStream = new SystemStream("test-system", "test-stream"); Set<Integer> partitions = new HashSet<>(); MessageCollector collector = mock(MessageCollector.class); doAnswer(invocation -> { OutgoingMessageEnvelope envelope = (OutgoingMessageEnvelope) invocation.getArguments()[0]; partitions.add((Integer) envelope.getPartitionKey()); assertEquals(envelope.getSystemStream(), systemStream); return null; }).when(collector).send(any()); ControlMessageSender sender = new ControlMessageSender(metadataCache); WatermarkMessage watermark = new WatermarkMessage(System.currentTimeMillis(), "task 0"); SystemStreamPartition ssp = new SystemStreamPartition(systemStream, new Partition(0)); sender.broadcastToOtherPartitions(watermark, ssp, collector); assertEquals(partitions.size(), 3); } }
/** * {@inheritDoc} */ @Override public synchronized void send(String source, OutgoingMessageEnvelope envelope) { checkForSendCallbackErrors("Received exception on message send"); String streamName = envelope.getSystemStream().getStream(); String streamId = physicalToStreamIds.getOrDefault(streamName, streamName); long beforeSendTimeMs = System.currentTimeMillis(); CompletableFuture<Void> sendResult = sendAsync(source, envelope); long afterSendTimeMs = System.currentTimeMillis(); long latencyMs = afterSendTimeMs - beforeSendTimeMs; sendLatency.get(streamId).update(latencyMs); aggSendLatency.update(latencyMs); pendingFutures.add(sendResult); // Auto update the metrics and possible throwable when futures are complete. sendResult.handle((aVoid, throwable) -> { long callbackLatencyMs = System.currentTimeMillis() - afterSendTimeMs; sendCallbackLatency.get(streamId).update(callbackLatencyMs); aggSendCallbackLatency.update(callbackLatencyMs); if (throwable != null) { sendErrors.get(streamId).inc(); aggSendErrors.inc(); LOG.error("Send message to event hub: {} failed with exception: ", streamId, throwable); sendExceptionOnCallback.compareAndSet(null, throwable); } return aVoid; }); }
/** * Sends a specified message envelope from a specified Samza source. * @param source String representing the source of the message. * @param envelope Aggregate object representing the serialized message to send from the source. */ @Override public void send(String source, OutgoingMessageEnvelope envelope) { Object key = envelope.getKey(); Object message = envelope.getMessage(); Object partitionKey; // We use the partition key from message if available, if not fallback to message key or use message as partition // key as the final resort. if (envelope.getPartitionKey() != null) { partitionKey = envelope.getPartitionKey(); } else if (key != null) { partitionKey = key; } else { partitionKey = message; } Preconditions.checkNotNull(partitionKey, "Failed to compute partition key for the message: " + envelope); int partition = Math.abs(partitionKey.hashCode()) % memoryManager.getPartitionCountForSystemStream(envelope.getSystemStream()); SystemStreamPartition ssp = new SystemStreamPartition(envelope.getSystemStream(), new Partition(partition)); memoryManager.put(ssp, key, message); }
private void sendControlMessage(ControlMessage message, MessageCollector collector) { OutgoingMessageEnvelope envelopeOut = new OutgoingMessageEnvelope(systemStream, 0, null, message); collector.send(envelopeOut); } }
protected void setSource(OutgoingMessageEnvelope envelope, IndexRequest indexRequest) { Object message = envelope.getMessage(); if (message instanceof byte[]) { indexRequest.source((byte[]) message); } else if (message instanceof Map) { indexRequest.source((Map) message); } else { throw new SamzaException("Unsupported message type: " + message.getClass().getCanonicalName()); } }
public void send(String source, OutgoingMessageEnvelope envelope) { if (mockConsumer != null) { MockCoordinatorStreamWrappedConsumer consumer = (MockCoordinatorStreamWrappedConsumer) mockConsumer; SystemStreamPartition ssp = new SystemStreamPartition(envelope.getSystemStream(), new Partition(0)); consumer.register(ssp, ""); try { consumer.addMessageEnvelope(new IncomingMessageEnvelope(ssp, "", envelope.getKey(), envelope.getMessage())); } catch (IOException | InterruptedException e) { e.printStackTrace(); } } else { envelopes.add(envelope); } }
assertEquals(2, envelopes.size()); assertEquals("key0", deserialize((byte[]) envelope0.getKey(), keyRef)[CoordinatorStreamMessage.KEY_INDEX]); Map<String, String> values = (Map<String, String>) deserialize((byte[]) envelope0.getMessage(), msgRef).get("values"); assertEquals("value0", values.get("value")); assertEquals("key1", deserialize((byte[]) envelope1.getKey(), keyRef)[CoordinatorStreamMessage.KEY_INDEX]); values = (Map<String, String>) deserialize((byte[]) envelope1.getMessage(), msgRef).get("values"); assertEquals("value1", values.get("value"));