protected Optional<String> getId(OutgoingMessageEnvelope envelope) { Object id = envelope.getKey(); if (id == null) { return Optional.absent(); } return Optional.of(id.toString()); }
@Override public void send(String source, OutgoingMessageEnvelope envelope) { LOG.info(String.format(String.format("Message %d :", messageCounter.incrementAndGet()))); String msg = String.format("OutputStream:%s Key:%s Value:%s", envelope.getSystemStream(), envelope.getKey(), new String((byte[]) envelope.getMessage())); LOG.info(msg); SamzaExecutor.saveOutputMessage(envelope); }
protected EventData createEventData(String streamId, OutgoingMessageEnvelope envelope) { Optional<Interceptor> interceptor = Optional.ofNullable(interceptors.getOrDefault(streamId, null)); byte[] eventValue = (byte[]) envelope.getMessage(); if (interceptor.isPresent()) { eventValue = interceptor.get().intercept(eventValue); } EventData eventData = new EventDataImpl(eventValue); eventData.getProperties().put(PRODUCE_TIMESTAMP, Long.toString(System.currentTimeMillis())); if (config.getSendKeyInEventProperties(systemName)) { String keyValue = ""; if (envelope.getKey() != null) { keyValue = (envelope.getKey() instanceof byte[]) ? new String((byte[]) envelope.getKey()) : envelope.getKey().toString(); } eventData.getProperties().put(KEY, keyValue); } return eventData; }
protected Object getEnvelopePartitionId(OutgoingMessageEnvelope envelope) { return envelope.getPartitionKey() == null ? envelope.getKey() : envelope.getPartitionKey(); }
@Override public void send(String source, OutgoingMessageEnvelope envelope) { String msg = String.format("OutputStream:%s Key:%s Value:%s", envelope.getSystemStream(), envelope.getKey(), envelope.getMessage()); LOG.info(msg); System.out.println(msg); }
/** * Sends a specified message envelope from a specified Samza source. * @param source String representing the source of the message. * @param envelope Aggregate object representing the serialized message to send from the source. */ @Override public void send(String source, OutgoingMessageEnvelope envelope) { Object key = envelope.getKey(); Object message = envelope.getMessage(); Object partitionKey; // We use the partition key from message if available, if not fallback to message key or use message as partition // key as the final resort. if (envelope.getPartitionKey() != null) { partitionKey = envelope.getPartitionKey(); } else if (key != null) { partitionKey = key; } else { partitionKey = message; } Preconditions.checkNotNull(partitionKey, "Failed to compute partition key for the message: " + envelope); int partition = Math.abs(partitionKey.hashCode()) % memoryManager.getPartitionCountForSystemStream(envelope.getSystemStream()); SystemStreamPartition ssp = new SystemStreamPartition(envelope.getSystemStream(), new Partition(partition)); memoryManager.put(ssp, key, message); }
@Override public void send(String source, OutgoingMessageEnvelope envelope) { SystemStream systemStream = envelope.getSystemStream(); List<SystemStreamPartition> sspForSystem = MSG_QUEUES.keySet().stream() .filter(ssp -> ssp.getSystemStream().equals(systemStream)) .collect(ArrayList::new, (l, ssp) -> l.add(ssp), (l1, l2) -> l1.addAll(l2)); if (sspForSystem.isEmpty()) { MSG_QUEUES.putIfAbsent(new SystemStreamPartition(systemStream, new Partition(0)), new ArrayList<>()); sspForSystem.add(new SystemStreamPartition(systemStream, new Partition(0))); } int partitionCount = sspForSystem.size(); int partitionId = envelope.getPartitionKey() == null ? envelope.getKey() == null ? this.seed.nextInt(partitionCount) : envelope.getKey().hashCode() % partitionCount : envelope.getPartitionKey().hashCode() % partitionCount; SystemStreamPartition ssp = new SystemStreamPartition(envelope.getSystemStream(), new Partition(partitionId)); List<IncomingMessageEnvelope> msgQueue = MSG_QUEUES.get(ssp); msgQueue.add(new IncomingMessageEnvelope(ssp, null, envelope.getKey(), envelope.getMessage())); }
/** * Sends a specified message envelope from a specified Samza source. * @param source String representing the source of the message. * @param envelope Aggregate object representing the serialized message to send from the source. */ @Override public void send(String source, OutgoingMessageEnvelope envelope) { Object key = envelope.getKey(); Object message = envelope.getMessage(); Object partitionKey; // We use the partition key from message if available, if not fallback to message key or use message as partition // key as the final resort. if (envelope.getPartitionKey() != null) { partitionKey = envelope.getPartitionKey(); } else if (key != null) { partitionKey = key; } else { partitionKey = message; } Preconditions.checkNotNull(partitionKey, "Failed to compute partition key for the message: " + envelope); int partition = Math.abs(partitionKey.hashCode()) % memoryManager.getPartitionCountForSystemStream(envelope.getSystemStream()); SystemStreamPartition ssp = new SystemStreamPartition(envelope.getSystemStream(), new Partition(partition)); memoryManager.put(ssp, key, message); }
/** * Sends a specified message envelope from a specified Samza source. * @param source String representing the source of the message. * @param envelope Aggregate object representing the serialized message to send from the source. */ @Override public void send(String source, OutgoingMessageEnvelope envelope) { Object key = envelope.getKey(); Object message = envelope.getMessage(); Object partitionKey; // We use the partition key from message if available, if not fallback to message key or use message as partition // key as the final resort. if (envelope.getPartitionKey() != null) { partitionKey = envelope.getPartitionKey(); } else if (key != null) { partitionKey = key; } else { partitionKey = message; } Preconditions.checkNotNull(partitionKey, "Failed to compute partition key for the message: " + envelope); int partition = Math.abs(partitionKey.hashCode()) % memoryManager.getPartitionCountForSystemStream(envelope.getSystemStream()); SystemStreamPartition ssp = new SystemStreamPartition(envelope.getSystemStream(), new Partition(partition)); memoryManager.put(ssp, key, message); }
/** * Sends a specified message envelope from a specified Samza source. * @param source String representing the source of the message. * @param envelope Aggregate object representing the serialized message to send from the source. */ @Override public void send(String source, OutgoingMessageEnvelope envelope) { Object key = envelope.getKey(); Object message = envelope.getMessage(); Object partitionKey; // We use the partition key from message if available, if not fallback to message key or use message as partition // key as the final resort. if (envelope.getPartitionKey() != null) { partitionKey = envelope.getPartitionKey(); } else if (key != null) { partitionKey = key; } else { partitionKey = message; } Preconditions.checkNotNull(partitionKey, "Failed to compute partition key for the message: " + envelope); int partition = Math.abs(partitionKey.hashCode()) % memoryManager.getPartitionCountForSystemStream(envelope.getSystemStream()); SystemStreamPartition ssp = new SystemStreamPartition(envelope.getSystemStream(), new Partition(partition)); memoryManager.put(ssp, key, message); }
/** * Sends a specified message envelope from a specified Samza source. * @param source String representing the source of the message. * @param envelope Aggregate object representing the serialized message to send from the source. */ @Override public void send(String source, OutgoingMessageEnvelope envelope) { Object key = envelope.getKey(); Object message = envelope.getMessage(); Object partitionKey; // We use the partition key from message if available, if not fallback to message key or use message as partition // key as the final resort. if (envelope.getPartitionKey() != null) { partitionKey = envelope.getPartitionKey(); } else if (key != null) { partitionKey = key; } else { partitionKey = message; } Preconditions.checkNotNull(partitionKey, "Failed to compute partition key for the message: " + envelope); int partition = Math.abs(partitionKey.hashCode()) % memoryManager.getPartitionCountForSystemStream(envelope.getSystemStream()); SystemStreamPartition ssp = new SystemStreamPartition(envelope.getSystemStream(), new Partition(partition)); memoryManager.put(ssp, key, message); }
public static CoordinatorStreamMessage deserializeCoordinatorStreamMessage(OutgoingMessageEnvelope msg) { JsonSerde<List<?>> keySerde = new JsonSerde<>(); Object[] keyArray = keySerde.fromBytes((byte[]) msg.getKey()).toArray(); JsonSerde<Map<String, Object>> msgSerde = new JsonSerde<>(); Map<String, Object> valueMap = msgSerde.fromBytes((byte[]) msg.getMessage()); return new CoordinatorStreamMessage(keyArray, valueMap); }
public void send(String source, OutgoingMessageEnvelope envelope) { if (mockConsumer != null) { MockCoordinatorStreamWrappedConsumer consumer = (MockCoordinatorStreamWrappedConsumer) mockConsumer; SystemStreamPartition ssp = new SystemStreamPartition(envelope.getSystemStream(), new Partition(0)); consumer.register(ssp, ""); try { consumer.addMessageEnvelope(new IncomingMessageEnvelope(ssp, "", envelope.getKey(), envelope.getMessage())); } catch (IOException | InterruptedException e) { e.printStackTrace(); } } else { envelopes.add(envelope); } }
assertEquals(2, envelopes.size()); assertEquals("key0", deserialize((byte[]) envelope0.getKey(), keyRef)[CoordinatorStreamMessage.KEY_INDEX]); Map<String, String> values = (Map<String, String>) deserialize((byte[]) envelope0.getMessage(), msgRef).get("values"); assertEquals("value0", values.get("value")); assertEquals("key1", deserialize((byte[]) envelope1.getKey(), keyRef)[CoordinatorStreamMessage.KEY_INDEX]); values = (Map<String, String>) deserialize((byte[]) envelope1.getMessage(), msgRef).get("values"); assertEquals("value1", values.get("value"));
}; assertEquals(3, envelopes.size()); assertEquals(new CoordinatorStreamMessage(setConfig1), new CoordinatorStreamMessage(deserialize((byte[]) envelope0.getKey(), keyRef), deserialize((byte[]) envelope0.getMessage(), msgRef))); assertEquals(new CoordinatorStreamMessage(setConfig2), new CoordinatorStreamMessage(deserialize((byte[]) envelope1.getKey(), keyRef), deserialize((byte[]) envelope1.getMessage(), msgRef))); assertEquals(new CoordinatorStreamMessage(delete), new CoordinatorStreamMessage(deserialize((byte[]) envelope2.getKey(), keyRef), deserialize((byte[]) envelope2.getMessage(), msgRef)));