private String getCompressedFormat(OutgoingMessageEnvelope envelope) { return new String((byte[]) envelope.getMessage()); }
@Override public void send(String source, OutgoingMessageEnvelope envelope) { LOG.info(String.format(String.format("Message %d :", messageCounter.incrementAndGet()))); String msg = String.format("OutputStream:%s Key:%s Value:%s", envelope.getSystemStream(), envelope.getKey(), new String((byte[]) envelope.getMessage())); LOG.info(msg); SamzaExecutor.saveOutputMessage(envelope); }
private String getPrettyFormat(OutgoingMessageEnvelope envelope) { lastErrorMsg = ""; String value = new String((byte[]) envelope.getMessage()); ObjectMapper mapper = new ObjectMapper(); String formattedValue; try { Object json = mapper.readValue(value, Object.class); formattedValue = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(json); } catch (IOException ex) { formattedValue = value; String msg = "getPrettyFormat failed with exception while formatting json "; lastErrorMsg = msg + ex.toString(); LOG.error(msg, ex); } return formattedValue; }
protected void setSource(OutgoingMessageEnvelope envelope, IndexRequest indexRequest) { Object message = envelope.getMessage(); if (message instanceof byte[]) { indexRequest.source((byte[]) message); } else if (message instanceof Map) { indexRequest.source((Map) message); } else { throw new SamzaException("Unsupported message type: " + message.getClass().getCanonicalName()); } }
protected EventData createEventData(String streamId, OutgoingMessageEnvelope envelope) { Optional<Interceptor> interceptor = Optional.ofNullable(interceptors.getOrDefault(streamId, null)); byte[] eventValue = (byte[]) envelope.getMessage(); if (interceptor.isPresent()) { eventValue = interceptor.get().intercept(eventValue); } EventData eventData = new EventDataImpl(eventValue); eventData.getProperties().put(PRODUCE_TIMESTAMP, Long.toString(System.currentTimeMillis())); if (config.getSendKeyInEventProperties(systemName)) { String keyValue = ""; if (envelope.getKey() != null) { keyValue = (envelope.getKey() instanceof byte[]) ? new String((byte[]) envelope.getKey()) : envelope.getKey().toString(); } eventData.getProperties().put(KEY, keyValue); } return eventData; }
@Override public void send(String source, OutgoingMessageEnvelope envelope) { messagesReceived.add(envelope.getMessage()); listeners.forEach((listener) -> listener.onSend(source, envelope)); }
@Override public void send(String source, OutgoingMessageEnvelope envelope) { messagesReceived.add(envelope.getMessage()); listeners.forEach((listener) -> listener.onSend(source, envelope)); }
@Override public void send(String source, OutgoingMessageEnvelope envelope) { String msg = String.format("OutputStream:%s Key:%s Value:%s", envelope.getSystemStream(), envelope.getKey(), envelope.getMessage()); LOG.info(msg); System.out.println(msg); }
@Test public void joinReverse() throws Exception { StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction()); StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc); List<Integer> output = new ArrayList<>(); MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage()); // push messages to second stream numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator)); // push messages to first stream with same keys numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator)); int outputSum = output.stream().reduce(0, (s, m) -> s + m); assertEquals(110, outputSum); }
@Test public void join() throws Exception { StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction()); StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc); List<Integer> output = new ArrayList<>(); MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage()); // push messages to first stream numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator)); // push messages to second stream with same keys numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator)); int outputSum = output.stream().reduce(0, (s, m) -> s + m); assertEquals(110, outputSum); }
@Test public void joinNoMatch() throws Exception { StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction()); StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc); List<Integer> output = new ArrayList<>(); MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage()); // push messages to first stream numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator)); // push messages to second stream with different keys numbers.forEach(n -> sot.process(new SecondStreamIME(n + 100, n), messageCollector, taskCoordinator)); assertTrue(output.isEmpty()); }
@Test public void joinNoMatchReverse() throws Exception { StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction()); StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc); List<Integer> output = new ArrayList<>(); MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage()); // push messages to second stream numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator)); // push messages to first stream with different keys numbers.forEach(n -> sot.process(new FirstStreamIME(n + 100, n), messageCollector, taskCoordinator)); assertTrue(output.isEmpty()); }
@Test public void joinRetainsMatchedMessagesReverse() throws Exception { StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction()); StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc); List<Integer> output = new ArrayList<>(); MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage()); // push messages to first stream numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator)); // push messages to second stream with same key numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator)); int outputSum = output.stream().reduce(0, (s, m) -> s + m); assertEquals(110, outputSum); output.clear(); // push messages to second stream with same keys once again. numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator)); int newOutputSum = output.stream().reduce(0, (s, m) -> s + m); assertEquals(110, newOutputSum); // should produce the same output as before }
@Override public void send(String source, OutgoingMessageEnvelope envelope) { SystemStream systemStream = envelope.getSystemStream(); List<SystemStreamPartition> sspForSystem = MSG_QUEUES.keySet().stream() .filter(ssp -> ssp.getSystemStream().equals(systemStream)) .collect(ArrayList::new, (l, ssp) -> l.add(ssp), (l1, l2) -> l1.addAll(l2)); if (sspForSystem.isEmpty()) { MSG_QUEUES.putIfAbsent(new SystemStreamPartition(systemStream, new Partition(0)), new ArrayList<>()); sspForSystem.add(new SystemStreamPartition(systemStream, new Partition(0))); } int partitionCount = sspForSystem.size(); int partitionId = envelope.getPartitionKey() == null ? envelope.getKey() == null ? this.seed.nextInt(partitionCount) : envelope.getKey().hashCode() % partitionCount : envelope.getPartitionKey().hashCode() % partitionCount; SystemStreamPartition ssp = new SystemStreamPartition(envelope.getSystemStream(), new Partition(partitionId)); List<IncomingMessageEnvelope> msgQueue = MSG_QUEUES.get(ssp); msgQueue.add(new IncomingMessageEnvelope(ssp, null, envelope.getKey(), envelope.getMessage())); }
@Test public void joinRetainsLatestMessageForKeyReverse() throws Exception { StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction()); StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc); List<Integer> output = new ArrayList<>(); MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage()); // push messages to second stream numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator)); // push messages to second stream again with same keys but different values numbers.forEach(n -> sot.process(new SecondStreamIME(n, 2 * n), messageCollector, taskCoordinator)); // push messages to first stream with same key numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator)); int outputSum = output.stream().reduce(0, (s, m) -> s + m); assertEquals(165, outputSum); // should use latest messages in the second stream }
@Test public void joinRetainsLatestMessageForKey() throws Exception { StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction()); StreamOperatorTask sot = createStreamOperatorTask(new SystemClock(), streamAppDesc); List<Integer> output = new ArrayList<>(); MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage()); // push messages to first stream numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator)); // push messages to first stream again with same keys but different values numbers.forEach(n -> sot.process(new FirstStreamIME(n, 2 * n), messageCollector, taskCoordinator)); // push messages to second stream with same key numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator)); int outputSum = output.stream().reduce(0, (s, m) -> s + m); assertEquals(165, outputSum); // should use latest messages in the first stream }
public static CoordinatorStreamMessage deserializeCoordinatorStreamMessage(OutgoingMessageEnvelope msg) { JsonSerde<List<?>> keySerde = new JsonSerde<>(); Object[] keyArray = keySerde.fromBytes((byte[]) msg.getKey()).toArray(); JsonSerde<Map<String, Object>> msgSerde = new JsonSerde<>(); Map<String, Object> valueMap = msgSerde.fromBytes((byte[]) msg.getMessage()); return new CoordinatorStreamMessage(keyArray, valueMap); }
@Test public void joinRemovesExpiredMessagesReverse() throws Exception { TestClock testClock = new TestClock(); StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction()); StreamOperatorTask sot = createStreamOperatorTask(testClock, streamAppDesc); List<Integer> output = new ArrayList<>(); MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage()); // push messages to second stream numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator)); testClock.advanceTime(JOIN_TTL.plus(Duration.ofMinutes(1))); // 1 minute after ttl sot.window(messageCollector, taskCoordinator); // should expire second stream messages // push messages to first stream with same key numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator)); assertTrue(output.isEmpty()); }
@Test public void joinRemovesExpiredMessages() throws Exception { TestClock testClock = new TestClock(); StreamApplicationDescriptorImpl streamAppDesc = this.getTestJoinStreamGraph(new TestJoinFunction()); StreamOperatorTask sot = createStreamOperatorTask(testClock, streamAppDesc); List<Integer> output = new ArrayList<>(); MessageCollector messageCollector = envelope -> output.add((Integer) envelope.getMessage()); // push messages to first stream numbers.forEach(n -> sot.process(new FirstStreamIME(n, n), messageCollector, taskCoordinator)); testClock.advanceTime(JOIN_TTL.plus(Duration.ofMinutes(1))); // 1 minute after ttl sot.window(messageCollector, taskCoordinator); // should expire first stream messages // push messages to second stream with same key numbers.forEach(n -> sot.process(new SecondStreamIME(n, n), messageCollector, taskCoordinator)); assertTrue(output.isEmpty()); }
public void send(String source, OutgoingMessageEnvelope envelope) { if (mockConsumer != null) { MockCoordinatorStreamWrappedConsumer consumer = (MockCoordinatorStreamWrappedConsumer) mockConsumer; SystemStreamPartition ssp = new SystemStreamPartition(envelope.getSystemStream(), new Partition(0)); consumer.register(ssp, ""); try { consumer.addMessageEnvelope(new IncomingMessageEnvelope(ssp, "", envelope.getKey(), envelope.getMessage())); } catch (IOException | InterruptedException e) { e.printStackTrace(); } } else { envelopes.add(envelope); } }