@Override public Object apply(IncomingMessageEnvelope ime) { Assert.notNull(ime, "ime is null"); KV<Object, Object> keyAndMessageKV = KV.of(ime.getKey(), ime.getMessage()); SamzaSqlRelMsgMetadata metadata = new SamzaSqlRelMsgMetadata(Instant.ofEpochMilli(ime.getEventTime()).toString(), Instant.ofEpochMilli(ime.getArrivalTime()).toString(), null); SamzaSqlInputMessage samzaMsg = SamzaSqlInputMessage.of(keyAndMessageKV, metadata); return samzaMsg; } }
/** * Returns all the messages from the earliest offset all the way to the latest. */ private void bootstrapMessagesFromStream() { synchronized (bootstrapLock) { while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); byte[] keyAsBytes = (byte[]) envelope.getKey(); Object[] keyArray = keySerde.fromBytes(keyAsBytes).toArray(); CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, new HashMap<>()); if (Objects.equals(coordinatorStreamMessage.getType(), type)) { if (envelope.getMessage() != null) { bootstrappedMessages.put(keyAsBytes, (byte[]) envelope.getMessage()); } else { bootstrappedMessages.remove(keyAsBytes); } } } } }
/** * Returns all the messages from the earliest offset all the way to the latest. */ private void bootstrapMessagesFromStream() { synchronized (bootstrapLock) { while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); byte[] keyAsBytes = (byte[]) envelope.getKey(); Object[] keyArray = keySerde.fromBytes(keyAsBytes).toArray(); CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, new HashMap<>()); if (Objects.equals(coordinatorStreamMessage.getType(), type)) { if (envelope.getMessage() != null) { bootstrappedMessages.put(keyAsBytes, (byte[]) envelope.getMessage()); } else { bootstrappedMessages.remove(keyAsBytes); } } } } }
/** * Returns all the messages from the earliest offset all the way to the latest. */ private void bootstrapMessagesFromStream() { synchronized (bootstrapLock) { while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); byte[] keyAsBytes = (byte[]) envelope.getKey(); Object[] keyArray = keySerde.fromBytes(keyAsBytes).toArray(); CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, new HashMap<>()); if (Objects.equals(coordinatorStreamMessage.getType(), type)) { if (envelope.getMessage() != null) { bootstrappedMessages.put(keyAsBytes, (byte[]) envelope.getMessage()); } else { bootstrappedMessages.remove(keyAsBytes); } } } } }
/** * Returns all the messages from the earliest offset all the way to the latest. */ private void bootstrapMessagesFromStream() { synchronized (bootstrapLock) { while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); byte[] keyAsBytes = (byte[]) envelope.getKey(); Object[] keyArray = keySerde.fromBytes(keyAsBytes).toArray(); CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, new HashMap<>()); if (Objects.equals(coordinatorStreamMessage.getType(), type)) { if (envelope.getMessage() != null) { bootstrappedMessages.put(keyAsBytes, (byte[]) envelope.getMessage()); } else { bootstrappedMessages.remove(keyAsBytes); } } } } }
@Override public Collection<Object> handleMessage(IncomingMessageEnvelope ime, MessageCollector collector, TaskCoordinator coordinator) { Object message; InputTransformer transformer = inputOpSpec.getTransformer(); if (transformer != null) { message = transformer.apply(ime); } else { message = this.inputOpSpec.isKeyed() ? KV.of(ime.getKey(), ime.getMessage()) : ime.getMessage(); } if (message != null) { return Collections.singletonList(message); } return Collections.emptyList(); }
@Override public Collection<Object> handleMessage(IncomingMessageEnvelope ime, MessageCollector collector, TaskCoordinator coordinator) { Object message; InputTransformer transformer = inputOpSpec.getTransformer(); if (transformer != null) { message = transformer.apply(ime); } else { message = this.inputOpSpec.isKeyed() ? KV.of(ime.getKey(), ime.getMessage()) : ime.getMessage(); } return Collections.singletonList(message); }
@Override public Collection<Object> handleMessage(IncomingMessageEnvelope ime, MessageCollector collector, TaskCoordinator coordinator) { Object message; InputTransformer transformer = inputOpSpec.getTransformer(); if (transformer != null) { message = transformer.apply(ime); } else { message = this.inputOpSpec.isKeyed() ? KV.of(ime.getKey(), ime.getMessage()) : ime.getMessage(); } return Collections.singletonList(message); }
/** * Returns all the messages from the earliest offset all the way to the latest. */ private void bootstrapMessagesFromStream() { synchronized (bootstrapLock) { while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); byte[] keyAsBytes = (byte[]) envelope.getKey(); Serde<List<?>> serde = new JsonSerde<>(); Object[] keyArray = serde.fromBytes(keyAsBytes).toArray(); CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, new HashMap<>()); if (Objects.equals(coordinatorStreamMessage.getType(), type)) { if (envelope.getMessage() != null) { bootstrappedMessages.put(coordinatorStreamMessage.getKey(), (byte[]) envelope.getMessage()); } else { bootstrappedMessages.remove(coordinatorStreamMessage.getKey()); } } } } }
@Override public Collection<Object> handleMessage(IncomingMessageEnvelope ime, MessageCollector collector, TaskCoordinator coordinator) { Object message; InputTransformer transformer = inputOpSpec.getTransformer(); if (transformer != null) { message = transformer.apply(ime); } else { message = this.inputOpSpec.isKeyed() ? KV.of(ime.getKey(), ime.getMessage()) : ime.getMessage(); } return Collections.singletonList(message); }
@Override public Collection<Object> handleMessage(IncomingMessageEnvelope ime, MessageCollector collector, TaskCoordinator coordinator) { Object message; InputTransformer transformer = inputOpSpec.getTransformer(); if (transformer != null) { message = transformer.apply(ime); } else { message = this.inputOpSpec.isKeyed() ? KV.of(ime.getKey(), ime.getMessage()) : ime.getMessage(); } return Collections.singletonList(message); }
/** * returns all unread messages of a specific type, after an iterator on the stream * * @param iterator the iterator pointing to an offset in the coordinator stream. All unread messages after this iterator are returned * @param type the type of the messages to be returned * @return a set of unread messages of a given type, after a given iterator */ public Set<CoordinatorStreamMessage> getUnreadMessages(SystemStreamPartitionIterator iterator, String type) { LinkedHashSet<CoordinatorStreamMessage> messages = new LinkedHashSet<CoordinatorStreamMessage>(); while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); Object[] keyArray = keySerde.fromBytes((byte[]) envelope.getKey()).toArray(); Map<String, Object> valueMap = null; if (envelope.getMessage() != null) { valueMap = messageSerde.fromBytes((byte[]) envelope.getMessage()); } CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, valueMap); if (type == null || type.equals(coordinatorStreamMessage.getType())) { messages.add(coordinatorStreamMessage); } } return messages; }
/** * returns all unread messages of a specific type, after an iterator on the stream * * @param iterator the iterator pointing to an offset in the coordinator stream. All unread messages after this iterator are returned * @param type the type of the messages to be returned * @return a set of unread messages of a given type, after a given iterator */ public Set<CoordinatorStreamMessage> getUnreadMessages(SystemStreamPartitionIterator iterator, String type) { LinkedHashSet<CoordinatorStreamMessage> messages = new LinkedHashSet<CoordinatorStreamMessage>(); while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); Object[] keyArray = keySerde.fromBytes((byte[]) envelope.getKey()).toArray(); Map<String, Object> valueMap = null; if (envelope.getMessage() != null) { valueMap = messageSerde.fromBytes((byte[]) envelope.getMessage()); } CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, valueMap); if (type == null || type.equals(coordinatorStreamMessage.getType())) { messages.add(coordinatorStreamMessage); } } return messages; }
/** * returns all unread messages of a specific type, after an iterator on the stream * * @param iterator the iterator pointing to an offset in the coordinator stream. All unread messages after this iterator are returned * @param type the type of the messages to be returned * @return a set of unread messages of a given type, after a given iterator */ public Set<CoordinatorStreamMessage> getUnreadMessages(SystemStreamPartitionIterator iterator, String type) { LinkedHashSet<CoordinatorStreamMessage> messages = new LinkedHashSet<CoordinatorStreamMessage>(); while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); Object[] keyArray = keySerde.fromBytes((byte[]) envelope.getKey()).toArray(); Map<String, Object> valueMap = null; if (envelope.getMessage() != null) { valueMap = messageSerde.fromBytes((byte[]) envelope.getMessage()); } CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, valueMap); if (type == null || type.equals(coordinatorStreamMessage.getType())) { messages.add(coordinatorStreamMessage); } } return messages; }
/** * returns all unread messages of a specific type, after an iterator on the stream * * @param iterator the iterator pointing to an offset in the coordinator stream. All unread messages after this iterator are returned * @param type the type of the messages to be returned * @return a set of unread messages of a given type, after a given iterator */ public Set<CoordinatorStreamMessage> getUnreadMessages(SystemStreamPartitionIterator iterator, String type) { LinkedHashSet<CoordinatorStreamMessage> messages = new LinkedHashSet<CoordinatorStreamMessage>(); while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); Object[] keyArray = keySerde.fromBytes((byte[]) envelope.getKey()).toArray(); Map<String, Object> valueMap = null; if (envelope.getMessage() != null) { valueMap = messageSerde.fromBytes((byte[]) envelope.getMessage()); } CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, valueMap); if (type == null || type.equals(coordinatorStreamMessage.getType())) { messages.add(coordinatorStreamMessage); } } return messages; }
/** * returns all unread messages of a specific type, after an iterator on the stream * * @param iterator the iterator pointing to an offset in the coordinator stream. All unread messages after this iterator are returned * @param type the type of the messages to be returned * @return a set of unread messages of a given type, after a given iterator */ public Set<CoordinatorStreamMessage> getUnreadMessages(SystemStreamPartitionIterator iterator, String type) { LinkedHashSet<CoordinatorStreamMessage> messages = new LinkedHashSet<CoordinatorStreamMessage>(); while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); Object[] keyArray = keySerde.fromBytes((byte[]) envelope.getKey()).toArray(); Map<String, Object> valueMap = null; if (envelope.getMessage() != null) { valueMap = messageSerde.fromBytes((byte[]) envelope.getMessage()); } CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, valueMap); if (type == null || type.equals(coordinatorStreamMessage.getType())) { messages.add(coordinatorStreamMessage); } } return messages; }
public IncomingMessageEnvelope readNext() { if (!hasNext()) { LOG.warn("Attempting to read more data when there aren't any. ssp=" + systemStreamPartition); return null; } // record the next offset before we read, so when the read fails and we reconnect, // we seek to the same offset that we try below curSingleFileOffset = curReader.nextOffset(); IncomingMessageEnvelope messageEnvelope = curReader.readNext(); // Copy everything except for the offset. Turn the single-file style offset into a multi-file one return new IncomingMessageEnvelope(messageEnvelope.getSystemStreamPartition(), getCurOffset(), messageEnvelope.getKey(), messageEnvelope.getMessage(), messageEnvelope.getSize(), messageEnvelope.getEventTime(), messageEnvelope.getArrivalTime()); }
private void verifyEvents(List<IncomingMessageEnvelope> messages, List<EventData> eventDataList, Interceptor interceptor) { Assert.assertEquals(messages.size(), eventDataList.size()); for (int i = 0; i < messages.size(); i++) { IncomingMessageEnvelope message = messages.get(i); EventData eventData = eventDataList.get(i); Assert.assertEquals(message.getKey(), eventData.getSystemProperties().getPartitionKey()); Assert.assertEquals(message.getMessage(), interceptor.intercept(eventData.getBytes())); Assert.assertEquals(message.getOffset(), eventData.getSystemProperties().getOffset()); } }
@Test public void testNullMessageWithValidMessageKey() { final String messageKey = "validKey"; SystemProducer systemProducer = systemFactory.getProducer(SYSTEM_NAME, config, mockRegistry); systemProducer.send(SOURCE, new OutgoingMessageEnvelope(SYSTEM_STREAM, messageKey, null)); SystemConsumer consumer = systemFactory.getConsumer(SYSTEM_NAME, config, mockRegistry); Set<SystemStreamPartition> sspsToPoll = IntStream.range(0, PARTITION_COUNT) .mapToObj(partition -> new SystemStreamPartition(SYSTEM_STREAM, new Partition(partition))) .collect(Collectors.toSet()); // register the consumer for ssps for (SystemStreamPartition ssp : sspsToPoll) { consumer.register(ssp, "0"); } List<IncomingMessageEnvelope> results = consumeRawMessages(consumer, sspsToPoll); assertEquals(1, results.size()); assertEquals(results.get(0).getKey(), messageKey); assertNull(results.get(0).getMessage()); }
private void verifyRecords(List<IncomingMessageEnvelope> outputRecords, List<Record> inputRecords, String shardId) { Iterator outputRecordsIter = outputRecords.iterator(); inputRecords.forEach(record -> { IncomingMessageEnvelope envelope = (IncomingMessageEnvelope) outputRecordsIter.next(); String outputKey = (String) envelope.getKey(); KinesisIncomingMessageEnvelope kinesisMessageEnvelope = (KinesisIncomingMessageEnvelope) envelope; Assert.assertEquals(outputKey, record.getPartitionKey()); Assert.assertEquals(kinesisMessageEnvelope.getSequenceNumber(), record.getSequenceNumber()); Assert.assertEquals(kinesisMessageEnvelope.getApproximateArrivalTimestamp(), record.getApproximateArrivalTimestamp()); Assert.assertEquals(kinesisMessageEnvelope.getShardId(), shardId); ByteBuffer outputData = ByteBuffer.wrap((byte[]) kinesisMessageEnvelope.getMessage()); record.getData().rewind(); Assert.assertTrue(outputData.equals(record.getData())); verifyOffset(envelope.getOffset(), record, shardId); }); }