@Override public Collection<Object> handleMessage(IncomingMessageEnvelope ime, MessageCollector collector, TaskCoordinator coordinator) { Object message; InputTransformer transformer = inputOpSpec.getTransformer(); if (transformer != null) { message = transformer.apply(ime); } else { message = this.inputOpSpec.isKeyed() ? KV.of(ime.getKey(), ime.getMessage()) : ime.getMessage(); } return Collections.singletonList(message); }
@Override public Collection<Object> handleMessage(IncomingMessageEnvelope ime, MessageCollector collector, TaskCoordinator coordinator) { Object message; InputTransformer transformer = inputOpSpec.getTransformer(); if (transformer != null) { message = transformer.apply(ime); } else { message = this.inputOpSpec.isKeyed() ? KV.of(ime.getKey(), ime.getMessage()) : ime.getMessage(); } if (message != null) { return Collections.singletonList(message); } return Collections.emptyList(); }
@Override public Collection<Object> handleMessage(IncomingMessageEnvelope ime, MessageCollector collector, TaskCoordinator coordinator) { Object message; InputTransformer transformer = inputOpSpec.getTransformer(); if (transformer != null) { message = transformer.apply(ime); } else { message = this.inputOpSpec.isKeyed() ? KV.of(ime.getKey(), ime.getMessage()) : ime.getMessage(); } return Collections.singletonList(message); }
@Override public Collection<Object> handleMessage(IncomingMessageEnvelope ime, MessageCollector collector, TaskCoordinator coordinator) { Object message; InputTransformer transformer = inputOpSpec.getTransformer(); if (transformer != null) { message = transformer.apply(ime); } else { message = this.inputOpSpec.isKeyed() ? KV.of(ime.getKey(), ime.getMessage()) : ime.getMessage(); } return Collections.singletonList(message); }
@Override public Collection<Object> handleMessage(IncomingMessageEnvelope ime, MessageCollector collector, TaskCoordinator coordinator) { Object message; InputTransformer transformer = inputOpSpec.getTransformer(); if (transformer != null) { message = transformer.apply(ime); } else { message = this.inputOpSpec.isKeyed() ? KV.of(ime.getKey(), ime.getMessage()) : ime.getMessage(); } return Collections.singletonList(message); }
/** * Returns all the messages from the earliest offset all the way to the latest. */ private void bootstrapMessagesFromStream() { synchronized (bootstrapLock) { while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); byte[] keyAsBytes = (byte[]) envelope.getKey(); Object[] keyArray = keySerde.fromBytes(keyAsBytes).toArray(); CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, new HashMap<>()); if (Objects.equals(coordinatorStreamMessage.getType(), type)) { if (envelope.getMessage() != null) { bootstrappedMessages.put(keyAsBytes, (byte[]) envelope.getMessage()); } else { bootstrappedMessages.remove(keyAsBytes); } } } } }
/** * Returns all the messages from the earliest offset all the way to the latest. */ private void bootstrapMessagesFromStream() { synchronized (bootstrapLock) { while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); byte[] keyAsBytes = (byte[]) envelope.getKey(); Object[] keyArray = keySerde.fromBytes(keyAsBytes).toArray(); CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, new HashMap<>()); if (Objects.equals(coordinatorStreamMessage.getType(), type)) { if (envelope.getMessage() != null) { bootstrappedMessages.put(keyAsBytes, (byte[]) envelope.getMessage()); } else { bootstrappedMessages.remove(keyAsBytes); } } } } }
/** * Returns all the messages from the earliest offset all the way to the latest. */ private void bootstrapMessagesFromStream() { synchronized (bootstrapLock) { while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); byte[] keyAsBytes = (byte[]) envelope.getKey(); Object[] keyArray = keySerde.fromBytes(keyAsBytes).toArray(); CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, new HashMap<>()); if (Objects.equals(coordinatorStreamMessage.getType(), type)) { if (envelope.getMessage() != null) { bootstrappedMessages.put(keyAsBytes, (byte[]) envelope.getMessage()); } else { bootstrappedMessages.remove(keyAsBytes); } } } } }
@Override public Object apply(IncomingMessageEnvelope ime) { Assert.notNull(ime, "ime is null"); KV<Object, Object> keyAndMessageKV = KV.of(ime.getKey(), ime.getMessage()); SamzaSqlRelMsgMetadata metadata = new SamzaSqlRelMsgMetadata(Instant.ofEpochMilli(ime.getEventTime()).toString(), Instant.ofEpochMilli(ime.getArrivalTime()).toString(), null); SamzaSqlInputMessage samzaMsg = SamzaSqlInputMessage.of(keyAndMessageKV, metadata); return samzaMsg; } }
/** * returns all unread messages of a specific type, after an iterator on the stream * * @param iterator the iterator pointing to an offset in the coordinator stream. All unread messages after this iterator are returned * @param type the type of the messages to be returned * @return a set of unread messages of a given type, after a given iterator */ public Set<CoordinatorStreamMessage> getUnreadMessages(SystemStreamPartitionIterator iterator, String type) { LinkedHashSet<CoordinatorStreamMessage> messages = new LinkedHashSet<CoordinatorStreamMessage>(); while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); Object[] keyArray = keySerde.fromBytes((byte[]) envelope.getKey()).toArray(); Map<String, Object> valueMap = null; if (envelope.getMessage() != null) { valueMap = messageSerde.fromBytes((byte[]) envelope.getMessage()); } CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, valueMap); if (type == null || type.equals(coordinatorStreamMessage.getType())) { messages.add(coordinatorStreamMessage); } } return messages; }
/** * Returns all the messages from the earliest offset all the way to the latest. */ private void bootstrapMessagesFromStream() { synchronized (bootstrapLock) { while (iterator.hasNext()) { IncomingMessageEnvelope envelope = iterator.next(); byte[] keyAsBytes = (byte[]) envelope.getKey(); Serde<List<?>> serde = new JsonSerde<>(); Object[] keyArray = serde.fromBytes(keyAsBytes).toArray(); CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, new HashMap<>()); if (Objects.equals(coordinatorStreamMessage.getType(), type)) { if (envelope.getMessage() != null) { bootstrappedMessages.put(coordinatorStreamMessage.getKey(), (byte[]) envelope.getMessage()); } else { bootstrappedMessages.remove(coordinatorStreamMessage.getKey()); } } } } }
public IncomingMessageEnvelope readNext() { if (!hasNext()) { LOG.warn("Attempting to read more data when there aren't any. ssp=" + systemStreamPartition); return null; } // record the next offset before we read, so when the read fails and we reconnect, // we seek to the same offset that we try below curSingleFileOffset = curReader.nextOffset(); IncomingMessageEnvelope messageEnvelope = curReader.readNext(); // Copy everything except for the offset. Turn the single-file style offset into a multi-file one return new IncomingMessageEnvelope(messageEnvelope.getSystemStreamPartition(), getCurOffset(), messageEnvelope.getKey(), messageEnvelope.getMessage(), messageEnvelope.getSize(), messageEnvelope.getEventTime(), messageEnvelope.getArrivalTime()); }
private void verifyEvents(List<IncomingMessageEnvelope> messages, List<EventData> eventDataList, Interceptor interceptor) { Assert.assertEquals(messages.size(), eventDataList.size()); for (int i = 0; i < messages.size(); i++) { IncomingMessageEnvelope message = messages.get(i); EventData eventData = eventDataList.get(i); Assert.assertEquals(message.getKey(), eventData.getSystemProperties().getPartitionKey()); Assert.assertEquals(message.getMessage(), interceptor.intercept(eventData.getBytes())); Assert.assertEquals(message.getOffset(), eventData.getSystemProperties().getOffset()); } }
@Test public void testNullMessageWithValidMessageKey() { final String messageKey = "validKey"; SystemProducer systemProducer = systemFactory.getProducer(SYSTEM_NAME, config, mockRegistry); systemProducer.send(SOURCE, new OutgoingMessageEnvelope(SYSTEM_STREAM, messageKey, null)); SystemConsumer consumer = systemFactory.getConsumer(SYSTEM_NAME, config, mockRegistry); Set<SystemStreamPartition> sspsToPoll = IntStream.range(0, PARTITION_COUNT) .mapToObj(partition -> new SystemStreamPartition(SYSTEM_STREAM, new Partition(partition))) .collect(Collectors.toSet()); // register the consumer for ssps for (SystemStreamPartition ssp : sspsToPoll) { consumer.register(ssp, "0"); } List<IncomingMessageEnvelope> results = consumeRawMessages(consumer, sspsToPoll); assertEquals(1, results.size()); assertEquals(results.get(0).getKey(), messageKey); assertNull(results.get(0).getMessage()); }
@Test public void testSequentialRead() throws Exception { SystemStreamPartition ssp = new SystemStreamPartition("hdfs", "testStream", new Partition(0)); MultiFileHdfsReader multiReader = new MultiFileHdfsReader(HdfsReaderFactory.ReaderType.AVRO, ssp, Arrays.asList(descriptors), "0:0"); int index = 0; while (multiReader.hasNext()) { GenericRecord record = (GenericRecord) multiReader.readNext().getMessage(); Assert.assertEquals(index % NUM_EVENTS, record.get(FIELD_1)); Assert.assertEquals("string_" + (index % NUM_EVENTS), record.get(FIELD_2).toString()); index++; } Assert.assertEquals(3 * NUM_EVENTS, index); multiReader.close(); }
@Override public void process( IncomingMessageEnvelope incomingMessageEnvelope, MessageCollector messageCollector, TaskCoordinator taskCoordinator) throws Exception { messageCollector.send( new OutgoingMessageEnvelope( new SystemStream(outputSystem, outputTopic), incomingMessageEnvelope.getMessage())); processedMessageCount++; if (processedMessageCount == expectedMessageCount) { taskCoordinator.shutdown(TaskCoordinator.RequestScope.ALL_TASKS_IN_CONTAINER); } } }
@Test public void testFileReopen() throws Exception { SystemStreamPartition ssp = new SystemStreamPartition("hdfs", "testStream", new Partition(0)); SingleFileHdfsReader reader = new AvroFileHdfsReader(ssp); reader.open(AVRO_FILE, "0"); int index = 0; for (;index < NUM_EVENTS / 2; index++) { GenericRecord record = (GenericRecord) reader.readNext().getMessage(); Assert.assertEquals(index, record.get(FIELD_1)); Assert.assertEquals("string_" + index, record.get(FIELD_2).toString()); } String offset = reader.nextOffset(); reader.close(); reader = new AvroFileHdfsReader(ssp); reader.open(AVRO_FILE, offset); for (;index < NUM_EVENTS; index++) { GenericRecord record = (GenericRecord) reader.readNext().getMessage(); Assert.assertEquals(index, record.get(FIELD_1)); Assert.assertEquals("string_" + index, record.get(FIELD_2).toString()); } Assert.assertEquals(NUM_EVENTS, index); reader.close(); }
@Test public void testSequentialRead() throws Exception { SystemStreamPartition ssp = new SystemStreamPartition("hdfs", "testStream", new Partition(0)); SingleFileHdfsReader reader = new AvroFileHdfsReader(ssp); reader.open(AVRO_FILE, "0"); int index = 0; while (reader.hasNext()) { GenericRecord record = (GenericRecord) reader.readNext().getMessage(); Assert.assertEquals(index, record.get(FIELD_1)); Assert.assertEquals("string_" + index, record.get(FIELD_2).toString()); index++; } Assert.assertEquals(NUM_EVENTS, index); reader.close(); }
@Test public void testHasNextShouldWork() { int numMessages = 10; MockSystemConsumer consumer = new MockSystemConsumer(numMessages); SystemStreamPartitionIterator iterator = new SystemStreamPartitionIterator(consumer, SSP); while (iterator.hasNext()) { assertEquals(--numMessages, iterator.next().getMessage()); } assertFalse(iterator.hasNext()); assertEquals(0, numMessages); }
@Test public void testNextWithoutHasNextCallShouldWorkWhenAvailableAndFailWhenNot() { int numMessages = 10; MockSystemConsumer consumer = new MockSystemConsumer(numMessages); SystemStreamPartitionIterator iterator = new SystemStreamPartitionIterator(consumer, SSP); for (int i = 0; i < numMessages; ++i) { assertEquals(numMessages - i - 1, iterator.next().getMessage()); } assertFalse(iterator.hasNext()); try { iterator.next(); fail("Expected not to get any more messages from iterator."); } catch (NoSuchElementException e) { // expected } }