/** * Handles the produce request from {@link InMemorySystemProducer} and populates the underlying message queue. * * @param ssp system stream partition * @param key key for message produced * @param message actual payload */ void put(SystemStreamPartition ssp, Object key, Object message) { List<IncomingMessageEnvelope> messages = bufferedMessages.get(ssp); String offset = String.valueOf(messages.size()); if (message instanceof EndOfStreamMessage) { offset = IncomingMessageEnvelope.END_OF_STREAM_OFFSET; } IncomingMessageEnvelope messageEnvelope = new IncomingMessageEnvelope(ssp, offset, key, message); bufferedMessages.get(ssp) .add(messageEnvelope); }
/** * Handles the produce request from {@link InMemorySystemProducer} and populates the underlying message queue. * * @param ssp system stream partition * @param key key for message produced * @param message actual payload */ void put(SystemStreamPartition ssp, Object key, Object message) { List<IncomingMessageEnvelope> messages = bufferedMessages.get(ssp); String offset = String.valueOf(messages.size()); if (message instanceof EndOfStreamMessage) { offset = IncomingMessageEnvelope.END_OF_STREAM_OFFSET; } IncomingMessageEnvelope messageEnvelope = new IncomingMessageEnvelope(ssp, offset, key, message); bufferedMessages.get(ssp) .add(messageEnvelope); }
/** * Handles the produce request from {@link InMemorySystemProducer} and populates the underlying message queue. * * @param ssp system stream partition * @param key key for message produced * @param message actual payload */ void put(SystemStreamPartition ssp, Object key, Object message) { List<IncomingMessageEnvelope> messages = bufferedMessages.get(ssp); String offset = String.valueOf(messages.size()); if (message instanceof EndOfStreamMessage) { offset = IncomingMessageEnvelope.END_OF_STREAM_OFFSET; } IncomingMessageEnvelope messageEnvelope = new IncomingMessageEnvelope(ssp, offset, key, message, 0, 0L, Instant.now().toEpochMilli()); bufferedMessages.get(ssp) .add(messageEnvelope); }
/** * Handles the produce request from {@link InMemorySystemProducer} and populates the underlying message queue. * * @param ssp system stream partition * @param key key for message produced * @param message actual payload */ void put(SystemStreamPartition ssp, Object key, Object message) { List<IncomingMessageEnvelope> messages = bufferedMessages.get(ssp); String offset = String.valueOf(messages.size()); if (message instanceof EndOfStreamMessage) { offset = IncomingMessageEnvelope.END_OF_STREAM_OFFSET; } IncomingMessageEnvelope messageEnvelope = new IncomingMessageEnvelope(ssp, offset, key, message); bufferedMessages.get(ssp) .add(messageEnvelope); }
/** * This method is deprecated in favor of WatermarkManager.buildEndOfStreamEnvelope(SystemStreamPartition ssp). * * @param ssp The SSP that is at end-of-stream. * @return an IncomingMessageEnvelope corresponding to end-of-stream for that SSP. */ public static IncomingMessageEnvelope buildEndOfStreamEnvelope(SystemStreamPartition ssp) { return new IncomingMessageEnvelope(ssp, END_OF_STREAM_OFFSET, null, new EndOfStreamMessage(null)); }
/** * This method is deprecated in favor of WatermarkManager.buildEndOfStreamEnvelope(SystemStreamPartition ssp). * * @param ssp The SSP that is at end-of-stream. * @return an IncomingMessageEnvelope corresponding to end-of-stream for that SSP. */ public static IncomingMessageEnvelope buildEndOfStreamEnvelope(SystemStreamPartition ssp) { return new IncomingMessageEnvelope(ssp, END_OF_STREAM_OFFSET, null, new EndOfStreamMessage(null)); }
public static IncomingMessageEnvelope buildWatermarkEnvelope(SystemStreamPartition ssp, long watermark) { return new IncomingMessageEnvelope(ssp, null, null, new WatermarkMessage(watermark, null)); }
public static IncomingMessageEnvelope buildWatermarkEnvelope(SystemStreamPartition ssp, long watermark) { return new IncomingMessageEnvelope(ssp, null, null, new WatermarkMessage(watermark, null)); }
@Override public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll(Set<SystemStreamPartition> systemStreamPartitions, long timeout) { Map<SystemStreamPartition, List<IncomingMessageEnvelope>> systemStreamPartitionEnvelopes = new HashMap<SystemStreamPartition, List<IncomingMessageEnvelope>>(); for (SystemStreamPartition systemStreamPartition : systemStreamPartitions) { List<IncomingMessageEnvelope> q = new ArrayList<IncomingMessageEnvelope>(); if (numPollReturnsWithMessages-- > 0) { q.add(new IncomingMessageEnvelope(SSP, "", null, numPollReturnsWithMessages)); } systemStreamPartitionEnvelopes.put(systemStreamPartition, q); } return systemStreamPartitionEnvelopes; } }
@Override public IncomingMessageEnvelope readNext() { // get checkpoint for THIS record String checkpoint = nextOffset(); GenericRecord record = fileReader.next(); if (fileReader.previousSync() != curBlockStart) { curBlockStart = fileReader.previousSync(); curRecordOffset = 0; } else { curRecordOffset++; } // avro schema doesn't necessarily have key field return new IncomingMessageEnvelope(systemStreamPartition, checkpoint, null, record); }
@Test public void testWithInputTransformer() { InputOperatorSpec inputOpSpec = new InputOperatorSpec("stream-id", null, null, IncomingMessageEnvelope::getOffset, true, "input-op-id"); InputOperatorImpl inputOperator = new InputOperatorImpl(inputOpSpec); IncomingMessageEnvelope ime = new IncomingMessageEnvelope(mock(SystemStreamPartition.class), "123", "key", "msg"); Collection<Object> results = inputOperator.handleMessage(ime, mock(MessageCollector.class), mock(TaskCoordinator.class)); Object result = results.iterator().next(); assertEquals("123", result); }
private IncomingMessageEnvelope createIncomingMessageEnvelope(CoordinatorStreamMessage message, SystemStreamPartition ssp) { try { byte[] key = SamzaObjectMapper.getObjectMapper().writeValueAsString(message.getKeyArray()).getBytes("UTF-8"); byte[] value = SamzaObjectMapper.getObjectMapper().writeValueAsString(message.getMessageMap()).getBytes("UTF-8"); return new IncomingMessageEnvelope(ssp, null, key, value); } catch (Exception e) { return null; } } }
@Test public void testWithFilteringInputTransformer() { InputOperatorSpec inputOpSpec = new InputOperatorSpec("stream-id", null, null, (ime) -> null, true, "input-op-id"); InputOperatorImpl inputOperator = new InputOperatorImpl(inputOpSpec); IncomingMessageEnvelope ime = new IncomingMessageEnvelope(mock(SystemStreamPartition.class), "123", "key", "msg"); Collection<Object> results = inputOperator.handleMessage(ime, mock(MessageCollector.class), mock(TaskCoordinator.class)); assertTrue("Transformer doesn't return any record. Expected an empty collection", results.isEmpty()); } }
@Test public void testWithUnkeyedInput() { InputOperatorImpl inputOperator = new InputOperatorImpl(new InputOperatorSpec("stream-id", null, null, null, false, "input-op-id")); IncomingMessageEnvelope ime = new IncomingMessageEnvelope(mock(SystemStreamPartition.class), "123", "key", "msg"); Collection<Object> results = inputOperator.handleMessage(ime, mock(MessageCollector.class), mock(TaskCoordinator.class)); Object result = results.iterator().next(); assertEquals("msg", result); }
public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll(Set<SystemStreamPartition> systemStreamPartitions, long timeout) throws InterruptedException { Map<SystemStreamPartition, List<IncomingMessageEnvelope>> map = new LinkedHashMap<SystemStreamPartition, List<IncomingMessageEnvelope>>(); assertEquals(1, systemStreamPartitions.size()); SystemStreamPartition systemStreamPartition = systemStreamPartitions.iterator().next(); assertEquals(expectedSystemStreamPartition, systemStreamPartition); if (pollCount++ == 0) { List<IncomingMessageEnvelope> list = new ArrayList<IncomingMessageEnvelope>(); SetConfig setConfig1 = new SetConfig("test", "job.name", "my-job-name"); SetConfig setConfig2 = new SetConfig("test", "job.id", "1234"); Delete delete = new Delete("test", "job.name", SetConfig.TYPE); list.add(new IncomingMessageEnvelope(systemStreamPartition, null, serialize(setConfig1.getKeyArray()), serialize(setConfig1.getMessageMap()))); list.add(new IncomingMessageEnvelope(systemStreamPartition, null, serialize(setConfig2.getKeyArray()), serialize(setConfig2.getMessageMap()))); list.add(new IncomingMessageEnvelope(systemStreamPartition, null, serialize(delete.getKeyArray()), delete.getMessageMap())); map.put(systemStreamPartition, list); } return map; }
@Test public void testWithKeyedInput() { InputOperatorImpl inputOperator = new InputOperatorImpl(new InputOperatorSpec("stream-id", null, null, null, true, "input-op-id")); IncomingMessageEnvelope ime = new IncomingMessageEnvelope(mock(SystemStreamPartition.class), "123", "key", "msg"); Collection<Object> results = inputOperator.handleMessage(ime, mock(MessageCollector.class), mock(TaskCoordinator.class)); Object result = results.iterator().next(); assertEquals("key", ((KV) result).getKey()); assertEquals("msg", ((KV) result).getValue()); }
public IncomingMessageEnvelope readNext() { if (!hasNext()) { LOG.warn("Attempting to read more data when there aren't any. ssp=" + systemStreamPartition); return null; } // record the next offset before we read, so when the read fails and we reconnect, // we seek to the same offset that we try below curSingleFileOffset = curReader.nextOffset(); IncomingMessageEnvelope messageEnvelope = curReader.readNext(); // Copy everything except for the offset. Turn the single-file style offset into a multi-file one return new IncomingMessageEnvelope(messageEnvelope.getSystemStreamPartition(), getCurOffset(), messageEnvelope.getKey(), messageEnvelope.getMessage(), messageEnvelope.getSize(), messageEnvelope.getEventTime(), messageEnvelope.getArrivalTime()); }
@Override public void send(String source, OutgoingMessageEnvelope envelope) { SystemStream systemStream = envelope.getSystemStream(); List<SystemStreamPartition> sspForSystem = MSG_QUEUES.keySet().stream() .filter(ssp -> ssp.getSystemStream().equals(systemStream)) .collect(ArrayList::new, (l, ssp) -> l.add(ssp), (l1, l2) -> l1.addAll(l2)); if (sspForSystem.isEmpty()) { MSG_QUEUES.putIfAbsent(new SystemStreamPartition(systemStream, new Partition(0)), new ArrayList<>()); sspForSystem.add(new SystemStreamPartition(systemStream, new Partition(0))); } int partitionCount = sspForSystem.size(); int partitionId = envelope.getPartitionKey() == null ? envelope.getKey() == null ? this.seed.nextInt(partitionCount) : envelope.getKey().hashCode() % partitionCount : envelope.getPartitionKey().hashCode() % partitionCount; SystemStreamPartition ssp = new SystemStreamPartition(envelope.getSystemStream(), new Partition(partitionId)); List<IncomingMessageEnvelope> msgQueue = MSG_QUEUES.get(ssp); msgQueue.add(new IncomingMessageEnvelope(ssp, null, envelope.getKey(), envelope.getMessage())); }
/** * Creates a new checkpoint envelope for the provided task, ssp and offset */ private IncomingMessageEnvelope newCheckpointEnvelope(TaskName taskName, SystemStreamPartition ssp, String offset) { KafkaCheckpointLogKey checkpointKey = new KafkaCheckpointLogKey("checkpoint", taskName, GROUPER_FACTORY_CLASS); KafkaCheckpointLogKeySerde checkpointKeySerde = new KafkaCheckpointLogKeySerde(); Checkpoint checkpointMsg = new Checkpoint(ImmutableMap.of(ssp, offset)); CheckpointSerde checkpointMsgSerde = new CheckpointSerde(); return new IncomingMessageEnvelope(CHECKPOINT_SSP, offset, checkpointKeySerde.toBytes(checkpointKey), checkpointMsgSerde.toBytes(checkpointMsg)); }
public void send(String source, OutgoingMessageEnvelope envelope) { if (mockConsumer != null) { MockCoordinatorStreamWrappedConsumer consumer = (MockCoordinatorStreamWrappedConsumer) mockConsumer; SystemStreamPartition ssp = new SystemStreamPartition(envelope.getSystemStream(), new Partition(0)); consumer.register(ssp, ""); try { consumer.addMessageEnvelope(new IncomingMessageEnvelope(ssp, "", envelope.getKey(), envelope.getMessage())); } catch (IOException | InterruptedException e) { e.printStackTrace(); } } else { envelopes.add(envelope); } }