private void startTransferThread() { try { // Serialize the key once, since we will use it for every event. final byte[] keyBytes = key.getBytes("UTF-8"); Runnable transferFromQueueToSystem = () -> { while (!Thread.currentThread().isInterrupted()) { try { byte[] serializedLogEvent = logQueue.take(); OutgoingMessageEnvelope outgoingMessageEnvelope = new OutgoingMessageEnvelope(systemStream, keyBytes, serializedLogEvent); systemProducer.send(SOURCE, outgoingMessageEnvelope); } catch (InterruptedException e) { // Preserve the interrupted status for the loop condition. Thread.currentThread().interrupt(); } catch (Throwable t) { log.error("Error sending StreamAppender event to SystemProducer", t); } } }; transferThread = new Thread(transferFromQueueToSystem); transferThread.setDaemon(true); transferThread.setName("Samza StreamAppender Producer " + transferThread.getName()); transferThread.start(); } catch (UnsupportedEncodingException e) { throw new SamzaException(String.format( "Container name: %s could not be encoded to bytes. StreamAppender cannot proceed.", key), e); } }
@Override public void put(byte[] key, byte[] value) { OutgoingMessageEnvelope envelope = new OutgoingMessageEnvelope(coordinatorSystemStream, 0, key, value); systemProducer.send(SOURCE, envelope); flush(); }
@Override public void put(byte[] key, byte[] value) { OutgoingMessageEnvelope envelope = new OutgoingMessageEnvelope(coordinatorSystemStream, 0, key, value); systemProducer.send(SOURCE, envelope); flush(); }
@Override public void put(byte[] key, byte[] value) { OutgoingMessageEnvelope envelope = new OutgoingMessageEnvelope(coordinatorSystemStream, 0, key, value); systemProducer.send(SOURCE, envelope); flush(); }
private void startTransferThread() { try { // Serialize the key once, since we will use it for every event. final byte[] keyBytes = key.getBytes("UTF-8"); Runnable transferFromQueueToSystem = () -> { while (!Thread.currentThread().isInterrupted()) { try { byte[] serializedLogEvent = logQueue.take(); OutgoingMessageEnvelope outgoingMessageEnvelope = new OutgoingMessageEnvelope(systemStream, keyBytes, serializedLogEvent); systemProducer.send(SOURCE, outgoingMessageEnvelope); } catch (InterruptedException e) { // Preserve the interrupted status for the loop condition. Thread.currentThread().interrupt(); } catch (Throwable t) { log.error("Error sending StreamAppender event to SystemProducer", t); } } }; transferThread = new Thread(transferFromQueueToSystem); transferThread.setDaemon(true); transferThread.setName("Samza StreamAppender Producer " + transferThread.getName()); transferThread.start(); } catch (UnsupportedEncodingException e) { throw new SamzaException(String.format( "Container name: %s could not be encoded to bytes. StreamAppender cannot proceed.", key), e); } }
@Override public void put(byte[] key, byte[] value) { OutgoingMessageEnvelope envelope = new OutgoingMessageEnvelope(coordinatorSystemStream, 0, key, value); systemProducer.send(SOURCE, envelope); flush(); }
@Override public void put(String key, byte[] value) { OutgoingMessageEnvelope envelope = new OutgoingMessageEnvelope(coordinatorSystemStream, 0, keySerde.toBytes(key), value); systemProducer.send(SOURCE, envelope); flush(); }
private void produceMessages(Object... events) { SystemProducer systemProducer = systemFactory.getProducer(SYSTEM_NAME, config, mockRegistry); Stream.of(events) .forEach(event -> systemProducer.send(SOURCE, new OutgoingMessageEnvelope(SYSTEM_STREAM, event))); }
@Test public void testSend() throws Exception { OutgoingMessageEnvelope envelope = mock(OutgoingMessageEnvelope.class); IndexRequest indexRequest = mock(IndexRequest.class); when(INDEX_REQUEST_FACTORY.getIndexRequest(envelope)).thenReturn(indexRequest); producer.send(SOURCE_ONE, envelope); verify(processorOne).add(indexRequest); }
/** * Serialize and send a coordinator stream message. * * @param message * The message to send. */ public void send(CoordinatorStreamMessage message) { log.debug("Sending {}", message); try { String source = message.getSource(); byte[] key = keySerde.toBytes(Arrays.asList(message.getKeyArray())); byte[] value = null; if (!message.isDelete()) { value = messageSerde.toBytes(message.getMessageMap()); } OutgoingMessageEnvelope envelope = new OutgoingMessageEnvelope(systemStream, Integer.valueOf(0), key, value); systemProducer.send(source, envelope); } catch (Exception e) { throw new SamzaException(e); } }
/** * Serialize and send a coordinator stream message. * * @param message * The message to send. */ public void send(CoordinatorStreamMessage message) { log.debug("Sending {}", message); try { String source = message.getSource(); byte[] key = keySerde.toBytes(Arrays.asList(message.getKeyArray())); byte[] value = null; if (!message.isDelete()) { value = messageSerde.toBytes(message.getMessageMap()); } OutgoingMessageEnvelope envelope = new OutgoingMessageEnvelope(systemStream, Integer.valueOf(0), key, value); systemProducer.send(source, envelope); } catch (Exception e) { throw new SamzaException(e); } }
/** * Serialize and send a coordinator stream message. * * @param message * The message to send. */ public void send(CoordinatorStreamMessage message) { log.debug("Sending {}", message); try { String source = message.getSource(); byte[] key = keySerde.toBytes(Arrays.asList(message.getKeyArray())); byte[] value = null; if (!message.isDelete()) { value = messageSerde.toBytes(message.getMessageMap()); } OutgoingMessageEnvelope envelope = new OutgoingMessageEnvelope(systemStream, Integer.valueOf(0), key, value); systemProducer.send(source, envelope); } catch (Exception e) { throw new SamzaException(e); } }
/** * Serialize and send a coordinator stream message. * * @param message * The message to send. */ public void send(CoordinatorStreamMessage message) { log.debug("Sending {}", message); try { String source = message.getSource(); byte[] key = keySerde.toBytes(Arrays.asList(message.getKeyArray())); byte[] value = null; if (!message.isDelete()) { value = messageSerde.toBytes(message.getMessageMap()); } OutgoingMessageEnvelope envelope = new OutgoingMessageEnvelope(systemStream, Integer.valueOf(0), key, value); systemProducer.send(source, envelope); } catch (Exception e) { throw new SamzaException(e); } }
/** * Serialize and send a coordinator stream message. * * @param message * The message to send. */ public void send(CoordinatorStreamMessage message) { log.debug("Sending {}", message); try { String source = message.getSource(); byte[] key = keySerde.toBytes(Arrays.asList(message.getKeyArray())); byte[] value = null; if (!message.isDelete()) { value = messageSerde.toBytes(message.getMessageMap()); } OutgoingMessageEnvelope envelope = new OutgoingMessageEnvelope(systemStream, Integer.valueOf(0), key, value); systemProducer.send(source, envelope); } catch (Exception e) { throw new SamzaException(e); } }
@Test(expected = NullPointerException.class) public void testNullMessageWithNullKey() { SystemProducer systemProducer = systemFactory.getProducer(SYSTEM_NAME, config, mockRegistry); systemProducer.send(SOURCE, new OutgoingMessageEnvelope(SYSTEM_STREAM, null)); }
@Test public void testNullMessageWithValidMessageKey() { final String messageKey = "validKey"; SystemProducer systemProducer = systemFactory.getProducer(SYSTEM_NAME, config, mockRegistry); systemProducer.send(SOURCE, new OutgoingMessageEnvelope(SYSTEM_STREAM, messageKey, null)); SystemConsumer consumer = systemFactory.getConsumer(SYSTEM_NAME, config, mockRegistry); Set<SystemStreamPartition> sspsToPoll = IntStream.range(0, PARTITION_COUNT) .mapToObj(partition -> new SystemStreamPartition(SYSTEM_STREAM, new Partition(partition))) .collect(Collectors.toSet()); // register the consumer for ssps for (SystemStreamPartition ssp : sspsToPoll) { consumer.register(ssp, "0"); } List<IncomingMessageEnvelope> results = consumeRawMessages(consumer, sspsToPoll); assertEquals(1, results.size()); assertEquals(results.get(0).getKey(), messageKey); assertNull(results.get(0).getMessage()); }
@Test public void testSend() { Config eventHubConfig = createEventHubConfig(); EventHubSystemFactory systemFactory = new EventHubSystemFactory(); SystemProducer systemProducer = systemFactory.getProducer(SYSTEM_NAME, eventHubConfig, new NoOpMetricsRegistry()); systemProducer.register(STREAM_NAME1); try { systemProducer.send(STREAM_NAME1, createMessageEnvelope(STREAM_NAME1)); Assert.fail("Sending event before starting producer should throw exception"); } catch (SamzaException e) { } systemProducer.start(); systemProducer.send(STREAM_NAME1, createMessageEnvelope(STREAM_NAME1)); try { systemProducer.send("unregistered_stream", createMessageEnvelope("unregistered_stream")); Assert.fail("Sending event to destination that is not registered should throw exception"); } catch (SamzaException e) { } try { systemProducer.register(STREAM_NAME2); Assert.fail("Trying to register after starting producer should throw exception"); } catch (SamzaException e) { } systemProducer.flush(STREAM_NAME1); systemProducer.stop(); }
@Test public void testSendToSpecificPartition() { Config eventHubConfig = MockEventHubConfigFactory.getEventHubConfig(EventHubSystemProducer.PartitioningMethod.PARTITION_KEY_AS_PARTITION); EventHubSystemFactory systemFactory = new EventHubSystemFactory(); SystemProducer systemProducer = systemFactory.getProducer(SYSTEM_NAME, eventHubConfig, new NoOpMetricsRegistry()); systemProducer.register(STREAM_NAME1); systemProducer.start(); for (int i = 0; i < 100; i++) { systemProducer.send(STREAM_NAME1, createMessageEnvelope(STREAM_NAME1, 0)); } systemProducer.flush(STREAM_NAME1); systemProducer.stop(); }