/** * Returns the period separated system stream name for the provided {@code systemStream}. For * example, SystemStream("kafka", "topic") would return "kafka.topic". * * @param systemStream the {@link SystemStream} to get the name for * @return the system stream name */ public static String getNameFromSystemStream(SystemStream systemStream) { return systemStream.getSystem() + "." + systemStream.getStream(); }
/** * Returns the period separated system stream name for the provided {@code systemStream}. For * example, SystemStream("kafka", "topic") would return "kafka.topic". * * @param systemStream the {@link SystemStream} to get the name for * @return the system stream name */ public static String getNameFromSystemStream(SystemStream systemStream) { return systemStream.getSystem() + "." + systemStream.getStream(); }
/** * Constructs a Samza stream object based upon an existing Samza stream. * @param other Reference to an already existing Samza stream. */ public SystemStream(SystemStream other) { this(other.getSystem(), other.getStream()); }
/** * Constructs a Samza stream partition object based upon another Samza stream and a specified partition. * @param other Reference to an already existing Samza stream. * @param partition Reference to an already existing Samza partition. */ public SystemStreamPartition(SystemStream other, Partition partition) { this(other.getSystem(), other.getStream(), partition); }
/** * Constructs a Samza stream object based upon an existing Samza stream. * @param other Reference to an already existing Samza stream. */ public SystemStream(SystemStream other) { this(other.getSystem(), other.getStream()); }
/** * Constructs a Samza stream partition object based upon another Samza stream and a specified partition. * @param other Reference to an already existing Samza stream. * @param partition Reference to an already existing Samza partition. */ public SystemStreamPartition(SystemStream other, Partition partition) { this(other.getSystem(), other.getStream(), partition); }
/** * Returns the period separated system stream name for the provided {@code systemStream}. For * example, SystemStream("kafka", "topic") would return "kafka.topic". * * @param systemStream the {@link SystemStream} to get the name for * @return the system stream name */ public static String getNameFromSystemStream(SystemStream systemStream) { return systemStream.getSystem() + "." + systemStream.getStream(); }
/** * Returns the period separated system stream name for the provided {@code systemStream}. For * example, SystemStream("kafka", "topic") would return "kafka.topic". * * @param systemStream the {@link SystemStream} to get the name for * @return the system stream name */ public static String getNameFromSystemStream(SystemStream systemStream) { return systemStream.getSystem() + "." + systemStream.getStream(); }
/** * Returns the period separated system stream name for the provided {@code systemStream}. For * example, SystemStream("kafka", "topic") would return "kafka.topic". * * @param systemStream the {@link SystemStream} to get the name for * @return the system stream name */ public static String getNameFromSystemStream(SystemStream systemStream) { return systemStream.getSystem() + "." + systemStream.getStream(); }
@Override public String getSchema(SystemStream systemStream) { String fileName = String.format("%s.avsc", systemStream.getStream()); File file = new File(schemaDir, fileName); try { return Schema.parse(file).toString(); } catch (IOException e) { throw new SamzaException(e); } } }
@Override public String getSchema(SystemStream systemStream) { return config.get(String.format(CFG_SOURCE_SCHEMA, systemStream.getSystem(), systemStream.getStream())); } }
void send(ControlMessage message, SystemStream systemStream, MessageCollector collector) { int partitionCount = getPartitionCount(systemStream); // We pick a partition based on topic hashcode to aggregate the control messages from upstream tasks // After aggregation the task will broadcast the results to other partitions int aggregatePartition = systemStream.getStream().hashCode() % partitionCount; LOG.debug(String.format("Send %s message from task %s to %s partition %s for aggregation", MessageType.of(message).name(), message.getTaskName(), systemStream, aggregatePartition)); OutgoingMessageEnvelope envelopeOut = new OutgoingMessageEnvelope(systemStream, aggregatePartition, null, message); collector.send(envelopeOut); }
protected IndexRequest getRequest(OutgoingMessageEnvelope envelope) { String[] parts = envelope.getSystemStream().getStream().split("/"); if (parts.length != 2) { throw new SamzaException("Elasticsearch stream name must match pattern {index}/{type}"); } String index = parts[0]; String type = parts[1]; return Requests.indexRequest(index).type(type); }
void send(ControlMessage message, SystemStream systemStream, MessageCollector collector) { int partitionCount = getPartitionCount(systemStream); // We pick a partition based on topic hashcode to aggregate the control messages from upstream tasks // After aggregation the task will broadcast the results to other partitions int aggregatePartition = systemStream.getStream().hashCode() % partitionCount; LOG.debug(String.format("Send %s message from task %s to %s partition %s for aggregation", MessageType.of(message).name(), message.getTaskName(), systemStream, aggregatePartition)); OutgoingMessageEnvelope envelopeOut = new OutgoingMessageEnvelope(systemStream, aggregatePartition, null, message); collector.send(envelopeOut); }
/** * Validates each changelog system-stream with its respective SystemAdmin. */ private void validateChangelogStreams() { LOG.info("Validating change log streams: " + changelogSystemStreams); for (SystemStream changelogSystemStream : changelogSystemStreams.values()) { SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(changelogSystemStream.getSystem()); StreamSpec changelogSpec = StreamSpec.createChangeLogStreamSpec(changelogSystemStream.getStream(), changelogSystemStream.getSystem(), maxChangeLogStreamPartitions); systemAdmin.validateStream(changelogSpec); } }
void send(ControlMessage message, SystemStream systemStream, MessageCollector collector) { int partitionCount = getPartitionCount(systemStream); // We pick a partition based on topic hashcode to aggregate the control messages from upstream tasks // After aggregation the task will broadcast the results to other partitions int aggregatePartition = systemStream.getStream().hashCode() % partitionCount; LOG.debug(String.format("Send %s message from task %s to %s partition %s for aggregation", MessageType.of(message).name(), message.getTaskName(), systemStream, aggregatePartition)); OutgoingMessageEnvelope envelopeOut = new OutgoingMessageEnvelope(systemStream, aggregatePartition, null, message); collector.send(envelopeOut); }
void send(ControlMessage message, SystemStream systemStream, MessageCollector collector) { int partitionCount = getPartitionCount(systemStream); // We pick a partition based on topic hashcode to aggregate the control messages from upstream tasks // After aggregation the task will broadcast the results to other partitions int aggregatePartition = systemStream.getStream().hashCode() % partitionCount; LOG.debug(String.format("Send %s message from task %s to %s partition %s for aggregation", MessageType.of(message).name(), message.getTaskName(), systemStream, aggregatePartition)); OutgoingMessageEnvelope envelopeOut = new OutgoingMessageEnvelope(systemStream, aggregatePartition, null, message); collector.send(envelopeOut); }
void send(ControlMessage message, SystemStream systemStream, MessageCollector collector) { int partitionCount = getPartitionCount(systemStream); // We pick a partition based on topic hashcode to aggregate the control messages from upstream tasks // After aggregation the task will broadcast the results to other partitions int aggregatePartition = systemStream.getStream().hashCode() % partitionCount; LOG.debug(String.format("Send %s message from task %s to %s partition %s for aggregation", MessageType.of(message).name(), message.getTaskName(), systemStream, aggregatePartition)); OutgoingMessageEnvelope envelopeOut = new OutgoingMessageEnvelope(systemStream, aggregatePartition, null, message); collector.send(envelopeOut); }
@Before public void setup() { when(SYSTEM.getStream()).thenReturn(INDEX + "/" + TYPE); }
@Test(expected=SamzaException.class) public void testGetIndexRequestInvalidStreamName() { when(SYSTEM.getStream()).thenReturn(INDEX); indexRequestFactory.getIndexRequest(new OutgoingMessageEnvelope(SYSTEM, EMPTY_MSG)); }