public SystemStream toSystemStream() { return new SystemStream(systemName, physicalName); }
/** * Constructs a Samza stream partition object based upon another Samza stream and a specified partition. * @param other Reference to an already existing Samza stream. * @param partition Reference to an already existing Samza partition. */ public SystemStreamPartition(SystemStream other, Partition partition) { this(other.getSystem(), other.getStream(), partition); }
long getWatermark(SystemStream systemStream) { return watermarkStates.entrySet().stream() .filter(entry -> entry.getKey().getSystemStream().equals(systemStream)) .map(entry -> entry.getValue().getWatermarkTime()) .min(Long::compare) .orElse(WATERMARK_NOT_EXIST); }
@Override public String getSchema(SystemStream systemStream) { String fileName = String.format("%s.avsc", systemStream.getStream()); File file = new File(schemaDir, fileName); try { return Schema.parse(file).toString(); } catch (IOException e) { throw new SamzaException(e); } } }
public String getSystemName() { return systemStream.getSystem(); }
if (currentPartitionCount != prevPartitionCount) { log.warn(String.format("Change of partition count detected in stream %s. old partition count: %d, current partition count: %d", systemStream.toString(), prevPartitionCount, currentPartitionCount)); if (currentPartitionCount > prevPartitionCount) { log.error(String.format("Shutting down (stateful) or restarting (stateless) the job since current " + "partition count %d is greater than the old partition count %d for stream %s.", currentPartitionCount, prevPartitionCount, systemStream.toString())); streamsChanged.add(systemStream); log.error(String.format("Error comparing partition count differences for stream: %s", metadataEntry.getKey().toString()));
protected IndexRequest getRequest(OutgoingMessageEnvelope envelope) { String[] parts = envelope.getSystemStream().getStream().split("/"); if (parts.length != 2) { throw new SamzaException("Elasticsearch stream name must match pattern {index}/{type}"); } String index = parts[0]; String type = parts[1]; return Requests.indexRequest(index).type(type); }
public CoordinatorStreamSystemProducer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemProducer systemProducer = systemFactory.getProducer(coordinatorSystemStream.getSystem(), config, registry); this.systemStream = coordinatorSystemStream; this.systemProducer = systemProducer; this.systemAdmin = systemAdmin; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
if (currentPartitionCount != prevPartitionCount) { log.warn(String.format("Change of partition count detected in stream %s. old partition count: %d, current partition count: %d", systemStream.toString(), prevPartitionCount, currentPartitionCount)); if (currentPartitionCount > prevPartitionCount) { log.error(String.format("Shutting down (stateful) or restarting (stateless) the job since current " + "partition count %d is greater than the old partition count %d for stream %s.", currentPartitionCount, prevPartitionCount, systemStream.toString())); streamsChanged.add(systemStream); log.error(String.format("Error comparing partition count differences for stream: %s", metadataEntry.getKey().toString()));
/** * Returns the period separated system stream name for the provided {@code systemStream}. For * example, SystemStream("kafka", "topic") would return "kafka.topic". * * @param systemStream the {@link SystemStream} to get the name for * @return the system stream name */ public static String getNameFromSystemStream(SystemStream systemStream) { return systemStream.getSystem() + "." + systemStream.getStream(); }
public SystemStream getSystemStream() { return new SystemStream(system, stream); }
void send(ControlMessage message, SystemStream systemStream, MessageCollector collector) { int partitionCount = getPartitionCount(systemStream); // We pick a partition based on topic hashcode to aggregate the control messages from upstream tasks // After aggregation the task will broadcast the results to other partitions int aggregatePartition = systemStream.getStream().hashCode() % partitionCount; LOG.debug(String.format("Send %s message from task %s to %s partition %s for aggregation", MessageType.of(message).name(), message.getTaskName(), systemStream, aggregatePartition)); OutgoingMessageEnvelope envelopeOut = new OutgoingMessageEnvelope(systemStream, aggregatePartition, null, message); collector.send(envelopeOut); }
public CoordinatorStreamStore(String namespace, Config config, MetricsRegistry metricsRegistry) { this.config = config; this.type = namespace; this.keySerde = new CoordinatorStreamKeySerde(type); this.coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); this.systemProducer = systemFactory.getProducer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry); this.systemConsumer = systemFactory.getConsumer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry); this.systemAdmin = systemFactory.getAdmin(this.coordinatorSystemStream.getSystem(), config); }
boolean isEndOfStream(SystemStream systemStream) { return eosStates.entrySet().stream() .filter(entry -> entry.getKey().getSystemStream().equals(systemStream)) .allMatch(entry -> entry.getValue().isEndOfStream()); }
if (currentPartitionCount != prevPartitionCount) { log.warn(String.format("Change of partition count detected in stream %s. old partition count: %d, current partition count: %d", systemStream.toString(), prevPartitionCount, currentPartitionCount)); if (currentPartitionCount > prevPartitionCount) { log.error(String.format("Shutting down (stateful) or restarting (stateless) the job since current " + "partition count %d is greater than the old partition count %d for stream %s.", currentPartitionCount, prevPartitionCount, systemStream.toString())); streamsChanged.add(systemStream); log.error(String.format("Error comparing partition count differences for stream: %s", metadataEntry.getKey().toString()));
public SystemStream toSystemStream() { return new SystemStream(systemName, physicalName); }
void send(ControlMessage message, SystemStream systemStream, MessageCollector collector) { int partitionCount = getPartitionCount(systemStream); // We pick a partition based on topic hashcode to aggregate the control messages from upstream tasks // After aggregation the task will broadcast the results to other partitions int aggregatePartition = systemStream.getStream().hashCode() % partitionCount; LOG.debug(String.format("Send %s message from task %s to %s partition %s for aggregation", MessageType.of(message).name(), message.getTaskName(), systemStream, aggregatePartition)); OutgoingMessageEnvelope envelopeOut = new OutgoingMessageEnvelope(systemStream, aggregatePartition, null, message); collector.send(envelopeOut); }
public CoordinatorStreamSystemProducer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemProducer systemProducer = systemFactory.getProducer(coordinatorSystemStream.getSystem(), config, registry); this.systemStream = coordinatorSystemStream; this.systemProducer = systemProducer; this.systemAdmin = systemAdmin; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
long getWatermark(SystemStream systemStream) { return watermarkStates.entrySet().stream() .filter(entry -> entry.getKey().getSystemStream().equals(systemStream)) .map(entry -> entry.getValue().getWatermarkTime()) .min(Long::compare) .orElse(WATERMARK_NOT_EXIST); }