public CoordinatorStreamSystemProducer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemProducer systemProducer = systemFactory.getProducer(coordinatorSystemStream.getSystem(), config, registry); this.systemStream = coordinatorSystemStream; this.systemProducer = systemProducer; this.systemAdmin = systemAdmin; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
/** * Builds coordinator system config for the {@param jobInstance}. * @param jobInstance the job instance to get the jobModel for. * @return the constructed coordinator system config. */ private Config getCoordinatorSystemConfig(JobInstance jobInstance) { try { InstallationRecord record = installFinder.getAllInstalledJobs().get(jobInstance); ConfigFactory configFactory = Util.getObj(taskResourceConfig.getJobConfigFactory(), ConfigFactory.class); Config config = configFactory.getConfig(new URI(String.format("file://%s", record.getConfigFilePath()))); Map<String, String> configMap = ImmutableMap.of(JobConfig.JOB_ID(), jobInstance.getJobId(), JobConfig.JOB_NAME(), jobInstance.getJobName()); return CoordinatorStreamUtil.buildCoordinatorStreamConfig(new MapConfig(ImmutableList.of(config, configMap))); } catch (Exception e) { LOG.error(String.format("Failed to get coordinator stream config for job : %s", jobInstance), e); throw new SamzaException(e); } }
@Before public void setup() { mockCoordinatorStreamSystemFactory = new MockCoordinatorStreamSystemFactory(); MockCoordinatorStreamSystemFactory.enableMockConsumerCache(); PowerMockito.mockStatic(CoordinatorStreamUtil.class); when(CoordinatorStreamUtil.getCoordinatorSystemFactory(anyObject())).thenReturn(mockCoordinatorStreamSystemFactory); when(CoordinatorStreamUtil.getCoordinatorSystemStream(anyObject())).thenReturn(new SystemStream("test-kafka", "test")); when(CoordinatorStreamUtil.getCoordinatorStreamName(anyObject(), anyObject())).thenReturn("test"); }
/** * Creates a coordinator stream kafka topic. */ private void createCoordinatorStream() { SystemAdmin coordinatorSystemAdmin = null; SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); coordinatorSystemAdmin = systemAdmins.getSystemAdmin(coordinatorSystemStream.getSystem()); String streamName = coordinatorSystemStream.getStream(); StreamSpec coordinatorSpec = StreamSpec.createCoordinatorStreamSpec(streamName, coordinatorSystemStream.getSystem()); if (coordinatorSystemAdmin.createStream(coordinatorSpec)) { LOG.info("Created coordinator stream: {}.", streamName); } else { LOG.info("Coordinator stream: {} already exists.", streamName); } }
private SystemStream getCoordinatorSystemStream(Config config) { assertNotNull(config.get("job.coordinator.system")); assertNotNull(config.get("job.name")); return new SystemStream(config.get("job.coordinator.system"), CoordinatorStreamUtil.getCoordinatorStreamName(config.get("job.name"), config.get("job.id") == null ? "1" : config.get("job.id"))); }
/** * Returns a consumer that sends all configs to the coordinator stream. * * @param config Along with the configs, you can pass checkpoints and changelog stream messages into the stream. * The expected pattern is cp:source:taskname -> ssp,offset for checkpoint (Use sspToString util) * ch:source:taskname -> changelogPartition for changelog * Everything else is processed as normal config */ @Override public SystemConsumer getConsumer(String systemName, Config config, MetricsRegistry registry) { if (useCachedConsumer && mockConsumer != null) { return mockConsumer; } String jobName = config.get("job.name"); String jobId = config.get("job.id"); if (jobName == null) { throw new ConfigException("Must define job.name."); } if (jobId == null) { jobId = "1"; } String streamName = CoordinatorStreamUtil.getCoordinatorStreamName(jobName, jobId); SystemStreamPartition systemStreamPartition = new SystemStreamPartition(systemName, streamName, new Partition(0)); mockConsumer = new MockCoordinatorStreamWrappedConsumer(systemStreamPartition, config); return mockConsumer; }
public CoordinatorStreamSystemProducer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemProducer systemProducer = systemFactory.getProducer(coordinatorSystemStream.getSystem(), config, registry); this.systemStream = coordinatorSystemStream; this.systemProducer = systemProducer; this.systemAdmin = systemAdmin; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
@Before public void setup() { MockCoordinatorStreamSystemFactory.enableMockConsumerCache(); mockCoordinatorStreamSystemFactory = new MockCoordinatorStreamSystemFactory(); PowerMockito.mockStatic(CoordinatorStreamUtil.class); when(CoordinatorStreamUtil.getCoordinatorSystemFactory(anyObject())).thenReturn(mockCoordinatorStreamSystemFactory); when(CoordinatorStreamUtil.getCoordinatorSystemStream(anyObject())).thenReturn(new SystemStream("test-kafka", "test")); when(CoordinatorStreamUtil.getCoordinatorStreamName(anyObject(), anyObject())).thenReturn("test"); }
public CoordinatorStreamSystemProducer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemProducer systemProducer = systemFactory.getProducer(coordinatorSystemStream.getSystem(), config, registry); this.systemStream = coordinatorSystemStream; this.systemProducer = systemProducer; this.systemAdmin = systemAdmin; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
@Before public void setUp() throws NoSuchFieldException, NoSuchMethodException { configMap = new HashMap<>(); configMap.put("job.name", "test-job"); configMap.put("job.coordinator.system", "kafka"); configMap.put("task.inputs", "kafka.topic1"); configMap.put("systems.kafka.samza.factory", "org.apache.samza.system.MockSystemFactory"); configMap.put("samza.cluster-manager.factory", "org.apache.samza.clustermanager.MockClusterResourceManagerFactory"); configMap.put("job.coordinator.monitor-partition-change.frequency.ms", "1"); MockSystemFactory.MSG_QUEUES.put(new SystemStreamPartition("kafka", "topic1", new Partition(0)), new ArrayList<>()); MockSystemFactory.MSG_QUEUES.put(new SystemStreamPartition("kafka", "__samza_coordinator_test-job_1", new Partition(0)), new ArrayList<>()); MockCoordinatorStreamSystemFactory.enableMockConsumerCache(); PowerMockito.mockStatic(CoordinatorStreamUtil.class); when(CoordinatorStreamUtil.getCoordinatorSystemFactory(anyObject())).thenReturn(new MockCoordinatorStreamSystemFactory()); when(CoordinatorStreamUtil.getCoordinatorSystemStream(anyObject())).thenReturn(new SystemStream("kafka", "test")); when(CoordinatorStreamUtil.getCoordinatorStreamName(anyObject(), anyObject())).thenReturn("test"); }
public CoordinatorStreamSystemProducer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemProducer systemProducer = systemFactory.getProducer(coordinatorSystemStream.getSystem(), config, registry); this.systemStream = coordinatorSystemStream; this.systemProducer = systemProducer; this.systemAdmin = systemAdmin; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
@Before public void setUp() { MockCoordinatorStreamSystemFactory.enableMockConsumerCache(); Map<String, String> configMap = ImmutableMap.of("job.name", "test-job", "job.coordinator.system", "test-kafka"); MockCoordinatorStreamSystemFactory systemFactory = new MockCoordinatorStreamSystemFactory(); PowerMockito.mockStatic(CoordinatorStreamUtil.class); when(CoordinatorStreamUtil.getCoordinatorSystemFactory(anyObject())).thenReturn(systemFactory); when(CoordinatorStreamUtil.getCoordinatorSystemStream(anyObject())).thenReturn(new SystemStream("test-kafka", "test")); when(CoordinatorStreamUtil.getCoordinatorStreamName(anyObject(), anyObject())).thenReturn("test"); coordinatorStreamStore = new CoordinatorStreamStore(SetTaskContainerMapping.TYPE, new MapConfig(configMap), new MetricsRegistryMap()); coordinatorStreamStore.init(); }
public CoordinatorStreamSystemProducer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemProducer systemProducer = systemFactory.getProducer(coordinatorSystemStream.getSystem(), config, registry); this.systemStream = coordinatorSystemStream; this.systemProducer = systemProducer; this.systemAdmin = systemAdmin; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
public CoordinatorStreamSystemConsumer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemConsumer systemConsumer = systemFactory.getConsumer(coordinatorSystemStream.getSystem(), config, registry); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); this.systemConsumer = systemConsumer; this.systemAdmin = systemAdmin; this.configMap = new HashMap<>(); this.isBootstrapped = false; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
public CoordinatorStreamSystemConsumer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemConsumer systemConsumer = systemFactory.getConsumer(coordinatorSystemStream.getSystem(), config, registry); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); this.systemConsumer = systemConsumer; this.systemAdmin = systemAdmin; this.configMap = new HashMap<>(); this.isBootstrapped = false; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
public CoordinatorStreamSystemConsumer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemConsumer systemConsumer = systemFactory.getConsumer(coordinatorSystemStream.getSystem(), config, registry); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); this.systemConsumer = systemConsumer; this.systemAdmin = systemAdmin; this.configMap = new HashMap<>(); this.isBootstrapped = false; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
public CoordinatorStreamSystemConsumer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemConsumer systemConsumer = systemFactory.getConsumer(coordinatorSystemStream.getSystem(), config, registry); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); this.systemConsumer = systemConsumer; this.systemAdmin = systemAdmin; this.configMap = new HashMap<>(); this.isBootstrapped = false; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
public CoordinatorStreamSystemConsumer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemConsumer systemConsumer = systemFactory.getConsumer(coordinatorSystemStream.getSystem(), config, registry); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); this.systemConsumer = systemConsumer; this.systemAdmin = systemAdmin; this.configMap = new HashMap<>(); this.isBootstrapped = false; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
public CoordinatorStreamStore(String namespace, Config config, MetricsRegistry metricsRegistry) { this.config = config; this.type = namespace; this.keySerde = new CoordinatorStreamKeySerde(type); this.coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); this.systemProducer = systemFactory.getProducer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry); this.systemConsumer = systemFactory.getConsumer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry); this.systemAdmin = systemFactory.getAdmin(this.coordinatorSystemStream.getSystem(), config); }
public CoordinatorStreamStore(String namespace, Config config, MetricsRegistry metricsRegistry) { this.config = config; this.type = namespace; this.keySerde = new JsonSerde<>(); this.coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); this.systemProducer = systemFactory.getProducer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry); this.systemConsumer = systemFactory.getConsumer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry); this.systemAdmin = systemFactory.getAdmin(this.coordinatorSystemStream.getSystem(), config); }