private SystemFactory newFactory(SystemProducer producer, SystemConsumer consumer, SystemAdmin admin) { SystemFactory factory = mock(SystemFactory.class); when(factory.getProducer(anyString(), any(Config.class), any(MetricsRegistry.class))).thenReturn(producer); when(factory.getConsumer(anyString(), any(Config.class), any(MetricsRegistry.class))).thenReturn(consumer); when(factory.getAdmin(anyString(), any(Config.class))).thenReturn(admin); return factory; }
public CoordinatorStreamSystemProducer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemProducer systemProducer = systemFactory.getProducer(coordinatorSystemStream.getSystem(), config, registry); this.systemStream = coordinatorSystemStream; this.systemProducer = systemProducer; this.systemAdmin = systemAdmin; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
public CoordinatorStreamSystemConsumer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemConsumer systemConsumer = systemFactory.getConsumer(coordinatorSystemStream.getSystem(), config, registry); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); this.systemConsumer = systemConsumer; this.systemAdmin = systemAdmin; this.configMap = new HashMap<>(); this.isBootstrapped = false; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
/** * Get {@link SystemAdmin} instances for all the systems defined in this config. * * @return map of system name to {@link SystemAdmin} */ public Map<String, SystemAdmin> getSystemAdmins() { return getSystemFactories().entrySet() .stream() .collect(Collectors.toMap(systemNameToFactoryEntry -> systemNameToFactoryEntry.getKey(), systemNameToFactoryEntry -> systemNameToFactoryEntry.getValue() .getAdmin(systemNameToFactoryEntry.getKey(), this))); }
/** * Creates SystemConsumer objects for store restoration, creating one consumer per system. */ private static Map<String, SystemConsumer> createStoreConsumers(Map<String, SystemStream> changelogSystemStreams, Map<String, SystemFactory> systemFactories, Config config, MetricsRegistry registry) { // Determine the set of systems being used across all stores Set<String> storeSystems = changelogSystemStreams.values().stream().map(SystemStream::getSystem).collect(Collectors.toSet()); // Create one consumer for each system in use, map with one entry for each such system Map<String, SystemConsumer> storeSystemConsumers = new HashMap<>(); // Map of each storeName to its respective systemConsumer Map<String, SystemConsumer> storeConsumers = new HashMap<>(); // Iterate over the list of storeSystems and create one sysConsumer per system for (String storeSystemName : storeSystems) { SystemFactory systemFactory = systemFactories.get(storeSystemName); if (systemFactory == null) { throw new SamzaException("Changelog system " + storeSystemName + " does not exist in config"); } storeSystemConsumers.put(storeSystemName, systemFactory.getConsumer(storeSystemName, config, registry)); } // Populate the map of storeName to its relevant systemConsumer for (String storeName : changelogSystemStreams.keySet()) { storeConsumers.put(storeName, storeSystemConsumers.get(changelogSystemStreams.get(storeName).getSystem())); } return storeConsumers; }
public CoordinatorStreamSystemProducer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemProducer systemProducer = systemFactory.getProducer(coordinatorSystemStream.getSystem(), config, registry); this.systemStream = coordinatorSystemStream; this.systemProducer = systemProducer; this.systemAdmin = systemAdmin; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
public CoordinatorStreamSystemConsumer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemConsumer systemConsumer = systemFactory.getConsumer(coordinatorSystemStream.getSystem(), config, registry); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); this.systemConsumer = systemConsumer; this.systemAdmin = systemAdmin; this.configMap = new HashMap<>(); this.isBootstrapped = false; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
/** * Get {@link SystemAdmin} instances for all the systems defined in this config. * * @return map of system name to {@link SystemAdmin} */ public Map<String, SystemAdmin> getSystemAdmins() { return getSystemFactories().entrySet() .stream() .collect(Collectors.toMap(systemNameToFactoryEntry -> systemNameToFactoryEntry.getKey(), systemNameToFactoryEntry -> systemNameToFactoryEntry.getValue() .getAdmin(systemNameToFactoryEntry.getKey(), this))); }
/** * get the SystemConsumers for the stores */ private HashMap<String, SystemConsumer> getStoreConsumers() { HashMap<String, SystemConsumer> storeConsumers = new HashMap<>(); Map<String, SystemFactory> systemFactories = new JavaSystemConfig(jobConfig).getSystemFactories(); for (Entry<String, SystemStream> entry : changeLogSystemStreams.entrySet()) { String storeSystem = entry.getValue().getSystem(); if (!systemFactories.containsKey(storeSystem)) { throw new SamzaException("Changelog system " + storeSystem + " for store " + entry.getKey() + " does not exist in the config."); } storeConsumers.put(entry.getKey(), systemFactories.get(storeSystem).getConsumer(storeSystem, jobConfig, new MetricsRegistryMap())); } return storeConsumers; }
public CoordinatorStreamStore(String namespace, Config config, MetricsRegistry metricsRegistry) { this.config = config; this.type = namespace; this.keySerde = new CoordinatorStreamKeySerde(type); this.coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); this.systemProducer = systemFactory.getProducer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry); this.systemConsumer = systemFactory.getConsumer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry); this.systemAdmin = systemFactory.getAdmin(this.coordinatorSystemStream.getSystem(), config); }
public CoordinatorStreamSystemProducer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemProducer systemProducer = systemFactory.getProducer(coordinatorSystemStream.getSystem(), config, registry); this.systemStream = coordinatorSystemStream; this.systemProducer = systemProducer; this.systemAdmin = systemAdmin; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
public CoordinatorStreamSystemConsumer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemConsumer systemConsumer = systemFactory.getConsumer(coordinatorSystemStream.getSystem(), config, registry); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); this.systemConsumer = systemConsumer; this.systemAdmin = systemAdmin; this.configMap = new HashMap<>(); this.isBootstrapped = false; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
/** * Get {@link SystemAdmin} instances for all the systems defined in this config. * * @return map of system name to {@link SystemAdmin} */ public Map<String, SystemAdmin> getSystemAdmins() { return getSystemFactories().entrySet() .stream() .collect(Collectors.toMap(systemNameToFactoryEntry -> systemNameToFactoryEntry.getKey(), systemNameToFactoryEntry -> systemNameToFactoryEntry.getValue() .getAdmin(systemNameToFactoryEntry.getKey(), this))); }
/** * get the SystemConsumers for the stores */ private HashMap<String, SystemConsumer> getStoreConsumers() { HashMap<String, SystemConsumer> storeConsumers = new HashMap<>(); Map<String, SystemFactory> systemFactories = new JavaSystemConfig(jobConfig).getSystemFactories(); for (Entry<String, SystemStream> entry : changeLogSystemStreams.entrySet()) { String storeSystem = entry.getValue().getSystem(); if (!systemFactories.containsKey(storeSystem)) { throw new SamzaException("Changelog system " + storeSystem + " for store " + entry.getKey() + " does not exist in the config."); } storeConsumers.put(entry.getKey(), systemFactories.get(storeSystem).getConsumer(storeSystem, jobConfig, new MetricsRegistryMap())); } return storeConsumers; }
public CoordinatorStreamStore(String namespace, Config config, MetricsRegistry metricsRegistry) { this.config = config; this.type = namespace; this.keySerde = new JsonSerde<>(); this.coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); this.systemProducer = systemFactory.getProducer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry); this.systemConsumer = systemFactory.getConsumer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry); this.systemAdmin = systemFactory.getAdmin(this.coordinatorSystemStream.getSystem(), config); }
public CoordinatorStreamSystemProducer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemProducer systemProducer = systemFactory.getProducer(coordinatorSystemStream.getSystem(), config, registry); this.systemStream = coordinatorSystemStream; this.systemProducer = systemProducer; this.systemAdmin = systemAdmin; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
public CoordinatorStreamSystemConsumer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemConsumer systemConsumer = systemFactory.getConsumer(coordinatorSystemStream.getSystem(), config, registry); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); this.systemConsumer = systemConsumer; this.systemAdmin = systemAdmin; this.configMap = new HashMap<>(); this.isBootstrapped = false; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
/** * Get {@link SystemAdmin} instances for all the systems defined in this config. * * @return map of system name to {@link SystemAdmin} */ public Map<String, SystemAdmin> getSystemAdmins() { return getSystemFactories().entrySet() .stream() .collect(Collectors.toMap(systemNameToFactoryEntry -> systemNameToFactoryEntry.getKey(), systemNameToFactoryEntry -> systemNameToFactoryEntry.getValue() .getAdmin(systemNameToFactoryEntry.getKey(), this))); }
/** * get the SystemConsumers for the stores */ private HashMap<String, SystemConsumer> getStoreConsumers() { HashMap<String, SystemConsumer> storeConsumers = new HashMap<>(); Map<String, SystemFactory> systemFactories = new JavaSystemConfig(jobConfig).getSystemFactories(); for (Entry<String, SystemStream> entry : changeLogSystemStreams.entrySet()) { String storeSystem = entry.getValue().getSystem(); if (!systemFactories.containsKey(storeSystem)) { throw new SamzaException("Changelog system " + storeSystem + " for store " + entry.getKey() + " does not exist in the config."); } storeConsumers.put(entry.getKey(), systemFactories.get(storeSystem).getConsumer(storeSystem, jobConfig, new MetricsRegistryMap())); } return storeConsumers; }
public CoordinatorStreamStore(String namespace, Config config, MetricsRegistry metricsRegistry) { this.config = config; this.type = namespace; this.keySerde = new JsonSerde<>(); this.coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); this.systemProducer = systemFactory.getProducer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry); this.systemConsumer = systemFactory.getConsumer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry); this.systemAdmin = systemFactory.getAdmin(this.coordinatorSystemStream.getSystem(), config); }