public GroupBySystemStreamPartition(Config config) { TaskConfigJava taskConfig = new TaskConfigJava(config); broadcastStreams = taskConfig.getBroadcastSystemStreamPartitions(); }
/** * Get the name of the checkpoint manager factory * * @return Name of checkpoint manager factory */ public String getCheckpointManagerFactoryName() { return get(CHECKPOINT_MANAGER_FACTORY, null); }
/** * @return the set of SystemStreams which were configured as bootstrap streams. */ public Set<SystemStream> getBootstrapStreams() { Set<SystemStream> bootstrapInputs = new HashSet<>(); Set<SystemStream> allInputs = taskConfigJava.getAllInputStreams(); for (SystemStream systemStream : allInputs) { if (streamConfig.getBootstrapEnabled(systemStream)) { bootstrapInputs.add(systemStream); } } return Collections.unmodifiableSet(bootstrapInputs); }
@Override public void start() { // No-op JobModel jobModel = null; try { jobModel = getJobModel(); CheckpointManager checkpointManager = new TaskConfigJava(jobModel.getConfig()).getCheckpointManager(null); if (checkpointManager != null) { checkpointManager.createResources(); } ChangelogStreamManager.createChangelogStreams(config, jobModel.maxChangeLogStreamPartitions); } catch (Exception e) { LOGGER.error("Exception while trying to getJobModel.", e); if (coordinatorListener != null) { coordinatorListener.onCoordinatorFailure(e); } } if (jobModel != null && jobModel.getContainers().containsKey(processorId)) { if (coordinatorListener != null) { coordinatorListener.onJobModelExpired(); coordinatorListener.onNewJobModel(processorId, jobModel); } } else { LOGGER.info("JobModel: {} does not contain processorId: {}. Stopping the JobCoordinator", jobModel, processorId); stop(); } }
@Override public SystemStreamPartitionGrouper getSystemStreamPartitionGrouper(Config config) { if (!(new TaskConfigJava(config).getBroadcastSystemStreams().isEmpty())) { throw new ConfigException("The job configured with AllSspToSingleTaskGrouper cannot have broadcast streams."); } String processors = config.get(JobConfig.PROCESSOR_LIST()); List<String> processorList = Arrays.asList(processors.split(",")); if (processorList.isEmpty()) { throw new SamzaException("processor list cannot be empty!"); } return new AllSspToSingleTaskGrouper(processorList); } }
private Optional<StreamPartitionCountMonitor> getPartitionCountMonitor(Config config, SystemAdmins systemAdmins) { StreamMetadataCache streamMetadata = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance()); Set<SystemStream> inputStreamsToMonitor = new TaskConfigJava(config).getAllInputStreams(); if (inputStreamsToMonitor.isEmpty()) { throw new SamzaException("Input streams to a job can not be empty."); } return Optional.of(new StreamPartitionCountMonitor(inputStreamsToMonitor, streamMetadata, metrics, new JobConfig(config).getMonitorPartitionChangeFrequency(), streamsChanged -> { // Fail the jobs with durable state store. Otherwise, application state.status remains UNDEFINED s.t. YARN job will be restarted if (hasDurableStores) { log.error("Input topic partition count changed in a job with durable state. Failing the job."); state.status = SamzaApplicationState.SamzaAppStatus.FAILED; } coordinatorException = new PartitionChangeException("Input topic partition count changes detected."); })); }
this.applicationDefinedContainerContextFactoryOptional = applicationDefinedContainerContextFactoryOptional; this.applicationDefinedTaskContextFactoryOptional = applicationDefinedTaskContextFactoryOptional; this.taskShutdownMs = new TaskConfigJava(config).getShutdownMs(); this.jobCoordinator = (jobCoordinator != null) ? jobCoordinator : createJobCoordinator(); this.jobCoordinatorListener = createJobCoordinatorListener();
public DefaultChooserConfig(Config config) { super(config); taskConfigJava = new TaskConfigJava(config); streamConfig = new StreamConfig(config); }
/** * Get the SystemStreams for the configured input and broadcast streams. * * @return the set of SystemStreams for both standard inputs and broadcast stream inputs. */ public Set<SystemStream> getAllInputStreams() { Set<SystemStream> allInputSS = new HashSet<>(); TaskConfig taskConfig = TaskConfig.Config2Task(this); allInputSS.addAll((Set<? extends SystemStream>) JavaConverters.setAsJavaSetConverter(taskConfig.getInputStreams()).asJava()); allInputSS.addAll(getBroadcastSystemStreams()); return Collections.unmodifiableSet(allInputSS); }
/** * Get the SystemStreams for the configured broadcast streams. * * @return the set of SystemStreams for which there are broadcast stream SSPs configured. */ public Set<SystemStream> getBroadcastSystemStreams() { Set<SystemStream> broadcastSS = new HashSet<>(); Set<SystemStreamPartition> broadcastSSPs = getBroadcastSystemStreamPartitions(); for (SystemStreamPartition bssp : broadcastSSPs) { broadcastSS.add(bssp.getSystemStream()); } return Collections.unmodifiableSet(broadcastSS); }
/** * Create the checkpoint manager * * @param metricsRegistry Registry of metrics to use. Can be null if not using metrics. * @return CheckpointManager object if checkpoint manager factory is configured, otherwise null. */ public CheckpointManager getCheckpointManager(MetricsRegistry metricsRegistry) { // Initialize checkpoint streams during job coordination String checkpointManagerFactoryName = getCheckpointManagerFactoryName(); if (StringUtils.isNotBlank(checkpointManagerFactoryName)) { CheckpointManager checkpointManager = Util.getObj(checkpointManagerFactoryName, CheckpointManagerFactory.class).getCheckpointManager(this, metricsRegistry); return checkpointManager; } return null; }
List<String> systemStreamPartitions = getList(BROADCAST_INPUT_STREAMS, Collections.<String>emptyList());
CheckpointManager checkpointManager = new TaskConfigJava(config).getCheckpointManager(metrics.getMetricsRegistry()); if (checkpointManager != null) { checkpointManager.createResources();
@Override public SystemStreamPartitionGrouper getSystemStreamPartitionGrouper(Config config) { if (!(new TaskConfigJava(config).getBroadcastSystemStreams().isEmpty())) { throw new ConfigException("The job configured with AllSspToSingleTaskGrouper cannot have broadcast streams."); } String processors = config.get(JobConfig.PROCESSOR_LIST()); List<String> processorList = Arrays.asList(processors.split(",")); if (processorList.isEmpty()) { throw new SamzaException("processor list cannot be empty!"); } return new AllSspToSingleTaskGrouper(processorList); } }
@VisibleForTesting StreamPartitionCountMonitor getPartitionCountMonitor() { StreamMetadataCache streamMetadata = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance()); Set<SystemStream> inputStreamsToMonitor = new TaskConfigJava(config).getAllInputStreams(); return new StreamPartitionCountMonitor( inputStreamsToMonitor, streamMetadata, metrics.getMetricsRegistry(), new JobConfig(config).getMonitorPartitionChangeFrequency(), streamsChanged -> { if (leaderElector.amILeader()) { debounceTimer.scheduleAfterDebounceTime(ON_PROCESSOR_CHANGE, 0, this::doOnProcessorChange); } }); }
this.applicationDefinedContainerContextFactoryOptional = applicationDefinedContainerContextFactoryOptional; this.applicationDefinedTaskContextFactoryOptional = applicationDefinedTaskContextFactoryOptional; this.taskShutdownMs = new TaskConfigJava(config).getShutdownMs(); this.jobCoordinator = (jobCoordinator != null) ? jobCoordinator : createJobCoordinator(); this.jobCoordinatorListener = createJobCoordinatorListener();
public DefaultChooserConfig(Config config) { super(config); taskConfigJava = new TaskConfigJava(config); streamConfig = new StreamConfig(config); }
/** * Get the SystemStreams for the configured input and broadcast streams. * * @return the set of SystemStreams for both standard inputs and broadcast stream inputs. */ public Set<SystemStream> getAllInputStreams() { Set<SystemStream> allInputSS = new HashSet<>(); TaskConfig taskConfig = TaskConfig.Config2Task(this); allInputSS.addAll((Set<? extends SystemStream>) JavaConverters.setAsJavaSetConverter(taskConfig.getInputStreams()).asJava()); allInputSS.addAll(getBroadcastSystemStreams()); return Collections.unmodifiableSet(allInputSS); }
/** * Get the SystemStreams for the configured broadcast streams. * * @return the set of SystemStreams for which there are broadcast stream SSPs configured. */ public Set<SystemStream> getBroadcastSystemStreams() { Set<SystemStream> broadcastSS = new HashSet<>(); Set<SystemStreamPartition> broadcastSSPs = getBroadcastSystemStreamPartitions(); for (SystemStreamPartition bssp : broadcastSSPs) { broadcastSS.add(bssp.getSystemStream()); } return Collections.unmodifiableSet(broadcastSS); }
/** * Create the checkpoint manager * * @param metricsRegistry Registry of metrics to use. Can be null if not using metrics. * @return CheckpointManager object if checkpoint manager factory is configured, otherwise null. */ public CheckpointManager getCheckpointManager(MetricsRegistry metricsRegistry) { // Initialize checkpoint streams during job coordination String checkpointManagerFactoryName = getCheckpointManagerFactoryName(); if (StringUtils.isNotBlank(checkpointManagerFactoryName)) { CheckpointManager checkpointManager = Util.getObj(checkpointManagerFactoryName, CheckpointManagerFactory.class).getCheckpointManager(this, metricsRegistry); return checkpointManager; } return null; }