public AbstractContainerAllocator(ClusterResourceManager containerProcessManager, ResourceRequestState resourceRequestState, Config config, SamzaApplicationState state) { ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config); this.clusterResourceManager = containerProcessManager; this.allocatorSleepIntervalMs = clusterManagerConfig.getAllocatorSleepTime(); this.resourceRequestState = resourceRequestState; this.containerMemoryMb = clusterManagerConfig.getContainerMemoryMb(); this.containerNumCpuCores = clusterManagerConfig.getNumCores(); this.taskConfig = new TaskConfig(config); this.state = state; this.config = config; }
public int getNumCores() { if (containsKey(CLUSTER_MANAGER_MAX_CORES)) { return getInt(CLUSTER_MANAGER_MAX_CORES); } else if (containsKey(CONTAINER_MAX_CPU_CORES)) { log.info("Configuration {} is deprecated. Please use {}", CONTAINER_MAX_CPU_CORES, CLUSTER_MANAGER_MAX_CORES); return getInt(CONTAINER_MAX_CPU_CORES); } else { return DEFAULT_CPU_CORES; } }
public String getContainerManagerClass() { return get(CLUSTER_MANAGER_FACTORY, CLUSTER_MANAGER_FACTORY_DEFAULT); }
public boolean getHostAffinityEnabled() { if (containsKey(JOB_HOST_AFFINITY_ENABLED)) { return getBoolean(JOB_HOST_AFFINITY_ENABLED); } else if (containsKey(HOST_AFFINITY_ENABLED)) { log.warn("Configuration {} is deprecated. Please use {}", HOST_AFFINITY_ENABLED, JOB_HOST_AFFINITY_ENABLED); return getBoolean(HOST_AFFINITY_ENABLED); } else { return false; } }
ContainerProcessManager(Config config, SamzaApplicationState state, MetricsRegistryMap registry, ClusterResourceManager resourceManager) { JobModelManager jobModelManager = state.jobModelManager; this.state = state; this.clusterManagerConfig = new ClusterManagerConfig(config); this.jobConfig = new JobConfig(config); this.hostAffinityEnabled = clusterManagerConfig.getHostAffinityEnabled(); this.clusterResourceManager = resourceManager; this.metrics = new ContainerProcessManagerMetrics(config, state, registry); if (this.hostAffinityEnabled) { this.containerAllocator = new HostAwareContainerAllocator(clusterResourceManager, clusterManagerConfig.getContainerRequestTimeout(), config, state); } else { this.containerAllocator = new ContainerAllocator(clusterResourceManager, config, state); } this.allocatorThread = new Thread(this.containerAllocator, "Container Allocator Thread"); log.info("finished initialization of samza task manager"); }
clusterManagerConfig = new ClusterManagerConfig(config); isJmxEnabled = clusterManagerConfig.getJmxEnabledOnJobCoordinator(); jobCoordinatorSleepInterval = clusterManagerConfig.getJobCoordinatorSleepInterval();
ContainerProcessManager(Config config, SamzaApplicationState state, MetricsRegistryMap registry, AbstractContainerAllocator allocator, ClusterResourceManager manager) { this.state = state; this.clusterManagerConfig = new ClusterManagerConfig(config); this.jobConfig = new JobConfig(config); this.hostAffinityEnabled = clusterManagerConfig.getHostAffinityEnabled(); this.clusterResourceManager = manager; this.metrics = new ContainerProcessManagerMetrics(config, state, registry); this.containerAllocator = allocator; this.allocatorThread = new Thread(this.containerAllocator, "Container Allocator Thread"); }
ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config); this.lifecycle = new SamzaYarnAppMasterLifecycle( clusterManagerConfig.getContainerMemoryMb(), clusterManagerConfig.getNumCores(), samzaAppState, state,
int retryCount = clusterManagerConfig.getContainerRetryCount(); int retryWindowMs = clusterManagerConfig.getContainerRetryWindowMs();
/** * Returns an instantiated {@link ResourceManagerFactory} from a {@link ClusterManagerConfig}. The * {@link ResourceManagerFactory} is used to return an implementation of a {@link ClusterResourceManager} * * @param clusterManagerConfig, the cluster manager config to parse. * */ private ResourceManagerFactory getContainerProcessManagerFactory(final ClusterManagerConfig clusterManagerConfig) { final String containerManagerFactoryClass = clusterManagerConfig.getContainerManagerClass(); final ResourceManagerFactory factory; try { factory = Util.getObj(containerManagerFactoryClass, ResourceManagerFactory.class); } catch (Exception e) { log.error("Exception when creating ContainerManager", e); throw new SamzaException(e); } return factory; }
ContainerProcessManager(Config config, SamzaApplicationState state, MetricsRegistryMap registry, ClusterResourceManager resourceManager) { JobModelManager jobModelManager = state.jobModelManager; this.state = state; this.clusterManagerConfig = new ClusterManagerConfig(config); this.jobConfig = new JobConfig(config); this.hostAffinityEnabled = clusterManagerConfig.getHostAffinityEnabled(); this.clusterResourceManager = resourceManager; this.metrics = new ContainerProcessManagerMetrics(config, state, registry); if (this.hostAffinityEnabled) { this.containerAllocator = new HostAwareContainerAllocator(clusterResourceManager, clusterManagerConfig.getContainerRequestTimeout(), config, state); } else { this.containerAllocator = new ContainerAllocator(clusterResourceManager, config, state); } this.allocatorThread = new Thread(this.containerAllocator, "Container Allocator Thread"); log.info("finished initialization of samza task manager"); }
clusterManagerConfig = new ClusterManagerConfig(config); isJmxEnabled = clusterManagerConfig.getJmxEnabledOnJobCoordinator(); jobCoordinatorSleepInterval = clusterManagerConfig.getJobCoordinatorSleepInterval();
ContainerProcessManager(Config config, SamzaApplicationState state, MetricsRegistryMap registry, AbstractContainerAllocator allocator, ClusterResourceManager manager) { this.state = state; this.clusterManagerConfig = new ClusterManagerConfig(config); this.jobConfig = new JobConfig(config); this.hostAffinityEnabled = clusterManagerConfig.getHostAffinityEnabled(); this.clusterResourceManager = manager; this.metrics = new ContainerProcessManagerMetrics(config, state, registry); this.containerAllocator = allocator; this.allocatorThread = new Thread(this.containerAllocator, "Container Allocator Thread"); }
public boolean getHostAffinityEnabled() { if (containsKey(CLUSTER_MANAGER_HOST_AFFINITY_ENABLED)) { return getBoolean(CLUSTER_MANAGER_HOST_AFFINITY_ENABLED); } else if (containsKey(HOST_AFFINITY_ENABLED)) { log.info("Configuration {} is deprecated. Please use {}", HOST_AFFINITY_ENABLED, CLUSTER_MANAGER_HOST_AFFINITY_ENABLED); return getBoolean(HOST_AFFINITY_ENABLED); } else { return false; } }
ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config); this.lifecycle = new SamzaYarnAppMasterLifecycle( clusterManagerConfig.getContainerMemoryMb(), clusterManagerConfig.getNumCores(), samzaAppState, state,
int retryCount = clusterManagerConfig.getContainerRetryCount(); int retryWindowMs = clusterManagerConfig.getContainerRetryWindowMs();
/** * Returns an instantiated {@link ResourceManagerFactory} from a {@link ClusterManagerConfig}. The * {@link ResourceManagerFactory} is used to return an implementation of a {@link ClusterResourceManager} * * @param clusterManagerConfig, the cluster manager config to parse. * */ private ResourceManagerFactory getContainerProcessManagerFactory(final ClusterManagerConfig clusterManagerConfig) { final String containerManagerFactoryClass = clusterManagerConfig.getContainerManagerClass(); final ResourceManagerFactory factory; try { factory = Util.getObj(containerManagerFactoryClass, ResourceManagerFactory.class); } catch (Exception e) { log.error("Exception when creating ContainerManager", e); throw new SamzaException(e); } return factory; }
public AbstractContainerAllocator(ClusterResourceManager containerProcessManager, ResourceRequestState resourceRequestState, Config config, SamzaApplicationState state) { ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config); this.clusterResourceManager = containerProcessManager; this.allocatorSleepIntervalMs = clusterManagerConfig.getAllocatorSleepTime(); this.resourceRequestState = resourceRequestState; this.containerMemoryMb = clusterManagerConfig.getContainerMemoryMb(); this.containerNumCpuCores = clusterManagerConfig.getNumCores(); this.taskConfig = new TaskConfig(config); this.state = state; this.config = config; }
public int getContainerRetryCount() { if (containsKey(CLUSTER_MANAGER_CONTAINER_RETRY_COUNT)) return getInt(CLUSTER_MANAGER_CONTAINER_RETRY_COUNT); else if (containsKey(CONTAINER_RETRY_COUNT)) { log.info("Configuration {} is deprecated. Please use {}", CONTAINER_RETRY_COUNT, CLUSTER_MANAGER_CONTAINER_RETRY_COUNT); return getInt(CONTAINER_RETRY_COUNT); } else { return DEFAULT_CONTAINER_RETRY_COUNT; } }
ContainerProcessManager(Config config, SamzaApplicationState state, MetricsRegistryMap registry, ClusterResourceManager resourceManager) { JobModelManager jobModelManager = state.jobModelManager; this.state = state; this.clusterManagerConfig = new ClusterManagerConfig(config); this.jobConfig = new JobConfig(config); this.hostAffinityEnabled = clusterManagerConfig.getHostAffinityEnabled(); this.clusterResourceManager = resourceManager; this.metrics = new ContainerProcessManagerMetrics(config, state, registry); if (this.hostAffinityEnabled) { this.containerAllocator = new HostAwareContainerAllocator(clusterResourceManager, clusterManagerConfig.getContainerRequestTimeout(), config, state); } else { this.containerAllocator = new ContainerAllocator(clusterResourceManager, config, state); } this.allocatorThread = new Thread(this.containerAllocator, "Container Allocator Thread"); log.info("finished initialization of samza task manager"); }