final MasterConfig config = new MasterConfig() .setZooKeeperConnectString(getZooKeeperConnectString()) .setZooKeeperSessionTimeoutMillis(getZooKeeperSessionTimeoutMillis()) .setZooKeeperConnectionTimeoutMillis(getZooKeeperConnectionTimeoutMillis()) .setZooKeeperClusterId(getZooKeeperClusterId()) .setNoZooKeeperMasterRegistration(getNoZooKeeperRegistration()) .setZooKeeperEnableAcls(getZooKeeperEnableAcls()) .setZookeeperAclAgentUser(getZooKeeperAclAgentUser()) .setZooKeeperAclAgentDigest(options.getString(zkAclAgentDigest.getDest())) .setZookeeperAclMasterUser(getZooKeeperAclMasterUser()) .setZooKeeperAclMasterPassword(masterPassword) .setDomain(getDomain()) .setName(getName()) .setStatsdHostPort(getStatsdHostPort()) .setInhibitMetrics(getInhibitMetrics()) .setSentryDsn(getSentryDsn()) .setServiceRegistryAddress(getServiceRegistryAddress()) .setServiceRegistrarPlugin(getServiceRegistrarPlugin()) .setAdminEndpoint(adminAddress) .setHttpEndpoint(httpAddress) .setKafkaBrokers(getKafkaBrokers()) .setPubsubPrefixes(getPubsubPrefixes()) .setStateDirectory(getStateDirectory()) .setAgentReapingTimeout(options.getLong(agentReapingTimeout.getDest())) .setJobRetention(options.getLong(jobRetention.getDest())) .setFfwdConfig(ffwdConfig(options)) .setWhitelistedCapabilities(ImmutableSet.copyOf( options.getList(whitelistedCapabilities.getDest()))) .setJobHistoryReapingEnabled(options.getBoolean(jobHistoryReapingEnabled.getDest()));
@Override protected void startUp() throws Exception { logBanner(); if (!config.getNoZooKeeperMasterRegistration()) { zkRegistrar.startAsync().awaitRunning(); } expiredJobReaper.startAsync().awaitRunning(); rollingUpdateService.startAsync().awaitRunning(); agentReaper.ifPresent(reaper -> reaper.startAsync().awaitRunning()); oldJobReaper.ifPresent(reaper -> reaper.startAsync().awaitRunning()); jobHistoryReaper.ifPresent(reaper -> reaper.startAsync().awaitRunning()); try { server.start(); } catch (Exception e) { log.error("Unable to start server, shutting down", e); server.stop(); } final ServiceRegistration serviceRegistration = ServiceRegistration.newBuilder() .endpoint("helios", "http", config.getHttpEndpoint().getPort(), config.getDomain(), config.getName()) .build(); registrar.register(serviceRegistration); }
public MasterMain(final CuratorClientFactory curatorClientFactory, final MasterConfig masterConfig, final LoggingConfig loggingConfig) { super(loggingConfig, masterConfig.getSentryDsn()); this.masterConfig = masterConfig; this.curatorClientFactory = curatorClientFactory; }
List<AuthInfo> authorization = null; final String masterUser = config.getZookeeperAclMasterUser(); final String masterPassword = config.getZooKeeperAclMasterPassword(); final String agentUser = config.getZookeeperAclAgentUser(); final String agentDigest = config.getZooKeeperAclAgentDigest(); if (config.isZooKeeperEnableAcls()) { if (isNullOrEmpty(masterUser) || isNullOrEmpty(masterPassword)) { throw new HeliosRuntimeException( config.getZooKeeperConnectionString(), config.getZooKeeperSessionTimeoutMillis(), config.getZooKeeperConnectionTimeoutMillis(), zooKeeperRetryPolicy, aclProvider, authorization); final ZooKeeperClient client = new DefaultZooKeeperClient(curator, config.getZooKeeperClusterId()); client.start(); zkRegistrar = ZooKeeperRegistrarService.newBuilder() .setZooKeeperClient(client) .setZooKeeperRegistrar(new MasterZooKeeperRegistrar(config.getName())) .build();
public MasterParser(final String... args) throws ArgumentParserException { super("helios-master", "Spotify Helios Master", args); final Namespace options = getNamespace(); final InetSocketAddress httpAddress = parseSocketAddress(options.getString(httpArg.getDest())); final MasterConfig config = new MasterConfig() .setZooKeeperConnectString(getZooKeeperConnectString()) .setZooKeeperSessionTimeoutMillis(getZooKeeperSessionTimeoutMillis()) .setZooKeeperConnectionTimeoutMillis(getZooKeeperConnectionTimeoutMillis()) .setZooKeeperNamespace(getZooKeeperNamespace()) .setZooKeeperClusterId(getZooKeeperClusterId()) .setNoZooKeeperMasterRegistration(getNoZooKeeperRegistration()) .setDomain(getDomain()) .setName(getName()) .setStatsdHostPort(getStatsdHostPort()) .setRiemannHostPort(getRiemannHostPort()) .setInhibitMetrics(getInhibitMetrics()) .setSentryDsn(getSentryDsn()) .setServiceRegistryAddress(getServiceRegistryAddress()) .setServiceRegistrarPlugin(getServiceRegistrarPlugin()) .setAdminPort(options.getInt(adminArg.getDest())) .setHttpEndpoint(httpAddress) .setKafkaBrokers(getKafkaBrokers()) .setStateDirectory(getStateDirectory()); this.masterConfig = config; }
if (config.isInhibitMetrics()) { metrics = new NoopMetrics(); } else { metrics = new MetricsImpl(metricsRegistry, MetricsImpl.Type.MASTER); metrics.start(); if (!Strings.isNullOrEmpty(config.getStatsdHostPort())) { environment.lifecycle().manage(new ManagedStatsdReporter(config.getStatsdHostPort(), metricsRegistry)); final FastForwardConfig ffwdConfig = config.getFfwdConfig(); if (ffwdConfig != null) { environment.lifecycle().manage(FastForwardReporter.create( final Path stateDirectory = config.getStateDirectory().toAbsolutePath().normalize(); if (!Files.exists(stateDirectory)) { try { new ZooKeeperMasterModel(zkClientProvider, config.getName(), eventSenders, DG_EVENT_TOPIC); this.registrar = createServiceRegistrar(config.getServiceRegistrarPlugin(), config.getServiceRegistryAddress(), config.getDomain()); if (config.getAgentReapingTimeout() > 0) { this.agentReaper = Optional.of(new DeadAgentReaper(model, config.getAgentReapingTimeout())); } else { log.info("Reaping of dead agents disabled");
config.getRiemannHostPort(), config.getName(), "helios-master"); final RiemannFacade riemannFacade = riemannSupport.getFacade(); log.info("Starting metrics"); final Metrics metrics; if (config.isInhibitMetrics()) { metrics = new NoopMetrics(); } else { metrics.start(); environment.lifecycle().manage(riemannSupport); environment.lifecycle().manage(new ManagedStatsdReporter(config.getStatsdHostPort(), "helios-master", metricsRegistry)); zooKeeperClient, modelReporter); final KafkaClientProvider kafkaClientProvider = new KafkaClientProvider( config.getKafkaBrokers()); final Path stateDirectory = config.getStateDirectory().toAbsolutePath().normalize(); if (!Files.exists(stateDirectory)) { try { new ZooKeeperMasterModel(zkClientProvider, config.getName(), kafkaSender); this.registrar = createServiceRegistrar(config.getServiceRegistrarPlugin(), config.getServiceRegistryAddress(), config.getDomain()); config.getHttpEndpoint(), config.getAdminPort(), false);
/** * Create a Zookeeper client and create the control and state nodes if needed. * * @param config The service configuration. * @return A zookeeper client. */ private ZooKeeperClient setupZookeeperClient(final MasterConfig config) { final RetryPolicy zooKeeperRetryPolicy = new ExponentialBackoffRetry(1000, 3); final CuratorFramework curator = curatorClientFactory.newClient( config.getZooKeeperConnectionString(), config.getZooKeeperSessionTimeoutMillis(), config.getZooKeeperConnectionTimeoutMillis(), zooKeeperRetryPolicy, config.getZooKeeperNamespace()); final ZooKeeperClient client = new DefaultZooKeeperClient(curator, config.getZooKeeperClusterId()); client.start(); zkRegistrar = new ZooKeeperRegistrar(client, new MasterZooKeeperRegistrar(config.getName())); return client; } }
@Override protected void startUp() throws Exception { logBanner(); if (!config.getNoZooKeeperMasterRegistration()) { zkRegistrar.startAsync().awaitRunning(); } expiredJobReaper.startAsync().awaitRunning(); rollingUpdateService.startAsync().awaitRunning(); try { server.start(); } catch (Exception e) { log.error("Unable to start server, shutting down", e); server.stop(); } final ServiceRegistration serviceRegistration = ServiceRegistration.newBuilder() .endpoint("helios", "http", config.getHttpEndpoint().getPort(), config.getDomain(), config.getName()) .build(); registrar.register(serviceRegistration); }
public MasterMain(final CuratorClientFactory curatorClientFactory, final MasterConfig masterConfig, final LoggingConfig loggingConfig) { super(loggingConfig, masterConfig.getSentryDsn()); this.masterConfig = masterConfig; this.curatorClientFactory = curatorClientFactory; }