private int getMaxConcurrentRunsOneFlow(final Props azkProps) { // The default threshold is set to 30 for now, in case some users are affected. We may // decrease this number in future, to better prevent DDos attacks. return azkProps.getInt(ConfigurationKeys.MAX_CONCURRENT_RUNS_ONEFLOW, DEFAULT_MAX_ONCURRENT_RUNS_ONEFLOW); }
private int getMaxConcurrentRunsOneFlow(final Props azkProps) { // The default threshold is set to 30 for now, in case some users are affected. We may // decrease this number in future, to better prevent DDos attacks. return azkProps.getInt(ConfigurationKeys.MAX_CONCURRENT_RUNS_ONEFLOW, DEFAULT_MAX_ONCURRENT_RUNS_ONEFLOW); }
public int getRetries() { return this.inputProps.getInt("retries", 0); }
private ExecutorService createExecutorInfoRefresherService() { return Executors.newFixedThreadPool(this.azkProps.getInt( ConfigurationKeys.EXECUTORINFO_REFRESH_MAX_THREADS, 5)); }
public AbstractMailer(final Props props, final EmailMessageCreator messageCreator) { this.azkabanName = props.getString("azkaban.name", "azkaban"); this.messageCreator = messageCreator; final long maxAttachmentSizeInMB = props.getInt("mail.max.attachment.size.mb", 100); this.attachmentMazSizeInByte = maxAttachmentSizeInMB * MB_IN_BYTES; }
/** * Constructor taking global props. */ @Inject public SessionCache(final Props props) { this.effectiveSessionTimeToLive = props.getLong(ConfigurationKeys.SESSION_TIME_TO_LIVE, DEFAULT_SESSION_TIME_TO_LIVE); this.cache = CacheBuilder.newBuilder() .maximumSize(props.getInt("max.num.sessions", MAX_NUM_SESSIONS)) .expireAfterAccess(effectiveSessionTimeToLive, TimeUnit.MILLISECONDS) .build(); }
@Inject public StorageCleaner(final Props props, final Storage storage, final DatabaseOperator databaseOperator) { this.storage = storage; this.databaseOperator = databaseOperator; this.maxArtifactsPerProject = props.getInt(AZKABAN_STORAGE_ARTIFACT_MAX_RETENTION, 0); checkArgument(this.maxArtifactsPerProject >= 0, String.format("Invalid value for %s : %d", AZKABAN_STORAGE_ARTIFACT_MAX_RETENTION, this.maxArtifactsPerProject)); if (isCleanupPermitted()) { log.info(String.format("%s Config: Max %d artifact(s) retained per project", AZKABAN_STORAGE_ARTIFACT_MAX_RETENTION, this.maxArtifactsPerProject)); } else { log.warn("Project cleanup disabled. All artifacts will be stored."); } }
@Inject AzkabanProjectLoader(final Props props, final ProjectLoader projectLoader, final StorageManager storageManager, final FlowLoaderFactory flowLoaderFactory, final ExecutorLoader executorLoader) { this.props = requireNonNull(props, "Props is null"); this.projectLoader = requireNonNull(projectLoader, "project Loader is null"); this.storageManager = requireNonNull(storageManager, "Storage Manager is null"); this.flowLoaderFactory = requireNonNull(flowLoaderFactory, "Flow Loader Factory is null"); this.tempDir = new File(props.getString(ConfigurationKeys.PROJECT_TEMP_DIR, "temp")); this.executorLoader = executorLoader; if (!this.tempDir.exists()) { log.info("Creating temp dir: " + this.tempDir.getAbsolutePath()); this.tempDir.mkdirs(); } else { log.info("Using temp dir: " + this.tempDir.getAbsolutePath()); } this.projectVersionRetention = props.getInt(ConfigurationKeys.PROJECT_VERSION_RETENTION, 3); log.info("Project version retention is set to " + this.projectVersionRetention); }
/** * Make sure all the job callback related properties are valid * * @return number of valid job callback properties. Mainly for testing purpose. */ public static int validate(final String jobName, final Props serverProps, final Props jobProps, final Collection<String> errors) { final int maxNumCallback = serverProps.getInt( JobCallbackConstants.MAX_CALLBACK_COUNT_PROPERTY_KEY, JobCallbackConstants.DEFAULT_MAX_CALLBACK_COUNT); final int maxPostBodyLength = serverProps.getInt(MAX_POST_BODY_LENGTH_PROPERTY_KEY, DEFAULT_POST_BODY_LENGTH); int totalCallbackCount = 0; for (final JobCallbackStatusEnum jobStatus : JobCallbackStatusEnum.values()) { totalCallbackCount += validateBasedOnStatus(jobProps, errors, jobStatus, maxNumCallback, maxPostBodyLength); } if (logger.isDebugEnabled()) { logger.debug("Found " + totalCallbackCount + " job callbacks for job " + jobName); } return totalCallbackCount; }
private QueueProcessorThread setupQueueProcessor() { return new QueueProcessorThread( this.azkProps.getBoolean(Constants.ConfigurationKeys.QUEUEPROCESSING_ENABLED, true), this.azkProps.getLong(Constants.ConfigurationKeys.ACTIVE_EXECUTOR_REFRESH_IN_MS, 50000), this.azkProps.getInt( Constants.ConfigurationKeys.ACTIVE_EXECUTOR_REFRESH_IN_NUM_FLOW, 5), this.azkProps.getInt( Constants.ConfigurationKeys.MAX_DISPATCHING_ERRORS_PERMITTED, this.activeExecutors.getAll().size()), this.sleepAfterDispatchFailure); }
private static void updateDerivedConfigs(final Props azkabanSettings) { final boolean isSslEnabled = azkabanSettings.getBoolean("jetty.use.ssl", true); final int port = isSslEnabled ? azkabanSettings.getInt("jetty.ssl.port", DEFAULT_SSL_PORT_NUMBER) : azkabanSettings.getInt("jetty.port", DEFAULT_PORT_NUMBER); // setting stats configuration for connectors final String hostname = azkabanSettings.getString("jetty.hostname", "localhost"); azkabanSettings.put("server.hostname", hostname); azkabanSettings.put("server.port", port); azkabanSettings.put("server.useSSL", String.valueOf(isSslEnabled)); }
@Inject public Emailer(final Props props, final CommonMetrics commonMetrics, final EmailMessageCreator messageCreator, final ExecutorLoader executorLoader) { super(props, messageCreator); this.executorLoader = requireNonNull(executorLoader, "executorLoader is null."); this.commonMetrics = requireNonNull(commonMetrics, "commonMetrics is null."); this.azkabanName = props.getString("azkaban.name", "azkaban"); final int mailTimeout = props.getInt("mail.timeout.millis", 30000); EmailMessage.setTimeout(mailTimeout); final int connectionTimeout = props.getInt("mail.connection.timeout.millis", 30000); EmailMessage.setConnectionTimeout(connectionTimeout); EmailMessage.setTotalAttachmentMaxSize(getAttachmentMaxSize()); this.clientHostname = props.getString(ConfigurationKeys.AZKABAN_WEBSERVER_EXTERNAL_HOSTNAME, props.getString("jetty.hostname", "localhost")); if (props.getBoolean("jetty.use.ssl", true)) { this.scheme = HTTPS; this.clientPortNumber = Integer.toString(props .getInt(ConfigurationKeys.AZKABAN_WEBSERVER_EXTERNAL_SSL_PORT, props.getInt("jetty.ssl.port", Constants.DEFAULT_SSL_PORT_NUMBER))); } else { this.scheme = HTTP; this.clientPortNumber = Integer.toString( props.getInt(ConfigurationKeys.AZKABAN_WEBSERVER_EXTERNAL_PORT, props.getInt("jetty.port", Constants.DEFAULT_PORT_NUMBER))); } }
@Inject public ExecutorHealthChecker(final Props azkProps, final ExecutorLoader executorLoader, final ExecutorApiGateway apiGateway, final AlerterHolder alerterHolder) { this.healthCheckIntervalMin = azkProps .getLong(ConfigurationKeys.AZKABAN_EXECUTOR_HEALTHCHECK_INTERVAL_MIN, DEFAULT_EXECUTOR_HEALTHCHECK_INTERVAL.toMinutes()); this.executorMaxFailureCount = azkProps.getInt(ConfigurationKeys .AZKABAN_EXECUTOR_MAX_FAILURE_COUNT, DEFAULT_EXECUTOR_MAX_FAILURE_COUNT); this.alertEmails = azkProps.getStringList(ConfigurationKeys.AZKABAN_ADMIN_ALERT_EMAIL); this.scheduler = Executors.newSingleThreadScheduledExecutor(); this.executorLoader = executorLoader; this.apiGateway = apiGateway; this.alerterHolder = alerterHolder; }
@Override public Server get() { requireNonNull(this.props); final int maxThreads = this.props .getInt("jetty.maxThreads", Constants.DEFAULT_JETTY_MAX_THREAD_COUNT); final boolean useSsl = this.props.getBoolean("jetty.use.ssl", true); final int port; final Server server = new Server(); if (useSsl) { final int sslPortNumber = this.props .getInt("jetty.ssl.port", Constants.DEFAULT_SSL_PORT_NUMBER); port = sslPortNumber; server.addConnector(getSslSocketConnector(sslPortNumber)); } else { port = this.props.getInt("jetty.port", Constants.DEFAULT_PORT_NUMBER); server.addConnector(getSocketConnector(port)); } // setting stats configuration for connectors setStatsOnConnectors(server); logger.info(String.format( "Starting %sserver on port: %d # Max threads: %d", useSsl ? "SSL " : "", port, maxThreads)); return server; }
private void createThreadPool() { final int maxThreads = this.props .getInt("jetty.maxThreads", Constants.DEFAULT_JETTY_MAX_THREAD_COUNT); final QueuedThreadPool httpThreadPool = new QueuedThreadPool(maxThreads); this.server.setThreadPool(httpThreadPool); addThreadPoolGauges(httpThreadPool); }
/** * Create Datasource from parameters in the properties */ public static AzkabanDataSource getDataSource(final Props props) { final String databaseType = props.getString("database.type"); AzkabanDataSource dataSource = null; if (databaseType.equals("mysql")) { final int port = props.getInt("mysql.port"); final String host = props.getString("mysql.host"); final String database = props.getString("mysql.database"); final String user = props.getString("mysql.user"); final String password = props.getString("mysql.password"); final int numConnections = props.getInt("mysql.numconnections"); dataSource = getMySQLDataSource(host, port, database, user, password, numConnections); } else if (databaseType.equals("h2")) { final String path = props.getString("h2.path"); final Path h2DbPath = Paths.get(path).toAbsolutePath(); logger.info("h2 DB path: " + h2DbPath); dataSource = getH2DataSource(h2DbPath); } return dataSource; }
@Inject public EmailMessageCreator(final Props props) { this.mailHost = props.getString("mail.host", "localhost"); this.mailPort = props.getInt("mail.port", DEFAULT_SMTP_PORT); this.mailUser = props.getString("mail.user", ""); this.mailPassword = props.getString("mail.password", ""); this.mailSender = props.getString("mail.sender", ""); this.tls = props.getString("mail.tls", "false"); this.usesAuth = props.getBoolean("mail.useAuth", true); }
@Override public void init(final ServletConfig config) throws ServletException { this.application = SERVICE_PROVIDER.getInstance(AzkabanWebServer.class); if (this.application == null) { throw new IllegalStateException( "No batch application is defined in the servlet context!"); } final Props props = this.application.getServerProps(); this.name = props.getString("azkaban.name", ""); this.label = props.getString("azkaban.label", ""); this.color = props.getString("azkaban.color", "#FF0000"); this.passwordPlaceholder = props.getString("azkaban.password.placeholder", "Password"); this.displayExecutionPageSize = props.getInt(ConfigurationKeys.DISPLAY_EXECUTION_PAGE_SIZE, 16); if (this.application instanceof AzkabanWebServer) { final AzkabanWebServer server = (AzkabanWebServer) this.application; this.viewerPlugins = PluginRegistry.getRegistry().getViewerPlugins(); this.triggerPlugins = new ArrayList<>(server.getTriggerPlugins().values()); } }
@Inject public MySQLDataSource(final Props props, final DBMetrics dbMetrics) { super(); this.dbMetrics = dbMetrics; final int port = props.getInt("mysql.port"); final String host = props.getString("mysql.host"); final String dbName = props.getString("mysql.database"); final String user = props.getString("mysql.user"); final String password = props.getString("mysql.password"); final int numConnections = props.getInt("mysql.numconnections"); final String url = "jdbc:mysql://" + (host + ":" + port + "/" + dbName); addConnectionProperty("useUnicode", "yes"); addConnectionProperty("characterEncoding", "UTF-8"); setDriverClassName("com.mysql.jdbc.Driver"); setUsername(user); setPassword(password); setUrl(url); setMaxTotal(numConnections); setValidationQuery("/* ping */ select 1"); setTestOnBorrow(true); } /**
@Override public void init(final ServletConfig config) throws ServletException { super.init(config); final AzkabanWebServer server = (AzkabanWebServer) getApplication(); this.projectManager = server.getProjectManager(); this.executorManagerAdapter = server.getExecutorManager(); this.scheduleManager = server.getScheduleManager(); this.userManager = server.getUserManager(); this.scheduler = server.getScheduler(); this.lockdownCreateProjects = server.getServerProps().getBoolean(LOCKDOWN_CREATE_PROJECTS_KEY, false); this.enableQuartz = server.getServerProps().getBoolean(ConfigurationKeys.ENABLE_QUARTZ, false); if (this.lockdownCreateProjects) { logger.info("Creation of projects is locked down"); } this.lockdownUploadProjects = server.getServerProps().getBoolean(LOCKDOWN_UPLOAD_PROJECTS_KEY, false); if (this.lockdownUploadProjects) { logger.info("Uploading of projects is locked down"); } this.downloadBufferSize = server.getServerProps().getInt(PROJECT_DOWNLOAD_BUFFER_SIZE_IN_BYTES, 8192); logger.info("downloadBufferSize: " + this.downloadBufferSize); }