/** * Extracts and returns a configuration map containing container related Twill settings. * * @param args the arguments to use for looking up resources configurations * @param containerMemory the container memory size in MB * @return a map of configurations */ public static Map<String, String> getTwillContainerConfigs(Map<String, String> args, int containerMemory) { Integer reservedMemory = getPositiveInt(args, RESERVED_MEMORY_KEY_OVERRIDE, "reserved memory size"); if (reservedMemory == null) { return Collections.emptyMap(); } int heapMemory = containerMemory - reservedMemory; if (heapMemory <= 0) { LOG.warn("Ignoring invalid reserved memory size '{}' from runtime arguments. " + "It must be smaller than container memory size '{}'", reservedMemory, containerMemory); return Collections.emptyMap(); } double ratio = ((double) heapMemory) / containerMemory; Map<String, String> config = new HashMap<>(); config.put(Configs.Keys.JAVA_RESERVED_MEMORY_MB, reservedMemory.toString()); config.put(Configs.Keys.HEAP_RESERVED_MIN_RATIO, String.format("%.2f", ratio)); return config; }
/** * Extracts and returns a configuration map containing container related Twill settings. * * @param args the arguments to use for looking up resources configurations * @param containerMemory the container memory size in MB * @return a map of configurations */ public static Map<String, String> getTwillContainerConfigs(Map<String, String> args, int containerMemory) { Integer reservedMemory = getPositiveInt(args, RESERVED_MEMORY_KEY_OVERRIDE, "reserved memory size"); if (reservedMemory == null) { return Collections.emptyMap(); } int heapMemory = containerMemory - reservedMemory; if (heapMemory <= 0) { LOG.warn("Ignoring invalid reserved memory size '{}' from runtime arguments. " + "It must be smaller than container memory size '{}'", reservedMemory, containerMemory); return Collections.emptyMap(); } double ratio = ((double) heapMemory) / containerMemory; Map<String, String> config = new HashMap<>(); config.put(Configs.Keys.JAVA_RESERVED_MEMORY_MB, reservedMemory.toString()); config.put(Configs.Keys.HEAP_RESERVED_MIN_RATIO, String.format("%.2f", ratio)); return config; }
/** * Returns the twill configurations to be used in {@link TwillPreparer#withConfiguration(Map)} based on the * runtime arguments. */ public static Map<String, String> getTwillApplicationConfigs(Map<String, String> args) { Map<String, String> result = new HashMap<>(); Integer maxAttempts = getPositiveInt(args, YARN_MAX_APP_ATTEMPTS, YARN_MAX_APP_ATTEMPTS); if (maxAttempts != null) { result.put(Configs.Keys.YARN_MAX_APP_ATTEMPTS, maxAttempts.toString()); } Long failureValidityInterval = getLong(args, YARN_ATTEMPT_FAILURES_VALIDITY_INTERVAL, YARN_ATTEMPT_FAILURES_VALIDITY_INTERVAL); if (failureValidityInterval != null) { result.put(Configs.Keys.YARN_ATTEMPT_FAILURES_VALIDITY_INTERVAL, failureValidityInterval.toString()); } return result; }
/** * Returns the twill configurations to be used in {@link TwillPreparer#withConfiguration(Map)} based on the * runtime arguments. */ public static Map<String, String> getTwillApplicationConfigs(Map<String, String> args) { Map<String, String> result = new HashMap<>(); Integer maxAttempts = getPositiveInt(args, YARN_MAX_APP_ATTEMPTS, YARN_MAX_APP_ATTEMPTS); if (maxAttempts != null) { result.put(Configs.Keys.YARN_MAX_APP_ATTEMPTS, maxAttempts.toString()); } Long failureValidityInterval = getLong(args, YARN_ATTEMPT_FAILURES_VALIDITY_INTERVAL, YARN_ATTEMPT_FAILURES_VALIDITY_INTERVAL); if (failureValidityInterval != null) { result.put(Configs.Keys.YARN_ATTEMPT_FAILURES_VALIDITY_INTERVAL, failureValidityInterval.toString()); } return result; }
/** * Validates the custom transaction timeout, if specified in the given arguments. If scope and name are not null, * validates only for that given scope (for example, flowlet.myFlowlet). * * @throws IllegalArgumentException if the transaction timeout exceeds the transaction timeout limit given in the * CConfiguration. */ public static void validateTransactionTimeout(Map<String, String> args, CConfiguration cConf, @Nullable String scope, @Nullable String name) { String argName = TRANSACTION_TIMEOUT; if (scope != null && name != null) { argName = RuntimeArguments.addScope(scope, name, TRANSACTION_TIMEOUT); } Integer timeout = getPositiveInt(args, argName, "transaction timeout"); if (timeout != null) { int maxTimeout = cConf.getInt(TxConstants.Manager.CFG_TX_MAX_TIMEOUT); if (timeout > maxTimeout) { throw new IllegalArgumentException(String.format( "Transaction timeout (%s) of %d seconds must not exceed the transaction timeout limit (%s) of %d", argName, timeout, TxConstants.Manager.CFG_TX_MAX_TIMEOUT, maxTimeout)); } } }
/** * Returns the {@link Resources} based on configurations in the given arguments. * * @param args the arguments to use for looking up resources configurations * @param defaultResources default resources to use if resources configurations are missing from the arguments. * If it is {@code null}, the default values in {@link Resources} will be used. */ public static Resources getResources(Map<String, String> args, @Nullable Resources defaultResources) { Integer memory = getPositiveInt(args, MEMORY_KEY, "memory size"); Integer cores = getPositiveInt(args, CORES_KEY, "number of cores"); defaultResources = defaultResources == null ? new Resources() : defaultResources; if (memory == null && cores == null) { return defaultResources; } return new Resources(memory != null ? memory : defaultResources.getMemoryMB(), cores != null ? cores : defaultResources.getVirtualCores()); }
/** * Returns the transaction timeout based on the given arguments or, as fallback, the CConfiguration. * * @return the integer value of the argument system.data.tx.timeout, or if that is not given in the arguments, * the value for data.tx.timeout from the CConfiguration. * @throws IllegalArgumentException if the transaction timeout exceeds the transaction timeout limit given by * the {@link TxConstants.Manager#CFG_TX_MAX_TIMEOUT} setting in the {@link CConfiguration} */ public static int getTransactionTimeout(Map<String, String> args, CConfiguration cConf) { Integer timeout = getPositiveInt(args, TRANSACTION_TIMEOUT, "transaction timeout"); if (timeout == null) { return cConf.getInt(TxConstants.Manager.CFG_TX_TIMEOUT); } int maxTimeout = cConf.getInt(TxConstants.Manager.CFG_TX_MAX_TIMEOUT); if (timeout > maxTimeout) { throw new IllegalArgumentException(String.format( "Transaction timeout (%s) of %d seconds must not exceed the transaction timeout limit (%s) of %d", TRANSACTION_TIMEOUT, timeout, TxConstants.Manager.CFG_TX_MAX_TIMEOUT, maxTimeout)); } return timeout; }
/** * Validates the custom transaction timeout, if specified in the given arguments. If scope and name are not null, * validates only for that given scope (for example, flowlet.myFlowlet). * * @throws IllegalArgumentException if the transaction timeout exceeds the transaction timeout limit given in the * CConfiguration. */ public static void validateTransactionTimeout(Map<String, String> args, CConfiguration cConf, @Nullable String scope, @Nullable String name) { String argName = TRANSACTION_TIMEOUT; if (scope != null && name != null) { argName = RuntimeArguments.addScope(scope, name, TRANSACTION_TIMEOUT); } Integer timeout = getPositiveInt(args, argName, "transaction timeout"); if (timeout != null) { int maxTimeout = cConf.getInt(TxConstants.Manager.CFG_TX_MAX_TIMEOUT); if (timeout > maxTimeout) { throw new IllegalArgumentException(String.format( "Transaction timeout (%s) of %d seconds must not exceed the transaction timeout limit (%s) of %d", argName, timeout, TxConstants.Manager.CFG_TX_MAX_TIMEOUT, maxTimeout)); } } }
/** * Returns the transaction timeout based on the given arguments or, as fallback, the CConfiguration. * * @return the integer value of the argument system.data.tx.timeout, or if that is not given in the arguments, * the value for data.tx.timeout from the CConfiguration. * @throws IllegalArgumentException if the transaction timeout exceeds the transaction timeout limit given by * the {@link TxConstants.Manager#CFG_TX_MAX_TIMEOUT} setting in the {@link CConfiguration} */ public static int getTransactionTimeout(Map<String, String> args, CConfiguration cConf) { Integer timeout = getPositiveInt(args, TRANSACTION_TIMEOUT, "transaction timeout"); if (timeout == null) { return cConf.getInt(TxConstants.Manager.CFG_TX_TIMEOUT); } int maxTimeout = cConf.getInt(TxConstants.Manager.CFG_TX_MAX_TIMEOUT); if (timeout > maxTimeout) { throw new IllegalArgumentException(String.format( "Transaction timeout (%s) of %d seconds must not exceed the transaction timeout limit (%s) of %d", TRANSACTION_TIMEOUT, timeout, TxConstants.Manager.CFG_TX_MAX_TIMEOUT, maxTimeout)); } return timeout; }
/** * Returns the {@link Resources} based on configurations in the given arguments. * * @param args the arguments to use for looking up resources configurations * @param defaultResources default resources to use if resources configurations are missing from the arguments. * If it is {@code null}, the default values in {@link Resources} will be used. */ public static Resources getResources(Map<String, String> args, @Nullable Resources defaultResources) { Integer memory = getPositiveInt(args, MEMORY_KEY, "memory size"); Integer cores = getPositiveInt(args, CORES_KEY, "number of cores"); defaultResources = defaultResources == null ? new Resources() : defaultResources; if (memory == null && cores == null) { return defaultResources; } return new Resources(memory != null ? memory : defaultResources.getMemoryMB(), cores != null ? cores : defaultResources.getVirtualCores()); }