/** * Sets the Spark memory overhead setting based on the runtime arguments of the given type. * * @param runtimeArgs the runtime arguments * @param type either {@code driver} or {@code executor} * @param containerSize the memory size in MB of the container * @param configs the configuration to be updated */ private void setMemoryOverhead(Map<String, String> runtimeArgs, String type, int containerSize, Map<String, String> configs) { Map<String, String> memoryConfigs = SystemArguments.getTwillContainerConfigs(runtimeArgs, containerSize); if (!memoryConfigs.containsKey(Configs.Keys.JAVA_RESERVED_MEMORY_MB)) { return; } configs.put("spark.yarn." + type + ".memoryOverhead", memoryConfigs.get(Configs.Keys.JAVA_RESERVED_MEMORY_MB)); }
/** * Sets the Spark memory overhead setting based on the runtime arguments of the given type. * * @param runtimeArgs the runtime arguments * @param type either {@code driver} or {@code executor} * @param containerSize the memory size in MB of the container * @param configs the configuration to be updated */ private void setMemoryOverhead(Map<String, String> runtimeArgs, String type, int containerSize, Map<String, String> configs) { Map<String, String> memoryConfigs = SystemArguments.getTwillContainerConfigs(runtimeArgs, containerSize); if (!memoryConfigs.containsKey(Configs.Keys.JAVA_RESERVED_MEMORY_MB)) { return; } configs.put("spark.yarn." + type + ".memoryOverhead", memoryConfigs.get(Configs.Keys.JAVA_RESERVED_MEMORY_MB)); }
/** * Sets the Spark memory overhead setting based on the runtime arguments of the given type. * * @param runtimeArgs the runtime arguments * @param type either {@code driver} or {@code executor} * @param containerSize the memory size in MB of the container * @param configs the configuration to be updated */ private void setMemoryOverhead(Map<String, String> runtimeArgs, String type, int containerSize, Map<String, String> configs) { Map<String, String> memoryConfigs = SystemArguments.getTwillContainerConfigs(runtimeArgs, containerSize); if (!memoryConfigs.containsKey(Configs.Keys.JAVA_RESERVED_MEMORY_MB)) { return; } configs.put("spark.yarn." + type + ".memoryOverhead", memoryConfigs.get(Configs.Keys.JAVA_RESERVED_MEMORY_MB)); }
public ProgramLaunchConfig addRunnable(String name, TwillRunnable runnable, int instances, Map<String, String> args, Resources defaultResources, @Nullable Integer maxRetries) { ResourceSpecification resourceSpec = createResourceSpec(SystemArguments.getResources(args, defaultResources), instances); Map<String, String> configs = SystemArguments.getTwillContainerConfigs(args, resourceSpec.getMemorySize()); Map<String, Level> logLevels = SystemArguments.getLogLevels(args); runnables.put(name, new RunnableDefinition(runnable, resourceSpec, configs, logLevels, maxRetries)); return this; }
public ProgramLaunchConfig addRunnable(String name, TwillRunnable runnable, int instances, Map<String, String> args, Resources defaultResources, @Nullable Integer maxRetries) { ResourceSpecification resourceSpec = createResourceSpec(SystemArguments.getResources(args, defaultResources), instances); Map<String, String> configs = SystemArguments.getTwillContainerConfigs(args, resourceSpec.getMemorySize()); Map<String, Level> logLevels = SystemArguments.getLogLevels(args); runnables.put(name, new RunnableDefinition(runnable, resourceSpec, configs, logLevels, maxRetries)); return this; }
Map<String, String> configs = SystemArguments.getTwillContainerConfigs(runtimeArgs, memoryMB); if (configs.containsKey(Configs.Keys.JAVA_RESERVED_MEMORY_MB)) { reservedMemory = Integer.parseInt(configs.get(Configs.Keys.JAVA_RESERVED_MEMORY_MB));
Map<String, String> configs = SystemArguments.getTwillContainerConfigs(runtimeArgs, memoryMB); if (configs.containsKey(Configs.Keys.JAVA_RESERVED_MEMORY_MB)) { reservedMemory = Integer.parseInt(configs.get(Configs.Keys.JAVA_RESERVED_MEMORY_MB));
Map<String, String> configs = SystemArguments.getTwillContainerConfigs( ImmutableMap.of(SystemArguments.RESERVED_MEMORY_KEY_OVERRIDE, "200"), 300); Assert.assertEquals(ImmutableMap.of(Configs.Keys.JAVA_RESERVED_MEMORY_MB, "200", configs = SystemArguments.getTwillContainerConfigs( ImmutableMap.of(SystemArguments.RESERVED_MEMORY_KEY_OVERRIDE, "-1"), 300); Assert.assertTrue(configs.isEmpty()); configs = SystemArguments.getTwillContainerConfigs( ImmutableMap.of(SystemArguments.RESERVED_MEMORY_KEY_OVERRIDE, "300"), 300); Assert.assertTrue(configs.isEmpty());