name().toLowerCase(), memoryMB, memorySource, maxHeapSize, maxHeapSource, vcores > 0 ? " Virtual cores is " + vcores + ", set " + vcoreSource + "." : ".");
name().toLowerCase(), memoryMB, memorySource, maxHeapSize, maxHeapSource, vcores > 0 ? " Virtual cores is " + vcores + ", set " + vcoreSource + "." : ".");
@Test public void testProgrammatic() { CConfiguration cConf = CConfiguration.create(); cConf.setInt(Configs.Keys.JAVA_RESERVED_MEMORY_MB, 300); Configuration hConf = new Configuration(); hConf.setInt(Job.MAP_MEMORY_MB, 3000); hConf.setInt(Job.MAP_CPU_VCORES, 5); // Always use configurations setup programmatically via job conf. MapReduceRuntimeService.TaskType.MAP.configure(hConf, cConf, Collections.emptyMap(), null); int maxHeapSize = org.apache.twill.internal.utils.Resources.computeMaxHeapSize( 3000, cConf.getInt(Configs.Keys.JAVA_RESERVED_MEMORY_MB), 0); validateResources(cConf, hConf, 3000, 5, maxHeapSize); // Even resources is provided via context, it is ignored. hConf = new Configuration(); hConf.setInt(Job.MAP_MEMORY_MB, 3000); hConf.setInt(Job.MAP_CPU_VCORES, 5); MapReduceRuntimeService.TaskType.MAP.configure(hConf, cConf, Collections.emptyMap(), new Resources(1234)); maxHeapSize = org.apache.twill.internal.utils.Resources.computeMaxHeapSize( 3000, cConf.getInt(Configs.Keys.JAVA_RESERVED_MEMORY_MB), 0); validateResources(cConf, hConf, 3000, 5, maxHeapSize); // Set the reserved memory via task arguments hConf = new Configuration(); hConf.setInt(Job.MAP_MEMORY_MB, 3000); hConf.setInt(Job.MAP_CPU_VCORES, 5); MapReduceRuntimeService.TaskType.MAP.configure( hConf, cConf, Collections.singletonMap("system.resources.reserved.memory.override", "2000"), null); validateResources(cConf, hConf, 3000, 5, 3000 - 2000); }
@Test public void testContextResources() { CConfiguration cConf = CConfiguration.create(); cConf.setInt(Configs.Keys.JAVA_RESERVED_MEMORY_MB, 300); Configuration hConf = new Configuration(); // Resources is set through context object // At runtime time, it is either from spec, from runtime arg of the program // or from the context.setResources call in the initialize() method) Resources resources = new Resources(2345, 8); MapReduceRuntimeService.TaskType.MAP.configure(hConf, cConf, Collections.emptyMap(), resources); int maxHeapSize = org.apache.twill.internal.utils.Resources.computeMaxHeapSize( 2345, cConf.getInt(Configs.Keys.JAVA_RESERVED_MEMORY_MB), 0); validateResources(cConf, hConf, 2345, 8, maxHeapSize); // Set the reserved memory via task arguments hConf = new Configuration(); MapReduceRuntimeService.TaskType.MAP.configure( hConf, cConf, Collections.singletonMap("system.resources.reserved.memory.override", "1000"), resources); validateResources(cConf, hConf, 2345, 8, 2345 - 1000); }
TaskType(String memoryConfKey, int defaultMemoryMB, String javaOptsKey) { this.memoryConfKey = memoryConfKey; this.defaultMemoryMB = defaultMemoryMB; this.javaOptsKey = javaOptsKey; String vcoreConfKey = null; int defaultVcores = 0; try { String confFieldName = name() + "_CPU_VCORES"; vcoreConfKey = Job.class.getField(confFieldName).get(null).toString(); String defaultValueFieldName = "DEFAULT_" + confFieldName; defaultVcores = (Integer) Job.class.getField(defaultValueFieldName).get(null); } catch (Exception e) { // OK to ignore // Some older version of hadoop-mr-client doesn't has the VCORES field as vcores was not supported in YARN. } this.vcoreConfKey = vcoreConfKey; this.defaultVcores = defaultVcores; }
TaskType(String memoryConfKey, int defaultMemoryMB, String javaOptsKey) { this.memoryConfKey = memoryConfKey; this.defaultMemoryMB = defaultMemoryMB; this.javaOptsKey = javaOptsKey; String vcoreConfKey = null; int defaultVcores = 0; try { String confFieldName = name() + "_CPU_VCORES"; vcoreConfKey = Job.class.getField(confFieldName).get(null).toString(); String defaultValueFieldName = "DEFAULT_" + confFieldName; defaultVcores = (Integer) Job.class.getField(defaultValueFieldName).get(null); } catch (Exception e) { // OK to ignore // Some older version of hadoop-mr-client doesn't has the VCORES field as vcores was not supported in YARN. } this.vcoreConfKey = vcoreConfKey; this.defaultVcores = defaultVcores; }
@Test public void testDefault() { CConfiguration cConf = CConfiguration.create(); Configuration hConf = new Configuration(); // Nothing is set. It should be using the default. MapReduceRuntimeService.TaskType.MAP.configure(hConf, cConf, Collections.emptyMap(), null); int maxHeapSize = org.apache.twill.internal.utils.Resources.computeMaxHeapSize( Job.DEFAULT_MAP_MEMORY_MB, cConf.getInt(Configs.Keys.JAVA_RESERVED_MEMORY_MB), cConf.getDouble(Configs.Keys.HEAP_RESERVED_MIN_RATIO)); validateResources(cConf, hConf, Job.DEFAULT_MAP_MEMORY_MB, Job.DEFAULT_MAP_CPU_VCORES, maxHeapSize); // Sets the reserved memory via task arguments hConf = new Configuration(); int reservedMemory = Job.DEFAULT_MAP_MEMORY_MB / 2; MapReduceRuntimeService.TaskType.MAP.configure( hConf, cConf, Collections.singletonMap("system.resources.reserved.memory.override", String.valueOf(reservedMemory)), null); validateResources(cConf, hConf, Job.DEFAULT_MAP_MEMORY_MB, Job.DEFAULT_MAP_CPU_VCORES, Job.DEFAULT_MAP_MEMORY_MB - reservedMemory); }