private Resource getContainerResource() { final long mem = taskManagerParameters.taskManagerTotalMemoryMB(); final int containerMemorySizeMB; if (mem <= Integer.MAX_VALUE) { containerMemorySizeMB = (int) mem; } else { containerMemorySizeMB = Integer.MAX_VALUE; LOG.error("Decreasing container size from {} MB to {} MB (integer value overflow)", mem, containerMemorySizeMB); } // Resource requirements for worker containers int taskManagerSlots = taskManagerParameters.numSlots(); int vcores = config.getInteger(YarnConfigOptions.VCORES, Math.max(taskManagerSlots, 1)); return Resource.newInstance(containerMemorySizeMB, vcores); }
public static MesosTaskManagerParameters createTmParameters(Configuration configuration, Logger log) { // TM configuration final MesosTaskManagerParameters taskManagerParameters = MesosTaskManagerParameters.create(configuration); log.info("TaskManagers will be created with {} task slots", taskManagerParameters.containeredParameters().numSlots()); log.info("TaskManagers will be started with container size {} MB, JVM heap size {} MB, " + "JVM direct memory limit {} MB, {} cpus, {} gpus", taskManagerParameters.containeredParameters().taskManagerTotalMemoryMB(), taskManagerParameters.containeredParameters().taskManagerHeapSizeMB(), taskManagerParameters.containeredParameters().taskManagerDirectMemoryLimitMB(), taskManagerParameters.cpus(), taskManagerParameters.gpus()); return taskManagerParameters; }
ContaineredTaskManagerParameters.create(config, taskManagerContainerMemory, slotsPerTaskManager); LOG.info("TaskManagers will be created with {} task slots", taskManagerParameters.numSlots()); LOG.info("TaskManagers will be started with container size {} MB, JVM heap size {} MB, " + "JVM direct memory limit {} MB",
cpus = Math.max(containeredParameters.numSlots(), 1.0);
private Resource getContainerResource() { final long mem = taskManagerParameters.taskManagerTotalMemoryMB(); final int containerMemorySizeMB; if (mem <= Integer.MAX_VALUE) { containerMemorySizeMB = (int) mem; } else { containerMemorySizeMB = Integer.MAX_VALUE; LOG.error("Decreasing container size from {} MB to {} MB (integer value overflow)", mem, containerMemorySizeMB); } // Resource requirements for worker containers int taskManagerSlots = taskManagerParameters.numSlots(); int vcores = config.getInteger(YarnConfigOptions.VCORES, Math.max(taskManagerSlots, 1)); return Resource.newInstance(containerMemorySizeMB, vcores); }
@Override protected void requestNewWorkers(int numWorkers) { final long mem = taskManagerParameters.taskManagerTotalMemoryMB(); final int containerMemorySizeMB; if (mem <= Integer.MAX_VALUE) { containerMemorySizeMB = (int) mem; } else { containerMemorySizeMB = Integer.MAX_VALUE; LOG.error("Decreasing container size from {} MB to {} MB (integer value overflow)", mem, containerMemorySizeMB); } for (int i = 0; i < numWorkers; i++) { numPendingContainerRequests++; LOG.info("Requesting new TaskManager container with {} megabytes memory. Pending requests: {}", containerMemorySizeMB, numPendingContainerRequests); // Priority for worker containers - priorities are intra-application Priority priority = Priority.newInstance(0); // Resource requirements for worker containers int taskManagerSlots = taskManagerParameters.numSlots(); int vcores = config.getInteger(YarnConfigOptions.VCORES, Math.max(taskManagerSlots, 1)); Resource capability = Resource.newInstance(containerMemorySizeMB, vcores); resourceManagerClient.addContainerRequest( new AMRMClient.ContainerRequest(capability, null, null, priority)); } // make sure we transmit the request fast and receive fast news of granted allocations resourceManagerClient.setHeartbeatInterval(FAST_YARN_HEARTBEAT_INTERVAL_MS); }
ContaineredTaskManagerParameters.create(config, taskManagerContainerMemory, slotsPerTaskManager); LOG.info("TaskManagers will be created with {} task slots", taskManagerParameters.numSlots()); LOG.info("TaskManagers will be started with container size {} MB, JVM heap size {} MB, " + "JVM direct memory limit {} MB",
ContaineredTaskManagerParameters.create(config, taskManagerContainerMemory, slotsPerTaskManager); LOG.info("TaskManagers will be created with {} task slots", taskManagerParameters.numSlots()); LOG.info("TaskManagers will be started with container size {} MB, JVM heap size {} MB, " + "JVM direct memory limit {} MB",