ContaineredTaskManagerParameters.create(flinkConfig, resource.getMemory(), numberOfTaskSlots);
ContaineredTaskManagerParameters.create(config, taskManagerContainerMemory, slotsPerTaskManager);
final ContaineredTaskManagerParameters containeredTaskManagerParameters = ContaineredTaskManagerParameters.create( configuration, taskManagerMemory,
ContaineredTaskManagerParameters containeredParameters = ContaineredTaskManagerParameters.create( flinkConfig, flinkConfig.getInteger(MESOS_RM_TASKS_MEMORY_MB),
public static ContaineredTaskManagerParameters create( Configuration config, long containerMemorySizeMB, int heapMemorySizeMB, int directMemorySizeMB, int numSlots, int youngMemorySizeMB, double containerCpuCore) { // obtain the additional environment variables from the configuration final HashMap<String, String> envVars = new HashMap<>(); final String prefix = ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX; for (String key : config.keySet()) { if (key.startsWith(prefix) && key.length() > prefix.length()) { // remove prefix String envVarKey = key.substring(prefix.length()); envVars.put(envVarKey, config.getString(key, null)); } } return new ContaineredTaskManagerParameters( containerMemorySizeMB, heapMemorySizeMB, directMemorySizeMB, numSlots, envVars, youngMemorySizeMB, containerCpuCore); } }
/** * Computes the parameters to be used to start a TaskManager Java process. * * @param config The Flink configuration. * @param containerMemoryMB The size of the complete container, in megabytes. * @return The parameters to start the TaskManager processes with. */ public static ContaineredTaskManagerParameters create( Configuration config, long containerMemoryMB, int numSlots) { // (1) try to compute how much memory used by container final long cutoffMB = calculateCutoffMB(config, containerMemoryMB); // (2) split the remaining Java memory between heap and off-heap final long heapSizeMB = TaskManagerServices.calculateHeapSizeMB(containerMemoryMB - cutoffMB, config); // use the cut-off memory for off-heap (that was its intention) final long offHeapSizeMB = containerMemoryMB - heapSizeMB; return create(config, containerMemoryMB, (int)heapSizeMB, (int)offHeapSizeMB, numSlots, 0); }
ContaineredTaskManagerParameters.create(flinkConfig, resource.getMemory(), numberOfTaskSlots);
private ContainerLaunchContext createTaskExecutorLaunchContext(Resource resource, String containerId, String host) throws Exception { // init the ContainerLaunchContext final String currDir = env.get(ApplicationConstants.Environment.PWD.key()); final ContaineredTaskManagerParameters taskManagerParameters = ContaineredTaskManagerParameters.create(flinkConfig, resource.getMemory(), numberOfTaskSlots); log.debug("TaskExecutor {} will be started with container size {} MB, JVM heap size {} MB, " + "JVM direct memory limit {} MB", containerId, taskManagerParameters.taskManagerTotalMemoryMB(), taskManagerParameters.taskManagerHeapSizeMB(), taskManagerParameters.taskManagerDirectMemoryLimitMB()); Configuration taskManagerConfig = BootstrapTools.cloneConfiguration(flinkConfig); log.debug("TaskManager configuration: {}", taskManagerConfig); ContainerLaunchContext taskExecutorLaunchContext = Utils.createTaskExecutorContext( flinkConfig, yarnConfig, env, taskManagerParameters, taskManagerConfig, currDir, YarnTaskExecutorRunner.class, log); // set a special environment variable to uniquely identify this container taskExecutorLaunchContext.getEnvironment() .put(ENV_FLINK_CONTAINER_ID, containerId); taskExecutorLaunchContext.getEnvironment() .put(ENV_FLINK_NODE_ID, host); return taskExecutorLaunchContext; }
ContaineredTaskManagerParameters.create(config, taskManagerContainerMemory, slotsPerTaskManager);
ContaineredTaskManagerParameters.create(config, taskManagerContainerMemory, slotsPerTaskManager);