public static MesosTaskManagerParameters createTmParameters(Configuration configuration, Logger log) { // TM configuration final MesosTaskManagerParameters taskManagerParameters = MesosTaskManagerParameters.create(configuration); log.info("TaskManagers will be created with {} task slots", taskManagerParameters.containeredParameters().numSlots()); log.info("TaskManagers will be started with container size {} MB, JVM heap size {} MB, " + "JVM direct memory limit {} MB, {} cpus, {} gpus", taskManagerParameters.containeredParameters().taskManagerTotalMemoryMB(), taskManagerParameters.containeredParameters().taskManagerHeapSizeMB(), taskManagerParameters.containeredParameters().taskManagerDirectMemoryLimitMB(), taskManagerParameters.cpus(), taskManagerParameters.gpus()); return taskManagerParameters; }
/** * Method to validate cluster specification before deploy it, it will throw * an {@link FlinkException} if the {@link ClusterSpecification} is invalid. * * @param clusterSpecification cluster specification to check against the configuration of the * AbstractYarnClusterDescriptor * @throws FlinkException if the cluster cannot be started with the provided {@link ClusterSpecification} */ private void validateClusterSpecification(ClusterSpecification clusterSpecification) throws FlinkException { try { final long taskManagerMemorySize = clusterSpecification.getTaskManagerMemoryMB(); // We do the validation by calling the calculation methods here // Internally these methods will check whether the cluster can be started with the provided // ClusterSpecification and the configured memory requirements final long cutoff = ContaineredTaskManagerParameters.calculateCutoffMB(flinkConfiguration, taskManagerMemorySize); TaskManagerServices.calculateHeapSizeMB(taskManagerMemorySize - cutoff, flinkConfiguration); } catch (IllegalArgumentException iae) { throw new FlinkException("Cannot fulfill the minimum memory requirements with the provided " + "cluster specification. Please increase the memory of the cluster.", iae); } }
for (Map.Entry<String, String> entry : params.containeredParameters().taskManagerEnv().entrySet()) { env.addVariables(variable(entry.getKey(), entry.getValue())); jvmArgs.append(" -Xms").append(tmParams.taskManagerHeapSizeMB()).append("m"); jvmArgs.append(" -Xmx").append(tmParams.taskManagerHeapSizeMB()).append("m"); if (tmParams.taskManagerDirectMemoryLimitMB() >= 0) { jvmArgs.append(" -XX:MaxDirectMemorySize=").append(tmParams.taskManagerDirectMemoryLimitMB()).append("m");
private Resource getContainerResource() { final long mem = taskManagerParameters.taskManagerTotalMemoryMB(); final int containerMemorySizeMB; if (mem <= Integer.MAX_VALUE) { containerMemorySizeMB = (int) mem; } else { containerMemorySizeMB = Integer.MAX_VALUE; LOG.error("Decreasing container size from {} MB to {} MB (integer value overflow)", mem, containerMemorySizeMB); } // Resource requirements for worker containers int taskManagerSlots = taskManagerParameters.numSlots(); int vcores = config.getInteger(YarnConfigOptions.VCORES, Math.max(taskManagerSlots, 1)); return Resource.newInstance(containerMemorySizeMB, vcores); }
params.add(String.format("-Xms%dm", tmParams.taskManagerHeapSizeMB())); params.add(String.format("-Xmx%dm", tmParams.taskManagerHeapSizeMB())); if (tmParams.taskManagerDirectMemoryLimitMB() >= 0) { params.add(String.format("-XX:MaxDirectMemorySize=%dm", tmParams.taskManagerDirectMemoryLimitMB()));
params.add(String.format("-Xms%dm", tmParams.taskManagerHeapSizeMB())); params.add(String.format("-Xmx%dm", tmParams.taskManagerHeapSizeMB())); if (tmParams.getYoungMemoryMB() > 0) { params.add(String.format("-Xmn%dm", tmParams.getYoungMemoryMB())); if (tmParams.taskManagerDirectMemoryLimitMB() >= 0) { params.add(String.format("-XX:MaxDirectMemorySize=%dm", tmParams.taskManagerDirectMemoryLimitMB()));
int numSlots) { final long cutoffMB = calculateCutoffMB(config, containerMemoryMB); return new ContaineredTaskManagerParameters( containerMemoryMB, heapSizeMB, offHeapSizeMB, numSlots, envVars);
final ContaineredTaskManagerParameters containeredTaskManagerParameters = ContaineredTaskManagerParameters.create( configuration, taskManagerMemory, slotsPerTaskManager); final long expectedHeadSize = containeredTaskManagerParameters.taskManagerHeapSizeMB() << 20L;
/** * Computes the parameters to be used to start a TaskManager Java process. * * @param config The Flink configuration. * @param containerMemoryMB The size of the complete container, in megabytes. * @return The parameters to start the TaskManager processes with. */ public static ContaineredTaskManagerParameters create( Configuration config, long containerMemoryMB, int numSlots) { // (1) try to compute how much memory used by container final long cutoffMB = calculateCutoffMB(config, containerMemoryMB); // (2) split the remaining Java memory between heap and off-heap final long heapSizeMB = TaskManagerServices.calculateHeapSizeMB(containerMemoryMB - cutoffMB, config); // use the cut-off memory for off-heap (that was its intention) final long offHeapSizeMB = containerMemoryMB - heapSizeMB; return create(config, containerMemoryMB, (int)heapSizeMB, (int)offHeapSizeMB, numSlots, 0); }
public static ContaineredTaskManagerParameters create( Configuration config, long containerMemorySizeMB, int heapMemorySizeMB, int directMemorySizeMB, int numSlots, int youngMemorySizeMB, double containerCpuCore) { // obtain the additional environment variables from the configuration final HashMap<String, String> envVars = new HashMap<>(); final String prefix = ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX; for (String key : config.keySet()) { if (key.startsWith(prefix) && key.length() > prefix.length()) { // remove prefix String envVarKey = key.substring(prefix.length()); envVars.put(envVarKey, config.getString(key, null)); } } return new ContaineredTaskManagerParameters( containerMemorySizeMB, heapMemorySizeMB, directMemorySizeMB, numSlots, envVars, youngMemorySizeMB, containerCpuCore); } }
String applicationMasterHostName = "localhost"; String webInterfaceURL = "foobar"; ContaineredTaskManagerParameters taskManagerParameters = new ContaineredTaskManagerParameters( 1L, 1L, 1L, 1, new HashMap<String, String>()); ContainerLaunchContext taskManagerLaunchContext = mock(ContainerLaunchContext.class);
containerEnv.putAll(tmParams.taskManagerEnv());
@Override public double getMemory() { return params.containeredParameters().taskManagerTotalMemoryMB(); }
params.add(String.format("-Xms%dm", tmParams.taskManagerHeapSizeMB())); params.add(String.format("-Xmx%dm", tmParams.taskManagerHeapSizeMB())); if (tmParams.taskManagerDirectMemoryLimitMB() >= 0) { params.add(String.format("-XX:MaxDirectMemorySize=%dm", tmParams.taskManagerDirectMemoryLimitMB()));
int numSlots) { final long cutoffMB = calculateCutoffMB(config, containerMemoryMB); return new ContaineredTaskManagerParameters( containerMemoryMB, heapSizeMB, offHeapSizeMB, numSlots, envVars);
private Resource getContainerResource() { final long mem = taskManagerParameters.taskManagerTotalMemoryMB(); final int containerMemorySizeMB; if (mem <= Integer.MAX_VALUE) { containerMemorySizeMB = (int) mem; } else { containerMemorySizeMB = Integer.MAX_VALUE; LOG.error("Decreasing container size from {} MB to {} MB (integer value overflow)", mem, containerMemorySizeMB); } // Resource requirements for worker containers int taskManagerSlots = taskManagerParameters.numSlots(); int vcores = config.getInteger(YarnConfigOptions.VCORES, Math.max(taskManagerSlots, 1)); return Resource.newInstance(containerMemorySizeMB, vcores); }
return new ContaineredTaskManagerParameters( containerMemoryMB, heapSizeMB, offHeapSize, numSlots, envVars);
containerEnv.putAll(tmParams.taskManagerEnv());
ContaineredTaskManagerParameters.create(flinkConfig, resource.getMemory(), numberOfTaskSlots); taskManagerParameters.taskManagerTotalMemoryMB(), taskManagerParameters.taskManagerHeapSizeMB(), taskManagerParameters.taskManagerDirectMemoryLimitMB());