taskManagerParameters.taskManagerDirectMemoryLimitMB()); Configuration taskManagerConfig = BootstrapTools.cloneConfiguration(flinkConfig);
taskManagerParameters.taskManagerDirectMemoryLimitMB()); Configuration taskManagerConfig = BootstrapTools.cloneConfiguration(flinkConfig);
private ContainerLaunchContext createTaskExecutorLaunchContext(Resource resource, String containerId, String host) throws Exception { // init the ContainerLaunchContext final String currDir = env.get(ApplicationConstants.Environment.PWD.key()); final ContaineredTaskManagerParameters taskManagerParameters = ContaineredTaskManagerParameters.create(flinkConfig, resource.getMemory(), numberOfTaskSlots); log.debug("TaskExecutor {} will be started with container size {} MB, JVM heap size {} MB, " + "JVM direct memory limit {} MB", containerId, taskManagerParameters.taskManagerTotalMemoryMB(), taskManagerParameters.taskManagerHeapSizeMB(), taskManagerParameters.taskManagerDirectMemoryLimitMB()); Configuration taskManagerConfig = BootstrapTools.cloneConfiguration(flinkConfig); log.debug("TaskManager configuration: {}", taskManagerConfig); ContainerLaunchContext taskExecutorLaunchContext = Utils.createTaskExecutorContext( flinkConfig, yarnConfig, env, taskManagerParameters, taskManagerConfig, currDir, YarnTaskExecutorRunner.class, log); // set a special environment variable to uniquely identify this container taskExecutorLaunchContext.getEnvironment() .put(ENV_FLINK_CONTAINER_ID, containerId); taskExecutorLaunchContext.getEnvironment() .put(ENV_FLINK_NODE_ID, host); return taskExecutorLaunchContext; }
/** * Generate a task manager configuration. * @param baseConfig Config to start from. * @param jobManagerHostname Job manager host name. * @param jobManagerPort Port of the job manager. * @param numSlots Number of slots to configure. * @param registrationTimeout Timeout for registration * @return TaskManager configuration */ public static Configuration generateTaskManagerConfiguration( Configuration baseConfig, String jobManagerHostname, int jobManagerPort, int numSlots, FiniteDuration registrationTimeout) { Configuration cfg = cloneConfiguration(baseConfig); if (jobManagerHostname != null && !jobManagerHostname.isEmpty()) { cfg.setString(JobManagerOptions.ADDRESS, jobManagerHostname); } if (jobManagerPort > 0) { cfg.setInteger(JobManagerOptions.PORT, jobManagerPort); } cfg.setString(TaskManagerOptions.REGISTRATION_TIMEOUT, registrationTimeout.toString()); if (numSlots != -1){ cfg.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, numSlots); } return cfg; }
/** * Generate a task manager configuration. * @param baseConfig Config to start from. * @param jobManagerHostname Job manager host name. * @param jobManagerPort Port of the job manager. * @param numSlots Number of slots to configure. * @param registrationTimeout Timeout for registration * @return TaskManager configuration */ public static Configuration generateTaskManagerConfiguration( Configuration baseConfig, String jobManagerHostname, int jobManagerPort, int numSlots, FiniteDuration registrationTimeout) { Configuration cfg = cloneConfiguration(baseConfig); if (jobManagerHostname != null && !jobManagerHostname.isEmpty()) { cfg.setString(JobManagerOptions.ADDRESS, jobManagerHostname); } if (jobManagerPort > 0) { cfg.setInteger(JobManagerOptions.PORT, jobManagerPort); } cfg.setString(TaskManagerOptions.REGISTRATION_TIMEOUT, registrationTimeout.toString()); if (numSlots != -1){ cfg.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, numSlots); } return cfg; }