@Override public double getMemory() { return params.containeredParameters().taskManagerTotalMemoryMB(); }
/** * Configures an artifact server to serve the artifacts associated with a container specification. * @param server the server to configure. * @param container the container with artifacts to serve. * @throws IOException if the artifacts cannot be accessed. */ static void configureArtifactServer(MesosArtifactServer server, ContainerSpecification container) throws IOException { // serve the artifacts associated with the container environment for (ContainerSpecification.Artifact artifact : container.getArtifacts()) { server.addPath(artifact.source, artifact.dest); } } }
public static ContainerSpecification createContainerSpec(Configuration configuration, Configuration dynamicProperties) throws Exception { // generate a container spec which conveys the artifacts/vars needed to launch a TM ContainerSpecification spec = new ContainerSpecification(); // propagate the AM dynamic configuration to the TM spec.getDynamicConfiguration().addAll(dynamicProperties); applyOverlays(configuration, spec); return spec; }
ContaineredTaskManagerParameters.create(flinkConfig, resource.getMemory(), numberOfTaskSlots); taskManagerParameters.taskManagerTotalMemoryMB(), taskManagerParameters.taskManagerHeapSizeMB(), taskManagerParameters.taskManagerDirectMemoryLimitMB()); Configuration taskManagerConfig = BootstrapTools.cloneConfiguration(flinkConfig);
public static MesosTaskManagerParameters createTmParameters(Configuration configuration, Logger log) { // TM configuration final MesosTaskManagerParameters taskManagerParameters = MesosTaskManagerParameters.create(configuration); log.info("TaskManagers will be created with {} task slots", taskManagerParameters.containeredParameters().numSlots()); log.info("TaskManagers will be started with container size {} MB, JVM heap size {} MB, " + "JVM direct memory limit {} MB, {} cpus, {} gpus", taskManagerParameters.containeredParameters().taskManagerTotalMemoryMB(), taskManagerParameters.containeredParameters().taskManagerHeapSizeMB(), taskManagerParameters.containeredParameters().taskManagerDirectMemoryLimitMB(), taskManagerParameters.cpus(), taskManagerParameters.gpus()); return taskManagerParameters; }
FileSystem.initialize(configuration); BootstrapTools.updateTmpDirectoriesInConfiguration(configuration, localDirs); TaskManagerRunner.runTaskManager(configuration, new ResourceID(containerId)); return null; });
Configuration dynamicProperties = BootstrapTools.parseDynamicProperties(cmd); LOG.debug("Mesos dynamic properties: {}", dynamicProperties); final ResourceID resourceId = new ResourceID(containerID); LOG.info("ResourceID assigned for this container: {}", resourceId);
actorSystem = BootstrapTools.startActorSystem( configuration, ownHostname.getCanonicalHostName(),
new File(workingDirectory, UUID.randomUUID() + "-taskmanager-conf.yaml"); log.debug("Writing TaskManager configuration to {}", taskManagerConfigFile.getAbsolutePath()); BootstrapTools.writeConfiguration(taskManagerConfig, taskManagerConfigFile); boolean hasLog4j = new File(workingDirectory, "log4j.properties").exists(); String launchCommand = BootstrapTools.getTaskManagerShellCommand( flinkConfig, tmParams, ".", ApplicationConstants.LOG_DIR_EXPANSION_VAR, hasLogback, hasLog4j, hasKrb5, taskManagerMainClass); containerEnv.putAll(tmParams.taskManagerEnv());
private Resource getContainerResource() { final long mem = taskManagerParameters.taskManagerTotalMemoryMB(); final int containerMemorySizeMB; if (mem <= Integer.MAX_VALUE) { containerMemorySizeMB = (int) mem; } else { containerMemorySizeMB = Integer.MAX_VALUE; LOG.error("Decreasing container size from {} MB to {} MB (integer value overflow)", mem, containerMemorySizeMB); } // Resource requirements for worker containers int taskManagerSlots = taskManagerParameters.numSlots(); int vcores = config.getInteger(YarnConfigOptions.VCORES, Math.max(taskManagerSlots, 1)); return Resource.newInstance(containerMemorySizeMB, vcores); }
BootstrapTools.substituteDeprecatedConfigPrefix(configuration, ConfigConstants.YARN_APPLICATION_MASTER_ENV_PREFIX, ResourceManagerOptions.CONTAINERIZED_MASTER_ENV_PREFIX); BootstrapTools.substituteDeprecatedConfigPrefix(configuration, ConfigConstants.YARN_TASK_MANAGER_ENV_PREFIX, ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX); BootstrapTools.updateTmpDirectoriesInConfiguration(configuration, localDirs);
@Override public int getPorts() { return extractPortKeys(containerSpec.getDynamicConfiguration()).size(); }
@Override protected void handleMessage(Object message) { // check for YARN specific actor messages first if (message instanceof ContainersAllocated) { containersAllocated(((ContainersAllocated) message).containers()); } else if (message instanceof ContainersComplete) { containersComplete(((ContainersComplete) message).containers()); } else { // message handled by the generic resource master code super.handleMessage(message); } }
ConfigConstants.DEFAULT_YARN_CONTAINER_START_COMMAND_TEMPLATE); final String amCommand = BootstrapTools.getStartCommand(commandTemplate, startCommandValues);
/** * Method to validate cluster specification before deploy it, it will throw * an {@link FlinkException} if the {@link ClusterSpecification} is invalid. * * @param clusterSpecification cluster specification to check against the configuration of the * AbstractYarnClusterDescriptor * @throws FlinkException if the cluster cannot be started with the provided {@link ClusterSpecification} */ private void validateClusterSpecification(ClusterSpecification clusterSpecification) throws FlinkException { try { final long taskManagerMemorySize = clusterSpecification.getTaskManagerMemoryMB(); // We do the validation by calling the calculation methods here // Internally these methods will check whether the cluster can be started with the provided // ClusterSpecification and the configured memory requirements final long cutoff = ContaineredTaskManagerParameters.calculateCutoffMB(flinkConfiguration, taskManagerMemorySize); TaskManagerServices.calculateHeapSizeMB(taskManagerMemorySize - cutoff, flinkConfiguration); } catch (IllegalArgumentException iae) { throw new FlinkException("Cannot fulfill the minimum memory requirements with the provided " + "cluster specification. Please increase the memory of the cluster.", iae); } }
/** * Loads the global configuration, adds the given dynamic properties configuration, and sets * the temp directory paths. * * @param dynamicProperties dynamic properties to integrate * @param log logger instance * @return the loaded and adapted global configuration */ public static Configuration loadConfiguration(Configuration dynamicProperties, Logger log) { Configuration configuration = GlobalConfiguration.loadConfigurationWithDynamicProperties(dynamicProperties); // read the environment variables final Map<String, String> envs = System.getenv(); final String tmpDirs = envs.get(MesosConfigKeys.ENV_FLINK_TMP_DIR); BootstrapTools.updateTmpDirectoriesInConfiguration(configuration, tmpDirs); return configuration; } }
public static void main(String[] args) { // startup checks and logging EnvironmentInformation.logEnvironmentInfo(LOG, MesosSessionClusterEntrypoint.class.getSimpleName(), args); SignalHandler.register(LOG); JvmShutdownSafeguard.installAsShutdownHook(LOG); // load configuration incl. dynamic properties CommandLineParser parser = new PosixParser(); CommandLine cmd; try { cmd = parser.parse(ALL_OPTIONS, args); } catch (Exception e){ LOG.error("Could not parse the command-line options.", e); System.exit(STARTUP_FAILURE_RETURN_CODE); return; } Configuration dynamicProperties = BootstrapTools.parseDynamicProperties(cmd); Configuration configuration = MesosEntrypointUtils.loadConfiguration(dynamicProperties, LOG); MesosSessionClusterEntrypoint clusterEntrypoint = new MesosSessionClusterEntrypoint(configuration, dynamicProperties); ClusterEntrypoint.runClusterEntrypoint(clusterEntrypoint); }
BootstrapTools.substituteDeprecatedConfigPrefix(configuration, ConfigConstants.YARN_APPLICATION_MASTER_ENV_PREFIX, ResourceManagerOptions.CONTAINERIZED_MASTER_ENV_PREFIX); BootstrapTools.substituteDeprecatedConfigPrefix(configuration, ConfigConstants.YARN_TASK_MANAGER_ENV_PREFIX, ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX); BootstrapTools.updateTmpDirectoriesInConfiguration(configuration, localDirs);
public static void main(String[] args) { // startup checks and logging EnvironmentInformation.logEnvironmentInfo(LOG, MesosJobClusterEntrypoint.class.getSimpleName(), args); SignalHandler.register(LOG); JvmShutdownSafeguard.installAsShutdownHook(LOG); // load configuration incl. dynamic properties CommandLineParser parser = new PosixParser(); CommandLine cmd; try { cmd = parser.parse(ALL_OPTIONS, args); } catch (Exception e){ LOG.error("Could not parse the command-line options.", e); System.exit(STARTUP_FAILURE_RETURN_CODE); return; } Configuration dynamicProperties = BootstrapTools.parseDynamicProperties(cmd); Configuration configuration = MesosEntrypointUtils.loadConfiguration(dynamicProperties, LOG); MesosJobClusterEntrypoint clusterEntrypoint = new MesosJobClusterEntrypoint(configuration, dynamicProperties); ClusterEntrypoint.runClusterEntrypoint(clusterEntrypoint); } }