public ProgramLaunchConfig addRunnable(String name, TwillRunnable runnable, int instances, Map<String, String> args, Resources defaultResources, @Nullable Integer maxRetries) { ResourceSpecification resourceSpec = createResourceSpec(SystemArguments.getResources(args, defaultResources), instances); Map<String, String> configs = SystemArguments.getTwillContainerConfigs(args, resourceSpec.getMemorySize()); Map<String, Level> logLevels = SystemArguments.getLogLevels(args); runnables.put(name, new RunnableDefinition(runnable, resourceSpec, configs, logLevels, maxRetries)); return this; }
/** * Get the profile id for the given program, given arguments for a run. All non-workflow program types will use the * native profile. Workflow program types will use whatever profile is specified in its arguments, or the native * profile if none is specified. * * @param programId program to get the profile for * @param args arguments for a program run * @return the profile id for the program run */ public static ProfileId getProfileIdForProgram(ProgramId programId, Map<String, String> args) { if (programId.getType() == ProgramType.WORKFLOW) { return getProfileIdFromArgs(programId.getNamespaceId(), args).orElse(ProfileId.NATIVE); } return ProfileId.NATIVE; }
/** * Get the profile id for the given program, given arguments for a run. All non-workflow program types will use the * native profile. Workflow program types will use whatever profile is specified in its arguments, or the native * profile if none is specified. * * @param programId program to get the profile for * @param args arguments for a program run * @return the profile id for the program run */ public static ProfileId getProfileIdForProgram(ProgramId programId, Map<String, String> args) { if (isProfileAllowed(programId.getType())) { return getProfileIdFromArgs(programId.getNamespaceId(), args).orElse(ProfileId.NATIVE); } return ProfileId.NATIVE; }
/** * Returns the {@link Resources} based on configurations in the given arguments. * * Same as calling {@link #getResources(Map, Resources)} with first argument from {@link Arguments#asMap()}. */ public static Resources getResources(Arguments args, @Nullable Resources defaultResources) { return getResources(args.asMap(), defaultResources); }
Resources resources = SystemArguments.getResources(ImmutableMap.of(), defaultResources); Assert.assertEquals(defaultResources, resources); resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.MEMORY_KEY, "10"), defaultResources); Assert.assertEquals(new Resources(10), resources); resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.CORES_KEY, "8"), defaultResources); Assert.assertEquals(new Resources(defaultResources.getMemoryMB(), 8), resources); resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.MEMORY_KEY, "10", SystemArguments.CORES_KEY, "8"), defaultResources); Assert.assertEquals(new Resources(10, 8), resources); resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.MEMORY_KEY, "-10"), defaultResources); Assert.assertEquals(defaultResources, resources); resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.CORES_KEY, "abc"), defaultResources); Assert.assertEquals(defaultResources, resources); resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.MEMORY_KEY, "xyz", SystemArguments.CORES_KEY, "8"), defaultResources); Assert.assertEquals(new Resources(defaultResources.getMemoryMB(), 8), resources); resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.MEMORY_KEY, "10", SystemArguments.CORES_KEY, "-8"), defaultResources); Assert.assertEquals(new Resources(10, defaultResources.getVirtualCores()), resources); resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.MEMORY_KEY, "-1", SystemArguments.CORES_KEY, "-8"), defaultResources);
/** * Sets the Spark memory overhead setting based on the runtime arguments of the given type. * * @param runtimeArgs the runtime arguments * @param type either {@code driver} or {@code executor} * @param containerSize the memory size in MB of the container * @param configs the configuration to be updated */ private void setMemoryOverhead(Map<String, String> runtimeArgs, String type, int containerSize, Map<String, String> configs) { Map<String, String> memoryConfigs = SystemArguments.getTwillContainerConfigs(runtimeArgs, containerSize); if (!memoryConfigs.containsKey(Configs.Keys.JAVA_RESERVED_MEMORY_MB)) { return; } configs.put("spark.yarn." + type + ".memoryOverhead", memoryConfigs.get(Configs.Keys.JAVA_RESERVED_MEMORY_MB)); }
@Override protected void doStart() { try { initializer.initialize(); SystemArguments.setLogLevel(programOptions.getUserArguments(), initializer); notifyStarted(); } catch (Throwable t) { notifyFailed(t); } }
@VisibleForTesting ProgramOptions createProgramOptions(ProgramId programId, Map<String, String> userArgs, Map<String, String> sysArgs, boolean debug) throws NotFoundException, ProfileConflictException { ProfileId profileId = SystemArguments.getProfileIdForProgram(programId, userArgs); Map<String, String> profileProperties = SystemArguments.getProfileProperties(userArgs); Profile profile = profileService.getProfile(profileId, profileProperties); if (profile.getStatus() == ProfileStatus.DISABLED) { SystemArguments.addProfileArgs(systemArgs, profile);
RetryStrategy retryStrategy = SystemArguments.getRetryStrategy(options.getUserArguments().asMap(), program.getType(), cConf); ArtifactManager artifactManager = artifactManagerFactory.create(program.getId().getNamespaceId(), retryStrategy);
twillPreparer.setLogLevels(transformLogLevels(SystemArguments.getLogLevels(userArgs))); twillConfigs.putAll(SystemArguments.getTwillApplicationConfigs(userArgs)); twillPreparer.withConfiguration(twillConfigs);
/** * Set the log level for the {@link LogAppenderInitializer}. * * @param args the arguments to use for looking up resources configurations * @param initializer the LogAppenderInitializer which will be used to set up the log level */ public static void setLogLevel(Arguments args, LogAppenderInitializer initializer) { initializer.setLogLevels(getLogLevels(args.asMap())); }
@Test public void testGetProgramProfile() { ProfileId profileId = NamespaceId.DEFAULT.profile("p"); Map<String, String> args = Collections.singletonMap(SystemArguments.PROFILE_NAME, profileId.getScopedName()); ApplicationId appId = NamespaceId.DEFAULT.app("a"); ProgramId mrId = appId.mr("mr"); ProgramId serviceId = appId.service("serv"); ProgramId sparkId = appId.spark("spark"); ProgramId workerId = appId.worker("worker"); ProgramId workflowID = appId.workflow("wf"); Assert.assertEquals(profileId, SystemArguments.getProfileIdForProgram(mrId, args)); Assert.assertEquals(ProfileId.NATIVE, SystemArguments.getProfileIdForProgram(serviceId, args)); Assert.assertEquals(profileId, SystemArguments.getProfileIdForProgram(sparkId, args)); Assert.assertEquals(ProfileId.NATIVE, SystemArguments.getProfileIdForProgram(workerId, args)); Assert.assertEquals(profileId, SystemArguments.getProfileIdForProgram(workflowID, args)); }
private TaskFields createTaskInfo(ProvisionerInfo provisionerInfo) { ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").workflow("wf").run(RunIds.generate()); Map<String, String> systemArgs = new HashMap<>(); Map<String, String> userArgs = new HashMap<>(); Profile profile = new Profile(ProfileId.NATIVE.getProfile(), "label", "desc", provisionerInfo); SystemArguments.addProfileArgs(systemArgs, profile); ProgramOptions programOptions = new SimpleProgramOptions(programRunId.getParent(), new BasicArguments(systemArgs), new BasicArguments(userArgs)); ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId(); ApplicationSpecification appSpec = new DefaultApplicationSpecification( "name", "1.0.0", "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); ProgramDescriptor programDescriptor = new ProgramDescriptor(programRunId.getParent(), appSpec); return new TaskFields(programDescriptor, programOptions, programRunId); }
/** * Returns the {@link Resources} based on configurations in the given arguments. * * Same as calling {@link #getResources(Map, Resources)} with first argument from {@link Arguments#asMap()}. */ public static Resources getResources(Arguments args, @Nullable Resources defaultResources) { return getResources(args.asMap(), defaultResources); }
/** * Sets the Spark memory overhead setting based on the runtime arguments of the given type. * * @param runtimeArgs the runtime arguments * @param type either {@code driver} or {@code executor} * @param containerSize the memory size in MB of the container * @param configs the configuration to be updated */ private void setMemoryOverhead(Map<String, String> runtimeArgs, String type, int containerSize, Map<String, String> configs) { Map<String, String> memoryConfigs = SystemArguments.getTwillContainerConfigs(runtimeArgs, containerSize); if (!memoryConfigs.containsKey(Configs.Keys.JAVA_RESERVED_MEMORY_MB)) { return; } configs.put("spark.yarn." + type + ".memoryOverhead", memoryConfigs.get(Configs.Keys.JAVA_RESERVED_MEMORY_MB)); }
@Override protected void doStart() { try { initializer.initialize(); SystemArguments.setLogLevel(programOptions.getUserArguments(), initializer); notifyStarted(); } catch (Throwable t) { notifyFailed(t); } }
@VisibleForTesting ProgramOptions createProgramOptions(ProgramId programId, Map<String, String> userArgs, Map<String, String> sysArgs, boolean debug) throws NotFoundException, ProfileConflictException { ProfileId profileId = SystemArguments.getProfileIdForProgram(programId, userArgs); Map<String, String> profileProperties = SystemArguments.getProfileProperties(userArgs); Profile profile = profileService.getProfile(profileId, profileProperties); if (profile.getStatus() == ProfileStatus.DISABLED) { SystemArguments.addProfileArgs(systemArgs, profile);
SystemArguments.getRetryStrategy(contextConfig.getProgramOptions().getUserArguments().asMap(), contextConfig.getProgramId().getType(), cConf); this.txClient = new RetryingLongTransactionSystemClient(injector.getInstance(TransactionSystemClient.class),
twillPreparer.setLogLevels(transformLogLevels(SystemArguments.getLogLevels(userArgs))); twillConfigs.putAll(SystemArguments.getTwillApplicationConfigs(userArgs)); twillPreparer.withConfiguration(twillConfigs);
/** * Set the log level for the {@link LogAppenderInitializer}. * * @param args the arguments to use for looking up resources configurations * @param initializer the LogAppenderInitializer which will be used to set up the log level */ public static void setLogLevel(Arguments args, LogAppenderInitializer initializer) { initializer.setLogLevels(getLogLevels(args.asMap())); }