/** * Returns the {@link Resources} based on configurations in the given arguments. * * Same as calling {@link #getResources(Map, Resources)} with first argument from {@link Arguments#asMap()}. */ public static Resources getResources(Arguments args, @Nullable Resources defaultResources) { return getResources(args.asMap(), defaultResources); }
/** * Returns the {@link Resources} based on configurations in the given arguments. * * Same as calling {@link #getResources(Map, Resources)} with first argument from {@link Arguments#asMap()}. */ public static Resources getResources(Arguments args, @Nullable Resources defaultResources) { return getResources(args.asMap(), defaultResources); }
BasicSparkClientContext(SparkRuntimeContext sparkRuntimeContext) { this.sparkRuntimeContext = sparkRuntimeContext; this.localizeResources = new HashMap<>(); this.additionalPythonLocations = new LinkedList<>(); SparkSpecification spec = sparkRuntimeContext.getSparkSpecification(); this.driverResources = SystemArguments.getResources(getDriverRuntimeArguments(), spec.getDriverResources()); this.executorResources = SystemArguments.getResources(getExecutorRuntimeArguments(), spec.getExecutorResources()); }
BasicSparkClientContext(SparkRuntimeContext sparkRuntimeContext) { this.sparkRuntimeContext = sparkRuntimeContext; this.localizeResources = new HashMap<>(); this.additionalPythonLocations = new LinkedList<>(); SparkSpecification spec = sparkRuntimeContext.getSparkSpecification(); this.driverResources = SystemArguments.getResources(getDriverRuntimeArguments(), spec.getDriverResources()); this.executorResources = SystemArguments.getResources(getExecutorRuntimeArguments(), spec.getExecutorResources()); }
BasicSparkClientContext(SparkRuntimeContext sparkRuntimeContext) { this.sparkRuntimeContext = sparkRuntimeContext; this.localizeResources = new HashMap<>(); this.additionalPythonLocations = new LinkedList<>(); SparkSpecification spec = sparkRuntimeContext.getSparkSpecification(); this.driverResources = SystemArguments.getResources(getDriverRuntimeArguments(), spec.getDriverResources()); this.executorResources = SystemArguments.getResources(getExecutorRuntimeArguments(), spec.getExecutorResources()); }
public ProgramLaunchConfig addRunnable(String name, TwillRunnable runnable, int instances, Map<String, String> args, Resources defaultResources, @Nullable Integer maxRetries) { ResourceSpecification resourceSpec = createResourceSpec(SystemArguments.getResources(args, defaultResources), instances); Map<String, String> configs = SystemArguments.getTwillContainerConfigs(args, resourceSpec.getMemorySize()); Map<String, Level> logLevels = SystemArguments.getLogLevels(args); runnables.put(name, new RunnableDefinition(runnable, resourceSpec, configs, logLevels, maxRetries)); return this; }
public ProgramLaunchConfig addRunnable(String name, TwillRunnable runnable, int instances, Map<String, String> args, Resources defaultResources, @Nullable Integer maxRetries) { ResourceSpecification resourceSpec = createResourceSpec(SystemArguments.getResources(args, defaultResources), instances); Map<String, String> configs = SystemArguments.getTwillContainerConfigs(args, resourceSpec.getMemorySize()); Map<String, Level> logLevels = SystemArguments.getLogLevels(args); runnables.put(name, new RunnableDefinition(runnable, resourceSpec, configs, logLevels, maxRetries)); return this; }
this.loggingContext = createLoggingContext(program.getId(), getRunId(), workflowProgramInfo); this.spec = spec; this.mapperResources = SystemArguments.getResources(getMapperRuntimeArguments(), spec.getMapperResources()); this.reducerResources = SystemArguments.getResources(getReducerRuntimeArguments(), spec.getReducerResources()); this.pluginArchive = pluginArchive; this.resourcesToLocalize = new HashMap<>();
this.loggingContext = createLoggingContext(program.getId(), getRunId(), workflowProgramInfo); this.spec = spec; this.mapperResources = SystemArguments.getResources(getMapperRuntimeArguments(), spec.getMapperResources()); this.reducerResources = SystemArguments.getResources(getReducerRuntimeArguments(), spec.getReducerResources()); this.streamAdmin = streamAdmin; this.pluginArchive = pluginArchive;
Resources resources = SystemArguments.getResources(ImmutableMap.of(), defaultResources); Assert.assertEquals(defaultResources, resources); resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.MEMORY_KEY, "10"), defaultResources); Assert.assertEquals(new Resources(10), resources); resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.CORES_KEY, "8"), defaultResources); Assert.assertEquals(new Resources(defaultResources.getMemoryMB(), 8), resources); resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.MEMORY_KEY, "10", SystemArguments.CORES_KEY, "8"), defaultResources); Assert.assertEquals(new Resources(10, 8), resources); resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.MEMORY_KEY, "-10"), defaultResources); Assert.assertEquals(defaultResources, resources); resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.CORES_KEY, "abc"), defaultResources); Assert.assertEquals(defaultResources, resources); resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.MEMORY_KEY, "xyz", SystemArguments.CORES_KEY, "8"), defaultResources); Assert.assertEquals(new Resources(defaultResources.getMemoryMB(), 8), resources); resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.MEMORY_KEY, "10", SystemArguments.CORES_KEY, "-8"), defaultResources); Assert.assertEquals(new Resources(10, defaultResources.getVirtualCores()), resources); resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.MEMORY_KEY, "-1", SystemArguments.CORES_KEY, "-8"), defaultResources);