private static MapReduceContextConfig createContextConfig() { Configuration conf = new Configuration(new YarnConfiguration()); conf.addResource(new Path(MRJobConfig.JOB_CONF_FILE)); return new MapReduceContextConfig(conf); }
private static MapReduceContextConfig createContextConfig() { Configuration conf = new Configuration(new YarnConfiguration()); conf.addResource(new Path(MRJobConfig.JOB_CONF_FILE)); return new MapReduceContextConfig(conf); }
public DistributedMapReduceTaskContextProvider(CConfiguration cConf, Configuration hConf, MapReduceClassLoader mapReduceClassLoader) { super(createInjector(cConf, hConf), mapReduceClassLoader); MapReduceContextConfig mapReduceContextConfig = new MapReduceContextConfig(hConf); Injector injector = getInjector(); Deque<Service> coreServices = new LinkedList<>(); coreServices.add(injector.getInstance(ZKClientService.class)); coreServices.add(injector.getInstance(MetricsCollectionService.class)); if (ProgramRunners.getClusterMode(mapReduceContextConfig.getProgramOptions()) == ClusterMode.ON_PREMISE) { coreServices.add(injector.getInstance(KafkaClientService.class)); } this.coreServices = coreServices; this.logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class); this.mapReduceContextConfig = new MapReduceContextConfig(hConf); }
public DistributedMapReduceTaskContextProvider(CConfiguration cConf, Configuration hConf, MapReduceClassLoader mapReduceClassLoader) { super(createInjector(cConf, hConf), mapReduceClassLoader); MapReduceContextConfig mapReduceContextConfig = new MapReduceContextConfig(hConf); Injector injector = getInjector(); Deque<Service> coreServices = new LinkedList<>(); coreServices.add(injector.getInstance(ZKClientService.class)); coreServices.add(injector.getInstance(MetricsCollectionService.class)); if (ProgramRunners.getClusterMode(mapReduceContextConfig.getProgramOptions()) == ClusterMode.ON_PREMISE) { coreServices.add(injector.getInstance(KafkaClientService.class)); } this.coreServices = coreServices; this.logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class); this.mapReduceContextConfig = new MapReduceContextConfig(hConf); }
private static Injector createInjector(CConfiguration cConf, Configuration hConf) { MapReduceContextConfig mapReduceContextConfig = new MapReduceContextConfig(hConf); // principal will be null if running on a kerberos distributed cluster ProgramOptions programOptions = mapReduceContextConfig.getProgramOptions(); Arguments systemArgs = programOptions.getArguments(); String runId = systemArgs.getOption(ProgramOptionConstants.RUN_ID); return Guice.createInjector( new DistributedProgramContainerModule(cConf, hConf, mapReduceContextConfig.getProgramId().run(runId), systemArgs) ); } }
private static Injector createInjector(CConfiguration cConf, Configuration hConf) { MapReduceContextConfig mapReduceContextConfig = new MapReduceContextConfig(hConf); // principal will be null if running on a kerberos distributed cluster ProgramOptions programOptions = mapReduceContextConfig.getProgramOptions(); Arguments systemArgs = programOptions.getArguments(); String runId = systemArgs.getOption(ProgramOptionConstants.RUN_ID); return Guice.createInjector( new DistributedProgramContainerModule(cConf, hConf, mapReduceContextConfig.getProgramId().run(runId), systemArgs) ); } }
@Test public void testManyMacrosInAppSpec() { Configuration hConf = new Configuration(); MapReduceContextConfig cfg = new MapReduceContextConfig(hConf); StringBuilder appCfg = new StringBuilder(); for (int i = 0; i < 100; i++) { appCfg.append("${").append(i).append("}"); hConf.setInt(String.valueOf(i), i); } ApplicationSpecification appSpec = new DefaultApplicationSpecification( "name", "desc", appCfg.toString(), new ArtifactId("artifact", new ArtifactVersion("1.0.0"), ArtifactScope.USER), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap() ); cfg.setApplicationSpecification(appSpec); Assert.assertEquals(appSpec.getConfiguration(), cfg.getApplicationSpecification().getConfiguration()); }
@Test public void testGetPluginsWithMacrosMoreThan20() { Configuration hConf = new Configuration(); MapReduceContextConfig cfg = new MapReduceContextConfig(hConf); Map<String, Plugin> mockPlugins = new HashMap<>(); ArtifactId artifactId = new ArtifactId("plugins", new ArtifactVersion("1.0.0"), ArtifactScope.SYSTEM); Map<String, String> properties = new HashMap<>(); properties.put("path", "${input.directory}/${a}${b}${c}${d}${e}${f}${g}${h}${i}${j}" + "${k}${l}${m}${n}${o}${p}${q}${r}${s}${t}${u}${v}${w}${x}${y}${z}.txt"); hConf.set("input.directory", "/dummy/path"); String[] alphabetsArr = {"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"}; for (String alphabet : alphabetsArr) { hConf.set(alphabet, alphabet); } Set<ArtifactId> parents = new LinkedHashSet<>(); Plugin filePlugin1 = new Plugin(parents, artifactId, new PluginClass("type", "name", "desc", "clsname", "cfgfield", Collections.emptyMap()), PluginProperties.builder().addAll(properties).build()); mockPlugins.put("File1", filePlugin1); hConf.set(MapReduceContextConfig.HCONF_ATTR_PLUGINS, GSON.toJson(mockPlugins)); Map<String, Plugin> plugins = cfg.getPlugins(); Assert.assertEquals(plugins, mockPlugins); } }
MapReduceContextConfig contextConfig = new MapReduceContextConfig(jobContext.getConfiguration());
MapReduceContextConfig contextConfig = new MapReduceContextConfig(jobContext.getConfiguration());
MapReduceContextConfig contextConfig = new MapReduceContextConfig(mapredConf); CConfiguration cConfCopy = CConfiguration.copy(cConf); if (hbaseDDLExecutorDirectory != null) {
MapReduceContextConfig contextConfig = new MapReduceContextConfig(mapredConf); CConfiguration cConfCopy = CConfiguration.copy(cConf); if (hbaseDDLExecutorDirectory != null) {
MapReduceContextConfig contextConfig = new MapReduceContextConfig(key.getConfiguration()); MapReduceClassLoader classLoader = MapReduceClassLoader.getFromConfiguration(key.getConfiguration());
MapReduceContextConfig contextConfig = new MapReduceContextConfig(key.getConfiguration()); MapReduceClassLoader classLoader = MapReduceClassLoader.getFromConfiguration(key.getConfiguration());
/** * Creates logging context for MapReduce program. If the program is started * by Workflow an instance of {@link WorkflowProgramLoggingContext} is returned, * otherwise an instance of {@link MapReduceLoggingContext} is returned. */ private LoggingContext createMapReduceLoggingContext() { MapReduceContextConfig contextConfig = new MapReduceContextConfig(parameters.getHConf()); ProgramId programId = contextConfig.getProgramId(); RunId runId = ProgramRunners.getRunId(contextConfig.getProgramOptions()); WorkflowProgramInfo workflowProgramInfo = contextConfig.getWorkflowProgramInfo(); if (workflowProgramInfo == null) { return new MapReduceLoggingContext(programId.getNamespace(), programId.getApplication(), programId.getProgram(), runId.getId()); } String workflowId = workflowProgramInfo.getName(); String workflowRunId = workflowProgramInfo.getRunId().getId(); return new WorkflowProgramLoggingContext(programId.getNamespace(), programId.getApplication(), workflowId, workflowRunId, ProgramType.MAPREDUCE, programId.getProgram(), runId.getId()); }
/** * Creates logging context for MapReduce program. If the program is started * by Workflow an instance of {@link WorkflowProgramLoggingContext} is returned, * otherwise an instance of {@link MapReduceLoggingContext} is returned. */ private LoggingContext createMapReduceLoggingContext() { MapReduceContextConfig contextConfig = new MapReduceContextConfig(parameters.getHConf()); ProgramId programId = contextConfig.getProgramId(); RunId runId = ProgramRunners.getRunId(contextConfig.getProgramOptions()); WorkflowProgramInfo workflowProgramInfo = contextConfig.getWorkflowProgramInfo(); if (workflowProgramInfo == null) { return new MapReduceLoggingContext(programId.getNamespace(), programId.getApplication(), programId.getProgram(), runId.getId()); } String workflowId = workflowProgramInfo.getName(); String workflowRunId = workflowProgramInfo.getRunId().getId(); return new WorkflowProgramLoggingContext(programId.getNamespace(), programId.getApplication(), workflowId, workflowRunId, ProgramType.MAPREDUCE, programId.getProgram(), runId.getId()); }