/** * Creates a local temporary directory for this MapReduce run. */ private File createTempDirectory() { ProgramId programId = context.getProgram().getId(); File tempDir = new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR), cConf.get(Constants.AppFabric.TEMP_DIR)).getAbsoluteFile(); File runtimeServiceDir = new File(tempDir, "runner"); File dir = new File(runtimeServiceDir, String.format("%s.%s.%s.%s.%s", programId.getType().name().toLowerCase(), programId.getNamespace(), programId.getApplication(), programId.getProgram(), context.getRunId().getId())); dir.mkdirs(); return dir; }
/** * Creates a local temporary directory for this MapReduce run. */ private File createTempDirectory() { ProgramId programId = context.getProgram().getId(); File tempDir = new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR), cConf.get(Constants.AppFabric.TEMP_DIR)).getAbsoluteFile(); File runtimeServiceDir = new File(tempDir, "runner"); File dir = new File(runtimeServiceDir, String.format("%s.%s.%s.%s.%s", programId.getType().name().toLowerCase(), programId.getNamespace(), programId.getApplication(), programId.getProgram(), context.getRunId().getId())); dir.mkdirs(); return dir; }
private String getJobName(BasicMapReduceContext context) { ProgramId programId = context.getProgram().getId(); // MRJobClient expects the following format (for RunId to be the first component) return String.format("%s.%s.%s.%s.%s", context.getRunId().getId(), ProgramType.MAPREDUCE.name().toLowerCase(), programId.getNamespace(), programId.getApplication(), programId.getProgram()); }
private String getJobName(BasicMapReduceContext context) { ProgramId programId = context.getProgram().getId(); // MRJobClient expects the following format (for RunId to be the first component) return String.format("%s.%s.%s.%s.%s", context.getRunId().getId(), ProgramType.MAPREDUCE.name().toLowerCase(), programId.getNamespace(), programId.getApplication(), programId.getProgram()); }
this.mapReduceRunId = context.getProgram().getId().run(context.getRunId().getId()); this.clusterMode = clusterMode;
this.mapReduceRunId = context.getProgram().getId().run(context.getRunId().getId()); this.clusterMode = clusterMode;
/** * Creates a temporary directory through the {@link LocationFactory} provided to this class. */ private Location createTempLocationDirectory() throws IOException { ProgramId programId = context.getProgram().getId(); String tempLocationName = String.format("%s/%s.%s.%s.%s.%s", cConf.get(Constants.AppFabric.TEMP_DIR), programId.getType().name().toLowerCase(), programId.getNamespace(), programId.getApplication(), programId.getProgram(), context.getRunId().getId()); Location location = locationFactory.get(programId.getNamespaceId()).append(tempLocationName); location.mkdirs(); return location; }
/** * Creates a temporary directory through the {@link LocationFactory} provided to this class. */ private Location createTempLocationDirectory() throws IOException { ProgramId programId = context.getProgram().getId(); String tempLocationName = String.format("%s/%s.%s.%s.%s.%s", cConf.get(Constants.AppFabric.TEMP_DIR), programId.getType().name().toLowerCase(), programId.getNamespace(), programId.getApplication(), programId.getProgram(), context.getRunId().getId()); Location location = locationFactory.get(programId.getNamespaceId()).append(tempLocationName); location.mkdirs(); return location; }
private void setDecoderForStream(StreamInputFormatProvider streamProvider, Job job, Map<String, String> inputFormatConfiguration, Class<? extends Mapper> mapperClass) { // For stream, we need to do two extra steps. // 1. stream usage registration since it only happens on client side. // 2. Infer the stream event decoder from Mapper/Reducer TypeToken<?> mapperTypeToken = mapperClass == null ? null : resolveClass(mapperClass, Mapper.class); Type inputValueType = getInputValueType(job.getConfiguration(), StreamEvent.class, mapperTypeToken); streamProvider.setDecoderType(inputFormatConfiguration, inputValueType); StreamId streamId = streamProvider.getStreamId(); try { streamAdmin.register(ImmutableList.of(context.getProgram().getId()), streamId); streamAdmin.addAccess(context.getProgram().getId().run(context.getRunId().getId()), streamId, AccessType.READ); } catch (Exception e) { LOG.warn("Failed to register usage {} -> {}", context.getProgram().getId(), streamId, e); } }
this.loggingContext = createLoggingContext(program.getId(), getRunId(), workflowProgramInfo); this.spec = spec; this.mapperResources = SystemArguments.getResources(getMapperRuntimeArguments(), spec.getMapperResources());
this.loggingContext = createLoggingContext(program.getId(), getRunId(), workflowProgramInfo); this.spec = spec; this.mapperResources = SystemArguments.getResources(getMapperRuntimeArguments(), spec.getMapperResources());