static synchronized String getOutputName(TaskAttemptContext context) { return context.getConfiguration().get("mapreduce.output.basename", "part") + "-" + NUMBER_FORMAT.format(context.getTaskAttemptID().getTaskID().getId()); }
private void futureGet(Context context) throws IOException { try { future.get(); } catch (Exception e) { throw new IOException("Failed to build cube in mapper " + context.getTaskAttemptID().getTaskID().getId(), e); } } }
@Override public void run(Context ctx) throws IOException, InterruptedException { System.out.println("Running task: " + ctx.getTaskAttemptID().getTaskID().getId()); latch.get("mapAwaitLatch").await(); mapExecCnt.incrementAndGet(); System.out.println("Completed task: " + ctx.getTaskAttemptID().getTaskID().getId()); } }
@Override public void run(Context ctx) throws IOException, InterruptedException { System.out.println("Running task: " + ctx.getTaskAttemptID().getTaskID().getId()); latch.get("reduceAwaitLatch").await(); reduceExecCnt.incrementAndGet(); System.out.println("Completed task: " + ctx.getTaskAttemptID().getTaskID().getId()); } }
@Override public void run(Context ctx) throws IOException, InterruptedException { System.out.println("Running task: " + ctx.getTaskAttemptID().getTaskID().getId()); latch.get("combineAwaitLatch").await(); combineExecCnt.incrementAndGet(); System.out.println("Completed task: " + ctx.getTaskAttemptID().getTaskID().getId()); } }
@Override public RecordReader<LongWritable, LongWritable> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { int taskId = context.getTaskAttemptID().getTaskID().getId(); int numMapTasks = context.getConfiguration().getInt(NUM_MAPS_KEY, NUM_MAPS); int numIterations = context.getConfiguration().getInt(NUM_IMPORT_ROUNDS_KEY, NUM_IMPORT_ROUNDS); int iteration = context.getConfiguration().getInt(ROUND_NUM_KEY, 0); taskId = taskId + iteration * numMapTasks; numMapTasks = numMapTasks * numIterations; long chainId = Math.abs(new Random().nextLong()); chainId = chainId - (chainId % numMapTasks) + taskId; // ensure that chainId is unique per task and across iterations LongWritable[] keys = new LongWritable[] {new LongWritable(chainId)}; return new FixedRecordReader<>(keys, keys); } }
@Override protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); Configuration conf = context.getConfiguration(); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(); String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME); CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName); cubeConfig = cube.getConfig(); baseCuboidId = cube.getCuboidScheduler().getBaseCuboidId(); baseCuboidRowCountInMappers = Lists.newLinkedList(); output = conf.get(BatchConstants.CFG_OUTPUT_PATH); samplingPercentage = Integer .parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT)); taskId = context.getTaskAttemptID().getTaskID().getId(); cuboidHLLMap = Maps.newHashMap(); }
/** * @param ctx Context for IO operations. */ public HadoopV2Context(HadoopV2TaskContext ctx) { super(ctx.jobConf(), ctx.jobContext().getJobID()); taskAttemptID = ctx.attemptId(); conf.set(MRJobConfig.ID, taskAttemptID.getJobID().toString()); conf.set(MRJobConfig.TASK_ID, taskAttemptID.getTaskID().toString()); conf.set(MRJobConfig.TASK_ATTEMPT_ID, taskAttemptID.toString()); output = ctx.output(); input = ctx.input(); this.ctx = ctx; }
cubeDesc = cube.getDescriptor(); taskId = context.getTaskAttemptID().getTaskID().getId();
/** * Write random values to the writer assuming a table created using * {@link #FAMILIES} as column family descriptors */ private void writeRandomKeyValues(RecordWriter<ImmutableBytesWritable, Cell> writer, TaskAttemptContext context, Set<byte[]> families, int numRows) throws IOException, InterruptedException { byte keyBytes[] = new byte[Bytes.SIZEOF_INT]; int valLength = 10; byte valBytes[] = new byte[valLength]; int taskId = context.getTaskAttemptID().getTaskID().getId(); assert taskId < Byte.MAX_VALUE : "Unit tests dont support > 127 tasks!"; final byte [] qualifier = Bytes.toBytes("data"); Random random = new Random(); for (int i = 0; i < numRows; i++) { Bytes.putInt(keyBytes, 0, i); random.nextBytes(valBytes); ImmutableBytesWritable key = new ImmutableBytesWritable(keyBytes); for (byte[] family : families) { Cell kv = new KeyValue(keyBytes, family, qualifier, valBytes); writer.write(key, kv); } } }
byte valBytes[] = new byte[valLength]; int taskId = context.getTaskAttemptID().getTaskID().getId(); assert taskId < Byte.MAX_VALUE : "Unit tests dont support > 127 tasks!";
byte valBytes[] = new byte[valLength]; int taskId = context.getTaskAttemptID().getTaskID().getId(); assert taskId < Byte.MAX_VALUE : "Unit tests dont support > 127 tasks!"; Random random = new Random();
@Override protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); Configuration conf = context.getConfiguration(); mos = new MultipleOutputs(context); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(); String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME); CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName); CubeDesc cubeDesc = cube.getDescriptor(); List<TblColRef> uhcColumns = cubeDesc.getAllUHCColumns(); int taskId = context.getTaskAttemptID().getTaskID().getId(); col = uhcColumns.get(taskId); logger.info("column name: " + col.getIdentity()); if (cube.getDescriptor().getShardByColumns().contains(col)) { //for ShardByColumns builder = DictionaryGenerator.newDictionaryBuilder(col.getType()); builder.init(null, 0, null); } else { //for GlobalDictionaryColumns String hdfsDir = conf.get(BatchConstants.CFG_GLOBAL_DICT_BASE_DIR); DictionaryInfo dictionaryInfo = new DictionaryInfo(col.getColumnDesc(), col.getDatatype()); String builderClass = cubeDesc.getDictionaryBuilderClass(col); builder = (IDictionaryBuilder) ClassUtil.newInstance(builderClass); builder.init(dictionaryInfo, 0, hdfsDir); } }
/** Get the task id */ public TaskID getTaskId() { return attemptId.getTaskID(); } /** Get the task type */
private String getOutFileName(TaskAttemptContext context, String prefix) { TaskID taskId = context.getTaskAttemptID().getTaskID(); int partition = taskId.getId(); NumberFormat nf = NumberFormat.getInstance(); nf.setMinimumIntegerDigits(5); nf.setGroupingUsed(false); StringBuilder result = new StringBuilder(); result.append(prefix); result.append("-"); result.append(nf.format(partition)); return result.toString(); }
private void futureGet(Context context) throws IOException { try { future.get(); } catch (Exception e) { throw new IOException("Failed to build cube in mapper " + context.getTaskAttemptID().getTaskID().getId(), e); } } }
@Deprecated public OnDiskMapOutput(TaskAttemptID mapId, TaskAttemptID reduceId, MergeManagerImpl<K,V> merger, long size, JobConf conf, MapOutputFile mapOutputFile, int fetcher, boolean primaryMapOutput) throws IOException { this(mapId, merger, size, conf, fetcher, primaryMapOutput, FileSystem.getLocal(conf).getRaw(), mapOutputFile.getInputFileForWrite(mapId.getTaskID(), size)); }
private void verifyPartitionAssignment(Context context) { if ("true".equals(System.getProperty("verifyPartitionAssignment", "true"))) { String partitionStr = context.getConfiguration().get("mapred.task.partition"); if (partitionStr == null) { partitionStr = context.getConfiguration().get("mapreduce.task.partition"); } int partition = Integer.parseInt(partitionStr); int taskId = context.getTaskAttemptID().getTaskID().getId(); Preconditions.checkArgument(partition == taskId, "mapred.task.partition: " + partition + " not equal to reducer taskId: " + taskId); } }
public static TaskAttemptId toYarn( org.apache.hadoop.mapreduce.TaskAttemptID id) { TaskAttemptId taskAttemptId = recordFactory.newRecordInstance(TaskAttemptId.class); taskAttemptId.setTaskId(toYarn(id.getTaskID())); taskAttemptId.setId(id.getId()); return taskAttemptId; }
private static TaskAttemptContext getTaskContext(TaskAttemptContext baseContext, Job job) { org.apache.hadoop.mapreduce.TaskAttemptID baseTaskId = baseContext.getTaskAttemptID(); // Create a task ID context with our specialized job ID. org.apache.hadoop.mapreduce.TaskAttemptID taskId; taskId = new org.apache.hadoop.mapreduce.TaskAttemptID(job.getJobID().getJtIdentifier(), job.getJobID().getId(), baseTaskId.isMap(), baseTaskId.getTaskID().getId(), baseTaskId.getId()); return new TaskAttemptContextWrapper(baseContext, job.getConfiguration(), taskId); }