/** * Read a splits file into a list of raw splits * @param in the stream to read from * @return the complete list of splits * @throws IOException */ static RawSplit[] readSplitFile(DataInput in) throws IOException { byte[] header = new byte[SPLIT_FILE_HEADER.length]; in.readFully(header); if (!Arrays.equals(SPLIT_FILE_HEADER, header)) { throw new IOException("Invalid header on split file"); } int vers = WritableUtils.readVInt(in); if (vers != CURRENT_SPLIT_FILE_VERSION) { throw new IOException("Unsupported split version " + vers); } int len = WritableUtils.readVInt(in); RawSplit[] result = new RawSplit[len]; for(int i=0; i < len; ++i) { result[i] = new RawSplit(); result[i].readFields(in); } return result; }
/** * Read a splits file into a list of raw splits * @param in the stream to read from * @return the complete list of splits * @throws IOException */ static RawSplit[] readSplitFile(DataInput in) throws IOException { byte[] header = new byte[SPLIT_FILE_HEADER.length]; in.readFully(header); if (!Arrays.equals(SPLIT_FILE_HEADER, header)) { throw new IOException("Invalid header on split file"); } int vers = WritableUtils.readVInt(in); if (vers != CURRENT_SPLIT_FILE_VERSION) { throw new IOException("Unsupported split version " + vers); } int len = WritableUtils.readVInt(in); RawSplit[] result = new RawSplit[len]; for(int i=0; i < len; ++i) { result[i] = new RawSplit(); result[i].readFields(in); } return result; }
serializer.open(buffer); for(T split: array) { RawSplit rawSplit = new RawSplit(); rawSplit.setClassName(split.getClass().getName()); buffer.reset();
DataOutputBuffer buffer = new DataOutputBuffer(); for(InputSplit split: splits) { RawSplit rawSplit = new RawSplit(); rawSplit.setClassName(split.getClass().getName()); buffer.reset();
private void initSetupCleanupTasks(String jobFile) { if (!jobSetupCleanupNeeded) { LOG.info("Setup/Cleanup not needed for job" + jobId); // nothing to initialize return; } // create cleanup two cleanup tips, one map and one reduce. cleanup = new TaskInProgress[2]; // cleanup map tip. This map doesn't use any splits. Just assign an empty // split. JobClient.RawSplit emptySplit = new JobClient.RawSplit(); cleanup[0] = new TaskInProgress(jobId, jobFile, emptySplit, conf, this, numMapTasks, 1); cleanup[0].setJobCleanupTask(); // cleanup reduce tip. cleanup[1] = new TaskInProgress(jobId, jobFile, numMapTasks, numReduceTasks, conf, this, 1); cleanup[1].setJobCleanupTask(); // create two setup tips, one map and one reduce. setup = new TaskInProgress[2]; // setup map tip. This map doesn't use any split. Just assign an empty // split. setup[0] = new TaskInProgress(jobId, jobFile, emptySplit, conf, this, numMapTasks + 1, 1); setup[0].setJobSetupTask(); // setup reduce tip. setup[1] = new TaskInProgress(jobId, jobFile, numMapTasks, numReduceTasks + 1, conf, this, 1); setup[1].setJobSetupTask(); }
/** Create the list of input splits and write them out in a file for *the JobTracker. The format is: * <format version> * <numSplits> * for each split: * <RawSplit> * @param splits the input splits to write out * @param out the stream to write to */ private void writeSplitsFile(InputSplit[] splits, FSDataOutputStream out) throws IOException { out.write(SPLIT_FILE_HEADER); WritableUtils.writeVInt(out, CURRENT_SPLIT_FILE_VERSION); WritableUtils.writeVInt(out, splits.length); DataOutputBuffer buffer = new DataOutputBuffer(); RawSplit rawSplit = new RawSplit(); for(InputSplit split: splits) { rawSplit.setClassName(split.getClass().getName()); buffer.reset(); split.write(buffer); rawSplit.setDataLength(split.getLength()); rawSplit.setBytes(buffer.getData(), 0, buffer.getLength()); rawSplit.setLocations(split.getLocations()); rawSplit.write(out); } }