public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { sequenceFileRecordReader.initialize(split, context); }
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { sequenceFileRecordReader.initialize(split, context); }
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { sequenceFileRecordReader.initialize(split, context); }
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { sequenceFileRecordReader.initialize(split, context); }
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { sequenceFileRecordReader.initialize(split, context); }
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { sequenceFileRecordReader.initialize(split, context); }
private void openForRead(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { reader = new SequenceFileRecordReader<>(); reader .initialize(new FileSplit(chunkFilePath, 0, getFileSize(chunkFilePath, configuration), null), taskAttemptContext); }
private void openForRead(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { reader = new SequenceFileRecordReader<>(); reader .initialize(new FileSplit(chunkFilePath, 0, getFileSize(chunkFilePath, configuration), null), taskAttemptContext); }
private void openForRead(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { reader = new SequenceFileRecordReader<K, V>(); reader.initialize(new FileSplit(chunkFilePath, 0, DistCpUtils.getFileSize(chunkFilePath, chunkContext.getConfiguration()), null), taskAttemptContext); }
private void openForRead(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { reader = new SequenceFileRecordReader<K, V>(); reader.initialize(new FileSplit(chunkFilePath, 0, DistCpUtils.getFileSize(chunkFilePath, chunkContext.getConfiguration()), null), taskAttemptContext); }
@Override public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { CombineFileSplit fSplit = (CombineFileSplit) split; super.initialize(new FileSplit(fSplit.getPath(index), 0, fSplit.getLength(index), fSplit.getLocations()), context); } }
private Stream<KV<Text, LongWritable>> extractResultsFromFile(String fileName) { try (SequenceFileRecordReader<Text, LongWritable> reader = new SequenceFileRecordReader<>()) { Path path = new Path(fileName); TaskAttemptContext taskContext = HadoopFormats.createTaskAttemptContext(new Configuration(), new JobID("readJob", 0), 0); reader.initialize( new FileSplit(path, 0L, Long.MAX_VALUE, new String[] {"localhost"}), taskContext); List<KV<Text, LongWritable>> result = new ArrayList<>(); while (reader.nextKeyValue()) { result.add( KV.of( new Text(reader.getCurrentKey().toString()), new LongWritable(reader.getCurrentValue().get()))); } return result.stream(); } catch (Exception e) { throw new RuntimeException(e); } }
@Override public Function<String, Stream<Pair<Text, LongWritable>>> extractOutputFunction( String outputDir, Configuration conf) { return part -> ExceptionUtils.unchecked( () -> { try (final SequenceFileRecordReader<Text, LongWritable> reader = new SequenceFileRecordReader<>()) { final Path path = new Path(outputDir + "/" + part); final TaskAttemptContext taskContext = HadoopUtils.createTaskContext(new Configuration(), HadoopUtils.getJobID(), 0); reader.initialize( new FileSplit(path, 0L, Long.MAX_VALUE, new String[] {"localhost"}), taskContext); final List<Pair<Text, LongWritable>> result = new ArrayList<>(); while (reader.nextKeyValue()) { result.add(Pair.of(reader.getCurrentKey(), reader.getCurrentValue())); } return result.stream(); } }); }