/** * Returns a split for each store files directory using the block location * of each file as locality reference. */ @Override public List<InputSplit> getSplits(JobContext job) throws IOException { List<InputSplit> splits = new ArrayList<>(); List<FileStatus> files = listStatus(job); Text key = new Text(); for (FileStatus file: files) { Path path = file.getPath(); FileSystem fs = path.getFileSystem(job.getConfiguration()); LineReader reader = new LineReader(fs.open(path)); long pos = 0; int n; try { while ((n = reader.readLine(key)) > 0) { String[] hosts = getStoreDirHosts(fs, path); splits.add(new FileSplit(path, pos, n, hosts)); pos += n; } } finally { reader.close(); } } return splits; }
FileStatus masterStat = fs.getFileStatus(masterIndexPath); masterIndexTimestamp = masterStat.getModificationTime(); lin = new LineReader(in, getConf()); read = lin.readLine(line); read = 0; aIn.seek(s.begin); aLin = new LineReader(aIn, getConf()); while (read + s.begin < s.end) { int tmp = aLin.readLine(line);
selectObjectContent = selectClient.getRecordsContent(selectObjectContentRequest); closer.register(selectObjectContent); reader = new LineReader(selectObjectContent, lineDelimiter.getBytes(StandardCharsets.UTF_8)); closer.register(reader); isFirstLine = false;
FileSystem fs = path.getFileSystem(job.getConfiguration()); FSDataInputStream fileIn = fs.open(path); LineReader in = new LineReader(fileIn, job.getConfiguration()); int lineLen = 0; while(true) {
public CustomRecordReader(CombineFileSplit split, TaskAttemptContext context, Integer index) throws IOException{ path = split.getPath(index); fs = path.getFileSystem(context.getConfiguration()); startOffset = split.getOffset(index); endOffset = startOffset + split.getLength(index); fileIn = fs.open(path); reader = new LineReader(fileIn); pos = startOffset; }
value.set(text.toString()); fileIn.seek(pos); in = new LineReader(fileIn, job); return true;
try { FSDataInputStream in = fs.open(fileName); lr = new LineReader(in, conf); Text line = new Text(); long begin = 0;
boolean skipFirstLine = false; if (codec != null) { in = new LineReader(codec.createInputStream(fileIn), job); end = Long.MAX_VALUE; } else { fileIn.seek(start); in = new LineReader(fileIn, job);
@Override protected void createInputReader(InputStream input, Configuration conf) throws IOException { maxLineLen = conf.getInt(MAX_LINE_LEN_CONF, Integer.MAX_VALUE); in_ = new LineReader(input, conf); }
@Override protected void createInputReader(InputStream input, Configuration conf) throws IOException { maxLineLen = conf.getInt(LzoLineRecordReader.MAX_LINE_LEN_CONF, Integer.MAX_VALUE); lineReader_ = new LineReader(input, conf); }
private static LineReader makeStream(String str, int bufsz) throws IOException { return new LineReader(new ByteArrayInputStream (str.getBytes("UTF-8")), bufsz); }
private LineReader makeStream(String str) throws IOException { return new LineReader(new ByteArrayInputStream (str.getBytes("UTF-8")), defaultConf); }
private LineReader makeStream(String str) throws IOException { return new LineReader(new ByteArrayInputStream (str.getBytes("UTF-8")), defaultConf); }
@Override protected void createInputReader(InputStream input, Configuration conf) throws IOException { in_ = new LineReader(input, conf); String fileURI = getFieldDefinitionFile(); Path path = new Path(fileURI); InputStream is = path.getFileSystem(conf).open(path); w3cLogParser_ = new W3CLogParser(is); is.close(); }
@Override public void initialize(PipeMapRed pipeMapRed) throws IOException { super.initialize(pipeMapRed); clientIn = pipeMapRed.getClientInput(); conf = pipeMapRed.getConfiguration(); lineReader = new LineReader((InputStream)clientIn, conf); key = new Text(); line = new Text(); }
private static LineReader makeStream(String str, int bufsz) throws IOException { return new LineReader(new ByteArrayInputStream (str.getBytes("UTF-8")), bufsz); }
private LineReader makeStream(String str) throws IOException { return new LineReader(new ByteArrayInputStream (str.getBytes("UTF-8")), defaultConf); }
private LineReader makeStream(String str) throws IOException { return new LineReader(new ByteArrayInputStream (str.getBytes("UTF-8")), defaultConf); }
@Override public void initialize(PipeMapRed pipeMapRed) throws IOException { super.initialize(pipeMapRed); clientIn = pipeMapRed.getClientInput(); conf = pipeMapRed.getConfiguration(); numKeyFields = pipeMapRed.getNumOfKeyFields(); separator = pipeMapRed.getFieldSeparator(); lineReader = new LineReader((InputStream)clientIn, conf); key = new Text(); value = new Text(); line = new Text(); }
@Override protected LineReader createReader(InputStream inputStream) throws IOException { LineReader lineReader = new LineReader(inputStream, delimiter); if (getContext().getStart() > 0) { processReadCount(lineReader.readLine(new Text())); } return lineReader; }