private void positionAtFirstRecord(FSDataInputStream stream) throws IOException { if (start > 0) { // Advance to the start of the first line in our slice. // We use a temporary LineReader to read a partial line and find the // start of the first one on or after our starting position. // In case our slice starts right at the beginning of a line, we need to back // up by one position and then discard the first line. start -= 1; stream.seek(start); LineReader reader = new LineReader(stream); int bytesRead = reader.readLine(buffer, (int)Math.min(MAX_LINE_LENGTH, end - start)); start = start + bytesRead; stream.seek(start); } // else // if start == 0 we're starting at the beginning of a line pos = start; }
private void positionAtFirstRecord(FSDataInputStream stream) throws IOException { if (start > 0) { stream.seek(start); } // we are now in a new chromosome/fragment, so read its name/index sequence // and reset position counter // index sequence LineReader reader = new LineReader(stream); int bytesRead = reader.readLine(buffer, (int)Math.min(MAX_LINE_LENGTH, end - start)); current_split_indexseq = buffer.toString(); // now get rid of '>' character current_split_indexseq = current_split_indexseq.substring(1,current_split_indexseq.length()); // initialize position counter current_split_pos = 1; if (logger.isDebugEnabled()) { logger.debug("read index sequence: {}", current_split_indexseq); } start = start + bytesRead; stream.seek(start); pos = start; }
private void positionAtFirstRecord(FSDataInputStream stream) throws IOException { if (start > 0) { stream.seek(start); } // we are now in a new chromosome/fragment, so read its name/index sequence // and reset position counter // index sequence LineReader reader = new LineReader(stream); int bytesRead = reader.readLine(buffer, (int)Math.min(MAX_LINE_LENGTH, end - start)); current_split_indexseq = buffer.toString(); // now get rid of '>' character current_split_indexseq = current_split_indexseq.substring(1,current_split_indexseq.length()); // initialize position counter current_split_pos = 1; logger.debug("read index sequence: {}"); start = start + bytesRead; stream.seek(start); pos = start; }
private void positionAtFirstRecord(FSDataInputStream stream) throws IOException { if (start > 0) { // Advance to the start of the first line in our slice. // We use a temporary LineReader to read a partial line and find the // start of the first one on or after our starting position. // In case our slice starts right at the beginning of a line, we need to back // up by one position and then discard the first line. start -= 1; stream.seek(start); LineReader reader = new LineReader(stream); int bytesRead = reader.readLine(buffer, (int)Math.min(MAX_LINE_LENGTH, end - start)); start = start + bytesRead; stream.seek(start); } // else // if start == 0 we're starting at the beginning of a line pos = start; }
private void positionAtFirstRecord(FSDataInputStream stream) throws IOException { if (start > 0) { // Advance to the start of the first line in our slice. // We use a temporary LineReader to read a partial line and find the // start of the first one on or after our starting position. // In case our slice starts right at the beginning of a line, we need to back // up by one position and then discard the first line. start -= 1; stream.seek(start); LineReader reader = new LineReader(stream); int bytesRead = reader.readLine(buffer, (int)Math.min(MAX_LINE_LENGTH, end - start)); start = start + bytesRead; stream.seek(start); } // else // if start == 0 we're starting at the beginning of a line pos = start; }
private void positionAtFirstRecord(FSDataInputStream stream) throws IOException { if (start > 0) { stream.seek(start); } // we are now in a new chromosome/fragment, so read its name/index sequence // and reset position counter // index sequence LineReader reader = new LineReader(stream); int bytesRead = reader.readLine(buffer, (int)Math.min(MAX_LINE_LENGTH, end - start)); current_split_indexseq = buffer.toString(); // now get rid of '>' character current_split_indexseq = current_split_indexseq.substring(1,current_split_indexseq.length()); // initialize position counter current_split_pos = 1; if (logger.isDebugEnabled()) { logger.debug("read index sequence: {}", current_split_indexseq); } start = start + bytesRead; stream.seek(start); pos = start; }
@Test public void testReadBufferedLine() throws IOException { reader = new LineReader(new ByteArrayInputStream(input22.getBytes()), 22); reader.readLine(dest); assertEquals("0123456789", dest.toString()); }
@Test public void testReadBeyondBuffer() throws IOException { reader = new LineReader(new ByteArrayInputStream(input22.getBytes()), 5); reader.readLine(dest); assertEquals("0123456789", dest.toString()); }
public FastaRecordReader(Configuration conf, FileSplit split) throws IOException { file = split.getPath(); start = split.getStart(); end = start + split.getLength(); current_split_pos = 1; FileSystem fs = file.getFileSystem(conf); FSDataInputStream fileIn = fs.open(file); CompressionCodecFactory codecFactory = new CompressionCodecFactory(conf); CompressionCodec codec = codecFactory.getCodec(file); if (codec == null) // no codec. Uncompressed file. { positionAtFirstRecord(fileIn); inputStream = fileIn; } else { // compressed file if (start != 0) throw new RuntimeException("Start position for compressed file is not 0! (found " + start + ")"); inputStream = codec.createInputStream(fileIn); end = Long.MAX_VALUE; // read until the end of the file } lineReader = new LineReader(inputStream); }
public FastqRecordReader(Configuration conf, FileSplit split) throws IOException { setConf(conf); file = split.getPath(); start = split.getStart(); end = start + split.getLength(); FileSystem fs = file.getFileSystem(conf); FSDataInputStream fileIn = fs.open(file); CompressionCodecFactory codecFactory = new CompressionCodecFactory(conf); CompressionCodec codec = codecFactory.getCodec(file); if (codec == null) { // no codec. Uncompressed file. positionAtFirstRecord(fileIn); inputStream = fileIn; } else { // compressed file if (start != 0) { throw new RuntimeException("Start position for compressed file is not 0! (found " + start + ")"); } inputStream = codec.createInputStream(fileIn); end = Long.MAX_VALUE; // read until the end of the file } lineReader = new LineReader(inputStream); }
public QseqRecordReader(Configuration conf, FileSplit split) throws IOException { setConf(conf); file = split.getPath(); start = split.getStart(); end = start + split.getLength(); FileSystem fs = file.getFileSystem(conf); FSDataInputStream fileIn = fs.open(file); CompressionCodecFactory codecFactory = new CompressionCodecFactory(conf); CompressionCodec codec = codecFactory.getCodec(file); if (codec == null) // no codec. Uncompressed file. { positionAtFirstRecord(fileIn); inputStream = fileIn; } else { // compressed file if (start != 0) throw new RuntimeException("Start position for compressed file is not 0! (found " + start + ")"); inputStream = codec.createInputStream(fileIn); end = Long.MAX_VALUE; // read until the end of the file } lineReader = new LineReader(inputStream); }
public FastqRecordReader(Configuration conf, FileSplit split) throws IOException { setConf(conf); file = split.getPath(); start = split.getStart(); end = start + split.getLength(); FileSystem fs = file.getFileSystem(conf); FSDataInputStream fileIn = fs.open(file); CompressionCodecFactory codecFactory = new CompressionCodecFactory(conf); CompressionCodec codec = codecFactory.getCodec(file); if (codec == null) // no codec. Uncompressed file. { positionAtFirstRecord(fileIn); inputStream = fileIn; } else { // compressed file if (start != 0) throw new RuntimeException("Start position for compressed file is not 0! (found " + start + ")"); inputStream = codec.createInputStream(fileIn); end = Long.MAX_VALUE; // read until the end of the file } lineReader = new LineReader(inputStream); }
public FastaRecordReader(Configuration conf, FileSplit split) throws IOException { setConf(conf); file = split.getPath(); start = split.getStart(); end = start + split.getLength(); current_split_pos = 1; FileSystem fs = file.getFileSystem(conf); FSDataInputStream fileIn = fs.open(file); CompressionCodecFactory codecFactory = new CompressionCodecFactory(conf); CompressionCodec codec = codecFactory.getCodec(file); if (codec == null) // no codec. Uncompressed file. { positionAtFirstRecord(fileIn); inputStream = fileIn; } else { // compressed file if (start != 0) throw new RuntimeException("Start position for compressed file is not 0! (found " + start + ")"); inputStream = codec.createInputStream(fileIn); end = Long.MAX_VALUE; // read until the end of the file } lineReader = new LineReader(inputStream); }
public FastqRecordReader(Configuration conf, FileSplit split) throws IOException { setConf(conf); file = split.getPath(); start = split.getStart(); end = start + split.getLength(); FileSystem fs = file.getFileSystem(conf); FSDataInputStream fileIn = fs.open(file); CompressionCodecFactory codecFactory = new CompressionCodecFactory(conf); CompressionCodec codec = codecFactory.getCodec(file); if (codec == null) // no codec. Uncompressed file. { positionAtFirstRecord(fileIn); inputStream = fileIn; } else { // compressed file if (start != 0) throw new RuntimeException("Start position for compressed file is not 0! (found " + start + ")"); inputStream = codec.createInputStream(fileIn); end = Long.MAX_VALUE; // read until the end of the file } lineReader = new LineReader(inputStream); }
@Test public void testSkipOnBufferedLine() throws IOException { reader = new LineReader(new ByteArrayInputStream(input22.getBytes()), 22); long skipped = reader.skip(1); assertEquals(1, skipped); reader.readLine(dest); assertEquals("123456789", dest.toString()); }
public QseqRecordReader(Configuration conf, FileSplit split) throws IOException { setConf(conf); file = split.getPath(); start = split.getStart(); end = start + split.getLength(); FileSystem fs = file.getFileSystem(conf); FSDataInputStream fileIn = fs.open(file); CompressionCodecFactory codecFactory = new CompressionCodecFactory(conf); CompressionCodec codec = codecFactory.getCodec(file); if (codec == null) // no codec. Uncompressed file. { positionAtFirstRecord(fileIn); inputStream = fileIn; } else { // compressed file if (start != 0) throw new RuntimeException("Start position for compressed file is not 0! (found " + start + ")"); inputStream = codec.createInputStream(fileIn); end = Long.MAX_VALUE; // read until the end of the file } lineReader = new LineReader(inputStream); }
public FastaRecordReader(Configuration conf, FileSplit split) throws IOException { setConf(conf); file = split.getPath(); start = split.getStart(); end = start + split.getLength(); current_split_pos = 1; FileSystem fs = file.getFileSystem(conf); FSDataInputStream fileIn = fs.open(file); CompressionCodecFactory codecFactory = new CompressionCodecFactory(conf); CompressionCodec codec = codecFactory.getCodec(file); if (codec == null) // no codec. Uncompressed file. { positionAtFirstRecord(fileIn); inputStream = fileIn; } else { // compressed file if (start != 0) throw new RuntimeException("Start position for compressed file is not 0! (found " + start + ")"); inputStream = codec.createInputStream(fileIn); end = Long.MAX_VALUE; // read until the end of the file } lineReader = new LineReader(inputStream); }
public QseqRecordReader(Configuration conf, FileSplit split) throws IOException { setConf(conf); file = split.getPath(); start = split.getStart(); end = start + split.getLength(); FileSystem fs = file.getFileSystem(conf); FSDataInputStream fileIn = fs.open(file); CompressionCodecFactory codecFactory = new CompressionCodecFactory(conf); CompressionCodec codec = codecFactory.getCodec(file); if (codec == null) // no codec. Uncompressed file. { positionAtFirstRecord(fileIn); inputStream = fileIn; } else { // compressed file if (start != 0) throw new RuntimeException("Start position for compressed file is not 0! (found " + start + ")"); inputStream = codec.createInputStream(fileIn); end = Long.MAX_VALUE; // read until the end of the file } lineReader = new LineReader(inputStream); }
@Test public void testSkipBeyondBuffer() throws IOException { reader = new LineReader(new ByteArrayInputStream(input22.getBytes()), 5); long skipped = reader.skip(11); assertEquals(11, skipped); reader.readLine(dest); assertEquals("0987654321", dest.toString()); }
@Test public void testSkipBeyondInput() throws IOException { reader = new LineReader(new ByteArrayInputStream(input10.getBytes()), 5); long skipped = reader.skip(11); assertEquals(10, skipped); skipped = reader.skip(11); assertEquals(0, skipped); }