/** Wraps a file given by a {@link File} object into an iterator. * * @param file a file. */ public static ByteIterator asByteIterator(final File file) throws IOException { return new ByteDataInputWrapper(new DataInputStream(new FastBufferedInputStream(new FileInputStream(file)))); } /** Wraps a file given by a pathname into an iterator.
public Document nextDocument() throws IOException { if ( index == size ) return null; if ( index == firstDocument[ f + 1 ] ) { fbis.close(); fbis = new FastBufferedInputStream( new FileInputStream( file[ ++f ] ) ); } readDocument( index, f, fbis ); return document( index++ ); } };
f = Arrays.binarySearch( firstDocument, index ); if ( f < 0 ) f = -f - 2; fbis = new FastBufferedInputStream( new FileInputStream( file[ f ] ) ); fbis.position( start ); final long end = pointers.get( f ).getLong( index - firstDocument[ f ] + 1 ); Arrays.fill( bufferSize, 0 ); String title; while( fbis.position() < end ) { l = readLine( fbis ); if ( startsWith( lineBuffer, META_MARKER ) ) { if ( openStream ) fbis.close();
FastBufferedInputStream fbis = new FastBufferedInputStream( is, bufferSize ); while ( ( l = fbis.readLine( buffer ) ) != -1 ) { if ( l == buffer.length ) { while ( ( l = fbis.readLine( buffer ) ) == buffer.length ); currInter = fbis.position(); pastHeader = true; oldPos = fbis.position(); fbis.close();
public Document getDocument( InputStream rawContent, Reference2ObjectMap<Enum<?>,Object> metadata ) throws IOException { final FastBufferedInputStream fbis = new FastBufferedInputStream( rawContent ); int startedHeader = 0; // 0 == false, 1 == true, 2 === header started and uri collected boolean foundDocNo = false; int l = fbis.readLine( buffer ); if ( l < 0 ) throw new EOFException(); if ( ! TRECDocumentCollection.equals( buffer, l, DOC_OPEN ) ) throw new IllegalStateException ( "Document does not start with <DOC>: " + new String( buffer, 0, l ) ); while ( ( l = fbis.readLine( buffer ) ) != -1 ) { if ( !foundDocNo && startsWith( buffer, l, DOCNO_OPEN ) ) { foundDocNo = true; metadata.put( MetadataKeys.TITLE, new String( buffer, DOCNO_OPEN.length, l - ( DOCNO_OPEN.length + DOCNO_CLOSE.length ) ) ); } switch ( startedHeader ) { case 0: if ( TRECDocumentCollection.equals( buffer, l, DOCHDR_OPEN ) ) startedHeader = 1; break; case 1: startedHeader = 2; metadata.put( MetadataKeys.URI, new String( buffer, 0, l ) ); break; case 2: } if ( TRECDocumentCollection.equals( buffer, l, DOCHDR_CLOSE ) ) break; } return new TRECHeaderDocument( metadata ); }
int result = remotedInputStream.read( readBuf, 0, len ); outputStream.writeInt( result ); if ( result > 0 ) outputStream.write( readBuf, 0, result ); outputStream.writeInt( remotedInputStream.read() ); outputStream.flush(); break; remotedInputStream.position( inputStream.readLong() ); break; outputStream.writeLong( remotedInputStream.position() ); outputStream.flush(); break; outputStream.writeLong( remotedInputStream.available() ); outputStream.flush(); break; outputStream.writeLong( remotedInputStream.skip( inputStream.readLong() ) ); outputStream.flush(); break; remotedInputStream.close();
this.filename = filename.toString(); inputStream = new FastBufferedInputStream(new FileInputStream(this.filename), bufferSize); decoder = (charset = Charset.forName(encoding)).newDecoder(); byte[] array = new byte[16]; while((len = inputStream.readLine(array, start, array.length - start, terminators)) == array.length - start) { start += len; array = ByteArrays.grow(array, array.length + 1); charBuffer = CharBuffer.wrap(new char[array.length]); inputStream.position(0); borders = new EliasFanoMonotoneLongBigList(count, inputStream.length(), new LongIterator() { long pos = 0; byte[] buffer = byteBuffer.array();
@Override public synchronized void close() { if (fbis == null) return; try { fbis.close(); } catch (final IOException e) { throw new RuntimeException(e); } finally { fbis = null; } }
/** Reads a line into the given byte array. * * @param array byte array where the next line will be stored. * @param terminators a set containing the line termination sequences that we want * to consider as valid. * @return the number of bytes actually placed in {@code array}, or -1 at end of file. * @see #readLine(byte[], int, int, EnumSet) */ public int readLine(final byte[] array, final EnumSet<LineTerminator> terminators) throws IOException { return readLine(array, 0, array.length, terminators); }
@Override public void reset() { index = 0; try { inputStream.position(0); } catch (IOException e) { LOG.error("Unable to reset iterator to position 0"); } }
public int read() throws IOException { if ( eof ) return -1; final int nextByte = super.read(); if ( nextByte == separator || nextByte == -1 ) { eof = true; return -1; } return nextByte; }
/** Resets the internal logic of this fast buffered input stream. * * @deprecated As of {@code fastutil} 5.0.4, replaced by {@link #flush()}. The old * semantics of this method does not contradict {@link InputStream}'s contract, as * the semantics of {@link #reset()} is undefined if {@link InputStream#markSupported()} * returns false. On the other hand, the name was really a poor choice. */ @Override @Deprecated public void reset() { flush(); } }
final FastBufferedInputStream fbis = gzipped ? new FastBufferedInputStream( new GZIPInputStream( new FileInputStream( f ) ) ) : new FastBufferedInputStream( new FileInputStream( f ) ); long position; for(;;) { position = fbis.position(); if ( readLine( fbis ) == -1 ) break; if ( startsWith( lineBuffer, DOC_MARKER ) ) p.add( position ); p.add( fbis.position() ); fbis.close();
FastBufferedInputStream fbis = new FastBufferedInputStream( is, bufferSize ); while ( ( l = fbis.readLine( buffer ) ) != -1 ) { if ( l == buffer.length ) { while ( ( l = fbis.readLine( buffer ) ) == buffer.length ); currInter = fbis.position(); pastHeader = true; oldPos = fbis.position(); fbis.close();
public Document getDocument( InputStream rawContent, Reference2ObjectMap<Enum<?>,Object> metadata ) throws IOException { final FastBufferedInputStream fbis = new FastBufferedInputStream( rawContent ); int startedHeader = 0; // 0 == false, 1 == true, 2 === header started and uri collected boolean foundDocNo = false; int l = fbis.readLine( buffer ); if ( l < 0 ) throw new EOFException(); if ( ! TRECDocumentCollection.equals( buffer, l, DOC_OPEN ) ) throw new IllegalStateException ( "Document does not start with <DOC>: " + new String( buffer, 0, l ) ); while ( ( l = fbis.readLine( buffer ) ) != -1 ) { if ( !foundDocNo && startsWith( buffer, l, DOCNO_OPEN ) ) { foundDocNo = true; metadata.put( MetadataKeys.TITLE, new String( buffer, DOCNO_OPEN.length, l - ( DOCNO_OPEN.length + DOCNO_CLOSE.length ) ) ); } switch ( startedHeader ) { case 0: if ( TRECDocumentCollection.equals( buffer, l, DOCHDR_OPEN ) ) startedHeader = 1; break; case 1: startedHeader = 2; metadata.put( MetadataKeys.URI, new String( buffer, 0, l ) ); break; case 2: } if ( TRECDocumentCollection.equals( buffer, l, DOCHDR_CLOSE ) ) break; } return new TRECHeaderDocument( metadata ); }
int result = remotedInputStream.read( readBuf, 0, len ); outputStream.writeInt( result ); if ( result > 0 ) outputStream.write( readBuf, 0, result ); outputStream.writeInt( remotedInputStream.read() ); outputStream.flush(); break; remotedInputStream.position( inputStream.readLong() ); break; outputStream.writeLong( remotedInputStream.position() ); outputStream.flush(); break; outputStream.writeLong( remotedInputStream.available() ); outputStream.flush(); break; outputStream.writeLong( remotedInputStream.skip( inputStream.readLong() ) ); outputStream.flush(); break; remotedInputStream.close();
this.filename = filename.toString(); inputStream = new FastBufferedInputStream(new FileInputStream(this.filename), bufferSize); decoder = (charset = Charset.forName(encoding)).newDecoder(); byte[] array = new byte[16]; while((len = inputStream.readLine(array, start, array.length - start, terminators)) == array.length - start) { start += len; array = ByteArrays.grow(array, array.length + 1); charBuffer = CharBuffer.wrap(new char[array.length]); inputStream.position(0); borders = new EliasFanoMonotoneLongBigList(count, inputStream.length(), new LongIterator() { long pos = 0; byte[] buffer = byteBuffer.array();