public SimpleFSIndexInput(String resourceDesc, SeekableByteChannel channel, IOContext context) throws IOException { super(resourceDesc, context); this.channel = channel; this.off = 0L; this.end = channel.size(); }
@Override public byte[] read() throws IOException { try (SeekableByteChannel channel = Files.newByteChannel(path, options)) { return ByteStreams.toByteArray(Channels.newInputStream(channel), channel.size()); } }
@Override public byte[] read() throws IOException { try (SeekableByteChannel channel = Files.newByteChannel(path, options)) { return ByteStreams.toByteArray(Channels.newInputStream(channel), channel.size()); } }
@Override public byte[] read() throws IOException { try (SeekableByteChannel channel = Files.newByteChannel(path, options)) { return ByteStreams.toByteArray(Channels.newInputStream(channel), channel.size()); } }
/** * Skips the given number of bytes or throws an EOFException if * skipping failed. */ private void skipBytes(final int count) throws IOException { long currentPosition = archive.position(); long newPosition = currentPosition + count; if (newPosition > archive.size()) { throw new EOFException(); } archive.position(newPosition); }
@Override public byte[] read() throws IOException { try (SeekableByteChannel channel = Files.newByteChannel(path, options)) { return com.facebook.presto.jdbc.internal.guava.io.Files.readFile( Channels.newInputStream(channel), channel.size()); } }
@Override public WorkUnit call() throws IOException { long pos = ((long) blockSize) * blockIndex; if (pos > chan.size()) { return this; } chan.position(pos); // read until buffer is full, or EOF while (chan.read(buf) > 0) {} ; return this; }
throw new IllegalArgumentException("bufSize must be positive"); this.size = chan.size(); if (bufSize > this.size) { this.bufSize = (int) this.size;
@Override public ByteBuffer call() throws IOException { long pos = ((long) blockSize) * blockIndex; if (pos > chan.size()) { return null; } if (pos < 0) { // This should never happen, if the code's correct. throw new IllegalArgumentException( "blockIndex " + blockIndex + " has position " + pos + ": negative position is not valid."); } chan.position(pos); // read until buffer is full, or EOF while (chan.read(buf) >= 0 && buf.hasRemaining()) {} return buf; }
@Test public void testEOF() throws Exception { SeekableByteChannel chan1 = Files.newByteChannel(input); SeekableByteChannel chan2 = SeekableByteChannelPrefetcher.addPrefetcher(1, Files.newByteChannel(input)); // read the final 128 bytes, exactly. testSeeking(chan1, chan2, (int) chan1.size() - 128); // read truncated because we're asking for beyond EOF testSeeking(chan1, chan2, (int) chan1.size() - 64); // read starting past EOF testSeeking(chan1, chan2, (int) chan1.size() + 128); // read more than a whole block past EOF testSeeking(chan1, chan2, (int) chan1.size() + 1024 * 2); }
@Test public void testNewByteChannelWrite() throws Exception { Path path = Paths.get(URI.create("gs://bucket/tests")); try (SeekableByteChannel output = Files.newByteChannel(path, WRITE)) { assertThat(output.position()).isEqualTo(0); assertThat(output.size()).isEqualTo(0); ByteBuffer buffer = ByteBuffer.wrap("filec".getBytes(UTF_8)); assertThat(output.write(buffer)).isEqualTo(5); assertThat(output.position()).isEqualTo(5); assertThat(output.size()).isEqualTo(5); buffer = ByteBuffer.wrap("onten".getBytes(UTF_8)); assertThat(output.write(buffer)).isEqualTo(5); assertThat(output.position()).isEqualTo(10); assertThat(output.size()).isEqualTo(10); } assertThat(new String(Files.readAllBytes(path), UTF_8)).isEqualTo("fileconten"); }
@Test public void testNewByteChannelRead_seekBeyondSize_reportsEofOnNextRead() throws Exception { Path path = Paths.get(URI.create("gs://lol/cat")); Files.write(path, "hellocat".getBytes(UTF_8)); try (SeekableByteChannel input = Files.newByteChannel(path)) { ByteBuffer buffer = ByteBuffer.allocate(5); input.position(10); assertThat(input.read(buffer)).isEqualTo(-1); input.position(11); assertThat(input.read(buffer)).isEqualTo(-1); assertThat(input.size()).isEqualTo(8); } }
@Test public void testCreateAndWrite() throws IOException { CloudStorageFileSystem testBucket = getTestBucket(); Path path = testBucket.getPath(PREFIX + randomSuffix()); // file shouldn't exist initially (see above). assertThat(Files.exists(path)).isFalse(); try { Files.createFile(path); Files.write(path, FILE_CONTENTS, UTF_8); // now it does. assertThat(Files.exists(path)).isTrue(); // let's check that the contents is OK. ByteArrayOutputStream wantBytes = new ByteArrayOutputStream(); PrintWriter writer = new PrintWriter(new OutputStreamWriter(wantBytes, UTF_8)); for (String content : FILE_CONTENTS) { writer.println(content); } writer.close(); SeekableByteChannel chan = Files.newByteChannel(path, StandardOpenOption.READ); byte[] gotBytes = new byte[(int) chan.size()]; readFully(chan, gotBytes); assertThat(gotBytes).isEqualTo(wantBytes.toByteArray()); } finally { // let's not leave files around Files.deleteIfExists(path); } }
@Test public void testWrite() throws IOException { CloudStorageFileSystem testBucket = getTestBucket(); Path path = testBucket.getPath(PREFIX + randomSuffix()); // file shouldn't exist initially. If it does it's either because it's a leftover // from a previous run (so we should delete the file) // or because we're misconfigured and pointing to an actually important file // (so we should absolutely not delete it). // So if the file's here, don't try to fix it automatically, let the user deal with it. assertThat(Files.exists(path)).isFalse(); try { Files.write(path, FILE_CONTENTS, UTF_8); // now it does. assertThat(Files.exists(path)).isTrue(); // let's check that the contents is OK. ByteArrayOutputStream wantBytes = new ByteArrayOutputStream(); PrintWriter writer = new PrintWriter(new OutputStreamWriter(wantBytes, UTF_8)); for (String content : FILE_CONTENTS) { writer.println(content); } writer.close(); SeekableByteChannel chan = Files.newByteChannel(path, StandardOpenOption.READ); byte[] gotBytes = new byte[(int) chan.size()]; readFully(chan, gotBytes); assertThat(gotBytes).isEqualTo(wantBytes.toByteArray()); } finally { // let's not leave files around Files.deleteIfExists(path); } }
@Test(timeout = 60_000) public void testReadByteChannel() throws IOException { CloudStorageFileSystem testBucket = getTestBucket(); Path path = testBucket.getPath(SML_FILE); long size = Files.size(path); SeekableByteChannel chan = Files.newByteChannel(path, StandardOpenOption.READ); assertThat(chan.size()).isEqualTo(size); ByteBuffer buf = ByteBuffer.allocate(SML_SIZE); int read = 0; while (chan.isOpen()) { int rc = chan.read(buf); assertThat(chan.size()).isEqualTo(size); if (rc < 0) { // EOF break; } assertThat(rc).isGreaterThan(0); read += rc; assertThat(chan.position()).isEqualTo(read); } assertThat(read).isEqualTo(size); byte[] expected = new byte[SML_SIZE]; new Random(SML_SIZE).nextBytes(expected); assertThat(Arrays.equals(buf.array(), expected)).isTrue(); }
@Test public void testPartialBuffers() throws Exception { SeekableByteChannel chan1 = Files.newByteChannel(input); SeekableByteChannel chan2 = SeekableByteChannelPrefetcher.addPrefetcher(1, Files.newByteChannel(input)); // get a partial buffer testSeeking(chan1, chan2, (int) chan1.size() - 127); // make sure normal reads can use the full buffer for (int i = 0; i < 2; i++) { testSeeking(chan1, chan2, i * 1024); } // get a partial buffer, replacing one of the full ones testSeeking(chan1, chan2, (int) chan1.size() - 127); // make sure the buffers are still OK for (int i = 0; i < 2; i++) { testSeeking(chan1, chan2, i * 1024); } }
@Test public void testSeek() throws IOException { CloudStorageFileSystem testBucket = getTestBucket(); Path path = testBucket.getPath(BIG_FILE); int size = BIG_SIZE; byte[] contents = randomContents(size); byte[] sample = new byte[100]; byte[] wanted; byte[] wanted2; SeekableByteChannel chan = Files.newByteChannel(path, StandardOpenOption.READ); assertThat(chan.size()).isEqualTo(size); // check seek int dest = size / 2; chan.position(dest); readFully(chan, sample); wanted = Arrays.copyOfRange(contents, dest, dest + 100); assertThat(wanted).isEqualTo(sample); // now go back and check the beginning // (we do 2 locations because 0 is sometimes a special case). chan.position(0); readFully(chan, sample); wanted2 = Arrays.copyOf(contents, 100); assertThat(wanted2).isEqualTo(sample); // if the two spots in the file have the same contents, then this isn't a good file for this // test. assertThat(wanted).isNotEqualTo(wanted2); }
@Test public void testSeek() throws Exception { SeekableByteChannel chan1 = Files.newByteChannel(input); SeekableByteChannel chan2 = SeekableByteChannelPrefetcher.addPrefetcher(1, Files.newByteChannel(input)); testSeeking(chan1, chan2, 1024); testSeeking(chan1, chan2, 1500); testSeeking(chan1, chan2, 128); testSeeking(chan1, chan2, 256); testSeeking(chan1, chan2, 128); // yes, testReading - let's make sure that reading more than one block still works // even after a seek. testReading(chan1, chan2, 1500); testSeeking(chan1, chan2, 2048); testSeeking(chan1, chan2, 0); testSeeking(chan1, chan2, 3000); testSeeking(chan1, chan2, 6000); testSeeking(chan1, chan2, (int) chan1.size() - 127); testSeeking(chan1, chan2, (int) chan1.size() - 128); testSeeking(chan1, chan2, (int) chan1.size() - 129); }
@Override public int available() throws IOException { if (channel instanceof SeekableByteChannel) { SeekableByteChannel sbc = (SeekableByteChannel) channel; long remainder = Math.max(0, sbc.size() - sbc.position()); return (remainder > Integer.MAX_VALUE) ? Integer.MAX_VALUE : (int)(remainder + buffer.remaining()); } return buffer.remaining(); } }