/** Example of reading the blob's content through a reader. */ // [TARGET reader(BlobSourceOption...)] public void reader() throws IOException { // [START reader] try (ReadChannel reader = blob.reader()) { ByteBuffer bytes = ByteBuffer.allocate(64 * 1024); while (reader.read(bytes) > 0) { bytes.flip(); // do something with bytes bytes.clear(); } } // [END reader] }
/** Example of reading a blob's content through a reader. */ // [TARGET reader(String, String, BlobSourceOption...)] // [VARIABLE "my_unique_bucket"] // [VARIABLE "my_blob_name"] public void readerFromStrings(String bucketName, String blobName) throws IOException { // [START readerFromStrings] try (ReadChannel reader = storage.reader(bucketName, blobName)) { ByteBuffer bytes = ByteBuffer.allocate(64 * 1024); while (reader.read(bytes) > 0) { bytes.flip(); // do something with bytes bytes.clear(); } } // [END readerFromStrings] }
/** Example of reading just a portion of the blob's content. */ // [TARGET reader(BlobSourceOption...)] // [VARIABLE 1] // [VARIABLE 8] public byte[] readContentRange(int start, int end) throws IOException { // [START readContentRange] try (ReadChannel reader = blob.reader()) { reader.seek(start); ByteBuffer bytes = ByteBuffer.allocate(end - start); reader.read(bytes); return bytes.array(); } // [END readContentRange] }
/** Example of reading a blob's content through a reader. */ // [TARGET reader(BlobId, BlobSourceOption...)] // [VARIABLE "my_unique_bucket"] // [VARIABLE "my_blob_name"] public void readerFromId(String bucketName, String blobName) throws IOException { // [START readerFromId] BlobId blobId = BlobId.of(bucketName, blobName); try (ReadChannel reader = storage.reader(blobId)) { ByteBuffer bytes = ByteBuffer.allocate(64 * 1024); while (reader.read(bytes) > 0) { bytes.flip(); // do something with bytes bytes.clear(); } } // [END readerFromId] }
/** * Downloads this blob to the given file path using specified blob read options. * * @param path destination * @param options blob read options * @throws StorageException upon failure */ public void downloadTo(Path path, BlobSourceOption... options) { try (OutputStream outputStream = Files.newOutputStream(path); ReadChannel reader = reader(options)) { WritableByteChannel channel = Channels.newChannel(outputStream); ByteBuffer bytes = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE); while (reader.read(bytes) > 0) { bytes.flip(); channel.write(bytes); bytes.clear(); } } catch (IOException e) { throw new StorageException(e); } }
@Override public int read(ByteBuffer dst) throws IOException { synchronized (this) { checkOpen(); int amt; final CloudStorageRetryHandler retryHandler = new CloudStorageRetryHandler(maxRetries, maxChannelReopens, config); dst.mark(); while (true) { try { dst.reset(); amt = channel.read(dst); break; } catch (StorageException exs) { // Will rethrow a StorageException if all retries/reopens are exhausted handleStorageException(exs, retryHandler); } } if (amt > 0) { position += amt; // This can only happen if the file changed under us and we didn't notice. if (position > size) { size = position; } } return amt; } }
@Test public void testRead() throws IOException { ByteBuffer buffer = ByteBuffer.allocate(1); when(gcsChannel.read(eq(buffer))).thenReturn(1); assertThat(chan.position()).isEqualTo(0L); assertThat(chan.read(buffer)).isEqualTo(1); assertThat(chan.position()).isEqualTo(1L); verify(gcsChannel).read(any(ByteBuffer.class)); verify(gcsChannel, times(3)).isOpen(); }
@Test public void testReadRetry() throws IOException { ByteBuffer buffer = ByteBuffer.allocate(1); when(gcsChannel.read(eq(buffer))) .thenThrow( new StorageException( new IOException( "outer", new IOException( "Connection closed prematurely: bytesRead = 33554432, Content-Length = 41943040")))) .thenReturn(1); assertThat(chan.position()).isEqualTo(0L); assertThat(chan.read(buffer)).isEqualTo(1); assertThat(chan.position()).isEqualTo(1L); verify(gcsChannel, times(2)).read(any(ByteBuffer.class)); }
@Test public void testReadRetryEventuallyGivesUp() throws IOException { ByteBuffer buffer = ByteBuffer.allocate(1); when(gcsChannel.read(eq(buffer))) .thenThrow( new StorageException( new IOException( "Connection closed prematurely: bytesRead = 33554432, Content-Length = 41943040"))) .thenThrow( new StorageException( new IOException( "Connection closed prematurely: bytesRead = 33554432, Content-Length = 41943040"))) .thenReturn(1); assertThat(chan.position()).isEqualTo(0L); thrown.expect(StorageException.class); chan.read(buffer); }
@Test public void testReadRetrySSLHandshake() throws IOException { ByteBuffer buffer = ByteBuffer.allocate(1); when(gcsChannel.read(eq(buffer))) .thenThrow( new StorageException( new IOException( "something", new IOException( "thing", new SSLHandshakeException("connection closed due to throttling"))))) .thenReturn(1); assertThat(chan.position()).isEqualTo(0L); assertThat(chan.read(buffer)).isEqualTo(1); assertThat(chan.position()).isEqualTo(1L); verify(gcsChannel, times(2)).read(any(ByteBuffer.class)); }
@Test public void testReadAndWriteChannels() throws IOException { String blobName = "test-read-and-write-channels-blob"; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); byte[] stringBytes; try (WriteChannel writer = storage.writer(blob)) { stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8); writer.write(ByteBuffer.wrap(BLOB_BYTE_CONTENT)); writer.write(ByteBuffer.wrap(stringBytes)); } ByteBuffer readBytes; ByteBuffer readStringBytes; try (ReadChannel reader = storage.reader(blob.getBlobId())) { readBytes = ByteBuffer.allocate(BLOB_BYTE_CONTENT.length); readStringBytes = ByteBuffer.allocate(stringBytes.length); reader.read(readBytes); reader.read(readStringBytes); } assertArrayEquals(BLOB_BYTE_CONTENT, readBytes.array()); assertEquals(BLOB_STRING_CONTENT, new String(readStringBytes.array(), UTF_8)); }
@Test public void testReadAndWriteChannelsWithDifferentFileSize() throws IOException { String blobNamePrefix = "test-read-and-write-channels-blob-"; int[] blobSizes = {0, 700, 1024 * 256, 2 * 1024 * 1024, 4 * 1024 * 1024, 4 * 1024 * 1024 + 1}; Random rnd = new Random(); for (int blobSize : blobSizes) { String blobName = blobNamePrefix + blobSize; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); byte[] bytes = new byte[blobSize]; rnd.nextBytes(bytes); try (WriteChannel writer = storage.writer(blob)) { writer.write(ByteBuffer.wrap(bytes)); } ByteArrayOutputStream output = new ByteArrayOutputStream(); try (ReadChannel reader = storage.reader(blob.getBlobId())) { ByteBuffer buffer = ByteBuffer.allocate(64 * 1024); while (reader.read(buffer) > 0) { buffer.flip(); output.write(buffer.array(), 0, buffer.limit()); buffer.clear(); } } assertArrayEquals(bytes, output.toByteArray()); assertTrue(storage.delete(BUCKET, blobName)); } }
@Test public void testReadCompressedBlob() throws IOException { String blobName = "test-read-compressed-blob"; BlobInfo blobInfo = BlobInfo.newBuilder(BlobId.of(BUCKET, blobName)) .setContentType("text/plain") .setContentEncoding("gzip") .build(); Blob blob = storage.create(blobInfo, COMPRESSED_CONTENT); try (ByteArrayOutputStream output = new ByteArrayOutputStream()) { try (ReadChannel reader = storage.reader(BlobId.of(BUCKET, blobName))) { reader.setChunkSize(8); ByteBuffer buffer = ByteBuffer.allocate(8); while (reader.read(buffer) != -1) { buffer.flip(); output.write(buffer.array(), 0, buffer.limit()); buffer.clear(); } } assertArrayEquals( BLOB_STRING_CONTENT.getBytes(UTF_8), storage.readAllBytes(BUCKET, blobName)); assertArrayEquals(COMPRESSED_CONTENT, output.toByteArray()); try (GZIPInputStream zipInput = new GZIPInputStream(new ByteArrayInputStream(output.toByteArray()))) { assertArrayEquals(BLOB_STRING_CONTENT.getBytes(UTF_8), ByteStreams.toByteArray(zipInput)); } } }
@Test public void testReaderWithOptions() throws IOException { byte[] result = new byte[DEFAULT_CHUNK_SIZE]; EasyMock.expect( storageRpcMock.read(BLOB_INFO2.toPb(), BLOB_SOURCE_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) .andReturn(Tuple.of("etag", result)); EasyMock.replay(storageRpcMock); initializeService(); ReadChannel channel = storage.reader( BUCKET_NAME1, BLOB_NAME2, BLOB_SOURCE_GENERATION, BLOB_SOURCE_METAGENERATION); assertNotNull(channel); assertTrue(channel.isOpen()); channel.read(ByteBuffer.allocate(42)); }
@Test public void testSaveAndRestore() throws IOException { byte[] firstResult = randomByteArray(DEFAULT_CHUNK_SIZE); byte[] secondResult = randomByteArray(DEFAULT_CHUNK_SIZE); ByteBuffer firstReadBuffer = ByteBuffer.allocate(42); ByteBuffer secondReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE); expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) .andReturn(Tuple.of("etag", firstResult)); expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 42, DEFAULT_CHUNK_SIZE)) .andReturn(Tuple.of("etag", secondResult)); replay(storageRpcMock); reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS); reader.read(firstReadBuffer); RestorableState<ReadChannel> readerState = reader.capture(); ReadChannel restoredReader = readerState.restore(); restoredReader.read(secondReadBuffer); assertArrayEquals( Arrays.copyOf(firstResult, firstReadBuffer.capacity()), firstReadBuffer.array()); assertArrayEquals(secondResult, secondReadBuffer.array()); }
@Test public void testReadAndWriteChannelWithEncryptionKey() throws IOException { String blobName = "test-read-write-channel-with-customer-key-blob"; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); byte[] stringBytes; try (WriteChannel writer = storage.writer(blob, Storage.BlobWriteOption.encryptionKey(BASE64_KEY))) { stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8); writer.write(ByteBuffer.wrap(BLOB_BYTE_CONTENT)); writer.write(ByteBuffer.wrap(stringBytes)); } ByteBuffer readBytes; ByteBuffer readStringBytes; try (ReadChannel reader = storage.reader(blob.getBlobId(), Storage.BlobSourceOption.decryptionKey(KEY))) { readBytes = ByteBuffer.allocate(BLOB_BYTE_CONTENT.length); readStringBytes = ByteBuffer.allocate(stringBytes.length); reader.read(readBytes); reader.read(readStringBytes); } assertArrayEquals(BLOB_BYTE_CONTENT, readBytes.array()); assertEquals(BLOB_STRING_CONTENT, new String(readStringBytes.array(), UTF_8)); assertTrue(storage.delete(BUCKET, blobName)); }
@Test public void testReaderWithDecryptionKey() throws IOException { byte[] result = new byte[DEFAULT_CHUNK_SIZE]; EasyMock.expect( storageRpcMock.read(BLOB_INFO2.toPb(), ENCRYPTION_KEY_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) .andReturn(Tuple.of("etag", result)) .times(2); EasyMock.replay(storageRpcMock); initializeService(); ReadChannel channel = storage.reader(BUCKET_NAME1, BLOB_NAME2, BlobSourceOption.decryptionKey(KEY)); assertNotNull(channel); assertTrue(channel.isOpen()); channel.read(ByteBuffer.allocate(42)); channel = storage.reader(BUCKET_NAME1, BLOB_NAME2, BlobSourceOption.decryptionKey(BASE64_KEY)); assertNotNull(channel); assertTrue(channel.isOpen()); channel.read(ByteBuffer.allocate(42)); }
@Test public void testReaderWithOptionsFromBlobId() throws IOException { byte[] result = new byte[DEFAULT_CHUNK_SIZE]; EasyMock.expect( storageRpcMock.read( BLOB_INFO1.getBlobId().toPb(), BLOB_SOURCE_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) .andReturn(Tuple.of("etag", result)); EasyMock.replay(storageRpcMock); initializeService(); ReadChannel channel = storage.reader( BLOB_INFO1.getBlobId(), BLOB_SOURCE_GENERATION_FROM_BLOB_ID, BLOB_SOURCE_METAGENERATION); assertNotNull(channel); assertTrue(channel.isOpen()); channel.read(ByteBuffer.allocate(42)); }
@Test public void testReadAndWriteCaptureChannels() throws IOException { String blobName = "test-read-and-write-capture-channels-blob"; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); byte[] stringBytes; WriteChannel writer = storage.writer(blob); stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8); writer.write(ByteBuffer.wrap(BLOB_BYTE_CONTENT)); RestorableState<WriteChannel> writerState = writer.capture(); WriteChannel secondWriter = writerState.restore(); secondWriter.write(ByteBuffer.wrap(stringBytes)); secondWriter.close(); ByteBuffer readBytes; ByteBuffer readStringBytes; ReadChannel reader = storage.reader(blob.getBlobId()); reader.setChunkSize(BLOB_BYTE_CONTENT.length); readBytes = ByteBuffer.allocate(BLOB_BYTE_CONTENT.length); reader.read(readBytes); RestorableState<ReadChannel> readerState = reader.capture(); ReadChannel secondReader = readerState.restore(); readStringBytes = ByteBuffer.allocate(stringBytes.length); secondReader.read(readStringBytes); reader.close(); secondReader.close(); assertArrayEquals(BLOB_BYTE_CONTENT, readBytes.array()); assertEquals(BLOB_STRING_CONTENT, new String(readStringBytes.array(), UTF_8)); assertTrue(storage.delete(BUCKET, blobName)); }