@Override public InputStreamWithHints openNextWithHints() throws IOException { if (!iterator.hasNext()) { return null; } final String key = iterator.next(); final GetObjectRequest request = new GetObjectRequest(bucket, key); S3Object object = new DefaultRetryable<S3Object>(format("Getting object '%s'", request.getKey())) { @Override public S3Object call() { return client.getObject(request); } }.executeWithCheckedException(retryExec, IOException.class); long objectSize = object.getObjectMetadata().getContentLength(); // Some plugin users are parsing this output to get file list. // Keep it for now but might be removed in the future. LOGGER.info("Open S3Object with bucket [{}], key [{}], with size [{}]", bucket, key, objectSize); InputStream inputStream = new ResumableInputStream(object.getObjectContent(), new S3InputStreamReopener(client, request, objectSize, retryExec)); return new InputStreamWithHints(inputStream, String.format("s3://%s/%s", bucket, key)); }
@Test public void reopenS3FileByReopener() throws Exception { String content = "value"; { // not retry doReturn(s3object("in/aa/a", content)).when(client).getObject(any(GetObjectRequest.class)); S3InputStreamReopener opener = new S3InputStreamReopener(client, new GetObjectRequest("my_bucket", "in/aa/a"), content.length()); try (InputStream in = opener.reopen(0, new RuntimeException())) { BufferedReader r = new BufferedReader(new InputStreamReader(in)); assertEquals("value", r.readLine()); } } { // retry once doThrow(new RuntimeException()).doReturn(s3object("in/aa/a", content)).when(client).getObject(any(GetObjectRequest.class)); S3InputStreamReopener opener = new S3InputStreamReopener( client, new GetObjectRequest("my_bucket", "in/aa/a"), content.length(), retryExecutor() .withInitialRetryWait(0) .withRetryLimit(1)); try (InputStream in = opener.reopen(0, new RuntimeException())) { BufferedReader r = new BufferedReader(new InputStreamReader(in)); assertEquals("value", r.readLine()); } } }
@Test(expected = AmazonClientException.class) public void reopenS3FileByReopener_on_retry_always_throw_exception() throws Exception { // always failed call with 2 retries doThrow(new AmazonClientException("This exception is thrown when retrying.")).when(client).getObject(any(GetObjectRequest.class)); S3InputStreamReopener opener = new S3InputStreamReopener( client, new GetObjectRequest("my_bucket", "in/aa/a"), "value".length(), retryExecutor() .withInitialRetryWait(0) .withRetryLimit(2)); try (InputStream in = opener.reopen(0, new AmazonClientException("This exception can be ignored"))) { fail("Should throw exception."); } }
@Test(expected = AmazonClientException.class) public void reopenS3FileByReopener_on_retry_gave_up_should_throw_original_exception() throws Exception { String content = "value"; doThrow(new AmazonClientException("no")).doReturn(s3object("in/aa/a", content)).when(client).getObject(any(GetObjectRequest.class)); S3InputStreamReopener opener = new S3InputStreamReopener( client, new GetObjectRequest("my_bucket", "in/aa/a"), content.length(), retryExecutor() .withInitialRetryWait(0) .withRetryLimit(0)); opener.reopen(0, new RuntimeException()); }