@Override public boolean delete(Bucket bucket, Map<Option, ?> options) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_DELETE_BUCKET); Scope scope = tracer.withSpan(span); try { storage .buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(Option.IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(Option.IF_METAGENERATION_NOT_MATCH.getLong(options)) .setUserProject(Option.USER_PROJECT.getString(options)) .execute(); return true; } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); StorageException serviceException = translate(ex); if (serviceException.getCode() == HTTP_NOT_FOUND) { return false; } throw serviceException; } finally { scope.close(); span.end(); } }
@Override public Tuple<String, Iterable<Bucket>> list(Map<Option, ?> options) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_LIST_BUCKETS); Scope scope = tracer.withSpan(span); try { Buckets buckets = storage .buckets() .list(this.options.getProjectId()) .setProjection(DEFAULT_PROJECTION) .setPrefix(Option.PREFIX.getString(options)) .setMaxResults(Option.MAX_RESULTS.getLong(options)) .setPageToken(Option.PAGE_TOKEN.getString(options)) .setFields(Option.FIELDS.getString(options)) .setUserProject(Option.USER_PROJECT.getString(options)) .execute(); return Tuple.<String, Iterable<Bucket>>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); throw translate(ex); } finally { scope.close(); span.end(); } }
batchHelper.queue( configureRequest( gcs.buckets().get(resourceId.getBucketName()), resourceId.getBucketName()), new JsonBatchCallback<Bucket>() { @Override
@VisibleForTesting void createBucket(String projectId, Bucket bucket, BackOff backoff, Sleeper sleeper) throws IOException { Storage.Buckets.Insert insertBucket = storageClient.buckets().insert(projectId, bucket); insertBucket.setPredefinedAcl("projectPrivate"); insertBucket.setPredefinedDefaultObjectAcl("projectPrivate");
configureRequest(gcs.buckets().delete(bucketName), bucketName);
@VisibleForTesting @Nullable Bucket getBucket(GcsPath path, BackOff backoff, Sleeper sleeper) throws IOException { Storage.Buckets.Get getBucket = storageClient.buckets().get(path.getBucket());
List<Bucket> allBuckets = new ArrayList<>(); Storage.Buckets.List listBucket = configureRequest(gcs.buckets().list(storageOptions.getProjectId()), null);
/** * See {@link GoogleCloudStorage#create(String, CreateBucketOptions)} for details about expected * behavior. */ @Override public void create(String bucketName, CreateBucketOptions options) throws IOException { logger.atFine().log("create(%s)", bucketName); Preconditions.checkArgument( !Strings.isNullOrEmpty(bucketName), "bucketName must not be null or empty"); checkNotNull(options, "options must not be null"); checkNotNull(storageOptions.getProjectId(), "projectId must not be null"); Bucket bucket = new Bucket(); bucket.setName(bucketName); bucket.setLocation(options.getLocation()); bucket.setStorageClass(options.getStorageClass()); Storage.Buckets.Insert insertBucket = configureRequest(gcs.buckets().insert(storageOptions.getProjectId(), bucket), bucketName); // TODO(user): To match the behavior of throwing FileNotFoundException for 404, we probably // want to throw org.apache.commons.io.FileExistsException for 409 here. try { ResilientOperation.retry( ResilientOperation.getGoogleRequestCallable(insertBucket), backOffFactory.newBackOff(), rateLimitedRetryDeterminer, IOException.class, sleeper); } catch (InterruptedException e) { throw new IOException(e); // From sleep } }
/** Tests that a retryable error is retried enough times. */ @Test public void testRetryableErrorRetryEnoughTimes() throws IOException { when(mockLowLevelRequest.execute()).thenReturn(mockLowLevelResponse); final int retries = 10; when(mockLowLevelResponse.getStatusCode()) .thenAnswer( new Answer<Integer>() { int n = 0; @Override public Integer answer(InvocationOnMock invocation) { return n++ < retries ? 503 : 9999; } }); Storage.Buckets.Get result = storage.buckets().get("test"); try { result.executeUnparsed(); fail(); } catch (IOException e) { } verify(mockHttpResponseInterceptor).interceptResponse(any(HttpResponse.class)); verify(mockLowLevelRequest, atLeastOnce()).addHeader(anyString(), anyString()); verify(mockLowLevelRequest, times(retries + 1)).setTimeout(anyInt(), anyInt()); verify(mockLowLevelRequest, times(retries + 1)).setWriteTimeout(anyInt()); verify(mockLowLevelRequest, times(retries + 1)).execute(); verify(mockLowLevelResponse, times(retries + 1)).getStatusCode(); expectedLogs.verifyWarn("performed 10 retries due to unsuccessful status codes"); }
@Override public Bucket get(Bucket bucket, Map<Option, ?> options) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_GET_BUCKET); Scope scope = tracer.withSpan(span); try { return storage .buckets() .get(bucket.getName()) .setProjection(DEFAULT_PROJECTION) .setIfMetagenerationMatch(Option.IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(Option.IF_METAGENERATION_NOT_MATCH.getLong(options)) .setFields(Option.FIELDS.getString(options)) .setUserProject(Option.USER_PROJECT.getString(options)) .execute(); } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); StorageException serviceException = translate(ex); if (serviceException.getCode() == HTTP_NOT_FOUND) { return null; } throw serviceException; } finally { scope.close(); span.end(); } }
@Override public boolean delete(Bucket bucket, Map<Option, ?> options) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_DELETE_BUCKET); Scope scope = tracer.withSpan(span); try { storage .buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(Option.IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(Option.IF_METAGENERATION_NOT_MATCH.getLong(options)) .setUserProject(Option.USER_PROJECT.getString(options)) .execute(); return true; } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); StorageException serviceException = translate(ex); if (serviceException.getCode() == HTTP_NOT_FOUND) { return false; } throw serviceException; } finally { scope.close(); span.end(); } }
@Override public Tuple<String, Iterable<Bucket>> list(Map<Option, ?> options) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_LIST_BUCKETS); Scope scope = tracer.withSpan(span); try { Buckets buckets = storage .buckets() .list(this.options.getProjectId()) .setProjection(DEFAULT_PROJECTION) .setPrefix(Option.PREFIX.getString(options)) .setMaxResults(Option.MAX_RESULTS.getLong(options)) .setPageToken(Option.PAGE_TOKEN.getString(options)) .setFields(Option.FIELDS.getString(options)) .setUserProject(Option.USER_PROJECT.getString(options)) .execute(); return Tuple.<String, Iterable<Bucket>>of(buckets.getNextPageToken(), buckets.getItems()); } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); throw translate(ex); } finally { scope.close(); span.end(); } }
/** * Gets the bucket with the given name. * * @param bucketName name of the bucket to get * @return the bucket with the given name or null if bucket not found * @throws IOException if the bucket exists but cannot be accessed */ private Bucket getBucket(String bucketName) throws IOException { logger.atFine().log("getBucket(%s)", bucketName); checkArgument(!Strings.isNullOrEmpty(bucketName), "bucketName must not be null or empty"); Storage.Buckets.Get getBucket = configureRequest(gcs.buckets().get(bucketName), bucketName); try { return getBucket.execute(); } catch (IOException e) { if (errorExtractor.itemNotFound(e)) { logger.atFine().withCause(e).log("getBucket(%s): not found", bucketName); return null; } throw new IOException("Error accessing Bucket " + bucketName, e); } }
/** Tests that a non-retriable error is not retried. */ @Test public void testErrorCodeForbidden() throws IOException { when(mockLowLevelRequest.execute()).thenReturn(mockLowLevelResponse); when(mockLowLevelResponse.getStatusCode()) .thenReturn(403) // Non-retryable error. .thenReturn(200); // Shouldn't happen. try { Storage.Buckets.Get result = storage.buckets().get("test"); HttpResponse response = result.executeUnparsed(); assertNotNull(response); } catch (HttpResponseException e) { assertThat(e.getMessage(), Matchers.containsString("403")); } verify(mockHttpResponseInterceptor).interceptResponse(any(HttpResponse.class)); verify(mockLowLevelRequest, atLeastOnce()).addHeader(anyString(), anyString()); verify(mockLowLevelRequest).setTimeout(anyInt(), anyInt()); verify(mockLowLevelRequest).setWriteTimeout(anyInt()); verify(mockLowLevelRequest).execute(); verify(mockLowLevelResponse).getStatusCode(); expectedLogs.verifyWarn("Request failed with code 403"); }
@Override public Bucket patch(Bucket bucket, Map<Option, ?> options) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_PATCH_BUCKET); Scope scope = tracer.withSpan(span); try { return storage .buckets() .patch(bucket.getName(), bucket) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(Option.PREDEFINED_ACL.getString(options)) .setPredefinedDefaultObjectAcl(Option.PREDEFINED_DEFAULT_OBJECT_ACL.getString(options)) .setIfMetagenerationMatch(Option.IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(Option.IF_METAGENERATION_NOT_MATCH.getLong(options)) .setUserProject(Option.USER_PROJECT.getString(options)) .execute(); } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); throw translate(ex); } finally { scope.close(); span.end(); } }
@Test public void testGetBucketNotExists() throws IOException { GcsOptions pipelineOptions = gcsOptionsWithTestCredential(); GcsUtil gcsUtil = pipelineOptions.getGcsUtil(); Storage mockStorage = Mockito.mock(Storage.class); gcsUtil.setStorageClient(mockStorage); Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class); Storage.Buckets.Get mockStorageGet = Mockito.mock(Storage.Buckets.Get.class); BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff()); when(mockStorage.buckets()).thenReturn(mockStorageObjects); when(mockStorageObjects.get("testbucket")).thenReturn(mockStorageGet); when(mockStorageGet.execute()) .thenThrow( googleJsonResponseException( HttpStatusCodes.STATUS_CODE_NOT_FOUND, "It don't exist", "Nothing here to see")); thrown.expect(FileNotFoundException.class); thrown.expectMessage("It don't exist"); gcsUtil.getBucket( GcsPath.fromComponents("testbucket", "testobject"), mockBackOff, new FastNanoClockAndSleeper()); }
@Test public void testBucketDoesNotExistBecauseOfAccessError() throws IOException { GcsOptions pipelineOptions = gcsOptionsWithTestCredential(); GcsUtil gcsUtil = pipelineOptions.getGcsUtil(); Storage mockStorage = Mockito.mock(Storage.class); gcsUtil.setStorageClient(mockStorage); Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class); Storage.Buckets.Get mockStorageGet = Mockito.mock(Storage.Buckets.Get.class); BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff()); GoogleJsonResponseException expectedException = googleJsonResponseException( HttpStatusCodes.STATUS_CODE_FORBIDDEN, "Waves hand mysteriously", "These aren't the buckets you're looking for"); when(mockStorage.buckets()).thenReturn(mockStorageObjects); when(mockStorageObjects.get("testbucket")).thenReturn(mockStorageGet); when(mockStorageGet.execute()).thenThrow(expectedException); assertFalse( gcsUtil.bucketAccessible( GcsPath.fromComponents("testbucket", "testobject"), mockBackOff, new FastNanoClockAndSleeper())); }
@Test public void testBucketDoesNotExist() throws IOException { GcsOptions pipelineOptions = gcsOptionsWithTestCredential(); GcsUtil gcsUtil = pipelineOptions.getGcsUtil(); Storage mockStorage = Mockito.mock(Storage.class); gcsUtil.setStorageClient(mockStorage); Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class); Storage.Buckets.Get mockStorageGet = Mockito.mock(Storage.Buckets.Get.class); BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff()); when(mockStorage.buckets()).thenReturn(mockStorageObjects); when(mockStorageObjects.get("testbucket")).thenReturn(mockStorageGet); when(mockStorageGet.execute()) .thenThrow( googleJsonResponseException( HttpStatusCodes.STATUS_CODE_NOT_FOUND, "It don't exist", "Nothing here to see")); assertFalse( gcsUtil.bucketAccessible( GcsPath.fromComponents("testbucket", "testobject"), mockBackOff, new FastNanoClockAndSleeper())); }
/** Tests that a retriable error is retried. */ @Test public void testRetryableError() throws IOException { when(mockLowLevelRequest.execute()) .thenReturn(mockLowLevelResponse) .thenReturn(mockLowLevelResponse) .thenReturn(mockLowLevelResponse); when(mockLowLevelResponse.getStatusCode()) .thenReturn(503) // Retryable .thenReturn(429) // We also retry on 429 Too Many Requests. .thenReturn(200); Storage.Buckets.Get result = storage.buckets().get("test"); HttpResponse response = result.executeUnparsed(); assertNotNull(response); verify(mockHttpResponseInterceptor).interceptResponse(any(HttpResponse.class)); verify(mockLowLevelRequest, atLeastOnce()).addHeader(anyString(), anyString()); verify(mockLowLevelRequest, times(3)).setTimeout(anyInt(), anyInt()); verify(mockLowLevelRequest, times(3)).setWriteTimeout(anyInt()); verify(mockLowLevelRequest, times(3)).execute(); verify(mockLowLevelResponse, times(3)).getStatusCode(); expectedLogs.verifyDebug("Request failed with code 503"); }