absoluteUri.getQuery(), absoluteUri.getFragment()); for (GcsPath entry : gcsUtil.expand(GcsPath.fromUri(gcsUriGlob))) { uris.add(entry.toUri());
/** * Creates an object in GCS. * * <p>Returns a WritableByteChannel that can be used to write data to the object. * * @param path the GCS file to write to * @param type the type of object, eg "text/plain". * @return a Callable object that encloses the operation. */ public WritableByteChannel create(GcsPath path, String type) throws IOException { return create(path, type, uploadBufferSizeBytes); }
/** * Calculate the checksums of a set of files on GCS. * @param gcsUtil - Used to retrieve the files. * @param gcsPaths - List of paths of the files. * @return A List of String representing the MD5 hashes of the files. */ public static List<String> getGcsFileChecksums(GcsUtil gcsUtil, List<GcsPath> gcsPaths) { List<String> checksums = new ArrayList<>(); try { for (StorageObjectOrIOException objectOrIOException : gcsUtil.getObjects(gcsPaths)) { IOException ex = objectOrIOException.ioException(); if (ex != null) { throw ex; } checksums.add(objectOrIOException.storageObject().getMd5Hash()); } } catch (IOException e) { throw new RuntimeException(e); } return checksums; } }
/** * Creates a {@link Bucket} under the specified project in Cloud Storage or propagates an * exception. */ public void createBucket(String projectId, Bucket bucket) throws IOException { createBucket(projectId, bucket, createBackOff(), Sleeper.DEFAULT); }
/** Returns whether the GCS bucket exists and is accessible. */ public boolean bucketAccessible(GcsPath path) throws IOException { return bucketAccessible(path, createBackOff(), Sleeper.DEFAULT); }
/** * Returns the project number of the project which owns this bucket. If the bucket exists, it must * be accessible otherwise the permissions exception will be propagated. If the bucket does not * exist, an exception will be thrown. */ public long bucketOwner(GcsPath path) throws IOException { return getBucket(path, createBackOff(), Sleeper.DEFAULT).getProjectNumber().longValue(); }
@Test(expected = RuntimeException.class) public void testPackageUploadFailsWhenIOExceptionThrown() throws Exception { File tmpFile = makeFileWithContents("file.txt", "This is a test!"); when(mockGcsUtil.getObjects(anyListOf(GcsPath.class))) .thenReturn( ImmutableList.of( StorageObjectOrIOException.create(new FileNotFoundException("some/path")))); when(mockGcsUtil.create(any(GcsPath.class), anyString())) .thenThrow(new IOException("Fake Exception: Upload error")); try (PackageUtil directPackageUtil = PackageUtil.withExecutorService(MoreExecutors.newDirectExecutorService())) { directPackageUtil.stageClasspathElements( ImmutableList.of(tmpFile.getAbsolutePath()), STAGING_PATH, fastNanoClockAndSleeper, createOptions); } finally { verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class)); verify(mockGcsUtil, times(5)).create(any(GcsPath.class), anyString()); verifyNoMoreInteractions(mockGcsUtil); } }
this.mockGcsUtil = mock(GcsUtil.class); when(mockGcsUtil.create(any(GcsPath.class), anyString())) .then( invocation -> StandardOpenOption.DELETE_ON_CLOSE)); when(mockGcsUtil.create(any(GcsPath.class), anyString(), anyInt())) .then( invocation -> StandardOpenOption.DELETE_ON_CLOSE)); when(mockGcsUtil.expand(any(GcsPath.class))) .then(invocation -> ImmutableList.of((GcsPath) invocation.getArguments()[0])); when(mockGcsUtil.bucketAccessible(GcsPath.fromUri(VALID_STAGING_BUCKET))).thenReturn(true); when(mockGcsUtil.bucketAccessible(GcsPath.fromUri(VALID_TEMP_BUCKET))).thenReturn(true); when(mockGcsUtil.bucketAccessible(GcsPath.fromUri(VALID_TEMP_BUCKET + "/staging/"))) .thenReturn(true); when(mockGcsUtil.bucketAccessible(GcsPath.fromUri(VALID_PROFILE_BUCKET))).thenReturn(true); when(mockGcsUtil.bucketAccessible(GcsPath.fromUri(NON_EXISTENT_BUCKET))).thenReturn(false); when(mockGcsUtil.getObjects(anyListOf(GcsPath.class))) .thenAnswer( invocationOnMock -> { when(mockGcsUtil.bucketAccessible(GcsPath.fromUri("gs://bucket/object"))).thenReturn(true);
@Test public void testNonExistentObjectReturnsEmptyResult() throws IOException { GcsOptions pipelineOptions = gcsOptionsWithTestCredential(); GcsUtil gcsUtil = pipelineOptions.getGcsUtil(); Storage mockStorage = Mockito.mock(Storage.class); gcsUtil.setStorageClient(mockStorage); Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class); Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class); GcsPath pattern = GcsPath.fromUri("gs://testbucket/testdirectory/nonexistentfile"); GoogleJsonResponseException expectedException = googleJsonResponseException( HttpStatusCodes.STATUS_CODE_NOT_FOUND, "It don't exist", "Nothing here to see"); when(mockStorage.objects()).thenReturn(mockStorageObjects); when(mockStorageObjects.get(pattern.getBucket(), pattern.getObject())) .thenReturn(mockStorageGet); when(mockStorageGet.execute()).thenThrow(expectedException); assertEquals(Collections.emptyList(), gcsUtil.expand(pattern)); }
private GcsUtil buildMockGcsUtil() throws IOException { GcsUtil mockGcsUtil = Mockito.mock(GcsUtil.class); // Any request to open gets a new bogus channel Mockito.when(mockGcsUtil.open(Mockito.any(GcsPath.class))) .then( invocation -> FileChannel.open( Files.createTempFile("channel-", ".tmp"), StandardOpenOption.CREATE, StandardOpenOption.DELETE_ON_CLOSE)); // Any request for expansion returns a list containing the original GcsPath // This is required to pass validation that occurs in TextIO during apply() Mockito.when(mockGcsUtil.expand(Mockito.any(GcsPath.class))) .then(invocation -> ImmutableList.of((GcsPath) invocation.getArguments()[0])); return mockGcsUtil; } }
private static DataflowPipelineOptions buildPipelineOptions(String... args) throws IOException { GcsUtil mockGcsUtil = mock(GcsUtil.class); when(mockGcsUtil.expand(any(GcsPath.class))) .then(invocation -> ImmutableList.of((GcsPath) invocation.getArguments()[0])); when(mockGcsUtil.bucketAccessible(any(GcsPath.class))).thenReturn(true); DataflowPipelineOptions options = PipelineOptionsFactory.fromArgs(args).as(DataflowPipelineOptions.class); options.setRunner(DataflowRunner.class); options.setGcpCredential(new TestCredential()); options.setJobName("some-job-name"); options.setProject("some-project"); options.setRegion("some-region"); options.setTempLocation(GcsPath.fromComponents("somebucket", "some/path").toString()); options.setFilesToStage(new ArrayList<>()); options.setGcsUtil(mockGcsUtil); return options; } }
private GcsUtil buildMockGcsUtil() throws IOException { GcsUtil mockGcsUtil = mock(GcsUtil.class); when(mockGcsUtil.create(any(GcsPath.class), anyString())) .then( invocation -> FileChannel.open( Files.createTempFile("channel-", ".tmp"), StandardOpenOption.CREATE, StandardOpenOption.DELETE_ON_CLOSE)); when(mockGcsUtil.expand(any(GcsPath.class))) .then(invocation -> ImmutableList.of((GcsPath) invocation.getArguments()[0])); return mockGcsUtil; }
@Test public void testBucketAccessible() throws IOException { GcsOptions pipelineOptions = gcsOptionsWithTestCredential(); GcsUtil gcsUtil = pipelineOptions.getGcsUtil(); Storage mockStorage = Mockito.mock(Storage.class); gcsUtil.setStorageClient(mockStorage); Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class); Storage.Buckets.Get mockStorageGet = Mockito.mock(Storage.Buckets.Get.class); BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff()); when(mockStorage.buckets()).thenReturn(mockStorageObjects); when(mockStorageObjects.get("testbucket")).thenReturn(mockStorageGet); when(mockStorageGet.execute()) .thenThrow(new SocketTimeoutException("SocketException")) .thenReturn(new Bucket()); assertTrue( gcsUtil.bucketAccessible( GcsPath.fromComponents("testbucket", "testobject"), mockBackOff, new FastNanoClockAndSleeper())); }
when(mockGcsUtil.listObjects(eq("testbucket"), anyString(), isNull(String.class))) .thenReturn(modelObjects); GcsPath.fromUri("gs://testbucket/testdirectory/otherfile")); when(mockGcsUtil.getObjects(eq(gcsPaths))) .thenReturn( ImmutableList.of(
/** * Returns the file size from GCS or throws {@link FileNotFoundException} if the resource does not * exist. */ @VisibleForTesting List<Long> fileSizes(List<GcsPath> paths) throws IOException { List<StorageObjectOrIOException> results = getObjects(paths); ImmutableList.Builder<Long> ret = ImmutableList.builder(); for (StorageObjectOrIOException result : results) { ret.add(toFileSize(result)); } return ret.build(); }
private void verifyPathIsAccessible(String path, String errorMessage) { GcsPath gcsPath = getGcsPath(path); try { checkArgument(gcpOptions.getGcsUtil().bucketAccessible(gcsPath), errorMessage, path); } catch (IOException e) { throw new RuntimeException( String.format("Unable to verify that GCS bucket gs://%s exists.", gcsPath.getBucket()), e); } }
@Override protected ReadableByteChannel open(GcsResourceId resourceId) throws IOException { return options.getGcsUtil().open(resourceId.getGcsPath()); }
@Test public void testCreateBucket() throws IOException { GcsOptions pipelineOptions = gcsOptionsWithTestCredential(); GcsUtil gcsUtil = pipelineOptions.getGcsUtil(); Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class); Storage mockStorage = Mockito.mock(Storage.class); gcsUtil.setStorageClient(mockStorage); Storage.Buckets.Insert mockStorageInsert = Mockito.mock(Storage.Buckets.Insert.class); BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff()); when(mockStorage.buckets()).thenReturn(mockStorageObjects); when(mockStorageObjects.insert(any(String.class), any(Bucket.class))) .thenReturn(mockStorageInsert); when(mockStorageInsert.execute()) .thenThrow(new SocketTimeoutException("SocketException")) .thenReturn(new Bucket()); gcsUtil.createBucket("a", new Bucket(), mockBackOff, new FastNanoClockAndSleeper()); }
@Test public void testGetBucket() throws IOException { GcsOptions pipelineOptions = gcsOptionsWithTestCredential(); GcsUtil gcsUtil = pipelineOptions.getGcsUtil(); Storage mockStorage = Mockito.mock(Storage.class); gcsUtil.setStorageClient(mockStorage); Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class); Storage.Buckets.Get mockStorageGet = Mockito.mock(Storage.Buckets.Get.class); BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff()); when(mockStorage.buckets()).thenReturn(mockStorageObjects); when(mockStorageObjects.get("testbucket")).thenReturn(mockStorageGet); when(mockStorageGet.execute()) .thenThrow(new SocketTimeoutException("SocketException")) .thenReturn(new Bucket()); assertNotNull( gcsUtil.getBucket( GcsPath.fromComponents("testbucket", "testobject"), mockBackOff, new FastNanoClockAndSleeper())); }
gcpOptions.getGcsUtil().createBucket(projectId, bucket); } catch (FileAlreadyExistsException e) { LOG.debug("Bucket '{}'' already exists, verifying access.", bucketName); long owner = gcpOptions.getGcsUtil().bucketOwner(GcsPath.fromComponents(bucketName, "")); checkArgument( owner == projectNumber,