public long size(final String bucket, final String path) throws IOException { return storage.objects().get(bucket, path).execute().getSize().longValue(); }
@Override public StorageObject get(StorageObject object, Map<Option, ?> options) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_GET_OBJECT); Scope scope = tracer.withSpan(span); try { return getCall(object, options).execute(); } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); StorageException serviceException = translate(ex); if (serviceException.getCode() == HTTP_NOT_FOUND) { return null; } throw serviceException; } finally { scope.close(); span.end(); } }
@Override public <T> T loadObject(String accountName, ObjectType objectType, String objectKey) throws IllegalArgumentException, NotFoundException { GoogleNamedAccountCredentials credentials = (GoogleNamedAccountCredentials)accountCredentialsRepository .getOne(accountName) .orElseThrow(() -> new IllegalArgumentException("Unable to resolve account " + accountName + ".")); Storage storage = credentials.getStorage(); String bucketName = credentials.getBucket(); StorageObject item; try { item = resolveSingularItem(objectType, objectKey, credentials, storage, bucketName); } catch (IllegalArgumentException e) { throw new NotFoundException(e.getMessage()); } try { StorageObject storageObject = storage.objects().get(bucketName, item.getName()).execute(); return deserialize(storage, storageObject, objectType.getTypeReference()); } catch (IOException e) { if (e instanceof HttpResponseException) { HttpResponseException hre = (HttpResponseException)e; log.error("Failed to load {} {}: {} {}", objectType.getGroup(), objectKey, hre.getStatusCode(), hre.getStatusMessage()); if (hre.getStatusCode() == 404) { throw new NotFoundException("No file at path " + item.getName() + "."); } } throw new IllegalStateException(e); } }
.objects() .get(projectName + "-cloud-pubsub-loadtest", filePath.getFileName().toString()) .execute() .getMd5Hash()); try (InputStream inputStream = Files.newInputStream(filePath, StandardOpenOption.READ)) {
/** * Gets the object with the given resourceId. * * @param resourceId identifies a StorageObject * @return the object with the given name or null if object not found * @throws IOException if the object exists but cannot be accessed */ private StorageObject getObject(StorageResourceId resourceId) throws IOException { logger.atFine().log("getObject(%s)", resourceId); Preconditions.checkArgument( resourceId.isStorageObject(), "Expected full StorageObject id, got %s", resourceId); String bucketName = resourceId.getBucketName(); String objectName = resourceId.getObjectName(); Storage.Objects.Get getObject = configureRequest(gcs.objects().get(bucketName, objectName), bucketName); try { return getObject.execute(); } catch (IOException e) { if (errorExtractor.itemNotFound(e)) { logger.atFine().withCause(e).log("getObject(%s): not found", resourceId); return null; } throw new IOException("Error accessing " + resourceId, e); } }
@Test public void testRetryFileSizeNonBatch() throws IOException { GcsOptions pipelineOptions = gcsOptionsWithTestCredential(); GcsUtil gcsUtil = pipelineOptions.getGcsUtil(); Storage mockStorage = Mockito.mock(Storage.class); gcsUtil.setStorageClient(mockStorage); Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class); Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class); BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.withMaxRetries(2).backoff()); when(mockStorage.objects()).thenReturn(mockStorageObjects); when(mockStorageObjects.get("testbucket", "testobject")).thenReturn(mockStorageGet); when(mockStorageGet.execute()) .thenThrow(new SocketTimeoutException("SocketException")) .thenThrow(new SocketTimeoutException("SocketException")) .thenReturn(new StorageObject().setSize(BigInteger.valueOf(1000))); assertEquals( 1000, gcsUtil .getObject( GcsPath.fromComponents("testbucket", "testobject"), mockBackOff, new FastNanoClockAndSleeper()) .getSize() .longValue()); assertEquals(BackOff.STOP, mockBackOff.nextBackOffMillis()); }
@Test public void testAccessDeniedObjectThrowsIOException() throws IOException { GcsOptions pipelineOptions = gcsOptionsWithTestCredential(); GcsUtil gcsUtil = pipelineOptions.getGcsUtil(); Storage mockStorage = Mockito.mock(Storage.class); gcsUtil.setStorageClient(mockStorage); Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class); Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class); GcsPath pattern = GcsPath.fromUri("gs://testbucket/testdirectory/accessdeniedfile"); GoogleJsonResponseException expectedException = googleJsonResponseException( HttpStatusCodes.STATUS_CODE_FORBIDDEN, "Waves hand mysteriously", "These aren't the buckets you're looking for"); when(mockStorage.objects()).thenReturn(mockStorageObjects); when(mockStorageObjects.get(pattern.getBucket(), pattern.getObject())) .thenReturn(mockStorageGet); when(mockStorageGet.execute()).thenThrow(expectedException); thrown.expect(IOException.class); thrown.expectMessage("Unable to get the file object for path"); gcsUtil.expand(pattern); }
@Test public void testNonExistentObjectReturnsEmptyResult() throws IOException { GcsOptions pipelineOptions = gcsOptionsWithTestCredential(); GcsUtil gcsUtil = pipelineOptions.getGcsUtil(); Storage mockStorage = Mockito.mock(Storage.class); gcsUtil.setStorageClient(mockStorage); Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class); Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class); GcsPath pattern = GcsPath.fromUri("gs://testbucket/testdirectory/nonexistentfile"); GoogleJsonResponseException expectedException = googleJsonResponseException( HttpStatusCodes.STATUS_CODE_NOT_FOUND, "It don't exist", "Nothing here to see"); when(mockStorage.objects()).thenReturn(mockStorageObjects); when(mockStorageObjects.get(pattern.getBucket(), pattern.getObject())) .thenReturn(mockStorageGet); when(mockStorageGet.execute()).thenThrow(expectedException); assertEquals(Collections.emptyList(), gcsUtil.expand(pattern)); }
@Override public StorageObject get(StorageObject object, Map<Option, ?> options) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_GET_OBJECT); Scope scope = tracer.withSpan(span); try { return getCall(object, options).execute(); } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); StorageException serviceException = translate(ex); if (serviceException.getCode() == HTTP_NOT_FOUND) { return null; } throw serviceException; } finally { scope.close(); span.end(); } }
Optional<StorageObject> stat(String bucket, String object) throws IOException { try { return Optional.of(client.objects() .get(bucket, object) .execute()); } catch (GoogleJsonResponseException e) { if (e.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) { return Optional.absent(); } throw e; } }
@Test public void testFileSizeNonBatch() throws Exception { GcsOptions pipelineOptions = gcsOptionsWithTestCredential(); GcsUtil gcsUtil = pipelineOptions.getGcsUtil(); Storage mockStorage = Mockito.mock(Storage.class); gcsUtil.setStorageClient(mockStorage); Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class); Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class); when(mockStorage.objects()).thenReturn(mockStorageObjects); when(mockStorageObjects.get("testbucket", "testobject")).thenReturn(mockStorageGet); when(mockStorageGet.execute()) .thenReturn(new StorageObject().setSize(BigInteger.valueOf(1000))); assertEquals(1000, gcsUtil.fileSize(GcsPath.fromComponents("testbucket", "testobject"))); }
private static boolean GCSURLExists(String url) { // ensure data is accessible try { // if we can read the size, then surely we can read the file GcsPath fn = GcsPath.fromUri(url); Storage.Objects storageClient = GCSOptions.Methods.createStorageClient(pipelineOptions, auth); Storage.Objects.Get getter = storageClient.get(fn.getBucket(), fn.getObject()); StorageObject object = getter.execute(); BigInteger size = object.getSize(); return true; } catch (Exception x) { return false; } }
private static boolean GCSURLExists(String url) { // ensure data is accessible try { // if we can read the size, then surely we can read the file GcsPath fn = GcsPath.fromUri(url); Storage.Objects storageClient = GCSOptions.Methods.createStorageClient(pipelineOptions, auth); Storage.Objects.Get getter = storageClient.get(fn.getBucket(), fn.getObject()); StorageObject object = getter.execute(); BigInteger size = object.getSize(); return true; } catch (Exception x) { return false; } }
/** * @param name of the file we're interested in * @return size of the file, in bytes * @throws IOException */ public long getFileSize(String bucket, String name) throws IOException { Storage.Objects.Get getObject = storage.objects().get(bucket, name); StorageObject object = getObject.execute(); BigInteger size = object.getSize(); if (size.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) { throw new RuntimeException("File size is too big for a long!"); } return size.longValue(); }
Storage.Objects.Get getObject = client.objects().get(bucketName, objectname); StorageObject object = getObject.execute(); System.out.println("Size is: " + object.getSize().longValue());
/** * @param name of the file we're interested in * @return size of the file, in bytes * @throws IOException */ public long getFileSize(String bucket, String name) throws IOException { Storage.Objects.Get getObject = storage.objects().get(bucket, name); StorageObject object = getObject.execute(); BigInteger size = object.getSize(); if (size.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) { throw new RuntimeException("File size is too big for a long!"); } return size.longValue(); }
private static void checkGcsUrlExists(String url) throws IOException { // Ensure data is accessible. // If we can read the size, then surely we can read the file. GcsPath fn = GcsPath.fromUri(url); Storage.Objects storageClient = GCSOptions.Methods.createStorageClient(pipelineOptions, auth); Storage.Objects.Get getter = storageClient.get(fn.getBucket(), fn.getObject()); StorageObject object = getter.execute(); BigInteger size = object.getSize(); }
@Override public StorageObject get(StorageObject object, Map<Option, ?> options) { try { return getRequest(object, options).execute(); } catch (IOException ex) { StorageException serviceException = translate(ex); if (serviceException.code() == HTTP_NOT_FOUND) { return null; } throw serviceException; } }
public static StorageObject get(Storage storage, String bucketName, String objectName) throws IOException { Storage.Objects.Get getObject = storage.objects().get(bucketName, objectName); return getObject.execute(); }
public String version(final String bucket, final String path) throws IOException { return storage.objects().get(bucket, path).execute().getEtag(); }