@Override public byte[] load(StorageObject from, Map<Option, ?> options) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_LOAD); Scope scope = tracer.withSpan(span); try { Storage.Objects.Get getRequest = storage .objects() .get(from.getBucket(), from.getName()) .setGeneration(from.getGeneration()) .setIfMetagenerationMatch(Option.IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(Option.IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(Option.IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(Option.IF_GENERATION_NOT_MATCH.getLong(options)) .setUserProject(Option.USER_PROJECT.getString(options)); setEncryptionHeaders(getRequest.getRequestHeaders(), ENCRYPTION_KEY_PREFIX, options); ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.executeMedia().download(out); return out.toByteArray(); } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); throw translate(ex); } finally { scope.close(); span.end(); } }
@Override public Policy setIamPolicy(String bucket, Policy policy, Map<Option, ?> options) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_SET_BUCKET_IAM_POLICY); Scope scope = tracer.withSpan(span); try { return storage .buckets() .setIamPolicy(bucket, policy) .setUserProject(Option.USER_PROJECT.getString(options)) .execute(); } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); throw translate(ex); } finally { scope.close(); span.end(); } }
private DefaultRpcBatch(Storage storage) { this.storage = storage; batches = new LinkedList<>(); // add OpenCensus HttpRequestInitializer batches.add(storage.batch(batchRequestInitializer)); }
@Override public String open(StorageObject object, Map<Option, ?> options) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_OPEN); Scope scope = tracer.withSpan(span); try { Insert req = storage.objects().insert(object.getBucket(), object); GenericUrl url = req.buildHttpRequest().getUrl(); String scheme = url.getScheme(); JsonFactory jsonFactory = storage.getJsonFactory(); HttpRequestFactory requestFactory = storage.getRequestFactory(); HttpRequest httpRequest = requestFactory.buildPostRequest(url, new JsonHttpContent(jsonFactory, object)); requestHeaders.set( "X-Upload-Content-Type", firstNonNull(object.getContentType(), "application/octet-stream")); String key = Option.CUSTOMER_SUPPLIED_KEY.getString(options); if (key != null) { BaseEncoding base64 = BaseEncoding.base64(); error.setCode(response.getStatusCode()); error.setMessage(response.getStatusMessage()); throw translate(error); throw translate(ex); } finally { scope.close();
/** {@inheritDoc} */ @Override public void registerAddresses(Collection<InetSocketAddress> addrs) throws IgniteSpiException { assert !F.isEmpty(addrs); init(); for (InetSocketAddress addr : addrs) { String key = keyFromAddr(addr); StorageObject object = new StorageObject(); object.setBucket(bucketName); object.setName(key); InputStreamContent content = new InputStreamContent("application/octet-stream", OBJECT_CONTENT); content.setLength(OBJECT_CONTENT.available()); try { Storage.Objects.Insert insertObject = storage.objects().insert(bucketName, object, content); insertObject.execute(); } catch (Exception e) { throw new IgniteSpiException("Failed to put entry [bucketName=" + bucketName + ", entry=" + key + ']', e); } } }
private Storage.Objects.Delete deleteCall(StorageObject blob, Map<Option, ?> options) throws IOException { return storage .objects() .delete(blob.getBucket(), blob.getName()) .setGeneration(blob.getGeneration()) .setIfMetagenerationMatch(Option.IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(Option.IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(Option.IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(Option.IF_GENERATION_NOT_MATCH.getLong(options)) .setUserProject(Option.USER_PROJECT.getString(options)); }
@Override public Tuple<String, byte[]> read( StorageObject from, Map<Option, ?> options, long position, int bytes) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_READ); Scope scope = tracer.withSpan(span); try { Get req = storage .objects() .get(from.getBucket(), from.getName()) .setGeneration(from.getGeneration()) .setIfMetagenerationMatch(Option.IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(Option.IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(Option.IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(Option.IF_GENERATION_NOT_MATCH.getLong(options)) .setUserProject(Option.USER_PROJECT.getString(options)); HttpHeaders requestHeaders = req.getRequestHeaders(); requestHeaders.setRange(range.toString()); setEncryptionHeaders(requestHeaders, ENCRYPTION_KEY_PREFIX, options); ByteArrayOutputStream output = new ByteArrayOutputStream(bytes); HttpResponse httpResponse = req.executeMedia(); IOUtils.copy(lowLevelHttpResponse.getContent(), output); } catch (IllegalAccessException | NoSuchFieldException ex) { throw new StorageException( BaseServiceException.UNKNOWN_CODE, "Error parsing gzip response", ex);
private Storage.Objects.Patch patchCall(StorageObject storageObject, Map<Option, ?> options) throws IOException { return storage .objects() .patch(storageObject.getBucket(), storageObject.getName(), storageObject) .setProjection(DEFAULT_PROJECTION) .setPredefinedAcl(Option.PREDEFINED_ACL.getString(options)) .setIfMetagenerationMatch(Option.IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(Option.IF_METAGENERATION_NOT_MATCH.getLong(options)) .setIfGenerationMatch(Option.IF_GENERATION_MATCH.getLong(options)) .setIfGenerationNotMatch(Option.IF_GENERATION_NOT_MATCH.getLong(options)) .setUserProject(Option.USER_PROJECT.getString(options)); }
/** {@inheritDoc} */ @Override public Collection<InetSocketAddress> getRegisteredAddresses() throws IgniteSpiException { init(); Collection<InetSocketAddress> addrs = new ArrayList<>(); try { Storage.Objects.List listObjects = storage.objects().list(bucketName); com.google.api.services.storage.model.Objects objects; do { objects = listObjects.execute(); if (objects == null || objects.getItems() == null) break; for (StorageObject object : objects.getItems()) addrs.add(addrFromString(object.getName())); listObjects.setPageToken(objects.getNextPageToken()); } while (null != objects.getNextPageToken()); } catch (Exception e) { throw new IgniteSpiException("Failed to get content from the bucket: " + bucketName, e); } return addrs; }
/** * Uploads data to an object in a bucket. * * @param name the name of the destination object. * @param contentType the MIME type of the data. * @param file the file to upload. * @param bucketName the name of the bucket to create the object in. */ public static void uploadFile( String name, String contentType, File file, String bucketName) throws IOException, GeneralSecurityException { InputStreamContent contentStream = new InputStreamContent( contentType, new FileInputStream(file)); // Setting the length improves upload performance contentStream.setLength(file.length()); StorageObject objectMetadata = new StorageObject() // Set the destination object name .setName(name) // Set the access control list to publicly read-only .setAcl(Arrays.asList( new ObjectAccessControl().setEntity("allUsers").setRole("READER"))); // Do the insert Storage client = StorageFactory.getService(); Storage.Objects.Insert insertRequest = client.objects().insert( bucketName, objectMetadata, contentStream); insertRequest.execute(); } // [END upload_stream]
@Override public String open(StorageObject object, Map<Option, ?> options) { try { Insert req = storage.objects().insert(object.getBucket(), object); GenericUrl url = req.buildHttpRequest().getUrl(); String scheme = url.getScheme(); url = new GenericUrl(scheme + "://" + host + path); url.set("uploadType", "resumable"); url.set("name", object.getName()); for (Option option : options.keySet()) { Object content = option.get(options); JsonFactory jsonFactory = storage.getJsonFactory(); HttpRequestFactory requestFactory = storage.getRequestFactory(); HttpRequest httpRequest = requestFactory.buildPostRequest(url, new JsonHttpContent(jsonFactory, object)); httpRequest.getHeaders().set("X-Upload-Content-Type", firstNonNull(object.getContentType(), "application/octet-stream")); HttpResponse response = httpRequest.execute(); if (response.getStatusCode() != 200) {
@Override public StorageObject create( StorageObject storageObject, final InputStream content, Map<Option, ?> options) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_CREATE_OBJECT); Scope scope = tracer.withSpan(span); try { Storage.Objects.Insert insert = storage .objects() .insert( storageObject.getBucket(), storageObject, new InputStreamContent(storageObject.getContentType(), content)); insert.getMediaHttpUploader().setDirectUploadEnabled(true); Boolean disableGzipContent = Option.IF_DISABLE_GZIP_CONTENT.getBoolean(options); if (disableGzipContent != null) { insert.setDisableGZipContent(disableGzipContent); } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); throw translate(ex); } finally { scope.close();
/** * Used by TEST SUITES only. Called through reflection. * * @param bucketName Bucket to delete. */ private void removeBucket(String bucketName) { init(); try { Storage.Buckets.Delete deleteBucket = storage.buckets().delete(bucketName); deleteBucket.execute(); } catch (Exception e) { throw new IgniteSpiException("Failed to remove the bucket: " + bucketName, e); } }
Storage.Objects.List objectsList = storage.objects().list(bucketName).setPrefix(rootFolder); Objects objects; String itemName = item.getName(); int indexOfLastSlash = itemName.lastIndexOf("/"); Map<String, Object> objectMetadataMap = new HashMap<>(); long updatedTimestamp = item.getUpdated().getValue();
@Override public boolean delete(Bucket bucket, Map<Option, ?> options) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_DELETE_BUCKET); Scope scope = tracer.withSpan(span); try { storage .buckets() .delete(bucket.getName()) .setIfMetagenerationMatch(Option.IF_METAGENERATION_MATCH.getLong(options)) .setIfMetagenerationNotMatch(Option.IF_METAGENERATION_NOT_MATCH.getLong(options)) .setUserProject(Option.USER_PROJECT.getString(options)) .execute(); return true; } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); StorageException serviceException = translate(ex); if (serviceException.getCode() == HTTP_NOT_FOUND) { return false; } throw serviceException; } finally { scope.close(); span.end(); } }
public static StorageObject uploadWithMetadata(Storage storage, StorageObject object, InputStream data) throws IOException { InputStreamContent mediaContent = new InputStreamContent(object.getContentType(), data); Storage.Objects.Insert insertObject = storage.objects().insert(object.getBucket(), object, mediaContent); insertObject.getMediaHttpUploader().setDisableGZipContent(true); return insertObject.execute(); }
@Override public Tuple<String, Iterable<StorageObject>> list(final String bucket, Map<Option, ?> options) { Span span = startSpan(HttpStorageRpcSpans.SPAN_NAME_LIST_OBJECTS); Scope scope = tracer.withSpan(span); try { Objects objects = storage .objects() .list(bucket) .setProjection(DEFAULT_PROJECTION) .setVersions(Option.VERSIONS.getBoolean(options)) .setDelimiter(Option.DELIMITER.getString(options)) .setPrefix(Option.PREFIX.getString(options)) .setMaxResults(Option.MAX_RESULTS.getLong(options)) .setPageToken(Option.PAGE_TOKEN.getString(options)) firstNonNull(objects.getItems(), ImmutableList.<StorageObject>of()), objects.getPrefixes() != null ? Lists.transform(objects.getPrefixes(), objectFromPrefix(bucket)) : ImmutableList.<StorageObject>of()); return Tuple.of(objects.getNextPageToken(), storageObjects); } catch (IOException ex) { span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage())); throw translate(ex); } finally { scope.close();