S3Service s3Service = new RestS3Service(awsCredentials); String key = formatLatLon(x, y) + ".tiff"; S3Object object = s3Service.getObject(awsBucketName, key);
RestS3Service s3Service = new RestS3Service(awsCredentials); S3Bucket[] myBuckets = s3Service.listAllBuckets(); System.out.println("How many buckets do I have in S3? " + myBuckets.length); S3Bucket testBucket = s3Service.createBucket(BUCKET_NAME); System.out.println("Created test bucket: " + testBucket.getName()); S3Bucket euBucket = s3Service.createBucket("eu-bucket", S3Bucket.LOCATION_EUROPE); S3Bucket usWestBucket = s3Service.createBucket("us-west-bucket", S3Bucket.LOCATION_US_WEST); S3Bucket asiaPacificBucket = s3Service.createBucket( "asia-pacific-bucket", S3Bucket.LOCATION_ASIA_PACIFIC); object = s3Service.putObject(testBucket, object); s3Service.putObject(testBucket, stringObject); s3Service.putObject(testBucket, fileObject); s3Service.putObject(testBucket, helloWorldObject); s3Service.putObject(testBucket, rrsObject); S3Object objectDetailsOnly = s3Service.getObjectDetails(testBucket, TEST_OBJECT_NAME); System.out.println("S3Object, details only: " + objectDetailsOnly); S3Object objectComplete = s3Service.getObject(testBucket, TEST_OBJECT_NAME); System.out.println("S3Object, complete: " + objectComplete); S3Object downloadedObject = s3Service.getObject(testBucket, TEST_OBJECT_NAME); String textData = ServiceUtils.readInputStreamToString(
RestS3Service service = new RestS3Service(credentials); S3Object[] objects = service.listObjects(s3BucketName, prefix, null, 1000);
@Override public boolean isInStorage() { try { RestS3Service service = null; try { service = new RestS3Service(new AWSCredentials(getAccess(), getSecret())); service.getObjectDetails(getBucket(), getPath()); return true; } finally { if (service != null) { service.shutdown(); } } } catch (ServiceException error) { return false; } }
RestS3Service s3Service = new RestS3Service(awsCredentials); s3Service.setRequesterPaysEnabled(true); S3Object object = s3Service.getObject("aws-publicdatasets", key); ar = ArcReaderFactory.getReader(object.getDataInputStream(), 1024 * 16);
public static boolean checkAuth(String awsAccessKey, String awsSecretKey) { AWSCredentials creds = null; try { creds = new AWSCredentials(awsAccessKey, awsSecretKey); RestS3Service s3Service = new RestS3Service(creds); s3Service.listAllBuckets(); return true; } catch (Exception e) { SDFSLogger.getLog().fatal("Unable to authenticate to AWS", e); return false; } }
@Override protected InputStream createData() throws IOException { RestS3Service service = null; try { service = new RestS3Service(new AWSCredentials(getAccess(), getSecret())); S3Object object = service.getObject(getBucket(), getPath()); return new ServiceInputStream(service, object.getDataInputStream()); } catch (ServiceException ex) { throw new IOException(String.format("Unable to store [%s] file!", getPath()), ex); } }
service = new RestS3Service(new AWSCredentials(getAccess(), getSecret())); S3Object object = new S3Object(getPath()); service.putObject(getBucket(), object); service.shutdown();
RestS3Service s3Service = new RestS3Service(awsCredentials); String signedPutUrl = s3Service.createSignedPutUrl( s3BucketName, objects[i].getKey(), objects[i].getMetadataMap(), expiryDate, false);
RestS3Service s3Service = new RestS3Service(awsCredentials); s3Service.setRequesterPaysEnabled(true); S3Object[] objects = s3Service.listObjects("aws-publicdatasets", "common-crawl/crawl-002/" + prefix, null); System.out.println("Fetched info for <" + objects.length + "> files");
public static boolean checkBucketUnique(String awsAccessKey, String awsSecretKey, String bucketName) { AWSCredentials creds = null; try { creds = new AWSCredentials(awsAccessKey, awsSecretKey); RestS3Service s3Service = new RestS3Service(creds); S3Bucket s3Bucket = s3Service.getBucket(bucketName); if (s3Bucket == null) { return true; // s3Bucket = s3Service.createBucket(bucketName); } return false; } catch (Exception e) { SDFSLogger.getLog().fatal("Unable to create aws bucket", e); return false; } }
RestS3Service s3Service = pool.borrowObject(); S3Bucket s3Bucket = s3Service.getBucket(this.name); s3Bucket = s3Service.createBucket(this.name); else s3Bucket = s3Service .createBucket(this.name, bucketLocation); SDFSLogger.getLog().info("created new store " + name); S3Object s3Object = new S3Object("bucketinfo"); s3Object.addMetadata("currentsize", "-1"); s3Object.addMetadata("currentcompressedsize", "-1"); s3Service.putObject(this.name, s3Object); } else { obj = s3Service.getObject(this.name, "bucketinfo"); } catch (Exception e) { SDFSLogger.getLog().info( s3Object.addMetadata("currentsize", "-1"); s3Object.addMetadata("currentcompressedsize", "-1"); s3Service.putObject(this.name, s3Object); } else { if (obj.containsMetadata("currentsize")) { .warn("The S3 objectstore DSE did not close correctly. Metadata tag currentsize was not added"); s3Service.updateObjectMetadata(this.name, obj);
@Override public void deleteChunk(byte[] hash, long start, int len) throws IOException { String hashString = this.getHashName(hash, Main.chunkStoreEncryptionEnabled); RestS3Service s3Service = null; try { this.chunks.invalidate(hashString); s3Service = pool.borrowObject(); StorageObject obj = s3Service.getObjectDetails(this.name, hashString); int size = Integer.parseInt((String) obj.getMetadata("size")); int compressedSize = Integer.parseInt((String) obj .getMetadata("compressedsize")); this.currentLength.addAndGet(-1 * size); this.compressedLength.addAndGet(-1 * compressedSize); s3Service.deleteObject(this.name, hashString); } catch (Exception e) { SDFSLogger.getLog() .warn("Unable to delete object " + hashString, e); } finally { pool.returnObject(s3Service); } }
@Override protected InputStream openObjectStream(S3Object object) throws IOException { try { // Get data of the given object and open an input stream return s3Client.getObject(object.getBucketName(), object.getKey()).getDataInputStream(); } catch (ServiceException e) { throw new IOException(e); } }
return; if (s3Client.isObjectInBucket(s3Bucket, s3Path)) { final S3Object[] list = s3Client.listObjects(s3Bucket, s3Path, ""); if (list.length == 0) { target.setAcl(GSAccessControlList.REST_CANNED_BUCKET_OWNER_FULL_CONTROL); s3Client.copyObject( s3Bucket, s3Path, false ); if (!s3Client.isObjectInBucket(targetS3Bucket, targetS3Path)) { throw new IOE( "After copy was reported as successful the file doesn't exist in the target location [%s]", if (s3Client.isObjectInBucket(targetS3Bucket, targetS3Path)) { log.info( "Not moving file [s3://%s/%s], already present in target location [s3://%s/%s]",
S3Object obj = s3Service.getObject(this.name, "bucketinfo"); obj.removeMetadata("currentsize"); obj.addMetadata("currentsize", obj.addMetadata("currentcompressedsize", Long.toString(this.compressedLength.get())); s3Service.updateObjectMetadata(this.name, obj); } catch (Exception e) { SDFSLogger.getLog().warn("error while closing bucket " + this.name,
StorageObjectsChunk objectsChunk; do { objectsChunk = s3Client.listObjectsChunked( bucket, prefix, final S3Object s3Object = s3Client.getObject(bucket, prefix); if (!s3Object.isDirectoryPlaceholder()) { objects.add(s3Object);
@Override public void kill(DataSegment segment) throws SegmentLoadingException { try { Map<String, Object> loadSpec = segment.getLoadSpec(); String s3Bucket = MapUtils.getString(loadSpec, "bucket"); String s3Path = MapUtils.getString(loadSpec, "key"); String s3DescriptorPath = S3Utils.descriptorPathForSegmentPath(s3Path); if (s3Client.isObjectInBucket(s3Bucket, s3Path)) { log.info("Removing index file[s3://%s/%s] from s3!", s3Bucket, s3Path); s3Client.deleteObject(s3Bucket, s3Path); } if (s3Client.isObjectInBucket(s3Bucket, s3DescriptorPath)) { log.info("Removing descriptor file[s3://%s/%s] from s3!", s3Bucket, s3DescriptorPath); s3Client.deleteObject(s3Bucket, s3DescriptorPath); } } catch (ServiceException e) { throw new SegmentLoadingException(e, "Couldn't kill segment[%s]: [%s]", segment.getIdentifier(), e); } }
private void putObject(String bucketName, String path, S3Object object) throws ServiceException { object.setBucketName(bucketName); object.setKey(path); if (!config.getDisableAcl()) { object.setAcl(GSAccessControlList.REST_CANNED_BUCKET_OWNER_FULL_CONTROL); } log.info("Pushing %s.", object); s3Client.putObject(bucketName, object); } }