protected void validateBucket(AmazonS3Client client, String bucketName) { log.debug("Looking for bucket: " + bucketName); if (client.doesBucketExist(bucketName)) { log.debug("Found bucket '" + bucketName + "' Validating permissions"); validatePermissions(client, bucketName); } else { log.info("Creating bucket " + bucketName); // If we create the bucket, we "own" it and by default have the "fullcontrol" permission client.createBucket(bucketName); } }
public static boolean checkBucketUnique(String awsAccessKey, String awsSecretKey, String bucketName) { AWSCredentials creds = null; try { creds = new BasicAWSCredentials(awsAccessKey, awsSecretKey); AmazonS3Client s3Service = new AmazonS3Client(creds); return s3Service.doesBucketExist(bucketName); } catch (Exception e) { SDFSLogger.getLog().fatal("Unable to create aws bucket", e); return false; } }
@Override public void close() { // backend is closing. abort all mulitpart uploads from start. if(s3service.doesBucketExist(bucket)) { tmx.abortMultipartUploads(bucket, startTime); } tmx.shutdownNow(); s3service.shutdown(); LOG.info("S3Backend closed."); }
@Override public void close() { // backend is closing. abort all mulitpart uploads from start. if(s3service.doesBucketExist(bucket)) { tmx.abortMultipartUploads(bucket, startTime); } tmx.shutdownNow(); s3service.shutdown(); LOG.info("S3Backend closed."); }
@Override public Bucket createBucket(String bucket) { checkNotBlank(bucket, "bucket"); checkArgument(!client.doesBucketExist(bucket), "bucket [%s] already exists"); return Bucket.copyOf(client.createBucket(bucket, region)); }
public static boolean checkBucketUnique(String awsAccessKey, String awsSecretKey, String bucketName) { BasicAWSCredentials creds = null; try { creds = new BasicAWSCredentials(awsAccessKey, awsSecretKey); @SuppressWarnings("deprecation") AmazonS3Client s3Service = new AmazonS3Client(creds); boolean exists = s3Service.doesBucketExist(bucketName); if (!exists) { s3Service.createBucket(bucketName); } return true; } catch (Exception e) { SDFSLogger.getLog().fatal("Unable to create aws bucket", e); return false; } }
/** * Returns the root URL where the bucket name is located. * <p>Please note that the URL does not contain the bucket name</p> * @param bucketName The bucket name * @return the root URL where the bucket name is located. * @throws S3Exception If something goes wrong. */ private String ensureBucketExists(String bucketName) { String bucketUrl = null; try { if (!s3Client.doesBucketExist(bucketName)) { LOG.info("Bucket {} doesn't exists...Creating one"); s3Client.createBucket(bucketName); LOG.info("Created bucket: {}", bucketName); } bucketUrl = s3Client.getResourceUrl(bucketName, null) + bucketName; } catch (AmazonClientException ace) { LOG.error("An error occurred while connecting to S3. Will not execute action" + " for bucket: {}", bucketName, ace); throw new S3Exception(ace); } return bucketUrl; }
@Override public void close() throws DataStoreException { super.close(); // backend is closing. abort all mulitpart uploads from start. if(s3service.doesBucketExist(bucket)) { tmx.abortMultipartUploads(bucket, startTime); } tmx.shutdownNow(); s3service.shutdown(); LOG.info("S3Backend closed."); }
@Override public void prepare(Map conf, TopologyContext context, OutputCollector collector) { super.prepare(conf, context, collector); bucketName = ConfUtils.getString(conf, BUCKET); boolean bucketExists = client.doesBucketExist(bucketName); if (!bucketExists) { String message = "Bucket " + bucketName + " does not exist"; throw new RuntimeException(message); } this.eventCounter = context.registerMetric("s3cache_counter", new MultiCountMetric(), 10); }
@Override public void close() throws DataStoreException { super.close(); // backend is closing. abort all mulitpart uploads from start. if(s3service.doesBucketExist(bucket)) { tmx.abortMultipartUploads(bucket, startTime); } tmx.shutdownNow(); s3service.shutdown(); LOG.info("S3Backend closed."); }
@Override public void prepare(Map conf, TopologyContext context, OutputCollector collector) { super.prepare(conf, context, collector); bucketName = ConfUtils.getString(conf, BUCKET); boolean bucketExists = client.doesBucketExist(bucketName); if (!bucketExists) { String message = "Bucket " + bucketName + " does not exist"; throw new RuntimeException(message); } this.eventCounter = context.registerMetric(getMetricPrefix() + "s3cache_counter", new MultiCountMetric(), 10); }
@Override public Task<Void> start() { s3Connection = new S3Connection(s3Configuration); // Check the bucket actually exists // TODO: Should we create it here if it doesn't? if(!s3Connection.getS3Client().doesBucketExist(bucketName)) { throw new UncheckedException("S3 bucket '" + bucketName + "' does not exist."); } return Task.done(); }
Main.writeThreads); AmazonS3Client s3Service = pool.borrowObject(); boolean exists = s3Service.doesBucketExist(this.name); if (!exists) { s3Service.createBucket(this.name);
@Override public Optional<ObjectMetadata> getOptionalMetadata(String bucket, String key) { // Blanks not allowed checkNotBlank(bucket, "bucket"); checkNotBlank(key, "key"); // Throw an exception if they are asking about a key inside a bucket that doesn't exist checkArgument(client.doesBucketExist(bucket), "bucket [%s] does not exist", bucket); try { // Attempt to get metadata for this object in this bucket org.kuali.common.aws.s3.model.ObjectMetadata omd = org.kuali.common.aws.s3.model.ObjectMetadata.copyOf(client.getObjectMetadata(bucket, key)); // If we successfully got metadata, the object definitely exists return Optional.of(omd); } catch (AmazonS3Exception ase) { if (ase.getStatusCode() == HTTP_NOT_FOUND) { // Object definitely doesn't exist // NOTE: We can only safely return false here because we've already checked to make sure the bucket exists // Amazon also returns a 404 for buckets that don't exist return absent(); } else { // Re-throw all other exceptions, as we now have no way to determine if the object exists. // A 403 permissions error could just mean we are being denied access to the bucket // It may, or may not, mean anything about the existence of the indicated key in the indicated bucket. // This is very similar to what the JDK does with file.exists() // The JDK throws SecurityException if a security manager exists and its SecurityManager.checkRead(String) method denies read access to the file or directory throw ase; } } }
@Override public Bucket getOrCreateBucketOwnedByMe(String bucket) { if (client.doesBucketExist(bucket)) { List<com.amazonaws.services.s3.model.Bucket> buckets = client.listBuckets(); for (com.amazonaws.services.s3.model.Bucket b : buckets) { if (b.getName().equals(bucket)) { return Bucket.copyOf(b); } } throw illegalState("bucket [%s] exists, but is not owned by the currently authenticated user", bucket); } else { return Bucket.copyOf(client.createBucket(bucket, region)); } }
/** * Creates a binary store with a connection to Amazon S3 * * @param accessKey AWS access key credential * @param secretKey AWS secret key credential * @param bucketName Name of the S3 bucket in which binary content will be stored * @param endPoint The S3 endpoint URL where the bucket will be accessed * @throws BinaryStoreException if S3 connection cannot be made to verify bucket */ public S3BinaryStore(String accessKey, String secretKey, String bucketName, String endPoint) throws BinaryStoreException { this.bucketName = bucketName; this.s3Client = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey)); // Support for compatible S3 storage systems if(endPoint != null) this.s3Client.setEndpoint(endPoint); this.fileSystemCache = TransientBinaryStore.get(); this.fileSystemCache.setMinimumBinarySizeInBytes(1L); // Ensure bucket exists try { if (!s3Client.doesBucketExist(bucketName)) { s3Client.createBucket(bucketName); } } catch (AmazonClientException e) { throw new BinaryStoreException(e); } }
AmazonS3Client s3service = Utils.openService(props); TransferManager tmx = new TransferManager(s3service); if (s3service.doesBucketExist(bucket)) { for (int i = 0; i < 4; i++) { tmx.abortMultipartUploads(bucket, date);
if (!s3Client.doesBucketExist(bucketName)) { s3Client.createBucket(bucketName);
client.setEndpoint(endpoint); if (!client.doesBucketExist(bucket)) { client.createBucket(bucket);
AmazonS3Client s3service = Utils.openService(props); TransferManager tmx = new TransferManager(s3service); if (s3service.doesBucketExist(bucket)) { for (int i = 0; i < 4; i++) { tmx.abortMultipartUploads(bucket, date);