private COSClient getCOSClient() { if (cosClient == null) { // 1 初始化用户身份信息(secretId, secretKey) COSCredentials cred = new BasicCOSCredentials(secretId, secretKey); // 2 设置bucket的区域, COS地域的简称请参照 https://cloud.tencent.com/document/product/436/6224 ClientConfig clientConfig = new ClientConfig(new Region(region)); cosClient = new COSClient(cred, clientConfig); } return cosClient; }
/** * Creates a new instance of {@link COSInputStream}. * * @param bucketName the name of the bucket * @param key the key of the file * @param client the client for COS * @param position the position to begin reading from * @param multiRangeChunkSize the chunk size to use on this stream */ COSInputStream(String bucketName, String key, COSClient client, long position, long multiRangeChunkSize) throws IOException { super(multiRangeChunkSize); mBucketName = bucketName; mKey = key; mCosClient = client; mPos = position; ObjectMetadata meta = mCosClient.getObjectMetadata(mBucketName, key); mContentLength = meta == null ? 0 : meta.getContentLength(); }
@Override protected boolean deleteObject(String key) { try { mClient.deleteObject(mBucketNameInternal, key); } catch (CosClientException e) { LOG.error("Failed to delete {}", key, e); return false; } return true; }
@Override public void store(InputStream inputStream, long contentLength, String contentType, String keyName) { try { // 简单文件上传, 最大支持 5 GB, 适用于小文件上传, 建议 20M以下的文件使用该接口 ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setContentLength(contentLength); objectMetadata.setContentType(contentType); // 对象键(Key)是对象在存储桶中的唯一标识。例如,在对象的访问域名 `bucket1-1250000000.cos.ap-guangzhou.myqcloud.com/doc1/pic1.jpg` 中,对象键为 doc1/pic1.jpg, 详情参考 [对象键](https://cloud.tencent.com/document/product/436/13324) PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, keyName, inputStream, objectMetadata); PutObjectResult putObjectResult = getCOSClient().putObject(putObjectRequest); } catch (Exception ex) { ex.printStackTrace(); } }
/** * Creates an COS {@code ClientConfiguration} using an Alluxio Configuration. * * @return the COS {@link ClientConfig} */ private static ClientConfig createCOSClientConfig(String regionName, UnderFileSystemConfiguration conf) { ClientConfig config = new ClientConfig(new Region(regionName)); config.setConnectionTimeout((int) conf.getMs(PropertyKey.COS_CONNECTION_TIMEOUT)); config.setSocketTimeout((int) conf.getMs(PropertyKey.COS_SOCKET_TIMEOUT)); config.setMaxConnectionsCount(conf.getInt(PropertyKey.COS_CONNECTION_MAX)); return config; }
/** * Constructs a new instance of {@link COSUnderFileSystem}. * * @param uri the {@link AlluxioURI} for this UFS * @param conf the configuration for this UFS * @param alluxioConf Alluxio configuration * @return the created {@link COSUnderFileSystem} instance */ public static COSUnderFileSystem createInstance(AlluxioURI uri, UnderFileSystemConfiguration conf, AlluxioConfiguration alluxioConf) throws Exception { String bucketName = UnderFileSystemUtils.getBucketName(uri); Preconditions.checkArgument(conf.isSet(PropertyKey.COS_ACCESS_KEY), "Property %s is required to connect to COS", PropertyKey.COS_ACCESS_KEY); Preconditions.checkArgument(conf.isSet(PropertyKey.COS_SECRET_KEY), "Property %s is required to connect to COS", PropertyKey.COS_SECRET_KEY); Preconditions.checkArgument(conf.isSet(PropertyKey.COS_REGION), "Property %s is required to connect to COS", PropertyKey.COS_REGION); Preconditions.checkArgument(conf.isSet(PropertyKey.COS_APP_ID), "Property %s is required to connect to COS", PropertyKey.COS_APP_ID); String accessKey = conf.get(PropertyKey.COS_ACCESS_KEY); String secretKey = conf.get(PropertyKey.COS_SECRET_KEY); String regionName = conf.get(PropertyKey.COS_REGION); String appId = conf.get(PropertyKey.COS_APP_ID); COSCredentials cred = new BasicCOSCredentials(accessKey, secretKey); ClientConfig clientConfig = createCOSClientConfig(regionName, conf); COSClient client = new COSClient(cred, clientConfig); return new COSUnderFileSystem(uri, client, bucketName, appId, conf, alluxioConf); }
@Override protected boolean createEmptyObject(String key) { try { ObjectMetadata objMeta = new ObjectMetadata(); objMeta.setContentLength(0); mClient.putObject(mBucketNameInternal, key, new ByteArrayInputStream(new byte[0]), objMeta); return true; } catch (CosClientException e) { LOG.error("Failed to create object: {}", key, e); return false; } }
/** * Closes this output stream. When an output stream is closed, the local temporary file is * uploaded to COS Service. Once the file is uploaded, the temporary file is deleted. */ @Override public void close() throws IOException { if (mClosed.getAndSet(true)) { return; } mLocalOutputStream.close(); try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(mFile))) { ObjectMetadata meta = new ObjectMetadata(); meta.setContentLength(mFile.length()); if (mHash != null) { byte[] hashBytes = mHash.digest(); meta.setContentMD5(new String(Base64.encodeBase64(hashBytes))); } mCosClient.putObject(mBucketName, mKey, in, meta); mFile.delete(); } catch (CosClientException e) { LOG.error("Failed to upload {}. Temporary file @ {}", mKey, mFile.getPath()); throw new IOException(e); } return; } }
@Override protected InputStream createStream(long startPos, long endPos) throws IOException { GetObjectRequest req = new GetObjectRequest(mBucketName, mKey); // COS returns entire object if we read past the end req.setRange(startPos, endPos < mContentLength ? endPos - 1 : mContentLength - 1); COSObject object = mCosClient.getObject(req); return new BufferedInputStream(object.getObjectContent()); } }
@Override protected ObjectStatus getObjectStatus(String key) { try { ObjectMetadata meta = mClient.getObjectMetadata(mBucketNameInternal, key); if (meta == null) { return null; } return new ObjectStatus(key, meta.getETag(), meta.getContentLength(), meta.getLastModified().getTime()); } catch (CosClientException e) { LOG.warn("Failed to get Object {}, return null", key, e); return null; } }
private ObjectListing getObjectListingChunk(ListObjectsRequest request) { ObjectListing result; try { result = mClient.listObjects(request); } catch (CosClientException e) { LOG.error("Failed to list path {}", request.getPrefix(), e); result = null; } return result; }
@Override public ObjectListingChunk getNextChunk() throws IOException { if (mResult.isTruncated()) { ObjectListing nextResult = mClient.listObjects(mRequest); if (nextResult != null) { return new COSObjectListingChunk(mRequest, nextResult); } } return null; } }
@Override public AccessControlList getBucketAcl(GetBucketAclRequest getBucketAclRequest) throws CosClientException, CosServiceException { String bucketName = getBucketAclRequest.getBucketName(); rejectNull(bucketName, "The bucket name parameter must be specified when requesting a bucket's ACL"); rejectNull(clientConfig.getRegion(), "region is null, region in clientConfig must be specified when requesting a bucket acl"); return getAcl(bucketName, null, null, getBucketAclRequest); }
@Override public COSObject getCOSObjectStream() { return getObject(getObjectRequest); } }, ServiceUtils.OVERWRITE_MODE);
@Override protected boolean copyObject(String src, String dst) { try { LOG.debug("Copying {} to {}", src, dst); mClient.copyObject(mBucketNameInternal, src, mBucketNameInternal, dst); return true; } catch (CosClientException e) { LOG.error("Failed to rename file {} to {}", src, dst, e); return false; } }
@Override public URL generatePresignedUrl(String bucketName, String key, Date expiration) throws CosClientException { return generatePresignedUrl(bucketName, key, expiration, HttpMethodName.GET); }
public void setCOSCredentialsProvider(COSCredentialsProvider credProvider) { rejectNull(credProvider, "credProvider must not be null"); this.credProvider = credProvider; }
@Override public void delete(String keyName) { try { getCOSClient().deleteObject(bucketName, keyName); } catch (Exception e) { e.printStackTrace(); } }
@Override public COSObject getCOSObjectStream() { return getObject(getObjectRequest); } }, ServiceUtils.OVERWRITE_MODE);
@Override public URL generatePresignedUrl(String bucketName, String key, Date expiration) throws CosClientException { return generatePresignedUrl(bucketName, key, expiration, HttpMethodName.GET); }