How to use

MultipartUploadSlicingAlgorithm

in

org.jclouds.blobstore.strategy.internal

protected long getNextChunkOffset() {
 long next = chunkOffset;
 chunkOffset += getChunkSize();
 return next;
}

/**
* Phase 1 of the algorithm.
* ChunkSize does not grow from a {@code MultipartUploadStrategy.DEFAULT_PART_SIZE}
* until we reach {@code MultipartUploadSlicingAlgorithm.MAGNITUDE_BASE} number of parts.
*/
@Test
public void testWhenChunkSizeHasToStartGrowing() {
 MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm(
    MIN_PART_SIZE, MAX_PART_SIZE, MAX_NUMBER_OF_PARTS);
 // upper limit while we still have exactly defaultPartSize chunkSize
 long length = MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE * MultipartUploadSlicingAlgorithm.DEFAULT_MAGNITUDE_BASE;
 long chunkSize = strategy.calculateChunkSize(length);
 assertEquals(chunkSize, MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE);
 assertEquals(strategy.getParts(), MultipartUploadSlicingAlgorithm.DEFAULT_MAGNITUDE_BASE - 1);
 assertEquals(strategy.getRemaining(), MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE);
 assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length);
 // then chunkSize is increasing
 length += 1;
 chunkSize = strategy.calculateChunkSize(length);
 assertEquals(chunkSize, MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE * 2);
 assertEquals(strategy.getParts(), MultipartUploadSlicingAlgorithm.DEFAULT_MAGNITUDE_BASE / 2);
 assertEquals(strategy.getRemaining(), 1);
 assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length);
}

@Beta
protected String putMultipartBlob(String container, Blob blob, PutOptions overrides, ListeningExecutorService executor) {
 ArrayList<ListenableFuture<MultipartPart>> parts = new ArrayList<ListenableFuture<MultipartPart>>();
 long contentLength = checkNotNull(blob.getMetadata().getContentMetadata().getContentLength(),
    "must provide content-length to use multi-part upload");
 MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm(
    getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts());
 long partSize = algorithm.calculateChunkSize(contentLength);
 MultipartUpload mpu = initiateMultipartUpload(container, blob.getMetadata(), partSize, overrides);
 int partNumber = 0;
 for (Payload payload : slicer.slice(blob.getPayload(), partSize)) {
   BlobUploader b =
      new BlobUploader(mpu, partNumber++, payload);
   parts.add(executor.submit(b));
 }
 return completeMultipartUpload(mpu, Futures.getUnchecked(Futures.allAsList(parts)));
}

long contentLength = blob.getMetadata().getContentMetadata().getContentLength();
MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm(
   getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts());
long partSize = algorithm.calculateChunkSize(contentLength);
int partNumber = 1;
while (partNumber <= algorithm.getParts()) {
 Payload payload = slicer.slice(blob.getPayload(), algorithm.getCopied(), partSize);
 BlobUploader b =
    new BlobUploader(mpu, partNumber++, payload);
 parts.add(executor.submit(b));
 algorithm.addCopied(partSize);
if (algorithm.getRemaining() != 0) {
 Payload payload = slicer.slice(blob.getPayload(), algorithm.getCopied(), algorithm.getRemaining());
 BlobUploader b =
    new BlobUploader(mpu, partNumber, payload);

/**
* Phase 2 of the algorithm.
* The number of parts does not grow from {@code MultipartUploadSlicingAlgorithm.MAGNITUDE_BASE}
* until we reach the {@code MultipartUploadStrategy.MAX_PART_SIZE}.
*/
@Test
public void testWhenPartsHasToStartGrowingFromMagnitudeBase() {
 MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm(
    MIN_PART_SIZE, MAX_PART_SIZE, MAX_NUMBER_OF_PARTS);
 // upper limit while we still have exactly MAGNITUDE_BASE parts (together with the remaining)
 long length = MAX_PART_SIZE * MultipartUploadSlicingAlgorithm.DEFAULT_MAGNITUDE_BASE;
 long chunkSize = strategy.calculateChunkSize(length);
 assertEquals(chunkSize, MAX_PART_SIZE);
 assertEquals(strategy.getParts(), MultipartUploadSlicingAlgorithm.DEFAULT_MAGNITUDE_BASE - 1);
 assertEquals(strategy.getRemaining(), MAX_PART_SIZE);
 assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length);
 // then the number of parts is increasing
 length += 1;
 chunkSize = strategy.calculateChunkSize(length);
 assertEquals(chunkSize, MAX_PART_SIZE);
 assertEquals(strategy.getParts(), MultipartUploadSlicingAlgorithm.DEFAULT_MAGNITUDE_BASE);
 assertEquals(strategy.getRemaining(), 1);
 assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length);
}

@Beta
protected String putMultipartBlob(String container, Blob blob, PutOptions overrides, ListeningExecutorService executor) {
 ArrayList<ListenableFuture<MultipartPart>> parts = new ArrayList<ListenableFuture<MultipartPart>>();
 long contentLength = checkNotNull(blob.getMetadata().getContentMetadata().getContentLength(),
    "must provide content-length to use multi-part upload");
 MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm(
    getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts());
 long partSize = algorithm.calculateChunkSize(contentLength);
 MultipartUpload mpu = initiateMultipartUpload(container, blob.getMetadata(), partSize, overrides);
 int partNumber = 0;
 for (Payload payload : slicer.slice(blob.getPayload(), partSize)) {
   BlobUploader b =
      new BlobUploader(mpu, partNumber++, payload);
   parts.add(executor.submit(b));
 }
 return completeMultipartUpload(mpu, Futures.getUnchecked(Futures.allAsList(parts)));
}

long contentLength = blob.getMetadata().getContentMetadata().getContentLength();
MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm(
   getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts());
long partSize = algorithm.calculateChunkSize(contentLength);
int partNumber = 1;
while (partNumber <= algorithm.getParts()) {
 Payload slice = slicer.slice(payload, algorithm.getCopied(), partSize);
 BlobUploader b = new BlobUploader(mpu, partNumber++, slice);
 parts.add(repeatable ? executor.submit(b) : Futures.immediateFuture(b.call()));
 algorithm.addCopied(partSize);
if (algorithm.getRemaining() != 0) {
 Payload slice = slicer.slice(payload, algorithm.getCopied(), algorithm.getRemaining());
 BlobUploader b = new BlobUploader(mpu, partNumber, slice);
 parts.add(repeatable ? executor.submit(b) : Futures.immediateFuture(b.call()));

  /**
  * Phase 3 of the algorithm.
  * The number of parts are increasing until {@code MAX_NUMBER_OF_PARTS}
  * while its size does not exceeds the {@code MultipartUploadStrategy.MAX_PART_SIZE}.
  */
  @Test
  public void testWhenPartsExceedsMaxNumberOfParts() {
   MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm(
      MIN_PART_SIZE, MAX_PART_SIZE, MAX_NUMBER_OF_PARTS);
   // upper limit while we still have exactly MAX_NUMBER_OF_PARTS parts (together with the remaining)
   long length = MAX_PART_SIZE * MAX_NUMBER_OF_PARTS;
   long chunkSize = strategy.calculateChunkSize(length);
   assertEquals(chunkSize, MAX_PART_SIZE);
   assertEquals(strategy.getParts(), MAX_NUMBER_OF_PARTS - 1);
   assertEquals(strategy.getRemaining(), MAX_PART_SIZE);
   assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length);

   // then the number of parts is increasing
   length += 1;
   chunkSize = strategy.calculateChunkSize(length);
   assertEquals(chunkSize, MAX_PART_SIZE);
   assertEquals(strategy.getParts(), MAX_NUMBER_OF_PARTS);
   assertEquals(strategy.getRemaining(), 1);
   assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length);
  }
}

@Beta
protected String putMultipartBlob(String container, Blob blob, PutOptions overrides, ListeningExecutorService executor) {
 ArrayList<ListenableFuture<MultipartPart>> parts = new ArrayList<ListenableFuture<MultipartPart>>();
 long contentLength = checkNotNull(blob.getMetadata().getContentMetadata().getContentLength(),
    "must provide content-length to use multi-part upload");
 MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm(
    getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts());
 long partSize = algorithm.calculateChunkSize(contentLength);
 MultipartUpload mpu = initiateMultipartUpload(container, blob.getMetadata(), partSize, overrides);
 int partNumber = 0;
 for (Payload payload : slicer.slice(blob.getPayload(), partSize)) {
   BlobUploader b =
      new BlobUploader(mpu, partNumber++, payload);
   parts.add(executor.submit(b));
 }
 return completeMultipartUpload(mpu, Futures.getUnchecked(Futures.allAsList(parts)));
}

protected long getNextChunkOffset() {
 long next = chunkOffset;
 chunkOffset += getChunkSize();
 return next;
}

MultipartUploadSlicingAlgorithm strategy = new MultipartUploadSlicingAlgorithm(
   MIN_PART_SIZE, MAX_PART_SIZE, MAX_NUMBER_OF_PARTS);
long chunkSize = strategy.calculateChunkSize(length);
assertEquals(chunkSize, MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE);
assertEquals(strategy.getParts(), 0);
assertEquals(strategy.getRemaining(), length);
assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length);
chunkSize = strategy.calculateChunkSize(length);
assertEquals(chunkSize, MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE);
assertEquals(strategy.getParts(), 0);
assertEquals(strategy.getRemaining(), length);
assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length);
chunkSize = strategy.calculateChunkSize(length);
assertEquals(chunkSize, MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE);
assertEquals(strategy.getParts(), 1);
assertEquals(strategy.getRemaining(), 1);
assertEquals(chunkSize * strategy.getParts() + strategy.getRemaining(), length);

@Beta
protected String putMultipartBlob(String container, Blob blob, PutOptions overrides, ListeningExecutorService executor) {
 ArrayList<ListenableFuture<MultipartPart>> parts = new ArrayList<ListenableFuture<MultipartPart>>();
 MultipartUpload mpu = initiateMultipartUpload(container, blob.getMetadata(), overrides);
 try {
   long contentLength = blob.getMetadata().getContentMetadata().getContentLength();
   MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm(
      getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts());
   long partSize = algorithm.calculateChunkSize(contentLength);
   int partNumber = 1;
   for (Payload payload : slicer.slice(blob.getPayload(), partSize)) {
    BlobUploader b =
       new BlobUploader(mpu, partNumber++, payload);
    parts.add(executor.submit(b));
   }
   return completeMultipartUpload(mpu, Futures.getUnchecked(Futures.allAsList(parts)));
 } catch (RuntimeException re) {
   abortMultipartUpload(mpu);
   throw re;
 }
}

protected long getNextChunkOffset() {
 long next = chunkOffset;
 chunkOffset += getChunkSize();
 return next;
}

@Beta
protected String putMultipartBlob(String container, Blob blob, PutOptions overrides, ListeningExecutorService executor) {
 ArrayList<ListenableFuture<MultipartPart>> parts = new ArrayList<ListenableFuture<MultipartPart>>();
 MultipartUpload mpu = initiateMultipartUpload(container, blob.getMetadata(), overrides);
 try {
   long contentLength = blob.getMetadata().getContentMetadata().getContentLength();
   MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm(
      getMinimumMultipartPartSize(), getMaximumMultipartPartSize(), getMaximumNumberOfParts());
   long partSize = algorithm.calculateChunkSize(contentLength);
   int partNumber = 1;
   for (Payload payload : slicer.slice(blob.getPayload(), partSize)) {
    BlobUploader b =
       new BlobUploader(mpu, partNumber++, payload);
    parts.add(executor.submit(b));
   }
   return completeMultipartUpload(mpu, Futures.getUnchecked(Futures.allAsList(parts)));
 } catch (RuntimeException re) {
   abortMultipartUpload(mpu);
   throw re;
 }
}

protected long getNextChunkOffset() {
 long next = chunkOffset;
 chunkOffset += getChunkSize();
 return next;
}

@Test(groups = { "integration", "live" })
public void testPutMultipartByteSource() throws Exception {
 long length = Math.max(getMinimumMultipartBlobSize(), MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE + 1);
 BlobStore blobStore = view.getBlobStore();
 MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm(
     blobStore.getMinimumMultipartPartSize(), blobStore.getMaximumMultipartPartSize(),
     blobStore.getMaximumNumberOfParts());
 // make sure that we are creating multiple parts
 assertThat(algorithm.calculateChunkSize(length)).isLessThan(length);
 ByteSource byteSource = TestUtils.randomByteSource().slice(0, length);
 Payload payload = new ByteSourcePayload(byteSource);
 HashCode hashCode = byteSource.hash(Hashing.md5());
 testPut(payload, hashCode, payload, length, new PutOptions().multipart(true));
}

@Test(groups = { "integration", "live" })
public void testPutMultipartInputStream() throws Exception {
 long length = Math.max(getMinimumMultipartBlobSize(), MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE + 1);
 BlobStore blobStore = view.getBlobStore();
 MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm(
     blobStore.getMinimumMultipartPartSize(), blobStore.getMaximumMultipartPartSize(),
     blobStore.getMaximumNumberOfParts());
 // make sure that we are creating multiple parts
 assertThat(algorithm.calculateChunkSize(length)).isLessThan(length);
 ByteSource byteSource = TestUtils.randomByteSource().slice(0, length);
 Payload payload = new InputStreamPayload(byteSource.openStream());
 testPut(payload, null, new ByteSourcePayload(byteSource), length, new PutOptions().multipart(true));
}