How to use

io.pravega.segmentstore.storage.rolling.RollingStorage

constructor

@Override
public Storage createStorageAdapter() {
  return new AsyncStorageWrapper(new RollingStorage(this.baseStorage), this.executor);
}

  @Override
  public Storage createStorageAdapter() {
    HDFSStorage s = new HDFSStorage(this.config);
    return new AsyncStorageWrapper(new RollingStorage(s), this.executor);
  }
}

  @Override
  public Storage createStorageAdapter() {
    FileSystemStorage s = new FileSystemStorage(this.config);
    return new AsyncStorageWrapper(new RollingStorage(s), this.executor);
  }
}

@Override
public Storage createStorageAdapter() {
  return new WatchableAsyncStorageWrapper(new RollingStorage(this.baseStorage), this.executor);
}

  @Override
  public Storage createStorageAdapter() {
    S3Config s3Config = new S3Config(config.getUrl())
        .withIdentity(config.getAccessKey())
        .withSecretKey(config.getSecretKey())
        .withNamespace(config.getNamespace());

    S3JerseyClient client = new S3JerseyClient(s3Config);
    ExtendedS3Storage s = new ExtendedS3Storage(client, this.config);
    return new AsyncStorageWrapper(new RollingStorage(s), this.executor);
  }
}

@Override
protected Storage createStorage() {
  return new AsyncStorageWrapper(new RollingStorage(new InMemoryStorage(), DEFAULT_ROLLING_POLICY), executorService());
}

protected Storage wrap(SyncStorage storage) {
  return new AsyncStorageWrapper(new RollingStorage(storage, new SegmentRollingPolicy(DEFAULT_ROLLING_SIZE)), executorService());
}

/**
 * Tests the ability to auto-refresh a Write Handle upon offset disagreement.
 */
@Test
public void testRefreshHandleBadOffset() throws Exception {
  // Write small and large writes, alternatively.
  @Cleanup
  val baseStorage = new InMemoryStorage();
  @Cleanup
  val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY);
  s.initialize(1);
  s.create(SEGMENT_NAME);
  val h1 = s.openWrite(SEGMENT_NAME);
  val h2 = s.openWrite(SEGMENT_NAME); // Open now, before writing, so we force a refresh.
  byte[] data = "data".getBytes();
  s.write(h1, 0, new ByteArrayInputStream(data), data.length);
  s.write(h2, data.length, new ByteArrayInputStream(data), data.length);
  // Check that no file has exceeded its maximum length.
  byte[] expectedData = new byte[data.length * 2];
  System.arraycopy(data, 0, expectedData, 0, data.length);
  System.arraycopy(data, 0, expectedData, data.length, data.length);
  checkWrittenData(expectedData, h2, s);
}

  @Override
  public Storage createStorageAdapter() {
    URI uri = URI.create(endpoint);
    S3Config s3Config = new S3Config(uri);
    s3Config = s3Config.withIdentity(config.getAccessKey()).withSecretKey(config.getSecretKey())
        .withRetryEnabled(false)
        .withInitialRetryDelay(1)
        .withProperty("com.sun.jersey.client.property.connectTimeout", 100);
    S3JerseyClient client = new S3ClientWrapper(s3Config, filesystemS3);
    return new AsyncStorageWrapper(new RollingStorage(new ExtendedS3Storage(client, config)), this.storageExecutor);
  }
}

TestContext(AttributeIndexConfig config, CachePolicy cachePolicy) {
  this.memoryStorage = new InMemoryStorage();
  this.memoryStorage.initialize(1);
  this.storage = new TestContext.TestStorage(new RollingStorage(this.memoryStorage, config.getAttributeSegmentRollingPolicy()), executorService());
  this.containerMetadata = new MetadataBuilder(CONTAINER_ID).build();
  this.cacheFactory = new InMemoryCacheFactory();
  this.cacheManager = new TestCacheManager(cachePolicy, executorService());
  val factory = new ContainerAttributeIndexFactoryImpl(config, this.cacheFactory, this.cacheManager, executorService());
  this.index = factory.createContainerAttributeIndex(this.containerMetadata, this.storage);
}

/**
 * Tests the ability to truncate Sealed Segments.
 */
@Test
public void testTruncateSealed() throws Exception {
  // Write small and large writes, alternatively.
  @Cleanup
  val baseStorage = new TestStorage();
  @Cleanup
  val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY);
  s.initialize(1);
  // Create a Segment, write some data, then seal it.
  s.create(SEGMENT_NAME);
  val appendHandle = (RollingSegmentHandle) s.openWrite(SEGMENT_NAME);
  val writeStream = new ByteArrayOutputStream();
  populate(s, appendHandle, writeStream);
  s.seal(appendHandle);
  byte[] writtenData = writeStream.toByteArray();
  val truncateHandle = (RollingSegmentHandle) s.openWrite(SEGMENT_NAME);
  Assert.assertTrue("Handle not read-only after sealing.", truncateHandle.isReadOnly());
  Assert.assertTrue("Handle not sealed after sealing.", truncateHandle.isSealed());
  // Test that truncate works in this scenario.
  testProgressiveTruncate(truncateHandle, truncateHandle, writtenData, s, baseStorage);
}

/**
 * Tests the ability to concat using the header file for those cases when native concat cannot be used because the
 * source Segment has multiple SegmentChunks.
 */
@Test
public void testConcatHeaderMultiFile() throws Exception {
  final int initialTargetLength = (int) DEFAULT_ROLLING_POLICY.getMaxLength() / 2;
  final String sourceSegmentName = "SourceSegment";
  @Cleanup
  val baseStorage = new InMemoryStorage();
  @Cleanup
  val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY);
  s.initialize(1);
  // Create a Target Segment and a Source Segment and write some data to them.
  s.create(SEGMENT_NAME);
  val targetHandle = (RollingSegmentHandle) s.openWrite(SEGMENT_NAME);
  val writeStream = new ByteArrayOutputStream();
  populate(s, targetHandle, 1, initialTargetLength, initialTargetLength, writeStream);
  s.create(sourceSegmentName);
  val sourceHandle = (RollingSegmentHandle) s.openWrite(sourceSegmentName);
  populate(s, sourceHandle, APPENDS_PER_SEGMENT, initialTargetLength, initialTargetLength, writeStream);
  s.seal(sourceHandle);
  // Concat and verify the handle has been updated accordingly.
  s.concat(targetHandle, initialTargetLength, sourceSegmentName);
  checkConcatResult(s, targetHandle, sourceSegmentName, 1 + sourceHandle.chunks().size(), initialTargetLength + (int) sourceHandle.length());
  checkWrittenData(writeStream.toByteArray(), s.openRead(SEGMENT_NAME), s);
}

val baseStorage = new TestStorage();
@Cleanup
val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY);
s.initialize(1);
s.create(SEGMENT_NAME);

/**
 * Tests the ability to concat using the header file for those cases when native concat cannot be used because the
 * source Segment has a single SegmentChunk, but it's too large to fit into the Target's active SegmentChunk.
 */
@Test
public void testConcatHeaderSingleFile() throws Exception {
  final int initialTargetLength = (int) DEFAULT_ROLLING_POLICY.getMaxLength() / 2;
  final int bigSourceLength = (int) DEFAULT_ROLLING_POLICY.getMaxLength() - initialTargetLength + 1;
  final String sourceSegmentName = "SourceSegment";
  @Cleanup
  val baseStorage = new InMemoryStorage();
  @Cleanup
  val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY);
  s.initialize(1);
  // Create a Target Segment and a Source Segment and write some data to them.
  s.create(SEGMENT_NAME);
  val targetHandle = (RollingSegmentHandle) s.openWrite(SEGMENT_NAME);
  val writeStream = new ByteArrayOutputStream();
  populate(s, targetHandle, 1, initialTargetLength, initialTargetLength, writeStream);
  s.create(sourceSegmentName);
  val sourceHandle = (RollingSegmentHandle) s.openWrite(sourceSegmentName);
  populate(s, sourceHandle, 1, bigSourceLength, bigSourceLength, writeStream);
  s.seal(sourceHandle);
  // Concat and verify the handle has been updated accordingly.
  s.concat(targetHandle, initialTargetLength, sourceSegmentName);
  checkConcatResult(s, targetHandle, sourceSegmentName, 2, initialTargetLength + bigSourceLength);
  checkWrittenData(writeStream.toByteArray(), s.openRead(SEGMENT_NAME), s);
}

val baseStorage = new TestStorage();
@Cleanup
val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY);
s.initialize(1);
s.create(SEGMENT_NAME);

/**
 * Tests the ability to use native concat for those cases when it's appropriate.
 */
@Test
public void testConcatNatively() throws Exception {
  final int initialTargetLength = (int) DEFAULT_ROLLING_POLICY.getMaxLength() / 2;
  final int initialSourceLength = (int) DEFAULT_ROLLING_POLICY.getMaxLength() - initialTargetLength;
  final String sourceSegmentName = "SourceSegment";
  @Cleanup
  val baseStorage = new InMemoryStorage();
  @Cleanup
  val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY);
  s.initialize(1);
  // Create a target Segment and write a little data to it.
  s.create(SEGMENT_NAME);
  val targetHandle = (RollingSegmentHandle) s.openWrite(SEGMENT_NAME);
  val writeStream = new ByteArrayOutputStream();
  populate(s, targetHandle, 1, initialTargetLength, initialTargetLength, writeStream);
  // Create a source Segment and write a little data to it, making sure it is small enough to fit into the target
  // when we need to concat.
  s.create(sourceSegmentName);
  val sourceHandle = (RollingSegmentHandle) s.openWrite(sourceSegmentName);
  populate(s, sourceHandle, 1, initialSourceLength, initialSourceLength, writeStream);
  s.seal(sourceHandle);
  // Concat and verify the handle has been updated accordingly.
  s.concat(targetHandle, initialTargetLength, sourceSegmentName);
  checkConcatResult(s, targetHandle, sourceSegmentName, 1, initialTargetLength + initialSourceLength);
  checkWrittenData(writeStream.toByteArray(), s.openRead(SEGMENT_NAME), s);
}

val baseStorage = new InMemoryStorage();
@Cleanup
val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY);
s.initialize(1);
s.create(SEGMENT_NAME);

val baseStorage = new TestStorage();
@Cleanup
val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY);
s.initialize(1);

val baseStorage = new TestStorage();
@Cleanup
val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY);
s.initialize(1);

/**
 * Tests the case when Create was interrupted after it created the Header file but before populating it.
 */
@Test
public void testCreateRecovery() throws Exception {
  @Cleanup
  val baseStorage = new TestStorage();
  @Cleanup
  val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY);
  s.initialize(1);
  // Create an empty header file. This simulates a create() operation that failed mid-way.
  baseStorage.create(StreamSegmentNameUtils.getHeaderSegmentName(SEGMENT_NAME));
  Assert.assertFalse("Not expecting Segment to exist.", s.exists(SEGMENT_NAME));
  AssertExtensions.assertThrows(
      "Not expecting Segment to exist (getStreamSegmentInfo).",
      () -> s.getStreamSegmentInfo(SEGMENT_NAME),
      ex -> ex instanceof StreamSegmentNotExistsException);
  AssertExtensions.assertThrows(
      "Not expecting Segment to exist (openHandle).",
      () -> s.openRead(SEGMENT_NAME),
      ex -> ex instanceof StreamSegmentNotExistsException);
  // Retry the operation and verify everything is in place.
  s.create(SEGMENT_NAME);
  val si = s.getStreamSegmentInfo(SEGMENT_NAME);
  Assert.assertEquals("Expected the Segment to have been created.", 0, si.getLength());
}