/** * Creates a new instance of the DurableLog class. * * @param config Durable Log Configuration. * @param metadata The StreamSegment Container Metadata for the container which this Durable Log is part of. * @param dataFrameLogFactory A DurableDataLogFactory which can be used to create instances of DataFrameLogs. * @param readIndex A ReadIndex which can be used to store newly processed appends. * @param executor The Executor to use for async operations. * @throws NullPointerException If any of the arguments are null. */ public DurableLog(DurableLogConfig config, UpdateableContainerMetadata metadata, DurableDataLogFactory dataFrameLogFactory, ReadIndex readIndex, ScheduledExecutorService executor) { Preconditions.checkNotNull(config, "config"); this.metadata = Preconditions.checkNotNull(metadata, "metadata"); Preconditions.checkNotNull(dataFrameLogFactory, "dataFrameLogFactory"); Preconditions.checkNotNull(readIndex, "readIndex"); this.executor = Preconditions.checkNotNull(executor, "executor"); this.durableDataLog = dataFrameLogFactory.createDurableDataLog(metadata.getContainerId()); assert this.durableDataLog != null : "dataFrameLogFactory created null durableDataLog."; this.traceObjectId = String.format("DurableLog[%s]", metadata.getContainerId()); this.inMemoryOperationLog = createInMemoryLog(); this.memoryStateUpdater = new MemoryStateUpdater(this.inMemoryOperationLog, readIndex, this::triggerTailReads); MetadataCheckpointPolicy checkpointPolicy = new MetadataCheckpointPolicy(config, this::queueMetadataCheckpoint, this.executor); this.operationProcessor = new OperationProcessor(this.metadata, this.memoryStateUpdater, this.durableDataLog, checkpointPolicy, executor); Services.onStop(this.operationProcessor, this::queueStoppedHandler, this::queueFailedHandler, this.executor); this.tailReads = new HashSet<>(); this.closed = new AtomicBoolean(); this.delayedStart = new CompletableFuture<>(); this.delayedStartRetry = Retry.withExpBackoff(config.getStartRetryDelay().toMillis(), 1, Integer.MAX_VALUE) .retryWhen(ex -> Exceptions.unwrap(ex) instanceof DataLogDisabledException); }
.retryingOn(NotDoneException.class) .throwingOn(RuntimeException.class) .runAsync(() -> controller.getCurrentSegments("test", "test")
public static RetryUnconditionally indefinitelyWithExpBackoff(String failureMessage) { Exceptions.checkNotNullOrEmpty(failureMessage, "failureMessage"); RetryWithBackoff params = new RetryWithBackoff(DEFAULT_RETRY_INIT_DELAY, DEFAULT_RETRY_MULTIPLIER, Integer.MAX_VALUE, DEFAULT_RETRY_MAX_DELAY); Consumer<Throwable> consumer = e -> { if (log.isDebugEnabled()) { log.debug(failureMessage); } else { log.warn(failureMessage); } }; return new RetryUnconditionally(consumer, params); }
.retryingOn(NotDoneException.class) .throwingOn(RuntimeException.class) .runAsync(() -> controller.getCurrentSegments("test", "test")
.retryingOn(ScaleOperationNotDoneException.class) .throwingOn(RuntimeException.class) .runAsync(() -> controller.getCurrentSegments(SCOPE, STREAM_NAME)
.retryingOn(NotDoneException.class) .throwingOn(RuntimeException.class) .runAsync(() -> controller.getCurrentSegments("test", "test")
Retry.indefinitelyWithExpBackoff(retrySchedule.getInitialMillis(), retrySchedule.getMultiplier(), retrySchedule.getMaxDelay(), t -> log.warn(writerId + " Failed to connect: ", t)) .runAsync(() -> {
this.retryConfig = Retry.withExpBackoff(config.getInitialBackoffMillis(), config.getBackoffMultiple(), config.getRetryAttempts(), config.getMaxBackoffMillis()) .retryingOn(StatusRuntimeException.class) .throwingOn(Exception.class);
.retryingOn(StoreException.DataNotFoundException.class) .throwingOn(IllegalStateException.class) .run(() -> { .retryingOn(IllegalStateException.class) .throwingOn(RuntimeException.class) .run(() -> {
.build())); Retry.withExpBackoff(500, 2, 10) .retryWhen(ex -> true) .run(() -> this.streamManager.get().createScope(SCOPE));
val finished = new CompletableFuture<Void>(); val retry = Retry.withExpBackoff(1, 2, expectedCount) .retryWhen(t -> { if (count.get() >= expectedCount) { finished.complete(null);
/** * Updates the reader group data at specified path by applying the updater method on the existing data. * It repeatedly invokes conditional update on specified path until is succeeds or max attempts (10) are exhausted. * * @param path Reader group node path. * @param updater Function to obtain the new data value from existing data value. * @throws Exception Throws exception thrown from Curator, or from application of updater method. */ private void updateReaderGroupData(String path, Function<ReaderGroupData, ReaderGroupData> updater) throws Exception { final long initialMillis = 100L; final int multiplier = 2; final int attempts = 10; final long maxDelay = 2000; Stat stat = new Stat(); Retry.withExpBackoff(initialMillis, multiplier, attempts, maxDelay) .retryingOn(KeeperException.BadVersionException.class) .throwingOn(Exception.class) .run(() -> { byte[] data = client.getData().storingStatIn(stat).forPath(path); ReaderGroupData groupData = groupDataSerializer.deserialize(ByteBuffer.wrap(data)); groupData = updater.apply(groupData); byte[] newData = groupDataSerializer.serialize(groupData).array(); client.setData() .withVersion(stat.getVersion()) .forPath(path, newData); return null; }); }
val waitOn = new CompletableFuture<Void>(); val retry = Retry.withExpBackoff(1, 2, 3) .retryWhen(t -> true); val error = new AtomicReference<Throwable>(); val p = new SequentialAsyncProcessor(
@Override public boolean write(ByteBuffer data, long expectedOffset) throws SegmentSealedException { synchronized (lock) { //Used to preserver order. long appendSequence = requestIdGenerator.get(); return retrySchedule.retryingOn(ConnectionFailedException.class) .throwingOn(SegmentSealedException.class) .run(() -> { if (client == null || client.isClosed()) { client = new RawClient(controller, connectionFactory, segmentId); long requestId = requestIdGenerator.get(); log.debug("Setting up append on segment: {}", segmentId); SetupAppend setup = new SetupAppend(requestId, writerId, segmentId.getScopedName(), delegationToken); val reply = client.sendRequest(requestId, setup); AppendSetup appendSetup = transformAppendSetup(reply.join()); if (appendSetup.getLastEventNumber() >= appendSequence) { return true; } } val request = new ConditionalAppend(writerId, appendSequence, expectedOffset, new Event(Unpooled.wrappedBuffer(data))); val reply = client.sendRequest(appendSequence, request); return transformDataAppended(reply.join()); }); } }
private CompletableFuture<Void> retryFutureInExecutor(final long delay, final int multiplier, final int attempts, final long maxDelay, final boolean success, final ScheduledExecutorService executorService) { loopCounter.set(0); accumulator.set(0); return Retry.withExpBackoff(delay, multiplier, attempts, maxDelay) .retryingOn(RetryableException.class) .throwingOn(NonretryableException.class) .runInExecutor(() -> { accumulator.getAndAdd(loopCounter.getAndIncrement()); int i = loopCounter.get(); log.debug("Loop counter = " + i); if (i % 10 == 0) { if (success) { log.debug("result = ", accumulator.get()); return; } else { throw new NonretryableException(); } } else { throw new RetryableException(); } }, executorService); }
private int retry(long delay, int multiplier, int attempts, long maxDelay, boolean success) { loopCounter.set(0); accumulator.set(0); return Retry.withExpBackoff(delay, multiplier, attempts, maxDelay) .retryingOn(RetryableException.class) .throwingOn(NonretryableException.class) .run(() -> { accumulator.getAndAdd(loopCounter.getAndIncrement()); int i = loopCounter.get(); log.debug("Loop counter = " + i); if (i % 10 == 0) { if (success) { return accumulator.get(); } else { throw new NonretryableException(); } } else { throw new RetryableException(); } }); }
/** * Invoke the simple scale up Test, produce traffic from multiple writers in parallel. * The test will periodically check if a scale event has occurred by talking to controller via * controller client. * * @throws InterruptedException if interrupted * @throws URISyntaxException If URI is invalid */ private CompletableFuture<Void> scaleUpTest() { ClientFactoryImpl clientFactory = getClientFactory(); ControllerImpl controller = getController(); final AtomicBoolean exit = new AtomicBoolean(false); createWriters(clientFactory, 6, SCOPE, SCALE_UP_STREAM_NAME); // overall wait for test to complete in 260 seconds (4.2 minutes) or scale up, whichever happens first. return Retry.withExpBackoff(10, 10, 30, Duration.ofSeconds(10).toMillis()) .retryingOn(ScaleOperationNotDoneException.class) .throwingOn(RuntimeException.class) .runAsync(() -> controller.getCurrentSegments(SCOPE, SCALE_UP_STREAM_NAME) .thenAccept(x -> { log.debug("size ==" + x.getSegments().size()); if (x.getSegments().size() == 1) { throw new ScaleOperationNotDoneException(); } else { log.info("scale up done successfully"); exit.set(true); } }), scaleExecutorService); }
/** * Invoke the scale up Test with transactional writes. Produce traffic from multiple writers in parallel. Each * writer writes using transactions. The test will periodically check if a scale event has occurred by talking to * controller via controller client. * * @throws InterruptedException if interrupted * @throws URISyntaxException If URI is invalid */ private CompletableFuture<Void> scaleUpTxnTest() { ControllerImpl controller = getController(); final AtomicBoolean exit = new AtomicBoolean(false); ClientFactoryImpl clientFactory = getClientFactory(); startWritingIntoTxn(clientFactory.createTransactionalEventWriter(SCALE_UP_TXN_STREAM_NAME, new JavaSerializer<>(), EventWriterConfig.builder().build()), exit); // overall wait for test to complete in 260 seconds (4.2 minutes) or scale up, whichever happens first. return Retry.withExpBackoff(10, 10, 30, Duration.ofSeconds(10).toMillis()) .retryingOn(ScaleOperationNotDoneException.class) .throwingOn(RuntimeException.class) .runAsync(() -> controller.getCurrentSegments(SCOPE, SCALE_UP_TXN_STREAM_NAME) .thenAccept(x -> { if (x.getSegments().size() == 1) { throw new ScaleOperationNotDoneException(); } else { log.info("txn test scale up done successfully"); exit.set(true); } }), scaleExecutorService); } }
private void checkReads(HashMap<String, ByteArrayOutputStream> segmentContents, StreamSegmentStore store) { for (Map.Entry<String, ByteArrayOutputStream> e : segmentContents.entrySet()) { String segmentName = e.getKey(); byte[] expectedData = e.getValue().toByteArray(); long segmentLength = store.getStreamSegmentInfo(segmentName, TIMEOUT).join().getLength(); Assert.assertEquals("Unexpected Read Index length for segment " + segmentName, expectedData.length, segmentLength); AtomicLong expectedCurrentOffset = new AtomicLong(0); // We retry a number of times on StreamSegmentNotExists. It is possible that waitForSegmentsInStorage may have // returned successfully because it detected the Segment was complete there, but the internal callback to the // ReadIndex (completeMerge) may not yet have been executed. The ReadIndex has a mechanism to cope with this, // but it only retries once, after a fixed time interval, which is more than generous on any system. // However, on very slow systems, it is possible that that callback may take a significant amount of time to even // begin executing, hence the trying to read data that was merged from a Transaction may result in a spurious // StreamSegmentNotExistsException. // This is gracefully handled by retries in AppendProcessor and/or Client, but in this case, we simply have to // do the retries ourselves, hoping that the callback eventually executes. Retry.withExpBackoff(100, 2, 10, TIMEOUT.toMillis() / 5) .retryWhen(ex -> Exceptions.unwrap(ex) instanceof StreamSegmentNotExistsException) .run(() -> { checkSegmentReads(segmentName, expectedCurrentOffset, segmentLength, store, expectedData); return null; }); } }
/** * Invoke the simple scale down Test, produce no into a stream. * The test will periodically check if a scale event has occurred by talking to controller via * controller client. * * @throws InterruptedException if interrupted * @throws URISyntaxException If URI is invalid */ private CompletableFuture<Void> scaleDownTest() { final ControllerImpl controller = getController(); // overall wait for test to complete in 260 seconds (4.2 minutes) or scale down, whichever happens first. return Retry.withExpBackoff(10, 10, 30, Duration.ofSeconds(10).toMillis()) .retryingOn(ScaleOperationNotDoneException.class) .throwingOn(RuntimeException.class) .runAsync(() -> controller.getCurrentSegments(SCOPE, SCALE_DOWN_STREAM_NAME) .thenAccept(x -> { if (x.getSegments().size() == 2) { throw new ScaleOperationNotDoneException(); } else { log.info("scale down done successfully"); } }), scaleExecutorService); }