private void assertAllTimeSpentInQueueing(QueryState expectedState, Consumer<QueryStateMachine> stateTransition) { TestingTicker ticker = new TestingTicker(); QueryStateMachine stateMachine = createQueryStateMachineWithTicker(ticker); ticker.increment(7, MILLISECONDS); stateTransition.accept(stateMachine); assertEquals(stateMachine.getQueryState(), expectedState); QueryStats queryStats = stateMachine.getQueryInfo(Optional.empty()).getQueryStats(); assertEquals(queryStats.getQueuedTime(), new Duration(7, MILLISECONDS)); assertEquals(queryStats.getResourceWaitingTime(), new Duration(0, MILLISECONDS)); assertEquals(queryStats.getTotalPlanningTime(), new Duration(0, MILLISECONDS)); assertEquals(queryStats.getExecutionTime(), new Duration(0, MILLISECONDS)); assertEquals(queryStats.getFinishingTime(), new Duration(0, MILLISECONDS)); }
@Test public void testIncreasingBatchSize() { int rows = 1024; // We deliberately do not set the ticker, so that the expression is always cheap and the batch size gets doubled until other limits are hit TestingTicker testingTicker = new TestingTicker(); ExpressionProfiler profiler = new ExpressionProfiler(testingTicker, SPLIT_RUN_QUANTA); PageProcessor pageProcessor = new PageProcessor( Optional.empty(), ImmutableList.of(new InputPageProjection(0, BIGINT)), OptionalInt.of(1), profiler); Slice[] slices = new Slice[rows]; Arrays.fill(slices, Slices.allocate(rows)); Page inputPage = new Page(createSlicesBlock(slices)); Iterator<Optional<Page>> output = processAndAssertRetainedPageSize(pageProcessor, inputPage); long previousPositionCount = 1; long totalPositionCount = 0; while (totalPositionCount < rows) { Optional<Page> page = output.next(); assertTrue(page.isPresent()); long positionCount = page.get().getPositionCount(); totalPositionCount += positionCount; // skip the first read && skip the last read, which can be a partial page if (positionCount > 1 && totalPositionCount != rows) { assertEquals(positionCount, previousPositionCount * 2); } previousPositionCount = positionCount; } }
@Override public ListenableFuture<?> processFor(Duration duration) { started.set(true); ticker.increment(quantaTimeMillis, MILLISECONDS); globalPhaser.arriveAndAwaitAdvance(); int phase = beginQuantaPhaser.arriveAndAwaitAdvance(); firstPhase.compareAndSet(-1, phase - 1); lastPhase.set(phase); endQuantaPhaser.arriveAndAwaitAdvance(); if (completedPhases.incrementAndGet() >= requiredPhases) { endQuantaPhaser.arriveAndDeregister(); beginQuantaPhaser.arriveAndDeregister(); globalPhaser.arriveAndDeregister(); completed.set(null); } return Futures.immediateFuture(null); }
@Test public void testDecreasingBatchSize() { int rows = 1024; // We set the expensive expression threshold to 0, so the expression is always considered expensive and the batch size gets halved until it becomes 1 TestingTicker testingTicker = new TestingTicker(); ExpressionProfiler profiler = new ExpressionProfiler(testingTicker, new Duration(0, MILLISECONDS)); PageProcessor pageProcessor = new PageProcessor( Optional.empty(), ImmutableList.of(new InputPageProjection(0, BIGINT)), OptionalInt.of(512), profiler); Slice[] slices = new Slice[rows]; Arrays.fill(slices, Slices.allocate(rows)); Page inputPage = new Page(createSlicesBlock(slices)); Iterator<Optional<Page>> output = processAndAssertRetainedPageSize(pageProcessor, inputPage); long previousPositionCount = 1; long totalPositionCount = 0; while (totalPositionCount < rows) { Optional<Page> page = output.next(); assertTrue(page.isPresent()); long positionCount = page.get().getPositionCount(); totalPositionCount += positionCount; // the batch size doesn't get smaller than 1 if (positionCount > 1 && previousPositionCount != 1) { assertEquals(positionCount, previousPositionCount / 2); } previousPositionCount = positionCount; } }
@Override public synchronized ListenableFuture<?> delay(Duration duration) { delays.add(duration); ticker.increment(duration.toMillis(), MILLISECONDS); return Futures.immediateFuture(null); } }
@Test public void testFailureInterval() { TestingTicker ticker = new TestingTicker(); ticker.increment(1, NANOSECONDS); Backoff backoff = new Backoff(1, new Duration(15, SECONDS), ticker, ImmutableList.of(new Duration(10, MILLISECONDS))); ticker.increment(10, MICROSECONDS); // verify initial state assertEquals(backoff.getFailureCount(), 0); assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 0); // first failure, should never fail assertFalse(backoff.failure()); assertEquals(backoff.getFailureCount(), 1); assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 0); ticker.increment(14, SECONDS); // second failure within the limit, should not fail assertFalse(backoff.failure()); assertEquals(backoff.getFailureCount(), 2); assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 14); ticker.increment(1, SECONDS); // final failure after the limit causes failure assertTrue(backoff.failure()); assertEquals(backoff.getFailureCount(), 3); assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 15); }
@Test public void testTaskHandle() { TestingTicker ticker = new TestingTicker(); TaskExecutor taskExecutor = new TaskExecutor(4, 8, 3, 4, ticker); taskExecutor.start(); try { TaskId taskId = new TaskId("test", 0, 0); TaskHandle taskHandle = taskExecutor.addTask(taskId, () -> 0, 10, new Duration(1, MILLISECONDS), OptionalInt.empty()); Phaser beginPhase = new Phaser(); beginPhase.register(); Phaser verificationComplete = new Phaser(); verificationComplete.register(); TestingJob driver1 = new TestingJob(ticker, new Phaser(), beginPhase, verificationComplete, 10, 0); TestingJob driver2 = new TestingJob(ticker, new Phaser(), beginPhase, verificationComplete, 10, 0); // force enqueue a split taskExecutor.enqueueSplits(taskHandle, true, ImmutableList.of(driver1)); assertEquals(taskHandle.getRunningLeafSplits(), 0); // normal enqueue a split taskExecutor.enqueueSplits(taskHandle, false, ImmutableList.of(driver2)); assertEquals(taskHandle.getRunningLeafSplits(), 1); // let the split continue to run beginPhase.arriveAndDeregister(); verificationComplete.arriveAndDeregister(); } finally { taskExecutor.stop(); } }
@Test public void timeTry() throws Exception { TimeStat stat = new TimeStat(ticker); try (BlockTimer ignored = stat.time()) { ticker.increment(10, TimeUnit.MILLISECONDS); } TimeDistribution allTime = stat.getAllTime(); assertEquals(allTime.getCount(), 1.0); assertEquals(allTime.getMin(), 0.010); assertEquals(allTime.getMax(), 0.010); }
@Test public void testMinTries() { TestingTicker ticker = new TestingTicker(); ticker.increment(1, NANOSECONDS); Backoff backoff = new Backoff(3, new Duration(1, NANOSECONDS), ticker, ImmutableList.of(new Duration(10, MILLISECONDS))); ticker.increment(10, MICROSECONDS); // verify initial state assertEquals(backoff.getFailureCount(), 0); assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 0); // first failure, should never fail assertFalse(backoff.failure()); assertEquals(backoff.getFailureCount(), 1); assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 0); ticker.increment(14, SECONDS); // second failure under min failures, should not fail assertFalse(backoff.failure()); assertEquals(backoff.getFailureCount(), 2); assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 14); ticker.increment(1, SECONDS); // last try failed assertTrue(backoff.failure()); assertEquals(backoff.getFailureCount(), 3); assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 15); }
TestingTicker ticker = new TestingTicker();
@Override public ListenableFuture<?> processFor(Duration duration) { started.set(true); ticker.increment(quantaTimeMillis, MILLISECONDS); globalPhaser.arriveAndAwaitAdvance(); int phase = beginQuantaPhaser.arriveAndAwaitAdvance(); firstPhase.compareAndSet(-1, phase - 1); lastPhase.set(phase); endQuantaPhaser.arriveAndAwaitAdvance(); if (completedPhases.incrementAndGet() >= requiredPhases) { endQuantaPhaser.arriveAndDeregister(); beginQuantaPhaser.arriveAndDeregister(); globalPhaser.arriveAndDeregister(); completed.set(null); } return Futures.immediateFuture(null); }
@Test public void testStartRequest() { TestingTicker ticker = new TestingTicker(); ticker.increment(1, NANOSECONDS); Backoff backoff = new Backoff(1, new Duration(15, SECONDS), ticker, ImmutableList.of(new Duration(10, MILLISECONDS))); ticker.increment(10, MICROSECONDS); assertFalse(backoff.failure()); assertEquals(backoff.getFailureCount(), 1); assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 0); assertEquals(backoff.getFailureRequestTimeTotal().roundTo(SECONDS), 0); ticker.increment(7, SECONDS); backoff.startRequest(); ticker.increment(7, SECONDS); assertFalse(backoff.failure()); assertEquals(backoff.getFailureCount(), 2); assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 14); // failed request took 7 seconds. assertEquals(backoff.getFailureRequestTimeTotal().roundTo(SECONDS), 7); ticker.increment(1, SECONDS); backoff.startRequest(); ticker.increment(1, SECONDS); assertTrue(backoff.failure()); assertEquals(backoff.getFailureCount(), 3); assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 16); // failed requests took 7+1 seconds. assertEquals(backoff.getFailureRequestTimeTotal().roundTo(SECONDS), 8); }
TestingTicker ticker = new TestingTicker(); TaskExecutor taskExecutor = new TaskExecutor(4, 16, 1, maxDriversPerTask, splitQueue, ticker); taskExecutor.start();
@Test public void timeTry() throws Exception { TimeStat stat = new TimeStat(ticker); try (BlockTimer ignored = stat.time()) { ticker.increment(10, TimeUnit.MILLISECONDS); } TimeDistribution allTime = stat.getAllTime(); assertEquals(allTime.getCount(), 1.0); assertEquals(allTime.getMin(), 0.010); assertEquals(allTime.getMax(), 0.010); }
@Test public void testPlanningTimeDuration() TestingTicker mockTicker = new TestingTicker(); QueryStateMachine stateMachine = createQueryStateMachineWithTicker(mockTicker); assertState(stateMachine, QUEUED); mockTicker.increment(50, MILLISECONDS); assertTrue(stateMachine.transitionToWaitingForResources()); assertState(stateMachine, WAITING_FOR_RESOURCES); mockTicker.increment(100, MILLISECONDS); assertTrue(stateMachine.transitionToPlanning()); assertState(stateMachine, PLANNING); mockTicker.increment(200, MILLISECONDS); assertTrue(stateMachine.transitionToStarting()); assertState(stateMachine, STARTING); mockTicker.increment(300, MILLISECONDS); assertTrue(stateMachine.transitionToRunning()); assertState(stateMachine, RUNNING); mockTicker.increment(400, MILLISECONDS); assertTrue(stateMachine.transitionToFinishing()); tryGetFutureValue(stateMachine.getStateChange(FINISHING), 2, SECONDS);
public MockMethodInvoker(Supplier<ListenableFuture<Object>> resultsSupplier) { this(resultsSupplier, new TestingTicker()); }
@Override public ListenableFuture<?> processFor(Duration duration) { started.set(true); ticker.increment(quantaTimeMillis, MILLISECONDS); globalPhaser.arriveAndAwaitAdvance(); int phase = beginQuantaPhaser.arriveAndAwaitAdvance(); firstPhase.compareAndSet(-1, phase - 1); lastPhase.set(phase); endQuantaPhaser.arriveAndAwaitAdvance(); if (completedPhases.incrementAndGet() >= requiredPhases) { endQuantaPhaser.arriveAndDeregister(); beginQuantaPhaser.arriveAndDeregister(); globalPhaser.arriveAndDeregister(); completed.set(null); } return Futures.immediateFuture(null); }
TestingTicker ticker = new TestingTicker(); ticker.increment(1, NANOSECONDS); assertEquals(NANOSECONDS.toSeconds(backoffDelay), 0); ticker.increment(backoffDelay, NANOSECONDS); assertEquals(NANOSECONDS.toSeconds(backoffDelay), 1); ticker.increment(backoffDelay, NANOSECONDS); assertEquals(NANOSECONDS.toSeconds(backoffDelay), 2); ticker.increment(backoffDelay, NANOSECONDS); assertEquals(NANOSECONDS.toSeconds(backoffDelay), 4); ticker.increment(backoffDelay, NANOSECONDS); assertEquals(NANOSECONDS.toSeconds(backoffDelay), 8); ticker.increment(backoffDelay, NANOSECONDS);
public MockMethodInvoker(Function<InvokeRequest, ListenableFuture<Object>> resultsSupplier) { this(resultsSupplier, new TestingTicker()); }
@Test public void time() throws Exception { TimeStat stat = new TimeStat(ticker); stat.time(() -> { ticker.increment(10, TimeUnit.MILLISECONDS); return null; }); TimeDistribution allTime = stat.getAllTime(); assertEquals(allTime.getCount(), 1.0); assertEquals(allTime.getMin(), 0.010); assertEquals(allTime.getMax(), 0.010); }