private boolean tryFetchLookupSourceProvider() { if (lookupSourceProvider == null) { if (!lookupSourceProviderFuture.isDone()) { return false; } lookupSourceProvider = requireNonNull(getDone(lookupSourceProviderFuture)); statisticsCounter.updateLookupSourcePositions(lookupSourceProvider.withLease(lookupSourceLease -> lookupSourceLease.getLookupSource().getJoinPositionCount())); } return true; }
private void finishLookupSourceUnspilling() { checkState(state == State.INPUT_UNSPILLING); if (!unspillInProgress.get().isDone()) { // Pages have not be unspilled yet. return; } // Use Queue so that Pages already consumed by Index are not retained by us. Queue<Page> pages = new ArrayDeque<>(getDone(unspillInProgress.get())); long memoryRetainedByRemainingPages = pages.stream() .mapToLong(Page::getRetainedSizeInBytes) .sum(); localUserMemoryContext.setBytes(memoryRetainedByRemainingPages + index.getEstimatedSize().toBytes()); while (!pages.isEmpty()) { Page next = pages.remove(); index.addPage(next); // There is no attempt to compact index, since unspilled pages are unlikely to have blocks with retained size > logical size. memoryRetainedByRemainingPages -= next.getRetainedSizeInBytes(); localUserMemoryContext.setBytes(memoryRetainedByRemainingPages + index.getEstimatedSize().toBytes()); } LookupSourceSupplier partition = buildLookupSource(); lookupSourceChecksum.ifPresent(checksum -> checkState(partition.checksum() == checksum, "Unspilled lookupSource checksum does not match original one")); localUserMemoryContext.setBytes(partition.get().getInMemorySizeInBytes()); spilledLookupSourceHandle.setLookupSource(partition); state = State.INPUT_UNSPILLED_AND_BUILT; }
PagesSpatialIndex pagesSpatialIndex = getDone(pagesSpatialIndexFuture); DriverYieldSignal yieldSignal = operatorContext.getDriverContext().getYieldSignal(); while (probePosition < probe.getPositionCount()) {
private boolean tryFetchLookupSourceProvider() { if (lookupSourceProvider == null) { if (!lookupSourceProviderFuture.isDone()) { return false; } lookupSourceProvider = requireNonNull(getDone(lookupSourceProviderFuture)); statisticsCounter.updateLookupSourcePositions(lookupSourceProvider.withLease(lookupSourceLease -> lookupSourceLease.getLookupSource().getJoinPositionCount())); } return true; }
private boolean tryFetchLookupSourceProvider() { if (lookupSourceProvider == null) { if (!lookupSourceProviderFuture.isDone()) { return false; } lookupSourceProvider = requireNonNull(getDone(lookupSourceProviderFuture)); statisticsCounter.updateLookupSourcePositions(lookupSourceProvider.withLease(lookupSourceLease -> lookupSourceLease.getLookupSource().getJoinPositionCount())); } return true; }
@Test public void testGetDone() { assertEquals(getDone(immediateFuture("Alice")), "Alice"); assertFailure(() -> getDone(immediateFailedFuture(new IllegalStateException("some failure"))), expect(IllegalStateException.class, "some failure")); assertFailure( () -> getDone(immediateFailedFuture(new IOException("some failure"))), expect(RuntimeException.class, "java.io.IOException: some failure", expect(IOException.class, "some failure"))); assertFailure(() -> getDone(SettableFuture.create()), expect(IllegalArgumentException.class, "future not done yet")); assertFailure(() -> getDone(null), expect(NullPointerException.class, "future is null")); }
@Test public void testGetDone() { assertEquals(getDone(immediateFuture("Alice")), "Alice"); assertFailure(() -> getDone(immediateFailedFuture(new IllegalStateException("some failure"))), expect(IllegalStateException.class, "some failure")); assertFailure( () -> getDone(immediateFailedFuture(new IOException("some failure"))), expect(RuntimeException.class, "java.io.IOException: some failure", expect(IOException.class, "some failure"))); assertFailure(() -> getDone(SettableFuture.create()), expect(IllegalArgumentException.class, "future not done yet")); assertFailure(() -> getDone(null), expect(NullPointerException.class, "future is null")); }
private void finishLookupSourceUnspilling() { checkState(state == State.INPUT_UNSPILLING); if (!unspillInProgress.get().isDone()) { // Pages have not be unspilled yet. return; } // Use Queue so that Pages already consumed by Index are not retained by us. Queue<Page> pages = new ArrayDeque<>(getDone(unspillInProgress.get())); long memoryRetainedByRemainingPages = pages.stream() .mapToLong(Page::getRetainedSizeInBytes) .sum(); localUserMemoryContext.setBytes(memoryRetainedByRemainingPages + index.getEstimatedSize().toBytes()); while (!pages.isEmpty()) { Page next = pages.remove(); index.addPage(next); // There is no attempt to compact index, since unspilled pages are unlikely to have blocks with retained size > logical size. memoryRetainedByRemainingPages -= next.getRetainedSizeInBytes(); localUserMemoryContext.setBytes(memoryRetainedByRemainingPages + index.getEstimatedSize().toBytes()); } LookupSourceSupplier partition = buildLookupSource(); lookupSourceChecksum.ifPresent(checksum -> checkState(partition.checksum() == checksum, "Unspilled lookupSource checksum does not match original one")); localUserMemoryContext.setBytes(partition.get().getInMemorySizeInBytes()); spilledLookupSourceHandle.setLookupSource(partition); state = State.INPUT_UNSPILLED_AND_BUILT; }
private void finishLookupSourceUnspilling() { checkState(state == State.INPUT_UNSPILLING); if (!unspillInProgress.get().isDone()) { // Pages have not be unspilled yet. return; } // Use Queue so that Pages already consumed by Index are not retained by us. Queue<Page> pages = new ArrayDeque<>(getDone(unspillInProgress.get())); long memoryRetainedByRemainingPages = pages.stream() .mapToLong(Page::getRetainedSizeInBytes) .sum(); localUserMemoryContext.setBytes(memoryRetainedByRemainingPages + index.getEstimatedSize().toBytes()); while (!pages.isEmpty()) { Page next = pages.remove(); index.addPage(next); // There is no attempt to compact index, since unspilled pages are unlikely to have blocks with retained size > logical size. memoryRetainedByRemainingPages -= next.getRetainedSizeInBytes(); localUserMemoryContext.setBytes(memoryRetainedByRemainingPages + index.getEstimatedSize().toBytes()); } LookupSourceSupplier partition = buildLookupSource(); lookupSourceChecksum.ifPresent(checksum -> checkState(partition.checksum() == checksum, "Unspilled lookupSource checksum does not match original one")); localUserMemoryContext.setBytes(partition.get().getInMemorySizeInBytes()); spilledLookupSourceHandle.setLookupSource(partition); state = State.INPUT_UNSPILLED_AND_BUILT; }
PagesSpatialIndex pagesSpatialIndex = getDone(pagesSpatialIndexFuture); DriverYieldSignal yieldSignal = operatorContext.getDriverContext().getYieldSignal(); while (probePosition < probe.getPositionCount()) {
PagesSpatialIndex pagesSpatialIndex = getDone(pagesSpatialIndexFuture); DriverYieldSignal yieldSignal = operatorContext.getDriverContext().getYieldSignal(); while (probePosition < probe.getPositionCount()) {