@Override public void load(LazyBlock lazyBlock) { if (block == null) { return; } lazyBlock.setBlock(coercer.apply(block.getLoadedBlock())); // clear reference to loader to free resources, since load was successful block = null; } }
private static boolean isNotLoadedLazyBlock(Block block) { return (block instanceof LazyBlock) && !((LazyBlock) block).isLoaded(); }
@Override public String getEncodingName() { assureLoaded(); return LazyBlockEncoding.NAME; }
private Page recordProcessedInput(Page page) { operatorContext.recordProcessedInput(0, page.getPositionCount()); // account processed bytes from lazy blocks only when they are loaded Block[] blocks = new Block[page.getChannelCount()]; for (int i = 0; i < page.getChannelCount(); ++i) { Block block = page.getBlock(i); if (block instanceof LazyBlock) { LazyBlock delegateLazyBlock = (LazyBlock) block; blocks[i] = new LazyBlock(page.getPositionCount(), lazyBlock -> { Block loadedBlock = delegateLazyBlock.getLoadedBlock(); operatorContext.recordProcessedInput(loadedBlock.getSizeInBytes(), 0L); lazyBlock.setBlock(loadedBlock); }); } else { operatorContext.recordProcessedInput(block.getSizeInBytes(), 0L); blocks[i] = block; } } return new Page(page.getPositionCount(), blocks); }
private Block createBlock(int currentPageSize, int fieldId) { int hiveColumnIndex = hiveColumnIndexes[fieldId]; return new LazyBlock( currentPageSize, new RcFileBlockLoader(hiveColumnIndex)); }
for (int i = 0; i < adaptedBlocks.length; i++) { Block block = dataPage.getBlock(i); if (block instanceof LazyBlock && !((LazyBlock) block).isLoaded()) { adaptedBlocks[i] = new LazyBlock(rowsToKeep.size(), new RowFilterLazyBlockLoader(dataPage.getBlock(i), rowsToKeep)); Block block = dataPage.getBlock(columnMapping.getIndex()); if (coercers[fieldId] != null) { block = new LazyBlock(batchSize, new CoercionLazyBlockLoader(block, coercers[fieldId]));
private Page recordProcessedInput(Page page) { operatorContext.recordProcessedInput(0, page.getPositionCount()); // account processed bytes from lazy blocks only when they are loaded Block[] blocks = new Block[page.getChannelCount()]; for (int i = 0; i < page.getChannelCount(); ++i) { Block block = page.getBlock(i); if (block instanceof LazyBlock) { LazyBlock delegateLazyBlock = (LazyBlock) block; blocks[i] = new LazyBlock(page.getPositionCount(), lazyBlock -> { Block loadedBlock = delegateLazyBlock.getLoadedBlock(); operatorContext.recordProcessedInput(loadedBlock.getSizeInBytes(), 0L); lazyBlock.setBlock(loadedBlock); }); } else { operatorContext.recordProcessedInput(block.getSizeInBytes(), 0L); blocks[i] = block; } } return new Page(page.getPositionCount(), blocks); }
@Test public void testSelectNoneFilterLazyLoad() { PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectNoneFilter()), ImmutableList.of(new InputPageProjection(1, BIGINT))); // if channel 1 is loaded, test will fail Page inputPage = new Page(createLongSequenceBlock(0, 100), new LazyBlock(100, lazyBlock -> { throw new AssertionError("Lazy block should not be loaded"); })); LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()); Iterator<Optional<Page>> output = pageProcessor.process(SESSION, new DriverYieldSignal(), memoryContext, inputPage); assertEquals(memoryContext.getBytes(), 0); List<Optional<Page>> outputPages = ImmutableList.copyOf(output); assertEquals(outputPages.size(), 0); }
@Test public void testProjectLazyLoad() { PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectAllFilter()), ImmutableList.of(new LazyPagePageProjection()), OptionalInt.of(MAX_BATCH_SIZE)); // if channel 1 is loaded, test will fail Page inputPage = new Page(createLongSequenceBlock(0, 100), new LazyBlock(100, lazyBlock -> { throw new AssertionError("Lazy block should not be loaded"); })); LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()); Iterator<Optional<Page>> output = pageProcessor.process(SESSION, new DriverYieldSignal(), memoryContext, inputPage); List<Optional<Page>> outputPages = ImmutableList.copyOf(output); assertEquals(outputPages.size(), 1); assertPageEquals(ImmutableList.of(BIGINT), outputPages.get(0).orElse(null), new Page(createLongSequenceBlock(0, 100))); }
@Override public final void load(LazyBlock lazyBlock) { if (loaded) { return; } checkState(batchId == expectedBatchId); try { Block block = recordReader.readBlock(type, columnIndex); lazyBlock.setBlock(block); } catch (IOException e) { throw new PrestoException(RAPTOR_ERROR, e); } loaded = true; } }
@Override public Block getLoadedBlock() { assureLoaded(); return block; }
private static boolean isNotLoadedLazyBlock(Block block) { return (block instanceof LazyBlock) && !((LazyBlock) block).isLoaded(); }
@Test public void testSelectNoneFilterLazyLoad() { PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectNoneFilter()), ImmutableList.of(new InputPageProjection(1, BIGINT))); // if channel 1 is loaded, test will fail Page inputPage = new Page(createLongSequenceBlock(0, 100), new LazyBlock(100, lazyBlock -> { throw new AssertionError("Lazy block should not be loaded"); })); LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()); Iterator<Optional<Page>> output = pageProcessor.process(SESSION, new DriverYieldSignal(), memoryContext, inputPage); assertEquals(memoryContext.getBytes(), 0); List<Optional<Page>> outputPages = ImmutableList.copyOf(output); assertEquals(outputPages.size(), 0); }
@Override public void load(LazyBlock lazyBlock) { if (block == null) { return; } lazyBlock.setBlock(block.getPositions(rowsToKeep.elements(), 0, rowsToKeep.size())); // clear reference to loader to free resources, since load was successful block = null; } }
@Override public byte getByte(int position, int offset) { assureLoaded(); return block.getByte(position, offset); }
@Test public void testProjectLazyLoad() { PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectAllFilter()), ImmutableList.of(new LazyPagePageProjection()), OptionalInt.of(MAX_BATCH_SIZE)); // if channel 1 is loaded, test will fail Page inputPage = new Page(createLongSequenceBlock(0, 100), new LazyBlock(100, lazyBlock -> { throw new AssertionError("Lazy block should not be loaded"); })); LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()); Iterator<Optional<Page>> output = pageProcessor.process(SESSION, new DriverYieldSignal(), memoryContext, inputPage); List<Optional<Page>> outputPages = ImmutableList.copyOf(output); assertEquals(outputPages.size(), 1); assertPageEquals(ImmutableList.of(BIGINT), outputPages.get(0).orElse(null), new Page(createLongSequenceBlock(0, 100))); }