private static void skipPages(ConnectorPageSource pageSource, int skipPages) { for (int i = 0; i < skipPages; i++) { checkState(!pageSource.isFinished(), "pageSource is unexpectedly finished"); pageSource.getNextPage(); } }
@Override public Page getNextPage() { Page nextPage = delegate.getNextPage(); if (nextPage == null) { return null; } Block[] blocks = Arrays.stream(delegateFieldIndex) .mapToObj(nextPage::getBlock) .toArray(Block[]::new); return new Page(nextPage.getPositionCount(), blocks); }
public static MaterializedResult materializeSourceDataStream(ConnectorSession session, ConnectorPageSource pageSource, List<Type> types) { MaterializedResult.Builder builder = resultBuilder(session, types); while (!pageSource.isFinished()) { Page outputPage = pageSource.getNextPage(); if (outputPage == null) { break; } builder.page(outputPage); } return builder.build(); }
private static void assertPageSource(List<Type> types, Iterator<?>[] valuesByField, ConnectorPageSource pageSource) { Page page; while ((page = pageSource.getNextPage()) != null) { for (int field = 0; field < page.getChannelCount(); field++) { Block block = page.getBlock(field); for (int i = 0; i < block.getPositionCount(); i++) { assertTrue(valuesByField[field].hasNext()); Object expected = valuesByField[field].next(); Object actual = decodeObject(types.get(field), block, i); assertEquals(actual, expected); } } } }
do { getFutureValue(pageSource.isBlocked()); Page page = pageSource.getNextPage(); if (page != null && page.getPositionCount() > 0) { checkSpatialPartitioningTable(!kdbTree.isPresent(), "Expected exactly one row for table %s, but found more", name);
private static PrestoThriftPageResult getRowsInternal(ConnectorPageSource pageSource, String tableName, List<String> columnNames, @Nullable PrestoThriftId nextToken) { // very inefficient implementation as it needs to re-generate all previous results to get the next page int skipPages = nextToken != null ? Ints.fromByteArray(nextToken.getId()) : 0; skipPages(pageSource, skipPages); Page page = null; while (!pageSource.isFinished() && page == null) { page = pageSource.getNextPage(); skipPages++; } PrestoThriftId newNextToken = pageSource.isFinished() ? null : new PrestoThriftId(Ints.toByteArray(skipPages)); return toThriftPage(page, types(tableName, columnNames), newNextToken); }
@Override public Page getOutput() { Page page = pageSource.getNextPage(); if (page == null) { return null; } // update operator stats long endCompletedBytes = pageSource.getCompletedBytes(); long endReadTimeNanos = pageSource.getReadTimeNanos(); operatorContext.recordRawInputWithTiming(endCompletedBytes - completedBytes, endReadTimeNanos - readTimeNanos); operatorContext.recordProcessedInput(page.getSizeInBytes(), page.getPositionCount()); completedBytes = endCompletedBytes; readTimeNanos = endReadTimeNanos; // assure the page is in memory before handing to another operator page = page.getLoadedPage(); return page; }
Page dataPage = delegate.getNextPage(); if (dataPage == null) { return null;
@Benchmark public List<Page> read(CompressionCounter counter) throws IOException { if (!fileFormat.supports(data)) { throw new RuntimeException(fileFormat + " does not support data set " + dataSet); } List<Page> pages = new ArrayList<>(100); try (ConnectorPageSource pageSource = fileFormat.createFileFormatReader( SESSION, HDFS_ENVIRONMENT, dataFile, data.getColumnNames(), data.getColumnTypes())) { while (!pageSource.isFinished()) { Page page = pageSource.getNextPage(); if (page != null) { pages.add(page.getLoadedPage()); } } } counter.inputSize += data.getSize(); counter.outputSize += dataFile.length(); return pages; }
@Override public Page getOutput() { if (split == null) { return null; } if (source == null) { source = pageSourceProvider.createPageSource(operatorContext.getSession(), split, columns); } Page page = source.getNextPage(); if (page != null) { // assure the page is in memory before handing to another operator page = page.getLoadedPage(); // update operator stats long endCompletedBytes = source.getCompletedBytes(); long endReadTimeNanos = source.getReadTimeNanos(); operatorContext.recordRawInputWithTiming(endCompletedBytes - completedBytes, endReadTimeNanos - readTimeNanos); operatorContext.recordProcessedInput(page.getSizeInBytes(), page.getPositionCount()); completedBytes = endCompletedBytes; readTimeNanos = endReadTimeNanos; } // updating system memory usage should happen after page is loaded. systemMemoryContext.setBytes(source.getSystemMemoryUsage()); return page; } }
Page nextPage = pageSource.getNextPage(); if (nextPage != null) { pages.add(nextPage.getLoadedPage());
while (totalRows < 20000) { assertFalse(pageSource.isFinished()); Page page = pageSource.getNextPage(); assertNotNull(page); Block block = page.getBlock(1); while (totalRows < 40000) { assertFalse(pageSource.isFinished()); Page page = pageSource.getNextPage(); assertNotNull(page); Block block = page.getBlock(1); while (totalRows < NUM_ROWS) { assertFalse(pageSource.isFinished()); Page page = pageSource.getNextPage(); assertNotNull(page); Block block = page.getBlock(1); assertNull(pageSource.getNextPage()); assertTrue(pageSource.isFinished()); assertEquals(pageSource.getSystemMemoryUsage(), 0);
private Page processPageSource() { DriverYieldSignal yieldSignal = operatorContext.getDriverContext().getYieldSignal(); if (!finishing && mergingOutput.needsInput() && !yieldSignal.isSet()) { Page page = pageSource.getNextPage(); finishing = pageSource.isFinished(); pageSourceMemoryContext.setBytes(pageSource.getSystemMemoryUsage()); if (page != null) { page = recordProcessedInput(page); // update operator stats long endCompletedBytes = pageSource.getCompletedBytes(); long endReadTimeNanos = pageSource.getReadTimeNanos(); operatorContext.recordRawInputWithTiming(endCompletedBytes - completedBytes, endReadTimeNanos - readTimeNanos); completedBytes = endCompletedBytes; readTimeNanos = endReadTimeNanos; Iterator<Optional<Page>> output = pageProcessor.process(operatorContext.getSession().toConnectorSession(), yieldSignal, pageProcessorMemoryContext, page); mergingOutput.addInput(output); } if (finishing) { mergingOutput.finish(); } } Page result = mergingOutput.getOutput(); outputMemoryContext.setBytes(mergingOutput.getRetainedSizeInBytes() + pageProcessorMemoryContext.getBytes()); return result; }
public SortedRowSource(ConnectorPageSource pageSource, List<Type> columnTypes, List<Integer> sortIndexes, List<SortOrder> sortOrders) { this.pageSource = requireNonNull(pageSource, "pageSource is null"); this.columnTypes = ImmutableList.copyOf(requireNonNull(columnTypes, "columnTypes is null")); this.sortIndexes = ImmutableList.copyOf(requireNonNull(sortIndexes, "sortIndexes is null")); this.sortOrders = ImmutableList.copyOf(requireNonNull(sortOrders, "sortOrders is null")); currentPage = pageSource.getNextPage(); currentPosition = 0; }
private static void skipPages(ConnectorPageSource pageSource, int skipPages) { for (int i = 0; i < skipPages; i++) { checkState(!pageSource.isFinished(), "pageSource is unexpectedly finished"); pageSource.getNextPage(); } }
private static Page getNextPage(ConnectorPageSource pageSource) { Page page = null; while (isNullOrEmptyPage(page) && !pageSource.isFinished()) { page = pageSource.getNextPage(); if (page != null) { page = page.getLoadedPage(); } } return page; }
public static MaterializedResult materializeSourceDataStream(ConnectorSession session, ConnectorPageSource pageSource, List<Type> types) { MaterializedResult.Builder builder = resultBuilder(session, types); while (!pageSource.isFinished()) { Page outputPage = pageSource.getNextPage(); if (outputPage == null) { break; } builder.page(outputPage); } return builder.build(); }
@Override public Page getOutput() { Page page = pageSource.getNextPage(); if (page == null) { return null; } // update operator stats long endCompletedBytes = pageSource.getCompletedBytes(); long endReadTimeNanos = pageSource.getReadTimeNanos(); operatorContext.recordGeneratedInput(endCompletedBytes - completedBytes, page.getPositionCount(), endReadTimeNanos - readTimeNanos); completedBytes = endCompletedBytes; readTimeNanos = endReadTimeNanos; return page; }
private static void assertPageSource(List<Type> types, Iterator<?>[] valuesByField, ConnectorPageSource pageSource) { Page page; while ((page = pageSource.getNextPage()) != null) { for (int field = 0; field < page.getChannelCount(); field++) { Block block = page.getBlock(field); for (int i = 0; i < block.getPositionCount(); i++) { assertTrue(valuesByField[field].hasNext()); Object expected = valuesByField[field].next(); Object actual = decodeObject(types.get(field), block, i); assertEquals(actual, expected); } } } }
private static PrestoThriftPageResult getRowsInternal(ConnectorPageSource pageSource, String tableName, List<String> columnNames, @Nullable PrestoThriftId nextToken) { // very inefficient implementation as it needs to re-generate all previous results to get the next page int skipPages = nextToken != null ? Ints.fromByteArray(nextToken.getId()) : 0; skipPages(pageSource, skipPages); Page page = null; while (!pageSource.isFinished() && page == null) { page = pageSource.getNextPage(); skipPages++; } PrestoThriftId newNextToken = pageSource.isFinished() ? null : new PrestoThriftId(Ints.toByteArray(skipPages)); return toThriftPage(page, types(tableName, columnNames), newNextToken); }