@Override public void reset() throws IOException { this.iterator = fragments.iterator(); this.currentScanner = getNextScanner(); }
Scanner scanner = new MergeScanner(conf, rootNode.getOutSchema(), outputMeta, new ArrayList<>(fragments)); scanner.init();
public MergeScanner(Configuration conf, Schema schema, TableMeta meta, List<Fragment> rawFragmentList, Schema target) throws IOException { this.schema = schema; this.meta = meta; this.target = target; this.fragments = new ArrayList<Fragment>(); long numBytes = 0; for (Fragment eachFileFragment: rawFragmentList) { long fragmentLength = TablespaceManager.guessFragmentVolume((TajoConf) conf, eachFileFragment); if (fragmentLength > 0) { numBytes += fragmentLength; fragments.add(eachFileFragment); } } // it should keep the input order. Otherwise, it causes wrong result of sort queries. this.reset(); if (currentScanner != null) { this.projectable = currentScanner.isProjectable(); this.selectable = currentScanner.isSelectable(); } tableStats = new TableStats(); tableStats.setNumBytes(numBytes); tableStats.setNumBlocks(fragments.size()); for(Column eachColumn: schema.getRootColumns()) { ColumnStats columnStats = new ColumnStats(eachColumn); tableStats.addColumnStat(columnStats); } }
private void initScanner(Schema projected) throws IOException { TableDesc table = plan.getTableDesc(); TableMeta meta = table.getMeta(); if (fragments.length > 1) { this.scanner = new MergeScanner( context.getConf(), plan.getPhysicalSchema(), meta, FragmentConvertor.convert(context.getConf(), fragments), projected ); } else { Tablespace tablespace = TablespaceManager.get(table.getUri()); this.scanner = tablespace.getScanner( meta, plan.getPhysicalSchema(), FragmentConvertor.convert(context.getConf(), fragments[0]), projected); } if (scanner.isSelectable()) { // TODO - isSelectable should be moved to FormatProperty scanner.setFilter(qual); } if (plan.hasLimit()) { scanner.setLimit(plan.getLimit()); } scanner.init(); }
public MergeScanner(Configuration conf, Schema schema, TableMeta meta, List<Fragment> rawFragmentList, Schema target) throws IOException { this.schema = schema; this.meta = meta; this.target = target; this.fragments = new ArrayList<>(); long numBytes = 0; for (Fragment eachFileFragment: rawFragmentList) { long fragmentLength = TablespaceManager.guessFragmentVolume((TajoConf) conf, eachFileFragment); if (fragmentLength > 0) { numBytes += fragmentLength; fragments.add(eachFileFragment); } } // it should keep the input order. Otherwise, it causes wrong result of sort queries. this.reset(); if (currentScanner != null) { this.projectable = currentScanner.isProjectable(); this.selectable = currentScanner.isSelectable(); } tableStats = new TableStats(); tableStats.setNumBytes(numBytes); tableStats.setNumBlocks(fragments.size()); for(Column eachColumn: schema.getRootColumns()) { ColumnStats columnStats = new ColumnStats(eachColumn); tableStats.addColumnStat(columnStats); } }
private void initScanner(Schema projected) throws IOException { TableDesc table = plan.getTableDesc(); TableMeta meta = table.getMeta(); if (fragments.length > 1) { this.scanner = new MergeScanner( context.getConf(), plan.getPhysicalSchema(), meta, FragmentConvertor.convert(context.getConf(), fragments), projected ); } else { Tablespace tablespace = TablespaceManager.get(table.getUri()); this.scanner = tablespace.getScanner( meta, plan.getPhysicalSchema(), FragmentConvertor.convert(context.getConf(), fragments[0]), projected); } if (scanner.isSelectable()) { // TODO - isSelectable should be moved to FormatProperty scanner.setFilter(qual); } if (plan.hasLimit()) { scanner.setLimit(plan.getLimit()); } scanner.init(); }
@Override public void reset() throws IOException { this.iterator = fragments.iterator(); if (currentScanner != null) { currentScanner.close(); } this.currentScanner = getNextScanner(); }
public MergeScanner(Configuration conf, Schema schema, TableMeta meta, List<FileFragment> rawFragmentList, Schema target) throws IOException { this.conf = conf; this.schema = schema; this.meta = meta; this.target = target; this.fragments = new ArrayList<FileFragment>(); long numBytes = 0; for (FileFragment eachFileFragment: rawFragmentList) { numBytes += eachFileFragment.getEndKey(); if (eachFileFragment.getEndKey() > 0) { fragments.add(eachFileFragment); } } // it should keep the input order. Otherwise, it causes wrong result of sort queries. this.reset(); if (currentScanner != null) { this.projectable = currentScanner.isProjectable(); this.selectable = currentScanner.isSelectable(); } tableStats = new TableStats(); tableStats.setNumBytes(numBytes); tableStats.setNumBlocks(fragments.size()); for(Column eachColumn: schema.getColumns()) { ColumnStats columnStats = new ColumnStats(eachColumn); tableStats.addColumnStat(columnStats); } }
Scanner scanner = new MergeScanner(conf, rootNode.getOutSchema(), outputMeta, new ArrayList<>(fragments)); scanner.init();
@Override public void reset() throws IOException { this.iterator = fragments.iterator(); if (currentScanner != null) { currentScanner.close(); } this.currentScanner = getNextScanner(); }
Scanner scanner = new MergeScanner(conf, rootNode.getOutSchema(), outputMeta, new ArrayList<>(fragments)); scanner.init(); Tuple tuple;
@Override public Tuple next() throws IOException { Tuple tuple; while (currentScanner != null) { tuple = currentScanner.next(); if (tuple != null) { return tuple; } // since read tuple is null, close the current scanner. if (currentScanner != null) { currentScanner.close(); TableStats scannerTableStsts = currentScanner.getInputStats(); if (scannerTableStsts != null) { tableStats.setReadBytes(tableStats.getReadBytes() + scannerTableStsts.getReadBytes()); tableStats.setNumRows(tableStats.getNumRows() + scannerTableStsts.getNumRows()); } currentScanner = null; } currentScanner = getNextScanner(); } return null; }
.build(); Scanner scanner = new MergeScanner(conf, schema, meta, Arrays.asList(fragment), targetSchema); assertEquals(isProjectableStorage(meta.getDataFormat()), scanner.isProjectable());
@Override public Tuple next() throws IOException { Tuple tuple; while (currentScanner != null) { tuple = currentScanner.next(); if (tuple != null) { return tuple; } // since read tuple is null, close the current scanner. if (currentScanner != null) { currentScanner.close(); TableStats scannerTableStsts = currentScanner.getInputStats(); if (scannerTableStsts != null) { tableStats.setReadBytes(tableStats.getReadBytes() + scannerTableStsts.getReadBytes()); tableStats.setNumRows(tableStats.getNumRows() + scannerTableStsts.getNumRows()); } currentScanner = null; } currentScanner = getNextScanner(); } return null; }
@Override public Tuple next() throws IOException { if (currentScanner != null) tuple = currentScanner.next(); if (tuple != null) { return tuple; } else { if (currentScanner != null) { currentScanner.close(); TableStats scannerTableStsts = currentScanner.getInputStats(); if (scannerTableStsts != null) { tableStats.setReadBytes(tableStats.getReadBytes() + scannerTableStsts.getReadBytes()); tableStats.setNumRows(tableStats.getNumRows() + scannerTableStsts.getNumRows()); } } currentScanner = getNextScanner(); if (currentScanner != null) { tuple = currentScanner.next(); } } return tuple; }