/** * closeTable * * @param table table to be closed. */ public void closeTable(Table table) { if (!tableCachingEnabled && table != null) { table.close(); } }
@Override public String toString() { StringBuilder sb = new StringBuilder("MaprDBJsonRecordReader[Table=") .append(table != null ? table.getPath() : null); if (reader != null) { sb.append(", Document ID=") .append(IdCodec.asString(reader.getId())); } sb.append(", reader=") .append(reader) .append(']'); return sb.toString(); } }
public void onRemoval(RemovalNotification<MapRDBTableCache.Key, Table> removal) { Table table = removal.getValue(); MapRDBTableCache.Key key = removal.getKey(); logger.debug("time {} closing the tablePath {} tableHandle {} index {} userName {}", System.nanoTime(), key.path == null ? "null" : key.path, table == null ? "null" : table, key.indexDesc == null ? "null" : key.indexDesc.getIndexName(), key.ugi.getUserName() == null ? "null" : key.ugi.getUserName()); table.close(); // close the table } };
@BeforeClass public static void setup_TestEncodedFieldPaths() throws Exception { try (Table table = DBTests.createOrReplaceTable(TABLE_NAME, ImmutableMap.of("codes", "codes"))) { tableCreated = true; tablePath = table.getPath().toUri().getPath(); DBTests.createIndex(TABLE_NAME, INDEX_NAME, new String[] {"age"}, new String[] {"name.last", "data.salary"}); DBTests.admin().getTableIndexes(table.getPath(), true); try (final InputStream in = TestEncodedFieldPaths.class.getResourceAsStream(JSON_FILE_URL); final DocumentStream stream = Json.newDocumentStream(in);) { table.insertOrReplace(stream); table.flush(); } // wait for the indexes to sync DBTests.waitForRowCount(table.getPath(), 5, INDEX_FLUSH_TIMEOUT); DBTests.waitForIndexFlush(table.getPath(), INDEX_FLUSH_TIMEOUT); } finally { test("ALTER SESSION SET `planner.disable_full_table_scan` = true"); } }
List<ScanRange> ranges = table.getMetaTable().getScanRanges(plugin.getRestrictedScanRangeSizeMB()); logger.debug("Num scan ranges for table {} = {}", table.getName(), ranges.size()); QueryCondition condition = r.getCondition(); List<RowkeyRange> rowkeyRanges = ((ConditionImpl)condition).getRowkeyRanges(); byte[] start = rowkeyRanges.get(0).getStartRow(); byte[] stop = rowkeyRanges.get(rowkeyRanges.size() - 1).getStopRow();
DBTests.setTableStatsSendInterval(1); if (admin.tableExists(tablePath)) { DocumentStream stream = Json.newDocumentStream(in)) { try { table.insert(stream, "rowid"); // insert a batch of document in stream } catch(Exception e) { System.out.println(stream.toString()); table.flush(); DBTests.waitForIndexFlush(table.getPath(), INDEX_FLUSH_TIMEOUT); Thread.sleep(200000);
Table createOrGetTable(String tableName, int recordNum) { if (admin.tableExists(tableName)) { return MapRDBImpl.getTable(tableName); // admin.deleteTable(tableName); } else { TableDescriptor desc = new TableDescriptorImpl(new Path(tableName)); int splits = (recordNum / SPLIT_SIZE) - (((recordNum % SPLIT_SIZE) > 1)? 0 : 1); String[] splitsStr = new String[splits]; StringBuilder strBuilder = new StringBuilder("Splits:"); for (int i = 0; i < splits; ++i) { splitsStr[i] = String.format("%d", (i+1)*SPLIT_SIZE); strBuilder.append(splitsStr[i] + ", "); } System.out.print(strBuilder.toString()); return admin.createTable(desc, splitsStr); } }
@BeforeClass public static void setup_TestSimpleJson() throws Exception { try (Table table = DBTests.createOrReplaceTable(TABLE_NAME); InputStream in = MaprDBTestsSuite.getJsonStream(JSON_FILE_URL); DocumentStream stream = Json.newDocumentStream(in)) { tableCreated = true; tablePath = table.getPath().toUri().getPath(); for (Document document : stream) { table.insert(document, "business_id"); } table.flush(); } }
/** * Get the estimated average rowsize. DO NOT call this API directly. * Call the stats API instead which modifies the counts based on preference options. * @param index, to use for generating the estimate * @return row count post filtering */ public MapRDBStatisticsPayload getAverageRowSizeStats(IndexDescriptor index) { IndexDesc indexDesc = null; double avgRowSize = AVG_ROWSIZE_UNKNOWN; if (index != null) { indexDesc = (IndexDesc)((MapRDBIndexDescriptor)index).getOriginalDesc(); } // If no index is specified, get it from the primary table if (indexDesc == null && scanSpec.isSecondaryIndex()) { throw new UnsupportedOperationException("getAverageRowSizeStats should be invoked on primary table"); } // Get the index table or primary table and use the DB API to get the estimated number of rows. For size estimates, // we assume that all the columns would be read from the disk. final Table table = this.formatPlugin.getJsonTableCache().getTable(scanSpec.getTableName(), indexDesc, getUserName()); if (table != null) { final MetaTable metaTable = table.getMetaTable(); if (metaTable != null) { avgRowSize = metaTable.getAverageRowSize(); } } logger.debug("index_plan_info: getEstimatedRowCount obtained from DB Client for {}: indexName: {}, indexInfo: {}, " + "avgRowSize: {}, estimatedSize {}", this, (indexDesc == null ? "null" : indexDesc.getIndexName()), (indexDesc == null ? "null" : indexDesc.getIndexInfo()), avgRowSize); return new MapRDBStatisticsPayload(ROWCOUNT_UNKNOWN, ROWCOUNT_UNKNOWN, avgRowSize); }
private void createIndex(Table table, String[] indexDef) throws Exception { if (indexDef == null) { // don't create index here. indexes may have been created return; } for (int i = 0; i < indexDef.length / 3; ++i) { String indexCmd = String.format("maprcli table index add" + " -path " + table.getPath() + " -index %s" + " -indexedfields '%s'" + ((indexDef[3 * i + 2].length()==0)?"":" -includedfields '%s'") + ((indexDef[3 * i].startsWith("hash"))? " -hashed true" : ""), indexDefInCommand(indexDef[3 * i]), // index name indexDefInCommand(indexDef[3 * i + 1]), // indexedfields indexDefInCommand(indexDef[3 * i + 2])); // includedfields System.out.println(indexCmd); TestCluster.runCommand(indexCmd); DBTests.admin().getTableIndexes(table.getPath(), true); } }
private void init() { try { // Get the fullTableRowCount only once i.e. if not already obtained before. if (fullTableRowCount == 0) { final Table t = this.formatPlugin.getJsonTableCache().getTable( scanSpec.getTableName(), scanSpec.getIndexDesc(), getUserName()); final MetaTable metaTable = t.getMetaTable(); // For condition null, we get full table stats. com.mapr.db.scan.ScanStats stats = metaTable.getScanStats(); fullTableRowCount = stats.getEstimatedNumRows(); fullTableEstimatedSize = stats.getEstimatedSize(); // MapRDB client can return invalid rowCount i.e. 0, especially right after table // creation. It takes 15 minutes before table stats are obtained and cached in client. // If we get 0 rowCount, fallback to getting rowCount using old admin API. if (fullTableRowCount == 0) { PluginCost pluginCostModel = formatPlugin.getPluginCostModel(); final int avgColumnSize = pluginCostModel.getAverageColumnSize(this); final int numColumns = (columns == null || columns.isEmpty() || Utilities.isStarQuery(columns)) ? STAR_COLS : columns.size(); MapRDBTableStats tableStats = new MapRDBTableStats(formatPlugin.getFsConf(), scanSpec.getTableName()); fullTableRowCount = tableStats.getNumRows(); fullTableEstimatedSize = fullTableRowCount * numColumns * avgColumnSize; } } } catch (Exception e) { throw new DrillRuntimeException("Error getting region info for table: " + scanSpec.getTableName() + (scanSpec.getIndexDesc() == null ? "" : (", index: " + scanSpec.getIndexName())), e); } }
@Override public void setup(OperatorContext context, OutputMutator output) throws ExecutionSetupException { this.vectorWriter = new VectorContainerWriter(output, unionEnabled); this.vectorWriterMutator = output; this.operatorContext = context; try { table.setOption(TableOption.EXCLUDEID, !includeId); documentStream = table.find(condition, scannedFields); documentIterator = documentStream.iterator(); setupWriter(); } catch (DBException ex) { throw new ExecutionSetupException(ex); } }
Admin admin = MaprDBTestsSuite.getAdmin(); if (admin != null) { if (admin.tableExists(PRIMARY_TABLE_NAME)) { admin.deleteTable(PRIMARY_TABLE_NAME);
void init(String host, String port) { try { admin = MapRDB.newAdmin(); initialized = true; gen = new LargeTableGen(admin); } catch (Exception e) { System.out.println("Connection to HBase threw" + e.getMessage()); } } }
@AfterClass public static void cleanupTests() throws Exception { synchronized (MaprDBTestsSuite.class) { if (initCount.decrementAndGet() == 0) { HBaseTestsSuite.tearDownCluster(); admin.close(); } } }
try { Set<DrillIndexDescriptor> idxSet = new HashSet<>(); Collection<IndexDesc> indexes = admin().getTableIndexes(new Path(tableName)); if (indexes.size() == 0 ) { logger.error("No index returned from Admin.getTableIndexes for table {}", tableName);
@SuppressWarnings("deprecation") private Admin admin() { assert getOriginalScan() instanceof MapRDBGroupScan; final MapRDBGroupScan dbGroupScan = (MapRDBGroupScan) getOriginalScan(); final UserGroupInformation currentUser = ImpersonationUtil.createProxyUgi(dbGroupScan.getUserName()); final Configuration conf = dbGroupScan.getFormatPlugin().getFsConf(); final Admin admin; try { admin = currentUser.doAs((PrivilegedExceptionAction<Admin>) () -> MapRDB.getAdmin(conf)); } catch (Exception e) { throw new DrillRuntimeException("Failed to get Admin instance for user: " + currentUser.getUserName(), e); } return admin; } }
@AfterClass public static void cleanupTableIndexes() throws Exception { Admin admin = MaprDBTestsSuite.getAdmin(); if (admin != null) { if (admin.tableExists(PRIMARY_TABLE_NAME)) { // admin.deleteTable(PRIMARY_TABLE_NAME); } } test(defaultnonCoveringSelectivityThreshold); }
@Override public void close() { if (documentStream != null) { documentStream.close(); } if (table != null) { table.close(); } }
@BeforeClass public static void setupTests() throws Exception { if (initCount.get() == 0) { synchronized (MaprDBTestsSuite.class) { if (initCount.get() == 0) { HBaseTestsSuite.configure(false /*manageHBaseCluster*/, true /*createTables*/); HBaseTestsSuite.initCluster(); // Sleep to allow table data to be flushed to tables. // Without this, the row count stats to return 0, // causing the planner to reject optimized plans. Thread.sleep(5000); admin = MapRDB.newAdmin(); conf = HBaseTestsSuite.getConf(); initCount.incrementAndGet(); // must increment while inside the synchronized block return; } } } initCount.incrementAndGet(); return; }