protected Table getDummyTable() throws SemanticException { Path dummyPath = createDummyFile(); Table desc = new Table(DUMMY_DATABASE, DUMMY_TABLE); desc.getTTable().getSd().setLocation(dummyPath.toString()); desc.getTTable().getSd().getSerdeInfo().setSerializationLib(NullStructSerDe.class.getName()); desc.setInputFormatClass(NullRowsInputFormat.class); desc.setOutputFormatClass(HiveIgnoreKeyTextOutputFormat.class); return desc; }
/** * Remove any created directories for CTEs. */ public void removeMaterializedCTEs() { // clean CTE tables for (Table materializedTable : cteTables.values()) { Path location = materializedTable.getDataLocation(); try { FileSystem fs = location.getFileSystem(conf); boolean status = fs.delete(location, true); LOG.info("Removed " + location + " for materialized " + materializedTable.getTableName() + ", status=" + status); } catch (IOException e) { // ignore LOG.warn("Error removing " + location + " for materialized " + materializedTable.getTableName() + ": " + StringUtils.stringifyException(e)); } } cteTables.clear(); }
public ReplaceTableStageableTableMetadata(Table referenceTable) { super(referenceTable, referenceTable.getDbName(), referenceTable.getTableName(), referenceTable.getDataLocation().toString()); }
/** * @return include the db name */ public String getCompleteName() { return getCompleteName(getDbName(), getTableName()); }
public TableSpec(Table table) { tableHandle = table; tableName = table.getDbName() + "." + table.getTableName(); specType = SpecType.TABLE_ONLY; }
@Test(expected = MetastoreException.class) public void testInvalidPartitionKeyName() throws HiveException, AlreadyExistsException, IOException, MetastoreException { Table table = createTestTable(); List<Partition> partitions = hive.getPartitions(table); assertEquals(2, partitions.size()); // add a fake partition dir on fs fs = partitions.get(0).getDataLocation().getFileSystem(hive.getConf()); Path fakePart = new Path(table.getDataLocation().toString(), "fakedate=2009-01-01/fakecity=sanjose"); fs.mkdirs(fakePart); fs.deleteOnExit(fakePart); checker.checkMetastore(catName, dbName, tableName, null, new CheckResult()); }
@Test public void testDataDeletion() throws HiveException, IOException, TException { Database db = new Database(); db.setName(dbName); hive.createDatabase(db); Table table = new Table(dbName, tableName); table.setDbName(dbName); table.setInputFormatClass(TextInputFormat.class); table.setOutputFormatClass(HiveIgnoreKeyTextOutputFormat.class); table.setPartCols(partCols); hive.createTable(table); table = hive.getTable(dbName, tableName); Path fakeTable = table.getPath().getParent().suffix( Path.SEPARATOR + "faketable"); fs = fakeTable.getFileSystem(hive.getConf()); fs.mkdirs(fakeTable); fs.deleteOnExit(fakeTable); Path fakePart = new Path(table.getDataLocation().toString(), "fakepartition=fakevalue"); fs.mkdirs(fakePart); fs.deleteOnExit(fakePart); hive.dropTable(dbName, tableName, true, true); assertFalse(fs.exists(fakePart)); hive.dropDatabase(dbName); assertFalse(fs.exists(fakeTable)); }
private static class ThreadLocalHive extends ThreadLocal<Hive> { @Override protected Hive initialValue() { return null; } @Override public synchronized void set(Hive hiveObj) { Hive currentHive = this.get(); if (currentHive != hiveObj) { // Remove/close current thread-local Hive object before overwriting with new Hive object. remove(); super.set(hiveObj); } } @Override public synchronized void remove() { Hive currentHive = this.get(); if (currentHive != null) { // Close the metastore connections before removing it from thread local hiveDB. currentHive.close(false); super.remove(); } } }
@Override public void handle(Context withinContext) throws Exception { LOG.info("Processing#{} ALTER_TABLE message : {}", fromEventId(), eventMessageAsJSON); Table qlMdTableBefore = new Table(before); if (!Utils .shouldReplicate(withinContext.replicationSpec, qlMdTableBefore, withinContext.hiveConf)) { return; } if (Scenario.ALTER == scenario) { withinContext.replicationSpec.setIsMetadataOnly(true); Table qlMdTableAfter = new Table(after); Path metaDataPath = new Path(withinContext.eventRoot, EximUtil.METADATA_NAME); // If we are not dumping metadata about a table, we shouldn't be dumping basic statistics // as well, since that won't be accurate. So reset them to what they would look like for an // empty table. if (withinContext.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY)) { qlMdTableAfter.setStatsStateLikeNewTable(); } EximUtil.createExportDump( metaDataPath.getFileSystem(withinContext.hiveConf), metaDataPath, qlMdTableAfter, null, withinContext.replicationSpec, withinContext.hiveConf); } DumpMetaData dmd = withinContext.createDmd(this); dmd.setPayload(eventMessageAsJSON); dmd.write(); }
private static Path genPartPathFromTable(Table tbl, Map<String, String> partSpec, Path tblDataLocationPath) throws MetaException { Path partPath = new Path(tbl.getDataLocation(), Warehouse.makePartPath(partSpec)); return new Path(tblDataLocationPath.toUri().getScheme(), tblDataLocationPath.toUri().getAuthority(), partPath.toUri().getPath()); }
public Path getDataLocation() { if (table.isPartitioned()) { if (tPartition.getSd() == null) return null; else return new Path(tPartition.getSd().getLocation()); } else { if (table.getTTable() == null || table.getTTable().getSd() == null) return null; else return new Path(table.getTTable().getSd().getLocation()); } }
String onClauseAsString, Table targetTable, boolean onlyHaveWhenNotMatchedClause) throws SemanticException { if (!conf.getBoolVar(HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK)) { LOG.info("Merge statement cardinality violation check is disabled: " + HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK.varname); .append("\n SELECT cardinality_violation(") .append(getSimpleTableName(target)).append(".ROW__ID"); addPartitionColsToSelect(targetTable.getPartCols(), rewrittenQueryStr, target); addPartitionColsToSelect(targetTable.getPartCols(), rewrittenQueryStr, target); format.processStorageFormat("TextFile"); Table table = db.newTable(tableName); table.setSerializationLib(format.getSerde()); List<FieldSchema> fields = new ArrayList<FieldSchema>(); fields.add(new FieldSchema("val", "int", null)); table.setFields(fields); table.setDataLocation(Warehouse.getDnsPath(new Path(SessionState.get().getTempTableSpace(), tableName), conf)); table.getTTable().setTemporary(true); table.setStoredAsSubDirectories(false); table.setInputFormatClass(format.getInputFormat()); table.setOutputFormatClass(format.getOutputFormat()); db.createTable(table, true);
/** * Creates path where partitions matching prefix should lie in filesystem * @param tbl table in which partition is * @return expected location of partitions matching prefix in filesystem */ public Path createPath(Table tbl) throws HiveException { String prefixSubdir; try { prefixSubdir = Warehouse.makePartName(fields, values); } catch (MetaException e) { throw new HiveException("Unable to get partitions directories prefix", e); } Path tableDir = tbl.getDataLocation(); if (tableDir == null) { throw new HiveException("Table has no location set"); } return new Path(tableDir, prefixSubdir); } /**
private void fireInsertEvent(Table tbl, Map<String, String> partitionSpec, List<Path> newFiles) throws HiveException { if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML)) { LOG.debug("Firing dml insert event"); if (tbl.isTemporary()) { LOG.debug("Not firing dml insert event as " + tbl.getTableName() + " is temporary"); return; FileSystem fileSystem = tbl.getDataLocation().getFileSystem(conf); FireEventRequestData data = new FireEventRequestData(); InsertEventRequestData insertData = new InsertEventRequestData(); if (newFiles != null && newFiles.size() > 0) { for (Path p : newFiles) { insertData.addToFilesAdded(p.toString()); FileChecksum cksum = fileSystem.getFileChecksum(p); StringUtils.byteToHexString(cksum.getBytes(), 0, cksum.getLength()); insertData.addToFilesAddedChecksum(checksumString); } else { rqst.setDbName(tbl.getDbName()); rqst.setTableName(tbl.getTableName()); if (partitionSpec != null && partitionSpec.size() > 0) { List<String> partVals = new ArrayList<String>(partitionSpec.size()); for (FieldSchema fs : tbl.getPartitionKeys()) { partVals.add(partitionSpec.get(fs.getName())); throw new HiveException(e);
if (crtView.getReplicationSpec().allowEventReplacementInto(oldview.getParameters())){ crtView.setReplace(true); // we replace existing view. } else { oldview.setViewOriginalText(crtView.getViewOriginalText()); oldview.setViewExpandedText(crtView.getViewExpandedText()); oldview.setFields(crtView.getSchema()); if (crtView.getComment() != null) { oldview.setProperty("comment", crtView.getComment()); oldview.getTTable().getParameters().putAll(crtView.getTblProps()); oldview.setPartCols(crtView.getPartCols()); if (crtView.getInputFormat() != null) { oldview.setInputFormatClass(crtView.getInputFormat()); oldview.setOutputFormatClass(crtView.getOutputFormat()); oldview.checkValidity(null); db.alterTable(crtView.getViewName(), oldview, false, null, true); addIfAbsentByName(new WriteEntity(oldview, WriteEntity.WriteType.DDL_NO_LOCK)); if (tbl.isMaterializedView()) { CreationMetadata cm = new CreationMetadata(MetaStoreUtils.getDefaultCatalog(conf), tbl.getDbName(), tbl.getTableName(), ImmutableSet.copyOf(crtView.getTablesUsed())); cm.setValidTxnList(conf.get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY)); tbl.getTTable().setCreationMetadata(cm);
Table tempTableObj = new Table(new org.apache.hadoop.hive.metastore.api.Table(table.getTTable())); String tempTblName = table.getTableName() + tempTblNameSuffix; tempTableObj.setTableName(tempTblName); tempTableObj.setFields(table.getAllCols()); tempTableObj.setPartCols(new ArrayList<>()); tempTableObj.setParameters(new HashMap<>()); tempTableObj.setDataLocation(new Path(fromURI)); if (inputFormatClassName != null && serDeClassName != null) { try { tempTableObj.setInputFormatClass(inputFormatClassName); tempTableObj.setSerializationLib(serDeClassName); } catch (HiveException e) { throw new SemanticException("Load Data: Failed to set inputFormat or SerDe"); addPartitionColsToInsert(table.getPartCols(), rewrittenQueryStr); rewrittenQueryStr.append(" select * from "); rewrittenQueryStr.append(tempTblName); HiveConf.setVar(conf, HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
private DatasetDescriptor createSourceDataset() { try { String sourceTable = getTable().getDbName() + "." + getTable().getTableName(); DatasetDescriptor source = new DatasetDescriptor(DatasetConstants.PLATFORM_HIVE, sourceTable); Path sourcePath = getTable().getDataLocation(); log.info(String.format("[%s]Source path %s being used in conversion", this.getClass().getName(), sourcePath)); String sourceLocation = Path.getPathWithoutSchemeAndAuthority(sourcePath).toString(); FileSystem sourceFs = sourcePath.getFileSystem(new Configuration()); source.addMetadata(DatasetConstants.FS_SCHEME, sourceFs.getScheme()); source.addMetadata(DatasetConstants.FS_LOCATION, sourceLocation); return source; } catch (IOException e) { throw new RuntimeException(e); } }
throw new HiveException("Internal error - write ID not set for MM conversion"); if (tbl.getPartitionKeys().size() > 0) { PartitionIterable parts = new PartitionIterable(db, tbl, null, HiveConf.getIntVar(conf, ConfVars.METASTORE_BATCH_RETRIEVE_MAX)); Iterator<Partition> partIter = parts.iterator(); while (partIter.hasNext()) { Partition part = partIter.next(); checkMmLb(part); Path src = part.getDataLocation(), tgt = new Path(src, mmDir); srcs.add(src); tgts.add(tgt); Path src = tbl.getDataLocation(), tgt = new Path(src, mmDir); srcs.add(src); tgts.add(tgt);
public static org.apache.hadoop.hive.metastore.api.Partition createMetaPartitionObject( Table tbl, Map<String, String> partSpec, Path location) throws HiveException { List<String> pvals = new ArrayList<String>(); for (FieldSchema field : tbl.getPartCols()) { String val = partSpec.get(field.getName()); if (val == null || val.isEmpty()) { throw new HiveException("partition spec is invalid; field " + field.getName() + " does not exist or is empty"); } pvals.add(val); } org.apache.hadoop.hive.metastore.api.Partition tpart = new org.apache.hadoop.hive.metastore.api.Partition(); tpart.setDbName(tbl.getDbName()); tpart.setTableName(tbl.getTableName()); tpart.setValues(pvals); if (!tbl.isView()) { tpart.setSd(tbl.getSd().deepCopy()); tpart.getSd().setLocation((location != null) ? location.toString() : null); } return tpart; }
tbl.getTTable().setPrivilegesIsSet(false); Assert.assertTrue(ft.getTTable().isSetId()); ft.getTTable().unsetId(); ft.checkValidity(hiveConf); assertEquals("Table names didn't match for table: " + tableName, tbl .getTableName(), ft.getTableName()); assertEquals("Table owners didn't match for table: " + tableName, tbl .getOwner(), ft.getOwner()); assertEquals("Table retention didn't match for table: " + tableName, tbl.getRetention(), ft.getRetention()); assertEquals("Data location is not set correctly", wh.getDefaultTablePath(hm.getDatabase(DEFAULT_DATABASE_NAME), tableName).toString(), ft.getDataLocation().toString()); tbl.setDataLocation(ft.getDataLocation()); tbl.setCreateTime(ft.getTTable().getCreateTime()); tbl.getParameters().put(hive_metastoreConstants.DDL_TIME, ft.getParameters().get(hive_metastoreConstants.DDL_TIME)); if (tbl.getTTable().isSetWriteId() != ft.getTTable().isSetWriteId()) { ft.getTTable().setWriteId(0); tbl.getTTable().setWriteId(0); tbl.getTTable().unsetId(); assertTrue("Tables doesn't match: " + tableName + " (" + ft.getTTable() + "; " + tbl.getTTable() + ")", ft.getTTable().equals(tbl.getTTable()));