@Test public void testRenameTable() throws Exception { HCatClient client = HCatClient.create(new Configuration(hcatConf)); String tableName = "temptable"; String newName = "mytable"; client.dropTable(null, tableName, true); client.dropTable(null, newName, true); ArrayList<HCatFieldSchema> cols = new ArrayList<HCatFieldSchema>(); cols.add(new HCatFieldSchema("id", Type.INT, "id columns")); cols.add(new HCatFieldSchema("value", Type.STRING, "id columns")); HCatCreateTableDesc tableDesc = HCatCreateTableDesc .create(null, tableName, cols).fileFormat("rcfile").build(); client.createTable(tableDesc); client.renameTable(null, tableName, newName); try { client.getTable(null, tableName); } catch (HCatException exp) { assertTrue("Unexpected exception message: " + exp.getMessage(), exp.getMessage().contains("NoSuchObjectException while fetching table")); } HCatTable newTable = client.getTable(null, newName); assertTrue(newTable != null); assertTrue(newTable.getTableName().equals(newName)); client.close(); }
@Test public void testPartitionsHCatClientImpl() throws Exception { HCatClient client = HCatClient.create(new Configuration(hcatConf)); String dbName = "ptnDB"; String tableName = "pageView"; client.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); client.createDatabase(dbDesc); ArrayList<HCatFieldSchema> cols = new ArrayList<HCatFieldSchema>(); cols.add(new HCatFieldSchema("userid", Type.INT, "id columns")); .fileFormat("sequenceFile"); HCatCreateTableDesc tableDesc = HCatCreateTableDesc.create(table, false).build(); client.createTable(tableDesc); table = client.getTable(dbName, tableName); client.addPartition(addPtn); client.addPartition(addPtn2); client.addPartition(addPtn3); List<HCatPartition> ptnList = client.listPartitionsByFilter(dbName, tableName, null); assertTrue(ptnList.size() == 3); HCatPartition ptn = client.getPartition(dbName, tableName, firstPtn); assertTrue(ptn != null);
private void init( HCatClient hcatClient, long eventFrom, int maxEvents, IMetaStoreClient.NotificationFilter filter) throws HCatException { // Simple implementation for now, this will later expand to do DAG evaluation. this.hcatClient = hcatClient; this.filter = filter; this.pos = eventFrom; if (maxEvents < 1){ // 0 or -1 implies fetch everything this.maxEvents = Integer.MAX_VALUE; } else { this.maxEvents = maxEvents; } batchSize = Integer.parseInt( hcatClient.getConfVal(HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX.varname,"50")); this.eventCount = 0; this.maxPos = hcatClient.getCurrentNotificationEventId(); }
private long renewToken(Token token, String metaStoreURI, String hiveMetaStorePrincipal) { HCatClient hcatClient = null; if (UserGroupInformation.isSecurityEnabled()) { try { String tokenStr = token.encodeToUrlString(); HiveConf hcatConf = createHiveConf(metaStoreURI, hiveMetaStorePrincipal); LOG.debug("renewing delegation tokens for principal={}", hiveMetaStorePrincipal); hcatClient = HCatClient.create(hcatConf); Long expiryTime = hcatClient.renewDelegationToken(tokenStr); LOG.info("Renewed delegation token. new expiryTime={}", expiryTime); return expiryTime; } catch (Exception ex) { throw new RuntimeException("Failed to renew delegation tokens.", ex); } finally { if (hcatClient != null) try { hcatClient.close(); } catch (HCatException e) { LOG.error(" Exception", e); } } } else { throw new RuntimeException("Security is not enabled for Hadoop"); } }
private Token<DelegationTokenIdentifier> getDelegationToken(HiveConf hcatConf, String metaStoreServicePrincipal, String topologySubmitterUser) throws IOException { LOG.info("Creating delegation tokens for principal={}", metaStoreServicePrincipal); HCatClient hcatClient = null; try { hcatClient = HCatClient.create(hcatConf); String delegationToken = hcatClient.getDelegationToken(topologySubmitterUser, metaStoreServicePrincipal); Token<DelegationTokenIdentifier> delegationTokenId = new Token<DelegationTokenIdentifier>(); delegationTokenId.decodeFromUrlString(delegationToken); DelegationTokenIdentifier d = new DelegationTokenIdentifier(); d.readFields(new DataInputStream(new ByteArrayInputStream( delegationTokenId.getIdentifier()))); LOG.info("Created Delegation Token for : " + d.getUser()); return delegationTokenId; } finally { if (hcatClient != null) hcatClient.close(); } }
cfg.set(HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX.varname,"10"); // set really low batch size to ensure batching cfg.set(HiveConf.ConfVars.HIVE_REPL_TASK_FACTORY.varname, EximReplicationTaskFactory.class.getName()); HCatClient sourceMetastore = HCatClient.create(cfg); long baseId = sourceMetastore.getCurrentNotificationEventId(); sourceMetastore.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); sourceMetastore.createDatabase(HCatCreateDBDesc.create(dbName).ifNotExists(false).build()); List<HCatFieldSchema> cols1 = HCatSchemaUtils.getHCatSchema("a:int,b:string").getFields(); HCatTable table1 = (new HCatTable(dbName, tblName1)).cols(cols1); sourceMetastore.createTable(HCatCreateTableDesc.create(table1).build()); List<HCatFieldSchema> pcols2 = HCatSchemaUtils.getHCatSchema("b:string").getFields(); HCatTable table2 = (new HCatTable(dbName, tblName2)).cols(cols2).partCols(pcols2); sourceMetastore.createTable(HCatCreateTableDesc.create(table2).build()); HCatTable table2Created = sourceMetastore.getTable(dbName,tblName2); HCatPartition ptn1 = (new HCatPartition(table2Created, ptnDesc1, makePartLocation(table2Created,ptnDesc1))); sourceMetastore.addPartition(HCatAddPartitionDesc.create(ptn1).build()); HCatPartition ptn = (new HCatPartition(table2Created, ptnDesc, makePartLocation(table2Created,ptnDesc))); sourceMetastore.addPartition(HCatAddPartitionDesc.create(ptn).build()); sourceMetastore.dropPartitions(dbName,tblName2,ptnDesc,true); sourceMetastore.dropTable(dbName, tblName1, true);
@Test public void testGetPartitionsWithPartialSpec() throws Exception { try { HCatClient client = HCatClient.create(new Configuration(hcatConf)); final String dbName = "myDb"; final String tableName = "myTable"; client.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); client.createDatabase(HCatCreateDBDesc.create(dbName).build()); List<HCatFieldSchema> columnSchema = Arrays.asList(new HCatFieldSchema("foo", Type.INT, ""), new HCatFieldSchema("bar", Type.STRING, "")); client.createTable(HCatCreateTableDesc.create(table, false).build()); table = client.getTable(dbName, tableName); assertNotNull("The created just now can't be null.", table); partitionSpec.put("grid", "AB"); partitionSpec.put("dt", "2011_12_31"); client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table, partitionSpec, makePartLocation(table,partitionSpec))).build()); partitionSpec.put("grid", "AB"); partitionSpec.put("dt", "2012_01_01"); client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table, partitionSpec, makePartLocation(table,partitionSpec))).build()); partitionSpec.put("dt", "2012_01_01"); partitionSpec.put("grid", "OB"); client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(table, partitionSpec, makePartLocation(table,partitionSpec))).build());
String tableTwo = "testTable2"; String tableThree = "testTable3"; HCatClient client = HCatClient.create(new Configuration(hcatConf)); client.dropDatabase(db, true, HCatClient.DropDBMode.CASCADE); client.createDatabase(dbDesc); List<String> dbNames = client.listDatabaseNamesByPattern("*"); assertTrue(dbNames.contains("default")); assertTrue(dbNames.contains(db)); HCatDatabase testDb = client.getDatabase(db); assertTrue(testDb.getComment() == null); assertTrue(testDb.getProperties().size() == 0); HCatCreateTableDesc tableDesc = HCatCreateTableDesc .create(db, tableOne, cols).fileFormat("rcfile").build(); client.createTable(tableDesc); HCatTable table1 = client.getTable(db, tableOne); assertTrue(table1.getInputFileFormat().equalsIgnoreCase( RCFileInputFormat.class.getName())); client.createTable(tableDesc); fail("Expected exception"); } catch (HCatException e) { client.dropTable(db, tableOne, true); HCatCreateTableDesc tableDesc2 = HCatCreateTableDesc.create(db, tableTwo, cols).fieldsTerminatedBy('\001').escapeChar('\002').linesTerminatedBy('\003'). mapKeysTerminatedBy('\004').collectionItemsTerminatedBy('\005').nullDefinedAs('\006').build();
@Test public void testCreateTableLike() throws Exception { HCatClient client = HCatClient.create(new Configuration(hcatConf)); String tableName = "tableone"; String cloneTable = "tabletwo"; client.dropTable(null, tableName, true); client.dropTable(null, cloneTable, true); ArrayList<HCatFieldSchema> cols = new ArrayList<HCatFieldSchema>(); cols.add(new HCatFieldSchema("id", Type.INT, "id columns")); cols.add(new HCatFieldSchema("value", Type.STRING, "id columns")); HCatCreateTableDesc tableDesc = HCatCreateTableDesc .create(null, tableName, cols).fileFormat("rcfile").build(); client.createTable(tableDesc); // create a new table similar to previous one. client.createTableLike(null, tableName, cloneTable, true, false, null); List<String> tables = client.listTableNamesByPattern(null, "table*"); assertTrue(tables.size() == 2); client.close(); }
@Test public void testGetMessageBusTopicName() throws Exception { try { HCatClient client = HCatClient.create(new Configuration(hcatConf)); String dbName = "testGetMessageBusTopicName_DBName"; String tableName = "testGetMessageBusTopicName_TableName"; client.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); client.createDatabase(HCatCreateDBDesc.create(dbName).build()); String messageBusTopicName = "MY.topic.name"; Map<String, String> tableProperties = new HashMap<String, String>(1); tableProperties.put(HCatConstants.HCAT_MSGBUS_TOPIC_NAME, messageBusTopicName); client.createTable(HCatCreateTableDesc.create(dbName, tableName, Arrays.asList(new HCatFieldSchema("foo", Type.STRING, ""))).tblProps(tableProperties).build()); assertEquals("MessageBus topic-name doesn't match!", messageBusTopicName, client.getMessageBusTopicName(dbName, tableName)); client.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); client.close(); } catch (Exception exception) { LOG.error("Unexpected exception.", exception); assertTrue("Unexpected exception:" + exception.getMessage(), false); } }
private HCatClient sourceMetaStore() throws HCatException { return HCatClient.create(new Configuration(hcatConf)); }
sourceMetaStore().dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); sourceMetaStore().createDatabase(HCatCreateDBDesc.create(dbName).build()); List<HCatFieldSchema> columnSchema = new ArrayList<HCatFieldSchema>( Arrays.asList(new HCatFieldSchema("foo", Type.INT, ""), sourceMetaStore().createTable(HCatCreateTableDesc.create(sourceTable).build()); sourceTable = sourceMetaStore().getTable(dbName, tableName); assertNotNull("Table couldn't be queried for. ", sourceTable); sourceMetaStore().addPartition(HCatAddPartitionDesc.create(sourcePartition_1).build()); assertEquals("Unexpected number of partitions. ", 1, sourceMetaStore().getPartitions(dbName, tableName).size()); HCatPartition addedPartition_1 = sourceMetaStore().getPartition(dbName, tableName, partitionSpec_1); assertEquals("Column schema doesn't match.", sourceTable.getCols(), addedPartition_1.getColumns()); assertEquals("InputFormat doesn't match.", sourceTable.getInputFileFormat(), addedPartition_1.getInputFormat()); targetMetaStore().dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); targetMetaStore().createDatabase(HCatCreateDBDesc.create(dbName).build()); HCatTable targetTable = targetMetaStore().deserializeTable(sourceMetaStore().serializeTable(sourceTable)); targetMetaStore().createTable(HCatCreateTableDesc.create(targetTable).build()); targetTable = targetMetaStore().getTable(dbName, tableName); .tblProps(tableParams) .serdeParam(serdeConstants.FIELD_DELIM, Character.toString('\001'));
HCatClient client = HCatClient.create(new Configuration(hcatConf)); String dbName = "testObjectNotFoundException_DBName"; String tableName = "testObjectNotFoundException_TableName"; client.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); client.getDatabase(dbName); assertTrue("Expected ObjectNotFoundException.", false); } catch(Exception exception) { client.createDatabase(HCatCreateDBDesc.create(dbName).build()); client.getTable(dbName, tableName); assertTrue("Expected ObjectNotFoundException.", false); } catch(Exception exception) { Arrays.asList(new HCatFieldSchema(partitionColumn, Type.STRING, ""))); HCatTable table = new HCatTable(dbName, tableName).cols(columns).partCols(partitionColumns); client.createTable(HCatCreateTableDesc.create(table, false).build()); HCatTable createdTable = client.getTable(dbName,tableName); client.getPartition(dbName, tableName, partitionSpec); assertTrue("Expected ObjectNotFoundException.", false); } catch(Exception exception) { client.addPartition(HCatAddPartitionDesc.create(new HCatPartition(createdTable, partitionSpec, makePartLocation(createdTable,partitionSpec))).build()); 0, client.listPartitionsByFilter(dbName, tableName, partitionColumn + " < 'foobar'").size());
@Test public void testUpdateTableSchema() throws Exception { try { HCatClient client = HCatClient.create(new Configuration(hcatConf)); final String dbName = "testUpdateTableSchema_DBName"; final String tableName = "testUpdateTableSchema_TableName"; client.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); client.createDatabase(HCatCreateDBDesc.create(dbName).build()); List<HCatFieldSchema> oldSchema = Arrays.asList(new HCatFieldSchema("foo", Type.INT, ""), new HCatFieldSchema("bar", Type.STRING, "")); client.createTable(HCatCreateTableDesc.create(dbName, tableName, oldSchema).build()); List<HCatFieldSchema> newSchema = Arrays.asList(new HCatFieldSchema("completely", Type.DOUBLE, ""), new HCatFieldSchema("new", Type.STRING, ""), new HCatFieldSchema("fields", Type.STRING, "")); client.updateTableSchema(dbName, tableName, newSchema); assertArrayEquals(newSchema.toArray(), client.getTable(dbName, tableName).getCols().toArray()); client.dropDatabase(dbName, false, HCatClient.DropDBMode.CASCADE); } catch (Exception exception) { LOG.error("Unexpected exception.", exception); assertTrue("Unexpected exception: " + exception.getMessage(), false); } }
@Test public void testDatabaseLocation() throws Exception { HCatClient client = HCatClient.create(new Configuration(hcatConf)); String dbName = "locationDB"; client.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); HCatCreateDBDesc dbDesc = HCatCreateDBDesc.create(dbName) .ifNotExists(true).location("/tmp/" + dbName).build(); client.createDatabase(dbDesc); HCatDatabase newDB = client.getDatabase(dbName); assertTrue(newDB.getLocation().matches(".*/tmp/" + dbName)); client.close(); }
sourceMetaStore().dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); sourceMetaStore().createDatabase(HCatCreateDBDesc.create(dbName).build()); List<HCatFieldSchema> columnSchema = new ArrayList<HCatFieldSchema>( Arrays.asList(new HCatFieldSchema("foo", Type.INT, ""), sourceMetaStore().createTable(HCatCreateTableDesc.create(sourceTable).build()); sourceTable = sourceMetaStore().getTable(dbName, tableName); assertNotNull("Table couldn't be queried for. ", sourceTable); sourceMetaStore().addPartition(HCatAddPartitionDesc.create(sourcePartition_1).build()); assertEquals("Unexpected number of partitions. ", 1, sourceMetaStore().getPartitions(dbName, tableName).size()); HCatPartition addedPartition_1 = sourceMetaStore().getPartition(dbName, tableName, partitionSpec_1); assertEquals("Column schema doesn't match.", sourceTable.getCols(), addedPartition_1.getColumns()); assertEquals("InputFormat doesn't match.", sourceTable.getInputFileFormat(), addedPartition_1.getInputFormat()); targetMetaStore().dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE); targetMetaStore().createDatabase(HCatCreateDBDesc.create(dbName).build()); HCatTable targetTable = targetMetaStore().deserializeTable(sourceMetaStore().serializeTable(sourceTable)); targetMetaStore().createTable(HCatCreateTableDesc.create(targetTable).build()); targetTable = targetMetaStore().getTable(dbName, tableName); .tblProps(tableParams) .serdeParam(serdeConstants.FIELD_DELIM, Character.toString('\001'));
@Test public void testDropTableException() throws Exception { HCatClient client = HCatClient.create(new Configuration(hcatConf)); String tableName = "tableToBeDropped"; boolean isExceptionCaught = false; client.dropTable(null, tableName, true); try { client.dropTable(null, tableName, false); } catch (Exception exp) { isExceptionCaught = true; assertTrue(exp instanceof HCatException); LOG.info("Drop Table Exception: " + exp.getCause()); } finally { client.close(); assertTrue("The expected exception was never thrown.", isExceptionCaught); } }
private static HCatTable getHCatTable(String databaseName, String tableName, HiveConf conf) { HCatClient client = null; try { client = HCatClient.create(conf); return client.getTable(databaseName, tableName); } catch (HCatException e) { throw new RuntimeException("Unable to get table from the metastore.", e); } finally { if (client != null) { try { client.close(); } catch (HCatException e) { throw new RuntimeException("Unable close client.", e); } } } }
@Override public void delete(URI uri, Context context) throws URIHandlerException { HCatClient client = ((HCatContext) context).getHCatClient(); try { HCatURI hcatUri = new HCatURI(uri); if (!hcatUri.getPartitionMap().isEmpty()) { client.dropPartitions(hcatUri.getDb(), hcatUri.getTable(), hcatUri.getPartitionMap(), true); } else { client.dropTable(hcatUri.getDb(), hcatUri.getTable(), true); } } catch (URISyntaxException e) { throw new HCatAccessorException(ErrorCode.E1501, e); } catch (HCatException e) { throw new HCatAccessorException(ErrorCode.E1501, e); } }
private void closeQuietly(HCatClient client, String delegationToken, boolean close) { if (close && client != null) { try { if(delegationToken != null && !delegationToken.isEmpty()) { client.cancelDelegationToken(delegationToken); } client.close(); } catch (Exception ignore) { XLog.getLog(HCatURIHandler.class).warn("Error closing hcat client", ignore); } } }