/** * Connects to HMS by creating HiveMetaStoreClient. * * @throws IOException if could not establish connection * @throws InterruptedException if connection was interrupted * @throws MetaException if other errors happened */ public void connect() throws IOException, InterruptedException, MetaException { if (client != null) { return; } client = hiveConnectionFactory.connect().getClient(); }
/** * Gets the HMS client connection object. * If will create a new connection if no connection object exists. * * @return The HMS client used to communication with the Hive MetaStore. * @throws Exception If it cannot connect to the HMS service. */ private HiveMetaStoreClient getHmsClient() throws Exception { if (hmsClient == null) { try { hmsClient = hmsConnectionFactory.connect().getClient(); } catch (Exception e) { LOGGER.error("Fail to connect to the HMS service: {}", e.getMessage()); throw e; } } return hmsClient; }
@Before public void setupMocks() throws Exception { reset(hmsConnectionMock, hmsClientMock, sentryStore); when(hmsConnectionMock.connect()).thenReturn(new HMSClient(hmsClientMock)); }
@Override ObjectMapping doTask() throws Exception { List<Partition> tblParts; HMSClient c = null; try (HMSClient client = clientFactory.connect()) { c = client; tblParts = client.getClient().getPartitionsByNames(dbName, tblName, partNames); } catch (Exception e) { if (c != null) { c.invalidate(); } throw e; } LOGGER.debug("Fetched partitions for db = {}, table = {}", dbName, tblName); Collection<String> partitionNames = new ArrayList<>(tblParts.size()); for (Partition part : tblParts) { if(part != null && part.getSd() != null) { String partPath = pathFromURI(part.getSd().getLocation()); if (partPath != null) { partitionNames.add(partPath.intern()); } } else { LOGGER.info("Partition or its storage descriptor is null while fetching partitions for db = {} table = {}", dbName, tblName); } } return new ObjectMapping(authName, partitionNames); } }
try (HMSClient client = clientFactory.connect()) { c = client; allDbStr = client.getClient().getAllDatabases();
ObjectMapping doTask() throws Exception { HMSClient c = null; try (HMSClient client = clientFactory.connect()) { c = client; List<Table> tables = client.getClient().getTableObjectsByName(dbName, tableNames);
@Override ObjectMapping doTask() throws Exception { HMSClient c = null; try (HMSClient client = clientFactory.connect()) { c = client; Database db = client.getClient().getDatabase(dbName); if (!dbName.equalsIgnoreCase(db.getName())) { LOGGER.warn("Database name {} does not match {}", db.getName(), dbName); return emptyObjectMapping; } List<String> allTblStr = client.getClient().getAllTables(dbName); // Count total number of tables tableCount.inc(allTblStr.size()); for (int i = 0; i < allTblStr.size(); i += maxTablesPerCall) { List<String> tablesToFetch = allTblStr.subList(i, Math.min(i + maxTablesPerCall, allTblStr.size())); Callable<CallResult> tableTask = new TableTask(db, tablesToFetch); results.add(threadPool.submit(tableTask)); } String dbPath = safeIntern(pathFromURI(db.getLocationUri())); return (dbPath != null) ? new ObjectMapping(dbName, dbPath) : emptyObjectMapping; } catch (Exception e) { if (c != null) { c.invalidate(); } throw e; } } }
@Test public void testGetEmptyNotificationsWhenHmsReturnsANullResponse() throws Exception { SentryStore store = Mockito.mock(SentryStore.class); HiveConnectionFactory hmsConnection = Mockito.mock(HiveConnectionFactory.class); HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class); Mockito.when(hmsConnection.connect()).thenReturn(new HMSClient(hmsClient)); try (HiveNotificationFetcher fetcher = new HiveNotificationFetcher(store, hmsConnection)) { List<NotificationEvent> events; Mockito.when(hmsClient.getNextNotification(0, Integer.MAX_VALUE, null)) .thenReturn(null); events = fetcher.fetchNotifications(0); assertTrue(events.isEmpty()); } }
@Test public void testGetEmptyNotificationsWhenHmsReturnsEmptyEvents() throws Exception { SentryStore store = Mockito.mock(SentryStore.class); HiveConnectionFactory hmsConnection = Mockito.mock(HiveConnectionFactory.class); HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class); Mockito.when(hmsConnection.connect()).thenReturn(new HMSClient(hmsClient)); try (HiveNotificationFetcher fetcher = new HiveNotificationFetcher(store, hmsConnection)) { List<NotificationEvent> events; Mockito.when(hmsClient.getNextNotification(0, Integer.MAX_VALUE, null)) .thenReturn(new NotificationEventResponse(Collections.<NotificationEvent>emptyList())); events = fetcher.fetchNotifications(0); assertTrue(events.isEmpty()); } }
@Test public void testGetAllNotificationsReturnedByHms() throws Exception { SentryStore store = Mockito.mock(SentryStore.class); HiveConnectionFactory hmsConnection = Mockito.mock(HiveConnectionFactory.class); HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class); Mockito.when(hmsConnection.connect()).thenReturn(new HMSClient(hmsClient)); try (HiveNotificationFetcher fetcher = new HiveNotificationFetcher(store, hmsConnection)) { List<NotificationEvent> events; Mockito.when(hmsClient.getNextNotification(0, Integer.MAX_VALUE, null)) .thenReturn(new NotificationEventResponse( Arrays.<NotificationEvent>asList( new NotificationEvent(1L, 0, "CREATE_DATABASE", ""), new NotificationEvent(2L, 0, "CREATE_TABLE", "") ) )); events = fetcher.fetchNotifications(0); assertEquals(2, events.size()); assertEquals(1, events.get(0).getEventId()); assertEquals("CREATE_DATABASE", events.get(0).getEventType()); assertEquals(2, events.get(1).getEventId()); assertEquals("CREATE_TABLE", events.get(1).getEventType()); } }
HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class); Mockito.when(hmsConnection.connect()).thenReturn(new HMSClient(hmsClient));
HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class); Mockito.when(hmsConnection.connect()).thenReturn(new HMSClient(hmsClient));