@Override public ILookupTable getCachedLookupTable(TableDesc tableDesc, ExtTableSnapshotInfo extTableSnapshotInfo, boolean buildIfNotExist) { String resourcePath = extTableSnapshotInfo.getResourcePath(); if (inBuildingTables.containsKey(resourcePath)) { logger.info("cache is in building for snapshot:" + resourcePath); return null; } CachedTableInfo cachedTableInfo = tablesCache.getIfPresent(resourcePath); if (cachedTableInfo == null) { if (buildIfNotExist) { buildSnapshotCache(tableDesc, extTableSnapshotInfo, getSourceLookupTable(tableDesc, extTableSnapshotInfo)); } logger.info("no available cache ready for the table snapshot:" + extTableSnapshotInfo.getResourcePath()); return null; } String[] keyColumns = extTableSnapshotInfo.getKeyColumns(); String dbPath = getSnapshotStorePath(extTableSnapshotInfo.getTableName(), extTableSnapshotInfo.getId()); return new RocksDBLookupTable(tableDesc, keyColumns, dbPath); }
/** * create ext table snapshot * @param signature * @param tableName * @param keyColumns *@param storageType * @param storageLocation @return created snapshot * @throws IOException */ public ExtTableSnapshotInfo createSnapshot(TableSignature signature, String tableName, String snapshotID, String[] keyColumns, int shardNum, String storageType, String storageLocation) throws IOException { ExtTableSnapshotInfo snapshot = new ExtTableSnapshotInfo(); snapshot.setUuid(snapshotID); snapshot.setSignature(signature); snapshot.setTableName(tableName); snapshot.setKeyColumns(keyColumns); snapshot.setStorageType(storageType); snapshot.setStorageLocationIdentifier(storageLocation); snapshot.setShardNum(shardNum); save(snapshot); return snapshot; }
private ExtTableSnapshotInfo checkDupByInfo(ExtTableSnapshotInfo snapshot) throws IOException { ResourceStore store = TableMetadataManager.getInstance(this.config).getStore(); String resourceDir = snapshot.getResourceDir(); NavigableSet<String> existings = store.listResources(resourceDir); if (existings == null) return null; TableSignature sig = snapshot.getSignature(); for (String existing : existings) { ExtTableSnapshotInfo existingSnapshot = load(existing); // direct load from store if (existingSnapshot != null && sig.equals(existingSnapshot.getSignature())) return existingSnapshot; } return null; }
/** * * @param signature source table signature * @param tableName * @return latest snapshot info * @throws IOException */ public ExtTableSnapshotInfo getLatestSnapshot(TableSignature signature, String tableName) throws IOException { ExtTableSnapshotInfo snapshot = new ExtTableSnapshotInfo(signature, tableName); snapshot.updateRandomUuid(); ExtTableSnapshotInfo dupSnapshot = checkDupByInfo(snapshot); return dupSnapshot; }
private void saveSnapshotCacheState(ExtTableSnapshotInfo extTableSnapshotInfo, String cachePath) { File stateFile = getCacheStateFile(getSnapshotCachePath(extTableSnapshotInfo.getTableName(), extTableSnapshotInfo.getId())); try { Files.write(CacheState.AVAILABLE.name(), stateFile, Charsets.UTF_8); tablesCache.put(extTableSnapshotInfo.getResourcePath(), new CachedTableInfo(cachePath)); } catch (IOException e) { throw new RuntimeException("error when write cache state for snapshot:" + extTableSnapshotInfo.getResourcePath()); } }
private ExtTableSnapshotInfo buildSnapshotCache(String snapshotID, int rowCnt) throws Exception { ExtTableSnapshotInfo snapshotInfo = new ExtTableSnapshotInfo(); snapshotInfo.setTableName(TABLE_COUNTRY); snapshotInfo.setUuid(snapshotID); snapshotInfo.setStorageType(MOCK_EXT_LOOKUP); snapshotInfo.setKeyColumns(new String[] { "COUNTRY" }); snapshotInfo.setRowCnt(rowCnt); snapshotInfo.setSignature(new TableSignature("/test", rowCnt, System.currentTimeMillis())); ExtTableSnapshotInfoManager.getInstance(kylinConfig).save(snapshotInfo); RocksDBLookupTableCache cache = RocksDBLookupTableCache.getInstance(kylinConfig); cache.buildSnapshotCache(tableDesc, snapshotInfo, getLookupTableWithRandomData(rowCnt)); while (cache.getCacheState(snapshotInfo) == CacheState.IN_BUILDING) { Thread.sleep(500); } return snapshotInfo; }
public String getResourcePath() { return getResourcePath(tableName, uuid); }
for (ExtTableSnapshotInfo extTableSnapshot : extTableSnapshots) { TableSnapshotResponse response = new TableSnapshotResponse(); response.setSnapshotID(extTableSnapshot.getId()); response.setSnapshotType(TableSnapshotResponse.TYPE_EXT); response.setLastBuildTime(extTableSnapshot.getLastBuildTime()); response.setStorageType(extTableSnapshot.getStorageType()); response.setSourceTableSize(extTableSnapshot.getSignature().getSize()); response.setSourceTableLastModifyTime(extTableSnapshot.getSignature().getLastModifiedTime()); response.setCubesAndSegmentsUsage(snapshotUsageMap.get(extTableSnapshot.getResourcePath())); result.add(response);
@Test public void testRestoreCacheFromFiles() throws Exception { String snapshotID = RandomUtil.randomUUID().toString(); String snapshotCacheBasePath = RocksDBLookupTableCache.getCacheBasePath(kylinConfig) + File.separator + TABLE_COUNTRY + File.separator + snapshotID; String dbPath = snapshotCacheBasePath + File.separator + "db"; RocksDBLookupBuilder builder = new RocksDBLookupBuilder(tableDesc, new String[] { "COUNTRY" }, dbPath); builder.build(getLookupTableWithRandomData(10000)); String stateFilePath = snapshotCacheBasePath + File.separator + "STATE"; Files.write(CacheState.AVAILABLE.name(), new File(stateFilePath), Charsets.UTF_8); RocksDBLookupTableCache cache = RocksDBLookupTableCache.getInstance(kylinConfig); ExtTableSnapshotInfo snapshotInfo = new ExtTableSnapshotInfo(); snapshotInfo.setTableName(TABLE_COUNTRY); snapshotInfo.setUuid(snapshotID); snapshotInfo.setStorageType(MOCK_EXT_LOOKUP); snapshotInfo.setKeyColumns(new String[] { "COUNTRY" }); ILookupTable lookupTable = cache.getCachedLookupTable(tableDesc, snapshotInfo, false); int rowCnt = 0; for (String[] strings : lookupTable) { rowCnt++; } lookupTable.close(); assertEquals(10000, rowCnt); }
public HBaseLookupTable(TableDesc tableDesc, ExtTableSnapshotInfo extTableSnapshot) { String tableName = extTableSnapshot.getStorageLocationIdentifier(); this.lookupTableName = TableName.valueOf(tableName); KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); Connection connection = HBaseConnection.get(kylinConfig.getStorageUrl()); try { table = connection.getTable(lookupTableName); } catch (IOException e) { throw new RuntimeException("error when connect HBase", e); } String[] keyColumns = extTableSnapshot.getKeyColumns(); encoder = new HBaseLookupRowEncoder(tableDesc, keyColumns, extTableSnapshot.getShardNum()); }
public static ILookupTable getExtLookupTable(TableDesc tableDesc, ExtTableSnapshotInfo extTableSnapshot) { IExtLookupTableCache extLookupTableCache = getExtLookupProvider(extTableSnapshot.getStorageType()).getLocalCache(); if (extLookupTableCache == null) { return getExtLookupTableWithoutCache(tableDesc, extTableSnapshot); } ILookupTable cachedLookupTable = extLookupTableCache.getCachedLookupTable(tableDesc, extTableSnapshot, true); if (cachedLookupTable != null) { logger.info("try to use cached lookup table:{}", extTableSnapshot.getResourcePath()); return cachedLookupTable; } logger.info("use ext lookup table:{}", extTableSnapshot.getResourcePath()); return getExtLookupTableWithoutCache(tableDesc, extTableSnapshot); }
long srcTableRowCnt = Long.parseLong(job.findExtraInfoBackward(BatchConstants.LOOKUP_EXT_SNAPSHOT_SRC_RECORD_CNT_PFX + tableName, "-1")); logger.info("update table:{} snapshot row count:{}", tableName, srcTableRowCnt); snapshot.setRowCnt(srcTableRowCnt); snapshot.setLastBuildTime(System.currentTimeMillis()); extTableSnapshotInfoManager.updateSnapshot(snapshot); String hTableName = snapshot.getStorageLocationIdentifier();
@Test public void testCheckCacheState() throws Exception { ExtTableSnapshotInfo snapshotInfo = buildSnapshotCache(RandomUtil.randomUUID().toString(), 1000); RocksDBLookupTableCache cache = RocksDBLookupTableCache.getInstance(kylinConfig); ILookupTable cachedLookupTable = cache.getCachedLookupTable(tableDesc, snapshotInfo, false); assertNotNull(cachedLookupTable); cachedLookupTable.close(); ExtTableSnapshotInfoManager.getInstance(kylinConfig).removeSnapshot(snapshotInfo.getTableName(), snapshotInfo.getId()); cache.checkCacheState(); String cacheLocalPath = cache.getSnapshotCachePath(snapshotInfo.getTableName(), snapshotInfo.getId()); // won't cleanup because it is newly created in last 1 hour assertTrue(new File(cacheLocalPath).exists()); // change the volatile value kylinConfig.setProperty("kylin.snapshot.ext.local.cache.check.volatile", "0"); cache.checkCacheState(); // this time it should be removed. assertFalse(new File(cacheLocalPath).exists()); cachedLookupTable = cache.getCachedLookupTable(tableDesc, snapshotInfo, false); assertNull(cachedLookupTable); }
public static ILookupTable getExtLookupTableWithoutCache(TableDesc tableDesc, ExtTableSnapshotInfo extTableSnapshot) { IExtLookupProvider provider = getExtLookupProvider(extTableSnapshot.getStorageType()); return provider.getLookupTable(tableDesc, extTableSnapshot); }
private static List<String> getAllUsedExtLookupTables() throws IOException { List<String> result = Lists.newArrayList(); KylinConfig config = KylinConfig.getInstanceFromEnv(); final Set<String> activeSnapshotSet = ExtTableSnapshotInfoManager.getInstance(config).getAllExtSnapshotResPaths(); for (String extSnapshotResource : activeSnapshotSet) { try { ExtTableSnapshotInfo extTableSnapshot = ExtTableSnapshotInfoManager.getInstance(config).getSnapshot( extSnapshotResource); if (extTableSnapshot != null) { if (ExtTableSnapshotInfo.STORAGE_TYPE_HBASE.equals(extTableSnapshot.getStorageType())) { result.add(extTableSnapshot.getStorageLocationIdentifier()); } } } catch (Exception e) { logger.error("error fetch ext table snapshot:" + extSnapshotResource, e); } } return result; }
private void removeSnapshotIfExist(ExtTableSnapshotInfoManager extSnapshotInfoManager, KylinConfig kylinConfig, String tableName, String lookupSnapshotID) throws IOException { ExtTableSnapshotInfo snapshotInfo = null; try { snapshotInfo = extSnapshotInfoManager.getSnapshot(tableName, lookupSnapshotID); } catch (Exception e) { // swallow the exception, means not snapshot exist of this snapshot id } if (snapshotInfo == null) { return; } logger.info("the table:{} snapshot:{} exist, remove it", tableName, lookupSnapshotID); extSnapshotInfoManager.removeSnapshot(tableName, lookupSnapshotID); String hTableName = snapshotInfo.getStorageLocationIdentifier(); logger.info("remove related HBase table:{} for snapshot:{}", hTableName, lookupSnapshotID); Connection conn = getHBaseConnection(kylinConfig); Admin admin = conn.getAdmin(); admin.deleteTable(TableName.valueOf(hTableName)); }
@Override public CacheState getCacheState(ExtTableSnapshotInfo extTableSnapshotInfo) { String resourcePath = extTableSnapshotInfo.getResourcePath(); if (inBuildingTables.containsKey(resourcePath)) { return CacheState.IN_BUILDING; } File stateFile = getCacheStateFile(getSnapshotCachePath(extTableSnapshotInfo.getTableName(), extTableSnapshotInfo.getId())); if (!stateFile.exists()) { return CacheState.NONE; } try { String stateString = Files.toString(stateFile, Charsets.UTF_8); return CacheState.valueOf(stateString); } catch (IOException e) { logger.error("error when read state file", e); } return CacheState.NONE; }
@Override public void removeSnapshotCache(ExtTableSnapshotInfo extTableSnapshotInfo) { tablesCache.invalidate(extTableSnapshotInfo.getResourcePath()); }
public HBaseLookupTable(TableDesc tableDesc, ExtTableSnapshotInfo extTableSnapshot) { String tableName = extTableSnapshot.getStorageLocationIdentifier(); this.lookupTableName = TableName.valueOf(tableName); KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); Connection connection = HBaseConnection.get(kylinConfig.getStorageUrl()); try { table = connection.getTable(lookupTableName); } catch (IOException e) { throw new RuntimeException("error when connect HBase", e); } String[] keyColumns = extTableSnapshot.getKeyColumns(); encoder = new HBaseLookupRowEncoder(tableDesc, keyColumns, extTableSnapshot.getShardNum()); }
public static ILookupTable getExtLookupTable(TableDesc tableDesc, ExtTableSnapshotInfo extTableSnapshot) { IExtLookupTableCache extLookupTableCache = getExtLookupProvider(extTableSnapshot.getStorageType()).getLocalCache(); if (extLookupTableCache == null) { return getExtLookupTableWithoutCache(tableDesc, extTableSnapshot); } ILookupTable cachedLookupTable = extLookupTableCache.getCachedLookupTable(tableDesc, extTableSnapshot, true); if (cachedLookupTable != null) { logger.info("try to use cached lookup table:{}", extTableSnapshot.getResourcePath()); return cachedLookupTable; } logger.info("use ext lookup table:{}", extTableSnapshot.getResourcePath()); return getExtLookupTableWithoutCache(tableDesc, extTableSnapshot); }