private static List<String> getAllUsedExtLookupTables() throws IOException { List<String> result = Lists.newArrayList(); KylinConfig config = KylinConfig.getInstanceFromEnv(); final Set<String> activeSnapshotSet = ExtTableSnapshotInfoManager.getInstance(config).getAllExtSnapshotResPaths(); for (String extSnapshotResource : activeSnapshotSet) { try { ExtTableSnapshotInfo extTableSnapshot = ExtTableSnapshotInfoManager.getInstance(config).getSnapshot( extSnapshotResource); if (extTableSnapshot != null) { if (ExtTableSnapshotInfo.STORAGE_TYPE_HBASE.equals(extTableSnapshot.getStorageType())) { result.add(extTableSnapshot.getStorageLocationIdentifier()); } } } catch (Exception e) { logger.error("error fetch ext table snapshot:" + extSnapshotResource, e); } } return result; }
public void removeSnapshotLocalCache(String tableName, String snapshotID) { ExtTableSnapshotInfoManager snapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(getConfig()); ExtTableSnapshotInfo extTableSnapshotInfo = snapshotInfoManager.getSnapshot(tableName, snapshotID); if (extTableSnapshotInfo == null) { throw new IllegalArgumentException( "cannot find ext snapshot info for table:" + tableName + " snapshot:" + snapshotID); } LookupProviderFactory.removeLocalCache(extTableSnapshotInfo); }
public String getSnapshotLocalCacheState(String tableName, String snapshotID) { ExtTableSnapshotInfoManager snapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(getConfig()); ExtTableSnapshotInfo extTableSnapshotInfo = snapshotInfoManager.getSnapshot(tableName, snapshotID); if (extTableSnapshotInfo == null) { throw new IllegalArgumentException( "cannot find ext snapshot info for table:" + tableName + " snapshot:" + snapshotID); } CacheState cacheState = LookupProviderFactory.getCacheState(extTableSnapshotInfo); return cacheState.name(); }
final Set<String> activeSnapshotSet = ExtTableSnapshotInfoManager.getInstance(config).getAllExtSnapshotResPaths();
public void updateSnapshotLocalCache(String project, String tableName, String snapshotID) { ExtTableSnapshotInfoManager snapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(getConfig()); ExtTableSnapshotInfo extTableSnapshotInfo = snapshotInfoManager.getSnapshot(tableName, snapshotID); TableDesc tableDesc = getTableManager().getTableDesc(tableName, project); if (extTableSnapshotInfo == null) { throw new IllegalArgumentException( "cannot find ext snapshot info for table:" + tableName + " snapshot:" + snapshotID); } LookupProviderFactory.rebuildLocalCache(tableDesc, extTableSnapshotInfo); }
private ILookupTable getExtLookupTable(CubeSegment cubeSegment, String tableName, SnapshotTableDesc snapshotTableDesc) { String snapshotResPath = getSnapshotResPath(cubeSegment, tableName, snapshotTableDesc); ExtTableSnapshotInfo extTableSnapshot = ExtTableSnapshotInfoManager.getInstance(config).getSnapshot( snapshotResPath); TableDesc tableDesc = getMetadataManager().getTableDesc(tableName, cubeSegment.getProject()); return LookupProviderFactory.getExtLookupTable(tableDesc, extTableSnapshot); }
DefaultChainedExecutable job = (DefaultChainedExecutable) execMgr.getJob(cubingJobID); ExtTableSnapshotInfoManager extTableSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(kylinConfig); ExtTableSnapshotInfo snapshot = extTableSnapshotInfoManager.getSnapshot(tableName, snapshotID); long srcTableRowCnt = Long.parseLong(job.findExtraInfoBackward(BatchConstants.LOOKUP_EXT_SNAPSHOT_SRC_RECORD_CNT_PFX + tableName, "-1"));
cube.getProject()); ExtTableSnapshotInfoManager extSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(kylinConfig); removeSnapshotIfExist(extSnapshotInfoManager, kylinConfig, tableName, lookupSnapshotID);
public void addMaterializeLookupTableSteps(LookupMaterializeContext context, String tableName, SnapshotTableDesc snapshotTableDesc) { KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); ExtTableSnapshotInfoManager extTableSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(kylinConfig); TableDesc tableDesc = TableMetadataManager.getInstance(kylinConfig).getTableDesc(tableName, cube.getProject()); IReadableTable sourceTable = SourceManager.createReadableTable(tableDesc, context.getJobFlow().getId()); try { ExtTableSnapshotInfo latestSnapshot = extTableSnapshotInfoManager.getLatestSnapshot( sourceTable.getSignature(), tableName); if (latestSnapshot != null) { logger.info("there is latest snapshot exist for table:{}, skip build snapshot step.", tableName); context.addLookupSnapshotPath(tableName, latestSnapshot.getResourcePath()); return; } } catch (IOException ioException) { throw new RuntimeException(ioException); } logger.info("add build snapshot steps for table:{}", tableName); String snapshotID = genLookupSnapshotID(); context.addLookupSnapshotPath(tableName, ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID)); addLookupTableConvertToHFilesStep(context.getJobFlow(), tableName, snapshotID); addLookupTableHFilesBulkLoadStep(context.getJobFlow(), tableName, snapshotID); if (snapshotTableDesc !=null && snapshotTableDesc.isEnableLocalCache()) { addUpdateSnapshotQueryCacheStep(context.getJobFlow(), tableName, snapshotID); } }
@Test public void testCheckCacheState() throws Exception { ExtTableSnapshotInfo snapshotInfo = buildSnapshotCache(RandomUtil.randomUUID().toString(), 1000); RocksDBLookupTableCache cache = RocksDBLookupTableCache.getInstance(kylinConfig); ILookupTable cachedLookupTable = cache.getCachedLookupTable(tableDesc, snapshotInfo, false); assertNotNull(cachedLookupTable); cachedLookupTable.close(); ExtTableSnapshotInfoManager.getInstance(kylinConfig).removeSnapshot(snapshotInfo.getTableName(), snapshotInfo.getId()); cache.checkCacheState(); String cacheLocalPath = cache.getSnapshotCachePath(snapshotInfo.getTableName(), snapshotInfo.getId()); // won't cleanup because it is newly created in last 1 hour assertTrue(new File(cacheLocalPath).exists()); // change the volatile value kylinConfig.setProperty("kylin.snapshot.ext.local.cache.check.volatile", "0"); cache.checkCacheState(); // this time it should be removed. assertFalse(new File(cacheLocalPath).exists()); cachedLookupTable = cache.getCachedLookupTable(tableDesc, snapshotInfo, false); assertNull(cachedLookupTable); }
List<TableSnapshotResponse> internalGetLookupTableSnapshots(String tableName, TableSignature signature) throws IOException { ExtTableSnapshotInfoManager extSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(getConfig()); SnapshotManager snapshotManager = SnapshotManager.getInstance(getConfig()); List<ExtTableSnapshotInfo> extTableSnapshots = extSnapshotInfoManager.getSnapshots(tableName);
private ExtTableSnapshotInfo buildSnapshotCache(String snapshotID, int rowCnt) throws Exception { ExtTableSnapshotInfo snapshotInfo = new ExtTableSnapshotInfo(); snapshotInfo.setTableName(TABLE_COUNTRY); snapshotInfo.setUuid(snapshotID); snapshotInfo.setStorageType(MOCK_EXT_LOOKUP); snapshotInfo.setKeyColumns(new String[] { "COUNTRY" }); snapshotInfo.setRowCnt(rowCnt); snapshotInfo.setSignature(new TableSignature("/test", rowCnt, System.currentTimeMillis())); ExtTableSnapshotInfoManager.getInstance(kylinConfig).save(snapshotInfo); RocksDBLookupTableCache cache = RocksDBLookupTableCache.getInstance(kylinConfig); cache.buildSnapshotCache(tableDesc, snapshotInfo, getLookupTableWithRandomData(rowCnt)); while (cache.getCacheState(snapshotInfo) == CacheState.IN_BUILDING) { Thread.sleep(500); } return snapshotInfo; }
final Set<String> activeSnapshotSet = ExtTableSnapshotInfoManager.getInstance(config).getAllExtSnapshotResPaths();
private ILookupTable getExtLookupTable(CubeSegment cubeSegment, String tableName, SnapshotTableDesc snapshotTableDesc) { String snapshotResPath = getSnapshotResPath(cubeSegment, tableName, snapshotTableDesc); ExtTableSnapshotInfo extTableSnapshot = ExtTableSnapshotInfoManager.getInstance(config).getSnapshot( snapshotResPath); TableDesc tableDesc = getMetadataManager().getTableDesc(tableName, cubeSegment.getProject()); return LookupProviderFactory.getExtLookupTable(tableDesc, extTableSnapshot); }
DefaultChainedExecutable job = (DefaultChainedExecutable) execMgr.getJob(cubingJobID); ExtTableSnapshotInfoManager extTableSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(kylinConfig); ExtTableSnapshotInfo snapshot = extTableSnapshotInfoManager.getSnapshot(tableName, snapshotID); long srcTableRowCnt = Long.parseLong(job.findExtraInfoBackward(BatchConstants.LOOKUP_EXT_SNAPSHOT_SRC_RECORD_CNT_PFX + tableName, "-1"));
cube.getProject()); ExtTableSnapshotInfoManager extSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(kylinConfig); removeSnapshotIfExist(extSnapshotInfoManager, kylinConfig, tableName, lookupSnapshotID);
public void addMaterializeLookupTableSteps(LookupMaterializeContext context, String tableName, SnapshotTableDesc snapshotTableDesc) { KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); ExtTableSnapshotInfoManager extTableSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(kylinConfig); TableDesc tableDesc = TableMetadataManager.getInstance(kylinConfig).getTableDesc(tableName, cube.getProject()); IReadableTable sourceTable = SourceManager.createReadableTable(tableDesc, context.getJobFlow().getId()); try { ExtTableSnapshotInfo latestSnapshot = extTableSnapshotInfoManager.getLatestSnapshot( sourceTable.getSignature(), tableName); if (latestSnapshot != null) { logger.info("there is latest snapshot exist for table:{}, skip build snapshot step.", tableName); context.addLookupSnapshotPath(tableName, latestSnapshot.getResourcePath()); return; } } catch (IOException ioException) { throw new RuntimeException(ioException); } logger.info("add build snapshot steps for table:{}", tableName); String snapshotID = genLookupSnapshotID(); context.addLookupSnapshotPath(tableName, ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID)); addLookupTableConvertToHFilesStep(context.getJobFlow(), tableName, snapshotID); addLookupTableHFilesBulkLoadStep(context.getJobFlow(), tableName, snapshotID); if (snapshotTableDesc !=null && snapshotTableDesc.isEnableLocalCache()) { addUpdateSnapshotQueryCacheStep(context.getJobFlow(), tableName, snapshotID); } }