public static String mockUrl(String tag, KylinConfig kylinConfig) { String str = kylinConfig.getMetadataUrlPrefix() + "@" + tag; return str; }
public boolean checkIfNeedMigrate(KylinConfig kylinConfig) throws IOException { ResourceStore store = ResourceStore.getStore(kylinConfig); if (!(store instanceof HBaseResourceStore)) { logger.info("HBase enviroment not found. Not necessary to migrate data"); return false; } String userTableName = kylinConfig.getMetadataUrlPrefix() + AclConstant.USER_TABLE_NAME; String aclTableName = kylinConfig.getMetadataUrlPrefix() + AclConstant.ACL_TABLE_NAME; if (needMigrateTable(aclTableName, store) || needMigrateTable(userTableName, store)) return true; return false; }
public void check(List<String> segFullNameList) { issueExistHTables = Lists.newArrayList(); inconsistentHTables = Lists.newArrayList(); for (String segFullName : segFullNameList) { String[] sepNameList = segFullName.split(","); try { HTableDescriptor hTableDescriptor = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0])); String host = hTableDescriptor.getValue(IRealizationConstants.HTableTag); if (!dstCfg.getMetadataUrlPrefix().equalsIgnoreCase(host)) { inconsistentHTables.add(segFullName); } } catch (IOException e) { issueExistHTables.add(segFullName); continue; } } }
public void check(List<String> segFullNameList) { issueExistHTables = Lists.newArrayList(); inconsistentHTables = Lists.newArrayList(); for (String segFullName : segFullNameList) { String[] sepNameList = StringUtil.splitByComma(segFullName); try { HTableDescriptor hTableDescriptor = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0])); String host = hTableDescriptor.getValue(IRealizationConstants.HTableTag); if (!dstCfg.getMetadataUrlPrefix().equalsIgnoreCase(host)) { inconsistentHTables.add(segFullName); } } catch (IOException e) { issueExistHTables.add(segFullName); continue; } } }
public Factory(KylinConfig config) { this.curator = getZKClient(config); this.zkPathBase = fixSlash(config.getZookeeperBasePath() + "/" + config.getMetadataUrlPrefix()); }
private void updateHtable(String tableName) throws IOException { HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName)); if (oldHostValue.equals(desc.getValue(IRealizationConstants.HTableTag))) { desc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix()); hbaseAdmin.disableTable(TableName.valueOf(tableName)); hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc); hbaseAdmin.enableTable(TableName.valueOf(tableName)); updatedResources.add(tableName); } }
public void fixInconsistent() throws IOException { if (ifFix == true) { for (String segFullName : inconsistentHTables) { String[] sepNameList = segFullName.split(","); HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0])); logger.info("Change the host of htable " + sepNameList[0] + "belonging to cube " + sepNameList[1] + " from " + desc.getValue(IRealizationConstants.HTableTag) + " to " + dstCfg.getMetadataUrlPrefix()); hbaseAdmin.disableTable(TableName.valueOf(sepNameList[0])); desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix()); hbaseAdmin.modifyTable(TableName.valueOf(sepNameList[0]), desc); hbaseAdmin.enableTable(TableName.valueOf(sepNameList[0])); } } else { logger.info("------ Inconsistent HTables Needed To Be Fixed ------"); for (String hTable : inconsistentHTables) { String[] sepNameList = hTable.split(","); logger.info(sepNameList[0] + " belonging to cube " + sepNameList[1]); } logger.info("----------------------------------------------------"); } }
String aclTableName = kylinConfig.getMetadataUrlPrefix() + AclConstant.ACL_TABLE_NAME; convertToResourceStore(kylinConfig, aclTableName, store, new ResultConverter() { @Override break; case AclConstant.USER_TABLE_NAME: String userTableName = kylinConfig.getMetadataUrlPrefix() + AclConstant.USER_TABLE_NAME;
public void migrate(KylinConfig kylinConfig) throws IOException { if (!checkIfNeedMigrate(kylinConfig)) { logger.info("Do not need to migrate acl table data"); return; } else { if (!kylinConfig.getServerMode().equals("all")) { throw new IllegalStateException( "Please make sure that you have config kylin.server.mode=all before migrating data"); } logger.info("Start to migrate acl table data"); ResourceStore store = ResourceStore.getStore(kylinConfig); String userTableName = kylinConfig.getMetadataUrlPrefix() + AclConstant.USER_TABLE_NAME; //System.out.println("user table name : " + userTableName); String aclTableName = kylinConfig.getMetadataUrlPrefix() + AclConstant.ACL_TABLE_NAME; if (needMigrateTable(aclTableName, store)) { logger.info("Migrate table : {}", aclTableName); migrate(store, AclConstant.ACL_TABLE_NAME, kylinConfig); } if (needMigrateTable(userTableName, store)) { logger.info("Migrate table : {}", userTableName); migrate(store, AclConstant.USER_TABLE_NAME, kylinConfig); } } }
public void fixInconsistent() throws IOException { if (ifFix == true) { for (String segFullName : inconsistentHTables) { String[] sepNameList = StringUtil.splitByComma(segFullName); HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0])); logger.info("Change the host of htable " + sepNameList[0] + "belonging to cube " + sepNameList[1] + " from " + desc.getValue(IRealizationConstants.HTableTag) + " to " + dstCfg.getMetadataUrlPrefix()); hbaseAdmin.disableTable(sepNameList[0]); desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix()); hbaseAdmin.modifyTable(sepNameList[0], desc); hbaseAdmin.enableTable(sepNameList[0]); } } else { logger.info("------ Inconsistent HTables Needed To Be Fixed ------"); for (String hTable : inconsistentHTables) { String[] sepNameList = StringUtil.splitByComma(hTable); logger.info(sepNameList[0] + " belonging to cube " + sepNameList[1]); } logger.info("----------------------------------------------------"); } }
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName)); hbaseAdmin.disableTable(tableName); desc.setValue(IRealizationConstants.HTableTag, srcConfig.getMetadataUrlPrefix()); hbaseAdmin.modifyTable(tableName, desc); hbaseAdmin.enableTable(tableName);
List<String> oldTables = getOldHTables(); if (oldTables != null && oldTables.size() > 0) { String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix(); Admin admin = null; try {
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName); hbaseAdmin.disableTable(tableName); desc.setValue(IRealizationConstants.HTableTag, srcConfig.getMetadataUrlPrefix()); hbaseAdmin.modifyTable(tableName, desc); hbaseAdmin.enableTable(tableName); Table destAclHtable = null; try { destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
for (HTableDescriptor desc : tableDescriptors) { String host = desc.getValue(IRealizationConstants.HTableTag); if (kylinConfig.getMetadataUrlPrefix().equalsIgnoreCase(host)) {
@Test public void testGetMetadataUrlPrefix() { KylinConfig config = KylinConfig.getInstanceFromEnv(); config.setMetadataUrl("testMetaPrefix@hbase"); assertEquals("testMetaPrefix", config.getMetadataUrlPrefix()); config.setMetadataUrl("testMetaPrefix@hdfs"); assertEquals("testMetaPrefix", config.getMetadataUrlPrefix()); config.setMetadataUrl("/kylin/temp"); assertEquals("/kylin/temp", config.getMetadataUrlPrefix()); }
hTableDesc.setCompactionEnabled(false); hTableDesc.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName()); hTableDesc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix()); hTableDesc.setValue(IRealizationConstants.HTableCreationTime, String.valueOf(System.currentTimeMillis())); String commitInfo = KylinVersion.getGitCommitInfo();
if (!host.equalsIgnoreCase(kylinConfig.getMetadataUrlPrefix())) { logger.warn("This server doesn't own this table: " + tableName); return false;
Table aclHtable = null; try { aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
Table aclHtable = null; try { aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
tableDesc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix()); tableDesc.setValue(IRealizationConstants.HTableCreationTime, String.valueOf(System.currentTimeMillis()));