public void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String copyAcl, String purgeAndDisable, String overwriteIfExists, String realExecute) throws IOException, InterruptedException { moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, copyAcl, purgeAndDisable, overwriteIfExists, realExecute); }
public static void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String copyAcl, String purgeAndDisable, String overwriteIfExists, String realExecute) throws IOException, InterruptedException { moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, copyAcl, purgeAndDisable, overwriteIfExists, realExecute); }
public void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String copyAcl, String purgeAndDisable, String overwriteIfExists, String realExecute, String migrateSegment) throws IOException, InterruptedException { moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, Boolean.parseBoolean(copyAcl), Boolean.parseBoolean(purgeAndDisable), Boolean.parseBoolean(overwriteIfExists), Boolean.parseBoolean(realExecute), Boolean.parseBoolean(migrateSegment)); }
private void extractResources(File destDir) { logger.info("The resource paths going to be extracted:"); for (String s : requiredResources) { logger.info(s + "(required)"); } try { KylinConfig srcConfig = KylinConfig.getInstanceFromEnv(); KylinConfig dstConfig = KylinConfig.createInstanceFromUri(destDir.getAbsolutePath()); new ResourceTool().copy(srcConfig, dstConfig, requiredResources); } catch (Exception e) { throw new RuntimeException("Failed to extract job resources. ", e); } }
public CubeMetadataUpgrade(String newMetadataUrl) { KylinConfig.destroyInstance(); config = KylinConfig.createInstanceFromUri(newMetadataUrl); store = ResourceStore.getStore(config); }
KylinConfig config = KylinConfig.createInstanceFromUri(commandLine.getOptionValue(OPTION_METADATA.getOpt())); String sql = commandLine.getOptionValue(OPTION_SQL.getOpt());
public static void dumpResources(KylinConfig kylinConfig, File metaDir, Set<String> dumpList) throws IOException { long startTime = System.currentTimeMillis(); ResourceStore from = ResourceStore.getStore(kylinConfig); KylinConfig localConfig = KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath()); ResourceStore to = ResourceStore.getStore(localConfig); for (String path : dumpList) { RawResource res = from.getResource(path); if (res == null) throw new IllegalStateException("No resource found at -- " + path); to.putResource(path, res.content(), res.lastModified()); res.content().close(); } logger.debug("Dump resources to {} took {} ms", metaDir, System.currentTimeMillis() - startTime); }
KylinConfig localConfig = KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath()); ResourceStore to = ResourceStore.getStore(localConfig); for (String path : dumpList) {
kylinConfig = KylinConfig.getInstanceFromEnv(); } else { kylinConfig = KylinConfig.createInstanceFromUri(dstCfgUri);
switch (cmd) { case "reset": tool.reset(args.length == 1 ? KylinConfig.getInstanceFromEnv() : KylinConfig.createInstanceFromUri(args[1])); break; case "list": break; case "download": tool.copyParallel(KylinConfig.getInstanceFromEnv(), KylinConfig.createInstanceFromUri(args[1]), "/"); System.out.println("Metadata backed up to " + args[1]); break; case "fetch": tool.copy(KylinConfig.getInstanceFromEnv(), KylinConfig.createInstanceFromUri(args[1]), args[2], true); break; case "upload": tool.copyParallel(KylinConfig.createInstanceFromUri(args[1]), KylinConfig.getInstanceFromEnv(), "/"); System.out.println("Metadata restored from " + args[1]); break;
kylinConfig = KylinConfig.getInstanceFromEnv(); } else { kylinConfig = KylinConfig.createInstanceFromUri(dstCfgUri);
public static void dumpAndUploadKylinPropsAndMetadata(Set<String> dumpList, KylinConfigExt kylinConfig, String metadataUrl) throws IOException { try (AutoDeleteDirectory tmpDir = new AutoDeleteDirectory("kylin_job_meta", ""); AutoDeleteDirectory metaDir = tmpDir.child("meta")) { // dump metadata JobRelatedMetaUtil.dumpResources(kylinConfig, metaDir.getFile(), dumpList); // dump metadata dumpResources(kylinConfig, metaDir.getFile(), dumpList); // write kylin.properties Properties props = kylinConfig.exportToProperties(); props.setProperty("kylin.metadata.url", metadataUrl); File kylinPropsFile = new File(metaDir.getFile(), "kylin.properties"); try (FileOutputStream os = new FileOutputStream(kylinPropsFile)) { props.store(os, kylinPropsFile.getAbsolutePath()); } KylinConfig dstConfig = KylinConfig.createKylinConfig(props); //upload metadata new ResourceTool().copy(KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath()), dstConfig); } } }
KylinConfig dstConfig = KylinConfig.createInstanceFromUri(dest);
private void injest(File metaRoot) throws IOException { KylinConfig srcConfig = KylinConfig.createInstanceFromUri(metaRoot.getAbsolutePath()); TableMetadataManager srcMetadataManager = TableMetadataManager.getInstance(srcConfig); DataModelManager srcModelManager = DataModelManager.getInstance(srcConfig); HybridManager srcHybridManager = HybridManager.getInstance(srcConfig); CubeManager srcCubeManager = CubeManager.getInstance(srcConfig); CubeDescManager srcCubeDescManager = CubeDescManager.getInstance(srcConfig); checkAndMark(srcMetadataManager, srcModelManager, srcHybridManager, srcCubeManager, srcCubeDescManager); new ResourceTool().copy(srcConfig, kylinConfig, Lists.newArrayList(requiredResources)); // clear the cache Broadcaster.getInstance(kylinConfig).notifyClearAll(); ProjectManager projectManager = ProjectManager.getInstance(kylinConfig); for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) { logger.info("add " + tableDesc + " to " + targetProjectName); projectManager.addTableDescToProject(Lists.newArrayList(tableDesc.getIdentity()).toArray(new String[0]), targetProjectName); } for (CubeInstance cube : srcCubeManager.listAllCubes()) { logger.info("add " + cube + " to " + targetProjectName); projectManager.addModelToProject(cube.getModel().getName(), targetProjectName); projectManager.moveRealizationToProject(RealizationType.CUBE, cube.getName(), targetProjectName, null); } }
Preconditions.checkState(files.length == 1); String dumpDir = files[0].getAbsolutePath(); KylinConfig instanceFromUri = KylinConfig.createInstanceFromUri(dumpDir); NavigableSet<String> tables = new ResourceTool().list(instanceFromUri, "table"); NavigableSet<String> tableExds = new ResourceTool().list(instanceFromUri, "table_exd");
private static void copy(String srcUri, String dstUri) throws IOException { System.out.println("Copy from " + srcUri + " to " + dstUri); KylinConfig srcConfig = KylinConfig.createInstanceFromUri(srcUri); KylinConfig dstConfig = KylinConfig.createInstanceFromUri(dstUri); copy(srcConfig, dstConfig); }
public static void deployMetadata(String localMetaData) throws IOException { // install metadata to hbase new ResourceTool().reset(config()); new ResourceTool().copy(KylinConfig.createInstanceFromUri(localMetaData), config()); // update cube desc signature. for (CubeInstance cube : CubeManager.getInstance(config()).listAllCubes()) { CubeDescManager.getInstance(config()).updateCubeDesc(cube.getDescriptor());//enforce signature updating } }
@Test public void testCopy() throws IOException { KylinConfig dstConfig = KylinConfig.createInstanceFromUri(dstPath); ResourceStore srcStore = ResourceStore.getStore(KylinConfig.getInstanceFromEnv()); ResourceStore dstStore = ResourceStore.getStore(dstConfig); //metadata under source path and destination path are not equal before copy Assert.assertNotEquals(srcStore.listResources("/"), dstStore.listResources("/")); new ResourceTool().copy(KylinConfig.getInstanceFromEnv(), dstConfig, "/"); //After copy, two paths have same metadata Assert.assertEquals(srcStore.listResources("/"), dstStore.listResources("/")); } }
private void dumpResources(KylinConfig kylinConfig, File metaDir, ArrayList<String> dumpList) throws IOException { ResourceStore from = ResourceStore.getStore(kylinConfig); KylinConfig localConfig = KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath()); ResourceStore to = ResourceStore.getStore(localConfig); for (String path : dumpList) { InputStream in = from.getResource(path); if (in == null) throw new IllegalStateException("No resource found at -- " + path); long ts = from.getResourceTimestamp(path); to.putResource(path, in, ts); //The following log is duplicate with in ResourceStore //log.info("Dumped resource " + path + " to " + metaDir.getAbsolutePath()); } }
public static void deployMetadata() throws IOException { // install metadata to hbase ResourceTool.reset(config()); ResourceTool.copy(KylinConfig.createInstanceFromUri(AbstractKylinTestCase.LOCALMETA_TEST_DATA), config()); // update cube desc signature. for (CubeInstance cube : CubeManager.getInstance(config()).listAllCubes()) { cube.getDescriptor().setSignature(cube.getDescriptor().calculateSignature()); CubeManager.getInstance(config()).updateCube(cube); } }