private PropertiesModuleManager getModulesManager() { String timestampFile = hubConfig.getHubProject().getUserModulesDeployTimestampFile(); PropertiesModuleManager pmm = new PropertiesModuleManager(timestampFile); // Need to delete ml-javaclient-utils timestamp file as well as modules present in the standard gradle locations are now // loaded by the modules loader in the parent class which adds these entries to the ml-javaclient-utils timestamp file String filePath = hubConfig.getAppConfig().getModuleTimestampsPath(); File defaultTimestampFile = new File(filePath); if (forceLoad) { pmm.deletePropertiesFile(); if (defaultTimestampFile.exists()){ defaultTimestampFile.delete(); } } return pmm; }
public FlowRunnerImpl(HubConfig hubConfig) { this.hubConfig = hubConfig; this.stagingClient = hubConfig.newStagingClient(); this.destinationDatabase = hubConfig.getDbName(DatabaseKind.FINAL); }
public String getDHFVersion() { return hubConfig.getDHFVersion(); }
@Override public boolean saveDbIndexes() { try { Path dir = hubConfig.getEntityDatabaseDir(); File finalFile = Paths.get(dir.toString(), HubConfig.FINAL_ENTITY_DATABASE_FILE).toFile(); File stagingFile = Paths.get(dir.toString(), HubConfig.STAGING_ENTITY_DATABASE_FILE).toFile(); if (!dir.toFile().exists()) { dir.toFile().mkdirs(); } List<JsonNode> entities = getAllEntities(); if (entities.size() > 0) { DbIndexGenerator generator = new DbIndexGenerator(hubConfig.newReverseFlowClient()); String indexes = generator.getIndexes(entities); // in order to make entity indexes ml-app-deployer compatible, add database-name keys. // ml-app-deployer removes these keys upon sending to marklogic. ObjectNode indexNode = (ObjectNode) mapper.readTree(indexes); indexNode.put("database-name", "%%mlFinalDbName%%"); mapper.writerWithDefaultPrettyPrinter().writeValue(finalFile, indexNode); indexNode.put("database-name", "%%mlStagingDbName%%"); mapper.writerWithDefaultPrettyPrinter().writeValue(stagingFile, indexNode); return true; } } catch (IOException e) { e.printStackTrace(); } return false; }
@Test public void testInit() throws IOException { HubConfig config = getHubFlowRunnerConfig(); config.createProject(PROJECT_PATH); config.setHttpName(DatabaseKind.STAGING, "my-crazy-test-staging"); config.setDbName(DatabaseKind.STAGING, "my-crazy-test-staging"); config.setForestsPerHost(DatabaseKind.STAGING, 100); config.setPort(DatabaseKind.STAGING, 1111); config.setHttpName(DatabaseKind.FINAL, "my-crazy-test-final"); config.setDbName(DatabaseKind.FINAL, "my-crazy-test-final"); config.setForestsPerHost(DatabaseKind.FINAL, 100); config.setPort(DatabaseKind.FINAL, 2222); config.setHttpName(DatabaseKind.JOB, "my-crazy-test-trace"); config.setDbName(DatabaseKind.JOB, "my-crazy-test-trace"); config.setForestsPerHost(DatabaseKind.JOB, 100); config.setPort(DatabaseKind.JOB, 3333); config.setForestsPerHost(DatabaseKind.MODULES,3); config.setForestsPerHost(DatabaseKind.STAGING_TRIGGERS, 4); config.setForestsPerHost(DatabaseKind.STAGING_SCHEMAS, 5); config.setHubRoleName("myrole"); config.setHubUserName("myuser"); config.initHubProject(); assertEquals(config.getAppConfig().getHost(), props.getProperty("mlHost"));
DatabaseClient stagingClient = hubConfig.newStagingClient(); DatabaseClient finalClient = hubConfig.newFinalClient(); Path userModulesPath = hubConfig.getHubPluginsDir(); String baseDir = userModulesPath.normalize().toAbsolutePath().toString(); Path startPath = userModulesPath.resolve("entities"); Path dir = Paths.get(hubConfig.getHubProject().getProjectDirString(), HubConfig.ENTITY_CONFIG_DIR); if (!dir.toFile().exists()) { dir.toFile().mkdirs(); XMLDocumentManager documentManager = hubConfig.newModulesDbClient().newXMLDocumentManager(); DocumentWriteSet documentWriteSet = documentManager.newWriteSet();
@Test public void testHubInfo() { HubConfig config = getHubFlowRunnerConfig(); ObjectMapper objmapper = new ObjectMapper(); try { JsonNode jsonNode = objmapper.readTree(config.getInfo()); assertEquals(jsonNode.get("stagingDbName").asText(), config.getDbName(DatabaseKind.STAGING)); assertEquals(jsonNode.get("stagingHttpName").asText(), config.getHttpName(DatabaseKind.STAGING)); assertEquals(jsonNode.get("finalForestsPerHost").asInt(), (int) config.getForestsPerHost(DatabaseKind.FINAL)); assertEquals(jsonNode.get("finalPort").asInt(), (int) config.getPort(DatabaseKind.FINAL)); } catch (Exception e) { throw new DataHubConfigurationException("Your datahub configuration could not serialize"); } } }
@Override public void execute(CommandContext context) { String timestampFile = hubConfig.getHubProject().getHubModulesDeployTimestampFile(); PropertiesModuleManager propsManager = new PropertiesModuleManager(timestampFile); propsManager.deletePropertiesFile(); DatabaseClient modulesClient = hubConfig.newModulesDbClient(); if (caughtException == null) { modulesLoader.loadModules("classpath*:/ml-modules", new DefaultModulesFinder(), modulesClient); modulesLoader.loadModules("classpath*:/ml-modules-staging", new SearchOptionsFinder(), hubConfig.newStagingClient()); modulesLoader.loadModules("classpath*:/ml-modules-traces", new SearchOptionsFinder(), hubConfig.newJobDbClient()); modulesLoader.loadModules("classpath*:/ml-modules-jobs", new SearchOptionsFinder(), hubConfig.newJobDbClient());
@Test public void getInputFlowOptions() throws Exception { Map<String, Object> options = ec.getInputFlowOptions("test-entity", "flow-name"); JSONAssert.assertEquals("{ \"input_file_path\": " + hubConfig.getHubProject().getProjectDirString() + " }", new ObjectMapper().writeValueAsString(options), true); }
throw new DataHubConfigurationException("DHF " + hubConfig.getDHFVersion() +" cannot deploy security to server version " + serverVersion); String modulesDatabaseName = hubConfig.getAppConfig().getModulesDatabaseName(); ManageConfig manageConfig = context.getManageClient().getManageConfig(); String securityUsername = manageConfig.getSecurityUsername(); String securityPassword = manageConfig.getSecurityPassword(); DatabaseClient installerClient = DatabaseClientFactory.newClient( hubConfig.getHost(), 8000, "Security", String modulesDatabaseName = hubConfig.getAppConfig().getModulesDatabaseName(); ManageClient manageClient = context.getManageClient();
public String getMarkLogicVersion() { // this call specifically needs to access marklogic without a known database ServerEvaluationCall eval = hubConfig.getAppConfig().newAppServicesDatabaseClient(null).newServerEval(); String xqy = "xdmp:version()"; EvalResultIterator result = eval.xquery(xqy).eval(); if (result.hasNext()) { return result.next().getString(); } else { throw new RuntimeException("Couldn't determine MarkLogic Version"); } }
@Override public void execute(CommandContext context) { AppConfig appConfig = context.getAppConfig(); DatabaseClient client = appConfig.newDatabaseClient(); EntityServicesManager mgr = new EntityServicesManager(client); CodeGenerationRequest request = createCodeGenerationRequest(); List<File> entityFiles = findEntityFiles(); if (!entityFiles.isEmpty()) { //create map of entity name -> entity definition file Map<String,File> entityNameFileMap = createEntityNameFileMap(entityFiles); logger.debug("Found the following entities->files: {} " + entityNameFileMap); filterEntities(entityNameFileMap); if (!entityNameFileMap.isEmpty()) { logger.warn("About to generate a template for the following entities: {} into directory {} ", entityNameFileMap.keySet(), hubConfig.getAppConfig().getSchemasPath()); for (File f : entityNameFileMap.values()) { GeneratedCode code = loadModelDefinition(request, f, mgr); generateExtractionTemplate(appConfig, code); } } } else { logger.info("No data hub entity files found under {} or its sub-directories.", hubConfig.getHubEntitiesDir()); } }
String modulesDatabaseName = hubConfig.getAppConfig().getModulesDatabaseName(); ManageConfig manageConfig = context.getManageClient().getManageConfig(); String securityUsername = manageConfig.getSecurityUsername(); String securityPassword = manageConfig.getSecurityPassword(); DatabaseClient installerClient = DatabaseClientFactory.newClient( hubConfig.getHost(), 8000, "Security",
bean.setUsername(hubConfig.getAppConfig().getAppServicesUsername()); bean.setPassword(hubConfig.getAppConfig().getAppServicesPassword()); bean.setModules_root("/"); if (hubConfig.getIsHostLoadBalancer()) { bean.setRestrict_hosts(true);
/** * When either staging-database.json or final-database.json is processed (and in the case of staging-database.json, * it could be under the hub config or user config directory), need to check to see if a same-named file is in the * entity config directory, where entity-specific database configuration is written. If so, it needs to be merged in * with the file being processed so that none of the entity configuration is lost. * * @param payloadNode * @return * @throws IOException */ private ObjectNode mergePayloadWithEntityConfigFileIfItExists(ObjectNode payloadNode) throws IOException { if (hubConfig.getEntityDatabaseDir() != null && this.databaseFilename != null) { File entityDatabaseDir = hubConfig.getEntityDatabaseDir().toFile(); if (entityDatabaseDir != null) { File entityDatabaseFile = new File(entityDatabaseDir, this.databaseFilename); if (entityDatabaseFile != null && entityDatabaseFile.exists()) { if (logger.isDebugEnabled()) { logger.debug("Merging in file: " + entityDatabaseFile.getAbsolutePath()); } ObjectNode entityConfigNode = (ObjectNode) ObjectMapperFactory.getObjectMapper().readTree(entityDatabaseFile); return JsonNodeUtil.mergeObjectNodes(payloadNode, entityConfigNode); } } } return payloadNode; }
@Test public void getFlowMlcpOptionsFromFile() throws Exception { setEnvConfig(); Map<String, Object> options = fm.getFlowMlcpOptionsFromFile("test-entity", "test-flow"); JSONAssert.assertEquals("{ \"input_file_path\": " + hubConfig.getHubProject().getProjectDirString() + " }", new ObjectMapper().writeValueAsString(options), true); }
protected void installUserModules(HubConfig hubConfig, boolean force) { logger.debug("Installing user modules into MarkLogic"); List<Command> commands = new ArrayList<>(); loadUserModulesCommand.setForceLoad(force); commands.add(loadUserModulesCommand); LoadModulesCommand loadModulesCommand = new LoadModulesCommand(); commands.add(loadModulesCommand); SimpleAppDeployer deployer = new SimpleAppDeployer(((HubConfigImpl)hubConfig).getManageClient(), ((HubConfigImpl)hubConfig).getAdminManager()); deployer.setCommands(commands); deployer.deploy(hubConfig.getAppConfig()); }
getModulesFile("/marklogic.rest.transform/test-input-transform/assets/transform.xqy")); String timestampFile = hubConfig.getHubProject().getUserModulesDeployTimestampFile(); PropertiesModuleManager propsManager = new PropertiesModuleManager(timestampFile); propsManager.initialize();
@Test public void testAmpLoading() { HubConfig config = getHubAdminConfig(); LoadHubModulesCommand loadHubModulesCommand = new LoadHubModulesCommand(); loadHubModulesCommand.setHubConfig(config); ManageClient manageClient = new ManageClient(new com.marklogic.mgmt.ManageConfig(host, 8002, secUser, secPassword)); CommandContext commandContext = new CommandContext(config.getAppConfig(), manageClient, null); deployHubAmpsCommand.setHubConfig(config); deployHubAmpsCommand.execute(commandContext); } }
getModulesFile("/marklogic.rest.transform/test-input-transform/assets/transform.xqy")); String timestampFile = hubConfig.getHubProject().getUserModulesDeployTimestampFile(); PropertiesModuleManager propsManager = new PropertiesModuleManager(timestampFile); propsManager.initialize();