protected String getModulesFile(String uri) { try { String contents = modMgr.read(uri).next().getContent(new StringHandle()).get(); return contents.replaceFirst("(\\(:|//)\\s+cache\\sbuster:.+\\n", ""); } catch (IllegalStateException e){ return null; } catch(Exception e) { e.printStackTrace(); } return null; }
DocumentPage page = docMgr.read("/generic/Pandakarlino.jpg", "/generic/dog.json", "/generic/foo1.txt", "/generic/foo.xml");
protected DocumentPage getDocs(QueryBatch batch) { GenericDocumentManager docMgr = batch.getClient().newDocumentManager(); if ( view != null ) docMgr.setSearchView(view); if ( categories != null ) docMgr.setMetadataCategories(categories); if ( nonDocumentFormat != null ) docMgr.setNonDocumentFormat(nonDocumentFormat); if ( consistentSnapshot == true ) { return ((GenericDocumentImpl) docMgr).read( batch.getServerTimestamp(), transform, batch.getItems() ); } else { return docMgr.read( transform, batch.getItems() ); } }
@Test public void testRunFlow() throws SAXException, IOException, ParserConfigurationException, XMLStreamException { addStagingDocs(); installModules(); assertEquals(2, getStagingDocCount()); assertEquals(0, getFinalDocCount()); getHubFlowRunnerConfig(); Flow flow1 = fm.getFlow("test", "my-test-flow1"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow1) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); getHubAdminConfig(); assertEquals(2, getStagingDocCount()); assertEquals(2, getFinalDocCount()); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized/harmonized1.xml"), finalDocMgr.read("/employee1.xml").next().getContent(new DOMHandle()).get() ); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized/harmonized2.xml"), finalDocMgr.read("/employee2.xml").next().getContent(new DOMHandle()).get()); DocumentMetadataHandle metadata = finalDocMgr.readMetadata("/employee1.xml", new DocumentMetadataHandle()); DocumentMetadataHandle.DocumentPermissions permissions = metadata.getPermissions(); assertEquals( permissions.get("harmonized-reader").toString(), "[READ]", "Default permissions on harmonized documents should contain harmonized-reader/read"); assertEquals(permissions.get("harmonized-updater").toString(), "[UPDATE]", "Default permissions on harmonized documents should contain harmonized-updater/update"); }
protected Document getModulesDocument(String uri) { return modMgr.read(uri).next().getContent(new DOMHandle()).get(); }
@Test public void testRunFlowWithBackwards() throws SAXException, IOException, ParserConfigurationException, XMLStreamException { addFinalDocs(); installModules(); assertEquals(0, getStagingDocCount()); assertEquals(2, getFinalDocCount()); getHubFlowRunnerConfig(); Flow flow1 = fm.getFlow("test", "my-test-flow1"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow1) .withBatchSize(10) .withThreadCount(1) .withSourceClient(getHubFlowRunnerConfig().newReverseFlowClient()) .withDestinationDatabase(HubConfig.DEFAULT_STAGING_NAME); flowRunner.run(); flowRunner.awaitCompletion(); getHubAdminConfig(); assertEquals(2, getStagingDocCount()); assertEquals(2, getFinalDocCount()); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized/harmonized1.xml"), stagingDocMgr.read("/employee1.xml").next().getContent(new DOMHandle()).get() ); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized/harmonized2.xml"), stagingDocMgr.read("/employee2.xml").next().getContent(new DOMHandle()).get()); DocumentMetadataHandle metadata = stagingDocMgr.readMetadata("/employee1.xml", new DocumentMetadataHandle()); DocumentMetadataHandle.DocumentPermissions permissions = metadata.getPermissions(); assertEquals(permissions.get("harmonized-reader").toString(), "[READ]", "Default permissions on harmonized documents should contain harmonized-reader/read"); assertEquals(permissions.get("harmonized-updater").toString(), "[UPDATE]", "Default permissions on harmonized documents should contain harmonized-updater/update"); }
@Test public void testEnvelopeSJS() throws IOException { scaffolding.createFlow(ENTITY, "testharmonize-sjs-json", FlowType.HARMONIZE, CodeFormat.JAVASCRIPT, DataFormat.JSON, false); //testing sjs JSON canonical instance Files.copy(getResourceStream("flow-runner-test/collector.sjs"), projectDir.resolve("plugins/entities/" + ENTITY + "/harmonize/testharmonize-sjs-json/collector.sjs"), StandardCopyOption.REPLACE_EXISTING); Files.copy(getResourceStream("flow-runner-test/contentTestingEnvelope.sjs"), projectDir.resolve("plugins/entities/" + ENTITY + "/harmonize/testharmonize-sjs-json/content.sjs"), StandardCopyOption.REPLACE_EXISTING); installUserModules(getHubAdminConfig(), false); Flow harmonizeFlow = fm.getFlow(ENTITY, "testharmonize-sjs-json", FlowType.HARMONIZE); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(harmonizeFlow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); JsonNode jsonEnvelope = finalDocMgr.read("1.json").next().getContent(new JacksonHandle()).get(); try { logger.debug(new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(jsonEnvelope)); } catch (JsonProcessingException e) { e.printStackTrace(); } assertNull(jsonEnvelope.get("envelope").get("instance").get("Person").get("$attachments")); assertNotNull(jsonEnvelope.get("envelope").get("attachments")); }
@Test public void testEnvelopeXQY() throws IOException { //testing xqy JSON canonical instance scaffolding.createFlow(ENTITY, "testharmonize-xqy-json", FlowType.HARMONIZE, CodeFormat.XQUERY, DataFormat.JSON, false); Files.copy(getResourceStream("flow-runner-test/collector2.xqy"), projectDir.resolve("plugins/entities/" + ENTITY + "/harmonize/testharmonize-xqy-json/collector.xqy"), StandardCopyOption.REPLACE_EXISTING); Files.copy(getResourceStream("flow-runner-test/content-testing-envelope.xqy"), projectDir.resolve("plugins/entities/" + ENTITY + "/harmonize/testharmonize-xqy-json/content.xqy"), StandardCopyOption.REPLACE_EXISTING); installUserModules(getHubAdminConfig(), false); Flow harmonizeFlow = fm.getFlow(ENTITY, "testharmonize-xqy-json", FlowType.HARMONIZE); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(harmonizeFlow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); JsonNode jsonEnvelope = finalDocMgr.read("2.json").next().getContent(new JacksonHandle()).get(); try { logger.debug(new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(jsonEnvelope)); } catch (JsonProcessingException e) { e.printStackTrace(); } assertNull(jsonEnvelope.get("envelope").get("instance").get("Person").get("$attachments")); assertNotNull(jsonEnvelope.get("envelope").get("attachments")); }
@Test public void runHarmonizeNoOptions() throws IOException, InterruptedException { deleteProjectDir(); createProjectDir(); //envConfig.setInitialized(true); //envConfig.setMlSettings(HubConfigBuilder.newHubConfigBuilder(PROJECT_PATH).withPropertiesFromEnvironment().build()); Path projectDir = Paths.get(".", PROJECT_PATH); scaffolding.createFlow(ENTITY, "sjs-json-harmonization-flow", FlowType.HARMONIZE, CodeFormat.JAVASCRIPT, DataFormat.JSON, false); Path harmonizeDir = projectDir.resolve("plugins/entities/" + ENTITY + "/harmonize"); FileUtil.copy(getResourceStream("flow-manager/sjs-harmonize-flow/headers.sjs"), harmonizeDir.resolve("sjs-json-harmonization-flow/headers.sjs").toFile()); installUserModules(getHubAdminConfig(), true); DocumentMetadataHandle meta = new DocumentMetadataHandle(); meta.getCollections().add(ENTITY); installStagingDoc("/staged.json", meta, "flow-manager/staged.json"); //envConfig.setMlSettings(HubConfigBuilder.newHubConfigBuilder(PROJECT_PATH).withPropertiesFromEnvironment().build()); setEnvConfig(); ObjectMapper mapper = new ObjectMapper(); JsonNode body = mapper.readTree("{\"batchSize\":1, \"threadCount\": 1}"); ResponseEntity<?> responseEntity = ec.runHarmonizeFlow(ENTITY, "sjs-json-harmonization-flow", body); assertEquals(HttpStatus.OK, responseEntity.getStatusCode()); // document takes a moment to arrive. Thread.sleep(3000); DocumentRecord doc = finalDocMgr.read("/staged.json").next(); JsonNode root = doc.getContent(new JacksonHandle()).get(); JsonNode env = root.path("envelope"); JsonNode headers = env.path("headers"); JsonNode optionNode = headers.path("test-option"); assertTrue(optionNode.isMissingNode()); }
DocumentRecord doc = finalDocMgr.read("/staged.json").next(); JsonNode root = doc.getContent(new JacksonHandle()).get(); JsonNode env = root.path("envelope");
String expected = getResource("flow-manager/final.json"); String actual = stagingDocMgr.read("/input.json").next().getContent(new StringHandle()).get(); JSONAssert.assertEquals(expected, actual, false);
@Test public void runXqyJsonFlowWithBinaryContent() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); enableTracing(); assertTrue(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeXqyJsonWithBinary"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(5, getFinalDocCount()); assertEquals(6, getTracingDocCount()); DocumentRecord doc = finalDocMgr.read("/doc/1.json").next(); String finalDoc= doc.getContent(new StringHandle()).get(); assertJsonEqual(getResource("tracing-test/traces/finalXqyJsonDoc.json"), finalDoc, true); JsonNode node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new JacksonHandle()).get(); assertEquals(4, node.get("trace").get("steps").size()); assertEquals("content", node.get("trace").get("steps").get(0).get("label").asText()); assertEquals(BINARY_HEX_ENCODED_XQY, node.get("trace").get("steps").get(0).get("output").asText().toLowerCase()); }
@Test public void runSjsJsonFlowWithBinaryContent() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); enableTracing(); assertTrue(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeSjsJsonWithBinary"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(5, getFinalDocCount()); assertEquals(6, getTracingDocCount()); DocumentRecord doc = finalDocMgr.read("1").next(); String finalDoc= doc.getContent(new StringHandle()).get(); assertJsonEqual(getResource("tracing-test/traces/finalSjsJsonDoc.json"), finalDoc, true); JsonNode node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new JacksonHandle()).get(); assertEquals(4, node.get("trace").get("steps").size()); assertEquals("content", node.get("trace").get("steps").get(0).get("label").asText()); assertEquals(BINARY_HEX_ENCODED_SJS, node.get("trace").get("steps").get(0).get("output").asText().toLowerCase()); }
moveMgr.stopJob(ticket); BytesHandle readContents = client.newDocumentManager().read("/doc/string", new BytesHandle()); System.out.println("Read doc, and format=[" + readContents.getFormat() + "]"); assertEquals("contents from db should match", docContents, new String(readContents.get()));
@Test public void testRunFlowWithHeader() throws SAXException, IOException, ParserConfigurationException, XMLStreamException { addStagingDocs(); HashMap<String, String> modules = new HashMap<>(); modules.put("/entities/test/harmonize/my-test-flow-with-header/flow.xml", "flow-manager-test/my-test-flow-with-header/flow.xml"); modules.put("/entities/test/harmonize/my-test-flow-with-header/collector.xqy", "flow-manager-test/my-test-flow-with-header/collector.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-header/content.xqy", "flow-manager-test/my-test-flow-with-header/content.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-header/headers.xqy", "flow-manager-test/my-test-flow-with-header/headers.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-header/triples.xqy", "flow-manager-test/my-test-flow-with-header/triples.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-header/writer.xqy", "flow-manager-test/my-test-flow-with-header/writer.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-header/main.xqy", "flow-manager-test/my-test-flow-with-header/main.xqy"); installModules(modules); assertEquals(2, getStagingDocCount()); assertEquals(0, getFinalDocCount()); getHubFlowRunnerConfig(); Flow flow1 = fm.getFlow("test", "my-test-flow-with-header"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow1) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); getHubAdminConfig(); assertEquals(2, getStagingDocCount()); assertEquals(2, getFinalDocCount()); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized-with-header/harmonized1.xml"), finalDocMgr.read("/employee1.xml").next().getContent(new DOMHandle()).get() ); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized-with-header/harmonized2.xml"), finalDocMgr.read("/employee2.xml").next().getContent(new DOMHandle()).get()); runInModules("xdmp:directory-delete(\"/entities/test/harmonize/my-test-flow-with-header/\")"); }
@Test public void testRunFlowWithAll() throws SAXException, IOException, ParserConfigurationException, XMLStreamException { addStagingDocs(); HashMap<String, String> modules = new HashMap<>(); modules.put("/entities/test/harmonize/my-test-flow-with-all/my-test-flow-with-all.xml", "flow-manager-test/my-test-flow-with-all/my-test-flow-with-all.xml"); modules.put("/entities/test/harmonize/my-test-flow-with-all/collector.xqy", "flow-manager-test/my-test-flow-with-all/collector.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-all/headers.xqy", "flow-manager-test/my-test-flow-with-all/headers.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-all/content.xqy", "flow-manager-test/my-test-flow-with-all/content.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-all/triples.xqy", "flow-manager-test/my-test-flow-with-all/triples.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-all/writer.xqy", "flow-manager-test/my-test-flow-with-all/writer.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-all/main.xqy", "flow-manager-test/my-test-flow-with-all/main.xqy"); installModules(modules); assertEquals(2, getStagingDocCount()); assertEquals(0, getFinalDocCount()); getHubFlowRunnerConfig(); Flow flow1 = fm.getFlow("test", "my-test-flow-with-all"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow1) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); getHubAdminConfig(); assertEquals(2, getStagingDocCount()); assertEquals(2, getFinalDocCount()); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized-with-all/harmonized1.xml"), finalDocMgr.read("/employee1.xml").next().getContent(new DOMHandle()).get() ); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized-with-all/harmonized2.xml"), finalDocMgr.read("/employee2.xml").next().getContent(new DOMHandle()).get()); runInModules("xdmp:directory-delete(\"/entities/test/harmonize/my-test-flow-with-all/\")"); }
@Test public void runXqyXmlFlowWithBinaryContent() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); enableTracing(); assertTrue(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeXqyXmlWithBinary"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(5, getFinalDocCount()); assertEquals(6, getTracingDocCount()); DocumentRecord doc = finalDocMgr.read("/doc/1.xml").next(); Document finalDoc = doc.getContent(new DOMHandle()).get(); assertXMLEqual(getXmlFromResource("tracing-test/traces/finalSjsXmlDoc.xml"), finalDoc); Document node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new DOMHandle()).get(); assertEquals(4, node.getElementsByTagName("step").getLength()); assertEquals("content", node.getElementsByTagName("label").item(0).getTextContent()); assertEquals(BINARY_HEX_ENCODED_XQY, node.getElementsByTagName("output").item(0).getTextContent().toLowerCase()); }
@Test public void runSjsXmlFlowWithBinaryContent() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); enableTracing(); assertTrue(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeSjsXmlWithBinary"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(5, getFinalDocCount()); assertEquals(6, getTracingDocCount()); DocumentRecord doc = finalDocMgr.read("1").next(); Document finalDoc = doc.getContent(new DOMHandle()).get(); //debugOutput(finalDoc); assertXMLEqual(getXmlFromResource("tracing-test/traces/finalSjsXmlDoc.xml"), finalDoc); Document node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new DOMHandle()).get(); assertEquals(4, node.getElementsByTagName("step").getLength()); assertEquals("content", node.getElementsByTagName("label").item(0).getTextContent()); assertEquals(BINARY_HEX_ENCODED_SJS, node.getElementsByTagName("output").item(0).getTextContent().toLowerCase()); }
@Test public void testProjectScaffolding() throws IOException { DatabaseClient stagingTriggersClient = null; DatabaseClient finalTriggersClient = null; DatabaseClient stagingSchemasClient = null; DatabaseClient finalSchemasClient = null; try { stagingTriggersClient = getClient(host, stagingPort, HubConfig.DEFAULT_STAGING_TRIGGERS_DB_NAME, user, password, stagingAuthMethod); finalTriggersClient = getClient(host, finalPort, HubConfig.DEFAULT_FINAL_TRIGGERS_DB_NAME, user, password, finalAuthMethod); stagingSchemasClient = getClient(host, stagingPort, HubConfig.DEFAULT_STAGING_SCHEMAS_DB_NAME, user, password, stagingAuthMethod); finalSchemasClient = getClient(host, finalPort, HubConfig.DEFAULT_FINAL_SCHEMAS_DB_NAME, user, password, finalAuthMethod); } catch (Exception e) { throw new RuntimeException(e); } //checking if modules are written to correct db assertNotNull(getModulesFile("/ext/sample-trigger.xqy")); assertNotNull(getModulesFile("/ext/sample-trigger.xqy")); //checking if tdes are written to correct db Document expectedXml = getXmlFromResource("data-hub-test/scaffolding/tdedoc.xml"); Document actualXml = stagingSchemasClient.newDocumentManager().read("/tde/tdedoc.xml").next().getContent(new DOMHandle()).get(); assertXMLEqual(expectedXml, actualXml); actualXml = finalSchemasClient.newDocumentManager().read("/tde/tdedoc.xml").next().getContent(new DOMHandle()).get(); assertXMLEqual(expectedXml, actualXml); //checking if triggers are written assertTrue(stagingTriggersClient.newServerEval().xquery("fn:count(fn:doc())").eval().next().getNumber().intValue() == 1); assertTrue(finalTriggersClient.newServerEval().xquery("fn:count(fn:doc())").eval().next().getNumber().intValue() == 1); }