private void deserialize(Node xml) { NodeList children = xml.getChildNodes(); for (int i = 0; i < children.getLength(); i++) { Node node = children.item(i); if (node.getNodeType() != Node.ELEMENT_NODE) { continue; } String nodeName = node.getLocalName(); switch(nodeName) { case "name": this.name = node.getTextContent(); break; case "flows": deserialize(node); break; case "flow": flows.add(FlowManager.flowFromXml((Element)node)); break; } } }
private void runHarmonizeFlow(String flowName, DatabaseClient srcClient, String destDb) { Flow harmonizeFlow = flowManager.getFlow("SupportCall", flowName, FlowType.HARMONIZE); FlowRunner flowRunner = flowManager.newFlowRunner().withFlow(harmonizeFlow).withBatchSize(3).withThreadCount(1) .withSourceClient(srcClient).withDestinationDatabase(destDb) .onItemComplete((String jobId, String itemId) -> { logger.debug("Completed item " + itemId); }).onItemFailed((String jobId, String itemId) -> { logger.debug("Failed item " + itemId); }); flowRunner.run(); flowRunner.awaitCompletion(); int finalCount = getFinalDocCount(); assertTrue( "After save, pii, this value is 16, before, it's 15. Actual is " + finalCount, finalCount == 15 || finalCount == 16); }
@Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { if (isFlowPropertiesFile(file) && modulesManager.hasFileBeenModifiedSinceLastLoaded(file.toFile())) { Flow flow = flowManager.getFlowFromProperties(file); StringHandle handle = new StringHandle(flow.serialize()); handle.setFormat(Format.XML); documentWriteSet.add(flow.getFlowDbPath(), handle); modulesManager.saveLastLoadedTimestamp(file.toFile(), new Date()); } return FileVisitResult.CONTINUE; } });
}); List<String> legacyFlows = flowManager.getLegacyFlows(); assertEquals(8, legacyFlows.size(), String.join("\n", legacyFlows)); assertEquals(8, flowManager.updateLegacyFlows("2.0.0").size()); // don't change this value assertEquals(0, flowManager.getLegacyFlows().size()); legacyFlows = flowManager.getLegacyFlows(); assertEquals(8, legacyFlows.size(), String.join("\n", legacyFlows)); assertEquals(8, flowManager.updateLegacyFlows("1.1.5").size()); assertEquals(0, flowManager.getLegacyFlows().size()); legacyFlows = flowManager.getLegacyFlows(); assertEquals(4, legacyFlows.size(), String.join("\n", legacyFlows)); assertEquals(4, flowManager.updateLegacyFlows("2.0.0").size()); assertEquals(0, flowManager.getLegacyFlows().size()); installUserModules(getHubAdminConfig(), true);
Flow flow = flowManager.getFlow(ENTITY, flowName, FlowType.INPUT);
assertEquals(0, fm.getLegacyFlows().size()); List<String> legacyFlows = fm.getLegacyFlows(); assertEquals(8, legacyFlows.size()); legacyFlows.sort(String::compareToIgnoreCase);
@Test public void testGetLocalFlows() throws IOException { createProjectDir(PROJECT_PATH); scaffolding.createEntity("my-entity"); assertEquals(0, fm.getLocalFlows().size()); CodeFormat[] codeFormats = new CodeFormat[] { CodeFormat.JAVASCRIPT, CodeFormat.XQUERY }; DataFormat[] dataFormats = new DataFormat[] { DataFormat.JSON, DataFormat.XML }; FlowType[] flowTypes = new FlowType[] { FlowType.INPUT, FlowType.HARMONIZE }; for (CodeFormat codeFormat : codeFormats) { for (DataFormat dataFormat : dataFormats) { for (FlowType flowType : flowTypes) { String flowName = flowType.toString() + "-" + codeFormat.toString() + "-" + dataFormat.toString(); scaffolding.createFlow("my-entity", flowName, flowType, codeFormat, dataFormat, false); } } } List<Flow> flows = fm.getLocalFlows(); assertEquals(8, flows.size()); for (Flow flow : flows) { assertEquals("my-entity", flow.getEntityName()); } FileUtils.deleteDirectory(Paths.get("./del-me-dir").toFile()); }
installModule("/entities/test/harmonize/my-test-flow2/my-test-flow2.xml", "flow-manager-test/my-test-flow1/my-test-flow2.xml"); List<Flow> flows = fm.getFlows("test"); assertEquals(2, flows.size());
Flow flow = flowManager.getFlow(ENTITY, flowName, FlowType.HARMONIZE);
@Override public void execute(CommandContext context) { List<String> legacyFlows = flowManager.getLegacyFlows(); if (legacyFlows.size() > 0) { throw new LegacyFlowsException(legacyFlows);
Flow harmonizeFlow = fm.getFlow(ENTITY, "testharmonize", FlowType.HARMONIZE); HashMap<String, Object> options = new HashMap<>(); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(harmonizeFlow) .withBatchSize(10)
Flow flow = flowManager.getFlow(ENTITY, flowName, FlowType.HARMONIZE);
@Override public List<Flow> getFlows(String entityName) { RequestParameters params = new RequestParameters(); params.add("entity-name", entityName); ServiceResultIterator resultItr = this.getServices().get(params); if (resultItr == null || ! resultItr.hasNext()) { return null; } ServiceResult res = resultItr.next(); DOMHandle handle = new DOMHandle(); Document parent = res.getContent(handle).get(); NodeList children = parent.getDocumentElement().getChildNodes(); ArrayList<Flow> flows = null; if (children.getLength() > 0) { flows = new ArrayList<>(); } Node node; for (int i = 0; i < children.getLength(); i++) { node = children.item(i); if (node.getNodeType() == Node.ELEMENT_NODE) { flows.add(FlowManager.flowFromXml((Element)children.item(i))); } } return flows; }
@Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { if (isFlowPropertiesFile(file) && modulesManager.hasFileBeenModifiedSinceLastLoaded(file.toFile())) { Flow flow = flowManager.getFlowFromProperties(file); StringHandle handle = new StringHandle(flow.serialize()); handle.setFormat(Format.XML); documentWriteSet.add(flow.getFlowDbPath(), handle); modulesManager.saveLastLoadedTimestamp(file.toFile(), new Date()); } return FileVisitResult.CONTINUE; } });
@Test public void testRunFlowWithHeader() throws SAXException, IOException, ParserConfigurationException, XMLStreamException { addStagingDocs(); HashMap<String, String> modules = new HashMap<>(); modules.put("/entities/test/harmonize/my-test-flow-with-header/flow.xml", "flow-manager-test/my-test-flow-with-header/flow.xml"); modules.put("/entities/test/harmonize/my-test-flow-with-header/collector.xqy", "flow-manager-test/my-test-flow-with-header/collector.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-header/content.xqy", "flow-manager-test/my-test-flow-with-header/content.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-header/headers.xqy", "flow-manager-test/my-test-flow-with-header/headers.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-header/triples.xqy", "flow-manager-test/my-test-flow-with-header/triples.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-header/writer.xqy", "flow-manager-test/my-test-flow-with-header/writer.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-header/main.xqy", "flow-manager-test/my-test-flow-with-header/main.xqy"); installModules(modules); assertEquals(2, getStagingDocCount()); assertEquals(0, getFinalDocCount()); getHubFlowRunnerConfig(); Flow flow1 = fm.getFlow("test", "my-test-flow-with-header"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow1) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); getHubAdminConfig(); assertEquals(2, getStagingDocCount()); assertEquals(2, getFinalDocCount()); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized-with-header/harmonized1.xml"), finalDocMgr.read("/employee1.xml").next().getContent(new DOMHandle()).get() ); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized-with-header/harmonized2.xml"), finalDocMgr.read("/employee2.xml").next().getContent(new DOMHandle()).get()); runInModules("xdmp:directory-delete(\"/entities/test/harmonize/my-test-flow-with-header/\")"); }
assertEquals(0, getJobDocCount()); Flow flow = flowManager.getFlow(ENTITY, flowName, FlowType.INPUT); String inputPath = getResourceFile("e2e-test/input/input" + fileSuffix + "." + dataFormat.toString()).getAbsolutePath(); String basePath = getResourceFile("e2e-test/input").getAbsolutePath();
@Override public Flow getFlow(String entityName, String flowName, FlowType flowType) { RequestParameters params = new RequestParameters(); params.add("entity-name", entityName); params.add("flow-name", flowName); if (flowType != null) { params.add("flow-type", flowType.toString()); } ServiceResultIterator resultItr = this.getServices().get(params); if (resultItr == null || ! resultItr.hasNext()) { return null; } ServiceResult res = resultItr.next(); DOMHandle handle = new DOMHandle(); Document parent = res.getContent(handle).get(); return FlowManager.flowFromXml(parent.getDocumentElement()); }
@Test public void testGetFlowFromProperties() throws IOException { scaffolding.createEntity("my-entity"); allCombos((codeFormat, dataFormat, flowType, useEs) -> { String flowName = flowType.toString() + "-" + codeFormat.toString() + "-" + dataFormat.toString(); scaffolding.createFlow("my-entity", flowName, flowType, codeFormat, dataFormat, false); }); allCombos((codeFormat, dataFormat, flowType, useEs) -> { String flowName = flowType.toString() + "-" + codeFormat.toString() + "-" + dataFormat.toString(); Path propertiesFile = Paths.get(PROJECT_PATH, "plugins", "entities", "my-entity", flowType.toString(), flowName, flowName + ".properties"); Flow flow = fm.getFlowFromProperties(propertiesFile); assertEquals(flowName, flow.getName()); assertEquals("my-entity", flow.getEntityName()); assertEquals(codeFormat, flow.getCodeFormat()); assertEquals(dataFormat, flow.getDataFormat()); assertEquals(flowType, flow.getType()); }); deleteProjectDir(); }
@Test public void testRunFlowWithAll() throws SAXException, IOException, ParserConfigurationException, XMLStreamException { addStagingDocs(); HashMap<String, String> modules = new HashMap<>(); modules.put("/entities/test/harmonize/my-test-flow-with-all/my-test-flow-with-all.xml", "flow-manager-test/my-test-flow-with-all/my-test-flow-with-all.xml"); modules.put("/entities/test/harmonize/my-test-flow-with-all/collector.xqy", "flow-manager-test/my-test-flow-with-all/collector.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-all/headers.xqy", "flow-manager-test/my-test-flow-with-all/headers.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-all/content.xqy", "flow-manager-test/my-test-flow-with-all/content.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-all/triples.xqy", "flow-manager-test/my-test-flow-with-all/triples.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-all/writer.xqy", "flow-manager-test/my-test-flow-with-all/writer.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-all/main.xqy", "flow-manager-test/my-test-flow-with-all/main.xqy"); installModules(modules); assertEquals(2, getStagingDocCount()); assertEquals(0, getFinalDocCount()); getHubFlowRunnerConfig(); Flow flow1 = fm.getFlow("test", "my-test-flow-with-all"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow1) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); getHubAdminConfig(); assertEquals(2, getStagingDocCount()); assertEquals(2, getFinalDocCount()); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized-with-all/harmonized1.xml"), finalDocMgr.read("/employee1.xml").next().getContent(new DOMHandle()).get() ); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized-with-all/harmonized2.xml"), finalDocMgr.read("/employee2.xml").next().getContent(new DOMHandle()).get()); runInModules("xdmp:directory-delete(\"/entities/test/harmonize/my-test-flow-with-all/\")"); }
@Test public void getTestFlow() { installModule("/entities/test/harmonize/my-test-flow1/my-test-flow1.xml", "flow-manager-test/my-test-flow1/my-test-flow1-json.xml"); Flow flow1 = fm.getFlow("test", "my-test-flow1"); assertEquals("my-test-flow1", flow1.getName()); assertEquals(CodeFormat.JAVASCRIPT, flow1.getCodeFormat()); assertEquals(DataFormat.JSON, flow1.getDataFormat()); assertEquals("test", flow1.getEntityName()); assertEquals(FlowType.HARMONIZE, flow1.getType()); Collector c = flow1.getCollector(); assertEquals(CodeFormat.JAVASCRIPT, c.getCodeFormat()); assertEquals("/entities/test/harmonize/my-test-flow1/collector.sjs", c.getModule()); MainPlugin main = flow1.getMain(); assertEquals(CodeFormat.JAVASCRIPT, main.getCodeFormat()); assertEquals("/entities/test/harmonize/my-test-flow1/main.sjs", main.getModule()); }