How to use

FlowManagerImpl

in

com.marklogic.hub.impl

@Override public Flow getFlow(String entityName, String flowName) {
  return getFlow(entityName, flowName, null);
}

@JsonIgnore
public void refreshProject(Properties properties, boolean loadGradleProperties) {
  loadConfigurationFromProperties(properties, loadGradleProperties);
  flowManager.setupClient();
  dataHub.wireClient();
  versions.setupClient();
}

@Override public List<String> getLegacyFlows() {
  List<String> oldFlows = new ArrayList<>();
  Path entitiesDir = hubConfig.getHubEntitiesDir();
  File[] entityDirs = entitiesDir.toFile().listFiles(pathname -> pathname.isDirectory());
  if (entityDirs != null) {
    for (File entityDir : entityDirs) {
      Path inputDir = entityDir.toPath().resolve("input");
      Path harmonizeDir = entityDir.toPath().resolve("harmonize");
      File[] inputFlows = inputDir.toFile().listFiles((pathname) -> pathname.isDirectory() && !pathname.getName().equals("REST"));
      addLegacyFlowToList(oldFlows, entityDir, inputFlows);
      File[] harmonizeFlows = harmonizeDir.toFile().listFiles((pathname) -> pathname.isDirectory() && !pathname.getName().equals("REST"));
      addLegacyFlowToList(oldFlows, entityDir, harmonizeFlows);
    }
  }
  return oldFlows;
}

@Test
public void testEnvelopeXQY() throws IOException {
  //testing xqy JSON canonical instance
  scaffolding.createFlow(ENTITY, "testharmonize-xqy-json", FlowType.HARMONIZE,
      CodeFormat.XQUERY, DataFormat.JSON, false);
  Files.copy(getResourceStream("flow-runner-test/collector2.xqy"),
      projectDir.resolve("plugins/entities/" + ENTITY + "/harmonize/testharmonize-xqy-json/collector.xqy"),
      StandardCopyOption.REPLACE_EXISTING);
  Files.copy(getResourceStream("flow-runner-test/content-testing-envelope.xqy"),
      projectDir.resolve("plugins/entities/" + ENTITY + "/harmonize/testharmonize-xqy-json/content.xqy"),
      StandardCopyOption.REPLACE_EXISTING);
  installUserModules(getHubAdminConfig(), false);
  Flow harmonizeFlow = fm.getFlow(ENTITY, "testharmonize-xqy-json",
      FlowType.HARMONIZE);
  FlowRunner flowRunner = fm.newFlowRunner()
      .withFlow(harmonizeFlow)
      .withBatchSize(10)
      .withThreadCount(1);
  flowRunner.run();
  flowRunner.awaitCompletion();
  JsonNode jsonEnvelope = finalDocMgr.read("2.json").next().getContent(new JacksonHandle()).get();
  try {
    logger.debug(new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(jsonEnvelope));
  } catch (JsonProcessingException e) {
    e.printStackTrace();
  }
  assertNull(jsonEnvelope.get("envelope").get("instance").get("Person").get("$attachments"));
  assertNotNull(jsonEnvelope.get("envelope").get("attachments"));
}

@Override public List<Flow> getLocalFlowsForEntity(String entityName) {
  return getLocalFlowsForEntity(entityName, null);
}

@Override public Flow getFlowFromProperties(Path propertiesFile) {
  String quotedSeparator = Pattern.quote(File.separator);
  /* Extract flowName and entityName from ..../plugins/entities/<entityName>/
   * input|harmonize/<flowName>/flowName.properties
   */
  String floweRegex = ".+" + "plugins" + quotedSeparator + "entities" + quotedSeparator + "(.+)"+ quotedSeparator 
      +"(input|harmonize)" + quotedSeparator + "(.+)" + quotedSeparator + ".+";        
  FlowType flowType = propertiesFile.toString().replaceAll(floweRegex, "$2").equals("input")
      ? FlowType.INPUT : FlowType.HARMONIZE;
  String entityName = propertiesFile.toString().replaceAll(floweRegex, "$1");
  return getLocalFlow(entityName, propertiesFile.getParent(), flowType);
}

@Override public List<Flow> getFlows(String entityName) {
  RequestParameters params = new RequestParameters();
  params.add("entity-name", entityName);
  ServiceResultIterator resultItr = this.getServices().get(params);
  if (resultItr == null || ! resultItr.hasNext()) {
    return null;
  }
  ServiceResult res = resultItr.next();
  DOMHandle handle = new DOMHandle();
  Document parent = res.getContent(handle).get();
  NodeList children = parent.getDocumentElement().getChildNodes();
  ArrayList<Flow> flows = null;
  if (children.getLength() > 0) {
    flows = new ArrayList<>();
  }
  Node node;
  for (int i = 0; i < children.getLength(); i++) {
    node = children.item(i);
    if (node.getNodeType() == Node.ELEMENT_NODE) {
      flows.add(FlowManager.flowFromXml((Element)children.item(i)));
    }
  }
  return flows;
}

List<String> flows = flowManager.updateLegacyFlows(currentVersion);
if (updatedFlows != null) {
  updatedFlows.addAll(flows);

@Test
public void testEnvelopeSJS() throws IOException {
  scaffolding.createFlow(ENTITY, "testharmonize-sjs-json", FlowType.HARMONIZE,
      CodeFormat.JAVASCRIPT, DataFormat.JSON, false);
  //testing sjs JSON canonical instance
  Files.copy(getResourceStream("flow-runner-test/collector.sjs"),
      projectDir.resolve("plugins/entities/" + ENTITY + "/harmonize/testharmonize-sjs-json/collector.sjs"),
      StandardCopyOption.REPLACE_EXISTING);
  Files.copy(getResourceStream("flow-runner-test/contentTestingEnvelope.sjs"),
      projectDir.resolve("plugins/entities/" + ENTITY + "/harmonize/testharmonize-sjs-json/content.sjs"),
      StandardCopyOption.REPLACE_EXISTING);
  installUserModules(getHubAdminConfig(), false);
  Flow harmonizeFlow = fm.getFlow(ENTITY, "testharmonize-sjs-json",
      FlowType.HARMONIZE);
  FlowRunner flowRunner = fm.newFlowRunner()
      .withFlow(harmonizeFlow)
      .withBatchSize(10)
      .withThreadCount(1);
  flowRunner.run();
  flowRunner.awaitCompletion();
  JsonNode jsonEnvelope = finalDocMgr.read("1.json").next().getContent(new JacksonHandle()).get();
  try {
    logger.debug(new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(jsonEnvelope));
  } catch (JsonProcessingException e) {
    e.printStackTrace();
  }
  assertNull(jsonEnvelope.get("envelope").get("instance").get("Person").get("$attachments"));
  assertNotNull(jsonEnvelope.get("envelope").get("attachments"));
}

@Override public List<Flow> getLocalFlows() {
  List<Flow> flows = new ArrayList<>();
  Path entitiesDir = hubConfig.getHubEntitiesDir();
  File[] entities = entitiesDir.toFile().listFiles((pathname -> pathname.isDirectory()));
  if (entities != null) {
    for (File entity : entities) {
      String entityName = entity.getName();
      flows.addAll(getLocalFlowsForEntity(entityName));
    }
  }
  return flows;
}

if (inputFlows != null) {
  for (File inputFlow : inputFlows) {
    Flow flow = getLocalFlow(entityName, inputFlow.toPath(), FlowType.INPUT);
    if (flow != null) {
      flows.add(flow);
if (harmonizeFlows != null) {
  for (File harmonizeFlow : harmonizeFlows) {
    Flow flow = getLocalFlow(entityName, harmonizeFlow.toPath(), FlowType.HARMONIZE);
    if (flow != null) {
      flows.add(flow);

@Override public Flow getFlow(String entityName, String flowName, FlowType flowType) {
  RequestParameters params = new RequestParameters();
  params.add("entity-name", entityName);
  params.add("flow-name", flowName);
  if (flowType != null) {
    params.add("flow-type", flowType.toString());
  }
  ServiceResultIterator resultItr = this.getServices().get(params);
  if (resultItr == null || ! resultItr.hasNext()) {
    return null;
  }
  ServiceResult res = resultItr.next();
  DOMHandle handle = new DOMHandle();
  Document parent = res.getContent(handle).get();
  return FlowManager.flowFromXml(parent.getDocumentElement());
}

@Test
public void runJSONErrorFlowWithoutTracing() {
  assertEquals(0, getFinalDocCount());
  assertEquals(0, getTracingDocCount());
  Tracing t = Tracing.create(flowRunnerClient);
  assertFalse(t.isEnabled());
  Flow flow = fm.getFlow("trace-entity", "tracemeJSONError");
  FlowRunner flowRunner = fm.newFlowRunner()
    .withFlow(flow)
    .withBatchSize(10)
    .withThreadCount(1);
  flowRunner.run();
  flowRunner.awaitCompletion();
  assertEquals(0, getFinalDocCount());
  assertEquals(5, getTracingDocCount());
  JsonNode node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new JacksonHandle()).get();
  System.out.println(node.asText());
  assertEquals(1, node.get("trace").get("steps").size());
  assertEquals("content", node.get("trace").get("steps").get(0).get("label").asText());
}

public void wireClients() {        
  fm.setupClient();
  dataHub.wireClient();
  versions.setupClient();
}
//Use this method sparingly as it slows down the test

  @Test
  public void runCollector() {
    assertEquals(0, getStagingDocCount());
    assertEquals(0, getFinalDocCount());
    Flow harmonizeFlow = fm.getFlow(ENTITY, "testharmonize",
      FlowType.HARMONIZE);
    HashMap<String, Object> options = new HashMap<>();

    // a sneaky attempt to test passing options. this value makes the collector work.
    options.put("returnStuff", true);
    FlowRunner flowRunner = fm.newFlowRunner()
      .withFlow(harmonizeFlow)
      .withBatchSize(10)
      .withThreadCount(1)
      .withOptions(options)
      .withStopOnFailure(true);
    JobTicket ticket = flowRunner.run();
    flowRunner.awaitCompletion();
    assertEquals(0, getFinalDocCount());

    JsonNode node = jobDocMgr.read("/jobs/" + ticket.getJobId() + ".json").next().getContent(new JacksonHandle()).get();
    assertEquals(ticket.getJobId(), node.get("jobId").asText());
    assertEquals(0, node.get("successfulEvents").asInt());
    assertEquals(0, node.get("failedEvents").asInt());
    assertEquals(0, node.get("failedBatches").asInt());
    assertEquals("FAILED", node.get("status").asText());

  }
}

  @Test
  public void runJSONWriterErrorFlowWithoutTracing() {
    assertEquals(0, getFinalDocCount());
    assertEquals(0, getTracingDocCount());

    Tracing t = Tracing.create(flowRunnerClient);
    assertFalse(t.isEnabled());

    Flow flow = fm.getFlow("trace-entity", "tracemeJSONWriterError");

    FlowRunner flowRunner = fm.newFlowRunner()
      .withFlow(flow)
      .withBatchSize(10)
      .withThreadCount(1);
    flowRunner.run();
    flowRunner.awaitCompletion();

    assertEquals(0, getFinalDocCount());
    assertEquals(5, getTracingDocCount());

    JsonNode node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new JacksonHandle()).get();
    assertEquals(1, node.get("trace").get("steps").size());
    assertEquals("writer", node.get("trace").get("steps").get(0).get("label").asText());
  }
}

@Test
public void runSjsJsonFlowWithBinaryContent() {
  assertEquals(0, getFinalDocCount());
  assertEquals(0, getTracingDocCount());
  Tracing t = Tracing.create(flowRunnerClient);
  assertFalse(t.isEnabled());
  enableTracing();
  assertTrue(t.isEnabled());
  Flow flow = fm.getFlow("trace-entity", "tracemeSjsJsonWithBinary");
  FlowRunner flowRunner = fm.newFlowRunner()
    .withFlow(flow)
    .withBatchSize(10)
    .withThreadCount(1);
  flowRunner.run();
  flowRunner.awaitCompletion();
  assertEquals(5, getFinalDocCount());
  assertEquals(6, getTracingDocCount());
  DocumentRecord doc = finalDocMgr.read("1").next();
  String finalDoc= doc.getContent(new StringHandle()).get();
  assertJsonEqual(getResource("tracing-test/traces/finalSjsJsonDoc.json"), finalDoc, true);
  JsonNode node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new JacksonHandle()).get();
  assertEquals(4, node.get("trace").get("steps").size());
  assertEquals("content", node.get("trace").get("steps").get(0).get("label").asText());
  assertEquals(BINARY_HEX_ENCODED_SJS, node.get("trace").get("steps").get(0).get("output").asText().toLowerCase());
}

@Test
public void runXqyJsonFlowWithBinaryContent() {
  assertEquals(0, getFinalDocCount());
  assertEquals(0, getTracingDocCount());
  Tracing t = Tracing.create(flowRunnerClient);
  assertFalse(t.isEnabled());
  enableTracing();
  assertTrue(t.isEnabled());
  Flow flow = fm.getFlow("trace-entity", "tracemeXqyJsonWithBinary");
  FlowRunner flowRunner = fm.newFlowRunner()
    .withFlow(flow)
    .withBatchSize(10)
    .withThreadCount(1);
  flowRunner.run();
  flowRunner.awaitCompletion();
  assertEquals(5, getFinalDocCount());
  assertEquals(6, getTracingDocCount());
  DocumentRecord doc = finalDocMgr.read("/doc/1.json").next();
  String finalDoc= doc.getContent(new StringHandle()).get();
  assertJsonEqual(getResource("tracing-test/traces/finalXqyJsonDoc.json"), finalDoc, true);
  JsonNode node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new JacksonHandle()).get();
  assertEquals(4, node.get("trace").get("steps").size());
  assertEquals("content", node.get("trace").get("steps").get(0).get("label").asText());
  assertEquals(BINARY_HEX_ENCODED_XQY, node.get("trace").get("steps").get(0).get("output").asText().toLowerCase());
}

Flow harmonizeFlow = fm.getFlow(ENTITY, "testharmonize-sjs-xml",
    FlowType.HARMONIZE);
FlowRunner flowRunner = fm.newFlowRunner()
    .withFlow(harmonizeFlow)
    .withBatchSize(10)

@Test
public void runXMLErrorFlowWithoutTracing() {
  assertEquals(0, getFinalDocCount());
  assertEquals(0, getTracingDocCount());
  Tracing t = Tracing.create(flowRunnerClient);
  assertFalse(t.isEnabled());
  Flow flow = fm.getFlow("trace-entity", "tracemeXMLError");
  FlowRunner flowRunner = fm.newFlowRunner()
    .withFlow(flow)
    .withBatchSize(10)
    .withThreadCount(1);
  flowRunner.run();
  flowRunner.awaitCompletion();
  assertEquals(0, getFinalDocCount());
  assertEquals(5, getTracingDocCount());
  Document node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new DOMHandle()).get();
  assertEquals(1, node.getElementsByTagName("step").getLength());
  assertEquals("content", node.getElementsByTagName("label").item(0).getTextContent());
}