com.marklogic.hub.impl.FlowManagerImpl java code examples

Add the Codota plugin to your IDE and get smart completions

private void myMethod () {
 Connection c =DataSource dataSource;dataSource.getConnection()
String url;DriverManager.getConnection(url)
IdentityDatabaseUtil.getDBConnection()
Smart code suggestions by Tabnine
}

@Override public Flow getFlow(String entityName, String flowName) {
  return getFlow(entityName, flowName, null);
}

@JsonIgnore
public void refreshProject(Properties properties, boolean loadGradleProperties) {
  loadConfigurationFromProperties(properties, loadGradleProperties);
  flowManager.setupClient();
  dataHub.wireClient();
  versions.setupClient();
}

@Override public List<String> getLegacyFlows() {
  List<String> oldFlows = new ArrayList<>();
  Path entitiesDir = hubConfig.getHubEntitiesDir();
  File[] entityDirs = entitiesDir.toFile().listFiles(pathname -> pathname.isDirectory());
  if (entityDirs != null) {
    for (File entityDir : entityDirs) {
      Path inputDir = entityDir.toPath().resolve("input");
      Path harmonizeDir = entityDir.toPath().resolve("harmonize");
      File[] inputFlows = inputDir.toFile().listFiles((pathname) -> pathname.isDirectory() && !pathname.getName().equals("REST"));
      addLegacyFlowToList(oldFlows, entityDir, inputFlows);
      File[] harmonizeFlows = harmonizeDir.toFile().listFiles((pathname) -> pathname.isDirectory() && !pathname.getName().equals("REST"));
      addLegacyFlowToList(oldFlows, entityDir, harmonizeFlows);
    }
  }
  return oldFlows;
}

@Test
public void testEnvelopeXQY() throws IOException {
  //testing xqy JSON canonical instance
  scaffolding.createFlow(ENTITY, "testharmonize-xqy-json", FlowType.HARMONIZE,
      CodeFormat.XQUERY, DataFormat.JSON, false);
  Files.copy(getResourceStream("flow-runner-test/collector2.xqy"),
      projectDir.resolve("plugins/entities/" + ENTITY + "/harmonize/testharmonize-xqy-json/collector.xqy"),
      StandardCopyOption.REPLACE_EXISTING);
  Files.copy(getResourceStream("flow-runner-test/content-testing-envelope.xqy"),
      projectDir.resolve("plugins/entities/" + ENTITY + "/harmonize/testharmonize-xqy-json/content.xqy"),
      StandardCopyOption.REPLACE_EXISTING);
  installUserModules(getHubAdminConfig(), false);
  Flow harmonizeFlow = fm.getFlow(ENTITY, "testharmonize-xqy-json",
      FlowType.HARMONIZE);
  FlowRunner flowRunner = fm.newFlowRunner()
      .withFlow(harmonizeFlow)
      .withBatchSize(10)
      .withThreadCount(1);
  flowRunner.run();
  flowRunner.awaitCompletion();
  JsonNode jsonEnvelope = finalDocMgr.read("2.json").next().getContent(new JacksonHandle()).get();
  try {
    logger.debug(new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(jsonEnvelope));
  } catch (JsonProcessingException e) {
    e.printStackTrace();
  }
  assertNull(jsonEnvelope.get("envelope").get("instance").get("Person").get("$attachments"));
  assertNotNull(jsonEnvelope.get("envelope").get("attachments"));
}

@Override public List<Flow> getLocalFlowsForEntity(String entityName) {
  return getLocalFlowsForEntity(entityName, null);
}

@Override public Flow getFlowFromProperties(Path propertiesFile) {
  String quotedSeparator = Pattern.quote(File.separator);
  /* Extract flowName and entityName from ..../plugins/entities/<entityName>/
   * input|harmonize/<flowName>/flowName.properties
   */
  String floweRegex = ".+" + "plugins" + quotedSeparator + "entities" + quotedSeparator + "(.+)"+ quotedSeparator 
      +"(input|harmonize)" + quotedSeparator + "(.+)" + quotedSeparator + ".+";        
  FlowType flowType = propertiesFile.toString().replaceAll(floweRegex, "$2").equals("input")
      ? FlowType.INPUT : FlowType.HARMONIZE;
  String entityName = propertiesFile.toString().replaceAll(floweRegex, "$1");
  return getLocalFlow(entityName, propertiesFile.getParent(), flowType);
}

@Override public List<Flow> getFlows(String entityName) {
  RequestParameters params = new RequestParameters();
  params.add("entity-name", entityName);
  ServiceResultIterator resultItr = this.getServices().get(params);
  if (resultItr == null || ! resultItr.hasNext()) {
    return null;
  }
  ServiceResult res = resultItr.next();
  DOMHandle handle = new DOMHandle();
  Document parent = res.getContent(handle).get();
  NodeList children = parent.getDocumentElement().getChildNodes();
  ArrayList<Flow> flows = null;
  if (children.getLength() > 0) {
    flows = new ArrayList<>();
  }
  Node node;
  for (int i = 0; i < children.getLength(); i++) {
    node = children.item(i);
    if (node.getNodeType() == Node.ELEMENT_NODE) {
      flows.add(FlowManager.flowFromXml((Element)children.item(i)));
    }
  }
  return flows;
}

List<String> flows = flowManager.updateLegacyFlows(currentVersion);
if (updatedFlows != null) {
  updatedFlows.addAll(flows);

@Test
public void testEnvelopeSJS() throws IOException {
  scaffolding.createFlow(ENTITY, "testharmonize-sjs-json", FlowType.HARMONIZE,
      CodeFormat.JAVASCRIPT, DataFormat.JSON, false);
  //testing sjs JSON canonical instance
  Files.copy(getResourceStream("flow-runner-test/collector.sjs"),
      projectDir.resolve("plugins/entities/" + ENTITY + "/harmonize/testharmonize-sjs-json/collector.sjs"),
      StandardCopyOption.REPLACE_EXISTING);
  Files.copy(getResourceStream("flow-runner-test/contentTestingEnvelope.sjs"),
      projectDir.resolve("plugins/entities/" + ENTITY + "/harmonize/testharmonize-sjs-json/content.sjs"),
      StandardCopyOption.REPLACE_EXISTING);
  installUserModules(getHubAdminConfig(), false);
  Flow harmonizeFlow = fm.getFlow(ENTITY, "testharmonize-sjs-json",
      FlowType.HARMONIZE);
  FlowRunner flowRunner = fm.newFlowRunner()
      .withFlow(harmonizeFlow)
      .withBatchSize(10)
      .withThreadCount(1);
  flowRunner.run();
  flowRunner.awaitCompletion();
  JsonNode jsonEnvelope = finalDocMgr.read("1.json").next().getContent(new JacksonHandle()).get();
  try {
    logger.debug(new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(jsonEnvelope));
  } catch (JsonProcessingException e) {
    e.printStackTrace();
  }
  assertNull(jsonEnvelope.get("envelope").get("instance").get("Person").get("$attachments"));
  assertNotNull(jsonEnvelope.get("envelope").get("attachments"));
}

@Override public List<Flow> getLocalFlows() {
  List<Flow> flows = new ArrayList<>();
  Path entitiesDir = hubConfig.getHubEntitiesDir();
  File[] entities = entitiesDir.toFile().listFiles((pathname -> pathname.isDirectory()));
  if (entities != null) {
    for (File entity : entities) {
      String entityName = entity.getName();
      flows.addAll(getLocalFlowsForEntity(entityName));
    }
  }
  return flows;
}

if (inputFlows != null) {
  for (File inputFlow : inputFlows) {
    Flow flow = getLocalFlow(entityName, inputFlow.toPath(), FlowType.INPUT);
    if (flow != null) {
      flows.add(flow);
if (harmonizeFlows != null) {
  for (File harmonizeFlow : harmonizeFlows) {
    Flow flow = getLocalFlow(entityName, harmonizeFlow.toPath(), FlowType.HARMONIZE);
    if (flow != null) {
      flows.add(flow);

@Override public Flow getFlow(String entityName, String flowName, FlowType flowType) {
  RequestParameters params = new RequestParameters();
  params.add("entity-name", entityName);
  params.add("flow-name", flowName);
  if (flowType != null) {
    params.add("flow-type", flowType.toString());
  }
  ServiceResultIterator resultItr = this.getServices().get(params);
  if (resultItr == null || ! resultItr.hasNext()) {
    return null;
  }
  ServiceResult res = resultItr.next();
  DOMHandle handle = new DOMHandle();
  Document parent = res.getContent(handle).get();
  return FlowManager.flowFromXml(parent.getDocumentElement());
}

@Test
public void runJSONErrorFlowWithoutTracing() {
  assertEquals(0, getFinalDocCount());
  assertEquals(0, getTracingDocCount());
  Tracing t = Tracing.create(flowRunnerClient);
  assertFalse(t.isEnabled());
  Flow flow = fm.getFlow("trace-entity", "tracemeJSONError");
  FlowRunner flowRunner = fm.newFlowRunner()
    .withFlow(flow)
    .withBatchSize(10)
    .withThreadCount(1);
  flowRunner.run();
  flowRunner.awaitCompletion();
  assertEquals(0, getFinalDocCount());
  assertEquals(5, getTracingDocCount());
  JsonNode node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new JacksonHandle()).get();
  System.out.println(node.asText());
  assertEquals(1, node.get("trace").get("steps").size());
  assertEquals("content", node.get("trace").get("steps").get(0).get("label").asText());
}

public void wireClients() {        
  fm.setupClient();
  dataHub.wireClient();
  versions.setupClient();
}
//Use this method sparingly as it slows down the test

  @Test
  public void runCollector() {
    assertEquals(0, getStagingDocCount());
    assertEquals(0, getFinalDocCount());
    Flow harmonizeFlow = fm.getFlow(ENTITY, "testharmonize",
      FlowType.HARMONIZE);
    HashMap<String, Object> options = new HashMap<>();

    // a sneaky attempt to test passing options. this value makes the collector work.
    options.put("returnStuff", true);
    FlowRunner flowRunner = fm.newFlowRunner()
      .withFlow(harmonizeFlow)
      .withBatchSize(10)
      .withThreadCount(1)
      .withOptions(options)
      .withStopOnFailure(true);
    JobTicket ticket = flowRunner.run();
    flowRunner.awaitCompletion();
    assertEquals(0, getFinalDocCount());

    JsonNode node = jobDocMgr.read("/jobs/" + ticket.getJobId() + ".json").next().getContent(new JacksonHandle()).get();
    assertEquals(ticket.getJobId(), node.get("jobId").asText());
    assertEquals(0, node.get("successfulEvents").asInt());
    assertEquals(0, node.get("failedEvents").asInt());
    assertEquals(0, node.get("failedBatches").asInt());
    assertEquals("FAILED", node.get("status").asText());

  }
}

  @Test
  public void runJSONWriterErrorFlowWithoutTracing() {
    assertEquals(0, getFinalDocCount());
    assertEquals(0, getTracingDocCount());

    Tracing t = Tracing.create(flowRunnerClient);
    assertFalse(t.isEnabled());

    Flow flow = fm.getFlow("trace-entity", "tracemeJSONWriterError");

    FlowRunner flowRunner = fm.newFlowRunner()
      .withFlow(flow)
      .withBatchSize(10)
      .withThreadCount(1);
    flowRunner.run();
    flowRunner.awaitCompletion();

    assertEquals(0, getFinalDocCount());
    assertEquals(5, getTracingDocCount());

    JsonNode node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new JacksonHandle()).get();
    assertEquals(1, node.get("trace").get("steps").size());
    assertEquals("writer", node.get("trace").get("steps").get(0).get("label").asText());
  }
}

@Test
public void runSjsJsonFlowWithBinaryContent() {
  assertEquals(0, getFinalDocCount());
  assertEquals(0, getTracingDocCount());
  Tracing t = Tracing.create(flowRunnerClient);
  assertFalse(t.isEnabled());
  enableTracing();
  assertTrue(t.isEnabled());
  Flow flow = fm.getFlow("trace-entity", "tracemeSjsJsonWithBinary");
  FlowRunner flowRunner = fm.newFlowRunner()
    .withFlow(flow)
    .withBatchSize(10)
    .withThreadCount(1);
  flowRunner.run();
  flowRunner.awaitCompletion();
  assertEquals(5, getFinalDocCount());
  assertEquals(6, getTracingDocCount());
  DocumentRecord doc = finalDocMgr.read("1").next();
  String finalDoc= doc.getContent(new StringHandle()).get();
  assertJsonEqual(getResource("tracing-test/traces/finalSjsJsonDoc.json"), finalDoc, true);
  JsonNode node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new JacksonHandle()).get();
  assertEquals(4, node.get("trace").get("steps").size());
  assertEquals("content", node.get("trace").get("steps").get(0).get("label").asText());
  assertEquals(BINARY_HEX_ENCODED_SJS, node.get("trace").get("steps").get(0).get("output").asText().toLowerCase());
}

@Test
public void runXqyJsonFlowWithBinaryContent() {
  assertEquals(0, getFinalDocCount());
  assertEquals(0, getTracingDocCount());
  Tracing t = Tracing.create(flowRunnerClient);
  assertFalse(t.isEnabled());
  enableTracing();
  assertTrue(t.isEnabled());
  Flow flow = fm.getFlow("trace-entity", "tracemeXqyJsonWithBinary");
  FlowRunner flowRunner = fm.newFlowRunner()
    .withFlow(flow)
    .withBatchSize(10)
    .withThreadCount(1);
  flowRunner.run();
  flowRunner.awaitCompletion();
  assertEquals(5, getFinalDocCount());
  assertEquals(6, getTracingDocCount());
  DocumentRecord doc = finalDocMgr.read("/doc/1.json").next();
  String finalDoc= doc.getContent(new StringHandle()).get();
  assertJsonEqual(getResource("tracing-test/traces/finalXqyJsonDoc.json"), finalDoc, true);
  JsonNode node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new JacksonHandle()).get();
  assertEquals(4, node.get("trace").get("steps").size());
  assertEquals("content", node.get("trace").get("steps").get(0).get("label").asText());
  assertEquals(BINARY_HEX_ENCODED_XQY, node.get("trace").get("steps").get(0).get("output").asText().toLowerCase());
}

Flow harmonizeFlow = fm.getFlow(ENTITY, "testharmonize-sjs-xml",
    FlowType.HARMONIZE);
FlowRunner flowRunner = fm.newFlowRunner()
    .withFlow(harmonizeFlow)
    .withBatchSize(10)

@Test
public void runXMLErrorFlowWithoutTracing() {
  assertEquals(0, getFinalDocCount());
  assertEquals(0, getTracingDocCount());
  Tracing t = Tracing.create(flowRunnerClient);
  assertFalse(t.isEnabled());
  Flow flow = fm.getFlow("trace-entity", "tracemeXMLError");
  FlowRunner flowRunner = fm.newFlowRunner()
    .withFlow(flow)
    .withBatchSize(10)
    .withThreadCount(1);
  flowRunner.run();
  flowRunner.awaitCompletion();
  assertEquals(0, getFinalDocCount());
  assertEquals(5, getTracingDocCount());
  Document node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new DOMHandle()).get();
  assertEquals(1, node.getElementsByTagName("step").getLength());
  assertEquals("content", node.getElementsByTagName("label").item(0).getTextContent());
}

Most used methods

Popular in Java

Parsing JSON documents to java classes using gson
getExternalFilesDir (Context)
getSystemService (Context)
setContentView (Activity)
Thread (java.lang)
A thread is a thread of execution in a program. The Java Virtual Machine allows an application to ha
HttpURLConnection (java.net)
An URLConnection for HTTP (RFC 2616 [http://tools.ietf.org/html/rfc2616]) used to send and receive d
SimpleDateFormat (java.text)
Formats and parses dates in a locale-sensitive manner. Formatting turns a Date into a String, and pa
Iterator (java.util)
An iterator over a sequence of objects, such as a collection.If a collection has been changed since
Set (java.util)
A Set is a data structure which does not allow duplicate elements.
JOptionPane (javax.swing)
Top 17 Free Sublime Text Plugins

How to useFlowManagerImpl in com.marklogic.hub.impl

Best Java code snippets using com.marklogic.hub.impl.FlowManagerImpl (Showing top 20 results out of 315)

How to use
FlowManagerImpl
in
com.marklogic.hub.impl