@Test public void testAdminWorkflow() throws Exception { testAdminProgram(appManager.getWorkflowManager(AdminApp.WORKFLOW_NAME)); }
File firstFileDone = new File(TMP_FOLDER.newFolder() + "/first.done"); WorkflowManager workflowManager = appManager.getWorkflowManager(WorkflowStatusTestApp.WORKFLOW_NAME); workflowManager.start(ImmutableMap.of("workflow.success.file", workflowSuccess.getAbsolutePath(), "action.success.file", actionSuccess.getAbsolutePath(),
@Test public void testSparkProgramStatusSchedule() throws Exception { ApplicationManager appManager = deploy(TestSparkApp.class); ScheduleId scheduleId = new ScheduleId(NamespaceId.DEFAULT.getNamespace(), TestSparkApp.class.getSimpleName(), "schedule"); appManager.enableSchedule(scheduleId); WorkflowManager workflowManager = appManager.getWorkflowManager(TestSparkApp.TriggeredWorkflow.class.getSimpleName()); int numRuns = workflowManager.getHistory(ProgramRunStatus.COMPLETED).size(); // Start the upstream program SparkManager sparkManager = appManager.getSparkManager(TestSparkApp.ScalaClassicSpark.class.getSimpleName()); sparkManager.start(); // Wait for the downstream to complete workflowManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES); // Run again with the kryo serializer sparkManager.start(Collections.singletonMap("spark.serializer", "org.apache.spark.serializer.KryoSerializer")); // Wait for the downstream to complete again workflowManager.waitForRuns(ProgramRunStatus.COMPLETED, numRuns + 2, 5, TimeUnit.MINUTES); }
@Category(XSlowTests.class) @Test public void testWorkflowLocalDatasets() throws Exception { ApplicationManager applicationManager = deployApplication(testSpace, WorkflowAppWithLocalDatasets.class); // Execute Workflow without keeping the local datasets after run Map<String, String> additionalParams = new HashMap<>(); String runId = executeWorkflow(applicationManager, additionalParams, 1); verifyWorkflowRun(runId, false, false, "COMPLETED"); additionalParams.put("dataset.wordcount.keep.local", "true"); runId = executeWorkflow(applicationManager, additionalParams, 2); verifyWorkflowRun(runId, true, false, "COMPLETED"); additionalParams.clear(); additionalParams.put("dataset.*.keep.local", "true"); runId = executeWorkflow(applicationManager, additionalParams, 3); verifyWorkflowRun(runId, true, true, "COMPLETED"); additionalParams.clear(); additionalParams.put("dataset.*.keep.local", "true"); additionalParams.put("destroy.throw.exception", "true"); runId = executeWorkflow(applicationManager, additionalParams, 4); verifyWorkflowRun(runId, true, true, "STARTED"); WorkflowManager wfManager = applicationManager.getWorkflowManager(WorkflowAppWithLocalDatasets.WORKFLOW_NAME); List<RunRecord> history = wfManager.getHistory(); Assert.assertEquals(4, history.size()); for (RunRecord record : history) { Assert.assertEquals(ProgramRunStatus.COMPLETED, record.getStatus()); } }
@Test public void testWorkflowCondition() throws Exception { ApplicationManager applicationManager = deployApplication(testSpace, ConditionalWorkflowApp.class); final WorkflowManager wfmanager = applicationManager.getWorkflowManager("ConditionalWorkflow"); wfmanager.start(ImmutableMap.of("configurable.condition", "true")); Tasks.waitFor(true, new Callable<Boolean>() { @Override public Boolean call() throws Exception { return wfmanager.getHistory(ProgramRunStatus.COMPLETED).size() == 1; } }, 30, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS); List<RunRecord> history = wfmanager.getHistory(); String pid = history.get(0).getPid(); WorkflowTokenNodeDetail tokenNodeDetail = wfmanager.getTokenAtNode(pid, "MyConfigurableCondition", WorkflowToken.Scope.USER, null); Map<String, String> expected = ImmutableMap.of("configurable.condition.initialize", "true", "configurable.condition.destroy", "true", "configurable.condition.apply", "true"); Assert.assertEquals(expected, tokenNodeDetail.getTokenDataAtNode()); tokenNodeDetail = wfmanager.getTokenAtNode(pid, "SimpleCondition", WorkflowToken.Scope.USER, null); expected = ImmutableMap.of("simple.condition.initialize", "true"); Assert.assertEquals(expected, tokenNodeDetail.getTokenDataAtNode()); tokenNodeDetail = wfmanager.getTokenAtNode(pid, "action2", WorkflowToken.Scope.USER, null); expected = ImmutableMap.of("action.name", "action2"); Assert.assertEquals(expected, tokenNodeDetail.getTokenDataAtNode()); }
private String executeWorkflow(ApplicationManager applicationManager, Map<String, String> additionalParams, int expectedComplete) throws Exception { WorkflowManager wfManager = applicationManager.getWorkflowManager(WorkflowAppWithLocalDatasets.WORKFLOW_NAME); Map<String, String> runtimeArgs = new HashMap<>(); File waitFile = new File(TMP_FOLDER.newFolder(), "/wait.file");
@Test public void testConcurrentRuns() throws Exception { ApplicationManager appManager = deployApplication(ConcurrentRunTestApp.class); WorkerManager workerManager = appManager.getWorkerManager(ConcurrentRunTestApp.TestWorker.class.getSimpleName()); workerManager.start(); // Start another time should fail as worker doesn't support concurrent run. workerManager.waitForRun(ProgramRunStatus.RUNNING, 10, TimeUnit.SECONDS); try { workerManager.start(); Assert.fail("Expected failure to start worker"); } catch (Exception e) { Assert.assertTrue(Throwables.getRootCause(e) instanceof ConflictException); } workerManager.waitForRun(ProgramRunStatus.RUNNING, 10, TimeUnit.SECONDS); workerManager.stop(); // Start the workflow File tmpDir = TEMP_FOLDER.newFolder(); File actionFile = new File(tmpDir, "action.file"); Map<String, String> args = Collections.singletonMap("action.file", actionFile.getAbsolutePath()); WorkflowManager workflowManager = appManager.getWorkflowManager( ConcurrentRunTestApp.TestWorkflow.class.getSimpleName()); // Starts two runs, both should succeed workflowManager.start(args); workflowManager.start(args); // Should get two active runs workflowManager.waitForRuns(ProgramRunStatus.RUNNING, 2, 10L, TimeUnit.SECONDS); // Touch the file to complete the workflow runs Files.touch(actionFile); workflowManager.waitForRuns(ProgramRunStatus.COMPLETED, 2, 10L, TimeUnit.SECONDS); }
args = RuntimeArguments.addScope(Scope.MAPREDUCE, ClusterNameTestApp.ClusterNameMapReduce.class.getSimpleName(), args); WorkflowManager workflowManager = appManager.getWorkflowManager( ClusterNameTestApp.ClusterNameWorkflow.class.getSimpleName()).start(args);
appManager.getWorkflowManager(workflowName); ScheduleManager scheduleManager = workflowManager.getSchedule(AppWithSchedule.EVERY_HOUR_SCHEDULE);
final WorkflowManager wfmanager = applicationManager.getWorkflowManager(AppWithSchedule.WORKFLOW_NAME); List<ScheduleDetail> schedules = wfmanager.getProgramSchedules(); Assert.assertEquals(2, schedules.size());
@Category(SlowTests.class) @Test public void testCustomActionDatasetAccess() throws Exception { addDatasetInstance("keyValueTable", DatasetWithCustomActionApp.CUSTOM_TABLE); addDatasetInstance("fileSet", DatasetWithCustomActionApp.CUSTOM_FILESET); ApplicationManager appManager = deployApplication(DatasetWithCustomActionApp.class); ServiceManager serviceManager = appManager.getServiceManager(DatasetWithCustomActionApp.CUSTOM_SERVICE).start(); serviceManager.waitForRun(ProgramRunStatus.RUNNING, 10, TimeUnit.SECONDS); WorkflowManager workflowManager = appManager.getWorkflowManager(DatasetWithCustomActionApp.CUSTOM_WORKFLOW).start(); workflowManager.waitForRun(ProgramRunStatus.COMPLETED, 2, TimeUnit.MINUTES); appManager.stopAll(); DataSetManager<KeyValueTable> outTableManager = getDataset(DatasetWithCustomActionApp.CUSTOM_TABLE); KeyValueTable outputTable = outTableManager.get(); Assert.assertEquals("world", Bytes.toString(outputTable.read("hello"))); Assert.assertEquals("service", Bytes.toString(outputTable.read("hi"))); Assert.assertEquals("another.world", Bytes.toString(outputTable.read("another.hello"))); DataSetManager<FileSet> outFileSetManager = getDataset(DatasetWithCustomActionApp.CUSTOM_FILESET); FileSet fs = outFileSetManager.get(); try (InputStream in = fs.getLocation("test").getInputStream()) { Assert.assertEquals(42, in.read()); } }
serviceManager.waitForStopped(10, TimeUnit.SECONDS); WorkflowManager workflowManager = appManager.getWorkflowManager(AppWithPlugin.WORKFLOW); workflowManager.start(); workflowManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES);
WorkerManager txWorkerManager = appManager.getWorkerManager(AppWithCustomTx.WORKER_TX) .start(txTimeoutArguments(txDefaulTimeoutWorker)); WorkflowManager txWFManager = appManager.getWorkflowManager(AppWithCustomTx.WORKFLOW_TX) .start(txTimeoutArguments(txDefaulTimeoutWorkflow)); WorkflowManager notxWFManager = appManager.getWorkflowManager(AppWithCustomTx.WORKFLOW_NOTX) .start(txTimeoutArguments(txDefaulTimeoutWorkflow, txDefaulTimeoutAction, "action", AppWithCustomTx.ACTION_NOTX));