@Override public Map<String, WorkflowNodeStateDetail> getWorkflowNodeStates(String workflowRunId) throws NotFoundException { try { ProgramRunId programRunId = workflowId.run(workflowRunId); return workflowClient.getWorkflowNodeStates(programRunId); } catch (IOException | UnauthenticatedException | UnauthorizedException e) { throw Throwables.propagate(e); } }
@Override public WorkflowTokenDetail getToken(String runId, @Nullable WorkflowToken.Scope scope, @Nullable String key) throws NotFoundException { try { return workflowClient.getWorkflowToken(workflowId.run(runId), scope, key); } catch (IOException | UnauthenticatedException | UnauthorizedException e) { throw Throwables.propagate(e); } }
@Override public WorkflowTokenNodeDetail getTokenAtNode(String runId, String nodeName, @Nullable WorkflowToken.Scope scope, @Nullable String key) throws NotFoundException { try { return workflowClient.getWorkflowTokenAtNode(workflowId.run(runId), nodeName, scope, key); } catch (IOException | UnauthenticatedException | UnauthorizedException e) { throw Throwables.propagate(e); } }
private WorkflowToken getWorkflowToken(String namespaceId, String appName, String workflow, String runId) throws NotFoundException { ApplicationId appId = new ApplicationId(namespaceId, appName); ApplicationSpecification appSpec = store.getApplication(appId); if (appSpec == null) { throw new NotFoundException(appId); } WorkflowId workflowId = appId.workflow(workflow); if (!appSpec.getWorkflows().containsKey(workflow)) { throw new NotFoundException(workflowId); } if (store.getRun(workflowId.run(runId)) == null) { throw new NotFoundException(workflowId.run(runId)); } return store.getWorkflowToken(workflowId, runId); }
private WorkflowToken getWorkflowToken(String namespaceId, String appName, String workflow, String runId) throws NotFoundException { ApplicationId appId = new ApplicationId(namespaceId, appName); ApplicationSpecification appSpec = store.getApplication(appId); if (appSpec == null) { throw new NotFoundException(appId); } WorkflowId workflowId = appId.workflow(workflow); if (!appSpec.getWorkflows().containsKey(workflow)) { throw new NotFoundException(workflowId); } if (store.getRun(workflowId.run(runId)) == null) { throw new NotFoundException(workflowId.run(runId)); } return store.getWorkflowToken(workflowId, runId); }
private void addWorkflowNodeState(ProgramRunId programRunId, Map<String, String> systemArgs, ProgramRunStatus status, @Nullable BasicThrowable failureCause, byte[] sourceId) { String workflowNodeId = systemArgs.get(ProgramOptionConstants.WORKFLOW_NODE_ID); String workflowName = systemArgs.get(ProgramOptionConstants.WORKFLOW_NAME); String workflowRun = systemArgs.get(ProgramOptionConstants.WORKFLOW_RUN_ID); ApplicationId appId = programRunId.getParent().getParent(); ProgramRunId workflowRunId = appId.workflow(workflowName).run(workflowRun); // Node states will be stored with following key: // workflowNodeState.namespace.app.WORKFLOW.workflowName.workflowRun.workflowNodeId MDSKey key = getProgramKeyBuilder(TYPE_WORKFLOW_NODE_STATE, workflowRunId).add(workflowNodeId).build(); WorkflowNodeStateDetail nodeStateDetail = new WorkflowNodeStateDetail(workflowNodeId, ProgramRunStatus.toNodeStatus(status), programRunId.getRun(), failureCause); write(key, nodeStateDetail); // Get the run record of the Workflow which started this program key = getProgramRunInvertedTimeKey(TYPE_RUN_RECORD_ACTIVE, workflowRunId, RunIds.getTime(workflowRun, TimeUnit.SECONDS)); RunRecordMeta record = get(key, RunRecordMeta.class); if (record != null) { // Update the parent Workflow run record by adding node id and program run id in the properties Map<String, String> properties = new HashMap<>(record.getProperties()); properties.put(workflowNodeId, programRunId.getRun()); write(key, RunRecordMeta.builder(record).setProperties(properties).setSourceId(sourceId).build()); } }
private void addWorkflowNodeState(ProgramRunId programRunId, Map<String, String> systemArgs, ProgramRunStatus status, @Nullable BasicThrowable failureCause, byte[] sourceId) { String workflowNodeId = systemArgs.get(ProgramOptionConstants.WORKFLOW_NODE_ID); String workflowName = systemArgs.get(ProgramOptionConstants.WORKFLOW_NAME); String workflowRun = systemArgs.get(ProgramOptionConstants.WORKFLOW_RUN_ID); ApplicationId appId = programRunId.getParent().getParent(); ProgramRunId workflowRunId = appId.workflow(workflowName).run(workflowRun); // Node states will be stored with following key: // workflowNodeState.namespace.app.WORKFLOW.workflowName.workflowRun.workflowNodeId MDSKey key = getProgramKeyBuilder(TYPE_WORKFLOW_NODE_STATE, workflowRunId).add(workflowNodeId).build(); WorkflowNodeStateDetail nodeStateDetail = new WorkflowNodeStateDetail(workflowNodeId, ProgramRunStatus.toNodeStatus(status), programRunId.getRun(), failureCause); write(key, nodeStateDetail); // Get the run record of the Workflow which started this program key = getProgramRunInvertedTimeKey(TYPE_RUN_RECORD_ACTIVE, workflowRunId, RunIds.getTime(workflowRun, TimeUnit.SECONDS)); RunRecordMeta record = get(key, RunRecordMeta.class); if (record != null) { // Update the parent Workflow run record by adding node id and program run id in the properties Map<String, String> properties = new HashMap<>(record.getProperties()); properties.put(workflowNodeId, programRunId.getRun()); write(key, RunRecordMeta.builder(record).setProperties(properties).setSourceId(sourceId).build()); } }
@Override public void perform(Arguments arguments, PrintStream printStream) throws Exception { String[] programIdParts = arguments.get(elementType.getArgumentName().toString()).split("\\."); if (programIdParts.length < 2) { throw new CommandInputError(this); } ProgramRunId programRunId = cliConfig.getCurrentNamespace().app(programIdParts[0]).workflow(programIdParts[1]) .run(arguments.get(ArgumentName.RUN_ID.toString())); Table table = getWorkflowNodeStates(programRunId); cliConfig.getTableRenderer().render(cliConfig, printStream, table); }
@Override public void perform(Arguments arguments, PrintStream printStream) throws Exception { String[] programIdParts = arguments.get(elementType.getArgumentName().toString()).split("\\."); if (programIdParts.length < 2) { throw new CommandInputError(this); } ProgramRunId programRunId = cliConfig.getCurrentNamespace().app(programIdParts[0]).workflow(programIdParts[1]) .run(arguments.get(ArgumentName.RUN_ID.toString())); Table table = getWorkflowLocalDatasets(programRunId); cliConfig.getTableRenderer().render(cliConfig, printStream, table); }
@Override public void perform(Arguments arguments, PrintStream printStream) throws Exception { String[] programIdParts = arguments.get(elementType.getArgumentName().toString()).split("\\."); if (programIdParts.length < 2) { throw new CommandInputError(this); } ProgramRunId programRunId = cliConfig.getCurrentNamespace().app(programIdParts[0]).workflow(programIdParts[1]) .run(arguments.get(ArgumentName.RUN_ID.toString())); Table table = getWorkflowLocalDatasets(programRunId); cliConfig.getTableRenderer().render(cliConfig, printStream, table); }
@Override public void perform(Arguments arguments, PrintStream printStream) throws Exception { String[] programIdParts = arguments.get(elementType.getArgumentName().toString()).split("\\."); if (programIdParts.length < 2) { throw new CommandInputError(this); } ProgramRunId programRunId = cliConfig.getCurrentNamespace().app(programIdParts[0]).workflow(programIdParts[1]) .run(arguments.get(ArgumentName.RUN_ID.toString())); Table table = getWorkflowNodeStates(programRunId); cliConfig.getTableRenderer().render(cliConfig, printStream, table); }
private TaskFields createTaskInfo(ProvisionerInfo provisionerInfo) { ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").workflow("wf").run(RunIds.generate()); Map<String, String> systemArgs = new HashMap<>(); Map<String, String> userArgs = new HashMap<>(); Profile profile = new Profile(ProfileId.NATIVE.getProfile(), "label", "desc", provisionerInfo); SystemArguments.addProfileArgs(systemArgs, profile); ProgramOptions programOptions = new SimpleProgramOptions(programRunId.getParent(), new BasicArguments(systemArgs), new BasicArguments(userArgs)); ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId(); ApplicationSpecification appSpec = new DefaultApplicationSpecification( "name", "1.0.0", "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); ProgramDescriptor programDescriptor = new ProgramDescriptor(programRunId.getParent(), appSpec); return new TaskFields(programDescriptor, programOptions, programRunId); }
@Test public void testWorkflow() throws InterruptedException, ExecutionException, TimeoutException { ProgramRunId workflowRunId = workflow1.run(RunIds.generate());
@Test public void testRoundingLogic() throws Exception { ProgramRunId runId = NamespaceId.DEFAULT.app("round").workflow("round").run(RunIds.generate()); ProfileId profileId = NamespaceId.DEFAULT.profile("roundProfile"); MetricsCollectionService collectionService = injector.getInstance(MetricsCollectionService.class);
@Test public void testProfileMetrics() throws Exception { ProgramRunId runId = NamespaceId.DEFAULT.app("myApp").workflow("myProgram").run(RunIds.generate()); ProfileId profileId = NamespaceId.DEFAULT.profile("myProfile"); MetricsCollectionService collectionService = injector.getInstance(MetricsCollectionService.class); MetricStore metricStore = injector.getInstance(MetricStore.class); // There are 5 nodes, we emit the metrics each 2 mins, so each time the node minute should go up by 10 min ProfileMetricService scheduledService = new ProfileMetricService(collectionService, runId, profileId, 5, 2, null); // emit and verify the results scheduledService.emitMetric(); Tasks.waitFor(10L, () -> getMetric(metricStore, runId, profileId, "system." + Constants.Metrics.Program.PROGRAM_NODE_MINUTES), 10, TimeUnit.SECONDS); scheduledService.emitMetric(); Tasks.waitFor(20L, () -> getMetric(metricStore, runId, profileId, "system." + Constants.Metrics.Program.PROGRAM_NODE_MINUTES), 10, TimeUnit.SECONDS); scheduledService.emitMetric(); Tasks.waitFor(30L, () -> getMetric(metricStore, runId, profileId, "system." + Constants.Metrics.Program.PROGRAM_NODE_MINUTES), 10, TimeUnit.SECONDS); }
@Test public void testProfileMetricsDeletion() throws Exception { ProfileId myProfile = NamespaceId.DEFAULT.profile("MyProfile"); Profile profile = new Profile("MyProfile", Profile.NATIVE.getLabel(), Profile.NATIVE.getDescription(), Profile.NATIVE.getScope(), Profile.NATIVE.getProvisioner()); ProgramRunId runId = NamespaceId.DEFAULT.app("myApp").workflow("myProgram").run(RunIds.generate()); // create and disable the profile profileService.saveProfile(myProfile, profile); profileService.disableProfile(myProfile); // emit some metrics MetricsCollectionService metricService = injector.getInstance(MetricsCollectionService.class); MetricsContext metricsContext = metricService.getContext(getMetricsTags(runId, myProfile)); metricsContext.increment(Constants.Metrics.Program.PROGRAM_NODE_MINUTES, 30L); MetricStore metricStore = injector.getInstance(MetricStore.class); Tasks.waitFor(30L, () -> getMetric(metricStore, runId, myProfile, "system." + Constants.Metrics.Program.PROGRAM_NODE_MINUTES), 10, TimeUnit.SECONDS); // delete and verify the metrics are gone profileService.deleteProfile(myProfile); Tasks.waitFor(0L, () -> getMetric(metricStore, runId, myProfile, "system." + Constants.Metrics.Program.PROGRAM_NODE_MINUTES), 10, TimeUnit.SECONDS); }
ProgramRunId programRunId = NamespaceId.DEFAULT.app("app1").workflow("testkill").run(runId); ProfileId profileId = NamespaceId.DEFAULT.profile("myProfile"); publishProgramStatus(programRunId, ProgramStatus.INITIALIZING);
@Test public void testProfileInRunRecord() throws Exception { AppMetadataStore store = getMetadataStore("testProfileInRunRecord"); TransactionExecutor txnl = getTxExecutor(store); ProgramRunId runId = NamespaceId.DEFAULT.app("myApp").workflow("myProgram").run(RunIds.generate()); ProfileId profileId = NamespaceId.DEFAULT.profile("MyProfile"); txnl.execute(() -> { long startSourceId = 1L; store.recordProgramProvisioning(runId, null, Collections.singletonMap(SystemArguments.PROFILE_NAME, profileId.getScopedName()), AppFabricTestHelper.createSourceId(startSourceId), ARTIFACT_ID); // the profile id should be there after the provisioning stage RunRecordMeta run = store.getRun(runId); Assert.assertNotNull(run); Assert.assertEquals(profileId, run.getProfileId()); store.recordProgramProvisioned(runId, 0, AppFabricTestHelper.createSourceId(startSourceId + 1)); store.recordProgramStart(runId, null, ImmutableMap.of(), AppFabricTestHelper.createSourceId(startSourceId + 2)); store.recordProgramRunning(runId, RunIds.getTime(runId.getRun(), TimeUnit.SECONDS), null, AppFabricTestHelper.createSourceId(startSourceId + 3)); store.recordProgramStop(runId, RunIds.getTime(runId.getRun(), TimeUnit.SECONDS), ProgramRunStatus.KILLED, null, AppFabricTestHelper.createSourceId(startSourceId + 4)); run = store.getRun(runId); Assert.assertNotNull(run); Assert.assertEquals(profileId, run.getProfileId()); }); }
ProgramRunId programRunId = NamespaceId.DEFAULT.app("app1").workflow("myworkflow").run(runId); ProfileId profileId = NamespaceId.DEFAULT.profile("myProfile"); publishProgramStatus(programRunId, ProgramStatus.INITIALIZING);