- Add the Codota plugin to your IDE and get smart completions
private void myMethod () {}
private void recordCompletedWorkflow(AppMetadataStore metaStore, WorkflowDataset workflowDataset, WorkflowId workflowId, String runId) { RunRecordMeta runRecord = metaStore.getRun(workflowId.run(runId)); if (runRecord == null) { return; ApplicationId app = workflowId.getParent(); ApplicationSpecification appSpec = getApplicationSpec(metaStore, app); if (appSpec == null || appSpec.getWorkflows() == null || appSpec.getWorkflows().get(workflowId.getProgram()) == null) { LOG.warn("Missing ApplicationSpecification for {}, " + "potentially caused by application removal right after stopping workflow {}", app, workflowId); WorkflowSpecification workflowSpec = appSpec.getWorkflows().get(workflowId.getProgram()); Map<String, WorkflowNode> nodeIdMap = workflowSpec.getNodeIdMap(); List<WorkflowDataset.ProgramRun> programRunsList = new ArrayList<>();
public WorkflowId workflow(String program) { return new WorkflowId(this, program); }
private static MDSKey.Builder getRowKeyBuilder(WorkflowId id, long time) { return new MDSKey.Builder().add(id.getNamespace()) .add(id.getApplication()).add(id.getVersion()).add(id.getProgram()).add(time); } }
public static Workflow fromEntityId(WorkflowId workflowId) { return new Workflow(Id.Application.fromEntityId(workflowId.getParent()), workflowId.getProgram()); } }
private List<ScheduleDetail> doList(WorkflowId workflow) throws IOException, UnauthenticatedException, NotFoundException, UnauthorizedException { String path = String.format("apps/%s/workflows/%s/schedules", workflow.getApplication(), workflow.getProgram()); URL url = config.resolveNamespacedURLV3(workflow.getNamespaceId(), path); HttpResponse response = restClient.execute(HttpMethod.GET, url, config.getAccessToken(), HttpURLConnection.HTTP_NOT_FOUND); if (HttpURLConnection.HTTP_NOT_FOUND == response.getResponseCode()) { throw new NotFoundException(workflow); } ObjectResponse<List<ScheduleDetail>> objectResponse = ObjectResponse.fromJsonBody(response, LIST_SCHEDULE_DETAIL_TYPE, GSON); return objectResponse.getResponseObject(); }
@Override public Map<String, WorkflowNodeStateDetail> getWorkflowNodeStates(String workflowRunId) throws NotFoundException { try { ProgramRunId programRunId = workflowId.run(workflowRunId); return workflowClient.getWorkflowNodeStates(programRunId); } catch (IOException | UnauthenticatedException | UnauthorizedException e) { throw Throwables.propagate(e); } }
new WorkflowSpecification("DummyClass", SCHEDULED_PROG1_ID.getProgram(), "scheduled workflow", Collections.emptyMap(), Collections.emptyList(), Collections.emptyMap(), Collections.emptyMap()); Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), ImmutableMap.of(SCHEDULED_PROG1_ID.getProgram(), scheduledWorfklow1), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); new WorkflowSpecification("DummyClass", SCHEDULED_PROG2_ID.getProgram(), "scheduled workflow", Collections.emptyMap(), Collections.emptyList(), Collections.emptyMap(), Collections.emptyMap()); Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), ImmutableMap.of(SCHEDULED_PROG2_ID.getProgram(), scheduledWorfklow2), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
String programName = programRun.getName(); ProgramType programType = programRun.getProgramType(); ProgramId program = new ProgramId(workflowId.getNamespace(), workflowId.getApplication(), programType, programName); String programRunId = programRun.getRunId();
@Test public void testSystemMetadata() { MetadataEntity entity = NamespaceId.DEFAULT.app("appX").workflow("wtf").toMetadataEntity(); store.replaceMetadata(MetadataScope.SYSTEM, new MetadataDataset.Record( entity, ImmutableMap.of("a", "b", "x", "y"), ImmutableSet.of("tag1", "tag2")), ImmutableSet.of("a"), ImmutableSet.of("x")); MetadataSearchResponse response = store.search( new SearchRequest(null, "tag1", Collections.emptySet(), SortInfo.DEFAULT, 0, Integer.MAX_VALUE, 0, null, false, EnumSet.allOf(EntityScope.class))); Set<MetadataSearchResultRecord> results = response.getResults(); Assert.assertEquals(1, results.size()); store.replaceMetadata(MetadataScope.SYSTEM, new MetadataDataset.Record( entity, ImmutableMap.of("a", "b", "x", "y"), ImmutableSet.of("tag1", "tag2")), ImmutableSet.of("a"), ImmutableSet.of("x")); response = store.search( new SearchRequest(null, "tag1", Collections.emptySet(), SortInfo.DEFAULT, 0, Integer.MAX_VALUE, 0, null, false, EnumSet.allOf(EntityScope.class))); results = response.getResults(); Assert.assertEquals(1, results.size()); }
@Override public WorkflowTokenDetail getToken(String runId, @Nullable WorkflowToken.Scope scope, @Nullable String key) throws NotFoundException { try { return workflowClient.getWorkflowToken(workflowId.run(runId), scope, key); } catch (IOException | UnauthenticatedException | UnauthorizedException e) { throw Throwables.propagate(e); } }
/** * Get the next scheduled run time of the program. A program may contain multiple schedules. * This method returns the next scheduled runtimes for all the schedules. This method only takes + into account schedules based on time. Schedules based on data are ignored. * * @param workflow Id of the Workflow for which to fetch next run times. * @return list of Scheduled runtimes for the Workflow. Empty list if there are no schedules. */ public List<ScheduledRuntime> nextRuntimes(WorkflowId workflow) throws IOException, UnauthenticatedException, NotFoundException, UnauthorizedException { String path = String.format("apps/%s/workflows/%s/nextruntime", workflow.getApplication(), workflow.getProgram()); URL url = config.resolveNamespacedURLV3(workflow.getNamespaceId(), path); HttpResponse response = restClient.execute(HttpMethod.GET, url, config.getAccessToken(), HttpURLConnection.HTTP_NOT_FOUND); if (HttpURLConnection.HTTP_NOT_FOUND == response.getResponseCode()) { throw new NotFoundException(workflow); } ObjectResponse<List<ScheduledRuntime>> objectResponse = ObjectResponse.fromJsonBody( response, new TypeToken<List<ScheduledRuntime>>() { }.getType(), GSON); return objectResponse.getResponseObject(); }
public static Workflow fromEntityId(WorkflowId workflowId) { return new Workflow(Id.Application.fromEntityId(workflowId.getParent()), workflowId.getProgram()); } }
String programName = programRun.getName(); ProgramType programType = programRun.getProgramType(); ProgramId program = new ProgramId(workflowId.getNamespace(), workflowId.getApplication(), programType, programName); String programRunId = programRun.getRunId();
@Test public void testSearchOnTagsUpdate() { MetadataEntity entity = NamespaceId.DEFAULT.app("appX").workflow("wtf").toMetadataEntity(); store.addTags(MetadataScope.SYSTEM, entity, ImmutableSet.of("tag1", "tag2")); Assert.assertEquals(ImmutableSet.of("tag1", "tag2"), store.getTags(MetadataScope.SYSTEM, entity)); MetadataSearchResponse response = store.search( new SearchRequest(null, "tag1", Collections.emptySet(), SortInfo.DEFAULT, 0, Integer.MAX_VALUE, 0, null, false, EnumSet.allOf(EntityScope.class))); Set<MetadataSearchResultRecord> results = response.getResults(); Assert.assertEquals(1, results.size()); // add an more tags store.addTags(MetadataScope.SYSTEM, entity, ImmutableSet.of("tag3", "tag4")); ImmutableSet<String> expectedTags = ImmutableSet.of("tag1", "tag2", "tag3", "tag4"); Assert.assertEquals(expectedTags, store.getTags(MetadataScope.SYSTEM, entity)); for (String expectedTag : expectedTags) { response = store.search( new SearchRequest(null, expectedTag, Collections.emptySet(), SortInfo.DEFAULT, 0, Integer.MAX_VALUE, 0, null, false, EnumSet.allOf(EntityScope.class))); results = response.getResults(); Assert.assertEquals(1, results.size()); } // add an empty set of tags. This should have no effect on retrieval or search of tags store.addTags(MetadataScope.SYSTEM, entity, ImmutableSet.of()); Assert.assertEquals(expectedTags, store.getTags(MetadataScope.SYSTEM, entity)); response = store.search( new SearchRequest(null, "tag1", Collections.emptySet(), SortInfo.DEFAULT, 0, Integer.MAX_VALUE, 0, null, false, EnumSet.allOf(EntityScope.class))); results = response.getResults(); Assert.assertEquals(1, results.size()); }
@Override public WorkflowTokenNodeDetail getTokenAtNode(String runId, String nodeName, @Nullable WorkflowToken.Scope scope, @Nullable String key) throws NotFoundException { try { return workflowClient.getWorkflowTokenAtNode(workflowId.run(runId), nodeName, scope, key); } catch (IOException | UnauthenticatedException | UnauthorizedException e) { throw Throwables.propagate(e); } }
private void recordCompletedWorkflow(AppMetadataStore metaStore, WorkflowDataset workflowDataset, WorkflowId workflowId, String runId) { RunRecordMeta runRecord = metaStore.getRun(workflowId.run(runId)); if (runRecord == null) { return; ApplicationId app = workflowId.getParent(); ApplicationSpecification appSpec = getApplicationSpec(metaStore, app); if (appSpec == null || appSpec.getWorkflows() == null || appSpec.getWorkflows().get(workflowId.getProgram()) == null) { LOG.warn("Missing ApplicationSpecification for {}, " + "potentially caused by application removal right after stopping workflow {}", app, workflowId); WorkflowSpecification workflowSpec = appSpec.getWorkflows().get(workflowId.getProgram()); Map<String, WorkflowNode> nodeIdMap = workflowSpec.getNodeIdMap(); List<WorkflowDataset.ProgramRun> programRunsList = new ArrayList<>();
public WorkflowId workflow(String program) { return new WorkflowId(this, program); }
private static MDSKey.Builder getRowKeyBuilder(WorkflowId id, long time) { return new MDSKey.Builder().add(id.getNamespace()) .add(id.getApplication()).add(id.getVersion()).add(id.getProgram()).add(time); } }
@Test public void testWorkflowTags() throws Exception { String appName = WorkflowAppWithFork.class.getSimpleName(); ApplicationId appId = NamespaceId.DEFAULT.app(appName); String workflowName = WorkflowAppWithFork.WorkflowWithFork.class.getSimpleName(); ArtifactId artifactId = NamespaceId.DEFAULT.artifact(appId.getApplication(), "1.0"); ApplicationWithPrograms appWithPrograms = createAppWithWorkflow(artifactId, appId, workflowName); WorkflowSpecification workflowSpec = appWithPrograms.getSpecification().getWorkflows().get(workflowName); SystemMetadataWriterStage systemMetadataWriterStage = new SystemMetadataWriterStage(metadataPublisher); StageContext stageContext = new StageContext(Object.class); systemMetadataWriterStage.process(stageContext); systemMetadataWriterStage.process(appWithPrograms); // verify that the workflow is not tagged with the fork node name. First wait for tags to show up Tasks.waitFor(false, () -> metadataStore.getTags( MetadataScope.SYSTEM, appId.workflow(workflowName).toMetadataEntity()).isEmpty(), 5, TimeUnit.SECONDS); Set<String> workflowSystemTags = metadataStore.getTags(MetadataScope.SYSTEM, appId.workflow(workflowName).toMetadataEntity()); Sets.SetView<String> intersection = Sets.intersection(workflowSystemTags, getWorkflowForkNodes(workflowSpec)); Assert.assertTrue("Workflows should not be tagged with fork node names, but found the following fork nodes " + "in the workflow's system tags: " + intersection, intersection.isEmpty()); // verify that metadata was added for the workflow's schedule. First wait for the metadata to show up Tasks.waitFor(false, () -> metadataStore.getProperties( MetadataScope.SYSTEM, appId.toMetadataEntity()).isEmpty(), 5, TimeUnit.SECONDS); Map<String, String> metadataProperties = metadataStore.getMetadata(MetadataScope.SYSTEM, appId.toMetadataEntity()).getProperties(); Assert.assertEquals(WorkflowAppWithFork.SCHED_NAME + ":testDescription", metadataProperties.get("schedule:" + WorkflowAppWithFork.SCHED_NAME)); }
private WorkflowToken getWorkflowToken(String namespaceId, String appName, String workflow, String runId) throws NotFoundException { ApplicationId appId = new ApplicationId(namespaceId, appName); ApplicationSpecification appSpec = store.getApplication(appId); if (appSpec == null) { throw new NotFoundException(appId); } WorkflowId workflowId = appId.workflow(workflow); if (!appSpec.getWorkflows().containsKey(workflow)) { throw new NotFoundException(workflowId); } if (store.getRun(workflowId.run(runId)) == null) { throw new NotFoundException(workflowId.run(runId)); } return store.getWorkflowToken(workflowId, runId); }