private Optional<Runnable> getEmitMetricsRunnable(ProgramRunId programRunId, RunRecordMeta recordedRunRecord, String metricName) { Optional<ProfileId> profile = SystemArguments.getProfileIdFromArgs(programRunId.getNamespaceId(), recordedRunRecord.getSystemArgs()); return profile.map(profileId -> () -> emitProfileMetrics(programRunId, profileId, metricName)); }
private Optional<Runnable> getEmitMetricsRunnable(ProgramRunId programRunId, RunRecordMeta recordedRunRecord, String metricName) { Optional<ProfileId> profile = SystemArguments.getProfileIdFromArgs(programRunId.getNamespaceId(), recordedRunRecord.getSystemArgs()); return profile.map(profileId -> () -> emitProfileMetrics(programRunId, profileId, metricName)); }
/** * Creates a {@link ProfileMetricService} for the profile being used for the given program run. */ private ProfileMetricService createProfileMetricsService(ProgramRunId programRunId, ProgramOptions programOptions, Cluster cluster) { Optional<ProfileId> profile = SystemArguments.getProfileIdFromArgs(programRunId.getNamespaceId(), programOptions.getArguments().asMap()); ProfileId profileId = profile.orElseThrow( () -> new IllegalStateException("Missing profile information for program run " + programRunId)); return new ProfileMetricService(metricsCollectionService, programRunId, profileId, cluster.getNodes().size(), monitorScheduler); }
/** * Creates a {@link ProfileMetricService} for the profile being used for the given program run. */ private ProfileMetricService createProfileMetricsService(ProgramRunId programRunId, ProgramOptions programOptions, Cluster cluster) { Optional<ProfileId> profile = SystemArguments.getProfileIdFromArgs(programRunId.getNamespaceId(), programOptions.getArguments().asMap()); ProfileId profileId = profile.orElseThrow( () -> new IllegalStateException("Missing profile information for program run " + programRunId)); return new ProfileMetricService(metricsCollectionService, programRunId, profileId, cluster.getNodes().size(), monitorScheduler); }
private URL getWorkflowLocalDatasetURL(ProgramRunId workflowRunId) throws MalformedURLException { String path = String.format("apps/%s/workflows/%s/runs/%s/localdatasets", workflowRunId.getApplication(), workflowRunId.getProgram(), workflowRunId.getRun()); NamespaceId namespaceId = workflowRunId.getNamespaceId(); return config.resolveNamespacedURLV3(namespaceId, path); }
@Override public <T extends Dataset> T getDataset(String namespace, String name, Map<String, String> arguments, AccessType accessType) throws DatasetInstantiationException { NamespaceId programNamespaceId = runtimeContext.getProgramRunId().getNamespaceId(); if (NamespaceId.SYSTEM.getNamespace().equalsIgnoreCase(namespace) && !NamespaceId.SYSTEM.equals(programNamespaceId)) { throw new DatasetInstantiationException(String.format("Dataset %s cannot be instantiated from %s namespace. " + "Cannot access %s namespace.", name, programNamespaceId, NamespaceId.SYSTEM)); } T dataset = datasetCache.getDataset(namespace, name, arguments, accessType); // Only call startTx if the dataset hasn't been seen before // It is ok because there is only one transaction in this DatasetContext // If a dataset instance is being reused, we don't need to call startTx again. // It's also true for the case when a dataset instance was released and reused. if (datasets.add(dataset) && dataset instanceof TransactionAware) { ((TransactionAware) dataset).startTx(transaction); } return dataset; }
@Override public <T extends Dataset> T getDataset(String namespace, String name, Map<String, String> arguments, AccessType accessType) throws DatasetInstantiationException { NamespaceId programNamespaceId = runtimeContext.getProgramRunId().getNamespaceId(); if (NamespaceId.SYSTEM.getNamespace().equalsIgnoreCase(namespace) && !NamespaceId.SYSTEM.equals(programNamespaceId)) { throw new DatasetInstantiationException(String.format("Dataset %s cannot be instantiated from %s namespace. " + "Cannot access %s namespace.", name, programNamespaceId, NamespaceId.SYSTEM)); } T dataset = datasetCache.getDataset(namespace, name, arguments, accessType); // Only call startTx if the dataset hasn't been seen before // It is ok because there is only one transaction in this DatasetContext // If a dataset instance is being reused, we don't need to call startTx again. // It's also true for the case when a dataset instance was released and reused. if (datasets.add(dataset) && dataset instanceof TransactionAware) { ((TransactionAware) dataset).startTx(transaction); } return dataset; }
@Override public <T extends Dataset> T getDataset(String namespace, String name, Map<String, String> arguments, AccessType accessType) throws DatasetInstantiationException { NamespaceId programNamespaceId = runtimeContext.getProgramRunId().getNamespaceId(); if (NamespaceId.SYSTEM.getNamespace().equalsIgnoreCase(namespace) && !NamespaceId.SYSTEM.equals(programNamespaceId)) { throw new DatasetInstantiationException(String.format("Dataset %s cannot be instantiated from %s namespace. " + "Cannot access %s namespace.", name, programNamespaceId, NamespaceId.SYSTEM)); } T dataset = datasetCache.getDataset(namespace, name, arguments, accessType); // Only call startTx if the dataset hasn't been seen before // It is ok because there is only one transaction in this DatasetContext // If a dataset instance is being reused, we don't need to call startTx again. // It's also true for the case when a dataset instance was released and reused. if (datasets.add(dataset) && dataset instanceof TransactionAware) { ((TransactionAware) dataset).startTx(transaction); } return dataset; }
/** * Retrieve the {@link WorkflowToken} for the specified workflow run filtered by the specified * {@link WorkflowToken.Scope} and key. * * @param workflowRunId the run id of the workflow * @param scope the specified {@link WorkflowToken.Scope}. If null, it returns keys for * {@link WorkflowToken.Scope#USER} * @param key the specified key. If null, it returns all keys in the specified {@link WorkflowToken.Scope} * @return {@link WorkflowTokenDetail} with the specified filters */ public WorkflowTokenDetail getWorkflowToken(ProgramRunId workflowRunId, @Nullable WorkflowToken.Scope scope, @Nullable String key) throws IOException, UnauthenticatedException, NotFoundException, UnauthorizedException { String path = String.format("apps/%s/workflows/%s/runs/%s/token", workflowRunId.getApplication(), workflowRunId.getProgram(), workflowRunId.getRun()); URL url = config.resolveNamespacedURLV3(workflowRunId.getNamespaceId(), appendScopeAndKeyToUrl(path, scope, key)); HttpResponse response = restClient.execute(HttpMethod.GET, url, config.getAccessToken(), HttpURLConnection.HTTP_NOT_FOUND); if (response.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { String msg = "Either the workflow or its run id"; if (key != null) { msg = String.format("%s or the specified key at the specified scope", msg); } throw new NotFoundException(workflowRunId, msg); } return ObjectResponse.fromJsonBody(response, WorkflowTokenDetail.class, GSON).getResponseObject(); }
/** * Retrieve the keys set by the specified node in the {@link WorkflowToken} for the specified workflow run * filtered by the specified {@link WorkflowToken.Scope} and key. * * @param workflowRunId the run id of the workflow * @param nodeName the name of the node * @param scope the specified {@link WorkflowToken.Scope} * @param key the specified key * @return {@link WorkflowTokenDetail} with the specified filters */ public WorkflowTokenNodeDetail getWorkflowTokenAtNode(ProgramRunId workflowRunId, String nodeName, @Nullable WorkflowToken.Scope scope, @Nullable String key) throws IOException, UnauthenticatedException, NotFoundException, UnauthorizedException { String path = String.format("apps/%s/workflows/%s/runs/%s/nodes/%s/token", workflowRunId.getApplication(), workflowRunId.getProgram(), workflowRunId.getRun(), nodeName); URL url = config.resolveNamespacedURLV3(workflowRunId.getNamespaceId(), appendScopeAndKeyToUrl(path, scope, key)); HttpResponse response = restClient.execute(HttpMethod.GET, url, config.getAccessToken(), HttpURLConnection.HTTP_NOT_FOUND); if (response.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { String msg = "Either the workflow or its run id"; if (key != null) { msg = String.format("%s or the specified key at the specified scope", msg); } throw new NotFoundException(workflowRunId, msg); } return ObjectResponse.fromJsonBody(response, WorkflowTokenNodeDetail.class, GSON).getResponseObject(); }
/** * Get node states associated with the Workflow run. * @param workflowRunId run id for the Workflow * @return the map of node id to the {@link WorkflowNodeStateDetail} * @throws IOException if the error occurred during executing the http request * @throws UnauthenticatedException if the request is not authorized successfully in the gateway server * @throws NotFoundException if the workflow with given runid not found */ public Map<String, WorkflowNodeStateDetail> getWorkflowNodeStates(ProgramRunId workflowRunId) throws IOException, UnauthenticatedException, NotFoundException, UnauthorizedException { String path = String.format("apps/%s/workflows/%s/runs/%s/nodes/state", workflowRunId.getApplication(), workflowRunId.getProgram(), workflowRunId.getRun()); NamespaceId namespaceId = workflowRunId.getNamespaceId(); URL urlPath = config.resolveNamespacedURLV3(namespaceId, path); HttpResponse response = restClient.execute(HttpMethod.GET, urlPath, config.getAccessToken(), HttpURLConnection.HTTP_NOT_FOUND); if (response.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { throw new NotFoundException(workflowRunId); } return ObjectResponse.fromJsonBody(response, new TypeToken<Map<String, WorkflowNodeStateDetail>>() { }).getResponseObject(); } }
Optional<ProfileId> profileId = SystemArguments.getProfileIdFromArgs(programRunId.getNamespaceId(), systemArgs); if (!profileId.isPresent()) { LOG.error("Ignoring unexpected request to record provisioning state for program run {} that does not have "
private void addRuns(Store store, ProgramRunId... runs) { for (ProgramRunId run : runs) { ArtifactId artifactId = run.getNamespaceId().artifact("testArtifact", "1.0").toApiArtifactId(); setStartAndRunning(store, run.getParent(), run.getEntityName(), artifactId); } }
Optional<ProfileId> profileId = SystemArguments.getProfileIdFromArgs(programRunId.getNamespaceId(), systemArgs); if (!profileId.isPresent()) { LOG.error("Ignoring unexpected request to record provisioning state for program run {} that does not have "
/** Adds runs which have workflows associated with them * * @param store store instance * @param workflowName name of the workflow * @param workflowRunId run ID associated with all program runs * @param runs list ofo runs to be added */ private void addWorkflowRuns(Store store, String workflowName, String workflowRunId, ProgramRunId... runs) { Map<String, String> workflowIDMap = new HashMap<>(); Map<String, String> emptyMap = ImmutableMap.of(); workflowIDMap.put(ProgramOptionConstants.WORKFLOW_NAME, workflowName); workflowIDMap.put(ProgramOptionConstants.WORKFLOW_NODE_ID, "workflowNodeId"); workflowIDMap.put(ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId); workflowIDMap.put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()); for (ProgramRunId run : runs) { ArtifactId artifactId = run.getNamespaceId().artifact("testArtifact", "1.0").toApiArtifactId(); store.setProvisioning(run, emptyMap, workflowIDMap, AppFabricTestHelper.createSourceId(++sourceId), artifactId); store.setProvisioned(run, 0, AppFabricTestHelper.createSourceId(++sourceId)); store.setStart(run, null, workflowIDMap, AppFabricTestHelper.createSourceId(++sourceId)); store.setRunning(run, RunIds.getTime(run.getRun(), TimeUnit.SECONDS) + 1, null, AppFabricTestHelper.createSourceId(++sourceId)); } }