public Topology(Topology other) { if (other != null) { setId(other.getId()); setVersionId(other.getVersionId()); setName(other.getName()); setDescription(other.getDescription()); setConfig(other.getConfig()); setNamespaceId(other.getNamespaceId()); setVersionTimestamp(other.getVersionTimestamp()); } // this will be re-generated during deployment. topologyDag = null; }
private TopologyData doExportTopology(Topology topology) throws Exception { TopologyDag dag = topologyDagBuilder.getDag(topology); topology.setTopologyDag(dag); TopologyData topologyData = new TopologyData(); TopologyExportVisitor exportVisitor = new TopologyExportVisitor(topology.getId(), topologyData, this); topologyData.setTopologyName(topology.getName()); topologyData.setConfig(topology.getConfig()); TopologyDag topologyDag = topology.getTopologyDag(); if (topologyDag != null) { topologyDag.traverse(exportVisitor); } topologyData.setMetadata(getTopologyEditorMetadata(topology.getId())); return topologyData; }
public Topology removeTopology(Long topologyId, Long versionId, boolean recurse) { Topology topology = new Topology(); topology.setId(topologyId); topology.setVersionId(versionId); if (recurse) { try { removeTopologyDependencies(topology.getId(), topology.getVersionId()); } catch (Exception ex) { LOG.error("Got exception while removing topology dependencies", ex); throw new RuntimeException(ex); } } Topology removedTopology = dao.remove(topology.getStorableKey()); removeTopologyVersionInfo(versionId); return removedTopology; }
private TopologySampling getSamplingInstance(Topology topology) { Namespace namespace = environmentService.getNamespace(topology.getNamespaceId()); if (namespace == null) { throw new RuntimeException("Corresponding namespace not found: " + topology.getNamespaceId()); } TopologySampling topologySampling = topologySamplingContainer.findInstance(namespace); if (topologySampling == null) { throw new RuntimeException("Can't find Topology sampling for such namespace " + topology.getNamespaceId()); } return topologySampling; } }
private String getTopologyTestRunEventLog(Topology topology) { return topologyTestRunResultDir + File.separator + "topology-test-run-event-topology-" + topology.getId() + "-" + UUID.randomUUID().toString() + ".log"; }
@PUT @Path("/topologies/{topologyId}") @Timed public Response addOrUpdateTopology (@PathParam("topologyId") Long topologyId, Topology topology, @Context SecurityContext securityContext) { SecurityUtil.checkRoleOrPermissions(authorizer, securityContext, Roles.ROLE_TOPOLOGY_SUPER_ADMIN, NAMESPACE, topologyId, WRITE); if (StringUtils.isEmpty(topology.getName())) { throw BadRequestException.missingParameter(Topology.NAME); } if (StringUtils.isEmpty(topology.getConfig())) { throw BadRequestException.missingParameter(Topology.CONFIG); } if (topology.getNamespaceId() == null) { throw BadRequestException.missingParameter(Topology.NAMESPACE_ID); } Topology existingTopology = catalogService.getTopology(topologyId); Topology result = catalogService.addOrUpdateTopology(topologyId, topology); if (existingTopology != null) { Long prevNamespaceId = existingTopology.getNamespaceId(); if (!result.getNamespaceId().equals(prevNamespaceId)) { LOG.info("Determined namespace change on topology: " + topologyId); // environment has changed: it should set 'reconfigure' to all components catalogService.setReconfigureOnAllComponentsInTopology(result); } } return WSUtils.respondEntity(result, OK); }
private List<QueryParam> queryParam(Topology topology) { List<QueryParam> queryParams = new ArrayList<>(); queryParams.add(new QueryParam(TopologyComponent.TOPOLOGYID, topology.getId().toString())); queryParams.add(new QueryParam(TopologyComponent.VERSIONID, topology.getVersionId().toString())); return queryParams; } }
TopologyMetricsService metricsService, StreamCatalogService catalogService) { LOG.debug("[START] enrichTopology - topology id: {}", topology.getId()); Stopwatch stopwatch = Stopwatch.createStarted(); Namespace namespace = environmentService.getNamespace(topology.getNamespaceId()); if (namespace != null) { namespaceName = namespace.getName(); detailedResponse.setRuntime(new TopologyRuntimeResponse(runtimeTopologyId, topologyMetric, latenciesTopN)); } catch (TopologyNotAliveException e) { LOG.debug("Topology {} is not alive", topology.getId()); detailedResponse = new TopologyDashboardResponse(topology, TopologyRunningStatus.NOT_RUNNING, namespaceName); catalogService.getTopologyState(topology.getId()) .ifPresent(state -> { if (TopologyStateFactory.getInstance().getTopologyState(state.getName()) == TopologyStates.TOPOLOGY_STATE_DEPLOYED) { LOG.debug("[END] enrichTopology - topology id: {}, elapsed: {} ms", topology.getId(), stopwatch.elapsed(TimeUnit.MILLISECONDS));
public Topology addTopology(Topology topology) { validateTopology(topology); boolean storedPlaceholderVersionTopology = false; if (topology.getId() == null) { topology.setId(this.dao.nextId(TOPOLOGY_NAMESPACE)); topology.setVersionId(PLACEHOLDER_ID); this.dao.add(topology); LOG.debug("Added topology {} with placeholder version", topology); storedPlaceholderVersionTopology = true; } long timestamp = System.currentTimeMillis(); topology.setVersionTimestamp(timestamp); TopologyVersion versionInfo = addCurrentTopologyVersionInfo(topology.getId(), timestamp); LOG.debug("Added version info {}", versionInfo); if (storedPlaceholderVersionTopology) { // remove topology with placeholder version first // WARN: don't use removeTopology since it also removes PLACEHOLDER topology version info! removeOnlyTopologyEntity(topology.getId(), topology.getVersionId()); } // put actual version id topology.setVersionId(versionInfo.getId()); this.dao.addOrUpdate(topology); LOG.debug("Added topology {}", topology); return topology; }
public Topology cloneTopology(Long namespaceId, Topology topology) throws Exception { Preconditions.checkNotNull(topology, "Topology does not exist"); TopologyData exported = new TopologyData(doExportTopology(topology)); Optional<String> latest = getLatestCloneName(exported.getTopologyName(), listTopologies()); exported.setTopologyName(getNextCloneName(latest.orElse(topology.getName()))); if (namespaceId == null) { namespaceId = topology.getNamespaceId(); } return importTopology(namespaceId, exported); }
public Topology importTopology(Long namespaceId, TopologyData topologyData) throws Exception { Preconditions.checkNotNull(topologyData); Topology newTopology = new Topology(); try { newTopology.setName(topologyData.getTopologyName()); newTopology.setConfig(topologyData.getConfig()); newTopology.setNamespaceId(namespaceId); addTopology(newTopology); } catch (Exception ex) { LOG.error("Got exception while importing the topology", ex); throw ex; } try { doImportTopology(newTopology, topologyData); } catch (Exception ex) { LOG.error("Got exception while importing the topology", ex); removeTopology(newTopology.getId(), true); throw ex; } return newTopology; }
/** * Clones the given version of the topology and all its dependencies to a new 'CURRENT' version. * The ids of the topology and its dependencies are retained. */ public Topology cloneTopologyVersion(Long topologyId, Long versionId) { Topology topology = getTopology(topologyId, versionId); if (topology != null) { try { topology = addTopology(new Topology(topology)); copyTopologyDependencies(topologyId, versionId, topology.getVersionId()); } catch (Exception ex) { LOG.error("Got exception while copying topology dependencies", ex); removeTopology(topology.getId(), topology.getVersionId(), true); throw new RuntimeException(ex); } } return topology; }
private Void runTestInBackground(TopologyActions topologyActions, Topology topology, TopologyTestRunHistory history, Map<String, TestRunSource> testRunSourceMap, Map<String, TestRunProcessor> testRunProcessorMap, Map<String, TestRunRulesProcessor> testRunRulesProcessorMap, Map<String, TestRunSink> testRunSinkMap, Map<String, List<Map<String, Object>>> expectedOutputRecordsMap, Optional<Long> durationSecs) throws IOException { TopologyLayout topologyLayout = CatalogToLayoutConverter.getTopologyLayout(topology, topology.getTopologyDag()); try { topologyActionsService.setUpClusterArtifacts(topology, topologyActions); String mavenArtifacts = topologyActionsService.setUpExtraJars(topology, topologyActions); topologyActions.runTest(topologyLayout, history, mavenArtifacts, testRunSourceMap, testRunProcessorMap, testRunRulesProcessorMap, testRunSinkMap, durationSecs); history.finishSuccessfully(); Map<String, List<Map<String, Object>>> actualOutputRecordsMap = parseTestRunOutputFiles(testRunSinkMap); history.setActualOutputRecords(objectMapper.writeValueAsString(actualOutputRecordsMap)); if (expectedOutputRecordsMap != null) { boolean matched = equalsOutputRecords(expectedOutputRecordsMap, actualOutputRecordsMap); history.setMatched(matched); } } catch (Throwable e) { LOG.warn("Exception thrown while running Topology as test mode. Marking as 'failed'. topology id: {}", topology.getId(), e); history.finishWithFailures(); } catalogService.addOrUpdateTopologyTestRunHistory(history.getId(), history); return null; }
@GET @Path("/topologies/{topologyId}/actions/export") @Produces(MediaType.APPLICATION_OCTET_STREAM) @Timed public Response exportTopology(@PathParam("topologyId") Long topologyId, @Context SecurityContext securityContext) throws Exception { SecurityUtil.checkRoleOrPermissions(authorizer, securityContext, Roles.ROLE_TOPOLOGY_SUPER_ADMIN, NAMESPACE, topologyId, READ, EXECUTE); Topology topology = catalogService.getTopology(topologyId); if (topology != null) { String exportedTopology = catalogService.exportTopology(topology); if (!StringUtils.isEmpty(exportedTopology)) { InputStream is = new ByteArrayInputStream(exportedTopology.getBytes(StandardCharsets.UTF_8)); return Response.status(OK) .entity(is) .header("Content-Disposition", "attachment; filename=\"" + topology.getName() + ".json\"") .build(); } } throw EntityNotFoundException.byId(topologyId.toString()); }
compared = c1.getTopology().getName().compareTo(c2.getTopology().getName()); break; case STATUS: break; case LAST_UPDATED: compared = c1.getTopology().getVersionTimestamp().compareTo(c2.getTopology().getVersionTimestamp()); break; default:
@Override public void deploy(TopologyContext context) throws Exception { TopologyActions topologyActions = context.getTopologyActions(); Topology topology = context.getTopology(); TopologyDag dag = topology.getTopologyDag(); TopologyLayout layout = CatalogToLayoutConverter.getTopologyLayout(topology, dag); if (dag == null) { throw new IllegalStateException("Topology dag not set up"); } try { context.setCurrentAction("Submitting topology to streaming engine"); String mavenArtifacts = context.getMavenArtifacts(); topologyActions.deploy(layout, mavenArtifacts, context, context.getAsUser()); context.setState(TOPOLOGY_STATE_DEPLOYED); context.setCurrentAction("Topology deployed"); } catch (Exception ex) { LOG.error("Error while trying to deploy the topology in the streaming engine", ex); LOG.error("Trying to kill any running instance of topology '{}'", context.getTopology().getName()); killTopologyIfRunning(context, layout); context.setState(TOPOLOGY_STATE_DEPLOYMENT_FAILED); context.setCurrentAction("Topology submission failed due to: " + ex); throw new IgnoreTransactionRollbackException(ex); } }
public Collection<Topology> listTopologies(List<QueryParam> queryParams) { Collection<Topology> topologies = this.dao.find(TOPOLOGY_NAMESPACE, queryParams); topologies.forEach(t -> t.setVersionTimestamp(getVersionTimestamp(t.getVersionId()))); return topologies; }
public String setUpExtraJars(Topology topology, TopologyActions topologyActions) throws IOException { StormTopologyDependenciesHandler extraJarsHandler = new StormTopologyDependenciesHandler(catalogService); topology.getTopologyDag().traverse(extraJarsHandler); Path extraJarsLocation = topologyActions.getExtraJarsLocation(CatalogToLayoutConverter.getTopologyLayout(topology)); makeEmptyDir(extraJarsLocation); Set<String> extraJars = new HashSet<>(); extraJars.addAll(extraJarsHandler.getExtraJars()); extraJars.addAll(getBundleJars(extraJarsHandler.getTopologyComponentBundleSet())); downloadAndCopyJars(extraJars, extraJarsLocation); return extraJarsHandler.getMavenDeps(); }
public TopologyTestRunHistory testRunTopology(Topology topology, TopologyTestRunCase testCase, Long durationSecs) throws Exception { TopologyDag dag = topologyDagBuilder.getDag(topology); topology.setTopologyDag(dag); ensureValid(dag); TopologyActions topologyActions = getTopologyActionsInstance(topology); LOG.debug("Running topology {} in test mode", topology); return topologyTestRunner.runTest(topologyActions, topology, testCase, durationSecs); }