private static void addContext(Profiler profiler, CeTask task) { profiler .logTimeLast(true) .addContext("project", task.getComponentKey()) .addContext("type", task.getType()) .addContext("id", task.getUuid()); String submitterLogin = task.getSubmitterLogin(); if (submitterLogin != null) { profiler.addContext("submitter", submitterLogin); } } }
private static Project createProject(org.sonar.ce.queue.CeTask ceTask) { return new ProjectImpl( ceTask.getComponentUuid(), ceTask.getComponentKey(), ceTask.getComponentName()); }
public CeTask build() { return new CeTask(this); } }
@Override public void onEnd(CeTask ceTask, CeActivityDto.Status status, @Nullable CeTaskResult taskResult, @Nullable Throwable error) { if (status == CeActivityDto.Status.SUCCESS) { return; } String projectUuid = ceTask.getComponentUuid(); if (!CeTaskTypes.REPORT.equals(ceTask.getType()) || projectUuid == null) { return; } if (notificationService.hasProjectSubscribersForTypes(projectUuid, singleton(ReportAnalysisFailureNotification.TYPE))) { try (DbSession dbSession = dbClient.openSession(false)) { ComponentDto projectDto = dbClient.componentDao().selectOrFailByUuid(dbSession, projectUuid); checkScopeAndQualifier(projectDto); CeActivityDto ceActivityDto = dbClient.ceActivityDao().selectByUuid(dbSession, ceTask.getUuid()) .orElseThrow(() -> new RowNotFoundException(format("CeActivity with uuid '%s' not found", ceTask.getUuid()))); ReportAnalysisFailureNotification taskFailureNotification = buildNotification(ceActivityDto, projectDto, error); notificationService.deliver(taskFailureNotificationSerializer.toNotification(taskFailureNotification)); } } }
public void initForTask(CeTask task) { MDC.put(MDC_CE_TASK_UUID, task.getUuid()); }
/** * @return a {@link Runnable} to execute some checks on the project at the end of the step */ private Runnable loadProject(ScannerReport.Metadata reportMetadata, Organization organization) { String reportProjectKey = projectKeyFromReport(reportMetadata); String componentKey = ceTask.getComponentKey(); if (componentKey == null) { throw MessageException.of(format( "Compute Engine task component key is null. Project with UUID %s must have been deleted since report was uploaded. Can not proceed.", ceTask.getComponentUuid())); } ComponentDto dto = toProject(reportProjectKey); analysisMetadata.setProject(new Project(dto.uuid(), dto.getDbKey(), dto.name(), dto.description())); return () -> { if (!componentKey.equals(reportProjectKey)) { throw MessageException.of(format( "ProjectKey in report (%s) is not consistent with projectKey under which the report has been submitted (%s)", reportProjectKey, componentKey)); } if (!dto.getOrganizationUuid().equals(organization.getUuid())) { throw MessageException.of(format("Project is not in the expected organization: %s", organization.getKey())); } if (dto.getMainBranchProjectUuid() != null) { throw MessageException.of("Project should not reference a branch"); } }; }
@Override public void handle(Request wsRequest, Response wsResponse) throws Exception { String organizationKey = wsRequest.getParam(PARAM_ORGANIZATION_KEY) .emptyAsNull() .or(defaultOrganizationProvider.get()::getKey); String projectKey = wsRequest.mandatoryParam(PARAM_PROJECT_KEY); String projectBranch = wsRequest.param(PARAM_PROJECT_BRANCH); String projectName = abbreviate(defaultIfBlank(wsRequest.param(PARAM_PROJECT_NAME), projectKey), MAX_COMPONENT_NAME_LENGTH); Map<String, String> characteristics = parseTaskCharacteristics(wsRequest); try (InputStream report = new BufferedInputStream(wsRequest.mandatoryParamAsPart(PARAM_REPORT_DATA).getInputStream())) { CeTask task = reportSubmitter.submit(organizationKey, projectKey, projectBranch, projectName, characteristics, report); Ce.SubmitResponse submitResponse = Ce.SubmitResponse.newBuilder() .setTaskId(task.getUuid()) .setProjectId(task.getComponentUuid()) .build(); WsUtils.writeProtobuf(submitResponse, wsRequest, wsResponse); } }
private void executeTask(CeTask task) { callListeners(t -> t.onStart(task)); Profiler ceProfiler = startActivityProfiler(task); CeActivityDto.Status status = CeActivityDto.Status.FAILED; CeTaskResult taskResult = null; Throwable error = null; try { // TODO delegate the message to the related task processor, according to task type Optional<CeTaskProcessor> taskProcessor = taskProcessorRepository.getForCeTask(task); if (taskProcessor.isPresent()) { taskResult = taskProcessor.get().process(task); status = CeActivityDto.Status.SUCCESS; } else { LOG.error("No CeTaskProcessor is defined for task of type {}. Plugin configuration may have changed", task.getType()); status = CeActivityDto.Status.FAILED; } } catch (MessageException e) { // error error = e; } catch (Throwable e) { // error LOG.error(format("Failed to execute task %s", task.getUuid()), e); error = e; } finally { finalizeTask(task, ceProfiler, status, taskResult, error); } }
@Override protected void execute(Context context) throws SQLException { Long unInitializedFileSources = context.prepareSelect("select count(1) from file_sources where line_count = ? and project_uuid = ?") .setInt(1, LINE_COUNT_NOT_POPULATED) .setString(2, ceTask.getComponentUuid()) .get(row -> row.getLong(1)); if (unInitializedFileSources != null && unInitializedFileSources > 0) { MassUpdate massUpdate = context.prepareMassUpdate(); massUpdate.select("select id,line_hashes from file_sources where line_count = ? and project_uuid = ?") .setInt(1, LINE_COUNT_NOT_POPULATED) .setString(2, ceTask.getComponentUuid()); massUpdate.update("update file_sources set line_count = ? where id = ?"); massUpdate.rowPluralName("line counts of sources of project " + ceTask.getComponentUuid()); massUpdate.execute(PopulateFileSourceLineCount::handle); } }
@Override public Optional<CeTaskProcessor> getForCeTask(CeTask ceTask) { return Optional.ofNullable(taskProcessorByCeTaskType.get(ceTask.getType())); }
private Organization loadOrganization(ScannerReport.Metadata reportMetadata) { try (DbSession dbSession = dbClient.openSession(false)) { Organization organization = toOrganization(dbSession, ceTask.getOrganizationUuid()); checkOrganizationKeyConsistency(reportMetadata, organization); analysisMetadata.setOrganization(organization); analysisMetadata.setOrganizationsEnabled(organizationFlags.isEnabled(dbSession)); return organization; } }
@Override public void dumpAsCsv(ScoreMatrix scoreMatrix) { if (configuration.getBoolean("sonar.filemove.dumpCsv").orElse(false)) { try { Path tempFile = Files.createTempFile(String.format("score-matrix-%s", ceTask.getUuid()), ".csv"); try (BufferedWriter writer = Files.newBufferedWriter(tempFile, UTF_8)) { writer.write(scoreMatrix.toCsv(';')); } LOG.info("File move similarity score matrix dumped as CSV: {}", tempFile); } catch (IOException e) { LOG.error("Failed to dump ScoreMatrix as CSV", e); } } } }
@Override public void execute() { try (DbSession dbSession = dbClient.openSession(false)) { Optional<CeTaskInputDao.DataStream> opt = dbClient.ceTaskInputDao().selectData(dbSession, task.getUuid()); if (opt.isPresent()) { File unzippedDir = tempFolder.newDir(); try (CeTaskInputDao.DataStream reportStream = opt.get(); InputStream zipStream = new BufferedInputStream(reportStream.getInputStream())) { ZipUtils.unzip(zipStream, unzippedDir); } catch (IOException e) { throw new IllegalStateException("Fail to extract report " + task.getUuid() + " from database", e); } reportDirectoryHolder.setDirectory(unzippedDir); } else { throw MessageException.of("Analysis report " + task.getUuid() + " is missing in database"); } } }
private Result findAndProcessTask() { if (!enabledCeWorkerController.isEnabled(this)) { return DISABLED; } Optional<CeTask> ceTask = tryAndFindTaskToExecute(); if (!ceTask.isPresent()) { return NO_TASK; } try (EnabledCeWorkerController.ProcessingRecorderHook processing = enabledCeWorkerController.registerProcessingFor(this)) { executeTask(ceTask.get()); } catch (Exception e) { LOG.error(format("An error occurred while executing task with uuid '%s'", ceTask.get().getUuid()), e); } return TASK_PROCESSED; }
private void finalizeTask(CeTask task, Profiler ceProfiler, CeActivityDto.Status status, @Nullable CeTaskResult taskResult, @Nullable Throwable error) { try { queue.remove(task, status, taskResult, error); } catch (Exception e) { String errorMessage = format("Failed to finalize task with uuid '%s' and persist its state to db", task.getUuid()); if (error instanceof MessageException) { LOG.error(format("%s. Task failed with MessageException \"%s\"", errorMessage, error.getMessage()), e); } else { LOG.error(errorMessage, e); } } finally { // finalize stopActivityProfiler(ceProfiler, task, status); callListeners(t -> t.onEnd(task, status, taskResult, error)); } }
@Override public void remove(CeTask task, CeActivityDto.Status status, @Nullable CeTaskResult taskResult, @Nullable Throwable error) { checkArgument(error == null || status == CeActivityDto.Status.FAILED, "Error can be provided only when status is FAILED"); try (DbSession dbSession = dbClient.openSession(false)) { CeQueueDto queueDto = dbClient.ceQueueDao().selectByUuid(dbSession, task.getUuid()) .orElseThrow(() -> new IllegalStateException("Task does not exist anymore: " + task)); CeActivityDto activityDto = new CeActivityDto(queueDto); activityDto.setStatus(status); updateQueueStatus(status, activityDto); updateTaskResult(activityDto, taskResult); updateError(activityDto, error); remove(dbSession, queueDto, activityDto); } }
@Override public void execute() { try (CloseableIterator<String> logsIterator = reportReader.readScannerLogs()) { if (logsIterator.hasNext()) { try (DbSession dbSession = dbClient.openSession(false)) { // in case the task was restarted, the context might have been already persisted // for total reliability, we rather delete the existing row as we don't want to assume the content // consistent with the report dbClient.ceScannerContextDao().deleteByUuids(dbSession, singleton(ceTask.getUuid())); dbSession.commit(); dbClient.ceScannerContextDao().insert(dbSession, ceTask.getUuid(), logsIterator); dbSession.commit(); } } } } }
private ProjectAnalysisImpl createProjectAnalysis(CeTask.Status status) { return new ProjectAnalysisImpl( createOrganization(), new CeTaskImpl(this.ceTask.getUuid(), status), createProject(this.ceTask), getAnalysis().orElse(null), getAnalysis().map(a -> a.getDate().getTime()).orElse(system2.now()), ScannerContextImpl.from(reportReader.readContextProperties()), status == SUCCESS ? createQualityGate() : null, createBranch()); }