public QueryResult<Job> queue(String studyStr, String jobName, String description, String executable, Job.Type type, Map<String, String> params, List<File> input, List<File> output, File outDir, Map<String, Object> attributes, String token) throws CatalogException { Job job = new Job(jobName, null, executable, type, input, output, outDir, params, -1) .setDescription(description) .setToolId("variant_index") .setAttributes(attributes); return create(studyStr, job, QueryOptions.empty(), token); }
ParamUtils.checkParameter(job.getToolId(), "toolId"); job.setId(ParamUtils.defaultString(job.getId(), job.getToolId() + "_" + TimeUtils.getTimeMillis())); job.setDescription(ParamUtils.defaultString(job.getDescription(), "")); job.setCreationDate(ParamUtils.defaultString(job.getCreationDate(), TimeUtils.getTime())); job.setStatus(ParamUtils.defaultObject(job.getStatus(), new Job.JobStatus(Job.JobStatus.PREPARED))); job.setInput(ParamUtils.defaultObject(job.getInput(), Collections.emptyList())); job.setOutput(ParamUtils.defaultObject(job.getOutput(), Collections.emptyList())); job.setExecution(ParamUtils.defaultObject(job.getExecution(), "")); job.setParams(ParamUtils.defaultObject(job.getParams(), HashMap::new)); job.setResourceManagerAttributes(ParamUtils.defaultObject(job.getResourceManagerAttributes(), HashMap::new)); job.setAttributes(ParamUtils.defaultObject(job.getAttributes(), HashMap::new)); job.setUserId(userId); job.setRelease(catalogManager.getStudyManager().getCurrentRelease(study, userId)); job.setOutDir(job.getOutDir() != null && StringUtils.isNotEmpty(job.getOutDir().getPath()) ? job.getOutDir() : null); for (File inputFile : job.getInput()) { String fileId = StringUtils.isEmpty(inputFile.getPath()) ? inputFile.getName() : inputFile.getPath(); try { job.setInput(inputFileList); for (File outputFile : job.getOutput()) { String fileId = StringUtils.isEmpty(outputFile.getPath()) ? outputFile.getName() : outputFile.getPath(); try { job.setOutput(outputFileList); if (job.getOutDir() != null) {
Path path = getJobTemporaryFolder(job.getUid(), tempJobFolder); try { catalogIOManager.createDirectory(path.toUri()); String stderr = path.resolve(job.getName() + '_' + job.getUid() + ".err").toString(); String stdout = path.resolve(job.getName() + '_' + job.getUid() + ".out").toString(); String userToken = catalogManager.getUserManager().getSystemTokenForUser(job.getUserId(), sessionId); if (job.getToolId().equals("opencga-analysis")) { commandLine.append(job.getExecution()).append(' '); } else { commandLine.append("tools execute "); commandLine.append("--job ").append(job.getUid()).append(' '); logger.info("Updating job {} from {} to {}", job.getUid(), Job.JobStatus.PREPARED, Job.JobStatus.QUEUED); try { job.getAttributes().put(Job.OPENCGA_TMP_DIR, path.toString()); job.getResourceManagerAttributes().put(AbstractExecutor.STDOUT, stdout); job.getResourceManagerAttributes().put(AbstractExecutor.STDERR, stderr); job.getResourceManagerAttributes().put(AbstractExecutor.OUTDIR, path.toString()); .append(JobDBAdaptor.QueryParams.ATTRIBUTES.key(), job.getAttributes()) .append(JobDBAdaptor.QueryParams.RESOURCE_MANAGER_ATTRIBUTES.key(), job.getResourceManagerAttributes()); QueryResult<Job> update = jobDBAdaptor.update(job.getUid(), params, QueryOptions.empty()); if (update.getNumResults() == 1) {
public int execute(Job job) throws CatalogException, AnalysisExecutionException { return execute(job.getToolId(), job.getExecution(), catalogManager.getJobManager().getStudyId(job.getUid()), job.getParams()); }
void cleanPrivateJobInformation(Job job) { // Remove the session id from the job attributes job.getAttributes().remove(Job.OPENCGA_TMP_DIR); job.getAttributes().remove(Job.OPENCGA_OUTPUT_DIR); job.getAttributes().remove(Job.OPENCGA_STUDY); ObjectMap params = new ObjectMap(JobDBAdaptor.QueryParams.ATTRIBUTES.key(), job.getAttributes()); try { dbAdaptorFactory.getCatalogJobDBAdaptor().update(job.getUid(), params, QueryOptions.empty()); } catch (CatalogException e) { logger.error("Could not remove session id from attributes of job {}. ", job.getUid(), e); } }
@Test public void updateInputAndOutputFiles() throws Exception { Job job = new Job() .setOutDir(new File().setUid(5)) .setStatus(new Job.JobStatus()); long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); job.setName("jobName1"); QueryResult<Job> insert = catalogJobDBAdaptor.insert(job, studyId, null); List<File> fileInput = Arrays.asList( new File().setUid(5L).setName("file1").setStatus(new File.FileStatus()), new File().setUid(6L).setName("file2").setStatus(new File.FileStatus()), new File().setUid(7L).setName("file3").setStatus(new File.FileStatus()) ); List<File> fileOutput = Arrays.asList( new File().setUid(15L).setName("file1").setStatus(new File.FileStatus()), new File().setUid(16L).setName("file2").setStatus(new File.FileStatus()), new File().setUid(17L).setName("file3").setStatus(new File.FileStatus()) ); ObjectMap params = new ObjectMap() .append(JobDBAdaptor.QueryParams.INPUT.key(), fileInput) .append(JobDBAdaptor.QueryParams.OUTPUT.key(), fileOutput); QueryResult<Job> update = catalogJobDBAdaptor.update(insert.first().getUid(), params, QueryOptions.empty()); assertEquals(3, update.first().getInput().size()); assertEquals(3, update.first().getOutput().size()); assertTrue(Arrays.asList(5L, 6L, 7L).containsAll(update.first().getInput().stream().map(File::getUid).collect(Collectors.toList()))); assertTrue(Arrays.asList(15L, 16L, 17L) .containsAll(update.first().getOutput().stream().map(File::getUid).collect(Collectors.toList()))); }
/** * Job methods * *************************** */ @Test public void testCreateJob() throws CatalogException, IOException { Query query = new Query(StudyDBAdaptor.QueryParams.OWNER.key(), "user"); String studyId = catalogManager.getStudyManager().get(query, null, sessionIdUser).first().getId(); File outDir = catalogManager.getFileManager().createFolder(studyId, Paths.get("jobs", "myJob").toString(), null, true, null, QueryOptions.empty(), sessionIdUser).first(); catalogManager.getJobManager().create(studyId,new Job().setId("myJob").setToolId("samtool").setDescription("description") .setOutDir(outDir).setExecution("echo \"Hello world!\"").setStatus(new Job.JobStatus(Job.JobStatus.PREPARED)), null, sessionIdUser); catalogManager.getJobManager().create(studyId, new Job().setId("myReadyJob").setToolId("samtool").setDescription("description") .setOutDir(outDir).setExecution("echo \"Hello world!\"").setStatus(new Job.JobStatus(Job.JobStatus.READY)), null, sessionIdUser); catalogManager.getJobManager().create(studyId, new Job().setId("myQueuedJob").setToolId("samtool").setDescription("description") .setOutDir(outDir).setExecution("echo \"Hello world!\"").setStatus(new Job.JobStatus(Job.JobStatus.QUEUED)), null, sessionIdUser); catalogManager.getJobManager().create(studyId, new Job().setId("myErrorJob").setToolId("samtool").setDescription("description") .setOutDir(outDir).setExecution("echo \"Hello world!\"").setStatus(new Job.JobStatus(Job.JobStatus.ERROR)), null, sessionIdUser); query = new Query() .append(JobDBAdaptor.QueryParams.STATUS_NAME.key(), Arrays.asList(Job.JobStatus.PREPARED, Job.JobStatus.QUEUED, Job.JobStatus.RUNNING, Job.JobStatus.DONE)); QueryResult<Job> unfinishedJobs = catalogManager.getJobManager().get(String.valueOf(studyId), query, null, sessionIdUser); assertEquals(2, unfinishedJobs.getNumResults()); QueryResult<Job> allJobs = catalogManager.getJobManager().get(String.valueOf(studyId), (Query) null, null, sessionIdUser); assertEquals(4, allJobs.getNumResults()); }
Path path = getJobTemporaryFolder(job.getUid(), tempJobFolder); try { catalogIOManager.createDirectory(path.toUri()); String stderr = path.resolve(job.getName() + '_' + job.getUid() + ".err").toString(); String stdout = path.resolve(job.getName() + '_' + job.getUid() + ".out").toString(); String userId = job.getUserId(); String userSessionId = null; try { StringBuilder commandLine = new StringBuilder(binHome).append(job.getExecutable()); job.getParams().put("outdir", path.toString()); if (job.getAttributes().get(INDEX_TYPE).toString().equalsIgnoreCase(VARIANT_TYPE)) { try { job.getParams().put("path", dbAdaptorFactory.getCatalogFileDBAdaptor().get(job.getOutDir().getUid(), QueryOptions.empty()).first().getId()); } catch (CatalogDBException e) { "L", "log-level", "merge", "o", "outdir", "overwrite-annotations", "path", "queue", "s", "study", "transform", "transformed-files", "resume", "load-split-data")); buildCommandLine(job.getParams(), commandLine, knownParams); } else { commandLine.append(" alignment index"); for (Map.Entry<String, String> param : job.getParams().entrySet()) { commandLine.append(' '); commandLine.append("--").append(param.getKey());
@Test public void getAllJobs() throws CatalogException { Job job = new Job() .setId("job1") .setToolId("toolId") .setOutDir(new File().setPath(data_d1_d2)); long job1 = catalogManager.getJobManager().create(studyFqn, job, null, ownerSessionId).first().getUid(); job.setId("job2"); long job2 = catalogManager.getJobManager().create(studyFqn, job, null, ownerSessionId).first().getUid(); job.setId("job3"); long job3 = catalogManager.getJobManager().create(studyFqn, job, null, ownerSessionId).first().getUid(); job.setId("job4"); long job4 = catalogManager.getJobManager().create(studyFqn, job, null, ownerSessionId).first().getUid(); checkGetAllJobs(Arrays.asList(job1, job2, job3, job4), ownerSessionId); //Owner can see everything checkGetAllJobs(Collections.emptyList(), externalSessionId); //Can't see inside data_d1_d2_d3 }
Job job1 = new Job() .setId("job1") .setCreationDate(TimeUtils.getTime()) .setStatus(new Job.JobStatus()); Date oneHourBack = cal.getTime(); Job job2 = new Job() .setId("job2") .setCreationDate(TimeUtils.getTime(oneHourBack)) .setStatus(new Job.JobStatus()); assertTrue("job1".equals(jobQueryResult1.getResult().get(0).getId())); assertTrue("job2".equals(jobQueryResult1.getResult().get(1).getId())); assertTrue("job2".equals(jobQueryResult2.getResult().get(0).getId())); assertTrue("job1".equals(jobQueryResult2.getResult().get(1).getId()));
@Override public Document convertToStorageType(Job object) { Document document = super.convertToStorageType(object); document.put("uid", object.getUid()); document.put("studyUid", object.getStudyUid()); document.put("outDir", convertFileToDocument(object.getOutDir())); document.put("input", convertFilesToDocument(object.getInput())); document.put("output", convertFilesToDocument(object.getOutput())); return document; }
Set<Long> fileIdsReferencedInJobs = new HashSet<>(); for (Job job : jobInputFiles.getResult()) { fileIdsReferencedInJobs.addAll(job.getInput().stream().map(File::getUid).collect(Collectors.toList())); jobMap.put(job.getUid(), job); fileIdsReferencedInJobs.addAll(job.getOutput().stream().map(File::getUid).collect(Collectors.toList())); jobMap.put(job.getUid(), job); fileIdsReferencedInJobs.add(job.getOutDir().getUid()); jobMap.put(job.getUid(), job); Job job = jobMap.get(jobAux.getUid()); List<File> inputFiles = new ArrayList<>(job.getInput().size()); List<File> attributeFiles = new ArrayList<>(job.getInput().size()); for (File file : job.getInput()) { if (relatedFileMap.containsKey(file.getUid())) { attributeFiles.add(relatedFileMap.get(file.getUid())); Map<String, Object> attributes = job.getAttributes(); ObjectMap opencgaAttributes; if (!attributes.containsKey(Constants.PRIVATE_OPENCGA_ATTRIBUTES)) { .append(JobDBAdaptor.QueryParams.ATTRIBUTES.key(), attributes) .append(JobDBAdaptor.QueryParams.INPUT.key(), inputFiles); jobDBAdaptor.update(job.getUid(), params, QueryOptions.empty()); Job job = jobMap.get(jobAux.getUid());
@Test public void deleteJobTest() throws CatalogException { long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); Job job = catalogJobDBAdaptor.insert(new Job("name", user3.getId(), "", "", "", new File().setUid(4), Collections.emptyList(), 1), studyId, null).first(); long jobId = job.getUid(); assertEquals(Job.JobStatus.PREPARED, job.getStatus().getName()); catalogJobDBAdaptor.delete(jobId); thrown.expect(CatalogDBException.class); thrown.expectMessage("not exist"); catalogJobDBAdaptor.get(jobId, QueryOptions.empty()); }
if (simulate) { //Simulate a job. Do not create it. jobQueryResult = new QueryResult<>("simulatedJob", (int) (System.currentTimeMillis() - start), 1, 1, "", "", Collections.singletonList( new Job(jobName, catalogManager.getUserManager().getUserId(sessionId), toolName, description, commandLine, outDir, inputFiles, 1))); } else { throw new AnalysisExecutionException(e.getCause()); jobQueryResult = catalogManager.getJobManager().get(job.getUid(), null, sessionId);
void executeJob(Job job, String token) { try { executorManager.execute(job, token); } catch (Exception e) { logger.error("Error executing job {}.", job.getUid(), e); } }
public File(String name, Type type, Format format, Bioformat bioformat, String path, String description, FileStatus status, long size, int release) { this(name, type, format, bioformat, null, path, null, TimeUtils.getTime(), TimeUtils.getTime(), description, status, false, size, null, new Experiment(), Collections.emptyList(), new Job(), Collections.emptyList(), new FileIndex(), release, Collections.emptyList(), Collections.emptyMap(), Collections.emptyMap()); }
@Override protected String getStatus(Job job) { String status; try { status = SgeManager.status(Objects.toString(job.getResourceManagerAttributes().get(Job.JOB_SCHEDULER_NAME))); } catch (Exception e) { logger.error("Could not obtain the status of the job {} from SGE. Error: {}", job.getUid(), e.getMessage()); return Job.JobStatus.UNKNOWN; } switch (status) { case SgeManager.ERROR: case SgeManager.EXECUTION_ERROR: return Job.JobStatus.ERROR; case SgeManager.FINISHED: return Job.JobStatus.READY; case SgeManager.QUEUED: return Job.JobStatus.QUEUED; case SgeManager.RUNNING: case SgeManager.TRANSFERRED: return Job.JobStatus.RUNNING; case SgeManager.UNKNOWN: default: return job.getStatus().getName(); } }
@Test public void SetVisitedJob() throws CatalogException { long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); Job jobBefore = catalogJobDBAdaptor.insert(new Job("name", user3.getId(), "", "", "", new File().setUid(4), Collections.emptyList(), 1), studyId, null).first(); long jobId = jobBefore.getUid(); assertTrue(!jobBefore.isVisited()); ObjectMap params = new ObjectMap(JobDBAdaptor.QueryParams.VISITED.key(), true); catalogJobDBAdaptor.update(jobBefore.getUid(), params, QueryOptions.empty()); Job jobAfter = catalogJobDBAdaptor.get(jobId, null).first(); assertTrue(jobAfter.isVisited()); }
logger.info("Ready to run {}", job.getCommandLine()); Command com = new Command(job.getCommandLine()); if (job.getResourceManagerAttributes().containsKey("STDOUT")) { URI sout = Paths.get((String) job.getResourceManagerAttributes().get("STDOUT")).toUri(); com.setOutputOutputStream(ioManager.createOutputStream(sout, false)); if (job.getResourceManagerAttributes().containsKey("STDERR")) { URI serr = Paths.get((String) job.getResourceManagerAttributes().get("STDERR")).toUri(); com.setErrorOutputStream(ioManager.createOutputStream(serr, false)); final long jobId = job.getUid(); logger.info("Executing job {}({})", job.getName(), job.getUid()); logger.debug("Executing commandLine {}", job.getCommandLine()); logger.info("=========================================="); System.err.println(); logger.info("Finished job {}({})", job.getName(), job.getUid()); logger.info("=========================================="); return catalogManager.getJobManager().get(job.getUid(), new QueryOptions(), sessionId);
@Override public QueryResult<Job> insert(Job job, long studyId, QueryOptions options) throws CatalogDBException { long startTime = startQuery(); this.dbAdaptorFactory.getCatalogStudyDBAdaptor().checkId(studyId); List<Bson> filterList = new ArrayList<>(); filterList.add(Filters.eq(QueryParams.ID.key(), job.getId())); filterList.add(Filters.eq(PRIVATE_STUDY_ID, studyId)); filterList.add(Filters.eq(QueryParams.STATUS_NAME.key(), Status.READY)); Bson bson = Filters.and(filterList); QueryResult<Long> count = jobCollection.count(bson); if (count.getResult().get(0) > 0) { throw new CatalogDBException("Job { id: '" + job.getId() + "'} already exists."); } long jobId = getNewId(); job.setUid(jobId); job.setStudyUid(studyId); if (StringUtils.isEmpty(job.getUuid())) { job.setUuid(UUIDUtils.generateOpenCGAUUID(UUIDUtils.Entity.JOB)); } Document jobObject = jobConverter.convertToStorageType(job); if (StringUtils.isNotEmpty(job.getCreationDate())) { jobObject.put(PRIVATE_CREATION_DATE, TimeUtils.toDate(job.getCreationDate())); } else { jobObject.put(PRIVATE_CREATION_DATE, TimeUtils.getDate()); } jobObject.put(PERMISSION_RULES_APPLIED, Collections.emptyList()); jobCollection.insert(jobObject, null); //TODO: Check results.get(0).getN() != 0 return endQuery("Create Job", startTime, get(jobId, filterOptions(options, FILTER_ROUTE_JOBS))); }