final CompletionListener listener = new CompletionListener(); jobsService.get().submitJob(JobRequest.newBuilder() .setSqlQuery(query) .setQueryType(QueryType.UI_INTERNAL_RUN)
@POST @Path("/{id}/cancel") public void cancelJob(@PathParam("id") String id) throws JobException { final String username = securityContext.getUserPrincipal().getName(); try { jobs.cancel(username, new JobId(id), String.format("Query cancelled by user '%s'", username)); } catch (JobNotFoundException e) { throw new NotFoundException(String.format("Could not find a job with id [%s]", id)); } } }
@GET @Produces(APPLICATION_JSON) public JobsUI getJobs( @QueryParam("filter") String filters, @QueryParam("sort") String sortColumn, @QueryParam("order") SortOrder order, @QueryParam("offset") @DefaultValue("0") int offset, @QueryParam("limit") @DefaultValue("100") int limit ) { final List<Job> jobs = ImmutableList.copyOf(jobsService.get().getAllJobs(filters, sortColumn, order, offset, limit, securityContext.getUserPrincipal().getName())); return new JobsUI( namespace, jobs, getNext(offset, limit, filters, sortColumn, order, jobs.size()) ); }
@Test public void testDsg2Internal() { Job job = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(new SqlQuery("select * from DG.dsg2", SampleDataPopulator.DEFAULT_USER_NAME)) .setQueryType(QueryType.UI_INTERNAL_PREVIEW) .build(), NoOpJobStatusListener.INSTANCE); job.getData().loadIfNecessary(); // internal jobs don't get counted assertEquals((int) jobsCount.get(dsg2), jobsService.getJobsCount(dsg2.toNamespaceKey())); assertEquals((int) jobsCount.get(sample2), jobsService.getJobsCount(sample2.toNamespaceKey())); }
jobsService.getJobsForDataset(datasetPath.toNamespaceKey(), null, query.getUsername(), MAX_JOBS_TO_SEARCH) : jobsService.getJobsForDataset(datasetPath.toNamespaceKey(), version, query.getUsername(), MAX_JOBS_TO_SEARCH); for (Job job : jobsForDataset) { if (job.getJobAttempt().getInfo().getQueryType() == queryType try { statusListener.jobCompleted(); return new JobUI(jobsService.getJob(job.getJobId())); } catch (RuntimeException | JobNotFoundException e) { logger.debug("job {} not found for dataset {}", job.getJobId().getId(), messagePath, e); return new JobUI(jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(query) .setQueryType(queryType)
doc("Submitting job for dataset testB.dsB1 dataset for version " + v2.getVersion()); l(JobsService.class).submitJob(JobRequest.newBuilder() .setSqlQuery(getQueryFromConfig(dsGet)) .setQueryType(QueryType.UI_PREVIEW) .build(), NoOpJobStatusListener.INSTANCE).getData().loadIfNecessary(); l(JobsService.class).submitJob(JobRequest.newBuilder() .setSqlQuery(new SqlQuery(dsGet.getSql(), dsGet.getContext(), USERNAME)) .setQueryType(QueryType.UI_PREVIEW) .build(), NoOpJobStatusListener.INSTANCE).getData().loadIfNecessary(); List<Job> jobs2 = ImmutableList.copyOf(jobsService.getJobsForDataset(new DatasetPath("testB.dsB1").toNamespaceKey(), 1000)); assertEquals(3, jobs2.size()); // we have already run that query for the latest version in the previous call jobs2 = ImmutableList.copyOf(jobsService.getJobsForDataset(new DatasetPath("testB.dsB1").toNamespaceKey(), dsGet.getDatasetVersion(), 1000)); assertEquals(dsB1JobsVersion, jobs2.size()); jobs2 = ImmutableList.copyOf(jobsService.getJobsForDataset(new DatasetPath("testB.dsB1").toNamespaceKey(), dsGet.getDatasetVersion(), DEFAULT_USERNAME, 1000)); assertEquals(1, jobs2.size()); jobs2 = ImmutableList.copyOf(jobsService.getJobsForDataset(new DatasetPath("testB.dsB1").toNamespaceKey(), v2, DEFAULT_USERNAME, 1000)); assertEquals(1, jobs2.size()); jobs2 = ImmutableList.copyOf(jobsService.getJobsForDataset(new DatasetPath("testB.dsB1").toNamespaceKey(), v2, USERNAME, 1000)); assertEquals(1, jobs2.size());
private static QueryProfile getQueryProfile(final String query) throws JobNotFoundException { final Job job = getJobsService().submitJob(JobRequest.newBuilder() .setSqlQuery(new SqlQuery(query, DEFAULT_USERNAME)) .setQueryType(QueryType.UI_INTERNAL_RUN) .setDatasetPath(DatasetPath.NONE.toNamespaceKey()) .setDatasetVersion(DatasetVersion.NONE) .build(), new NoOpJobStatusListener()); job.getData().loadIfNecessary(); return getJobsService().getProfile(job.getJobId(), 0); }
@GET @Path("/{id}") public JobStatus getJobStatus(@PathParam("id") String id) { try { Job job = jobs.getJob(new JobId(id), securityContext.getUserPrincipal().getName()); return JobStatus.fromJob(job); } catch (JobNotFoundException e) { throw new NotFoundException(String.format("Could not find a job with id [%s]", id)); } }
final JobsService jobsService = l(JobsService.class); final DatasetPath ds1 = new DatasetPath("s.ds1"); Job job1_0 = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(new SqlQuery("select * from LocalFS1.\"dac-sample1.json\" limit 10", ds1.toParentPathList(), DEFAULT_USERNAME)) .setDatasetPath(ds1.toNamespaceKey()) job1_0.getData().loadIfNecessary(); JobDataFragment result = job1_0.getData().truncate(10); JobDataFragment storedResult = jobsService.getJob(job1_0.getJobId()).getData().truncate(10); for (Field column: result.getSchema()) { assertTrue(storedResult.getSchema().getFields().contains(column));
@Test public void cancelledDownloadJob() throws Exception { final DatasetPath dsPath = new DatasetPath("DG.testDS2"); DatasetUI ds = createDatasetFromSQLAndSave(dsPath,"select * from DG.dsg1 --- comment", asList("cp")); final Job job = datasetService.prepareDownload(dsPath, ds.getDatasetVersion(), DownloadFormat.CSV, 500, SampleDataPopulator.DEFAULT_USER_NAME); l(JobsService.class).cancel(SampleDataPopulator.DEFAULT_USER_NAME, job.getJobId(), "because I can"); job.getData().loadIfNecessary(); if (l(JobsService.class).getJob(job.getJobId()).getJobAttempt().getState() == JobState.CANCELED) { try { datasetService.downloadData(job.getJobAttempt().getInfo().getDownloadInfo(), SampleDataPopulator.DEFAULT_USER_NAME); fail(); } catch (Exception e) { assertTrue(e instanceof FileNotFoundException); } } }
private boolean recordHeader(OutputStream output, JobId id, User user, String submissionId) throws UserNotFoundException, IOException, JobNotFoundException { SupportHeader header = new SupportHeader(); header.setClusterInfo(getClusterInfo()); header.setJob(jobsService.get().getJob(id).getJobAttempt()); Submission submission = new Submission() .setSubmissionId(submissionId) .setDate(System.currentTimeMillis()) .setEmail(user.getEmail()) .setFirst(user.getFirstName()) .setLast(user.getLastName()); header.setSubmission(submission); // record the dremio version that was used to run the query in the header header.setDremioVersion(jobsService.get().getProfile(id, 0).getDremioVersion()); ProtostuffUtil.toJSON(output, header, SupportHeader.getSchema(), false); return true; }
public long getJobsCount(List<NamespaceKey> datasetPaths) { long jobCount = 0; for (Integer count : jobsService.getJobsCount(datasetPaths)) { if (count != null) { jobCount += count; } } return jobCount; }
@Test public void testQueryParser() throws Exception { TestSpacesStoragePlugin.setup(getCurrentDremioDaemon()); getPreview(getDataset(new DatasetPath("testA.dsA1"))); List<Job> jobs = ImmutableList.copyOf(l(JobsService.class).getJobsForDataset(new DatasetPath("testA.dsA1").toNamespaceKey(), 1000)); assertNotNull(jobs); assertTrue(jobs.size() > 0); JobUI job1 = expectSuccess(getBuilder(getAPIv2().path("job/" + jobs.get(0).getJobId().getId())).buildGet(), JobUI.class); assertEquals(jobs.get(0).getJobId(), job1.getJobId()); final Job job = l(JobsService.class).getJob(jobs.get(0).getJobId()); final JobDetails jobDetails = job.getJobAttempt().getDetails(); final JobStats jobStats = job.getJobAttempt().getStats(); assertEquals(1, jobDetails.getTableDatasetProfilesList().size()); assertEquals(1000, (long)jobDetails.getOutputRecords()); // leaf limit is 10k assertEquals(16250, (long) jobDetails.getDataVolume()); assertEquals(16250, (long) jobStats.getOutputBytes()); assertEquals(16250, (long) jobStats.getInputBytes()); assertEquals(1000, (long) jobStats.getInputRecords()); assertEquals(1000, (long) jobStats.getOutputRecords()); } }
private QueryProfile recordProfile(OutputStream out, JobId id, int attempt) throws IOException, JobNotFoundException { QueryProfile profile = jobsService.get().getProfile(id, attempt); ProtostuffUtil.toJSON(out, profile, SchemaUserBitShared.QueryProfile.WRITE, false); return profile; }
Iterable<Job> jobs = jobsService.getJobsForDataset( new NamespaceKey(currentDataset.getFullPathList()), currentDataset.getVersion(), 1); final JobState jobState;
@Test public void testDsg1Unknown() { Job job = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(new SqlQuery("select * from DG.dsg1", SampleDataPopulator.DEFAULT_USER_NAME)) .setQueryType(QueryType.UNKNOWN) .build(), NoOpJobStatusListener.INSTANCE); job.getData().loadIfNecessary(); // unkown jobs are not counted assertEquals((int)jobsCount.get(dsg1), jobsService.getJobsCount(dsg1.toNamespaceKey())); assertEquals((int) jobsCount.get(sample1), jobsService.getJobsCount(sample1.toNamespaceKey())); }
l(JobsService.class).submitJob(JobRequest.newBuilder() .setSqlQuery(getQueryFromConfig(dsGet)) .setQueryType(QueryType.UI_RUN) .build(), NoOpJobStatusListener.INSTANCE).getData().loadIfNecessary(); List<Job> jobs = ImmutableList.copyOf(jobsService.getJobsForDataset(new DatasetPath("testB.dsB1").toNamespaceKey(), dsGet.getDatasetVersion(), 1000)); assertEquals(dsB1JobsVersion, jobs.size()); jobs = ImmutableList.copyOf(jobsService.getJobsForDataset(new DatasetPath("testB.dsB1").toNamespaceKey(), new DatasetVersion(v2.getVersion()), 1000)); assertEquals(v2.getVersion(), 1, jobs.size()); jobs = ImmutableList.copyOf(jobsService.getJobsForDataset(new DatasetPath("testB.dsB1").toNamespaceKey(), dsGet.getDatasetVersion(), 1000)); assertEquals(dsB1JobsVersion, jobs.size()); jobs = ImmutableList.copyOf(jobsService.getJobsForDataset(new DatasetPath("testB.dsB1").toNamespaceKey(), 1000));
@GET @Path("/details") @Produces(APPLICATION_JSON) public JobDetailsUI getJobDetail(@PathParam("jobId") String jobId) throws JobResourceNotFoundException { JobId id = new JobId(jobId); final Job job; try { job = jobsService.getJob(id, securityContext.getUserPrincipal().getName()); } catch (JobNotFoundException e) { throw JobResourceNotFoundException.fromJobNotFoundException(e); } return JobDetailsUI.of(job); }
public int getJobsCount(NamespaceKey path) { return jobsService.getJobsCount(path); }
@GET @Path("/{queryid}.json") @Produces(APPLICATION_JSON) public String getProfileJSON(@PathParam("queryid") String queryId, @QueryParam("attempt") @DefaultValue("0") int attempt) throws IOException { final QueryProfile profile; try { profile = jobsService.getProfile(new JobId(queryId), attempt); } catch (JobNotFoundException ignored) { // TODO: should this be JobResourceNotFoundException? throw new NotFoundException(format("Profile for JobId [%s] and Attempt [%d] not found.", queryId, attempt)); } return new String(SERIALIZER.serialize(profile)); }