public JobData(Job job, int offset, int limit) { this.delegate = job.getData().range(offset, limit); this.job = job; }
public JobUI(Job job) { this.jobId = job.getJobId(); this.attempts = FluentIterable.from(job.getAttempts()) .transform(new Function<JobAttempt, JobAttemptUI>() { @Override public JobAttemptUI apply(JobAttempt input) { return toUI(input); } }).toList(); this.data = new JobDataWrapper(job.getData()); }
private void createAndSaveRefresh(final JobDetails details, final RefreshDecision decision) { final boolean isFull = decision.getAccelerationSettings().getMethod() == RefreshMethod.FULL; final long updateId = isFull ? -1L : getUpdateId(job.getJobId(), job.getData()); final MaterializationMetrics metrics = ReflectionUtils.computeMetrics(job); final List<DataPartition> dataPartitions = ReflectionUtils.computeDataPartitions(job.getJobAttempt().getInfo()); final List<String> refreshPath = ReflectionUtils.getRefreshPath(job.getJobId(), job.getData(), accelerationBasePath); final Refresh refresh = ReflectionUtils.createRefresh(reflection.getId(), refreshPath, decision.getSeriesId(), decision.getSeriesOrdinal(), updateId, details, metrics, dataPartitions); logger.trace("Refresh created: {}", refresh); materializationStore.save(refresh); logger.debug("materialization {} was written to {}", ReflectionUtils.getId(materialization), PathUtils.constructFullPath(refreshPath)); }
@Test public void testDownloadJsonRest() throws Exception { final String downloadPath = new DatasetVersionResourcePath(new DatasetPath("DG.dsg1"), dsg1.getVersion()).toString(); InitialDownloadResponse initialDownloadResponse = expectSuccess( getBuilder(getAPIv2().path(downloadPath).path("download").queryParam("downloadFormat", DownloadFormat.JSON)).buildGet(), InitialDownloadResponse.class); // wait for job final Job job = l(JobsService.class).getJob(initialDownloadResponse.getJobId()); job.getData().loadIfNecessary(); // get job data Response response = getBuilder(getAPIv2().path(initialDownloadResponse.getDownloadUrl())).buildGet().invoke(); validateAllRows(readDataJson((InputStream)response.getEntity())); }
@Test public void testDownloadCsvRest() throws Exception { final String downloadPath = new DatasetVersionResourcePath(new DatasetPath("DG.dsg1"), dsg1.getVersion()).toString(); InitialDownloadResponse initialDownloadResponse = expectSuccess( getBuilder(getAPIv2().path(downloadPath).path("download").queryParam("downloadFormat", DownloadFormat.CSV)).buildGet(), InitialDownloadResponse.class); // wait for job final Job job = l(JobsService.class).getJob(initialDownloadResponse.getJobId()); job.getData().loadIfNecessary(); // get job data Response response = getBuilder(getAPIv2().path(initialDownloadResponse.getDownloadUrl())).buildGet().invoke(); validateAllRows(readDataCsv((InputStream)response.getEntity())); }
protected void setSystemOption(String optionName, String optionValue) { final String query = String.format("ALTER SYSTEM SET \"%s\"=%s", optionName, optionValue); final Job job = getJobsService().submitJob(JobRequest.newBuilder() .setSqlQuery(new SqlQuery(query, DEFAULT_USERNAME)) .setQueryType(QueryType.UI_INTERNAL_RUN) .setDatasetPath(DatasetPath.NONE.toNamespaceKey()) .build(), NoOpJobStatusListener.INSTANCE); job.getData().loadIfNecessary(); }
@Test public void testAlterOption() throws Exception { SqlQuery ctas = getQueryFromSQL("alter session set \"planner.enable_multiphase_agg\"=true"); Job ctasJob = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(ctas) .build(), NoOpJobStatusListener.INSTANCE); ctasJob.getData().loadIfNecessary(); }
@Test public void testExplain() throws Exception { SqlQuery ctas = getQueryFromSQL("EXPLAIN PLAN FOR SELECT * FROM sys.version"); Job ctasJob = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(ctas) .build(), NoOpJobStatusListener.INSTANCE); ctasJob.getData().loadIfNecessary(); }
@Test public void testAliasedQuery() throws Exception { SqlQuery ctas = getQueryFromSQL("SHOW SCHEMAS"); Job ctasJob = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(ctas) .build(), NoOpJobStatusListener.INSTANCE); ctasJob.getData().loadIfNecessary(); }
@Test public void testDsg1Unknown() { Job job = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(new SqlQuery("select * from DG.dsg1", SampleDataPopulator.DEFAULT_USER_NAME)) .setQueryType(QueryType.UNKNOWN) .build(), NoOpJobStatusListener.INSTANCE); job.getData().loadIfNecessary(); // unkown jobs are not counted assertEquals((int)jobsCount.get(dsg1), jobsService.getJobsCount(dsg1.toNamespaceKey())); assertEquals((int) jobsCount.get(sample1), jobsService.getJobsCount(sample1.toNamespaceKey())); }
@Test public void testDsg2Internal() { Job job = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(new SqlQuery("select * from DG.dsg2", SampleDataPopulator.DEFAULT_USER_NAME)) .setQueryType(QueryType.UI_INTERNAL_PREVIEW) .build(), NoOpJobStatusListener.INSTANCE); job.getData().loadIfNecessary(); // internal jobs don't get counted assertEquals((int) jobsCount.get(dsg2), jobsService.getJobsCount(dsg2.toNamespaceKey())); assertEquals((int) jobsCount.get(sample2), jobsService.getJobsCount(sample2.toNamespaceKey())); }
@Test public void testVirtualDatasetWithNotNullFields() throws Exception { final LocalJobsService jobsService = (LocalJobsService) l(JobsService.class); expectSuccess(getBuilder(getAPIv2().path("space/space1")).buildPut(Entity.json(new Space(null, "space1", null, null, null, 0, null))), Space.class); final String pathName = "space1.v1"; final DatasetPath numbersJsonPath = new DatasetPath(pathName); DatasetUI numbersJsonVD = createDatasetFromSQLAndSave(numbersJsonPath, "select row_number() over (order by a) as rnk, a from cp.\"json/numbers.json\"", asList("cp")); final SqlQuery query = getQueryFromSQL(String.format("select t1.rnk, t1.a from %s t1 join %s t2 on t1.rnk = t2.rnk+1", pathName, pathName)); final Job ctasJob = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(query) .build(), NoOpJobStatusListener.INSTANCE); ctasJob.getData().loadIfNecessary(); }
@Test public void testReflectionsTable() { jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(getQueryFromSQL("SELECT * FROM sys.reflections")) .setDatasetPath(NONE_PATH) .build(), NoOpJobStatusListener.INSTANCE).getData().loadIfNecessary(); }
@Test public void testMaterializationsTable() { jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(getQueryFromSQL("SELECT * FROM sys.materializations")) .setDatasetPath(NONE_PATH) .build(), NoOpJobStatusListener.INSTANCE).getData().loadIfNecessary(); } }
@Test public void testDownloadCsv() throws Exception { Job job = datasetService.prepareDownload(dsg1DatasetPath, dsg1.getVersion(), DownloadFormat.CSV, -1, SampleDataPopulator.DEFAULT_USER_NAME); job.getData().loadIfNecessary(); DownloadDataResponse downloadDataResponse = datasetService.downloadData(job.getJobAttempt().getInfo().getDownloadInfo(), SampleDataPopulator.DEFAULT_USER_NAME); validateAllRows(readDataCsv(downloadDataResponse.getInput())); }
@Test public void testDownloadJson() throws Exception { Job job = datasetService.prepareDownload(dsg1DatasetPath, dsg1.getVersion(), DownloadFormat.JSON, -1, SampleDataPopulator.DEFAULT_USER_NAME); job.getData().loadIfNecessary(); DownloadDataResponse downloadDataResponse = datasetService.downloadData(job.getJobAttempt().getInfo().getDownloadInfo(), SampleDataPopulator.DEFAULT_USER_NAME); validateAllRows(readDataJson(downloadDataResponse.getInput())); }
@Test public void testQueryOnFile() throws Exception { final JobsService jobService = l(JobsService.class); Job job = jobService.submitJob(JobRequest.newBuilder() .setSqlQuery(new SqlQuery("SELECT * FROM dachdfs_test.dir1.json.\"users.json\"", SampleDataPopulator.DEFAULT_USER_NAME)) .build(), NoOpJobStatusListener.INSTANCE); JobDataFragment jobData = job.getData().truncate(500); assertEquals(3, jobData.getReturnedRowCount()); assertEquals(2, jobData.getSchema().getFieldCount()); } }
private static QueryProfile getQueryProfile(final String query) throws JobNotFoundException { final Job job = getJobsService().submitJob(JobRequest.newBuilder() .setSqlQuery(new SqlQuery(query, DEFAULT_USERNAME)) .setQueryType(QueryType.UI_INTERNAL_RUN) .setDatasetPath(DatasetPath.NONE.toNamespaceKey()) .setDatasetVersion(DatasetVersion.NONE) .build(), new NoOpJobStatusListener()); job.getData().loadIfNecessary(); return getJobsService().getProfile(job.getJobId(), 0); }
@Test public void testDsg2UI() { Job job = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(new SqlQuery("select * from DG.dsg2", SampleDataPopulator.DEFAULT_USER_NAME)) .setQueryType(QueryType.UI_RUN) .build(), NoOpJobStatusListener.INSTANCE); job.getData().loadIfNecessary(); assertEquals(inc(dsg2), jobsService.getJobsCount(dsg2.toNamespaceKey())); assertEquals(inc(sample2), jobsService.getJobsCount(sample2.toNamespaceKey())); }
@Test public void testDsg1External() { Job job = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(new SqlQuery("select * from DG.dsg1", SampleDataPopulator.DEFAULT_USER_NAME)) .setQueryType(QueryType.UI_RUN) .build(), NoOpJobStatusListener.INSTANCE); job.getData().loadIfNecessary(); assertEquals(inc(dsg1), jobsService.getJobsCount(dsg1.toNamespaceKey())); assertEquals(inc(sample1), jobsService.getJobsCount(sample1.toNamespaceKey())); }