protected String getCatalogOutdirId(String studyStr, ObjectMap options, String sessionId) throws CatalogException { String catalogOutDirId; if (isCatalogPathDefined(options)) { String catalogOutDirIdStr = options.getString(CATALOG_PATH); catalogOutDirId = catalogManager.getFileManager() .getUid(catalogOutDirIdStr, studyStr, sessionId).getResource().getId(); } else { catalogOutDirId = null; } return catalogOutDirId; }
try { job.getParams().put("path", dbAdaptorFactory.getCatalogFileDBAdaptor().get(job.getOutDir().getUid(), QueryOptions.empty()).first().getId()); } catch (CatalogDBException e) { e.printStackTrace();
@Test public void testAnnotateCreateAndLoad() throws Exception { DummyVariantDBAdaptor dbAdaptor = mockVariantDBAdaptor(); List<File> files = annotate(new Query(), new QueryOptions(VariantAnnotationManager.CREATE, true)); verify(dbAdaptor, atLeastOnce()).iterator(any(Query.class), any()); verify(dbAdaptor, never()).updateAnnotations(any(), anyLong(), any()); verify(dbAdaptor, never()).updateCustomAnnotations(any(), any(), any(), anyLong(), any()); assertEquals(1, files.size()); checkAnnotation(v -> false); QueryOptions config = new QueryOptions(VariantAnnotationManager.LOAD_FILE, files.get(0).getId()); dbAdaptor = mockVariantDBAdaptor(); annotate(new Query(), config); verify(dbAdaptor, atLeastOnce()).updateAnnotations(any(), anyLong(), any()); verify(dbAdaptor, never()).updateCustomAnnotations(any(), any(), any(), anyLong(), any()); verify(dbAdaptor, never()).iterator(any(Query.class), any()); checkAnnotation(v -> true); }
public static File create(String resourceName, boolean indexed) throws IOException, CatalogException { File file; URI uri = getResourceUri(resourceName); file = fileMetadataReader.create(studyId, uri, "data/vcfs/", "", true, null, sessionId).first(); catalogFileUtils.upload(uri, file, null, sessionId, false, false, true, false, Long.MAX_VALUE); if (indexed) { FileIndex fileIndex = new FileIndex("user", "today", new FileIndex.IndexStatus(FileIndex.IndexStatus.READY), 1234, Collections.emptyMap()); catalogManager.getFileManager().setFileIndex(studyId, file.getPath(), fileIndex, sessionId); indexedFiles.add((int) file.getUid()); List<String> samples = catalogManager.getCohortManager().getSamples(studyId, cohortId, null, sessionId).getResult().stream().map(Sample::getId).collect(Collectors.toList()); samples.addAll(file.getSamples().stream().map(Sample::getId).collect(Collectors.toList())); catalogManager.getCohortManager().update(studyId, cohortId, new ObjectMap(CohortDBAdaptor.QueryParams.SAMPLES.key(), samples), true, null, sessionId); } return catalogManager.getFileManager().get(studyId, file.getId(), null, sessionId).first(); }
private File getOriginalFromTransformed(String study, File file, String sessionId) throws URISyntaxException, CatalogException { // Look for the vcf file String vcfId = null; // Matchup variant files, if missing if (file.getRelatedFiles() == null || file.getRelatedFiles().isEmpty()) { catalogManager.getFileManager().matchUpVariantFiles(null, Collections.singletonList(file), sessionId); } for (File.RelatedFile relatedFile : file.getRelatedFiles()) { if (File.RelatedFile.Relation.PRODUCED_FROM.equals(relatedFile.getRelation())) { long fileUid = relatedFile.getFileId(); // FIXME!!! vcfId = catalogManager.getFileManager().get(study, new Query(UID.key(), fileUid), new QueryOptions(QueryOptions.INCLUDE, ID.key()), sessionId).first().getId(); break; } } if (vcfId == null) { logger.error("This code should never be executed. Every transformed avro file should come from a registered vcf file"); throw new CatalogException("Internal error. No vcf file could be found for file " + file.getPath()); } QueryResult<File> vcfQueryResult = fileManager.get(study, vcfId, FILE_GET_QUERY_OPTIONS, sessionId); if (vcfQueryResult.getNumResults() != 1) { logger.error("This code should never be executed. No vcf file could be found for vcf id " + vcfId); throw new CatalogException("Internal error. No vcf file could be found under id " + vcfId); } file = vcfQueryResult.first(); return file; }
@Test public void testCustomAnnotation() throws Exception { annotate(new Query(), new QueryOptions()); checkAnnotation(v -> true); DummyVariantDBAdaptor dbAdaptor = mockVariantDBAdaptor(); File file = opencga.createFile(studyId, "custom_annotation/myannot.gff", sessionId); QueryOptions options = new QueryOptions() .append(VariantAnnotationManager.LOAD_FILE, file.getId()) .append(VariantAnnotationManager.CUSTOM_ANNOTATION_KEY, "myAnnot"); options.put(StorageOperation.CATALOG_PATH, outputId); variantManager.annotate(String.valueOf(studyId), new Query(), opencga.createTmpOutdir(studyId, "annot", sessionId), options, sessionId); verify(dbAdaptor, atLeastOnce()).updateCustomAnnotations(any(), matches("myAnnot"), any(), anyLong(), any()); file = opencga.createFile(studyId, "custom_annotation/myannot.bed", sessionId); options = new QueryOptions() .append(VariantAnnotationManager.LOAD_FILE, file.getId()) .append(VariantAnnotationManager.CUSTOM_ANNOTATION_KEY, "myAnnot2"); options.put(StorageOperation.CATALOG_PATH, outputId); variantManager.annotate(String.valueOf(studyId), new Query(), opencga.createTmpOutdir(studyId, "annot", sessionId), options, sessionId); verify(dbAdaptor, atLeastOnce()).updateCustomAnnotations(any(), matches("myAnnot2"), any(), anyLong(), any()); }
protected File create(String studyId, URI uri, String path) throws IOException, CatalogException { File file; file = fileMetadataReader.create(studyId, uri, path, "", true, null, sessionId).first(); // File.Format format = FormatDetector.detect(uri); // File.Bioformat bioformat = BioformatDetector.detect(uri); // file = catalogManager.createFile(studyId, format, bioformat, "data/vcfs/", "", true, -1, sessionId).first(); catalogFileUtils.upload(uri, file, null, sessionId, false, false, true, false, Long.MAX_VALUE); return catalogManager.getFileManager().get(studyId, file.getId(), null, sessionId).first(); }
public File beforeAggregated(String fileName, Aggregation aggregation) throws Exception { Map<String, Object> attributes; if (aggregation != null) { attributes = Collections.singletonMap(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), aggregation); } else { attributes = Collections.emptyMap(); } catalogManager.getStudyManager().update(studyId, new ObjectMap(StudyDBAdaptor.QueryParams.ATTRIBUTES.key(), attributes), null, sessionId); File file1 = opencga.createFile(studyId, fileName, sessionId); // coh0 = catalogManager.createCohort(studyId, "coh0", Cohort.Type.CONTROL_SET, "", file1.getSampleIds(), null, sessionId).first().getId(); QueryOptions queryOptions = new QueryOptions(VariantStorageEngine.Options.ANNOTATE.key(), false); queryOptions.putIfNotNull(StorageOperation.CATALOG_PATH, outputId); variantManager.index(studyId, file1.getId(), createTmpOutdir(file1), queryOptions, sessionId); return file1; }
inputFile = catalogManager.getFileManager().get(studyId, inputFile.getId(), null, sessionId).first(); assertNotNull(inputFile.getIndex().getTransformedFile()); String transformedFileId = catalogManager.getFileManager().get(studyId, new Query(FileDBAdaptor.QueryParams.UID.key(), inputFile.getIndex().getTransformedFile().getId()), new QueryOptions(), sessionId).first().getId();
@Test public void testIndexWithAggregatedStats() throws Exception { QueryOptions queryOptions = new QueryOptions(VariantStorageEngine.Options.ANNOTATE.key(), false) .append(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), Aggregation.BASIC); queryOptions.put(VariantStorageEngine.Options.CALCULATE_STATS.key(), true); queryOptions.putIfNotNull(StorageOperation.CATALOG_PATH, outputId); variantManager.index(studyFqn, files.get(0).getId(), opencga.createTmpOutdir(studyId, "index", sessionId), queryOptions, sessionId); assertEquals(0, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.READY, getDefaultCohort(studyId).getStatus().getName()); StatsVariantStorageTest.checkCalculatedAggregatedStats(Collections.singleton(DEFAULT_COHORT), dbName); }
protected void checkEtlResults(String studyId, List<StoragePipelineResult> etlResults, String expectedStatus) throws CatalogException { for (StoragePipelineResult etlResult : etlResults) { File input = catalogManager.getFileManager().get(studyId, new Query(FileDBAdaptor.QueryParams.URI.key(), etlResult.getInput()), null, sessionId).first(); String indexedFileId; if (input.getRelatedFiles().isEmpty()) { indexedFileId = input.getId(); } else { long indexedFileUid = input.getRelatedFiles().get(0).getFileId(); indexedFileId = catalogManager.getFileManager().get(studyId, new Query(FileDBAdaptor.QueryParams.UID.key(), indexedFileUid), new QueryOptions(), sessionId).first().getId(); } assertEquals(expectedStatus, catalogManager.getFileManager().get(studyId, indexedFileId, null, sessionId).first().getIndex().getStatus().getName()); System.out.println("etlResult = " + etlResult); } }
@Test public void testIndexWithStatsLowerCaseAggregationType() throws Exception { QueryOptions queryOptions = new QueryOptions(VariantStorageEngine.Options.ANNOTATE.key(), false); queryOptions.put(VariantStorageEngine.Options.CALCULATE_STATS.key(), true); queryOptions.put(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), "none"); queryOptions.putIfNotNull(StorageOperation.CATALOG_PATH, outputId); variantManager.index(studyId, getFile(0).getId(), newTmpOutdir(), queryOptions, sessionId); assertEquals(500, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.READY, getDefaultCohort(studyId).getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); }
public void before () throws Exception { File file = opencga.createFile(studyId, "1000g_batches/1-500.filtered.10k.chr22.phase3_shapeit2_mvncall_integrated_v5.20130502.genotypes.vcf.gz", sessionId); for (int i = 0; i < coh.length; i++) { Cohort cohort = catalogManager.getCohortManager().create(studyId, "coh" + i, Study.Type.CONTROL_SET, "", file.getSamples().subList(file.getSamples() .size() / coh.length * i, file.getSamples().size() / coh.length * (i + 1)), null, null, sessionId).first(); coh[i] = cohort.getId(); } QueryOptions queryOptions = new QueryOptions(VariantStorageEngine.Options.ANNOTATE.key(), false); queryOptions.putIfNotNull(StorageOperation.CATALOG_PATH, outputId); variantManager.index(studyId, file.getId(), createTmpOutdir(file), queryOptions, sessionId); all = catalogManager.getCohortManager().get(studyId, new Query(CohortDBAdaptor.QueryParams.ID.key(), DEFAULT_COHORT), new QueryOptions(), sessionId).first().getId(); }
public File createFile(String studyId, String resourceName, String sessionId) throws IOException, CatalogException { File file; URI uri = getResourceUri(resourceName); CatalogManager catalogManager = getCatalogManager(); file = new FileMetadataReader(catalogManager).create(studyId, uri, "data/vcfs/", "", true, null, sessionId).first(); new FileUtils(catalogManager).upload(uri, file, null, sessionId, false, false, true, false, Long.MAX_VALUE); return catalogManager.getFileManager().get(studyId, file.getId(), null, sessionId).first(); }
@Test public void testIndexWithStatsWrongAggregationType() throws Exception { QueryOptions queryOptions = new QueryOptions(VariantStorageEngine.Options.ANNOTATE.key(), false); queryOptions.put(VariantStorageEngine.Options.CALCULATE_STATS.key(), true); queryOptions.put(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), "wrong_type"); queryOptions.putIfNotNull(StorageOperation.CATALOG_PATH, outputId); try { variantManager.index(studyId, getFile(0).getId(), newTmpOutdir(), queryOptions, sessionId); fail("Expected StoragePipelineException exception"); } catch (StoragePipelineException e) { assertEquals(0, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.NONE, getDefaultCohort(studyId).getStatus().getName()); assertEquals(FileIndex.IndexStatus.TRANSFORMED, catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getIndex().getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); } queryOptions.put(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), "none"); // File already transformed queryOptions.put(VariantFileIndexerStorageOperation.LOAD, true); variantManager.index(studyId, getFile(0).getId(), newTmpOutdir(), queryOptions, sessionId); assertEquals(500, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.READY, getDefaultCohort(studyId).getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); }
variantManager.index(studyId, getFile(0).getId(), newTmpOutdir(), queryOptions, sessionId); assertEquals(500, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.NONE, getDefaultCohort(studyId).getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); variantManager.index(studyId, getFile(1).getId(), newTmpOutdir(), queryOptions, sessionId); assertEquals(1000, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.NONE, getDefaultCohort(studyId).getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(1).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); variantManager.index(studyId, getFile(2).getId(), newTmpOutdir(), queryOptions, sessionId); assertEquals(1500, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.READY, getDefaultCohort(studyId).getStatus().getName()); new Query(CohortDBAdaptor.QueryParams.ID.key(), DEFAULT_COHORT), new QueryOptions(), sessionId).first()), catalogManager, dbName, sessionId); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(2).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); variantManager.index(studyId, getFile(3).getId(), newTmpOutdir(), queryOptions, sessionId); assertEquals(2000, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.INVALID, getDefaultCohort(studyId).getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(3).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); variantManager.index(studyId, getFile(4).getId(), newTmpOutdir(), queryOptions, sessionId); assertEquals(2504, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.READY, getDefaultCohort(studyId).getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(4).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); checkCalculatedStats(studyId, Collections.singletonMap(DEFAULT_COHORT, catalogManager.getCohortManager().get(studyId, new Query(CohortDBAdaptor.QueryParams.ID.key(), DEFAULT_COHORT), new QueryOptions(), sessionId).first()), catalogManager,
@Test public void testByStepsMultiRelease() throws Exception { List<File> inputFiles = new ArrayList<>(); File transformFile; for (int i = 77; i <= 79; i++) { inputFiles.add(create("platinum/1K.end.platinum-genomes-vcf-NA128" + i + "_S1.genome.vcf.gz")); } for (File inputFile : inputFiles) { transformFile = transformFile(inputFile, new QueryOptions()); loadFile(transformFile, new QueryOptions(), outputId); opencga.getCatalogManager().getProjectManager().incrementRelease(projectAlias, sessionId); } int i = 1; for (File inputFile : inputFiles) { inputFile = opencga.getCatalogManager().getFileManager().get(studyId, inputFile.getId(), null, sessionId).first(); assertEquals(1, inputFile.getRelease()); assertEquals(i, inputFile.getIndex().getRelease()); i++; } variantManager.iterator(new Query(VariantQueryParam.STUDY.key(), studyId), new QueryOptions(), sessionId).forEachRemaining(variant -> { System.out.println("variant = " + variant); }); }
variantManager.index(studyId, inputFile.getId(), outdir, queryOptions, sessionId); inputFile = catalogManager.getFileManager().get(studyId, inputFile.getId(), null, sessionId).first(); assertEquals(FileIndex.IndexStatus.TRANSFORMED, inputFile.getIndex().getStatus().getName()); assertNotNull(inputFile.getStats().get(FileMetadataReader.VARIANT_FILE_STATS));
studyFqn = study.getFqn(); outputId = catalogManager.getFileManager().createFolder(studyFqn, Paths.get("data", "index").toString(), null, true, null, QueryOptions.empty(), sessionId).first().getId(); outputPath = "data/index/"; studyId2 = catalogManager.getStudyManager().create(projectId, "s2", "s2", "s2", Study.Type.CASE_CONTROL, null, "Study " + "2", null, null, null, null, Collections.singletonMap(File.Bioformat.VARIANT, new DataStore(getStorageEngine(), dbName)), null, Collections.singletonMap(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), getAggregation()), null, sessionId).first().getId(); outputId2 = catalogManager.getFileManager().createFolder(studyId2, Paths.get("data", "index").toString(), null, true, null, QueryOptions.empty(), sessionId).first().getId();
@Test public void parseQuery() throws Exception { assertEquals("user@p1:s1", parseValue(STUDY, "s1")); assertEquals("user@p1:s1,user@p1:s2", parseValue(STUDY, "s1,s2")); assertEquals("!user@p1:s1,user@p1:s2", parseValue(STUDY, "!s1,s2")); assertEquals("user@p1:s2;!user@p1:s1;user@p1:s3", parseValue(STUDY, "user@p1:s2;!s1;p1:s3")); assertEquals(file1.getName(), parseValue("s1", FILE, file1.getName())); assertEquals(file1.getName(), parseValue("s1", FILE, file1.getId())); assertEquals(file1.getName(), parseValue("s1", FILE, file1.getPath())); assertEquals(file1.getName() + "," + file2.getName(), parseValue("s1", FILE, file1.getPath() + "," + file2.getPath())); assertEquals(file1.getName() + ";" + file2.getName(), parseValue("s1", FILE, file1.getPath() + ";" + file2.getPath())); // assertEquals("file1.vcf", parseValue("s1", FILE, String.valueOf(file1.getUid()))); assertEquals("sample1:HOM_ALT;sample2:HET_REF", parseValue("s1", GENOTYPE, "sample1:HOM_ALT;sample2:HET_REF")); assertEquals("sample1:HOM_ALT,sample2:HET_REF", parseValue("s1", GENOTYPE, "sample1:HOM_ALT,sample2:HET_REF")); }