public void checkCohortCanBeDeleted(Cohort cohort) throws CatalogException { // Check if the cohort is different from DEFAULT_COHORT if (StudyEntry.DEFAULT_COHORT.equals(cohort.getId())) { throw new CatalogException("Cohort " + StudyEntry.DEFAULT_COHORT + " cannot be deleted."); } // Check if the cohort can be deleted if (cohort.getStatus() != null && cohort.getStatus().getName() != null && !cohort.getStatus().getName().equals(Cohort.CohortStatus.NONE)) { throw new CatalogException("The cohort is used in storage."); } }
static CatalogException unableToCalculateCohortReady(Cohort cohort) { return new CatalogException("Unable to calculate stats for cohort " + "{ uid: " + cohort.getUid() + " id: \"" + cohort.getId() + "\" }" + " with status \"" + cohort.getStatus().getName() + "\". " + "Resume or update stats for continue calculation"); }
static CatalogException unableToCalculateCohortCalculating(Cohort cohort) { return new CatalogException("Unable to calculate stats for cohort " + "{ uid: " + cohort.getUid() + " id: \"" + cohort.getId() + "\" }" + " with status \"" + cohort.getStatus().getName() + "\". " + "Resume for continue calculation."); }
protected List<StoragePipelineResult> loadFiles(List<File> files, List<File> expectedLoadedFiles, QueryOptions queryOptions, String outputId) throws Exception { queryOptions.append(VariantFileIndexerStorageOperation.TRANSFORM, false); queryOptions.append(VariantFileIndexerStorageOperation.LOAD, true); queryOptions.append(StorageOperation.CATALOG_PATH, outputId); boolean calculateStats = queryOptions.getBoolean(VariantStorageEngine.Options.CALCULATE_STATS.key()); String studyId = catalogManager.getFileManager().getStudy(files.get(0), sessionId).getId(); List<String> fileIds = files.stream().map(File::getId).collect(Collectors.toList()); String outdir = opencga.createTmpOutdir(studyId, "_LOAD_", sessionId); List<StoragePipelineResult> etlResults = variantManager.index(studyId, fileIds, outdir, queryOptions, sessionId); assertEquals(expectedLoadedFiles.size(), etlResults.size()); checkEtlResults(studyId, etlResults, FileIndex.IndexStatus.READY); Cohort defaultCohort = getDefaultCohort(studyId); for (File file : expectedLoadedFiles) { List<Long> samplesInFile = file.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()); List<Long> samplesInCohort = defaultCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()); assertTrue(samplesInCohort.containsAll(samplesInFile)); } if (calculateStats) { assertEquals(Cohort.CohortStatus.READY, defaultCohort.getStatus().getName()); checkCalculatedStats(studyId, Collections.singletonMap(DEFAULT_COHORT, defaultCohort), catalogManager, dbName, sessionId); } return etlResults; }
@Test public void testCalculateAggregatedExacStats() throws Exception { beforeAggregated("exachead.vcf.gz", Aggregation.EXAC); String tagMap = getResourceUri("exac-tag-mapping.properties").getPath(); List<String> cohortIds = createCohorts(sessionId, studyId, tagMap, catalogManager, logger) .stream().map(Cohort::getId).map(Object::toString).collect(Collectors.toList()); QueryOptions options = new QueryOptions(VariantStorageEngine.Options.AGGREGATION_MAPPING_PROPERTIES.key(), tagMap); calculateStats(options, cohortIds); List<Cohort> cohorts = catalogManager.getCohortManager().get(studyId, (Query) null, null, sessionId).getResult(); Set<String> cohortNames = cohorts .stream() .map(Cohort::getId) .collect(Collectors.toSet()); assertEquals(8, cohortNames.size()); for (Cohort cohort : cohorts) { assertEquals(Cohort.CohortStatus.READY, cohort.getStatus().getName()); } // checkCalculatedAggregatedStats(cohorts, dbName); }
public static void checkCalculatedStats(String studyId, Map<String, Cohort> cohorts, CatalogManager catalogManager, String dbName, String sessionId) throws Exception { VariantDBAdaptor dbAdaptor = StorageEngineFactory.get().getVariantStorageEngine(null, dbName).getDBAdaptor(); for (Variant variant : dbAdaptor) { for (StudyEntry sourceEntry : variant.getStudies()) { assertEquals("In variant " + variant.toString(), cohorts.size(), sourceEntry.getStats().size()); for (Map.Entry<String, VariantStats> entry : sourceEntry.getStats().entrySet()) { assertTrue("In variant " + variant.toString(), cohorts.containsKey(entry.getKey())); if (cohorts.get(entry.getKey()) != null) { assertEquals("Variant: " + variant.toString() + " does not have the correct number of samples in cohort '" + entry.getKey() + "'. jsonVariant: " + variant.toJson() , cohorts.get(entry.getKey()).getSamples().size(), entry.getValue().getGenotypeCount().values().stream().reduce(Integer::sum).orElse(0).intValue()); } } } } for (Cohort cohort : cohorts.values()) { cohort = catalogManager.getCohortManager().get(studyId, cohort.getId(), null, sessionId).first(); assertEquals(Cohort.CohortStatus.READY, cohort.getStatus().getName()); } }
@Test public void testCalculateAggregatedExacStatsExplicitCohorts() throws Exception { beforeAggregated("exachead.vcf.gz", Aggregation.EXAC); String tagMap = getResourceUri("exac-tag-mapping.properties").getPath(); QueryOptions options = new QueryOptions(VariantStorageEngine.Options.AGGREGATION_MAPPING_PROPERTIES.key(), tagMap); calculateStats(options, Arrays.asList("AFR", "ALL", "AMR", "EAS", "FIN", "NFE", "OTH", "SAS")); List<Cohort> cohorts = catalogManager.getCohortManager().get(studyId, (Query) null, null, sessionId).getResult(); Set<String> cohortNames = cohorts .stream() .map(Cohort::getId) .collect(Collectors.toSet()); assertEquals(8, cohortNames.size()); for (Cohort cohort : cohorts) { assertEquals(Cohort.CohortStatus.READY, cohort.getStatus().getName()); } // checkCalculatedAggregatedStats(cohorts, dbName); }
@Test public void testCalculateInvalidStats() throws Exception { before(); calculateStats(coh[0]); DummyVariantStorageEngine vsm = mockVariantStorageManager(); String message = "Error"; doThrow(new StorageEngineException(message)).when(vsm).calculateStats(any(), any(List.class), any()); doThrow(new StorageEngineException(message)).when(vsm).calculateStats(any(), any(Map.class), any()); try { calculateStats(coh[1]); fail(); } catch (StorageEngineException e) { assertEquals(message, e.getCause().getMessage()); } Cohort coh1 = catalogManager.getCohortManager().get(studyId, coh[1], null, sessionId).first(); assertEquals(Cohort.CohortStatus.INVALID, coh1.getStatus().getName()); vsm = mockVariantStorageManager(); calculateStats(coh[1]); }
@Test public void testCalculateAggregatedExacStatsWithoutCohorts() throws Exception { beforeAggregated("exachead.vcf.gz", Aggregation.EXAC); String tagMap = getResourceUri("exac-tag-mapping.properties").getPath(); QueryOptions options = new QueryOptions(VariantStorageEngine.Options.AGGREGATION_MAPPING_PROPERTIES.key(), tagMap); calculateStats(options); List<Cohort> cohorts = catalogManager.getCohortManager().get(studyId, (Query) null, null, sessionId).getResult(); Set<String> cohortNames = cohorts .stream() .map(Cohort::getId) .collect(Collectors.toSet()); assertEquals(8, cohortNames.size()); for (Cohort cohort : cohorts) { assertEquals(Cohort.CohortStatus.READY, cohort.getStatus().getName()); } // checkCalculatedAggregatedStats(cohorts, dbName); }
private void removeFile(List<File> files, QueryOptions options, String outputId) throws Exception { List<String> fileIds = files.stream().map(File::getId).collect(Collectors.toList()); Study study = catalogManager.getFileManager().getStudy(files.get(0), sessionId); String studyId = study.getFqn(); List<File> removedFiles = variantManager.removeFile(fileIds, studyId, sessionId, new QueryOptions()); assertEquals(files.size(), removedFiles.size()); Cohort all = catalogManager.getCohortManager().get(studyId, new Query(CohortDBAdaptor.QueryParams.ID.key(), StudyEntry.DEFAULT_COHORT), null, sessionId).first(); Set<Long> allSampleIds = all.getSamples().stream().map(Sample::getUid).collect(Collectors.toSet()); assertThat(all.getStatus().getName(), anyOf(is(Cohort.CohortStatus.INVALID), is(Cohort.CohortStatus.NONE))); Set<Long> loadedSamples = catalogManager.getFileManager().get(studyId, new Query(FileDBAdaptor.QueryParams.INDEX_STATUS_NAME.key (), FileIndex.IndexStatus.READY), null, sessionId) .getResult() .stream() .flatMap(f -> f.getSamples().stream()) .map(Sample::getUid) .collect(Collectors.toSet()); assertEquals(loadedSamples, allSampleIds); for (File file : removedFiles) { assertEquals(FileIndex.IndexStatus.TRANSFORMED, file.getIndex().getStatus().getName()); } }
private String updateDefaultCohortStatus(Study study, String status, String sessionId) throws CatalogException { Query query = new Query(CohortDBAdaptor.QueryParams.ID.key(), StudyEntry.DEFAULT_COHORT); Cohort defaultCohort = catalogManager.getCohortManager().get(study.getFqn(), query, new QueryOptions(), sessionId).first(); String prevStatus = defaultCohort.getStatus().getName(); catalogManager.getCohortManager().setStatus(study.getFqn(), defaultCohort.getId(), status, null, sessionId); return prevStatus; }
assertEquals(Cohort.CohortStatus.NONE, catalogManager.getCohortManager().get(studyId, coh[0], null, sessionId).first().getStatus().getName()); assertEquals(Cohort.CohortStatus.READY, catalogManager.getCohortManager().get(studyId, coh[0], null, sessionId).first().getStatus().getName()); assertEquals(Cohort.CohortStatus.READY, catalogManager.getCohortManager().get(studyId, coh[0], null, sessionId).first().getStatus().getName()); .collect(Collectors.toList()); catalogManager.getCohortManager().update(studyId, coh[0], new ObjectMap("samples", newCohort), new QueryOptions(), sessionId); assertEquals(Cohort.CohortStatus.INVALID, catalogManager.getCohortManager().get(studyId, coh[0], null, sessionId).first().getStatus().getName()); assertEquals(Cohort.CohortStatus.READY, catalogManager.getCohortManager().get(studyId, coh[0], null, sessionId).first().getStatus().getName()); cohorts.put("coh0", catalogManager.getCohortManager().get(studyId, coh[0], null, sessionId).first()); checkCalculatedStats(cohorts);
@Test public void testIndexWithAggregatedStats() throws Exception { QueryOptions queryOptions = new QueryOptions(VariantStorageEngine.Options.ANNOTATE.key(), false) .append(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), Aggregation.BASIC); queryOptions.put(VariantStorageEngine.Options.CALCULATE_STATS.key(), true); queryOptions.putIfNotNull(StorageOperation.CATALOG_PATH, outputId); variantManager.index(studyFqn, files.get(0).getId(), opencga.createTmpOutdir(studyId, "index", sessionId), queryOptions, sessionId); assertEquals(0, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.READY, getDefaultCohort(studyId).getStatus().getName()); StatsVariantStorageTest.checkCalculatedAggregatedStats(Collections.singleton(DEFAULT_COHORT), dbName); }
@Test public void testCalculateStatsGroups() throws Exception { before(); Map<String, Cohort> cohorts = new HashMap<>(); calculateStats(new QueryOptions(), coh[0], coh[1], coh[2]); cohorts.put(coh[0], catalogManager.getCohortManager().get(studyId, coh[0], null, sessionId).first()); cohorts.put(coh[1], catalogManager.getCohortManager().get(studyId, coh[1], null, sessionId).first()); cohorts.put(coh[2], catalogManager.getCohortManager().get(studyId, coh[2], null, sessionId).first()); checkCalculatedStats(cohorts); try { calculateStats(new QueryOptions(), all, coh[3], "-" + coh[4]); fail(); } catch (CatalogException e) { logger.info("received expected exception. this is OK, there is no cohort " + ("-" + coh[4]) + '\n'); } assertEquals(Cohort.CohortStatus.NONE, catalogManager.getCohortManager().get(studyId, "ALL", null, sessionId).first().getStatus().getName()); assertEquals(Cohort.CohortStatus.NONE, catalogManager.getCohortManager().get(studyId, coh[3], null, sessionId).first().getStatus().getName()); assertEquals(Cohort.CohortStatus.NONE, catalogManager.getCohortManager().get(studyId, coh[4], null, sessionId).first().getStatus().getName()); calculateStats(new QueryOptions(), all, coh[3], coh[4]); cohorts.put(DEFAULT_COHORT, catalogManager.getCohortManager().get(studyId, DEFAULT_COHORT, null, sessionId).first()); cohorts.put(coh[3], catalogManager.getCohortManager().get(studyId, coh[3], null, sessionId).first()); cohorts.put(coh[4], catalogManager.getCohortManager().get(studyId, coh[4], null, sessionId).first()); checkCalculatedStats(cohorts); }
@Test public void testDeleteCohort() throws CatalogException, IOException { String studyId = "user@1000G:phase1"; Sample sampleId1 = catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_1"), new QueryOptions(), sessionIdUser).first(); Sample sampleId2 = catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_2"), new QueryOptions(), sessionIdUser).first(); Sample sampleId3 = catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_3"), new QueryOptions(), sessionIdUser).first(); Cohort myCohort = catalogManager.getCohortManager().create(studyId, new Cohort().setId("MyCohort").setType(Study.Type.FAMILY) .setSamples(Arrays.asList(sampleId1, sampleId2, sampleId3)), null, sessionIdUser).first(); assertEquals("MyCohort", myCohort.getId()); assertEquals(3, myCohort.getSamples().size()); assertTrue(myCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId1.getUid())); assertTrue(myCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId2.getUid())); assertTrue(myCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId3.getUid())); WriteResult deleteResult = catalogManager.getCohortManager().delete(studyId, new Query(CohortDBAdaptor.QueryParams.UID.key(), myCohort.getUid()), null, sessionIdUser); assertEquals(1, deleteResult.getNumModified()); Query query = new Query() .append(CohortDBAdaptor.QueryParams.UID.key(), myCohort.getUid()) .append(CohortDBAdaptor.QueryParams.STATUS_NAME.key(), "!=" + Cohort.CohortStatus.READY); Cohort cohort = catalogManager.getCohortManager().get(studyId, query, null, sessionIdUser).first(); assertEquals(Status.DELETED, cohort.getStatus().getName()); }
cohort.setRelease(studyManager.getCurrentRelease(study, userId)); cohort.setStats(ParamUtils.defaultObject(cohort.getStats(), Collections::emptyMap)); cohort.setStatus(ParamUtils.defaultObject(cohort.getStatus(), Cohort.CohortStatus::new)); cohort.setSamples(ParamUtils.defaultObject(cohort.getSamples(), Collections::emptyList));
@Test public void testIndexWithStatsLowerCaseAggregationType() throws Exception { QueryOptions queryOptions = new QueryOptions(VariantStorageEngine.Options.ANNOTATE.key(), false); queryOptions.put(VariantStorageEngine.Options.CALCULATE_STATS.key(), true); queryOptions.put(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), "none"); queryOptions.putIfNotNull(StorageOperation.CATALOG_PATH, outputId); variantManager.index(studyId, getFile(0).getId(), newTmpOutdir(), queryOptions, sessionId); assertEquals(500, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.READY, getDefaultCohort(studyId).getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); }
@Override public QueryResult<Cohort> delete(long id, QueryOptions queryOptions) throws CatalogDBException { long startTime = startQuery(); checkId(id); // Check if the cohort is active Query query = new Query(QueryParams.UID.key(), id) .append(QueryParams.STATUS_NAME.key(), "!=" + Status.DELETED); if (count(query).first() == 0) { query.put(QueryParams.STATUS_NAME.key(), Status.DELETED); QueryOptions options = new QueryOptions(MongoDBCollection.INCLUDE, QueryParams.STATUS_NAME.key()); Cohort cohort = get(query, options).first(); throw new CatalogDBException("The cohort {" + id + "} was already " + cohort.getStatus().getName()); } // Change the status of the cohort to deleted setStatus(id, Status.DELETED); query = new Query(QueryParams.UID.key(), id).append(QueryParams.STATUS_NAME.key(), Status.DELETED); return endQuery("Delete cohort", startTime, get(query, null)); }
@Test public void testIndexWithStatsWrongAggregationType() throws Exception { QueryOptions queryOptions = new QueryOptions(VariantStorageEngine.Options.ANNOTATE.key(), false); queryOptions.put(VariantStorageEngine.Options.CALCULATE_STATS.key(), true); queryOptions.put(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), "wrong_type"); queryOptions.putIfNotNull(StorageOperation.CATALOG_PATH, outputId); try { variantManager.index(studyId, getFile(0).getId(), newTmpOutdir(), queryOptions, sessionId); fail("Expected StoragePipelineException exception"); } catch (StoragePipelineException e) { assertEquals(0, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.NONE, getDefaultCohort(studyId).getStatus().getName()); assertEquals(FileIndex.IndexStatus.TRANSFORMED, catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getIndex().getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); } queryOptions.put(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), "none"); // File already transformed queryOptions.put(VariantFileIndexerStorageOperation.LOAD, true); variantManager.index(studyId, getFile(0).getId(), newTmpOutdir(), queryOptions, sessionId); assertEquals(500, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.READY, getDefaultCohort(studyId).getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); }