throw new IllegalStateException("Unknown status " + cohort.getStatus().getName()); cohortMap.put(cohort.getId(), cohort.getSamples().stream().map(Sample::getId).collect(Collectors.toList()));
throw new CatalogException("No cohort " + cohortStr + " found in study " + studyStr); if (cohortQueryResult.first().getSamples().size() == 0) { return new QueryResult<>("Samples from cohort " + cohortStr); cohortQueryResult.getDbTime(), cohortQueryResult.first().getSamples().size(), cohortQueryResult.first().getSamples().size(), cohortQueryResult.getWarningMsg(), cohortQueryResult.getErrorMsg(), cohortQueryResult.first().getSamples());
@Override public Document convertToStorageType(Cohort object, List<VariableSet> variableSetList) { Document document = super.convertToStorageType(object, variableSetList); document.remove(CohortDBAdaptor.QueryParams.ANNOTATION_SETS.key()); document.put("uid", object.getUid()); document.put("studyUid", object.getStudyUid()); document.put("samples", convertSamplesToDocument(object.getSamples())); return document; }
public static void checkCalculatedStats(String studyId, Map<String, Cohort> cohorts, CatalogManager catalogManager, String dbName, String sessionId) throws Exception { VariantDBAdaptor dbAdaptor = StorageEngineFactory.get().getVariantStorageEngine(null, dbName).getDBAdaptor(); for (Variant variant : dbAdaptor) { for (StudyEntry sourceEntry : variant.getStudies()) { assertEquals("In variant " + variant.toString(), cohorts.size(), sourceEntry.getStats().size()); for (Map.Entry<String, VariantStats> entry : sourceEntry.getStats().entrySet()) { assertTrue("In variant " + variant.toString(), cohorts.containsKey(entry.getKey())); if (cohorts.get(entry.getKey()) != null) { assertEquals("Variant: " + variant.toString() + " does not have the correct number of samples in cohort '" + entry.getKey() + "'. jsonVariant: " + variant.toJson() , cohorts.get(entry.getKey()).getSamples().size(), entry.getValue().getGenotypeCount().values().stream().reduce(Integer::sum).orElse(0).intValue()); } } } } for (Cohort cohort : cohorts.values()) { cohort = catalogManager.getCohortManager().get(studyId, cohort.getId(), null, sessionId).first(); assertEquals(Cohort.CohortStatus.READY, cohort.getStatus().getName()); } }
cohortSolrModel.setStatus(cohort.getStatus().getName()); if (cohort.getSamples() != null) { cohortSolrModel.setNumSamples(cohort.getSamples().size()); } else { cohortSolrModel.setNumSamples(0);
protected List<StoragePipelineResult> loadFiles(List<File> files, List<File> expectedLoadedFiles, QueryOptions queryOptions, String outputId) throws Exception { queryOptions.append(VariantFileIndexerStorageOperation.TRANSFORM, false); queryOptions.append(VariantFileIndexerStorageOperation.LOAD, true); queryOptions.append(StorageOperation.CATALOG_PATH, outputId); boolean calculateStats = queryOptions.getBoolean(VariantStorageEngine.Options.CALCULATE_STATS.key()); String studyId = catalogManager.getFileManager().getStudy(files.get(0), sessionId).getId(); List<String> fileIds = files.stream().map(File::getId).collect(Collectors.toList()); String outdir = opencga.createTmpOutdir(studyId, "_LOAD_", sessionId); List<StoragePipelineResult> etlResults = variantManager.index(studyId, fileIds, outdir, queryOptions, sessionId); assertEquals(expectedLoadedFiles.size(), etlResults.size()); checkEtlResults(studyId, etlResults, FileIndex.IndexStatus.READY); Cohort defaultCohort = getDefaultCohort(studyId); for (File file : expectedLoadedFiles) { List<Long> samplesInFile = file.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()); List<Long> samplesInCohort = defaultCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()); assertTrue(samplesInCohort.containsAll(samplesInFile)); } if (calculateStats) { assertEquals(Cohort.CohortStatus.READY, defaultCohort.getStatus().getName()); checkCalculatedStats(studyId, Collections.singletonMap(DEFAULT_COHORT, defaultCohort), catalogManager, dbName, sessionId); } return etlResults; }
@Test public void testUpdateCohort() throws CatalogException { Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), new QueryOptions(), sessionIdUser).first(); Sample sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), new QueryOptions(), sessionIdUser).first(); Sample sampleId3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_3"), new QueryOptions(), sessionIdUser).first(); Sample sampleId4 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_4"), new QueryOptions(), sessionIdUser).first(); Sample sampleId5 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_5"), new QueryOptions(), sessionIdUser).first(); Cohort myCohort = catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort").setType(Study.Type.FAMILY) .setSamples(Arrays.asList(sampleId1, sampleId2, sampleId3)), null, sessionIdUser).first(); assertEquals("MyCohort", myCohort.getId()); assertEquals(3, myCohort.getSamples().size()); assertTrue(myCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId1.getUid())); assertTrue(myCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId2.getUid())); assertTrue(myCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId3.getUid())); Cohort myModifiedCohort = catalogManager.getCohortManager().update(studyFqn, myCohort.getId(), new ObjectMap("samples", Arrays.asList(sampleId1.getId(), sampleId3.getId(), sampleId4.getId(), sampleId5.getId())) .append(CohortDBAdaptor.QueryParams.ID.key(), "myModifiedCohort"), new QueryOptions(), sessionIdUser).first(); assertEquals("myModifiedCohort", myModifiedCohort.getId()); assertEquals(4, myModifiedCohort.getSamples().size()); assertTrue(myModifiedCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId1.getUid())); assertTrue(myModifiedCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId3.getUid())); assertTrue(myModifiedCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId4.getUid())); assertTrue(myModifiedCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId5.getUid())); }
/** * Sample methods * *************************** */ /* * Cohort methods * */ @Test public void testCreateCohort() throws CatalogException { Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), new QueryOptions(), sessionIdUser).first(); Sample sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), new QueryOptions(), sessionIdUser).first(); Sample sampleId3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_3"), new QueryOptions(), sessionIdUser).first(); Cohort myCohort = catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort") .setSamples(Arrays.asList(sampleId1, sampleId2, sampleId3)), null, sessionIdUser).first(); assertEquals("MyCohort", myCohort.getId()); assertEquals(3, myCohort.getSamples().size()); assertTrue(myCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId1.getUid())); assertTrue(myCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId2.getUid())); assertTrue(myCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId3.getUid())); }
private void removeFile(List<File> files, QueryOptions options, String outputId) throws Exception { List<String> fileIds = files.stream().map(File::getId).collect(Collectors.toList()); Study study = catalogManager.getFileManager().getStudy(files.get(0), sessionId); String studyId = study.getFqn(); List<File> removedFiles = variantManager.removeFile(fileIds, studyId, sessionId, new QueryOptions()); assertEquals(files.size(), removedFiles.size()); Cohort all = catalogManager.getCohortManager().get(studyId, new Query(CohortDBAdaptor.QueryParams.ID.key(), StudyEntry.DEFAULT_COHORT), null, sessionId).first(); Set<Long> allSampleIds = all.getSamples().stream().map(Sample::getUid).collect(Collectors.toSet()); assertThat(all.getStatus().getName(), anyOf(is(Cohort.CohortStatus.INVALID), is(Cohort.CohortStatus.NONE))); Set<Long> loadedSamples = catalogManager.getFileManager().get(studyId, new Query(FileDBAdaptor.QueryParams.INDEX_STATUS_NAME.key (), FileIndex.IndexStatus.READY), null, sessionId) .getResult() .stream() .flatMap(f -> f.getSamples().stream()) .map(Sample::getUid) .collect(Collectors.toSet()); assertEquals(loadedSamples, allSampleIds); for (File file : removedFiles) { assertEquals(FileIndex.IndexStatus.TRANSFORMED, file.getIndex().getStatus().getName()); } }
ParamUtils.checkObj(cohort.getSamples(), "Sample list"); cohort.setType(ParamUtils.defaultObject(cohort.getType(), Study.Type.COLLECTION)); cohort.setCreationDate(TimeUtils.getTime()); cohort.setStats(ParamUtils.defaultObject(cohort.getStats(), Collections::emptyMap)); cohort.setStatus(ParamUtils.defaultObject(cohort.getStatus(), Cohort.CohortStatus::new)); cohort.setSamples(ParamUtils.defaultObject(cohort.getSamples(), Collections::emptyList)); if (!cohort.getSamples().isEmpty()) { Query query = new Query() .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(SampleDBAdaptor.QueryParams.UID.key(), cohort.getSamples().stream() .map(Sample::getUid) .collect(Collectors.toList())); QueryResult<Long> count = sampleDBAdaptor.count(query); if (count.first() != cohort.getSamples().size()) { throw new CatalogException("Error: Some samples do not exist in the study " + study.getFqn());
.get(studyConfiguration.getStudyName(), cohortName, null, sessionId).first(); List<String> cohortFromCatalog = defaultCohort .getSamples() .stream() .map(Sample::getId) Cohort cohort = iterator.next(); if (cohort.getStatus() != null && cohort.getStatus().getName().equals(Cohort.CohortStatus.INVALID)) { if (cohort.getSamples().size() != studyConfiguration.getCohorts().get(cohortId).size()) { Set<String> cohortFromCatalog = cohort.getSamples() .stream() .map(Sample::getId)
private void removeStudy(Object study, QueryOptions options) throws Exception { variantManager.removeStudy(study.toString(), sessionId, new QueryOptions()); Query query = new Query(FileDBAdaptor.QueryParams.INDEX_STATUS_NAME.key(), FileIndex.IndexStatus.READY); assertEquals(0L, catalogManager.getFileManager().count(study.toString(), query, sessionId).getNumTotalResults()); Cohort all = catalogManager.getCohortManager().get(studyId, new Query(CohortDBAdaptor.QueryParams.ID.key(), StudyEntry.DEFAULT_COHORT), null, sessionId).first(); assertTrue(all.getSamples().isEmpty()); }
@Test public void testDeleteCohort() throws CatalogException, IOException { String studyId = "user@1000G:phase1"; Sample sampleId1 = catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_1"), new QueryOptions(), sessionIdUser).first(); Sample sampleId2 = catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_2"), new QueryOptions(), sessionIdUser).first(); Sample sampleId3 = catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_3"), new QueryOptions(), sessionIdUser).first(); Cohort myCohort = catalogManager.getCohortManager().create(studyId, new Cohort().setId("MyCohort").setType(Study.Type.FAMILY) .setSamples(Arrays.asList(sampleId1, sampleId2, sampleId3)), null, sessionIdUser).first(); assertEquals("MyCohort", myCohort.getId()); assertEquals(3, myCohort.getSamples().size()); assertTrue(myCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId1.getUid())); assertTrue(myCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId2.getUid())); assertTrue(myCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId3.getUid())); WriteResult deleteResult = catalogManager.getCohortManager().delete(studyId, new Query(CohortDBAdaptor.QueryParams.UID.key(), myCohort.getUid()), null, sessionIdUser); assertEquals(1, deleteResult.getNumModified()); Query query = new Query() .append(CohortDBAdaptor.QueryParams.UID.key(), myCohort.getUid()) .append(CohortDBAdaptor.QueryParams.STATUS_NAME.key(), "!=" + Cohort.CohortStatus.READY); Cohort cohort = catalogManager.getCohortManager().get(studyId, query, null, sessionIdUser).first(); assertEquals(Status.DELETED, cohort.getStatus().getName()); }
@Test public void testIndexWithAggregatedStats() throws Exception { QueryOptions queryOptions = new QueryOptions(VariantStorageEngine.Options.ANNOTATE.key(), false) .append(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), Aggregation.BASIC); queryOptions.put(VariantStorageEngine.Options.CALCULATE_STATS.key(), true); queryOptions.putIfNotNull(StorageOperation.CATALOG_PATH, outputId); variantManager.index(studyFqn, files.get(0).getId(), opencga.createTmpOutdir(studyId, "index", sessionId), queryOptions, sessionId); assertEquals(0, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.READY, getDefaultCohort(studyId).getStatus().getName()); StatsVariantStorageTest.checkCalculatedAggregatedStats(Collections.singleton(DEFAULT_COHORT), dbName); }
List<Long> samplesInCohort = defaultCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()); for (File file : expectedLoadedFiles) { Long[] samplesInFiles = file.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).toArray(new Long[0]);
assertEquals(cohortSolrModel.getNumSamples(), cohort.getSamples().size());
@Test public void testIndexWithStatsLowerCaseAggregationType() throws Exception { QueryOptions queryOptions = new QueryOptions(VariantStorageEngine.Options.ANNOTATE.key(), false); queryOptions.put(VariantStorageEngine.Options.CALCULATE_STATS.key(), true); queryOptions.put(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), "none"); queryOptions.putIfNotNull(StorageOperation.CATALOG_PATH, outputId); variantManager.index(studyId, getFile(0).getId(), newTmpOutdir(), queryOptions, sessionId); assertEquals(500, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.READY, getDefaultCohort(studyId).getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); }
assertEquals(Cohort.CohortStatus.READY, catalogManager.getCohortManager().get(studyId, coh[0], null, sessionId).first().getStatus().getName()); List<String> newCohort = catalogManager.getCohortManager().get(studyId, coh[0], null, sessionId).first().getSamples().stream() .map(Sample::getId) .skip(10).limit(100)
assertEquals(500, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.NONE, getDefaultCohort(studyId).getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); assertEquals(1000, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.NONE, getDefaultCohort(studyId).getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(1).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); assertEquals(1500, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.READY, getDefaultCohort(studyId).getStatus().getName()); checkCalculatedStats(studyId, Collections.singletonMap(DEFAULT_COHORT, catalogManager.getCohortManager().get(studyId, assertEquals(2000, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.INVALID, getDefaultCohort(studyId).getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(3).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); assertEquals(2504, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.READY, getDefaultCohort(studyId).getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(4).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS));
@Test public void testIndexWithStatsWrongAggregationType() throws Exception { QueryOptions queryOptions = new QueryOptions(VariantStorageEngine.Options.ANNOTATE.key(), false); queryOptions.put(VariantStorageEngine.Options.CALCULATE_STATS.key(), true); queryOptions.put(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), "wrong_type"); queryOptions.putIfNotNull(StorageOperation.CATALOG_PATH, outputId); try { variantManager.index(studyId, getFile(0).getId(), newTmpOutdir(), queryOptions, sessionId); fail("Expected StoragePipelineException exception"); } catch (StoragePipelineException e) { assertEquals(0, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.NONE, getDefaultCohort(studyId).getStatus().getName()); assertEquals(FileIndex.IndexStatus.TRANSFORMED, catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getIndex().getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); } queryOptions.put(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), "none"); // File already transformed queryOptions.put(VariantFileIndexerStorageOperation.LOAD, true); variantManager.index(studyId, getFile(0).getId(), newTmpOutdir(), queryOptions, sessionId); assertEquals(500, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.READY, getDefaultCohort(studyId).getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); }