protected void updateCohorts(String studyId, List<String> cohortIds, String sessionId, String status, String message) throws CatalogException { for (String cohortId : cohortIds) { catalogManager.getCohortManager().setStatus(studyId, cohortId, status, message, sessionId); } }
@Override protected List<String> validate(String defaultStudyStr, List<String> values, Integer release, VariantQueryParam param, String sessionId) throws CatalogException { if (release == null) { AbstractManager.MyResources<Cohort> uids = catalogManager.getCohortManager().getUids(values, defaultStudyStr, sessionId); return uids.getResourceList().stream().map(Cohort::getId).collect(Collectors.toList()); } else { return validate(defaultStudyStr, values, release, param, catalogManager.getCohortManager(), Cohort::getId, Cohort::getRelease, null, sessionId); } } }
private Cohort createDefaultCohort(Study study, String sessionId) throws CatalogException { return catalogManager.getCohortManager().create(study.getId(), StudyEntry.DEFAULT_COHORT, Study.Type.COLLECTION, DEFAULT_COHORT_DESCRIPTION, Collections.emptyList(), null, null, sessionId).first(); }
private String updateDefaultCohortStatus(Study study, String status, String sessionId) throws CatalogException { Query query = new Query(CohortDBAdaptor.QueryParams.ID.key(), StudyEntry.DEFAULT_COHORT); Cohort defaultCohort = catalogManager.getCohortManager().get(study.getFqn(), query, new QueryOptions(), sessionId).first(); String prevStatus = defaultCohort.getStatus().getName(); catalogManager.getCohortManager().setStatus(study.getFqn(), defaultCohort.getId(), status, null, sessionId); return prevStatus; }
public Map<String, Object> facet(String studyStr, String fileFields, String sampleFields, String individualFields, String cohortFields, String familyFields, boolean defaultStats, String sessionId) throws CatalogException, IOException { Map<String, Object> result = new HashMap<>(); result.put("sample", catalogManager.getSampleManager().facet(studyStr, new Query(), setFacetFields(sampleFields), defaultStats, sessionId)); result.put("file", catalogManager.getFileManager().facet(studyStr, new Query(), setFacetFields(fileFields), defaultStats, sessionId)); result.put("individual", catalogManager.getIndividualManager().facet(studyStr, new Query(), setFacetFields(individualFields), defaultStats, sessionId)); result.put("family", catalogManager.getFamilyManager().facet(studyStr, new Query(), setFacetFields(familyFields), defaultStats, sessionId)); result.put("cohort", catalogManager.getCohortManager().facet(studyStr, new Query(), setFacetFields(cohortFields), defaultStats, sessionId)); return result; }
public void calculateAggregatedStats(QueryOptions options) throws Exception { // coh0 = catalogManager.createCohort(studyId, "ALL", Cohort.Type.COLLECTION, "", file.getSampleIds(), null, sessionId).first().getId(); String cohId = catalogManager.getCohortManager().get(studyId, (Query) null, null, sessionId).first().getId(); calculateStats(cohId, options); checkCalculatedAggregatedStats(Collections.singleton(DEFAULT_COHORT), dbName); }
@Test public void testCalculateAggregatedExacStats() throws Exception { beforeAggregated("exachead.vcf.gz", Aggregation.EXAC); String tagMap = getResourceUri("exac-tag-mapping.properties").getPath(); List<String> cohortIds = createCohorts(sessionId, studyId, tagMap, catalogManager, logger) .stream().map(Cohort::getId).map(Object::toString).collect(Collectors.toList()); QueryOptions options = new QueryOptions(VariantStorageEngine.Options.AGGREGATION_MAPPING_PROPERTIES.key(), tagMap); calculateStats(options, cohortIds); List<Cohort> cohorts = catalogManager.getCohortManager().get(studyId, (Query) null, null, sessionId).getResult(); Set<String> cohortNames = cohorts .stream() .map(Cohort::getId) .collect(Collectors.toSet()); assertEquals(8, cohortNames.size()); for (Cohort cohort : cohorts) { assertEquals(Cohort.CohortStatus.READY, cohort.getStatus().getName()); } // checkCalculatedAggregatedStats(cohorts, dbName); }
public static void checkCalculatedStats(String studyId, Map<String, Cohort> cohorts, CatalogManager catalogManager, String dbName, String sessionId) throws Exception { VariantDBAdaptor dbAdaptor = StorageEngineFactory.get().getVariantStorageEngine(null, dbName).getDBAdaptor(); for (Variant variant : dbAdaptor) { for (StudyEntry sourceEntry : variant.getStudies()) { assertEquals("In variant " + variant.toString(), cohorts.size(), sourceEntry.getStats().size()); for (Map.Entry<String, VariantStats> entry : sourceEntry.getStats().entrySet()) { assertTrue("In variant " + variant.toString(), cohorts.containsKey(entry.getKey())); if (cohorts.get(entry.getKey()) != null) { assertEquals("Variant: " + variant.toString() + " does not have the correct number of samples in cohort '" + entry.getKey() + "'. jsonVariant: " + variant.toJson() , cohorts.get(entry.getKey()).getSamples().size(), entry.getValue().getGenotypeCount().values().stream().reduce(Integer::sum).orElse(0).intValue()); } } } } for (Cohort cohort : cohorts.values()) { cohort = catalogManager.getCohortManager().get(studyId, cohort.getId(), null, sessionId).first(); assertEquals(Cohort.CohortStatus.READY, cohort.getStatus().getName()); } }
public static File create(String resourceName, boolean indexed) throws IOException, CatalogException { File file; URI uri = getResourceUri(resourceName); file = fileMetadataReader.create(studyId, uri, "data/vcfs/", "", true, null, sessionId).first(); catalogFileUtils.upload(uri, file, null, sessionId, false, false, true, false, Long.MAX_VALUE); if (indexed) { FileIndex fileIndex = new FileIndex("user", "today", new FileIndex.IndexStatus(FileIndex.IndexStatus.READY), 1234, Collections.emptyMap()); catalogManager.getFileManager().setFileIndex(studyId, file.getPath(), fileIndex, sessionId); indexedFiles.add((int) file.getUid()); List<String> samples = catalogManager.getCohortManager().getSamples(studyId, cohortId, null, sessionId).getResult().stream().map(Sample::getId).collect(Collectors.toList()); samples.addAll(file.getSamples().stream().map(Sample::getId).collect(Collectors.toList())); catalogManager.getCohortManager().update(studyId, cohortId, new ObjectMap(CohortDBAdaptor.QueryParams.SAMPLES.key(), samples), true, null, sessionId); } return catalogManager.getFileManager().get(studyId, file.getId(), null, sessionId).first(); }
@Test public void testCalculateAggregatedExacStatsExplicitCohorts() throws Exception { beforeAggregated("exachead.vcf.gz", Aggregation.EXAC); String tagMap = getResourceUri("exac-tag-mapping.properties").getPath(); QueryOptions options = new QueryOptions(VariantStorageEngine.Options.AGGREGATION_MAPPING_PROPERTIES.key(), tagMap); calculateStats(options, Arrays.asList("AFR", "ALL", "AMR", "EAS", "FIN", "NFE", "OTH", "SAS")); List<Cohort> cohorts = catalogManager.getCohortManager().get(studyId, (Query) null, null, sessionId).getResult(); Set<String> cohortNames = cohorts .stream() .map(Cohort::getId) .collect(Collectors.toSet()); assertEquals(8, cohortNames.size()); for (Cohort cohort : cohorts) { assertEquals(Cohort.CohortStatus.READY, cohort.getStatus().getName()); } // checkCalculatedAggregatedStats(cohorts, dbName); }
@Test public void readCohort() throws CatalogException { assertEquals(1, catalogManager.getCohortManager().get(studyFqn, new Query(), null, ownerSessionId).getNumResults()); assertEquals(0, catalogManager.getCohortManager().get(studyFqn, new Query(), null, externalSessionId).getNumResults()); }
@Test public void testCalculateAggregatedExacStatsWithoutCohorts() throws Exception { beforeAggregated("exachead.vcf.gz", Aggregation.EXAC); String tagMap = getResourceUri("exac-tag-mapping.properties").getPath(); QueryOptions options = new QueryOptions(VariantStorageEngine.Options.AGGREGATION_MAPPING_PROPERTIES.key(), tagMap); calculateStats(options); List<Cohort> cohorts = catalogManager.getCohortManager().get(studyId, (Query) null, null, sessionId).getResult(); Set<String> cohortNames = cohorts .stream() .map(Cohort::getId) .collect(Collectors.toSet()); assertEquals(8, cohortNames.size()); for (Cohort cohort : cohorts) { assertEquals(Cohort.CohortStatus.READY, cohort.getStatus().getName()); } // checkCalculatedAggregatedStats(cohorts, dbName); }
@Test public void testCalculateInvalidStats() throws Exception { before(); calculateStats(coh[0]); DummyVariantStorageEngine vsm = mockVariantStorageManager(); String message = "Error"; doThrow(new StorageEngineException(message)).when(vsm).calculateStats(any(), any(List.class), any()); doThrow(new StorageEngineException(message)).when(vsm).calculateStats(any(), any(Map.class), any()); try { calculateStats(coh[1]); fail(); } catch (StorageEngineException e) { assertEquals(message, e.getCause().getMessage()); } Cohort coh1 = catalogManager.getCohortManager().get(studyId, coh[1], null, sessionId).first(); assertEquals(Cohort.CohortStatus.INVALID, coh1.getStatus().getName()); vsm = mockVariantStorageManager(); calculateStats(coh[1]); }
@Test public void testResumeCalculateStats() throws Exception { before(); calculateStats(coh[0]); catalogManager.getCohortManager().setStatus(studyId, coh[1], Cohort.CohortStatus.CALCULATING, "", sessionId); Cohort coh1 = catalogManager.getCohortManager().get(studyId, coh[1], null, sessionId).first(); Exception expected = VariantStatsStorageOperation.unableToCalculateCohortCalculating(coh1); try { calculateStats(coh[1]); fail(); } catch (Exception e) { assertThat(e, instanceOf(expected.getClass())); assertThat(e, hasMessage(is(expected.getMessage()))); } calculateStats(coh[1], new QueryOptions(VariantStorageEngine.Options.RESUME.key(), true)); }
protected Cohort getDefaultCohort(String studyId) throws CatalogException { return catalogManager.getCohortManager().get(studyId, new Query(CohortDBAdaptor.QueryParams.ID.key(), DEFAULT_COHORT), new QueryOptions(), sessionId).first(); }
public void before () throws Exception { File file = opencga.createFile(studyId, "1000g_batches/1-500.filtered.10k.chr22.phase3_shapeit2_mvncall_integrated_v5.20130502.genotypes.vcf.gz", sessionId); for (int i = 0; i < coh.length; i++) { Cohort cohort = catalogManager.getCohortManager().create(studyId, "coh" + i, Study.Type.CONTROL_SET, "", file.getSamples().subList(file.getSamples() .size() / coh.length * i, file.getSamples().size() / coh.length * (i + 1)), null, null, sessionId).first(); coh[i] = cohort.getId(); } QueryOptions queryOptions = new QueryOptions(VariantStorageEngine.Options.ANNOTATE.key(), false); queryOptions.putIfNotNull(StorageOperation.CATALOG_PATH, outputId); variantManager.index(studyId, file.getId(), createTmpOutdir(file), queryOptions, sessionId); all = catalogManager.getCohortManager().get(studyId, new Query(CohortDBAdaptor.QueryParams.ID.key(), DEFAULT_COHORT), new QueryOptions(), sessionId).first().getId(); }
private void removeStudy(Object study, QueryOptions options) throws Exception { variantManager.removeStudy(study.toString(), sessionId, new QueryOptions()); Query query = new Query(FileDBAdaptor.QueryParams.INDEX_STATUS_NAME.key(), FileIndex.IndexStatus.READY); assertEquals(0L, catalogManager.getFileManager().count(study.toString(), query, sessionId).getNumTotalResults()); Cohort all = catalogManager.getCohortManager().get(studyId, new Query(CohortDBAdaptor.QueryParams.ID.key(), StudyEntry.DEFAULT_COHORT), null, sessionId).first(); assertTrue(all.getSamples().isEmpty()); }
@Test public void testCreateCohortAlreadyExisting() throws CatalogException { Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), new QueryOptions(), sessionIdUser).first(); catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort").setType(Study.Type.FAMILY) .setSamples(Collections.singletonList(sampleId1)), null, sessionIdUser).first(); thrown.expect(CatalogDBException.class); catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort").setType(Study.Type.FAMILY) .setSamples(Collections.singletonList(sampleId1)), null, sessionIdUser).first(); }
@Test public void testDeleteCohortWithStats() throws Exception { QueryOptions queryOptions = new QueryOptions(VariantStorageEngine.Options.ANNOTATE.key(), false) .append(VariantStorageEngine.Options.CALCULATE_STATS.key(), true); File transformedFile = transformFile(getFile(0), queryOptions); loadFile(transformedFile, queryOptions, outputId); WriteResult result = catalogManager.getCohortManager().delete(studyFqn, new Query(CohortDBAdaptor.QueryParams.ID.key(), "ALL"), null, sessionId); assertEquals(0, result.getNumModified()); assertTrue(result.getFailed().get(0).getMessage().contains("ALL cannot be deleted")); }
@Test public void testCreateCohortFail() throws CatalogException { thrown.expect(CatalogException.class); List<Sample> sampleList = Arrays.asList(new Sample().setUid(23L), new Sample().setUid(4L), new Sample().setUid(5L)); catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort").setType(Study.Type.FAMILY).setSamples(sampleList), null, sessionIdUser); }