public VariantSearchManager getVariantSearchManager() throws StorageEngineException { if (variantSearchManager.get() == null) { synchronized (variantSearchManager) { if (variantSearchManager.get() == null) { // TODO One day we should use reflection here reading from storage-configuration.yml variantSearchManager.set(new VariantSearchManager(getStudyConfigurationManager(), configuration)); } } } return variantSearchManager.get(); }
public VariantExporter(VariantStorageEngine engine) throws StorageEngineException { this(engine, new VariantMetadataFactory(engine.getStudyConfigurationManager())); }
/** * Exports the result of the given query and the associated metadata. * @param outputFile Optional output file. If null or empty, will print into the Standard output. Won't export any metadata. * @param outputFormat Variant output format * @param query Query with the variants to export * @param queryOptions Query options * @throws IOException If there is any IO error * @throws StorageEngineException If there is any error exporting variants */ public void exportData(URI outputFile, VariantOutputFormat outputFormat, Query query, QueryOptions queryOptions) throws IOException, StorageEngineException { exportData(outputFile, outputFormat, new VariantMetadataFactory(getStudyConfigurationManager() ), query, queryOptions); }
public void calculateStats(String study, Map<String, ? extends Collection<String>> cohorts, QueryOptions options) throws StorageEngineException, IOException { VariantStatisticsManager statisticsManager = newVariantStatisticsManager(); StudyConfigurationManager scm = getStudyConfigurationManager(); scm.lockAndUpdate(study, sc -> { scm.registerCohorts(sc, cohorts); return sc; }); statisticsManager.calculateStatistics(study, new ArrayList<>(cohorts.keySet()), options); }
private Map<String, List<Sample>> checkSamplesPermissions(Query query, QueryOptions queryOptions, String sessionId) throws CatalogException, StorageEngineException, IOException { String study = catalogUtils.getAnyStudy(query, sessionId); DataStore dataStore = getDataStore(study, sessionId); VariantStorageEngine variantStorageEngine = getVariantStorageEngine(dataStore); return checkSamplesPermissions(query, queryOptions, variantStorageEngine.getStudyConfigurationManager(), sessionId); }
public StudyConfiguration updateCatalogFromStudyConfiguration(String sessionId, String study, DataStore dataStore) throws IOException, CatalogException, StorageEngineException { CatalogStudyConfigurationFactory studyConfigurationFactory = new CatalogStudyConfigurationFactory(catalogManager); StudyConfigurationManager studyConfigurationManager = getVariantStorageEngine(dataStore).getStudyConfigurationManager(); StudyConfiguration studyConfiguration = studyConfigurationManager.getStudyConfiguration(study, null).first(); if (studyConfiguration != null) { // Update Catalog file and cohort status. studyConfigurationFactory.updateCatalogFromStudyConfiguration(studyConfiguration, sessionId); } return studyConfiguration; }
public void saveAnnotation(String project, String annotationName, ObjectMap params, String sessionId) throws StorageEngineException, VariantAnnotatorException, CatalogException, IllegalAccessException, InstantiationException, ClassNotFoundException { DataStore dataStore = getDataStoreByProjectId(project, sessionId); VariantStorageEngine variantStorageEngine = storageEngineFactory.getVariantStorageEngine(dataStore.getStorageEngine(), dataStore.getDbName()); StorageOperation.updateProjectMetadata(catalogManager, variantStorageEngine.getStudyConfigurationManager(), project, sessionId); variantStorageEngine.saveAnnotation(annotationName, params); }
public void deleteAnnotation(String project, String annotationName, ObjectMap params, String sessionId) throws StorageEngineException, VariantAnnotatorException, CatalogException, IllegalAccessException, InstantiationException, ClassNotFoundException { DataStore dataStore = getDataStoreByProjectId(project, sessionId); VariantStorageEngine variantStorageEngine = storageEngineFactory.getVariantStorageEngine(dataStore.getStorageEngine(), dataStore.getDbName()); StorageOperation.updateProjectMetadata(catalogManager, variantStorageEngine.getStudyConfigurationManager(), project, sessionId); variantStorageEngine.deleteAnnotation(annotationName, params); }
public VariantDBIterator iterator(Query query, QueryOptions queryOptions, String sessionId) throws CatalogException, StorageEngineException { String study = catalogUtils.getAnyStudy(query, sessionId); DataStore dataStore = getDataStore(study, sessionId); VariantStorageEngine storageEngine = getVariantStorageEngine(dataStore); catalogUtils.parseQuery(query, sessionId); checkSamplesPermissions(query, queryOptions, storageEngine.getStudyConfigurationManager(), sessionId); return storageEngine.iterator(query, queryOptions); }
private <R> R secure(Query query, QueryOptions queryOptions, String sessionId, VariantReadOperation<R> supplier) throws CatalogException, StorageEngineException, IOException { String study = catalogUtils.getAnyStudy(query, sessionId); catalogUtils.parseQuery(query, sessionId); DataStore dataStore = getDataStore(study, sessionId); VariantStorageEngine variantStorageEngine = getVariantStorageEngine(dataStore); checkSamplesPermissions(query, queryOptions, variantStorageEngine.getStudyConfigurationManager(), sessionId); return supplier.apply(variantStorageEngine); } private <R> R secure(Query facetedQuery, Query query, QueryOptions queryOptions,
@Test public void testResumeOnError() throws Exception { variantStorageEngine.getStudyConfigurationManager().lockAndUpdate(STUDY_NAME, sc -> { Integer id = sc.getSearchIndexedSampleSets().get(sc.getSampleIds().get(samples1.get(0))); sc.getSearchIndexedSampleSetsStatus().put(id, BatchFileOperation.Status.ERROR); return sc; }); variantStorageEngine.searchIndexSamples(STUDY_NAME, samples1); }
/** * Provide a new VariantAnnotationManager for creating and loading annotations. * * @param params Other params * @return A new instance of VariantAnnotationManager * @throws StorageEngineException if there is an error creating the VariantAnnotationManager * @throws VariantAnnotatorException if there is an error creating the VariantAnnotator */ protected final VariantAnnotationManager newVariantAnnotationManager(ObjectMap params) throws StorageEngineException, VariantAnnotatorException { ProjectMetadata projectMetadata = getStudyConfigurationManager().getProjectMetadata(getMergedOptions(params)).first(); VariantAnnotator annotator = VariantAnnotatorFactory.buildVariantAnnotator( configuration, getStorageEngineId(), projectMetadata, params); return newVariantAnnotationManager(annotator); }
@Test public void testResumeFail() throws Exception { variantStorageEngine.getStudyConfigurationManager().lockAndUpdate(STUDY_NAME, sc -> { Integer id = sc.getSearchIndexedSampleSets().get(sc.getSampleIds().get(samples1.get(0))); sc.getSearchIndexedSampleSetsStatus().put(id, BatchFileOperation.Status.RUNNING); return sc; }); thrown.expectMessage("Samples already being indexed. Resume operation to continue."); variantStorageEngine.searchIndexSamples(STUDY_NAME, samples1); }
@Before public void before() throws Exception { solr.configure(variantStorageEngine); if (!loaded) { load(); loaded = true; } variantStorageEngine.getStudyConfigurationManager().lockAndUpdate(STUDY_NAME, sc -> { for (Integer id : sc.getSearchIndexedSampleSetsStatus().keySet()) { sc.getSearchIndexedSampleSetsStatus().put(id, BatchFileOperation.Status.READY); } COLLECTION_1 = "opencga_variants_test_1_" + sc.getSearchIndexedSampleSets().get(sc.getSampleIds().get(samples1.get(0))); COLLECTION_2 = "opencga_variants_test_1_" + sc.getSearchIndexedSampleSets().get(sc.getSampleIds().get(samples2.get(0))); return sc; }); }
@Test public void testResumeWhileRunning() throws Exception { variantStorageEngine.getStudyConfigurationManager().lockAndUpdate(STUDY_NAME, sc -> { Integer id = sc.getSearchIndexedSampleSets().get(sc.getSampleIds().get(samples1.get(0))); sc.getSearchIndexedSampleSetsStatus().put(id, BatchFileOperation.Status.RUNNING); return sc; }); variantStorageEngine.getOptions().put(VariantStorageEngine.Options.RESUME.key(), true); variantStorageEngine.searchIndexSamples(STUDY_NAME, samples1); }
public CellBaseUtils getCellBaseUtils() throws StorageEngineException { if (cellBaseUtils == null) { final ProjectMetadata metadata = getStudyConfigurationManager().getProjectMetadata(getOptions()).first(); String species = metadata.getSpecies(); String assembly = metadata.getAssembly(); ClientConfiguration clientConfiguration = configuration.getCellbase().toClientConfiguration(); if (StringUtils.isEmpty(species)) { species = clientConfiguration.getDefaultSpecies(); } species = toCellBaseSpeciesName(species); cellBaseUtils = new CellBaseUtils(new CellBaseClient(species, assembly, clientConfiguration)); } return cellBaseUtils; }
@Override public StudyConfigurationManager getStudyConfigurationManager() throws StorageEngineException { ObjectMap options = getOptions(); if (studyConfigurationManager != null) { return studyConfigurationManager; } else if (!options.getString(FileStudyConfigurationAdaptor.STUDY_CONFIGURATION_PATH, "").isEmpty()) { return super.getStudyConfigurationManager(); } else { MongoDataStoreManager mongoDataStoreManager = getMongoDataStoreManager(); MongoDataStore db = mongoDataStoreManager.get( getMongoCredentials().getMongoDbName(), getMongoCredentials().getMongoDBConfiguration()); studyConfigurationManager = new StudyConfigurationManager(new MongoDBVariantStorageMetadataDBAdaptorFactory(db, options)); return studyConfigurationManager; } }
@Test public void testDoQuerySearchManagerSpecificSearchIndexSamplesNotReadyCollections() throws Exception { Query query = new Query(SAMPLE.key(), samples1); check(COLLECTION_1, query, QueryOptions.empty()); variantStorageEngine.getStudyConfigurationManager().lockAndUpdate(STUDY_NAME, sc -> { Integer id = sc.getSearchIndexedSampleSets().get(sc.getSampleIds().get(samples1.get(0))); sc.getSearchIndexedSampleSetsStatus().put(id, BatchFileOperation.Status.RUNNING); return sc; }); check(null, query, QueryOptions.empty()); }
protected void check(String collection, Query query, QueryOptions options) throws StorageEngineException { assertEquals(query.toJson() + " " + options.toJson(), collection, VariantSearchUtils.inferSpecificSearchIndexSamplesCollection(query, options, variantStorageEngine.getStudyConfigurationManager(), DB_NAME)); }
@Test @Override public void multiRegionIndex() throws Exception { super.multiRegionIndex(); checkLoadedVariants(); StudyConfiguration studyConfiguration = variantStorageEngine.getStudyConfigurationManager() .getStudyConfiguration(1, null).first(); for (BatchFileOperation batchFileOperation : studyConfiguration.getBatches()) { assertEquals(MongoDBVariantOptions.DIRECT_LOAD.key(), batchFileOperation.getOperationName()); } }