public final ObjectMap getMergedOptions(Map<? extends String, ?> params) { ObjectMap options = new ObjectMap(getOptions()); if (params != null) { params.forEach(options::putIfNotNull); } return options; }
private void remove() throws Exception { StorageVariantCommandOptions.VariantRemoveCommandOptions cliOptions = variantCommandOptions.variantRemoveCommandOptions; variantStorageEngine.getOptions().put(VariantStorageEngine.Options.RESUME.key(), cliOptions.resume); variantStorageEngine.getOptions().putAll(cliOptions.commonOptions.params); if (cliOptions.files.size() == 1 && cliOptions.files.get(0).equalsIgnoreCase(VariantQueryUtils.ALL)) { variantStorageEngine.removeStudy(cliOptions.study); } else { variantStorageEngine.removeFiles(cliOptions.study, cliOptions.files); } }
/** * Index the given input files. By default, executes the steps in {@link VariantStoragePipeline}. * * Will create a {@link #newStoragePipeline} for each input file. * * @param inputFiles Input files to index * @param outdirUri Output directory for possible intermediate files * @param doExtract Execute extract step {@link VariantStoragePipeline#extract} * @param doTransform Execute transform step {@link VariantStoragePipeline#transform} * @param doLoad Execute load step {@link VariantStoragePipeline#load} * @return List of {@link StoragePipelineResult}, one for each input file. * @throws StorageEngineException If there is any problem related with the StorageEngine */ @Override public List<StoragePipelineResult> index(List<URI> inputFiles, URI outdirUri, boolean doExtract, boolean doTransform, boolean doLoad) throws StorageEngineException { List<StoragePipelineResult> results = super.index(inputFiles, outdirUri, doExtract, doTransform, doLoad); if (doLoad) { annotateLoadedFiles(outdirUri, inputFiles, results, getOptions()); calculateStatsForLoadedFiles(outdirUri, inputFiles, results, getOptions()); searchIndexLoadedFiles(inputFiles, getOptions()); } return results; }
ObjectMap options = new ObjectMap(engine.getOptions()) .append(VariantExporterDriver.OUTPUT_PARAM, outputFileUri.toString()) .append(VariantExporterDriver.OUTPUT_FORMAT_PARAM, outputFormat.toString()); mrExecutor.run(VariantExporterDriver.class, args, engine.getOptions(), "Export variants");
protected void setDefaultTimeout(QueryOptions options) { int defaultTimeout = getOptions().getInt(DEFAULT_TIMEOUT.key(), DEFAULT_TIMEOUT.defaultValue()); int maxTimeout = getOptions().getInt(MAX_TIMEOUT.key(), MAX_TIMEOUT.defaultValue()); int timeout = options.getInt(QueryOptions.TIMEOUT, defaultTimeout); if (timeout > maxTimeout) { throw new VariantQueryException("Invalid timeout '" + timeout + "'. Max timeout is " + maxTimeout); } else if (timeout < 0) { throw new VariantQueryException("Invalid timeout '" + timeout + "'. Timeout must be positive"); } options.put(QueryOptions.TIMEOUT, timeout); }
/** * Atomically updates the studyConfiguration before removing samples. * * @param study Study * @param files Files to remove * @return FileIds to remove * @throws StorageEngineException StorageEngineException */ protected BatchFileOperation preRemoveFiles(String study, List<String> files) throws StorageEngineException { List<Integer> fileIds = new ArrayList<>(); AtomicReference<BatchFileOperation> batchFileOperation = new AtomicReference<>(); getStudyConfigurationManager().lockAndUpdate(study, studyConfiguration -> { fileIds.addAll(getStudyConfigurationManager().getFileIdsFromStudy(files, studyConfiguration)); boolean resume = getOptions().getBoolean(RESUME.key(), RESUME.defaultValue()); batchFileOperation.set(addBatchOperation(studyConfiguration, REMOVE_OPERATION_NAME, fileIds, resume, BatchFileOperation.Type.REMOVE)); if (!studyConfiguration.getIndexedFiles().containsAll(fileIds)) { // Remove indexed files to get non indexed files fileIds.removeAll(studyConfiguration.getIndexedFiles()); throw new StorageEngineException("Unable to remove non indexed files: " + fileIds); } return studyConfiguration; }); return batchFileOperation.get(); }
public void searchIndexSamples(String study, List<String> samples, QueryOptions queryOptions, String sessionId) throws StorageEngineException, IOException, VariantSearchException, IllegalAccessException, ClassNotFoundException, InstantiationException, CatalogException { DataStore dataStore = getDataStore(study, sessionId); VariantStorageEngine variantStorageEngine = storageEngineFactory.getVariantStorageEngine(dataStore.getStorageEngine(), dataStore.getDbName()); variantStorageEngine.getOptions().putAll(queryOptions); variantStorageEngine.searchIndexSamples(study, samples); }
public void removeSearchIndexSamples(String study, List<String> samples, QueryOptions queryOptions, String sessionId) throws CatalogException, IllegalAccessException, InstantiationException, ClassNotFoundException, StorageEngineException, VariantSearchException { DataStore dataStore = getDataStore(study, sessionId); VariantStorageEngine variantStorageEngine = storageEngineFactory.getVariantStorageEngine(dataStore.getStorageEngine(), dataStore.getDbName()); variantStorageEngine.getOptions().putAll(queryOptions); variantStorageEngine.removeSearchIndexSamples(study, samples); }
public void importData(StudyInfo studyInfo, URI inputUri, String sessionId) throws IOException, StorageEngineException { VariantMetadataImporter variantMetadataImporter; variantMetadataImporter = new CatalogVariantMetadataImporter(studyInfo.getStudyFQN(), inputUri, sessionId); try { DataStore dataStore = studyInfo.getDataStores().get(File.Bioformat.VARIANT); VariantStorageEngine variantStorageEngine = getVariantStorageEngine(dataStore); ObjectMap options = variantStorageEngine.getOptions(); VariantMetadata metadata; StudyConfiguration studyConfiguration; try (StudyConfigurationManager scm = variantStorageEngine.getStudyConfigurationManager()) { metadata = variantMetadataImporter.importMetaData(inputUri, scm); studyConfiguration = scm.getStudyConfiguration(((int) studyInfo.getStudyUid()), null).first(); } variantStorageEngine.importData(inputUri, metadata, Collections.singletonList(studyConfiguration), options); } catch (Exception e) { logger.error("Error importing data"); throw e; } }
public void removeStudy(StudyInfo studyInfo, QueryOptions options, String sessionId) throws CatalogException, StorageEngineException, IOException { // We get the credentials of the Datastore to insert the variants DataStore dataStore = studyInfo.getDataStores().get(File.Bioformat.VARIANT); // Update study configuration BEFORE executing the operation and fetching files from Catalog updateCatalogFromStudyConfiguration(sessionId, studyInfo.getStudyFQN(), dataStore); VariantStorageEngine variantStorageEngine = getVariantStorageEngine(dataStore); variantStorageEngine.getOptions().putAll(options); variantStorageEngine.removeStudy(studyInfo.getStudyFQN()); // Update study configuration to synchronize updateCatalogFromStudyConfiguration(sessionId, studyInfo.getStudyFQN(), dataStore); }
@Test public void testResumeWhileRunning() throws Exception { variantStorageEngine.getStudyConfigurationManager().lockAndUpdate(STUDY_NAME, sc -> { Integer id = sc.getSearchIndexedSampleSets().get(sc.getSampleIds().get(samples1.get(0))); sc.getSearchIndexedSampleSetsStatus().put(id, BatchFileOperation.Status.RUNNING); return sc; }); variantStorageEngine.getOptions().put(VariantStorageEngine.Options.RESUME.key(), true); variantStorageEngine.searchIndexSamples(STUDY_NAME, samples1); }
public CellBaseUtils getCellBaseUtils() throws StorageEngineException { if (cellBaseUtils == null) { final ProjectMetadata metadata = getStudyConfigurationManager().getProjectMetadata(getOptions()).first(); String species = metadata.getSpecies(); String assembly = metadata.getAssembly(); ClientConfiguration clientConfiguration = configuration.getCellbase().toClientConfiguration(); if (StringUtils.isEmpty(species)) { species = clientConfiguration.getDefaultSpecies(); } species = toCellBaseSpeciesName(species); cellBaseUtils = new CellBaseUtils(new CellBaseClient(species, assembly, clientConfiguration)); } return cellBaseUtils; }
public List<File> removeFiles(StudyInfo studyInfo, QueryOptions options, String sessionId) throws CatalogException, StorageEngineException, IOException { // We get the credentials of the Datastore to insert the variants DataStore dataStore = studyInfo.getDataStores().get(File.Bioformat.VARIANT); // Update study configuration BEFORE executing the operation and fetching files from Catalog updateCatalogFromStudyConfiguration(sessionId, studyInfo.getStudyFQN(), dataStore); List<String> fileNames = new ArrayList<>(studyInfo.getFileInfos().size()); List<String> filePaths = new ArrayList<>(studyInfo.getFileInfos().size()); for (FileInfo fileInfo : studyInfo.getFileInfos()) { File file = catalogManager.getFileManager().get(studyInfo.getStudyFQN(), fileInfo.getPath(), null, sessionId).first(); if (file.getIndex().getStatus().getName().equals(FileIndex.IndexStatus.READY)) { fileNames.add(fileInfo.getName()); filePaths.add(fileInfo.getPath()); } else { throw new CatalogException("Unable to remove variants from file " + file.getName() + ". " + "IndexStatus = " + file.getIndex().getStatus().getName()); } } if (fileNames.isEmpty()) { throw new CatalogException("Nothing to do!"); } VariantStorageEngine variantStorageEngine = getVariantStorageEngine(dataStore); variantStorageEngine.getOptions().putAll(options); variantStorageEngine.removeFiles(studyInfo.getStudyFQN(), fileNames); // Update study configuration to synchronize updateCatalogFromStudyConfiguration(sessionId, studyInfo.getStudyFQN(), dataStore); return catalogManager.getFileManager().get(studyInfo.getStudyFQN(), new Query(FileDBAdaptor.QueryParams.PATH.key(), filePaths), new QueryOptions(), sessionId) .getResult(); }
dbAdaptor.getStudyConfigurationManager().updateStudyConfiguration(studyConfiguration, null); options.put(VariantStorageEngine.Options.STUDY.key(), "S_" + studyId); storageEngine.getOptions().putAll(options); storageEngine.getOptions().put(VariantStorageEngine.Options.RELEASE.key(), release++); storageEngine.index(inputFiles.subList(0, 2), outputUri, true, true, true); storageEngine.getOptions().put(VariantStorageEngine.Options.RELEASE.key(), release++); storageEngine.index(inputFiles.subList(2, 4), outputUri, true, true, true);
boolean resume = getOptions().getBoolean(RESUME.key(), RESUME.defaultValue()); id.set(getStudyConfigurationManager().registerSearchIndexSamples(studyConfiguration, samples, resume)); return studyConfiguration;
@Before public void before() throws Exception { clearDB(DB_NAME); dbAdaptor = getVariantStorageEngine().getDBAdaptor(); VariantStorageEngine storageEngine = getVariantStorageEngine(); List<URI> inputFiles = new ArrayList<>(); // 12877, 12889 and 12890 have an overlapping at M:16184:C:A , M:16184:C:- , M:16184:CC:- // for (int i : new int[]{0, 12, 13}) { for (int fileId = 12877; fileId < 12877 + 17; fileId++) { String fileName = "1K.end.platinum-genomes-vcf-NA" + fileId + "_S1.genome.vcf.gz"; inputFiles.add(getResourceUri("platinum/" + fileName)); } storageEngine.getOptions().put(VariantStorageEngine.Options.STUDY.key(), STUDY_NAME); storageEngine.getOptions().put(VariantStorageEngine.Options.CALCULATE_STATS.key(), false); // storageEngine.index(inputFiles.subList(0, inputFiles.size()/2), outputUri, true, true, true); // storageEngine.index(inputFiles.subList(inputFiles.size()/2, inputFiles.size()), outputUri, true, true, true); storageEngine.index(inputFiles, outputUri, true, true, true); studyConfiguration = storageEngine.getStudyConfigurationManager().getStudyConfiguration(STUDY_ID, null).first(); }
storageEngine.getOptions().putAll(options); storageEngine.index(inputFiles, outputUri, true, true, true);
@Test public void testMultiAnnotations() throws Exception { VariantStorageEngine variantStorageEngine = getVariantStorageEngine(); runDefaultETL(smallInputUri, variantStorageEngine, newStudyConfiguration(), new ObjectMap(VariantStorageEngine.Options.ANNOTATE.key(), false)); variantStorageEngine.getOptions() .append(VARIANT_ANNOTATOR_CLASSNAME, TestAnnotator.class.getName()) .append(ANNOTATOR, VariantAnnotatorFactory.AnnotationSource.OTHER); variantStorageEngine.saveAnnotation("v0", new ObjectMap()); variantStorageEngine.annotate(new Query(), new ObjectMap(TestAnnotator.ANNOT_KEY, "v1").append(OVERWRITE_ANNOTATIONS, true)); variantStorageEngine.saveAnnotation("v1", new ObjectMap()); variantStorageEngine.annotate(new Query(), new ObjectMap(TestAnnotator.ANNOT_KEY, "v2").append(OVERWRITE_ANNOTATIONS, true)); variantStorageEngine.saveAnnotation("v2", new ObjectMap()); variantStorageEngine.annotate(new Query(VariantQueryParam.REGION.key(), "1"), new ObjectMap(TestAnnotator.ANNOT_KEY, "v3").append(OVERWRITE_ANNOTATIONS, true)); assertEquals(0, variantStorageEngine.getAnnotation("v0", null, null).getResult().size()); checkAnnotationSnapshot(variantStorageEngine, "v1", "v1"); checkAnnotationSnapshot(variantStorageEngine, "v2", "v2"); checkAnnotationSnapshot(variantStorageEngine, VariantAnnotationManager.CURRENT, VariantAnnotationManager.CURRENT, "v3", new Query(VariantQueryParam.REGION.key(), "1")); checkAnnotationSnapshot(variantStorageEngine, VariantAnnotationManager.CURRENT, "v2", "v2", new Query(VariantQueryParam.REGION.key(), "2")); variantStorageEngine.deleteAnnotation("v1", new ObjectMap()); testQueries(variantStorageEngine); }
@Test public void testChangeAnnotator() throws Exception { VariantStorageEngine variantStorageEngine = getVariantStorageEngine(); runDefaultETL(smallInputUri, variantStorageEngine, newStudyConfiguration(), new ObjectMap(VariantStorageEngine.Options.ANNOTATE.key(), false)); variantStorageEngine.getOptions() .append(VARIANT_ANNOTATOR_CLASSNAME, TestAnnotator.class.getName()) .append(ANNOTATOR, VariantAnnotatorFactory.AnnotationSource.OTHER); // First annotation. Should run ok. variantStorageEngine.annotate(new Query(), new ObjectMap(TestAnnotator.ANNOT_KEY, "v1")); assertEquals("v1", variantStorageEngine.getStudyConfigurationManager().getProjectMetadata().first().getAnnotation().getCurrent().getAnnotator().getVersion()); // Second annotation. New annotator. Overwrite. Should run ok. variantStorageEngine.annotate(new Query(), new ObjectMap(TestAnnotator.ANNOT_KEY, "v2").append(OVERWRITE_ANNOTATIONS, true)); assertEquals("v2", variantStorageEngine.getStudyConfigurationManager().getProjectMetadata().first().getAnnotation().getCurrent().getAnnotator().getVersion()); // Third annotation. New annotator. Do not overwrite. Should fail. thrown.expect(VariantAnnotatorException.class); variantStorageEngine.annotate(new Query(), new ObjectMap(TestAnnotator.ANNOT_KEY, "v3").append(OVERWRITE_ANNOTATIONS, false)); }
@Test public void testChangeAnnotatorFail() throws Exception { VariantStorageEngine variantStorageEngine = getVariantStorageEngine(); runDefaultETL(smallInputUri, variantStorageEngine, newStudyConfiguration(), new ObjectMap(VariantStorageEngine.Options.ANNOTATE.key(), false)); variantStorageEngine.getOptions() .append(VARIANT_ANNOTATOR_CLASSNAME, TestAnnotator.class.getName()) .append(ANNOTATOR, VariantAnnotatorFactory.AnnotationSource.OTHER); // First annotation. Should run ok. variantStorageEngine.annotate(new Query(), new ObjectMap(TestAnnotator.ANNOT_KEY, "v1")); try { // Second annotation. New annotator. Overwrite. Fail annotation variantStorageEngine.annotate(new Query(), new ObjectMap(TestAnnotator.ANNOT_KEY, "v2") .append(TestAnnotator.FAIL, true) .append(OVERWRITE_ANNOTATIONS, true)); fail("Expected to fail!"); } catch (VariantAnnotatorException e) { e.printStackTrace(); // Annotator information does not change assertEquals("v1", variantStorageEngine.getStudyConfigurationManager().getProjectMetadata().first().getAnnotation().getCurrent().getAnnotator().getVersion()); } // Second annotation bis. New annotator. Overwrite. variantStorageEngine.annotate(new Query(), new ObjectMap(TestAnnotator.ANNOT_KEY, "v2") .append(TestAnnotator.FAIL, false) .append(OVERWRITE_ANNOTATIONS, true)); assertEquals("v2", variantStorageEngine.getStudyConfigurationManager().getProjectMetadata().first().getAnnotation().getCurrent().getAnnotator().getVersion()); }