public StorageEngineConfiguration getStorageEngine() { return getStorageEngine(defaultStorageEngineId); }
public ObjectMap getOptions() { return configuration.getStorageEngine(storageEngineId).getVariant().getOptions(); }
public VariantStoragePipeline(StorageConfiguration configuration, String storageEngineId, VariantDBAdaptor dbAdaptor, VariantReaderUtils variantReaderUtils) { this(configuration, storageEngineId, dbAdaptor, variantReaderUtils, new ObjectMap(configuration.getStorageEngine(storageEngineId).getVariant().getOptions())); }
private void configure(ClientCliOptionsParser.CommonOptions commonOptions, String dbName) throws Exception { this.logFile = commonOptions.logFile; /** * Getting VariantStorageEngine * We need to find out the Storage Engine Id to be used * If not storage engine is passed then the default is taken from storage-configuration.yml file **/ this.storageEngine = (storageEngine != null && !storageEngine.isEmpty()) ? storageEngine : configuration.getDefaultStorageEngineId(); logger.debug("Storage Engine set to '{}'", this.storageEngine); this.storageConfiguration = configuration.getStorageEngine(storageEngine); // TODO: Start passing catalogManager StorageEngineFactory storageEngineFactory = StorageEngineFactory.get(configuration); if (storageEngine == null || storageEngine.isEmpty()) { this.alignmentStorageManager = storageEngineFactory.getAlignmentStorageEngine(null, dbName); } else { this.alignmentStorageManager = storageEngineFactory.getAlignmentStorageEngine(storageEngine, dbName); } }
private void configure(GeneralCliOptions.CommonOptions commonOptions, String dbName) throws Exception { this.logFile = commonOptions.logFile; /** * Getting VariantStorageEngine * We need to find out the Storage Engine Id to be used * If not storage engine is passed then the default is taken from storage-configuration.yml file **/ this.storageEngine = (storageEngine != null && !storageEngine.isEmpty()) ? storageEngine : configuration.getDefaultStorageEngineId(); logger.debug("Storage Engine set to '{}'", this.storageEngine); this.storageConfiguration = configuration.getStorageEngine(storageEngine); // TODO: Start passing catalogManager StorageEngineFactory storageEngineFactory = StorageEngineFactory.get(configuration); if (storageEngine == null || storageEngine.isEmpty()) { this.variantStorageEngine = storageEngineFactory.getVariantStorageEngine(null, dbName); } else { this.variantStorageEngine = storageEngineFactory.getVariantStorageEngine(storageEngine, dbName); } }
MongoCredentials getMongoCredentials() { // If no database name is provided, read from the configuration file if (StringUtils.isEmpty(dbName)) { dbName = getOptions().getString(DB_NAME.key(), DB_NAME.defaultValue()); } DatabaseCredentials database = configuration.getStorageEngine(STORAGE_ENGINE_ID).getVariant().getDatabase(); try { return new MongoCredentials(database, dbName); } catch (IllegalOpenCGACredentialsException e) { throw Throwables.propagate(e); } }
@Override public void removeFiles(String study, List<String> files) throws StorageEngineException { BatchFileOperation batchFileOperation = preRemoveFiles(study, files); List<Integer> fileIds = batchFileOperation.getFileIds(); ObjectMap options = new ObjectMap(configuration.getStorageEngine(STORAGE_ENGINE_ID).getVariant().getOptions()); StudyConfigurationManager scm = getStudyConfigurationManager(); Integer studyId = scm.getStudyId(study, null); Thread hook = scm.buildShutdownHook(REMOVE_OPERATION_NAME, studyId, fileIds); try { Runtime.getRuntime().addShutdownHook(hook); getDBAdaptor().removeFiles(study, files, batchFileOperation.getTimestamp(), new QueryOptions(options)); postRemoveFiles(study, fileIds, false); } catch (Exception e) { postRemoveFiles(study, fileIds, true); throw e; } finally { Runtime.getRuntime().removeShutdownHook(hook); } }
@Test public void transformIsolated() throws Exception { ObjectMap params = new ObjectMap(); URI outputUri = newOutputUri(); VariantStorageEngine variantStorageManager = getVariantStorageEngine(); variantStorageManager.getConfiguration().getStorageEngine(variantStorageManager.getStorageEngineId()).getVariant().getDatabase() .setHosts(Collections.singletonList("1.1.1.1")); StoragePipelineResult etlResult = runETL(variantStorageManager, smallInputUri, outputUri, params, true, true, false); System.out.println("etlResult = " + etlResult); String[] malformedFiles = Paths.get(outputUri).toFile().list((dir, name) -> name.contains(MALFORMED_FILE)); assertEquals(0, malformedFiles.length); }
private VariantDBAdaptor getVariantDBAdaptor(Request request) throws IllegalAccessException, InstantiationException, ClassNotFoundException, StorageEngineException { // Setting storageEngine and database parameters. If the storageEngine is not provided then the server default is used String storageEngine = genericGrpcService.getDefaultStorageEngine(); if (StringUtils.isNotEmpty(request.getStorageEngine())) { storageEngine = request.getStorageEngine(); } String database = genericGrpcService .getStorageConfiguration().getStorageEngine(storageEngine).getVariant().getOptions().getString("database.name"); if (StringUtils.isNotEmpty(request.getDatabase())) { database = request.getDatabase(); } // Creating the VariantDBAdaptor to the parsed storageEngine and database VariantDBAdaptor variantDBAdaptor = GenericGrpcService.storageEngineFactory .getVariantStorageEngine(storageEngine, database).getDBAdaptor(); // logger.debug("Connection to {}:{} in {}ms", storageEngine, database, System.currentTimeMillis() - start); return variantDBAdaptor; } }
@Test public void transformToJsonSTDOUT() throws Exception { URI outputUri = newOutputUri(); VariantStorageEngine variantStorageManager = getVariantStorageEngine(); File outputFile = Paths.get(outputUri.resolve("outputFile.json")).toFile(); try (PrintStream os = new PrintStream(new FileOutputStream(outputFile))) { System.setOut(os); ObjectMap options = variantStorageManager.getConfiguration() .getStorageEngine(variantStorageManager.getStorageEngineId()).getVariant().getOptions(); options.append(VariantStorageEngine.Options.STDOUT.key(), true); options.append(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "json"); variantStorageManager.index(Collections.singletonList(smallInputUri), outputUri, true, true, false).get(0); } finally { System.setOut(out); } assertEquals(999, countLines(outputFile)); }
@Test public void transformFromSTDIN() throws Exception { URI outputUri = newOutputUri(); VariantStorageEngine variantStorageManager = getVariantStorageEngine(); InputStream inputStream = FileUtils.newInputStream(Paths.get(smallInputUri)); System.setIn(inputStream); ObjectMap options = variantStorageManager.getConfiguration() .getStorageEngine(variantStorageManager.getStorageEngineId()).getVariant().getOptions(); options.append(VariantStorageEngine.Options.STDIN.key(), true); URI inputFile = Paths.get(smallInputUri).getFileName().toUri(); System.out.println("inputFile = " + inputFile); assertFalse("File should not exist in that specific location", Paths.get(inputFile).toFile().exists()); StoragePipelineResult storagePipelineResult = variantStorageManager.index(Collections.singletonList(inputFile), outputUri, true, true, false).get(0); assertEquals(999, countLinesFromAvro(Paths.get(storagePipelineResult.getTransformResult()).toFile())); }
@Test public void transformToAvroSTDOUT() throws Exception { URI outputUri = newOutputUri(); VariantStorageEngine variantStorageManager = getVariantStorageEngine(); File outputFile = Paths.get(outputUri.resolve("outputFile.avro")).toFile(); try (PrintStream os = new PrintStream(new FileOutputStream(outputFile))) { System.setOut(os); ObjectMap options = variantStorageManager.getConfiguration() .getStorageEngine(variantStorageManager.getStorageEngineId()).getVariant().getOptions(); options.append(VariantStorageEngine.Options.STDOUT.key(), true); options.append(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "avro"); variantStorageManager.index(Collections.singletonList(smallInputUri), outputUri, true, true, false).get(0); } finally { System.setOut(out); } assertEquals(999, countLinesFromAvro(outputFile)); }
public VariantMongoDBAdaptor(MongoDataStoreManager mongoManager, MongoCredentials credentials, String variantsCollectionName, StudyConfigurationManager studyConfigurationManager, StorageConfiguration storageConfiguration) throws UnknownHostException { // MongoDB configuration this.closeConnection = false; this.credentials = credentials; this.mongoManager = mongoManager; db = mongoManager.get(credentials.getMongoDbName(), credentials.getMongoDBConfiguration()); collectionName = variantsCollectionName; variantsCollection = db.getCollection(collectionName); this.studyConfigurationManager = studyConfigurationManager; this.storageConfiguration = storageConfiguration; StorageEngineConfiguration storageEngineConfiguration = storageConfiguration.getStorageEngine(MongoDBVariantStorageEngine.STORAGE_ENGINE_ID); this.configuration = storageEngineConfiguration == null || storageEngineConfiguration.getVariant().getOptions() == null ? new ObjectMap() : storageEngineConfiguration.getVariant().getOptions(); queryParser = new VariantMongoDBQueryParser(studyConfigurationManager); NUMBER_INSTANCES.incrementAndGet(); }
@Override default HadoopVariantStorageEngine getVariantStorageEngine() throws Exception { synchronized (manager) { if (manager.get() == null) { manager.set(new HadoopVariantStorageEngine()); } } HadoopVariantStorageEngine manager = HadoopVariantStorageTest.manager.get(); //Make a copy of the configuration Configuration conf = new Configuration(false); HBaseConfiguration.merge(conf, HadoopVariantStorageTest.configuration.get()); StorageConfiguration storageConfiguration = getStorageConfiguration(conf); storageConfiguration.getStorageEngine(HadoopVariantStorageEngine.STORAGE_ENGINE_ID) .getVariant() .getOptions() .putAll(getOtherStorageConfigurationOptions()); manager.setConfiguration(storageConfiguration, HadoopVariantStorageEngine.STORAGE_ENGINE_ID, VariantStorageBaseTest.DB_NAME); manager.mrExecutor = new TestMRExecutor(conf); manager.conf = conf; return manager; }
@Test public void testMultipleFilesProtoConcurrent() throws Exception { List<URI> protoFiles = new LinkedList<>(); HadoopVariantStorageEngine variantStorageManager = getVariantStorageEngine(); ObjectMap options = variantStorageManager.getConfiguration().getStorageEngine(variantStorageManager.getStorageEngineId()).getVariant().getOptions(); options.put(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "proto"); options.put(VariantStorageEngine.Options.STUDY.key(), STUDY_NAME); List<URI> inputFiles = new LinkedList<>(); // for (int fileId = 12877; fileId <= 12893; fileId++) { for (int fileId = 12877; fileId <= 12879; fileId++) { String fileName = "platinum/1K.end.platinum-genomes-vcf-NA" + fileId + "_S1.genome.vcf.gz"; // inputFiles.add(getResourceUri(fileName)); List<StoragePipelineResult> results = variantStorageManager.index(Collections.singletonList(getResourceUri(fileName)), outputUri, true, true, false); protoFiles.add(results.get(0).getTransformResult()); } // dbAdaptor.getStudyConfigurationManager().updateStudyConfiguration(studyConfiguration, null); protoFiles = protoFiles.subList(0,2); // TODO remove List<StoragePipelineResult> index2 = variantStorageManager.index(protoFiles, outputUri, false, false, true); System.out.println(index2); }
@Test public void testVariantFactory() throws Exception { ObjectMap options = new ObjectMap(VariantAnnotationManager.ANNOTATOR, VariantAnnotatorFactory.AnnotationSource.CELLBASE_DB_ADAPTOR); VariantAnnotator variantAnnotator = VariantAnnotatorFactory.buildVariantAnnotator(storageConfiguration, storageConfiguration.getDefaultStorageEngineId(), projectMetadata, options); assertThat(variantAnnotator, is(instanceOf(CellBaseDirectVariantAnnotator.class))); storageConfiguration.getStorageEngine().getVariant().getOptions().put(VariantAnnotationManager.ANNOTATOR, VariantAnnotatorFactory.AnnotationSource.CELLBASE_DB_ADAPTOR.toString()); options = new ObjectMap(); variantAnnotator = VariantAnnotatorFactory.buildVariantAnnotator(storageConfiguration, storageConfiguration.getDefaultStorageEngineId(), projectMetadata, options); assertThat(variantAnnotator, is(instanceOf(CellBaseDirectVariantAnnotator.class))); options = new ObjectMap(VariantAnnotationManager.ANNOTATOR, null); variantAnnotator = VariantAnnotatorFactory.buildVariantAnnotator(storageConfiguration, storageConfiguration.getDefaultStorageEngineId(), projectMetadata, options); assertThat(variantAnnotator, is(instanceOf(CellBaseDirectVariantAnnotator.class))); storageConfiguration.getStorageEngine().getVariant().getOptions().put(VariantAnnotationManager.ANNOTATOR, VariantAnnotatorFactory.AnnotationSource.CELLBASE_REST); options = new ObjectMap(VariantAnnotationManager.ANNOTATOR, null); variantAnnotator = VariantAnnotatorFactory.buildVariantAnnotator(storageConfiguration, storageConfiguration.getDefaultStorageEngineId(), projectMetadata, options); assertThat(variantAnnotator, is(instanceOf(CellBaseRestVariantAnnotator.class))); storageConfiguration.getStorageEngine().getVariant().getOptions().remove(VariantAnnotationManager.ANNOTATOR); options = new ObjectMap(); variantAnnotator = VariantAnnotatorFactory.buildVariantAnnotator(storageConfiguration, storageConfiguration.getDefaultStorageEngineId(), projectMetadata, options); assertThat(variantAnnotator, is(instanceOf(CellBaseRestVariantAnnotator.class))); options = new ObjectMap(VariantAnnotationManager.ANNOTATOR, VariantAnnotatorFactory.AnnotationSource.OTHER.toString()) .append(VariantAnnotationManager.VARIANT_ANNOTATOR_CLASSNAME, TestCellBaseRestVariantAnnotator.class.getName()); variantAnnotator = VariantAnnotatorFactory.buildVariantAnnotator(storageConfiguration, storageConfiguration.getDefaultStorageEngineId(), projectMetadata, options); assertThat(variantAnnotator, is(instanceOf(TestCellBaseRestVariantAnnotator.class))); }
public static StoragePipelineResult runETL(VariantStorageEngine variantStorageManager, URI inputUri, URI outputUri, ObjectMap params, boolean doExtract, boolean doTransform, boolean doLoad) throws IOException, FileFormatException, StorageEngineException { variantStorageManager.getConfiguration() .getStorageEngine(variantStorageManager.getStorageEngineId()).getVariant().getOptions().putAll(params); StoragePipelineResult storagePipelineResult = variantStorageManager.index(Collections.singletonList(inputUri), outputUri, doExtract, doTransform, doLoad).get(0); checkFileExists(storagePipelineResult.getExtractResult()); checkFileExists(storagePipelineResult.getPreTransformResult()); checkFileExists(storagePipelineResult.getTransformResult()); checkFileExists(storagePipelineResult.getPostTransformResult()); checkFileExists(storagePipelineResult.getPreLoadResult()); checkFileExists(storagePipelineResult.getLoadResult()); checkFileExists(storagePipelineResult.getPostLoadResult()); return storagePipelineResult; }
@Test public void testTwoFilesConcurrent() throws Exception { StudyConfiguration studyConfiguration = VariantStorageBaseTest.newStudyConfiguration(); HadoopVariantStorageEngine variantStorageManager = getVariantStorageEngine(); ObjectMap options = variantStorageManager.getConfiguration().getStorageEngine(variantStorageManager.getStorageEngineId()).getVariant().getOptions(); options.put(HadoopVariantStorageEngine.HADOOP_LOAD_DIRECT, true); options.put(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "proto"); options.put(VariantStorageEngine.Options.STUDY.key(), studyConfiguration.getStudyName()); List<URI> inputFiles = Arrays.asList(getResourceUri("s1.genome.vcf"), getResourceUri("s2.genome.vcf")); List<StoragePipelineResult> index = variantStorageManager.index(inputFiles, outputUri, true, true, true); VariantHadoopDBAdaptor dbAdaptor = variantStorageManager.getDBAdaptor(); studyConfiguration = dbAdaptor.getStudyConfigurationManager().getStudyConfiguration(studyConfiguration.getStudyId(), null).first(); for (StoragePipelineResult storagePipelineResult : index) { System.out.println(storagePipelineResult); } printVariants(studyConfiguration, dbAdaptor, newOutputUri()); for (Variant variant : dbAdaptor) { System.out.println("variant = " + variant); } checkLoadedFilesS1S2(studyConfiguration, dbAdaptor); assertThat(studyConfiguration.getIndexedFiles(), hasItems(1, 2)); }
private StudyConfiguration load(ObjectMap extraParams, List<URI> inputFiles) throws Exception { StudyConfiguration studyConfiguration = VariantStorageBaseTest.newStudyConfiguration(); HadoopVariantStorageEngine variantStorageManager = getVariantStorageEngine(); VariantHadoopDBAdaptor dbAdaptor = variantStorageManager.getDBAdaptor(); ObjectMap options = variantStorageManager.getConfiguration().getStorageEngine(variantStorageManager.getStorageEngineId()).getVariant().getOptions(); options.put(VariantStorageEngine.Options.STUDY.key(), studyConfiguration.getStudyName()); options.put(VariantStorageEngine.Options.GVCF.key(), true); options.put(HadoopVariantStorageEngine.VARIANT_TABLE_INDEXES_SKIP, true); options.put(HadoopVariantStorageEngine.HADOOP_LOAD_ARCHIVE_BATCH_SIZE, 1); options.put(VariantStorageEngine.Options.MERGE_MODE.key(), VariantStorageEngine.MergeMode.BASIC); options.putAll(extraParams); List<StoragePipelineResult> index = variantStorageManager.index(inputFiles, outputUri, true, true, true); for (StoragePipelineResult storagePipelineResult : index) { System.out.println(storagePipelineResult); } URI outputUri = newOutputUri(1); studyConfiguration = dbAdaptor.getStudyConfigurationManager().getStudyConfiguration(studyConfiguration.getStudyId(), null).first(); printVariants(studyConfiguration, dbAdaptor, outputUri); return studyConfiguration; }
public static void removeFile(HadoopVariantStorageEngine variantStorageManager, String dbName, int fileId, StudyConfiguration studyConfiguration, Map<? extends String, ?> otherParams) throws Exception { ObjectMap params = new ObjectMap() .append(VariantStorageEngine.Options.STUDY.key(), studyConfiguration.getStudyName()) .append(VariantStorageEngine.Options.DB_NAME.key(), dbName); if (otherParams != null) { params.putAll(otherParams); } variantStorageManager.getConfiguration().getStorageEngine(variantStorageManager.getStorageEngineId()).getVariant().getOptions() .putAll(params); variantStorageManager.removeFile(studyConfiguration.getStudyName(), fileId); studyConfiguration.copy( variantStorageManager .getDBAdaptor() .getStudyConfigurationManager() .getStudyConfiguration(studyConfiguration.getStudyId(), null) .first()); // return variantStorageManager.readVariantSource(etlResult.getTransformResult(), new ObjectMap()); }