public ObjectMap getOptions() { return configuration.getStorageEngine(storageEngineId).getVariant().getOptions(); }
private HBaseCredentials buildCredentials(String table) throws StorageEngineException { StorageEtlConfiguration vStore = configuration.getStorageEngine(STORAGE_ENGINE_ID).getVariant(); DatabaseCredentials db = vStore.getDatabase(); String user = db.getUser(); String pass = db.getPassword();
switch (type) { case ALIGNMENT: clazz = this.storageConfiguration.getStorageEngine(storageEngineId).getAlignment().getManager(); break; case VARIANT: clazz = this.storageConfiguration.getStorageEngine(storageEngineId).getVariant().getManager(); break; default:
static StorageConfiguration updateStorageConfiguration(StorageConfiguration storageConfiguration, Configuration conf) throws IOException { storageConfiguration.setDefaultStorageEngineId(HadoopVariantStorageEngine.STORAGE_ENGINE_ID); StorageEtlConfiguration variantConfiguration = storageConfiguration.getStorageEngine(HadoopVariantStorageEngine.STORAGE_ENGINE_ID).getVariant(); ObjectMap options = variantConfiguration.getOptions(); options.put(VariantStorageEngine.Options.MERGE_MODE.key(), VariantStorageEngine.MergeMode.BASIC); variantConfiguration.getDatabase().setHosts(Collections.singletonList("hbase://" + HadoopVariantStorageTest.configuration.get().get(HConstants.ZOOKEEPER_QUORUM))); return storageConfiguration;
new StorageEtlConfiguration("org.opencb.opencga.storage.mongodb.alignment.MongoDBAlignmentStorageManager", new ObjectMap (), new DatabaseCredentials(Arrays.asList("mongodb-dev:27017"), "user", "password")), new StorageEtlConfiguration("org.opencb.opencga.storage.mongodb.alignment.MongoDBVariantStorageManager", new ObjectMap(), new DatabaseCredentials(Arrays.asList("mongodb-dev:27017"), "user", "password")), options); new StorageEtlConfiguration("org.opencb.opencga.storage.hadoop.alignment.HadoopAlignmentStorageManager", new ObjectMap(), new DatabaseCredentials(Arrays.asList("who-master:60000"), "user", "password")), new StorageEtlConfiguration("org.opencb.opencga.storage.hadoop.alignment.HadoopVariantStorageManager", new ObjectMap(), new DatabaseCredentials(Arrays.asList("who-master:60000"), "user", "password")), options);
storageConfiguration.getStorageEngines().add(new StorageEngineConfiguration( STORAGE_ENGINE_DUMMY, new StorageEtlConfiguration(), new StorageEtlConfiguration(DummyVariantStorageEngine.class.getName(), new ObjectMap(), new DatabaseCredentials()), new ObjectMap() ));
private void annotationDelete() throws VariantAnnotatorException, StorageEngineException { StorageVariantCommandOptions.AnnotationDeleteCommandOptions cliOptions = variantCommandOptions.annotationDeleteCommandOptions; ObjectMap options = storageConfiguration.getVariant().getOptions(); options.putAll(cliOptions.commonOptions.params); variantStorageEngine.deleteAnnotation(cliOptions.annotationId, options); }
MongoCredentials getMongoCredentials() { // If no database name is provided, read from the configuration file if (StringUtils.isEmpty(dbName)) { dbName = getOptions().getString(DB_NAME.key(), DB_NAME.defaultValue()); } DatabaseCredentials database = configuration.getStorageEngine(STORAGE_ENGINE_ID).getVariant().getDatabase(); try { return new MongoCredentials(database, dbName); } catch (IllegalOpenCGACredentialsException e) { throw Throwables.propagate(e); } }
private void annotationSave() throws VariantAnnotatorException, StorageEngineException { StorageVariantCommandOptions.AnnotationSaveCommandOptions cliOptions = variantCommandOptions.annotationSaveCommandOptions; ObjectMap options = storageConfiguration.getVariant().getOptions(); options.putAll(cliOptions.commonOptions.params); variantStorageEngine.saveAnnotation(cliOptions.annotationId, options); }
@Test public void transformIsolated() throws Exception { ObjectMap params = new ObjectMap(); URI outputUri = newOutputUri(); VariantStorageEngine variantStorageManager = getVariantStorageEngine(); variantStorageManager.getConfiguration().getStorageEngine(variantStorageManager.getStorageEngineId()).getVariant().getDatabase() .setHosts(Collections.singletonList("1.1.1.1")); StoragePipelineResult etlResult = runETL(variantStorageManager, smallInputUri, outputUri, params, true, true, false); System.out.println("etlResult = " + etlResult); String[] malformedFiles = Paths.get(outputUri).toFile().list((dir, name) -> name.contains(MALFORMED_FILE)); assertEquals(0, malformedFiles.length); }
public VariantStoragePipeline(StorageConfiguration configuration, String storageEngineId, VariantDBAdaptor dbAdaptor, VariantReaderUtils variantReaderUtils) { this(configuration, storageEngineId, dbAdaptor, variantReaderUtils, new ObjectMap(configuration.getStorageEngine(storageEngineId).getVariant().getOptions())); }
StorageVariantCommandOptions.VariantStatsCommandOptions statsVariantsCommandOptions = variantCommandOptions.statsVariantsCommandOptions; QueryOptions options = new QueryOptions(storageConfiguration.getVariant().getOptions()); options.put(VariantStorageEngine.Options.OVERWRITE_STATS.key(), statsVariantsCommandOptions.overwriteStats); options.put(VariantStorageEngine.Options.UPDATE_STATS.key(), statsVariantsCommandOptions.updateStats);
private void fillGaps() throws StorageEngineException { StorageVariantCommandOptions.FillGapsCommandOptions fillGapsCommandOptions = variantCommandOptions.fillGapsCommandOptions; ObjectMap options = storageConfiguration.getVariant().getOptions(); options.put(VariantStorageEngine.Options.RESUME.key(), fillGapsCommandOptions.resume); options.putAll(fillGapsCommandOptions.commonOptions.params); variantStorageEngine.fillGaps(fillGapsCommandOptions.study, fillGapsCommandOptions.samples, options); }
private void fillMissing() throws StorageEngineException { StorageVariantCommandOptions.FillMissingCommandOptions cliOptions = variantCommandOptions.fillMissingCommandOptions; ObjectMap options = storageConfiguration.getVariant().getOptions(); options.put(VariantStorageEngine.Options.RESUME.key(), cliOptions.resume); options.putAll(cliOptions.commonOptions.params); variantStorageEngine.fillMissing(cliOptions.study, options, cliOptions.overwrite); }
public VariantMongoDBAdaptor(MongoDataStoreManager mongoManager, MongoCredentials credentials, String variantsCollectionName, StudyConfigurationManager studyConfigurationManager, StorageConfiguration storageConfiguration) throws UnknownHostException { // MongoDB configuration this.closeConnection = false; this.credentials = credentials; this.mongoManager = mongoManager; db = mongoManager.get(credentials.getMongoDbName(), credentials.getMongoDBConfiguration()); collectionName = variantsCollectionName; variantsCollection = db.getCollection(collectionName); this.studyConfigurationManager = studyConfigurationManager; this.storageConfiguration = storageConfiguration; StorageEngineConfiguration storageEngineConfiguration = storageConfiguration.getStorageEngine(MongoDBVariantStorageEngine.STORAGE_ENGINE_ID); this.configuration = storageEngineConfiguration == null || storageEngineConfiguration.getVariant().getOptions() == null ? new ObjectMap() : storageEngineConfiguration.getVariant().getOptions(); queryParser = new VariantMongoDBQueryParser(studyConfigurationManager); NUMBER_INSTANCES.incrementAndGet(); }
ObjectMap alignmentOptions = storageConfiguration.getAlignment().getOptions();
storageConfiguration.getVariant().getOptions().putAll(variantQueryCommandOptions.commonOptions.params);
public static VariantAnnotator buildVariantAnnotator(StorageConfiguration configuration, String storageEngineId, ProjectMetadata projectMetadata, ObjectMap options) throws VariantAnnotatorException { ObjectMap storageOptions = new ObjectMap(configuration.getStorageEngine(storageEngineId).getVariant().getOptions()); if (options != null) { options.forEach(storageOptions::putIfNotNull);
@Override public void removeFiles(String study, List<String> files) throws StorageEngineException { BatchFileOperation batchFileOperation = preRemoveFiles(study, files); List<Integer> fileIds = batchFileOperation.getFileIds(); ObjectMap options = new ObjectMap(configuration.getStorageEngine(STORAGE_ENGINE_ID).getVariant().getOptions()); StudyConfigurationManager scm = getStudyConfigurationManager(); Integer studyId = scm.getStudyId(study, null); Thread hook = scm.buildShutdownHook(REMOVE_OPERATION_NAME, studyId, fileIds); try { Runtime.getRuntime().addShutdownHook(hook); getDBAdaptor().removeFiles(study, files, batchFileOperation.getTimestamp(), new QueryOptions(options)); postRemoveFiles(study, fileIds, false); } catch (Exception e) { postRemoveFiles(study, fileIds, true); throw e; } finally { Runtime.getRuntime().removeShutdownHook(hook); } }
try { Runtime.getRuntime().addShutdownHook(hook); ObjectMap options = new ObjectMap(configuration.getStorageEngine(STORAGE_ENGINE_ID).getVariant().getOptions()); getDBAdaptor().removeStudy(studyName, batchFileOperation.get().getTimestamp(), new QueryOptions(options));