/** * Get fixed format for the VARCHAR ARRAY sample columns. * @param studyConfiguration StudyConfiguration * @return List of fixed formats */ public static List<String> getFixedFormat(StudyConfiguration studyConfiguration) { List<String> format; List<String> extraFields = studyConfiguration.getAttributes().getAsStringList(Options.EXTRA_GENOTYPE_FIELDS.key()); if (extraFields.isEmpty()) { extraFields = Collections.singletonList(VariantMerger.GENOTYPE_FILTER_KEY); } boolean excludeGenotypes = studyConfiguration.getAttributes() .getBoolean(Options.EXCLUDE_GENOTYPES.key(), Options.EXCLUDE_GENOTYPES.defaultValue()); if (excludeGenotypes) { format = extraFields; } else { format = new ArrayList<>(1 + extraFields.size()); format.add(VariantMerger.GT_KEY); format.addAll(extraFields); } return format; }
public static MergeMode from(ObjectMap options) { String mergeModeStr = options.getString(Options.MERGE_MODE.key(), Options.MERGE_MODE.defaultValue().toString()); return MergeMode.valueOf(mergeModeStr.toUpperCase()); } }
@Override protected void securePreLoad(StudyConfiguration studyConfiguration, VariantFileMetadata fileMetadata) throws StorageEngineException { super.securePreLoad(studyConfiguration, fileMetadata); if (options.getBoolean(VariantStorageEngine.Options.LOAD_SPLIT_DATA.key(), VariantStorageEngine.Options.LOAD_SPLIT_DATA.defaultValue())) { throw new StorageEngineException("Unable to load split data in " + STORAGE_ENGINE_ID); } final AtomicInteger ongoingLoads = new AtomicInteger(1); // this boolean resume = options.getBoolean(VariantStorageEngine.Options.RESUME.key(), VariantStorageEngine.Options.RESUME.defaultValue()); List<Integer> fileIds = Collections.singletonList(getFileId()); StudyConfigurationManager.addBatchOperation(studyConfiguration, OPERATION_NAME, fileIds, resume, BatchFileOperation.Type.LOAD, operation -> { if (operation.getOperationName().equals(OPERATION_NAME)) { if (operation.currentStatus().equals(BatchFileOperation.Status.ERROR)) { Integer fileId = operation.getFileIds().get(0); String fileName = studyConfiguration.getFileIds().inverse().get(fileId); logger.warn("Pending load operation for file " + fileName + " (" + fileId + ')'); } else { ongoingLoads.incrementAndGet(); } return true; } else { return false; } }); if (ongoingLoads.get() > 1) { logger.info("There are " + ongoingLoads.get() + " concurrent load operations"); } }
AtomicReference<BatchFileOperation> batchFileOperation = new AtomicReference<>(); int studyId = scm.lockAndUpdate(studyName, studyConfiguration -> { boolean resume = getOptions().getBoolean(RESUME.key(), RESUME.defaultValue()); batchFileOperation.set(StudyConfigurationManager.addBatchOperation( studyConfiguration,
VariantStorageEngine.Options.UPDATE_STATS.defaultValue()); boolean overwrite = options.getBoolean(VariantStorageEngine.Options.OVERWRITE_STATS.key(), VariantStorageEngine.Options.OVERWRITE_STATS.defaultValue()); Query query = VariantStatisticsManager.buildInputQuery(readStudyConfiguration(), cohorts, overwrite, updateStats, options); QueryOptions queryOptions = VariantStatisticsManager.buildIncludeExclude();
throws StoragePipelineException { if (files != null && !files.isEmpty() && options != null && options.getBoolean(Options.CALCULATE_STATS.key(), Options.CALCULATE_STATS.defaultValue())) {
} else { sampling = options.getInt(APPROXIMATE_COUNT_SAMPLING_SIZE.key(), getOptions().getInt(APPROXIMATE_COUNT_SAMPLING_SIZE.key(), APPROXIMATE_COUNT_SAMPLING_SIZE.defaultValue())); QueryOptions queryOptions = new QueryOptions(QueryOptions.INCLUDE, VariantField.ID).append(QueryOptions.LIMIT, sampling);
boolean active = searchActiveAndAlive(); if (!getOptions().getBoolean(INTERSECT_ACTIVE.key(), INTERSECT_ACTIVE.defaultValue()) || useSearchIndex.equals(UseSearchIndex.NO)) { } else if (getOptions().getBoolean(INTERSECT_ALWAYS.key(), INTERSECT_ALWAYS.defaultValue())) { int intersectParamsThreshold = getOptions().getInt(INTERSECT_PARAMS_THRESHOLD.key(), INTERSECT_PARAMS_THRESHOLD.defaultValue()); intersect = coveredParams.size() >= intersectParamsThreshold;
boolean resume = getOptions().getBoolean(RESUME.key(), RESUME.defaultValue()); id.set(getStudyConfigurationManager().registerSearchIndexSamples(studyConfiguration, samples, resume)); return studyConfiguration;
if (files != null && !files.isEmpty() && options.getBoolean(Options.ANNOTATE.key(), Options.ANNOTATE.defaultValue())) { try { VariantDBAdaptor dbAdaptor = getDBAdaptor();
@Override public URI preTransform(URI input) throws StorageEngineException, IOException, FileFormatException { String fileName = VariantReaderUtils.getFileName(input); String study = options.getString(Options.STUDY.key()); boolean isolate = options.getBoolean(Options.ISOLATE_FILE_FROM_STUDY_CONFIGURATION.key(), Options.ISOLATE_FILE_FROM_STUDY_CONFIGURATION.defaultValue()); StudyConfiguration studyConfiguration; if (isolate) { logger.debug("Isolated study configuration"); studyConfiguration = new StudyConfiguration(-1, "unknown", -1, fileName); studyConfiguration.setAggregationStr(options.getString(Options.AGGREGATED_TYPE.key(), Options.AGGREGATED_TYPE.defaultValue().toString())); options.put(Options.ISOLATE_FILE_FROM_STUDY_CONFIGURATION.key(), true); } else { StudyConfigurationManager scm = dbAdaptor.getStudyConfigurationManager(); checkExistsStudyConfiguration(null); studyConfiguration = scm.lockAndUpdate(study, existingStudyConfiguration -> { if (existingStudyConfiguration.getAggregation() == null) { existingStudyConfiguration.setAggregationStr(options.getString(Options.AGGREGATED_TYPE.key(), Options.AGGREGATED_TYPE.defaultValue().toString())); } setFileId(scm.registerFile(existingStudyConfiguration, input.getPath())); return existingStudyConfiguration; }); } privateStudyConfiguration = studyConfiguration; return input; }
public QueryResult<ProjectMetadata> getProjectMetadata(ObjectMap options) throws StorageEngineException { QueryResult<ProjectMetadata> queryResult = getProjectMetadata(); ProjectMetadata projectMetadata = queryResult.first(); if (options != null && (projectMetadata == null || StringUtils.isEmpty(projectMetadata.getSpecies()) && options.containsKey(VariantAnnotationManager.SPECIES) || StringUtils.isEmpty(projectMetadata.getAssembly()) && options.containsKey(VariantAnnotationManager.ASSEMBLY))) { projectMetadata = lockAndUpdateProject(pm -> { if (pm == null) { pm = new ProjectMetadata(); } if (pm.getRelease() <= 0) { pm.setRelease(options.getInt(VariantStorageEngine.Options.RELEASE.key(), VariantStorageEngine.Options.RELEASE.defaultValue())); } if (StringUtils.isEmpty(pm.getSpecies())) { pm.setSpecies(toCellBaseSpeciesName(options.getString(VariantAnnotationManager.SPECIES))); } if (StringUtils.isEmpty(pm.getAssembly())) { pm.setAssembly(options.getString(VariantAnnotationManager.ASSEMBLY)); } return pm; }); queryResult.setResult(Collections.singletonList(projectMetadata)); queryResult.setNumResults(1); queryResult.setNumTotalResults(1); } return queryResult; }
@Override public URI preTransform(URI input) throws StorageEngineException, IOException, FileFormatException { logger.info("PreTransform: " + input); // ObjectMap options = configuration.getStorageEngine(STORAGE_ENGINE_ID).getVariant().getOptions(); if (!options.containsKey(VariantStorageEngine.Options.TRANSFORM_FORMAT.key())) { options.put(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), VariantStorageEngine.Options.TRANSFORM_FORMAT.defaultValue()); } String transVal = options.getString(VariantStorageEngine.Options.TRANSFORM_FORMAT.key()); switch (transVal){ case "avro": case "proto": break; default: throw new IllegalArgumentException("Output format " + transVal + " not supported for Hadoop!"); } // non gVCF files are supported. Don't force all files to be gVCF // if (!options.containsKey(VariantStorageEngine.Options.GVCF.key())) { // options.put(VariantStorageEngine.Options.GVCF.key(), true); // } // boolean isGvcf = options.getBoolean(VariantStorageEngine.Options.GVCF.key()); // if (!isGvcf) { // throw new NotImplementedException("Only GVCF format supported!!!"); // } return super.preTransform(input); }
/** * Atomically updates the studyConfiguration before removing samples. * * @param study Study * @param files Files to remove * @return FileIds to remove * @throws StorageEngineException StorageEngineException */ protected BatchFileOperation preRemoveFiles(String study, List<String> files) throws StorageEngineException { List<Integer> fileIds = new ArrayList<>(); AtomicReference<BatchFileOperation> batchFileOperation = new AtomicReference<>(); getStudyConfigurationManager().lockAndUpdate(study, studyConfiguration -> { fileIds.addAll(getStudyConfigurationManager().getFileIdsFromStudy(files, studyConfiguration)); boolean resume = getOptions().getBoolean(RESUME.key(), RESUME.defaultValue()); batchFileOperation.set(addBatchOperation(studyConfiguration, REMOVE_OPERATION_NAME, fileIds, resume, BatchFileOperation.Type.REMOVE)); if (!studyConfiguration.getIndexedFiles().containsAll(fileIds)) { // Remove indexed files to get non indexed files fileIds.removeAll(studyConfiguration.getIndexedFiles()); throw new StorageEngineException("Unable to remove non indexed files: " + fileIds); } return studyConfiguration; }); return batchFileOperation.get(); }
public static boolean isResume(ObjectMap options) { return options.getBoolean(Options.RESUME.key(), Options.RESUME.defaultValue()); }
protected StudyConfiguration checkExistsStudyConfiguration(StudyConfiguration studyConfiguration) throws StorageEngineException { if (studyConfiguration == null) { studyConfiguration = getStudyConfiguration(); if (studyConfiguration == null) { String studyName = options.getString(Options.STUDY.key(), Options.STUDY.defaultValue()); logger.info("Creating a new StudyConfiguration '{}'", studyName); studyConfiguration = getStudyConfigurationManager().createStudy(studyName); } } privateStudyConfiguration = studyConfiguration; setStudyId(studyConfiguration.getStudyId()); return studyConfiguration; }
public static boolean getExcludeGenotypes(StudyConfiguration studyConfiguration) { return studyConfiguration.getAttributes().getBoolean(VariantStorageEngine.Options.EXCLUDE_GENOTYPES.key(), VariantStorageEngine.Options.EXCLUDE_GENOTYPES.defaultValue()); } }
protected void setDefaultTimeout(QueryOptions options) { int defaultTimeout = getOptions().getInt(DEFAULT_TIMEOUT.key(), DEFAULT_TIMEOUT.defaultValue()); int maxTimeout = getOptions().getInt(MAX_TIMEOUT.key(), MAX_TIMEOUT.defaultValue()); int timeout = options.getInt(QueryOptions.TIMEOUT, defaultTimeout); if (timeout > maxTimeout) { throw new VariantQueryException("Invalid timeout '" + timeout + "'. Max timeout is " + maxTimeout); } else if (timeout < 0) { throw new VariantQueryException("Invalid timeout '" + timeout + "'. Timeout must be positive"); } options.put(QueryOptions.TIMEOUT, timeout); }
protected VariantFileMetadata createEmptyVariantFileMetadata(Path input) { VariantFileMetadata fileMetadata = VariantReaderUtils.createEmptyVariantFileMetadata(input); int fileId; if (options.getBoolean(Options.ISOLATE_FILE_FROM_STUDY_CONFIGURATION.key(), Options.ISOLATE_FILE_FROM_STUDY_CONFIGURATION .defaultValue())) { fileId = -1; } else { fileId = getFileId(); } return fileMetadata.setId(Integer.toString(fileId)); }
MongoCredentials getMongoCredentials() { // If no database name is provided, read from the configuration file if (StringUtils.isEmpty(dbName)) { dbName = getOptions().getString(DB_NAME.key(), DB_NAME.defaultValue()); } DatabaseCredentials database = configuration.getStorageEngine(STORAGE_ENGINE_ID).getVariant().getDatabase(); try { return new MongoCredentials(database, dbName); } catch (IllegalOpenCGACredentialsException e) { throw Throwables.propagate(e); } }