private List<String> buildFormat(StudyConfiguration studyConfiguration) { List<String> format = new LinkedList<>(); if (!excludeGenotypes) { format.add(VariantMerger.GT_KEY); } format.addAll(studyConfiguration.getAttributes().getAsStringList(VariantStorageEngine.Options.EXTRA_GENOTYPE_FIELDS.key())); return format; }
public static MergeMode from(ObjectMap options) { String mergeModeStr = options.getString(Options.MERGE_MODE.key(), Options.MERGE_MODE.defaultValue().toString()); return MergeMode.valueOf(mergeModeStr.toUpperCase()); } }
@Test public void corruptedTransformTest() throws Exception { ObjectMap params = new ObjectMap(VariantStorageEngine.Options.TRANSFORM_FAIL_ON_MALFORMED_VARIANT.key(), false); URI outputUri = newOutputUri(); StoragePipelineResult result = runETL(getVariantStorageEngine(), corruptedInputUri, outputUri, params, true, true, false); String[] malformedFiles = Paths.get(outputUri).toFile().list((dir, name) -> name.contains(MALFORMED_FILE)); assertEquals(1, malformedFiles.length); assertEquals(2, result.getTransformStats().getInt("malformed lines")); }
protected void setDefaultTimeout(QueryOptions options) { int defaultTimeout = getOptions().getInt(DEFAULT_TIMEOUT.key(), DEFAULT_TIMEOUT.defaultValue()); int maxTimeout = getOptions().getInt(MAX_TIMEOUT.key(), MAX_TIMEOUT.defaultValue()); int timeout = options.getInt(QueryOptions.TIMEOUT, defaultTimeout); if (timeout > maxTimeout) { throw new VariantQueryException("Invalid timeout '" + timeout + "'. Max timeout is " + maxTimeout); } else if (timeout < 0) { throw new VariantQueryException("Invalid timeout '" + timeout + "'. Timeout must be positive"); } options.put(QueryOptions.TIMEOUT, timeout); }
@Test public void testCalculateAggregatedStatsNonAggregatedStudy() throws Exception { beforeAggregated("variant-test-aggregated-file.vcf.gz", null); calculateAggregatedStats(new QueryOptions(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), Aggregation.BASIC)); Study study = catalogManager.getStudyManager().get(studyId, null, sessionId).first(); String agg = study.getAttributes().get(VariantStorageEngine.Options.AGGREGATED_TYPE.key()).toString(); assertNotNull(agg); assertEquals(Aggregation.BASIC.toString(), agg); }
@Before public void setUp() throws Exception { runDefaultETL(inputUri, variantStorageEngine, newStudyConfiguration(), new QueryOptions() // .append(VariantStorageEngine.Options.EXTRA_GENOTYPE_FIELDS.key(), "GL,DS") .append(VariantStorageEngine.Options.ANNOTATE.key(), false)); }
@Override public Map<String, ?> getOtherStorageConfigurationOptions() { return new ObjectMap(HadoopVariantStorageEngine.VARIANT_TABLE_INDEXES_SKIP, true) .append(HadoopVariantStorageEngine.STATS_LOCAL, true) .append(VariantStorageEngine.Options.MERGE_MODE.key(), VariantStorageEngine.MergeMode.BASIC); } }
@Before public void setUp() throws Exception { clearDB(DB_NAME); ObjectMap params = new ObjectMap(VariantStorageEngine.Options.CALCULATE_STATS.key(), false) .append(VariantStorageEngine.Options.ANNOTATE.key(), false); runDefaultETL(inputUri, variantStorageEngine, newStudyConfiguration(), params); projectMetadata = new ProjectMetadata("hsapiens", "grch37", 1); }
@Override public URI postTransform(URI input) throws IOException, FileFormatException { // Delete isolated storage configuration if (options.getBoolean(Options.ISOLATE_FILE_FROM_STUDY_CONFIGURATION.key())) { privateStudyConfiguration = null; } return input; }
@Override @Before public void before() throws Exception { studyConfiguration = newStudyConfiguration(); studyConfiguration.setAggregation(getAggregationType()); clearDB(DB_NAME); inputUri = getInputUri(); runDefaultETL(inputUri, getVariantStorageEngine(), studyConfiguration, new ObjectMap(VariantStorageEngine.Options.ANNOTATE.key(), false) .append(VariantStorageEngine.Options.CALCULATE_STATS.key(), false)); dbAdaptor = getVariantStorageEngine().getDBAdaptor(); }
@Override protected void securePreLoad(StudyConfiguration studyConfiguration, VariantFileMetadata fileMetadata) throws StorageEngineException { super.securePreLoad(studyConfiguration, fileMetadata); MergeMode mergeMode; if (!studyConfiguration.getAttributes().containsKey(Options.MERGE_MODE.key())) { mergeMode = MergeMode.from(options); studyConfiguration.getAttributes().put(Options.MERGE_MODE.key(), mergeMode); } else { options.put(MERGE_MODE.key(), MergeMode.from(studyConfiguration.getAttributes())); } }
@Test public void testCalculateAggregatedExacStatsWrongExplicitCohorts() throws Exception { beforeAggregated("exachead.vcf.gz", Aggregation.EXAC); String tagMap = getResourceUri("exac-tag-mapping.properties").getPath(); QueryOptions options = new QueryOptions(VariantStorageEngine.Options.AGGREGATION_MAPPING_PROPERTIES.key(), tagMap); thrown.expectMessage(VariantStatsStorageOperation.differentCohortsThanMappingFile().getMessage()); calculateStats(options, Arrays.asList("AFR", "ALL")); }
@Before public void setUp() throws Exception { studyConfiguration = newStudyConfiguration(); runDefaultETL(smallInputUri, variantStorageEngine, studyConfiguration, new ObjectMap(VariantStorageEngine.Options.EXTRA_GENOTYPE_FIELDS.key(), "GL,DS")); }
@Override public ObjectMap getOptions() { return new ObjectMap(VariantStorageEngine.Options.MERGE_MODE.key(), VariantStorageEngine.MergeMode.BASIC); } }
@Test public void testMultipleFilesConcurrentMergeBasic() throws Exception { testMultipleFilesConcurrent(new ObjectMap(VariantStorageEngine.Options.MERGE_MODE.key(), VariantStorageEngine.MergeMode.BASIC) .append(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "avro") .append(HadoopVariantStorageEngine.HADOOP_LOAD_VARIANT_BATCH_SIZE, 5) .append(HadoopVariantStorageEngine.HADOOP_LOAD_ARCHIVE_BATCH_SIZE, 5)); }
@Override public Map<String, ?> getOtherStorageConfigurationOptions() { return new ObjectMap(HadoopVariantStorageEngine.VARIANT_TABLE_INDEXES_SKIP, true) .append(VariantStorageEngine.Options.MERGE_MODE.key(), VariantStorageEngine.MergeMode.BASIC) .append(HadoopVariantStorageEngine.STATS_LOCAL, false); } }
private void fillMissing() throws StorageEngineException { StorageVariantCommandOptions.FillMissingCommandOptions cliOptions = variantCommandOptions.fillMissingCommandOptions; ObjectMap options = storageConfiguration.getVariant().getOptions(); options.put(VariantStorageEngine.Options.RESUME.key(), cliOptions.resume); options.putAll(cliOptions.commonOptions.params); variantStorageEngine.fillMissing(cliOptions.study, options, cliOptions.overwrite); }
private void fillGaps() throws StorageEngineException { StorageVariantCommandOptions.FillGapsCommandOptions fillGapsCommandOptions = variantCommandOptions.fillGapsCommandOptions; ObjectMap options = storageConfiguration.getVariant().getOptions(); options.put(VariantStorageEngine.Options.RESUME.key(), fillGapsCommandOptions.resume); options.putAll(fillGapsCommandOptions.commonOptions.params); variantStorageEngine.fillGaps(fillGapsCommandOptions.study, fillGapsCommandOptions.samples, options); }
@Override public ObjectMap getExtraOptions() { return new ObjectMap(VariantStorageEngine.Options.MERGE_MODE.key(), VariantStorageEngine.MergeMode.BASIC); } }
@Override @Before public void before() throws Exception { studyConfiguration = newStudyConfiguration(); clearDB(DB_NAME); runDefaultETL(inputUri, getVariantStorageEngine(), studyConfiguration, new ObjectMap(VariantStorageEngine.Options.ANNOTATE.key(), false)); dbAdaptor = getVariantStorageEngine().getDBAdaptor(); }