@Test public void avroBasicIndex() throws Exception { clearDB(DB_NAME); StudyConfiguration studyConfiguration = newStudyConfiguration(); StoragePipelineResult etlResult = runDefaultETL(smallInputUri, variantStorageEngine, studyConfiguration, new ObjectMap(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "avro")); assertTrue("Incorrect transform file extension " + etlResult.getTransformResult() + ". Expected 'variants.avro.gz'", Paths.get(etlResult.getTransformResult()).toFile().getName().endsWith("variants.avro.gz")); assertEquals(1, studyConfiguration.getIndexedFiles().size()); VariantFileMetadata fileMetadata = checkTransformedVariants(etlResult.getTransformResult(), studyConfiguration); checkLoadedVariants(variantStorageEngine.getDBAdaptor(), studyConfiguration, true, false, true, getExpectedNumLoadedVariants (fileMetadata)); }
@Test public void basicIndex() throws Exception { clearDB(DB_NAME); StudyConfiguration studyConfiguration = newStudyConfiguration(); StoragePipelineResult etlResult = runDefaultETL(smallInputUri, variantStorageEngine, studyConfiguration, new ObjectMap(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "json")); assertTrue("Incorrect transform file extension " + etlResult.getTransformResult() + ". Expected 'variants.json.gz'", Paths.get(etlResult.getTransformResult()).toFile().getName().endsWith("variants.json.gz")); VariantFileMetadata fileMetadata = variantStorageEngine.getVariantReaderUtils().readVariantFileMetadata(etlResult.getTransformResult()); assertEquals(1, studyConfiguration.getIndexedFiles().size()); checkTransformedVariants(etlResult.getTransformResult(), studyConfiguration); checkLoadedVariants(variantStorageEngine.getDBAdaptor(), studyConfiguration, true, false, true, getExpectedNumLoadedVariants(fileMetadata)); }
@Test public void loadFromSTDIN() throws Exception { clearDB(DB_NAME); StudyConfiguration studyConfiguration = newStudyConfiguration(); StoragePipelineResult etlResult = runDefaultETL(smallInputUri, variantStorageEngine, studyConfiguration, new ObjectMap(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "avro"), true, false); VariantFileMetadata fileMetadata = checkTransformedVariants(etlResult.getTransformResult(), studyConfiguration); Path tempFile = Paths.get(outputUri).resolve("temp_file"); Files.move(Paths.get(etlResult.getTransformResult()), tempFile); assertFalse(Files.exists(Paths.get(etlResult.getTransformResult()))); InputStream in = System.in; try (InputStream is = new FileInputStream(tempFile.toFile())) { System.setIn(is); variantStorageEngine.getConfiguration() .getStorageEngine(variantStorageEngine.getStorageEngineId()).getVariant().getOptions() .put(VariantStorageEngine.Options.STDIN.key(), true); variantStorageEngine.index(Collections.singletonList(etlResult.getTransformResult()), outputUri, false, false, true); } finally { System.setIn(in); } studyConfiguration = variantStorageEngine.getStudyConfigurationManager().getStudyConfiguration(STUDY_NAME, null).first(); assertEquals(1, studyConfiguration.getIndexedFiles().size()); checkLoadedVariants(variantStorageEngine.getDBAdaptor(), studyConfiguration, true, false, true, getExpectedNumLoadedVariants(fileMetadata)); }
clearDB(DB_NAME); ObjectMap params = new ObjectMap(); StudyConfiguration studyConfiguration = newStudyConfiguration(); params.put(VariantStorageEngine.Options.STUDY.key(), studyConfiguration.getStudyName()); params.put(VariantStorageEngine.Options.COMPRESS_METHOD.key(), "snappy");
public void indexWithOtherFields(String extraFields) throws Exception { StudyConfiguration studyConfiguration = newStudyConfiguration(); StoragePipelineResult etlResult = runDefaultETL(smallInputUri, getVariantStorageEngine(), studyConfiguration, new ObjectMap(VariantStorageEngine.Options.EXTRA_GENOTYPE_FIELDS.key(), extraFields)
clearDB(DB_NAME); ObjectMap params = new ObjectMap(); StudyConfiguration studyConfiguration = newStudyConfiguration(); params.put(VariantStorageEngine.Options.STUDY.key(), studyConfiguration.getStudyName()); params.put(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "json");
@Test public void indexWithoutOtherFields() throws Exception { StudyConfiguration studyConfiguration = newStudyConfiguration(); runDefaultETL(smallInputUri, getVariantStorageEngine(), studyConfiguration, new ObjectMap(VariantStorageEngine.Options.EXTRA_GENOTYPE_FIELDS.key(), VariantQueryUtils.NONE) .append(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "avro") .append(VariantStorageEngine.Options.ANNOTATE.key(), false) .append(VariantStorageEngine.Options.CALCULATE_STATS.key(), false) ); for (Variant variant : variantStorageEngine.getDBAdaptor()) { assertEquals("GT", variant.getStudy(STUDY_NAME).getFormatAsString()); } }
@Test @Ignore public void insertVariantIntoSolr() throws Exception { clearDB(DB_NAME); ObjectMap params = new ObjectMap(); StudyConfiguration studyConfiguration = newStudyConfiguration(); params.put(VariantStorageEngine.Options.STUDY.key(), studyConfiguration.getStudyName()); params.put(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "json"); params.put(VariantStorageEngine.Options.COMPRESS_METHOD.key(), "gZiP"); params.put(VariantStorageEngine.Options.TRANSFORM_THREADS.key(), 1); params.put(VariantStorageEngine.Options.LOAD_THREADS.key(), 1); params.put(VariantStorageEngine.Options.ANNOTATE.key(), true); runETL(variantStorageEngine, params, true, true, true); VariantDBAdaptor dbAdaptor = getVariantStorageEngine().getDBAdaptor(); VariantSearchManager variantSearchManager = new VariantSearchManager(null, variantStorageEngine.getConfiguration()); // FIXME Collection is not in the configuration any more // variantSearchManager.load(variantStorageEngine.getConfiguration().getSearch().getCollection(), dbAdaptor.iterator()); }