public VariantSearchManager configure(VariantStorageEngine variantStorageEngine) throws StorageEngineException { variantStorageEngine.getConfiguration().getSearch().setMode("core"); variantStorageEngine.getConfiguration().getSearch().setActive(true); VariantSearchManager variantSearchManager = variantStorageEngine.getVariantSearchManager(); variantSearchManager.setSolrManager(new SolrManager(solrClient, "localhost", "core", variantStorageEngine.getConfiguration().getSearch().getTimeout())); return variantSearchManager; }
@Test public void transformIsolated() throws Exception { ObjectMap params = new ObjectMap(); URI outputUri = newOutputUri(); VariantStorageEngine variantStorageManager = getVariantStorageEngine(); variantStorageManager.getConfiguration().getStorageEngine(variantStorageManager.getStorageEngineId()).getVariant().getDatabase() .setHosts(Collections.singletonList("1.1.1.1")); StoragePipelineResult etlResult = runETL(variantStorageManager, smallInputUri, outputUri, params, true, true, false); System.out.println("etlResult = " + etlResult); String[] malformedFiles = Paths.get(outputUri).toFile().list((dir, name) -> name.contains(MALFORMED_FILE)); assertEquals(0, malformedFiles.length); }
@Test public void transformFromSTDIN() throws Exception { URI outputUri = newOutputUri(); VariantStorageEngine variantStorageManager = getVariantStorageEngine(); InputStream inputStream = FileUtils.newInputStream(Paths.get(smallInputUri)); System.setIn(inputStream); ObjectMap options = variantStorageManager.getConfiguration() .getStorageEngine(variantStorageManager.getStorageEngineId()).getVariant().getOptions(); options.append(VariantStorageEngine.Options.STDIN.key(), true); URI inputFile = Paths.get(smallInputUri).getFileName().toUri(); System.out.println("inputFile = " + inputFile); assertFalse("File should not exist in that specific location", Paths.get(inputFile).toFile().exists()); StoragePipelineResult storagePipelineResult = variantStorageManager.index(Collections.singletonList(inputFile), outputUri, true, true, false).get(0); assertEquals(999, countLinesFromAvro(Paths.get(storagePipelineResult.getTransformResult()).toFile())); }
@Test public void transformToAvroSTDOUT() throws Exception { URI outputUri = newOutputUri(); VariantStorageEngine variantStorageManager = getVariantStorageEngine(); File outputFile = Paths.get(outputUri.resolve("outputFile.avro")).toFile(); try (PrintStream os = new PrintStream(new FileOutputStream(outputFile))) { System.setOut(os); ObjectMap options = variantStorageManager.getConfiguration() .getStorageEngine(variantStorageManager.getStorageEngineId()).getVariant().getOptions(); options.append(VariantStorageEngine.Options.STDOUT.key(), true); options.append(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "avro"); variantStorageManager.index(Collections.singletonList(smallInputUri), outputUri, true, true, false).get(0); } finally { System.setOut(out); } assertEquals(999, countLinesFromAvro(outputFile)); }
@Test public void transformToJsonSTDOUT() throws Exception { URI outputUri = newOutputUri(); VariantStorageEngine variantStorageManager = getVariantStorageEngine(); File outputFile = Paths.get(outputUri.resolve("outputFile.json")).toFile(); try (PrintStream os = new PrintStream(new FileOutputStream(outputFile))) { System.setOut(os); ObjectMap options = variantStorageManager.getConfiguration() .getStorageEngine(variantStorageManager.getStorageEngineId()).getVariant().getOptions(); options.append(VariantStorageEngine.Options.STDOUT.key(), true); options.append(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "json"); variantStorageManager.index(Collections.singletonList(smallInputUri), outputUri, true, true, false).get(0); } finally { System.setOut(out); } assertEquals(999, countLines(outputFile)); }
String dbName = (searchOptions.dbName == null ? "variants" : searchOptions.dbName); variantStorageEngine.getConfiguration().getSearch().setHost(solrUrl); variantStorageEngine.getConfiguration()); boolean querying = true; QueryOptions options = new QueryOptions(); String mode = variantStorageEngine.getConfiguration().getSearch().getMode(); if (querying) { if ("cloud".equals(mode)) {
@Test public void testFailSearchNotAvailable() throws StorageEngineException { VariantQueryException exception = new VariantQueryException("Unable to use search index. SearchEngine is not available"); thrown.expect(exception.getClass()); thrown.expectMessage(exception.getMessage()); variantStorageEngine.getConfiguration().getSearch().setActive(false); variantStorageEngine.doIntersectWithSearch(new Query(ANNOT_TRAIT.key(), "myTrait"), new QueryOptions(USE_SEARCH_INDEX, VariantStorageEngine.UseSearchIndex.YES)); }
@Test public void loadFromSTDIN() throws Exception { clearDB(DB_NAME); StudyConfiguration studyConfiguration = newStudyConfiguration(); StoragePipelineResult etlResult = runDefaultETL(smallInputUri, variantStorageEngine, studyConfiguration, new ObjectMap(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "avro"), true, false); VariantFileMetadata fileMetadata = checkTransformedVariants(etlResult.getTransformResult(), studyConfiguration); Path tempFile = Paths.get(outputUri).resolve("temp_file"); Files.move(Paths.get(etlResult.getTransformResult()), tempFile); assertFalse(Files.exists(Paths.get(etlResult.getTransformResult()))); InputStream in = System.in; try (InputStream is = new FileInputStream(tempFile.toFile())) { System.setIn(is); variantStorageEngine.getConfiguration() .getStorageEngine(variantStorageEngine.getStorageEngineId()).getVariant().getOptions() .put(VariantStorageEngine.Options.STDIN.key(), true); variantStorageEngine.index(Collections.singletonList(etlResult.getTransformResult()), outputUri, false, false, true); } finally { System.setIn(in); } studyConfiguration = variantStorageEngine.getStudyConfigurationManager().getStudyConfiguration(STUDY_NAME, null).first(); assertEquals(1, studyConfiguration.getIndexedFiles().size()); checkLoadedVariants(variantStorageEngine.getDBAdaptor(), studyConfiguration, true, false, true, getExpectedNumLoadedVariants(fileMetadata)); }
VariantDBAdaptor dbAdaptor = variantStorageEngine.getDBAdaptor(); VariantAnnotator annotator = VariantAnnotatorFactory.buildVariantAnnotator(variantStorageEngine.getConfiguration(), variantStorageEngine.getStorageEngineId(), projectMetadata); DefaultVariantAnnotationManager annotationManager = new DefaultVariantAnnotationManager(annotator, dbAdaptor);
VariantDBAdaptor dbAdaptor = variantStorageEngine.getDBAdaptor(); VariantAnnotator annotator = VariantAnnotatorFactory.buildVariantAnnotator(variantStorageEngine.getConfiguration(), variantStorageEngine.getStorageEngineId(), projectMetadata); DefaultVariantAnnotationManager annotationManager = new DefaultVariantAnnotationManager(annotator, dbAdaptor);
VariantDBAdaptor dbAdaptor = variantStorageEngine.getDBAdaptor(); VariantAnnotator annotator = VariantAnnotatorFactory.buildVariantAnnotator(variantStorageEngine.getConfiguration(), variantStorageEngine.getStorageEngineId(), projectMetadata); DefaultVariantAnnotationManager annotationManager = new DefaultVariantAnnotationManager(annotator, dbAdaptor);
public static StoragePipelineResult runETL(VariantStorageEngine variantStorageManager, URI inputUri, URI outputUri, ObjectMap params, boolean doExtract, boolean doTransform, boolean doLoad) throws IOException, FileFormatException, StorageEngineException { variantStorageManager.getConfiguration() .getStorageEngine(variantStorageManager.getStorageEngineId()).getVariant().getOptions().putAll(params); StoragePipelineResult storagePipelineResult = variantStorageManager.index(Collections.singletonList(inputUri), outputUri, doExtract, doTransform, doLoad).get(0); checkFileExists(storagePipelineResult.getExtractResult()); checkFileExists(storagePipelineResult.getPreTransformResult()); checkFileExists(storagePipelineResult.getTransformResult()); checkFileExists(storagePipelineResult.getPostTransformResult()); checkFileExists(storagePipelineResult.getPreLoadResult()); checkFileExists(storagePipelineResult.getLoadResult()); checkFileExists(storagePipelineResult.getPostLoadResult()); return storagePipelineResult; }
List<Variant> variants = reader.read(limit); CellBaseClient cellBaseClient = new CellBaseClient(variantStorageEngine.getConfiguration().getCellbase().toClientConfiguration()); QueryResponse<VariantAnnotation> queryResponse = cellBaseClient.getVariantClient().getAnnotationByVariantIds(variants.stream().map(Variant::toString).collect(Collectors.toList()), QueryOptions.empty());
@Test @Ignore public void insertVariantIntoSolr() throws Exception { clearDB(DB_NAME); ObjectMap params = new ObjectMap(); StudyConfiguration studyConfiguration = newStudyConfiguration(); params.put(VariantStorageEngine.Options.STUDY.key(), studyConfiguration.getStudyName()); params.put(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "json"); params.put(VariantStorageEngine.Options.COMPRESS_METHOD.key(), "gZiP"); params.put(VariantStorageEngine.Options.TRANSFORM_THREADS.key(), 1); params.put(VariantStorageEngine.Options.LOAD_THREADS.key(), 1); params.put(VariantStorageEngine.Options.ANNOTATE.key(), true); runETL(variantStorageEngine, params, true, true, true); VariantDBAdaptor dbAdaptor = getVariantStorageEngine().getDBAdaptor(); VariantSearchManager variantSearchManager = new VariantSearchManager(null, variantStorageEngine.getConfiguration()); // FIXME Collection is not in the configuration any more // variantSearchManager.load(variantStorageEngine.getConfiguration().getSearch().getCollection(), dbAdaptor.iterator()); }
variantStorageManager.getConfiguration().getStorageEngine(variantStorageManager.getStorageEngineId()).getVariant().getOptions() .append(VariantStorageEngine.Options.STUDY.key(), studyConfigurationBatchFile.getStudyName()) .putAll(options);