protected VariantDBIterator getVariantsToIndex(boolean overwrite, Query query, QueryOptions queryOptions, VariantDBAdaptor dbAdaptor) throws StorageEngineException { if (!overwrite) { query.put(VariantQueryUtils.VARIANTS_TO_INDEX.key(), true); } return dbAdaptor.iterator(query, queryOptions); }
/** * Gets iterator from OpenCGA Variant database. **/ private Iterator<Variant> obtainIterator(VariantDBAdaptor variantDBAdaptor, Query query, QueryOptions options) { Query iteratorQuery = query != null ? query : new Query(); //Parse query options QueryOptions iteratorQueryOptions = options != null ? options : new QueryOptions(); // TODO rethink this way to refer to the Variant fields (through DBObjectToVariantConverter) List<String> include = Arrays.asList("chromosome", "start", "end", "alternate", "reference", "sourceEntries"); iteratorQueryOptions.add("include", include); return variantDBAdaptor.iterator(iteratorQuery, iteratorQueryOptions); }
@Override public void get(Request request, StreamObserver<Variant> responseObserver) { try { // Creating the datastore Query and QueryOptions objects from the gRPC request Map of Strings Query query = genericGrpcService.createQuery(request); QueryOptions queryOptions = genericGrpcService.createQueryOptions(request); // checkAuthorizedHosts(query, request.getIp()); VariantDBAdaptor variantDBAdaptor = getVariantDBAdaptor(request); // Iterator iterator = geneDBAdaptor.nativeIterator(query, queryOptions); Iterator iterator = variantDBAdaptor.iterator(query, queryOptions); while (iterator.hasNext()) { org.opencb.biodata.models.variant.Variant next = (org.opencb.biodata.models.variant.Variant) iterator.next(); Variant convert = convert(next); responseObserver.onNext(convert); } responseObserver.onCompleted(); variantDBAdaptor.close(); } catch (IllegalAccessException | InstantiationException | ClassNotFoundException | StorageEngineException e) { e.printStackTrace(); // } catch (NotAuthorizedHostException e) { // e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } }
public VariantDBIterator iterator(Query query, QueryOptions options) { query = preProcessQuery(query, options); return dbAdaptor.iterator(query, options); }
/** /** * Fetch all variants resulting of executing the query in the database. Returned fields are taken from * the 'include' and 'exclude' fields at options. * * @param variants Iterator with variants to filter * @param query Query to be executed in the database to filter variants * @param options Query modifiers, accepted values are: include, exclude, limit, skip, sort and count * @return A QueryResult with the result of the query */ default VariantQueryResult<Variant> get(Iterator<?> variants, Query query, QueryOptions options) { return iterator(variants, query, options) .toQueryResult(getSamplesMetadataIfRequested(query, options, getStudyConfigurationManager())); }
@Test public void multiIterator() throws Exception { List<String> variantsToQuery = allVariants.getResult() .stream() .filter(v -> !v.isSymbolic()) .map(Variant::toString) .limit(allVariants.getResult().size() / 2) .collect(Collectors.toList()); VariantDBIterator iterator = dbAdaptor.iterator(variantsToQuery.iterator(), new Query(), new QueryOptions()); QueryResult<Variant> queryResult = iterator.toQueryResult(); assertEquals(variantsToQuery.size(), queryResult.getResult().size()); }
/** * The query won't use the search index, either because is not available, not necessary, or forbidden. * * @param query Query * @param options QueryOptions * @param iterator Shall the resulting object be an iterator instead of a QueryResult * @return QueryResult or Iterator with the variants that matches the query * @throws StorageEngineException StorageEngineException */ protected Object getOrIteratorNotSearchIndex(Query query, QueryOptions options, boolean iterator) throws StorageEngineException { VariantDBAdaptor dbAdaptor = getDBAdaptor(); if (iterator) { return dbAdaptor.iterator(query, options); } else { setDefaultTimeout(options); return dbAdaptor.get(query, options).setSource(getStorageEngineId()); } }
public int export(Path outputVcf, Query query, QueryOptions options) throws IOException { try (GZIPOutputStream outputStream = new GZIPOutputStream(new FileOutputStream(outputVcf.toFile()))) { DataWriter<Variant> writer = new VariantWriterFactory(dbAdaptor).newDataWriter(VariantWriterFactory.VariantOutputFormat.VCF_GZ, outputStream, query, options); writer.open(); writer.pre(); dbAdaptor.iterator(query, options).forEachRemaining(writer::write); writer.post(); writer.close(); } return 0; }
.append(VariantQueryParam.SAMPLE.key(), samples); VariantDBIterator iterator = dbAdaptor.iterator(query, queryOptions);
@Test public void testNativeQuery() { int count = 0; for (VariantDBIterator iterator = dbAdaptor.iterator(new Query(), new QueryOptions(VariantHadoopDBAdaptor.NATIVE, true)); iterator.hasNext();) { Variant variant = iterator.next(); // System.out.println(variant.toJson()); count++; } Assert.assertEquals(dbAdaptor.count(new Query()).first().intValue(), count); }
static void checkCohorts(VariantDBAdaptor dbAdaptor, StudyConfiguration studyConfiguration) { for (VariantDBIterator iterator = dbAdaptor.iterator(new Query(), null); iterator.hasNext(); ) { Variant variant = iterator.next();
protected void checkCorrectness(URI file) throws StorageEngineException, NonStandardCompliantSampleField { Map<String, Variant> expectedVariants = readVariants(file); VariantDBIterator variants = variantStorageEngine.getDBAdaptor() .iterator(new Query(VariantQueryParam.STUDY.key(), "s1") .append(VariantQueryParam.FILE.key(), Paths.get(file).getFileName().toString()) .append(VariantQueryParam.INCLUDE_GENOTYPE.key(), true), new QueryOptions(QueryOptions.SORT, true)); while (variants.hasNext()) { Variant actual = variants.next(); Variant expected = expectedVariants.get(actual.toString()); // Check variants are correct assertNotNull(actual.toString(), expected); assertEquals(expected.toString(), actual.toString()); assertEquals(expected.getSv(), actual.getSv()); StudyEntry actualStudyEntry = actual.getStudies().get(0); StudyEntry expectedStudyEntry = expected.getStudies().get(0); // Check GTs are correct assertEquals(expectedStudyEntry.getSamplesData().stream().map(l -> Collections.singletonList(l.get(0))).collect(Collectors.toList()), actualStudyEntry.getSamplesData()); // Check File Attributes are correct expectedStudyEntry.getFiles().get(0).setFileId(""); actualStudyEntry.getFiles().get(0).setFileId(""); assertEquals(expectedStudyEntry.getFiles().get(0), actualStudyEntry.getFiles().get(0)); if (actual.getAlternate().equals("<DEL:ME:ALU>") || actual.getType().equals(VariantType.BREAKEND)) { System.err.println("WARN: Variant " + actual + (actual.getAnnotation() == null ? " without annotation" : " with annotation")); } else { assertNotNull(actual.toString(), actual.getAnnotation()); } } }
VariantDBIterator iterator = dbAdaptor.iterator(new Query(VariantQueryParam.STUDY.key(), studyConfigurationMultiFile.getStudyId() + "," + studyConfigurationSingleFile.getStudyId()).append(VariantQueryParam.UNKNOWN_GENOTYPE.key(), "0/0"), new QueryOptions()); int numVariants = 0;
@Test public void testArchiveIterator() { int count = 0; Query query = new Query(VariantQueryParam.STUDY.key(), studyConfiguration.getStudyId()) .append(VariantQueryParam.FILE.key(), UriUtils.fileName(smallInputUri)); for (VariantDBIterator iterator = dbAdaptor.iterator(query, new QueryOptions("archive", true)); iterator.hasNext(); ) { Variant variant = iterator.next(); // System.out.println(variant.toJson()); count++; } Assert.assertEquals(fileMetadata.getStats().getNumVariants(), count); }
logger.debug("Intersect query " + engineQuery.toJson() + " options " + options.toJson()); if (iterator) { return dbAdaptor.iterator(variantsIterator, engineQuery, options); } else { setDefaultTimeout(options);
@Test public void indexWithOtherFieldsNoGT() throws Exception { //GL:DP:GU:TU:AU:CU StudyConfiguration studyConfiguration = newStudyConfiguration(); StoragePipelineResult etlResult = runDefaultETL(getResourceUri("variant-test-somatic.vcf"), getVariantStorageEngine(), studyConfiguration, new ObjectMap(VariantStorageEngine.Options.EXTRA_GENOTYPE_FIELDS.key(), Arrays.asList("GL", "DP", "AU", "CU", "GU", "TU")) // .append(VariantStorageEngine.Options.FILE_ID.key(), 2) .append(VariantStorageEngine.Options.ANNOTATE.key(), false) .append(VariantStorageEngine.Options.CALCULATE_STATS.key(), false) ); VariantDBIterator iterator = getVariantStorageEngine().getDBAdaptor().iterator(new Query(VariantQueryParam.UNKNOWN_GENOTYPE.key(), "./."), new QueryOptions()); while (iterator.hasNext()) { Variant variant = iterator.next(); assertThat(variant.getStudy(STUDY_NAME).getSampleData("SAMPLE_1", "GT"), anyOf(is("./."), is("."))); assertNotNull(variant.getStudy(STUDY_NAME).getSampleData("SAMPLE_1", "DP")); assertNotNull(variant.getStudy(STUDY_NAME).getSampleData("SAMPLE_1", "GL")); assertNotNull(variant.getStudy(STUDY_NAME).getSampleData("SAMPLE_1", "AU")); assertNotNull(variant.getStudy(STUDY_NAME).getSampleData("SAMPLE_1", "CU")); assertNotNull(variant.getStudy(STUDY_NAME).getSampleData("SAMPLE_1", "GU")); assertNotNull(variant.getStudy(STUDY_NAME).getSampleData("SAMPLE_1", "TU")); } }
VariantDBIterator iterator = dbAdaptor.iterator(new Query(VariantQueryParam.INCLUDE_SAMPLE.key(), "SAMPLE_1") .append(VariantQueryParam.INCLUDE_FILE.key(), VariantQueryUtils.ALL), new QueryOptions()); iterator.forEachRemaining(variant -> { iterator = dbAdaptor.iterator(new Query(VariantQueryParam.INCLUDE_SAMPLE.key(), "SAMPLE_2") .append(VariantQueryParam.INCLUDE_FILE.key(), VariantQueryUtils.ALL), new QueryOptions()); iterator.forEachRemaining(variant -> { iterator = dbAdaptor.iterator(new Query(VariantQueryParam.INCLUDE_SAMPLE.key(), "SAMPLE_2") .append(VariantQueryParam.FILE.key(), "variant-test-somatic_2.vcf"), new QueryOptions()); iterator.forEachRemaining(variant -> {