public VariantExporter(VariantStorageEngine engine, VariantMetadataFactory metadataFactory) throws StorageEngineException { this.engine = engine; variantWriterFactory = new VariantWriterFactory(engine.getDBAdaptor()); this.metadataFactory = metadataFactory; }
/** * Exports the result of the given query and the associated metadata. * @param outputFileUri Optional output file. If null or empty, will print into the Standard output. Won't export any metadata. * @param outputFormat Variant Output format. * @param query Query with the variants to export * @param queryOptions Query options * @throws IOException If there is any IO error * @throws StorageEngineException If there is any error exporting variants */ public void export(@Nullable URI outputFileUri, VariantOutputFormat outputFormat, Query query, QueryOptions queryOptions) throws IOException, StorageEngineException { String outputFile = null; if (outputFileUri != null) { outputFile = outputFileUri.getPath(); } outputFile = VariantWriterFactory.checkOutput(outputFile, outputFormat); try (OutputStream os = VariantWriterFactory.getOutputStream(outputFile, outputFormat)) { boolean logProgress = !VariantWriterFactory.isStandardOutput(outputFile); exportData(os, outputFormat, query, queryOptions, logProgress); } if (metadataFactory != null && !VariantWriterFactory.isStandardOutput(outputFile)) { VariantMetadata metadata = metadataFactory.makeVariantMetadata(query, queryOptions); writeMetadata(metadata, outputFile + METADATA_FILE_EXTENSION); } }
public static OutputStream getOutputStream(String output, String outputFormatStr) throws IOException { VariantOutputFormat outputFormat = toOutputFormat(outputFormatStr, output); return getOutputStream(output, outputFormat); }
private DataWriter<Variant> configureWriter(final TaskAttemptContext job, OutputStream fileOut) throws IOException { // job.getCounter(VcfDataWriter.class.getName(), "failed").increment(0); // init final Configuration conf = job.getConfiguration(); VariantOutputFormat outputFormat = VariantOutputFormat.valueOf(conf.get(OUTPUT_FORMAT_PARAM)); DataWriter<Variant> dataWriter; VariantTableHelper helper = new VariantTableHelper(conf); try (StudyConfigurationManager scm = new StudyConfigurationManager(new HBaseVariantStorageMetadataDBAdaptorFactory(helper))) { VariantWriterFactory writerFactory = new VariantWriterFactory(scm); Query query = VariantMapReduceUtil.getQueryFromConfig(conf); QueryOptions options = VariantMapReduceUtil.getQueryOptionsFromConfig(conf); dataWriter = writerFactory.newDataWriter(outputFormat, fileOut, query, options); // dataWriter.setConverterErrorListener((v, e) -> // job.getCounter(VcfDataWriter.class.getName(), "failed").increment(1)); dataWriter.open(); dataWriter.pre(); return dataWriter; } }
DataWriter<Variant> variantDataWriter = variantWriterFactory.newDataWriter(outputFormat, outputStream, query, queryOptions);
@Test public void checkOutputTest() throws Exception { String path = ""; assertEquals(path + "myFile.json.gz", VariantWriterFactory.checkOutput(path + "myFile", JSON_GZ)); assertEquals(path + "myFile.json.gz", VariantWriterFactory.checkOutput(path + "myFile.json", JSON_GZ)); assertEquals(path + "myFile.json.gz", VariantWriterFactory.checkOutput(path + "myFile.json.gz", JSON_GZ)); assertEquals(path + "myFile.json.gz", VariantWriterFactory.checkOutput(path + "myFile.json.gz.", JSON_GZ)); assertEquals(path + "myFile.stats.tsv.gz", VariantWriterFactory.checkOutput(path + "myFile.", STATS_GZ)); assertEquals(path + "myFile.stats.tsv.gz", VariantWriterFactory.checkOutput(path + "myFile", STATS_GZ)); assertEquals(path + "myFile.stats.tsv.gz", VariantWriterFactory.checkOutput(path + "myFile.stats", STATS_GZ)); assertEquals(path + "myFile.stats.tsv.gz", VariantWriterFactory.checkOutput(path + "myFile.stats.", STATS_GZ)); assertEquals(path + "myFile.stats.tsv.gz", VariantWriterFactory.checkOutput(path + "myFile.stats.tsv", STATS_GZ)); assertEquals(path + "myFile.stats.tsv.gz", VariantWriterFactory.checkOutput(path + "myFile.stats.tsv.gz", STATS_GZ)); }
public static Query parseQuery(StorageVariantCommandOptions.VariantQueryCommandOptions queryVariantsOptions, List<String> studyNames) throws Exception { VariantWriterFactory.VariantOutputFormat of = VariantWriterFactory.toOutputFormat(queryVariantsOptions.outputFormat, null); return parseGenericVariantQuery(queryVariantsOptions, queryVariantsOptions.study, studyNames, queryVariantsOptions.commonQueryOptions.count, of); }
StudyConfiguration sc = getStudyConfiguration(query, true); List<String> cohorts = new ArrayList<>(sc.getCohortIds().keySet()); throw variantFormatNotSupported(outputFormat.toString());
/** * Transform the string to a valid output format. * If none, VCF by default. * * @param outputFormatStr Output format as String * @param output Output file * @return Valid VariantOutputFormat * @throws IllegalArgumentException if the outputFormatStr is not valid */ public static VariantOutputFormat toOutputFormat(String outputFormatStr, String output) { if (!StringUtils.isEmpty(outputFormatStr)) { outputFormatStr = outputFormatStr.replace('.', '_'); return VariantOutputFormat.valueOf(outputFormatStr.toUpperCase()); } else if (isStandardOutput(output)) { return VCF; } else { return VCF_GZ; } }
public int export(Path outputVcf, Query query, QueryOptions options) throws IOException { try (GZIPOutputStream outputStream = new GZIPOutputStream(new FileOutputStream(outputVcf.toFile()))) { DataWriter<Variant> writer = new VariantWriterFactory(dbAdaptor).newDataWriter(VariantWriterFactory.VariantOutputFormat.VCF_GZ, outputStream, query, options); writer.open(); writer.pre(); dbAdaptor.iterator(query, options).forEachRemaining(writer::write); writer.post(); writer.close(); } return 0; }
@Test public void checkBadOutputTest() throws Exception { thrown.expect(IllegalArgumentException.class); VariantWriterFactory.checkOutput("path/", JSON_GZ); } }
.toOutputFormat(variantQueryCommandOptions.outputFormat, variantQueryCommandOptions.commonQueryOptions.output); variantStorageEngine.exportData(uri, of, query, options);
public static OutputStream getOutputStream(String output, VariantOutputFormat outputFormat) throws IOException { boolean gzip = outputFormat.isGzip(); // output format has priority over output name OutputStream outputStream; if (isStandardOutput(output)) { // Unclosable OutputStream outputStream = new UnclosableOutputStream(System.out); } else { outputStream = new FileOutputStream(output); logger.debug("writing to %s", output); } // If compressed a GZip output stream is used if (gzip && outputFormat != VariantOutputFormat.AVRO_GZ) { outputStream = new GZIPOutputStream(outputStream); } else { outputStream = new BufferedOutputStream(outputStream); } logger.debug("using {} output stream", gzip ? "gzipped" : "plain"); return outputStream; }
private static void printVcf(StudyConfiguration studyConfiguration, VariantHadoopDBAdaptor dbAdaptor, Path outDir) throws IOException { try (OutputStream os = new FileOutputStream(outDir.resolve("variant." + studyConfiguration.getStudyName() + ".vcf").toFile())) { Query query = new Query(VariantQueryParam.STUDY.key(), studyConfiguration.getStudyName()).append(VariantQueryParam.UNKNOWN_GENOTYPE.key(), "."); QueryOptions queryOptions = new QueryOptions(); DataWriter<Variant> writer = new VariantWriterFactory(dbAdaptor).newDataWriter(VariantWriterFactory.VariantOutputFormat.VCF, os, query, queryOptions); writer.open(); writer.pre(); writer.write(dbAdaptor.get(query, queryOptions).getResult()); writer.post(); writer.close(); } }
public static String checkOutput(@Nullable String output, VariantOutputFormat outputFormat) throws IOException { if (isStandardOutput(output)) {
URI outputFile = null; final Path outdir; if (!VariantWriterFactory.isStandardOutput(outputStr)) { URI outdirUri = null; try {