public static Path getMetaFromTransformedFile(Path variantsFile) { return Paths.get(getMetaFromTransformedFile(variantsFile.toString())); }
@Override public boolean open() { try { if (outdir != null) { String filePath = fileMetadata.getPath(); String fileName = Paths.get(filePath).getFileName().toString(); String output = Paths.get(outdir.toString(), fileName).toAbsolutePath().toString() + "." + VariantReaderUtils.VARIANTS_FILE + ".json.gz"; variantsStream = new GZIPOutputStream(new FileOutputStream(output)); fileStream = new GZIPOutputStream(new FileOutputStream(VariantReaderUtils.getMetaFromTransformedFile(output))); } } catch (IOException ex) { throw new UncheckedIOException(ex); } return true; }
protected static VariantJsonReader getVariantJsonReader(Path input, VariantStudyMetadata metadata) throws StorageEngineException { VariantJsonReader variantJsonReader; if (isJson(input.toString())) { Path sourceFile = getMetaFromTransformedFile(input.toAbsolutePath()); variantJsonReader = new VariantJsonReader(metadata, input.toAbsolutePath().toString(), sourceFile.toAbsolutePath().toString()); } else { throw variantInputNotSupported(input); } return variantJsonReader; }
protected static VariantAvroReader getVariantAvroReader(Path input, VariantStudyMetadata metadata, boolean stdin) throws StorageEngineException { VariantAvroReader variantAvroReader; if (isAvro(input.toString())) { String sourceFile = getMetaFromTransformedFile(input.toAbsolutePath().toString()); if (stdin) { variantAvroReader = new VariantAvroReader(System.in, new File(sourceFile), metadata); } else { variantAvroReader = new VariantAvroReader(input.toAbsolutePath().toFile(), new File(sourceFile), metadata); } } else { throw variantInputNotSupported(input); } return variantAvroReader; }
@Override public VariantFileMetadata readVariantFileMetadata(URI input) throws StorageEngineException { VariantFileMetadata source; if (input.getScheme() == null || input.getScheme().startsWith("file")) { return VariantReaderUtils.readVariantFileMetadata(Paths.get(input.getPath()), null); } Path metaPath = new Path(VariantReaderUtils.getMetaFromTransformedFile(input.toString())); FileSystem fs = null; try { fs = FileSystem.get(conf); } catch (IOException e) { throw new StorageEngineException("Unable to get FileSystem", e); } try ( InputStream inputStream = new GZIPInputStream(fs.open(metaPath)) ) { source = VariantReaderUtils.readVariantFileMetadataFromJson(inputStream); } catch (IOException e) { throw new StorageEngineException("Unable to read VariantFileMetadata", e); } return source; } }
URI fileInput = URI.create(VariantReaderUtils.getMetaFromTransformedFile(input.toString())); org.apache.hadoop.fs.Path fileOutputPath = new org.apache.hadoop.fs.Path( output.resolve(Paths.get(fileInput.getPath()).getFileName().toString()));
protected void load(URI input, int studyId, int fileId) throws StorageEngineException { URI vcfMeta = URI.create(VariantReaderUtils.getMetaFromTransformedFile(input.toString())); String hadoopRoute = options.getString(HADOOP_BIN, "hadoop"); String jar = getJarWithDependencies(); Class execClass = ArchiveDriver.class; String executable = hadoopRoute + " jar " + jar + " " + execClass.getName(); String args = ArchiveDriver.buildCommandLineArgs(input, vcfMeta, variantsTableCredentials.toString(), getArchiveTable(), studyId, fileId, options); long startTime = System.currentTimeMillis(); logger.info("------------------------------------------------------"); logger.info("Loading file {} into archive table '{}'", fileId, getArchiveTable()); logger.debug(executable + " " + args); logger.info("------------------------------------------------------"); int exitValue = mrExecutor.run(executable, args); logger.info("------------------------------------------------------"); logger.info("Exit value: {}", exitValue); logger.info("Total time: {}s", (System.currentTimeMillis() - startTime) / 1000.0); if (exitValue != 0) { throw new StorageEngineException("Error loading file " + input + " into archive table \"" + getArchiveTable() + "\""); } }
@Test public void getMetaTest() { assertEquals("s1.genome.vcf.file.json.gz", VariantReaderUtils.getMetaFromTransformedFile("s1.genome.vcf.variants.json.gz")); assertEquals("s1.genome.vcf.file.json.gz", VariantReaderUtils.getMetaFromTransformedFile("s1.genome.vcf.variants.json.snappy")); assertEquals("s1.genome.vcf.file.json.gz", VariantReaderUtils.getMetaFromTransformedFile("s1.genome.vcf.variants.json")); assertEquals("s1.genome.vcf.file.json.gz", VariantReaderUtils.getMetaFromTransformedFile("s1.genome.vcf.variants.avro.gz")); assertEquals("s1.genome.vcf.file.json.gz", VariantReaderUtils.getMetaFromTransformedFile("s1.genome.vcf.variants.avro.snappy")); assertEquals("s1.genome.vcf.file.json.gz", VariantReaderUtils.getMetaFromTransformedFile("s1.genome.vcf.variants.avro")); assertEquals("s1.genome.vcf.file.json.gz", VariantReaderUtils.getMetaFromTransformedFile("s1.genome.vcf.variants.proto.gz")); assertEquals("s1.genome.vcf.file.json.gz", VariantReaderUtils.getMetaFromTransformedFile("s1.genome.vcf.variants.proto.snappy")); assertEquals("s1.genome.vcf.file.json.gz", VariantReaderUtils.getMetaFromTransformedFile("s1.genome.vcf.variants.proto")); assertEquals("s1_variants.genomes.vcf.gz.file.json.gz", VariantReaderUtils.getMetaFromTransformedFile("s1_variants.genomes.vcf.gz.variants.avro.gz")); thrown.expect(IllegalArgumentException.class); VariantReaderUtils.getMetaFromTransformedFile("s1.genome.vcf.gz"); }
input = getMetaFromTransformedFile(input);
Path outputMetaFile = VariantReaderUtils.getMetaFromTransformedFile(outputVariantsFile);
@Test public void testDeleteTransformedFile() throws Exception { QueryOptions queryOptions = new QueryOptions(VariantStorageEngine.Options.ANNOTATE.key(), false) .append(VariantStorageEngine.Options.CALCULATE_STATS.key(), false); File inputFile = getFile(0); File transformedFile = transformFile(inputFile, queryOptions); catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.NAME.key(), transformedFile.getName()), new ObjectMap(FileManager.SKIP_TRASH, true), sessionId); catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.NAME.key(), VariantReaderUtils.getMetaFromTransformedFile(transformedFile.getName())), new ObjectMap(FileManager.SKIP_TRASH, true), sessionId); indexFile(inputFile, queryOptions, outputId); }