public static StoragePipelineResult runETL(VariantStorageEngine variantStorageManager, ObjectMap options, boolean doExtract, boolean doTransform, boolean doLoad) throws IOException, FileFormatException, StorageEngineException { return runETL(variantStorageManager, inputUri, outputUri, options, doExtract, doTransform, doLoad); }
public static StoragePipelineResult runDefaultETL(URI inputUri, VariantStorageEngine variantStorageManager, StudyConfiguration studyConfiguration, ObjectMap params, boolean doTransform, boolean doLoad) throws URISyntaxException, IOException, FileFormatException, StorageEngineException { ObjectMap newParams = new ObjectMap(params); // newParams.put(VariantStorageEngine.Options.STUDY_CONFIGURATION.key(), studyConfiguration); newParams.putIfAbsent(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), studyConfiguration.getAggregation()); // newParams.putIfAbsent(VariantStorageEngine.Options.STUDY_ID.key(), studyConfiguration.getStudyId()); newParams.putIfAbsent(VariantStorageEngine.Options.STUDY.key(), studyConfiguration.getStudyName()); // newParams.putIfAbsent(VariantStorageEngine.Options.FILE_ID.key(), FILE_ID); // Default value is already avro // newParams.putIfAbsent(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "avro"); newParams.putIfAbsent(VariantStorageEngine.Options.ANNOTATE.key(), true); newParams.putIfAbsent(VariantAnnotationManager.SPECIES, "hsapiens"); newParams.putIfAbsent(VariantAnnotationManager.ASSEMBLY, "GRch37"); newParams.putIfAbsent(VariantStorageEngine.Options.CALCULATE_STATS.key(), true); StoragePipelineResult storagePipelineResult = runETL(variantStorageManager, inputUri, outputUri, newParams, true, doTransform, doLoad); try (VariantDBAdaptor dbAdaptor = variantStorageManager.getDBAdaptor()) { StudyConfiguration newStudyConfiguration = dbAdaptor.getStudyConfigurationManager().getStudyConfiguration(studyConfiguration.getStudyName(), null).first(); if (newStudyConfiguration != null) { studyConfiguration.copy(newStudyConfiguration); } } return storagePipelineResult; }
public static VariantFileMetadata loadFile( HadoopVariantStorageEngine variantStorageManager, String dbName, URI outputUri, String resourceName, StudyConfiguration studyConfiguration, Map<? extends String, ?> otherParams, boolean doTransform, boolean loadArchive, boolean loadVariant) throws Exception { URI fileInputUri = VariantStorageBaseTest.getResourceUri(resourceName); ObjectMap params = new ObjectMap(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "proto") .append(VariantStorageEngine.Options.STUDY.key(), studyConfiguration.getStudyName()) .append(VariantStorageEngine.Options.DB_NAME.key(), dbName).append(VariantStorageEngine.Options.ANNOTATE.key(), false) .append(VariantAnnotationManager.SPECIES, "hsapiens").append(VariantAnnotationManager.ASSEMBLY, "GRch37") .append(VariantStorageEngine.Options.CALCULATE_STATS.key(), false) .append(HadoopVariantStorageEngine.HADOOP_LOAD_DIRECT, true) .append(HadoopVariantStorageEngine.HADOOP_LOAD_ARCHIVE, loadArchive) .append(HadoopVariantStorageEngine.HADOOP_LOAD_VARIANT, loadVariant); if (otherParams != null) { params.putAll(otherParams); } // if (fileId > 0) { // params.append(VariantStorageEngine.Options.FILE_ID.key(), fileId); // } StoragePipelineResult etlResult = VariantStorageBaseTest.runETL(variantStorageManager, fileInputUri, outputUri, params, doTransform, doTransform, loadArchive || loadVariant); StudyConfiguration updatedStudyConfiguration = variantStorageManager.getDBAdaptor().getStudyConfigurationManager().getStudyConfiguration(studyConfiguration.getStudyId(), null).first(); if (updatedStudyConfiguration != null) { studyConfiguration.copy(updatedStudyConfiguration); } return variantStorageManager.readVariantFileMetadata(doTransform ? etlResult.getTransformResult() : etlResult.getInput()); }