protected final URI postLoad(URI input, URI output, List<Integer> fileIds) throws StorageEngineException { if (fileIds == null || fileIds.isEmpty()) { fileIds = Collections.singletonList(getFileId()); } int studyId = getStudyId(); long lock = dbAdaptor.getStudyConfigurationManager().lockStudy(studyId); // Check loaded variants BEFORE updating the StudyConfiguration checkLoadedVariants(fileIds, getStudyConfiguration()); StudyConfiguration studyConfiguration; try { //Update StudyConfiguration studyConfiguration = getStudyConfiguration(true); securePostLoad(fileIds, studyConfiguration); dbAdaptor.getStudyConfigurationManager().updateStudyConfiguration(studyConfiguration, new QueryOptions()); } finally { dbAdaptor.getStudyConfigurationManager().unLockStudy(studyId, lock); } return input; }
@Override public URI preLoad(URI input, URI output) throws StorageEngineException { getOrCreateStudyConfiguration(false); int studyId = getStudyId(); VariantFileMetadata fileMetadata = readVariantFileMetadata(input); //Get the studyConfiguration. If there is no StudyConfiguration, create a empty one. dbAdaptor.getStudyConfigurationManager().lockAndUpdate(studyId, studyConfiguration -> { securePreLoad(studyConfiguration, fileMetadata); privateStudyConfiguration = studyConfiguration; return studyConfiguration; }); return input; }
@Override public URI postLoad(URI input, URI output) throws StorageEngineException { return postLoad(input, output, null); }
protected StudyConfiguration getOrCreateStudyConfiguration(boolean forceFetch) throws StorageEngineException { return checkExistsStudyConfiguration(getStudyConfiguration(forceFetch)); }
protected StudyConfiguration checkExistsStudyConfiguration(StudyConfiguration studyConfiguration) throws StorageEngineException { if (studyConfiguration == null) { studyConfiguration = getStudyConfiguration(); if (studyConfiguration == null) { String studyName = options.getString(Options.STUDY.key(), Options.STUDY.defaultValue()); logger.info("Creating a new StudyConfiguration '{}'", studyName); studyConfiguration = getStudyConfigurationManager().createStudy(studyName); } } privateStudyConfiguration = studyConfiguration; setStudyId(studyConfiguration.getStudyId()); return studyConfiguration; }
@Override public void securePostLoad(List<Integer> fileIds, StudyConfiguration studyConfiguration) throws StorageEngineException { super.securePostLoad(fileIds, studyConfiguration); studyConfiguration.getAttributes().put(MISSING_GENOTYPES_UPDATED, false); }
@Override protected void securePreLoad(StudyConfiguration studyConfiguration, VariantFileMetadata source) throws StorageEngineException { super.securePreLoad(studyConfiguration, source); List<Integer> fileIds = Collections.singletonList(getFileId()); BatchFileOperation op = new BatchFileOperation("load", fileIds, 1, BatchFileOperation.Type.LOAD); op.addStatus(BatchFileOperation.Status.RUNNING); studyConfiguration.getBatches().add(op); }
String filePath = storageETL.readVariantFileMetadata(nextUri).getPath(); String fileName = Paths.get(filePath).getFileName().toString(); StudyConfiguration studyConfiguration = storageETL.getStudyConfiguration(); } finally { try { storageETL.close(); } catch (StorageEngineException e) { logger.error("Issue closing DB connection ", e);
String studyName = statsVariantsCommandOptions.study; String filename = outputUri.equals(directoryUri) ? VariantStoragePipeline.buildFilename(studyName, 0) : Paths.get(outputUri.getPath()).getFileName().toString(); filename += '.' + TimeUtils.getTime();
public URI preLoad(URI input, URI output) throws StorageEngineException { URI uri = super.preLoad(input, output); if (isResumeStage(options)) { logger.info("Resume stage load."); // Clean stage collection? } return uri; }
public final StudyConfiguration getStudyConfiguration() throws StorageEngineException { return getStudyConfiguration(false); }
@Override public void securePostLoad(List<Integer> fileIds, StudyConfiguration studyConfiguration) throws StorageEngineException { super.securePostLoad(fileIds, studyConfiguration); BatchFileOperation.Status status = dbAdaptor.getStudyConfigurationManager() .setStatus(studyConfiguration, BatchFileOperation.Status.READY, "load", fileIds); if (status != BatchFileOperation.Status.DONE) { logger.warn("Unexpected status " + status); } }
@Override protected void securePreLoad(StudyConfiguration studyConfiguration, VariantFileMetadata fileMetadata) throws StorageEngineException { super.securePreLoad(studyConfiguration, fileMetadata); MergeMode mergeMode; if (!studyConfiguration.getAttributes().containsKey(Options.MERGE_MODE.key())) { mergeMode = MergeMode.from(options); studyConfiguration.getAttributes().put(Options.MERGE_MODE.key(), mergeMode); } else { options.put(MERGE_MODE.key(), MergeMode.from(studyConfiguration.getAttributes())); } }
.buildFilename(studyConfiguration.getStudyName(), fileIds.get(0)) + "." + TimeUtils.getTime()); statsOptions.put(DefaultVariantStatisticsManager.OUTPUT, statsOutputUri.toString());
@Override public URI preLoad(URI input, URI output) throws StorageEngineException { super.preLoad(input, output); try { ArchiveTableHelper.createArchiveTableIfNeeded(dbAdaptor.getGenomeHelper(), getArchiveTable(), dbAdaptor.getConnection()); } catch (IOException e) { throw new StorageHadoopException("Issue creating table " + getArchiveTable(), e); } try { VariantTableHelper.createVariantTableIfNeeded(dbAdaptor.getGenomeHelper(), variantsTableCredentials.getTable(), dbAdaptor.getConnection()); } catch (IOException e) { throw new StorageHadoopException("Issue creating table " + variantsTableCredentials.getTable(), e); } return input; }
protected int getStudyId() throws StorageEngineException { if (privateStudyId == null) { privateStudyId = getStudyConfiguration().getStudyId(); return privateStudyId; } else { return privateStudyId; } }
@Override public void securePostLoad(List<Integer> fileIds, StudyConfiguration studyConfiguration) throws StorageEngineException { super.securePostLoad(fileIds, studyConfiguration); boolean direct = options.getBoolean(DIRECT_LOAD.key(), DIRECT_LOAD.defaultValue()); if (direct) { BatchFileOperation.Status status = setStatus(studyConfiguration, BatchFileOperation.Status.READY, DIRECT_LOAD.key(), fileIds); if (status != BatchFileOperation.Status.DONE) { logger.warn("Unexpected status " + status); } } else { BatchFileOperation.Status status = setStatus(studyConfiguration, BatchFileOperation.Status.READY, MERGE.key(), fileIds); if (status != BatchFileOperation.Status.DONE) { logger.warn("Unexpected status " + status); } } Set<String> genotypes = new HashSet<>(studyConfiguration.getAttributes().getAsStringList(LOADED_GENOTYPES.key())); genotypes.addAll(writeResult.getGenotypes()); studyConfiguration.getAttributes().put(LOADED_GENOTYPES.key(), genotypes); }
@Override protected void securePreLoad(StudyConfiguration studyConfiguration, VariantFileMetadata source) throws StorageEngineException { super.securePreLoad(studyConfiguration, source); int fileId = getFileId();
@Override public URI postLoad(URI input, URI output) throws StorageEngineException { StudyConfigurationManager scm = getStudyConfigurationManager(); try { int studyId = getStudyId(); VariantFileMetadata fileMetadata = readVariantFileMetadata(input); fileMetadata.setId(String.valueOf(getFileId())); scm.updateVariantFileMetadata(studyId, fileMetadata); dbAdaptor.getVariantFileMetadataDBAdaptor().updateLoadedFilesSummary(studyId, Collections.singletonList(getFileId())); } catch (IOException e) { throw new StorageEngineException("Error storing VariantFileMetadata for file " + getFileId(), e); } registerLoadedFiles(Collections.singletonList(getFileId())); // This method checks the loaded variants (if possible) and adds the loaded files to the studyConfiguration super.postLoad(input, output); return input; }
File[] statsFile1 = getTmpRootDir().toFile().listFiles((dir, name1) -> name1.startsWith(VariantStoragePipeline.buildFilename(studyConfiguration.getStudyName(), fileIdChr1)) && name1.contains("variants")); File[] statsFile2 = getTmpRootDir().toFile().listFiles((dir, name1) -> name1.startsWith(VariantStoragePipeline.buildFilename(studyConfiguration.getStudyName(), fileIdChr22)) && name1.contains("variants")); assertEquals(1, statsFile1.length);