private long getTransformedFileIdFromOriginal(File file) throws CatalogException { long transformedFile = file.getIndex() != null && file.getIndex().getTransformedFile() != null ? file.getIndex().getTransformedFile().getId() : -1; if (transformedFile == -1) { logger.error("This code should never be executed. Every vcf file containing the transformed status should have" + " a registered transformed file"); throw new CatalogException("Internal error. No transformed file could be found for file " + file.getUid()); } return transformedFile; }
@Override protected List<String> validate(String defaultStudyStr, List<String> values, Integer release, VariantQueryParam param, String sessionId) throws CatalogException { if (release == null) { AbstractManager.MyResources<File> uids = catalogManager.getFileManager().getUids(values, defaultStudyStr, sessionId); return uids.getResourceList().stream().map(File::getName).collect(Collectors.toList()); } else { return validate(defaultStudyStr, values, release, param, catalogManager.getFileManager(), File::getName, file -> ((int) file.getIndex().getRelease()), file -> { if (file.getIndex() == null || file.getIndex().getStatus() == null || file.getIndex().getStatus().getName() == null || !file.getIndex().getStatus().getName().equals(Status.READY)) { throw new VariantQueryException("File '" + file.getName() + "' is not indexed"); } }, sessionId); } } }
if (file.getStatus().getName().equals(File.FileStatus.READY) && file.getFormat().equals(File.Format.VCF)) { String indexStatus; if (file.getIndex() != null && file.getIndex().getStatus() != null && file.getIndex().getStatus().getName() != null) { indexStatus = file.getIndex().getStatus().getName(); } else { indexStatus = FileIndex.IndexStatus.NONE;
while (iterator.hasNext()) { File next = iterator.next(); String status = next.getIndex().getStatus().getName(); switch (status) { case FileIndex.IndexStatus.READY: next.getIndex().setTransformedFile(null); filesToUpdate.put(next.getUid(), next.getIndex()); break; case FileIndex.IndexStatus.TRANSFORMED: next.getIndex().setTransformedFile(null); next.getIndex().getStatus().setName(FileIndex.IndexStatus.NONE); filesToUpdate.put(next.getUid(), next.getIndex()); break; default:
public List<File> removeFiles(StudyInfo studyInfo, QueryOptions options, String sessionId) throws CatalogException, StorageEngineException, IOException { // We get the credentials of the Datastore to insert the variants DataStore dataStore = studyInfo.getDataStores().get(File.Bioformat.VARIANT); // Update study configuration BEFORE executing the operation and fetching files from Catalog updateCatalogFromStudyConfiguration(sessionId, studyInfo.getStudyFQN(), dataStore); List<String> fileNames = new ArrayList<>(studyInfo.getFileInfos().size()); List<String> filePaths = new ArrayList<>(studyInfo.getFileInfos().size()); for (FileInfo fileInfo : studyInfo.getFileInfos()) { File file = catalogManager.getFileManager().get(studyInfo.getStudyFQN(), fileInfo.getPath(), null, sessionId).first(); if (file.getIndex().getStatus().getName().equals(FileIndex.IndexStatus.READY)) { fileNames.add(fileInfo.getName()); filePaths.add(fileInfo.getPath()); } else { throw new CatalogException("Unable to remove variants from file " + file.getName() + ". " + "IndexStatus = " + file.getIndex().getStatus().getName()); } } if (fileNames.isEmpty()) { throw new CatalogException("Nothing to do!"); } VariantStorageEngine variantStorageEngine = getVariantStorageEngine(dataStore); variantStorageEngine.getOptions().putAll(options); variantStorageEngine.removeFiles(studyInfo.getStudyFQN(), fileNames); // Update study configuration to synchronize updateCatalogFromStudyConfiguration(sessionId, studyInfo.getStudyFQN(), dataStore); return catalogManager.getFileManager().get(studyInfo.getStudyFQN(), new Query(FileDBAdaptor.QueryParams.PATH.key(), filePaths), new QueryOptions(), sessionId) .getResult(); }
private void checkStudyConfiguration(Study study, StudyConfiguration studyConfiguration) throws CatalogException { assertEquals("user@p1:s1", studyConfiguration.getStudyName()); assertEquals(study.getUid(), studyConfiguration.getStudyId()); assertTrue(studyConfiguration.getInvalidStats().isEmpty()); for (Map.Entry<String, Integer> entry : studyConfiguration.getFileIds().entrySet()) { File file = catalogManager.getFileManager().get(studyConfiguration.getStudyName(), studyConfiguration.getFileIds().inverse().get(entry.getValue()), null, sessionId).first(); assertEquals(file.getName(), entry.getKey()); int id = (int) file.getUid(); assertEquals(file.getSamples().stream().map(Sample::getUid).map(Long::intValue).collect(Collectors.toSet()), studyConfiguration.getSamplesInFiles().get((id))); if (file.getIndex() == null || file.getIndex().getStatus() == null || file.getIndex().getStatus().getName() == null || !file.getIndex().getStatus().getName().equals(FileIndex.IndexStatus.READY)) { assertFalse(studyConfiguration.getIndexedFiles().contains(id)); // assertFalse("Should not contain header for file " + file.getId(), studyConfiguration.getHeaders().containsKey(id)); } // else { // assertTrue(studyConfiguration.getIndexedFiles().contains(id)); // assertTrue("Missing header for file " + file.getId(), studyConfiguration.getHeaders().containsKey(id)); // assertTrue("Missing header for file " + file.getId(), !studyConfiguration.getHeaders().get(id).isEmpty()); // } } }
private void removeFile(List<File> files, QueryOptions options, String outputId) throws Exception { List<String> fileIds = files.stream().map(File::getId).collect(Collectors.toList()); Study study = catalogManager.getFileManager().getStudy(files.get(0), sessionId); String studyId = study.getFqn(); List<File> removedFiles = variantManager.removeFile(fileIds, studyId, sessionId, new QueryOptions()); assertEquals(files.size(), removedFiles.size()); Cohort all = catalogManager.getCohortManager().get(studyId, new Query(CohortDBAdaptor.QueryParams.ID.key(), StudyEntry.DEFAULT_COHORT), null, sessionId).first(); Set<Long> allSampleIds = all.getSamples().stream().map(Sample::getUid).collect(Collectors.toSet()); assertThat(all.getStatus().getName(), anyOf(is(Cohort.CohortStatus.INVALID), is(Cohort.CohortStatus.NONE))); Set<Long> loadedSamples = catalogManager.getFileManager().get(studyId, new Query(FileDBAdaptor.QueryParams.INDEX_STATUS_NAME.key (), FileIndex.IndexStatus.READY), null, sessionId) .getResult() .stream() .flatMap(f -> f.getSamples().stream()) .map(Sample::getUid) .collect(Collectors.toSet()); assertEquals(loadedSamples, allSampleIds); for (File file : removedFiles) { assertEquals(FileIndex.IndexStatus.TRANSFORMED, file.getIndex().getStatus().getName()); } }
assertEquals(FileIndex.IndexStatus.READY, catalogManager.getFileManager().get(studyId, nonIndexedFile.getName(), null, sessionId).first().getIndex().getStatus().getName()); studyConfigurationFactory.updateCatalogFromStudyConfiguration(sc, sessionId); assertEquals(FileIndex.IndexStatus.INDEXING, catalogManager.getFileManager().get(studyId, nonIndexedFile.getName(), null, sessionId).first().getIndex().getStatus().getName());
@Test public void testUpdateIndexStatus() throws CatalogException { QueryResult<File> fileResult = fileManager.create(studyFqn, File.Type.FILE, File.Format.VCF, File.Bioformat.VARIANT, "data/test.vcf", "", "description", new File.FileStatus(File.FileStatus.STAGE), 0, -1, Collections.emptyList(), -1, Collections.emptyMap(), Collections.emptyMap(), true, null, new QueryOptions(), sessionIdUser); fileManager.updateFileIndexStatus(fileResult.first(), FileIndex.IndexStatus.TRANSFORMED, null, sessionIdUser); QueryResult<File> read = fileManager.get(studyFqn, fileResult.first().getPath(), new QueryOptions(), sessionIdUser); assertEquals(FileIndex.IndexStatus.TRANSFORMED, read.first().getIndex().getStatus().getName()); }
@Test public void testIndex1() throws Exception { indexFile(inputFile1, new QueryOptions(), outputId); Assert.assertEquals(FileIndex.IndexStatus.READY, catalogManager.getFileManager().get(studyId, inputFile1.getPath(), null, sessionId).first().getIndex().getStatus().getName()); Assert.assertNull(catalogManager.getFileManager().get(studyId, inputFile2.getPath(), null, sessionId).first().getIndex().getStatus()); }
@Test public void testBySteps2() throws Exception { File transformFile = transformFile(inputFile2, new QueryOptions()); Assert.assertNull(catalogManager.getFileManager().get(studyId, inputFile1.getPath(), null, sessionId).first().getIndex().getStatus()); Assert.assertEquals(FileIndex.IndexStatus.TRANSFORMED, catalogManager.getFileManager().get(studyId, inputFile2.getPath(), null, sessionId).first().getIndex().getStatus().getName()); loadFile(transformFile, new QueryOptions(), outputId); Assert.assertNull(catalogManager.getFileManager().get(studyId, inputFile1.getPath(), null, sessionId).first().getIndex().getStatus()); Assert.assertEquals(FileIndex.IndexStatus.READY, catalogManager.getFileManager().get(studyId, inputFile2.getPath(), null, sessionId).first().getIndex().getStatus().getName()); } }
@Test public void testBySteps1() throws Exception { File transformFile = transformFile(inputFile1, new QueryOptions()); Assert.assertEquals(FileIndex.IndexStatus.TRANSFORMED, catalogManager.getFileManager().get(studyId, inputFile1.getPath(), null, sessionId).first().getIndex().getStatus().getName()); Assert.assertNull(catalogManager.getFileManager().get(studyId, inputFile2.getPath(), null, sessionId).first().getIndex().getStatus()); loadFile(transformFile, new QueryOptions(), outputId); Assert.assertEquals(FileIndex.IndexStatus.READY, catalogManager.getFileManager().get(studyId, inputFile1.getPath(), null, sessionId).first().getIndex().getStatus().getName()); Assert.assertNull(catalogManager.getFileManager().get(studyId, inputFile2.getPath(), null, sessionId).first().getIndex().getStatus()); }
@Test public void testIndex2() throws Exception { indexFile(inputFile2, new QueryOptions(), outputId); Assert.assertNull(catalogManager.getFileManager().get(studyId, inputFile1.getPath(), null, sessionId).first().getIndex().getStatus()); Assert.assertEquals(FileIndex.IndexStatus.READY, catalogManager.getFileManager().get(studyId, inputFile2.getPath(), null, sessionId).first().getIndex().getStatus().getName()); }
public QueryResult<FileIndex> updateFileIndexStatus(File file, String newStatus, String message, Integer release, String sessionId) throws CatalogException { String userId = catalogManager.getUserManager().getUserId(sessionId); Long studyId = file.getStudyUid(); authorizationManager.checkFilePermission(studyId, file.getUid(), userId, FileAclEntry.FilePermissions.WRITE); FileIndex index = file.getIndex(); if (index != null) { if (!FileIndex.IndexStatus.isValid(newStatus)) { throw new CatalogException("The status " + newStatus + " is not a valid status."); } else { index.setStatus(new FileIndex.IndexStatus(newStatus, message)); } } else { index = new FileIndex(userId, TimeUtils.getTime(), new FileIndex.IndexStatus(newStatus), -1, new ObjectMap()); } if (release != null) { if (newStatus.equals(FileIndex.IndexStatus.READY)) { index.setRelease(release); } } ObjectMap params = new ObjectMap(FileDBAdaptor.QueryParams.INDEX.key(), index); fileDBAdaptor.update(file.getUid(), params, QueryOptions.empty()); auditManager.recordUpdate(AuditRecord.Resource.file, file.getUid(), userId, params, null, null); return new QueryResult<>("Update file index", 0, 1, 1, "", "", Arrays.asList(index)); }
protected void checkEtlResults(String studyId, List<StoragePipelineResult> etlResults, String expectedStatus) throws CatalogException { for (StoragePipelineResult etlResult : etlResults) { File input = catalogManager.getFileManager().get(studyId, new Query(FileDBAdaptor.QueryParams.URI.key(), etlResult.getInput()), null, sessionId).first(); String indexedFileId; if (input.getRelatedFiles().isEmpty()) { indexedFileId = input.getId(); } else { long indexedFileUid = input.getRelatedFiles().get(0).getFileId(); indexedFileId = catalogManager.getFileManager().get(studyId, new Query(FileDBAdaptor.QueryParams.UID.key(), indexedFileUid), new QueryOptions(), sessionId).first().getId(); } assertEquals(expectedStatus, catalogManager.getFileManager().get(studyId, indexedFileId, null, sessionId).first().getIndex().getStatus().getName()); System.out.println("etlResult = " + etlResult); } }
@Test public void testByStepsMultiRelease() throws Exception { List<File> inputFiles = new ArrayList<>(); File transformFile; for (int i = 77; i <= 79; i++) { inputFiles.add(create("platinum/1K.end.platinum-genomes-vcf-NA128" + i + "_S1.genome.vcf.gz")); } for (File inputFile : inputFiles) { transformFile = transformFile(inputFile, new QueryOptions()); loadFile(transformFile, new QueryOptions(), outputId); opencga.getCatalogManager().getProjectManager().incrementRelease(projectAlias, sessionId); } int i = 1; for (File inputFile : inputFiles) { inputFile = opencga.getCatalogManager().getFileManager().get(studyId, inputFile.getId(), null, sessionId).first(); assertEquals(1, inputFile.getRelease()); assertEquals(i, inputFile.getIndex().getRelease()); i++; } variantManager.iterator(new Query(VariantQueryParam.STUDY.key(), studyId), new QueryOptions(), sessionId).forEachRemaining(variant -> { System.out.println("variant = " + variant); }); }
variantManager.index(studyId, inputFile.getId(), outdir, queryOptions, sessionId); inputFile = catalogManager.getFileManager().get(studyId, inputFile.getId(), null, sessionId).first(); assertEquals(FileIndex.IndexStatus.TRANSFORMED, inputFile.getIndex().getStatus().getName()); assertNotNull(inputFile.getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); assertEquals(inputFile.getUid(), relatedFiles.get(0).getFileId()); assertEquals(transformedFile.getUid(), inputFile.getIndex().getTransformedFile().getId());
@Test public void testIndexWithStatsWrongAggregationType() throws Exception { QueryOptions queryOptions = new QueryOptions(VariantStorageEngine.Options.ANNOTATE.key(), false); queryOptions.put(VariantStorageEngine.Options.CALCULATE_STATS.key(), true); queryOptions.put(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), "wrong_type"); queryOptions.putIfNotNull(StorageOperation.CATALOG_PATH, outputId); try { variantManager.index(studyId, getFile(0).getId(), newTmpOutdir(), queryOptions, sessionId); fail("Expected StoragePipelineException exception"); } catch (StoragePipelineException e) { assertEquals(0, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.NONE, getDefaultCohort(studyId).getStatus().getName()); assertEquals(FileIndex.IndexStatus.TRANSFORMED, catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getIndex().getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); } queryOptions.put(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), "none"); // File already transformed queryOptions.put(VariantFileIndexerStorageOperation.LOAD, true); variantManager.index(studyId, getFile(0).getId(), newTmpOutdir(), queryOptions, sessionId); assertEquals(500, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.READY, getDefaultCohort(studyId).getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); }