public File(String name, Type type, Format format, Bioformat bioformat, URI uri, String path, String checksum, String creationDate, String modificationDate, String description, FileStatus status, boolean external, long size, Software software, Experiment experiment, List<Sample> samples, Job job, List<RelatedFile> relatedFiles, FileIndex index, int release, List<AnnotationSet> annotationSets, Map<String, Object> stats, Map<String, Object> attributes) { this.id = StringUtils.isNotEmpty(path) ? StringUtils.replace(path, "/", ":") : path; this.name = name; this.type = type; this.format = format; this.bioformat = bioformat; this.uri = uri; this.path = path; this.checksum = checksum; this.creationDate = creationDate; this.modificationDate = modificationDate; this.description = description; this.status = status; this.release = release; this.external = external; this.size = size; this.software = software; this.experiment = experiment; this.samples = samples; this.tags = Collections.emptyList(); this.job = job; this.relatedFiles = relatedFiles; this.index = index != null ? index : new FileIndex(); this.annotationSets = annotationSets; this.stats = stats; this.attributes = attributes; }
if (file.getStatus().getName().equals(File.FileStatus.READY) && file.getFormat().equals(File.Format.VCF)) { String indexStatus; if (file.getIndex() != null && file.getIndex().getStatus() != null && file.getIndex().getStatus().getName() != null) { indexStatus = file.getIndex().getStatus().getName(); } else { indexStatus = FileIndex.IndexStatus.NONE;
public boolean hasTransformedFile(FileIndex index) { return index.getTransformedFile() != null && index.getTransformedFile().getId() > 0; }
@Override protected List<String> validate(String defaultStudyStr, List<String> values, Integer release, VariantQueryParam param, String sessionId) throws CatalogException { if (release == null) { AbstractManager.MyResources<File> uids = catalogManager.getFileManager().getUids(values, defaultStudyStr, sessionId); return uids.getResourceList().stream().map(File::getName).collect(Collectors.toList()); } else { return validate(defaultStudyStr, values, release, param, catalogManager.getFileManager(), File::getName, file -> ((int) file.getIndex().getRelease()), file -> { if (file.getIndex() == null || file.getIndex().getStatus() == null || file.getIndex().getStatus().getName() == null || !file.getIndex().getStatus().getName().equals(Status.READY)) { throw new VariantQueryException("File '" + file.getName() + "' is not indexed"); } }, sessionId); } } }
&& file.getIndex().getTransformedFile() != null && file.getIndex().getTransformedFile().getId() != transformedFile.getUid()); || file.getIndex().getStatus() == null || file.getIndex().getStatus().getName() == null || file.getIndex().getStatus().getName().equals(FileIndex.IndexStatus.NONE)); if (index.getTransformedFile() == null) { index.setTransformedFile(new FileIndex.TransformedFile(transformedFile.getUid(), json.getUid())); if (vcf.getIndex() != null && vcf.getIndex().getStatus() != null && vcf.getIndex().getStatus().getName() != null) { status = vcf.getIndex().getStatus().getName(); index.setStatus(new FileIndex.IndexStatus(FileIndex.IndexStatus.TRANSFORMED, "Found transformed file"));
numFiles++; File file = iterator.next(); String status = file.getIndex() == null || file.getIndex().getStatus() == null ? FileIndex.IndexStatus.NONE : file.getIndex().getStatus().getName(); if (!status.equals(FileIndex.IndexStatus.READY)) { final FileIndex index; index = file.getIndex() == null ? new FileIndex() : file.getIndex(); if (index.getStatus() == null) { index.setStatus(new FileIndex.IndexStatus()); file.getIndex().getStatus().getName(), FileIndex.IndexStatus.READY); index.getStatus().setName(FileIndex.IndexStatus.READY); catalogManager.getFileManager().setFileIndex(studyConfiguration.getStudyName(), file.getPath(), index, sessionId); index = file.getIndex() == null ? new FileIndex() : file.getIndex(); String prevStatus = index.getStatus().getName(); String newStatus; if (hasTransformedFile(index)) {
switch (index.getStatus().getName()) { case FileIndex.IndexStatus.NONE: case FileIndex.IndexStatus.TRANSFORMED: indexStatusMessage = "Unexpected index status. Expected " + FileIndex.IndexStatus.TRANSFORMING + ", " + FileIndex.IndexStatus.LOADING + " or " + FileIndex.IndexStatus.INDEXING + " and got " + index.getStatus(); logger.warn(indexStatusMessage); case FileIndex.IndexStatus.READY: //Do not show warn message when index status is READY. indexStatusName = index.getStatus().getName(); break; case FileIndex.IndexStatus.TRANSFORMING: case FileIndex.IndexStatus.LOADING: if (jobFailed) { if (indexedFile.getIndex().getTransformedFile() == null) { indexStatusName = FileIndex.IndexStatus.NONE; } else { } else if (indexedFile.getIndex().getTransformedFile() != null) { throw new IllegalStateException("Unknown Index Status " + index.getStatus().getName());
public QueryResult<FileIndex> updateFileIndexStatus(File file, String newStatus, String message, Integer release, String sessionId) throws CatalogException { String userId = catalogManager.getUserManager().getUserId(sessionId); Long studyId = file.getStudyUid(); authorizationManager.checkFilePermission(studyId, file.getUid(), userId, FileAclEntry.FilePermissions.WRITE); FileIndex index = file.getIndex(); if (index != null) { if (!FileIndex.IndexStatus.isValid(newStatus)) { throw new CatalogException("The status " + newStatus + " is not a valid status."); } else { index.setStatus(new FileIndex.IndexStatus(newStatus, message)); } } else { index = new FileIndex(userId, TimeUtils.getTime(), new FileIndex.IndexStatus(newStatus), -1, new ObjectMap()); } if (release != null) { if (newStatus.equals(FileIndex.IndexStatus.READY)) { index.setRelease(release); } } ObjectMap params = new ObjectMap(FileDBAdaptor.QueryParams.INDEX.key(), index); fileDBAdaptor.update(file.getUid(), params, QueryOptions.empty()); auditManager.recordUpdate(AuditRecord.Resource.file, file.getUid(), userId, params, null, null); return new QueryResult<>("Update file index", 0, 1, 1, "", "", Arrays.asList(index)); }
while (iterator.hasNext()) { File next = iterator.next(); String status = next.getIndex().getStatus().getName(); switch (status) { case FileIndex.IndexStatus.READY: next.getIndex().setTransformedFile(null); filesToUpdate.put(next.getUid(), next.getIndex()); break; case FileIndex.IndexStatus.TRANSFORMED: next.getIndex().setTransformedFile(null); next.getIndex().getStatus().setName(FileIndex.IndexStatus.NONE); filesToUpdate.put(next.getUid(), next.getIndex()); break;
@Test public void testByStepsMultiRelease() throws Exception { List<File> inputFiles = new ArrayList<>(); File transformFile; for (int i = 77; i <= 79; i++) { inputFiles.add(create("platinum/1K.end.platinum-genomes-vcf-NA128" + i + "_S1.genome.vcf.gz")); } for (File inputFile : inputFiles) { transformFile = transformFile(inputFile, new QueryOptions()); loadFile(transformFile, new QueryOptions(), outputId); opencga.getCatalogManager().getProjectManager().incrementRelease(projectAlias, sessionId); } int i = 1; for (File inputFile : inputFiles) { inputFile = opencga.getCatalogManager().getFileManager().get(studyId, inputFile.getId(), null, sessionId).first(); assertEquals(1, inputFile.getRelease()); assertEquals(i, inputFile.getIndex().getRelease()); i++; } variantManager.iterator(new Query(VariantQueryParam.STUDY.key(), studyId), new QueryOptions(), sessionId).forEachRemaining(variant -> { System.out.println("variant = " + variant); }); }
variantManager.index(studyId, inputFile.getId(), outdir, queryOptions, sessionId); inputFile = catalogManager.getFileManager().get(studyId, inputFile.getId(), null, sessionId).first(); assertEquals(FileIndex.IndexStatus.TRANSFORMED, inputFile.getIndex().getStatus().getName()); assertNotNull(inputFile.getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); assertEquals(inputFile.getUid(), relatedFiles.get(0).getFileId()); assertEquals(transformedFile.getUid(), inputFile.getIndex().getTransformedFile().getId());
if (fileAux.getIndex() != null && fileAux.getIndex().getStatus() != null && !FileIndex.IndexStatus.NONE.equals(fileAux.getIndex().getStatus().getName()) && !FileIndex.IndexStatus.TRANSFORMED.equals(fileAux.getIndex().getStatus().getName())) { throw new CatalogException("Cannot delete file: " + fileAux.getName() + ". The index status is " + fileAux.getIndex().getStatus().getName()); while (iterator.hasNext()) { File next = iterator.next(); String status = next.getIndex().getStatus().getName(); switch (status) { case FileIndex.IndexStatus.READY: next.getIndex().setTransformedFile(null); break; case FileIndex.IndexStatus.TRANSFORMED: next.getIndex().setTransformedFile(null); next.getIndex().getStatus().setName(FileIndex.IndexStatus.NONE); break; case FileIndex.IndexStatus.NONE:
public File(String name, Type type, Format format, Bioformat bioformat, String path, String description, FileStatus status, long size, int release) { this(name, type, format, bioformat, null, path, null, TimeUtils.getTime(), TimeUtils.getTime(), description, status, false, size, null, new Experiment(), Collections.emptyList(), new Job(), Collections.emptyList(), new FileIndex(), release, Collections.emptyList(), Collections.emptyMap(), Collections.emptyMap()); }
private long getTransformedFileIdFromOriginal(File file) throws CatalogException { long transformedFile = file.getIndex() != null && file.getIndex().getTransformedFile() != null ? file.getIndex().getTransformedFile().getId() : -1; if (transformedFile == -1) { logger.error("This code should never be executed. Every vcf file containing the transformed status should have" + " a registered transformed file"); throw new CatalogException("Internal error. No transformed file could be found for file " + file.getUid()); } return transformedFile; }
public List<File> removeFiles(StudyInfo studyInfo, QueryOptions options, String sessionId) throws CatalogException, StorageEngineException, IOException { // We get the credentials of the Datastore to insert the variants DataStore dataStore = studyInfo.getDataStores().get(File.Bioformat.VARIANT); // Update study configuration BEFORE executing the operation and fetching files from Catalog updateCatalogFromStudyConfiguration(sessionId, studyInfo.getStudyFQN(), dataStore); List<String> fileNames = new ArrayList<>(studyInfo.getFileInfos().size()); List<String> filePaths = new ArrayList<>(studyInfo.getFileInfos().size()); for (FileInfo fileInfo : studyInfo.getFileInfos()) { File file = catalogManager.getFileManager().get(studyInfo.getStudyFQN(), fileInfo.getPath(), null, sessionId).first(); if (file.getIndex().getStatus().getName().equals(FileIndex.IndexStatus.READY)) { fileNames.add(fileInfo.getName()); filePaths.add(fileInfo.getPath()); } else { throw new CatalogException("Unable to remove variants from file " + file.getName() + ". " + "IndexStatus = " + file.getIndex().getStatus().getName()); } } if (fileNames.isEmpty()) { throw new CatalogException("Nothing to do!"); } VariantStorageEngine variantStorageEngine = getVariantStorageEngine(dataStore); variantStorageEngine.getOptions().putAll(options); variantStorageEngine.removeFiles(studyInfo.getStudyFQN(), fileNames); // Update study configuration to synchronize updateCatalogFromStudyConfiguration(sessionId, studyInfo.getStudyFQN(), dataStore); return catalogManager.getFileManager().get(studyInfo.getStudyFQN(), new Query(FileDBAdaptor.QueryParams.PATH.key(), filePaths), new QueryOptions(), sessionId) .getResult(); }
public File(Type type, Format format, Bioformat bioformat, String path, String description, FileStatus status, long size, List<Sample> samples, long jobId, Software software, Map<String, Object> stats, Map<String, Object> attributes) { this("", type, format, bioformat, null, path, null, TimeUtils.getTime(), TimeUtils.getTime(), description, status, false, size, software, new Experiment(), samples, new Job().setUid(jobId), Collections.emptyList(), new FileIndex(), -1, Collections.emptyList(), stats, attributes); }
assertNotNull(inputFile.getIndex().getTransformedFile()); String transformedFileId = catalogManager.getFileManager().get(studyId, new Query(FileDBAdaptor.QueryParams.UID.key(), inputFile.getIndex().getTransformedFile().getId()), new QueryOptions(), sessionId).first().getId();
private void removeFile(List<File> files, QueryOptions options, String outputId) throws Exception { List<String> fileIds = files.stream().map(File::getId).collect(Collectors.toList()); Study study = catalogManager.getFileManager().getStudy(files.get(0), sessionId); String studyId = study.getFqn(); List<File> removedFiles = variantManager.removeFile(fileIds, studyId, sessionId, new QueryOptions()); assertEquals(files.size(), removedFiles.size()); Cohort all = catalogManager.getCohortManager().get(studyId, new Query(CohortDBAdaptor.QueryParams.ID.key(), StudyEntry.DEFAULT_COHORT), null, sessionId).first(); Set<Long> allSampleIds = all.getSamples().stream().map(Sample::getUid).collect(Collectors.toSet()); assertThat(all.getStatus().getName(), anyOf(is(Cohort.CohortStatus.INVALID), is(Cohort.CohortStatus.NONE))); Set<Long> loadedSamples = catalogManager.getFileManager().get(studyId, new Query(FileDBAdaptor.QueryParams.INDEX_STATUS_NAME.key (), FileIndex.IndexStatus.READY), null, sessionId) .getResult() .stream() .flatMap(f -> f.getSamples().stream()) .map(Sample::getUid) .collect(Collectors.toSet()); assertEquals(loadedSamples, allSampleIds); for (File file : removedFiles) { assertEquals(FileIndex.IndexStatus.TRANSFORMED, file.getIndex().getStatus().getName()); } }
file.setIndex(new FileIndex("", "", new FileIndex.IndexStatus(Status.READY, ""), -1, Collections.emptyMap())); file.setSamples(samples);