ParamUtils.checkPath(file.getPath(), "path"); file.setType(ParamUtils.defaultObject(file.getType(), File.Type.FILE)); file.setFormat(ParamUtils.defaultObject(file.getFormat(), File.Format.PLAIN)); file.setBioformat(ParamUtils.defaultObject(file.getBioformat(), File.Bioformat.NONE)); file.setDescription(ParamUtils.defaultString(file.getDescription(), "")); file.setRelatedFiles(ParamUtils.defaultObject(file.getRelatedFiles(), ArrayList::new)); file.setCreationDate(TimeUtils.getTime()); file.setModificationDate(file.getCreationDate()); if (file.getType() == File.Type.FILE) { file.setStatus(ParamUtils.defaultObject(file.getStatus(), new File.FileStatus(File.FileStatus.STAGE))); } else { file.setStatus(ParamUtils.defaultObject(file.getStatus(), new File.FileStatus(File.FileStatus.READY))); if (file.getSize() < 0) { throw new CatalogException("Error: DiskUsage can't be negative!"); file.setSamples(ParamUtils.defaultObject(file.getSamples(), ArrayList::new)); for (Sample sample : file.getSamples()) { if (sample.getUid() <= 0 || !sampleDBAdaptor.exists(sample.getUid())) { throw new CatalogException("Sample { id: " + sample.getUid() + "} does not exist."); if (file.getJob() != null && file.getJob().getUid() > 0 && !jobDBAdaptor.exists(file.getJob().getUid())) { throw new CatalogException("Job { id: " + file.getJob().getUid() + "} does not exist."); file.setStats(ParamUtils.defaultObject(file.getStats(), HashMap::new)); file.setAttributes(ParamUtils.defaultObject(file.getAttributes(), HashMap::new)); if (file.getType() == File.Type.DIRECTORY && !file.getPath().endsWith("/")) { file.setPath(file.getPath() + "/");
if (VariantReaderUtils.isTransformedVariants(file.getName())) { if (transformedFiles != null) { if (!avroUri.equals(file.getUri())) { throw new CatalogException("The avro file " + file.getName() + " in " + file.getUri() + " has been associated a " + "different uri " + avroUri + " by the user."); if (file.getFormat().equals(File.Format.VCF) || file.getFormat().equals(File.Format.GVCF)) { String status = file.getIndex() == null || file.getIndex().getStatus() == null ? FileIndex.IndexStatus.NONE : file.getIndex().getStatus().getName(); switch (status) { case FileIndex.IndexStatus.NONE: fileUris.add(UriUtils.createUri(transformedFiles.get(i))); } else { logger.warn("Cannot load vcf file " + file.getName() + " if no avro file is provided."); case FileIndex.IndexStatus.LOADING: if (!resume) { logger.warn("Unable to load this file. Already being loaded. Skipping file {}", file.getName()); break; transformedToOrigFileIdsMap.put(getTransformedFileIdFromOriginal(file), file.getUid()); } else { fileUris.add(transformed.getUri()); logger.warn("We can only load files previously transformed. Skipping file {}", file.getName()); break;
@Deprecated public QueryResult<Job> create(long studyId, String name, String toolName, String description, String executor, Map<String, String> params, String commandLine, URI tmpOutDirUri, long outDirId, List<File> inputFiles, List<File> outputFiles, Map<String, Object> attributes, Map<String, Object> resourceManagerAttributes, Job.JobStatus status, long startTime, long endTime, QueryOptions options, String sessionId) throws CatalogException { Job job = new Job(-1, null, name, "", toolName, null, "", description, startTime, endTime, executor, "", commandLine, false, status, -1, new File().setUid(outDirId), inputFiles, outputFiles, Collections.emptyList(), params, -1, attributes, resourceManagerAttributes); return create(String.valueOf(studyId), job, options, sessionId); }
/** * Check status. To upload a fileObject, the catalog file entry must be in status "uploading". * Set "ignoreStatus = true" to ignore the actual file status and replace the content. * * @throws org.opencb.opencga.catalog.exceptions.CatalogIOException */ private void checkStatus(File file) throws CatalogIOException { if (!file.getStatus().getName().equals(File.FileStatus.STAGE)) { throw new CatalogIOException("File status is already uploaded and ready! " + "file:{ path: '" + file.getPath() + "', id:" + file.getPath() + ", status: '" + file.getStatus().getName() + "' } " + "Needs 'ignoreStatus = true' for continue."); } }
private void checkAlignmentBioformat(File file) throws CatalogException { if (!file.getBioformat().equals(File.Bioformat.ALIGNMENT)) { throw new CatalogException("File " + file.getName() + " not supported. Expecting an alignment file."); } }
fileSolrModel.setUid(file.getUid()); fileSolrModel.setName(file.getName()); fileSolrModel.setStudyId(study.getFqn().replace(":", "__")); fileSolrModel.setType(file.getType().name()); if (file.getFormat() != null) { fileSolrModel.setFormat(file.getFormat().name()); if (file.getBioformat() != null) { fileSolrModel.setBioformat(file.getBioformat().name()); fileSolrModel.setRelease(file.getRelease()); Date date = TimeUtils.toDate(file.getCreationDate()); LocalDate localDate = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); fileSolrModel.setCreationDay(localDate.getDayOfMonth()); fileSolrModel.setCreationDayOfWeek(localDate.getDayOfWeek().toString()); fileSolrModel.setStatus(file.getStatus().getName()); fileSolrModel.setStatus(file.getStatus().getName()); fileSolrModel.setExternal(file.isExternal()); fileSolrModel.setSize(file.getSize()); if (file.getSoftware() != null) { fileSolrModel.setSoftware(file.getSoftware().getName()); fileSolrModel.setNumSamples(file.getSamples() != null ? file.getSamples().size() : 0); fileSolrModel.setNumRelatedFiles(file.getRelatedFiles() != null ? file.getRelatedFiles().size() : 0); fileSolrModel.setAnnotations(SolrConverterUtil.populateAnnotations(variableMap, file.getAnnotationSets()));
@Test public void FileToSolrTest() { Study study = new Study().setFqn("user@project:study").setAttributes(new HashMap<>()); File file = new File("name", File.Type.FILE, File.Format.BAM, File.Bioformat.MICROARRAY_EXPRESSION_ONECHANNEL_AGILENT, "test/base", "convertorTest", new File.FileStatus("READY"), 1111L, 2); file.setUid(111).setSamples(Arrays.asList(new Sample().setId("1"), new Sample().setId("2"))) .setSoftware(new Software().setName("Software")); file.setAnnotationSets(AnnotationHelper.createAnnotation()); assert (fileSolrModel.getUid() == file.getUid()); assert (fileSolrModel.getName().equals(file.getName())); assert (fileSolrModel.getType().equals(file.getType().name())); assert (fileSolrModel.getFormat().equals(file.getFormat().name())); assert (fileSolrModel.getBioformat().equals(file.getBioformat().name())); assert (fileSolrModel.getRelease() == file.getRelease()); Date date = TimeUtils.toDate(file.getCreationDate()); LocalDate localDate = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); assertEquals(localDate.getDayOfWeek().toString(), fileSolrModel.getCreationDayOfWeek()); assert (fileSolrModel.getStatus().equals(file.getStatus().getName())); assert (fileSolrModel.isExternal() == file.isExternal()); assert (fileSolrModel.getSize() == file.getSize()); assert (fileSolrModel.getSoftware().equals(file.getSoftware().getName())); assert (fileSolrModel.getNumSamples() == file.getSamples().size());
@Test public void testGetMetadataFromVcf() throws CatalogException, FileNotFoundException { File file = catalogManager.getFileManager().upload(study.getFqn(), vcfFileUri, new File().setPath(folder.getPath() + VCF_FILE_NAME), false, false, sessionIdUser).first(); assertTrue(file.getSize() > 0); file = FileMetadataReader.get(catalogManager). setMetadataInformation(file, null, null, sessionIdUser, false); assertEquals(File.FileStatus.READY, file.getStatus().getName()); assertEquals(File.Format.VCF, file.getFormat()); assertEquals(File.Bioformat.VARIANT, file.getBioformat()); assertNotNull(file.getAttributes().get(VARIANT_FILE_METADATA)); assertEquals(4, file.getSamples().size()); assertEquals(expectedSampleNames, ((Map<String, Object>) file.getAttributes().get(VARIANT_FILE_METADATA)).get("sampleIds")); catalogManager.getSampleManager().get(study.getFqn(), new Query(SampleDBAdaptor.QueryParams.ID.key(), file.getSamples().stream().map(Sample::getId).collect(Collectors.toList())), new QueryOptions(), sessionIdUser).getResult(); assertTrue(expectedSampleNames.containsAll(file.getSamples().stream().map(Sample::getId).collect(Collectors.toSet()))); }
switch (field) { case "name": value = file.getName(); break; case "format": value = file.getFormat().name(); break; case "bioformat": value = file.getFormat().name(); break; case "path": value = file.getPath(); break; case "description": value = file.getDescription(); break; case ADD: case SET: file.setDescription(hookConfiguration.getWhat()); break; case REMOVE: file.setDescription(""); break; default: if (file.getTags() != null) { tagsCopy.addAll(file.getTags());
@Test public void testDeleteExisting() throws IOException, CatalogException { QueryResult<File> queryResult = catalogManager.getFileManager().upload(study.getFqn(), CatalogManagerTest.createDebugFile().toURI(), new File().setPath(folder.getPath() + "file1.txt"), false, false, sessionIdUser); File file = queryResult.first(); CatalogManagerTest.createDebugFile(directory.resolve("file1.txt").toString()); List<File> files = new FileScanner(catalogManager).scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.DELETE, false, true, sessionIdUser); files.forEach((File f) -> assertFalse(f.getAttributes().containsKey("checksum"))); assertEquals(File.FileStatus.DELETED, getFile(file.getUid()).getStatus().getName()); }
@Test public void testReplaceExisting() throws IOException, CatalogException { // Create and register file1.txt and s/file2.txt File file = catalogManager.getFileManager().upload(study.getFqn(), CatalogManagerTest.createDebugFile().toURI(), new File().setPath(folder.getPath() + "file1.txt"), false, false, true, true, sessionIdUser).first(); catalogManager.getFileManager().upload(study.getFqn(), CatalogManagerTest.createDebugFile().toURI(), new File().setPath(folder.getPath() + "s/file2.txt"), false, true, true, true, sessionIdUser).first(); // Create same file structure, and replace CatalogManagerTest.createDebugFile(directory.resolve("file1.txt").toString()); Files.createDirectory(directory.resolve("s/")); CatalogManagerTest.createDebugFile(directory.resolve("s/file2.txt").toString()); FileScanner fileScanner = new FileScanner(catalogManager); fileScanner.scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.REPLACE, true, true, sessionIdUser); File replacedFile = catalogManager.getFileManager().get(study.getFqn(), file.getPath(), null, sessionIdUser).first(); assertEquals(File.FileStatus.READY, replacedFile.getStatus().getName()); assertEquals(file.getUid(), replacedFile.getUid()); assertNotEquals(replacedFile.getChecksum(), file.getChecksum()); assertEquals(replacedFile.getChecksum(), catalogManager.getCatalogIOManagerFactory().getDefault().calculateChecksum(replacedFile.getUri())); }
@Test public void deleteFolderTest2() throws CatalogException, IOException { List<File> folderFiles = new LinkedList<>(); File folder = createBasicDirectoryFileTestEnvironment(folderFiles); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(catalogManager.getFileManager().getUri(folder)); for (File file : folderFiles) { assertTrue(ioManager.exists(catalogManager.getFileManager().getUri(file))); } catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()), null, sessionIdUser); Query query = new Query() .append(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()) .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.TRASHED); File fileTmp = catalogManager.getFileManager().get(studyFqn, query, QueryOptions.empty(), sessionIdUser).first(); assertEquals("Folder name should not be modified", folder.getPath(), fileTmp.getPath()); assertEquals("Status should be to TRASHED", File.FileStatus.TRASHED, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", folder.getName(), fileTmp.getName()); assertTrue(ioManager.exists(fileTmp.getUri())); for (File file : folderFiles) { query.put(FileDBAdaptor.QueryParams.UID.key(), file.getUid()); fileTmp = fileManager.get(studyFqn, query, QueryOptions.empty(), sessionIdUser).first(); assertEquals("Folder name should not be modified", file.getPath(), fileTmp.getPath()); assertEquals("Status should be to TRASHED", File.FileStatus.TRASHED, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", file.getName(), fileTmp.getName()); assertTrue("File uri: " + fileTmp.getUri() + " should exist", ioManager.exists(fileTmp.getUri())); } }
indexedFile = catalogManager.getFileManager().get(study.getFqn(), indexedFile.getId(), null, sessionId).first(); StoragePipelineResult storagePipelineResult = map.get(indexedFile.getName()); String indexStatusMessage = null; if (indexedFile.getIndex() != null) { FileIndex index = indexedFile.getIndex(); switch (index.getStatus().getName()) { case FileIndex.IndexStatus.NONE: case FileIndex.IndexStatus.LOADING: if (jobFailed) { if (indexedFile.getIndex().getTransformedFile() == null) { indexStatusName = FileIndex.IndexStatus.NONE; } else { } else if (indexedFile.getIndex().getTransformedFile() != null) { throw new CatalogException("Critical error. Empty index parameter in file " + indexedFile.getUid()); Map<String, Object> attributes = indexedFile.getAttributes(); attributes.put("storagePipelineResult", storagePipelineResult); ObjectMap params = new ObjectMap(FileDBAdaptor.QueryParams.ATTRIBUTES.key(), attributes); fileManager.update(study.getFqn(), indexedFile.getPath(), params, new QueryOptions(), sessionId);
for (File file : files) { FileInfo fileInfo = new FileInfo(); fileInfo.setFileUid(file.getUid()); Path path = Paths.get(file.getUri().getRawPath()); fileInfo.setPath(file.getPath()); fileInfo.setFilePath(path); fileInfo.setName(file.getName()); fileInfo.setBioformat(file.getBioformat()); fileInfo.setFormat(file.getFormat());
@Test public void testCreate() throws CatalogException { QueryResult<File> fileQueryResult = FileMetadataReader.get(catalogManager). create(study.getFqn(), vcfFileUri, folder.getPath() + VCF_FILE_NAME, "", false, null, sessionIdUser); File file = fileQueryResult.first(); assertEquals(File.FileStatus.STAGE, file.getStatus().getName()); assertEquals(File.Format.VCF, file.getFormat()); assertEquals(File.Bioformat.VARIANT, file.getBioformat()); assertNotNull(file.getAttributes().get(VARIANT_FILE_METADATA)); assertEquals(4, file.getSamples().size()); assertEquals(21499, file.getSize()); new FileUtils(catalogManager).upload(vcfFileUri, file, null, sessionIdUser, false, false, true, true, Integer.MAX_VALUE); file = catalogManager.getFileManager().get(study.getFqn(), file.getPath(), null, sessionIdUser).first(); assertEquals(File.FileStatus.READY, file.getStatus().getName()); assertEquals(File.Format.VCF, file.getFormat()); assertEquals(File.Bioformat.VARIANT, file.getBioformat()); assertNotNull(file.getAttributes().get(VARIANT_FILE_METADATA)); assertNotNull(((Map) file.getAttributes().get(VARIANT_FILE_METADATA)).get("sampleIds")); assertEquals(4, ((List) ((Map) file.getAttributes().get(VARIANT_FILE_METADATA)).get("sampleIds")).size()); assertNotNull(((Map) file.getAttributes().get(VARIANT_FILE_METADATA)).get("header")); assertEquals(4, file.getSamples().size()); assertEquals(21499, file.getSize()); }
variantManager.index(studyId, inputFile.getId(), outdir, queryOptions, sessionId); inputFile = catalogManager.getFileManager().get(studyId, inputFile.getId(), null, sessionId).first(); assertEquals(FileIndex.IndexStatus.TRANSFORMED, inputFile.getIndex().getStatus().getName()); assertNotNull(inputFile.getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); .append(FileDBAdaptor.QueryParams.NAME.key(), "~" + inputFile.getName() + ".variants.(json|avro)"); List<File.RelatedFile> relatedFiles = transformedFile.getRelatedFiles().stream() .filter(relatedFile -> relatedFile.getRelation().equals(File.RelatedFile.Relation.PRODUCED_FROM)) .collect(Collectors.toList()); assertEquals(1, relatedFiles.size()); assertEquals(inputFile.getUid(), relatedFiles.get(0).getFileId()); assertEquals(transformedFile.getUid(), inputFile.getIndex().getTransformedFile().getId());
.map(Sample::getUid) .collect(Collectors.toList()); samples.add(files.get(0).getSamples().get(0).getUid()); File nonIndexedFile = files.stream().filter(file -> !indexedFiles.contains(((int) file.getUid()))).findFirst().orElse(null); assertNotNull(nonIndexedFile); sc.getIndexedFiles().add(((int) nonIndexedFile.getUid())); sc.getFileIds().put(nonIndexedFile.getName(), ((int) nonIndexedFile.getUid())); scm.updateStudyConfiguration(sc, null); assertEquals(FileIndex.IndexStatus.READY, catalogManager.getFileManager().get(studyId, nonIndexedFile.getName(), null, sessionId).first().getIndex().getStatus().getName()); nonIndexedFile = files.stream().filter(file -> !indexedFiles.contains(((int) file.getUid()))).findFirst().orElse(null); assertNotNull(nonIndexedFile); sc.getBatches() .add(new BatchFileOperation("LOAD", Collections.singletonList(((int) nonIndexedFile.getUid())), 1L, BatchFileOperation.Type.LOAD) .addStatus(BatchFileOperation.Status.RUNNING)); sc.getFileIds().put(nonIndexedFile.getName(), ((int) nonIndexedFile.getUid())); assertEquals(FileIndex.IndexStatus.INDEXING, catalogManager.getFileManager().get(studyId, nonIndexedFile.getName(), null, sessionId).first().getIndex().getStatus().getName());
@Test public void testLinkFolder4() throws CatalogException, IOException { URI uri = Paths.get(getStudyURI()).resolve("data").toUri(); ObjectMap params = new ObjectMap("parents", true); QueryResult<File> allFiles = link(uri, "test/myLinkedFolder/", studyFqn, params, sessionIdUser); assertEquals(6, allFiles.getNumResults()); QueryResult<File> sameAllFiles = link(uri, "test/myLinkedFolder/", studyFqn, params, sessionIdUser); assertEquals(allFiles.getNumResults(), sameAllFiles.getNumResults()); List<File> result = allFiles.getResult(); for (int i = 0; i < result.size(); i++) { assertEquals(allFiles.getResult().get(i).getUid(), sameAllFiles.getResult().get(i).getUid()); assertEquals(allFiles.getResult().get(i).getPath(), sameAllFiles.getResult().get(i).getPath()); assertEquals(allFiles.getResult().get(i).getUri(), sameAllFiles.getResult().get(i).getUri()); } thrown.expect(CatalogException.class); thrown.expectMessage("already linked"); link(uri, "data", studyFqn, new ObjectMap(), sessionIdUser); }
@Test public void testIndex() throws Exception { URI uri = getClass().getResource("/biofiles/variant-test-file.vcf.gz").toURI(); File file = fileManager.link(studyFqn, uri, "", null, sessionIdUser).first(); assertEquals(4, file.getSamples().size()); assertEquals(File.Format.VCF, file.getFormat()); assertEquals(File.Bioformat.VARIANT, file.getBioformat()); Job job = fileManager.index(studyFqn, Collections.singletonList(file.getName()), "VCF", null, sessionIdUser).first(); assertEquals(file.getUid(), job.getInput().get(0).getUid()); }
if (file.getType() == File.Type.DIRECTORY) { return file; if (format != File.Format.UNKNOWN && !format.equals(file.getFormat())) { modifyParams.put(FileDBAdaptor.QueryParams.FORMAT.key(), format); file.setFormat(format); if (bioformat != File.Bioformat.NONE && !bioformat.equals(file.getBioformat())) { modifyParams.put(FileDBAdaptor.QueryParams.BIOFORMAT.key(), bioformat); file.setBioformat(bioformat); catalogManager.getFileManager().setDiskUsage(study.getFqn(), file.getPath(), modifyParams.getLong(FileDBAdaptor.QueryParams.SIZE.key()), sessionId); modifyParams.remove(FileDBAdaptor.QueryParams.SIZE.key()); catalogManager.getFileManager().setModificationDate(study.getFqn(), file.getPath(), modifyParams.getString(FileDBAdaptor.QueryParams.MODIFICATION_DATE.key()), sessionId); modifyParams.remove(FileDBAdaptor.QueryParams.MODIFICATION_DATE.key()); .setUri(study.getFqn(), file.getPath(), modifyParams.getString(FileDBAdaptor.QueryParams.URI.key()), sessionId); modifyParams.remove(FileDBAdaptor.QueryParams.URI.key()); catalogManager.getFileManager().update(study.getFqn(), file.getPath(), modifyParams, new QueryOptions(), sessionId); return catalogManager.getFileManager().get(study.getFqn(), file.getPath(), options, sessionId).first();