public FileDaemon(int period, int deleteDelay, String sessionId, CatalogManager catalogManager) throws CatalogDBException { super(period, sessionId, catalogManager); this.deleteDelay = deleteDelay; this.deleteDelayMillis = (long) (deleteDelay * 24 * 60 * 60 * 1000); this.catalogFileUtils = new FileUtils(catalogManager); }
/** * Upload file to a created entry file in Catalog. * * @param sourceUri File URI to be moved into Catalog workspace * @param file File from Catalog * @param sourceChecksum Optional: Source checksum * @param sessionId Valid sessionID to modify the File in Catalog * @param ignoreStatus Ignore the status (uploading, uploaded, ready) from Catalog * @param overwrite Overwrite if there is a file in the target * @param deleteSource After moving, delete file. If false, force copy. * @param calculateChecksum Calculate checksum * @throws CatalogException CatalogException */ public void upload(URI sourceUri, File file, String sourceChecksum, String sessionId, boolean ignoreStatus, boolean overwrite, boolean deleteSource, boolean calculateChecksum) throws CatalogException { upload(sourceUri, file, sourceChecksum, sessionId, ignoreStatus, overwrite, deleteSource, calculateChecksum, Long.MAX_VALUE); }
public static File.Bioformat detectBioformat(URI uri) { return detectBioformat(uri, detectFormat(uri), detectCompression(uri)); }
@Deprecated /** * Link an external file into Catalog. Adds an URI to the field File::uri * A file with a uri is an "external" file. Files inside external folders are not external. * * @param file Already existing file. Should be "STAGE" or "MISSING", or have relink == true * @param calculateChecksum Calculate checksum from the new linked file * @param externalUri External file to link * @param createFoundFiles Create a simple catalog file entry for found files. Used only for link folders * @param relink Change the existing link of a file. * @param sessionId User's sessionId * @return Modified file * @throws CatalogException CatalogException */ public File link(File file, boolean calculateChecksum, URI externalUri, boolean createFoundFiles, boolean relink, String sessionId) throws CatalogException { ParamUtils.checkObj(file, "file"); ParamUtils.checkObj(externalUri, "externalUri"); checkCanLinkFile(file, relink); if (file.getType() == File.Type.DIRECTORY) { return linkFolder(file, calculateChecksum, externalUri, createFoundFiles, relink, sessionId); } else { return linkFile(file, calculateChecksum, externalUri, sessionId); } }
@Deprecated /** * Creates and links an external folder. * * @param studyId StudyId where to place the folder * @param filePath Path to create the folder * @param parents Make parent directories as needed * @param description Folder description * @param calculateChecksum Calculate checksum from the new linked file * @param externalUri External file to link * @param createFoundFiles Create a simple catalog file entry for found files. * @param relink Change the existing link of a file. * @param sessionId User's sessionId * @return Created folder * @throws CatalogException CatalogException */ public File linkFolder(long studyId, String filePath, boolean parents, String description, boolean calculateChecksum, URI externalUri, boolean createFoundFiles, boolean relink, String sessionId) throws CatalogException { ParamUtils.checkObj(externalUri, "externalUri"); File folder = catalogManager.getFileManager().createFolder(Long.toString(studyId), Paths.get(filePath).toString(), new File.FileStatus(File.FileStatus.STAGE), parents, description, null, sessionId).first(); checkCanLinkFile(folder, relink); return linkFolder(folder, calculateChecksum, externalUri, createFoundFiles, relink, sessionId); }
public File createFile(String studyId, String resourceName, String sessionId) throws IOException, CatalogException { File file; URI uri = getResourceUri(resourceName); CatalogManager catalogManager = getCatalogManager(); file = new FileMetadataReader(catalogManager).create(studyId, uri, "data/vcfs/", "", true, null, sessionId).first(); new FileUtils(catalogManager).upload(uri, file, null, sessionId, false, false, true, false, Long.MAX_VALUE); return catalogManager.getFileManager().get(studyId, file.getId(), null, sessionId).first(); }
File newFile = catalogManager.getFileManager().create(study.getFqn(), File.Type.FILE, null, null, relativePath, null, "", null, 0, -1, null, (long) -1, null, null, true, null, null, sessionId).first(); upload(uri, newFile, null, sessionId, false, false, false, calculateChecksum); checkFile(study.getFqn(), existingFile, calculateChecksum, sessionId); updateFileAttributes(folder, null, externalUri, objectMap, sessionId); return catalogManager.getFileManager().get(folder.getUid(), null, sessionId).first();
returnedFile = catalogFileUtils.checkFile(studyFqn, file, true, userSessionId); catalogFileUtils.upload(sourceUri, file, null, userSessionId, false, false, false, true); fileUri = catalogManager.getFileManager().getUri(file); file = catalogManager.getFileManager().get(studyFqn, file.getPath(), null, userSessionId).first(); returnedFile = catalogFileUtils.checkFile(studyFqn, file, true, userSessionId); returnedFile = catalogFileUtils.checkFile(studyFqn, file, true, userSessionId); returnedFile = catalogFileUtils.checkFile(studyFqn, file, true, userSessionId); os.write('\n'); os.close(); returnedFile = catalogFileUtils.checkFile(studyFqn, file, true, userSessionId); returnedFile = catalogFileUtils.checkFile(studyFqn, file, true, userSessionId); returnedFile = catalogFileUtils.checkFile(studyFqn, file, true, userSessionId);
targetChecksum = sourceChecksum; updateFileAttributes(file, targetChecksum, targetUri, new ObjectMap("status.name", File.FileStatus.READY), sessionId); return; checkStatus(file); checkTarget(file, targetUri, targetIOManager, overwrite); copy(sourceIOManager, sourceUri, targetIOManager, targetUri); } catch (CatalogIOException | IOException e) { throw new CatalogIOException("Error while copying file. ", e); updateFileAttributes(file, sourceChecksum, targetUri, new ObjectMap(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.READY), sessionId);
File.Format format = FileUtils.detectFormat(fileUri); File.Bioformat bioformat = FileUtils.detectBioformat(fileUri); modifyParams.putAll(catalogFileUtils.getModifiedFileAttributes(file, fileUri, false));
@Test public void detectAlignment() { assertEquals(File.Bioformat.ALIGNMENT, FileUtils.detectBioformat(URI.create("file:///test.bam"))); assertEquals(File.Bioformat.ALIGNMENT, FileUtils.detectBioformat(URI.create("file:///test.sam"))); assertEquals(File.Bioformat.ALIGNMENT, FileUtils.detectBioformat(URI.create("file:///test.cram"))); assertEquals(File.Bioformat.ALIGNMENT, FileUtils.detectBioformat(URI.create("file:///test.sam.gz"))); assertEquals(File.Bioformat.ALIGNMENT, FileUtils.detectBioformat(URI.create("file:///test.bam.alignments.json"))); assertEquals(File.Bioformat.ALIGNMENT, FileUtils.detectBioformat(URI.create("file:///test.bam.alignments.json.gz"))); assertEquals(File.Bioformat.ALIGNMENT, FileUtils.detectBioformat(URI.create("file:///test.bam.alignments.json.snz"))); assertEquals(File.Bioformat.ALIGNMENT, FileUtils.detectBioformat(URI.create("file:///test.bam.alignments.avro"))); assertEquals(File.Bioformat.ALIGNMENT, FileUtils.detectBioformat(URI.create("file:///test.bam.alignments.avro.gz"))); assertEquals(File.Bioformat.ALIGNMENT, FileUtils.detectBioformat(URI.create("file:///test.bam.alignments.avro.snz"))); }
File.Format format = FileUtils.detectFormat(fileUri); File.Bioformat bioformat = FileUtils.detectBioformat(fileUri);
checkStatus(file); checkTarget(file, targetUri, targetIOManager, overwrite); updateFileAttributes(file, checksum, targetUri, new ObjectMap(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.READY), sessionId);
/** * Check tracking from all files from a study. * * Set file status {@link File.FileStatus#MISSING} if the file (fileUri) is unreachable * Set file status to {@link File.FileStatus#READY} if was {@link File.FileStatus#MISSING} and file (fileUri) is reachable * * @param study The study to check * @param sessionId User sessionId * @param calculateChecksum Calculate checksum for "found files" * @throws CatalogException if a Catalog error occurs * @return found and lost files */ public List<File> checkStudyFiles(Study study, boolean calculateChecksum, String sessionId) throws CatalogException { Query query = new Query(); query.put(FileDBAdaptor.QueryParams.STATUS_NAME.key(), Arrays.asList( File.FileStatus.READY, File.FileStatus.MISSING, File.FileStatus.TRASHED)); QueryResult<File> files = catalogManager.getFileManager().get(study.getFqn(), query, new QueryOptions(), sessionId); List<File> modifiedFiles = new LinkedList<>(); for (File file : files.getResult()) { File checkedFile = catalogFileUtils.checkFile(study.getFqn(), file, calculateChecksum, sessionId); if (checkedFile != file) { modifiedFiles.add(checkedFile); } } return modifiedFiles; }
@Deprecated public void delete(long fileId, String sessionId) throws CatalogException { delete(catalogManager.getFileManager().get(fileId, null, sessionId).first(), sessionId); }
@Test public void testDownloadAndHeadFile() throws CatalogException, IOException, InterruptedException { FileUtils catalogFileUtils = new FileUtils(catalogManager); File file = catalogManager.getFileManager().create(studyFqn, File.Type.FILE, File.Format.PLAIN, File.Bioformat .VARIANT, "data/" + fileName, null, "description", null, 0, -1, null, (long) -1, null, null, true, null, null, sessionIdUser).first(); catalogFileUtils.upload(is, file, sessionIdUser, false, false, true); is.close();
@Test public void detectSequence() { assertEquals(File.Bioformat.SEQUENCE, FileUtils.detectBioformat(URI.create("file:///test.fastq"))); assertEquals(File.Bioformat.SEQUENCE, FileUtils.detectBioformat(URI.create("file:///test.fastq.gz"))); }
new org.opencb.opencga.catalog.managers.FileUtils(catalogManager).upload(sourceUri, fileQueryResult.first(), null, sessionId, ignoreStatus, overwrite, deleteSource, calculateChecksum, Long.MAX_VALUE);
public FileScanner(CatalogManager catalogManager) { this.catalogManager = catalogManager; catalogFileUtils = new FileUtils(catalogManager); }