/** * Get a ObjectMap with some fields if they have been modified. * size * modificationDate * checksum * uri * * @param file file * @param fileUri If null, calls to getFileUri() * <p> * TODO: Lazy checksum: Only calculate checksum if the size has changed. * @param calculateChecksum Calculate checksum to check if have changed * @return ObjectMap ObjectMap * @throws CatalogException CatalogException */ public ObjectMap getModifiedFileAttributes(File file, URI fileUri, boolean calculateChecksum) throws CatalogException { if (fileUri == null) { fileUri = catalogManager.getFileManager().getUri(file); } String checksum = null; if (calculateChecksum) { checksum = catalogManager.getCatalogIOManagerFactory().get(fileUri).calculateChecksum(fileUri); } return getModifiedFileAttributes(file, checksum, fileUri, null); }
parameters = ParamUtils.defaultObject(parameters, ObjectMap::new); if (fileUri == null) { fileUri = catalogManager.getFileManager().getUri(file);
/** * Scan the study folder, add all untracked files and check tracking. * * @param study Study to resync * @param calculateChecksum Calculates checksum of all the files in the directory to scan * @param sessionId User sessionId * @return New, lost and found files * @throws CatalogException if a Catalog error occurs * @throws IOException if an I/O error occurs */ public List<File> reSync(Study study, boolean calculateChecksum, String sessionId) throws CatalogException, IOException { // File root = catalogManager.getAllFiles(studyId, new QueryOptions("path", ""), sessionId).first(); Query query = new Query(); query.put(FileDBAdaptor.QueryParams.URI.key(), "~.*"); //Where URI exists query.put(FileDBAdaptor.QueryParams.TYPE.key(), File.Type.DIRECTORY); List<File> files = catalogManager.getFileManager().get(study.getFqn(), query, null, sessionId).getResult(); List<File> scan = new LinkedList<>(); for (File file : files) { scan.addAll(scan(file, catalogManager.getFileManager().getUri(file), FileScannerPolicy.REPLACE, calculateChecksum, false, sessionId)); } // TODO: Scan per file scan.addAll(checkStudyFiles(study, calculateChecksum, sessionId)); return scan; }
public DataInputStream download(String studyStr, String fileStr, int start, int limit, QueryOptions options, String sessionId) throws CatalogException { MyResource<File> resource = getUid(fileStr, studyStr, sessionId); authorizationManager.checkFilePermission(resource.getStudy().getUid(), resource.getResource().getUid(), resource.getUser(), FileAclEntry.FilePermissions.DOWNLOAD); URI fileUri = getUri(resource.getResource()); return catalogIOManagerFactory.get(fileUri).getFileObject(fileUri, start, limit); }
URI fileUri = catalogManager.getFileManager().getUri(file); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(fileUri); if (ioManager.isDirectory(fileUri)) {
URI fileUri = catalogManager.getFileManager().getUri(pedFile); Study study = catalogManager.getFileManager().getStudy(pedFile, sessionId); long auxTime;
public DataInputStream grep(String studyStr, String fileStr, String pattern, QueryOptions options, String sessionId) throws CatalogException { MyResource<File> resource = getUid(fileStr, studyStr, sessionId); authorizationManager.checkFilePermission(resource.getStudy().getUid(), resource.getResource().getUid(), resource.getUser(), FileAclEntry.FilePermissions.VIEW); URI fileUri = getUri(resource.getResource()); boolean ignoreCase = options.getBoolean("ignoreCase"); boolean multi = options.getBoolean("multi"); return catalogIOManagerFactory.get(fileUri).getGrepFileObject(fileUri, pattern, ignoreCase, multi); }
CatalogIOManager targetIOManager; try { targetUri = catalogManager.getFileManager().getUri(file); targetIOManager = catalogManager.getCatalogIOManagerFactory().get(targetUri.getScheme()); } catch (CatalogDBException | CatalogIOException e) {
@Test public void testGetBasicMetadata() throws CatalogException, IOException { File file = catalogManager.getFileManager().create(study.getFqn(), new File().setPath(folder.getPath() + "test.txt"), false, StringUtils.randomString(1000), null, sessionIdUser).first(); assertEquals(1000, file.getSize()); String creationDate = file.getCreationDate(); String modificationDate = file.getModificationDate(); URI fileUri = catalogManager.getFileManager().getUri(file); try { Thread.sleep(1000); //Sleep 1 second to see changes on the "modificationDate" } catch (InterruptedException ignored) {} OutputStream outputStream = new FileOutputStream(Paths.get(fileUri).toFile(), true); byte[] bytes2 = StringUtils.randomString(100).getBytes(); outputStream.write(bytes2); outputStream.close(); file = FileMetadataReader.get(catalogManager). setMetadataInformation(file, null, null, sessionIdUser, false); assertEquals(1000 + bytes2.length, file.getSize()); assertTrue(TimeUtils.toDate(modificationDate).getTime() < TimeUtils.toDate(file.getModificationDate()).getTime()); assertEquals(creationDate, file.getCreationDate()); }
assertTrue(paths.contains("data/new/folder/")); URI uri = catalogManager.getFileManager().getUri(folder); assertTrue(!catalogManager.getCatalogIOManagerFactory().get(uri).exists(uri));
case File.FileStatus.READY: case File.FileStatus.MISSING: { URI fileUri = catalogManager.getFileManager().getUri(file); if (!catalogManager.getCatalogIOManagerFactory().get(fileUri).exists(fileUri)) { logger.warn("File { id:" + file.getPath() + ", path:\"" + file.getPath() + "\" } lost tracking from file " + fileUri); URI fileUri = catalogManager.getFileManager().getUri(file); if (!catalogManager.getCatalogIOManagerFactory().get(fileUri).exists(fileUri)) { modifiedFile = file;
@Test public void testIndexByStepsExternallyTransformed() throws Exception { QueryOptions queryOptions = new QueryOptions(VariantFileIndexerStorageOperation.TRANSFORM, true) // TODO: Should work without isolating transformation? .append(VariantStorageEngine.Options.ISOLATE_FILE_FROM_STUDY_CONFIGURATION.key(), true); // File transformFile = transformFile(getFile(0), queryOptions); String outdir = opencga.createTmpOutdir(studyId, "_TRANSFORM_", sessionId); List<StoragePipelineResult> etlResults = variantManager.index(studyId, getFile(0).getPath(), outdir, queryOptions, sessionId); File transformFile = null; create(studyId2, catalogManager.getFileManager().getUri(getFile(0))); for (java.io.File file : Paths.get(UriUtils.createUri(outdir)).toFile().listFiles()) { File f = create(studyId2, file.toURI()); if (VariantReaderUtils.isTransformedVariants(file.toString())) { assertNull(transformFile); transformFile = f; } } assertNotNull(transformFile); catalogManager.getFileManager().matchUpVariantFiles(studyId2, singletonList(transformFile), sessionId); queryOptions = new QueryOptions().append(VariantStorageEngine.Options.ANNOTATE.key(), false) .append(VariantStorageEngine.Options.CALCULATE_STATS.key(), true); loadFile(transformFile, queryOptions, outputId2); }
@Test public void deleteFoldersTest() throws CatalogException, IOException { List<File> folderFiles = new LinkedList<>(); File folder = prepareFiles(folderFiles); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(catalogManager.getFileManager().getUri(folder)); for (File file : folderFiles) { assertTrue(ioManager.exists(catalogManager.getFileManager().getUri(file))); } catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()), null, userSessionId); Query query = new Query() .append(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()) .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.TRASHED); QueryResult<File> fileQueryResult = catalogManager.getFileManager().get(studyFqn, query, QueryOptions.empty(), userSessionId); assertTrue(ioManager.exists(fileQueryResult.first().getUri())); for (File file : folderFiles) { assertTrue("File uri: " + file.getUri() + " should exist", ioManager.exists(file.getUri())); } }
fileUri = catalogManager.getFileManager().getUri(file); file = catalogManager.getFileManager().get(studyFqn, file.getPath(), null, userSessionId).first(); returnedFile = catalogFileUtils.checkFile(studyFqn, file, true, userSessionId);
@Test public void deleteFolderTest3() throws CatalogException, IOException { List<File> folderFiles = new LinkedList<>(); File folder = createBasicDirectoryFileTestEnvironment(folderFiles); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(catalogManager.getFileManager().getUri(folder)); for (File file : folderFiles) { assertTrue(ioManager.exists(catalogManager.getFileManager().getUri(file))); } catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()), new ObjectMap(FileManager.SKIP_TRASH, true), sessionIdUser); Query query = new Query() .append(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()) .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.PENDING_DELETE); File fileTmp = fileManager.get(studyFqn, query, QueryOptions.empty(), sessionIdUser).first(); String myPath = Paths.get(folder.getPath()) + AbstractManager.INTERNAL_DELIMITER + "DELETED"; assertTrue("Folder name should have been modified", fileTmp.getPath().contains(myPath)); assertEquals("Status should be to PENDING_DELETE", File.FileStatus.PENDING_DELETE, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", folder.getName(), fileTmp.getName()); assertTrue(ioManager.exists(fileTmp.getUri())); for (File file : folderFiles) { query.put(FileDBAdaptor.QueryParams.UID.key(), file.getUid()); fileTmp = fileManager.get(studyFqn, query, QueryOptions.empty(), sessionIdUser).first(); assertTrue("Folder name should have been modified", fileTmp.getPath().contains(myPath)); assertEquals("Status should be to PENDING_DELETE", File.FileStatus.PENDING_DELETE, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", file.getName(), fileTmp.getName()); assertTrue("File uri: " + fileTmp.getUri() + " should exist", ioManager.exists(fileTmp.getUri())); } }
File folder = createBasicDirectoryFileTestEnvironment(folderFiles); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(catalogManager.getFileManager().getUri(folder)); for (File file : folderFiles) { assertTrue(ioManager.exists(catalogManager.getFileManager().getUri(file)));
File folder = createBasicDirectoryFileTestEnvironment(folderFiles); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(catalogManager.getFileManager().getUri(folder)); for (File file : folderFiles) { assertTrue(ioManager.exists(catalogManager.getFileManager().getUri(file)));
@Test public void deleteFolderTest() throws CatalogException, IOException { List<File> folderFiles = new LinkedList<>(); File folder = createBasicDirectoryFileTestEnvironment(folderFiles); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(catalogManager.getFileManager().getUri(folder)); for (File file : folderFiles) { assertTrue(ioManager.exists(catalogManager.getFileManager().getUri(file))); } catalogManager.getFileManager().create(studyFqn, File.Type.FILE, File.Format.PLAIN, File.Bioformat.NONE, "folder/subfolder/subsubfolder/my_staged.txt", null, null, new File.FileStatus(File.FileStatus.STAGE), (long) 0, (long) -1, null, (long) -1, null, null, true, null, null, sessionIdUser).first(); WriteResult deleteResult = catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()), null, sessionIdUser); assertEquals(0, deleteResult.getNumModified()); File fileTmp = catalogManager.getFileManager().get(studyFqn, folder.getPath(), null, sessionIdUser).first(); assertEquals("Folder name should not be modified", folder.getPath(), fileTmp.getPath()); assertTrue(ioManager.exists(fileTmp.getUri())); for (File file : folderFiles) { fileTmp = catalogManager.getFileManager().get(studyFqn, file.getPath(), null, sessionIdUser).first(); assertEquals("File name should not be modified", file.getPath(), fileTmp.getPath()); assertTrue("File uri: " + fileTmp.getUri() + " should exist", ioManager.exists(fileTmp.getUri())); } }
List<File> folderFiles = new LinkedList<>(); File folder = prepareFiles(folderFiles); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(catalogManager.getFileManager().getUri(folder)); for (File file : folderFiles) { assertTrue(ioManager.exists(catalogManager.getFileManager().getUri(file)));
@Test public void deleteFolderTest2() throws CatalogException, IOException { List<File> folderFiles = new LinkedList<>(); File folder = createBasicDirectoryFileTestEnvironment(folderFiles); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(catalogManager.getFileManager().getUri(folder)); for (File file : folderFiles) { assertTrue(ioManager.exists(catalogManager.getFileManager().getUri(file))); } catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()), null, sessionIdUser); Query query = new Query() .append(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()) .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.TRASHED); File fileTmp = catalogManager.getFileManager().get(studyFqn, query, QueryOptions.empty(), sessionIdUser).first(); assertEquals("Folder name should not be modified", folder.getPath(), fileTmp.getPath()); assertEquals("Status should be to TRASHED", File.FileStatus.TRASHED, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", folder.getName(), fileTmp.getName()); assertTrue(ioManager.exists(fileTmp.getUri())); for (File file : folderFiles) { query.put(FileDBAdaptor.QueryParams.UID.key(), file.getUid()); fileTmp = fileManager.get(studyFqn, query, QueryOptions.empty(), sessionIdUser).first(); assertEquals("Folder name should not be modified", file.getPath(), fileTmp.getPath()); assertEquals("Status should be to TRASHED", File.FileStatus.TRASHED, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", file.getName(), fileTmp.getName()); assertTrue("File uri: " + fileTmp.getUri() + " should exist", ioManager.exists(fileTmp.getUri())); } }