/** * @param sourceIOManager Source IOManager * @param source Source file * @param targetIOManager Target IOManager * @param target Target file * @throws IOException * @throws org.opencb.opencga.catalog.exceptions.CatalogIOException */ private void copy(CatalogIOManager sourceIOManager, URI source, CatalogIOManager targetIOManager, URI target) throws IOException, CatalogIOException { logger.info("Copying file from {} to {}", source, target); //If it's the same IOManager, copy intra FS if (sourceIOManager == targetIOManager) { sourceIOManager.copyFile(source, target); return; } //TODO: Implement copies inter FS // HDFS -> POSIX // POSIX -> HDFS logger.info("Unable to copy directly from {} to {} . Doing manual copy.", source.getScheme(), target.getScheme()); DataInputStream fileObject = sourceIOManager.getFileObject(source, -1, -1); targetIOManager.createFile(target, fileObject); //throw new CatalogIOManagerException("Unable to copy from " + source.getScheme() + " to " + target.getScheme()); }
public URI createUser(String userId) throws CatalogIOException { checkParam(userId); URI usersUri = getUsersUri(); checkDirectoryUri(usersUri, true); URI userPath = getUserUri(userId); try { if (!exists(userPath)) { createDirectory(userPath); createDirectory(userPath.resolve(CatalogIOManager.USER_PROJECTS_FOLDER)); createDirectory(userPath.resolve(CatalogIOManager.USER_BIN_FOLDER)); return userPath; } } catch (CatalogIOException e) { throw e; } return null; }
public URI createStudy(String userId, String projectId, String studyId) throws CatalogIOException { checkParam(studyId); URI projectUri = getProjectUri(userId, projectId); checkDirectoryUri(projectUri, true); URI studyUri = getStudyUri(userId, projectId, studyId); return createStudy(studyUri); }
public DataInputStream getFileObject(URI studyUri, String objectId, int start, int limit) throws CatalogIOException { URI fileUri = getFileUri(studyUri, objectId); return getFileObject(fileUri, start, limit); }
public boolean existProject(String userId, String projectId) throws CatalogIOException { return exists(getProjectUri(userId, projectId)); }
public URI createFolder(URI studyUri, String folderName, boolean parent) throws CatalogIOException { checkParam(folderName); if (!folderName.endsWith("/")) { folderName += "/"; } checkDirectoryUri(studyUri, true); URI folderUri = null; try { folderUri = studyUri.resolve(new URI(null, folderName, null)); } catch (URISyntaxException e) { throw CatalogIOException.uriSyntaxException(folderName, e); } try { if (!exists(folderUri)) { if (parent) { createDirectory(folderUri, true); } else { checkDirectoryUri(folderUri.resolve(".."), true); createDirectory(folderUri); } } } catch (CatalogIOException e) { throw new CatalogIOException("createFolder(): could not create the directory", e); } return folderUri; }
logger.info("Deleting " + JOB_STATUS_FILE + " file: {}", path.toUri()); try { ioManager.deleteFile(path.toUri()); } catch (CatalogIOException e) { logger.error("Could not delete " + JOB_STATUS_FILE + " file"); throw e; if (!ioManager.exists(tmpOutDirUri)) { logger.warn("Output folder doesn't exist"); return; uriList = ioManager.listFiles(tmpOutDirUri); } catch (CatalogIOException e) { logger.warn("Could not obtain the URI of the files within the directory {}", tmpOutDirUri); ioManager.deleteDirectory(tmpOutDirUri); } catch (CatalogIOException e) { if (ioManager.exists(tmpOutDirUri)) { logger.error("Could not delete empty directory {}. Error: {}", tmpOutDirUri, e.getMessage()); throw e;
public URI createProject(String userId, String projectId) throws CatalogIOException { checkParam(projectId); URI projectUri = getProjectUri(userId, projectId); try { if (!exists(projectUri)) { projectUri = createDirectory(projectUri, true); } } catch (CatalogIOException e) { throw new CatalogIOException("createProject(): could not create the project folder", e); } return projectUri; }
/** * This method creates the folders and workspace structure for storing the OpenCGA data. I * * @throws CatalogIOException CatalogIOException */ public void setup() throws CatalogIOException { setConfiguration(configuration); if (!exists(rootDir)) { logger.info("Initializing CatalogIOManager. Creating main folder '" + rootDir + "'"); createDirectory(rootDir, true); } checkDirectoryUri(rootDir, true); if (!exists(jobsDir)) { logger.info("Initializing CatalogIOManager. Creating jobs folder '" + jobsDir + "'"); createDirectory(jobsDir); } checkDirectoryUri(jobsDir, true); if (!exists(rootDir.resolve(OPENCGA_USERS_FOLDER))) { createDirectory(rootDir.resolve(OPENCGA_USERS_FOLDER)); } if (!exists(rootDir.resolve(OPENCGA_ANONYMOUS_USERS_FOLDER))) { createDirectory(rootDir.resolve(OPENCGA_ANONYMOUS_USERS_FOLDER)); } if (!exists(rootDir.resolve(OPENCGA_BIN_FOLDER))) { createDirectory(rootDir.resolve(OPENCGA_BIN_FOLDER)); } }
/** * Check if there is any file in the URI target. * * @throws org.opencb.opencga.catalog.exceptions.CatalogIOException */ private void checkTarget(File file, URI targetUri, CatalogIOManager targetIOManager, boolean overwrite) throws CatalogIOException { if (!overwrite && targetIOManager.exists(targetUri)) { throw new CatalogIOException("There is a file in the target!" + "file:{id:" + file.getPath() + ", targetUri: '" + targetUri + "' } " + "Needs 'overwrite = true' for continue."); } }
@Test public void testCreateStudy() throws Exception { String userId = "imedina"; String projectId = "1000g"; Path userPath = Paths.get(posixCatalogIOManager.createUser(userId)); Path projectPath = Paths.get(posixCatalogIOManager.createProject(userId, projectId)); assertTrue(Files.exists(projectPath)); assertEquals(userPath.toString() + "/projects/" + projectId, projectPath.toString()); Path studyPath = Paths.get(posixCatalogIOManager.createStudy(userId, projectId, "phase1")); assertTrue(Files.exists(studyPath)); // assertTrue(Files.exists(studyPath.resolve("data"))); // assertTrue(Files.exists(studyPath.resolve("analysis"))); assertEquals(projectPath.toString() + "/phase1", studyPath.toString()); // posixIOManager.deleteStudy(userId, projectId, "phase1"); // assertFalse(Files.exists(studyPath)); // // posixIOManager.deleteProject(userId, projectId); // assertFalse(Files.exists(projectPath)); // // posixIOManager.deleteUser(userId); // assertFalse(Files.exists(studyPath)); }
public URI createStudy(URI studyUri) throws CatalogIOException { checkUriScheme(studyUri); try { if (!exists(studyUri)) { studyUri = createDirectory(studyUri); } } catch (CatalogIOException e) { throw new CatalogIOException("createStudy method: could not create the study folder: " + e.toString(), e); } return studyUri; }
@Test public void testCreateAccount() throws Exception { String userId = "imedina"; URI userUri = posixCatalogIOManager.createUser(userId); Path userPath = Paths.get(userUri); assertTrue(Files.exists(userPath)); assertEquals("/tmp/opencga/users/" + userId + "/", userUri.getPath()); posixCatalogIOManager.deleteUser(userId); assertFalse(Files.exists(userPath)); }
if (calculateChecksum) { logger.debug("SourceURI equals to TargetURI. Only calculate checksum and update file entry"); targetChecksum = targetIOManager.calculateChecksum(targetUri); } else { logger.debug("SourceURI equals to TargetURI. Only update file entry"); long size = sourceIOManager.getFileSize(sourceUri); sourceChecksum = sourceIOManager.calculateChecksum(sourceUri); } else { //Do not calculate checksum. sourceChecksum = "null"; try { logger.info("Moving file {} -> {}", sourceUri, targetUri); sourceIOManager.moveFile(sourceUri, targetUri); targetChecksum = sourceChecksum; fileMoved = true; targetChecksum = targetIOManager.calculateChecksum(targetUri); } else { targetChecksum = sourceChecksum; try { logger.info("Checksum fail. Delete target file."); targetIOManager.deleteFile(targetUri); } catch (CatalogIOException e) { e.printStackTrace(); logger.info("Deleting source file {} after moving", sourceUri);
public URI createDirectory(URI uri) throws CatalogIOException { return createDirectory(uri, false); }
private QueryResult<File> create(Study study, File file, boolean parents, String content, QueryOptions options, String sessionId) throws CatalogException { QueryResult<File> queryResult = register(study, file, parents, options, sessionId); if (file.getType() == File.Type.FILE && StringUtils.isNotEmpty(content)) { CatalogIOManager ioManager = catalogIOManagerFactory.getDefault(); // We set parents to true because the file has been successfully registered, which means the directories are already registered // in catalog ioManager.createDirectory(Paths.get(file.getUri()).getParent().toUri(), true); InputStream inputStream = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); ioManager.createFile(file.getUri(), inputStream); // Update file parameters ObjectMap params = new ObjectMap() .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.READY) .append(FileDBAdaptor.QueryParams.SIZE.key(), ioManager.getFileSize(file.getUri())); queryResult = fileDBAdaptor.update(file.getUid(), params, QueryOptions.empty()); } return queryResult; }
if (!ioManager.exists(entry.getValue())) { untrackedFiles.put(entry.getKey(), entry.getValue()); continue; Stream<URI> files = ioManager.listFilesStream(entry.getValue());
URI fileUri = catalogManager.getFileManager().getUri(file); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(fileUri); if (ioManager.isDirectory(fileUri)) { if (!ioManager.listFiles(fileUri).isEmpty()) { throw new CatalogIOException("Unable to delete folder " + fileUri + ". Folder is not empty!"); ioManager.deleteFile(fileUri);
@Test public void testCreateFile() throws CatalogException, IOException { String content = "This is the content\tof the file"; try { fileManager.create(studyFqn3, File.Type.FILE, File.Format.UNKNOWN, File.Bioformat.UNKNOWN, "data/test/myTest/myFile.txt", null, null, new File.FileStatus(File.FileStatus.READY), 0, -1, null, -1, null, null, false, "This is the content\tof the file", null, sessionIdUser2); fail("An error should be raised because parents is false"); } catch (CatalogException e) { System.out.println("Correct"); } QueryResult<File> fileQueryResult = fileManager.create(studyFqn3, File.Type.FILE, File.Format.UNKNOWN, File.Bioformat.UNKNOWN, "data/test/myTest/myFile.txt", null, null, new File.FileStatus(File.FileStatus.READY), 0, -1, null, -1, null, null, true, content, null, sessionIdUser2); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(fileQueryResult.first().getUri()); assertTrue(ioManager.exists(fileQueryResult.first().getUri())); DataInputStream fileObject = ioManager.getFileObject(fileQueryResult.first().getUri(), -1, -1); assertEquals(content, fileObject.readLine()); }
@Deprecated private File linkFile(File file, boolean calculateChecksum, URI externalUri, String sessionId) throws CatalogException { String checksum = null; CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(externalUri); if (ioManager.isDirectory(externalUri)) { throw new CatalogIOException("Can't link file '" + file.getPath() + "' with a folder uri " + externalUri); } if (calculateChecksum) { try { checksum = ioManager.calculateChecksum(externalUri); } catch (CatalogIOException e) { e.printStackTrace(); } } ObjectMap objectMap = new ObjectMap("uri", externalUri); objectMap.put(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.READY); updateFileAttributes(file, checksum, externalUri, objectMap, sessionId); return catalogManager.getFileManager().get(file.getUid(), null, sessionId).first(); }