/** * Upload file to a created entry file in Catalog. * * @param sourceUri File URI to be moved into Catalog workspace * @param file File from Catalog * @param sourceChecksum Optional: Source checksum * @param sessionId Valid sessionID to modify the File in Catalog * @param ignoreStatus Ignore the status (uploading, uploaded, ready) from Catalog * @param overwrite Overwrite if there is a file in the target * @param deleteSource After moving, delete file. If false, force copy. * @param calculateChecksum Calculate checksum * @throws CatalogException CatalogException */ public void upload(URI sourceUri, File file, String sourceChecksum, String sessionId, boolean ignoreStatus, boolean overwrite, boolean deleteSource, boolean calculateChecksum) throws CatalogException { upload(sourceUri, file, sourceChecksum, sessionId, ignoreStatus, overwrite, deleteSource, calculateChecksum, Long.MAX_VALUE); }
File newFile = catalogManager.getFileManager().create(study.getFqn(), File.Type.FILE, null, null, relativePath, null, "", null, 0, -1, null, (long) -1, null, null, true, null, null, sessionId).first(); upload(uri, newFile, null, sessionId, false, false, false, calculateChecksum);
@Test public void updateTest() throws IOException, CatalogException { QueryResult<File> fileQueryResult; URI sourceUri; sourceUri = CatalogManagerTest.createDebugFile().toURI(); fileQueryResult = catalogManager.getFileManager().create(studyFqn, File.Type.FILE, File.Format.PLAIN, File .Bioformat.NONE, "item." + TimeUtils.getTimeMillis() + ".txt", null, "file at root", null, 0, -1, null, (long) -1, null, null, true, null, null, userSessionId); catalogFileUtils.upload(sourceUri, fileQueryResult.getResult().get(0), null, userSessionId, false, false, true, false, 1000); sourceUri = CatalogManagerTest.createDebugFile().toURI(); fileQueryResult = catalogManager.getFileManager().create(studyFqn, File.Type.FILE, File.Format.PLAIN, File .Bioformat.NONE, "item." + TimeUtils.getTimeMillis() + ".txt", null, "file at root", null, 0, -1, null, (long) -1, null, null, true, null, null, userSessionId); catalogFileUtils.upload(sourceUri, fileQueryResult.getResult().get(0), null, userSessionId, false, false, true, false, 100000000); sourceUri = CatalogManagerTest.createDebugFile().toURI(); fileQueryResult = catalogManager.getFileManager().create(studyFqn, File.Type.FILE, File.Format.PLAIN, File .Bioformat.NONE, "item." + TimeUtils.getTimeMillis() + ".txt", null, "file at root", null, 0, -1, null, (long) -1, null, null, true, null, null, userSessionId); catalogFileUtils.upload(sourceUri, fileQueryResult.getResult().get(0), null, userSessionId, false, false, true, true); } //
public static File create(String resourceName, boolean indexed) throws IOException, CatalogException { File file; URI uri = getResourceUri(resourceName); file = fileMetadataReader.create(studyId, uri, "data/vcfs/", "", true, null, sessionId).first(); catalogFileUtils.upload(uri, file, null, sessionId, false, false, true, false, Long.MAX_VALUE); if (indexed) { FileIndex fileIndex = new FileIndex("user", "today", new FileIndex.IndexStatus(FileIndex.IndexStatus.READY), 1234, Collections.emptyMap()); catalogManager.getFileManager().setFileIndex(studyId, file.getPath(), fileIndex, sessionId); indexedFiles.add((int) file.getUid()); List<String> samples = catalogManager.getCohortManager().getSamples(studyId, cohortId, null, sessionId).getResult().stream().map(Sample::getId).collect(Collectors.toList()); samples.addAll(file.getSamples().stream().map(Sample::getId).collect(Collectors.toList())); catalogManager.getCohortManager().update(studyId, cohortId, new ObjectMap(CohortDBAdaptor.QueryParams.SAMPLES.key(), samples), true, null, sessionId); } return catalogManager.getFileManager().get(studyId, file.getId(), null, sessionId).first(); }
new org.opencb.opencga.catalog.managers.FileUtils(catalogManager).upload(sourceUri, fileQueryResult.first(), null, sessionId, ignoreStatus, overwrite, deleteSource, calculateChecksum, Long.MAX_VALUE);
File file = catalogManager.getFileManager().create(studyFqn, File.Type.FILE, File.Format.PLAIN, File.Bioformat .VARIANT, "data/" + fileName, null, "description", null, 0, -1, null, (long) -1, null, null, true, null, null, sessionIdUser).first(); catalogFileUtils.upload(is, file, sessionIdUser, false, false, true); is.close();
protected File create(String studyId, URI uri, String path) throws IOException, CatalogException { File file; file = fileMetadataReader.create(studyId, uri, path, "", true, null, sessionId).first(); // File.Format format = FormatDetector.detect(uri); // File.Bioformat bioformat = BioformatDetector.detect(uri); // file = catalogManager.createFile(studyId, format, bioformat, "data/vcfs/", "", true, -1, sessionId).first(); catalogFileUtils.upload(uri, file, null, sessionId, false, false, true, false, Long.MAX_VALUE); return catalogManager.getFileManager().get(studyId, file.getId(), null, sessionId).first(); }
@Test public void testDownloadFile() throws CatalogException, IOException, InterruptedException { String fileName = "item." + TimeUtils.getTimeMillis() + ".vcf"; int fileSize = 200; byte[] bytesOrig = StringUtils.randomString(fileSize).getBytes(); QueryResult<File> queryResult = catalogManager.getFileManager().create(studyFqn, File.Type.FILE, File.Format.PLAIN, File.Bioformat.NONE, "data/" + fileName, null, "description", new File.FileStatus(File.FileStatus.STAGE), 0, -1, null, -1, null, null, true, null, null, sessionIdUser); new FileUtils(catalogManager).upload(new ByteArrayInputStream(bytesOrig), queryResult.first(), sessionIdUser, false, false, true); File file = catalogManager.getFileManager().get(studyFqn, queryResult.first().getPath(), null, sessionIdUser).first(); DataInputStream dis = catalogManager.getFileManager().download(studyFqn, file.getPath(), -1, -1, null, sessionIdUser); byte[] bytes = new byte[fileSize]; dis.read(bytes, 0, fileSize); assertTrue(Arrays.equals(bytesOrig, bytes)); }
@BeforeClass public static void beforeClass() throws IOException, CatalogException, URISyntaxException { Configuration configuration = Configuration.load(CatalogSampleAnnotationsLoaderTest.class.getClassLoader() .getClass().getResource("/configuration-test.yml").openStream()); configuration.getAdmin().setAlgorithm("HS256"); catalogManager = new CatalogManager(configuration); catalogManager.deleteCatalogDB(true); catalogManager.installCatalogDB("dummy", "admin"); loader = new CatalogSampleAnnotationsLoader(catalogManager); String pedFileName = "20130606_g1k.ped"; URL pedFileURL = CatalogSampleAnnotationsLoader.class.getClassLoader().getResource(pedFileName); pedigree = loader.readPedigree(pedFileURL.getPath()); userId = "user1"; catalogManager.getUserManager().create(userId, userId, "asdasd@asd.asd", userId, "", -1L, Account.FULL, QueryOptions.empty(), null); sessionId = catalogManager.getUserManager().login(userId, userId); Project project = catalogManager.getProjectManager().create("def", "default", "", "ACME", "Homo sapiens", null, null, "GRCh38", new QueryOptions(), sessionId).getResult().get(0); Study study = catalogManager.getStudyManager().create(project.getFqn(), "def", null, "default", Study.Type.FAMILY, null, "", null, null, null, null, null, null, null, null, sessionId).getResult().get(0); studyId = study.getFqn(); pedFile = catalogManager.getFileManager().create(studyId, File.Type.FILE, File.Format.PED, File.Bioformat .OTHER_PED, "data/" + pedFileName, null, "", null, 0, -1, null, (long) -1, null, null, true, null, null, sessionId) .getResult().get(0); new FileUtils(catalogManager).upload(pedFileURL.toURI(), pedFile, null, sessionId, false, false, false, true, 10000000); pedFile = catalogManager.getFileManager().get(studyId, pedFile.getPath(), null, sessionId).getResult().get(0); }
@Test public void testCreate() throws CatalogException { QueryResult<File> fileQueryResult = FileMetadataReader.get(catalogManager). create(study.getFqn(), vcfFileUri, folder.getPath() + VCF_FILE_NAME, "", false, null, sessionIdUser); File file = fileQueryResult.first(); assertEquals(File.FileStatus.STAGE, file.getStatus().getName()); assertEquals(File.Format.VCF, file.getFormat()); assertEquals(File.Bioformat.VARIANT, file.getBioformat()); assertNotNull(file.getAttributes().get(VARIANT_FILE_METADATA)); assertEquals(4, file.getSamples().size()); assertEquals(21499, file.getSize()); new FileUtils(catalogManager).upload(vcfFileUri, file, null, sessionIdUser, false, false, true, true, Integer.MAX_VALUE); file = catalogManager.getFileManager().get(study.getFqn(), file.getPath(), null, sessionIdUser).first(); assertEquals(File.FileStatus.READY, file.getStatus().getName()); assertEquals(File.Format.VCF, file.getFormat()); assertEquals(File.Bioformat.VARIANT, file.getBioformat()); assertNotNull(file.getAttributes().get(VARIANT_FILE_METADATA)); assertNotNull(((Map) file.getAttributes().get(VARIANT_FILE_METADATA)).get("sampleIds")); assertEquals(4, ((List) ((Map) file.getAttributes().get(VARIANT_FILE_METADATA)).get("sampleIds")).size()); assertNotNull(((Map) file.getAttributes().get(VARIANT_FILE_METADATA)).get("header")); assertEquals(4, file.getSamples().size()); assertEquals(21499, file.getSize()); }
public File createFile(String studyId, String resourceName, String sessionId) throws IOException, CatalogException { File file; URI uri = getResourceUri(resourceName); CatalogManager catalogManager = getCatalogManager(); file = new FileMetadataReader(catalogManager).create(studyId, uri, "data/vcfs/", "", true, null, sessionId).first(); new FileUtils(catalogManager).upload(uri, file, null, sessionId, false, false, true, false, Long.MAX_VALUE); return catalogManager.getFileManager().get(studyId, file.getId(), null, sessionId).first(); }
catalogFileUtils.upload(fileTest.toURI(), fileResult.first(), null, sessionIdUser, false, false, true, true); assertTrue("File deleted", !fileTest.exists()); "data/" + fileName, null, "description", null, 0, -1, null, (long) -1, null, null, true, null, null, sessionIdUser); fileTest = createDebugFile(); catalogFileUtils.upload(fileTest.toURI(), fileResult.first(), null, sessionIdUser, false, false, false, true); assertTrue("File not deleted", fileTest.exists()); assertTrue(fileTest.delete());
@Test public void deleteFilesTest2() throws CatalogException, IOException { QueryResult<File> queryResult = catalogManager.getFileManager().create(studyFqn, File.Type.FILE, File.Format.PLAIN, File.Bioformat.NONE, "my.txt", null, "", new File.FileStatus(File.FileStatus.STAGE), 0, -1, null, -1, null, null, false, null, null, userSessionId); new FileUtils(catalogManager).upload(new ByteArrayInputStream(StringUtils.randomString(200) .getBytes()), queryResult.first(), userSessionId, false, false, true); File file = catalogManager.getFileManager().get(studyFqn, queryResult.first().getPath(), null, userSessionId).first(); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(file.getUri()); assertTrue(ioManager.exists(file.getUri())); catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), null, userSessionId); assertTrue(ioManager.exists(file.getUri())); // catalogFileUtils.delete(file.getId(), userSessionId); // assertTrue(!ioManager.exists(catalogManager.getFileUri(catalogManager.getFile(file.getId(), userSessionId).first()))); }
catalogFileUtils.upload(sourceUri, file, null, userSessionId, false, false, false, true); fileUri = catalogManager.getFileManager().getUri(file); file = catalogManager.getFileManager().get(studyFqn, file.getPath(), null, userSessionId).first();
new FileUtils(catalogManager).upload(new ByteArrayInputStream(StringUtils.randomString(200).getBytes()), queryResult1.first(), userSessionId, false, false, true); new FileUtils(catalogManager).upload(new ByteArrayInputStream(StringUtils.randomString(200).getBytes()), queryResult.first(), userSessionId, false, false, true); File toTrash = catalogManager.getFileManager().get(studyFqn, queryResult.first().getPath(), null, userSessionId).first(); catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), toTrash.getUid()),