public static void insertPedigreeFile(CatalogManager catalogManager, Path inputFile, String sessionId) throws CatalogException, FileNotFoundException { String path = "data/peds"; URI sourceUri = inputFile.toUri(); File file = catalogManager.getFileManager().upload("user1@default:study1", new FileInputStream(new java.io.File(sourceUri)), new File().setPath(Paths.get(path, inputFile.getFileName().toString()).toString()), false, true, sessionId).first(); // Load samples using the pedigree file CatalogSampleAnnotationsLoader catalogSampleAnnotationsLoader = new CatalogSampleAnnotationsLoader(catalogManager); catalogSampleAnnotationsLoader.loadSampleAnnotations(file, null, sessionId); } //
new File().setPath(filePath), overwrite, true, calculateChecksum, deleteSource, sessionId).first();
@Test public void testDeleteExisting() throws IOException, CatalogException { QueryResult<File> queryResult = catalogManager.getFileManager().upload(study.getFqn(), CatalogManagerTest.createDebugFile().toURI(), new File().setPath(folder.getPath() + "file1.txt"), false, false, sessionIdUser); File file = queryResult.first(); CatalogManagerTest.createDebugFile(directory.resolve("file1.txt").toString()); List<File> files = new FileScanner(catalogManager).scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.DELETE, false, true, sessionIdUser); files.forEach((File f) -> assertFalse(f.getAttributes().containsKey("checksum"))); assertEquals(File.FileStatus.DELETED, getFile(file.getUid()).getStatus().getName()); }
private File prepareFiles(List<File> folderFiles) throws CatalogException { File folder = catalogManager.getFileManager().createFolder(studyFqn, Paths.get("folder").toString(), null, false, null, QueryOptions.empty(), userSessionId).first(); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/my.txt"), false, StringUtils.randomString(200), null, userSessionId).first() ); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/my2.txt"), false, StringUtils.randomString(200), null, userSessionId).first() ); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/my3.txt"), false, StringUtils.randomString(200), null, userSessionId).first() ); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/subfolder/my4.txt"), true, StringUtils.randomString(200), null, userSessionId).first() ); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/subfolder/my5.txt"), false, StringUtils.randomString(200), null, userSessionId).first() ); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/subfolder/subsubfolder/my6.txt"), true, StringUtils.randomString(200), null, userSessionId).first() ); return folder; }
private File createBasicDirectoryFileTestEnvironment(List<File> folderFiles) throws CatalogException { File folder = catalogManager.getFileManager().createFolder(studyFqn, Paths.get("folder").toString(), null, false, null, QueryOptions.empty(), sessionIdUser).first(); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/my.txt"), false, StringUtils.randomString(200), null, sessionIdUser).first() ); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/my2.txt"), false, StringUtils.randomString(200), null, sessionIdUser).first() ); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/my3.txt"), false, StringUtils.randomString(200), null, sessionIdUser).first() ); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/subfolder/my4.txt"), true, StringUtils.randomString(200), null, sessionIdUser).first() ); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/subfolder/my5.txt"), false, StringUtils.randomString(200), null, sessionIdUser).first() ); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/subfolder/subsubfolder/my6.txt"), true, StringUtils.randomString(200), null, sessionIdUser).first() ); return folder; }
@Test public void renameFileTest() throws CatalogException { QueryResult<File> queryResult1 = catalogManager.getFileManager().create(studyFqn, new File().setPath("data/file.txt"), true, StringUtils.randomString(200), null, sessionIdUser); assertEquals(1, queryResult1.getNumResults()); QueryResult<File> queryResult = catalogManager.getFileManager().create(studyFqn, new File().setPath("data/nested/folder/file2.txt"), true, StringUtils.randomString(200), null, sessionIdUser); assertEquals(1, queryResult.getNumResults()); catalogManager.getFileManager().rename(studyFqn, "data/nested/", "nested2", sessionIdUser); Set<String> paths = catalogManager.getFileManager().get(studyFqn, new Query(), new QueryOptions(), sessionIdUser) .getResult() .stream().map(File::getPath).collect(Collectors.toSet()); assertTrue(paths.contains("data/nested2/")); assertFalse(paths.contains("data/nested/")); assertTrue(paths.contains("data/nested2/folder/")); assertTrue(paths.contains("data/nested2/folder/file2.txt")); assertTrue(paths.contains("data/file.txt")); catalogManager.getFileManager().rename(studyFqn, "data/", "Data", sessionIdUser); paths = catalogManager.getFileManager().get(studyFqn, new Query(), new QueryOptions(), sessionIdUser).getResult() .stream().map(File::getPath).collect(Collectors.toSet()); assertTrue(paths.contains("Data/")); assertTrue(paths.contains("Data/file.txt")); assertTrue(paths.contains("Data/nested2/")); assertTrue(paths.contains("Data/nested2/folder/")); assertTrue(paths.contains("Data/nested2/folder/file2.txt")); }
@Test public void testGetMetadataFromVcf() throws CatalogException, FileNotFoundException { File file = catalogManager.getFileManager().upload(study.getFqn(), vcfFileUri, new File().setPath(folder.getPath() + VCF_FILE_NAME), false, false, sessionIdUser).first(); assertTrue(file.getSize() > 0); file = FileMetadataReader.get(catalogManager). setMetadataInformation(file, null, null, sessionIdUser, false); assertEquals(File.FileStatus.READY, file.getStatus().getName()); assertEquals(File.Format.VCF, file.getFormat()); assertEquals(File.Bioformat.VARIANT, file.getBioformat()); assertNotNull(file.getAttributes().get(VARIANT_FILE_METADATA)); assertEquals(4, file.getSamples().size()); assertEquals(expectedSampleNames, ((Map<String, Object>) file.getAttributes().get(VARIANT_FILE_METADATA)).get("sampleIds")); catalogManager.getSampleManager().get(study.getFqn(), new Query(SampleDBAdaptor.QueryParams.ID.key(), file.getSamples().stream().map(Sample::getId).collect(Collectors.toList())), new QueryOptions(), sessionIdUser).getResult(); assertTrue(expectedSampleNames.containsAll(file.getSamples().stream().map(Sample::getId).collect(Collectors.toSet()))); }
@Test public void testGetBasicMetadata() throws CatalogException, IOException { File file = catalogManager.getFileManager().create(study.getFqn(), new File().setPath(folder.getPath() + "test.txt"), false, StringUtils.randomString(1000), null, sessionIdUser).first(); assertEquals(1000, file.getSize()); String creationDate = file.getCreationDate(); String modificationDate = file.getModificationDate(); URI fileUri = catalogManager.getFileManager().getUri(file); try { Thread.sleep(1000); //Sleep 1 second to see changes on the "modificationDate" } catch (InterruptedException ignored) {} OutputStream outputStream = new FileOutputStream(Paths.get(fileUri).toFile(), true); byte[] bytes2 = StringUtils.randomString(100).getBytes(); outputStream.write(bytes2); outputStream.close(); file = FileMetadataReader.get(catalogManager). setMetadataInformation(file, null, null, sessionIdUser, false); assertEquals(1000 + bytes2.length, file.getSize()); assertTrue(TimeUtils.toDate(modificationDate).getTime() < TimeUtils.toDate(file.getModificationDate()).getTime()); assertEquals(creationDate, file.getCreationDate()); }
@Test public void testReplaceExisting() throws IOException, CatalogException { // Create and register file1.txt and s/file2.txt File file = catalogManager.getFileManager().upload(study.getFqn(), CatalogManagerTest.createDebugFile().toURI(), new File().setPath(folder.getPath() + "file1.txt"), false, false, true, true, sessionIdUser).first(); catalogManager.getFileManager().upload(study.getFqn(), CatalogManagerTest.createDebugFile().toURI(), new File().setPath(folder.getPath() + "s/file2.txt"), false, true, true, true, sessionIdUser).first(); // Create same file structure, and replace CatalogManagerTest.createDebugFile(directory.resolve("file1.txt").toString()); Files.createDirectory(directory.resolve("s/")); CatalogManagerTest.createDebugFile(directory.resolve("s/file2.txt").toString()); FileScanner fileScanner = new FileScanner(catalogManager); fileScanner.scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.REPLACE, true, true, sessionIdUser); File replacedFile = catalogManager.getFileManager().get(study.getFqn(), file.getPath(), null, sessionIdUser).first(); assertEquals(File.FileStatus.READY, replacedFile.getStatus().getName()); assertEquals(file.getUid(), replacedFile.getUid()); assertNotEquals(replacedFile.getChecksum(), file.getChecksum()); assertEquals(replacedFile.getChecksum(), catalogManager.getCatalogIOManagerFactory().getDefault().calculateChecksum(replacedFile.getUri())); }
QueryResult<File> queryResult = catalogManager.getFileManager().create(studyFqn, new File().setPath("data/" + fileName), false, StringUtils.randomString(200), null, sessionIdUser); assertTrue("", queryResult.first().getStatus().getName().equals(File.FileStatus.READY)); fileTest = createDebugFile(); catalogManager.getFileManager().upload(studyFqn, fileTest.toURI(), new File().setPath("data/deletable/folder/" + fileName), false, true, sessionIdUser); new File().setPath("data/deletable/" + fileName), false, true, false, false, sessionIdUser); assertTrue(fileTest.delete()); assertEquals(1, fileQueryResult.getNumResults()); fileTest = createDebugFile(); fileQueryResult = catalogManager.getFileManager().upload(studyFqn2, fileTest.toURI(), new File().setPath(fileName).setDescription("file at root"), false, true, false, false, sessionIdUser); assertEquals(1, fileQueryResult.getNumResults()); assertTrue(fileTest.delete()); long size = Files.size(fileTest.toPath()); catalogManager.getFileManager().upload(studyFqn2, fileTest.toURI(), new File().setPath(fileName).setDescription("file at root"), false, true, sessionIdUser);
@Test public void testGetMetadataFromBam() throws CatalogException, FileNotFoundException { File file = catalogManager.getFileManager().upload(study.getFqn(), bamFileUri, new File().setPath(folder.getPath() + BAM_FILE_NAME), false, false, sessionIdUser).first(); assertTrue(file.getSize() > 0); file = FileMetadataReader.get(catalogManager). setMetadataInformation(file, null, null, sessionIdUser, false); assertEquals(File.FileStatus.READY, file.getStatus().getName()); // assertEquals(File.Format.GZIP, file.getFormat()); assertEquals(File.Bioformat.ALIGNMENT, file.getBioformat()); assertNotNull(file.getAttributes().get("alignmentHeader")); assertEquals(1, file.getSamples().size()); assertEquals("HG00096", catalogManager.getSampleManager().get(study.getFqn(), file.getSamples().get(0).getId(), null, sessionIdUser).first().getId()); }
@Test public void testDoNotOverwriteSampleIds() throws CatalogException, FileNotFoundException { File file = catalogManager.getFileManager().upload(study.getFqn(), vcfFileUri, new File().setPath(folder.getPath() + VCF_FILE_NAME), false, false, sessionIdUser).first(); assertEquals(File.FileStatus.READY, file.getStatus().getName()); assertEquals(File.Format.VCF, file.getFormat()); assertEquals(File.Bioformat.VARIANT, file.getBioformat()); assertNotNull(file.getAttributes().get(VARIANT_FILE_METADATA)); assertEquals(4, file.getSamples().size()); //Add a sampleId String sampleId = catalogManager.getSampleManager().create(study.getFqn(), new Sample().setId("Bad_Sample"), null, sessionIdUser) .first().getId(); catalogManager.getFileManager().update(study.getFqn(), file.getPath(), new ObjectMap(FileDBAdaptor.QueryParams.SAMPLES.key(), Collections.singletonList(sampleId)), new QueryOptions(), sessionIdUser); file = catalogManager.getFileManager().get(study.getFqn(), file.getPath(), null, sessionIdUser).first(); assertEquals(5, file.getSamples().size()); assertEquals(sampleId, file.getSamples().get(4).getId()); }
@Test public void getAllJobs() throws CatalogException { Job job = new Job() .setId("job1") .setToolId("toolId") .setOutDir(new File().setPath(data_d1_d2)); long job1 = catalogManager.getJobManager().create(studyFqn, job, null, ownerSessionId).first().getUid(); job.setId("job2"); long job2 = catalogManager.getJobManager().create(studyFqn, job, null, ownerSessionId).first().getUid(); job.setId("job3"); long job3 = catalogManager.getJobManager().create(studyFqn, job, null, ownerSessionId).first().getUid(); job.setId("job4"); long job4 = catalogManager.getJobManager().create(studyFqn, job, null, ownerSessionId).first().getUid(); checkGetAllJobs(Arrays.asList(job1, job2, job3, job4), ownerSessionId); //Owner can see everything checkGetAllJobs(Collections.emptyList(), externalSessionId); //Can't see inside data_d1_d2_d3 }
@Test public void testDeleteTrashed() throws IOException, CatalogException { QueryResult<File> queryResult = catalogManager.getFileManager().upload(study.getFqn(), CatalogManagerTest.createDebugFile().toURI(), new File().setPath(folder.getPath() + "file1.txt"), false, false, sessionIdUser); File file = queryResult.first(); catalogManager.getFileManager().delete(study.getFqn(), new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), new QueryOptions(), sessionIdUser); QueryResult<File> fileQueryResult = catalogManager.getFileManager().get(study.getFqn(), new Query() .append(FileDBAdaptor.QueryParams.UID.key(), file.getUid()) .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), "!=EMPTY"), new QueryOptions(), sessionIdUser); file = fileQueryResult.first(); assertEquals(File.FileStatus.TRASHED, file.getStatus().getName()); }
data_d1_d2_d3_d4 = catalogManager.getFileManager().createFolder(studyFqn, Paths.get("data/d1/d2/d3/d4/").toString(), null, false, null, QueryOptions.empty(), ownerSessionId).first().getPath(); QueryResult<File> queryResult = catalogManager.getFileManager().create(studyFqn, new File().setPath("data/d1/d2/d3/d4/my.txt"), false, "file content", null, ownerSessionId);
new File().setPath(testFolder.getPath() + "test_1K.txt.gz"), false, StringUtils.randomString(1000), null, sessionIdUser); new File().setPath(testFolder.getPath() + "test_0.5K.txt").setBioformat(File.Bioformat.DATAMATRIX_EXPRESSION), false, StringUtils.randomString(500), null, sessionIdUser); new File().setPath(testFolder.getPath() + "test_0.1K.png").setFormat(File.Format.IMAGE), false, StringUtils.randomString(100), null, sessionIdUser);