@Override public void configure(SyncStorage source, Iterator<SyncFilter> filters, SyncStorage target) { super.configure(source, filters, target); File rootFile = createFile(config.getPath()); if (source == this) { if (!rootFile.exists()) throw new ConfigurationException("the source " + rootFile + " does not exist"); if (config.getModifiedSince() != null) { modifiedSince = Iso8601Util.parse(config.getModifiedSince()); if (modifiedSince == null) throw new ConfigurationException("could not parse modified-since"); } if (config.getDeleteCheckScript() != null) { File deleteCheckScript = new File(config.getDeleteCheckScript()); if (!deleteCheckScript.exists()) throw new ConfigurationException("delete check script " + deleteCheckScript + " does not exist"); } if (config.getExcludedPaths() != null) { excludedPathPatterns = new ArrayList<>(); for (String pattern : config.getExcludedPaths()) { excludedPathPatterns.add(Pattern.compile(pattern)); } } } }
@Override public String getRelativePath(String identifier, boolean directory) { String relativePath = createFile(identifier).getAbsolutePath(); File rootFile = createFile(config.getPath()); if (!config.isUseAbsolutePath() && relativePath.startsWith(rootFile.getAbsolutePath())) { relativePath = relativePath.substring(rootFile.getAbsolutePath().length()); } if (File.separatorChar == '\\') { relativePath = relativePath.replace('\\', '/'); } if (relativePath.startsWith("/")) { relativePath = relativePath.substring(1); } return relativePath; }
@Override public void delete(String identifier) { File deleteCheckScript = null; if (config.getDeleteCheckScript() != null) deleteCheckScript = new File(config.getDeleteCheckScript()); delete(identifier, config.getDeleteOlderThan(), deleteCheckScript); }
mimeMap.put("foo", "x-bar"); mimeMap.put("aaa", "application/octet-stream"); FilesystemConfig object = new FilesystemConfig(); object.setPath("/foo/bar"); object.setUseAbsolutePath(true); object.setFollowLinks(true); object.setDeleteOlderThan(0); object.setDeleteCheckScript("foo.sh"); object.setModifiedSince("2015-01-01T00:00:00Z"); object.setExcludedPaths(new String[]{".*\\.bak", ".*/\\.snapshot"}); object.setStoreMetadata(true); FilesystemConfig xObject = (FilesystemConfig) unmarshaller.unmarshal(new StringReader(xml)); Assert.assertEquals(object.getPath(), xObject.getPath()); Assert.assertEquals(object.isUseAbsolutePath(), xObject.isUseAbsolutePath()); Assert.assertEquals(object.isFollowLinks(), xObject.isFollowLinks()); Assert.assertEquals(object.getDeleteOlderThan(), xObject.getDeleteOlderThan()); Assert.assertEquals(object.getDeleteCheckScript(), xObject.getDeleteCheckScript()); Assert.assertEquals(object.getModifiedSince(), xObject.getModifiedSince()); Assert.assertArrayEquals(object.getExcludedPaths(), xObject.getExcludedPaths());
@Override public void configure(SyncStorage source, Iterator<SyncFilter> filters, SyncStorage target) { super.configure(source, filters, target); if (config.getLocalCacheRoot() == null) throw new ConfigurationException("must specify a cache root"); File cacheRoot = new File(config.getLocalCacheRoot()); if (!cacheRoot.exists() || !cacheRoot.isDirectory() || !cacheRoot.canWrite() || cacheRoot.list().length != 0) throw new ConfigurationException(cacheRoot + " is not a writable empty directory"); // split plugin chain for each side of the cache (source -> cache and cache -> target) List<SyncFilter> filtersBeforeCache = new ArrayList<>(); List<SyncFilter> filtersAfterCache = new ArrayList<>(); boolean beforeCache = true; while (filters.hasNext()) { SyncFilter filter = filters.next(); if (filter == this) beforeCache = false; else if (beforeCache) filtersBeforeCache.add(filter); else filtersAfterCache.add(filter); } FilesystemConfig cacheConfig = new FilesystemConfig(); cacheConfig.setPath(config.getLocalCacheRoot()); cacheTarget = new FilesystemStorage(); cacheTarget.setConfig(cacheConfig); cacheTarget.setOptions(options); cacheTarget.configure(source, filtersBeforeCache.iterator(), cacheTarget); cacheSource = new FilesystemStorage(); cacheSource.setConfig(cacheConfig); cacheSource.setOptions(options); cacheSource.configure(cacheSource, filtersAfterCache.iterator(), target); }
@Test public void testFilesystem() { final File tempDir = new File("/tmp/ecs-sync-filesystem-test"); // File.createTempFile("ecs-sync-filesystem-test", "dir"); tempDir.mkdir(); tempDir.deleteOnExit(); if (!tempDir.exists() || !tempDir.isDirectory()) throw new RuntimeException("unable to make temp dir"); FilesystemConfig filesystemConfig = new FilesystemConfig(); filesystemConfig.setPath(tempDir.getPath()); filesystemConfig.setStoreMetadata(true); multiEndToEndTest(filesystemConfig, new TestConfig(), false); new File(tempDir, ObjectMetadata.METADATA_DIR).delete(); // delete this so the temp dir can go away }
@Test public void testExcludeFilter() throws Exception { Path file = Paths.get("."); FilesystemConfig fsConfig = new FilesystemConfig(); fsConfig.setPath(file.toString()); fsConfig.setExcludedPaths(new String[]{"(.*/)?\\.[^/]*", "(.*/)?[^/]*foo[^/]*", "(.*/)?[^/]*\\.bin"}); FilesystemStorage storage = new FilesystemStorage(); storage.setConfig(fsConfig); storage.configure(storage, null, null); DirectoryStream.Filter<Path> filter = storage.getFilter(); String[] positiveTests = new String[]{"bar.txt", "a.out", "this has spaces", "n.o.t.h.i.n.g"}; for (String test : positiveTests) { Assert.assertTrue("filter should have accepted " + test, filter.accept(file.resolve(test))); } String[] negativeTests = new String[]{".svn", ".snapshots", ".f.o.o", "foo.txt", "ffoobar", "mywarez.bin", "in.the.round.bin"}; for (String test : negativeTests) { Assert.assertFalse("filter should have rejected " + test, filter.accept(file.resolve(test))); } }
@Test public void testFilesystemCli() throws Exception { File sourceFile = new File("/tmp/foo"); File targetFile = new File("/tmp/bar"); String[] args = new String[]{ "-source", "file://" + sourceFile, "-target", "file://" + targetFile, "--source-use-absolute-path", "--target-include-base-dir" }; CliConfig cliConfig = CliHelper.parseCliConfig(args); SyncConfig syncConfig = CliHelper.parseSyncConfig(cliConfig, args); Object source = syncConfig.getSource(); Assert.assertNotNull("source is null", source); Assert.assertTrue("source is not FilesystemSource", source instanceof FilesystemConfig); FilesystemConfig fsSource = (FilesystemConfig) source; Object target = syncConfig.getTarget(); Assert.assertNotNull("target is null", target); Assert.assertTrue("target is not FilesystemTarget", target instanceof FilesystemConfig); FilesystemConfig fsTarget = (FilesystemConfig) target; Assert.assertEquals("source file mismatch", sourceFile.getPath(), fsSource.getPath()); Assert.assertTrue("source use-absolute-path should be enabled", fsSource.isUseAbsolutePath()); Assert.assertEquals("target file mismatch", targetFile.getPath(), fsTarget.getPath()); Assert.assertTrue("target include-base-dir should be enabled", fsTarget.isIncludeBaseDir()); }
FilesystemConfig fsConfig = new FilesystemConfig(); fsConfig.setPath(tempDir.getPath()); fsConfig.setModifiedSince(Iso8601Util.format(modifiedSince));
@Test public void testFilesystemUriParsing() { FilesystemConfig fsConfig = new FilesystemConfig(); String uri = "file:///foo/bar", path = "/foo/bar", newUri = "file:/foo/bar"; ConfigUtil.parseUri(fsConfig, uri); Assert.assertEquals(path, fsConfig.getPath()); Assert.assertEquals(newUri, ConfigUtil.generateUri(fsConfig)); uri = "file://foo/bar"; path = "foo/bar"; newUri = "file:foo/bar"; ConfigUtil.parseUri(fsConfig, uri); Assert.assertEquals(path, fsConfig.getPath()); Assert.assertEquals(newUri, ConfigUtil.generateUri(fsConfig)); } }
@Override public Iterable<ObjectSummary> allObjects() { ObjectSummary rootSummary = createSummary(config.getPath()); if (rootSummary.isDirectory() && !config.isIncludeBaseDir()) return children(rootSummary); else return Collections.singletonList(rootSummary); }
protected LinkOption[] getLinkOptions() { return config.isFollowLinks() ? new LinkOption[0] : new LinkOption[]{LinkOption.NOFOLLOW_LINKS}; }
@Override public String getIdentifier(String relativePath, boolean directory) { return createFile(config.getPath(), relativePath).getPath(); }
@Test public void testSingleFile() throws Exception { String name = "single-file-test"; File sFile = new File(sourceDir, name); File tFile = new File(targetDir, name); int size = 100 * 1024; StreamUtil.copy(new RandomInputStream(size), new FileOutputStream(sFile), size); FilesystemConfig sConfig = new FilesystemConfig(); sConfig.setPath(sFile.getAbsolutePath()); FilesystemConfig tConfig = new FilesystemConfig(); tConfig.setPath(tFile.getAbsolutePath()); SyncConfig syncConfig = new SyncConfig().withSource(sConfig).withTarget(tConfig); EcsSync sync = new EcsSync(); sync.setSyncConfig(syncConfig); sync.run(); Assert.assertArrayEquals(Files.readAllBytes(sFile.toPath()), Files.readAllBytes(tFile.toPath())); } }
FilesystemConfig fsConfig = new FilesystemConfig(); fsConfig.setPath(sourceDir.getPath()); fsConfig.setStoreMetadata(true); fsConfig.setPath(targetDir.getPath());
private ObjectSummary createSummary(File file) { boolean link = isSymLink(file); boolean directory = file.isDirectory() && (config.isFollowLinks() || !link); long size = directory || link ? 0 : file.length(); return new ObjectSummary(file.getPath(), directory, size); }
@Test public void testSingleFileFromFilesystem() throws Exception { FilesystemConfig sourceConfig = new FilesystemConfig(); sourceConfig.setPath(filesystemSourceFile.getAbsolutePath()); NfsConfig targetConfig = getNfsConfig(nfs); targetConfig.setSubPath(targetDirectoryName + NfsFile.separator + testFileName); SyncConfig syncConfig = new SyncConfig().withSource(sourceConfig).withTarget(targetConfig); EcsSync sync = new EcsSync(); sync.setSyncConfig(syncConfig); sync.run(); byte[] sourceBytes = Files.readAllBytes(filesystemSourceFile.toPath()); byte[] targetBytes = readBytes(targetFile); Assert.assertArrayEquals(sourceBytes, targetBytes); }
.withDiscardData(false); FilesystemConfig tmpConfig = new FilesystemConfig(); tmpConfig.setPath(tempDir.getPath()); tmpConfig.setStoreMetadata(true);
private InputStream readDataStream(String identifier) { try { File file = createFile(identifier); if (!config.isFollowLinks() && isSymLink(file)) return new ByteArrayInputStream(new byte[0]); else return createInputStream(file); } catch (IOException e) { throw new RuntimeException(e); } }
@Test public void testSingleFileToFilesystem() throws Exception { NfsConfig sourceConfig = getNfsConfig(nfs); sourceConfig.setSubPath(sourceDirectoryName + NfsFile.separator + testFileName); FilesystemConfig targetConfig = new FilesystemConfig(); targetConfig.setPath(filesystemTargetFile.getAbsolutePath()); SyncConfig syncConfig = new SyncConfig().withSource(sourceConfig).withTarget(targetConfig); EcsSync sync = new EcsSync(); sync.setSyncConfig(syncConfig); sync.run(); byte[] sourceBytes = readBytes(sourceFile); byte[] targetBytes = Files.readAllBytes(filesystemTargetFile.toPath()); Assert.assertArrayEquals(sourceBytes, targetBytes); }