private synchronized void applyRetention() { long aggregateFileSize = 0; // compute when enabled if (retentionConfig.getAggregateFileSize() > 0) { for (Path path : retainedPaths) { File file = path.toFile(); aggregateFileSize += file.length(); // 0 if doesn't exist } } if (retentionConfig.evaluate(retainedPaths.size(), aggregateFileSize)) { Path oldestPath = retainedPaths.remove(0); File file = oldestPath.toFile(); trace.info("deleting file {}", file); file.delete(); } }
@Override public String toString() { return String.format("fileCount:%d aggSize:%d ageSec:%d", getFileCount(), getAggregateFileSize(), getAgeSec()); }
/** same as {@code newConfig(0, aggregateFileSize, 0, 0)} * * @param aggregateFileSize remove a file when {@code aggregateFileSize} would be exceeded. 0 to disable. * @return the retention config */ public static FileWriterRetentionConfig newAggregateFileSizeBasedConfig(long aggregateFileSize) { if (aggregateFileSize < 1) throw new IllegalArgumentException("aggregateFileSize"); return newConfig(0, aggregateFileSize, 0, 0); } /** same as {@code newConfig(0, 0, ageSe, periodMsecc)}
private static <T> void checkFileWriterConfig(FileWriterRetentionConfig cfg, int fileCnt, long aggSize, long ageSec, long periodMsec) { assertEquals(fileCnt, cfg.getFileCount()); assertEquals(aggSize, cfg.getAggregateFileSize()); assertEquals(ageSec, cfg.getAgeSec()); assertEquals(periodMsec, cfg.getPeriodMsec()); cfg.toString(); int falseFileCnt = fileCnt-1; int trueFileCnt = fileCnt+1; long falseAggSize = aggSize-1; long trueAggSize = aggSize+1; assertFalse("fileCnt:"+fileCnt+" aggSize:"+aggSize, cfg.evaluate(falseFileCnt, falseAggSize)); if (fileCnt!=0) assertTrue("fileCnt:"+fileCnt+" aggSize:"+aggSize, cfg.evaluate(trueFileCnt, falseAggSize)); if (aggSize!=0) assertTrue("fileCnt:"+fileCnt+" aggSize:"+aggSize, cfg.evaluate(falseFileCnt, trueAggSize)); }
/** * Create a new file writer policy instance. * <p> * The configuration is: * <ul> * <li>10 second time based active file flushing</li> * <li>1MB file size based active file cycling</li> * <li>10 file retention count</li> * </ul> * The active and final file pathname behavior is specified in * {@link #FileWriterPolicy(FileWriterFlushConfig, FileWriterCycleConfig, FileWriterRetentionConfig)} */ public FileWriterPolicy() { this(FileWriterFlushConfig.newTimeBasedConfig(TimeUnit.SECONDS.toMillis(10)), FileWriterCycleConfig.newFileSizeBasedConfig(1*1024*1024), FileWriterRetentionConfig.newFileCountBasedConfig(10)); }
@Test public void testRetentionConfig() throws Exception { FileWriterRetentionConfig cfg; cfg = FileWriterRetentionConfig.newFileCountBasedConfig(2); checkFileWriterConfig(cfg, 2, 0, 0, 0); expectIAE(() -> FileWriterRetentionConfig.newFileCountBasedConfig(0)); cfg = FileWriterRetentionConfig.newAggregateFileSizeBasedConfig(3); checkFileWriterConfig(cfg, 0, 3, 0, 0); expectIAE(() -> FileWriterRetentionConfig.newAggregateFileSizeBasedConfig(0)); cfg = FileWriterRetentionConfig.newAgeBasedConfig(10,11); checkFileWriterConfig(cfg, 0, 0, 10, 11); expectIAE(() -> FileWriterRetentionConfig.newAgeBasedConfig(0,1)); expectIAE(() -> FileWriterRetentionConfig.newAgeBasedConfig(1,0)); cfg = FileWriterRetentionConfig.newConfig(1, 2, 3, 0); checkFileWriterConfig(cfg, 1, 2, 3, 0); expectIAE(() -> FileWriterRetentionConfig.newConfig(0, 0, 0, 0)); expectIAE(() -> FileWriterRetentionConfig.newConfig(0, 0, 1, 0)); expectIAE(() -> FileWriterRetentionConfig.newConfig(0, 0, 0, 1)); expectIAE(() -> FileWriterRetentionConfig.newConfig(-1, 0, 0, 0)); expectIAE(() -> FileWriterRetentionConfig.newConfig(0, -1, 0, 0)); expectIAE(() -> FileWriterRetentionConfig.newConfig(0, 0, -1, 0)); expectIAE(() -> FileWriterRetentionConfig.newConfig(0, 0, 0, -1)); }
periodMsec, periodMsec, TimeUnit.MILLISECONDS); if (retentionConfig.getAgeSec() > 0) { long periodMsec = retentionConfig.getPeriodMsec(); getExecutor().scheduleAtFixedRate( () -> applyTimeBasedRetention(),
private synchronized void applyTimeBasedRetention() { long now = System.currentTimeMillis(); long minTime = now - TimeUnit.SECONDS.toMillis(retentionConfig.getAgeSec()); ArrayList<Path> toDelete = new ArrayList<>(); for (Path path : retainedPaths) { // oldest first File file = path.toFile(); if (file.lastModified() < minTime) toDelete.add(path); else break; } for (Path path : toDelete) { trace.info("deleting file {}", path); path.toFile().delete(); } retainedPaths.removeAll(toDelete); }
@Override public String toString() { return String.format("basePathname:%s [retention: %s] [cycle: %s] [flush: %s]", basePathname, retentionConfig.toString(), cycleConfig.toString(), flushConfig.toString()); }
@Test public void testAllTimeBased() throws Exception { // exercise case with multiple timer based policies Topology t = newTopology("testAllTimeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // keep all given age and TMO_SEC int ageSec = 10; long periodMsec = TimeUnit.SECONDS.toMillis(1); // net one tuple per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newTimeBasedConfig(TimeUnit.MILLISECONDS.toMillis(250)), FileWriterCycleConfig.newConfig(1, 2000, TimeUnit.SECONDS.toMillis(10), null), FileWriterRetentionConfig.newAgeBasedConfig(ageSec, periodMsec) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
/** * Create a new configuration. * * @param fileCount remove a file when {@code fileCount} would be exceeded. 0 to disable. * @param aggregateFileSize remove a file when {@code aggregateFileSize} would be exceeded. 0 to disable. * @param ageSec remove a file that's older than {@code ageSec} seconds. 0 to disable. * @param periodMsec frequency for checking for ageSec based removal. 0 to disable.] * @return the retention config */ public static FileWriterRetentionConfig newConfig(int fileCount, long aggregateFileSize, long ageSec, long periodMsec) { return new FileWriterRetentionConfig(fileCount, aggregateFileSize, ageSec, periodMsec); }
@Test public void testFlushTupleBased() throws Exception { Topology t = newTopology("testFlushTupleBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newPredicateBasedConfig( tuple -> tuple.startsWith("1-") || tuple.startsWith("3-")), FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), // all in 1 file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(1), FileWriterRetentionConfig.newAgeBasedConfig(ageSec, periodMsec) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy);
FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(1), // one per file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(contents, () -> basePath.toString(), () -> policy);
/** same as {@code newConfig(fileCount, 0, 0, 0)} * * @param fileCount remove a file when {@code fileCount} would be exceeded. 0 to disable. * @return the retention config */ public static FileWriterRetentionConfig newFileCountBasedConfig(int fileCount) { if (fileCount < 1) throw new IllegalArgumentException("fileCount"); return newConfig(fileCount, 0, 0, 0); } /** same as {@code newConfig(0, aggregateFileSize, 0, 0)}
@Test public void testCycleTupleBased() throws Exception { Topology t = newTopology("testCycleTupleBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // a tuple based config / predicate. in this case should end up with 3 files. // flush on the last tuple too to ensure the test completes before TMO. Predicate<String> cycleIt = tuple -> tuple.startsWith("1-") || tuple.startsWith("3-") || tuple.equals(lines[lines.length-1]); List<List<String>> expResults = buildExpResults(lines, cycleIt); assertEquals(3, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newPredicateBasedConfig(cycleIt), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
/** same as {@code newConfig(0, 0, ageSe, periodMsecc)} * * @param ageSec remove a file that's older than {@code ageSec} seconds. 0 to disable. * @param periodMsec frequency for checking for ageSec based removal. 0 to disable.] * @return the retention config */ public static FileWriterRetentionConfig newAgeBasedConfig(long ageSec, long periodMsec) { if (ageSec < 1) throw new IllegalArgumentException("ageSec"); if (periodMsec < 1) throw new IllegalArgumentException("periodMsec"); return newConfig(0, 0, ageSec, periodMsec); }
@Test public void testFlushImplicit() throws Exception { Topology t = newTopology("testFlushImplicit"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testFlushCntBased() throws Exception { Topology t = newTopology("testFlushCntBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newCountBasedConfig(1), // every tuple FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), // all in 1 file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testCompressedFileWriterPolicy() throws Exception { Topology t = newTopology("testCompressedFileWriterPolicy"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net 2 tuples per file int cntTuples = 2; AtomicInteger cnt = new AtomicInteger(); Predicate<String> cycleIt = tuple -> cnt.incrementAndGet() % cntTuples == 0; List<List<String>> expResults = buildExpResults(lines, cycleIt); assertEquals(lines.length / cntTuples, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new CompressedFileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(cntTuples), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }