/** * Create a new file writer policy instance. * <p> * The configuration is: * <ul> * <li>10 second time based active file flushing</li> * <li>1MB file size based active file cycling</li> * <li>10 file retention count</li> * </ul> * The active and final file pathname behavior is specified in * {@link #FileWriterPolicy(FileWriterFlushConfig, FileWriterCycleConfig, FileWriterRetentionConfig)} */ public FileWriterPolicy() { this(FileWriterFlushConfig.newTimeBasedConfig(TimeUnit.SECONDS.toMillis(10)), FileWriterCycleConfig.newFileSizeBasedConfig(1*1024*1024), FileWriterRetentionConfig.newFileCountBasedConfig(10)); }
@Test public void testAllTimeBased() throws Exception { // exercise case with multiple timer based policies Topology t = newTopology("testAllTimeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // keep all given age and TMO_SEC int ageSec = 10; long periodMsec = TimeUnit.SECONDS.toMillis(1); // net one tuple per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newTimeBasedConfig(TimeUnit.MILLISECONDS.toMillis(250)), FileWriterCycleConfig.newConfig(1, 2000, TimeUnit.SECONDS.toMillis(10), null), FileWriterRetentionConfig.newAgeBasedConfig(ageSec, periodMsec) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testFlushTimeBased() throws Exception { Topology t = newTopology("testFlushTimeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); // add delay so time flush happens int throttleSec = 1; TStream<String> s = PlumbingStreams.blockingThrottle( t.strings(lines), throttleSec, TimeUnit.SECONDS); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newTimeBasedConfig(TimeUnit.MILLISECONDS.toMillis(250)), FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), // all in 1 file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, (lines.length*throttleSec)+TMO_SEC, basePath, expResults); }
@Test public void testFlushConfig() throws Exception { FileWriterFlushConfig<String> cfg; String trueTuple = "true"; String falseTuple = "false"; Predicate<String> p = tuple -> tuple.equals("true"); cfg = FileWriterFlushConfig.newImplicitConfig(); checkFileWriterConfig(cfg, 0, 0, null, trueTuple, falseTuple); cfg = FileWriterFlushConfig.newCountBasedConfig(3); checkFileWriterConfig(cfg, 3, 0, null, trueTuple, falseTuple); expectIAE(() -> FileWriterFlushConfig.newCountBasedConfig(0)); cfg = FileWriterFlushConfig.newTimeBasedConfig(10); checkFileWriterConfig(cfg, 0, 10, null, trueTuple, falseTuple); expectIAE(() -> FileWriterFlushConfig.newTimeBasedConfig(0)); cfg = FileWriterFlushConfig.newPredicateBasedConfig(p); checkFileWriterConfig(cfg, 0, 0, p, trueTuple, falseTuple); expectIAE(() -> FileWriterFlushConfig.newPredicateBasedConfig(null)); cfg = FileWriterFlushConfig.newConfig(1, 2, p); checkFileWriterConfig(cfg, 1, 2, p, trueTuple, falseTuple); cfg = FileWriterFlushConfig.newConfig(0, 0, null); checkFileWriterConfig(cfg, 0, 0, null, trueTuple, falseTuple); expectIAE(() -> FileWriterFlushConfig.newConfig(-1, 0, null)); expectIAE(() -> FileWriterFlushConfig.newConfig(0, -1, null)); }