FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(1), // one per file FileWriterRetentionConfig.newFileCountBasedConfig(10)
@Test public void testCycleTupleBased() throws Exception { Topology t = newTopology("testCycleTupleBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // a tuple based config / predicate. in this case should end up with 3 files. // flush on the last tuple too to ensure the test completes before TMO. Predicate<String> cycleIt = tuple -> tuple.startsWith("1-") || tuple.startsWith("3-") || tuple.equals(lines[lines.length-1]); List<List<String>> expResults = buildExpResults(lines, cycleIt); assertEquals(3, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newPredicateBasedConfig(cycleIt), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testRetainAggSizeBased() throws Exception { // more aggsize than configured; only keep aggsize worth Topology t = newTopology("testRetainAggSizeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuple per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); // agg size only enough for last two lines long aggregateFileSize = 2 * (("1-"+getStr()).getBytes(StandardCharsets.UTF_8).length + 1/*eol*/); expResults.remove(0); expResults.remove(0); assertEquals(2, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(1), FileWriterRetentionConfig.newAggregateFileSizeBasedConfig(aggregateFileSize) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testFlushImplicit() throws Exception { Topology t = newTopology("testFlushImplicit"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testCompressedFileWriterPolicy() throws Exception { Topology t = newTopology("testCompressedFileWriterPolicy"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net 2 tuples per file int cntTuples = 2; AtomicInteger cnt = new AtomicInteger(); Predicate<String> cycleIt = tuple -> cnt.incrementAndGet() % cntTuples == 0; List<List<String>> expResults = buildExpResults(lines, cycleIt); assertEquals(lines.length / cntTuples, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new CompressedFileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(cntTuples), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testCycleCntBased() throws Exception { Topology t = newTopology("testCycleCntBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net two tuples per file int cntTuples = 2; AtomicInteger cnt = new AtomicInteger(); Predicate<String> cycleIt = tuple -> cnt.incrementAndGet() % cntTuples == 0; List<List<String>> expResults = buildExpResults(lines, cycleIt); assertEquals(lines.length / cntTuples, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(cntTuples), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testRetainCntBased() throws Exception { // more lines than configured retained numFiles; only keep the last numFiles Topology t = newTopology("testRetainCntBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuples per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); int keepCnt = 2; // only keep the last n files for (int i = 0; i < keepCnt; i++) expResults.remove(0); assertEquals(keepCnt, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(1), FileWriterRetentionConfig.newFileCountBasedConfig(keepCnt) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testCycleTimeBased() throws Exception { Topology t = newTopology("testCycleTimeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuple per file with 250msec cycle config and 1 throttle List<List<String>> expResults = buildExpResults(lines, tuple -> true); // add delay so time cycle happens // also verifies only cycle if there's something to cycle // (i.e., these cycles happen faster than tuples are written) int throttleSec = 1; TStream<String> s = PlumbingStreams.blockingThrottle( t.strings(lines), throttleSec, TimeUnit.SECONDS); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newTimeBasedConfig(TimeUnit.MILLISECONDS.toMillis(250)), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, (lines.length*throttleSec)+TMO_SEC, basePath, expResults); }
FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(1), FileWriterRetentionConfig.newAgeBasedConfig(ageSec, periodMsec)
@Test public void testManyFiles() throws Exception { Topology t = newTopology("testManyFiles"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuples per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); // in this config files are create very fast hence they end // up exercising the _<n> suffix to basePath_YYYYMMDD_HHMMSS TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), // no extra flush FileWriterCycleConfig.newCountBasedConfig(1), // yield one line per file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testCycleSizeBased() throws Exception { Topology t = newTopology("testCycleSizeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuple per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); int fileSize = 2; TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newFileSizeBasedConfig(fileSize), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testManyFilesSlow() throws Exception { Topology t = newTopology("testManyFilesSlow"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuples per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); // add delay so we get different files w/o a _<n> suffix int throttleSec = 2; TStream<String> s = PlumbingStreams.blockingThrottle( t.strings(lines), throttleSec, TimeUnit.SECONDS); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), // no extra flush FileWriterCycleConfig.newCountBasedConfig(1), // yield one line per file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, (lines.length*throttleSec)+TMO_SEC, basePath, expResults); }
@Test public void testFlushConfig() throws Exception { FileWriterFlushConfig<String> cfg; String trueTuple = "true"; String falseTuple = "false"; Predicate<String> p = tuple -> tuple.equals("true"); cfg = FileWriterFlushConfig.newImplicitConfig(); checkFileWriterConfig(cfg, 0, 0, null, trueTuple, falseTuple); cfg = FileWriterFlushConfig.newCountBasedConfig(3); checkFileWriterConfig(cfg, 3, 0, null, trueTuple, falseTuple); expectIAE(() -> FileWriterFlushConfig.newCountBasedConfig(0)); cfg = FileWriterFlushConfig.newTimeBasedConfig(10); checkFileWriterConfig(cfg, 0, 10, null, trueTuple, falseTuple); expectIAE(() -> FileWriterFlushConfig.newTimeBasedConfig(0)); cfg = FileWriterFlushConfig.newPredicateBasedConfig(p); checkFileWriterConfig(cfg, 0, 0, p, trueTuple, falseTuple); expectIAE(() -> FileWriterFlushConfig.newPredicateBasedConfig(null)); cfg = FileWriterFlushConfig.newConfig(1, 2, p); checkFileWriterConfig(cfg, 1, 2, p, trueTuple, falseTuple); cfg = FileWriterFlushConfig.newConfig(0, 0, null); checkFileWriterConfig(cfg, 0, 0, null, trueTuple, falseTuple); expectIAE(() -> FileWriterFlushConfig.newConfig(-1, 0, null)); expectIAE(() -> FileWriterFlushConfig.newConfig(0, -1, null)); }