@Test public void testFlushTupleBased() throws Exception { Topology t = newTopology("testFlushTupleBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newPredicateBasedConfig( tuple -> tuple.startsWith("1-") || tuple.startsWith("3-")), FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), // all in 1 file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
FileWriterCycleConfig.newCountBasedConfig(1), // one per file FileWriterRetentionConfig.newFileCountBasedConfig(10) );
@Test public void testRetainAggSizeBased() throws Exception { // more aggsize than configured; only keep aggsize worth Topology t = newTopology("testRetainAggSizeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuple per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); // agg size only enough for last two lines long aggregateFileSize = 2 * (("1-"+getStr()).getBytes(StandardCharsets.UTF_8).length + 1/*eol*/); expResults.remove(0); expResults.remove(0); assertEquals(2, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(1), FileWriterRetentionConfig.newAggregateFileSizeBasedConfig(aggregateFileSize) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testFlushImplicit() throws Exception { Topology t = newTopology("testFlushImplicit"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testFlushCntBased() throws Exception { Topology t = newTopology("testFlushCntBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newCountBasedConfig(1), // every tuple FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), // all in 1 file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testCompressedFileWriterPolicy() throws Exception { Topology t = newTopology("testCompressedFileWriterPolicy"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net 2 tuples per file int cntTuples = 2; AtomicInteger cnt = new AtomicInteger(); Predicate<String> cycleIt = tuple -> cnt.incrementAndGet() % cntTuples == 0; List<List<String>> expResults = buildExpResults(lines, cycleIt); assertEquals(lines.length / cntTuples, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new CompressedFileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(cntTuples), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testFlushTimeBased() throws Exception { Topology t = newTopology("testFlushTimeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); // add delay so time flush happens int throttleSec = 1; TStream<String> s = PlumbingStreams.blockingThrottle( t.strings(lines), throttleSec, TimeUnit.SECONDS); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newTimeBasedConfig(TimeUnit.MILLISECONDS.toMillis(250)), FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), // all in 1 file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, (lines.length*throttleSec)+TMO_SEC, basePath, expResults); }
@Test public void testCycleCntBased() throws Exception { Topology t = newTopology("testCycleCntBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net two tuples per file int cntTuples = 2; AtomicInteger cnt = new AtomicInteger(); Predicate<String> cycleIt = tuple -> cnt.incrementAndGet() % cntTuples == 0; List<List<String>> expResults = buildExpResults(lines, cycleIt); assertEquals(lines.length / cntTuples, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(cntTuples), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testRetainCntBased() throws Exception { // more lines than configured retained numFiles; only keep the last numFiles Topology t = newTopology("testRetainCntBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuples per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); int keepCnt = 2; // only keep the last n files for (int i = 0; i < keepCnt; i++) expResults.remove(0); assertEquals(keepCnt, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(1), FileWriterRetentionConfig.newFileCountBasedConfig(keepCnt) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
FileWriterCycleConfig.newCountBasedConfig(1), FileWriterRetentionConfig.newAgeBasedConfig(ageSec, periodMsec) );
@Test public void testManyFiles() throws Exception { Topology t = newTopology("testManyFiles"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuples per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); // in this config files are create very fast hence they end // up exercising the _<n> suffix to basePath_YYYYMMDD_HHMMSS TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), // no extra flush FileWriterCycleConfig.newCountBasedConfig(1), // yield one line per file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testManyFilesSlow() throws Exception { Topology t = newTopology("testManyFilesSlow"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuples per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); // add delay so we get different files w/o a _<n> suffix int throttleSec = 2; TStream<String> s = PlumbingStreams.blockingThrottle( t.strings(lines), throttleSec, TimeUnit.SECONDS); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), // no extra flush FileWriterCycleConfig.newCountBasedConfig(1), // yield one line per file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, (lines.length*throttleSec)+TMO_SEC, basePath, expResults); }
expectIAE(() -> FileWriterCycleConfig.newFileSizeBasedConfig(0)); cfg = FileWriterCycleConfig.newCountBasedConfig(3); checkFileWriterConfig(cfg, 0, 3, 0, null, trueTuple, falseTuple); expectIAE(() -> FileWriterCycleConfig.newCountBasedConfig(0));