@Test public void testDefaultConfig() throws Exception { FileWriterPolicy<String> policy = new FileWriterPolicy<>(); checkFileWriterConfig(policy.getFlushConfig(), 0, TimeUnit.SECONDS.toMillis(10), null, null, null); checkFileWriterConfig(policy.getCycleConfig(), 1*1024*1024, 0, 0, null, null, null); checkFileWriterConfig(policy.getRetentionConfig(), 10, 0, 0, 0); policy.toString(); policy.close(); }
@Override public synchronized Path closeActiveFile(Path path) throws IOException { int tmpCurTupleCnt = curTupleCnt; resetActiveFileInfo(); Path finalPath = hookGenerateFinalFilePath(path); trace.trace("closing active file nTuples={}, finalPath={}", tmpCurTupleCnt, finalPath); hookRenameFile(path, finalPath); retainedPaths.add(finalPath); applyRetention(); return finalPath; }
getExecutor().scheduleAtFixedRate( () -> { try { this.flushable.flush(); } catch (IOException e) { /*ignore*/ } getExecutor().scheduleAtFixedRate( () -> { try { this.closeable.close(); } catch (IOException e) { /*ignore*/ } getExecutor().scheduleAtFixedRate( () -> applyTimeBasedRetention(), periodMsec, periodMsec, TimeUnit.MILLISECONDS);
@Test public void testFlushTupleBased() throws Exception { Topology t = newTopology("testFlushTupleBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newPredicateBasedConfig( tuple -> tuple.startsWith("1-") || tuple.startsWith("3-")), FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), // all in 1 file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Override public Path getNextActiveFilePath() { Path path = hookGenerateNextActiveFilePath(); trace.trace("next active file path={}", path); return path; }
@Override protected Path hookGenerateFinalFilePath(Path path) { // finalPath = the normal finalPath + SUFFIX Path finalPath = super.hookGenerateFinalFilePath(path); finalPath = finalPath.getParent().resolve(finalPath.getFileName() + SUFFIX); return finalPath; }
contents, throttleSec, TimeUnit.SECONDS); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(1), // one per file
@Test public void testCycleTupleBased() throws Exception { Topology t = newTopology("testCycleTupleBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // a tuple based config / predicate. in this case should end up with 3 files. // flush on the last tuple too to ensure the test completes before TMO. Predicate<String> cycleIt = tuple -> tuple.startsWith("1-") || tuple.startsWith("3-") || tuple.equals(lines[lines.length-1]); List<List<String>> expResults = buildExpResults(lines, cycleIt); assertEquals(3, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newPredicateBasedConfig(cycleIt), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testAllTimeBased() throws Exception { // exercise case with multiple timer based policies Topology t = newTopology("testAllTimeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // keep all given age and TMO_SEC int ageSec = 10; long periodMsec = TimeUnit.SECONDS.toMillis(1); // net one tuple per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newTimeBasedConfig(TimeUnit.MILLISECONDS.toMillis(250)), FileWriterCycleConfig.newConfig(1, 2000, TimeUnit.SECONDS.toMillis(10), null), FileWriterRetentionConfig.newAgeBasedConfig(ageSec, periodMsec) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testRetainAggSizeBased() throws Exception { // more aggsize than configured; only keep aggsize worth Topology t = newTopology("testRetainAggSizeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuple per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); // agg size only enough for last two lines long aggregateFileSize = 2 * (("1-"+getStr()).getBytes(StandardCharsets.UTF_8).length + 1/*eol*/); expResults.remove(0); expResults.remove(0); assertEquals(2, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(1), FileWriterRetentionConfig.newAggregateFileSizeBasedConfig(aggregateFileSize) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testFlushImplicit() throws Exception { Topology t = newTopology("testFlushImplicit"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testFlushCntBased() throws Exception { Topology t = newTopology("testFlushCntBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newCountBasedConfig(1), // every tuple FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), // all in 1 file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testFlushTimeBased() throws Exception { Topology t = newTopology("testFlushTimeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); // add delay so time flush happens int throttleSec = 1; TStream<String> s = PlumbingStreams.blockingThrottle( t.strings(lines), throttleSec, TimeUnit.SECONDS); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newTimeBasedConfig(TimeUnit.MILLISECONDS.toMillis(250)), FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), // all in 1 file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, (lines.length*throttleSec)+TMO_SEC, basePath, expResults); }
@Test public void testCycleCntBased() throws Exception { Topology t = newTopology("testCycleCntBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net two tuples per file int cntTuples = 2; AtomicInteger cnt = new AtomicInteger(); Predicate<String> cycleIt = tuple -> cnt.incrementAndGet() % cntTuples == 0; List<List<String>> expResults = buildExpResults(lines, cycleIt); assertEquals(lines.length / cntTuples, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(cntTuples), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testRetainCntBased() throws Exception { // more lines than configured retained numFiles; only keep the last numFiles Topology t = newTopology("testRetainCntBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuples per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); int keepCnt = 2; // only keep the last n files for (int i = 0; i < keepCnt; i++) expResults.remove(0); assertEquals(keepCnt, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(1), FileWriterRetentionConfig.newFileCountBasedConfig(keepCnt) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
t.strings(lines), throttleSec, TimeUnit.SECONDS); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(1),
@Test public void testCycleTimeBased() throws Exception { Topology t = newTopology("testCycleTimeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuple per file with 250msec cycle config and 1 throttle List<List<String>> expResults = buildExpResults(lines, tuple -> true); // add delay so time cycle happens // also verifies only cycle if there's something to cycle // (i.e., these cycles happen faster than tuples are written) int throttleSec = 1; TStream<String> s = PlumbingStreams.blockingThrottle( t.strings(lines), throttleSec, TimeUnit.SECONDS); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newTimeBasedConfig(TimeUnit.MILLISECONDS.toMillis(250)), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, (lines.length*throttleSec)+TMO_SEC, basePath, expResults); }
@Test public void testManyFiles() throws Exception { Topology t = newTopology("testManyFiles"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuples per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); // in this config files are create very fast hence they end // up exercising the _<n> suffix to basePath_YYYYMMDD_HHMMSS TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), // no extra flush FileWriterCycleConfig.newCountBasedConfig(1), // yield one line per file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testCycleSizeBased() throws Exception { Topology t = newTopology("testCycleSizeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuple per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); int fileSize = 2; TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newFileSizeBasedConfig(fileSize), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testManyFilesSlow() throws Exception { Topology t = newTopology("testManyFilesSlow"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuples per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); // add delay so we get different files w/o a _<n> suffix int throttleSec = 2; TStream<String> s = PlumbingStreams.blockingThrottle( t.strings(lines), throttleSec, TimeUnit.SECONDS); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), // no extra flush FileWriterCycleConfig.newCountBasedConfig(1), // yield one line per file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, (lines.length*throttleSec)+TMO_SEC, basePath, expResults); }