@Override public void wrote(T tuple, long nbytes) { curSize += nbytes; curTupleCnt++; flushIt = flushConfig.evaluate(curTupleCnt, tuple); cycleIt = cycleConfig.evaluate(curSize, curTupleCnt, tuple); }
@Override public String toString() { return String.format("fileSize:%d cntTuples:%d periodMsec:%d tuplePredicate:%s", getFileSize(), getCntTuples(), getPeriodMsec(), getTuplePredicate() == null ? "no" : "yes"); } }
/** same as {@code newConfig(0, 0, 0, tuplePredicate)} * * @param <T> Tuple type * @param tuplePredicate cycle if {@code tuplePredicate} returns true. null to disable. * @return the cycle configuration */ public static <T> FileWriterCycleConfig<T> newPredicateBasedConfig(Predicate<T> tuplePredicate) { return newConfig(0, 0, 0, tuplePredicate); }
private static <T> void checkFileWriterConfig(FileWriterCycleConfig<T> cfg, long fileSize, int cntTuples, long periodMsec, Predicate<T> tuplePredicate, T trueTuple, T falseTuple) { assertEquals(fileSize, cfg.getFileSize()); assertEquals(cntTuples, cfg.getCntTuples()); assertEquals(periodMsec, cfg.getPeriodMsec()); assertEquals(tuplePredicate, cfg.getTuplePredicate()); cfg.toString(); long falseFileSize = fileSize-1; long trueFileSize = fileSize+1; int falseNTuples = cntTuples==1 ? 0 : cntTuples+1; int trueNTuples = 3*cntTuples; assertFalse("fileSize:"+fileSize+" cntTuples:"+cntTuples+" pred:"+tuplePredicate, cfg.evaluate(falseFileSize, falseNTuples, falseTuple)); if (fileSize!=0) assertTrue("fileSize:"+fileSize+" cntTuples:"+cntTuples+" pred:"+tuplePredicate, cfg.evaluate(trueFileSize, trueNTuples, falseTuple)); if (cntTuples!=0) assertTrue("fileSize:"+fileSize+" cntTuples:"+cntTuples+" pred:"+tuplePredicate, cfg.evaluate(falseFileSize, trueNTuples, falseTuple)); if (tuplePredicate!=null) assertTrue("fileSize:"+fileSize+" cntTuples:"+cntTuples+" pred:"+tuplePredicate, cfg.evaluate(falseFileSize, falseNTuples, trueTuple)); }
Predicate<String> p = tuple -> tuple.equals("true"); cfg = FileWriterCycleConfig.newFileSizeBasedConfig(2); checkFileWriterConfig(cfg, 2, 0, 0, null, trueTuple, falseTuple); expectIAE(() -> FileWriterCycleConfig.newFileSizeBasedConfig(0)); cfg = FileWriterCycleConfig.newCountBasedConfig(3); checkFileWriterConfig(cfg, 0, 3, 0, null, trueTuple, falseTuple); expectIAE(() -> FileWriterCycleConfig.newCountBasedConfig(0)); cfg = FileWriterCycleConfig.newTimeBasedConfig(10); checkFileWriterConfig(cfg, 0, 0, 10, null, trueTuple, falseTuple); expectIAE(() -> FileWriterCycleConfig.newTimeBasedConfig(0)); cfg = FileWriterCycleConfig.newPredicateBasedConfig(p); checkFileWriterConfig(cfg, 0, 0, 0, p, trueTuple, falseTuple); expectIAE(() -> FileWriterCycleConfig.newPredicateBasedConfig(null)); cfg = FileWriterCycleConfig.newConfig(1, 2, 3, p); checkFileWriterConfig(cfg, 1, 2, 3, p, trueTuple, falseTuple); expectIAE(() -> FileWriterCycleConfig.newConfig(0, 0, 0, null)); expectIAE(() -> FileWriterCycleConfig.newConfig(-1, 0, 0, null)); expectIAE(() -> FileWriterCycleConfig.newConfig(0, -1, 0, null)); expectIAE(() -> FileWriterCycleConfig.newConfig(0, 0, -1, null));
@Test public void testFlushTupleBased() throws Exception { Topology t = newTopology("testFlushTupleBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newPredicateBasedConfig( tuple -> tuple.startsWith("1-") || tuple.startsWith("3-")), FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), // all in 1 file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
/** * Create a new file writer policy instance. * <p> * The configuration is: * <ul> * <li>10 second time based active file flushing</li> * <li>1MB file size based active file cycling</li> * <li>10 file retention count</li> * </ul> * The active and final file pathname behavior is specified in * {@link #FileWriterPolicy(FileWriterFlushConfig, FileWriterCycleConfig, FileWriterRetentionConfig)} */ public FileWriterPolicy() { this(FileWriterFlushConfig.newTimeBasedConfig(TimeUnit.SECONDS.toMillis(10)), FileWriterCycleConfig.newFileSizeBasedConfig(1*1024*1024), FileWriterRetentionConfig.newFileCountBasedConfig(10)); }
@Override public String toString() { return String.format("basePathname:%s [retention: %s] [cycle: %s] [flush: %s]", basePathname, retentionConfig.toString(), cycleConfig.toString(), flushConfig.toString()); }
periodMsec, periodMsec, TimeUnit.MILLISECONDS); if (cycleConfig.getPeriodMsec() > 0) { long periodMsec = cycleConfig.getPeriodMsec(); getExecutor().scheduleAtFixedRate( () -> { try { this.closeable.close(); }
/** * Create a new configuration. * <p> * At least one configuration mode must be enabled. * @param <T> Tuple type * @param fileSize cycle after {@code fileSize} bytes have been written. 0 to disable. * @param cntTuples cycle after every {@code cntTuple} tuples have been written. 0 to disable. * @param periodMsec cycle after {@code periodMsec} has elapsed since the last time based cycle. 0 to disable. * @param tuplePredicate cycle if {@code tuplePredicate} returns true. null to disable. * @return the cycle configuration */ public static <T> FileWriterCycleConfig<T> newConfig(long fileSize, int cntTuples, long periodMsec, Predicate<T> tuplePredicate) { return new FileWriterCycleConfig<>(fileSize, cntTuples, periodMsec, tuplePredicate); }
FileWriterCycleConfig.newCountBasedConfig(1), // one per file FileWriterRetentionConfig.newFileCountBasedConfig(10) );
@Test public void testCycleSizeBased() throws Exception { Topology t = newTopology("testCycleSizeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuple per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); int fileSize = 2; TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newFileSizeBasedConfig(fileSize), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
/** same as {@code newConfig(0, 0, periodMsec, null)} * * @param <T> Tuple type * @param periodMsec cycle after {@code periodMsec} has elapsed since the last time based cycle. 0 to disable. * @return the cycle configuration */ public static <T> FileWriterCycleConfig<T> newTimeBasedConfig(long periodMsec) { if (periodMsec < 1) throw new IllegalArgumentException("periodMsec"); return newConfig(0, 0, periodMsec, null); } /** same as {@code newConfig(0, 0, 0, tuplePredicate)}
@Test public void testRetainAggSizeBased() throws Exception { // more aggsize than configured; only keep aggsize worth Topology t = newTopology("testRetainAggSizeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuple per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); // agg size only enough for last two lines long aggregateFileSize = 2 * (("1-"+getStr()).getBytes(StandardCharsets.UTF_8).length + 1/*eol*/); expResults.remove(0); expResults.remove(0); assertEquals(2, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(1), FileWriterRetentionConfig.newAggregateFileSizeBasedConfig(aggregateFileSize) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
/** same as {@code newConfig(fileSize, 0, 0, null)} * * @param <T> Tuple type * @param fileSize cycle after {@code fileSize} bytes have been written. 0 to disable. * @return the cycle configuration */ public static <T> FileWriterCycleConfig<T> newFileSizeBasedConfig(long fileSize) { if (fileSize < 1) throw new IllegalArgumentException("fileSize"); return newConfig(fileSize, 0, 0, null); } /** same as {@code newConfig(0, cntTuples, 0, null)}
@Test public void testFlushImplicit() throws Exception { Topology t = newTopology("testFlushImplicit"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
/** same as {@code newConfig(0, cntTuples, 0, null)} * * @param <T> Tuple type * @param cntTuples cycle after every {@code cntTuple} tuples have been written. 0 to disable. * @return the cycle configuration */ public static <T> FileWriterCycleConfig<T> newCountBasedConfig(int cntTuples) { if (cntTuples < 1) throw new IllegalArgumentException("cntTuples"); return newConfig(0, cntTuples, 0, null); } /** same as {@code newConfig(0, 0, periodMsec, null)}
@Test public void testFlushCntBased() throws Exception { Topology t = newTopology("testFlushCntBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newCountBasedConfig(1), // every tuple FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), // all in 1 file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testAllTimeBased() throws Exception { // exercise case with multiple timer based policies Topology t = newTopology("testAllTimeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // keep all given age and TMO_SEC int ageSec = 10; long periodMsec = TimeUnit.SECONDS.toMillis(1); // net one tuple per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newTimeBasedConfig(TimeUnit.MILLISECONDS.toMillis(250)), FileWriterCycleConfig.newConfig(1, 2000, TimeUnit.SECONDS.toMillis(10), null), FileWriterRetentionConfig.newAgeBasedConfig(ageSec, periodMsec) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testCompressedFileWriterPolicy() throws Exception { Topology t = newTopology("testCompressedFileWriterPolicy"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net 2 tuples per file int cntTuples = 2; AtomicInteger cnt = new AtomicInteger(); Predicate<String> cycleIt = tuple -> cnt.incrementAndGet() % cntTuples == 0; List<List<String>> expResults = buildExpResults(lines, cycleIt); assertEquals(lines.length / cntTuples, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new CompressedFileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(cntTuples), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }