/** same as {@code newConfig(0, 0, 0, tuplePredicate)} * * @param <T> Tuple type * @param tuplePredicate cycle if {@code tuplePredicate} returns true. null to disable. * @return the cycle configuration */ public static <T> FileWriterCycleConfig<T> newPredicateBasedConfig(Predicate<T> tuplePredicate) { return newConfig(0, 0, 0, tuplePredicate); }
/** * Create a new configuration. * * @param fileCount remove a file when {@code fileCount} would be exceeded. 0 to disable. * @param aggregateFileSize remove a file when {@code aggregateFileSize} would be exceeded. 0 to disable. * @param ageSec remove a file that's older than {@code ageSec} seconds. 0 to disable. * @param periodMsec frequency for checking for ageSec based removal. 0 to disable.] * @return the retention config */ public static FileWriterRetentionConfig newConfig(int fileCount, long aggregateFileSize, long ageSec, long periodMsec) { return new FileWriterRetentionConfig(fileCount, aggregateFileSize, ageSec, periodMsec); }
@Override public String toString() { return String.format("fileSize:%d cntTuples:%d periodMsec:%d tuplePredicate:%s", getFileSize(), getCntTuples(), getPeriodMsec(), getTuplePredicate() == null ? "no" : "yes"); } }
/** * Create a new file writer policy instance. * <p> * The configuration is: * <ul> * <li>10 second time based active file flushing</li> * <li>1MB file size based active file cycling</li> * <li>10 file retention count</li> * </ul> * The active and final file pathname behavior is specified in * {@link #FileWriterPolicy(FileWriterFlushConfig, FileWriterCycleConfig, FileWriterRetentionConfig)} */ public FileWriterPolicy() { this(FileWriterFlushConfig.newTimeBasedConfig(TimeUnit.SECONDS.toMillis(10)), FileWriterCycleConfig.newFileSizeBasedConfig(1*1024*1024), FileWriterRetentionConfig.newFileCountBasedConfig(10)); }
@Override public String toString() { return String.format("fileCount:%d aggSize:%d ageSec:%d", getFileCount(), getAggregateFileSize(), getAgeSec()); }
@Override public String toString() { return String.format("cntTuples:%d periodMsec:%d tuplePredicate:%s", getCntTuples(), getPeriodMsec(), getTuplePredicate() == null ? "no" : "yes"); }
/** * Create a new configuration. * <p> * The underlying buffered writer's automatic flushing is used. * <p> * Same as {@code newConfig(0, 0, null)} * * @param <T> Tuple type * @return the flush configuration */ public static <T> FileWriterFlushConfig<T> newImplicitConfig() { return newConfig(0,0,null); } /** same as {@code newConfig(cntTuples, 0, null)}
/** * Declare a stream containing the absolute pathname of * newly created file names from watching {@code directory}. * <p> * This is the same as {@code directoryWatcher(t, () -> dir, null)}. * * @param te topology element whose topology the watcher will be added to * @param directory * Name of the directory to watch. * @return Stream containing absolute pathnames of newly created files in * {@code directory}. */ public static TStream<String> directoryWatcher(TopologyElement te, Supplier<String> directory) { return directoryWatcher(te, directory, null); }
/** same as {@code newConfig(0, aggregateFileSize, 0, 0)} * * @param aggregateFileSize remove a file when {@code aggregateFileSize} would be exceeded. 0 to disable. * @return the retention config */ public static FileWriterRetentionConfig newAggregateFileSizeBasedConfig(long aggregateFileSize) { if (aggregateFileSize < 1) throw new IllegalArgumentException("aggregateFileSize"); return newConfig(0, aggregateFileSize, 0, 0); } /** same as {@code newConfig(0, 0, ageSe, periodMsecc)}
/** * Create a new configuration. * <p> * At least one configuration mode must be enabled. * @param <T> Tuple type * @param fileSize cycle after {@code fileSize} bytes have been written. 0 to disable. * @param cntTuples cycle after every {@code cntTuple} tuples have been written. 0 to disable. * @param periodMsec cycle after {@code periodMsec} has elapsed since the last time based cycle. 0 to disable. * @param tuplePredicate cycle if {@code tuplePredicate} returns true. null to disable. * @return the cycle configuration */ public static <T> FileWriterCycleConfig<T> newConfig(long fileSize, int cntTuples, long periodMsec, Predicate<T> tuplePredicate) { return new FileWriterCycleConfig<>(fileSize, cntTuples, periodMsec, tuplePredicate); }
/** * Create a new configuration. * <p> * If nothing specific is specified, the underlying buffered * writer's automatic flushing is utilized. * * @param <T> Tuple type * @param cntTuples flush every {@code cntTuple} tuples written. 0 to disable. * @param periodMsec flush every {@code periodMsec} milliseconds. 0 to disable. * @param tuplePredicate flush if {@code tuplePredicate} is true. null to disable. * @return the flush configuration */ public static <T> FileWriterFlushConfig<T> newConfig(int cntTuples, long periodMsec, Predicate<T> tuplePredicate) { return new FileWriterFlushConfig<>(cntTuples, periodMsec, tuplePredicate); }
@Override public Path getNextActiveFilePath() { Path path = hookGenerateNextActiveFilePath(); trace.trace("next active file path={}", path); return path; }
@Override protected Path hookGenerateFinalFilePath(Path path) { // finalPath = the normal finalPath + SUFFIX Path finalPath = super.hookGenerateFinalFilePath(path); finalPath = finalPath.getParent().resolve(finalPath.getFileName() + SUFFIX); return finalPath; }
/** same as {@code newConfig(0, periodMsec, null)} * * @param <T> Tuple type * @param periodMsec flush every {@code periodMsec} milliseconds. 0 to disable. * @return the flush configuration */ public static <T> FileWriterFlushConfig<T> newTimeBasedConfig(long periodMsec) { if (periodMsec < 1) throw new IllegalArgumentException("periodMsec"); return newConfig(0, periodMsec, null); } /** same as {@code newConfig(0, 0, tuplePredicate)}
/** same as {@code newConfig(fileSize, 0, 0, null)} * * @param <T> Tuple type * @param fileSize cycle after {@code fileSize} bytes have been written. 0 to disable. * @return the cycle configuration */ public static <T> FileWriterCycleConfig<T> newFileSizeBasedConfig(long fileSize) { if (fileSize < 1) throw new IllegalArgumentException("fileSize"); return newConfig(fileSize, 0, 0, null); } /** same as {@code newConfig(0, cntTuples, 0, null)}
/** same as {@code newConfig(0, 0, tuplePredicate)} * * @param <T> Tuple type * @param tuplePredicate flush if {@code tuplePredicate} is true. null to disable. * @return the flush configuration */ public static <T> FileWriterFlushConfig<T> newPredicateBasedConfig(Predicate<T> tuplePredicate) { if (tuplePredicate == null) throw new IllegalArgumentException("tuplePredicate"); return newConfig(0, 0, tuplePredicate); } /**
/** same as {@code newConfig(0, cntTuples, 0, null)} * * @param <T> Tuple type * @param cntTuples cycle after every {@code cntTuple} tuples have been written. 0 to disable. * @return the cycle configuration */ public static <T> FileWriterCycleConfig<T> newCountBasedConfig(int cntTuples) { if (cntTuples < 1) throw new IllegalArgumentException("cntTuples"); return newConfig(0, cntTuples, 0, null); } /** same as {@code newConfig(0, 0, periodMsec, null)}
/** same as {@code newConfig(cntTuples, 0, null)} * * @param <T> Tuple type * @param cntTuples flush every {@code cntTuple} tuples written. 0 to disable. * @return the flush configuration */ public static <T> FileWriterFlushConfig<T> newCountBasedConfig(int cntTuples) { if (cntTuples < 1) throw new IllegalArgumentException("cntTuples"); return newConfig(cntTuples, 0, null); } /** same as {@code newConfig(0, periodMsec, null)}