@Override protected void init(@Nonnull Context context) throws Exception { Path path = Paths.get(this.path, context.jetInstance().getName() + '-' + context.globalProcessorIndex()); writer = Files.newBufferedWriter(path, StandardCharsets.UTF_8); }
/** * Returns a supplier of processors for {@link AvroSinks#files}. */ @Nonnull public static <D> ProcessorMetaSupplier writeFilesP( @Nonnull String directoryName, @Nonnull DistributedSupplier<Schema> schemaSupplier, @Nonnull DistributedSupplier<DatumWriter<D>> datumWriterSupplier ) { return ProcessorMetaSupplier.of( WriteBufferedP.<DataFileWriter<D>, D>supplier( context -> createWriter(Paths.get(directoryName), context.globalProcessorIndex(), schemaSupplier, datumWriterSupplier), DataFileWriter::append, DataFileWriter::flush, DataFileWriter::close ), 1); }
@Override protected void init(@Nonnull Context context) throws Exception { processorIndex = sharedFileSystem ? context.globalProcessorIndex() : context.localProcessorIndex(); parallelism = sharedFileSystem ? context.totalParallelism() : context.localParallelism(); try (DirectoryStream<Path> directoryStream = Files.newDirectoryStream(watchedDirectory)) { for (Path file : directoryStream) { if (Files.isRegularFile(file)) { // Negative offset means "initial offset", needed to skip the first line fileOffsets.put(file, new FileOffset(-Files.size(file), "")); } } } watcher = FileSystems.getDefault().newWatchService(); watchedDirectory.register(watcher, WATCH_EVENT_KINDS, WATCH_EVENT_MODIFIERS); getLogger().info("Started to watch directory: " + watchedDirectory); }
/** * Use {@link SinkProcessors#writeFileP(String, DistributedFunction, Charset, boolean)} */ public static <T> ProcessorMetaSupplier metaSupplier( @Nonnull String directoryName, @Nonnull DistributedFunction<? super T, ? extends String> toStringFn, @Nonnull String charset, boolean append) { return ProcessorMetaSupplier.preferLocalParallelismOne(writeBufferedP( ctx -> createBufferedWriter(Paths.get(directoryName), ctx.globalProcessorIndex(), charset, append), (fileWriter, item) -> { fileWriter.write(toStringFn.apply((T) item)); fileWriter.newLine(); }, BufferedWriter::flush, BufferedWriter::close )); }
@Override protected void init(@Nonnull Context context) { processorIndex = context.globalProcessorIndex(); totalParallelism = context.totalParallelism(); snapshottingEnabled = context.snapshottingEnabled(); consumer = new KafkaConsumer<>(properties); assignPartitions(false); }
@Override protected void init(@Nonnull Context context) { this.connection = connectionSupplier.get(); this.parallelism = context.totalParallelism(); this.index = context.globalProcessorIndex(); }
@Override protected void init(@Nonnull Context context) throws Exception { processorIndex = sharedFileSystem ? context.globalProcessorIndex() : context.localProcessorIndex(); parallelism = sharedFileSystem ? context.totalParallelism() : context.localParallelism(); directoryStream = Files.newDirectoryStream(directory, glob); outputTraverser = Traversers.traverseIterator(directoryStream.iterator()) .filter(this::shouldProcessEvent) .flatMap(this::processFile); }
@Override protected void init(@Nonnull Context context) { processorIndex = context.globalProcessorIndex(); totalParallelism = context.totalParallelism(); snapshottingEnabled = context.snapshottingEnabled(); consumer = new KafkaConsumer<>(properties); assignPartitions(false); }
@Override public String toString() { String jobPrefix = context.jobConfig().getName() == null ? "" : context.jobConfig().getName() + "/"; return "ProcessorTasklet{" + jobPrefix + context.vertexName() + '#' + context.globalProcessorIndex() + '}'; }