/** * Use {@link SinkProcessors#writeJdbcP}. */ public static <T> ProcessorMetaSupplier metaSupplier( @Nonnull String updateQuery, @Nonnull DistributedSupplier<? extends Connection> connectionSupplier, @Nonnull DistributedBiConsumer<? super PreparedStatement, ? super T> bindFn ) { return ProcessorMetaSupplier.preferLocalParallelismOne(() -> new WriteJdbcP<>(updateQuery, connectionSupplier, bindFn)); }
/** * Internal API, use {@link SourceProcessors#streamSocketP(String, int, Charset)}. */ public static ProcessorMetaSupplier supplier(String host, int port, @Nonnull String charset) { return ProcessorMetaSupplier.preferLocalParallelismOne( () -> new StreamSocketP(host, port, Charset.forName(charset))); } }
public static BatchSource<User> customSource() { return Sources.batchFromProcessor("custom-source", preferLocalParallelismOne(CustomSourceP::new)); } }
/** * Returns a sink which discards all received items. */ @Nonnull public static <T> Sink<T> noop() { return fromProcessor("noop", preferLocalParallelismOne(noopP())); }
public static <T> ProcessorMetaSupplier supplier( @Nonnull String connectionURL, @Nonnull String query, @Nonnull DistributedFunction<? super ResultSet, ? extends T> mapOutputFn ) { return ProcessorMetaSupplier.forceTotalParallelismOne(ProcessorSupplier.of(() -> new ReadJdbcP<>( () -> DriverManager.getConnection(connectionURL), (connection, parallelism, index) -> { PreparedStatement statement = connection.prepareStatement(query); try { return statement.executeQuery(); } catch (SQLException e) { statement.close(); throw e; } }, mapOutputFn) )); }
/** * Returns a supplier of processors for * {@link Sinks#socket(String, int)}. */ public static <T> ProcessorMetaSupplier writeSocketP( @Nonnull String host, int port, @Nonnull DistributedFunction<? super T, ? extends String> toStringFn, @Nonnull Charset charset ) { checkSerializable(toStringFn, "toStringFn"); String charsetName = charset.name(); return preferLocalParallelismOne(writeBufferedP( index -> new BufferedWriter(new OutputStreamWriter(new Socket(host, port).getOutputStream(), charsetName)), (bufferedWriter, item) -> { @SuppressWarnings("unchecked") T t = (T) item; bufferedWriter.write(toStringFn.apply(t)); bufferedWriter.write('\n'); }, BufferedWriter::flush, BufferedWriter::close )); }
/** * Returns a meta-supplier of processors for a sink vertex that logs all * the data items it receives. The log category is {@code * com.hazelcast.jet.impl.processor.PeekWrappedP.<vertexName>#<processorIndex>} * and the level is INFO. {@link Watermark} items are always logged, but at * FINE level; they are <em>not</em> passed to {@code toStringFn}. * <p> * The vertex logs each item on whichever cluster member it happens to * receive it. Its primary purpose is for development, when running Jet on * a local machine. * * @param toStringFn a function that returns a string representation of a stream item * @param <T> stream item type */ @Nonnull public static <T> ProcessorMetaSupplier writeLoggerP( @Nonnull DistributedFunction<T, ? extends CharSequence> toStringFn ) { checkSerializable(toStringFn, "toStringFn"); return preferLocalParallelismOne(() -> new WriteLoggerP<>(toStringFn)); }
public static ProcessorMetaSupplier streamTwitterP(Properties properties, List<String> terms) { return preferLocalParallelismOne(ProcessorSupplier.of(() -> new StreamTwitterP(properties, terms))); }
/** * Use {@link SinkProcessors#writeFileP(String, DistributedFunction, Charset, boolean)} */ public static <T> ProcessorMetaSupplier metaSupplier( @Nonnull String directoryName, @Nonnull DistributedFunction<? super T, ? extends String> toStringFn, @Nonnull String charset, boolean append) { return ProcessorMetaSupplier.preferLocalParallelismOne(writeBufferedP( ctx -> createBufferedWriter(Paths.get(directoryName), ctx.globalProcessorIndex(), charset, append), (fileWriter, item) -> { fileWriter.write(toStringFn.apply((T) item)); fileWriter.newLine(); }, BufferedWriter::flush, BufferedWriter::close )); }
@Nonnull @SuppressWarnings("unchecked") public static <T, K, V> ProcessorMetaSupplier updateMapSupplier( @Nonnull String name, @Nullable ClientConfig clientConfig, @Nonnull DistributedFunction<? super T, ? extends K> toKeyFn, @Nonnull DistributedFunction<? super T, ? extends EntryProcessor<K, V>> toEntryProcessorFn ) { checkSerializable(toKeyFn, "toKeyFn"); checkSerializable(toEntryProcessorFn, "toEntryProcessorFn"); boolean isLocal = clientConfig == null; return preferLocalParallelismOne(new EntryProcessorWriterSupplier<>( name, asXmlString(clientConfig), toKeyFn, toEntryProcessorFn, isLocal ) ); }
@Nonnull public static ProcessorMetaSupplier writeCacheSupplier(@Nonnull String name, @Nullable ClientConfig clientConfig) { boolean isLocal = clientConfig == null; return preferLocalParallelismOne(new HazelcastWriterSupplier<>( asXmlString(clientConfig), instance -> procContext -> new ArrayMap(), ArrayMap::add, CacheFlush.flushToCache(name, isLocal), DistributedConsumer.noop() )); }
@Nonnull public static ProcessorMetaSupplier writeListSupplier(@Nonnull String name, @Nullable ClientConfig clientConfig) { boolean isLocal = clientConfig == null; return preferLocalParallelismOne(new HazelcastWriterSupplier<>( asXmlString(clientConfig), instance -> procContext -> new ArrayList<>(), ArrayList::add, instance -> { IList<Object> list = instance.getList(name); return buffer -> { try { list.addAll(buffer); } catch (HazelcastInstanceNotActiveException e) { throw handleInstanceNotActive(e, isLocal); } buffer.clear(); }; }, DistributedConsumer.noop() )); }
@Nonnull @SuppressWarnings("unchecked") public static <T, K, V> ProcessorMetaSupplier updateMapSupplier( @Nonnull String mapName, @Nullable ClientConfig clientConfig, @Nonnull DistributedFunction<? super T, ? extends K> toKeyFn, @Nonnull DistributedBiFunction<? super V, ? super T, ? extends V> updateFn ) { checkSerializable(toKeyFn, "toKeyFn"); checkSerializable(updateFn, "updateFn"); boolean isLocal = clientConfig == null; return preferLocalParallelismOne(new HazelcastWriterSupplier<UpdateMapContext<K, V, T>, T>( asXmlString(clientConfig), instance -> procContext -> new UpdateMapContext<>(instance, mapName, toKeyFn, updateFn, isLocal), UpdateMapContext::add, instance -> UpdateMapContext::flush, UpdateMapContext::finish )); }
@Nonnull @SuppressWarnings("unchecked") public static ProcessorMetaSupplier writeMapSupplier(@Nonnull String name, @Nullable ClientConfig clientConfig) { boolean isLocal = clientConfig == null; return preferLocalParallelismOne(new HazelcastWriterSupplier<>( asXmlString(clientConfig), instance -> procContext -> new ArrayMap(), ArrayMap::add, instance -> { IMap map = instance.getMap(name); return buffer -> { try { map.putAll(buffer); } catch (HazelcastInstanceNotActiveException e) { throw handleInstanceNotActive(e, isLocal); } buffer.clear(); }; }, DistributedConsumer.noop() )); }
@Test(timeout = 20000) public void test() { Pipeline p = Pipeline.create(); p.drawFrom(Sources.batchFromProcessor("source", preferLocalParallelismOne(CustomSourceP::new))) .drainTo(Sinks.fromProcessor("sink", preferLocalParallelismOne(CustomSinkP::new))); jetInstance.newJob(p).join(); }