@Override public IdentifiedDataSerializable create(int typeId) { switch (typeId) { case DAG: return new DAG(); case EDGE: return new Edge(); case VERTEX: return new Vertex(); case APPLY_FN_ENTRY_PROCESSOR: return new ApplyFnEntryProcessor(); default: throw new IllegalArgumentException("Unknown type id " + typeId); } } }
public static void main(String[] args) throws Exception { System.setProperty("hazelcast.logging.type", "log4j"); Jet.newJetInstance(); JetInstance jet = Jet.newJetInstance(); try { IMapJet<Object, Object> map = jet.getMap("map"); range(0, COUNT).parallel().forEach(i -> map.put("key-" + i, i)); DAG dag = new DAG(); Vertex source = dag.newVertex("map-source", SourceProcessors.readMapP(map.getName())); Vertex sink = dag.newVertex("file-sink", new WriteFilePSupplier(OUTPUT_FOLDER)); dag.edge(between(source, sink)); jet.newJob(dag).join(); System.out.println("\nHazelcast IMap dumped to folder " + new File(OUTPUT_FOLDER).getAbsolutePath()); } finally { Jet.shutdownAll(); } } }
@Nonnull private static DAG buildDag(Map<String, Long> counts) { final Pattern delimiter = Pattern.compile("\\W+"); DAG dag = new DAG(); Vertex source = dag.newVertex("source", DocLinesP::new); Vertex tokenize = dag.newVertex("tokenize", flatMapP((String line) -> traverseArray(delimiter.split(line.toLowerCase())) .filter(word -> !word.isEmpty())) ); Vertex aggregate = dag.newVertex("aggregate", aggregateByKeyP(singletonList(wholeItem()), counting(), Util::entry)); Vertex sink = dag.newVertex("sink", () -> new MapSinkP(counts)); return dag.edge(between(source.localParallelism(1), tokenize)) .edge(between(tokenize, aggregate).partitioned(wholeItem(), HASH_CODE)) .edge(between(aggregate, sink)); }
public static CompletableFuture<Void> copyMapUsingJob(JetInstance instance, int queueSize, String sourceMap, String targetMap) { DAG dag = new DAG(); Vertex source = dag.newVertex("readMap(" + sourceMap + ')', readMapP(sourceMap)); Vertex sink = dag.newVertex("writeMap(" + targetMap + ')', writeMapP(targetMap)); dag.edge(between(source, sink).setConfig(new EdgeConfig().setQueueSize(queueSize))); JobConfig jobConfig = new JobConfig() .setName("copy-" + sourceMap + "-to-" + targetMap); return instance.newJob(dag, jobConfig).getFuture(); } }
private static DAG buildDag() { DAG dag = new DAG(); DistributedToLongFunction<? super Trade> timestampFn = Trade::getTime; DistributedFunction<? super Trade, ?> keyFn = Trade::getTicker;
final Pattern delimiter = Pattern.compile("\\W+"); DAG dag = new DAG();
private static DAG buildDag() { DAG dag = new DAG(); SlidingWindowPolicy winPolicy = slidingWinPolicy(SLIDING_WINDOW_LENGTH_MILLIS, SLIDE_STEP_MILLIS);
public static void main(String[] args) { System.setProperty("hazelcast.logging.type", "log4j"); JetInstance instance = Jet.newJetInstance(); Jet.newJetInstance(); try { TickerInfo.populateMap(instance.getHazelcastInstance().getReplicatedMap("tickersInfo")); DAG dag = new DAG(); Vertex tradesSource = dag.newVertex("tradesSource", GenerateTradesP::new); Vertex enrichment = dag.newVertex("enrichment", mapUsingContextP(replicatedMapContext("tickersInfo"), (ReplicatedMap<String, TickerInfo> map, Trade item) -> tuple2(item, map.get(item.getTicker())))); Vertex sink = dag.newVertex("sink", DiagnosticProcessors.writeLoggerP()); tradesSource.localParallelism(1); dag .edge(between(tradesSource, enrichment)) .edge(between(enrichment, sink)); instance.newJob(dag).join(); } finally { Jet.shutdownAll(); } } }
private static DAG createDag() { DistributedFunction<Entry<Entry<?, String>, ?>, String> byWord = item -> item.getKey().getValue(); DAG dag = new DAG();
TickerInfo.populateMap(instance.getMap(TICKER_INFO_MAP_NAME)); DAG dag = new DAG();