congrats Icon
New! Announcing Tabnine Chat Beta
Learn More
Tabnine Logo
StreamStageWithKey
Code IndexAdd Tabnine to your IDE (free)

How to use
StreamStageWithKey
in
com.hazelcast.jet.pipeline

Best Java code snippets using com.hazelcast.jet.pipeline.StreamStageWithKey (Showing top 12 results out of 315)

origin: hazelcast/hazelcast-jet

@Nonnull @Override
public <R> StreamStage<R> rollingAggregate(@Nonnull AggregateOperation1<? super T, ?, ? extends R> aggrOp) {
  return groupingKey(constantKey()).rollingAggregate(aggrOp, (k, v) -> v);
}
origin: hazelcast/hazelcast-jet

@SuppressWarnings("unchecked")
public <T> Tag<T> add(StreamStageWithKey<T, K> stage) {
  ComputeStageImplBase computeStage = ((StageWithGroupingBase) stage).computeStage;
  ensureJetEvents(computeStage, "This pipeline stage");
  upstreamStages.add(computeStage);
  keyFns.add(stage.keyFn());
  return (Tag<T>) tag(upstreamStages.size() - 1);
}
origin: hazelcast/hazelcast-jet-demos

.window(sliding(MINUTES.toMillis(120), MINUTES.toMillis(15)))
.aggregate(linearTrend(CarCount::getTime, CarCount::getCount))
.map((TimestampedEntry<String, Double> e) ->
origin: hazelcast/hazelcast-jet-code-samples

private static Pipeline buildPipeline() {
  Pipeline p = Pipeline.create();
  p.drawFrom(Sources.<Trade, Integer, Trade>mapJournal(TRADES_MAP_NAME,
      DistributedPredicate.alwaysTrue(), EventJournalMapEvent::getNewValue, START_FROM_CURRENT))
   .addTimestamps(Trade::getTime, 3000)
   .groupingKey(Trade::getTicker)
   .window(WindowDefinition.sliding(SLIDING_WINDOW_LENGTH_MILLIS, SLIDE_STEP_MILLIS))
   .aggregate(counting(),
       (winStart, winEnd, key, result) -> String.format("%s %5s %4d", toLocalTime(winEnd), key, result))
   .drainTo(Sinks.logger());
  return p;
}
origin: hazelcast/hazelcast-jet-code-samples

private static Pipeline buildPipeline() {
  Pipeline p = Pipeline.create();
  p.drawFrom(Sources.<PriceUpdateEvent, String, Tuple2<Integer, Long>>mapJournal(
      "prices",
      mapPutEvents(),
      e -> new PriceUpdateEvent(e.getKey(), e.getNewValue().f0(), e.getNewValue().f1()),
      START_FROM_CURRENT
  ))
   .addTimestamps(PriceUpdateEvent::timestamp, LAG_SECONDS * 1000)
   .setLocalParallelism(1)
   .groupingKey(PriceUpdateEvent::ticker)
   .window(WindowDefinition.sliding(WINDOW_SIZE_SECONDS * 1000, 1000))
   .aggregate(AggregateOperations.counting())
   .drainTo(Sinks.logger());
  return p;
}
origin: hazelcast/hazelcast-jet-code-samples

private static Pipeline buildPipeline() {
  Pipeline p = Pipeline.create();
  p.drawFrom(Sources.<Entry<String, Integer>, Integer, Entry<String, Integer>>mapJournal(TRADES_MAP_NAME,
      DistributedPredicate.alwaysTrue(), EventJournalMapEvent::getNewValue, START_FROM_CURRENT))
   .groupingKey(Entry::getKey)
   .rollingAggregate(summingLong(Entry::getValue))
   .drainTo(Sinks.map(VOLUME_MAP_NAME));
  return p;
}
origin: hazelcast/hazelcast-jet

  @SuppressWarnings("unchecked")
  private <T1, T2, A, R, OUT> StreamStage<OUT> attachAggregate3(
      @Nonnull StreamStageWithKey<T1, ? extends K> stage1,
      @Nonnull StreamStageWithKey<T2, ? extends K> stage2,
      @Nonnull AggregateOperation3<? super T, ? super T1, ? super T2, A, R> aggrOp,
      @Nonnull KeyedWindowResultFunction<? super K, ? super R, ? extends OUT> mapToOutputFn
  ) {
    Transform transform1 = ((StageWithGroupingBase) stage1).computeStage.transform;
    Transform transform2 = ((StageWithGroupingBase) stage2).computeStage.transform;
    JetEventFunctionAdapter fnAdapter = ADAPT_TO_JET_EVENT;
    return computeStage.attach(new WindowGroupTransform<K, R, JetEvent<OUT>>(
            asList(computeStage.transform, transform1, transform2),
            wDef,
            asList(fnAdapter.adaptKeyFn(keyFn()),
                fnAdapter.adaptKeyFn(stage1.keyFn()),
                fnAdapter.adaptKeyFn(stage2.keyFn())),
            adaptAggregateOperation3(aggrOp),
            fnAdapter.adaptKeyedWindowResultFn(mapToOutputFn)
        ),
        fnAdapter);
  }
}
origin: hazelcast/hazelcast-jet-code-samples

private static Pipeline buildPipeline() {
  // we'll calculate two aggregations over the same input data:
  // 1. number of viewed product listings
  // 2. set of purchased product IDs
  // Output of the aggregation will be List{Integer, Set<String>}
  AggregateOperation1<ProductEvent, ?, Tuple2<Long, Set<String>>> aggrOp = allOf(
      summingLong(e -> e.getProductEventType() == VIEW_LISTING ? 1 : 0),
      mapping(e -> e.getProductEventType() == PURCHASE ? e.getProductId() : null, toSet())
  );
  Pipeline p = Pipeline.create();
  p.drawFrom(Sources.<ProductEvent>streamFromProcessor("generator",
      ProcessorMetaSupplier.of(GenerateEventsP::new, 1)))
   .addTimestamps(ProductEvent::getTimestamp, 0)
   .groupingKey(ProductEvent::getUserId)
   .window(WindowDefinition.session(SESSION_TIMEOUT))
   .aggregate(aggrOp, SessionWindow::sessionToString)
   .drainTo(Sinks.logger());
  return p;
}
origin: hazelcast/hazelcast-jet-code-samples

private static Pipeline buildPipeline() {
  Pipeline p = Pipeline.create();
  p.drawFrom(Sources.<Trade, Integer, Trade>mapJournal(TRADES_MAP_NAME,
      DistributedPredicate.alwaysTrue(), EventJournalMapEvent::getNewValue, START_FROM_CURRENT))
   .groupingKey(Trade::getTicker)
   .rollingAggregate(summingLong(Trade::getPrice))
   .drainTo(Sinks.map(VOLUME_MAP_NAME));
  return p;
}
origin: hazelcast/hazelcast-jet

@SuppressWarnings("unchecked")
private <T1, A, R, OUT> StreamStage<OUT> attachAggregate2(
    @Nonnull StreamStageWithKey<T1, ? extends K> stage1,
    @Nonnull AggregateOperation2<? super T, ? super T1, A, R> aggrOp,
    @Nonnull KeyedWindowResultFunction<? super K, ? super R, ? extends OUT> mapToOutputFn
) {
  Transform upstream1 = ((StageWithGroupingBase) stage1).computeStage.transform;
  JetEventFunctionAdapter fnAdapter = ADAPT_TO_JET_EVENT;
  return computeStage.attach(new WindowGroupTransform<K, R, JetEvent<OUT>>(
          asList(computeStage.transform, upstream1),
          wDef,
          asList(fnAdapter.adaptKeyFn(keyFn()),
              fnAdapter.adaptKeyFn(stage1.keyFn())),
          adaptAggregateOperation2(aggrOp),
          fnAdapter.adaptKeyedWindowResultFn(mapToOutputFn)
      ),
      fnAdapter);
}
origin: hazelcast/hazelcast-jet-code-samples

@SuppressWarnings("Convert2MethodRef") // https://bugs.openjdk.java.net/browse/JDK-8154236
private static Pipeline coGroup() {
  Pipeline p = Pipeline.create();
  StreamStageWithKey<PageVisit, Integer> pageVisits = p
      .drawFrom(Sources.<PageVisit, Integer, PageVisit>mapJournal(PAGE_VISIT,
          mapPutEvents(), mapEventNewValue(), START_FROM_OLDEST))
      .addTimestamps(pv -> pv.timestamp(), 100)
      .groupingKey(pv -> pv.userId());
  StreamStageWithKey<Payment, Integer> payments = p
      .drawFrom(Sources.<Payment, Integer, Payment>mapJournal(PAYMENT,
          mapPutEvents(), mapEventNewValue(), START_FROM_OLDEST))
      .addTimestamps(pm -> pm.timestamp(), 100)
      .groupingKey(pm -> pm.userId());
  StreamStageWithKey<AddToCart, Integer> addToCarts = p
      .drawFrom(Sources.<AddToCart, Integer, AddToCart>mapJournal(ADD_TO_CART,
          mapPutEvents(), mapEventNewValue(), START_FROM_OLDEST))
      .addTimestamps(atc -> atc.timestamp(), 100)
      .groupingKey(atc -> atc.userId());
  StageWithKeyAndWindow<PageVisit, Integer> windowStage = pageVisits.window(sliding(10, 1));
  StreamStage<TimestampedEntry<Integer, Tuple3<List<PageVisit>, List<AddToCart>, List<Payment>>>> coGrouped =
      windowStage.aggregate3(toList(), addToCarts, toList(), payments, toList());
  coGrouped.drainTo(Sinks.logger());
  return p;
}
origin: hazelcast/hazelcast-jet-code-samples

@SuppressWarnings("Convert2MethodRef") // https://bugs.openjdk.java.net/browse/JDK-8154236
private static Pipeline coGroupWithBuilder() {
  Pipeline p = Pipeline.create();
  StreamStageWithKey<PageVisit, Integer> pageVisits = p
      .drawFrom(Sources.<PageVisit, Integer, PageVisit>mapJournal(PAGE_VISIT,
          mapPutEvents(), mapEventNewValue(), START_FROM_OLDEST))
      .addTimestamps(pv -> pv.timestamp(), 100)
      .groupingKey(pv -> pv.userId());
  StreamStageWithKey<AddToCart, Integer> addToCarts = p
      .drawFrom(Sources.<AddToCart, Integer, AddToCart>mapJournal(ADD_TO_CART,
          mapPutEvents(), mapEventNewValue(), START_FROM_OLDEST))
      .addTimestamps(atc -> atc.timestamp(), 100)
      .groupingKey(atc -> atc.userId());
  StreamStageWithKey<Payment, Integer> payments = p
      .drawFrom(Sources.<Payment, Integer, Payment>mapJournal(PAYMENT,
          mapPutEvents(), mapEventNewValue(), START_FROM_OLDEST))
      .addTimestamps(pm -> pm.timestamp(), 100)
      .groupingKey(pm -> pm.userId());
  StageWithKeyAndWindow<PageVisit, Integer> windowStage = pageVisits.window(sliding(10, 1));
  WindowGroupAggregateBuilder<Integer, List<PageVisit>> builder = windowStage.aggregateBuilder(toList());
  Tag<List<PageVisit>> pageVisitTag = builder.tag0();
  Tag<List<AddToCart>> addToCartTag = builder.add(addToCarts, toList());
  Tag<List<Payment>> paymentTag = builder.add(payments, toList());
  StreamStage<TimestampedEntry<Integer, Tuple3<List<PageVisit>, List<AddToCart>, List<Payment>>>> coGrouped =
      builder.build((winStart, winEnd, key, r) -> new TimestampedEntry<>(
          winEnd, key, tuple3(r.get(pageVisitTag), r.get(addToCartTag), r.get(paymentTag))));
  coGrouped.drainTo(Sinks.logger());
  return p;
}
com.hazelcast.jet.pipelineStreamStageWithKey

Javadoc

An intermediate step while constructing a windowed group-and-aggregate pipeline stage. It captures the grouping key and offers a method to specify the window definition.

Most used methods

  • window
    Adds the definition of the window to use in the group-and-aggregate pipeline stage being constructed
  • rollingAggregate
  • keyFn

Popular in Java

  • Updating database using SQL prepared statement
  • notifyDataSetChanged (ArrayAdapter)
  • scheduleAtFixedRate (ScheduledExecutorService)
  • getOriginalFilename (MultipartFile)
    Return the original filename in the client's filesystem.This may contain path information depending
  • HttpServer (com.sun.net.httpserver)
    This class implements a simple HTTP server. A HttpServer is bound to an IP address and port number a
  • Hashtable (java.util)
    A plug-in replacement for JDK1.5 java.util.Hashtable. This version is based on org.cliffc.high_scale
  • Set (java.util)
    A Set is a data structure which does not allow duplicate elements.
  • ConcurrentHashMap (java.util.concurrent)
    A plug-in replacement for JDK1.5 java.util.concurrent.ConcurrentHashMap. This version is based on or
  • JPanel (javax.swing)
  • Get (org.apache.hadoop.hbase.client)
    Used to perform Get operations on a single row. To get everything for a row, instantiate a Get objec
  • Top Vim plugins
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now