congrats Icon
New! Tabnine Pro 14-day free trial
Start a free trial
Tabnine Logo
StreamStageWithKey
Code IndexAdd Tabnine to your IDE (free)

How to use
StreamStageWithKey
in
com.hazelcast.jet.pipeline

Best Java code snippets using com.hazelcast.jet.pipeline.StreamStageWithKey (Showing top 12 results out of 315)

origin: hazelcast/hazelcast-jet

@Nonnull @Override
public <R> StreamStage<R> rollingAggregate(@Nonnull AggregateOperation1<? super T, ?, ? extends R> aggrOp) {
  return groupingKey(constantKey()).rollingAggregate(aggrOp, (k, v) -> v);
}
origin: hazelcast/hazelcast-jet

@SuppressWarnings("unchecked")
public <T> Tag<T> add(StreamStageWithKey<T, K> stage) {
  ComputeStageImplBase computeStage = ((StageWithGroupingBase) stage).computeStage;
  ensureJetEvents(computeStage, "This pipeline stage");
  upstreamStages.add(computeStage);
  keyFns.add(stage.keyFn());
  return (Tag<T>) tag(upstreamStages.size() - 1);
}
origin: hazelcast/hazelcast-jet-demos

.window(sliding(MINUTES.toMillis(120), MINUTES.toMillis(15)))
.aggregate(linearTrend(CarCount::getTime, CarCount::getCount))
.map((TimestampedEntry<String, Double> e) ->
origin: hazelcast/hazelcast-jet-code-samples

private static Pipeline buildPipeline() {
  Pipeline p = Pipeline.create();
  p.drawFrom(Sources.<Trade, Integer, Trade>mapJournal(TRADES_MAP_NAME,
      DistributedPredicate.alwaysTrue(), EventJournalMapEvent::getNewValue, START_FROM_CURRENT))
   .addTimestamps(Trade::getTime, 3000)
   .groupingKey(Trade::getTicker)
   .window(WindowDefinition.sliding(SLIDING_WINDOW_LENGTH_MILLIS, SLIDE_STEP_MILLIS))
   .aggregate(counting(),
       (winStart, winEnd, key, result) -> String.format("%s %5s %4d", toLocalTime(winEnd), key, result))
   .drainTo(Sinks.logger());
  return p;
}
origin: hazelcast/hazelcast-jet-code-samples

private static Pipeline buildPipeline() {
  Pipeline p = Pipeline.create();
  p.drawFrom(Sources.<PriceUpdateEvent, String, Tuple2<Integer, Long>>mapJournal(
      "prices",
      mapPutEvents(),
      e -> new PriceUpdateEvent(e.getKey(), e.getNewValue().f0(), e.getNewValue().f1()),
      START_FROM_CURRENT
  ))
   .addTimestamps(PriceUpdateEvent::timestamp, LAG_SECONDS * 1000)
   .setLocalParallelism(1)
   .groupingKey(PriceUpdateEvent::ticker)
   .window(WindowDefinition.sliding(WINDOW_SIZE_SECONDS * 1000, 1000))
   .aggregate(AggregateOperations.counting())
   .drainTo(Sinks.logger());
  return p;
}
origin: hazelcast/hazelcast-jet-code-samples

private static Pipeline buildPipeline() {
  Pipeline p = Pipeline.create();
  p.drawFrom(Sources.<Entry<String, Integer>, Integer, Entry<String, Integer>>mapJournal(TRADES_MAP_NAME,
      DistributedPredicate.alwaysTrue(), EventJournalMapEvent::getNewValue, START_FROM_CURRENT))
   .groupingKey(Entry::getKey)
   .rollingAggregate(summingLong(Entry::getValue))
   .drainTo(Sinks.map(VOLUME_MAP_NAME));
  return p;
}
origin: hazelcast/hazelcast-jet

  @SuppressWarnings("unchecked")
  private <T1, T2, A, R, OUT> StreamStage<OUT> attachAggregate3(
      @Nonnull StreamStageWithKey<T1, ? extends K> stage1,
      @Nonnull StreamStageWithKey<T2, ? extends K> stage2,
      @Nonnull AggregateOperation3<? super T, ? super T1, ? super T2, A, R> aggrOp,
      @Nonnull KeyedWindowResultFunction<? super K, ? super R, ? extends OUT> mapToOutputFn
  ) {
    Transform transform1 = ((StageWithGroupingBase) stage1).computeStage.transform;
    Transform transform2 = ((StageWithGroupingBase) stage2).computeStage.transform;
    JetEventFunctionAdapter fnAdapter = ADAPT_TO_JET_EVENT;
    return computeStage.attach(new WindowGroupTransform<K, R, JetEvent<OUT>>(
            asList(computeStage.transform, transform1, transform2),
            wDef,
            asList(fnAdapter.adaptKeyFn(keyFn()),
                fnAdapter.adaptKeyFn(stage1.keyFn()),
                fnAdapter.adaptKeyFn(stage2.keyFn())),
            adaptAggregateOperation3(aggrOp),
            fnAdapter.adaptKeyedWindowResultFn(mapToOutputFn)
        ),
        fnAdapter);
  }
}
origin: hazelcast/hazelcast-jet-code-samples

private static Pipeline buildPipeline() {
  // we'll calculate two aggregations over the same input data:
  // 1. number of viewed product listings
  // 2. set of purchased product IDs
  // Output of the aggregation will be List{Integer, Set<String>}
  AggregateOperation1<ProductEvent, ?, Tuple2<Long, Set<String>>> aggrOp = allOf(
      summingLong(e -> e.getProductEventType() == VIEW_LISTING ? 1 : 0),
      mapping(e -> e.getProductEventType() == PURCHASE ? e.getProductId() : null, toSet())
  );
  Pipeline p = Pipeline.create();
  p.drawFrom(Sources.<ProductEvent>streamFromProcessor("generator",
      ProcessorMetaSupplier.of(GenerateEventsP::new, 1)))
   .addTimestamps(ProductEvent::getTimestamp, 0)
   .groupingKey(ProductEvent::getUserId)
   .window(WindowDefinition.session(SESSION_TIMEOUT))
   .aggregate(aggrOp, SessionWindow::sessionToString)
   .drainTo(Sinks.logger());
  return p;
}
origin: hazelcast/hazelcast-jet-code-samples

private static Pipeline buildPipeline() {
  Pipeline p = Pipeline.create();
  p.drawFrom(Sources.<Trade, Integer, Trade>mapJournal(TRADES_MAP_NAME,
      DistributedPredicate.alwaysTrue(), EventJournalMapEvent::getNewValue, START_FROM_CURRENT))
   .groupingKey(Trade::getTicker)
   .rollingAggregate(summingLong(Trade::getPrice))
   .drainTo(Sinks.map(VOLUME_MAP_NAME));
  return p;
}
origin: hazelcast/hazelcast-jet

@SuppressWarnings("unchecked")
private <T1, A, R, OUT> StreamStage<OUT> attachAggregate2(
    @Nonnull StreamStageWithKey<T1, ? extends K> stage1,
    @Nonnull AggregateOperation2<? super T, ? super T1, A, R> aggrOp,
    @Nonnull KeyedWindowResultFunction<? super K, ? super R, ? extends OUT> mapToOutputFn
) {
  Transform upstream1 = ((StageWithGroupingBase) stage1).computeStage.transform;
  JetEventFunctionAdapter fnAdapter = ADAPT_TO_JET_EVENT;
  return computeStage.attach(new WindowGroupTransform<K, R, JetEvent<OUT>>(
          asList(computeStage.transform, upstream1),
          wDef,
          asList(fnAdapter.adaptKeyFn(keyFn()),
              fnAdapter.adaptKeyFn(stage1.keyFn())),
          adaptAggregateOperation2(aggrOp),
          fnAdapter.adaptKeyedWindowResultFn(mapToOutputFn)
      ),
      fnAdapter);
}
origin: hazelcast/hazelcast-jet-code-samples

@SuppressWarnings("Convert2MethodRef") // https://bugs.openjdk.java.net/browse/JDK-8154236
private static Pipeline coGroup() {
  Pipeline p = Pipeline.create();
  StreamStageWithKey<PageVisit, Integer> pageVisits = p
      .drawFrom(Sources.<PageVisit, Integer, PageVisit>mapJournal(PAGE_VISIT,
          mapPutEvents(), mapEventNewValue(), START_FROM_OLDEST))
      .addTimestamps(pv -> pv.timestamp(), 100)
      .groupingKey(pv -> pv.userId());
  StreamStageWithKey<Payment, Integer> payments = p
      .drawFrom(Sources.<Payment, Integer, Payment>mapJournal(PAYMENT,
          mapPutEvents(), mapEventNewValue(), START_FROM_OLDEST))
      .addTimestamps(pm -> pm.timestamp(), 100)
      .groupingKey(pm -> pm.userId());
  StreamStageWithKey<AddToCart, Integer> addToCarts = p
      .drawFrom(Sources.<AddToCart, Integer, AddToCart>mapJournal(ADD_TO_CART,
          mapPutEvents(), mapEventNewValue(), START_FROM_OLDEST))
      .addTimestamps(atc -> atc.timestamp(), 100)
      .groupingKey(atc -> atc.userId());
  StageWithKeyAndWindow<PageVisit, Integer> windowStage = pageVisits.window(sliding(10, 1));
  StreamStage<TimestampedEntry<Integer, Tuple3<List<PageVisit>, List<AddToCart>, List<Payment>>>> coGrouped =
      windowStage.aggregate3(toList(), addToCarts, toList(), payments, toList());
  coGrouped.drainTo(Sinks.logger());
  return p;
}
origin: hazelcast/hazelcast-jet-code-samples

@SuppressWarnings("Convert2MethodRef") // https://bugs.openjdk.java.net/browse/JDK-8154236
private static Pipeline coGroupWithBuilder() {
  Pipeline p = Pipeline.create();
  StreamStageWithKey<PageVisit, Integer> pageVisits = p
      .drawFrom(Sources.<PageVisit, Integer, PageVisit>mapJournal(PAGE_VISIT,
          mapPutEvents(), mapEventNewValue(), START_FROM_OLDEST))
      .addTimestamps(pv -> pv.timestamp(), 100)
      .groupingKey(pv -> pv.userId());
  StreamStageWithKey<AddToCart, Integer> addToCarts = p
      .drawFrom(Sources.<AddToCart, Integer, AddToCart>mapJournal(ADD_TO_CART,
          mapPutEvents(), mapEventNewValue(), START_FROM_OLDEST))
      .addTimestamps(atc -> atc.timestamp(), 100)
      .groupingKey(atc -> atc.userId());
  StreamStageWithKey<Payment, Integer> payments = p
      .drawFrom(Sources.<Payment, Integer, Payment>mapJournal(PAYMENT,
          mapPutEvents(), mapEventNewValue(), START_FROM_OLDEST))
      .addTimestamps(pm -> pm.timestamp(), 100)
      .groupingKey(pm -> pm.userId());
  StageWithKeyAndWindow<PageVisit, Integer> windowStage = pageVisits.window(sliding(10, 1));
  WindowGroupAggregateBuilder<Integer, List<PageVisit>> builder = windowStage.aggregateBuilder(toList());
  Tag<List<PageVisit>> pageVisitTag = builder.tag0();
  Tag<List<AddToCart>> addToCartTag = builder.add(addToCarts, toList());
  Tag<List<Payment>> paymentTag = builder.add(payments, toList());
  StreamStage<TimestampedEntry<Integer, Tuple3<List<PageVisit>, List<AddToCart>, List<Payment>>>> coGrouped =
      builder.build((winStart, winEnd, key, r) -> new TimestampedEntry<>(
          winEnd, key, tuple3(r.get(pageVisitTag), r.get(addToCartTag), r.get(paymentTag))));
  coGrouped.drainTo(Sinks.logger());
  return p;
}
com.hazelcast.jet.pipelineStreamStageWithKey

Javadoc

An intermediate step while constructing a windowed group-and-aggregate pipeline stage. It captures the grouping key and offers a method to specify the window definition.

Most used methods

  • window
    Adds the definition of the window to use in the group-and-aggregate pipeline stage being constructed
  • rollingAggregate
  • keyFn

Popular in Java

  • Reading from database using SQL prepared statement
  • getContentResolver (Context)
  • getExternalFilesDir (Context)
  • setRequestProperty (URLConnection)
  • HttpURLConnection (java.net)
    An URLConnection for HTTP (RFC 2616 [http://tools.ietf.org/html/rfc2616]) used to send and receive d
  • Timestamp (java.sql)
    A Java representation of the SQL TIMESTAMP type. It provides the capability of representing the SQL
  • JFileChooser (javax.swing)
  • Get (org.apache.hadoop.hbase.client)
    Used to perform Get operations on a single row. To get everything for a row, instantiate a Get objec
  • Loader (org.hibernate.loader)
    Abstract superclass of object loading (and querying) strategies. This class implements useful common
  • Scheduler (org.quartz)
    This is the main interface of a Quartz Scheduler. A Scheduler maintains a registry of org.quartz.Job
  • Top plugins for WebStorm
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimAtomGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyStudentsTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now