/** * Returns a {@code PTransform} that takes an input {@code PCollection<KV<K, Long>>} and returns a * {@code PCollection<KV<K, Long>>} that contains an output element mapping each distinct key in * the input {@code PCollection} to the sum of the values associated with that key in the input * {@code PCollection}. */ public static <K> Combine.PerKey<K, Long, Long> longsPerKey() { return Combine.perKey(Sum.ofLongs()); }
/** * Returns a {@code PTransform} that takes an input {@code PCollection<Long>} and returns a {@code * PCollection<Long>} whose contents is the sum of the input {@code PCollection}'s elements, or * {@code 0} if there are no elements. */ public static Combine.Globally<Long, Long> longsGlobally() { return Combine.globally(Sum.ofLongs()); }
@VisibleForTesting AdaptiveThrottler(long samplePeriodMs, long sampleUpdateMs, double overloadRatio, Random random) { allRequests = new MovingFunction( samplePeriodMs, sampleUpdateMs, 1 /* numSignificantBuckets */, 1 /* numSignificantSamples */, Sum.ofLongs()); successfulRequests = new MovingFunction( samplePeriodMs, sampleUpdateMs, 1 /* numSignificantBuckets */, 1 /* numSignificantSamples */, Sum.ofLongs()); this.overloadRatio = overloadRatio; this.random = random; }
public MovingAverage( long samplePeriodMs, long sampleUpdateMs, int numSignificantBuckets, int numSignificantSamples) { sum = new MovingFunction( samplePeriodMs, sampleUpdateMs, numSignificantBuckets, numSignificantSamples, Sum.ofLongs()); count = new MovingFunction( samplePeriodMs, sampleUpdateMs, numSignificantBuckets, numSignificantSamples, Sum.ofLongs()); }
/** {@link CombineFn} for Sum based on {@link Sum} and {@link Combine.BinaryCombineFn}. */ static CombineFn createSum(Schema.TypeName fieldType) { switch (fieldType) { case INT32: return Sum.ofIntegers(); case INT16: return new ShortSum(); case BYTE: return new ByteSum(); case INT64: return Sum.ofLongs(); case FLOAT: return new FloatSum(); case DOUBLE: return Sum.ofDoubles(); case DECIMAL: return new BigDecimalSum(); default: throw new UnsupportedOperationException( String.format("[%s] is not support in SUM", fieldType)); } }
@Test public void testSumLongFn() { testCombineFn(Sum.ofLongs(), Lists.newArrayList(1L, 2L, 3L, 4L), 10L); }
@Test public void testAccumulatorCombiningStateWithUnderlying() throws CannotProvideCoderException { CopyOnAccessInMemoryStateInternals<String> underlying = CopyOnAccessInMemoryStateInternals.withUnderlying(key, null); CombineFn<Long, long[], Long> sumLongFn = Sum.ofLongs(); StateNamespace namespace = new StateNamespaceForTest("foo"); CoderRegistry reg = pipeline.getCoderRegistry(); StateTag<CombiningState<Long, long[], Long>> stateTag = StateTags.combiningValue( "summer", sumLongFn.getAccumulatorCoder(reg, reg.getCoder(Long.class)), sumLongFn); GroupingState<Long, Long> underlyingValue = underlying.state(namespace, stateTag); assertThat(underlyingValue.read(), equalTo(0L)); underlyingValue.add(1L); assertThat(underlyingValue.read(), equalTo(1L)); CopyOnAccessInMemoryStateInternals<String> internals = CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying); GroupingState<Long, Long> copyOnAccessState = internals.state(namespace, stateTag); assertThat(copyOnAccessState.read(), equalTo(1L)); copyOnAccessState.add(4L); assertThat(copyOnAccessState.read(), equalTo(5L)); assertThat(underlyingValue.read(), equalTo(1L)); GroupingState<Long, Long> reReadUnderlyingValue = underlying.state(namespace, stateTag); assertThat(underlyingValue.read(), equalTo(reReadUnderlyingValue.read())); }
@Test public void testAccumulatorCombiningStateWithUnderlying() throws CannotProvideCoderException { CopyOnAccessInMemoryStateInternals<String> underlying = CopyOnAccessInMemoryStateInternals.withUnderlying(key, null); CombineFn<Long, long[], Long> sumLongFn = Sum.ofLongs(); StateNamespace namespace = new StateNamespaceForTest("foo"); CoderRegistry reg = pipeline.getCoderRegistry(); StateTag<CombiningState<Long, long[], Long>> stateTag = StateTags.combiningValue( "summer", sumLongFn.getAccumulatorCoder(reg, reg.getCoder(Long.class)), sumLongFn); GroupingState<Long, Long> underlyingValue = underlying.state(namespace, stateTag); assertThat(underlyingValue.read(), equalTo(0L)); underlyingValue.add(1L); assertThat(underlyingValue.read(), equalTo(1L)); CopyOnAccessInMemoryStateInternals<String> internals = CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying); GroupingState<Long, Long> copyOnAccessState = internals.state(namespace, stateTag); assertThat(copyOnAccessState.read(), equalTo(1L)); copyOnAccessState.add(4L); assertThat(copyOnAccessState.read(), equalTo(5L)); assertThat(underlyingValue.read(), equalTo(1L)); GroupingState<Long, Long> reReadUnderlyingValue = underlying.state(namespace, stateTag); assertThat(underlyingValue.read(), equalTo(reReadUnderlyingValue.read())); }
@Test public void testLongStats() { for (TestCase<Long> t : LONG_CASES) { assertEquals(t.sum, Sum.ofLongs().apply(t.data)); assertEquals(t.min, Min.ofLongs().apply(t.data)); assertEquals(t.max, Max.ofLongs().apply(t.data)); assertEquals(t.mean, Mean.<Long>of().apply(t.data)); } } }
VarLongCoder.of(), AppliedCombineFn.withInputCoder( Sum.ofLongs(), CoderRegistry.createDefault(), KvCoder.of(VarLongCoder.of(), VarLongCoder.of())));
VarLongCoder.of(), AppliedCombineFn.withInputCoder( Sum.ofLongs(), CoderRegistry.createDefault(), KvCoder.of(VarLongCoder.of(), VarLongCoder.of())));
@Test public void getEnvironmentCombine() throws IOException { SdkComponents components = SdkComponents.create(); components.registerEnvironment(Environments.createDockerEnvironment("java")); CombinePayload payload = CombinePayload.newBuilder() .setCombineFn(CombineTranslation.toProto(Sum.ofLongs(), components)) .build(); RehydratedComponents rehydratedComponents = RehydratedComponents.forComponents(components.toComponents()); PTransform builder = PTransform.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.COMBINE_PER_KEY_TRANSFORM_URN) .setPayload(payload.toByteString()) .build()) .build(); Environment env = Environments.getEnvironment(builder, rehydratedComponents).get(); assertThat( env, equalTo( components .toComponents() .getEnvironmentsOrThrow(payload.getCombineFn().getEnvironmentId()))); } }
@Test public void testGetAccumulatorCoderEquals() { Combine.BinaryCombineIntegerFn sumIntegerFn = Sum.ofIntegers(); assertEquals( sumIntegerFn.getAccumulatorCoder(STANDARD_REGISTRY, VarIntCoder.of()), sumIntegerFn.getAccumulatorCoder(STANDARD_REGISTRY, VarIntCoder.of())); assertNotEquals( sumIntegerFn.getAccumulatorCoder(STANDARD_REGISTRY, VarIntCoder.of()), sumIntegerFn.getAccumulatorCoder(STANDARD_REGISTRY, BigEndianIntegerCoder.of())); Combine.BinaryCombineLongFn sumLongFn = Sum.ofLongs(); assertEquals( sumLongFn.getAccumulatorCoder(STANDARD_REGISTRY, VarLongCoder.of()), sumLongFn.getAccumulatorCoder(STANDARD_REGISTRY, VarLongCoder.of())); assertNotEquals( sumLongFn.getAccumulatorCoder(STANDARD_REGISTRY, VarLongCoder.of()), sumLongFn.getAccumulatorCoder(STANDARD_REGISTRY, BigEndianLongCoder.of())); Combine.BinaryCombineDoubleFn sumDoubleFn = Sum.ofDoubles(); assertEquals( sumDoubleFn.getAccumulatorCoder(STANDARD_REGISTRY, DoubleCoder.of()), sumDoubleFn.getAccumulatorCoder(STANDARD_REGISTRY, DoubleCoder.of())); } }
@Test @Category(NeedsRunner.class) public void testGloballyWithSchemaAggregateFn() { Collection<AggregatePojos> elements = ImmutableList.of( new AggregatePojos(1, 1, 2), new AggregatePojos(2, 1, 3), new AggregatePojos(3, 2, 4), new AggregatePojos(4, 2, 5)); PCollection<Row> aggregate = pipeline .apply(Create.of(elements)) .apply( Group.<AggregatePojos>globally() .aggregateField("field1", Sum.ofLongs(), "field1_sum") .aggregateField("field3", Sum.ofIntegers(), "field3_sum") .aggregateField("field1", Top.largestLongsFn(1), "field1_top")); Schema aggregateSchema = Schema.builder() .addInt64Field("field1_sum") .addInt32Field("field3_sum") .addArrayField("field1_top", FieldType.INT64) .build(); Row expectedRow = Row.withSchema(aggregateSchema).addValues(10L, 14).addArray(4L).build(); PAssert.that(aggregate).containsInAnyOrder(expectedRow); pipeline.run(); }
.apply( Group.<OuterAggregate>globally() .aggregateFields(field1Selector, Sum.ofLongs(), "field1_sum") .aggregateFields(field3Selector, Sum.ofIntegers(), "field3_sum") .aggregateFields(field1Selector, Top.largestLongsFn(1), "field1_top"));
.apply( Group.<AggregatePojos>byFieldNames("field2") .aggregateField("field1", Sum.ofLongs(), "field1_sum") .aggregateField("field3", Sum.ofIntegers(), "field3_sum") .aggregateField("field1", Top.largestLongsFn(1), "field1_top"));
.apply( Group.<OuterAggregate>byFieldAccessDescriptor(field2Selector) .aggregateFields(field1Selector, Sum.ofLongs(), "field1_sum") .aggregateFields(field3Selector, Sum.ofIntegers(), "field3_sum") .aggregateFields(field1Selector, Top.largestLongsFn(1), "field1_top"));