public DoubleSortedAction() { super(s -> s.sorted(), DoubleStream.class, SORTED); }
@Override public DoubleStream sorted() { return wrap(stream().sorted()); }
@Override @SuppressWarnings("unchecked") public TS build(boolean parallel) { final TS built = previous().build(parallel); if (built instanceof Stream<?>) { if (comparator == null) { return (TS) ((Stream<T>) built).sorted(); } else { return (TS) ((Stream<T>) built).sorted(comparator); } } else if (built instanceof IntStream) { return (TS) ((IntStream) built).sorted(); } else if (built instanceof LongStream) { return (TS) ((LongStream) built).sorted(); } else if (built instanceof DoubleStream) { return (TS) ((DoubleStream) built).sorted(); } else { throw new UnsupportedOperationException( "Built stream did not match any known stream interface." ); } } }
private void testAggregationDoubles(InternalAggregationFunction function, Page page, double maxError, double... inputs) { assertAggregation(function, QDIGEST_EQUALITY, "test multiple positions", page, getExpectedValueDoubles(maxError, inputs)); // test scalars List<Double> rows = Arrays.stream(inputs).sorted().boxed().collect(Collectors.toList()); SqlVarbinary returned = (SqlVarbinary) AggregationTestUtils.aggregation(function, page); assertPercentileWithinError(StandardTypes.DOUBLE, returned, maxError, rows, 0.1, 0.5, 0.9, 0.99); }
private void assertPercentilesWithinError(String type, SqlVarbinary binary, double error, List<? extends Number> rows, double[] percentiles) { List<Double> boxedPercentiles = Arrays.stream(percentiles).sorted().boxed().collect(toImmutableList()); List<Number> lowerBounds = boxedPercentiles.stream().map(percentile -> getLowerBound(error, rows, percentile)).collect(toImmutableList()); List<Number> upperBounds = boxedPercentiles.stream().map(percentile -> getUpperBound(error, rows, percentile)).collect(toImmutableList()); // Ensure that the lower bound of each item in the distribution is not greater than the chosen quantiles functionAssertions.assertFunction( format( "zip_with(values_at_quantiles(CAST(X'%s' AS qdigest(%s)), ARRAY[%s]), ARRAY[%s], (value, lowerbound) -> value >= lowerbound)", binary.toString().replaceAll("\\s+", " "), type, ARRAY_JOINER.join(boxedPercentiles), ARRAY_JOINER.join(lowerBounds)), METADATA.getType(parseTypeSignature("array(boolean)")), Collections.nCopies(percentiles.length, true)); // Ensure that the upper bound of each item in the distribution is not less than the chosen quantiles functionAssertions.assertFunction( format( "zip_with(values_at_quantiles(CAST(X'%s' AS qdigest(%s)), ARRAY[%s]), ARRAY[%s], (value, upperbound) -> value <= upperbound)", binary.toString().replaceAll("\\s+", " "), type, ARRAY_JOINER.join(boxedPercentiles), ARRAY_JOINER.join(upperBounds)), METADATA.getType(parseTypeSignature("array(boolean)")), Collections.nCopies(percentiles.length, true)); }
public DoubleSortedAction() { super(s -> s.sorted(), DoubleStream.class, SORTED); }
@Override public ExDoubleStream sorted() { return ExDoubleStream.of(stream.sorted()); }
@Override public DoubleStream sorted() { return new DoubleResourceStream(stream.sorted(), closeOnTerminalOperation, resources); }
/** * @param inputCol The column values to convert to percentiles, remains unmodified * @param outputCol The percentile value that corresponds to the given column value */ @Override protected void applyFunction(final double[] inputCol, final double[] outputCol) { // sort the column, and, for each value in the column, store the *min* position in the sorted array final double[] sortedInputCol = Arrays.stream(inputCol).sorted().toArray(); final Map<Double, Integer> map = new HashMap<>(); for (int i = sortedInputCol.length - 1; i >= 0; --i) { // increment by one so that the max value has 100th percentile map.put(sortedInputCol[i], i + 1); } // normalize the position for each value by the size of the column final double norm = inputCol.length; for (int i = 0; i < inputCol.length; ++i) { outputCol[i] = map.get(inputCol[i]) / norm; } } }
@Override public MDoubleStream sorted(boolean ascending) { if (ascending) { return new LocalDoubleStream(stream.sorted()); } return new LocalDoubleStream(stream.mapToObj(Double::valueOf) .sorted((d1, d2) -> -Double.compare(d1, d2)) .mapToDouble(d -> d)); }
@Override public Object doWork(Object value) throws IOException { if(!(value instanceof List<?>)){ throw new StreamEvaluatorException("List value expected but found type %s for value %s", value.getClass().getName(), value.toString()); } EmpiricalDistribution empiricalDistribution = new EmpiricalDistribution(); double[] backingValues = ((List<?>)value).stream().mapToDouble(innerValue -> ((Number)innerValue).doubleValue()).sorted().toArray(); empiricalDistribution.load(backingValues); return empiricalDistribution; } }
/** * Normalize distribution tensor. * * @return the tensor */ public Tensor normalizeDistribution() { double[] sortedValues = Arrays.stream(getData()).sorted().toArray(); Tensor result = map(v -> Math.abs(((double) Arrays.binarySearch(sortedValues, v)) / ((double) sortedValues.length))); return result; }
/** * Gets percentile. * * @param percentile the percentile * @return the percentile */ public synchronized Double getPercentile(final double percentile) { if (null == values) return Double.NaN; return values.parallelStream().flatMapToDouble(x -> Arrays.stream(x)).sorted().skip((int) (percentile * values.size())).findFirst().orElse(Double.NaN); }
@Override public DoubleStream sorted() { return wrap(stream().sorted()); }
@Override public DoubleStreamEx sorted() { return new DoubleStreamEx(stream().sorted(), context); }
double[] monthsArray = Random.doubles(12, 0, 100).sorted().toArray();
private void testAggregationDoubles(InternalAggregationFunction function, Page page, double maxError, double... inputs) { assertAggregation(function, QDIGEST_EQUALITY, "test multiple positions", page, getExpectedValueDoubles(maxError, inputs)); // test scalars List<Double> rows = Arrays.stream(inputs).sorted().boxed().collect(Collectors.toList()); SqlVarbinary returned = (SqlVarbinary) AggregationTestUtils.aggregation(function, page); assertPercentileWithinError(StandardTypes.DOUBLE, returned, maxError, rows, 0.1, 0.5, 0.9, 0.99); }
private void testAggregationDoubles(InternalAggregationFunction function, Page page, double maxError, double... inputs) { assertAggregation(function, QDIGEST_EQUALITY, "test multiple positions", page, getExpectedValueDoubles(maxError, inputs)); // test scalars List<Double> rows = Arrays.stream(inputs).sorted().boxed().collect(Collectors.toList()); SqlVarbinary returned = (SqlVarbinary) AggregationTestUtils.aggregation(function, page); assertPercentileWithinError(StandardTypes.DOUBLE, returned, maxError, rows, 0.1, 0.5, 0.9, 0.99); }
private void assertPercentilesWithinError(String type, SqlVarbinary binary, double error, List<? extends Number> rows, double[] percentiles) { List<Double> boxedPercentiles = Arrays.stream(percentiles).sorted().boxed().collect(toImmutableList()); List<Number> lowerBounds = boxedPercentiles.stream().map(percentile -> getLowerBound(error, rows, percentile)).collect(toImmutableList()); List<Number> upperBounds = boxedPercentiles.stream().map(percentile -> getUpperBound(error, rows, percentile)).collect(toImmutableList()); // Ensure that the lower bound of each item in the distribution is not greater than the chosen quantiles functionAssertions.assertFunction( format( "zip_with(values_at_quantiles(CAST(X'%s' AS qdigest(%s)), ARRAY[%s]), ARRAY[%s], (value, lowerbound) -> value >= lowerbound)", binary.toString().replaceAll("\\s+", " "), type, ARRAY_JOINER.join(boxedPercentiles), ARRAY_JOINER.join(lowerBounds)), METADATA.getType(parseTypeSignature("array(boolean)")), Collections.nCopies(percentiles.length, true)); // Ensure that the upper bound of each item in the distribution is not less than the chosen quantiles functionAssertions.assertFunction( format( "zip_with(values_at_quantiles(CAST(X'%s' AS qdigest(%s)), ARRAY[%s]), ARRAY[%s], (value, upperbound) -> value <= upperbound)", binary.toString().replaceAll("\\s+", " "), type, ARRAY_JOINER.join(boxedPercentiles), ARRAY_JOINER.join(upperBounds)), METADATA.getType(parseTypeSignature("array(boolean)")), Collections.nCopies(percentiles.length, true)); }
private void assertPercentilesWithinError(String type, SqlVarbinary binary, double error, List<? extends Number> rows, double[] percentiles) { List<Double> boxedPercentiles = Arrays.stream(percentiles).sorted().boxed().collect(toImmutableList()); List<Number> lowerBounds = boxedPercentiles.stream().map(percentile -> getLowerBound(error, rows, percentile)).collect(toImmutableList()); List<Number> upperBounds = boxedPercentiles.stream().map(percentile -> getUpperBound(error, rows, percentile)).collect(toImmutableList()); // Ensure that the lower bound of each item in the distribution is not greater than the chosen quantiles functionAssertions.assertFunction( format( "zip_with(values_at_quantiles(CAST(X'%s' AS qdigest(%s)), ARRAY[%s]), ARRAY[%s], (value, lowerbound) -> value >= lowerbound)", binary.toString().replaceAll("\\s+", " "), type, ARRAY_JOINER.join(boxedPercentiles), ARRAY_JOINER.join(lowerBounds)), METADATA.getType(parseTypeSignature("array(boolean)")), Collections.nCopies(percentiles.length, true)); // Ensure that the upper bound of each item in the distribution is not less than the chosen quantiles functionAssertions.assertFunction( format( "zip_with(values_at_quantiles(CAST(X'%s' AS qdigest(%s)), ARRAY[%s]), ARRAY[%s], (value, upperbound) -> value <= upperbound)", binary.toString().replaceAll("\\s+", " "), type, ARRAY_JOINER.join(boxedPercentiles), ARRAY_JOINER.join(upperBounds)), METADATA.getType(parseTypeSignature("array(boolean)")), Collections.nCopies(percentiles.length, true)); }