public static Object unboxPossibleOptional(Object result) { if (result instanceof Optional) { Optional optional = (Optional) result; if (optional.isPresent()) { return optional.get(); } else { return null; if (optional.isPresent()) { return optional.getAsInt(); } else { return null; if (optional.isPresent()) { return optional.getAsDouble(); } else { return null; if (optional.isPresent()) { return optional.getAsLong(); } else { return null;
/** * Returns the last element of the specified stream, or {@link OptionalLong#empty} if the stream * is empty. * * <p>Equivalent to {@code stream.reduce((a, b) -> b)}, but may perform significantly better. This * method's runtime will be between O(log n) and O(n), performing better on <a * href="http://gee.cs.oswego.edu/dl/html/StreamParallelGuidance.html">efficiently splittable</a> * streams. * * @see LongStream#findFirst() * @throws NullPointerException if the last element of the stream is null */ public static OptionalLong findLast(LongStream stream) { // findLast(Stream) does some allocation, so we might as well box some more java.util.Optional<Long> boxedLast = findLast(stream.boxed()); return boxedLast.isPresent() ? OptionalLong.of(boxedLast.get()) : OptionalLong.empty(); }
@Override public long getLastSequenceId() { // Return the highest sequence id across all partitions. This will be correct, // since there is a single id generator across all partitions for the same producer return producers.stream().map(Producer::getLastSequenceId).mapToLong(Long::longValue).max().orElse(-1); }
private static OptionalLong toMetastoreDistinctValuesCount(OptionalLong distinctValuesCount, OptionalLong nullsCount) { // metastore counts null as a distinct value if (distinctValuesCount.isPresent() && nullsCount.isPresent()) { return OptionalLong.of(distinctValuesCount.getAsLong() + (nullsCount.getAsLong() > 0 ? 1 : 0)); } return OptionalLong.empty(); }
/** * If a value is present in {@code optional}, returns a stream containing only that element, * otherwise returns an empty stream. * * <p><b>Java 9 users:</b> use {@code optional.stream()} instead. */ public static LongStream stream(OptionalLong optional) { return optional.isPresent() ? LongStream.of(optional.getAsLong()) : LongStream.empty(); }
.orElseThrow(() -> new AssertionError("Table does not exist: " + tableName)); assertEqualsIgnoreOrder(partitionNames, CREATE_TABLE_PARTITIONED_DATA.getMaterializedRows().stream() .map(row -> "ds=" + row.getField(CREATE_TABLE_PARTITIONED_DATA.getTypes().size() - 1)) .collect(toList())); assertEquals(partitions.size(), partitionNames.size()); for (String partitionName : partitionNames) { Partition partition = partitions.get(partitionName).get(); assertEquals(partition.getParameters().get(PRESTO_VERSION_NAME), TEST_SERVER_VERSION); assertEquals(partition.getParameters().get(PRESTO_QUERY_ID_NAME), queryId); MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.of(storageFormat)); assertEqualsIgnoreOrder(result.getMaterializedRows(), CREATE_TABLE_PARTITIONED_DATA.getMaterializedRows()); assertEquals(partitionStatistics.getRowCount().getAsLong(), 1L); assertEquals(partitionStatistics.getFileCount().getAsLong(), 1L); assertGreaterThan(partitionStatistics.getInMemoryDataSizeInBytes().getAsLong(), 0L); assertGreaterThan(partitionStatistics.getOnDiskDataSizeInBytes().getAsLong(), 0L); sink.appendPage(CREATE_TABLE_PARTITIONED_DATA_2ND.toPage()); Collection<Slice> fragments = getFutureValue(sink.finish()); metadata.finishInsert(session, insertTableHandle, fragments, ImmutableList.of()); List<ColumnHandle> columnHandles = filterNonHiddenColumnHandles(metadata.getColumnHandles(session, tableHandle).values()); MaterializedResult result = readTable(transaction, tableHandle, columnHandles, newSession(), TupleDomain.all(), OptionalInt.empty(), Optional.empty()); assertEqualsIgnoreOrder(result.getMaterializedRows(), CREATE_TABLE_PARTITIONED_DATA.getMaterializedRows());
.filter(column -> column.getName().equals("ds")) .map(ColumnMetadata::getName) .collect(toList()); Table table = transaction.getMetastore(tableName.getSchemaName()).getTable(tableName.getSchemaName(), tableName.getTableName()).get(); assertEquals(table.getStorage().getStorageFormat().getInputFormat(), storageFormat.getInputFormat()); MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.of(storageFormat)); assertEquals(result.getRowCount(), 0); assertEquals(statistics.getRowCount().getAsLong(), 0L); assertEquals(statistics.getFileCount().getAsLong(), 0L); assertEquals(statistics.getInMemoryDataSizeInBytes().getAsLong(), 0L); assertEquals(statistics.getOnDiskDataSizeInBytes().getAsLong(), 0L);
.map(source -> new TaskSource( source.getPlanNodeId(), source.getSplits().stream() .filter(scheduledSplit -> scheduledSplit.getSequenceId() > currentMaxAcknowledgedSplit) .collect(Collectors.toSet()), .flatMap(source -> source.getSplits().stream()) .mapToLong(ScheduledSplit::getSequenceId) .max() .orElse(maxAcknowledgedSplit); return updatedUnpartitionedSources;
@Test(dataProvider = "caches") @CacheSpec(implementation = Implementation.Caffeine, requiresWeakOrSoft = true, expireAfterAccess = Expire.DISABLED, expireAfterWrite = Expire.DISABLED, maximumSize = Maximum.UNREACHABLE, weigher = CacheWeigher.COLLECTION, population = Population.EMPTY, stats = Stats.ENABLED, removalListener = Listener.DEFAULT, writer = Writer.DISABLED) public void put_weighted(Cache<Integer, List<Integer>> cache, CacheContext context) { Integer key = context.absentKey(); cache.put(key, ImmutableList.of(1)); GcFinalization.awaitFullGc(); assertThat(cache.asMap().put(key, ImmutableList.of(1, 2, 3)), context.isStrongValues() ? is(ImmutableList.of(1)) : nullValue()); assertThat(cache.policy().eviction().get().weightedSize().getAsLong(), is(3L)); }
private Map<String, Map<String, HiveColumnStatistics>> getPartitionColumnStatistics( String databaseName, String tableName, Set<String> partitionNames, List<String> columnNames, Map<String, OptionalLong> partitionRowCounts) { return getMetastorePartitionColumnStatistics(databaseName, tableName, partitionNames, columnNames).entrySet().stream() .filter(entry -> !entry.getValue().isEmpty()) .collect(toImmutableMap( Map.Entry::getKey, entry -> groupStatisticsByColumn(entry.getValue(), partitionRowCounts.getOrDefault(entry.getKey(), OptionalLong.empty())))); }
ConnectorOutputTableHandle outputHandle = metadata.beginCreateTable(session, tableMetadata, Optional.empty()); metadata.finishCreateTable(session, outputHandle, fragments, ImmutableList.of()); MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.of(storageFormat)); assertEqualsIgnoreOrder(result.getMaterializedRows(), CREATE_TABLE_DATA.getMaterializedRows()); Table table = getMetastoreClient().getTable(tableName.getSchemaName(), tableName.getTableName()).get(); assertEquals(table.getParameters().get(PRESTO_VERSION_NAME), TEST_SERVER_VERSION); assertEquals(table.getParameters().get(PRESTO_QUERY_ID_NAME), queryId); assertEquals(statistics.getRowCount().getAsLong(), CREATE_TABLE_DATA.getRowCount()); assertEquals(statistics.getFileCount().getAsLong(), 1L); assertGreaterThan(statistics.getInMemoryDataSizeInBytes().getAsLong(), 0L); assertGreaterThan(statistics.getOnDiskDataSizeInBytes().getAsLong(), 0L);
private synchronized boolean allAssignmentsHavePropagated(Iterable<QueryExecution> queries) { if (nodes.isEmpty()) { // Assignments can't have propagated, if there are no visible nodes. return false; } long newestAssignment = ImmutableList.copyOf(queries).stream() .map(QueryExecution::getMemoryPool) .mapToLong(VersionedMemoryPoolId::getVersion) .min() .orElse(-1); long mostOutOfDateNode = nodes.values().stream() .mapToLong(RemoteNodeMemory::getCurrentAssignmentVersion) .min() .orElse(Long.MAX_VALUE); return newestAssignment <= mostOutOfDateNode; }
columnStatistics.getMaxValueSizeInBytes().ifPresent(maxValueSizeInBytes -> checkStatistics(maxValueSizeInBytes >= 0, table, partition, column, "maxValueSizeInBytes must be greater than or equal to zero: %s", maxValueSizeInBytes)); columnStatistics.getTotalSizeInBytes().ifPresent(totalSizeInBytes -> checkStatistics(totalSizeInBytes >= 0, table, partition, column, "totalSizeInBytes must be greater than or equal to zero: %s", totalSizeInBytes)); columnStatistics.getNullsCount().ifPresent(nullsCount -> { checkStatistics(nullsCount >= 0, table, partition, column, "nullsCount must be greater than or equal to zero: %s", nullsCount); if (rowCount.isPresent()) { checkStatistics( nullsCount <= rowCount.getAsLong(), table, partition, "nullsCount must be less than or equal to rowCount. nullsCount: %s. rowCount: %s.", nullsCount, rowCount.getAsLong()); columnStatistics.getDistinctValuesCount().ifPresent(distinctValuesCount -> { checkStatistics(distinctValuesCount >= 0, table, partition, column, "distinctValuesCount must be greater than or equal to zero: %s", distinctValuesCount); if (rowCount.isPresent()) { checkStatistics( distinctValuesCount <= rowCount.getAsLong(), table, partition, "distinctValuesCount must be less than or equal to rowCount. distinctValuesCount: %s. rowCount: %s.", distinctValuesCount, rowCount.getAsLong()); if (rowCount.isPresent() && columnStatistics.getNullsCount().isPresent()) {
private OptionalLong getMinCreatedAt(Map<String, QueryBuilder> filters, QueryBuilder esQuery) { String facetNameAndField = CREATED_AT.getFieldName(); SearchRequestBuilder esRequest = client .prepareSearch(INDEX_TYPE_ISSUE) .setSize(0); BoolQueryBuilder esFilter = boolQuery(); filters.values().stream().filter(Objects::nonNull).forEach(esFilter::must); if (esFilter.hasClauses()) { esRequest.setQuery(QueryBuilders.boolQuery().must(esQuery).filter(esFilter)); } else { esRequest.setQuery(esQuery); } esRequest.addAggregation(AggregationBuilders.min(facetNameAndField).field(facetNameAndField)); Min minValue = esRequest.get().getAggregations().get(facetNameAndField); double actualValue = minValue.getValue(); if (Double.isInfinite(actualValue)) { return OptionalLong.empty(); } return OptionalLong.of((long)actualValue); }
private static OptionalLong getDistinctValuesCount(HiveColumnStatistics statistics) { if (statistics.getBooleanStatistics().isPresent() && statistics.getBooleanStatistics().get().getFalseCount().isPresent() && statistics.getBooleanStatistics().get().getTrueCount().isPresent()) { long falseCount = statistics.getBooleanStatistics().get().getFalseCount().getAsLong(); long trueCount = statistics.getBooleanStatistics().get().getTrueCount().getAsLong(); return OptionalLong.of((falseCount > 0 ? 1 : 0) + (trueCount > 0 ? 1 : 0)); } if (statistics.getDistinctValuesCount().isPresent()) { return statistics.getDistinctValuesCount(); } return OptionalLong.empty(); }
Integer value = TypeUtils.castToInt(obj); if (value == null) { return (T) OptionalInt.empty(); } else { return (T) OptionalInt.of(value); Long value = TypeUtils.castToLong(obj); if (value == null) { return (T) OptionalLong.empty(); } else { return (T) OptionalLong.of(value); Double value = TypeUtils.castToDouble(obj); if (value == null) { return (T) OptionalDouble.empty(); } else { return (T) OptionalDouble.of(value); return (T) Optional.empty(); return (T) Optional.of(value);
@Override public Map<String, String> getAttributes(final RecordSchema schema) { final Map<String, String> attributes = new HashMap<>(4); final SchemaIdentifier id = schema.getIdentifier(); final Long schemaId = id.getIdentifier().getAsLong(); final Integer schemaVersion = id.getVersion().getAsInt(); attributes.put(HortonworksAttributeSchemaReferenceStrategy.SCHEMA_ID_ATTRIBUTE, String.valueOf(schemaId)); attributes.put(HortonworksAttributeSchemaReferenceStrategy.SCHEMA_VERSION_ATTRIBUTE, String.valueOf(schemaVersion)); attributes.put(HortonworksAttributeSchemaReferenceStrategy.SCHEMA_PROTOCOL_VERSION_ATTRIBUTE, String.valueOf(LATEST_PROTOCOL_VERSION)); if (id.getBranch().isPresent()) { attributes.put(SCHEMA_BRANCH_ATTRIBUTE, id.getBranch().get()); } return attributes; }
default long getLastMajorCompactionTimestamp(TableName table) { return getLiveServerMetrics().values().stream() .flatMap(s -> s.getRegionMetrics().values().stream()) .filter(r -> RegionInfo.getTable(r.getRegionName()).equals(table)) .mapToLong(RegionMetrics::getLastMajorCompactionTimestamp).min().orElse(0); }