public double[] metricPercentilesAsArray(long[] counts) { double[] result = new double[7]; if (isEmpty(counts)) { Arrays.fill(result, Double.NaN); return result; } double[] offsetPercentiles = new double[] { 0.5, 0.75, 0.95, 0.98, 0.99 }; EstimatedHistogram metric = new EstimatedHistogram(counts); if (metric.isOverflowed()) { System.err.println(String.format("EstimatedHistogram overflowed larger than %s, unable to calculate percentiles", metric.getLargestBucketOffset())); for (int i = 0; i < result.length; i++) result[i] = Double.NaN; } else { for (int i = 0; i < offsetPercentiles.length; i++) result[i] = metric.percentile(offsetPercentiles[i]); } result[5] = metric.min(); result[6] = metric.max(); return result; }
estimatedRowSizePercentiles[i] = partitionSizeHist.percentile(offsetPercentiles[i]); estimatedColumnCountPercentiles[i] = columnCountHist.percentile(offsetPercentiles[i]);
estimatedRowSizePercentiles[i] = partitionSizeHist.percentile(offsetPercentiles[i]); estimatedColumnCountPercentiles[i] = columnCountHist.percentile(offsetPercentiles[i]);
estimatedRowSizePercentiles[i] = partitionSizeHist.percentile(offsetPercentiles[i]); estimatedColumnCountPercentiles[i] = columnCountHist.percentile(offsetPercentiles[i]);
sstable.buildSummary(false, false, Downsampling.BASE_SAMPLING_LEVEL); long indexFileLength = new File(descriptor.filenameFor(Component.PRIMARY_INDEX)).length(); int dataBufferSize = sstable.optimizationStrategy.bufferSize(statsMetadata.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile())); int indexBufferSize = sstable.optimizationStrategy.bufferSize(indexFileLength / sstable.indexSummary.size()); sstable.ifile = ibuilder.bufferSize(indexBufferSize).complete();
sstable.buildSummary(false, false, Downsampling.BASE_SAMPLING_LEVEL); long indexFileLength = new File(descriptor.filenameFor(Component.PRIMARY_INDEX)).length(); int dataBufferSize = sstable.optimizationStrategy.bufferSize(statsMetadata.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile())); int indexBufferSize = sstable.optimizationStrategy.bufferSize(indexFileLength / sstable.indexSummary.size()); sstable.ifile = ibuilder.bufferSize(indexBufferSize).complete();
sstable.buildSummary(false, false, Downsampling.BASE_SAMPLING_LEVEL); long indexFileLength = new File(descriptor.filenameFor(Component.PRIMARY_INDEX)).length(); int dataBufferSize = sstable.optimizationStrategy.bufferSize(statsMetadata.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile())); int indexBufferSize = sstable.optimizationStrategy.bufferSize(indexFileLength / sstable.indexSummary.size()); sstable.ifile = ibuilder.bufferSize(indexBufferSize).complete();
protected ReusablePartitionData createBuffer(int blocksCount) { int estimatedRowCount = 16; int columnCount = metadata().partitionColumns().regulars.size(); if (columnCount == 0 || metadata().clusteringColumns().isEmpty()) { estimatedRowCount = 1; } else { try { // To avoid wasted resizing we guess-estimate the number of rows we're likely to read. For that // we use the stats on the number of rows per partition for that sstable. // FIXME: so far we only keep stats on cells, so to get a rough estimate on the number of rows, // we divide by the number of regular columns the table has. We should fix once we collect the // stats on rows int estimatedRowsPerPartition = (int)(sstable.getEstimatedColumnCount().percentile(0.75) / columnCount); estimatedRowCount = Math.max(estimatedRowsPerPartition / blocksCount, 1); } catch (IllegalStateException e) { // The EstimatedHistogram mean() method can throw this (if it overflows). While such overflow // shouldn't happen, it's not worth taking the risk of letting the exception bubble up. } } return new ReusablePartitionData(metadata(), partitionKey(), columns(), estimatedRowCount); }
protected ReusablePartitionData createBuffer(int blocksCount) { int estimatedRowCount = 16; int columnCount = metadata().partitionColumns().regulars.size(); if (columnCount == 0 || metadata().clusteringColumns().isEmpty()) { estimatedRowCount = 1; } else { try { // To avoid wasted resizing we guess-estimate the number of rows we're likely to read. For that // we use the stats on the number of rows per partition for that sstable. // FIXME: so far we only keep stats on cells, so to get a rough estimate on the number of rows, // we divide by the number of regular columns the table has. We should fix once we collect the // stats on rows int estimatedRowsPerPartition = (int)(sstable.getEstimatedColumnCount().percentile(0.75) / columnCount); estimatedRowCount = Math.max(estimatedRowsPerPartition / blocksCount, 1); } catch (IllegalStateException e) { // The EstimatedHistogram mean() method can throw this (if it overflows). While such overflow // shouldn't happen, it's not worth taking the risk of letting the exception bubble up. } } return new ReusablePartitionData(metadata(), partitionKey(), columns(), estimatedRowCount); }
protected ReusablePartitionData createBuffer(int blocksCount) { int estimatedRowCount = 16; int columnCount = metadata().partitionColumns().regulars.size(); if (columnCount == 0 || metadata().clusteringColumns().isEmpty()) { estimatedRowCount = 1; } else { try { // To avoid wasted resizing we guess-estimate the number of rows we're likely to read. For that // we use the stats on the number of rows per partition for that sstable. // FIXME: so far we only keep stats on cells, so to get a rough estimate on the number of rows, // we divide by the number of regular columns the table has. We should fix once we collect the // stats on rows int estimatedRowsPerPartition = (int)(sstable.getEstimatedColumnCount().percentile(0.75) / columnCount); estimatedRowCount = Math.max(estimatedRowsPerPartition / blocksCount, 1); } catch (IllegalStateException e) { // The EstimatedHistogram mean() method can throw this (if it overflows). While such overflow // shouldn't happen, it's not worth taking the risk of letting the exception bubble up. } } return new ReusablePartitionData(metadata(), partitionKey(), columns(), estimatedRowCount); }
int dataBufferSize = optimizationStrategy.bufferSize(stats.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile())); int indexBufferSize = optimizationStrategy.bufferSize(indexFileLength / indexSummary.size()); FileHandle ifile = iwriter.builder.bufferSize(indexBufferSize).complete();
int dataBufferSize = optimizationStrategy.bufferSize(stats.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile())); int indexBufferSize = optimizationStrategy.bufferSize(indexFileLength / indexSummary.size()); FileHandle ifile = iwriter.builder.bufferSize(indexBufferSize).complete();
int dataBufferSize = optimizationStrategy.bufferSize(stats.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile())); int indexBufferSize = optimizationStrategy.bufferSize(indexFileLength / indexSummary.size()); FileHandle ifile = iwriter.builder.bufferSize(indexBufferSize).complete();
@SuppressWarnings("resource") public SSTableReader openEarly() { // find the max (exclusive) readable key IndexSummaryBuilder.ReadableBoundary boundary = iwriter.getMaxReadable(); if (boundary == null) return null; StatsMetadata stats = statsMetadata(); assert boundary.indexLength > 0 && boundary.dataLength > 0; // open the reader early IndexSummary indexSummary = iwriter.summary.build(metadata.partitioner, boundary); long indexFileLength = new File(descriptor.filenameFor(Component.PRIMARY_INDEX)).length(); int indexBufferSize = optimizationStrategy.bufferSize(indexFileLength / indexSummary.size()); FileHandle ifile = iwriter.builder.bufferSize(indexBufferSize).complete(boundary.indexLength); if (compression) dbuilder.withCompressionMetadata(((CompressedSequentialWriter) dataFile).open(boundary.dataLength)); int dataBufferSize = optimizationStrategy.bufferSize(stats.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile())); FileHandle dfile = dbuilder.bufferSize(dataBufferSize).complete(boundary.dataLength); invalidateCacheAtBoundary(dfile); SSTableReader sstable = SSTableReader.internalOpen(descriptor, components, metadata, ifile, dfile, indexSummary, iwriter.bf.sharedCopy(), maxDataAge, stats, SSTableReader.OpenReason.EARLY, header); // now it's open, find the ACTUAL last readable key (i.e. for which the data file has also been flushed) sstable.first = getMinimalKey(first); sstable.last = getMinimalKey(boundary.lastKey); return sstable; }
@SuppressWarnings("resource") public SSTableReader openEarly() { // find the max (exclusive) readable key IndexSummaryBuilder.ReadableBoundary boundary = iwriter.getMaxReadable(); if (boundary == null) return null; StatsMetadata stats = statsMetadata(); assert boundary.indexLength > 0 && boundary.dataLength > 0; // open the reader early IndexSummary indexSummary = iwriter.summary.build(metadata.partitioner, boundary); long indexFileLength = new File(descriptor.filenameFor(Component.PRIMARY_INDEX)).length(); int indexBufferSize = optimizationStrategy.bufferSize(indexFileLength / indexSummary.size()); FileHandle ifile = iwriter.builder.bufferSize(indexBufferSize).complete(boundary.indexLength); if (compression) dbuilder.withCompressionMetadata(((CompressedSequentialWriter) dataFile).open(boundary.dataLength)); int dataBufferSize = optimizationStrategy.bufferSize(stats.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile())); FileHandle dfile = dbuilder.bufferSize(dataBufferSize).complete(boundary.dataLength); invalidateCacheAtBoundary(dfile); SSTableReader sstable = SSTableReader.internalOpen(descriptor, components, metadata, ifile, dfile, indexSummary, iwriter.bf.sharedCopy(), maxDataAge, stats, SSTableReader.OpenReason.EARLY, header); // now it's open, find the ACTUAL last readable key (i.e. for which the data file has also been flushed) sstable.first = getMinimalKey(first); sstable.last = getMinimalKey(boundary.lastKey); return sstable; }
@SuppressWarnings("resource") public SSTableReader openEarly() { // find the max (exclusive) readable key IndexSummaryBuilder.ReadableBoundary boundary = iwriter.getMaxReadable(); if (boundary == null) return null; StatsMetadata stats = statsMetadata(); assert boundary.indexLength > 0 && boundary.dataLength > 0; // open the reader early IndexSummary indexSummary = iwriter.summary.build(metadata.partitioner, boundary); long indexFileLength = new File(descriptor.filenameFor(Component.PRIMARY_INDEX)).length(); int indexBufferSize = optimizationStrategy.bufferSize(indexFileLength / indexSummary.size()); FileHandle ifile = iwriter.builder.bufferSize(indexBufferSize).complete(boundary.indexLength); if (compression) dbuilder.withCompressionMetadata(((CompressedSequentialWriter) dataFile).open(boundary.dataLength)); int dataBufferSize = optimizationStrategy.bufferSize(stats.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile())); FileHandle dfile = dbuilder.bufferSize(dataBufferSize).complete(boundary.dataLength); invalidateCacheAtBoundary(dfile); SSTableReader sstable = SSTableReader.internalOpen(descriptor, components, metadata, ifile, dfile, indexSummary, iwriter.bf.sharedCopy(), maxDataAge, stats, SSTableReader.OpenReason.EARLY, header); // now it's open, find the ACTUAL last readable key (i.e. for which the data file has also been flushed) sstable.first = getMinimalKey(first); sstable.last = getMinimalKey(boundary.lastKey); return sstable; }
int dataBufferSize = optimizationStrategy.bufferSize(sstableMetadata.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile()));
buildSummary(recreateBloomFilter, summaryLoaded, Downsampling.BASE_SAMPLING_LEVEL); int dataBufferSize = optimizationStrategy.bufferSize(sstableMetadata.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile()));
buildSummary(recreateBloomFilter, summaryLoaded, Downsampling.BASE_SAMPLING_LEVEL); int dataBufferSize = optimizationStrategy.bufferSize(sstableMetadata.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile()));
int dataBufferSize = optimizationStrategy.bufferSize(sstableMetadata.estimatedPartitionSize.percentile(DatabaseDescriptor.getDiskOptimizationEstimatePercentile()));