public IntColumn append(int i) { data.add(i); return this; }
public TimeColumn appendInternal(int f) { data.add(f); return this; }
public DateColumn appendInternal(int f) { data.add(f); return this; }
private IntArrayList generatePositionList(int numRows, int numPositions) { IntArrayList positions = new IntArrayList(numPositions); for (int i = 0; i < numPositions; i++) { positions.add((7 * i + 3) % numRows); } Collections.sort(positions); return positions; } }
private void initFloatValues() { if (predicate != null) { return; } synchronized (initLock) { if (predicate != null) { return; } IntArrayList floatBits = new IntArrayList(values.size()); for (String value : values) { Float floatValue = Floats.tryParse(value); if (floatValue != null) { floatBits.add(Float.floatToIntBits(floatValue)); } } if (floatBits.size() > NUMERIC_HASHING_THRESHOLD) { final IntOpenHashSet floatBitsHashSet = new IntOpenHashSet(floatBits); predicate = input -> floatBitsHashSet.contains(Float.floatToIntBits(input)); } else { final int[] floatBitsArray = floatBits.toIntArray(); Arrays.sort(floatBitsArray); predicate = input -> Arrays.binarySearch(floatBitsArray, Float.floatToIntBits(input)) >= 0; } } }
public IntArrayList computeEligibleRowIds(Page page) { IntArrayList ids = new IntArrayList(page.getPositionCount()); Page bucketColumnsPage = extractColumns(page, bucketColumns); for (int position = 0; position < page.getPositionCount(); position++) { int bucket = getHiveBucket(tableBucketCount, typeInfoList, bucketColumnsPage, position); if ((bucket - bucketToKeep) % partitionBucketCount != 0) { throw new PrestoException(HIVE_INVALID_BUCKET_FILES, format( "A row that is supposed to be in bucket %s is encountered. Only rows in bucket %s (modulo %s) are expected", bucket, bucketToKeep % partitionBucketCount, partitionBucketCount)); } if (bucket == bucketToKeep) { ids.add(position); } } return ids; } }
private synchronized IntArrayList partitionPage(Page page, IntPredicate spillPartitionMask) { IntArrayList unspilledPositions = new IntArrayList(); for (int position = 0; position < page.getPositionCount(); position++) { int partition = partitionFunction.getPartition(page, position); if (!spillPartitionMask.test(partition)) { unspilledPositions.add(position); continue; } spilledPartitions.add(partition); PageBuilder pageBuilder = pageBuilders[partition]; pageBuilder.declarePosition(); for (int channel = 0; channel < types.size(); channel++) { Type type = types.get(channel); type.appendTo(page.getBlock(channel), position, pageBuilder.getBlockBuilder(channel)); } } return unspilledPositions; }
public void generateDataFrameIndex() { for (int i = 0; i < df.getData().get(index).size(); i++) { if (!dataTable.containsKey((int) df.get(i, index))) { dataTable.put((int) df.get(i, index), new IntArrayList()); } dataTable.get((int) df.get(i, index)).add(index); } }
@Override public void append(String value) throws NoKeysAvailableException { int key; if (value == null || StringColumnType.missingValueIndicator().equals(value)) { key = MISSING_VALUE; put(key, StringColumnType.missingValueIndicator()); } else { key = getKeyForValue(value); } if (key == DEFAULT_RETURN_VALUE) { key = getValueId(); put(key, value); } values.add(key); }
links = new IntArrayList(); links.add(to); checkState(positionLinks.putIfAbsent(from, links) == null, "sorted links is corrupted"); return from;
@Override public IntColumn top(int n) { final IntArrayList top = new IntArrayList(); final int[] values = data.toIntArray(); IntArrays.parallelQuickSort(values, descendingComparator); for (int i = 0; i < n && i < values.length; i++) { top.add(values[i]); } return new IntColumn(name() + "[Top " + n + "]", top); }
@Override public IntColumn bottom(final int n) { final IntArrayList bottom = new IntArrayList(); final int[] values = data.toIntArray(); IntArrays.parallelQuickSort(values); for (int i = 0; i < n && i < values.length; i++) { bottom.add(values[i]); } return new IntColumn(name() + "[Bottoms " + n + "]", bottom); }
@Override public synchronized void accept(Page page) { // reset the assignment lists for (IntList partitionAssignment : partitionAssignments) { partitionAssignment.clear(); } // assign each row to a partition for (int position = 0; position < page.getPositionCount(); position++) { int partition = partitionGenerator.getPartition(page, position); partitionAssignments[partition].add(position); } // build a page for each partition Block[] outputBlocks = new Block[page.getChannelCount()]; for (int partition = 0; partition < buffers.size(); partition++) { IntArrayList positions = partitionAssignments[partition]; if (!positions.isEmpty()) { for (int i = 0; i < page.getChannelCount(); i++) { outputBlocks[i] = page.getBlock(i).copyPositions(positions.elements(), 0, positions.size()); } Page pageSplit = new Page(positions.size(), outputBlocks); memoryManager.updateMemoryUsage(pageSplit.getRetainedSizeInBytes()); buffers.get(partition).accept(new PageReference(pageSplit, 1, () -> memoryManager.updateMemoryUsage(-pageSplit.getRetainedSizeInBytes()))); } } }
private void constructColumnIndices(RowSequentialAccessSparseMatrix rowMatrix) { IntArrayList[] columnIndicesList = new IntArrayList[rowMatrix.columnSize()]; for (int columnIndex = 0; columnIndex < rowMatrix.columnSize(); columnIndex++) { columnIndicesList[columnIndex] = new IntArrayList(); } for (MatrixEntry matrixEntry : rowMatrix) { int rowIndex = matrixEntry.row(); int columnIndex = matrixEntry.column(); columnIndicesList[columnIndex].add(rowIndex); } columnIndices = new int[rowMatrix.columnSize()][]; for (int columnIndex = 0; columnIndex < rowMatrix.columnSize(); columnIndex++) { int size = columnIndicesList[columnIndex].size(); columnIndices[columnIndex] = new int[size]; Collections.sort(columnIndicesList[columnIndex]); for (int position = 0; position < size; position++) { columnIndices[columnIndex][position] = columnIndicesList[columnIndex].getInt(position); } } }
private static Block[] createKeyValueBlock(int positionCount, Block keys, Block values, int[] lengths) { if (!hasNull(keys)) { return new Block[] {keys, values}; } // // Map entries with a null key are skipped in the Hive ORC reader, so skip them here also // IntArrayList nonNullPositions = new IntArrayList(keys.getPositionCount()); int position = 0; for (int mapIndex = 0; mapIndex < positionCount; mapIndex++) { int length = lengths[mapIndex]; for (int entryIndex = 0; entryIndex < length; entryIndex++) { if (keys.isNull(position)) { // key is null, so remove this entry from the map lengths[mapIndex]--; } else { nonNullPositions.add(position); } position++; } } Block newKeys = keys.copyPositions(nonNullPositions.elements(), 0, nonNullPositions.size()); Block newValues = values.copyPositions(nonNullPositions.elements(), 0, nonNullPositions.size()); return new Block[] {newKeys, newValues}; }
public RowSequentialAccessSparseMatrix(int rows, int columns, DataFrame df, DataFrameIndex rowDataFrameIndex, int itemColumn, int valueColumn) { super(rows, columns); rowVectors = new VectorBasedSequentialSparseVector[rows]; for (int rowIndex = 0; rowIndex < rows; rowIndex++) { IntArrayList positions = rowDataFrameIndex.getIndices(rowIndex); IntArrayList indices = new IntArrayList(positions.size()); DoubleArrayList doubleValues = new DoubleArrayList(positions.size()); for (int position : positions) { indices.add((int) df.get(itemColumn,position)); doubleValues.add((double) df.get(valueColumn,position)); } rowVectors[rowIndex] = new VectorBasedSequentialSparseVector( columns, indices, doubleValues); } }
probeIndexBuilder.add(position); estimatedProbeBlockBytes += Integer.BYTES;