@Override public T next() { if (!hasNext()) { throw new NoSuchElementException(); } return indexed.get(currIndex++); }
@Override public String toString() { return "BlockCompressedColumnarDoubles_Anonymous{" + "currBufferNum=" + currBufferNum + ", sizePer=" + sizePer + ", numChunks=" + singleThreadedDoubleBuffers.size() + ", totalSize=" + totalSize + '}'; } }
private void checkBasicAPIs(String[] strings, Indexed<String> index, boolean allowReverseLookup) { Assert.assertEquals(strings.length, index.size()); for (int i = 0; i < strings.length; i++) { Assert.assertEquals(strings[i], index.get(i)); } if (allowReverseLookup) { HashMap<String, Integer> mixedUp = new HashMap<>(); for (int i = 0; i < strings.length; i++) { mixedUp.put(strings[i], i); } for (Map.Entry<String, Integer> entry : mixedUp.entrySet()) { Assert.assertEquals(entry.getValue().intValue(), index.indexOf(entry.getKey())); } } else { try { index.indexOf("xxx"); Assert.fail("should throw exception"); } catch (UnsupportedOperationException e) { // not supported } } }
int allocationSize = indexed.size() * Integer.BYTES; log.debug("Allocating dictionary merging direct buffer with size[%,d]", allocationSize); mergeBufferTotalSize += allocationSize; directBufferAllocations.add(new Pair<>(conversionDirectBuffer, allocationSize)); } else { conversions[i] = IntBuffer.allocate(indexed.size()); mergeBufferTotalSize += indexed.size(); indexed.iterator(), NullHandling::nullToEmptyIfNeeded
@Test public void testSanity() { final String[] strings = {"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l"}; Indexed<String> indexed = GenericIndexed.fromArray(strings, GenericIndexed.STRING_STRATEGY); checkBasicAPIs(strings, indexed, true); Assert.assertEquals(-13, indexed.indexOf("q")); Assert.assertEquals(-9, indexed.indexOf("howdydo")); Assert.assertEquals(-1, indexed.indexOf("1111")); }
Assert.assertEquals(2, queryable.getAvailableDimensions().size()); Assert.assertEquals("host", queryable.getAvailableDimensions().get(0)); Assert.assertEquals("spatial", queryable.getAvailableDimensions().get(1)); Assert.assertEquals(ImmutableList.of("visited_sum", "unique_hosts"), queryable.getAvailableMetrics());
DictionaryMergeIterator(Indexed<String>[] dimValueLookups, boolean useDirect) { pQueue = new PriorityQueue<>(dimValueLookups.length, NULLS_FIRST_PEEKING_COMPARATOR); conversions = new IntBuffer[dimValueLookups.length]; for (int i = 0; i < conversions.length; i++) { if (dimValueLookups[i] == null) { continue; } Indexed<String> indexed = dimValueLookups[i]; if (useDirect) { int allocationSize = indexed.size() * Integer.BYTES; log.info("Allocating dictionary merging direct buffer with size[%,d]", allocationSize); final ByteBuffer conversionDirectBuffer = ByteBuffer.allocateDirect(allocationSize); conversions[i] = conversionDirectBuffer.asIntBuffer(); directBufferAllocations.add(new Pair<>(conversionDirectBuffer, allocationSize)); } else { conversions[i] = IntBuffer.allocate(indexed.size()); } final PeekingIterator<String> iter = Iterators.peekingIterator( Iterators.transform( indexed.iterator(), NullHandling::nullToEmptyIfNeeded ) ); if (iter.hasNext()) { pQueue.add(Pair.of(i, iter)); } } }
protected void loadBuffer(int bufferNum) { CloseQuietly.close(holder); holder = singleThreadedBuffers.get(bufferNum); ByteBuffer bb = holder.get(); ByteOrder byteOrder = bb.order(); // slice() makes the buffer's position = 0 buffer = bb.slice().order(byteOrder); currBufferNum = bufferNum; bigEndian = byteOrder.equals(ByteOrder.BIG_ENDIAN); }
@Override public String toString() { return "CompressedVSizeColumnarInts{" + "currBufferNum=" + currBufferNum + ", sizePer=" + sizePer + ", numChunks=" + singleThreadedBuffers.size() + ", totalSize=" + totalSize + '}'; }
/** * Checks if the suffix of strings.get(i) matches the suffix of this matcher. The first prefix.length characters * of s are ignored. This method is useful if you've already independently verified the prefix. This method * evalutes strings.get(i) lazily to save time when it isn't necessary to actually look at the string. */ public boolean matchesSuffixOnly(final Indexed<String> strings, final int i) { if (suffixMatch == SuffixMatch.MATCH_ANY) { return true; } else if (suffixMatch == SuffixMatch.MATCH_EMPTY) { final String s = strings.get(i); return s == null ? matches(null) : s.length() == prefix.length(); } else { // suffixMatch is MATCH_PATTERN final String s = strings.get(i); return matches(s); } }
@Override public String toString() { return "CompressedIntsIndexedSupplier_Anonymous{" + "currBufferNum=" + currBufferNum + ", sizePer=" + sizePer + ", numChunks=" + singleThreadedIntBuffers.size() + ", totalSize=" + totalSize + '}'; }
private int indexOf(Indexed<T> indexed, @Nullable T value) { if (!allowReverseLookup) { throw new UnsupportedOperationException("Reverse lookup not allowed."); } int minIndex = 0; int maxIndex = size - 1; while (minIndex <= maxIndex) { int currIndex = (minIndex + maxIndex) >>> 1; T currValue = indexed.get(currIndex); int comparison = strategy.compare(currValue, value); if (comparison == 0) { return currIndex; } if (comparison < 0) { minIndex = currIndex + 1; } else { maxIndex = currIndex - 1; } } return -(minIndex + 1); }
@Override public String toString() { return "BlockCompressedColumnarLongs_Anonymous{" + "currBufferNum=" + currBufferNum + ", sizePer=" + sizePer + ", numChunks=" + singleThreadedLongBuffers.size() + ", totalSize=" + totalSize + '}'; } }
protected void loadBuffer(int bufferNum) { CloseQuietly.close(holder); holder = singleThreadedDoubleBuffers.get(bufferNum); // asDoubleBuffer() makes the doubleBuffer's position = 0 doubleBuffer = holder.get().asDoubleBuffer(); currBufferNum = bufferNum; }
@Override public String toString() { return "BlockCompressedColumnarFloats_Anonymous{" + "currBufferNum=" + currBufferNum + ", sizePer=" + sizePer + ", numChunks=" + singleThreadedFloatBuffers.size() + ", totalSize=" + totalSize + '}'; } }
protected void loadBuffer(int bufferNum) { CloseQuietly.close(holder); holder = singleThreadedFloatBuffers.get(bufferNum); // asFloatBuffer() makes the floatBuffer's position = 0 floatBuffer = holder.get().asFloatBuffer(); currBufferNum = bufferNum; }