private final static void checkOrder(DiskRangeList prev, DiskRangeList next, DiskRangeList ref) { if (prev.getEnd() <= next.getOffset()) return; assertInvalidOrder(ref.prev == null ? ref : ref.prev, prev, next); }
public static DiskRangeList findUpperBound(DiskRangeList ranges, long offset) { while (ranges.getEnd() <= offset) { if (ranges.next.getOffset() < ranges.getEnd()) { throwRangesError(ranges, offset, offset); } ranges = ranges.next; } return ranges; }
public static DiskRangeList findUpperBound(DiskRangeList ranges, long offset) { while (ranges.getEnd() <= offset) { if (ranges.next.getOffset() < ranges.getEnd()) { throwRangesError(ranges, offset, offset); } ranges = ranges.next; } return ranges; }
public DiskRangeList getIncompleteCbs( DiskRangeList ranges, long baseOffset, BooleanRef gotAllData) { DiskRangeList prev = ranges.prev; if (prev == null) { prev = new MutateHelper(ranges); } DiskRangeList current = ranges; gotAllData.value = true; // Assume by default that we would find everything. while (current != null) { // We assume ranges in "ranges" are non-overlapping; thus, we will save next in advance. DiskRangeList check = current; current = current.next; if (check.hasData()) continue; Integer badLength = cache.get(Long.valueOf(check.getOffset() + baseOffset)); if (badLength == null || badLength < check.getLength()) { gotAllData.value = false; continue; } // We could just remove here and handle the missing tail during read, but that can be // dangerous; let's explicitly add an incomplete CB. check.replaceSelfWith(new IncompleteCb(check.getOffset(), check.getEnd())); } return prev.next; }
/** Finds compressed offset in a stream and makes sure iter points to its position. This may be necessary for obscure combinations of compression and encoding boundaries. */ private static DiskRangeList findExactPosition(DiskRangeList ranges, long offset) { if (offset < 0) return ranges; ranges = findUpperBound(ranges, offset); ranges = findLowerBound(ranges, offset); if (offset < ranges.getOffset() || offset >= ranges.getEnd()) { throwRangesError(ranges, offset, offset); } return ranges; }
List<ByteBuffer> toReleaseCopies, List<ProcCacheChunk> toDecompress, List<IncompleteCb> badEstimates) throws IOException { if (cOffset > current.getOffset()) { next = current.next; if (next != null && (endCOffset >= 0 && currentOffset < endCOffset) && next.getOffset() >= endCOffset) { throw new IOException("Expected data at " + currentOffset + " (reading until " + endCOffset + "), but the next buffer starts at " + next.getOffset()); lastUncompressed = (newCached == null) ? lastUncompressed : newCached; next = (newCached != null) ? newCached.next : null; currentOffset = (next != null) ? next.getOffset() : -1;
/** Finds compressed offset in a stream and makes sure iter points to its position. This may be necessary for obscure combinations of compression and encoding boundaries. */ private static DiskRangeList findExactPosition(DiskRangeList ranges, long offset) { if (offset < 0) return ranges; ranges = findUpperBound(ranges, offset); ranges = findLowerBound(ranges, offset); if (offset < ranges.getOffset() || offset >= ranges.getEnd()) { throwRangesError(ranges, offset, offset); } return ranges; }
List<ByteBuffer> toReleaseCopies, List<ProcCacheChunk> toDecompress, List<IncompleteCb> badEstimates) throws IOException { if (cOffset > current.getOffset()) { next = current.next; if (next != null && (endCOffset >= 0 && currentOffset < endCOffset) && next.getOffset() >= endCOffset) { throw new IOException("Expected data at " + currentOffset + " (reading until " + endCOffset + "), but the next buffer starts at " + next.getOffset()); lastUncompressed = (newCached == null) ? lastUncompressed : newCached; next = (newCached != null) ? newCached.next : null; currentOffset = (next != null) ? next.getOffset() : -1;
private void getOverlappingRanges(long baseOffset, DiskRangeList currentNotCached, ConcurrentSkipListMap<Long, LlapDataBuffer> cache, DiskRangeListFactory factory, BooleanRef gotAllData) { long absOffset = currentNotCached.getOffset() + baseOffset; if (!doAssumeGranularBlocks) {
private static DiskRangeList findIntersectingPosition( DiskRangeList ranges, long offset, long end) { if (offset < 0) return ranges; ranges = findUpperBound(ranges, offset); ranges = findLowerBound(ranges, end); // We are now on some intersecting buffer, find the first intersecting buffer. while (ranges.prev != null && ranges.prev.getEnd() > offset) { if (ranges.prev.getEnd() > ranges.getOffset()) { throwRangesError(ranges, offset, end); } ranges = ranges.prev; } return ranges; }
trace.logStartRead(current); if (streamOffset > current.getOffset()) { break; // We have no data from this point on (could be unneeded), skip. assert partOffset <= current.getOffset(); if (partOffset == current.getOffset() && current instanceof CacheChunk) { assert current.getOffset() == partOffset && current.getEnd() == partEnd; lastUncompressed = (CacheChunk)current; current = current.next; continue; if (current.getOffset() >= partEnd) { continue; // We have no data at all for this part of the stream (could be unneeded), skip. boolean noMoreDataForPart = (next == null || next.getOffset() >= partEnd); if (noMoreDataForPart && hasEntirePartTo < partEnd && candidateCached != null) { + current.getOffset() + ", " + current.getEnd() + ")"); trace.logUncompressedData(current.getOffset(), current.getEnd()); BufferChunk curBc = (BufferChunk)current; hasEntirePartTo = (hasEntirePartTo == current.getOffset()) ? current.getEnd() : -1; if (hasEntirePartTo == -1) {
private static DiskRangeList findIntersectingPosition( DiskRangeList ranges, long offset, long end) { if (offset < 0) return ranges; ranges = findUpperBound(ranges, offset); ranges = findLowerBound(ranges, end); // We are now on some intersecting buffer, find the first intersecting buffer. while (ranges.prev != null && ranges.prev.getEnd() > offset) { if (ranges.prev.getEnd() > ranges.getOffset()) { throwRangesError(ranges, offset, end); } ranges = ranges.prev; } return ranges; }
if (streamOffset > current.getOffset()) { break; // We have no data from this point on (could be unneeded), skip. assert partOffset <= current.getOffset(); if (partOffset == current.getOffset() && current instanceof CacheChunk) { assert current.getOffset() == partOffset && current.getEnd() == partEnd; lastUncompressed = (CacheChunk)current; current = current.next; continue; if (current.getOffset() >= partEnd) { continue; // We have no data at all for this part of the stream (could be unneeded), skip. boolean noMoreDataForPart = (next == null || next.getOffset() >= partEnd); if (noMoreDataForPart && hasEntirePartTo < partEnd && candidateCached != null) { + current.getOffset() + ", " + current.getEnd() + ")"); hasEntirePartTo = (hasEntirePartTo == current.getOffset()) ? current.getEnd() : -1; if (hasEntirePartTo == -1) {
while (drl != null) { assert drl.hasData(); long from = drl.getOffset(), to = drl.getEnd(); int offsetFromReadStart = (int)(from - readStartPos), candidateSize = (int)(to - from); ByteBuffer data = drl.getData().duplicate(); DiskRangeList candidate = current; current = current.next; long from = candidate.getOffset(), to = candidate.getEnd();
public void verifyResult(DiskRangeList result, long... vals) { for (int i = 0; i < vals.length; i += 3) { switch ((int)vals[i]) { case INCOMPLETE: assertTrue(result instanceof IncompleteCb); break; case DRL: assertFalse(result instanceof IncompleteCb); break; default: fail(); } assertEquals(vals[i + 1], result.getOffset()); assertEquals(vals[i + 2], result.getEnd()); result = result.next; } assertNull(result); }
private final static void checkOrder(DiskRangeList prev, DiskRangeList next, DiskRangeList ref) { if (prev.getEnd() <= next.getOffset()) return; assertInvalidOrder(ref.prev == null ? ref : ref.prev, prev, next); }
public DiskRangeList getIncompleteCbs( DiskRangeList ranges, long baseOffset, BooleanRef gotAllData) { DiskRangeList prev = ranges.prev; if (prev == null) { prev = new MutateHelper(ranges); } DiskRangeList current = ranges; gotAllData.value = true; // Assume by default that we would find everything. while (current != null) { // We assume ranges in "ranges" are non-overlapping; thus, we will save next in advance. DiskRangeList check = current; current = current.next; if (check.hasData()) continue; Integer badLength = cache.get(Long.valueOf(check.getOffset() + baseOffset)); if (badLength == null || badLength < check.getLength()) { gotAllData.value = false; continue; } // We could just remove here and handle the missing tail during read, but that can be // dangerous; let's explicitly add an incomplete CB. check.replaceSelfWith(new IncompleteCb(check.getOffset(), check.getEnd())); } return prev.next; }