public void prepareToCommit() { // this method should only be called when we've finished appending records, so we truncate the // memory we're using to the exact amount required to represent it before building our summary entries.setCapacity(entries.length()); offsets.setCapacity(offsets.length()); }
/** * * @param decoratedKey the key for this record * @param indexStart the position in the index file this record begins * @param indexEnd the position in the index file we need to be able to read to (exclusive) to read this record * @param dataEnd the position in the data file we need to be able to read to (exclusive) to read this record * a value of 0 indicates we are not tracking readable boundaries */ public IndexSummaryBuilder maybeAddEntry(DecoratedKey decoratedKey, long indexStart, long indexEnd, long dataEnd) { if (keysWritten == nextSamplePosition) { assert entries.length() <= Integer.MAX_VALUE; offsets.writeInt((int) entries.length()); entries.write(decoratedKey.getKey()); entries.writeLong(indexStart); setNextSamplePosition(keysWritten); } else if (dataEnd != 0 && keysWritten + 1 == nextSamplePosition) { // this is the last key in this summary interval, so stash it ReadableBoundary boundary = new ReadableBoundary(decoratedKey, indexEnd, dataEnd, (int)(offsets.length() / 4), entries.length()); lastReadableByData.put(dataEnd, boundary); lastReadableByIndex.put(indexEnd, boundary); } keysWritten++; return this; }
@Override protected void reallocate(long count) { long newCapacity = calculateNewSize(count); if (newCapacity != capacity()) { long position = length(); ByteOrder order = buffer.order(); SafeMemory oldBuffer = memory; memory = this.memory.copy(newCapacity); buffer = tailBuffer(memory); int newPosition = (int) (position - tailOffset(memory)); buffer.position(newPosition); buffer.order(order); oldBuffer.free(); } }
public IndexSummary build(IPartitioner partitioner, ReadableBoundary boundary) { assert entries.length() > 0; int count = (int) (offsets.length() / 4); long entriesLength = entries.length(); if (boundary != null) { count = boundary.summaryCount; entriesLength = boundary.entriesLength; } int sizeAtFullSampling = (int) Math.ceil(keysWritten / (double) minIndexInterval); assert count > 0; return new IndexSummary(partitioner, offsets.currentBuffer().sharedCopy(), count, entries.currentBuffer().sharedCopy(), entriesLength, sizeAtFullSampling, minIndexInterval, samplingLevel); }
public IndexSummaryBuilder(long expectedKeys, int minIndexInterval, int samplingLevel) { this.samplingLevel = samplingLevel; this.startPoints = Downsampling.getStartPoints(BASE_SAMPLING_LEVEL, samplingLevel); long maxExpectedEntries = expectedKeys / minIndexInterval; if (maxExpectedEntries > Integer.MAX_VALUE) { // that's a _lot_ of keys, and a very low min index interval int effectiveMinInterval = (int) Math.ceil((double) Integer.MAX_VALUE / expectedKeys); maxExpectedEntries = expectedKeys / effectiveMinInterval; assert maxExpectedEntries <= Integer.MAX_VALUE : maxExpectedEntries; logger.warn("min_index_interval of {} is too low for {} expected keys; using interval of {} instead", minIndexInterval, expectedKeys, effectiveMinInterval); this.minIndexInterval = effectiveMinInterval; } else { this.minIndexInterval = minIndexInterval; } // for initializing data structures, adjust our estimates based on the sampling level maxExpectedEntries = Math.max(1, (maxExpectedEntries * samplingLevel) / BASE_SAMPLING_LEVEL); offsets = new SafeMemoryWriter(4 * maxExpectedEntries).order(ByteOrder.nativeOrder()); entries = new SafeMemoryWriter(40 * maxExpectedEntries).order(ByteOrder.nativeOrder()); // the summary will always contain the first index entry (downsampling will never remove it) nextSamplePosition = 0; indexIntervalMatches++; }
public IndexSummaryBuilder(long expectedKeys, int minIndexInterval, int samplingLevel) { this.samplingLevel = samplingLevel; this.startPoints = Downsampling.getStartPoints(BASE_SAMPLING_LEVEL, samplingLevel); long maxExpectedEntries = expectedKeys / minIndexInterval; if (maxExpectedEntries > Integer.MAX_VALUE) { // that's a _lot_ of keys, and a very low min index interval int effectiveMinInterval = (int) Math.ceil((double) Integer.MAX_VALUE / expectedKeys); maxExpectedEntries = expectedKeys / effectiveMinInterval; assert maxExpectedEntries <= Integer.MAX_VALUE : maxExpectedEntries; logger.warn("min_index_interval of {} is too low for {} expected keys; using interval of {} instead", minIndexInterval, expectedKeys, effectiveMinInterval); this.minIndexInterval = effectiveMinInterval; } else { this.minIndexInterval = minIndexInterval; } // for initializing data structures, adjust our estimates based on the sampling level maxExpectedEntries = Math.max(1, (maxExpectedEntries * samplingLevel) / BASE_SAMPLING_LEVEL); offsets = new SafeMemoryWriter(4 * maxExpectedEntries).withByteOrder(ByteOrder.nativeOrder()); entries = new SafeMemoryWriter(40 * maxExpectedEntries).withByteOrder(ByteOrder.nativeOrder()); // the summary will always contain the first index entry (downsampling will never remove it) nextSamplePosition = 0; indexIntervalMatches++; }
public void setCapacity(long newCapacity) { if (newCapacity != capacity()) { SafeMemory oldBuffer = buffer; buffer = this.buffer.copy(newCapacity); oldBuffer.free(); } }
private long ensureCapacity(long size) { long newLength = this.length + size; if (newLength > buffer.size()) setCapacity(Math.max(newLength, buffer.size() + (buffer.size() / 2))); return newLength; }
/** * * @param decoratedKey the key for this record * @param indexStart the position in the index file this record begins * @param indexEnd the position in the index file we need to be able to read to (exclusive) to read this record * @param dataEnd the position in the data file we need to be able to read to (exclusive) to read this record * a value of 0 indicates we are not tracking readable boundaries */ public IndexSummaryBuilder maybeAddEntry(DecoratedKey decoratedKey, long indexStart, long indexEnd, long dataEnd) throws IOException { if (keysWritten == nextSamplePosition) { assert entries.length() <= Integer.MAX_VALUE; offsets.writeInt((int) entries.length()); entries.write(decoratedKey.getKey()); entries.writeLong(indexStart); setNextSamplePosition(keysWritten); } else if (dataEnd != 0 && keysWritten + 1 == nextSamplePosition) { // this is the last key in this summary interval, so stash it ReadableBoundary boundary = new ReadableBoundary(decoratedKey, indexEnd, dataEnd, (int)(offsets.length() / 4), entries.length()); lastReadableByData.put(dataEnd, boundary); lastReadableByIndex.put(indexEnd, boundary); } keysWritten++; return this; }
@Override protected void reallocate(long count) { long newCapacity = calculateNewSize(count); if (newCapacity != capacity()) { long position = length(); ByteOrder order = buffer.order(); SafeMemory oldBuffer = memory; memory = this.memory.copy(newCapacity); buffer = tailBuffer(memory); int newPosition = (int) (position - tailOffset(memory)); buffer.position(newPosition); buffer.order(order); oldBuffer.free(); } }
public IndexSummary build(IPartitioner partitioner, ReadableBoundary boundary) { assert entries.length() > 0; int count = (int) (offsets.length() / 4); long entriesLength = entries.length(); if (boundary != null) { count = boundary.summaryCount; entriesLength = boundary.entriesLength; } int sizeAtFullSampling = (int) Math.ceil(keysWritten / (double) minIndexInterval); assert count > 0; return new IndexSummary(partitioner, offsets.currentBuffer().sharedCopy(), count, entries.currentBuffer().sharedCopy(), entriesLength, sizeAtFullSampling, minIndexInterval, samplingLevel); }
offsets = new SafeMemoryWriter(4 * maxExpectedEntries).order(ByteOrder.nativeOrder()); entries = new SafeMemoryWriter(expectedEntrySize * maxExpectedEntries).order(ByteOrder.nativeOrder());
if ((entries.length() + getEntrySize(decoratedKey)) <= Integer.MAX_VALUE) offsets.writeInt((int) entries.length()); entries.write(decoratedKey.getKey()); entries.writeLong(indexStart); setNextSamplePosition(keysWritten); ReadableBoundary boundary = new ReadableBoundary(decoratedKey, indexEnd, dataEnd, (int) (offsets.length() / 4), entries.length()); lastReadableByData.put(dataEnd, boundary); lastReadableByIndex.put(indexEnd, boundary);
@Override protected void reallocate(long count) { long newCapacity = calculateNewSize(count); if (newCapacity != capacity()) { long position = length(); ByteOrder order = buffer.order(); SafeMemory oldBuffer = memory; memory = this.memory.copy(newCapacity); buffer = tailBuffer(memory); int newPosition = (int) (position - tailOffset(memory)); buffer.position(newPosition); buffer.order(order); oldBuffer.free(); } }
public void prepareToCommit() { // this method should only be called when we've finished appending records, so we truncate the // memory we're using to the exact amount required to represent it before building our summary entries.setCapacity(entries.length()); offsets.setCapacity(offsets.length()); }
public IndexSummary build(IPartitioner partitioner, ReadableBoundary boundary) { assert entries.length() > 0; int count = (int) (offsets.length() / 4); long entriesLength = entries.length(); if (boundary != null) { count = boundary.summaryCount; entriesLength = boundary.entriesLength; } int sizeAtFullSampling = (int) Math.ceil(keysWritten / (double) minIndexInterval); assert count > 0; return new IndexSummary(partitioner, offsets.currentBuffer().sharedCopy(), count, entries.currentBuffer().sharedCopy(), entriesLength, sizeAtFullSampling, minIndexInterval, samplingLevel); }
offsets = new SafeMemoryWriter(4 * maxExpectedEntries).order(ByteOrder.nativeOrder()); entries = new SafeMemoryWriter(expectedEntrySize * maxExpectedEntries).order(ByteOrder.nativeOrder());
if ((entries.length() + getEntrySize(decoratedKey)) <= Integer.MAX_VALUE) offsets.writeInt((int) entries.length()); entries.write(decoratedKey.getKey()); entries.writeLong(indexStart); setNextSamplePosition(keysWritten); ReadableBoundary boundary = new ReadableBoundary(decoratedKey, indexEnd, dataEnd, (int) (offsets.length() / 4), entries.length()); lastReadableByData.put(dataEnd, boundary); lastReadableByIndex.put(indexEnd, boundary);