int estimateBytes(long curBlockPos) { BlockWriter b = new BlockWriter( INDEX_BLOCK_TYPE, keyType, MAX_BLOCK_SIZE, Math.max(restartInterval, entries.size() / MAX_RESTARTS)); try { for (Entry e : entries) { b.mustAdd(e); } if (cur != null) { b.mustAdd(new IndexEntry(cur.lastKey(), curBlockPos)); } } catch (BlockSizeTooSmallException e) { return b.currentSize(); } return b.currentSize(); }
void mustAdd(Entry entry) throws BlockSizeTooSmallException { if (!tryAdd(entry, true)) { // Insanely long names need a larger block size. throw blockSizeTooSmall(entry); } }
private void finishLogSection() throws IOException { if (cur != null && cur.blockType() == LOG_BLOCK_TYPE) { logs.finishSectionMaybeWriteIndex(); } }
boolean tryAdd(Entry entry) { if (entry instanceof ObjEntry && computeBlockBytes(entry.sizeBytes(), 1) > blockLimitBytes) { // If the ObjEntry has so many ref block pointers that its // encoding overflows any block, reconfigure it to tell readers to // instead scan all refs for this ObjectId. That significantly // shrinks the entry to a very small size, which may now fit into // this block. ((ObjEntry) entry).markScanRequired(); } if (tryAdd(entry, true)) { return true; } else if (nextShouldBeRestart()) { // It was time for another restart, but the entry doesn't fit // with its complete key, as the block is nearly full. Try to // force it to fit with prefix compression rather than waste // the tail of the block with padding. return tryAdd(entry, false); } return false; }
private void writeMultiLevelIndex(List<IndexEntry> keys) throws IOException { levels = 1; while (maxIndexLevels == 0 || levels < maxIndexLevels) { keys = writeOneLevel(keys); if (keys == null) { return; } levels++; } // When maxIndexLevels has restricted the writer, write one // index block with the entire remaining set of keys. BlockWriter b = new BlockWriter( INDEX_BLOCK_TYPE, keyType, MAX_BLOCK_SIZE, Math.max(restartInterval, keys.size() / MAX_RESTARTS)); for (Entry e : keys) { b.mustAdd(e); } rootPosition = out.size(); b.writeTo(out); }
private boolean tryAdd(Entry entry, boolean tryRestart) { byte[] key = entry.key; int prefixLen = 0; boolean restart = tryRestart && nextShouldBeRestart(); if (!restart) { Entry priorEntry = entries.get(entries.size() - 1); byte[] prior = priorEntry.key; prefixLen = commonPrefix(prior, prior.length, key); if (prefixLen <= 5 /* "refs/" */ && keyType == REF_BLOCK_TYPE) { // Force restart points at transitions between namespaces // such as "refs/heads/" to "refs/tags/". restart = true; prefixLen = 0; } else if (prefixLen == 0) { restart = true; } } entry.restart = restart; entry.prefixLen = prefixLen; int entryBytes = entry.sizeBytes(); if (computeBlockBytes(entryBytes, restart) > blockLimitBytes) { return false; } entriesSumBytes += entryBytes; entries.add(entry); if (restart) { restartCnt++; } return true; }
private void beginBlock(BlockWriter.Entry entry) throws BlockSizeTooSmallException { byte blockType = entry.blockType(); int bs = out.bytesAvailableInBlock(); cur = new BlockWriter(blockType, idx.keyType, bs, restartInterval); cur.mustAdd(entry); }
static int compare(Entry ea, Entry eb) { byte[] a = ea.key; byte[] b = eb.key; return BlockWriter.compare(a, 0, a.length, b, 0, b.length); }
int sz = cur.currentSize(); bytes += sz; if (cur.blockType() == REF_BLOCK_TYPE) { idx = refs.idx; } else if (cur.blockType() == LOG_BLOCK_TYPE) { idx = logs.idx;
private int computeBlockBytes(int entryBytes, boolean restart) { return computeBlockBytes( entriesSumBytes + entryBytes, restartCnt + (restart ? 1 : 0)); }
int sizeBytes() { int sfxLen = key.length - prefixLen; int sfx = encodeSuffixAndType(sfxLen, valueType()); return computeVarintSize(prefixLen) + computeVarintSize(sfx) + sfxLen + valueSize(); }
BlockWriter(byte type, byte kt, int bs, int ri) { blockType = type; keyType = kt; blockLimitBytes = bs; restartInterval = ri; entries = new ArrayList<>(estimateEntryCount(type, kt, bs)); }
boolean tryAdd(Entry entry) { if (entry instanceof ObjEntry && computeBlockBytes(entry.sizeBytes(), 1) > blockLimitBytes) { // If the ObjEntry has so many ref block pointers that its // encoding overflows any block, reconfigure it to tell readers to // instead scan all refs for this ObjectId. That significantly // shrinks the entry to a very small size, which may now fit into // this block. ((ObjEntry) entry).markScanRequired(); } if (tryAdd(entry, true)) { return true; } else if (nextShouldBeRestart()) { // It was time for another restart, but the entry doesn't fit // with its complete key, as the block is nearly full. Try to // force it to fit with prefix compression rather than waste // the tail of the block with padding. return tryAdd(entry, false); } return false; }
private void writeMultiLevelIndex(List<IndexEntry> keys) throws IOException { levels = 1; while (maxIndexLevels == 0 || levels < maxIndexLevels) { keys = writeOneLevel(keys); if (keys == null) { return; } levels++; } // When maxIndexLevels has restricted the writer, write one // index block with the entire remaining set of keys. BlockWriter b = new BlockWriter( INDEX_BLOCK_TYPE, keyType, MAX_BLOCK_SIZE, Math.max(restartInterval, keys.size() / MAX_RESTARTS)); for (Entry e : keys) { b.mustAdd(e); } rootPosition = out.size(); b.writeTo(out); }
private boolean tryAdd(Entry entry, boolean tryRestart) { byte[] key = entry.key; int prefixLen = 0; boolean restart = tryRestart && nextShouldBeRestart(); if (!restart) { Entry priorEntry = entries.get(entries.size() - 1); byte[] prior = priorEntry.key; prefixLen = commonPrefix(prior, prior.length, key); if (prefixLen <= 5 /* "refs/" */ && keyType == REF_BLOCK_TYPE) { // Force restart points at transitions between namespaces // such as "refs/heads/" to "refs/tags/". restart = true; prefixLen = 0; } else if (prefixLen == 0) { restart = true; } } entry.restart = restart; entry.prefixLen = prefixLen; int entryBytes = entry.sizeBytes(); if (computeBlockBytes(entryBytes, restart) > blockLimitBytes) { return false; } entriesSumBytes += entryBytes; entries.add(entry); if (restart) { restartCnt++; } return true; }
private void beginBlock(BlockWriter.Entry entry) throws BlockSizeTooSmallException { byte blockType = entry.blockType(); int bs = out.bytesAvailableInBlock(); cur = new BlockWriter(blockType, idx.keyType, bs, restartInterval); cur.mustAdd(entry); }
boolean match(byte[] match, boolean matchIsPrefix) { int len = nameLen; if (blockType == LOG_BLOCK_TYPE) { len -= 9; } if (matchIsPrefix) { return len >= match.length && compare( match, 0, match.length, nameBuf, 0, match.length) == 0; } return compare(match, 0, match.length, nameBuf, 0, len) == 0; }
int sz = cur.currentSize(); bytes += sz; if (cur.blockType() == REF_BLOCK_TYPE) { idx = refs.idx; } else if (cur.blockType() == LOG_BLOCK_TYPE) { idx = logs.idx;