/** Encode the Metadata. The caller must ensure that all segments have encoded keys first. * @throws MissingKeyException This indicates disk corruption or a bug (e.g. not all segments * had encoded keys). Since we don't checksum the blocks, there isn't much point in trying to * recover from losing a key; but at least we can detect that there was a problem. * * (Package-visible for unit tests) */ Metadata encodeMetadata() throws IOException, MissingKeyException { ClientCHK[] dataKeys = new ClientCHK[totalDataBlocks + crossCheckBlocks * segments.length]; ClientCHK[] checkKeys = new ClientCHK[totalCheckBlocks]; int dataPtr = 0; int checkPtr = 0; for(int segNo = 0; segNo < segments.length; segNo++) { SplitFileInserterSegmentStorage segment = segments[segNo]; for(int i=0;i<segment.dataBlockCount+segment.crossCheckBlockCount;i++) { dataKeys[dataPtr++] = segment.readKey(i); } for(int i=0;i<segment.checkBlockCount;i++) { checkKeys[checkPtr++] = segment.readKey(i+segment.dataBlockCount+segment.crossCheckBlockCount); } } assert(dataPtr == dataKeys.length); assert(checkPtr == checkKeys.length); return new Metadata(splitfileType, dataKeys, checkKeys, segmentSize, checkSegmentSize, deductBlocksFromSegments, clientMetadata, dataLength, archiveType, compressionCodec, decompressedLength, isMetadata, hashes, hashThisLayerOnly, origDataSize, origCompressedDataSize, topRequiredBlocks, topTotalBlocks, topDontCompress, cmode, splitfileCryptoAlgorithm, splitfileCryptoKey, specifySplitfileKeyInMetadata, crossCheckBlocks); }
void setKey(int blockNumber, ClientCHK key) throws IOException { if(logMINOR) Logger.minor(this, "Setting key "+key+" for block "+blockNumber+" on "+this, new Exception("debug")); try { ClientCHK oldKey = readKey(blockNumber); if(!oldKey.equals(key)) throw new IOException("Key for block has changed! Data corruption or bugs in SplitFileInserter code"); } catch (MissingKeyException e) { // Ok. writeKey(blockNumber, key); } // Must be called either way as we don't regenerate blocksHaveKeys on startup. setHasKey(blockNumber); }
/** Called on startup to check which keys we actually have. Does nothing unless the segment * claims to have been encoded already. FIXME consider calling this later on for robustness, * but we would then need to re-encode ... */ public void checkKeys() { synchronized(this) { if(!encoded) return; } try { for(int i=0;i<totalBlockCount;i++) { readKey(i); } } catch (IOException e) { parent.failOnDiskError(e); return; } catch (MissingKeyException e) { // Easy to recover so may as well... Logger.error(this, "Missing key even though segment encoded. Recovering by re-encoding..."); synchronized(this) { encoded = false; } return; } }
blockChooser.consecutiveRNFsCountAsSuccess > 0) { try { readKey(blockNo); blockChooser.onRNF(blockNo); parent.clearCooldown();
public void testSmallSplitfileHasKeys() throws IOException, InsertException, MissingKeyException { Random r = new Random(12121); long size = 65536; // Exact multiple, so no last block LockableRandomAccessBuffer data = generateData(r, size); HashResult[] hashes = getHashes(data); MyCallback cb = new MyCallback(); InsertContext context = baseContext.clone(); context.earlyEncode = true; KeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, false, context, cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0); storage.start(); cb.waitForFinishedEncode(); assertEquals(storage.segments.length, 1); assertEquals(storage.segments[0].dataBlockCount, 2); assertEquals(storage.segments[0].checkBlockCount, 3); assertEquals(storage.segments[0].crossCheckBlockCount, 0); cb.waitForHasKeys(); for(int i=0;i<storage.segments[0].dataBlockCount+storage.segments[0].checkBlockCount+storage.segments[0].crossCheckBlockCount;i++) storage.segments[0].readKey(i); storage.encodeMetadata(); assertTrue(storage.getStatus() == Status.ENCODED); }