metaBucket.free(); copyBucket.free(); return new TestSplitfile(data, m1, originalDataBlocks, originalCheckBlocks, dataKeys, checkKeys, cryptoKey, cryptoAlgorithm, segmentDataBlockCount, segmentCheckBlockCount, persistent);
public void testChooseKeyCooldown() throws CHKEncodeException, IOException, MetadataUnresolvedException, MetadataParseException, FetchException, InterruptedException { int dataBlocks = 3, checkBlocks = 3; int COOLDOWN_TIME = 200; TestSplitfile test = TestSplitfile.constructSingleSegment(dataBlocks*BLOCK_SIZE, checkBlocks, null, false); StorageCallback cb = test.createStorageCallback(); FetchContext ctx = test.makeFetchContext(); ctx.maxSplitfileBlockRetries = 5; ctx.setCooldownRetries(3); ctx.setCooldownTime(COOLDOWN_TIME, true); SplitFileFetcherStorage storage = test.createStorage(cb, ctx);
@Override public void queueHeal(byte[] data, byte[] cryptoKey, byte cryptoAlgorithm) { assertTrue(Arrays.equals(cryptoKey, splitfile.cryptoKey)); assertEquals(cryptoAlgorithm, splitfile.cryptoAlgorithm); int x = -1; boolean progress = false; while((x = splitfile.findCheckBlock(data, x)) != -1) { synchronized(this) { encodedBlocks[x+splitfile.dataBlocks.length] = true; } progress = true; } if(!progress) { // Data block? while((x = splitfile.findDataBlock(data, x)) != -1) { synchronized(this) { encodedBlocks[x] = true; } progress = true; } } if(!progress) { System.err.println("Queued healing block not in the original block list"); assertTrue(false); } assertTrue(progress); }
long size = 32768*2-1; assertTrue(dataBlocks * (long)BLOCK_SIZE >= size); TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, null, true); StorageCallback cb = test.createStorageCallback(); SplitFileFetcherStorage storage = test.createStorage(cb); storage = test.createStorage(cb, test.makeFetchContext(), cb.getRAF()); SplitFileFetcherSegmentStorage segment = storage.segments[0]; assertFalse(segment.corruptMetadata()); hits[block] = true; assertFalse(segment.hasStartedDecode()); assertTrue(segment.onGotKey(test.getCHK(block), test.encodeBlock(block))); cb.markDownloadedBlock(block); if(i != test.dataBlocks.length-1) { storage = test.createStorage(cb, test.makeFetchContext(), cb.getRAF()); segment = storage.segments[0]; storage.start(false); cb.waitForFinished(); cb.checkFailed(); test.verifyOutput(storage); cb.checkFailed(); storage.finishedFetcher();
long size = 32768*2-1; assertTrue(dataBlocks * (long)BLOCK_SIZE >= size); TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, null, true); StorageCallback cb = test.createStorageCallback(); SplitFileFetcherStorage storage = test.createStorage(cb); storage = test.createStorage(cb, test.makeFetchContext(), cb.getRAF()); SplitFileFetcherSegmentStorage segment = storage.segments[0]; assertFalse(segment.corruptMetadata()); hits[block] = true; assertFalse(segment.hasStartedDecode()); assertTrue(segment.onGotKey(test.getCHK(block), test.encodeBlock(block))); cb.markDownloadedBlock(block); cb.waitForFinished(); cb.checkFailed(); test.verifyOutput(storage); cb.checkFailed(); storage.finishedFetcher();
private void testRandomMixtureMultiSegment(TestSplitfile test) throws CHKEncodeException, IOException, FetchException, MetadataParseException { StorageCallback cb = test.createStorageCallback(); SplitFileFetcherStorage storage = test.createStorage(cb); int total = test.dataBlocks.length+test.checkBlocks.length; for(SplitFileFetcherSegmentStorage segment : storage.segments) { } while (hits[block]); hits[block] = true; SplitFileFetcherSegmentStorage segment = storage.segments[test.segmentFor(block)]; if(segment.hasStartedDecode()) { i--; continue; assertTrue(segment.onGotKey(test.getCHK(block), test.encodeBlock(block))); cb.markDownloadedBlock(block); cb.waitForFinished(); cb.checkFailed(); test.verifyOutput(storage); cb.checkFailed(); storage.finishedFetcher();
metaBucket.free(); copyBucket.free(); return new TestSplitfile(data, m1, originalDataBlocks, originalCheckBlocks, dataKeys, checkKeys, cryptoKey, cryptoAlgorithm, null, null, persistent);
private void testRandomMixture(TestSplitfile test) throws FetchException, MetadataParseException, IOException, CHKEncodeException { StorageCallback cb = test.createStorageCallback(); SplitFileFetcherStorage storage = test.createStorage(cb); SplitFileFetcherSegmentStorage segment = storage.segments[0]; int total = test.dataBlocks.length+test.checkBlocks.length; hits[block] = true; assertFalse(segment.hasStartedDecode()); assertTrue(segment.onGotKey(test.getCHK(block), test.encodeBlock(block))); cb.markDownloadedBlock(block); cb.waitForFinished(); cb.checkFailed(); test.verifyOutput(storage); cb.checkFailed(); storage.finishedFetcher();
private void testCheckBlocksOnly(TestSplitfile test) throws IOException, CHKEncodeException, FetchException, MetadataParseException { StorageCallback cb = test.createStorageCallback(); SplitFileFetcherStorage storage = test.createStorage(cb); SplitFileFetcherSegmentStorage segment = storage.segments[0]; for(int i=0;i<test.dataBlocks.length;i++) { assertTrue(segment.onGotKey(test.checkKeys[i].getNodeCHK(), test.encodeCheckBlock(i))); cb.markDownloadedBlock(i + test.dataBlocks.length); cb.waitForFinished(); cb.checkFailed(); test.verifyOutput(storage); cb.checkFailed(); storage.finishedFetcher();
long size = 32768*2-1; assertTrue(dataBlocks * (long)BLOCK_SIZE >= size); TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, null, true); StorageCallback cb = test.createStorageCallback(); FetchContext ctx = test.makeFetchContext(); ctx.maxSplitfileBlockRetries = 2; SplitFileFetcherStorage storage = test.createStorage(cb, ctx); storage = test.createStorage(cb, ctx, cb.getRAF()); for(int i=0;i<3;i++) { boolean[] tried = new boolean[dataBlocks+checkBlocks]; storage = test.createStorage(cb, ctx, cb.getRAF()); assertTrue(i != 2); storage.start(false);
private void testDataBlocksOnly(TestSplitfile test) throws IOException, CHKEncodeException, FetchException, MetadataParseException { StorageCallback cb = test.createStorageCallback(); SplitFileFetcherStorage storage = test.createStorage(cb); SplitFileFetcherSegmentStorage segment = storage.segments[0]; for(int i=0;i<test.checkBlocks.length;i++) { segment.onNonFatalFailure(test.dataBlocks.length+i); } for(int i=0;i<test.dataBlocks.length;i++) { assertFalse(segment.hasStartedDecode()); assertTrue(segment.onGotKey(test.dataKeys[i].getNodeCHK(), test.encodeDataBlock(i))); cb.markDownloadedBlock(i); } cb.checkFailed(); assertTrue(segment.hasStartedDecode()); cb.checkFailed(); waitForDecode(segment); cb.checkFailed(); cb.waitForFinished(); cb.checkFailed(); test.verifyOutput(storage); cb.checkFailed(); storage.finishedFetcher(); cb.checkFailed(); waitForFinished(segment); cb.checkFailed(); cb.waitForFree(storage); cb.checkFailed(); }
public void testPersistenceReloadThenChooseKey() throws IOException, StorageFormatException, CHKEncodeException, MetadataUnresolvedException, MetadataParseException, FetchException { int dataBlocks = 2; int checkBlocks = 3; long size = 32768*2-1; assertTrue(dataBlocks * (long)BLOCK_SIZE >= size); TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, null, true); StorageCallback cb = test.createStorageCallback(); FetchContext ctx = test.makeFetchContext(); ctx.maxSplitfileBlockRetries = 2; SplitFileFetcherStorage storage = test.createStorage(cb, ctx); // No need to shutdown the old storage. storage = test.createStorage(cb, ctx, cb.getRAF()); for(int i=0;i<3;i++) { boolean[] tried = new boolean[dataBlocks+checkBlocks]; innerChooseKeyTest(dataBlocks, checkBlocks, storage.segments[0], tried, test, false); } test.fetchingKeys.clear(); assertEquals(storage.chooseRandomKey(), null); cb.waitForFailed(); }
public void testChooseKeyThreeTries() throws CHKEncodeException, IOException, MetadataUnresolvedException, MetadataParseException, FetchException { int dataBlocks = 3, checkBlocks = 3; TestSplitfile test = TestSplitfile.constructSingleSegment(dataBlocks*BLOCK_SIZE, checkBlocks, null, false); StorageCallback cb = test.createStorageCallback(); FetchContext ctx = test.makeFetchContext(); ctx.maxSplitfileBlockRetries = 2; SplitFileFetcherStorage storage = test.createStorage(cb, ctx); for(int i=0;i<3;i++) { boolean[] tried = new boolean[dataBlocks+checkBlocks]; innerChooseKeyTest(dataBlocks, checkBlocks, storage.segments[0], tried, test, false); } assertEquals(storage.chooseRandomKey(), null); cb.waitForFailed(); }
public SplitFileFetcherStorage createStorage(StorageCallback cb) throws FetchException, MetadataParseException, IOException { return createStorage(cb, makeFetchContext()); }
private void testSingleSegment(int dataBlocks, int checkBlocks, long size) throws CHKEncodeException, IOException, FetchException, MetadataParseException, MetadataUnresolvedException { assertTrue(dataBlocks * (long)BLOCK_SIZE >= size); TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, null, false); testDataBlocksOnly(test); if(checkBlocks >= dataBlocks) testCheckBlocksOnly(test); testRandomMixture(test); test.free(); }
public CHKBlock encodeBlock(int block) throws CHKEncodeException { if(block < dataBlocks.length) return encodeDataBlock(block); else return encodeCheckBlock(block - dataBlocks.length); }
/** Test persistence: Create and then reload. Don't do anything. */ public void testPersistenceReload() throws CHKEncodeException, IOException, MetadataUnresolvedException, MetadataParseException, FetchException, StorageFormatException { int dataBlocks = 2; int checkBlocks = 3; long size = 32768*2-1; assertTrue(dataBlocks * (long)BLOCK_SIZE >= size); TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, null, true); StorageCallback cb = test.createStorageCallback(); SplitFileFetcherStorage storage = test.createStorage(cb); // No need to shutdown the old storage. storage = test.createStorage(cb, test.makeFetchContext(), cb.getRAF()); storage.close(); }
private void testMultiSegment(long size, int[] segmentDataBlockCount, int[] segmentCheckBlockCount, int segmentSize, int checkSegmentSize, int deductBlocksFromSegments, CompatibilityMode topCompatibilityMode) throws CHKEncodeException, IOException, MetadataUnresolvedException, MetadataParseException, FetchException { TestSplitfile test = TestSplitfile.constructMultipleSegments(size, segmentDataBlockCount, segmentCheckBlockCount, segmentSize, checkSegmentSize, deductBlocksFromSegments, topCompatibilityMode, null, false); testRandomMixtureMultiSegment(test); test.free(); }
public void testChooseKeyOneTry() throws CHKEncodeException, IOException, MetadataUnresolvedException, MetadataParseException, FetchException { int dataBlocks = 3, checkBlocks = 3; TestSplitfile test = TestSplitfile.constructSingleSegment(dataBlocks*BLOCK_SIZE, checkBlocks, null, false); StorageCallback cb = test.createStorageCallback(); FetchContext ctx = test.makeFetchContext(); ctx.maxSplitfileBlockRetries = 0; SplitFileFetcherStorage storage = test.createStorage(cb, ctx); boolean[] tried = new boolean[dataBlocks+checkBlocks]; innerChooseKeyTest(dataBlocks, checkBlocks, storage.segments[0], tried, test, false); assertEquals(storage.chooseRandomKey(), null); cb.waitForFailed(); }
public void testWriteReadSegmentKeys() throws FetchException, MetadataParseException, IOException, CHKEncodeException, MetadataUnresolvedException, ChecksumFailedException { int dataBlocks = 3, checkBlocks = 3; TestSplitfile test = TestSplitfile.constructSingleSegment(dataBlocks*BLOCK_SIZE, checkBlocks, null, true); StorageCallback cb = test.createStorageCallback(); SplitFileFetcherStorage storage = test.createStorage(cb); SplitFileFetcherSegmentStorage segment = storage.segments[0]; SplitFileSegmentKeys keys = segment.getSegmentKeys(); SplitFileSegmentKeys moreKeys = segment.readSegmentKeys(); assertTrue(keys.equals(moreKeys)); storage.close(); }