for(int i=0;i<dataBlocks+checkBlocks;i++) { int chosen = storage.chooseRandomKey(); assertTrue(chosen != -1); assertFalse(tried[chosen]); tried[chosen] = true; Key k = test.getCHK(chosen); assertTrue(b); assertEquals(storage.chooseRandomKey(), -1); assertEquals(storage.getOverallCooldownTime(), Long.MAX_VALUE); assertEquals(storage.getOverallCooldownTime(), 0); assertTrue(storage.getOverallCooldownTime() != Long.MAX_VALUE || storage.hasFailed());
static TestSplitfile constructSingleSegment(long size, int checkBlocks, String mime, boolean persistent) throws IOException, CHKEncodeException, MetadataUnresolvedException, MetadataParseException { assertTrue(checkBlocks <= MAX_SEGMENT_SIZE); assertTrue(size < MAX_SEGMENT_SIZE * (long)BLOCK_SIZE); Bucket data = makeRandomBucket(size); byte[][] originalDataBlocks = splitAndPadBlocks(data, size); int dataBlocks = originalDataBlocks.length; assertTrue(dataBlocks <= MAX_SEGMENT_SIZE); assertTrue(dataBlocks + checkBlocks <= MAX_SEGMENT_SIZE); byte[][] originalCheckBlocks = constructBlocks(checkBlocks); codec.encode(originalDataBlocks, originalCheckBlocks, falseArray(checkBlocks), BLOCK_SIZE); ClientMetadata cm = new ClientMetadata(mime); byte[] cryptoKey = randomKey(); byte cryptoAlgorithm = Key.ALGO_AES_CTR_256_SHA256; ClientCHK[] dataKeys = makeKeys(originalDataBlocks, cryptoKey, cryptoAlgorithm); ClientCHK[] checkKeys = makeKeys(originalCheckBlocks, cryptoKey, cryptoAlgorithm); Metadata m = new Metadata(SplitfileAlgorithm.ONION_STANDARD, dataKeys, checkKeys, dataBlocks, checkBlocks, 0, cm, size, null, null, size, false, null, null, size, size, dataBlocks,
public void testChooseKeyOneTry() throws CHKEncodeException, IOException, MetadataUnresolvedException, MetadataParseException, FetchException { int dataBlocks = 3, checkBlocks = 3; TestSplitfile test = TestSplitfile.constructSingleSegment(dataBlocks*BLOCK_SIZE, checkBlocks, null, false); StorageCallback cb = test.createStorageCallback(); FetchContext ctx = test.makeFetchContext(); ctx.maxSplitfileBlockRetries = 0; SplitFileFetcherStorage storage = test.createStorage(cb, ctx); boolean[] tried = new boolean[dataBlocks+checkBlocks]; innerChooseKeyTest(dataBlocks, checkBlocks, storage.segments[0], tried, test, false); assertEquals(storage.chooseRandomKey(), null); cb.waitForFailed(); }
private void testSingleSegment(int dataBlocks, int checkBlocks, long size) throws CHKEncodeException, IOException, FetchException, MetadataParseException, MetadataUnresolvedException { assertTrue(dataBlocks * (long)BLOCK_SIZE >= size); TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, null, false); testDataBlocksOnly(test); if(checkBlocks >= dataBlocks) testCheckBlocksOnly(test); testRandomMixture(test); test.free(); }
private void testDataBlocksOnly(TestSplitfile test) throws IOException, CHKEncodeException, FetchException, MetadataParseException { StorageCallback cb = test.createStorageCallback(); SplitFileFetcherStorage storage = test.createStorage(cb); SplitFileFetcherSegmentStorage segment = storage.segments[0]; for(int i=0;i<test.checkBlocks.length;i++) { segment.onNonFatalFailure(test.dataBlocks.length+i); } for(int i=0;i<test.dataBlocks.length;i++) { assertFalse(segment.hasStartedDecode()); assertTrue(segment.onGotKey(test.dataKeys[i].getNodeCHK(), test.encodeDataBlock(i))); cb.markDownloadedBlock(i); } cb.checkFailed(); assertTrue(segment.hasStartedDecode()); cb.checkFailed(); waitForDecode(segment); cb.checkFailed(); cb.waitForFinished(); cb.checkFailed(); test.verifyOutput(storage); cb.checkFailed(); storage.finishedFetcher(); cb.checkFailed(); waitForFinished(segment); cb.checkFailed(); cb.waitForFree(storage); cb.checkFailed(); }
for(int i=0;i<3;i++) { boolean[] tried = new boolean[dataBlocks+checkBlocks]; innerChooseKeyTest(dataBlocks, checkBlocks, storage.segments[0], tried, test, true); assertTrue(storage.segments[0].getOverallCooldownTime() > now); assertFalse(storage.segments[0].getOverallCooldownTime() == Long.MAX_VALUE); assertEquals(storage.chooseRandomKey(), null); Thread.sleep((long)(COOLDOWN_TIME+COOLDOWN_TIME/2+1)); cb.checkFailed(); innerChooseKeyTest(dataBlocks, checkBlocks, storage.segments[0], tried, test, true);
continue; assertTrue(segment.onGotKey(test.getCHK(block), test.encodeBlock(block))); cb.markDownloadedBlock(block); printChosenBlocks(hits); cb.checkFailed(); for(SplitFileFetcherSegmentStorage segment : storage.segments) assertTrue(segment.hasStartedDecode()); // All segments have started decoding. cb.checkFailed(); for(SplitFileFetcherSegmentStorage segment : storage.segments) waitForDecode(segment); cb.checkFailed(); cb.waitForFinished(); cb.checkFailed(); for(SplitFileFetcherSegmentStorage segment : storage.segments) waitForFinished(segment); cb.checkFailed(); cb.waitForFree(storage);
public void testPersistenceReloadThenChooseKey() throws IOException, StorageFormatException, CHKEncodeException, MetadataUnresolvedException, MetadataParseException, FetchException { int dataBlocks = 2; int checkBlocks = 3; long size = 32768*2-1; assertTrue(dataBlocks * (long)BLOCK_SIZE >= size); TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, null, true); StorageCallback cb = test.createStorageCallback(); FetchContext ctx = test.makeFetchContext(); ctx.maxSplitfileBlockRetries = 2; SplitFileFetcherStorage storage = test.createStorage(cb, ctx); // No need to shutdown the old storage. storage = test.createStorage(cb, ctx, cb.getRAF()); for(int i=0;i<3;i++) { boolean[] tried = new boolean[dataBlocks+checkBlocks]; innerChooseKeyTest(dataBlocks, checkBlocks, storage.segments[0], tried, test, false); } test.fetchingKeys.clear(); assertEquals(storage.chooseRandomKey(), null); cb.waitForFailed(); }
public void testWriteReadSegmentKeys() throws FetchException, MetadataParseException, IOException, CHKEncodeException, MetadataUnresolvedException, ChecksumFailedException { int dataBlocks = 3, checkBlocks = 3; TestSplitfile test = TestSplitfile.constructSingleSegment(dataBlocks*BLOCK_SIZE, checkBlocks, null, true); StorageCallback cb = test.createStorageCallback(); SplitFileFetcherStorage storage = test.createStorage(cb); SplitFileFetcherSegmentStorage segment = storage.segments[0]; SplitFileSegmentKeys keys = segment.getSegmentKeys(); SplitFileSegmentKeys moreKeys = segment.readSegmentKeys(); assertTrue(keys.equals(moreKeys)); storage.close(); }
private void waitForFinished(SplitFileFetcherSegmentStorage segment) { while(!segment.isFinished()) { assertFalse(segment.hasFailed()); try { Thread.sleep(1); } catch (InterruptedException e) { // Ignore. } } }
} while (hits[block]); hits[block] = true; assertFalse(segment.hasStartedDecode()); assertTrue(segment.onGotKey(test.getCHK(block), test.encodeBlock(block))); cb.markDownloadedBlock(block); assertTrue(segment.hasStartedDecode()); cb.checkFailed(); waitForDecode(segment); cb.checkFailed(); cb.waitForFinished(); storage.finishedFetcher(); cb.checkFailed(); waitForFinished(segment); cb.checkFailed(); cb.waitForFree(storage);
int checkBlocks = 3; long size = 32768*2-1; assertTrue(dataBlocks * (long)BLOCK_SIZE >= size); TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, null, true); StorageCallback cb = test.createStorageCallback(); for(int i=0;i<3;i++) { boolean[] tried = new boolean[dataBlocks+checkBlocks]; innerChooseKeyTest(dataBlocks, checkBlocks, storage.segments[0], tried, test, false); assertTrue(i != 2); storage.start(false); } catch (FetchException e) { assertEquals(storage.chooseRandomKey(), null); cb.waitForFailed();
/** Test persistence: Create and then reload. Don't do anything. */ public void testPersistenceReload() throws CHKEncodeException, IOException, MetadataUnresolvedException, MetadataParseException, FetchException, StorageFormatException { int dataBlocks = 2; int checkBlocks = 3; long size = 32768*2-1; assertTrue(dataBlocks * (long)BLOCK_SIZE >= size); TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, null, true); StorageCallback cb = test.createStorageCallback(); SplitFileFetcherStorage storage = test.createStorage(cb); // No need to shutdown the old storage. storage = test.createStorage(cb, test.makeFetchContext(), cb.getRAF()); storage.close(); }
private void waitForDecode(SplitFileFetcherSegmentStorage segment) { while(!segment.hasSucceeded()) { assertFalse(segment.hasFailed()); try { Thread.sleep(1); } catch (InterruptedException e) { // Ignore. } } }
int deductBlocksFromSegments, CompatibilityMode topCompatibilityMode, String mime, boolean persistent) throws IOException, CHKEncodeException, MetadataUnresolvedException, MetadataParseException { int dataBlocks = sum(segmentDataBlockCount); int checkBlocks = sum(segmentCheckBlockCount); int segments = segmentDataBlockCount.length; assertEquals((size + BLOCK_SIZE - 1) / BLOCK_SIZE, dataBlocks); assertEquals(segments, segmentCheckBlockCount.length); Bucket data = makeRandomBucket(size); byte[][] originalDataBlocks = splitAndPadBlocks(data, size); byte[][] originalCheckBlocks = constructBlocks(checkBlocks); int startDataBlock = 0; int startCheckBlock = 0; byte[][] segmentDataBlocks = Arrays.copyOfRange(originalDataBlocks, startDataBlock, startDataBlock + segmentDataBlockCount[seg]); byte[][] segmentCheckBlocks = Arrays.copyOfRange(originalCheckBlocks, startCheckBlock, startCheckBlock + segmentCheckBlockCount[seg]); codec.encode(segmentDataBlocks, segmentCheckBlocks, falseArray(segmentCheckBlocks.length), BLOCK_SIZE); startDataBlock += segmentDataBlockCount[seg]; startCheckBlock += segmentCheckBlockCount[seg]; byte[] cryptoKey = randomKey(); byte cryptoAlgorithm = Key.ALGO_AES_CTR_256_SHA256; ClientCHK[] dataKeys = makeKeys(originalDataBlocks, cryptoKey, cryptoAlgorithm); ClientCHK[] checkKeys = makeKeys(originalCheckBlocks, cryptoKey, cryptoAlgorithm); Metadata m = new Metadata(SplitfileAlgorithm.ONION_STANDARD, dataKeys, checkKeys, segmentSize, checkSegmentSize, deductBlocksFromSegments, cm, size, null, null, size, false, null, null,
int checkBlocks = 3; long size = 32768*2-1; assertTrue(dataBlocks * (long)BLOCK_SIZE >= size); TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, null, true); StorageCallback cb = test.createStorageCallback(); assertFalse(segment.corruptMetadata()); int total = test.dataBlocks.length+test.checkBlocks.length; for(int i=0;i<total;i++) } while (hits[block]); hits[block] = true; assertFalse(segment.hasStartedDecode()); assertTrue(segment.onGotKey(test.getCHK(block), test.encodeBlock(block))); cb.markDownloadedBlock(block); assertTrue(segment.hasStartedDecode()); cb.checkFailed(); waitForDecode(segment); cb.checkFailed(); cb.waitForFinished(); storage.finishedFetcher(); cb.checkFailed(); waitForFinished(segment); cb.checkFailed(); cb.waitForFree(storage);
public void testChooseKeyThreeTries() throws CHKEncodeException, IOException, MetadataUnresolvedException, MetadataParseException, FetchException { int dataBlocks = 3, checkBlocks = 3; TestSplitfile test = TestSplitfile.constructSingleSegment(dataBlocks*BLOCK_SIZE, checkBlocks, null, false); StorageCallback cb = test.createStorageCallback(); FetchContext ctx = test.makeFetchContext(); ctx.maxSplitfileBlockRetries = 2; SplitFileFetcherStorage storage = test.createStorage(cb, ctx); for(int i=0;i<3;i++) { boolean[] tried = new boolean[dataBlocks+checkBlocks]; innerChooseKeyTest(dataBlocks, checkBlocks, storage.segments[0], tried, test, false); } assertEquals(storage.chooseRandomKey(), null); cb.waitForFailed(); }
assertFalse(segment.hasStartedDecode()); assertTrue(segment.onGotKey(test.checkKeys[i].getNodeCHK(), test.encodeCheckBlock(i))); cb.markDownloadedBlock(i + test.dataBlocks.length); assertTrue(segment.hasStartedDecode()); cb.checkFailed(); waitForDecode(segment); cb.checkFailed(); cb.waitForFinished(); storage.finishedFetcher(); cb.checkFailed(); waitForFinished(segment); cb.checkFailed(); cb.waitForFree(storage);
int checkBlocks = 3; long size = 32768*2-1; assertTrue(dataBlocks * (long)BLOCK_SIZE >= size); TestSplitfile test = TestSplitfile.constructSingleSegment(size, checkBlocks, null, true); StorageCallback cb = test.createStorageCallback(); assertFalse(segment.corruptMetadata()); int total = test.dataBlocks.length+test.checkBlocks.length; for(int i=0;i<total;i++) } while (hits[block]); hits[block] = true; assertFalse(segment.hasStartedDecode()); assertTrue(segment.onGotKey(test.getCHK(block), test.encodeBlock(block))); cb.markDownloadedBlock(block); if(i != test.dataBlocks.length-1) { assertTrue(segment.hasStartedDecode()); cb.checkFailed(); waitForDecode(segment); cb.checkFailed(); cb.waitForFinished(); storage.finishedFetcher(); cb.checkFailed(); waitForFinished(segment); cb.checkFailed(); cb.waitForFree(storage);