@Override public boolean run(ClientContext context) { long now = System.currentTimeMillis(); long wakeupTime; synchronized(cooldownLock) { if(cooldownTime < now) return false; long oldCooldownTime = overallCooldownWakeupTime; if(overallCooldownWakeupTime > now) return false; // Wait for it to wake up. wakeupTime = Long.MAX_VALUE; for(SplitFileFetcherSegmentStorage segment : segments) { long segmentTime = segment.getOverallCooldownTime(); if(segmentTime < now) return false; wakeupTime = Math.min(segmentTime, wakeupTime); } overallCooldownWakeupTime = wakeupTime; if(overallCooldownWakeupTime < oldCooldownTime) return false; } fetcher.reduceCooldown(wakeupTime); return false; }
public synchronized void getUnfetchedKeys(List<Key> keys) throws IOException { if(finished || tryDecode) return; SplitFileSegmentKeys keyList = getSegmentKeys(); for(int i=0;i<totalBlocks();i++) { if(!blockChooser.hasSucceeded(i)) keys.add(keyList.getNodeKey(i, null, false)); } }
/** Should be called before scheduling, unlike restart(). Doesn't lock, i.e. part of * construction. But we must have read metadata on the regular segments first, which won't be * true in the constructor. */ public void checkBlocks() { for(int i=0;i<totalBlocks;i++) { if(segments[i].hasBlock(blockNumbers[i])) { blocksFound[i] = true; totalFound++; } } }
private void waitForDecode(SplitFileFetcherSegmentStorage segment) { while(!segment.hasSucceeded()) { assertFalse(segment.hasFailed()); try { Thread.sleep(1); } catch (InterruptedException e) { // Ignore. } } }
private void waitForFinished(SplitFileFetcherSegmentStorage segment) { while(!segment.isFinished()) { assertFalse(segment.hasFailed()); try { Thread.sleep(1); } catch (InterruptedException e) { // Ignore. } } }
int totalBlocks = totalBlocks(); byte[][] allBlocks = readAllBlocks(); SplitFileSegmentKeys keys = getSegmentKeys(); if(allBlocks == null || keys == null) { return; if(fetchedCount < blocksForDecode()) { int oldBlocksFetchedCount = blockChooser.successCount(); blockChooser.replaceSuccesses(used); if(fetchedCount < blocksForDecode()) { writeMetadata(); boolean wasCorrupt; synchronized(this) { byte[][] dataBlocks = new byte[blocksForDecode()][]; byte[][] checkBlocks = new byte[this.checkBlocks][]; if(blockNumber < blocksForDecode()) validDataBlocks++; if(blockNumber < dataBlocks.length) maybeBlocks.clear(); maybeBlocks = null; if(validBlocks < blocksForDecode()) { writeMetadata(); boolean wasCorrupt;
int countCrossCheckBlocks = 0; for(int i=0;i<segments.length;i++) { segments[i] = new SplitFileFetcherSegmentStorage(this, dis, i, maxRetries != -1, dataOffset, completeViaTruncation ? crossCheckBlocksOffset : -1, segmentKeysOffset, segmentStatusOffset, keysFetching); dataOffset += crossCheckBlocks * CHKBlock.DATA_LENGTH; segmentKeysOffset += SplitFileFetcherSegmentStorage.storedKeysLength(dataBlocks+crossCheckBlocks, checkBlocks, splitfileSingleCryptoKey != null, checksumLength); segmentStatusOffset += SplitFileFetcherSegmentStorage.paddedStoredSegmentStatusLength(dataBlocks, checkBlocks, crossCheckBlocks, maxRetries != -1, checksumLength, true); if(dataOffset > rafLength) boolean needsDecode = false; try { segment.readMetadata(); if(segment.hasFailed()) { raf.close(); if(segment.needsDecode()) needsDecode = true; if(needsDecode) { SplitFileFetcherSegmentStorage segment = segments[i]; try { segment.readSegmentKeys(); } catch (ChecksumFailedException e) { throw new StorageFormatException("Keys corrupted");
SplitFileFetcherSegmentStorage segment = storage.segments[0]; for(int i=0;i<test.dataBlocks.length;i++) { segment.onNonFatalFailure(i); segment.onNonFatalFailure(i+test.dataBlocks.length); assertFalse(segment.hasStartedDecode()); assertTrue(segment.onGotKey(test.checkKeys[i].getNodeCHK(), test.encodeCheckBlock(i))); cb.markDownloadedBlock(i + test.dataBlocks.length); assertTrue(segment.hasStartedDecode()); cb.checkFailed(); waitForDecode(segment);
assertFalse(segment.corruptMetadata()); int total = test.dataBlocks.length+test.checkBlocks.length; for(int i=0;i<total;i++) segment.onNonFatalFailure(i); // We want healing on all blocks that aren't found. boolean[] hits = new boolean[total]; for(int i=0;i<test.dataBlocks.length;i++) { } while (hits[block]); hits[block] = true; assertFalse(segment.hasStartedDecode()); assertTrue(segment.onGotKey(test.getCHK(block), test.encodeBlock(block))); cb.markDownloadedBlock(block); assertTrue(segment.hasStartedDecode()); cb.checkFailed(); waitForDecode(segment);
keys.clear(); for(int i=0;i<dataBlocks+checkBlocks;i++) { int chosen = storage.chooseRandomKey(); assertTrue(chosen != -1); assertFalse(tried[chosen]); assertEquals(storage.chooseRandomKey(), -1); assertEquals(storage.getOverallCooldownTime(), Long.MAX_VALUE); storage.onNonFatalFailure(i); assertEquals(storage.getOverallCooldownTime(), 0); assertTrue(storage.getOverallCooldownTime() != Long.MAX_VALUE || storage.hasFailed());
/** Add a random block that has not been added already or decoded already. * @throws IOException */ private boolean addRandomBlock(SplitFileInserterStorage storage, SplitFileFetcherStorage fetcherStorage, Random random) throws IOException { int segCount = storage.segments.length; boolean[] exhaustedSegments = new boolean[segCount]; for(int i=0;i<segCount;i++) { while(true) { int segNo = random.nextInt(segCount); if(exhaustedSegments[segNo]) continue; SplitFileFetcherSegmentStorage segment = fetcherStorage.segments[segNo]; if(segment.isDecodingOrFinished()) { exhaustedSegments[segNo] = true; break; } while(true) { int blockNo = random.nextInt(segment.totalBlocks()); if(segment.hasBlock(blockNo)) { continue; } ClientCHKBlock block = storage.segments[segNo].encodeBlock(blockNo); boolean success = segment.onGotKey(block.getClientKey().getNodeCHK(), block.getBlock()); assertTrue(success); return true; } } } return false; }
int total = test.dataBlocks.length+test.checkBlocks.length; for(SplitFileFetcherSegmentStorage segment : storage.segments) { for(int i=0;i<segment.totalBlocks();i++) segment.onNonFatalFailure(i); // We want healing on all blocks that aren't found. hits[block] = true; SplitFileFetcherSegmentStorage segment = storage.segments[test.segmentFor(block)]; if(segment.hasStartedDecode()) { i--; continue; assertTrue(segment.onGotKey(test.getCHK(block), test.encodeBlock(block))); cb.markDownloadedBlock(block); cb.checkFailed(); for(SplitFileFetcherSegmentStorage segment : storage.segments) assertTrue(segment.hasStartedDecode()); // All segments have started decoding. cb.checkFailed(); for(SplitFileFetcherSegmentStorage segment : storage.segments)
public synchronized ClientCHK getKey(int blockNum) { SplitFileSegmentKeys keys; try { keys = getSegmentKeys(); } catch (IOException e) { return null; } if(keys == null) return null; return keys.getKey(blockNum, null, false); }
@Override public boolean run(ClientContext context) { try { // FIXME CPU USAGE Add another API to the segment to avoid re-decoding. SplitFileSegmentKeys keys = segments[blockNo].getSegmentKeys(); if(keys == null) return false; boolean success = segments[blockNo].innerOnGotKey(key.getNodeCHK(), block, keys, blockNumbers[blockNo], data); if(success) { if(logMINOR) Logger.minor(this, "Successfully decoded cross-segment block"); } else { // Not really a big deal, but potentially interesting... Logger.warning(this, "Decoded cross-segment block but not wanted by segment"); } } catch (IOException e) { parent.failOnDiskError(e); return true; } return false; } });
public void testWriteReadSegmentKeys() throws FetchException, MetadataParseException, IOException, CHKEncodeException, MetadataUnresolvedException, ChecksumFailedException { int dataBlocks = 3, checkBlocks = 3; TestSplitfile test = TestSplitfile.constructSingleSegment(dataBlocks*BLOCK_SIZE, checkBlocks, null, true); StorageCallback cb = test.createStorageCallback(); SplitFileFetcherStorage storage = test.createStorage(cb); SplitFileFetcherSegmentStorage segment = storage.segments[0]; SplitFileSegmentKeys keys = segment.getSegmentKeys(); SplitFileSegmentKeys moreKeys = segment.readSegmentKeys(); assertTrue(keys.equals(moreKeys)); storage.close(); }
fetched[blockNo] = true; ClientCHKBlock block = inserterSegment.encodeBlock(blockNo); assertFalse(fetcherSegment.hasStartedDecode()); boolean success = fetcherSegment.onGotKey(block.getClientKey().getNodeCHK(), block.getBlock()); assertTrue(success); fcb.checkFailed(); assertTrue(fetcherSegment.hasStartedDecode()); fcb.checkFailed(); waitForDecode(fetcherSegment);
public SplitFileSegmentKeys getSegmentKeys() throws IOException { synchronized(this) { if(keysCache != null) { SplitFileSegmentKeys cached = keysCache.get(); if(cached != null) return cached; } SplitFileSegmentKeys keys; try { keys = readSegmentKeys(); } catch (ChecksumFailedException e) { Logger.error(this, "Keys corrupted on "+this+" !"); // Treat as IOException, i.e. fatal. FIXME! throw new IOException(e); } if(keys == null) return keys; keysCache = new SoftReference<SplitFileSegmentKeys>(keys); return keys; } }
found = storage.segments[i].onGotKey((NodeCHK)key, (CHKBlock)block); } catch (IOException e) { fetcher.failOnDiskError(e);
private boolean allSucceeded() { for(SplitFileFetcherSegmentStorage segment : segments) { if(!segment.hasSucceeded()) return false; } return true; }
private boolean allFinished() { // First, are any of the segments still working, that is, are they able to send requests, // or are they decoding/encoding? for(SplitFileFetcherSegmentStorage segment : segments) { if(!segment.isFinished()) return false; } // We cannot proceed unless none of the cross-segments is decoding. if(crossSegments != null) { for(SplitFileFetcherCrossSegmentStorage segment : crossSegments) { if(segment.isDecoding()) return false; } } return true; }