/** Generate keys for each block and record them. * @throws IOException */ private void generateKeys(byte[][] dataBlocks, int offset) throws IOException { for(int i=0;i<dataBlocks.length;i++) { setKey(i + offset, encodeBlock(dataBlocks[i]).getClientKey()); } }
public ClientCHKBlock encodeBlock(int blockNo) throws IOException { if(parent.isFinishing()) { throw new IOException("Already finishing reading block "+blockNo+" for "+this+" for "+parent); } synchronized(this) { if(this.blockChooser.hasSucceeded(blockNo)) { Logger.error(this, "Already inserted block "+blockNo+" for "+this+" for "+parent); throw new IOException("Already inserted block "+blockNo+" for "+this+" for "+parent); } } byte[] buf = readBlock(blockNo); return encodeBlock(buf); }
private void writeCheckBlock(int checkBlockNo, byte[] buf) throws IOException { parent.writeCheckBlock(segNo, checkBlockNo, buf); if(DEBUG_ENCODE) { SplitFileInserterSegmentStorage segment = segments[checkBlockNo + dataBlockCount]; ClientCHK key = segment.encodeBlock(buf).getClientKey(); segment.setKey(blockNumbers[checkBlockNo + dataBlockCount], key); } }
private byte[][] readDataBlocks() throws IOException { RAFLock lock = parent.lockUnderlying(); try { byte[][] data = new byte[dataBlockCount][]; for(int i=0;i<dataBlockCount;i++) { data[i] = segments[i].readDataBlock(blockNumbers[i]); if(DEBUG_ENCODE) { ClientCHK key = segments[i].encodeBlock(data[i]).getClientKey(); segments[i].setKey(blockNumbers[i], key); } } return data; } finally { lock.unlock(); } }
/** Add a random block that has not been added already or decoded already. * @throws IOException */ private boolean addRandomBlock(SplitFileInserterStorage storage, SplitFileFetcherStorage fetcherStorage, Random random) throws IOException { int segCount = storage.segments.length; boolean[] exhaustedSegments = new boolean[segCount]; for(int i=0;i<segCount;i++) { while(true) { int segNo = random.nextInt(segCount); if(exhaustedSegments[segNo]) continue; SplitFileFetcherSegmentStorage segment = fetcherStorage.segments[segNo]; if(segment.isDecodingOrFinished()) { exhaustedSegments[segNo] = true; break; } while(true) { int blockNo = random.nextInt(segment.totalBlocks()); if(segment.hasBlock(blockNo)) { continue; } ClientCHKBlock block = storage.segments[segNo].encodeBlock(blockNo); boolean success = segment.onGotKey(block.getClientKey().getNodeCHK(), block.getBlock()); assertTrue(success); return true; } } } return false; }
final BlockInsert token = (BlockInsert) request.token; try { ClientCHKBlock clientBlock = token.segment.encodeBlock(token.blockNumber); CHKBlock block = clientBlock.getBlock(); final ClientCHK key = clientBlock.getClientKey();
public void testSmallSplitfileCompletion() throws IOException, InsertException, MissingKeyException { Random r = new Random(12121); long size = 65536; // Exact multiple, so no last block LockableRandomAccessBuffer data = generateData(r, size); HashResult[] hashes = getHashes(data); MyCallback cb = new MyCallback(); InsertContext context = baseContext.clone(); KeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, false, context, cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0); storage.start(); cb.waitForFinishedEncode(); assertEquals(storage.segments.length, 1); SplitFileInserterSegmentStorage segment = storage.segments[0]; assertEquals(segment.dataBlockCount, 2); assertEquals(segment.checkBlockCount, 3); assertEquals(segment.crossCheckBlockCount, 0); assertEquals(storage.getStatus(), Status.ENCODED); for(int i=0;i<segment.totalBlockCount;i++) { segment.onInsertedBlock(i, segment.encodeBlock(i).getClientKey()); } cb.waitForSucceededInsert(); assertEquals(storage.getStatus(), Status.SUCCEEDED); }
chosenBlocks[chosen.segment.segNo][chosen.blockNumber] = true; chosen.segment.onInsertedBlock(chosen.blockNumber, chosen.segment.encodeBlock(chosen.blockNumber).getClientKey());
SplitFileFetcherSegmentStorage fetcherSegment = fetcherStorage.segments[segNo]; for(int blockNo=0;blockNo<inserterSegment.dataBlockCount;blockNo++) { ClientCHKBlock block = inserterSegment.encodeBlock(blockNo); boolean success = fetcherSegment.onGotKey(block.getClientKey().getNodeCHK(), block.getBlock()); assertTrue(success);
assertTrue(resumed.getStatus() == Status.ENCODED); for(int i=0;i<segment.totalBlockCount;i++) { segment.onInsertedBlock(i, segment.encodeBlock(i).getClientKey());
assertEquals(segment.crossCheckBlockCount, 0); assertTrue(resumed.getStatus() == Status.ENCODED); segment.onInsertedBlock(i, segment.encodeBlock(i).getClientKey());
assertEquals(segment.crossCheckBlockCount, 0); assertTrue(resumed.getStatus() == Status.ENCODED); segment.onInsertedBlock(i, segment.encodeBlock(i).getClientKey());
assertFalse(chosenBlocks[chosen.blockNumber]); chosenBlocks[chosen.blockNumber] = true; segment.onInsertedBlock(chosen.blockNumber, segment.encodeBlock(chosen.blockNumber).getClientKey());
assertFalse(chosenBlocks[chosen.blockNumber]); chosenBlocks[chosen.blockNumber] = true; segment.setKey(chosen.blockNumber, segment.encodeBlock(chosen.blockNumber).getClientKey()); segment.onFailure(chosen.blockNumber, new InsertException(InsertExceptionMode.ROUTE_NOT_FOUND));
assertFalse(chosenBlocks[chosen.blockNumber]); chosenBlocks[chosen.blockNumber] = true; segment.onInsertedBlock(chosen.blockNumber, segment.encodeBlock(chosen.blockNumber).getClientKey());
assertEquals(segment.crossCheckBlockCount, 0); assertEquals(storage.getStatus(), Status.ENCODED); segment.setKey(0, segment.encodeBlock(0).getClientKey()); segment.onFailure(0, new InsertException(InsertExceptionMode.ROUTE_NOT_FOUND)); assertEquals(storage.getStatus(), Status.ENCODED);
assertFalse(chosenBlocks[chosen.blockNumber]); chosenBlocks[chosen.blockNumber] = true; segment.onInsertedBlock(chosen.blockNumber, segment.encodeBlock(chosen.blockNumber).getClientKey());
assertFalse(chosenBlocks[chosen.blockNumber]); chosenBlocks[chosen.blockNumber] = true; segment.onInsertedBlock(chosen.blockNumber, segment.encodeBlock(chosen.blockNumber).getClientKey());
} while (fetched[blockNo]); fetched[blockNo] = true; ClientCHKBlock block = inserterSegment.encodeBlock(blockNo); assertFalse(fetcherSegment.hasStartedDecode()); boolean success = fetcherSegment.onGotKey(block.getClientKey().getNodeCHK(), block.getBlock());
assertFalse(chosenBlocks[chosen.blockNumber]); chosenBlocks[chosen.blockNumber] = true; segment.onInsertedBlock(chosen.blockNumber, segment.encodeBlock(chosen.blockNumber).getClientKey());