@Override public SendableRequestItem chooseKey(KeysFetchingLocally keys, ClientContext context) { return storage.chooseBlock(); }
@Override public boolean run(ClientContext context) { try { Metadata metadata = storage.encodeMetadata(); reportMetadata(metadata); if(ctx.getCHKOnly) onSucceeded(metadata); } catch (IOException e) { storage.fail(new InsertException(InsertExceptionMode.BUCKET_ERROR, e, null)); } catch (MissingKeyException e) { storage.fail(new InsertException(InsertExceptionMode.BUCKET_ERROR, "Lost one or more keys", e, null)); } return false; }
/** @return -1 if the insert has finished, 0 if has blocks to send, otherwise Long.MAX_VALUE. */ public long getWakeupTime(ClientContext context, long now) { // LOCKING: hasFinished() uses (this), separate from cooldownLock. // It is safe to use both here (on the request selection thread), one after the other. if (hasFinished()) return -1; if (noBlocksToSend()) return Long.MAX_VALUE; else return 0; }
public void onFailure(int blockNo, InsertException e) { if(logMINOR) Logger.minor(this, "Failed block "+blockNo+" with "+e+" for "+this+" for "+parent); if(parent.hasFinished()) return; // Race condition possible as this is a callback parent.addFailure(e); if(e.isFatal()) { parent.failFatalErrorInBlock(); } else { if(e.mode == InsertExceptionMode.ROUTE_NOT_FOUND && readKey(blockNo); blockChooser.onRNF(blockNo); parent.clearCooldown(); return; } catch (MissingKeyException e1) { Logger.error(this, "RNF but no key on block "+blockNo+" on "+this); } catch (IOException e1) { if(parent.hasFinished()) return; // Race condition possible as this is a callback parent.failOnDiskError(e1); return; parent.failTooManyRetriesInBlock(); return; parent.failTooManyRetriesInBlock(); } else { if(blockChooser.maxRetries >= 0) lazyWriteMetadata(); parent.clearCooldown();
private void testEncodeAfterShutdownCrossSegment(long size) throws InsertException, IOException, MissingKeyException, StorageFormatException, ChecksumFailedException, ResumeFailedException, MetadataUnresolvedException { Random r = new Random(12121); LockableRandomAccessBuffer data = generateData(r, size); HashResult[] hashes = getHashes(data); MyCallback cb = new MyCallback(); MyKeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, true, baseContext.clone(), cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0); executor.waitForIdle(); // Has not encoded anything. for(SplitFileInserterSegmentStorage segment : storage.segments) assert(!segment.isFinishedEncoding()); SplitFileInserterStorage resumed = new SplitFileInserterStorage(storage.getRAF(), data, cb, r, memoryLimitedJobRunner, jobRunner, ticker, keys, fg, persistentFileTracker, null); resumed.start(); cb.waitForFinishedEncode(); cb.waitForHasKeys(); executor.waitForIdle(); resumed.encodeMetadata(); assertTrue(resumed.getStatus() == Status.ENCODED); resumed.originalData.free(); resumed.getRAF().free(); }
public void testSmallSplitfileNoLastBlock() throws IOException, InsertException { Random r = new Random(12121); long size = 65536; // Exact multiple, so no last block LockableRandomAccessBuffer data = generateData(r, size); HashResult[] hashes = getHashes(data); MyCallback cb = new MyCallback(); KeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, false, baseContext.clone(), cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0); storage.start(); cb.waitForFinishedEncode(); assertEquals(storage.segments.length, 1); assertEquals(storage.segments[0].dataBlockCount, 2); assertEquals(storage.segments[0].checkBlockCount, 3); assertEquals(storage.segments[0].crossCheckBlockCount, 0); assertTrue(storage.getStatus() == Status.ENCODED); }
context.consecutiveRNFsCountAsSuccess = 2; MyKeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, false, context, cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0); storage.start(); cb.waitForFinishedEncode(); assertEquals(storage.segments.length, 1); assertEquals(segment.checkBlockCount, 3); assertEquals(segment.crossCheckBlockCount, 0); assertEquals(storage.getStatus(), Status.ENCODED); boolean[] chosenBlocks = new boolean[segment.totalBlockCount]; assertFalse(storage.noBlocksToSend()); chosenBlocks[chosen.blockNumber] = true; assertNull(storage.chooseBlock()); assertTrue(storage.noBlocksToSend()); for(int i=0;i<segment.totalBlockCount;i++) { segment.onFailure(i, new InsertException(InsertExceptionMode.ROUTE_NOT_FOUND)); assertFalse(storage.noBlocksToSend()); assertEquals(storage.getStatus(), Status.SUCCEEDED);
MyCallback cb = new MyCallback(); MyKeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, true, baseContext.clone(), cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0); storage.start(); cb.waitForFinishedEncode(); cb.waitForHasKeys(); executor.waitForIdle(); Metadata metadata = storage.encodeMetadata(); assertTrue(storage.getStatus() == Status.ENCODED); Bucket mBucket1 = bigBucketFactory.makeBucket(-1); DataOutputStream os = new DataOutputStream(mBucket1.getOutputStream()); metadata.writeTo(os); os.close(); SplitFileInserterStorage resumed = new SplitFileInserterStorage(storage.getRAF(), data, cb, r, memoryLimitedJobRunner, jobRunner, ticker, keys, fg, persistentFileTracker, null); Metadata metadata2 = storage.encodeMetadata(); Bucket mBucket2 = bigBucketFactory.makeBucket(-1); os = new DataOutputStream(mBucket2.getOutputStream()); int totalBlocks = storage.getTotalBlockCount(); assertEquals(totalBlocks, resumed.getTotalBlockCount()); for(int i=0;i<totalBlocks;i++) { BlockInsert chosen = resumed.chooseBlock(); if(chosen == null) { assertFalse(true);
/** Status. Generally depends on the status of the individual segments... * Not persisted: Can be deduced from the state of the segments, except for the last 3 states, * which are only used during completion (we don't keep the storage around once we're * finished). */ enum Status { NOT_STARTED, STARTED, ENCODED_CROSS_SEGMENTS, ENCODED, GENERATING_METADATA, SUCCEEDED, FAILED }
MyCallback cb = new MyCallback(); KeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, false, baseContext.clone(), cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0); storage.start(); cb.waitForFinishedEncode(); assertEquals(storage.segments[0].checkBlockCount, 3); assertEquals(storage.segments[0].crossCheckBlockCount, 0); assertTrue(Arrays.equals(storage.readSegmentDataBlock(0, 0), Arrays.copyOfRange(originalData, 0, CHKBlock.DATA_LENGTH))); int truncateLength = (int) (size % CHKBlock.DATA_LENGTH); long offsetLastBlock = size - truncateLength; byte[] buf = storage.readSegmentDataBlock(0, 1); assert(buf.length == CHKBlock.DATA_LENGTH); byte[] truncated = Arrays.copyOfRange(buf, 0, truncateLength); assertEquals(originalLastBlock.length, truncated.length); assertTrue(Arrays.equals(originalLastBlock, truncated)); assertTrue(storage.getStatus() == Status.ENCODED);
this.freeData = freeData; try { storage = new SplitFileInserterStorage(originalData, decompressedLength, this, compressionCodec, meta, isMetadata, archiveType, context.getRandomAccessBufferFactory(persistent), persistent, throw new InsertException(InsertExceptionMode.BUCKET_ERROR, e, null); this.raf = storage.getRAF(); this.sender = new SplitFileInserterSender(this, storage); this.realTime = realTime;
synchronized(this) { assert(failing == null); if(hasFinished()) return false; status = Status.GENERATING_METADATA; Metadata metadata = encodeMetadata(); synchronized(this) { status = Status.SUCCEEDED;
raf.onResume(context); originalData.onResume(context); this.storage = new SplitFileInserterStorage(raf, originalData, this, context.fastWeakRandom, context.memoryLimitedJobRunner, context.getJobRunner(true), context.ticker, context.getChkInsertScheduler(realTime).fetchingKeys(), context.persistentFG, context.persistentFileTracker, context.getPersistentMasterSecret()); storage.onResume(context); this.sender = new SplitFileInserterSender(this, storage); schedule(context);
public byte[] readDataBlock(int blockNo) throws IOException { assert(blockNo >= 0 && blockNo < dataBlockCount); return parent.readSegmentDataBlock(segNo, blockNo); }
private boolean maybeFail() { // Might have failed. // Have to check segments before checking for failure because of race conditions. if(allSegmentsCompletedOrFailed()) { InsertException e = null; synchronized(this) { if(failing == null) return false; e = failing; if(hasFinished()) { if(logMINOR) Logger.minor(this, "Maybe fail returning true because already finished"); return true; } status = Status.FAILED; } if(logMINOR) Logger.minor(this, "Maybe fail returning true with error "+e); callback.onFailed(e); return true; } else { return false; } }
@Override public void schedule(ClientContext context) throws InsertException { cb.onBlockSetFinished(this, context); storage.start(); if(!ctx.getCHKOnly) { sender.clearWakeupTime(context); sender.schedule(context); } }
public void testSmallSplitfileHasKeys() throws IOException, InsertException, MissingKeyException { Random r = new Random(12121); long size = 65536; // Exact multiple, so no last block LockableRandomAccessBuffer data = generateData(r, size); HashResult[] hashes = getHashes(data); MyCallback cb = new MyCallback(); InsertContext context = baseContext.clone(); context.earlyEncode = true; KeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, false, context, cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0); storage.start(); cb.waitForFinishedEncode(); assertEquals(storage.segments.length, 1); assertEquals(storage.segments[0].dataBlockCount, 2); assertEquals(storage.segments[0].checkBlockCount, 3); assertEquals(storage.segments[0].crossCheckBlockCount, 0); cb.waitForHasKeys(); for(int i=0;i<storage.segments[0].dataBlockCount+storage.segments[0].checkBlockCount+storage.segments[0].crossCheckBlockCount;i++) storage.segments[0].readKey(i); storage.encodeMetadata(); assertTrue(storage.getStatus() == Status.ENCODED); }
public void testSmallSplitfileCompletion() throws IOException, InsertException, MissingKeyException { Random r = new Random(12121); long size = 65536; // Exact multiple, so no last block LockableRandomAccessBuffer data = generateData(r, size); HashResult[] hashes = getHashes(data); MyCallback cb = new MyCallback(); InsertContext context = baseContext.clone(); KeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, false, context, cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0); storage.start(); cb.waitForFinishedEncode(); assertEquals(storage.segments.length, 1); SplitFileInserterSegmentStorage segment = storage.segments[0]; assertEquals(segment.dataBlockCount, 2); assertEquals(segment.checkBlockCount, 3); assertEquals(segment.crossCheckBlockCount, 0); assertEquals(storage.getStatus(), Status.ENCODED); for(int i=0;i<segment.totalBlockCount;i++) { segment.onInsertedBlock(i, segment.encodeBlock(i).getClientKey()); } cb.waitForSucceededInsert(); assertEquals(storage.getStatus(), Status.SUCCEEDED); }
context.maxInsertRetries = 2; MyKeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, false, context, cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0); storage.start(); cb.waitForFinishedEncode(); assertEquals(storage.segments.length, 1); assertEquals(segment.checkBlockCount, 3); assertEquals(segment.crossCheckBlockCount, 0); assertEquals(storage.getStatus(), Status.ENCODED); boolean[] chosenBlocks = new boolean[segment.totalBlockCount]; assertFalse(storage.noBlocksToSend()); chosenBlocks[chosen.blockNumber] = true; assertNull(storage.chooseBlock()); assertTrue(storage.noBlocksToSend()); for(int i=0;i<segment.totalBlockCount;i++) { assertFalse(storage.noBlocksToSend()); assertEquals(storage.getStatus(), Status.SUCCEEDED);
MyCallback cb = new MyCallback(); KeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, true, baseContext.clone(), cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0); storage.start(); cb.waitForFinishedEncode(); assertEquals(storage.segments.length, 1); assertEquals(storage.segments[0].checkBlockCount, 3); assertEquals(storage.segments[0].crossCheckBlockCount, 0); assertTrue(storage.getStatus() == Status.ENCODED); SplitFileInserterStorage resumed = null; for(int i=0;i<storage.segments[0].totalBlockCount;i++) { executor.waitForIdle(); resumed = new SplitFileInserterStorage(storage.getRAF(), data, cb, r, memoryLimitedJobRunner, jobRunner, ticker, keys, fg, persistentFileTracker, null); assertEquals(resumed.segments.length, 1); assertEquals(segment.checkBlockCount, 3); assertEquals(segment.crossCheckBlockCount, 0); assertTrue(resumed.getStatus() == Status.ENCODED); segment.onInsertedBlock(i, segment.encodeBlock(i).getClientKey()); assertEquals(Status.SUCCEEDED, resumed.getStatus());