MyCallback cb = new MyCallback(); KeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, false, baseContext.clone(), cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker,
this.freeData = freeData; try { storage = new SplitFileInserterStorage(originalData, decompressedLength, this, compressionCodec, meta, isMetadata, archiveType, context.getRandomAccessBufferFactory(persistent), persistent,
private void testEncodeAfterShutdownCrossSegment(long size) throws InsertException, IOException, MissingKeyException, StorageFormatException, ChecksumFailedException, ResumeFailedException, MetadataUnresolvedException { Random r = new Random(12121); LockableRandomAccessBuffer data = generateData(r, size); HashResult[] hashes = getHashes(data); MyCallback cb = new MyCallback(); MyKeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, true, baseContext.clone(), cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0); executor.waitForIdle(); // Has not encoded anything. for(SplitFileInserterSegmentStorage segment : storage.segments) assert(!segment.isFinishedEncoding()); SplitFileInserterStorage resumed = new SplitFileInserterStorage(storage.getRAF(), data, cb, r, memoryLimitedJobRunner, jobRunner, ticker, keys, fg, persistentFileTracker, null); resumed.start(); cb.waitForFinishedEncode(); cb.waitForHasKeys(); executor.waitForIdle(); resumed.encodeMetadata(); assertTrue(resumed.getStatus() == Status.ENCODED); resumed.originalData.free(); resumed.getRAF().free(); }
public void testSmallSplitfileNoLastBlock() throws IOException, InsertException { Random r = new Random(12121); long size = 65536; // Exact multiple, so no last block LockableRandomAccessBuffer data = generateData(r, size); HashResult[] hashes = getHashes(data); MyCallback cb = new MyCallback(); KeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, false, baseContext.clone(), cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0); storage.start(); cb.waitForFinishedEncode(); assertEquals(storage.segments.length, 1); assertEquals(storage.segments[0].dataBlockCount, 2); assertEquals(storage.segments[0].checkBlockCount, 3); assertEquals(storage.segments[0].crossCheckBlockCount, 0); assertTrue(storage.getStatus() == Status.ENCODED); }
MyCallback cb = new MyCallback(); MyKeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, true, baseContext.clone(), cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, metadata.writeTo(os); os.close(); SplitFileInserterStorage resumed = new SplitFileInserterStorage(storage.getRAF(), data, cb, r, memoryLimitedJobRunner, jobRunner, ticker, keys, fg, persistentFileTracker, null);
public void testSmallSplitfileHasKeys() throws IOException, InsertException, MissingKeyException { Random r = new Random(12121); long size = 65536; // Exact multiple, so no last block LockableRandomAccessBuffer data = generateData(r, size); HashResult[] hashes = getHashes(data); MyCallback cb = new MyCallback(); InsertContext context = baseContext.clone(); context.earlyEncode = true; KeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, false, context, cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0); storage.start(); cb.waitForFinishedEncode(); assertEquals(storage.segments.length, 1); assertEquals(storage.segments[0].dataBlockCount, 2); assertEquals(storage.segments[0].checkBlockCount, 3); assertEquals(storage.segments[0].crossCheckBlockCount, 0); cb.waitForHasKeys(); for(int i=0;i<storage.segments[0].dataBlockCount+storage.segments[0].checkBlockCount+storage.segments[0].crossCheckBlockCount;i++) storage.segments[0].readKey(i); storage.encodeMetadata(); assertTrue(storage.getStatus() == Status.ENCODED); }
MyCallback cb = new MyCallback(); KeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, true, baseContext.clone(), cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, assertTrue(storage.getStatus() == Status.ENCODED); executor.waitForIdle(); SplitFileInserterStorage resumed = new SplitFileInserterStorage(storage.getRAF(), data, cb, r, memoryLimitedJobRunner, jobRunner, ticker, keys, fg, persistentFileTracker, null); assertEquals(resumed.segments.length, 1);
MyCallback cb = new MyCallback(); KeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, true, baseContext.clone(), cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, for(int i=0;i<storage.segments[0].totalBlockCount;i++) { executor.waitForIdle(); resumed = new SplitFileInserterStorage(storage.getRAF(), data, cb, r, memoryLimitedJobRunner, jobRunner, ticker, keys, fg, persistentFileTracker, null); assertEquals(resumed.segments.length, 1);
MyCallback cb = new MyCallback(); KeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, true, baseContext.clone(), cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, for(int i=0;i<storage.segments[0].totalBlockCount;i++) { executor.waitForIdle(); resumed = new SplitFileInserterStorage(storage.getRAF(), data, cb, r, memoryLimitedJobRunner, jobRunner, ticker, keys, fg, persistentFileTracker, null); assertEquals(resumed.segments.length, 1);
public void testSmallSplitfileCompletion() throws IOException, InsertException, MissingKeyException { Random r = new Random(12121); long size = 65536; // Exact multiple, so no last block LockableRandomAccessBuffer data = generateData(r, size); HashResult[] hashes = getHashes(data); MyCallback cb = new MyCallback(); InsertContext context = baseContext.clone(); KeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, false, context, cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0); storage.start(); cb.waitForFinishedEncode(); assertEquals(storage.segments.length, 1); SplitFileInserterSegmentStorage segment = storage.segments[0]; assertEquals(segment.dataBlockCount, 2); assertEquals(segment.checkBlockCount, 3); assertEquals(segment.crossCheckBlockCount, 0); assertEquals(storage.getStatus(), Status.ENCODED); for(int i=0;i<segment.totalBlockCount;i++) { segment.onInsertedBlock(i, segment.encodeBlock(i).getClientKey()); } cb.waitForSucceededInsert(); assertEquals(storage.getStatus(), Status.SUCCEEDED); }
raf.onResume(context); originalData.onResume(context); this.storage = new SplitFileInserterStorage(raf, originalData, this, context.fastWeakRandom, context.memoryLimitedJobRunner, context.getJobRunner(true), context.ticker, context.getChkInsertScheduler(realTime).fetchingKeys(), context.persistentFG,
public void testCancel() throws IOException, InsertException, MissingKeyException { Random r = new Random(12124); long size = 32768*6; BarrierRandomAccessBuffer data = new BarrierRandomAccessBuffer(generateData(r, size)); HashResult[] hashes = getHashes(data); data.pause(); MyCallback cb = new MyCallback(); InsertContext context = baseContext.clone(); context.earlyEncode = true; KeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, false, context, cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0); storage.start(); assertEquals(storage.getStatus(), Status.STARTED); assertEquals(storage.segments.length, 1); SplitFileInserterSegmentStorage segment = storage.segments[0]; segment.onFailure(0, new InsertException(InsertExceptionMode.INTERNAL_ERROR)); data.proceed(); // Now it will complete encoding, and then report in, and then fail. try { cb.waitForFinishedEncode(); assertFalse(true); // Should have failed. } catch (InsertException e) { assertFalse(segment.isEncoding()); assertEquals(storage.getStatus(), Status.FAILED); } }
context.consecutiveRNFsCountAsSuccess = 0; context.maxInsertRetries = 2; SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, true, context, cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, resumed = new SplitFileInserterStorage(storage.getRAF(), data, cb, r, memoryLimitedJobRunner, jobRunner, ticker, keys, fg, persistentFileTracker, null); assertEquals(resumed.segments.length, 1);
context.maxInsertRetries = 2; KeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, false, context, cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker,
context.earlyEncode = true; KeysFetchingLocally keysFetching = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, false, context, cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker,
context.maxInsertRetries = 2; KeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, false, context, cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker,
context.maxInsertRetries = 2; MyKeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, false, context, cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker,
context.maxInsertRetries = 2; ClientRequestSelector keys = new ClientRequestSelector(true, false, false, null); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, false, context, cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker,
context.consecutiveRNFsCountAsSuccess = 3; MyKeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, false, context, cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker,
context.earlyEncode = true; KeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, false, context, cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker,