/** * Write a log record as control record. The method will be used by Monitor Service to enforce a new inprogress segment. * * @param record * log record * @return future of the write */ public Future<DLSN> writeControlRecord(final LogRecord record) { record.setControl(); return write(record); }
public Future<DLSN> write(byte[] data) { requestPos += data.length; Future<DLSN> writeResult = logWriter.write(new LogRecord(requestPos, data)); return writeResult.addEventListener(new WriteCompleteListener(requestPos)); }
static void writeRecordsForNonBlockingReads(DistributedLogConfiguration conf, DistributedLogManager dlm, boolean recover, long segmentSize) throws Exception { long txId = 1; for (long i = 0; i < 3; i++) { BKAsyncLogWriter writer = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned(); for (long j = 1; j < segmentSize; j++) { FutureUtils.result(writer.write(DLMTestUtil.getLogRecordInstance(txId++))); } if (recover) { FutureUtils.result(writer.write(DLMTestUtil.getLogRecordInstance(txId++))); TimeUnit.MILLISECONDS.sleep(300); writer.abort(); LOG.debug("Recovering Segments"); BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true); FutureUtils.result(blplm.recoverIncompleteLogSegments()); FutureUtils.result(blplm.asyncClose()); LOG.debug("Recovered Segments"); } else { FutureUtils.result(writer.write(DLMTestUtil.getLogRecordInstance(txId++))); writer.closeAndComplete(); } TimeUnit.MILLISECONDS.sleep(300); } }
@Test(timeout = 60000) public void testReaderLockCloseInAcquireCallback() throws Exception { final String name = runtime.getMethodName(); DistributedLogManager dlm = createNewDLM(conf, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.closeAndComplete(); final CountDownLatch latch = new CountDownLatch(1); Future<AsyncLogReader> futureReader1 = dlm.getAsyncLogReaderWithLock(DLSN.InitialDLSN); futureReader1.flatMap(new ExceptionalFunction<AsyncLogReader, Future<Void>>() { @Override public Future<Void> applyE(AsyncLogReader reader) throws IOException { return reader.asyncClose().map(new AbstractFunction1<Void, Void>() { @Override public Void apply(Void result) { latch.countDown(); return null; } }); } }); latch.await(); dlm.close(); }
@Test(timeout = 60000) public void testAsyncWritePendingWritesAbortedWhenLedgerRollTriggerFails() throws Exception { String name = runtime.getMethodName(); DistributedLogConfiguration confLocal = new DistributedLogConfiguration(); confLocal.loadConf(testConf); confLocal.setOutputBufferSize(1024); confLocal.setMaxLogSegmentBytes(1024); confLocal.setLogSegmentRollingIntervalMinutes(0); DistributedLogManager dlm = createNewDLM(confLocal, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm.startAsyncLogSegmentNonPartitioned()); // Write one record larger than max seg size. Ledger doesn't roll until next write. int txid = 1; LogRecord record = DLMTestUtil.getLogRecordInstance(txid++, 2048); Future<DLSN> result = writer.write(record); DLSN dlsn = Await.result(result, Duration.fromSeconds(10)); assertEquals(1, dlsn.getLogSegmentSequenceNo()); record = DLMTestUtil.getLogRecordInstance(txid++, MAX_LOGRECORD_SIZE + 1); result = writer.write(record); validateFutureFailed(result, LogRecordTooLongException.class); record = DLMTestUtil.getLogRecordInstance(txid++, MAX_LOGRECORD_SIZE + 1); result = writer.write(record); validateFutureFailed(result, WriteException.class); record = DLMTestUtil.getLogRecordInstance(txid++, MAX_LOGRECORD_SIZE + 1); result = writer.write(record); validateFutureFailed(result, WriteException.class); writer.closeAndComplete(); dlm.close(); }
@Test(timeout = 60000) public void testReaderLockSharedDlmDoesNotConflict() throws Exception { String name = runtime.getMethodName(); DistributedLogManager dlm0 = createNewDLM(conf, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm0.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.write(DLMTestUtil.getLogRecordInstance(2L)); writer.closeAndComplete(); DistributedLogManager dlm1 = createNewDLM(conf, name); Future<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN); Future<AsyncLogReader> futureReader2 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN); // Both use the same client id, so there's no lock conflict. Not necessarily ideal, but how the // system currently works. Await.result(futureReader1); Await.result(futureReader2); dlm0.close(); dlm1.close(); }
@Test(timeout = 60000) public void testOutstandingWriteLimitBlockAllLimitWithDarkmode() throws Exception { DistributedLogConfiguration confLocal = new DistributedLogConfiguration(); confLocal.addConfiguration(testConf); confLocal.setOutputBufferSize(0); confLocal.setImmediateFlushEnabled(true); confLocal.setPerWriterOutstandingWriteLimit(0); confLocal.setOutstandingWriteLimitDarkmode(true); DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName()); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm.startAsyncLogSegmentNonPartitioned()); ArrayList<Future<DLSN>> results = new ArrayList<Future<DLSN>>(1000); for (int i = 0; i < 1000; i++) { results.add(writer.write(DLMTestUtil.getLogRecordInstance(1L))); } for (Future<DLSN> result : results) { Await.result(result); } writer.closeAndComplete(); dlm.close(); }
@Test(timeout = 60000) public void testReaderLockFutureCancelledWhileLocked() throws Exception { String name = runtime.getMethodName(); DistributedLogManager dlm0 = createNewDLM(conf, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm0.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.write(DLMTestUtil.getLogRecordInstance(2L)); writer.closeAndComplete(); DistributedLogManager dlm1 = createNewDLM(conf, name); Future<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN); // Must not throw or cancel or do anything bad, future already completed. Await.result(futureReader1); FutureUtils.cancel(futureReader1); AsyncLogReader reader1 = Await.result(futureReader1); Await.result(reader1.readNext()); dlm0.close(); dlm1.close(); }
@Test(timeout = 60000) public void testReaderLockDlmClosed() throws Exception { String name = runtime.getMethodName(); DistributedLogManager dlm0 = createNewDLM(conf, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm0.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.write(DLMTestUtil.getLogRecordInstance(2L)); writer.closeAndComplete(); DistributedLogManager dlm1 = createNewDLM(conf, name); Future<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN); Await.result(futureReader1); BKDistributedLogManager dlm2 = (BKDistributedLogManager) createNewDLM(conf, name); Future<AsyncLogReader> futureReader2 = dlm2.getAsyncLogReaderWithLock(DLSN.InitialDLSN); dlm2.close(); try { Await.result(futureReader2); fail("should have thrown exception!"); } catch (LockClosedException ex) { } catch (LockCancelledException ex) { } dlm0.close(); dlm1.close(); }
@Test(timeout = 60000) public void testReaderLockSessionExpires() throws Exception { String name = runtime.getMethodName(); DistributedLogManager dlm0 = createNewDLM(conf, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm0.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.write(DLMTestUtil.getLogRecordInstance(2L)); writer.closeAndComplete(); DistributedLogManager dlm1 = createNewDLM(conf, name); Future<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN); AsyncLogReader reader1 = Await.result(futureReader1); ZooKeeperClientUtils.expireSession(((BKDistributedLogManager)dlm1).getWriterZKC(), zkServers, 1000); // The result of expireSession is somewhat non-deterministic with this lock. // It may fail with LockingException or it may succesfully reacquire, so for // the moment rather than make it deterministic we accept either result. boolean success = false; try { Await.result(reader1.readNext()); success = true; } catch (LockingException ex) { } if (success) { Await.result(reader1.readNext()); } Utils.close(reader1); dlm0.close(); dlm1.close(); }
Await.result(writer.write(DLMTestUtil.getLogRecordInstance(0L)));
@Test(timeout = 60000) public void testReaderLockFutureCancelledWhileWaiting() throws Exception { String name = runtime.getMethodName(); DistributedLogManager dlm0 = createNewDLM(conf, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm0.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.write(DLMTestUtil.getLogRecordInstance(2L)); writer.closeAndComplete(); DistributedLogManager dlm1 = createNewDLM(conf, name); Future<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN); AsyncLogReader reader1 = Await.result(futureReader1); DistributedLogManager dlm2 = createNewDLM(conf, name); Future<AsyncLogReader> futureReader2 = dlm2.getAsyncLogReaderWithLock(DLSN.InitialDLSN); try { FutureUtils.cancel(futureReader2); Await.result(futureReader2); fail("Should fail getting log reader as it is cancelled"); } catch (LockClosedException ex) { } catch (LockCancelledException ex) { } catch (OwnershipAcquireFailedException oafe) { } futureReader2 = dlm2.getAsyncLogReaderWithLock(DLSN.InitialDLSN); Utils.close(reader1); Await.result(futureReader2); dlm0.close(); dlm1.close(); dlm2.close(); }
(BKAsyncLogWriter)(dlm.startAsyncLogSegmentNonPartitioned()); Await.result(writer.write(DLMTestUtil.getLogRecordInstance(1L))); writer.abort();
ArrayList<Future<DLSN>> results = new ArrayList<Future<DLSN>>(1000); for (int i = 0; i < 1000; i++) { results.add(writer.write(DLMTestUtil.getLogRecordInstance(1L)));
int i = 1; for (; i <= NUM_RECORDS; i++) { Await.result(writer.write(DLMTestUtil.getLogRecordInstance(i))); assertEquals("last tx id should become " + i, i, writer.getLastTxId()); Await.result(writer.write(DLMTestUtil.getLogRecordInstance(i))); fail("Should have thrown"); } catch (EndOfStreamException ex) {
@Test(timeout = 60000) public void testMarkEndOfStreamAtBeginningOfSegment() throws Exception { String name = runtime.getMethodName(); DistributedLogConfiguration confLocal = new DistributedLogConfiguration(); confLocal.addConfiguration(testConf); confLocal.setOutputBufferSize(0); confLocal.setImmediateFlushEnabled(true); confLocal.setPeriodicFlushFrequencyMilliSeconds(0); DistributedLogManager dlm = createNewDLM(confLocal, name); BKAsyncLogWriter writer = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned(); Await.result(writer.markEndOfStream()); try { Await.result(writer.write(DLMTestUtil.getLogRecordInstance(1))); fail("Should have thrown"); } catch (EndOfStreamException ex) { } BKAsyncLogReaderDLSN reader = (BKAsyncLogReaderDLSN) dlm.getAsyncLogReader(DLSN.InitialDLSN); try { LogRecord record = Await.result(reader.readNext()); fail("Should have thrown"); } catch (EndOfStreamException ex) { } }
@Test(timeout = 60000) public void testReaderLockIfLockPathDoesntExist() throws Exception { final String name = runtime.getMethodName(); DistributedLogManager dlm = createNewDLM(conf, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.closeAndComplete(); Future<AsyncLogReader> futureReader1 = dlm.getAsyncLogReaderWithLock(DLSN.InitialDLSN); BKAsyncLogReaderDLSN reader1 = (BKAsyncLogReaderDLSN) Await.result(futureReader1); LogRecordWithDLSN record = Await.result(reader1.readNext()); assertEquals(1L, record.getTransactionId()); assertEquals(0L, record.getSequenceId()); DLMTestUtil.verifyLogRecord(record); String readLockPath = reader1.bkLedgerManager.getReadLockPath(); Utils.close(reader1); // simulate a old stream created without readlock path writer.bkDistributedLogManager.getWriterZKC().get().delete(readLockPath, -1); Future<AsyncLogReader> futureReader2 = dlm.getAsyncLogReaderWithLock(DLSN.InitialDLSN); AsyncLogReader reader2 = Await.result(futureReader2); record = Await.result(reader2.readNext()); assertEquals(1L, record.getTransactionId()); assertEquals(0L, record.getSequenceId()); DLMTestUtil.verifyLogRecord(record); }