static List<LogRecord> getLogRecordInstanceList(long firstTxId, int count, int size) { List<LogRecord> logrecs = new ArrayList<LogRecord>(count); for (long i = 0; i < count; i++) { logrecs.add(getLogRecordInstance(firstTxId + i, size)); } return logrecs; }
@Override public void run() { long txid = nextTxId.getAndIncrement(); LogRecord record = DLMTestUtil.getLogRecordInstance(txid); try { out.write(record); } catch (IOException e) { // ignore the ioe } } }, 0, 400, TimeUnit.MILLISECONDS);
@Override public void run() { try { long txid = 2; DLSN dlsn = DLSN.InvalidDLSN; while (running.get()) { limiter.acquire(); long curTxId = txid++; dlsn = FutureUtils.result(writer.write(DLMTestUtil.getLogRecordInstance(curTxId))); writeCount.incrementAndGet(); if (curTxId % 1000 == 0) { LOG.info("writer write {}", curTxId); } } LOG.info("Completed writing record at {}", dlsn); Utils.close(writer); } catch (DLInterruptedException die) { Thread.currentThread().interrupt(); } catch (IOException e) { } } };
public static void generateCompletedLogSegments(DistributedLogManager manager, DistributedLogConfiguration conf, long numCompletedSegments, long segmentSize) throws Exception { BKDistributedLogManager dlm = (BKDistributedLogManager) manager; long txid = 1L; for (long i = 0; i < numCompletedSegments; i++) { BKSyncLogWriter writer = dlm.startLogSegmentNonPartitioned(); for (long j = 1; j <= segmentSize; j++) { writer.write(DLMTestUtil.getLogRecordInstance(txid++)); } writer.closeAndComplete(); } }
@Test(timeout = 60000) public void testReaderLockCloseInAcquireCallback() throws Exception { final String name = runtime.getMethodName(); DistributedLogManager dlm = createNewDLM(conf, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.closeAndComplete(); final CountDownLatch latch = new CountDownLatch(1); Future<AsyncLogReader> futureReader1 = dlm.getAsyncLogReaderWithLock(DLSN.InitialDLSN); futureReader1.flatMap(new ExceptionalFunction<AsyncLogReader, Future<Void>>() { @Override public Future<Void> applyE(AsyncLogReader reader) throws IOException { return reader.asyncClose().map(new AbstractFunction1<Void, Void>() { @Override public Void apply(Void result) { latch.countDown(); return null; } }); } }); latch.await(); dlm.close(); }
@Test(timeout = 60000) public void testTwoWritersOnLockDisabled() throws Exception { DistributedLogConfiguration confLocal = new DistributedLogConfiguration(); confLocal.addConfiguration(conf); confLocal.setOutputBufferSize(0); confLocal.setWriteLockEnabled(false); String name = "distrlog-two-writers-lock-disabled"; DistributedLogManager manager = createNewDLM(confLocal, name); AsyncLogWriter writer1 = FutureUtils.result(manager.openAsyncLogWriter()); FutureUtils.result(writer1.write(DLMTestUtil.getLogRecordInstance(1L))); AsyncLogWriter writer2 = FutureUtils.result(manager.openAsyncLogWriter()); FutureUtils.result(writer2.write(DLMTestUtil.getLogRecordInstance(2L))); // write a record to writer 1 again try { FutureUtils.result(writer1.write(DLMTestUtil.getLogRecordInstance(3L))); fail("Should fail writing record to writer 1 again as writer 2 took over the ownership"); } catch (BKTransmitException bkte) { assertEquals(BKException.Code.LedgerFencedException, bkte.getBKResultCode()); } }
@Test(timeout = 60000) public void testReaderLockSharedDlmDoesNotConflict() throws Exception { String name = runtime.getMethodName(); DistributedLogManager dlm0 = createNewDLM(conf, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm0.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.write(DLMTestUtil.getLogRecordInstance(2L)); writer.closeAndComplete(); DistributedLogManager dlm1 = createNewDLM(conf, name); Future<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN); Future<AsyncLogReader> futureReader2 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN); // Both use the same client id, so there's no lock conflict. Not necessarily ideal, but how the // system currently works. Await.result(futureReader1); Await.result(futureReader2); dlm0.close(); dlm1.close(); }
@Test(timeout = 60000) public void testSimpleWrite() throws Exception { BKDistributedLogManager dlm = createNewDLM(conf, "distrlog-simplewrite"); BKSyncLogWriter out = dlm.startLogSegmentNonPartitioned(); for (long i = 1; i <= 100; i++) { LogRecord op = DLMTestUtil.getLogRecordInstance(i); out.write(op); } BKLogSegmentWriter perStreamLogWriter = out.getCachedLogWriter(); out.closeAndComplete(); BKLogWriteHandler blplm = dlm.createWriteHandler(true); assertNotNull(zkc.exists(blplm.completedLedgerZNode(1, 100, perStreamLogWriter.getLogSegmentSequenceNumber()), false)); FutureUtils.result(blplm.asyncClose()); }
@Test(timeout = 60000) public void testOutstandingWriteLimitBlockAllLimitWithDarkmode() throws Exception { DistributedLogConfiguration confLocal = new DistributedLogConfiguration(); confLocal.addConfiguration(testConf); confLocal.setOutputBufferSize(0); confLocal.setImmediateFlushEnabled(true); confLocal.setPerWriterOutstandingWriteLimit(0); confLocal.setOutstandingWriteLimitDarkmode(true); DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName()); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm.startAsyncLogSegmentNonPartitioned()); ArrayList<Future<DLSN>> results = new ArrayList<Future<DLSN>>(1000); for (int i = 0; i < 1000; i++) { results.add(writer.write(DLMTestUtil.getLogRecordInstance(1L))); } for (Future<DLSN> result : results) { Await.result(result); } writer.closeAndComplete(); dlm.close(); }
@Test(timeout = 60000) public void testNumberOfTransactions() throws Exception { String name = "distrlog-txncount"; DistributedLogManager dlm = createNewDLM(conf, name); BKSyncLogWriter out = (BKSyncLogWriter)dlm.startLogSegmentNonPartitioned(); for (long i = 1; i <= 100; i++) { LogRecord op = DLMTestUtil.getLogRecordInstance(i); out.write(op); } out.closeAndComplete(); long numTrans = DLMTestUtil.getNumberofLogRecords(createNewDLM(conf, name), 1); assertEquals(100, numTrans); dlm.close(); }
@Test(timeout = 60000) public void testSimpleRead() throws Exception { String name = "distrlog-simpleread"; DistributedLogManager dlm = createNewDLM(conf, name); final long numTransactions = 10000; BKSyncLogWriter out = (BKSyncLogWriter)dlm.startLogSegmentNonPartitioned(); for (long i = 1; i <= numTransactions; i++) { LogRecord op = DLMTestUtil.getLogRecordInstance(i); out.write(op); } out.closeAndComplete(); assertEquals(numTransactions, DLMTestUtil.getNumberofLogRecords(createNewDLM(conf, name), 1)); dlm.close(); }
@Test(timeout = 60000) public void testReaderLockFutureCancelledWhileLocked() throws Exception { String name = runtime.getMethodName(); DistributedLogManager dlm0 = createNewDLM(conf, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm0.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.write(DLMTestUtil.getLogRecordInstance(2L)); writer.closeAndComplete(); DistributedLogManager dlm1 = createNewDLM(conf, name); Future<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN); // Must not throw or cancel or do anything bad, future already completed. Await.result(futureReader1); FutureUtils.cancel(futureReader1); AsyncLogReader reader1 = Await.result(futureReader1); Await.result(reader1.readNext()); dlm0.close(); dlm1.close(); }
@Test(timeout = 60000) public void testReaderLockDlmClosed() throws Exception { String name = runtime.getMethodName(); DistributedLogManager dlm0 = createNewDLM(conf, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm0.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.write(DLMTestUtil.getLogRecordInstance(2L)); writer.closeAndComplete(); DistributedLogManager dlm1 = createNewDLM(conf, name); Future<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN); Await.result(futureReader1); BKDistributedLogManager dlm2 = (BKDistributedLogManager) createNewDLM(conf, name); Future<AsyncLogReader> futureReader2 = dlm2.getAsyncLogReaderWithLock(DLSN.InitialDLSN); dlm2.close(); try { Await.result(futureReader2); fail("should have thrown exception!"); } catch (LockClosedException ex) { } catch (LockCancelledException ex) { } dlm0.close(); dlm1.close(); }
private long writeAndMarkEndOfStream(DistributedLogManager dlm, long txid) throws Exception { for (long i = 0; i < 3; i++) { long start = txid; BKSyncLogWriter writer = (BKSyncLogWriter)dlm.startLogSegmentNonPartitioned(); for (long j = 1; j <= DEFAULT_SEGMENT_SIZE; j++) { writer.write(DLMTestUtil.getLogRecordInstance(txid++)); } BKLogSegmentWriter perStreamLogWriter = writer.getCachedLogWriter(); if (i < 2) { writer.closeAndComplete(); BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true); assertNotNull(zkc.exists(blplm.completedLedgerZNode(start, txid - 1, perStreamLogWriter.getLogSegmentSequenceNumber()), false)); FutureUtils.result(blplm.asyncClose()); } else { writer.markEndOfStream(); BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true); assertNotNull(zkc.exists(blplm.completedLedgerZNode(start, DistributedLogConstants.MAX_TXID, perStreamLogWriter.getLogSegmentSequenceNumber()), false)); FutureUtils.result(blplm.asyncClose()); } } return txid; }
@Test(timeout = 60000) public void testSimpleRecovery() throws Exception { DLMTestUtil.BKLogPartitionWriteHandlerAndClients bkdlmAndClients = createNewBKDLM(conf, "distrlog-simplerecovery"); BKLogSegmentWriter out = bkdlmAndClients.getWriteHandler().startLogSegment(1); long txid = 1; for (long i = 1; i <= 100; i++) { LogRecord op = DLMTestUtil.getLogRecordInstance(txid++); out.write(op); if ((i % 10) == 0) { FutureUtils.result(out.flushAndCommit()); } } FutureUtils.result(out.flushAndCommit()); Abortables.abort(out, false); FutureUtils.result(out.asyncClose()); assertNull(zkc.exists(bkdlmAndClients.getWriteHandler().completedLedgerZNode(1, 100, out.getLogSegmentSequenceNumber()), false)); assertNotNull(zkc.exists(bkdlmAndClients.getWriteHandler().inprogressZNode(out.getLogSegmentId(), 1, out.getLogSegmentSequenceNumber()), false)); FutureUtils.result(bkdlmAndClients.getWriteHandler().recoverIncompleteLogSegments()); assertNotNull(zkc.exists(bkdlmAndClients.getWriteHandler().completedLedgerZNode(1, 100, out.getLogSegmentSequenceNumber()), false)); assertNull(zkc.exists(bkdlmAndClients.getWriteHandler().inprogressZNode(out.getLogSegmentId(), 1, out.getLogSegmentSequenceNumber()), false)); }
private void initDlogMeta(String dlNamespace, String un, String streamName) throws Exception { URI uri = createDLMURI(dlNamespace); DistributedLogConfiguration newConf = new DistributedLogConfiguration(); newConf.addConfiguration(conf); newConf.setCreateStreamIfNotExists(true); newConf.setZkAclId(un); DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder() .conf(newConf).uri(uri).build(); DistributedLogManager dlm = namespace.openLog(streamName); LogWriter writer = dlm.startLogSegmentNonPartitioned(); for (int i = 0; i < 10; i++) { writer.write(DLMTestUtil.getLogRecordInstance(1L)); } writer.close(); dlm.close(); namespace.close(); }
@Test(timeout = 60000) public void testGetLastTxId() throws Exception { String name = runtime.getMethodName(); DistributedLogConfiguration confLocal = new DistributedLogConfiguration(); confLocal.addConfiguration(testConf); confLocal.setOutputBufferSize(0); confLocal.setImmediateFlushEnabled(true); DistributedLogManager dlm = createNewDLM(confLocal, name); AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned(); int numRecords = 10; for (int i = 0; i < numRecords; i++) { Await.result(writer.write(DLMTestUtil.getLogRecordInstance(i))); assertEquals("last tx id should become " + i, i, writer.getLastTxId()); } // open a writer to recover the inprogress log segment AsyncLogWriter recoverWriter = dlm.startAsyncLogSegmentNonPartitioned(); assertEquals("recovered last tx id should be " + (numRecords - 1), numRecords - 1, recoverWriter.getLastTxId()); }
/** * Non durable write should fail if writer is marked as end of stream. * * @throws Exception */ @Test(timeout = 60000) public void testNondurableWriteAfterEndOfStream() throws Exception { DistributedLogConfiguration confLocal = newLocalConf(); confLocal.setImmediateFlushEnabled(false); confLocal.setOutputBufferSize(Integer.MAX_VALUE); confLocal.setPeriodicFlushFrequencyMilliSeconds(0); confLocal.setDurableWriteEnabled(false); ZKDistributedLock lock = createLock("/test/lock-" + runtime.getMethodName(), zkc, true); BKLogSegmentWriter writer = createLogSegmentWriter(confLocal, 0L, -1L, lock); FutureUtils.result(writer.markEndOfStream()); try { Await.result(writer.asyncWrite(DLMTestUtil.getLogRecordInstance(1))); fail("Should fail the write if the writer is marked as end of stream"); } catch (EndOfStreamException we) { // expected } closeWriterAndLock(writer, lock); }
/** * Non durable write should fail if writer is marked as end of stream. * * @throws Exception */ @Test(timeout = 60000) public void testNondurableWrite() throws Exception { DistributedLogConfiguration confLocal = newLocalConf(); confLocal.setImmediateFlushEnabled(false); confLocal.setOutputBufferSize(Integer.MAX_VALUE); confLocal.setPeriodicFlushFrequencyMilliSeconds(0); confLocal.setDurableWriteEnabled(false); ZKDistributedLock lock = createLock("/test/lock-" + runtime.getMethodName(), zkc, true); BKLogSegmentWriter writer = createLogSegmentWriter(confLocal, 0L, -1L, lock); assertEquals(DLSN.InvalidDLSN, Await.result(writer.asyncWrite(DLMTestUtil.getLogRecordInstance(2)))); assertEquals(-1L, ((BKLogSegmentEntryWriter) writer.getEntryWriter()) .getLedgerHandle().getLastAddPushed()); closeWriterAndLock(writer, lock); } }
@Test(timeout = 60000) public void testReaderLockIfLockPathDoesntExist() throws Exception { final String name = runtime.getMethodName(); DistributedLogManager dlm = createNewDLM(conf, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.closeAndComplete(); Future<AsyncLogReader> futureReader1 = dlm.getAsyncLogReaderWithLock(DLSN.InitialDLSN); BKAsyncLogReaderDLSN reader1 = (BKAsyncLogReaderDLSN) Await.result(futureReader1); LogRecordWithDLSN record = Await.result(reader1.readNext()); assertEquals(1L, record.getTransactionId()); assertEquals(0L, record.getSequenceId()); DLMTestUtil.verifyLogRecord(record); String readLockPath = reader1.bkLedgerManager.getReadLockPath(); Utils.close(reader1); // simulate a old stream created without readlock path writer.bkDistributedLogManager.getWriterZKC().get().delete(readLockPath, -1); Future<AsyncLogReader> futureReader2 = dlm.getAsyncLogReaderWithLock(DLSN.InitialDLSN); AsyncLogReader reader2 = Await.result(futureReader2); record = Await.result(reader2.readNext()); assertEquals(1L, record.getTransactionId()); assertEquals(0L, record.getSequenceId()); DLMTestUtil.verifyLogRecord(record); }