/** * Start a new log segment in a BookKeeper ledger. * First ensure that we have the write lock for this journal. * Then create a ledger and stream based on that ledger. * The ledger id is written to the inprogress znode, so that in the * case of a crash, a recovery process can find the ledger we were writing * to when we crashed. * * @param txId First transaction id to be written to the stream * @return * @throws IOException */ public BKLogSegmentWriter startLogSegment(long txId) throws IOException { return startLogSegment(txId, false, false); }
@Test(timeout = 60000) public void testMaxLogRecSize() throws Exception { DLMTestUtil.BKLogPartitionWriteHandlerAndClients bkdlmAndClients = createNewBKDLM(conf, "distrlog-maxlogRecSize"); long txid = 1; BKLogSegmentWriter out = bkdlmAndClients.getWriteHandler().startLogSegment(1); boolean exceptionEncountered = false; try { LogRecord op = new LogRecord(txid, DLMTestUtil.repeatString( DLMTestUtil.repeatString("abcdefgh", 256), 512).getBytes()); out.write(op); } catch (LogRecordTooLongException exc) { exceptionEncountered = true; } finally { FutureUtils.result(out.asyncClose()); } bkdlmAndClients.close(); assert(exceptionEncountered); }
@Test(timeout = 60000) public void testMaxTransmissionSize() throws Exception { DistributedLogConfiguration confLocal = new DistributedLogConfiguration(); confLocal.loadConf(conf); confLocal.setOutputBufferSize(1024 * 1024); DLMTestUtil.BKLogPartitionWriteHandlerAndClients bkdlmAndClients = createNewBKDLM(confLocal, "distrlog-transmissionSize"); long txid = 1; BKLogSegmentWriter out = bkdlmAndClients.getWriteHandler().startLogSegment(1); boolean exceptionEncountered = false; byte[] largePayload = DLMTestUtil.repeatString(DLMTestUtil.repeatString("abcdefgh", 256), 256).getBytes(); try { while (txid < 3) { LogRecord op = new LogRecord(txid, largePayload); out.write(op); txid++; } } catch (LogRecordTooLongException exc) { exceptionEncountered = true; } finally { FutureUtils.result(out.asyncClose()); } bkdlmAndClients.close(); assert(!exceptionEncountered); }
@Test(timeout = 60000) public void testRecoveryEmptyLedger() throws Exception { DLMTestUtil.BKLogPartitionWriteHandlerAndClients bkdlmAndClients = createNewBKDLM(conf, "distrlog-recovery-empty-ledger"); BKLogSegmentWriter out = bkdlmAndClients.getWriteHandler().startLogSegment(1); long txid = 1; for (long i = 1; i <= 100; i++) { LogRecord op = DLMTestUtil.getLogRecordInstance(txid++); out.write(op); if ((i % 10) == 0) { FutureUtils.result(out.flushAndCommit()); } } FutureUtils.result(out.flushAndCommit()); FutureUtils.result(out.asyncClose()); bkdlmAndClients.getWriteHandler().completeAndCloseLogSegment(out.getLogSegmentSequenceNumber(), out.getLogSegmentId(), 1, 100, 100); assertNotNull(zkc.exists(bkdlmAndClients.getWriteHandler().completedLedgerZNode(1, 100, out.getLogSegmentSequenceNumber()), false)); BKLogSegmentWriter outEmpty = bkdlmAndClients.getWriteHandler().startLogSegment(101); Abortables.abort(outEmpty, false); assertNull(zkc.exists(bkdlmAndClients.getWriteHandler().completedLedgerZNode(101, 101, outEmpty.getLogSegmentSequenceNumber()), false)); assertNotNull(zkc.exists(bkdlmAndClients.getWriteHandler().inprogressZNode(outEmpty.getLogSegmentId(), 101, outEmpty.getLogSegmentSequenceNumber()), false)); FutureUtils.result(bkdlmAndClients.getWriteHandler().recoverIncompleteLogSegments()); assertNull(zkc.exists(bkdlmAndClients.getWriteHandler().inprogressZNode(outEmpty.getLogSegmentId(), outEmpty.getLogSegmentSequenceNumber(), 101), false)); assertNotNull(zkc.exists(bkdlmAndClients.getWriteHandler().completedLedgerZNode(101, 101, outEmpty.getLogSegmentSequenceNumber()), false)); }
private void prepareLogSegments(String name, int numSegments, int numEntriesPerSegment) throws Exception { DLMTestUtil.BKLogPartitionWriteHandlerAndClients bkdlmAndClients = createNewBKDLM(conf, name); long txid = 1; for (int sid = 0; sid < numSegments; ++sid) { BKLogSegmentWriter out = bkdlmAndClients.getWriteHandler().startLogSegment(txid); for (int eid = 0; eid < numEntriesPerSegment; ++eid) { LogRecord record = DLMTestUtil.getLargeLogRecordInstance(txid); out.write(record); ++txid; } FutureUtils.result(out.asyncClose()); bkdlmAndClients.getWriteHandler().completeAndCloseLogSegment( out.getLogSegmentSequenceNumber(), out.getLogSegmentId(), 1 + sid * numEntriesPerSegment, (sid + 1) * numEntriesPerSegment, numEntriesPerSegment); } bkdlmAndClients.close(); }
@Test(timeout = 60000) public void testSimpleRecovery() throws Exception { DLMTestUtil.BKLogPartitionWriteHandlerAndClients bkdlmAndClients = createNewBKDLM(conf, "distrlog-simplerecovery"); BKLogSegmentWriter out = bkdlmAndClients.getWriteHandler().startLogSegment(1); long txid = 1; for (long i = 1; i <= 100; i++) { LogRecord op = DLMTestUtil.getLogRecordInstance(txid++); out.write(op); if ((i % 10) == 0) { FutureUtils.result(out.flushAndCommit()); } } FutureUtils.result(out.flushAndCommit()); Abortables.abort(out, false); FutureUtils.result(out.asyncClose()); assertNull(zkc.exists(bkdlmAndClients.getWriteHandler().completedLedgerZNode(1, 100, out.getLogSegmentSequenceNumber()), false)); assertNotNull(zkc.exists(bkdlmAndClients.getWriteHandler().inprogressZNode(out.getLogSegmentId(), 1, out.getLogSegmentSequenceNumber()), false)); FutureUtils.result(bkdlmAndClients.getWriteHandler().recoverIncompleteLogSegments()); assertNotNull(zkc.exists(bkdlmAndClients.getWriteHandler().completedLedgerZNode(1, 100, out.getLogSegmentSequenceNumber()), false)); assertNull(zkc.exists(bkdlmAndClients.getWriteHandler().inprogressZNode(out.getLogSegmentId(), 1, out.getLogSegmentSequenceNumber()), false)); }
long txid = 1; DLMTestUtil.BKLogPartitionWriteHandlerAndClients bkdlmAndClients = createNewBKDLM(conf, "distrlog-allbookiefailure"); BKLogSegmentWriter out = bkdlmAndClients.getWriteHandler().startLogSegment(txid); out = bkdlmAndClients.getWriteHandler().startLogSegment(txid); for (long i = 1; i <= 3; i++) { LogRecord op = DLMTestUtil.getLogRecordInstance(txid++);
long txid = 1; DLMTestUtil.BKLogPartitionWriteHandlerAndClients bkdlmAndClients = createNewBKDLM(conf, "distrlog-onebookiefailure"); BKLogSegmentWriter out = bkdlmAndClients.getWriteHandler().startLogSegment(txid); for (long i = 1; i <= 3; i++) { LogRecord op = DLMTestUtil.getLogRecordInstance(txid++);
zkc.exists(blplm.completedLedgerZNode(start, txid - 1, perStreamLogWriter.getLogSegmentSequenceNumber()), false)); BKLogSegmentWriter writer = blplm.startLogSegment(txid - 1); blplm.completeAndCloseLogSegment(writer.getLogSegmentSequenceNumber(), writer.getLogSegmentId(), txid - 1, txid - 1, 0);
zkc.exists(blplm.completedLedgerZNode(start, txid - 1, writer.getLogSegmentSequenceNumber()), false)); BKLogSegmentWriter perStreamLogWriter = blplm.startLogSegment(txid - 1); blplm.completeAndCloseLogSegment(perStreamLogWriter.getLogSegmentSequenceNumber(), perStreamLogWriter.getLogSegmentId(), txid - 1, txid - 1, 0);