private BookKeeperClient getBookKeeperClient(com.twitter.distributedlog.DistributedLogManagerFactory factory) { DistributedLogNamespace namespace = factory.getNamespace(); assert(namespace instanceof BKDistributedLogNamespace); return ((BKDistributedLogNamespace) namespace).getReaderBKC(); }
@Override public DistributedLogManager openLog(String logName) throws InvalidStreamNameException, IOException { return openLog(logName, Optional.<DistributedLogConfiguration>absent(), Optional.<DynamicDistributedLogConfiguration>absent()); }
/** * Close the distributed log manager factory, freeing any resources it may hold. */ public void close() { namespace.close(); } }
.setEnableLedgerAllocatorPool(true) .setLedgerAllocatorPoolName("test"); BKDistributedLogNamespace namespace = BKDistributedLogNamespace.newBuilder().conf(conf).uri(uri).build(); namespace.createLog(streamName); MaxLogSegmentSequenceNo max1 = getMaxLogSegmentSequenceNo(namespace.getSharedWriterZKCForDL(), uri, streamName, conf); assertEquals(DistributedLogConstants.UNASSIGNED_LOGSEGMENT_SEQNO, max1.getSequenceNumber()); DistributedLogManager dlm = namespace.openLog(streamName); final int numSegments = 3; for (int i = 0; i < numSegments; i++) { out.closeAndComplete(); MaxLogSegmentSequenceNo max2 = getMaxLogSegmentSequenceNo(namespace.getSharedWriterZKCForDL(), uri, streamName, conf); assertEquals(3, max2.getSequenceNumber()); updateMaxLogSegmentSequenceNo(namespace.getSharedWriterZKCForDL(), uri, streamName, conf, new byte[0]); DistributedLogManager dlm1 = namespace.openLog(streamName); try { dlm1.startLogSegmentNonPartitioned(); updateMaxLogSegmentSequenceNo(namespace.getSharedWriterZKCForDL(), uri, streamName, conf, "invalid-max".getBytes(UTF_8)); DistributedLogManager dlm2 = namespace.openLog(streamName); try { dlm2.startLogSegmentNonPartitioned(); namespace.close();
BKDistributedLogNamespace namespace = BKDistributedLogNamespace.newBuilder() .conf(conf).uri(uri).build(); namespace.openLog(".test1"); fail("Should fail to create invalid stream .test"); } catch (InvalidStreamNameException isne) { DistributedLogManager dlm = namespace.openLog("test1"); LogWriter writer = dlm.startLogSegmentNonPartitioned(); writer.write(DLMTestUtil.getLogRecordInstance(1)); namespace.openLog(".test2"); fail("Should fail to create invalid stream .test2"); } catch (InvalidStreamNameException isne) { namespace.openLog("/test2"); fail("should fail to create invalid stream /test2"); } catch (InvalidStreamNameException isne) { namespace.openLog(streamName); fail("should fail to create invalid stream " + streamName); } catch (InvalidStreamNameException isne) { namespace.openLog(streamName); fail("should fail to create invalid stream " + streamName); } catch (InvalidStreamNameException isne) { namespace.openLog("test_2-3"); LogWriter newWriter = newDLM.startLogSegmentNonPartitioned();
BKDistributedLogNamespace namespace = BKDistributedLogNamespace.newBuilder() .conf(conf).uri(uri).build(); assertTrue(namespace.logExists(name)); assertFalse(namespace.logExists("non-existent-log")); URI nonExistentUri = createDLMURI("/" + "non-existent-ns"); DistributedLogNamespace nonExistentNS = DistributedLogNamespaceBuilder.newBuilder() Iterator<String> logIter = namespace.getLogs(); while(logIter.hasNext()) { String log = logIter.next(); for(Map.Entry<String, byte[]> logEntry: namespace.enumerateLogsWithMetadataInNamespace().entrySet()) { assertEquals(name, new String(logEntry.getValue()));
URI uri = createDLMURI("/" + name); ensureURICreated(uri); BKDistributedLogNamespace namespace = BKDistributedLogNamespace.newBuilder() .conf(conf).uri(uri).build(); DistributedLogManager[] dlms = new DistributedLogManager[count]; for (int s = 0; s < count; s++) { if (shareBK) { dlms[s] = namespace.createDistributedLogManager(name + String.format("%d", s), DistributedLogManagerFactory.ClientSharingOption.SharedClients); } else { dlms[s] = namespace.createDistributedLogManager(name + String.format("%d", s), DistributedLogManagerFactory.ClientSharingOption.SharedZKClientPerStreamBKClient); dlms[0] = namespace.createDistributedLogManager(name + String.format("%d", 0), DistributedLogManagerFactory.ClientSharingOption.SharedClients); } else { dlms[0] = namespace.createDistributedLogManager(name + String.format("%d", 0), DistributedLogManagerFactory.ClientSharingOption.SharedZKClientPerStreamBKClient); namespace.close();
/** {@inheritDoc} */ @Override public void setConf(Configuration configuration) { this.conf = configuration; dlConf = new DistributedLogConfiguration(); dlUri = URI.create(configuration.get(DL_URI, "")); streamName = configuration.get(DL_STREAM, ""); try { namespace = BKDistributedLogNamespace.newBuilder() .conf(dlConf) .uri(dlUri) .build(); dlm = namespace.openLog(streamName); } catch (IOException e) { throw new RuntimeException(e); } }
static MetadataAccessor createNewMetadataAccessor(DistributedLogConfiguration conf, String name, URI uri) throws Exception { // TODO: Metadata Accessor seems to be a legacy object which only used by kestrel // (we might consider deprecating this) BKDistributedLogNamespace namespace = BKDistributedLogNamespace.newBuilder() .conf(conf).uri(uri).build(); return namespace.createMetadataAccessor(name); }
public void testClientSharingOptions() throws Exception { URI uri = createDLMURI("/clientSharingOptions"); BKDistributedLogNamespace namespace = BKDistributedLogNamespace.newBuilder() .conf(conf).uri(uri).build(); BKDistributedLogManager bkdlm1 = (BKDistributedLogManager)namespace.createDistributedLogManager("perstream1", DistributedLogManagerFactory.ClientSharingOption.PerStreamClients); BKDistributedLogManager bkdlm2 = (BKDistributedLogManager)namespace.createDistributedLogManager("perstream2", DistributedLogManagerFactory.ClientSharingOption.PerStreamClients); BKDistributedLogManager bkdlm1 = (BKDistributedLogManager)namespace.createDistributedLogManager("sharedZK1", DistributedLogManagerFactory.ClientSharingOption.SharedZKClientPerStreamBKClient); BKDistributedLogManager bkdlm2 = (BKDistributedLogManager)namespace.createDistributedLogManager("sharedZK2", DistributedLogManagerFactory.ClientSharingOption.SharedZKClientPerStreamBKClient); BKDistributedLogManager bkdlm1 = (BKDistributedLogManager)namespace.createDistributedLogManager("sharedBoth1", DistributedLogManagerFactory.ClientSharingOption.SharedClients); BKDistributedLogManager bkdlm2 = (BKDistributedLogManager)namespace.createDistributedLogManager("sharedBoth2", DistributedLogManagerFactory.ClientSharingOption.SharedClients);
/** * Create a DistributedLogManager for <i>nameOfLogStream</i>, with default shared clients. * * @param nameOfLogStream * name of log stream * @return distributedlog manager * @throws com.twitter.distributedlog.exceptions.InvalidStreamNameException if stream name is invalid * @throws IOException */ public DistributedLogManager createDistributedLogManagerWithSharedClients(String nameOfLogStream) throws InvalidStreamNameException, IOException { return createDistributedLogManager(nameOfLogStream, ClientSharingOption.SharedClients); }
return BKDistributedLogNamespace.newBuilder() .conf(_conf) .uri(bkUri)
.uri(uri) .build(); DistributedLogManager dlm = namespace.openLog("test-stream"); FailpointUtils.setFailpoint(FailpointUtils.FailPointName.FP_StartLogSegmentOnAssignLogSegmentSequenceNumber, FailpointUtils.FailPointActions.FailPointAction_Throw); LedgerAllocator allocator = namespace.getLedgerAllocator(); assertTrue(allocator instanceof LedgerAllocatorPool); LedgerAllocatorPool allocatorPool = (LedgerAllocatorPool) allocator;
private ZooKeeperClient getZooKeeperClient(com.twitter.distributedlog.DistributedLogManagerFactory factory) { DistributedLogNamespace namespace = factory.getNamespace(); assert(namespace instanceof BKDistributedLogNamespace); return ((BKDistributedLogNamespace) namespace).getSharedWriterZKCForDL(); }
/** * This method is to initialize the metadata for a unpartitioned stream with name <i>streamName</i>. * * TODO: after 0.2 is upgraded to 0.3, remove this. * * @param streamName * stream name. * @throws IOException */ public void createUnpartitionedStream(final String streamName) throws IOException { namespace.createLog(streamName); }
public Map<String, byte[]> enumerateLogsWithMetadataInNamespace() throws IOException, IllegalArgumentException { return namespace.enumerateLogsWithMetadataInNamespace(); }
public MetadataAccessor createMetadataAccessor(String nameOfMetadataNode) throws InvalidStreamNameException, IOException { return namespace.createMetadataAccessor(nameOfMetadataNode); }
public Collection<String> enumerateAllLogsInNamespace() throws IOException, IllegalArgumentException { if (bkdlConfig.isFederatedNamespace()) { throw new UnsupportedOperationException("Use DistributedLogNamespace methods for federated namespace"); } return Sets.newHashSet(getLogs()); }
/** * Create Log Segment for an pre-create stream. No max ledger sequence number recorded. */ @Test(timeout = 60000) public void testCreateLogSegmentOnPrecreatedStream() throws Exception { URI uri = createURI(); String streamName = testName.getMethodName(); DistributedLogConfiguration conf = new DistributedLogConfiguration() .setLockTimeout(99999) .setOutputBufferSize(0) .setImmediateFlushEnabled(true) .setEnableLedgerAllocatorPool(true) .setLedgerAllocatorPoolName("test"); BKDistributedLogNamespace namespace = BKDistributedLogNamespace.newBuilder().conf(conf).uri(uri).build(); namespace.createLog(streamName); MaxLogSegmentSequenceNo max1 = getMaxLogSegmentSequenceNo(namespace.getSharedWriterZKCForDL(), uri, streamName, conf); assertEquals(DistributedLogConstants.UNASSIGNED_LOGSEGMENT_SEQNO, max1.getSequenceNumber()); DistributedLogManager dlm = namespace.openLog(streamName); final int numSegments = 3; for (int i = 0; i < numSegments; i++) { BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned(); out.write(DLMTestUtil.getLogRecordInstance(i)); out.closeAndComplete(); } MaxLogSegmentSequenceNo max2 = getMaxLogSegmentSequenceNo(namespace.getSharedWriterZKCForDL(), uri, streamName, conf); assertEquals(3, max2.getSequenceNumber()); dlm.close(); namespace.close(); }
Optional<DynamicDistributedLogConfiguration> dynamicStreamConfiguration) throws InvalidStreamNameException, IOException { return namespace.createDistributedLogManager( nameOfLogStream, clientSharingOption,