/** * Initialize the 'clients' member with the configured number of * clients. */ private void initClients() throws DBException { synchronized (KuduYCSBClient.class) { if (!clients.isEmpty()) { return; } Properties prop = getProperties(); String masterAddresses = prop.getProperty(MASTER_ADDRESSES_OPT, "localhost:7051"); LOG.debug("Connecting to the masters at {}", masterAddresses); int numClients = getIntFromProp(prop, NUM_CLIENTS_OPT, DEFAULT_NUM_CLIENTS); for (int i = 0; i < numClients; i++) { clients.add(new KuduClient.KuduClientBuilder(masterAddresses) .defaultSocketReadTimeoutMs(DEFAULT_SLEEP) .defaultOperationTimeoutMs(DEFAULT_SLEEP) .defaultAdminOperationTimeoutMs(DEFAULT_SLEEP) .build()); } } }
@Singleton @Provides KuduClientSession createKuduClientSession( KuduConnectorId connectorId, KuduClientConfig config) { requireNonNull(config, "config is null"); KuduClient.KuduClientBuilder builder = new KuduClient.KuduClientBuilder(config.getMasterAddresses()); builder.defaultAdminOperationTimeoutMs(config.getDefaultAdminOperationTimeout().toMillis()); builder.defaultOperationTimeoutMs(config.getDefaultOperationTimeout().toMillis()); builder.defaultSocketReadTimeoutMs(config.getDefaultSocketReadTimeout().toMillis()); if (config.isDisableStatistics()) { builder.disableStatistics(); } KuduClient client = builder.build(); SchemaEmulation strategy; if (config.isSchemaEmulationEnabled()) { strategy = new SchemaEmulationByTableNameConvention(config.getSchemaEmulationPrefix()); } else { strategy = new NoSchemaEmulation(); } return new KuduClientSession(connectorId, client, strategy); } }
public static void createKuduTable(String tableName, int tablets, int replicas, int rows) throws Exception { try (KuduClient client = new KuduClient.KuduClientBuilder(KUDU_MASTER).build()) {
/** * Test that the client properly falls back to the old GetMasterRegistration * RPC when connecting to a master which does not support the new * ConnectToMaster RPC. */ @Test(timeout=60000) public void testFallbackConnectRpc() throws Exception { MiniKuduCluster cluster = new MiniKuduCluster.MiniKuduClusterBuilder() .addMasterServerFlag("--master_support_connect_to_master_rpc=0") .numMasterServers(1) .numTabletServers(0) .build(); KuduClient c = null; try { c = new KuduClient.KuduClientBuilder(cluster.getMasterAddressesAsString()) .build(); // Call some method which uses the master. This forces us to connect // and verifies that the fallback works. c.listTabletServers(); } finally { if (c != null) { c.close(); } cluster.shutdown(); } }
@Singleton @Provides KuduClientSession createKuduClientSession( KuduConnectorId connectorId, KuduClientConfig config) { requireNonNull(config, "config is null"); KuduClient.KuduClientBuilder builder = new KuduClient.KuduClientBuilder(config.getMasterAddresses()); builder.defaultAdminOperationTimeoutMs(config.getDefaultAdminOperationTimeout().toMillis()); builder.defaultOperationTimeoutMs(config.getDefaultOperationTimeout().toMillis()); builder.defaultSocketReadTimeoutMs(config.getDefaultSocketReadTimeout().toMillis()); if (config.isDisableStatistics()) { builder.disableStatistics(); } KuduClient client = builder.build(); SchemaEmulation strategy; if (config.isSchemaEmulationEnabled()) { strategy = new SchemaEmulationByTableNameConvention(config.getSchemaEmulationPrefix()); } else { strategy = new NoSchemaEmulation(); } return new KuduClientSession(connectorId, client, strategy); } }
private void startCluster(Set<Option> opts) throws IOException { MiniKuduClusterBuilder mcb = new MiniKuduClusterBuilder(); mcb.enableKerberos(); if (opts.contains(Option.LONG_LEADER_ELECTION)) { mcb.addMasterServerFlag("--leader_failure_max_missed_heartbeat_periods=10.0"); } if (opts.contains(Option.SHORT_TOKENS_AND_TICKETS)) { mcb.addMasterServerFlag("--authn_token_validity_seconds=" + TICKET_LIFETIME_SECS) .kdcRenewLifetime(RENEWABLE_LIFETIME_SECS + "s") .kdcTicketLifetime(TICKET_LIFETIME_SECS + "s"); } miniCluster = mcb.numMasterServers(3) .numTabletServers(opts.contains(Option.START_TSERVERS) ? 3 : 0) .build(); miniCluster.kinit("test-admin"); client = new KuduClient.KuduClientBuilder(miniCluster.getMasterAddressesAsString()).build(); // TODO(todd): it seems that exportAuthenticationCredentials() doesn't properly retry // in the case that there is no leader, even though NoLeaderFoundException is a RecoverableException. // So, we have to use a hack of calling listTabletServers, which _does_ properly retry, // in order to wait for the masters to elect a leader. client.listTabletServers(); }
/** * Regression test for some log spew which occurred in short-lived client instances which * had outbound connections. */ @Test(timeout = 100000) public void testCloseShortlyAfterOpen() throws Exception { CapturingLogAppender cla = new CapturingLogAppender(); try (Closeable c = cla.attach()) { try (KuduClient localClient = new KuduClient.KuduClientBuilder(harness.getMasterAddressesAsString()).build()) { // Force the client to connect to the masters. localClient.exportAuthenticationCredentials(); } } // Ensure there is no log spew due to an unexpected lost connection. String exception_text = cla.getAppendedText(); assertFalse("Unexpected exception:\n" + exception_text, exception_text.contains("lost connection to peer")); }
/** * Creates a local aclient that we auto-close while buffering one row, then makes sure that after * closing that we can read the row. */ @Test(timeout = 100000) public void testAutoClose() throws Exception { try (KuduClient localClient = new KuduClient.KuduClientBuilder(harness.getMasterAddressesAsString()).build()) { localClient.createTable(TABLE_NAME, basicSchema, getBasicCreateTableOptions()); KuduTable table = localClient.openTable(TABLE_NAME); KuduSession session = localClient.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); Insert insert = createBasicSchemaInsert(table, 0); session.apply(insert); } KuduTable table = client.openTable(TABLE_NAME); AsyncKuduScanner scanner = new AsyncKuduScanner.AsyncKuduScannerBuilder(asyncClient, table).build(); assertEquals(1, countRowsInScan(scanner)); }
@Override public Token obtainToken() throws Exception { try { if (UserGroupInformation.getCurrentUser().isFromKeytab()) { // If we're using a keytab, we're probably in cluster mode // can we rely on Spark already having logged in at least once? UserGroupInformation.getCurrentUser().reloginFromKeytab(); } KuduClient client = new KuduClient.KuduClientBuilder(kuduMasterAddresses).build(); byte[] token = client.exportAuthenticationCredentials(); client.close(); LOG.debug("Obtained new Kudu token for {}", kuduMasterAddresses); return SecurityUtils.createToken(token); } catch (Exception e) { LOG.error("Could not obtain new security token from {}", kuduMasterAddresses); throw e; } }
@Override public void run() { try (KuduClient contextClient = new KuduClient.KuduClientBuilder(masterAddresses) .defaultAdminOperationTimeoutMs(operationTimeoutMs) .build()) { KuduScanner scanner = KuduScanToken.deserializeIntoScanner(serializedToken, contextClient); try { int localCount = 0; while (scanner.hasMoreRows()) { localCount += Iterators.size(scanner.nextRows()); } count.addAndGet(localCount); } finally { scanner.close(); } } catch (Exception e) { LOG.error("exception in parallel token scanner", e); } } });
@Singleton @Provides KuduClientSession createKuduClientSession( KuduConnectorId connectorId, KuduClientConfig config) { requireNonNull(config, "config is null"); KuduClient.KuduClientBuilder builder = new KuduClient.KuduClientBuilder(config.getMasterAddresses()); builder.defaultAdminOperationTimeoutMs(config.getDefaultAdminOperationTimeout().toMillis()); builder.defaultOperationTimeoutMs(config.getDefaultOperationTimeout().toMillis()); builder.defaultSocketReadTimeoutMs(config.getDefaultSocketReadTimeout().toMillis()); if (config.isDisableStatistics()) { builder.disableStatistics(); } KuduClient client = builder.build(); String tenant = config.getTenant(); return new NativeKuduClientSession(connectorId, client, tenant); } }
@Override public Void call() throws Exception { for (int i = 0; i < 5; i++) { try (KuduClient c = new KuduClient.KuduClientBuilder(harness.getMasterAddressesAsString()) .build()) { KuduTable table = c.openTable(TABLE_NAME); for (int j = 0; j < 5; j++) { KuduScanToken.KuduScanTokenBuilder scanBuilder = c.newScanTokenBuilder(table); scanBuilder.build(); c.asyncClient.emptyTabletsCacheForTable(table.getTableId()); } } } return null; } }));
private KuduClient createClient() { return new KuduClient.KuduClientBuilder(miniCluster.getMasterAddressesAsString()).build(); }
@Test(timeout = 50000) public void testKerberos() throws Exception { FakeDNS.getInstance().install(); try (MiniKuduCluster cluster = new MiniKuduCluster.MiniKuduClusterBuilder() .numMasterServers(NUM_MASTERS) .numTabletServers(NUM_TABLET_SERVERS) .enableKerberos() .build()) { KuduClient client = new KuduClientBuilder(cluster.getMasterAddressesAsString()).build(); ListTablesResponse resp = client.getTablesList(); assertTrue(resp.getTablesList().isEmpty()); assertNull(client.getHiveMetastoreConfig()); } }
KuduConnection(Config config, byte[] token) { client = new KuduClient.KuduClientBuilder(config.getString(CONNECTION_CONFIG_NAME)).build(); if (token != null) { client.importAuthenticationCredentials(token); } session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); session.setMutationBufferSpace(10000); session.setIgnoreAllDuplicateRows(KuduUtils.doesInsertIgnoreDuplicates(config)); }
@Test(timeout = 50000) public void testHiveMetastoreIntegration() throws Exception { try (MiniKuduCluster cluster = new MiniKuduCluster.MiniKuduClusterBuilder() .numMasterServers(NUM_MASTERS) .numTabletServers(NUM_TABLET_SERVERS) .enableHiveMetastoreIntegration() .build()) { KuduClient client = new KuduClientBuilder(cluster.getMasterAddressesAsString()).build(); assertNotNull(client.getHiveMetastoreConfig()); } }
@Override public void initialize(Map<String, Object> puProperties) { reader = new KuduDBEntityReader(kunderaMetadata); setExternalProperties(puProperties); initializePropertyReader(); PersistenceUnitMetadata pum = kunderaMetadata.getApplicationMetadata() .getPersistenceUnitMetadata(getPersistenceUnit()); Properties pumProps = pum.getProperties(); if (puProperties != null) { pumProps.putAll(puProperties); } String kuduMasterHost = (String) pumProps.getProperty("kundera.nodes"); String kuduMasterPort = (String) pumProps.getProperty("kundera.port"); if (kuduMasterHost == null || kuduMasterPort == null) { throw new KunderaException("Hostname/IP or Port is null."); } kuduClient = new KuduClient.KuduClientBuilder(kuduMasterHost + ":" + kuduMasterPort).build(); }
@Override protected boolean initiateClient() { for (String host : hosts) { if (host == null || !StringUtils.isNumeric(port) || port.isEmpty()) { logger.error("Host or port should not be null / port should be numeric"); throw new IllegalArgumentException("Host or port should not be null / port should be numeric"); } try { client = new KuduClient.KuduClientBuilder(host + ":" + port).build(); } catch (Exception e) { logger.error("Database host cannot be resolved, Caused by: " + e.getMessage()); throw new SchemaGenerationException("Database host cannot be resolved, Caused by: " + e.getMessage()); } } return true; }
protected KuduClient createClient(final String masters) { return new KuduClient.KuduClientBuilder(masters).build(); }
public KuduStoragePlugin(KuduStoragePluginConfig configuration, DrillbitContext context, String name) throws IOException { super(context, name); this.schemaFactory = new KuduSchemaFactory(this, name); this.engineConfig = configuration; this.client = new KuduClient.KuduClientBuilder(configuration.getMasterAddresses()).build(); }