@Override protected boolean initiateClient() { for (String host : hosts) { if (host == null || !StringUtils.isNumeric(port) || port.isEmpty()) { logger.error("Host or port should not be null / port should be numeric"); throw new IllegalArgumentException("Host or port should not be null / port should be numeric"); } try { client = new KuduClient.KuduClientBuilder(host + ":" + port).build(); } catch (Exception e) { logger.error("Database host cannot be resolved, Caused by: " + e.getMessage()); throw new SchemaGenerationException("Database host cannot be resolved, Caused by: " + e.getMessage()); } } return true; }
/** * Initialize the 'clients' member with the configured number of * clients. */ private void initClients() throws DBException { synchronized (KuduYCSBClient.class) { if (!clients.isEmpty()) { return; } Properties prop = getProperties(); String masterAddresses = prop.getProperty(MASTER_ADDRESSES_OPT, "localhost:7051"); LOG.debug("Connecting to the masters at {}", masterAddresses); int numClients = getIntFromProp(prop, NUM_CLIENTS_OPT, DEFAULT_NUM_CLIENTS); for (int i = 0; i < numClients; i++) { clients.add(new KuduClient.KuduClientBuilder(masterAddresses) .defaultSocketReadTimeoutMs(DEFAULT_SLEEP) .defaultOperationTimeoutMs(DEFAULT_SLEEP) .defaultAdminOperationTimeoutMs(DEFAULT_SLEEP) .build()); } } }
@Singleton @Provides KuduClientSession createKuduClientSession( KuduConnectorId connectorId, KuduClientConfig config) { requireNonNull(config, "config is null"); KuduClient.KuduClientBuilder builder = new KuduClient.KuduClientBuilder(config.getMasterAddresses()); builder.defaultAdminOperationTimeoutMs(config.getDefaultAdminOperationTimeout().toMillis()); builder.defaultOperationTimeoutMs(config.getDefaultOperationTimeout().toMillis()); builder.defaultSocketReadTimeoutMs(config.getDefaultSocketReadTimeout().toMillis()); if (config.isDisableStatistics()) { builder.disableStatistics(); } KuduClient client = builder.build(); SchemaEmulation strategy; if (config.isSchemaEmulationEnabled()) { strategy = new SchemaEmulationByTableNameConvention(config.getSchemaEmulationPrefix()); } else { strategy = new NoSchemaEmulation(); } return new KuduClientSession(connectorId, client, strategy); } }
System.clearProperty(SecurityUtil.KUDU_TICKETCACHE_PROPERTY); try { KuduClient newClient = new KuduClient.KuduClientBuilder( miniCluster.getMasterAddressesAsString()).build();
KuduClient c = null; try { c = new KuduClient.KuduClientBuilder(masterAddr).build();
checkNotNull(builder.tableName,"Kudu table cannot be null"); builderForThisConnection = builder; KuduClient.KuduClientBuilder kuduClientBuilder = new KuduClient.KuduClientBuilder(builder.mastersCollection); if (builder.isOperationTimeOutSet) { kuduClientBuilder.defaultOperationTimeoutMs(builder.operationTimeOutMs); kuduClientBuilder.bossCount(builder.numBossThreads); kuduClientBuilder.workerCount(builder.workerThreads); kuduClientBuilder.defaultSocketReadTimeoutMs(builder.socketReadTimeOutMs); kuduClient = kuduClientBuilder.build(); kuduSession = kuduClient.newSession(); if (builder.isFlushModeSet) {
@Test(timeout = 100000) public void testCustomNioExecutor() throws Exception { long startTime = System.nanoTime(); final KuduClient localClient = new KuduClient.KuduClientBuilder(harness.getMasterAddressesAsString()) .nioExecutors(Executors.newFixedThreadPool(1), Executors.newFixedThreadPool(2)) .bossCount(1) .workerCount(2) .build(); long buildTime = (System.nanoTime() - startTime) / 1000000000L; assertTrue("Building KuduClient is slow, maybe netty get stuck", buildTime < 3);
try (KuduClient alterClient = new KuduClient.KuduClientBuilder(harness.getMasterAddressesAsString()) .defaultAdminOperationTimeoutMs(harness.DEFAULT_SLEEP) .build()) { AlterTableOptions alter = new AlterTableOptions(); lower = basicSchema.newPartialRow();
cluster.kdestroy(); try (Closeable c = cla.attach(); KuduClient client = new KuduClient.KuduClientBuilder(cluster.getMasterAddressesAsString()) .build() ) { assertTrue(client.getTablesList().getTablesList().isEmpty());
public void testLowTimeouts() throws Exception { KuduClient lowTimeoutsClient = new KuduClient.KuduClientBuilder(harness.getMasterAddressesAsString()) .defaultAdminOperationTimeoutMs(1) .defaultOperationTimeoutMs(1) .build();
/** * Test that, if the masters are down when we attempt to connect, we don't end up * logging any nonsensical stack traces including Netty internals. */ @Test(timeout = 100000) public void testNoLogSpewOnConnectionRefused() throws Exception { CapturingLogAppender cla = new CapturingLogAppender(); try (Closeable c = cla.attach()) { harness.killAllMasterServers(); try (KuduClient localClient = new KuduClient.KuduClientBuilder(harness.getMasterAddressesAsString()).build()) { // Force the client to connect to the masters. localClient.exportAuthenticationCredentials(); fail("Should have failed to connect."); } catch (NoLeaderFoundException e) { assertTrue("Bad exception string: " + e.getMessage(), e.getMessage().matches(".*Master config .+ has no leader. " + "Exceptions received:.*Connection refused.*Connection refused" + ".*Connection refused.*")); } } finally { harness.startAllMasterServers(); } // Ensure there is no log spew due to an unexpected lost connection. String logText = cla.getAppendedText(); assertFalse("Should not claim to have lost a connection in the log", logText.contains("lost connection to peer")); assertFalse("Should not have netty spew in log", logText.contains("socket.nio.AbstractNioSelector")); }
new KuduClient.KuduClientBuilder(harness.getMasterAddressesAsString()).build()) { if (killBefore == KillBefore.CREATE_TABLE) { harness.killLeaderMasterServer();
new KuduClient.KuduClientBuilder(harness.getMasterAddressesAsString()).build()) { client.listTabletServers(); fail(); new KuduClient.KuduClientBuilder(harness.getMasterAddressesAsString()).build()) { Client.AuthenticationCredentialsPB credentials = Client.AuthenticationCredentialsPB.newBuilder().setRealUser("token-user").build();
/** * Test that the client properly falls back to the old GetMasterRegistration * RPC when connecting to a master which does not support the new * ConnectToMaster RPC. */ @Test(timeout=60000) public void testFallbackConnectRpc() throws Exception { MiniKuduCluster cluster = new MiniKuduCluster.MiniKuduClusterBuilder() .addMasterServerFlag("--master_support_connect_to_master_rpc=0") .numMasterServers(1) .numTabletServers(0) .build(); KuduClient c = null; try { c = new KuduClient.KuduClientBuilder(cluster.getMasterAddressesAsString()) .build(); // Call some method which uses the master. This forces us to connect // and verifies that the fallback works. c.listTabletServers(); } finally { if (c != null) { c.close(); } cluster.shutdown(); } }
@Singleton @Provides KuduClientSession createKuduClientSession( KuduConnectorId connectorId, KuduClientConfig config) { requireNonNull(config, "config is null"); KuduClient.KuduClientBuilder builder = new KuduClient.KuduClientBuilder(config.getMasterAddresses()); builder.defaultAdminOperationTimeoutMs(config.getDefaultAdminOperationTimeout().toMillis()); builder.defaultOperationTimeoutMs(config.getDefaultOperationTimeout().toMillis()); builder.defaultSocketReadTimeoutMs(config.getDefaultSocketReadTimeout().toMillis()); if (config.isDisableStatistics()) { builder.disableStatistics(); } KuduClient client = builder.build(); SchemaEmulation strategy; if (config.isSchemaEmulationEnabled()) { strategy = new SchemaEmulationByTableNameConvention(config.getSchemaEmulationPrefix()); } else { strategy = new NoSchemaEmulation(); } return new KuduClientSession(connectorId, client, strategy); } }
private void startCluster(Set<Option> opts) throws IOException { MiniKuduClusterBuilder mcb = new MiniKuduClusterBuilder(); mcb.enableKerberos(); if (opts.contains(Option.LONG_LEADER_ELECTION)) { mcb.addMasterServerFlag("--leader_failure_max_missed_heartbeat_periods=10.0"); } if (opts.contains(Option.SHORT_TOKENS_AND_TICKETS)) { mcb.addMasterServerFlag("--authn_token_validity_seconds=" + TICKET_LIFETIME_SECS) .kdcRenewLifetime(RENEWABLE_LIFETIME_SECS + "s") .kdcTicketLifetime(TICKET_LIFETIME_SECS + "s"); } miniCluster = mcb.numMasterServers(3) .numTabletServers(opts.contains(Option.START_TSERVERS) ? 3 : 0) .build(); miniCluster.kinit("test-admin"); client = new KuduClient.KuduClientBuilder(miniCluster.getMasterAddressesAsString()).build(); // TODO(todd): it seems that exportAuthenticationCredentials() doesn't properly retry // in the case that there is no leader, even though NoLeaderFoundException is a RecoverableException. // So, we have to use a hack of calling listTabletServers, which _does_ properly retry, // in order to wait for the masters to elect a leader. client.listTabletServers(); }
/** * Regression test for some log spew which occurred in short-lived client instances which * had outbound connections. */ @Test(timeout = 100000) public void testCloseShortlyAfterOpen() throws Exception { CapturingLogAppender cla = new CapturingLogAppender(); try (Closeable c = cla.attach()) { try (KuduClient localClient = new KuduClient.KuduClientBuilder(harness.getMasterAddressesAsString()).build()) { // Force the client to connect to the masters. localClient.exportAuthenticationCredentials(); } } // Ensure there is no log spew due to an unexpected lost connection. String exception_text = cla.getAppendedText(); assertFalse("Unexpected exception:\n" + exception_text, exception_text.contains("lost connection to peer")); }
/** * Creates a local aclient that we auto-close while buffering one row, then makes sure that after * closing that we can read the row. */ @Test(timeout = 100000) public void testAutoClose() throws Exception { try (KuduClient localClient = new KuduClient.KuduClientBuilder(harness.getMasterAddressesAsString()).build()) { localClient.createTable(TABLE_NAME, basicSchema, getBasicCreateTableOptions()); KuduTable table = localClient.openTable(TABLE_NAME); KuduSession session = localClient.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); Insert insert = createBasicSchemaInsert(table, 0); session.apply(insert); } KuduTable table = client.openTable(TABLE_NAME); AsyncKuduScanner scanner = new AsyncKuduScanner.AsyncKuduScannerBuilder(asyncClient, table).build(); assertEquals(1, countRowsInScan(scanner)); }
@Override public Token obtainToken() throws Exception { try { if (UserGroupInformation.getCurrentUser().isFromKeytab()) { // If we're using a keytab, we're probably in cluster mode // can we rely on Spark already having logged in at least once? UserGroupInformation.getCurrentUser().reloginFromKeytab(); } KuduClient client = new KuduClient.KuduClientBuilder(kuduMasterAddresses).build(); byte[] token = client.exportAuthenticationCredentials(); client.close(); LOG.debug("Obtained new Kudu token for {}", kuduMasterAddresses); return SecurityUtils.createToken(token); } catch (Exception e) { LOG.error("Could not obtain new security token from {}", kuduMasterAddresses); throw e; } }
@Override public void run() { try (KuduClient contextClient = new KuduClient.KuduClientBuilder(masterAddresses) .defaultAdminOperationTimeoutMs(operationTimeoutMs) .build()) { KuduScanner scanner = KuduScanToken.deserializeIntoScanner(serializedToken, contextClient); try { int localCount = 0; while (scanner.hasMoreRows()) { localCount += Iterators.size(scanner.nextRows()); } count.addAndGet(localCount); } finally { scanner.close(); } } catch (Exception e) { LOG.error("exception in parallel token scanner", e); } } });