public ExceededCpuLimitException(Duration duration) { super(EXCEEDED_CPU_LIMIT, "Exceeded CPU limit of " + duration.toString()); } }
config.set(S3_MAX_BACKOFF_TIME, maxBackoffTime.toString()); config.set(S3_MAX_RETRY_TIME, maxRetryTime.toString()); config.set(S3_CONNECT_TIMEOUT, connectTimeout.toString()); config.set(S3_SOCKET_TIMEOUT, socketTimeout.toString()); config.set(S3_STAGING_DIRECTORY, stagingDirectory.toString()); config.setInt(S3_MAX_CONNECTIONS, maxConnections);
public static void assertOperatorIsUnblocked(Operator operator, Duration timeout) { if (!waitForOperatorToUnblock(operator, timeout)) { fail("Operator is expected to be unblocked within " + timeout.toString()); } }
public static void assertOperatorIsBlocked(Operator operator, Duration timeout) { if (waitForOperatorToUnblock(operator, timeout)) { fail("Operator is expected to be blocked for at least " + timeout.toString()); } }
public static void copyTpchTables( QueryRunner queryRunner, String sourceCatalog, String sourceSchema, Session session, Iterable<TpchTable<?>> tables) { log.info("Loading data from %s.%s...", sourceCatalog, sourceSchema); long startTime = System.nanoTime(); for (TpchTable<?> table : tables) { copyTable(queryRunner, sourceCatalog, sourceSchema, table.getTableName().toLowerCase(ENGLISH), session); } log.info("Loading from %s.%s complete in %s", sourceCatalog, sourceSchema, nanosSince(startTime).toString(SECONDS)); }
this.stagingDirectory = new File(conf.get(S3_STAGING_DIRECTORY, defaults.getS3StagingDirectory().toString())); this.maxAttempts = conf.getInt(S3_MAX_CLIENT_RETRIES, defaults.getS3MaxClientRetries()) + 1; this.maxBackoffTime = Duration.valueOf(conf.get(S3_MAX_BACKOFF_TIME, defaults.getS3MaxBackoffTime().toString())); this.maxRetryTime = Duration.valueOf(conf.get(S3_MAX_RETRY_TIME, defaults.getS3MaxRetryTime().toString())); int maxErrorRetries = conf.getInt(S3_MAX_ERROR_RETRIES, defaults.getS3MaxErrorRetries()); boolean sslEnabled = conf.getBoolean(S3_SSL_ENABLED, defaults.isS3SslEnabled()); Duration connectTimeout = Duration.valueOf(conf.get(S3_CONNECT_TIMEOUT, defaults.getS3ConnectTimeout().toString())); Duration socketTimeout = Duration.valueOf(conf.get(S3_SOCKET_TIMEOUT, defaults.getS3SocketTimeout().toString())); int maxConnections = conf.getInt(S3_MAX_CONNECTIONS, defaults.getS3MaxConnections()); this.multiPartUploadMinFileSize = conf.getLong(S3_MULTIPART_MIN_FILE_SIZE, defaults.getS3MultipartMinFileSize().toBytes());
private static void copyTpchTables( QueryRunner queryRunner, String sourceCatalog, String sourceSchema, Session session, Iterable<TpchTable<?>> tables) { LOG.info("Loading data from %s.%s...", sourceCatalog, sourceSchema); long startTime = System.nanoTime(); for (TpchTable<?> table : tables) { copyTable(queryRunner, sourceCatalog, session, sourceSchema, table); } LOG.info("Loading from %s.%s complete in %s", sourceCatalog, sourceSchema, nanosSince(startTime).toString(SECONDS)); }
public static void copyTpchTablesBucketed( QueryRunner queryRunner, String sourceCatalog, String sourceSchema, Session session, Iterable<TpchTable<?>> tables) { log.info("Loading data from %s.%s...", sourceCatalog, sourceSchema); long startTime = System.nanoTime(); for (TpchTable<?> table : tables) { copyTableBucketed(queryRunner, new QualifiedObjectName(sourceCatalog, sourceSchema, table.getTableName().toLowerCase(ENGLISH)), session); } log.info("Loading from %s.%s complete in %s", sourceCatalog, sourceSchema, nanosSince(startTime).toString(SECONDS)); }
"10ms", false, duration -> Duration.valueOf(duration.toString()).toString(), object -> object);
int maxErrorRetries = config.getInt(S3_MAX_ERROR_RETRIES, defaults.getS3MaxErrorRetries()); boolean sslEnabled = config.getBoolean(S3_SSL_ENABLED, defaults.isS3SslEnabled()); Duration connectTimeout = Duration.valueOf(config.get(S3_CONNECT_TIMEOUT, defaults.getS3ConnectTimeout().toString())); Duration socketTimeout = Duration.valueOf(config.get(S3_SOCKET_TIMEOUT, defaults.getS3SocketTimeout().toString())); int maxConnections = config.getInt(S3_SELECT_PUSHDOWN_MAX_CONNECTIONS, clientConfig.getS3SelectPushdownMaxConnections());
private synchronized void scheduleNextRequest() { // stopped or done? TaskStatus taskStatus = getTaskStatus(); if (!running || taskStatus.getState().isDone()) { return; } // outstanding request? if (future != null && !future.isDone()) { // this should never happen log.error("Can not reschedule update because an update is already running"); return; } // if throttled due to error, asynchronously wait for timeout and try again ListenableFuture<?> errorRateLimit = errorTracker.acquireRequestPermit(); if (!errorRateLimit.isDone()) { errorRateLimit.addListener(this::scheduleNextRequest, executor); return; } Request request = prepareGet() .setUri(uriBuilderFrom(taskStatus.getSelf()).appendPath("status").build()) .setHeader(CONTENT_TYPE, JSON_UTF_8.toString()) .setHeader(PRESTO_CURRENT_STATE, taskStatus.getState().toString()) .setHeader(PRESTO_MAX_WAIT, refreshMaxWait.toString()) .build(); errorTracker.startRequest(); future = httpClient.executeAsync(request, createFullJsonResponseHandler(taskStatusCodec)); currentRequestStartNanos.set(System.nanoTime()); Futures.addCallback(future, new SimpleHttpResponseHandler<>(this, request.getUri(), stats), executor); }
public static DistributedQueryRunner createRedisQueryRunner(EmbeddedRedis embeddedRedis, String dataFormat, Iterable<TpchTable<?>> tables) throws Exception { DistributedQueryRunner queryRunner = null; try { queryRunner = new DistributedQueryRunner(createSession(), 2); queryRunner.installPlugin(new TpchPlugin()); queryRunner.createCatalog("tpch", "tpch"); embeddedRedis.start(); Map<SchemaTableName, RedisTableDescription> tableDescriptions = createTpchTableDescriptions(queryRunner.getCoordinator().getMetadata(), tables, dataFormat); installRedisPlugin(embeddedRedis, queryRunner, tableDescriptions); TestingPrestoClient prestoClient = queryRunner.getClient(); log.info("Loading data..."); long startTime = System.nanoTime(); for (TpchTable<?> table : tables) { loadTpchTable(embeddedRedis, prestoClient, table, dataFormat); } log.info("Loading complete in %s", nanosSince(startTime).toString(SECONDS)); embeddedRedis.destroyJedisPool(); return queryRunner; } catch (Throwable e) { closeAllSuppress(e, queryRunner, embeddedRedis); throw e; } }
public static DistributedQueryRunner createElasticsearchQueryRunner(EmbeddedElasticsearchNode embeddedElasticsearchNode, Iterable<TpchTable<?>> tables) throws Exception { DistributedQueryRunner queryRunner = null; try { queryRunner = DistributedQueryRunner.builder(createSession()) .setNodeCount(NODE_COUNT) .build(); queryRunner.installPlugin(new TpchPlugin()); queryRunner.createCatalog("tpch", "tpch"); embeddedElasticsearchNode.start(); ElasticsearchTableDescriptionProvider tableDescriptions = createTableDescriptions(queryRunner.getCoordinator().getMetadata()); TestingElasticsearchConnectorFactory testFactory = new TestingElasticsearchConnectorFactory(tableDescriptions); installElasticsearchPlugin(queryRunner, testFactory); TestingPrestoClient prestoClient = queryRunner.getClient(); LOG.info("Loading data..."); long startTime = System.nanoTime(); for (TpchTable<?> table : tables) { loadTpchTopic(embeddedElasticsearchNode, prestoClient, table); } LOG.info("Loading complete in %s", nanosSince(startTime).toString(SECONDS)); return queryRunner; } catch (Exception e) { closeAllSuppress(e, queryRunner, embeddedElasticsearchNode); throw e; } }
public static DistributedQueryRunner createKafkaQueryRunner(EmbeddedKafka embeddedKafka, Iterable<TpchTable<?>> tables) throws Exception { DistributedQueryRunner queryRunner = null; try { queryRunner = new DistributedQueryRunner(createSession(), 2); queryRunner.installPlugin(new TpchPlugin()); queryRunner.createCatalog("tpch", "tpch"); embeddedKafka.start(); for (TpchTable<?> table : tables) { embeddedKafka.createTopics(kafkaTopicName(table)); } Map<SchemaTableName, KafkaTopicDescription> topicDescriptions = createTpchTopicDescriptions(queryRunner.getCoordinator().getMetadata(), tables); installKafkaPlugin(embeddedKafka, queryRunner, topicDescriptions); TestingPrestoClient prestoClient = queryRunner.getClient(); log.info("Loading data..."); long startTime = System.nanoTime(); for (TpchTable<?> table : tables) { loadTpchTopic(embeddedKafka, prestoClient, table); } log.info("Loading complete in %s", nanosSince(startTime).toString(SECONDS)); return queryRunner; } catch (Throwable e) { closeAllSuppress(e, queryRunner, embeddedKafka); throw e; } }
public ExceededCpuLimitException(Duration duration) { super(EXCEEDED_CPU_LIMIT, "Exceeded CPU limit of " + duration.toString()); } }
@JsonValue @Override public String toString() { return toString(unit); }
@Override public String value() { return duration.toString(); }
public static void assertOperatorIsUnblocked(Operator operator, Duration timeout) { if (!waitForOperatorToUnblock(operator, timeout)) { fail("Operator is expected to be unblocked within " + timeout.toString()); } }