@Override public QueryExecution executeRawQuery(RequestContext context, String query, ZoneId timezone, Map<String, String> sessionParameters) { return executeRawQuery(context, query, timezone, sessionParameters, null); }
@Subscribe public void onCreateProject(ProjectCreatedEvent event) { executor.executeRawStatement(format("CREATE TABLE %s(id VARCHAR, %s VARCHAR, " + "created_at TIMESTAMP, merged_at TIMESTAMP)", executor.formatTableReference(event.project, QualifiedName.of(ANONYMOUS_ID_MAPPING), Optional.empty(), ImmutableMap.of()), checkCollection(projectConfig.getUserColumn()))); } }
public QueryExecution executeRawStatement(RequestContext context, String query, ZoneId timezone, Map<String, String> sessionProperties, String catalog, String user, boolean update) { return internalExecuteRawQuery(context, query, createSession(catalog, timezone, sessionProperties, user), update); }
public QueryExecution executeRawQuery(RequestContext context, String query, ZoneId timezone, Map<String, String> sessionProperties, String catalog) { if (sessionProperties.containsKey("external.source_options")) { String encodedKey = sessionProperties.get("external.source_options"); Map<String, DataSourceType> params; if (encodedKey != null) { params = JsonHelper.read(getDecoder().decode(encodedKey), new TypeReference<Map<String, DataSourceType>>() { }); } else { params = new HashMap<>(); } if (params.size() == 1) { Map.Entry<String, DataSourceType> next = params.entrySet().iterator().next(); QueryExecution singleQueryExecution = getSingleQueryExecution(query, next.getValue()); if (singleQueryExecution != null) { return singleQueryExecution; } } } return executeRawStatement(context, query, timezone, sessionProperties, catalog, null, false); }
@Override public QueryExecution executeRawStatement(RequestContext context, String query, Map<String, String> sessionProperties) { return executeRawStatement(context, query, ZoneOffset.UTC, sessionProperties, null, null, true); }
@BeforeSuite @Override public void setup() throws Exception { testingEnvironment = new TestingEnvironment(); PrestoConfig prestoConfig = testingEnvironment.getPrestoConfig(); metastore = new PrestoRakamRaptorMetastore(testingEnvironment.getPrestoMetastore(), new EventBus(), new ProjectConfig(), prestoConfig); metastore.setup(); prestoQueryExecutor = new PrestoQueryExecutor(new ProjectConfig(), prestoConfig, null, metastore); materializedViewService = new PrestoMaterializedViewService( new PrestoConfig(), prestoQueryExecutor, metastore, getDatabaseMetadataStore(), Clock.systemUTC()); eventStore = new TestingPrestoEventStore(prestoQueryExecutor, prestoConfig); super.setup(); }
@Override public List<String> getEventFilterPredicate(String project, List<EventFilter> eventFilter) { return eventFilter.stream().map(f -> String.format("id in (%s)", String.format(getEventFilterQuery(project, f), executor.formatTableReference(project, QualifiedName.of(f.collection), Optional.empty(), ImmutableMap.of())))) .collect(Collectors.toList()); }
String suffix = node.getSuffix(); if ("materialized".equals(prefix)) { return getTableReference(project, MATERIALIZED_VIEW_PREFIX + suffix, sample); } else if ("collection".equals(prefix) || (prefix == null && !"users".equals(suffix) && !"_all".equals(suffix))) { return getTableReference(project, suffix, sample); } else { String encodedKey = sessionParameters.get("external.source_options"); checkTableColumn(prestoConfig.getCheckpointColumn()), sharedColumns.isEmpty() ? "1" : sharedColumns, getTableReference(project, collection, sample))) .collect(Collectors.joining(" union all ")) + ") _all"; } else {
@Override public int[] storeBatch(List<Event> events) { for (Map.Entry<String, List<Event>> collection : events.stream().collect(Collectors.groupingBy(e -> e.collection())).entrySet()) { QueryResult join = queryExecutor.executeRawStatement(String.format("INSERT INTO %s.%s.%s (_shard_time, %s) (%s)", config.getColdStorageConnector(), checkProject(events.get(0).project(), '"'), checkCollection(collection.getKey()), collection.getValue().get(0).schema().stream().map(e -> ValidationUtil.checkCollection(e.getName())) .collect(Collectors.joining(", ")), collection.getValue().stream() .map(e -> buildValues(e.properties(), e.schema())) .collect(Collectors.joining(" union all ")))) .getResult().join(); if (join.isFailed()) { try { Thread.sleep(300000); } catch (InterruptedException e) { e.printStackTrace(); } throw new IllegalStateException(join.getError().message); } } return EventStore.SUCCESSFUL_BATCH; }
@BeforeSuite public void setup() throws Exception { testingEnvironment = new TestingEnvironment(); PrestoConfig prestoConfig = testingEnvironment.getPrestoConfig(); InMemoryQueryMetadataStore queryMetadataStore = new InMemoryQueryMetadataStore(); metastore = new PrestoRakamRaptorMetastore(testingEnvironment.getPrestoMetastore(), new EventBus(), new ProjectConfig(), prestoConfig); metastore.setup(); PrestoQueryExecutor queryExecutor = new PrestoQueryExecutor(new ProjectConfig(), prestoConfig, null, metastore); PrestoMaterializedViewService materializedViewService = new PrestoMaterializedViewService( new PrestoConfig(), queryExecutor, metastore, queryMetadataStore, Clock.systemUTC()); QueryExecutorService queryExecutorService = new QueryExecutorService(queryExecutor, metastore, materializedViewService, '"'); retentionQueryExecutor = new PrestoRetentionQueryExecutor(new ProjectConfig(), queryExecutorService, metastore, materializedViewService, new UserPluginConfig()); testingPrestoEventStore = new TestingPrestoEventStore(queryExecutor, prestoConfig); // TODO: Presto throws "No node available" error, find a way to avoid this ugly hack. Thread.sleep(1000); super.setup(); }
@Override public QueryExecution executeRawQuery(RequestContext context, String query, Map<String, String> sessionProperties) { return executeRawQuery(context, query, ZoneOffset.UTC, sessionProperties, null); }
@Override public void store(Event event) { queryExecutor.executeRawStatement(String.format("INSERT INTO %s.%s.%s VALUES %s", config.getColdStorageConnector(), event.project(), event.collection(), buildValues(event.properties(), event.schema()))); }
@BeforeSuite @Override public void setup() throws Exception { testingEnvironment = new TestingEnvironment(); PrestoConfig prestoConfig = testingEnvironment.getPrestoConfig(); JDBCConfig postgresqlConfig = testingEnvironment.getPostgresqlConfig(); metastoreDataSource = JDBCPoolDataSource.getOrCreateDataSource(postgresqlConfig); queryMetadataStore = new InMemoryQueryMetadataStore(); metastore = new PrestoRakamRaptorMetastore(testingEnvironment.getPrestoMetastore(), new EventBus(), new ProjectConfig(), prestoConfig); metastore.setup(); prestoQueryExecutor = new PrestoQueryExecutor(new ProjectConfig(), prestoConfig, null, metastore); PrestoMaterializedViewService materializedViewService = new PrestoMaterializedViewService( new PrestoConfig(), prestoQueryExecutor, metastore, queryMetadataStore, Clock.systemUTC()); QueryExecutorService queryExecutorService = new QueryExecutorService(prestoQueryExecutor, metastore, materializedViewService, '"'); eventExplorer = new PrestoEventExplorer(new ProjectConfig(), queryExecutorService, materializedViewService); eventStore = new TestingPrestoEventStore(prestoQueryExecutor, prestoConfig); super.setup(); }
return; prestoExecutor.executeRawQuery(new RequestContext(project, null), query + " limit 1000").getResult() .thenAccept(r -> { lastOffsets = offsets;
@BeforeSuite @Override public void setup() throws Exception { testingEnvironment = new TestingEnvironment(); PrestoConfig prestoConfig = testingEnvironment.getPrestoConfig(); InMemoryQueryMetadataStore inMemoryQueryMetadataStore = new InMemoryQueryMetadataStore(); JDBCPoolDataSource prestoMetastore = testingEnvironment.getPrestoMetastore(); EventBus eventBus = new EventBus(); metastore = new PrestoRakamRaptorMetastore(prestoMetastore, eventBus, new ProjectConfig(), prestoConfig); metastore.setup(); PrestoQueryExecutor prestoQueryExecutor = new PrestoQueryExecutor(new ProjectConfig(), prestoConfig, null, metastore); PrestoMaterializedViewService materializedViewService = new PrestoMaterializedViewService( new PrestoConfig(), prestoQueryExecutor, metastore, inMemoryQueryMetadataStore, Clock.systemUTC()); QueryExecutorService queryExecutorService = new QueryExecutorService(prestoQueryExecutor, metastore, materializedViewService, '"'); FastGenericFunnelQueryExecutor fastGenericFunnelQueryExecutor = new FastGenericFunnelQueryExecutor(queryExecutorService, new ProjectConfig(), metastore); PrestoApproxFunnelQueryExecutor prestoApproxFunnelQueryExecutor = new PrestoApproxFunnelQueryExecutor(new ProjectConfig(), queryExecutorService, metastore); funnelQueryExecutor = new PrestoFunnelQueryExecutor(new ProjectConfig(), new PrestoConfig(), fastGenericFunnelQueryExecutor, prestoApproxFunnelQueryExecutor, metastore, prestoQueryExecutor, new UserPluginConfig()); testingPrestoEventStore = new TestingPrestoEventStore(prestoQueryExecutor, prestoConfig); super.setup(); }
return executor.executeRawQuery(context, format("select collection, json from (%s) order by %s desc limit %d", sqlQuery, checkTableColumn(projectConfig.getTimeColumn()), limit)) .getResult() .thenApply(result -> {