public ClientSession createSession(String catalog, ZoneId timezone, Map<String, String> sessionProperties, String user) { return new ClientSession( prestoConfig.getAddress(), user == null ? "rakam" : user, "rakam", ImmutableSet.of(), null, catalog == null ? "default" : catalog, "default", TimeZone.getTimeZone(timezone == null ? ZoneOffset.UTC : timezone).getID(), Locale.ENGLISH, sessionProperties, ImmutableMap.of(), null, false, new Duration(1, TimeUnit.MINUTES)); }
@Override public Set<String> load(String project) throws Exception { try (Connection conn = prestoConnectionFactory.openConnection()) { HashSet<String> tables = new HashSet<>(); ResultSet tableRs = conn.getMetaData().getTables(config.getColdStorageConnector(), project, null, new String[]{"TABLE"}); while (tableRs.next()) { String tableName = tableRs.getString("table_name"); if (!tableName.startsWith(PrestoMaterializedViewService.MATERIALIZED_VIEW_PREFIX)) { tables.add(tableName); } } return tables; } } });
private String getTableReference(String project, String tableName, Optional<QuerySampling> sample) { String hotStorageConnector = prestoConfig.getHotStorageConnector(); String table = checkCollection(project) + "." + checkCollection(tableName) + sample.map(e -> " TABLESAMPLE " + e.method.name() + "(" + e.percentage + ")").orElse(""); if (hotStorageConnector != null) { return "((select * from " + prestoConfig.getColdStorageConnector() + "." + table + " union all " + "select * from " + hotStorageConnector + "." + table + ")" + " as " + tableName + ")"; } else { return prestoConfig.getColdStorageConnector() + "." + table; } } }
prestoConfig = new PrestoConfig() .setAddress(URI.create(prestoUrl)) .setStreamingConnector("streaming") .setColdStorageConnector("rakam_raptor"); LOGGER.info("Presto started on " + prestoUrl); .setUsername("sa").setPassword("")); System.out.println(prestoConfig.getAddress());
this.prestoConfig = prestoConfig; defaultSession = new ClientSession( prestoConfig.getAddress(), "rakam", "api-server", ImmutableSet.of(), null, prestoConfig.getColdStorageConnector(), "default", TimeZone.getTimeZone(ZoneOffset.UTC).getID(),
queryEnd += format(", \"%s\" %s", prestoConfig.getCheckpointColumn(), toSql(TIMESTAMP)); prestoConfig.getColdStorageConnector(), project, checkCollection(collection), queryEnd, properties); .filter(field -> schemaFields.stream().noneMatch(f -> f.getName().equals(field.getName()))) .forEach(f -> { if (f.getName().equals(prestoConfig.getCheckpointColumn())) { throw new RakamException("Checkpoint column is reserved", BAD_REQUEST);
prestoConfig.getHotStorageConnector() + "." + project + "." + e.getCollection(), offsetBefore, offsetNow,
private boolean filterTables(String tableName, String tableColumn) { return !tableName.startsWith(MATERIALIZED_VIEW_PREFIX) && !tableColumn.startsWith("$") && !tableName.startsWith("$") && !tableColumn.equals(prestoConfig.getCheckpointColumn()); }
@BeforeSuite @Override public void setup() throws Exception { testingEnvironment = new TestingEnvironment(); PrestoConfig prestoConfig = testingEnvironment.getPrestoConfig(); metastore = new PrestoRakamRaptorMetastore(testingEnvironment.getPrestoMetastore(), new EventBus(), new ProjectConfig(), prestoConfig); metastore.setup(); prestoQueryExecutor = new PrestoQueryExecutor(new ProjectConfig(), prestoConfig, null, metastore); materializedViewService = new PrestoMaterializedViewService( new PrestoConfig(), prestoQueryExecutor, metastore, getDatabaseMetadataStore(), Clock.systemUTC()); eventStore = new TestingPrestoEventStore(prestoQueryExecutor, prestoConfig); super.setup(); }
@Override public ProjectCollection getUserTable(String project, boolean isEventFilterActive) { if (isEventFilterActive) { return new ProjectCollection(prestoConfig.getUserConnector() + ".users", project); } return new ProjectCollection("users", project); } }
@Override public void createProject(String project) { if (config.getExistingProjects().contains(project)) { checkProject(project); try (Handle handle = dbi.open()) { handle.createStatement("INSERT INTO project (name) VALUES(:name)") .bind("name", project) .execute(); } catch (Exception e) { if (getProjects().contains(project)) { throw new RakamException("The project already exists", BAD_REQUEST); } } super.onCreateProject(project); return; } throw new RakamException(NOT_IMPLEMENTED); }
@Override public List<SchemaField> getCollection(String project, String collection) { return dao.listTableColumns(project, collection).stream() // this field should be removed since the server sets it .filter(a -> !a.getColumnName().equals(prestoConfig.getCheckpointColumn())) .map(column -> { TypeSignature typeSignature = column.getDataType().getTypeSignature(); return new SchemaField(column.getColumnName(), PrestoQueryExecution.fromPrestoType(typeSignature.getBase(), typeSignature.getParameters().stream() .filter(param -> param.getKind() == TYPE) .map(param -> param.getTypeSignature().getBase()).iterator())); }).collect(Collectors.toList()); }
@BeforeSuite @Override public void setup() throws Exception { testingEnvironment = new TestingEnvironment(); PrestoConfig prestoConfig = testingEnvironment.getPrestoConfig(); InMemoryQueryMetadataStore inMemoryQueryMetadataStore = new InMemoryQueryMetadataStore(); JDBCPoolDataSource prestoMetastore = testingEnvironment.getPrestoMetastore(); EventBus eventBus = new EventBus(); metastore = new PrestoRakamRaptorMetastore(prestoMetastore, eventBus, new ProjectConfig(), prestoConfig); metastore.setup(); PrestoQueryExecutor prestoQueryExecutor = new PrestoQueryExecutor(new ProjectConfig(), prestoConfig, null, metastore); PrestoMaterializedViewService materializedViewService = new PrestoMaterializedViewService( new PrestoConfig(), prestoQueryExecutor, metastore, inMemoryQueryMetadataStore, Clock.systemUTC()); QueryExecutorService queryExecutorService = new QueryExecutorService(prestoQueryExecutor, metastore, materializedViewService, '"'); FastGenericFunnelQueryExecutor fastGenericFunnelQueryExecutor = new FastGenericFunnelQueryExecutor(queryExecutorService, new ProjectConfig(), metastore); PrestoApproxFunnelQueryExecutor prestoApproxFunnelQueryExecutor = new PrestoApproxFunnelQueryExecutor(new ProjectConfig(), queryExecutorService, metastore); funnelQueryExecutor = new PrestoFunnelQueryExecutor(new ProjectConfig(), new PrestoConfig(), fastGenericFunnelQueryExecutor, prestoApproxFunnelQueryExecutor, metastore, prestoQueryExecutor, new UserPluginConfig()); testingPrestoEventStore = new TestingPrestoEventStore(prestoQueryExecutor, prestoConfig); super.setup(); }
@Override public List<SchemaField> load(ProjectCollection key) throws Exception { try (Connection conn = prestoConnectionFactory.openConnection()) { ResultSet dbColumns = conn.getMetaData().getColumns(config.getColdStorageConnector(), key.project, key.collection, null); List<SchemaField> schema = convertToSchema(dbColumns); if (schema == null) { return ImmutableList.of(); } return schema; } } });
properties.put("user", "presto-rakam"); return DriverManager.getConnection(String.format("jdbc:presto://%s:%d", config.getAddress().getHost(), config.getAddress().getPort()), properties); };
.filter(col -> collections.stream().allMatch(list -> list.getValue().contains(col))) .map(f -> f.getName()) .filter(f -> !f.equals(prestoConfig.getCheckpointColumn())) .collect(Collectors.joining(", ")); .map(collection -> format("select '%s' as _collection, %s, %s from %s", collection, checkTableColumn(prestoConfig.getCheckpointColumn()), sharedColumns.isEmpty() ? "1" : sharedColumns, getTableReference(project, collection, sample))) .collect(Collectors.joining(" union all ")) + ") _all"; } else { return "(select cast(null as varchar) as _collection, cast(null as timestamp) as " + checkTableColumn(prestoConfig.getCheckpointColumn()) + ", null as _user, null as " + checkTableColumn(projectConfig.getTimeColumn()) + " limit 0) _all";
@BeforeSuite public void setup() throws Exception { testingEnvironment = new TestingEnvironment(); PrestoConfig prestoConfig = testingEnvironment.getPrestoConfig(); InMemoryQueryMetadataStore queryMetadataStore = new InMemoryQueryMetadataStore(); metastore = new PrestoRakamRaptorMetastore(testingEnvironment.getPrestoMetastore(), new EventBus(), new ProjectConfig(), prestoConfig); metastore.setup(); PrestoQueryExecutor queryExecutor = new PrestoQueryExecutor(new ProjectConfig(), prestoConfig, null, metastore); PrestoMaterializedViewService materializedViewService = new PrestoMaterializedViewService( new PrestoConfig(), queryExecutor, metastore, queryMetadataStore, Clock.systemUTC()); QueryExecutorService queryExecutorService = new QueryExecutorService(queryExecutor, metastore, materializedViewService, '"'); retentionQueryExecutor = new PrestoRetentionQueryExecutor(new ProjectConfig(), queryExecutorService, metastore, materializedViewService, new UserPluginConfig()); testingPrestoEventStore = new TestingPrestoEventStore(queryExecutor, prestoConfig); // TODO: Presto throws "No node available" error, find a way to avoid this ugly hack. Thread.sleep(1000); super.setup(); }
public String convertFunnel(String project, String connectorField, int idx, FunnelStep funnelStep, Optional<String> dimension, Optional<String> segment, LocalDate startDate, LocalDate endDate) { Optional<String> filterExp = funnelStep.getExpression().map(value -> RakamSqlFormatter.formatExpression(value, name -> name.getParts().stream().map(e -> formatIdentifier(e, '"')).collect(Collectors.joining(".")), name -> formatIdentifier("step" + idx, '"') + "." + name, '"')); String format = format("SELECT %s %s, %d as step, %s.%s from %s.%s.%s %s %s %s", dimension.map(ValidationUtil::checkTableColumn).map(v -> "step" + idx + "." + v).map(v -> segment.isPresent() ? applySegment(v, segment) + " as \"" + dimension.orElse("") + "_segment\"" + "," : v + ",").orElse(""), userMappingEnabled ? format("coalesce(mapping._user, %s._user, %s) as _user", "step" + idx, format(connectorField, "step" + idx)) : format(connectorField, "step" + idx), idx + 1, "step" + idx, checkTableColumn(projectConfig.getTimeColumn()), prestoConfig.getColdStorageConnector(), checkProject(project, '"'), checkCollection(funnelStep.getCollection()), "step" + idx, userMappingEnabled ? format("left join %s.%s mapping on (%s.%s is null and mapping.created_at >= date '%s' and mapping.merged_at <= date '%s' and mapping.id = %s.%s)", project, checkCollection(ANONYMOUS_ID_MAPPING), "step" + idx, checkTableColumn(projectConfig.getUserColumn()), startDate.format(ISO_LOCAL_DATE), endDate.format(ISO_LOCAL_DATE), "step" + idx, checkTableColumn(projectConfig.getUserColumn())) : "", filterExp.map(v -> "where " + v).orElse("")); return format; }
@Inject public PrestoEventStream(@ForStreamer HttpClient httpClient, AWSKinesisModule.PrestoStreamConfig config, PrestoConfig prestoConfig) { this.httpClient = httpClient; this.streamingPort = config.getPort(); this.prestoAddress = prestoConfig.getAddress(); this.queryCodec = JsonCodec.jsonCodec(StreamQuery.class); }
checkTableColumn(prestoConfig.getCheckpointColumn()), predicate); }, '"'); queryExecutor.formatTableReference(context.project, name, Optional.empty(), ImmutableMap.of()), format("WHERE %s > from_unixtime(%d)", checkTableColumn(prestoConfig.getCheckpointColumn()), lastUpdated.getEpochSecond())); return collection; }, '"');