@Override public Set<String> load(String project) throws Exception { try (Connection conn = prestoConnectionFactory.openConnection()) { HashSet<String> tables = new HashSet<>(); ResultSet tableRs = conn.getMetaData().getTables(config.getColdStorageConnector(), project, null, new String[]{"TABLE"}); while (tableRs.next()) { String tableName = tableRs.getString("table_name"); if (!tableName.startsWith(PrestoMaterializedViewService.MATERIALIZED_VIEW_PREFIX)) { tables.add(tableName); } } return tables; } } });
@Override public List<SchemaField> load(ProjectCollection key) throws Exception { try (Connection conn = prestoConnectionFactory.openConnection()) { ResultSet dbColumns = conn.getMetaData().getColumns(config.getColdStorageConnector(), key.project, key.collection, null); List<SchemaField> schema = convertToSchema(dbColumns); if (schema == null) { return ImmutableList.of(); } return schema; } } });
private String getTableReference(String project, String tableName, Optional<QuerySampling> sample) { String hotStorageConnector = prestoConfig.getHotStorageConnector(); String table = checkCollection(project) + "." + checkCollection(tableName) + sample.map(e -> " TABLESAMPLE " + e.method.name() + "(" + e.percentage + ")").orElse(""); if (hotStorageConnector != null) { return "((select * from " + prestoConfig.getColdStorageConnector() + "." + table + " union all " + "select * from " + hotStorageConnector + "." + table + ")" + " as " + tableName + ")"; } else { return prestoConfig.getColdStorageConnector() + "." + table; } } }
public String convertFunnel(String project, String connectorField, int idx, FunnelStep funnelStep, Optional<String> dimension, Optional<String> segment, LocalDate startDate, LocalDate endDate) { Optional<String> filterExp = funnelStep.getExpression().map(value -> RakamSqlFormatter.formatExpression(value, name -> name.getParts().stream().map(e -> formatIdentifier(e, '"')).collect(Collectors.joining(".")), name -> formatIdentifier("step" + idx, '"') + "." + name, '"')); String format = format("SELECT %s %s, %d as step, %s.%s from %s.%s.%s %s %s %s", dimension.map(ValidationUtil::checkTableColumn).map(v -> "step" + idx + "." + v).map(v -> segment.isPresent() ? applySegment(v, segment) + " as \"" + dimension.orElse("") + "_segment\"" + "," : v + ",").orElse(""), userMappingEnabled ? format("coalesce(mapping._user, %s._user, %s) as _user", "step" + idx, format(connectorField, "step" + idx)) : format(connectorField, "step" + idx), idx + 1, "step" + idx, checkTableColumn(projectConfig.getTimeColumn()), prestoConfig.getColdStorageConnector(), checkProject(project, '"'), checkCollection(funnelStep.getCollection()), "step" + idx, userMappingEnabled ? format("left join %s.%s mapping on (%s.%s is null and mapping.created_at >= date '%s' and mapping.merged_at <= date '%s' and mapping.id = %s.%s)", project, checkCollection(ANONYMOUS_ID_MAPPING), "step" + idx, checkTableColumn(projectConfig.getUserColumn()), startDate.format(ISO_LOCAL_DATE), endDate.format(ISO_LOCAL_DATE), "step" + idx, checkTableColumn(projectConfig.getUserColumn())) : "", filterExp.map(v -> "where " + v).orElse("")); return format; }
String query; try (Connection connection = prestoConnectionFactory.openConnection()) { ResultSet columns = connection.getMetaData().getColumns(config.getColdStorageConnector(), project, collection, null); HashSet<String> strings = new HashSet<>(); while (columns.next()) { return currentFields; query = format("CREATE TABLE %s.\"%s\".\"%s\" (%s) WITH (temporal_column = '%s') ", config.getColdStorageConnector(), project, collection, queryEnd, checkLiteral(projectConfig.getTimeColumn())); connection.createStatement().execute(query); }) .map(f -> format("ALTER TABLE %s.\"%s\".\"%s\" ADD COLUMN \"%s\" %s", config.getColdStorageConnector(), project, collection, f.getName(), toSql(f.getType()))) .forEach(q -> {
if ("rakam_raptor".equals(prestoConfig.getColdStorageConnector())) { metadataDataSource = bindJDBCConfig(binder, "presto.metastore.jdbc"); if ("rakam_raptor".equals(prestoConfig.getColdStorageConnector())) { implementation = PrestoRakamRaptorMetastore.class; } else {
format(" }' as json, %s from %s where %s = %s %s", checkTableColumn(projectConfig.getTimeColumn()), "\"" + prestoConfig.getColdStorageConnector() + "\"" + ".\"" + context.project + "\"." + checkCollection(entry.getKey()), checkTableColumn(projectConfig.getUserColumn()), userType.get().isNumeric() ? user : "'" + user + "'",
"api-server", ImmutableSet.of(), null, prestoConfig.getColdStorageConnector(), "default", TimeZone.getTimeZone(ZoneOffset.UTC).getID(),
prestoConfig.getColdStorageConnector(), project, checkCollection(collection), queryEnd, properties);
prestoQuery = format("SELECT DISTINCT %s FROM %s.%s.%s TABLESAMPLE BERNOULLI (%f) WHERE %s IS NOT NULL", checkTableColumn(attribute), prestoConfig.getColdStorageConnector(), checkProject(project, '"'), checkCollection(collection, '"'),
@Override public int[] storeBatch(List<Event> events) { for (Map.Entry<String, List<Event>> collection : events.stream().collect(Collectors.groupingBy(e -> e.collection())).entrySet()) { QueryResult join = queryExecutor.executeRawStatement(String.format("INSERT INTO %s.%s.%s (_shard_time, %s) (%s)", config.getColdStorageConnector(), checkProject(events.get(0).project(), '"'), checkCollection(collection.getKey()), collection.getValue().get(0).schema().stream().map(e -> ValidationUtil.checkCollection(e.getName())) .collect(Collectors.joining(", ")), collection.getValue().stream() .map(e -> buildValues(e.properties(), e.schema())) .collect(Collectors.joining(" union all ")))) .getResult().join(); if (join.isFailed()) { try { Thread.sleep(300000); } catch (InterruptedException e) { e.printStackTrace(); } throw new IllegalStateException(join.getError().message); } } return EventStore.SUCCESSFUL_BATCH; }
@Override public void deleteProject(String project) { checkProject(project); try (Handle handle = dbi.open()) { handle.createStatement("delete from project where name = :project") .bind("project", project).execute(); } for (String collectionName : getCollectionNames(project)) { String query = format("DROP TABLE %s.\"%s\".\"%s\"", prestoConfig.getColdStorageConnector(), project, collectionName); QueryResult join = new PrestoQueryExecution(defaultSession, query, true).getResult().join(); if (join.isFailed()) { LOGGER.error("Error while deleting table %s.%s : %s", project, collectionName, join.getError().toString()); } } super.onDeleteProject(project); }
@Override public void store(Event event) { queryExecutor.executeRawStatement(String.format("INSERT INTO %s.%s.%s VALUES %s", config.getColdStorageConnector(), event.project(), event.collection(), buildValues(event.properties(), event.schema()))); }