@Override public List<SchemaField> getOrCreateCollectionFields(String project, String collection, Set<SchemaField> fields) { ValidationUtil.checkCollectionValid(collection); return getOrCreateCollectionFields(project, collection, fields, fields.size() + 1); }
@Override public Map<String, List<SchemaField>> getCollections(String project) { return getTables(project, this::filterTables); }
private static String sqlColumnType(FieldType type) { JDBCType jdbcType = jdbcType(type); if (jdbcType != null) { switch (jdbcType) { case BOOLEAN: return "boolean"; case BIGINT: return "bigint"; case DOUBLE: return "double"; case INTEGER: return "int"; case VARBINARY: return format("varbinary(%s)", MAX_BINARY_INDEX_SIZE); } } return null; }
public List<SchemaField> getOrCreateCollectionFields(String project, String collection, Set<SchemaField> fields, int tryCount) { String query; List<SchemaField> schemaFields = getCollection(project, collection); List<SchemaField> lastFields; Table tableInformation = dao.getTableInformation(project, collection); if (!getProjects().contains(project)) { throw new NotExistsException("Project"); return f; }) .map(f -> format("\"%s\" %s", f.getName(), toSql(f.getType()))) .collect(Collectors.joining(", ")); if (queryEnd.isEmpty()) { queryEnd += format(", \"%s\" %s", prestoConfig.getCheckpointColumn(), toSql(TIMESTAMP)); if (join.getError().message.contains("exists") || join.getError().message.equals(METADATA_ERROR)) { if (tryCount > 0) { return getOrCreateCollectionFields(project, collection, fields, tryCount - 1); } else { String description = format("%s.%s: %s: %s", addColumn(tableInformation, project, collection, f.getName(), f.getType()); } catch (Exception e) { if (e.getMessage().equals(METADATA_ERROR) && (e.getCause().getMessage().contains(JDBI_CONFLICT_ERROR) || e.getCause().getMessage().contains(JDBI_DUPLICATE_ERROR))) { for (int i = 0; i < 50; i++) { try { addColumn(tableInformation, project, collection, f.getName(), f.getType()); } catch (Exception ex) {
@BeforeSuite @Override public void setup() throws Exception { testingEnvironment = new TestingEnvironment(); PrestoConfig prestoConfig = testingEnvironment.getPrestoConfig(); metastore = new PrestoRakamRaptorMetastore(testingEnvironment.getPrestoMetastore(), new EventBus(), new ProjectConfig(), prestoConfig); metastore.setup(); prestoQueryExecutor = new PrestoQueryExecutor(new ProjectConfig(), prestoConfig, null, metastore); materializedViewService = new PrestoMaterializedViewService( new PrestoConfig(), prestoQueryExecutor, metastore, getDatabaseMetadataStore(), Clock.systemUTC()); eventStore = new TestingPrestoEventStore(prestoQueryExecutor, prestoConfig); super.setup(); }
private void addColumn(Table table, String schema, String tableName, String columnName, FieldType fieldType) { List<TableColumn> existingColumns = dao.listTableColumns(schema, tableName); TableColumn lastColumn = existingColumns.get(existingColumns.size() - 1); long columnId = lastColumn.getColumnId() + 1; int ordinalPosition = existingColumns.size(); String type = TypeSignature.parseTypeSignature(toSql(fieldType)).toString().toLowerCase(ENGLISH); daoTransaction(dbi, MetadataDao.class, dao -> { dao.insertColumn(table.getTableId(), columnId, columnName, ordinalPosition, type, null, null); dao.updateTableVersion(table.getTableId(), System.currentTimeMillis()); }); String columnType = sqlColumnType(fieldType); if (columnType == null) { return; } String sql = format("ALTER TABLE %s ADD COLUMN (%s %s, %s %s)", shardIndexTable(table.getTableId()), minColumn(columnId), columnType, maxColumn(columnId), columnType); try (Handle handle = dbi.open()) { handle.execute(sql); } catch (DBIException e) { throw metadataError(e); } }
public static <T> void daoTransaction(IDBI dbi, Class<T> daoType, Consumer<T> callback) { runTransaction(dbi, (handle, status) -> { callback.accept(handle.attach(daoType)); return null; }); }
@Override public void createProject(String project) { checkProject(project); try (Handle handle = dbi.open()) { try { handle.createStatement("INSERT INTO project (name) VALUES(:name)") .bind("name", project) .execute(); } catch (Exception e) { if (getProjects().contains(project)) { throw new AlreadyExistsException("project", BAD_REQUEST); } } } super.onCreateProject(project); }
@Override public void deleteProject(String project) { checkProject(project); try (Handle handle = dbi.open()) { handle.createStatement("delete from project where name = :project") .bind("project", project).execute(); } for (String collectionName : getCollectionNames(project)) { String query = format("DROP TABLE %s.\"%s\".\"%s\"", prestoConfig.getColdStorageConnector(), project, collectionName); QueryResult join = new PrestoQueryExecution(defaultSession, query, true).getResult().join(); if (join.isFailed()) { LOGGER.error("Error while deleting table %s.%s : %s", project, collectionName, join.getError().toString()); } } super.onDeleteProject(project); }
@BeforeMethod public void setUpMethod() throws Exception { TestingEnvironment testingEnvironment = new TestingEnvironment(); metastore = new PrestoRakamRaptorMetastore(testingEnvironment.getPrestoMetastore(), new EventBus(), new ProjectConfig(), testingEnvironment.getPrestoConfig()); metastore.setup(); }
@BeforeSuite public void setup() throws Exception { testingEnvironment = new TestingEnvironment(); PrestoConfig prestoConfig = testingEnvironment.getPrestoConfig(); InMemoryQueryMetadataStore queryMetadataStore = new InMemoryQueryMetadataStore(); metastore = new PrestoRakamRaptorMetastore(testingEnvironment.getPrestoMetastore(), new EventBus(), new ProjectConfig(), prestoConfig); metastore.setup(); PrestoQueryExecutor queryExecutor = new PrestoQueryExecutor(new ProjectConfig(), prestoConfig, null, metastore); PrestoMaterializedViewService materializedViewService = new PrestoMaterializedViewService( new PrestoConfig(), queryExecutor, metastore, queryMetadataStore, Clock.systemUTC()); QueryExecutorService queryExecutorService = new QueryExecutorService(queryExecutor, metastore, materializedViewService, '"'); retentionQueryExecutor = new PrestoRetentionQueryExecutor(new ProjectConfig(), queryExecutorService, metastore, materializedViewService, new UserPluginConfig()); testingPrestoEventStore = new TestingPrestoEventStore(queryExecutor, prestoConfig); // TODO: Presto throws "No node available" error, find a way to avoid this ugly hack. Thread.sleep(1000); super.setup(); }
@BeforeSuite @Override public void setup() throws Exception { testingEnvironment = new TestingEnvironment(); PrestoConfig prestoConfig = testingEnvironment.getPrestoConfig(); JDBCConfig postgresqlConfig = testingEnvironment.getPostgresqlConfig(); metastoreDataSource = JDBCPoolDataSource.getOrCreateDataSource(postgresqlConfig); queryMetadataStore = new InMemoryQueryMetadataStore(); metastore = new PrestoRakamRaptorMetastore(testingEnvironment.getPrestoMetastore(), new EventBus(), new ProjectConfig(), prestoConfig); metastore.setup(); prestoQueryExecutor = new PrestoQueryExecutor(new ProjectConfig(), prestoConfig, null, metastore); PrestoMaterializedViewService materializedViewService = new PrestoMaterializedViewService( new PrestoConfig(), prestoQueryExecutor, metastore, queryMetadataStore, Clock.systemUTC()); QueryExecutorService queryExecutorService = new QueryExecutorService(prestoQueryExecutor, metastore, materializedViewService, '"'); eventExplorer = new PrestoEventExplorer(new ProjectConfig(), queryExecutorService, materializedViewService); eventStore = new TestingPrestoEventStore(prestoQueryExecutor, prestoConfig); super.setup(); }
@Override public Map<String, List<SchemaField>> getSchemas(String project, Predicate<String> filter) { return getTables(project, (t, c) -> filter.test(t)); }
@BeforeSuite @Override public void setup() throws Exception { testingEnvironment = new TestingEnvironment(); PrestoConfig prestoConfig = testingEnvironment.getPrestoConfig(); InMemoryQueryMetadataStore inMemoryQueryMetadataStore = new InMemoryQueryMetadataStore(); JDBCPoolDataSource prestoMetastore = testingEnvironment.getPrestoMetastore(); EventBus eventBus = new EventBus(); metastore = new PrestoRakamRaptorMetastore(prestoMetastore, eventBus, new ProjectConfig(), prestoConfig); metastore.setup(); PrestoQueryExecutor prestoQueryExecutor = new PrestoQueryExecutor(new ProjectConfig(), prestoConfig, null, metastore); PrestoMaterializedViewService materializedViewService = new PrestoMaterializedViewService( new PrestoConfig(), prestoQueryExecutor, metastore, inMemoryQueryMetadataStore, Clock.systemUTC()); QueryExecutorService queryExecutorService = new QueryExecutorService(prestoQueryExecutor, metastore, materializedViewService, '"'); FastGenericFunnelQueryExecutor fastGenericFunnelQueryExecutor = new FastGenericFunnelQueryExecutor(queryExecutorService, new ProjectConfig(), metastore); PrestoApproxFunnelQueryExecutor prestoApproxFunnelQueryExecutor = new PrestoApproxFunnelQueryExecutor(new ProjectConfig(), queryExecutorService, metastore); funnelQueryExecutor = new PrestoFunnelQueryExecutor(new ProjectConfig(), new PrestoConfig(), fastGenericFunnelQueryExecutor, prestoApproxFunnelQueryExecutor, metastore, prestoQueryExecutor, new UserPluginConfig()); testingPrestoEventStore = new TestingPrestoEventStore(prestoQueryExecutor, prestoConfig); super.setup(); }