private void logQuery(Operation op, ServiceDocumentQueryResult rsp, QuerySpecification qs, String sql, long queryTime) { if (isDebugQuery()) { appendDebugInfo(op, "ExecuteQuery time ms: " + queryTime, null, "ExecuteQuery count: " + rsp.documentCount, null, "Total query time ms: " + TimeUnit.MICROSECONDS.toMillis(rsp.queryTimeMicros), null, "QuerySpecification", Utils.toJsonHtml(qs), "SQL", prettySqlStatement(sql)); } // Log slow queries if (this.logSlowQueryThresholdMicros > 0 && rsp.queryTimeMicros > this.logSlowQueryThresholdMicros) { long totalQueryTimeMillis = TimeUnit.MICROSECONDS.toMillis(rsp.queryTimeMicros); Map<String, Object> map = new HashMap<>(); map.put("totalQueryTimeMillis", totalQueryTimeMillis); map.put("executeQueryTimeMillis", queryTime); map.put("documentCount", rsp.documentCount); map.put("sql", sql); map.put("querySpecification", qs); logger.warning(String.format("Slow SQL Query, %d ms: %s", totalQueryTimeMillis, Utils.toJson(map))); } }
/** * Performs SQL initialization for the given document type, e.g. creating a dedicated table for * it, etc. * * Expected to be called once per document type upon host startup. */ public void initForDocument(String factoryLink, Class<? extends ServiceDocument> documentType, ServiceDocumentDescription sdd) { this.schemaManager.addFactory(factoryLink, documentType, sdd); TableDescription desc = this.schemaManager.getTableDescriptionForFactoryLink(factoryLink); if (desc != null) { ensureTableExists(desc); } }
private Set<String> getKindScope(QuerySpecification qs) { Set<String> kindScope = new HashSet<>(4); findKindScopeFromQuery(qs.query, kindScope); if (kindScope.isEmpty()) { findFactoryScopeFromQuery(qs.query, kindScope); } if (kindScope.isEmpty()) { logger.fine(() -> String.format("Missing kind scope in query: %s", Utils.toJsonHtml(qs))); // Returning all scopes to allow do a UNION of all tables // Adding to existing kindScope mutable set to avoid Kryo cloning issues kindScope.addAll(this.schemaManager.getServiceDocuments()); } if (kindScope.isEmpty()) { throw new IllegalArgumentException("Cannot create SQL query without a kind scope"); } return kindScope; }
if (qs.context.kindScope == null) { qs.context.kindScope = getKindScope(qs); tables = kindScopeToTableDescriptions(qs.context.kindScope); qs.context.nativeSearcher = tables; return queryIndex(op, tables, null, postgresPage, task.querySpec.options, task.querySpec, task.querySpec.resultLimit, task.documentExpirationTimeMicros, task.nodeSelectorLink, task.indexLink);
String where = tq != null ? tq : PostgresQueryConverter.convert(qs.query, qs.context, td); where = updateQuery(op, td, where, queryStartTimeMicros, qs, false); if (where == null) { return rsp; String where = tq != null ? tq : PostgresQueryConverter.convert(qs.query, qs.context, td); where = updateQuery(op, td, where, queryStartTimeMicros, qs, false); if (where == null) { return null; logger.info(String.format( "SQL query: \n>>>>>>>>>> SQL BEGIN >>>>>>>>>>\n%s\n<<<<<<<<<< SQL END <<<<<<<<<<\n", prettySqlStatement(sql))); bottom = processQueryResults(qs, options, count, rsp, rs, queryStartTimeMicros, nodeSelectorLink, true); if (hasOffset) { bottom); rsp.nextPageLink = createNextPage(op, qs, null, sortFields, !hasOffset ? after : null, hasOffset ? offset : null, null, logQuery(op, rsp, qs, sql, queryTime);
loadDoc(visitor, rs); final String link = visitor.documentSelfLink; final String json = visitor.jsonSerializedState; || options.contains(QueryOption.OWNER_SELECTION) || options.contains(QueryOption.EXPAND_SELECTED_FIELDS)) { state = getStateFromPostgresDocument(null, visitor, link); if (!processQueryResultsForOwnerSelection(json, state, nodeSelectorPath)) { continue; o = toJsonObject(state); processQueryResultsForSelectLinks(qs, rsp, visitor, rs, link, state);
loadDoc(visitor, rs); state = getStateFromPostgresDocument(null, visitor, link); if (state == null) { logger.warning(() -> String.format("Skipping link term %s for %s, can "
PostgresSchemaManager getPostgresSchemaManager() { return this.postgresDocumentIndexService.getDao().getPostgresSchemaManager(); }
/** * Traverses the given query and find documentSelfLink constraints from which the factory query * scope is deducted. * * Takes all links into account no matter if they MUST_NOT_OCCUR - no need for such optimization * for now as our support for multiple kind scope will cope with that. */ private void findKindScopeFromQuery(Query query, Set<String> result) { if (query.booleanClauses != null) { for (Query innerQuery : query.booleanClauses) { findKindScopeFromQuery(innerQuery, result); } return; } if (query.term != null && ServiceDocument.FIELD_NAME_KIND.equals(query.term.propertyName) && query.occurance != QueryTask.Query.Occurance.MUST_NOT_OCCUR) { result.add(query.term.matchValue); } }
/** * Traverses the given query and find documentSelfLink constraints from which the factory query * scope is deducted. * * Takes all links into account no matter if they MUST_NOT_OCCUR - no need for such optimization * for now as our support for multiple kind scope will cope with that. */ private void findFactoryScopeFromQuery(Query query, Set<String> result) { if (query.booleanClauses != null) { for (Query innerQuery : query.booleanClauses) { findFactoryScopeFromQuery(innerQuery, result); } return; } if (query.term != null && ServiceDocument.FIELD_NAME_SELF_LINK.equals(query.term.propertyName) && !UriUtils.URI_WILDCARD_CHAR.equals(query.term.matchValue)) { String factoryLink = UriUtils.getParentPath(query.term.matchValue); if (factoryLink != null && !factoryLink.isEmpty() && !UriUtils.URI_PATH_CHAR.equals(factoryLink) && !factoryLink.contains(UriUtils.URI_WILDCARD_CHAR)) { String documentKind = this.schemaManager.getDocumentKindForFactoryLink(factoryLink); if (documentKind != null) { result.add(documentKind); } } } }
public PostgresDocumentIndexService(ServiceHost host, DataSource ds) { super(ServiceDocument.class); toggleOption(ServiceOption.CORE, true); toggleOption(ServiceOption.PERIODIC_MAINTENANCE, true); // TODO: define how often to expire documents setMaintenanceIntervalMicros(TimeUnit.SECONDS.toMicros(1)); setHost(host); this.ds = ds; this.dao = new PostgresServiceDocumentDao(host, this, ds); }
private void deleteAllDocumentsForSelfLink(Connection conn, String tableName, Operation postOrDelete, String link, ServiceDocument state) throws Exception { dao.deleteDocument(conn, tableName, link); postOrDelete.complete(); adjustTimeSeriesStat(STAT_NAME_SERVICE_DELETE_COUNT, AGGREGATION_TYPE_SUM, 1); logFine("%s expired", link); if (state == null) { return; } applyActiveQueries(postOrDelete, state, null); // remove service, if its running // Broadcasting delete to all nodes, to make sure owner node stop the service // TODO: Find better solution, all nodes query for expiration and stop if service owner? // TODO: Consider skipping delete for IMMUTABLE and non-periodic services, since they will // stop on idle // TODO: Why handleDelete is not called? Noticed same behavior with lucene Operation delete = Operation.createDelete(this, state.documentSelfLink) .setBodyNoCloning(state) .disableFailureLogging(true) .addPragmaDirective(Operation.PRAGMA_DIRECTIVE_NO_INDEX_UPDATE) .setReferer(getUri()); getHost().broadcastRequest(ServiceUriPaths.DEFAULT_NODE_SELECTOR, false, delete); }
String where = tq != null ? tq : PostgresQueryConverter.convert(qs.query, qs.context, td); where = updateQuery(op, td, where, startTimeMillis, qs, true); if (where == null) { return rsp; String where = tq != null ? tq : PostgresQueryConverter.convert(qs.query, qs.context, td); where = updateQuery(op, td, where, startTimeMillis, qs, true); if (where == null) { return null; logger.info(String.format( "SQL query: \n>>>>>>>>>> SQL BEGIN >>>>>>>>>>\n%s\n<<<<<<<<<< SQL END <<<<<<<<<<\n", prettySqlStatement(sql))); rsp.nextPageLink = createNextPage(op, qs, tq, groupSortFields, null, 0, groupLimit + groupOffset, expirationTimeMicros, indexLink, nodeSelectorLink, String pageLink = createNextPage(op, qsPerGroup, null, null, null, 0, null, expirationTimeMicros, indexLink, nodeSelectorLink, false); logQuery(op, rsp, qs, sql, queryTime);
loadDoc(visitor, rs); return getStateFromPostgresDocument(tableDescription, visitor, selfLink);
private void applyDocumentExpirationPolicy(long deadline) throws Exception { // TODO: need better solution to expire documents, this can be very slow to have // deletion in batches across tables int limit = expiredDocumentSearchThreshold; long now = Utils.getNowMicrosUtc(); for (TableDescription tableDescription : this.dao.getPostgresSchemaManager().getTableDescriptions()) { if (Utils.getSystemNowMicrosUtc() >= deadline || limit <= 0) { break; } int expired = applyDocumentExpirationPolicyForTable(tableDescription, now, deadline, limit); limit -= expired; } }
if (isDebugQuery()) { appendDebugInfo(op, "Auth Query", Utils.toJsonHtml(resourceQuery), "Auth SQL", prettySqlStatement(sb.toString()));
dao.loadDoc(visitor, rs); try { serviceDocument = this.dao .getStateFromPostgresDocument(tableDescription, visitor, documentSelfLink); } catch (Exception e) {