private Range createUniversalPluginScanRange(String namespace, @Nullable String type) { List<Field<?>> keys = new ArrayList<>(); keys.add(Fields.stringField(StoreDefinition.ArtifactStore.NAMESPACE_FIELD, namespace)); if (type != null) { keys.add(Fields.stringField(StoreDefinition.ArtifactStore.PLUGIN_TYPE_FIELD, type)); } return Range.singleton(keys); }
/** * Create a range with a begin and an end. * * @param begin the fields forming the beginning of the range * @param beginBound the match type for the begin fields * @param end the fields forming the end of the range * @param endBound the match type for the end fields * @return a range object */ public static Range create(Collection<Field<?>> begin, Bound beginBound, Collection<Field<?>> end, Bound endBound) { return new Range(begin, beginBound, end, endBound); }
private void appendRange(StringBuilder statement, Range range) { appendScanBound(statement, range.getBegin(), range.getBeginBound().equals(Range.Bound.INCLUSIVE) ? ">=" : ">"); if (!range.getBegin().isEmpty() && !range.getEnd().isEmpty()) { statement.append(" AND "); } appendScanBound(statement, range.getEnd(), range.getEndBound().equals(Range.Bound.INCLUSIVE) ? "<=" : "<"); }
private String getDeleteAllStatement(Range range) { StringBuilder statement = new StringBuilder("DELETE FROM ").append(tableSchema.getTableId().getName()); if (!range.getBegin().isEmpty() || !range.getEnd().isEmpty()) { statement.append(" WHERE "); appendRange(statement, range); } return statement.toString(); }
@Test public void testSimpleScan() throws Exception { int max = 100; List<Collection<Field<?>>> expected = writeSimpleStructuredRows(max, ""); List<Collection<Field<?>>> actual = scanSimpleStructuredRows( Range.create(Collections.singleton(Fields.intField(KEY, 5)), Range.Bound.INCLUSIVE, Collections.singleton(Fields.intField(KEY, 15)), Range.Bound.EXCLUSIVE), max); Assert.assertEquals(expected.subList(5, 15), actual); actual = scanSimpleStructuredRows( Range.create(Collections.singleton(Fields.intField(KEY, 5)), Range.Bound.EXCLUSIVE, Collections.singleton(Fields.intField(KEY, 15)), Range.Bound.INCLUSIVE), max); Assert.assertEquals(expected.subList(6, 16), actual); actual = scanSimpleStructuredRows( Range.singleton(Collections.singleton(Fields.intField(KEY, 46))), max); Assert.assertEquals(expected.subList(46, 47), actual); // TODO: test invalid range // TODO: test begin only range // TODO: test end only range }
getTransactionRunner().run(context -> { StructuredTable table = context.getTable(SIMPLE_TABLE); Range range = Range.create(Arrays.asList(Fields.intField(KEY, 6), Fields.longField(KEY2, 6L)), Range.Bound.INCLUSIVE, Arrays.asList(Fields.intField(KEY, 8), Fields.longField(KEY2, 8L)), Assert.assertEquals(expected, scanSimpleStructuredRows(Range.all(), max)); getTransactionRunner().run(context -> { StructuredTable table = context.getTable(SIMPLE_TABLE); Range range = Range.create(Collections.singletonList(Fields.intField(KEY, 2)), Range.Bound.INCLUSIVE, Collections.singletonList(Fields.intField(KEY, 5)), Range.Bound.EXCLUSIVE); table.deleteAll(range); }); Assert.assertEquals(expected, scanSimpleStructuredRows(Range.all(), max)); Range range = Range.create(Collections.singletonList(Fields.intField(KEY, max + 1)), Range.Bound.INCLUSIVE, Collections.singletonList(Fields.intField(KEY, max + 5)), Range.Bound.EXCLUSIVE); table.deleteAll(range); }); Assert.assertEquals(expected, scanSimpleStructuredRows(Range.all(), max)); getTransactionRunner().run(context -> { StructuredTable table = context.getTable(SIMPLE_TABLE); table.deleteAll(Range.all()); });
@Test public void testMultipleKeyScan() throws Exception { int max = 10; // Write rows and read them, the rows will have keys (0, 0L), (2, 2L), ..., (9, 9L) List<Collection<Field<?>>> expected = writeSimpleStructuredRows(max, ""); List<Collection<Field<?>>> actual = readSimpleStructuredRows(max); Assert.assertEquals(expected, actual); // scan from (1, 8L) inclusive to (3, 3L) inclusive, should return (2, 2L) and (3, 3L) actual = scanSimpleStructuredRows( Range.create(ImmutableList.of(Fields.intField(KEY, 1), Fields.longField(KEY2, 8L)), Range.Bound.INCLUSIVE, ImmutableList.of(Fields.intField(KEY, 3), Fields.longField(KEY2, 3L)), Range.Bound.INCLUSIVE), max); Assert.assertEquals(expected.subList(2, 4), actual); // scan from (1, 8L) inclusive to (3, 3L) exclusive, should only return (2, 2L) actual = scanSimpleStructuredRows( Range.create(ImmutableList.of(Fields.intField(KEY, 1), Fields.longField(KEY2, 8L)), Range.Bound.INCLUSIVE, ImmutableList.of(Fields.intField(KEY, 3), Fields.longField(KEY2, 3L)), Range.Bound.EXCLUSIVE), max); Assert.assertEquals(expected.subList(2, 3), actual); }
/** * Get the scan query for the range given. For example, if the range provides key1, key2 as the begin and end to * scan, both rows are inclusive, it will generate the following query: * SELECT * FROM simpletable WHERE (key1,key2)>=(?,?) AND (key1,key2)<=(?,?) LIMIT 10; * * @param range the range to scan. * @param limit limit number of row * @return the scan query */ private String getScanQuery(Range range, int limit) { StringBuilder queryString = new StringBuilder("SELECT * FROM ").append(tableSchema.getTableId().getName()); if (range.getBegin().isEmpty() && range.getEnd().isEmpty()) { return queryString.append(" LIMIT ").append(limit).append(";").toString(); } queryString.append(" WHERE "); appendRange(queryString, range); queryString.append(" LIMIT ").append(limit).append(";"); return queryString.toString(); }
private Range createArtifactScanRange(NamespaceId namespace) { Field<String> stringField = Fields.stringField(StoreDefinition.ArtifactStore.ARTIFACT_NAMESPACE_FIELD, namespace.getNamespace()); return Range.singleton(Collections.singleton(stringField)); }
private Scanner getScanner(Range keyRange) { // the method will always prepend the table name as prefix byte[] begin = convertKeyToBytes(keyRange.getBegin(), true); byte[] end = convertKeyToBytes(keyRange.getEnd(), true); // Table.scan() start key is inclusive by default, and if it is EXCLUSTIVE, we want to ensure the start keys are // not empty so that we do not scan from the start of some other table if (!keyRange.getBegin().isEmpty() && keyRange.getBeginBound() == Range.Bound.EXCLUSIVE) { begin = Bytes.stopKeyForPrefix(begin); } // Table.scan() stop key is exclusive by default, so when the end keys are not specifies, we will need to scan to // the end of table, which will be the default table prefix + 1. if (keyRange.getEnd().isEmpty() || keyRange.getEndBound() == Range.Bound.INCLUSIVE) { end = Bytes.stopKeyForPrefix(end); } return table.scan(begin, end); }
@Override public void deleteAll(Range keyRange) throws InvalidFieldException, IOException { LOG.trace("Table {}: DeleteAll with range {}", tableSchema.getTableId(), keyRange); fieldValidator.validatePrimaryKeys(keyRange.getBegin(), true); fieldValidator.validatePrimaryKeys(keyRange.getEnd(), true); String sql = getDeleteAllStatement(keyRange); try (PreparedStatement statement = connection.prepareStatement(sql)) { int index = 1; if (keyRange.getBegin() != null) { for (Field<?> key : keyRange.getBegin()) { setField(statement, key, index); index++; } } if (keyRange.getEnd() != null) { for (Field<?> key : keyRange.getEnd()) { setField(statement, key, index); index++; } } LOG.trace("SQL statement: {}", statement); statement.executeUpdate(); } catch (SQLException e) { throw new IOException(String.format("Failed to delete the rows from table %s with range %s", tableSchema.getTableId().getName(), keyRange), e); } }
/** * Creates a range that only matches one element. This range will read all elements which is equal to the * given keys. * * @param singleton the fields forming the singleton range * @return a range object */ public static Range singleton(Collection<Field<?>> singleton) { return new Range(singleton, Bound.INCLUSIVE, singleton, Bound.INCLUSIVE); }
private Range createAppClassRange(NamespaceId namespace) { return Range.singleton(Collections.singleton(Fields.stringField(StoreDefinition.ArtifactStore.NAMESPACE_FIELD, namespace.getNamespace()))); }
@Override public CloseableIterator<StructuredRow> scan(Range keyRange, int limit) throws InvalidFieldException, IOException { LOG.trace("Table {}: Scan range {} with limit {}", tableSchema.getTableId(), keyRange, limit); fieldValidator.validatePrimaryKeys(keyRange.getBegin(), true); fieldValidator.validatePrimaryKeys(keyRange.getEnd(), true); String scanQuery = getScanQuery(keyRange, limit); // We don't close the statement here because once it is closed, the result set is also closed. try { PreparedStatement statement = connection.prepareStatement(scanQuery); int index = 1; if (keyRange.getBegin() != null) { for (Field<?> key : keyRange.getBegin()) { setField(statement, key, index); index++; } } if (keyRange.getEnd() != null) { for (Field<?> key : keyRange.getEnd()) { setField(statement, key, index); index++; } } LOG.trace("SQL statement: {}", statement); ResultSet resultSet = statement.executeQuery(); return new ResultSetIterator(statement, resultSet, tableSchema); } catch (SQLException e) { throw new IOException(String.format("Failed to scan from table %s with range %s", tableSchema.getTableId().getName(), keyRange), e); } }
/** * Create a range that only has an end point. It will include all values less than (or equal to) the end point. * * @param end the fields forming the end of the range * @param endBound the match type for the end fields * @return a range object */ public static Range to(Collection<Field<?>> end, Bound endBound) { return new Range(Collections.emptySet(), Bound.INCLUSIVE, end, endBound); }
private Range createPluginScanRange(Id.Artifact parentArtifactId, @Nullable String type) { List<Field<?>> keys = new ArrayList<>(); keys.add(Fields.stringField(StoreDefinition.ArtifactStore.PARENT_NAMESPACE_FIELD, parentArtifactId.getNamespace().getId())); keys.add(Fields.stringField(StoreDefinition.ArtifactStore.PARENT_NAME_FIELD, parentArtifactId.getName())); if (type != null) { keys.add(Fields.stringField(StoreDefinition.ArtifactStore.PLUGIN_TYPE_FIELD, type)); } return Range.singleton(keys); }
/** * Create a range that starts from a begin point, but does not have an end. It will include all values that are * greater than (or equal to) the begin point. * * @param begin the fields forming the beginning of the range * @param beginBound the match type for the begin fields * @return a range object */ public static Range from(Collection<Field<?>> begin, Bound beginBound) { return new Range(begin, beginBound, Collections.emptySet(), Bound.INCLUSIVE); }
/** * Get all application classes that belong to the specified namespace. * Results are returned as a sorted map from artifact to application classes in that artifact. * Map entries are sorted by the artifact. * * @param namespace the namespace from which to get application classes * @return an unmodifiable map of artifact to a list of all application classes in that artifact. * The map will never be null. If there are no application classes, an empty map will be returned. */ public SortedMap<ArtifactDescriptor, List<ApplicationClass>> getApplicationClasses(NamespaceId namespace) { return TransactionRunners.run(transactionRunner, context -> { SortedMap<ArtifactDescriptor, List<ApplicationClass>> result = Maps.newTreeMap(); StructuredTable table = context.getTable(StoreDefinition.ArtifactStore.APP_DATA_TABLE); Collection<Field<?>> keys = Collections.singleton( Fields.stringField(StoreDefinition.ArtifactStore.NAMESPACE_FIELD, namespace.getNamespace())); try (CloseableIterator<StructuredRow> iterator = table.scan(Range.singleton(keys), Integer.MAX_VALUE)) { while (iterator.hasNext()) { StructuredRow row = iterator.next(); Map.Entry<ArtifactDescriptor, ApplicationClass> entry = extractApplicationClass(row); List<ApplicationClass> existingAppClasses = result.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()); existingAppClasses.add(entry.getValue()); } } return Collections.unmodifiableSortedMap(result); }); }
/** * Create a range that matches all the elements. * * @return a range object */ public static Range all() { return new Range(Collections.emptySet(), Bound.INCLUSIVE, Collections.emptySet(), Bound.INCLUSIVE); }
private List<ArtifactDetail> getArtifacts(StructuredTable artifactDataTable, ArtifactRange range, int limit, ArtifactSortOrder order) throws IOException { Collection<Field<?>> keys = Arrays.asList(Fields.stringField(StoreDefinition.ArtifactStore.ARTIFACT_NAMESPACE_FIELD, range.getNamespace()), Fields.stringField(StoreDefinition.ArtifactStore.ARTIFACT_NAME_FIELD, range.getName())); // here we have to query with Integer.MAX_VALUE since we want to sort the result try (CloseableIterator<StructuredRow> iterator = artifactDataTable.scan(Range.singleton(keys), Integer.MAX_VALUE)) { return getArtifacts(iterator, limit, order, range); } }