private RevTree forceTreeId(RevTreeBuilder b, ObjectId treeId) { RevTree tree = b.build(); long size = tree.size(); int childTreeCount = tree.numTrees(); ImmutableList<Node> trees = tree.trees(); ImmutableList<Node> features = tree.features(); SortedMap<Integer, Bucket> buckets = tree.buckets(); RevTree fakenId = RevTreeBuilder.create(treeId, size, childTreeCount, trees, features, buckets); return fakenId; }
public @Test void testRemoveWorldPoints() { List<Node> nodes = IndexTestSupport.createWorldPointsNodes(5); final RevTreeBuilder builder = createBuiler(); nodes.forEach((n) -> assertTrue(builder.put(n))); RevTree tree = builder.build(); assertEquals(nodes.size(), tree.size()); List<Node> removes = nodes.subList(0, nodes.size() / 2); final RevTreeBuilder builder2 = createBuiler(tree); removes.forEach((n) -> assertTrue(builder2.remove(n))); RevTree tree2 = builder2.build(); assertEquals(nodes.size() - removes.size(), tree2.size()); }
@Override public boolean feature(final @Nullable NodeRef left, @Nullable final NodeRef right) { final boolean cancelled = progress.isCanceled(); if (!cancelled) { if (left == null) { Node node = right.getNode(); boolean put = builder.put(node); checkState(put, "Node was not added to index: %s", node); } else if (right == null) { Node node = left.getNode(); boolean removed = builder.remove(node); checkState(removed, "Node was not removed from index: %s", node); } else { Node lnode = left.getNode(); Node rnode = right.getNode(); boolean updated = builder.update(lnode, rnode); if (!lnode.equals(rnode)) { checkState(updated, "Node %s was not updated to %s", lnode, rnode); } } progress.setProgress(count.incrementAndGet()); } return !cancelled; }
private RevTree createLeafTree(Collection<Node> featureNodes) { int childTreeCount = 0; ImmutableList<Node> trees = null; List<Node> sorted = new ArrayList<Node>(featureNodes); Collections.sort(sorted, CanonicalNodeOrder.INSTANCE); ImmutableList<Node> features = ImmutableList.copyOf(sorted); ImmutableSortedMap<Integer, Bucket> buckets = null; int size = features.size(); RevTree tree = RevTreeBuilder.build(size, childTreeCount, trees, features, buckets); return tree; }
private void addNode(RevTreeBuilder tree, int i) { Node ref = createNode(i); tree.put(ref); }
public static RevTree createLeafTree(long size, Collection<Node> features, Collection<Node> trees) { Preconditions.checkNotNull(features); Preconditions.checkNotNull(trees); ImmutableList<Node> featuresList = ImmutableList.of(); ImmutableList<Node> treesList = ImmutableList.of(); if (!features.isEmpty()) { featuresList = NODE_STORAGE_ORDER.immutableSortedCopy(features); } if (!trees.isEmpty()) { treesList = NODE_STORAGE_ORDER.immutableSortedCopy(trees); } final ObjectId id = HashObject.hashTree(treesList, featuresList, null); return RevTreeBuilder.create(id, size, trees.size(), treesList, featuresList, null); }
public static RevTree createFeaturesTree(ObjectStore source, final String namePrefix, final int numEntries, final int startIndex, boolean randomIds) { RevTree tree = createFeaturesTreeBuilder(source, namePrefix, numEntries, startIndex, randomIds).build(); source.put(tree); return tree; }
boolean put = builder.put(right); checkState(put, "Node was not added to index: %s", right); } else if (right == null) { boolean removed = builder.remove(left); checkState(removed, "Node was not removed from index: %s", left); } else { boolean updated = builder.update(left, right); if (!left.equals(right)) { checkState(updated, "Node %s was not updated to %s", left, right);
public static RevTreeBuilder createLargeFeaturesTreeBuilder(ObjectDatabase source, final String namePrefix, final int numEntries, final int startIndex, boolean randomIds) { RevTreeBuilder tree = CanonicalTreeBuilder.create(source); for (int i = startIndex; i < startIndex + numEntries; i++) { tree.put(featureNode(namePrefix, i, randomIds)); } return tree; }
private RevTree createNodeTree(long size, int numTrees, TreeMap<Integer, Bucket> buckets) { ImmutableSortedMap<Integer, Bucket> innerTrees = ImmutableSortedMap.copyOf(buckets); final ObjectId id = HashObject.hashTree(null, null, innerTrees); return RevTreeBuilder.create(id, size, numTrees, null, null, innerTrees); }
public static RevTree createTreesTree(ObjectStore source, int numSubTrees, int featuresPerSubtre, ObjectId metadataId) { RevTree tree = createTreesTreeBuilder(source, numSubTrees, featuresPerSubtre, metadataId) .build(); source.put(tree); return tree; }
public @Test void testRemoveWorldPointsWholeQuadrant() { List<Node> nodes = IndexTestSupport.createWorldPointsNodes(5); final RevTreeBuilder builder = createBuiler(); nodes.forEach((n) -> assertTrue(builder.put(n))); RevTree tree = builder.build(); assertEquals(nodes.size(), tree.size()); final Envelope NEBounds = new Envelope(0, 180, 0, 90); List<Node> removes = new ArrayList<>(); for (Node n : nodes) { if (NEBounds.contains(n.bounds().orNull())) { removes.add(n); } } final RevTreeBuilder builder2 = createBuiler(tree); removes.forEach((n) -> assertTrue(builder2.remove(n))); RevTree tree2 = builder2.build(); assertEquals(nodes.size() - removes.size(), tree2.size()); } }
public static RevTreeBuilder createFeaturesTreeBuilder(ObjectStore source, final String namePrefix, final int numEntries, final int startIndex, boolean randomIds) { RevTreeBuilder tree = CanonicalTreeBuilder.create(source); for (int i = startIndex; i < startIndex + numEntries; i++) { tree.put(featureNode(namePrefix, i, randomIds)); } return tree; }
id = HashObject.hashTree(trees, features, buckets); RevTree tree = RevTreeBuilder.create(id, size, treeCount, trees, features, buckets); return tree;
protected ObjectId createAndSaveTree(final List<Node> nodes, final boolean insertInListOrder) throws Exception { List<Node> insert = nodes; if (!insertInListOrder) { insert = new ArrayList<>(nodes); Collections.shuffle(insert); } RevTreeBuilder treeBuilder = createBuiler(); nodes.forEach((n) -> treeBuilder.put(n)); RevTree tree = treeBuilder.build(); assertTrue(objectStore.exists(tree.getId())); assertEquals(nodes.size(), tree.size()); return tree.getId(); }
private RevTree buildLeafTree(DAG root) { if (state.isCancelled()) { return null; } checkState(root.numBuckets() == 0); final ImmutableList<Node> children; { Set<NodeId> childrenIds = new HashSet<>(); root.forEachChild((id) -> childrenIds.add(id)); children = toNodes(childrenIds); } ImmutableList<Node> treesList = ImmutableList.copyOf(Iterables.filter(children, n -> n.getType().equals(TYPE.TREE) && !n.getObjectId().isNull())); ImmutableList<Node> featuresList = ImmutableList.copyOf(Iterables.filter(children, n -> n.getType().equals(TYPE.FEATURE) && !n.getObjectId().isNull())); final long size = sumTreeSizes(treesList) + featuresList.size(); final int childTreeCount = treesList.size(); ImmutableSortedMap<Integer, Bucket> buckets = null; if (state.isCancelled()) { return null; } RevTree tree = RevTreeBuilder.build(size, childTreeCount, treesList, featuresList, buckets); return tree; }
private RevTreeBuilder createTree(final int size, final Iterable<Node> nodes, final RevTreeBuilder b, Stopwatch totalTime) { Preconditions.checkArgument(!totalTime.isRunning()); System.err.print("\tInserting nodes..."); int count = 0, s = 0; final int step = size / 100; Stopwatch sw = Stopwatch.createUnstarted(); // Iterable<List<Node>> partitions = Iterables.partition(nodes, 100_000); // for (List<Node> partition : partitions) { sw.start(); totalTime.start(); for (Node n : nodes) { count++; s++; b.put(n); if (s == step) { s = 0; System.err.print('#'); } } sw.stop(); totalTime.stop(); // } System.err.printf("\n%,d nodes inserted in %s\n", count, sw); System.err.flush(); return b; }