@Override public boolean close() { iterator.close(); return true; }
public MongoPersistentCursor(MongoDBCollection collection, Bson query, Bson projection, QueryOptions options, int batchSize, int limit, int skip) { this.options = options; this.query = query; this.projection = projection; this.collection = collection; if (batchSize > 0) { this.batchSize = batchSize; } if (limit > 0) { this.limit = limit; } if (skip > 0) { this.skip = skip; } reset(); }
protected void reset() { count = 0; exceptions = 0; resume(null); }
MongoPersistentCursor iterator = new MongoPersistentCursor(variantsCollection, purgeQuery, queryParser.createProjection( new Query(), new QueryOptions(QueryOptions.INCLUDE, DocumentToVariantConverter.REQUIRED_FIELDS_SET)), new QueryOptions()); List<Document> documentsToInsert = new ArrayList<>(deleteBatchSize); while (iterator.hasNext()) { Document next = iterator.next(); documentsToDelete.add(next.getString("_id")); next.append(DocumentToTrashVariantConverter.TIMESTAMP_FIELD, ts); documentsToInsert.add(next); if (documentsToDelete.size() == deleteBatchSize || !iterator.hasNext()) { if (documentsToDelete.isEmpty()) {
@Override public boolean open() { //Filter documents with the selected studyId and chromosomes //Sorting by _id // FindIterable<Document> iterable = stageCollection.nativeQuery().find(getQuery(), // new QueryOptions(QueryOptions.SORT, Sorts.ascending("_id")) // ); // iterable.batchSize(20); // this.iterator = iterable.iterator(); QueryOptions options = getQueryOptions(); Bson query = getQuery(); iterator = new MongoPersistentCursor(stageCollection, query, null, options) .setBatchSize(20); return true; }
if (iterator.hasNext()) { last = iterator.next(); list.add(last); if (iterator.hasNext()) { int end = lastVar.getEnd(); String chr = lastVar.getChromosome(); while (iterator.hasNext()) { next = iterator.next(); Variant nextVar = MongoDBVariantStageLoader.STAGE_TO_VARIANT_CONVERTER.convertToDataModelType(next);
@Override public boolean hasNext() { try { return mongoCursor.hasNext(); } catch (MongoCursorNotFoundException e) { error(e); return mongoCursor.hasNext(); } }
static VariantMongoDBIterator persistentIterator(MongoDBCollection collection, Bson query, Bson projection, QueryOptions options, DocumentToVariantConverter converter) { StopWatch watch = StopWatch.createStarted(); MongoPersistentCursor cursor = new MongoPersistentCursor(collection, query, projection, options); VariantMongoDBIterator iterator = new VariantMongoDBIterator(cursor, converter); iterator.timeFetching += watch.getNanoTime(); return iterator; }
protected MongoPersistentCursor resume(Object lastObjectId) { Bson query; if (lastObjectId != null) { query = Filters.and(Filters.gt("_id", lastObjectId), this.query); } else { query = this.query; } FindIterable<Document> iterable = newFindIterable(query, this.projection, this.options); if (!options.containsKey(QueryOptions.SORT)) { iterable.sort(Sorts.ascending("$natural")); } mongoCursor = iterable .batchSize(batchSize) .limit(limit) .skip(skip) .iterator(); return this; }
@Override public Document next() { Document next; try { next = mongoCursor.next(); } catch (MongoCursorNotFoundException e) { error(e); next = mongoCursor.next(); } count++; lastId = next.get("_id"); return next; }
public MongoCursor<Document> nativeIterator(Query query, QueryOptions options, boolean persistent) { if (query == null) { query = new Query(); } if (options == null) { options = new QueryOptions(); } Document mongoQuery = queryParser.parseQuery(query); System.out.println("mongoQuery = " + mongoQuery); Document projection = queryParser.createProjection(query, options); System.out.println("projection = " + projection); options.putIfAbsent(MongoDBCollection.BATCH_SIZE, 100); if (persistent) { logger.debug("Using mongodb persistent iterator"); return new MongoPersistentCursor(variantsCollection, mongoQuery, projection, options); } else { FindIterable<Document> dbCursor = variantsCollection.nativeQuery().find(mongoQuery, projection, options); return dbCursor.iterator(); } }
@Override public Document tryNext() { Document next; try { next = mongoCursor.tryNext(); } catch (MongoCursorNotFoundException e) { error(e); next = mongoCursor.tryNext(); } if (next != null) { count++; lastId = next.get("_id"); } return next; }
private void error(MongoCursorNotFoundException e) { logger.warn("Resuming after " + MongoCursorNotFoundException.class, e); exceptions++; resume(lastId); }