public void testExceptions(QueryBatcher queryBatcher, int expectedSuccesses, int expectedFailures) { final AtomicInteger successfulBatchCount = new AtomicInteger(); final AtomicInteger failureBatchCount = new AtomicInteger(); queryBatcher .withBatchSize(1) .onUrisReady( batch -> successfulBatchCount.incrementAndGet() ) .onQueryFailure( queryThrowable -> failureBatchCount.incrementAndGet() ); moveMgr.startJob(queryBatcher); queryBatcher.awaitCompletion(); moveMgr.stopJob(queryBatcher); assertEquals(expectedSuccesses, successfulBatchCount.get()); assertEquals(expectedFailures, failureBatchCount.get()); }
.withBatchSize(batchSize) .withThreadCount(threadCount) .withJobId(jobId)
QueryBatcher batcher = moveMgr.newQueryBatcher(new StructuredQueryBuilder().collection(qhbTestCollection)) .withBatchSize(20) .withThreadCount(20);
QueryBatcher getUris = moveMgr.newQueryBatcher(query) .withThreadCount(5) .withBatchSize(100) .onUrisReady( new UrisToWriterListener(writer) ) .onUrisReady(batch -> successDocs1.addAndGet(batch.getItems().length)) .withBatchSize(19) .onUrisReady(batch -> successDocs2.addAndGet(batch.getItems().length)) .onQueryFailure( throwable -> {
QueryBatcher getUris = moveMgr.newQueryBatcher(query) .withThreadCount(6) .withBatchSize(5000) .onUrisReady( batch -> uris.addAll(Arrays.asList(batch.getItems())) ) .onQueryFailure( throwable -> { QueryBatcher performDelete = moveMgr.newQueryBatcher(uris.iterator()) .withThreadCount(2) .withBatchSize(99) .onUrisReady(new DeleteListener()) .onUrisReady(batch -> successDocs.addAndGet(batch.getItems().length))
StringBuilder failures = new StringBuilder(); QueryBatcher batcher = moveMgr.newQueryBatcher(uris.iterator()) .withBatchSize(1) .onUrisReady( new ApplyTransformListener()
QueryBatcher getUris = moveMgr.newQueryBatcher(query) .withForestConfig(ffg) .withBatchSize(2) .withThreadCount(5) .onUrisReady(batch -> retrievedUris.addAll(Arrays.asList(batch.getItems())) )
QueryBatcher exportBatcher = moveMgr.newQueryBatcher(query) .withThreadCount(1) .withBatchSize(30) .withConsistentSnapshot() .onUrisReady( .withBatchSize(10) .withConsistentSnapshot() .onUrisReady(new DeleteListener())
final StringBuffer failures = new StringBuffer(); queryBatcher .withBatchSize(batchSize) .withThreadCount(threadCount) .onUrisReady(
QueryBatcher qb = moveMgr.newQueryBatcher(uris.iterator()) .withThreadCount(2) .withBatchSize(99) .withConsistentSnapshot() .onUrisReady(new ExportListener()
.withBatchSize(5000) .onUrisReady( new UrisToWriterListener(writer) ) .onQueryFailure(exception -> exception.printStackTrace());
moveMgr.newQueryBatcher(query) .withThreadCount(5) .withBatchSize(10) .onUrisReady(exportListener) .onQueryFailure( throwable -> throwable.printStackTrace() );
final AtomicInteger count3 = new AtomicInteger(0); QueryBatcher batcher3 = moveMgr.newQueryBatcher(query3) .withBatchSize(100) .onUrisReady(batch -> count3.addAndGet(batch.getItems().length)) .onQueryFailure((throwable) -> throwable.printStackTrace());
Hashtable<String,AtomicInteger> urisDeleted = new Hashtable<>(); QueryBatcher batcher = moveMgr.newQueryBatcher(query) .withBatchSize(1) .withConsistentSnapshot() .onUrisReady(
@Test public void testQueryBatcher() { client.newDocumentManager().writeAs(collection + "/test1.json", meta, "[true]"); client.newDocumentManager().writeAs(collection + "/test1.xml", meta, "<xml/>"); client.newDocumentManager().writeAs(collection + "/test1.txt", meta, "text"); assertEquals(3, client.newQueryManager().search(collectionQuery, new SearchHandle()).getTotalResults()); StructuredQueryDefinition query = collectionQuery; // begin copy from "Using QueryBatcher" in src/main/java/com/marklogic/datamovement/package-info.java QueryBatcher qhb = dataMovementManager.newQueryBatcher(query) .withBatchSize(1000) .withThreadCount(20) .withConsistentSnapshot() .onUrisReady(batch -> { for ( String uri : batch.getItems() ) { if ( uri.endsWith(".txt") ) { client.newDocumentManager().delete(uri); } } }) .onQueryFailure(queryBatchException -> queryBatchException.printStackTrace()); JobTicket ticket = dataMovementManager.startJob(qhb); qhb.awaitCompletion(); dataMovementManager.stopJob(ticket); // end copy from "Using QueryBatcher" in src/main/java/com/marklogic/datamovement/package-info.java SearchHandle results = client.newQueryManager().search(collectionQuery, new SearchHandle()); assertEquals(2, results.getTotalResults()); for ( MatchDocumentSummary match : results.getMatchResults() ) { assertTrue(match.getUri().matches(".*/test1.(json|xml)")); } }
new StructuredQueryBuilder().collection(collection) .withBatchSize(10) .onUrisReady(new DeleteListener()) .withConsistentSnapshot();