@Override public QueryBatcherJobTicket run(DatabaseClient databaseClient) { DataMovementManager dmm = this.dataMovementManager != null ? this.dataMovementManager : databaseClient.newDataMovementManager(); String jobDescription = getJobDescription(); if (jobDescription != null && logger.isInfoEnabled()) { logger.info(jobDescription); } QueryBatcherBuilder builder = newQueryBatcherBuilder(); QueryBatcher queryBatcher = builder.buildQueryBatcher(databaseClient, dmm); prepareQueryBatcher(queryBatcher); JobTicket jobTicket = dmm.startJob(queryBatcher); if (awaitCompletion) { queryBatcher.awaitCompletion(); if (stopJobAfterCompletion) { dmm.stopJob(queryBatcher); } if (jobDescription != null && logger.isInfoEnabled()) { logger.info("Completed: " + jobDescription); } } return new QueryBatcherJobTicket(dmm, queryBatcher, jobTicket); }
public void testExceptions(QueryBatcher queryBatcher, int expectedSuccesses, int expectedFailures) { final AtomicInteger successfulBatchCount = new AtomicInteger(); final AtomicInteger failureBatchCount = new AtomicInteger(); queryBatcher .withBatchSize(1) .onUrisReady( batch -> successfulBatchCount.incrementAndGet() ) .onQueryFailure( queryThrowable -> failureBatchCount.incrementAndGet() ); moveMgr.startJob(queryBatcher); queryBatcher.awaitCompletion(); moveMgr.stopJob(queryBatcher); assertEquals(expectedSuccesses, successfulBatchCount.get()); assertEquals(expectedFailures, failureBatchCount.get()); }
private void testListenerException(QueryBatchListener listener) { final AtomicInteger failureBatchCount = new AtomicInteger(); Iterator<String> iterator = Arrays.asList(new String[] {uri1}).iterator(); QueryBatcher queryBatcher = moveMgr.newQueryBatcher(iterator) .onUrisReady( batch -> logger.debug("uri={}", batch.getItems()[0]) ) .onUrisReady(listener) .onQueryFailure( queryThrowable -> failureBatchCount.incrementAndGet() ); moveMgr.startJob(queryBatcher); queryBatcher.awaitCompletion(); moveMgr.stopJob(queryBatcher); // there should be no failure sent to the QueryBatcher onQueryFailure listeners assertEquals(0, failureBatchCount.get()); }
batcher.awaitCompletion(Long.MAX_VALUE, TimeUnit.DAYS); batcher.awaitCompletion(Long.MAX_VALUE, TimeUnit.DAYS);
}); moveMgr.startJob(getUris); getUris.awaitCompletion(); writer.flush(); }); moveMgr.startJob(doNothing); doNothing.awaitCompletion(); moveMgr.stopJob(doNothing);
}); moveMgr.startJob(getUris); getUris.awaitCompletion(); if ( failures.length() > 0 ) fail(failures.toString()); }); moveMgr.startJob(performDelete); performDelete.awaitCompletion();
batcher.awaitCompletion(); moveMgr.stopJob(ticket);
JobTicket jobTicket = dmm.startJob(batcher); batcher.awaitCompletion(); dmm.stopJob(batcher); dmm.release(); jobTicket = dmm.startJob(batcher); batcher.awaitCompletion(); dmm.stopJob(batcher); dmm.release();
queryBatcher.awaitCompletion(); moveMgr.stopJob(queryBatcher); assertTrue("Close method is not called on QueryBatchListener", calledBatchListener.get());
.onQueryFailure(exception -> exception.printStackTrace() ); moveMgr.startJob(getUris); getUris.awaitCompletion();
.onQueryFailure(failListener); moveMgr.startJob(deleteBatcher); deleteBatcher.awaitCompletion(); exportBatcher.awaitCompletion();
.onQueryFailure(exception -> exception.printStackTrace()); JobTicket getUrisTicket = dataMovementManager.startJob(getUris); getUris.awaitCompletion(); dataMovementManager.stopJob(getUrisTicket); writer.flush(); .onQueryFailure(exception-> exception.printStackTrace()); JobTicket ticket = dataMovementManager.startJob(performDelete); performDelete.awaitCompletion(); dataMovementManager.stopJob(ticket);
.onQueryFailure(exception -> exception.printStackTrace()); moveMgr.startJob(qb); qb.awaitCompletion(); moveMgr.stopJob(qb);
boolean finished = queryJob.awaitCompletion(3, TimeUnit.MINUTES); if ( finished == false ) { throw new IllegalStateException("ERROR: Job did not finish within three minutes");
.withConsistentSnapshot(); JobTicket ticket2 = moveMgr.startJob( batcher ); batcher.awaitCompletion(); moveMgr.stopJob(ticket2); System.out.println("DEBUG: count2=[" + count2 + "]"); .onQueryFailure((throwable) -> throwable.printStackTrace()); JobTicket ticket3 = moveMgr.startJob( batcher3 ); batcher3.awaitCompletion(); moveMgr.stopJob(ticket3);
@Test public void testResultReplace() throws Exception { DocumentMetadataHandle meta = new DocumentMetadataHandle().withCollections(collection); // write the document client.newDocumentManager().writeAs(collection + "/test1.json", meta, "{ \"testProperty\": \"test1\" }"); StructuredQueryDefinition query = sqb.value(sqb.jsonProperty("testProperty"), "test1"); ServerTransform transform = new ServerTransform(transformName1) .addParameter("newValue", "test1a"); ApplyTransformListener listener = new ApplyTransformListener() .withTransform(transform) .withApplyResult(ApplyResult.REPLACE); QueryBatcher batcher = moveMgr.newQueryBatcher(query) .onUrisReady(listener); JobTicket ticket = moveMgr.startJob( batcher ); batcher.awaitCompletion(); moveMgr.stopJob(ticket); JsonNode docContents = docMgr.readAs(collection + "/test1.json", JsonNode.class); assertEquals( "the transform should have changed testProperty to 'test1a'", "test1a", docContents.get("testProperty").textValue() ); }
@Test public void testResultIgnore() throws Exception { DocumentMetadataHandle meta = new DocumentMetadataHandle().withCollections(collection); // write the document client.newDocumentManager().writeAs(collection + "/test2.json", meta, "{ \"testProperty\": \"test2\" }"); StructuredQueryDefinition query = sqb.value(sqb.jsonProperty("testProperty"), "test2"); ServerTransform transform = new ServerTransform(transformName2) .addParameter("newValue", "test2a"); ApplyTransformListener listener = new ApplyTransformListener() .withTransform(transform) .withApplyResult(ApplyResult.IGNORE); QueryBatcher batcher = moveMgr.newQueryBatcher(query) .onUrisReady(listener); JobTicket ticket = moveMgr.startJob( batcher ); batcher.awaitCompletion(); moveMgr.stopJob(ticket); JsonNode docContents = docMgr.readAs(collection + "/test2.json", JsonNode.class); assertEquals( "the transform should have changed testProperty to 'test2a'", "test2a", docContents.get("testProperty").textValue() ); }
@Test public void testQueryBatcher() { client.newDocumentManager().writeAs(collection + "/test1.json", meta, "[true]"); client.newDocumentManager().writeAs(collection + "/test1.xml", meta, "<xml/>"); client.newDocumentManager().writeAs(collection + "/test1.txt", meta, "text"); assertEquals(3, client.newQueryManager().search(collectionQuery, new SearchHandle()).getTotalResults()); StructuredQueryDefinition query = collectionQuery; // begin copy from "Using QueryBatcher" in src/main/java/com/marklogic/datamovement/package-info.java QueryBatcher qhb = dataMovementManager.newQueryBatcher(query) .withBatchSize(1000) .withThreadCount(20) .withConsistentSnapshot() .onUrisReady(batch -> { for ( String uri : batch.getItems() ) { if ( uri.endsWith(".txt") ) { client.newDocumentManager().delete(uri); } } }) .onQueryFailure(queryBatchException -> queryBatchException.printStackTrace()); JobTicket ticket = dataMovementManager.startJob(qhb); qhb.awaitCompletion(); dataMovementManager.stopJob(ticket); // end copy from "Using QueryBatcher" in src/main/java/com/marklogic/datamovement/package-info.java SearchHandle results = client.newQueryManager().search(collectionQuery, new SearchHandle()); assertEquals(2, results.getTotalResults()); for ( MatchDocumentSummary match : results.getMatchResults() ) { assertTrue(match.getUri().matches(".*/test1.(json|xml)")); } }
.withConsistentSnapshot(); JobTicket ticket = moveMgr.startJob(queryBatcher); queryBatcher.awaitCompletion(); moveMgr.stopJob(ticket);