private void testListenerException(QueryBatchListener listener) { final AtomicInteger failureBatchCount = new AtomicInteger(); Iterator<String> iterator = Arrays.asList(new String[] {uri1}).iterator(); QueryBatcher queryBatcher = moveMgr.newQueryBatcher(iterator) .onUrisReady( batch -> logger.debug("uri={}", batch.getItems()[0]) ) .onUrisReady(listener) .onQueryFailure( queryThrowable -> failureBatchCount.incrementAndGet() ); moveMgr.startJob(queryBatcher); queryBatcher.awaitCompletion(); moveMgr.stopJob(queryBatcher); // there should be no failure sent to the QueryBatcher onQueryFailure listeners assertEquals(0, failureBatchCount.get()); }
@Test public void testIssue793() { WriteBatcher batcher = moveMgr.newWriteBatcher(); batcher.addAs("test.txt", "test"); moveMgr.startJob(batcher); moveMgr.stopJob(batcher); } }
/** If less than minHosts are left, calls stopJob. * * @param numHosts the minimum number of hosts before this will call dataMovementMangaer.stopJob(batcher) * * @return this instance (for method chaining) */ public HostAvailabilityListener withMinHosts(int numHosts) { if (moveMgr.getConnectionType() == DatabaseClient.ConnectionType.GATEWAY) { if (numHosts != 1) { throw new IllegalArgumentException("numHosts must be 1 when using only the primary host for the connection"); } } else { if (numHosts <= 0) throw new IllegalArgumentException("numHosts must be > 0"); // TODO: use existing forest configuration instead of refreshing? int numConfigHosts = moveMgr.readForestConfig().getPreferredHosts().length; if (numHosts > numConfigHosts) throw new IllegalArgumentException ("numHosts must be less than or equal to the number of hosts in the cluster"); } this.minHosts = numHosts; return this; }
@Override public void initialize() { writeBatcher = this.dataMovementManager.newWriteBatcher(); writeBatcher.withThreadCount(threadCount); writeBatcher.withBatchSize(batchSize); jobTicket = dataMovementManager.startJob(writeBatcher); }
WriteBatcher batcher1 = moveMgr.newWriteBatcher() .withBatchSize(100) .onBatchFailure((batch, throwable) -> throwable.printStackTrace()); JobTicket ticket1 = moveMgr.startJob( batcher1 ); for ( int i=0; i < numDocs; i++) { batcher1.addAs(collection + "/test_doc_" + i + ".json", meta, "{ \"testProperty\": \"test3\" }"); moveMgr.stopJob(ticket1); .onSuccess(batch -> count2.addAndGet(batch.getItems().length)) .onBatchFailure((batch, throwable) -> throwable.printStackTrace()); QueryBatcher batcher = moveMgr.newQueryBatcher(query2) .onUrisReady(listener) .withConsistentSnapshot(); JobTicket ticket2 = moveMgr.startJob( batcher ); batcher.awaitCompletion(); moveMgr.stopJob(ticket2); System.out.println("DEBUG: count2=[" + count2 + "]"); assertEquals( "exactly " + numDocs + " docs should have been transformed", numDocs, count2.get()); query3.setCollections(collection); final AtomicInteger count3 = new AtomicInteger(0); QueryBatcher batcher3 = moveMgr.newQueryBatcher(query3) .withBatchSize(100) .onUrisReady(batch -> count3.addAndGet(batch.getItems().length)) .onQueryFailure((throwable) -> throwable.printStackTrace()); JobTicket ticket3 = moveMgr.startJob( batcher3 ); batcher3.awaitCompletion(); moveMgr.stopJob(ticket3);
StructuredQueryDefinition query = null; if (jobIds == null) { batcher = dmm.newQueryBatcher(emptyQuery); batcher = dmm.newQueryBatcher(sqb.value(sqb.jsonProperty("jobId"), jobIds)); JobTicket jobTicket = dmm.startJob(batcher); dmm.stopJob(batcher); dmm.release(); JobReport report = dmm.getJobReport(jobTicket); long jobCount = report.getSuccessEventsCount(); response.totalJobs = jobCount; batcher = dmm.newQueryBatcher(emptyQuery); batcher = dmm.newQueryBatcher(sqb.value(sqb.element(new QName("jobId")), jobIds)); jobTicket = dmm.startJob(batcher); dmm.stopJob(batcher); dmm.release(); report = dmm.getJobReport(jobTicket); long traceCount = report.getSuccessEventsCount(); response.totalTraces = traceCount;
.newWriteBatcher() .withJobName("Load jobs") .withBatchSize(50); JobTicket ticket = dmm.startJob(writer); dmm.stopJob(ticket); dmm.release(); .newWriteBatcher() .withJobName("Load traces"); ticket = dmm.startJob(writer); dmm.stopJob(ticket); dmm.release();
@Override public QueryBatcherJobTicket run(DatabaseClient databaseClient) { DataMovementManager dmm = this.dataMovementManager != null ? this.dataMovementManager : databaseClient.newDataMovementManager(); String jobDescription = getJobDescription(); if (jobDescription != null && logger.isInfoEnabled()) { logger.info(jobDescription); } QueryBatcherBuilder builder = newQueryBatcherBuilder(); QueryBatcher queryBatcher = builder.buildQueryBatcher(databaseClient, dmm); prepareQueryBatcher(queryBatcher); JobTicket jobTicket = dmm.startJob(queryBatcher); if (awaitCompletion) { queryBatcher.awaitCompletion(); if (stopJobAfterCompletion) { dmm.stopJob(queryBatcher); } if (jobDescription != null && logger.isInfoEnabled()) { logger.info("Completed: " + jobDescription); } } return new QueryBatcherJobTicket(dmm, queryBatcher, jobTicket); }
final StringBuffer failures = new StringBuffer(); final int expectedBatches = (int) Math.ceil(totalDocCount / expectedBatchSize); WriteBatcher batcher = moveMgr.newWriteBatcher() .withBatchSize(batchSize) .withThreadCount(batcherThreadCount) long batchMinTime = new Date().getTime(); assertFalse("Job should not be started yet", batcher.isStarted()); moveMgr.startJob(batcher); assertTrue("Job should be started now", batcher.isStarted()); JobTicket ticket = moveMgr.getActiveJob(writeBatcherJobId); assertEquals(batchSize, batcher.getBatchSize()); assertEquals(writeBatcherJobId, batcher.getJobId()); JobReport report = moveMgr.getJobReport(ticket); moveMgr.stopJob(ticket); assertEquals("there should be " + successfulCount + " docs in the collection", successfulCount.get(), docs.getTotalSize()); report = moveMgr.getJobReport(ticket); long maxTime = new Date().getTime(); Date batchDate = batchTimestamp.get().getTime();
moveMgr.startJob(queryBatcher); long reportStartTime = new Date().getTime(); JobTicket ticket = moveMgr.getActiveJob(queryBatcherJobId); assertTrue("Job should be started now", queryBatcher.isStarted()); assertEquals(queryBatcherJobName, ticket.getBatcher().getJobName()); JobReport report = moveMgr.getJobReport(ticket); moveMgr.stopJob(ticket.getBatcher()); report = moveMgr.getJobReport(ticket); long maxTime = new Date().getTime(); Date batchDate = batchTimestamp.get().getTime(); if (moveMgr.getConnectionType() == DatabaseClient.ConnectionType.GATEWAY) {
Set<String> uris = Collections.synchronizedSet(new HashSet<>()); StringBuilder failures = new StringBuilder(); QueryBatcher getUris = moveMgr.newQueryBatcher(query) .withThreadCount(6) .withBatchSize(5000) failures.append("ERROR:[" + throwable + "]\n"); }); moveMgr.startJob(getUris); getUris.awaitCompletion(); if ( failures.length() > 0 ) fail(failures.toString()); Set<String> uris2 = Collections.synchronizedSet(new HashSet<>()); StringBuilder failures2 = new StringBuilder(); QueryBatcher performDelete = moveMgr.newQueryBatcher(uris.iterator()) .withThreadCount(2) .withBatchSize(99) failures2.append("ERROR:[" + throwable + "]\n"); }); moveMgr.startJob(performDelete); performDelete.awaitCompletion();
/** * Apply the given listener on batches on documents matching the given structured query. * * @param urisReadyListener * @param queryDefinition * @return */ public QueryBatcherJobTicket applyOnStructuredQuery(QueryBatchListener urisReadyListener, StructuredQueryDefinition queryDefinition) { return apply(urisReadyListener, dataMovementManager.newQueryBatcher(queryDefinition)); }
WriteBatcher batcher = moveMgr.newWriteBatcher(); moveMgr.startJob(batcher); String[] uris = new String[100]; for ( int i=0; i < 100; i++ ) { moveMgr.newQueryBatcher(query) .withThreadCount(5) .withBatchSize(10) .onUrisReady(exportListener) .onQueryFailure( throwable -> throwable.printStackTrace() ); moveMgr.startJob( queryJob );
@Test public void testPrematureStopIteratorJob() { // this test can't actually validate that the message is printed, so a human // must check the logging output // Expected something like: // 18:14:58.420 [main] WARN c.m.c.d.impl.QueryBatcherImpl - QueryBatcher instance "unnamed" stopped before all results were processed List<String> list = new ArrayList<>(); list.add("firstUri.txt"); QueryBatcher batcher = moveMgr.newQueryBatcher(list.iterator()); moveMgr.stopJob(batcher); }
public void testExceptions(DocumentWriteSet docs, int expectedSuccesses, int expectedFailures) { WriteBatcher batcher = moveMgr.newWriteBatcher() .onBatchSuccess( batch -> { throw new InternalError(errorMessage); } ) .onBatchFailure( (batch, throwable) -> { throw new InternalError(errorMessage); } ); testExceptions(batcher, docs, expectedSuccesses, expectedFailures); batcher = moveMgr.newWriteBatcher() .onBatchSuccess( batch -> { throw new RuntimeException(errorMessage); } ) .onBatchFailure( (batch, throwable) -> { throw new RuntimeException(errorMessage); } ); testExceptions(batcher, docs, expectedSuccesses, expectedFailures); cleanupDocs(docs); }
/** * Manages refreshing the forests and hosts and retrying events after a host * becomes unavailable. * @param moveMgr the DataMovementManager (used to call readForestConfig to reset after black-listing an unavailable host) */ public HostAvailabilityListener(DataMovementManager moveMgr) { if (moveMgr == null) throw new IllegalArgumentException("moveMgr must not be null"); this.moveMgr = moveMgr; if (moveMgr.getConnectionType() == DatabaseClient.ConnectionType.DIRECT) { hostUnavailableExceptions.add(SocketException.class); hostUnavailableExceptions.add(SSLException.class); hostUnavailableExceptions.add(UnknownHostException.class); } }
private void scheduleForestResynch(Batcher batcher, String host) { // cancel any previously scheduled re-sync if ( future != null ) future.cancel(false); // schedule a re-sync with the server forest config future = Executors.newScheduledThreadPool(1) .schedule( () -> { if ( batcher.isStopped() ) { logger.debug("Job \"{}\" is stopped, so cancelling re-sync with the server forest config", batcher.getJobName()); } else { ForestConfiguration updatedForestConfig = moveMgr.readForestConfig(); logger.info("it's been {} since host {} failed, opening communication to all server hosts [{}]", suspendTimeForHostUnavailable.toString(), host, Arrays.asList(updatedForestConfig.getPreferredHosts())); // set the forestConfig back to whatever the server says it is batcher.withForestConfig(updatedForestConfig); } } , suspendTimeForHostUnavailable.toMillis(), TimeUnit.MILLISECONDS); }
public String getSummaryReport() { JobReport report = moveMgr.getJobReport(ticket); return "batches: " + report.getSuccessBatchesCount() + ", docs: " + report.getSuccessEventsCount() + ", failures: " + report.getFailureEventsCount(); } }
WriteBatcher writeBatcher = moveMgr.newWriteBatcher() .withBatchSize(10) .withForestConfig(ffg) moveMgr.stopJob(writeBatcher); QueryBatcher getUris = moveMgr.newQueryBatcher(query) .withForestConfig(ffg) .withBatchSize(2) .onUrisReady(batch -> retrievedUris.addAll(Arrays.asList(batch.getItems())) ) .onQueryFailure(exception -> exception.printStackTrace() ); moveMgr.startJob(getUris); getUris.awaitCompletion();