/** * Returns the host your application should talk to for this forest. If * getAlternateHost() is not null, return it. Otherwise, if * getOpenReplicaHost() is not null, return it. Otherwise, if * getRequestHost() is not null, return it. Otherwise, return getHost(). * * @return the preferred host for this forest */ public default String getPreferredHost() { if ( getAlternateHost() != null ) { return getAlternateHost(); } else if ( getOpenReplicaHost() != null ) { return getOpenReplicaHost(); } else if ( getRequestHost() != null ) { return getRequestHost(); } else { return getHost(); } }
String openReplicaHost = forest.getOpenReplicaHost(); if ( openReplicaHost != null ) openReplicaHost = openReplicaHost.toLowerCase(); if ( renames.containsKey(openReplicaHost) ) { openReplicaHost = renames.get(openReplicaHost); String requestHost = forest.getRequestHost(); if ( requestHost != null ) requestHost = requestHost.toLowerCase(); if ( renames.containsKey(requestHost) ) { requestHost = renames.get(requestHost); String alternateHost = forest.getAlternateHost(); if ( alternateHost != null ) alternateHost = alternateHost.toLowerCase(); if ( renames.containsKey(alternateHost) ) { alternateHost = renames.get(alternateHost); String host = forest.getHost(); if ( host != null ) host = host.toLowerCase(); if ( renames.containsKey(host) ) { host = renames.get(host); return new ForestImpl(host, openReplicaHost, requestHost, alternateHost, forest.getDatabaseName(), forest.getForestName(), forest.getForestId(), forest.isUpdateable(), false); }).toArray(Forest[]::new); if((blackList.contains(forest.getPreferredHost()) || blackList.contains(forest.getHost())) && forest.getPreferredHostType() == HostType.REQUEST_HOST) { hostArray = new String[] {forest.getOpenReplicaHost(), forest.getAlternateHost()}; } else { hostArray = new String[] {forest.getHost(), forest.getOpenReplicaHost(), forest.getAlternateHost(), forest.getRequestHost()};
/** * Returns the preferred host type for this forest. If * getAlternateHost() is not null, return ALTERNATE_HOST. Otherwise, if * getOpenReplicaHost() is not null, return OPEN_REPLICA_HOST. Otherwise, * if getRequestHost() is not null, return REQUEST_HOST. * Otherwise, return FOREST_HOST. * * @return the host type of the forest */ public default HostType getPreferredHostType() { if ( getAlternateHost() != null ) { return HostType.ALTERNATE_HOST; } else if ( getOpenReplicaHost() != null ) { return HostType.OPEN_REPLICA_HOST; } else if ( getRequestHost() != null ) { return HostType.REQUEST_HOST; } else { return HostType.FOREST_HOST; } } }
@Override public synchronized QueryBatcher withForestConfig(ForestConfiguration forestConfig) { super.withForestConfig(forestConfig); Forest[] forests = forestConfig.listForests(); Set<Forest> oldForests = new HashSet<>(forestResults.keySet()); Map<String,Forest> hosts = new HashMap<>(); for ( Forest forest : forests ) { if ( forest.getPreferredHost() == null ) throw new IllegalStateException("Hostname must not be null for any forest"); hosts.put(forest.getPreferredHost(), forest); if ( forestResults.get(forest) == null ) forestResults.put(forest, new AtomicLong()); if ( forestIsDone.get(forest) == null ) forestIsDone.put(forest, new AtomicBoolean(false)); if ( retryForestMap.get(forest) == null ) retryForestMap.put(forest, new AtomicInteger(0)); } logger.info("(withForestConfig) Using forests on {} hosts for \"{}\"", hosts.keySet(), forests[0].getDatabaseName()); List<DatabaseClient> newClientList = new ArrayList<>(); for ( String host : hosts.keySet() ) { Forest forest = hosts.get(host); DatabaseClient client = getMoveMgr().getForestClient(forest); newClientList.add(client); } clientList.set(newClientList); boolean started = (threadPool != null); if ( started == true && oldForests.size() > 0 ) calucluateDeltas(oldForests, forests); return this; }
if ( forest.getPreferredHost() == null ) { throw new IllegalStateException("Hostname must not be null for any forest"); hosts.put(forest.getPreferredHost(), forest); if(forest.getPreferredHostType() == HostType.REQUEST_HOST && !forest.getHost().toLowerCase().equals(forest.getRequestHost().toLowerCase())) { if(hosts.containsKey(forest.getHost())) hosts.remove(forest.getHost()); logger.info("(withForestConfig) Using forests on {} hosts for \"{}\"", hosts.keySet(), forests[0].getDatabaseName());
@Test public void testArgs() throws Exception { if (moveMgr.getConnectionType() == DatabaseClient.ConnectionType.GATEWAY) return; int defaultPort = client.getPort(); Class<?> defaultAuthContext = client.getSecurityContext().getClass(); ForestConfiguration forestConfig = moveMgr.readForestConfig(); Forest[] forests = forestConfig.listForests(); String defaultDatabase = forests[0].getDatabaseName(); // expect three forests per node assertTrue(forests.length % 3 == 0); for ( Forest forest : forests ) { DatabaseClient forestClient = ((DataMovementManagerImpl) moveMgr).getForestClient(forest); // not all forests for a database are on the same host, so all we // can check is that the hostname is not null assertNotNull(forest.getHost()); // not all hosts have the original REST server, but all hosts have the uber port assertEquals(defaultPort, forestClient.getPort()); assertEquals(defaultDatabase, forest.getDatabaseName()); assertEquals(defaultAuthContext, forestClient.getSecurityContext().getClass()); assertEquals(true, forest.isUpdateable()); if ( forest.getForestName() == null || ! forest.getForestName().startsWith("java-unittest-") ) { fail("Unexpected forestName \"" + forest.getForestName() + "\""); } } } }
FilteredForestConfiguration existingFilteredForestConfiguration = (FilteredForestConfiguration) existingForestConfig; for( Forest forest : existingFilteredForestConfiguration.listForests()) { if(forest.getPreferredHostType() == HostType.REQUEST_HOST && !forest.getHost().toLowerCase().equals(forest.getRequestHost().toLowerCase())) { if(preferredHostsList.contains(forest.getHost())) { preferredHostsList.remove(forest.getHost());
private void retry(QueryEvent queryEvent, boolean callFailListeners) { if ( isStopped() == true ) { logger.warn("Job is now stopped, aborting the retry"); return; } Forest retryForest = null; for ( Forest forest : getForestConfig().listForests() ) { if ( forest.equals(queryEvent.getForest()) ) { // while forest and queryEvent.getForest() have equivalent forest id, // we expect forest to have the currently available host info retryForest = forest; break; } } if ( retryForest == null ) { throw new IllegalStateException("Forest for queryEvent (" + queryEvent.getForest().getForestName() + ") is not in current getForestConfig()"); } // we're obviously not done with this forest forestIsDone.get(retryForest).set(false); retryForestMap.get(retryForest).incrementAndGet(); long start = queryEvent.getForestResultsSoFar() + 1; logger.trace("retryForest {} on retryHost {} at start {}", retryForest.getForestName(), retryForest.getPreferredHost(), start); QueryTask runnable = new QueryTask(getMoveMgr(), this, retryForest, query, queryEvent.getForestBatchNumber(), start, queryEvent.getJobBatchNumber(), callFailListeners); runnable.run(); } /*
totalResults.addAndGet(batch.getItems().length); if(queryBatcherChecks) { String forestName = batch.getForest().getForestName(); matches.add(uri); batchDatabaseName.set(batch.getForest().getDatabaseName());
private List<String> getHostNames(Collection<Forest> forests) { return forests.stream().map((forest)->forest.getPreferredHost()).distinct().collect(Collectors.toList()); }
private List<String> getForestNames(Collection<Forest> forests) { return forests.stream().map((forest)->forest.getForestName()).collect(Collectors.toList()); }
public DatabaseClient getForestClient(Forest forest) { if ( forest == null ) throw new IllegalArgumentException("forest must not be null"); if (getConnectionType() == DatabaseClient.ConnectionType.GATEWAY) { return getPrimaryClient(); } String hostName = forest.getPreferredHost(); String key = hostName; DatabaseClient client = clientMap.get(key); if ( client != null ) return client; // since this is shared across threads, let's get an exclusive lock on it before updating it synchronized(clientMap) { // just to avoid creating unnecessary DatabaseClient instances, let's check one more time if // another thread just barely inserted an instance that matches client = clientMap.get(key); if ( client != null ) return client; client = DatabaseClientFactory.newClient( hostName, primaryClient.getPort(), forest.getDatabaseName(), primaryClient.getSecurityContext() ); clientMap.put(key, client); } return client; }
/** * A utility method to return the list of hosts a Batcher should use when talking * to this database. The list is retrieved by calling getPreferredHost() on each * Forest. * * @return the list of hosts a Batcher should use */ public default String[] getPreferredHosts() { return Stream.of(listForests()).map( (forest) -> forest.getPreferredHost()).distinct().toArray(String[]::new); } }
if ( isDone.get() == true ) { logger.error("Attempt to query forest '{}' forestBatchNum {} with start {} after the last batch " + "for that forest has already been retrieved", forest.getForestName(), forestBatchNum, start); return; forest.getForestName(), forestBatchNum, start); return; try ( UrisHandle results = queryMgr.uris(query, handle, start, null, forest.getForestName()) ) { batch.getJobBatchNumber(), batch.getJobResultsSoFar(), forest.getForestName());
assertEquals("host1", filteredForests[0].getHost()); assertEquals("alternatehost1", filteredForests[0].getPreferredHost()); assertEquals("openreplicahost1", filteredForests[0].getOpenReplicaHost()); assertEquals("requesthost1", filteredForests[0].getRequestHost()); assertEquals("alternatehost1", filteredForests[0].getAlternateHost()); assertEquals("forestId1", filteredForests[0].getForestId()); assertFalse("openreplicahost2".equals(filteredForests[1].getOpenReplicaHost())); assertFalse("openreplicahost2".equals(filteredForests[1].getPreferredHost())); assertEquals("host2", filteredForests[1].getHost()); assertEquals(null, filteredForests[1].getRequestHost()); assertEquals(null, filteredForests[1].getAlternateHost()); assertEquals("forestId2", filteredForests[1].getForestId()); assertFalse("alternatehost3".equals(filteredForests[2].getAlternateHost())); assertFalse("alternatehost3".equals(filteredForests[2].getPreferredHost())); assertEquals("host3", filteredForests[2].getHost()); assertEquals(null, filteredForests[2].getOpenReplicaHost()); assertEquals(null, filteredForests[2].getRequestHost()); assertEquals("forestId3", filteredForests[2].getForestId()); assertFalse("requestHost4".equals(filteredForests[3].getRequestHost())); assertFalse("requestHost4".equals(filteredForests[3].getPreferredHost())); assertFalse("host4".equals(filteredForests[3].getHost())); assertEquals(null, filteredForests[3].getOpenReplicaHost()); assertEquals(null, filteredForests[3].getAlternateHost()); assertEquals("forestId4", filteredForests[3].getForestId());
@Test public void testWithWriteAndQueryBatcher() throws Exception{ if (moveMgr.getConnectionType() == DatabaseClient.ConnectionType.GATEWAY) return; ForestConfiguration forestConfig = moveMgr.readForestConfig(); long hostNum = Stream.of(forestConfig.listForests()).map(forest->forest.getPreferredHost()).distinct().count(); if ( hostNum <= 1 ) return; // we're not in a cluster, so this test isn't valid String host1 = forestConfig.listForests()[0].getPreferredHost(); FilteredForestConfiguration ffg = new FilteredForestConfiguration(forestConfig) .withRenamedHost(host1, Inet4Address.getByName(host1).getHostAddress()); runWithWriteAndQueryBatcher(ffg); ffg = new FilteredForestConfiguration(forestConfig) .withWhiteList(host1); runWithWriteAndQueryBatcher(ffg); ffg = new FilteredForestConfiguration(forestConfig) .withBlackList(host1); runWithWriteAndQueryBatcher(ffg); }
/** * This implements the QueryFailureListener interface * * @param queryBatch the exception with information about the failed query attempt */ public void processFailure(QueryBatchException queryBatch) { boolean isHostUnavailableException = processException(queryBatch.getBatcher(), queryBatch, queryBatch.getClient().getHost()); if ( isHostUnavailableException == true ) { try { logger.warn("Retrying failed batch: {}, results so far: {}, forest: {}, forestBatch: {}, forest results so far: {}", queryBatch.getJobBatchNumber(), queryBatch.getJobResultsSoFar(), queryBatch.getForest().getForestName(), queryBatch.getForestBatchNumber(), queryBatch.getForestResultsSoFar()); queryBatch.getBatcher().retryWithFailureListeners(queryBatch); } catch (RuntimeException e) { logger.error("Exception during retry", e); processFailure(new QueryBatchException(queryBatch, e)); } } }
assertEquals("host1", filteredForests[0].getHost()); assertFalse("alternatehost1".equals(filteredForests[0].getAlternateHost())); assertFalse("alternatehost1".equals(filteredForests[0].getPreferredHost())); assertFalse("openreplicahost1".equals(filteredForests[0].getOpenReplicaHost())); assertFalse("requesthost1".equals(filteredForests[0].getRequestHost())); assertEquals("forestId1", filteredForests[0].getForestId()); assertEquals("openreplicahost2", filteredForests[1].getOpenReplicaHost()); assertEquals("openreplicahost2", filteredForests[1].getPreferredHost()); assertEquals("host2", filteredForests[1].getHost()); assertEquals(null, filteredForests[1].getAlternateHost()); assertEquals(null, filteredForests[1].getRequestHost()); assertEquals("forestId2", filteredForests[1].getForestId()); assertEquals("alternatehost3", filteredForests[2].getAlternateHost()); assertEquals("alternatehost3", filteredForests[2].getPreferredHost()); assertEquals("host3", filteredForests[2].getHost()); assertEquals(null, filteredForests[2].getOpenReplicaHost()); assertEquals(null, filteredForests[1].getRequestHost()); assertEquals("forestId3", filteredForests[2].getForestId()); assertEquals("requesthost4", filteredForests[3].getRequestHost()); assertEquals("requesthost4", filteredForests[3].getPreferredHost()); assertEquals("host4", filteredForests[3].getHost()); assertEquals(null, filteredForests[3].getOpenReplicaHost());
@Test public void testWithInvalidHosts() throws Exception{ if (moveMgr.getConnectionType() == DatabaseClient.ConnectionType.GATEWAY) return; ForestConfiguration forestConfig = moveMgr.readForestConfig(); String host1 = forestConfig.listForests()[0].getPreferredHost(); FilteredForestConfiguration ffg = new FilteredForestConfiguration(forestConfig) .withRenamedHost("someInvalidHostName", "anotherInvalidHostName"); runWithWriteAndQueryBatcher(ffg); ffg = new FilteredForestConfiguration(forestConfig) .withBlackList("someInvalidHostName"); runWithWriteAndQueryBatcher(ffg); ffg = new FilteredForestConfiguration(forestConfig) .withWhiteList("someInvalidHostName") .withWhiteList(host1); runWithWriteAndQueryBatcher(ffg); }
.evalAs(String.class); logger.info("DEBUG: [LegalHoldsTest] urisToDelete =[" + urisToDelete + "]"); logger.debug("DEBUG: [LegalHoldsTest] batch.getForest().getForestName()=[" + batch.getForest().getForestName() + "]"); logger.info("DEBUG: [LegalHoldsTest] uri =[" + uri + "]"); synchronized (urisDeleted) { AtomicInteger deleted = urisDeleted.get(batch.getForest().getForestName()); if ( deleted == null ) { deleted = new AtomicInteger(); urisDeleted.put(batch.getForest().getForestName(), deleted);