/** * Initialize any state for this DB. Called once per DB instance; there is one DB instance per * client thread. */ @Override public void init() throws DBException { Properties props = getProperties(); commitTime = Integer .parseInt(props.getProperty("solr.commit.within.time", DEFAULT_COMMIT_WITHIN_TIME)); batchMode = Boolean.parseBoolean(props.getProperty("solr.batch.mode", DEFAULT_BATCH_MODE)); String jaasConfPath = props.getProperty("solr.jaas.conf.path"); if(jaasConfPath != null) { System.setProperty("java.security.auth.login.config", jaasConfPath); HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer()); } // Check if Solr cluster is running in SolrCloud or Stand-alone mode Boolean cloudMode = Boolean.parseBoolean(props.getProperty("solr.cloud", DEFAULT_CLOUD_MODE)); System.err.println("Solr Cloud Mode = " + cloudMode); if (cloudMode) { System.err.println("Solr Zookeeper Remote Hosts = " + props.getProperty("solr.zookeeper.hosts", DEFAULT_ZOOKEEPER_HOSTS)); client = new CloudSolrClient.Builder().withZkHost( Arrays.asList(props.getProperty("solr.zookeeper.hosts", DEFAULT_ZOOKEEPER_HOSTS).split(","))).build(); } else { client = new HttpSolrClient.Builder(props.getProperty("solr.base.url", DEFAULT_SOLR_BASE_URL)).build(); } }
private DirectSolrInputDocumentWriter createCloudSolrWriter(Context context, Map<String, String> indexConnectionParams) throws IOException { String indexZkHost = indexConnectionParams.get(SolrConnectionParams.ZOOKEEPER); String collectionName = indexConnectionParams.get(SolrConnectionParams.COLLECTION); if (indexZkHost == null) { throw new IllegalStateException("No index ZK host defined"); } if (collectionName == null) { throw new IllegalStateException("No collection name defined"); } CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(indexZkHost).build(); int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(context.getConfiguration()); solrServer.setZkClientTimeout(zkSessionTimeout); solrServer.setZkConnectTimeout(zkSessionTimeout); solrServer.setDefaultCollection(collectionName); return new DirectSolrInputDocumentWriter(context.getConfiguration().get(INDEX_NAME_CONF_KEY), solrServer); }
private DirectSolrInputDocumentWriter createCloudSolrWriter( Context context, Map<String, String> indexConnectionParams, String uniqueKeyField) throws IOException { String indexZkHost = indexConnectionParams.get(SolrConnectionParams.ZOOKEEPER); String collectionName = indexConnectionParams.get(SolrConnectionParams.COLLECTION); if (indexZkHost == null) { throw new IllegalStateException("No index ZK host defined"); } if (collectionName == null) { throw new IllegalStateException("No collection name defined"); } CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(indexZkHost).build(); int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(context.getConfiguration()); solrServer.setZkClientTimeout(zkSessionTimeout); solrServer.setZkConnectTimeout(zkSessionTimeout); solrServer.setDefaultCollection(collectionName); solrServer.setIdField(uniqueKeyField); return new DirectSolrInputDocumentWriter(context.getConfiguration().get(INDEX_NAME_CONF_KEY), solrServer); }
.withZkHost(zkHost) .build(); cloudSolrClient.setDefaultCollection(collectionName);
/** * Instantiate Chronix Client. This should be called by subclasses' @OnScheduled method to create a client * if one does not yet exist. If called when scheduled, closeClient() should be called by the subclasses' @OnStopped * method so the client will be destroyed when the processor is stopped. * * @param context The context for this processor * @throws ProcessException if an error occurs while creating an Chronix client */ protected void createSolrClient(ControllerServiceInitializationContext context) throws ProcessException { if (solr != null) { return; } // create a solr client final boolean isCloud = context.getPropertyValue(SOLR_CLOUD).asBoolean(); final String connectionString = context.getPropertyValue(SOLR_CONNECTION_STRING).asString(); final String collection = context.getPropertyValue(SOLR_COLLECTION).asString(); if (isCloud) { //logInfo("creating solrCloudClient on $solrUrl for collection $collection"); CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder().withZkHost(connectionString).build(); cloudSolrClient.setDefaultCollection(collection); cloudSolrClient.setZkClientTimeout(30000); cloudSolrClient.setZkConnectTimeout(30000); solr = cloudSolrClient; } else { // logInfo(s"creating HttpSolrClient on $solrUrl for collection $collection") solr = new HttpSolrClient.Builder(connectionString + "/" + collection).build(); } }
/** * Create a Solr collection with a given number of shards. * * @param collectionName name of the collection to be created * @param configName name of the config for the collection * @param numShards number of shards in the collection */ public void createCollection(String collectionName, String configName, int numShards) throws IOException { int TIMEOUT = 30; // seconds CloudSolrClient solrClient = new CloudSolrClient.Builder().withZkHost(getZkConnectString()).build(); solrClient.setDefaultCollection(collectionName); CollectionAdminRequest.Create request = CollectionAdminRequest.createCollection( collectionName, configName, numShards, 1); request.setMaxShardsPerNode(numShards); RequestStatusState state; try { state = request.processAndWait(solrClient, TIMEOUT); } catch (SolrServerException | InterruptedException e) { throw new RuntimeException(e); } if (state != RequestStatusState.COMPLETED && state != RequestStatusState.RUNNING) { throw new IllegalStateException("Unexpected state: " + state); } waitForRecoveriesToFinish( collectionName, solrClient.getZkStateReader(), true, true, TIMEOUT); }
/** * Creates a SolrCloud collection if it doesn't already exist with the given config set. * * @deprecated since 1.2.0 use {@link SolrCollectionInitializer} instead. * @param zkHost host and optional port of one of a zookeeper instance (localhost:2181) * @param collectionName name of the collection to be created * @param configSetName name of the config set to use * @param configSetPath location of the config set (folder containing solr configs such as * solrconfig.xml) * @throws IOException if something goes wrong with the upload * @throws SolrServerException if something goes wrong with the upload */ @Deprecated public static void initSolr( String zkHost, String collectionName, String configSetName, Path configSetPath) throws IOException, SolrServerException { try (ZkClientClusterStateProvider provider = new ZkClientClusterStateProvider(zkHost)) { provider.uploadConfig(configSetPath, configSetName); } try (SolrClient solr = new CloudSolrClient.Builder().withZkHost(zkHost).build()) { List<String> existingCollectionNames = CollectionAdminRequest.listCollections(solr); if (!existingCollectionNames.contains(collectionName)) { solr.request(CollectionAdminRequest.createCollection(collectionName, configSetName, 1, 1)); } } }
public SolrClient getSolrServer() { if (zkHost != null && zkHost.length() > 0) { if (collectionName == null || collectionName.length() == 0) { throw new MorphlineCompilationException("Parameter 'zkHost' requires that you also pass parameter 'collection'", config); } CloudSolrClient cloudSolrClient = new Builder() .withZkHost(zkHost) .build(); cloudSolrClient.setDefaultCollection(collectionName); cloudSolrClient.setZkClientTimeout(zkClientSessionTimeout); cloudSolrClient.setZkConnectTimeout(zkClientConnectTimeout); return cloudSolrClient; } else { if (solrUrl == null && solrHomeDir != null) { CoreContainer coreContainer = new CoreContainer(solrHomeDir); coreContainer.load(); EmbeddedSolrServer embeddedSolrServer = new EmbeddedSolrServer(coreContainer, collectionName); return embeddedSolrServer; } if (solrUrl == null || solrUrl.length() == 0) { throw new MorphlineCompilationException("Missing parameter 'solrUrl'", config); } int solrServerNumThreads = 2; int solrServerQueueLength = solrServerNumThreads; SolrClient server = new SafeConcurrentUpdateSolrServer(solrUrl, solrServerQueueLength, solrServerNumThreads); return server; } }
public static SolrClient getClient(Map stormConf, String boltType) { String zkHost = ConfUtils.getString(stormConf, "solr." + boltType + ".zkhost", null); String solrUrl = ConfUtils.getString(stormConf, "solr." + boltType + ".url", null); String collection = ConfUtils.getString(stormConf, "solr." + boltType + ".collection", null); int queueSize = ConfUtils.getInt(stormConf, "solr." + boltType + ".queueSize", -1); SolrClient client; if (StringUtils.isNotBlank(zkHost)) { client = new CloudSolrClient.Builder().withZkHost(zkHost).build(); if (StringUtils.isNotBlank(collection)) { ((CloudSolrClient) client).setDefaultCollection(collection); } } else if (StringUtils.isNotBlank(solrUrl)) { if (queueSize == -1) { client = new HttpSolrClient.Builder(solrUrl).build(); } else { client = new ConcurrentUpdateSolrClient.Builder(solrUrl) .withQueueSize(queueSize).build(); } } else { throw new RuntimeException( "SolrClient should have zk or solr URL set up"); } return client; }
private Set<SolrClient> createSolrServers(Map<String, String> indexConnectionParams, String uniqueKeyField) throws MalformedURLException { String solrMode = getSolrMode(indexConnectionParams); if (solrMode.equals("cloud")) { String indexZkHost = indexConnectionParams.get(SolrConnectionParams.ZOOKEEPER); String collectionName = indexConnectionParams.get(SolrConnectionParams.COLLECTION); CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(indexZkHost).build(); int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(getConf()); solrServer.setZkClientTimeout(zkSessionTimeout); solrServer.setZkConnectTimeout(zkSessionTimeout); solrServer.setDefaultCollection(collectionName); solrServer.setIdField(uniqueKeyField); return Collections.singleton((SolrClient)solrServer); } else if (solrMode.equals("classic")) { PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager(); connectionManager.setDefaultMaxPerRoute(getSolrMaxConnectionsPerRoute(indexConnectionParams)); connectionManager.setMaxTotal(getSolrMaxConnectionsTotal(indexConnectionParams)); HttpClient httpClient = new DefaultHttpClient(connectionManager); return new HashSet<SolrClient>(createHttpSolrClients(indexConnectionParams, httpClient)); } else { throw new RuntimeException("Only 'cloud' and 'classic' are valid values for solr.mode, but got " + solrMode); } }
/** * Method to initialize a Solr collection using the provided default configs. * * @deprecated since 1.2.0 use {@link SolrCollectionInitializer} instead. * @param zkHost host and optional port of one of a zookeeper instance (localhost:2181) * @param collectionName name of the collection to be created * @throws IOException if something goes wrong with the upload * @throws SolrServerException if something goes wrong with the upload */ @Deprecated public static void initSolr(String zkHost, String collectionName) throws IOException, SolrServerException { try (SolrClient solr = new CloudSolrClient.Builder().withZkHost(zkHost).build()) { List<String> existingCollectionNames = CollectionAdminRequest.listCollections(solr); if (!existingCollectionNames.contains(collectionName)) { solr.request(CollectionAdminRequest.createCollection(collectionName, 1, 1)); } } } }
private Set<SolrClient> createSolrClients(Map<String, String> indexConnectionParams) throws MalformedURLException { String solrMode = getSolrMode(indexConnectionParams); if (solrMode.equals("cloud")) { String indexZkHost = indexConnectionParams.get(SolrConnectionParams.ZOOKEEPER); String collectionName = indexConnectionParams.get(SolrConnectionParams.COLLECTION); CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(indexZkHost).build(); int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(getConf()); solrServer.setZkClientTimeout(zkSessionTimeout); solrServer.setZkConnectTimeout(zkSessionTimeout); solrServer.setDefaultCollection(collectionName); return Collections.singleton((SolrClient)solrServer); } else if (solrMode.equals("classic")) { PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager(); connectionManager.setDefaultMaxPerRoute(getSolrMaxConnectionsPerRoute(indexConnectionParams)); connectionManager.setMaxTotal(getSolrMaxConnectionsTotal(indexConnectionParams)); HttpClient httpClient = new DefaultHttpClient(connectionManager); return new HashSet<SolrClient>(createHttpSolrClients(indexConnectionParams, httpClient)); } else { throw new RuntimeException("Only 'cloud' and 'classic' are valid values for solr.mode, but got " + solrMode); } }
public SolrCollectionManager() { // build client to connect to solr SolrFlintConfig config = SolrFlintConfig.getInstance(); // use cloud? Collection<String> zkhosts = config.getZKHosts(); if (zkhosts != null && !zkhosts.isEmpty()) { this.defaultShards = zkhosts.size(); this.defaultReplicas = 1; this._solr = new CloudSolrClient.Builder().withZkHost(zkhosts).build(); } else { this.defaultShards = 1; this.defaultReplicas = 1; this._solr = new HttpSolrClient.Builder(config.getServerURL()).allowCompression(true).build(); } }
public SolrIndexIO(Index index) { this._collection = index.getIndexID(); // build client to connect to solr SolrFlintConfig config = SolrFlintConfig.getInstance(); // use cloud? Collection<String> zkhosts = config.getZKHosts(); if (zkhosts != null && !zkhosts.isEmpty()) { this._client = new CloudSolrClient.Builder().withZkHost(zkhosts).build(); } else { this._client = new HttpSolrClient.Builder(config.getServerURL()).allowCompression(true).build(); } }
public static SolrClient createCloudSolrClient(Map<String, String> connectionParameters, String uniqueKeyField, int zkSessionTimeout) { String solrZk = connectionParameters.get(SolrConnectionParams.ZOOKEEPER); CloudSolrClient solr = new CloudSolrClient.Builder().withZkHost(solrZk).build(); solr.setZkClientTimeout(zkSessionTimeout); solr.setZkConnectTimeout(zkSessionTimeout); String collection = connectionParameters.get(SolrConnectionParams.COLLECTION); solr.setDefaultCollection(collection); solr.setIdField(uniqueKeyField); return solr; }
public SolrCollectionManager(Collection<String> zkhosts) { this.defaultShards = zkhosts.size(); this.defaultReplicas = 1; this._solr = new CloudSolrClient.Builder().withZkHost(zkhosts).build(); }
public static SolrClient createCloudSolrClient(Map<String, String> connectionParameters, int zkSessionTimeout) { String solrZk = connectionParameters.get(SolrConnectionParams.ZOOKEEPER); CloudSolrClient solr = new CloudSolrClient.Builder().withZkHost(solrZk).build(); solr.setZkClientTimeout(zkSessionTimeout); solr.setZkConnectTimeout(zkSessionTimeout); String collection = connectionParameters.get(SolrConnectionParams.COLLECTION); solr.setDefaultCollection(collection); return solr; }
private CloudSolrClient getCloudServer() { CloudSolrClient rVal = null; if (this.getZkHost() != null && this.getCollection() != null) { rVal = new CloudSolrClient.Builder().withZkHost(zkHost).build(); rVal.setDefaultCollection(this.getCollection()); } return rVal; }
/** * Builds a Solr client using the ZK hosts from the global config. * @return SolrClient */ public SolrClient getSolrClient() { return new CloudSolrClient.Builder().withZkHost(getZkHosts()).build(); }
/** * Builds a Solr client using the ZK hosts specified. * @return SolrClient */ public SolrClient getSolrClient(List<String> zkHosts) { return new CloudSolrClient.Builder().withZkHost(zkHosts).build(); }