CloudSolrClient cs = (CloudSolrClient) server; if (StringUtils.isBlank(cs.getDefaultCollection())) { cs.setDefaultCollection(getPrimaryName());
CloudSolrClient cs = (CloudSolrClient) server; if (StringUtils.isBlank(cs.getDefaultCollection())) { cs.setDefaultCollection(getReindexName());
cloudSolrClient.setDefaultCollection(collection); cloudSolrClient.setZkClientTimeout(zkClientTimeout); cloudSolrClient.setZkConnectTimeout(zkConnectionTimeout);
@Bean public SolrClient solrClient() { CloudSolrClient client = new CloudSolrClient(env.getRequiredProperty("spring.data.solr.zk-host")); client.setDefaultCollection("participant_"+env.getActiveProfiles()[0]); return client; }
public SolrHiveWriter(JobConf conf){ //初始化solrclient// 必须在这里初始化,否则,提取到一个公用类里面初始化会报错,因为MR的进程JVM和客户端的类是独立的 if(conf.get(Conf.IS_SOLRCLOUD).equals("1")) { //solrcloud模式 sc=new CloudSolrClient(conf.get(Conf.SOLR_URL).trim(),lbHttpSolrClient);//设置Cloud的client ((CloudSolrClient)sc).setDefaultCollection(conf.get(Conf.COLLECTION_NAME));//设置集合名 // ((CloudSolrClient)sc).setParallelUpdates(false);//取消并行更新 }else{ //普通模式 sc = new HttpSolrClient(conf.get(Conf.SOLR_URL),httpClient); } this.batchSize=Integer.parseInt(conf.get(Conf.SOLR_CURSOR_BATCH_SIZE)); log.info("批处理提交数量:{}",batchSize); }
@Override public void init(Configuration c) throws WriterException { String serverName = c.get("solr.cloud.address"); if (serverName == null) { throw new WriterException("Solr server address is not defined."); } String collection = c.get("solr.collection", "logs"); if(client == null) { client = new CloudSolrClient(serverName); client.setDefaultCollection(collection); } }
private SolrClient getSolrclient(String core) { SolrClient solrClient; if (solrClients.containsKey(core)) { //we have a client for this core return solrClients.get(core); } else { if (isCloud) { solrClient = new CloudSolrClient(zkHost); ((CloudSolrClient) solrClient).setDefaultCollection(core); } else { solrClient = new HttpSolrClient("http://" + solrHost + "/solr/" + core); } solrClients.put(core, solrClient); } return solrClient; }
@SuppressWarnings("rawtypes") @Override // FIXME: to be handled better public void init(NamedList args) { super.init(args); final String zkAddress = System.getProperty("zkHost"); if (isNotNullOrEmptyString(zkAddress)) { this.server= new CloudSolrClient(zkAddress); this.server.setDefaultCollection("store"); } }
protected CloudSolrClient createCloudClient(String defaultCollection) { CloudSolrClient client = getCloudSolrClient(zkServer.getZkAddress(), random().nextBoolean(), 30000, 60000); if (defaultCollection != null) client.setDefaultCollection(defaultCollection); return client; }
@Override public RecordReader getRecordReader(InputSplit split, JobConf conf, Reporter reporter) throws IOException { //初始化solrclient// 必须在这里初始化,否则,提取到一个公用类里面初始化会报错,因为MR的进程JVM和客户端的类是独立的 if(conf.get(Conf.IS_SOLRCLOUD).equals("1")) { //solrcloud模式 sc=new CloudSolrClient(conf.get(Conf.SOLR_URL).trim(),httpClient);//设置Cloud的client ((CloudSolrClient)sc).setDefaultCollection(conf.get(Conf.COLLECTION_NAME));//设置集合名 // ((CloudSolrClient)sc).setParallelUpdates(false); }else{ //普通模式 sc = new HttpSolrClient(conf.get(Conf.SOLR_URL),httpClient); } //第三个参数是游标的初始化查询参数 return new SolrReader(sc,new SolrQuery(),"*",conf); } }
private DirectSolrInputDocumentWriter createCloudSolrWriter(Context context, Map<String, String> indexConnectionParams) throws IOException { String indexZkHost = indexConnectionParams.get(SolrConnectionParams.ZOOKEEPER); String collectionName = indexConnectionParams.get(SolrConnectionParams.COLLECTION); if (indexZkHost == null) { throw new IllegalStateException("No index ZK host defined"); } if (collectionName == null) { throw new IllegalStateException("No collection name defined"); } CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(indexZkHost).build(); int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(context.getConfiguration()); solrServer.setZkClientTimeout(zkSessionTimeout); solrServer.setZkConnectTimeout(zkSessionTimeout); solrServer.setDefaultCollection(collectionName); return new DirectSolrInputDocumentWriter(context.getConfiguration().get(INDEX_NAME_CONF_KEY), solrServer); }
public static SolrClient createCloudSolrClient(Map<String, String> connectionParameters, int zkSessionTimeout) { String solrZk = connectionParameters.get(SolrConnectionParams.ZOOKEEPER); CloudSolrClient solr = new CloudSolrClient.Builder().withZkHost(solrZk).build(); solr.setZkClientTimeout(zkSessionTimeout); solr.setZkConnectTimeout(zkSessionTimeout); String collection = connectionParameters.get(SolrConnectionParams.COLLECTION); solr.setDefaultCollection(collection); return solr; }
private DirectSolrInputDocumentWriter createCloudSolrWriter( Context context, Map<String, String> indexConnectionParams, String uniqueKeyField) throws IOException { String indexZkHost = indexConnectionParams.get(SolrConnectionParams.ZOOKEEPER); String collectionName = indexConnectionParams.get(SolrConnectionParams.COLLECTION); if (indexZkHost == null) { throw new IllegalStateException("No index ZK host defined"); } if (collectionName == null) { throw new IllegalStateException("No collection name defined"); } CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(indexZkHost).build(); int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(context.getConfiguration()); solrServer.setZkClientTimeout(zkSessionTimeout); solrServer.setZkConnectTimeout(zkSessionTimeout); solrServer.setDefaultCollection(collectionName); solrServer.setIdField(uniqueKeyField); return new DirectSolrInputDocumentWriter(context.getConfiguration().get(INDEX_NAME_CONF_KEY), solrServer); }
public static SolrClient createCloudSolrClient(Map<String, String> connectionParameters, String uniqueKeyField, int zkSessionTimeout) { String solrZk = connectionParameters.get(SolrConnectionParams.ZOOKEEPER); CloudSolrClient solr = new CloudSolrClient.Builder().withZkHost(solrZk).build(); solr.setZkClientTimeout(zkSessionTimeout); solr.setZkConnectTimeout(zkSessionTimeout); String collection = connectionParameters.get(SolrConnectionParams.COLLECTION); solr.setDefaultCollection(collection); solr.setIdField(uniqueKeyField); return solr; }
protected CloudSolrClient getCommonCloudSolrClient() { synchronized (this) { if (commonCloudSolrClient == null) { commonCloudSolrClient = getCloudSolrClient(zkServer.getZkAddress(), random().nextBoolean(), 5000, 120000); commonCloudSolrClient.setDefaultCollection(DEFAULT_COLLECTION); commonCloudSolrClient.connect(); log.info("Created commonCloudSolrClient with updatesToLeaders={} and parallelUpdates={}", commonCloudSolrClient.isUpdatesToLeaders(), commonCloudSolrClient.isParallelUpdates()); } } return commonCloudSolrClient; }
private Set<SolrClient> createSolrClients(Map<String, String> indexConnectionParams) throws MalformedURLException { String solrMode = getSolrMode(indexConnectionParams); if (solrMode.equals("cloud")) { String indexZkHost = indexConnectionParams.get(SolrConnectionParams.ZOOKEEPER); String collectionName = indexConnectionParams.get(SolrConnectionParams.COLLECTION); CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(indexZkHost).build(); int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(getConf()); solrServer.setZkClientTimeout(zkSessionTimeout); solrServer.setZkConnectTimeout(zkSessionTimeout); solrServer.setDefaultCollection(collectionName); return Collections.singleton((SolrClient)solrServer); } else if (solrMode.equals("classic")) { PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager(); connectionManager.setDefaultMaxPerRoute(getSolrMaxConnectionsPerRoute(indexConnectionParams)); connectionManager.setMaxTotal(getSolrMaxConnectionsTotal(indexConnectionParams)); HttpClient httpClient = new DefaultHttpClient(connectionManager); return new HashSet<SolrClient>(createHttpSolrClients(indexConnectionParams, httpClient)); } else { throw new RuntimeException("Only 'cloud' and 'classic' are valid values for solr.mode, but got " + solrMode); } }
private Set<SolrClient> createSolrServers(Map<String, String> indexConnectionParams, String uniqueKeyField) throws MalformedURLException { String solrMode = getSolrMode(indexConnectionParams); if (solrMode.equals("cloud")) { String indexZkHost = indexConnectionParams.get(SolrConnectionParams.ZOOKEEPER); String collectionName = indexConnectionParams.get(SolrConnectionParams.COLLECTION); CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(indexZkHost).build(); int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(getConf()); solrServer.setZkClientTimeout(zkSessionTimeout); solrServer.setZkConnectTimeout(zkSessionTimeout); solrServer.setDefaultCollection(collectionName); solrServer.setIdField(uniqueKeyField); return Collections.singleton((SolrClient)solrServer); } else if (solrMode.equals("classic")) { PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager(); connectionManager.setDefaultMaxPerRoute(getSolrMaxConnectionsPerRoute(indexConnectionParams)); connectionManager.setMaxTotal(getSolrMaxConnectionsTotal(indexConnectionParams)); HttpClient httpClient = new DefaultHttpClient(connectionManager); return new HashSet<SolrClient>(createHttpSolrClients(indexConnectionParams, httpClient)); } else { throw new RuntimeException("Only 'cloud' and 'classic' are valid values for solr.mode, but got " + solrMode); } }
private CloudSolrClient getCloudServer() { CloudSolrClient rVal = null; if (this.getZkHost() != null && this.getCollection() != null) { rVal = new CloudSolrClient.Builder().withZkHost(zkHost).build(); rVal.setDefaultCollection(this.getCollection()); } return rVal; }
public SolrClient getSolrServer() { if (zkHost != null && zkHost.length() > 0) { if (collectionName == null || collectionName.length() == 0) { throw new MorphlineCompilationException("Parameter 'zkHost' requires that you also pass parameter 'collection'", config); } CloudSolrClient cloudSolrClient = new Builder() .withZkHost(zkHost) .build(); cloudSolrClient.setDefaultCollection(collectionName); cloudSolrClient.setZkClientTimeout(zkClientSessionTimeout); cloudSolrClient.setZkConnectTimeout(zkClientConnectTimeout); return cloudSolrClient; } else { if (solrUrl == null && solrHomeDir != null) { CoreContainer coreContainer = new CoreContainer(solrHomeDir); coreContainer.load(); EmbeddedSolrServer embeddedSolrServer = new EmbeddedSolrServer(coreContainer, collectionName); return embeddedSolrServer; } if (solrUrl == null || solrUrl.length() == 0) { throw new MorphlineCompilationException("Missing parameter 'solrUrl'", config); } int solrServerNumThreads = 2; int solrServerQueueLength = solrServerNumThreads; SolrClient server = new SafeConcurrentUpdateSolrServer(solrUrl, solrServerQueueLength, solrServerNumThreads); return server; } }
public static SolrClient getClient(Map stormConf, String boltType) { String zkHost = ConfUtils.getString(stormConf, "solr." + boltType + ".zkhost", null); String solrUrl = ConfUtils.getString(stormConf, "solr." + boltType + ".url", null); String collection = ConfUtils.getString(stormConf, "solr." + boltType + ".collection", null); int queueSize = ConfUtils.getInt(stormConf, "solr." + boltType + ".queueSize", -1); SolrClient client; if (StringUtils.isNotBlank(zkHost)) { client = new CloudSolrClient.Builder().withZkHost(zkHost).build(); if (StringUtils.isNotBlank(collection)) { ((CloudSolrClient) client).setDefaultCollection(collection); } } else if (StringUtils.isNotBlank(solrUrl)) { if (queueSize == -1) { client = new HttpSolrClient.Builder(solrUrl).build(); } else { client = new ConcurrentUpdateSolrClient.Builder(solrUrl) .withQueueSize(queueSize).build(); } } else { throw new RuntimeException( "SolrClient should have zk or solr URL set up"); } return client; }