/** * Creates a new core, associated with a collection, in Solr. */ public void createCore(String coreName, String collectionName, String configName, int numShards) throws IOException { createCore(coreName, collectionName, configName, numShards, null); }
/** * Instantiate with the name of the configuration directory and the ZooKeeper used to access it. */ public SolrConfigLoader(String collection, ZooKeeper zooKeeper) { super(Paths.get("solr")); this.zk = zooKeeper; this.configZkPath = getCollectionConfigPath(collection); }
/** * Utility method to upload a Solr config into ZooKeeper. This method only allows to supply schema and * solrconf, if you want to upload a full directory use {@link #uploadConfig(String, File)}. */ public void uploadConfig(String confName, byte[] schema, byte[] solrconf) throws IOException { // Write schema & solrconf to temporary dir, upload dir, delete tmp dir File tmpConfDir = Files.createTempDir(); Files.copy(ByteStreams.newInputStreamSupplier(schema), new File(tmpConfDir, "schema.xml")); Files.copy(ByteStreams.newInputStreamSupplier(solrconf), new File(tmpConfDir, "solrconfig.xml")); uploadConfig(confName, tmpConfDir); FileUtils.deleteDirectory(tmpConfDir); }
private Set<SolrClient> createSolrClients(Map<String, String> indexConnectionParams) throws MalformedURLException { String solrMode = getSolrMode(indexConnectionParams); if (solrMode.equals("cloud")) { String indexZkHost = indexConnectionParams.get(SolrConnectionParams.ZOOKEEPER); String collectionName = indexConnectionParams.get(SolrConnectionParams.COLLECTION); CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(indexZkHost).build(); int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(getConf()); solrServer.setZkClientTimeout(zkSessionTimeout); solrServer.setZkConnectTimeout(zkSessionTimeout); solrServer.setDefaultCollection(collectionName); return Collections.singleton((SolrClient)solrServer); } else if (solrMode.equals("classic")) { PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager(); connectionManager.setDefaultMaxPerRoute(getSolrMaxConnectionsPerRoute(indexConnectionParams)); connectionManager.setMaxTotal(getSolrMaxConnectionsTotal(indexConnectionParams)); HttpClient httpClient = new DefaultHttpClient(connectionManager); return new HashSet<SolrClient>(createHttpSolrClients(indexConnectionParams, httpClient)); } else { throw new RuntimeException("Only 'cloud' and 'classic' are valid values for solr.mode, but got " + solrMode); } }
public static List<SolrClient> createHttpSolrClients(Map<String, String> connectionParams, HttpClient httpClient) { List<SolrClient> result = Lists.newArrayList(); for (String shard : SolrConnectionParamUtil.getShards(connectionParams)) { result.add(new HttpSolrClient.Builder(shard).withHttpClient(httpClient).build()); } if (result.size() == 0) { throw new RuntimeException( String.format("You need to specify at least one solr shard connection parameter (%s0={url})", SolrConnectionParams.SOLR_SHARD_PREFIX)); } return result; }
private DirectSolrClassicInputDocumentWriter createClassicSolrWriter(Context context, Map<String, String> indexConnectionParams) throws IOException { PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager(); connectionManager.setDefaultMaxPerRoute(getSolrMaxConnectionsPerRoute(indexConnectionParams)); connectionManager.setMaxTotal(getSolrMaxConnectionsTotal(indexConnectionParams)); HttpClient httpClient = new DefaultHttpClient(connectionManager); List<SolrClient> solrServers = createHttpSolrClients(indexConnectionParams, httpClient); return new DirectSolrClassicInputDocumentWriter( context.getConfiguration().get(INDEX_NAME_CONF_KEY), solrServers); }
public void start() throws Exception { // Make the Solr home directory this.tmpDir = Files.createTempDir(); this.solrHomeDir = new File(tmpDir, "home"); if (!this.solrHomeDir.mkdir()) { throw new RuntimeException("Failed to create directory " + this.solrHomeDir.getAbsolutePath()); } writeSolrXml(); // Set required system properties System.setProperty("solr.solr.home", solrHomeDir.getAbsolutePath()); System.setProperty("zkHost", zkConnectString); System.setProperty("solr.port", Integer.toString(solrPort)); for (Entry<String, String> entry : configProperties.entrySet()) { System.setProperty(entry.getKey().toString(), entry.getValue()); } jettySolrRunner = createServer(); jettySolrRunner.start(); }
private Indexer createIndexer(String indexName, Context context, IndexerConf indexerConf, String tableName, ResultToSolrMapper mapper, Map<String, String> indexConnectionParams) throws IOException, SharderException { Configuration conf = context.getConfiguration(); if (conf.getBoolean(INDEX_DIRECT_WRITE_CONF_KEY, false)) { String solrMode = getSolrMode(indexConnectionParams); if (solrMode.equals("cloud")) { DirectSolrInputDocumentWriter writer = createCloudSolrWriter(context, indexConnectionParams); solrDocWriter = wrapInBufferedWriter(context, writer); return Indexer.createIndexer(indexName, indexerConf, tableName, mapper, null, null, solrDocWriter); } else if (solrMode.equals("classic")) { DirectSolrClassicInputDocumentWriter classicSolrWriter = createClassicSolrWriter(context, indexConnectionParams); Sharder sharder = createSharder(indexConnectionParams, classicSolrWriter.getNumServers()); solrDocWriter = wrapInBufferedWriter(context, classicSolrWriter); return Indexer.createIndexer(indexName, indexerConf, tableName, mapper, null, sharder, solrDocWriter); } else { throw new RuntimeException("Only 'cloud' and 'classic' are valid values for solr.mode, but got " + solrMode); } } else { solrDocWriter = new MapReduceSolrInputDocumentWriter(context); return Indexer.createIndexer(indexName, indexerConf, tableName, mapper, null, null, solrDocWriter); } }
/** * Create a Solr collection with a given number of shards. * * @param collectionName name of the collection to be created * @param configName name of the config for the collection * @param numShards number of shards in the collection */ public void createCollection(String collectionName, String configName, int numShards) throws IOException { int TIMEOUT = 30; // seconds CloudSolrClient solrClient = new CloudSolrClient.Builder().withZkHost(getZkConnectString()).build(); solrClient.setDefaultCollection(collectionName); CollectionAdminRequest.Create request = CollectionAdminRequest.createCollection( collectionName, configName, numShards, 1); request.setMaxShardsPerNode(numShards); RequestStatusState state; try { state = request.processAndWait(solrClient, TIMEOUT); } catch (SolrServerException | InterruptedException e) { throw new RuntimeException(e); } if (state != RequestStatusState.COMPLETED && state != RequestStatusState.RUNNING) { throw new IllegalStateException("Unexpected state: " + state); } waitForRecoveriesToFinish( collectionName, solrClient.getZkStateReader(), true, true, TIMEOUT); }
private Set<SolrClient> createSolrServers(Map<String, String> indexConnectionParams, String uniqueKeyField) throws MalformedURLException { String solrMode = getSolrMode(indexConnectionParams); if (solrMode.equals("cloud")) { String indexZkHost = indexConnectionParams.get(SolrConnectionParams.ZOOKEEPER); String collectionName = indexConnectionParams.get(SolrConnectionParams.COLLECTION); CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(indexZkHost).build(); int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(getConf()); solrServer.setZkClientTimeout(zkSessionTimeout); solrServer.setZkConnectTimeout(zkSessionTimeout); solrServer.setDefaultCollection(collectionName); solrServer.setIdField(uniqueKeyField); return Collections.singleton((SolrClient)solrServer); } else if (solrMode.equals("classic")) { PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager(); connectionManager.setDefaultMaxPerRoute(getSolrMaxConnectionsPerRoute(indexConnectionParams)); connectionManager.setMaxTotal(getSolrMaxConnectionsTotal(indexConnectionParams)); HttpClient httpClient = new DefaultHttpClient(connectionManager); return new HashSet<SolrClient>(createHttpSolrClients(indexConnectionParams, httpClient)); } else { throw new RuntimeException("Only 'cloud' and 'classic' are valid values for solr.mode, but got " + solrMode); } }
public static List<SolrClient> createHttpSolrClients(Map<String, String> connectionParams, HttpClient httpClient) { List<SolrClient> result = Lists.newArrayList(); for (String shard : SolrConnectionParamUtil.getShards(connectionParams)) { result.add(new HttpSolrClient.Builder(shard).withHttpClient(httpClient).build()); } if (result.size() == 0) { throw new RuntimeException( String.format("You need to specify at least one solr shard connection parameter (%s0={url})", SolrConnectionParams.SOLR_SHARD_PREFIX)); } return result; }
private DirectSolrClassicInputDocumentWriter createClassicSolrWriter(Context context, Map<String, String> indexConnectionParams) throws IOException { PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager(); connectionManager.setDefaultMaxPerRoute(getSolrMaxConnectionsPerRoute(indexConnectionParams)); connectionManager.setMaxTotal(getSolrMaxConnectionsTotal(indexConnectionParams)); HttpClient httpClient = new DefaultHttpClient(connectionManager); List<SolrClient> solrServers = createHttpSolrClients(indexConnectionParams, httpClient); return new DirectSolrClassicInputDocumentWriter( context.getConfiguration().get(INDEX_NAME_CONF_KEY), solrServers); }
public void start() throws Exception { // Make the Solr home directory this.tmpDir = Files.createTempDir(); this.solrHomeDir = new File(tmpDir, "home"); if (!this.solrHomeDir.mkdir()) { throw new RuntimeException("Failed to create directory " + this.solrHomeDir.getAbsolutePath()); } writeSolrXml(); // Set required system properties System.setProperty("solr.solr.home", solrHomeDir.getAbsolutePath()); System.setProperty("zkHost", zkConnectString); System.setProperty("solr.port", Integer.toString(solrPort)); for (Entry<String, String> entry : configProperties.entrySet()) { System.setProperty(entry.getKey().toString(), entry.getValue()); } jettySolrRunner = createServer(); jettySolrRunner.start(); }
private Indexer createIndexer(String indexName, Context context, IndexerConf indexerConf, String tableName, ResultToSolrMapper mapper, Map<String, String> indexConnectionParams) throws IOException, SharderException { Configuration conf = context.getConfiguration(); if (conf.getBoolean(INDEX_DIRECT_WRITE_CONF_KEY, false)) { String solrMode = getSolrMode(indexConnectionParams); if (solrMode.equals("cloud")) { DirectSolrInputDocumentWriter writer = createCloudSolrWriter( context, indexConnectionParams, indexerConf.getUniqueKeyField()); solrDocWriter = wrapInBufferedWriter(context, writer); return Indexer.createIndexer(indexName, indexerConf, tableName, mapper, null, null, solrDocWriter); } else if (solrMode.equals("classic")) { DirectSolrClassicInputDocumentWriter classicSolrWriter = createClassicSolrWriter(context, indexConnectionParams); Sharder sharder = createSharder(indexConnectionParams, classicSolrWriter.getNumServers()); solrDocWriter = wrapInBufferedWriter(context, classicSolrWriter); return Indexer.createIndexer(indexName, indexerConf, tableName, mapper, null, sharder, solrDocWriter); } else { throw new RuntimeException("Only 'cloud' and 'classic' are valid values for solr.mode, but got " + solrMode); } } else { solrDocWriter = new MapReduceSolrInputDocumentWriter(context); return Indexer.createIndexer(indexName, indexerConf, tableName, mapper, null, null, solrDocWriter); } }
/** * Create a Solr collection with a given number of shards. * * @param collectionName name of the collection to be created * @param configName name of the config for the collection * @param numShards number of shards in the collection */ public void createCollection(String collectionName, String configName, int numShards) throws IOException { for (int shardIndex = 0; shardIndex < numShards; shardIndex++) { String coreName = String.format("%s_shard%d", collectionName, shardIndex + 1); createCore(coreName, collectionName, configName, numShards, coreName + "_data"); } }
/** * Instantiate with the name of the configuration directory and the ZooKeeper used to access it. */ public SolrConfigLoader(String collection, ZooKeeper zooKeeper) { super(Paths.get("solr")); this.zk = zooKeeper; this.configZkPath = getCollectionConfigPath(collection); }
/** * Utility method to upload a Solr config into ZooKeeper. This method only allows to supply schema and * solrconf, if you want to upload a full directory use {@link #uploadConfig(String, File)}. */ public void uploadConfig(String confName, byte[] schema, byte[] solrconf) throws IOException { // Write schema & solrconf to temporary dir, upload dir, delete tmp dir File tmpConfDir = Files.createTempDir(); Files.copy(ByteStreams.newInputStreamSupplier(schema), new File(tmpConfDir, "managed-schema")); Files.copy(ByteStreams.newInputStreamSupplier(solrconf), new File(tmpConfDir, "solrconfig.xml")); uploadConfig(confName, tmpConfDir); FileUtils.deleteDirectory(tmpConfDir); }
private String[] createBatchArguments(IndexerDefinition indexer) { String[] batchIndexArguments = indexer.getBatchIndexCliArgumentsOrDefault(); List<String> args = Lists.newArrayList(); String mode = Optional.fromNullable(indexer.getConnectionParams().get(SolrConnectionParams.MODE)).or("cloud").toLowerCase(); if ("cloud".equals(mode)) { // cloud mode is the default args.add("--zk-host"); args.add(indexer.getConnectionParams().get(SolrConnectionParams.ZOOKEEPER)); } else { for (String shard : SolrConnectionParamUtil.getShards(indexer.getConnectionParams())) { args.add("--shard-url"); args.add(shard); } } args.add("--hbase-indexer-zk"); args.add(zkConnectString); args.add("--hbase-indexer-name"); args.add(indexer.getName()); args.add("--reducers"); args.add("0"); // additional arguments that were configured on the index (e.g. HBase scan parameters) args.addAll(Lists.newArrayList(batchIndexArguments)); return args.toArray(new String[args.size()]); }
private String[] createBatchArguments(IndexerDefinition indexer) { String[] batchIndexArguments = indexer.getBatchIndexCliArgumentsOrDefault(); List<String> args = Lists.newArrayList(); String mode = Optional.fromNullable(indexer.getConnectionParams().get(SolrConnectionParams.MODE)).or("cloud").toLowerCase(); if ("cloud".equals(mode)) { // cloud mode is the default args.add("--zk-host"); args.add(indexer.getConnectionParams().get(SolrConnectionParams.ZOOKEEPER)); } else { for (String shard : SolrConnectionParamUtil.getShards(indexer.getConnectionParams())) { args.add("--shard-url"); args.add(shard); } } args.add("--hbase-indexer-zk"); args.add(zkConnectString); args.add("--hbase-indexer-name"); args.add(indexer.getName()); args.add("--reducers"); args.add("0"); // additional arguments that were configured on the index (e.g. HBase scan parameters) args.addAll(Lists.newArrayList(batchIndexArguments)); return args.toArray(new String[args.size()]); }
if (SolrConnectionParamUtil.getShards(connectionParams).size() == 0) { throw new CliException("ERROR: You need at least one shard when using solr classic");