private String[] createBatchArguments(IndexerDefinition indexer) { String[] batchIndexArguments = indexer.getBatchIndexCliArgumentsOrDefault(); List<String> args = Lists.newArrayList(); String mode = Optional.fromNullable(indexer.getConnectionParams().get(SolrConnectionParams.MODE)).or("cloud").toLowerCase(); if ("cloud".equals(mode)) { // cloud mode is the default args.add("--zk-host"); args.add(indexer.getConnectionParams().get(SolrConnectionParams.ZOOKEEPER)); } else { for (String shard : SolrConnectionParamUtil.getShards(indexer.getConnectionParams())) { args.add("--shard-url"); args.add(shard); } } args.add("--hbase-indexer-zk"); args.add(zkConnectString); args.add("--hbase-indexer-name"); args.add(indexer.getName()); args.add("--reducers"); args.add("0"); // additional arguments that were configured on the index (e.g. HBase scan parameters) args.addAll(Lists.newArrayList(batchIndexArguments)); return args.toArray(new String[args.size()]); }
private String[] createBatchArguments(IndexerDefinition indexer) { String[] batchIndexArguments = indexer.getBatchIndexCliArgumentsOrDefault(); List<String> args = Lists.newArrayList(); String mode = Optional.fromNullable(indexer.getConnectionParams().get(SolrConnectionParams.MODE)).or("cloud").toLowerCase(); if ("cloud".equals(mode)) { // cloud mode is the default args.add("--zk-host"); args.add(indexer.getConnectionParams().get(SolrConnectionParams.ZOOKEEPER)); } else { for (String shard : SolrConnectionParamUtil.getShards(indexer.getConnectionParams())) { args.add("--shard-url"); args.add(shard); } } args.add("--hbase-indexer-zk"); args.add(zkConnectString); args.add("--hbase-indexer-name"); args.add(indexer.getName()); args.add("--reducers"); args.add("0"); // additional arguments that were configured on the index (e.g. HBase scan parameters) args.addAll(Lists.newArrayList(batchIndexArguments)); return args.toArray(new String[args.size()]); }
public IndexInfo(IndexerDefinition indexDefinition, IndexerConf indexerConf, RepositoryManager repositoryManager) throws IndexerConfException, RepositoryException, InterruptedException { this.indexDefinition = indexDefinition; this.indexerConf = indexerConf; String repoParam = indexDefinition.getConnectionParams().get(LResultToSolrMapper.REPO_KEY); LRepository repository = repositoryName == null ? repositoryManager.getDefaultRepository() : repositoryManager.getRepository(repositoryName); repositoryName = repoParam == null ? RepoAndTableUtil.DEFAULT_REPOSITORY : repoParam; this.lilyIndexerConf = LilyIndexerConfBuilder.build(new ByteArrayInputStream(indexDefinition.getConfiguration()), repository); }
private void restartIndexer(IndexerDefinition indexerDef) { IndexerHandle handle = indexers.get(indexerDef.getName()); if (handle.indexerDef.getOccVersion() >= indexerDef.getOccVersion()) { return; } boolean relevantChanges = !Arrays.equals(handle.indexerDef.getConfiguration(), indexerDef.getConfiguration()) || Objects.equal(handle.indexerDef.getConnectionType(), indexerDef.getConnectionType()) || !Objects.equal(handle.indexerDef.getConnectionParams(), indexerDef.getConnectionParams()); if (!relevantChanges) { return; } if (stopIndexer(indexerDef.getName())) { startIndexer(indexerDef); } }
private void restartIndexer(IndexerDefinition indexerDef) { IndexerHandle handle = indexers.get(indexerDef.getName()); if (handle.indexerDef.getOccVersion() >= indexerDef.getOccVersion()) { return; } boolean relevantChanges = !Arrays.equals(handle.indexerDef.getConfiguration(), indexerDef.getConfiguration()) || Objects.equal(handle.indexerDef.getConnectionType(), indexerDef.getConnectionType()) || !Objects.equal(handle.indexerDef.getConnectionParams(), indexerDef.getConnectionParams()); if (!relevantChanges) { return; } if (stopIndexer(indexerDef.getName())) { startIndexer(indexerDef); } }
private String fetchIndexerTableName(String indexerName) throws Exception{ // best effort since this could be a pattern ... IndexerDefinition indexerDefinition = get(indexerName); IndexerComponentFactory factory = IndexerComponentFactoryUtil.getComponentFactory(indexerDefinition.getIndexerComponentFactory(), new ByteArrayInputStream(indexerDefinition.getConfiguration()), indexerDefinition.getConnectionParams()); String tableName = factory.createIndexerConf().getTable(); // TODO we should fail if the table does not exist return tableName; }
private String fetchIndexerTableName(String indexerName) throws Exception{ // best effort since this could be a pattern ... IndexerDefinition indexerDefinition = get(indexerName); IndexerComponentFactory factory = IndexerComponentFactoryUtil.getComponentFactory(indexerDefinition.getIndexerComponentFactory(), new ByteArrayInputStream(indexerDefinition.getConfiguration()), indexerDefinition.getConnectionParams()); String tableName = factory.createIndexerConf().getTable(); // TODO we should fail if the table does not exist return tableName; }
hbaseIndexerComponentFactory = indexerDefinition.getIndexerComponentFactory(); configuration = indexerDefinition.getConfiguration(); if (indexerDefinition.getConnectionParams() != null) { indexConnectionParams.putAll(indexerDefinition.getConnectionParams());
ps.println(" + Connection type: " + indexer.getConnectionType()); ps.println(" + Connection params:"); if (indexer.getConnectionParams() == null) { ps.println(" + (none)"); } else { for (Map.Entry<String, String> entry : indexer.getConnectionParams().entrySet()) { ps.println(" + " + entry.getKey() + " = " + entry.getValue());
hbaseIndexerComponentFactory = indexerDefinition.getIndexerComponentFactory(); configuration = indexerDefinition.getConfiguration(); if (indexerDefinition.getConnectionParams() != null) { indexConnectionParams.putAll(indexerDefinition.getConnectionParams());
IndexerComponentFactory factory = IndexerComponentFactoryUtil.getComponentFactory(indexDef.getIndexerComponentFactory(), new ByteArrayInputStream(indexDef.getConfiguration()), indexDef.getConnectionParams()); factory.configure(new ByteArrayInputStream(indexerConfXml), indexDef.getConnectionParams()); indexerConf = factory.createIndexerConf();
ps.println(" + Connection type: " + indexer.getConnectionType()); ps.println(" + Connection params:"); if (indexer.getConnectionParams() == null) { ps.println(" + (none)"); } else { for (Map.Entry<String, String> entry : indexer.getConnectionParams().entrySet()) { ps.println(" + " + entry.getKey() + " = " + entry.getValue());
IndexerComponentFactory factory = IndexerComponentFactoryUtil.getComponentFactory(indexerDef.getIndexerComponentFactory(), new ByteArrayInputStream(indexerDef.getConfiguration()), indexerDef.getConnectionParams()); IndexerConf indexerConf = factory.createIndexerConf(); Map<String, String> connectionParams = indexerDef.getConnectionParams(); String solrMode = SolrConnectionParamUtil.getSolrMode(connectionParams); if (solrMode.equals("cloud")) {
IndexerComponentFactory factory = IndexerComponentFactoryUtil.getComponentFactory(indexer.getIndexerComponentFactory(), new ByteArrayInputStream(indexer.getConfiguration()), indexer.getConnectionParams()); IndexerConf indexerConf = factory.createIndexerConf(); if (indexerConf.tableNameIsRegex()) {
public IndexerDefinitionBuilder startFrom(IndexerDefinition existingDefinition) { this.name = existingDefinition.getName(); this.lifecycleState = existingDefinition.getLifecycleState(); this.batchIndexingState = existingDefinition.getBatchIndexingState(); this.incrementalIndexingState = existingDefinition.getIncrementalIndexingState(); this.subscriptionId = existingDefinition.getSubscriptionId(); this.configuration = existingDefinition.getConfiguration(); this.indexerComponentFactory = existingDefinition.getIndexerComponentFactory(); this.connectionType = existingDefinition.getConnectionType(); this.connectionParams = existingDefinition.getConnectionParams(); this.defaultBatchIndexCliArguments = existingDefinition.getDefaultBatchIndexCliArguments(); this.batchIndexCliArguments = existingDefinition.getBatchIndexCliArguments(); this.lastBatchBuildInfo = existingDefinition.getLastBatchBuildInfo(); this.activeBatchBuildInfo = existingDefinition.getActiveBatchBuildInfo(); this.subscriptionTimestamp = existingDefinition.getSubscriptionTimestamp(); this.occVersion = existingDefinition.getOccVersion(); return this; }
public IndexerDefinitionBuilder startFrom(IndexerDefinition existingDefinition) { this.name = existingDefinition.getName(); this.lifecycleState = existingDefinition.getLifecycleState(); this.batchIndexingState = existingDefinition.getBatchIndexingState(); this.incrementalIndexingState = existingDefinition.getIncrementalIndexingState(); this.subscriptionId = existingDefinition.getSubscriptionId(); this.configuration = existingDefinition.getConfiguration(); this.indexerComponentFactory = existingDefinition.getIndexerComponentFactory(); this.connectionType = existingDefinition.getConnectionType(); this.connectionParams = existingDefinition.getConnectionParams(); this.defaultBatchIndexCliArguments = existingDefinition.getDefaultBatchIndexCliArguments(); this.batchIndexCliArguments = existingDefinition.getBatchIndexCliArguments(); this.lastBatchBuildInfo = existingDefinition.getLastBatchBuildInfo(); this.activeBatchBuildInfo = existingDefinition.getActiveBatchBuildInfo(); this.subscriptionTimestamp = existingDefinition.getSubscriptionTimestamp(); this.occVersion = existingDefinition.getOccVersion(); return this; }
node.put("connectionType", indexer.getConnectionType()); if (indexer.getConnectionParams() != null) { ObjectNode paramsNode = node.putObject("connectionParams"); for (Map.Entry<String, String> entry : indexer.getConnectionParams().entrySet()) { paramsNode.put(entry.getKey(), entry.getValue());
node.put("connectionType", indexer.getConnectionType()); if (indexer.getConnectionParams() != null) { ObjectNode paramsNode = node.putObject("connectionParams"); for (Map.Entry<String, String> entry : indexer.getConnectionParams().entrySet()) { paramsNode.put(entry.getKey(), entry.getValue());
oldIndexerDef != null ? oldIndexerDef.getConnectionParams() : null); if (connectionParams != null) builder.connectionParams(connectionParams);
oldIndexerDef != null ? oldIndexerDef.getConnectionParams() : null); if (connectionParams != null) builder.connectionParams(connectionParams);