public IndexerDefinitionBuilder startFrom(IndexerDefinition existingDefinition) { this.name = existingDefinition.getName(); this.lifecycleState = existingDefinition.getLifecycleState(); this.batchIndexingState = existingDefinition.getBatchIndexingState(); this.incrementalIndexingState = existingDefinition.getIncrementalIndexingState(); this.subscriptionId = existingDefinition.getSubscriptionId(); this.configuration = existingDefinition.getConfiguration(); this.indexerComponentFactory = existingDefinition.getIndexerComponentFactory(); this.connectionType = existingDefinition.getConnectionType(); this.connectionParams = existingDefinition.getConnectionParams(); this.defaultBatchIndexCliArguments = existingDefinition.getDefaultBatchIndexCliArguments(); this.batchIndexCliArguments = existingDefinition.getBatchIndexCliArguments(); this.lastBatchBuildInfo = existingDefinition.getLastBatchBuildInfo(); this.activeBatchBuildInfo = existingDefinition.getActiveBatchBuildInfo(); this.subscriptionTimestamp = existingDefinition.getSubscriptionTimestamp(); this.occVersion = existingDefinition.getOccVersion(); return this; }
private boolean needsSubscriptionIdUnassigned(IndexerDefinition indexer) { return indexer.getIncrementalIndexingState() == IncrementalIndexingState.DO_NOT_SUBSCRIBE && indexer.getSubscriptionId() != null; }
private boolean shouldRunIndexer(IndexerDefinition indexerDef) { return indexerDef.getIncrementalIndexingState() == IncrementalIndexingState.SUBSCRIBE_AND_CONSUME && indexerDef.getSubscriptionId() != null && !indexerDef.getLifecycleState().isDeleteState(); }
if (indexerProcessIds.containsKey(indexerDef.getName())) { indexerProcessRegistry.unregisterIndexerProcess(indexerProcessIds.remove(indexerDef.getName())); indexerProcessId = indexerProcessRegistry.registerIndexerProcess(indexerDef.getName(), hostName); indexerProcessIds.put(indexerDef.getName(), indexerProcessId); IndexerComponentFactory factory = IndexerComponentFactoryUtil.getComponentFactory(indexerDef.getIndexerComponentFactory(), new ByteArrayInputStream(indexerDef.getConfiguration()), indexerDef.getConnectionParams()); IndexerConf indexerConf = factory.createIndexerConf(); ResultToSolrMapper mapper = factory.createMapper(indexerDef.getName()); if (indexerDef.getConnectionType() == null || indexerDef.getConnectionType().equals("solr")) { Map<String, String> connectionParams = indexerDef.getConnectionParams(); String solrMode = SolrConnectionParamUtil.getSolrMode(connectionParams); if (solrMode.equals("cloud")) { int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(hbaseConf); solrWriter = new DirectSolrInputDocumentWriter(indexerDef.getName(), createCloudSolrClient(connectionParams, zkSessionTimeout)); } else if (solrMode.equals("classic")) { connectionManager = new PoolingClientConnectionManager(); solrWriter = new DirectSolrClassicInputDocumentWriter(indexerDef.getName(), solrServers); sharder = createSharder(connectionParams, solrServers.size()); } else { "Invalid connection type: " + indexerDef.getConnectionType() + ". Only 'solr' is supported"); Indexer indexer = Indexer.createIndexer(indexerDef.getName(), indexerConf, indexerConf.getTable(), mapper, htablePool, sharder, solrWriter); IndexingEventListener eventListener = new IndexingEventListener(
private String fetchIndexerTableName(String indexerName) throws Exception{ // best effort since this could be a pattern ... IndexerDefinition indexerDefinition = get(indexerName); IndexerComponentFactory factory = IndexerComponentFactoryUtil.getComponentFactory(indexerDefinition.getIndexerComponentFactory(), new ByteArrayInputStream(indexerDefinition.getConfiguration()), indexerDefinition.getConnectionParams()); String tableName = factory.createIndexerConf().getTable(); // TODO we should fail if the table does not exist return tableName; }
private void assertValid(IndexerDefinition indexer) throws IndexerValidityException { if (indexer.getName() == null || indexer.getName().length() == 0) throw new IndexerValidityException("Name should not be null or zero-length"); if (indexer.getConfiguration() == null) throw new IndexerValidityException("Configuration should not be null."); if (indexer.getLifecycleState() == null) throw new IndexerValidityException("General state should not be null."); if (indexer.getBatchIndexingState() == null) throw new IndexerValidityException("Build state should not be null."); if (indexer.getIncrementalIndexingState() == null) throw new IndexerValidityException("Update state should not be null."); if (indexer.getBatchIndexCliArguments() != null && indexer.getBatchIndexingState() != BatchIndexingState.BUILD_REQUESTED) { throw new IndexerValidityException("The build state must be set to BUILD_REQUESTED when setting batchIndexCliArguments");
private void restartIndexer(IndexerDefinition indexerDef) { IndexerHandle handle = indexers.get(indexerDef.getName()); if (handle.indexerDef.getOccVersion() >= indexerDef.getOccVersion()) { return; } boolean relevantChanges = !Arrays.equals(handle.indexerDef.getConfiguration(), indexerDef.getConfiguration()) || Objects.equal(handle.indexerDef.getConnectionType(), indexerDef.getConnectionType()) || !Objects.equal(handle.indexerDef.getConnectionParams(), indexerDef.getConnectionParams()); if (!relevantChanges) { return; } if (stopIndexer(indexerDef.getName())) { startIndexer(indexerDef); } }
byte[] indexerConfXml = indexDef.getConfiguration(); IndexerConf indexerConf = null; if (!"org.lilyproject.indexer.hbase.mapper.LilyIndexerComponentFactory".equals(indexDef.getIndexerComponentFactory())) { continue; IndexerComponentFactory factory = IndexerComponentFactoryUtil.getComponentFactory(indexDef.getIndexerComponentFactory(), new ByteArrayInputStream(indexDef.getConfiguration()), indexDef.getConnectionParams()); factory.configure(new ByteArrayInputStream(indexerConfXml), indexDef.getConnectionParams()); indexerConf = factory.createIndexerConf(); newIndexInfos.put(indexDef.getName(), new IndexInfo(indexDef, indexerConf, repositoryManager)); } catch (Throwable t) { log.error("Error parsing indexer conf", t);
IndexerDefinition currentIndexer = getFreshIndexer(indexer.getName()); if (currentIndexer.getLifecycleState() == LifecycleState.DELETE_REQUESTED || currentIndexer.getLifecycleState() == LifecycleState.DELETING) { throw new IndexerUpdateException("An indexer in state " + indexer.getLifecycleState() + " cannot be modified."); if (indexer.getBatchIndexingState() == BatchIndexingState.BUILD_REQUESTED && currentIndexer.getBatchIndexingState() != BatchIndexingState.INACTIVE && currentIndexer.getBatchIndexingState() != BatchIndexingState.BUILD_REQUESTED) { throw new IndexerUpdateException("Cannot move batch indexing state from " + currentIndexer.getBatchIndexingState() + " to " + indexer.getBatchIndexingState()); if (currentIndexer.getLifecycleState() == LifecycleState.DELETE_REQUESTED) { throw new IndexerUpdateException("An indexer in the state " + LifecycleState.DELETE_REQUESTED + " cannot be updated."); if (!Objects.equal(currentIndexer.getActiveBatchBuildInfo(), indexer.getActiveBatchBuildInfo())) { throw new IndexerUpdateException("The active batch build info cannot be modified by users."); if (!Objects.equal(currentIndexer.getLastBatchBuildInfo(), indexer.getLastBatchBuildInfo())) { throw new IndexerUpdateException("The last batch build info cannot be modified by users.");
private String[] createBatchArguments(IndexerDefinition indexer) { String[] batchIndexArguments = indexer.getBatchIndexCliArgumentsOrDefault(); List<String> args = Lists.newArrayList(); String mode = Optional.fromNullable(indexer.getConnectionParams().get(SolrConnectionParams.MODE)).or("cloud").toLowerCase(); if ("cloud".equals(mode)) { // cloud mode is the default args.add("--zk-host"); args.add(indexer.getConnectionParams().get(SolrConnectionParams.ZOOKEEPER)); } else { for (String shard : SolrConnectionParamUtil.getShards(indexer.getConnectionParams())) { args.add("--shard-url"); args.add(shard); } } args.add("--hbase-indexer-zk"); args.add(zkConnectString); args.add("--hbase-indexer-name"); args.add(indexer.getName()); args.add("--reducers"); args.add("0"); // additional arguments that were configured on the index (e.g. HBase scan parameters) args.addAll(Lists.newArrayList(batchIndexArguments)); return args.toArray(new String[args.size()]); }
log.debug("EventWorker indexer.getLifecycleState: " + indexer.getLifecycleState()); if (indexer.getLifecycleState() == IndexerDefinition.LifecycleState.DELETE_REQUESTED) { canBeDeleted = true; String queueSubscriptionId = indexer.getSubscriptionId(); log.debug("EventWorker before removeSubscription for " + queueSubscriptionId); if (queueSubscriptionId != null) { sepModel.removeSubscription(indexer.getSubscriptionId()); if (indexer.getActiveBatchBuildInfo() != null) { JobClient jobClient = getJobClient(); Set<String> jobs = indexer.getActiveBatchBuildInfo().getMapReduceJobTrackingUrls().keySet(); for (String jobId : jobs) { RunningJob job = jobClient.getJob(jobId); indexerModel.updateIndexerInternal(indexer); } else if (indexer.getLifecycleState() == IndexerDefinition.LifecycleState.DELETING) { if (indexer.getActiveBatchBuildInfo() == null) { canBeDeleted = true;
oldIndexerDef != null ? oldIndexerDef.getConnectionParams() : null); if (connectionParams != null) builder.connectionParams(connectionParams); if (oldIndexerDef == null || oldIndexerDef.getIndexerComponentFactory() == null) builder.indexerComponentFactory(indexerComponentFactoryOption.value(options));
public IndexInfo(IndexerDefinition indexDefinition, IndexerConf indexerConf, RepositoryManager repositoryManager) throws IndexerConfException, RepositoryException, InterruptedException { this.indexDefinition = indexDefinition; this.indexerConf = indexerConf; String repoParam = indexDefinition.getConnectionParams().get(LResultToSolrMapper.REPO_KEY); LRepository repository = repositoryName == null ? repositoryManager.getDefaultRepository() : repositoryManager.getRepository(repositoryName); repositoryName = repoParam == null ? RepoAndTableUtil.DEFAULT_REPOSITORY : repoParam; this.lilyIndexerConf = LilyIndexerConfBuilder.build(new ByteArrayInputStream(indexDefinition.getConfiguration()), repository); }
private boolean needsBatchBuildStart(IndexerDefinition indexer) { return !indexer.getLifecycleState().isDeleteState() && indexer.getBatchIndexingState() == BatchIndexingState.BUILD_REQUESTED && indexer.getActiveBatchBuildInfo() == null; }
log.debug("Checking batch state for " + indexerDefinition.getName()); BatchBuildInfo batchBuildInfo = indexerDefinition.getActiveBatchBuildInfo(); if (batchBuildInfo != null) { Set<String> jobs = batchBuildInfo.getMapReduceJobTrackingUrls().keySet(); job = jobClient.getJob(JobID.forName(jobId)); } catch (IOException e) { log.error("Could not get job " + jobId + " for index " + indexerDefinition.getName() + " while checking active build info.",e); batchDone = false; } catch (IOException e) { log.error("Could not get jobstate for job " + jobId + " for index " + indexerDefinition.getName() + " while checking active build info.",e); batchDone = false; break; } else { log.warn("Could not find job " + jobId + " while checking active batch builds for indexer " + indexerDefinition.getName()); markBatchBuildCompleted(indexerDefinition.getName(), overAllSuccess); } else { executor.schedule(this, pollInterval, TimeUnit.MILLISECONDS);
IndexerDefinition newIndexer = input.getNewIndexer(); if (!newIndexer.getName().equals(indexName)) { throw new IndexerServerException(HttpServletResponse.SC_BAD_REQUEST, new IndexerUpdateException("Requested update of indexer: " + indexName + " but indexer parameter has name: " + newIndexer.getName())); try { IndexerDefinition oldIndexer = model.getFreshIndexer(indexName); if (!oldIndexer.equals(baseIndexer)) { throw new IndexerServerException(HttpServletResponse.SC_BAD_REQUEST, new IndexerConcurrentModificationException( if (newIndexer.equals(oldIndexer)) { return toJsonResultBytes("Indexer " + indexName + " already matches the specified settings, did not update it."); } else { && newIndexer.getLifecycleState() == LifecycleState.DELETE_REQUESTED; model.unlockIndexer(lock, ignoreMissing);
/** * Get a single index configuration (as stored in zookeeper). */ @GET @Path("{name}/config") @Produces("application/json") public Response getConfig(@PathParam("name") String name) throws IndexerNotFoundException, IOException { IndexerDefinition index = getModel().getIndexer(name); ObjectMapper m = new ObjectMapper(); ObjectNode json = m.createObjectNode(); json.put("occVersion", index.getOccVersion()); json.put("config", new String(index.getConfiguration(), Charsets.UTF_8)); return Response.ok(m.writeValueAsString(json), new MediaType("application", "json")).build(); }
@Override public void addIndexer(IndexerDefinition indexer) throws IndexerExistsException, IndexerModelException, IndexerValidityException { assertValid(indexer); if (indexer.getIncrementalIndexingState() != IndexerDefinition.IncrementalIndexingState.DO_NOT_SUBSCRIBE) { indexer = new IndexerDefinitionBuilder().startFrom(indexer).subscriptionTimestamp(System.currentTimeMillis()).build(); } final String indexerPath = indexerCollectionPath + "/" + indexer.getName(); final byte[] data = IndexerDefinitionJsonSerDeser.INSTANCE.toJsonBytes(indexer); try { zk.retryOperation(new ZooKeeperOperation<String>() { @Override public String execute() throws KeeperException, InterruptedException { return zk.create(indexerPath, data, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); } }); } catch (KeeperException.NodeExistsException e) { throw new IndexerExistsException(indexer.getName()); } catch (Exception e) { throw new IndexerModelException("Error creating indexer.", e); } }
if (indexer.getLifecycleState() == IndexerDefinition.LifecycleState.DELETE_REQUESTED || indexer.getLifecycleState() == IndexerDefinition.LifecycleState.DELETING) { prepareDeleteIndex(indexer.getName()); for (IndexerLifecycleListener lifecycleListener : lifecycleListeners) { lifecycleListener.onDelete(indexer); assignSubscription(indexer.getName()); for (IndexerLifecycleListener lifecycleListener : lifecycleListeners) { lifecycleListener.onSubscribe(indexer); unassignSubscription(indexer.getName()); for (IndexerLifecycleListener lifecycleListener : lifecycleListeners) { lifecycleListener.onUnsubscribe(indexer); startFullIndexBuild(indexer.getName()); for (IndexerLifecycleListener lifecycleListener : lifecycleListeners) { lifecycleListener.onBatchBuild(indexer);