@Override public Stat execute() throws KeeperException, InterruptedException { return zk.setData(indexerCollectionPathSlash + indexer.getName(), newData, indexer.getOccVersion()); } });
private String[] createBatchArguments(IndexerDefinition indexer) { String[] batchIndexArguments = indexer.getBatchIndexCliArgumentsOrDefault(); List<String> args = Lists.newArrayList(); String mode = Optional.fromNullable(indexer.getConnectionParams().get(SolrConnectionParams.MODE)).or("cloud").toLowerCase(); if ("cloud".equals(mode)) { // cloud mode is the default args.add("--zk-host"); args.add(indexer.getConnectionParams().get(SolrConnectionParams.ZOOKEEPER)); } else { for (String shard : SolrConnectionParamUtil.getShards(indexer.getConnectionParams())) { args.add("--shard-url"); args.add(shard); } } args.add("--hbase-indexer-zk"); args.add(zkConnectString); args.add("--hbase-indexer-name"); args.add(indexer.getName()); args.add("--reducers"); args.add("0"); // additional arguments that were configured on the index (e.g. HBase scan parameters) args.addAll(Lists.newArrayList(batchIndexArguments)); return args.toArray(new String[args.size()]); }
private String[] createBatchArguments(IndexerDefinition indexer) { String[] batchIndexArguments = indexer.getBatchIndexCliArgumentsOrDefault(); List<String> args = Lists.newArrayList(); String mode = Optional.fromNullable(indexer.getConnectionParams().get(SolrConnectionParams.MODE)).or("cloud").toLowerCase(); if ("cloud".equals(mode)) { // cloud mode is the default args.add("--zk-host"); args.add(indexer.getConnectionParams().get(SolrConnectionParams.ZOOKEEPER)); } else { for (String shard : SolrConnectionParamUtil.getShards(indexer.getConnectionParams())) { args.add("--shard-url"); args.add(shard); } } args.add("--hbase-indexer-zk"); args.add(zkConnectString); args.add("--hbase-indexer-name"); args.add(indexer.getName()); args.add("--reducers"); args.add("0"); // additional arguments that were configured on the index (e.g. HBase scan parameters) args.addAll(Lists.newArrayList(batchIndexArguments)); return args.toArray(new String[args.size()]); }
@Override public Stat execute() throws KeeperException, InterruptedException { return zk.setData(indexerCollectionPathSlash + indexer.getName(), newData, indexer.getOccVersion()); } });
private IndexerDefinition getFreshIndexer(String indexerName, String path) throws Exception { // FixMe: should we have an individual GetIndexer command so we don't have to iterate? HttpGet httpGet = new HttpGet(path); byte [] response = HttpUtil.sendRequest(httpGet); List<IndexerDefinitionBuilder> builders = IndexerDefinitionsJsonSerDeser.INSTANCE.fromJsonBytes(response); for (IndexerDefinitionBuilder builder : builders) { IndexerDefinition def = builder.build(); if (indexerName.equals(def.getName())) { return def; } } return null; } }
@Override public void run(OptionSet options) throws Exception { super.run(options); IndexerDefinition indexer = null; try { IndexerDefinitionBuilder builder = buildIndexerDefinition(options, null); indexer = builder.build(); } catch (IllegalArgumentException e) { System.err.printf("Error adding indexer: %s\n", e.getMessage()); return; } model.addIndexer(indexer); System.out.println("Indexer added: " + indexer.getName()); } }
@Override public void activateAsLeader() throws Exception { log.info("Starting up as indexer master."); // Start these processes, but it is not until we have registered our model listener // that these will receive work. eventWorker.start(); Collection<IndexerDefinition> indexers = indexerModel.getIndexers(listener); // Rather than performing any work that might to be done for the indexers here, // we push out fake events. This way there's only one place where these actions // need to be performed. for (IndexerDefinition index : indexers) { eventWorker.putEvent(new IndexerModelEvent(INDEXER_UPDATED, index.getName())); } log.info("Startup as indexer master successful."); }
@Override public void activateAsLeader() throws Exception { log.info("Starting up as indexer master."); // Start these processes, but it is not until we have registered our model listener // that these will receive work. eventWorker.start(); Collection<IndexerDefinition> indexers = indexerModel.getIndexers(listener); // Rather than performing any work that might to be done for the indexers here, // we push out fake events. This way there's only one place where these actions // need to be performed. for (IndexerDefinition index : indexers) { eventWorker.putEvent(new IndexerModelEvent(INDEXER_UPDATED, index.getName())); } log.info("Startup as indexer master successful."); }
@PostConstruct public void start() throws LeaderElectionSetupException, IOException, InterruptedException, KeeperException { leaderElection = new LeaderElection(zk, "Indexer Master", hbaseConf.get(ConfKeys.ZK_ROOT_NODE) + "/masters", new MyLeaderElectionCallback()); for (IndexerDefinition indexerDefinition : indexerModel.getIndexers()) { executor.schedule(new BatchStateUpdater(indexerDefinition.getName(), indexerModel, getJobClient(), executor, batchStatePollInterval), batchStatePollInterval, TimeUnit.MILLISECONDS); } }
private void restartIndexer(IndexerDefinition indexerDef) { IndexerHandle handle = indexers.get(indexerDef.getName()); if (handle.indexerDef.getOccVersion() >= indexerDef.getOccVersion()) { return; } boolean relevantChanges = !Arrays.equals(handle.indexerDef.getConfiguration(), indexerDef.getConfiguration()) || Objects.equal(handle.indexerDef.getConnectionType(), indexerDef.getConnectionType()) || !Objects.equal(handle.indexerDef.getConnectionParams(), indexerDef.getConnectionParams()); if (!relevantChanges) { return; } if (stopIndexer(indexerDef.getName())) { startIndexer(indexerDef); } }
@Override public void updateIndexerInternal(final IndexerDefinition indexer) throws InterruptedException, KeeperException, IndexerNotFoundException, IndexerConcurrentModificationException, IndexerValidityException { assertValid(indexer); final byte[] newData = IndexerDefinitionJsonSerDeser.INSTANCE.toJsonBytes(indexer); try { zk.retryOperation(new ZooKeeperOperation<Stat>() { @Override public Stat execute() throws KeeperException, InterruptedException { return zk.setData(indexerCollectionPathSlash + indexer.getName(), newData, indexer.getOccVersion()); } }); } catch (KeeperException.NoNodeException e) { throw new IndexerNotFoundException(indexer.getName()); } catch (KeeperException.BadVersionException e) { throw new IndexerConcurrentModificationException(indexer.getName()); } }
@Override public void updateIndexerInternal(final IndexerDefinition indexer) throws InterruptedException, KeeperException, IndexerNotFoundException, IndexerConcurrentModificationException, IndexerValidityException { assertValid(indexer); final byte[] newData = IndexerDefinitionJsonSerDeser.INSTANCE.toJsonBytes(indexer); try { zk.retryOperation(new ZooKeeperOperation<Stat>() { @Override public Stat execute() throws KeeperException, InterruptedException { return zk.setData(indexerCollectionPathSlash + indexer.getName(), newData, indexer.getOccVersion()); } }); } catch (KeeperException.NoNodeException e) { throw new IndexerNotFoundException(indexer.getName()); } catch (KeeperException.BadVersionException e) { throw new IndexerConcurrentModificationException(indexer.getName()); } }
private void restartIndexer(IndexerDefinition indexerDef) { IndexerHandle handle = indexers.get(indexerDef.getName()); if (handle.indexerDef.getOccVersion() >= indexerDef.getOccVersion()) { return; } boolean relevantChanges = !Arrays.equals(handle.indexerDef.getConfiguration(), indexerDef.getConfiguration()) || Objects.equal(handle.indexerDef.getConnectionType(), indexerDef.getConnectionType()) || !Objects.equal(handle.indexerDef.getConnectionParams(), indexerDef.getConnectionParams()); if (!relevantChanges) { return; } if (stopIndexer(indexerDef.getName())) { startIndexer(indexerDef); } }
@Override public void run(OptionSet options) throws Exception { super.run(options); IndexerDefinition indexer = null; try { IndexerDefinitionBuilder builder = buildIndexerDefinition(options, null); indexer = builder.build(); } catch (IllegalArgumentException e) { System.err.printf("Error adding indexer: %s\n", e.getMessage()); return; } if (!options.has("http")) { model.addIndexer(indexer); System.out.println("Indexer added: " + indexer.getName()); } else { addIndexerHttp(options, indexer); } }
/** * see {@link CliCompatibleIndexResource#post} */ @POST @Consumes("application/json") @Produces("application/json") @Override public byte[] post(@Context SecurityContext securityContext, byte [] jsonBytes) throws IndexerServerException { if (authzBinding == null) { throwNullBindingException(); } IndexerDefinition def = getIndexerFromJson(jsonBytes); try { authzBinding.authorize(getSubject(securityContext), new Indexer(def.getName()), EnumSet.of(IndexerModelAction.WRITE)); } catch (SentryUserException e) { throw new IndexerServerException(HttpServletResponse.SC_UNAUTHORIZED, new SentryBindingException(e)); } return super.post(securityContext, jsonBytes); }
/** * Add an indexer definition */ @POST @Consumes("application/json") @Produces("application/json") public byte[] post(@Context SecurityContext securityContext, byte [] jsonBytes) throws IndexerServerException { WriteableIndexerModel model = getModel(); IndexerDefinitionBuilder builder = getBuilderFromJson(jsonBytes, null); IndexerDefinition indexer = builder.build(); try { model.addIndexer(indexer); } catch (IndexerExistsException iee) { throw new IndexerServerException(HttpServletResponse.SC_BAD_REQUEST, iee); } catch (IndexerValidityException ive) { throw new IndexerServerException(HttpServletResponse.SC_BAD_REQUEST, ive); } catch (IndexerModelException ime) { throw new IndexerServerException(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, ime); } return toJsonResultBytes("Indexer added: " + indexer.getName()); }
public IndexerDefinitionBuilder startFrom(IndexerDefinition existingDefinition) { this.name = existingDefinition.getName(); this.lifecycleState = existingDefinition.getLifecycleState(); this.batchIndexingState = existingDefinition.getBatchIndexingState(); this.incrementalIndexingState = existingDefinition.getIncrementalIndexingState(); this.subscriptionId = existingDefinition.getSubscriptionId(); this.configuration = existingDefinition.getConfiguration(); this.indexerComponentFactory = existingDefinition.getIndexerComponentFactory(); this.connectionType = existingDefinition.getConnectionType(); this.connectionParams = existingDefinition.getConnectionParams(); this.defaultBatchIndexCliArguments = existingDefinition.getDefaultBatchIndexCliArguments(); this.batchIndexCliArguments = existingDefinition.getBatchIndexCliArguments(); this.lastBatchBuildInfo = existingDefinition.getLastBatchBuildInfo(); this.activeBatchBuildInfo = existingDefinition.getActiveBatchBuildInfo(); this.subscriptionTimestamp = existingDefinition.getSubscriptionTimestamp(); this.occVersion = existingDefinition.getOccVersion(); return this; }
public IndexerDefinitionBuilder startFrom(IndexerDefinition existingDefinition) { this.name = existingDefinition.getName(); this.lifecycleState = existingDefinition.getLifecycleState(); this.batchIndexingState = existingDefinition.getBatchIndexingState(); this.incrementalIndexingState = existingDefinition.getIncrementalIndexingState(); this.subscriptionId = existingDefinition.getSubscriptionId(); this.configuration = existingDefinition.getConfiguration(); this.indexerComponentFactory = existingDefinition.getIndexerComponentFactory(); this.connectionType = existingDefinition.getConnectionType(); this.connectionParams = existingDefinition.getConnectionParams(); this.defaultBatchIndexCliArguments = existingDefinition.getDefaultBatchIndexCliArguments(); this.batchIndexCliArguments = existingDefinition.getBatchIndexCliArguments(); this.lastBatchBuildInfo = existingDefinition.getLastBatchBuildInfo(); this.activeBatchBuildInfo = existingDefinition.getActiveBatchBuildInfo(); this.subscriptionTimestamp = existingDefinition.getSubscriptionTimestamp(); this.occVersion = existingDefinition.getOccVersion(); return this; }