@Inject public WorkerCuratorCoordinator( ObjectMapper jsonMapper, IndexerZkConfig indexerZkConfig, RemoteTaskRunnerConfig config, CuratorFramework curatorFramework, Worker worker ) { this.jsonMapper = jsonMapper; this.config = config; this.curatorFramework = curatorFramework; this.worker = worker; this.announcer = new Announcer(curatorFramework, Execs.directExecutor()); this.baseAnnouncementsPath = getPath(Arrays.asList(indexerZkConfig.getAnnouncementsPath(), worker.getHost())); this.baseTaskPath = getPath(Arrays.asList(indexerZkConfig.getTasksPath(), worker.getHost())); this.baseStatusPath = getPath(Arrays.asList(indexerZkConfig.getStatusPath(), worker.getHost())); }
private String defaultIndexerPath(final String subPath) { return ZKPaths.makePath(getBase(), subPath); }
public String getAnnouncementsPath() { return announcementsPath == null ? defaultIndexerPath("announcements") : announcementsPath; }
workers = cf.getChildren().forPath(indexerZkConfig.getStatusPath()); final String workerAnnouncePath = JOINER.join(indexerZkConfig.getAnnouncementsPath(), workerId); final String workerStatusPath = JOINER.join(indexerZkConfig.getStatusPath(), workerId); if (!zkWorkers.containsKey(workerId) && cf.checkExists().forPath(workerAnnouncePath) == null) { try {
@Test public void testExactConfig() { final Injector injector = Initialization.makeInjectorWithModules( GuiceInjectors.makeStartupInjector(), ImmutableList.of(simpleZkConfigModule) ); propertyValues.setProperty(zkServiceConfigString + ".base", "/druid/metrics"); JsonConfigurator configurator = injector.getBinding(JsonConfigurator.class).getProvider().get(); JsonConfigProvider<ZkPathsConfig> zkPathsConfig = JsonConfigProvider.of( zkServiceConfigString, ZkPathsConfig.class ); zkPathsConfig.inject(propertyValues, configurator); ZkPathsConfig zkPathsConfig1 = zkPathsConfig.get().get(); IndexerZkConfig indexerZkConfig = new IndexerZkConfig(zkPathsConfig1, null, null, null, null); Assert.assertEquals("/druid/metrics/indexer", indexerZkConfig.getBase()); Assert.assertEquals("/druid/metrics/indexer/announcements", indexerZkConfig.getAnnouncementsPath()); }
@Test public void testIndexerBaseOverride() { final String overrideValue = "/foo/bar/baz"; final String indexerPropertyKey = indexerPropertyString + ".base"; final String priorValue = System.getProperty(indexerPropertyKey); System.setProperty(indexerPropertyKey, overrideValue); // Set it here so that the binding picks it up final Injector injector = Initialization.makeInjectorWithModules( GuiceInjectors.makeStartupInjector(), ImmutableList.of(simpleZkConfigModule) ); propertyValues.clear(); propertyValues.setProperty(indexerPropertyKey, overrideValue); // Have to set it here as well annoyingly enough JsonConfigurator configurator = injector.getBinding(JsonConfigurator.class).getProvider().get(); JsonConfigProvider<IndexerZkConfig> indexerPathsConfig = JsonConfigProvider.of( indexerPropertyString, IndexerZkConfig.class ); indexerPathsConfig.inject(propertyValues, configurator); IndexerZkConfig indexerZkConfig = indexerPathsConfig.get().get(); // Rewind value before we potentially fail if (priorValue == null) { System.clearProperty(indexerPropertyKey); } else { System.setProperty(indexerPropertyKey, priorValue); } Assert.assertEquals(overrideValue, indexerZkConfig.getBase()); Assert.assertEquals(overrideValue + "/announcements", indexerZkConfig.getAnnouncementsPath()); }
protected List<String> getAssignedTasks(Worker worker) throws Exception { final List<String> assignedTasks = Lists.newArrayList( cf.getChildren().forPath(JOINER.join(indexerZkConfig.getTasksPath(), worker.getHost())) ); for (Map.Entry<String, RemoteTaskRunnerWorkItem> entry : runningTasks.entrySet()) { if (entry.getValue() == null) { log.error( "Huh? null work item for [%s]", entry.getKey() ); } else if (entry.getValue().getWorker() == null) { log.error("Huh? no worker for [%s]", entry.getKey()); } else if (entry.getValue().getWorker().getHost().equalsIgnoreCase(worker.getHost())) { log.info("[%s]: Found [%s] running", worker.getHost(), entry.getKey()); assignedTasks.add(entry.getKey()); } } log.info("[%s]: Found %d tasks assigned", worker.getHost(), assignedTasks.size()); return assignedTasks; }
this.indexerZkConfig = indexerZkConfig; this.cf = cf; this.workerPathCache = pathChildrenCacheFactory.build().make(cf, indexerZkConfig.getAnnouncementsPath()); this.workerStatusPathChildrenCacheExecutor = PathChildrenCacheFactory.Builder.createDefaultExecutor(); this.workerStatusPathChildrenCacheFactory = pathChildrenCacheFactory
List<String> workers; try { workers = cf.getChildren().forPath(indexerZkConfig.getStatusPath()); String workerStatusPath = JOINER.join(indexerZkConfig.getStatusPath(), workerId);
@Test public void testFullOverride() throws Exception { final DefaultObjectMapper mapper = new DefaultObjectMapper(); final ZkPathsConfig zkPathsConfig = new ZkPathsConfig(); IndexerZkConfig indexerZkConfig = new IndexerZkConfig( zkPathsConfig, "/druid/prod", "/druid/prod/a", "/druid/prod/t", "/druid/prod/s" ); Map<String, String> value = mapper.readValue( mapper.writeValueAsString(indexerZkConfig), JacksonUtils.TYPE_REFERENCE_MAP_STRING_STRING ); IndexerZkConfig newConfig = new IndexerZkConfig( zkPathsConfig, value.get("base"), value.get("announcementsPath"), value.get("tasksPath"), value.get("statusPath") ); Assert.assertEquals(indexerZkConfig, newConfig); } }
public String getBase() { return base == null ? getZkPathsConfig().defaultPath("indexer") : base; }
@Test public void testNullConfig() { propertyValues.clear(); final Injector injector = Initialization.makeInjectorWithModules( GuiceInjectors.makeStartupInjector(), ImmutableList.of(simpleZkConfigModule) ); JsonConfigurator configurator = injector.getBinding(JsonConfigurator.class).getProvider().get(); JsonConfigProvider<ZkPathsConfig> zkPathsConfig = JsonConfigProvider.of(zkServiceConfigString, ZkPathsConfig.class); zkPathsConfig.inject(propertyValues, configurator); JsonConfigProvider<IndexerZkConfig> indexerZkConfig = JsonConfigProvider.of( indexerPropertyString, IndexerZkConfig.class ); indexerZkConfig.inject(propertyValues, configurator); Assert.assertEquals("/druid/indexer/tasks", indexerZkConfig.get().get().getTasksPath()); }
workers = cf.getChildren().forPath(indexerZkConfig.getStatusPath()); final String workerAnnouncePath = JOINER.join(indexerZkConfig.getAnnouncementsPath(), workerId); final String workerStatusPath = JOINER.join(indexerZkConfig.getStatusPath(), workerId); if (!zkWorkers.containsKey(workerId) && cf.checkExists().forPath(workerAnnouncePath) == null) { try {
this.indexerZkConfig = indexerZkConfig; this.cf = cf; this.workerPathCache = pathChildrenCacheFactory.build().make(cf, indexerZkConfig.getAnnouncementsPath()); this.workerStatusPathChildrenCacheExecutor = PathChildrenCacheFactory.Builder.createDefaultExecutor(); this.workerStatusPathChildrenCacheFactory = pathChildrenCacheFactory
final String workerId = worker.getHost(); log.info("Cleaning up task[%s] on worker[%s]", taskId, workerId); final String statusPath = JOINER.join(indexerZkConfig.getStatusPath(), workerId, taskId); try { cf.delete().guaranteed().forPath(statusPath);
public RemoteTaskRunner makeRemoteTaskRunner( RemoteTaskRunnerConfig config, ProvisioningStrategy<WorkerTaskRunner> provisioningStrategy ) { RemoteTaskRunner remoteTaskRunner = new TestableRemoteTaskRunner( jsonMapper, config, new IndexerZkConfig( new ZkPathsConfig() { @Override public String getBase() { return basePath; } }, null, null, null, null ), cf, new PathChildrenCacheFactory.Builder(), null, DSuppliers.of(new AtomicReference<>(DefaultWorkerBehaviorConfig.defaultConfig())), provisioningStrategy ); remoteTaskRunner.start(); return remoteTaskRunner; }
public String getBase() { return base == null ? getZkPathsConfig().defaultPath("indexer") : base; }
@Inject public WorkerCuratorCoordinator( ObjectMapper jsonMapper, IndexerZkConfig indexerZkConfig, RemoteTaskRunnerConfig config, CuratorFramework curatorFramework, Worker worker ) { this.jsonMapper = jsonMapper; this.config = config; this.curatorFramework = curatorFramework; this.worker = worker; this.announcer = new Announcer(curatorFramework, MoreExecutors.sameThreadExecutor()); this.baseAnnouncementsPath = getPath(Arrays.asList(indexerZkConfig.getAnnouncementsPath(), worker.getHost())); this.baseTaskPath = getPath(Arrays.asList(indexerZkConfig.getTasksPath(), worker.getHost())); this.baseStatusPath = getPath(Arrays.asList(indexerZkConfig.getStatusPath(), worker.getHost())); }
JOINER.join(indexerZkConfig.getTasksPath(), worker, task.getId()), CreateMode.EPHEMERAL, jsonMapper.writeValueAsBytes(task),
private String defaultIndexerPath(final String subPath) { return ZKPaths.makePath(getBase(), subPath); }