/** * Build the {@link HelixManager} for the Service Master. */ private HelixManager buildHelixManager(Config config, String zkConnectionString) { String helixClusterName = config.getString(ServiceConfigKeys.HELIX_CLUSTER_NAME_KEY); String helixInstanceName = ConfigUtils.getString(config, ServiceConfigKeys.HELIX_INSTANCE_NAME_KEY, GobblinServiceManager.class.getSimpleName()); LOGGER.info("Creating Helix cluster if not already present [overwrite = false]: " + zkConnectionString); HelixUtils.createGobblinHelixCluster(zkConnectionString, helixClusterName, false); return HelixUtils.buildHelixManager(helixInstanceName, helixClusterName, zkConnectionString); }
/** {@inheritDoc} */ @Override public void onDeleteSpec(URI deletedSpecURI, String deletedSpecVersion) { if (this.helixManager.isPresent() && !this.helixManager.get().isConnected()) { // Specs in store will be notified when Scheduler is added as listener to FlowCatalog, so ignore // .. Specs if in cluster mode and Helix is not yet initialized _log.info("System not yet initialized. Skipping Spec Deletion: " + deletedSpecURI); return; } _log.info("Spec deletion detected: " + deletedSpecURI + "/" + deletedSpecVersion); if (!isActive && helixManager.isPresent()) { _log.info("Scheduler running in slave mode, forward Spec delete via Helix message to master: " + deletedSpecURI); HelixUtils.sendUserDefinedMessage(ServiceConfigKeys.HELIX_FLOWSPEC_REMOVE, deletedSpecURI.toString() + ":" + deletedSpecVersion, UUID.randomUUID().toString(), InstanceType.CONTROLLER, helixManager.get(), _log); return; } try { this.scheduledFlowSpecs.remove(deletedSpecURI.toString()); unscheduleJob(deletedSpecURI.toString()); } catch (JobException e) { _log.warn(String.format("Spec with URI: %s was not unscheduled cleaning", deletedSpecURI), e); } }
/** * Create a Helix cluster for the Gobblin Cluster application. * * @param zkConnectionString the ZooKeeper connection string * @param clusterName the Helix cluster name */ public static void createGobblinHelixCluster(String zkConnectionString, String clusterName) { createGobblinHelixCluster(zkConnectionString, clusterName, true); }
if (!isActive && helixManager.isPresent()) { _log.info("Scheduler running in slave mode, forward Spec add via Helix message to master: " + addedSpec); HelixUtils.sendUserDefinedMessage(ServiceConfigKeys.HELIX_FLOWSPEC_ADD, addedSpec.getUri().toString(), UUID.randomUUID().toString(), InstanceType.CONTROLLER, helixManager.get(), _log); return;
/** {@inheritDoc} */ @Override public void onUpdateSpec(Spec updatedSpec) { if (this.helixManager.isPresent() && !this.helixManager.get().isConnected()) { // Specs in store will be notified when Scheduler is added as listener to FlowCatalog, so ignore // .. Specs if in cluster mode and Helix is not yet initialized _log.info("System not yet initialized. Skipping Spec Update: " + updatedSpec); return; } _log.info("Spec changed: " + updatedSpec); if (!(updatedSpec instanceof FlowSpec)) { return; } if (!isActive && helixManager.isPresent()) { _log.info("Scheduler running in slave mode, forward Spec update via Helix message to master: " + updatedSpec); HelixUtils.sendUserDefinedMessage(ServiceConfigKeys.HELIX_FLOWSPEC_UPDATE, updatedSpec.getUri().toString(), UUID.randomUUID().toString(), InstanceType.CONTROLLER, helixManager.get(), _log); return; } try { onDeleteSpec(updatedSpec.getUri(), updatedSpec.getVersion()); } catch (Exception e) { _log.error("Failed to update Spec: " + updatedSpec, e); } try { onAddSpec(updatedSpec); } catch (Exception e) { _log.error("Failed to update Spec: " + updatedSpec, e); } }