public AzureCoordinationUtils(Config config) { azureConfig = new AzureConfig(config); this.client = new AzureClient(azureConfig.getAzureConnectionString()); }
public String getAzureConnectionString() { if (!containsKey(AZURE_STORAGE_CONNECT)) { throw new ConfigException("Missing " + AZURE_STORAGE_CONNECT + " config!"); } return get(AZURE_STORAGE_CONNECT); }
@Override public DistributedLockWithState getLockWithState(String lockId) { BlobUtils blob = new BlobUtils(client, azureConfig.getAzureContainerName(), azureConfig.getAzureBlobName() + lockId, azureConfig.getAzureBlobLength()); return new AzureLock(blob); }
/** * Creates an instance of Azure job coordinator, along with references to Azure leader elector, Azure Blob and Azure Table. * @param config User defined config */ public AzureJobCoordinator(String processorId, Config config, MetricsRegistry metricsRegistry) { //TODO: Cleanup previous values in the table when barrier times out. this.processorId = processorId; this.config = config; currentJMVersion = new AtomicReference<>(INITIAL_STATE); AzureConfig azureConfig = new AzureConfig(config); AzureClient client = new AzureClient(azureConfig.getAzureConnectionString()); leaderBlob = new BlobUtils(client, azureConfig.getAzureContainerName(), azureConfig.getAzureBlobName(), azureConfig.getAzureBlobLength()); errorHandler = (errorMsg) -> { LOG.error(errorMsg); stop(); }; table = new TableUtils(client, azureConfig.getAzureTableName(), INITIAL_STATE); azureLeaderElector = new AzureLeaderElector(new LeaseBlobManager(leaderBlob.getBlob())); azureLeaderElector.setLeaderElectorListener(new AzureLeaderElectorListener()); versionUpgradeDetected = new AtomicBoolean(false); heartbeat = new HeartbeatScheduler(errorHandler, table, currentJMVersion, processorId); versionUpgrade = new JMVersionUpgradeScheduler(errorHandler, leaderBlob, currentJMVersion, versionUpgradeDetected, processorId); leaderAlive = new LeaderLivenessCheckScheduler(errorHandler, table, leaderBlob, currentJMVersion, INITIAL_STATE); leaderBarrierScheduler = null; renewLease = null; liveness = null; }
public long getAzureBlobLength() { return getLong(AZURE_PAGEBLOB_LENGTH, DEFAULT_AZURE_PAGEBLOB_LENGTH); }
@Override public CheckpointManager getCheckpointManager(Config config, MetricsRegistry registry) { JobConfig jobConfig = new JobConfig(config); return new AzureCheckpointManager(new AzureConfig(config), jobConfig.getName()); } }
AzureCheckpointManager(AzureConfig azureConfig, Option<String> jobName) { if (!jobName.isDefined()) { throw new AzureException("Jobs must have a name to use Azure Checkpoint Manager"); } // Remove invalid characters jobTableName = jobName.get().replaceAll(REGEX_TABLE_NAME, ""); storageConnectionString = azureConfig.getAzureConnectionString(); azureClient = new AzureClient(storageConnectionString); }