public boolean onStart(CallbackData data, CallbackContext context) { ClusterAction jobAction = data.getJob().getClusterAction(); if (startTriggerActions.contains(jobAction)) { if (onStartUrl != null) { LOG.debug("sending request to {} before performing {} on cluster {}", onStartUrl, jobAction, data.getCluster().getId()); sendPost(onStartUrl, data, context); } } return true; }
@Override public void run() { CallbackData callbackData = gson.fromJson(gElement.getElement().getValue(), CallbackData.class); CallbackContext callbackContext = new CallbackContext(clusterStoreService, userStore, callbackData.getCluster().getAccount()); switch (callbackData.getType()) { case START: onStart(callbackData, callbackContext); break; case SUCCESS: clusterCallback.onSuccess(callbackData, callbackContext); break; case FAILURE: clusterCallback.onFailure(callbackData, callbackContext); break; default: LOG.error("Unknown callback type {}", callbackData.getType()); } }
private void sendPost(String url, CallbackData data, CallbackContext context) { HttpPost post = new HttpPost(url); Set<Node> nodes; try { nodes = context.getClusterStoreView().getClusterNodes(data.getCluster().getId()); } catch (Exception e) { LOG.error("Unable to fetch nodes for cluster {}, not sending post request.", data.getCluster().getId()); return; } try { JsonObject body = new JsonObject(); body.add("cluster", gson.toJsonTree(data.getCluster())); body.add("job", gson.toJsonTree(data.getJob())); body.add("nodes", gson.toJsonTree(nodes)); post.setEntity(new StringEntity(gson.toJson(body))); httpClient.execute(post); } catch (UnsupportedEncodingException e) { LOG.warn("Exception setting http post body", e); } catch (ClientProtocolException e) { LOG.warn("Exception executing http post callback to " + url, e); } catch (IOException e) { LOG.warn("Exception executing http post callback to " + url, e); } catch (Exception e) { LOG.warn("Exception executing http post callback to " + url, e); } finally { post.releaseConnection(); } } }
@Test public void testCalls() { HttpPostClusterCallback callback = new HttpPostClusterCallback(); String base = "http://" + host + ":" + port; conf = Configuration.create(); conf.set(Constants.HttpCallback.START_URL, base + "/start/endpoint"); conf.set(Constants.HttpCallback.SUCCESS_URL, base + "/success/endpoint"); conf.set(Constants.HttpCallback.FAILURE_URL, base + "/failure/endpoint"); callback.initialize(conf); ClusterJob job = new ClusterJob(new JobId(cluster.getId(), 1), ClusterAction.CLUSTER_CREATE); CallbackData data = new CallbackData(CallbackData.Type.START, cluster, job); CallbackContext context = new CallbackContext(clusterStoreService, userStore, cluster.getAccount()); callback.onStart(data, context); callback.onSuccess(data, context); callback.onSuccess(data, context); callback.onFailure(data, context); Assert.assertEquals(handler.getStartCount(), 1); Assert.assertEquals(handler.getFailureCount(), 1); Assert.assertEquals(handler.getSuccessCount(), 2); }
@Test public void testOnStartIsTrueWithBadURL() { HttpPostClusterCallback callback = new HttpPostClusterCallback(); conf = Configuration.create(); conf.set(Constants.HttpCallback.START_URL, "malformed-url"); callback.initialize(conf); ClusterJob job = new ClusterJob(new JobId(cluster.getId(), 1), ClusterAction.CLUSTER_CREATE); CallbackData data = new CallbackData(CallbackData.Type.START, cluster, job); CallbackContext context = new CallbackContext(clusterStoreService, userStore, cluster.getAccount()); Assert.assertTrue(callback.onStart(data, context)); }
@BeforeClass public static void setupTestClass() throws Exception { handler = new DummyHandler(); service = new DummyService(0, handler); service.startAndWait(); port = service.getBindAddress().getPort(); host = service.getBindAddress().getHostName(); clusterStoreService.getView(cluster.getAccount()).writeCluster(cluster); clusterStoreService.getSystemView().writeNode(Entities.ClusterExample.NODE1); clusterStoreService.getSystemView().writeNode(Entities.ClusterExample.NODE2); }
@Test(timeout = 20000) public void testFalseOnStartStopsJob() throws Exception { String tenantId = "q"; ClusterScheduler clusterScheduler = injector.getInstance(ClusterScheduler.class); clusterQueues.add(tenantId, new Element(cluster.getId(), ClusterAction.CLUSTER_CREATE.name())); clusterScheduler.run(); CallbackScheduler callbackScheduler = injector.getInstance(CallbackScheduler.class); // should be no job in the queue until the start callback runs Assert.assertEquals(0, jobQueues.size(tenantId)); // tell mock callback to return false for onStart callback mockClusterCallback.setReturnOnStart(false); // wait for start callback to finish waitForCallback(callbackScheduler); Assert.assertEquals(CallbackData.Type.START, mockClusterCallback.getReceivedCallbacks().get(0).getType()); // wait for fail callback to finish if (mockClusterCallback.getReceivedCallbacks().size() < 2) { waitForCallback(callbackScheduler); } Assert.assertEquals(CallbackData.Type.FAILURE, mockClusterCallback.getReceivedCallbacks().get(1).getType()); // there also should not be any jobs in the queue Assert.assertEquals(0, jobQueues.size(tenantId)); }
private void onStart(CallbackData callbackData, CallbackContext callbackContext) { ClusterJob job = callbackData.getJob(); Cluster cluster = callbackData.getCluster(); try { if (clusterCallback.onStart(callbackData, callbackContext)) { String jobId = callbackData.getJob().getJobId(); jobQueues.add(gElement.getQueueName(), new Element(jobId)); LOG.debug("added job {} to job queue", jobId); } else { switch (job.getClusterAction()) { case CLUSTER_CREATE: taskService.failJobAndTerminateCluster(job, cluster, "Cluster creation stopped by failed start callback."); break; default: // failed to plan means the job should fail, but state has already been changed so the cluster // state in the db is inconsistent with reality. // TODO: Should revert it here but need versioning or cluster history or something to that effect. taskService.failJobAndSetClusterStatus( job, cluster, Cluster.Status.INCONSISTENT, "Failed to schedule the " + job.getClusterAction() + " operation."); break; } } } catch (Exception e) { LOG.error("Exception failing job {} for cluster {}", job.getJobId(), cluster.getId(), e); } } }
private void testCallbacks(boolean failJob) throws Exception { ClusterScheduler clusterScheduler = injector.getInstance(ClusterScheduler.class); String tenantId = cluster.getAccount().getTenantId(); clusterQueues.add(tenantId, new Element(cluster.getId(), ClusterAction.CLUSTER_CREATE.name())); clusterScheduler.run(); CallbackScheduler callbackScheduler = injector.getInstance(CallbackScheduler.class); // should be no job in the queue until the start callback runs Assert.assertEquals(0, jobQueues.size(tenantId)); waitForCallback(callbackScheduler); Assert.assertEquals(CallbackData.Type.START, mockClusterCallback.getReceivedCallbacks().get(0).getType()); JobScheduler jobScheduler = injector.getInstance(JobScheduler.class); jobScheduler.run(); // take tasks until there are no more TakeTaskRequest takeRequest = new TakeTaskRequest("consumer1", PROVISIONER_ID, tenantId); SchedulableTask task = TestHelper.takeTask(getInternalServerUrl(), takeRequest); while (task != null) { FinishTaskRequest finishRequest = new FinishTaskRequest("consumer1", PROVISIONER_ID, tenantId, task.getTaskId(), null, null, failJob ? 1 : 0, null, null, null); TestHelper.finishTask(getInternalServerUrl(), finishRequest); jobScheduler.run(); jobScheduler.run(); task = TestHelper.takeTask(getInternalServerUrl(), takeRequest); } jobScheduler.run(); waitForCallback(callbackScheduler); // at this point, the failure callback should have run Assert.assertEquals(failJob ? CallbackData.Type.FAILURE : CallbackData.Type.SUCCESS, mockClusterCallback.getReceivedCallbacks().get(1).getType()); }
/** * Sets the status of the given job to {@link ClusterJob.Status#RUNNING} and add it to the queue to be run. * * @param job Job to start. * @param cluster Cluster the job is for. * @throws IOException */ public void startJob(ClusterJob job, Cluster cluster) throws IOException { // TODO: wrap in a transaction LOG.debug("Starting job {} for cluster {}", job.getJobId(), cluster.getId()); job.setJobStatus(ClusterJob.Status.RUNNING); // Note: writing job status as RUNNING, will allow other operations on the job // (like cancel, etc.) to happen in parallel. clusterStore.writeClusterJob(job); callbackQueues.add(cluster.getAccount().getTenantId(), new Element(gson.toJson(new CallbackData(CallbackData.Type.START, cluster, job)))); }
@AfterClass public static void cleanupTestClass() { service.stopAndWait(); } }
@Inject private CallbackScheduler(@Named("scheduler.id") String id, @Named("callback.executor.service") ListeningExecutorService executorService, TaskService taskService, ClusterCallback clusterCallback, Configuration conf, ClusterStoreService clusterStoreService, UserStore userStore, Gson gson, QueueService queueService) { this.id = id; this.executorService = executorService; this.taskService = taskService; this.clusterCallback = clusterCallback; this.clusterCallback.initialize(conf); this.gson = gson; this.callbackQueues = queueService.getQueueGroup(QueueType.CALLBACK); this.jobQueues = queueService.getQueueGroup(QueueType.JOB); this.clusterStoreService = clusterStoreService; this.userStore = userStore; }
@Before public void setupTest() { handler.clear(); }
private void waitForCallback(CallbackScheduler callbackScheduler) throws InterruptedException { int initialSize = mockClusterCallback.getReceivedCallbacks().size(); int size = initialSize; callbackScheduler.run(); while (size == initialSize) { size = mockClusterCallback.getReceivedCallbacks().size(); TimeUnit.MILLISECONDS.sleep(20); } }
this.onSuccessUrl = conf.get(Constants.HttpCallback.SUCCESS_URL); this.onFailureUrl = conf.get(Constants.HttpCallback.FAILURE_URL); this.startTriggerActions = parseActionsString(conf.get(Constants.HttpCallback.START_TRIGGERS, Constants.HttpCallback.DEFAULT_START_TRIGGERS)); this.successTriggerActions = parseActionsString(conf.get(Constants.HttpCallback.SUCCESS_TRIGGERS, Constants.HttpCallback.DEFAULT_SUCCESS_TRIGGERS)); this.failureTriggerActions = parseActionsString(conf.get(Constants.HttpCallback.FAILURE_TRIGGERS, Constants.HttpCallback.DEFAULT_FAILURE_TRIGGERS)); if (onStartUrl != null) {
@After public void cleanupTest() throws Exception { jobQueues.removeAll(); clusterQueues.removeAll(); solverQueues.removeAll(); provisionerQueues.removeAll(); callbackQueues.removeAll(); mockClusterCallback.clear(); }
@Test public void testTriggers() { HttpPostClusterCallback callback = new HttpPostClusterCallback(); String base = "http://" + host + ":" + port; conf = Configuration.create(); conf.set(Constants.HttpCallback.START_URL, base + "/start/endpoint"); conf.set(Constants.HttpCallback.START_TRIGGERS, ClusterAction.CLUSTER_CONFIGURE.name()); callback.initialize(conf); // should not get triggered ClusterJob job = new ClusterJob(new JobId(cluster.getId(), 1), ClusterAction.CLUSTER_CREATE); CallbackData data = new CallbackData(CallbackData.Type.START, cluster, job); CallbackContext context = new CallbackContext(clusterStoreService, userStore, cluster.getAccount()); callback.onStart(data, context); Assert.assertEquals(0, handler.getStartCount()); // should get triggered job = new ClusterJob(new JobId(cluster.getId(), 1), ClusterAction.CLUSTER_CONFIGURE); data = new CallbackData(CallbackData.Type.START, cluster, job); callback.onStart(data, context); Assert.assertEquals(1, handler.getStartCount()); }
public void onSuccess(CallbackData data, CallbackContext context) { ClusterAction jobAction = data.getJob().getClusterAction(); if (successTriggerActions.contains(data.getJob().getClusterAction())) { if (onSuccessUrl != null) { LOG.debug("{} completed successfully on cluster {}, sending request to {}", jobAction, data.getCluster().getId(), onSuccessUrl); sendPost(onSuccessUrl, data, context); } } }
/** * Sets the status of the given job to {@link ClusterJob.Status#FAILED} and the status of the cluster to some given * status. * * @param job Job to fail. * @param cluster Cluster to set the status for. * @param status Status to set the cluster to. * @param message Error message. * @throws IOException * @throws IllegalAccessException */ public void failJobAndSetClusterStatus(ClusterJob job, Cluster cluster, Cluster.Status status, String message) throws IOException, IllegalAccessException { cluster.setStatus(status); clusterStore.writeCluster(cluster); job.setJobStatus(ClusterJob.Status.FAILED); if (message != null) { job.setStatusMessage(message); } clusterStore.writeClusterJob(job); serverStats.getFailedClusterStats().incrementStat(job.getClusterAction()); callbackQueues.add(cluster.getAccount().getTenantId(), new Element(gson.toJson(new CallbackData(CallbackData.Type.FAILURE, cluster, job)))); }
@Override public void onFailure(CallbackData data, CallbackContext context) { ClusterAction jobAction = data.getJob().getClusterAction(); if (failureTriggerActions.contains(data.getJob().getClusterAction())) { if (onFailureUrl != null) { LOG.debug("{} failed on cluster {}, sending request to {}", jobAction, data.getCluster().getId(), onFailureUrl); sendPost(onFailureUrl, data, context); } } }