@Override protected void _terminate(TaskListener arg0) throws IOException, InterruptedException { //TODO: Check when this method is getting called and code accordingly LOGGER.log(Level.INFO, "AzureVMAgent: _terminate: called for agent {0}", getNodeName()); ProvisioningActivity activity = CloudStatistics.get().getActivityFor(this); if (activity != null) { activity.enterIfNotAlready(ProvisioningActivity.Phase.COMPLETED); } }
@Override public @Nonnull State dispose() throws Throwable { try { return inner.dispose(); } catch (Throwable ex) { CloudStatistics statistics = CloudStatistics.get(); ProvisioningActivity activity = statistics.getPotentiallyCompletedActivityFor(provisioningId); if (activity != null) { PhaseExecutionAttachment.ExceptionAttachment attachment = new PhaseExecutionAttachment.ExceptionAttachment( ProvisioningActivity.Status.WARN, ex ); statistics.attach(activity, ProvisioningActivity.Phase.COMPLETED, attachment); } throw ex; } }
@Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; RecordDisposal that = (RecordDisposal) o; if (!inner.equals(that.inner)) return false; return provisioningId.equals(that.provisioningId); }
@Restricted(NoExternalUse.class) public @Nonnull JCloudsSlave provisionSlave(JCloudsSlaveTemplate template) throws IOException, Openstack.ActionFailed{ CloudStatistics.ProvisioningListener provisioningListener = CloudStatistics.ProvisioningListener.get(); ProvisioningActivity.Id id = new ProvisioningActivity.Id(this.name, template.name); JCloudsSlave node; try { provisioningListener.onStarted(id); node = template.provisionSlave(this, id); provisioningListener.onComplete(id, node); } catch (Throwable ex) { provisioningListener.onFailure(id, ex); throw ex; } Jenkins.getActiveInstance().addNode(node); return node; }
public void cleanCloudStatistics() { Jenkins jenkins = Jenkins.getInstance(); Set<ProvisioningActivity.Id> plannedNodesSet = new HashSet<>(); for (NodeProvisioner.PlannedNode node : jenkins.unlabeledNodeProvisioner.getPendingLaunches()) { if (node instanceof TrackedItem) { plannedNodesSet.add(((TrackedItem) node).getId()); } } for (Label l : jenkins.getLabels()) { for (NodeProvisioner.PlannedNode node : l.nodeProvisioner.getPendingLaunches()) { if (node instanceof TrackedItem) { plannedNodesSet.add(((TrackedItem) node).getId()); } } } for (Node node : jenkins.getNodes()) { if (node instanceof TrackedItem) { plannedNodesSet.add(((TrackedItem) node).getId()); } } Collection<ProvisioningActivity> activities = CloudStatistics.get().getNotCompletedActivities(); for (ProvisioningActivity activity : activities) { if (activity.getCurrentPhase().equals(ProvisioningActivity.Phase.PROVISIONING) && !plannedNodesSet.contains(activity.getId())) { Exception e = new Exception(String.format("Node %s has lost. Mark as failure", activity.getId().toString())); CloudStatistics.ProvisioningListener.get().onFailure(activity.getId(), e); } } }
@Override protected void _terminate(TaskListener listener) { CloudStatistics cloudStatistics = CloudStatistics.get(); ProvisioningActivity activity = cloudStatistics.getActivityFor(this); if (activity != null) { activity.enterIfNotAlready(ProvisioningActivity.Phase.COMPLETED); // Attach what is likely a reason for the termination OfflineCause offlineCause = getFatalOfflineCause(); if (offlineCause != null) { PhaseExecutionAttachment attachment = new PhaseExecutionAttachment(ProvisioningActivity.Status.WARN, offlineCause.toString()); cloudStatistics.attach(activity, ProvisioningActivity.Phase.COMPLETED, attachment); } } // Wrap deletion disposables into statistics tracking disposables AsyncResourceDisposer.get().dispose( new RecordDisposal( new DestroyMachine(cloudName, nodeId), provisioningId ) ); }
final int index = i; final ProvisioningActivity.Id provisioningId = new ProvisioningActivity.Id(this.name, template.getTemplateName()); plannedNodes.add(new TrackedPlannedNode( provisioningId, template.getNoOfParallelJobs(),
private String getServerName() { CloudStatistics cs = CloudStatistics.get(); next_number: for (;;) { // Using static counter to ensure colliding template names (between clouds) will not cause a clash String nameCandidate = name + "-" + nodeCounter.getAndIncrement(); // Collide with existing node - quite likely from this cloud if (Jenkins.getInstance().getNode(nameCandidate) != null) continue; // Collide with node being provisioned (at this point this plugin does not assign final name before launch // is completed) or recently used name (just to avoid confusion). for (ProvisioningActivity provisioningActivity : cs.getActivities()) { if (nameCandidate.equals(provisioningActivity.getId().getNodeName())) { continue next_number; } } return nameCandidate; } }
@Override public boolean isOutOfScope(@Nonnull Server server) { if (Jenkins.getActiveInstance().getNode(specifier) != null) return false; // The node may be provisioned or deleted at the moment - do not interfere for (ProvisioningActivity pa : CloudStatistics.get().getActivities()) { if (specifier.equals(pa.getName())) { switch (pa.getCurrentPhase()) { case PROVISIONING: return false; // Node not yet created case LAUNCHING: case OPERATING: LOGGER.warning("Node does not exist for " + pa.getCurrentPhase() + " " + specifier); return false; case COMPLETED: return true; } assert false: "Unreachable"; } } Date created = server.getCreated(); if (created != null) { long now = System.currentTimeMillis(); long ageHours = (now - created.getTime()) / (1000 * 60 * 60); if (ageHours < 1) return false; // Make sure not to throw it away during launching } // Resolving activity by name is not reliable as they may not be assigned yet or the activity might already // be rotated when in completed state. In the latter case it should be treated as out of scope. return true; } }
@Override public void tearDown(Run<?, ?> run, FilePath workspace, Launcher launcher, TaskListener listener) throws IOException, InterruptedException { LOG.info("Shutting down slave"); listener.getLogger().println("Shutting down slave '" + slaveName + "'."); final Node slaveNode = Jenkins.getInstance().getNode(slaveName); if (isNull(slaveNode)) { throw new IllegalStateException("Can't get node " + slaveName); } final DockerSlaveSingle node = (DockerSlaveSingle) slaveNode; try { node.terminate(); } catch (Throwable e) { LOG.error("Can't terminate node", e); CloudStatistics.ProvisioningListener.get().onFailure(node.getId(), e); } } }
JCloudsSlave doProvisionFromTemplate(final JCloudsSlaveTemplate t) throws IOException { final StringWriter sw = new StringWriter(); final StreamTaskListener listener = new StreamTaskListener(sw); final ProvisioningActivity.Id provisioningId = new ProvisioningActivity.Id(this.name, t.name); JCloudsSlave node = t.provisionSlave(listener, provisioningId); Jenkins.getInstance().addNode(node); return node; }
/** * Return the number of active nodes provisioned using this template. * * @param onlyNewComputers Whether or not to limit the count to unused machines. */ public int getActiveNodesTotal(boolean onlyNewComputers) { int totalServers = 0; for (Node node : Jenkins.getActiveInstance().getNodes()) { if (node instanceof JCloudsSlave) { JCloudsSlave slave = (JCloudsSlave) node; if (slave.getId().getCloudName().equals(cloud.name) && slave.getId().getTemplateName().equals(name)) { JCloudsComputer computer = slave.getComputer(); if (computer != null && !computer.isPendingDelete() && !(computer.getOfflineCause() instanceof OfflineCause.UserCause)) { if (onlyNewComputers) { if (!computer.isUsed()) { totalServers++; } } else { totalServers++; } } } } } return totalServers; }
@Override public int hashCode() { return Objects.hashCode(inner.hashCode(), provisioningId.hashCode()); } }
final ProvisioningActivity.Id id = new ProvisioningActivity.Id(getDisplayName(), t.getDockerContainerLifecycle().getImage()); r.add(new TrackedPlannedNode( id, t.getNumExecutors(), Computer.threadPoolForRemoting.submit(() -> { get().onStarted(id); try { final DockerSlave dockerSlave = provisionWithWait(t, id); get().onComplete(id, dockerSlave); //rename return dockerSlave; } catch (Exception ex) { LOG.error("Error in provisioning; template='{}' for cloud='{}'", t, getDisplayName(), ex); get().onFailure(id, ex); throw Throwables.propagate(ex); } finally {
/** * Destroy the node. * If stopOnTerminate is {@code true}, calls {@link ComputeService#suspendNode}, * otherwise {@link ComputeService#destroyNode}. */ @Override protected void _terminate(TaskListener listener) throws IOException, InterruptedException { final ComputeService compute = JCloudsCloud.getByName(cloudName).getCompute(); if (compute.getNodeMetadata(nodeId) != null && compute.getNodeMetadata(nodeId).getStatus().equals(NodeMetadata.Status.RUNNING)) { if (stopOnTerminate) { LOGGER.info("Suspending slave : " + getNodeName()); compute.suspendNode(nodeId); } else { LOGGER.info("Terminating slave : " + getNodeName()); compute.destroyNode(nodeId); } } else { LOGGER.info("Slave " + getNodeName() + " is already not running."); } ProvisioningActivity activity = CloudStatistics.get().getActivityFor(this); if (activity != null) { activity.enterIfNotAlready(ProvisioningActivity.Phase.COMPLETED); } }
@Override public Collection<NodeProvisioner.PlannedNode> provision(Label label, int excessWorkload) { Queue<JCloudsSlaveTemplate> templateProvider = getAvailableTemplateProvider(label); List<PlannedNode> plannedNodeList = new ArrayList<>(); while (excessWorkload > 0 && !Jenkins.getActiveInstance().isQuietingDown() && !Jenkins.getActiveInstance().isTerminating()) { final JCloudsSlaveTemplate template = templateProvider.poll(); if (template == null) { LOGGER.info("Instance cap exceeded on all available templates"); break; } LOGGER.fine("Provisioning slave for " + label + " from template " + template.name); int numExecutors = template.getEffectiveSlaveOptions().getNumExecutors(); ProvisioningActivity.Id id = new ProvisioningActivity.Id(this.name, template.name); Future<Node> task = Computer.threadPoolForRemoting.submit(new NodeCallable(this, template, id)); plannedNodeList.add(new TrackedPlannedNode(id, numExecutors, task)); excessWorkload -= numExecutors; } return plannedNodeList; }
/** * Should a slave be retained to meet the minimum instances constraint? */ public static boolean shouldSlaveBeRetained(JCloudsSlave slave) { String templateName = slave.getId().getTemplateName(); String cloudName = slave.getId().getCloudName(); if (templateName != null && cloudName != null) { JCloudsCloud cloud = JCloudsCloud.getByName(cloudName); if (cloud != null) { JCloudsSlaveTemplate template = cloud.getTemplate(templateName); if (template != null) { SlaveOptions slaveOptions = template.getEffectiveSlaveOptions(); Integer instancesMin = slaveOptions.getInstancesMin(); JCloudsComputer computer = slave.getComputer(); Integer retentionTime = slaveOptions.getRetentionTime(); if (instancesMin > 0 && computer != null) { if (retentionTime != 0 && (template.getActiveNodesTotal(false) - 1) < instancesMin) { return true; } if (retentionTime == 0 && !computer.isUsed() && (template.getActiveNodesTotal(true) - 1) < instancesMin) { return true; } } } } } return false; }
ProvisioningActivity activity = CloudStatistics.get().getActivityFor(this); if (nonNull(activity)) { activity.enterIfNotAlready(ProvisioningActivity.Phase.COMPLETED);
final ProvisioningActivity.Id provisioningId = new ProvisioningActivity.Id(this.name, template.name); plannedNodeList.add(new TrackedPlannedNode(provisioningId, template.getNumExecutors(), Computer.threadPoolForRemoting.submit(new Callable<Node>() { public Node call() throws Exception {
ProvisioningActivity activity = CloudStatistics.get().getActivityFor(this); if (nonNull(activity)) { activity.enterIfNotAlready(ProvisioningActivity.Phase.COMPLETED);