/** Returns true if this filter matches the given host properties */ public boolean matches(String hostname, String flavor, Optional<ClusterMembership> membership) { if ( ! hostnames.isEmpty() && ! hostnames.contains(hostname)) return false; if ( ! flavors.isEmpty() && ! flavors.contains(flavor)) return false; if ( ! clusterTypes.isEmpty() && ! (membership.isPresent() && clusterTypes.contains(membership.get().cluster().type()))) return false; if ( ! clusterIds.isEmpty() && ! (membership.isPresent() && clusterIds.contains(membership.get().cluster().id()))) return false; return true; }
private int totalAllocatedTo(ClusterSpec cluster) { int count = 0; for (Map.Entry<ClusterSpec, List<HostSpec>> allocation : allocations.entrySet()) { if ( ! allocation.getKey().type().equals(cluster.type())) continue; if ( ! allocation.getKey().id().equals(cluster.id())) continue; count += allocation.getValue().size(); } return count; }
private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, String flavor, int nodesInGroup, int startIndex, boolean canFail) { List<HostSpec> allocation = allocations.getOrDefault(clusterGroup, new ArrayList<>()); allocations.put(clusterGroup, allocation); int nextIndex = nextIndexInCluster.getOrDefault(new Pair<>(clusterGroup.type(), clusterGroup.id()), startIndex); while (allocation.size() < nodesInGroup) { if (freeNodes.get(flavor).isEmpty()) { if (canFail) throw new IllegalArgumentException("Insufficient capacity of flavor '" + flavor + "'"); else break; } Host newHost = freeNodes.removeValue(flavor, 0); ClusterMembership membership = ClusterMembership.from(clusterGroup, nextIndex++); allocation.add(new HostSpec(newHost.hostname(), newHost.aliases(), newHost.flavor(), Optional.of(membership), newHost.version())); } nextIndexInCluster.put(new Pair<>(clusterGroup.type(), clusterGroup.id()), nextIndex); while (allocation.size() > nodesInGroup) allocation.remove(0); return allocation; }
/** * Provision load balancer(s) for given application. * * If the application has multiple container clusters, one load balancer will be provisioned for each cluster. */ public Map<LoadBalancerId, LoadBalancer> provision(ApplicationId application) { try (Mutex applicationLock = nodeRepository.lock(application)) { try (Mutex loadBalancersLock = db.lockLoadBalancers()) { Map<LoadBalancerId, LoadBalancer> loadBalancers = new LinkedHashMap<>(); for (Map.Entry<ClusterSpec, List<Node>> kv : activeContainers(application).entrySet()) { LoadBalancer loadBalancer = create(application, kv.getKey().id(), kv.getValue()) .with(kv.getKey().rotations()); loadBalancers.put(loadBalancer.id(), loadBalancer); db.writeLoadBalancer(loadBalancer); } return Collections.unmodifiableMap(loadBalancers); } } }
/** * Returns a list of the nodes which are * in groups with index number above or equal the group count */ private List<Node> findNodesInRemovableGroups(ApplicationId application, ClusterSpec requestedCluster, int wantedGroups) { List<Node> surplusNodes = new ArrayList<>(0); for (Node node : nodeRepository.getNodes(application, Node.State.active)) { ClusterSpec nodeCluster = node.allocation().get().membership().cluster(); if ( ! nodeCluster.id().equals(requestedCluster.id())) continue; if ( ! nodeCluster.type().equals(requestedCluster.type())) continue; if (nodeCluster.group().get().index() >= wantedGroups) surplusNodes.add(node); } return surplusNodes; }
NodePrioritizer(List<Node> allNodes, ApplicationId appId, ClusterSpec clusterSpec, NodeSpec nodeSpec, int spares, NameResolver nameResolver) { this.allNodes = Collections.unmodifiableList(allNodes); this.requestedNodes = nodeSpec; this.clusterSpec = clusterSpec; this.appId = appId; this.nameResolver = nameResolver; this.spareHosts = findSpareHosts(allNodes, spares); this.capacity = new DockerHostCapacity(allNodes); long nofFailedNodes = allNodes.stream() .filter(node -> node.state().equals(Node.State.failed)) .filter(node -> node.allocation().isPresent()) .filter(node -> node.allocation().get().owner().equals(appId)) .filter(node -> node.allocation().get().membership().cluster().id().equals(clusterSpec.id())) .count(); long nofNodesInCluster = allNodes.stream() .filter(node -> node.allocation().isPresent()) .filter(node -> node.allocation().get().owner().equals(appId)) .filter(node -> node.allocation().get().membership().cluster().id().equals(clusterSpec.id())) .count(); this.isAllocatingForReplacement = isReplacement(nofNodesInCluster, nofFailedNodes); this.isDocker = isDocker(); }
/** * Returns the highest index number of all active and failed nodes in this cluster, or -1 if there are no nodes. * We include failed nodes to avoid reusing the index of the failed node in the case where the failed node is the * node with the highest index. */ private int findHighestIndex(ApplicationId application, ClusterSpec cluster) { int highestIndex = -1; for (Node node : nodeRepository.getNodes(application, Node.State.active, Node.State.inactive, Node.State.parked, Node.State.failed)) { ClusterSpec nodeCluster = node.allocation().get().membership().cluster(); if ( ! nodeCluster.id().equals(cluster.id())) continue; if ( ! nodeCluster.type().equals(cluster.type())) continue; highestIndex = Math.max(node.allocation().get().membership().index(), highestIndex); } return highestIndex; }
protected String toStringValue() { return cluster.type().name() + "/" + cluster.id().value() + (cluster.group().isPresent() ? "/" + cluster.group().get().index() : "") + "/" + index + ( cluster.isExclusive() ? "/exclusive" : "") + ( retired ? "/retired" : "") + ( !cluster.rotations().isEmpty() ? "/" + rotationsAsString(cluster.rotations()) : ""); }
Map<ClusterSpec.Id, Set<Node>> nodesByCluster = getNodesBelongingToApplication(allNodes, applicationId).stream() .collect(Collectors.groupingBy( node -> node.allocation().get().membership().cluster().id(), Collectors.toSet())); Map<ClusterSpec.Id, Set<Node>> retireableNodesByCluster = nodesByCluster.entrySet().stream()
private String getHostFromVespaCertificate(List<SubjectAlternativeName> sans) { // TODO Remove this branch once all BM nodes are gone if (sans.stream().anyMatch(san -> san.getValue().endsWith("ostk.yahoo.cloud"))) { return getHostFromCalypsoCertificate(sans); } VespaUniqueInstanceId instanceId = VespaUniqueInstanceId.fromDottedString(getUniqueInstanceId(sans)); if (!zone.environment().value().equals(instanceId.environment())) throw new NodeIdentifierException("Invalid environment: " + instanceId.environment()); if (!zone.region().value().equals(instanceId.region())) throw new NodeIdentifierException("Invalid region(): " + instanceId.region()); List<Node> applicationNodes = nodeRepository.getNodes(ApplicationId.from(instanceId.tenant(), instanceId.application(), instanceId.instance())); return applicationNodes.stream() .filter( node -> node.allocation() .map(allocation -> allocation.membership().index() == instanceId.clusterIndex() && allocation.membership().cluster().id().value().equals(instanceId.clusterId())) .orElse(false)) .map(Node::hostname) .findFirst() .orElseThrow(() -> new NodeIdentifierException("Could not find any node with instance id: " + instanceId.asDottedString())); }
@Override public Map<ApplicationInstanceReference, ApplicationInstance> getAllApplicationInstances() { // Convert apps information to the response payload to return Map<ApplicationInstanceReference, ApplicationInstance> status = new HashMap<>(); for (Map.Entry<ApplicationId, MockDeployer.ApplicationContext> app : apps.entrySet()) { Set<ServiceInstance> serviceInstances = new HashSet<>(); for (Node node : nodeRepository.getNodes(app.getValue().id(), Node.State.active)) { serviceInstances.add(new ServiceInstance(new ConfigId("configid"), new HostName(node.hostname()), getHostStatus(node.hostname()))); } Set<ServiceCluster> serviceClusters = new HashSet<>(); serviceClusters.add(new ServiceCluster(new ClusterId(app.getValue().clusterContexts().get(0).cluster().id().value()), new ServiceType("serviceType"), serviceInstances)); TenantId tenantId = new TenantId(app.getKey().tenant().value()); ApplicationInstanceId applicationInstanceId = new ApplicationInstanceId(app.getKey().application().value()); status.put(new ApplicationInstanceReference(tenantId, applicationInstanceId), new ApplicationInstance(tenantId, applicationInstanceId, serviceClusters)); } return status; }
private void toSlime(ClusterMembership membership, Cursor object) { object.setString("clustertype", membership.cluster().type().name()); object.setString("clusterid", membership.cluster().id().value()); object.setString("group", String.valueOf(membership.cluster().group().get().index())); object.setLong("index", membership.index()); object.setBool("retired", membership.retired()); }
"app", toApp(applicationId), "clustertype", allocation.get().membership().cluster().type().name(), "clusterid", allocation.get().membership().cluster().id().value());