private static Zone getZone() { if (!isConfigServer()) { return Zone.defaultZone(); } CloudConfigInstallVariables cloudConfigVariables = new CloudConfigInstallVariables(); if (!cloudConfigVariables.hostedVespa().orElse(false)) { return Zone.defaultZone(); } RegionName region = cloudConfigVariables.region().map(RegionName::from).orElseGet(RegionName::defaultName); Environment environment = cloudConfigVariables.environment().map(Environment::from).orElseGet(Environment::defaultEnvironment); SystemName system = cloudConfigVariables.system().map(SystemName::from).orElseGet(SystemName::defaultSystem); return new Zone(system, environment, region); }
private String getConfigserverIdentityName() { return String.format("%s.provider_%s_%s", zone.system() == SystemName.main ? "vespa.vespa" : "vespa.vespa.cd", zone.environment().value(), zone.region().value()); } }
/** * Returns the distribution bits this cluster should use. * On Hosted Vespa this is hardcoded and not computed from the nodes because reducing the number of nodes is a common * operation, while reducing the number of distribution bits can lead to consistency problems. * This hardcoded value should work fine from 1-200 nodes. Those who have more will need to set this value * in config and not remove it again if they reduce the node count. */ public int distributionBits() { if (zone.environment() == Environment.prod && ! zone.equals(Zone.defaultZone())) { return 16; } else { // hosted test zone, or self-hosted system // hosted test zones: have few nodes and use visiting in tests: This is slow with 16 bits (to many buckets) // self hosted systems: should probably default to 16 bits, but the transition may cause problems return DistributionBitCalculator.getDistributionBits(getNodeCountPerGroup(), getDistributionMode()); } }
private void preprocessXML(File destination, File inputXml, Zone zone) throws ParserConfigurationException, TransformerException, SAXException, IOException { Document document = new XmlPreProcessor(appDir, inputXml, zone.environment(), zone.region()).run(); Transformer transformer = TransformerFactory.newInstance().newTransformer(); try (FileOutputStream outputStream = new FileOutputStream(destination)) { transformer.transform(new DOMSource(document), new StreamResult(outputStream)); } }
public FailedExpirer(NodeRepository nodeRepository, Zone zone, Clock clock, Duration interval, JobControl jobControl) { super(nodeRepository, interval, jobControl); this.nodeRepository = nodeRepository; this.zone = zone; this.clock = clock; if (zone.system() == SystemName.main) { if (zone.environment() == Environment.staging || zone.environment() == Environment.test) { defaultExpiry = Duration.ofHours(1); } else { defaultExpiry = Duration.ofDays(4); } containerExpiry = Duration.ofHours(1); } else { defaultExpiry = containerExpiry = Duration.ofMinutes(30); } }
/** * Whether or not the nodes requested can share physical host with other applications. * A security feature which only makes sense for prod. */ public boolean decideExclusivity(boolean requestedExclusivity) { return requestedExclusivity && zone.environment() == Environment.prod; }
public Flavor decideFlavor(Capacity requestedCapacity, ClusterSpec cluster) { // for now, always use the requested flavor if a docker flavor is requested Optional<String> requestedFlavor = requestedCapacity.flavor(); if (requestedFlavor.isPresent() && flavors.getFlavorOrThrow(requestedFlavor.get()).getType() == Flavor.Type.DOCKER_CONTAINER) return flavors.getFlavorOrThrow(requestedFlavor.get()); String defaultFlavorName = zone.defaultFlavor(cluster.type()); if (zone.system() == SystemName.cd) return flavors.getFlavorOrThrow(requestedFlavor.orElse(defaultFlavorName)); switch(zone.environment()) { case dev : case test : case staging : return flavors.getFlavorOrThrow(defaultFlavorName); default : return flavors.getFlavorOrThrow(requestedFlavor.orElse(defaultFlavorName)); } }
public VespaModelFactory(Version version, ConfigModelRegistry configModelRegistry, Clock clock) { this.version = version; if (configModelRegistry == null) { this.configModelRegistry = new NullConfigModelRegistry(); log.info("Will not load config models from plugins, as no registry is available"); } else { this.configModelRegistry = configModelRegistry; } this.modelImporters = Collections.emptyList(); this.zone = Zone.defaultZone(); this.clock = clock; }
/** Do not use */ public static Zone defaultZone() { return new Zone(SystemName.defaultSystem(), Environment.defaultEnvironment(), RegionName.defaultName()); }
private static String buildJvmGCOptions(Zone zone, String jvmGCOPtions, boolean isHostedVespa) { if (jvmGCOPtions != null) { return jvmGCOPtions; } else if ((zone.system() == SystemName.dev) || isHostedVespa) { return null; } else { return ContainerCluster.G1GC; } } private static String getJvmOptions(ContainerCluster cluster, Element nodesElement, DeployLogger deployLogger) {
NodeRetirer.SPARE_NODES_POLICY, FlavorSpareCount.constructFlavorSpareCountGraph(zone.nodeFlavors().get().getFlavors())); nodeRetirer = new NodeRetirer(nodeRepository, flavorSpareChecker, durationFromEnv("retire_interval").orElse(defaults.nodeRetirerInterval), deployer, jobControl, policy);
private boolean zoneHasActiveRotation(Zone zone, DeploymentSpec spec) { return spec.zones().stream() .anyMatch(declaredZone -> declaredZone.deploysTo(zone.environment(), Optional.of(zone.region())) && declaredZone.active()); }
private boolean checkForClashingParentHost() { return nodeRepository.zone().system() == SystemName.main && nodeRepository.zone().environment().isProduction(); }
/** * Throw if the node count is 1 for container and content clusters and we're in a production zone * * @return the argument node count * @throws IllegalArgumentException if only one node is requested and we can fail */ private int ensureRedundancy(int nodeCount, ClusterSpec.Type clusterType, boolean canFail) { if (canFail && nodeCount == 1 && Arrays.asList(ClusterSpec.Type.container, ClusterSpec.Type.content).contains(clusterType) && zone.environment().isProduction()) throw new IllegalArgumentException("Deployments to prod require at least 2 nodes per cluster for redundancy"); return nodeCount; }
/** * Constructor * @param flavors flavors to have in node repo */ public MockNodeRepository(MockCurator curator, NodeFlavors flavors) { super(flavors, curator, Clock.fixed(Instant.ofEpochMilli(123), ZoneId.of("Z")), Zone.defaultZone(), new MockNameResolver() .addRecord("test-container-1", "::2") .mockAnyLookup(), new DockerImage("docker-registry.domain.tld:8080/dist/vespa"), true); this.flavors = flavors; curator.setZooKeeperEnsembleConnectionSpec("cfg1:1234,cfg2:1234,cfg3:1234"); populate(); }
public void setContainerCluster(ContainerCluster containerCluster) { this.containerCluster = containerCluster; // If we are in a config server cluster the correct zone is propagated through cloud config options, // not through config to deployment options (see StandaloneContainerApplication.scala), // so we need to propagate the zone options into the container from here Environment environment = options.environment().isPresent() ? Environment.from(options.environment().get()) : Environment.defaultEnvironment(); RegionName region = options.region().isPresent() ? RegionName.from(options.region().get()) : RegionName.defaultName(); SystemName system = options.system().isPresent() ? SystemName.from(options.system().get()) : SystemName.defaultSystem(); containerCluster.setZone(new Zone(system, environment, region)); }
@Inject public AuthorizationFilter(Zone zone, NodeRepository nodeRepository, NodeRepositoryConfig nodeRepositoryConfig) { this( new Authorizer( zone.system(), nodeRepository, Stream.concat( Stream.of(HostName.getLocalhost()), Stream.of(nodeRepositoryConfig.hostnameWhitelist().split(",")) ).filter(hostname -> !hostname.isEmpty()).collect(Collectors.toSet())), AuthorizationFilter::logAndReject ); }
@Override public boolean isActive() { if(zone.system() == SystemName.cd) { return zone.environment() == Environment.dev || zone.environment() == Environment.prod; } if (zone.system() == SystemName.main) { if (zone.region().equals(RegionName.from("us-east-3"))) { return zone.environment() == Environment.perf || zone.environment() == Environment.prod; } else if (zone.region().equals(RegionName.from("us-west-1"))) { return zone.environment() == Environment.prod; } else if (zone.region().equals(RegionName.from("us-central-1"))) { return zone.environment() == Environment.prod; } else if (zone.region().equals(RegionName.from("ap-southeast-1"))) { return zone.environment() == Environment.prod; } else if (zone.region().equals(RegionName.from("ap-northeast-1"))) { return zone.environment() == Environment.prod; } else if (zone.region().equals(RegionName.from("ap-northeast-2"))) { return zone.environment() == Environment.prod; } else if (zone.region().equals(RegionName.from("eu-west-1"))) { return zone.environment() == Environment.prod; } } return false; }
private void addIdentityProvider(ContainerCluster cluster, List<ConfigServerSpec> configServerSpecs, HostName loadBalancerName, URI ztsUrl, String athenzDnsSuffix, Zone zone, DeploymentSpec spec) { spec.athenzDomain().ifPresent(domain -> { AthenzService service = spec.athenzService(zone.environment(), zone.region()) .orElseThrow(() -> new RuntimeException("Missing Athenz service configuration")); String zoneDnsSuffix = zone.environment().value() + "-" + zone.region().value() + "." + athenzDnsSuffix; IdentityProvider identityProvider = new IdentityProvider(domain, service, getLoadBalancerName(loadBalancerName, configServerSpecs), ztsUrl, zoneDnsSuffix, zone); cluster.addComponent(identityProvider); cluster.getContainers().forEach(container -> { container.setProp("identity.domain", domain.value()); container.setProp("identity.service", service.value()); }); }); }
DefaultTimes(Zone zone) { failGrace = Duration.ofMinutes(60); periodicRedeployInterval = Duration.ofMinutes(30); redeployMaintainerInterval = Duration.ofMinutes(1); operatorChangeRedeployInterval = Duration.ofMinutes(1); failedExpirerInterval = Duration.ofMinutes(10); provisionedExpiry = Duration.ofHours(4); rebootInterval = Duration.ofDays(30); nodeRetirerInterval = Duration.ofMinutes(30); metricsInterval = Duration.ofMinutes(1); infrastructureProvisionInterval = Duration.ofMinutes(3); throttlePolicy = NodeFailer.ThrottlePolicy.hosted; loadBalancerExpiry = Duration.ofHours(1); reservationExpiry = Duration.ofMinutes(20); // Need to be long enough for deployment to be finished for all config model versions hostProvisionerInterval = Duration.ofMinutes(5); hostDeprovisionerInterval = Duration.ofMinutes(5); if (zone.environment().equals(Environment.prod) && zone.system() != SystemName.cd) { inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy retiredInterval = Duration.ofMinutes(29); dirtyExpiry = Duration.ofHours(2); // enough time to clean the node retiredExpiry = Duration.ofDays(4); // give up migrating data after 4 days } else { inactiveExpiry = Duration.ofSeconds(2); // support interactive wipe start over retiredInterval = Duration.ofMinutes(1); dirtyExpiry = Duration.ofMinutes(30); retiredExpiry = Duration.ofMinutes(20); } }