gcloud: Fallback zone support (WIP)

mantle/cmd/kola/options.go

@@ -121,7 +121,7 @@ func init() {
 	// gcp-specific options
 	sv(&kola.GCPOptions.Image, "gcp-image", "", "GCP image, full api endpoints names are accepted if resource is in a different project")
 	sv(&kola.GCPOptions.Project, "gcp-project", "fedora-coreos-devel", "GCP project name")
-	sv(&kola.GCPOptions.Zone, "gcp-zone", "us-central1-a", "GCP zone name")
+	sv(&kola.GCPOptions.PreferredZone, "gcp-zone", "us-central1-a", "Preferred GCP zone name, if the resources to this zone are depleted, we will fallback to another zone in the same region")
 	sv(&kola.GCPOptions.MachineType, "gcp-machinetype", "", "GCP machine type")
 	sv(&kola.GCPOptions.DiskType, "gcp-disktype", "", "GCP disk type (default pd-ssd)")
 	sv(&kola.GCPOptions.Network, "gcp-network", "default", "GCP network")

mantle/cmd/ore/gcloud/destroy.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/coreos/coreos-assembler/mantle/platform/machine/gcloud"
 	"github.com/spf13/cobra"
 )
 
@@ -54,7 +55,12 @@ func runDestroy(cmd *cobra.Command, args []string) {
 
 	var count int
 	for _, vm := range vms {
-		if err := api.TerminateInstance(vm.Name); err != nil {
+		zone, err := gcloud.ExtractZoneFromInstance(vm)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Could not get vm's zone from URL %s: %v", vm.Zone, err)
+			os.Exit(1)
+		}
+		if err := api.TerminateInstance(vm.Name, zone); err != nil {
 			fmt.Fprintf(os.Stderr, "Failed destroying vm: %v\n", err)
 			os.Exit(1)
 		}

mantle/cmd/ore/gcloud/gcloud.go

@@ -41,7 +41,7 @@ func init() {
 
 	sv(&opts.Image, "image", "", "image name")
 	sv(&opts.Project, "project", "fedora-coreos-devel", "project")
-	sv(&opts.Zone, "zone", "us-central1-a", "zone")
+	sv(&opts.PreferredZone, "zone", "us-central1-a", "zone")
 	sv(&opts.MachineType, "machinetype", "n1-standard-1", "machine type")
 	sv(&opts.DiskType, "disktype", "pd-ssd", "disk type")
 	sv(&opts.BaseName, "basename", "kola", "instance name prefix")

mantle/platform/api/gcloud/api.go

@@ -17,10 +17,13 @@ package gcloud
 
 import (
 	"context"
-	"google.golang.org/api/option"
+	"fmt"
 	"net/http"
+	"regexp"
 	"time"
 
+	"google.golang.org/api/option"
+
 	"github.com/coreos/pkg/capnslog"
 	"google.golang.org/api/compute/v1"
 
@@ -35,7 +38,7 @@ var (
 type Options struct {
 	Image            string
 	Project          string
-	Zone             string
+	PreferredZone    string
 	MachineType      string
 	DiskType         string
 	Network          string
@@ -50,6 +53,68 @@ type API struct {
 	client  *http.Client
 	compute *compute.Service
 	options *Options
+	zones   []string
+}
+
+// This regex should match all standard (non-AI) zones
+// See: https://docs.cloud.google.com/compute/docs/regions-zones
+var standardZoneRegexp = regexp.MustCompile(`^([a-z]+-[a-z]+\d+)-[a-z]$`)
+
+// zones are in the form "us-central1-a" and the region would be "us-central1"
+// See: https://docs.cloud.google.com/compute/docs/regions-zones
+func extractRegionFromZone(zone string) (string, error) {
+	matches := standardZoneRegexp.FindStringSubmatch(zone)
+	if matches == nil {
+		return "", fmt.Errorf("zone %q does not match expected format {region}-{letter}", zone)
+	}
+	return matches[1], nil
+}
+
+func getAvailableZones(computeService *compute.Service, opts *Options) ([]string, error) {
+	if opts.MachineType == "" {
+		return []string{opts.PreferredZone}, nil
+	}
+
+	list, err := computeService.MachineTypes.AggregatedList(opts.Project).
+		Filter("name=" + opts.MachineType).Do()
+
+	if err != nil {
+		return nil, err
+	}
+
+	targetRegion, err := extractRegionFromZone(opts.PreferredZone)
+	if err != nil {
+		return nil, fmt.Errorf("could not extract region from zone %q: %w", opts.PreferredZone, err)
+	}
+
+	zones := []string{}
+	for _, scopedList := range list.Items {
+		// There should be either 1 or 0 MachineTypes
+		// 0 if this zone does not have the required machine type
+		// 1 if this zone does have the required machine type
+		if len(scopedList.MachineTypes) == 0 {
+			continue
+		}
+		if len(scopedList.MachineTypes) > 1 {
+			plog.Warningf("Unexpected: got %d machine types for filter name=%s", len(scopedList.MachineTypes), opts.MachineType)
+			continue
+		}
+		zone := scopedList.MachineTypes[0].Zone
+		if region, err := extractRegionFromZone(zone); err == nil && region == targetRegion {
+			// If the preferred zone can be used, it should be the first zone that we use,
+			// so we will make add it to the start of the list, rather than the end.
+			if zone == opts.PreferredZone {
+				zones = append([]string{zone}, zones...)
+			} else {
+				zones = append(zones, zone)
+			}
+		}
+	}
+
+	if len(zones) == 0 {
+		return zones, fmt.Errorf("no zones in region %s for machine type %s were found", targetRegion, opts.MachineType)
+	}
+	return zones, nil
 }
 
 func New(opts *Options) (*API, error) {
@@ -83,6 +148,12 @@ func New(opts *Options) (*API, error) {
 		return nil, err
 	}
 
+	zones, err := getAvailableZones(computeService, opts)
+	if err != nil {
+		plog.Warningf("Failed to discover available zones: %v. Falling back to preferred zone (%s) only.", err, opts.PreferredZone)
+		zones = []string{opts.PreferredZone}
+	}
+
 	if opts.ServiceAcct == "" {
 		proj, err := computeService.Projects.Get(opts.Project).Do()
 		if err != nil {
@@ -95,6 +166,7 @@ func New(opts *Options) (*API, error) {
 		client:  client,
 		compute: computeService,
 		options: opts,
+		zones:   zones,
 	}
 
 	return api, nil
@@ -105,5 +177,10 @@ func (a *API) Client() *http.Client {
 }
 
 func (a *API) GC(gracePeriod time.Duration) error {
-	return a.gcInstances(gracePeriod)
+	for _, zone := range a.zones {
+		if err := a.gcInstances(gracePeriod, zone); err != nil {
+			return err
+		}
+	}
+	return nil
 }