Convert OEM packages to sysext, drop several images, revamp Vagrant support

.github/workflows/ci.yaml

@@ -189,25 +189,15 @@ jobs:
 
           images_out="images"
 
-          has_packet=0
-          has_pxe=0
-          formats="${IMAGE_FORMATS}"
-          for format in "${formats}";do
-            [[ "${format}" = 'packet' ]] || [[ "${format}" = 'equinix_metal' ]] && has_packet=1
-            [[ "${format}" = 'pxe' ]] && has_pxe=1
-          done
-
-          [[ ${has_packet} -eq 1 ]] && [[ ${has_pxe} -eq 0 ]] && set -- 'pxe' "${@}"
-          if echo "$formats" | tr ' ' '\n' | grep -q '^vmware'; then
-            formats=$(echo "$formats" | tr ' ' '\n' | sed '/vmware.*/d')
-            formats+=" vmware vmware_insecure vmware_ova vmware_raw"
+          printf -v formats "%s\n" ${formats}
+          if grep -q '^vmware' <<< "${formats}"; then
+              formats=$(grep -v '^vmware' <<< "${formats}")
+              printf -v formats "%s\n" ${formats} vmware vmware_ova vmware_raw
           fi
-          if echo "$formats" | tr ' ' '\n' | grep -q -P '^(ami|aws)'; then
-            formats=$(echo "$formats" | tr ' ' '\n' | sed '/ami.*/d' | sed '/aws/d')
-            formats+=" ami ami_vmdk"
+          if printf "%s\n" $formats | grep -q '^ami\|^aws'; then
+              formats=$(grep -v '^ami\|^aws' <<< "${formats}")
+              printf -v formats "%s\n" ${formats} ami ami_vmdk
           fi
-          # Keep compatibility with SDK scripts where "equinix_metal" remains unknown.
-          formats=$(echo "$formats" | tr ' ' '\n' | sed 's/equinix_metal/packet/g')
 
           for format in ${formats}; do
               if [ "${format}" = qemu ] || [ "${format}" = qemu_uefi_secure ]; then

build_library/check_root

@@ -107,7 +107,6 @@ IGNORE_SHEBANG = (
     "*/python[0-9].[0-9][0-9]/cgi.py",
     "*/usr/lib64/modules/*/source/scripts/*",
     "*/usr/lib/modules/*/source/scripts/*",
-    "*/usr/share/nova-agent/*/etc/gentoo/nova-agent",
     "*/tmp/*",
     "*/Documentation/*",
     "*/doc/*",

build_library/disk_layout.json

@@ -98,14 +98,6 @@
         "blocks":"33845248"
       }
     },
-    "onmetal":{
-      "7":{
-        "label":"config-2",
-        "fs_label":"config-2",
-        "type":"data",
-        "fs_type":"ext2"
-      }
-    },
     "container":{
       "1":{
         "type":"blank"

build_library/extra_sysexts.sh

@@ -11,19 +11,3 @@ EXTRA_SYSEXTS=(
   "python|dev-lang/python,dev-python/pip"
   "zfs|sys-fs/zfs"
 )
-
-_get_unversioned_sysext_packages_unsorted() {
-  for sysext in "${EXTRA_SYSEXTS[@]}"; do
-    IFS="|" read -r _ PACKAGE_ATOMS _ <<< "$sysext"
-
-    IFS=,
-    for atom in $PACKAGE_ATOMS; do
-       qatom "$atom" -F "%{CATEGORY}/%{PN}"
-    done
-    unset IFS
-  done
-}
-
-get_unversioned_sysext_packages() {
-  _get_unversioned_sysext_packages_unsorted | sort | uniq
-}

sdk_container/src/third_party/coreos-overlay/coreos-base/oem-vagrant/files/grub.cfg → build_library/oem/akamai/grub.cfg

@@ -1,3 +1,3 @@
 # Flatcar GRUB settings
 
-set oem_id="vagrant"
+set oem_id="akamai"

sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/ami/grub.cfg.frag → build_library/oem/ami/grub.cfg

@@ -1,4 +1,6 @@
-# Flatcar GRUB settings for AMI
+# Flatcar GRUB settings
+
+set oem_id="ec2"
 
 # Blacklist the Xen framebuffer module so it doesn't get loaded at boot
 # Disable `ens3` style names, so eth0 is used for both ixgbevf or xen.

sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/azure/grub.cfg.frag → build_library/oem/azure/grub.cfg

@@ -1,3 +1,6 @@
+# Flatcar GRUB settings
+
+set oem_id="azure"
 set linux_append="flatcar.autologin"
 
 # Azure only has a serial console.

build_library/oem/digitalocean/grub.cfg

@@ -0,0 +1,3 @@
+# Flatcar GRUB settings
+
+set oem_id="digitalocean"

sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/gce/grub.cfg.frag → build_library/oem/gce/grub.cfg

@@ -1,3 +1,7 @@
+# Flatcar GRUB settings
+
+set oem_id="gce"
+
 # GCE only has a serial console.
 set linux_console="console=ttyS0,115200n8"
 serial com0 --speed=115200 --word=8 --parity=no

sdk_container/src/third_party/coreos-overlay/coreos-base/oem-rackspace/files/grub.cfg → build_library/oem/hetzner/grub.cfg

@@ -1,3 +1,3 @@
 # Flatcar GRUB settings
 
-set oem_id="rackspace"
+set oem_id="hetzner"

build_library/oem/kubevirt/grub.cfg

@@ -0,0 +1,3 @@
+# Flatcar GRUB settings
+
+set oem_id="kubevirt"

build_library/oem/nutanix/grub.cfg

@@ -0,0 +1,3 @@
+# Flatcar GRUB settings
+
+set oem_id="nutanix"

build_library/oem/openstack/grub.cfg

@@ -0,0 +1,4 @@
+# Flatcar GRUB settings
+
+set oem_id="openstack"
+set linux_append="flatcar.autologin"

build_library/oem/proxmoxve/grub.cfg

@@ -0,0 +1,4 @@
+# Flatcar GRUB settings
+
+set oem_id="proxmoxve"
+set linux_append="flatcar.autologin"

build_library/oem/qemu/grub.cfg

@@ -0,0 +1,4 @@
+# Flatcar GRUB settings
+
+set oem_id="qemu"
+set linux_append="flatcar.autologin"

sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/scaleway/grub.cfg.frag → build_library/oem/scaleway/grub.cfg

@@ -1 +1,4 @@
+# Flatcar GRUB settings
+
+set oem_id="scaleway"
 set linux_console="console=ttyS0,115200n8 earlycon=ttyS0,115200"

build_library/oem/stackit/grub.cfg

@@ -0,0 +1,3 @@
+# Flatcar GRUB settings
+
+set oem_id="openstack"

sdk_container/src/third_party/coreos-overlay/coreos-base/oem-vagrant/files/box/Vagrantfile → build_library/oem/vagrant/build/box/Vagrantfile

@@ -1,11 +1,8 @@
 # -*- mode: ruby -*-
 # # vi: set ft=ruby :
 
-if Vagrant::VERSION < "1.6.0"
-  raise "Need at least vagrant version 1.6.0, please update"
-end
+Vagrant.require_version ">= 2.2.5"
 
-require_relative 'change_host_name.rb'
 require_relative 'configure_networks.rb'
 require_relative 'base_mac.rb'
 
@@ -27,6 +24,10 @@ Vagrant.configure("2") do |config|
     # Fix docker not being able to resolve private registry in VirtualBox
     vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
     vb.customize ["modifyvm", :id, "--natdnsproxy1", "on"]
+
+    if File.exist?("config.ign")
+      vb.customize ["guestproperty", "set", :id, "/Ignition/Config", File.read("config.ign")]
+    end
   end
 
   config.vm.provider :vmware_fusion do |vf|
@@ -38,4 +39,9 @@ Vagrant.configure("2") do |config|
     prl.check_guest_tools = false
     prl.functional_psf    = false
   end
- end
+
+  if File.exist?("user-data")
+    config.vm.provision :file, :source => "user-data", :destination => "/tmp/vagrantfile-user-data"
+    config.vm.provision :shell, :inline => "mv /tmp/vagrantfile-user-data /var/lib/flatcar-vagrant/", :privileged => true
+  end
+end

sdk_container/src/third_party/coreos-overlay/coreos-base/oem-vagrant-virtualbox/files/box/configure_networks.rb → build_library/oem/vagrant/build/box/configure_networks.rb

@@ -1,8 +1,9 @@
 # -*- mode: ruby -*-
 # # vi: set ft=ruby :
 
-# NOTE: This monkey-patching is done to disable to old cloud config based
-# configure_networks built into the upstream vagrant project
+# NOTE: This monkey-patching is done to force cloud-init over NetworkManager.
+# Vagrant attempts to detect cloud-init, but Flatcar doesn't have an executable
+# under that name, only coreos-cloudinit.
 
 require Vagrant.source_root.join("plugins/guests/coreos/cap/configure_networks.rb")
 
@@ -11,6 +12,7 @@ module GuestCoreOS
     module Cap
       class ConfigureNetworks
         def self.configure_networks(machine, networks)
+          configure_networks_cloud_init(machine, networks)
         end
       end
     end

build_library/oem/vmware/grub.cfg

@@ -0,0 +1,4 @@
+# Flatcar GRUB settings
+
+set oem_id="vmware"
+set linux_append="flatcar.autologin"

build_library/oem_sysexts.sh

@@ -20,42 +20,35 @@ get_oem_overlay_root() {
   printf '%s' "${overlay_root}"
 }
 
-_get_oem_ids() {
-  local arch list_var_name
-  arch=${1}; shift
-  list_var_name=${1}; shift
-
-  local overlay_root
-  overlay_root=$(get_oem_overlay_root)
-
-  local -a ebuilds=("${overlay_root}/coreos-base/common-oem-files/common-oem-files-"*'.ebuild')
-  if [[ ${#ebuilds[@]} -eq 0 ]] || [[ ! -e ${ebuilds[0]} ]]; then
-    echo "No coreos-base/common-oem-files ebuilds?!" >&2
-    exit 1
-  fi
-
-  # This defines local COMMON_OEMIDS, AMD64_ONLY_OEMIDS,
-  # ARM64_ONLY_OEMIDS and OEMIDS variable. We don't use the last
-  # one. Also defines global-by-default EAPI, which we make local
-  # here to avoid making it global.
-  local EAPI
-  source "${ebuilds[0]}" flatcar-local-variables
-
-  local -n arch_oemids_ref="${arch^^}_ONLY_OEMIDS"
-  local all_oemids=(
-    "${COMMON_OEMIDS[@]}"
-    "${arch_oemids_ref[@]}"
-  )
-
-  mapfile -t "${list_var_name}" < <(printf '%s\n' "${all_oemids[@]}" | sort)
-}
-
 # Gets a list of OEMs that are using sysexts.
 #
 # 1 - arch
 # 2 - name of an array variable to store the result in
 get_oem_id_list() {
-  _get_oem_ids "$@"
+  local arch=${1}; shift
+  local -n list_var_ref=${1}; shift
+
+  local overlay_root dir ebuild regex
+  overlay_root=$(get_oem_overlay_root)
+
+  for dir in "${overlay_root}"/coreos-base/oem-*; do
+    for ebuild in "${dir}"/*.ebuild; do
+      if [[ ! -e ${ebuild} ]]; then
+        echo "No coreos-base/oem-* ebuilds?!" >&2
+        exit 1
+      fi
+
+      # Check the KEYWORDS by sourcing the ebuild. We can't rely on Portage
+      # because this needs to work outside the SDK. OEM ebuilds are relatively
+      # boring, so this should be sufficient. This doesn't check whether the
+      # KEYWORDS are stable, but that shouldn't matter.
+      regex="\b${arch}\b"
+      if ( set +eu; . "${ebuild}" &>/dev/null; [[ ${KEYWORDS} =~ ${regex} ]] ); then
+        list_var_ref+=( "${dir##*/oem-}" )
+        break
+      fi
+    done
+  done
 }
 
 # Gets a list of OEM sysext descriptors.
@@ -65,19 +58,14 @@ get_oem_id_list() {
 #
 # Format: "name|metapackage|useflags"
 get_oem_sysext_matrix() {
-  local arch list_var_name
-  arch=${1}; shift
-  list_var_name=${1}; shift
+  local arch=${1}; shift
+  declare -n list_var_ref=${1}; shift
 
   local -a oem_ids
-  _get_oem_ids "${arch}" oem_ids
+  get_oem_id_list "${arch}" oem_ids
 
-  local -a matrix=()
   local oem_id
   for oem_id in "${oem_ids[@]}"; do
-    matrix+=("oem-${oem_id}|coreos-base/oem-${oem_id}|${oem_id}")
+    list_var_ref+=( "oem-${oem_id}|coreos-base/oem-${oem_id}|" )
   done
-
-  local -n matrix_ref="${list_var_name}"
-  matrix_ref=("${matrix[@]}")
 }

build_library/sysext_prod_builder

@@ -15,6 +15,7 @@ SCRIPT_ROOT="$(cd "$(dirname "$(readlink -f "$0")")/../"; pwd)"
 assert_inside_chroot
 switch_to_strict_mode
 
+. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1
 
 # Create a sysext from a package and install it to the OS image.