feat: owasp security event logging for user, password, and shutdown events

.github/workflows/10-daily-unit-lint.yml

@@ -28,6 +28,7 @@ jobs:
         run: tox -e hypothesis-slow
   devel_tests:
     name: "Tip: Python"
+    if: github.repository == 'canonical/cloud-init'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -44,14 +45,15 @@ jobs:
       - name: Run unittest
         run: tox -e py3 -- --color=yes
   format_tip:
+    name: "Tip: Lint"
+    if: github.repository == 'canonical/cloud-init'
+    runs-on: ubuntu-latest
     env:
       FORCE_COLOR: 1
     strategy:
       fail-fast: false
       matrix:
         env: [tip-ruff, tip-mypy, tip-pylint, tip-black, tip-isort, py3-fast]
-    name: "Tip: Lint"
-    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -65,6 +67,7 @@ jobs:
         run: tox
   macos:
     name: Python ${{matrix.python-version}} ${{ matrix.slug }}
+    if: github.repository == 'canonical/cloud-init'
     runs-on: macos-latest
     strategy:
       matrix:

.github/workflows/21-pr-check-format.yml

@@ -13,11 +13,11 @@ defaults:
     shell: sh -ex {0}
 
 jobs:
-  check_format:
+  check_linters:
     strategy:
       fail-fast: false
       matrix:
-        env: [ruff, mypy, pylint, black, isort]
+        env: [check_format]
     name: Check ${{ matrix.env }}
     runs-on: ubuntu-latest
     env:
@@ -30,6 +30,7 @@ jobs:
         run: |
           sudo DEBIAN_FRONTEND=noninteractive apt-get -qy update
           sudo DEBIAN_FRONTEND=noninteractive apt-get -qy install tox
+          sudo snap install shfmt
 
       - name: Print version
         run: python3 --version
@@ -39,20 +40,12 @@ jobs:
           # matrix env: not to be confused w/environment variables or testenv
           TOXENV: ${{ matrix.env }}
         run: tox
-  schema-format:
-    strategy:
-      fail-fast: false
-    name: Check json format
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
-      - name: Test format
+      - name: Formatting check failed
+        if: failure()
         run: |
-          tools/check_json_format.sh cloudinit/config/schemas/schema-cloud-config-v1.json
-          tools/check_json_format.sh cloudinit/config/schemas/schema-network-config-v1.json
-          tools/check_json_format.sh cloudinit/config/schemas/versions.schema.cloud-config.json
+          echo "[31mResolve formatting errors with `tox -e do_format` (requires shfmt to format shell).\e[0m"
+          echo "[31mFor mypy and pylint failures see the warnings above.\e[0m"
 
   doc:
     strategy:

.github/workflows/24-pr-integration.yml

@@ -67,4 +67,12 @@ jobs:
           echo "[lxd]" > /home/$USER/.config/pycloudlib.toml
       - name: Run integration Tests
         run: |
-          CLOUD_INIT_CLOUD_INIT_SOURCE="$(ls ${{ runner.temp }}/cloud-init-base*.deb)" CLOUD_INIT_OS_IMAGE=${{ env.RELEASE }} tox -e integration-tests-ci -- --color=yes tests/integration_tests/
+          CLOUD_INIT_CLOUD_INIT_SOURCE="$(ls ${{ runner.temp }}/cloud-init-base*.deb)" CLOUD_INIT_OS_IMAGE=${{ env.RELEASE }} CLOUD_INIT_LOCAL_LOG_PATH=./cloudinit_logs tox -e integration-tests-ci -- --color=yes tests/integration_tests/
+      - name: Upload cloudinit logs on failure
+        if: failure()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f  # v6.0.0
+        with:
+          name: 'cloudinit-logs'
+          path: './cloudinit_logs'
+          retention-days: 3
+          if-no-files-found: ignore

.pylintrc

@@ -32,32 +32,3 @@ output-format=parseable
 
 # Just the errors please, no full report
 reports=no
-
-
-[TYPECHECK]
-
-# List of module names for which member attributes should not be checked
-# (useful for modules/projects where namespaces are manipulated during runtime
-# and thus existing member attributes cannot be deduced by static analysis. It
-# supports qualified module names, as well as Unix pattern matching.
-ignored-modules=
- http.client,
- httplib,
- pkg_resources,
- # cloud_tests requirements.
- boto3,
- botocore,
- paramiko,
- pylxd,
- simplestreams
-
-# List of class names for which member attributes should not be checked (useful
-# for classes with dynamically set attributes). This supports the use of
-# qualified names.
-# argparse.Namespace from https://github.com/PyCQA/pylint/issues/2413
-ignored-classes=argparse.Namespace,optparse.Values,thread._local,ImageManager,ContainerManager
-
-# List of members which are set dynamically and missed by pylint inference
-# system, and so shouldn't trigger E1101 when accessed. Python regular
-# expressions are accepted.
-generated-members=types,http.client,command_handlers,m_.*,enter_context

Makefile

@@ -9,40 +9,18 @@ distro ?= redhat
 
 GENERATOR_F=./systemd/cloud-init-generator
 DS_IDENTIFY=./tools/ds-identify
-BENCHMARK=./tools/benchmark.sh
 
 
 all: check
 
 check: test
 
-style-check: lint
-
-lint:
-	@$(CWD)/tools/run-lint
-
 unittest: clean_pyc
 	$(PYTHON) -m pytest -v tests/unittests cloudinit
 
 render-template:
 	$(PYTHON) ./tools/render-template --variant=$(VARIANT) $(FILE) $(subst .tmpl,,$(FILE))
 
-# from systemd-generator(7) regarding generators:
-# "We do recommend C code however, since generators are executed
-# synchronously and hence delay the entire boot if they are slow."
-#
-# Our generator is a shell script. Make it easy to measure the
-# generator. This should be monitored for performance regressions
-benchmark-generator: FILE=$(GENERATOR_F).tmpl
-benchmark-generator: VARIANT="benchmark"
-benchmark-generator: export ITER=$(NUM_ITER)
-benchmark-generator: render-template
-	$(BENCHMARK) $(GENERATOR_F)
-
-benchmark-ds-identify: export ITER=$(NUM_ITER)
-benchmark-ds-identify:
-	$(BENCHMARK) $(DS_IDENTIFY)
-
 ci-deps-ubuntu:
 	@$(PYTHON) $(CWD)/tools/read-dependencies --distro ubuntu --test-distro
 
@@ -103,13 +81,6 @@ deb-src:
 doc:
 	tox -e doc
 
-fmt:
-	tox -e do_format && tox -e check_format
-
-fmt-tip:
-	tox -e do_format_tip && tox -e check_format_tip
-
-
-.PHONY: all check test lint clean rpm srpm deb deb-src clean_pyc
-.PHONY: unittest style-check render-template benchmark-generator
+.PHONY: all check test clean rpm srpm deb deb-src clean_pyc
+.PHONY: unittest render-template
 .PHONY: clean_pytest clean_packaging clean_release doc

cloudinit/config/cc_set_passwords.py

@@ -11,7 +11,7 @@
 import random
 import re
 import string
-from typing import List
+from typing import List, Tuple
 
 from cloudinit import features, lifecycle, subp, util
 from cloudinit.cloud import Cloud
@@ -32,7 +32,7 @@
 LOG = logging.getLogger(__name__)
 
 
-def get_users_by_type(users_list: list, pw_type: str) -> list:
+def get_users_by_type(users_list: list, pw_type: str) -> List[Tuple[str, str]]:
     """either password or type: RANDOM is required, user is always required"""
     return (
         []

cloudinit/config/cc_users_groups.py

@@ -7,7 +7,9 @@
 """Users and Groups: Configure users and groups"""
 
 import logging
+from typing import List, Union
 
+from cloudinit import lifecycle
 from cloudinit.cloud import Cloud
 
 # Ensure this is aliased to a name not 'distros'
@@ -32,6 +34,37 @@
 NEED_HOME = ("ssh_authorized_keys", "ssh_import_id", "ssh_redirect_user")
 
 
+def _normalize_user_groups(
+    user: str, groups: Union[str, List[str], dict]
+) -> List[str]:
+    if not groups:
+        return []
+
+    if isinstance(groups, str):
+        return [group.strip() for group in groups.split(",") if group.strip()]
+
+    if isinstance(groups, dict):
+        lifecycle.deprecate(
+            deprecated=f"The user {user} has a 'groups' config value "
+            "of type dict",
+            deprecated_version="22.3",
+            extra_message="Use a comma-delimited string or "
+            "array instead: group1,group2.",
+        )
+        return list(groups)
+
+    if isinstance(groups, list):
+        if not all(isinstance(group, str) for group in groups):
+            raise TypeError(
+                f"Not creating user {user}. 'groups' must contain only "
+                "string values."
+            )
+        return groups
+    raise TypeError(
+        f"Not creating user {user}. 'groups' must be a string, list, or dict."
+    )
+
+
 def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None:
     (users, groups) = ug_util.normalize_users_groups(cfg, cloud.distro)
     (default_user, _user_config) = ug_util.extract_default(users)
@@ -41,6 +74,7 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None:
         cloud.distro.create_group(name, members)
 
     for user, config in users.items():
+        user_groups = _normalize_user_groups(user, config.pop("groups", []))
 
         no_home = [key for key in NO_HOME if config.get(key)]
         need_home = [key for key in NEED_HOME if config.get(key)]
@@ -77,4 +111,4 @@ def handle(name: str, cfg: Config, cloud: Cloud, args: list) -> None:
                 config["ssh_redirect_user"] = default_user
                 config["cloud_public_ssh_keys"] = cloud_keys
 
-        cloud.distro.create_user(user, **config)
+        cloud.distro.create_user(user, groups=user_groups, **config)