From 331604c51f35c51ffb8faae8f833918124dbf4a6 Mon Sep 17 00:00:00 2001 From: Juanpe Araque Date: Sun, 10 May 2026 22:24:08 +0200 Subject: [PATCH 1/6] Migrate validate config from datadog_checks_dev to ddev Port the `validate config` command, the `tooling/configuration/` spec loader, and the `tooling/config_validator/` legacy YAML linter from `datadog_checks_dev` into ddev: - ddev/src/ddev/cli/validate/config.py: native command using the ddev Application/Integration model, with the same CLI surface (`[CHECK]`, `--sync/-s`, `--verbose/-v`). - ddev/src/ddev/validation/configuration/: spec loader, validator, template engine, and consumers (ExampleConsumer + ModelConsumer). The bundled templates directory moves with it. ModelConsumer .create_code_formatter() now takes an optional repo_path argument so callers can pass app.repo.path instead of relying on legacy get_root. - ddev/src/ddev/validation/config_spec/: legacy YAML linter ported one-to-one. Public surface (validate_config, SEVERITY_ERROR, SEVERITY_WARNING) unchanged. - Legacy command file, its entry in ALL_COMMANDS, and the legacy test_config.py are deleted; the bundled my_check / tokumx fixtures stay because validate models tests still consume them. - Tests rewritten as plain pytest functions in ddev/tests/cli/validate /test_config.py using fake_repo/fake_extras_repo/fake_marketplace_repo. PR 1.13 (validate models) will reuse `ddev.validation.configuration` unchanged. --- .../dev/tooling/commands/validate/__init__.py | 2 - .../dev/tooling/commands/validate/config.py | 247 ------ .../tooling/commands/validate/test_config.py | 73 -- ddev/src/ddev/cli/validate/__init__.py | 2 +- ddev/src/ddev/cli/validate/config.py | 303 +++++++ .../ddev/validation/config_spec/__init__.py | 3 + .../validation/config_spec/config_block.py | 431 ++++++++++ ddev/src/ddev/validation/config_spec/utils.py | 81 ++ .../ddev/validation/config_spec/validator.py | 160 ++++ .../config_spec/validator_errors.py | 20 + .../ddev/validation/configuration/__init__.py | 6 + .../validation/configuration/constants.py | 50 ++ .../configuration/consumers/__init__.py | 7 + .../configuration/consumers/example.py | 330 ++++++++ .../configuration/consumers/model/__init__.py | 0 .../consumers/model/model_consumer.py | 260 ++++++ .../consumers/model/model_file.py | 209 +++++ .../consumers/model/model_info.py | 97 +++ .../consumers/openapi_document.py | 168 ++++ .../src/ddev/validation/configuration/core.py | 57 ++ .../src/ddev/validation/configuration/spec.py | 756 ++++++++++++++++++ .../ddev/validation/configuration/template.py | 185 +++++ .../templates/ad_identifiers.yaml | 10 + .../templates/common/perf_counters.yaml | 142 ++++ .../configuration/templates/init_config.yaml | 4 + .../templates/init_config/db.yaml | 18 + .../templates/init_config/default.yaml | 1 + .../templates/init_config/http.yaml | 49 ++ .../templates/init_config/jmx.yaml | 60 ++ .../templates/init_config/openmetrics.yaml | 2 + .../init_config/openmetrics_legacy.yaml | 2 + .../init_config/openmetrics_legacy_base | 0 .../templates/init_config/perf_counters.yaml | 9 + .../templates/init_config/service.yaml | 9 + .../templates/init_config/tags.yaml | 13 + .../configuration/templates/instances.yaml | 5 + .../templates/instances/all_integrations.yaml | 3 + .../configuration/templates/instances/db.yaml | 102 +++ .../templates/instances/default.yaml | 47 ++ .../templates/instances/global.yaml | 19 + .../templates/instances/http.yaml | 440 ++++++++++ .../templates/instances/jmx.yaml | 181 +++++ .../templates/instances/openmetrics.yaml | 448 +++++++++++ .../instances/openmetrics_legacy.yaml | 5 + .../instances/openmetrics_legacy_base.yaml | 254 ++++++ .../templates/instances/pdh_legacy.yaml | 75 ++ .../templates/instances/perf_counters.yaml | 64 ++ .../templates/instances/service.yaml | 9 + .../templates/instances/tags.yaml | 14 + .../templates/instances/tls.yaml | 82 ++ .../configuration/templates/logs.yaml | 16 + ddev/tests/cli/validate/test_config.py | 138 ++++ 52 files changed, 5345 insertions(+), 323 deletions(-) delete mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/config.py delete mode 100644 datadog_checks_dev/tests/tooling/commands/validate/test_config.py create mode 100644 ddev/src/ddev/cli/validate/config.py create mode 100644 ddev/src/ddev/validation/config_spec/__init__.py create mode 100644 ddev/src/ddev/validation/config_spec/config_block.py create mode 100644 ddev/src/ddev/validation/config_spec/utils.py create mode 100644 ddev/src/ddev/validation/config_spec/validator.py create mode 100644 ddev/src/ddev/validation/config_spec/validator_errors.py create mode 100644 ddev/src/ddev/validation/configuration/__init__.py create mode 100644 ddev/src/ddev/validation/configuration/constants.py create mode 100644 ddev/src/ddev/validation/configuration/consumers/__init__.py create mode 100644 ddev/src/ddev/validation/configuration/consumers/example.py create mode 100644 ddev/src/ddev/validation/configuration/consumers/model/__init__.py create mode 100644 ddev/src/ddev/validation/configuration/consumers/model/model_consumer.py create mode 100644 ddev/src/ddev/validation/configuration/consumers/model/model_file.py create mode 100644 ddev/src/ddev/validation/configuration/consumers/model/model_info.py create mode 100644 ddev/src/ddev/validation/configuration/consumers/openapi_document.py create mode 100644 ddev/src/ddev/validation/configuration/core.py create mode 100644 ddev/src/ddev/validation/configuration/spec.py create mode 100644 ddev/src/ddev/validation/configuration/template.py create mode 100644 ddev/src/ddev/validation/configuration/templates/ad_identifiers.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/common/perf_counters.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/init_config.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/init_config/db.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/init_config/default.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/init_config/http.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/init_config/jmx.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/init_config/openmetrics.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/init_config/openmetrics_legacy.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/init_config/openmetrics_legacy_base create mode 100644 ddev/src/ddev/validation/configuration/templates/init_config/perf_counters.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/init_config/service.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/init_config/tags.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/instances.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/instances/all_integrations.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/instances/db.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/instances/default.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/instances/global.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/instances/http.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/instances/jmx.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/instances/openmetrics.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/instances/openmetrics_legacy.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/instances/openmetrics_legacy_base.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/instances/pdh_legacy.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/instances/perf_counters.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/instances/service.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/instances/tags.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/instances/tls.yaml create mode 100644 ddev/src/ddev/validation/configuration/templates/logs.yaml create mode 100644 ddev/tests/cli/validate/test_config.py diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/__init__.py b/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/__init__.py index afb0e185282b9..f7872fd215ba2 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/__init__.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/__init__.py @@ -7,7 +7,6 @@ from .agent_reqs import agent_reqs from .agent_signature import legacy_signature from .codeowners import codeowners -from .config import config from .dashboards import dashboards from .dep import dep from .eula import eula @@ -25,7 +24,6 @@ ALL_COMMANDS = ( agent_reqs, codeowners, - config, dashboards, dep, eula, diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/config.py b/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/config.py deleted file mode 100644 index 87a6e22233ce0..0000000000000 --- a/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/config.py +++ /dev/null @@ -1,247 +0,0 @@ -# (C) Datadog, Inc. 2018-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) -import difflib -import os -import re - -import click -import yaml - -from datadog_checks.dev.fs import basepath, file_exists, path_exists, path_join, read_file, write_file -from datadog_checks.dev.tooling.commands.console import ( - CONTEXT_SETTINGS, - abort, - annotate_error, - echo_failure, - echo_info, - echo_success, - echo_waiting, - echo_warning, -) -from datadog_checks.dev.tooling.config_validator.validator import validate_config -from datadog_checks.dev.tooling.config_validator.validator_errors import SEVERITY_ERROR, SEVERITY_WARNING -from datadog_checks.dev.tooling.configuration import ConfigSpec -from datadog_checks.dev.tooling.configuration.consumers import ExampleConsumer -from datadog_checks.dev.tooling.constants import get_root -from datadog_checks.dev.tooling.testing import process_checks_option -from datadog_checks.dev.tooling.utils import ( - complete_valid_checks, - get_config_files, - get_data_directory, - get_version_string, -) - -FILE_INDENT = ' ' * 8 - -IGNORE_DEFAULT_INSTANCE = {'ceph', 'dotnetclr', 'gunicorn', 'marathon', 'pgbouncer', 'process', 'supervisord'} - - -@click.command(context_settings=CONTEXT_SETTINGS, short_help='Validate default configuration files') -@click.argument('check', shell_complete=complete_valid_checks, required=False) -@click.option('--sync', '-s', is_flag=True, help='Generate example configuration files based on specifications') -@click.option('--verbose', '-v', is_flag=True, help='Verbose mode') -@click.pass_context -def config(ctx, check, sync, verbose): - """Validate default configuration files. - - If `check` is specified, only the check will be validated, if check value is 'changed' will only apply to changed - checks, an 'all' or empty `check` value will validate all README files. - """ - - repo_choice = ctx.obj['repo_choice'] - if repo_choice == 'agent': - checks = ['agent'] - else: - checks = process_checks_option(check, source='valid_checks', extend_changed=True) - - is_core_check = ctx.obj['repo_choice'] == 'core' - - files_failed = {} - files_warned = {} - file_counter = [] - - echo_waiting(f'Validating default configuration files for {len(checks)} checks...') - for check in checks: - if check in ( - 'ddev', - 'datadog_checks_dev', - 'datadog_checks_base', - 'datadog_checks_dependency_provider', - 'datadog_checks_downloader', - ): - echo_info(f'Skipping {check}, it does not need an Agent-level config.') - continue - check_display_queue = [] - - spec_file_path = path_join(get_root(), check, 'assets', 'configuration', 'spec.yaml') - if not file_exists(spec_file_path): - example_location = get_data_directory(check) - - # If there's an example file in core and no spec file, we should fail - if is_core_check and path_exists(example_location) and len(os.listdir(example_location)) > 0: - file_counter.append(None) - files_failed[spec_file_path] = True - - check_display_queue.append( - lambda spec_file_path=spec_file_path, check=check: echo_failure( - f"Did not find spec file {spec_file_path} for check {check}" - ) - ) - - validate_config_legacy(check, check_display_queue, files_failed, files_warned, file_counter) - if verbose: - check_display_queue.append(lambda: echo_warning('No spec found', indent=True)) - if check_display_queue: - echo_info(f'{check}:') - for display in check_display_queue: - display() - continue - - file_counter.append(None) - - # source is the default file name - if check == 'agent': - source = 'datadog' - version = None - else: - source = check - version = get_version_string(check) - - spec_file_content = read_file(spec_file_path) - - if not validate_default_template(spec_file_content): - message = "Missing default template in init_config or instances section" - files_failed[spec_file_path] = True - check_display_queue.append(lambda message=message, **kwargs: echo_failure(message, **kwargs)) - annotate_error(spec_file_path, message) - - spec = ConfigSpec(spec_file_content, source=source, version=version) - spec.load() - if spec.errors: - files_failed[spec_file_path] = True - for error in spec.errors: - check_display_queue.append(lambda error=error, **kwargs: echo_failure(error, **kwargs)) - else: - example_location = get_data_directory(check) - example_consumer = ExampleConsumer(spec.data) - for example_file, (contents, errors) in example_consumer.render().items(): - file_counter.append(None) - example_file_path = path_join(example_location, example_file) - if errors: - files_failed[example_file_path] = True - for error in errors: - check_display_queue.append(lambda error=error, **kwargs: echo_failure(error, **kwargs)) - else: - if not file_exists(example_file_path) or read_file(example_file_path) != contents: - if sync: - echo_info(f"Writing config file to `{example_file_path}`") - write_file(example_file_path, contents) - else: - files_failed[example_file_path] = True - message = f'File `{example_file}` is not in sync, run "ddev validate config {check} -s"' - if file_exists(example_file_path): - example_file = read_file(example_file_path) - for diff_line in difflib.context_diff( - example_file.splitlines(), contents.splitlines(), "current", "expected" - ): - message += f'\n{diff_line}' - check_display_queue.append( - lambda message=message, **kwargs: echo_failure(message, **kwargs) - ) - annotate_error(example_file_path, message) - - if check_display_queue or verbose: - echo_info(f'{check}:') - if verbose: - check_display_queue.append(lambda **kwargs: echo_info('Valid spec', **kwargs)) - for display in check_display_queue: - display(indent=True) - - num_files = len(file_counter) - files_failed = len(files_failed) - files_warned = len(files_warned) - files_passed = num_files - (files_failed + files_warned) - - if files_failed or files_warned: - click.echo() - - if files_failed: - echo_failure(f'Files with errors: {files_failed}') - - if files_warned: - echo_warning(f'Files with warnings: {files_warned}') - - if files_passed: - if files_failed or files_warned: - echo_success(f'Files valid: {files_passed}') - else: - echo_success(f'All {num_files} configuration files are valid!') - - if files_failed: - abort() - - -def validate_default_template(spec_file): - if 'template: init_config' not in spec_file or 'template: instances' not in spec_file: - # This config spec does not have init_config or instances - return True - - templates = { - 'intances': [f'template: init_config/{t}' for t in ['default', 'openmetrics_legacy', 'openmetrics', 'jmx']], - 'init_config': [f'template: init_config/{t}' for t in ['default', 'openmetrics_legacy', 'openmetrics', 'jmx']], - } - # We want both instances and init_config to have at least one template present. - return all(any(re.search(t, spec_file) for t in tpls) for tpls in templates) - - -def validate_config_legacy(check, check_display_queue, files_failed, files_warned, file_counter): - config_files = get_config_files(check) - for config_file in config_files: - file_counter.append(None) - file_name = basepath(config_file) - try: - file_data = read_file(config_file) - config_data = yaml.safe_load(file_data) - except Exception as e: - files_failed[config_file] = True - - # We must convert to text here to free Exception object before it goes out of scope - error = str(e) - - check_display_queue.append(lambda file_name=file_name: echo_info(f'{file_name}:', indent=True)) - check_display_queue.append(lambda: echo_failure('Invalid YAML -', indent=FILE_INDENT)) - check_display_queue.append(lambda error=error: echo_info(error, indent=FILE_INDENT * 2)) - continue - - file_display_queue = [] - errors = validate_config(file_data) - for err in errors: - err_msg = str(err) - if err.severity == SEVERITY_ERROR: - file_display_queue.append(lambda x=err_msg: echo_failure(x, indent=FILE_INDENT)) - files_failed[config_file] = True - elif err.severity == SEVERITY_WARNING: - file_display_queue.append(lambda x=err_msg: echo_warning(x, indent=FILE_INDENT)) - files_warned[config_file] = True - else: - file_display_queue.append(lambda x=err_msg: echo_info(x, indent=FILE_INDENT)) - - # Verify there is an `instances` section - if 'instances' not in config_data: - files_failed[config_file] = True - message = 'Missing `instances` section' - file_display_queue.append(lambda message=message: echo_failure(message, indent=FILE_INDENT)) - annotate_error(file_name, message) - # Verify there is a default instance - else: - instances = config_data['instances'] - if check not in IGNORE_DEFAULT_INSTANCE and not isinstance(instances, list): - files_failed[config_file] = True - message = 'No default instance' - file_display_queue.append(lambda message=message: echo_failure(message, indent=FILE_INDENT)) - annotate_error(file_name, message) - - if file_display_queue: - check_display_queue.append(lambda x=file_name: echo_info(f'{x}:', indent=True)) - check_display_queue.extend(file_display_queue) diff --git a/datadog_checks_dev/tests/tooling/commands/validate/test_config.py b/datadog_checks_dev/tests/tooling/commands/validate/test_config.py deleted file mode 100644 index 010997b7121a2..0000000000000 --- a/datadog_checks_dev/tests/tooling/commands/validate/test_config.py +++ /dev/null @@ -1,73 +0,0 @@ -# (C) Datadog, Inc. 2023-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) - -import os -import shutil -import sys - -import pytest -from click.testing import CliRunner - -from datadog_checks.dev import run_command - - -@pytest.mark.parametrize( - 'repo,expect_failure', - [ - ("core", True), - ("extras", False), - ("marketplace", False), - ("internal", False), - ], -) -def test_validate_config_spec_file_mandatory_in_core(repo, expect_failure): - runner = CliRunner() - - with runner.isolated_filesystem(): - # Generate the check structure - working_repo = 'integrations-{}'.format(repo) - shutil.copytree( - os.path.dirname(os.path.realpath(__file__)) + "/data/my_check", "./{}/my_check".format(working_repo) - ) - os.chdir(working_repo) - os.remove("my_check/assets/configuration/spec.yaml") - - result = run_command( - [sys.executable, '-m', 'datadog_checks.dev', '--here', 'validate', 'config', 'my_check'], - capture=True, - ) - - if expect_failure: - assert 1 == result.code - assert 'Files with errors: 1' in result.stdout - else: - assert 0 == result.code - - assert 'Validating default configuration files for 1 checks...' in result.stdout - assert 'my_check:' in result.stdout - assert 'Did not find spec file' in result.stdout - assert '' == result.stderr - - -@pytest.mark.parametrize( - 'check', - [ - 'ddev', - 'datadog_checks_dev', - 'datadog_checks_base', - 'datadog_checks_dependency_provider', - 'datadog_checks_downloader', - ], -) -def test_configless_check(check): - """ - Test that a check without a config just passes. - """ - - result = run_command( - [sys.executable, '-m', 'datadog_checks.dev', '--here', 'validate', 'config', check], - capture=True, - ) - assert result.code == 0 - assert f'Skipping {check}, it does not need an Agent-level config.' in result.stdout diff --git a/ddev/src/ddev/cli/validate/__init__.py b/ddev/src/ddev/cli/validate/__init__.py index a711e7d149f35..2efb541b92534 100644 --- a/ddev/src/ddev/cli/validate/__init__.py +++ b/ddev/src/ddev/cli/validate/__init__.py @@ -5,7 +5,6 @@ from datadog_checks.dev.tooling.commands.validate.agent_reqs import agent_reqs from datadog_checks.dev.tooling.commands.validate.agent_signature import legacy_signature from datadog_checks.dev.tooling.commands.validate.codeowners import codeowners -from datadog_checks.dev.tooling.commands.validate.config import config from datadog_checks.dev.tooling.commands.validate.dashboards import dashboards from datadog_checks.dev.tooling.commands.validate.dep import dep from datadog_checks.dev.tooling.commands.validate.eula import eula @@ -22,6 +21,7 @@ from ddev.cli.validate.all import all from ddev.cli.validate.ci import ci +from ddev.cli.validate.config import config from ddev.cli.validate.http import http from ddev.cli.validate.labeler import labeler from ddev.cli.validate.licenses import licenses diff --git a/ddev/src/ddev/cli/validate/config.py b/ddev/src/ddev/cli/validate/config.py new file mode 100644 index 0000000000000..1695d6fe307a1 --- /dev/null +++ b/ddev/src/ddev/cli/validate/config.py @@ -0,0 +1,303 @@ +# (C) Datadog, Inc. 2018-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + +import difflib +import os +import re +from typing import TYPE_CHECKING + +import click + +if TYPE_CHECKING: + from ddev.cli.application import Application + from ddev.integration.core import Integration + +DEFAULT_INDENT = ' ' * 4 +FILE_INDENT = ' ' * 8 + +IGNORE_DEFAULT_INSTANCE = {'ceph', 'dotnetclr', 'gunicorn', 'marathon', 'pgbouncer', 'process', 'supervisord'} + +CONFIGLESS_CHECKS = { + 'ddev', + 'datadog_checks_dev', + 'datadog_checks_base', + 'datadog_checks_dependency_provider', + 'datadog_checks_downloader', +} + + +def _validate_default_template(spec_file: str) -> bool: + if 'template: init_config' not in spec_file or 'template: instances' not in spec_file: + return True + + templates = { + 'intances': [f'template: init_config/{t}' for t in ['default', 'openmetrics_legacy', 'openmetrics', 'jmx']], + 'init_config': [f'template: init_config/{t}' for t in ['default', 'openmetrics_legacy', 'openmetrics', 'jmx']], + } + return all(any(re.search(t, spec_file) for t in tpls) for tpls in templates) + + +def _data_directory(app: Application, check_name: str): + """Return the directory where rendered example files live for the given check name.""" + if check_name == 'agent': + return app.repo.path / 'pkg' / 'config' + return app.repo.path / check_name / 'datadog_checks' / check_name / 'data' + + +def _spec_file_path(app: Application, check_name: str): + if check_name == 'agent': + return app.repo.path / 'pkg' / 'config' / 'spec.yaml' + return app.repo.path / check_name / 'assets' / 'configuration' / 'spec.yaml' + + +def _check_version(integration: Integration | None, check_name: str) -> str | None: + """Return the version string used as a default in the spec, mirroring legacy behaviour.""" + if check_name == 'agent': + return None + if integration is None: + return None + about_file = integration.package_directory / '__about__.py' + if not about_file.is_file(): + return None + match = re.search(r'^__version__ *= *(?:[\'"])(.+?)(?:[\'"])', about_file.read_text(), re.MULTILINE) + return match.group(1) if match else None + + +def _legacy_config_files(app: Application, check_name: str) -> list: + """Return any legacy free-form config files that exist for the check.""" + if check_name == 'agent': + config_template = app.repo.path / 'pkg' / 'config' / 'config_template.yaml' + return [config_template] if config_template.is_file() else [] + + if check_name in CONFIGLESS_CHECKS: + return [] + + data_dir = app.repo.path / check_name / 'datadog_checks' / check_name / 'data' + candidates = [ + data_dir / 'auto_conf.yaml', + data_dir / 'conf.yaml.default', + data_dir / 'conf.yaml.example', + ] + return sorted([path for path in candidates if path.is_file()]) + + +def _validate_config_legacy( + app: Application, + check_name: str, + check_display_queue: list, + files_failed: dict, + files_warned: dict, + file_counter: list, +) -> None: + import yaml + + from ddev.validation.config_spec.validator import validate_config as validate_config_yaml + from ddev.validation.config_spec.validator_errors import SEVERITY_ERROR, SEVERITY_WARNING + + for config_file in _legacy_config_files(app, check_name): + file_counter.append(None) + file_name = config_file.name + try: + file_data = config_file.read_text() + config_data = yaml.safe_load(file_data) + except Exception as e: + files_failed[str(config_file)] = True + error = str(e) + + check_display_queue.append(lambda file_name=file_name: app.display_info(f'{file_name}:', indent=DEFAULT_INDENT)) + check_display_queue.append(lambda: app.display_error('Invalid YAML -', indent=FILE_INDENT)) + check_display_queue.append(lambda error=error: app.display_info(error, indent=FILE_INDENT * 2)) + continue + + file_display_queue = [] + errors = validate_config_yaml(file_data) + for err in errors: + err_msg = str(err) + if err.severity == SEVERITY_ERROR: + file_display_queue.append(lambda x=err_msg: app.display_error(x, indent=FILE_INDENT)) + files_failed[str(config_file)] = True + elif err.severity == SEVERITY_WARNING: + file_display_queue.append(lambda x=err_msg: app.display_warning(x, indent=FILE_INDENT)) + files_warned[str(config_file)] = True + else: + file_display_queue.append(lambda x=err_msg: app.display_info(x, indent=FILE_INDENT)) + + if 'instances' not in (config_data or {}): + files_failed[str(config_file)] = True + message = 'Missing `instances` section' + file_display_queue.append(lambda message=message: app.display_error(message, indent=FILE_INDENT)) + else: + instances = config_data['instances'] + if check_name not in IGNORE_DEFAULT_INSTANCE and not isinstance(instances, list): + files_failed[str(config_file)] = True + message = 'No default instance' + file_display_queue.append(lambda message=message: app.display_error(message, indent=FILE_INDENT)) + + if file_display_queue: + check_display_queue.append(lambda x=file_name: app.display_info(f'{x}:', indent=DEFAULT_INDENT)) + check_display_queue.extend(file_display_queue) + + +def _iter_target_checks(app: Application, check: str | None) -> list[str]: + """Resolve the user input into the list of check names to validate, mirroring legacy semantics.""" + if app.repo.name == 'agent': + return ['agent'] + + selection: tuple[str, ...] + if check is None or check.lower() == 'all': + selection = () + names = sorted({integration.name for integration in app.repo.integrations.iter_all(selection)}) + elif check.lower() == 'changed': + changed = sorted({integration.name for integration in app.repo.integrations.iter_changed_code()}) + valid = {integration.name for integration in app.repo.integrations.iter_all(())} + names = [name for name in changed if name in valid] + if 'datadog_checks_dev' in names or 'datadog_checks_base' in names: + names = sorted(valid) + else: + names = [check] + + return names + + +@click.command(short_help='Validate default configuration files') +@click.argument('check', required=False) +@click.option('--sync', '-s', is_flag=True, help='Generate example configuration files based on specifications') +@click.option('--verbose', '-v', is_flag=True, help='Verbose mode') +@click.pass_context +def config(ctx: click.Context, check: str | None, sync: bool, verbose: bool) -> None: + """Validate default configuration files. + + If `check` is specified, only the check will be validated, if check value is 'changed' will only apply to changed + checks, an 'all' or empty `check` value will validate all README files. + """ + from ddev.validation.configuration import ConfigSpec + from ddev.validation.configuration.consumers import ExampleConsumer + + app: Application = ctx.obj + checks = _iter_target_checks(app, check) + is_core_check = app.repo.name == 'core' + + files_failed: dict = {} + files_warned: dict = {} + file_counter: list = [] + + app.display_waiting(f'Validating default configuration files for {len(checks)} checks...') + for check_name in checks: + if check_name in CONFIGLESS_CHECKS: + app.display_info(f'Skipping {check_name}, it does not need an Agent-level config.') + continue + + check_display_queue: list = [] + + spec_file_path = _spec_file_path(app, check_name) + if not spec_file_path.is_file(): + example_location = _data_directory(app, check_name) + + # If there's an example file in core and no spec file, we should fail + if is_core_check and example_location.is_dir() and len(os.listdir(example_location)) > 0: + file_counter.append(None) + files_failed[str(spec_file_path)] = True + + check_display_queue.append( + lambda spec_file_path=spec_file_path, check_name=check_name: app.display_error( + f"Did not find spec file {spec_file_path} for check {check_name}" + ) + ) + + _validate_config_legacy(app, check_name, check_display_queue, files_failed, files_warned, file_counter) + if verbose: + check_display_queue.append(lambda: app.display_warning('No spec found', indent=DEFAULT_INDENT)) + if check_display_queue: + app.display_info(f'{check_name}:') + for display in check_display_queue: + display() + continue + + file_counter.append(None) + + if check_name == 'agent': + source = 'datadog' + integration = None + else: + source = check_name + try: + integration = app.repo.integrations.get(check_name) + except OSError: + integration = None + version = _check_version(integration, check_name) + + spec_file_content = spec_file_path.read_text() + + if not _validate_default_template(spec_file_content): + message = "Missing default template in init_config or instances section" + files_failed[str(spec_file_path)] = True + check_display_queue.append(lambda message=message, **kwargs: app.display_error(message, **kwargs)) + + spec = ConfigSpec(spec_file_content, source=source, version=version) + spec.load() + if spec.errors: + files_failed[str(spec_file_path)] = True + for error in spec.errors: + check_display_queue.append(lambda error=error, **kwargs: app.display_error(error, **kwargs)) + else: + example_location = _data_directory(app, check_name) + example_consumer = ExampleConsumer(spec.data) + for example_file, (contents, errors) in example_consumer.render().items(): + file_counter.append(None) + example_file_path = example_location / example_file + if errors: + files_failed[str(example_file_path)] = True + for error in errors: + check_display_queue.append(lambda error=error, **kwargs: app.display_error(error, **kwargs)) + else: + if not example_file_path.is_file() or example_file_path.read_text() != contents: + if sync: + app.display_info(f"Writing config file to `{example_file_path}`") + example_file_path.parent.mkdir(parents=True, exist_ok=True) + example_file_path.write_text(contents) + else: + files_failed[str(example_file_path)] = True + message = ( + f'File `{example_file}` is not in sync, run "ddev validate config {check_name} -s"' + ) + if example_file_path.is_file(): + current_file = example_file_path.read_text() + for diff_line in difflib.context_diff( + current_file.splitlines(), contents.splitlines(), "current", "expected" + ): + message += f'\n{diff_line}' + check_display_queue.append( + lambda message=message, **kwargs: app.display_error(message, **kwargs) + ) + + if check_display_queue or verbose: + app.display_info(f'{check_name}:') + if verbose: + check_display_queue.append(lambda **kwargs: app.display_info('Valid spec', **kwargs)) + for display in check_display_queue: + display(indent=DEFAULT_INDENT) + + num_files = len(file_counter) + files_failed_count = len(files_failed) + files_warned_count = len(files_warned) + files_passed = num_files - (files_failed_count + files_warned_count) + + if files_failed_count or files_warned_count: + click.echo() + + if files_failed_count: + app.display_error(f'Files with errors: {files_failed_count}') + + if files_warned_count: + app.display_warning(f'Files with warnings: {files_warned_count}') + + if files_passed: + if files_failed_count or files_warned_count: + app.display_success(f'Files valid: {files_passed}') + else: + app.display_success(f'All {num_files} configuration files are valid!') + + if files_failed_count: + app.abort() diff --git a/ddev/src/ddev/validation/config_spec/__init__.py b/ddev/src/ddev/validation/config_spec/__init__.py new file mode 100644 index 0000000000000..a149d84c103fd --- /dev/null +++ b/ddev/src/ddev/validation/config_spec/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2019-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/ddev/src/ddev/validation/config_spec/config_block.py b/ddev/src/ddev/validation/config_spec/config_block.py new file mode 100644 index 0000000000000..65ffdd00c8edb --- /dev/null +++ b/ddev/src/ddev/validation/config_spec/config_block.py @@ -0,0 +1,431 @@ +# (C) Datadog, Inc. 2019-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import re + +from .utils import get_indent, is_at_least_indented, is_blank, is_exactly_indented +from .validator_errors import SEVERITY_WARNING, ValidatorError + +# The maximum length authorized for comments. It accounts for text after '##' and does not apply to the @param +# declaration +MAX_COMMENT_LENGTH = 120 + +# The list of accepted type as regex. Anything starting with list is allowed. +ACCEPTED_VAR_TYPE_REGEX = [ + "^boolean$", + "^string$", + "^integer$", + "^double$", + "^float$", + "^object$", + "^list.*$", + "^dictionary$", +] + +# Regex used to parse the fields of the '## @param' declaration +PARAM_REGEX = "^## @param ([a-zA-Z0-9_\\-]+) +- (.+) - (required|optional)( - default: .*)?$" + +# Regex to parse a 'key: value' item +VAR_REGEX = "^(# |)(\\w+): ([a-zA-Z0-9_><]+)$" + +# Regex to identify an object (it ends with a colon). No comment allowed after the colon but the whole line +# can be a comment. +OBJECT_REGEX = "^(# |)([a-zA-Z0-9_\\-]+): *$" + +# Regex to identify a comment. Note that 'INDENT' needs to be updated with the number of spaces expected +COMMENT_REGEX = "^ {INDENT}##(.*)$" + +# Regex to match any comment with no respect to indentation +INCORRECTLY_INDENTED_COMMENT_REGEX = "^ *##(.*)$" + + +class ParamProperties: + """Class to represent a parameter declared using the '@param' annotation""" + + def __init__(self, var_name, type_name, required=True, default_value=None): + self.var_name = var_name + self.type_name = type_name + self.required = required + self.default_value = default_value + + @classmethod + def parse_from_string(cls, idx, config_lines, indent, errors): + if not is_exactly_indented(config_lines[idx], indent): + errors.append(ValidatorError("Content is not correctly indented", idx)) + return None + + current_line = config_lines[idx][indent:] + + if not current_line.startswith("## @param"): + errors.append(ValidatorError("Expecting @param declaration", idx)) + return None + + m = re.match(PARAM_REGEX, current_line) + if m is None: + errors.append(ValidatorError("Invalid @param declaration", idx)) + return None + + if m.group(3) == "optional": + def_val = None + if m.group(4): # If there is a default value + def_val = m.group(4)[12:] + + return cls(m.group(1), m.group(2), False, def_val) + + return cls(m.group(1), m.group(2), True) + + +class ConfigBlock: + """Class to represent a 'configuration block' which is the definition of a variable, with the @param annotation, + its description and its content. + """ + + def __init__(self, param_prop, description, line, length, should_recurse=False): + """ + :param param_prop: ParamProperties instance + :param description: The description preceding the variable declaration + :param line: The first line of this block + :param length: The number of lines this block takes. (see should_recurse) + :param should_recurse: Whether or not the content of the block must be analyzed. See _should_recurse function + """ + self.param_prop = param_prop + self.description = description + self.line = line + self.length = length + self.should_recurse = should_recurse + + def validate(self, errors): + """Method to return a list of errors and warnings about an already parsed block""" + self._validate_description(errors) + self._validate_type(errors) + + def _validate_description(self, errors): + """Check if the block has a description and lines are not too long.""" + if self.description is None: + # There was a error reading description, which has already been recorded. + return + + if self.description.strip() == '': + param_name = self.param_prop.var_name + errors.append(ValidatorError(f"Empty description for {param_name}", self.line, SEVERITY_WARNING)) + + for i, line in enumerate(self.description.splitlines()): + if len(line) > MAX_COMMENT_LENGTH and not line.endswith("#noqa"): + err_string = f"Description too long [{line[:30]}...] ({len(line)}/{MAX_COMMENT_LENGTH})" + errors.append(ValidatorError(err_string, self.line + i + 1)) + + def _validate_type(self, errors): + """Check if the block has a valid type""" + if self.param_prop is None: + return + + for regex in ACCEPTED_VAR_TYPE_REGEX: + if re.match(regex, self.param_prop.type_name): + break + else: + errors.append(ValidatorError(f"Type {self.param_prop.type_name} is not accepted", self.line)) + + @classmethod + def parse_from_strings(cls, start, config_lines, indent, errors): + """Main method used to parse a block starting at line 'start' with a given indentation.""" + idx = start + + # Let's first check if the block is a simple comment. If so, let's return and go to the next block + if _is_comment(start, config_lines, indent, errors): + comment, end = _parse_comment(start, config_lines) + return cls(None, comment, start, end - start) + + # Let's get to the end of the block supposing it is formatted correctly (@param line, description, empty + # comment, then the actual content). If it fails, let's ignore the whole block and its potential + # sub-blocks. + end = _get_end_of_param_declaration_block(start, len(config_lines), config_lines, indent, errors) + if end is None: + default_end = _get_next_block_in_case_of_failure(start, config_lines) + return cls(None, None, start, default_end - start) + + block_len = end - start + + # Parsing the @param line + param_prop = ParamProperties.parse_from_string(idx, config_lines, indent, errors) + if param_prop is None: + return cls(None, None, start, block_len) + + # If var is indicated as list, recompute end of block knowing it is a list + if param_prop.type_name.startswith('list'): + end = _get_end_of_param_declaration_block( + start, len(config_lines), config_lines, indent, errors, is_list=True + ) + if end is None: + default_end = _get_next_block_in_case_of_failure(start, config_lines) + return cls(None, None, start, default_end - start) + block_len = end - start + + # Parsing the description + idx += 1 + description, idx = _parse_description(idx, end, config_lines, indent, errors) + if idx is None: + return cls(param_prop, None, start, block_len) + + # We recurse if the variable is an object and contains at least one member with description + is_object = _is_object(idx, config_lines, indent, param_prop, errors) + if not is_object: + return cls(param_prop, description, start, block_len) + + should_recurse, next_block = _should_recurse(idx, config_lines, indent) + if should_recurse: + # If we recurse we use block_len, pointing to the next sub-block + return cls(param_prop, description, start, block_len, should_recurse=True) + + # If we don't recurse we use the next_object variable to point to the next block with the same or less + # indentation and thus ignore sub-blocks. + block_len = next_block - start + return cls(param_prop, description, start, block_len) + + +def _get_end_of_param_declaration_block(start, end, config_lines, indent, errors, is_list=False): + """Here we suppose the config block is correctly formatted (@param, description, empty comment then the + actual content) and try to return the line of any data coming after. In case of an object we point to its first + member. In case of a list or a simple variable we point to the next element. + """ + + if not is_exactly_indented(config_lines[start], indent): + other_indent = get_indent(config_lines[start]) + errors.append(ValidatorError(f"Unexpected indentation, expecting {indent} not {other_indent}", start)) + return None + + if not config_lines[start].startswith(' ' * indent + "## @param"): + errors.append(ValidatorError("Expecting @param declaration", start)) + return None + + # Going through the description + idx = start + 1 + while idx < end: + if is_blank(config_lines[idx]): + errors.append(ValidatorError("Blank line when reading description", idx)) + return None + if not is_exactly_indented(config_lines[idx], indent): + other_indent = get_indent(config_lines[idx]) + err_string = f"Unexpected indentation, expecting {indent} not {other_indent}" + errors.append(ValidatorError(err_string, idx)) + return None + + current_line = config_lines[idx][indent:] + if current_line[0:2] == '##': + # This is still the description + idx += 1 + elif current_line[0] == '#': + # This is the last line of the description + idx += 1 + break + else: + errors.append(ValidatorError(f"Cannot find end of block starting at line {start}", idx)) + return None + + # Now analyze the actual content + idx += 1 + while idx < end: + if is_blank(config_lines[idx]): + idx += 1 + continue + if not is_at_least_indented(config_lines[idx], indent): + # We reached a surrounding block, thus this block has ended. + break + + current_line = config_lines[idx][indent:] + + if current_line[0:2] == '# ': + # Commented data + idx += 1 + elif current_line.lstrip(' ').startswith('- '): + # The object is a list of things, let's get to the end of that object + is_list = True + idx += 1 + elif is_list and is_at_least_indented(config_lines[idx], indent + 1): + idx += 1 + else: + break + + return idx + + +def _parse_description(idx, end, config_lines, indent, errors): + """With idx pointing to the beginning of a description, it reads line by line and build the string. It returns + the string and a pointer to the end of the description. + """ + + description = [] + while idx < end: + if not is_exactly_indented(config_lines[idx], indent): + if is_blank(config_lines[idx]): + errors.append(ValidatorError("Reached end of description without marker", idx)) + else: + errors.append(ValidatorError("Description is not correctly indented", idx)) + return None, None + current_line = config_lines[idx][indent:] + + if current_line.startswith("##"): + description.append(current_line[2:]) + idx += 1 + elif current_line[0] == '#' and is_blank(current_line[1:]): + # End of description + description = '\n'.join(description) + idx += 1 + break + else: + description = '\n'.join(description) + errors.append(ValidatorError("Reached end of description without marker", idx)) + return description, None + else: + # EOF reached without end of description + errors.append(ValidatorError("Reached EOF while reading description", idx)) + return None, None + + return description, idx + + +def _is_object(idx, config_lines, indent, param_prop, errors): + """With idx pointing to the beginning of a 'key: value' variable declaration, this function returns true + if the variable is declared as an object in the @param declaration and if value is not on the same line. + """ + + if not is_exactly_indented(config_lines[idx], indent): + errors.append(ValidatorError("Content is not correctly indented", idx)) + return False + + current_line = config_lines[idx][indent:] + + if param_prop.type_name == 'object': + # The variable to be parsed is an object and thus requires to go recursively + if re.match(OBJECT_REGEX, current_line) is None: + err_string = f"Parameter {param_prop.var_name} is declared as object but isn't one" + errors.append(ValidatorError(err_string, idx)) + return False + return True + + return False + + +def _should_recurse(start, config_lines, initial_indent): + """Sometimes object are not expected to document every parameter and contain all the description at the top. + This is valid, therefore we should not recurse and analyze what's next after the whole object. Returns whether or + not to recurse as long a pointer to the next same-level element (not sub-object) + """ + idx = start + 1 + end = len(config_lines) + + # First get indent of the first member + while idx < end: + current_line = config_lines[idx] + if is_blank(current_line) or current_line[initial_indent:].startswith('# '): + idx += 1 + continue + break + else: + # Reached EOF + return False, idx + + current_line = config_lines[idx] + first_member_indent = get_indent(current_line) + + if first_member_indent <= initial_indent: + # We reached the end of object without any data + return False, idx + + # Second check if there is description inside the object + while idx < end: + current_line = config_lines[idx] + if is_blank(current_line): + idx += 1 + continue + if get_indent(current_line) <= initial_indent: + # We reached end of object + return False, idx + if current_line.startswith(' ' * first_member_indent + "## @param"): + # We found some description with the correct indentation. Let's recurse in that object + return True, None + idx += 1 + + return False, idx + + +def _is_comment(start, config_lines, indent, errors): + """Returns true if the block starting at line 'start' only contains comments and no @param declaration nor + setting any variable. + """ + + regex = COMMENT_REGEX.replace('INDENT', str(indent)) + idx = start + end = len(config_lines) + if "## @param" in config_lines[idx]: + # If we see @param, no matter how correctly formatted it is, we expect it to be a param declaration + return False + + while idx < end: + current_line = config_lines[idx] + if re.match(regex, current_line): + idx += 1 + continue + elif is_blank(current_line): + # End of block with only ## comments, the whole block is indeed only a comment + return True + elif re.match(INCORRECTLY_INDENTED_COMMENT_REGEX, current_line): + # This is still a comment but incorrectly indented + errors.append(ValidatorError("Comment block incorrectly indented", idx)) + idx += 1 + continue + else: + return False + + return True + + +def _parse_comment(start, config_lines): + """If we are reading a multiline comment, let's read it and find the index of the next block to be parsed. + Return a tuple of the comment and the index of the next block""" + idx = start + end = len(config_lines) + comment = [] + # Get to end of multiline comment + while idx < end: + current_line = config_lines[idx] + if re.match(INCORRECTLY_INDENTED_COMMENT_REGEX, current_line): + content = current_line.lstrip(' ')[2:] + comment.append(content) + idx += 1 + continue + break + + comment = '\n'.join(comment) + # Get to next data + while idx < end: + current_line = config_lines[idx] + if is_blank(current_line): + idx += 1 + continue + break + return comment, idx + + +def _get_next_block_in_case_of_failure(start, config_lines): + """If we can't read a param declaration correctly, let's default to this method to continue parsing + from the next block""" + idx = start + end = len(config_lines) + initial_indent = get_indent(config_lines[idx]) + + # First get rid of blank lines and comments + while idx < end: + current_line = config_lines[idx] + if is_blank(current_line) or current_line.lstrip(' ').startswith('#'): + idx += 1 + continue + break + + # Now wait for the first same-level comment to get back on track + while idx < end: + current_line = config_lines[idx] + if is_exactly_indented(current_line, initial_indent) and current_line[initial_indent] == '#': + break + idx += 1 + + return idx diff --git a/ddev/src/ddev/validation/config_spec/utils.py b/ddev/src/ddev/validation/config_spec/utils.py new file mode 100644 index 0000000000000..4e70152ca7375 --- /dev/null +++ b/ddev/src/ddev/validation/config_spec/utils.py @@ -0,0 +1,81 @@ +# (C) Datadog, Inc. 2019-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + + +def get_end_of_part(config_lines, start_line, indent=None): + """Returns the end of a yaml block for a given start line. + + If the indent parameter is unspecified, the start_line needs to be + the start of a block. The indentation of that line will be used. + You can specify the indent parameter to reach the end of the block + with the provided indentation. + + :param config_lines: The yaml data as an array of strings + :param start_line: Where to start looking for the end of a block + :param indent: Optional, see description + :return: The line at which the block ends. + None if line is empty or if there is no block + """ + i = start_line + end = len(config_lines) + if i >= end: + return end + + if is_blank(config_lines[i]): + return None + + if indent is None: + indent = get_indent(config_lines[i]) + + i += 1 + has_seen_data = False + while i < end: + if is_blank(config_lines[i]): + i += 1 + elif is_at_least_indented(config_lines[i], indent + 1): + i += 1 + has_seen_data = True + else: + end = i + + if not has_seen_data: + return None + + return end + + +def get_indent(line): + """Returns the indentation of a given line. According to YAML specs, indentation for list item does + not start at the hyphen + """ + if is_blank(line): + return 0 + + stripped = line.lstrip(' ') + if stripped.startswith('- '): + stripped = stripped[2:].lstrip(' ') + # This is a list item + + return len(line) - len(stripped) + + +def is_blank(line): + """Returns true if the line is empty. + A single hyphen in a line is considered as blank + """ + return line.isspace() or not line or line.strip() == '-' + + +def is_exactly_indented(line, indent): + """Returns true if the line has the expected indentation. Empty line has no indentation""" + if is_blank(line): + return False + return get_indent(line) == indent + + +def is_at_least_indented(line, indent): + """Returns true if the line has at least the expected indentation. Empty line has no indentation""" + if is_blank(line): + return False + return get_indent(line) >= indent diff --git a/ddev/src/ddev/validation/config_spec/validator.py b/ddev/src/ddev/validation/config_spec/validator.py new file mode 100644 index 0000000000000..40fa105c89cf4 --- /dev/null +++ b/ddev/src/ddev/validation/config_spec/validator.py @@ -0,0 +1,160 @@ +# (C) Datadog, Inc. 2019-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from .config_block import ConfigBlock +from .utils import get_end_of_part, get_indent, is_at_least_indented, is_blank +from .validator_errors import SEVERITY_WARNING, ValidatorError + + +def validate_config(config): + """Function used to validate a whole yaml configuration file. Will check if there are both the init_config and + the instances sections. And will parse using the _parse_for_config_blocks function + """ + errors = [] + blocks = [] # This will store ConfigBlocks as a tree + config_lines = config.split('\n') + + init_config_line = -1 + instances_line = -1 + for i, line in enumerate(config_lines): + if line.startswith("init_config:"): + init_config_line = i + if line != "init_config:": + errors.append(ValidatorError("Expected no data after ':'", i, SEVERITY_WARNING)) + if line.startswith("instances:"): + instances_line = i + if line != "instances:": + errors.append(ValidatorError("Expected no data after ':'", i, SEVERITY_WARNING)) + + if init_config_line == -1: + errors.append(ValidatorError("Missing `init_config` section", None)) + return errors + + if instances_line == -1: + errors.append(ValidatorError("Missing `instances` section", None)) + return errors + + # parse init_config data + blocks.append(_parse_init_config(config_lines, init_config_line, errors)) + + # parse instances data + instances_end = get_end_of_part(config_lines, instances_line) + if instances_end is None: + errors.append(ValidatorError("Malformed file, cannot find end of part 'instances'", instances_line)) + return errors + blocks.append(_parse_for_config_blocks(config_lines, instances_line + 1, instances_end, errors)) + + _check_no_duplicate_names(blocks, errors) + _validate_blocks(blocks, errors) + return errors + + +def _parse_init_config(config_lines, init_config_start_line, errors): + """Function used to parse the init_config section and return the list of 'ConfigBlock' + It first checks if the section contains data or not. If not, it returns an empty list. Otherwise + it will use the _parse_for_config_blocks function to parse it between the beginning and the end of the part + """ + blocks = [] + idx = init_config_start_line + 1 + + # Check if the init_config part contains data or not + while idx < len(config_lines): + current_line = config_lines[idx] + if is_blank(current_line): + idx += 1 + continue + elif is_at_least_indented(current_line, 1): + # There is data in 'init_config' + break + else: + # There is no data, do not try to parse the init_config + return blocks + + end = get_end_of_part(config_lines, init_config_start_line) + if end is None: + errors.append(ValidatorError("Malformed file, cannot find end of part 'init_config'", init_config_start_line)) + return blocks + + return _parse_for_config_blocks(config_lines, init_config_start_line + 1, end, errors) + + +def _parse_for_config_blocks(config_lines, start, end, errors): + """The function basically do all the work. It reads the config from start, removes blank lines first then when + it first sees data, it sets the 'indent' variable once for all. All blocks read in a given function call must + have the same indentation. Sub-blocks are parsed recursively and thus the 'indent' variable is given a new value. + Once a block is parsed the function will either recurse if the block requires it (see ConfigBlock), or it will go + to the next block and iterate. + """ + idx = start + blocks = [] + + # Go to the first line with data (see 'is_blank') + while idx < end: + if is_blank(config_lines[idx]): + idx += 1 + continue + break + else: + return blocks + + # All blocks of a same level must have the same indentation. Let's use the first one to compare them + indent = get_indent(config_lines[idx]) + + while idx < end: + current_line = config_lines[idx] + if is_blank(current_line): + idx += 1 + continue + if not is_at_least_indented(current_line, indent): + errors.append(ValidatorError("Content is not correctly indented - skipping rest of file", idx)) + # File will not be able to be parsed correctly if indentation is wrong + return blocks + + cfg_block = ConfigBlock.parse_from_strings(idx, config_lines, indent, errors) + # Even if there has been an issue when parsing the block, cfg_block.length always point to another block + # (either a sub-block or not) or to EOF + idx += cfg_block.length + blocks.append(cfg_block) + + if cfg_block.should_recurse: + # new_end points to the next line having the same indent as the cfg_block + new_end = get_end_of_part(config_lines, idx, indent=indent) + if new_end is None: + block_name = cfg_block.param_prop.var_name if cfg_block.param_prop else "?" + err_string = f"The object {block_name} cannot be parsed correctly, check indentation" + errors.append(ValidatorError(err_string, idx)) + return blocks + if new_end > end: + new_end = end + blocks += _parse_for_config_blocks(config_lines, idx, new_end, errors) + idx = new_end + + return blocks + + +def _check_no_duplicate_names(blocks, errors): + """blocks contains ConfigBlocks as a tree. This function makes sure that each yaml object has no duplicates + variables and return a list of errors to be displayed if duplicates are found. The @param declaration needs to + be there for this to correctly identify a variable. + """ + same_level_blocks = [b for b in blocks if isinstance(b, ConfigBlock)] + names_list = [b.param_prop.var_name for b in same_level_blocks if b.param_prop] + duplicates = {x for x in names_list if names_list.count(x) > 1} + for dup in duplicates: + errors.append(ValidatorError(f"Duplicate variable with name {dup}", None)) + + sub_lists_of_other_blocks = [b for b in blocks if isinstance(b, list)] + for l in sub_lists_of_other_blocks: + _check_no_duplicate_names(l, errors) + + +def _validate_blocks(blocks, errors): + """blocks contains ConfigBlocks as a tree. This function iterate over it to run the validate method on each + ConfigBlock and append errors to the provided array if needed. + """ + leaves = [b for b in blocks if isinstance(b, ConfigBlock)] + for b in leaves: + b.validate(errors) + nodes = [b for b in blocks if isinstance(b, list)] + for n in nodes: + _validate_blocks(n, errors) diff --git a/ddev/src/ddev/validation/config_spec/validator_errors.py b/ddev/src/ddev/validation/config_spec/validator_errors.py new file mode 100644 index 0000000000000..70716fb6ec48d --- /dev/null +++ b/ddev/src/ddev/validation/config_spec/validator_errors.py @@ -0,0 +1,20 @@ +# (C) Datadog, Inc. 2019-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +SEVERITY_ERROR = 0 +SEVERITY_WARNING = 1 + + +class ValidatorError: + def __init__(self, error_str, line_number, severity=SEVERITY_ERROR): + self.error_str = error_str + self.severity = severity + self.line_number = line_number + + def __repr__(self): + if self.line_number is None: + return self.error_str + return f"(L{self.line_number + 1}) {self.error_str}" + + def __str__(self): + return self.__repr__() diff --git a/ddev/src/ddev/validation/configuration/__init__.py b/ddev/src/ddev/validation/configuration/__init__.py new file mode 100644 index 0000000000000..1b8c78395b9a8 --- /dev/null +++ b/ddev/src/ddev/validation/configuration/__init__.py @@ -0,0 +1,6 @@ +# (C) Datadog, Inc. 2019-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from ddev.validation.configuration.core import ConfigSpec + +__all__ = ['ConfigSpec'] diff --git a/ddev/src/ddev/validation/configuration/constants.py b/ddev/src/ddev/validation/configuration/constants.py new file mode 100644 index 0000000000000..92943e7fc98f3 --- /dev/null +++ b/ddev/src/ddev/validation/configuration/constants.py @@ -0,0 +1,50 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# https://swagger.io/docs/specification/data-models/data-types/ +OPENAPI_DATA_TYPES = { + 'array', + 'boolean', + 'integer', + 'number', + 'object', + 'string', +} + +# https://spec.openapis.org/oas/v3.0.3#properties +OPENAPI_SCHEMA_PROPERTIES = { + 'additionalProperties', + 'allOf', + 'anyOf', + 'default', + 'description', + 'enum', + 'exclusiveMaximum', + 'exclusiveMinimum', + 'format', + 'items', + 'maxItems', + 'maxLength', + 'maxProperties', + 'maximum', + 'minItems', + 'minLength', + 'minimum', + 'multipleOf', + 'not', + 'oneOf', + 'pattern', + 'properties', + 'required', + 'title', + 'type', +} + +# Allowed metadata values for integration `formats` annotations in spec options/properties. +ALLOWED_FORMATS = { + 'java_jvm_options', + 'path', + 'port', + 'url', +} diff --git a/ddev/src/ddev/validation/configuration/consumers/__init__.py b/ddev/src/ddev/validation/configuration/consumers/__init__.py new file mode 100644 index 0000000000000..ca2a859fbf6ec --- /dev/null +++ b/ddev/src/ddev/validation/configuration/consumers/__init__.py @@ -0,0 +1,7 @@ +# (C) Datadog, Inc. 2019-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from ddev.validation.configuration.consumers.example import ExampleConsumer +from ddev.validation.configuration.consumers.model.model_consumer import ModelConsumer + +__all__ = ['ExampleConsumer', 'ModelConsumer'] diff --git a/ddev/src/ddev/validation/configuration/consumers/example.py b/ddev/src/ddev/validation/configuration/consumers/example.py new file mode 100644 index 0000000000000..08dfc7a61c846 --- /dev/null +++ b/ddev/src/ddev/validation/configuration/consumers/example.py @@ -0,0 +1,330 @@ +# (C) Datadog, Inc. 2019-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from io import StringIO + +import yaml + +from ddev.validation.configuration.constants import OPENAPI_SCHEMA_PROPERTIES + +ALLOWED_OPTION_FIELDS = { + 'name', + 'description', + 'required', + 'hidden', + 'display_priority', + 'deprecation', + 'legacy', + 'multiple', + 'multiple_instances_defined', + 'metadata_tags', + 'options', + 'value', + 'secret', + 'enabled', + 'example', + 'template', + 'overrides', + 'fleet_configurable', + 'formats', +} +ALLOWED_VALUE_FIELDS = OPENAPI_SCHEMA_PROPERTIES | { + 'example', + 'display_default', + 'compact_example', + 'require_trusted_provider', +} +DESCRIPTION_LINE_LENGTH_LIMIT = 120 + + +class OptionWriter(object): + def __init__(self): + self.writer = StringIO() + self.errors = [] + + def write(self, *strings): + for s in strings: + self.writer.write(s) + + def new_error(self, s): + self.errors.append(s) + + @property + def contents(self): + return self.writer.getvalue() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.writer.close() + + +def construct_yaml(obj, **kwargs): + kwargs.setdefault('default_flow_style', False) + return yaml.safe_dump(obj, sort_keys=False, **kwargs) + + +def value_type_string(value): + if 'anyOf' in value: + return ' or '.join(value_type_string(type_data) for type_data in value['anyOf']) + else: + value_type = value['type'] + if value_type == 'object': + return 'mapping' + elif value_type == 'array': + items = value['items'] + if 'anyOf' in items: + return f'(list of {value_type_string(items)})' + else: + item_type = items['type'] + if item_type == 'object': + return 'list of mappings' + elif item_type == 'array': + return 'list of lists' + else: + return f'list of {item_type}s' + else: + return value_type + + +def option_enabled(option): + if 'enabled' in option: + return bool(option['enabled']) + + return option['required'] + + +def validate_fields( + fields_dict: dict[str, any], option_name: str, allowed_fields: set[str], field_level: str, writer: OptionWriter +): + invalid_fields = [field for field in fields_dict if field not in allowed_fields] + + if invalid_fields: + invalid_fields_str = '\n'.join(f" - {field!r}" for field in invalid_fields) + writer.new_error( + f"Option name {option_name!r} contains the following invalid {field_level} fields:\n{invalid_fields_str}" + ) + + +def write_description(option, writer, indent, option_type): + description = option['description'] + deprecation = option['deprecation'] + if deprecation: + description += '\n\n<<< DEPRECATED >>>\n\n' + for key, info in option['deprecation'].items(): + key_part = f'{key}: ' + info_pad = ' ' * len(key_part) + description += key_part + + for i, line in enumerate(info.splitlines()): + if i > 0: + description += info_pad + + description += f'{line}\n' + + for line in description.splitlines(): + if line: + line = f'{indent}## {line}' + if len(line) > DESCRIPTION_LINE_LENGTH_LIMIT and ' /noqa' not in line: + extra_characters = len(line) - DESCRIPTION_LINE_LENGTH_LIMIT + writer.new_error( + 'Description line length of {} `{}` was over the limit by {} character{}'.format( + option_type, option['name'], extra_characters, 's' if extra_characters > 1 else '' + ) + ) + elif ' /noqa' in line: + line = line.replace(' /noqa', '') + writer.write(line) + else: + writer.write(indent, '##') + + writer.write('\n') + + +def write_option(option, writer, indent='', start_list=False): + option_name = option['name'] + + validate_fields(option, option_name, ALLOWED_OPTION_FIELDS, 'option-level', writer) + + if 'value' in option: + value = option['value'] + required = option['required'] + writer.write( + indent, + '## @param ', + option_name, + ' - ', + value_type_string(value), + ' - ', + 'required' if required else 'optional', + ) + + validate_fields(value, option_name, ALLOWED_VALUE_FIELDS, 'value-level', writer) + + example = value.get('example') + example_type = type(example) + if not required: + default = value.get('display_default', value.get('default')) + if default is not None: + default_type = type(default) + if default is not None and str(default).lower() != 'none': + if default_type is str: + writer.write(' - default: ', default) + elif default_type is bool: + writer.write(' - default: ', 'true' if default else 'false') + else: + writer.write(' - default: ', repr(default)) + elif 'display_default' not in value or 'default' in value: + if example_type is bool: + writer.write(' - default: ', 'true' if example else 'false') + elif example_type in (int, float): + writer.write(' - default: ', str(example)) + elif example_type is str: + if example and not (example[0] == '<' and example[-1] == '>'): + writer.write(' - default: ', example) + + writer.write('\n') + + write_description(option, writer, indent, 'option') + + writer.write(indent, '#\n') + + if start_list: + option_yaml = construct_yaml([{option_name: example}]) + indent = indent[:-2] + else: + if value.get('compact_example') and example_type is list: + option_yaml_lines = [f'{option_name}:'] + for item in example: + # Solitary strings are given an ellipsis after, prevent that + if isinstance(item, str): + compacted_item = construct_yaml(item, default_flow_style=True, default_style='"') + else: + # Compact examples should stay on one line to prevent weird line wraps. + compacted_item = construct_yaml(item, default_flow_style=True, width=float('inf')) + + option_yaml_lines.append(f'- {compacted_item.strip()}') + + option_yaml = '\n'.join(option_yaml_lines) + else: + option_yaml = construct_yaml({option_name: example}) + + example_indent = ' ' if example_type is list and example else '' + for i, line in enumerate(option_yaml.splitlines()): + writer.write(indent) + if not option_enabled(option): + writer.write('# ') + + if i > 0: + writer.write(example_indent) + + writer.write(line, '\n') + else: + write_description(option, writer, indent, 'section') + + writer.write(indent, '#\n') + + if 'options' in option: + multiple = option['multiple'] + multiple_instances_defined = option.get('multiple_instances_defined') + + if not option_enabled(option): + writer.write(indent, '# ', option_name, ':', '\n') + else: + writer.write(indent, option_name, ':', '\n') + + if multiple and multiple_instances_defined: + for idx, instance in enumerate(option['options']): + if idx == 0: + start_list = True + write_sub_option( + instance, writer, indent, multiple, include_top_description=True, start_list=start_list + ) + else: + write_sub_option(option, writer, indent, multiple) + + # For sections that prefer to document everything in the description, like `logs` + else: + example = option.get('example', [] if option.get('multiple', False) else {}) + option_yaml = construct_yaml({option_name: example}) + + example_indent = ' ' if type(example) is list and example else '' + for i, line in enumerate(option_yaml.splitlines()): + if not option_enabled(option): + writer.write(indent, '# ') + + if i > 0: + writer.write(example_indent) + + writer.write(line, '\n') + + +def write_sub_option(option, writer, indent, multiple, include_top_description=False, start_list=False): + options = sorted(option['options'], key=lambda opt: -opt['display_priority']) + next_indent = indent + ' ' + + if options: + for i, opt in enumerate(options): + if opt['hidden']: + continue + + writer.write('\n') + if i == 0 and multiple: + if include_top_description and option.get('description'): + write_description(option, writer, next_indent, 'option') + if start_list and 'options' in opt: + writer.write(indent, ' -\n') + if option_enabled(opt): + write_option(opt, writer, next_indent, start_list=True) + else: + writer.write(next_indent[:-2], '-\n') + write_option(opt, writer, next_indent) + else: + write_option(opt, writer, next_indent) + elif multiple: + writer.write('\n', next_indent[:-2], '- {}\n') + + +class ExampleConsumer(object): + def __init__(self, spec): + self.spec = spec + + def render(self): + files = {} + + for file in self.spec['files']: + with OptionWriter() as writer: + options = file['options'] + num_options = len(options) + for i, option in enumerate(options, 1): + if option['hidden']: + continue + + write_option(option, writer) + + # No new line necessary after the last option + if i != num_options: + writer.write('\n') + + if writer.errors: + has_option_level_errors = any('option-level' in error for error in writer.errors) + has_value_level_errors = any('value-level' in error for error in writer.errors) + + if has_option_level_errors or has_value_level_errors: + valid_fields = [] + + if has_option_level_errors: + fields_list = '\n'.join(f" - {field!r}" for field in sorted(ALLOWED_OPTION_FIELDS)) + valid_fields.append(f"Option-level fields must be one of the following:\n{fields_list}") + + if has_value_level_errors: + fields_list = '\n'.join(f" - {field!r}" for field in sorted(ALLOWED_VALUE_FIELDS)) + valid_fields.append(f"Value-level fields must be one of the following:\n{fields_list}") + + if valid_fields: + writer.errors.append('\n'.join(valid_fields)) + + files[file['example_name']] = (writer.contents, writer.errors) + + return files diff --git a/ddev/src/ddev/validation/configuration/consumers/model/__init__.py b/ddev/src/ddev/validation/configuration/consumers/model/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/ddev/src/ddev/validation/configuration/consumers/model/model_consumer.py b/ddev/src/ddev/validation/configuration/consumers/model/model_consumer.py new file mode 100644 index 0000000000000..4f9bd3cd44073 --- /dev/null +++ b/ddev/src/ddev/validation/configuration/consumers/model/model_consumer.py @@ -0,0 +1,260 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import warnings +from pathlib import Path +from typing import Dict, List, Tuple + +import yaml +from datamodel_code_generator import DataModelType +from datamodel_code_generator.format import CodeFormatter, PythonVersion +from datamodel_code_generator.model import get_data_model_types +from datamodel_code_generator.parser import LiteralType +from datamodel_code_generator.parser.openapi import OpenAPIParser + +from ddev.validation.configuration.consumers.model.model_file import build_model_file +from ddev.validation.configuration.consumers.model.model_info import ModelInfo +from ddev.validation.configuration.consumers.openapi_document import build_openapi_document + +PYTHON_VERSION = PythonVersion.PY_39 + +VALIDATORS_DOCUMENTATION = '''# Here you can include additional config validators or transformers +# +# def initialize_instance(values, **kwargs): +# if 'my_option' not in values and 'my_legacy_option' in values: +# values['my_option'] = values['my_legacy_option'] +# if values.get('my_number') > 10: +# raise ValueError('my_number max value is 10, got %s' % str(values.get('my_number'))) +# +# return values +''' + + +class ModelConsumer: + def __init__(self, spec: dict, code_formatter: CodeFormatter = None): + self.spec = spec + self.code_formatter = code_formatter or self.create_code_formatter() + + def render(self) -> Dict[str, Dict[str, str]]: + """ + Returns a dictionary containing for each spec file the list of rendered models + """ + # { spec_file_name: {model_file_name: model_file_contents } + rendered_files = {} + + for spec_file in self.spec['files']: + # (, (, )) + model_files: Dict[str, Tuple[str, List[str]]] = {} + # Contains pairs of model_id and schema_name. eg ('instance', 'InstanceConfig') + package_info: List[Tuple[str, str]] = [] + model_info = ModelInfo() + + # Sections are init_config and instances + for section in sorted(spec_file['options'], key=lambda s: s['name']): + ( + section_package_info, + section_model_files, + section_model_info, + ) = self._process_section(section) + package_info.extend(section_package_info) + model_files.update(section_model_files) + model_info.update(section_model_info) + + # Logs-only integrations + if not model_files: + continue + + model_files.update(self._build_model_files(model_info, package_info)) + rendered_files[spec_file['name']] = {file_name: model_files[file_name] for file_name in sorted(model_files)} + + return rendered_files + + def _process_section(self, section) -> (List[Tuple[str, str]], dict, ModelInfo): + # Values to return + # [(model_id, schema_name)] + package_info: List[Tuple[str, str]] = [] + # { model_file_name: (model_file_contents, errors) } + model_files: Dict[str, Tuple[str, List[str]]] = {} + model_info = ModelInfo() + + errors: List[str] = [] + section_name = section['name'] + if section_name == 'init_config': + model_id = 'shared' + model_file_name = f'{model_id}.py' + schema_name = 'SharedConfig' + elif section_name == 'instances': + model_id = 'instance' + model_file_name = f'{model_id}.py' + schema_name = 'InstanceConfig' + if section['multiple_instances_defined']: + section = self._merge_instances(section, errors) + # Skip anything checks don't use directly + else: + return ( + package_info, + model_files, + model_info, + ) + + package_info.append((model_id, schema_name)) + (section_openapi_document, model_info) = build_openapi_document(section, model_id, schema_name, errors) + + model_types = get_data_model_types(DataModelType.PydanticV2BaseModel, target_python_version=PYTHON_VERSION) + try: + section_parser = OpenAPIParser( + yaml.safe_dump(section_openapi_document), + data_model_type=model_types.data_model, + data_model_root_type=model_types.root_model, + data_model_field_type=model_types.field_model, + data_type_manager_type=model_types.data_type_manager, + dump_resolve_reference_action=model_types.dump_resolve_reference_action, + enum_field_as_literal=LiteralType.All, + encoding='utf-8', + enable_faux_immutability=True, + use_standard_collections=True, + strip_default_none=True, + # https://github.com/koxudaxi/datamodel-code-generator/pull/173 + field_constraints=True, + ) + # https://github.com/pydantic/pydantic/issues/6422 + # https://github.com/pydantic/pydantic/issues/6467#issuecomment-1623680485 + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + parsed_section = section_parser.parse() + except Exception as e: + errors.append(f'Error parsing the OpenAPI schema `{schema_name}`: {e}') + model_files[model_file_name] = ('', errors) + return ( + package_info, + model_files, + model_info, + ) + + model_file_contents = build_model_file( + parsed_section, + model_id, + section_name, + model_info, + self.code_formatter, + ) + # instance.py or shared.py + model_files[model_file_name] = (model_file_contents, errors) + return ( + package_info, + model_files, + model_info, + ) + + def _build_model_files( + self, model_info: ModelInfo, package_info: List[Tuple[str, str]] + ) -> Dict[str, Tuple[str, List]]: + """Builds the model files others than instace.py and shared.py + In particular it builds, if relevant: + - defaults.py + - deprecations.py + - __init__.py + - validators.py + Returns a Dict[ file_name, Tuple[file_contents, List[errors])] + """ + model_files = {} + if model_info.defaults_file_lines: + defaults_file_contents = self._build_defaults_file(model_info) + model_files['defaults.py'] = (f'\n{defaults_file_contents}', []) + + if model_info.deprecation_data: + deprecations_file_contents = self._build_deprecation_file(model_info.deprecation_data) + model_files['deprecations.py'] = (deprecations_file_contents, []) + + package_root_lines = ModelConsumer._build_package_root(package_info) + model_files['__init__.py'] = ('\n'.join(package_root_lines), []) + + # Custom + model_files['validators.py'] = (VALIDATORS_DOCUMENTATION, []) + return model_files + + def _merge_instances(self, section: dict, errors: List[str]) -> dict: + """Builds a new, unified, section by merging multiple + :param section: The section to unify + :param errors: The list where to add errors + """ + new_section = { + 'name': section['name'], + 'options': [], + } + # If one of these option is different for 2 options with the same name, an error is raised + required_consistent_options = ['required', 'deprecation', 'metadata_tags'] + # Cache the option index to ease option checking before merging + options_name_idx = {} + + for instance in section['options']: + for opt in instance['options']: + if options_name_idx.get(opt['name']) is not None: + cached_opt = new_section['options'][options_name_idx[opt['name']]] + + for opt_name in required_consistent_options: + if cached_opt[opt_name] != opt[opt_name]: + errors.append( + f'Options {cached_opt} and {opt} have a different value for attribute `{opt_name}`' + ) + if cached_opt['value']['type'] != opt['value']['type']: + errors.append(f'Options {cached_opt} and {opt} have a different value for attribute `type`') + + else: + new_section['options'].append(opt) + options_name_idx[opt['name']] = len(new_section['options']) - 1 + + return new_section + + @staticmethod + def create_code_formatter(repo_path: str | Path | None = None): + if repo_path is None: + return CodeFormatter(PYTHON_VERSION) + path = Path(repo_path) + return CodeFormatter(PYTHON_VERSION, settings_path=path if path.is_dir() else None) + + def _build_deprecation_file(self, deprecation_data): + file_needs_formatting = False + deprecations_file_lines = [] + for model_id, deprecations in deprecation_data.items(): + deprecations_file_lines.append('') + deprecations_file_lines.append('') + deprecations_file_lines.append(f'def {model_id}():') + deprecations_file_lines.append(f' return {deprecations!r}') + if len(deprecations_file_lines[-1]) > 120: + file_needs_formatting = True + + deprecations_file_lines.append('') + deprecations_file_contents = '\n'.join(deprecations_file_lines) + if file_needs_formatting: + deprecations_file_contents = self.code_formatter.apply_black(deprecations_file_contents) + return deprecations_file_contents + + @staticmethod + def _build_package_root(package_info): + package_info.sort() + package_root_lines = [] + for model_id, schema_name in package_info: + package_root_lines.append(f'from .{model_id} import {schema_name}') + + package_root_lines.append('') + package_root_lines.append('') + package_root_lines.append('class ConfigMixin:') + for model_id, schema_name in package_info: + package_root_lines.append(f' _config_model_{model_id}: {schema_name}') + for model_id, schema_name in package_info: + property_name = 'config' if model_id == 'instance' else f'{model_id}_config' + package_root_lines.append('') + package_root_lines.append(' @property') + package_root_lines.append(f' def {property_name}(self) -> {schema_name}:') + package_root_lines.append(f' return self._config_model_{model_id}') + + package_root_lines.append('') + return package_root_lines + + def _build_defaults_file(self, model_info: ModelInfo): + model_info.defaults_file_lines.append('') + defaults_file_contents = '\n'.join(model_info.defaults_file_lines) + if model_info.defaults_file_needs_value_normalization: + defaults_file_contents = self.code_formatter.apply_black(defaults_file_contents) + return defaults_file_contents diff --git a/ddev/src/ddev/validation/configuration/consumers/model/model_file.py b/ddev/src/ddev/validation/configuration/consumers/model/model_file.py new file mode 100644 index 0000000000000..980107b17c05b --- /dev/null +++ b/ddev/src/ddev/validation/configuration/consumers/model/model_file.py @@ -0,0 +1,209 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from datamodel_code_generator.format import CodeFormatter + +from ddev.validation.configuration.consumers.model.model_info import ModelInfo + + +def build_model_file( + parsed_document: str, + model_id: str, + section_name: str, + model_info: ModelInfo, + code_formatter: CodeFormatter, +): + """ + :param parsed_document: OpenApi parsed document + :param model_id: instance or shared + :param section_name: init or instances + :param model_info: Information to build the model file + :param code_formatter: + """ + # Whether or not there are options with default values + options_with_defaults = len(model_info.defaults_file_lines) > 0 + model_file_lines = parsed_document.splitlines() + _add_imports(model_file_lines, options_with_defaults, len(model_info.deprecation_data)) + _fix_types(model_file_lines) + + # Add constant for secure fields requiring trusted provider validation + if model_info.require_trusted_providers: + _add_secure_fields_constant(model_file_lines, model_info.require_trusted_providers) + + if model_id in model_info.deprecation_data: + model_file_lines += _define_deprecation_functions(model_id, section_name) + + model_file_lines += _define_validator_functions( + model_id, model_info.validator_data, options_with_defaults, bool(model_info.require_trusted_providers) + ) + + config_lines = [] + for i, line in enumerate(model_file_lines): + if line.startswith(' model_config = ConfigDict('): + config_lines.append(i) + + extra_config_lines = [' arbitrary_types_allowed=True,'] + for i, line_number in enumerate(config_lines): + index = line_number + (len(extra_config_lines) * i) + 1 + for line in extra_config_lines: + model_file_lines.insert(index, line) + + if i == len(config_lines) - 1: + model_file_lines.insert(index, ' validate_default=True,') + + model_file_lines.append('') + model_file_contents = '\n'.join(model_file_lines) + if any(len(line) > 120 for line in model_file_lines): + model_file_contents = code_formatter.apply_black(model_file_contents) + return model_file_contents + + +def _add_imports(model_file_lines, need_defaults, need_deprecations): + import_lines = [] + mapping_found = False + typing_location = -1 + + for i, line in enumerate(model_file_lines): + if line.startswith('from '): + import_lines.append(i) + if line.startswith('from typing '): + typing_location = i + elif 'dict[' in line: + mapping_found = True + + # pydantic imports + final_import_line = import_lines[-1] + for index in reversed(import_lines): + line = model_file_lines[index] + if line.startswith('from pydantic '): + model_file_lines[index] += ', field_validator, model_validator' + break + + if mapping_found: + if typing_location == -1: + insertion_index = import_lines[0] + 1 + model_file_lines.insert(insertion_index, 'from types import MappingProxyType') + model_file_lines.insert(insertion_index, '') + final_import_line += 2 + else: + model_file_lines.insert(typing_location, 'from types import MappingProxyType') + final_import_line += 1 + + local_imports = ['validators'] + if need_defaults: + local_imports.append('defaults') + if need_deprecations: + local_imports.append('deprecations') + + local_import_start_location = final_import_line + 1 + for line in reversed( + ( + '', + 'from datadog_checks.base.utils.functions import identity', + 'from datadog_checks.base.utils.models import validation', + '', + f'from . import {", ".join(sorted(local_imports))}', + ) + ): + model_file_lines.insert(local_import_start_location, line) + + +def _fix_types(model_file_lines): + for i, line in enumerate(model_file_lines): + line = model_file_lines[i] = line.replace('dict[', 'MappingProxyType[') + if 'list[' not in line: + continue + + buffer = bytearray() + containers = [] + + for char in line: + if char == '[': + if buffer[-4:] == b'list': + containers.append(True) + buffer[-4:] = b'tuple' + else: + containers.append(False) + elif char == ']' and containers.pop(): + buffer.extend(b', ...') + + buffer.append(ord(char)) + + model_file_lines[i] = buffer.decode('utf-8') + + +def _add_secure_fields_constant(model_file_lines, require_trusted_providers): + """Add SECURE_FIELD_NAMES constant before the first class definition.""" + class_line_index = next((i for i, line in enumerate(model_file_lines) if line.startswith('class ')), None) + if class_line_index is not None: + fields_str = ', '.join(f'{name!r}' for name in sorted(require_trusted_providers)) + model_file_lines[class_line_index:class_line_index] = [ + '', + f'SECURE_FIELD_NAMES = frozenset([{fields_str}])', + '', + ] + + +def _define_deprecation_functions(model_id, section_name): + model_file_lines = [''] + model_file_lines.append(" @model_validator(mode='before')") + model_file_lines.append(' def _handle_deprecations(cls, values, info):') + model_file_lines.append(" fields = info.context['configured_fields']") + model_file_lines.append( + f' validation.utils.handle_deprecations(' + f'{section_name!r}, deprecations.{model_id}(), fields, info.context)' + ) + model_file_lines.append(' return values') + return model_file_lines + + +def _define_validator_functions(model_id, validator_data, need_defaults, has_require_trusted_providers=False): + model_file_lines = [''] + model_file_lines.append(" @model_validator(mode='before')") + model_file_lines.append(' def _initial_validation(cls, values):') + model_file_lines.append( + f" return validation.core.initialize_config(" + f"getattr(validators, 'initialize_{model_id}', identity)(values))" + ) + + model_file_lines.append('') + model_file_lines.append(" @field_validator('*', mode='before')") + model_file_lines.append(' def _validate(cls, value, info):') + model_file_lines.append(' field = cls.model_fields[info.field_name]') + model_file_lines.append(' field_name = field.alias or info.field_name') + model_file_lines.append(" if field_name in info.context['configured_fields']:") + model_file_lines.append( + f" value = getattr(validators, f'{model_id}_{{info.field_name}}', identity)(value, field=field)" + ) + + for option_name, import_paths in sorted(validator_data): + model_file_lines.append('') + model_file_lines.append(f' if info.field_name == {option_name!r}:') + for import_path in import_paths: + model_file_lines.append(f' value = validators.{import_path}(value, field=field)') + + # Add security validation for fields requiring trusted provider + if has_require_trusted_providers: + model_file_lines.append('') + model_file_lines.append(' if info.field_name in SECURE_FIELD_NAMES:') + model_file_lines.append( + " validation.security.check_field_trusted_provider(" + "info.field_name, value, info.context.get('security_config'))" + ) + + if need_defaults: + model_file_lines.append(' else:') + model_file_lines.append( + f" value = getattr(defaults, f'{model_id}_{{info.field_name}}', lambda: value)()" + ) + + model_file_lines.append('') + model_file_lines.append(' return validation.utils.make_immutable(value)') + + model_file_lines.append('') + model_file_lines.append(" @model_validator(mode='after')") + model_file_lines.append(' def _final_validation(cls, model):') + model_file_lines.append( + f" return validation.core.check_model(getattr(validators, 'check_{model_id}', identity)(model))" + ) + return model_file_lines diff --git a/ddev/src/ddev/validation/configuration/consumers/model/model_info.py b/ddev/src/ddev/validation/configuration/consumers/model/model_info.py new file mode 100644 index 0000000000000..4ba48b167784e --- /dev/null +++ b/ddev/src/ddev/validation/configuration/consumers/model/model_info.py @@ -0,0 +1,97 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from collections import defaultdict + +# Singleton allowing `None` to be a valid default value +NO_DEFAULT = object() + + +class ModelInfo: + def __init__(self): + self.defaults_file_needs_value_normalization = False + # Contains function definitions as text for options that are optional so they have a default value + self.defaults_file_lines: list[str] = [] + self.validator_data = [] + self.deprecation_data = defaultdict(dict) + self.require_trusted_providers: list[str] = [] + + def update(self, section_model): + """ + Updates this model with another ModelInfo + """ + self.defaults_file_needs_value_normalization += section_model.defaults_file_needs_value_normalization + self.defaults_file_lines.extend(section_model.defaults_file_lines) + self.validator_data.extend(section_model.validator_data) + self.deprecation_data.update(section_model.deprecation_data) + self.require_trusted_providers.extend(section_model.require_trusted_providers) + + def add_type_validators(self, type_data: dict, option_name: str, normalized_option_name: str) -> list[str]: + """ + :param type_data: dict with the option type information + :param option_name: The option name + :param normalized_option_name: Normalized option name + :returns: A list of errors + """ + validator_data = [] + errors = [] + validators = type_data.pop('validators', []) + if not isinstance(validators, list): + errors.append(f'Config spec property `{option_name}.value.validators` must be an array') + elif validators: + for i, import_path in enumerate(validators, 1): + if not isinstance(import_path, str): + errors.append( + f'Entry #{i} of config spec property `{option_name}.value.validators` must be a string' + ) + break + else: + validator_data.append((normalized_option_name, validators)) + self.validator_data += validator_data + return errors + + def add_deprecation(self, model_id: str, option_name: str, deprecation_info: dict): + """ + :param model_id: 'shared' or 'instance' Used for the function name + :param option_name: The option name + :deprecation_info: Deprecation option information + """ + self.deprecation_data[model_id][option_name] = deprecation_info + + def add_defaults(self, model_id: str, normalized_option_name: str, type_data: dict): + """ + :param model_id: 'shared' or 'instance' Used for the function name + :param normalized_option_name: Used to build the function name + :type_data: dict containing all the relevant information to build the function + """ + default_value = self._get_default_value(type_data) + if default_value is not NO_DEFAULT: + self.defaults_file_needs_value_normalization = True + self.defaults_file_lines.extend(['', '', f'def {model_id}_{normalized_option_name}():']) + self.defaults_file_lines.append(f' return {default_value!r}') + + @staticmethod + def _get_default_value(type_data): + if 'default' in type_data: + return type_data['default'] + elif 'display_default' in type_data: + display_default = type_data['display_default'] + if display_default is None: + return NO_DEFAULT + else: + return display_default + elif 'type' not in type_data or type_data['type'] in ('array', 'object'): + return NO_DEFAULT + + example = type_data['example'] + if type_data['type'] == 'string': + if ModelInfo._example_looks_informative(example): + return NO_DEFAULT + elif isinstance(example, str): + return NO_DEFAULT + + return example + + @staticmethod + def _example_looks_informative(example): + return '<' in example and '>' in example and example == example.upper() diff --git a/ddev/src/ddev/validation/configuration/consumers/openapi_document.py b/ddev/src/ddev/validation/configuration/consumers/openapi_document.py new file mode 100644 index 0000000000000..4a3ba29e03a52 --- /dev/null +++ b/ddev/src/ddev/validation/configuration/consumers/openapi_document.py @@ -0,0 +1,168 @@ +# (C) Datadog, Inc. 2021-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from keyword import iskeyword +from typing import List + +from pydantic import BaseModel + +from ddev.validation.configuration.constants import OPENAPI_SCHEMA_PROPERTIES +from ddev.validation.configuration.consumers.model.model_info import ModelInfo + +# We don't need any self-documenting features +ALLOWED_TYPE_FIELDS = OPENAPI_SCHEMA_PROPERTIES - {'default', 'description', 'example', 'title'} + + +def build_openapi_document(section: dict, model_id: str, schema_name: str, errors: List[str]) -> (dict, ModelInfo): + """ + :param section: The section on a config spec: ie: init_config or instances + :param model_id: The model id, which is either 'shared' or 'instance' + :param schema_name: The specific model class name which is either SharedConfig or InstanceConfig + :param errors: Array where to write error messages + :return: openapi_document, model_info + :rtype: (dict, ModelInfo) + """ + # We want to create something like: + # + # paths: + # /instance: + # get: + # responses: + # '200': + # content: + # application/json: + # schema: + # $ref: '#/components/schemas/InstanceConfig' + # components: + # schemas: + # InstanceConfig: + # required: + # - endpoint + # properties: + # endpoint: + # ... + # timeout: + # ... + # ... + openapi_document = { + 'paths': { + f'/{model_id}': { + 'get': { + 'responses': { + '200': { + 'content': {'application/json': {'schema': {'$ref': f'#/components/schemas/{schema_name}'}}} + } + } + } + } + }, + 'components': {'schemas': {}}, + } + schema = {} + options = {} + required_options = [] + + schema['properties'] = options + schema['required'] = required_options + openapi_document['components']['schemas'][schema_name] = schema + + model_info = ModelInfo() + options_seen = set() + + for section_option in sorted(section['options'], key=lambda o: (o['name'], o['hidden'])): + option_name = section_option['name'] + normalized_option_name = _normalize_option_name(option_name) + + if normalized_option_name in options_seen: + continue + else: + options_seen.add(normalized_option_name) + + type_data = _build_type_data(section_option) + if not type_data: + errors.append(f'Option `{option_name}` must have a `value` or `options` attribute') + continue + options[option_name] = type_data + + if section_option['deprecation']: + model_info.add_deprecation(model_id, option_name, section_option['deprecation']) + + if type_data and type_data.get('require_trusted_provider', False): + model_info.require_trusted_providers.append(normalized_option_name) + + if section_option['required']: + required_options.append(option_name) + else: + model_info.add_defaults(model_id, normalized_option_name, type_data) + + validator_errors = model_info.add_type_validators(type_data, option_name, normalized_option_name) + errors.extend(validator_errors) + + # Remove fields that aren't part of the OpenAPI specification + for extra_field in set(type_data) - ALLOWED_TYPE_FIELDS: + type_data.pop(extra_field, None) + + _sanitize_openapi_object_properties(type_data) + return ( + openapi_document, + model_info, + ) + + +def _normalize_option_name(option_name): + # https://github.com/koxudaxi/datamodel-code-generator/blob/0.8.3/datamodel_code_generator/model/base.py#L82-L84 + if iskeyword(option_name) or hasattr(BaseModel, option_name): + option_name += '_' + + return option_name.replace('-', '_') + + +def _build_type_data(section_option: dict) -> dict: + """ + Builds the data structure with the type information (example, default value, nested types...) + """ + type_data = None + if 'value' in section_option: + # Simple type like str, number + type_data = section_option['value'] + # Some integrations (like `mysql`) have options that are grouped under a top-level option + elif 'options' in section_option: + # Object type + nested_properties = [] + type_data = {'type': 'object', 'properties': nested_properties} + for nested_option in section_option['options']: + nested_type_data = nested_option['value'] + + # Remove fields that aren't part of the OpenAPI specification + for extra_field in set(nested_type_data) - ALLOWED_TYPE_FIELDS: + nested_type_data.pop(extra_field, None) + + nested_properties.append({'name': nested_option['name'], **nested_type_data}) + return type_data + + +def _sanitize_openapi_object_properties(value): + if 'anyOf' in value: + for data in value['anyOf']: + _sanitize_openapi_object_properties(data) + return + + value_type = value['type'] + if value_type == 'array': + _sanitize_openapi_object_properties(value['items']) + elif value_type == 'object': + spec_properties = value.pop('properties') + properties = value['properties'] = {} + + # The config spec `properties` object modifier is not a map, but rather a list of maps with a + # required `name` attribute. This is so consumers will load objects consistently regardless of + # language guarantees regarding map key order. + for spec_prop in spec_properties: + name = spec_prop.pop('name') + properties[name] = spec_prop + _sanitize_openapi_object_properties(spec_prop) + + if 'additionalProperties' in value: + additional_properties = value['additionalProperties'] + if isinstance(additional_properties, dict): + _sanitize_openapi_object_properties(additional_properties) diff --git a/ddev/src/ddev/validation/configuration/core.py b/ddev/src/ddev/validation/configuration/core.py new file mode 100644 index 0000000000000..fe18bd11b200c --- /dev/null +++ b/ddev/src/ddev/validation/configuration/core.py @@ -0,0 +1,57 @@ +# (C) Datadog, Inc. 2019-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + +import yaml + +from ddev.validation.configuration.spec import spec_validator +from ddev.validation.configuration.template import ConfigTemplates + + +class ConfigSpec: + def __init__( + self, + contents: str, + template_paths: list[str] | None = None, + source: str | None = None, + version: str | None = None, + ): + """ + Parameters: + + contents: + the raw text contents of a spec + template_paths: + a sequence of directories that will take precedence when looking for templates + source: + a textual representation of what the spec refers to, usually an integration name + version: + the version of the spec to default to if the spec does not define one + """ + self.contents = contents + self.source = source + self.version = version + self.templates = ConfigTemplates(template_paths) + self.data: dict | None = None + self.errors: list[str] = [] + + def load(self) -> None: + """ + This function de-serializes the specification and: + 1. fills in default values + 2. populates any selected templates + 3. accumulates all error/warning messages + If the `errors` attribute is empty after this is called, the `data` attribute + will be the fully resolved spec object. + """ + if self.data is not None and not self.errors: + return + + try: + self.data = yaml.safe_load(self.contents) + except Exception as e: + self.errors.append(f'{self.source}: Unable to parse the configuration specification: {e}') + return + + spec_validator(self.data, self) diff --git a/ddev/src/ddev/validation/configuration/spec.py b/ddev/src/ddev/validation/configuration/spec.py new file mode 100644 index 0000000000000..41c4699dfae11 --- /dev/null +++ b/ddev/src/ddev/validation/configuration/spec.py @@ -0,0 +1,756 @@ +# (C) Datadog, Inc. 2019-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from .constants import ALLOWED_FORMATS, OPENAPI_DATA_TYPES + + +def spec_validator(spec: dict, loader) -> None: + if not isinstance(spec, dict): + loader.errors.append(f'{loader.source}: Configuration specifications must be a mapping object') + return + + release_version = spec.setdefault('version', loader.version) + if not release_version: + loader.errors.append( + f'{loader.source}: Configuration specifications must contain a top-level `version` attribute' + ) + return + elif not isinstance(release_version, str): + loader.errors.append(f'{loader.source}: The top-level `version` attribute must be a string') + return + + if 'files' not in spec: + loader.errors.append( + f'{loader.source}: Configuration specifications must contain a top-level `files` attribute' + ) + return + + files = spec['files'] + if not isinstance(files, list): + loader.errors.append(f'{loader.source}: The top-level `files` attribute must be an array') + return + + files_validator(files, loader) + + +def files_validator(files, loader) -> None: + num_files = len(files) + file_names_origin = {} + example_file_names_origin = {} + for file_index, config_file in enumerate(files, 1): + if not isinstance(config_file, dict): + loader.errors.append(f'{loader.source}, file #{file_index}: File attribute must be a mapping object') + continue + + if 'name' not in config_file: + loader.errors.append( + '{}, file #{}: Every file must contain a `name` attribute representing the ' + 'final destination the Agent loads'.format(loader.source, file_index) + ) + continue + + file_name = config_file['name'] + if not isinstance(file_name, str): + loader.errors.append(f'{loader.source}, file #{file_index}: Attribute `name` must be a string') + continue + + if file_name in file_names_origin: + loader.errors.append( + '{}, file #{}: File name `{}` already used by file #{}'.format( + loader.source, file_index, file_name, file_names_origin[file_name] + ) + ) + else: + file_names_origin[file_name] = file_index + + if file_name == 'auto_conf.yaml' or file_name == 'conf.yaml.default': + if 'example_name' in config_file and config_file['example_name'] != file_name: + loader.errors.append( + '{}, file #{}: Example file name `{}` should be `{}`'.format( + loader.source, file_index, config_file['example_name'], file_name + ) + ) + + example_file_name = config_file.setdefault('example_name', file_name) + else: + if num_files == 1: + expected_name = f"{normalize_source_name(loader.source or 'conf')}.yaml" + if file_name != expected_name: + loader.errors.append( + '{}, file #{}: File name `{}` should be `{}`'.format( + loader.source, file_index, file_name, expected_name + ) + ) + + example_file_name = config_file.setdefault('example_name', 'conf.yaml.example') + + if not isinstance(example_file_name, str): + loader.errors.append(f'{loader.source}, file #{file_index}: Attribute `example_name` must be a string') + + if example_file_name in example_file_names_origin: + loader.errors.append( + '{}, file #{}: Example file name `{}` already used by file #{}'.format( + loader.source, file_index, example_file_name, example_file_names_origin[example_file_name] + ) + ) + else: + example_file_names_origin[example_file_name] = file_index + + if 'options' not in config_file: + loader.errors.append(f'{loader.source}, {file_name}: Every file must contain an `options` attribute') + continue + + options = config_file['options'] + if not isinstance(options, list): + loader.errors.append(f'{loader.source}, {file_name}: The `options` attribute must be an array') + continue + + options_validator(options, loader, file_name) + + +def options_validator(options, loader, file_name, *sections): + sections_display = ', '.join(sections) + if sections_display: + sections_display += ', ' + + overrides = {} + override_errors = [] + + option_names_origin = {} + hide_template = False + for option_index, option in enumerate(options, 1): + if not isinstance(option, dict): + loader.errors.append( + '{}, {}, {}option #{}: Option attribute must be a mapping object'.format( + loader.source, file_name, sections_display, option_index + ) + ) + continue + + templates_resolved = False + while 'template' in option: + hide_template = option.get('hidden', False) + + overrides.update(option.pop('overrides', {})) + try: + template = loader.templates.load(option.pop('template')) + except Exception as e: + loader.errors.append(f'{loader.source}, {file_name}, {sections_display}option #{option_index}: {e}') + break + + else: + # Handle the case where a template name is overriden + if 'name' in option: + template['name'] = option['name'] + + errors = loader.templates.apply_overrides(template, overrides) + if errors: + override_errors.append((option_index, errors)) + + if isinstance(template, dict): + template.update(option) + option = template + options[option_index - 1] = template + elif isinstance(template, list): + if template: + option = template[0] + for item_index, template_item in enumerate(template): + options.insert(option_index + item_index, template_item) + + # Delete what's at the current index + options.pop(option_index - 1) + + # Perform this check once again + if not isinstance(option, dict): + loader.errors.append( + '{}, {}, {}option #{}: Template option must be a mapping object'.format( + loader.source, file_name, sections_display, option_index + ) + ) + break + else: + loader.errors.append( + '{}, {}, {}option #{}: Template refers to an empty array'.format( + loader.source, file_name, sections_display, option_index + ) + ) + break + else: + loader.errors.append( + '{}, {}, {}option #{}: Template does not refer to a mapping object nor array'.format( + loader.source, file_name, sections_display, option_index + ) + ) + break + + # Only set upon success or if there were no templates + else: + templates_resolved = True + + if not templates_resolved: + continue + + if 'name' not in option: + loader.errors.append( + '{}, {}, {}option #{}: Every option must contain a `name` attribute'.format( + loader.source, file_name, sections_display, option_index + ) + ) + continue + + option_name = option['name'] + if not isinstance(option_name, str): + loader.errors.append( + '{}, {}, {}option #{}: Attribute `name` must be a string'.format( + loader.source, file_name, sections_display, option_index + ) + ) + + option.setdefault('hidden', hide_template) + if not isinstance(option['hidden'], bool): + loader.errors.append( + '{}, {}, {}{}: Attribute `hidden` must be true or false'.format( + loader.source, file_name, sections_display, option_name + ) + ) + + if option_name in option_names_origin: + if not option['hidden']: + loader.errors.append( + '{}, {}, {}option #{}: Option name `{}` already used by option #{}'.format( + loader.source, + file_name, + sections_display, + option_index, + option_name, + option_names_origin[option_name], + ) + ) + else: + option_names_origin[option_name] = option_index + + if 'description' not in option: + loader.errors.append( + '{}, {}, {}{}: Every option must contain a `description` attribute'.format( + loader.source, file_name, sections_display, option_name + ) + ) + continue + + description = option['description'] + if not isinstance(description, str): + loader.errors.append( + '{}, {}, {}{}: Attribute `description` must be a string'.format( + loader.source, file_name, sections_display, option_name + ) + ) + + option.setdefault('required', False) + if not isinstance(option['required'], bool): + loader.errors.append( + '{}, {}, {}{}: Attribute `required` must be true or false'.format( + loader.source, file_name, sections_display, option_name + ) + ) + + option.setdefault('display_priority', 0) + if not isinstance(option['display_priority'], int): + loader.errors.append( + '{}, {}, {}{}: Attribute `display_priority` must be an integer'.format( + loader.source, file_name, sections_display, option_name + ) + ) + + option.setdefault('deprecation', {}) + if not isinstance(option['deprecation'], dict): + loader.errors.append( + '{}, {}, {}{}: Attribute `deprecation` must be a mapping object'.format( + loader.source, file_name, sections_display, option_name + ) + ) + else: + for key, info in option['deprecation'].items(): + if not isinstance(info, str): + loader.errors.append( + '{}, {}, {}{}: Key `{}` for attribute `deprecation` must be a string'.format( + loader.source, file_name, sections_display, option_name, key + ) + ) + + option.setdefault('metadata_tags', []) + if not isinstance(option['metadata_tags'], list): + loader.errors.append( + '{}, {}, {}{}: Attribute `metadata_tags` must be an array'.format( + loader.source, file_name, sections_display, option_name + ) + ) + else: + for metadata_tag in option['metadata_tags']: + if not isinstance(metadata_tag, str): + loader.errors.append( + '{}, {}, {}{}: Attribute `metadata_tags` must only contain strings'.format( + loader.source, file_name, sections_display, option_name + ) + ) + + if 'formats' in option: + formats_validator(option['formats'], loader, file_name, sections_display, option_name) + + if 'value' in option and 'options' in option: + loader.errors.append( + '{}, {}, {}{}: An option cannot contain both `value` and `options` attributes'.format( + loader.source, file_name, sections_display, option_name + ) + ) + continue + + if 'value' in option: + value = option['value'] + if not isinstance(value, dict): + loader.errors.append( + '{}, {}, {}{}: Attribute `value` must be a mapping object'.format( + loader.source, file_name, sections_display, option_name + ) + ) + continue + + option.setdefault('secret', False) + if not isinstance(option['secret'], bool): + loader.errors.append( + '{}, {}, {}{}: Attribute `secret` must be true or false'.format( + loader.source, file_name, sections_display, option_name + ) + ) + + value_validator(value, loader, file_name, sections_display, option_name, depth=0) + elif 'options' in option: + nested_options = option['options'] + if not isinstance(nested_options, list): + loader.errors.append( + '{}, {}, {}{}: The `options` attribute must be an array'.format( + loader.source, file_name, sections_display, option_name + ) + ) + continue + + option.setdefault('multiple', False) + if not isinstance(option['multiple'], bool): + loader.errors.append( + '{}, {}, {}{}: Attribute `multiple` must be true or false'.format( + loader.source, file_name, sections_display, option_name + ) + ) + + option.setdefault('multiple_instances_defined', False) + if not isinstance(option['multiple_instances_defined'], bool): + loader.errors.append( + '{}, {}, {}{}: Attribute `multiple` must be true or false'.format( + loader.source, file_name, sections_display, option_name + ) + ) + + previous_sections = list(sections) + previous_sections.append(option_name) + options_validator(nested_options, loader, file_name, *previous_sections) + + # If there are unused overrides, add the associated error messages + if overrides: + for option_index, errors in override_errors: + error_message = '\n'.join(errors) + loader.errors.append( + f'{loader.source}, {file_name}, {sections_display}option #{option_index}: {error_message}' + ) + + +def formats_validator(formats, loader, file_name, sections_display, option_name, property_name=None): + property_context = '' + if property_name is not None: + property_context = f' for property `{property_name}`' + + if not isinstance(formats, list): + loader.errors.append( + '{}, {}, {}{}: Attribute `formats`{} must be an array'.format( + loader.source, file_name, sections_display, option_name, property_context + ) + ) + return + + if not formats: + loader.errors.append( + '{}, {}, {}{}: Attribute `formats`{} must contain at least one entry'.format( + loader.source, file_name, sections_display, option_name, property_context + ) + ) + return + + if any(not isinstance(fmt, str) for fmt in formats): + loader.errors.append( + '{}, {}, {}{}: Attribute `formats`{} must only contain strings'.format( + loader.source, file_name, sections_display, option_name, property_context + ) + ) + return + + seen = set() + duplicates = set() + for fmt in formats: + if fmt in seen: + duplicates.add(fmt) + else: + seen.add(fmt) + + if duplicates: + duplicate_display = ', '.join(sorted(duplicates)) + loader.errors.append( + '{}, {}, {}{}: Attribute `formats`{} contains duplicate entries: {}'.format( + loader.source, file_name, sections_display, option_name, property_context, duplicate_display + ) + ) + + invalid_formats = sorted(set(formats) - ALLOWED_FORMATS) + if invalid_formats: + valid_formats = ' | '.join(sorted(ALLOWED_FORMATS)) + invalid_display = ', '.join(invalid_formats) + loader.errors.append( + '{}, {}, {}{}: Attribute `formats`{} contains unknown value(s): {}, valid values are {}'.format( + loader.source, + file_name, + sections_display, + option_name, + property_context, + invalid_display, + valid_formats, + ) + ) + + +def value_validator(value, loader, file_name, sections_display, option_name, depth=0): + # Validate require_trusted_provider property if present + if 'require_trusted_provider' in value and not isinstance(value['require_trusted_provider'], bool): + loader.errors.append( + '{}, {}, {}{}: Attribute `require_trusted_provider` must be true or false'.format( + loader.source, file_name, sections_display, option_name + ) + ) + + if 'anyOf' in value: + if 'type' in value: + loader.errors.append( + '{}, {}, {}{}: Values must contain either a `type` or `anyOf` attribute, not both'.format( + loader.source, file_name, sections_display, option_name + ) + ) + return + + one_of = value['anyOf'] + if not isinstance(one_of, list): + loader.errors.append( + '{}, {}, {}{}: Attribute `anyOf` must be an array'.format( + loader.source, file_name, sections_display, option_name + ) + ) + return + elif len(one_of) == 1: + loader.errors.append( + '{}, {}, {}{}: Attribute `anyOf` contains a single type, use the `type` attribute instead'.format( + loader.source, file_name, sections_display, option_name + ) + ) + return + + for i, type_data in enumerate(one_of, 1): + if not isinstance(type_data, dict): + loader.errors.append( + '{}, {}, {}{}: Type #{} of attribute `anyOf` must be a mapping'.format( + loader.source, file_name, sections_display, option_name, i + ) + ) + return + + value_validator(type_data, loader, file_name, sections_display, option_name, depth=depth + 1) + + if not depth and value.get('example') is None: + value['example'] = default_option_example(option_name) + + return + elif 'type' not in value: + loader.errors.append( + '{}, {}, {}{}: Every value must contain a `type` attribute'.format( + loader.source, file_name, sections_display, option_name + ) + ) + return + + value_type = value['type'] + if not isinstance(value_type, str): + loader.errors.append( + '{}, {}, {}{}: Attribute `type` must be a string'.format( + loader.source, file_name, sections_display, option_name + ) + ) + return + + if value_type == 'string': + if 'example' not in value: + if not depth: + value['example'] = default_option_example(option_name) + elif not isinstance(value['example'], str): + loader.errors.append( + '{}, {}, {}{}: Attribute `example` for `type` {} must be a string'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + + if 'pattern' in value and not isinstance(value['pattern'], str): + loader.errors.append( + '{}, {}, {}{}: Attribute `pattern` for `type` {} must be a string'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + elif value_type in ('integer', 'number'): + if 'example' not in value: + if not depth: + value['example'] = default_option_example(option_name) + elif not isinstance(value['example'], (int, float)): + loader.errors.append( + '{}, {}, {}{}: Attribute `example` for `type` {} must be a number'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + + minimum_valid = True + maximum_valid = True + + if 'minimum' in value and not isinstance(value['minimum'], (int, float)): + loader.errors.append( + '{}, {}, {}{}: Attribute `minimum` for `type` {} must be a number'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + minimum_valid = False + + if 'maximum' in value and not isinstance(value['maximum'], (int, float)): + loader.errors.append( + '{}, {}, {}{}: Attribute `maximum` for `type` {} must be a number'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + maximum_valid = False + + if ( + 'minimum' in value + and 'maximum' in value + and minimum_valid + and maximum_valid + and value['maximum'] <= value['minimum'] + ): + loader.errors.append( + '{}, {}, {}{}: Attribute `maximum` for `type` {} must be greater than attribute `minimum`'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + elif value_type == 'boolean': + if 'example' not in value: + if not depth: + loader.errors.append( + '{}, {}, {}{}: Every {} must contain a default `example` attribute'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + elif not isinstance(value['example'], bool): + loader.errors.append( + '{}, {}, {}{}: Attribute `example` for `type` {} must be true or false'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + elif value_type == 'array': + if 'example' not in value: + if not depth: + value['example'] = [] + elif not isinstance(value['example'], list): + loader.errors.append( + '{}, {}, {}{}: Attribute `example` for `type` {} must be an array'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + + if 'uniqueItems' in value and not isinstance(value['uniqueItems'], bool): + loader.errors.append( + '{}, {}, {}{}: Attribute `uniqueItems` for `type` {} must be true or false'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + + min_items_valid = True + max_items_valid = True + + if 'minItems' in value and not isinstance(value['minItems'], int): + loader.errors.append( + '{}, {}, {}{}: Attribute `minItems` for `type` {} must be an integer'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + min_items_valid = False + + if 'maxItems' in value and not isinstance(value['maxItems'], int): + loader.errors.append( + '{}, {}, {}{}: Attribute `maxItems` for `type` {} must be an integer'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + max_items_valid = False + + if ( + 'minItems' in value + and 'maxItems' in value + and min_items_valid + and max_items_valid + and value['maxItems'] <= value['minItems'] + ): + loader.errors.append( + '{}, {}, {}{}: Attribute `maxItems` for `type` {} must be greater than attribute `minItems`'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + + if 'items' not in value: + loader.errors.append( + '{}, {}, {}{}: Every {} must contain an `items` attribute'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + return + + items = value['items'] + if not isinstance(items, dict): + loader.errors.append( + '{}, {}, {}{}: Attribute `items` for `type` {} must be a mapping object'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + return + + value_validator(items, loader, file_name, sections_display, option_name, depth=depth + 1) + elif value_type == 'object': + if 'example' not in value: + if not depth: + value['example'] = {} + elif not isinstance(value['example'], dict): + loader.errors.append( + '{}, {}, {}{}: Attribute `example` for `type` {} must be a mapping object'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + + required = value.get('required') + if 'required' in value: + if not isinstance(required, list): + loader.errors.append( + '{}, {}, {}{}: Attribute `required` for `type` {} must be an array'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + required = None + elif not required: + loader.errors.append( + '{}, {}, {}{}: Remove attribute `required` for `type` {} if no properties are required'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + elif len(required) - len(set(required)): + loader.errors.append( + '{}, {}, {}{}: All entries in attribute `required` for `type` {} must be unique'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + + properties = value.setdefault('properties', []) + if not isinstance(properties, list): + loader.errors.append( + '{}, {}, {}{}: Attribute `properties` for `type` {} must be an array'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + return + + new_depth = depth + 1 + property_names = [] + for prop in properties: + if not isinstance(prop, dict): + loader.errors.append( + '{}, {}, {}{}: Every entry in `properties` for `type` {} must be a mapping object'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + + if 'name' not in prop: + loader.errors.append( + '{}, {}, {}{}: Every entry in `properties` for `type` {} must contain a `name` attribute'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + continue + + name = prop['name'] + if not isinstance(name, str): + loader.errors.append( + '{}, {}, {}{}: Attribute `name` for `type` {} must be a string'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + continue + + property_names.append(name) + + if 'formats' in prop: + formats_validator(prop['formats'], loader, file_name, sections_display, option_name, property_name=name) + + value_validator(prop, loader, file_name, sections_display, option_name, depth=new_depth) + + if len(property_names) - len(set(property_names)): + loader.errors.append( + '{}, {}, {}{}: All entries in attribute `properties` for `type` {} must have unique names'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + + if required and set(required).difference(property_names): + loader.errors.append( + '{}, {}, {}{}: All entries in attribute `required` for `type` ' + '{} must be defined in the `properties` attribute'.format( + loader.source, file_name, sections_display, option_name, value_type + ) + ) + + if 'additionalProperties' in value: + additional_properties = value['additionalProperties'] + if additional_properties is True: + return + elif not isinstance(additional_properties, dict): + loader.errors.append( + '{}, {}, {}{}: Attribute `additionalProperties` for `type` {} must be a mapping or set ' + 'to `true`'.format(loader.source, file_name, sections_display, option_name, value_type) + ) + return + + value_validator(additional_properties, loader, file_name, sections_display, option_name, depth=new_depth) + else: + loader.errors.append( + '{}, {}, {}{}: Unknown type `{}`, valid types are {}'.format( + loader.source, + file_name, + sections_display, + option_name, + value_type, + ' | '.join(sorted(OPENAPI_DATA_TYPES)), + ) + ) + + +def default_option_example(option_name): + return f'<{option_name.upper()}>' + + +def normalize_source_name(source_name): + return source_name.lower().replace(' ', '_') diff --git a/ddev/src/ddev/validation/configuration/template.py b/ddev/src/ddev/validation/configuration/template.py new file mode 100644 index 0000000000000..6f26a3b23333f --- /dev/null +++ b/ddev/src/ddev/validation/configuration/template.py @@ -0,0 +1,185 @@ +# (C) Datadog, Inc. 2019-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + +import os +from copy import deepcopy +from pathlib import Path + +import yaml + +TEMPLATES_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'templates') +VALID_EXTENSIONS = ('yaml', 'yml') + + +class ConfigTemplates: + def __init__(self, paths: list[str] | None = None): + self.templates: dict = {} + self.paths: list[str] = [] + + if paths: + self.paths.extend(paths) + + self.paths.append(TEMPLATES_DIR) + + def load(self, template: str): + path_parts = template.split('/') + branches = path_parts.pop().split('.') + path_parts.append(branches.pop(0)) + + possible_template_paths = ( + f'{os.path.join(path, *path_parts)}.{extension}' for path in self.paths for extension in VALID_EXTENSIONS + ) + + for template_path in possible_template_paths: + if Path(template_path).is_file(): + break + else: + raise ValueError(f"Template `{'/'.join(path_parts)}` does not exist") + + if template_path in self.templates: + data = self.templates[template_path] + else: + try: + data = yaml.safe_load(Path(template_path).read_text()) + except Exception as e: + raise ValueError(f'Unable to parse template `{template_path}`: {e}') + + self.templates[template_path] = data + + data = deepcopy(data) + for i, branch in enumerate(branches): + if isinstance(data, dict): + if branch in data: + data = data[branch] + else: + raise ValueError( + f"Template `{'/'.join(path_parts)}` has no element `{'.'.join(branches[: i + 1])}`" + ) + elif isinstance(data, list): + for item in data: + if isinstance(item, dict) and item.get('name') == branch: + data = item + break + else: + raise ValueError( + 'Template `{}` has no named element `{}`'.format( + '/'.join(path_parts), '.'.join(branches[: i + 1]) + ) + ) + else: + raise ValueError( + 'Template `{}.{}` does not refer to a mapping, rather it is type `{}`'.format( + '/'.join(path_parts), '.'.join(branches[:i]), type(data).__name__ + ) + ) + + return data + + def _expand_and_find(self, items: list, name: str, visited: set | None = None) -> dict | None: + """Find a named item in a list, expanding nested template refs in-place as needed. + + When the named item lives inside a nested ``- template:`` reference, that reference + is loaded and spliced into ``items`` in-place so that subsequent spec.py processing + sees the already-expanded data with any overrides applied. + """ + if visited is None: + visited = set() + + for item in items: + if isinstance(item, dict) and item.get('name') == name: + return item + + for idx, item in enumerate(items): + if not isinstance(item, dict) or 'name' in item or 'template' not in item: + continue + tmpl_name = item['template'] + if tmpl_name in visited: + continue + visited.add(tmpl_name) + try: + nested = self.load(tmpl_name) + except Exception: + continue + + if isinstance(nested, dict): + if nested.get('name') == name: + items[idx] = nested + return nested + elif isinstance(nested, list): + found = self._expand_and_find(nested, name, visited) + if found is not None: + items[idx : idx + 1] = nested + return found + + return None + + def apply_overrides(self, template, overrides): + errors = [] + + for override, value in sorted(overrides.items()): + root = template + override_keys = override.split('.') + final_key = override_keys.pop() + + intermediate_error = '' + + # Iterate through all but the last key, attempting to find a dictionary at every step + for i, key in enumerate(override_keys): + if isinstance(root, dict): + if i == 0 and root.get('name') == key: + continue + + if key in root: + root = root[key] + else: + intermediate_error = ( + f"Template override `{'.'.join(override_keys[:i])}` has no named mapping `{key}`" + ) + break + elif isinstance(root, list): + found = self._expand_and_find(root, key) + if found is not None: + root = found + else: + intermediate_error = ( + f"Template override `{'.'.join(override_keys[:i])}` has no named mapping `{key}`" + ) + break + else: + intermediate_error = ( + f"Template override `{'.'.join(override_keys[:i])}` does not refer to a mapping" + ) + break + + if intermediate_error: + errors.append(intermediate_error) + continue + + # Force assign the desired value to the final key + if isinstance(root, dict): + root[final_key] = value + elif isinstance(root, list): + found = self._expand_and_find(root, final_key) + if found is not None: + for i, item in enumerate(root): + if item is found: + root[i] = value + break + else: + intermediate_error = 'Template override has no named mapping `{}`'.format( + '.'.join(override_keys) if override_keys else override + ) + else: + intermediate_error = 'Template override `{}` does not refer to a mapping'.format( + '.'.join(override_keys) if override_keys else override + ) + + if intermediate_error: + errors.append(intermediate_error) + continue + + overrides.pop(override) + + return errors diff --git a/ddev/src/ddev/validation/configuration/templates/ad_identifiers.yaml b/ddev/src/ddev/validation/configuration/templates/ad_identifiers.yaml new file mode 100644 index 0000000000000..fac5f94e3007a --- /dev/null +++ b/ddev/src/ddev/validation/configuration/templates/ad_identifiers.yaml @@ -0,0 +1,10 @@ +name: ad_identifiers +required: true +value: + type: array + items: + type: string +description: | + A list of container identifiers that are used by Autodiscovery to identify + which container the check should be run against. For more information, see: + https://docs.datadoghq.com/agent/guide/ad_identifiers/ diff --git a/ddev/src/ddev/validation/configuration/templates/common/perf_counters.yaml b/ddev/src/ddev/validation/configuration/templates/common/perf_counters.yaml new file mode 100644 index 0000000000000..907d772c65e18 --- /dev/null +++ b/ddev/src/ddev/validation/configuration/templates/common/perf_counters.yaml @@ -0,0 +1,142 @@ +- name: metrics + display_priority: 0 + fleet_configurable: true + description: | + This mapping defines which metrics to collect from the performance + counters on the `server`. For more information, see: + https://docs.microsoft.com/en-us/windows/win32/perfctrs/about-performance-counters + + The top-level keys are the names of the desired performance objects: + + metrics: + System: + : ... + : ... + LogicalDisk: + : ... + : ... + + The available performance object options are: + + name (required): This becomes the prefix of all metrics submitted for each counter. + counters (required): This is the list of counters to collect. + tag_name: This is the name of the tag used for instances. For example, if the tag name for + the `LogicalDisk` performance object is `disk`, a possible tag would be `disk:C`. + If not set, the default tag name is `instance`. + include: This is the list of regular expressions used to select which instances to monitor. + If not set, all instances are monitored. + exclude: This is the list of regular expressions used to select which instances to ignore. + If not set, no instances are ignored. Note: `_Total` instances are ignored by default; + set `include_total` to `true` to collect them. + include_total: Whether to collect the `_Total` aggregate instance for this performance object. + Defaults to `false`. Most performance objects report `_Total` as the sum of the + individual instances, in which case it is preferable to compute the total in + Datadog. However, some perf objects (e.g. `MSExchangeTransport Queues`) report + data on `_Total` that is not derivable from the visible instances, and in that + case opting in restores the missing data without forcing the collection of every + individual instance. + include_fast: This is the list of wildcards or exact instance names used to select which + instances to monitor. It is faster than the regular expression `include` filter + because it relies on the Windows PDH built-in wildcard filtering. + instance_counts: This is a mapping used to select the count of instances to submit, where each + key is a count type and the value is the metric name to use, ignoring `name`. + The `total` count type represents the total number of encountered instances. + The `monitored` count type represents the number of monitored instances after + `include`/`exclude` filtering. The `unique` count type represents the number + of unique instance names that are monitored. + use_localized_counters: Whether or not performance object and counter names should refer to their + locale-specific versions rather than by their English name. This overrides + any defined value in `init_config`. + + The key for each counter object represents the name of the desired counter. + Counters can be defined in the following ways: + + 1. Mapping the counter name to a string. If a value is a string, then it represents the suffix of the sent metric + name, for example: + + counters: + - '% Free Space': usable + - Current Disk Queue Length: queue_length.current + + 2. Mapping the counter name to a set of options. If a value is a mapping, then it must have a `name` or + `metric_name` key that represents the suffix or full metric name of the + sent metric name, for example: + + counters: + - '% Free Space': + name: usable + type: rate + - Current Disk Queue Length: + metric_name: queue_length.current + average: true + + The available counter options are: + + type: This represents how the metric is handled, defaulting to `gauge`. The available types are: + gauge, rate, count, monotonic_count, service_check, temporal_percent, and time_elapsed. + average: When there are multiple values for the same instance name (for example, multiple processes + spawned with the same name) the check submits the sum. Setting this option to `true` + instructs the check to calculate the average instead. + aggregate: Whether or not to send an additional metric that is the aggregation of all values for + every monitored instance. If `average` is set to `true` the check submits the average as + a metric suffixed by `avg`, otherwise it submits the sum as a metric suffixed by `sum`. + If this is set to `only`, the check does not submit a metric per instance. + metric_name: This represents the full metric name in lieu of a `name` key and is not be prefixed by + the parent object's `name` key. + value: + type: object + additionalProperties: + type: object + required: + - name + - counters + properties: + - name: name + type: string + - name: tag_name + type: string + - name: include + type: array + items: + type: string + - name: exclude + type: array + items: + type: string + - name: include_total + type: boolean + - name: instance_counts + type: object + properties: + - name: total + type: string + - name: monitored + type: string + - name: unique + type: string + - name: use_localized_counters + type: boolean + - name: counters + type: array + items: + type: object + additionalProperties: + anyOf: + - type: string + - type: object + properties: + - name: name + type: string + - name: type + type: string + - name: average + type: boolean + - name: aggregate + anyOf: + - type: boolean + - type: string + enum: + - only + - name: metric_name + type: string + additionalProperties: true diff --git a/ddev/src/ddev/validation/configuration/templates/init_config.yaml b/ddev/src/ddev/validation/configuration/templates/init_config.yaml new file mode 100644 index 0000000000000..935923de9e84f --- /dev/null +++ b/ddev/src/ddev/validation/configuration/templates/init_config.yaml @@ -0,0 +1,4 @@ +name: init_config +enabled: true +description: | + All options defined here are available to all instances. diff --git a/ddev/src/ddev/validation/configuration/templates/init_config/db.yaml b/ddev/src/ddev/validation/configuration/templates/init_config/db.yaml new file mode 100644 index 0000000000000..2921014995581 --- /dev/null +++ b/ddev/src/ddev/validation/configuration/templates/init_config/db.yaml @@ -0,0 +1,18 @@ +- name: global_custom_queries + display_priority: 0 + fleet_configurable: true + description: | + See `custom_queries` defined below. + + Global custom queries can be applied to all instances using the + `use_global_custom_queries` setting at the instance level. + value: + type: array + items: + type: object + display_default: null + example: + - query: + columns: + tags: + collection_interval: diff --git a/ddev/src/ddev/validation/configuration/templates/init_config/default.yaml b/ddev/src/ddev/validation/configuration/templates/init_config/default.yaml new file mode 100644 index 0000000000000..fa3a001e465cc --- /dev/null +++ b/ddev/src/ddev/validation/configuration/templates/init_config/default.yaml @@ -0,0 +1 @@ +- template: init_config/service diff --git a/ddev/src/ddev/validation/configuration/templates/init_config/http.yaml b/ddev/src/ddev/validation/configuration/templates/init_config/http.yaml new file mode 100644 index 0000000000000..0ec19428d8624 --- /dev/null +++ b/ddev/src/ddev/validation/configuration/templates/init_config/http.yaml @@ -0,0 +1,49 @@ +- name: proxy + display_priority: 0 + fleet_configurable: true + value: + example: + http: http://: + https: https://: + no_proxy: + - + - + type: object + properties: + - name: http + type: string + - name: https + type: string + - name: no_proxy + type: array + items: + type: string + description: | + Set HTTP or HTTPS proxies for all instances. Use the `no_proxy` list + to specify hosts that must bypass proxies. + + The SOCKS protocol is also supported like so: + + socks5://user:pass@host:port + + Using the scheme `socks5` causes the DNS resolution to happen on the + client, rather than on the proxy server. This is in line with `curl`, + which uses the scheme to decide whether to do the DNS resolution on + the client or proxy. If you want to resolve the domains on the proxy + server, use `socks5h` as the scheme. +- name: skip_proxy + display_priority: 0 + fleet_configurable: true + value: + example: false + type: boolean + description: | + If set to `true`, this makes the check bypass any proxy + settings enabled and attempt to reach services directly. +- name: timeout + display_priority: 0 + fleet_configurable: true + value: + example: 10 + type: number + description: The timeout for connecting to services. diff --git a/ddev/src/ddev/validation/configuration/templates/init_config/jmx.yaml b/ddev/src/ddev/validation/configuration/templates/init_config/jmx.yaml new file mode 100644 index 0000000000000..f95c855aca782 --- /dev/null +++ b/ddev/src/ddev/validation/configuration/templates/init_config/jmx.yaml @@ -0,0 +1,60 @@ +- name: is_jmx + display_priority: 0 + required: true + description: Whether or not this file is a configuration for a JMX integration. + value: + example: true + type: boolean + fleet_configurable: true +- name: collect_default_metrics + display_priority: 0 + description: Whether or not the check should collect all default metrics. + enabled: true + value: + display_default: false + example: true + type: boolean + fleet_configurable: true +- name: new_gc_metrics + display_priority: 0 + description: | + Set to true to use better metric names for garbage collection metrics. + jvm.gc.cms.count => jvm.gc.minor_collection_count + jvm.gc.major_collection_count + jvm.gc.parnew.time => jvm.gc.minor_collection_time + jvm.gc.major_collection_time + The default value is false to ensure backward compatibility. + enabled: true + value: + display_default: false + example: true + type: boolean + fleet_configurable: true +- name: service_check_prefix + display_priority: 0 + description: | + Custom service check prefix. e.g. `my_prefix` to get a service check called `my_prefix.can_connect`. + If not set, the default service check used is the integration name. + value: + type: string + fleet_configurable: true +- name: conf + display_priority: 0 + description: | + The list of metrics to be collected by the integration + Read http://docs.datadoghq.com/integrations/java/ to learn how to customize it + The default metrics to be collected are kept in metrics.yaml, but you can still + add your own metrics here. + fleet_configurable: true + value: + example: + - include: + bean: + attribute: + MyAttribute: + alias: my.metric.name + metric_type: gauge + type: array + items: + type: object +- template: init_config/default diff --git a/ddev/src/ddev/validation/configuration/templates/init_config/openmetrics.yaml b/ddev/src/ddev/validation/configuration/templates/init_config/openmetrics.yaml new file mode 100644 index 0000000000000..769c0f5aad1fc --- /dev/null +++ b/ddev/src/ddev/validation/configuration/templates/init_config/openmetrics.yaml @@ -0,0 +1,2 @@ +- template: init_config/http +- template: init_config/default diff --git a/ddev/src/ddev/validation/configuration/templates/init_config/openmetrics_legacy.yaml b/ddev/src/ddev/validation/configuration/templates/init_config/openmetrics_legacy.yaml new file mode 100644 index 0000000000000..769c0f5aad1fc --- /dev/null +++ b/ddev/src/ddev/validation/configuration/templates/init_config/openmetrics_legacy.yaml @@ -0,0 +1,2 @@ +- template: init_config/http +- template: init_config/default diff --git a/ddev/src/ddev/validation/configuration/templates/init_config/openmetrics_legacy_base b/ddev/src/ddev/validation/configuration/templates/init_config/openmetrics_legacy_base new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/ddev/src/ddev/validation/configuration/templates/init_config/perf_counters.yaml b/ddev/src/ddev/validation/configuration/templates/init_config/perf_counters.yaml new file mode 100644 index 0000000000000..a9854dfba183d --- /dev/null +++ b/ddev/src/ddev/validation/configuration/templates/init_config/perf_counters.yaml @@ -0,0 +1,9 @@ +- name: use_localized_counters + display_priority: 0 + description: | + Whether performance object and counter names should refer to their + locale-specific versions rather than their English name. + fleet_configurable: true + value: + example: false + type: boolean diff --git a/ddev/src/ddev/validation/configuration/templates/init_config/service.yaml b/ddev/src/ddev/validation/configuration/templates/init_config/service.yaml new file mode 100644 index 0000000000000..897ffe7f06a26 --- /dev/null +++ b/ddev/src/ddev/validation/configuration/templates/init_config/service.yaml @@ -0,0 +1,9 @@ +name: service +display_priority: 0 +value: + type: string +description: | + Attach the tag `service:` to every metric, event, and service check emitted by this integration. + + Additionally, this sets the default `service` for every log source. +fleet_configurable: true diff --git a/ddev/src/ddev/validation/configuration/templates/init_config/tags.yaml b/ddev/src/ddev/validation/configuration/templates/init_config/tags.yaml new file mode 100644 index 0000000000000..450af0a0f9559 --- /dev/null +++ b/ddev/src/ddev/validation/configuration/templates/init_config/tags.yaml @@ -0,0 +1,13 @@ +name: tags +display_priority: 0 +value: + example: + - : + - : + type: array + items: + type: string +description: | + A list of tags to attach to every metric and service check emitted by this integration. + + Learn more about tagging at https://docs.datadoghq.com/tagging diff --git a/ddev/src/ddev/validation/configuration/templates/instances.yaml b/ddev/src/ddev/validation/configuration/templates/instances.yaml new file mode 100644 index 0000000000000..5d07c820fd6f2 --- /dev/null +++ b/ddev/src/ddev/validation/configuration/templates/instances.yaml @@ -0,0 +1,5 @@ +name: instances +multiple: true +enabled: true +description: | + Every instance is scheduled independently of the others. diff --git a/ddev/src/ddev/validation/configuration/templates/instances/all_integrations.yaml b/ddev/src/ddev/validation/configuration/templates/instances/all_integrations.yaml new file mode 100644 index 0000000000000..48900d29ec229 --- /dev/null +++ b/ddev/src/ddev/validation/configuration/templates/instances/all_integrations.yaml @@ -0,0 +1,3 @@ +- template: instances/tags +- template: instances/service +- template: instances/global diff --git a/ddev/src/ddev/validation/configuration/templates/instances/db.yaml b/ddev/src/ddev/validation/configuration/templates/instances/db.yaml new file mode 100644 index 0000000000000..957919ce7fc8f --- /dev/null +++ b/ddev/src/ddev/validation/configuration/templates/instances/db.yaml @@ -0,0 +1,102 @@ +- name: only_custom_queries + display_priority: 0 + fleet_configurable: true + description: | + Set this parameter to `true` if you want to skip the integration's default metrics collection. + Only metrics specified in `custom_queries` will be collected. + value: + type: boolean + example: false +- name: use_global_custom_queries + display_priority: 0 + fleet_configurable: true + description: | + How `global_custom_queries` should be used for this instance. There are 3 options: + + 1. true - `global_custom_queries` override `custom_queries`. + 2. false - `custom_queries` override `global_custom_queries`. + 3. extend - `global_custom_queries` are used in addition to any `custom_queries`. + value: + type: string + example: "true" +- name: custom_queries + display_priority: 0 + fleet_configurable: true + description: | + Each query must have 2 fields, and can have a third optional field: + + 1. query - The SQL to execute. It can be a simple statement or a multi-line script. + Use the pipe `|` if you require a multi-line script. + 2. columns - The list representing each column, ordered sequentially from left to right. + The number of columns must equal the number of columns returned in the query. + There are 2 required pieces of data: + 1. name - The suffix to append to `.` to form + the full metric name. If `type` is a `tag` type, this column is considered a tag and applied + to every metric collected by this particular query. + 2. type - The submission method (gauge, monotonic_count, etc.). + This can also be set to the following `tag` types to tag each metric in the row with the name + and value of the item in this column: + 1. tag - This is the default tag type + 2. tag_list - This allows multiple values to be attached to the tag name. For example: + ``` + query = { + "name": "example", + "query": "...", + "columns": [ + {"name": "server_tag", "type": "tag_list"}, + {"name": "foo", "type": "gauge"}, + ] + } + ``` + May result in: + ``` + gauge("foo", tags=["server_tag:us", "server_tag:primary", "server_tag:default"]) + gauge("foo", tags=["server_tag:eu"]) + gauge("foo", tags=["server_tag:eu", "server_tag:primary"]) + ``` + 3. tag_not_null - This only sets tags in the metric if the value is not null + You can use the `count` type to perform aggregation for queries that return multiple rows with + the same or no tags. + Columns without a name are ignored. To skip a column, enter: + ``` + - {} + ``` + 3. tags (optional) - A list of tags to apply to each metric. + 4. collection_interval (optional) - The frequency at which to collect the metrics. + If collection_interval is not set, the query will be run every check run. + If the collection interval is less than check collection interval, the query will be run every check + run. + If the collection interval is greater than check collection interval, the query will NOT BE RUN + exactly at the collection interval. + The query will be run at the next check run after the collection interval has passed. + 5. metric_prefix (optional) - The prefix to apply to each metric. + value: + type: array + items: + type: object + properties: + - name: query + type: string + - name: columns + type: array + items: + type: object + - name: tags + type: array + items: + type: string + - name: collection_interval + type: integer + - name: metric_prefix + type: string + example: + - query: SELECT foo, COUNT(*) FROM table.events GROUP BY foo + columns: + - name: foo + type: tag + - name: event.total + type: gauge + tags: + - test: + collection_interval: 30 + metric_prefix: foo_prefix diff --git a/ddev/src/ddev/validation/configuration/templates/instances/default.yaml b/ddev/src/ddev/validation/configuration/templates/instances/default.yaml new file mode 100644 index 0000000000000..945b1a94aa530 --- /dev/null +++ b/ddev/src/ddev/validation/configuration/templates/instances/default.yaml @@ -0,0 +1,47 @@ +- template: instances/all_integrations +- name: disable_generic_tags + display_priority: 0 + description: | + Generic tags such as `cluster` will be replaced by _cluster to avoid + getting mixed with other integration tags. + value: + type: boolean + display_default: false + example: true + hidden: true + fleet_configurable: true +- name: enable_legacy_tags_normalization + display_priority: 0 + description: | + Whether to enable legacy tags normalization. When enabled (default), hyphens in tag + values are replaced with underscores. Set to `false` to preserve hyphens, which is + consistent with Datadog's tag format. + value: + type: boolean + example: false + display_default: true + hidden: true + fleet_configurable: true +- name: metric_patterns + display_priority: 0 + description: | + A mapping of metrics to include or exclude, with each entry being a regular expression. + + Metrics defined in `exclude` will take precedence in case of overlap. + value: + example: + include: + - + exclude: + - + type: object + properties: + - name: include + type: array + items: + type: string + - name: exclude + type: array + items: + type: string + fleet_configurable: true diff --git a/ddev/src/ddev/validation/configuration/templates/instances/global.yaml b/ddev/src/ddev/validation/configuration/templates/instances/global.yaml new file mode 100644 index 0000000000000..73f3f23246ed8 --- /dev/null +++ b/ddev/src/ddev/validation/configuration/templates/instances/global.yaml @@ -0,0 +1,19 @@ +- name: min_collection_interval + display_priority: 0 + value: + example: 15 + type: number + description: | + This changes the collection interval of the check. For more information, see: + https://docs.datadoghq.com/developers/write_agent_check/#collection-interval + fleet_configurable: true +- name: empty_default_hostname + display_priority: 0 + value: + example: false + type: boolean + description: | + This forces the check to send metrics with no hostname. + + This is useful for cluster-level checks. + fleet_configurable: true diff --git a/ddev/src/ddev/validation/configuration/templates/instances/http.yaml b/ddev/src/ddev/validation/configuration/templates/instances/http.yaml new file mode 100644 index 0000000000000..1332e0b8538d6 --- /dev/null +++ b/ddev/src/ddev/validation/configuration/templates/instances/http.yaml @@ -0,0 +1,440 @@ +- name: proxy + display_priority: 0 + fleet_configurable: true + value: + example: + http: http://: + https: https://: + no_proxy: + - + - + type: object + properties: + - name: http + type: string + - name: https + type: string + - name: no_proxy + type: array + items: + type: string + description: | + This overrides the `proxy` setting in `init_config`. + + Set HTTP or HTTPS proxies for this instance. Use the `no_proxy` list + to specify hosts that must bypass proxies. + + The SOCKS protocol is also supported, for example: + + socks5://user:pass@host:port + + Using the scheme `socks5` causes the DNS resolution to happen on the + client, rather than on the proxy server. This is in line with `curl`, + which uses the scheme to decide whether to do the DNS resolution on + the client or proxy. If you want to resolve the domains on the proxy + server, use `socks5h` as the scheme. +- name: skip_proxy + display_priority: 0 + fleet_configurable: true + value: + example: false + type: boolean + description: | + This overrides the `skip_proxy` setting in `init_config`. + + If set to `true`, this makes the check bypass any proxy + settings enabled and attempt to reach services directly. +- name: auth_type + display_priority: 0 + value: + example: basic + type: string + description: | + The type of authentication to use. The available types (and related options) are: + ``` + - basic + |__ username + |__ password + |__ use_legacy_auth_encoding + - digest + |__ username + |__ password + - ntlm + |__ ntlm_domain + |__ password + - kerberos + |__ kerberos_auth + |__ kerberos_cache + |__ kerberos_delegate + |__ kerberos_force_initiate + |__ kerberos_hostname + |__ kerberos_keytab + |__ kerberos_principal + - aws + |__ aws_region + |__ aws_host + |__ aws_service + ``` + The `aws` auth type relies on boto3 to automatically gather AWS credentials, for example: from `.aws/credentials`. /noqa + Details: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html#configuring-credentials /noqa + + fleet_configurable: true +- name: use_legacy_auth_encoding + display_priority: 0 + fleet_configurable: true + value: + example: true + type: boolean + description: | + When `auth_type` is set to `basic`, this determines whether to encode as `latin1` rather than `utf-8`. +- name: username + display_priority: 0 + value: + type: string + description: The username to use if services are behind basic or digest auth. + fleet_configurable: true +- name: password + display_priority: 0 + secret: true + value: + type: string + description: The password to use if services are behind basic or NTLM auth. + fleet_configurable: true +- name: ntlm_domain + display_priority: 0 + fleet_configurable: true + value: + example: \ + type: string + description: | + If your services use NTLM authentication, specify + the domain used in the check. For NTLM Auth, append + the username to domain, not as the `username` parameter. +- name: kerberos_auth + display_priority: 0 + fleet_configurable: true + value: + example: disabled + type: string + enum: + - required + - optional + - disabled + description: | + If your services use Kerberos authentication, you can specify the Kerberos + strategy to use between: + + - required + - optional + - disabled + + See https://github.com/requests/requests-kerberos#mutual-authentication +- name: kerberos_cache + display_priority: 0 + fleet_configurable: true + value: + type: string + require_trusted_provider: true + description: | + Sets the KRB5CCNAME environment variable. + It should point to a credential cache with a valid TGT. +- name: kerberos_delegate + display_priority: 0 + fleet_configurable: true + value: + example: false + type: boolean + description: | + Set to `true` to enable Kerberos delegation of credentials to a server that requests delegation. + + See https://github.com/requests/requests-kerberos#delegation +- name: kerberos_force_initiate + display_priority: 0 + fleet_configurable: true + value: + example: false + type: boolean + description: | + Set to `true` to preemptively initiate the Kerberos GSS exchange and + present a Kerberos ticket on the initial request (and all subsequent). + + See https://github.com/requests/requests-kerberos#preemptive-authentication +- name: kerberos_hostname + display_priority: 0 + fleet_configurable: true + value: + type: string + description: | + Override the hostname used for the Kerberos GSS exchange if its DNS name doesn't + match its Kerberos hostname, for example: behind a content switch or load balancer. + + See https://github.com/requests/requests-kerberos#hostname-override +- name: kerberos_principal + display_priority: 0 + fleet_configurable: true + value: + type: string + description: | + Set an explicit principal, to force Kerberos to look for a + matching credential cache for the named user. + + See https://github.com/requests/requests-kerberos#explicit-principal +- name: kerberos_keytab + display_priority: 0 + fleet_configurable: true + formats: ["path"] + value: + example: + type: string + require_trusted_provider: true + description: Set the path to your Kerberos key tab file. +- name: auth_token + display_priority: 0 + secret: true + fleet_configurable: true + value: + example: + reader: + type: + : + : + writer: + type: + : + : + type: object + require_trusted_provider: true + properties: + - name: reader + type: object + properties: [] + - name: writer + type: object + properties: [] + description: | + This allows for the use of authentication information from dynamic sources. + Both a reader and writer must be configured. + + The available readers are: + + - type: file + path (required): The absolute path for the file to read from. + pattern: A regular expression pattern with a single capture group used to find the + token rather than using the entire file, for example: Your secret is (.+) + - type: oauth + url (required): The token endpoint. + client_id (required): The client identifier. + client_secret (required): The client secret. + basic_auth: Whether the provider expects credentials to be transmitted in + an HTTP Basic Auth header. The default is: false + options: Mapping of additional options to pass to the provider, such as the audience + or the scope. For example: + options: + audience: https://example.com + scope: read:example + + The available writers are: + + - type: header + name (required): The name of the field, for example: Authorization + value: The template value, for example `Bearer `. The default is: + placeholder: The substring in `value` to replace with the token, defaults to: +- name: aws_region + display_priority: 0 + fleet_configurable: true + value: + type: string + description: | + If your services require AWS Signature Version 4 signing, set the region. + + See https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html +- name: aws_host + display_priority: 0 + fleet_configurable: true + value: + type: string + description: | + If your services require AWS Signature Version 4 signing, set the host. + This only needs the hostname and does not require the protocol (HTTP, HTTPS, and more). + For example, if connecting to https://us-east-1.amazonaws.com/, set `aws_host` to `us-east-1.amazonaws.com`. + + Note: This setting is not necessary for official integrations. + + See https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html +- name: aws_service + display_priority: 0 + fleet_configurable: true + value: + type: string + description: | + If your services require AWS Signature Version 4 signing, set the service code. For a list + of available service codes, see https://docs.aws.amazon.com/general/latest/gr/rande.html + + Note: This setting is not necessary for official integrations. + + See https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html +- name: tls_verify + display_priority: 0 + fleet_configurable: true + value: + example: true + type: boolean + description: Instructs the check to validate the TLS certificate of services. +- name: tls_use_host_header + display_priority: 0 + fleet_configurable: true + value: + example: false + type: boolean + description: | + If a `Host` header is set, this enables its use for SNI (matching against the TLS certificate CN or SAN). +- name: tls_ignore_warning + display_priority: 0 + fleet_configurable: true + value: + example: false + type: boolean + description: | + If `tls_verify` is disabled, security warnings are logged by the check. + Disable those by setting `tls_ignore_warning` to true. +- name: tls_cert + display_priority: 0 + fleet_configurable: true + value: + example: + type: string + require_trusted_provider: true + description: | + The path to a single file in PEM format containing a certificate as well as any + number of CA certificates needed to establish the certificate's authenticity for + use when connecting to services. It may also contain an unencrypted private key to use. +- name: tls_private_key + display_priority: 0 + fleet_configurable: true + value: + example: + type: string + require_trusted_provider: true + description: | + The unencrypted private key to use for `tls_cert` when connecting to services. This is + required if `tls_cert` is set and it does not already contain a private key. +- name: tls_ca_cert + display_priority: 0 + fleet_configurable: true + value: + example: + type: string + require_trusted_provider: true + description: | + The path to a file of concatenated CA certificates in PEM format or a directory + containing several CA certificates in PEM format. If a directory, the directory + must have been processed using the `openssl rehash` command. See: + https://www.openssl.org/docs/man3.2/man1/c_rehash.html +- name: tls_protocols_allowed + display_priority: 0 + fleet_configurable: true + value: + example: + - 'SSLv3' + - 'TLSv1.2' + - 'TLSv1.3' + type: array + items: + type: string + description: | + The expected versions of TLS/SSL when fetching intermediate certificates. + Only `SSLv3`, `TLSv1.2`, `TLSv1.3` are allowed by default. The possible values are: + SSLv3 + TLSv1 + TLSv1.1 + TLSv1.2 + TLSv1.3 +- name: tls_ciphers + display_priority: 0 + description: | + The list of ciphers suites to use when connecting to an endpoint. If not specified, + `ALL` ciphers are used. For list of ciphers see: + https://www.openssl.org/docs/man1.0.2/man1/ciphers.html + fleet_configurable: true + value: + type: array + items: + type: string + example: + - 'TLS_AES_256_GCM_SHA384' + - 'TLS_CHACHA20_POLY1305_SHA256' + - 'TLS_AES_128_GCM_SHA256' +- name: headers + display_priority: 0 + fleet_configurable: true + value: + example: + Host: + X-Auth-Token: + type: object + description: | + The headers parameter allows you to send specific headers with every request. + You can use it for explicitly specifying the host header or adding headers for + authorization purposes. + + This overrides any default headers. +- name: extra_headers + display_priority: 0 + fleet_configurable: true + value: + example: + Host: + X-Auth-Token: + type: object + description: Additional headers to send with every request. +- name: timeout + display_priority: 0 + value: + example: 10 + type: number + description: | + The timeout for accessing services. + + This overrides the `timeout` setting in `init_config`. + fleet_configurable: true +- name: connect_timeout + display_priority: 0 + fleet_configurable: true + value: + type: number + description: The connect timeout for accessing services. Defaults to `timeout`. +- name: read_timeout + display_priority: 0 + fleet_configurable: true + value: + type: number + description: The read timeout for accessing services. Defaults to `timeout`. +- name: request_size + display_priority: 0 + description: | + The number of kibibytes (KiB) to read from streaming HTTP responses at a time. + fleet_configurable: true + value: + example: 16 + type: number +- name: log_requests + display_priority: 0 + fleet_configurable: true + value: + example: false + type: boolean + description: Whether or not to debug log the HTTP(S) requests made, including the method and URL. +- name: persist_connections + display_priority: 0 + fleet_configurable: true + value: + example: false + type: boolean + description: Whether or not to persist cookies and use connection pooling for improved performance. +- name: allow_redirects + display_priority: 0 + fleet_configurable: true + value: + example: true + type: boolean + description: Whether or not to allow URL redirection. diff --git a/ddev/src/ddev/validation/configuration/templates/instances/jmx.yaml b/ddev/src/ddev/validation/configuration/templates/instances/jmx.yaml new file mode 100644 index 0000000000000..18c20caa2738b --- /dev/null +++ b/ddev/src/ddev/validation/configuration/templates/instances/jmx.yaml @@ -0,0 +1,181 @@ +- name: host + display_priority: 0 + required: true + description: JMX hostname to connect to. + value: + type: string + fleet_configurable: true + +- name: port + display_priority: 0 + required: true + description: JMX port to connect to. + formats: ["port"] + value: + type: integer + fleet_configurable: true + +- name: is_jmx + display_priority: 0 + description: | + Whether or not this instance is a configuration for a JMX integration. + If `is_jmx` is set to true at the init_config level, this flag is ignored. + hidden: true + value: + example: false + type: boolean + fleet_configurable: true + +- name: jmx_url + display_priority: 0 + description: JMX URL to connect to. Can be used instead of host/port configs. + hidden: true + formats: ["url"] + value: + type: string + fleet_configurable: true + +- name: user + display_priority: 0 + description: User to use when connecting to JMX. + value: + type: string + fleet_configurable: true + +- name: password + display_priority: 0 + description: Password to use when connecting to JMX. + secret: true + value: + type: string + fleet_configurable: true + +- name: process_name_regex + display_priority: 0 + description: | + Instead of using a host and port, the Agent can connect using the attach API. + This requires the JDK to be installed and the path to tools.jar to be set below. + Note: It needs to be set when process_name_regex parameter is set + e.g. .*process_name.* + + Note: tools.jar was removed in Java 9: https://openjdk.java.net/jeps/220. + This option is supported in Java 8 and below. + fleet_configurable: true + value: + type: string + +- name: tools_jar_path + display_priority: 0 + description: | + The tools.jar path to be used with the `process_name_regex` parameter, + for example: /usr/lib/jvm/java-7-openjdk-amd64/lib/tools.jar + + Note: tools.jar was removed in Java 9: https://openjdk.java.net/jeps/220. + This option is supported in Java 8 and below. + fleet_configurable: true + formats: ["path"] + value: + type: string + require_trusted_provider: true + +- name: name + display_priority: 0 + description: Set the instance name to be used as the `instance` tag. + value: + type: string + + fleet_configurable: true +- name: java_bin_path + display_priority: 0 + description: "`java_bin_path` should be set if the Agent cannot find your java executable." + fleet_configurable: true + value: + type: string + require_trusted_provider: true + +- name: java_options + display_priority: 0 + description: 'A list of Java JVM options, for example: "-Xmx200m -Xms50m".' + fleet_configurable: true + formats: ["java_jvm_options"] + value: + type: string + +- name: trust_store_path + display_priority: 0 + description: | + The path to your trusted store. + `trust_store_path` should be set if SSL is enabled. + formats: ["path"] + fleet_configurable: true + value: + type: string + require_trusted_provider: true + +- name: trust_store_password + display_priority: 0 + description: | + The password for your TrustStore.jks file. + `trust_store_password` should be set if SSL is enabled. + secret: true + value: + type: string + fleet_configurable: true + +- name: key_store_path + display_priority: 0 + description: | + The path to your key store. + `key_store_path` should be set if client authentication is enabled on the target JVM. + formats: ["path"] + value: + type: string + require_trusted_provider: true + fleet_configurable: true + +- name: key_store_password + display_priority: 0 + description: | + The password to your key store. + `key_store_password` should be set if client authentication is enabled on the target JVM. + secret: true + value: + type: string + fleet_configurable: true + +- name: rmi_registry_ssl + display_priority: 0 + description: Whether or not the Agent should connect to the RMI registry using SSL. + fleet_configurable: true + value: + example: false + type: boolean + +- name: rmi_connection_timeout + display_priority: 0 + description: The connection timeout, in milliseconds, when connecting to a remote JVM. + fleet_configurable: true + value: + example: 20000 + type: number + +- name: rmi_client_timeout + display_priority: 0 + description: | + The timeout to consider a remote connection, already successfully established, as lost. + If a connected remote JVM does not reply after `rmi_client_timeout` milliseconds jmxfetch + will give up on that connection and retry. + fleet_configurable: true + value: + example: 15000 + type: number + +- name: collect_default_jvm_metrics + display_priority: 0 + description: Configures the collection of default JVM metrics. + value: + example: true + type: boolean + fleet_configurable: true + +- template: instances/all_integrations diff --git a/ddev/src/ddev/validation/configuration/templates/instances/openmetrics.yaml b/ddev/src/ddev/validation/configuration/templates/instances/openmetrics.yaml new file mode 100644 index 0000000000000..e79a3c82648d1 --- /dev/null +++ b/ddev/src/ddev/validation/configuration/templates/instances/openmetrics.yaml @@ -0,0 +1,448 @@ +- name: openmetrics_endpoint + display_priority: 0 + required: true + description: | + The URL exposing metrics in the OpenMetrics format. + fleet_configurable: true + value: + type: string +- name: namespace + display_priority: 0 + hidden: true + description: | + The namespace to be prepended to all metrics. + fleet_configurable: true + value: + type: string + pattern: '\w*' +- name: raw_metric_prefix + display_priority: 0 + description: | + A prefix that is removed from all exposed metric names, if present. + All configuration options will use the prefix-less name. + fleet_configurable: true + value: + type: string + example: _ + display_default: null +# Only shown for the generic `openmetrics` integration. Lets a user define which metrics they want to collect. +- name: metrics + display_priority: 0 + hidden: true + description: | + This list defines which metrics to collect from the `openmetrics_endpoint`. + Metrics may be defined in 3 ways: + + 1. If the item is a string, then it represents the exposed metric name, and + the sent metric name is identical. For example: + ``` + metrics: + - + - + ``` + 2. If the item is a mapping, then the keys represent the exposed metric names. + + 1. If a value is a string, then it represents the sent metric name. For example: + ``` + metrics: + - : + - : + ``` + 2. If a value is a mapping, then it must have a `name` and, optionally, a `type` key. + The `name` represents the sent metric name, and the `type` represents how + the metric should be handled, overriding any type information the endpoint + may provide. For example: + ``` + metrics: + - : + name: + type: + - : + name: + type: + ``` + The supported native types are `gauge`, `counter`, `histogram`, and `summary`. + + Note: To collect counter metrics with names ending in `_total`, specify the metric name without the `_total` + suffix. For example, to collect the counter metric `promhttp_metric_handler_requests_total`, specify + `promhttp_metric_handler_requests`. This submits to Datadog the metric name appended with `.count`. + For more information, see: + https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#suffixes + + Regular expressions may be used to match the exposed metric names, for example: + ``` + metrics: + - ^network_(ingress|egress)_.+ + - .+: + type: gauge + ```` + fleet_configurable: true + value: + example: + - + - : + - : + name: + type: + type: array + items: + anyOf: + - type: string + - type: object + additionalProperties: + anyOf: + - type: string + - type: object + properties: + - name: name + type: string + - name: type + type: string + additionalProperties: true +# Hidden for the generic `openmetrics` integration, which uses the `metrics` field. +# For custom openmetrics-based integrations this allows users to specify metrics in addition to the built-in ones. +- name: extra_metrics + display_priority: 0 + description: | + This list defines metrics to collect from the `openmetrics_endpoint`, in addition to + what the check collects by default. If the check already collects a metric, then + metric definitions here take precedence. Metrics may be defined in 3 ways: + + 1. If the item is a string, then it represents the exposed metric name, and + the sent metric name will be identical. For example: + ``` + extra_metrics: + - + - + ``` + 2. If the item is a mapping, then the keys represent the exposed metric names. + + 1. If a value is a string, then it represents the sent metric name. For example: + ``` + extra_metrics: + - : + - : + ``` + 2. If a value is a mapping, then it must have a `name` and/or `type` key. + The `name` represents the sent metric name, and the `type` represents how + the metric should be handled, overriding any type information the endpoint + may provide. For example: + ``` + extra_metrics: + - : + name: + type: + - : + name: + type: + ``` + The supported native types are `gauge`, `counter`, `histogram`, and `summary`. + + Note: To collect counter metrics with names ending in `_total`, specify the metric name without the `_total` + suffix. For example, to collect the counter metric `promhttp_metric_handler_requests_total`, specify + `promhttp_metric_handler_requests`. This submits to Datadog the metric name appended with `.count`. + For more information, see: + https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#suffixes + + Regular expressions may be used to match the exposed metric names, for example: + ``` + extra_metrics: + - ^network_(ingress|egress)_.+ + - .+: + type: gauge + ``` + fleet_configurable: true + value: + example: + - + - : + - : + name: + type: + type: array + items: + anyOf: + - type: string + - type: object + additionalProperties: + anyOf: + - type: string + - type: object + properties: + - name: name + type: string + - name: type + type: string + additionalProperties: true +- name: exclude_metrics + display_priority: 0 + description: | + A list of metrics to exclude, with each entry being either + the exact metric name or a regular expression. + + In order to exclude all metrics but the ones matching a specific filter, + you can use a negative lookahead regex like: + - ^(?!foo).*$ + fleet_configurable: true + value: + type: array + items: + type: string +- name: exclude_metrics_by_labels + display_priority: 0 + description: | + A mapping of labels to exclude metrics with matching label name and their corresponding metric values. To match + all values of a label, set it to `true`. + + Note: Label filtering happens before `rename_labels`. + + For example, the following configuration instructs the check to exclude all metrics with + a label `worker` or a label `pid` with the value of either `23` or `42`. + + exclude_metrics_by_labels: + worker: true + pid: + - '23' + - '42' + fleet_configurable: true + value: + type: object + additionalProperties: + anyOf: + - type: boolean + - type: array + items: + type: string +- name: tag_by_endpoint + display_priority: 0 + description: Whether to include an endpoint tag or not. + fleet_configurable: true + hidden: true + value: + type: boolean + example: False + display_default: True +- name: exclude_labels + display_priority: 0 + description: | + A list of labels to exclude, useful for high cardinality values like timestamps or UUIDs. + May be used in conjunction with `include_labels`. + Labels defined in `exclude_labels` will take precedence in case of overlap. + + Note: Label filtering happens before `rename_labels`. + fleet_configurable: true + value: + type: array + items: + type: string +- name: include_labels + display_priority: 0 + description: | + A list of labels to include. May be used in conjunction with `exclude_labels`. + Labels defined in `exclude_labels` will take precedence in case of overlap. + + Note: Label filtering happens before `rename_labels`. + fleet_configurable: true + value: + type: array + items: + type: string +- name: rename_labels + display_priority: 0 + description: | + A mapping of label names to their new names. + fleet_configurable: true + value: + example: + : + : + type: object +- name: enable_health_service_check + display_priority: 0 + description: | + Whether or not to send a service check named `.openmetrics.health` which reports + the health of the `openmetrics_endpoint`. + fleet_configurable: true + value: + example: true + type: boolean +- name: ignore_connection_errors + display_priority: 0 + description: | + Whether or not to ignore connection errors when scraping `openmetrics_endpoint`. + fleet_configurable: true + value: + example: false + type: boolean +- name: hostname_label + display_priority: 0 + description: | + Override the hostname for every metric submission with the value of one of its labels. + fleet_configurable: true + value: + type: string +- name: hostname_format + display_priority: 0 + description: | + When `hostname_label` is set, this instructs the check how to format the values. The string + `` is replaced by the value of the label defined by `hostname_label`. + fleet_configurable: true + value: + example: + type: string +- name: collect_histogram_buckets + display_priority: 0 + description: | + Whether or not to send histogram buckets. + fleet_configurable: true + value: + example: true + type: boolean +- name: non_cumulative_histogram_buckets + display_priority: 0 + fleet_configurable: true + description: | + Whether or not histogram buckets are non-cumulative and to come with a `lower_bound` tag. + value: + example: false + type: boolean +- name: histogram_buckets_as_distributions + display_priority: 0 + description: | + Whether or not to send histogram buckets as Datadog distribution metrics. This implicitly + enables the `collect_histogram_buckets` and `non_cumulative_histogram_buckets` options. + + Learn more about distribution metrics: + https://docs.datadoghq.com/developers/metrics/types/?tab=distribution#metric-types + fleet_configurable: true + value: + example: false + type: boolean +- name: collect_counters_with_distributions + display_priority: 0 + description: | + Whether or not to also collect the observation counter metrics ending in `.sum` and `.count` + when sending histogram buckets as Datadog distribution metrics. This implicitly enables the + `histogram_buckets_as_distributions` option. + fleet_configurable: true + value: + example: false + type: boolean +- name: use_process_start_time + display_priority: 0 + description: | + Whether to enable a heuristic for reporting counter values on the first scrape. When true, + the first time an endpoint is scraped, check `process_start_time_seconds` to decide whether zero + initial value can be assumed for counters. This requires keeping metrics in memory until the entire + response is received. + fleet_configurable: true + value: + example: false + type: boolean +- name: share_labels + display_priority: 0 + description: | + This mapping allows for the sharing of labels across multiple metrics. The keys represent the + exposed metrics from which to share labels, and the values are mappings that configure the + sharing behavior. Each mapping must have at least one of the following keys: + + - labels - This is a list of labels to share. All labels are shared if this is not set. + - match - This is a list of labels to match on other metrics as a condition for sharing. + - values - This is a list of allowed values as a condition for sharing. + + To unconditionally share all labels of a metric, set it to `true`. + + For example, the following configuration instructs the check to apply all labels from `metric_a` + to all other metrics, the `node` label from `metric_b` to only those metrics that have a `pod` + label value that matches the `pod` label value of `metric_b`, and all labels from `metric_c` + to all other metrics if their value is equal to `23` or `42`. + fleet_configurable: true + value: + example: + metric_a: true + metric_b: + labels: + - node + match: + - pod + metric_c: + values: + - 23 + - 42 + type: object + additionalProperties: + anyOf: + - type: boolean + - type: object + properties: + - name: labels + type: array + items: + type: string + - name: match + type: array + items: + type: string +- name: cache_shared_labels + display_priority: 0 + description: | + When `share_labels` is set, it instructs the check to cache labels collected from the first payload + for improved performance. + + Set this to `false` to compute label sharing for every payload at the risk of potentially increased memory usage. + fleet_configurable: true + value: + type: boolean + example: true +- name: raw_line_filters + display_priority: 0 + description: | + A list of regular expressions used to exclude lines read from the `openmetrics_endpoint` + from being parsed. + fleet_configurable: true + value: + type: array + items: + type: string +- name: cache_metric_wildcards + display_priority: 0 + description: | + Whether or not to cache data from metrics that are defined by regular expressions rather + than the full metric name. + fleet_configurable: true + value: + example: true + type: boolean +- name: use_latest_spec + display_priority: 0 + description: | + Whether or not the parser will strictly adhere to the OpenMetrics specification, + regardless of the received `Content-Type` header. + fleet_configurable: true + hidden: true + value: + example: false + type: boolean +- name: telemetry + display_priority: 0 + description: | + Whether or not to submit metrics prefixed by `.telemetry.` for debugging purposes. + fleet_configurable: true + value: + example: false + type: boolean +- name: ignore_tags + display_priority: 0 + description: | + A list of regular expressions used to ignore tags added by Autodiscovery and entries in the `tags` option. + fleet_configurable: true + value: + type: array + items: + type: string + example: + - + - + - +- template: instances/http +- template: instances/default diff --git a/ddev/src/ddev/validation/configuration/templates/instances/openmetrics_legacy.yaml b/ddev/src/ddev/validation/configuration/templates/instances/openmetrics_legacy.yaml new file mode 100644 index 0000000000000..8f99dbe0c9cb0 --- /dev/null +++ b/ddev/src/ddev/validation/configuration/templates/instances/openmetrics_legacy.yaml @@ -0,0 +1,5 @@ +- template: instances/openmetrics_legacy_base +- template: instances/http + overrides: + request_size.value.example: 10 +- template: instances/default diff --git a/ddev/src/ddev/validation/configuration/templates/instances/openmetrics_legacy_base.yaml b/ddev/src/ddev/validation/configuration/templates/instances/openmetrics_legacy_base.yaml new file mode 100644 index 0000000000000..37be55bd5491e --- /dev/null +++ b/ddev/src/ddev/validation/configuration/templates/instances/openmetrics_legacy_base.yaml @@ -0,0 +1,254 @@ +- name: prometheus_url + display_priority: 0 + required: true + description: The URL where your application metrics are exposed by Prometheus. + fleet_configurable: true + value: + type: string +- name: namespace + display_priority: 0 + hidden: true + description: The namespace to be prepended to all metrics. + fleet_configurable: true + value: + type: string + example: service +- name: metrics + display_priority: 0 + hidden: true + description: | + List of metrics to be fetched from the prometheus endpoint, if there's a + value it'll be renamed. This list should contain at least one metric. + fleet_configurable: true + value: + type: array + example: + - processor: cpu + - memory: mem + - io + items: + anyOf: + - type: string + - type: object + additionalProperties: + type: string +- name: prometheus_metrics_prefix + display_priority: 0 + description: Removes a given from exposed Prometheus metrics. + fleet_configurable: true + value: + type: string + example: _ + display_default: null +- name: health_service_check + display_priority: 0 + description: | + Send a service check reporting about the health of the Prometheus endpoint. + The service check is named .prometheus.health + fleet_configurable: true + value: + type: boolean + example: true +- name: label_to_hostname + display_priority: 0 + description: Override the hostname with the value of one label. + fleet_configurable: true + value: + type: string + example: