From 40dbe22ce0527c0a186eb792784b09b66eea1e47 Mon Sep 17 00:00:00 2001 From: Matej Kubinec Date: Wed, 11 Mar 2026 10:04:15 +0100 Subject: [PATCH] PMM-14852 Add option to use different engine in psmdb --- pmm_psmdb-pbm_setup/docker-compose-rs.yaml | 36 +++++++-- .../docker-compose-sharded.yaml | 45 +++++++++++ pmm_psmdb-pbm_setup/render-mongod-conf.sh | 80 +++++++++++++++++++ pmm_psmdb-pbm_setup/start-with-engine.sh | 41 ++++++++++ pmm_qa/pmm-framework.py | 7 +- pmm_qa/scripts/database_options.py | 2 +- 6 files changed, 201 insertions(+), 10 deletions(-) create mode 100755 pmm_psmdb-pbm_setup/render-mongod-conf.sh create mode 100755 pmm_psmdb-pbm_setup/start-with-engine.sh diff --git a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml index f4a89d39..70cbab8b 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-rs.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-rs.yaml @@ -34,6 +34,7 @@ services: volumes: - ./conf/pbm:/etc/pbm - ./conf/mongod-rs:/etc/mongod + - ./render-mongod-conf.sh:/usr/local/bin/render-mongod-conf.sh:ro - ./conf/datagen:/etc/datagen:ro - /sys/fs/cgroup:/sys/fs/cgroup:rw - /tmp/backup_data:/tmp/backup_data:rw @@ -46,9 +47,12 @@ services: PMM_AGENT_SERVER_USERNAME: admin PMM_AGENT_SERVER_PASSWORD: ${ADMIN_PASSWORD:-password} PMM_AGENT_SERVER_INSECURE_TLS: 1 + ENGINE: ${ENGINE:-wiredTiger} + REPLSET_NAME: rs + SHARD_ROLE: "" container_name: rs101 hostname: rs101 - entrypoint: bash -c "chown -R mongod:mongod /keytabs && exec /usr/sbin/init" + entrypoint: bash -c "bash /usr/local/bin/render-mongod-conf.sh && chown -R mongod:mongod /keytabs && exec /usr/sbin/init" rs102: depends_on: @@ -67,6 +71,7 @@ services: volumes: - ./conf/pbm:/etc/pbm - ./conf/mongod-rs:/etc/mongod + - ./render-mongod-conf.sh:/usr/local/bin/render-mongod-conf.sh:ro - /sys/fs/cgroup:/sys/fs/cgroup:rw - /tmp/backup_data:/tmp/backup_data:rw - keytabs:/keytabs @@ -78,9 +83,12 @@ services: PMM_AGENT_SERVER_USERNAME: admin PMM_AGENT_SERVER_PASSWORD: ${ADMIN_PASSWORD:-password} PMM_AGENT_SERVER_INSECURE_TLS: 1 + ENGINE: ${ENGINE:-wiredTiger} + REPLSET_NAME: rs + SHARD_ROLE: "" container_name: rs102 hostname: rs102 - entrypoint: bash -c "chown -R mongod:mongod /keytabs && exec /usr/sbin/init" + entrypoint: bash -c "bash /usr/local/bin/render-mongod-conf.sh && chown -R mongod:mongod /keytabs && exec /usr/sbin/init" rs103: depends_on: @@ -99,6 +107,7 @@ services: volumes: - ./conf/pbm:/etc/pbm - ./conf/mongod-rs:/etc/mongod + - ./render-mongod-conf.sh:/usr/local/bin/render-mongod-conf.sh:ro - /sys/fs/cgroup:/sys/fs/cgroup:rw - /tmp/backup_data:/tmp/backup_data:rw - keytabs:/keytabs @@ -110,9 +119,12 @@ services: PMM_AGENT_SERVER_USERNAME: admin PMM_AGENT_SERVER_PASSWORD: ${ADMIN_PASSWORD:-password} PMM_AGENT_SERVER_INSECURE_TLS: 1 + ENGINE: ${ENGINE:-wiredTiger} + REPLSET_NAME: rs + SHARD_ROLE: "" container_name: rs103 hostname: rs103 - entrypoint: bash -c "chown -R mongod:mongod /keytabs && exec /usr/sbin/init" + entrypoint: bash -c "bash /usr/local/bin/render-mongod-conf.sh && chown -R mongod:mongod /keytabs && exec /usr/sbin/init" rs201: depends_on: @@ -133,6 +145,7 @@ services: volumes: - ./conf/pbm:/etc/pbm - ./conf/mongod-rs:/etc/mongod + - ./render-mongod-conf.sh:/usr/local/bin/render-mongod-conf.sh:ro - /sys/fs/cgroup:/sys/fs/cgroup:rw - /tmp/backup_data:/tmp/backup_data:rw - keytabs:/keytabs @@ -144,9 +157,12 @@ services: PMM_AGENT_SERVER_USERNAME: admin PMM_AGENT_SERVER_PASSWORD: ${ADMIN_PASSWORD:-password} PMM_AGENT_SERVER_INSECURE_TLS: 1 + ENGINE: ${ENGINE:-wiredTiger} + REPLSET_NAME: rs + SHARD_ROLE: "" container_name: rs201 hostname: rs201 - entrypoint: bash -c "chown -R mongod:mongod /keytabs && exec /usr/sbin/init" + entrypoint: bash -c "bash /usr/local/bin/render-mongod-conf.sh && chown -R mongod:mongod /keytabs && exec /usr/sbin/init" rs202: depends_on: @@ -165,6 +181,7 @@ services: volumes: - ./conf/pbm:/etc/pbm - ./conf/mongod-rs:/etc/mongod + - ./render-mongod-conf.sh:/usr/local/bin/render-mongod-conf.sh:ro - /sys/fs/cgroup:/sys/fs/cgroup:rw - /tmp/backup_data:/tmp/backup_data:rw - keytabs:/keytabs @@ -176,9 +193,12 @@ services: PMM_AGENT_SERVER_USERNAME: admin PMM_AGENT_SERVER_PASSWORD: ${ADMIN_PASSWORD:-password} PMM_AGENT_SERVER_INSECURE_TLS: 1 + ENGINE: ${ENGINE:-wiredTiger} + REPLSET_NAME: rs + SHARD_ROLE: "" container_name: rs202 hostname: rs202 - entrypoint: bash -c "chown -R mongod:mongod /keytabs && exec /usr/sbin/init" + entrypoint: bash -c "bash /usr/local/bin/render-mongod-conf.sh && chown -R mongod:mongod /keytabs && exec /usr/sbin/init" rs203: depends_on: @@ -197,6 +217,7 @@ services: volumes: - ./conf/pbm:/etc/pbm - ./conf/mongod-rs:/etc/mongod + - ./render-mongod-conf.sh:/usr/local/bin/render-mongod-conf.sh:ro - /sys/fs/cgroup:/sys/fs/cgroup:rw - /tmp/backup_data:/tmp/backup_data:rw - keytabs:/keytabs @@ -208,9 +229,12 @@ services: PMM_AGENT_SERVER_USERNAME: admin PMM_AGENT_SERVER_PASSWORD: ${ADMIN_PASSWORD:-password} PMM_AGENT_SERVER_INSECURE_TLS: 1 + ENGINE: ${ENGINE:-wiredTiger} + REPLSET_NAME: rs + SHARD_ROLE: "" container_name: rs203 hostname: rs203 - entrypoint: bash -c "chown -R mongod:mongod /keytabs && exec /usr/sbin/init" + entrypoint: bash -c "bash /usr/local/bin/render-mongod-conf.sh && chown -R mongod:mongod /keytabs && exec /usr/sbin/init" minio: image: minio/minio diff --git a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml index 4f4598a5..a81ca2b5 100644 --- a/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml +++ b/pmm_psmdb-pbm_setup/docker-compose-sharded.yaml @@ -20,6 +20,7 @@ services: volumes: - ./conf/pbm:/etc/pbm - ./conf/mongod-rs1:/etc/mongod + - ./render-mongod-conf.sh:/usr/local/bin/render-mongod-conf.sh:ro - /sys/fs/cgroup:/sys/fs/cgroup:rw - /tmp/mongodb:/tmp privileged: true @@ -30,8 +31,12 @@ services: - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=${ADMIN_PASSWORD:-password}" - "PMM_AGENT_SERVER_INSECURE_TLS=1" + - "ENGINE=${ENGINE:-wiredTiger}" + - "REPLSET_NAME=rs1" + - "SHARD_ROLE=shardsvr" container_name: rs101 hostname: rs101 + entrypoint: bash -c "bash /usr/local/bin/render-mongod-conf.sh && exec /usr/sbin/init" ports: - "27027:27017" networks: @@ -44,6 +49,7 @@ services: volumes: - ./conf/pbm:/etc/pbm - ./conf/mongod-rs1:/etc/mongod + - ./render-mongod-conf.sh:/usr/local/bin/render-mongod-conf.sh:ro - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true cgroup: host @@ -53,8 +59,12 @@ services: - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=${ADMIN_PASSWORD:-password}" - "PMM_AGENT_SERVER_INSECURE_TLS=1" + - "ENGINE=${ENGINE:-wiredTiger}" + - "REPLSET_NAME=rs1" + - "SHARD_ROLE=shardsvr" container_name: rs102 hostname: rs102 + entrypoint: bash -c "bash /usr/local/bin/render-mongod-conf.sh && exec /usr/sbin/init" networks: - test-network @@ -65,6 +75,7 @@ services: volumes: - ./conf/pbm:/etc/pbm - ./conf/mongod-rs1:/etc/mongod + - ./render-mongod-conf.sh:/usr/local/bin/render-mongod-conf.sh:ro - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true cgroup: host @@ -74,8 +85,12 @@ services: - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=${ADMIN_PASSWORD:-password}" - "PMM_AGENT_SERVER_INSECURE_TLS=1" + - "ENGINE=${ENGINE:-wiredTiger}" + - "REPLSET_NAME=rs1" + - "SHARD_ROLE=shardsvr" container_name: rs103 hostname: rs103 + entrypoint: bash -c "bash /usr/local/bin/render-mongod-conf.sh && exec /usr/sbin/init" networks: - test-network @@ -86,6 +101,7 @@ services: volumes: - ./conf/pbm:/etc/pbm - ./conf/mongod-rs2:/etc/mongod + - ./render-mongod-conf.sh:/usr/local/bin/render-mongod-conf.sh:ro - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true cgroup: host @@ -95,8 +111,12 @@ services: - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=${ADMIN_PASSWORD:-password}" - "PMM_AGENT_SERVER_INSECURE_TLS=1" + - "ENGINE=${ENGINE:-wiredTiger}" + - "REPLSET_NAME=rs2" + - "SHARD_ROLE=shardsvr" container_name: rs201 hostname: rs201 + entrypoint: bash -c "bash /usr/local/bin/render-mongod-conf.sh && exec /usr/sbin/init" networks: - test-network @@ -107,6 +127,7 @@ services: volumes: - ./conf/pbm:/etc/pbm - ./conf/mongod-rs2:/etc/mongod + - ./render-mongod-conf.sh:/usr/local/bin/render-mongod-conf.sh:ro - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true cgroup: host @@ -116,8 +137,12 @@ services: - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=${ADMIN_PASSWORD:-password}" - "PMM_AGENT_SERVER_INSECURE_TLS=1" + - "ENGINE=${ENGINE:-wiredTiger}" + - "REPLSET_NAME=rs2" + - "SHARD_ROLE=shardsvr" container_name: rs202 hostname: rs202 + entrypoint: bash -c "bash /usr/local/bin/render-mongod-conf.sh && exec /usr/sbin/init" networks: - test-network @@ -128,6 +153,7 @@ services: volumes: - ./conf/pbm:/etc/pbm - ./conf/mongod-rs2:/etc/mongod + - ./render-mongod-conf.sh:/usr/local/bin/render-mongod-conf.sh:ro - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true cgroup: host @@ -137,8 +163,12 @@ services: - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=${ADMIN_PASSWORD:-password}" - "PMM_AGENT_SERVER_INSECURE_TLS=1" + - "ENGINE=${ENGINE:-wiredTiger}" + - "REPLSET_NAME=rs2" + - "SHARD_ROLE=shardsvr" container_name: rs203 hostname: rs203 + entrypoint: bash -c "bash /usr/local/bin/render-mongod-conf.sh && exec /usr/sbin/init" networks: - test-network @@ -149,6 +179,7 @@ services: volumes: - ./conf/pbm:/etc/pbm - ./conf/mongod-cfg:/etc/mongod + - ./render-mongod-conf.sh:/usr/local/bin/render-mongod-conf.sh:ro - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true cgroup: host @@ -158,8 +189,12 @@ services: - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=${ADMIN_PASSWORD:-password}" - "PMM_AGENT_SERVER_INSECURE_TLS=1" + - "ENGINE=${ENGINE:-wiredTiger}" + - "REPLSET_NAME=rscfg" + - "SHARD_ROLE=configsvr" container_name: rscfg01 hostname: rscfg01 + entrypoint: bash -c "bash /usr/local/bin/render-mongod-conf.sh && exec /usr/sbin/init" networks: - test-network @@ -170,6 +205,7 @@ services: volumes: - ./conf/pbm:/etc/pbm - ./conf/mongod-cfg:/etc/mongod + - ./render-mongod-conf.sh:/usr/local/bin/render-mongod-conf.sh:ro - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true cgroup: host @@ -179,8 +215,12 @@ services: - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=${ADMIN_PASSWORD:-password}" - "PMM_AGENT_SERVER_INSECURE_TLS=1" + - "ENGINE=${ENGINE:-wiredTiger}" + - "REPLSET_NAME=rscfg" + - "SHARD_ROLE=configsvr" container_name: rscfg02 hostname: rscfg02 + entrypoint: bash -c "bash /usr/local/bin/render-mongod-conf.sh && exec /usr/sbin/init" networks: - test-network @@ -191,6 +231,7 @@ services: volumes: - ./conf/pbm:/etc/pbm - ./conf/mongod-cfg:/etc/mongod + - ./render-mongod-conf.sh:/usr/local/bin/render-mongod-conf.sh:ro - /sys/fs/cgroup:/sys/fs/cgroup:rw privileged: true cgroup: host @@ -200,8 +241,12 @@ services: - "PMM_AGENT_SERVER_USERNAME=admin" - "PMM_AGENT_SERVER_PASSWORD=${ADMIN_PASSWORD:-password}" - "PMM_AGENT_SERVER_INSECURE_TLS=1" + - "ENGINE=${ENGINE:-wiredTiger}" + - "REPLSET_NAME=rscfg" + - "SHARD_ROLE=configsvr" container_name: rscfg03 hostname: rscfg03 + entrypoint: bash -c "bash /usr/local/bin/render-mongod-conf.sh && exec /usr/sbin/init" networks: - test-network diff --git a/pmm_psmdb-pbm_setup/render-mongod-conf.sh b/pmm_psmdb-pbm_setup/render-mongod-conf.sh new file mode 100755 index 00000000..f49e27d6 --- /dev/null +++ b/pmm_psmdb-pbm_setup/render-mongod-conf.sh @@ -0,0 +1,80 @@ +#!/bin/bash +set -euo pipefail + +engine_raw="${ENGINE:-wiredTiger}" +replset_name="${REPLSET_NAME:-rs}" +shard_role="${SHARD_ROLE:-}" + +engine="$(echo "${engine_raw}" | tr '[:upper:]' '[:lower:]')" +target_conf="/tmp/mongod.dynamic.conf" +sysconfig_file="/etc/sysconfig/mongod" + +case "${engine}" in + wiredtiger|wired_tiger) + storage_block="$(cat <<'EOF' +storage: + dbPath: /var/lib/mongo + journal: + enabled: true + wiredTiger: + engineConfig: + cacheSizeGB: 1 +EOF +)" + ;; + inmemory|in_memory) + storage_block="$(cat <<'EOF' +storage: + dbPath: /var/lib/mongo + engine: inMemory + inMemory: + engineConfig: + inMemorySizeGB: 1 +EOF +)" + ;; + *) + echo "Unsupported ENGINE '${engine_raw}'. Expected: wiredTiger or inmemory." >&2 + exit 1 + ;; +esac + +{ + printf "%s\n\n" "${storage_block}" + cat < "${target_conf}" + +if [[ -f "${sysconfig_file}" ]]; then + sed -i "s|^OPTIONS=.*|OPTIONS=\"-f ${target_conf}\"|" "${sysconfig_file}" +else + echo "OPTIONS=\"-f ${target_conf}\"" > "${sysconfig_file}" +fi diff --git a/pmm_psmdb-pbm_setup/start-with-engine.sh b/pmm_psmdb-pbm_setup/start-with-engine.sh new file mode 100755 index 00000000..c2ad07b3 --- /dev/null +++ b/pmm_psmdb-pbm_setup/start-with-engine.sh @@ -0,0 +1,41 @@ +#!/bin/bash +set -euo pipefail + +engine_raw="${ENGINE:-wiredTiger}" +setup_type_raw="${MONGO_SETUP_TYPE:-pss}" + +engine="$(echo "${engine_raw}" | tr '[:upper:]' '[:lower:]')" +setup_type="$(echo "${setup_type_raw}" | tr '[:upper:]' '[:lower:]')" + +base_script="" + +case "${setup_type}" in + pss|psa) + base_script="start-rs-only.sh" + ;; + shards|sharding) + base_script="start-sharded-no-server.sh" + ;; + *) + echo "Unsupported MONGO_SETUP_TYPE '${setup_type_raw}'. Expected: pss, psa, shards, sharding." + exit 1 + ;; +esac + +if [[ ! -f "${base_script}" ]]; then + echo "Required setup script '${base_script}' not found in $(pwd)." + if [[ "${setup_type}" == "shards" || "${setup_type}" == "sharding" ]]; then + echo "For sharding, generate '${base_script}' first using the existing pmm framework flow." + fi + exit 1 +fi + +case "${engine}" in + wiredtiger|wired_tiger|inmemory|in_memory) + exec bash "./${base_script}" "$@" + ;; + *) + echo "Unsupported ENGINE '${engine_raw}'. Expected: wiredTiger or inmemory." + exit 1 + ;; +esac diff --git a/pmm_qa/pmm-framework.py b/pmm_qa/pmm-framework.py index 083e280e..a4b3f5d5 100755 --- a/pmm_qa/pmm-framework.py +++ b/pmm_qa/pmm-framework.py @@ -517,6 +517,7 @@ def setup_psmdb(db_type, db_version=None, db_config=None, args=None): 'PMM_CLIENT_VERSION': get_value('CLIENT_VERSION', db_type, args, db_config), 'COMPOSE_PROFILES': get_value('COMPOSE_PROFILES', db_type, args, db_config), 'MONGO_SETUP_TYPE': get_value('SETUP_TYPE', db_type, args, db_config), + 'ENGINE': get_value('ENGINE', db_type, args, db_config), 'OL_VERSION': get_value('OL_VERSION', db_type, args, db_config), 'GSSAPI': get_value('GSSAPI', db_type, args, db_config), 'TESTS': 'no', @@ -528,10 +529,10 @@ def setup_psmdb(db_type, db_version=None, db_config=None, args=None): setup_type = get_value('SETUP_TYPE', db_type, args, db_config).lower() if setup_type in ("pss", "psa"): - shell_scripts = ['start-rs-only.sh'] + shell_scripts = ['start-with-engine.sh'] elif setup_type in ("shards", "sharding"): - shell_scripts = ['start-sharded-no-server.sh'] - mongo_sharding_setup(shell_scripts[0], args) + shell_scripts = ['start-with-engine.sh'] + mongo_sharding_setup('start-sharded-no-server.sh', args) # Execute shell scripts if not shell_scripts == []: diff --git a/pmm_qa/scripts/database_options.py b/pmm_qa/scripts/database_options.py index c84436be..3339d4f9 100644 --- a/pmm_qa/scripts/database_options.py +++ b/pmm_qa/scripts/database_options.py @@ -2,7 +2,7 @@ "PSMDB": { "versions": ["4.4", "5.0", "6.0", "7.0", "8.0", "latest"], "configurations": {"CLIENT_VERSION": "3-dev-latest", "SETUP_TYPE": "pss", "COMPOSE_PROFILES": "classic", - "TARBALL": "", "OL_VERSION": "9", "GSSAPI": "false"} + "TARBALL": "", "OL_VERSION": "9", "GSSAPI": "false", "ENGINE": "wiredTiger"} }, "MLAUNCH_PSMDB": { "versions": ["4.4", "5.0", "6.0", "7.0", "8.0"],