diff --git a/.github/workflows/python-c-ext-sanity-check.yml b/.github/workflows/python-c-ext-sanity-check.yml index becc1bf2c..de33c52f2 100644 --- a/.github/workflows/python-c-ext-sanity-check.yml +++ b/.github/workflows/python-c-ext-sanity-check.yml @@ -6,6 +6,11 @@ name: Python C-Extension Sanity Checks on: + # env.paths-ignore-splint: Don't run 'splint' on these paths + # Format: PATH1|PATH2|PATH3 + env: + - paths-ignore-splint: "mig/src/lustreclient/.*$" + # Triggers the workflow on push or pull request events but only for this git branch push: paths-ignore: @@ -24,6 +29,7 @@ on: - 'mig/apache/**' - 'mig/bin/**' - 'mig/java-bin/**' + - 'mig/src/lustreclient/**' - '**/*.py' - '**/*.js' branches: @@ -52,6 +58,7 @@ on: - 'mig/apache/**' - 'mig/bin/**' - 'mig/java-bin/**' + - 'mig/src/lustreclient/**' - '**/*.py' - '**/*.js' branches: @@ -90,9 +97,9 @@ jobs: run: | # NOTE: we only run splint error check for changed C files to limit noise # NOTE: point splint to Ubuntu's custom /usr/include/python3.x for Python.h - echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$')" + echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.paths-ignore-splint }}' | grep -E '\.c$')" # NOTE: show splint warnings but don't fail unless it found critical errors - git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list $(python3-config --includes) &> splint.log || true + git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.paths-ignore-splint }}' | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list $(python3-config --includes) &> splint.log || true [ ! -e splint.log ] || cat splint.log [ ! -e splint.log ] || ! grep -q ' Cannot continue' splint.log @@ -116,10 +123,10 @@ jobs: run: | # NOTE: we only run splint error check for changed C files to limit noise # NOTE: point splint to Ubuntu's custom /usr/include/python3.x for Python.h - echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$')" + echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.paths-ignore-splint }}' | grep -E '\.c$')" # NOTE: splint complains about NATIVE_TSS_KEY_T in system header here # NOTE: show splint warnings but don't fail unless it found critical errors - git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list -DNATIVE_TSS_KEY_T=char $(python3-config --includes) &> splint.log || true + git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.paths-ignore-splint }}' | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list -DNATIVE_TSS_KEY_T=char $(python3-config --includes) &> splint.log || true [ ! -e splint.log ] || cat splint.log [ ! -e splint.log ] || ! grep -q ' Cannot continue' splint.log @@ -156,10 +163,10 @@ jobs: # NOTE: perms are not right inside container so repeat what checkout module does. git config --global --add safe.directory "$PWD" # NOTE: we only run splint error check for changed C files to limit noise - echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$')" + echo "Lint changed code files: $(git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.paths-ignore-splint }}' | grep -E '\.c$')" echo "with splint from $(which splint)" ls -l /bin/splint # NOTE: show splint warnings but don't fail unless it found critical errors - git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list &> splint.log || true + git diff --diff-filter=ACMRTB --name-only HEAD^1 -- | grep -Ev '${{ env.paths-ignore-splint }}' | grep -E '\.c$' | xargs -r splint +posixlib -D__gnuc_va_list=va_list &> splint.log || true [ ! -e splint.log ] || cat splint.log [ ! -e splint.log ] || ! grep -q ' Cannot continue' splint.log diff --git a/mig/install/MiGserver-template.conf b/mig/install/MiGserver-template.conf index 3f02388a5..568cc737f 100644 --- a/mig/install/MiGserver-template.conf +++ b/mig/install/MiGserver-template.conf @@ -550,6 +550,7 @@ default_mount_re = SSHFS-2.X-1 [QUOTA] backend = __QUOTA_BACKEND__ +update_interval = __QUOTA_UPDATE_INTERVAL__ user_limit = __QUOTA_USER_LIMIT__ vgrid_limit = __QUOTA_VGRID_LIMIT__ diff --git a/mig/install/generateconfs.py b/mig/install/generateconfs.py index 411af14b3..2299233cb 100755 --- a/mig/install/generateconfs.py +++ b/mig/install/generateconfs.py @@ -252,6 +252,7 @@ def main(argv, _generate_confs=generate_confs, _print=print): 'seafile_seafhttp_port', 'seafile_client_port', 'seafile_quota', + 'quota_update_interval', 'quota_user_limit', 'quota_vgrid_limit', 'wwwserve_max_bytes', diff --git a/mig/install/miglustrequota-template.sh.cronjob b/mig/install/miglustrequota-template.sh.cronjob deleted file mode 100755 index 48f18050a..000000000 --- a/mig/install/miglustrequota-template.sh.cronjob +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# -# Run lustre quota for MiG servers -# -# The script depends on a miglustrequota setup -# (please refer to mig/src/pylustrequota/README). -# -# IMPORTANT: if placed in /etc/cron.X the script filename must be -# something consisting entirely of upper and lower case letters, digits, -# underscores, and hyphens. I.e. if the script name contains e.g. a period, -# '.', it will be silently ignored! -# This is a limitation on the run-parts wrapper used by cron -# (see man run-parts for the rationale behind this). - -# By default bash silently ignores and continues on most errors but we can set -# options to e.g. catch uninitialized variables and errors as explained in: -# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ -# NOTE: 'set -eE' exits on non-zero exit codes to add safety and as recommended -# best-practice (CWE-252, CWE-248, ...), yet, in some cases it hurts more to -# exit midway, so it can be a trade-off. -set -eEuo pipefail - -# Send output to another email address -#MAILTO="root" - -MIG_CONF=__MIG_CODE__/server/MiGserver.conf - -# Specify if migrid runs natively or inside containers with lustre at host. -# Value is the container manager (docker, podman, or empty string for none) -container_manager="" -container="migrid-lustre-quota" - -# Look in miglustrequota install dir first -export PATH="/usr/local/bin:${PATH}" - -if [[ $(id -u) -ne 0 ]]; then - echo "Please run $0 as root" - exit 1 -fi - -if [ -z "${container_manager}" ]; then - miglustrequota=$(which "miglustrequota.py" 2>/dev/null) - if [ ! -x "${miglustrequota}" ]; then - echo "ERROR: Missing miglustrequota.py" - exit 1 - fi - quota_cmd="${miglustrequota} -c ${MIG_CONF}" -else - check_cmd="${container_manager} container ls -a | grep -q '${container}'" - eval "$check_cmd" - ret=$? - if [ "$ret" -ne 0 ]; then - echo "ERROR: Missing ${container} container" - exit 1 - fi - quota_cmd="${container_manager} start -a ${container}" -fi - -eval "$quota_cmd" -ret=$? - -exit $ret diff --git a/mig/install/migrid-init.d-deb-template b/mig/install/migrid-init.d-deb-template index d6ac02101..87351aa07 100755 --- a/mig/install/migrid-init.d-deb-template +++ b/mig/install/migrid-init.d-deb-template @@ -43,6 +43,9 @@ if [ -z "$PYTHONPATH" ]; then else PYTHONPATH=${MIG_PATH}:$PYTHONPATH fi +# Make sure '/usr/local/(s)bin' is in path and force lookup order +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + # you probably do not want to modify these... PID_DIR=${PID_DIR:-/var/run} MIG_LOG=${MIG_STATE}/log @@ -62,13 +65,14 @@ MIG_FTPS=${MIG_CODE}/server/grid_ftps.py MIG_NOTIFY=${MIG_CODE}/server/grid_notify.py MIG_IMNOTIFY=${MIG_CODE}/server/grid_imnotify.py MIG_VMPROXY=${MIG_CODE}/server/grid_vmproxy.py +MIG_QUOTA=${MIG_CODE}/server/grid_quota.py MIG_CHKUSERROOT=${MIG_CODE}/server/chkuserroot.py MIG_CHKSIDROOT=${MIG_CODE}/server/chksidroot.py show_usage() { echo "Usage: migrid {start|stop|status|restart|reload}[daemon DAEMON]" echo "where daemon is left out for all or given along with DAEMON as one of the following" - echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all)" + echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all)" } check_enabled() { @@ -264,6 +268,18 @@ start_vmproxy() { log_end_msg 1 || true fi } +start_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Starting MiG quota daemon" ${SHORT_NAME} || true + if start-stop-daemon --start --quiet --oknodo --pidfile ${PID_FILE} --make-pidfile --user root --chuid root --background --name ${SHORT_NAME} --startas ${DAEMON_PATH} ; then + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} start_sftpsubsys() { check_enabled "sftp_subsys" || return 0 DAEMON_PATH=${MIG_SFTPSUBSYS} @@ -292,6 +308,7 @@ start_all() { start_notify start_imnotify start_vmproxy + start_quota return 0 } @@ -524,6 +541,19 @@ stop_vmproxy() { log_end_msg 1 || true fi } +stop_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Stopping MiG quota" ${SHORT_NAME} || true + if start-stop-daemon --stop --quiet --oknodo --pidfile ${PID_FILE} ; then + rm -f ${PID_FILE} + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} stop_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -563,6 +593,7 @@ stop_all() { stop_notify stop_imnotify stop_vmproxy + stop_quota return 0 } @@ -735,6 +766,18 @@ reload_vmproxy() { log_end_msg 1 || true fi } +reload_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Reloading MiG quota" ${SHORT_NAME} || true + if start-stop-daemon --stop --signal HUP --quiet --oknodo --pidfile ${PID_FILE} ; then + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} reload_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -787,6 +830,7 @@ reload_all() { reload_notify reload_imnotify reload_vmproxy + reload_quota # Apache helpers to verify proper chrooting reload_chkuserroot reload_chksidroot @@ -891,6 +935,13 @@ status_vmproxy() { PID_FILE="$PID_DIR/${SHORT_NAME}.pid" status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME} } +status_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME} +} status_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -929,6 +980,7 @@ status_all() { status_notify status_imnotify status_vmproxy + status_quota return 0 } @@ -940,7 +992,7 @@ test -f ${MIG_SCRIPT} || exit 0 # Force valid target case "$2" in - script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all) + script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all) TARGET="$2" ;; '') diff --git a/mig/install/migrid-init.d-rh-template b/mig/install/migrid-init.d-rh-template index c4bfad648..911c5bb3c 100755 --- a/mig/install/migrid-init.d-rh-template +++ b/mig/install/migrid-init.d-rh-template @@ -35,6 +35,7 @@ # processname: grid_notify.py # processname: grid_imnotify.py # processname: grid_vmproxy.py +# processname: grid_quota.py # processname: sshd # config: /etc/sysconfig/migrid # @@ -74,6 +75,9 @@ if [ -z "$PYTHONPATH" ]; then else export PYTHONPATH=${MIG_PATH}:$PYTHONPATH fi +# Make sure '/usr/local/(s)bin' is in path and force lookup order +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + # you probably do not want to modify these... PID_DIR=${PID_DIR:-/var/run} MIG_LOG=${MIG_STATE}/log @@ -93,13 +97,14 @@ MIG_FTPS=${MIG_CODE}/server/grid_ftps.py MIG_NOTIFY=${MIG_CODE}/server/grid_notify.py MIG_IMNOTIFY=${MIG_CODE}/server/grid_imnotify.py MIG_VMPROXY=${MIG_CODE}/server/grid_vmproxy.py +MIG_QUOTA=${MIG_CODE}/server/grid_quota.py MIG_CHKUSERROOT=${MIG_CODE}/server/chkuserroot.py MIG_CHKSIDROOT=${MIG_CODE}/server/chksidroot.py show_usage() { echo "Usage: migrid {start|stop|status|restart|reload}[daemon DAEMON]" echo "where daemon is left out for all or given along with DAEMON as one of the following" - echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all)" + echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all)" } check_enabled() { @@ -353,6 +358,21 @@ start_vmproxy() { [ $RET2 -ne 0 ] && echo "Warning: vmproxy not started." echo } +start_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Starting MiG quota daemon: $SHORT_NAME" + daemon --user root --pidfile ${PID_FILE} \ + "${DAEMON_PATH} >> ${MIG_LOG}/quota.out 2>&1 &" + fallback_save_pid "$DAEMON_PATH" "$PID_FILE" "$!" + RET2=$? + [ $RET2 -eq 0 ] && success + echo + [ $RET2 -ne 0 ] && echo "Warning: quota not started." + echo +} start_sftpsubsys() { check_enabled "sftp_subsys" || return DAEMON_PATH=${MIG_SFTPSUBSYS} @@ -385,6 +405,7 @@ start_all() { start_notify start_imnotify start_vmproxy + start_quota return 0 } @@ -545,6 +566,15 @@ stop_vmproxy() { killproc ${DAEMON_PATH} echo } +stop_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Shutting down MiG quota: $SHORT_NAME " + killproc ${DAEMON_PATH} + echo +} stop_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -588,6 +618,7 @@ stop_all() { stop_notify stop_imnotify stop_vmproxy + stop_quota return 0 } @@ -717,6 +748,15 @@ reload_vmproxy() { killproc ${DAEMON_PATH} -HUP echo } +reload_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Reloading MiG quota: $SHORT_NAME " + killproc ${DAEMON_PATH} -HUP + echo +} reload_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -773,6 +813,7 @@ reload_all() { reload_notify reload_imnotify reload_vmproxy + reload_quota # Apache helpers to verify proper chrooting reload_chkuserroot reload_chksidroot @@ -877,6 +918,13 @@ status_vmproxy() { PID_FILE="$PID_DIR/${SHORT_NAME}.pid" status ${DAEMON_PATH} } +status_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + status ${DAEMON_PATH} +} status_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -916,6 +964,7 @@ status_all() { status_notify status_imnotify status_vmproxy + status_quota return 0 } @@ -927,7 +976,7 @@ test -f ${MIG_SCRIPT} || exit 0 # Force valid target case "$2" in - script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all) + script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all) TARGET="$2" ;; '') diff --git a/mig/src/pylustrequota/bin/miglustrequota.py b/mig/lib/lustrequota.py old mode 100755 new mode 100644 similarity index 56% rename from mig/src/pylustrequota/bin/miglustrequota.py rename to mig/lib/lustrequota.py index b471bce4a..d335554d3 --- a/mig/src/pylustrequota/bin/miglustrequota.py +++ b/mig/lib/lustrequota.py @@ -3,8 +3,8 @@ # # --- BEGIN_HEADER --- # -# mig_lustre_quota - MiG lustre quota manager -# Copyright (C) 2003-2025 The MiG Project lead by the Science HPC Center at UCPH +# lustrequota - helpers to support lustre quota +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH # # This file is part of MiG. # @@ -26,66 +26,71 @@ # --- END_HEADER --- # -"""Assign lustre project id's to new users and vgrids, -set default quota on new entries and update existing quotas if changed. -Fetch the number of files and bytes used by each project id. -""" +"""helpers to support lustre quota""" import os -import sys -import time import stat -import getopt +import time import shlex import subprocess -import psutil + +# NOTE: we rely on psutil to resolve lustre mount point +try: + import psutil +except ImportError: + psutil = None from mig.shared.base import force_unicode -from mig.shared.conf import get_configuration_object from mig.shared.fileio import unpickle, pickle, save_json, makedirs_rec, \ make_symlink -from mig.shared.logger import daemon_logger - -from pylustrequota.lfs import lfs_set_project_id, lfs_get_project_quota, \ - lfs_set_project_quota - - -supported_quota_backends = ['lustre', 'lustre-gocryptfs'] +try: + from lustreclient.lfs import lfs_set_project_id, lfs_get_project_quota, \ + lfs_set_project_quota +except ImportError: + lfs_set_project_id = None + lfs_get_project_quota = None + lfs_set_project_quota = None -def usage(name=sys.argv[0]): - """Usage help""" - msg = """Usage: %(name)s [OPTIONS] -Where OPTIONS may be one or more of: - -h|--help Show this help - -v|--verbose Verbose output - -q|--quiet No stdout/stderr output - -c PATH|--config=PATH Path to config file - -l PATH|--lustre-basepath Path to lustre base - -g PATH|--gocryptfs-sock Path to gocryptfs socket -""" % {'name': name} - print(msg, file=sys.stderr) +def __get_lustre_basepath(configuration, lustre_basepath=None): + """If *lustre_basepath* is provided then check it, + otherwise try to resolve it""" + if psutil is None: + return None + valid_lustre_basepath = None + for mount in psutil.disk_partitions(all=True): + if mount.fstype == "lustre": + if lustre_basepath \ + and lustre_basepath.startswith(mount.mountpoint) \ + and os.path.isdir(lustre_basepath): + valid_lustre_basepath = lustre_basepath + break + elif mount.mountpoint.endswith(configuration.server_fqdn): + valid_lustre_basepath = mount.mountpoint + else: + check_lustre_basepath = os.path.join(mount.mountpoint, + configuration.server_fqdn) + if os.path.isdir(check_lustre_basepath): + valid_lustre_basepath = check_lustre_basepath + break -def INFO(configuration, msg, verbose=False): - """log info and print to stdout on verbose""" - configuration.logger.info(msg) - if verbose: - print(msg) - + return valid_lustre_basepath -def ERROR(configuration, msg, quiet=False): - """log error and print to stderr on verbose""" - configuration.logger.error(msg) - if not quiet: - print("ERROR: %s" % msg, file=sys.stderr) +def __get_gocryptfs_socket(configuration, gocryptfs_sock=None): + """If *gocryptfs_sock* is provided then check it, + otherwise return default if it exists""" + valid_gocryptfs_sock = None + if gocryptfs_sock is None: + gocryptfs_sock = "/var/run/gocryptfs.%s.sock" \ + % configuration.server_fqdn + if os.path.exists(gocryptfs_sock): + gocryptfs_sock_stat = os.lstat(gocryptfs_sock) + if stat.S_ISSOCK(gocryptfs_sock_stat.st_mode): + valid_gocryptfs_sock = gocryptfs_sock -def DEBUG(configuration, msg, verbose=False): - """log debug and print to stderr on verbose""" - configuration.logger.debug(msg) - if verbose and configuration.loglevel == 'debug': - print("DEBUG: %s" % msg, file=sys.stderr) + return valid_gocryptfs_sock def __shellexec(configuration, @@ -155,15 +160,65 @@ def __shellexec(configuration, return (rc, stdout, stderr) +def __set_project_id(configuration, + lustre_basepath, + quota_datapath, + quota_name, + quota_lustre_pid): + """Set lustre project *quota_lustre_pid* + Find the next *free* project id (PID) if *quota_lustre_pid* is occupied + NOTE: lustre uses a global counter for project id's (PID) + That means that different datasets and sub-mounts + share the same project id counter + # TODO: Add 'lustre_pid' offset support to configuration ? + """ + + # Find next unused lustre project id + + max_lustre_pid = 4294967294 + logger = configuration.logger + next_lustre_pid = quota_lustre_pid + while next_lustre_pid < max_lustre_pid: + (rc, currfiles, _, _, _) \ + = lfs_get_project_quota(lustre_basepath, next_lustre_pid) + if rc != 0: + logger.error("Failed to fetch quota for lustre project id: %d, %r" + % (next_lustre_pid, lustre_basepath) + + ", rc: %d" % rc) + return -1 + if currfiles == 0: + break + logger.info("Skipping project id: %d" + % next_lustre_pid + + " already registered with %d files" + % currfiles) + next_lustre_pid += 1 + + if next_lustre_pid == max_lustre_pid: + logger.error("Reached max lustre project id: %d" % max_lustre_pid) + return -1 + + # Set new project id + + logger.info("Setting lustre project id: %d for %r: %r" + % (next_lustre_pid, quota_name, quota_datapath)) + rc = lfs_set_project_id(quota_datapath, next_lustre_pid, 1) + if rc != 0: + logger.error("Failed to set lustre project id: %d for %r: %r" + % (next_lustre_pid, quota_name, quota_datapath) + + ", rc: %d" % rc) + return -1 + + return next_lustre_pid + + def __update_quota(configuration, lustre_basepath, lustre_setting, quota_name, quota_type, gocryptfs_sock, - timestamp, - verbose, - quiet): + timestamp): """Update quota for *quota_name*, if new entry then assign lustre project id and set default quota. If existing entry then update quota settings if changed @@ -173,9 +228,8 @@ def __update_quota(configuration, quota_limits_changed = False next_lustre_pid = lustre_setting.get('next_pid', -1) if next_lustre_pid == -1: - msg = "Invalid lustre quota next_pid: %d for: %r" \ - % (next_lustre_pid, quota_name) - ERROR(configuration, msg, quiet) + logger.error("Invalid lustre quota next_pid: %d for: %r" + % (next_lustre_pid, quota_name)) return False if quota_type == 'vgrid': default_quota_limit = configuration.quota_vgrid_limit @@ -197,9 +251,8 @@ def __update_quota(configuration, if os.path.exists(quota_filepath): quota = unpickle(quota_filepath, logger) if not quota: - msg = "Failed to load quota settings for: %r from %r" \ - % (quota_name, quota_filepath) - ERROR(configuration, msg, quiet) + logger.error("Failed to load quota settings for: %r from %r" + % (quota_name, quota_filepath)) return False else: quota = {'lustre_pid': next_lustre_pid, @@ -211,9 +264,8 @@ def __update_quota(configuration, quota_lustre_pid = quota.get('lustre_pid', -1) if quota_lustre_pid == -1: - msg = "Invalid quota lustre pid: %d for %r" \ - % (quota_lustre_pid, quota_name) - ERROR(configuration, msg, quiet) + logger.error("Invalid quota lustre pid: %d for %r" + % (quota_lustre_pid, quota_name)) return False # Resolve quota data path @@ -236,54 +288,46 @@ def __update_quota(configuration, quota_datapath = os.path.join(lustre_basepath, encoded_path) else: - msg = "Failed to resolve encrypted path for: %r" \ - % quota_name \ - + ", rc: %d, error: %s" \ - % (rc, stderr) - ERROR(configuration, msg, quiet) + logger.error("Failed to resolve encrypted path for: %r" + % quota_name + + ", rc: %d, error: %s" + % (rc, stderr)) return False else: - ERROR(configuration, - "Invalid quota backend: %r" % configuration.quota_backend, - quiet) + logger.error("Invalid quota backend: %r" + % configuration.quota_backend) return False # Skip non-dir entries if not os.path.isdir(quota_datapath): - msg = "Skipping non-dir entry: %r: %r" \ - % (quota_name, quota_datapath) - DEBUG(configuration, msg, verbose) + logger.debug("Skipping non-dir entry: %r: %r" + % (quota_name, quota_datapath)) return True # If new entry then set lustre project id - + new_lustre_pid = -1 if quota_lustre_pid == next_lustre_pid: - # TODO: Mask out path's from log if gocryptfs ? - msg = "Setting lustre project id: %d for %r: %r" \ - % (quota_lustre_pid, quota_name, quota_datapath) - INFO(configuration, msg) - rc = lfs_set_project_id(quota_datapath, quota_lustre_pid, 1) - if rc == 0: - lustre_setting['next_pid'] = quota_lustre_pid + 1 - else: - msg = "Failed to set lustre project id: %d for %r: %r" \ - % (quota_lustre_pid, quota_name, quota_datapath) \ - + ", rc: %d" \ - % rc - ERROR(configuration, msg, quiet) + new_lustre_pid = __set_project_id(configuration, + lustre_basepath, + quota_datapath, + quota_name, + quota_lustre_pid) + if new_lustre_pid == -1: + logger.error("Failed to set project id: %d, %r, %r" + % (new_lustre_pid, quota_name, quota_datapath)) return False + lustre_setting['next_pid'] = new_lustre_pid + 1 + quota_lustre_pid = new_lustre_pid # Get current quota values for lustre_pid (rc, currfiles, currbytes, softlimit_bytes, hardlimit_bytes) \ = lfs_get_project_quota(quota_datapath, quota_lustre_pid) if rc != 0: - msg = "Failed to fetch quota for lustre project id: %d, %r, %r" \ - % (quota_lustre_pid, quota_name, quota_datapath) \ - + ", rc: %d" \ - % rc - ERROR(configuration, msg, quiet) + logger.error("Failed to fetch quota for lustre project id: %d, %r, %r" + % (quota_lustre_pid, quota_name, quota_datapath) + + ", rc: %d" % rc) return False # Update quota info @@ -295,7 +339,7 @@ def __update_quota(configuration, # If new entry use default quota # and update quota if changed - if quota_lustre_pid == next_lustre_pid: + if new_lustre_pid > -1: quota_limits_changed = True quota['softlimit_bytes'] = default_quota_limit quota['hardlimit_bytes'] = default_quota_limit @@ -312,15 +356,14 @@ def __update_quota(configuration, quota['hardlimit_bytes'], ) if rc != 0: - msg = "Failed to set quota limit: %d/%d" \ - % (softlimit_bytes, - hardlimit_bytes) \ - + " for lustre project id: %d, %r, %r, rc: %d" \ - % (quota_lustre_pid, - quota_name, - quota_datapath, - rc) - ERROR(configuration, msg, quiet) + logger.error("Failed to set quota limit: %d/%d" + % (softlimit_bytes, + hardlimit_bytes) + + " for lustre project id: %d, %r, %r, rc: %d" + % (quota_lustre_pid, + quota_name, + quota_datapath, + rc)) return False # Save current quota @@ -331,28 +374,26 @@ def __update_quota(configuration, str(timestamp)) if not os.path.exists(new_quota_basepath) \ and not makedirs_rec(new_quota_basepath, configuration): - msg = "Failed to create new quota base path: %r" \ - % new_quota_basepath - ERROR(configuration, msg, quiet) + logger.error("Failed to create new quota base path: %r" + % new_quota_basepath) return False new_quota_filepath_pck = os.path.join(new_quota_basepath, "%s.pck" % quota_name) status = pickle(quota, new_quota_filepath_pck, logger) if not status: - msg = "Failed to save quota for: %r to %r" \ - % (quota_name, new_quota_filepath_pck) - ERROR(configuration, msg, quiet) + logger.error("Failed to save quota for: %r to %r" + % (quota_name, new_quota_filepath_pck)) return False + new_quota_filepath_json = os.path.join(new_quota_basepath, "%s.json" % quota_name) status = save_json(quota, new_quota_filepath_json, logger) if not status: - msg = "Failed to save quota for: %r to %r" \ - % (quota_name, new_quota_filepath_json) - ERROR(configuration, msg, quiet) + logger.error("Failed to save quota for: %r to %r" + % (quota_name, new_quota_filepath_json)) return False # Create symlink to new quota @@ -362,24 +403,50 @@ def __update_quota(configuration, logger, force=True) if not status: - msg = "Failed to make quota symlink for: %r: %r -> %r" \ - % (quota_name, new_quota_filepath_pck, quota_filepath) - ERROR(configuration, msg, quiet) + logger.error("Failed to make quota symlink for: %r: %r -> %r" + % (quota_name, new_quota_filepath_pck, quota_filepath)) return False return True -def update_quota(configuration, - lustre_basepath, - gocryptfs_sock, - verbose, - quiet): - """Update lustre quotas for users and vgrids""" +def update_lustre_quota(configuration): + """Update lustre quota for users and vgrids""" logger = configuration.logger + + # Check if lustreclient module was imported correctly + + if lfs_set_project_id is None \ + or lfs_get_project_quota is None \ + or lfs_set_project_quota is None: + logger.error("Failed to import lustreclient module") + return False + retval = True timestamp = int(time.time()) + # Get lustre_basepath + + lustre_basepath = __get_lustre_basepath(configuration) + if lustre_basepath: + logger.debug("Using lustre basepath: %r" + % lustre_basepath) + else: + logger.error("Found no valid lustre mounts for: %s" + % configuration.server_fqdn) + return False + + # Get gocryptfs socket if enabled + + if configuration.quota_backend == "lustre-gocryptfs": + gocryptfs_sock = __get_gocryptfs_socket(configuration) + if gocryptfs_sock: + logger.debug("Using gocryptfs socket: %r" + % gocryptfs_sock) + else: + logger.error("Missing gocryptfs socket") + return False + # Load lustre quota settings lustre_setting_filepath = os.path.join(configuration.quota_home, @@ -389,14 +456,14 @@ def update_quota(configuration, lustre_setting = unpickle(lustre_setting_filepath, logger) if not lustre_setting: - msg = "Failed to load lustre quota: %r" % lustre_setting_filepath - ERROR(configuration, msg, quiet) + logger.error("Failed to load lustre quota: %r" + % lustre_setting_filepath) return False else: lustre_setting = {'next_pid': 1, 'mtime': 0} - # Update quotas + # Update quota for quota_type in ('vgrid', 'user'): if quota_type == 'vgrid': @@ -410,8 +477,7 @@ def update_quota(configuration, for entry in it: if not os.path.isdir(entry.path): # Only take dirs into account - msg = "Skiping non-dir path: %r" % entry.path - DEBUG(configuration, msg, verbose) + logger.debug("Skiping non-dir path: %r" % entry.path) continue status = __update_quota(configuration, lustre_basepath, @@ -419,10 +485,7 @@ def update_quota(configuration, entry.name, quota_type, gocryptfs_sock, - timestamp, - verbose, - quiet, - ) + timestamp) if not status: retval = False @@ -433,135 +496,7 @@ def update_quota(configuration, lustre_setting_filepath, logger) if not status: - msg = "Failed to save lustra quota settings: %r" \ - % lustre_setting_filepath - ERROR(configuration, msg, quiet) - - return retval - - -def main(): - retval = True - verbose = False - quiet = False - config_file = None - lustre_basepath = None - gocryptfs_sock = None - try: - opts, args = getopt.getopt(sys.argv[1:], "hvqc:l:g:", - ["help", "verbose", "quiet", "config=", - "--lustre-basepath", "--gocryptfs-sock="]) - for opt, arg in opts: - if opt in ("-h", "--help"): - usage() - sys.exit() - elif opt in ("-v", "--verbose"): - verbose = True - elif opt in ("-q", "--quiet"): - quiet = True - elif opt in ("-c", "--config"): - config_file = arg - elif opt in ("-l", "--lustre-basepath"): - lustre_basepath = arg - elif opt in ("-g", "--gocryptfs-sock"): - gocryptfs_sock = arg - except Exception as err: - print(err, file=sys.stderr) - usage() - return 1 - - if quiet: - verbose = False - - # Initialize configuration - - try: - configuration = get_configuration_object(config_file=config_file) - except Exception as err: - print(err, file=sys.stderr) - usage() - return 1 - - # Use separate logger - - logger = daemon_logger("quota", - configuration.user_quota_log, - configuration.loglevel) - configuration.logger = logger - if configuration.quota_backend not in supported_quota_backends: - msg = "Quota backend: %s not in supported backends: %s" \ - % (configuration.quota_backend, - ", ".join(supported_quota_backends)) - ERROR(configuration, msg, quiet) - return False - - # If lustre_basepath is provided then check it, - # otherwise try to resolve it - - valid_lustre_basepath = None - for mount in psutil.disk_partitions(all=True): - if mount.fstype == "lustre": - if lustre_basepath \ - and lustre_basepath.startswith(mount.mountpoint) \ - and os.path.isdir(lustre_basepath): - valid_lustre_basepath = lustre_basepath - break - elif mount.mountpoint.endswith(configuration.server_fqdn): - valid_lustre_basepath = mount.mountpoint - else: - check_lustre_basepath = os.path.join(mount.mountpoint, - configuration.server_fqdn) - if os.path.isdir(check_lustre_basepath): - valid_lustre_basepath = check_lustre_basepath - break - - if valid_lustre_basepath is None: - if lustre_basepath: - msg = "Lustre base: %r is NOT mounted" % lustre_basepath - else: - msg = "Found no valid lustre mounts for: %s" \ - % configuration.server_fqdn - ERROR(configuration, msg, quiet) - return False + logger.error("Failed to save lustra quota settings: %r" + % lustre_setting_filepath) - INFO(configuration, - "Using lustre basepath: %r" % valid_lustre_basepath, - verbose) - - # Check gocryptfs socket - - if configuration.quota_backend == "lustre-gocryptfs": - check_gocryptfs_sock = gocryptfs_sock - if check_gocryptfs_sock is None: - check_gocryptfs_sock = "/var/run/gocryptfs.%s.sock" \ - % configuration.server_fqdn - if os.path.exists(check_gocryptfs_sock): - gocryptfs_sock_stat = os.lstat(check_gocryptfs_sock) - if stat.S_ISSOCK(gocryptfs_sock_stat.st_mode): - gocryptfs_sock = check_gocryptfs_sock - if gocryptfs_sock: - INFO(configuration, - "Using gocryptfs socket: %r" % gocryptfs_sock, - verbose) - else: - ERROR(configuration, - "Missing gocryptfs socket: %r" % check_gocryptfs_sock, - quiet) - return False - - # Perform update - - retval = update_quota(configuration, - valid_lustre_basepath, - gocryptfs_sock, - verbose, - quiet) return retval - - -if __name__ == "__main__": - status = main() - if status: - sys.exit(0) - else: - sys.exit(1) diff --git a/mig/lib/quota.py b/mig/lib/quota.py new file mode 100644 index 000000000..e93830d28 --- /dev/null +++ b/mig/lib/quota.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# quota - helpers to support storage quota +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. +# +# --- END_HEADER --- +# + +"""helpers to support storage quota""" + +from mig.lib.lustrequota import update_lustre_quota + + +supported_quota_backends = ['lustre', 'lustre-gocryptfs'] + + +def update_quota(configuration): + """Update quota for users and vgrids""" + retval = False + logger = configuration.logger + if configuration.quota_backend == 'lustre' \ + or configuration.quota_backend == 'lustre-gocryptfs': + retval = update_lustre_quota(configuration) + else: + logger.error("quota_backend: %r not in supported_quota_backends: %r" + % (configuration.quota_backend, + supported_quota_backends)) + + return retval diff --git a/mig/server/grid_quota.py b/mig/server/grid_quota.py new file mode 120000 index 000000000..5d3c8e420 --- /dev/null +++ b/mig/server/grid_quota.py @@ -0,0 +1 @@ +../../sbin/grid_quota.py \ No newline at end of file diff --git a/mig/shared/configuration.py b/mig/shared/configuration.py index 2f872694a..8ff46f01d 100644 --- a/mig/shared/configuration.py +++ b/mig/shared/configuration.py @@ -397,6 +397,7 @@ def fix_missing(config_file, verbose=True): 'vgrid_recipes_home': '.workflow_recipes_home/', 'vgrid_history_home': '.workflow_history_home/'} quota_section = {'backend': 'lustre', + 'update_interval': 3600, 'user_limit': 1024**4, 'vgrid_limit': 1024**4} defaults = { @@ -1737,6 +1738,9 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False, if config.has_option('GLOBAL', 'user_shared_dhparams'): self.user_shared_dhparams = config.get('GLOBAL', 'user_shared_dhparams') + if config.has_option('GLOBAL', 'user_quota_log'): + self.user_quota_log = config.get('GLOBAL', + 'user_quota_log') if config.has_option('GLOBAL', 'public_key_file'): self.public_key_file = config.get('GLOBAL', 'public_key_file') if config.has_option('GLOBAL', 'smtp_sender'): @@ -2002,6 +2006,9 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False, if config.has_option('QUOTA', 'backend'): self.quota_backend = config.get( 'QUOTA', 'backend') + if config.has_option('QUOTA', 'update_interval'): + self.quota_update_interval = config.getint( + 'QUOTA', 'update_interval') if config.has_option('QUOTA', 'user_limit'): self.quota_user_limit = config.getint( 'QUOTA', 'user_limit') diff --git a/mig/shared/install.py b/mig/shared/install.py index 92a1ab997..c9579fc60 100644 --- a/mig/shared/install.py +++ b/mig/shared/install.py @@ -533,6 +533,7 @@ def generate_confs( gdp_id_scramble='safe_hash', gdp_path_scramble='safe_encrypt', quota_backend='lustre', + quota_update_interval=3600, quota_user_limit=(1024**4), quota_vgrid_limit=(1024**4), ca_fqdn='', @@ -859,6 +860,7 @@ def _generate_confs_prepare( gdp_id_scramble, gdp_path_scramble, quota_backend, + quota_update_interval, quota_user_limit, quota_vgrid_limit, ca_fqdn, @@ -1117,6 +1119,7 @@ def _generate_confs_prepare( user_dict['__PUBLIC_ALIAS_HTTPS_LISTEN__'] = listen_clause user_dict['__STATUS_ALIAS_HTTPS_LISTEN__'] = listen_clause user_dict['__QUOTA_BACKEND__'] = quota_backend + user_dict['__QUOTA_UPDATE_INTERVAL__'] = "%s" % quota_update_interval user_dict['__QUOTA_USER_LIMIT__'] = "%s" % quota_user_limit user_dict['__QUOTA_VGRID_LIMIT__'] = "%s" % quota_vgrid_limit user_dict['__CA_FQDN__'] = ca_fqdn @@ -2343,7 +2346,6 @@ def _generate_confs_writefiles(options, user_dict, insert_list=[], cleanup_list= ("migacctexpire-template.sh.cronjob", "migacctexpire"), ("migverifyarchives-template.sh.cronjob", "migverifyarchives"), ("migstats-template.sh.cronjob", "migstats"), - ("miglustrequota-template.sh.cronjob", "miglustrequota"), ] overrides_out_name = { 'apache.initd': _override_apache_initd @@ -2522,11 +2524,11 @@ def _generate_confs_instructions(options, user_dict): /etc/cron.daily/ until the janitor service is ready to take care of those tasks. -The migcheckssl, migverifyarchives, migstats, migacctexpire and miglustrequota +The migcheckssl, migverifyarchives, migstats and migacctexpire files are cron scripts to automatically check for LetsEncrypt certificate renewal, run pending archive verification before sending a copy to tape, save -various usage stats, generate account expire stats and create/update lustre -quota. +various usage stats and generate account expire stats. + You can install them with: chmod 700 %(destination)s/migcheckssl sudo cp %(destination)s/migcheckssl /etc/cron.daily/ @@ -2536,8 +2538,6 @@ def _generate_confs_instructions(options, user_dict): sudo cp %(destination)s/migstats /etc/cron.weekly/ chmod 700 %(destination)s/migacctexpire sudo cp %(destination)s/migacctexpire /etc/cron.monthly/ -chmod 700 %(destination)s/miglustrequota -sudo cp %(destination)s/miglustrequota /etc/cron.hourly/ ''' % instructions_dict instructions_path = os.path.join( diff --git a/mig/src/libpam-mig/libpam_mig.c b/mig/src/libpam-mig/libpam_mig.c index b030fb340..4ca6f89a1 100644 --- a/mig/src/libpam-mig/libpam_mig.c +++ b/mig/src/libpam-mig/libpam_mig.c @@ -32,6 +32,7 @@ * */ + #include #include #include diff --git a/mig/src/pylustrequota/README b/mig/src/lustreclient/README similarity index 83% rename from mig/src/pylustrequota/README rename to mig/src/lustreclient/README index 22be7628b..46dc4d186 100644 --- a/mig/src/pylustrequota/README +++ b/mig/src/lustreclient/README @@ -1,7 +1,7 @@ -This folder contains a module for MiG lustre quota +This folder contains a module for MiG lustre client python extension To suppert containerized MiG (where lustre is mounted outside the container) -lustre quota functionality is compiled statically into this module. +lustre client functionality is compiled statically into this module. Install lustre dependencies (Rocky 9): ====================================== @@ -18,7 +18,7 @@ dnf --enablerepo=crb install \ libnl3-devel.x86_64 \ libyaml-devel \ krb5-devel.x86_64 -git clone git://git.whamcloud.com/fs/lustre-release.git +git clone https://github.com/lustre/lustre-release cd lustre-release && git checkout ${VERSION} ; cd - cd lustre-release && sh ./autogen.sh ; cd - cd lustre-release && ./configure --disable-server --enable-quota --enable-utils --enable-gss ; cd - diff --git a/mig/src/pylustrequota/pylustrequota/__init__.py b/mig/src/lustreclient/lustreclient/__init__.py similarity index 77% rename from mig/src/pylustrequota/pylustrequota/__init__.py rename to mig/src/lustreclient/lustreclient/__init__.py index 10abf6cd7..78e0d2121 100644 --- a/mig/src/pylustrequota/pylustrequota/__init__.py +++ b/mig/src/lustreclient/lustreclient/__init__.py @@ -3,7 +3,7 @@ # # --- BEGIN_HEADER --- # -# __init__ - luste quota python extensions +# __init__ - lustre client python extension # Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH # # This file is part of MiG. @@ -24,7 +24,7 @@ # # -- END_HEADER --- # -"""This package provide luste quota functionality""" +"""This package provide lustre client functionality""" __dummy = True @@ -34,11 +34,11 @@ # All sub modules to load in case of 'from X import *' __all__ = [] - + # Collect all package information here for easy use from scripts and helpers -package_name = 'Lustre Quota Python extension' -short_name = 'pylustrequota' +package_name = 'Lustre Client Python extension' +short_name = 'lustreclient' # IMPORTANT: Please keep version in sync with doc-src/README.t2t @@ -46,18 +46,18 @@ version_suffix = '' version_string = '.'.join([str(i) for i in version_tuple]) + version_suffix package_version = '%s %s' % (package_name, version_string) -project_team = 'The MiG Project lead by Brian Vinter' -project_email = 'info@erda.dk' -maintainer_team = 'The pylustrequota maintainers' -maintainer_email = 'info@erda.dk' -project_url = 'https://github.com/ucphhpc/pylustrequota' -download_url = 'https://github.com/ucphhpc/pylustrequota/releases' +project_team = 'The MiG Project by the Science HPC Center at UCPH' +project_email = 'info@migrid.org' +maintainer_team = 'The migrid.org maintainers' +maintainer_email = 'info@migrid.org' +project_url = 'https://github.com/ucphhpc/migrid-sync' +download_url = 'https://github.com/ucphhpc/migrid-sync/releases' license_name = 'GNU GPL v2' short_desc = \ - 'Python quota extension for lustre' + 'Lustre client python extension' long_desc = \ - """Python quota extension for for lustre: -Documentation: https://github.com/ucphhpc/pylustrequota + """Lustre client python extension: +Documentation: https://github.com/ucphhpc/migrid-sync """ project_class = [ 'Development Status :: 1 - Beta', @@ -72,7 +72,6 @@ 'Python', 'Python C extensions', 'lustre', - 'rsync', ] # Requirements diff --git a/mig/src/pylustrequota/pylustrequota/lfs.c b/mig/src/lustreclient/lustreclient/lfs.c similarity index 98% rename from mig/src/pylustrequota/pylustrequota/lfs.c rename to mig/src/lustreclient/lustreclient/lfs.c index 1082d9deb..5e0b0437d 100644 --- a/mig/src/pylustrequota/pylustrequota/lfs.c +++ b/mig/src/lustreclient/lustreclient/lfs.c @@ -1,7 +1,7 @@ /* --- BEGIN_HEADER --- -lfs - Shared lustre library functions for Python lustre quota -Copyright (C) 2003-2024 The MiG Project lead by Brian Vinter +lfs - Shared lustre library functions for Python lustre client +Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH This file is part of MiG. MiG is free software: you can redistribute it and/or modify @@ -385,4 +385,4 @@ PyMODINIT_FUNC PyInit_lfs(void) { } return module; -} \ No newline at end of file +} diff --git a/mig/src/pylustrequota/setup.cfg b/mig/src/lustreclient/setup.cfg similarity index 84% rename from mig/src/pylustrequota/setup.cfg rename to mig/src/lustreclient/setup.cfg index 2118cc8e3..75012b93e 100644 --- a/mig/src/pylustrequota/setup.cfg +++ b/mig/src/lustreclient/setup.cfg @@ -1,7 +1,7 @@ # --- BEGIN_HEADER --- # -# setup.cfg - setup configuration file for python lustre quota -# Copyright (C) 2003-2024 The MiG Project lead by Brian Vinter +# setup.cfg - setup configuration file for lustre client python extension +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH # # This file is part of MiG. # diff --git a/mig/src/pylustrequota/setup.py b/mig/src/lustreclient/setup.py similarity index 86% rename from mig/src/pylustrequota/setup.py rename to mig/src/lustreclient/setup.py index 830eeca69..7693e4fd5 100644 --- a/mig/src/pylustrequota/setup.py +++ b/mig/src/lustreclient/setup.py @@ -3,8 +3,8 @@ # # --- BEGIN_HEADER --- # -# setup.py - Setup for python luste quota -# Copyright (C) 2003-2024 The MiG Project lead by Brian Vinter +# setup.py - Setup for lustre client python extension +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH # # This file is part of MiG. # @@ -27,7 +27,7 @@ from setuptools import setup, Extension -from pylustrequota import version_string, short_name, project_team, \ +from lustreclient import version_string, short_name, project_team, \ project_email, short_desc, long_desc, project_url, download_url, \ license_name, project_class, project_keywords, versioned_requires, \ project_requires, project_extras, project_platforms, maintainer_team, \ @@ -51,14 +51,13 @@ install_requires=versioned_requires, requires=project_requires, extras_require=project_extras, - scripts=['bin/miglustrequota.py', - ], - packages=['pylustrequota'], - package_dir={'pylustrequota': 'pylustrequota', + scripts=[], + packages=['lustreclient'], + package_dir={'lustreclient': 'lustreclient', }, package_data={}, ext_modules=[ - Extension('pylustrequota.lfs', + Extension('lustreclient.lfs', include_dirs=['/usr/include', '/usr/include/python3', 'lustre-release/libcfs/include', @@ -69,7 +68,7 @@ ], library_dirs=[], libraries=[], - sources=['pylustrequota/lfs.c', + sources=['lustreclient/lfs.c', 'lustre-release/lustre/utils/lfs_project.c'], extra_objects=[ 'lustre-release/lustre/utils/.libs/liblustreapi.a'], diff --git a/sbin/grid_quota.py b/sbin/grid_quota.py new file mode 100755 index 000000000..48d2d401e --- /dev/null +++ b/sbin/grid_quota.py @@ -0,0 +1,144 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# grid_quota - daemon to create storage quota +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# -- END_HEADER --- +# + +"""Daemon that create storage quota""" + +from __future__ import absolute_import, print_function + +import os +import sys +import time +import traceback +import datetime + +from mig.lib.daemon import check_run, check_stop, interruptible_sleep, \ + register_run_handler, register_stop_handler, reset_run, stop_running +from mig.lib.quota import update_quota, supported_quota_backends +from mig.shared.conf import get_configuration_object +from mig.shared.logger import daemon_logger, register_hangup_handler + + +if __name__ == "__main__": + print( + """This is the MiG quota daemon which collects storage quota + information for users and vgrids. + +Set the MIG_CONF environment to the server configuration path +unless it is available in mig/server/MiGserver.conf +""" + ) + # Force no log init since we use separate logger + configuration = get_configuration_object(skip_log=True) + + log_level = configuration.loglevel + if sys.argv[1:] and sys.argv[1] in ["debug", "info", "warning", "error"]: + log_level = sys.argv[1] + + # Use separate logger + + logger = daemon_logger("quota", + configuration.user_quota_log, + log_level) + configuration.logger = logger + + # Check if quota is enabled + + if not configuration.site_enable_quota: + msg = "Quota support is disabled in configuration!" + logger.error(msg) + print("%s ERROR: %s" + % (datetime.datetime.now(), msg), + file=sys.stderr) + sys.exit(1) + + # Check quota backend + + if configuration.quota_backend not in supported_quota_backends: + msg = "Quota backend: %s not in supported backends: %s" \ + % (configuration.quota_backend, + ", ".join(supported_quota_backends)) + logger.error(msg) + print("%s ERROR: %s" + % (datetime.datetime.now(), msg), + file=sys.stderr) + sys.exit(1) + + # Allow e.g. logrotate to force log re-open after rotates + register_hangup_handler(configuration) + + # Allow trigger next run on SIGCONT to main process + register_run_handler(configuration) + + # Allow clean shutdown on SIGINT only to main process + register_stop_handler(configuration) + + throttle_secs = float(configuration.quota_update_interval) + main_pid = os.getpid() + msg = "(%s) Starting quota daemon with throttle: %d secs" \ + % (main_pid, throttle_secs) + logger.info(msg) + print("%s %s" % (datetime.datetime.now(), msg)) + + throttle = False + while not check_stop(): + try: + if throttle: + interruptible_sleep(configuration, throttle_secs, + (check_run, check_stop)) + reset_run() + if check_stop(): + break + t1 = time.time() + status = update_quota(configuration) + t2 = time.time() + msg = "(%s) Updated quota in %d secs with status: %s" \ + % (os.getpid(), int(t2-t1), status) + logger.info(msg) + print("%s %s" % (datetime.datetime.now(), msg)) + throttle = True + except KeyboardInterrupt: + stop_running() + # NOTE: we can't be sure if SIGINT was sent to only main process + # so we make sure to propagate to monitor child + msg = "(%s) Interrupt requested - shutdown" \ + % os.getpid() + logger.info(msg) + print("%s %s" % (datetime.datetime.now(), msg)) + except Exception as exc: + throttle = True + msg = "(%s) Caught unexpected exception:\n%s" \ + % (os.getpid(), traceback.format_exc()) + logger.error(msg) + print("%s ERROR: %s" + % (datetime.datetime.now(), msg), + file=sys.stderr) + + msg = "(%s) Quota daemon shutting down" % main_pid + logger.info(msg) + print("%s %s" % (datetime.datetime.now(), msg)) + + sys.exit(0) diff --git a/tests/fixture/confs-stdlocal/MiGserver.conf b/tests/fixture/confs-stdlocal/MiGserver.conf index 9520491ab..a9983263f 100644 --- a/tests/fixture/confs-stdlocal/MiGserver.conf +++ b/tests/fixture/confs-stdlocal/MiGserver.conf @@ -550,6 +550,7 @@ default_mount_re = SSHFS-2.X-1 [QUOTA] backend = lustre +update_interval = 3600 user_limit = 1099511627776 vgrid_limit = 1099511627776 diff --git a/tests/fixture/confs-stdlocal/miglustrequota b/tests/fixture/confs-stdlocal/miglustrequota deleted file mode 100644 index 062f246bc..000000000 --- a/tests/fixture/confs-stdlocal/miglustrequota +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# -# Run lustre quota for MiG servers -# -# The script depends on a miglustrequota setup -# (please refer to mig/src/pylustrequota/README). -# -# IMPORTANT: if placed in /etc/cron.X the script filename must be -# something consisting entirely of upper and lower case letters, digits, -# underscores, and hyphens. I.e. if the script name contains e.g. a period, -# '.', it will be silently ignored! -# This is a limitation on the run-parts wrapper used by cron -# (see man run-parts for the rationale behind this). - -# By default bash silently ignores and continues on most errors but we can set -# options to e.g. catch uninitialized variables and errors as explained in: -# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ -# NOTE: 'set -eE' exits on non-zero exit codes to add safety and as recommended -# best-practice (CWE-252, CWE-248, ...), yet, in some cases it hurts more to -# exit midway, so it can be a trade-off. -set -eEuo pipefail - -# Send output to another email address -#MAILTO="root" - -MIG_CONF=/home/mig/mig/server/MiGserver.conf - -# Specify if migrid runs natively or inside containers with lustre at host. -# Value is the container manager (docker, podman, or empty string for none) -container_manager="" -container="migrid-lustre-quota" - -# Look in miglustrequota install dir first -export PATH="/usr/local/bin:${PATH}" - -if [[ $(id -u) -ne 0 ]]; then - echo "Please run $0 as root" - exit 1 -fi - -if [ -z "${container_manager}" ]; then - miglustrequota=$(which "miglustrequota.py" 2>/dev/null) - if [ ! -x "${miglustrequota}" ]; then - echo "ERROR: Missing miglustrequota.py" - exit 1 - fi - quota_cmd="${miglustrequota} -c ${MIG_CONF}" -else - check_cmd="${container_manager} container ls -a | grep -q '${container}'" - eval "$check_cmd" - ret=$? - if [ "$ret" -ne 0 ]; then - echo "ERROR: Missing ${container} container" - exit 1 - fi - quota_cmd="${container_manager} start -a ${container}" -fi - -eval "$quota_cmd" -ret=$? - -exit $ret diff --git a/tests/fixture/confs-stdlocal/migrid-init.d-deb b/tests/fixture/confs-stdlocal/migrid-init.d-deb index d6ac02101..87351aa07 100755 --- a/tests/fixture/confs-stdlocal/migrid-init.d-deb +++ b/tests/fixture/confs-stdlocal/migrid-init.d-deb @@ -43,6 +43,9 @@ if [ -z "$PYTHONPATH" ]; then else PYTHONPATH=${MIG_PATH}:$PYTHONPATH fi +# Make sure '/usr/local/(s)bin' is in path and force lookup order +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + # you probably do not want to modify these... PID_DIR=${PID_DIR:-/var/run} MIG_LOG=${MIG_STATE}/log @@ -62,13 +65,14 @@ MIG_FTPS=${MIG_CODE}/server/grid_ftps.py MIG_NOTIFY=${MIG_CODE}/server/grid_notify.py MIG_IMNOTIFY=${MIG_CODE}/server/grid_imnotify.py MIG_VMPROXY=${MIG_CODE}/server/grid_vmproxy.py +MIG_QUOTA=${MIG_CODE}/server/grid_quota.py MIG_CHKUSERROOT=${MIG_CODE}/server/chkuserroot.py MIG_CHKSIDROOT=${MIG_CODE}/server/chksidroot.py show_usage() { echo "Usage: migrid {start|stop|status|restart|reload}[daemon DAEMON]" echo "where daemon is left out for all or given along with DAEMON as one of the following" - echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all)" + echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all)" } check_enabled() { @@ -264,6 +268,18 @@ start_vmproxy() { log_end_msg 1 || true fi } +start_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Starting MiG quota daemon" ${SHORT_NAME} || true + if start-stop-daemon --start --quiet --oknodo --pidfile ${PID_FILE} --make-pidfile --user root --chuid root --background --name ${SHORT_NAME} --startas ${DAEMON_PATH} ; then + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} start_sftpsubsys() { check_enabled "sftp_subsys" || return 0 DAEMON_PATH=${MIG_SFTPSUBSYS} @@ -292,6 +308,7 @@ start_all() { start_notify start_imnotify start_vmproxy + start_quota return 0 } @@ -524,6 +541,19 @@ stop_vmproxy() { log_end_msg 1 || true fi } +stop_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Stopping MiG quota" ${SHORT_NAME} || true + if start-stop-daemon --stop --quiet --oknodo --pidfile ${PID_FILE} ; then + rm -f ${PID_FILE} + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} stop_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -563,6 +593,7 @@ stop_all() { stop_notify stop_imnotify stop_vmproxy + stop_quota return 0 } @@ -735,6 +766,18 @@ reload_vmproxy() { log_end_msg 1 || true fi } +reload_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Reloading MiG quota" ${SHORT_NAME} || true + if start-stop-daemon --stop --signal HUP --quiet --oknodo --pidfile ${PID_FILE} ; then + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} reload_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -787,6 +830,7 @@ reload_all() { reload_notify reload_imnotify reload_vmproxy + reload_quota # Apache helpers to verify proper chrooting reload_chkuserroot reload_chksidroot @@ -891,6 +935,13 @@ status_vmproxy() { PID_FILE="$PID_DIR/${SHORT_NAME}.pid" status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME} } +status_quota() { + check_enabled "quota" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME} +} status_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -929,6 +980,7 @@ status_all() { status_notify status_imnotify status_vmproxy + status_quota return 0 } @@ -940,7 +992,7 @@ test -f ${MIG_SCRIPT} || exit 0 # Force valid target case "$2" in - script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all) + script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all) TARGET="$2" ;; '') diff --git a/tests/fixture/confs-stdlocal/migrid-init.d-rh b/tests/fixture/confs-stdlocal/migrid-init.d-rh index c4bfad648..911c5bb3c 100755 --- a/tests/fixture/confs-stdlocal/migrid-init.d-rh +++ b/tests/fixture/confs-stdlocal/migrid-init.d-rh @@ -35,6 +35,7 @@ # processname: grid_notify.py # processname: grid_imnotify.py # processname: grid_vmproxy.py +# processname: grid_quota.py # processname: sshd # config: /etc/sysconfig/migrid # @@ -74,6 +75,9 @@ if [ -z "$PYTHONPATH" ]; then else export PYTHONPATH=${MIG_PATH}:$PYTHONPATH fi +# Make sure '/usr/local/(s)bin' is in path and force lookup order +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + # you probably do not want to modify these... PID_DIR=${PID_DIR:-/var/run} MIG_LOG=${MIG_STATE}/log @@ -93,13 +97,14 @@ MIG_FTPS=${MIG_CODE}/server/grid_ftps.py MIG_NOTIFY=${MIG_CODE}/server/grid_notify.py MIG_IMNOTIFY=${MIG_CODE}/server/grid_imnotify.py MIG_VMPROXY=${MIG_CODE}/server/grid_vmproxy.py +MIG_QUOTA=${MIG_CODE}/server/grid_quota.py MIG_CHKUSERROOT=${MIG_CODE}/server/chkuserroot.py MIG_CHKSIDROOT=${MIG_CODE}/server/chksidroot.py show_usage() { echo "Usage: migrid {start|stop|status|restart|reload}[daemon DAEMON]" echo "where daemon is left out for all or given along with DAEMON as one of the following" - echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all)" + echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all)" } check_enabled() { @@ -353,6 +358,21 @@ start_vmproxy() { [ $RET2 -ne 0 ] && echo "Warning: vmproxy not started." echo } +start_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Starting MiG quota daemon: $SHORT_NAME" + daemon --user root --pidfile ${PID_FILE} \ + "${DAEMON_PATH} >> ${MIG_LOG}/quota.out 2>&1 &" + fallback_save_pid "$DAEMON_PATH" "$PID_FILE" "$!" + RET2=$? + [ $RET2 -eq 0 ] && success + echo + [ $RET2 -ne 0 ] && echo "Warning: quota not started." + echo +} start_sftpsubsys() { check_enabled "sftp_subsys" || return DAEMON_PATH=${MIG_SFTPSUBSYS} @@ -385,6 +405,7 @@ start_all() { start_notify start_imnotify start_vmproxy + start_quota return 0 } @@ -545,6 +566,15 @@ stop_vmproxy() { killproc ${DAEMON_PATH} echo } +stop_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Shutting down MiG quota: $SHORT_NAME " + killproc ${DAEMON_PATH} + echo +} stop_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -588,6 +618,7 @@ stop_all() { stop_notify stop_imnotify stop_vmproxy + stop_quota return 0 } @@ -717,6 +748,15 @@ reload_vmproxy() { killproc ${DAEMON_PATH} -HUP echo } +reload_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Reloading MiG quota: $SHORT_NAME " + killproc ${DAEMON_PATH} -HUP + echo +} reload_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -773,6 +813,7 @@ reload_all() { reload_notify reload_imnotify reload_vmproxy + reload_quota # Apache helpers to verify proper chrooting reload_chkuserroot reload_chksidroot @@ -877,6 +918,13 @@ status_vmproxy() { PID_FILE="$PID_DIR/${SHORT_NAME}.pid" status ${DAEMON_PATH} } +status_quota() { + check_enabled "quota" || return + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + status ${DAEMON_PATH} +} status_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -916,6 +964,7 @@ status_all() { status_notify status_imnotify status_vmproxy + status_quota return 0 } @@ -927,7 +976,7 @@ test -f ${MIG_SCRIPT} || exit 0 # Force valid target case "$2" in - script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|all) + script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all) TARGET="$2" ;; '') diff --git a/tests/test_mig_lib_quota.py b/tests/test_mig_lib_quota.py new file mode 100644 index 000000000..25dd1328a --- /dev/null +++ b/tests/test_mig_lib_quota.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# test_mig_lib_quota - unit test of the corresponding mig lib module +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. +# +# --- END_HEADER --- +# + +"""Unit tests for the migrid module pointed to in the filename""" + +from mig.lib.quota import update_quota +from tests.support import MigTestCase + + +class MigLibQouta(MigTestCase): + """Unit tests for quota related helper functions""" + + def _provide_configuration(self): + """Prepare isolated test config""" + return 'testconfig' + + def before_each(self): + """Set up test configuration and reset state before each test""" + pass + + def test_invalid_quota_backend(self): + """Test invalid quota_backend in configuration""" + self.configuration.quota_backend = "NEVERNEVER" + with self.assertLogs(level='DEBUG') as log_capture: + update_quota(self.configuration) + self.assertTrue(any("quota_backend: 'NEVERNEVER' not in supported_quota_backends" in msg + for msg in log_capture.output))