Feature: Add external database volume and external database connection

.editorconfig

@@ -0,0 +1,6 @@
+# shfmt options
+binary_next_line = false
+switch_case_indent = false
+space_redirects = false
+keep_padding = false
+function_next_line = false

README.md

@@ -4,7 +4,7 @@
 - Python >= 3.5 (with setuptools)
 - Docker >= 20.10.13
 - Docker compose >= 2.0
-- RAM memory: At least 4Gb for instance, preferrably 8Gb.
+- RAM memory: At least 4Gb for instance, preferably 8Gb.
 
 On Ubuntu 22.04:
 
@@ -69,7 +69,7 @@ Create a dhis2-data image from a .sql.gz SQL file and the apps and documents (or
 $ d2-docker create data docker.eyeseetea.com/eyeseetea/dhis2-data:2.37.9-sierra --sql=sierra-db.sql.gz [--apps-dir=path/to/apps] [--documents-dir=path/to/document] [--datavalues-dir=path/to/dataValue]
 ```
 
-There are demo database files at [databases.dhis2.org](https://databases.dhis2.org/) that may be used for testing purposses. The database downloaded should correspond to the core version created; if there is no database file for the created core version, a prior version of the database should work.
+There are demo database files at [databases.dhis2.org](https://databases.dhis2.org/) that may be used for testing purposes. The database downloaded should correspond to the core version created; if there is no database file for the created core version, a prior version of the database should work.
 
 ### Start a DHIS2 instance
 
@@ -94,6 +94,10 @@ Some notes:
 - Use option `--java-opts="JAVA_OPTS"` to override the default JAVA_OPTS for the Tomcat process. That's tipically used to set the maximum/initial Heap Memory size (for example: `--java-opts="-Xmx3500m -Xms2500m"`)
 - Use option `--postgis-version=13-3.1-alpine` to specify the PostGIS version to use. By default, 10-2.5-alpine is used.
 - Use option `--debug-port=PORT` to specify the debug port of the Tomcat process.
+- Use option `--external-db-volume=VOLUME_ABSOLUTE_PATH` to create or use a persistent volume for the database located at `VOLUME_ABSOLUTE_PATH`.
+- Use option `--external-db-url=POSTGRES_URL` to connect to an external PostgreSQL database instead of using the DB container. The URL should be in the format `postgresql://USER:PASSWORD@HOST:PORT/DBNAME`. Note that you have to configure the DB to accept connections from the docker internal network.
+- The previous `--external-db-volume` and `--external-db-url` options are mutually exclusive.
+- Use option `--load-dump-from-data` with `--external-db-url` to import the SQL dump to the external database. Equivalent to running without `-k`/`--keep-containers`. The receiving DB should have the appropriate config (DB owner, user permissions, postgis extension).
 
 #### Custom DHIS2 dhis.conf
 
@@ -420,5 +424,5 @@ To remove glowroot from a container you must:
 - connect to the core container (`docker exec -it ${core_instance_name} bash`)
 - inside the core container, remove the `/opt/glowroot.zip` file
 - inside the core container, remove the `/opt/glowroot` folder
-- inside the core container, remove the `/usr/local/tomcat/bin/setenv.sh` file. (This is not extrictly necessary as the jar file will no longer exist and won't be able to start, but if it is not removed, some warnings/errors may be generated upon tomcat start)
+- inside the core container, remove the `/usr/local/tomcat/bin/setenv.sh` file. (This is not strictly necessary as the jar file will no longer exist and won't be able to start, but if it is not removed, some warnings/errors may be generated upon tomcat start)
 - exit the core container and restart it (`docker restart ${core_instance_name}`)

src/d2_docker/commands/start.py

@@ -41,8 +41,23 @@ def setup(parser):
     parser.add_argument("--postgis-version", type=str, help="Set PostGIS database version")
     parser.add_argument("--enable-postgres-queries-logging", action="store_true",
                         help="Enable Postgres queries logging")
-
     parser.add_argument("--glowroot-port", metavar="PORT", help="Set glowroot port")
+    parser.add_argument(
+        "--external-db-volume",
+        metavar="DIRECTORY",
+        help="Directory for external database volume",
+    )
+    parser.add_argument(
+        "--external-db-url",
+        type=str,
+        metavar="postgresql://user:pass@host:port/dbname",
+        help="Use external PostgreSQL database"
+    )
+    parser.add_argument(
+        "--load-dump-from-data",
+        action="store_true",
+        help="Load database dump from data container (only with --external-db-url)",
+    )
 
 
 def run(args):
@@ -54,9 +69,36 @@ def run(args):
         image2 = args.image
 
     args.image = image2
+
+    check_conflicting_external_params(args)
+
+    if args.external_db_volume:
+        check_db_volume_path(args.external_db_volume)
+
+    if args.external_db_url:
+        utils.validate_external_db_connection(args.external_db_url)
+
     start(args)
 
 
+def check_conflicting_external_params(args):
+    if args.external_db_volume and args.external_db_url:
+        msg = "--external-db-volume and --external-db-url are mutually exclusive"
+        raise utils.D2DockerError(msg)
+    if args.load_dump_from_data and not args.external_db_url:
+        msg = "--load-dump-from-data can only be used with --external-db-url"
+        raise utils.D2DockerError(msg)
+
+
+def check_db_volume_path(external_db_volume):
+    if not os.path.isabs(external_db_volume):
+        msg = "--external-db-volume must be an absolute path: {}".format(external_db_volume)
+        raise utils.D2DockerError(msg)
+    if not os.path.exists(external_db_volume):
+        msg = "--external-db-volume path does not exist: {}".format(external_db_volume)
+        raise utils.D2DockerError(msg)
+
+
 def import_from_file(images_path):
     dhis2_data_image_re = "/{}:".format(utils.DHIS2_DATA_IMAGE)
     result = utils.load_images_file(images_path)
@@ -85,10 +127,13 @@ def start(args):
     override_containers = not args.keep_containers
 
     if args.pull:
-        utils.run_docker_compose(["pull"], image_name, core_image=core_image)
+        utils.run_docker_compose(["pull"], image_name, core_image=core_image,
+                                 external_db_volume=args.external_db_volume)
 
     if override_containers:
-        utils.run_docker_compose(["down", "--volumes"], image_name, core_image=core_image)
+        utils.run_docker_compose(["down", "--volumes"], image_name, core_image=core_image,
+                                 external_db_volume=args.external_db_volume,
+                                 external_db_url=args.external_db_url)
 
     up_args = filter(
         bool, ["--force-recreate" if override_containers else None, "-d" if args.detach else None]
@@ -115,7 +160,10 @@ def start(args):
             java_opts=args.java_opts,
             postgis_version=args.postgis_version,
             enable_postgres_queries_logging=args.enable_postgres_queries_logging,
-            glowroot_port=args.glowroot_port
+            glowroot_port=args.glowroot_port,
+            external_db_volume=args.external_db_volume,
+            external_db_url=args.external_db_url,
+            load_dump_from_data=args.load_dump_from_data,
         )
 
     if args.detach:

src/d2_docker/config/dhis2-core-start.sh

@@ -10,17 +10,23 @@ set -e -u -o pipefail
 #
 
 # Global: LOAD_FROM_DATA="yes" | "no"
+# Global: LOAD_DUMP_FROM_DATA="yes" | "no"
+# Global: EXTERNAL_DB_URL=string (optional)
 # Global: DEPLOY_PATH=string
 # Global: DHIS2_AUTH=string
 
 export PGPASSWORD="dhis"
+db_url=""
+if [[ -n "$EXTERNAL_DB_URL" ]]; then
+    db_url="${EXTERNAL_DB_URL//localhost/host.docker.internal}"
+fi
 
 dhis2_url="http://localhost:8080/$DEPLOY_PATH"
 dhis2_url_with_auth="http://$DHIS2_AUTH@localhost:8080/$DEPLOY_PATH"
-psql_base_cmd="psql --quiet -h db -U dhis dhis2"
+psql_base_cmd="psql --quiet ${db_url:-"-h db -U dhis dhis2"}"
 psql_cmd="$psql_base_cmd -v ON_ERROR_STOP=0"
 psql_strict_cmd="$psql_base_cmd -v ON_ERROR_STOP=1"
-pgrestore_cmd="pg_restore -h db -U dhis -d dhis2"
+pgrestore_cmd="pg_restore ${db_url:-"-h db -U dhis -d dhis2"}"
 configdir="/config"
 homedir="/dhis2-home-files"
 scripts_dir="/data/scripts"
@@ -37,7 +43,12 @@ debug() {
 }
 
 run_sql_files() {
-    base_db_path=$(test "${LOAD_FROM_DATA}" = "yes" && echo "$root_db_path" || echo "$post_db_path")
+    if { [ -z "$db_url" ] && [ "${LOAD_FROM_DATA}" = "yes" ]; } ||
+        { [ -n "$db_url" ] && [ "${LOAD_FROM_DATA}" = "yes" ] && [ "${LOAD_DUMP_FROM_DATA}" = "yes" ]; }; then
+        base_db_path="$root_db_path"
+    else
+        base_db_path="$post_db_path"
+    fi
     debug "Files in data path"
     find "$base_db_path" >&2
 

src/d2_docker/docker-compose.yml

@@ -17,6 +17,8 @@ services:
             CATALINA_OPTS: "-Dcontext.path=${DEPLOY_PATH} -Xdebug -Xrunjdwp:transport=dt_socket,address=0.0.0.0:8000,server=y,suspend=n"
             JAVA_OPTS: "-Xmx7500m -Xms4000m ${JAVA_OPTS}"
             LOAD_FROM_DATA: "${LOAD_FROM_DATA}"
+            EXTERNAL_DB_URL: "${EXTERNAL_DB_URL}"
+            LOAD_DUMP_FROM_DATA: "${LOAD_DUMP_FROM_DATA}"
             DEPLOY_PATH: "${DEPLOY_PATH}"
             DHIS2_AUTH: "${DHIS2_AUTH}"
         entrypoint: bash /config/dhis2-core-entrypoint.sh

src/d2_docker/utils.py

@@ -10,9 +10,10 @@
 import time
 import yaml
 import urllib.request
+from urllib.parse import urlparse
 from setuptools._distutils import dir_util
 from pathlib import Path
-from typing import Optional
+from typing import Dict, Optional
 
 import d2_docker
 from d2_docker.glowroot import get_port_glowroot
@@ -25,6 +26,7 @@
 PROJECT_DIR = os.path.dirname(os.path.realpath(__file__))
 ROOT_PATH = os.environ.get("ROOT_PATH") or PROJECT_DIR
 
+
 def get_dhis2_war_url(version):
     match = (re.match(r"^(\d+.\d+)", version) if version.startswith("2.")
              else re.match(r"^(\d+)", version))
@@ -263,6 +265,9 @@ def run_docker_compose(
     postgis_version=None,
     enable_postgres_queries_logging=False,
     glowroot_port=None,
+    external_db_volume=None,
+    external_db_url=None,
+    load_dump_from_data=False,
     **kwargs,
 ):
     """
@@ -279,6 +284,19 @@ def run_docker_compose(
     post_sql_dir_abs = get_absdir_for_docker_volume(post_sql_dir)
     scripts_dir_abs = get_absdir_for_docker_volume(scripts_dir)
 
+    if external_db_url and not dhis_conf:
+        logger.info("External DB URL provided, updating dhis.conf")
+        db_config = parse_postgres_url(external_db_url)
+        if not db_config:
+            raise D2DockerError("Invalid PostgreSQL URL format")
+
+        dhis_conf_file = build_dhis_conf_from_external_db(
+            jdbc_url=db_config['url'],
+            username=db_config['user'],
+            password=db_config['password']
+        )
+        dhis_conf = dhis_conf_file.name
+
     env_pairs = [
         ("DHIS2_DATA_IMAGE", final_image_name),
         ("DHIS2_CORE_PORT", str(port)) if port else None,
@@ -299,7 +317,10 @@ def run_docker_compose(
         # Add ROOT_PATH from environment (required when run inside a docker)
         ("ROOT_PATH", ROOT_PATH),
         ("PSQL_ENABLE_QUERY_LOGS", "") if not enable_postgres_queries_logging else None,
-        ("GLOWROOT_PORT", get_port_glowroot(glowroot_port))
+        ("GLOWROOT_PORT", get_port_glowroot(glowroot_port)),
+        ("EXTERNAL_DB_VOLUME", external_db_volume) if external_db_volume else None,
+        ("EXTERNAL_DB_URL", external_db_url) if external_db_url else None,
+        ("LOAD_DUMP_FROM_DATA", "yes" if load_dump_from_data else "no"),
     ]
     env = dict((k, v) for (k, v) in [pair for pair in env_pairs if pair] if v is not None)
 
@@ -310,6 +331,21 @@ def process_yaml(data):
         for env_port in env_ports:
             if env_port not in env:
                 core["ports"] = [port for port in core["ports"] if env_port not in port]
+        if "EXTERNAL_DB_VOLUME" in env:
+            data["volumes"]["pgdata"] = {
+                'driver': 'local',
+                'driver_opts': {
+                    'type': 'none',
+                    'o': 'bind',
+                    'device': external_db_volume
+                }
+            }
+        if "EXTERNAL_DB_URL" in env:
+            del data["services"]["db"]
+            del data["volumes"]["pgdata"]
+            core = data["services"]["core"]
+            core["depends_on"] = [dep for dep in core["depends_on"] if dep != "db"]
+            core["extra_hosts"] = ["host.docker.internal:host-gateway"]
 
         return data
 
@@ -335,6 +371,92 @@ def build_docker_compose(process_yaml):
 
     return temp_compose
 
+
+def validate_external_db_connection(db_url):
+    """Validate connection to external PostgreSQL database."""
+    logger.info("Validating external database connection...")
+    try:
+        psql_cmd = ["psql", "-d", db_url, "-c", "SELECT now();"]
+        run(psql_cmd, capture_output=True)
+        logger.info("External database validation successful")
+    except Exception as e:
+        raise D2DockerError(f"External database validation failed: {e}")
+
+
+def parse_postgres_url(url: str) -> Optional[Dict[str, str]]:
+    """Parse PostgreSQL connection URL.
+
+    Args:
+        url (str): PostgreSQL connection URL in the format postgresql://user:pass@host:port/dbname
+
+    Raises:
+        D2DockerError: If the URL is invalid.
+
+    Returns:
+        Optional[Dict[str, str]]: Dictionary with keys 'url', 'user', 'password' or None if url is empty.
+    """
+    if not url:
+        return None
+
+    try:
+        parsed = urlparse(url)
+        if parsed.scheme not in ['postgresql']:
+            raise D2DockerError(f"Invalid PostgreSQL URL scheme: {parsed.scheme}")
+
+        if None in [parsed.username, parsed.password, parsed.hostname, parsed.path]:
+            raise D2DockerError(f"Missing components in PostgreSQL URL: {url}")
+
+        hostname = "host.docker.internal" if parsed.hostname == "localhost" else parsed.hostname
+
+        port = ":"+str(parsed.port) if parsed.port else ""
+
+        return {
+            'url': "jdbc:"+parsed.scheme+"://"+str(hostname)+port+parsed.path,
+            'user': str(parsed.username),
+            'password': str(parsed.password),
+        }
+    except Exception as e:
+        raise D2DockerError(f"Failed to parse PostgreSQL URL: {e}")
+
+
+def build_dhis_conf_from_external_db(jdbc_url, username, password):
+    base_config_path = os.path.join(ROOT_PATH, "config", "DHIS2_home", "dhis.conf")
+    with open(base_config_path, 'r') as file:
+        config = file.read()
+
+    # Update connection URL
+    config = re.sub(
+        r'^connection\.url\s*=.*$',
+        f'connection.url = {jdbc_url}',
+        config,
+        flags=re.MULTILINE
+    )
+
+    if username:
+        config = re.sub(
+            r'^connection\.username\s*=.*$',
+            f'connection.username = {username}',
+            config,
+            flags=re.MULTILINE
+        )
+
+    if password:
+        config = re.sub(
+            r'^connection\.password\s*=.*$',
+            f'connection.password = {password}',
+            config,
+            flags=re.MULTILINE
+        )
+
+    with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.conf') as temp_conf:
+        temp_conf.write(config)
+
+    os.chmod(temp_conf.name, 0o644)
+
+    atexit.register(lambda: os.remove(temp_conf.name))
+    return temp_conf
+
+
 def get_config_path(default_filename, path):
     return os.path.abspath(path) if path else get_config_file(default_filename)
 
@@ -470,6 +592,7 @@ def export_data_from_image(source_image, dest_path):
 
 # https://github.com/dhis2/dhis2-core/blob/master/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceDomain.java#L35
 
+
 default_folders = [
     "apps",
     "dataValue",
@@ -482,6 +605,7 @@ def export_data_from_image(source_image, dest_path):
     "jobData"
 ]
 
+
 def export_data_from_running_containers(image_name, containers, destination, folders=None):
     """Export data (db + apps + documents) from a running Docker container to some folder."""
     logger.info("Copy Dhis2 apps")