From 7de407bb4fec46dbd80ebb7c53d6d84a4fc6007b Mon Sep 17 00:00:00 2001 From: Andrii Sheiko Date: Thu, 26 Feb 2026 14:46:41 +0200 Subject: [PATCH 1/3] added docker_mode for MYSQL_SSH (mysqldump) --- rsnapshot_backup/rsnapshot_backup.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/rsnapshot_backup/rsnapshot_backup.py b/rsnapshot_backup/rsnapshot_backup.py index 2b2bf39..45e4c94 100755 --- a/rsnapshot_backup/rsnapshot_backup.py +++ b/rsnapshot_backup/rsnapshot_backup.py @@ -925,7 +925,7 @@ def run_cmd_pipe(cmd): if item["source"] == "ALL": script_dump_part = textwrap.dedent( """\ - mysql --defaults-file=/etc/mysql/debian.cnf --skip-column-names --batch -e "SHOW DATABASES;" | grep -v -e information_schema -e performance_schema {grep_db_filter} > {mysql_dump_dir}/db_list.txt + {dump_run} 'mysql {dump_creds} --skip-column-names --batch -e "SHOW DATABASES;"' | grep -v -e information_schema -e performance_schema {grep_db_filter} > {mysql_dump_dir}/db_list.txt WAS_ERR=0 for db in $(cat {mysql_dump_dir}/db_list.txt); do set +e @@ -933,7 +933,7 @@ def run_cmd_pipe(cmd): {exec_before_dump} if [[ $? -ne 0 ]]; then WAS_ERR=1; fi for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do - {dump_prefix_cmd} mysqldump --defaults-file=/etc/mysql/debian.cnf --force --opt --single-transaction --quick --skip-lock-tables {mysql_events} --databases $db --max_allowed_packet=1G {mysqldump_args} | gzip > {mysql_dump_dir}/$db.gz + {dump_run} '{dump_prefix_cmd} mysqldump {dump_creds} --force --opt --single-transaction --quick --skip-lock-tables {mysql_events} --databases $1 --max_allowed_packet=1G {mysqldump_args}' -- $db | gzip > {mysql_dump_dir}/$db.gz if [[ $? -ne 0 ]]; then WAS_ERR=1 echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" @@ -960,7 +960,9 @@ def run_cmd_pipe(cmd): exec_after_dump=item["exec_after_dump"], mysqldump_args=item["mysqldump_args"], grep_db_filter=grep_db_filter, - dump_attempts=item["dump_attempts"] + dump_attempts=item["dump_attempts"], + dump_run="docker exec {container} sh -lc".format(container=item["docker_container"]) if item["docker_mode"] else "bash -c", + dump_creds="-u\"$MYSQL_USER\" -p\"$MYSQL_PASSWORD\"" if item["docker_mode"] else "--defaults-file=/etc/mysql/debian.cnf" ) else: script_dump_part = textwrap.dedent( @@ -971,7 +973,7 @@ def run_cmd_pipe(cmd): {exec_before_dump} if [[ $? -ne 0 ]]; then WAS_ERR=1; fi for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do - {dump_prefix_cmd} mysqldump --defaults-file=/etc/mysql/debian.cnf --force --opt --single-transaction --quick --skip-lock-tables {mysql_events} --databases {source} --max_allowed_packet=1G {mysqldump_args} | gzip > {mysql_dump_dir}/{source}.gz + {dump_run} '{dump_prefix_cmd} mysqldump {dump_creds} --force --opt --single-transaction --quick --skip-lock-tables {mysql_events} --databases {source} --max_allowed_packet=1G {mysqldump_args}' | gzip > {mysql_dump_dir}/{source}.gz if [[ $? -ne 0 ]]; then WAS_ERR=1 echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" @@ -998,7 +1000,9 @@ def run_cmd_pipe(cmd): mysqldump_args=item["mysqldump_args"], grep_db_filter=grep_db_filter, source=item["source"], - dump_attempts=item["dump_attempts"] + dump_attempts=item["dump_attempts"], + dump_run="docker exec {container} sh -lc".format(container=item["docker_container"]) if item["docker_mode"] else "bash -c", + dump_creds="-u\"$MYSQL_USER\" -p\"$MYSQL_PASSWORD\"" if item["docker_mode"] else "--defaults-file=/etc/mysql/debian.cnf" ) # If hourly retains are used keep dumps only for 59 minutes @@ -1007,7 +1011,7 @@ def run_cmd_pipe(cmd): #!/bin/bash set -e - ssh {ssh_args} -p {port} {user}@{host} ' + ssh {ssh_args} -p {port} {user}@{host} bash -se <<'EOF' set -x set -e set -o pipefail @@ -1021,7 +1025,7 @@ def run_cmd_pipe(cmd): cd {mysql_dump_dir} find {mysql_dump_dir} -type f -name "*.gz" -mmin +{mmin} -delete {script_dump_part} - ' + EOF """ ).format( ssh_args=ssh_args, From e195bfd09ab839f36c8d443b8a3e99f190dd7a90 Mon Sep 17 00:00:00 2001 From: Andrii Sheiko Date: Fri, 27 Feb 2026 13:35:22 +0200 Subject: [PATCH 2/3] added docker_mode for MYSQL_SSH (mysqldump): fix --- rsnapshot_backup/rsnapshot_backup.py | 312 ++++++++++++++++++--------- 1 file changed, 211 insertions(+), 101 deletions(-) diff --git a/rsnapshot_backup/rsnapshot_backup.py b/rsnapshot_backup/rsnapshot_backup.py index 45e4c94..29e9905 100755 --- a/rsnapshot_backup/rsnapshot_backup.py +++ b/rsnapshot_backup/rsnapshot_backup.py @@ -921,122 +921,232 @@ def run_cmd_pipe(cmd): ) else: + if item["docker_mode"]: + if item["source"] == "ALL": + script_dump_part = textwrap.dedent( + """\ + docker exec {container} sh -lc 'mysql -u"$MYSQL_USER" -p"$MYSQL_PASSWORD" --skip-column-names --batch -e "SHOW DATABASES;"' | grep -v -e information_schema -e performance_schema {grep_db_filter} > {mysql_dump_dir}/db_list.txt + WAS_ERR=0 + for db in $(cat {mysql_dump_dir}/db_list.txt); do + set +e + if [[ ! -f {mysql_dump_dir}/$db.gz ]]; then + {exec_before_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do + docker exec {container} sh -lc '{dump_prefix_cmd} mysqldump -u"$MYSQL_USER" -p"$MYSQL_PASSWORD" --force --opt --single-transaction --quick --skip-lock-tables {mysql_events} --databases $1 --max_allowed_packet=1G {mysqldump_args}' -- $db | gzip > {mysql_dump_dir}/$db.gz + if [[ $? -ne 0 ]]; then + WAS_ERR=1 + echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" + else + echo "NOTICE: Dump succeeded, attempt $DUMP_ATTEMPT of {dump_attempts}" + break + fi + done + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + {exec_after_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + else + echo "NOTICE: Valid dump already exists, skipping" + fi + set -e + done + if [[ $WAS_ERR -ne 0 ]]; then false; else true; fi + """ + ).format( + mysql_dump_dir=item["mysql_dump_dir"], + mysql_events="" if item["mysql_noevents"] else "--events", + dump_prefix_cmd=item["dump_prefix_cmd"], + exec_before_dump=item["exec_before_dump"], + exec_after_dump=item["exec_after_dump"], + mysqldump_args=item["mysqldump_args"], + grep_db_filter=grep_db_filter, + dump_attempts=item["dump_attempts"], + container=item["docker_container"] + ) + else: + script_dump_part = textwrap.dedent( + """\ + WAS_ERR=0 + set +e + if [[ ! -f {mysql_dump_dir}/{source}.gz ]]; then + {exec_before_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do + docker exec {container} sh -lc '{dump_prefix_cmd} mysqldump -u"$MYSQL_USER" -p"$MYSQL_PASSWORD" --force --opt --single-transaction --quick --skip-lock-tables {mysql_events} --databases {source} --max_allowed_packet=1G {mysqldump_args}' | gzip > {mysql_dump_dir}/{source}.gz + if [[ $? -ne 0 ]]; then + WAS_ERR=1 + echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" + else + echo "NOTICE: Dump succeeded, attempt $DUMP_ATTEMPT of {dump_attempts}" + break + fi + done + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + {exec_after_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + else + echo "NOTICE: Valid dump already exists, skipping" + fi + set -e + if [[ $WAS_ERR -ne 0 ]]; then false; else true; fi + """ + ).format( + mysql_dump_dir=item["mysql_dump_dir"], + mysql_events="" if item["mysql_noevents"] else "--events", + dump_prefix_cmd=item["dump_prefix_cmd"], + exec_before_dump=item["exec_before_dump"], + exec_after_dump=item["exec_after_dump"], + mysqldump_args=item["mysqldump_args"], + grep_db_filter=grep_db_filter, + source=item["source"], + dump_attempts=item["dump_attempts"], + container=item["docker_container"] + ) - if item["source"] == "ALL": - script_dump_part = textwrap.dedent( + # If hourly retains are used keep dumps only for 59 minutes + script = textwrap.dedent( """\ - {dump_run} 'mysql {dump_creds} --skip-column-names --batch -e "SHOW DATABASES;"' | grep -v -e information_schema -e performance_schema {grep_db_filter} > {mysql_dump_dir}/db_list.txt - WAS_ERR=0 - for db in $(cat {mysql_dump_dir}/db_list.txt); do - set +e - if [[ ! -f {mysql_dump_dir}/$db.gz ]]; then - {exec_before_dump} - if [[ $? -ne 0 ]]; then WAS_ERR=1; fi - for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do - {dump_run} '{dump_prefix_cmd} mysqldump {dump_creds} --force --opt --single-transaction --quick --skip-lock-tables {mysql_events} --databases $1 --max_allowed_packet=1G {mysqldump_args}' -- $db | gzip > {mysql_dump_dir}/$db.gz - if [[ $? -ne 0 ]]; then - WAS_ERR=1 - echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" - else - echo "NOTICE: Dump succeeded, attempt $DUMP_ATTEMPT of {dump_attempts}" - break - fi - done - if [[ $? -ne 0 ]]; then WAS_ERR=1; fi - {exec_after_dump} - if [[ $? -ne 0 ]]; then WAS_ERR=1; fi - else - echo "NOTICE: Valid dump already exists, skipping" - fi - set -e - done - if [[ $WAS_ERR -ne 0 ]]; then false; else true; fi + #!/bin/bash + set -e + + ssh {ssh_args} -p {port} {user}@{host} bash -se <<'EOF' + set -x + set -e + set -o pipefail + mkdir -p {mysql_dump_dir} + chmod 700 {mysql_dump_dir} + while [[ -d {mysql_dump_dir}/dump.lock ]]; do + sleep 5 + done + mkdir {mysql_dump_dir}/dump.lock + trap "rm -rf {mysql_dump_dir}/dump.lock" 0 + cd {mysql_dump_dir} + find {mysql_dump_dir} -type f -name "*.gz" -mmin +{mmin} -delete + {script_dump_part} + EOF """ ).format( + ssh_args=ssh_args, + port=item["connect_port"], + user=item["connect_user"], + host=item["connect_host"], mysql_dump_dir=item["mysql_dump_dir"], - mysql_events="" if item["mysql_noevents"] else "--events", - dump_prefix_cmd=item["dump_prefix_cmd"], - exec_before_dump=item["exec_before_dump"], - exec_after_dump=item["exec_after_dump"], - mysqldump_args=item["mysqldump_args"], - grep_db_filter=grep_db_filter, - dump_attempts=item["dump_attempts"], - dump_run="docker exec {container} sh -lc".format(container=item["docker_container"]) if item["docker_mode"] else "bash -c", - dump_creds="-u\"$MYSQL_USER\" -p\"$MYSQL_PASSWORD\"" if item["docker_mode"] else "--defaults-file=/etc/mysql/debian.cnf" + mmin="59" if "retain_hourly" in item else "720", + script_dump_part=script_dump_part ) + else: - script_dump_part = textwrap.dedent( - """\ - WAS_ERR=0 - set +e - if [[ ! -f {mysql_dump_dir}/{source}.gz ]]; then - {exec_before_dump} - if [[ $? -ne 0 ]]; then WAS_ERR=1; fi - for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do - {dump_run} '{dump_prefix_cmd} mysqldump {dump_creds} --force --opt --single-transaction --quick --skip-lock-tables {mysql_events} --databases {source} --max_allowed_packet=1G {mysqldump_args}' | gzip > {mysql_dump_dir}/{source}.gz - if [[ $? -ne 0 ]]; then - WAS_ERR=1 - echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" + if item["source"] == "ALL": + script_dump_part = textwrap.dedent( + """\ + mysql --defaults-file=/etc/mysql/debian.cnf --skip-column-names --batch -e "SHOW DATABASES;" | grep -v -e information_schema -e performance_schema {grep_db_filter} > {mysql_dump_dir}/db_list.txt + WAS_ERR=0 + for db in $(cat {mysql_dump_dir}/db_list.txt); do + set +e + if [[ ! -f {mysql_dump_dir}/$db.gz ]]; then + {exec_before_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do + {dump_prefix_cmd} mysqldump --defaults-file=/etc/mysql/debian.cnf --force --opt --single-transaction --quick --skip-lock-tables {mysql_events} --databases $db --max_allowed_packet=1G {mysqldump_args} | gzip > {mysql_dump_dir}/$db.gz + if [[ $? -ne 0 ]]; then + WAS_ERR=1 + echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" + else + echo "NOTICE: Dump succeeded, attempt $DUMP_ATTEMPT of {dump_attempts}" + break + fi + done + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + {exec_after_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi else - echo "NOTICE: Dump succeeded, attempt $DUMP_ATTEMPT of {dump_attempts}" - break + echo "NOTICE: Valid dump already exists, skipping" fi - done - if [[ $? -ne 0 ]]; then WAS_ERR=1; fi - {exec_after_dump} - if [[ $? -ne 0 ]]; then WAS_ERR=1; fi - else - echo "NOTICE: Valid dump already exists, skipping" - fi + set -e + done + if [[ $WAS_ERR -ne 0 ]]; then false; else true; fi + """ + ).format( + mysql_dump_dir=item["mysql_dump_dir"], + mysql_events="" if item["mysql_noevents"] else "--events", + dump_prefix_cmd=item["dump_prefix_cmd"], + exec_before_dump=item["exec_before_dump"], + exec_after_dump=item["exec_after_dump"], + mysqldump_args=item["mysqldump_args"], + grep_db_filter=grep_db_filter, + dump_attempts=item["dump_attempts"] + ) + else: + script_dump_part = textwrap.dedent( + """\ + WAS_ERR=0 + set +e + if [[ ! -f {mysql_dump_dir}/{source}.gz ]]; then + {exec_before_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do + {dump_prefix_cmd} mysqldump --defaults-file=/etc/mysql/debian.cnf --force --opt --single-transaction --quick --skip-lock-tables {mysql_events} --databases {source} --max_allowed_packet=1G {mysqldump_args} | gzip > {mysql_dump_dir}/{source}.gz + if [[ $? -ne 0 ]]; then + WAS_ERR=1 + echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" + else + echo "NOTICE: Dump succeeded, attempt $DUMP_ATTEMPT of {dump_attempts}" + break + fi + done + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + {exec_after_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + else + echo "NOTICE: Valid dump already exists, skipping" + fi + set -e + if [[ $WAS_ERR -ne 0 ]]; then false; else true; fi + """ + ).format( + mysql_dump_dir=item["mysql_dump_dir"], + mysql_events="" if item["mysql_noevents"] else "--events", + dump_prefix_cmd=item["dump_prefix_cmd"], + exec_before_dump=item["exec_before_dump"], + exec_after_dump=item["exec_after_dump"], + mysqldump_args=item["mysqldump_args"], + grep_db_filter=grep_db_filter, + source=item["source"], + dump_attempts=item["dump_attempts"] + ) + + # If hourly retains are used keep dumps only for 59 minutes + script = textwrap.dedent( + """\ + #!/bin/bash set -e - if [[ $WAS_ERR -ne 0 ]]; then false; else true; fi + + ssh {ssh_args} -p {port} {user}@{host} ' + set -x + set -e + set -o pipefail + mkdir -p {mysql_dump_dir} + chmod 700 {mysql_dump_dir} + while [[ -d {mysql_dump_dir}/dump.lock ]]; do + sleep 5 + done + mkdir {mysql_dump_dir}/dump.lock + trap "rm -rf {mysql_dump_dir}/dump.lock" 0 + cd {mysql_dump_dir} + find {mysql_dump_dir} -type f -name "*.gz" -mmin +{mmin} -delete + {script_dump_part} + ' """ ).format( + ssh_args=ssh_args, + port=item["connect_port"], + user=item["connect_user"], + host=item["connect_host"], mysql_dump_dir=item["mysql_dump_dir"], - mysql_events="" if item["mysql_noevents"] else "--events", - dump_prefix_cmd=item["dump_prefix_cmd"], - exec_before_dump=item["exec_before_dump"], - exec_after_dump=item["exec_after_dump"], - mysqldump_args=item["mysqldump_args"], - grep_db_filter=grep_db_filter, - source=item["source"], - dump_attempts=item["dump_attempts"], - dump_run="docker exec {container} sh -lc".format(container=item["docker_container"]) if item["docker_mode"] else "bash -c", - dump_creds="-u\"$MYSQL_USER\" -p\"$MYSQL_PASSWORD\"" if item["docker_mode"] else "--defaults-file=/etc/mysql/debian.cnf" + mmin="59" if "retain_hourly" in item else "720", + script_dump_part=script_dump_part ) - # If hourly retains are used keep dumps only for 59 minutes - script = textwrap.dedent( - """\ - #!/bin/bash - set -e - - ssh {ssh_args} -p {port} {user}@{host} bash -se <<'EOF' - set -x - set -e - set -o pipefail - mkdir -p {mysql_dump_dir} - chmod 700 {mysql_dump_dir} - while [[ -d {mysql_dump_dir}/dump.lock ]]; do - sleep 5 - done - mkdir {mysql_dump_dir}/dump.lock - trap "rm -rf {mysql_dump_dir}/dump.lock" 0 - cd {mysql_dump_dir} - find {mysql_dump_dir} -type f -name "*.gz" -mmin +{mmin} -delete - {script_dump_part} - EOF - """ - ).format( - ssh_args=ssh_args, - port=item["connect_port"], - user=item["connect_user"], - host=item["connect_host"], - mysql_dump_dir=item["mysql_dump_dir"], - mmin="59" if "retain_hourly" in item else "720", - script_dump_part=script_dump_part - ) - if item["type"] == "POSTGRESQL_SSH": # --verbose is needed for Completed on signature in dumps, and sdterr shouldn't be redirected to /dev/null From d20731194c2cbe96811e8f03dacc04dc458a977b Mon Sep 17 00:00:00 2001 From: Andrii Sheiko Date: Fri, 27 Feb 2026 11:52:22 +0000 Subject: [PATCH 3/3] update docker_mode for postgresql --- rsnapshot_backup/rsnapshot_backup.py | 461 ++++++++++++++++++--------- 1 file changed, 312 insertions(+), 149 deletions(-) diff --git a/rsnapshot_backup/rsnapshot_backup.py b/rsnapshot_backup/rsnapshot_backup.py index 29e9905..902653f 100755 --- a/rsnapshot_backup/rsnapshot_backup.py +++ b/rsnapshot_backup/rsnapshot_backup.py @@ -303,8 +303,6 @@ def run_cmd_pipe(cmd): item["docker_mode"] = False if "docker_container" not in item: item["docker_container"] = "empty" - if "db_user" not in item: - item["db_user"] = None # Check before_backup_check and skip item if failed # It is needed for both rotations and sync @@ -1155,178 +1153,343 @@ def run_cmd_pipe(cmd): # grep output should be put into stderr again # https://unix.stackexchange.com/questions/3514/how-to-grep-standard-error-stream-stderr # allow pg_dump.*: connecting just to see what it does - if item["source"] == "ALL": + + if item["docker_mode"]: + if item["source"] == "ALL": + + if "postgresql_dump_type" in item and item["postgresql_dump_type"] == "directory": + pg_dump_line_pipe_part = "" + pg_dump_format_part = "--format=directory --file={postgresql_dump_dir}/$db".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + if_exists_part = "-d {postgresql_dump_dir}/$db".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + mkdir_chown_part = "mkdir -p {postgresql_dump_dir}/$db && chown postgres:postgres {postgresql_dump_dir}/$db".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + else: + pg_dump_line_pipe_part = "| gzip > {postgresql_dump_dir}/$db.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + pg_dump_format_part = "--format=plain" + if_exists_part = "-f {postgresql_dump_dir}/$db.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + mkdir_chown_part = "" + + script_dump_part = textwrap.dedent( + """\ + docker exec -u postgres {container} sh -lc 'echo SELECT datname FROM pg_database | psql -U "$POSTGRES_USER" --no-align -t template1' {grep_db_filter} | grep -v -e template0 -e template1 > {postgresql_dump_dir}/db_list.txt + WAS_ERR=0 + for db in $(cat {postgresql_dump_dir}/db_list.txt); do + set +e + if [[ ! {if_exists_part} ]]; then + {mkdir_chown_part} + {exec_before_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do + docker exec -u postgres {container} sh -lc '{dump_prefix_cmd} pg_dump -U "$POSTGRES_USER" --create {postgresql_clean} {pg_dump_args} {pg_dump_format_part} --verbose $1' -- $db 2> >({pg_dump_filter}) {pg_dump_line_pipe_part} + if [[ $? -ne 0 ]]; then + WAS_ERR=1 + echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" + else + echo "NOTICE: Dump succeeded, attempt $DUMP_ATTEMPT of {dump_attempts}" + break + fi + done + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + {exec_after_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + else + echo "NOTICE: Valid dump already exists, skipping" + fi + set -e + done + if [[ $WAS_ERR -ne 0 ]]; then false; else true; fi + """ + ).format( + postgresql_dump_dir=item["postgresql_dump_dir"], + postgresql_clean="" if item["postgresql_noclean"] else "--clean --if-exists", + dump_prefix_cmd=item["dump_prefix_cmd"], + exec_before_dump=item["exec_before_dump"], + exec_after_dump=item["exec_after_dump"], + pg_dump_args=item["pg_dump_args"], + grep_db_filter=grep_db_filter, + pg_dump_filter=item["pg_dump_filter"], + dump_attempts=item["dump_attempts"], + pg_dump_line_pipe_part=pg_dump_line_pipe_part, + pg_dump_format_part=pg_dump_format_part, + if_exists_part=if_exists_part, + mkdir_chown_part=mkdir_chown_part, + container=item["docker_container"] + ) + else: + + if "postgresql_dump_type" in item and item["postgresql_dump_type"] == "directory": + pg_dump_line_pipe_part = "" + pg_dump_format_part = "--format=directory --file={postgresql_dump_dir}/{source}".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) + if_exists_part = "-d {postgresql_dump_dir}/{source}".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) + mkdir_chown_part = "mkdir -p {postgresql_dump_dir}/{source} && chown postgres:postgres {postgresql_dump_dir}/{source}".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) + else: + pg_dump_line_pipe_part = "| gzip > {postgresql_dump_dir}/{source}.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) + pg_dump_format_part = "--format=plain" + if_exists_part = "-f {postgresql_dump_dir}/{source}.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) + mkdir_chown_part = "" + + script_dump_part = textwrap.dedent( + """\ + WAS_ERR=0 + set +e + if [[ ! {if_exists_part} ]]; then + {mkdir_chown_part} + {exec_before_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do + docker exec -u postgres {container} sh -lc '{dump_prefix_cmd} pg_dump -U \"$POSTGRES_USER\" --create {postgresql_clean} {pg_dump_args} {pg_dump_format_part} --verbose {source}' 2> >({pg_dump_filter}) {pg_dump_line_pipe_part} + if [[ $? -ne 0 ]]; then + WAS_ERR=1 + echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" + else + echo "NOTICE: Dump succeeded, attempt $DUMP_ATTEMPT of {dump_attempts}" + break + fi + done + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + {exec_after_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + else + echo "NOTICE: Valid dump already exists, skipping" + fi + set -e + if [[ $WAS_ERR -ne 0 ]]; then false; else true; fi + """ + ).format( + postgresql_dump_dir=item["postgresql_dump_dir"], + postgresql_clean="" if item["postgresql_noclean"] else "--clean --if-exists", + dump_prefix_cmd=item["dump_prefix_cmd"], + exec_before_dump=item["exec_before_dump"], + exec_after_dump=item["exec_after_dump"], + pg_dump_args=item["pg_dump_args"], + grep_db_filter=grep_db_filter, + source=item["source"], + pg_dump_filter=item["pg_dump_filter"], + dump_attempts=item["dump_attempts"], + pg_dump_line_pipe_part=pg_dump_line_pipe_part, + pg_dump_format_part=pg_dump_format_part, + if_exists_part=if_exists_part, + mkdir_chown_part=mkdir_chown_part, + container=item["docker_container"] + ) if "postgresql_dump_type" in item and item["postgresql_dump_type"] == "directory": - pg_dump_line_pipe_part = "" - pg_dump_format_part = "--format=directory --file={postgresql_dump_dir}/$db".format(postgresql_dump_dir=item["postgresql_dump_dir"]) - if_exists_part = "-d {postgresql_dump_dir}/$db".format(postgresql_dump_dir=item["postgresql_dump_dir"]) - mkdir_chown_part = "mkdir -p {postgresql_dump_dir}/$db && chown postgres:postgres {postgresql_dump_dir}/$db".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + # Name mask is not needed for directory dumps, as directories are used as db dump folders only + find_part = '-type d' + # With directory format pg_dump writes by itself to the directory, chown needed for pg_dump to be able to traverse to dump directory + chown_part = "chown postgres:postgres {postgresql_dump_dir}".format(postgresql_dump_dir=item["postgresql_dump_dir"]) else: - pg_dump_line_pipe_part = "| gzip > {postgresql_dump_dir}/$db.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"]) - pg_dump_format_part = "--format=plain" - if_exists_part = "-f {postgresql_dump_dir}/$db.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"]) - mkdir_chown_part = "" + find_part = '-type f -name "*.gz"' + chown_part = "" - script_dump_part = textwrap.dedent( + # If hourly retains are used keep dumps only for 59 minutes + script = textwrap.dedent( """\ - {pg_run} "echo SELECT datname FROM pg_database | psql {pg_user} --no-align -t template1" {grep_db_filter} | grep -v -e template0 -e template1 > {postgresql_dump_dir}/db_list.txt - WAS_ERR=0 - for db in $(cat {postgresql_dump_dir}/db_list.txt); do - set +e - if [[ ! {if_exists_part} ]]; then - {mkdir_chown_part} - {exec_before_dump} - if [[ $? -ne 0 ]]; then WAS_ERR=1; fi - for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do - {pg_run} "{dump_prefix_cmd} pg_dump {pg_user} --create {postgresql_clean} {pg_dump_args} {pg_dump_format_part} --verbose $db" 2> >({pg_dump_filter}) {pg_dump_line_pipe_part} - if [[ $? -ne 0 ]]; then - WAS_ERR=1 - echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" - else - echo "NOTICE: Dump succeeded, attempt $DUMP_ATTEMPT of {dump_attempts}" - break - fi - done - if [[ $? -ne 0 ]]; then WAS_ERR=1; fi - {exec_after_dump} - if [[ $? -ne 0 ]]; then WAS_ERR=1; fi - else - echo "NOTICE: Valid dump already exists, skipping" - fi - set -e - done - if [[ $WAS_ERR -ne 0 ]]; then false; else true; fi + #!/bin/bash + set -e + + ssh {ssh_args} -p {port} {user}@{host} bash -se <<'EOF' + set -x + set -e + set -o pipefail + mkdir -p {postgresql_dump_dir} + chmod 700 {postgresql_dump_dir} + {chown_part} + while [[ -d {postgresql_dump_dir}/dump.lock ]]; do + sleep 5 + done + mkdir {postgresql_dump_dir}/dump.lock + trap "rm -rf {postgresql_dump_dir}/dump.lock" 0 + cd {postgresql_dump_dir} + find {postgresql_dump_dir} {find_part} -mmin +{mmin} -exec rm -rf {{}} + + {exec_before_dump} + {comment_out_pg_dumpall} docker exec -u postgres {container} sh -lc 'pg_dumpall -U "$POSTGRES_USER" --clean --if-exists --schema-only --verbose' 2> >({pg_dump_filter}) | gzip > {postgresql_dump_dir}/globals.gz + {exec_after_dump} + {script_dump_part} + EOF """ ).format( + ssh_args=ssh_args, + port=item["connect_port"], + user=item["connect_user"], + host=item["connect_host"], postgresql_dump_dir=item["postgresql_dump_dir"], - postgresql_clean="" if item["postgresql_noclean"] else "--clean --if-exists", - dump_prefix_cmd=item["dump_prefix_cmd"], + mmin="59" if "retain_hourly" in item else "720", + script_dump_part=script_dump_part, + pg_dump_filter=item["pg_dump_filter"], exec_before_dump=item["exec_before_dump"], exec_after_dump=item["exec_after_dump"], - pg_dump_args=item["pg_dump_args"], - grep_db_filter=grep_db_filter, - pg_dump_filter=item["pg_dump_filter"], - dump_attempts=item["dump_attempts"], - pg_dump_line_pipe_part=pg_dump_line_pipe_part, - pg_dump_format_part=pg_dump_format_part, - if_exists_part=if_exists_part, - mkdir_chown_part=mkdir_chown_part, - pg_run="docker exec -u postgres {container} sh -lc".format(container=item["docker_container"]) if item["docker_mode"] else "su - postgres -c", - pg_user="-U {db_user}".format(db_user=item["db_user"]) if item["db_user"] else "" + find_part=find_part, + chown_part=chown_part, + comment_out_pg_dumpall="#" if item["postgresql_skip_globals"] else "", + container=item["docker_container"] ) else: + if item["source"] == "ALL": + + if "postgresql_dump_type" in item and item["postgresql_dump_type"] == "directory": + pg_dump_line_pipe_part = "" + pg_dump_format_part = "--format=directory --file={postgresql_dump_dir}/$db".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + if_exists_part = "-d {postgresql_dump_dir}/$db".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + mkdir_chown_part = "mkdir -p {postgresql_dump_dir}/$db && chown postgres:postgres {postgresql_dump_dir}/$db".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + else: + pg_dump_line_pipe_part = "| gzip > {postgresql_dump_dir}/$db.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + pg_dump_format_part = "--format=plain" + if_exists_part = "-f {postgresql_dump_dir}/$db.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + mkdir_chown_part = "" + + script_dump_part = textwrap.dedent( + """\ + su - postgres -c "echo SELECT datname FROM pg_database | psql --no-align -t template1" {grep_db_filter} | grep -v -e template0 -e template1 > {postgresql_dump_dir}/db_list.txt + WAS_ERR=0 + for db in $(cat {postgresql_dump_dir}/db_list.txt); do + set +e + if [[ ! {if_exists_part} ]]; then + {mkdir_chown_part} + {exec_before_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do + su - postgres -c "{dump_prefix_cmd} pg_dump --create {postgresql_clean} {pg_dump_args} {pg_dump_format_part} --verbose $db" 2> >({pg_dump_filter}) {pg_dump_line_pipe_part} + if [[ $? -ne 0 ]]; then + WAS_ERR=1 + echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" + else + echo "NOTICE: Dump succeeded, attempt $DUMP_ATTEMPT of {dump_attempts}" + break + fi + done + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + {exec_after_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + else + echo "NOTICE: Valid dump already exists, skipping" + fi + set -e + done + if [[ $WAS_ERR -ne 0 ]]; then false; else true; fi + """ + ).format( + postgresql_dump_dir=item["postgresql_dump_dir"], + postgresql_clean="" if item["postgresql_noclean"] else "--clean --if-exists", + dump_prefix_cmd=item["dump_prefix_cmd"], + exec_before_dump=item["exec_before_dump"], + exec_after_dump=item["exec_after_dump"], + pg_dump_args=item["pg_dump_args"], + grep_db_filter=grep_db_filter, + pg_dump_filter=item["pg_dump_filter"], + dump_attempts=item["dump_attempts"], + pg_dump_line_pipe_part=pg_dump_line_pipe_part, + pg_dump_format_part=pg_dump_format_part, + if_exists_part=if_exists_part, + mkdir_chown_part=mkdir_chown_part + ) + else: + + if "postgresql_dump_type" in item and item["postgresql_dump_type"] == "directory": + pg_dump_line_pipe_part = "" + pg_dump_format_part = "--format=directory --file={postgresql_dump_dir}/{source}".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) + if_exists_part = "-d {postgresql_dump_dir}/{source}".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) + mkdir_chown_part = "mkdir -p {postgresql_dump_dir}/{source} && chown postgres:postgres {postgresql_dump_dir}/{source}".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) + else: + pg_dump_line_pipe_part = "| gzip > {postgresql_dump_dir}/{source}.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) + pg_dump_format_part = "--format=plain" + if_exists_part = "-f {postgresql_dump_dir}/{source}.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) + mkdir_chown_part = "" + + script_dump_part = textwrap.dedent( + """\ + WAS_ERR=0 + set +e + if [[ ! {if_exists_part} ]]; then + {mkdir_chown_part} + {exec_before_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do + su - postgres -c "{dump_prefix_cmd} pg_dump --create {postgresql_clean} {pg_dump_args} {pg_dump_format_part} --verbose {source}" 2> >({pg_dump_filter}) {pg_dump_line_pipe_part} + if [[ $? -ne 0 ]]; then + WAS_ERR=1 + echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" + else + echo "NOTICE: Dump succeeded, attempt $DUMP_ATTEMPT of {dump_attempts}" + break + fi + done + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + {exec_after_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + else + echo "NOTICE: Valid dump already exists, skipping" + fi + set -e + if [[ $WAS_ERR -ne 0 ]]; then false; else true; fi + """ + ).format( + postgresql_dump_dir=item["postgresql_dump_dir"], + postgresql_clean="" if item["postgresql_noclean"] else "--clean --if-exists", + dump_prefix_cmd=item["dump_prefix_cmd"], + exec_before_dump=item["exec_before_dump"], + exec_after_dump=item["exec_after_dump"], + pg_dump_args=item["pg_dump_args"], + grep_db_filter=grep_db_filter, + source=item["source"], + pg_dump_filter=item["pg_dump_filter"], + dump_attempts=item["dump_attempts"], + pg_dump_line_pipe_part=pg_dump_line_pipe_part, + pg_dump_format_part=pg_dump_format_part, + if_exists_part=if_exists_part, + mkdir_chown_part=mkdir_chown_part + ) if "postgresql_dump_type" in item and item["postgresql_dump_type"] == "directory": - pg_dump_line_pipe_part = "" - pg_dump_format_part = "--format=directory --file={postgresql_dump_dir}/{source}".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) - if_exists_part = "-d {postgresql_dump_dir}/{source}".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) - mkdir_chown_part = "mkdir -p {postgresql_dump_dir}/{source} && chown postgres:postgres {postgresql_dump_dir}/{source}".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) + # Name mask is not needed for directory dumps, as directories are used as db dump folders only + find_part = '-type d' + # With directory format pg_dump writes by itself to the directory, chown needed for pg_dump to be able to traverse to dump directory + chown_part = "chown postgres:postgres {postgresql_dump_dir}".format(postgresql_dump_dir=item["postgresql_dump_dir"]) else: - pg_dump_line_pipe_part = "| gzip > {postgresql_dump_dir}/{source}.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) - pg_dump_format_part = "--format=plain" - if_exists_part = "-f {postgresql_dump_dir}/{source}.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) - mkdir_chown_part = "" + find_part = '-type f -name "*.gz"' + chown_part = "" - script_dump_part = textwrap.dedent( + # If hourly retains are used keep dumps only for 59 minutes + script = textwrap.dedent( """\ - WAS_ERR=0 - set +e - if [[ ! {if_exists_part} ]]; then - {mkdir_chown_part} - {exec_before_dump} - if [[ $? -ne 0 ]]; then WAS_ERR=1; fi - for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do - {pg_run} "{dump_prefix_cmd} pg_dump {pg_user} --create {postgresql_clean} {pg_dump_args} {pg_dump_format_part} --verbose {source}" 2> >({pg_dump_filter}) {pg_dump_line_pipe_part} - if [[ $? -ne 0 ]]; then - WAS_ERR=1 - echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" - else - echo "NOTICE: Dump succeeded, attempt $DUMP_ATTEMPT of {dump_attempts}" - break - fi - done - if [[ $? -ne 0 ]]; then WAS_ERR=1; fi - {exec_after_dump} - if [[ $? -ne 0 ]]; then WAS_ERR=1; fi - else - echo "NOTICE: Valid dump already exists, skipping" - fi + #!/bin/bash set -e - if [[ $WAS_ERR -ne 0 ]]; then false; else true; fi + + ssh {ssh_args} -p {port} {user}@{host} ' + set -x + set -e + set -o pipefail + mkdir -p {postgresql_dump_dir} + chmod 700 {postgresql_dump_dir} + {chown_part} + while [[ -d {postgresql_dump_dir}/dump.lock ]]; do + sleep 5 + done + mkdir {postgresql_dump_dir}/dump.lock + trap "rm -rf {postgresql_dump_dir}/dump.lock" 0 + cd {postgresql_dump_dir} + find {postgresql_dump_dir} {find_part} -mmin +{mmin} -exec rm -rf {{}} + + {exec_before_dump} + {comment_out_pg_dumpall}su - postgres -c "pg_dumpall --clean --if-exists --schema-only --verbose" 2> >({pg_dump_filter}) | gzip > {postgresql_dump_dir}/globals.gz + {exec_after_dump} + {script_dump_part} + ' """ ).format( + ssh_args=ssh_args, + port=item["connect_port"], + user=item["connect_user"], + host=item["connect_host"], postgresql_dump_dir=item["postgresql_dump_dir"], - postgresql_clean="" if item["postgresql_noclean"] else "--clean --if-exists", - dump_prefix_cmd=item["dump_prefix_cmd"], + mmin="59" if "retain_hourly" in item else "720", + script_dump_part=script_dump_part, + pg_dump_filter=item["pg_dump_filter"], exec_before_dump=item["exec_before_dump"], exec_after_dump=item["exec_after_dump"], - pg_dump_args=item["pg_dump_args"], - grep_db_filter=grep_db_filter, - source=item["source"], - pg_dump_filter=item["pg_dump_filter"], - dump_attempts=item["dump_attempts"], - pg_dump_line_pipe_part=pg_dump_line_pipe_part, - pg_dump_format_part=pg_dump_format_part, - if_exists_part=if_exists_part, - mkdir_chown_part=mkdir_chown_part, - pg_run="docker exec -u postgres {container} sh -lc".format(container=item["docker_container"]) if item["docker_mode"] else "su - postgres -c", - pg_user="-U {db_user}".format(db_user=item["db_user"]) if item["db_user"] else "" + find_part=find_part, + chown_part=chown_part, + comment_out_pg_dumpall="#" if item["postgresql_skip_globals"] else "" ) - if "postgresql_dump_type" in item and item["postgresql_dump_type"] == "directory": - # Name mask is not needed for directory dumps, as directories are used as db dump folders only - find_part = '-type d' - # With directory format pg_dump writes by itself to the directory, chown needed for pg_dump to be able to traverse to dump directory - chown_part = "chown postgres:postgres {postgresql_dump_dir}".format(postgresql_dump_dir=item["postgresql_dump_dir"]) - else: - find_part = '-type f -name "*.gz"' - chown_part = "" - - # If hourly retains are used keep dumps only for 59 minutes - script = textwrap.dedent( - """\ - #!/bin/bash - set -e - - ssh {ssh_args} -p {port} {user}@{host} ' - set -x - set -e - set -o pipefail - mkdir -p {postgresql_dump_dir} - chmod 700 {postgresql_dump_dir} - {chown_part} - while [[ -d {postgresql_dump_dir}/dump.lock ]]; do - sleep 5 - done - mkdir {postgresql_dump_dir}/dump.lock - trap "rm -rf {postgresql_dump_dir}/dump.lock" 0 - cd {postgresql_dump_dir} - find {postgresql_dump_dir} {find_part} -mmin +{mmin} -exec rm -rf {{}} + - {exec_before_dump} - {comment_out_pg_dumpall}{pg_run} "pg_dumpall {pg_user} --clean --if-exists --schema-only --verbose" 2> >({pg_dump_filter}) | gzip > {postgresql_dump_dir}/globals.gz - {exec_after_dump} - {script_dump_part} - ' - """ - ).format( - ssh_args=ssh_args, - port=item["connect_port"], - user=item["connect_user"], - host=item["connect_host"], - postgresql_dump_dir=item["postgresql_dump_dir"], - mmin="59" if "retain_hourly" in item else "720", - script_dump_part=script_dump_part, - pg_dump_filter=item["pg_dump_filter"], - exec_before_dump=item["exec_before_dump"], - exec_after_dump=item["exec_after_dump"], - find_part=find_part, - chown_part=chown_part, - comment_out_pg_dumpall="#" if item["postgresql_skip_globals"] else "", - pg_run="docker exec -u postgres {container} sh -lc".format(container=item["docker_container"]) if item["docker_mode"] else "su - postgres -c", - pg_user="-U {db_user}".format(db_user=item["db_user"]) if item["db_user"] else "" - ) - if item["type"] == "MONGODB_SSH": if item["source"] == "ALL":