diff --git a/rsnapshot_backup/rsnapshot_backup.py b/rsnapshot_backup/rsnapshot_backup.py index 2b2bf39..902653f 100755 --- a/rsnapshot_backup/rsnapshot_backup.py +++ b/rsnapshot_backup/rsnapshot_backup.py @@ -303,8 +303,6 @@ def run_cmd_pipe(cmd): item["docker_mode"] = False if "docker_container" not in item: item["docker_container"] = "empty" - if "db_user" not in item: - item["db_user"] = None # Check before_backup_check and skip item if failed # It is needed for both rotations and sync @@ -921,19 +919,267 @@ def run_cmd_pipe(cmd): ) else: + if item["docker_mode"]: + if item["source"] == "ALL": + script_dump_part = textwrap.dedent( + """\ + docker exec {container} sh -lc 'mysql -u"$MYSQL_USER" -p"$MYSQL_PASSWORD" --skip-column-names --batch -e "SHOW DATABASES;"' | grep -v -e information_schema -e performance_schema {grep_db_filter} > {mysql_dump_dir}/db_list.txt + WAS_ERR=0 + for db in $(cat {mysql_dump_dir}/db_list.txt); do + set +e + if [[ ! -f {mysql_dump_dir}/$db.gz ]]; then + {exec_before_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do + docker exec {container} sh -lc '{dump_prefix_cmd} mysqldump -u"$MYSQL_USER" -p"$MYSQL_PASSWORD" --force --opt --single-transaction --quick --skip-lock-tables {mysql_events} --databases $1 --max_allowed_packet=1G {mysqldump_args}' -- $db | gzip > {mysql_dump_dir}/$db.gz + if [[ $? -ne 0 ]]; then + WAS_ERR=1 + echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" + else + echo "NOTICE: Dump succeeded, attempt $DUMP_ATTEMPT of {dump_attempts}" + break + fi + done + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + {exec_after_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + else + echo "NOTICE: Valid dump already exists, skipping" + fi + set -e + done + if [[ $WAS_ERR -ne 0 ]]; then false; else true; fi + """ + ).format( + mysql_dump_dir=item["mysql_dump_dir"], + mysql_events="" if item["mysql_noevents"] else "--events", + dump_prefix_cmd=item["dump_prefix_cmd"], + exec_before_dump=item["exec_before_dump"], + exec_after_dump=item["exec_after_dump"], + mysqldump_args=item["mysqldump_args"], + grep_db_filter=grep_db_filter, + dump_attempts=item["dump_attempts"], + container=item["docker_container"] + ) + else: + script_dump_part = textwrap.dedent( + """\ + WAS_ERR=0 + set +e + if [[ ! -f {mysql_dump_dir}/{source}.gz ]]; then + {exec_before_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do + docker exec {container} sh -lc '{dump_prefix_cmd} mysqldump -u"$MYSQL_USER" -p"$MYSQL_PASSWORD" --force --opt --single-transaction --quick --skip-lock-tables {mysql_events} --databases {source} --max_allowed_packet=1G {mysqldump_args}' | gzip > {mysql_dump_dir}/{source}.gz + if [[ $? -ne 0 ]]; then + WAS_ERR=1 + echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" + else + echo "NOTICE: Dump succeeded, attempt $DUMP_ATTEMPT of {dump_attempts}" + break + fi + done + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + {exec_after_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + else + echo "NOTICE: Valid dump already exists, skipping" + fi + set -e + if [[ $WAS_ERR -ne 0 ]]; then false; else true; fi + """ + ).format( + mysql_dump_dir=item["mysql_dump_dir"], + mysql_events="" if item["mysql_noevents"] else "--events", + dump_prefix_cmd=item["dump_prefix_cmd"], + exec_before_dump=item["exec_before_dump"], + exec_after_dump=item["exec_after_dump"], + mysqldump_args=item["mysqldump_args"], + grep_db_filter=grep_db_filter, + source=item["source"], + dump_attempts=item["dump_attempts"], + container=item["docker_container"] + ) + + # If hourly retains are used keep dumps only for 59 minutes + script = textwrap.dedent( + """\ + #!/bin/bash + set -e + ssh {ssh_args} -p {port} {user}@{host} bash -se <<'EOF' + set -x + set -e + set -o pipefail + mkdir -p {mysql_dump_dir} + chmod 700 {mysql_dump_dir} + while [[ -d {mysql_dump_dir}/dump.lock ]]; do + sleep 5 + done + mkdir {mysql_dump_dir}/dump.lock + trap "rm -rf {mysql_dump_dir}/dump.lock" 0 + cd {mysql_dump_dir} + find {mysql_dump_dir} -type f -name "*.gz" -mmin +{mmin} -delete + {script_dump_part} + EOF + """ + ).format( + ssh_args=ssh_args, + port=item["connect_port"], + user=item["connect_user"], + host=item["connect_host"], + mysql_dump_dir=item["mysql_dump_dir"], + mmin="59" if "retain_hourly" in item else "720", + script_dump_part=script_dump_part + ) + + else: + if item["source"] == "ALL": + script_dump_part = textwrap.dedent( + """\ + mysql --defaults-file=/etc/mysql/debian.cnf --skip-column-names --batch -e "SHOW DATABASES;" | grep -v -e information_schema -e performance_schema {grep_db_filter} > {mysql_dump_dir}/db_list.txt + WAS_ERR=0 + for db in $(cat {mysql_dump_dir}/db_list.txt); do + set +e + if [[ ! -f {mysql_dump_dir}/$db.gz ]]; then + {exec_before_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do + {dump_prefix_cmd} mysqldump --defaults-file=/etc/mysql/debian.cnf --force --opt --single-transaction --quick --skip-lock-tables {mysql_events} --databases $db --max_allowed_packet=1G {mysqldump_args} | gzip > {mysql_dump_dir}/$db.gz + if [[ $? -ne 0 ]]; then + WAS_ERR=1 + echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" + else + echo "NOTICE: Dump succeeded, attempt $DUMP_ATTEMPT of {dump_attempts}" + break + fi + done + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + {exec_after_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + else + echo "NOTICE: Valid dump already exists, skipping" + fi + set -e + done + if [[ $WAS_ERR -ne 0 ]]; then false; else true; fi + """ + ).format( + mysql_dump_dir=item["mysql_dump_dir"], + mysql_events="" if item["mysql_noevents"] else "--events", + dump_prefix_cmd=item["dump_prefix_cmd"], + exec_before_dump=item["exec_before_dump"], + exec_after_dump=item["exec_after_dump"], + mysqldump_args=item["mysqldump_args"], + grep_db_filter=grep_db_filter, + dump_attempts=item["dump_attempts"] + ) + else: + script_dump_part = textwrap.dedent( + """\ + WAS_ERR=0 + set +e + if [[ ! -f {mysql_dump_dir}/{source}.gz ]]; then + {exec_before_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do + {dump_prefix_cmd} mysqldump --defaults-file=/etc/mysql/debian.cnf --force --opt --single-transaction --quick --skip-lock-tables {mysql_events} --databases {source} --max_allowed_packet=1G {mysqldump_args} | gzip > {mysql_dump_dir}/{source}.gz + if [[ $? -ne 0 ]]; then + WAS_ERR=1 + echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" + else + echo "NOTICE: Dump succeeded, attempt $DUMP_ATTEMPT of {dump_attempts}" + break + fi + done + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + {exec_after_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + else + echo "NOTICE: Valid dump already exists, skipping" + fi + set -e + if [[ $WAS_ERR -ne 0 ]]; then false; else true; fi + """ + ).format( + mysql_dump_dir=item["mysql_dump_dir"], + mysql_events="" if item["mysql_noevents"] else "--events", + dump_prefix_cmd=item["dump_prefix_cmd"], + exec_before_dump=item["exec_before_dump"], + exec_after_dump=item["exec_after_dump"], + mysqldump_args=item["mysqldump_args"], + grep_db_filter=grep_db_filter, + source=item["source"], + dump_attempts=item["dump_attempts"] + ) + + # If hourly retains are used keep dumps only for 59 minutes + script = textwrap.dedent( + """\ + #!/bin/bash + set -e + + ssh {ssh_args} -p {port} {user}@{host} ' + set -x + set -e + set -o pipefail + mkdir -p {mysql_dump_dir} + chmod 700 {mysql_dump_dir} + while [[ -d {mysql_dump_dir}/dump.lock ]]; do + sleep 5 + done + mkdir {mysql_dump_dir}/dump.lock + trap "rm -rf {mysql_dump_dir}/dump.lock" 0 + cd {mysql_dump_dir} + find {mysql_dump_dir} -type f -name "*.gz" -mmin +{mmin} -delete + {script_dump_part} + ' + """ + ).format( + ssh_args=ssh_args, + port=item["connect_port"], + user=item["connect_user"], + host=item["connect_host"], + mysql_dump_dir=item["mysql_dump_dir"], + mmin="59" if "retain_hourly" in item else "720", + script_dump_part=script_dump_part + ) + + if item["type"] == "POSTGRESQL_SSH": + + # --verbose is needed for Completed on signature in dumps, and sdterr shouldn't be redirected to /dev/null + # and it produces a lot of noise, we need to filter it + # 2>&1 cannot be used before | gzip + # grep output should be put into stderr again + # https://unix.stackexchange.com/questions/3514/how-to-grep-standard-error-stream-stderr + # allow pg_dump.*: connecting just to see what it does + + if item["docker_mode"]: if item["source"] == "ALL": + + if "postgresql_dump_type" in item and item["postgresql_dump_type"] == "directory": + pg_dump_line_pipe_part = "" + pg_dump_format_part = "--format=directory --file={postgresql_dump_dir}/$db".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + if_exists_part = "-d {postgresql_dump_dir}/$db".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + mkdir_chown_part = "mkdir -p {postgresql_dump_dir}/$db && chown postgres:postgres {postgresql_dump_dir}/$db".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + else: + pg_dump_line_pipe_part = "| gzip > {postgresql_dump_dir}/$db.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + pg_dump_format_part = "--format=plain" + if_exists_part = "-f {postgresql_dump_dir}/$db.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + mkdir_chown_part = "" + script_dump_part = textwrap.dedent( """\ - mysql --defaults-file=/etc/mysql/debian.cnf --skip-column-names --batch -e "SHOW DATABASES;" | grep -v -e information_schema -e performance_schema {grep_db_filter} > {mysql_dump_dir}/db_list.txt + docker exec -u postgres {container} sh -lc 'echo SELECT datname FROM pg_database | psql -U "$POSTGRES_USER" --no-align -t template1' {grep_db_filter} | grep -v -e template0 -e template1 > {postgresql_dump_dir}/db_list.txt WAS_ERR=0 - for db in $(cat {mysql_dump_dir}/db_list.txt); do + for db in $(cat {postgresql_dump_dir}/db_list.txt); do set +e - if [[ ! -f {mysql_dump_dir}/$db.gz ]]; then + if [[ ! {if_exists_part} ]]; then + {mkdir_chown_part} {exec_before_dump} if [[ $? -ne 0 ]]; then WAS_ERR=1; fi for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do - {dump_prefix_cmd} mysqldump --defaults-file=/etc/mysql/debian.cnf --force --opt --single-transaction --quick --skip-lock-tables {mysql_events} --databases $db --max_allowed_packet=1G {mysqldump_args} | gzip > {mysql_dump_dir}/$db.gz + docker exec -u postgres {container} sh -lc '{dump_prefix_cmd} pg_dump -U "$POSTGRES_USER" --create {postgresql_clean} {pg_dump_args} {pg_dump_format_part} --verbose $1' -- $db 2> >({pg_dump_filter}) {pg_dump_line_pipe_part} if [[ $? -ne 0 ]]; then WAS_ERR=1 echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" @@ -953,25 +1199,44 @@ def run_cmd_pipe(cmd): if [[ $WAS_ERR -ne 0 ]]; then false; else true; fi """ ).format( - mysql_dump_dir=item["mysql_dump_dir"], - mysql_events="" if item["mysql_noevents"] else "--events", + postgresql_dump_dir=item["postgresql_dump_dir"], + postgresql_clean="" if item["postgresql_noclean"] else "--clean --if-exists", dump_prefix_cmd=item["dump_prefix_cmd"], exec_before_dump=item["exec_before_dump"], exec_after_dump=item["exec_after_dump"], - mysqldump_args=item["mysqldump_args"], + pg_dump_args=item["pg_dump_args"], grep_db_filter=grep_db_filter, - dump_attempts=item["dump_attempts"] + pg_dump_filter=item["pg_dump_filter"], + dump_attempts=item["dump_attempts"], + pg_dump_line_pipe_part=pg_dump_line_pipe_part, + pg_dump_format_part=pg_dump_format_part, + if_exists_part=if_exists_part, + mkdir_chown_part=mkdir_chown_part, + container=item["docker_container"] ) else: + + if "postgresql_dump_type" in item and item["postgresql_dump_type"] == "directory": + pg_dump_line_pipe_part = "" + pg_dump_format_part = "--format=directory --file={postgresql_dump_dir}/{source}".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) + if_exists_part = "-d {postgresql_dump_dir}/{source}".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) + mkdir_chown_part = "mkdir -p {postgresql_dump_dir}/{source} && chown postgres:postgres {postgresql_dump_dir}/{source}".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) + else: + pg_dump_line_pipe_part = "| gzip > {postgresql_dump_dir}/{source}.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) + pg_dump_format_part = "--format=plain" + if_exists_part = "-f {postgresql_dump_dir}/{source}.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) + mkdir_chown_part = "" + script_dump_part = textwrap.dedent( """\ WAS_ERR=0 set +e - if [[ ! -f {mysql_dump_dir}/{source}.gz ]]; then + if [[ ! {if_exists_part} ]]; then + {mkdir_chown_part} {exec_before_dump} if [[ $? -ne 0 ]]; then WAS_ERR=1; fi for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do - {dump_prefix_cmd} mysqldump --defaults-file=/etc/mysql/debian.cnf --force --opt --single-transaction --quick --skip-lock-tables {mysql_events} --databases {source} --max_allowed_packet=1G {mysqldump_args} | gzip > {mysql_dump_dir}/{source}.gz + docker exec -u postgres {container} sh -lc '{dump_prefix_cmd} pg_dump -U \"$POSTGRES_USER\" --create {postgresql_clean} {pg_dump_args} {pg_dump_format_part} --verbose {source}' 2> >({pg_dump_filter}) {pg_dump_line_pipe_part} if [[ $? -ne 0 ]]; then WAS_ERR=1 echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" @@ -990,229 +1255,241 @@ def run_cmd_pipe(cmd): if [[ $WAS_ERR -ne 0 ]]; then false; else true; fi """ ).format( - mysql_dump_dir=item["mysql_dump_dir"], - mysql_events="" if item["mysql_noevents"] else "--events", + postgresql_dump_dir=item["postgresql_dump_dir"], + postgresql_clean="" if item["postgresql_noclean"] else "--clean --if-exists", dump_prefix_cmd=item["dump_prefix_cmd"], exec_before_dump=item["exec_before_dump"], exec_after_dump=item["exec_after_dump"], - mysqldump_args=item["mysqldump_args"], + pg_dump_args=item["pg_dump_args"], grep_db_filter=grep_db_filter, source=item["source"], - dump_attempts=item["dump_attempts"] + pg_dump_filter=item["pg_dump_filter"], + dump_attempts=item["dump_attempts"], + pg_dump_line_pipe_part=pg_dump_line_pipe_part, + pg_dump_format_part=pg_dump_format_part, + if_exists_part=if_exists_part, + mkdir_chown_part=mkdir_chown_part, + container=item["docker_container"] ) + if "postgresql_dump_type" in item and item["postgresql_dump_type"] == "directory": + # Name mask is not needed for directory dumps, as directories are used as db dump folders only + find_part = '-type d' + # With directory format pg_dump writes by itself to the directory, chown needed for pg_dump to be able to traverse to dump directory + chown_part = "chown postgres:postgres {postgresql_dump_dir}".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + else: + find_part = '-type f -name "*.gz"' + chown_part = "" + # If hourly retains are used keep dumps only for 59 minutes script = textwrap.dedent( """\ #!/bin/bash set -e - ssh {ssh_args} -p {port} {user}@{host} ' + ssh {ssh_args} -p {port} {user}@{host} bash -se <<'EOF' set -x set -e set -o pipefail - mkdir -p {mysql_dump_dir} - chmod 700 {mysql_dump_dir} - while [[ -d {mysql_dump_dir}/dump.lock ]]; do + mkdir -p {postgresql_dump_dir} + chmod 700 {postgresql_dump_dir} + {chown_part} + while [[ -d {postgresql_dump_dir}/dump.lock ]]; do sleep 5 done - mkdir {mysql_dump_dir}/dump.lock - trap "rm -rf {mysql_dump_dir}/dump.lock" 0 - cd {mysql_dump_dir} - find {mysql_dump_dir} -type f -name "*.gz" -mmin +{mmin} -delete + mkdir {postgresql_dump_dir}/dump.lock + trap "rm -rf {postgresql_dump_dir}/dump.lock" 0 + cd {postgresql_dump_dir} + find {postgresql_dump_dir} {find_part} -mmin +{mmin} -exec rm -rf {{}} + + {exec_before_dump} + {comment_out_pg_dumpall} docker exec -u postgres {container} sh -lc 'pg_dumpall -U "$POSTGRES_USER" --clean --if-exists --schema-only --verbose' 2> >({pg_dump_filter}) | gzip > {postgresql_dump_dir}/globals.gz + {exec_after_dump} {script_dump_part} - ' + EOF """ ).format( ssh_args=ssh_args, port=item["connect_port"], user=item["connect_user"], host=item["connect_host"], - mysql_dump_dir=item["mysql_dump_dir"], + postgresql_dump_dir=item["postgresql_dump_dir"], mmin="59" if "retain_hourly" in item else "720", - script_dump_part=script_dump_part + script_dump_part=script_dump_part, + pg_dump_filter=item["pg_dump_filter"], + exec_before_dump=item["exec_before_dump"], + exec_after_dump=item["exec_after_dump"], + find_part=find_part, + chown_part=chown_part, + comment_out_pg_dumpall="#" if item["postgresql_skip_globals"] else "", + container=item["docker_container"] ) + else: + if item["source"] == "ALL": - if item["type"] == "POSTGRESQL_SSH": - - # --verbose is needed for Completed on signature in dumps, and sdterr shouldn't be redirected to /dev/null - # and it produces a lot of noise, we need to filter it - # 2>&1 cannot be used before | gzip - # grep output should be put into stderr again - # https://unix.stackexchange.com/questions/3514/how-to-grep-standard-error-stream-stderr - # allow pg_dump.*: connecting just to see what it does - if item["source"] == "ALL": + if "postgresql_dump_type" in item and item["postgresql_dump_type"] == "directory": + pg_dump_line_pipe_part = "" + pg_dump_format_part = "--format=directory --file={postgresql_dump_dir}/$db".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + if_exists_part = "-d {postgresql_dump_dir}/$db".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + mkdir_chown_part = "mkdir -p {postgresql_dump_dir}/$db && chown postgres:postgres {postgresql_dump_dir}/$db".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + else: + pg_dump_line_pipe_part = "| gzip > {postgresql_dump_dir}/$db.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + pg_dump_format_part = "--format=plain" + if_exists_part = "-f {postgresql_dump_dir}/$db.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + mkdir_chown_part = "" - if "postgresql_dump_type" in item and item["postgresql_dump_type"] == "directory": - pg_dump_line_pipe_part = "" - pg_dump_format_part = "--format=directory --file={postgresql_dump_dir}/$db".format(postgresql_dump_dir=item["postgresql_dump_dir"]) - if_exists_part = "-d {postgresql_dump_dir}/$db".format(postgresql_dump_dir=item["postgresql_dump_dir"]) - mkdir_chown_part = "mkdir -p {postgresql_dump_dir}/$db && chown postgres:postgres {postgresql_dump_dir}/$db".format(postgresql_dump_dir=item["postgresql_dump_dir"]) + script_dump_part = textwrap.dedent( + """\ + su - postgres -c "echo SELECT datname FROM pg_database | psql --no-align -t template1" {grep_db_filter} | grep -v -e template0 -e template1 > {postgresql_dump_dir}/db_list.txt + WAS_ERR=0 + for db in $(cat {postgresql_dump_dir}/db_list.txt); do + set +e + if [[ ! {if_exists_part} ]]; then + {mkdir_chown_part} + {exec_before_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do + su - postgres -c "{dump_prefix_cmd} pg_dump --create {postgresql_clean} {pg_dump_args} {pg_dump_format_part} --verbose $db" 2> >({pg_dump_filter}) {pg_dump_line_pipe_part} + if [[ $? -ne 0 ]]; then + WAS_ERR=1 + echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" + else + echo "NOTICE: Dump succeeded, attempt $DUMP_ATTEMPT of {dump_attempts}" + break + fi + done + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + {exec_after_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + else + echo "NOTICE: Valid dump already exists, skipping" + fi + set -e + done + if [[ $WAS_ERR -ne 0 ]]; then false; else true; fi + """ + ).format( + postgresql_dump_dir=item["postgresql_dump_dir"], + postgresql_clean="" if item["postgresql_noclean"] else "--clean --if-exists", + dump_prefix_cmd=item["dump_prefix_cmd"], + exec_before_dump=item["exec_before_dump"], + exec_after_dump=item["exec_after_dump"], + pg_dump_args=item["pg_dump_args"], + grep_db_filter=grep_db_filter, + pg_dump_filter=item["pg_dump_filter"], + dump_attempts=item["dump_attempts"], + pg_dump_line_pipe_part=pg_dump_line_pipe_part, + pg_dump_format_part=pg_dump_format_part, + if_exists_part=if_exists_part, + mkdir_chown_part=mkdir_chown_part + ) else: - pg_dump_line_pipe_part = "| gzip > {postgresql_dump_dir}/$db.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"]) - pg_dump_format_part = "--format=plain" - if_exists_part = "-f {postgresql_dump_dir}/$db.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"]) - mkdir_chown_part = "" - script_dump_part = textwrap.dedent( - """\ - {pg_run} "echo SELECT datname FROM pg_database | psql {pg_user} --no-align -t template1" {grep_db_filter} | grep -v -e template0 -e template1 > {postgresql_dump_dir}/db_list.txt - WAS_ERR=0 - for db in $(cat {postgresql_dump_dir}/db_list.txt); do - set +e - if [[ ! {if_exists_part} ]]; then - {mkdir_chown_part} - {exec_before_dump} - if [[ $? -ne 0 ]]; then WAS_ERR=1; fi - for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do - {pg_run} "{dump_prefix_cmd} pg_dump {pg_user} --create {postgresql_clean} {pg_dump_args} {pg_dump_format_part} --verbose $db" 2> >({pg_dump_filter}) {pg_dump_line_pipe_part} - if [[ $? -ne 0 ]]; then - WAS_ERR=1 - echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" - else - echo "NOTICE: Dump succeeded, attempt $DUMP_ATTEMPT of {dump_attempts}" - break - fi - done - if [[ $? -ne 0 ]]; then WAS_ERR=1; fi - {exec_after_dump} - if [[ $? -ne 0 ]]; then WAS_ERR=1; fi - else - echo "NOTICE: Valid dump already exists, skipping" - fi - set -e - done - if [[ $WAS_ERR -ne 0 ]]; then false; else true; fi - """ - ).format( - postgresql_dump_dir=item["postgresql_dump_dir"], - postgresql_clean="" if item["postgresql_noclean"] else "--clean --if-exists", - dump_prefix_cmd=item["dump_prefix_cmd"], - exec_before_dump=item["exec_before_dump"], - exec_after_dump=item["exec_after_dump"], - pg_dump_args=item["pg_dump_args"], - grep_db_filter=grep_db_filter, - pg_dump_filter=item["pg_dump_filter"], - dump_attempts=item["dump_attempts"], - pg_dump_line_pipe_part=pg_dump_line_pipe_part, - pg_dump_format_part=pg_dump_format_part, - if_exists_part=if_exists_part, - mkdir_chown_part=mkdir_chown_part, - pg_run="docker exec -u postgres {container} sh -lc".format(container=item["docker_container"]) if item["docker_mode"] else "su - postgres -c", - pg_user="-U {db_user}".format(db_user=item["db_user"]) if item["db_user"] else "" - ) - else: + if "postgresql_dump_type" in item and item["postgresql_dump_type"] == "directory": + pg_dump_line_pipe_part = "" + pg_dump_format_part = "--format=directory --file={postgresql_dump_dir}/{source}".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) + if_exists_part = "-d {postgresql_dump_dir}/{source}".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) + mkdir_chown_part = "mkdir -p {postgresql_dump_dir}/{source} && chown postgres:postgres {postgresql_dump_dir}/{source}".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) + else: + pg_dump_line_pipe_part = "| gzip > {postgresql_dump_dir}/{source}.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) + pg_dump_format_part = "--format=plain" + if_exists_part = "-f {postgresql_dump_dir}/{source}.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) + mkdir_chown_part = "" + + script_dump_part = textwrap.dedent( + """\ + WAS_ERR=0 + set +e + if [[ ! {if_exists_part} ]]; then + {mkdir_chown_part} + {exec_before_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do + su - postgres -c "{dump_prefix_cmd} pg_dump --create {postgresql_clean} {pg_dump_args} {pg_dump_format_part} --verbose {source}" 2> >({pg_dump_filter}) {pg_dump_line_pipe_part} + if [[ $? -ne 0 ]]; then + WAS_ERR=1 + echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" + else + echo "NOTICE: Dump succeeded, attempt $DUMP_ATTEMPT of {dump_attempts}" + break + fi + done + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + {exec_after_dump} + if [[ $? -ne 0 ]]; then WAS_ERR=1; fi + else + echo "NOTICE: Valid dump already exists, skipping" + fi + set -e + if [[ $WAS_ERR -ne 0 ]]; then false; else true; fi + """ + ).format( + postgresql_dump_dir=item["postgresql_dump_dir"], + postgresql_clean="" if item["postgresql_noclean"] else "--clean --if-exists", + dump_prefix_cmd=item["dump_prefix_cmd"], + exec_before_dump=item["exec_before_dump"], + exec_after_dump=item["exec_after_dump"], + pg_dump_args=item["pg_dump_args"], + grep_db_filter=grep_db_filter, + source=item["source"], + pg_dump_filter=item["pg_dump_filter"], + dump_attempts=item["dump_attempts"], + pg_dump_line_pipe_part=pg_dump_line_pipe_part, + pg_dump_format_part=pg_dump_format_part, + if_exists_part=if_exists_part, + mkdir_chown_part=mkdir_chown_part + ) if "postgresql_dump_type" in item and item["postgresql_dump_type"] == "directory": - pg_dump_line_pipe_part = "" - pg_dump_format_part = "--format=directory --file={postgresql_dump_dir}/{source}".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) - if_exists_part = "-d {postgresql_dump_dir}/{source}".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) - mkdir_chown_part = "mkdir -p {postgresql_dump_dir}/{source} && chown postgres:postgres {postgresql_dump_dir}/{source}".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) + # Name mask is not needed for directory dumps, as directories are used as db dump folders only + find_part = '-type d' + # With directory format pg_dump writes by itself to the directory, chown needed for pg_dump to be able to traverse to dump directory + chown_part = "chown postgres:postgres {postgresql_dump_dir}".format(postgresql_dump_dir=item["postgresql_dump_dir"]) else: - pg_dump_line_pipe_part = "| gzip > {postgresql_dump_dir}/{source}.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) - pg_dump_format_part = "--format=plain" - if_exists_part = "-f {postgresql_dump_dir}/{source}.gz".format(postgresql_dump_dir=item["postgresql_dump_dir"], source=item["source"]) - mkdir_chown_part = "" + find_part = '-type f -name "*.gz"' + chown_part = "" - script_dump_part = textwrap.dedent( + # If hourly retains are used keep dumps only for 59 minutes + script = textwrap.dedent( """\ - WAS_ERR=0 - set +e - if [[ ! {if_exists_part} ]]; then - {mkdir_chown_part} - {exec_before_dump} - if [[ $? -ne 0 ]]; then WAS_ERR=1; fi - for DUMP_ATTEMPT in $(seq 1 {dump_attempts}); do - {pg_run} "{dump_prefix_cmd} pg_dump {pg_user} --create {postgresql_clean} {pg_dump_args} {pg_dump_format_part} --verbose {source}" 2> >({pg_dump_filter}) {pg_dump_line_pipe_part} - if [[ $? -ne 0 ]]; then - WAS_ERR=1 - echo "ERROR: Dump failed, attempt $DUMP_ATTEMPT of {dump_attempts}" - else - echo "NOTICE: Dump succeeded, attempt $DUMP_ATTEMPT of {dump_attempts}" - break - fi - done - if [[ $? -ne 0 ]]; then WAS_ERR=1; fi - {exec_after_dump} - if [[ $? -ne 0 ]]; then WAS_ERR=1; fi - else - echo "NOTICE: Valid dump already exists, skipping" - fi + #!/bin/bash set -e - if [[ $WAS_ERR -ne 0 ]]; then false; else true; fi + + ssh {ssh_args} -p {port} {user}@{host} ' + set -x + set -e + set -o pipefail + mkdir -p {postgresql_dump_dir} + chmod 700 {postgresql_dump_dir} + {chown_part} + while [[ -d {postgresql_dump_dir}/dump.lock ]]; do + sleep 5 + done + mkdir {postgresql_dump_dir}/dump.lock + trap "rm -rf {postgresql_dump_dir}/dump.lock" 0 + cd {postgresql_dump_dir} + find {postgresql_dump_dir} {find_part} -mmin +{mmin} -exec rm -rf {{}} + + {exec_before_dump} + {comment_out_pg_dumpall}su - postgres -c "pg_dumpall --clean --if-exists --schema-only --verbose" 2> >({pg_dump_filter}) | gzip > {postgresql_dump_dir}/globals.gz + {exec_after_dump} + {script_dump_part} + ' """ ).format( + ssh_args=ssh_args, + port=item["connect_port"], + user=item["connect_user"], + host=item["connect_host"], postgresql_dump_dir=item["postgresql_dump_dir"], - postgresql_clean="" if item["postgresql_noclean"] else "--clean --if-exists", - dump_prefix_cmd=item["dump_prefix_cmd"], + mmin="59" if "retain_hourly" in item else "720", + script_dump_part=script_dump_part, + pg_dump_filter=item["pg_dump_filter"], exec_before_dump=item["exec_before_dump"], exec_after_dump=item["exec_after_dump"], - pg_dump_args=item["pg_dump_args"], - grep_db_filter=grep_db_filter, - source=item["source"], - pg_dump_filter=item["pg_dump_filter"], - dump_attempts=item["dump_attempts"], - pg_dump_line_pipe_part=pg_dump_line_pipe_part, - pg_dump_format_part=pg_dump_format_part, - if_exists_part=if_exists_part, - mkdir_chown_part=mkdir_chown_part, - pg_run="docker exec -u postgres {container} sh -lc".format(container=item["docker_container"]) if item["docker_mode"] else "su - postgres -c", - pg_user="-U {db_user}".format(db_user=item["db_user"]) if item["db_user"] else "" + find_part=find_part, + chown_part=chown_part, + comment_out_pg_dumpall="#" if item["postgresql_skip_globals"] else "" ) - if "postgresql_dump_type" in item and item["postgresql_dump_type"] == "directory": - # Name mask is not needed for directory dumps, as directories are used as db dump folders only - find_part = '-type d' - # With directory format pg_dump writes by itself to the directory, chown needed for pg_dump to be able to traverse to dump directory - chown_part = "chown postgres:postgres {postgresql_dump_dir}".format(postgresql_dump_dir=item["postgresql_dump_dir"]) - else: - find_part = '-type f -name "*.gz"' - chown_part = "" - - # If hourly retains are used keep dumps only for 59 minutes - script = textwrap.dedent( - """\ - #!/bin/bash - set -e - - ssh {ssh_args} -p {port} {user}@{host} ' - set -x - set -e - set -o pipefail - mkdir -p {postgresql_dump_dir} - chmod 700 {postgresql_dump_dir} - {chown_part} - while [[ -d {postgresql_dump_dir}/dump.lock ]]; do - sleep 5 - done - mkdir {postgresql_dump_dir}/dump.lock - trap "rm -rf {postgresql_dump_dir}/dump.lock" 0 - cd {postgresql_dump_dir} - find {postgresql_dump_dir} {find_part} -mmin +{mmin} -exec rm -rf {{}} + - {exec_before_dump} - {comment_out_pg_dumpall}{pg_run} "pg_dumpall {pg_user} --clean --if-exists --schema-only --verbose" 2> >({pg_dump_filter}) | gzip > {postgresql_dump_dir}/globals.gz - {exec_after_dump} - {script_dump_part} - ' - """ - ).format( - ssh_args=ssh_args, - port=item["connect_port"], - user=item["connect_user"], - host=item["connect_host"], - postgresql_dump_dir=item["postgresql_dump_dir"], - mmin="59" if "retain_hourly" in item else "720", - script_dump_part=script_dump_part, - pg_dump_filter=item["pg_dump_filter"], - exec_before_dump=item["exec_before_dump"], - exec_after_dump=item["exec_after_dump"], - find_part=find_part, - chown_part=chown_part, - comment_out_pg_dumpall="#" if item["postgresql_skip_globals"] else "", - pg_run="docker exec -u postgres {container} sh -lc".format(container=item["docker_container"]) if item["docker_mode"] else "su - postgres -c", - pg_user="-U {db_user}".format(db_user=item["db_user"]) if item["db_user"] else "" - ) - if item["type"] == "MONGODB_SSH": if item["source"] == "ALL":