From 97bff5561448c9b22c8b8257c4206d851bf99ad4 Mon Sep 17 00:00:00 2001 From: Lutz Bender Date: Tue, 27 Aug 2024 13:03:43 +0200 Subject: [PATCH 1/9] improve checksum calculation --- runs/backup.sh | 51 +++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/runs/backup.sh b/runs/backup.sh index cf6b94f9fd..5964a370b1 100755 --- a/runs/backup.sh +++ b/runs/backup.sh @@ -4,6 +4,8 @@ OPENWBDIRNAME=${OPENWBBASEDIR##*/} OPENWBDIRNAME=${OPENWBDIRNAME:-/} TARBASEDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) BACKUPDIR="$OPENWBBASEDIR/data/backup" +RAMDISKDIR="$OPENWBBASEDIR/ramdisk" +TEMPDIR="$RAMDISKDIR/temp" LOGFILE="$OPENWBBASEDIR/data/log/backup.log" useExtendedFilename=$1 @@ -51,42 +53,41 @@ fi --directory="/var/lib/" \ "mosquitto/" "mosquitto_local/" echo "adding git information" - git branch --no-color --show-current >"$OPENWBBASEDIR/ramdisk/GIT_BRANCH" - git log --pretty='format:%H' -n1 >"$OPENWBBASEDIR/ramdisk/GIT_HASH" - echo "branch: $(<"$OPENWBBASEDIR/ramdisk/GIT_BRANCH") commit-hash: $(<"$OPENWBBASEDIR/ramdisk/GIT_HASH")" + git branch --no-color --show-current >"$RAMDISKDIR/GIT_BRANCH" + git log --pretty='format:%H' -n1 >"$RAMDISKDIR/GIT_HASH" + echo "branch: $(<"$RAMDISKDIR/GIT_BRANCH") commit-hash: $(<"$RAMDISKDIR/GIT_HASH")" tar --verbose --append \ --file="$BACKUPFILE" \ - --directory="$OPENWBBASEDIR/ramdisk/" \ + --directory="$RAMDISKDIR/" \ "GIT_BRANCH" "GIT_HASH" echo "calculating checksums" - # openwb directory - find "$OPENWBBASEDIR" \( \ - -path "$OPENWBBASEDIR/.git" -o \ - -path "$OPENWBBASEDIR/data/backup" -o \ - -path "$OPENWBBASEDIR/.pytest" -o \ - -name "__pycache__" -o \ - -path "$OPENWBBASEDIR/.pytest_cache" -o \ - -path "$OPENWBBASEDIR/ramdisk" -o \ - -name "backup.log" \ - \) -prune -o \ - -type f -print0 | xargs -0 sha256sum | sed -n "s|$TARBASEDIR/||p" >"$OPENWBBASEDIR/ramdisk/SHA256SUM" - # configuration file - echo -n "/home/openwb/configuration.json" | xargs -0 sha256sum | sed -n "s|/home/openwb/||p" >>"$OPENWBBASEDIR/ramdisk/SHA256SUM" - # git info files - find "$OPENWBBASEDIR/ramdisk/GIT_"* \ - -type f -print0 | xargs -0 sha256sum | sed -n "s|$OPENWBBASEDIR/ramdisk/||p" >>"$OPENWBBASEDIR/ramdisk/SHA256SUM" - # mosquitto databases - find "/var/lib/mosquitto"* \ - -type f -print0 | xargs -0 sudo sha256sum | sed -n "s|/var/lib/||p" >>"$OPENWBBASEDIR/ramdisk/SHA256SUM" + IFS=$'\n' + mapfile -t file_list < <(tar -tf "$BACKUPFILE") + mkdir -p "$TEMPDIR" + # process each file + for file in "${file_list[@]}"; do + # skip directories + if [[ $file =~ /$ ]]; then + echo "skipping directory $file" + continue + fi + # extract the file + tar -xf "$BACKUPFILE" -C "$TEMPDIR" "$file" + # calculate the checksum + sha256sum "$TEMPDIR/$file" | sed -n "s|$TEMPDIR/||p" >> "$RAMDISKDIR/SHA256SUM" + # remove the file + rm -f "$TEMPDIR/$file" + done tar --verbose --append \ --file="$BACKUPFILE" \ - --directory="$OPENWBBASEDIR/ramdisk/" \ + --directory="$RAMDISKDIR/" \ "SHA256SUM" # cleanup echo "removing temporary files" - rm -v "$OPENWBBASEDIR/ramdisk/GIT_BRANCH" "$OPENWBBASEDIR/ramdisk/GIT_HASH" "$OPENWBBASEDIR/ramdisk/SHA256SUM" + rm -v "$RAMDISKDIR/GIT_BRANCH" "$RAMDISKDIR/GIT_HASH" "$RAMDISKDIR/SHA256SUM" + rm -rf "${TEMPDIR:?}/" tar --append \ --file="$BACKUPFILE" \ --directory="$OPENWBBASEDIR/data/log/" \ From a713ab6d49683d02039079ff6e945615ac0a6950 Mon Sep 17 00:00:00 2001 From: Lutz Bender Date: Tue, 27 Aug 2024 13:05:26 +0200 Subject: [PATCH 2/9] only include relevant files --- runs/backup.sh | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/runs/backup.sh b/runs/backup.sh index 5964a370b1..079415d7fe 100755 --- a/runs/backup.sh +++ b/runs/backup.sh @@ -35,13 +35,10 @@ fi tar --verbose --create \ --file="$BACKUPFILE" \ --directory="$TARBASEDIR/" \ - --exclude="$OPENWBDIRNAME/data/backup/*.tar" \ - --exclude="$OPENWBDIRNAME/data/log/backup.log" \ - --exclude="$OPENWBDIRNAME/.git" \ - --exclude "$OPENWBDIRNAME/ramdisk" \ - --exclude "__pycache__" \ - --exclude "$OPENWBDIRNAME/.pytest_cache" \ - "$OPENWBDIRNAME" + "$OPENWBDIRNAME/data/charge_log" \ + "$OPENWBDIRNAME/data/daily_log" \ + "$OPENWBDIRNAME/data/monthly_log" \ + "$OPENWBDIRNAME/data/log/uuid" echo "adding configuration file" sudo tar --verbose --append \ --file="$BACKUPFILE" \ @@ -52,6 +49,7 @@ fi --file="$BACKUPFILE" \ --directory="/var/lib/" \ "mosquitto/" "mosquitto_local/" + # ToDo: add mosquitto configuration files echo "adding git information" git branch --no-color --show-current >"$RAMDISKDIR/GIT_BRANCH" git log --pretty='format:%H' -n1 >"$RAMDISKDIR/GIT_HASH" From ddbf88ec7088b64661f0e460d807e9f929f8f125 Mon Sep 17 00:00:00 2001 From: Lutz Bender Date: Tue, 1 Oct 2024 09:03:55 +0200 Subject: [PATCH 3/9] extended log messages --- runs/backup.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/runs/backup.sh b/runs/backup.sh index 079415d7fe..ffe0a99f95 100755 --- a/runs/backup.sh +++ b/runs/backup.sh @@ -6,7 +6,8 @@ TARBASEDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) BACKUPDIR="$OPENWBBASEDIR/data/backup" RAMDISKDIR="$OPENWBBASEDIR/ramdisk" TEMPDIR="$RAMDISKDIR/temp" -LOGFILE="$OPENWBBASEDIR/data/log/backup.log" +LOGDIR="$OPENWBBASEDIR/data/log" +LOGFILE="$LOGDIR/backup.log" useExtendedFilename=$1 if ((useExtendedFilename == 1)); then @@ -18,6 +19,18 @@ else fi { + echo "starting backup script" + echo "environment:" + echo " OPENWBBASEDIR: $OPENWBBASEDIR" + echo " OPENWBDIRNAME: $OPENWBDIRNAME" + echo " TARBASEDIR: $TARBASEDIR" + echo " BACKUPDIR: $BACKUPDIR" + echo " RAMDISKDIR: $RAMDISKDIR" + echo " TEMPDIR: $TEMPDIR" + echo " LOGDIR: $LOGDIR" + echo " LOGFILE: $LOGFILE" + echo " FILENAME: $FILENAME" + echo "deleting old backup files if present in '$BACKUPDIR'" # remove old backup files rm -v "$BACKUPDIR/"* @@ -88,7 +101,7 @@ fi rm -rf "${TEMPDIR:?}/" tar --append \ --file="$BACKUPFILE" \ - --directory="$OPENWBBASEDIR/data/log/" \ + --directory="$LOGDIR/" \ "backup.log" echo "zipping archive" gzip --verbose "$BACKUPFILE" From be7c2e2180904dca84d6436073ed4d3492bcb5cf Mon Sep 17 00:00:00 2001 From: Lutz Bender Date: Wed, 2 Oct 2024 08:02:07 +0200 Subject: [PATCH 4/9] fix setting file permissions --- runs/atreboot.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runs/atreboot.sh b/runs/atreboot.sh index 0d33db394b..7b57b33b37 100755 --- a/runs/atreboot.sh +++ b/runs/atreboot.sh @@ -398,8 +398,8 @@ chmod 666 "$LOGFILE" fi # set restore dir permissions to allow file upload for apache - sudo chgrp www-data "${OPENWBBASEDIR}/data/restore" "${OPENWBBASEDIR}/data/restore/"* "${OPENWBBASEDIR}/data/data_migration" "${OPENWBBASEDIR}/data/data_migration/"* - sudo chmod g+w "${OPENWBBASEDIR}/data/restore" "${OPENWBBASEDIR}/data/restore/"* "${OPENWBBASEDIR}/data/data_migration" "${OPENWBBASEDIR}/data/data_migration/"* + sudo chgrp -R www-data "${OPENWBBASEDIR}/data/restore/." "${OPENWBBASEDIR}/data/data_migration/." + sudo chmod -R g+w "${OPENWBBASEDIR}/data/restore/." "${OPENWBBASEDIR}/data/data_migration/." # cleanup some folders folder="${OPENWBBASEDIR}/data/data_migration/var" From 471760a5d64d72a2b7c8a83e1056dbcbc9fd48fc Mon Sep 17 00:00:00 2001 From: Lutz Bender Date: Fri, 20 Dec 2024 09:55:34 +0100 Subject: [PATCH 5/9] add mosquitto configuration --- runs/backup.sh | 6 +++++- runs/restore.sh | 21 ++++++++++++++++----- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/runs/backup.sh b/runs/backup.sh index ffe0a99f95..c7f36cf824 100755 --- a/runs/backup.sh +++ b/runs/backup.sh @@ -62,7 +62,11 @@ fi --file="$BACKUPFILE" \ --directory="/var/lib/" \ "mosquitto/" "mosquitto_local/" - # ToDo: add mosquitto configuration files + echo "adding mosquitto configuration" + sudo tar --verbose --append \ + --file="$BACKUPFILE" \ + --directory="/etc/mosquitto/" \ + "conf_local.d/" echo "adding git information" git branch --no-color --show-current >"$RAMDISKDIR/GIT_BRANCH" git log --pretty='format:%H' -n1 >"$RAMDISKDIR/GIT_HASH" diff --git a/runs/restore.sh b/runs/restore.sh index d030f94055..4db1fcd4e1 100755 --- a/runs/restore.sh +++ b/runs/restore.sh @@ -3,8 +3,9 @@ OPENWB_BASE_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) RESTORE_DIR="$OPENWB_BASE_DIR/data/restore" SOURCE_FILE="$RESTORE_DIR/restore.tar.gz" WORKING_DIR="/home/openwb/openwb_restore" -MOSQUITTO_DIR="/var/lib/mosquitto" -MOSQUITTO_LOCAL_DIR="/var/lib/mosquitto_local" +MOSQUITTO_DB_DIR="/var/lib/mosquitto" +MOSQUITTO_LOCAL_DB_DIR="/var/lib/mosquitto_local" +MOSQUITTO_CONF_DIR="/etc/mosquitto" LOG_FILE="$OPENWB_BASE_DIR/data/log/restore.log" { @@ -54,18 +55,28 @@ LOG_FILE="$OPENWB_BASE_DIR/data/log/restore.log" echo "Backup does not contain configuration. Skipping restore." fi echo "****************************************" - echo "Step 5: restore mosquitto db" + echo "Step 5.1: restore mosquitto db" if [[ -f "${WORKING_DIR}/mosquitto/mosquitto.db" ]]; then - sudo mv -v -f "${WORKING_DIR}/mosquitto/mosquitto.db" "$MOSQUITTO_DIR/mosquitto.db" + sudo mv -v -f "${WORKING_DIR}/mosquitto/mosquitto.db" "$MOSQUITTO_DB_DIR/mosquitto.db" else echo "Backup does not contain mosquitto.db. Skipping restore." fi if [[ -f "${WORKING_DIR}/mosquitto_local/mosquitto.db" ]]; then - sudo mv -v -f "${WORKING_DIR}/mosquitto_local/mosquitto.db" "$MOSQUITTO_LOCAL_DIR/mosquitto.db" + sudo mv -v -f "${WORKING_DIR}/mosquitto_local/mosquitto.db" "$MOSQUITTO_LOCAL_DB_DIR/mosquitto.db" else echo "Backup does not contain local mosquitto.db. Skipping restore." fi echo "****************************************" + echo "Step 5.2: restore mosquitto configuration" + if [[ -d "${WORKING_DIR}/conf_local.d" ]]; then + # remove old configuration + sudo rm -v -r "$MOSQUITTO_CONF_DIR/conf_local.d" + # copy configuration + sudo cp -v -p -r "${WORKING_DIR}/conf_local.d" "$MOSQUITTO_CONF_DIR/" + else + echo "Backup does not contain mosquitto configuration. Skipping restore." + fi + echo "****************************************" echo "Step 6: cleanup after restore" sudo rm "$SOURCE_FILE" sudo rm -R "$WORKING_DIR" From 650438537cc0dddf21c3f85035c950c72b73ea3d Mon Sep 17 00:00:00 2001 From: Lutz Bender Date: Fri, 20 Dec 2024 10:47:56 +0100 Subject: [PATCH 6/9] fix directory ownership --- runs/prepare_restore.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/runs/prepare_restore.sh b/runs/prepare_restore.sh index 86f8c2513c..f12edb2df2 100755 --- a/runs/prepare_restore.sh +++ b/runs/prepare_restore.sh @@ -15,11 +15,13 @@ declare resultStatus mkdir -p "$WORKING_DIR" echo "****************************************" echo "Step 2: extract archive to working directory" - # extracting as root preserves file owner/group and permissions! + # extracting as root preserves file owner/group and permissions, but directory owner/group is root! if ! sudo tar --verbose --extract --file="$SOURCE_FILE" --directory="$WORKING_DIR"; then resultMessage="Beim Entpacken des Archivs ist ein Fehler aufgetreten!" resultStatus=1 else + echo "fixing directory owner/group" + sudo chown --verbose openwb:openwb "$WORKING_DIR/openWB" "$WORKING_DIR/openWB/data" "$WORKING_DIR/openWB/data/log" echo "****************************************" echo "Step 3: validating extracted files" if [[ ! -f "$WORKING_DIR/SHA256SUM" ]] || From 51bf4b148bdd54df95787e89726efe41214097a8 Mon Sep 17 00:00:00 2001 From: Lutz Bender Date: Fri, 20 Dec 2024 10:51:10 +0100 Subject: [PATCH 7/9] add info message in restore preparation --- runs/prepare_restore.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runs/prepare_restore.sh b/runs/prepare_restore.sh index f12edb2df2..d3e2c73689 100755 --- a/runs/prepare_restore.sh +++ b/runs/prepare_restore.sh @@ -36,6 +36,9 @@ declare resultStatus if [[ ! -f "$WORKING_DIR/configuration.json" ]]; then echo "configuration missing; continue anyway" fi + if [[ ! -d "$WORKING_DIR/conf_local.d" ]]; then + echo "mosquitto configuration missing; continue anyway" + fi if ! (cd "$WORKING_DIR" && sudo sha256sum --quiet --check "SHA256SUM"); then resultMessage="Einige Dateien wurden gelöscht oder bearbeitet!" resultStatus=1 From 0f145fd08437a4203eb13a7bcbc2631ffcfeaa5f Mon Sep 17 00:00:00 2001 From: Lutz Bender Date: Thu, 2 Jan 2025 09:29:37 +0100 Subject: [PATCH 8/9] add boot config --- runs/backup.sh | 4 ++++ runs/prepare_restore.sh | 3 +++ runs/restore.sh | 9 ++++++++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/runs/backup.sh b/runs/backup.sh index c7f36cf824..db825e3379 100755 --- a/runs/backup.sh +++ b/runs/backup.sh @@ -67,6 +67,10 @@ fi --file="$BACKUPFILE" \ --directory="/etc/mosquitto/" \ "conf_local.d/" + echo "adding boot file" + sudo tar --verbose --append \ + --file="$BACKUPFILE" \ + "/boot/config.txt" echo "adding git information" git branch --no-color --show-current >"$RAMDISKDIR/GIT_BRANCH" git log --pretty='format:%H' -n1 >"$RAMDISKDIR/GIT_HASH" diff --git a/runs/prepare_restore.sh b/runs/prepare_restore.sh index d3e2c73689..ab89ecfe43 100755 --- a/runs/prepare_restore.sh +++ b/runs/prepare_restore.sh @@ -39,6 +39,9 @@ declare resultStatus if [[ ! -d "$WORKING_DIR/conf_local.d" ]]; then echo "mosquitto configuration missing; continue anyway" fi + if [[ ! -d "$WORKING_DIR/boot" ]]; then + echo "boot file missing; continue anyway" + fi if ! (cd "$WORKING_DIR" && sudo sha256sum --quiet --check "SHA256SUM"); then resultMessage="Einige Dateien wurden gelöscht oder bearbeitet!" resultStatus=1 diff --git a/runs/restore.sh b/runs/restore.sh index 4db1fcd4e1..20a36c07b9 100755 --- a/runs/restore.sh +++ b/runs/restore.sh @@ -77,7 +77,14 @@ LOG_FILE="$OPENWB_BASE_DIR/data/log/restore.log" echo "Backup does not contain mosquitto configuration. Skipping restore." fi echo "****************************************" - echo "Step 6: cleanup after restore" + echo "Step 6: restore boot file" + if [[ -f "${WORKING_DIR}/boot/config.txt" ]]; then + sudo mv -v -f "${WORKING_DIR}/boot/config.txt" "/boot/" + else + echo "Backup does not contain boot file. Skipping restore." + fi + echo "****************************************" + echo "Step 7: cleanup after restore" sudo rm "$SOURCE_FILE" sudo rm -R "$WORKING_DIR" echo "****************************************" From 46f8b8023cd728f5f0e007a0635dc6cce1931a3e Mon Sep 17 00:00:00 2001 From: Lutz Bender Date: Thu, 2 Jan 2025 13:20:48 +0100 Subject: [PATCH 9/9] keep cloud configuration on factory reset --- packages/helpermodules/subdata.py | 8 +++++--- runs/factory_reset.sh | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/packages/helpermodules/subdata.py b/packages/helpermodules/subdata.py index f843a7ceb2..85d0543703 100644 --- a/packages/helpermodules/subdata.py +++ b/packages/helpermodules/subdata.py @@ -898,9 +898,11 @@ def process_system_topic(self, client: mqtt.Client, var: dict, msg: mqtt.MQTTMes self.set_json_payload(var["system"].data["backup_cloud"], msg) elif ("openWB/system/dataprotection_acknowledged" == msg.topic and decode_payload(msg.payload) is False): - Pub().pub("openWB/set/command/removeCloudBridge/todo", { - "command": "removeCloudBridge" - }) + if self.event_subdata_initialized.is_set(): + Pub().pub("openWB/set/command/removeCloudBridge/todo", + {"command": "removeCloudBridge"}) + else: + log.debug("skipping data protection message on startup") elif re.search("^.+/io/[0-9]+/config$", msg.topic) is not None: index = get_index(msg.topic) if decode_payload(msg.payload) == "": diff --git a/runs/factory_reset.sh b/runs/factory_reset.sh index 86475465d1..26de55c79c 100755 --- a/runs/factory_reset.sh +++ b/runs/factory_reset.sh @@ -8,12 +8,28 @@ fi case "$1" in "clearall") + echo "checking for configured cloud bridge..." + cloud_bridge=$(timeout 1 mosquitto_sub -t 'openWB/system/mqtt/bridge/+' | grep -E '"is_openwb_cloud": ?true') + if [ -n "$cloud_bridge" ]; then + echo "cloud bridge is configured" + valid_partner_ids=$(timeout 1 mosquitto_sub -t 'openWB/system/mqtt/valid_partner_ids' -C 1) + else + echo "no cloud bridge configured" + fi echo "deleting retained message store of external mosquitto..." timeout 3 mosquitto_sub -t '#' --remove-retained --retained-only echo "deleting retained message store of internal mosquitto..." timeout 3 mosquitto_sub -t '#' --remove-retained --retained-only -p 1886 echo "deleting log data" rm -r "$OPENWBBASEDIR/data/charge_log/"* "$OPENWBBASEDIR/data/daily_log/"* "$OPENWBBASEDIR/data/log/"*.log "$OPENWBBASEDIR/data/monthly_log/"* + echo "reset display rotation" + sudo sed -i "s/^lcd_rotate=[0-3]$/lcd_rotate=0/" "/boot/config.txt" + if [ -n "$cloud_bridge" ]; then + echo "restore cloud bridge configuration: $cloud_bridge" + mosquitto_pub -t 'openWB/command/max_id/mqtt_bridge' -r -m 1 -p 1886 + mosquitto_pub -t 'openWB/system/mqtt/valid_partner_ids' -r -m "$valid_partner_ids" -p 1886 + mosquitto_pub -t 'openWB/system/mqtt/bridge/0' -r -m "$cloud_bridge" -p 1886 + fi echo "all done";; *) echo "please pass \"clearall\" as parameter if you really want to reset all data stored in the internal and external broker"