From edf57b76f50c4b009e4f0e739439ec9b8e2c9648 Mon Sep 17 00:00:00 2001 From: danielsufrin Date: Tue, 13 Nov 2018 01:47:08 -0500 Subject: [PATCH 1/7] making soft and hard links work --- create-changelog-to-go-from-old-to-new.sh | 33 ++++++++++++++++++++--- expose-filesystem-root.sh | 3 +++ shell.Dockerfile | 5 ++-- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/create-changelog-to-go-from-old-to-new.sh b/create-changelog-to-go-from-old-to-new.sh index 6e5a891..471257a 100755 --- a/create-changelog-to-go-from-old-to-new.sh +++ b/create-changelog-to-go-from-old-to-new.sh @@ -2,13 +2,38 @@ # Use rsync to figure out what changes are necessary to go from old image contents to new echo "List of changes necessary to go from old image contents to new: " -rsync -a -x --human-readable --delete-after --checksum --dry-run --itemize-changes --exclude .docker-image-diff "$RESTRICT_DIFF_TO_PATH/" "rsync://rsync@old/root$RESTRICT_DIFF_TO_PATH/" | tee $OUT/changes.rsync.log + +echo "[root]" > /tmp/.rsyncd.conf; +echo "path = /" >> /tmp/.rsyncd.conf; +echo "read only = false" >> /tmp/.rsyncd.conf; +echo "uid=root" >> /tmp/.rsyncd.conf; +echo "gid=root" >> /tmp/.rsyncd.conf; + +echo rsync -a -H -x --human-readable --delete-after --checksum --dry-run --itemize-changes --exclude .docker-image-diff "$RESTRICT_DIFF_TO_PATH/" "rsync://rsync@old/root$RESTRICT_DIFF_TO_PATH/" | tee $OUT/changes.rsync.log + +rsync -a -H -x --human-readable --delete-after --checksum --dry-run --itemize-changes --exclude .docker-image-diff "$RESTRICT_DIFF_TO_PATH/" "rsync://rsync@old/root$RESTRICT_DIFF_TO_PATH/" | tee $OUT/changes.rsync.log # Add files to add to a tar archive -cat $OUT/changes.rsync.log | grep '^ $OUT/files-to-add.list -tar -cf $OUT/files-to-add.tar -T $OUT/files-to-add.list 2>&1 | grep -v "Removing leading" +echo "adding to tar " + +cat $OUT/changes.rsync.log | grep -E '^ $OUT/files-to-add_2.list + +cat $OUT/files-to-add_2.list | grep -oP '^\K.*?(?= ->| =>|$)' > $OUT/files-to-add.list + +pwd +ls -a +cd .. +pwd +ls -a + +tar -cf $OUT/files-to-add.tar -T $OUT/files-to-add.list 2>&1 +#| grep -v "Removing leading" +cd /.docker-image-diff/ +ls + # Add files to remove to a list +echo "List to delete " cat $OUT/changes.rsync.log | grep '^*deleting' | while read -a cols; do echo "$RESTRICT_DIFF_TO_PATH/"${cols[@]:1}; done > $OUT/files-to-remove.list # Informational output @@ -20,6 +45,6 @@ echo "Number of files to remove: " wc $OUT/files-to-remove.list echo echo "Changes not accounted for: " -cat $OUT/changes.rsync.log | grep -v '^ /tmp/.rsyncd.conf; echo "path = /" >> /tmp/.rsyncd.conf; echo "read only = false" >> /tmp/.rsyncd.conf; +echo "uid = 0" >> /tmp/.rsyncd.conf; +echo "gid = 0" >> /tmp/.rsyncd.conf; + rsync --daemon --port 873 --no-detach -vv --config /tmp/.rsyncd.conf diff --git a/shell.Dockerfile b/shell.Dockerfile index 6507555..1af932f 100644 --- a/shell.Dockerfile +++ b/shell.Dockerfile @@ -1,2 +1,3 @@ -FROM debian:jessie -RUN apt-get update && apt-get install gettext -y +FROM fedora:28 +RUN dnf update -y +RUN dnf install -y gettext-devel intltool From 500c7d3f0dd5398028f1b68f5db82f1602857c54 Mon Sep 17 00:00:00 2001 From: danielsufrin Date: Tue, 13 Nov 2018 12:04:44 -0500 Subject: [PATCH 2/7] making sure files-to-remove.list is deleted even if it is empty --- template.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template.Dockerfile b/template.Dockerfile index 801e2d8..57bcd03 100644 --- a/template.Dockerfile +++ b/template.Dockerfile @@ -1,4 +1,4 @@ FROM ${OLD_IMAGE} ADD files-to-add.tar / ADD files-to-remove.list /.files-to-remove.list -RUN if [ -s /.files-to-remove.list ]; then xargs -d '\n' -a /.files-to-remove.list rm && rm /.files-to-remove.list; fi +RUN if [ -s /.files-to-remove.list ]; then xargs -d '\n' -a /.files-to-remove.list rm; fi; rm /.files-to-remove.list; From 3fbd10f71b653fb5b923c69447399c2c9d8905cc Mon Sep 17 00:00:00 2001 From: danielsufrin Date: Tue, 13 Nov 2018 12:28:13 -0500 Subject: [PATCH 3/7] making minimal diff from develop --- create-changelog-to-go-from-old-to-new.sh | 28 ++--------------------- expose-filesystem-root.sh | 3 --- shrink_images.sh | 10 ++++++++ 3 files changed, 12 insertions(+), 29 deletions(-) create mode 100755 shrink_images.sh diff --git a/create-changelog-to-go-from-old-to-new.sh b/create-changelog-to-go-from-old-to-new.sh index 471257a..8b573a1 100755 --- a/create-changelog-to-go-from-old-to-new.sh +++ b/create-changelog-to-go-from-old-to-new.sh @@ -2,38 +2,14 @@ # Use rsync to figure out what changes are necessary to go from old image contents to new echo "List of changes necessary to go from old image contents to new: " - -echo "[root]" > /tmp/.rsyncd.conf; -echo "path = /" >> /tmp/.rsyncd.conf; -echo "read only = false" >> /tmp/.rsyncd.conf; -echo "uid=root" >> /tmp/.rsyncd.conf; -echo "gid=root" >> /tmp/.rsyncd.conf; - -echo rsync -a -H -x --human-readable --delete-after --checksum --dry-run --itemize-changes --exclude .docker-image-diff "$RESTRICT_DIFF_TO_PATH/" "rsync://rsync@old/root$RESTRICT_DIFF_TO_PATH/" | tee $OUT/changes.rsync.log - -rsync -a -H -x --human-readable --delete-after --checksum --dry-run --itemize-changes --exclude .docker-image-diff "$RESTRICT_DIFF_TO_PATH/" "rsync://rsync@old/root$RESTRICT_DIFF_TO_PATH/" | tee $OUT/changes.rsync.log +rsync -a -x --human-readable --delete-after --checksum --dry-run --itemize-changes --exclude .docker-image-diff "$RESTRICT_DIFF_TO_PATH/" "rsync://rsync@old/root$RESTRICT_DIFF_TO_PATH/" | tee $OUT/changes.rsync.log # Add files to add to a tar archive -echo "adding to tar " - cat $OUT/changes.rsync.log | grep -E '^ $OUT/files-to-add_2.list - cat $OUT/files-to-add_2.list | grep -oP '^\K.*?(?= ->| =>|$)' > $OUT/files-to-add.list - -pwd -ls -a -cd .. -pwd -ls -a - -tar -cf $OUT/files-to-add.tar -T $OUT/files-to-add.list 2>&1 -#| grep -v "Removing leading" -cd /.docker-image-diff/ -ls - +tar -cf $OUT/files-to-add.tar -T $OUT/files-to-add.list 2>&1 -C / # Add files to remove to a list -echo "List to delete " cat $OUT/changes.rsync.log | grep '^*deleting' | while read -a cols; do echo "$RESTRICT_DIFF_TO_PATH/"${cols[@]:1}; done > $OUT/files-to-remove.list # Informational output diff --git a/expose-filesystem-root.sh b/expose-filesystem-root.sh index 7bbcfc2..9c70bb7 100755 --- a/expose-filesystem-root.sh +++ b/expose-filesystem-root.sh @@ -3,7 +3,4 @@ echo "[root]" > /tmp/.rsyncd.conf; echo "path = /" >> /tmp/.rsyncd.conf; echo "read only = false" >> /tmp/.rsyncd.conf; -echo "uid = 0" >> /tmp/.rsyncd.conf; -echo "gid = 0" >> /tmp/.rsyncd.conf; - rsync --daemon --port 873 --no-detach -vv --config /tmp/.rsyncd.conf diff --git a/shrink_images.sh b/shrink_images.sh new file mode 100755 index 0000000..a16ac33 --- /dev/null +++ b/shrink_images.sh @@ -0,0 +1,10 @@ +cd docker-diff-based-layers +export RESTRICT_DIFF_TO_PATH= +export OLD_IMAGE=$1 +export NEW_IMAGE=$2 +docker-compose -f rsync-image-diff.docker-compose.yml up +docker-compose -f shell.docker-compose.yml -f process-image-diff.docker-compose.yml run --rm shell ./generate-dockerfile.sh + +cd output; docker build -t $3 .; cd .. + +cd .. \ No newline at end of file From fcf4e6c22c8a86275439775fc4e6c9c946629c2e Mon Sep 17 00:00:00 2001 From: danielsufrin Date: Tue, 13 Nov 2018 12:28:33 -0500 Subject: [PATCH 4/7] restoring shell.Dockerfile --- shell.Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/shell.Dockerfile b/shell.Dockerfile index 1af932f..6507555 100644 --- a/shell.Dockerfile +++ b/shell.Dockerfile @@ -1,3 +1,2 @@ -FROM fedora:28 -RUN dnf update -y -RUN dnf install -y gettext-devel intltool +FROM debian:jessie +RUN apt-get update && apt-get install gettext -y From 203d2d4c9a116d587decc4aa75c2cc036a163c98 Mon Sep 17 00:00:00 2001 From: danielsufrin Date: Tue, 13 Nov 2018 12:44:05 -0500 Subject: [PATCH 5/7] making sure hard links match beginning of line, improved comments --- create-changelog-to-go-from-old-to-new.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/create-changelog-to-go-from-old-to-new.sh b/create-changelog-to-go-from-old-to-new.sh index 8b573a1..57e6147 100755 --- a/create-changelog-to-go-from-old-to-new.sh +++ b/create-changelog-to-go-from-old-to-new.sh @@ -5,7 +5,9 @@ echo "List of changes necessary to go from old image contents to new: " rsync -a -x --human-readable --delete-after --checksum --dry-run --itemize-changes --exclude .docker-image-diff "$RESTRICT_DIFF_TO_PATH/" "rsync://rsync@old/root$RESTRICT_DIFF_TO_PATH/" | tee $OUT/changes.rsync.log # Add files to add to a tar archive -cat $OUT/changes.rsync.log | grep -E '^ $OUT/files-to-add_2.list +cat $OUT/changes.rsync.log | grep -E '^ $OUT/files-to-add_2.list + +# soft links (^cL in the grep above) have -> in the list which must be removed. Hard links ^hf have => cat $OUT/files-to-add_2.list | grep -oP '^\K.*?(?= ->| =>|$)' > $OUT/files-to-add.list tar -cf $OUT/files-to-add.tar -T $OUT/files-to-add.list 2>&1 -C / @@ -21,6 +23,6 @@ echo "Number of files to remove: " wc $OUT/files-to-remove.list echo echo "Changes not accounted for: " -cat $OUT/changes.rsync.log | grep -v -E '^ Date: Tue, 13 Nov 2018 12:47:08 -0500 Subject: [PATCH 6/7] cleaning the shrink_images.sh script --- shrink_images.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/shrink_images.sh b/shrink_images.sh index a16ac33..efcf2b8 100755 --- a/shrink_images.sh +++ b/shrink_images.sh @@ -1,10 +1,7 @@ -cd docker-diff-based-layers -export RESTRICT_DIFF_TO_PATH= +echo "Usage shrink_images.sh [RESTRICT_DIFF_TO_PATH]" +export RESTRICT_DIFF_TO_PATH=$4 export OLD_IMAGE=$1 export NEW_IMAGE=$2 docker-compose -f rsync-image-diff.docker-compose.yml up docker-compose -f shell.docker-compose.yml -f process-image-diff.docker-compose.yml run --rm shell ./generate-dockerfile.sh - cd output; docker build -t $3 .; cd .. - -cd .. \ No newline at end of file From 788cf73b4ef37a6e001d52c429f9aa3bed2e9970 Mon Sep 17 00:00:00 2001 From: danielsufrin Date: Tue, 13 Nov 2018 13:05:43 -0500 Subject: [PATCH 7/7] ignoring the rsync conf file --- create-changelog-to-go-from-old-to-new.sh | 4 ++-- expose-filesystem-root.sh | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/create-changelog-to-go-from-old-to-new.sh b/create-changelog-to-go-from-old-to-new.sh index 57e6147..3a90597 100755 --- a/create-changelog-to-go-from-old-to-new.sh +++ b/create-changelog-to-go-from-old-to-new.sh @@ -9,10 +9,10 @@ cat $OUT/changes.rsync.log | grep -E '^ in the list which must be removed. Hard links ^hf have => cat $OUT/files-to-add_2.list | grep -oP '^\K.*?(?= ->| =>|$)' > $OUT/files-to-add.list -tar -cf $OUT/files-to-add.tar -T $OUT/files-to-add.list 2>&1 -C / +tar -C / -cf $OUT/files-to-add.tar -T $OUT/files-to-add.list 2>&1 # Add files to remove to a list -cat $OUT/changes.rsync.log | grep '^*deleting' | while read -a cols; do echo "$RESTRICT_DIFF_TO_PATH/"${cols[@]:1}; done > $OUT/files-to-remove.list +cat $OUT/changes.rsync.log | grep '^*deleting' | grep -v 'tmp\/\.rsyncd\.conf' | while read -a cols; do echo "$RESTRICT_DIFF_TO_PATH/"${cols[@]:1}; done > $OUT/files-to-remove.list # Informational output echo diff --git a/expose-filesystem-root.sh b/expose-filesystem-root.sh index 9c70bb7..7bbcfc2 100755 --- a/expose-filesystem-root.sh +++ b/expose-filesystem-root.sh @@ -3,4 +3,7 @@ echo "[root]" > /tmp/.rsyncd.conf; echo "path = /" >> /tmp/.rsyncd.conf; echo "read only = false" >> /tmp/.rsyncd.conf; +echo "uid = 0" >> /tmp/.rsyncd.conf; +echo "gid = 0" >> /tmp/.rsyncd.conf; + rsync --daemon --port 873 --no-detach -vv --config /tmp/.rsyncd.conf