Skip to content

Commit af610b3

Browse files
author
Jason Mobarak
committed
Pull ccache from S3
1 parent 2fe5fd2 commit af610b3

File tree

9 files changed

+177
-11
lines changed

9 files changed

+177
-11
lines changed

‎.travis.yml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ matrix:
1818
include:
1919
- env:
2020
- VARIANT=obfuscator
21-
- ARCH=x86,arm
21+
- ARCH=arm,x86
2222
before_install:
2323
- pip install --user --upgrade awscli
2424
- sudo apt-get update
@@ -33,12 +33,13 @@ matrix:
3333

3434
- env:
3535
- VARIANT=vanilla
36-
- ARCH=x86,arm
36+
- ARCH=arm,x86
3737
before_install:
3838
- pip install --user --upgrade awscli
3939
- sudo apt-get update
4040
- sudo apt-get install -y jq
4141
script:
42+
- ./pull_ccache.bash --variant=$VARIANT --arch=$ARCH
4243
- VARIANT=$VARIANT ./.travis.sh
4344
after_success:
4445
- travis_wait tar -C output -cJf "llvm-$VARIANT-$ARCH.txz" .

‎Makefile‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ SCRIPTS := .travis.sh \
66
run_build_shell.bash \
77
stage_sysroot.bash \
88
do_clang_build.bash \
9+
push_ccache.bash \
10+
pull_ccache.bash \
911

1012
all: check base build
1113

@@ -39,14 +41,24 @@ build-example: check-build_example
3941
run: check-run_build_shell
4042
$(CURDIR)/run_build_shell.bash $(NO_TTY_ARG)
4143

42-
clean-vanilla:
44+
clean-vanilla-build:
4345
docker volume rm vanilla-llvm-build || :
46+
docker volume rm vanilla-llvm-ccache || :
47+
48+
clean-vanilla-src:
4449
docker volume rm vanilla-llvm || :
4550

46-
clean-obfuscator:
51+
clean-vanilla: clean-vanilla-build clean-vanilla-src
52+
53+
clean-obfuscator-build:
4754
docker volume rm obfuscator-llvm-build || :
55+
docker volume rm obfuscator-llvm-ccache || :
56+
57+
clean-obfuscator-src:
4858
docker volume rm obfuscator-llvm || :
4959

60+
clean-obfuscator: clean-obfuscator-build clean-obfuscator-src
61+
5062
clean-build: clean-vanilla clean-obfuscator
5163

5264
clean: clean-build

‎build.bash‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ IFS=$'\n\t'
1616
DOCKER_NAMETAG=$(cat docker_nametag)
1717

1818
mkdir -p output/opt
19-
mkdir -p output/ccache
2019

2120
VERBOSE=
2221
NO_TTY=
@@ -27,7 +26,7 @@ while [[ $# -gt 0 ]]; do
2726
--arch=x86) ARCH="X86"; shift ;;
2827
--arch=arm) ARCH="ARM"; shift ;;
2928
--arch=arm,x86) ARCH="ARM\\;X86"; shift ;;
30-
--arch=x86,arm) ARCH="X86\\;ARM"; shift ;;
29+
--arch=x86,arm) ARCH="ARM\\;X86"; shift ;;
3130

3231
--variant=vanilla) VARIANT="vanilla"; shift ;;
3332
--variant=obfuscator) VARIANT="obfuscator"; shift ;;
@@ -94,14 +93,12 @@ else
9493
INTERACTIVE=()
9594
fi
9695

97-
# -v $VARIANT-llvm-ccache:/work/ccache \
98-
9996
# shellcheck disable=SC2068
10097
docker run ${INTERACTIVE[@]:-} --rm \
10198
-v "$PWD/output/opt:/opt" \
10299
-v "$PWD/patches:/patches" \
103100
-v "$PWD:/this_dir" \
104-
-v "$PWD/output/ccache:/work/ccache" \
101+
-v $VARIANT-llvm-ccache:/work/ccache \
105102
-v $VARIANT-llvm:/work/$VARIANT-llvm \
106103
-v $VARIANT-llvm-build:/work/build \
107104
-e VARIANT=$VARIANT -e ARCH=$ARCH \

‎describe_repo.bash‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
#!/usr/bin/env bash
2+
git describe --tags --always --dirty

‎publish.sh‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ fi
2121
REPO="${PWD##*/}"
2222
BUCKET="${BUCKET:-llvm-obfuscator-arm}"
2323

24-
BUILD_VERSION="$(git describe --tags --dirty --always)"
24+
BUILD_VERSION="$(./describe_repo.bash)"
2525
BUILD_PATH="$REPO/$BUILD_VERSION"
2626
if [[ ! -z "$PRODUCT_VERSION" ]]; then
2727
BUILD_PATH="$BUILD_PATH/$PRODUCT_VERSION"
@@ -43,7 +43,6 @@ for file in "$@"; do
4343
OBJECT="s3://$BUCKET/$KEY"
4444
if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
4545
if [[ "$TRAVIS_BRANCH" == master || "$TRAVIS_TAG" == v* || "$TRAVIS_BRANCH" == v*-release ]]; then
46-
OBJECT="s3://$BUCKET/$KEY"
4746
aws s3 cp "$file" "$OBJECT"
4847
fi
4948
else

‎pull_ccache.bash‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright (C) 2018 Swift Navigation Inc.
4+
# Contact: Swift Navigation <dev@swiftnav.com>
5+
#
6+
# This source is subject to the license found in the file 'LICENSE' which must
7+
# be be distributed together with this source. All other rights reserved.
8+
#
9+
# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND,
10+
# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED
11+
# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE.
12+
13+
set -euo pipefail
14+
IFS=$'\n\t'
15+
16+
while [[ $# -gt 0 ]]; do
17+
case $1 in
18+
19+
--arch=x86) ARCH="X86"; shift ;;
20+
--arch=arm) ARCH="ARM"; shift ;;
21+
--arch=arm,x86) ARCH="ARM\\;X86"; shift ;;
22+
--arch=x86,arm) ARCH="ARM\\;X86"; shift ;;
23+
24+
--variant=vanilla) VARIANT="vanilla"; shift ;;
25+
--variant=obfuscator) VARIANT="obfuscator"; shift ;;
26+
27+
--verbose) VERBOSE="-v"; shift ;;
28+
--no-tty) NO_TTY=--no-tty; shift ;;
29+
30+
*) shift ;;
31+
esac
32+
done
33+
34+
if [[ -z "${ARCH:-}" ]]; then
35+
echo "Error: must specify --arch=<arm|x86|arm,x86|x86,arm>"
36+
exit 1
37+
fi
38+
39+
if [[ -z "${VARIANT:-}" ]]; then
40+
echo "Error: must a variant to --variant=<vanilla|obfuscator>"
41+
exit 1
42+
fi
43+
44+
BUILD_VERSION="$(./describe_repo.bash)"
45+
ARCH="$(echo ${ARCH} | sed 's@\\;@-@g')"
46+
47+
CCACHE_ARCHIVE="ccache-${VARIANT}-${ARCH}-${BUILD_VERSION}.tbz2"
48+
49+
./s3_download.bash ${CCACHE_ARCHIVE}
50+
51+
docker run ${INTERACTIVE[@]:-} --rm \
52+
-v "$PWD:/this_dir" \
53+
-v $VARIANT-llvm-ccache:/work/ccache \
54+
"$DOCKER_NAMETAG-$VARIANT" \
55+
/bin/bash -c "tar -xjf ${CCACHE_ARCHIVE} -C /work/ccache ."

‎push_ccache.bash‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright (C) 2018 Swift Navigation Inc.
4+
# Contact: Swift Navigation <dev@swiftnav.com>
5+
#
6+
# This source is subject to the license found in the file 'LICENSE' which must
7+
# be be distributed together with this source. All other rights reserved.
8+
#
9+
# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND,
10+
# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED
11+
# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE.
12+
13+
set -euo pipefail
14+
IFS=$'\n\t'
15+
16+
[[ -z "${DEBUG:-}" ]] || set -x
17+
18+
DOCKER_NAMETAG=$(cat docker_nametag)
19+
20+
while [[ $# -gt 0 ]]; do
21+
case $1 in
22+
23+
--arch=x86) ARCH="X86"; shift ;;
24+
--arch=arm) ARCH="ARM"; shift ;;
25+
--arch=arm,x86) ARCH="ARM\\;X86"; shift ;;
26+
--arch=x86,arm) ARCH="ARM\\;X86"; shift ;;
27+
28+
--variant=vanilla) VARIANT="vanilla"; shift ;;
29+
--variant=obfuscator) VARIANT="obfuscator"; shift ;;
30+
31+
--verbose) VERBOSE="-v"; shift ;;
32+
--no-tty) NO_TTY=--no-tty; shift ;;
33+
34+
*) shift ;;
35+
esac
36+
done
37+
38+
if [[ -z "${ARCH:-}" ]]; then
39+
echo "Error: must specify --arch=<arm|x86|arm,x86|x86,arm>"
40+
exit 1
41+
fi
42+
43+
if [[ -z "${VARIANT:-}" ]]; then
44+
echo "Error: must a variant to build --variant=<vanilla|obfuscator>"
45+
exit 1
46+
fi
47+
48+
BUILD_VERSION="$(./describe_repo.bash)"
49+
ARCH="$(echo ${ARCH} | sed 's@\\;@-@g')"
50+
51+
CCACHE_ARCHIVE="ccache-${VARIANT}-${ARCH}-${BUILD_VERSION}.tbz2"
52+
53+
docker run -it --rm \
54+
-v "$PWD:/this_dir" \
55+
-v $VARIANT-llvm-ccache:/work/ccache \
56+
"$DOCKER_NAMETAG-$VARIANT" \
57+
/bin/bash -c "tar -cjf ${CCACHE_ARCHIVE} -C /work/ccache ."
58+
59+
./publish.sh "${CCACHE_ARCHIVE}"

‎run_build_shell.bash‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,10 @@ docker run -i -t --rm \
3737
-v "$PWD/bin:/wrapper-bin" \
3838
-v "$PWD/patches:/patches" \
3939
-v "$PWD:/this_dir" \
40+
-v $VARIANT-llvm-ccache:/work/ccache \
4041
-v $VARIANT-llvm:/work/$VARIANT-llvm \
4142
-v $VARIANT-llvm-build:/work/build \
43+
-e CCACHE_DIR=/work/ccache \
4244
"$DOCKER_NAMETAG-$VARIANT" \
4345
/bin/bash -c "export PATH=/opt/llvm-$VARIANT/bin:/opt/llvm-$VARIANT/wrappers/bin:\$PATH; \
4446
cp -v /this_dir/cpp_wrapper.c /work/cpp_wrapper.c \

‎s3_download.bash‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
3+
# Copyright (C) 2016 Swift Navigation Inc.
4+
# Contact: Fergus Noble <fergus@swiftnav.com>
5+
#
6+
# This source is subject to the license found in the file 'LICENSE' which must
7+
# be be distributed together with this source. All other rights reserved.
8+
#
9+
# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND,
10+
# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED
11+
# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE.
12+
#
13+
# Script for downloading firmware and NAP binaries from S3 to be incorporated
14+
# into the Linux image.
15+
16+
set -xe
17+
18+
REPO="${PWD##*/}"
19+
BUCKET="${BUCKET:-llvm-obfuscator-arm}"
20+
21+
BUILD_VERSION="$(describe_repo.bash)"
22+
BUILD_PATH="$REPO/$BUILD_VERSION"
23+
if [[ ! -z "$PRODUCT_VERSION" ]]; then
24+
BUILD_PATH="$BUILD_PATH/$PRODUCT_VERSION"
25+
fi
26+
if [[ ! -z "$PRODUCT_REV" ]]; then
27+
BUILD_PATH="$BUILD_PATH/$PRODUCT_REV"
28+
fi
29+
if [[ ! -z "$PRODUCT_TYPE" ]]; then
30+
BUILD_PATH="$BUILD_PATH/$PRODUCT_TYPE"
31+
fi
32+
33+
echo "Downloading $* to $BUILD_PATH"
34+
35+
for file in "$@"; do
36+
KEY="$BUILD_PATH/$(basename "$file")"
37+
OBJECT="s3://$BUCKET/$KEY"
38+
aws s3 cp "$OBJECT" "$file"
39+
done

0 commit comments

Comments
 (0)