readies sync (fixes macOS build) (#265)

* readies sync

* installation of gnu utils in system-setup.py

* circleci: debug macOS

* circleci: debug macOS 2

* circleci: debug macOS 3

* circleci: debug macOS 4

* circleci: debug macOS 5

* Added double-panda test for macOS

* circleci: workaround for python ssl

* circleci: debug macOS 6

* circleci: debug macOS 7

* pack.sh fixes

* Test-related changes in Makefile

* Test-related changes in Makefile 1

* Improvements for tests in Makefile

* pack.sh fix

Author: rafie

.circleci/config.yml

@@ -16,7 +16,7 @@ commands:
           name: Install dependencies
           command: |
             ./opt/readies/bin/getpy
-            ./opt/system-setup.py
+            BREW_NO_UPDATE=1 ./opt/system-setup.py
             git clone git://github.com/antirez/redis.git --branch 5.0.7
             (cd redis && make malloc=libc -j $(nproc) && make install)
             redis-server --version
@@ -72,6 +72,10 @@ jobs:
     macos:
       xcode: 10.2.1
     steps:
+      - run:
+          name: Fix macOS Python installation
+          command: |
+            brew reinstall -f python2
       - ci_steps:
           platform: macosx
   build-multiarch-docker:
@@ -162,7 +166,6 @@ workflows:
       - build-macos:
           filters:
             branches:
-              # only: /.*/
               ignore: /.*/
             tags:
               only: /^v[0-9].*/
@@ -182,9 +185,6 @@ workflows:
           requires:
             - build
           filters:
-#             # uncomment to debug
-#            tags:
-#              only: /.*/
             branches:
               only: master
       - deploy_package:

opt/Makefile

@@ -12,8 +12,12 @@ make build    # compile and link
 make clean    # remove build artifacts
   ALL=1         # remove entire artifacts directory
 make test     # run tests
+  TEST=test       # run only test `test` with Redis output
+  TEST_ARGS=args  # add extra RLTest `args`
 make pack     # create installation packages
   PACK_DEPS=0   # do not pack dependencies
+  INTO=dir      # place artifacts in `dir`
+  BRANCH=name   # use `name` as branch name
 make deploy   # copy packages to S3
 make release  # release a version
 
@@ -123,9 +127,9 @@ fetch deps:
 
 pack:
 ifneq ($(PACK_DEPS),0)
-	$(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) DEPS=1 ./pack.sh
+	$(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) BRANCH=$(BRANCH) INTO=$(INTO) DEPS=1 ./pack.sh
 else
-	$(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) DEPS=0 ./pack.sh
+	$(SHOW)DEVICE=$(DEVICE) BINDIR=$(BINROOT) INSTALL_DIR=$(INSTALL_DIR) BRANCH=$(BRANCH) INTO=$(INTO) DEPS=0 ./pack.sh
 endif
 
 #----------------------------------------------------------------------------------------------
@@ -134,21 +138,37 @@ TEST_REPORT_DIR ?= $(PWD)
 
 ifeq ($(TEST),)
 TEST=basic_tests.py
+PYDEBUG=
+else
+TEST_ARGS += -s
+PYDEBUG=1
 endif
 
+TEST_PREFIX=set -e; cd $(ROOT)/test
+TEST_CMD=\
+	DEVICE=$(DEVICE) PYDEBUG=$(PYDEBUG) \
+	python3 -m RLTest $(TEST_ARGS) --test $(TEST) --module $(INSTALL_DIR)/redisai.so
+
+GEN ?= 1
+SLAVES ?= 1
+AOF ?= 1
+
 test:
 ifneq ($(NO_LFS),1)
 	$(SHOW)if [ "$(git lfs env > /dev/null 2>&1 ; echo $?)" != "0" ]; then cd $(ROOT); git lfs install; fi
 	$(SHOW)cd $(ROOT); git lfs pull
 endif
-	$(SHOW)set -e ;\
-	cd $(ROOT)/test ;\
-	export DEVICE=$(DEVICE) ;\
-	python3 -m RLTest $(TEST_ARGS) --test $(TEST) --module $(INSTALL_DIR)/redisai.so ;\
-	python3 -m RLTest $(TEST_ARGS) --test $(TEST) --module $(INSTALL_DIR)/redisai.so --use-slaves ;\
-	python3 -m RLTest $(TEST_ARGS) --test $(TEST) --module $(INSTALL_DIR)/redisai.so --use-aof 
-	# Commented until the MacOS CI is available
-	# python3 -m RLTest $(TEST_ARGS) --test double-panda.py --module $(INSTALL_DIR)/redisai.so
+ifeq ($(GEN),1)
+	$(SHOW)$(TEST_PREFIX); $(TEST_CMD)
+endif
+ifeq ($(AOF),1)
+	$(SHOW)$(TEST_PREFIX); printf "\nTests with --use-aof:\n\n" ;\
+	$(TEST_CMD) --use-aof
+endif
+ifeq ($(SLAVES),1)
+	$(SHOW)$(TEST_PREFIX); printf "\nTests with --use-slaves:\n\n" ;\
+	$(TEST_CMD) --use-slaves
+endif
 
 #----------------------------------------------------------------------------------------------
 

opt/pack.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+[[ $IGNERR == 1 ]] || set -e
+[[ $VERBOSE == 1 ]] && set -x
+
 [[ -z $DEVICE ]] && { echo DEVICE undefined; exit 1; }
 [[ -z $BINDIR ]] && { echo BINDIR undefined; exit 1; }
 [[ -z $INSTALL_DIR ]] && { echo INSTALL_DIR undefined; exit 1; }
@@ -9,7 +12,7 @@ HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 . $HERE/readies/shibumi/functions
 ROOT=$(realpath $HERE/..)
 
-RAMP_PROG=ramp
+RAMP_PROG="python3 -m RAMP.ramp"
 REDIS_ENT_LIB_PATH=/opt/redislabs/lib
 
 BINDIR=$(realpath $BINDIR)
@@ -18,30 +21,37 @@ INSTALL_DIR=$(realpath $INSTALL_DIR)
 export LC_ALL=C.UTF-8
 export LANG=C.UTF-8
 
+export ARCH=$($ROOT/opt/readies/bin/platform --arch)
+export OS=$($ROOT/opt/readies/bin/platform --os)
+export OSNICK=$($ROOT/opt/readies/bin/platform --osnick)
+
 pack_ramp() {
 	echo "Building RAMP file ..."
 	cd $ROOT
+	
+	local STEM=$PRODUCT.$OS-$OSNICK-$ARCH
+	local FQ_VER=$GIT_VER
+	local FQ_PACKAGE=$STEM.$FQ_VER
+
+	# this is only to extract {semantic_version} into VERSION
 	RAMPOUT=$(mktemp /tmp/ramp.XXXXXX)
 	$RAMP_PROG pack -m $ROOT/ramp.yml -o $BINDIR/$PRODUCT.{os}-{architecture}.{semantic_version}.zip $INSTALL_DIR/$PRODUCT.so 2> /dev/null | grep '.zip' > $RAMPOUT
-	realpath $(tail -1 $RAMPOUT) > $BINDIR/PACKAGE
-	cat $BINDIR/PACKAGE | sed -e "s/[^.]*\.[^.]*\.\(.*\)\.zip/\1/" > $BINDIR/VERSION
-	cat $BINDIR/PACKAGE | sed -e "s/[^.]*\.\([^.]*\)\..*\.zip/\1/" > $BINDIR/OSARCH
-	PACKAGE=$(cat $BINDIR/PACKAGE)
-	VERSION=$(cat $BINDIR/VERSION)
-	OSARCH=$(cat $BINDIR/OSARCH)
-	$RAMP_PROG pack -m $ROOT/ramp.yml -o "$BINDIR/$PRODUCT.{os}-{architecture}.{semantic_version}.zip" \
-		-c "BACKENDSPATH $REDIS_ENT_LIB_PATH/$PRODUCT-$DEVICE-$VERSION/backends" $INSTALL_DIR/$PRODUCT.so 2> /dev/null | grep '.zip' > $RAMPOUT
-	rm -f $RAMPOUT
-	export PACK_FNAME="$(basename $PACKAGE)"
+	local rampfile=`realpath $(tail -1 $RAMPOUT)`
+	rm -f $rampfile $RAMPOUT
+	echo `basename $rampfile` | sed -e "s/[^.]*\.[^.]*\.\(.*\)\.zip/\1/" > $BINDIR/VERSION
+	export VERSION=$(cat $BINDIR/VERSION)
+
+	$RAMP_PROG pack -m $ROOT/ramp.yml -o $BINDIR/$FQ_PACKAGE.zip \
+		-c "BACKENDSPATH $REDIS_ENT_LIB_PATH/$PRODUCT-$DEVICE-$VERSION/backends" $INSTALL_DIR/$PRODUCT.so > /dev/null 2>&1
 
 	cd "$BINDIR"
-	RAMP_STEM=$PRODUCT.$OSARCH
-	ln -sf $PACK_FNAME $RAMP_STEM.latest.zip
-	[[ ! -z $BRANCH ]] && ln -sf $PACK_FNAME $RAMP_STEM.${BRANCH}.zip
-	ln -sf $PACK_FNAME $RAMP_STEM.${GIT_VER}.zip
+	ln -sf $FQ_PACKAGE.zip $STEM.$VERSION.zip
+	ln -sf $FQ_PACKAGE.zip $STEM.latest.zip
+	# [[ ! -z $BRANCH ]] && ln -sf $FQ_PACKAGE.zip $STEM.$BRANCH.zip
 
-	export RELEASE_ARTIFACTS="$RELEASE_ARTIFACTS $PACK_FNAME $RAMP_STEM.latest.zip"
-	export DEV_ARTIFACTS="$DEV_ARTIFACTS $RAMP_STEM.${BRANCH}.zip $RAMP_STEM.${GIT_VER}.zip"
+	export RELEASE_ARTIFACTS="$RELEASE_ARTIFACTS $STEM.$VERSION.zip $RAMP_STEM.latest.zip"
+	export DEV_ARTIFACTS="$DEV_ARTIFACTS $FQ_PACKAGE.zip $STEM.$BRANCH.zip"
+	# [[ ! -z $BRANCH ]] && export DEV_ARTIFACTS="$DEV_ARTIFACTS $DEPS.$BRANCH.tgz"
 
 	echo "Done."
 }
@@ -50,35 +60,37 @@ pack_deps() {
 	echo "Building dependencies file ..."
 
 	cd $ROOT
-	PACK_FNAME=$(basename `cat $BINDIR/PACKAGE`)
-	ARCHOSVER=$(echo "$PACK_FNAME" | sed -e "s/^[^.]*\.\([^.]*\..*\)\.zip/\1/")
+
 	VERSION=$(cat $BINDIR/VERSION)
+	local VARIANT=$OS-$OSNICK-$ARCH.$GIT_VER
+
+	local STEM=$PRODUCT-$DEVICE-dependencies.$OS-$OSNICK-$ARCH
+	local FQ_VER=$GIT_VER
+	local FQ_PACKAGE=$STEM.$FQ_VER
+	
 	cd $INSTALL_DIR
+	local BACKENDS_DIR=$PRODUCT-$DEVICE-$VERSION
 	if [[ ! -h backends ]]; then
 		[[ ! -d backends ]] && { echo "install-$DEVICE/backend directory not found." ; exit 1; }
-		rm -rf $PRODUCT-$DEVICE-$VERSION
-		mkdir $PRODUCT-$DEVICE-$VERSION
+		rm -rf $BACKENDS_DIR
+		mkdir $BACKENDS_DIR
 
-		mv backends $PRODUCT-$DEVICE-$VERSION
-		ln -s $PRODUCT-$DEVICE-$VERSION/backends backends
+		mv backends $BACKENDS_DIR
+		ln -s $BACKENDS_DIR/backends backends
 	fi
-	find $PRODUCT-$DEVICE-$VERSION -name "*.so*" | xargs tar pczf $BINDIR/$PRODUCT-$DEVICE-dependencies.$ARCHOSVER.tgz
-	export DEPS_FNAME="$PRODUCT-$DEVICE-dependencies.$ARCHOSVER.tgz"
+	find $BACKENDS_DIR -name "*.so*" | xargs tar pczf $BINDIR/$FQ_PACKAGE.tgz
 
 	cd "$BINDIR"
-	DEPS_STEM=$PRODUCT-$DEVICE-dependencies.$OSARCH
-	ln -sf $DEPS_FNAME $DEPS_STEM.latest.tgz
-	[[ ! -z $BRANCH ]] && ln -sf $DEPS_FNAME $DEPS_STEM.${BRANCH}.tgz
-	ln -sf $DEPS_FNAME $DEPS_STEM.${GIT_VER}.tgz
+	ln -sf $FQ_PACKAGE.tgz $STEM.$VERSION.tgz
+	ln -sf $FQ_PACKAGE.tgz $STEM.latest.tgz
+	# [[ ! -z $BRANCH ]] && ln -sf $FQ_PACKAGE.tgz $STEM.$BRANCH.tgz
 
-	export RELEASE_ARTIFACTS="$RELEASE_ARTIFACTS $DEPS_FNAME $DEPS_STEM.latest.tgz"
-	export DEV_ARTIFACTS="$DEV_ARTIFACTS $DEPS_STEM.${BRANCH}.tgz $DEPS_STEM.${GIT_VER}.tgz"
+	export RELEASE_ARTIFACTS="$RELEASE_ARTIFACTS $STEM.$VERSION.tgz $STEM.latest.tgz"
+	export DEV_ARTIFACTS="$DEV_ARTIFACTS $FQ_PACKAGE.tgz"
+	# [[ ! -z $BRANCH ]] && export DEV_ARTIFACTS="$DEV_ARTIFACTS $STEM.$BRANCH.tgz"
 
 	echo "Done."
- }
-
-set -e
-[[ $VERBOSE == 1 ]] && set -x
+}
 
 if [[ $1 == --help || $1 == help ]]; then
 	cat <<-END
@@ -103,16 +115,20 @@ PRODUCT_LIB=$PRODUCT.so
 
 GIT_VER=""
 if [[ -d $ROOT/.git ]]; then
-	GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+	if [[ ! -z $BRANCH ]]; then
+		GIT_BRANCH="$BRANCH"
+	else
+		GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+	fi
 	GIT_COMMIT=$(git describe --always --abbrev=7 --dirty="+")
 	GIT_VER="${GIT_BRANCH}-${GIT_COMMIT}"
-fi
-
-OSX=""
-if [[ $($HERE/readies/bin/platform --os) == macosx ]]; then
-	# macOS: ramp is installed here
-	OSX=1
-	export PATH=$PATH:$HOME/Library/Python/2.7/bin
+else
+	if [[ ! -z $BRANCH ]]; then
+		GIT_BRANCH="$BRANCH"
+	else
+		GIT_BRANCH=unknown
+	fi
+	GIT_VER="$GIT_BRANCH"
 fi
 
 if ! command -v redis-server > /dev/null; then
@@ -126,7 +142,7 @@ pack_ramp
 if [[ ! -z $INTO ]]; then
 	mkdir -p $INTO
 	cd $INTO
-	mkdir release branch
+	mkdir -p release branch
 
 	for f in $RELEASE_ARTIFACTS; do
 		[[ -f $BINDIR/$f ]] && cp $BINDIR/$f release/

opt/readies/bin/platform

@@ -49,4 +49,5 @@ if ret == "":
     else:
         nick = ""
     ret = os + " " + platform.os_ver + nick + " " + platform.arch
-print(ret.strip())
+if ret.strip() != "":
+	print(ret.strip())

opt/readies/paella/__init__.py

@@ -26,7 +26,7 @@ def __setattr__(self,name,value):
 
 #----------------------------------------------------------------------------------------------
 
-Global.bb = bb
+Global.BB = bb
 Global.eprint = eprint
 Global.fatal = fatal
 Global.cwd = cwd

opt/readies/paella/debug.py

@@ -3,11 +3,14 @@
 
 #----------------------------------------------------------------------------------------------
 
-if 'PYDEBUG' in os.environ:
+pydebug = os.environ.get('PYDEBUG', '')
+if pydebug == '1' or pydebug == 'pudb':
     try:
         from pudb import set_trace as bb
     except ImportError:
         from pdb import set_trace as bb
+elif pydebug == 'pdb':
+    from pdb import set_trace as bb
 else:
 	def bb(): pass
 

opt/readies/paella/files.py

@@ -2,8 +2,11 @@
 from contextlib import contextmanager
 import os
 import os.path
-import urllib.request
 import tempfile
+try:
+    from urllib2 import urlopen
+except:
+    from urllib.request import urlopen
 
 #----------------------------------------------------------------------------------------------
 
@@ -38,7 +41,10 @@ def wget(url, dest="", tempdir=False):
             dest = tempfilepath()
         elif tempdir:
             dest = os.path.join('/tmp', dest)
-    urllib.request.urlretrieve(url, dest)
+    ufile = urlopen(url)
+    data = ufile.read()
+    with open(dest, "wb") as file:
+        file.write(data)
     return os.path.abspath(dest)
 
 #----------------------------------------------------------------------------------------------
@@ -59,3 +65,6 @@ def mkdir_p(dir):
         os.makedirs(dir, exist_ok=True)
 
 #----------------------------------------------------------------------------------------------
+
+def relpath(dir, rel):
+    return os.path.abspath(os.path.join(dir, rel))