Skip to content

Commit 6c33359

Browse files
author
DvirDukhan
authored
Merge branch 'master' into patch-1
2 parents 8b53bef + 310f8c1 commit 6c33359

37 files changed

+590
-176
lines changed

‎.circleci/config.yml‎

Lines changed: 52 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -43,17 +43,6 @@ commands:
4343
circleci step halt
4444
fi
4545
46-
setup-executor:
47-
steps:
48-
- run:
49-
name: Setup executor
50-
command: |
51-
sudo apt-get -qq update
52-
sudo apt-get -q install -y git openssh-client curl ca-certificates make tar gzip
53-
bash <(curl -fsSL https://get.docker.com)
54-
# - setup_remote_docker:
55-
# docker_layer_caching: true
56-
5746
relocate-docker-storage:
5847
steps:
5948
- run:
@@ -138,54 +127,6 @@ commands:
138127
- store_artifacts:
139128
path: tests/logs
140129

141-
platforms-build-steps:
142-
steps:
143-
- abort_for_docs
144-
- abort_for_noci
145-
- early_return_for_forked_pull_requests
146-
- checkout-all
147-
- relocate-docker-storage
148-
- setup-automation
149-
- run:
150-
name: Build for platform
151-
command: |
152-
pushd opt/build/docker
153-
docker login -u redisfab -p $DOCKER_REDISFAB_PWD
154-
for osnick in bionic xenial; do
155-
make CPU=1 OSNICK=$osnick ARTIFACTS=1 VERBOSE=1 build publish
156-
make GPU=1 OSNICK=$osnick ARTIFACTS=1 VERBOSE=1 build publish
157-
docker image prune -f
158-
done
159-
popd > /dev/null
160-
logstar=bin/artifacts/tests-logs-cpu.tgz
161-
logsdir=tests/logs/cpu
162-
mkdir -p $logsdir
163-
if [[ -e $logstar ]]; then tar -C $logsdir -xzf $logstar; fi
164-
(cd bin/artifacts; tar -cf snapshots.tar snapshots/)
165-
no_output_timeout: 40m
166-
- persist_to_workspace:
167-
root: bin/
168-
paths:
169-
- artifacts/*.zip
170-
- artifacts/*.tgz
171-
- artifacts/*.tar
172-
- store_artifacts:
173-
path: test/logs
174-
175-
deploy-steps:
176-
parameters:
177-
from:
178-
type: string
179-
steps:
180-
- abort_for_docs
181-
- abort_for_noci
182-
- early_return_for_forked_pull_requests
183-
- run:
184-
name: Deploy to S3
185-
command: |
186-
du -ah --apparent-size artifacts/*
187-
aws s3 cp artifacts/ s3://redismodules/$PACKAGE_NAME/ --acl public-read --recursive --exclude "*" --include "*.zip" --include "*.tgz"
188-
189130
jobs:
190131
lint:
191132
docker:
@@ -207,16 +148,43 @@ jobs:
207148
- build-steps:
208149
platform: debian
209150

210-
# this build runs on a fixed machine, due to a storage need
211-
# nothing about it necessitates the machine itself, other than the need for more disk.
212151
platforms-build:
213-
machine:
214-
enabled: true
215-
docker_layer_caching: true
216-
resource_class: medium
217-
image: ubuntu-2004:202101-01
152+
parameters:
153+
lite: # LITE value during make
154+
type: string
155+
osnick: # OSNICK value for the base platform of the docker
156+
type: string
157+
target: # CPU|GPU
158+
type: string
159+
docker:
160+
- image: redisfab/rmbuilder:6.2.1-x64-buster
218161
steps:
219-
- platforms-build-steps
162+
- abort_for_docs
163+
- abort_for_noci
164+
- early_return_for_forked_pull_requests
165+
- setup_remote_docker
166+
- checkout-all
167+
- setup-automation
168+
- run:
169+
name: Build for platform
170+
command: |
171+
bash <(curl -fsSL https://get.docker.com)
172+
pushd opt/build/docker
173+
docker login -u redisfab -p $DOCKER_REDISFAB_PWD
174+
make build <<parameters.target>> OSNICK=<<parameters.osnick>> ARTIFACTS=1 VERBOSE=1 <<parameters.lite>>
175+
popd > /dev/null
176+
logstar=bin/artifacts/tests-logs-cpu.tgz
177+
logsdir=tests/logs/cpu
178+
mkdir -p $logsdir
179+
if [[ -e $logstar ]]; then tar -C $logsdir -xzf $logstar; fi
180+
# (cd bin/artifacts; tar -cf snapshots.tar snapshots/)
181+
no_output_timeout: 40m
182+
- persist_to_workspace:
183+
root: bin/
184+
paths:
185+
- artifacts/*
186+
- store_artifacts:
187+
path: test/logs
220188

221189
coverage:
222190
docker:
@@ -414,14 +382,7 @@ jobs:
414382
- run:
415383
name: Deploy Snapshots to S3
416384
command: |
417-
cd workspace/artifacts
418-
for f in snapshots*.tar; do
419-
echo "Extracting $f ..."
420-
tar xf $f
421-
done
422-
echo "... done."
423-
du -ah --apparent-size *
424-
cd snapshots
385+
cd workspace/artifacts/snapshots
425386
for f in `ls *.zip *.tgz`; do
426387
aws s3 cp --no-progress $f s3://redismodules/$PACKAGE_NAME/snapshots/ --acl public-read
427388
done
@@ -518,11 +479,12 @@ on-version-tags: &on-version-tags
518479
tags:
519480
only: /^v[0-9].*/
520481

521-
on-master-and-version-tags: &on-master-and-version-tags
482+
on-master-version-tags-and-dockertests: &on-master-version-tags-and-dockertests
522483
filters:
523484
branches:
524485
only:
525486
- master
487+
- /.*dockertest$/
526488
tags:
527489
only: /^v[0-9].*/
528490

@@ -544,14 +506,26 @@ workflows:
544506
version: 2
545507
build_and_package:
546508
jobs:
509+
547510
- lint:
548511
<<: *on-any-branch
549512
- build-and-test:
550513
<<: *on-any-branch-but-tags
551514
<<: *after-linter
552515
- platforms-build:
553516
<<: *after-build-and-test
554-
<<: *on-master-and-version-tags
517+
<<: *on-master-version-tags-and-dockertests
518+
matrix:
519+
parameters:
520+
osnick:
521+
- xenial
522+
- bionic
523+
lite:
524+
- "REDISAI_LITE=0 publish"
525+
- "REDISAI_LITE=1"
526+
target:
527+
- "CPU=1"
528+
- "GPU=1"
555529
- coverage:
556530
context: common
557531
<<: *on-dev-branches
@@ -572,7 +546,7 @@ workflows:
572546
- deploy-snapshot:
573547
context: common
574548
<<: *after-platform-builds
575-
<<: *on-integ-branch
549+
<<: *on-master-version-tags-and-dockertests
576550
- deploy-release:
577551
context: common
578552
<<: *after-platform-builds

‎CMakeLists.txt‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ option(BUILD_TF "Build the TensorFlow backend" ON)
5555
option(BUILD_TFLITE "Build the TensorFlow Lite backend" ON)
5656
option(BUILD_ORT "Build the ONNXRuntime backend" ON)
5757
option(BUILD_TORCH "Build the PyTorch backend" ON)
58-
58+
option(BUILD_REDISAI_LITE "Build the RedisAI Lite Varient" OFF)
5959
#----------------------------------------------------------------------------------------------
6060

6161
FUNCTION(ADD_LDFLAGS _TARGET NEW_FLAGS)
@@ -114,6 +114,9 @@ ENDIF()
114114

115115
SET(CMAKE_C_STANDARD 11)
116116
ADD_DEFINITIONS(-DREDISMODULE_EXPERIMENTAL_API)
117+
if(BUILD_REDISAI_LITE)
118+
ADD_DEFINITIONS(-DREDISAI_LITE)
119+
ENDIF()
117120

118121
# SET(CUDA_TOOLKIT_ROOT_DIR /usr/local/cuda-10.0)
119122

‎Dockerfile‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ ARG OS=debian:buster-slim
1212
ARG ARCH=x64
1313

1414
ARG PACK=0
15+
ARG REDISAI_LITE=0
1516
ARG TEST=0
1617

1718
#----------------------------------------------------------------------------------------------
@@ -22,6 +23,9 @@ ARG OSNICK
2223
ARG OS
2324
ARG ARCH
2425
ARG REDIS_VER
26+
ARG REDISAI_LITE
27+
ARG PACK
28+
ARG TEST
2529

2630
RUN echo "Building for ${OSNICK} (${OS}) for ${ARCH} [with Redis ${REDIS_VER}]"
2731

@@ -40,18 +44,16 @@ RUN if [ "$DEPS_ARGS" = "" ]; then ./get_deps.sh cpu; else env $DEPS_ARGS ./get_
4044

4145
ARG BUILD_ARGS=""
4246
ADD ./ /build
43-
RUN bash -l -c "make -C opt build $BUILD_ARGS SHOW=1"
47+
RUN bash -l -c "make -C opt build REDISAI_LITE=${REDISAI_LITE} $BUILD_ARGS SHOW=1"
4448

45-
ARG PACK
46-
ARG TEST
4749

4850
RUN mkdir -p bin/artifacts
4951
RUN set -e ;\
50-
if [ "$PACK" = "1" ]; then bash -l -c "make -C opt pack"; fi
52+
if [ "$PACK" = "1" ]; then bash -l -c "make -C opt pack REDISAI_LITE=${REDISAI_LITE}"; fi
5153

5254
RUN set -e ;\
5355
if [ "$TEST" = "1" ]; then \
54-
bash -l -c "TEST= make -C opt test $BUILD_ARGS NO_LFS=1" ;\
56+
bash -l -c "TEST= make -C opt test REDISAI_LITE=${REDISAI_LITE} $BUILD_ARGS NO_LFS=1" ;\
5557
if [[ -d test/logs ]]; then \
5658
tar -C test/logs -czf bin/artifacts/test-logs-cpu.tgz . ;\
5759
fi ;\

‎Dockerfile.arm‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ ARG OS=debian:buster-slim
1212
ARG ARCH=arm64v8
1313

1414
ARG PACK=0
15+
ARG REDISAI_LITE=0
1516
ARG TEST=0
1617

1718
#----------------------------------------------------------------------------------------------
@@ -66,7 +67,7 @@ ARG REDIS_VER
6667
ARG PACK
6768

6869
RUN if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1; fi
69-
RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi
70+
RUN if [ ! -z $(command -v yum) ]; then yum install -y libgomp; fi
7071

7172
ENV REDIS_MODULES /usr/lib/redis/modules
7273
ENV LD_LIBRARY_PATH $REDIS_MODULES

‎Dockerfile.gpu‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ ARG ARCH=x64
1414
ARG CUDA_VER=11.0-cudnn8
1515

1616
ARG PACK=0
17+
ARG REDISAI_LITE=0
1718
ARG TEST=0
1819

1920
#----------------------------------------------------------------------------------------------
@@ -51,11 +52,12 @@ ADD ./ /build
5152
RUN bash -l -c "make -C opt build GPU=1 $BUILD_ARGS SHOW=1"
5253

5354
ARG PACK
55+
ARG REDISAI_LITE
5456
ARG TEST
5557

5658
RUN mkdir -p bin/artifacts
5759
RUN set -e ;\
58-
if [ "$PACK" = "1" ]; then bash -l -c "make -C opt pack GPU=1"; fi
60+
if [ "$PACK" = "1" ]; then bash -l -c "make -C opt pack GPU=1 REDISAI_LITE=$REDISAI_LITE"; fi
5961

6062
RUN set -e ;\
6163
if [ "$TEST" = "1" ]; then \

‎Dockerfile.jetson‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ ARG CUDA_VER=10.2-cudnn7
1616
ARG L4T_VER=r32.4.4
1717

1818
ARG PACK=0
19+
ARG REDISAI_LITE=0
1920
ARG TEST=0
2021

2122
#----------------------------------------------------------------------------------------------
@@ -57,6 +58,7 @@ RUN bash -c "set -e ;\
5758
make -C opt build $BUILD_ARGS SHOW=1"
5859

5960
ARG PACK
61+
ARG REDISAI_LITE
6062
ARG TEST
6163

6264
RUN mkdir -p bin/artifacts

‎README.md‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[![GitHub issues](https://img.shields.io/github/release/RedisAI/RedisAI.svg?sort=semver)](https://github.com/RedisAI/RedisAI/releases/latest)
22
[![CircleCI](https://circleci.com/gh/RedisAI/RedisAI/tree/master.svg?style=svg)](https://circleci.com/gh/RedisAI/RedisAI/tree/master)
3-
[![Dockerhub](https://img.shields.io/badge/dockerhub-redislabs%2Fredisai-blue)](https://hub.docker.com/r/redislabs/redisai/tags/)
3+
[![Dockerhub](https://img.shields.io/badge/dockerhub-redislabs%2Fredisai-blue)](https://hub.docker.com/r/redislabs/redisai/tags/)
44
[![codecov](https://codecov.io/gh/RedisAI/RedisAI/branch/master/graph/badge.svg)](https://codecov.io/gh/RedisAI/RedisAI)
55
[![Total alerts](https://img.shields.io/lgtm/alerts/g/RedisAI/RedisAI.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/RedisAI/RedisAI/alerts/)
66

@@ -69,7 +69,7 @@ redis-cli
6969

7070
## Building
7171

72-
You should obtain the module's source code and submodule using git like so:
72+
You should obtain the module's source code and submodule using git like so:
7373

7474
```sh
7575
git clone --recursive https://github.com/RedisAI/RedisAI
@@ -116,14 +116,21 @@ redis-server --loadmodule install-cpu/redisai.so
116116

117117
Some languages have client libraries that provide support for RedisAI's commands:
118118

119-
| Project | Language | License | Author | URL |
120-
| ------- | -------- | ------- | ------ | --- |
121-
| JRedisAI | Java | BSD-3 | [RedisLabs](https://redislabs.com/) | [Github](https://github.com/RedisAI/JRedisAI) |
122-
| redisai-py | Python | BSD-3 | [RedisLabs](https://redislabs.com/) | [Github](https://github.com/RedisAI/redisai-py) |
123-
| redisai-go | Go | BSD-3 | [RedisLabs](https://redislabs.com/) | [Github](https://github.com/RedisAI/redisai-go) |
124-
| redisai-js | Typescript/Javascript | BSD-3 | [RedisLabs](https://redislabs.com/) | [Github](https://github.com/RedisAI/redisai-js) |
125-
| redis-modules-sdk | TypeScript | BSD-3-Clause | [Dani Tseitlin](https://github.com/danitseitlin) | [Github](https://github.com/danitseitlin/redis-modules-sdk) |
126-
| redis-modules-java | Java | Apache-2.0 | [dengliming](https://github.com/dengliming) | [Github](https://github.com/dengliming/redis-modules-java) |
119+
| Project | Language | License | Author | URL |
120+
| ------- | -------- | ------- | ------ | --- |
121+
| JRedisAI | Java | BSD-3 | [RedisLabs](https://redislabs.com/) | [Github](https://github.com/RedisAI/JRedisAI) |
122+
| redisai-py | Python | BSD-3 | [RedisLabs](https://redislabs.com/) | [Github](https://github.com/RedisAI/redisai-py) |
123+
| redisai-go | Go | BSD-3 | [RedisLabs](https://redislabs.com/) | [Github](https://github.com/RedisAI/redisai-go) |
124+
| redisai-js | Typescript/Javascript | BSD-3 | [RedisLabs](https://redislabs.com/) | [Github](https://github.com/RedisAI/redisai-js) |
125+
| redis-modules-sdk | TypeScript | BSD-3-Clause | [Dani Tseitlin](https://github.com/danitseitlin) | [Github](https://github.com/danitseitlin/redis-modules-sdk) |
126+
| redis-modules-java | Java | Apache-2.0 | [dengliming](https://github.com/dengliming) | [Github](https://github.com/dengliming/redis-modules-java) |
127+
| smartredis | C++ | BSD-2-Clause | [Cray Labs](https://github.com/CrayLabs) | [Github](https://github.com/CrayLabs/SmartRedis) |
128+
| smartredis | C | BSD-2-Clause | [Cray Labs](https://github.com/CrayLabs) | [Github](https://github.com/CrayLabs/SmartRedis) |
129+
| smartredis | Fortran | BSD-2-Clause | [Cray Labs](https://github.com/CrayLabs) | [Github](https://github.com/CrayLabs/SmartRedis) |
130+
| smartredis | Python | BSD-2-Clause | [Cray Labs](https://github.com/CrayLabs) | [Github](https://github.com/CrayLabs/SmartRedis) |
131+
132+
133+
127134

128135

129136
## Backend Dependancy

‎docs/clients.md‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,19 @@
22

33
Some languages already have client libraries that provide support for RedisAI's commands. The following table lists the known ones:
44

5-
| Project | Language | License | Author | URL |
6-
| ------- | -------- | ------- | ------ | --- |
7-
| JRedisAI | Java | BSD-3 | [RedisLabs](https://redislabs.com/) | [Github](https://github.com/RedisAI/JRedisAI) |
8-
| redisai-py | Python | BSD-3 | [RedisLabs](https://redislabs.com/) | [Github](https://github.com/RedisAI/redisai-py) |
9-
| redisai-go | Go | BSD-3 | [RedisLabs](https://redislabs.com/) | [Github](https://github.com/RedisAI/redisai-go) |
10-
| redisai-js | Typescript/Javascript | BSD-3 | [RedisLabs](https://redislabs.com/) | [Github](https://github.com/RedisAI/redisai-js) |
11-
| redis-modules-sdk | TypeScript | BSD-3-Clause | [Dani Tseitlin](https://github.com/danitseitlin) | [Github](https://github.com/danitseitlin/redis-modules-sdk) |
12-
| redis-modules-java | Java | Apache-2.0 | [dengliming](https://github.com/dengliming) | [Github](https://github.com/dengliming/redis-modules-java) |
5+
| Project | Language | License | Author | URL |
6+
| ------- | -------- | ------- | ------ | --- |
7+
| JRedisAI | Java | BSD-3 | [RedisLabs](https://redislabs.com/) | [Github](https://github.com/RedisAI/JRedisAI) |
8+
| redisai-py | Python | BSD-3 | [RedisLabs](https://redislabs.com/) | [Github](https://github.com/RedisAI/redisai-py) |
9+
| redisai-go | Go | BSD-3 | [RedisLabs](https://redislabs.com/) | [Github](https://github.com/RedisAI/redisai-go) |
10+
| redisai-js | Typescript/Javascript | BSD-3 | [RedisLabs](https://redislabs.com/) | [Github](https://github.com/RedisAI/redisai-js) |
11+
| redis-modules-sdk | TypeScript | BSD-3-Clause | [Dani Tseitlin](https://github.com/danitseitlin) | [Github](https://github.com/danitseitlin/redis-modules-sdk) |
12+
| redis-modules-java | Java | Apache-2.0 | [dengliming](https://github.com/dengliming) | [Github](https://github.com/dengliming/redis-modules-java) |
13+
| smartredis | C++ | BSD-2-Clause | [Cray Labs](https://github.com/CrayLabs) | [Github](https://github.com/CrayLabs/SmartRedis) |
14+
| smartredis | C | BSD-2-Clause | [Cray Labs](https://github.com/CrayLabs) | [Github](https://github.com/CrayLabs/SmartRedis) |
15+
| smartredis | Fortran | BSD-2-Clause | [Cray Labs](https://github.com/CrayLabs) | [Github](https://github.com/CrayLabs/SmartRedis) |
16+
| smartredis | Python | BSD-2-Clause | [Cray Labs](https://github.com/CrayLabs) | [Github](https://github.com/CrayLabs/SmartRedis) |
17+
1318

1419

1520
The full documentation for RedisAI's API can be found at the [Commands page](commands.md).

0 commit comments

Comments
 (0)