Test DVC remote mechanism in CI

.github/workflows/test-and-publish-on-tag.yml

@@ -7,11 +7,8 @@ on:
     tags:
       - "ws-*@*"
 
-env:
-  REGISTRY: docker.io
-
 jobs:
-  build-test-push:
+  test-on-tag:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout from Git
@@ -28,73 +25,44 @@ jobs:
           sudo dpkg -i hurl_5.0.1_amd64.deb
           hurl --version
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Parse tag to get service info
-        id: parse
-        run: |
-          TAG=${{ github.ref_name }}
-          SERVICE_INTERMEDIATE=${TAG#ws-}
-          SERVICE_NAME=${SERVICE_INTERMEDIATE%@*}
-          VERSION=${TAG##*@}
-          echo "service_name=$SERVICE_NAME" >> $GITHUB_OUTPUT
-          echo "version=$VERSION" >> $GITHUB_OUTPUT
-          echo "image_name=cnrsinist/ws-$SERVICE_NAME" >> $GITHUB_OUTPUT
-
       - name: Build .env from Github Actions secrets
         shell: bash
         run: |
           bin/create-env.sh ${{ github.ref_name }} ${{ secrets.WEBDAV_LOGIN }} ${{ secrets.WEBDAV_PASSWORD }} ${{ secrets.WEBDAV_URL }} ${{secrets.OPENALEX_API_KEY}} ${{secrets.UNPAYWALL_API_KEY}} ${{secrets.CROSSREF_API_KEY}} ${{secrets.LIBRARIES_IO_API_KEY}} ${{secrets.ORCID_CLIENT_ID}} ${{secrets.ORCID_SECRET}} ${{secrets.ILAAS_API_KEY}}
           ls -l services/*/.env
 
-      - name: Build Docker image
-        uses: docker/build-push-action@v6
-        with:
-          context: services/${{ steps.parse.outputs.service_name }}
-          push: false
-          load: true
-          tags: |
-            ${{ steps.parse.outputs.image_name }}:latest
-            ${{ steps.parse.outputs.image_name }}:${{ steps.parse.outputs.version }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-      - name: Start container for testing
-        run: |
-          docker run --name test-container --rm --detach -p 31976:31976 ${{ steps.parse.outputs.image_name }}:latest
-
-      - name: Wait for server to be ready
-        run: |
-          echo "Waiting for http://localhost:31976"
-          for i in {1..10}; do
-            if curl -s http://localhost:31976 > /dev/null 2>&1; then
-              echo "Server is ready!"
-              exit 0
-            fi
-            echo "Attempt $i/10 - waiting..."
-            sleep 30
-          done
-          echo "Server did not become ready"
-          exit 1
-
-      - name: Run Hurl tests
+      - name: Build Docker Image & Test
+        shell: bash
         run: |
-          cd services/${{ steps.parse.outputs.service_name }}
-          hurl --variable host=http://localhost:31976 --variable blocked=true --test --jobs 1 tests.hurl
+          bin/test-on-tag.sh ${{ github.ref_name }}
 
-      - name: Stop test container
-        if: always()
-        run: |
-          docker stop test-container || true
+  build-on-tag:
+    needs:
+      - test-on-tag
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout from Git
+        uses: actions/checkout@v4
 
       - name: Docker Login
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
-      - name: Push Docker image
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 24
+
+      - name: Build .env from Github Actions secrets
+        shell: bash
+        run: |
+          bin/create-env.sh ${{ github.ref_name }} ${{ secrets.WEBDAV_LOGIN }} ${{ secrets.WEBDAV_PASSWORD }} ${{ secrets.WEBDAV_URL }} ${{secrets.OPENALEX_API_KEY}} ${{secrets.UNPAYWALL_API_KEY}} ${{secrets.CROSSREF_API_KEY}} ${{secrets.LIBRARIES_IO_API_KEY}} ${{secrets.ORCID_CLIENT_ID}} ${{secrets.ORCID_SECRET}} ${{secrets.ILAAS_API_KEY}}
+          ls -l services/*/.env
+
+      - name: Build & Push Docker Image
+        shell: bash
         run: |
-          docker push ${{ steps.parse.outputs.image_name }}:latest
-          docker push ${{ steps.parse.outputs.image_name }}:${{ steps.parse.outputs.version }}
+          echo "Building and pushing image for ${{ github.ref_name }}"
+          bin/build-and-push.sh ${{ github.ref_name }}

bin/build-and-push.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# tag should be in the format:
+# ws-<service-name>@<version>
+TAG=$1
+SERVICE_INTERMEDIATE=${TAG#ws-} # remove ws- part
+SERVICE_NAME=${SERVICE_INTERMEDIATE%@*} # remove version part
+
+if [ ! -d "services/$SERVICE_NAME" ]; then
+    echo "Could not find directory services/$SERVICE_NAME"
+    exit 1
+fi
+
+echo "Building $TAG"
+
+cd "services/$SERVICE_NAME"
+npm run build
+npm run publish

bin/build-and-test.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -eu
+
+# Wait until $1 is available, retrying every $3 milliseconds up to $2 times.
+wait_for_url () {
+    echo "Waiting for $1"
+    printf 'GET %s\nHTTP 200' "$1" | hurl --retry "$2" --retry-interval "$3" --verbose> /dev/null;
+    return 0
+}
+
+# branch name should be in the format:
+# services/<service-name>/<comment>
+BRANCH_NAME=$1
+SERVICE_INTERMEDIATE=${BRANCH_NAME#services/} # remove services/ part
+SERVICE_NAME=${SERVICE_INTERMEDIATE%/*}
+
+echo "Starting container of $SERVICE_NAME"
+cd "services/$SERVICE_NAME"
+npm run start:dev
+
+echo "Waiting server to be ready"
+wait_for_url "http://localhost:31976" 10 30000
+
+echo "Running hurl tests"
+hurl --jobs 1 --variable host=http://localhost:31976 --test tests.hurl
+
+echo "Stopping container of $SERVICE_NAME"
+npm run stop:dev
+
+echo "Done"

services/affiliation-rnsr/Dockerfile

@@ -2,7 +2,7 @@
 FROM python:3.9-slim-bullseye AS dvc-files
 WORKDIR /dvc
 RUN apt-get update && apt-get -y --no-install-recommends install git=1:2.30.2-1+deb11u5 && apt-get clean && rm -rf /var/lib/apt/lists/*
-RUN pip install --no-cache-dir dvc[webdav]==3.39.0
+RUN pip install --no-cache-dir dvc[webdav]==3.67.1
 RUN --mount=type=secret,id=webdav_login \
     --mount=type=secret,id=webdav_password \
     --mount=type=secret,id=webdav_url \
@@ -11,8 +11,9 @@ RUN --mount=type=secret,id=webdav_login \
     dvc remote add -d webdav-remote "$(cat /run/secrets/webdav_url)" && \
     dvc remote modify --local webdav-remote user "$(cat /run/secrets/webdav_login)" && \
     dvc remote modify --local webdav-remote password "$(cat /run/secrets/webdav_password)"
+
 COPY ./v3/affiliation/models.dvc /dvc
-RUN dvc pull -v
+RUN dvc pull --allow-missing --jobs 1 --verbose
 
 FROM cnrsinist/ezs-python-server:py3.9-no24-1.0.13
 

services/affiliation-rnsr/README.md

@@ -17,7 +17,8 @@ Nécessite les variables d'environnement:
 
 `npm run build:dev` et `npm start:dev` importent le fichier `.env` quand il existe.
 
-> 📗 Suggestion: déclarez les variables dans le fichier `.env` de cette manière:
+> [!TIP]  
+> Déclarez les variables dans le fichier `.env` de cette manière:
 >
 > ```bash
 > export WEBDAV_URL=webdavs://your.webdav.com/dvc

services/affiliation-rnsr/tests.hurl

@@ -1,3 +1,17 @@
+# To test locally, use:
+# hurl --test --variable host=http://localhost:31976 --jobs 1 services/affiliation-rnsr/tests.hurl
+# npm run test:local affiliation-rnsr
+#
+# To test remotely, use:
+# hurl --test --variable host=https://affiliation-rnsr.services.istex.fr --jobs 1 services/affiliation-rnsr/tests.hurl
+# npm run test:remote affiliation-rnsr
+#
+# The service should be launched as a local server. URL: http://localhost:31976
+# Just launch:
+# npm -w services/affiliation-rnsr run start:dev
+# npm run generate:example-tests services/affiliation-rnsr
+# npm -w services/affiliation-rnsr run stop:dev
+
 POST {{host}}/v3/affiliation/rnsr?indent=true
 content-type: application/json
 [