IVS-719 - Scaling

.env

@@ -1,51 +1,59 @@
-# variables in Docker Compose
-DEBUG = True
-ENV = Development
-PUBLIC_URL = http://localhost
-
-# Certbot
-CERTBOT_DOMAIN = _
-CERTBOT_EMAIL = <CERTBOT_EMAIL>
-
-# Django
-MEDIA_ROOT = /files_storage
-DJANGO_DB = postgresql
-DJANGO_SECRET_KEY = django-insecure-um7-^+&jbk_=80*xcc9uf4nh$4koida7)ja&6!vb*$8@n288jk
-DJANGO_ALLOWED_HOSTS = localhost
-DJANGO_TRUSTED_ORIGINS = http://localhost:3000 http://localhost http://localhost:8000
-DJANGO_LOG_LEVEL = INFO
-GHERKIN_LOG_FOLDER = /gherkin_logs
-DJANGO_GUNICORN_WORKERS = 3
-DJANGO_GUNICORN_THREADS_PER_WORKER = 4
-
-# DB
-POSTGRES_HOST = db
-POSTGRES_NAME = postgres
-POSTGRES_USER = postgres
-POSTGRES_PASSWORD = postgres
-POSTGRES_PORT = 5432
-
-# Worker
-REDIS_PORT = 6379
-CELERY_BROKER_URL = redis://redis:6379/0
-CELERY_TASK_SOFT_TIME_LIMIT = 3600
-CELERY_TASK_TIME_LIMIT = 4000
-TASK_TIMEOUT_LIMIT = 3600
-DJANGO_DB_USER_CONTEXT = SYSTEM
-DJANGO_DB_BULK_CREATE_BATCH_SIZE = 1000
-CELERY_CONCURRENCY = 4
-
-# Email
-MAILGUN_API_URL = <MG_API_URL>
-MAILGUN_API_KEY = <MG_API_KEY>
-MAILGUN_FROM_NAME = Validation Service
-MAILGUN_FROM_EMAIL = noreply@localhost
-ADMIN_EMAIL = noreply@localhost
-CONTACT_EMAIL = noreply@localhost
-
-# IAM
-B2C_CLIENT_ID = <B2C_CLIENT_ID>
-B2C_CLIENT_SECRET = <B2C_CLIENT_SECRET>
-B2C_AUTHORITY = <B2C_AUTHORITY>
-B2C_USER_FLOW = <B2C_USER_FLOW>
-USE_WHITELIST = False
+# variables in Docker Compose
+DEBUG = True
+ENV = Development
+PUBLIC_URL = http://localhost
+
+# Certbot
+CERTBOT_DOMAIN = _
+CERTBOT_EMAIL = <CERTBOT_EMAIL>
+
+# Django
+MEDIA_ROOT = /files_storage
+DJANGO_DB = postgresql
+DJANGO_SECRET_KEY = django-insecure-um7-^+&jbk_=80*xcc9uf4nh$4koida7)ja&6!vb*$8@n288jk
+DJANGO_ALLOWED_HOSTS = localhost
+DJANGO_TRUSTED_ORIGINS = http://localhost:3000 http://localhost http://localhost:8000
+DJANGO_LOG_LEVEL = INFO
+GHERKIN_LOG_FOLDER = /gherkin_logs
+DJANGO_GUNICORN_WORKERS = 3
+DJANGO_GUNICORN_THREADS_PER_WORKER = 4
+
+# DB
+POSTGRES_HOST = db
+POSTGRES_NAME = postgres
+POSTGRES_USER = postgres
+POSTGRES_PASSWORD = postgres
+POSTGRES_PORT = 5432
+
+# Worker
+REDIS_PORT = 6379
+CELERY_BROKER_URL = redis://redis:6379/0
+CELERY_TASK_SOFT_TIME_LIMIT = 3600
+CELERY_TASK_TIME_LIMIT = 4000
+TASK_TIMEOUT_LIMIT = 3600
+DJANGO_DB_USER_CONTEXT = SYSTEM
+DJANGO_DB_BULK_CREATE_BATCH_SIZE = 1000
+CELERY_CONCURRENCY = 4
+
+# Email
+MAILGUN_API_URL = <MG_API_URL>
+MAILGUN_API_KEY = <MG_API_KEY>
+MAILGUN_FROM_NAME = Validation Service
+MAILGUN_FROM_EMAIL = noreply@localhost
+ADMIN_EMAIL = noreply@localhost
+CONTACT_EMAIL = noreply@localhost
+
+# IAM
+B2C_CLIENT_ID = <B2C_CLIENT_ID>
+B2C_CLIENT_SECRET = <B2C_CLIENT_SECRET>
+B2C_AUTHORITY = <B2C_AUTHORITY>
+B2C_USER_FLOW = <B2C_USER_FLOW>
+USE_WHITELIST = False
+
+# Swarm (ignored by docker compose)
+# REGISTRY=localhost:5000
+# NFS_SERVER_IP=10.0.0.1
+# WORKER_CPU_LIMIT=2.0
+# WORKER_CPU_RESERVATION=1.0
+# WORKER_MEMORY_LIMIT=2G
+# WORKER_MEMORY_RESERVATION=1G

Makefile

@@ -26,6 +26,88 @@ start-infra-only:
 stop:
 	docker compose down
 
+# --- Docker Swarm ---
+
+REGISTRY ?= localhost:5000
+WORKERS  ?= 2
+ENV_FILE ?= .env
+SWARM_VARS = REGISTRY CERTBOT_DOMAIN CERTBOT_EMAIL NFS_SERVER_IP WORKER_CPU_LIMIT WORKER_MEMORY_LIMIT WORKER_CPU_RESERVATION WORKER_MEMORY_RESERVATION
+SWARM_ENV = ENV_FILE="$(ENV_FILE)" $(foreach v,$(SWARM_VARS),$(v)="$(shell grep '^$(v)=' $(ENV_FILE) | head -1 | cut -d= -f2-)")
+
+start-swarm:
+	env $(SWARM_ENV) envsubst < docker-compose.swarm.yml | docker stack deploy -c - --with-registry-auth validate
+
+start-swarm-nodb:
+	env $(SWARM_ENV) envsubst < docker-compose.swarm.nodb.yml | docker stack deploy -c - --with-registry-auth validate
+
+start-swarm-local:
+	env $(SWARM_ENV) envsubst < docker-compose.swarm.yml > /tmp/_swarm.yml && \
+	env $(SWARM_ENV) envsubst < docker-compose.swarm.local.yml > /tmp/_swarm.local.yml && \
+	docker stack deploy -c /tmp/_swarm.yml -c /tmp/_swarm.local.yml --with-registry-auth validate && \
+	rm -f /tmp/_swarm.yml /tmp/_swarm.local.yml
+
+stop-swarm:
+	docker stack rm validate
+
+scale-workers:
+	docker service scale validate_worker=$(WORKERS)
+
+set-worker-limits:
+	docker service update \
+		$(if $(CPU),--limit-cpu $(CPU)) \
+		$(if $(MEM),--limit-memory $(MEM)) \
+		$(if $(CPU_RES),--reserve-cpu $(CPU_RES)) \
+		$(if $(MEM_RES),--reserve-memory $(MEM_RES)) \
+		validate_worker
+
+swarm-push: build
+	docker tag buildingsmart/validationsvc-backend $(REGISTRY)/validationsvc-backend
+	docker tag buildingsmart/validationsvc-frontend $(REGISTRY)/validationsvc-frontend
+	docker push $(REGISTRY)/validationsvc-backend
+	docker push $(REGISTRY)/validationsvc-frontend
+
+swarm-status:
+	@docker service ls
+	@echo "---"
+	@docker service ps validate_worker
+
+# Add a worker node to the Swarm cluster
+# Usage: make add-worker NAME=dev-vm-worker-1 ENV_FILE=.env.DEV_SWARM
+# Reads SWARM_WORKER_N entries and SWARM_SSH_USER from ENV_FILE
+add-worker:
+	@test -n "$(NAME)" || (echo "Usage: make add-worker NAME=<worker-name> ENV_FILE=.env.DEV_SWARM" && exit 1)
+	$(eval SSH_USER := $(shell grep '^SWARM_SSH_USER=' $(ENV_FILE) | head -1 | cut -d= -f2-))
+	$(eval MANAGER_IP := $(shell grep '^NFS_SERVER_IP=' $(ENV_FILE) | head -1 | cut -d= -f2-))
+	$(eval WORKER_IP := $(shell grep '^SWARM_WORKER_' $(ENV_FILE) | grep '$(NAME)' | head -1 | cut -d: -f2))
+	@test -n "$(WORKER_IP)" || (echo "ERROR: Worker '$(NAME)' not found in $(ENV_FILE). Add it as: SWARM_WORKER_N=$(NAME):<ip>" && exit 1)
+	@test -n "$(MANAGER_IP)" || (echo "ERROR: NFS_SERVER_IP not set in $(ENV_FILE)" && exit 1)
+	@test -n "$(SSH_USER)" || (echo "ERROR: SWARM_SSH_USER not set in $(ENV_FILE)" && exit 1)
+	@echo "==> Installing Docker on $(NAME) ($(WORKER_IP))..."
+	sudo -u $(SSH_USER) ssh -o StrictHostKeyChecking=no $(SSH_USER)@$(WORKER_IP) "curl -fsSL https://get.docker.com | sh"
+	@echo "==> Configuring insecure registry ($(MANAGER_IP):5000)..."
+	sudo -u $(SSH_USER) ssh -o StrictHostKeyChecking=no $(SSH_USER)@$(WORKER_IP) 'echo '"'"'{ "insecure-registries": ["$(MANAGER_IP):5000"] }'"'"' | sudo tee /etc/docker/daemon.json && sudo systemctl restart docker'
+	@echo "==> Joining Swarm..."
+	sudo -u $(SSH_USER) ssh -o StrictHostKeyChecking=no $(SSH_USER)@$(WORKER_IP) "sudo docker swarm join --token $$(sudo docker swarm join-token worker -q) $(MANAGER_IP):2377"
+	@echo "==> Done! Node list:"
+	sudo docker node ls
+
+# Remove a worker node from the Swarm cluster
+# Usage: make remove-worker NAME=dev-vm-worker-1 ENV_FILE=.env.DEV_SWARM
+remove-worker:
+	@test -n "$(NAME)" || (echo "Usage: make remove-worker NAME=dev-vm-worker-1 ENV_FILE=.env.DEV_SWARM" && exit 1)
+	$(eval SSH_USER := $(shell grep '^SWARM_SSH_USER=' $(ENV_FILE) | head -1 | cut -d= -f2-))
+	$(eval WORKER_IP := $(shell grep '^SWARM_WORKER_' $(ENV_FILE) | grep '$(NAME)' | head -1 | cut -d: -f2))
+	@echo "==> Draining $(NAME)..."
+	sudo docker node update --availability drain $(NAME)
+	@echo "==> Leaving swarm..."
+	-sudo -u $(SSH_USER) ssh -o StrictHostKeyChecking=no $(SSH_USER)@$(WORKER_IP) "sudo docker swarm leave"
+	@echo "==> Waiting for node to go down..."
+	@for i in 1 2 3 4 5 6; do sleep 5; sudo docker node ls --format '{{.Hostname}} {{.Status}}' | grep -q '$(NAME) Down' && break; echo "    waiting..."; done
+	@echo "==> Removing node..."
+	sudo docker node rm $(NAME)
+	@echo "==> Done! Don't forget to remove the SWARM_WORKER entry from $(ENV_FILE)"
+	sudo docker node ls
+
 build:
 	docker compose build \
 	--build-arg GIT_COMMIT_HASH="$$(git rev-parse --short HEAD)" \
@@ -83,7 +165,7 @@ e2e-test: start-infra
 	cd e2e && npm install && npm run install-playwright && npm run test
 
 e2e-test-report: start-infra
-	cd e2e && npm install && npm run install-playwright && npm run test:html && npm run test:report
+	cd e2e && npm install && npm run inst1all-playwright && npm run test:html && npm run test:report
 
 BRANCH   ?= main                 
 SUBTREES := \

backend/apps/ifc_validation/api/v1/views.py

@@ -195,6 +195,7 @@ def post(self, request, *args, **kwargs):
                     #file = os.path.join(MEDIA_ROOT, uploaded_file['file_name'])                   
                     #uploaded_file['size'] = os.path.getsize(file)
                     uploaded_file['size'] = file_length
+                    f.seek(0)
                     instance = serializer.save()
 
                     # submit task for background execution

backend/apps/ifc_validation/tasks/processing/instance_completion.py

@@ -18,7 +18,7 @@
     import itertools
     import functools
 
-    file_path, step_ids = file_path, step_ids = json.load(sys.stdin)
+    file_path, step_ids = json.load(sys.stdin)
     ifc_file = ifcopenshell.open(file_path)
     def filter_serializable(v):
         def inner(k, v):

backend/core/settings.py

@@ -246,8 +246,10 @@
         "USER": os.environ.get("POSTGRES_USER", "postgres"),
         "PASSWORD": os.environ.get("POSTGRES_PASSWORD", "postgres"),
         "PORT": int(os.environ.get("POSTGRES_PORT", "5432")),
+        "CONN_MAX_AGE": int(os.environ.get("POSTGRES_CONN_MAX_AGE", 600)),
+        "CONN_HEALTH_CHECKS": True,
         "OPTIONS": {
-            "pool": True,
+            "pool": False,
         },
     },
 }

docker-compose.swarm.local.yml

@@ -0,0 +1,67 @@
+# Override: single-node local testing (no NFS, no ClamAV, reduced replicas)
+#
+# Usage:
+#   docker stack deploy -c docker-compose.swarm.yml -c docker-compose.swarm.local.yml validate
+#
+# For production/NFS testing, use docker-compose.swarm.yml directly.
+
+services:
+
+  frontend:
+    environment:
+      CERTBOT_DOMAIN: _
+      CERTBOT_EMAIL: x
+
+  backend:
+    deploy:
+      replicas: 1
+
+  worker:
+    entrypoint: /bin/sh
+    command:
+      - -c
+      - |
+        set -e
+        until cd /files_storage; do echo "Waiting for files_storage..."; done
+        until cd /app/backend; do echo "Waiting for server volume..."; done
+        while ! pg_isready -h "$$POSTGRES_HOST" -p "$$POSTGRES_PORT" -d "$$POSTGRES_NAME" -U "$$POSTGRES_USER" 2>/dev/null; do
+          echo "Waiting for DB..."
+          sleep 5
+        done
+        echo "DB is ready. Starting worker (no ClamAV)."
+        rm -f /usr/bin/clamdscan /usr/bin/clamscan 2>/dev/null || true
+        CELERY_CONCURRENCY=$${CELERY_CONCURRENCY:-4}
+        echo "Celery concurrency: $$CELERY_CONCURRENCY"
+        celery --app=core worker --loglevel=info --concurrency $$CELERY_CONCURRENCY --task-events --hostname=worker@%n
+    deploy:
+      replicas: 1
+      resources:
+        limits:
+          cpus: "2.0"
+          memory: 2G
+        reservations:
+          cpus: "0.5"
+          memory: 512M
+
+  scheduler:
+    entrypoint: /bin/sh
+    command:
+      - -c
+      - |
+        set -e
+        until cd /files_storage; do echo "Waiting for files_storage..."; done
+        until cd /app/backend; do echo "Waiting for server volume..."; done
+        while ! pg_isready -h "$$POSTGRES_HOST" -p "$$POSTGRES_PORT" -d "$$POSTGRES_NAME" -U "$$POSTGRES_USER" 2>/dev/null; do
+          echo "Waiting for DB..."
+          sleep 5
+        done
+        echo "DB is ready. Starting scheduler (no ClamAV)."
+        rm -f /usr/bin/clamdscan /usr/bin/clamscan 2>/dev/null || true
+        CELERY_CONCURRENCY=$${CELERY_CONCURRENCY:-4}
+        celery --app=core worker --beat --loglevel=info --concurrency $$CELERY_CONCURRENCY --task-events --hostname=worker-beat@%n
+
+volumes:
+  files_data:
+    driver: local
+  gherkin_rules_log_data:
+    driver: local