Cleanups

.github/workflows/smoke-nifi-services.yml

@@ -33,9 +33,7 @@ jobs:
       - name: Start NiFi services (build)
         run: |
           set -euo pipefail
-          source deploy/export_env_vars.sh
-          set -euo pipefail
-          docker compose -f deploy/services.dev.yml up -d --build nifi nifi-nginx nifi-registry-flow
+          make -C deploy start-nifi-dev-build
 
       - name: Smoke tests
         run: |
@@ -44,7 +42,7 @@ jobs:
           retries=30
           delay=15
           for attempt in $(seq 1 $retries); do
-            if ./scripts/smoke_nifi_services.sh; then
+            if ./scripts/tests/smoke_nifi_services.sh; then
               exit 0
             fi
             echo "Attempt ${attempt}/${retries} failed. Sleeping ${delay}s..."

deploy/Makefile

@@ -25,17 +25,23 @@ load-env:
 
 show-env:
 	${WITH_ENV} >/dev/null 2>&1; printenv | sort
+
+fix-nifi-registry-perms:
+	$(WITH_ENV) SKIP_EXPORT_ENV=1 ../nifi/fix_nifi_registry_perms.sh $(COMPOSE_FILE)
 
 # start services
 
 start-nifi:
-	$(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) nifi nifi-nginx nifi-registry-flow
+	$(WITH_ENV) SKIP_EXPORT_ENV=1 ../nifi/fix_nifi_registry_perms.sh services.yml; \
+	docker compose -f services.yml $(DC_START_CMD) nifi nifi-nginx nifi-registry-flow
 
 start-nifi-dev:
-	$(WITH_ENV) docker compose -f services.dev.yml $(DC_START_CMD) nifi nifi-nginx nifi-registry-flow
+	$(WITH_ENV) SKIP_EXPORT_ENV=1 ../nifi/fix_nifi_registry_perms.sh services.dev.yml; \
+	docker compose -f services.dev.yml $(DC_START_CMD) nifi nifi-nginx nifi-registry-flow
 
 start-nifi-dev-build:
-	$(WITH_ENV) docker compose -f services.dev.yml up -d --build nifi nifi-nginx nifi-registry-flow
+	$(WITH_ENV) SKIP_EXPORT_ENV=1 ../nifi/fix_nifi_registry_perms.sh services.dev.yml; \
+	docker compose -f services.dev.yml up -d --build nifi nifi-nginx nifi-registry-flow
 
 start-elastic:
 	$(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) elasticsearch-1 elasticsearch-2 kibana
@@ -101,7 +107,7 @@ start-data-infra: start-nifi start-elastic start-samples
 
 start-all: start-data-infra start-jupyter start-medcat-service start-ocr-services
 
-.PHONY: start-all start-data-infra start-nifi start-nifi-dev start-nifi-dev-build start-elastic start-samples start-jupyter
+.PHONY: start-all start-data-infra start-nifi start-nifi-dev start-nifi-dev-build start-elastic start-samples start-jupyter fix-nifi-registry-perms
 
 
 # stop services
@@ -112,6 +118,28 @@ stop-nifi:
 stop-nifi-dev:
 	$(WITH_ENV) docker compose -f services.dev.yml $(DC_STOP_CMD) nifi nifi-nginx nifi-registry-flow
 
+delete-nifi-containers:
+	$(WITH_ENV) docker compose -f services.yml rm -f -s nifi nifi-nginx nifi-registry-flow
+
+delete-nifi-dev-containers:
+	$(WITH_ENV) docker compose -f services.dev.yml rm -f -s nifi nifi-nginx nifi-registry-flow
+
+delete-nifi-images:
+	$(WITH_ENV) images="$$(docker compose -f services.yml config --images nifi nifi-nginx nifi-registry-flow | sort -u)"; \
+	if [ -n "$$images" ]; then \
+		docker image rm -f $$images; \
+	else \
+		echo "No NiFi images found in services.yml"; \
+	fi
+
+delete-nifi-dev-images:
+	$(WITH_ENV) images="$$(docker compose -f services.dev.yml config --images nifi nifi-nginx nifi-registry-flow | sort -u)"; \
+	if [ -n "$$images" ]; then \
+		docker image rm -f $$images; \
+	else \
+		echo "No NiFi images found in services.dev.yml"; \
+	fi
+
 stop-elastic:
 	$(WITH_ENV) docker compose -f services.yml $(DC_STOP_CMD) elasticsearch-1 elasticsearch-2 kibana
 
@@ -173,7 +201,7 @@ stop-data-infra: stop-nifi stop-elastic stop-samples
 
 stop-all: stop-data-infra stop-jupyter stop-medcat-service stop-ocr-services
 
-.PHONY: stop-data-infra stop-nifi stop-nifi-dev stop-elastic stop-samples stop-jupyter
+.PHONY: stop-data-infra stop-nifi stop-nifi-dev delete-nifi-containers delete-nifi-dev-containers delete-nifi-images delete-nifi-dev-images stop-elastic stop-samples stop-jupyter
 
 
 # cleanup

deploy/elasticsearch.env

@@ -7,7 +7,7 @@
 ELASTICSEARCH_VERSION=opensearch
 
 # possible values :
-#   - elasticsearch                         : docker.elastic.co/elasticsearch/elasticsearch:8.18.2
+#   - elasticsearch                         : docker.elastic.co/elasticsearch/elasticsearch:8.19.11
 #   - elasticsearch (custom cogstack image) : cogstacksystems/cogstack-elasticsearch:latest 
 #   - opensearch                            : opensearchproject/opensearch:3.4.0
 # the custom cogstack image is always based on the last image of ES native 
@@ -89,6 +89,8 @@ ELASTICSEARCH_SECURITY_DIR=../security/certificates/elastic/
 
 # MEMORY CONFIG
 ELASTICSEARCH_JAVA_OPTS="-Xms512m -Xmx512m -Des.failure_store_feature_flag_enabled=true"
+ES_JAVA_OPTS=$ELASTICSEARCH_JAVA_OPTS
+OPENSEARCH_JAVA_OPTS=$ELASTICSEARCH_JAVA_OPTS
 
 ELASTICSEARCH_DOCKER_CPU_MIN=1
 ELASTICSEARCH_DOCKER_CPU_MAX=1
@@ -163,7 +165,7 @@ KIBANA_VERSION=opensearch-dashboards
 KIBANA_CONFIG_FILE_VERSION=opensearch_dashboards
 
 # possible values:
-#   - elasticsearch                         : docker.elastic.co/kibana/kibana:8.18.2
+#   - elasticsearch                         : docker.elastic.co/kibana/kibana:8.19.11
 #   - elasticsearch (custom cogstack image) : cogstacksystems/cogstack-kibana:latest
 #   - opensearch                            : opensearchproject/opensearch-dashboards:3.4.0
 # the custom cogstack image is always based on the last image of ES native 
@@ -205,7 +207,7 @@ ELASTICSEARCH_XPACK_SECURITY_REPORTING_ENCRYPTION_KEY="e0Y1gTxHWOopIWMTtpjQsDS6K
 
 ######################################################################### METRICBEAT Env vars ##########################################################################
 
-METRICBEAT_IMAGE="docker.elastic.co/beats/metricbeat:8.18.2"
+METRICBEAT_IMAGE="docker.elastic.co/beats/metricbeat:8.19.11"
 
 METRICBEAT_DOCKER_SHM=512m
 METRICBEAT_DOCKER_CPU_MIN=1
@@ -222,7 +224,7 @@ FILEBEAT_STARTUP_COMMAND="-e --strict.perms=false"
 
 FILEBEAT_HOST="https://elasticsearch-1:9200"
 
-FILEBEAT_IMAGE="docker.elastic.co/beats/filebeat:8.18.2"
+FILEBEAT_IMAGE="docker.elastic.co/beats/filebeat:8.19.11"
 
 
 FILEBEAT_DOCKER_SHM=512m

deploy/export_env_vars.sh

@@ -41,22 +41,6 @@ env_files=(
   "$SERVICES_DIR/cogstack-nlp/medcat-service/env/medcat.env"
 )
 
-LINT_SCRIPT="$SCRIPT_DIR/../nifi/user_scripts/utils/lint_env.py"
-
-if [ -e "$LINT_SCRIPT" ]; then
-    chmod +x $LINT_SCRIPT
-fi
-
-if [ -x "$LINT_SCRIPT" ]; then
-  echo "🔍 Validating env files..."
-  if ! python3 "$LINT_SCRIPT" "${env_files[@]}"; then
-    echo "❌ Env validation failed. Fix the errors above before continuing."
-    exit 1
-  fi
-else
-  echo "⚠️  Skipping env validation; $LINT_SCRIPT not found or not executable."
-fi
-
 for env_file in "${env_files[@]}"; do
   if [ -f "$env_file" ]; then
     echo "✅ Sourcing $env_file"

deploy/network_settings.env

@@ -18,4 +18,4 @@ HTTP_PROXY=""
 NO_PROXY=""
 no_proxy=""
 http_proxy=""
-https_proxy=""
+https_proxy=""

deploy/nifi.env

@@ -41,6 +41,9 @@ NIFI_DATA_PATH="../data/"
 
 NIFI_TOOLKIT_VERSION=$NIFI_VERSION
 
+# this is to mount medcat models (optional)
+NIFI_MEDCAT_SERVICE_MODEL_PRODUCTION_PATH="../services/cogstack-nlp/medcat-service/models/"
+
 #### Port and network settings
 NIFI_WEB_PROXY_CONTEXT_PATH="/nifi"
 

deploy/services.dev.yml

@@ -63,11 +63,11 @@ x-nifi-common: &nifi-common
 
 x-nifi-volumes: &nifi-volumes
   # Drivers
-  - ../nifi/drivers:/opt/nifi/drivers
+  - ../nifi/drivers:/opt/nifi/drivers:ro
 
   # User overrides bundled in the image
   - ../nifi/user_scripts:/opt/nifi/user_scripts:rw
-  - ../nifi/user_schemas:/opt/nifi/user_schemas:rw
+  - ../nifi/user_schemas:/opt/nifi/user_schemas:ro
 
   # Python processors (NiFi 2.x)
   - ../nifi/user_python_extensions:/opt/nifi/nifi-current/python_extensions:rw
@@ -84,9 +84,6 @@ x-nifi-volumes: &nifi-volumes
   # Ingest data directory
   - ./${NIFI_DATA_PATH:-../data/}:/data/:rw
 
-  # DB schemas
-  - ../services/cogstack-db/:/opt/cogstack-db/:rw
-
   # MedCAT models
   - ./${RES_MEDCAT_SERVICE_MODEL_PRODUCTION_PATH:-../services/cogstack-nlp/medcat-service/models/}:/opt/models:rw
 
@@ -127,21 +124,10 @@ services:
     build:
        context: ..
        dockerfile: nifi/Dockerfile
-       args:
-           HTTP_PROXY: $HTTP_PROXY
-           HTTPS_PROXY: $HTTPS_PROXY
-           no_proxy: $no_proxy
     container_name: cogstack-nifi
     hostname: nifi
     shm_size: ${NIFI_DOCKER_SHM_SIZE:-"1g"}
     environment:
-      - USER_ID=${NIFI_UID:-1000}
-      - GROUP_ID=${NIFI_GID:-1000}
-      - NIFI_WEB_PROXY_HOST=${NIFI_WEB_PROXY_HOST:-"localhost:8443"}
-      - NIFI_WEB_PROXY_CONTEXT_PATH=${NIFI_WEB_PROXY_CONTEXT_PATH:-"/nifi"}
-      - NIFI_INTERNAL_PORT=${NIFI_INTERNAL_PORT:-8443}
-      - NIFI_OUTPUT_PORT=${NIFI_OUTPUT_PORT:-8082}
-      - NIFI_INPUT_SOCKET_PORT=${NIFI_INPUT_SOCKET_PORT:-10000}
       - PYTHONPATH=${NIFI_PYTHONPATH:-/opt/nifi/nifi-current/python/framework}
       - JVM_OPTS="${NIFI_JVM_OPTS:--XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+ParallelRefProcEnabled -Djava.security.egd=file:/dev/./urandom}"
     deploy:
@@ -170,13 +156,7 @@ services:
     hostname: nifi-registry
     container_name: cogstack-nifi-registry-flow
     shm_size: ${NIFI_DOCKER_REGISTRY_SHM_SIZE:-1g}
-    user: root
     environment:
-      - http_proxy=$HTTP_PROXY
-      - https_proxy=$HTTPS_PROXY
-      - no_proxy=$no_proxy
-      - USER_ID=${NIFI_UID:-1000}
-      - GROUP_ID=${NIFI_GID:-1000}
       - KEYSTORE_PATH=${NIFI_REGISTRY_KEYSTORE_PATH:-/security/certificates/nifi/nifi-keystore.jks}
       - KEYSTORE_TYPE=${NIFI_KEYSTORE_TYPE:-jks}
       - KEYSTORE_PASSWORD=${NIFI_KEYSTORE_PASSWORD:-"cogstackNifi"}
@@ -186,9 +166,6 @@ services:
       - TRUSTSTORE_TYPE=${NIFI_TRUSTSTORE_TYPE:-jks}
       - INITIAL_ADMIN_IDENTITY=${NIFI_INITIAL_ADMIN_IDENTITY:-"cogstack"}
       - AUTH=${NIFI_AUTH:-"tls"}
-      - NIFI_REGISTRY_DB_DIR=${NIFI_REGISTRY_DB_DIR:-/opt/nifi-registry/nifi-registry-current/database}
-      #- NIFI_REGISTRY_FLOW_PROVIDER=${NIFI_REGISTRY_FLOW_PROVIDER:-file}
-      - NIFI_REGISTRY_FLOW_STORAGE_DIR=${NIFI_REGISTRY_FLOW_STORAGE_DIR:-/opt/nifi-registry/nifi-registry-current/flow_storage}
     deploy:
       resources:
         limits:
@@ -203,11 +180,6 @@ services:
     ports:
       - "${NIFI_REGISTRY_FLOW_OUTPUT_PORT:-8083}:${NIFI_REGISTRY_FLOW_INPUT_PORT:-18443}"
 
-    entrypoint: bash -c "chown -R nifi:nifi /opt/nifi-registry/nifi-registry-current/database && \
-      chown -R nifi:nifi /opt/nifi-registry/nifi-registry-current/flow_storage && \
-      chown -R nifi:nifi /opt/nifi-registry/nifi-registry-current/work && \
-      chown -R nifi:nifi /opt/nifi-registry/nifi-registry-current/logs && \
-      bash /opt/nifi-registry/scripts/start.sh"
     logging: *nifi-logging-common
 
   nifi-nginx: