From daee08d145728d3d39df4b0498686134d169c99e Mon Sep 17 00:00:00 2001 From: fe51 <55736935+fe51@users.noreply.github.com> Date: Mon, 2 Mar 2026 21:35:01 +0100 Subject: [PATCH] refacto init_script container and add pose image at init --- containers/init_script/Dockerfile | 10 +- containers/init_script/init_script.py | 213 ++++++++++++++---- containers/init_script/requirements.txt | 6 + containers/init_script/utils.py | 124 ++++++++++ containers/notebooks/app/camera_status.ipynb | 2 +- .../notebooks/app/continuous_alert.ipynb | 2 +- .../notebooks/app/send_real_alerts.ipynb | 7 + 7 files changed, 314 insertions(+), 50 deletions(-) create mode 100644 containers/init_script/requirements.txt create mode 100644 containers/init_script/utils.py diff --git a/containers/init_script/Dockerfile b/containers/init_script/Dockerfile index 6ec887b..6c1dc33 100644 --- a/containers/init_script/Dockerfile +++ b/containers/init_script/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8.16-slim +FROM python:3.11-slim # hadolint ignore=DL3008 RUN apt-get update \ @@ -6,11 +6,15 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +# Copy requirements file +COPY requirements.txt /tmp/ + # hadolint ignore=DL3013 -RUN pip install --no-cache-dir pandas python-dotenv==1.0.1 boto3==1.34.90 requests +RUN pip install --no-cache-dir -r /tmp/requirements.txt -# Copy your initialization script into the container +# Copy initialization scripts into the container COPY init_script.py /usr/local/bin/ +COPY utils.py /usr/local/bin/ # Set execute permission on the script RUN chmod +x /usr/local/bin/init_script.py diff --git a/containers/init_script/init_script.py b/containers/init_script/init_script.py index 2bcf2d4..94f0594 100644 --- a/containers/init_script/init_script.py +++ b/containers/init_script/init_script.py @@ -1,59 +1,34 @@ #  TODO REFACTOR : use the init scripts which are in the test_update_pi directory #!/usr/bin/env python - -from typing import Any, Dict, Optional -import pandas as pd import sys import logging import os +import glob +import itertools +import random +import io +import pandas as pd import requests -import json +from PIL import Image +from pyroclient import Client + +# Import utility functions +from utils import ( + get_token, + api_request, + read_json_file, + write_json_file, + download_images_if_needed, +) logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s") +# Base URL for Client initialization (like in notebook: "http://api:5050") +base_url = os.environ.get("API_URL") +# API URL with /api/v1 for direct API requests +api_url = base_url + "/api/v1" -def get_token(api_url: str, login: str, pwd: str) -> str: - response = requests.post( - f"{api_url}/login/creds", - data={"username": login, "password": pwd}, - timeout=5, - ) - if response.status_code != 200: - raise ValueError(response.json()["detail"]) - return response.json()["access_token"] - - -def api_request( - method_type: str, - route: str, - headers=Dict[str, str], - payload: Optional[Dict[str, Any]] = None, -): - kwargs = {"json": payload} if isinstance(payload, dict) else {} - - response = getattr(requests, method_type)(route, headers=headers, **kwargs) - try: - detail = response.json() - except (requests.exceptions.JSONDecodeError, KeyError): - detail = response.text - assert response.status_code // 100 == 2, print(detail) - return response.json() - - -# Function to read JSON data from a file -def read_json_file(file_path): - with open(file_path, "r") as file: - return json.load(file) - - -# Function to write JSON data to a file -def write_json_file(file_path, data): - with open(file_path, "w") as file: - json.dump(data, file, indent=4) - - -api_url = os.environ.get("API_URL") + "/api/v1" superuser_login = os.environ.get("SUPERADMIN_LOGIN") superuser_pwd = os.environ.get("SUPERADMIN_PWD") slack_hook = os.environ.get("SLACK_HOOK") @@ -74,6 +49,10 @@ def write_json_file(file_path, data): cameras = cameras.fillna("") poses = pd.read_csv(f"data/csv/API_DATA{sub_path} - poses.csv") +# ============================================================================ +# ORGAS CREATION +# ============================================================================ + for orga in organizations.itertuples(index=False): logging.info(f"saving orga : {orga.name}") payload = {"name": orga.name} @@ -90,6 +69,9 @@ def write_json_file(file_path, data): payload, ) +# ============================================================================ +# USERS CREATION +# ============================================================================ for user in users.itertuples(index=False): logging.info(f"saving user : {user.login}") @@ -105,6 +87,10 @@ def write_json_file(file_path, data): data = read_json_file(credentials_path) data_wildfire = read_json_file(credentials_path_etl) +# ============================================================================ +# CAMERAS CREATION +# ============================================================================ + for camera in cameras.itertuples(index=False): logging.info(f"saving camera : {camera.name}") payload = { @@ -132,6 +118,10 @@ def write_json_file(file_path, data): write_json_file(credentials_path, data) write_json_file(credentials_path_etl, data_wildfire) +# ============================================================================ +# POSES CREATION +# ============================================================================ + logging.info("creating poses") for pose in poses.itertuples(index=False): payload = { @@ -142,6 +132,139 @@ def write_json_file(file_path, data): api_request("post", f"{api_url}/poses/", superuser_auth, payload) +# ============================================================================ +# CAMERA STATUS UPDATES AND POSE IMAGE UPLOADS +# ============================================================================ + +# Constants +BASE_DIRECTORY = "data" +SAMPLE_PATH = "last_image_cameras" +IMAGES_URL = "https://github.com/pyronear/pyro-envdev/releases/download/v0.0.1/last_image_cameras.zip" + +# Predefined list of image filenames for pose images +# (to be filled with actual filenames) +POSE_IMAGE_FILES = [ + "11-20251001151013-d6de7b82.jpg", + "17-20251001143249-dc04ad6f.jpg", + "22-20251001150858-6578cf9e.jpg", + "59-20251001151203-354621da.jpg", + "69-20251003170645-721ec72e.jpg", +] + +# Cameras to skip for pose images (one per organization) +# Format: {organization_id: camera_id_to_skip} +CAMERAS_TO_SKIP_POSE_IMAGES = { + 2: 8, # Organization 2: skip camera 1 (videlles-01) + 3: 14, # Organization 3: skip camera 9 (brison-01) +} + +# Download images if needed +logging.info("Checking for image files...") +download_images_if_needed(BASE_DIRECTORY, SAMPLE_PATH, IMAGES_URL) + +# Get all image files +image_dir = os.path.join(BASE_DIRECTORY, SAMPLE_PATH) +all_images = glob.glob(os.path.join(image_dir, "*.jpg")) + +if not all_images: + logging.warning( + "No images found in directory, skipping camera and pose image updates" + ) +else: + # Cycle images for camera last image updates + images_cycle = itertools.cycle(all_images) + + # Prepare pose images + pose_images_full_paths = [ + os.path.join(image_dir, fname) + for fname in POSE_IMAGE_FILES + if os.path.exists(os.path.join(image_dir, fname)) + ] + + if not pose_images_full_paths: + logging.warning("No valid pose images found in predefined list") + + logging.info("Updating cameras (heartbeat, last image) and pose images...") + + # Create admin client once for all operations + admin_token = superuser_auth["Authorization"].replace("Bearer ", "") + admin_client = Client(admin_token, base_url) + + # Fetch all cameras using pyroclient + cameras_response = admin_client.fetch_cameras().json() + + # Process each camera + for camera_data in cameras_response: + camera_id = camera_data["id"] + camera_name = camera_data["name"] + org_id = camera_data["organization_id"] + + logging.info(f"Processing camera {camera_name} (id: {camera_id})...") + + try: + result = api_request( + "post", f"{api_url}/cameras/{camera_id}/token", superuser_auth + ) + camera_token = result["access_token"] + + camera_client = Client(camera_token, base_url) + + # Update heartbeat + camera_client.heartbeat() + logging.info(" ✓ Heartbeat updated") + + # Update camera last image + img_file = next(images_cycle) + stream = io.BytesIO() + im = Image.open(img_file) + im.save(stream, format="JPEG", quality=80) + stream.seek(0) + camera_client.update_last_image(stream.getvalue()) + logging.info(" ✓ Last image updated") + + # Fetch camera details including poses + camera_details = requests.get( + f"{api_url}/cameras/{camera_id}", headers=superuser_auth, timeout=10 + ).json() + + camera_poses = camera_details.get("poses", []) + + if not camera_poses: + logging.info(" No poses for this camera") + elif CAMERAS_TO_SKIP_POSE_IMAGES.get(org_id) == camera_id: + logging.info(f" Skipping pose images (org {org_id} skip rule)") + elif not pose_images_full_paths: + logging.info(" No pose images available") + else: + # Upload pose images + num_poses = len(camera_poses) + num_images = min(num_poses, len(pose_images_full_paths)) + selected_images = random.sample(pose_images_full_paths, num_images) + + logging.info(f" Uploading {num_images} pose images...") + for pose_data, image_path in zip(camera_poses, selected_images): + try: + with open(image_path, "rb") as f: + image_data = f.read() + + # time.sleep(2) + admin_client.update_pose_image(pose_data["id"], image_data) + logging.info( + f"✓ Pose {pose_data['id']} (azimuth {pose_data['azimuth']})" + ) + except Exception as e: + logging.error(f" ✗ Pose {pose_data['id']}: {e}") + + logging.info(f" Completed camera {camera_name}") + + except Exception as e: + logging.error(f" Error processing camera {camera_name}: {e}") + continue + + logging.info("All camera and pose updates completed") + +logging.info("Initialization script completed successfully") + # Load environment variables from .env file # load_dotenv() diff --git a/containers/init_script/requirements.txt b/containers/init_script/requirements.txt new file mode 100644 index 0000000..0242cea --- /dev/null +++ b/containers/init_script/requirements.txt @@ -0,0 +1,6 @@ +pandas +python-dotenv==1.0.1 +boto3==1.34.90 +requests +pillow +pyroclient @ git+https://github.com/pyronear/pyro-api.git@main#subdirectory=client diff --git a/containers/init_script/utils.py b/containers/init_script/utils.py new file mode 100644 index 0000000..11b7b32 --- /dev/null +++ b/containers/init_script/utils.py @@ -0,0 +1,124 @@ +# Copyright (C) 2020-2026, Pyronear. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to +# for full license details. + +""" +Utility functions for initialization script. +Provides helper functions for API communication, file I/O, and image management. +""" + +from typing import Any, Dict, Optional +import os +import io +import json +import logging +import zipfile +import requests + + +def get_token(api_url: str, login: str, pwd: str) -> str: + """ + Authenticate with API and return access token. + + Args: + api_url: Base API URL (e.g., "http://api:5050/api/v1") + login: Username + pwd: Password + + Returns: + Access token string + """ + response = requests.post( + f"{api_url}/login/creds", + data={"username": login, "password": pwd}, + timeout=5, + ) + if response.status_code != 200: + raise ValueError(response.json()["detail"]) + return response.json()["access_token"] + + +def api_request( + method_type: str, + route: str, + headers: Dict[str, str], + payload: Optional[Dict[str, Any]] = None, +): + """ + Make a generic API request. + + Args: + method_type: HTTP method (get, post, patch, delete) + route: Full API route URL + headers: Request headers including authorization + payload: Optional request payload + + Returns: + JSON response data + """ + kwargs = {"json": payload} if isinstance(payload, dict) else {} + + response = getattr(requests, method_type)(route, headers=headers, **kwargs) + try: + detail = response.json() + except (requests.exceptions.JSONDecodeError, KeyError): + detail = response.text + assert response.status_code // 100 == 2, print(detail) + return response.json() + + +def read_json_file(file_path: str) -> Dict[str, Any]: + """ + Read JSON data from a file. + + Args: + file_path: Path to JSON file + + Returns: + Parsed JSON data as dictionary + """ + with open(file_path, "r") as file: + return json.load(file) + + +def write_json_file(file_path: str, data: Dict[str, Any]) -> None: + """ + Write JSON data to a file. + + Args: + file_path: Path to output file + data: Data to write as JSON + """ + with open(file_path, "w") as file: + json.dump(data, file, indent=4) + + +def download_images_if_needed(base_directory: str, sample_path: str, url: str) -> None: + """ + Download and extract images if directory doesn't exist. + + Args: + base_directory: Base directory for images (e.g., "data") + sample_path: Subdirectory name (e.g., "last_image_cameras") + url: URL to download zip file from + """ + full_path = os.path.join(base_directory, sample_path) + if not os.path.isdir(full_path): + logging.info(f"Images not found at {full_path}, downloading...") + + # Download the zip file + response = requests.get(url, timeout=30) + response.raise_for_status() + + # Create parent directory if it doesn't exist + os.makedirs(base_directory, exist_ok=True) + + # Extract the zip file + zip_content = zipfile.ZipFile(io.BytesIO(response.content)) + zip_content.extractall(base_directory) + + logging.info(f"Images downloaded and extracted to {full_path}") + else: + logging.info(f"Directory {sample_path} exists, skipping download") diff --git a/containers/notebooks/app/camera_status.ipynb b/containers/notebooks/app/camera_status.ipynb index 162567a..400b348 100644 --- a/containers/notebooks/app/camera_status.ipynb +++ b/containers/notebooks/app/camera_status.ipynb @@ -122,7 +122,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.7" + "version": "3.11.14" } }, "nbformat": 4, diff --git a/containers/notebooks/app/continuous_alert.ipynb b/containers/notebooks/app/continuous_alert.ipynb index 2f668b4..634bac0 100644 --- a/containers/notebooks/app/continuous_alert.ipynb +++ b/containers/notebooks/app/continuous_alert.ipynb @@ -131,7 +131,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.20" + "version": "3.11.14" } }, "nbformat": 4, diff --git a/containers/notebooks/app/send_real_alerts.ipynb b/containers/notebooks/app/send_real_alerts.ipynb index 4d22449..07f3bb5 100644 --- a/containers/notebooks/app/send_real_alerts.ipynb +++ b/containers/notebooks/app/send_real_alerts.ipynb @@ -336,6 +336,13 @@ "send_triangulated_alerts(cam_triangulation, API_URL, Client, admin_access_token)\n" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null,