From 12a43ad8e76fbe454a4418f3f75bbec90ea113dc Mon Sep 17 00:00:00 2001 From: MAULIK MK <214461757+maulik-mk@users.noreply.github.com> Date: Tue, 17 Feb 2026 18:32:25 +0530 Subject: [PATCH 01/12] chore: Add initial project setup with package.json and update .gitignore (#1) --- .gitignore | 4 ++++ package.json | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 package.json diff --git a/.gitignore b/.gitignore index 9a5aced..c0e2b02 100644 --- a/.gitignore +++ b/.gitignore @@ -137,3 +137,7 @@ dist # Vite logs files vite.config.js.timestamp-* vite.config.ts.timestamp-* + +.DS_Store +video +output \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..a196956 --- /dev/null +++ b/package.json @@ -0,0 +1,27 @@ +{ + "name": "worker", + "version": "0.0.0", + "description": "FFmpeg worker", + "repository": { + "type": "git", + "url": "git+https://github.com/maulik-mk/mp.ii-worker.git" + }, + "keywords": [ + "ffmpeg", + "worker", + "video processing", + "video encoding", + "video decoding", + "video transcoding", + "video streaming", + "video processing worker", + "video processing worker", + "nodjs" + ], + "author": "Maulik MK", + "license": "ISC", + "bugs": { + "url": "https://github.com/maulik-mk/mp.ii-worker/issues" + }, + "homepage": "https://github.com/maulik-mk/mp.ii-worker#readme" +} From 056690ab218283a8715a1c9b5cb0b69605d93a54 Mon Sep 17 00:00:00 2001 From: MAULIK MK <214461757+maulik-mk@users.noreply.github.com> Date: Tue, 17 Feb 2026 19:01:06 +0530 Subject: [PATCH 02/12] feat: Implement CI pipeline for building and testing the FFmpeg Docker image (#2) --- .github/workflows/ci.yml | 67 +++++++++++ Dockerfile | 130 ++++++++++++++++++++++ test/ffmpeg.test.sh | 232 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 429 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 Dockerfile create mode 100644 test/ffmpeg.test.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e7c64a8 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,67 @@ +name: CI + +on: + push: + branches: [ "main", "master"] + pull_request: + branches: [ "main", "master"] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-and-test: + name: Build & Test (Ubuntu) + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + steps: + # 1. Checkout the repository code + - name: Checkout Code + uses: actions/checkout@v4 + + # 2. Log in to GitHub Container Registry + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GHCR_TOKEN }} + + # 3. Download a sample test video (1080p, ~2MB) + # Dynamically to keep the git repo small. + # The script expects 'video', so we simulate that structure. + - name: Download Test Asset + run: | + mkdir -p video + echo "Downloading sample video..." + # Using a reliable test video source (Big Buck Bunny stable link) + curl -L -o video/test.mp4 https://test-videos.co.uk/vids/bigbuckbunny/mp4/av1/1080/Big_Buck_Bunny_1080_10s_5MB.mp4 + ls -lh video/ + + # 4. Run the Test Script + # Since ffmpeg.test.sh handles 'docker build', we just run it directly. + - name: Run Test Suite + run: | + chmod +x test/ffmpeg.test.sh + bash test/ffmpeg.test.sh + + # 5. Push Image to GHCR (Only on Main) + - name: Push to GHCR + if: github.ref == 'refs/heads/main' + run: | + IMAGE_ID=ghcr.io/${{ github.repository }} + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + + # Tag the image built by ffmpeg.test.sh with the proper GHCR name + docker tag worker-ffmpeg $IMAGE_ID:latest + docker tag worker-ffmpeg $IMAGE_ID:${{ github.sha }} + + echo "Pushing $IMAGE_ID:latest" + docker push $IMAGE_ID:latest + docker push $IMAGE_ID:${{ github.sha }} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c98b2c0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,130 @@ +# ── STAGE 1: BUILDER ───────────────────────────────────────────────────────── +# Alpine 3.21 chosen for minimal footprint (~5MB base). +# We compile from source to control exactly which libraries are linked. +# ───────────────────────────────────────────────────────────────────────────── +FROM alpine:3.21 AS builder + +# Link to GitHub Repository for Package Visibility +LABEL org.opencontainers.image.source=https://github.com/maulik-mk/mp.ii-worker + +# 1. Install Build Dependencies +# We only install what's strictly necessary for compilation. +# - build-base: GCC, Make, libc-dev (standard build toolchain) +# - pkgconf: Helper for library path resolution +# - nasm/yasm: Assemblers required for x264 SIMD optimizations (CRITICAL for perf) +# - x264-dev: H.264 video encoder headers +# - fdk-aac-dev: High-quality AAC audio encoder headers (better than native aac) +RUN apk add --no-cache \ + build-base \ + pkgconf \ + nasm \ + yasm \ + x264-dev \ + fdk-aac-dev + +# 2. Download FFmpeg Source +ARG FFMPEG_VERSION=7.1 +RUN wget -q https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \ + tar xjf ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \ + rm ffmpeg-${FFMPEG_VERSION}.tar.bz2 + +WORKDIR /ffmpeg-${FFMPEG_VERSION} + +# 3. Configure & Compile +# ONLY (H.264 + AAC). +# +# Flags explained: +# --enable-small: Optimize for size +# --disable-network: Attack surface reduction. +# --disable-autodetect: Deterministic build. +# --disable-*: We strip all GUI dependencies (SDL, X11, XCB) and hardware accelerators +# --extra-cflags: "-O2" for standard optimization. "-march=armv8-a" matches target arch. +RUN ./configure \ + --prefix=/usr/local \ + --enable-gpl \ + --enable-nonfree \ + --enable-small \ + \ + # ── Core Capabilities ── \ + --enable-libx264 \ + --enable-libfdk-aac \ + \ + # ── Bloat Removal Strategy ── \ + --disable-doc \ + --disable-debug \ + --disable-ffplay \ + --disable-network \ + --disable-autodetect \ + \ + # ── GUI & System Dependencies Strip ── \ + --disable-sdl2 \ + --disable-libxcb \ + --disable-libxcb-shm \ + --disable-libxcb-xfixes \ + --disable-libxcb-shape \ + --disable-xlib \ + \ + # ── Hardware Acceleration Strip (CPU-only target) ── \ + --disable-vaapi \ + --disable-vdpau \ + --disable-videotoolbox \ + --disable-audiotoolbox \ + --disable-cuda \ + --disable-cuvid \ + --disable-nvenc \ + --disable-nvdec \ + \ + # ── Device Strip ── \ + --disable-indevs \ + --disable-outdevs \ + \ + # ── Compiler Optimizations ── \ + --extra-cflags="-O2 -march=armv8-a" \ + \ + && make -j$(nproc) \ + && make install \ + # Binary Stripping: Removes debug symbols (~80% size reduction on binary) + && strip /usr/local/bin/ffmpeg /usr/local/bin/ffprobe + +# ── STAGE 2: RUNTIME ───────────────────────────────────────────────────────── +FROM alpine:3.21 + +# 1. Install Runtime Dependencies +# These are the shared libraries our compiled FFmpeg binary links against. +# Without these, the binary will fail with "not found" errors. +# - x264-libs: H.264 runtime +# - fdk-aac: AAC runtime +# - ca-certificates: Required we ever need to fetch HTTPS or HTTP +# clean apk cache immediately to keep layer size minimal. +RUN apk add --no-cache \ + x264-libs \ + fdk-aac \ + ca-certificates \ + && rm -rf /var/cache/apk/* + +# 2. Copy Artifacts +# Bringing over ONLY the compiled binaries from Stage 1. +COPY --from=builder /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg +COPY --from=builder /usr/local/bin/ffprobe /usr/local/bin/ffprobe + +# 3. Security Hardening +# - Create specific directories for input/output to control scope. +# - Create a non-root 'ffmpeg' user/group. +# - Chown directories to this user. +# - Switch USER context. +# Ideally, we should run with read-only root filesystem if possible. +RUN mkdir -p /input /output && \ + addgroup -S ffmpeg && adduser -S ffmpeg -G ffmpeg && \ + chown -R ffmpeg:ffmpeg /input /output + +USER ffmpeg +WORKDIR /work + +# 4. Verification Step +# Fails the build immediately if the binary is broken/missing libs. +RUN ffmpeg -version && ffprobe -version + +# Entrypoint configuration +# Allows passing arguments directly to docker run, e.g., "docker run img -i ..." +ENTRYPOINT ["ffmpeg"] +CMD ["-version"] diff --git a/test/ffmpeg.test.sh b/test/ffmpeg.test.sh new file mode 100644 index 0000000..68d06b6 --- /dev/null +++ b/test/ffmpeg.test.sh @@ -0,0 +1,232 @@ +#!/bin/bash + +# Enable strict mode: +# -e: Exit immediately if a command exits with a non-zero status. +# -u: Treat unset variables as an error when substituting. +# -o pipefail: The return value of a pipeline is the status of the last command to exit with a non-zero status. +set -euo pipefail + +# ------------------------------------------------------------------------------ +# Configuration +# ------------------------------------------------------------------------------ +IMAGE_NAME="worker-ffmpeg" + +# Resource limits for testing +CONTAINER_MEM_LIMIT="300m" +CONTAINER_CPU_LIMIT="1" + +# Paths +# script is run from the project root. +VIDEO_DIR="video" +OUTPUT_DIR="output" +TEST_VIDEO_FILE=$(ls "$VIDEO_DIR" | head -n 1) + +# colors. +BOLD="\033[1m" +GREEN="\033[32m" +CYAN="\033[36m" +YELLOW="\033[33m" +RED="\033[31m" +RESET="\033[0m" + +log_info() { + echo -e "${CYAN}[INFO]${RESET} $1" +} + +log_success() { + echo -e "${GREEN}[OK]${RESET} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${RESET} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${RESET} $1" >&2 +} + +log_header() { + echo -e "\n${BOLD}=== $1 ===${RESET}" +} + +# ------------------------------------------------------------------------------ +# Steps +# ------------------------------------------------------------------------------ + +# Ensure Docker is installed and running +check_prerequisites() { + if ! command -v docker &> /dev/null; then + log_error "Docker is not installed or not in PATH." + exit 1 + fi +} + +# Step 1: Build the Docker image +build_image() { + log_header "Step 1: Building Docker Image" + log_info "Initiating build for image '${IMAGE_NAME}'..." + docker build -t "${IMAGE_NAME}" . + log_success "Docker image build completed successfully." +} + +# Step 2: Verify FFmpeg version +verify_ffmpeg_version() { + log_header "Step 2: Verifying FFmpeg Version" + local output + output=$(docker run --rm "${IMAGE_NAME}" -version) + echo "$output" | head -n 1 +} + +# Step 3: Verify FFprobe version +verify_ffprobe_version() { + log_header "Step 3: Verifying FFprobe Version" + local output + output=$(docker run --rm --entrypoint ffprobe "${IMAGE_NAME}" -version) + echo "$output" | head -n 1 +} + +# Step 4: Verify supported codecs +verify_codecs() { + log_header "Step 4: Validating Supported Codecs" + log_info "Checking for required libraries: libx264 (H.264) and libfdk_aac (AAC)..." + + local codecs + if codecs=$(docker run --rm "${IMAGE_NAME}" -codecs 2>/dev/null | grep -E "libx264|libfdk_aac"); then + echo "$codecs" + log_success "Required codecs verified successfully." + else + log_error "Critical Error: Required codecs (libx264, libfdk_aac) are missing from the image." + exit 1 + fi +} + +# Step 5: Transcode Test +run_transcode_test() { + log_header "Step 5: Transcoding Test" + local input_path="${VIDEO_DIR}/${TEST_VIDEO_FILE}" + local output_file="${OUTPUT_DIR}/test_720p.mp4" + + if [[ ! -f "$input_path" ]]; then + log_warn "Input file not found at '${input_path}'. Skipping transcoding test." + return + fi + + # Prepare output directory + mkdir -p "${OUTPUT_DIR}" + rm -f "${output_file}" + + log_info "Starting transcoding process (Memory Limit: ${CONTAINER_MEM_LIMIT})..." + + # Run FFmpeg in Docker + docker run --rm \ + --memory="${CONTAINER_MEM_LIMIT}" \ + --cpus="${CONTAINER_CPU_LIMIT}" \ + -v "$(pwd)/${VIDEO_DIR}:/input:ro" \ + -v "$(pwd)/${OUTPUT_DIR}:/output" \ + "${IMAGE_NAME}" \ + -y \ + -hide_banner -loglevel error \ + -stats \ + -i "/input/${TEST_VIDEO_FILE}" \ + -threads 1 \ + -filter_threads 1 \ + -vf "scale=1280:720" \ + -c:v libx264 -preset ultrafast -b:v 2800k \ + # First 10 seconds of the video only + -t 10 \ + -movflags +faststart \ + "/output/$(basename "${output_file}")" + + if [[ -f "${output_file}" ]]; then + log_success "Transcoding complete. Output file created at: ${output_file}" + ls -lh "${output_file}" + else + log_error "Transcoding failed. No output file was generated." + exit 1 + fi +} + +# Step 6: HLS Adaptive Bitrate Test +run_hls_test() { + log_header "Step 6: HLS Adaptive Bitrate Test" + local input_path="${VIDEO_DIR}/${TEST_VIDEO_FILE}" + + if [[ ! -f "$input_path" ]]; then + log_warn "Input file not found. Skipping HLS test." + return + fi + + local hls_root="${OUTPUT_DIR}/hls" + rm -rf "${hls_root}" + mkdir -p "${hls_root}"/{360p,720p,1080p} + + # Define common options to reduce repetition + local docker_opts="--rm --memory=${CONTAINER_MEM_LIMIT} --cpus=${CONTAINER_CPU_LIMIT} -v $(pwd)/${VIDEO_DIR}:/input:ro -v $(pwd)/${OUTPUT_DIR}/hls:/output" + local ffmpeg_opts="-hide_banner -loglevel error -stats -threads 1 -filter_threads 1" + local hls_flags="-f hls -hls_time 4 -hls_playlist_type vod -hls_list_size 0" + + # Pass 1: 360p + log_info "Generating 360p HLS stream segment..." + docker run ${docker_opts} "${IMAGE_NAME}" ${ffmpeg_opts} \ + -i "/input/${TEST_VIDEO_FILE}" -t 10 \ + -vf "scale=640:360" -c:v libx264 -preset ultrafast -b:v 800k \ + ${hls_flags} -hls_segment_filename '/output/360p/segment_%03d.ts' -y '/output/360p/stream.m3u8' + + # Pass 2: 720p + log_info "Generating 720p HLS stream segment..." + docker run ${docker_opts} "${IMAGE_NAME}" ${ffmpeg_opts} \ + -i "/input/${TEST_VIDEO_FILE}" -t 10 \ + -vf "scale=1280:720" -c:v libx264 -preset ultrafast -b:v 2800k \ + ${hls_flags} -hls_segment_filename '/output/720p/segment_%03d.ts' -y '/output/720p/stream.m3u8' + + # Pass 3: 1080p + log_info "Generating 1080p HLS stream segment..." + docker run ${docker_opts} "${IMAGE_NAME}" ${ffmpeg_opts} \ + -i "/input/${TEST_VIDEO_FILE}" -t 10 \ + -vf "scale=1920:1080" -c:v libx264 -preset ultrafast -b:v 5000k \ + ${hls_flags} -hls_segment_filename '/output/1080p/segment_%03d.ts' -y '/output/1080p/stream.m3u8' + + # Generate Master Playlist + cat > "${hls_root}/master.m3u8" < Date: Tue, 17 Feb 2026 19:43:13 +0530 Subject: [PATCH 03/12] fix: Remove architecture-specific optimization flag from FFmpeg build (#4) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c98b2c0..2aa6c16 100644 --- a/Dockerfile +++ b/Dockerfile @@ -79,7 +79,7 @@ RUN ./configure \ --disable-outdevs \ \ # ── Compiler Optimizations ── \ - --extra-cflags="-O2 -march=armv8-a" \ + --extra-cflags="-O2" \ \ && make -j$(nproc) \ && make install \ From 342927b1aac794226414374b119b1a1aded144de Mon Sep 17 00:00:00 2001 From: MAULIK MK <214461757+maulik-mk@users.noreply.github.com> Date: Tue, 17 Feb 2026 20:03:47 +0530 Subject: [PATCH 04/12] fix: ensure ffmpeg test script always specifies output file (#5) --- test/ffmpeg.test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ffmpeg.test.sh b/test/ffmpeg.test.sh index 68d06b6..e5fcd27 100644 --- a/test/ffmpeg.test.sh +++ b/test/ffmpeg.test.sh @@ -118,6 +118,7 @@ run_transcode_test() { log_info "Starting transcoding process (Memory Limit: ${CONTAINER_MEM_LIMIT})..." # Run FFmpeg in Docker + # First 10 seconds of the video only docker run --rm \ --memory="${CONTAINER_MEM_LIMIT}" \ --cpus="${CONTAINER_CPU_LIMIT}" \ @@ -132,7 +133,6 @@ run_transcode_test() { -filter_threads 1 \ -vf "scale=1280:720" \ -c:v libx264 -preset ultrafast -b:v 2800k \ - # First 10 seconds of the video only -t 10 \ -movflags +faststart \ "/output/$(basename "${output_file}")" From 39580d381847e5a7cc65b20df2b38c968605b017 Mon Sep 17 00:00:00 2001 From: MAULIK MK <214461757+maulik-mk@users.noreply.github.com> Date: Tue, 17 Feb 2026 22:16:20 +0530 Subject: [PATCH 05/12] [fix] [CI] : Grant full permissions to the test output directory. (#7) - Enable CI for the dev branch. - permissions to the ffmpeg test output directory. - Update CI test video download to use H264 codec instead of AV1. - test: add 777 permissions to HLS output directory in ffmpeg test --- .github/workflows/ci.yml | 6 +++--- test/ffmpeg.test.sh | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7c64a8..bc917fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: push: - branches: [ "main", "master"] + branches: [ "main", "master", 'dev'] pull_request: - branches: [ "main", "master"] + branches: [ "main", "master", 'dev'] concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -40,7 +40,7 @@ jobs: mkdir -p video echo "Downloading sample video..." # Using a reliable test video source (Big Buck Bunny stable link) - curl -L -o video/test.mp4 https://test-videos.co.uk/vids/bigbuckbunny/mp4/av1/1080/Big_Buck_Bunny_1080_10s_5MB.mp4 + curl -L -o video/test.mp4 https://test-videos.co.uk/vids/bigbuckbunny/mp4/h264/1080/Big_Buck_Bunny_1080_10s_5MB.mp4 ls -lh video/ # 4. Run the Test Script diff --git a/test/ffmpeg.test.sh b/test/ffmpeg.test.sh index e5fcd27..df6c028 100644 --- a/test/ffmpeg.test.sh +++ b/test/ffmpeg.test.sh @@ -113,6 +113,7 @@ run_transcode_test() { # Prepare output directory mkdir -p "${OUTPUT_DIR}" + chmod 777 "${OUTPUT_DIR}" rm -f "${output_file}" log_info "Starting transcoding process (Memory Limit: ${CONTAINER_MEM_LIMIT})..." @@ -159,6 +160,7 @@ run_hls_test() { local hls_root="${OUTPUT_DIR}/hls" rm -rf "${hls_root}" mkdir -p "${hls_root}"/{360p,720p,1080p} + chmod -R 777 "${hls_root}" # Define common options to reduce repetition local docker_opts="--rm --memory=${CONTAINER_MEM_LIMIT} --cpus=${CONTAINER_CPU_LIMIT} -v $(pwd)/${VIDEO_DIR}:/input:ro -v $(pwd)/${OUTPUT_DIR}/hls:/output" From 3534e5160441d55e4bde5dfe0e5498f644a7a0be Mon Sep 17 00:00:00 2001 From: MAULIK MK <214461757+maulik-mk@users.noreply.github.com> Date: Tue, 17 Feb 2026 23:45:40 +0530 Subject: [PATCH 06/12] Release : v0.1.0 (#9) - Sync changes of v0.1.0 PR --- .github/workflows/ci.yml | 42 +++++++++++++++++++++++++++------------- package.json | 2 +- test/ffmpeg.test.sh | 8 +++++++- 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc917fa..aff66eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,43 +24,59 @@ jobs: - name: Checkout Code uses: actions/checkout@v4 - # 2. Log in to GitHub Container Registry + # Setup Docker Buildx to enable advanced build features and caching + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + # Build and cache Docker layers using GitHub Actions cache backend + # 'load: true' exports the image to the local Docker daemon for testing + - name: Build and Load Docker Image + uses: docker/build-push-action@v5 + with: + context: . + load: true + tags: worker-ffmpeg:latest + cache-from: type=gha + cache-to: type=gha,mode=max + + # Authenticate to GHCR for image publishing - name: Log in to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} - password: ${{ secrets.GHCR_TOKEN }} + password: ${{ secrets.GITHUB_TOKEN }} - # 3. Download a sample test video (1080p, ~2MB) - # Dynamically to keep the git repo small. - # The script expects 'video', so we simulate that structure. + # Fetch reference test asset (1080p H.264) + # Dynamic download avoids repository bloat - name: Download Test Asset run: | mkdir -p video echo "Downloading sample video..." - # Using a reliable test video source (Big Buck Bunny stable link) + # Source: Big Buck Bunny (Stable URL) curl -L -o video/test.mp4 https://test-videos.co.uk/vids/bigbuckbunny/mp4/h264/1080/Big_Buck_Bunny_1080_10s_5MB.mp4 ls -lh video/ - # 4. Run the Test Script - # Since ffmpeg.test.sh handles 'docker build', we just run it directly. + # Execute integration test suite + # Patch script to skip redundant build step (using cached image) - name: Run Test Suite + env: + SKIP_BUILD: 'true' run: | chmod +x test/ffmpeg.test.sh bash test/ffmpeg.test.sh - # 5. Push Image to GHCR (Only on Main) + # Publish artifact to GHCR (Production branch only) - name: Push to GHCR if: github.ref == 'refs/heads/main' run: | IMAGE_ID=ghcr.io/${{ github.repository }} - # Change all uppercase to lowercase + # Normalize repository name to lowercase IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') - # Tag the image built by ffmpeg.test.sh with the proper GHCR name - docker tag worker-ffmpeg $IMAGE_ID:latest - docker tag worker-ffmpeg $IMAGE_ID:${{ github.sha }} + # Apply semantic tags (latest, SHA) for versioning + docker tag worker-ffmpeg:latest $IMAGE_ID:latest + docker tag worker-ffmpeg:latest $IMAGE_ID:${{ github.sha }} echo "Pushing $IMAGE_ID:latest" docker push $IMAGE_ID:latest diff --git a/package.json b/package.json index a196956..dbdc58f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "worker", - "version": "0.0.0", + "version": "0.1.0", "description": "FFmpeg worker", "repository": { "type": "git", diff --git a/test/ffmpeg.test.sh b/test/ffmpeg.test.sh index df6c028..32029d5 100644 --- a/test/ffmpeg.test.sh +++ b/test/ffmpeg.test.sh @@ -221,7 +221,13 @@ print_stats() { # Main Execution main() { check_prerequisites - build_image + + if [[ "${SKIP_BUILD:-false}" != "true" ]]; then + build_image + else + log_info "Skipping build step (SKIP_BUILD=true)..." + fi + verify_ffmpeg_version verify_ffprobe_version verify_codecs From aa1f2d84a8b0eafa140ec8a14de96a145bb76968 Mon Sep 17 00:00:00 2001 From: MAULIK MK <214461757+maulik-mk@users.noreply.github.com> Date: Sat, 21 Mar 2026 15:03:13 +0530 Subject: [PATCH 07/12] feat: implement DDD worker architecture with FFmpeg HLS encoding pipeline (#11) This commit introduces the foundational architecture for the video processing worker. It establishes a complete pipeline for downloading, encoding, and uploading video content using Domain-Driven Design principles. Key features added: - Infrastructure Setup: Integrated BullMQ for highly reliable background video job processing and setup PostgreSQL connection pooling. - Azure Storage Integration: Added services for fetching raw media blobs and robust uploading of generated output segments/manifests back to Azure Blob Storage. - FFmpeg HLS Processing: Added core FFmpeg adapters to handle video probing, content complexity analysis, multi-profile audio/video rendering, and HLS (m3u8) playlist generation. - DDD Architecture: Organized the codebase into independent `domain` models, `application` use cases (`video.process.ts`), and `infrastructure` implementations. --- .dockerignore | 18 + .env.example | 49 + .prettierrc | 13 + Dockerfile | 290 ++- package.json | 57 +- pnpm-lock.yaml | 1835 +++++++++++++++++ pnpm-workspace.yaml | 3 + sql/main.sql | 57 + src/application/video.process.ts | 155 ++ src/config/env.ts | 45 + src/domain/errors.ts | 38 + src/domain/job.interface.ts | 93 + src/infrastructure/db/db.ts | 138 ++ src/infrastructure/ffmpeg/adapter.ts | 169 ++ src/infrastructure/ffmpeg/constants.ts | 15 + src/infrastructure/ffmpeg/core/complexity.ts | 192 ++ src/infrastructure/ffmpeg/core/hash.ts | 15 + src/infrastructure/ffmpeg/core/probe.ts | 105 + src/infrastructure/ffmpeg/core/runner.ts | 153 ++ src/infrastructure/ffmpeg/encoding/flags.ts | 164 ++ .../ffmpeg/encoding/profiles.ts | 110 + .../ffmpeg/encoding/profiles/audio.json | 51 + .../ffmpeg/encoding/profiles/video.json | 338 +++ src/infrastructure/ffmpeg/hls/pipeline.ts | 143 ++ src/infrastructure/ffmpeg/hls/playlist.ts | 232 +++ src/infrastructure/ffmpeg/index.ts | 1 + src/infrastructure/ffmpeg/types.ts | 52 + src/infrastructure/queue/video.worker.ts | 89 + src/infrastructure/storage/azure.service.ts | 111 + src/server.ts | 104 + test/ffmpeg.test.sh | 14 +- test/queue-job.test.local.ts | 193 ++ tsconfig.json | 24 + 33 files changed, 4953 insertions(+), 113 deletions(-) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 .prettierrc create mode 100644 pnpm-lock.yaml create mode 100644 pnpm-workspace.yaml create mode 100644 sql/main.sql create mode 100644 src/application/video.process.ts create mode 100644 src/config/env.ts create mode 100644 src/domain/errors.ts create mode 100644 src/domain/job.interface.ts create mode 100644 src/infrastructure/db/db.ts create mode 100644 src/infrastructure/ffmpeg/adapter.ts create mode 100644 src/infrastructure/ffmpeg/constants.ts create mode 100644 src/infrastructure/ffmpeg/core/complexity.ts create mode 100644 src/infrastructure/ffmpeg/core/hash.ts create mode 100644 src/infrastructure/ffmpeg/core/probe.ts create mode 100644 src/infrastructure/ffmpeg/core/runner.ts create mode 100644 src/infrastructure/ffmpeg/encoding/flags.ts create mode 100644 src/infrastructure/ffmpeg/encoding/profiles.ts create mode 100644 src/infrastructure/ffmpeg/encoding/profiles/audio.json create mode 100644 src/infrastructure/ffmpeg/encoding/profiles/video.json create mode 100644 src/infrastructure/ffmpeg/hls/pipeline.ts create mode 100644 src/infrastructure/ffmpeg/hls/playlist.ts create mode 100644 src/infrastructure/ffmpeg/index.ts create mode 100644 src/infrastructure/ffmpeg/types.ts create mode 100644 src/infrastructure/queue/video.worker.ts create mode 100644 src/infrastructure/storage/azure.service.ts create mode 100644 src/server.ts create mode 100644 test/queue-job.test.local.ts create mode 100644 tsconfig.json diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..144d08f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +node_modules +.git +.github +dist +video +test +*.log +*.md +.env +.env.* +!.env.example +.DS_Store +.gitignore +.dockerignore +.eslintcache +coverage +.nyc_output +docs \ No newline at end of file diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..29ef4ee --- /dev/null +++ b/.env.example @@ -0,0 +1,49 @@ +# ============================================================================== +# SHARED (Required for BOTH Production & Development) +# ============================================================================== + +# Core System Services +PORT=3000 +REDIS_URL=rediss://user:password@host:port +DATABASE_URL=postgresql://user:password@host:port/dbname +CORS_ORIGIN=* + +# Azure Blob Storage (Shared Configuration) +AZURE_STORAGE_CONTAINER_NAME=video-assets +CONTAINER_DIRECTORY_1=encoded/hls/v1 +AZURE_UPLOAD_BATCH_SIZE=20 +AZURE_UPLOAD_RETRIES=3 + +# Queue Stability +WORKER_CONCURRENCY=1 +JOB_LOCK_DURATION_MS=120000 +JOB_LOCK_RENEW_MS=30000 + +# Video Pipeline Settings +# "SINGLE_FILE" (Byte-range fMP4) or "SEGMENTED" (Standard chunks) +HLS_OUTPUT_MODE="SEGMENTED" + + +# ============================================================================== +# PRODUCTION ONLY +# ============================================================================== +NODE_ENV=production + +# Azure Managed Identity URL (Replaces the connection string in production for zero-trust security) +AZURE_STORAGE_ACCOUNT_URL=https://.blob.core.windows.net + + +# ============================================================================== +# DEVELOPMENT ONLY +# ============================================================================== +NODE_ENV=development + +# Azure Connection String (Used locally before deploying to Managed Identity infrastructure) +AZURE_STORAGE_CONNECTION_STRING=DefaultEndpointsProtocol=https;AccountName=... + +# Dev / Testing Overrides +# Truncates video source to N seconds to test pipelines quickly without rendering full video +TEST_DURATION_SECONDS=15 + +# Mock Payload Data (Used by test/queue-job.test.local.ts) +RAW_VIDEO_SOURCE_URL=http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4 \ No newline at end of file diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..3393f79 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,13 @@ +{ + "semi": true, + "trailingComma": "all", + "singleQuote": true, + "printWidth": 100, + "tabWidth": 3, + "useTabs": false, + "arrowParens": "always", + "endOfLine": "lf", + "bracketSpacing": true, + "proseWrap": "always", + "embeddedLanguageFormatting": "auto" +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 2aa6c16..4ea1852 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,28 +1,77 @@ -# ── STAGE 1: BUILDER ───────────────────────────────────────────────────────── -# Alpine 3.21 chosen for minimal footprint (~5MB base). -# We compile from source to control exactly which libraries are linked. -# ───────────────────────────────────────────────────────────────────────────── -FROM alpine:3.21 AS builder - -# Link to GitHub Repository for Package Visibility -LABEL org.opencontainers.image.source=https://github.com/maulik-mk/mp.ii-worker - -# 1. Install Build Dependencies -# We only install what's strictly necessary for compilation. -# - build-base: GCC, Make, libc-dev (standard build toolchain) -# - pkgconf: Helper for library path resolution -# - nasm/yasm: Assemblers required for x264 SIMD optimizations (CRITICAL for perf) -# - x264-dev: H.264 video encoder headers -# - fdk-aac-dev: High-quality AAC audio encoder headers (better than native aac) -RUN apk add --no-cache \ - build-base \ - pkgconf \ +# ============================================================================= +# PROJECT: FFmpeg Video Processing Worker (Node.js) +# AUTHOR: Maulik M. Kadeval +# +# COPYRIGHT & THIRD-PARTY ATTRIBUTIONS: +# This image compiles and integrates several high-performance media libraries. +# The respective trademarks, copyrights, and licenses belong to their original +# creators and organizations: +# +# * FFmpeg - Copyright (c) The FFmpeg Developers (https://ffmpeg.org) +# * VMAF - Copyright (c) Netflix, Inc. (https://github.com/Netflix/vmaf) +# * libfdk-aac - Copyright (c) Fraunhofer IIS (https://www.iis.fraunhofer.de) +# * libx264 - Copyright (c) VideoLAN Organization (https://www.videolan.org) +# * libx265 - Copyright (c) MulticoreWare, Inc. (https://x265.com) +# * libopus - Copyright (c) Xiph.Org Foundation (https://opus-codec.org) +# * libsoxr - Copyright (c) The SoX Resampler Authors (https://sourceforge.net/p/soxr) +# * libzimg - Copyright (c) Sekrit-Twc (https://github.com/sekrit-twc/zimg) +# +# NOTE: This build enables `--nonfree` components (libfdk-aac). Ensure compliance +# with licensing requirements if distributing this image commercially. +# +# Base Image: Ubuntu 24.04 (Noble Numbat, glibc 2.39). +# ============================================================================= + +FROM ubuntu:24.04 AS builder + +# Suppress interactive prompts from apt during the build process +ENV DEBIAN_FRONTEND=noninteractive + +# ----------------------------------------------------------------------------- +# 1. Install System & Build Dependencies +# Retrieves compilation toolchains (GCC, Make, CMake, Meson) and development +# headers for the third-party audio/video codecs. +# ----------------------------------------------------------------------------- +RUN apt-get update && apt-get install -y --no-install-recommends \ + software-properties-common && \ + add-apt-repository -y universe && \ + add-apt-repository -y multiverse && \ + apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + pkg-config \ nasm \ yasm \ - x264-dev \ - fdk-aac-dev + wget \ + git \ + cmake \ + ninja-build \ + meson \ + ca-certificates \ + libx264-dev \ + libx265-dev \ + libfdk-aac-dev \ + libzimg-dev \ + libssl-dev \ + libsoxr-dev \ + libopus-dev \ + && rm -rf /var/lib/apt/lists/* + +# ----------------------------------------------------------------------------- +# 2. Compile Netflix VMAF (Video Multi-Method Assessment Fusion) +# Clones the official Netflix repository and builds the libvmaf library for +# AI-driven perceptual video quality metrics. +# ----------------------------------------------------------------------------- +RUN git clone --depth 1 https://github.com/Netflix/vmaf.git /tmp/vmaf && \ + cd /tmp/vmaf/libvmaf && \ + meson setup build --buildtype release --prefix=/usr/local && \ + ninja -vC build install && \ + mkdir -p /usr/local/share/model && \ + cp -r ../model/* /usr/local/share/model/ && \ + rm -rf /tmp/vmaf -# 2. Download FFmpeg Source +# ----------------------------------------------------------------------------- +# 3. Download FFmpeg Source Code +# ----------------------------------------------------------------------------- ARG FFMPEG_VERSION=7.1 RUN wget -q https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \ tar xjf ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \ @@ -30,41 +79,42 @@ RUN wget -q https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \ WORKDIR /ffmpeg-${FFMPEG_VERSION} -# 3. Configure & Compile -# ONLY (H.264 + AAC). -# -# Flags explained: -# --enable-small: Optimize for size -# --disable-network: Attack surface reduction. -# --disable-autodetect: Deterministic build. -# --disable-*: We strip all GUI dependencies (SDL, X11, XCB) and hardware accelerators -# --extra-cflags: "-O2" for standard optimization. "-march=armv8-a" matches target arch. -RUN ./configure \ +# ----------------------------------------------------------------------------- +# 4. Configure and Compile FFmpeg +# Integrates all previously installed libraries (x264, x265, VMAF, Opus, SoXR). +# PKG_CONFIG_PATH ensures the custom-built libvmaf is located successfully. +# ----------------------------------------------------------------------------- +RUN PKG_CONFIG_PATH=/usr/local/lib/aarch64-linux-gnu/pkgconfig:/usr/local/lib/x86_64-linux-gnu/pkgconfig:/usr/local/lib/pkgconfig \ + ./configure \ --prefix=/usr/local \ --enable-gpl \ --enable-nonfree \ - --enable-small \ - \ - # ── Core Capabilities ── \ + --enable-version3 \ + --enable-swresample \ + --enable-libsoxr \ + --enable-libopus \ --enable-libx264 \ + --enable-libx265 \ --enable-libfdk-aac \ - \ - # ── Bloat Removal Strategy ── \ + --enable-libzimg \ + --enable-libvmaf \ + --enable-encoder=eac3 \ + --enable-encoder=ac3 \ + --enable-encoder=aac \ + --enable-openssl \ + --enable-protocol=https \ + --enable-protocol=http \ + --enable-protocol=file \ --disable-doc \ --disable-debug \ --disable-ffplay \ - --disable-network \ --disable-autodetect \ - \ - # ── GUI & System Dependencies Strip ── \ --disable-sdl2 \ --disable-libxcb \ --disable-libxcb-shm \ --disable-libxcb-xfixes \ --disable-libxcb-shape \ --disable-xlib \ - \ - # ── Hardware Acceleration Strip (CPU-only target) ── \ --disable-vaapi \ --disable-vdpau \ --disable-videotoolbox \ @@ -73,58 +123,126 @@ RUN ./configure \ --disable-cuvid \ --disable-nvenc \ --disable-nvdec \ - \ - # ── Device Strip ── \ --disable-indevs \ --disable-outdevs \ - \ - # ── Compiler Optimizations ── \ --extra-cflags="-O2" \ - \ && make -j$(nproc) \ && make install \ - # Binary Stripping: Removes debug symbols (~80% size reduction on binary) && strip /usr/local/bin/ffmpeg /usr/local/bin/ffprobe -# ── STAGE 2: RUNTIME ───────────────────────────────────────────────────────── -FROM alpine:3.21 - -# 1. Install Runtime Dependencies -# These are the shared libraries our compiled FFmpeg binary links against. -# Without these, the binary will fail with "not found" errors. -# - x264-libs: H.264 runtime -# - fdk-aac: AAC runtime -# - ca-certificates: Required we ever need to fetch HTTPS or HTTP -# clean apk cache immediately to keep layer size minimal. -RUN apk add --no-cache \ - x264-libs \ - fdk-aac \ + +# ============================================================================= +# Stage 2 — Node.js Application Build +# Securely installs Node.js and compiles the TypeScript worker codebase. +# ============================================================================= +FROM ubuntu:24.04 AS node-builder + +ENV DEBIAN_FRONTEND=noninteractive + +# Securely install NodeSource repository and pnpm package manager +RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates gnupg && \ + mkdir -p /etc/apt/keyrings && \ + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ + NODE_MAJOR=24 && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ + apt-get update && apt-get install -y --no-install-recommends nodejs && \ + npm install -g pnpm && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY package.json pnpm-lock.yaml ./ +RUN pnpm install --frozen-lockfile + +COPY tsconfig.json ./ +COPY src ./src +RUN pnpm run build + + +# ============================================================================= +# Stage 3 — Production Runtime +# Final optimized image containing the compiled Node.js application, the custom +# FFmpeg binary, and minimal shared runtime libraries. +# ============================================================================= +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +# ----------------------------------------------------------------------------- +# Metadata & OCI Labels +# ----------------------------------------------------------------------------- +ARG VERSION=0.1.0 +ARG BUILD_DATE=unknown + +LABEL org.opencontainers.image.title="ffmpeg-queue-worker-node" \ + org.opencontainers.image.description="FFmpeg 7.1 (w/ VMAF) & Node.js Video Job Worker" \ + org.opencontainers.image.version="${VERSION}" \ + org.opencontainers.image.created="${BUILD_DATE}" \ + org.opencontainers.image.authors="Maulik M. Kadeval" \ + org.opencontainers.image.source="https://github.com/maulik-mk/ffmpeg-queue-worker-node.git" \ + com.bitflow.project.code="P1/25-26/bd/A1-WK" \ + com.bitflow.project.id="P1" \ + com.bitflow.project.cycle="25-26" \ + com.bitflow.project.dept="bd" \ + com.bitflow.project.app="A1" \ + com.bitflow.project.role="WK" + +# ----------------------------------------------------------------------------- +# 1. Install Runtime Dependencies & Node.js +# Fetches the shared objects (.so files) required by the compiled FFmpeg binary. +# ----------------------------------------------------------------------------- +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl \ + wget \ ca-certificates \ - && rm -rf /var/cache/apk/* + gnupg \ + libx265-199 \ + libx264-164 \ + libfdk-aac2 \ + libzimg2 \ + libsoxr0 \ + libopus0 \ + && mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \ + && NODE_MAJOR=24 \ + && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list \ + && apt-get update && apt-get install -y --no-install-recommends nodejs \ + && npm install -g pnpm \ + && rm -rf /var/lib/apt/lists/* -# 2. Copy Artifacts -# Bringing over ONLY the compiled binaries from Stage 1. -COPY --from=builder /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg +# ----------------------------------------------------------------------------- +# 2. Transfer FFmpeg & VMAF Assets from Builder Stage +# ----------------------------------------------------------------------------- +COPY --from=builder /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg COPY --from=builder /usr/local/bin/ffprobe /usr/local/bin/ffprobe +COPY --from=builder /usr/local/lib/*-linux-gnu/libvmaf.so* /usr/local/lib/ +COPY --from=builder /usr/local/share/model /usr/local/share/model + +# Update the dynamic linker cache so FFmpeg locates libvmaf and system libraries +RUN ldconfig + +# ----------------------------------------------------------------------------- +# 3. Setup Node.js Application Environment +# ----------------------------------------------------------------------------- +WORKDIR /app +COPY package.json pnpm-lock.yaml ./ +RUN pnpm install --prod --frozen-lockfile +COPY --from=node-builder /app/dist ./dist + +# ----------------------------------------------------------------------------- +# 4. Security Hardening & Process Management +# Creates a non-root 'worker' user to safely execute the application. +# ----------------------------------------------------------------------------- +RUN mkdir -p /tmp/worker && \ + groupadd -r worker && useradd -r -g worker worker && \ + chown -R worker:worker /app /tmp/worker + +USER worker + +ENV NODE_ENV=production +ENV PORT=3000 + +# Docker Healthcheck to ensure the Node.js API/Worker is responding +HEALTHCHECK --interval=30s --timeout=3s \ + CMD wget --no-verbose --tries=1 --spider http://localhost:3000/health || exit 1 -# 3. Security Hardening -# - Create specific directories for input/output to control scope. -# - Create a non-root 'ffmpeg' user/group. -# - Chown directories to this user. -# - Switch USER context. -# Ideally, we should run with read-only root filesystem if possible. -RUN mkdir -p /input /output && \ - addgroup -S ffmpeg && adduser -S ffmpeg -G ffmpeg && \ - chown -R ffmpeg:ffmpeg /input /output - -USER ffmpeg -WORKDIR /work - -# 4. Verification Step -# Fails the build immediately if the binary is broken/missing libs. -RUN ffmpeg -version && ffprobe -version - -# Entrypoint configuration -# Allows passing arguments directly to docker run, e.g., "docker run img -i ..." -ENTRYPOINT ["ffmpeg"] -CMD ["-version"] +CMD ["node", "dist/server.js"] \ No newline at end of file diff --git a/package.json b/package.json index dbdc58f..3b483a5 100644 --- a/package.json +++ b/package.json @@ -1,27 +1,44 @@ { "name": "worker", "version": "0.1.0", - "description": "FFmpeg worker", + "description": "FFmpeg Worker Service (TypeScript)", "repository": { "type": "git", - "url": "git+https://github.com/maulik-mk/mp.ii-worker.git" + "url": "git+https://github.com/maulik-mk/ffmpeg-queue-worker-node.git" }, - "keywords": [ - "ffmpeg", - "worker", - "video processing", - "video encoding", - "video decoding", - "video transcoding", - "video streaming", - "video processing worker", - "video processing worker", - "nodjs" - ], - "author": "Maulik MK", - "license": "ISC", - "bugs": { - "url": "https://github.com/maulik-mk/mp.ii-worker/issues" + "main": "dist/server.js", + "type": "module", + "packageManager": "pnpm@10.32.1+sha512.a706938f0e89ac1456b6563eab4edf1d1faf3368d1191fc5c59790e96dc918e4456ab2e67d613de1043d2e8c81f87303e6b40d4ffeca9df15ef1ad567348f2be", + "scripts": { + "build": "rm -rf dist && tsc && find src -name '*.json' | while read f; do mkdir -p dist/$(dirname ${f#src/}) && cp $f dist/${f#src/}; done", + "dev:local:format": "prettier --write \"src/**/*.{ts,json}\"", + "dev:local:build:docker": "docker build -t worker-local-dev .", + "dev:local:test:job": "tsx --env-file=.env test/queue-job.test.local.ts", + "dev:local:run:docker": "docker run --env-file .env worker-local-dev", + "dev:local:e2e": "pnpm run dev:local:format && pnpm run dev:local:build:docker && pnpm run dev:local:test:job && pnpm run dev:local:run:docker" }, - "homepage": "https://github.com/maulik-mk/mp.ii-worker#readme" -} + "dependencies": { + "@azure/identity": "^4.13.1", + "@azure/storage-blob": "^12.31.0", + "@fastify/cors": "9.0.1", + "@fastify/helmet": "11.1.1", + "@fastify/rate-limit": "9.1.0", + "bullmq": "^5.0.0", + "fastify": "^4.26.0", + "glob": "^11.0.0", + "pg": "^8.13.0", + "pino": "^8.19.0", + "uuid": "^13.0.0", + "zod": "^3.22.0" + }, + "devDependencies": { + "@types/node": "^20.11.0", + "@types/pg": "^8.11.0", + "prettier": "^3.2.0", + "tsx": "^4.7.0", + "typescript": "^5.3.0" + }, + "engines": { + "node": ">=24" + } +} \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml new file mode 100644 index 0000000..2800053 --- /dev/null +++ b/pnpm-lock.yaml @@ -0,0 +1,1835 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@azure/identity': + specifier: ^4.13.1 + version: 4.13.1 + '@azure/storage-blob': + specifier: ^12.31.0 + version: 12.31.0 + '@fastify/cors': + specifier: 9.0.1 + version: 9.0.1 + '@fastify/helmet': + specifier: 11.1.1 + version: 11.1.1 + '@fastify/rate-limit': + specifier: 9.1.0 + version: 9.1.0 + bullmq: + specifier: ^5.0.0 + version: 5.71.0 + fastify: + specifier: ^4.26.0 + version: 4.29.1 + glob: + specifier: ^11.0.0 + version: 11.1.0 + pg: + specifier: ^8.13.0 + version: 8.20.0 + pino: + specifier: ^8.19.0 + version: 8.21.0 + uuid: + specifier: ^13.0.0 + version: 13.0.0 + zod: + specifier: ^3.22.0 + version: 3.25.76 + devDependencies: + '@types/node': + specifier: ^20.11.0 + version: 20.19.37 + '@types/pg': + specifier: ^8.11.0 + version: 8.18.0 + prettier: + specifier: ^3.2.0 + version: 3.8.1 + tsx: + specifier: ^4.7.0 + version: 4.21.0 + typescript: + specifier: ^5.3.0 + version: 5.9.3 + +packages: + + '@azure/abort-controller@2.1.2': + resolution: {integrity: sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==} + engines: {node: '>=18.0.0'} + + '@azure/core-auth@1.10.1': + resolution: {integrity: sha512-ykRMW8PjVAn+RS6ww5cmK9U2CyH9p4Q88YJwvUslfuMmN98w/2rdGRLPqJYObapBCdzBVeDgYWdJnFPFb7qzpg==} + engines: {node: '>=20.0.0'} + + '@azure/core-client@1.10.1': + resolution: {integrity: sha512-Nh5PhEOeY6PrnxNPsEHRr9eimxLwgLlpmguQaHKBinFYA/RU9+kOYVOQqOrTsCL+KSxrLLl1gD8Dk5BFW/7l/w==} + engines: {node: '>=20.0.0'} + + '@azure/core-http-compat@2.3.2': + resolution: {integrity: sha512-Tf6ltdKzOJEgxZeWLCjMxrxbodB/ZeCbzzA1A2qHbhzAjzjHoBVSUeSl/baT/oHAxhc4qdqVaDKnc2+iE932gw==} + engines: {node: '>=20.0.0'} + peerDependencies: + '@azure/core-client': ^1.10.0 + '@azure/core-rest-pipeline': ^1.22.0 + + '@azure/core-lro@2.7.2': + resolution: {integrity: sha512-0YIpccoX8m/k00O7mDDMdJpbr6mf1yWo2dfmxt5A8XVZVVMz2SSKaEbMCeJRvgQ0IaSlqhjT47p4hVIRRy90xw==} + engines: {node: '>=18.0.0'} + + '@azure/core-paging@1.6.2': + resolution: {integrity: sha512-YKWi9YuCU04B55h25cnOYZHxXYtEvQEbKST5vqRga7hWY9ydd3FZHdeQF8pyh+acWZvppw13M/LMGx0LABUVMA==} + engines: {node: '>=18.0.0'} + + '@azure/core-rest-pipeline@1.23.0': + resolution: {integrity: sha512-Evs1INHo+jUjwHi1T6SG6Ua/LHOQBCLuKEEE6efIpt4ZOoNonaT1kP32GoOcdNDbfqsD2445CPri3MubBy5DEQ==} + engines: {node: '>=20.0.0'} + + '@azure/core-tracing@1.3.1': + resolution: {integrity: sha512-9MWKevR7Hz8kNzzPLfX4EAtGM2b8mr50HPDBvio96bURP/9C+HjdH3sBlLSNNrvRAr5/k/svoH457gB5IKpmwQ==} + engines: {node: '>=20.0.0'} + + '@azure/core-util@1.13.1': + resolution: {integrity: sha512-XPArKLzsvl0Hf0CaGyKHUyVgF7oDnhKoP85Xv6M4StF/1AhfORhZudHtOyf2s+FcbuQ9dPRAjB8J2KvRRMUK2A==} + engines: {node: '>=20.0.0'} + + '@azure/core-xml@1.5.0': + resolution: {integrity: sha512-D/sdlJBMJfx7gqoj66PKVmhDDaU6TKA49ptcolxdas29X7AfvLTmfAGLjAcIMBK7UZ2o4lygHIqVckOlQU3xWw==} + engines: {node: '>=20.0.0'} + + '@azure/identity@4.13.1': + resolution: {integrity: sha512-5C/2WD5Vb1lHnZS16dNQRPMjN6oV/Upba+C9nBIs15PmOi6A3ZGs4Lr2u60zw4S04gi+u3cEXiqTVP7M4Pz3kw==} + engines: {node: '>=20.0.0'} + + '@azure/logger@1.3.0': + resolution: {integrity: sha512-fCqPIfOcLE+CGqGPd66c8bZpwAji98tZ4JI9i/mlTNTlsIWslCfpg48s/ypyLxZTump5sypjrKn2/kY7q8oAbA==} + engines: {node: '>=20.0.0'} + + '@azure/msal-browser@5.6.1': + resolution: {integrity: sha512-Ylmp8yngH7YRLV5mA1aF4CNS6WsJTPbVXaA0Tb1x1Gv/J3BM3hE4Q7nDaf7dRfU00FcxDBBudTjqlpH74ZSsgw==} + engines: {node: '>=0.8.0'} + + '@azure/msal-common@16.4.0': + resolution: {integrity: sha512-twXt09PYtj1PffNNIAzQlrBd0DS91cdA6i1gAfzJ6BnPM4xNk5k9q/5xna7jLIjU3Jnp0slKYtucshGM8OGNAw==} + engines: {node: '>=0.8.0'} + + '@azure/msal-node@5.1.1': + resolution: {integrity: sha512-71grXU6+5hl+3CL3joOxlj/AW6rmhthuTlG0fRqsTrhPArQBpZuUFzCIlKOGdcafLUa/i1hBdV78ZxJdlvRA+g==} + engines: {node: '>=20'} + + '@azure/storage-blob@12.31.0': + resolution: {integrity: sha512-DBgNv10aCSxopt92DkTDD0o9xScXeBqPKGmR50FPZQaEcH4JLQ+GEOGEDv19V5BMkB7kxr+m4h6il/cCDPvmHg==} + engines: {node: '>=20.0.0'} + + '@azure/storage-common@12.3.0': + resolution: {integrity: sha512-/OFHhy86aG5Pe8dP5tsp+BuJ25JOAl9yaMU3WZbkeoiFMHFtJ7tu5ili7qEdBXNW9G5lDB19trwyI6V49F/8iQ==} + engines: {node: '>=20.0.0'} + + '@esbuild/aix-ppc64@0.27.4': + resolution: {integrity: sha512-cQPwL2mp2nSmHHJlCyoXgHGhbEPMrEEU5xhkcy3Hs/O7nGZqEpZ2sUtLaL9MORLtDfRvVl2/3PAuEkYZH0Ty8Q==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.27.4': + resolution: {integrity: sha512-gdLscB7v75wRfu7QSm/zg6Rx29VLdy9eTr2t44sfTW7CxwAtQghZ4ZnqHk3/ogz7xao0QAgrkradbBzcqFPasw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.27.4': + resolution: {integrity: sha512-X9bUgvxiC8CHAGKYufLIHGXPJWnr0OCdR0anD2e21vdvgCI8lIfqFbnoeOz7lBjdrAGUhqLZLcQo6MLhTO2DKQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.27.4': + resolution: {integrity: sha512-PzPFnBNVF292sfpfhiyiXCGSn9HZg5BcAz+ivBuSsl6Rk4ga1oEXAamhOXRFyMcjwr2DVtm40G65N3GLeH1Lvw==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.27.4': + resolution: {integrity: sha512-b7xaGIwdJlht8ZFCvMkpDN6uiSmnxxK56N2GDTMYPr2/gzvfdQN8rTfBsvVKmIVY/X7EM+/hJKEIbbHs9oA4tQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.27.4': + resolution: {integrity: sha512-sR+OiKLwd15nmCdqpXMnuJ9W2kpy0KigzqScqHI3Hqwr7IXxBp3Yva+yJwoqh7rE8V77tdoheRYataNKL4QrPw==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.27.4': + resolution: {integrity: sha512-jnfpKe+p79tCnm4GVav68A7tUFeKQwQyLgESwEAUzyxk/TJr4QdGog9sqWNcUbr/bZt/O/HXouspuQDd9JxFSw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.27.4': + resolution: {integrity: sha512-2kb4ceA/CpfUrIcTUl1wrP/9ad9Atrp5J94Lq69w7UwOMolPIGrfLSvAKJp0RTvkPPyn6CIWrNy13kyLikZRZQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.27.4': + resolution: {integrity: sha512-7nQOttdzVGth1iz57kxg9uCz57dxQLHWxopL6mYuYthohPKEK0vU0C3O21CcBK6KDlkYVcnDXY099HcCDXd9dA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.27.4': + resolution: {integrity: sha512-aBYgcIxX/wd5n2ys0yESGeYMGF+pv6g0DhZr3G1ZG4jMfruU9Tl1i2Z+Wnj9/KjGz1lTLCcorqE2viePZqj4Eg==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.27.4': + resolution: {integrity: sha512-oPtixtAIzgvzYcKBQM/qZ3R+9TEUd1aNJQu0HhGyqtx6oS7qTpvjheIWBbes4+qu1bNlo2V4cbkISr8q6gRBFA==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.27.4': + resolution: {integrity: sha512-8mL/vh8qeCoRcFH2nM8wm5uJP+ZcVYGGayMavi8GmRJjuI3g1v6Z7Ni0JJKAJW+m0EtUuARb6Lmp4hMjzCBWzA==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.27.4': + resolution: {integrity: sha512-1RdrWFFiiLIW7LQq9Q2NES+HiD4NyT8Itj9AUeCl0IVCA459WnPhREKgwrpaIfTOe+/2rdntisegiPWn/r/aAw==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.27.4': + resolution: {integrity: sha512-tLCwNG47l3sd9lpfyx9LAGEGItCUeRCWeAx6x2Jmbav65nAwoPXfewtAdtbtit/pJFLUWOhpv0FpS6GQAmPrHA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.27.4': + resolution: {integrity: sha512-BnASypppbUWyqjd1KIpU4AUBiIhVr6YlHx/cnPgqEkNoVOhHg+YiSVxM1RLfiy4t9cAulbRGTNCKOcqHrEQLIw==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.27.4': + resolution: {integrity: sha512-+eUqgb/Z7vxVLezG8bVB9SfBie89gMueS+I0xYh2tJdw3vqA/0ImZJ2ROeWwVJN59ihBeZ7Tu92dF/5dy5FttA==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.27.4': + resolution: {integrity: sha512-S5qOXrKV8BQEzJPVxAwnryi2+Iq5pB40gTEIT69BQONqR7JH1EPIcQ/Uiv9mCnn05jff9umq/5nqzxlqTOg9NA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.27.4': + resolution: {integrity: sha512-xHT8X4sb0GS8qTqiwzHqpY00C95DPAq7nAwX35Ie/s+LO9830hrMd3oX0ZMKLvy7vsonee73x0lmcdOVXFzd6Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.27.4': + resolution: {integrity: sha512-RugOvOdXfdyi5Tyv40kgQnI0byv66BFgAqjdgtAKqHoZTbTF2QqfQrFwa7cHEORJf6X2ht+l9ABLMP0dnKYsgg==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.27.4': + resolution: {integrity: sha512-2MyL3IAaTX+1/qP0O1SwskwcwCoOI4kV2IBX1xYnDDqthmq5ArrW94qSIKCAuRraMgPOmG0RDTA74mzYNQA9ow==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.27.4': + resolution: {integrity: sha512-u8fg/jQ5aQDfsnIV6+KwLOf1CmJnfu1ShpwqdwC0uA7ZPwFws55Ngc12vBdeUdnuWoQYx/SOQLGDcdlfXhYmXQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.27.4': + resolution: {integrity: sha512-JkTZrl6VbyO8lDQO3yv26nNr2RM2yZzNrNHEsj9bm6dOwwu9OYN28CjzZkH57bh4w0I2F7IodpQvUAEd1mbWXg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.27.4': + resolution: {integrity: sha512-/gOzgaewZJfeJTlsWhvUEmUG4tWEY2Spp5M20INYRg2ZKl9QPO3QEEgPeRtLjEWSW8FilRNacPOg8R1uaYkA6g==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.27.4': + resolution: {integrity: sha512-Z9SExBg2y32smoDQdf1HRwHRt6vAHLXcxD2uGgO/v2jK7Y718Ix4ndsbNMU/+1Qiem9OiOdaqitioZwxivhXYg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.27.4': + resolution: {integrity: sha512-DAyGLS0Jz5G5iixEbMHi5KdiApqHBWMGzTtMiJ72ZOLhbu/bzxgAe8Ue8CTS3n3HbIUHQz/L51yMdGMeoxXNJw==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.27.4': + resolution: {integrity: sha512-+knoa0BDoeXgkNvvV1vvbZX4+hizelrkwmGJBdT17t8FNPwG2lKemmuMZlmaNQ3ws3DKKCxpb4zRZEIp3UxFCg==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@fastify/ajv-compiler@3.6.0': + resolution: {integrity: sha512-LwdXQJjmMD+GwLOkP7TVC68qa+pSSogeWWmznRJ/coyTcfe9qA05AHFSe1eZFwK6q+xVRpChnvFUkf1iYaSZsQ==} + + '@fastify/cors@9.0.1': + resolution: {integrity: sha512-YY9Ho3ovI+QHIL2hW+9X4XqQjXLjJqsU+sMV/xFsxZkE8p3GNnYVFpoOxF7SsP5ZL76gwvbo3V9L+FIekBGU4Q==} + + '@fastify/error@3.4.1': + resolution: {integrity: sha512-wWSvph+29GR783IhmvdwWnN4bUxTD01Vm5Xad4i7i1VuAOItLvbPAb69sb0IQ2N57yprvhNIwAP5B6xfKTmjmQ==} + + '@fastify/fast-json-stringify-compiler@4.3.0': + resolution: {integrity: sha512-aZAXGYo6m22Fk1zZzEUKBvut/CIIQe/BapEORnxiD5Qr0kPHqqI69NtEMCme74h+at72sPhbkb4ZrLd1W3KRLA==} + + '@fastify/helmet@11.1.1': + resolution: {integrity: sha512-pjJxjk6SLEimITWadtYIXt6wBMfFC1I6OQyH/jYVCqSAn36sgAIFjeNiibHtifjCd+e25442pObis3Rjtame6A==} + + '@fastify/merge-json-schemas@0.1.1': + resolution: {integrity: sha512-fERDVz7topgNjtXsJTTW1JKLy0rhuLRcquYqNR9rF7OcVpCa2OVW49ZPDIhaRRCaUuvVxI+N416xUoF76HNSXA==} + + '@fastify/rate-limit@9.1.0': + resolution: {integrity: sha512-h5dZWCkuZXN0PxwqaFQLxeln8/LNwQwH9popywmDCFdKfgpi4b/HoMH1lluy6P+30CG9yzzpSpwTCIPNB9T1JA==} + + '@ioredis/commands@1.5.0': + resolution: {integrity: sha512-eUgLqrMf8nJkZxT24JvVRrQya1vZkQh8BBeYNwGDqa5I0VUi8ACx7uFvAaLxintokpTenkK6DASvo/bvNbBGow==} + + '@isaacs/cliui@9.0.0': + resolution: {integrity: sha512-AokJm4tuBHillT+FpMtxQ60n8ObyXBatq7jD2/JA9dxbDDokKQm8KMht5ibGzLVU9IJDIKK4TPKgMHEYMn3lMg==} + engines: {node: '>=18'} + + '@lukeed/ms@2.0.2': + resolution: {integrity: sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA==} + engines: {node: '>=8'} + + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': + resolution: {integrity: sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==} + cpu: [arm64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3': + resolution: {integrity: sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==} + cpu: [x64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3': + resolution: {integrity: sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==} + cpu: [arm64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3': + resolution: {integrity: sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==} + cpu: [arm] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3': + resolution: {integrity: sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==} + cpu: [x64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3': + resolution: {integrity: sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==} + cpu: [x64] + os: [win32] + + '@pinojs/redact@0.4.0': + resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} + + '@types/node@20.19.37': + resolution: {integrity: sha512-8kzdPJ3FsNsVIurqBs7oodNnCEVbni9yUEkaHbgptDACOPW04jimGagZ51E6+lXUwJjgnBw+hyko/lkFWCldqw==} + + '@types/pg@8.18.0': + resolution: {integrity: sha512-gT+oueVQkqnj6ajGJXblFR4iavIXWsGAFCk3dP4Kki5+a9R4NMt0JARdk6s8cUKcfUoqP5dAtDSLU8xYUTFV+Q==} + + '@typespec/ts-http-runtime@0.3.4': + resolution: {integrity: sha512-CI0NhTrz4EBaa0U+HaaUZrJhPoso8sG7ZFya8uQoBA57fjzrjRSv87ekCjLZOFExN+gXE/z0xuN2QfH4H2HrLQ==} + engines: {node: '>=20.0.0'} + + abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + + abstract-logging@2.0.1: + resolution: {integrity: sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==} + + agent-base@7.1.4: + resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} + engines: {node: '>= 14'} + + ajv-formats@2.1.1: + resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==} + peerDependencies: + ajv: ^8.0.0 + peerDependenciesMeta: + ajv: + optional: true + + ajv-formats@3.0.1: + resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==} + peerDependencies: + ajv: ^8.0.0 + peerDependenciesMeta: + ajv: + optional: true + + ajv@8.18.0: + resolution: {integrity: sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==} + + atomic-sleep@1.0.0: + resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} + engines: {node: '>=8.0.0'} + + avvio@8.4.0: + resolution: {integrity: sha512-CDSwaxINFy59iNwhYnkvALBwZiTydGkOecZyPkqBpABYR1KqGEsET0VOOYDwtleZSUIdeY36DC2bSZ24CO1igA==} + + balanced-match@4.0.4: + resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==} + engines: {node: 18 || 20 || >=22} + + base64-js@1.5.1: + resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + + brace-expansion@5.0.4: + resolution: {integrity: sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==} + engines: {node: 18 || 20 || >=22} + + buffer-equal-constant-time@1.0.1: + resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} + + buffer@6.0.3: + resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + + bullmq@5.71.0: + resolution: {integrity: sha512-aeNWh4drsafSKnAJeiNH/nZP/5O8ZdtdMbnOPZmpjXj7NZUP5YC901U3bIH41iZValm7d1i3c34ojv7q31m30w==} + + bundle-name@4.1.0: + resolution: {integrity: sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==} + engines: {node: '>=18'} + + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + + cookie@0.7.2: + resolution: {integrity: sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==} + engines: {node: '>= 0.6'} + + cron-parser@4.9.0: + resolution: {integrity: sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==} + engines: {node: '>=12.0.0'} + + cross-spawn@7.0.6: + resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} + engines: {node: '>= 8'} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + default-browser-id@5.0.1: + resolution: {integrity: sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q==} + engines: {node: '>=18'} + + default-browser@5.5.0: + resolution: {integrity: sha512-H9LMLr5zwIbSxrmvikGuI/5KGhZ8E2zH3stkMgM5LpOWDutGM2JZaj460Udnf1a+946zc7YBgrqEWwbk7zHvGw==} + engines: {node: '>=18'} + + define-lazy-prop@3.0.0: + resolution: {integrity: sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==} + engines: {node: '>=12'} + + denque@2.1.0: + resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} + engines: {node: '>=0.10'} + + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + ecdsa-sig-formatter@1.0.11: + resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} + + esbuild@0.27.4: + resolution: {integrity: sha512-Rq4vbHnYkK5fws5NF7MYTU68FPRE1ajX7heQ/8QXXWqNgqqJ/GkmmyxIzUnf2Sr/bakf8l54716CcMGHYhMrrQ==} + engines: {node: '>=18'} + hasBin: true + + event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + + events@3.3.0: + resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} + engines: {node: '>=0.8.x'} + + fast-content-type-parse@1.1.0: + resolution: {integrity: sha512-fBHHqSTFLVnR61C+gltJuE5GkVQMV0S2nqUO8TJ+5Z3qAKG8vAx4FKai1s5jq/inV1+sREynIWSuQ6HgoSXpDQ==} + + fast-decode-uri-component@1.0.1: + resolution: {integrity: sha512-WKgKWg5eUxvRZGwW8FvfbaH7AXSh2cL+3j5fMGzUMCxWBJ3dV3a7Wz8y2f/uQ0e3B6WmodD3oS54jTQ9HVTIIg==} + + fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + + fast-json-stringify@5.16.1: + resolution: {integrity: sha512-KAdnLvy1yu/XrRtP+LJnxbBGrhN+xXu+gt3EUvZhYGKCr3lFHq/7UFJHHFgmJKoqlh6B40bZLEv7w46B0mqn1g==} + + fast-querystring@1.1.2: + resolution: {integrity: sha512-g6KuKWmFXc0fID8WWH0jit4g0AGBoJhCkJMb1RmbsSEUNvQ+ZC8D6CUZ+GtF8nMzSPXnhiePyyqqipzNNEnHjg==} + + fast-redact@3.5.0: + resolution: {integrity: sha512-dwsoQlS7h9hMeYUq1W++23NDcBLV4KqONnITDV9DjfS3q1SgDGVrBdvvTLUotWtPSD7asWDV9/CmsZPy8Hf70A==} + engines: {node: '>=6'} + + fast-uri@2.4.0: + resolution: {integrity: sha512-ypuAmmMKInk5q7XcepxlnUWDLWv4GFtaJqAzWKqn62IpQ3pejtr5dTVbt3vwqVaMKmkNR55sTT+CqUKIaT21BA==} + + fast-uri@3.1.0: + resolution: {integrity: sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==} + + fast-xml-builder@1.1.4: + resolution: {integrity: sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg==} + + fast-xml-parser@5.5.6: + resolution: {integrity: sha512-3+fdZyBRVg29n4rXP0joHthhcHdPUHaIC16cuyyd1iLsuaO6Vea36MPrxgAzbZna8lhvZeRL8Bc9GP56/J9xEw==} + hasBin: true + + fastify-plugin@4.5.1: + resolution: {integrity: sha512-stRHYGeuqpEZTL1Ef0Ovr2ltazUT9g844X5z/zEBFLG8RYlpDiOCIG+ATvYEp+/zmc7sN29mcIMp8gvYplYPIQ==} + + fastify@4.29.1: + resolution: {integrity: sha512-m2kMNHIG92tSNWv+Z3UeTR9AWLLuo7KctC7mlFPtMEVrfjIhmQhkQnT9v15qA/BfVq3vvj134Y0jl9SBje3jXQ==} + + fastq@1.20.1: + resolution: {integrity: sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==} + + find-my-way@8.2.2: + resolution: {integrity: sha512-Dobi7gcTEq8yszimcfp/R7+owiT4WncAJ7VTTgFH1jYJ5GaG1FbhjwDG820hptN0QDFvzVY3RfCzdInvGPGzjA==} + engines: {node: '>=14'} + + foreground-child@3.3.1: + resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} + engines: {node: '>=14'} + + forwarded@0.2.0: + resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} + engines: {node: '>= 0.6'} + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + get-tsconfig@4.13.6: + resolution: {integrity: sha512-shZT/QMiSHc/YBLxxOkMtgSid5HFoauqCE3/exfsEcwg1WkeqjG+V40yBbBrsD+jW2HDXcs28xOfcbm2jI8Ddw==} + + glob@11.1.0: + resolution: {integrity: sha512-vuNwKSaKiqm7g0THUBu2x7ckSs3XJLXE+2ssL7/MfTGPLLcrJQ/4Uq1CjPTtO5cCIiRxqvN6Twy1qOwhL0Xjcw==} + engines: {node: 20 || >=22} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me + hasBin: true + + helmet@7.2.0: + resolution: {integrity: sha512-ZRiwvN089JfMXokizgqEPXsl2Guk094yExfoDXR0cBYWxtBbaSww/w+vT4WEJsBW2iTUi1GgZ6swmoug3Oy4Xw==} + engines: {node: '>=16.0.0'} + + http-proxy-agent@7.0.2: + resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==} + engines: {node: '>= 14'} + + https-proxy-agent@7.0.6: + resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} + engines: {node: '>= 14'} + + ieee754@1.2.1: + resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + + ioredis@5.9.3: + resolution: {integrity: sha512-VI5tMCdeoxZWU5vjHWsiE/Su76JGhBvWF1MJnV9ZtGltHk9BmD48oDq8Tj8haZ85aceXZMxLNDQZRVo5QKNgXA==} + engines: {node: '>=12.22.0'} + + ipaddr.js@1.9.1: + resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} + engines: {node: '>= 0.10'} + + is-docker@3.0.0: + resolution: {integrity: sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + hasBin: true + + is-inside-container@1.0.0: + resolution: {integrity: sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==} + engines: {node: '>=14.16'} + hasBin: true + + is-wsl@3.1.1: + resolution: {integrity: sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==} + engines: {node: '>=16'} + + isexe@2.0.0: + resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + + jackspeak@4.2.3: + resolution: {integrity: sha512-ykkVRwrYvFm1nb2AJfKKYPr0emF6IiXDYUaFx4Zn9ZuIH7MrzEZ3sD5RlqGXNRpHtvUHJyOnCEFxOlNDtGo7wg==} + engines: {node: 20 || >=22} + + json-schema-ref-resolver@1.0.1: + resolution: {integrity: sha512-EJAj1pgHc1hxF6vo2Z3s69fMjO1INq6eGHXZ8Z6wCQeldCuwxGK9Sxf4/cScGn3FZubCVUehfWtcDM/PLteCQw==} + + json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + + jsonwebtoken@9.0.3: + resolution: {integrity: sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==} + engines: {node: '>=12', npm: '>=6'} + + jwa@2.0.1: + resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} + + jws@4.0.1: + resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} + + light-my-request@5.14.0: + resolution: {integrity: sha512-aORPWntbpH5esaYpGOOmri0OHDOe3wC5M2MQxZ9dvMLZm6DnaAn0kJlcbU9hwsQgLzmZyReKwFwwPkR+nHu5kA==} + + lodash.defaults@4.2.0: + resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} + + lodash.includes@4.3.0: + resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==} + + lodash.isarguments@3.1.0: + resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} + + lodash.isboolean@3.0.3: + resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==} + + lodash.isinteger@4.0.4: + resolution: {integrity: sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==} + + lodash.isnumber@3.0.3: + resolution: {integrity: sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==} + + lodash.isplainobject@4.0.6: + resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==} + + lodash.isstring@4.0.1: + resolution: {integrity: sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==} + + lodash.once@4.1.1: + resolution: {integrity: sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==} + + lru-cache@11.2.7: + resolution: {integrity: sha512-aY/R+aEsRelme17KGQa/1ZSIpLpNYYrhcrepKTZgE+W3WM16YMCaPwOHLHsmopZHELU0Ojin1lPVxKR0MihncA==} + engines: {node: 20 || >=22} + + luxon@3.7.2: + resolution: {integrity: sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==} + engines: {node: '>=12'} + + minimatch@10.2.4: + resolution: {integrity: sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==} + engines: {node: 18 || 20 || >=22} + + minipass@7.1.3: + resolution: {integrity: sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==} + engines: {node: '>=16 || 14 >=14.17'} + + mnemonist@0.39.6: + resolution: {integrity: sha512-A/0v5Z59y63US00cRSLiloEIw3t5G+MiKz4BhX21FI+YBJXBOGW0ohFxTxO08dsOYlzxo87T7vGfZKYp2bcAWA==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + msgpackr-extract@3.0.3: + resolution: {integrity: sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==} + hasBin: true + + msgpackr@1.11.5: + resolution: {integrity: sha512-UjkUHN0yqp9RWKy0Lplhh+wlpdt9oQBYgULZOiFhV3VclSF1JnSQWZ5r9gORQlNYaUKQoR8itv7g7z1xDDuACA==} + + node-abort-controller@3.1.1: + resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==} + + node-gyp-build-optional-packages@5.2.2: + resolution: {integrity: sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==} + hasBin: true + + obliterator@2.0.5: + resolution: {integrity: sha512-42CPE9AhahZRsMNslczq0ctAEtqk8Eka26QofnqC346BZdHDySk3LWka23LI7ULIw11NmltpiLagIq8gBozxTw==} + + on-exit-leak-free@2.1.2: + resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==} + engines: {node: '>=14.0.0'} + + open@10.2.0: + resolution: {integrity: sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==} + engines: {node: '>=18'} + + package-json-from-dist@1.0.1: + resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} + + path-expression-matcher@1.1.3: + resolution: {integrity: sha512-qdVgY8KXmVdJZRSS1JdEPOKPdTiEK/pi0RkcT2sw1RhXxohdujUlJFPuS1TSkevZ9vzd3ZlL7ULl1MHGTApKzQ==} + engines: {node: '>=14.0.0'} + + path-key@3.1.1: + resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} + engines: {node: '>=8'} + + path-scurry@2.0.2: + resolution: {integrity: sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==} + engines: {node: 18 || 20 || >=22} + + pg-cloudflare@1.3.0: + resolution: {integrity: sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==} + + pg-connection-string@2.12.0: + resolution: {integrity: sha512-U7qg+bpswf3Cs5xLzRqbXbQl85ng0mfSV/J0nnA31MCLgvEaAo7CIhmeyrmJpOr7o+zm0rXK+hNnT5l9RHkCkQ==} + + pg-int8@1.0.1: + resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} + engines: {node: '>=4.0.0'} + + pg-pool@3.13.0: + resolution: {integrity: sha512-gB+R+Xud1gLFuRD/QgOIgGOBE2KCQPaPwkzBBGC9oG69pHTkhQeIuejVIk3/cnDyX39av2AxomQiyPT13WKHQA==} + peerDependencies: + pg: '>=8.0' + + pg-protocol@1.13.0: + resolution: {integrity: sha512-zzdvXfS6v89r6v7OcFCHfHlyG/wvry1ALxZo4LqgUoy7W9xhBDMaqOuMiF3qEV45VqsN6rdlcehHrfDtlCPc8w==} + + pg-types@2.2.0: + resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} + engines: {node: '>=4'} + + pg@8.20.0: + resolution: {integrity: sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==} + engines: {node: '>= 16.0.0'} + peerDependencies: + pg-native: '>=3.0.1' + peerDependenciesMeta: + pg-native: + optional: true + + pgpass@1.0.5: + resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} + + pino-abstract-transport@1.2.0: + resolution: {integrity: sha512-Guhh8EZfPCfH+PMXAb6rKOjGQEoy0xlAIn+irODG5kgfYV+BQ0rGYYWTIel3P5mmyXqkYkPmdIkywsn6QKUR1Q==} + + pino-abstract-transport@2.0.0: + resolution: {integrity: sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw==} + + pino-std-serializers@6.2.2: + resolution: {integrity: sha512-cHjPPsE+vhj/tnhCy/wiMh3M3z3h/j15zHQX+S9GkTBgqJuTuJzYJ4gUyACLhDaJ7kk9ba9iRDmbH2tJU03OiA==} + + pino-std-serializers@7.1.0: + resolution: {integrity: sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==} + + pino@8.21.0: + resolution: {integrity: sha512-ip4qdzjkAyDDZklUaZkcRFb2iA118H9SgRh8yzTkSQK8HilsOJF7rSY8HoW5+I0M46AZgX/pxbprf2vvzQCE0Q==} + hasBin: true + + pino@9.14.0: + resolution: {integrity: sha512-8OEwKp5juEvb/MjpIc4hjqfgCNysrS94RIOMXYvpYCdm/jglrKEiAYmiumbmGhCvs+IcInsphYDFwqrjr7398w==} + hasBin: true + + postgres-array@2.0.0: + resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} + engines: {node: '>=4'} + + postgres-bytea@1.0.1: + resolution: {integrity: sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==} + engines: {node: '>=0.10.0'} + + postgres-date@1.0.7: + resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} + engines: {node: '>=0.10.0'} + + postgres-interval@1.2.0: + resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} + engines: {node: '>=0.10.0'} + + prettier@3.8.1: + resolution: {integrity: sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==} + engines: {node: '>=14'} + hasBin: true + + process-warning@3.0.0: + resolution: {integrity: sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ==} + + process-warning@5.0.0: + resolution: {integrity: sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==} + + process@0.11.10: + resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} + engines: {node: '>= 0.6.0'} + + proxy-addr@2.0.7: + resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} + engines: {node: '>= 0.10'} + + quick-format-unescaped@4.0.4: + resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==} + + readable-stream@4.7.0: + resolution: {integrity: sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + real-require@0.2.0: + resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==} + engines: {node: '>= 12.13.0'} + + redis-errors@1.2.0: + resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} + engines: {node: '>=4'} + + redis-parser@3.0.0: + resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} + engines: {node: '>=4'} + + require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + + resolve-pkg-maps@1.0.0: + resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} + + ret@0.4.3: + resolution: {integrity: sha512-0f4Memo5QP7WQyUEAYUO3esD/XjOc3Zjjg5CPsAq1p8sIu0XPeMbHJemKA0BO7tV0X7+A0FoEpbmHXWxPyD3wQ==} + engines: {node: '>=10'} + + reusify@1.1.0: + resolution: {integrity: sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==} + engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + + rfdc@1.4.1: + resolution: {integrity: sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==} + + run-applescript@7.1.0: + resolution: {integrity: sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==} + engines: {node: '>=18'} + + safe-buffer@5.2.1: + resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + + safe-regex2@3.1.0: + resolution: {integrity: sha512-RAAZAGbap2kBfbVhvmnTFv73NWLMvDGOITFYTZBAaY8eR+Ir4ef7Up/e7amo+y1+AH+3PtLkrt9mvcTsG9LXug==} + + safe-stable-stringify@2.5.0: + resolution: {integrity: sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==} + engines: {node: '>=10'} + + secure-json-parse@2.7.0: + resolution: {integrity: sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==} + + semver@7.7.4: + resolution: {integrity: sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==} + engines: {node: '>=10'} + hasBin: true + + set-cookie-parser@2.7.2: + resolution: {integrity: sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==} + + shebang-command@2.0.0: + resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} + engines: {node: '>=8'} + + shebang-regex@3.0.0: + resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} + engines: {node: '>=8'} + + signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + + sonic-boom@3.8.1: + resolution: {integrity: sha512-y4Z8LCDBuum+PBP3lSV7RHrXscqksve/bi0as7mhwVnBW+/wUqKT/2Kb7um8yqcFy0duYbbPxzt89Zy2nOCaxg==} + + sonic-boom@4.2.1: + resolution: {integrity: sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==} + + split2@4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} + engines: {node: '>= 10.x'} + + standard-as-callback@2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + + string_decoder@1.3.0: + resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + + strnum@2.2.0: + resolution: {integrity: sha512-Y7Bj8XyJxnPAORMZj/xltsfo55uOiyHcU2tnAVzHUnSJR/KsEX+9RoDeXEnsXtl/CX4fAcrt64gZ13aGaWPeBg==} + + thread-stream@2.7.0: + resolution: {integrity: sha512-qQiRWsU/wvNolI6tbbCKd9iKaTnCXsTwVxhhKM6nctPdujTyztjlbUkUTUymidWcMnZ5pWR0ej4a0tjsW021vw==} + + thread-stream@3.1.0: + resolution: {integrity: sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==} + + toad-cache@3.7.0: + resolution: {integrity: sha512-/m8M+2BJUpoJdgAHoG+baCwBT+tf2VraSfkBgl0Y00qIWt41DJ8R5B8nsEw0I58YwF5IZH6z24/2TobDKnqSWw==} + engines: {node: '>=12'} + + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + + tsx@4.21.0: + resolution: {integrity: sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==} + engines: {node: '>=18.0.0'} + hasBin: true + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + + uuid@11.1.0: + resolution: {integrity: sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==} + hasBin: true + + uuid@13.0.0: + resolution: {integrity: sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==} + hasBin: true + + uuid@8.3.2: + resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} + hasBin: true + + which@2.0.2: + resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} + engines: {node: '>= 8'} + hasBin: true + + wsl-utils@0.1.0: + resolution: {integrity: sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw==} + engines: {node: '>=18'} + + xtend@4.0.2: + resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} + engines: {node: '>=0.4'} + + zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + +snapshots: + + '@azure/abort-controller@2.1.2': + dependencies: + tslib: 2.8.1 + + '@azure/core-auth@1.10.1': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-util': 1.13.1 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/core-client@1.10.1': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-auth': 1.10.1 + '@azure/core-rest-pipeline': 1.23.0 + '@azure/core-tracing': 1.3.1 + '@azure/core-util': 1.13.1 + '@azure/logger': 1.3.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/core-http-compat@2.3.2(@azure/core-client@1.10.1)(@azure/core-rest-pipeline@1.23.0)': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-client': 1.10.1 + '@azure/core-rest-pipeline': 1.23.0 + + '@azure/core-lro@2.7.2': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-util': 1.13.1 + '@azure/logger': 1.3.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/core-paging@1.6.2': + dependencies: + tslib: 2.8.1 + + '@azure/core-rest-pipeline@1.23.0': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-auth': 1.10.1 + '@azure/core-tracing': 1.3.1 + '@azure/core-util': 1.13.1 + '@azure/logger': 1.3.0 + '@typespec/ts-http-runtime': 0.3.4 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/core-tracing@1.3.1': + dependencies: + tslib: 2.8.1 + + '@azure/core-util@1.13.1': + dependencies: + '@azure/abort-controller': 2.1.2 + '@typespec/ts-http-runtime': 0.3.4 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/core-xml@1.5.0': + dependencies: + fast-xml-parser: 5.5.6 + tslib: 2.8.1 + + '@azure/identity@4.13.1': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-auth': 1.10.1 + '@azure/core-client': 1.10.1 + '@azure/core-rest-pipeline': 1.23.0 + '@azure/core-tracing': 1.3.1 + '@azure/core-util': 1.13.1 + '@azure/logger': 1.3.0 + '@azure/msal-browser': 5.6.1 + '@azure/msal-node': 5.1.1 + open: 10.2.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/logger@1.3.0': + dependencies: + '@typespec/ts-http-runtime': 0.3.4 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/msal-browser@5.6.1': + dependencies: + '@azure/msal-common': 16.4.0 + + '@azure/msal-common@16.4.0': {} + + '@azure/msal-node@5.1.1': + dependencies: + '@azure/msal-common': 16.4.0 + jsonwebtoken: 9.0.3 + uuid: 8.3.2 + + '@azure/storage-blob@12.31.0': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-auth': 1.10.1 + '@azure/core-client': 1.10.1 + '@azure/core-http-compat': 2.3.2(@azure/core-client@1.10.1)(@azure/core-rest-pipeline@1.23.0) + '@azure/core-lro': 2.7.2 + '@azure/core-paging': 1.6.2 + '@azure/core-rest-pipeline': 1.23.0 + '@azure/core-tracing': 1.3.1 + '@azure/core-util': 1.13.1 + '@azure/core-xml': 1.5.0 + '@azure/logger': 1.3.0 + '@azure/storage-common': 12.3.0(@azure/core-client@1.10.1) + events: 3.3.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/storage-common@12.3.0(@azure/core-client@1.10.1)': + dependencies: + '@azure/abort-controller': 2.1.2 + '@azure/core-auth': 1.10.1 + '@azure/core-http-compat': 2.3.2(@azure/core-client@1.10.1)(@azure/core-rest-pipeline@1.23.0) + '@azure/core-rest-pipeline': 1.23.0 + '@azure/core-tracing': 1.3.1 + '@azure/core-util': 1.13.1 + '@azure/logger': 1.3.0 + events: 3.3.0 + tslib: 2.8.1 + transitivePeerDependencies: + - '@azure/core-client' + - supports-color + + '@esbuild/aix-ppc64@0.27.4': + optional: true + + '@esbuild/android-arm64@0.27.4': + optional: true + + '@esbuild/android-arm@0.27.4': + optional: true + + '@esbuild/android-x64@0.27.4': + optional: true + + '@esbuild/darwin-arm64@0.27.4': + optional: true + + '@esbuild/darwin-x64@0.27.4': + optional: true + + '@esbuild/freebsd-arm64@0.27.4': + optional: true + + '@esbuild/freebsd-x64@0.27.4': + optional: true + + '@esbuild/linux-arm64@0.27.4': + optional: true + + '@esbuild/linux-arm@0.27.4': + optional: true + + '@esbuild/linux-ia32@0.27.4': + optional: true + + '@esbuild/linux-loong64@0.27.4': + optional: true + + '@esbuild/linux-mips64el@0.27.4': + optional: true + + '@esbuild/linux-ppc64@0.27.4': + optional: true + + '@esbuild/linux-riscv64@0.27.4': + optional: true + + '@esbuild/linux-s390x@0.27.4': + optional: true + + '@esbuild/linux-x64@0.27.4': + optional: true + + '@esbuild/netbsd-arm64@0.27.4': + optional: true + + '@esbuild/netbsd-x64@0.27.4': + optional: true + + '@esbuild/openbsd-arm64@0.27.4': + optional: true + + '@esbuild/openbsd-x64@0.27.4': + optional: true + + '@esbuild/openharmony-arm64@0.27.4': + optional: true + + '@esbuild/sunos-x64@0.27.4': + optional: true + + '@esbuild/win32-arm64@0.27.4': + optional: true + + '@esbuild/win32-ia32@0.27.4': + optional: true + + '@esbuild/win32-x64@0.27.4': + optional: true + + '@fastify/ajv-compiler@3.6.0': + dependencies: + ajv: 8.18.0 + ajv-formats: 2.1.1(ajv@8.18.0) + fast-uri: 2.4.0 + + '@fastify/cors@9.0.1': + dependencies: + fastify-plugin: 4.5.1 + mnemonist: 0.39.6 + + '@fastify/error@3.4.1': {} + + '@fastify/fast-json-stringify-compiler@4.3.0': + dependencies: + fast-json-stringify: 5.16.1 + + '@fastify/helmet@11.1.1': + dependencies: + fastify-plugin: 4.5.1 + helmet: 7.2.0 + + '@fastify/merge-json-schemas@0.1.1': + dependencies: + fast-deep-equal: 3.1.3 + + '@fastify/rate-limit@9.1.0': + dependencies: + '@lukeed/ms': 2.0.2 + fastify-plugin: 4.5.1 + toad-cache: 3.7.0 + + '@ioredis/commands@1.5.0': {} + + '@isaacs/cliui@9.0.0': {} + + '@lukeed/ms@2.0.2': {} + + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3': + optional: true + + '@pinojs/redact@0.4.0': {} + + '@types/node@20.19.37': + dependencies: + undici-types: 6.21.0 + + '@types/pg@8.18.0': + dependencies: + '@types/node': 20.19.37 + pg-protocol: 1.13.0 + pg-types: 2.2.0 + + '@typespec/ts-http-runtime@0.3.4': + dependencies: + http-proxy-agent: 7.0.2 + https-proxy-agent: 7.0.6 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + abort-controller@3.0.0: + dependencies: + event-target-shim: 5.0.1 + + abstract-logging@2.0.1: {} + + agent-base@7.1.4: {} + + ajv-formats@2.1.1(ajv@8.18.0): + optionalDependencies: + ajv: 8.18.0 + + ajv-formats@3.0.1(ajv@8.18.0): + optionalDependencies: + ajv: 8.18.0 + + ajv@8.18.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-uri: 3.1.0 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + + atomic-sleep@1.0.0: {} + + avvio@8.4.0: + dependencies: + '@fastify/error': 3.4.1 + fastq: 1.20.1 + + balanced-match@4.0.4: {} + + base64-js@1.5.1: {} + + brace-expansion@5.0.4: + dependencies: + balanced-match: 4.0.4 + + buffer-equal-constant-time@1.0.1: {} + + buffer@6.0.3: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + + bullmq@5.71.0: + dependencies: + cron-parser: 4.9.0 + ioredis: 5.9.3 + msgpackr: 1.11.5 + node-abort-controller: 3.1.1 + semver: 7.7.4 + tslib: 2.8.1 + uuid: 11.1.0 + transitivePeerDependencies: + - supports-color + + bundle-name@4.1.0: + dependencies: + run-applescript: 7.1.0 + + cluster-key-slot@1.1.2: {} + + cookie@0.7.2: {} + + cron-parser@4.9.0: + dependencies: + luxon: 3.7.2 + + cross-spawn@7.0.6: + dependencies: + path-key: 3.1.1 + shebang-command: 2.0.0 + which: 2.0.2 + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + default-browser-id@5.0.1: {} + + default-browser@5.5.0: + dependencies: + bundle-name: 4.1.0 + default-browser-id: 5.0.1 + + define-lazy-prop@3.0.0: {} + + denque@2.1.0: {} + + detect-libc@2.1.2: + optional: true + + ecdsa-sig-formatter@1.0.11: + dependencies: + safe-buffer: 5.2.1 + + esbuild@0.27.4: + optionalDependencies: + '@esbuild/aix-ppc64': 0.27.4 + '@esbuild/android-arm': 0.27.4 + '@esbuild/android-arm64': 0.27.4 + '@esbuild/android-x64': 0.27.4 + '@esbuild/darwin-arm64': 0.27.4 + '@esbuild/darwin-x64': 0.27.4 + '@esbuild/freebsd-arm64': 0.27.4 + '@esbuild/freebsd-x64': 0.27.4 + '@esbuild/linux-arm': 0.27.4 + '@esbuild/linux-arm64': 0.27.4 + '@esbuild/linux-ia32': 0.27.4 + '@esbuild/linux-loong64': 0.27.4 + '@esbuild/linux-mips64el': 0.27.4 + '@esbuild/linux-ppc64': 0.27.4 + '@esbuild/linux-riscv64': 0.27.4 + '@esbuild/linux-s390x': 0.27.4 + '@esbuild/linux-x64': 0.27.4 + '@esbuild/netbsd-arm64': 0.27.4 + '@esbuild/netbsd-x64': 0.27.4 + '@esbuild/openbsd-arm64': 0.27.4 + '@esbuild/openbsd-x64': 0.27.4 + '@esbuild/openharmony-arm64': 0.27.4 + '@esbuild/sunos-x64': 0.27.4 + '@esbuild/win32-arm64': 0.27.4 + '@esbuild/win32-ia32': 0.27.4 + '@esbuild/win32-x64': 0.27.4 + + event-target-shim@5.0.1: {} + + events@3.3.0: {} + + fast-content-type-parse@1.1.0: {} + + fast-decode-uri-component@1.0.1: {} + + fast-deep-equal@3.1.3: {} + + fast-json-stringify@5.16.1: + dependencies: + '@fastify/merge-json-schemas': 0.1.1 + ajv: 8.18.0 + ajv-formats: 3.0.1(ajv@8.18.0) + fast-deep-equal: 3.1.3 + fast-uri: 2.4.0 + json-schema-ref-resolver: 1.0.1 + rfdc: 1.4.1 + + fast-querystring@1.1.2: + dependencies: + fast-decode-uri-component: 1.0.1 + + fast-redact@3.5.0: {} + + fast-uri@2.4.0: {} + + fast-uri@3.1.0: {} + + fast-xml-builder@1.1.4: + dependencies: + path-expression-matcher: 1.1.3 + + fast-xml-parser@5.5.6: + dependencies: + fast-xml-builder: 1.1.4 + path-expression-matcher: 1.1.3 + strnum: 2.2.0 + + fastify-plugin@4.5.1: {} + + fastify@4.29.1: + dependencies: + '@fastify/ajv-compiler': 3.6.0 + '@fastify/error': 3.4.1 + '@fastify/fast-json-stringify-compiler': 4.3.0 + abstract-logging: 2.0.1 + avvio: 8.4.0 + fast-content-type-parse: 1.1.0 + fast-json-stringify: 5.16.1 + find-my-way: 8.2.2 + light-my-request: 5.14.0 + pino: 9.14.0 + process-warning: 3.0.0 + proxy-addr: 2.0.7 + rfdc: 1.4.1 + secure-json-parse: 2.7.0 + semver: 7.7.4 + toad-cache: 3.7.0 + + fastq@1.20.1: + dependencies: + reusify: 1.1.0 + + find-my-way@8.2.2: + dependencies: + fast-deep-equal: 3.1.3 + fast-querystring: 1.1.2 + safe-regex2: 3.1.0 + + foreground-child@3.3.1: + dependencies: + cross-spawn: 7.0.6 + signal-exit: 4.1.0 + + forwarded@0.2.0: {} + + fsevents@2.3.3: + optional: true + + get-tsconfig@4.13.6: + dependencies: + resolve-pkg-maps: 1.0.0 + + glob@11.1.0: + dependencies: + foreground-child: 3.3.1 + jackspeak: 4.2.3 + minimatch: 10.2.4 + minipass: 7.1.3 + package-json-from-dist: 1.0.1 + path-scurry: 2.0.2 + + helmet@7.2.0: {} + + http-proxy-agent@7.0.2: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + https-proxy-agent@7.0.6: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + ieee754@1.2.1: {} + + ioredis@5.9.3: + dependencies: + '@ioredis/commands': 1.5.0 + cluster-key-slot: 1.1.2 + debug: 4.4.3 + denque: 2.1.0 + lodash.defaults: 4.2.0 + lodash.isarguments: 3.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + + ipaddr.js@1.9.1: {} + + is-docker@3.0.0: {} + + is-inside-container@1.0.0: + dependencies: + is-docker: 3.0.0 + + is-wsl@3.1.1: + dependencies: + is-inside-container: 1.0.0 + + isexe@2.0.0: {} + + jackspeak@4.2.3: + dependencies: + '@isaacs/cliui': 9.0.0 + + json-schema-ref-resolver@1.0.1: + dependencies: + fast-deep-equal: 3.1.3 + + json-schema-traverse@1.0.0: {} + + jsonwebtoken@9.0.3: + dependencies: + jws: 4.0.1 + lodash.includes: 4.3.0 + lodash.isboolean: 3.0.3 + lodash.isinteger: 4.0.4 + lodash.isnumber: 3.0.3 + lodash.isplainobject: 4.0.6 + lodash.isstring: 4.0.1 + lodash.once: 4.1.1 + ms: 2.1.3 + semver: 7.7.4 + + jwa@2.0.1: + dependencies: + buffer-equal-constant-time: 1.0.1 + ecdsa-sig-formatter: 1.0.11 + safe-buffer: 5.2.1 + + jws@4.0.1: + dependencies: + jwa: 2.0.1 + safe-buffer: 5.2.1 + + light-my-request@5.14.0: + dependencies: + cookie: 0.7.2 + process-warning: 3.0.0 + set-cookie-parser: 2.7.2 + + lodash.defaults@4.2.0: {} + + lodash.includes@4.3.0: {} + + lodash.isarguments@3.1.0: {} + + lodash.isboolean@3.0.3: {} + + lodash.isinteger@4.0.4: {} + + lodash.isnumber@3.0.3: {} + + lodash.isplainobject@4.0.6: {} + + lodash.isstring@4.0.1: {} + + lodash.once@4.1.1: {} + + lru-cache@11.2.7: {} + + luxon@3.7.2: {} + + minimatch@10.2.4: + dependencies: + brace-expansion: 5.0.4 + + minipass@7.1.3: {} + + mnemonist@0.39.6: + dependencies: + obliterator: 2.0.5 + + ms@2.1.3: {} + + msgpackr-extract@3.0.3: + dependencies: + node-gyp-build-optional-packages: 5.2.2 + optionalDependencies: + '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.3 + optional: true + + msgpackr@1.11.5: + optionalDependencies: + msgpackr-extract: 3.0.3 + + node-abort-controller@3.1.1: {} + + node-gyp-build-optional-packages@5.2.2: + dependencies: + detect-libc: 2.1.2 + optional: true + + obliterator@2.0.5: {} + + on-exit-leak-free@2.1.2: {} + + open@10.2.0: + dependencies: + default-browser: 5.5.0 + define-lazy-prop: 3.0.0 + is-inside-container: 1.0.0 + wsl-utils: 0.1.0 + + package-json-from-dist@1.0.1: {} + + path-expression-matcher@1.1.3: {} + + path-key@3.1.1: {} + + path-scurry@2.0.2: + dependencies: + lru-cache: 11.2.7 + minipass: 7.1.3 + + pg-cloudflare@1.3.0: + optional: true + + pg-connection-string@2.12.0: {} + + pg-int8@1.0.1: {} + + pg-pool@3.13.0(pg@8.20.0): + dependencies: + pg: 8.20.0 + + pg-protocol@1.13.0: {} + + pg-types@2.2.0: + dependencies: + pg-int8: 1.0.1 + postgres-array: 2.0.0 + postgres-bytea: 1.0.1 + postgres-date: 1.0.7 + postgres-interval: 1.2.0 + + pg@8.20.0: + dependencies: + pg-connection-string: 2.12.0 + pg-pool: 3.13.0(pg@8.20.0) + pg-protocol: 1.13.0 + pg-types: 2.2.0 + pgpass: 1.0.5 + optionalDependencies: + pg-cloudflare: 1.3.0 + + pgpass@1.0.5: + dependencies: + split2: 4.2.0 + + pino-abstract-transport@1.2.0: + dependencies: + readable-stream: 4.7.0 + split2: 4.2.0 + + pino-abstract-transport@2.0.0: + dependencies: + split2: 4.2.0 + + pino-std-serializers@6.2.2: {} + + pino-std-serializers@7.1.0: {} + + pino@8.21.0: + dependencies: + atomic-sleep: 1.0.0 + fast-redact: 3.5.0 + on-exit-leak-free: 2.1.2 + pino-abstract-transport: 1.2.0 + pino-std-serializers: 6.2.2 + process-warning: 3.0.0 + quick-format-unescaped: 4.0.4 + real-require: 0.2.0 + safe-stable-stringify: 2.5.0 + sonic-boom: 3.8.1 + thread-stream: 2.7.0 + + pino@9.14.0: + dependencies: + '@pinojs/redact': 0.4.0 + atomic-sleep: 1.0.0 + on-exit-leak-free: 2.1.2 + pino-abstract-transport: 2.0.0 + pino-std-serializers: 7.1.0 + process-warning: 5.0.0 + quick-format-unescaped: 4.0.4 + real-require: 0.2.0 + safe-stable-stringify: 2.5.0 + sonic-boom: 4.2.1 + thread-stream: 3.1.0 + + postgres-array@2.0.0: {} + + postgres-bytea@1.0.1: {} + + postgres-date@1.0.7: {} + + postgres-interval@1.2.0: + dependencies: + xtend: 4.0.2 + + prettier@3.8.1: {} + + process-warning@3.0.0: {} + + process-warning@5.0.0: {} + + process@0.11.10: {} + + proxy-addr@2.0.7: + dependencies: + forwarded: 0.2.0 + ipaddr.js: 1.9.1 + + quick-format-unescaped@4.0.4: {} + + readable-stream@4.7.0: + dependencies: + abort-controller: 3.0.0 + buffer: 6.0.3 + events: 3.3.0 + process: 0.11.10 + string_decoder: 1.3.0 + + real-require@0.2.0: {} + + redis-errors@1.2.0: {} + + redis-parser@3.0.0: + dependencies: + redis-errors: 1.2.0 + + require-from-string@2.0.2: {} + + resolve-pkg-maps@1.0.0: {} + + ret@0.4.3: {} + + reusify@1.1.0: {} + + rfdc@1.4.1: {} + + run-applescript@7.1.0: {} + + safe-buffer@5.2.1: {} + + safe-regex2@3.1.0: + dependencies: + ret: 0.4.3 + + safe-stable-stringify@2.5.0: {} + + secure-json-parse@2.7.0: {} + + semver@7.7.4: {} + + set-cookie-parser@2.7.2: {} + + shebang-command@2.0.0: + dependencies: + shebang-regex: 3.0.0 + + shebang-regex@3.0.0: {} + + signal-exit@4.1.0: {} + + sonic-boom@3.8.1: + dependencies: + atomic-sleep: 1.0.0 + + sonic-boom@4.2.1: + dependencies: + atomic-sleep: 1.0.0 + + split2@4.2.0: {} + + standard-as-callback@2.1.0: {} + + string_decoder@1.3.0: + dependencies: + safe-buffer: 5.2.1 + + strnum@2.2.0: {} + + thread-stream@2.7.0: + dependencies: + real-require: 0.2.0 + + thread-stream@3.1.0: + dependencies: + real-require: 0.2.0 + + toad-cache@3.7.0: {} + + tslib@2.8.1: {} + + tsx@4.21.0: + dependencies: + esbuild: 0.27.4 + get-tsconfig: 4.13.6 + optionalDependencies: + fsevents: 2.3.3 + + typescript@5.9.3: {} + + undici-types@6.21.0: {} + + uuid@11.1.0: {} + + uuid@13.0.0: {} + + uuid@8.3.2: {} + + which@2.0.2: + dependencies: + isexe: 2.0.0 + + wsl-utils@0.1.0: + dependencies: + is-wsl: 3.1.1 + + xtend@4.0.2: {} + + zod@3.25.76: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 0000000..96c13c9 --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1,3 @@ +ignoredBuiltDependencies: + - esbuild + - msgpackr-extract diff --git a/sql/main.sql b/sql/main.sql new file mode 100644 index 0000000..5a18f87 --- /dev/null +++ b/sql/main.sql @@ -0,0 +1,57 @@ +CREATE TABLE IF NOT EXISTS videos ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + source_url TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'queued' + CHECK (status IN ('queued', 'processing', 'transcoding', 'uploading', 'completed', 'failed')), + playlist_url TEXT, + sprite_url TEXT, + poster_url TEXT, + error_message TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS video_metadata ( + video_id TEXT PRIMARY KEY REFERENCES videos(id) ON DELETE CASCADE, + duration NUMERIC(10, 6) NOT NULL, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + codec TEXT NOT NULL, + frame_rate NUMERIC(6, 3) DEFAULT 0, + bit_rate BIGINT DEFAULT 0, + size_bytes BIGINT DEFAULT 0, + video_range TEXT DEFAULT 'SDR', + aspect_ratio TEXT, + created_at TIMESTAMPTZ DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS video_renditions ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + video_id TEXT REFERENCES videos(id) ON DELETE CASCADE, + resolution TEXT NOT NULL, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + bitrate INTEGER, + url TEXT, + created_at TIMESTAMPTZ DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_videos_user_id ON videos(user_id); +CREATE INDEX IF NOT EXISTS idx_videos_status ON videos(status); +CREATE INDEX IF NOT EXISTS idx_vmeta_video_id ON video_metadata(video_id); +CREATE INDEX IF NOT EXISTS idx_vrend_video_id ON video_renditions(video_id); + +CREATE OR REPLACE FUNCTION update_updated_at_column() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS set_updated_at ON videos; +CREATE TRIGGER set_updated_at + BEFORE UPDATE ON videos + FOR EACH ROW + EXECUTE FUNCTION update_updated_at_column(); diff --git a/src/application/video.process.ts b/src/application/video.process.ts new file mode 100644 index 0000000..21f9009 --- /dev/null +++ b/src/application/video.process.ts @@ -0,0 +1,155 @@ +import type { + JobData, + ProcessVideoUseCase, + StorageProvider, + TranscodeProvider, + VideoRepository, + ProgressCallback, +} from '../domain/job.interface.js'; +import { WorkerError } from '../domain/errors.js'; +import { pino } from 'pino'; + +const logger = pino({ name: 'ProcessVideo' }); + +/** + * Orchestrates the core video processing pipeline: Probe -> Transcode -> Upload. + * + * @remarks + * - Idempotency: If a job fails midway, retrying it will safely overwrite existing partial state. + * - Cleanup: Guaranteed to remove local intermediate files on both success and failure pathways. + */ +export class ProcessVideo implements ProcessVideoUseCase { + constructor( + private readonly ffmpeg: TranscodeProvider, + private readonly storage: StorageProvider, + private readonly db: VideoRepository, + ) {} + + /** + * Executes the transcoding pipeline and synchronizes state with the database and webhook. + * + * @param job - Job payload from BullMQ. `videoId` acts as the idempotency key in DB/Storage. + * @throws {WorkerError} If any step fails. Process catches this, cleans up, and rethrows + * so the BullMQ wrapper can handle the retry/failure logic based on `.retryable`. + */ + async execute(job: JobData, onProgress?: ProgressCallback): Promise { + const { videoId, sourceUrl, webhookUrl } = job; + logger.info({ videoId, sourceUrl, webhookUrl }, 'Starting video processing pipeline'); + + await this.db.updateStatus(videoId, 'processing'); + + try { + logger.info({ videoId }, 'Step 1/3: Probing source'); + const probeResult = await this.ffmpeg.probe(sourceUrl); + + await this.db.updateStatus(videoId, 'processing'); + logger.info( + { + videoId, + duration: probeResult.duration, + resolution: `${probeResult.width}x${probeResult.height}`, + }, + 'Probe complete', + ); + + await this.db.saveMetadata(videoId, probeResult); + + logger.info({ videoId }, 'Step 2/3: Transcoding HLS'); + await this.db.updateStatus(videoId, 'transcoding'); + + const { outputDir, renditions } = await this.ffmpeg.transcodeHLS( + sourceUrl, + videoId, + probeResult.width, + probeResult.height, + probeResult.duration, + onProgress, + probeResult.frameRate, + probeResult.audioStreams, + probeResult.videoRange, + ); + + logger.info({ videoId }, 'Step 3/3: Uploading HLS Master'); + await this.db.updateStatus(videoId, 'uploading'); + + const masterPlaylistUrl = await this.storage.uploadHLS(outputDir, videoId, onProgress); + + if (renditions && renditions.length > 0) { + const baseUrl = masterPlaylistUrl.substring(0, masterPlaylistUrl.lastIndexOf('/') + 1); + const fullRenditions = renditions.map((r) => ({ + ...r, + url: `${baseUrl}${r.url}`, + })); + await this.db.saveRenditions(videoId, fullRenditions); + } + + logger.info({ videoId }, 'Cleaning up worker directories'); + await this.ffmpeg.cleanup(videoId); + + await this.db.updateStatus(videoId, 'completed', { + playlistUrl: masterPlaylistUrl, + probeResult, + }); + + logger.info( + { videoId, playlistUrl: masterPlaylistUrl }, + 'Pipeline completed successfully', + ); + + if (webhookUrl) { + logger.info({ webhookUrl, videoId }, 'Dialing completion webhook'); + try { + await fetch(webhookUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + videoId, + status: 'COMPLETED', + playlistUrl: masterPlaylistUrl, + probeResult, + }), + }); + } catch (whErr) { + logger.warn({ err: whErr, webhookUrl }, 'Failed to deliver completion webhook'); + } + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + const isRetryable = error instanceof WorkerError ? error.retryable : true; + + logger.error({ err: error, videoId, retryable: isRetryable }, 'Pipeline failed'); + + try { + await this.db.updateStatus(videoId, 'failed', { error: errorMessage }); + } catch (dbErr) { + logger.error({ err: dbErr, videoId }, 'Failed to update DB with error status'); + } + + try { + await this.ffmpeg.cleanup(videoId); + } catch (cleanupErr) { + logger.warn({ err: cleanupErr, videoId }, 'Cleanup failed after error (non-critical)'); + } + + if (webhookUrl) { + logger.info({ webhookUrl, videoId }, 'Dialing failure webhook'); + try { + await fetch(webhookUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + videoId, + status: 'FAILED', + error: errorMessage, + retryable: isRetryable, + }), + }); + } catch (whErr) { + logger.warn({ err: whErr, webhookUrl }, 'Failed to deliver failure webhook'); + } + } + + throw error; + } + } +} diff --git a/src/config/env.ts b/src/config/env.ts new file mode 100644 index 0000000..ac27d9d --- /dev/null +++ b/src/config/env.ts @@ -0,0 +1,45 @@ +/** + * Runtime configuration validated via Zod schema. + * Designed to fail-fast: if any required variable is missing or malformed, the process crashes + * immediately before attempting to bind ports or connect to the database. + */ +import { z } from 'zod'; + +/** + * Strips literal quotes injected by `.env` loaders to prevent DSN/connection string parsing errors. + */ +const unquotedString = z.string().transform((s) => s.replace(/^["'](.*?)["']$/, '$1')); + +const envSchema = z.object({ + NODE_ENV: unquotedString + .pipe(z.enum(['development', 'production', 'test'])) + .default('development'), + PORT: z.coerce.number().default(3000), + REDIS_URL: unquotedString.pipe(z.string().url()), + WORKER_CONCURRENCY: z.coerce.number().default(1), + + TEST_DURATION_SECONDS: z.coerce.number().optional(), + HLS_OUTPUT_MODE: unquotedString.pipe(z.enum(['SINGLE_FILE', 'SEGMENTED'])).default('SEGMENTED'), + + JOB_LOCK_DURATION_MS: z.coerce.number().default(120000), + JOB_LOCK_RENEW_MS: z.coerce.number().default(30000), + + AZURE_UPLOAD_BATCH_SIZE: z.coerce.number().default(20), + AZURE_UPLOAD_RETRIES: z.coerce.number().default(3), + AZURE_STORAGE_CONNECTION_STRING: unquotedString.optional(), + AZURE_STORAGE_ACCOUNT_URL: unquotedString.optional(), + AZURE_STORAGE_CONTAINER_NAME: unquotedString.pipe( + z.string().min(1, 'AZURE_STORAGE_CONTAINER_NAME is required'), + ), + CONTAINER_DIRECTORY_1: unquotedString.pipe( + z.string().min(3, 'CONTAINER_DIRECTORY_1 is required'), + ), + CORS_ORIGIN: unquotedString.default('*'), + DATABASE_URL: unquotedString.pipe(z.string().min(1, 'DATABASE_URL is required')), +}); + +/** + * Validated application configuration. Extracted directly from `process.env`. + * @throws {ZodError} If validation fails during module load. + */ +export const config: z.infer = envSchema.parse(process.env); diff --git a/src/domain/errors.ts b/src/domain/errors.ts new file mode 100644 index 0000000..a129695 --- /dev/null +++ b/src/domain/errors.ts @@ -0,0 +1,38 @@ +/** + * Base custom exception for all domain errors. Wraps the underlying `.cause` for traceability. + * Features a `retryable` flag that signals to the queue wrapper whether to re-queue the job + * (e.g., transient network issue) or fail it permanently (e.g., validation failure). + */ +export class WorkerError extends Error { + public readonly retryable: boolean; + + constructor(message: string, retryable: boolean, options?: ErrorOptions) { + super(message, options); + this.name = this.constructor.name; + this.retryable = retryable; + } +} + +export class SourceNotFoundError extends WorkerError { + constructor(url: string, cause?: Error) { + super(`Source video not found: ${url}`, false, { cause }); + } +} + +export class ValidationError extends WorkerError { + constructor(message: string) { + super(message, false); + } +} + +export class TranscodeError extends WorkerError { + constructor(message: string, cause?: Error) { + super(message, true, { cause }); + } +} + +export class UploadError extends WorkerError { + constructor(message: string, cause?: Error) { + super(message, true, { cause }); + } +} diff --git a/src/domain/job.interface.ts b/src/domain/job.interface.ts new file mode 100644 index 0000000..ca4fae0 --- /dev/null +++ b/src/domain/job.interface.ts @@ -0,0 +1,93 @@ +export interface JobData { + videoId: string; + sourceUrl: string; + userId: string; + webhookUrl?: string; +} + +export interface AudioStreamInfo { + index: number; + language: string; + channels: number; + title: string; +} + +export interface ProbeResult { + duration: number; + width: number; + height: number; + aspectRatio: string; + originalAspectRatio: number; + codec: string; + fileSize: number; + frameRate: number; + audioStreams: AudioStreamInfo[]; + videoRange: string; +} + +export interface VideoRendition { + resolution: string; + width: number; + height: number; + bitrate: number; + url: string; +} + +export interface TranscodeResult { + outputDir: string; + renditions: VideoRendition[]; +} + +export type JobStatus = + | 'queued' + | 'processing' + | 'transcoding' + | 'uploading' + | 'completed' + | 'failed'; + +export interface VideoRepository { + updateStatus( + videoId: string, + status: JobStatus, + metadata?: Record, + ): Promise; + + saveMetadata(videoId: string, metadata: ProbeResult): Promise; + saveRenditions(videoId: string, renditions: VideoRendition[]): Promise; +} + +/** + * Dependency-inversion interface for blob storage providers (e.g., Azure Blob Storage, AWS S3). + */ +export interface StorageProvider { + uploadHLS(folderPath: string, videoId: string, onProgress?: ProgressCallback): Promise; +} + +export type ProgressCallback = (data: { variant: string; percent: number }) => void; + +/** + * Domain boundary around the native FFmpeg system binary. + * Implementors must guarantee that `cleanup()` removes all trailing artifacts from the OS. + */ +export interface TranscodeProvider { + probe(sourceUrl: string): Promise; + + transcodeHLS( + sourceUrl: string, + videoId: string, + sourceWidth: number, + sourceHeight: number, + sourceDuration: number, + onProgress?: ProgressCallback, + sourceFrameRate?: number, + audioStreams?: AudioStreamInfo[], + videoRange?: string, + ): Promise; + + cleanup(videoId: string): Promise; +} + +export interface ProcessVideoUseCase { + execute(job: JobData, onProgress?: ProgressCallback): Promise; +} diff --git a/src/infrastructure/db/db.ts b/src/infrastructure/db/db.ts new file mode 100644 index 0000000..539f15a --- /dev/null +++ b/src/infrastructure/db/db.ts @@ -0,0 +1,138 @@ +import pg from 'pg'; +import { pino } from 'pino'; +import type { + VideoRepository, + JobStatus, + ProbeResult, + VideoRendition, +} from '../../domain/job.interface.js'; + +const logger = pino({ name: 'PostgresVideoRepository' }); + +/** + * PostgreSQL adapter for persisting video state and metadata. + * + * @remarks + * - Manages its own connection pool. Must call `close()` during graceful shutdown to prevent connection leaks. + * - Upserts are used for metadata (`ON CONFLICT (video_id) DO UPDATE`) to safely support job retries (idempotency). + * - `updateStatus` serves as a sanity check: it intentionally throws if the video ID vanishes mid-process. + */ +export class PostgresVideoRepository implements VideoRepository { + private readonly pool: pg.Pool; + + constructor(connectionString: string) { + this.pool = new pg.Pool({ + connectionString, + ssl: { + rejectUnauthorized: false, + }, + }); + } + + async updateStatus( + videoId: string, + status: JobStatus, + metadata?: Record, + ): Promise { + logger.info({ videoId, status }, 'Updating video status'); + + const fields: string[] = ['status = $1', 'updated_at = NOW()']; + const values: unknown[] = [status]; + let paramIndex = 2; + + if (metadata?.playlistUrl) { + fields.push(`playlist_url = $${paramIndex++}`); + values.push(metadata.playlistUrl); + } + if (metadata?.spriteUrl) { + fields.push(`sprite_url = $${paramIndex++}`); + values.push(metadata.spriteUrl); + } + if (metadata?.posterUrl) { + fields.push(`poster_url = $${paramIndex++}`); + values.push(metadata.posterUrl); + } + if (metadata?.error) { + fields.push(`error_message = $${paramIndex++}`); + values.push(metadata.error); + } + + values.push(videoId); + const query = `UPDATE videos SET ${fields.join(', ')} WHERE id = $${paramIndex} RETURNING id`; + + const result = await this.pool.query(query, values); + + if (result.rowCount === 0) { + logger.warn({ videoId, status }, 'Video ID not found in database (update skipped)'); + throw new Error(`Video ID ${videoId} not found in database`); + } + + logger.info({ videoId, status }, 'Video status updated'); + } + + async saveMetadata(videoId: string, meta: ProbeResult): Promise { + const query = ` + INSERT INTO video_metadata (video_id, duration, width, height, codec, size_bytes, frame_rate, aspect_ratio, video_range) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (video_id) DO UPDATE SET + duration = EXCLUDED.duration, + width = EXCLUDED.width, + height = EXCLUDED.height, + codec = EXCLUDED.codec, + size_bytes = EXCLUDED.size_bytes, + frame_rate = EXCLUDED.frame_rate, + aspect_ratio = EXCLUDED.aspect_ratio, + video_range = EXCLUDED.video_range + `; + + try { + await this.pool.query(query, [ + videoId, + meta.duration, + meta.width, + meta.height, + meta.codec, + meta.fileSize, + meta.frameRate, + meta.aspectRatio, + meta.videoRange, + ]); + logger.info({ videoId }, 'Video metadata saved'); + } catch (err) { + logger.error({ videoId, err }, 'Failed to save video metadata'); + throw new Error(`Failed to save metadata: ${(err as Error).message}`); + } + } + + async saveRenditions(videoId: string, renditions: VideoRendition[]): Promise { + if (!renditions || renditions.length === 0) return; + + const client = await this.pool.connect(); + try { + await client.query('BEGIN'); + await client.query('DELETE FROM video_renditions WHERE video_id = $1', [videoId]); + + const query = ` + INSERT INTO video_renditions (video_id, resolution, width, height, bitrate, url) + VALUES ($1, $2, $3, $4, $5, $6) + `; + + for (const r of renditions) { + await client.query(query, [videoId, r.resolution, r.width, r.height, r.bitrate, r.url]); + } + + await client.query('COMMIT'); + logger.info({ videoId, count: renditions.length }, 'Video renditions saved'); + } catch (err) { + await client.query('ROLLBACK'); + logger.error({ videoId, err }, 'Failed to save renditions'); + throw new Error(`Failed to save renditions: ${(err as Error).message}`); + } finally { + client.release(); + } + } + + async close(): Promise { + await this.pool.end(); + } +} diff --git a/src/infrastructure/ffmpeg/adapter.ts b/src/infrastructure/ffmpeg/adapter.ts new file mode 100644 index 0000000..1501fef --- /dev/null +++ b/src/infrastructure/ffmpeg/adapter.ts @@ -0,0 +1,169 @@ +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { pino } from 'pino'; +import type { + TranscodeProvider, + ProbeResult, + ProgressCallback, + TranscodeResult, + AudioStreamInfo, +} from '../../domain/job.interface.js'; +import { DEFAULT_WORK_DIR } from './types.js'; +import { + filterActiveVideoProfiles, + computeVideoMetadata, + computeAudioMetadata, +} from './encoding/profiles.js'; +import { probe } from './core/probe.js'; +import { probeComplexity } from './core/complexity.js'; +import { processMasterPipeline } from './hls/pipeline.js'; +import { writeMasterPlaylist } from './hls/playlist.js'; +import { HLS_CONSTANTS } from './constants.js'; +import { generateTierUuid, blobPathFromUuid } from './core/hash.js'; + +const logger = pino({ name: 'FFmpegAdapter' }); + +const SCHEMA_VERSION = 'v1'; + +const ISO_639_1_MAP: Record = { + eng: 'en', + hin: 'hi', + spa: 'es', + fra: 'fr', + deu: 'de', + jpn: 'ja', + und: 'und', +}; + +/** + * The "Brain" of the transcoding engine. + * + * @remarks + * - Orchestrates the entire lifecycle: Probing -> Complexity Analysis -> Transcoding -> Manifest Mapping. + * - Implements a Dispersed Hash Tree schema (via `blobPathFromUuid`) to prevent directory iteration attacks in public storage. + * - Employs a "Smart Per-Title" intelligence: Probes the file's visual complexity before assigning final bitrates and renditions. + */ +export class FFmpegAdapter implements TranscodeProvider { + constructor(private readonly workDir: string = DEFAULT_WORK_DIR) {} + async probe(sourceUrl: string): Promise { + return probe(sourceUrl); + } + + async transcodeHLS( + sourceUrl: string, + videoId: string, + sourceWidth: number, + sourceHeight: number, + sourceDuration: number, + onProgress?: ProgressCallback, + sourceFrameRate?: number, + audioStreams: AudioStreamInfo[] = [], + videoRange?: string, + ): Promise { + const outputDir = path.join(this.workDir, videoId, 'hls'); + const activeProfiles = filterActiveVideoProfiles(sourceWidth, sourceHeight, videoRange); + + logger.info({ videoId }, 'Analyzing video complexity for Smart Per-Title Bitrate adaptation'); + + const { multiplier: complexityMultiplier } = await probeComplexity( + sourceUrl, + sourceDuration, + videoId, + sourceWidth, + sourceHeight, + ); + + const rawVideoVariants = computeVideoMetadata( + activeProfiles, + sourceWidth, + sourceHeight, + complexityMultiplier, + ); + + const rawAudioRenditions = computeAudioMetadata(audioStreams); + const hlsTime = rawVideoVariants[0]?.hlsTime ?? 6; + + const videoVariants = rawVideoVariants.map((v) => { + const tierId = generateTierUuid(); + const relativeUrl = path.join(SCHEMA_VERSION, blobPathFromUuid(tierId)); + return { ...v, relativeUrl }; + }); + + const audioRenditions = rawAudioRenditions.map((a) => { + const lang2Letter = ISO_639_1_MAP[a.language] || a.language; + const uniqueAudioName = `${lang2Letter}_${a.name}`; + const tierId = generateTierUuid(); + const relativeUrl = path.join(SCHEMA_VERSION, blobPathFromUuid(tierId)); + return { ...a, relativeUrl }; + }); + + for (const v of videoVariants) { + await fs.mkdir(path.join(outputDir, v.relativeUrl), { recursive: true }); + } + for (const a of audioRenditions) { + await fs.mkdir(path.join(outputDir, a.relativeUrl), { recursive: true }); + } + + await processMasterPipeline( + sourceUrl, + outputDir, + videoId, + videoVariants, + audioRenditions, + hlsTime, + onProgress, + sourceFrameRate, + sourceDuration, + videoRange, + ); + + const fileExists = async (p: string) => + fs + .stat(p) + .then(() => true) + .catch(() => false); + + const validVideoVariants = []; + for (const v of videoVariants) { + const p = path.join(outputDir, v.relativeUrl, HLS_CONSTANTS.VARIANT_PLAYLIST_NAME); + if (await fileExists(p)) validVideoVariants.push(v); + } + + if (validVideoVariants.length === 0) { + throw new Error( + 'All video encoding phases failed. No valid video segments were generated.', + ); + } + + const validAudioRenditions = []; + + for (const a of audioRenditions) { + const p = path.join(outputDir, a.relativeUrl, HLS_CONSTANTS.VARIANT_PLAYLIST_NAME); + if (await fileExists(p)) validAudioRenditions.push(a); + } + + await writeMasterPlaylist( + outputDir, + validVideoVariants, + validAudioRenditions, + sourceFrameRate, + ); + + logger.info({ videoId }, 'All variants transcoded, Dispersed Hash Tree master written'); + + const renditions = validVideoVariants.map((v) => ({ + resolution: v.name, + width: v.actualWidth, + height: v.actualHeight, + bitrate: v.bitrate, + url: `../${v.relativeUrl}/${HLS_CONSTANTS.VARIANT_PLAYLIST_NAME}`, + })); + + return { outputDir, renditions }; + } + + async cleanup(videoId: string): Promise { + const dir = path.join(this.workDir, videoId); + await fs.rm(dir, { recursive: true, force: true }); + } +} diff --git a/src/infrastructure/ffmpeg/constants.ts b/src/infrastructure/ffmpeg/constants.ts new file mode 100644 index 0000000..241e4d8 --- /dev/null +++ b/src/infrastructure/ffmpeg/constants.ts @@ -0,0 +1,15 @@ +export const HLS_CONSTANTS = { + MASTER_PLAYLIST_NAME: 'playlist.m3u8', + + VIDEO_SEGMENT_NAME: 'data_%03d.m4s', + SINGLE_VIDEO_NAME: 'data.m4s', + + INIT_SEGMENT_NAME: 'init.mp4', + + VARIANT_PLAYLIST_NAME: 'manifest.m3u8', + + AUDIO_TIERS: { + SURROUND: 'a1', + STEREO: 'a2', + }, +} as const; diff --git a/src/infrastructure/ffmpeg/core/complexity.ts b/src/infrastructure/ffmpeg/core/complexity.ts new file mode 100644 index 0000000..f97f723 --- /dev/null +++ b/src/infrastructure/ffmpeg/core/complexity.ts @@ -0,0 +1,192 @@ +import { pino } from 'pino'; +import { spawn } from 'node:child_process'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { TranscodeError } from '../../../domain/errors.js'; + +const logger = pino({ name: 'PerTitleVMAF' }); + +export interface ComplexityResult { + multiplier: number; + sampleBitrate: number; +} + +async function runCommand(args: string[]): Promise { + return new Promise((resolve, reject) => { + const proc = spawn('ffmpeg', args); + let output = ''; + + proc.stderr.on('data', (data) => { + output += data.toString(); + }); + + proc.on('close', (code) => { + if (code !== 0) reject(new Error(`FFmpeg failed with code ${code}:\n${output}`)); + else resolve(output); + }); + }); +} + +/** + * Orchestrates a mathematically-driven Smart Per-Title Bitrate adaptation using Netflix's VMAF model. + * + * @remarks + * - Phase 1: Generates a near-lossless reference encode (CRF 10, ultrafast) to establish a pristine baseline. + * - Phase 2: Encodes empirical test points (CRF 19, 23, 27) and scores them against the reference using the VMAF neural network. + * - Phase 3: Interpolates a Rate-Distortion curve to find the exact bits-per-second required to hit a perceptive VMAF score of 95.0. + * - Bounding: Applies strict OTT resolution safety floors to prevent bitrate explosion on noisy/grainy source files. + */ +export async function probeComplexity( + sourceUrl: string, + duration: number, + videoId: string, + sourceWidth: number, + sourceHeight: number, +): Promise { + const workDir = `/tmp/worker/${videoId}/vmaf`; + await fs.mkdir(workDir, { recursive: true }); + + const refPath = path.join(workDir, 'ref.mkv'); + + logger.info({ videoId }, 'Phase 1: Generating 100% Whole-Timeline Reference (CFR Enforced)'); + + try { + await runCommand([ + '-i', + sourceUrl, + '-map', + '0:v:0', + '-an', + '-sn', + '-c:v', + 'libx264', + '-crf', + '10', + '-preset', + 'ultrafast', + '-fps_mode', + 'cfr', + '-pix_fmt', + 'yuv420p', + '-y', + refPath, + ]); + + logger.info({ videoId }, 'Phase 2: Encoding Full Grid & Running VMAF AI Analysis'); + + const crfs = [19, 23, 27]; + const results: { crf: number; vmaf: number; kbps: number }[] = []; + + for (const crf of crfs) { + const testPath = path.join(workDir, `test_${crf}.mkv`); + + await runCommand([ + '-i', + refPath, + '-c:v', + 'libx264', + '-crf', + String(crf), + '-preset', + 'superfast', + '-fps_mode', + 'cfr', + '-y', + testPath, + ]); + + const stat = await fs.stat(testPath); + const kbps = (stat.size * 8) / duration / 1000; + + const vmafOut = await runCommand([ + '-i', + testPath, + '-i', + refPath, + '-lavfi', + '[0:v]setpts=PTS-STARTPTS,scale=1920:1080:flags=bicubic[dist];[1:v]setpts=PTS-STARTPTS,scale=1920:1080:flags=bicubic[ref];[dist][ref]libvmaf=model=path=/usr/local/share/model/vmaf_v0.6.1.json:n_subsample=24:n_threads=4:pool=harmonic_mean', + '-f', + 'null', + '-', + ]); + + const match = vmafOut.match(/VMAF score: ([\d.]+)/); + const vmaf = match ? parseFloat(match[1]) : 0; + + results.push({ crf, vmaf, kbps }); + } + + logger.info( + { videoId, curve: results }, + 'Phase 3: Full-Timeline Rate-Distortion Curve mathematically mapped', + ); + + const TARGET_VMAF = 95.0; + results.sort((a, b) => a.vmaf - b.vmaf); + + let optimalKbps = 0; + + if (TARGET_VMAF <= results[0].vmaf) { + optimalKbps = results[0].kbps; + } else if (TARGET_VMAF >= results[results.length - 1].vmaf) { + optimalKbps = results[results.length - 1].kbps; + } else { + for (let i = 0; i < results.length - 1; i++) { + if (TARGET_VMAF >= results[i].vmaf && TARGET_VMAF <= results[i + 1].vmaf) { + const p1 = results[i]; + const p2 = results[i + 1]; + const slope = (p2.kbps - p1.kbps) / (p2.vmaf - p1.vmaf); + optimalKbps = p1.kbps + slope * (TARGET_VMAF - p1.vmaf); + break; + } + } + } + + const is4K = sourceWidth >= 3840 || sourceHeight >= 2160; + const is1080p = (sourceWidth >= 1920 || sourceHeight >= 1080) && !is4K; + + const BASELINE_KBPS = is4K ? 20000 : is1080p ? 6000 : 3000; + + if (!optimalKbps || isNaN(optimalKbps) || optimalKbps <= 0) { + logger.warn( + { videoId }, + 'VMAF calculation yielded invalid optimalKbps. Falling back to BASELINE.', + ); + optimalKbps = BASELINE_KBPS; + } + + let multiplier = optimalKbps / BASELINE_KBPS; + + const minFloor = is4K ? 0.85 : 0.6; + const maxCeil = is4K ? 1.5 : 2.0; + + multiplier = Math.max(minFloor, Math.min(multiplier, maxCeil)); + + logger.info( + { + videoId, + optimalKbps: Math.round(optimalKbps), + baselineKbps: BASELINE_KBPS, + finalMultiplier: multiplier, + is4K, + }, + 'Real Per-Title VMAF calculated with OTT Resolution Quality Floors applied', + ); + + await fs.rm(workDir, { recursive: true, force: true }).catch(() => {}); + + return { multiplier, sampleBitrate: Math.round(optimalKbps) }; + } catch (error) { + logger.error( + { videoId, err: error }, + 'Real VMAF probe failed. Strict mode active: Failing the pipeline.', + ); + + await fs.rm(workDir, { recursive: true, force: true }).catch(() => {}); + + throw new TranscodeError( + `VMAF Complexity Probe failed: ${error instanceof Error ? error.message : String(error)}`, + error instanceof Error ? error : undefined, + ); + } +} diff --git a/src/infrastructure/ffmpeg/core/hash.ts b/src/infrastructure/ffmpeg/core/hash.ts new file mode 100644 index 0000000..84ad4c4 --- /dev/null +++ b/src/infrastructure/ffmpeg/core/hash.ts @@ -0,0 +1,15 @@ +import { v7 as uuidv7 } from 'uuid'; + +export function generateTierUuid(): string { + return uuidv7(); +} + +export function blobPathFromUuid(id: string): string { + const cleanId = id.toLowerCase().replace(/-/g, ''); + + const p1 = cleanId.slice(0, 2); + const p2 = cleanId.slice(2, 4); + const p3 = cleanId.slice(4, 6); + + return `${p1}/${p2}/${p3}/${id}`; +} diff --git a/src/infrastructure/ffmpeg/core/probe.ts b/src/infrastructure/ffmpeg/core/probe.ts new file mode 100644 index 0000000..fe4def3 --- /dev/null +++ b/src/infrastructure/ffmpeg/core/probe.ts @@ -0,0 +1,105 @@ +import { pino } from 'pino'; +import type { ProbeResult, AudioStreamInfo } from '../../../domain/job.interface.js'; +import { ValidationError } from '../../../domain/errors.js'; +import { runFFprobe } from './runner.js'; + +const logger = pino({ name: 'ProbeCommand' }); +const MAX_DURATION_SECONDS = 7200; +const MAX_FILE_SIZE_BYTES = 10 * 1024 * 1024 * 1024; + +export async function probe(sourceUrl: string): Promise { + logger.info({ sourceUrl }, 'Probing source video'); + + const data = await runFFprobe(sourceUrl); + + const videoStream = data.streams.find((s) => s.codec_type === 'video'); + if (!videoStream) { + throw new ValidationError('No video stream found in source'); + } + + const duration = data.format.duration ?? 0; + const fileSize = data.format.size ?? 0; + const width = videoStream.width ?? 0; + const height = videoStream.height ?? 0; + const codec = videoStream.codec_name ?? 'unknown'; + + const rFrameRate = videoStream.r_frame_rate ?? '30/1'; + const [fpNum, fpDen] = rFrameRate.split('/').map(Number); + const frameRate = fpDen ? Math.round(fpNum / fpDen) : fpNum; + + const rawAudioStreams = data.streams.filter((s) => s.codec_type === 'audio'); + + const audioStreams: AudioStreamInfo[] = rawAudioStreams.map((stream, arrayIndex) => { + const s = stream as any; + const tags = s.tags || {}; + + const lang = tags.language || tags.LANGUAGE || 'und'; + + const title = tags.title || tags.TITLE || tags.handler_name || `Track ${arrayIndex + 1}`; + + return { + index: arrayIndex, + language: lang.toLowerCase().slice(0, 3), + channels: s.channels ?? 2, + title: title, + }; + }); + + if (audioStreams.length === 0) { + audioStreams.push({ index: -1, language: 'und', channels: 2, title: 'Track 1' }); + } + + let videoRange = 'SDR'; + const colorTransfer = (videoStream as any).color_transfer?.toLowerCase() || ''; + if (colorTransfer.includes('smpte2084')) { + videoRange = 'PQ'; + } else if (colorTransfer.includes('arib-std-b67') || colorTransfer.includes('hlg')) { + videoRange = 'HLG'; + } + + if (duration > MAX_DURATION_SECONDS) { + throw new ValidationError( + `Video too long: ${Math.round(duration)}s (max: ${MAX_DURATION_SECONDS}s)`, + ); + } + + if (fileSize > MAX_FILE_SIZE_BYTES) { + throw new ValidationError( + `File too large: ${Math.round(fileSize / 1024 / 1024)}MB (max: ${MAX_FILE_SIZE_BYTES / 1024 / 1024}MB)`, + ); + } + + if (height < 240) { + throw new ValidationError(`Resolution too low: ${width}x${height} (min: 240p)`); + } + + const originalAspectRatio = width / height; + let aspectRatio = '16:9'; + + const ratioStr = originalAspectRatio.toFixed(2); + if (ratioStr === '1.78') aspectRatio = '16:9'; + else if (ratioStr === '0.56') aspectRatio = '9:16'; + else if (ratioStr === '1.85') aspectRatio = '1.85:1'; + else if (ratioStr === '2.00') aspectRatio = '2.00:1'; + else if (ratioStr === '2.39') aspectRatio = '2.39:1'; + else aspectRatio = `${ratioStr}:1`; + + const result: ProbeResult = { + duration, + width, + height, + aspectRatio, + originalAspectRatio, + codec, + fileSize, + frameRate, + audioStreams, + videoRange, + }; + + logger.info( + { sourceUrl, audioLanguagesDetected: audioStreams.map((a) => a.language), videoRange }, + 'Probe complete', + ); + return result; +} diff --git a/src/infrastructure/ffmpeg/core/runner.ts b/src/infrastructure/ffmpeg/core/runner.ts new file mode 100644 index 0000000..1688108 --- /dev/null +++ b/src/infrastructure/ffmpeg/core/runner.ts @@ -0,0 +1,153 @@ +import { spawn } from 'node:child_process'; +import { pino } from 'pino'; +import { TranscodeError, ValidationError, SourceNotFoundError } from '../../../domain/errors.js'; +import type { RunOptions } from '../types.js'; + +const logger = pino({ name: 'FFmpegRunner' }); + +/** + * Robust child-process wrapper for invoking the FFmpeg binary safely in Node.js. + * + * @remarks + * - Parses FFmpeg's chaotic `stderr` payload to extract frame-accurate progression values. + * - Maintains a rolling buffer of `stderr` (hard-capped at 50kb) to prevent V8 memory leaks during hours-long encodes. + * - Maps raw exit codes and the tail of the stderr buffer into actionable `TranscodeError` domain exceptions. + */ +export function runFFmpeg(opts: RunOptions): Promise { + const { args, label, videoId, onProgress, duration } = opts; + + return new Promise((resolve, reject) => { + const cmd = `ffmpeg ${args.join(' ')}`; + logger.info({ videoId, label, cmd }, 'FFmpeg command started'); + + const proc = spawn('ffmpeg', args, { stdio: ['ignore', 'ignore', 'pipe'] }); + let stderrBuffer = ''; + let lastLogPercent = -1; + + proc.stderr.on('data', (chunk: Buffer) => { + const text = chunk.toString(); + stderrBuffer += text; + + if (stderrBuffer.length > 50000) { + stderrBuffer = stderrBuffer.slice(-25000); + } + + if (onProgress || duration) { + const match = text.match(/time=(\d+):(\d+):(\d+)\.(\d+)/); + if (match) { + const [, h, m, s] = match.map(Number); + const seconds = h * 3600 + m * 60 + s; + + let percent = seconds; + + if (duration && duration > 0) { + percent = Math.min(100, Math.round((seconds / duration) * 100)); + + if (percent % 5 === 0 && percent !== lastLogPercent) { + logger.info( + { videoId, label, progress: `${percent}%` }, + 'Processing video...', + ); + lastLogPercent = percent; + } + } + + if (onProgress) { + onProgress({ variant: label, percent }); + } + } + } + + for (const line of text.split('\n')) { + const lower = line.toLowerCase(); + if (lower.includes('error') || lower.includes('crypto')) { + logger.warn({ videoId, label, stderr: line.trim() }, 'FFmpeg stderr'); + } + } + }); + + proc.on('close', (code) => { + if (code === 0) { + if (duration) logger.info({ videoId, label, progress: '100%' }, 'Processing complete!'); + resolve(); + } else { + const lastLines = stderrBuffer.split('\n').slice(-5).join('\n'); + logger.error({ videoId, label, code, stderr: lastLines }, 'FFmpeg process failed'); + reject(new TranscodeError(`FFmpeg ${label} exited with code ${code}: ${lastLines}`)); + } + }); + + proc.on('error', (err) => { + logger.error({ err, videoId, label }, 'FFmpeg spawn error'); + reject(new TranscodeError(`Failed to spawn FFmpeg for ${label}: ${err.message}`, err)); + }); + }); +} + +interface FfprobeStream { + codec_type: string; + codec_name?: string; + width?: number; + height?: number; + r_frame_rate?: string; + channels?: number; +} + +interface FfprobeFormat { + duration?: number; + size?: number; +} + +interface FfprobeOutput { + streams: FfprobeStream[]; + format: FfprobeFormat; +} + +export function runFFprobe(sourceUrl: string): Promise { + const args = [ + '-v', + 'quiet', + '-print_format', + 'json', + '-show_format', + '-show_streams', + sourceUrl, + ]; + + return new Promise((resolve, reject) => { + logger.info({ sourceUrl }, 'Probing source'); + + const proc = spawn('ffprobe', args, { stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = ''; + let stderr = ''; + + proc.stdout.on('data', (chunk: Buffer) => { + stdout += chunk.toString(); + }); + proc.stderr.on('data', (chunk: Buffer) => { + stderr += chunk.toString(); + }); + + proc.on('close', (code) => { + if (code !== 0) { + if (stderr.includes('404') || stderr.includes('Server returned')) { + return reject(new SourceNotFoundError(sourceUrl)); + } + return reject( + new ValidationError(`ffprobe failed (code ${code}): ${stderr.slice(-200)}`), + ); + } + + try { + const result = JSON.parse(stdout) as FfprobeOutput; + resolve(result); + } catch { + reject(new ValidationError(`Failed to parse ffprobe output: ${stdout.slice(0, 200)}`)); + } + }); + + proc.on('error', (err) => { + reject(new ValidationError(`Failed to spawn ffprobe: ${err.message}`)); + }); + }); +} diff --git a/src/infrastructure/ffmpeg/encoding/flags.ts b/src/infrastructure/ffmpeg/encoding/flags.ts new file mode 100644 index 0000000..5495a9a --- /dev/null +++ b/src/infrastructure/ffmpeg/encoding/flags.ts @@ -0,0 +1,164 @@ +import path from 'node:path'; +import { config } from '../../../config/env.js'; +import type { VideoVariantMeta, AudioVariantMeta } from '../types.js'; +import { HLS_CONSTANTS } from '../constants.js'; + +export interface FrameRateInfo { + ffmpegFraction: string; + appleFormat: string; + gopSize: number; +} + +export function getBroadcastFrameRate(sourceFps?: number): FrameRateInfo | null { + if (!sourceFps) return null; + + const targetFps = Math.min(sourceFps, 30); + const eps = 0.05; + + let fraction = `${Math.round(targetFps * 1000)}/1000`; + let exactFps = targetFps; + + if (Math.abs(targetFps - 23.976) < eps) { + fraction = '24000/1001'; + exactFps = 24000 / 1001; + } else if (Math.abs(targetFps - 29.97) < eps) { + fraction = '30000/1001'; + exactFps = 30000 / 1001; + } else { + const rounded = Math.round(targetFps); + if (Math.abs(targetFps - rounded) < eps) { + fraction = `${rounded}/1`; + exactFps = rounded; + } + } + + return { + ffmpegFraction: fraction, + appleFormat: exactFps.toFixed(3), + gopSize: Math.round(exactFps * 2), + }; +} + +export function hlsOutputFlags(hlsTime: number, outputDir: string): string[] { + return [ + '-hls_fmp4_init_filename', + HLS_CONSTANTS.INIT_SEGMENT_NAME, + '-movflags', + '+frag_keyframe+empty_moov+default_base_moof+cmaf+omit_tfhd_offset', + '-f', + 'hls', + '-hls_time', + String(hlsTime), + '-hls_list_size', + '0', + '-hls_playlist_type', + 'vod', + '-hls_segment_type', + 'fmp4', + '-hls_flags', + config.HLS_OUTPUT_MODE === 'SINGLE_FILE' + ? '+independent_segments+single_file+round_durations' + : '+independent_segments+round_durations', + '-hls_segment_filename', + config.HLS_OUTPUT_MODE === 'SINGLE_FILE' + ? path.join(outputDir, HLS_CONSTANTS.SINGLE_VIDEO_NAME) + : path.join(outputDir, HLS_CONSTANTS.VIDEO_SEGMENT_NAME), + '-avoid_negative_ts', + 'make_zero', + '-fflags', + '+genpts', + '-use_stream_ids_as_track_ids', + '1', + '-video_track_timescale', + '90000', + ]; +} + +export function videoEncoderFlags(variant: VideoVariantMeta, sourceFrameRate?: number): string[] { + const fpsInfo = getBroadcastFrameRate(sourceFrameRate); + const gopSize = fpsInfo ? fpsInfo.gopSize : 48; + + const codec = variant.videoCodec || 'libx264'; + const isHevc = codec === 'libx265'; + const isHdr = variant.profile === 'main10'; + + const colorPrimaries = isHdr ? 'bt2020' : 'bt709'; + const colorTransfer = isHdr ? 'smpte2084' : 'bt709'; + const colorMatrix = isHdr ? 'bt2020nc' : 'bt709'; + const pixFmt = isHdr ? 'yuv420p10le' : 'yuv420p'; + + const baseFlags: string[] = [ + '-c:v', + codec, + '-tag:v', + isHevc ? 'hvc1' : variant.videoCodecTag.substring(0, 4), + '-preset', + variant.preset, + ...(fpsInfo ? ['-r', fpsInfo.ffmpegFraction, '-fps_mode', 'cfr'] : []), + ...(variant.profile ? ['-profile:v', variant.profile] : []), + ...(variant.level ? ['-level', variant.level] : []), + '-pix_fmt', + pixFmt, + '-colorspace', + colorMatrix, + '-color_primaries', + colorPrimaries, + '-color_trc', + colorTransfer, + '-color_range', + 'tv', + '-crf', + '23', + '-maxrate', + String(variant.maxrate), + '-bufsize', + String(variant.bufsize), + '-b:v', + String(variant.bitrate), + '-g', + String(gopSize), + '-keyint_min', + String(gopSize), + '-sc_threshold', + '0', + ]; + + if (isHevc) { + baseFlags.push( + '-x265-params', + `no-open-gop=1:keyint=${gopSize}:min-keyint=${gopSize}:info=0:colorprim=${colorPrimaries}:transfer=${colorTransfer}:colormatrix=${colorMatrix}`, + '-flags', + '+global_header', + ); + } else { + baseFlags.push('-flags', '+cgop+global_header'); + } + + return baseFlags; +} + +export function videoFilterChain(width: number, height: number): string { + return [ + `scale=${width}:${height}:force_original_aspect_ratio=disable`, + 'setsar=1/1', + 'unsharp=3:3:0.5:3:3:0.5', + ].join(','); +} + +export function audioEncoderFlags(audio: AudioVariantMeta): string[] { + const flags = [ + '-c:a', + audio.codec, + '-b:a', + String(audio.bitrate), + '-ac', + String(audio.channels), + '-ar', + String(audio.sampleRate), + ]; + + const afFilter = 'aresample=async=1:first_pts=0'; + flags.push('-af', afFilter); + + return flags; +} diff --git a/src/infrastructure/ffmpeg/encoding/profiles.ts b/src/infrastructure/ffmpeg/encoding/profiles.ts new file mode 100644 index 0000000..bd0f132 --- /dev/null +++ b/src/infrastructure/ffmpeg/encoding/profiles.ts @@ -0,0 +1,110 @@ +import { createRequire } from 'node:module'; +import type { VideoProfile, AudioProfile, VideoVariantMeta, AudioVariantMeta } from '../types.js'; +import type { AudioStreamInfo } from '../../../domain/job.interface.js'; + +const require = createRequire(import.meta.url); +const videoConfig = require('./profiles/video.json') as Record; +const audioConfig = require('./profiles/audio.json') as AudioProfile[]; + +const VIDEO_PROFILES: VideoProfile[] = [ + ...(videoConfig.h264_sdr || []), + ...(videoConfig.h265_sdr || []), + ...(videoConfig.h265_hdr || []), +]; + +const AUDIO_PROFILES: AudioProfile[] = audioConfig; + +export function filterActiveVideoProfiles( + sourceWidth: number, + sourceHeight: number, + videoRange: string = 'SDR', +): VideoProfile[] { + const isVertical = sourceHeight > sourceWidth; + + let compatibleProfiles = VIDEO_PROFILES; + if (videoRange === 'SDR') { + compatibleProfiles = VIDEO_PROFILES.filter((p) => !p.name.includes('hdr')); + } + + const active = compatibleProfiles.filter((v) => { + const standardWidth = Math.round((v.height * 16) / 9); + if (isVertical) { + return sourceHeight >= standardWidth || sourceWidth >= v.height; + } else { + return sourceWidth >= standardWidth || sourceHeight >= v.height; + } + }); + + if (active.length === 0) { + active.push(compatibleProfiles[0] || VIDEO_PROFILES[0]); + } + + return active; +} + +export function computeVideoMetadata( + profiles: VideoProfile[], + sourceWidth: number, + sourceHeight: number, + complexityMultiplier: number = 1.0, +): Omit[] { + const activeProfiles = profiles; + + const isVertical = sourceHeight > sourceWidth; + + return activeProfiles.map((profile) => { + const standardWidth = Math.round((profile.height * 16) / 9); + let maxBoxWidth = standardWidth; + let maxBoxHeight = profile.height; + + if (isVertical) { + maxBoxWidth = profile.height; + maxBoxHeight = standardWidth; + } + + const scaleWidth = maxBoxWidth / sourceWidth; + const scaleHeight = maxBoxHeight / sourceHeight; + const scale = Math.min(scaleWidth, scaleHeight, 1.0); + + const outWidth = sourceWidth * scale; + const outHeight = sourceHeight * scale; + + const actualWidth = Math.round(outWidth / 2) * 2; + const actualHeight = Math.round(outHeight / 2) * 2; + + const dynamicMaxrate = Math.round(profile.maxrate * complexityMultiplier); + const dynamicBufsize = Math.round(profile.bufsize * complexityMultiplier); + const dynamicBitrate = Math.round(profile.bitrate * complexityMultiplier); + + return { + ...profile, + actualWidth, + actualHeight, + bitrate: dynamicBitrate, + maxrate: dynamicMaxrate, + bufsize: dynamicBufsize, + }; + }); +} + +export function computeAudioMetadata( + sourceAudioStreams: AudioStreamInfo[] = [], +): Omit[] { + const renditions: Omit[] = []; + + for (const stream of sourceAudioStreams) { + for (const profile of AUDIO_PROFILES) { + if (profile.hardwareProfile && stream.channels < 2) continue; + + renditions.push({ + ...profile, + sourceChannels: stream.channels, + language: stream.language, + streamIndex: stream.index, + title: stream.title, + }); + } + } + + return renditions; +} diff --git a/src/infrastructure/ffmpeg/encoding/profiles/audio.json b/src/infrastructure/ffmpeg/encoding/profiles/audio.json new file mode 100644 index 0000000..c152592 --- /dev/null +++ b/src/infrastructure/ffmpeg/encoding/profiles/audio.json @@ -0,0 +1,51 @@ +[ + { + "name": "aud_aac_he2_t1", + "channels": 2, + "codec": "libfdk_aac", + "profile": "aac_he_v2", + "bitrate": 32000, + "sampleRate": 44100, + "hardwareProfile": false + }, + { + "name": "aud_aac_lc_t2", + "channels": 2, + "codec": "aac", + "bitrate": 64000, + "sampleRate": 44100, + "hardwareProfile": false + }, + { + "name": "aud_aac_lc_t3", + "channels": 2, + "codec": "aac", + "bitrate": 128000, + "sampleRate": 44100, + "hardwareProfile": false + }, + { + "name": "aud_aac_lc_t4", + "channels": 2, + "codec": "aac", + "bitrate": 160000, + "sampleRate": 48000, + "hardwareProfile": false + }, + { + "name": "aud_eac3_51_t1", + "channels": 6, + "codec": "eac3", + "bitrate": 448000, + "sampleRate": 48000, + "hardwareProfile": true + }, + { + "name": "aud_ac3_51_t1", + "channels": 6, + "codec": "ac3", + "bitrate": 384000, + "sampleRate": 48000, + "hardwareProfile": true + } +] diff --git a/src/infrastructure/ffmpeg/encoding/profiles/video.json b/src/infrastructure/ffmpeg/encoding/profiles/video.json new file mode 100644 index 0000000..42fd7e2 --- /dev/null +++ b/src/infrastructure/ffmpeg/encoding/profiles/video.json @@ -0,0 +1,338 @@ +{ + "h264_sdr": [ + { + "name": "vod_avc_sdr_t1", + "width": 416, + "height": 234, + "bitrate": 130000, + "maxrate": 145000, + "bufsize": 290000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "preset": "slow", + "profile": "baseline", + "level": "3.0", + "videoCodecTag": "avc1.42E01E" + }, + { + "name": "vod_avc_sdr_t2", + "width": 480, + "height": 270, + "bitrate": 365000, + "maxrate": 400000, + "bufsize": 800000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "preset": "slow", + "profile": "baseline", + "level": "3.0", + "videoCodecTag": "avc1.42E01E" + }, + { + "name": "vod_avc_sdr_t3", + "width": 640, + "height": 360, + "bitrate": 730000, + "maxrate": 800000, + "bufsize": 1600000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "preset": "slow", + "profile": "main", + "level": "3.0", + "videoCodecTag": "avc1.4D401E" + }, + { + "name": "vod_avc_sdr_t4", + "width": 960, + "height": 540, + "bitrate": 2000000, + "maxrate": 2200000, + "bufsize": 4400000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "preset": "slow", + "profile": "high", + "level": "3.1", + "videoCodecTag": "avc1.64001F" + }, + { + "name": "vod_avc_sdr_t5", + "width": 1280, + "height": 720, + "bitrate": 3000000, + "maxrate": 3300000, + "bufsize": 6000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "preset": "slow", + "profile": "high", + "level": "3.1", + "videoCodecTag": "avc1.64001F" + }, + { + "name": "vod_avc_sdr_t6", + "width": 1920, + "height": 1080, + "bitrate": 6000000, + "maxrate": 6600000, + "bufsize": 12000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "preset": "slow", + "profile": "high", + "level": "4.0", + "videoCodecTag": "avc1.640028" + } + ], + "h265_sdr": [ + { + "name": "vod_hevc_sdr_t1", + "width": 416, + "height": 234, + "bitrate": 130000, + "maxrate": 145000, + "bufsize": 290000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main", + "level": "3.1", + "videoCodecTag": "hvc1.1.4.L93.B0" + }, + { + "name": "vod_hevc_sdr_t2", + "width": 480, + "height": 270, + "bitrate": 250000, + "maxrate": 280000, + "bufsize": 500000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main", + "level": "3.1", + "videoCodecTag": "hvc1.1.4.L93.B0" + }, + { + "name": "vod_hevc_sdr_t3", + "width": 640, + "height": 360, + "bitrate": 575000, + "maxrate": 630000, + "bufsize": 1200000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main", + "level": "3.1", + "videoCodecTag": "hvc1.1.4.L93.B0" + }, + { + "name": "vod_hevc_sdr_t4", + "width": 960, + "height": 540, + "bitrate": 1100000, + "maxrate": 1200000, + "bufsize": 2400000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main", + "level": "3.1", + "videoCodecTag": "hvc1.1.4.L93.B0" + }, + { + "name": "vod_hevc_sdr_t5", + "width": 1280, + "height": 720, + "bitrate": 2100000, + "maxrate": 2300000, + "bufsize": 4200000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main", + "level": "4.0", + "videoCodecTag": "hvc1.1.4.L120.B0" + }, + { + "name": "vod_hevc_sdr_t6", + "width": 1920, + "height": 1080, + "bitrate": 6800000, + "maxrate": 7200000, + "bufsize": 12000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main", + "level": "4.0", + "videoCodecTag": "hvc1.1.4.L120.B0" + }, + { + "name": "vod_hevc_sdr_t7", + "width": 2560, + "height": 1440, + "bitrate": 11600000, + "maxrate": 12500000, + "bufsize": 20000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main", + "level": "5.0", + "videoCodecTag": "hvc1.1.4.L150.B0" + }, + { + "name": "vod_hevc_sdr_t8", + "width": 3840, + "height": 2160, + "bitrate": 20000000, + "maxrate": 22000000, + "bufsize": 30000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main", + "level": "5.0", + "videoCodecTag": "hvc1.1.4.L150.B0" + } + ], + "h265_hdr": [ + { + "name": "vod_hevc_hdr_t1", + "width": 416, + "height": 234, + "bitrate": 130000, + "maxrate": 145000, + "bufsize": 290000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main10", + "level": "3.1", + "videoCodecTag": "hvc1.2.4.L93.B0" + }, + { + "name": "vod_hevc_hdr_t2", + "width": 480, + "height": 270, + "bitrate": 300000, + "maxrate": 330000, + "bufsize": 600000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main10", + "level": "3.1", + "videoCodecTag": "hvc1.2.4.L93.B0" + }, + { + "name": "vod_hevc_hdr_t3", + "width": 640, + "height": 360, + "bitrate": 690000, + "maxrate": 750000, + "bufsize": 1300000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main10", + "level": "3.1", + "videoCodecTag": "hvc1.2.4.L93.B0" + }, + { + "name": "vod_hevc_hdr_t4", + "width": 960, + "height": 540, + "bitrate": 1300000, + "maxrate": 1450000, + "bufsize": 2600000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main10", + "level": "3.1", + "videoCodecTag": "hvc1.2.4.L93.B0" + }, + { + "name": "vod_hevc_hdr_t5", + "width": 1280, + "height": 720, + "bitrate": 2500000, + "maxrate": 2700000, + "bufsize": 5000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main10", + "level": "4.0", + "videoCodecTag": "hvc1.2.4.L120.B0" + }, + { + "name": "vod_hevc_hdr_t6", + "width": 1920, + "height": 1080, + "bitrate": 8100000, + "maxrate": 8800000, + "bufsize": 14000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main10", + "level": "4.0", + "videoCodecTag": "hvc1.2.4.L120.B0" + }, + { + "name": "vod_hevc_hdr_t7", + "width": 2560, + "height": 1440, + "bitrate": 14000000, + "maxrate": 15000000, + "bufsize": 22000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main10", + "level": "5.0", + "videoCodecTag": "hvc1.2.4.L150.B0" + }, + { + "name": "vod_hevc_hdr_t8", + "width": 3840, + "height": 2160, + "bitrate": 24000000, + "maxrate": 26000000, + "bufsize": 32000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main10", + "level": "5.0", + "videoCodecTag": "hvc1.2.4.L150.B0" + } + ] +} diff --git a/src/infrastructure/ffmpeg/hls/pipeline.ts b/src/infrastructure/ffmpeg/hls/pipeline.ts new file mode 100644 index 0000000..5dcbc24 --- /dev/null +++ b/src/infrastructure/ffmpeg/hls/pipeline.ts @@ -0,0 +1,143 @@ +import path from 'node:path'; +import { config } from '../../../config/env.js'; +import type { ProgressCallback } from '../../../domain/job.interface.js'; +import type { VideoVariantMeta, AudioVariantMeta } from '../types.js'; +import { + videoEncoderFlags, + videoFilterChain, + audioEncoderFlags, + hlsOutputFlags, +} from '../encoding/flags.js'; +import { runFFmpeg } from '../core/runner.js'; +import { HLS_CONSTANTS } from '../constants.js'; + +/** + * Executes the multi-phase FFmpeg transcoding pipeline using dynamically generated `filter_complex` graphs. + * + * @remarks + * - Phases are executed sequentially (Audio -> H.264 -> H.265) to strictly bound peak active memory usage. + * - The `filter_complex` graph splits the decoded input stream in memory, avoiding redundant decodes per resolution. + * - Aggregates and normalizes percentage callbacks across phases using algorithmic weighting based on codec complexity. + */ +export async function processMasterPipeline( + sourceUrl: string, + outputDir: string, + videoId: string, + videoVariants: VideoVariantMeta[], + audioRenditions: AudioVariantMeta[], + hlsTime: number, + onProgress?: ProgressCallback, + sourceFrameRate?: number, + sourceDuration?: number, + videoRange: string = 'SDR', +): Promise { + const h264Sdr = videoVariants.filter((v) => v.videoCodec === 'libx264'); + const h265Sdr = videoVariants.filter( + (v) => v.videoCodec === 'libx265' && v.profile !== 'main10', + ); + const h265Hdr = videoVariants.filter( + (v) => v.videoCodec === 'libx265' && v.profile === 'main10', + ); + + let currentBaseProgress = 0; + const weightAudio = audioRenditions.length > 0 ? 10 : 0; + const weightH264 = h264Sdr.length > 0 ? 20 : 0; + const weightH265Sdr = h265Sdr.length > 0 ? 30 : 0; + const weightH265Hdr = h265Hdr.length > 0 ? 40 : 0; + const totalWeight = weightAudio + weightH264 + weightH265Sdr + weightH265Hdr; + + const runPhase = async (label: string, weight: number, buildArgs: () => string[]) => { + const args = buildArgs(); + await runFFmpeg({ + args, + label, + videoId, + duration: config.TEST_DURATION_SECONDS || sourceDuration, + onProgress: (p) => { + if (onProgress) { + const scaledProgress = currentBaseProgress + p.percent * (weight / totalWeight); + onProgress({ variant: label, percent: scaledProgress }); + } + }, + }); + currentBaseProgress += (weight / totalWeight) * 100; + }; + + const getBaseInputArgs = () => { + const args = []; + + if (config.TEST_DURATION_SECONDS) { + args.push('-t', String(config.TEST_DURATION_SECONDS)); + } + args.push('-i', sourceUrl); + return args; + }; + + if (audioRenditions.length > 0) { + await runPhase('Phase_1_Audio', weightAudio, () => { + const args = [...getBaseInputArgs()]; + + audioRenditions.forEach((audio) => { + const audioDir = path.join(outputDir, audio.relativeUrl); + const manifestPath = path.join(audioDir, HLS_CONSTANTS.VARIANT_PLAYLIST_NAME); + const streamMap = audio.streamIndex !== -1 ? `0:a:${audio.streamIndex}?` : '0:a:0?'; + + args.push( + '-map', + streamMap, + ...audioEncoderFlags(audio), + ...hlsOutputFlags(hlsTime, audioDir), + manifestPath, + ); + }); + return args; + }); + } + + const buildVideoPhaseArgs = (variants: VideoVariantMeta[], isHdr: boolean) => { + const args = [...getBaseInputArgs()]; + const filtergraph: string[] = []; + + let preFilter = isHdr ? '[0:v:0]format=yuv420p10le' : '[0:v:0]format=yuv420p'; + + if (!isHdr && (videoRange === 'PQ' || videoRange === 'HLG')) { + preFilter = `[0:v:0]zscale=transfer=linear:npl=100,format=gbrpf32le,tonemap=hable:desat=0,zscale=transfer=bt709:matrix=bt709:primaries=bt709:range=tv,format=yuv420p`; + } + if (!isHdr) preFilter += `,hqdn3d=3:3:2:2`; + + if (variants.length > 1) { + const splits = variants.map((_, i) => `[split_${i}]`).join(''); + filtergraph.push(`${preFilter},split=${variants.length}${splits}`); + } else { + filtergraph.push(`${preFilter}[split_0]`); + } + + variants.forEach((variant, i) => { + const scaleFilter = videoFilterChain(variant.actualWidth, variant.actualHeight); + filtergraph.push(`[split_${i}]${scaleFilter}[vout${i}]`); + }); + + args.push('-filter_complex', filtergraph.join('; ')); + + variants.forEach((variant, index) => { + const variantDir = path.join(outputDir, variant.relativeUrl); + const manifestPath = path.join(variantDir, HLS_CONSTANTS.VARIANT_PLAYLIST_NAME); + + args.push( + '-map', + `[vout${index}]`, + ...videoEncoderFlags(variant, sourceFrameRate), + ...hlsOutputFlags(hlsTime, variantDir), + manifestPath, + ); + }); + return args; + }; + + if (h264Sdr.length > 0) + await runPhase('Phase_2_H264_SDR', weightH264, () => buildVideoPhaseArgs(h264Sdr, false)); + if (h265Sdr.length > 0) + await runPhase('Phase_3_H265_SDR', weightH265Sdr, () => buildVideoPhaseArgs(h265Sdr, false)); + if (h265Hdr.length > 0) + await runPhase('Phase_4_H265_HDR', weightH265Hdr, () => buildVideoPhaseArgs(h265Hdr, true)); +} diff --git a/src/infrastructure/ffmpeg/hls/playlist.ts b/src/infrastructure/ffmpeg/hls/playlist.ts new file mode 100644 index 0000000..94530b6 --- /dev/null +++ b/src/infrastructure/ffmpeg/hls/playlist.ts @@ -0,0 +1,232 @@ +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { pino } from 'pino'; +import type { VideoVariantMeta, AudioVariantMeta } from '../types.js'; +import { HLS_CONSTANTS } from '../constants.js'; +import { getBroadcastFrameRate } from '../encoding/flags.js'; + +const logger = pino({ name: 'PlaylistWriter' }); + +const ISO_639_1_MAP: Record = { + eng: 'en', + hin: 'hi', + spa: 'es', + fra: 'fr', + deu: 'de', + jpn: 'ja', + und: 'und', +}; + +const LANGUAGE_NAMES: Record = { + eng: 'English', + hin: 'Hindi', + spa: 'Spanish (LA)', + fra: 'Français', + deu: 'Deutsch', + jpn: 'Japanese', + und: 'Unknown', +}; + +function getAspectRatioString(width: number, height: number): string { + const ratio = width / height; + if (Math.abs(ratio - 16 / 9) < 0.05) return '16:9'; + if (Math.abs(ratio - 9 / 16) < 0.05) return '9:16'; + if (Math.abs(ratio - 1.85) < 0.05) return '1.85:1'; + if (Math.abs(ratio - 2.0) < 0.05) return '2.00:1'; + if (Math.abs(ratio - 2.39) < 0.05) return '2.39:1'; + return `${ratio.toFixed(2)}:1`; +} + +function getPairedAudioNames(videoHeight: number, availableAudio: AudioVariantMeta[]): string[] { + if (!availableAudio || availableAudio.length === 0) return []; + + const hasAudio = (name: string) => availableAudio.some((a) => a.name === name); + const hardware: string[] = []; + if (hasAudio('aud_ac3_51_t1')) hardware.push('aud_ac3_51_t1'); + if (hasAudio('aud_eac3_51_t1')) hardware.push('aud_eac3_51_t1'); + + let stereo = availableAudio[0].name; + if (videoHeight > 720 && hasAudio('aud_aac_lc_t4')) stereo = 'aud_aac_lc_t4'; + else if (videoHeight > 432 && hasAudio('aud_aac_lc_t3')) stereo = 'aud_aac_lc_t3'; + else if (videoHeight > 270 && hasAudio('aud_aac_lc_t2')) stereo = 'aud_aac_lc_t2'; + else if (hasAudio('aud_aac_he2_t1')) stereo = 'aud_aac_he2_t1'; + + return [stereo, ...hardware]; +} + +async function getBandwidthForDir(dirPath: string): Promise<{ peak: number; avg: number }> { + try { + const manifestPath = path.join(dirPath, HLS_CONSTANTS.VARIANT_PLAYLIST_NAME); + const manifestContent = await fs.readFile(manifestPath, 'utf8'); + const lines = manifestContent.split('\n'); + + let totalBits = 0; + let totalDuration = 0; + let peakBitrate = 0; + let currentDuration = 0; + let currentByteRangeLength = 0; + + for (const line of lines) { + const trimmed = line.trim(); + if (trimmed.startsWith('#EXTINF:')) + currentDuration = parseFloat(trimmed.split(':')[1].split(',')[0]); + else if (trimmed.startsWith('#EXT-X-BYTERANGE:')) + currentByteRangeLength = parseInt(trimmed.substring(17).split('@')[0], 10); + else if ( + !trimmed.startsWith('#') && + (trimmed.endsWith('.m4s') || trimmed.endsWith('.mp4')) + ) { + let bits = 0; + if (currentByteRangeLength > 0) { + bits = currentByteRangeLength * 8; + currentByteRangeLength = 0; + } else { + const stat = await fs.stat(path.join(dirPath, trimmed.split('?')[0])); + bits = stat.size * 8; + } + const bitrate = currentDuration > 0 ? Math.round(bits / currentDuration) : 0; + totalBits += bits; + totalDuration += currentDuration; + if (bitrate > peakBitrate) peakBitrate = bitrate; + } + } + if (totalDuration === 0) return { peak: 0, avg: 0 }; + return { peak: peakBitrate, avg: Math.round(totalBits / totalDuration) }; + } catch (e) { + return { peak: 0, avg: 0 }; + } +} + +export async function writeMasterPlaylist( + outputDir: string, + videoVariants: VideoVariantMeta[], + audioRenditions: AudioVariantMeta[], + sourceFrameRate?: number, +): Promise { + if (!videoVariants || videoVariants.length === 0) { + throw new Error('Cannot write master playlist: No valid video variants provided.'); + } + + const lines = ['#EXTM3U', '#EXT-X-VERSION:7', '#EXT-X-INDEPENDENT-SEGMENTS', '']; + + const defaultVariant = videoVariants.reduce((prev, curr) => + Math.abs(curr.maxrate - 2000000) < Math.abs(prev.maxrate - 2000000) ? curr : prev, + ); + const orderedVariants = [ + defaultVariant, + ...videoVariants.filter((v) => v.name !== defaultVariant.name), + ]; + + const variantAudioMap = new Map(); + const usedAudioNames = new Set(); + + for (const v of orderedVariants) { + const paired = getPairedAudioNames(v.actualHeight, audioRenditions); + variantAudioMap.set(v.name, paired); + paired.forEach((name) => usedAudioNames.add(name)); + } + + let currentLangGroup = ''; + for (const audio of audioRenditions) { + if (!usedAudioNames.has(audio.name)) continue; + + const displayName = !audio.title.startsWith('Track ') + ? audio.title + : LANGUAGE_NAMES[audio.language] || audio.language.toUpperCase(); + + if (audio.language !== currentLangGroup) { + lines.push(`#-- ${displayName} --`); + currentLangGroup = audio.language; + } + + const langAttr = + audio.language === 'und' + ? '' + : `LANGUAGE="${ISO_639_1_MAP[audio.language] || audio.language}",`; + const relativeUri = `../${audio.relativeUrl}/${HLS_CONSTANTS.VARIANT_PLAYLIST_NAME}`; + + lines.push( + `#EXT-X-MEDIA:TYPE=AUDIO,NAME="${displayName}",GROUP-ID="${audio.name}",${langAttr}DEFAULT=${audio.streamIndex === 0 ? 'YES' : 'NO'},AUTOSELECT=YES,CHANNELS="${audio.channels}",URI="${relativeUri}"`, + ); + } + lines.push(''); + + for (const v of orderedVariants) { + const videoDir = path.join(outputDir, v.relativeUrl); + const videoBw = await getBandwidthForDir(videoDir); + + const pairedAudioNames = variantAudioMap.get(v.name) || []; + const trueVideoAvg = videoBw.avg > 0 ? videoBw.avg : v.maxrate * 0.55; + const trueVideoPeak = videoBw.peak > 0 ? videoBw.peak : v.maxrate; + const actualVideoRange = v.profile === 'main10' ? 'PQ' : 'SDR'; + const relativeUri = `../${v.relativeUrl}/${HLS_CONSTANTS.VARIANT_PLAYLIST_NAME}`; + const fpsInfo = getBroadcastFrameRate(sourceFrameRate); + const frameRateString = fpsInfo ? fpsInfo.appleFormat : (v.frameRate ?? 30).toFixed(3); + + if (pairedAudioNames.length === 0) { + lines.push( + `#-- stream_${v.name.padEnd(16)} ${`${v.actualWidth}x${v.actualHeight}`.padEnd(11)} AR: ${getAspectRatioString(v.actualWidth, v.actualHeight).padEnd(8)} ${v.videoCodecTag.padEnd(14)} regular avg: ${(trueVideoAvg / 1_000_000).toFixed(1).padStart(4)} Mbps max: ${(trueVideoPeak / 1_000_000).toFixed(1).padStart(4)} Mbps --`, + ); + + const attributes = [ + `AVERAGE-BANDWIDTH=${Math.round(trueVideoAvg * 1.05)}`, + `BANDWIDTH=${Math.round(trueVideoPeak * 1.05)}`, + `VIDEO-RANGE=${actualVideoRange}`, + `CLOSED-CAPTIONS=NONE`, + `CODECS="${v.videoCodecTag}"`, + `FRAME-RATE=${frameRateString}`, + `RESOLUTION=${v.actualWidth}x${v.actualHeight}`, + ].join(','); + + lines.push(`#EXT-X-STREAM-INF:${attributes}`, relativeUri); + continue; + } + + const repAudio = + audioRenditions.find((ar) => ar.name === pairedAudioNames[0]) || audioRenditions[0]; + const repAudioBw = await getBandwidthForDir(path.join(outputDir, repAudio.relativeUrl)); + const trueRepAudioAvg = repAudioBw.avg > 0 ? repAudioBw.avg : repAudio.bitrate; + const trueRepAudioPeak = repAudioBw.peak > 0 ? repAudioBw.peak : repAudio.bitrate; + + const commentPeak = Math.round((trueVideoPeak + trueRepAudioPeak) * 1.05); + const commentAvg = Math.round((trueVideoAvg + trueRepAudioAvg) * 1.05); + + lines.push( + `#-- stream_${v.name.padEnd(16)} ${`${v.actualWidth}x${v.actualHeight}`.padEnd(11)} AR: ${getAspectRatioString(v.actualWidth, v.actualHeight).padEnd(8)} ${v.videoCodecTag.padEnd(14)} regular avg: ${(commentAvg / 1_000_000).toFixed(1).padStart(4)} Mbps max: ${(commentPeak / 1_000_000).toFixed(1).padStart(4)} Mbps --`, + ); + + for (const audioName of pairedAudioNames) { + const a = audioRenditions.find((ar) => ar.name === audioName) || audioRenditions[0]; + const audioBw = await getBandwidthForDir(path.join(outputDir, a.relativeUrl)); + + const localPeakBandwidth = Math.round( + (trueVideoPeak + (audioBw.peak > 0 ? audioBw.peak : a.bitrate)) * 1.05, + ); + const localAvgBandwidth = Math.round( + (trueVideoAvg + (audioBw.avg > 0 ? audioBw.avg : a.bitrate)) * 1.05, + ); + + let audioCodecTag = 'mp4a.40.2'; + if (a.codec === 'libfdk_aac' && a.profile === 'aac_he_v2') audioCodecTag = 'mp4a.40.29'; + else if (a.codec === 'ac3') audioCodecTag = 'ac-3'; + else if (a.codec === 'eac3') audioCodecTag = 'ec-3'; + + const attributes = [ + `AVERAGE-BANDWIDTH=${localAvgBandwidth}`, + `BANDWIDTH=${localPeakBandwidth}`, + `VIDEO-RANGE=${actualVideoRange}`, + `CLOSED-CAPTIONS=NONE`, + `CODECS="${v.videoCodecTag},${audioCodecTag}"`, + `AUDIO="${audioName}"`, + `FRAME-RATE=${frameRateString}`, + `RESOLUTION=${v.actualWidth}x${v.actualHeight}`, + ].join(','); + + lines.push(`#EXT-X-STREAM-INF:${attributes}`, relativeUri); + } + } + + const masterPath = path.join(outputDir, HLS_CONSTANTS.MASTER_PLAYLIST_NAME); + await fs.writeFile(masterPath, lines.join('\n')); + logger.info({ masterPath }, 'Master playlist written'); +} diff --git a/src/infrastructure/ffmpeg/index.ts b/src/infrastructure/ffmpeg/index.ts new file mode 100644 index 0000000..a71d546 --- /dev/null +++ b/src/infrastructure/ffmpeg/index.ts @@ -0,0 +1 @@ +export { FFmpegAdapter } from './adapter.js'; diff --git a/src/infrastructure/ffmpeg/types.ts b/src/infrastructure/ffmpeg/types.ts new file mode 100644 index 0000000..c7805d4 --- /dev/null +++ b/src/infrastructure/ffmpeg/types.ts @@ -0,0 +1,52 @@ +import type { ProgressCallback } from '../../domain/job.interface.js'; + +export interface VideoProfile { + name: string; + width: number; + height: number; + bitrate: number; + maxrate: number; + bufsize: number; + hlsTime?: number; + frameRate?: number; + videoCodec?: string; + videoCodecTag: string; + videoRange?: string; + preset: string; + profile?: string; + level?: string; +} + +export interface AudioProfile { + name: string; + channels: number; + codec: string; + profile?: string; + bitrate: number; + sampleRate: number; + hardwareProfile: boolean; +} + +export type VideoVariantMeta = VideoProfile & { + actualWidth: number; + actualHeight: number; + relativeUrl: string; +}; + +export type AudioVariantMeta = AudioProfile & { + sourceChannels: number; + language: string; + streamIndex: number; + title: string; + relativeUrl: string; +}; + +export interface RunOptions { + args: string[]; + label: string; + videoId: string; + onProgress?: ProgressCallback; + duration?: number; +} + +export const DEFAULT_WORK_DIR = '/tmp/worker'; diff --git a/src/infrastructure/queue/video.worker.ts b/src/infrastructure/queue/video.worker.ts new file mode 100644 index 0000000..6d89e18 --- /dev/null +++ b/src/infrastructure/queue/video.worker.ts @@ -0,0 +1,89 @@ +import { Worker, Job, UnrecoverableError } from 'bullmq'; +import { config } from '../../config/env.js'; +import { pino } from 'pino'; +import type { JobData, ProcessVideoUseCase } from '../../domain/job.interface.js'; + +const logger = pino({ name: 'QueueWorker' }); + +/** + * BullMQ consumer that binds the Redis queue to the `ProcessVideo` domain logic. + * + * @remarks + * - Maps domain progress callbacks directly to BullMQ `job.updateProgress`. + * - Relies on Redis lock durations (`config.JOB_LOCK_DURATION_MS`) to detect computationally locked/crashed workers. + * - Does not throw on failure; relies on BullMQ's internal retry mechanism and `.on('failed')` listeners. + */ +export class VideoWorker { + private readonly worker: Worker; + + constructor(private readonly processVideo: ProcessVideoUseCase) { + const redisUrl = new URL(config.REDIS_URL); + const connection = { + host: redisUrl.hostname, + port: parseInt(redisUrl.port, 10), + username: redisUrl.username || undefined, + password: redisUrl.password || undefined, + tls: redisUrl.protocol === 'rediss:' ? { rejectUnauthorized: false } : undefined, + keepAlive: 10000, + enableReadyCheck: false, + maxRetriesPerRequest: null, + }; + + this.worker = new Worker( + 'video-processing', + async (job: Job) => { + logger.info( + { jobId: job.id, videoId: job.data.videoId, attempt: job.attemptsMade + 1 }, + 'Processing job', + ); + + await this.processVideo.execute(job.data, (progress) => { + job.updateProgress(progress).catch(() => {}); + }); + }, + { + connection, + concurrency: config.WORKER_CONCURRENCY, + autorun: false, + lockDuration: config.JOB_LOCK_DURATION_MS, + lockRenewTime: config.JOB_LOCK_RENEW_MS, + }, + ); + + this.worker.on('completed', (job: Job) => { + logger.info( + { + jobId: job.id, + videoId: job.data.videoId, + duration: `${Date.now() - job.timestamp}ms`, + }, + 'Job completed', + ); + }); + + this.worker.on('failed', (job: Job | undefined, err: Error) => { + logger.error( + { jobId: job?.id, videoId: job?.data?.videoId, attempt: job?.attemptsMade, err }, + 'Job failed', + ); + }); + + this.worker.on('stalled', (jobId: string) => { + logger.warn({ jobId }, 'Job stalled — may need manual retry'); + }); + + this.worker.on('error', (err: Error) => { + logger.error({ err }, 'Worker error'); + }); + } + + start(): void { + this.worker.run(); + logger.info('Worker started listening on queue: video-processing'); + } + + async close(): Promise { + await this.worker.close(); + logger.info('Worker shut down gracefully'); + } +} diff --git a/src/infrastructure/storage/azure.service.ts b/src/infrastructure/storage/azure.service.ts new file mode 100644 index 0000000..2e7f468 --- /dev/null +++ b/src/infrastructure/storage/azure.service.ts @@ -0,0 +1,111 @@ +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { BlobServiceClient } from '@azure/storage-blob'; +import { DefaultAzureCredential } from '@azure/identity'; +import { pino } from 'pino'; +import { config } from '../../config/env.js'; +import type { ProgressCallback } from '../../domain/job.interface.js'; + +const logger = pino({ name: 'AzureStorage' }); + +/** + * Azure Blob Storage adapter for uploading generated HLS playlists and segments. + * + * @remarks + * - Recursively scans the local output directory and mirrors the final structure to the Blob container. + * - Automatically infers and injects correct `Content-Type` headers (`application/vnd.apple.mpegurl` or `video/mp4`). + * - Security note: Uses `DefaultAzureCredential` in production (Managed Identity), falling back to connection strings locally. + */ +export class AzureStorageService { + private readonly blobServiceClient: BlobServiceClient; + private readonly containerName = config.AZURE_STORAGE_CONTAINER_NAME; + private readonly envDirectory = config.CONTAINER_DIRECTORY_1; + + constructor() { + if (config.NODE_ENV === 'production') { + if (!config.AZURE_STORAGE_ACCOUNT_URL) { + throw new Error( + 'AZURE_STORAGE_ACCOUNT_URL is required in production for Managed Identity', + ); + } + this.blobServiceClient = new BlobServiceClient( + config.AZURE_STORAGE_ACCOUNT_URL, + new DefaultAzureCredential(), + ); + logger.info('Azure Storage authenticated via Managed Identity'); + } else { + if (!config.AZURE_STORAGE_CONNECTION_STRING) { + throw new Error('AZURE_STORAGE_CONNECTION_STRING is required in development'); + } + this.blobServiceClient = BlobServiceClient.fromConnectionString( + config.AZURE_STORAGE_CONNECTION_STRING, + ); + logger.info('Azure Storage authenticated via Connection String'); + } + } + + private async getFilesRecursive(dir: string): Promise { + const entries = await fs.readdir(dir, { withFileTypes: true }); + const files = await Promise.all( + entries.map((entry) => { + const res = path.resolve(dir, entry.name); + return entry.isDirectory() ? this.getFilesRecursive(res) : res; + }), + ); + return Array.prototype.concat(...files) as string[]; + } + + async uploadHLS( + folderPath: string, + videoId: string, + onProgress?: ProgressCallback, + ): Promise { + const containerClient = this.blobServiceClient.getContainerClient(this.containerName); + await containerClient.createIfNotExists({ access: 'blob' }); + + const files = await this.getFilesRecursive(folderPath); + let uploadedCount = 0; + let masterPlaylistUrl = ''; + + for (const filePath of files) { + const relativeToHlsDir = path.relative(folderPath, filePath).replace(/\\/g, '/'); + let blobPath = ''; + + if (relativeToHlsDir === 'playlist.m3u8') { + blobPath = `${this.envDirectory}/${videoId}/playlist.m3u8`; + } else if (relativeToHlsDir.startsWith('v1/')) { + blobPath = `${this.envDirectory}/${relativeToHlsDir}`; + } else { + blobPath = `${this.envDirectory}/${videoId}/${relativeToHlsDir}`; + } + + const blockBlobClient = containerClient.getBlockBlobClient(blobPath); + let contentType = 'application/octet-stream'; + + if (filePath.endsWith('.m3u8')) { + contentType = 'application/vnd.apple.mpegurl'; + } else if (filePath.endsWith('.m4s') || filePath.endsWith('.mp4')) { + contentType = 'video/mp4'; + } + + const fileBuffer = await fs.readFile(filePath); + await blockBlobClient.uploadData(fileBuffer, { + blobHTTPHeaders: { + blobContentType: contentType, + }, + }); + + if (relativeToHlsDir === 'playlist.m3u8') { + masterPlaylistUrl = blockBlobClient.url; + } + + uploadedCount++; + if (onProgress) { + onProgress({ variant: 'Azure Upload', percent: (uploadedCount / files.length) * 100 }); + } + } + + logger.info({ videoId, uploadedFiles: uploadedCount }, 'Azure upload complete'); + return masterPlaylistUrl; + } +} diff --git a/src/server.ts b/src/server.ts new file mode 100644 index 0000000..d5d4568 --- /dev/null +++ b/src/server.ts @@ -0,0 +1,104 @@ +/** + * Application entry point: Bootstraps the HTTP server and BullMQ worker. + * + * @remarks + * - Wires the dependency injection graph for the video processing pipeline. + * - Exposes Kubernetes-compatible readiness (`/ready`) and liveness (`/health`) probes. + * - Enforces a strict 30s graceful shutdown timeout on SIGINT/SIGTERM to prevent + * orphaned pods if external dependencies (e.g., Azure Storage, DB) hang. + */ +import Fastify, { FastifyInstance } from 'fastify'; +import cors from '@fastify/cors'; +import helmet from '@fastify/helmet'; +import rateLimit from '@fastify/rate-limit'; +import { config } from './config/env.js'; +import { VideoWorker } from './infrastructure/queue/video.worker.js'; +import { ProcessVideo } from './application/video.process.js'; +import { FFmpegAdapter } from './infrastructure/ffmpeg/index.js'; +import { AzureStorageService } from './infrastructure/storage/azure.service.js'; +import { PostgresVideoRepository } from './infrastructure/db/db.js'; + +const server: FastifyInstance = Fastify({ + logger: { + level: config.NODE_ENV === 'production' ? 'info' : 'debug', + }, +}); + +await server.register(helmet); + +const allowedOrigins = config.CORS_ORIGIN === '*' ? '*' : config.CORS_ORIGIN.split(','); +await server.register(cors, { + origin: allowedOrigins, +}); + +await server.register(rateLimit, { + max: 100, + timeWindow: '1 minute', +}); + +server.get('/health', async () => ({ + status: 'ok', + uptime: process.uptime(), + timestamp: new Date().toISOString(), +})); + +let isReady = false; + +server.get('/ready', async (_req, reply) => { + if (isReady) return { status: 'ready' }; + return reply.status(503).send({ status: 'not_ready' }); +}); + +/** + * Bootstraps external dependencies, starts the worker pipeline, and binds the server. + * + * Flow during SIGINT/SIGTERM: + * 1. Sets `isReady = false` (returns HTTP 503) to drain load balancer traffic. + * 2. Waits for the active BullMQ worker job to finish (up to 30s). + * 3. Gracefully closes DB connections and the HTTP server. + */ +const start = async () => { + try { + const ffmpeg = new FFmpegAdapter(); + const storage = new AzureStorageService(); + + const db = new PostgresVideoRepository(config.DATABASE_URL); + const processVideo = new ProcessVideo(ffmpeg, storage, db); + const worker = new VideoWorker(processVideo); + worker.start(); + + await server.listen({ port: config.PORT, host: '0.0.0.0' }); + isReady = true; + server.log.info(`Worker service ready on port ${config.PORT}`); + + const shutdown = async (signal: string) => { + server.log.info(`Received ${signal}, shutting down...`); + isReady = false; + + const timeout = setTimeout(() => { + server.log.error('Shutdown timed out after 30s, forcing exit'); + process.exit(1); + }, 30_000); + + try { + await worker.close(); + await db.close(); + await server.close(); + clearTimeout(timeout); + process.exit(0); + } catch (err) { + server.log.error({ err }, 'Error during shutdown'); + clearTimeout(timeout); + process.exit(1); + } + }; + + process.on('SIGINT', () => shutdown('SIGINT')); + process.on('SIGTERM', () => shutdown('SIGTERM')); + } catch (err) { + server.log.error(err); + process.exit(1); + } +}; + +start(); diff --git a/test/ffmpeg.test.sh b/test/ffmpeg.test.sh index 32029d5..85a54fc 100644 --- a/test/ffmpeg.test.sh +++ b/test/ffmpeg.test.sh @@ -73,7 +73,7 @@ build_image() { verify_ffmpeg_version() { log_header "Step 2: Verifying FFmpeg Version" local output - output=$(docker run --rm "${IMAGE_NAME}" -version) + output=$(docker run --rm "${IMAGE_NAME}" ffmpeg -version) echo "$output" | head -n 1 } @@ -81,7 +81,7 @@ verify_ffmpeg_version() { verify_ffprobe_version() { log_header "Step 3: Verifying FFprobe Version" local output - output=$(docker run --rm --entrypoint ffprobe "${IMAGE_NAME}" -version) + output=$(docker run --rm "${IMAGE_NAME}" ffprobe -version) echo "$output" | head -n 1 } @@ -91,7 +91,7 @@ verify_codecs() { log_info "Checking for required libraries: libx264 (H.264) and libfdk_aac (AAC)..." local codecs - if codecs=$(docker run --rm "${IMAGE_NAME}" -codecs 2>/dev/null | grep -E "libx264|libfdk_aac"); then + if codecs=$(docker run --rm "${IMAGE_NAME}" ffmpeg -codecs 2>/dev/null | grep -E "libx264|libfdk_aac"); then echo "$codecs" log_success "Required codecs verified successfully." else @@ -126,7 +126,7 @@ run_transcode_test() { -v "$(pwd)/${VIDEO_DIR}:/input:ro" \ -v "$(pwd)/${OUTPUT_DIR}:/output" \ "${IMAGE_NAME}" \ - -y \ + ffmpeg -y \ -hide_banner -loglevel error \ -stats \ -i "/input/${TEST_VIDEO_FILE}" \ @@ -169,21 +169,21 @@ run_hls_test() { # Pass 1: 360p log_info "Generating 360p HLS stream segment..." - docker run ${docker_opts} "${IMAGE_NAME}" ${ffmpeg_opts} \ + docker run ${docker_opts} "${IMAGE_NAME}" ffmpeg ${ffmpeg_opts} \ -i "/input/${TEST_VIDEO_FILE}" -t 10 \ -vf "scale=640:360" -c:v libx264 -preset ultrafast -b:v 800k \ ${hls_flags} -hls_segment_filename '/output/360p/segment_%03d.ts' -y '/output/360p/stream.m3u8' # Pass 2: 720p log_info "Generating 720p HLS stream segment..." - docker run ${docker_opts} "${IMAGE_NAME}" ${ffmpeg_opts} \ + docker run ${docker_opts} "${IMAGE_NAME}" ffmpeg ${ffmpeg_opts} \ -i "/input/${TEST_VIDEO_FILE}" -t 10 \ -vf "scale=1280:720" -c:v libx264 -preset ultrafast -b:v 2800k \ ${hls_flags} -hls_segment_filename '/output/720p/segment_%03d.ts' -y '/output/720p/stream.m3u8' # Pass 3: 1080p log_info "Generating 1080p HLS stream segment..." - docker run ${docker_opts} "${IMAGE_NAME}" ${ffmpeg_opts} \ + docker run ${docker_opts} "${IMAGE_NAME}" ffmpeg ${ffmpeg_opts} \ -i "/input/${TEST_VIDEO_FILE}" -t 10 \ -vf "scale=1920:1080" -c:v libx264 -preset ultrafast -b:v 5000k \ ${hls_flags} -hls_segment_filename '/output/1080p/segment_%03d.ts' -y '/output/1080p/stream.m3u8' diff --git a/test/queue-job.test.local.ts b/test/queue-job.test.local.ts new file mode 100644 index 0000000..dfe56bf --- /dev/null +++ b/test/queue-job.test.local.ts @@ -0,0 +1,193 @@ +/** + * Local E2E demo script used to test the video-processing pipeline. + * + * This file acts as a lightweight simulation of the main backend. It inserts + * a test record into Postgres and enqueues a BullMQ job so the FFmpeg worker + * (microservice) can pick it up and process the video. + * + * The purpose of this script is to validate the full processing flow locally: + * + * Backend (simulated) → Postgres → BullMQ Queue → FFmpeg Worker → Video Processing + * + * This is only a small demo/test @utility and is not part of the production + * backend logic. + * + * ----------------------------------------------------------------------------- + * Required environment variables (defined in `.env`) + * @See [.env.example](https://github.com/maulik-mk/ffmpeg-queue-worker-node/blob/main/.env.example) + * ----------------------------------------------------------------------------- + * + * | Variable | Description | + * | ---------------------- | ------------------------------------------------ | + * | `REDIS_URL` | Redis connection URL (`redis://` or `rediss://`) | + * | `DATABASE_URL` | Postgres database URL | + * | `RAW_VIDEO_SOURCE_URL` | Public URL of the source video to process | + * + * ----------------------------------------------------------------------------- + * Optional environment variables + * ----------------------------------------------------------------------------- + * + * | Variable | Default | Description | + * | ----------------- | ------------- | ------------------------------------ | + * | `HLS_OUTPUT_MODE` | `"SEGMENTED"` | `"SEGMENTED"` or `"SINGLE_FILE"` | + * + * ----------------------------------------------------------------------------- + * HLS Output Modes + * ----------------------------------------------------------------------------- + * + * SEGMENTED + * --------- + * FFmpeg outputs each HLS segment as a separate fMP4 (`.m4s`) file along with + * an initialization segment. This is the standard HLS VOD layout. + * + * Player behavior: + * - Fetches small segments on demand + * - Playback can begin after the first segment + * - Adaptive bitrate switching happens at segment boundaries + * - Seeking targets individual segments + * + * Infrastructure implications: + * - Per-segment CDN caching and invalidation + * - Parallel uploads to Azure Blob Storage + * - Larger object count per video + * + * SINGLE_FILE + * ----------- + * FFmpeg appends all media data into a single fMP4 file per variant and + * references byte ranges via `EXT-X-BYTERANGE`. + * + * Player behavior: + * - Uses HTTP Range requests + * - Playback, ABR switching, and seeking work normally on compliant CDNs + * - May buffer if byte-range caching is poorly supported + * + * Infrastructure implications: + * - Fewer objects stored in Azure Blob Storage + * - Simpler storage management and cleanup + * - Entire variant must upload before the manifest becomes usable + * + * ----------------------------------------------------------------------------- + * Usage + * ----------------------------------------------------------------------------- + * + * ```sh + * pnpm run dev:local:test:job + * pnpm run dev:local:e2e + * ``` + * + * ----------------------------------------------------------------------------- + * References + * ----------------------------------------------------------------------------- + * + * @See `../.env.example` for the full list of supported environment variables. + */ + +import { Queue } from "bullmq"; +import { pino } from "pino"; +import { v7 as uuidv7 } from "uuid"; + +const logger = pino({ name: "QueueTestJob" }); +const QUEUE_NAME = "video-processing"; + +const REQUIRED_ENV = ["REDIS_URL", "DATABASE_URL", "RAW_VIDEO_SOURCE_URL"] as const; + +/** + * Validates that all required environment variables are present. + * + * @returns Resolved environment configuration. + * @throws Logs missing keys and exits with code `1` if any are absent. + */ +function validateEnv(): { + redisUrl: string; + databaseUrl: string; + sourceUrl: string; + hlsOutputMode: string; +} { + const missing = REQUIRED_ENV.filter((key) => !process.env[key]); + if (missing.length > 0) { + logger.error( + { missing }, + `Missing required environment variable(s). Add them to your .env file and retry.`, + ); + process.exit(1); + } + + return { + redisUrl: process.env.REDIS_URL!, + databaseUrl: process.env.DATABASE_URL!, + sourceUrl: process.env.RAW_VIDEO_SOURCE_URL!, + hlsOutputMode: process.env.HLS_OUTPUT_MODE ?? "SEGMENTED", + }; +} + +/** + * Parses a Redis URL into a BullMQ-compatible connection object. + * + * @param redisUrl - Full Redis connection URL (`redis://` or `rediss://`). + * @returns Connection options including optional TLS configuration. + */ +function parseRedisConnection(redisUrl: string) { + const url = new URL(redisUrl); + return { + host: url.hostname, + port: parseInt(url.port), + username: url.username, + password: url.password, + tls: url.protocol === "rediss:" ? { rejectUnauthorized: false } : undefined, + }; +} + +/** + * Entry point — connects to Redis, seeds Postgres, and enqueues a + * `process-video` job. + */ +async function main() { + const env = validateEnv(); + const videoId = uuidv7(); + + // #1. Connect to Redis + logger.info("Connecting to Redis…"); + const connection = parseRedisConnection(env.redisUrl); + const queue = new Queue(QUEUE_NAME, { connection }); + + // #2. Seed the database + logger.info({ videoId }, "Seeding Postgres…"); + const pg = await import("pg"); + const connectionString = env.databaseUrl; + const pool = new pg.default.Pool({ + connectionString, + ssl: { + rejectUnauthorized: false + } + }); + + try { + await pool.query( + `INSERT INTO videos (id, user_id, source_url, status) VALUES ($1, $2, $3, $4)`, + [videoId, "test-user-local", env.sourceUrl, "queued"], + ); + } catch (err) { + logger.error({ err }, "Failed to seed database"); + await pool.end(); + await queue.close(); + process.exit(1); + } + await pool.end(); + logger.info("Database seeded successfully"); + + // #3. Enqueue the job + logger.info({ videoId, sourceUrl: env.sourceUrl }, "Adding job to queue…"); + + await queue.add("process-video", { + videoId, + sourceUrl: env.sourceUrl, + userId: "test-user-local", + hlsOutputMode: env.hlsOutputMode, + webhookUrl: null, + }); + + logger.info("Job added successfully — start the worker to process it."); + await queue.close(); +} + +main(); diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..8f00d06 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,24 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "rootDir": "./src", + "outDir": "./dist", + "strict": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "skipLibCheck": true, + "noEmit": false, + "declaration": true, + "sourceMap": true, + "resolveJsonModule": true + }, + "include": [ + "src/**/*" + ], + "exclude": [ + "node_modules", + "test" + ] +} \ No newline at end of file From 3cd877c81d6087e7f6db98ec0658a6fb372efaaa Mon Sep 17 00:00:00 2001 From: MAULIK MK <214461757+maulik-mk@users.noreply.github.com> Date: Sat, 21 Mar 2026 16:09:39 +0530 Subject: [PATCH 08/12] CI: add test scripts and enhance CI workflow (#12) * chore(ci): add test scripts and enhance CI workflow - Add scripts to download test video and run ffmpeg tests - Refactor CI workflow for better Docker versioning, labeling, and image push logic - Integrate new scripts into CI for improved test automation * fix(ci): update script paths for downloading test assets and running tests * fix(ci): make scripts in .github/scripts executable --- .github/scripts/download_test_video.sh | 7 ++ .github/scripts/run_tests.sh | 5 ++ .github/workflows/ci.yml | 108 ++++++++++++++++++------- 3 files changed, 89 insertions(+), 31 deletions(-) create mode 100755 .github/scripts/download_test_video.sh create mode 100755 .github/scripts/run_tests.sh diff --git a/.github/scripts/download_test_video.sh b/.github/scripts/download_test_video.sh new file mode 100755 index 0000000..411ca90 --- /dev/null +++ b/.github/scripts/download_test_video.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -e + +mkdir -p video +echo "Downloading sample video..." +curl -L -o video/test.mp4 https://test-videos.co.uk/vids/bigbuckbunny/mp4/h264/1080/Big_Buck_Bunny_1080_10s_5MB.mp4 +ls -lh video/ \ No newline at end of file diff --git a/.github/scripts/run_tests.sh b/.github/scripts/run_tests.sh new file mode 100755 index 0000000..7af13ce --- /dev/null +++ b/.github/scripts/run_tests.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -e + +chmod +x test/ffmpeg.test.sh +bash test/ffmpeg.test.sh \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aff66eb..cc92793 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: push: - branches: [ "main", "master", 'dev'] + branches: [ "main", "master", "dev" ] pull_request: - branches: [ "main", "master", 'dev'] + branches: [ "main", "master", "dev" ] concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -24,60 +24,106 @@ jobs: - name: Checkout Code uses: actions/checkout@v4 - # Setup Docker Buildx to enable advanced build features and caching + # 2. Extract Dockerfile VERSION + - name: Read Dockerfile VERSION + id: docker_version + run: | + VERSION=$(grep -E '^ARG VERSION=' Dockerfile | cut -d '=' -f2 | tr -d '"') + echo "VERSION=$VERSION" >> $GITHUB_ENV + echo "Found Dockerfile VERSION=$VERSION" + if [[ "$VERSION" =~ \.0$ ]]; then + echo "IS_MAJOR_MINOR=true" >> $GITHUB_ENV + else + echo "IS_MAJOR_MINOR=false" >> $GITHUB_ENV + fi + + # 3. Set Build Date + - name: Set Build Date + if: github.ref == 'refs/heads/main' + run: | + BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + echo "BUILD_DATE=$BUILD_DATE" >> $GITHUB_ENV + echo "Build date: $BUILD_DATE" + + # 4. Extract all labels from Dockerfile + - name: Extract Dockerfile Labels + id: docker_labels + run: | + echo "Extracting labels from Dockerfile..." + # Remove leading LABEL and combine into single line (multi-line support) + LABELS=$(grep '^LABEL ' Dockerfile | sed 's/^LABEL //' | tr '\n' ' ') + # Replace ARG placeholders with ENV values + LABELS="${LABELS//\${VERSION}/${{ env.VERSION }}}" + LABELS="${LABELS//\${BUILD_DATE}/${{ env.BUILD_DATE }}}" + echo "DOCKER_LABELS=$LABELS" >> $GITHUB_ENV + echo "Extracted labels: $LABELS" + + # 5. Check if Docker image VERSION exists in GHCR + - name: Check if Docker image VERSION exists + if: github.ref == 'refs/heads/main' && env.IS_MAJOR_MINOR == 'true' + id: check_version + run: | + EXISTING_TAG=$(curl -s -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + "https://ghcr.io/v2/${{ github.repository }}/tags/list" | jq -r '.tags[]?' | grep "^${{ env.VERSION }}$" || true) + echo "EXISTING_TAG=$EXISTING_TAG" >> $GITHUB_ENV + if [ "$EXISTING_TAG" = "${{ env.VERSION }}" ]; then + echo "Docker image VERSION=${{ env.VERSION }} already exists. Skipping build/push." + else + echo "Docker image VERSION=${{ env.VERSION }} is new. Will build/push." + fi + + # 6. Set up Docker Buildx - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - # Build and cache Docker layers using GitHub Actions cache backend - # 'load: true' exports the image to the local Docker daemon for testing + # 7. Build Docker image (only if new VERSION) - name: Build and Load Docker Image + if: github.ref == 'refs/heads/main' && env.EXISTING_TAG == '' && env.IS_MAJOR_MINOR == 'true' uses: docker/build-push-action@v5 with: context: . load: true - tags: worker-ffmpeg:latest + build-args: | + VERSION=${{ env.VERSION }} + BUILD_DATE=${{ env.BUILD_DATE }} + labels: ${{ env.DOCKER_LABELS }} + tags: | + worker-ffmpeg:${{ env.VERSION }} + worker-ffmpeg:latest cache-from: type=gha cache-to: type=gha,mode=max - # Authenticate to GHCR for image publishing + # 8. Log in to GHCR - name: Log in to GitHub Container Registry + if: github.ref == 'refs/heads/main' && env.EXISTING_TAG == '' && env.IS_MAJOR_MINOR == 'true' uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - # Fetch reference test asset (1080p H.264) - # Dynamic download avoids repository bloat + # 9. Download Test Asset - name: Download Test Asset - run: | - mkdir -p video - echo "Downloading sample video..." - # Source: Big Buck Bunny (Stable URL) - curl -L -o video/test.mp4 https://test-videos.co.uk/vids/bigbuckbunny/mp4/h264/1080/Big_Buck_Bunny_1080_10s_5MB.mp4 - ls -lh video/ - - # Execute integration test suite - # Patch script to skip redundant build step (using cached image) + run: .github/scripts/download_test_video.sh + + # 10. Run Test Suite - name: Run Test Suite - env: - SKIP_BUILD: 'true' - run: | - chmod +x test/ffmpeg.test.sh - bash test/ffmpeg.test.sh + run: .github/scripts/run_tests.sh - # Publish artifact to GHCR (Production branch only) + # 11. Push Docker image to GHCR (only if new VERSION) - name: Push to GHCR - if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/main' && env.EXISTING_TAG == '' && env.IS_MAJOR_MINOR == 'true' run: | IMAGE_ID=ghcr.io/${{ github.repository }} - # Normalize repository name to lowercase IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') - - # Apply semantic tags (latest, SHA) for versioning + + # Tag images + docker tag worker-ffmpeg:${{ env.VERSION }} $IMAGE_ID:${{ env.VERSION }} docker tag worker-ffmpeg:latest $IMAGE_ID:latest docker tag worker-ffmpeg:latest $IMAGE_ID:${{ github.sha }} - - echo "Pushing $IMAGE_ID:latest" + + # Push images + echo "Pushing $IMAGE_ID:${{ env.VERSION }}" + docker push $IMAGE_ID:${{ env.VERSION }} docker push $IMAGE_ID:latest - docker push $IMAGE_ID:${{ github.sha }} + docker push $IMAGE_ID:${{ github.sha }} \ No newline at end of file From 89a3537fa4b82beae3639ba3966eca88988cd8b7 Mon Sep 17 00:00:00 2001 From: MAULIK MK <214461757+maulik-mk@users.noreply.github.com> Date: Sat, 21 Mar 2026 17:01:57 +0530 Subject: [PATCH 09/12] Release : v0.2.0 (#14) - Sync changes of v0.2.0 PR --- Dockerfile | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4ea1852..a29a39b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -170,7 +170,7 @@ ENV DEBIAN_FRONTEND=noninteractive # ----------------------------------------------------------------------------- # Metadata & OCI Labels # ----------------------------------------------------------------------------- -ARG VERSION=0.1.0 +ARG VERSION=0.2.0 ARG BUILD_DATE=unknown LABEL org.opencontainers.image.title="ffmpeg-queue-worker-node" \ diff --git a/package.json b/package.json index 3b483a5..c73ed2c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "worker", - "version": "0.1.0", + "version": "0.2.0", "description": "FFmpeg Worker Service (TypeScript)", "repository": { "type": "git", From 4545375bf6e51c2c2cb64c6ba4923e4402346c53 Mon Sep 17 00:00:00 2001 From: MAULIK MK <214461757+maulik-mk@users.noreply.github.com> Date: Wed, 1 Apr 2026 22:38:59 +0530 Subject: [PATCH 10/12] feat: optimize database connection pooling (#15) --- .env.example | 2 +- src/infrastructure/db/db.ts | 6 +++--- test/queue-job.test.local.ts | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index 29ef4ee..620cb72 100644 --- a/.env.example +++ b/.env.example @@ -5,7 +5,7 @@ # Core System Services PORT=3000 REDIS_URL=rediss://user:password@host:port -DATABASE_URL=postgresql://user:password@host:port/dbname +DATABASE_URL=postgresql://postgres.ABC-WASD-XYZ:[PASSWORD]@[aws-1-ap-south-1].pooler.supabase.com:6543/postgres CORS_ORIGIN=* # Azure Blob Storage (Shared Configuration) diff --git a/src/infrastructure/db/db.ts b/src/infrastructure/db/db.ts index 539f15a..f6ff9f8 100644 --- a/src/infrastructure/db/db.ts +++ b/src/infrastructure/db/db.ts @@ -23,9 +23,9 @@ export class PostgresVideoRepository implements VideoRepository { constructor(connectionString: string) { this.pool = new pg.Pool({ connectionString, - ssl: { - rejectUnauthorized: false, - }, + max: 20, + idleTimeoutMillis: 30000, + connectionTimeoutMillis: 5000, }); } diff --git a/test/queue-job.test.local.ts b/test/queue-job.test.local.ts index dfe56bf..e188546 100644 --- a/test/queue-job.test.local.ts +++ b/test/queue-job.test.local.ts @@ -156,9 +156,9 @@ async function main() { const connectionString = env.databaseUrl; const pool = new pg.default.Pool({ connectionString, - ssl: { - rejectUnauthorized: false - } + max: 20, + idleTimeoutMillis: 30000, + connectionTimeoutMillis: 5000, }); try { From 4fc1d030c240cf45213cf47c799f3dbf3dbd533e Mon Sep 17 00:00:00 2001 From: MAULIK MK <214461757+maulik-mk@users.noreply.github.com> Date: Sun, 5 Apr 2026 18:17:54 +0530 Subject: [PATCH 11/12] feat: migrate VMAF to ABR ladder (#16) - Migrate VMAF complexity probe to static ABR bitrate ladder. - Support upto 4K Dolby Vision and and 16ch audio - For more details #16 --- .env.example | 35 +- .gitignore | 6 +- src/application/video.process.ts | 49 ++- src/config/env.ts | 12 +- src/domain/job.interface.ts | 1 + src/infrastructure/ffmpeg/adapter.ts | 19 +- src/infrastructure/ffmpeg/constants.ts | 10 - src/infrastructure/ffmpeg/core/complexity.ts | 192 ---------- src/infrastructure/ffmpeg/core/probe.ts | 9 +- .../ffmpeg/encoding/ABR/audio/audio.json | 47 +++ .../ffmpeg/encoding/ABR/video/avc.sdr.json | 206 +++++++++++ .../ffmpeg/encoding/ABR/video/dvh.pq.json | 223 ++++++++++++ .../ffmpeg/encoding/ABR/video/hvc.pq.json | 223 ++++++++++++ .../ffmpeg/encoding/ABR/video/hvc.sdr.json | 223 ++++++++++++ src/infrastructure/ffmpeg/encoding/flags.ts | 130 ++++++- .../ffmpeg/encoding/profiles.ts | 132 +++++-- .../ffmpeg/encoding/profiles/audio.json | 51 --- .../ffmpeg/encoding/profiles/video.json | 338 ------------------ src/infrastructure/ffmpeg/hls/pipeline.ts | 89 ++++- src/infrastructure/ffmpeg/hls/playlist.ts | 163 ++++++--- src/infrastructure/ffmpeg/types.ts | 7 + src/infrastructure/storage/azure.service.ts | 98 +++-- 22 files changed, 1495 insertions(+), 768 deletions(-) delete mode 100644 src/infrastructure/ffmpeg/core/complexity.ts create mode 100644 src/infrastructure/ffmpeg/encoding/ABR/audio/audio.json create mode 100644 src/infrastructure/ffmpeg/encoding/ABR/video/avc.sdr.json create mode 100644 src/infrastructure/ffmpeg/encoding/ABR/video/dvh.pq.json create mode 100644 src/infrastructure/ffmpeg/encoding/ABR/video/hvc.pq.json create mode 100644 src/infrastructure/ffmpeg/encoding/ABR/video/hvc.sdr.json delete mode 100644 src/infrastructure/ffmpeg/encoding/profiles/audio.json delete mode 100644 src/infrastructure/ffmpeg/encoding/profiles/video.json diff --git a/.env.example b/.env.example index 620cb72..c8add50 100644 --- a/.env.example +++ b/.env.example @@ -15,14 +15,45 @@ AZURE_UPLOAD_BATCH_SIZE=20 AZURE_UPLOAD_RETRIES=3 # Queue Stability +# CONCURRENCY = number of parallel jobs. Each job spawns FFmpeg which uses FFMPEG_THREADS cores. +# Set to 1 for max single-job speed, or 2 to process 2 videos simultaneously (cores split between them). WORKER_CONCURRENCY=1 -JOB_LOCK_DURATION_MS=120000 -JOB_LOCK_RENEW_MS=30000 + +# Lock must survive the entire encode pipeline (can take 30+ minutes for full-length content). +# Renewal interval should be aggressive (15s) to survive CPU-starved Node.js event loops. +JOB_LOCK_DURATION_MS=1800000 +JOB_LOCK_RENEW_MS=15000 # Video Pipeline Settings # "SINGLE_FILE" (Byte-range fMP4) or "SEGMENTED" (Standard chunks) HLS_OUTPUT_MODE="SEGMENTED" +# CDN base URL prepended to HLS segment and init-segment URIs in variant manifests. +# Leave unset for relative paths (local dev). Set to full CDN URL for production. +DOMAIN_SUBDOMAIN_NAME=https://vod-cdn.{SUBDOMAIN}.{DOMAIN}.com + +# ============================================================================== +# PERFORMANCE TUNING +# ============================================================================== + +# Global FFmpeg thread count. 0 = auto-detect (recommended). +# FFmpeg uses this for demuxing, filtering, and muxing threads. +FFMPEG_THREADS=0 + +# x265 (HEVC/Dolby Vision) thread pool size. Set to your vCPU count for max utilization. +# This is the BIGGEST performance lever — pools=none previously disabled ALL threading. +# Example: 32-core machine → X265_POOL_SIZE=32 +X265_POOL_SIZE=32 + +# x265 frame-level parallelism. How many frames encode simultaneously. +# 4 is optimal for most machines. Higher values use more RAM but increase throughput. +# Rule of thumb: 2-6 depending on available RAM (each frame buffer ~50-200MB for 4K). +X265_FRAME_THREADS=4 + +# Developer Override: Force the system to use ONLY one group of profiles. +# Values: 'avc_sdr', 'hvc_sdr', 'hvc_pq', 'dvh_pq', 'ALL' +TEST_VIDEO_PROFILE=ALL + # ============================================================================== # PRODUCTION ONLY diff --git a/.gitignore b/.gitignore index c0e2b02..ec2a396 100644 --- a/.gitignore +++ b/.gitignore @@ -139,5 +139,7 @@ vite.config.js.timestamp-* vite.config.ts.timestamp-* .DS_Store -video -output \ No newline at end of file +output + +# DEVELOPMENT ONLY local files +tmp \ No newline at end of file diff --git a/src/application/video.process.ts b/src/application/video.process.ts index 21f9009..5dc5737 100644 --- a/src/application/video.process.ts +++ b/src/application/video.process.ts @@ -1,3 +1,7 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; import type { JobData, ProcessVideoUseCase, @@ -12,11 +16,7 @@ import { pino } from 'pino'; const logger = pino({ name: 'ProcessVideo' }); /** - * Orchestrates the core video processing pipeline: Probe -> Transcode -> Upload. - * - * @remarks - * - Idempotency: If a job fails midway, retrying it will safely overwrite existing partial state. - * - Cleanup: Guaranteed to remove local intermediate files on both success and failure pathways. + * Orchestrates the core video processing pipeline: Download -> Probe -> Transcode -> Upload. */ export class ProcessVideo implements ProcessVideoUseCase { constructor( @@ -25,13 +25,6 @@ export class ProcessVideo implements ProcessVideoUseCase { private readonly db: VideoRepository, ) {} - /** - * Executes the transcoding pipeline and synchronizes state with the database and webhook. - * - * @param job - Job payload from BullMQ. `videoId` acts as the idempotency key in DB/Storage. - * @throws {WorkerError} If any step fails. Process catches this, cleans up, and rethrows - * so the BullMQ wrapper can handle the retry/failure logic based on `.retryable`. - */ async execute(job: JobData, onProgress?: ProgressCallback): Promise { const { videoId, sourceUrl, webhookUrl } = job; logger.info({ videoId, sourceUrl, webhookUrl }, 'Starting video processing pipeline'); @@ -39,15 +32,41 @@ export class ProcessVideo implements ProcessVideoUseCase { await this.db.updateStatus(videoId, 'processing'); try { - logger.info({ videoId }, 'Step 1/3: Probing source'); - const probeResult = await this.ffmpeg.probe(sourceUrl); + const parsedUrl = new URL(sourceUrl); + const extension = path.extname(parsedUrl.pathname); + + logger.info({ videoId, extension }, 'Step 0/3: Downloading source video locally'); + + const workDir = `/tmp/worker/${videoId}`; + await fs.promises.mkdir(workDir, { recursive: true }); + + const localSourcePath = path.join(workDir, `source${extension}`); + + const response = await fetch(sourceUrl); + if (!response.ok) { + throw new Error( + `Failed to download source video: ${response.status} ${response.statusText}`, + ); + } + if (!response.body) { + throw new Error('Response body from source video is empty'); + } + await pipeline( + Readable.fromWeb(response.body as any), + fs.createWriteStream(localSourcePath), + ); + logger.info({ videoId }, 'Source video successfully downloaded to worker disk'); + + logger.info({ videoId }, 'Step 1/3: Probing source'); + const probeResult = await this.ffmpeg.probe(localSourcePath); await this.db.updateStatus(videoId, 'processing'); logger.info( { videoId, duration: probeResult.duration, resolution: `${probeResult.width}x${probeResult.height}`, + format: extension, }, 'Probe complete', ); @@ -58,7 +77,7 @@ export class ProcessVideo implements ProcessVideoUseCase { await this.db.updateStatus(videoId, 'transcoding'); const { outputDir, renditions } = await this.ffmpeg.transcodeHLS( - sourceUrl, + localSourcePath, videoId, probeResult.width, probeResult.height, diff --git a/src/config/env.ts b/src/config/env.ts index ac27d9d..00f03a8 100644 --- a/src/config/env.ts +++ b/src/config/env.ts @@ -19,10 +19,13 @@ const envSchema = z.object({ WORKER_CONCURRENCY: z.coerce.number().default(1), TEST_DURATION_SECONDS: z.coerce.number().optional(), + TEST_VIDEO_PROFILE: unquotedString + .pipe(z.enum(['avc_sdr', 'hvc_sdr', 'hvc_pq', 'dvh_pq', 'ALL'])) + .default('ALL'), HLS_OUTPUT_MODE: unquotedString.pipe(z.enum(['SINGLE_FILE', 'SEGMENTED'])).default('SEGMENTED'), - JOB_LOCK_DURATION_MS: z.coerce.number().default(120000), - JOB_LOCK_RENEW_MS: z.coerce.number().default(30000), + JOB_LOCK_DURATION_MS: z.coerce.number().default(1800000), + JOB_LOCK_RENEW_MS: z.coerce.number().default(15000), AZURE_UPLOAD_BATCH_SIZE: z.coerce.number().default(20), AZURE_UPLOAD_RETRIES: z.coerce.number().default(3), @@ -36,6 +39,11 @@ const envSchema = z.object({ ), CORS_ORIGIN: unquotedString.default('*'), DATABASE_URL: unquotedString.pipe(z.string().min(1, 'DATABASE_URL is required')), + DOMAIN_SUBDOMAIN_NAME: unquotedString.optional(), + + FFMPEG_THREADS: z.coerce.number().default(0), + X265_POOL_SIZE: z.coerce.number().default(32), + X265_FRAME_THREADS: z.coerce.number().default(4), }); /** diff --git a/src/domain/job.interface.ts b/src/domain/job.interface.ts index ca4fae0..38733be 100644 --- a/src/domain/job.interface.ts +++ b/src/domain/job.interface.ts @@ -7,6 +7,7 @@ export interface JobData { export interface AudioStreamInfo { index: number; + codec: string; language: string; channels: number; title: string; diff --git a/src/infrastructure/ffmpeg/adapter.ts b/src/infrastructure/ffmpeg/adapter.ts index 1501fef..667015e 100644 --- a/src/infrastructure/ffmpeg/adapter.ts +++ b/src/infrastructure/ffmpeg/adapter.ts @@ -15,7 +15,6 @@ import { computeAudioMetadata, } from './encoding/profiles.js'; import { probe } from './core/probe.js'; -import { probeComplexity } from './core/complexity.js'; import { processMasterPipeline } from './hls/pipeline.js'; import { writeMasterPlaylist } from './hls/playlist.js'; import { HLS_CONSTANTS } from './constants.js'; @@ -35,14 +34,6 @@ const ISO_639_1_MAP: Record = { und: 'und', }; -/** - * The "Brain" of the transcoding engine. - * - * @remarks - * - Orchestrates the entire lifecycle: Probing -> Complexity Analysis -> Transcoding -> Manifest Mapping. - * - Implements a Dispersed Hash Tree schema (via `blobPathFromUuid`) to prevent directory iteration attacks in public storage. - * - Employs a "Smart Per-Title" intelligence: Probes the file's visual complexity before assigning final bitrates and renditions. - */ export class FFmpegAdapter implements TranscodeProvider { constructor(private readonly workDir: string = DEFAULT_WORK_DIR) {} async probe(sourceUrl: string): Promise { @@ -63,15 +54,7 @@ export class FFmpegAdapter implements TranscodeProvider { const outputDir = path.join(this.workDir, videoId, 'hls'); const activeProfiles = filterActiveVideoProfiles(sourceWidth, sourceHeight, videoRange); - logger.info({ videoId }, 'Analyzing video complexity for Smart Per-Title Bitrate adaptation'); - - const { multiplier: complexityMultiplier } = await probeComplexity( - sourceUrl, - sourceDuration, - videoId, - sourceWidth, - sourceHeight, - ); + const complexityMultiplier = 1.0; const rawVideoVariants = computeVideoMetadata( activeProfiles, diff --git a/src/infrastructure/ffmpeg/constants.ts b/src/infrastructure/ffmpeg/constants.ts index 241e4d8..6ebf917 100644 --- a/src/infrastructure/ffmpeg/constants.ts +++ b/src/infrastructure/ffmpeg/constants.ts @@ -1,15 +1,5 @@ export const HLS_CONSTANTS = { MASTER_PLAYLIST_NAME: 'playlist.m3u8', - VIDEO_SEGMENT_NAME: 'data_%03d.m4s', - SINGLE_VIDEO_NAME: 'data.m4s', - - INIT_SEGMENT_NAME: 'init.mp4', - VARIANT_PLAYLIST_NAME: 'manifest.m3u8', - - AUDIO_TIERS: { - SURROUND: 'a1', - STEREO: 'a2', - }, } as const; diff --git a/src/infrastructure/ffmpeg/core/complexity.ts b/src/infrastructure/ffmpeg/core/complexity.ts deleted file mode 100644 index f97f723..0000000 --- a/src/infrastructure/ffmpeg/core/complexity.ts +++ /dev/null @@ -1,192 +0,0 @@ -import { pino } from 'pino'; -import { spawn } from 'node:child_process'; -import fs from 'node:fs/promises'; -import path from 'node:path'; -import { TranscodeError } from '../../../domain/errors.js'; - -const logger = pino({ name: 'PerTitleVMAF' }); - -export interface ComplexityResult { - multiplier: number; - sampleBitrate: number; -} - -async function runCommand(args: string[]): Promise { - return new Promise((resolve, reject) => { - const proc = spawn('ffmpeg', args); - let output = ''; - - proc.stderr.on('data', (data) => { - output += data.toString(); - }); - - proc.on('close', (code) => { - if (code !== 0) reject(new Error(`FFmpeg failed with code ${code}:\n${output}`)); - else resolve(output); - }); - }); -} - -/** - * Orchestrates a mathematically-driven Smart Per-Title Bitrate adaptation using Netflix's VMAF model. - * - * @remarks - * - Phase 1: Generates a near-lossless reference encode (CRF 10, ultrafast) to establish a pristine baseline. - * - Phase 2: Encodes empirical test points (CRF 19, 23, 27) and scores them against the reference using the VMAF neural network. - * - Phase 3: Interpolates a Rate-Distortion curve to find the exact bits-per-second required to hit a perceptive VMAF score of 95.0. - * - Bounding: Applies strict OTT resolution safety floors to prevent bitrate explosion on noisy/grainy source files. - */ -export async function probeComplexity( - sourceUrl: string, - duration: number, - videoId: string, - sourceWidth: number, - sourceHeight: number, -): Promise { - const workDir = `/tmp/worker/${videoId}/vmaf`; - await fs.mkdir(workDir, { recursive: true }); - - const refPath = path.join(workDir, 'ref.mkv'); - - logger.info({ videoId }, 'Phase 1: Generating 100% Whole-Timeline Reference (CFR Enforced)'); - - try { - await runCommand([ - '-i', - sourceUrl, - '-map', - '0:v:0', - '-an', - '-sn', - '-c:v', - 'libx264', - '-crf', - '10', - '-preset', - 'ultrafast', - '-fps_mode', - 'cfr', - '-pix_fmt', - 'yuv420p', - '-y', - refPath, - ]); - - logger.info({ videoId }, 'Phase 2: Encoding Full Grid & Running VMAF AI Analysis'); - - const crfs = [19, 23, 27]; - const results: { crf: number; vmaf: number; kbps: number }[] = []; - - for (const crf of crfs) { - const testPath = path.join(workDir, `test_${crf}.mkv`); - - await runCommand([ - '-i', - refPath, - '-c:v', - 'libx264', - '-crf', - String(crf), - '-preset', - 'superfast', - '-fps_mode', - 'cfr', - '-y', - testPath, - ]); - - const stat = await fs.stat(testPath); - const kbps = (stat.size * 8) / duration / 1000; - - const vmafOut = await runCommand([ - '-i', - testPath, - '-i', - refPath, - '-lavfi', - '[0:v]setpts=PTS-STARTPTS,scale=1920:1080:flags=bicubic[dist];[1:v]setpts=PTS-STARTPTS,scale=1920:1080:flags=bicubic[ref];[dist][ref]libvmaf=model=path=/usr/local/share/model/vmaf_v0.6.1.json:n_subsample=24:n_threads=4:pool=harmonic_mean', - '-f', - 'null', - '-', - ]); - - const match = vmafOut.match(/VMAF score: ([\d.]+)/); - const vmaf = match ? parseFloat(match[1]) : 0; - - results.push({ crf, vmaf, kbps }); - } - - logger.info( - { videoId, curve: results }, - 'Phase 3: Full-Timeline Rate-Distortion Curve mathematically mapped', - ); - - const TARGET_VMAF = 95.0; - results.sort((a, b) => a.vmaf - b.vmaf); - - let optimalKbps = 0; - - if (TARGET_VMAF <= results[0].vmaf) { - optimalKbps = results[0].kbps; - } else if (TARGET_VMAF >= results[results.length - 1].vmaf) { - optimalKbps = results[results.length - 1].kbps; - } else { - for (let i = 0; i < results.length - 1; i++) { - if (TARGET_VMAF >= results[i].vmaf && TARGET_VMAF <= results[i + 1].vmaf) { - const p1 = results[i]; - const p2 = results[i + 1]; - const slope = (p2.kbps - p1.kbps) / (p2.vmaf - p1.vmaf); - optimalKbps = p1.kbps + slope * (TARGET_VMAF - p1.vmaf); - break; - } - } - } - - const is4K = sourceWidth >= 3840 || sourceHeight >= 2160; - const is1080p = (sourceWidth >= 1920 || sourceHeight >= 1080) && !is4K; - - const BASELINE_KBPS = is4K ? 20000 : is1080p ? 6000 : 3000; - - if (!optimalKbps || isNaN(optimalKbps) || optimalKbps <= 0) { - logger.warn( - { videoId }, - 'VMAF calculation yielded invalid optimalKbps. Falling back to BASELINE.', - ); - optimalKbps = BASELINE_KBPS; - } - - let multiplier = optimalKbps / BASELINE_KBPS; - - const minFloor = is4K ? 0.85 : 0.6; - const maxCeil = is4K ? 1.5 : 2.0; - - multiplier = Math.max(minFloor, Math.min(multiplier, maxCeil)); - - logger.info( - { - videoId, - optimalKbps: Math.round(optimalKbps), - baselineKbps: BASELINE_KBPS, - finalMultiplier: multiplier, - is4K, - }, - 'Real Per-Title VMAF calculated with OTT Resolution Quality Floors applied', - ); - - await fs.rm(workDir, { recursive: true, force: true }).catch(() => {}); - - return { multiplier, sampleBitrate: Math.round(optimalKbps) }; - } catch (error) { - logger.error( - { videoId, err: error }, - 'Real VMAF probe failed. Strict mode active: Failing the pipeline.', - ); - - await fs.rm(workDir, { recursive: true, force: true }).catch(() => {}); - - throw new TranscodeError( - `VMAF Complexity Probe failed: ${error instanceof Error ? error.message : String(error)}`, - error instanceof Error ? error : undefined, - ); - } -} diff --git a/src/infrastructure/ffmpeg/core/probe.ts b/src/infrastructure/ffmpeg/core/probe.ts index fe4def3..542d3b8 100644 --- a/src/infrastructure/ffmpeg/core/probe.ts +++ b/src/infrastructure/ffmpeg/core/probe.ts @@ -39,6 +39,7 @@ export async function probe(sourceUrl: string): Promise { return { index: arrayIndex, + codec: s.codec_name || 'aac', language: lang.toLowerCase().slice(0, 3), channels: s.channels ?? 2, title: title, @@ -46,7 +47,13 @@ export async function probe(sourceUrl: string): Promise { }); if (audioStreams.length === 0) { - audioStreams.push({ index: -1, language: 'und', channels: 2, title: 'Track 1' }); + audioStreams.push({ + index: -1, + codec: 'aac', + language: 'und', + channels: 2, + title: 'Track 1', + }); } let videoRange = 'SDR'; diff --git a/src/infrastructure/ffmpeg/encoding/ABR/audio/audio.json b/src/infrastructure/ffmpeg/encoding/ABR/audio/audio.json new file mode 100644 index 0000000..9bcb7db --- /dev/null +++ b/src/infrastructure/ffmpeg/encoding/ABR/audio/audio.json @@ -0,0 +1,47 @@ +[ + { + "name": "audio-stereo-64", + "channels": 2, + "codec": "libfdk_aac", + "profile": "aac_low", + "bitrate": 64000, + "sampleRate": 48000, + "hardwareProfile": false + }, + { + "name": "audio-stereo-128", + "channels": 2, + "codec": "libfdk_aac", + "profile": "aac_low", + "bitrate": 128000, + "sampleRate": 48000, + "hardwareProfile": false + }, + { + "name": "audio-stereo-160", + "channels": 2, + "codec": "libfdk_aac", + "profile": "aac_low", + "bitrate": 160000, + "sampleRate": 48000, + "hardwareProfile": false + }, + { + "name": "audio-ac3", + "channels": 6, + "codec": "ac3", + "bitrate": 384000, + "sampleRate": 48000, + "hardwareProfile": true + }, + { + "name": "audio-atmos", + "channels": 16, + "codec": "eac3", + "bitrate": 768000, + "sampleRate": 48000, + "hardwareProfile": true, + "isCinemaMaster": true, + "isAtmos": true + } +] diff --git a/src/infrastructure/ffmpeg/encoding/ABR/video/avc.sdr.json b/src/infrastructure/ffmpeg/encoding/ABR/video/avc.sdr.json new file mode 100644 index 0000000..04ac8d0 --- /dev/null +++ b/src/infrastructure/ffmpeg/encoding/ABR/video/avc.sdr.json @@ -0,0 +1,206 @@ +[ + { + "name": "video_qtc301", + "width": 480, + "height": 270, + "bitrate": 250000, + "maxrate": 275000, + "bufsize": 500000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "videoRange": "SDR", + "preset": "slow", + "profile": "high", + "level": "3.1", + "crf": 23, + "videoCodecTag": "avc1.64001f" + }, + { + "name": "video_qtc305", + "width": 544, + "height": 306, + "bitrate": 350000, + "maxrate": 385000, + "bufsize": 700000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "videoRange": "SDR", + "preset": "slow", + "profile": "high", + "level": "3.1", + "crf": 23, + "videoCodecTag": "avc1.64001f" + }, + { + "name": "video_qtc310", + "width": 608, + "height": 342, + "bitrate": 500000, + "maxrate": 550000, + "bufsize": 1000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "videoRange": "SDR", + "preset": "slow", + "profile": "high", + "level": "3.1", + "crf": 23, + "videoCodecTag": "avc1.64001f" + }, + { + "name": "video_qtc311", + "width": 672, + "height": 378, + "bitrate": 750000, + "maxrate": 825000, + "bufsize": 1500000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "videoRange": "SDR", + "preset": "slow", + "profile": "high", + "level": "3.1", + "crf": 23, + "videoCodecTag": "avc1.64001f" + }, + { + "name": "video_qtc315", + "width": 768, + "height": 432, + "bitrate": 1100000, + "maxrate": 1210000, + "bufsize": 2200000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "videoRange": "SDR", + "preset": "slow", + "profile": "high", + "level": "3.1", + "crf": 21, + "videoCodecTag": "avc1.64001f" + }, + { + "name": "video_qtc320", + "width": 864, + "height": 486, + "bitrate": 1500000, + "maxrate": 1650000, + "bufsize": 3000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "videoRange": "SDR", + "preset": "slow", + "profile": "high", + "level": "3.1", + "crf": 20, + "videoCodecTag": "avc1.64001f" + }, + { + "name": "video_qtc322", + "width": 1024, + "height": 576, + "bitrate": 2100000, + "maxrate": 2310000, + "bufsize": 4200000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "videoRange": "SDR", + "preset": "slow", + "profile": "high", + "level": "3.1", + "crf": 20, + "videoCodecTag": "avc1.64001f" + }, + { + "name": "video_qtc325", + "width": 1280, + "height": 720, + "bitrate": 2800000, + "maxrate": 3080000, + "bufsize": 5600000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "videoRange": "SDR", + "preset": "slow", + "profile": "high", + "level": "3.2", + "crf": 19, + "videoCodecTag": "avc1.640020" + }, + { + "name": "video_qtc330", + "width": 1280, + "height": 720, + "bitrate": 4500000, + "maxrate": 4950000, + "bufsize": 9000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "videoRange": "SDR", + "preset": "slow", + "profile": "high", + "level": "3.2", + "crf": 18, + "videoCodecTag": "avc1.640020" + }, + { + "name": "video_qtc335", + "width": 1920, + "height": 1080, + "bitrate": 6500000, + "maxrate": 7150000, + "bufsize": 13000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "videoRange": "SDR", + "preset": "slow", + "profile": "high", + "level": "4.0", + "crf": 20, + "videoCodecTag": "avc1.640028" + }, + { + "name": "video_qtc340", + "width": 1920, + "height": 1080, + "bitrate": 9700000, + "maxrate": 10670000, + "bufsize": 19400000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "videoRange": "SDR", + "preset": "slow", + "profile": "high", + "level": "4.0", + "crf": 18, + "videoCodecTag": "avc1.640028" + }, + { + "name": "video_qtc345", + "width": 1920, + "height": 1080, + "bitrate": 13500000, + "maxrate": 14850000, + "bufsize": 27000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "videoRange": "SDR", + "preset": "slow", + "profile": "high", + "level": "4.0", + "crf": 15, + "videoCodecTag": "avc1.640028" + } +] diff --git a/src/infrastructure/ffmpeg/encoding/ABR/video/dvh.pq.json b/src/infrastructure/ffmpeg/encoding/ABR/video/dvh.pq.json new file mode 100644 index 0000000..3d50b9a --- /dev/null +++ b/src/infrastructure/ffmpeg/encoding/ABR/video/dvh.pq.json @@ -0,0 +1,223 @@ +[ + { + "name": "video_qtc901", + "width": 480, + "height": 270, + "bitrate": 300000, + "maxrate": 330000, + "bufsize": 600000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 23, + "videoCodecTag": "dvh1.05.01" + }, + { + "name": "video_qtc910", + "width": 608, + "height": 342, + "bitrate": 480000, + "maxrate": 528000, + "bufsize": 960000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 23, + "videoCodecTag": "dvh1.05.01" + }, + { + "name": "video_qtc911", + "width": 672, + "height": 378, + "bitrate": 690000, + "maxrate": 759000, + "bufsize": 1300000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 23, + "videoCodecTag": "dvh1.05.01" + }, + { + "name": "video_qtc915", + "width": 768, + "height": 432, + "bitrate": 1000000, + "maxrate": 1100000, + "bufsize": 2000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 21, + "videoCodecTag": "dvh1.05.01" + }, + { + "name": "video_qtc920", + "width": 864, + "height": 486, + "bitrate": 1300000, + "maxrate": 1400000, + "bufsize": 2600000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 20, + "videoCodecTag": "dvh1.05.01" + }, + { + "name": "video_qtc922", + "width": 1024, + "height": 576, + "bitrate": 1800000, + "maxrate": 1900000, + "bufsize": 3600000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 20, + "videoCodecTag": "dvh1.05.01" + }, + { + "name": "video_qtc925", + "width": 1280, + "height": 720, + "bitrate": 2500000, + "maxrate": 2700000, + "bufsize": 5000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 19, + "videoCodecTag": "dvh1.05.01" + }, + { + "name": "video_qtc930", + "width": 1280, + "height": 720, + "bitrate": 3200000, + "maxrate": 3500000, + "bufsize": 6400000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 18, + "videoCodecTag": "dvh1.05.01" + }, + { + "name": "video_qtc935", + "width": 1920, + "height": 1080, + "bitrate": 3200000, + "maxrate": 3500000, + "bufsize": 6400000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 20, + "videoCodecTag": "dvh1.05.03" + }, + { + "name": "video_qtc945", + "width": 1920, + "height": 1080, + "bitrate": 8100000, + "maxrate": 8900000, + "bufsize": 16200000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 15, + "videoCodecTag": "dvh1.05.03" + }, + { + "name": "video_qtc950", + "width": 2560, + "height": 1440, + "bitrate": 14000000, + "maxrate": 15400000, + "bufsize": 28000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "5.1", + "crf": 18, + "videoCodecTag": "dvh1.05.06" + }, + { + "name": "video_qtc955", + "width": 3840, + "height": 2160, + "bitrate": 14000000, + "maxrate": 15400000, + "bufsize": 28000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "5.1", + "crf": 16, + "videoCodecTag": "dvh1.05.06" + }, + { + "name": "video_qtc960", + "width": 3840, + "height": 2160, + "bitrate": 24000000, + "maxrate": 26400000, + "bufsize": 48000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "5.1", + "crf": 13, + "videoCodecTag": "dvh1.05.06" + } +] diff --git a/src/infrastructure/ffmpeg/encoding/ABR/video/hvc.pq.json b/src/infrastructure/ffmpeg/encoding/ABR/video/hvc.pq.json new file mode 100644 index 0000000..483ef71 --- /dev/null +++ b/src/infrastructure/ffmpeg/encoding/ABR/video/hvc.pq.json @@ -0,0 +1,223 @@ +[ + { + "name": "video_qtc701", + "width": 480, + "height": 270, + "bitrate": 300000, + "maxrate": 330000, + "bufsize": 600000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 24, + "videoCodecTag": "hvc1.2.20000000.L123.B0" + }, + { + "name": "video_qtc710", + "width": 608, + "height": 342, + "bitrate": 480000, + "maxrate": 528000, + "bufsize": 960000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 24, + "videoCodecTag": "hvc1.2.20000000.L123.B0" + }, + { + "name": "video_qtc711", + "width": 672, + "height": 378, + "bitrate": 690000, + "maxrate": 759000, + "bufsize": 1300000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 24, + "videoCodecTag": "hvc1.2.20000000.L123.B0" + }, + { + "name": "video_qtc715", + "width": 768, + "height": 432, + "bitrate": 1000000, + "maxrate": 1100000, + "bufsize": 2000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 22, + "videoCodecTag": "hvc1.2.20000000.L123.B0" + }, + { + "name": "video_qtc720", + "width": 864, + "height": 486, + "bitrate": 1300000, + "maxrate": 1400000, + "bufsize": 2600000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 21, + "videoCodecTag": "hvc1.2.20000000.L123.B0" + }, + { + "name": "video_qtc722", + "width": 1024, + "height": 576, + "bitrate": 1800000, + "maxrate": 1900000, + "bufsize": 3600000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 21, + "videoCodecTag": "hvc1.2.20000000.L123.B0" + }, + { + "name": "video_qtc725", + "width": 1280, + "height": 720, + "bitrate": 2500000, + "maxrate": 2700000, + "bufsize": 5000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 20, + "videoCodecTag": "hvc1.2.20000000.L123.B0" + }, + { + "name": "video_qtc730", + "width": 1280, + "height": 720, + "bitrate": 3200000, + "maxrate": 3500000, + "bufsize": 6400000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 19, + "videoCodecTag": "hvc1.2.20000000.L123.B0" + }, + { + "name": "video_qtc735", + "width": 1920, + "height": 1080, + "bitrate": 3200000, + "maxrate": 3500000, + "bufsize": 6400000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 21, + "videoCodecTag": "hvc1.2.20000000.L123.B0" + }, + { + "name": "video_qtc745", + "width": 1920, + "height": 1080, + "bitrate": 8100000, + "maxrate": 8900000, + "bufsize": 16200000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 16, + "videoCodecTag": "hvc1.2.20000000.L123.B0" + }, + { + "name": "video_qtc750", + "width": 2560, + "height": 1440, + "bitrate": 14000000, + "maxrate": 15400000, + "bufsize": 28000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "5.1", + "crf": 19, + "videoCodecTag": "hvc1.2.20000000.H150.B0" + }, + { + "name": "video_qtc755", + "width": 3840, + "height": 2160, + "bitrate": 14000000, + "maxrate": 15400000, + "bufsize": 28000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "5.1", + "crf": 17, + "videoCodecTag": "hvc1.2.20000000.H150.B0" + }, + { + "name": "video_qtc760", + "width": 3840, + "height": 2160, + "bitrate": 24000000, + "maxrate": 26400000, + "bufsize": 48000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "PQ", + "preset": "slow", + "profile": "main10", + "level": "5.1", + "crf": 14, + "videoCodecTag": "hvc1.2.20000000.H150.B0" + } +] diff --git a/src/infrastructure/ffmpeg/encoding/ABR/video/hvc.sdr.json b/src/infrastructure/ffmpeg/encoding/ABR/video/hvc.sdr.json new file mode 100644 index 0000000..4a9d576 --- /dev/null +++ b/src/infrastructure/ffmpeg/encoding/ABR/video/hvc.sdr.json @@ -0,0 +1,223 @@ +[ + { + "name": "video_qtc501", + "width": 480, + "height": 270, + "bitrate": 250000, + "maxrate": 275000, + "bufsize": 500000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "SDR", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 25, + "videoCodecTag": "hvc1.2.20000000.L123.B0" + }, + { + "name": "video_qtc510", + "width": 608, + "height": 342, + "bitrate": 400000, + "maxrate": 440000, + "bufsize": 800000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "SDR", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 25, + "videoCodecTag": "hvc1.2.20000000.L123.B0" + }, + { + "name": "video_qtc511", + "width": 672, + "height": 378, + "bitrate": 575000, + "maxrate": 632000, + "bufsize": 1100000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "SDR", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 25, + "videoCodecTag": "hvc1.2.20000000.L123.B0" + }, + { + "name": "video_qtc515", + "width": 768, + "height": 432, + "bitrate": 825000, + "maxrate": 907000, + "bufsize": 1600000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "SDR", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 23, + "videoCodecTag": "hvc1.2.20000000.L123.B0" + }, + { + "name": "video_qtc520", + "width": 864, + "height": 486, + "bitrate": 1100000, + "maxrate": 1200000, + "bufsize": 2200000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "SDR", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 22, + "videoCodecTag": "hvc1.2.20000000.L123.B0" + }, + { + "name": "video_qtc522", + "width": 1024, + "height": 576, + "bitrate": 1500000, + "maxrate": 1600000, + "bufsize": 3000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "SDR", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 22, + "videoCodecTag": "hvc1.2.20000000.L123.B0" + }, + { + "name": "video_qtc525", + "width": 1280, + "height": 720, + "bitrate": 2100000, + "maxrate": 2300000, + "bufsize": 4200000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "SDR", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 21, + "videoCodecTag": "hvc1.2.20000000.L123.B0" + }, + { + "name": "video_qtc530", + "width": 1280, + "height": 720, + "bitrate": 2700000, + "maxrate": 2900000, + "bufsize": 5400000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "SDR", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 20, + "videoCodecTag": "hvc1.2.20000000.L123.B0" + }, + { + "name": "video_qtc535", + "width": 1920, + "height": 1080, + "bitrate": 2700000, + "maxrate": 2900000, + "bufsize": 5400000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "SDR", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 22, + "videoCodecTag": "hvc1.2.20000000.L123.B0" + }, + { + "name": "video_qtc545", + "width": 1920, + "height": 1080, + "bitrate": 6800000, + "maxrate": 7400000, + "bufsize": 13600000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "SDR", + "preset": "slow", + "profile": "main10", + "level": "4.1", + "crf": 17, + "videoCodecTag": "hvc1.2.20000000.L123.B0" + }, + { + "name": "video_qtc550", + "width": 2560, + "height": 1440, + "bitrate": 11600000, + "maxrate": 12700000, + "bufsize": 23200000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "SDR", + "preset": "slow", + "profile": "main10", + "level": "5.1", + "crf": 20, + "videoCodecTag": "hvc1.2.20000000.H150.B0" + }, + { + "name": "video_qtc555", + "width": 3840, + "height": 2160, + "bitrate": 11600000, + "maxrate": 12700000, + "bufsize": 23200000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "SDR", + "preset": "slow", + "profile": "main10", + "level": "5.1", + "crf": 18, + "videoCodecTag": "hvc1.2.20000000.H150.B0" + }, + { + "name": "video_qtc560", + "width": 3840, + "height": 2160, + "bitrate": 20000000, + "maxrate": 22000000, + "bufsize": 40000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "videoRange": "SDR", + "preset": "slow", + "profile": "main10", + "level": "5.1", + "crf": 15, + "videoCodecTag": "hvc1.2.20000000.H150.B0" + } +] diff --git a/src/infrastructure/ffmpeg/encoding/flags.ts b/src/infrastructure/ffmpeg/encoding/flags.ts index 5495a9a..eab7150 100644 --- a/src/infrastructure/ffmpeg/encoding/flags.ts +++ b/src/infrastructure/ffmpeg/encoding/flags.ts @@ -5,7 +5,7 @@ import { HLS_CONSTANTS } from '../constants.js'; export interface FrameRateInfo { ffmpegFraction: string; - appleFormat: string; + aFormat: string; gopSize: number; } @@ -34,15 +34,37 @@ export function getBroadcastFrameRate(sourceFps?: number): FrameRateInfo | null return { ffmpegFraction: fraction, - appleFormat: exactFps.toFixed(3), + aFormat: exactFps.toFixed(3), gopSize: Math.round(exactFps * 2), }; } -export function hlsOutputFlags(hlsTime: number, outputDir: string): string[] { - return [ +export function hlsOutputFlags( + hlsTime: number, + outputDir: string, + videoId: string, + variant?: VideoVariantMeta, + audio?: AudioVariantMeta, + baseUrl?: string, +): string[] { + let segmentPattern = 'data_%03d.m4s'; + let initPattern = '000.mp4'; + + if (variant) { + let codec = variant.videoCodecTag.substring(0, 4); + if (codec.startsWith('dv')) codec = 'dovi'; + const base = `${videoId}_${variant.name}_${codec}_${variant.actualWidth}x${variant.actualHeight}`; + segmentPattern = `${base}_--%d.m4s`; + initPattern = `${base}.mp4`; + } else if (audio) { + const base = `${videoId}_audio_${audio.language}_${audio.name}`; + segmentPattern = `${base}--%d.m4s`; + initPattern = `${base}.mp4`; + } + + const flags = [ '-hls_fmp4_init_filename', - HLS_CONSTANTS.INIT_SEGMENT_NAME, + initPattern, '-movflags', '+frag_keyframe+empty_moov+default_base_moof+cmaf+omit_tfhd_offset', '-f', @@ -60,9 +82,7 @@ export function hlsOutputFlags(hlsTime: number, outputDir: string): string[] { ? '+independent_segments+single_file+round_durations' : '+independent_segments+round_durations', '-hls_segment_filename', - config.HLS_OUTPUT_MODE === 'SINGLE_FILE' - ? path.join(outputDir, HLS_CONSTANTS.SINGLE_VIDEO_NAME) - : path.join(outputDir, HLS_CONSTANTS.VIDEO_SEGMENT_NAME), + path.join(outputDir, segmentPattern), '-avoid_negative_ts', 'make_zero', '-fflags', @@ -72,6 +92,12 @@ export function hlsOutputFlags(hlsTime: number, outputDir: string): string[] { '-video_track_timescale', '90000', ]; + + if (baseUrl) { + flags.push('-hls_base_url', baseUrl); + } + + return flags; } export function videoEncoderFlags(variant: VideoVariantMeta, sourceFrameRate?: number): string[] { @@ -80,12 +106,17 @@ export function videoEncoderFlags(variant: VideoVariantMeta, sourceFrameRate?: n const codec = variant.videoCodec || 'libx264'; const isHevc = codec === 'libx265'; - const isHdr = variant.profile === 'main10'; + const isHdr = + (variant.videoRange === 'PQ' || variant.videoRange === 'HLG') && variant.profile === 'main10'; const colorPrimaries = isHdr ? 'bt2020' : 'bt709'; - const colorTransfer = isHdr ? 'smpte2084' : 'bt709'; + const colorTransfer = isHdr + ? variant.videoRange === 'HLG' + ? 'arib-std-b67' + : 'smpte2084' + : 'bt709'; const colorMatrix = isHdr ? 'bt2020nc' : 'bt709'; - const pixFmt = isHdr ? 'yuv420p10le' : 'yuv420p'; + const pixFmt = variant.profile === 'main10' ? 'yuv420p10le' : 'yuv420p'; const baseFlags: string[] = [ '-c:v', @@ -94,6 +125,8 @@ export function videoEncoderFlags(variant: VideoVariantMeta, sourceFrameRate?: n isHevc ? 'hvc1' : variant.videoCodecTag.substring(0, 4), '-preset', variant.preset, + '-tune', + 'film', ...(fpsInfo ? ['-r', fpsInfo.ffmpegFraction, '-fps_mode', 'cfr'] : []), ...(variant.profile ? ['-profile:v', variant.profile] : []), ...(variant.level ? ['-level', variant.level] : []), @@ -108,7 +141,7 @@ export function videoEncoderFlags(variant: VideoVariantMeta, sourceFrameRate?: n '-color_range', 'tv', '-crf', - '23', + String(variant.crf || 23), '-maxrate', String(variant.maxrate), '-bufsize', @@ -121,17 +154,39 @@ export function videoEncoderFlags(variant: VideoVariantMeta, sourceFrameRate?: n String(gopSize), '-sc_threshold', '0', + '-threads', + String(config.FFMPEG_THREADS), ]; if (isHevc) { + const isDvh = variant.videoCodecTag.startsWith('dvh1'); + const dvProfile = variant.videoCodecTag.includes('.05.') ? '5' : '8.1'; + const dvhParam = isDvh + ? `:dolby-vision-profile=${dvProfile}:dolby-vision-rpu=filename:hdr10-opt=1` + : ''; + + const defaultMasterDisplay = + 'G(13250,34500)B(7500,3000)R(34000,16000)WP(15635,16450)L(10000000,1)'; + const defaultMaxCll = '1000,400'; + const isPQ = colorTransfer === 'smpte2084'; + const hdr10Params = isPQ + ? `:hdr10-opt=1:repeat-headers=1:master-display=${defaultMasterDisplay}:max-cll=${defaultMaxCll}` + : ':repeat-headers=1'; + const extraHdrParams = isHdr ? hdr10Params : ''; + baseFlags.push( '-x265-params', - `no-open-gop=1:keyint=${gopSize}:min-keyint=${gopSize}:info=0:colorprim=${colorPrimaries}:transfer=${colorTransfer}:colormatrix=${colorMatrix}`, + `pools=${config.X265_POOL_SIZE}:frame-threads=${config.X265_FRAME_THREADS}:wpp=1:pmode=1:pme=1:no-open-gop=1:scenecut=0:keyint=${gopSize}:min-keyint=${gopSize}:info=0:colorprim=${colorPrimaries}:transfer=${colorTransfer}:colormatrix=${colorMatrix}${extraHdrParams}${dvhParam}`, '-flags', '+global_header', ); } else { - baseFlags.push('-flags', '+cgop+global_header'); + baseFlags.push( + '-x264-params', + `threads=${config.FFMPEG_THREADS === 0 ? 'auto' : config.FFMPEG_THREADS}`, + '-flags', + '+cgop+global_header', + ); } return baseFlags; @@ -139,13 +194,16 @@ export function videoEncoderFlags(variant: VideoVariantMeta, sourceFrameRate?: n export function videoFilterChain(width: number, height: number): string { return [ - `scale=${width}:${height}:force_original_aspect_ratio=disable`, + `scale=${width}:${height}:force_original_aspect_ratio=disable:flags=lanczos`, 'setsar=1/1', - 'unsharp=3:3:0.5:3:3:0.5', ].join(','); } export function audioEncoderFlags(audio: AudioVariantMeta): string[] { + if (audio.isAtmos) { + return ['-c:a', 'copy']; + } + const flags = [ '-c:a', audio.codec, @@ -157,8 +215,46 @@ export function audioEncoderFlags(audio: AudioVariantMeta): string[] { String(audio.sampleRate), ]; - const afFilter = 'aresample=async=1:first_pts=0'; + let afFilter = ''; + + if (audio.sourceChannels >= 6 && audio.channels === 2) { + afFilter += 'pan=stereo|FL=FL+0.707*FC+0.707*SL+0.2*LFE|FR=FR+0.707*FC+0.707*SR+0.2*LFE,'; + } + + const resampleParams = 'async=1:first_pts=0:resampler=soxr:precision=28:dither_method=shibata'; + afFilter += `aresample=${resampleParams},`; + + if (!audio.isCinemaMaster) { + afFilter += 'loudnorm=I=-24:LRA=15:TP=-2.0,'; + } + + let layout = 'stereo'; + if (audio.channels === 6) layout = '5.1'; + else if (audio.channels === 8) layout = '7.1'; + else if (audio.channels === 10) layout = '5.1.4'; + else if (audio.channels === 12) layout = '7.1.4'; + + const format = audio.channels === 2 ? 's16' : 'fltp'; + afFilter += `aformat=sample_rates=${audio.sampleRate}:channel_layouts=${layout}:sample_fmts=${format}`; + flags.push('-af', afFilter); + const bitsPerSample = audio.isCinemaMaster ? '24' : '16'; + flags.push('-bits_per_raw_sample', bitsPerSample); + + if (audio.profile) { + flags.push('-profile:a', audio.profile); + } + + if (audio.codec === 'libfdk_aac') { + if (!audio.profile || audio.profile === 'aac_low') { + flags.push('-afterburner', '1'); + } + } else if (audio.codec === 'aac') { + flags.push('-cutoff', '0'); + } else if (audio.codec === 'ac3' || audio.codec === 'eac3') { + flags.push('-dialnorm', '-24'); + } + return flags; } diff --git a/src/infrastructure/ffmpeg/encoding/profiles.ts b/src/infrastructure/ffmpeg/encoding/profiles.ts index bd0f132..cdbae90 100644 --- a/src/infrastructure/ffmpeg/encoding/profiles.ts +++ b/src/infrastructure/ffmpeg/encoding/profiles.ts @@ -1,31 +1,67 @@ import { createRequire } from 'node:module'; +import { config } from '../../../config/env.js'; import type { VideoProfile, AudioProfile, VideoVariantMeta, AudioVariantMeta } from '../types.js'; import type { AudioStreamInfo } from '../../../domain/job.interface.js'; const require = createRequire(import.meta.url); -const videoConfig = require('./profiles/video.json') as Record; -const audioConfig = require('./profiles/audio.json') as AudioProfile[]; + +const ABR_VIDEO = { + avc_sdr: (require('./ABR/video/avc.sdr.json') as VideoProfile[]).map((v, i) => ({ + ...v, + tierNumber: i + 1, + })), + hvc_sdr: (require('./ABR/video/hvc.sdr.json') as VideoProfile[]).map((v, i) => ({ + ...v, + tierNumber: i + 1, + })), + hvc_pq: (require('./ABR/video/hvc.pq.json') as VideoProfile[]).map((v, i) => ({ + ...v, + tierNumber: i + 1, + })), + dvh_pq: (require('./ABR/video/dvh.pq.json') as VideoProfile[]).map((v, i) => ({ + ...v, + tierNumber: i + 1, + })), +}; + +const cleanDomain = config.DOMAIN_SUBDOMAIN_NAME?.replace(/^https?:\/\//, '').replace(/\/$/, ''); + +const ABR_AUDIO = (require('./ABR/audio/audio.json') as AudioProfile[]).map((a) => ({ + ...a, + groupId: cleanDomain ? `${a.name}-${cleanDomain}` : a.name, + name: a.name, +})); const VIDEO_PROFILES: VideoProfile[] = [ - ...(videoConfig.h264_sdr || []), - ...(videoConfig.h265_sdr || []), - ...(videoConfig.h265_hdr || []), + ...ABR_VIDEO.avc_sdr, + ...ABR_VIDEO.hvc_sdr, + ...ABR_VIDEO.hvc_pq, + ...ABR_VIDEO.dvh_pq, ]; -const AUDIO_PROFILES: AudioProfile[] = audioConfig; +const AUDIO_PROFILES: AudioProfile[] = ABR_AUDIO; export function filterActiveVideoProfiles( sourceWidth: number, sourceHeight: number, videoRange: string = 'SDR', ): VideoProfile[] { - const isVertical = sourceHeight > sourceWidth; - - let compatibleProfiles = VIDEO_PROFILES; - if (videoRange === 'SDR') { - compatibleProfiles = VIDEO_PROFILES.filter((p) => !p.name.includes('hdr')); + let compatibleProfiles: VideoProfile[] = []; + + // Developer Override: Force a specific profile group for testing + if (config.TEST_VIDEO_PROFILE && config.TEST_VIDEO_PROFILE !== 'ALL') { + const forcedKey = config.TEST_VIDEO_PROFILE as keyof typeof ABR_VIDEO; + compatibleProfiles = [...ABR_VIDEO[forcedKey]]; + } else if (videoRange === 'SDR') { + compatibleProfiles = [...ABR_VIDEO.avc_sdr, ...ABR_VIDEO.hvc_sdr]; + } else if (videoRange === 'PQ' || videoRange === 'HLG') { + compatibleProfiles = [...ABR_VIDEO.hvc_pq, ...ABR_VIDEO.dvh_pq]; + } else { + compatibleProfiles = VIDEO_PROFILES; } + const isVertical = sourceHeight > sourceWidth; + const active = compatibleProfiles.filter((v) => { const standardWidth = Math.round((v.height * 16) / 9); if (isVertical) { @@ -36,7 +72,7 @@ export function filterActiveVideoProfiles( }); if (active.length === 0) { - active.push(compatibleProfiles[0] || VIDEO_PROFILES[0]); + active.push(compatibleProfiles[0] || ABR_VIDEO.avc_sdr[0]); } return active; @@ -49,28 +85,56 @@ export function computeVideoMetadata( complexityMultiplier: number = 1.0, ): Omit[] { const activeProfiles = profiles; - - const isVertical = sourceHeight > sourceWidth; + const sourceArea = sourceWidth * sourceHeight; + const sourceAspectRatio = sourceWidth / sourceHeight; return activeProfiles.map((profile) => { const standardWidth = Math.round((profile.height * 16) / 9); - let maxBoxWidth = standardWidth; - let maxBoxHeight = profile.height; + const targetArea = standardWidth * profile.height; - if (isVertical) { - maxBoxWidth = profile.height; - maxBoxHeight = standardWidth; + let scale = Math.sqrt(targetArea / sourceArea); + scale = Math.min(scale, 1.0); + + let maxWidthLimit = Infinity; + let maxHeightLimit = Infinity; + + if (standardWidth >= 1920) { + maxWidthLimit = standardWidth; + maxHeightLimit = profile.height; + } else if (standardWidth <= 864) { + maxWidthLimit = 864; + maxHeightLimit = 486; + } else { + maxWidthLimit = standardWidth * 1.25; + maxHeightLimit = profile.height * 1.25; } - const scaleWidth = maxBoxWidth / sourceWidth; - const scaleHeight = maxBoxHeight / sourceHeight; - const scale = Math.min(scaleWidth, scaleHeight, 1.0); + if (sourceHeight > sourceWidth) { + const temp = maxWidthLimit; + maxWidthLimit = maxHeightLimit; + maxHeightLimit = temp; + } - const outWidth = sourceWidth * scale; - const outHeight = sourceHeight * scale; + if (sourceWidth * scale > maxWidthLimit) { + scale = maxWidthLimit / sourceWidth; + } + if (sourceHeight * scale > maxHeightLimit) { + scale = maxHeightLimit / sourceHeight; + } - const actualWidth = Math.round(outWidth / 2) * 2; - const actualHeight = Math.round(outHeight / 2) * 2; + let exactWidth, exactHeight, actualWidth, actualHeight; + + if (sourceWidth >= sourceHeight) { + exactHeight = sourceHeight * scale; + actualHeight = Math.floor(exactHeight / 2) * 2; + exactWidth = actualHeight * sourceAspectRatio; + actualWidth = Math.round(exactWidth / 2) * 2; + } else { + exactWidth = sourceWidth * scale; + actualWidth = Math.floor(exactWidth / 2) * 2; + exactHeight = actualWidth / sourceAspectRatio; + actualHeight = Math.round(exactHeight / 2) * 2; + } const dynamicMaxrate = Math.round(profile.maxrate * complexityMultiplier); const dynamicBufsize = Math.round(profile.bufsize * complexityMultiplier); @@ -89,19 +153,31 @@ export function computeVideoMetadata( export function computeAudioMetadata( sourceAudioStreams: AudioStreamInfo[] = [], -): Omit[] { - const renditions: Omit[] = []; +): AudioVariantMeta[] { + const renditions: AudioVariantMeta[] = []; for (const stream of sourceAudioStreams) { for (const profile of AUDIO_PROFILES) { if (profile.hardwareProfile && stream.channels < 2) continue; + const isAtmosSource = + stream.codec === 'eac3' || + stream.codec === 'dca' || + stream.codec === 'dts' || + stream.codec === 'truehd'; + + if (profile.isAtmos && !isAtmosSource) continue; + renditions.push({ ...profile, + groupId: profile.groupId, sourceChannels: stream.channels, + sourceCodec: stream.codec, language: stream.language, streamIndex: stream.index, title: stream.title, + relativeUrl: '', + isAtmos: profile.isAtmos && isAtmosSource, }); } } diff --git a/src/infrastructure/ffmpeg/encoding/profiles/audio.json b/src/infrastructure/ffmpeg/encoding/profiles/audio.json deleted file mode 100644 index c152592..0000000 --- a/src/infrastructure/ffmpeg/encoding/profiles/audio.json +++ /dev/null @@ -1,51 +0,0 @@ -[ - { - "name": "aud_aac_he2_t1", - "channels": 2, - "codec": "libfdk_aac", - "profile": "aac_he_v2", - "bitrate": 32000, - "sampleRate": 44100, - "hardwareProfile": false - }, - { - "name": "aud_aac_lc_t2", - "channels": 2, - "codec": "aac", - "bitrate": 64000, - "sampleRate": 44100, - "hardwareProfile": false - }, - { - "name": "aud_aac_lc_t3", - "channels": 2, - "codec": "aac", - "bitrate": 128000, - "sampleRate": 44100, - "hardwareProfile": false - }, - { - "name": "aud_aac_lc_t4", - "channels": 2, - "codec": "aac", - "bitrate": 160000, - "sampleRate": 48000, - "hardwareProfile": false - }, - { - "name": "aud_eac3_51_t1", - "channels": 6, - "codec": "eac3", - "bitrate": 448000, - "sampleRate": 48000, - "hardwareProfile": true - }, - { - "name": "aud_ac3_51_t1", - "channels": 6, - "codec": "ac3", - "bitrate": 384000, - "sampleRate": 48000, - "hardwareProfile": true - } -] diff --git a/src/infrastructure/ffmpeg/encoding/profiles/video.json b/src/infrastructure/ffmpeg/encoding/profiles/video.json deleted file mode 100644 index 42fd7e2..0000000 --- a/src/infrastructure/ffmpeg/encoding/profiles/video.json +++ /dev/null @@ -1,338 +0,0 @@ -{ - "h264_sdr": [ - { - "name": "vod_avc_sdr_t1", - "width": 416, - "height": 234, - "bitrate": 130000, - "maxrate": 145000, - "bufsize": 290000, - "frameRate": 30, - "hlsTime": 6, - "videoCodec": "libx264", - "preset": "slow", - "profile": "baseline", - "level": "3.0", - "videoCodecTag": "avc1.42E01E" - }, - { - "name": "vod_avc_sdr_t2", - "width": 480, - "height": 270, - "bitrate": 365000, - "maxrate": 400000, - "bufsize": 800000, - "frameRate": 30, - "hlsTime": 6, - "videoCodec": "libx264", - "preset": "slow", - "profile": "baseline", - "level": "3.0", - "videoCodecTag": "avc1.42E01E" - }, - { - "name": "vod_avc_sdr_t3", - "width": 640, - "height": 360, - "bitrate": 730000, - "maxrate": 800000, - "bufsize": 1600000, - "frameRate": 30, - "hlsTime": 6, - "videoCodec": "libx264", - "preset": "slow", - "profile": "main", - "level": "3.0", - "videoCodecTag": "avc1.4D401E" - }, - { - "name": "vod_avc_sdr_t4", - "width": 960, - "height": 540, - "bitrate": 2000000, - "maxrate": 2200000, - "bufsize": 4400000, - "frameRate": 30, - "hlsTime": 6, - "videoCodec": "libx264", - "preset": "slow", - "profile": "high", - "level": "3.1", - "videoCodecTag": "avc1.64001F" - }, - { - "name": "vod_avc_sdr_t5", - "width": 1280, - "height": 720, - "bitrate": 3000000, - "maxrate": 3300000, - "bufsize": 6000000, - "frameRate": 30, - "hlsTime": 6, - "videoCodec": "libx264", - "preset": "slow", - "profile": "high", - "level": "3.1", - "videoCodecTag": "avc1.64001F" - }, - { - "name": "vod_avc_sdr_t6", - "width": 1920, - "height": 1080, - "bitrate": 6000000, - "maxrate": 6600000, - "bufsize": 12000000, - "frameRate": 30, - "hlsTime": 6, - "videoCodec": "libx264", - "preset": "slow", - "profile": "high", - "level": "4.0", - "videoCodecTag": "avc1.640028" - } - ], - "h265_sdr": [ - { - "name": "vod_hevc_sdr_t1", - "width": 416, - "height": 234, - "bitrate": 130000, - "maxrate": 145000, - "bufsize": 290000, - "frameRate": 30, - "hlsTime": 6, - "videoCodec": "libx265", - "preset": "slow", - "profile": "main", - "level": "3.1", - "videoCodecTag": "hvc1.1.4.L93.B0" - }, - { - "name": "vod_hevc_sdr_t2", - "width": 480, - "height": 270, - "bitrate": 250000, - "maxrate": 280000, - "bufsize": 500000, - "frameRate": 30, - "hlsTime": 6, - "videoCodec": "libx265", - "preset": "slow", - "profile": "main", - "level": "3.1", - "videoCodecTag": "hvc1.1.4.L93.B0" - }, - { - "name": "vod_hevc_sdr_t3", - "width": 640, - "height": 360, - "bitrate": 575000, - "maxrate": 630000, - "bufsize": 1200000, - "frameRate": 30, - "hlsTime": 6, - "videoCodec": "libx265", - "preset": "slow", - "profile": "main", - "level": "3.1", - "videoCodecTag": "hvc1.1.4.L93.B0" - }, - { - "name": "vod_hevc_sdr_t4", - "width": 960, - "height": 540, - "bitrate": 1100000, - "maxrate": 1200000, - "bufsize": 2400000, - "frameRate": 30, - "hlsTime": 6, - "videoCodec": "libx265", - "preset": "slow", - "profile": "main", - "level": "3.1", - "videoCodecTag": "hvc1.1.4.L93.B0" - }, - { - "name": "vod_hevc_sdr_t5", - "width": 1280, - "height": 720, - "bitrate": 2100000, - "maxrate": 2300000, - "bufsize": 4200000, - "frameRate": 30, - "hlsTime": 6, - "videoCodec": "libx265", - "preset": "slow", - "profile": "main", - "level": "4.0", - "videoCodecTag": "hvc1.1.4.L120.B0" - }, - { - "name": "vod_hevc_sdr_t6", - "width": 1920, - "height": 1080, - "bitrate": 6800000, - "maxrate": 7200000, - "bufsize": 12000000, - "frameRate": 30, - "hlsTime": 6, - "videoCodec": "libx265", - "preset": "slow", - "profile": "main", - "level": "4.0", - "videoCodecTag": "hvc1.1.4.L120.B0" - }, - { - "name": "vod_hevc_sdr_t7", - "width": 2560, - "height": 1440, - "bitrate": 11600000, - "maxrate": 12500000, - "bufsize": 20000000, - "frameRate": 30, - "hlsTime": 6, - "videoCodec": "libx265", - "preset": "slow", - "profile": "main", - "level": "5.0", - "videoCodecTag": "hvc1.1.4.L150.B0" - }, - { - "name": "vod_hevc_sdr_t8", - "width": 3840, - "height": 2160, - "bitrate": 20000000, - "maxrate": 22000000, - "bufsize": 30000000, - "frameRate": 30, - "hlsTime": 6, - "videoCodec": "libx265", - "preset": "slow", - "profile": "main", - "level": "5.0", - "videoCodecTag": "hvc1.1.4.L150.B0" - } - ], - "h265_hdr": [ - { - "name": "vod_hevc_hdr_t1", - "width": 416, - "height": 234, - "bitrate": 130000, - "maxrate": 145000, - "bufsize": 290000, - "frameRate": 30, - "hlsTime": 6, - "videoCodec": "libx265", - "preset": "slow", - "profile": "main10", - "level": "3.1", - "videoCodecTag": "hvc1.2.4.L93.B0" - }, - { - "name": "vod_hevc_hdr_t2", - "width": 480, - "height": 270, - "bitrate": 300000, - "maxrate": 330000, - "bufsize": 600000, - "frameRate": 30, - "hlsTime": 6, - "videoCodec": "libx265", - "preset": "slow", - "profile": "main10", - "level": "3.1", - "videoCodecTag": "hvc1.2.4.L93.B0" - }, - { - "name": "vod_hevc_hdr_t3", - "width": 640, - "height": 360, - "bitrate": 690000, - "maxrate": 750000, - "bufsize": 1300000, - "frameRate": 30, - "hlsTime": 6, - "videoCodec": "libx265", - "preset": "slow", - "profile": "main10", - "level": "3.1", - "videoCodecTag": "hvc1.2.4.L93.B0" - }, - { - "name": "vod_hevc_hdr_t4", - "width": 960, - "height": 540, - "bitrate": 1300000, - "maxrate": 1450000, - "bufsize": 2600000, - "frameRate": 30, - "hlsTime": 6, - "videoCodec": "libx265", - "preset": "slow", - "profile": "main10", - "level": "3.1", - "videoCodecTag": "hvc1.2.4.L93.B0" - }, - { - "name": "vod_hevc_hdr_t5", - "width": 1280, - "height": 720, - "bitrate": 2500000, - "maxrate": 2700000, - "bufsize": 5000000, - "frameRate": 30, - "hlsTime": 6, - "videoCodec": "libx265", - "preset": "slow", - "profile": "main10", - "level": "4.0", - "videoCodecTag": "hvc1.2.4.L120.B0" - }, - { - "name": "vod_hevc_hdr_t6", - "width": 1920, - "height": 1080, - "bitrate": 8100000, - "maxrate": 8800000, - "bufsize": 14000000, - "frameRate": 30, - "hlsTime": 6, - "videoCodec": "libx265", - "preset": "slow", - "profile": "main10", - "level": "4.0", - "videoCodecTag": "hvc1.2.4.L120.B0" - }, - { - "name": "vod_hevc_hdr_t7", - "width": 2560, - "height": 1440, - "bitrate": 14000000, - "maxrate": 15000000, - "bufsize": 22000000, - "frameRate": 30, - "hlsTime": 6, - "videoCodec": "libx265", - "preset": "slow", - "profile": "main10", - "level": "5.0", - "videoCodecTag": "hvc1.2.4.L150.B0" - }, - { - "name": "vod_hevc_hdr_t8", - "width": 3840, - "height": 2160, - "bitrate": 24000000, - "maxrate": 26000000, - "bufsize": 32000000, - "frameRate": 30, - "hlsTime": 6, - "videoCodec": "libx265", - "preset": "slow", - "profile": "main10", - "level": "5.0", - "videoCodecTag": "hvc1.2.4.L150.B0" - } - ] -} diff --git a/src/infrastructure/ffmpeg/hls/pipeline.ts b/src/infrastructure/ffmpeg/hls/pipeline.ts index 5dcbc24..b1a20d2 100644 --- a/src/infrastructure/ffmpeg/hls/pipeline.ts +++ b/src/infrastructure/ffmpeg/hls/pipeline.ts @@ -1,4 +1,6 @@ import path from 'node:path'; +import fs from 'node:fs/promises'; +import { pino } from 'pino'; import { config } from '../../../config/env.js'; import type { ProgressCallback } from '../../../domain/job.interface.js'; import type { VideoVariantMeta, AudioVariantMeta } from '../types.js'; @@ -11,14 +13,33 @@ import { import { runFFmpeg } from '../core/runner.js'; import { HLS_CONSTANTS } from '../constants.js'; +const logger = pino({ name: 'HlsPipeline' }); + /** - * Executes the multi-phase FFmpeg transcoding pipeline using dynamically generated `filter_complex` graphs. - * - * @remarks - * - Phases are executed sequentially (Audio -> H.264 -> H.265) to strictly bound peak active memory usage. - * - The `filter_complex` graph splits the decoded input stream in memory, avoiding redundant decodes per resolution. - * - Aggregates and normalizes percentage callbacks across phases using algorithmic weighting based on codec complexity. + * Post-process variant manifests to fix init segment URIs. + * FFmpeg's -hls_base_url only applies to .m4s segment URIs, NOT to #EXT-X-MAP:URI (init segments). + * This function prepends the CDN base URL to the init segment filename. */ +async function fixInitSegmentUrls( + outputDir: string, + relativeUrl: string, + baseUrl: string | undefined, +): Promise { + if (!baseUrl) return; + const manifestPath = path.join(outputDir, relativeUrl, HLS_CONSTANTS.VARIANT_PLAYLIST_NAME); + try { + let content = await fs.readFile(manifestPath, 'utf8'); + // Match: #EXT-X-MAP:URI="filename.mp4" (bare filename without http) + content = content.replace( + /#EXT-X-MAP:URI="(?!https?:\/\/)([^"]+)"/g, + `#EXT-X-MAP:URI="${baseUrl}$1"`, + ); + await fs.writeFile(manifestPath, content); + } catch (err) { + logger.warn({ manifestPath, err }, 'Could not fix init segment URL'); + } +} + export async function processMasterPipeline( sourceUrl: string, outputDir: string, @@ -33,11 +54,9 @@ export async function processMasterPipeline( ): Promise { const h264Sdr = videoVariants.filter((v) => v.videoCodec === 'libx264'); const h265Sdr = videoVariants.filter( - (v) => v.videoCodec === 'libx265' && v.profile !== 'main10', - ); - const h265Hdr = videoVariants.filter( - (v) => v.videoCodec === 'libx265' && v.profile === 'main10', + (v) => v.videoCodec === 'libx265' && v.videoRange === 'SDR', ); + const h265Hdr = videoVariants.filter((v) => v.videoCodec === 'libx265' && v.videoRange === 'PQ'); let currentBaseProgress = 0; const weightAudio = audioRenditions.length > 0 ? 10 : 0; @@ -66,9 +85,15 @@ export async function processMasterPipeline( const getBaseInputArgs = () => { const args = []; + args.push('-drc_scale', '0'); + if (config.TEST_DURATION_SECONDS) { args.push('-t', String(config.TEST_DURATION_SECONDS)); } + + if (sourceUrl.startsWith('http')) { + args.push('-reconnect', '1', '-reconnect_streamed', '1', '-reconnect_delay_max', '5'); + } args.push('-i', sourceUrl); return args; }; @@ -82,28 +107,48 @@ export async function processMasterPipeline( const manifestPath = path.join(audioDir, HLS_CONSTANTS.VARIANT_PLAYLIST_NAME); const streamMap = audio.streamIndex !== -1 ? `0:a:${audio.streamIndex}?` : '0:a:0?'; + const baseUrl = config.DOMAIN_SUBDOMAIN_NAME + ? `${config.DOMAIN_SUBDOMAIN_NAME}/${config.AZURE_STORAGE_CONTAINER_NAME}/${config.CONTAINER_DIRECTORY_1}/${audio.relativeUrl}/` + : undefined; + args.push( '-map', streamMap, ...audioEncoderFlags(audio), - ...hlsOutputFlags(hlsTime, audioDir), + ...hlsOutputFlags(hlsTime, audioDir, videoId, undefined, audio, baseUrl), manifestPath, ); }); return args; }); + + // Fix init segment URLs for all audio renditions + for (const audio of audioRenditions) { + const baseUrl = config.DOMAIN_SUBDOMAIN_NAME + ? `${config.DOMAIN_SUBDOMAIN_NAME}/${config.AZURE_STORAGE_CONTAINER_NAME}/${config.CONTAINER_DIRECTORY_1}/${audio.relativeUrl}/` + : undefined; + await fixInitSegmentUrls(outputDir, audio.relativeUrl, baseUrl); + } } const buildVideoPhaseArgs = (variants: VideoVariantMeta[], isHdr: boolean) => { const args = [...getBaseInputArgs()]; const filtergraph: string[] = []; - let preFilter = isHdr ? '[0:v:0]format=yuv420p10le' : '[0:v:0]format=yuv420p'; + let preFilter = ''; - if (!isHdr && (videoRange === 'PQ' || videoRange === 'HLG')) { - preFilter = `[0:v:0]zscale=transfer=linear:npl=100,format=gbrpf32le,tonemap=hable:desat=0,zscale=transfer=bt709:matrix=bt709:primaries=bt709:range=tv,format=yuv420p`; + if (isHdr) { + preFilter = '[0:v:0]format=yuv420p10le'; + } else { + if (videoRange === 'PQ') { + preFilter = `[0:v:0]zscale=tin=smpte2084:min=bt2020nc:pin=bt2020:rin=tv:t=linear:npl=100,format=gbrpf32le,zscale=p=bt709,tonemap=tonemap=hable:desat=0,zscale=t=bt709:m=bt709:r=tv,format=yuv420p`; + } else if (videoRange === 'HLG') { + preFilter = `[0:v:0]zscale=tin=arib-std-b67:min=bt2020nc:pin=bt2020:rin=tv:t=linear:npl=100,format=gbrpf32le,zscale=p=bt709,tonemap=tonemap=hable:desat=0,zscale=t=bt709:m=bt709:r=tv,format=yuv420p`; + } else { + preFilter = '[0:v:0]format=yuv420p'; + } + // preFilter += `,hqdn3d=3:3:2:2`; } - if (!isHdr) preFilter += `,hqdn3d=3:3:2:2`; if (variants.length > 1) { const splits = variants.map((_, i) => `[split_${i}]`).join(''); @@ -123,11 +168,15 @@ export async function processMasterPipeline( const variantDir = path.join(outputDir, variant.relativeUrl); const manifestPath = path.join(variantDir, HLS_CONSTANTS.VARIANT_PLAYLIST_NAME); + const baseUrl = config.DOMAIN_SUBDOMAIN_NAME + ? `${config.DOMAIN_SUBDOMAIN_NAME}/${config.AZURE_STORAGE_CONTAINER_NAME}/${config.CONTAINER_DIRECTORY_1}/${variant.relativeUrl}/` + : undefined; + args.push( '-map', `[vout${index}]`, ...videoEncoderFlags(variant, sourceFrameRate), - ...hlsOutputFlags(hlsTime, variantDir), + ...hlsOutputFlags(hlsTime, variantDir, videoId, variant, undefined, baseUrl), manifestPath, ); }); @@ -140,4 +189,12 @@ export async function processMasterPipeline( await runPhase('Phase_3_H265_SDR', weightH265Sdr, () => buildVideoPhaseArgs(h265Sdr, false)); if (h265Hdr.length > 0) await runPhase('Phase_4_H265_HDR', weightH265Hdr, () => buildVideoPhaseArgs(h265Hdr, true)); + + // Fix init segment URLs for all video variants + for (const v of videoVariants) { + const baseUrl = config.DOMAIN_SUBDOMAIN_NAME + ? `${config.DOMAIN_SUBDOMAIN_NAME}/${config.AZURE_STORAGE_CONTAINER_NAME}/${config.CONTAINER_DIRECTORY_1}/${v.relativeUrl}/` + : undefined; + await fixInitSegmentUrls(outputDir, v.relativeUrl, baseUrl); + } } diff --git a/src/infrastructure/ffmpeg/hls/playlist.ts b/src/infrastructure/ffmpeg/hls/playlist.ts index 94530b6..279cfea 100644 --- a/src/infrastructure/ffmpeg/hls/playlist.ts +++ b/src/infrastructure/ffmpeg/hls/playlist.ts @@ -1,6 +1,8 @@ import path from 'node:path'; import fs from 'node:fs/promises'; +import { createHash } from 'node:crypto'; import { pino } from 'pino'; +import { config } from '../../../config/env.js'; import type { VideoVariantMeta, AudioVariantMeta } from '../types.js'; import { HLS_CONSTANTS } from '../constants.js'; import { getBroadcastFrameRate } from '../encoding/flags.js'; @@ -29,29 +31,55 @@ const LANGUAGE_NAMES: Record = { function getAspectRatioString(width: number, height: number): string { const ratio = width / height; - if (Math.abs(ratio - 16 / 9) < 0.05) return '16:9'; - if (Math.abs(ratio - 9 / 16) < 0.05) return '9:16'; - if (Math.abs(ratio - 1.85) < 0.05) return '1.85:1'; - if (Math.abs(ratio - 2.0) < 0.05) return '2.00:1'; - if (Math.abs(ratio - 2.39) < 0.05) return '2.39:1'; + if (Math.abs(ratio - 16 / 9) < 0.02) return '16:9'; + if (Math.abs(ratio - 9 / 16) < 0.02) return '9:16'; + if (Math.abs(ratio - 4 / 3) < 0.02) return '4:3'; + if (Math.abs(ratio - 1.0) < 0.02) return '1:1'; + if (Math.abs(ratio - 1.85) < 0.02) return '1.85:1'; + if (Math.abs(ratio - 2.0) < 0.02) return '2.00:1'; + if (Math.abs(ratio - 2.35) < 0.02) return '2.35:1'; + if (Math.abs(ratio - 2.39) < 0.02) return '2.39:1'; + if (Math.abs(ratio - 1.9) < 0.02) return '1.90:1'; return `${ratio.toFixed(2)}:1`; } -function getPairedAudioNames(videoHeight: number, availableAudio: AudioVariantMeta[]): string[] { +function formatBitrate(bps: number): string { + if (bps < 1_000_000) { + return `${Math.round(bps / 1000) + .toString() + .padStart(4)} kbps`; + } + return `${(bps / 1_000_000).toFixed(1).padStart(4)} Mbps`; +} + +function getStableId(name: string, extra?: string): string { + const input = `${name}${extra ? `-${extra}` : ''}`; + return createHash('sha256').update(input).digest('hex'); +} + +function getPairedAudioNames( + videoWidth: number, + videoHeight: number, + availableAudio: AudioVariantMeta[], +): string[] { if (!availableAudio || availableAudio.length === 0) return []; + const maxDim = Math.max(videoWidth, videoHeight); const hasAudio = (name: string) => availableAudio.some((a) => a.name === name); + const getGroupId = (name: string) => + availableAudio.find((a) => a.name === name)?.groupId || name; + const hardware: string[] = []; - if (hasAudio('aud_ac3_51_t1')) hardware.push('aud_ac3_51_t1'); - if (hasAudio('aud_eac3_51_t1')) hardware.push('aud_eac3_51_t1'); + if (hasAudio('audio-ac3')) hardware.push(getGroupId('audio-ac3')); + if (hasAudio('audio-atmos')) hardware.push(getGroupId('audio-atmos')); - let stereo = availableAudio[0].name; - if (videoHeight > 720 && hasAudio('aud_aac_lc_t4')) stereo = 'aud_aac_lc_t4'; - else if (videoHeight > 432 && hasAudio('aud_aac_lc_t3')) stereo = 'aud_aac_lc_t3'; - else if (videoHeight > 270 && hasAudio('aud_aac_lc_t2')) stereo = 'aud_aac_lc_t2'; - else if (hasAudio('aud_aac_he2_t1')) stereo = 'aud_aac_he2_t1'; + let stereoName = 'audio-stereo-64'; + if (maxDim >= 1280 && hasAudio('audio-stereo-160')) stereoName = 'audio-stereo-160'; + else if (maxDim >= 854 && hasAudio('audio-stereo-128')) stereoName = 'audio-stereo-128'; + else if (hasAudio('audio-stereo-64')) stereoName = 'audio-stereo-64'; + else stereoName = availableAudio[0].name; - return [stereo, ...hardware]; + return [getGroupId(stereoName), ...hardware]; } async function getBandwidthForDir(dirPath: string): Promise<{ peak: number; avg: number }> { @@ -81,7 +109,9 @@ async function getBandwidthForDir(dirPath: string): Promise<{ peak: number; avg: bits = currentByteRangeLength * 8; currentByteRangeLength = 0; } else { - const stat = await fs.stat(path.join(dirPath, trimmed.split('?')[0])); + const filename = trimmed.split('/').pop()?.split('?')[0]; + if (!filename) continue; + const stat = await fs.stat(path.join(dirPath, filename)); bits = stat.size * 8; } const bitrate = currentDuration > 0 ? Math.round(bits / currentDuration) : 0; @@ -109,26 +139,41 @@ export async function writeMasterPlaylist( const lines = ['#EXTM3U', '#EXT-X-VERSION:7', '#EXT-X-INDEPENDENT-SEGMENTS', '']; - const defaultVariant = videoVariants.reduce((prev, curr) => - Math.abs(curr.maxrate - 2000000) < Math.abs(prev.maxrate - 2000000) ? curr : prev, - ); - const orderedVariants = [ - defaultVariant, - ...videoVariants.filter((v) => v.name !== defaultVariant.name), - ]; + const getCodecWeight = (v: VideoVariantMeta) => { + if (v.videoCodecTag.startsWith('avc')) return 1; + if (v.videoCodecTag.startsWith('hvc') && v.videoRange === 'SDR') return 2; + if (v.videoCodecTag.startsWith('hvc') && v.videoRange === 'PQ') return 3; + if (v.videoCodecTag.startsWith('dv')) return 4; + return 5; + }; + + const anchorVariants = videoVariants.filter((v) => v.tierNumber === 7); + const otherVariants = videoVariants.filter((v) => v.tierNumber !== 7); + + anchorVariants.sort((a, b) => getCodecWeight(a) - getCodecWeight(b)); + + otherVariants.sort((a, b) => { + const weightA = getCodecWeight(a); + const weightB = getCodecWeight(b); + if (weightA !== weightB) return weightA - weightB; + return a.maxrate - b.maxrate; + }); + + const orderedVariants = [...anchorVariants, ...otherVariants]; const variantAudioMap = new Map(); const usedAudioNames = new Set(); for (const v of orderedVariants) { - const paired = getPairedAudioNames(v.actualHeight, audioRenditions); + const paired = getPairedAudioNames(v.actualWidth, v.actualHeight, audioRenditions); variantAudioMap.set(v.name, paired); paired.forEach((name) => usedAudioNames.add(name)); } let currentLangGroup = ''; for (const audio of audioRenditions) { - if (!usedAudioNames.has(audio.name)) continue; + const audioGroupId = audio.groupId || audio.name; + if (!usedAudioNames.has(audioGroupId)) continue; const displayName = !audio.title.startsWith('Track ') ? audio.title @@ -144,9 +189,14 @@ export async function writeMasterPlaylist( ? '' : `LANGUAGE="${ISO_639_1_MAP[audio.language] || audio.language}",`; const relativeUri = `../${audio.relativeUrl}/${HLS_CONSTANTS.VARIANT_PLAYLIST_NAME}`; + const stableRenditionId = getStableId(audio.name, audio.language); + + const channelAttr = audio.isAtmos ? '16/JOC' : audio.channels.toString(); lines.push( - `#EXT-X-MEDIA:TYPE=AUDIO,NAME="${displayName}",GROUP-ID="${audio.name}",${langAttr}DEFAULT=${audio.streamIndex === 0 ? 'YES' : 'NO'},AUTOSELECT=YES,CHANNELS="${audio.channels}",URI="${relativeUri}"`, + `#EXT-X-MEDIA:TYPE=AUDIO,NAME="${displayName}",GROUP-ID="${audioGroupId}",${langAttr}DEFAULT=${ + audio.streamIndex === 0 ? 'YES' : 'NO' + },AUTOSELECT=YES,CHANNELS="${channelAttr}",STABLE-RENDITION-ID="${stableRenditionId}",URI="${relativeUri}"`, ); } lines.push(''); @@ -156,21 +206,23 @@ export async function writeMasterPlaylist( const videoBw = await getBandwidthForDir(videoDir); const pairedAudioNames = variantAudioMap.get(v.name) || []; - const trueVideoAvg = videoBw.avg > 0 ? videoBw.avg : v.maxrate * 0.55; + const trueVideoAvg = videoBw.avg > 0 ? videoBw.avg : v.bitrate * 0.9; const trueVideoPeak = videoBw.peak > 0 ? videoBw.peak : v.maxrate; - const actualVideoRange = v.profile === 'main10' ? 'PQ' : 'SDR'; + + const isDovi = v.videoCodecTag.startsWith('dv'); + const actualVideoRange = isDovi || v.profile === 'main10' ? 'PQ' : 'SDR'; const relativeUri = `../${v.relativeUrl}/${HLS_CONSTANTS.VARIANT_PLAYLIST_NAME}`; const fpsInfo = getBroadcastFrameRate(sourceFrameRate); - const frameRateString = fpsInfo ? fpsInfo.appleFormat : (v.frameRate ?? 30).toFixed(3); + const frameRateString = fpsInfo ? fpsInfo.aFormat : (v.frameRate ?? 30).toFixed(3); if (pairedAudioNames.length === 0) { lines.push( - `#-- stream_${v.name.padEnd(16)} ${`${v.actualWidth}x${v.actualHeight}`.padEnd(11)} AR: ${getAspectRatioString(v.actualWidth, v.actualHeight).padEnd(8)} ${v.videoCodecTag.padEnd(14)} regular avg: ${(trueVideoAvg / 1_000_000).toFixed(1).padStart(4)} Mbps max: ${(trueVideoPeak / 1_000_000).toFixed(1).padStart(4)} Mbps --`, + `#-- ${v.name.padEnd(16)} ${`${v.actualWidth}x${v.actualHeight}`.padEnd(11)} AR: ${getAspectRatioString(v.actualWidth, v.actualHeight).padEnd(8)} ${v.videoCodecTag.padEnd(14)} regular avg: ${formatBitrate(trueVideoAvg)} max: ${formatBitrate(trueVideoPeak)} --`, ); const attributes = [ - `AVERAGE-BANDWIDTH=${Math.round(trueVideoAvg * 1.05)}`, - `BANDWIDTH=${Math.round(trueVideoPeak * 1.05)}`, + `AVERAGE-BANDWIDTH=${Math.round(trueVideoAvg)}`, + `BANDWIDTH=${trueVideoPeak}`, `VIDEO-RANGE=${actualVideoRange}`, `CLOSED-CAPTIONS=NONE`, `CODECS="${v.videoCodecTag}"`, @@ -183,43 +235,62 @@ export async function writeMasterPlaylist( } const repAudio = - audioRenditions.find((ar) => ar.name === pairedAudioNames[0]) || audioRenditions[0]; + audioRenditions.find((ar) => (ar.groupId || ar.name) === pairedAudioNames[0]) || + audioRenditions[0]; const repAudioBw = await getBandwidthForDir(path.join(outputDir, repAudio.relativeUrl)); const trueRepAudioAvg = repAudioBw.avg > 0 ? repAudioBw.avg : repAudio.bitrate; const trueRepAudioPeak = repAudioBw.peak > 0 ? repAudioBw.peak : repAudio.bitrate; - const commentPeak = Math.round((trueVideoPeak + trueRepAudioPeak) * 1.05); - const commentAvg = Math.round((trueVideoAvg + trueRepAudioAvg) * 1.05); - lines.push( - `#-- stream_${v.name.padEnd(16)} ${`${v.actualWidth}x${v.actualHeight}`.padEnd(11)} AR: ${getAspectRatioString(v.actualWidth, v.actualHeight).padEnd(8)} ${v.videoCodecTag.padEnd(14)} regular avg: ${(commentAvg / 1_000_000).toFixed(1).padStart(4)} Mbps max: ${(commentPeak / 1_000_000).toFixed(1).padStart(4)} Mbps --`, + `#-- ${v.name.padEnd(16)} ${`${v.actualWidth}x${v.actualHeight}`.padEnd( + 11, + )} AR: ${getAspectRatioString(v.actualWidth, v.actualHeight).padEnd( + 8, + )} ${v.videoCodecTag.padEnd(14)} regular avg: ${formatBitrate( + Math.round(trueVideoAvg + trueRepAudioAvg), + )} max: ${formatBitrate(trueVideoPeak + trueRepAudioPeak)} --`, ); - for (const audioName of pairedAudioNames) { - const a = audioRenditions.find((ar) => ar.name === audioName) || audioRenditions[0]; + for (const audioGroupId of pairedAudioNames) { + const a = + audioRenditions.find((ar) => (ar.groupId || ar.name) === audioGroupId) || + audioRenditions[0]; const audioBw = await getBandwidthForDir(path.join(outputDir, a.relativeUrl)); - const localPeakBandwidth = Math.round( - (trueVideoPeak + (audioBw.peak > 0 ? audioBw.peak : a.bitrate)) * 1.05, - ); - const localAvgBandwidth = Math.round( - (trueVideoAvg + (audioBw.avg > 0 ? audioBw.avg : a.bitrate)) * 1.05, - ); + const audioAvg = audioBw.avg > 0 ? audioBw.avg : a.bitrate; + const audioPeak = audioBw.peak > 0 ? audioBw.peak : a.bitrate; + + const localAvgBandwidth = Math.round(trueVideoAvg + audioAvg); + const localPeakBandwidth = Math.round(trueVideoPeak + audioPeak); let audioCodecTag = 'mp4a.40.2'; - if (a.codec === 'libfdk_aac' && a.profile === 'aac_he_v2') audioCodecTag = 'mp4a.40.29'; + if (a.profile === 'aac_he') audioCodecTag = 'mp4a.40.5'; + else if (a.profile === 'aac_he_v2') audioCodecTag = 'mp4a.40.29'; else if (a.codec === 'ac3') audioCodecTag = 'ac-3'; else if (a.codec === 'eac3') audioCodecTag = 'ec-3'; + const maxDim = Math.max(v.actualWidth, v.actualHeight); + const isHighDef = maxDim >= 1280; + const isUltraHighDef = + maxDim >= 2560 || v.profile === 'main10' || v.videoCodecTag.startsWith('dvh1'); + + let hdcpLevel = 'NONE'; + if (isUltraHighDef) hdcpLevel = 'TYPE-1'; + else if (isHighDef) hdcpLevel = 'TYPE-0'; + + const stableVariantId = getStableId(v.name, v.videoCodecTag); + const attributes = [ `AVERAGE-BANDWIDTH=${localAvgBandwidth}`, `BANDWIDTH=${localPeakBandwidth}`, `VIDEO-RANGE=${actualVideoRange}`, `CLOSED-CAPTIONS=NONE`, `CODECS="${v.videoCodecTag},${audioCodecTag}"`, - `AUDIO="${audioName}"`, + `AUDIO="${audioGroupId}"`, `FRAME-RATE=${frameRateString}`, + `HDCP-LEVEL=${hdcpLevel}`, `RESOLUTION=${v.actualWidth}x${v.actualHeight}`, + `STABLE-VARIANT-ID="${stableVariantId}"`, ].join(','); lines.push(`#EXT-X-STREAM-INF:${attributes}`, relativeUri); diff --git a/src/infrastructure/ffmpeg/types.ts b/src/infrastructure/ffmpeg/types.ts index c7805d4..378047c 100644 --- a/src/infrastructure/ffmpeg/types.ts +++ b/src/infrastructure/ffmpeg/types.ts @@ -1,6 +1,7 @@ import type { ProgressCallback } from '../../domain/job.interface.js'; export interface VideoProfile { + tierNumber?: number; name: string; width: number; height: number; @@ -15,16 +16,20 @@ export interface VideoProfile { preset: string; profile?: string; level?: string; + crf?: number; } export interface AudioProfile { name: string; + groupId?: string; channels: number; codec: string; profile?: string; bitrate: number; sampleRate: number; hardwareProfile: boolean; + isAtmos?: boolean; + isCinemaMaster?: boolean; } export type VideoVariantMeta = VideoProfile & { @@ -34,7 +39,9 @@ export type VideoVariantMeta = VideoProfile & { }; export type AudioVariantMeta = AudioProfile & { + groupId?: string; sourceChannels: number; + sourceCodec?: string; language: string; streamIndex: number; title: string; diff --git a/src/infrastructure/storage/azure.service.ts b/src/infrastructure/storage/azure.service.ts index 2e7f468..a6774f8 100644 --- a/src/infrastructure/storage/azure.service.ts +++ b/src/infrastructure/storage/azure.service.ts @@ -5,6 +5,7 @@ import { DefaultAzureCredential } from '@azure/identity'; import { pino } from 'pino'; import { config } from '../../config/env.js'; import type { ProgressCallback } from '../../domain/job.interface.js'; +import { HLS_CONSTANTS } from '../ffmpeg/constants.js'; const logger = pino({ name: 'AzureStorage' }); @@ -67,43 +68,80 @@ export class AzureStorageService { let uploadedCount = 0; let masterPlaylistUrl = ''; - for (const filePath of files) { - const relativeToHlsDir = path.relative(folderPath, filePath).replace(/\\/g, '/'); - let blobPath = ''; + let currentIndex = 0; + const totalFiles = files.length; - if (relativeToHlsDir === 'playlist.m3u8') { - blobPath = `${this.envDirectory}/${videoId}/playlist.m3u8`; - } else if (relativeToHlsDir.startsWith('v1/')) { - blobPath = `${this.envDirectory}/${relativeToHlsDir}`; - } else { - blobPath = `${this.envDirectory}/${videoId}/${relativeToHlsDir}`; - } + const uploadWorker = async () => { + while (currentIndex < totalFiles) { + const fileIndex = currentIndex++; + const filePath = files[fileIndex]; - const blockBlobClient = containerClient.getBlockBlobClient(blobPath); - let contentType = 'application/octet-stream'; + const relativeToHlsDir = path.relative(folderPath, filePath).replace(/\\/g, '/'); + let blobPath = ''; - if (filePath.endsWith('.m3u8')) { - contentType = 'application/vnd.apple.mpegurl'; - } else if (filePath.endsWith('.m4s') || filePath.endsWith('.mp4')) { - contentType = 'video/mp4'; - } + if (relativeToHlsDir === HLS_CONSTANTS.MASTER_PLAYLIST_NAME) { + blobPath = `${this.envDirectory}/${videoId}/${HLS_CONSTANTS.MASTER_PLAYLIST_NAME}`; + } else if (relativeToHlsDir.startsWith('v1/')) { + blobPath = `${this.envDirectory}/${relativeToHlsDir}`; + } else { + blobPath = `${this.envDirectory}/${videoId}/${relativeToHlsDir}`; + } - const fileBuffer = await fs.readFile(filePath); - await blockBlobClient.uploadData(fileBuffer, { - blobHTTPHeaders: { - blobContentType: contentType, - }, - }); + const blockBlobClient = containerClient.getBlockBlobClient(blobPath); + let contentType = 'application/octet-stream'; - if (relativeToHlsDir === 'playlist.m3u8') { - masterPlaylistUrl = blockBlobClient.url; - } + if (filePath.endsWith('.m3u8')) { + contentType = 'application/vnd.apple.mpegurl'; + } else if (filePath.endsWith('.m4s')) { + contentType = 'application/octet-stream'; + } else if (filePath.endsWith('.mp4')) { + contentType = 'video/mp4'; + } + + let attempts = 0; + const maxRetries = config.AZURE_UPLOAD_RETRIES; + let success = false; + + while (attempts < maxRetries && !success) { + try { + attempts++; + await blockBlobClient.uploadFile(filePath, { + blobHTTPHeaders: { + blobContentType: contentType, + }, + }); + success = true; + } catch (error) { + if (attempts >= maxRetries) { + logger.error( + { videoId, filePath, attempts }, + 'Failed to upload Blob file after max retries', + ); + throw error; + } + await new Promise((res) => setTimeout(res, 1000 * attempts)); + } + } - uploadedCount++; - if (onProgress) { - onProgress({ variant: 'Azure Upload', percent: (uploadedCount / files.length) * 100 }); + if (relativeToHlsDir === HLS_CONSTANTS.MASTER_PLAYLIST_NAME) { + masterPlaylistUrl = blockBlobClient.url; + } + + uploadedCount++; + if (onProgress) { + const percent = Math.round((uploadedCount / totalFiles) * 100); + const prevPercent = Math.round(((uploadedCount - 1) / totalFiles) * 100); + if (percent > prevPercent) { + onProgress({ variant: 'Azure Upload', percent }); + } + } } - } + }; + + const concurrency = Math.min(config.AZURE_UPLOAD_BATCH_SIZE, totalFiles); + const workers = Array.from({ length: concurrency }).map(() => uploadWorker()); + + await Promise.all(workers); logger.info({ videoId, uploadedFiles: uploadedCount }, 'Azure upload complete'); return masterPlaylistUrl; From d047f4a598c5137d6e6a4d67ee4aca366628fa27 Mon Sep 17 00:00:00 2001 From: MAULIK MK <214461757+maulik-mk@users.noreply.github.com> Date: Sun, 5 Apr 2026 18:48:08 +0530 Subject: [PATCH 12/12] Release/v0.3.0 (#18) - Sync changes of v0.3.0 PR --- Dockerfile | 2 +- package.json | 2 +- pnpm-lock.yaml | 334 ++++++++--------- src/infrastructure/ffmpeg/core/complexity.ts | 192 ++++++++++ .../ffmpeg/encoding/profiles/audio.json | 51 +++ .../ffmpeg/encoding/profiles/video.json | 338 ++++++++++++++++++ 6 files changed, 750 insertions(+), 169 deletions(-) create mode 100644 src/infrastructure/ffmpeg/core/complexity.ts create mode 100644 src/infrastructure/ffmpeg/encoding/profiles/audio.json create mode 100644 src/infrastructure/ffmpeg/encoding/profiles/video.json diff --git a/Dockerfile b/Dockerfile index a29a39b..abb7665 100644 --- a/Dockerfile +++ b/Dockerfile @@ -170,7 +170,7 @@ ENV DEBIAN_FRONTEND=noninteractive # ----------------------------------------------------------------------------- # Metadata & OCI Labels # ----------------------------------------------------------------------------- -ARG VERSION=0.2.0 +ARG VERSION=0.3.0 ARG BUILD_DATE=unknown LABEL org.opencontainers.image.title="ffmpeg-queue-worker-node" \ diff --git a/package.json b/package.json index c73ed2c..fcc036b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "worker", - "version": "0.2.0", + "version": "0.3.0", "description": "FFmpeg Worker Service (TypeScript)", "repository": { "type": "git", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2800053..620dec5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -25,7 +25,7 @@ importers: version: 9.1.0 bullmq: specifier: ^5.0.0 - version: 5.71.0 + version: 5.73.0 fastify: specifier: ^4.26.0 version: 4.29.1 @@ -47,10 +47,10 @@ importers: devDependencies: '@types/node': specifier: ^20.11.0 - version: 20.19.37 + version: 20.19.39 '@types/pg': specifier: ^8.11.0 - version: 8.18.0 + version: 8.20.0 prettier: specifier: ^3.2.0 version: 3.8.1 @@ -114,16 +114,16 @@ packages: resolution: {integrity: sha512-fCqPIfOcLE+CGqGPd66c8bZpwAji98tZ4JI9i/mlTNTlsIWslCfpg48s/ypyLxZTump5sypjrKn2/kY7q8oAbA==} engines: {node: '>=20.0.0'} - '@azure/msal-browser@5.6.1': - resolution: {integrity: sha512-Ylmp8yngH7YRLV5mA1aF4CNS6WsJTPbVXaA0Tb1x1Gv/J3BM3hE4Q7nDaf7dRfU00FcxDBBudTjqlpH74ZSsgw==} + '@azure/msal-browser@5.6.3': + resolution: {integrity: sha512-sTjMtUm+bJpENU/1WlRzHEsgEHppZDZ1EtNyaOODg/sQBtMxxJzGB+MOCM+T2Q5Qe1fKBrdxUmjyRxm0r7Ez9w==} engines: {node: '>=0.8.0'} - '@azure/msal-common@16.4.0': - resolution: {integrity: sha512-twXt09PYtj1PffNNIAzQlrBd0DS91cdA6i1gAfzJ6BnPM4xNk5k9q/5xna7jLIjU3Jnp0slKYtucshGM8OGNAw==} + '@azure/msal-common@16.4.1': + resolution: {integrity: sha512-Bl8f+w37xkXsYh7QRkAKCFGYtWMYuOVO7Lv+BxILrvGz3HbIEF22Pt0ugyj0QPOl6NLrHcnNUQ9yeew98P/5iw==} engines: {node: '>=0.8.0'} - '@azure/msal-node@5.1.1': - resolution: {integrity: sha512-71grXU6+5hl+3CL3joOxlj/AW6rmhthuTlG0fRqsTrhPArQBpZuUFzCIlKOGdcafLUa/i1hBdV78ZxJdlvRA+g==} + '@azure/msal-node@5.1.2': + resolution: {integrity: sha512-DoeSJ9U5KPAIZoHsPywvfEj2MhBniQe0+FSpjLUTdWoIkI999GB5USkW6nNEHnIaLVxROHXvprWA1KzdS1VQ4A==} engines: {node: '>=20'} '@azure/storage-blob@12.31.0': @@ -134,158 +134,158 @@ packages: resolution: {integrity: sha512-/OFHhy86aG5Pe8dP5tsp+BuJ25JOAl9yaMU3WZbkeoiFMHFtJ7tu5ili7qEdBXNW9G5lDB19trwyI6V49F/8iQ==} engines: {node: '>=20.0.0'} - '@esbuild/aix-ppc64@0.27.4': - resolution: {integrity: sha512-cQPwL2mp2nSmHHJlCyoXgHGhbEPMrEEU5xhkcy3Hs/O7nGZqEpZ2sUtLaL9MORLtDfRvVl2/3PAuEkYZH0Ty8Q==} + '@esbuild/aix-ppc64@0.27.7': + resolution: {integrity: sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==} engines: {node: '>=18'} cpu: [ppc64] os: [aix] - '@esbuild/android-arm64@0.27.4': - resolution: {integrity: sha512-gdLscB7v75wRfu7QSm/zg6Rx29VLdy9eTr2t44sfTW7CxwAtQghZ4ZnqHk3/ogz7xao0QAgrkradbBzcqFPasw==} + '@esbuild/android-arm64@0.27.7': + resolution: {integrity: sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==} engines: {node: '>=18'} cpu: [arm64] os: [android] - '@esbuild/android-arm@0.27.4': - resolution: {integrity: sha512-X9bUgvxiC8CHAGKYufLIHGXPJWnr0OCdR0anD2e21vdvgCI8lIfqFbnoeOz7lBjdrAGUhqLZLcQo6MLhTO2DKQ==} + '@esbuild/android-arm@0.27.7': + resolution: {integrity: sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==} engines: {node: '>=18'} cpu: [arm] os: [android] - '@esbuild/android-x64@0.27.4': - resolution: {integrity: sha512-PzPFnBNVF292sfpfhiyiXCGSn9HZg5BcAz+ivBuSsl6Rk4ga1oEXAamhOXRFyMcjwr2DVtm40G65N3GLeH1Lvw==} + '@esbuild/android-x64@0.27.7': + resolution: {integrity: sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==} engines: {node: '>=18'} cpu: [x64] os: [android] - '@esbuild/darwin-arm64@0.27.4': - resolution: {integrity: sha512-b7xaGIwdJlht8ZFCvMkpDN6uiSmnxxK56N2GDTMYPr2/gzvfdQN8rTfBsvVKmIVY/X7EM+/hJKEIbbHs9oA4tQ==} + '@esbuild/darwin-arm64@0.27.7': + resolution: {integrity: sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==} engines: {node: '>=18'} cpu: [arm64] os: [darwin] - '@esbuild/darwin-x64@0.27.4': - resolution: {integrity: sha512-sR+OiKLwd15nmCdqpXMnuJ9W2kpy0KigzqScqHI3Hqwr7IXxBp3Yva+yJwoqh7rE8V77tdoheRYataNKL4QrPw==} + '@esbuild/darwin-x64@0.27.7': + resolution: {integrity: sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==} engines: {node: '>=18'} cpu: [x64] os: [darwin] - '@esbuild/freebsd-arm64@0.27.4': - resolution: {integrity: sha512-jnfpKe+p79tCnm4GVav68A7tUFeKQwQyLgESwEAUzyxk/TJr4QdGog9sqWNcUbr/bZt/O/HXouspuQDd9JxFSw==} + '@esbuild/freebsd-arm64@0.27.7': + resolution: {integrity: sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==} engines: {node: '>=18'} cpu: [arm64] os: [freebsd] - '@esbuild/freebsd-x64@0.27.4': - resolution: {integrity: sha512-2kb4ceA/CpfUrIcTUl1wrP/9ad9Atrp5J94Lq69w7UwOMolPIGrfLSvAKJp0RTvkPPyn6CIWrNy13kyLikZRZQ==} + '@esbuild/freebsd-x64@0.27.7': + resolution: {integrity: sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==} engines: {node: '>=18'} cpu: [x64] os: [freebsd] - '@esbuild/linux-arm64@0.27.4': - resolution: {integrity: sha512-7nQOttdzVGth1iz57kxg9uCz57dxQLHWxopL6mYuYthohPKEK0vU0C3O21CcBK6KDlkYVcnDXY099HcCDXd9dA==} + '@esbuild/linux-arm64@0.27.7': + resolution: {integrity: sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==} engines: {node: '>=18'} cpu: [arm64] os: [linux] - '@esbuild/linux-arm@0.27.4': - resolution: {integrity: sha512-aBYgcIxX/wd5n2ys0yESGeYMGF+pv6g0DhZr3G1ZG4jMfruU9Tl1i2Z+Wnj9/KjGz1lTLCcorqE2viePZqj4Eg==} + '@esbuild/linux-arm@0.27.7': + resolution: {integrity: sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==} engines: {node: '>=18'} cpu: [arm] os: [linux] - '@esbuild/linux-ia32@0.27.4': - resolution: {integrity: sha512-oPtixtAIzgvzYcKBQM/qZ3R+9TEUd1aNJQu0HhGyqtx6oS7qTpvjheIWBbes4+qu1bNlo2V4cbkISr8q6gRBFA==} + '@esbuild/linux-ia32@0.27.7': + resolution: {integrity: sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==} engines: {node: '>=18'} cpu: [ia32] os: [linux] - '@esbuild/linux-loong64@0.27.4': - resolution: {integrity: sha512-8mL/vh8qeCoRcFH2nM8wm5uJP+ZcVYGGayMavi8GmRJjuI3g1v6Z7Ni0JJKAJW+m0EtUuARb6Lmp4hMjzCBWzA==} + '@esbuild/linux-loong64@0.27.7': + resolution: {integrity: sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==} engines: {node: '>=18'} cpu: [loong64] os: [linux] - '@esbuild/linux-mips64el@0.27.4': - resolution: {integrity: sha512-1RdrWFFiiLIW7LQq9Q2NES+HiD4NyT8Itj9AUeCl0IVCA459WnPhREKgwrpaIfTOe+/2rdntisegiPWn/r/aAw==} + '@esbuild/linux-mips64el@0.27.7': + resolution: {integrity: sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==} engines: {node: '>=18'} cpu: [mips64el] os: [linux] - '@esbuild/linux-ppc64@0.27.4': - resolution: {integrity: sha512-tLCwNG47l3sd9lpfyx9LAGEGItCUeRCWeAx6x2Jmbav65nAwoPXfewtAdtbtit/pJFLUWOhpv0FpS6GQAmPrHA==} + '@esbuild/linux-ppc64@0.27.7': + resolution: {integrity: sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==} engines: {node: '>=18'} cpu: [ppc64] os: [linux] - '@esbuild/linux-riscv64@0.27.4': - resolution: {integrity: sha512-BnASypppbUWyqjd1KIpU4AUBiIhVr6YlHx/cnPgqEkNoVOhHg+YiSVxM1RLfiy4t9cAulbRGTNCKOcqHrEQLIw==} + '@esbuild/linux-riscv64@0.27.7': + resolution: {integrity: sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==} engines: {node: '>=18'} cpu: [riscv64] os: [linux] - '@esbuild/linux-s390x@0.27.4': - resolution: {integrity: sha512-+eUqgb/Z7vxVLezG8bVB9SfBie89gMueS+I0xYh2tJdw3vqA/0ImZJ2ROeWwVJN59ihBeZ7Tu92dF/5dy5FttA==} + '@esbuild/linux-s390x@0.27.7': + resolution: {integrity: sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==} engines: {node: '>=18'} cpu: [s390x] os: [linux] - '@esbuild/linux-x64@0.27.4': - resolution: {integrity: sha512-S5qOXrKV8BQEzJPVxAwnryi2+Iq5pB40gTEIT69BQONqR7JH1EPIcQ/Uiv9mCnn05jff9umq/5nqzxlqTOg9NA==} + '@esbuild/linux-x64@0.27.7': + resolution: {integrity: sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==} engines: {node: '>=18'} cpu: [x64] os: [linux] - '@esbuild/netbsd-arm64@0.27.4': - resolution: {integrity: sha512-xHT8X4sb0GS8qTqiwzHqpY00C95DPAq7nAwX35Ie/s+LO9830hrMd3oX0ZMKLvy7vsonee73x0lmcdOVXFzd6Q==} + '@esbuild/netbsd-arm64@0.27.7': + resolution: {integrity: sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==} engines: {node: '>=18'} cpu: [arm64] os: [netbsd] - '@esbuild/netbsd-x64@0.27.4': - resolution: {integrity: sha512-RugOvOdXfdyi5Tyv40kgQnI0byv66BFgAqjdgtAKqHoZTbTF2QqfQrFwa7cHEORJf6X2ht+l9ABLMP0dnKYsgg==} + '@esbuild/netbsd-x64@0.27.7': + resolution: {integrity: sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==} engines: {node: '>=18'} cpu: [x64] os: [netbsd] - '@esbuild/openbsd-arm64@0.27.4': - resolution: {integrity: sha512-2MyL3IAaTX+1/qP0O1SwskwcwCoOI4kV2IBX1xYnDDqthmq5ArrW94qSIKCAuRraMgPOmG0RDTA74mzYNQA9ow==} + '@esbuild/openbsd-arm64@0.27.7': + resolution: {integrity: sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==} engines: {node: '>=18'} cpu: [arm64] os: [openbsd] - '@esbuild/openbsd-x64@0.27.4': - resolution: {integrity: sha512-u8fg/jQ5aQDfsnIV6+KwLOf1CmJnfu1ShpwqdwC0uA7ZPwFws55Ngc12vBdeUdnuWoQYx/SOQLGDcdlfXhYmXQ==} + '@esbuild/openbsd-x64@0.27.7': + resolution: {integrity: sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==} engines: {node: '>=18'} cpu: [x64] os: [openbsd] - '@esbuild/openharmony-arm64@0.27.4': - resolution: {integrity: sha512-JkTZrl6VbyO8lDQO3yv26nNr2RM2yZzNrNHEsj9bm6dOwwu9OYN28CjzZkH57bh4w0I2F7IodpQvUAEd1mbWXg==} + '@esbuild/openharmony-arm64@0.27.7': + resolution: {integrity: sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==} engines: {node: '>=18'} cpu: [arm64] os: [openharmony] - '@esbuild/sunos-x64@0.27.4': - resolution: {integrity: sha512-/gOzgaewZJfeJTlsWhvUEmUG4tWEY2Spp5M20INYRg2ZKl9QPO3QEEgPeRtLjEWSW8FilRNacPOg8R1uaYkA6g==} + '@esbuild/sunos-x64@0.27.7': + resolution: {integrity: sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==} engines: {node: '>=18'} cpu: [x64] os: [sunos] - '@esbuild/win32-arm64@0.27.4': - resolution: {integrity: sha512-Z9SExBg2y32smoDQdf1HRwHRt6vAHLXcxD2uGgO/v2jK7Y718Ix4ndsbNMU/+1Qiem9OiOdaqitioZwxivhXYg==} + '@esbuild/win32-arm64@0.27.7': + resolution: {integrity: sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==} engines: {node: '>=18'} cpu: [arm64] os: [win32] - '@esbuild/win32-ia32@0.27.4': - resolution: {integrity: sha512-DAyGLS0Jz5G5iixEbMHi5KdiApqHBWMGzTtMiJ72ZOLhbu/bzxgAe8Ue8CTS3n3HbIUHQz/L51yMdGMeoxXNJw==} + '@esbuild/win32-ia32@0.27.7': + resolution: {integrity: sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==} engines: {node: '>=18'} cpu: [ia32] os: [win32] - '@esbuild/win32-x64@0.27.4': - resolution: {integrity: sha512-+knoa0BDoeXgkNvvV1vvbZX4+hizelrkwmGJBdT17t8FNPwG2lKemmuMZlmaNQ3ws3DKKCxpb4zRZEIp3UxFCg==} + '@esbuild/win32-x64@0.27.7': + resolution: {integrity: sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==} engines: {node: '>=18'} cpu: [x64] os: [win32] @@ -311,8 +311,8 @@ packages: '@fastify/rate-limit@9.1.0': resolution: {integrity: sha512-h5dZWCkuZXN0PxwqaFQLxeln8/LNwQwH9popywmDCFdKfgpi4b/HoMH1lluy6P+30CG9yzzpSpwTCIPNB9T1JA==} - '@ioredis/commands@1.5.0': - resolution: {integrity: sha512-eUgLqrMf8nJkZxT24JvVRrQya1vZkQh8BBeYNwGDqa5I0VUi8ACx7uFvAaLxintokpTenkK6DASvo/bvNbBGow==} + '@ioredis/commands@1.5.1': + resolution: {integrity: sha512-JH8ZL/ywcJyR9MmJ5BNqZllXNZQqQbnVZOqpPQqE1vHiFgAw4NHbvE0FOduNU8IX9babitBT46571OnPTT0Zcw==} '@isaacs/cliui@9.0.0': resolution: {integrity: sha512-AokJm4tuBHillT+FpMtxQ60n8ObyXBatq7jD2/JA9dxbDDokKQm8KMht5ibGzLVU9IJDIKK4TPKgMHEYMn3lMg==} @@ -355,11 +355,11 @@ packages: '@pinojs/redact@0.4.0': resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} - '@types/node@20.19.37': - resolution: {integrity: sha512-8kzdPJ3FsNsVIurqBs7oodNnCEVbni9yUEkaHbgptDACOPW04jimGagZ51E6+lXUwJjgnBw+hyko/lkFWCldqw==} + '@types/node@20.19.39': + resolution: {integrity: sha512-orrrD74MBUyK8jOAD/r0+lfa1I2MO6I+vAkmAWzMYbCcgrN4lCrmK52gRFQq/JRxfYPfonkr4b0jcY7Olqdqbw==} - '@types/pg@8.18.0': - resolution: {integrity: sha512-gT+oueVQkqnj6ajGJXblFR4iavIXWsGAFCk3dP4Kki5+a9R4NMt0JARdk6s8cUKcfUoqP5dAtDSLU8xYUTFV+Q==} + '@types/pg@8.20.0': + resolution: {integrity: sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow==} '@typespec/ts-http-runtime@0.3.4': resolution: {integrity: sha512-CI0NhTrz4EBaa0U+HaaUZrJhPoso8sG7ZFya8uQoBA57fjzrjRSv87ekCjLZOFExN+gXE/z0xuN2QfH4H2HrLQ==} @@ -409,8 +409,8 @@ packages: base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} - brace-expansion@5.0.4: - resolution: {integrity: sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==} + brace-expansion@5.0.5: + resolution: {integrity: sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==} engines: {node: 18 || 20 || >=22} buffer-equal-constant-time@1.0.1: @@ -419,8 +419,8 @@ packages: buffer@6.0.3: resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} - bullmq@5.71.0: - resolution: {integrity: sha512-aeNWh4drsafSKnAJeiNH/nZP/5O8ZdtdMbnOPZmpjXj7NZUP5YC901U3bIH41iZValm7d1i3c34ojv7q31m30w==} + bullmq@5.73.0: + resolution: {integrity: sha512-uX8RbQaBbzk0H9JYXKGrNxpDqFcDBQFFKCyKarMjtfYHuct5X48M2LUq3Q9FXt/P2kWzPrqYlNnNqsico7ty5A==} bundle-name@4.1.0: resolution: {integrity: sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==} @@ -474,8 +474,8 @@ packages: ecdsa-sig-formatter@1.0.11: resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} - esbuild@0.27.4: - resolution: {integrity: sha512-Rq4vbHnYkK5fws5NF7MYTU68FPRE1ajX7heQ/8QXXWqNgqqJ/GkmmyxIzUnf2Sr/bakf8l54716CcMGHYhMrrQ==} + esbuild@0.27.7: + resolution: {integrity: sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==} engines: {node: '>=18'} hasBin: true @@ -515,8 +515,8 @@ packages: fast-xml-builder@1.1.4: resolution: {integrity: sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg==} - fast-xml-parser@5.5.6: - resolution: {integrity: sha512-3+fdZyBRVg29n4rXP0joHthhcHdPUHaIC16cuyyd1iLsuaO6Vea36MPrxgAzbZna8lhvZeRL8Bc9GP56/J9xEw==} + fast-xml-parser@5.5.10: + resolution: {integrity: sha512-go2J2xODMc32hT+4Xr/bBGXMaIoiCwrwp2mMtAvKyvEFW6S/v5Gn2pBmE4nvbwNjGhpcAiOwEv7R6/GZ6XRa9w==} hasBin: true fastify-plugin@4.5.1: @@ -545,8 +545,8 @@ packages: engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] - get-tsconfig@4.13.6: - resolution: {integrity: sha512-shZT/QMiSHc/YBLxxOkMtgSid5HFoauqCE3/exfsEcwg1WkeqjG+V40yBbBrsD+jW2HDXcs28xOfcbm2jI8Ddw==} + get-tsconfig@4.13.7: + resolution: {integrity: sha512-7tN6rFgBlMgpBML5j8typ92BKFi2sFQvIdpAqLA2beia5avZDrMs0FLZiM5etShWq5irVyGcGMEA1jcDaK7A/Q==} glob@11.1.0: resolution: {integrity: sha512-vuNwKSaKiqm7g0THUBu2x7ckSs3XJLXE+2ssL7/MfTGPLLcrJQ/4Uq1CjPTtO5cCIiRxqvN6Twy1qOwhL0Xjcw==} @@ -569,8 +569,8 @@ packages: ieee754@1.2.1: resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} - ioredis@5.9.3: - resolution: {integrity: sha512-VI5tMCdeoxZWU5vjHWsiE/Su76JGhBvWF1MJnV9ZtGltHk9BmD48oDq8Tj8haZ85aceXZMxLNDQZRVo5QKNgXA==} + ioredis@5.10.1: + resolution: {integrity: sha512-HuEDBTI70aYdx1v6U97SbNx9F1+svQKBDo30o0b9fw055LMepzpOOd0Ccg9Q6tbqmBSJaMuY0fB7yw9/vjBYCA==} engines: {node: '>=12.22.0'} ipaddr.js@1.9.1: @@ -652,8 +652,8 @@ packages: resolution: {integrity: sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==} engines: {node: '>=12'} - minimatch@10.2.4: - resolution: {integrity: sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==} + minimatch@10.2.5: + resolution: {integrity: sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==} engines: {node: 18 || 20 || >=22} minipass@7.1.3: @@ -694,8 +694,8 @@ packages: package-json-from-dist@1.0.1: resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} - path-expression-matcher@1.1.3: - resolution: {integrity: sha512-qdVgY8KXmVdJZRSS1JdEPOKPdTiEK/pi0RkcT2sw1RhXxohdujUlJFPuS1TSkevZ9vzd3ZlL7ULl1MHGTApKzQ==} + path-expression-matcher@1.2.1: + resolution: {integrity: sha512-d7gQQmLvAKXKXE2GeP9apIGbMYKz88zWdsn/BN2HRWVQsDFdUY36WSLTY0Jvd4HWi7Fb30gQ62oAOzdgJA6fZw==} engines: {node: '>=14.0.0'} path-key@3.1.1: @@ -885,8 +885,8 @@ packages: string_decoder@1.3.0: resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} - strnum@2.2.0: - resolution: {integrity: sha512-Y7Bj8XyJxnPAORMZj/xltsfo55uOiyHcU2tnAVzHUnSJR/KsEX+9RoDeXEnsXtl/CX4fAcrt64gZ13aGaWPeBg==} + strnum@2.2.2: + resolution: {integrity: sha512-DnR90I+jtXNSTXWdwrEy9FakW7UX+qUZg28gj5fk2vxxl7uS/3bpI4fjFYVmdK9etptYBPNkpahuQnEwhwECqA==} thread-stream@2.7.0: resolution: {integrity: sha512-qQiRWsU/wvNolI6tbbCKd9iKaTnCXsTwVxhhKM6nctPdujTyztjlbUkUTUymidWcMnZ5pWR0ej4a0tjsW021vw==} @@ -1013,7 +1013,7 @@ snapshots: '@azure/core-xml@1.5.0': dependencies: - fast-xml-parser: 5.5.6 + fast-xml-parser: 5.5.10 tslib: 2.8.1 '@azure/identity@4.13.1': @@ -1025,8 +1025,8 @@ snapshots: '@azure/core-tracing': 1.3.1 '@azure/core-util': 1.13.1 '@azure/logger': 1.3.0 - '@azure/msal-browser': 5.6.1 - '@azure/msal-node': 5.1.1 + '@azure/msal-browser': 5.6.3 + '@azure/msal-node': 5.1.2 open: 10.2.0 tslib: 2.8.1 transitivePeerDependencies: @@ -1039,15 +1039,15 @@ snapshots: transitivePeerDependencies: - supports-color - '@azure/msal-browser@5.6.1': + '@azure/msal-browser@5.6.3': dependencies: - '@azure/msal-common': 16.4.0 + '@azure/msal-common': 16.4.1 - '@azure/msal-common@16.4.0': {} + '@azure/msal-common@16.4.1': {} - '@azure/msal-node@5.1.1': + '@azure/msal-node@5.1.2': dependencies: - '@azure/msal-common': 16.4.0 + '@azure/msal-common': 16.4.1 jsonwebtoken: 9.0.3 uuid: 8.3.2 @@ -1085,82 +1085,82 @@ snapshots: - '@azure/core-client' - supports-color - '@esbuild/aix-ppc64@0.27.4': + '@esbuild/aix-ppc64@0.27.7': optional: true - '@esbuild/android-arm64@0.27.4': + '@esbuild/android-arm64@0.27.7': optional: true - '@esbuild/android-arm@0.27.4': + '@esbuild/android-arm@0.27.7': optional: true - '@esbuild/android-x64@0.27.4': + '@esbuild/android-x64@0.27.7': optional: true - '@esbuild/darwin-arm64@0.27.4': + '@esbuild/darwin-arm64@0.27.7': optional: true - '@esbuild/darwin-x64@0.27.4': + '@esbuild/darwin-x64@0.27.7': optional: true - '@esbuild/freebsd-arm64@0.27.4': + '@esbuild/freebsd-arm64@0.27.7': optional: true - '@esbuild/freebsd-x64@0.27.4': + '@esbuild/freebsd-x64@0.27.7': optional: true - '@esbuild/linux-arm64@0.27.4': + '@esbuild/linux-arm64@0.27.7': optional: true - '@esbuild/linux-arm@0.27.4': + '@esbuild/linux-arm@0.27.7': optional: true - '@esbuild/linux-ia32@0.27.4': + '@esbuild/linux-ia32@0.27.7': optional: true - '@esbuild/linux-loong64@0.27.4': + '@esbuild/linux-loong64@0.27.7': optional: true - '@esbuild/linux-mips64el@0.27.4': + '@esbuild/linux-mips64el@0.27.7': optional: true - '@esbuild/linux-ppc64@0.27.4': + '@esbuild/linux-ppc64@0.27.7': optional: true - '@esbuild/linux-riscv64@0.27.4': + '@esbuild/linux-riscv64@0.27.7': optional: true - '@esbuild/linux-s390x@0.27.4': + '@esbuild/linux-s390x@0.27.7': optional: true - '@esbuild/linux-x64@0.27.4': + '@esbuild/linux-x64@0.27.7': optional: true - '@esbuild/netbsd-arm64@0.27.4': + '@esbuild/netbsd-arm64@0.27.7': optional: true - '@esbuild/netbsd-x64@0.27.4': + '@esbuild/netbsd-x64@0.27.7': optional: true - '@esbuild/openbsd-arm64@0.27.4': + '@esbuild/openbsd-arm64@0.27.7': optional: true - '@esbuild/openbsd-x64@0.27.4': + '@esbuild/openbsd-x64@0.27.7': optional: true - '@esbuild/openharmony-arm64@0.27.4': + '@esbuild/openharmony-arm64@0.27.7': optional: true - '@esbuild/sunos-x64@0.27.4': + '@esbuild/sunos-x64@0.27.7': optional: true - '@esbuild/win32-arm64@0.27.4': + '@esbuild/win32-arm64@0.27.7': optional: true - '@esbuild/win32-ia32@0.27.4': + '@esbuild/win32-ia32@0.27.7': optional: true - '@esbuild/win32-x64@0.27.4': + '@esbuild/win32-x64@0.27.7': optional: true '@fastify/ajv-compiler@3.6.0': @@ -1195,7 +1195,7 @@ snapshots: fastify-plugin: 4.5.1 toad-cache: 3.7.0 - '@ioredis/commands@1.5.0': {} + '@ioredis/commands@1.5.1': {} '@isaacs/cliui@9.0.0': {} @@ -1221,13 +1221,13 @@ snapshots: '@pinojs/redact@0.4.0': {} - '@types/node@20.19.37': + '@types/node@20.19.39': dependencies: undici-types: 6.21.0 - '@types/pg@8.18.0': + '@types/pg@8.20.0': dependencies: - '@types/node': 20.19.37 + '@types/node': 20.19.39 pg-protocol: 1.13.0 pg-types: 2.2.0 @@ -1273,7 +1273,7 @@ snapshots: base64-js@1.5.1: {} - brace-expansion@5.0.4: + brace-expansion@5.0.5: dependencies: balanced-match: 4.0.4 @@ -1284,10 +1284,10 @@ snapshots: base64-js: 1.5.1 ieee754: 1.2.1 - bullmq@5.71.0: + bullmq@5.73.0: dependencies: cron-parser: 4.9.0 - ioredis: 5.9.3 + ioredis: 5.10.1 msgpackr: 1.11.5 node-abort-controller: 3.1.1 semver: 7.7.4 @@ -1336,34 +1336,34 @@ snapshots: dependencies: safe-buffer: 5.2.1 - esbuild@0.27.4: + esbuild@0.27.7: optionalDependencies: - '@esbuild/aix-ppc64': 0.27.4 - '@esbuild/android-arm': 0.27.4 - '@esbuild/android-arm64': 0.27.4 - '@esbuild/android-x64': 0.27.4 - '@esbuild/darwin-arm64': 0.27.4 - '@esbuild/darwin-x64': 0.27.4 - '@esbuild/freebsd-arm64': 0.27.4 - '@esbuild/freebsd-x64': 0.27.4 - '@esbuild/linux-arm': 0.27.4 - '@esbuild/linux-arm64': 0.27.4 - '@esbuild/linux-ia32': 0.27.4 - '@esbuild/linux-loong64': 0.27.4 - '@esbuild/linux-mips64el': 0.27.4 - '@esbuild/linux-ppc64': 0.27.4 - '@esbuild/linux-riscv64': 0.27.4 - '@esbuild/linux-s390x': 0.27.4 - '@esbuild/linux-x64': 0.27.4 - '@esbuild/netbsd-arm64': 0.27.4 - '@esbuild/netbsd-x64': 0.27.4 - '@esbuild/openbsd-arm64': 0.27.4 - '@esbuild/openbsd-x64': 0.27.4 - '@esbuild/openharmony-arm64': 0.27.4 - '@esbuild/sunos-x64': 0.27.4 - '@esbuild/win32-arm64': 0.27.4 - '@esbuild/win32-ia32': 0.27.4 - '@esbuild/win32-x64': 0.27.4 + '@esbuild/aix-ppc64': 0.27.7 + '@esbuild/android-arm': 0.27.7 + '@esbuild/android-arm64': 0.27.7 + '@esbuild/android-x64': 0.27.7 + '@esbuild/darwin-arm64': 0.27.7 + '@esbuild/darwin-x64': 0.27.7 + '@esbuild/freebsd-arm64': 0.27.7 + '@esbuild/freebsd-x64': 0.27.7 + '@esbuild/linux-arm': 0.27.7 + '@esbuild/linux-arm64': 0.27.7 + '@esbuild/linux-ia32': 0.27.7 + '@esbuild/linux-loong64': 0.27.7 + '@esbuild/linux-mips64el': 0.27.7 + '@esbuild/linux-ppc64': 0.27.7 + '@esbuild/linux-riscv64': 0.27.7 + '@esbuild/linux-s390x': 0.27.7 + '@esbuild/linux-x64': 0.27.7 + '@esbuild/netbsd-arm64': 0.27.7 + '@esbuild/netbsd-x64': 0.27.7 + '@esbuild/openbsd-arm64': 0.27.7 + '@esbuild/openbsd-x64': 0.27.7 + '@esbuild/openharmony-arm64': 0.27.7 + '@esbuild/sunos-x64': 0.27.7 + '@esbuild/win32-arm64': 0.27.7 + '@esbuild/win32-ia32': 0.27.7 + '@esbuild/win32-x64': 0.27.7 event-target-shim@5.0.1: {} @@ -1397,13 +1397,13 @@ snapshots: fast-xml-builder@1.1.4: dependencies: - path-expression-matcher: 1.1.3 + path-expression-matcher: 1.2.1 - fast-xml-parser@5.5.6: + fast-xml-parser@5.5.10: dependencies: fast-xml-builder: 1.1.4 - path-expression-matcher: 1.1.3 - strnum: 2.2.0 + path-expression-matcher: 1.2.1 + strnum: 2.2.2 fastify-plugin@4.5.1: {} @@ -1446,7 +1446,7 @@ snapshots: fsevents@2.3.3: optional: true - get-tsconfig@4.13.6: + get-tsconfig@4.13.7: dependencies: resolve-pkg-maps: 1.0.0 @@ -1454,7 +1454,7 @@ snapshots: dependencies: foreground-child: 3.3.1 jackspeak: 4.2.3 - minimatch: 10.2.4 + minimatch: 10.2.5 minipass: 7.1.3 package-json-from-dist: 1.0.1 path-scurry: 2.0.2 @@ -1477,9 +1477,9 @@ snapshots: ieee754@1.2.1: {} - ioredis@5.9.3: + ioredis@5.10.1: dependencies: - '@ioredis/commands': 1.5.0 + '@ioredis/commands': 1.5.1 cluster-key-slot: 1.1.2 debug: 4.4.3 denque: 2.1.0 @@ -1567,9 +1567,9 @@ snapshots: luxon@3.7.2: {} - minimatch@10.2.4: + minimatch@10.2.5: dependencies: - brace-expansion: 5.0.4 + brace-expansion: 5.0.5 minipass@7.1.3: {} @@ -1615,7 +1615,7 @@ snapshots: package-json-from-dist@1.0.1: {} - path-expression-matcher@1.1.3: {} + path-expression-matcher@1.2.1: {} path-key@3.1.1: {} @@ -1791,7 +1791,7 @@ snapshots: dependencies: safe-buffer: 5.2.1 - strnum@2.2.0: {} + strnum@2.2.2: {} thread-stream@2.7.0: dependencies: @@ -1807,8 +1807,8 @@ snapshots: tsx@4.21.0: dependencies: - esbuild: 0.27.4 - get-tsconfig: 4.13.6 + esbuild: 0.27.7 + get-tsconfig: 4.13.7 optionalDependencies: fsevents: 2.3.3 diff --git a/src/infrastructure/ffmpeg/core/complexity.ts b/src/infrastructure/ffmpeg/core/complexity.ts new file mode 100644 index 0000000..f97f723 --- /dev/null +++ b/src/infrastructure/ffmpeg/core/complexity.ts @@ -0,0 +1,192 @@ +import { pino } from 'pino'; +import { spawn } from 'node:child_process'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { TranscodeError } from '../../../domain/errors.js'; + +const logger = pino({ name: 'PerTitleVMAF' }); + +export interface ComplexityResult { + multiplier: number; + sampleBitrate: number; +} + +async function runCommand(args: string[]): Promise { + return new Promise((resolve, reject) => { + const proc = spawn('ffmpeg', args); + let output = ''; + + proc.stderr.on('data', (data) => { + output += data.toString(); + }); + + proc.on('close', (code) => { + if (code !== 0) reject(new Error(`FFmpeg failed with code ${code}:\n${output}`)); + else resolve(output); + }); + }); +} + +/** + * Orchestrates a mathematically-driven Smart Per-Title Bitrate adaptation using Netflix's VMAF model. + * + * @remarks + * - Phase 1: Generates a near-lossless reference encode (CRF 10, ultrafast) to establish a pristine baseline. + * - Phase 2: Encodes empirical test points (CRF 19, 23, 27) and scores them against the reference using the VMAF neural network. + * - Phase 3: Interpolates a Rate-Distortion curve to find the exact bits-per-second required to hit a perceptive VMAF score of 95.0. + * - Bounding: Applies strict OTT resolution safety floors to prevent bitrate explosion on noisy/grainy source files. + */ +export async function probeComplexity( + sourceUrl: string, + duration: number, + videoId: string, + sourceWidth: number, + sourceHeight: number, +): Promise { + const workDir = `/tmp/worker/${videoId}/vmaf`; + await fs.mkdir(workDir, { recursive: true }); + + const refPath = path.join(workDir, 'ref.mkv'); + + logger.info({ videoId }, 'Phase 1: Generating 100% Whole-Timeline Reference (CFR Enforced)'); + + try { + await runCommand([ + '-i', + sourceUrl, + '-map', + '0:v:0', + '-an', + '-sn', + '-c:v', + 'libx264', + '-crf', + '10', + '-preset', + 'ultrafast', + '-fps_mode', + 'cfr', + '-pix_fmt', + 'yuv420p', + '-y', + refPath, + ]); + + logger.info({ videoId }, 'Phase 2: Encoding Full Grid & Running VMAF AI Analysis'); + + const crfs = [19, 23, 27]; + const results: { crf: number; vmaf: number; kbps: number }[] = []; + + for (const crf of crfs) { + const testPath = path.join(workDir, `test_${crf}.mkv`); + + await runCommand([ + '-i', + refPath, + '-c:v', + 'libx264', + '-crf', + String(crf), + '-preset', + 'superfast', + '-fps_mode', + 'cfr', + '-y', + testPath, + ]); + + const stat = await fs.stat(testPath); + const kbps = (stat.size * 8) / duration / 1000; + + const vmafOut = await runCommand([ + '-i', + testPath, + '-i', + refPath, + '-lavfi', + '[0:v]setpts=PTS-STARTPTS,scale=1920:1080:flags=bicubic[dist];[1:v]setpts=PTS-STARTPTS,scale=1920:1080:flags=bicubic[ref];[dist][ref]libvmaf=model=path=/usr/local/share/model/vmaf_v0.6.1.json:n_subsample=24:n_threads=4:pool=harmonic_mean', + '-f', + 'null', + '-', + ]); + + const match = vmafOut.match(/VMAF score: ([\d.]+)/); + const vmaf = match ? parseFloat(match[1]) : 0; + + results.push({ crf, vmaf, kbps }); + } + + logger.info( + { videoId, curve: results }, + 'Phase 3: Full-Timeline Rate-Distortion Curve mathematically mapped', + ); + + const TARGET_VMAF = 95.0; + results.sort((a, b) => a.vmaf - b.vmaf); + + let optimalKbps = 0; + + if (TARGET_VMAF <= results[0].vmaf) { + optimalKbps = results[0].kbps; + } else if (TARGET_VMAF >= results[results.length - 1].vmaf) { + optimalKbps = results[results.length - 1].kbps; + } else { + for (let i = 0; i < results.length - 1; i++) { + if (TARGET_VMAF >= results[i].vmaf && TARGET_VMAF <= results[i + 1].vmaf) { + const p1 = results[i]; + const p2 = results[i + 1]; + const slope = (p2.kbps - p1.kbps) / (p2.vmaf - p1.vmaf); + optimalKbps = p1.kbps + slope * (TARGET_VMAF - p1.vmaf); + break; + } + } + } + + const is4K = sourceWidth >= 3840 || sourceHeight >= 2160; + const is1080p = (sourceWidth >= 1920 || sourceHeight >= 1080) && !is4K; + + const BASELINE_KBPS = is4K ? 20000 : is1080p ? 6000 : 3000; + + if (!optimalKbps || isNaN(optimalKbps) || optimalKbps <= 0) { + logger.warn( + { videoId }, + 'VMAF calculation yielded invalid optimalKbps. Falling back to BASELINE.', + ); + optimalKbps = BASELINE_KBPS; + } + + let multiplier = optimalKbps / BASELINE_KBPS; + + const minFloor = is4K ? 0.85 : 0.6; + const maxCeil = is4K ? 1.5 : 2.0; + + multiplier = Math.max(minFloor, Math.min(multiplier, maxCeil)); + + logger.info( + { + videoId, + optimalKbps: Math.round(optimalKbps), + baselineKbps: BASELINE_KBPS, + finalMultiplier: multiplier, + is4K, + }, + 'Real Per-Title VMAF calculated with OTT Resolution Quality Floors applied', + ); + + await fs.rm(workDir, { recursive: true, force: true }).catch(() => {}); + + return { multiplier, sampleBitrate: Math.round(optimalKbps) }; + } catch (error) { + logger.error( + { videoId, err: error }, + 'Real VMAF probe failed. Strict mode active: Failing the pipeline.', + ); + + await fs.rm(workDir, { recursive: true, force: true }).catch(() => {}); + + throw new TranscodeError( + `VMAF Complexity Probe failed: ${error instanceof Error ? error.message : String(error)}`, + error instanceof Error ? error : undefined, + ); + } +} diff --git a/src/infrastructure/ffmpeg/encoding/profiles/audio.json b/src/infrastructure/ffmpeg/encoding/profiles/audio.json new file mode 100644 index 0000000..c152592 --- /dev/null +++ b/src/infrastructure/ffmpeg/encoding/profiles/audio.json @@ -0,0 +1,51 @@ +[ + { + "name": "aud_aac_he2_t1", + "channels": 2, + "codec": "libfdk_aac", + "profile": "aac_he_v2", + "bitrate": 32000, + "sampleRate": 44100, + "hardwareProfile": false + }, + { + "name": "aud_aac_lc_t2", + "channels": 2, + "codec": "aac", + "bitrate": 64000, + "sampleRate": 44100, + "hardwareProfile": false + }, + { + "name": "aud_aac_lc_t3", + "channels": 2, + "codec": "aac", + "bitrate": 128000, + "sampleRate": 44100, + "hardwareProfile": false + }, + { + "name": "aud_aac_lc_t4", + "channels": 2, + "codec": "aac", + "bitrate": 160000, + "sampleRate": 48000, + "hardwareProfile": false + }, + { + "name": "aud_eac3_51_t1", + "channels": 6, + "codec": "eac3", + "bitrate": 448000, + "sampleRate": 48000, + "hardwareProfile": true + }, + { + "name": "aud_ac3_51_t1", + "channels": 6, + "codec": "ac3", + "bitrate": 384000, + "sampleRate": 48000, + "hardwareProfile": true + } +] diff --git a/src/infrastructure/ffmpeg/encoding/profiles/video.json b/src/infrastructure/ffmpeg/encoding/profiles/video.json new file mode 100644 index 0000000..42fd7e2 --- /dev/null +++ b/src/infrastructure/ffmpeg/encoding/profiles/video.json @@ -0,0 +1,338 @@ +{ + "h264_sdr": [ + { + "name": "vod_avc_sdr_t1", + "width": 416, + "height": 234, + "bitrate": 130000, + "maxrate": 145000, + "bufsize": 290000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "preset": "slow", + "profile": "baseline", + "level": "3.0", + "videoCodecTag": "avc1.42E01E" + }, + { + "name": "vod_avc_sdr_t2", + "width": 480, + "height": 270, + "bitrate": 365000, + "maxrate": 400000, + "bufsize": 800000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "preset": "slow", + "profile": "baseline", + "level": "3.0", + "videoCodecTag": "avc1.42E01E" + }, + { + "name": "vod_avc_sdr_t3", + "width": 640, + "height": 360, + "bitrate": 730000, + "maxrate": 800000, + "bufsize": 1600000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "preset": "slow", + "profile": "main", + "level": "3.0", + "videoCodecTag": "avc1.4D401E" + }, + { + "name": "vod_avc_sdr_t4", + "width": 960, + "height": 540, + "bitrate": 2000000, + "maxrate": 2200000, + "bufsize": 4400000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "preset": "slow", + "profile": "high", + "level": "3.1", + "videoCodecTag": "avc1.64001F" + }, + { + "name": "vod_avc_sdr_t5", + "width": 1280, + "height": 720, + "bitrate": 3000000, + "maxrate": 3300000, + "bufsize": 6000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "preset": "slow", + "profile": "high", + "level": "3.1", + "videoCodecTag": "avc1.64001F" + }, + { + "name": "vod_avc_sdr_t6", + "width": 1920, + "height": 1080, + "bitrate": 6000000, + "maxrate": 6600000, + "bufsize": 12000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx264", + "preset": "slow", + "profile": "high", + "level": "4.0", + "videoCodecTag": "avc1.640028" + } + ], + "h265_sdr": [ + { + "name": "vod_hevc_sdr_t1", + "width": 416, + "height": 234, + "bitrate": 130000, + "maxrate": 145000, + "bufsize": 290000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main", + "level": "3.1", + "videoCodecTag": "hvc1.1.4.L93.B0" + }, + { + "name": "vod_hevc_sdr_t2", + "width": 480, + "height": 270, + "bitrate": 250000, + "maxrate": 280000, + "bufsize": 500000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main", + "level": "3.1", + "videoCodecTag": "hvc1.1.4.L93.B0" + }, + { + "name": "vod_hevc_sdr_t3", + "width": 640, + "height": 360, + "bitrate": 575000, + "maxrate": 630000, + "bufsize": 1200000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main", + "level": "3.1", + "videoCodecTag": "hvc1.1.4.L93.B0" + }, + { + "name": "vod_hevc_sdr_t4", + "width": 960, + "height": 540, + "bitrate": 1100000, + "maxrate": 1200000, + "bufsize": 2400000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main", + "level": "3.1", + "videoCodecTag": "hvc1.1.4.L93.B0" + }, + { + "name": "vod_hevc_sdr_t5", + "width": 1280, + "height": 720, + "bitrate": 2100000, + "maxrate": 2300000, + "bufsize": 4200000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main", + "level": "4.0", + "videoCodecTag": "hvc1.1.4.L120.B0" + }, + { + "name": "vod_hevc_sdr_t6", + "width": 1920, + "height": 1080, + "bitrate": 6800000, + "maxrate": 7200000, + "bufsize": 12000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main", + "level": "4.0", + "videoCodecTag": "hvc1.1.4.L120.B0" + }, + { + "name": "vod_hevc_sdr_t7", + "width": 2560, + "height": 1440, + "bitrate": 11600000, + "maxrate": 12500000, + "bufsize": 20000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main", + "level": "5.0", + "videoCodecTag": "hvc1.1.4.L150.B0" + }, + { + "name": "vod_hevc_sdr_t8", + "width": 3840, + "height": 2160, + "bitrate": 20000000, + "maxrate": 22000000, + "bufsize": 30000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main", + "level": "5.0", + "videoCodecTag": "hvc1.1.4.L150.B0" + } + ], + "h265_hdr": [ + { + "name": "vod_hevc_hdr_t1", + "width": 416, + "height": 234, + "bitrate": 130000, + "maxrate": 145000, + "bufsize": 290000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main10", + "level": "3.1", + "videoCodecTag": "hvc1.2.4.L93.B0" + }, + { + "name": "vod_hevc_hdr_t2", + "width": 480, + "height": 270, + "bitrate": 300000, + "maxrate": 330000, + "bufsize": 600000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main10", + "level": "3.1", + "videoCodecTag": "hvc1.2.4.L93.B0" + }, + { + "name": "vod_hevc_hdr_t3", + "width": 640, + "height": 360, + "bitrate": 690000, + "maxrate": 750000, + "bufsize": 1300000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main10", + "level": "3.1", + "videoCodecTag": "hvc1.2.4.L93.B0" + }, + { + "name": "vod_hevc_hdr_t4", + "width": 960, + "height": 540, + "bitrate": 1300000, + "maxrate": 1450000, + "bufsize": 2600000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main10", + "level": "3.1", + "videoCodecTag": "hvc1.2.4.L93.B0" + }, + { + "name": "vod_hevc_hdr_t5", + "width": 1280, + "height": 720, + "bitrate": 2500000, + "maxrate": 2700000, + "bufsize": 5000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main10", + "level": "4.0", + "videoCodecTag": "hvc1.2.4.L120.B0" + }, + { + "name": "vod_hevc_hdr_t6", + "width": 1920, + "height": 1080, + "bitrate": 8100000, + "maxrate": 8800000, + "bufsize": 14000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main10", + "level": "4.0", + "videoCodecTag": "hvc1.2.4.L120.B0" + }, + { + "name": "vod_hevc_hdr_t7", + "width": 2560, + "height": 1440, + "bitrate": 14000000, + "maxrate": 15000000, + "bufsize": 22000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main10", + "level": "5.0", + "videoCodecTag": "hvc1.2.4.L150.B0" + }, + { + "name": "vod_hevc_hdr_t8", + "width": 3840, + "height": 2160, + "bitrate": 24000000, + "maxrate": 26000000, + "bufsize": 32000000, + "frameRate": 30, + "hlsTime": 6, + "videoCodec": "libx265", + "preset": "slow", + "profile": "main10", + "level": "5.0", + "videoCodecTag": "hvc1.2.4.L150.B0" + } + ] +}