dunedaq-develop-cpp-ci.yml

 
name: build-develop-datafilter-alma9

on:
  push:
    branches: 
      - develop
      - patch/*
      - prep-release/*
    paths-ignore:
      - 'docs/**'
      - '.github/**'
  pull_request:
    branches:
      - develop
      - patch/*
      - prep-release/*
  schedule:
    - cron: "08 22 * * *"
  workflow_dispatch:
    inputs:
      test_duration:
        description: 'Duration for continuous testing (in minutes)'
        required: false
        default: '60'
      test_interval:
        description: 'Interval between test runs (in seconds)'
        required: false
        default: '300'
      enable_continuous_test:
        description: 'Enable continuous testing'
        required: false
        type: boolean
        default: false

jobs:
  build_against_dev_release:
    name: Build DUNEDAQ DataFilter V5 on Alma9
    runs-on: ubuntu-latest
    timeout-minutes: 480
    strategy:
      matrix:
        include:
          - image: "almalinux:9"
            os_name: "alma9"
            link: "last_successful_alma9"
    container:
      image: ${{ matrix.image }}
      options: --privileged
    defaults:
      run:
        shell: bash
    
    steps:
      - name: Install CVMFS and dependencies
        run: |
          dnf install -y epel-release
          dnf install -y \
            https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest.noarch.rpm
          dnf install -y cvmfs cvmfs-config-default
          dnf install -y --allowerasing git wget curl cmake gcc-c++ make python3 python3-pip openssl-devel libunwind-devel
          
          # Setup CVMFS
          mkdir -p /etc/cvmfs
          cat > /etc/cvmfs/default.local <<EOF
          CVMFS_REPOSITORIES=dunedaq.opensciencegrid.org,dunedaq-development.opensciencegrid.org
          CVMFS_HTTP_PROXY=DIRECT
          CVMFS_CLIENT_PROFILE=single
          CVMFS_QUOTA_LIMIT=10000
          EOF
          
          # Mount CVMFS
          mkdir -p /cvmfs/dunedaq.opensciencegrid.org
          mkdir -p /cvmfs/dunedaq-development.opensciencegrid.org
          mount -t cvmfs dunedaq.opensciencegrid.org /cvmfs/dunedaq.opensciencegrid.org || true
          mount -t cvmfs dunedaq-development.opensciencegrid.org /cvmfs/dunedaq-development.opensciencegrid.org || true
          
          # Verify CVMFS is mounted
          echo "CVMFS mounted FS"
          ls -la /cvmfs/dunedaq.opensciencegrid.org/
          ls /cvmfs/dunedaq-development.opensciencegrid.org/
      
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          path: ${{ github.repository }}
      
      - name: Setup dev area
        run: |
          #source /cvmfs/dunedaq.opensciencegrid.org/tools/dbt/latest/env.sh           
          source `realpath /cvmfs/dunedaq.opensciencegrid.org/spack-externals/spack-installation/share/spack/setup-env.sh`
          source /cvmfs/dunedaq.opensciencegrid.org/setup_dunedaq.sh
          setup_dbt latest
          echo "=> pwd"
          pwd 
          echo "=>ls dunedaq-development"
          ls /cvmfs/dunedaq-development.opensciencegrid.org/
          # need to remove it first
          rm -fr test-area
          dbt-create -b candidate fddaq-v5.3.2-rc3-a9 ./test-area
          echo "ls test-area"
          ls test-area
          echo "pwd"
          pwd
          echo "ls"
          ls 
          #echo "find"
          #find .
          cd test-area
          source env.sh
          echo "env"
          env
          dbt-workarea-env || true  
          echo "=>repos pwd"
          pwd
          cd sourcecode
          echo "=> ls"
          ls
          git clone https://github.com/DUNE-DAQ/daqsystemtest.git
          git clone https://github.com/DUNE-DAQ/fddaqconf.git 
          git clone https://github.com/DUNE-DAQ/hdf5libs.git 
          git clone https://github.com/DUNE-DAQ/detdataformats.git
          git clone https://github.com/DUNE-DAQ/fddetdataformats.git
          git clone https://github.com/DUNE-DAQ/datafilter.git 
          git clone https://github.com/DUNE-DAQ/appfwk.git
          cd fddaqconf
          git checkout 0575bb1ade1632c2679f4c33ca499a1dbf8175a1
          cd ..
          
          cd appfwk
          git checkout 1f8ce77ccdf23b8e068b8096218cfb26dd87d80e
          cd ..
          
          cd hdf5libs
          git checkout e1dc1f0b19bd64dbae5a36f5f525d70d4de4f6cb
          cd ..
          
          cd fddetdataformats
          git checkout 240f71fe8391a7d04db048f111e182ee8313de34
          cd ..
          
          echo "from Build"
          echo "=>pwd"
          pwd
          
          dbt-workarea-env
          dbt-build -j$(nproc)
        continue-on-error: true
      
      - name: Run tests (optional)
        run: |
          cd /__w/datafilter/datafilter/test-area
          source env.sh
          dbt-workarea-env
          
          # Extract paths from OKS config
          CONFIG_XML="${PWD}/sourcecode/datafilter/test/config/dfSession.data.xml"
          
          # Replace output directory and input directory with work area using sed
          WORK_AREA="${PWD}"
          sed -i "s|/lcg/storage18/dune/chen|${WORK_AREA}/output|g" "${CONFIG_XML}"
          sed -i "s|/lcg/storage19/test-area/dune/trigger_records/sourcehdf5|${WORK_AREA}/input|g" "${CONFIG_XML}"
          echo "Updated config - odir now points to: ${WORK_AREA}/output"
          echo "Updated config - storage_pathname now points to: ${WORK_AREA}/input"
          
          # Get output directory (odir from FilterResultWriter) - search within 10 lines after class match
          OUTPUT_DIR=$(grep -A10 'obj class="FilterResultWriter"' "${CONFIG_XML}" | grep 'name="odir"' | grep -oP 'val="\K[^"]+')
          echo "Output directory from config: ${OUTPUT_DIR}"
          
          # Get input directory (storage_pathname from TRDispatcher) - search within 10 lines after class match
          INPUT_DIR=$(grep -A10 'obj class="TRDispatcher"' "${CONFIG_XML}" | grep 'name="storage_pathname"' | grep -oP 'val="\K[^"]+')
          echo "Input directory from config: ${INPUT_DIR}"
          
          # Create directories
          mkdir -p "${OUTPUT_DIR}"
          mkdir -p "${INPUT_DIR}"
          
          # Set OKS config path
          OKS_CONFIG="${CONFIG_XML}"
          
          # Function to cleanup processes
          cleanup() {
            echo "Cleaning up processes..."
            pkill -TERM -f "trdispatcher|filterorchestrator|datafilter2|filterresultwriter" || true
            sleep 2
            pkill -KILL -f "trdispatcher|filterorchestrator|datafilter2|filterresultwriter" || true
          }
          trap cleanup EXIT
          
          # Start the apps in background from test/apps directory
          APPS_DIR="${PWD}/sourcecode/datafilter/test/apps"
          cd "${APPS_DIR}"
          echo "Running apps from: ${APPS_DIR}"
          
          echo "Starting TRDispatcher..."
          trdispatcher  > trd.log 2>&1 &
          TRD_PID=$!
          
          echo "Starting FilterOrchestrator..."
          filterorchestrator  > fo.log 2>&1 &
          FO_PID=$!
          
          echo "Starting DataFilter..."
          datafilter2  > df.log 2>&1 &
          DF_PID=$!
          
          echo "Starting FilterResultWriter..."
          filterresultwriter  > frw.log 2>&1 &
          FRW_PID=$!
          
          echo "All apps started. PIDs: TRD=$TRD_PID FO=$FO_PID DF=$DF_PID FRW=$FRW_PID"
          
          # Wait for hdf5 files in output directory (up to 5 minutes)
          MAX_WAIT=300
          INTERVAL=10
          ELAPSED=0
          
          while [ $ELAPSED -lt $MAX_WAIT ]; do
            H5_COUNT=$(find "${OUTPUT_DIR}" -name "*.hdf5" 2>/dev/null | wc -l)
            echo "Elapsed: ${ELAPSED}s, HDF5 files found: ${H5_COUNT}"
            
            if [ ${H5_COUNT} -ge 5 ]; then
              echo "SUCCESS: Found ${H5_COUNT} hdf5 files (>=5 required)"
              find "${OUTPUT_DIR}" -name "*.hdf5" 2>/dev/null
              exit 0
            fi
            
            sleep ${INTERVAL}
            ELAPSED=$((ELAPSED + INTERVAL))
          done
          
          echo "TIMEOUT: Only found ${H5_COUNT} hdf5 files after ${MAX_WAIT}s"
          echo "=== TRDispatcher log ==="
          cat "${APPS_DIR}/trd.log" 2>/dev/null || true
          echo "=== FilterOrchestrator log ==="
          cat "${APPS_DIR}/fo.log" 2>/dev/null || true
          echo "=== DataFilter log ==="
          cat "${APPS_DIR}/df.log" 2>/dev/null || true
          echo "=== FilterResultWriter log ==="
          cat "${APPS_DIR}/frw.log" 2>/dev/null || true
          exit 1

      - name: Integration test - run apps and check hdf5 output
        run: |
          cd /__w/datafilter/datafilter/test-area
          source env.sh
          dbt-workarea-env
          
          # Extract paths from OKS config
          CONFIG_XML="${PWD}/sourcecode/datafilter/test/config/dfSession.data.xml"
          
          # Replace output directory and input directory with work area using sed
          WORK_AREA="${PWD}"
          sed -i "s|/lcg/storage18/dune/chen|${WORK_AREA}/output|g" "${CONFIG_XML}"
          sed -i "s|/lcg/storage19/test-area/dune/trigger_records/sourcehdf5|${WORK_AREA}/input|g" "${CONFIG_XML}"
          echo "Updated config - odir now points to: ${WORK_AREA}/output"
          echo "Updated config - storage_pathname now points to: ${WORK_AREA}/input"
          
          # Get output directory (odir from FilterResultWriter) - search within 10 lines after class match
          OUTPUT_DIR=$(grep -A10 'obj class="FilterResultWriter"' "${CONFIG_XML}" | grep 'name="odir"' | grep -oP 'val="\K[^"]+')
          echo "Output directory from config: ${OUTPUT_DIR}"
          
          # Get input directory (storage_pathname from TRDispatcher) - search within 10 lines after class match
          INPUT_DIR=$(grep -A10 'obj class="TRDispatcher"' "${CONFIG_XML}" | grep 'name="storage_pathname"' | grep -oP 'val="\K[^"]+')
          echo "Input directory from config: ${INPUT_DIR}"
          
          # Create directories
          mkdir -p "${OUTPUT_DIR}"
          mkdir -p "${INPUT_DIR}"
          
          # Set OKS config path
          OKS_CONFIG="${CONFIG_XML}"
          
          # Function to cleanup processes
          cleanup() {
            echo "Cleaning up processes..."
            pkill -TERM -f "trdispatcher|filterorchestrator|datafilter2|filterresultwriter" || true
            sleep 2
            pkill -KILL -f "trdispatcher|filterorchestrator|datafilter2|filterresultwriter" || true
          }
          trap cleanup EXIT
          
          # Start the apps in background from test/apps directory
          APPS_DIR="${PWD}/sourcecode/datafilter/test/apps"
          cd "${APPS_DIR}"
          echo "Running apps from: ${APPS_DIR}"
          
          echo "Starting TRDispatcher..."
          # trdispatcher -n TRDispatcher_0 -s test-session -x ${OKS_CONFIG} > trd.log 2>&1 &
          trdispatcher > trd.log 2>&1 &
          TRD_PID=$!
          
          echo "Starting FilterOrchestrator..."
          #filterorchestrator -n FilterOrchestrator_0 -s test-session -x ${OKS_CONFIG} > fo.log 2>&1 &
          filterorchestrator > fo.log 2>&1 &
          FO_PID=$!
          
          echo "Starting DataFilter..."
          # datafilter2 -n DataFilter_0 -s test-session -x ${OKS_CONFIG} > df.log 2>&1 &
          datafilter2 > df.log 2>&1 &
          DF_PID=$!
          
          echo "Starting FilterResultWriter..."
          # filterresultwriter -n FilterResultWriter_0 -s test-session -x ${OKS_CONFIG} > frw.log 2>&1 &
          filterresultwriter > frw.log 2>&1 &
          FRW_PID=$!
          
          echo "All apps started. PIDs: TRD=$TRD_PID FO=$FO_PID DF=$DF_PID FRW=$FRW_PID"
          
          # Wait for hdf5 files in output directory (up to 5 minutes)
          MAX_WAIT=300
          INTERVAL=10
          ELAPSED=0
          
          while [ $ELAPSED -lt $MAX_WAIT ]; do
            H5_COUNT=$(find "${OUTPUT_DIR}" -name "*.hdf5" 2>/dev/null | wc -l)
            echo "Elapsed: ${ELAPSED}s, HDF5 files found: ${H5_COUNT}"
            
            if [ ${H5_COUNT} -ge 5 ]; then
              echo "SUCCESS: Found ${H5_COUNT} hdf5 files (>=5 required)"
              find "${OUTPUT_DIR}" -name "*.hdf5" 2>/dev/null
              exit 0
            fi
            
            sleep ${INTERVAL}
            ELAPSED=$((ELAPSED + INTERVAL))
          done
          
          echo "TIMEOUT: Only found ${H5_COUNT} hdf5 files after ${MAX_WAIT}s"
          echo "=== TRDispatcher log ==="
          cat "${APPS_DIR}/trd.log" 2>/dev/null || true
          echo "=== FilterOrchestrator log ==="
          cat "${APPS_DIR}/fo.log" 2>/dev/null || true
          echo "=== DataFilter log ==="
          cat "${APPS_DIR}/df.log" 2>/dev/null || true
          echo "=== FilterResultWriter log ==="
          cat "${APPS_DIR}/frw.log" 2>/dev/null || true
          exit 1
        continue-on-error: false
      
      - name: Continuous Testing
        if: ${{ github.event.inputs.enable_continuous_test == 'true' }}
        run: |
          source dbt-env.sh
          dbt-workarea-env
          
          DURATION=${{ github.event.inputs.test_duration }}
          INTERVAL=${{ github.event.inputs.test_interval }}
          END_TIME=$((SECONDS + DURATION * 60))
          TEST_COUNT=0
          FAILED_TESTS=0
          
          echo "Starting continuous testing for ${DURATION} minutes"
          echo "Test interval: ${INTERVAL} seconds"
          echo "=========================================="
          
          while [ $SECONDS -lt $END_TIME ]; do
            TEST_COUNT=$((TEST_COUNT + 1))
            CURRENT_TIME=$(date '+%Y-%m-%d %H:%M:%S')
            echo ""
            echo "=========================================="
            echo "Test Run #${TEST_COUNT} at ${CURRENT_TIME}"
            echo "=========================================="
            
            # Add your actual test commands here
            # Example test commands:
            # dbt-test
            # pytest tests/
            # ./run_integration_tests.sh
            
            if ! echo "Running test suite..."; then
              FAILED_TESTS=$((FAILED_TESTS + 1))
              echo "ERROR: Test run #${TEST_COUNT} failed!"
            else
              echo "SUCCESS: Test run #${TEST_COUNT} passed"
            fi
            
            REMAINING=$((END_TIME - SECONDS))
            if [ $REMAINING -gt $INTERVAL ]; then
              echo "Next test in ${INTERVAL} seconds..."
              sleep $INTERVAL
            else
              echo "Continuous testing period ending..."
              break
            fi
          done
          
          echo ""
          echo "=========================================="
          echo "Continuous Testing Summary"
          echo "=========================================="
          echo "Total test runs: ${TEST_COUNT}"
          echo "Failed tests: ${FAILED_TESTS}"
          echo "Success rate: $(( (TEST_COUNT - FAILED_TESTS) * 100 / TEST_COUNT ))%"
          
          if [ $FAILED_TESTS -gt 0 ]; then
            echo "WARNING: ${FAILED_TESTS} test runs failed"
            exit 1
          fi
      
      - name: Archive build artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: build-artifacts-${{ matrix.os_name }}
          path: |
            build/
            log/
          retention-days: 7

      - name: Send email notification on failure
        if: failure()
        uses: dawidd6/action-send-mail@v3
        with:
          server_address: ${{ secrets.SMTP_SERVER }}
          server_port: ${{ secrets.SMTP_PORT }}
          username: ${{ secrets.SMTP_USERNAME }}
          password: ${{ secrets.SMTP_PASSWORD }}
          subject: "Workflow failed: ${{ github.workflow }} - ${{ github.repository }}"
          body: |
            The workflow ${{ github.workflow }} has failed.
            
            Repository: ${{ github.repository }}
            Branch: ${{ github.ref }}
            Commit: ${{ github.sha }}
            Run ID: ${{ github.run_id }}
            
            View the run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
          to: ${{ secrets.NOTIFY_EMAIL }}
          from: ${{ secrets.SMTP_USERNAME }}