added test coverage for ci (#792)

* added test coverage for ci

* merged coverage job into build, removed all but one step for testing

* test code for meeting

* Call-Graph Improvements (#785)

* minor fix in CHA and RTA resolvers

* Several small improvements for call-graph resolving

* Fix out-of-bounds access in getVarTypeFromIR()

* Add address-taken functions caching in base resolver

* Some cleanup in resolvers

* pre-commit

* Fix bug in the overloads of buildLLVMBasedCallGraph() that takes a CallGraphAnalysisType

* Dbg-Info based Type Matching (#791)

* moved stripPointerTypes() + debug prints

* MetadataKind approach

* I think we don't get around using getName()

* unittests still fail

* bugfix + cleanup

* Some cleanup

* Implement allocated-types collection in terms of debug info

* Fix libdeps

* Remove some unneeded includes

---------

Co-authored-by: Fabian Schiebel <52407375+fabianbs96@users.noreply.github.com>
Co-authored-by: Fabian Schiebel <fabian.schiebel@iem.fraunhofer.de>

* Some fixes in the IterativeIDESolver (#782)

* Fix the IterativeIDESolver by allowing to analyze problems that override extend() and combine()

* Use the logger in the IterativeIDESolver

* Make it possible to solve the IDEFeatureTaintAnalysis with the IterativeIDESolver

* Fix errors introduced by merge

* Bump dependencies (#784)

* Remove boost from BitVectorSet (#788)

* Remove boost from BitVectorSet

* Minor deboostifying

* Apply review comment on Bit-index-inbounds check + simplify operator<<

* GCC Compatibility (#787)

* Make phasar compile with gcc (g++-11) and uncover (and fix) two bugs related to json ser/de with LLVMAliasSet and LLVMBasedICFG that were not detected untio now, because of nlohmann's implicit conversions feature

* Address review comments

* Simplify isLLVMZeroValue according to the discussion in #787

* backup for rebase

* cmake works, g++ fails

* working version (with local hacks -20)

* added regex for filter, it is not working

* fixed filtering + cleanup

* basic ci impl, lib coverage, fixed release cmake error

* added cmake-scripts to submodule index

* set llvm-cov before code-cov script

* fixed version of llvm-cov

* testing v19 due to ci issues

* testing cov only when in debug build type

* fixed all build type checks

* added -v to see invocation of error in ci

* only run coverage when matrix.build is Debug

* Debug code + potential bug found?

* only run cov when PHASAR_DEBUG_LIBDEPS is off

* removed debug code, added messages for cov paths

* testing llvm cov 19

* hardcoded llvm-19 for cov

* adding llvm-19 as dependency for cov tools

* disabling asan for release for now

* removed unneccesary char

* development merge + json external

* Is the sanitizer the problem?

* fixed cmake failing when in debug w/o -DCODE_COVERAGE=ON

* added coverage report artifact uploading

* fixed ci

* fixed filtering not working correctly on runner

* docker sanitizer back ON + new matrix entry DebugCov

* Docker sanitizer still fails, disabling for now

* Removed coverage of unittests

* cleanup

* Some cleanup

* Add demangling to cov report

* artifact should be entire all-merged/ folder

* external diff

* removed rename of index.html

* Revert json downgrade

---------

Co-authored-by: Fabian Schiebel <52407375+fabianbs96@users.noreply.github.com>
Co-authored-by: Fabian Schiebel <fabian.schiebel@iem.fraunhofer.de>
Co-authored-by: Fabian Schiebel <fabianbs@mail.upb.de>

Author: mxHuber

.github/workflows/ci.yml

@@ -16,7 +16,7 @@ jobs:
       matrix:
         os: [ubuntu-24.04, ubuntu-24.04-arm]
         compiler: [ [clang++-19, clang-19, "clang-19 libclang-rt-19-dev"] ]
-        build: [ Debug, Release, DebugLibdeps ]
+        build: [ Debug, Release, DebugLibdeps, DebugCov ]
         include:
           - build: Debug
             cmake_build_type: Debug
@@ -27,11 +27,17 @@ jobs:
           - build: DebugLibdeps
             cmake_build_type: Debug
             flags: -DPHASAR_DEBUG_LIBDEPS=ON -DBUILD_SHARED_LIBS=ON
+          - build: DebugCov
+            cmake_build_type: Debug
+            flags: -DCODE_COVERAGE=ON
+            extra_dependencies: llvm-19 # For coverage
         exclude:
           - os: ubuntu-24.04-arm
             build: Debug
           - os: ubuntu-24.04-arm
             build: DebugLibdeps
+          - os: ubuntu-24.04-arm
+            build: DebugCov
 
     continue-on-error: false
     steps:
@@ -44,7 +50,7 @@ jobs:
       - name: Install Phasar Dependencies
         shell: bash
         run: |
-          ./utils/InstallAptDependencies.sh --noninteractive tzdata ${{ matrix.compiler[2] }}
+          ./utils/InstallAptDependencies.sh --noninteractive tzdata ${{ matrix.compiler[2] }} ${{ matrix.extra_dependencies }}
 
       - name: Building Phasar in ${{ matrix.build }} with ${{ matrix.compiler[0] }}
         env:
@@ -77,3 +83,18 @@ jobs:
           cd ./examples/how-to
           cmake -S . -B build -Dphasar_ROOT="$PHASAR_ROOT_DIR/INSTALL"
           cmake --build ./build --target run_sample_programs
+
+      - name: Check test coverage and generate HTML report
+        if: matrix.build == 'DebugCov'
+        shell: bash
+        run: |
+          cmake --build ./build --target ccov-all
+
+      - name: Upload coverage HTML report artifact
+        if: matrix.build == 'DebugCov'
+        uses: actions/upload-artifact@v4
+        with:
+          name: CoverageReport
+          path: ./build/ccov/all-merged/
+          if-no-files-found: error
+          retention-days: 7

CMakeLists.txt

@@ -268,11 +268,43 @@ find_package(Threads)
 set(CMAKE_CXX_CLANG_TIDY "")
 
 # Nlohmann JSON
-
 include(add_nlohmann_json)
 add_nlohmann_json()
 add_json_schema_validator()
 
+# Coverage
+if (CODE_COVERAGE)
+  set(CODE_COVERAGE_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/code-coverage.cmake")
+  file(DOWNLOAD "https://raw.githubusercontent.com/StableCoder/cmake-scripts/refs/tags/25.08/code-coverage.cmake" "${CODE_COVERAGE_SCRIPT}"
+    EXPECTED_HASH SHA256=cad4dd2bdf944f77627687e0298debc8341bfdeffaf949937f319c7c4ff3fdaf
+    TLS_VERIFY ON
+    STATUS CODE_COVERAGE_SCRIPT_DOWNLOAD_STATUS
+  )
+  message(STATUS "Download code-coverage.cmake: ${CODE_COVERAGE_SCRIPT_DOWNLOAD_STATUS}")
+
+  if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+    string(REGEX MATCH "^[0-9]+" COMPILER_VERSION_MAJOR "${CMAKE_CXX_COMPILER_VERSION}")
+    message(STATUS "Clang major version: ${COMPILER_VERSION_MAJOR}")
+    set(LLVM_TOOL_VERSION_SUFFIX "-${COMPILER_VERSION_MAJOR}")
+  endif()
+
+  # pre-populate these cmake variables as we require specific versions:
+  find_program(LLVM_COV_PATH "llvm-cov${LLVM_TOOL_VERSION_SUFFIX}" REQUIRED)
+  find_program(LLVM_PROFDATA_PATH "llvm-profdata${LLVM_TOOL_VERSION_SUFFIX}" REQUIRED)
+  message(STATUS "Found llvm-cov: ${LLVM_COV_PATH}")
+  message(STATUS "Found llvm-profdata: ${LLVM_PROFDATA_PATH}")
+
+  include("${CODE_COVERAGE_SCRIPT}")
+
+  find_program(cxxfilt "c++filt" QUIET)
+  if (cxxfilt)
+    set(DEMANGLER_OPTIONS LLVM_COV_HTML_OPTIONS "-Xdemangler=${cxxfilt}")
+  endif()
+
+  set(COV_FILEPATH_FILTER "(usr/*|external/|unittests/)")
+  add_code_coverage_all_targets(EXCLUDE ${COV_FILEPATH_FILTER} ${DEMANGLER_OPTIONS})
+endif()
+
 # Googletest
 if (NOT PHASAR_IN_TREE)
   if(PHASAR_BUILD_UNITTESTS AND NOT TARGET gtest)

bootstrap.sh

@@ -97,6 +97,7 @@ fi
 tmp_dir=$(mktemp -d "llvm-build.XXXXXXXX" --tmpdir)
 ./utils/install-llvm.sh "${NUM_THREADS}" "${tmp_dir}" "${LLVM_INSTALL_DIR}" ${LLVM_RELEASE}
 rm -rf "${tmp_dir}"
+
 echo "dependencies successfully installed"
 
 # *Always* set the LLVM root to ensure the Phasar script uses the proper toolchain

cmake/phasar_macros.cmake

@@ -35,6 +35,10 @@ function(add_phasar_unittest test_name)
   )
   set_tests_properties("${test}" PROPERTIES LABELS "all")
   set(CTEST_OUTPUT_ON_FAILURE ON)
+
+  if (CODE_COVERAGE)
+    target_code_coverage(${test} AUTO ALL)
+  endif()
 endfunction()
 
 function(validate_binary_version result item)
@@ -351,6 +355,10 @@ function(add_phasar_library name)
   endif()
 
   set_property(GLOBAL APPEND PROPERTY LLVM_EXPORTS ${name})
+
+  if (CODE_COVERAGE)
+    target_code_coverage(${name} AUTO ALL)
+  endif()
 endfunction(add_phasar_library)
 
 macro(subdirlist result curdir)