Dev/py unit tests and refactor ci

.github/workflows/ci.yml

@@ -6,8 +6,8 @@ on:
     branches: [main]
 
 jobs:
-  build:
-    name: Build project
+  test-debug:
+    name: Build project and test in debug mode
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -20,13 +20,16 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup Python 3.10
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.10'
 
+      - name: Install pytest
+        run: pip install pytest
+
       - name: Install Boost Python and Python dev headers
         run: |
-          sudo apt-get install -y \
+          sudo apt-get update && sudo apt-get install -y \
             libboost-python-dev \
             python3-dev
 
@@ -49,37 +52,78 @@ jobs:
 
       - name: Build in Debug Mode
         run: |
-          mkdir build && cd build && cmake -DCMake_Build_Type=Debug ..
+          mkdir build && cd build 
+          cmake -DCMake_Build_Type=Debug -DBUILD_TESTS=ON ..
           make
 
+      - name: Run Unit tests
+        run: cd build && ctest --output-on-failure --verbose
+
+      - name: Upload artifacts
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: debug-build-output
+          path: build/
+          retention-days: 2
+
+  test-release:
+    name: Build project and test in relese mode
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        gcc-version: [13]
+        cmake-version: ['3.31.3']
+      fail-fast: true
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install pytest
+        run: pip install pytest
+
+      - name: Install Boost Python and Python dev headers
+        run: |
+          sudo apt-get update && sudo apt-get install -y \
+            libboost-python-dev \
+            python3-dev
+
+      - name: Setup CMake
+        uses: jwlawson/actions-setup-cmake@v2
+        with:
+          cmake-version: ${{ matrix.cmake-version }}
+
+      - name: Setup GCC
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y gcc-${{ matrix.gcc-version }} g++-${{ matrix.gcc-version }}
+          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${{ matrix.gcc-version }} 100
+          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${{ matrix.gcc-version }} 100
+
+      - name: Verify versions
+        run: |
+          gcc --version
+          cmake --version
+
       - name: Build in Release Mode
         run: |
-          cd build
-          echo "Deleting the debug build files in $(pwd)/build" 
-          rm -rf *
-          cmake -DCMake_Build_Type=Release ..
+          mkdir build && cd build 
+          cmake -DCMake_Build_Type=Release -DBUILD_TESTS=ON ..
           make
 
+      - name: Run Unit tests
+        run: cd build && ctest --output-on-failure --verbose
+
       - name: Upload artifacts
+        if: failure()
         uses: actions/upload-artifact@v4
         with:
-          name: build-output
+          name: release-build-output
           path: build/
-          retention-days: 1
-
-  # example of artifacts
-  #    - name: Upload artifacts
-  #      uses: actions/upload-artifact@v4
-  #      with:
-  #        name: build-${{ matrix.gcc-version }}-${{ github.run_id }}
-  #        path: output/
-  #
-  #test:
-  #  needs: build  # Runs after build job
-  #  runs-on: ubuntu-latest
-  #  steps:
-  #    - name: Download artifact
-  #      uses: actions/download-artifact@v4
-  #      with:
-  #        name: build-output
-  #        path: output/
+          retention-days: 2

.gitignore

@@ -1,4 +1,6 @@
 build
 .vscode
 unit_tests_backend
-*.txt
+*.txt
+python_lib/dl_lib/_compiled
+*__pycache__*

CMakeLists.txt

@@ -5,11 +5,10 @@ project(dllib VERSION 1.0.0 LANGUAGES CXX)
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
-set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR})
-set(LIBRARY_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR})
 
 # to link shared libs. TODO: can we get rid of this, as it may cost runtime?
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+set(PYTHON_MODULE_DIR "${CMAKE_SOURCE_DIR}/python_lib/dl_lib/_compiled")
 
 # to enable find boost, see https://stackoverflow.com/a/79147222
 if(POLICY CMP0167)
@@ -104,10 +103,9 @@ include_directories("${PROJECT_SOURCE_DIR}/src"
                     "${PROJECT_SOURCE_DIR}/examples")
 
 add_subdirectory(src)
-add_subdirectory(examples)
 
-option(BUILD_TESTING "Build tests" ON)
-if(BUILD_TESTING)
+option(BUILD_TESTS "Build tests" OFF)
+if(BUILD_TESTS)
     enable_testing()
     add_subdirectory(tests)
 endif()

python_lib/dl_lib/__init__.py

@@ -0,0 +1,3 @@
+from ._compiled._core import Tensor, Dimension, Device, Ones, Zeros, Gaussian
+
+__all__ = ['Tensor', 'Device', 'Dimension']

python_lib/dl_lib/nn/__init__.py

@@ -0,0 +1,4 @@
+#from .._compiled._layers import FfLayer, ReLU
+#from .._compiled._core import Tensor  # re-export if needed
+
+#__all__ = ['FfLayer', 'ReLU']

readme.md

@@ -2,6 +2,10 @@
 
 A from-scratch deep learning framework in modern C++ with Python bindings.
 
+## Motivation
+
+Built to understand deep learning frameworks from first principles - from computational graphs to gradient computation to optimization algorithms.
+
 ## Features
 
 - **Computational Graph**: Dynamic graph construction with automatic differentiation
@@ -38,15 +42,16 @@ Roadmap:
 mkdir build && cd build
 cmake ..
 make
-./run_tests
+ctest
 ```
 
 ## Required
 
-- Python 3 (we test with 3.10, but it should work with any version)
+- Compiler capable of C++20 at least (we test with gcc 12.3.0)
 - Boost Python
 - Cmake > 3.24
-- Compiler capable of C++20 at least (we test with gcc 12.3.0)
+- Python 3 (we test with 3.10, but it should work with any version)
+- pytest for unit tests (we use 9.0.2)
 
 ## Troubleshooting
 
@@ -55,10 +60,6 @@ make
 
 The implementation of the Python wrapper does not work on MSVC6/7 in its current form. This is due to an issue that arises from Boost Python in combination with these compilers. Workarounds are proposed, but not implemented. More information here [here](https://beta.boost.org/doc/libs/develop/libs/python/doc/html/tutorial/tutorial/exposing.html).
 
-## Motivation
-
-Built to understand deep learning frameworks from first principles - from computational graphs to gradient computation to optimization algorithms.
-
 ## License
 
 MIT

src/CMakeLists.txt

@@ -2,12 +2,12 @@
 add_subdirectory(backend)
 add_subdirectory(python)
 
-target_link_libraries(py_data_modeling 
+target_link_libraries(_core 
                       ${Boost_LIBRARIES} 
                       ${PYTHON_LIBRARIES} 
                       BackendCore)
 
-target_include_directories(py_data_modeling PRIVATE 
+target_include_directories(_core PRIVATE 
                            ${PYTHON_INCLUDE_DIRS} 
                            ${Boost_INCLUDE_DIRS})
 

src/backend/computational_graph/getter_node.cpp

@@ -15,6 +15,23 @@ using namespace std;
 using namespace graph;
 
 vector< shared_ptr<Tensor> > GetterNode::backward(const Tensor& upstreamGrad) {
-  assert(!upstreamGrad.getRequiresGrad());
-  return { make_shared<Tensor>(upstreamGrad.createDeepCopy()) };
+  // upstreamGrad is scalar by definition
+  assert(!upstreamGrad.getRequiresGrad() && upstreamGrad.getDims().nDims()==1);
+
+  auto res = make_shared<Tensor>(parents[0]->getDims(), parents[0]->getDevice(), false);
+  for(tensorSize_t i=0; i<res->getSize(); i++){
+    res->setItem(0, i);
+  }
+
+  if(std::holds_alternative<tensorSize_t>(idx)){
+    res->setItem(upstreamGrad.getItem(0), std::get<tensorSize_t>(idx));
+  }
+  else if(std::holds_alternative<multiDimIdx_t>(idx)){
+    res->setItem(upstreamGrad.getItem(0), std::get<multiDimIdx_t>(idx));
+  }
+  else{
+    __throw_runtime_error("Idx variant in unexpected state");
+  }
+
+  return { std::move(res) };
 }

src/backend/computational_graph/getter_node.h

@@ -13,6 +13,9 @@
 
 #include "graph_node.h"
 
+#include <vector>
+#include <variant>
+
 namespace graph{
   /**
    * @brief When calling a get function, say as in 
@@ -21,9 +24,17 @@ namespace graph{
    * 
    */
   class GetterNode final : public GraphNode {
+    using multiDimIdx_t = std::vector<tensorDim_t>;
+
+    private:
+      const std::variant<tensorSize_t, multiDimIdx_t> idx;
+
     public:
-      explicit GetterNode(std::shared_ptr<Tensor> t) 
-        : GraphNode({std::move(t)}) {}
+      explicit GetterNode(std::shared_ptr<Tensor> t, const tensorSize_t idx) 
+        : GraphNode({std::move(t)}), idx{idx} {}
+
+      explicit GetterNode(std::shared_ptr<Tensor> t, const multiDimIdx_t& idx) 
+        : GraphNode({std::move(t)}), idx{idx} {}
 
       GetterNode(const GetterNode& other) = delete;
       GetterNode& operator=(const GetterNode& other) = delete;

src/backend/computational_graph/graph_creation.cpp

@@ -103,7 +103,7 @@ shared_ptr<Tensor> graph::get(const shared_ptr<Tensor>& t, tensorSize_t idx) {
                              t->getDevice());
 
   if(t->getRequiresGrad()){
-    res->setCgNode(std::make_shared<graph::GetterNode>(t));
+    res->setCgNode(std::make_shared<graph::GetterNode>(t, idx));
     assert(res->getRequiresGrad());
   }
   return res;
@@ -115,13 +115,25 @@ shared_ptr<Tensor> graph::get(const shared_ptr<Tensor>& t, tensorSize_t idx) {
  * 
  * loss = loss + other.get(i), we need to make sure get(i) can map to computational graph.
  */
-shared_ptr<Tensor> graph::get(const shared_ptr<Tensor>& t, vector<tensorDim_t>&& idx) {
+shared_ptr<Tensor> graph::get(const shared_ptr<Tensor>& t, const vector<tensorDim_t>& idx) {
   ftype val = t->getItem(std::move(idx));
   auto res = make_shared<Tensor>(std::vector<tensorDim_t>{1}, std::vector<ftype>{val}, 
                              t->getDevice());
   if(t->getRequiresGrad()){
-    res->setCgNode(std::make_shared<graph::GetterNode>(t));
+    res->setCgNode(std::make_shared<graph::GetterNode>(t, idx));
     assert(res->getRequiresGrad());
   }
   return res;
+}
+
+/**
+ * @brief Takes the sum of the whole tensor, then returns result as vector.
+ */
+shared_ptr<Tensor> graph::sumTensor(const shared_ptr<Tensor> t) {
+  auto res = make_shared<Tensor>(std::vector<tensorDim_t>{1}, std::vector<ftype>{0.0}, 
+                                 t->getDevice(), t->getRequiresGrad());
+  for(tensorSize_t i=0; i<t->getSize(); i++){
+    res = graph::add(res, graph::get(t, i));
+  }
+  return res;
 }

src/backend/computational_graph/graph_creation.h

@@ -16,22 +16,25 @@
 #include <memory>
 
 namespace graph {
+  // Artithmetic operations
   std::shared_ptr<Tensor> mul(const std::shared_ptr<Tensor> left, const std::shared_ptr<Tensor> right);
-
-  std::shared_ptr<Tensor> add(const std::shared_ptr<Tensor> left, const std::shared_ptr<Tensor> right);
-
-  std::shared_ptr<Tensor> matmul(const std::shared_ptr<Tensor> left, const std::shared_ptr<Tensor> right);
-
   std::shared_ptr<Tensor> mul(const std::shared_ptr<Tensor> left, ftype scalar); 
   std::shared_ptr<Tensor> mul(ftype scalar, const std::shared_ptr<Tensor> left); 
 
+  std::shared_ptr<Tensor> add(const std::shared_ptr<Tensor> left, const std::shared_ptr<Tensor> right);
   std::shared_ptr<Tensor> add(const std::shared_ptr<Tensor> left, ftype scalar);    
   std::shared_ptr<Tensor> add(ftype scalar, const std::shared_ptr<Tensor> left);
 
+  std::shared_ptr<Tensor> matmul(const std::shared_ptr<Tensor> left, const std::shared_ptr<Tensor> right);
+
   std::shared_ptr<Tensor> sub(const std::shared_ptr<Tensor> left, ftype scalar);
   std::shared_ptr<Tensor> div(const std::shared_ptr<Tensor> left, ftype scalar);
 
+  // Getter methods
   std::shared_ptr<Tensor> get(const std::shared_ptr<Tensor>& t, tensorSize_t idx);
-  std::shared_ptr<Tensor> get(const std::shared_ptr<Tensor>& t, std::vector<tensorDim_t>&& idx);
+  std::shared_ptr<Tensor> get(const std::shared_ptr<Tensor>& t, const std::vector<tensorDim_t>& idx);
+
+  // Composite operations
+  std::shared_ptr<Tensor> sumTensor(const std::shared_ptr<Tensor> t);
 }
 

src/backend/computational_graph/topological_sort.cpp

@@ -36,10 +36,11 @@ bool TopologicalSort::hasCycles(const Tensor* root) {
     assert(start->cgNode);
 
     stack<const Tensor*> tStack;
+    unordered_set<const Tensor*> visited;
 
-    auto pushParentsOnStack = [&tStack](const Tensor* t){
+    auto pushParentsOnStack = [&tStack, &visited](const Tensor* t){
       for(auto parent: t->cgNode->getParents()){
-        if(parent->cgNode){
+        if(parent->cgNode && !visited.contains(parent.get())){
           tStack.push(parent.get());
         }
       }
@@ -57,6 +58,7 @@ bool TopologicalSort::hasCycles(const Tensor* root) {
       tStack.pop();
 
       pushParentsOnStack(t);
+      visited.insert(t);
     }
 
     return false;