Updated codebase to use QuestDB instead of streaming files from S3

.github/workflows/build.yml

@@ -15,18 +15,20 @@ jobs:
           distribution: 'zulu' # See 'Supported distributions' for available options
           java-version: '21'
       - uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: '9.0.x'
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
       - name: Cache SonarCloud packages
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ~\.sonar\cache
           key: ${{ runner.os }}-sonar
           restore-keys: ${{ runner.os }}-sonar
       - name: Cache SonarCloud scanner
         id: cache-sonar-scanner
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: .\.sonar\scanner
           key: ${{ runner.os }}-sonar-scanner
@@ -37,19 +39,19 @@ jobs:
         run: |
           New-Item -Path .\.sonar\scanner -ItemType Directory
           dotnet tool update dotnet-sonarscanner --tool-path .\.sonar\scanner
-      - name: Test
-        run: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=opencover
-      - name: Begin
+      - name: Begin SonarCloud Analysis
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         shell: powershell
-        run: .\.sonar\scanner\dotnet-sonarscanner begin /k:"mccaffers_backtesting-engine" /o:"mccaffers" /d:sonar.login="${{ secrets.SONAR_TOKEN }}" /d:sonar.host.url="https://sonarcloud.io" /d:sonar.cs.opencover.reportsPaths="./tests/coverage.opencover.xml"        
+        run: .\.sonar\scanner\dotnet-sonarscanner begin /k:"mccaffers_backtesting-engine" /o:"mccaffers" /d:sonar.login="${{ secrets.SONAR_TOKEN }}" /d:sonar.cs.opencover.reportsPaths="./tests/coverage.opencover.xml"
       - name: Build
-        run: dotnet build   
-      - name: Sonar End
+        run: dotnet build
+      - name: Test
+        run: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=opencover
+      - name: End SonarCloud Analysis
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         shell: powershell
         run: .\.sonar\scanner\dotnet-sonarscanner end /d:sonar.login="${{ secrets.SONAR_TOKEN }}"

.github/workflows/codeql-analysis.yml

@@ -36,16 +36,17 @@ jobs:
 
     steps:
 
-    - uses: actions/setup-dotnet@v1
+    - uses: actions/checkout@v5
+    - name: Setup dotnet
+      uses: actions/setup-dotnet@v5
       with:
-        dotnet-version: 8.0.x
-
-    - name: Checkout repository
-      uses: actions/checkout@v2
+        dotnet-version: | 
+          8.0.x
+          9.0.x
 
     # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+    - name: CodeQL Initialize
+      uses: github/codeql-action/init@v4
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -56,7 +57,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v4
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -65,9 +66,8 @@ jobs:
     #    and modify them (or add more) to build your code if your project
     #    uses a compiled language
 
-#     - name: build
-#       run: dotnet build
-
+    # - name: build
+    #   run: dotnet build
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v4

.gitignore

@@ -37,4 +37,13 @@ engine.zip
 tickdata/
 scripts/data.sh
 src/ui/src/libs/
-scripts/backtestings/data.sh
+scripts/backtestings/data.sh
+
+.env
+.env.*
+!.env.example
+*.pem
+*.key
+*.p12
+appsettings.*.json
+!appsettings.Development.json

README.md

@@ -1,65 +1,102 @@
 ## C# Backtesting Engine
 
-### Active development
+> [!NOTE]
+> This project is maintained but no longer under active development. It receives periodic updates and improvements, but my primary focus has shifted to a **C++ implementation** for further experimentation. Check out the progress on the high-performance C++ version here: **[backtesting-engine-cpp](https://github.com/mccaffers/backtesting-engine-cpp)**
 
-Working build! Requires local tick data. Evaluate locally with `dotnet test` - 49 tests should pass, 1 skipped 
+### Background
 
-## About The Project
+This is a high-performance C# backtesting engine designed to analyze financial data and evaluate multiple trading strategies at scale. The engine leverages **QuestDB** for efficient time-series data storage and retrieval, providing significantly faster query performance and more robust handling of tick-level financial data compared to traditional file-based approaches. This project serves as a practical demonstration of quantitative engineering capabilities, combining financial domain knowledge with modern software architecture and cloud infrastructure.
 
-I'm developing a high-performance C# backtesting engine designed to analyze financial data and evaluate multiple trading strategies at scale.
+> I initiated this project to deepen my understanding of financial markets while showcasing technical expertise through detailed documentation and transparent decision-making. By building a production-grade backtesting system from the ground up I've been able to explore real-world challenges in quantitative finance—from data pipeline optimisation to strategy evaluation frameworks. The project also initially investigates the practical benefits of cloud services, specifically AWS, for horizontally scaling strategy permutations and experiments. <br/><br/> This cloud-native approach dramatically reduces the time required to analyze results and generate actionable insights, enabling rapid iteration on trading hypotheses and more comprehensive strategy exploration than would be feasible on local infrastructure for better cost and performance. <br/></br>Do read more of my blog detailing the development process: https://mccaffers.com/quantitative_engineering/building_a_backtesting_system/
 
 ![alt](images/development_active.svg) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=mccaffers_backtesting-engine&metric=alert_status)](https://sonarcloud.io/summary/overall?id=mccaffers_backtesting-engine) [![Build](https://github.com/mccaffers/backtesting-engine/actions/workflows/build.yml/badge.svg)](https://github.com/mccaffers/backtesting-engine/actions/workflows/build.yml) [![Bugs](https://sonarcloud.io/api/project_badges/measure?project=mccaffers_backtesting-engine&metric=bugs)](https://sonarcloud.io/summary/new_code?id=mccaffers_backtesting-engine) [![Coverage](https://sonarcloud.io/api/project_badges/measure?project=mccaffers_backtesting-engine&metric=coverage)](https://sonarcloud.io/summary/new_code?id=mccaffers_backtesting-engine)
 
-
-I'm extracting results and creating various graphs for trend analyses using SciPy for calculations and Plotly for visualization.
+### Results
+Results are extracted and analyzed using various visualization tools for trend analysis.
 
 ![alt text](images/random-indices-sp500-variable.svg)
 
 *Read more results on https://mccaffers.com/randomly_trading/*
 
-## Features
+### Features
 
-- [x] Multiple symbol ingest with time synchronisation
-- [x] 50 xUnit testing across Trade Management, Ingest, Reporting & Utilities
+- [x] **QuestDB Integration** - High-performance time-series database for tick data
+- [x] 24 xUnit tests covering Trade Management, Ingest, Reporting & Utilities
 - [x] Trade Environment
-    * Trade Excution
+    * Trade Execution
     * Equity Monitoring
+    * Position Management
 - [x] Reporting (ElasticSearch)
+- [x] Environment-based configuration
 
 ## Getting Started
 
-To begin with, ensure you can run the application with `dotnet test`
+### Prerequisites
+
+* .NET 9
+* QuestDB (localhost:8812 for jdbc client)
+* ElasticSearch for reporting (optional, will report to terminal if not present)
+
+### Running the Engine
 
-**Local Terminal**
+1. **Start QuestDB** locally (follow [QuestDB installation guide](https://questdb.io/docs/get-started/docker/))
 
-sh ./scripts/backtesting/run.sh
+2. **Configure your strategy** - Edit the JSON payload in `./scripts/run.sh` to set:
+   - Trading symbols (e.g., EURUSD)
+   - Strategy parameters (stop distance, limit distance, etc.)
+   - Backtesting period (LAST_MONTHS)
+   - Environment variables (Elasticsearch URI, username, password)
 
-**Local Web & Terminal** (requires canvasjs.min.js)
+3. **Run the engine**:
+   ```bash
+   ./scripts/run.sh
+   ```
 
-sh ./scripts/backtesting/web.sh
+4. **Run tests**:
+   ```bash
+   dotnet test
+   ```
+   Check, 24 tests should pass successfully.
 
-## Dependencies
+## Why QuestDB?
 
-* dotnet v8
-* ElasticSearch for reporting
-* CanvasJS used for charting
-  * JS libaries `canvasjs.min.js` to be placed within ./src/ui/src/libs/ folder
-  * Charting & Web use, need to accept dotnet's certificates `dotnet dev-certs https --trust`
-* Some financial tick data in CSV format, in the /src/tickdata folder (example provided)
+QuestDB offers several advantages for financial backtesting:
+- **Fast ingestion** - Optimized for high-frequency time-series data
+- **SQL interface** - Familiar query language with time-series extensions
+- **Low latency** - Microsecond-level query performance
+- **Efficient storage** - Column-oriented storage reduces disk footprint
 
-    ```bash
-    # ./tickdata/{symbol}/2020.csv:
-    UTC,AskPrice,BidPrice,AskVolume,BidVolume
-    2018-01-01T01:00:00.594+00:00,1.35104,1.35065,1.5,0.75
-    ```
+## Configuration
 
-## Debugging
+Environment variables are managed through the JSON payload in `./scripts/run.sh`:
 
-If you have changed target frameworks, make sure to update the program path in .vscode `launch.json`
+```json
+{
+  "RUN_ID": "LOCAL#RANDOM_ID#",
+  "SYMBOLS": "EURUSD",
+  "LAST_MONTHS": 2,
+  "REPORT_TO_ELASTICSEARCH": true,
+  "STRATEGY": {
+    "TRADING_VARIABLES": {
+      "STRATEGY": "RandomStrategy",
+      "STOP_DISTANCE_IN_PIPS": "100",
+      "LIMIT_DISTANCE_IN_PIPS": "9"
+    }
+  }
+}
+```
 
-## Random Trading Strategy
-    accountEquity=0
-    maximumDrawndownPercentage=0
+Set your Elasticsearch credentials:
+```bash
+export ELASTICSEARCH_URI="http://localhost:9200"
+export ELASTICSEARCH_USERNAME="your_username"
+export ELASTICSEARCH_PASSWORD="your_password"
+```
+
+## Contributing
+
+Issues and pull requests are welcome! Please raise any bugs or feature requests on the [GitHub repository](https://github.com/mccaffers/backtesting-engine/issues).
 
 ## License
+
 [MIT](https://choosealicense.com/licenses/mit/)

backtesting-engine.sln

@@ -8,12 +8,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "backtesting-engine", "src\b
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "interfaces", "src\Interfaces\interfaces.csproj", "{230D296C-4E8B-4E3F-BE23-CB99D9818E77}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "live", "live", "{104C4FAE-5AE5-49EB-BDCE-15CCE3D1D058}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ingest", "ingest", "{3D53D1C2-65DC-4FE7-8311-C1BDBFF24989}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ingest", "src\live\ingest\src\ingest.csproj", "{7B4CDB5A-BFFE-458C-9602-C183039FCF9E}"
-EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "models", "src\Models\models.csproj", "{D202732F-B3CC-43A8-B190-84484B64C35D}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "strategies", "src\strategies\strategies.csproj", "{802C0BC2-75BC-44A3-BCD2-539947B7D345}"
@@ -22,10 +16,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "utilities", "src\utilities\
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "test", "tests\test.csproj", "{AA34B532-DB36-4247-BB9E-306B2D876A32}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "webserver", "src\webserver\webserver.csproj", "{7AC7DAF2-8F5E-4270-860C-30B14B7BE5F2}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "webutils", "src\webutils\webutils.csproj", "{913D643B-FB8D-4CD3-BD77-3C1F86F1C4D3}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -43,10 +33,6 @@ Global
 		{230D296C-4E8B-4E3F-BE23-CB99D9818E77}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{230D296C-4E8B-4E3F-BE23-CB99D9818E77}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{230D296C-4E8B-4E3F-BE23-CB99D9818E77}.Release|Any CPU.Build.0 = Release|Any CPU
-		{7B4CDB5A-BFFE-458C-9602-C183039FCF9E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{7B4CDB5A-BFFE-458C-9602-C183039FCF9E}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{7B4CDB5A-BFFE-458C-9602-C183039FCF9E}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{7B4CDB5A-BFFE-458C-9602-C183039FCF9E}.Release|Any CPU.Build.0 = Release|Any CPU
 		{D202732F-B3CC-43A8-B190-84484B64C35D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{D202732F-B3CC-43A8-B190-84484B64C35D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{D202732F-B3CC-43A8-B190-84484B64C35D}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -63,25 +49,12 @@ Global
 		{AA34B532-DB36-4247-BB9E-306B2D876A32}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{AA34B532-DB36-4247-BB9E-306B2D876A32}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{AA34B532-DB36-4247-BB9E-306B2D876A32}.Release|Any CPU.Build.0 = Release|Any CPU
-		{7AC7DAF2-8F5E-4270-860C-30B14B7BE5F2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{7AC7DAF2-8F5E-4270-860C-30B14B7BE5F2}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{7AC7DAF2-8F5E-4270-860C-30B14B7BE5F2}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{7AC7DAF2-8F5E-4270-860C-30B14B7BE5F2}.Release|Any CPU.Build.0 = Release|Any CPU
-		{913D643B-FB8D-4CD3-BD77-3C1F86F1C4D3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{913D643B-FB8D-4CD3-BD77-3C1F86F1C4D3}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{913D643B-FB8D-4CD3-BD77-3C1F86F1C4D3}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{913D643B-FB8D-4CD3-BD77-3C1F86F1C4D3}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{A1E8C629-D887-45F8-A4F9-41AA2B64A9D0} = {34DC4AAB-D45F-424A-AC69-30631C0F870A}
 		{230D296C-4E8B-4E3F-BE23-CB99D9818E77} = {34DC4AAB-D45F-424A-AC69-30631C0F870A}
-		{104C4FAE-5AE5-49EB-BDCE-15CCE3D1D058} = {34DC4AAB-D45F-424A-AC69-30631C0F870A}
-		{3D53D1C2-65DC-4FE7-8311-C1BDBFF24989} = {104C4FAE-5AE5-49EB-BDCE-15CCE3D1D058}
-		{7B4CDB5A-BFFE-458C-9602-C183039FCF9E} = {3D53D1C2-65DC-4FE7-8311-C1BDBFF24989}
 		{D202732F-B3CC-43A8-B190-84484B64C35D} = {34DC4AAB-D45F-424A-AC69-30631C0F870A}
 		{802C0BC2-75BC-44A3-BCD2-539947B7D345} = {34DC4AAB-D45F-424A-AC69-30631C0F870A}
 		{373C9641-BF7E-4873-B653-F31F97B183C6} = {34DC4AAB-D45F-424A-AC69-30631C0F870A}
-		{7AC7DAF2-8F5E-4270-860C-30B14B7BE5F2} = {34DC4AAB-D45F-424A-AC69-30631C0F870A}
-		{913D643B-FB8D-4CD3-BD77-3C1F86F1C4D3} = {34DC4AAB-D45F-424A-AC69-30631C0F870A}
 	EndGlobalSection
 EndGlobal