WIP: merge turbostream DO NOT MERGE

.github/workflows/perf-ci.yml

@@ -2,7 +2,7 @@ name: perf-ci
 
 on:
   pull_request:
-    branches: [ 2.0 ]
+    branches: [ main ]
 
 jobs:
   test-performance:

.github/workflows/pre-commit.yml

@@ -0,0 +1,28 @@
+name: Pre-commit
+
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+
+jobs:
+  pre-commit:
+    name: pre-commit
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install pre-commit
+        run: python -m pip install -U pip pre-commit
+
+      - name: Run pre-commit
+        run: pre-commit run --all-files --show-diff-on-failure

.github/workflows/render-ci.yml

@@ -0,0 +1,59 @@
+name: render-ci
+
+on:
+  pull_request:
+    branches: [ main ]
+  push:
+    branches: [ main ]
+
+jobs:
+  test-render:
+    name: test-render - ubuntu-latest
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Check disk space (before cleanup)
+        run: df -h
+
+      - name: Free up disk space
+        run: |
+          echo "Before cleanup:"
+          df -h
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
+          sudo apt-get clean
+          sudo rm -rf ~/.cache/pip /tmp/* /var/tmp/*
+          echo "After cleanup:"
+          df -h
+
+      - name: Install system dependencies for headless rendering
+        run: |
+          sudo apt update
+          sudo apt install -y ffmpeg xvfb build-essential
+
+      - name: Upgrade pip
+        run: python -m pip install -U pip
+
+      - name: Install pufferlib
+        run: |
+          pip install -e .[cpu] --no-cache-dir
+        env:
+          TMPDIR: ${{ runner.temp }}/build
+          PIP_NO_CACHE_DIR: 1
+
+      - name: Compile C extensions
+        run: python setup.py build_ext --inplace --force
+
+      - name: Check disk space (after install)
+        run: df -h
+
+      - name: Run render test
+        run: python tests/test_drive_render.py
+        timeout-minutes: 15

.github/workflows/train-ci.yml

@@ -2,9 +2,9 @@ name: train-ci
 
 on:
   pull_request:
-    branches: [ 2.0 ]
+    branches: [ main ]
   push:
-    branches: [ 2.0 ]
+    branches: [ main ]
 
 jobs:
   test-training:

.github/workflows/training-test.yml

@@ -0,0 +1,37 @@
+name: Training Test
+
+on:
+  pull_request:
+    types:
+      - closed
+    branches: [ develop ]
+
+jobs:
+  run-training-test:
+    name: Run Training Test
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Authenticate to Google Cloud
+        uses: 'google-github-actions/auth@v2'
+        with:
+          credentials_json: '${{ secrets.GCP_CREDENTIALS }}'
+
+      - name: Configure Docker for GCP Artifact Registry
+        run: gcloud auth configure-docker europe-west4-docker.pkg.dev
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies for mlops.py
+        run: pip install pyyaml google-cloud-aiplatform
+
+      - name: Run training test
+        run: python mlops/mlops.py train-test --verbose
+        timeout-minutes: 40

.github/workflows/utest.yml

@@ -2,9 +2,9 @@ name: Unit tests
 
 on:
   pull_request:
-    branches: [ 2.0 ]
+    branches: [ main ]
   push:
-    branches: [ 2.0 ]
+    branches: [ main ]
 
 jobs:
   test:

.gitignore

@@ -141,13 +141,13 @@ dmypy.json
 # Pyre type checker
 .pyre/
 .pylintrc
-.vscode/
 
 # ROMs
 *.gb
 
 checkpoints/
 experiments/
+benchmark*/
 wandb/
 .neptune/
 raylib*/
@@ -163,14 +163,9 @@ pufferlib/ocean/impulse_wars/benchmark/
 
 # Ignore data files
 data/
-pufferlib/resources/drive/binaries/
-pufferlib/resources/drive/binaries/training/
-pufferlib/resources/drive/binaries/validation/
-
-# But keep map_000.bin for the training test
-!pufferlib/resources/drive/binaries/map_000.bin
-!pufferlib/resources/drive/binaries/training/map_000.bin
-pufferlib/resources/drive/sanity/sanity_binaries/
+pufferlib/resources/drive/binaries/*/
+!pufferlib/resources/drive/binaries/carla/
+!pufferlib/resources/drive/binaries/carla/**
 
 # Compiled drive binary in root
 /drive
@@ -187,6 +182,5 @@ pufferlib/resources/drive/output_agent.gif pufferlib/resources/drive/output.gif
 artifacts/
 # Local drive renders
 pufferlib/resources/drive/output*.gif
-emsdk/
-docs/book/*
-!docs/book/assets/
+*.mp4
+.vscode/

.pre-commit-config.yaml

@@ -26,4 +26,4 @@ repos:
   rev: 'v21.1.5'
   hooks:
   -   id: clang-format
-      exclude: '\.(js|html)$|^build_web/|^docs/assets/'
+      types_or: [c, c++]