Migration to async Copilot Usage Metrics API - Full Implementation (All Phases Complete)

.env

@@ -6,7 +6,7 @@ NUXT_PUBLIC_IS_DATA_MOCKED=true
 NUXT_PUBLIC_SCOPE=organization
 
 # Determines the enterprise or organization name to target API calls.
-NUXT_PUBLIC_GITHUB_ORG=octodemo
+NUXT_PUBLIC_GITHUB_ORG=cody-test-org
 
 NUXT_PUBLIC_GITHUB_ENT=
 
@@ -17,7 +17,7 @@ NUXT_PUBLIC_USING_GITHUB_AUTH=false
 
 # Determines the GitHub Personal Access Token to use for API calls.
 # Create with scopes copilot, manage_billing:copilot or manage_billing:enterprise, read:enterprise AND read:org
-# NUXT_GITHUB_TOKEN=<TOKEN>
+NUXT_GITHUB_TOKEN=<TOKEN>
 
 NUXT_SESSION_PASSWORD=something_long_and_random_thats_at_least_32_characters
 
@@ -27,3 +27,6 @@ NUXT_OAUTH_GITHUB_CLIENT_SECRET=
 
 # to use a corporate proxy
 # HTTP_PROXY=http://proxy.company.com:8080
+
+ENABLE_HISTORICAL_MODE=true
+NUXT_PUBLIC_USE_NEW_API=true

.env.example

@@ -0,0 +1,139 @@
+# ==============================================================================
+# GitHub Copilot Metrics Viewer Configuration
+# ==============================================================================
+
+# ------------------------------------------------------------------------------
+# Core Configuration
+# ------------------------------------------------------------------------------
+
+# Determines if mocked data should be used instead of making API calls.
+NUXT_PUBLIC_IS_DATA_MOCKED=true
+
+# Determines the scope of the API calls. 
+# Can be 'enterprise', 'organization', 'team-organization', or 'team-enterprise'
+NUXT_PUBLIC_SCOPE=organization
+
+# Determines the enterprise or organization name to target API calls.
+NUXT_PUBLIC_GITHUB_ORG=octodemo
+NUXT_PUBLIC_GITHUB_ENT=
+
+# Determines the team name if exists to target API calls.
+NUXT_PUBLIC_GITHUB_TEAM=
+
+# Enable GitHub OAuth authentication
+NUXT_PUBLIC_USING_GITHUB_AUTH=false
+
+# ------------------------------------------------------------------------------
+# Authentication
+# ------------------------------------------------------------------------------
+
+# GitHub Personal Access Token for API calls
+# Required scopes: copilot, manage_billing:copilot, manage_billing:enterprise, read:enterprise, read:org
+# NUXT_GITHUB_TOKEN=ghp_your_token_here
+
+# Session password for encrypting user sessions (REQUIRED!)
+# Must be at least 32 characters long
+NUXT_SESSION_PASSWORD=something_long_and_random_thats_at_least_32_characters
+
+# GitHub OAuth App credentials (if using GitHub auth)
+NUXT_OAUTH_GITHUB_CLIENT_ID=
+NUXT_OAUTH_GITHUB_CLIENT_SECRET=
+
+# ------------------------------------------------------------------------------
+# API Migration Configuration (New Features)
+# ------------------------------------------------------------------------------
+
+# Use the LEGACY Copilot Metrics API (deprecated, shuts down April 2, 2026)
+# Default is false — the new download-based API is used unless this is explicitly set to true.
+# Only set USE_LEGACY_API=true if you have a specific reason to use the old /copilot/metrics endpoint.
+USE_LEGACY_API=false
+
+# Enable storage-backed historical mode
+# Set to 'true' to query storage for date ranges instead of live API calls
+# Requires storage backend configuration (see below)
+NUXT_PUBLIC_ENABLE_HISTORICAL_MODE=false
+
+# ------------------------------------------------------------------------------
+# Storage Configuration
+# ------------------------------------------------------------------------------
+
+# Storage backend for metrics data
+# Options: filesystem (dev), postgresql (prod), redis, mongodb
+# Configured in nuxt.config.ts nitro.storage section
+
+# For PostgreSQL backend (recommended for production):
+# DATABASE_URL=postgresql://user:password@localhost:5432/copilot_metrics
+
+# For Redis backend:
+# REDIS_URL=redis://localhost:6379
+
+# For filesystem backend (default, development only):
+# Data stored in ./.data/metrics directory (already configured)
+
+# ------------------------------------------------------------------------------
+# Data Sync Configuration
+# ------------------------------------------------------------------------------
+
+# Enable automatic daily sync (for background scheduler)
+# Set to 'true' to enable the scheduled task
+SYNC_ENABLED=false
+
+# Cron schedule for sync job (default: 2 AM daily)
+# Format: minute hour day month weekday
+SYNC_SCHEDULE=0 2 * * *
+
+# Number of days to backfill on first sync
+SYNC_BACKFILL_DAYS=28
+
+# Data retention period (days)
+SYNC_RETENTION_DAYS=365
+
+# ------------------------------------------------------------------------------
+# Network Configuration
+# ------------------------------------------------------------------------------
+
+# HTTP Proxy (for corporate environments)
+# HTTP_PROXY=http://proxy.company.com:8080
+
+# Custom CA certificate path (for corporate proxies with custom CA)
+# CUSTOM_CA_PATH=/path/to/ca-cert.pem
+
+# Server port (default: 80 in Docker, 3000 in dev)
+# NITRO_PORT=3000
+
+# ------------------------------------------------------------------------------
+# Admin API Configuration
+# ------------------------------------------------------------------------------
+
+# ⚠️ SECURITY WARNING: Admin endpoints are NOT authenticated by default!
+#
+# The /api/admin/* endpoints (sync trigger, status) are currently UNPROTECTED.
+# If you expose these endpoints publicly, you MUST secure them via:
+#   - Reverse proxy authentication (nginx, Apache, etc.)
+#   - API gateway with auth (AWS API Gateway, Azure APIM, etc.)
+#   - Firewall rules restricting to internal networks only
+#   - VPN or private network access only
+#
+# ADMIN_API_SECRET is reserved for future native authentication but has NO EFFECT
+# in the current version. Do NOT rely on this variable for security.
+# ADMIN_API_SECRET=your_secure_random_secret_here
+
+# ==============================================================================
+# Migration Notes
+# ==============================================================================
+
+# The legacy GitHub Copilot Metrics API will shut down on April 2, 2026.
+# This application uses the new download-based API by default. No configuration needed.
+#
+# API modes:
+#   Default (new API) — download-based reports, richer data, model/feature breakdowns
+#   Legacy (opt-in)   — set USE_LEGACY_API=true to use deprecated /copilot/metrics
+#
+# Recommended setup:
+#   1. Test with mock data (IS_DATA_MOCKED=true)
+#   2. Set real GitHub token (NUXT_GITHUB_TOKEN)
+#   3. Configure PostgreSQL + enable historical mode (ENABLE_HISTORICAL_MODE=true)
+#   4. Enable sync (SYNC_ENABLED=true)
+#
+# See MIGRATION_GUIDE.md for full details
+# ==============================================================================

.github/workflows/deploy_to_ghcr.yml

@@ -55,6 +55,45 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           push: true
 
+  push_sync_to_ghcr:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout GitHub Action
+        uses: actions/checkout@v4
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ghcr.io/${{ github.repository }}-sync
+          labels: |
+            org.opencontainers.image.title=copilot-metrics-viewer-sync
+            org.opencontainers.image.description=Sync service for GitHub Copilot metrics data
+            org.opencontainers.image.source=${{ github.repository }}
+          tags: |
+            type=sha
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Build and push sync image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: Dockerfile.sync
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          push: true
+
   deploy:
     runs-on: ubuntu-latest
     needs: push_to_ghcr

.github/workflows/deploy_to_ghcr_tag_release.yaml

@@ -102,3 +102,45 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           push: true
+
+  push_sync_to_ghcr:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout GitHub Action
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.release.tag_name }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ghcr.io/${{ github.repository }}-sync
+          labels: |
+            org.opencontainers.image.title=copilot-metrics-viewer-sync
+            org.opencontainers.image.description=Sync service for GitHub Copilot metrics data
+            org.opencontainers.image.source=${{ github.repository }}
+          tags: |
+            type=raw,value=latest,enable=true
+            type=sha
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Build and push sync image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: Dockerfile.sync
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          push: true

.github/workflows/playwright.yml

@@ -1,4 +1,6 @@
 name: Playwright Tests
+permissions:
+  contents: read
 on:
   push:
     branches: [ main, master ]
@@ -19,6 +21,20 @@ jobs:
         load: true
         tags: ui:test
 
+  test-docker-sync:
+    timeout-minutes: 15
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+    - name: Build sync image
+      uses: docker/build-push-action@v6
+      with:
+        file: Dockerfile.sync
+        load: true
+        tags: sync:test
+
   test:
     timeout-minutes: 10
     permissions:
@@ -44,7 +60,7 @@ jobs:
           NUXT_SESSION_PASSWORD=foo-foo-foo-foo-foo-foo-foo-foo-foo-foo-foo-foo \
           NUXT_PUBLIC_GITHUB_ORG=octodemo \
           NUXT_PUBLIC_IS_DATA_MOCKED=true \
-          RUN_COMMAND='npm run preview' npm run test:e2e
+          RUN_COMMAND='npm run preview' npx playwright test --grep-invert "@seed|@storage"
     - uses: actions/upload-artifact@v4
       if: ${{ !cancelled() }}
       with:
@@ -72,10 +88,84 @@ jobs:
           -e NUXT_SESSION_PASSWORD=foo-foo-foo-foo-foo-foo-foo-foo-foo-foo-foo-foo \
           -e NUXT_PUBLIC_GITHUB_ORG=octodemo \
           -e NUXT_PUBLIC_IS_DATA_MOCKED=true \
-          app:pw
+          app:pw npx playwright test --grep-invert "@seed|@storage"
     - uses: actions/upload-artifact@v4
       if: always()
       with:
         name: playwright-report-docker
         path: test-results-docker/
         retention-days: 30
+
+  test-storage-pipeline:
+    timeout-minutes: 15
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:16-alpine
+        env:
+          POSTGRES_USER: testuser
+          POSTGRES_PASSWORD: testpass
+          POSTGRES_DB: copilot_metrics
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-node@v4
+      with:
+        node-version: lts/*
+    - name: Install dependencies
+      run: npm ci
+    - name: Build the app
+      run: npm run build
+    - name: Install Playwright Browsers
+      run: npx playwright install --with-deps chromium
+    - name: "Phase 1: Seed DB with mock data"
+      run: |
+        NUXT_SESSION_PASSWORD=foo-foo-foo-foo-foo-foo-foo-foo-foo-foo-foo-foo \
+        NUXT_PUBLIC_GITHUB_ORG=storage-test-org \
+        NUXT_PUBLIC_IS_DATA_MOCKED=true \
+        ENABLE_HISTORICAL_MODE=true \
+        DATABASE_URL=postgresql://testuser:testpass@localhost:5432/copilot_metrics \
+        node .output/server/index.mjs &
+        SERVER_PID=$!
+
+        # Wait for server to be ready
+        for i in $(seq 1 30); do
+          curl -sf http://localhost:3000/api/health > /dev/null 2>&1 && break
+          sleep 1
+        done
+
+        CI=true npx playwright test --grep "@seed" --project=chromium
+
+        kill $SERVER_PID
+        wait $SERVER_PID 2>/dev/null || true
+    - name: "Phase 2: Verify dashboard reads from DB"
+      run: |
+        NUXT_SESSION_PASSWORD=foo-foo-foo-foo-foo-foo-foo-foo-foo-foo-foo-foo \
+        NUXT_PUBLIC_GITHUB_ORG=storage-test-org \
+        NUXT_PUBLIC_IS_DATA_MOCKED=false \
+        ENABLE_HISTORICAL_MODE=true \
+        DATABASE_URL=postgresql://testuser:testpass@localhost:5432/copilot_metrics \
+        node .output/server/index.mjs &
+        SERVER_PID=$!
+
+        for i in $(seq 1 30); do
+          curl -sf http://localhost:3000/api/health > /dev/null 2>&1 && break
+          sleep 1
+        done
+
+        CI=true npx playwright test --grep "@storage" --project=chromium
+
+        kill $SERVER_PID
+        wait $SERVER_PID 2>/dev/null || true
+    - uses: actions/upload-artifact@v4
+      if: ${{ !cancelled() }}
+      with:
+        name: playwright-report-storage
+        path: playwright-report/
+        retention-days: 30

.gitignore

@@ -29,4 +29,5 @@ playwright-logs
 
 # Test results
 test-results
+test-results-old
 results.xml

.nuxtrc

@@ -0,0 +1 @@
+setups.@nuxt/test-utils="3.23.0"