From d799aba6e98d3b8aff1ca94f01138ab4ec283e1d Mon Sep 17 00:00:00 2001 From: Rex P Date: Mon, 29 Dec 2025 13:16:09 +1100 Subject: [PATCH 1/6] ci: Add snapshot tests for changes in the API --- gcp/api/cloudbuild.yaml | 8 +++ gcp/api/run_apitester.py | 55 +++++++++++++++++++ gcp/api/run_tests_e2e.sh | 28 ++++++++++ tools/apitester/README.md | 3 +- .../__snapshots__/cassette_TestCommand.snap | 16 +++--- .../cassette_TestCommand_CallAnalysis.snap | 6 +- .../cassette_TestCommand_Transitive.snap | 4 +- .../__snapshots__/cassette_single_query.snap | 10 +++- tools/apitester/internal/vcr/interactions.go | 1 + 9 files changed, 114 insertions(+), 17 deletions(-) create mode 100644 gcp/api/run_apitester.py create mode 100755 gcp/api/run_tests_e2e.sh diff --git a/gcp/api/cloudbuild.yaml b/gcp/api/cloudbuild.yaml index 7190530a129..d27f2b9e47a 100644 --- a/gcp/api/cloudbuild.yaml +++ b/gcp/api/cloudbuild.yaml @@ -43,4 +43,12 @@ steps: - CLOUDBUILD=1 waitFor: ['init', 'sync'] +- name: 'gcr.io/oss-vdb/ci' + id: 'api-snapshot-tests' + dir: gcp/api + args: ['bash', '-ex', 'run_tests_e2e.sh', '/workspace/dummy.json'] + env: + - CLOUDBUILD=1 + waitFor: ['init', 'sync'] + timeout: 7200s diff --git a/gcp/api/run_apitester.py b/gcp/api/run_apitester.py new file mode 100644 index 00000000000..007803d34fc --- /dev/null +++ b/gcp/api/run_apitester.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +import os +import sys +import subprocess +import time +import test_server + +_PORT = 8080 + +def main(): + if len(sys.argv) < 2: + print(f'Usage: {sys.argv[0]} path/to/credential.json') + sys.exit(1) + + credential_path = sys.argv[1] + + # Ensure Docker image is pulled + subprocess.run( + ['docker', 'pull', 'gcr.io/endpoints-release/endpoints-runtime:2'], + check=True) + + print("Starting test server...") + server = test_server.start(credential_path, port=_PORT) + + # Wait for server to start up + time.sleep(10) + + try: + # Determine API URL + if os.getenv('CLOUDBUILD'): + host = test_server.get_cloudbuild_esp_host() + else: + host = 'localhost' + + api_base_url = f"{host}:{_PORT}" + print(f"Running Go tests against {api_base_url}") + + env = os.environ.copy() + env['OSV_API_BASE_URL'] = api_base_url + + # Go tests path + # Assuming this script is in gcp/api/ + go_test_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../tools/apitester')) + + cmd = ['go', 'test', './...'] + print(f"Executing: {' '.join(cmd)} in {go_test_dir}") + + subprocess.run(cmd, cwd=go_test_dir, env=env, check=True) + + finally: + print("Stopping test server...") + server.stop() + +if __name__ == '__main__': + main() diff --git a/gcp/api/run_tests_e2e.sh b/gcp/api/run_tests_e2e.sh new file mode 100755 index 00000000000..0ebd0bd8bdf --- /dev/null +++ b/gcp/api/run_tests_e2e.sh @@ -0,0 +1,28 @@ +#!/bin/bash -x +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "Usage: $0 /path/to/credential.json" + exit 1 +fi + +export GOOGLE_CLOUD_PROJECT=oss-vdb-test OSV_VULNERABILITIES_BUCKET=osv-test-vulnerabilities + +# Try to start docker if not running (mostly for CI) +service docker start || true + +set -e + +poetry run python run_apitester.py "$1" diff --git a/tools/apitester/README.md b/tools/apitester/README.md index 6b7d142f140..719ae68737a 100644 --- a/tools/apitester/README.md +++ b/tools/apitester/README.md @@ -46,4 +46,5 @@ Before the test suite is actually run, the cassettes will be "cleaned" so that - the `response` is property is not present, to reduce the size of each cassette By default, requests are made against the local instance of the API, but you can -use the `OSV_API_BASE_URL` to point it against other instances. +use the `OSV_API_BASE_URL` to point it against other instances. +E.g. `OSV_API_BASE_URL=api.test.osv.dev go test ./...` diff --git a/tools/apitester/__snapshots__/cassette_TestCommand.snap b/tools/apitester/__snapshots__/cassette_TestCommand.snap index 59066590159..cb8a161e484 100755 --- a/tools/apitester/__snapshots__/cassette_TestCommand.snap +++ b/tools/apitester/__snapshots__/cassette_TestCommand.snap @@ -3141,11 +3141,11 @@ }, { "id": "PYSEC-2023-192", - "modified": "" + "modified": "" }, { "id": "PYSEC-2023-212", - "modified": "" + "modified": "" } ] }, @@ -3189,11 +3189,11 @@ }, { "id": "PYSEC-2023-192", - "modified": "" + "modified": "" }, { "id": "PYSEC-2023-212", - "modified": "" + "modified": "" } ] }, @@ -3619,11 +3619,11 @@ }, { "id": "PYSEC-2023-192", - "modified": "" + "modified": "" }, { "id": "PYSEC-2023-212", - "modified": "" + "modified": "" } ] }, @@ -3667,11 +3667,11 @@ }, { "id": "PYSEC-2023-192", - "modified": "" + "modified": "" }, { "id": "PYSEC-2023-212", - "modified": "" + "modified": "" } ] }, diff --git a/tools/apitester/__snapshots__/cassette_TestCommand_CallAnalysis.snap b/tools/apitester/__snapshots__/cassette_TestCommand_CallAnalysis.snap index c7fa922010d..012857de860 100755 --- a/tools/apitester/__snapshots__/cassette_TestCommand_CallAnalysis.snap +++ b/tools/apitester/__snapshots__/cassette_TestCommand_CallAnalysis.snap @@ -6,7 +6,7 @@ "vulns": [ { "id": "GHSA-c3h9-896r-86jm", - "modified": "" + "modified": "" }, { "id": "GO-2021-0053", @@ -74,7 +74,7 @@ "vulns": [ { "id": "GHSA-c3h9-896r-86jm", - "modified": "" + "modified": "" }, { "id": "GO-2021-0053", @@ -94,7 +94,7 @@ "vulns": [ { "id": "GHSA-c3h9-896r-86jm", - "modified": "" + "modified": "" }, { "id": "GO-2021-0053", diff --git a/tools/apitester/__snapshots__/cassette_TestCommand_Transitive.snap b/tools/apitester/__snapshots__/cassette_TestCommand_Transitive.snap index 66c1c427573..59adbdea6f7 100755 --- a/tools/apitester/__snapshots__/cassette_TestCommand_Transitive.snap +++ b/tools/apitester/__snapshots__/cassette_TestCommand_Transitive.snap @@ -634,11 +634,11 @@ }, { "id": "PYSEC-2023-192", - "modified": "" + "modified": "" }, { "id": "PYSEC-2023-212", - "modified": "" + "modified": "" } ] }, diff --git a/tools/apitester/__snapshots__/cassette_single_query.snap b/tools/apitester/__snapshots__/cassette_single_query.snap index 680b407c069..5181c5d1562 100755 --- a/tools/apitester/__snapshots__/cassette_single_query.snap +++ b/tools/apitester/__snapshots__/cassette_single_query.snap @@ -1467,7 +1467,6 @@ "related": [ "ALSA-2025:1671", "ALSA-2025:1673", - "CGA-962m-89hc-rmjq", "RLSA-2025:1673", "SUSE-SU-2024:2784-1", "SUSE-SU-2024:2930-1", @@ -1546,7 +1545,6 @@ "modified": "", "published": "2024-09-11T10:15:02.883Z", "related": [ - "CGA-g55g-qx76-5fjj", "SUSE-SU-2024:3202-1", "SUSE-SU-2024:3203-1", "SUSE-SU-2024:3204-1", @@ -1710,6 +1708,7 @@ "modified": "", "published": "2025-02-05T10:15:22.710Z", "related": [ + "CGA-gr5c-pjrp-3fmw", "MGASA-2025-0123", "SUSE-SU-2025:0369-1", "SUSE-SU-2025:0370-1", @@ -1768,7 +1767,11 @@ "aliases": ["CURL-CVE-2025-0665"], "modified": "", "published": "2025-02-05T10:15:22.857Z", - "related": ["MGASA-2025-0123", "openSUSE-SU-2025:14809-1"], + "related": [ + "CGA-h2f8-6v5h-2qcp", + "MGASA-2025-0123", + "openSUSE-SU-2025:14809-1" + ], "references": [ { "type": "ADVISORY", @@ -1830,6 +1833,7 @@ "modified": "", "published": "2025-02-05T10:15:22.980Z", "related": [ + "CGA-378j-cghq-mmhg", "MGASA-2025-0123", "SUSE-SU-2025:0369-1", "SUSE-SU-2025:0370-1", diff --git a/tools/apitester/internal/vcr/interactions.go b/tools/apitester/internal/vcr/interactions.go index b61b1bdbc6b..a5bd12ab00a 100644 --- a/tools/apitester/internal/vcr/interactions.go +++ b/tools/apitester/internal/vcr/interactions.go @@ -32,6 +32,7 @@ func Play(t *testing.T, interaction *cassette.Interaction) *http.Response { } req.URL.Host = fetchAPIBaseURL() + req.Host = req.URL.Host req.Header.Set("User-Agent", "osv.dev/apitester") req.ContentLength = -1 From dd78e5ba0d98be8e06ae5d9bddf1297fa26cba9e Mon Sep 17 00:00:00 2001 From: Rex P Date: Mon, 29 Dec 2025 13:30:55 +1100 Subject: [PATCH 2/6] Format and lintttt --- gcp/api/run_apitester.py | 89 +++++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 43 deletions(-) diff --git a/gcp/api/run_apitester.py b/gcp/api/run_apitester.py index 007803d34fc..f8f261094f2 100644 --- a/gcp/api/run_apitester.py +++ b/gcp/api/run_apitester.py @@ -7,49 +7,52 @@ _PORT = 8080 + def main(): - if len(sys.argv) < 2: - print(f'Usage: {sys.argv[0]} path/to/credential.json') - sys.exit(1) - - credential_path = sys.argv[1] - - # Ensure Docker image is pulled - subprocess.run( - ['docker', 'pull', 'gcr.io/endpoints-release/endpoints-runtime:2'], - check=True) - - print("Starting test server...") - server = test_server.start(credential_path, port=_PORT) - - # Wait for server to start up - time.sleep(10) - - try: - # Determine API URL - if os.getenv('CLOUDBUILD'): - host = test_server.get_cloudbuild_esp_host() - else: - host = 'localhost' - - api_base_url = f"{host}:{_PORT}" - print(f"Running Go tests against {api_base_url}") - - env = os.environ.copy() - env['OSV_API_BASE_URL'] = api_base_url - - # Go tests path - # Assuming this script is in gcp/api/ - go_test_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../tools/apitester')) - - cmd = ['go', 'test', './...'] - print(f"Executing: {' '.join(cmd)} in {go_test_dir}") - - subprocess.run(cmd, cwd=go_test_dir, env=env, check=True) - - finally: - print("Stopping test server...") - server.stop() + if len(sys.argv) < 2: + print(f'Usage: {sys.argv[0]} path/to/credential.json') + sys.exit(1) + + credential_path = sys.argv[1] + + # Ensure Docker image is pulled + subprocess.run( + ['docker', 'pull', 'gcr.io/endpoints-release/endpoints-runtime:2'], + check=True) + + print("Starting test server...") + server = test_server.start(credential_path, port=_PORT) + + # Wait for server to start up + time.sleep(10) + + try: + # Determine API URL + if os.getenv('CLOUDBUILD'): + host = test_server.get_cloudbuild_esp_host() + else: + host = 'localhost' + + api_base_url = f"{host}:{_PORT}" + print(f"Running Go tests against {api_base_url}") + + env = os.environ.copy() + env['OSV_API_BASE_URL'] = api_base_url + + # Go tests path + # Assuming this script is in gcp/api/ + go_test_dir = os.path.abspath( + os.path.join(os.path.dirname(__file__), '../../tools/apitester')) + + cmd = ['go', 'test', './...'] + print(f"Executing: {' '.join(cmd)} in {go_test_dir}") + + subprocess.run(cmd, cwd=go_test_dir, env=env, check=True) + + finally: + print("Stopping test server...") + server.stop() + if __name__ == '__main__': - main() + main() From d803086099b25256c3bc2694cb2726a86043a849 Mon Sep 17 00:00:00 2001 From: Rex P Date: Mon, 29 Dec 2025 14:30:24 +1100 Subject: [PATCH 3/6] Fix more lints --- gcp/api/run_apitester.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/gcp/api/run_apitester.py b/gcp/api/run_apitester.py index f8f261094f2..281063a320b 100644 --- a/gcp/api/run_apitester.py +++ b/gcp/api/run_apitester.py @@ -1,4 +1,18 @@ -#!/usr/bin/env python3 +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Run E2E golang cassette API tests.""" + import os import sys import subprocess From e9dfdb78e422575acc82491fc4c9ed76c31873fb Mon Sep 17 00:00:00 2001 From: Rex P Date: Mon, 29 Dec 2025 14:50:41 +1100 Subject: [PATCH 4/6] Do we actually need cloudbuild esp url? --- gcp/api/run_apitester.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/gcp/api/run_apitester.py b/gcp/api/run_apitester.py index 281063a320b..8d9c996f0cf 100644 --- a/gcp/api/run_apitester.py +++ b/gcp/api/run_apitester.py @@ -42,10 +42,7 @@ def main(): try: # Determine API URL - if os.getenv('CLOUDBUILD'): - host = test_server.get_cloudbuild_esp_host() - else: - host = 'localhost' + host = 'localhost' api_base_url = f"{host}:{_PORT}" print(f"Running Go tests against {api_base_url}") From cf363328c83a8894fd359a699895fe01165d4095 Mon Sep 17 00:00:00 2001 From: Rex P Date: Mon, 29 Dec 2025 15:00:45 +1100 Subject: [PATCH 5/6] Ok maybe it was there for a reason --- gcp/api/run_apitester.py | 5 ++++- tools/apitester/internal/vcr/interactions.go | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/gcp/api/run_apitester.py b/gcp/api/run_apitester.py index 8d9c996f0cf..281063a320b 100644 --- a/gcp/api/run_apitester.py +++ b/gcp/api/run_apitester.py @@ -42,7 +42,10 @@ def main(): try: # Determine API URL - host = 'localhost' + if os.getenv('CLOUDBUILD'): + host = test_server.get_cloudbuild_esp_host() + else: + host = 'localhost' api_base_url = f"{host}:{_PORT}" print(f"Running Go tests against {api_base_url}") diff --git a/tools/apitester/internal/vcr/interactions.go b/tools/apitester/internal/vcr/interactions.go index a5bd12ab00a..c3d41e49961 100644 --- a/tools/apitester/internal/vcr/interactions.go +++ b/tools/apitester/internal/vcr/interactions.go @@ -3,6 +3,7 @@ package vcr import ( "net/http" "os" + "strings" "testing" "gopkg.in/dnaeon/go-vcr.v4/pkg/cassette" @@ -36,7 +37,7 @@ func Play(t *testing.T, interaction *cassette.Interaction) *http.Response { req.Header.Set("User-Agent", "osv.dev/apitester") req.ContentLength = -1 - if req.URL.Hostname() == "localhost" || req.URL.Hostname() == "127.0.0.1" { + if req.URL.Hostname() == "localhost" || req.URL.Hostname() == "127.0.0.1" || strings.HasPrefix(req.URL.Hostname(), "192.168.") { req.URL.Scheme = "http" } From 6e311ab173e01117a50c9af4cb3905c41b6ed4ef Mon Sep 17 00:00:00 2001 From: Rex P Date: Mon, 29 Dec 2025 15:09:51 +1100 Subject: [PATCH 6/6] Sequential run? --- gcp/api/cloudbuild.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gcp/api/cloudbuild.yaml b/gcp/api/cloudbuild.yaml index d27f2b9e47a..754459fbf7e 100644 --- a/gcp/api/cloudbuild.yaml +++ b/gcp/api/cloudbuild.yaml @@ -49,6 +49,7 @@ steps: args: ['bash', '-ex', 'run_tests_e2e.sh', '/workspace/dummy.json'] env: - CLOUDBUILD=1 - waitFor: ['init', 'sync'] + # Don't run at the same time as api-tests + waitFor: ['init', 'sync', 'api-tests'] timeout: 7200s