From a5b3a710e9ec9a61542198c2529434017aef374e Mon Sep 17 00:00:00 2001 From: michaelkedar Date: Wed, 17 Jun 2026 06:02:58 +0000 Subject: [PATCH 1/5] final migration --- .gitignore | 1 + AGENTS.md | 12 +++--- Makefile | 29 +++++++++----- deployment/build-and-stage.yaml | 24 ++++++----- deployment/clouddeploy/osv-api/run-prod.yaml | 4 ++ .../clouddeploy/osv-api/run-staging.yaml | 6 +++ .../IntegrationTests_api_query_response.txt | 40 +++++++++---------- gcp/api/integration_tests.py | 6 +-- gcp/api/test_server.py | 14 +++++-- go/cmd/api-devserver/main.go | 1 + go/cmd/api/main.go | 17 ++++++-- go/internal/api/query_affected.go | 3 ++ go/internal/api/server.go | 9 +++++ 13 files changed, 110 insertions(+), 56 deletions(-) diff --git a/.gitignore b/.gitignore index 8b232e74d70..e6b838c720d 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,5 @@ temp/* gcp/api/v1/osv/** .hypothesis go/api-devserver +go/api esp.log \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 5cbe47518ae..b84c1cbfa7a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -163,7 +163,7 @@ Many tests use expected outputs saved directly in the source tree: ### Local API Server Development (Go-native) - To run the public OSV API server locally using the native Go implementation alongside the ESPv2 proxy (which transcodes HTTP/JSON REST requests to gRPC): ```bash - make run-go-api + make run-api-server ``` --- @@ -246,11 +246,11 @@ The `osv` folder is a shared Python package. Since the primary API server and so ## GCP Component Architecture (`gcp/`) Contains deployment setups, workers running in GKE, Cloud Functions, and the user-facing website and API. -### 1. API Server (`gcp/api/`) -- **Status**: **Active (Python)**. -- Serves the public HTTP API for querying vulnerabilities by package or version. -- **Deployment Target**: **Google Cloud Run** (managed via Cloud Deploy pipeline `osv-api`). -- *Note*: Plans exist to migrate this to Go in the near future, but it currently remains in Python. +### 1. API Server (`go/cmd/api/`) +- **Status**: **Active (Go)**. +- Serves the public OSV gRPC API server (transcoded to HTTP/JSON REST via ESPv2). +- **Deployment Target**: **Google Cloud Run** (managed via Cloud Deploy pipeline `osv-api` deploying to `osv-grpc-backend`). +- *Note*: Fully migrated from Python to Go. The legacy Python implementation remains in `gcp/api/` but is retired. ### 2. Website (`gcp/website/`) - **Status**: **Active**. diff --git a/Makefile b/Makefile index 3f61fc74532..1ea832aa6ff 100644 --- a/Makefile +++ b/Makefile @@ -45,14 +45,16 @@ go-tests: api-server-tests: test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1) + cd go && go build -o ./api ./cmd/api cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest . - cd gcp/api && ./run_tests.sh $(HOME)/.config/gcloud/application_default_credentials.json - cd gcp/api && ./run_tests_e2e.sh $(HOME)/.config/gcloud/application_default_credentials.json + cd gcp/api && OSV_USE_GO_BACKEND=1 ./run_tests.sh $(HOME)/.config/gcloud/application_default_credentials.json + cd gcp/api && OSV_USE_GO_BACKEND=1 ./run_tests_e2e.sh $(HOME)/.config/gcloud/application_default_credentials.json update-api-snapshots: test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1) + cd go && go build -o ./api ./cmd/api cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest . - cd gcp/api && UPDATE_SNAPS=true ./run_tests_e2e.sh $(HOME)/.config/gcloud/application_default_credentials.json + cd gcp/api && UPDATE_SNAPS=true OSV_USE_GO_BACKEND=1 ./run_tests_e2e.sh $(HOME)/.config/gcloud/application_default_credentials.json lint: GOTOOLCHAIN=go1.26.3 $(run-cmd) tools/lint_and_format.sh @@ -103,20 +105,27 @@ run-website-emulator: cd gcp/website/blog && hugo --buildFuture -d ../dist/static/blog cd gcp/website && $(install-cmd) && DATASTORE_EMULATOR_PORT=5002 $(run-cmd) python frontend_emulator.py +# Run with `make run-api-server ARGS=--no-backend` to launch esp without backend. +# Run the Go developer server orchestrator (launches both ESPv2 and the Go API server). # Run with `make run-api-server ARGS=--no-backend` to launch esp without backend. run-api-server: - test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1) - cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest . - cd gcp/api && $(install-cmd) && GOOGLE_CLOUD_PROJECT=oss-vdb OSV_VULNERABILITIES_BUCKET=osv-vulnerabilities $(run-cmd) python test_server.py $(HOME)/.config/gcloud/application_default_credentials.json $(ARGS) - -# Run the native Go developer server orchestrator (launches both ESPv2 and the Go API server). -# Run with `run-go-api ARGS=--no-backend` to launch esp without backend. -run-go-api: test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1) docker inspect osv/esp:latest >/dev/null 2>&1 || (cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest .) @cd go && go build -o ./api-devserver ./cmd/api-devserver && (GOOGLE_CLOUD_PROJECT=oss-vdb OSV_VULNERABILITIES_BUCKET=osv-vulnerabilities ./api-devserver $(ARGS); EXIT_CODE=$$?; rm -f ./api-devserver; exit $$EXIT_CODE) +# Run the Go developer server orchestrator against the staging/test environment. run-api-server-test: + test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1) + docker inspect osv/esp:latest >/dev/null 2>&1 || (cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest .) + @cd go && go build -o ./api-devserver ./cmd/api-devserver && (GOOGLE_CLOUD_PROJECT=oss-vdb-test OSV_VULNERABILITIES_BUCKET=osv-test-vulnerabilities ./api-devserver $(ARGS); EXIT_CODE=$$?; rm -f ./api-devserver; exit $$EXIT_CODE) + +# Legacy Python API server targets +run-python-api-server: + test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1) + cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest . + cd gcp/api && $(install-cmd) && GOOGLE_CLOUD_PROJECT=oss-vdb OSV_VULNERABILITIES_BUCKET=osv-vulnerabilities $(run-cmd) python test_server.py $(HOME)/.config/gcloud/application_default_credentials.json $(ARGS) + +run-python-api-server-test: test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1) cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest . cd gcp/api && $(install-cmd) && GOOGLE_CLOUD_PROJECT=oss-vdb-test OSV_VULNERABILITIES_BUCKET=osv-test-vulnerabilities $(run-cmd) python test_server.py $(HOME)/.config/gcloud/application_default_credentials.json $(ARGS) diff --git a/deployment/build-and-stage.yaml b/deployment/build-and-stage.yaml index 6f1dacbb37b..522a39ac740 100644 --- a/deployment/build-and-stage.yaml +++ b/deployment/build-and-stage.yaml @@ -210,6 +210,20 @@ steps: args: ['push', '--all-tags', 'gcr.io/oss-vdb/gitter'] waitFor: ['build-gitter', 'cloud-build-queue'] +- name: gcr.io/cloud-builders/docker + entrypoint: 'bash' + args: ['-c', 'docker pull gcr.io/oss-vdb/osv-server:latest || exit 0'] + id: 'pull-osv-server' + waitFor: ['setup'] +- name: gcr.io/cloud-builders/docker + args: ['buildx', 'build', '-t', 'gcr.io/oss-vdb/osv-server:latest', '-t', 'gcr.io/oss-vdb/osv-server:$COMMIT_SHA', '--target', 'api', '--build-context', 'bindings=../bindings', '-f', 'Dockerfile', '--cache-from', 'gcr.io/oss-vdb/osv-server:latest', '--pull', '.'] + dir: 'go' + id: 'build-osv-server' + waitFor: ['pull-osv-server', 'build-gitter'] +- name: gcr.io/cloud-builders/docker + args: ['push', '--all-tags', 'gcr.io/oss-vdb/osv-server'] + waitFor: ['build-osv-server', 'cloud-build-queue'] + # Build/push staging-api-test images to gcr.io/oss-vdb-test. - name: gcr.io/cloud-builders/docker args: ['build', '-t', 'gcr.io/oss-vdb-test/staging-api-test:latest', '-t', 'gcr.io/oss-vdb-test/staging-api-test:$COMMIT_SHA', '.'] @@ -352,16 +366,6 @@ steps: args: ['push', '--all-tags', 'gcr.io/oss-vdb/run_first_package_finder'] waitFor: ['build-first-package-finder', 'cloud-build-queue'] - -# Build/push api backend -- name: 'gcr.io/cloud-builders/docker' - args: ['build', '-t', 'gcr.io/oss-vdb/osv-server:latest', '-t', 'gcr.io/oss-vdb/osv-server:$COMMIT_SHA', '-f', 'gcp/api/Dockerfile', '.'] - id: 'build-osv-server' - waitFor: ['setup'] -- name: 'gcr.io/cloud-builders/docker' - args: ['push', '--all-tags', 'gcr.io/oss-vdb/osv-server'] - waitFor: ['build-osv-server', 'cloud-build-queue'] - # Build/push Debian copyright mirror image - name: gcr.io/cloud-builders/docker args: ['build', '-t', 'gcr.io/oss-vdb/debian-copyright-mirror:latest', '-t', 'gcr.io/oss-vdb/debian-copyright-mirror:$COMMIT_SHA', '.'] diff --git a/deployment/clouddeploy/osv-api/run-prod.yaml b/deployment/clouddeploy/osv-api/run-prod.yaml index a35228df38b..8fb560a67fd 100644 --- a/deployment/clouddeploy/osv-api/run-prod.yaml +++ b/deployment/clouddeploy/osv-api/run-prod.yaml @@ -11,8 +11,12 @@ spec: containers: - image: osv-server env: + - name: GOOGLE_CLOUD_PROJECT + value: oss-vdb - name: OSV_VULNERABILITIES_BUCKET value: osv-vulnerabilities + - name: FAILED_TASKS_TOPIC + value: failed-tasks resources: limits: cpu: 2 diff --git a/deployment/clouddeploy/osv-api/run-staging.yaml b/deployment/clouddeploy/osv-api/run-staging.yaml index 27c17dc5117..00c67f4f159 100644 --- a/deployment/clouddeploy/osv-api/run-staging.yaml +++ b/deployment/clouddeploy/osv-api/run-staging.yaml @@ -11,8 +11,14 @@ spec: containers: - image: osv-server env: + - name: GOOGLE_CLOUD_PROJECT + value: oss-vdb-test - name: OSV_VULNERABILITIES_BUCKET value: osv-test-vulnerabilities + - name: FAILED_TASKS_TOPIC + value: failed-tasks + - name: OSV_VERBOSE_LOGGING + value: "true" resources: limits: cpu: 2 diff --git a/gcp/api/fixtures/IntegrationTests_api_query_response.txt b/gcp/api/fixtures/IntegrationTests_api_query_response.txt index 2ae4c52f345..2104a2792d9 100644 --- a/gcp/api/fixtures/IntegrationTests_api_query_response.txt +++ b/gcp/api/fixtures/IntegrationTests_api_query_response.txt @@ -1,25 +1,25 @@ [ '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "crossbeam-utils", "purl": ' '"pkg:cargo/crossbeam-utils"}, "version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "crossbeam-utils", "purl": ' '"pkg:cargo/crossbeam-utils"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "crossbeam-utils", "purl": ' '"pkg:cargo/crossbeam-utils@0.8.5"}, "version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "crossbeam-utils", "purl": ' '"pkg:cargo/crossbeam-utils@0.8.5"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "crossbeam-utils", "purl": ' '"pkg:pypi/numpy"}, "version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "crossbeam-utils", "purl": ' '"pkg:pypi/numpy"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "crossbeam-utils", "purl": ' '"pkg:pypi/numpy@8.24.0"}, "version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "crossbeam-utils", "purl": ' '"pkg:pypi/numpy@8.24.0"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' @@ -30,24 +30,24 @@ '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "numpy", "purl": ' '"pkg:cargo/crossbeam-utils"}, "version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "numpy", "purl": ' '"pkg:cargo/crossbeam-utils"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "numpy", "purl": ' '"pkg:cargo/crossbeam-utils@0.8.5"}, "version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "numpy", "purl": ' '"pkg:cargo/crossbeam-utils@0.8.5"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "numpy", "purl": "pkg:pypi/numpy"}, ' '"version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "numpy", "purl": "pkg:pypi/numpy"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "numpy", "purl": ' '"pkg:pypi/numpy@8.24.0"}, "version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "name": "numpy", "purl": ' '"pkg:pypi/numpy@8.24.0"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' @@ -57,22 +57,22 @@ '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "purl": "pkg:cargo/crossbeam-utils"}, ' '"version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "purl": "pkg:cargo/crossbeam-utils"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "purl": "pkg:cargo/crossbeam-utils@0.8.5"}, ' '"version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "purl": "pkg:cargo/crossbeam-utils@0.8.5"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "purl": "pkg:pypi/numpy"}, "version": ' '"0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "purl": "pkg:pypi/numpy"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "purl": "pkg:pypi/numpy@8.24.0"}, "version": ' '"0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io", "purl": "pkg:pypi/numpy@8.24.0"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"ecosystem": "crates.io"}, "version": "0.8.5"}\n', @@ -81,22 +81,22 @@ '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "crossbeam-utils", "purl": "pkg:cargo/crossbeam-utils"}, ' '"version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "crossbeam-utils", "purl": "pkg:cargo/crossbeam-utils"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "crossbeam-utils", "purl": "pkg:cargo/crossbeam-utils@0.8.5"}, ' '"version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "crossbeam-utils", "purl": "pkg:cargo/crossbeam-utils@0.8.5"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "crossbeam-utils", "purl": "pkg:pypi/numpy"}, "version": ' '"0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "crossbeam-utils", "purl": "pkg:pypi/numpy"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "crossbeam-utils", "purl": "pkg:pypi/numpy@8.24.0"}, "version": ' '"0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "crossbeam-utils", "purl": "pkg:pypi/numpy@8.24.0"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "crossbeam-utils"}, "version": "0.8.5"}\n', @@ -105,20 +105,20 @@ '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "numpy", "purl": "pkg:cargo/crossbeam-utils"}, "version": ' '"0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "numpy", "purl": "pkg:cargo/crossbeam-utils"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "numpy", "purl": "pkg:cargo/crossbeam-utils@0.8.5"}, "version": ' '"0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "numpy", "purl": "pkg:cargo/crossbeam-utils@0.8.5"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "numpy", "purl": "pkg:pypi/numpy"}, "version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "numpy", "purl": "pkg:pypi/numpy"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "numpy", "purl": "pkg:pypi/numpy@8.24.0"}, "version": "0.8.5"}\n', - '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' + '200:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "numpy", "purl": "pkg:pypi/numpy@8.24.0"}}\n', '400:{"commit": "d374094d8c49b6b7d288f307e11217ec5a502391", "package": ' '{"name": "numpy"}, "version": "0.8.5"}\n', diff --git a/gcp/api/integration_tests.py b/gcp/api/integration_tests.py index abfe61bd573..9b2aa36255c 100644 --- a/gcp/api/integration_tests.py +++ b/gcp/api/integration_tests.py @@ -29,7 +29,7 @@ from osv import tests _PORT = 8080 -_TIMEOUT = 10 # Timeout for HTTP(S) requests +_TIMEOUT = 15 # Timeout for HTTP(S) requests _LONG_TESTS = os.getenv('LONG_TESTS') _TEST_DATA_DIR = 'fixtures' _BASE_QUERY = '/v1/query' @@ -399,7 +399,7 @@ def test_query_invalid_ecosystem(self): self.assert_results_equal({ 'code': 3, - 'message': 'Invalid ecosystem.' + 'message': 'invalid ecosystem' }, response.json()) def test_query_unknown_purl_invalid_semver(self): @@ -697,7 +697,7 @@ def test_get_vuln_by_alias_not_in_db(self): self.assert_results_equal( { 'code': 5, - 'message': 'Bug not found, but the following aliases were: ' + 'message': 'Vulnerability not found, but the following aliases were: ' 'MAL-2024-2291' }, response.json()) diff --git a/gcp/api/test_server.py b/gcp/api/test_server.py index ea7efbda0d0..7570f1b40ab 100644 --- a/gcp/api/test_server.py +++ b/gcp/api/test_server.py @@ -49,11 +49,17 @@ def start_backend(port, log_path): log_handle = open(log_path, 'w') env = os.environ.copy() + if os.getenv('OSV_USE_GO_BACKEND') == '1': + # Use the Go backend. We assume it has been compiled. + # Path is relative to gcp/api/ directory. + backend_cmd = ['../../go/api', f'--port={port}'] + print(f"Starting Go backend: {' '.join(backend_cmd)}") + else: + backend_cmd = [sys.executable, 'server.py', f'--port={port}'] + print(f"Starting Python backend: {' '.join(backend_cmd)}") + backend_proc = subprocess.Popen( - [sys.executable, 'server.py', f'--port={port}'], - env=env, - stdout=log_handle, - stderr=subprocess.STDOUT) + backend_cmd, env=env, stdout=log_handle, stderr=subprocess.STDOUT) return backend_proc diff --git a/go/cmd/api-devserver/main.go b/go/cmd/api-devserver/main.go index 7da15163458..b6e72a74dfc 100644 --- a/go/cmd/api-devserver/main.go +++ b/go/cmd/api-devserver/main.go @@ -205,6 +205,7 @@ func runBackend(ctx context.Context, port int) { repoIndexStore := db.NewRepoIndexStore(dbClient) if err := api.RunServer(ctx, api.ServerOptions{ Port: port, + Local: true, VerboseLogs: true, VulnStore: vulnStore, RelationsStore: relationsStore, diff --git a/go/cmd/api/main.go b/go/cmd/api/main.go index 60117288427..1cbeaa0c155 100644 --- a/go/cmd/api/main.go +++ b/go/cmd/api/main.go @@ -33,12 +33,22 @@ func run() error { logger.InitGlobalLogger() defer logger.Close() - port := flag.Int("port", 8000, "port for the OSV API") - flag.Parse() - ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() + defaultPort := 8000 + if portStr := os.Getenv("PORT"); portStr != "" { + if p, err := strconv.Atoi(portStr); err == nil { + defaultPort = p + } else { + logger.ErrorContext(ctx, "Invalid PORT environment variable, using default", slog.Any("error", err)) + } + } + + port := flag.Int("port", defaultPort, "port for the OSV API") + local := flag.Bool("local", false, "enable gRPC reflection for local debugging") + flag.Parse() + project := os.Getenv("GOOGLE_CLOUD_PROJECT") if project == "" { // Fallback to metadata server for Cloud Run @@ -126,6 +136,7 @@ func run() error { return api.RunServer(ctx, api.ServerOptions{ Port: *port, + Local: *local, VerboseLogs: verboseLogs, VulnStore: vulnStore, RelationsStore: relationsStore, diff --git a/go/internal/api/query_affected.go b/go/internal/api/query_affected.go index dc3bf5af3c7..d98580520ed 100644 --- a/go/internal/api/query_affected.go +++ b/go/internal/api/query_affected.go @@ -449,6 +449,9 @@ func (s *server) parseQuery(query *pb.Query) (parsedQueryInfo, error) { if qi.packageName == "" { return parsedQueryInfo{}, status.Error(codes.InvalidArgument, "invalid query") } + if qi.ecosystem == "" && qi.version == "" { + return parsedQueryInfo{}, status.Error(codes.InvalidArgument, "invalid query") + } return qi, nil } diff --git a/go/internal/api/server.go b/go/internal/api/server.go index 671cb185c0a..f9404c35a0e 100644 --- a/go/internal/api/server.go +++ b/go/internal/api/server.go @@ -13,6 +13,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/health" healthgrpc "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/reflection" pb "osv.dev/bindings/go/api" ) @@ -34,6 +35,8 @@ type server struct { type ServerOptions struct { Port int + // Local controls whether to enable gRPC server reflection. + Local bool // VerboseLogs controls whether to log verbose information, // including per-request data. VerboseLogs bool @@ -66,11 +69,17 @@ func RunServer(ctx context.Context, opts ServerOptions) error { }) healthServer := health.NewServer() + healthServer.SetServingStatus("osv.v1.OSV", healthgrpc.HealthCheckResponse_SERVING) + healthServer.SetServingStatus("", healthgrpc.HealthCheckResponse_SERVING) healthgrpc.RegisterHealthServer(s, healthServer) // Start background dependency health monitor go monitorDatabaseHealth(ctx, healthServer, opts.VulnStore, opts.HealthCheckInterval, opts.HealthCheckThreshold) + if opts.Local { + reflection.Register(s) + } + logger.InfoContext(ctx, "server listening", "port", opts.Port) serveErr := make(chan error, 1) From 1bfc2ee4560444d78701a1a0d13366919b6a89dd Mon Sep 17 00:00:00 2001 From: michaelkedar Date: Wed, 17 Jun 2026 06:16:36 +0000 Subject: [PATCH 2/5] lint --- gcp/api/integration_tests.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gcp/api/integration_tests.py b/gcp/api/integration_tests.py index 9b2aa36255c..dca149f4922 100644 --- a/gcp/api/integration_tests.py +++ b/gcp/api/integration_tests.py @@ -697,8 +697,9 @@ def test_get_vuln_by_alias_not_in_db(self): self.assert_results_equal( { 'code': 5, - 'message': 'Vulnerability not found, but the following aliases were: ' - 'MAL-2024-2291' + 'message': + 'Vulnerability not found, but the following aliases were: ' + 'MAL-2024-2291' }, response.json()) def test_query_batch(self): From 2ebeba568d8a4212985d241efb9be6f0ec7f78a2 Mon Sep 17 00:00:00 2001 From: michaelkedar Date: Wed, 17 Jun 2026 23:32:20 +0000 Subject: [PATCH 3/5] use go in cloud build --- gcp/api/cloudbuild.yaml | 1 + gcp/api/snapshot_tests.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/gcp/api/cloudbuild.yaml b/gcp/api/cloudbuild.yaml index 9e4d4c876bd..6fe1ab901fb 100644 --- a/gcp/api/cloudbuild.yaml +++ b/gcp/api/cloudbuild.yaml @@ -47,3 +47,4 @@ options: logging: CLOUD_LOGGING_ONLY env: - CLOUDBUILD=1 + - OSV_USE_GO_BACKEND=1 diff --git a/gcp/api/snapshot_tests.yaml b/gcp/api/snapshot_tests.yaml index fbcabdf22b2..14cf478f0b7 100644 --- a/gcp/api/snapshot_tests.yaml +++ b/gcp/api/snapshot_tests.yaml @@ -38,3 +38,4 @@ options: logging: CLOUD_LOGGING_ONLY env: - CLOUDBUILD=1 + - OSV_USE_GO_BACKEND=1 From d7531a18f856f26162770e83198966915cc58486 Mon Sep 17 00:00:00 2001 From: michaelkedar Date: Wed, 17 Jun 2026 23:49:41 +0000 Subject: [PATCH 4/5] build the go binary we're testing --- gcp/api/cloudbuild.yaml | 8 +++++++- gcp/api/snapshot_tests.yaml | 7 +++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/gcp/api/cloudbuild.yaml b/gcp/api/cloudbuild.yaml index 6fe1ab901fb..e0b5cb62c06 100644 --- a/gcp/api/cloudbuild.yaml +++ b/gcp/api/cloudbuild.yaml @@ -34,12 +34,18 @@ steps: args: ['poetry', 'sync'] waitFor: ['-'] +- name: 'gcr.io/oss-vdb/ci' + id: 'build-go-backend' + dir: go + args: ['go', 'build', '-o', 'api', './cmd/api'] + waitFor: ['init'] + - name: 'gcr.io/oss-vdb/ci' id: 'api-tests' dir: gcp/api #TODO: Update test scripts to support not supplying a credential. args: ['bash', '-ex', 'run_tests.sh', '/workspace/dummy.json'] - waitFor: ['init', 'sync'] + waitFor: ['build-go-backend', 'sync'] timeout: 7200s serviceAccount: 'projects/oss-vdb/serviceAccounts/api-e2e-tester@oss-vdb.iam.gserviceaccount.com' diff --git a/gcp/api/snapshot_tests.yaml b/gcp/api/snapshot_tests.yaml index 14cf478f0b7..4f4381dd621 100644 --- a/gcp/api/snapshot_tests.yaml +++ b/gcp/api/snapshot_tests.yaml @@ -27,10 +27,17 @@ steps: dir: gcp/api args: ['poetry', 'sync'] +- name: 'gcr.io/oss-vdb/ci' + id: 'build-go-backend' + dir: go + args: ['go', 'build', '-o', 'api', './cmd/api'] + waitFor: ['-'] + - name: 'gcr.io/oss-vdb/ci' id: 'api-snapshot-tests' dir: gcp/api args: ['bash', '-ex', 'run_tests_e2e.sh', '/workspace/dummy.json'] + waitFor: ['build-go-backend', 'sync'] timeout: 7200s serviceAccount: 'projects/oss-vdb/serviceAccounts/api-e2e-tester@oss-vdb.iam.gserviceaccount.com' From 7769a226ff9bdb61b1a62c4695684bb506d49424 Mon Sep 17 00:00:00 2001 From: michaelkedar Date: Mon, 6 Jul 2026 06:57:08 +0000 Subject: [PATCH 5/5] fix name normalization --- .../database/datastore/affected_versions.go | 3 ++ .../datastore/vulnerability_matching.go | 12 ++++- .../datastore/vulnerability_matching_test.go | 46 +++++++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/go/internal/database/datastore/affected_versions.go b/go/internal/database/datastore/affected_versions.go index 2191b0f62f5..443a28a622b 100644 --- a/go/internal/database/datastore/affected_versions.go +++ b/go/internal/database/datastore/affected_versions.go @@ -54,6 +54,9 @@ func computeAffectedVersions(vuln *osvschema.Vulnerability) []AffectedVersions { pkgName := affected.GetPackage().GetName() eHelper, exists := ecosystem.DefaultProvider.Get(pkgEcosystem) + if exists { + pkgName = ecosystem.NormalizePackageName(eHelper, pkgName) + } // TODO(michaelkedar): Matching the current behavior of the API, // where GIT tags match to the first git repo in the ranges list, even if diff --git a/go/internal/database/datastore/vulnerability_matching.go b/go/internal/database/datastore/vulnerability_matching.go index 07842408153..dd7f94ae22b 100644 --- a/go/internal/database/datastore/vulnerability_matching.go +++ b/go/internal/database/datastore/vulnerability_matching.go @@ -179,10 +179,18 @@ func (s *VulnerabilityStore) MatchPackages(ctx context.Context, ecosystem, name, // Ideally, we'd check both unnormalized and normalized named at once, if there // is no provided ecosystem (even though that is not explicitly supported), but - // Datastore cannot give cursors for 'OR' queries, so just only normalize if - // this is explicitly GIT. + // Datastore cannot give cursors for 'OR' queries, so normalize if an ecosystem + // is explicitly provided or if this is explicitly GIT. if ecosystem == "GIT" { name = normalizeRepo(name) + } else if ecosystem != "" { + provider := s.ecosystems + if provider == nil { + provider = eco.DefaultProvider + } + if sys, ok := provider.Get(ecosystem); ok { + name = eco.NormalizePackageName(sys, name) + } } q := datastore.NewQuery("AffectedVersions"). FilterField("name", "=", name). diff --git a/go/internal/database/datastore/vulnerability_matching_test.go b/go/internal/database/datastore/vulnerability_matching_test.go index 71c9472c9a6..9ce3c428f70 100644 --- a/go/internal/database/datastore/vulnerability_matching_test.go +++ b/go/internal/database/datastore/vulnerability_matching_test.go @@ -751,3 +751,49 @@ func TestParsePythonCursor(t *testing.T) { }) } } + +func TestVulnerabilityStore_MatchPackages_Normalization(t *testing.T) { + ctx := context.Background() + dsClient := testutils.MustNewDatastoreClientForTesting(t) + store := NewVulnerabilityStore(VulnStoreConfig{ + Client: dsClient, + GCS: testutils.NewMockStorage(), + }) + + key := datastore.IncompleteKey("AffectedVersions", nil) + entity := AffectedVersions{ + VulnID: "VULN-PYPI", + Ecosystem: "PyPI", + Name: "foo-bar", + Versions: []string{"1.0.0"}, + } + if _, err := dsClient.Put(ctx, key, &entity); err != nil { + t.Fatalf("Failed to setup datastore test data: %v", err) + } + + queries := []string{ + "foo-bar", + "Foo_Bar", + "foo.bar", + "FOO---BAR", + } + + for _, qName := range queries { + t.Run(qName, func(t *testing.T) { + iterSeq := store.MatchPackages(ctx, "PyPI", qName, "", "") + next, stop := iter.Pull2(iterSeq) + defer stop() + + match, err, ok := next() + if !ok { + t.Fatalf("MatchPackages(PyPI, %q) returned no results, want VULN-PYPI", qName) + } + if err != nil { + t.Fatalf("MatchPackages(PyPI, %q) error: %v", qName, err) + } + if match.ID != "VULN-PYPI" { + t.Errorf("MatchPackages(PyPI, %q) got ID %q, want VULN-PYPI", qName, match.ID) + } + }) + } +}