feat(api): make prod api use new datastore entities (#4058)

Switch production over to use new `AffectedVersions` for matching.
There's now a bunch of unused code in server.py that I will remove in a
follow-up PR.

Author: michaelkedar

Makefile

@@ -71,7 +71,7 @@ run-website-emulator:
 run-api-server:
 	test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth login --update-adc'"; exit 1)
 	cd gcp/api && docker build -f Dockerfile.esp -t osv/esp:latest .
-	cd gcp/api && $(install-cmd) && GOOGLE_CLOUD_PROJECT=oss-vdb $(run-cmd) python test_server.py $(HOME)/.config/gcloud/application_default_credentials.json $(ARGS)# Run with `make run-api-server ARGS=--no-backend` to launch esp without backend.
+	cd gcp/api && $(install-cmd) && GOOGLE_CLOUD_PROJECT=oss-vdb OSV_VULNERABILITIES_BUCKET=osv-vulnerabilities $(run-cmd) python test_server.py $(HOME)/.config/gcloud/application_default_credentials.json $(ARGS)# Run with `make run-api-server ARGS=--no-backend` to launch esp without backend.
 
 run-api-server-test:
 	test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth login --update-adc'"; exit 1)

deployment/clouddeploy/osv-api/run-prod.yaml

@@ -10,6 +10,9 @@ spec:
     spec:
       containers:
       - image: osv-server
+        env:
+        - name: OSV_VULNERABILITIES_BUCKET
+          value: osv-vulnerabilities
         resources:
           limits:
             cpu: 2

gcp/api/integration_tests.py

@@ -173,17 +173,20 @@ def test_query_version(self):
         timeout=_TIMEOUT)
     self.assert_results_equal({'vulns': [self._VULN_744]}, response.json())
 
-    response = requests.post(
-        _api() + _BASE_QUERY,
-        data=json.dumps({
-            'version': '2.1.2-rc',
-            'package': {
-                'name': 'mruby',
-            }
-        }),
-        timeout=_TIMEOUT)
-
-    self.assert_results_equal({'vulns': [self._VULN_744]}, response.json())
+    # NOTE(michaelkedar): version queries without ecosystem specified is not
+    # officially supported. Since our change to matching logic, this test
+    # would now return >50 vulnerabilities across 4 ecosystems.
+    # response = requests.post(
+    #     _api() + _BASE_QUERY,
+    #     data=json.dumps({
+    #         'version': '2.1.2-rc',
+    #         'package': {
+    #             'name': 'mruby',
+    #         }
+    #     }),
+    #     timeout=_TIMEOUT)
+
+    # self.assert_results_equal({'vulns': [self._VULN_744]}, response.json())
     # self.assertEqual(
     #   response.text,
     #   '{"code":3,"message":"Ecosystem not specified"}')
@@ -262,6 +265,7 @@ def test_query_semver(self):
         go_2021_0052,
         ghsa_3vp4_m3rf_835h,
     ]
+    expected_vulns.sort(key=lambda x: x['id'])
 
     # Test that a SemVer (believed to be vulnerable) version and an ecosystem
     # returns expected vulnerabilities.

gcp/api/run_tests.sh

@@ -18,7 +18,7 @@ if [ $# -lt 1 ]; then
   exit 1
 fi
 
-export GOOGLE_CLOUD_PROJECT=oss-vdb
+export GOOGLE_CLOUD_PROJECT=oss-vdb OSV_VULNERABILITIES_BUCKET=osv-vulnerabilities
 service docker start
 
 # Set -e later as service docker start should be able to successfully fail

gcp/api/server.py

@@ -173,28 +173,10 @@ def GetVulnById(self, request, context: grpc.ServicerContext):
       context.abort(grpc.StatusCode.INVALID_ARGUMENT, 'ID too long')
       return None
 
-    if get_gcp_project() in ('oss-vdb-test', 'test-osv'):
-      # Get vuln from GCS
-      try:
-        return osv.gcs.get_by_id(request.id)
-      except exceptions.NotFound:
-        # Check for aliases
-        alias_group = yield osv.AliasGroup.query(
-            osv.AliasGroup.bug_ids == request.id).get_async()
-        if alias_group:
-          alias_string = ' '.join([
-              f'{alias}' for alias in alias_group.bug_ids if alias != request.id
-          ])
-          context.abort(
-              grpc.StatusCode.NOT_FOUND,
-              f'Bug not found, but the following aliases were: {alias_string}')
-          return None
-        context.abort(grpc.StatusCode.NOT_FOUND, 'Bug not found.')
-        return None
-
-    bug = yield osv.Bug.query(osv.Bug.db_id == request.id).get_async()
-
-    if not bug:
+    # Get vuln from GCS
+    try:
+      return osv.gcs.get_by_id(request.id)
+    except exceptions.NotFound:
       # Check for aliases
       alias_group = yield osv.AliasGroup.query(
           osv.AliasGroup.bug_ids == request.id).get_async()
@@ -209,17 +191,6 @@ def GetVulnById(self, request, context: grpc.ServicerContext):
       context.abort(grpc.StatusCode.NOT_FOUND, 'Bug not found.')
       return None
 
-    if bug.status == osv.BugStatus.UNPROCESSED:
-      context.abort(grpc.StatusCode.NOT_FOUND, 'Bug not found.')
-      return None
-
-    if not bug.public:
-      context.abort(grpc.StatusCode.PERMISSION_DENIED, 'Permission denied.')
-      return None
-
-    resp = yield bug_to_response(bug, include_details=True)
-    return resp
-
   @ndb_context
   @trace_filter.log_trace
   @ndb.synctasklet
@@ -876,18 +847,10 @@ def to_response(b: osv.Bug):
       return None
 
     bugs = yield query_by_commit(context, commit_bytes, to_response=to_response)
-  elif package_name and get_gcp_project() in ('oss-vdb-test', 'test-osv'):
+  elif package_name:
     # New Database table & GCS querying
     bugs = yield query_package(context, package_name, ecosystem, version,
                                include_details)
-  # Version query needs to include a package.
-  elif package_name and version:
-    bugs = yield query_by_version(
-        context, package_name, ecosystem, version, to_response=to_response)
-  elif package_name and ecosystem:
-    # Package specified without version.
-    bugs = yield query_by_package(
-        context, package_name, ecosystem, to_response=to_response)
   else:
     context.service_context.abort(grpc.StatusCode.INVALID_ARGUMENT,
                                   'Invalid query.')