Merge branch 'google:master' into master

Author: CharlyReux

.github/workflows/publish-to-pypi.yaml

@@ -44,7 +44,7 @@ jobs:
         build
         --sdist --wheel --outdir dist/ .
     - name: Publish distribution to PyPI
-      uses: pypa/gh-action-pypi-publish@e53eb8b103ffcb59469888563dc324e3c8ba6f06 # v1.8.12
+      uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14
       with:
         password: ${{ secrets.PYPI_API_TOKEN }}
         packages_dir: dist/

.github/workflows/scorecards.yml

@@ -50,6 +50,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@928ff8c822d966a999092a6a35e32177899afb7c # v2.24.6
+        uses: github/codeql-action/upload-sarif@e56cfd0877b4826be144d11aa31e6c64a55828e9 # v2.24.7
         with:
           sarif_file: results.sarif

Pipfile.lock

@@ -123,6 +123,20 @@ steps:
   args: ['push', '--all-tags', 'gcr.io/oss-vdb/alpine-cve-convert']
   waitFor: ['build-alpine-cve-convert', 'cloud-build-queue']
 
+- name: 'gcr.io/cloud-builders/docker'
+  entrypoint: 'bash'
+  args: ['-c', 'docker pull gcr.io/oss-vdb/debian-cve-convert:latest || exit 0']
+  id: 'pull-debian-cve-convert'
+  waitFor: ['setup']
+- name: gcr.io/cloud-builders/docker
+  args: ['build', '-t', 'gcr.io/oss-vdb/debian-cve-convert:latest', '-t', 'gcr.io/oss-vdb/debian-cve-convert:$COMMIT_SHA', '-f', 'cmd/debian/Dockerfile', '--cache-from', 'gcr.io/oss-vdb/debian-cve-convert:latest', '--pull', '.']
+  dir: 'vulnfeeds'
+  id: 'build-debian-cve-convert'
+  waitFor: ['pull-debian-cve-convert']
+- name: gcr.io/cloud-builders/docker
+  args: ['push', '--all-tags', 'gcr.io/oss-vdb/debian-cve-convert']
+  waitFor: ['build-debian-cve-convert', 'cloud-build-queue']
+
 - name: 'gcr.io/cloud-builders/docker'
   entrypoint: 'bash'
   args: ['-c', 'docker pull gcr.io/oss-vdb/combine-to-osv:latest || exit 0']
@@ -263,6 +277,7 @@ steps:
      debian-convert=gcr.io/oss-vdb/debian-convert:$COMMIT_SHA,\
      combine-to-osv=gcr.io/oss-vdb/combine-to-osv:$COMMIT_SHA,\
      alpine-cve-convert=gcr.io/oss-vdb/alpine-cve-convert:$COMMIT_SHA,\
+     debian-cve-convert=gcr.io/oss-vdb/debian-cve-convert:$COMMIT_SHA,\
      debian-copyright-mirror=gcr.io/oss-vdb/debian-copyright-mirror:$COMMIT_SHA,\
      cpe-repo-gen=gcr.io/oss-vdb/cpe-repo-gen:$COMMIT_SHA,\
      nvd-cve-osv=gcr.io/oss-vdb/nvd-cve-osv:$COMMIT_SHA,\
@@ -312,6 +327,7 @@ images:
 - 'gcr.io/oss-vdb/alias-computation:$COMMIT_SHA'
 - 'gcr.io/oss-vdb/cron:$COMMIT_SHA'
 - 'gcr.io/oss-vdb/alpine-cve-convert:$COMMIT_SHA'
+- 'gcr.io/oss-vdb/debian-cve-convert:$COMMIT_SHA'
 - 'gcr.io/oss-vdb/combine-to-osv:$COMMIT_SHA'
 - 'gcr.io/oss-vdb/indexer:$COMMIT_SHA'
 - 'gcr.io/oss-vdb/debian-convert:$COMMIT_SHA'

deployment/build-and-stage.yaml

@@ -0,0 +1,35 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: debian-cve-convert
+spec:
+  schedule: "0 */1 * * *"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      activeDeadlineSeconds: 3600
+      template:
+        spec:
+          containers:
+          - name: debian-cve-convert
+            image: debian-cve-convert
+            imagePullPolicy: Always
+            env:
+            - name: GOOGLE_CLOUD_PROJECT
+              value: oss-vdb-test
+            - name: OUTPUT_GCS_BUCKET
+              value: osv-test-cve-osv-conversion
+            securityContext:
+              privileged: true
+            resources:
+              requests:
+                cpu: 1
+                memory: "1G"
+              limits:
+                cpu: 1
+                memory: "2G"
+          restartPolicy: OnFailure
+          volumes:
+            - name: "ssd"
+              hostPath:
+                path: "/mnt/disks/ssd0"

deployment/clouddeploy/gke-workers/environments/oss-vdb-test/debian-cve-convert.yaml

@@ -1,5 +1,6 @@
 resources:
 - ../../base
+- debian-cve-convert.yaml
 patches:
 - path: workers.yaml
 - path: scaler.yaml

deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml

@@ -5,12 +5,21 @@ metadata:
 description: OSV API pipeline
 serialPipeline:
   stages:
-  - targetId: staging-api
+  - targetId: staging-api-multi
     profiles: [ staging ]
-  - targetId: production-api
+  - targetId: production-api-multi
     profiles: [ prod ]
 ---
 
+apiVersion: deploy.cloud.google.com/v1
+kind: Target
+metadata:
+  name: staging-api-multi
+description: multi-target oss-vdb-test API instances
+multiTarget:
+  targetIds: [staging-api, staging-api-batch]
+---
+
 apiVersion: deploy.cloud.google.com/v1
 kind: Target
 metadata:
@@ -25,6 +34,32 @@ executionConfigs:
   serviceAccount: [email protected]
 ---
 
+apiVersion: deploy.cloud.google.com/v1
+kind: Target
+metadata:
+  name: staging-api-batch
+description: oss-vdb-test API batch query instance
+run:
+  location: projects/oss-vdb-test/locations/us-central1
+deployParameters:
+  serviceName: "osv-grpc-backend-batch"
+  containerConcurrency: "1"
+executionConfigs:
+- usages:
+  - RENDER
+  - DEPLOY
+  serviceAccount: [email protected]
+---
+
+apiVersion: deploy.cloud.google.com/v1
+kind: Target
+metadata:
+  name: production-api-multi
+description: multi-target oss-vdb API instances
+multiTarget:
+  targetIds: [production-api, production-api-batch]
+---
+
 apiVersion: deploy.cloud.google.com/v1
 kind: Target
 metadata:
@@ -37,3 +72,20 @@ executionConfigs:
   - RENDER
   - DEPLOY
   serviceAccount: [email protected]
+---
+
+apiVersion: deploy.cloud.google.com/v1
+kind: Target
+metadata:
+  name: production-api-batch
+description: oss-vdb API batch query instance
+run:
+  location: projects/oss-vdb/locations/us-central1
+deployParameters:
+  serviceName: "osv-grpc-backend-batch"
+  containerConcurrency: "1"
+executionConfigs:
+- usages:
+  - RENDER
+  - DEPLOY
+  serviceAccount: [email protected]

deployment/clouddeploy/osv-api/clouddeploy.yaml

@@ -1,9 +1,12 @@
 apiVersion: serving.knative.dev/v1
 kind: Service
 metadata:
-  name: osv-grpc-backend
+  name: osv-grpc-backend # from-param: ${serviceName}
 spec:
   template:
+    metadata:
+      annotations:
+        autoscaling.knative.dev/maxScale: '300'
     spec:
       containers:
       - image: osv-server
@@ -13,11 +16,13 @@ spec:
         startupProbe:
           grpc:
             service: osv.v1.OSV
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
         livenessProbe:
           grpc:
             service: osv.v1.OSV
           timeoutSeconds: 5
           failureThreshold: 3
           periodSeconds: 10
       timeoutSeconds: 60
-      containerConcurrency: 10
+      containerConcurrency: 5 # from-param: ${containerConcurrency}

deployment/clouddeploy/osv-api/run.yaml

@@ -21,7 +21,7 @@ steps:
 - name: gcr.io/cloud-builders/gcloud
   args: ['deploy', 'releases', 'promote', '--quiet', '--release=osv-$SHORT_SHA', '--region=us-central1', '--delivery-pipeline=gke-workers', '--to-target=production-workers', '--annotations=tag=$TAG_NAME']
 - name: gcr.io/cloud-builders/gcloud
-  args: ['deploy', 'releases', 'promote', '--quiet', '--release=osv-$SHORT_SHA', '--region=us-central1', '--delivery-pipeline=osv-api', '--to-target=production-api', '--annotations=tag=$TAG_NAME']
+  args: ['deploy', 'releases', 'promote', '--quiet', '--release=osv-$SHORT_SHA', '--region=us-central1', '--delivery-pipeline=osv-api', '--to-target=production-api-multi', '--annotations=tag=$TAG_NAME']
 - name: gcr.io/cloud-builders/gcloud
   args: ['deploy', 'releases', 'promote', '--quiet', '--release=osv-$SHORT_SHA', '--region=us-central1', '--delivery-pipeline=gke-indexer', '--to-target=production-indexer', '--annotations=tag=$TAG_NAME']
 
@@ -36,6 +36,8 @@ steps:
   args: ['container', 'images', 'add-tag', '--quiet', 'gcr.io/oss-vdb/exporter:$COMMIT_SHA', 'gcr.io/oss-vdb/exporter:$TAG_NAME']
 - name: gcr.io/cloud-builders/gcloud
   args: ['container', 'images', 'add-tag', '--quiet', 'gcr.io/oss-vdb/alpine-cve-convert:$COMMIT_SHA', 'gcr.io/oss-vdb/alpine-cve-convert:$TAG_NAME']
+- name: gcr.io/cloud-builders/gcloud
+  args: ['container', 'images', 'add-tag', '--quiet', 'gcr.io/oss-vdb/debian-cve-convert:$COMMIT_SHA', 'gcr.io/oss-vdb/debian-cve-convert:$TAG_NAME']
 - name: gcr.io/cloud-builders/gcloud
   args: ['container', 'images', 'add-tag', '--quiet', 'gcr.io/oss-vdb/combine-to-osv:$COMMIT_SHA', 'gcr.io/oss-vdb/combine-to-osv:$TAG_NAME']
 - name: gcr.io/cloud-builders/gcloud

deployment/deploy-prod.yaml

@@ -31,3 +31,6 @@ backend:
     - selector: "*"
       address: ${backend_url}
       deadline: 60
+    - selector: "osv.v1.OSV.QueryAffectedBatch"
+      address: ${backend_batch_url}
+      deadline: 60