Actually upload the generated sitemaps (and perform logging) (#2309)

Turns out, the original code wasn't doing any uploading 🤦

Author: another-rex

deployment/clouddeploy/gke-workers/base/generate-sitemap.yaml

@@ -13,7 +13,7 @@ spec:
           - name: generate-sitemap-cron
             image: cron
             imagePullPolicy: Always
-            command: ["/usr/local/bin/generate_sitemap/generate_sitemap.py", "--base_url", "$BASE_URL"]
+            command: ["/usr/local/bin/generate_sitemap/generate_and_upload.sh"]
             resources:
               requests:
                 cpu: 1

deployment/clouddeploy/gke-workers/environments/oss-vdb-test/generate-sitemap.yaml

@@ -14,3 +14,5 @@ spec:
               value: "https://test.osv.dev"
             - name: GOOGLE_CLOUD_PROJECT
               value: oss-vdb-test
+            - name: OUTPUT_GCS_BUCKET
+              value: test-osv-dev-sitemap

deployment/clouddeploy/gke-workers/environments/oss-vdb/generate-sitemap.yaml

@@ -14,3 +14,5 @@ spec:
               value: "https://osv.dev"
             - name: GOOGLE_CLOUD_PROJECT
               value: oss-vdb
+            - name: OUTPUT_GCS_BUCKET
+              value: osv-dev-sitemap

docker/cron/generate_sitemap/generate_and_upload.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+
+SITEMAP_OUTPUT="sitemap_output/"
+OUTPUT_BUCKET="${OUTPUT_GCS_BUCKET:=test-osv-dev-sitemap}"
+BASE_URL_PATH="${BASE_URL:=https://test.osv.dev}"
+
+echo "Begin sitemap generation for $BASE_URL_PATH"
+
+./generate_sitemap.py --base_url $BASE_URL_PATH
+
+echo "Begin Syncing with cloud to $OUTPUT_BUCKET"
+
+gsutil -m rsync -c -d $SITEMAP_OUTPUT "gs://$OUTPUT_BUCKET/"

docker/cron/generate_sitemap/generate_sitemap.py

@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Generate sitemap."""
+import logging
 import sys
 import os
 import osv
@@ -59,17 +60,18 @@ def get_sitemap_url_for_ecosystem(ecosystem: str, base_url: str) -> str:
 
 def generate_sitemap_for_ecosystem(ecosystem: str, base_url: str) -> None:
   """Generate a sitemap for the give n ecosystem."""
+  logging.info('Generating sitemap for ecosystem "%s".', ecosystem)
   vulnerability_ids = fetch_vulnerability_ids(ecosystem)
   filename = get_sitemap_filename_for_ecosystem(ecosystem)
   urlset = Element(
-      'urlset', xmlns="http://www.sitemaps.org/schemas/sitemap/0.9")
+      'urlset', xmlns='http://www.sitemaps.org/schemas/sitemap/0.9')
 
   # TODO: For large ecosystems with over 50,000 vulnerabilities, generate
   # multiple sitemaps.
   for vuln in vulnerability_ids[:_SITEMAP_URL_LIMIT]:
     url = SubElement(urlset, 'url')
     loc = SubElement(url, 'loc')
-    loc.text = f"{base_url}/vulnerability/{vuln}"
+    loc.text = f'{base_url}/vulnerability/{vuln}'
     lastmod = SubElement(url, 'lastmod')
     lastmod.text = datetime.datetime.now().isoformat()
 
@@ -79,11 +81,12 @@ def generate_sitemap_for_ecosystem(ecosystem: str, base_url: str) -> None:
 
 def generate_sitemap_index(ecosystems: set[str], base_url: str) -> None:
   """Generate a sitemap index."""
+  logging.info('Generating sitemap index.')
   sitemapindex = Element(
-      'sitemapindex', xmlns="http://www.sitemaps.org/schemas/sitemap/0.9")
+      'sitemapindex', xmlns='http://www.sitemaps.org/schemas/sitemap/0.9')
 
   for ecosystem in ecosystems:
-    sitemap = SubElement(sitemapindex, "sitemap")
+    sitemap = SubElement(sitemapindex, 'sitemap')
     loc = SubElement(sitemap, 'loc')
     loc.text = get_sitemap_url_for_ecosystem(ecosystem, base_url)
     lastmod = SubElement(sitemap, 'lastmod')

gcp/appengine/frontend_handlers.py

@@ -117,8 +117,8 @@ def index():
 
 @blueprint.route('/robots.txt')
 def robots():
-  response = make_response(f"Sitemap: {request.host_url}sitemap_index.xml\n")
-  response.mimetype = "text/plain"
+  response = make_response(f'Sitemap: {request.host_url}sitemap_index.xml\n')
+  response.mimetype = 'text/plain'
   return response