break it up

brabster · brabster · commit 82d8862a0aaf · 2025-11-30T21:46:14.000Z
diff --git a/.devcontainer/pypi_vulnerabilities.code-workspace b/.devcontainer/pypi_vulnerabilities.code-workspace
diff --git a/.github/workflows/deploy_safety.yml b/.github/workflows/deploy_safety.yml
@@ -24,7 +24,6 @@ jobs:
       - name: load_prod_safety_db
         uses: ./.github/actions/run_in_venv
         with:
-          working-dir: .
+          working-dir: ./safety_etl
           script: |
-            dbt run-operation ensure_datasets
-            python etl/safety_db/load_missing_partitions.py --dataset ${DBT_DATASET}_internal
+            python ./load_missing_partitions.py --dataset ${DBT_DATASET}_impl
diff --git a/.github/workflows/deploy_semver_udf.yml b/.github/workflows/deploy_semver_udf.yml
@@ -0,0 +1,29 @@
+on:
+  workflow_dispatch: {}
+  workflow_call: {}
+  
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    env:
+      PIP_REQUIRE_VIRTUALENV: true
+      DBT_MAX_GIGABYTES_BILLED: ${{ vars.DBT_MAX_GIGABYTES_BILLED }}
+    permissions:
+        contents: read
+        id-token: write
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+      
+      - name: setup_workflow
+        uses: ./.github/actions/setup_default_workflow
+        with:
+          workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
+      
+      - name: deploy udfs
+        uses: ./.github/actions/run_in_venv
+        with:
+          working-dir: ./semver_udf
+          script: |
+            dbt build
diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
@@ -16,12 +16,21 @@ jobs:
       id-token: write
     secrets: inherit
 
+  refresh-semver-udf:
+    uses: ./.github/workflows/deploy_semver_udf.yml
+    permissions:
+      contents: read
+      id-token: write
+    secrets: inherit
+
   refresh-safety:
     uses: ./.github/workflows/deploy_safety.yml
     permissions:
       contents: read
       id-token: write
     secrets: inherit
+    needs:
+      - refresh-semver-udf
 
   refresh_vulns:
     uses: ./.github/workflows/deploy_vulns.yml
@@ -30,8 +39,9 @@ jobs:
       id-token: write
       actions: read
       pages: write
+    secrets: inherit
     needs:
+      - refresh-semver-udf
       - refresh-safety
       - refresh-pypi
-    secrets: inherit
     
diff --git a/.safety-policy.yml b/.safety-policy.yml
@@ -1,4 +1,2 @@
 security:
-  ignore-vulnerabilities:
-    '73530':
-      reason: No details of vulnerability
+  ignore-vulnerabilities: {}
diff --git a/dbt_shared/macros/environment/grant_schemas.sql b/dbt_shared/macros/environment/grant_schemas.sql
@@ -0,0 +1,34 @@
+{% macro grant_schemas(schemas, grant_view_to=[], revoke_view_from=[]) -%}
+{% if execute %}
+{% for schema in schemas %}
+    {% if grant_view_to|length > 0 and not schema.endswith('_impl') %}
+
+        {%- set grant_sql -%}
+        GRANT `roles/bigquery.dataViewer`
+        ON SCHEMA {{ schema }}
+        TO
+            {% for principal in grant_view_to %}
+            '{{ principal }}'{{ ", " if not loop.last else "" }}
+            {% endfor %}
+        {%- endset -%}
+
+        {%- do run_query(grant_sql) -%}
+    {% endif %}
+
+    {% if revoke_view_from|length > 0 %}
+    {%- set revoke_sql -%}
+    REVOKE `roles/bigquery.dataViewer`
+    ON SCHEMA {{ schema }}
+    FROM
+        {% for principal in revoke_view_from %}
+        '{{ principal }}'{{ ", " if not loop.last else "" }}
+        {% endfor %}
+    {%- endset -%}
+
+    {%- do run_query(revoke_sql) -%}
+    {% endif %}
+
+{% endfor %}
+{% endif %}
+
+{%- endmacro %}
diff --git a/pypi/models/published/file_downloads.sql b/pypi/models/published/file_downloads.sql
diff --git a/pypi/models/published/file_downloads.yml b/pypi/models/published/file_downloads.yml
diff --git a/pypi_vulnerabilities.code-workspace b/pypi_vulnerabilities.code-workspace
@@ -0,0 +1,37 @@
+{
+	"folders": [
+		{
+			"name": "pypi",
+			"path": "pypi"
+		},
+		{
+			"name": "safety",
+			"path": "safety"
+		},
+		{
+			"name": "safety_etl",
+			"path": "safety_etl"
+		},
+		{
+			"name": "semver_udf",
+			"path": "semver_udf"
+		},
+		{
+			"name": "dbt_shared",
+			"path": "dbt_shared"
+		},
+		{
+			"name": "vscode_shared",
+			"path": "vscode_shared"
+		},
+		{
+			"name": ".github",
+			"path": ".github"
+		},
+		{
+			"name": "pypi_vulnerabilities",
+			"path": "."
+		}
+	],
+	"settings": {}
+}
diff --git a/safety/dbt_project.yml b/safety/dbt_project.yml
@@ -0,0 +1,36 @@
+name: 'product' # only referred to in this config file
+version: '1.0.0'
+config-version: 2
+
+profile: 'current'
+
+macro-paths: ["macros", "../dbt_shared/macros"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+model-paths: ["models"]
+snapshot-paths: ["snapshots"]
+
+clean-targets:
+  - "target"
+  - "dbt_packages"
+
+models:
+  +labels:
+    stability: unstable
+    data_classification: public
+  +persist_docs:
+    relation: true
+    columns: true
+  # product:
+  #   impl:
+  #     +schema: impl
+
+tests:
+  product: {}
+
+on-run-start:
+  - '{{ ensure_dataset(is_public=env_var("DBT_PUBLIC")) }}'
+
+on-run-end:
+  - '{{ drop_redundant_models(dry_run=false) }}'
diff --git a/safety/models/published/safety_db_metrics.sql b/safety/models/published/safety_db_metrics.sql
@@ -0,0 +1,7 @@
+
+SELECT
+    commit_timestamp,
+    TO_JSON_STRING(commit) commit_details,
+    COUNT(1) package_count
+FROM {{ source('safety_db', 'safety_db_history') }}
+GROUP BY commit_timestamp, commit_details
diff --git a/safety/models/published/safety_db_metrics.yml b/safety/models/published/safety_db_metrics.yml
@@ -0,0 +1,22 @@
+version: 2
+
+models:
+  - name: safety_db_metrics
+    description: Metrics about Safety DB contents available
+    columns:
+      - name: commit_timestamp
+        description: Timestamp of the commit
+        tests:
+          - not_null
+          - unique
+      - name: commit_details
+        description: Summary of commit details encoded as JSON
+        tests:
+          - not_null
+          - unique
+      - name: package_count
+        description: Number of packages with at least one vulnerability record in the commit
+        tests:
+          - dbt_utils.accepted_range:
+              arguments:
+                min_value: 0
diff --git a/safety/models/published/safety_vulnerabilities.sql b/safety/models/published/safety_vulnerabilities.sql
@@ -0,0 +1,22 @@
+
+WITH dated AS (
+  SELECT
+    package,
+    DATE(commit_timestamp) commit_date,
+    vulnerability.specs specs,
+    vulnerability.cve cve,
+    {{ source('semver_udf', 'multi_spec_has_at_least_one_upper_bound') }}(vulnerability.specs) fix_was_available,
+    ROW_NUMBER() OVER(PARTITION BY package, vulnerability.cve ORDER BY commit_timestamp) - 1 previous_commits
+  FROM {{ source('safety_db', 'safety_db_history') }}
+      CROSS JOIN UNNEST(vulnerabilities) vulnerability
+)
+
+SELECT
+    *,
+    COALESCE(
+      LEAD(commit_date) OVER (
+        PARTITION BY package, cve
+        ORDER BY commit_date),
+      DATE_ADD(commit_date, INTERVAL 1 MONTH)) until_date,
+    previous_commits = 0 is_first_commit
+FROM dated
diff --git a/safety/models/published/safety_vulnerabilities.yml b/safety/models/published/safety_vulnerabilities.yml
@@ -0,0 +1,44 @@
+version: 2
+
+models:
+  - name: safety_vulnerabilities
+    description: Row-per-vulnerability view over the safety database
+    # tests:
+    # there are rows that have all these columns the same but different specs?
+    #   - dbt_utils.unique_combination_of_columns:
+    #       combination_of_columns:
+    #         - commit_date
+    #         - package
+    #         - cve
+    columns:
+      - name: commit_date
+        description: Date of the Safety commit, first date when a particular vulnerability record was in effect
+        tests:
+          - not_null
+      - name: until_date
+        description: Date of the next commit
+      - name: package
+        description: Package associated with vulnerability
+        tests:
+          - not_null
+      - name: specs
+        description: Version range vulnerability applies to, as a semver constraint
+        tests:
+          - not_null
+      - name: cve
+        description: Vulnerability identifier - may not actually be a CVE! May be null.
+      - name: previous_commits
+        description: Number of times the vulnerability has appeared in a safety DB commit before, 0 = first appearance
+        tests:
+          - not_null
+          - dbt_utils.accepted_range:
+              arguments:
+                min_value: 0
+      - name: fix_was_available
+        description: Whether a fix appeared to be available at the time, based on the version constraints
+        tests:
+          - not_null
+      - name: is_first_commit
+        description: True when this is the first appearance of the vulnerability for the package and version specification
+        tests:
+          - not_null
diff --git a/safety/models/sources/safety_db.yml b/safety/models/sources/safety_db.yml
@@ -0,0 +1,24 @@
+version: 2
+
+sources:
+  - name: safety_db
+    schema: '{{ target.schema }}_impl'
+    description: Contents of public Safety DB, see https://github.com/pyupio/safety-db
+    tables:
+      - name: safety_db_history
+        description: History of SafetyDB public changes. This data is updated once per month. Schema definition in etl/safety_schema.json
+        columns:
+          - name: package
+          - name: commit
+          - name: commit.sha
+          - name: commit.timestamp
+          - name: commit.message
+          - name: commit.file_url
+          - name: vulnerabilities
+          - name: vulnerabilities.v
+          - name: vulnerabilities.specs
+          - name: vulnerabilities.more_info_path
+          - name: vulnerabilities.id
+          - name: vulnerabilities.cve
+          - name: vulnerabilities.advisory
+
diff --git a/safety/models/sources/semver_udf.yml b/safety/models/sources/semver_udf.yml
@@ -0,0 +1,6 @@
+sources:
+  - name: semver_udf
+    tables:
+      - name: multi_spec_has_at_least_one_upper_bound
+      - name: matches_multi_spec
+      - name: matches_spec
diff --git a/safety/packages.yml b/safety/packages.yml
@@ -0,0 +1,3 @@
+packages:
+  - package: dbt-labs/dbt_utils
+    version: ">=1.1.1"
diff --git a/safety/profiles.yml b/safety/profiles.yml
@@ -0,0 +1,13 @@
+current:
+  outputs:
+    current:
+      dataset: safetycli
+      location: "{{ env_var('DBT_LOCATION') }}"
+      method: oauth
+      priority: interactive
+      project: "{{ env_var('GOOGLE_CLOUD_PROJECT') }}"
+      threads: "{{ env_var('DBT_THREADS', '8') | int }}"
+      type: bigquery
+      # 50GB limit default, overridden in actions variables
+      maximum_bytes_billed: "{{ env_var('DBT_MAX_GIGABYTES_BILLED', '50') | int * (1024 * 1024 * 1024) }}"
+  target: current
diff --git a/safety/requirements.txt b/safety/requirements.txt
@@ -0,0 +1 @@
+dbt-bigquery>=1.7.0
diff --git a/safety/settings.json b/safety/settings.json
@@ -0,0 +1 @@
+../vscode_shared/settings.json
diff --git a/safety_etl/.vscode b/safety_etl/.vscode
@@ -0,0 +1 @@
+../vscode_shared
diff --git a/safety_etl/bigquery.py b/safety_etl/bigquery.py
diff --git a/safety_etl/github.py b/safety_etl/github.py
diff --git a/safety_etl/load_missing_partitions.py b/safety_etl/load_missing_partitions.py
@@ -9,7 +9,7 @@
 
 def parse_args():
     parser = argparse.ArgumentParser()
-    parser.add_argument("--project_id", default=os.environ.get("DBT_PROJECT"))
+    parser.add_argument("--project_id", default=os.environ.get("GOOGLE_CLOUD_PROJECT"))
     parser.add_argument("--location", default=os.environ.get("DBT_LOCATION"))
     parser.add_argument("--dataset", default=os.environ.get("DBT_DATASET"))
     parser.add_argument("--history_table", default="safety_db_history")
diff --git a/safety_etl/requirements.txt b/safety_etl/requirements.txt
@@ -0,0 +1 @@
+google-cloud-bigquery>=2.3.0
diff --git a/safety_etl/safety_schema.json b/safety_etl/safety_schema.json
