WIP: load new cve's via cpe

Nils Kreiner · Nils Kreiner · commit 742b2b1dffcb · 2025-04-13T16:17:49.000+02:00
diff --git a/backend/__init__.py b/backend/__init__.py
diff --git a/backend/analyzer/manager/project_manager.py b/backend/analyzer/manager/project_manager.py
@@ -4,7 +4,7 @@
 from django.db import DatabaseError
 
 from analyzer.manager.cve_manager import CVEObjectManager
-from analyzer.models import Project, Report, Dependency
+from analyzer.models import Project, Report, Dependency, CPEObject
 from analyzer.parser.types import ParseResult
 from utilities.helperclass import hash_key
 
@@ -124,6 +124,12 @@ def _update_dependencies(self, data: dict[str, ParseResult]):
         dependency_object.path = data.get(new_dependency_id).path
         dependency_object.in_use = True
         dependency_object.save()
+        for new_cpe_id in data.get(new_dependency_id).cpe_ids:
+          cpe_object = CPEObject.objects.get_or_create(
+            dependency=dependency_object,
+            cpe_id=new_cpe_id
+          )[0]
+          cpe_object.save()
 
         for vulnerability in data.get(new_dependency_id).vulnerabilities:
           cve_object = CVEObjectManager(vulnerability).get()
diff --git a/backend/analyzer/migrations/0003_project_auto_update_alter_dependency_package_manager_and_more.py b/backend/analyzer/migrations/0003_project_auto_update_alter_dependency_package_manager_and_more.py
@@ -0,0 +1,28 @@
+# Generated by Django 5.1.2 on 2025-03-04 15:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('analyzer', '0002_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='project',
+            name='auto_update',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AlterField(
+            model_name='dependency',
+            name='package_manager',
+            field=models.CharField(default='NA', max_length=255),
+        ),
+        migrations.AlterField(
+            model_name='report',
+            name='overall_cvss_severity',
+            field=models.CharField(max_length=255, null=True),
+        ),
+    ]
diff --git a/backend/analyzer/migrations/0004_cpeobject.py b/backend/analyzer/migrations/0004_cpeobject.py
@@ -0,0 +1,25 @@
+# Generated by Django 5.1.2 on 2025-03-11 18:07
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('analyzer', '0003_project_auto_update_alter_dependency_package_manager_and_more'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CPEObject',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('cpe_id', models.CharField(max_length=255, unique=True)),
+                ('dependency', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='analyzer.dependency')),
+            ],
+            options={
+                'db_table': 'securecheckplus_cpe_object',
+            },
+        ),
+    ]
diff --git a/backend/analyzer/models.py b/backend/analyzer/models.py
@@ -19,6 +19,7 @@ class Meta:
                                             choices=constants.Threshold.choices,
                                             default=constants.Threshold.MEDIUM.name)
     api_key_hash = models.CharField(null=True, max_length=100)
+    auto_update = models.BooleanField(default=False)
 
     @property
     def dependency_count(self):
@@ -147,3 +148,10 @@ class Meta:
     score_metric_data = models.TextField(default=constants.DEFAULT_SCORE_METRIC_DATA)
     overall_cvss_score = models.DecimalField(max_digits=3, decimal_places=1, null=True)
     overall_cvss_severity = models.CharField(max_length=255, null=True)
+
+class CPEObject(models.Model):
+    class Meta:
+        db_table = constants.DB_SCHEMA_PREFIX + "cpe_object"
+
+    dependency = models.ForeignKey(Dependency, on_delete=models.CASCADE)
+    cpe_id = models.CharField(max_length=255, blank=False, unique=True)
diff --git a/backend/analyzer/parser/owasp_parser.py b/backend/analyzer/parser/owasp_parser.py
@@ -99,9 +99,12 @@ def parse_json(json_data: str or dict) -> dict[str, ParseResult]:
                            dependency.get("vulnerabilities", []) if
                            vuln["source"] in ["NVD", "OSSINDEX"]]
 
+        cpe_ids = [vuln["id"] for vuln in
+                            dependency.get("vulnerabilityIds", [])]
+
         result = ParseResult(dependency_name, version, path,
                              dependency_license,
-                             vulnerabilities, package_manager)
+                             vulnerabilities, package_manager, cpe_ids)
         data[f"{dependency_name}:{version}"] = result
 
     return data
diff --git a/backend/analyzer/parser/types.py b/backend/analyzer/parser/types.py
@@ -9,3 +9,4 @@ class ParseResult:
     license: str
     vulnerabilities: list[str]
     package_manager: str
+    cpe_ids: list[str]
diff --git a/backend/analyzer/services/cve_fetcher.py b/backend/analyzer/services/cve_fetcher.py
@@ -55,14 +55,15 @@ class CVEFetcher:
         "baseSeverity": "N/A"
     }
 
-    def __init__(self, cve_id: str):
+    def __init__(self, cve_id: str = None, cpe_id: str = None):
         """
         Initializes the CVEFetcher with a specific CVE ID.
 
         Args:
             cve_id (str): The CVE identifier to fetch data for.
         """
         self.cve_id = cve_id
+        self.cpe_id = cpe_id
         self.data = {}
         self.successful = False
 
@@ -78,7 +79,7 @@ def fetch_from_nist_gov(self):
         """
         try:
             headers = {"apiKey": NVD_API_KEY}
-            url = parse.urlunparse(NVD_ADDRESS) + self.cve_id
+            url = parse.urlunparse(NVD_ADDRESS) + "?cveId=" + self.cve_id
             logger.info(f"Fetching CVE data from NIST for CVE ID: {self.cve_id} using URL: {url}")
             response = requests.get(url, headers=headers)
 
@@ -127,6 +128,71 @@ def fetch_from_nist_gov(self):
         except (ValueError, KeyError) as e:
             logger.error(f"Failed to retrieve or process CVE data for CVE ID: {self.cve_id}. Error: {e}")
 
+    def fetch_from_nist_by_cpe(self):
+        """
+        Fetches CVE data from the NIST government server for a given CPE ID.
+
+        The function will retrieve all CVEs associated with the provided CPE ID.
+        """
+        if not self.cpe_id:
+            logger.error("CPE ID is required for this fetcher.")
+            return
+
+        try:
+            headers = {"apiKey": NVD_API_KEY}
+            url = parse.urlunparse(NVD_ADDRESS) + "?cpeName=" + self.cpe_id
+            logger.info(f"Fetching CVE data from NIST for CPE ID: {self.cpe_id} using URL: {url}")
+            response = self._make_request(url, headers)
+
+            if response:
+                self._process_cve_data(response)
+                self.successful = True
+                logger.info(f"Successfully fetched CVE data for CPE ID: {self.cpe_id}")
+
+        except Exception as e:
+            logger.error(f"Failed to fetch CVE data for CPE ID: {self.cpe_id}. Error: {e}")
+
+
+    def _make_request(self, url, headers):
+        """
+        Makes a GET request to the provided URL with the given headers and handles the response.
+        """
+        try:
+            response = requests.get(url, headers=headers)
+
+            if response.status_code != 200:
+                logger.warning(f"Failed to fetch CVE data for CPE ID: {self.cpe_id}. HTTP status: {response.status_code}")
+                return None
+
+            return response.json()
+
+        except requests.RequestException as e:
+            logger.error(f"Request failed for CPE ID: {self.cpe_id}. Error: {e}")
+            return None
+
+    def _process_cve_data(self, response_json):
+        """
+        Processes the CVE data from the response and fetches additional CVE data for each CVE ID.
+        """
+        if not isinstance(response_json, dict) or "vulnerabilities" not in response_json:
+            raise ValueError(f"Invalid response structure for CPE ID: {self.cpe_id}")
+
+        cve_list = response_json["vulnerabilities"]
+        for cve_entry in cve_list:
+            cve_id = cve_entry.get("cve", {}).get("id", None)
+            if cve_id:
+                self.fetch_cve_by_id(cve_id)
+
+    def fetch_cve_by_id(self, cve_id):
+        """
+        Fetches CVE data for a specific CVE ID (helper function for fetching individual CVEs).
+        """
+        # Reuse the original `fetch_from_nist_gov` logic here for individual CVEs
+        logger.info(f"Fetching individual CVE data for {cve_id}")
+        fetcher = CVEFetcher(cve_id=cve_id)
+        fetcher.fetch_from_nist_gov()
+        self.data[cve_id] = fetcher.data
+
     def fetch_epss(self):
         """
         Fetches the Exploit Prediction Scoring System (EPSS) score for the given CVE ID.
diff --git a/backend/securecheckplus/settings.py b/backend/securecheckplus/settings.py
@@ -157,6 +157,7 @@ def get_env_variable_or_shutdown_gracefully(var_name):
     "django.contrib.messages",
     "django.contrib.staticfiles",
     "rest_framework",
+    "django_apscheduler",
 ]
 
 MIDDLEWARE = [
diff --git a/backend/utilities/constants.py b/backend/utilities/constants.py
@@ -3,7 +3,7 @@
 
 SERVER_MAIL_ADDRESS = "securecheckplus@thisisnotanofficialmail.de"
 
-NVD_ADDRESS = parse.urlparse("https://services.nvd.nist.gov/rest/json/cves/2.0?cveId=")
+NVD_ADDRESS = parse.urlparse("https://services.nvd.nist.gov/rest/json/cves/2.0")
 
 EPSS_ADDRESS = parse.urlparse("https://api.first.org/data/v1/epss?cve=")
 
diff --git a/backend/webserver/management/__init__.py b/backend/webserver/management/__init__.py
diff --git a/backend/webserver/management/commands/__init__.py b/backend/webserver/management/commands/__init__.py
diff --git a/backend/webserver/management/commands/runapscheduler.py b/backend/webserver/management/commands/runapscheduler.py
@@ -0,0 +1,103 @@
+import logging
+
+from django.conf import settings
+
+from apscheduler.schedulers.blocking import BlockingScheduler
+from apscheduler.triggers.cron import CronTrigger
+from django.core.management.base import BaseCommand
+from django_apscheduler.jobstores import DjangoJobStore
+from django_apscheduler.models import DjangoJobExecution
+from django_apscheduler import util
+from analyzer.models import Project, CVEObject, CPEObject, Dependency, Report
+from analyzer.manager.cve_manager import CVEObjectManager
+from analyzer.services.cve_fetcher import CVEFetcher
+
+logger = logging.getLogger(__name__)
+
+
+def update_cves():
+    logger.info("start job")
+    projects = Project.objects.filter(auto_update=True)
+    for project in projects:
+         if project.auto_update:
+            cve_objects = CVEObject.objects.filter(report__dependency__project=project)
+            CVEObjectManager(cve_objects).update_cve()
+
+def update_deps():
+    logger.info("start job")
+    projects = Project.objects.filter(auto_update=True)
+    cpes_total = []
+    for project in projects:
+        dependencies = Dependency.objects.filter(project=project)
+        for dependency_object in dependencies:
+            cpes = CPEObject.objects.filter(dependency=dependency_object)
+            for cpe_object in cpes:
+                cpes_total.append(cpe_object.cpe_id)
+                cve_fetcher = CVEFetcher(cpe_id=cpe_object.cpe_id)
+                cve_fetcher.fetch_from_nist_by_cpe()
+                if cve_fetcher.successful:
+                    for cve in cve_fetcher.data:
+                        cve_object = CVEObjectManager(cve).get()
+                        report = Report.objects.get_or_create(dependency=dependency_object,
+                                                              cve_object=cve_object)[0]
+                        logger.info(f"data: {cve}")
+                        logger.info(f"report: {report}")
+    logger.info(f"cpes: {cpes_total}")
+
+
+
+
+
+
+# The `close_old_connections` decorator ensures that database connections, that have become
+# unusable or are obsolete, are closed before and after your job has run. You should use it
+# to wrap any jobs that you schedule that access the Django database in any way.
+@util.close_old_connections
+def delete_old_job_executions(max_age=604_800):
+    """
+    This job deletes APScheduler job execution entries older than `max_age` from the database.
+    It helps to prevent the database from filling up with old historical records that are no
+    longer useful.
+
+    :param max_age: The maximum length of time to retain historical job execution records.
+                    Defaults to 7 days.
+    """
+    DjangoJobExecution.objects.delete_old_job_executions(max_age)
+
+
+class Command(BaseCommand):
+    help = "Runs APScheduler."
+
+    def handle(self, *args, **options):
+        scheduler = BlockingScheduler(timezone=settings.TIME_ZONE)
+        scheduler.add_jobstore(DjangoJobStore(), "default")
+
+        scheduler.add_job(
+            update_deps,
+            trigger=CronTrigger(second="*/59"),  # Every 59 seconds
+            id="update_projects",  # The `id` assigned to each job MUST be unique
+            max_instances=1,
+            replace_existing=True,
+        )
+        logger.info("Added job 'update_projects'.")
+
+        scheduler.add_job(
+            delete_old_job_executions,
+            trigger=CronTrigger(
+                day_of_week="mon", hour="00", minute="00"
+            ),  # Midnight on Monday, before start of the next work week.
+            id="delete_old_job_executions",
+            max_instances=1,
+            replace_existing=True,
+        )
+        logger.info(
+            "Added weekly job: 'delete_old_job_executions'."
+        )
+
+        try:
+            logger.info("Starting scheduler...")
+            scheduler.start()
+        except KeyboardInterrupt:
+            logger.info("Stopping scheduler...")
+            scheduler.shutdown()
+            logger.info("Scheduler shut down successfully!")
diff --git a/backend/webserver/serializer/project_serializer.py b/backend/webserver/serializer/project_serializer.py
@@ -9,12 +9,13 @@
 class ProjectBasicSerializer(serializers.ModelSerializer):
     class Meta:
         model = Project
-        fields = ["projectId", "projectName", "updated", "deploymentThreshold"]
+        fields = ["projectId", "projectName", "updated", "deploymentThreshold", "autoUpdate"]
 
     projectId = serializers.CharField(source="project_id", read_only=True)
     projectName = serializers.CharField(source="project_name", allow_blank=True)
     updated = serializers.DateTimeField(default=datetime.now(), read_only=True)
     deploymentThreshold = serializers.ChoiceField(source="deployment_threshold", choices=Threshold.names)
+    autoUpdate = serializers.BooleanField(source="auto_update")
 
 
 class ProjectDetailSerializer(ProjectBasicSerializer):
@@ -42,7 +43,7 @@ class ProjectSummarySerializer(ProjectBasicSerializer):
     class Meta:
         model = Project
         fields = ["projectId", "projectName", "updated", "vulnerabilityCount",
-                  "misconfigurationCount", "riskValue", "requirementsFulfilled"]
+                  "misconfigurationCount", "riskValue", "requirementsFulfilled", "autoUpdate"]
 
     vulnerabilityCount = serializers.SerializerMethodField(method_name="get_vulnerability_count", read_only=True)
     misconfigurationCount = serializers.ReadOnlyField(default=12, read_only=True)  # TODO upcoming feature
diff --git a/frontend/src/components/ProjectCard.tsx b/frontend/src/components/ProjectCard.tsx
@@ -14,6 +14,7 @@ import {colors} from "../style/globalStyle"
 import localization from "../utilities/localization"
 import {useLocation, useNavigate} from "react-router-dom";
 import {useState} from "react";
+import UpdateButton from "./UpdateButton";
 
 export interface Props {
     /**
@@ -40,6 +41,7 @@ export default function ProjectCard(props: Props) {
     const navigate = useNavigate()
     const location = useLocation()
     const [checked, setChecked] = useState(false);
+
     const sentToProject = () => {
         if (location.pathname.endsWith("projects")) {
             navigate(props.project.projectId)
@@ -48,7 +50,6 @@ export default function ProjectCard(props: Props) {
         }
     }
 
-
     function handleCheckboxChangeAndReturnState(){
         let selectedProjectsIds = sessionStorage.getItem("selectedProjectsIds")
         let selectedProjectArray: string[] = selectedProjectsIds ? JSON.parse(selectedProjectsIds) : [];
@@ -80,6 +81,9 @@ export default function ProjectCard(props: Props) {
                         {/*<Stack marginLeft={"1rem"}>*/}
                         {/*    <Typography sx={{projectGroupStyle}}> {"Projektgruppen Name"}</Typography>*/}
                         {/*</Stack>*/}
+                        <Stack>
+                            <UpdateButton project={props.project}/>
+                        </Stack>
                     </Stack>
                     <Stack sx={{alignSelf:"center", marginTop:"1rem"}}>
                         <Stack direction={"row"} alignItems={"center"}>
diff --git a/frontend/src/components/UpdateButton.tsx b/frontend/src/components/UpdateButton.tsx

Original file line number	Diff line number	Diff line change
`@@ -157,6 +157,7 @@ def get_env_variable_or_shutdown_gracefully(var_name):`
`157`	`157`	`"django.contrib.messages",`
`158`	`158`	`"django.contrib.staticfiles",`
`159`	`159`	`"rest_framework",`
	`160`	`+ "django_apscheduler",`
`160`	`161`	`]`
`161`	`162`
`162`	`163`	`MIDDLEWARE = [`