Merge branch 'white/staging' into white/master

dkraus · dkraus · commit 41c899ab6f55 · 2024-09-20T15:52:43.000-03:00
diff --git a/.gitlab/ci/testing/.pretesting-gitlab-ci.yml b/.gitlab/ci/testing/.pretesting-gitlab-ci.yml
@@ -98,7 +98,7 @@ flake8:
     stage: pre_testing
     script:
       - pip install flake8
-      - flake8 .
+      - flake8 . --exclude=.git
     rules:
         - !reference [ .ignore-on-tag, rules ]
         - when: on_success
diff --git a/CHANGELOG/5.7.0/community.md b/CHANGELOG/5.7.0/community.md
@@ -0,0 +1,5 @@
+ * [ADD] Added bulk update feature for custom attributes. #7716
+ * [FIX] Fix hostnames not working on pipelines conditions. #7807
+ * [FIX] Allow services to be searchable. #7514
+ * [FIX] Fixed crash on unsupported image format upload. #7710
+ * [FIX] Fixed service based jobs not working for assets #7778
diff --git a/CHANGELOG/5.7.0/date.md b/CHANGELOG/5.7.0/date.md
@@ -0,0 +1 @@
+Sep 20th, 2024
diff --git a/RELEASE.md b/RELEASE.md
@@ -1,6 +1,14 @@
 New features in the latest update
 =====================================
 
+5.7.0 [Sep 20th, 2024]:
+---
+ * [ADD] Added bulk update feature for custom attributes. #7716
+ * [FIX] Allow services to be searchable. #7514
+ * [FIX] Fixed crash on unsupported image format upload. #7710
+ * [FIX] Fixed service based jobs not working for assets #7778
+ * [FIX] Fixed hostnames not working on pipelines conditions. #7807
+
 5.6.1 [Aug 28th, 2024]:
 ---
  * [FIX] Resolved an issue with filtering by Custom Attributes. #7800
diff --git a/faraday/__init__.py b/faraday/__init__.py
@@ -4,5 +4,5 @@
 See the file 'doc/LICENSE' for the license information
 """
 
-__version__ = '5.6.1'
+__version__ = '5.7.0'
 __license_version__ = __version__
diff --git a/faraday/openapi/faraday_swagger.json b/faraday/openapi/faraday_swagger.json
@@ -1,7 +1,7 @@
 {
     "info": {
         "description": "The Faraday REST API enables you to interact with [our server](https://github.com/infobyte/faraday).\nUse this API to interact or integrate with Faraday server. This page documents the REST API, with HTTP response codes and example requests and responses.",
-        "title": "Faraday 5.6.0 API",
+        "title": "Faraday 5.7.0 API",
         "version": "v3"
     },
     "security": [
@@ -2417,6 +2417,47 @@
                 }
             }
         },
+        "/_api/v3/ws/{workspace_name}/services/filter": {
+            "get": {
+                "tags": [
+                    "Filter",
+                    "Service"
+                ],
+                "description": "Filters, sorts and groups workspaced objects using a json with parameters. These parameters must be part of the model.",
+                "parameters": [
+                    {
+                        "in": "query",
+                        "name": "q",
+                        "description": "recursive json with filters that supports operators. The json could also contain sort and group."
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "returns filtered, sorted and grouped results",
+                        "content": {
+                            "application/json": {
+                                "schema": {
+                                    "$ref": "#/components/schemas/FlaskRestless"
+                                }
+                            }
+                        }
+                    },
+                    "400": {
+                        "description": "invalid q was sent to the server"
+                    }
+                }
+            },
+            "parameters": [
+                {
+                    "name": "workspace_name",
+                    "in": "path",
+                    "schema": {
+                        "type": "string"
+                    },
+                    "required": true
+                }
+            ]
+        },
         "/_api/v3/services": {
             "delete": {
                 "tags": [
@@ -8802,4 +8843,4 @@
             "name": "settings"
         }
     ]
-}
+}
diff --git a/faraday/server/api/modules/services.py b/faraday/server/api/modules/services.py
@@ -26,7 +26,8 @@
     FilterSetMeta,
     FilterAlchemyMixin,
     BulkDeleteWorkspacedMixin,
-    BulkUpdateWorkspacedMixin
+    BulkUpdateWorkspacedMixin,
+    FilterWorkspacedMixin,
 )
 from faraday.server.schemas import (
     MetadataSchema,
@@ -134,7 +135,12 @@ class Meta(FilterSetMeta):
         operators = (operators.Equal,)
 
 
-class ServiceView(PaginatedMixin, FilterAlchemyMixin, ReadWriteWorkspacedView, BulkDeleteWorkspacedMixin, BulkUpdateWorkspacedMixin):
+class ServiceView(PaginatedMixin,
+                  FilterAlchemyMixin,
+                  ReadWriteWorkspacedView,
+                  BulkDeleteWorkspacedMixin,
+                  BulkUpdateWorkspacedMixin,
+                  FilterWorkspacedMixin):
 
     route_base = 'services'
     model_class = Service
diff --git a/faraday/server/api/modules/vulns.py b/faraday/server/api/modules/vulns.py
@@ -83,7 +83,7 @@
     FaradayCustomField,
     PrimaryKeyRelatedField,
 )
-from faraday.server.utils.vulns import parse_cve_references_and_policyviolations, update_one_host_severity_stat
+from faraday.server.utils.vulns import parse_cve_references_and_policyviolations, update_one_host_severity_stat, bulk_update_custom_attributes
 from faraday.server.debouncer import debounce_workspace_update
 
 vulns_api = Blueprint('vulns_api', __name__)
@@ -819,6 +819,9 @@ def _perform_bulk_update(self, ids, data, workspace_name=None, **kwargs):
         kwargs['returning'] = returning_rows
         if workspace_name:
             debounce_workspace_update(workspace_name)
+
+        if (len(data) > 0 and len(ids) > 0) and 'custom_fields' in data.keys():
+            return bulk_update_custom_attributes(ids, data)
         return super()._perform_bulk_update(ids, data, workspace_name, **kwargs)
 
     def put(self, object_id, workspace_name=None, **kwargs):
diff --git a/faraday/server/api/modules/vulns_context.py b/faraday/server/api/modules/vulns_context.py
@@ -59,6 +59,7 @@
 )
 from faraday.server.utils.export import export_vulns_to_csv
 from faraday.server.utils.filters import FlaskRestlessSchema
+from faraday.server.utils.vulns import bulk_update_custom_attributes
 from faraday.server.api.modules.vulns import (
     EvidenceSchema,
     VulnerabilitySchema,
@@ -144,6 +145,8 @@ def _perform_bulk_update(self, ids, data, **kwargs):
             Vulnerability.service_id,
         ]
         kwargs['returning'] = returning_rows
+        if (len(data) > 0 and len(ids) > 0) and "custom_fields" in data.keys():
+            return bulk_update_custom_attributes(ids, data)
         return super()._perform_bulk_update(ids, data, **kwargs)
 
     def _get_eagerloaded_query(self, *args, **kwargs):
diff --git a/faraday/server/api/modules/workflow.py b/faraday/server/api/modules/workflow.py
@@ -194,6 +194,13 @@
     ]
 }
 
+service_datatypes = {
+    "service/name": "string",
+    "service/port": "int",
+    "service/status": "string",
+    "service/version": "string",
+}
+
 order_regex = re.compile(r"^$|^\d+(-\d+)*$")
 
 WORKFLOW_LIMIT = 2
diff --git a/faraday/server/fields.py b/faraday/server/fields.py
@@ -14,7 +14,7 @@
 from depot.fields.upload import UploadedFile
 from depot.io.utils import file_from_content, INMEMORY_FILESIZE
 from depot.manager import DepotManager
-from PIL import Image
+from PIL import Image, UnidentifiedImageError
 from sqlalchemy.dialects.postgresql.json import JSONB
 from webargs.core import ValidationError
 
@@ -49,7 +49,10 @@ def process_content(self, content, filename=None, content_type=None):
         image_format = imghdr.what(None, h=content[:32])
         if image_format:
             content_type = f'image/{image_format}'
-            self.generate_thumbnail(content)
+            try:
+                self.generate_thumbnail(content)
+            except UnidentifiedImageError:
+                pass
         return super().process_content(
                 content, filename, content_type)
 
diff --git a/faraday/server/models.py b/faraday/server/models.py
@@ -1195,13 +1195,6 @@ class Host(Metadata):
         cascade="all, delete-orphan"
     )
 
-    @property
-    def service(self):
-        if self.services:
-            return self.services[0]
-        else:
-            return None
-
     # 1 workspace <--> N hosts
     # 1 to N (the FK is placed in the child) and bidirectional (backref)
     workspace_id = Column(Integer, ForeignKey('workspace.id', ondelete='CASCADE'), index=True, nullable=False)
diff --git a/faraday/server/utils/filters.py b/faraday/server/utils/filters.py
@@ -227,6 +227,11 @@ def _model_class(self):
         return User
 
 
+class FlaskRestlessServiceFilterSchema(FlaskRestlessFilterSchema):
+    def _model_class(self):
+        return Service
+
+
 class FlaskRestlessOperator(Schema):
     _or = fields.Nested("self", attribute='or', data_key='or')
     _and = fields.Nested("self", attribute='and', data_key='and')
@@ -237,6 +242,7 @@ class FlaskRestlessOperator(Schema):
         FlaskRestlessWorkspaceFilterSchema,
         FlaskRestlessUserFilterSchema,
         FlaskRestlessVulnerabilityTemplateFilterSchema,
+        FlaskRestlessServiceFilterSchema,
     ]
 
     def load(
diff --git a/faraday/server/utils/vulns.py b/faraday/server/utils/vulns.py
@@ -1,18 +1,22 @@
 import re
 import logging
+import json
 
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.inspection import inspect
+from sqlalchemy import func, update
 
 from faraday.server.models import (
     CVE,
     db,
     Reference,
     PolicyViolation,
-    OWASP
+    OWASP,
+    VulnerabilityGeneric
 )
 from faraday.server.utils.database import is_unique_constraint_violation
 from faraday.server.utils.reference import create_reference
+from flask import jsonify
 
 logger = logging.getLogger(__name__)
 
@@ -260,3 +264,40 @@ def update_one_host_severity_stat(vulnerability):
         else:
             logger.warning("Nor service nor host found in vulnerability ", vulnerability)
     return hosts, services
+
+
+def bulk_update_custom_attributes(vuln_ids: list, data: dict):
+    """
+    Updates or adds specified custom attribute for a list of vulnerabilities without
+    overwriting existing custom attributes.
+
+    :param vuln_ids: List of vulnerability IDs to update.
+    :param data: Dictionary of custom field to update/add.
+    :return: Flask Response object
+    """
+
+    try:
+        for key, value in data["custom_fields"].items():
+            # Prepare the path and value for jsonb_set
+            key_path = f'{{{key}}}'  # e.g., '{string}'
+            value_json = json.dumps(value)
+            # Perform the bulk update
+            stmt = (
+                update(VulnerabilityGeneric)
+                .where(VulnerabilityGeneric.id.in_(vuln_ids))
+                .values(
+                    custom_fields=func.jsonb_set(
+                        func.coalesce(VulnerabilityGeneric.custom_fields, '{}'),  # Handle NULLs
+                        key_path,
+                        value_json,
+                        True  # Create missing
+                    )
+                )
+            )
+            db.session.execute(stmt)
+        db.session.commit()
+        return jsonify({"updated": len(vuln_ids)})
+
+    except Exception as e:
+        db.session.rollback()
+        return jsonify({"error": str(e)})
diff --git a/faraday/server/utils/workflows.py b/faraday/server/utils/workflows.py
@@ -17,6 +17,7 @@
     OPERATORS,
     fields_lookup,
     rules_attributes,
+    service_datatypes,
 )
 from faraday.server.extensions import socketio
 from faraday.server.models import (
@@ -181,24 +182,43 @@ def _process_field_data(obj, field):
 
     # special case for service_id
     if field[0] == "service_id":
-        return obj, "service_id"
+        return [obj], "service_id"
+
+    # special case for host services
+    if obj.__class__.__name__.lower() == "host" and field[0] == "service":
+        field = ["services"] + field[1:]
+
+    if field[0] == "hostnames":
+        return [obj], "hostnames"
 
     for route in field:
+
+        # break if previous iteration was a list
+        if isinstance(obj, list):
+            field = route
+            break
+
         obj_data_in_field = getattr(obj, route, None)
         if obj_data_in_field is None:
             raise ValueError(f"Field \"{route}\" not found in object {obj}")
         if isinstance(obj_data_in_field, db.Model):
             obj = obj_data_in_field
+        elif isinstance(obj_data_in_field, list):
+            obj = obj_data_in_field
         else:
             field = route
             break
-    return obj, field
 
+    if isinstance(field, list):
+        field = field[-1]
 
-def _check_leaf(obj, condition):
+    if not isinstance(obj, list):
+        obj = [obj]
+    return obj, field
 
-    target_obj, field = _process_field_data(obj, condition.field)
-    model_data = getattr(target_obj, field, None)
+
+def _perform_leaf_check(obj, condition, field):
+    model_data = getattr(obj, field, None)
 
     operator = OPERATORS.get(condition.operator, None)
     if operator is None:
@@ -207,7 +227,10 @@ def _check_leaf(obj, condition):
     class_name = obj.__class__.__name__.lower()
     class_name = "vulnerability" if "web" in class_name else class_name
 
-    data_type = [x.get("type") for x in rules_attributes[class_name] if x.get("name") == condition.field][0]
+    if class_name != "service":
+        data_type = [x.get("type") for x in rules_attributes[class_name] if x.get("name") == condition.field][0]
+    else:
+        data_type = service_datatypes.get(condition.field, None)
     data = condition.data
 
     if data_type == "string":
@@ -241,6 +264,14 @@ def _check_leaf(obj, condition):
     return operator(model_data, data)
 
 
+def _check_leaf(obj, condition):
+    target_objs, field = _process_field_data(obj, condition.field)
+    results = []
+    for target_obj in target_objs:
+        results.append(_perform_leaf_check(target_obj, condition, field))
+    return any(results)
+
+
 def _check_condition(obj, condition):
     if condition.type == "leaf":
         try:
diff --git a/pynixify/packages/faraday-agent-parameters-types/default.nix b/pynixify/packages/faraday-agent-parameters-types/default.nix
@@ -6,12 +6,12 @@
 
 buildPythonPackage rec {
   pname = "faraday-agent-parameters-types";
-  version = "1.7.0";
+  version = "1.7.1";
 
   src = fetchPypi {
     inherit version;
     pname = "faraday_agent_parameters_types";
-    sha256 = "03b8s649gvl25wrhx7wz4n1fhr2v8yxgfsijky1a1zi1r8xd6nn4";
+    sha256 = "0k5gzgfaagh9sr5jbysshjhhyzxhmsky3mwkq5n320d7m7zzk6na";
   };
 
   buildInputs = [ pytest-runner ];
diff --git a/pynixify/packages/faraday-plugins/default.nix b/pynixify/packages/faraday-plugins/default.nix
diff --git a/pynixify/packages/faradaysec/default.nix b/pynixify/packages/faradaysec/default.nix
diff --git a/requirements.txt b/requirements.txt
diff --git a/tests/test_api_vulnerability.py b/tests/test_api_vulnerability.py
diff --git a/tests/test_api_vulnerability_context.py b/tests/test_api_vulnerability_context.py
diff --git a/tests/test_api_workflow.py b/tests/test_api_workflow.py

Original file line number	Diff line number	Diff line change
`@@ -194,6 +194,13 @@`
`194`	`194`	`]`
`195`	`195`	`}`
`196`	`196`
	`197`	`+service_datatypes = {`
	`198`	`+ "service/name": "string",`
	`199`	`+ "service/port": "int",`
	`200`	`+ "service/status": "string",`
	`201`	`+ "service/version": "string",`
	`202`	`+}`
	`203`	`+`
`197`	`204`	`order_regex = re.compile(r"^$\|^\d+(-\d+)*$")`
`198`	`205`
`199`	`206`	`WORKFLOW_LIMIT = 2`