Merge branch 'master' of github.com:oasis-open/cti-stix-validator

chrisr3d · chrisr3d · commit fbcf543b242b · 2026-02-09T14:33:59.000+01:00
diff --git a/.github/workflows/python-ci-tests.yml b/.github/workflows/python-ci-tests.yml
@@ -39,7 +39,8 @@ jobs:
       run: |
         tox
     - name: Upload coverage information to Codecov
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4.2.0
       with:
+        token: ${{ secrets.CODECOV_TOKEN }}
         fail_ci_if_error: false # optional (default = false)
         verbose: true # optional (default = false)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -6,7 +6,7 @@ repos:
         exclude: ^stix2validator/(v20|v21)/assets/.*.csv$
     -   id: check-merge-conflict
 -   repo: https://github.com/PyCQA/flake8
-    rev: 5.0.4
+    rev: 7.3.0
     hooks:
     -   id: flake8
         name: Check project styling
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -8,7 +8,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.12"
+    python: "3.14"
 
 # Build documentation in the "docs/" directory with Sphinx
 sphinx:
diff --git a/README.rst b/README.rst
@@ -144,8 +144,9 @@ repositories/maintainers-guide#additionalMaintainers>`_.
 .. Initial Maintainers: Greg Back & Ivan Kirillov & Chris Lenk
 
 * `Emily Ratliff <mailto:emily.ratliff@ibm.com>`__; GitHub ID: `https://github.com/ejratl <https://github.com/ejratl>`_; WWW: `IBM <http://www.ibm.com/>`__
-*  `Jason Keirstead <mailto:Jason.Keirstead@ca.ibm.com>`__; GitHub ID: `https://github.com/JasonKeirstead <https://github.com/JasonKeirstead>`_; WWW: `IBM <http://www.ibm.com/>`__
 * `Rich Piazza <mailto:rpiazza@mitre.org>`_; GitHub ID: `https://github.com/rpiazza <https://github.com/rpiazza>`_; WWW: `MITRE <http://www.mitre.org/>`_
+* `Alexandre Dulaunoy <mailto:alexandre.dulaunoy@circl.lu>`__; GitHub ID: https://github.com/adulau; WWW: `CIRCL <http://www.circl.lu/>`__
+* `Christian Studer <mailto:christian.studer@circl.lu>`__; GitHub ID: https://github.com/chrisr3d; WWW: `CIRCL <http://www.circl.lu/>`__
 
 
 .. _aboutOpenRepos:
diff --git a/docs/best-practices.rst b/docs/best-practices.rst
@@ -99,8 +99,8 @@ Mandatory Checks - STIX 2.1
 |                                 | RFC 5646 language code                 | language code.                         |
 +---------------------------------+----------------------------------------+----------------------------------------+
 | software_language               | the 'language' property of software    | The 'languages' property of object     |
-|                                 | objects is a valid ISO 639-2 language  | '<identifier>' contains an invalid     |
-|                                 | code                                   | code ('<lang>').                       |
+|                                 | objects is a valid RFC 5646 language   | '<identifier>' contains an invalid     |
+|                                 | code (ISO 639-2 accepted with warning) | code ('<lang>').                       |
 +---------------------------------+----------------------------------------+----------------------------------------+
 | patterns                        | that the syntax of the pattern of an   | '<object>' is not a valid observable   |
 |                                 | indicator is valid, and that objects   | type name                              |
diff --git a/pinning-schema.md b/pinning-schema.md
@@ -12,6 +12,6 @@ Output confirms the schema 2.1 correct commit is pinned:
 Entering 'stix2validator/schemas-2.0'
 stix2validator/schemas-2.0 b155093705ab4934ee29e7ba4dc99ed053cd4e7f
 Entering 'stix2validator/schemas-2.1'
-stix2validator/schemas-2.1 4d010b385b83e53ea322357783ff3be3da5820fb
+stix2validator/schemas-2.1 c4f8d589acf2bdb3783655c89e0ffb6e150006ae
 
-Push from local to my repo (fork of Oasis) which then feeds the change to the PR on Oasis.  I've updated the comments in the PR to confirm.
+Push from local to my repo (fork of Oasis) which then feeds the change to the PR on Oasis.  I've updated the comments in the PR to confirm.
diff --git a/stix2validator/schemas-2.1 b/stix2validator/schemas-2.1
@@ -1 +1 @@
-Subproject commit 4d010b385b83e53ea322357783ff3be3da5820fb
+Subproject commit c4f8d589acf2bdb3783655c89e0ffb6e150006ae
diff --git a/stix2validator/test/v21/__init__.py b/stix2validator/test/v21/__init__.py
@@ -33,7 +33,6 @@ def assertTrueWithOptions(self, instance, **kwargs):
         else:
             options = ValidationOptions(strict=True, **kwargs)
         results = validate_parsed_json(instance, options)
-        print_results(results)
         self.assertTrue(results.is_valid)
 
     def assertFalseWithOptions(self, instance, **kwargs):
diff --git a/stix2validator/test/v21/observed_data_tests.py b/stix2validator/test/v21/observed_data_tests.py
@@ -643,9 +643,41 @@ def test_software_language(self):
         }
         self.assertFalseWithOptions(observed_data)
 
-        observed_data['languages'][0] = 'eng'
+        # Test with RFC 5646 language code (preferred)
+        observed_data['languages'][0] = 'en'
         self.assertTrueWithOptions(observed_data)
 
+    def test_software_language_iso639_compatibility(self):
+        """Test that ISO 639-2 codes are accepted with warnings for backward compatibility"""
+        import io
+        import logging
+
+        # Capture log output
+        log_capture = io.StringIO()
+        handler = logging.StreamHandler(log_capture)
+        handler.setLevel(logging.WARNING)
+        logger = logging.getLogger('stix2validator.v21.musts')
+        logger.addHandler(handler)
+        logger.setLevel(logging.WARNING)
+
+        observed_data = {
+            "type": "software",
+            "id": "software--ff1e0780-358c-5808-a8c7-d0fca4ef6ef4",
+            "name": "word",
+            "languages": ["eng"]  # ISO 639-2 code
+        }
+
+        # Should be valid (backward compatibility)
+        self.assertTrueWithOptions(observed_data)
+
+        # Should generate a warning
+        log_output = log_capture.getvalue()
+        self.assertIn("ISO 639-2 language code", log_output)
+        self.assertIn("RFC 5646 language codes are preferred", log_output)
+
+        # Clean up
+        logger.removeHandler(handler)
+
     def test_software_cpe(self):
         observed_data = {
             "type": "software",
diff --git a/stix2validator/v21/musts.py b/stix2validator/v21/musts.py
@@ -411,16 +411,30 @@ def language(instance):
 
 @cyber_observable_check("2.1")
 def software_language(instance):
-    """Ensure the 'language' property of software objects is a valid ISO 639-2
-    language code.
+    """Ensure the 'language' property of software objects is a valid RFC 5646
+    language code. ISO 639-2 codes are accepted for backward compatibility
+    but will generate warnings.
     """
     if ('type' in instance and instance['type'] == 'software' and
             'languages' in instance):
         for lang in instance['languages']:
-            if lang not in enums.SOFTWARE_LANG_CODES:
+            if lang in enums.LANG_CODES:
+                # Valid RFC 5646 language code
+                continue
+            elif lang in enums.SOFTWARE_LANG_CODES:
+                # ISO 639-2 code - accept but warn
+                import logging
+                logger = logging.getLogger(__name__)
+                logger.warning("The 'languages' property of object '%s' "
+                               "contains an ISO 639-2 language code ('%s'). "
+                               "RFC 5646 language codes are preferred for STIX 2.1. "
+                               "Consider updating to RFC 5646 format."
+                               % (instance['id'], lang))
+            else:
+                # Invalid language code
                 yield JSONError("The 'languages' property of object '%s' "
-                                "contains an invalid ISO 639-2 language "
-                                " code ('%s')."
+                                "contains an invalid language code ('%s'). "
+                                "Expected RFC 5646 language code."
                                 % (instance['id'], lang), instance['id'])
 
 
diff --git a/stix2validator/validator.py b/stix2validator/validator.py
@@ -558,6 +558,12 @@ def patch_schema(schema_data: dict, schema_path: str) -> dict:
     return schema_data
 
 
+_HTTP_SCHEMAS = dict()
+
+
+_FILE_SCHEMAS = dict()
+
+
 def retrieve_from_filesystem(schema_path_uri: str, schema_dir: str) -> Resource:
     """Callback to retrieve a schema given its path.
 
@@ -569,11 +575,17 @@ def retrieve_from_filesystem(schema_path_uri: str, schema_dir: str) -> Resource:
         A resource loaded with the content.
     """
     if schema_path_uri.startswith("http://") or schema_path_uri.startswith("https://"):
+        if schema_path_uri in _HTTP_SCHEMAS:
+            return _HTTP_SCHEMAS[schema_path_uri]
         response = requests.get(schema_path_uri, allow_redirects=True)
         schema = response.json()
         schema = patch_schema(schema, schema_path_uri)
-        return Resource.from_contents(schema)
+        schema_resource = Resource.from_contents(schema)
+        _HTTP_SCHEMAS[schema_path_uri] = schema_resource
+        return schema_resource
     else:
+        if schema_path_uri in _FILE_SCHEMAS:
+            return _FILE_SCHEMAS[schema_path_uri]
         schema_path = pathlib.Path(schema_path_uri)
         if schema_path.is_absolute() or schema_path_uri.startswith("file://"):
             is_relative = False
@@ -585,9 +597,14 @@ def retrieve_from_filesystem(schema_path_uri: str, schema_dir: str) -> Resource:
             schema = json.load(f)
         schema = patch_schema(schema, schema_path)
         if is_relative:
-            return Resource.opaque(schema)
+            schema_resource = Resource.opaque(schema)
         else:
-            return Resource.from_contents(schema)
+            schema_resource = Resource.from_contents(schema)
+        _FILE_SCHEMAS[schema_path_uri] = schema_resource
+        return schema_resource
+
+
+_PATCHED_SCHEMAS = dict()
 
 
 def load_validator(schema_path, schema, schema_dir):
@@ -600,7 +617,11 @@ def load_validator(schema_path, schema, schema_dir):
     Returns:
         An instance of Draft202012Validator.
     """
-    schema = patch_schema(schema, schema_path)
+    if schema_path in _PATCHED_SCHEMAS:
+        schema = _PATCHED_SCHEMAS[schema_path]
+    else:
+        schema = patch_schema(schema, schema_path)
+        _PATCHED_SCHEMAS[schema_path] = schema
     retrieve_callback = functools.partial(retrieve_from_filesystem, schema_dir=schema_dir)
     registry = Registry(
         retrieve=retrieve_callback
@@ -609,17 +630,27 @@ def load_validator(schema_path, schema, schema_dir):
     return validator
 
 
+_SCHEMAS_FOUND = dict()
+
+
 def find_schema(schema_dir, name):
     """Search the `schema_dir` directory for a schema called `name`.json.
     Return the file path of the first match it finds.
     """
+    if schema_dir in _SCHEMAS_FOUND and name in _SCHEMAS_FOUND[schema_dir]:
+        return _SCHEMAS_FOUND[schema_dir][name]
+
     schema_filename = name + '.json'
 
     for root, dirnames, filenames in os.walk(schema_dir, followlinks=True):
         if "examples" in root:
             continue
         if schema_filename in filenames:
-            return os.path.join(root, schema_filename)
+            schema_path = os.path.join(root, schema_filename)
+            if schema_dir not in _SCHEMAS_FOUND:
+                _SCHEMAS_FOUND[schema_dir] = dict()
+            _SCHEMAS_FOUND[schema_dir][name] = schema_path
+            return schema_path
 
 
 def load_schema(schema_path):
