more validators

Author: enthec-opensource

.github/workflows/scripts/icon_path_validator.py

@@ -2,13 +2,19 @@
 import pathlib
 import string
 from typing import Final
+from xml.etree import ElementTree
 
 
 class InvalidStructureException(Exception):
     def __init__(self, msg: str):
         super().__init__(msg)
 
 
+class InvalidSVGException(Exception):
+    def __init__(self, msg: str):
+        super().__init__(msg)
+
+
 class IconValidator:
     def __init__(self):
         self._SOURCE_DIR: Final[str] = "src"
@@ -23,6 +29,16 @@ def validate(self) -> None:
         for file in self._FULL_IMAGES_DIR.iterdir():
             if file.name not in json_icons:
                 raise InvalidStructureException(f"{file.name} must be used, {file} isn't used!")
+            if file.name.lower().endswith(".svg"):
+                self._validate_svg(file)
+
+    def _validate_svg(self, file: pathlib.Path) -> None:
+        try:
+            with file.open("r", encoding="utf8") as f:
+                content: str = f.read()
+            ElementTree.fromstring(content)
+        except ElementTree.ParseError as e:
+            raise InvalidSVGException(f"Invalid SVG '{file.name}': {e}")
 
     def get_json_icons(self) -> set[str]:
         letters: list[str] = list(string.ascii_lowercase)

.github/workflows/scripts/schema_validator.py

@@ -0,0 +1,38 @@
+import json
+import pathlib
+from typing import Final
+
+from jsonschema import validate, ValidationError
+
+
+class SchemaValidationException(Exception):
+    def __init__(self, msg: str):
+        super().__init__(msg)
+
+
+class SchemaValidator:
+    def __init__(self):
+        self._SOURCE_DIR: Final[str] = "src"
+        self._TECH_DIR: Final[str] = "technologies"
+        self._FULL_TECH_DIR: Final[pathlib.Path] = pathlib.Path(self._SOURCE_DIR).joinpath(self._TECH_DIR)
+        self._SCHEMA_FILE: Final[pathlib.Path] = pathlib.Path("schema.json")
+
+    def validate(self) -> None:
+        if not self._SCHEMA_FILE.is_file():
+            raise FileNotFoundError(f"Schema file '{self._SCHEMA_FILE}' not found!")
+        with self._SCHEMA_FILE.open("r", encoding="utf8") as f:
+            schema: dict = json.load(f)
+        for tech_file in sorted(self._FULL_TECH_DIR.iterdir()):
+            if not tech_file.name.endswith(".json"):
+                continue
+            with tech_file.open("r", encoding="utf8") as f:
+                technologies: dict = json.load(f)
+            try:
+                validate(instance=technologies, schema=schema)
+            except ValidationError as e:
+                path: str = " -> ".join(str(p) for p in e.absolute_path) if e.absolute_path else "root"
+                raise SchemaValidationException(f"{tech_file.name}: {e.message} (at {path})")
+
+
+if __name__ == '__main__':
+    SchemaValidator().validate()

.github/workflows/scripts/technology_validator.py

@@ -74,6 +74,16 @@ def __init__(self, msg: str):
         super().__init__(msg)
 
 
+class TechNotFoundException(Exception):
+    def __init__(self, msg: str):
+        super().__init__(msg)
+
+
+class InvalidURLException(Exception):
+    def __init__(self, msg: str):
+        super().__init__(msg)
+
+
 class AbstractValidator:
     def __init__(self, required: bool = False):
         self._required = required
@@ -185,6 +195,27 @@ def get_type(self) -> list[Type]:
         return [str]
 
 
+class URLValidator(StringValidator):
+    def __init__(self, required: bool = False):
+        super().__init__(required)
+        self._url_pattern: Final[re.Pattern] = re.compile(
+            r"^https?://"
+            r"(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)+"
+            r"[A-Za-z0-9-]{2,}"
+            r"(?::\d+)?"
+            r"(?:/[^\s]*)?"
+            r"$"
+        )
+
+    def _validate(self, tech_name: str, data: Any) -> bool:
+        if not super()._validate(tech_name, data):
+            return False
+        if not self._url_pattern.match(data):
+            self._set_custom_error(InvalidURLException(f"Tech '{tech_name}' has invalid URL: '{data}'"))
+            return False
+        return True
+
+
 class BoolValidator(AbstractValidator):
     def get_type(self) -> list[Type]:
         return [bool]
@@ -317,6 +348,22 @@ def _validate(self, tech_name: str, data: Any) -> bool:
         return True
 
 
+class ReferenceValidator(ArrayValidator):
+    def __init__(self, all_techs: set[str]):
+        super().__init__()
+        self._all_techs: Final[set[str]] = all_techs
+
+    def _validate(self, tech_name: str, data: Any) -> bool:
+        if not super()._validate(tech_name, data):
+            return False
+        for ref in data:
+            clean_ref: str = ref.split(r"\;")[0]
+            if clean_ref not in self._all_techs:
+                self._set_custom_error(TechNotFoundException(f"Tech '{tech_name}' references '{clean_ref}' but it doesn't exist!"))
+                return False
+        return True
+
+
 class TechnologiesValidator:
     def __init__(self, file_name: str):
         self._SOURCE_DIR: Final[str] = "src"
@@ -328,18 +375,19 @@ def __init__(self, file_name: str):
         self._IMAGES_DIR: Final[str] = "images"
         self._ICONS_DIR: Final[str] = "icons"
         self._ICONS: Final[list[str]] = [icon.name for icon in pathlib.Path(self._SOURCE_DIR).joinpath(self._IMAGES_DIR).joinpath(self._ICONS_DIR).iterdir()]
+        self._ALL_TECHS: Final[set[str]] = self._get_all_tech_names()
         self._validators: dict[str, AbstractValidator] = {  # TODO confidence and version validator
             "cats": CategoryValidator(self._CATEGORIES, True),
-            "website": StringValidator(True),
+            "website": URLValidator(True),
             "description": StringValidator(),
             "icon": IconValidator(self._ICONS),
             "cpe": CPEValidator(),
             "saas": BoolValidator(),
             "oss": BoolValidator(),
             "pricing": PricingValidator(),
-            "implies": ArrayValidator(),  # TODO cat validation
-            "requires": ArrayValidator(),  # TODO ^
-            "excludes": ArrayValidator(),  # TODO ^
+            "implies": ReferenceValidator(self._ALL_TECHS),
+            "requires": ReferenceValidator(self._ALL_TECHS),
+            "excludes": ReferenceValidator(self._ALL_TECHS),
             "requiresCategory": CategoryValidator(self._CATEGORIES),
             "cookies": DictValidator(contains_regex=True),
             "dom": DomValidator(),
@@ -385,6 +433,16 @@ def _duplicate_key_validator(cls, pairs: list[tuple[str, Any]]) -> dict[str, Any
             result[key] = value
         return result
 
+    def _get_all_tech_names(self) -> set[str]:
+        all_techs: set[str] = set()
+        for letter in list(string.ascii_lowercase) + ["_"]:
+            tech_file: pathlib.Path = self._FULL_TECH_DIR.joinpath(f"{letter}.json")
+            if tech_file.exists():
+                with tech_file.open("r", encoding="utf8") as f:
+                    technologies: dict = json.load(f)
+                all_techs.update(technologies.keys())
+        return all_techs
+
 
 class TechnologyProcessor:
     def __init__(self, tech_name: str, tech_data: dict, validators: dict[str, AbstractValidator]):

.github/workflows/validate.yml

@@ -18,6 +18,27 @@ jobs:
       - name: run structure validator
         run: python3 .github/workflows/scripts/structure_validator.py
 
+  validate_schema:
+    runs-on: ubuntu-22.04
+    needs: validate_structure
+    strategy:
+      matrix:
+        python-version: [ "3.12" ]
+    steps:
+      - name: checkout repository
+        uses: actions/checkout@v4
+
+      - name: set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: install dependencies
+        run: python3 -m pip install jsonschema
+
+      - name: run schema validator
+        run: python3 .github/workflows/scripts/schema_validator.py
+
   validate_categories:
     runs-on: ubuntu-22.04
     needs: validate_structure

src/technologies/d.json

@@ -2266,6 +2266,7 @@
     "cats": [
       1
     ],
+    "cpe": "cpe:2.3:a:monospace:directus:*:*:*:*:*:node.js:*:*",
     "description": "Directus is a free and open-source headless CMS framework for managing custom SQL-based databases.",
     "headers": {
       "x-powered-by": "^Directus$"

src/technologies/g.json

@@ -479,6 +479,7 @@
       57,
       12
     ],
+    "cpe": "cpe:2.3:a:gatsbyjs:gatsby:*:*:*:*:*:node.js:*:*",
     "description": "Gatsby is a React-based open-source framework with performance, scalability and security built-in.",
     "dom": [
       "div#___gatsby, style#gatsby-inlined-css"
@@ -1409,6 +1410,7 @@
       1,
       11
     ],
+    "cpe": "cpe:2.3:a:ghost:ghost:*:*:*:*:*:node.js:*:*",
     "description": "Ghost is a powerful app for new-media creators to publish, share, and grow a business around their content.",
     "headers": {
       "X-Ghost-Cache-Status": ""

src/technologies/j.json

@@ -674,6 +674,7 @@
     "cats": [
       22
     ],
+    "cpe": "cpe:2.3:a:eclipse:jetty:*:*:*:*:*:*:*:*",
     "description": "Jetty is an open-source web server and servlet container known for its scalability and efficiency, supporting protocols like HTTP and WebSocket for various applications from development tools to cloud services.",
     "headers": {
       "Server": "Jetty(?:\\(([\\d\\.]*\\d+))?\\;version:\\1"

src/technologies/k.json

@@ -2003,6 +2003,7 @@
       18,
       22
     ],
+    "cpe": "cpe:2.3:a:koajs:koa:*:*:*:*:*:node.js:*:*",
     "headers": {
       "X-Powered-By": "^koa$"
     },
@@ -2274,6 +2275,7 @@
     "cats": [
       64
     ],
+    "cpe": "cpe:2.3:a:konghq:kong:*:*:*:*:*:*:*:*",
     "description": "Kong is an open-source API gateway and platform that acts as middleware between compute clients and the API-centric applications.",
     "headers": {
       "via": "^kong/([\\d\\.]+)(?:.+)?$\\;version:\\1"

src/technologies/m.json

@@ -6701,7 +6701,7 @@
     "meta": {
       "generator": "Mijnwebwinkel"
     },
-    "website": "https://www.myonlinestore.com/ "
+    "website": "https://www.myonlinestore.com/"
   },
   "MyRest": {
     "cats": [

src/technologies/n.json

@@ -2536,6 +2536,7 @@
       18,
       57
     ],
+    "cpe": "cpe:2.3:a:nuxt:nuxt:*:*:*:*:*:*:*:*",
     "description": "Nuxt is a Vue framework for developing modern web applications.",
     "dom": [
       "div[id^='__nuxt']"