Taxonomy on startup (#561)

* Add convenience script for build command. Should maybe merge later

* Avoid start taxonomy on startup

If we do want to roll out taxonomy updates automatically, we
probably want to regularly spin up the taxonomy container to
allow updates while the REST API is live.

* Extract method so module can also be used on its own without cli

* Import specified taxonomy on startup

* Add documentation on taxonomies and its service

* Predefine the import folder of taxonomies

Author: PGijsbers

data/taxonomies/.gitkeep

@@ -3,6 +3,7 @@ services:
     stdin_open: true
     volumes:
       - ./src:/app:ro
+      - ${DATA_PATH}/taxonomies:/data/taxonomies
     command: python main.py
 
   fill-db-with-examples:

docker-compose.dev.yaml

@@ -237,9 +237,9 @@ services:
         condition: service_completed_successfully
 
   taxonomy:
+    profiles: ["taxonomy"]
     container_name: taxonomy
     image: aiod_metadata_catalogue
     volumes:
-      - ./data/taxonomies.json:/data/taxonomies.json
-      - ./src:/app
+      - ${DATA_PATH}/taxonomies/taxonomies.json:/data/taxonomies.json
     command: python taxonomies/synchronize_taxonomy.py

docker-compose.yaml

@@ -0,0 +1,25 @@
+# Taxonomies
+
+AI-on-Demand uses [taxonomies](https://en.wikipedia.org/wiki/Taxonomy) to standardize terms across assets, for example, for [licenses](https://aiod-dev.i3a.es/docs#/Taxonomies/license_licenses_get), [business sectors](https://aiod-dev.i3a.es/docs#/Taxonomies/industrial_sector_industrial_sectors_get), or [news categories](https://aiod-dev.i3a.es/docs#/Taxonomies/news_category_news_categorys_get).
+These taxonomies are defined by the [conceptual model](https://github.com/aiondemand/metadata-schema).
+Each term in a taxonomy has a specific definition and may have subterms defined. e.g., the business sector `construction` has subsectors for `infrastructure` and `buildings`, which each may also have subterms.
+
+## Importing the Taxonomy
+The JSON file produced by the export script in the metadata repository can be used as input for the `taxonomy` service of this project's `docker compose`.
+Put the JSON file at `${DATA_PATH}/taxonomies/taxonomies.json`, where `DATA_PATH` is an environment variable (typically set from the `.env` file).
+Then invoke docker compose with the "taxonomy" profile. The script will then:
+
+ - invalidate all old known terms that are no longer part of the taxonomy: assets which already use them will keep them, but they may not be added to new items.
+ - update existing terms, e.g., with new definitions
+ - add new terms to the taxonomy
+
+## Development Taxonomy
+When using the development configuration, you put a file in the `${DATA_PATH}/taxonomies` directory and point to it from the API configuration file (`config.override.toml`), where `${DATA_PATH}/taxonomies` will be mounted as `/data/taxonomies`. For example, with `DATA_PATH=./data` and `./data/taxonomies/example_taxonomies.json` present:
+
+```toml
+[dev]
+taxonomy="/data/taxonomies/example_taxonomies.json"
+```
+
+It is not intended to use this in production, since it might result in accidentally overwriting the taxonomies.
+In production, use the `taxonomy` service discussed above.

docs/developer/schema/taxonomies.md

@@ -56,6 +56,7 @@ nav:
           - 'Relationships': developer/schema/relationships.md
           - 'Objects': developer/schema/objects.md
           - 'Schema Migration': developer/schema/migration.md
+          - 'Taxonomies': developer/schema/taxonomies.md
       - 'Elastic Search': developer/elastic_search.md
       - 'Scripts': developer/scripts.md
       - 'Release Workflow': developer/releases.md

mkdocs.yaml

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+profiles=""
+for arg in "$@"; do
+  profiles+="--profile $arg "
+done
+
+NC='\033[0m' # No Color
+CYAN='\033[1;36m'
+GREEN='\033[0;32m'
+
+source .env
+[ ! -f override.env ] && touch override.env
+source override.env
+
+if [[ "${USE_LOCAL_DEV}" == "true" ]]; then
+  compose_with_dev="-f docker-compose.dev.yaml"
+  profiles+="--profile nginx"
+  echo -e "Launching ${CYAN}with${NC} local changes."
+else
+  compose_with_dev=""
+  echo -e "Launching ${GREEN}without${NC} local changes."
+fi
+
+command="docker compose --env-file=.env --env-file=override.env -f docker-compose.yaml ${compose_with_dev} ${profiles} build"
+echo "${command}"
+eval "${command}"

scripts/build.sh

@@ -29,6 +29,7 @@ request_timeout = 10  # seconds
 log_level = "INFO"  # Python log levels: https://docs.python.org/3/library/logging.html#logging-levels
 disable_reviews = false  # set to true to automatically publish submissions
 url_prefix = ""
+taxonomy = ""  # Set to a JSON file containing a taxonomy to put into the database on startup
 
 # Authentication and authorization
 [keycloak]

src/config.default.toml

@@ -8,6 +8,7 @@
 import argparse
 from datetime import datetime, timezone
 import logging
+from pathlib import Path
 
 import pkg_resources
 import uvicorn
@@ -28,6 +29,7 @@
 from database.model.platform.platform_names import PlatformName
 from database.session import EngineSingleton, DbSession
 from database.setup import create_database, database_exists
+from taxonomies.synchronize_taxonomy import synchronize_taxonomy_from_file
 from triggers import disable_review_process, enable_review_process
 from error_handling import http_exception_handler
 from database.model.agent.agent import Agent
@@ -105,6 +107,11 @@ def create_app() -> FastAPI:
         drop_database = build_database_setting == "drop-then-build"
         build_database(drop_database=drop_database)
 
+    if taxonomy_path := DEV_CONFIG.get("taxonomy"):
+        if not (taxonomy_file := Path(taxonomy_path)).is_file():
+            raise ValueError(f"dev.taxonomy must be a path to a file, but is {taxonomy_path!r}.")
+        synchronize_taxonomy_from_file(taxonomy_file)
+
     pyproject_toml = pkg_resources.get_distribution("aiod_metadata_catalogue")
     app = build_app(url_prefix=DEV_CONFIG.get("url_prefix", ""), version=pyproject_toml.version)
     return app

src/main.py

@@ -30,7 +30,7 @@ def parse_args():
 class Term(NamedTuple):
     name: str
     definition: str
-    children: list[Self]  #  type: ignore[valid-type]
+    children: list[Self]  # type: ignore[valid-type]
 
 
 type_by_name: dict[str, type] = {
@@ -116,17 +116,21 @@ def synchronize_term(term: Taxonomy):
         synchronize_term(term)
 
 
-def main():
-    logging.basicConfig(level=logging.INFO)
-    logging.info("Starting synchronization script.")
-    args = parse_args()
-    taxonomies = load_taxonomies_from_json(args.definitions_file)
+def synchronize_taxonomy_from_file(file: Path) -> None:
+    taxonomies = load_taxonomies_from_json(file)
     with DbSession(autoflush=False) as session:
         for type_, definitions in taxonomies:
             synchronize(type_, definitions, session)
         logging.info("Committing changes to database.")
         session.commit()
 
 
+def main():
+    logging.basicConfig(level=logging.INFO)
+    logging.info("Starting synchronization script.")
+    args = parse_args()
+    synchronize_taxonomy_from_file(args.definitions_file)
+
+
 if __name__ == "__main__":
     main()