chore: update ekglib docs and components

Author: jgeluk

.vscode/settings.json

@@ -1,14 +1,11 @@
 {
-    "python.testing.unittestArgs": [
-        "-v",
-        "-s",
-        "./tests",
-        "-p",
-        "test_*.py"
-    ],
-    "python.testing.pytestEnabled": false,
-    "python.testing.nosetestsEnabled": false,
-    "python.testing.unittestEnabled": true,
-    "python.linting.pylintEnabled": true,
-    "python.linting.enabled": true
-}
+  // Python formatter configuration
+  "[python]": {
+    "editor.defaultFormatter": "charliermarsh.ruff",
+    "editor.formatOnSave": true,
+    "editor.codeActionsOnSave": {
+      "source.fixAll": "explicit",
+      "source.organizeImports": "explicit"
+    }
+  }
+}

README.md

@@ -1,7 +1,5 @@
 # ekglib
 
-> ⚠️ **Warning:** This is a pre-alpha project. Everything in this repo is subject to heavy change.
-
 A Python Library for various tasks in an EKG DataOps operation.
 
 ## Badges
@@ -13,39 +11,67 @@ A Python Library for various tasks in an EKG DataOps operation.
 [![Linting & Formatting: ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 [![Type checking: mypy](https://img.shields.io/badge/type%20checking-mypy-blue.svg)](https://mypy.readthedocs.io/)
 
-**Links:**
+## Metadata Parsers
 
-- 📖 [Documentation](https://ekgf.github.io/ekglib/)
-- 🐛 [Issue Tracker](https://github.com/EKGF/ekglib/issues)
-- 💬 [Discussions](https://github.com/EKGF/ekglib/discussions)
+- [Concept Parser](concept_parser/README.md)
+- [Persona Parser](persona_parser/README.md)
+- [Story Validate Rule Parser](dataops_rule_parser/README.md)
+- [Story Vaidate Rules Capture](dataops_rules_capture/README.md)
+- [Story Validate Rules Executor](dataops_rules_execute/README.md)
+- [Use Case Parser](use_case_parser/README.md)
+- [User Story Parser](user_story_parser/README.md)
 
----
+## Capture Steps
 
-## Metadata Parsers
+- [Xlsx Parser](xlsx_parser/README.md)
+- [LDAP Parser](ldap_parser/README.md)
 
-- [Concept Parser](concept_parser/)
-- [Persona Parser](persona_parser/)
-- [Story Validate Rule Parser](dataops_rule_parser/)
-- [Story Vaidate Rules Capture](dataops_rules_capture/)
-- [Story Validate Rules Executor](dataops_rules_execute/)
-- [Use Case Parser](use_case_parser/)
-- [User Story Parser](user_story_parser/)
+## Maturity Model Tools
 
-## Capture Steps
+- [Maturity Model Parser](maturity_model_parser/README.md)
+
+## Pipelines and Export
+
+- [Pipeline Framework](pipeline/README.md)
+- [Step Export](step_export/README.md)
+
+## LDAP Variants
+
+- [LDAP Parser to File](ldap_parser_to_file/README.md)
+- [LDAP Parser to S3](ldap_parser_to_s3/README.md)
 
-- [Xlsx Parser](xlsx_parser/)
-- [LDAP Parser](ldap_parser/)
+## Storage and Data Access
 
----
+- [S3 Helpers](s3/README.md)
+- [Data Sources](data_source/README.md)
+- [Datasets](dataset/README.md)
+
+## Knowledge Graph and SPARQL Utilities
+
+- [KG IRI Utilities](kgiri/README.md)
+- [SPARQL Helpers](sparql/README.md)
+- [Namespaces](namespace/README.md)
+- [Ontologies and Resources](resources/README.md)
+
+## Core Utilities
+
+- [Logging Utilities](log/README.md)
+- [String Utilities](string/README.md)
+- [Git Utilities](git/README.md)
+- [Exceptions](exceptions/README.md)
+- [MIME Helpers](mime/README.md)
+- [Main CLI Entrypoint](main/README.md)
 
 ## Installation
 
-**Using `uv` (recommended)**
+`ekglib` is not yet published on PyPI. You can install it directly from GitHub.
 
-If you are using `uv` to manage your project, add `ekglib` as a dependency:
+**Using `uv` from GitHub (recommended)**
+
+Add `ekglib` as a dependency from GitHub:
 
 ```bash
-uv add ekglib
+uv add --git https://github.com/EKGF/ekglib.git
 ```
 
 You can then run the provided CLI tools via `uv`:
@@ -59,19 +85,19 @@ uv run pipeline-example --help
 To install the CLI tools as global commands (similar to `pipx`):
 
 ```bash
-uv tool install ekglib
+uv tool install --git https://github.com/EKGF/ekglib.git
 
 xlsx-parser --help
 user-story-parser --help
 pipeline-example --help
 ```
 
-**Using `pip`**
+**Using `pip` from GitHub**
 
-If you prefer to use `pip` directly:
+If you prefer to use `pip`, you can install from the GitHub repo:
 
 ```bash
-python -m pip install ekglib
+python -m pip install "git+https://github.com/EKGF/ekglib.git"
 ```
 
 The console scripts will then be available on your `PATH`:
@@ -82,8 +108,6 @@ user-story-parser --help
 pipeline-example --help
 ```
 
----
-
 ## Development setup (from source)
 
 If you cloned this repository and want to work on `ekglib` itself:
@@ -94,8 +118,6 @@ uv sync
 
 This will create and populate a virtual environment using `uv` based on `pyproject.toml`.
 
----
-
 ## Tests
 
 To run all tests:
@@ -110,8 +132,6 @@ To run a single test:
 uv run pytest tests/<path-to-test> -k <name-of-test>
 ```
 
----
-
 ## Packaging
 
 ```bash

pyproject.toml

@@ -34,7 +34,6 @@ dependencies = [
     "python-dateutil",
     "rdflib",
     "requests",
-    "setuptools",
     "SPARQLWrapper",
     "stringcase",
     "toml",

src/ekglib/concept_parser/parse.py

@@ -30,7 +30,7 @@ def __init__(self, input_file_name, verbose):
             for s, p, o in self.g:
                 print((s, p, o))
 
-    def check(self):
+    def check(self) -> None:
         for concept in self.g.subjects(RDF.type, CONCEPT.Concept):
             log_item('Concept', concept)
             for rdfsLabel in self.g.objects(concept, RDFS.label):
@@ -41,7 +41,7 @@ def check(self):
         # TODO: Nothing much happens here yet
         #
 
-    def dump(self, output_file):
+    def dump(self, output_file: str | Path | None) -> None:
         if not output_file:
             warning('You did not specify an output file, no output file created')
             return

src/ekglib/data_source/README.md

@@ -0,0 +1,7 @@
+# data_source
+
+Abstractions and helpers for describing and working with input data sources in `ekglib` pipelines and parsers.
+
+This package provides a small set of reusable building blocks used across multiple components.
+
+

src/ekglib/data_source/various.py

@@ -1,8 +1,9 @@
 import os
 from argparse import ArgumentParser
+from typing import Any
 
 
-def set_cli_params(parser: ArgumentParser) -> None:
+def set_cli_params(parser: ArgumentParser) -> Any:
     ekg_data_source_code = os.getenv('EKG_DATA_SOURCE_CODE', None)
     group = parser.add_argument_group('Data Source')
     if ekg_data_source_code:

src/ekglib/dataops_rule_parser/parse.py

@@ -5,19 +5,15 @@
 
 import owlrl
 import rdflib
-from rdflib import Graph, Literal, URIRef, OWL, RDF, RDFS, XSD
+from rdflib import OWL, RDF, RDFS, XSD, Graph, Literal, URIRef
 
 from ..data_source import set_cli_params as data_source_set_cli_params
-from ..kgiri import (
-    EKG_NS,
-    set_kgiri_base,
-    set_kgiri_base_replace,
-    set_cli_params as kgiri_set_cli_params,
-    kgiri_replace_iri_in_literal,
-)
-from ..log import error, warning, log_error, log_list, log_item, log_iri
+from ..kgiri import EKG_NS, kgiri_replace_iri_in_literal
+from ..kgiri import set_cli_params as kgiri_set_cli_params
+from ..kgiri import set_kgiri_base, set_kgiri_base_replace
+from ..log import error, log_error, log_iri, log_item, log_list, warning
 from ..main import load_rdf_file_into_graph
-from ..namespace import RULE, PROV, RAW, DATAOPS, DATASET
+from ..namespace import DATAOPS, DATASET, PROV, RAW, RULE
 
 OWL._fail = (
     False  # workaround for this issue: https://github.com/RDFLib/OWL-RL/issues/53
@@ -95,11 +91,11 @@ def load_ontology(self, ontology_file_name: Path):
         log_item('Loading Ontology', ontology_file_name)
         load_rdf_file_into_graph(self.g, ontology_file_name)
 
-    def load_ontologies(self):
+    def load_ontologies(self) -> None:
         for ontology_file_name in ontology_file_names:
             self.load_ontology(self.ontologies_root / ontology_file_name)
 
-    def rdfs_infer(self):
+    def rdfs_infer(self) -> None:
         owlrl.RDFSClosure.RDFS_Semantics(self.g, True, True, True)
         closure_class = owlrl.return_closure_class(
             owl_closure=True, rdfs_closure=True, owl_extras=True, trimming=True
@@ -172,10 +168,10 @@ def set_sort_key(self):
             return '99-obfuscate'
         return f'10-{set_key}'
 
-    def key(self):
+    def key(self) -> str:
         return f'{self.set_key()}-{self.rule_file.parent.stem}'
 
-    def sort_key(self):
+    def sort_key(self) -> str:
         return f'{self.set_sort_key()}-{self.rule_file.parent.stem}'
 
     def create_rule_set(self):
@@ -221,7 +217,7 @@ def check_rule(self, rule_iri) -> bool:
     # Check to see if in the same directory as the rule.ttl file we have a .sparql_endpoint file
     # with the given name. If so, return its contents as a Literal.
     #
-    def check_sparql_file_name(self, sparql_file_name):
+    def check_sparql_file_name(self, sparql_file_name: str) -> Literal | None:
         if self.verbose:
             log_item('Checking', self.rule_file.parent / sparql_file_name)
         sparql_file_name_full_path = self.rule_file.parent / sparql_file_name
@@ -233,7 +229,9 @@ def check_sparql_file_name(self, sparql_file_name):
         )
         return rdflib.Literal(sparql_file_name_full_path.read_text())
 
-    def process_sparql_literal(self, rule, sparql_literal):
+    def process_sparql_literal(
+        self, rule: URIRef, sparql_literal: Literal | None
+    ) -> None:
         if not sparql_literal:
             return
         self.replace_literal_triple(
@@ -249,16 +247,18 @@ def process_sparql_literal(self, rule, sparql_literal):
     #
     # Add a triple to the graph with the given sparql_endpoint literal.
     #
-    def add_literal_triple(self, s, p, o):
+    def add_literal_triple(self, s: URIRef, p: URIRef, o: Literal) -> None:
         if self.verbose:
             print('Adding triple <{0}> - <{1}> - "{2}"'.format(s, p, o))
         self.g.add((s, p, o))
 
-    def replace_literal_triple(self, s, p1, p2, o):
+    def replace_literal_triple(
+        self, s: URIRef, p1: URIRef, p2: URIRef, o: Literal
+    ) -> None:
         self.g.remove((s, p1, None))
         self.add_literal_triple(s, p2, o)
 
-    def dump(self, output_file) -> int:
+    def dump(self, output_file: str | Path | None) -> int:
         if not output_file:
             warning('You did not specify an output file, no output file created')
             return 1
@@ -311,3 +311,5 @@ def main() -> int:
 
 if __name__ == '__main__':
     exit(main())
+if __name__ == '__main__':
+    exit(main())

src/ekglib/dataops_rules_capture/capture.py

@@ -103,7 +103,7 @@ def capture_rule_file(self, rule_file: Path, rule_directory_iri: URIRef):
         for triple in processor.g:
             self.g.add(triple)
 
-    def s3_file_name(self):
+    def s3_file_name(self) -> str:
         return f'raw-data-dataops-rules-{self.data_source_code}.ttl.gz'
 
     def export(self) -> int:

src/ekglib/dataset/README.md

@@ -0,0 +1,7 @@
+# dataset
+
+Helpers and types for representing datasets used by `ekglib` tools and pipelines.
+
+These utilities are typically consumed by higher-level parsers and exporters rather than used directly.
+
+

src/ekglib/exceptions/README.md

@@ -0,0 +1,7 @@
+# exceptions
+
+Common exception types and error utilities shared across `ekglib`.
+
+Centralizing these exceptions helps keep error handling consistent between components.
+
+