diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f66604..4b00209 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](https://semver.org/) +## [Unreleased] + +### Changed + +- all: update dev-dependencies +- plugin: tailored for Corporate Memory v25.1.x (cmem-plugin-base >= v4.9.0+) +- plugin: example test code now uses integrated Context classes + + ## [7.1.0] 2025-02-10 ### Changed diff --git a/src/.gitignore b/src/.gitignore index 2cb7280..214e10a 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -147,3 +147,5 @@ co artifacts .DS_Store .task +__metadata__.json + diff --git a/src/.gitlab-ci.yml b/src/.gitlab-ci.yml index cd56453..4eaa50f 100644 --- a/src/.gitlab-ci.yml +++ b/src/.gitlab-ci.yml @@ -46,6 +46,7 @@ pytest: - task check:pytest artifacts: when: always + name: artifacts-and__metadata__ reports: coverage_report: coverage_format: cobertura @@ -53,10 +54,8 @@ pytest: junit: - dist/junit-pytest.xml paths: - - dist/badge-coverage.svg - - dist/badge-tests.svg - - dist/coverage - - dist/coverage.xml + - __metadata__.json + - dist/* safety: stage: test diff --git a/src/README-public.md.jinja b/src/README-public.md.jinja index c0f2eef..a441202 100644 --- a/src/README-public.md.jinja +++ b/src/README-public.md.jinja @@ -11,7 +11,8 @@ cmemc admin workspace python install {{ package_name }} ``` {%- endif %} {% if github_page -%}[![workflow]({{ github_page }}/actions/workflows/check.yml/badge.svg)]({{ github_page}}/actions){%- endif %} {% if pypi -%}[![pypi version](https://img.shields.io/pypi/v/{{ package_name }})](https://pypi.org/project/{{ package_name }}){%- endif %} {% if pypi -%}[![license](https://img.shields.io/pypi/l/{{ package_name }})](https://pypi.org/project/{{ package_name }}){%- endif %} -[![poetry][poetry-shield]][poetry-link] [![ruff][ruff-shield]][ruff-link] [![mypy][mypy-shield]][mypy-link] [![copier][copier-shield]][copier] +[![poetry][poetry-shield]][poetry-link] [![ruff][ruff-shield]][ruff-link] [![mypy][mypy-shield]][mypy-link] [![copier][copier-shield]][copier] [![{cimd metadata enabled}][cmid-metadata-enabled]][cimd] + {% if project_type == 'plugin' -%}[cmem-link]: https://documentation.eccenca.com [cmem-shield]: https://img.shields.io/endpoint?url=https://dev.documentation.eccenca.com/badge.json{%- endif %} @@ -23,4 +24,6 @@ cmemc admin workspace python install {{ package_name }} [mypy-shield]: https://www.mypy-lang.org/static/mypy_badge.svg [copier]: https://copier.readthedocs.io/ [copier-shield]: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/copier-org/copier/master/img/badge/badge-grayscale-inverted-border-purple.json +[cimd]: https://github.com/seebi/cimd +[cmid-metadata-enabled]: https://img.shields.io/badge/%7Bcimd%7D-metadata_enabled-gray?labelColor=orange diff --git a/src/README.md.jinja b/src/README.md.jinja index 83508cc..c2e8454 100644 --- a/src/README.md.jinja +++ b/src/README.md.jinja @@ -3,7 +3,7 @@ {{ project_description }} {% if project_type == 'plugin' -%}[![eccenca Corporate Memory][cmem-shield]][cmem-link] {%- endif %}{% if github_page -%}[![workflow]({{ github_page }}/actions/workflows/check.yml/badge.svg)]({{ github_page}}/actions){%- endif %} {% if pypi -%}[![pypi version](https://img.shields.io/pypi/v/{{ package_name }})](https://pypi.org/project/{{ package_name}}){%- endif %} {% if pypi -%}[![license](https://img.shields.io/pypi/l/{{ package_name }})](https://pypi.org/project/{{ package_name}}){%- endif %} -[![poetry][poetry-shield]][poetry-link] [![ruff][ruff-shield]][ruff-link] [![mypy][mypy-shield]][mypy-link] [![copier][copier-shield]][copier] + [![poetry][poetry-shield]][poetry-link] [![ruff][ruff-shield]][ruff-link] [![mypy][mypy-shield]][mypy-link] [![copier][copier-shield]][copier] [![{cimd metadata enabled}][cmid-metadata-enabled]][cimd] ## Development @@ -21,3 +21,6 @@ [mypy-shield]: https://www.mypy-lang.org/static/mypy_badge.svg [copier]: https://copier.readthedocs.io/ [copier-shield]: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/copier-org/copier/master/img/badge/badge-grayscale-inverted-border-purple.json +[cimd]: https://github.com/seebi/cimd +[cmid-metadata-enabled]: https://img.shields.io/badge/%7Bcimd%7D-metadata_enabled-gray?labelColor=orange + diff --git a/src/Taskfile.yaml b/src/Taskfile.yaml index 1d04220..229e84c 100644 --- a/src/Taskfile.yaml +++ b/src/Taskfile.yaml @@ -93,7 +93,7 @@ tasks: clean: desc: Removes dist, *.pyc and some caches cmds: - - rm -rf {{.DIST_DIR}} .mypy_cache .pytest_cache + - rm -rf {{.DIST_DIR}} .mypy_cache .pytest_cache __metadata__.json - find . -name "*.pyc" -print0 | xargs -0 rm || echo "" # }}} @@ -120,29 +120,67 @@ tasks: # --memray is not used on windows - platforms: [windows] cmd: > - poetry run pytest --junitxml={{.JUNIT_FILE}} - --cov-report term --cov-report xml:{{.COVERAGE_FILE}} - --cov-report html:{{.COVERAGE_DIR}} --cov={{.PACKAGE}} - --html={{.HTML_FILE}} --self-contained-html + poetry run pytest + --cov={{.PACKAGE}} + --junitxml={{.PYTEST_XML}} + --html={{.PYTEST_HTML}} --self-contained-html + --cov-report term + --cov-report xml:{{.COVERAGE_FILE}} + --cov-report html:{{.COVERAGE_DIR}} + | tee {{.PYTEST_LOG}} - platforms: [darwin, linux] cmd: > - poetry run pytest --memray --junitxml={{.JUNIT_FILE}} - --cov-report term --cov-report xml:{{.COVERAGE_FILE}} - --cov-report html:{{.COVERAGE_DIR}} --cov={{.PACKAGE}} - --html={{.HTML_FILE}} --self-contained-html - - cmd: > - poetry run genbadge coverage -l - -i {{.COVERAGE_FILE}} -o {{.BADGE_COVERAGE}} - - cmd: > - poetry run genbadge tests -l - -i {{.JUNIT_FILE}} -o {{.BADGE_TESTS}} + poetry run pytest + --cov={{.PACKAGE}} + --memray --most-allocations=1 + --junitxml={{.PYTEST_XML}} + --html={{.PYTEST_HTML}} --self-contained-html + --cov-report term + --cov-report xml:{{.COVERAGE_FILE}} + --cov-report html:{{.COVERAGE_DIR}} + | tee {{.PYTEST_LOG}} + - > + poetry run genbadge coverage -l + -i {{.COVERAGE_FILE}} -o {{.BADGE_COVERAGE}} + - > + poetry run genbadge tests -l + -i {{.PYTEST_XML}} -o {{.BADGE_TESTS}} + - poetry run cimd extract coverage-xml {{.COVERAGE_FILE}} + - poetry run cimd extract junit-xml {{.PYTEST_XML}} + - poetry run cimd delete --field value 0 + - | + if [[ "$CI_JOB_URL" != "" ]]; then + poetry run cimd extend gitlab-link coverage-xml-line-rate --artifact-path {{.COVERAGE_DIR}}/index.html + poetry run cimd extend gitlab-link "junit-xml.*" --artifact-path {{.PYTEST_HTML}} + fi + - platforms: [darwin, linux] + task: check:pytest:cimd vars: BADGE_COVERAGE: ./{{.DIST_DIR}}/badge-coverage.svg BADGE_TESTS: ./{{.DIST_DIR}}/badge-tests.svg COVERAGE_DIR: ./{{.DIST_DIR}}/coverage COVERAGE_FILE: ./{{.DIST_DIR}}/coverage.xml - HTML_FILE: ./{{.DIST_DIR}}/pytest.html - JUNIT_FILE: ./{{.DIST_DIR}}/junit-pytest.xml + PYTEST_HTML: ./{{.DIST_DIR}}/pytest.html + PYTEST_XML: ./{{.DIST_DIR}}/junit-pytest.xml + PYTEST_LOG: ./{{.DIST_DIR}}/pytest.log + + check:pytest:cimd: + desc: prepare cimd metadata for pytest stage + internal: true + cmds: + - > + poetry run cimd add memray-max-memory {{.MAX_MEMORY}} + --label "Max Memory" + --description "Highest total memory allocated in a single test" + --image "https://img.shields.io/badge/Max%20Memory-{{.MAX_MEMORY}}-yellow" + - | + if [[ "$CI_JOB_URL" != "" ]]; then + poetry run cimd extend gitlab-link "memray.*" --artifact-path {{.PYTEST_LOG}} + fi + vars: + PYTEST_LOG: ./{{.DIST_DIR}}/pytest.log + MAX_MEMORY: + sh: cat {{.PYTEST_LOG}} | grep "Total memory allocated" | cut -d ":" -f 2 | tr -d "[:blank:]" check:mypy: desc: Complain about typing errors diff --git a/src/pyproject.toml.jinja b/src/pyproject.toml.jinja index 749a9a2..44cbff8 100644 --- a/src/pyproject.toml.jinja +++ b/src/pyproject.toml.jinja @@ -20,22 +20,23 @@ python = "^3.11"{%- else %} python = ">=3.11, ^3"{%- endif %} {% if project_type == 'plugin' -%}[tool.poetry.dependencies.cmem-plugin-base] -version = "^4.8.0" +version = "^4.9.0" allow-prereleases = false [tool.poetry.group.dev.dependencies.cmem-cmemc] version = "^24.3.0"{%- endif %} [tool.poetry.group.dev.dependencies] +cimd = "^0" genbadge = {extras = ["coverage"], version = "^1.1.1"} -mypy = "^1.14.1" -pip = "^25.0" +mypy = "^1.15.0" +pip = "^25.0.1" pytest = "^8.3.4" pytest-cov = "^6.0.0" pytest-dotenv = "^0.5.2" pytest-html = "^4.1.1" pytest-memray = { version = "^1.7.0", markers = "platform_system != 'Windows'" } -ruff = "^0.9.4" +ruff = "^0.9.7" safety = "^1.10.3" [build-system] diff --git a/src/tests/{% if project_type == 'plugin' %}test_example.py{% endif %}.jinja b/src/tests/{% if project_type == 'plugin' %}test_example.py{% endif %}.jinja index ba9e497..2c7c119 100644 --- a/src/tests/{% if project_type == 'plugin' %}test_example.py{% endif %}.jinja +++ b/src/tests/{% if project_type == 'plugin' %}test_example.py{% endif %}.jinja @@ -4,6 +4,9 @@ Remove this and other example files after bootstrapping your project. """ import io +import os +from collections.abc import Generator +from typing import Any import pytest from cmem.cmempy.workspace.projects.datasets.dataset import make_new_dataset @@ -12,40 +15,47 @@ from cmem.cmempy.workspace.projects.resources.resource import ( create_resource, get_resource_response, ) +from cmem_plugin_base.testing import TestExecutionContext from {{ package_dir }}.example_transform import Lifetime from {{ package_dir }}.example_workflow import DollyPlugin -from tests.utils import TestExecutionContext, needs_cmem -PROJECT_NAME = "{{ package_dir }}_test_project" -DATASET_NAME = "sample_dataset" -RESOURCE_NAME = "sample_dataset.txt" -DATASET_TYPE = "text" +needs_cmem = pytest.mark.skipif( + os.environ.get("CMEM_BASE_URI", "") == "", reason="Needs CMEM configuration" +) + + +class BuildProject: + """Build Project Fixture""" + + project_name = "{{ package_dir }}_test_project" + dataset_name = "sample_dataset" + resource_name = "sample_dataset.txt" + dataset_type = "text" + dataset_content = "{{ package_dir }} plugin sample file." @pytest.fixture -def di_environment() -> object: +def build_project() -> Generator[BuildProject, Any, None]: """Provide the DI build project incl. assets.""" - make_new_project(PROJECT_NAME) + _ = BuildProject() + make_new_project(_.project_name) make_new_dataset( - project_name=PROJECT_NAME, - dataset_name=DATASET_NAME, - dataset_type=DATASET_TYPE, - parameters={"file": RESOURCE_NAME}, + project_name=_.project_name, + dataset_name=_.dataset_name, + dataset_type=_.dataset_type, + parameters={"file": _.resource_name}, autoconfigure=False, ) - with io.StringIO("{{ package_name }} plugin sample file.") as response_file: + with io.StringIO(_.dataset_content) as response_file: create_resource( - project_name=PROJECT_NAME, - resource_name=RESOURCE_NAME, + project_name=_.project_name, + resource_name=_.resource_name, file_resource=response_file, replace=True, ) - yield { - "project": PROJECT_NAME, - "dataset": RESOURCE_NAME, - } - delete_project(PROJECT_NAME) + yield _ + delete_project(_.project_name) @needs_cmem @@ -74,7 +84,7 @@ def test_transform_execution_with_inputs() -> None: @needs_cmem -def test_integration_placeholder(di_environment: dict) -> None: +def test_integration_placeholder(build_project: BuildProject) -> None: """Placeholder to write integration testcase with cmem""" - with get_resource_response(di_environment["project"], di_environment["dataset"]) as response: - assert response.text != "" + with get_resource_response(build_project.project_name, build_project.resource_name) as response: + assert response.text == build_project.dataset_content diff --git a/src/tests/{% if project_type == 'plugin' %}utils.py{% endif %}.jinja b/src/tests/{% if project_type == 'plugin' %}utils.py{% endif %}.jinja deleted file mode 100644 index 1a1c381..0000000 --- a/src/tests/{% if project_type == 'plugin' %}utils.py{% endif %}.jinja +++ /dev/null @@ -1,74 +0,0 @@ -"""Testing utilities. - -Remove this and other example files after bootstrapping your project. -""" - -import os -from typing import ClassVar - -import pytest - -# check for cmem environment and skip if not present -from cmem.cmempy.api import get_token -from cmem.cmempy.config import get_oauth_default_credentials -from cmem_plugin_base.dataintegration.context import ( - ExecutionContext, - PluginContext, - ReportContext, - TaskContext, - UserContext, -) - -needs_cmem = pytest.mark.skipif( - os.environ.get("CMEM_BASE_URI", "") == "", reason="Needs CMEM configuration" -) - - -class TestUserContext(UserContext): - """dummy user context that can be used in tests""" - - __test__ = False - default_credential: ClassVar[dict] = {} - - def __init__(self): - # get access token from default service account - if not TestUserContext.default_credential: - TestUserContext.default_credential = get_oauth_default_credentials() - access_token = get_token(_oauth_credentials=TestUserContext.default_credential)[ - "access_token" - ] - self.token = lambda: access_token - - -class TestPluginContext(PluginContext): - """dummy plugin context that can be used in tests""" - - __test__ = False - - def __init__( - self, - project_id: str = "dummyProject", - ): - self.project_id = project_id - self.user = TestUserContext() - - -class TestTaskContext(TaskContext): - """dummy Task context that can be used in tests""" - - __test__ = False - - def __init__(self, project_id: str = "dummyProject", task_id: str = "dummyTask"): - self.project_id = lambda: project_id - self.task_id = lambda: task_id - - -class TestExecutionContext(ExecutionContext): - """dummy execution context that can be used in tests""" - - __test__ = False - - def __init__(self, project_id: str = "dummyProject", task_id: str = "dummyTask"): - self.report = ReportContext() - self.task = TestTaskContext(project_id=project_id, task_id=task_id) - self.user = TestUserContext()