From ea4070d4570bfdd36adb345be5d011e25f29d590 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Thu, 31 Jul 2025 12:01:58 -0700 Subject: [PATCH 1/7] [FEATURE] add flak8-cq to codeql-analysis.yml Signed-off-by: Mr. Walls --- .github/workflows/codeql-analysis.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d243379..66aab91 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -13,7 +13,7 @@ name: "CodeQL" on: push: - branches: [ master, stable ] + branches: [ master, stable, feature-flake8-87 ] pull_request: # The branches below must be a subset of the branches above branches: [ stable ] @@ -21,7 +21,7 @@ on: - cron: '17 5 * * 1' # Declare default permissions as read only. -permissions: read-all +permissions: {} jobs: analyze: @@ -71,6 +71,13 @@ jobs: #- run: | # make bootstrap # make release + - name: Flake8 Scan + uses: reactive-firewall/flake8-cq@5a4f0f0e90a5c94b3f0fa1e659f2ec565b76be35 # v1.6a0 + with: # optional arguments + config: '.flake8.ini' + match: '**/*.py' + publish-artifacts: false + if: ${{ success() }} - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 From 922dd776a6dc12d57b0f3fef05b70d54beccefe1 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Thu, 31 Jul 2025 12:35:25 -0700 Subject: [PATCH 2/7] [UPDATE] version bump for flake8-cq --- .github/workflows/codeql-analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 66aab91..4a49fc7 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -72,7 +72,7 @@ jobs: # make bootstrap # make release - name: Flake8 Scan - uses: reactive-firewall/flake8-cq@5a4f0f0e90a5c94b3f0fa1e659f2ec565b76be35 # v1.6a0 + uses: reactive-firewall/flake8-cq@cfe54a7621fbb7290a68d1097e8e63532cb5e57d # v1.6a2 with: # optional arguments config: '.flake8.ini' match: '**/*.py' From b6d35bd7b0b445cb3f6a2a6d586b8383422c012c Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Mon, 11 Aug 2025 14:28:24 -0700 Subject: [PATCH 3/7] [UPDATE] Version bump for flake8-cq to v2.0 --- .github/workflows/codeql-analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4a49fc7..4b601d2 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -72,7 +72,7 @@ jobs: # make bootstrap # make release - name: Flake8 Scan - uses: reactive-firewall/flake8-cq@cfe54a7621fbb7290a68d1097e8e63532cb5e57d # v1.6a2 + uses: reactive-firewall/flake8-cq@v2 # v2.0 with: # optional arguments config: '.flake8.ini' match: '**/*.py' From bfcf1bfd3d1078e8286ab2e49b1740cab74e9955 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Tue, 12 Aug 2025 15:02:04 -0700 Subject: [PATCH 4/7] [PATCH] Apply changes from review (- WIP PR #89 -) --- .circleci/config.yml | 2 +- docs/conf.py | 357 ++++++++++++++++++++++++++++++++------- docs/utils.py | 335 ++++++++++++++++++++++++++++++++++++ pythonrepo/__init__.py | 21 +-- pythonrepo/pythonrepo.py | 62 +++---- tests/__init__.py | 29 ++-- tests/context.py | 7 +- 7 files changed, 687 insertions(+), 126 deletions(-) create mode 100644 docs/utils.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 144e304..83f8d6c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,7 +33,7 @@ commands: parameters: python-version: type: string - default: "3.12" + default: "3.13" jobs: build: diff --git a/docs/conf.py b/docs/conf.py index 314411b..362f1b4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,68 +11,149 @@ # All configuration values have a default; values that are commented out # serve to show the default. +"""Sphinx documentation configuration file. + +This module contains configuration settings for building the project's documentation +using Sphinx. It sets up extensions, themes, and various build parameters. + +Attributes: + DOCS_BUILD_REF (str): The Git reference used in GitHub links, defaults to 'stable'. + needs_sphinx (str): Minimum required Sphinx version. + extensions (list): Sphinx extensions to be used. + autodoc2_packages (list): Packages to be documented using autodoc2. + project (str): Name of the project. + copyright (str): Copyright information. + version (str): Short X.Y version. + release (str): Full version including alpha/beta/rc tags. + +Example: + >>> import conf + >>> conf.project + 'python-repo' + >>> conf.version + '1.1' +""" + import sys import os +from urllib.parse import quote + +# If you have custom documentation extensions, add them here +sys.path.insert(0, os.path.abspath("..")) +from docs.utils import ( + _validate_git_ref, # noqa + slugify_header, # noqa + sanitize_url, # noqa + sanitize_intersphinx_mapping # noqa +) + +# Define the branch reference for linkcode_resolve +DOCS_BUILD_REF: str = _validate_git_ref(os.environ.get("DOCS_BUILD_REF", "HEAD")) +""" +The Git reference used in the GitHub links. +Used by linkcode_resolve() to generate GitHub links. Accepts most git references +(commit hash or branch name). +Value: + str: The Git reference, defaults to 'HEAD' (latest) if DOCS_BUILD_REF environment + variable is not set. +""" + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath('../')) -sys.path.insert(1, os.path.abspath('./pythonrepo')) + +# sys.path.insert(1, os.path.abspath("tests")) # uncomment to add tests docs +sys.path.insert(1, os.path.abspath("pythonrepo")) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = '5.3' +needs_sphinx: str = "7.3" -# Add any Sphinx extension module names here, as strings. They can be extensions +# Add any Sphinx extension module names below, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -# for md us 'autodoc2' (pip install sphinx-autodoc2) -# for rst use 'sphinx.ext.autodoc' + +# Hints: +# for GHF md try 'sphinx.ext.githubpages', 'sphinxcontrib.mermaid' and 'myst_parser' +# (pip install 'sphinxcontrib-mermaid') +# (pip install 'myst-parser[linkify]') +# for reST ONLY projects, can disable 'myst_parser' +# Auto-documenting +# for md use 'autodoc2' (pip install sphinx-autodoc2) +# for reST use 'sphinx.ext.autodoc' + +# Add any Sphinx extension module names here, as strings. +# type-hint extensions: list[str] extensions = [ - 'sphinx.ext.napoleon', 'autodoc2', 'sphinx.ext.autosummary', - 'sphinx.ext.githubpages', 'myst_parser', - 'sphinx.ext.doctest', 'sphinx.ext.todo', - 'sphinx.ext.linkcode', 'sphinx.ext.viewcode' - ] + "sphinx.ext.napoleon", + "autodoc2", + "sphinx.ext.autosectionlabel", + "sphinx.ext.githubpages", + "sphinxcontrib.mermaid", + "myst_parser", + "sphinx_design", + "sphinx.ext.autosummary", + "sphinx.ext.doctest", + "sphinx.ext.todo", + "sphinx.ext.linkcode", + "sphinx.ext.viewcode", + "sphinx.ext.intersphinx", +] -# for md auto-docs +# for md auto-docs (comment or remove to use default RST mode) +# type-hint autodoc2_packages: list[str] autodoc2_packages = [ "pythonrepo", "tests", ] +# also for md auto-docs to use myst (comment or remove to use default RST mode) +# type-hint autodoc2_render_plugin: str autodoc2_render_plugin = "myst" # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +# type-hint source_suffix: list[str] +templates_path = ["_templates"] # The suffix of source filenames. +# type-hint source_suffix: dict source_suffix = { - '.md': 'markdown', - '.txt': 'markdown', - '.rst': 'restructuredtext', + ".yml": "yaml", + ".toml": "toml", + ".md": "markdown", + ".txt": "markdown", + "Makefile": "makefile", + ".rst": "restructuredtext", } # The encoding of source files. Official sphinx docs recommend utf-8-sig. -source_encoding = 'utf-8-sig' +# type-hint source_encoding: str +source_encoding = "utf-8-sig" # The master toctree document. -master_doc = 'toc' +# type-hint master_doc: str +master_doc = "toc" # General information about the project. -project = u'python_template' -copyright = u'2017-2024, reactive-firewall' +# type-hint project: str +project = "pythonrepo" + +# add the by-line or string for author(s) here +authors: str = "reactive-firewall" -# The version info for the project you're documenting, acts as replacement for +# type-hint copyright: str +copyright = f"2017-2025, {authors}" + +# The version info for the project yo"re documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = 'v2.0' +version = "v2.0" # The full version, including alpha/beta/rc tags. -release = 'v2.0.0-alpha' +release = "v2.0.0b0" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -83,11 +164,25 @@ # today = '' # Else, today_fmt is used as the format for a strftime call. # today_fmt = '%B %d, %Y' -today_fmt = '%Y.%B.%d' +today_fmt = "%Y.%B.%d" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build', '.github', '.circleci', '.DS_Store', '**/.git', 'dist', 'tests'] +# type-hint exclude_patterns: list[str] +exclude_patterns = [ + "*~", + "www", + "dist", + "_build", + ".github", + "**/docs", + "**/.git", + ".DS_Store", + ".circleci", + "codecov_env", + "../tests/tests/**", + f"../{project}/{project}/**", +] # The reST default role (used for this markup: `text`) to use for all documents. # default_role = None @@ -107,8 +202,35 @@ show_authors = True # The name of the Pygments (syntax highlighting) style to use. -# pygments_style = 'py' -pygments_style = 'default' +# pygments_style = "default" + +# For GHF md try these themes (pip install 'sphinxawesome-theme>=5.2') +# pygments_style = "github" +# and for dark-mode +# pygments_style_dark = "github-dark" + +# and for improved highlighting, try these advanced highlighting settings: + +# pygments_options: dict = { +# "tabsize": 4, +# "stripall": False, +# "encoding": "utf-8", +# } + +# pygments_yaml_options: dict = { +# "tabsize": 2, +# "stripall": True, +# "encoding": "utf-8", +# } + +# highlight_options = { +# "default": pygments_options, +# "python": pygments_options, +# "yaml": pygments_yaml_options, +# "ini": pygments_yaml_options, +# "makefile": pygments_options, +# } + # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] @@ -120,7 +242,8 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'sphinxawesome_theme' +# (pip install 'sphinxawesome-theme>=5.2') +html_theme = "sphinxawesome_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -135,7 +258,7 @@ # html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -html_short_title = 'Project Docs' +html_short_title = "Project Docs" # The name of an image file (relative to this directory) to place at the top # of the sidebar. @@ -146,16 +269,17 @@ # pixels large. # html_favicon = None -html_permalinks_icon = '#' +html_permalinks_icon = "#" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. # html_last_updated_fmt = '%b %d, %Y' +html_last_updated_fmt = today_fmt.strip() # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. @@ -172,7 +296,7 @@ # html_domain_indices = True # If false, no index is generated. -# html_use_index = True +html_use_index = True # If true, the index is split into individual pages for each letter. # html_split_index = False @@ -184,7 +308,7 @@ # html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -# html_show_copyright = True +html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the @@ -195,26 +319,71 @@ # html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'python_repo_doc' +htmlhelp_basename = "python_repo_doc" + +# -- Options for Mermaid diagrams ----------------------------------------------- +# see https://github.com/mgaitan/sphinxcontrib-mermaid?tab=readme-ov-file#markdown-support + +# GFM style mermaid use zoom +mermaid_d3_zoom = True +# Mermaid Diagram Themes +mermaid_params = ["--theme", "default", "--backgroundColor", "transparent"] +# for darkmode try: +# mermaid_params = ["--theme", "dark", "--backgroundColor", "transparent"] # -- Options for MyST markdown parser ------------------------------------------- -# see https://myst-parser.readthedocs.io/en/latest/syntax/roles-and-directives.html#syntax-directives +# see https://myst-parser.readthedocs.io/en/latest/syntax/roles-and-directives.html # be more like GFM with style -myst_enable_extensions = ('tasklist', 'strikethrough', 'fieldlist') +myst_enable_extensions = ("tasklist", "strikethrough", "fieldlist", "linkify") # for GFM diagrams and interoperability with other Markdown renderers -myst_fence_as_directive = ('mermaid', 'suggestion', 'note') +myst_fence_as_directive = ("mermaid", "suggestion", "note") + +# Add linkify configuration +myst_linkify_fuzzy_links = False # Focus only on github markdown +# NOTE: We keep myst_gfm_only = False because setting it to True +# caused a regression where the 'toc.md' contents fence broke. +# Re-enabling GitHub Flavored Markdown exclusively +# should be approached with caution to avoid that issue. myst_gfm_only = False +# html metadata for MyST content +myst_html_meta = { + "github_url": sanitize_url(f"https://github.com/{authors}/{project}"), +} -#heading_anchors = 1 +# For GH-style admonitions to MyST conversion (still have to use directive with MyST) +myst_admonition_aliases = { + "note": "note", + "warning": "warning", + "important": "important", + "tip": "tip", + "caution": "caution" +} + +# how deep should markdown headers have anchors be generated +heading_anchors = 3 + +# Enable header anchors as requested +myst_heading_anchors = 3 + +# For better slug generation in header references (MUST be imported as above) +myst_heading_slug_func = slugify_header # -- Options for napoleon ext -------------------------------------------------- +# also for GHF md try these +napoleon_google_docstring = True +napoleon_numpy_docstring = False + +# exclude 'private' when it has docstrings +napoleon_include_private_with_doc = False +napoleon_include_special_with_doc = False + # include __init__ when it has docstrings napoleon_include_init_with_doc = True @@ -237,13 +406,7 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ( - u'index', - u'Documentation.tex', - u'python repo Template Documentation', - u'reactive-firewall', - u'manual' - ), + ("index", "Documentation.tex", f"{project} Documentation", f"{authors}", "manual"), ] # The name of an image file (relative to this directory) to place at the top of @@ -271,15 +434,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - ( - u'index', - u'python_repo', - u'python_repo Template Documentation', - [u'reactive-firewall'], - 1 - ) -] +man_pages = [("index", f"{project}", f"{project} Documentation", [f"{authors}"], 8)] # If true, show URL addresses after external links. # man_show_urls = False @@ -292,13 +447,13 @@ # dir menu entry, description, category) texinfo_documents = [ ( - u'index', - u'Python Repo', - u'python_repo Template Documentation', - u'reactive-firewall', - u'python_repo Template', - u'python_repo Template.', - u'Miscellaneous' + "index", + f"{project}", + f"{project} Documentation", + f"{authors}", + f"{project}", + f"{project} Python Module.", + "Miscellaneous" ), ] @@ -312,8 +467,80 @@ # texinfo_show_urls = 'footnote' # -- Link resolver ------------------------------------------------------------- -def linkcode_resolve(domain, info): - if domain != 'py' or not info.get('module'): + +linkcode_url_prefix: str = sanitize_url(f"https://github.com/{authors}/{project}") + +suffix: str = "/issues/%s" + +extlinks: dict[str,tuple] = { + "issue": ( + f"{linkcode_url_prefix}/{suffix}", + "issue #%s" + ) +} + +# try to link with official python3 documentation. +# see https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html for more +intersphinx_mapping = sanitize_intersphinx_mapping( + { + "python": ("https://docs.python.org/3", (None, "python-inv.txt")), + "PEP": ("https://peps.python.org", (None, "pep-inv.txt")), + }, +) + + +def linkcode_resolve(domain: Any, info: Any) -> str: + """ + Resolves selectively linking to GitHub source-code for the multicast module. + + See https://www.sphinx-doc.org/en/master/usage/extensions/linkcode.html for more details. + + Unit-Testing: + + First set up test fixtures by importing conf. + + >>> import docs.conf as _conf + >>> + + Testcase 1: Test function with input. + + >>> _conf.linkcode_resolve is not None + True + >>> test_text = "multicast.env" # this is resolved + >>> bad_input = False # this is invalid + >>> res_text = _conf.linkcode_resolve("py", info={"module": test_text}) + >>> res_text is not None + True + >>> type(res_text) is type(str()) + True + >>> len(res_text) > 0 + True + >>> res_text is not test_text + True + >>> _conf.linkcode_resolve("py", info={"module": test_text,}) == res_text + True + >>> _conf.linkcode_resolve("py", info={"module": bad_input,}) is None + True + >>> len(res_text) > 0 + True + >>> + >>> # cleanup from unit-test + >>> del bad_input + >>> del test_text + >>> del res_text + >>> + """ + if not isinstance(domain, str) or domain != "py": + return None + if not isinstance(info, dict) or "module" not in info or not info["module"]: return None - filename = info['module'].replace('.', '/') - return "https://github.com/reactive-firewall/python-repo/blob/stable/%s.py" % filename + if not isinstance(info["module"], str): + return None + filename = info["module"].replace(".", "/") + theResult = f"{linkcode_url_prefix}/blob/{DOCS_BUILD_REF}/{filename}.py" + if "/{project}.py" in theResult: + theResult = theResult.replace(f"/{project}.py", f"/{project}/__init__.py") + if "/tests.py" in theResult: + theResult = theResult.replace("/tests.py", "/tests/__init__.py") + return sanitize_url(quote(theResult, safe=":/-._")) + diff --git a/docs/utils.py b/docs/utils.py new file mode 100644 index 0000000..449aaa7 --- /dev/null +++ b/docs/utils.py @@ -0,0 +1,335 @@ +#! /usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Python Repo Template +# .................................. +# Copyright (c) 2024-2025, Mr. Walls +# .................................. +# Licensed under MIT (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# .......................................... +# https://github.com/reactive-firewall/python-repo/LICENSE.md +# .......................................... +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import unicodedata +from urllib.parse import ( + ParseResult, # noqa + urlparse, # noqa + urlunparse, # noqa + quote # noqa +) + + +# Git reference validation pattern +# Enforces: +# - Must start with alphanumeric character +# - Can contain alphanumeric characters, underscore, hyphen, forward slash, and dot +GIT_REF_PATTERN: str = r'^[a-zA-Z0-9][a-zA-Z0-9_\-./]*$' + + +# URL allowed scheme list +# Enforces: +# - URLs Must start with https +URL_ALLOWED_SCHEMES: frozenset = frozenset({"https"}) + + +# URL allowed domain list +# Enforces: +# - URLs Must belong to one of these domains +URL_ALLOWED_NETLOCS: frozenset = frozenset({ + "github.com", "gist.github.com", "readthedocs.com", "docs.python.org", "peps.python.org", +}) + + +# Maximum allowed URL length +MAX_URL_LENGTH: int = 2048 # Common browser limit +"""Maximum allowed length for URL validation. + +Should be large enough for most URLs but no larger than common browser limits. + +Unit-Testing: + + First set up test fixtures by importing utils. + + >>> import docs.utils as _utils + >>> + + >>> _utils.MAX_URL_LENGTH is not None + True + >>> type(_utils.MAX_URL_LENGTH) is type(int()) + True + >>> _utils.MAX_URL_LENGTH > 0 + True + >>> _utils.MAX_URL_LENGTH >= 256 + True + >>> _utils.MAX_URL_LENGTH <= 2048 + True + >>> + +""" + + +# Error messages for URL validation +INVALID_LENGTH_ERROR: str = f"URL exceeds maximum length of {MAX_URL_LENGTH} characters." +"""Length error message for URL validation. + +Unit-Testing: + + First set up test fixtures by importing utils. + + >>> import docs.utils as _utils + >>> + + >>> _utils.INVALID_LENGTH_ERROR is not None + True + >>> type(_utils.INVALID_LENGTH_ERROR) is type(str()) + True + >>> len(_utils.INVALID_LENGTH_ERROR) > 0 + True + >>> + +""" + + +INVALID_SCHEME_ERROR: str = "Invalid URL scheme. Only 'https' is allowed." +"""Scheme error message for URL validation. + +Unit-Testing: + + First set up test fixtures by importing utils. + + >>> import docs.utils as _utils + >>> + + >>> _utils.INVALID_SCHEME_ERROR is not None + True + >>> type(_utils.INVALID_SCHEME_ERROR) is type(str()) + True + >>> len(_utils.INVALID_SCHEME_ERROR) > 0 + True + >>> + +""" + + +INVALID_DOMAIN_ERROR: str = f"Invalid or untrusted domain. Only {URL_ALLOWED_NETLOCS} are allowed." +"""Domain error message for URL validation. + +Unit-Testing: + + First set up test fixtures by importing utils. + + >>> import docs.utils as _utils + >>> + + >>> _utils.INVALID_DOMAIN_ERROR is not None + True + >>> type(_utils.INVALID_DOMAIN_ERROR) is type(str()) + True + >>> len(_utils.INVALID_DOMAIN_ERROR) > 0 + True + >>> + +""" + + +def _validate_git_ref(ref: str) -> str: + """ + Validate if the provided string is a valid Git reference. + + Git reference naming rules: + - Must start with an alphanumeric character + - Can contain alphanumeric characters, underscore, hyphen, forward slash, and dot + - Cannot contain consecutive dots (..) + Args: + ref (str) -- The Git reference to validate. + + Returns: + str -- The validated Git reference. + + Raises: + ValueError -- If the reference contains invalid characters. + + Meta-Testing: + + Testcase 1: Valid reference. + + >>> _validate_git_ref('main') + 'main' + + Testcase 2: Valid reference with special characters. + + >>> _validate_git_ref('feature/new-feature') + 'feature/new-feature' + + Testcase 3: Invalid reference with disallowed characters. + + >>> _validate_git_ref('invalid$ref') #doctest: +IGNORE_EXCEPTION_DETAIL +ELLIPSIS + Traceback (most recent call last): + ... + ValueError: Invalid Git reference: invalid$ref + + Testcase 4: Empty reference. + + >>> _validate_git_ref('') #doctest: +IGNORE_EXCEPTION_DETAIL +ELLIPSIS + Traceback (most recent call last): + ... + ValueError: Invalid Git reference:... + + Testcase 5: Invalid reference with disallowed dot-dot characters. + + >>> _validate_git_ref('invalid..ref') #doctest: +IGNORE_EXCEPTION_DETAIL +ELLIPSIS + Traceback (most recent call last): + ... + ValueError: Invalid Git reference: invalid..ref + """ + if not re.match(GIT_REF_PATTERN, ref) or ".." in ref: + raise ValueError(f"Invalid Git reference: {ref}") + return ref + + +def slugify_header(s: str) -> str: + """ + Convert header text to a URL-friendly slug. + + This function transforms header text into a URL-friendly slug by removing special characters, + converting to lowercase, and replacing consecutive spaces or dashes with a single dash. + The resulting slug is suitable for use in header anchors and URL paths. + + Arguments: + s (str) -- The header text to be converted into a slug. + + Returns: + str -- A URL-friendly slug derived from the input text. + + Unit-Testing: + + Testcase 1: Basic header with spaces and special characters. + + >>> slugify_header("Hello, World!") + 'hello-world' + + Testcase 2: Header with multiple spaces and mixed case. + + >>> slugify_header(" API Documentation ") + 'api-documentation' + + Testcase 3: Header with consecutive spaces and dashes. + + >>> slugify_header("API -- Documentation") + 'api-documentation' + + Testcase 4: Header with Unicode characters and accents. + + >>> slugify_header("über café 123") + 'über-café-123' + + Testcase 5: Header with special markdown characters. + + >>> slugify_header("[CEP-7] Documentation *Guide*") + 'cep-7-documentation-guide' + """ + # First Normalize Unicode characters to prevent homograph attacks + text: str = unicodedata.normalize('NFKC', s) + # Then, remove special characters and convert to lowercase + text = re.sub(r'[^\w\- ]', "", text).strip().lower() + # Then replace consecutive spaces or dashes with a single dash + return re.sub(r'[-\s]+', "-", text) + + +def sanitize_url(url: str) -> str: + """ + Sanitize and validate a URL according to allowed schemes and domains. + + This function validates that the URL uses an allowed scheme (https) and points + to a trusted domain, then safely encodes its path and query components. + + Args: + url (str) -- The URL to sanitize. + + Returns: + str -- The sanitized URL. + + Raises: + ValueError -- If the URL has an invalid scheme or points to an untrusted domain. + + + Unit-Testing: + + Testcase 0: First set up test fixtures by importing utils. + + >>> import docs.utils as _utils + >>> + + Testcase 1: Basic URL with spaces and special characters. + + >>> url_fxtr = "https://github.com/user/Hello World!" + >>> _utils.sanitize_url(url_fxtr) + 'https://github.com/user/Hello%20World%21' + >>> + + """ + # Validate length + if len(url) > MAX_URL_LENGTH: + raise ValueError(INVALID_LENGTH_ERROR) + parsed_url: ParseResult = urlparse(url) + # Validate scheme + if parsed_url.scheme not in URL_ALLOWED_SCHEMES: + raise ValueError(INVALID_SCHEME_ERROR) + # Validate netloc + if parsed_url.netloc not in URL_ALLOWED_NETLOCS: + raise ValueError(INVALID_DOMAIN_ERROR) + # Normalize netloc to prevent homograph attacks + sanitized_netloc: str = unicodedata.normalize('NFKC', parsed_url.netloc) + # Sanitize path and query - using the safe parameter to preserve URL structure + sanitized_path: str = quote(unicodedata.normalize('NFKC', parsed_url.path), safe="/=") + sanitized_query: str = quote(parsed_url.query, safe="&=") + # Reconstruct the sanitized URL + return urlunparse(( + parsed_url.scheme, + sanitized_netloc, + sanitized_path, + parsed_url.params, + sanitized_query, + parsed_url.fragment, + )) + + +def sanitize_intersphinx_mapping(mapping: dict) -> dict: + """ + Sanitize URLs in an intersphinx mapping dictionary. + + This function applies URL sanitization to each URL in the mapping while + preserving the associated extra values. + + Args: + mapping (dict) -- A dictionary mapping names to tuples of (url, extra_value). + + Returns: + dict -- A dictionary with the same structure but with sanitized URLs. + + Unit-Testing: + + Testcase 1: Basic intersphinx mapping. + + >>> mapping = {'python': ('https://docs.python.org/3', None)} + >>> sanitize_intersphinx_mapping(mapping) + {'python': ('https://docs.python.org/3', None)} + + Testcase 2: Mapping with URL containing special characters. + + >>> mapping = {'project': ('https://github.com/user/project with spaces', None)} + >>> result = sanitize_intersphinx_mapping(mapping) + >>> result['project'][0] + 'https://github.com/user/project%20with%20spaces' + >>> + + """ + return {key: (sanitize_url(url), extra_value) for key, (url, extra_value) in mapping.items()} diff --git a/pythonrepo/__init__.py b/pythonrepo/__init__.py index 40154f0..be019bc 100644 --- a/pythonrepo/__init__.py +++ b/pythonrepo/__init__.py @@ -3,13 +3,13 @@ # Python Repo Template # .................................. -# Copyright (c) 2017-2024, Kendrick Walls +# Copyright (c) 2017-2025, Mr. Walls # .................................. # Licensed under MIT (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # .......................................... -# http://www.github.com/reactive-firewall/python-repo/LICENSE.md +# https://github.com/reactive-firewall/python-repo/LICENSE.md # .......................................... # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -19,25 +19,26 @@ """Python Repo.""" -__module__ = """pythonrepo""" +__module__: str = """pythonrepo""" """This is pythonrepo module Template.""" -__version__ = """1.1.5""" -"""This is version 1.1.5 of pythonrepo Template""" +__version__: str = """2.0.0""" +"""This is version 2.0.0 of pythonrepo Template""" try: import sys import os if str(__module__) in __file__: # pragma: no branch - __parentPath = os.path.join( - os.path.dirname(__file__), '..' + __parentPath: str = os.path.join( + os.path.dirname(__file__), "..", ) - sys.path.insert(0, os.path.abspath(__parentPath)) -except Exception as err: + if str(os.path.abspath(__parentPath)) not in sys.path: # pragma: no branch + sys.path.insert(0, os.path.abspath(__parentPath)) +except ImportError as err: baton = ImportError(err, str("[CWE-758] Module failed completely.")) baton.module = __module__ baton.path = __file__ baton.__cause__ = err - raise baton + raise baton from err diff --git a/pythonrepo/pythonrepo.py b/pythonrepo/pythonrepo.py index e785263..d88d6cf 100644 --- a/pythonrepo/pythonrepo.py +++ b/pythonrepo/pythonrepo.py @@ -18,7 +18,7 @@ # limitations under the License. -__module__ = """pythonrepo.pythonrepo""" +__module__: str = """pythonrepo.pythonrepo""" """This is pythonrepo component Template.""" @@ -32,23 +32,23 @@ baton.path = __file__ baton.__cause__ = err # Throw more relevant Error - raise baton + raise baton from err -from . import __version__ +from . import __version__ as __version__ # noqa -__prog__ = str(__module__) +__prog__: str = str(__module__) """The name of this program is PythonRepo""" -__description__ = str( +__description__: str = str( """Add a Description Here""" ) """Contains the description of the program.""" -__epilog__ = str( +__epilog__: str = str( """Add an epilog here.""" ) """Contains the short epilog of the program CLI help text.""" @@ -57,7 +57,7 @@ # Add your functions here -def NoOp(*args, **kwargs): +def NoOp(*args, **kwargs) -> None: """The meaning of Nothing.""" return None @@ -65,38 +65,42 @@ def NoOp(*args, **kwargs): # More boiler-plate-code -TASK_OPTIONS = { # skipcq: PTC-W0020 - 'noop': NoOp +TASK_OPTIONS: dict = { # skipcq: PTC-W0020 + "noop": NoOp, } """The callable function tasks of this program.""" -def parseArgs(arguments=None): - """Parses the CLI arguments. See argparse.ArgumentParser for more. - param str - arguments - the array of arguments to parse. - Usually sys.argv[1:] - returns argparse.Namespace - the Namespace parsed with +def parseArgs(arguments=None) -> argparse.Namespace: + """Parses the CLI arguments. + + See `argparse.ArgumentParser` for more. + + Parameters: + arguments (list) - the array of arguments to parse. + Usually `sys.argv[1:]`. + + Returns: + result (tuple(argparse.Namespace, list)) - the Namespace parsed with the key-value pairs. """ - parser = argparse.ArgumentParser( + parser: argparse.ArgumentParser = argparse.ArgumentParser( prog=__prog__, description=__description__, - epilog=__epilog__ + epilog=__epilog__, ) parser.add_argument( - 'some_task', choices=TASK_OPTIONS.keys(), - help='the help text for this option.' + "some_task", choices=sorted(TASK_OPTIONS.keys()), + help="the help text for this option." ) parser.add_argument( - '-V', '--version', - action='version', version=str( - "%(prog)s {version}" - ).format(version=str(__version__)) + "-V", "--version", + action="version", version=f"{parser.prog} {__version__}" ) return parser.parse_known_args(arguments) -def useTool(tool, *arguments): +def useTool(tool, *arguments) -> Any: """Handler for launching the functions.""" if (tool is not None) and (tool in TASK_OPTIONS): try: @@ -109,28 +113,28 @@ def useTool(tool, *arguments): return None -def main(*argv): +def main(*argv) -> None: """The Main Event.""" try: try: args, extra = parseArgs(*argv) service_cmd = args.some_task useTool(service_cmd, extra) - except Exception: + except Exception as _cause: w = str("WARNING - An error occurred while") w += str("handling the arguments.") w += str(" Cascading failure.") print(w) - sys.exit(2) - except Exception: + raise SystemExit(2) from _cause + except BaseException as _panic: # nocq e = str("CRITICAL - An error occurred while handling") e += str("the cascading failure.") print(e) - sys.exit(3) + raise SystemExit(3) from _panic sys.exit(0) -if __name__ in '__main__': +if __name__ == "__main__": # deepsource overlooks the readability of "if main" type code here. (See PTC-W0048) if (sys.argv is not None) and (len(sys.argv) >= 1): main(sys.argv[1:]) diff --git a/tests/__init__.py b/tests/__init__.py index 7eb262c..aa88beb 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -2,13 +2,13 @@ # Python Repo Template # .................................. -# Copyright (c) 2017-2024, Kendrick Walls +# Copyright (c) 2017-2025, Mr. Walls # .................................. # Licensed under MIT (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # .......................................... -# http://www.github.com/reactive-firewall/python-repo/LICENSE.md +# https://github.com/reactive-firewall/python-repo/LICENSE.md # .......................................... # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -18,7 +18,7 @@ """Python Repo Testing Module.""" -__module__ = """tests""" +__module__: str = """tests""" """This is pythonrepo testing module Template.""" @@ -27,24 +27,17 @@ import sys import os sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), str('..')))) - except Exception as ImportErr: - print(str('')) - print(str(type(ImportErr))) - print(str(ImportErr)) - print(str((ImportErr.args))) - print(str('')) - ImportErr = None - del ImportErr - raise ImportError(str("Test module failed completely.")) + except (ImportError,OSError,TypeError,ValueError,AttributeError,IndexError) as ImportErr: + raise ImportError("[CWE-758] Test module failed completely.") from ImportErr from tests import context as context # skipcq: PYL-C0414 - if context.__name__ is None: - raise ImportError(str("Test module failed to import even the context framework.")) + if not hasattr(context, '__name__') or not context.__name__: # pragma: no branch + raise ImportError("[CWE-758] Failed to import context") from None from tests import profiling as profiling # skipcq: PYL-C0414 - if profiling.__name__ is None: - raise ImportError(str("Test module failed to import even the profiling framework.")) + if not hasattr(profiling, '__name__') or not profiling.__name__: # pragma: no branch + raise ImportError("[CWE-758] Failed to import the profiling framework.") from None from tests import test_basic - if test_basic.__name__ is None: - raise ImportError(str("Test module failed to import even the basic tests.")) + if not hasattr(test_basic, '__name__') or not test_basic.__name__: # pragma: no branch + raise ImportError("[CWE-758] Failed to import the basic tests.") from None except Exception as badErr: baton = ImportError(badErr, str("[CWE-758] Test module failed completely.")) baton.module = __module__ diff --git a/tests/context.py b/tests/context.py index bf4b164..f6159fd 100644 --- a/tests/context.py +++ b/tests/context.py @@ -17,13 +17,12 @@ # limitations under the License. -__module__ = """tests.context""" +__module__ = """tests""" """This is pythonrepo testing module Template.""" -__module__ = """tests""" +__name__ = f"""{module}.context""" # skipcq: PYL-W0622 -__name__ = """tests.context""" # skipcq: PYL-W0622 __doc__ = """ @@ -66,6 +65,7 @@ except ImportError as err: # pragma: no branch raise ModuleNotFoundError("[CWE-440] OS Failed to import.") from err + try: if 'tests.profiling' not in sys.modules: import tests.profiling as profiling @@ -74,6 +74,7 @@ except ImportError as err: # pragma: no branch raise ModuleNotFoundError("[CWE-440] profiling Failed to import.") from err + try: import pythonrepo as pythonrepo # skipcq: PYL-C0414 if pythonrepo.__name__ is None: From 16ba9197d4362d06361f1693a3a0c1d79e86d5a2 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Tue, 12 Aug 2025 18:35:58 -0700 Subject: [PATCH 5/7] [PATCH] fixup for PR #89 regressions and linting --- docs/conf.py | 23 +- pythonrepo/__init__.py | 2 +- pythonrepo/pythonrepo.py | 4 +- test-requirements.txt | 37 +-- tests/__init__.py | 2 +- tests/check_spelling | 2 +- tests/context.py | 482 ++++++++++++++++++++++++++++++++++++++- tests/test_usage.py | 123 ++-------- 8 files changed, 533 insertions(+), 142 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 362f1b4..52bf8a9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -29,9 +29,9 @@ Example: >>> import conf >>> conf.project - 'python-repo' + 'pythonrepo' >>> conf.version - '1.1' + '2.0' """ import sys @@ -41,7 +41,7 @@ from urllib.parse import quote # If you have custom documentation extensions, add them here -sys.path.insert(0, os.path.abspath("..")) +import docs.utils from docs.utils import ( _validate_git_ref, # noqa slugify_header, # noqa @@ -64,6 +64,7 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath("..")) # sys.path.insert(1, os.path.abspath("tests")) # uncomment to add tests docs sys.path.insert(1, os.path.abspath("pythonrepo")) @@ -151,9 +152,9 @@ # built documents. # # The short X.Y version. -version = "v2.0" +version = "2.0" # The full version, including alpha/beta/rc tags. -release = "v2.0.0b0" +release = f"v${version}.0b0" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -258,7 +259,7 @@ # html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -html_short_title = "Project Docs" +html_short_title = "Project Docs" # The name of an image file (relative to this directory) to place at the top # of the sidebar. @@ -472,7 +473,7 @@ suffix: str = "/issues/%s" -extlinks: dict[str,tuple] = { +extlinks: dict[str, tuple] = { "issue": ( f"{linkcode_url_prefix}/{suffix}", "issue #%s" @@ -481,7 +482,7 @@ # try to link with official python3 documentation. # see https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html for more -intersphinx_mapping = sanitize_intersphinx_mapping( +intersphinx_mapping: dict[str, tuple] = sanitize_intersphinx_mapping( { "python": ("https://docs.python.org/3", (None, "python-inv.txt")), "PEP": ("https://peps.python.org", (None, "pep-inv.txt")), @@ -489,9 +490,9 @@ ) -def linkcode_resolve(domain: Any, info: Any) -> str: +def linkcode_resolve(domain: any, info: any) -> str: """ - Resolves selectively linking to GitHub source-code for the multicast module. + Resolves selectively linking to GitHub source-code for the project module. See https://www.sphinx-doc.org/en/master/usage/extensions/linkcode.html for more details. @@ -506,7 +507,7 @@ def linkcode_resolve(domain: Any, info: Any) -> str: >>> _conf.linkcode_resolve is not None True - >>> test_text = "multicast.env" # this is resolved + >>> test_text = "pythonrepo.env" # this is resolved >>> bad_input = False # this is invalid >>> res_text = _conf.linkcode_resolve("py", info={"module": test_text}) >>> res_text is not None diff --git a/pythonrepo/__init__.py b/pythonrepo/__init__.py index be019bc..9b8b708 100644 --- a/pythonrepo/__init__.py +++ b/pythonrepo/__init__.py @@ -35,7 +35,7 @@ os.path.dirname(__file__), "..", ) if str(os.path.abspath(__parentPath)) not in sys.path: # pragma: no branch - sys.path.insert(0, os.path.abspath(__parentPath)) + sys.path.insert(0, os.path.abspath(__parentPath)) # pragma: no cover except ImportError as err: baton = ImportError(err, str("[CWE-758] Module failed completely.")) baton.module = __module__ diff --git a/pythonrepo/pythonrepo.py b/pythonrepo/pythonrepo.py index d88d6cf..abcd8d3 100644 --- a/pythonrepo/pythonrepo.py +++ b/pythonrepo/pythonrepo.py @@ -100,7 +100,7 @@ def parseArgs(arguments=None) -> argparse.Namespace: return parser.parse_known_args(arguments) -def useTool(tool, *arguments) -> Any: +def useTool(tool, *arguments) -> any: """Handler for launching the functions.""" if (tool is not None) and (tool in TASK_OPTIONS): try: @@ -128,7 +128,7 @@ def main(*argv) -> None: raise SystemExit(2) from _cause except BaseException as _panic: # nocq e = str("CRITICAL - An error occurred while handling") - e += str("the cascading failure.") + e += str(" the cascading failure.") print(e) raise SystemExit(3) from _panic sys.exit(0) diff --git a/test-requirements.txt b/test-requirements.txt index 6ede96c..2f0a442 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -13,25 +13,32 @@ tox>=3.0.0 setuptools>=75.0 # virtualenv - MIT license virtualenv>=20.26.6 -# pip - MIT license -pip>=24.3.1 # build - MIT license build>=1.1.1, !=1.2.2.post1 -# python-repo - MIT -#-e git+https://github.com/reactive-firewall/python-repo.git#egg=python-repo # wheel - MIT license -wheel>=0.44 +wheel>=0.45 +# pip - MIT license +pip>=25.1.1 # # TESTING ONLY - Do NOT report issues with these optionals on python-repo # -flake8>=5.0 -pyflakes>=2.5.0 -pep8>=1.0 -pytest>=7 -pytest-checkdocs>=2.4 -pytest-cov>=4.0.0 +# flake8 - MIT license +flake8>=7.2.0 +# flake8-comprehensions - MIT License +flake8-comprehensions>=3.16.0 +# pyflakes - MIT license +# pyflakes>=3.3.2 +# pep8 - MIT license +pep8>=1.7 +# pytest - MIT license +pytest>=7.0, !=8.1.0 +# pytest-doctestplus - BSD license +pytest-doctestplus>=1.4.0 +# pytest-cov - MIT license +pytest-cov>=6.1.1 +# pytest-enabler - MIT license pytest-enabler>=1.0.1 -pytest-flake8>=1.0 -coverage >= 6.3 -pytest-cov >= 4.0.0; -pytest-enabler >= 1.0.1 +# pytest-flake8 - BSD license - removed from tests in v2.0.9a3 +# pytest-flake8>=1.0.7 +# coverage - Apache 2.0 license +coverage>=7.2 diff --git a/tests/__init__.py b/tests/__init__.py index aa88beb..7324b0e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -27,7 +27,7 @@ import sys import os sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), str('..')))) - except (ImportError,OSError,TypeError,ValueError,AttributeError,IndexError) as ImportErr: + except (ImportError, OSError, TypeError, ValueError, AttributeError, IndexError) as ImportErr: raise ImportError("[CWE-758] Test module failed completely.") from ImportErr from tests import context as context # skipcq: PYL-C0414 if not hasattr(context, '__name__') or not context.__name__: # pragma: no branch diff --git a/tests/check_spelling b/tests/check_spelling index 15b0e77..a47265d 100755 --- a/tests/check_spelling +++ b/tests/check_spelling @@ -97,7 +97,7 @@ umask 137 # force utf-8 for spelling export LC_CTYPE="${LC_CTYPE:-en_US.UTF-8}" -LOCK_FILE="${TMPDIR:-/tmp}/org.pak.multicast.spell-check-shell" +LOCK_FILE="${TMPDIR:-/tmp}/org.pak.pythonrepo.spell-check-shell" EXIT_CODE=1 # Function to check if a command exists. diff --git a/tests/context.py b/tests/context.py index f6159fd..eeb87ee 100644 --- a/tests/context.py +++ b/tests/context.py @@ -21,7 +21,7 @@ """This is pythonrepo testing module Template.""" -__name__ = f"""{module}.context""" # skipcq: PYL-W0622 +__name__ = f"""{__module__}.context""" # skipcq: PYL-W0622 __doc__ = """ @@ -74,6 +74,10 @@ except ImportError as err: # pragma: no branch raise ModuleNotFoundError("[CWE-440] profiling Failed to import.") from err +try: + import subprocess +except ImportError as _cause: # pragma: no branch + raise ModuleNotFoundError("[CWE-440] subprocess Failed to import.") from _cause try: import pythonrepo as pythonrepo # skipcq: PYL-C0414 @@ -85,3 +89,479 @@ baton.path = __file__ baton.__cause__ = badErr raise baton + + +def getCoverageCommand() -> str: + """ + Function for backend coverage command. + Rather than just return the sys.executable which will usually be a python implementation, + this function will search for a coverage tool to allow coverage testing to continue beyond + the process fork of typical cli testing. + + Meta Testing: + + First set up test fixtures by importing test context. + + >>> import tests.context as _context + >>> + + Testcase 1: function should have a output. + + >>> _context.getCoverageCommand() is None + False + >>> + + + """ + thecov = "exit 1 ; #" + try: + thecov = checkPythonCommand(["command", "-v", "coverage"]) + _unsafe_cov = checkPythonCommand(["which", "coverage"]) + if (str("/coverage") in str(thecov) or str("/coverage") in str(_unsafe_cov)): + thecov = str("coverage") # skipcq: TCV-002 + elif str("/coverage3") in str(checkPythonCommand(["command", "-v", "coverage3"])): + thecov = str("coverage3") # skipcq: TCV-002 + else: # pragma: no branch + thecov = "exit 1 ; #" # skipcq: TCV-002 + except Exception: # pragma: no branch + thecov = "exit 1 ; #" # handled error by suppressing it and indicating caller should abort. + return str(thecov) + + +def __check_cov_before_py() -> str: + """ + Utility Function to check for coverage availability before just using plain python. + Rather than just return the sys.executable which will usually be a python implementation, + this function will search for a coverage tool before falling back on just plain python. + + Meta Testing: + + First set up test fixtures by importing test context. + + >>> import tests.context as _context + >>> + + Testcase 1: function should have a output. + + >>> _context.__check_cov_before_py() is not None + True + >>> + + Testcase 2: function should have a string output of python or coverage. + + >>> _test_fixture = _context.__check_cov_before_py() + >>> isinstance(_test_fixture, type(str(""))) + True + >>> (str("python") in _test_fixture) or (str("coverage") in _test_fixture) + True + >>> + + + """ + thepython = str(sys.executable) + thecov = getCoverageCommand() + if (str("coverage") in str(thecov)) and (sys.version_info >= (3, 7)): + thepython = str(f"{str(thecov)} run -p") # skipcq: TCV-002 + else: # pragma: no branch + try: + # pylint: disable=cyclic-import - skipcq: PYL-R0401, PYL-C0414 + import coverage as coverage # skipcq: PYL-C0414 + if coverage.__name__ is not None: + thepython = str("{} -m coverage run -p").format(str(sys.executable)) + except Exception: + thepython = str(sys.executable) # handled error by falling back on faile-safe value. + return thepython + + +def getPythonCommand() -> str: + """ + Function for backend python command. + Rather than just return the sys.executable which will usually be a python implementation, + this function will search for a coverage tool with getCoverageCommand() first. + + Meta Testing: + + First set up test fixtures by importing test context. + + >>> import tests.context as _context + >>> + + Testcase 1: function should have a output. + + >>> _context.getPythonCommand() is not None + True + >>> + + """ + thepython = "python" + try: + thepython = __check_cov_before_py() + except Exception: # pragma: no branch + thepython = "exit 1 ; #" + try: + thepython = str(sys.executable) + except Exception: + thepython = "exit 1 ; #" # handled error by suppressing it and indicating exit. + return str(thepython) + + +def taint_command_args(args: (list, tuple)) -> list: + """Validate and sanitize command arguments for security. + + This function validates the command (first argument) against a whitelist + and sanitizes all arguments to prevent command injection attacks. + + Args: + args (list): Command arguments to validate + + Returns: + list: Sanitized command arguments + + Raises: + CommandExecutionError: If validation fails + + Meta Testing: + + >>> import tests.context as _context + >>> import sys as _sys + >>> + + Testcase 1: Function should validate and return unmodified Python command. + + >>> test_fixture = ['python', '-m', 'pytest'] + >>> _context.taint_command_args(test_fixture) + ['python', '-m', 'pytest'] + >>> + + Testcase 2: Function should handle sys.executable path. + + >>> test_fixture = [str(_sys.executable), '-m', 'coverage', 'run'] + >>> result = _context.taint_command_args(test_fixture) #doctest: +ELLIPSIS + >>> str('python') in str(result[0]) or str('coverage') in str(result[0]) + True + >>> result[1:] == ['-m', 'coverage', 'run'] + True + >>> + + Testcase 3: Function should reject disallowed commands. + + >>> test_fixture = ['rm', '-rf', '/'] + >>> _context.taint_command_args(test_fixture) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ValueError: Command 'rm' is not allowed... + >>> + + Testcase 4: Function should validate input types. + + >>> test_fixture = None + >>> _context.taint_command_args(test_fixture) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ValueError: Invalid command arguments + >>> + >>> test_fixture = "python -m pytest" # String instead of list + >>> _context.taint_command_args(test_fixture) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ValueError: Invalid command arguments + >>> + + Testcase 5: Function should handle coverage command variations. + + >>> test_fixture = [str(_sys.executable), 'coverage', 'run', '--source=pythonrepo'] + >>> _context.taint_command_args(test_fixture) #doctest: +ELLIPSIS + [...'coverage', 'run', '--source=pythonrepo'] + >>> + >>> test_fixture = ['coverage', 'run', '--source=pythonrepo'] + >>> _context.taint_command_args(test_fixture) #doctest: +ELLIPSIS + ['exit 1 ; #', 'run',...'run', '--source=pythonrepo'] + >>> + >>> test_fixture = ['coverage3', 'run', '--source=.'] + >>> _context.taint_command_args(test_fixture) #doctest: +ELLIPSIS + ['exit 1 ; #', 'run',...'--source=.'] + >>> + + Testcase 6: Function should handle case-insensitive command validation. + + >>> test_fixture = ['Python3', '-m', 'pytest'] + >>> _context.taint_command_args(test_fixture) + ['Python3', '-m', 'pytest'] + >>> + >>> test_fixture = ['COVERAGE', 'run'] + >>> _context.taint_command_args(test_fixture) #doctest: +ELLIPSIS + [...'COVERAGE', 'run'...] + >>> + """ + if not args or not isinstance(args, (list, tuple)): + raise ValueError("Invalid command arguments") + # Validate the command (first argument) + allowed_commands = { + "python", "python3", "coverage", "coverage3", + sys.executable, # Allow the current Python interpreter + } + command = str(args[0]).lower() + # Extract base command name for exact matching + # Handle both path separators (/ for Unix, \ for Windows) + command_base = command.split("/")[-1].split("\\")[-1] + # Check if command is allowed (exact match on base name or full path match with sys.executable) + if command_base not in allowed_commands and command != str(sys.executable).lower(): + raise ValueError( + f"Command '{command}' is not allowed. Allowed commands: {allowed_commands}", + ) + # Sanitize all arguments to prevent injection + tainted_args = [str(arg) for arg in args] + # Special handling for coverage commands + if "coverage" in command: + tainted_args = checkCovCommand(*tainted_args) + # Sanitize all arguments to prevent injection + return tainted_args + + +def validateCommandArgs(args: list) -> None: + """ + Validates command arguments to ensure they do not contain null characters. + + Args: + args (list): A list of command arguments to be validated. + + Raises: + ValueError: If any argument contains a null character. + """ + if (args is None) or (args == [None]) or (len(args) <= 0): # pragma: no branch + # skipcq: TCV-002 + raise ValueError("[CWE-1286] args must be an array of positional arguments") from None + for arg in args: + if isinstance(arg, str) and "\x00" in arg: + raise ValueError("[CWE-20] Null characters are not allowed in command arguments.") + + +def checkStrOrByte(theInput): + """ + Converts the input to a string if possible, otherwise returns it as bytes. + + This utility function is designed to handle both string and byte inputs, + ensuring consistent output type. It attempts to decode byte inputs to UTF-8 + strings, falling back to bytes if decoding fails. + + Args: + theInput: The input to be checked and potentially converted. + Can be None, str, bytes, or any other type. + + Returns: + str: If the input is already a string or can be decoded to UTF-8. + bytes: If the input is bytes and cannot be decoded to UTF-8. + None: If the input is None. + + Meta Testing: + + First set up test fixtures by importing test context. + + >>> import tests.context as _context + >>> + + Testcase 1: Input is a string. + + >>> _context.checkStrOrByte("Hello") + 'Hello' + >>> + + Testcase 2: Input is UTF-8 decodable bytes. + + >>> _context.checkStrOrByte(b"Hello") + 'Hello' + >>> + + Testcase 3: Input is bytes that are not UTF-8 decodable. + + >>> _context.checkStrOrByte(b'\\xff\\xfe') + b'\xff\xfe' + >>> + + Testcase 4: Input is None. + + >>> _context.checkStrOrByte(None) is None + True + >>> + + Testcase 5: Input is an empty string. + + >>> _context.checkStrOrByte("") + '' + >>> + + Testcase 6: Input is empty bytes. + + >>> _context.checkStrOrByte(b"") + '' + >>> + + + """ + theOutput = None + if theInput is not None: # pragma: no branch + theOutput = theInput + try: + if isinstance(theInput, bytes): + theOutput = theInput.decode("UTF-8") + except UnicodeDecodeError: # pragma: no branch + theOutput = bytes(theInput) + return theOutput + + +def checkCovCommand(*args): # skipcq: PYL-W0102 - [] != [None] + """ + Modifies the input command arguments to include coverage-related options when applicable. + + This utility function checks if the first argument contains "coverage" and, if so, + modifies the argument list to include additional coverage run options. It's primarily + used internally by other functions in the testing framework. + Not intended to be run directly. + + Args: + *args (list): A list of command arguments; should not be pass None. + + Returns: + list: The modified list of arguments with 'coverage run' options added as applicable. + + Meta Testing: + + First set up test fixtures by importing test context. + + >>> import tests.context as _context + >>> + + Testcase 1: Function should return unmodified arguments if 'coverage' is missing. + + >>> _context.checkCovCommand("python", "script.py") + ['python', 'script.py'] + + Testcase 2: Function should modify arguments when 'coverage' is the first argument. + A.) Missing 'run' + + >>> _context.checkCovCommand("coverage", "script.py") #doctest: +ELLIPSIS + ['...', 'run', '-p', '--context=Integration', '--source=pythonrepo', 'script.py'] + + Testcase 3: Function should modify arguments when 'coverage run' is in the first argument. + A.) NOT missing 'run' + + >>> _context.checkCovCommand("coverage run", "script.py") #doctest: +ELLIPSIS + ['...', 'run', '-p', '--context=Integration', '--source=pythonrepo', 'script.py'] + + Testcase 4: Function should handle coverage command with full path. + + >>> _context.checkCovCommand("/usr/bin/coverage", "test.py") #doctest: +ELLIPSIS + ['...', 'run', '-p', '--context=Integration', '--source=pythonrepo', 'test.py'] + + Testcase 5: Function should handle coverage invoked via sys.executable. + + >>> import sys as _sys + >>> test_fixture = [str("{} -m coverage run").format(_sys.executable), "test.py"] + >>> _context.checkCovCommand(*test_fixture) #doctest: +ELLIPSIS + [..., '-m', 'coverage', 'run', '-p', '...', '--source=pythonrepo', 'test.py'] + + + """ + if sys.__name__ is None: # pragma: no branch + raise ImportError("[CWE-758] Failed to import system.") from None + if not args or args[0] is None: + # skipcq: TCV-002 + raise ValueError("[CWE-1286] args must be an array of positional arguments") from None + else: + args = [*args] # convert to an array + if str("coverage") in args[0]: + i = 1 + if str(f"{str(sys.executable)} -m coverage") in str(args[0]): # pragma: no branch + args[0] = str(sys.executable) + args.insert(1, str("-m")) + args.insert(2, str("coverage")) + i += 2 + else: # pragma: no branch + args[0] = str(getCoverageCommand()) + extra_args = ["run", "-p", "--context=Integration", "--source=pythonrepo"] + # PEP-279 - see https://www.python.org/dev/peps/pep-0279/ + for k, ktem in enumerate(extra_args): + offset = i + k + args.insert(offset, ktem) + return [*args] + + +def checkPythonCommand(args, stderr=None): + """ + Execute a Python command and return its output. + + This function is a wrapper around subprocess.check_output with additional + error handling and output processing. It's designed to execute Python + commands or coverage commands, making it useful for running tests and + collecting coverage data. + + Args: + args (list): A list of command arguments to be executed. + stderr (Optional[int]): File descriptor for stderr redirection. + Defaults to None. + + Returns: + str: The command output as a string, with any byte output decoded to UTF-8. + + Raises: + subprocess.CalledProcessError: If the command returns a non-zero exit status. + + Meta Testing: + + First set up test fixtures by importing test context. + + >>> import sys as _sys + >>> import tests.context as _context + >>> + + Testcase 1: Function should have an output when provided valid arguments. + + >>> test_fixture_1 = [str(_sys.executable), '-c', 'print("Hello, World!")'] + >>> _context.checkPythonCommand(test_fixture_1) + 'Hello, World!\\n' + + Testcase 2: Function should capture stderr when specified. + + >>> import subprocess as _subprocess + >>> test_args_2 = [ + ... str(_sys.executable), '-c', 'import sys; print("Error", file=sys.stderr)' + ... ] + >>> + >>> _context.checkPythonCommand(test_args_2, stderr=_subprocess.STDOUT) + 'Error\\n' + + Testcase 3: Function should handle exceptions and return output. + + >>> test_fixture_e = [str(_sys.executable), '-c', 'raise ValueError("Test error")'] + >>> _context.checkPythonCommand( + ... test_fixture_e, stderr=_subprocess.STDOUT + ... ) #doctest: +ELLIPSIS + 'Traceback (most recent call last):\\n...ValueError...' + + Testcase 4: Function should return the output as a string. + + >>> test_fixture_s = [str(_sys.executable), '-c', 'print(b"Bytes output")'] + >>> isinstance(_context.checkPythonCommand( + ... test_fixture_s, stderr=_subprocess.STDOUT + ... ), str) + True + + + """ + theOutput = None + try: + if (args is None) or (args == [None]) or (len(args) <= 0): # pragma: no branch + theOutput = None # None is safer than subprocess.check_output(["exit 1 ; #"]) + else: + validateCommandArgs(args) + if str("coverage") in args[0]: + args = checkCovCommand(*args) + # Validate and sanitize command arguments + safe_args = taint_command_args(args) + theOutput = subprocess.check_output(safe_args, stderr=stderr) + except Exception as _cause: # pragma: no branch + theOutput = None + try: + if _cause.output is not None: + theOutput = _cause.output + except Exception: + theOutput = None # suppress all errors + theOutput = checkStrOrByte(theOutput) + return theOutput diff --git a/tests/test_usage.py b/tests/test_usage.py index 9af1a7e..2d7a225 100644 --- a/tests/test_usage.py +++ b/tests/test_usage.py @@ -23,76 +23,10 @@ import sys import tests.profiling as profiling - -def getPythonCommand(): - """Function for backend python command with cross-python coverage support.""" - thepython = "exit 1 ; #" - try: - thepython = checkPythonCommand(["which", "coverage"]) - if (str("/coverage") in str(thepython)) and (sys.version_info >= (3, 3)): - thepython = str("coverage run -p") - elif (str("/coverage") in str(thepython)) and (sys.version_info <= (3, 2)): - try: - import coverage - if coverage.__name__ is not None: - thepython = str("{} -m coverage run -p").format(str(sys.executable)) - else: - thepython = str(sys.executable) - except Exception: - thepython = str(sys.executable) - else: - thepython = str(sys.executable) - except Exception: - thepython = "exit 1 ; #" - try: - thepython = str(sys.executable) - except Exception: - thepython = "exit 1 ; #" - return str(thepython) - - -def buildPythonCommand(args=None): - """Function for building backend subprocess command line.""" - theArgs = args - # you need to change this to the name of your project - __project__ = str("pythonrepo") - try: - if args is None or (args == [None]): - theArgs = ["exit 1 ; #"] - else: - theArgs = args - if str("coverage ") in str(theArgs[0]): - if str("{} -m coverage ").format(str(sys.executable)) in str(theArgs[0]): - theArgs[0] = str(sys.executable) - theArgs.insert(1, str("-m")) - theArgs.insert(2, str("coverage")) - theArgs.insert(3, str("run")) - theArgs.insert(4, str("-p")) - theArgs.insert(4, str("--source={}").format(__project__)) - else: - theArgs[0] = str("coverage") - theArgs.insert(1, str("run")) - theArgs.insert(2, str("-p")) - theArgs.insert(2, str("--source={}").format(__project__)) - except Exception: - theArgs = ["exit 1 ; #"] - return theArgs - - -def checkPythonCommand(args=None, stderr=None): - """Function for backend subprocess check_output command like testing with coverage support.""" - theOutput = None - try: - taintArgs = buildPythonCommand(args) - theOutput = subprocess.check_output(taintArgs, stderr=stderr) - except Exception: - theOutput = None - try: - if isinstance(theOutput, bytes): - theOutput = theOutput.decode('utf8') - except UnicodeDecodeError: - theOutput = bytes(theOutput) - return theOutput +from tests.context import ( + getPythonCommand, + checkPythonCommand, +) @profiling.do_cprofile @@ -106,21 +40,6 @@ def timePythonCommand(args=None, stderr=None): return checkPythonCommand(args, stderr) -def checkPythonErrors(args=None, stderr=None): - """Function like checkPythonCommand, but with error passing.""" - theOutput = None - try: - taintArgs = buildPythonCommand(args) - theOutput = subprocess.check_output(taintArgs, stderr=stderr) - if isinstance(theOutput, bytes): - # default to utf8 your mileage may vary - theOutput = theOutput.decode('utf8') - except Exception as err: - theOutput = None - raise RuntimeError(err) - return theOutput - - def debugBlob(blob=None): """In case you need it.""" try: @@ -143,7 +62,7 @@ def debugBlob(blob=None): def debugIfNoneResult(thepython, theArgs, theOutput): """In case you need it.""" try: - if (str(theOutput) is not None): + if not (str(theOutput) is None or str(theOutput) == str(None)): theResult = True else: theResult = False @@ -208,14 +127,11 @@ def test_template_case(self): theOutputtext = str(repr(bytes(theOutputtext))) # ADD REAL VERSION TEST HERE theResult = debugIfNoneResult(thepython, args, theOutputtext) + self.assertTrue(theResult) # or simply: self.assertIsNotNone(theOutputtext) except Exception as err: - print(str("")) - print(str(type(err))) - print(str(err)) - print(str((err.args))) - print(str("")) + self.fail(err) err = None del err # skipcq: PTC-W0043 theResult = False @@ -249,11 +165,7 @@ def test_profile_template_case(self): # or simply: self.assertIsNotNone(theOutputtext) except Exception as err: - print(str("")) - print(str(type(err))) - print(str(err)) - print(str((err.args))) - print(str("")) + self.fail(err) err = None del err # skipcq: PTC-W0043 theResult = False @@ -261,7 +173,7 @@ def test_profile_template_case(self): @unittest.expectedFailure def test_fail_template_case(self): - """Test case template for profiling.""" + """Test case template for profiling with an expected failure.""" theResult = False thepython = getPythonCommand() if (thepython is not None): @@ -284,21 +196,16 @@ def test_fail_template_case(self): theOutputtext = theOutputtext.decode('utf8') except UnicodeDecodeError: theOutputtext = str(repr(bytes(theOutputtext))) - theResult = debugIfNoneResult(thepython, args, theOutputtext) + theResult = not debugIfNoneResult(thepython, args, theOutputtext) # or simply: - self.assertIsNotNone(theOutputtext) + self.assertIsNone(theOutputtext) except Exception as err: - print(str("")) - print(str(type(err))) - print(str(err)) - print(str((err.args))) - print(str("")) + self.fail(err) err = None del err # skipcq: PTC-W0043 theResult = False self.assertTrue(theResult) - @unittest.expectedFailure def test_bad_template_case(self): """Test case template for profiling.""" theResult = False @@ -327,11 +234,7 @@ def test_bad_template_case(self): # or simply: self.assertIsNotNone(theOutputtext) except Exception as err: - print(str("")) - print(str(type(err))) - print(str(err)) - print(str((err.args))) - print(str("")) + self.fail(err) err = None del err # skipcq: PTC-W0043 theResult = False From af120579f81ba9a69d2c319be99a7f59c84d1656 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Tue, 12 Aug 2025 18:44:21 -0700 Subject: [PATCH 6/7] [STYLE] Suppressed some minor noisy linter alerts. --- pythonrepo/pythonrepo.py | 2 +- tests/context.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pythonrepo/pythonrepo.py b/pythonrepo/pythonrepo.py index abcd8d3..d8f23b9 100644 --- a/pythonrepo/pythonrepo.py +++ b/pythonrepo/pythonrepo.py @@ -35,7 +35,7 @@ raise baton from err -from . import __version__ as __version__ # noqa +from . import __version__ __prog__: str = str(__module__) diff --git a/tests/context.py b/tests/context.py index eeb87ee..5905537 100644 --- a/tests/context.py +++ b/tests/context.py @@ -461,7 +461,7 @@ def checkCovCommand(*args): # skipcq: PYL-W0102 - [] != [None] """ if sys.__name__ is None: # pragma: no branch raise ImportError("[CWE-758] Failed to import system.") from None - if not args or args[0] is None: + if not args or args[0] is None: # skipcq: PYL-R1720 # skipcq: TCV-002 raise ValueError("[CWE-1286] args must be an array of positional arguments") from None else: From 1d37669de0720ed8f05acb3f70ca283775cc3d80 Mon Sep 17 00:00:00 2001 From: "Mr. Walls" Date: Tue, 12 Aug 2025 19:05:52 -0700 Subject: [PATCH 7/7] [STYLE] Minor improvement for code style --- pythonrepo/pythonrepo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonrepo/pythonrepo.py b/pythonrepo/pythonrepo.py index d8f23b9..b9233b5 100644 --- a/pythonrepo/pythonrepo.py +++ b/pythonrepo/pythonrepo.py @@ -71,7 +71,7 @@ def NoOp(*args, **kwargs) -> None: """The callable function tasks of this program.""" -def parseArgs(arguments=None) -> argparse.Namespace: +def parseArgs(arguments=None) -> tuple[argparse.Namespace, list[str]]: """Parses the CLI arguments. See `argparse.ArgumentParser` for more.