From db451706df389ad8336465dbdb0686eb37962ca7 Mon Sep 17 00:00:00 2001 From: "p.sacharias" Date: Mon, 10 Mar 2025 16:24:02 +0000 Subject: [PATCH 01/19] Initial commit --- .devcontainer/.dfetch_data.yaml | 10 + .devcontainer/Dockerfile | 26 ++ .devcontainer/LICENSE | 21 ++ .devcontainer/devcontainer.json | 38 +++ .gitignore | 13 + .pre-commit-config.yaml | 18 ++ LICENSE | 42 ++-- README.md | 4 +- dfetch.yaml | 13 + dfetch_hub/__init__.py | 1 + dfetch_hub/example_gui/gui.py | 293 ++++++++++++++++++++++ dfetch_hub/project/cli.py | 47 ++++ dfetch_hub/project/export.py | 33 +++ dfetch_hub/project/input_parser.py | 24 ++ dfetch_hub/project/project_finder.py | 174 +++++++++++++ dfetch_hub/project/project_parser.py | 47 ++++ dfetch_hub/project/project_sources.py | 108 ++++++++ dfetch_hub/project/remote_datasource.py | 141 +++++++++++ doc/index.md | 35 +++ doc/program_components.puml | 31 +++ projects.yaml | 315 ++++++++++++++++++++++++ pyproject.toml | 45 ++++ test/test_cli.py | 33 +++ test/test_common.py | 20 ++ test/test_dfetch_export.py | 51 ++++ test/test_input_parser.py | 39 +++ test/test_project_finder.py | 49 ++++ test/test_project_parser.py | 120 +++++++++ test/test_project_sources.py | 73 ++++++ test/testdata/dfetch00.yaml | 16 ++ test/testdata/dfetch01.yaml | 23 ++ test/testdata/dfetch_export.yaml | 15 ++ test/testdata/sources00.yaml | 3 + test/testdata/versions.yaml | 5 + test/testdata/versions01.yaml | 41 +++ 35 files changed, 1944 insertions(+), 23 deletions(-) create mode 100644 .devcontainer/.dfetch_data.yaml create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/LICENSE create mode 100644 .devcontainer/devcontainer.json create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 dfetch.yaml create mode 100644 dfetch_hub/__init__.py create mode 100644 dfetch_hub/example_gui/gui.py create mode 100644 dfetch_hub/project/cli.py create mode 100644 dfetch_hub/project/export.py create mode 100644 dfetch_hub/project/input_parser.py create mode 100644 dfetch_hub/project/project_finder.py create mode 100644 dfetch_hub/project/project_parser.py create mode 100644 dfetch_hub/project/project_sources.py create mode 100644 dfetch_hub/project/remote_datasource.py create mode 100644 doc/index.md create mode 100644 doc/program_components.puml create mode 100644 projects.yaml create mode 100644 pyproject.toml create mode 100644 test/test_cli.py create mode 100644 test/test_common.py create mode 100644 test/test_dfetch_export.py create mode 100644 test/test_input_parser.py create mode 100644 test/test_project_finder.py create mode 100644 test/test_project_parser.py create mode 100644 test/test_project_sources.py create mode 100644 test/testdata/dfetch00.yaml create mode 100644 test/testdata/dfetch01.yaml create mode 100644 test/testdata/dfetch_export.yaml create mode 100644 test/testdata/sources00.yaml create mode 100644 test/testdata/versions.yaml create mode 100644 test/testdata/versions01.yaml diff --git a/.devcontainer/.dfetch_data.yaml b/.devcontainer/.dfetch_data.yaml new file mode 100644 index 0000000..9ef1aec --- /dev/null +++ b/.devcontainer/.dfetch_data.yaml @@ -0,0 +1,10 @@ +# This is a generated file by dfetch. Don't edit this, but edit the manifest. +# For more info see https://dfetch.rtfd.io/en/latest/getting_started.html +dfetch: + branch: main + hash: 9bb1320bd8367d6a33ecc4150e319108 + last_fetch: 10/03/2025, 16:16:46 + patch: '' + remote_url: https://github.com/dfetch-org/dfetch + revision: c037c000e200704f3db1ac6af1aceeb79cb1954c + tag: '' diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..47c3efb --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,26 @@ +FROM mcr.microsoft.com/devcontainers/python:1-3.12-bullseye + +# Install dependencies +# pv is required for asciicasts +RUN apt-get update && apt-get install --no-install-recommends -y \ + pv=1.6.6-1+b1 \ + subversion=1.14.1-3+deb11u1 && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /workspaces/dfetch + +# Add a non-root user (dev) +RUN useradd -m dev && chown -R dev:dev /workspaces/dfetch + +USER dev + +ENV PATH="/home/dev/.local/bin:${PATH}" + +COPY --chown=dev:dev . . + +RUN pip install --no-cache-dir --root-user-action=ignore --upgrade pip==24.3.1 \ + && pip install --no-cache-dir --root-user-action=ignore -e .[development,docs,test,casts] \ + && pre-commit install --install-hooks + +# Set bash as the default shell +SHELL ["/bin/bash", "-ec"] diff --git a/.devcontainer/LICENSE b/.devcontainer/LICENSE new file mode 100644 index 0000000..f81e253 --- /dev/null +++ b/.devcontainer/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 dfetch-org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..3cc8f4d --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,38 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/python +{ + "name": "Python 3", + "build": { + "dockerfile": "Dockerfile", + "context": ".." + }, + "postCreateCommand": "pip install -e .[development,docs,casts]", + "customizations": { + "vscode": { + "extensions": [ + "lextudio.restructuredtext", + "alexkrechik.cucumberautocomplete", + "ms-python.python", + "ms-python.isort", + "ms-python.black-formatter", + "ms-python.debugpy", + "mhutchie.git-graph", + "tamasfe.even-better-toml", + "trond-snekvik.simple-rst", + "jebbs.plantuml", + "jimasp.behave-vsc" + ], + "settings": { + "terminal.integrated.profiles.linux": { + "bash": { + "path": "bash", + "icon": "terminal-bash" + } + }, + "terminal.integrated.defaultProfile.linux": "bash" + } + } + }, + "workspaceFolder": "/workspaces/dfetch", + "remoteUser": "dev" +} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7085d73 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +__pycache__ +.coverage +.mypy_cache +.pytest_cache +.vscode +build +coverage.xml +dfetch_hub.egg-info +dist +doc/_build +doc/landing-page/_build +example/Tests/ +venv* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..d345313 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +repos: +- repo: local + hooks: + - id: isort + name: import sorting + entry: isort + language: python + - id: black + name: black formatter + entry: black + language: system + types: [file, python] + - id: pylint + name: lint python files + entry: pylint + language: system + files: ^src/ + types: [file, python] \ No newline at end of file diff --git a/LICENSE b/LICENSE index 435ccda..7db949e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,21 @@ -MIT License - -Copyright (c) 2025 dfetch-org - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +MIT License + +Copyright (c) 2025 dfetch-org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 16b3be0..9be2b37 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ -# dfetch-hub -Explorer for finding new projects to dfetch +# dfetch-hub +Explorer for finding new projects to dfetch diff --git a/dfetch.yaml b/dfetch.yaml new file mode 100644 index 0000000..0d88e2c --- /dev/null +++ b/dfetch.yaml @@ -0,0 +1,13 @@ +manifest: + version: 0.0 # DFetch Module syntax version + + remotes: # declare common sources in one place + - name: github + url-base: https://github.com/ + + projects: + - name: dfetch-devcontainer + dst: .devcontainer + repo-path: dfetch-org/dfetch + src: .devcontainer + branch: main diff --git a/dfetch_hub/__init__.py b/dfetch_hub/__init__.py new file mode 100644 index 0000000..88dd0d9 --- /dev/null +++ b/dfetch_hub/__init__.py @@ -0,0 +1 @@ +# file required to run gui from project path \ No newline at end of file diff --git a/dfetch_hub/example_gui/gui.py b/dfetch_hub/example_gui/gui.py new file mode 100644 index 0000000..5ada823 --- /dev/null +++ b/dfetch_hub/example_gui/gui.py @@ -0,0 +1,293 @@ +"""sample of a possible nicegui based gui""" +from nicegui import events, ui +from thefuzz import fuzz + +from dfetch_hub.project.project_finder import GitProjectFinder +from dfetch_hub.project.project_sources import RemoteSource, SourceList + + +def main(): + """main gui runner""" + ui.context.sl = SourceList() + ui.context.pf = [] + header() + with ui.column().classes("w-1/3 mx-auto mt-10"): + show_sources() + url_input() + sources_input() + ui.run(title="dfetch project viewer", reconnect_timeout=30) + + +def header(): + """main gui header""" + with ui.header().classes("bg-black text-white p-4"): + with ui.row().classes( + "container mx-auto flex justify-between items-center space-x-4" + ): + ui.link("Sources", target="/sources").classes( + "text-white hover:text-gray-400 no-underline" + ) + ui.link("Projects", target="/projects").classes( + "text-white hover:text-gray-400 no-underline" + ) + ui.link("Filters", target="/filters").classes( + "text-white hover:text-gray-400 no-underline" + ) + + +@ui.page("/sources") +def sources_page(): + """page to enter sources to search""" + header() + with ui.column().classes("w-1/3 mx-auto mt-10"): + show_sources() + url_input() + sources_input() + + +def add_projects_to_page(): + """add list of project finder results to page""" + if not ui.context.pf: + ui.context.pf = [] + ui.context.projects = [] + for source in ui.context.sl.get_remotes(): + if source.url not in [pf.url for pf in ui.context.pf]: + print(f"adding source {source.url}") + if hasattr(source, "exclusions"): + pf = GitProjectFinder(source.url, source.exclusions) + else: + pf = GitProjectFinder(source.url) + ui.context.pf += [pf] + for pf in ui.context.pf: + try: + add_project_finder_to_page(pf) + except ValueError as e: + ui.notification(f"{e}") + + +def add_project_finder_to_page(pf, projects=None): + """add single project finder result to page""" + if not projects: + projects = pf.list_projects() + ui.context.projects += [ + project for project in projects if project not in ui.context.projects + ] + ui.label(f"{pf.url}").classes("text-xl font-bold mb-4 text-center") + with ui.row().classes("grid grid-cols-2 gap-4 overflow-auto max-h-screen w-2/3"): + for project in projects: + add_project_to_page(project) + + +def add_project_to_page(project): + """add single project to page""" + with ui.card().classes( + "bg-black text-white p-6 rounded shadow-lg \ + text-center w-full h-40 flex justify-center items-center" + ): + ui.link(text=project.name, target=f"/project_data/{project.name}").classes( + "text-white hover:text-gray-400 no-underline" + ) + + +@ui.page("/projects/") +def projects_page(): + """projects for source""" + header() + search_input = ui.input(placeholder="Search packages").classes("flex-grow") + search_input.on_value_change(lambda e: update_autocomplete(e.value)) + ui.context.project_col = ui.column().classes("w-2/3 mx-auto mt-10") + with ui.context.project_col: + try: + add_projects_to_page() + except AttributeError as e: + print(e) + ui.navigate.to("/sources") + + +def update_autocomplete(value): + """autocomplete for project search""" + ui.context.project_col.clear() + with ui.context.project_col: + if not value: + print("adding all projects") + add_projects_to_page() + else: + print(f"adding projects matching {value}") + for pf in ui.context.pf: + sorted_list = [] + for project in pf.list_projects(): + url, repo_path, src = ( + fuzz.ratio(value, project.url), + -fuzz.ratio(value, project.repo_path), + -fuzz.ratio(value, project.src), + ) + print( + f"ratios {project.url}{project.repo_path}{project.src}\ +- {fuzz.ratio(value,project.url)}\ +- {fuzz.ratio(value,project.repo_path)}\ +- {fuzz.ratio(value,project.src)}" + ) + ratio = max( + fuzz.ratio(value, project.url), + fuzz.ratio(value, project.repo_path), + fuzz.ratio(value, project.src), + ) + if ratio > 30 or url > 20 or repo_path > 20 or src > 20: + sorted_list += [(ratio, project)] + sorted_list.sort(key=lambda i: i[0], reverse=True) + for ratio, project in sorted_list: + add_project_to_page(project) + + +@ui.page("/project_data/{name}") +def projects_data_page(name: str): + """data for project""" + header() + with ui.column().classes("w-5/6 items-center mx-auto mt-10"): + try: + found_project = None + for project in ui.context.projects: + if project.name == name: + found_project = project + break + if found_project: + project_representation(found_project) + else: + ui.label( + f"could not find project in \ +{[project.name for project in ui.context.projects]}" + ) + except AttributeError: + ui.label(f"{name} was not found") + + +@ui.page("/filters") +def filters_page(): + """page showing exclusions per source""" + header() + ui.notify("no sources present, redirecting to sources") + if hasattr(ui.context, "pf") and len(ui.context.pf) > 0: + with ui.column(): + for pf in ui.context.pf: + with ui.row(): + ui.label(f"{pf.url}") + filter_in = ui.input(placeholder="enter filter regex") + ui.button( + "add exclusion", + on_click=lambda pf=pf, filter_in=filter_in: add_exclusion( + pf, filter_in.value + ), + ) + if ( + hasattr(pf, "exclusions") + and pf.exclusions + and len(pf.exclusions) > 0 + ): + with ui.column(): + for excl in pf.exclusions: + ui.label(f"{excl}") + ui.button("store", on_click=presist_sources) + else: + ui.navigate.to("/sources") + + +def add_exclusion(pf, regex): + """add exclusion for the project finder for a source""" + ui.notify(f"adding exclusion {regex} to projects on url {pf.url}") + project = [ + project for project in ui.context.sl.get_remotes() if project.url == pf.url + ][0] + project.add_exclusion(regex) + pf.add_exclusion(regex) + pf.filter_projects() + + +def presist_sources(): + """persist entered sources to file""" + sl = ui.context.sl + ui.download(sl.as_yaml().encode("utf-8"), filename="sources.yaml") + + +def url_input(): + """url input page""" + url_search_field = ui.input(placeholder="enter url to list packages").classes( + "w-full p-2 text-lg border border-gray-300 rounded" + ) + ui.button( + text="get projects", on_click=lambda a: get_projects(url_search_field.value) + ).classes("bg-black text-white px-4 py-2 rounded hover:bg-gray-800 mt-4") + + +def sources_input(): + """input sources file""" + ui.upload( + on_upload=lambda e: handle_upload(e) # pylint:disable = unnecessary-lambda + ).props("accept=.yaml").classes("max-w-full") + + +def handle_upload(file: events.UploadEventArguments): + """handle upload of sources file""" + ui.context.sl = SourceList.from_yaml(file.content.read()) + ui.notify(f"uploaded {file.name}") + + +def get_projects(url): + """handling of project search""" + if url and len(url) > 5: # what is min valid url len? + name = url.split("/")[-1] + ui.context.sl.add_remote(RemoteSource({"name": name, "url-base": url})) + ui.navigate.to("/projects/") + + +def project_representation(project): + """project representation""" + ui.label(project.name).classes("text-h5 text-black mb-5") + + with ui.row().classes("justify-between m-20"): + # Empty space in column 1 (1-2) + with ui.column().classes("w-full sm:w-1/12"): + pass # No content here (empty) + + # Column 1 (2-4), spanning 3 parts + with ui.column().classes("w-full sm:w-3/12"): + ui.link( + project.url, target=f"http://{project.url}/{project.repo_path}" + ).classes("text-body1 text-black") + ui.label(f"Source - {project.src if project.src else "/"}").classes( + "text-body2 text-black" + ) + ui.label(f"vcs - {project.vcs}") + + # Column 2 (4-7), spanning 3 parts + with ui.column().classes("w-full sm:w-3/12"): + ui.label("branches").classes("text-h5 text-black mb-3") + for branch in project.versions.branches: + revision_representation(branch) + + # Column 3 (7-10), spanning 3 parts + with ui.column().classes("w-full sm:w-3/12"): + ui.label("tags").classes("text-h5 text-black mb-3") + for tag in project.versions.tags: + revision_representation(tag) + + # Empty space in column 3 (10-12) + with ui.column().classes("w-full sm:w-1/12"): + pass # No content here (empty) + + +def revision_representation(rev): + """revision representation""" + ui.label(f"revision {rev.name} - {rev.revision}").classes( + "text-body2 text-black mb-2" + ) + + +def show_sources(): + """show sources in source view""" + if hasattr(ui.context, "pf"): + for pf in ui.context.pf: + ui.label(f"{pf.url}") + + +if __name__ in ("__main__", "__mp_main__"): + main() diff --git a/dfetch_hub/project/cli.py b/dfetch_hub/project/cli.py new file mode 100644 index 0000000..2cc2696 --- /dev/null +++ b/dfetch_hub/project/cli.py @@ -0,0 +1,47 @@ +""" + +""" + +import argparse + +from dfetch_hub.project.input_parser import InputParser +from dfetch_hub.project.project_finder import GitProjectFinder +from dfetch_hub.project.project_parser import ProjectParser +from dfetch_hub.project.project_sources import SourceList + +# from project.cli_disp import CliDisp + + +def main(parser: argparse.ArgumentParser): + """main command line interface for program""" + args = parser.parse_args() + if not args.url and not args.dfetch_source: + parser.print_help() + raise ValueError("no url or dfetch manifest found") + input_args_parser = InputParser(args) + url_list = input_args_parser.get_urls() + if args.persist_sources: + sources_list = SourceList.from_input_parser(input_args_parser) + with open("sources.yaml", "w", encoding="utf-8") as sources_file: + sources_file.write(sources_list.as_yaml()) + parser = ProjectParser() + for url in url_list: + gpf = GitProjectFinder(url, args.project_exclude_pattern) + projects = gpf.list_projects() + for project in projects: + parser.add_project(project) + with open("projects.yaml", "w", encoding="utf-8") as datasource: + datasource.write(parser.get_projects_as_yaml()) + + +if __name__ == "__main__": + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument("-u", "--url", required=False, nargs="+") + arg_parser.add_argument("-ds", "--dfetch-source", required=False) + arg_parser.add_argument( + "-pep", "--project-exclude-pattern", required=False, nargs="+" + ) + arg_parser.add_argument( + "-ps", "--persist-sources", required=False, action="store_true" + ) + main(arg_parser) diff --git a/dfetch_hub/project/export.py b/dfetch_hub/project/export.py new file mode 100644 index 0000000..286ba61 --- /dev/null +++ b/dfetch_hub/project/export.py @@ -0,0 +1,33 @@ +"""export module""" +from abc import ABC +from dfetch.manifest.manifest import Manifest, ManifestDict +from dfetch.manifest.project import ProjectEntry, ProjectEntryDict +from dfetch.manifest.remote import Remote, RemoteDict + +class Export(ABC): + + def export(self): + pass + +class DfetchExport(Export): + + def __init__(self, entries=None): + if entries: + self._entries = entries + else: + self._entries = [] + + def add_entry(self, entry): + self._entries += [entry] + + @property + def entries(self): + return self._entries + + def export(self, path=None): + remotes = [] # TODO: bundle projects with shared path in remotes + projects = [ProjectEntryDict(name=entry.name, revision=entry.revision, src=entry.src, url=entry.url, repo_path=entry.repo_path, vcs=entry.vcs) for entry in self._entries] + as_dict = ManifestDict(version = Manifest.CURRENT_VERSION, remotes=remotes, projects=projects) + if not path: + path = "dfetch.yaml" + Manifest(as_dict).dump(path) \ No newline at end of file diff --git a/dfetch_hub/project/input_parser.py b/dfetch_hub/project/input_parser.py new file mode 100644 index 0000000..57120ae --- /dev/null +++ b/dfetch_hub/project/input_parser.py @@ -0,0 +1,24 @@ +"""input parser module""" + +from typing import Sequence + +from dfetch.manifest.manifest import Manifest + + +class InputParser: # pylint:disable=too-few-public-methods + """parser for url or dfetch file input""" + + def __init__(self, args): + self.args = args + + def get_urls(self) -> Sequence[str]: + """get urls for input""" + if self.args.url: + if isinstance(self.args.url, list): + return self.args.url + return [self.args.url] + return self._parse_dfetch_remotes(self.args.dfetch_source) + + def _parse_dfetch_remotes(self, dfetch_path) -> Sequence[str]: + manifest = Manifest.from_file(dfetch_path) + return [project.remote_url for project in manifest.projects] diff --git a/dfetch_hub/project/project_finder.py b/dfetch_hub/project/project_finder.py new file mode 100644 index 0000000..2ad65bb --- /dev/null +++ b/dfetch_hub/project/project_finder.py @@ -0,0 +1,174 @@ +"""project finder module""" + +import logging +import os +import re +import sys +from abc import abstractmethod +from contextlib import chdir +from typing import Optional, Sequence + +from dfetch.util.cmdline import SubprocessCommandError, run_on_cmdline + +from dfetch_hub.project.remote_datasource import RemoteProject + +WORKDIR = "tmp" + + +class ProjectFinder: + """class to find projects in repositories""" + + def __init__(self, url: str, exclusions: Optional[Sequence[str]] = None): + self._url = url + self._logger = logging.getLogger() + self._projects: list[str] = [] + self._exclusions = exclusions + + @property + def url(self): + """repo url""" + return self._url + + @abstractmethod + def list_projects(self): + """list all projects in a repo""" + raise AssertionError("abstractmethod") + + def filter_exclusions(self, paths: Sequence[str]): + """filter exclusions from list of projects""" + filtered_paths = [] + for path in paths: + path_allowed = True + if self._exclusions: + for exclusion in self._exclusions: + try: + re.compile(exclusion) + except re.error as exc: + raise ValueError( + f"regex of exclusion is invalid, {exclusion}" + ) from exc + path_allowed = path_allowed and not re.search(exclusion, path) + if not path_allowed: + break + if path_allowed and path not in filtered_paths: + filtered_paths += [path] + else: + filtered_paths = paths + return filtered_paths + + @property + def exclusions(self): + """get exclusion for project finder""" + return self._exclusions + + def add_exclusion(self, exclusion): + """add an exclusion regex""" + if exclusion: + if not self._exclusions: + self._exclusions = [] + self._exclusions += [exclusion] + print(f"exclusions are {self.exclusions}") + + def filter_projects(self): + """filter projects on exclusions""" + for project in self._projects: + path = f"{project.url, project.repo_path, project.src}" + path_allowed = True + if self._exclusions: + for exclusion in self._exclusions: + try: + re.compile(exclusion) + except re.error as exc: + raise ValueError( + f"regex of exclusion is invalid, {exclusion}" + ) from exc + path_allowed = path_allowed and not re.search(exclusion, path) + if not path_allowed: + break + if not path_allowed: + self._projects.remove(project) + + +class GitProjectFinder(ProjectFinder): + """git implementation of project finder""" + + def list_projects(self): + """list all git projects in a git repo""" + if not self._projects: + if os.path.exists(WORKDIR) and os.path.isdir(WORKDIR): + if sys.platform == "win32": + os.system(f"rmdir /S /Q {WORKDIR}") + elif sys.platform == "linux": + os.system(f"rm -rf {WORKDIR}") + try: + result = run_on_cmdline( + self._logger, f"git clone --no-checkout {self._url} {WORKDIR}" + ) + with chdir(WORKDIR): + result = run_on_cmdline(self._logger, "git status") + # More matching (specific types, add interface) + # keep matched on (e.g. project x matched on ...) + res = re.findall( + r"\sdeleted:\s+(.*(?:README|LICENSE|CHANGELOG|Readme|readme|License|license).*)", # pylint:disable=line-too-long + result.stdout.decode("utf-8"), + ) + result = run_on_cmdline( + self._logger, "git for-each-ref refs/remotes/origin" + ) + branches = re.findall( + r"([a-f0-9]*)\scommit\s.*/origin/(.*)", + result.stdout.decode("utf-8"), + ) + result = run_on_cmdline(self._logger, "git for-each-ref refs/tags") + tags = re.findall( + r"([a-f0-9]*)\s(?:(?:commit)|(?:tag))\s*refs/tags/(.*)", + result.stdout.decode("utf-8"), + ) + except SubprocessCommandError as exc: + raise ValueError( + f"could not find repository at url {self._url}" + ) from exc + finally: + if sys.platform == "win32": + os.system(f"rmdir /S /Q {WORKDIR}") + elif sys.platform == "linux": + os.system(f"rm -rf {WORKDIR}") + paths = set() + for path in res: + if "/" in path: + paths.add(f"{path.rsplit("/", maxsplit=1)[0].strip(" ")}") + else: + paths.add("") + filtered_paths = self.filter_exclusions(paths) + self._projects = self._projects_from_paths(filtered_paths, branches, tags) + return self._projects + + def _projects_from_paths( + self, paths: Sequence[str], branches=Sequence[str], tags=Sequence[str] + ): + projects = [] + for path in paths: + if "/" in path: + name = path.rsplit("/", maxsplit=1)[1] + elif len(path) > 1: + name = path + else: + name = self.url.rsplit("/", maxsplit=1)[1] + base_url, repo_path = _base_url(self.url) + src = path + vcs = "git" + project = RemoteProject(name, base_url, repo_path, src, vcs) + project.versions.vcs = vcs + project.add_versions(branches, tags) + projects += [project] + return projects + + +def _base_url(url): + if "://" in url: + url = url.split("://", maxsplit=1)[1] + if "/" in url: + url, repo_path = url.split("/", maxsplit=1) + else: + repo_path = "" + return url, repo_path diff --git a/dfetch_hub/project/project_parser.py b/dfetch_hub/project/project_parser.py new file mode 100644 index 0000000..176def5 --- /dev/null +++ b/dfetch_hub/project/project_parser.py @@ -0,0 +1,47 @@ +"""project parser module""" + +from typing import List + +import yaml + +from dfetch_hub.project.remote_datasource import RemoteProject + + +class ProjectParser: + """class that parses python projects + + - used on projects found by project finder + - parsed into sources which can be stored and monitored + """ + + def __init__(self): + self._projects: List[RemoteProject] = [] + + def add_project(self, new_project: RemoteProject): + """add a project""" + if new_project not in self._projects: + self._projects += [new_project] + + def get_projects(self): + """get all projects""" + return self._projects + + def get_projects_as_yaml(self): + """get yaml representation of projects""" + yaml_str = "" + yaml_obj = {"projects": []} + for project in self._projects: + yaml_obj["projects"] += [project.as_yaml()] + yaml_str = yaml.dump(yaml_obj) + return yaml_str + + @classmethod + def from_yaml(cls, yaml_file): + """create parser from yaml file""" + with open(yaml_file, "r", encoding="utf-8") as yamlf: + instance = cls() + yaml_data = yaml.load(yamlf.read(), Loader=yaml.Loader) + for project in yaml_data["projects"]: + parsed_project = RemoteProject.from_yaml(project) + instance.add_project(parsed_project) + return instance diff --git a/dfetch_hub/project/project_sources.py b/dfetch_hub/project/project_sources.py new file mode 100644 index 0000000..58ef314 --- /dev/null +++ b/dfetch_hub/project/project_sources.py @@ -0,0 +1,108 @@ +"""project sources module""" + +from typing import Optional, Sequence + +import yaml +from dfetch.manifest.remote import Remote + +from dfetch_hub.project.input_parser import InputParser + + +class RemoteSource(Remote): + """class representing source for projects""" + + def __init__(self, args): + super().__init__(args) + self.exclusions: Optional[Sequence] = None + + def add_exclusion(self, exclusion_regex: str): + """add exclusion to project source""" + if not self.exclusions: + self.exclusions = [exclusion_regex] + else: + self.exclusions += [exclusion_regex] + + def as_yaml(self): + """get yaml representation""" + yaml_data = super().as_yaml() + yaml_data["exclusions"] = self.exclusions + return {k: v for k, v in yaml_data.items() if v} + + def __eq__(self, other): + if isinstance(other, RemoteSource): + if hasattr(self, "exclusions"): + if not hasattr(other, "exclusions"): + return False + return ( + self.name == other.name + and self.url == other.url + and self.exclusions == other.exclusions + ) + if hasattr(other, "exclusions"): + return False + return self.name == other.name and self.url == other.url + return False + + +class SourceList: + """class representing a sequence of project sources""" + + CURRENT_VERSION = "0.0" + + def __init__(self): + self._sources: Sequence[RemoteSource] = [] + + def add_remote(self, source: RemoteSource): + """add source""" + self._sources += [source] + + def get_remotes(self) -> list[RemoteSource]: + """get list of sources""" + return self._sources + + def as_yaml(self): + """yaml representation""" + versiondata = {"version": self.CURRENT_VERSION} + remotes_data = {"remotes": [source.as_yaml() for source in self._sources]} + yamldata = {"source-list": [versiondata, remotes_data]} + return yaml.dump(yamldata) + + @classmethod + def from_yaml(cls, yaml_data): + """load from sources files""" + if not yaml_data: + raise ValueError("failed to load data from file") + instance = cls() + yaml_data = yaml.load(yaml_data, Loader=yaml.Loader) + assert yaml_data, "file should have data" + assert yaml_data["source-list"], "file should have list of sources" + version = [i["version"] for i in yaml_data["source-list"] if "version" in i][0] + remotes = [i["remotes"] for i in yaml_data["source-list"] if "remotes" in i][0] + if version != cls.CURRENT_VERSION: + raise ValueError("invalid version") + + for source in remotes: + src = RemoteSource({"name": source["name"], "url-base": source["url-base"]}) + if "exclusions" in source: + for excl in source["exclusions"]: + src.add_exclusion(excl) + instance.add_remote(src) + return instance + + @classmethod + def from_input_parser(cls, parser: InputParser): + """generate instance from parser""" + instance = cls() + for url in parser.get_urls(): + name = url.split("/")[-1] + src = RemoteSource({"name": name, "url-base": url}) + instance.add_remote(src) + return instance + + def __eq__(self, other): + if isinstance(other, SourceList): + return ( + self._sources == other._sources + and self.CURRENT_VERSION == other.CURRENT_VERSION + ) + return False diff --git a/dfetch_hub/project/remote_datasource.py b/dfetch_hub/project/remote_datasource.py new file mode 100644 index 0000000..40bbb99 --- /dev/null +++ b/dfetch_hub/project/remote_datasource.py @@ -0,0 +1,141 @@ +"""remote datasource module""" + +from dataclasses import dataclass +from typing import Sequence, Tuple + + +@dataclass +class RemoteRef: + """representation of a single remote reference""" + + name: str + revision: str # chosen for dfetch naming + + def as_yaml(self): + """yaml representation of reference""" + yamldata = {"name": self.name, "revision": self.revision} + return {k: v for k, v in yamldata.items() if v} + + def __eq__(self, other): + if isinstance(other, str): + return self.name == other + if isinstance(other, RemoteRef): + return self.name == other.name and self.revision == other.revision + return False + + +class RemoteProjectVersions: + """representation of collection of versions for project""" + + def __init__(self, vcs=None): + self.tags = [] + self.branches = [] + self.vcs = vcs + + def add_tags(self, tags): + """add tags""" + for hash_val, tag_name in tags: + if tag_name not in [tag.name for tag in self.tags]: + self.tags += [RemoteRef(tag_name, hash_val)] + + def add_branches(self, branches): + """add branches""" + for hash_val, branch_name in branches: + if branch_name not in [branch.name for branch in self.branches]: + self.branches += [RemoteRef(branch_name, hash_val)] + + @property + def default(self): + """get default branch""" + if not self.vcs or self.vcs == "git": + return "main" if "main" in self.branches else "master" + if self.vcs == "svn": + return "trunk" + raise ValueError("no default version known for repository") + + def as_yaml(self): + """get yaml representation""" + default = None + try: + default = self.default + except ValueError: + pass + yamldata = { + "default": default, + "tags": [tag.as_yaml() for tag in self.tags], + "branches": [branch.as_yaml() for branch in self.branches], + } + return {k: v for k, v in yamldata.items() if v} + + def __eq__(self, other): + if isinstance(other, RemoteProjectVersions): + return other.branches == self.branches and other.tags == self.tags + return False + + +class RemoteProject: + """representation of remote repository project""" + + def __init__( + self, name, url, repo_path, src, vcs, versions=None + ): # pylint:disable=too-many-arguments,too-many-positional-arguments + self.name = name + self.url = url + self.repo_path = repo_path + self.src = src + self.vcs = vcs + self.versions = versions if versions else RemoteProjectVersions() + + def add_versions( + self, branches=Sequence[Tuple[str, str]], tags=Sequence[Tuple[str, str]] + ): + """add branches and tags""" + if not hasattr(self, "versions"): + self.versions = RemoteProjectVersions(self.vcs) + self.versions.add_branches(branches) + self.versions.add_tags(tags) + + def as_yaml(self): + """get yaml representation""" + yamldata = { + "name": self.name, + "versions": ( + None if not hasattr(self, "versions") else self.versions.as_yaml() + ), + "src": self.src, + "url": self.url, + "repo-path": self.repo_path, + "vcs": None if not hasattr(self, "vcs") else self.vcs, + } + return {k: v for k, v in yamldata.items() if v} + + @classmethod + def from_yaml(cls, yaml_data): + """build project from yaml representation""" + src = None if "src" not in yaml_data else yaml_data["src"] + versions = None if "versions" not in yaml_data else yaml_data["versions"] + parsed = cls( + yaml_data["name"], + yaml_data["url"], + yaml_data["repo-path"], + src, + vcs=yaml_data["vcs"], + ) + if versions: + branches = [ + (branch["revision"], branch["name"]) for branch in versions["branches"] + ] + tags = [(tag["revision"], tag["name"]) for tag in versions["tags"]] + parsed.add_versions(branches=branches, tags=tags) + return parsed + + def __eq__(self, other): + if isinstance(other, str): + return other == self.name + if isinstance(other, RemoteProject): + return ( + other.name == self.name + and other.url == self.url + and other.repo_path == self.repo_path + ) + return False diff --git a/doc/index.md b/doc/index.md new file mode 100644 index 0000000..755cb35 --- /dev/null +++ b/doc/index.md @@ -0,0 +1,35 @@ +# Remote Project Overview + +- [Remote Project Overview](#remote-project-overview) + - [Idea](#idea) + - [File types](#file-types) + - [Source list](#source-list) + - [Project lists](#project-lists) + - [Output file](#output-file) + +## Idea + +- url to list project + versions +- dfetch.yaml formatter to output selected items + +## File types + +### Source list + +Source list is a type used to list a collection of project sources. +A project source is a place to look for projects. +Optionally a list of exclusions can be added so certain patterns that look like potential projects but are not projects are excluded. + +The source list can be used to share project locations and exclusions between users. + +### Project lists + +The project list is the main overview containing all the projects in the sources. +It is populated by the information gathered by the `ProjectFinder` when it is passed to the `ProjectParser` by looking in the project sources locations and parsing the information there. +The `ProjectParser` is the module that can use a project list as a source of projects and generate entries for dependency fetchers. +The current use case in mind is `dFetch`, but in the future e.g. `git submodule`'s or other dependency fetchers could be used. + +### Output file + +The output of this project is a source file which can be used by other tools to import the selected projects. +Our first use case is to create `dFetch` manifest files, which can then be used to import the selected versions of projects into other projects. diff --git a/doc/program_components.puml b/doc/program_components.puml new file mode 100644 index 0000000..b139e0c --- /dev/null +++ b/doc/program_components.puml @@ -0,0 +1,31 @@ +@startuml program_components +!include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Component.puml + +Person(user, "user", "person that uses project") + +Container(input_parser, "input parser", "parses input to program project sources") +Container(project_finder, "project finder", "searches repositories for projects") +Container(project_parser, "project parser", "parses repositories data to project data") +Container(export, "export", "select data and export") +Container_Ext(dfetch, "fetch dependencies", "update project with requested dependencies") + +System(project_sources, "project sources file", "contains project sources") +System(project_list, "project data list", "contains all projects for given sources") +System(output_file, "dfetch (or other format) outfile", "contains required info to get project") + +Rel(user, input_parser, "add input", "url, dfetch file, ...") + +Rel(input_parser, project_sources, "generates", "persist data") +Rel_R(input_parser, project_finder, "add parsed sources", "add info required to parse projects") + +Rel_U(project_sources, project_finder, "add parsed sources", "add info required to parse projects") + +Rel_R(project_finder, project_parser, "info to create projects") +Rel(project_parser, project_list, "generates", "persist data") + +Rel_R(project_parser, export, "forward project versions") +Rel(export, output_file, "generates", "persist data") + +Rel(output_file, dfetch, "let dfetch use", "dfetch manifest") + +@enduml diff --git a/projects.yaml b/projects.yaml new file mode 100644 index 0000000..cc7d545 --- /dev/null +++ b/projects.yaml @@ -0,0 +1,315 @@ +projects: +- name: dfetch + repo-path: dfetch-org/dfetch + url: github.com + vcs: git + versions: + branches: + - name: HEAD + revision: c037c000e200704f3db1ac6af1aceeb79cb1954c + - name: add-more-checkers + revision: fa464794a55d2bc2e64a029a8ac1152d1e057088 + - name: dependabot/docker/dot-devcontainer/devcontainers/python-3.13-bullseye + revision: 4777980b443154f0c00f32f5e77a1d2a5d8dec61 + - name: dependabot/pip/main/cyclonedx-python-lib-5.0.1 + revision: 4cd13444f74c7817e212f043dc1e3a75bab8e379 + - name: main + revision: c037c000e200704f3db1ac6af1aceeb79cb1954c + - name: prototype-extended-data + revision: 635cf9711577f446b1a7c9d4048bd6a3854eb872 + default: main + tags: + - name: 0.1.0 + revision: 3733a64356b8899b43d4c9145a7c7ca8aa529927 + - name: 0.1.1 + revision: 7ef62b1cf04f1f3c30a57b0138b5d7b50ec076b7 + - name: 0.2.0 + revision: 3cd50945cb736d8aadce5d8194ea746f454b0e55 + - name: 0.3.0 + revision: 74f1bb642800ff6f561d8bcd1aad01fce7e993e4 + - name: 0.4.0 + revision: 4d960baa0725bf805fecd08ad9f60a4f706e4be0 + - name: 0.5.1 + revision: 0422cae6f3d7f46f8667af3fce67b607dca076df + - name: 0.6.0 + revision: ad0a73489e762e83f911deabe18f6764678bb013 + - name: 0.7.0 + revision: 192466458d69530a149311a98ed95efe51dbcf90 + - name: 0.8.0 + revision: 43319554160abb9ebad64614fd1c8d2beaa2296e + - name: 0.9.0 + revision: 069c65f74000ef0f906fc5574023f6d2eaeb6c5b + - name: 0.9.1 + revision: 73a84be9c02d492d6d39bbaaf9df218696f56ab1 + - name: v0.0.1 + revision: b4c04095990e2abc18a925e82c678d50d90f16af + - name: v0.0.2 + revision: 91aa476b8cd63ae2faf4de19d078f95e29b16bd3 + - name: v0.0.3 + revision: cecd0f2752a25f5af7e2568dac0a1b0432a149c6 + - name: v0.0.4 + revision: 9dbca3302925062e42740bccab00051a112b9cf6 + - name: v0.0.5 + revision: 9267fb1125e2efc05c32b4f9250a2cedf395753f + - name: v0.0.6 + revision: 7e6933bded164a917cbb1a510c67d976656263d2 + - name: v0.0.7 + revision: 1e47a197fdf23cd55ba58d4b7b3a2491cb550302 + - name: v0.0.8 + revision: f8a4541891194c07b953ef6af2c251fe40a8987e + - name: v0.0.9 + revision: 422c7d9d9578ccb8d79d8a6b14dac6f6e30bc4f8 + - name: v0.5.0 + revision: fbedf548fc2929250bdcc4252af298658b36b01c +- name: demo-magic + repo-path: dfetch-org/dfetch + src: doc/generate-casts/demo-magic + url: github.com + vcs: git + versions: + branches: + - name: HEAD + revision: c037c000e200704f3db1ac6af1aceeb79cb1954c + - name: add-more-checkers + revision: fa464794a55d2bc2e64a029a8ac1152d1e057088 + - name: dependabot/docker/dot-devcontainer/devcontainers/python-3.13-bullseye + revision: 4777980b443154f0c00f32f5e77a1d2a5d8dec61 + - name: dependabot/pip/main/cyclonedx-python-lib-5.0.1 + revision: 4cd13444f74c7817e212f043dc1e3a75bab8e379 + - name: main + revision: c037c000e200704f3db1ac6af1aceeb79cb1954c + - name: prototype-extended-data + revision: 635cf9711577f446b1a7c9d4048bd6a3854eb872 + default: main + tags: + - name: 0.1.0 + revision: 3733a64356b8899b43d4c9145a7c7ca8aa529927 + - name: 0.1.1 + revision: 7ef62b1cf04f1f3c30a57b0138b5d7b50ec076b7 + - name: 0.2.0 + revision: 3cd50945cb736d8aadce5d8194ea746f454b0e55 + - name: 0.3.0 + revision: 74f1bb642800ff6f561d8bcd1aad01fce7e993e4 + - name: 0.4.0 + revision: 4d960baa0725bf805fecd08ad9f60a4f706e4be0 + - name: 0.5.1 + revision: 0422cae6f3d7f46f8667af3fce67b607dca076df + - name: 0.6.0 + revision: ad0a73489e762e83f911deabe18f6764678bb013 + - name: 0.7.0 + revision: 192466458d69530a149311a98ed95efe51dbcf90 + - name: 0.8.0 + revision: 43319554160abb9ebad64614fd1c8d2beaa2296e + - name: 0.9.0 + revision: 069c65f74000ef0f906fc5574023f6d2eaeb6c5b + - name: 0.9.1 + revision: 73a84be9c02d492d6d39bbaaf9df218696f56ab1 + - name: v0.0.1 + revision: b4c04095990e2abc18a925e82c678d50d90f16af + - name: v0.0.2 + revision: 91aa476b8cd63ae2faf4de19d078f95e29b16bd3 + - name: v0.0.3 + revision: cecd0f2752a25f5af7e2568dac0a1b0432a149c6 + - name: v0.0.4 + revision: 9dbca3302925062e42740bccab00051a112b9cf6 + - name: v0.0.5 + revision: 9267fb1125e2efc05c32b4f9250a2cedf395753f + - name: v0.0.6 + revision: 7e6933bded164a917cbb1a510c67d976656263d2 + - name: v0.0.7 + revision: 1e47a197fdf23cd55ba58d4b7b3a2491cb550302 + - name: v0.0.8 + revision: f8a4541891194c07b953ef6af2c251fe40a8987e + - name: v0.0.9 + revision: 422c7d9d9578ccb8d79d8a6b14dac6f6e30bc4f8 + - name: v0.5.0 + revision: fbedf548fc2929250bdcc4252af298658b36b01c +- name: generate-casts + repo-path: dfetch-org/dfetch + src: doc/generate-casts + url: github.com + vcs: git + versions: + branches: + - name: HEAD + revision: c037c000e200704f3db1ac6af1aceeb79cb1954c + - name: add-more-checkers + revision: fa464794a55d2bc2e64a029a8ac1152d1e057088 + - name: dependabot/docker/dot-devcontainer/devcontainers/python-3.13-bullseye + revision: 4777980b443154f0c00f32f5e77a1d2a5d8dec61 + - name: dependabot/pip/main/cyclonedx-python-lib-5.0.1 + revision: 4cd13444f74c7817e212f043dc1e3a75bab8e379 + - name: main + revision: c037c000e200704f3db1ac6af1aceeb79cb1954c + - name: prototype-extended-data + revision: 635cf9711577f446b1a7c9d4048bd6a3854eb872 + default: main + tags: + - name: 0.1.0 + revision: 3733a64356b8899b43d4c9145a7c7ca8aa529927 + - name: 0.1.1 + revision: 7ef62b1cf04f1f3c30a57b0138b5d7b50ec076b7 + - name: 0.2.0 + revision: 3cd50945cb736d8aadce5d8194ea746f454b0e55 + - name: 0.3.0 + revision: 74f1bb642800ff6f561d8bcd1aad01fce7e993e4 + - name: 0.4.0 + revision: 4d960baa0725bf805fecd08ad9f60a4f706e4be0 + - name: 0.5.1 + revision: 0422cae6f3d7f46f8667af3fce67b607dca076df + - name: 0.6.0 + revision: ad0a73489e762e83f911deabe18f6764678bb013 + - name: 0.7.0 + revision: 192466458d69530a149311a98ed95efe51dbcf90 + - name: 0.8.0 + revision: 43319554160abb9ebad64614fd1c8d2beaa2296e + - name: 0.9.0 + revision: 069c65f74000ef0f906fc5574023f6d2eaeb6c5b + - name: 0.9.1 + revision: 73a84be9c02d492d6d39bbaaf9df218696f56ab1 + - name: v0.0.1 + revision: b4c04095990e2abc18a925e82c678d50d90f16af + - name: v0.0.2 + revision: 91aa476b8cd63ae2faf4de19d078f95e29b16bd3 + - name: v0.0.3 + revision: cecd0f2752a25f5af7e2568dac0a1b0432a149c6 + - name: v0.0.4 + revision: 9dbca3302925062e42740bccab00051a112b9cf6 + - name: v0.0.5 + revision: 9267fb1125e2efc05c32b4f9250a2cedf395753f + - name: v0.0.6 + revision: 7e6933bded164a917cbb1a510c67d976656263d2 + - name: v0.0.7 + revision: 1e47a197fdf23cd55ba58d4b7b3a2491cb550302 + - name: v0.0.8 + revision: f8a4541891194c07b953ef6af2c251fe40a8987e + - name: v0.0.9 + revision: 422c7d9d9578ccb8d79d8a6b14dac6f6e30bc4f8 + - name: v0.5.0 + revision: fbedf548fc2929250bdcc4252af298658b36b01c +- name: plantuml-c4 + repo-path: dfetch-org/dfetch + src: doc/static/uml/styles/plantuml-c4 + url: github.com + vcs: git + versions: + branches: + - name: HEAD + revision: c037c000e200704f3db1ac6af1aceeb79cb1954c + - name: add-more-checkers + revision: fa464794a55d2bc2e64a029a8ac1152d1e057088 + - name: dependabot/docker/dot-devcontainer/devcontainers/python-3.13-bullseye + revision: 4777980b443154f0c00f32f5e77a1d2a5d8dec61 + - name: dependabot/pip/main/cyclonedx-python-lib-5.0.1 + revision: 4cd13444f74c7817e212f043dc1e3a75bab8e379 + - name: main + revision: c037c000e200704f3db1ac6af1aceeb79cb1954c + - name: prototype-extended-data + revision: 635cf9711577f446b1a7c9d4048bd6a3854eb872 + default: main + tags: + - name: 0.1.0 + revision: 3733a64356b8899b43d4c9145a7c7ca8aa529927 + - name: 0.1.1 + revision: 7ef62b1cf04f1f3c30a57b0138b5d7b50ec076b7 + - name: 0.2.0 + revision: 3cd50945cb736d8aadce5d8194ea746f454b0e55 + - name: 0.3.0 + revision: 74f1bb642800ff6f561d8bcd1aad01fce7e993e4 + - name: 0.4.0 + revision: 4d960baa0725bf805fecd08ad9f60a4f706e4be0 + - name: 0.5.1 + revision: 0422cae6f3d7f46f8667af3fce67b607dca076df + - name: 0.6.0 + revision: ad0a73489e762e83f911deabe18f6764678bb013 + - name: 0.7.0 + revision: 192466458d69530a149311a98ed95efe51dbcf90 + - name: 0.8.0 + revision: 43319554160abb9ebad64614fd1c8d2beaa2296e + - name: 0.9.0 + revision: 069c65f74000ef0f906fc5574023f6d2eaeb6c5b + - name: 0.9.1 + revision: 73a84be9c02d492d6d39bbaaf9df218696f56ab1 + - name: v0.0.1 + revision: b4c04095990e2abc18a925e82c678d50d90f16af + - name: v0.0.2 + revision: 91aa476b8cd63ae2faf4de19d078f95e29b16bd3 + - name: v0.0.3 + revision: cecd0f2752a25f5af7e2568dac0a1b0432a149c6 + - name: v0.0.4 + revision: 9dbca3302925062e42740bccab00051a112b9cf6 + - name: v0.0.5 + revision: 9267fb1125e2efc05c32b4f9250a2cedf395753f + - name: v0.0.6 + revision: 7e6933bded164a917cbb1a510c67d976656263d2 + - name: v0.0.7 + revision: 1e47a197fdf23cd55ba58d4b7b3a2491cb550302 + - name: v0.0.8 + revision: f8a4541891194c07b953ef6af2c251fe40a8987e + - name: v0.0.9 + revision: 422c7d9d9578ccb8d79d8a6b14dac6f6e30bc4f8 + - name: v0.5.0 + revision: fbedf548fc2929250bdcc4252af298658b36b01c +- name: features + repo-path: dfetch-org/dfetch + src: features + url: github.com + vcs: git + versions: + branches: + - name: HEAD + revision: c037c000e200704f3db1ac6af1aceeb79cb1954c + - name: add-more-checkers + revision: fa464794a55d2bc2e64a029a8ac1152d1e057088 + - name: dependabot/docker/dot-devcontainer/devcontainers/python-3.13-bullseye + revision: 4777980b443154f0c00f32f5e77a1d2a5d8dec61 + - name: dependabot/pip/main/cyclonedx-python-lib-5.0.1 + revision: 4cd13444f74c7817e212f043dc1e3a75bab8e379 + - name: main + revision: c037c000e200704f3db1ac6af1aceeb79cb1954c + - name: prototype-extended-data + revision: 635cf9711577f446b1a7c9d4048bd6a3854eb872 + default: main + tags: + - name: 0.1.0 + revision: 3733a64356b8899b43d4c9145a7c7ca8aa529927 + - name: 0.1.1 + revision: 7ef62b1cf04f1f3c30a57b0138b5d7b50ec076b7 + - name: 0.2.0 + revision: 3cd50945cb736d8aadce5d8194ea746f454b0e55 + - name: 0.3.0 + revision: 74f1bb642800ff6f561d8bcd1aad01fce7e993e4 + - name: 0.4.0 + revision: 4d960baa0725bf805fecd08ad9f60a4f706e4be0 + - name: 0.5.1 + revision: 0422cae6f3d7f46f8667af3fce67b607dca076df + - name: 0.6.0 + revision: ad0a73489e762e83f911deabe18f6764678bb013 + - name: 0.7.0 + revision: 192466458d69530a149311a98ed95efe51dbcf90 + - name: 0.8.0 + revision: 43319554160abb9ebad64614fd1c8d2beaa2296e + - name: 0.9.0 + revision: 069c65f74000ef0f906fc5574023f6d2eaeb6c5b + - name: 0.9.1 + revision: 73a84be9c02d492d6d39bbaaf9df218696f56ab1 + - name: v0.0.1 + revision: b4c04095990e2abc18a925e82c678d50d90f16af + - name: v0.0.2 + revision: 91aa476b8cd63ae2faf4de19d078f95e29b16bd3 + - name: v0.0.3 + revision: cecd0f2752a25f5af7e2568dac0a1b0432a149c6 + - name: v0.0.4 + revision: 9dbca3302925062e42740bccab00051a112b9cf6 + - name: v0.0.5 + revision: 9267fb1125e2efc05c32b4f9250a2cedf395753f + - name: v0.0.6 + revision: 7e6933bded164a917cbb1a510c67d976656263d2 + - name: v0.0.7 + revision: 1e47a197fdf23cd55ba58d4b7b3a2491cb550302 + - name: v0.0.8 + revision: f8a4541891194c07b953ef6af2c251fe40a8987e + - name: v0.0.9 + revision: 422c7d9d9578ccb8d79d8a6b14dac6f6e30bc4f8 + - name: v0.5.0 + revision: fbedf548fc2929250bdcc4252af298658b36b01c diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..7a761b4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,45 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "dfetch_hub" +version = "0.0.1" +authors = [ + { name="p sacharias", email="p.sacharias@gmail.com" }, +] +description = "Dfetch Hub" +readme = "README.md" +requires-python = ">=3.8" +classifiers = [ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", +] +dependencies = [ + "dfetch==0.9.1", + "PyYAML==6.0.2" +] + +[project.optional-dependencies] # https://packaging.python.org/en/latest/guides/writing-pyproject-toml/#writing-pyproject-toml +development = [ + "isort==6.0.0", + "mypy==1.14.1", + "pre-commit==4.1.0", + "pylint==3.3.4", + "pytest==8.3.4", + "pytest-cov==6.0.0", + "vcrpy==7.0.0", +] +gui = [ + "thefuzz==0.22.1", + "nicegui==2.11.1" +] +[project.urls] +Homepage = "https://github.com/dfetch-org/dfetch-hub" +Issues = "https://github.com/dfetch-org/dfetch-hub/issues" + +[project.scripts] +DfetchHub-cli = "dfetch_hub.project.cli:main_cli" + +[tool.pylint.format] +max-line-length = "88" diff --git a/test/test_cli.py b/test/test_cli.py new file mode 100644 index 0000000..2084d35 --- /dev/null +++ b/test/test_cli.py @@ -0,0 +1,33 @@ +"""test cli module""" +import pytest +from test_common import ParserMock +from dfetch_hub.project.cli import main + +@pytest.fixture +def parser_no_args(): + return ParserMock() + + +@pytest.fixture +def parser_url(): + return ParserMock(url="https://github.com/cpputest/") + + +@pytest.fixture +def parser_dfetch(): + return ParserMock(dfetch_source="test/dfetch.yaml") + + +def test_no_parameters(parser_no_args): + with pytest.raises(ValueError): + main(parser_no_args) + + +def test_url_parameter(parser_url): + with pytest.raises(ValueError): + main(parser_url) + + +def test_dfetch_parameter(parser_dfetch): + with pytest.raises(FileNotFoundError): + main(parser_dfetch) diff --git a/test/test_common.py b/test/test_common.py new file mode 100644 index 0000000..29754bf --- /dev/null +++ b/test/test_common.py @@ -0,0 +1,20 @@ +"""common test utilities file""" + + +class Args: + def __init__(self, url=None, dfetch_source=None): + self.url = url + self.dfetch_source = dfetch_source + self.project_exclude_pattern = [] + self.persist_sources = False + + +class ParserMock: + def __init__(self, url=None, dfetch_source=None): + self.args = Args(url, dfetch_source) + + def parse_args(self): + return self.args + + def print_help(self): + print("help") diff --git a/test/test_dfetch_export.py b/test/test_dfetch_export.py new file mode 100644 index 0000000..dfac915 --- /dev/null +++ b/test/test_dfetch_export.py @@ -0,0 +1,51 @@ +"""test dfetch export functionality""" +import os +import pytest + +from dfetch_hub.project.export import DfetchExport +from dataclasses import dataclass + +@dataclass +class EntryMock: + name:str + revision:str + src:str + url:str + repo_path:str + vcs:str="git" + +@pytest.fixture +def entry(): + return EntryMock("test", "0123456789abcdef", "/src", "http://test.test", "test_path") + +@pytest.fixture +def entries(): + entry = EntryMock("test", "0123456789abcdef", "/src", "http://test.test", "test_path") + entry2 = EntryMock("test2", "0123456789abcdef2", "/src2", "http://test.test2", "test_path2") + return [entry, entry2] + +def test_add_entry(entry): + export = DfetchExport() + export.add_entry(entry) + assert len(export.entries) == 1 + assert entry in export.entries + +def test_from_entry(entry): + export = DfetchExport([entry]) + assert len(export.entries) == 1 + assert entry in export.entries + +def test_multiple_entries(entries): + export = DfetchExport() + for entry in entries: + export.add_entry(entry) + assert len(export.entries) == len(entries) + for entry in entries: + assert entry in export.entries + +def test_yaml_file(entries): + export = DfetchExport() + for entry in entries: + export.add_entry(entry) + export.export("test/testdata/dfetch_export.yaml") + assert os.path.exists("test/testdata/dfetch_export.yaml") diff --git a/test/test_input_parser.py b/test/test_input_parser.py new file mode 100644 index 0000000..2e17d99 --- /dev/null +++ b/test/test_input_parser.py @@ -0,0 +1,39 @@ +"""test input parser functionality""" + +import pytest +from test_common import Args + +from dfetch_hub.project.input_parser import InputParser + + +def test_input_url(): + url = "http://www.github.com" + mock = Args(url) + assert [url] == InputParser(mock).get_urls() + + +def test_input_multiple_url(): + url = ["http://www.github.com", "http://www.example.com"] + mock = Args(url) + assert url == InputParser(mock).get_urls() + + +def test_input_dfetch_single_remote(): + dfetch_file_name = "test/testdata/dfetch00.yaml" + mock = Args(dfetch_source=dfetch_file_name) + assert [ + "https://github.com/cpputest/cpputest.git", + "https://github.com/zserge/jsmn.git", + ] == InputParser(mock).get_urls() + + +def test_input_dfetch_multiple_remotes(): + dfetch_file_name = "test/testdata/dfetch01.yaml" + mock = Args(dfetch_source=dfetch_file_name) + urls = InputParser(mock).get_urls() + assert 3 == len(urls) + assert [ + "https://github.com/cpputest/cpputest.git", + "https://github.com/zserge/jsmn.git", + "https://gitlab.com/ShacharKraus/pyfixed", + ] == urls diff --git a/test/test_project_finder.py b/test/test_project_finder.py new file mode 100644 index 0000000..fc83d78 --- /dev/null +++ b/test/test_project_finder.py @@ -0,0 +1,49 @@ +"""test project finder functionality""" + +import pytest + +from dfetch_hub.project.project_finder import GitProjectFinder + + +def test_find_cpputest_github(): + url = "https://github.com/cpputest/cpputest.git" + gpf = GitProjectFinder(url) + projects = gpf.list_projects() + assert len(projects) == 13 + assert "cpputest.git" in projects + assert "examples" in projects + assert "CppUTest" in projects + + +def test_find_jasmine_github(): + url = "https://github.com/zserge/jsmn.git" + gpf = GitProjectFinder(url) + projects = gpf.list_projects() + assert len(projects) == 1 + assert "jsmn.git" in projects + + +def test_find_pyfixed_gitlab(): + url = "https://gitlab.com/ShacharKraus/pyfixed" + gpf = GitProjectFinder(url) + projects = gpf.list_projects() + assert len(projects) == 1 + assert "pyfixed" in projects + +def test_find_cpputest_github_exclusion_filer(): + url = "https://github.com/cpputest/cpputest.git" + exclusions = ["platforms.*", ".*examples.*", "scripts"] + gpf = GitProjectFinder(url, exclusions=exclusions) + projects = gpf.list_projects() + assert len(projects) == 3 + assert "cpputest.git" in projects + assert "CppUTest" in projects + assert "Symbian" in projects + + +def test_find_cpputest_github_invalid_regex(): + url = "https://github.com/cpputest/cpputest.git" + exclusions = ["*examples*"] + with pytest.raises(ValueError): + gpf = GitProjectFinder(url, exclusions=exclusions) + projects = gpf.list_projects() diff --git a/test/test_project_parser.py b/test/test_project_parser.py new file mode 100644 index 0000000..eb04e13 --- /dev/null +++ b/test/test_project_parser.py @@ -0,0 +1,120 @@ +"""test project parser functionality""" + +import pytest +import yaml + +from dfetch_hub.project.project_parser import ProjectParser, RemoteProject + + +def get_sample_project(): + name = "CppUTest" + url = "https://github.com/cpputest/" + src = "" + repo_path = "cpputest.git" + vcs = "git" + return RemoteProject(name, url, repo_path, src, vcs) + + +@pytest.fixture +def project(): + return get_sample_project() + + +@pytest.fixture +def project_w_version(): + project = get_sample_project() + tags = [ + ("aabbccddeeff", "1.0.0"), + ("aaaaaaaaaaaa", "1.1.1"), + ("bbbbbbbbbbbb", "best_tag_ever"), + ] + branches = [ + ("aabbccddaabb", "dev"), + ("bbccddeeffaa", "master"), + ("ccddeeffaabb", "some_old_forgotten_branch"), + ] + project.add_versions(branches, tags) + return project + + +@pytest.fixture +def projects_w_version(): + project_1 = get_sample_project() + project_2 = get_sample_project() + tags1 = [ + ("aabbccddeeff", "1.0.0"), + ("aaaaaaaaaaaa", "1.1.1"), + ("bbbbbbbbbbbb", "best_tag_ever"), + ] + tags2 = [ + ("aabbccddeeff", "2.0.0"), + ("bbddeeffaacc", "2.1.1"), + ("0123456789a", "worst_tag_ever"), + ] + branches = [ + ("aabbccddaabb", "dev"), + ("bbccddeeffaa", "master"), + ("ccddeeffaabb", "some_old_forgotten_branch"), + ] + project_1.add_versions(branches, tags1) + project_2.add_versions(branches, tags2) + return (project_1, project_2) + + +def test_add_project(project): + parser = ProjectParser() + parser.add_project(project) + assert len(parser.get_projects()) == 1 + assert parser.get_projects()[0] == project + + +def test_get_as_yaml(project): + parser = ProjectParser() + parser.add_project(project) + yaml_proj = parser.get_projects_as_yaml() + assert yaml.load(yaml_proj, yaml.Loader) + + +def test_from_yaml(project): + parser = ProjectParser() + yaml_file = "test/testdata/versions.yaml" + parser = ProjectParser.from_yaml(yaml_file) + assert len(parser.get_projects()) == 1 + assert parser.get_projects()[0] == project + + +def test_project_version(project_w_version): + parser = ProjectParser() + project = project_w_version + parser.add_project(project) + assert len(parser.get_projects()) == 1 + assert parser.get_projects()[0] == project + assert "1.0.0" in project.versions.tags + assert "dev" in project.versions.branches + + +def test_two_projects_version(projects_w_version): + parser = ProjectParser() + project1, project2 = projects_w_version + parser.add_project(project1) + parser.add_project(project2) + assert len(parser.get_projects()) == 1 + assert parser.get_projects()[0] == project1 + assert "1.0.0" in project1.versions.tags + assert "dev" in project1.versions.branches + assert "2.0.0" not in project1.versions.tags + assert "2.0.0" in project2.versions.tags + assert "dev" in project2.versions.branches + + +def test_versions_from_yaml(projects_w_version): + parser = ProjectParser() + yaml_file = "test/testdata/versions01.yaml" + parser = ProjectParser.from_yaml(yaml_file) + project1, project2 = projects_w_version + project2.name = "project_2" + project_1_from_file = parser.get_projects()[0] + assert len(parser.get_projects()) == 2 + assert project_1_from_file == project1 + assert parser.get_projects()[1] == project2 + assert project_1_from_file.versions == project1.versions diff --git a/test/test_project_sources.py b/test/test_project_sources.py new file mode 100644 index 0000000..6920eb0 --- /dev/null +++ b/test/test_project_sources.py @@ -0,0 +1,73 @@ +"""test project sources functionality""" + +import pytest + +from dfetch_hub.project.project_sources import RemoteSource, SourceList + + +@pytest.fixture +def input_parser(): + class InputParserMock: + def get_urls(self): + return ["https://github.com/cpputest/cpputest.git"] + + return InputParserMock() + + +def test_add_remote(): + sl = SourceList() + ps = RemoteSource( + {"name": "cpputest", "url-base": "https://github.com/cpputest/cpputest.git"} + ) + sl.add_remote(ps) + assert len(sl.get_remotes()) == 1 + assert sl.get_remotes()[0] == ps + + +def test_add_multiple_remotes(): + sl = SourceList() + ps1 = RemoteSource( + {"name": "cpputest", "url-base": "https://github.com/cpputest/cpputest.git"} + ) + ps2 = RemoteSource( + {"name": "other_repo", "url-base": "https://github.com/cpputest/other_repo.git"} + ) + sl.add_remote(ps1) + sl.add_remote(ps2) + assert len(sl.get_remotes()) == 2 + assert sl.get_remotes() == [ps1, ps2] + + +def test_yaml(): + sl = SourceList() + ps1 = RemoteSource( + {"name": "cpputest", "url-base": "https://github.com/cpputest/cpputest.git"} + ) + ps2 = RemoteSource( + {"name": "other_repo", "url-base": "https://github.com/cpputest/other_repo.git"} + ) + sl.add_remote(ps1) + sl.add_remote(ps2) + sl2 = SourceList.from_yaml(sl.as_yaml()) + assert sl2 == sl + + +def test_input_parser(input_parser): + sl = SourceList.from_input_parser(input_parser) + assert len(sl.get_remotes()) == 1 + ps = RemoteSource( + {"name": "cpputest.git", "url-base": "https://github.com/cpputest/cpputest.git"} + ) + assert ps in sl.get_remotes() + + +def test_yaml_with_exclusions(): + sl = SourceList() + ps = RemoteSource( + {"name": "cpputest", "url-base": "https://github.com/cpputest/cpputest.git"} + ) + ps.add_exclusion("test/.*") + ps.add_exclusion("module*") + sl.add_remote(ps) + sl2 = SourceList.from_yaml(sl.as_yaml()) + assert sl2 == sl diff --git a/test/testdata/dfetch00.yaml b/test/testdata/dfetch00.yaml new file mode 100644 index 0000000..195d971 --- /dev/null +++ b/test/testdata/dfetch00.yaml @@ -0,0 +1,16 @@ +manifest: + version: 0.0 # DFetch Module syntax version + + remotes: # declare common sources in one place + - name: github + url-base: https://github.com/ + + + projects: + - name: cpputest + dst: cpputest/src/ # Destination of this project (relative to this file) + repo-path: cpputest/cpputest.git # Use default github remote + tag: v3.4 # tag + + - name: jsmn # without destination, defaults to project name + repo-path: zserge/jsmn.git # only repo-path is enough diff --git a/test/testdata/dfetch01.yaml b/test/testdata/dfetch01.yaml new file mode 100644 index 0000000..b3c15a9 --- /dev/null +++ b/test/testdata/dfetch01.yaml @@ -0,0 +1,23 @@ +manifest: + version: 0.0 # DFetch Module syntax version + + remotes: # declare common sources in one place + - name: github + url-base: https://github.com/ + + - name: gitlab + url-base: https://gitlab.com/ + + projects: + - name: cpputest + dst: cpputest/src/ # Destination of this project (relative to this file) + repo-path: cpputest/cpputest.git # Use default github remote + tag: v3.4 # tag + + - name: jsmn # without destination, defaults to project name + repo-path: zserge/jsmn.git # only repo-path is enough + + - name: pyfixed + dst: pyfixed + repo-path: ShacharKraus/pyfixed + remote: gitlab \ No newline at end of file diff --git a/test/testdata/dfetch_export.yaml b/test/testdata/dfetch_export.yaml new file mode 100644 index 0000000..ef11d2d --- /dev/null +++ b/test/testdata/dfetch_export.yaml @@ -0,0 +1,15 @@ +manifest: + version: '0.0' + + projects: + - name: test + revision: 0123456789abcdef + src: /src + url: http://test.test + vcs: git + + - name: test2 + revision: 0123456789abcdef2 + src: /src2 + url: http://test.test2 + vcs: git diff --git a/test/testdata/sources00.yaml b/test/testdata/sources00.yaml new file mode 100644 index 0000000..abcf872 --- /dev/null +++ b/test/testdata/sources00.yaml @@ -0,0 +1,3 @@ +sources: +- url: https://github.com/cpputest/cpputest.git +- url: https://github.com/cpputest/other_repo.git diff --git a/test/testdata/versions.yaml b/test/testdata/versions.yaml new file mode 100644 index 0000000..2f4d4b1 --- /dev/null +++ b/test/testdata/versions.yaml @@ -0,0 +1,5 @@ +projects: +- name: CppUTest + repo-path: cpputest.git + url: https://github.com/cpputest/ + vcs: git \ No newline at end of file diff --git a/test/testdata/versions01.yaml b/test/testdata/versions01.yaml new file mode 100644 index 0000000..449e8b1 --- /dev/null +++ b/test/testdata/versions01.yaml @@ -0,0 +1,41 @@ +projects: +- name: CppUTest + repo-path: cpputest.git + url: https://github.com/cpputest/ + vcs: git + versions: + branches: + - revision: aabbccddaabb + name: dev + - revision: bbccddeeffaa + name: master + - revision: ccddeeffaabb + name: some_old_forgotten_branch + default: master + tags: + - revision: aabbccddeeff + name: 1.0.0 + - revision: aaaaaaaaaaaa + name: 1.1.1 + - revision: bbbbbbbbbbbb + name: best_tag_ever +- name: project_2 + repo-path: cpputest.git + url: https://github.com/cpputest/ + vcs: git + versions: + branches: + - revision: aabbccddaabb + name: dev + - revision: bbccddeeffaa + name: master + - revision: ccddeeffaabb + name: some_old_forgotten_branch + default: master + tags: + - revision: aabbccddeeff + name: 2.0.0 + - revision: bbddeeffaacc + name: 2.1.1 + - revision: 0123456789a + name: worst_tag_ever From 37aa1b343bfd079922ab9e3b2da83c377c07854d Mon Sep 17 00:00:00 2001 From: "p.sacharias" Date: Mon, 10 Mar 2025 16:24:17 +0000 Subject: [PATCH 02/19] patch devcontainer for dfetch-hub --- .devcontainer/Dockerfile | 4 ++-- .devcontainer/devcontainer.json | 2 +- dfetch-devcontainer.patch | 29 +++++++++++++++++++++++++++++ dfetch.yaml | 1 + 4 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 dfetch-devcontainer.patch diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 47c3efb..d537212 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -7,10 +7,10 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ subversion=1.14.1-3+deb11u1 && \ rm -rf /var/lib/apt/lists/* -WORKDIR /workspaces/dfetch +WORKDIR /workspaces/dfetch-hub # Add a non-root user (dev) -RUN useradd -m dev && chown -R dev:dev /workspaces/dfetch +RUN useradd -m dev && chown -R dev:dev /workspaces/dfetch-hub USER dev diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3cc8f4d..263377c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -33,6 +33,6 @@ } } }, - "workspaceFolder": "/workspaces/dfetch", + "workspaceFolder": "/workspaces/dfetch-hub", "remoteUser": "dev" } diff --git a/dfetch-devcontainer.patch b/dfetch-devcontainer.patch new file mode 100644 index 0000000..6c4cffe --- /dev/null +++ b/dfetch-devcontainer.patch @@ -0,0 +1,29 @@ +diff --git c/Dockerfile w/Dockerfile +index 47c3efb..d537212 100644 +--- c/Dockerfile ++++ w/Dockerfile +@@ -7,10 +7,10 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ + subversion=1.14.1-3+deb11u1 && \ + rm -rf /var/lib/apt/lists/* + +-WORKDIR /workspaces/dfetch ++WORKDIR /workspaces/dfetch-hub + + # Add a non-root user (dev) +-RUN useradd -m dev && chown -R dev:dev /workspaces/dfetch ++RUN useradd -m dev && chown -R dev:dev /workspaces/dfetch-hub + + USER dev + +diff --git c/devcontainer.json w/devcontainer.json +index 3cc8f4d..263377c 100644 +--- c/devcontainer.json ++++ w/devcontainer.json +@@ -33,6 +33,6 @@ + } + } + }, +- "workspaceFolder": "/workspaces/dfetch", ++ "workspaceFolder": "/workspaces/dfetch-hub", + "remoteUser": "dev" + } diff --git a/dfetch.yaml b/dfetch.yaml index 0d88e2c..356ad84 100644 --- a/dfetch.yaml +++ b/dfetch.yaml @@ -11,3 +11,4 @@ manifest: repo-path: dfetch-org/dfetch src: .devcontainer branch: main + patch: dfetch-devcontainer.patch From 341901792be0b77e2a854dc343e4546eda472cfc Mon Sep 17 00:00:00 2001 From: Ben Spoor <37540691+ben-edna@users.noreply.github.com> Date: Wed, 9 Apr 2025 20:32:27 +0200 Subject: [PATCH 03/19] Update README.md Add codespaces link --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 9be2b37..b5191cb 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,5 @@ # dfetch-hub + +[![Contribute with Codespaces](https://img.shields.io/static/v1?label=Codespaces&message=Open&color=blue)](https://codespaces.new/dfetch-org/dfetch-hub) + Explorer for finding new projects to dfetch From fccd334e4886694ef806f9b5f37d64ef391badd2 Mon Sep 17 00:00:00 2001 From: Ben Spoor <37540691+ben-edna@users.noreply.github.com> Date: Wed, 9 Apr 2025 18:39:08 +0000 Subject: [PATCH 04/19] Auto-format --- dfetch_hub/__init__.py | 2 +- dfetch_hub/example_gui/gui.py | 1 + dfetch_hub/project/cli.py | 4 +--- dfetch_hub/project/export.py | 24 +++++++++++++++++++---- test/test_cli.py | 3 +++ test/test_dfetch_export.py | 37 ++++++++++++++++++++++++----------- test/test_project_finder.py | 1 + 7 files changed, 53 insertions(+), 19 deletions(-) diff --git a/dfetch_hub/__init__.py b/dfetch_hub/__init__.py index 88dd0d9..8e13402 100644 --- a/dfetch_hub/__init__.py +++ b/dfetch_hub/__init__.py @@ -1 +1 @@ -# file required to run gui from project path \ No newline at end of file +# file required to run gui from project path diff --git a/dfetch_hub/example_gui/gui.py b/dfetch_hub/example_gui/gui.py index 5ada823..16b429f 100644 --- a/dfetch_hub/example_gui/gui.py +++ b/dfetch_hub/example_gui/gui.py @@ -1,4 +1,5 @@ """sample of a possible nicegui based gui""" + from nicegui import events, ui from thefuzz import fuzz diff --git a/dfetch_hub/project/cli.py b/dfetch_hub/project/cli.py index 2cc2696..ce0901a 100644 --- a/dfetch_hub/project/cli.py +++ b/dfetch_hub/project/cli.py @@ -1,6 +1,4 @@ -""" - -""" +""" """ import argparse diff --git a/dfetch_hub/project/export.py b/dfetch_hub/project/export.py index 286ba61..6272fbf 100644 --- a/dfetch_hub/project/export.py +++ b/dfetch_hub/project/export.py @@ -1,14 +1,18 @@ """export module""" + from abc import ABC + from dfetch.manifest.manifest import Manifest, ManifestDict from dfetch.manifest.project import ProjectEntry, ProjectEntryDict from dfetch.manifest.remote import Remote, RemoteDict + class Export(ABC): def export(self): pass + class DfetchExport(Export): def __init__(self, entries=None): @@ -25,9 +29,21 @@ def entries(self): return self._entries def export(self, path=None): - remotes = [] # TODO: bundle projects with shared path in remotes - projects = [ProjectEntryDict(name=entry.name, revision=entry.revision, src=entry.src, url=entry.url, repo_path=entry.repo_path, vcs=entry.vcs) for entry in self._entries] - as_dict = ManifestDict(version = Manifest.CURRENT_VERSION, remotes=remotes, projects=projects) + remotes = [] # TODO: bundle projects with shared path in remotes + projects = [ + ProjectEntryDict( + name=entry.name, + revision=entry.revision, + src=entry.src, + url=entry.url, + repo_path=entry.repo_path, + vcs=entry.vcs, + ) + for entry in self._entries + ] + as_dict = ManifestDict( + version=Manifest.CURRENT_VERSION, remotes=remotes, projects=projects + ) if not path: path = "dfetch.yaml" - Manifest(as_dict).dump(path) \ No newline at end of file + Manifest(as_dict).dump(path) diff --git a/test/test_cli.py b/test/test_cli.py index 2084d35..1dd6ffb 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -1,8 +1,11 @@ """test cli module""" + import pytest from test_common import ParserMock + from dfetch_hub.project.cli import main + @pytest.fixture def parser_no_args(): return ParserMock() diff --git a/test/test_dfetch_export.py b/test/test_dfetch_export.py index dfac915..036bee3 100644 --- a/test/test_dfetch_export.py +++ b/test/test_dfetch_export.py @@ -1,40 +1,54 @@ """test dfetch export functionality""" + import os +from dataclasses import dataclass + import pytest from dfetch_hub.project.export import DfetchExport -from dataclasses import dataclass + @dataclass class EntryMock: - name:str - revision:str - src:str - url:str - repo_path:str - vcs:str="git" + name: str + revision: str + src: str + url: str + repo_path: str + vcs: str = "git" + @pytest.fixture def entry(): - return EntryMock("test", "0123456789abcdef", "/src", "http://test.test", "test_path") + return EntryMock( + "test", "0123456789abcdef", "/src", "http://test.test", "test_path" + ) + @pytest.fixture def entries(): - entry = EntryMock("test", "0123456789abcdef", "/src", "http://test.test", "test_path") - entry2 = EntryMock("test2", "0123456789abcdef2", "/src2", "http://test.test2", "test_path2") + entry = EntryMock( + "test", "0123456789abcdef", "/src", "http://test.test", "test_path" + ) + entry2 = EntryMock( + "test2", "0123456789abcdef2", "/src2", "http://test.test2", "test_path2" + ) return [entry, entry2] + def test_add_entry(entry): - export = DfetchExport() + export = DfetchExport() export.add_entry(entry) assert len(export.entries) == 1 assert entry in export.entries + def test_from_entry(entry): export = DfetchExport([entry]) assert len(export.entries) == 1 assert entry in export.entries + def test_multiple_entries(entries): export = DfetchExport() for entry in entries: @@ -43,6 +57,7 @@ def test_multiple_entries(entries): for entry in entries: assert entry in export.entries + def test_yaml_file(entries): export = DfetchExport() for entry in entries: diff --git a/test/test_project_finder.py b/test/test_project_finder.py index fc83d78..6f46a81 100644 --- a/test/test_project_finder.py +++ b/test/test_project_finder.py @@ -30,6 +30,7 @@ def test_find_pyfixed_gitlab(): assert len(projects) == 1 assert "pyfixed" in projects + def test_find_cpputest_github_exclusion_filer(): url = "https://github.com/cpputest/cpputest.git" exclusions = ["platforms.*", ".*examples.*", "scripts"] From 03e1710bdcdda031ab054180cff3a493fbcd3712 Mon Sep 17 00:00:00 2001 From: Ben Spoor <37540691+ben-edna@users.noreply.github.com> Date: Wed, 9 Apr 2025 18:55:07 +0000 Subject: [PATCH 05/19] Install gui deps in devcontainer --- .devcontainer/.dfetch_data.yaml | 8 ++++---- .devcontainer/Dockerfile | 6 ++++-- dfetch-devcontainer.patch | 23 ++++++++++++++++------- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/.devcontainer/.dfetch_data.yaml b/.devcontainer/.dfetch_data.yaml index 9ef1aec..a81fba6 100644 --- a/.devcontainer/.dfetch_data.yaml +++ b/.devcontainer/.dfetch_data.yaml @@ -2,9 +2,9 @@ # For more info see https://dfetch.rtfd.io/en/latest/getting_started.html dfetch: branch: main - hash: 9bb1320bd8367d6a33ecc4150e319108 - last_fetch: 10/03/2025, 16:16:46 - patch: '' + hash: f72611479707b1b327416d294585fb5a + last_fetch: 09/04/2025, 18:59:35 + patch: dfetch-devcontainer.patch remote_url: https://github.com/dfetch-org/dfetch - revision: c037c000e200704f3db1ac6af1aceeb79cb1954c + revision: 89e8c77621e62b6ef657c4358b350a959c82af16 tag: '' diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index d537212..4d68823 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -15,11 +15,13 @@ RUN useradd -m dev && chown -R dev:dev /workspaces/dfetch-hub USER dev ENV PATH="/home/dev/.local/bin:${PATH}" +ENV PYTHONPATH="/home/dev/.local/lib/python3.12" +ENV PYTHONUSERBASE="/home/dev/.local" COPY --chown=dev:dev . . -RUN pip install --no-cache-dir --root-user-action=ignore --upgrade pip==24.3.1 \ - && pip install --no-cache-dir --root-user-action=ignore -e .[development,docs,test,casts] \ +RUN pip install --no-cache-dir --root-user-action=ignore --upgrade pip==25.0.1 \ + && pip install --no-cache-dir --root-user-action=ignore -e .[gui] \ && pre-commit install --install-hooks # Set bash as the default shell diff --git a/dfetch-devcontainer.patch b/dfetch-devcontainer.patch index 6c4cffe..5d5ff84 100644 --- a/dfetch-devcontainer.patch +++ b/dfetch-devcontainer.patch @@ -1,7 +1,7 @@ -diff --git c/Dockerfile w/Dockerfile -index 47c3efb..d537212 100644 ---- c/Dockerfile -+++ w/Dockerfile +diff --git a/Dockerfile b/Dockerfile +index b3e8653..4d68823 100644 +--- a/Dockerfile ++++ b/Dockerfile @@ -7,10 +7,10 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ subversion=1.14.1-3+deb11u1 && \ rm -rf /var/lib/apt/lists/* @@ -15,10 +15,19 @@ index 47c3efb..d537212 100644 USER dev -diff --git c/devcontainer.json w/devcontainer.json +@@ -21,7 +21,7 @@ ENV PYTHONUSERBASE="/home/dev/.local" + COPY --chown=dev:dev . . + + RUN pip install --no-cache-dir --root-user-action=ignore --upgrade pip==25.0.1 \ +- && pip install --no-cache-dir --root-user-action=ignore -e .[development,docs,test,casts] \ ++ && pip install --no-cache-dir --root-user-action=ignore -e .[gui] \ + && pre-commit install --install-hooks + + # Set bash as the default shell +diff --git a/devcontainer.json b/devcontainer.json index 3cc8f4d..263377c 100644 ---- c/devcontainer.json -+++ w/devcontainer.json +--- a/devcontainer.json ++++ b/devcontainer.json @@ -33,6 +33,6 @@ } } From d13104b3b4285584b567ea1f6e98b40f2b790f5b Mon Sep 17 00:00:00 2001 From: Ben Spoor <37540691+ben-edna@users.noreply.github.com> Date: Wed, 9 Apr 2025 19:00:28 +0000 Subject: [PATCH 06/19] Fix main cli --- dfetch_hub/project/cli.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dfetch_hub/project/cli.py b/dfetch_hub/project/cli.py index ce0901a..55f54ac 100644 --- a/dfetch_hub/project/cli.py +++ b/dfetch_hub/project/cli.py @@ -32,7 +32,7 @@ def main(parser: argparse.ArgumentParser): datasource.write(parser.get_projects_as_yaml()) -if __name__ == "__main__": +def main_cli(): arg_parser = argparse.ArgumentParser() arg_parser.add_argument("-u", "--url", required=False, nargs="+") arg_parser.add_argument("-ds", "--dfetch-source", required=False) @@ -43,3 +43,7 @@ def main(parser: argparse.ArgumentParser): "-ps", "--persist-sources", required=False, action="store_true" ) main(arg_parser) + + +if __name__ == "__main__": + main_cli() From 932357dbe7688c764735d5c93f7206f6d7079a3f Mon Sep 17 00:00:00 2001 From: Ben Spoor <37540691+ben-edna@users.noreply.github.com> Date: Wed, 9 Apr 2025 19:03:02 +0000 Subject: [PATCH 07/19] Add code-workspace --- dfetch-hub.code-workspace | 142 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 dfetch-hub.code-workspace diff --git a/dfetch-hub.code-workspace b/dfetch-hub.code-workspace new file mode 100644 index 0000000..5d866c3 --- /dev/null +++ b/dfetch-hub.code-workspace @@ -0,0 +1,142 @@ +{ + "folders": [ + { + "path": "." + } + ], + "settings": { + "[windows]": { + "python.defaultInterpreterPath": "${workspaceFolder}/venv/Scripts/python", + }, + "editor.trimAutoWhitespace": true, + "editor.codeActionsOnSave": { + "source.organizeImports": "explicit" + }, + "isort.check": true, + "restructuredtext.linter.run": "onType", + "restructuredtext.linter.doc8.extraArgs": [ + "--config", + "${workspaceFolder}/pyproject.toml" + ], + "python.testing.pytestArgs": [ + "tests" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "python.testing.autoTestDiscoverOnSaveEnabled": true, + "cucumberautocomplete.steps": [ + "features/steps/*.py" + ], + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter", + "editor.formatOnSave": true, + }, + }, + "launch": { + "version": "0.2.0", + "configurations": [ + { + "name": "DFetch hub gui", + "type": "debugpy", + "request": "launch", + "module": "dfetch_hub.example_gui.gui", + "justMyCode": false, + "args": [ + ] + }, + { + "name": "DFetch hub cli", + "type": "debugpy", + "request": "launch", + "module": "dfetch_hub.project.cli", + "justMyCode": false, + "args": ["-u", "https://github.com/dfetch-org/dfetch.git" + ] + } + ] + }, + "tasks": { + "version": "2.0.0", + "tasks": [ + { + "label": "Build Docs", + "type": "shell", + "linux": { + "command": "make" + }, + "windows": { + "command": "make.bat" + }, + "args": [ + "html" + ], + "options": { + "cwd": "${workspaceFolder}/doc" + }, + "group": { + "kind": "build", + "isDefault": true + }, + "presentation": { + "reveal": "always", + "panel": "shared" + } + }, + { + "label": "Check quality (pre-commit)", + "type": "shell", + "command": "pre-commit", + "args": [ + "run", + "--all-files" + ], + "options": { + "cwd": "${workspaceFolder}" + }, + "group": { + "kind": "test", + "isDefault": false + }, + "presentation": { + "reveal": "always", + "panel": "shared" + } + }, + { + "label": "Run All Unit Tests", + "type": "shell", + "command": "python", + "args": [ + "-m", + "pytest", + "test" + ], + "options": { + "cwd": "${workspaceFolder}" + }, + "group": { + "kind": "test", + "isDefault": false + }, + "presentation": { + "reveal": "always", + "panel": "shared" + } + }, + ] + }, + "extensions": { + "recommendations": [ + "bungcip.better-toml", + "jebbs.plantuml", + "lextudio.restructuredtext", + "ms-python.black-formatter", + "ms-python.debugpy", + "ms-python.isort", + "ms-python.pylint", + "ms-python.python", + "trond-snekvik.simple-rst", + "jimasp.behave-vsc" + ] + } +} \ No newline at end of file From 4977a4ad24a7972b6ec42a06572b56da2107c78a Mon Sep 17 00:00:00 2001 From: Ben Spoor <37540691+ben-edna@users.noreply.github.com> Date: Wed, 9 Apr 2025 19:06:23 +0000 Subject: [PATCH 08/19] Add some basic info to README --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index b5191cb..cb7677b 100644 --- a/README.md +++ b/README.md @@ -3,3 +3,23 @@ [![Contribute with Codespaces](https://img.shields.io/static/v1?label=Codespaces&message=Open&color=blue)](https://codespaces.new/dfetch-org/dfetch-hub) Explorer for finding new projects to dfetch + +## Setup + +```console +pip install -e .[gui] +``` + +## Basic usage + +### CLI + +```console +DfetchHub-cli -u "https://github.com/dfetch-org/dfetch.git" +``` + +### Gui + +```console +python -m "dfetch_hub.example_gui.gui" +``` From a7cf1eacb8dd707b7aad9b82d74d5218ea0dc0e6 Mon Sep 17 00:00:00 2001 From: Ben Spoor <37540691+ben-edna@users.noreply.github.com> Date: Wed, 9 Apr 2025 19:13:53 +0000 Subject: [PATCH 09/19] Also install development dependencies --- .devcontainer/.dfetch_data.yaml | 4 ++-- .devcontainer/Dockerfile | 2 +- dfetch-devcontainer.patch | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.devcontainer/.dfetch_data.yaml b/.devcontainer/.dfetch_data.yaml index a81fba6..95e5105 100644 --- a/.devcontainer/.dfetch_data.yaml +++ b/.devcontainer/.dfetch_data.yaml @@ -2,8 +2,8 @@ # For more info see https://dfetch.rtfd.io/en/latest/getting_started.html dfetch: branch: main - hash: f72611479707b1b327416d294585fb5a - last_fetch: 09/04/2025, 18:59:35 + hash: f2a3ac0bd99186f66dbca1f5c37b09f8 + last_fetch: 09/04/2025, 19:11:43 patch: dfetch-devcontainer.patch remote_url: https://github.com/dfetch-org/dfetch revision: 89e8c77621e62b6ef657c4358b350a959c82af16 diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 4d68823..0a1ea92 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -21,7 +21,7 @@ ENV PYTHONUSERBASE="/home/dev/.local" COPY --chown=dev:dev . . RUN pip install --no-cache-dir --root-user-action=ignore --upgrade pip==25.0.1 \ - && pip install --no-cache-dir --root-user-action=ignore -e .[gui] \ + && pip install --no-cache-dir --root-user-action=ignore -e .[development,gui] \ && pre-commit install --install-hooks # Set bash as the default shell diff --git a/dfetch-devcontainer.patch b/dfetch-devcontainer.patch index 5d5ff84..0ad94ba 100644 --- a/dfetch-devcontainer.patch +++ b/dfetch-devcontainer.patch @@ -20,7 +20,7 @@ index b3e8653..4d68823 100644 RUN pip install --no-cache-dir --root-user-action=ignore --upgrade pip==25.0.1 \ - && pip install --no-cache-dir --root-user-action=ignore -e .[development,docs,test,casts] \ -+ && pip install --no-cache-dir --root-user-action=ignore -e .[gui] \ ++ && pip install --no-cache-dir --root-user-action=ignore -e .[development,gui] \ && pre-commit install --install-hooks # Set bash as the default shell From 18dfb2dfbc442f143e3e0ec986710263ce7a0d4f Mon Sep 17 00:00:00 2001 From: Ben Spoor <37540691+ben-edna@users.noreply.github.com> Date: Wed, 9 Apr 2025 19:14:06 +0000 Subject: [PATCH 10/19] Upgrade dfetch --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7a761b4..0152afd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ - "dfetch==0.9.1", + "dfetch==0.10.0", "PyYAML==6.0.2" ] From 0dbd8ec4e2cd97630352fe821a404d8f546e7205 Mon Sep 17 00:00:00 2001 From: Ben Spoor <37540691+ben-edna@users.noreply.github.com> Date: Wed, 9 Apr 2025 19:14:20 +0000 Subject: [PATCH 11/19] Add first workflow --- .github/workflows/test.yml | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..6ef56d7 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,40 @@ +name: Test + +on: + push: + branches: + - main + pull_request: + types: [opened, synchronize, reopened] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + pip install .[gui,development] + + # - run: codespell # Check for typo's + - run: isort --diff dfetch_hub # Checks import order + - run: black --check dfetch_hub # Checks code style + # - run: flake8 dfetch_hub # Checks pep8 conformance + - run: pylint dfetch_hub # Checks pep8 conformance + # - run: ruff check dfetch # Check using ruff + - run: mypy --strict dfetch_hub # Check types + # - run: pyright . # Check types + # - run: doc8 doc # Checks documentation + # - run: pydocstyle dfetch # Checks doc strings + # - run: bandit -r dfetch # Checks security issues + # - run: xenon -b B -m A -a A dfetch # Check code quality + - run: pytest --cov=dfetch_hub test # Run tests + # - run: coverage run --source=dfetch --append -m behave features # Run features tests + # - run: coverage xml -o coverage.xml # Create XML report + # - run: pyroma --directory --min=10 . # Check pyproject From 081880cc1e800e47965ac9faa1a665af80d89ae5 Mon Sep 17 00:00:00 2001 From: Ben Spoor <37540691+ben-edna@users.noreply.github.com> Date: Wed, 9 Apr 2025 19:15:18 +0000 Subject: [PATCH 12/19] Add dependabot config --- .github/dependabot.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..f268356 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,24 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" + # Raise pull requests for version updates + # to pip against the `develop` branch + target-branch: "dev" + # Labels on pull requests for version updates only + labels: + - "dependencies" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + - package-ecosystem: "devcontainers" + directory: "/" + schedule: + interval: "daily" + - package-ecosystem: "docker" + directory: "/.devcontainer" + schedule: + interval: "daily" \ No newline at end of file From 9bc8b805ea70cc11954032291a39e5e4e7d4a609 Mon Sep 17 00:00:00 2001 From: Ben Spoor <37540691+ben-edna@users.noreply.github.com> Date: Wed, 9 Apr 2025 19:18:22 +0000 Subject: [PATCH 13/19] Add black --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 0152afd..737c01c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ [project.optional-dependencies] # https://packaging.python.org/en/latest/guides/writing-pyproject-toml/#writing-pyproject-toml development = [ + "black==25.1.0", "isort==6.0.0", "mypy==1.14.1", "pre-commit==4.1.0", From c8b1db92e66ef00bec00000c3740bf41f8f42757 Mon Sep 17 00:00:00 2001 From: Ben Spoor <37540691+ben-edna@users.noreply.github.com> Date: Wed, 9 Apr 2025 20:28:20 +0000 Subject: [PATCH 14/19] Fix mypy issues --- dfetch_hub/example_gui/gui.py | 45 ++++++++++++---------- dfetch_hub/project/cli.py | 10 ++--- dfetch_hub/project/export.py | 28 ++++++++++---- dfetch_hub/project/input_parser.py | 5 ++- dfetch_hub/project/project_finder.py | 33 ++++++++-------- dfetch_hub/project/project_parser.py | 14 +++---- dfetch_hub/project/project_sources.py | 50 ++++++++++++++----------- dfetch_hub/project/remote_datasource.py | 42 ++++++++++++--------- pyproject.toml | 10 +++++ 9 files changed, 142 insertions(+), 95 deletions(-) diff --git a/dfetch_hub/example_gui/gui.py b/dfetch_hub/example_gui/gui.py index 16b429f..ff3b5e5 100644 --- a/dfetch_hub/example_gui/gui.py +++ b/dfetch_hub/example_gui/gui.py @@ -1,13 +1,16 @@ """sample of a possible nicegui based gui""" +from typing import Optional, Sequence + from nicegui import events, ui from thefuzz import fuzz -from dfetch_hub.project.project_finder import GitProjectFinder +from dfetch_hub.project.project_finder import GitProjectFinder, ProjectFinder from dfetch_hub.project.project_sources import RemoteSource, SourceList +from dfetch_hub.project.remote_datasource import RemoteProject, RemoteRef -def main(): +def main() -> None: """main gui runner""" ui.context.sl = SourceList() ui.context.pf = [] @@ -19,7 +22,7 @@ def main(): ui.run(title="dfetch project viewer", reconnect_timeout=30) -def header(): +def header() -> None: """main gui header""" with ui.header().classes("bg-black text-white p-4"): with ui.row().classes( @@ -37,7 +40,7 @@ def header(): @ui.page("/sources") -def sources_page(): +def sources_page() -> None: """page to enter sources to search""" header() with ui.column().classes("w-1/3 mx-auto mt-10"): @@ -46,7 +49,7 @@ def sources_page(): sources_input() -def add_projects_to_page(): +def add_projects_to_page() -> None: """add list of project finder results to page""" if not ui.context.pf: ui.context.pf = [] @@ -66,7 +69,9 @@ def add_projects_to_page(): ui.notification(f"{e}") -def add_project_finder_to_page(pf, projects=None): +def add_project_finder_to_page( + pf: ProjectFinder, projects: Optional[Sequence[RemoteProject]] = None +) -> None: """add single project finder result to page""" if not projects: projects = pf.list_projects() @@ -79,7 +84,7 @@ def add_project_finder_to_page(pf, projects=None): add_project_to_page(project) -def add_project_to_page(project): +def add_project_to_page(project: RemoteProject) -> None: """add single project to page""" with ui.card().classes( "bg-black text-white p-6 rounded shadow-lg \ @@ -91,7 +96,7 @@ def add_project_to_page(project): @ui.page("/projects/") -def projects_page(): +def projects_page() -> None: """projects for source""" header() search_input = ui.input(placeholder="Search packages").classes("flex-grow") @@ -105,7 +110,7 @@ def projects_page(): ui.navigate.to("/sources") -def update_autocomplete(value): +def update_autocomplete(value: str) -> None: """autocomplete for project search""" ui.context.project_col.clear() with ui.context.project_col: @@ -141,7 +146,7 @@ def update_autocomplete(value): @ui.page("/project_data/{name}") -def projects_data_page(name: str): +def projects_data_page(name: str) -> None: """data for project""" header() with ui.column().classes("w-5/6 items-center mx-auto mt-10"): @@ -163,7 +168,7 @@ def projects_data_page(name: str): @ui.page("/filters") -def filters_page(): +def filters_page() -> None: """page showing exclusions per source""" header() ui.notify("no sources present, redirecting to sources") @@ -192,7 +197,7 @@ def filters_page(): ui.navigate.to("/sources") -def add_exclusion(pf, regex): +def add_exclusion(pf: ProjectFinder, regex: str) -> None: """add exclusion for the project finder for a source""" ui.notify(f"adding exclusion {regex} to projects on url {pf.url}") project = [ @@ -203,13 +208,13 @@ def add_exclusion(pf, regex): pf.filter_projects() -def presist_sources(): +def presist_sources() -> None: """persist entered sources to file""" sl = ui.context.sl ui.download(sl.as_yaml().encode("utf-8"), filename="sources.yaml") -def url_input(): +def url_input() -> None: """url input page""" url_search_field = ui.input(placeholder="enter url to list packages").classes( "w-full p-2 text-lg border border-gray-300 rounded" @@ -219,20 +224,20 @@ def url_input(): ).classes("bg-black text-white px-4 py-2 rounded hover:bg-gray-800 mt-4") -def sources_input(): +def sources_input() -> None: """input sources file""" ui.upload( on_upload=lambda e: handle_upload(e) # pylint:disable = unnecessary-lambda ).props("accept=.yaml").classes("max-w-full") -def handle_upload(file: events.UploadEventArguments): +def handle_upload(file: events.UploadEventArguments) -> None: """handle upload of sources file""" ui.context.sl = SourceList.from_yaml(file.content.read()) ui.notify(f"uploaded {file.name}") -def get_projects(url): +def get_projects(url: str) -> None: """handling of project search""" if url and len(url) > 5: # what is min valid url len? name = url.split("/")[-1] @@ -240,7 +245,7 @@ def get_projects(url): ui.navigate.to("/projects/") -def project_representation(project): +def project_representation(project: RemoteProject) -> None: """project representation""" ui.label(project.name).classes("text-h5 text-black mb-5") @@ -276,14 +281,14 @@ def project_representation(project): pass # No content here (empty) -def revision_representation(rev): +def revision_representation(rev: RemoteRef) -> None: """revision representation""" ui.label(f"revision {rev.name} - {rev.revision}").classes( "text-body2 text-black mb-2" ) -def show_sources(): +def show_sources() -> None: """show sources in source view""" if hasattr(ui.context, "pf"): for pf in ui.context.pf: diff --git a/dfetch_hub/project/cli.py b/dfetch_hub/project/cli.py index 55f54ac..3dd8fd6 100644 --- a/dfetch_hub/project/cli.py +++ b/dfetch_hub/project/cli.py @@ -10,7 +10,7 @@ # from project.cli_disp import CliDisp -def main(parser: argparse.ArgumentParser): +def main(parser: argparse.ArgumentParser) -> None: """main command line interface for program""" args = parser.parse_args() if not args.url and not args.dfetch_source: @@ -22,17 +22,17 @@ def main(parser: argparse.ArgumentParser): sources_list = SourceList.from_input_parser(input_args_parser) with open("sources.yaml", "w", encoding="utf-8") as sources_file: sources_file.write(sources_list.as_yaml()) - parser = ProjectParser() + project_parser = ProjectParser() for url in url_list: gpf = GitProjectFinder(url, args.project_exclude_pattern) projects = gpf.list_projects() for project in projects: - parser.add_project(project) + project_parser.add_project(project) with open("projects.yaml", "w", encoding="utf-8") as datasource: - datasource.write(parser.get_projects_as_yaml()) + datasource.write(project_parser.get_projects_as_yaml()) -def main_cli(): +def main_cli() -> None: arg_parser = argparse.ArgumentParser() arg_parser.add_argument("-u", "--url", required=False, nargs="+") arg_parser.add_argument("-ds", "--dfetch-source", required=False) diff --git a/dfetch_hub/project/export.py b/dfetch_hub/project/export.py index 6272fbf..47c4879 100644 --- a/dfetch_hub/project/export.py +++ b/dfetch_hub/project/export.py @@ -1,35 +1,49 @@ """export module""" from abc import ABC +from dataclasses import dataclass +from typing import List, Optional, Sequence, Union from dfetch.manifest.manifest import Manifest, ManifestDict -from dfetch.manifest.project import ProjectEntry, ProjectEntryDict +from dfetch.manifest.project import ProjectEntryDict from dfetch.manifest.remote import Remote, RemoteDict +@dataclass +class Entry: + name: str + revision: str + src: str + url: str + repo_path: str + vcs: str = "git" + + class Export(ABC): - def export(self): + def export(self) -> None: pass class DfetchExport(Export): - def __init__(self, entries=None): + def __init__(self, entries: Optional[List[Entry]] = None): if entries: self._entries = entries else: self._entries = [] - def add_entry(self, entry): + def add_entry(self, entry: Entry) -> None: self._entries += [entry] @property - def entries(self): + def entries(self) -> List[Entry]: return self._entries - def export(self, path=None): - remotes = [] # TODO: bundle projects with shared path in remotes + def export(self, path: str = "") -> None: + remotes: Sequence[Union[RemoteDict, Remote]] = ( + [] + ) # TODO: bundle projects with shared path in remotes projects = [ ProjectEntryDict( name=entry.name, diff --git a/dfetch_hub/project/input_parser.py b/dfetch_hub/project/input_parser.py index 57120ae..740ee06 100644 --- a/dfetch_hub/project/input_parser.py +++ b/dfetch_hub/project/input_parser.py @@ -1,5 +1,6 @@ """input parser module""" +from argparse import Namespace from typing import Sequence from dfetch.manifest.manifest import Manifest @@ -8,7 +9,7 @@ class InputParser: # pylint:disable=too-few-public-methods """parser for url or dfetch file input""" - def __init__(self, args): + def __init__(self, args: Namespace): self.args = args def get_urls(self) -> Sequence[str]: @@ -19,6 +20,6 @@ def get_urls(self) -> Sequence[str]: return [self.args.url] return self._parse_dfetch_remotes(self.args.dfetch_source) - def _parse_dfetch_remotes(self, dfetch_path) -> Sequence[str]: + def _parse_dfetch_remotes(self, dfetch_path: str) -> Sequence[str]: manifest = Manifest.from_file(dfetch_path) return [project.remote_url for project in manifest.projects] diff --git a/dfetch_hub/project/project_finder.py b/dfetch_hub/project/project_finder.py index 2ad65bb..7ec390f 100644 --- a/dfetch_hub/project/project_finder.py +++ b/dfetch_hub/project/project_finder.py @@ -6,7 +6,7 @@ import sys from abc import abstractmethod from contextlib import chdir -from typing import Optional, Sequence +from typing import List, Optional, Sequence, Set, Tuple, Union from dfetch.util.cmdline import SubprocessCommandError, run_on_cmdline @@ -18,23 +18,23 @@ class ProjectFinder: """class to find projects in repositories""" - def __init__(self, url: str, exclusions: Optional[Sequence[str]] = None): + def __init__(self, url: str, exclusions: Optional[List[str]] = None): self._url = url self._logger = logging.getLogger() - self._projects: list[str] = [] - self._exclusions = exclusions + self._projects: list[RemoteProject] = [] + self._exclusions: List[str] = exclusions or [] @property - def url(self): + def url(self) -> str: """repo url""" return self._url @abstractmethod - def list_projects(self): + def list_projects(self) -> list[RemoteProject]: """list all projects in a repo""" raise AssertionError("abstractmethod") - def filter_exclusions(self, paths: Sequence[str]): + def filter_exclusions(self, paths: Union[List[str], Set[str]]) -> List[str]: """filter exclusions from list of projects""" filtered_paths = [] for path in paths: @@ -53,15 +53,15 @@ def filter_exclusions(self, paths: Sequence[str]): if path_allowed and path not in filtered_paths: filtered_paths += [path] else: - filtered_paths = paths + filtered_paths = list(paths) return filtered_paths @property - def exclusions(self): + def exclusions(self) -> Optional[List[str]]: """get exclusion for project finder""" return self._exclusions - def add_exclusion(self, exclusion): + def add_exclusion(self, exclusion: Optional[str]) -> None: """add an exclusion regex""" if exclusion: if not self._exclusions: @@ -69,7 +69,7 @@ def add_exclusion(self, exclusion): self._exclusions += [exclusion] print(f"exclusions are {self.exclusions}") - def filter_projects(self): + def filter_projects(self) -> None: """filter projects on exclusions""" for project in self._projects: path = f"{project.url, project.repo_path, project.src}" @@ -92,7 +92,7 @@ def filter_projects(self): class GitProjectFinder(ProjectFinder): """git implementation of project finder""" - def list_projects(self): + def list_projects(self) -> List[RemoteProject]: """list all git projects in a git repo""" if not self._projects: if os.path.exists(WORKDIR) and os.path.isdir(WORKDIR): @@ -144,8 +144,11 @@ def list_projects(self): return self._projects def _projects_from_paths( - self, paths: Sequence[str], branches=Sequence[str], tags=Sequence[str] - ): + self, + paths: Sequence[str], + branches: Sequence[Tuple[str, str]], + tags: Sequence[Tuple[str, str]], + ) -> List[RemoteProject]: projects = [] for path in paths: if "/" in path: @@ -164,7 +167,7 @@ def _projects_from_paths( return projects -def _base_url(url): +def _base_url(url: str) -> Tuple[str, str]: if "://" in url: url = url.split("://", maxsplit=1)[1] if "/" in url: diff --git a/dfetch_hub/project/project_parser.py b/dfetch_hub/project/project_parser.py index 176def5..17dff0d 100644 --- a/dfetch_hub/project/project_parser.py +++ b/dfetch_hub/project/project_parser.py @@ -1,6 +1,6 @@ """project parser module""" -from typing import List +from typing import Any, Dict, List import yaml @@ -14,29 +14,29 @@ class ProjectParser: - parsed into sources which can be stored and monitored """ - def __init__(self): + def __init__(self) -> None: self._projects: List[RemoteProject] = [] - def add_project(self, new_project: RemoteProject): + def add_project(self, new_project: RemoteProject) -> None: """add a project""" if new_project not in self._projects: self._projects += [new_project] - def get_projects(self): + def get_projects(self) -> List[RemoteProject]: """get all projects""" return self._projects - def get_projects_as_yaml(self): + def get_projects_as_yaml(self) -> str: """get yaml representation of projects""" yaml_str = "" - yaml_obj = {"projects": []} + yaml_obj: Dict[str, Any] = {"projects": []} for project in self._projects: yaml_obj["projects"] += [project.as_yaml()] yaml_str = yaml.dump(yaml_obj) return yaml_str @classmethod - def from_yaml(cls, yaml_file): + def from_yaml(cls, yaml_file: str) -> "ProjectParser": """create parser from yaml file""" with open(yaml_file, "r", encoding="utf-8") as yamlf: instance = cls() diff --git a/dfetch_hub/project/project_sources.py b/dfetch_hub/project/project_sources.py index 58ef314..4f91d72 100644 --- a/dfetch_hub/project/project_sources.py +++ b/dfetch_hub/project/project_sources.py @@ -1,6 +1,7 @@ """project sources module""" -from typing import Optional, Sequence +from argparse import Namespace +from typing import Any, Dict, List, Optional, Union import yaml from dfetch.manifest.remote import Remote @@ -8,27 +9,27 @@ from dfetch_hub.project.input_parser import InputParser -class RemoteSource(Remote): +class RemoteSource(Remote): # type: ignore """class representing source for projects""" - def __init__(self, args): + def __init__(self, args: Union[Namespace, Dict[str, str]]): super().__init__(args) - self.exclusions: Optional[Sequence] = None + self.exclusions: List[str] = [] - def add_exclusion(self, exclusion_regex: str): + def add_exclusion(self, exclusion_regex: str) -> None: """add exclusion to project source""" if not self.exclusions: self.exclusions = [exclusion_regex] else: self.exclusions += [exclusion_regex] - def as_yaml(self): + def as_yaml(self) -> Dict[str, Any]: """get yaml representation""" yaml_data = super().as_yaml() yaml_data["exclusions"] = self.exclusions return {k: v for k, v in yaml_data.items() if v} - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: if isinstance(other, RemoteSource): if hasattr(self, "exclusions"): if not hasattr(other, "exclusions"): @@ -40,7 +41,7 @@ def __eq__(self, other): ) if hasattr(other, "exclusions"): return False - return self.name == other.name and self.url == other.url + return bool(self.name == other.name and self.url == other.url) return False @@ -49,18 +50,18 @@ class SourceList: CURRENT_VERSION = "0.0" - def __init__(self): - self._sources: Sequence[RemoteSource] = [] + def __init__(self) -> None: + self._sources: List[RemoteSource] = [] - def add_remote(self, source: RemoteSource): + def add_remote(self, source: RemoteSource) -> None: """add source""" self._sources += [source] - def get_remotes(self) -> list[RemoteSource]: + def get_remotes(self) -> List[RemoteSource]: """get list of sources""" return self._sources - def as_yaml(self): + def as_yaml(self) -> str: """yaml representation""" versiondata = {"version": self.CURRENT_VERSION} remotes_data = {"remotes": [source.as_yaml() for source in self._sources]} @@ -68,16 +69,23 @@ def as_yaml(self): return yaml.dump(yamldata) @classmethod - def from_yaml(cls, yaml_data): + def from_yaml(cls, yaml_data: Union[str, bytes]) -> "SourceList": """load from sources files""" if not yaml_data: raise ValueError("failed to load data from file") + instance = cls() - yaml_data = yaml.load(yaml_data, Loader=yaml.Loader) - assert yaml_data, "file should have data" - assert yaml_data["source-list"], "file should have list of sources" - version = [i["version"] for i in yaml_data["source-list"] if "version" in i][0] - remotes = [i["remotes"] for i in yaml_data["source-list"] if "remotes" in i][0] + + parsed_yaml: Optional[Dict[str, Any]] = yaml.load(yaml_data, Loader=yaml.Loader) + if not parsed_yaml: + raise RuntimeError("file should have data") + assert parsed_yaml["source-list"], "file should have list of sources" + version = [i["version"] for i in parsed_yaml["source-list"] if "version" in i][ + 0 + ] + remotes = [i["remotes"] for i in parsed_yaml["source-list"] if "remotes" in i][ + 0 + ] if version != cls.CURRENT_VERSION: raise ValueError("invalid version") @@ -90,7 +98,7 @@ def from_yaml(cls, yaml_data): return instance @classmethod - def from_input_parser(cls, parser: InputParser): + def from_input_parser(cls, parser: InputParser) -> "SourceList": """generate instance from parser""" instance = cls() for url in parser.get_urls(): @@ -99,7 +107,7 @@ def from_input_parser(cls, parser: InputParser): instance.add_remote(src) return instance - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: if isinstance(other, SourceList): return ( self._sources == other._sources diff --git a/dfetch_hub/project/remote_datasource.py b/dfetch_hub/project/remote_datasource.py index 40bbb99..948ff50 100644 --- a/dfetch_hub/project/remote_datasource.py +++ b/dfetch_hub/project/remote_datasource.py @@ -1,7 +1,7 @@ """remote datasource module""" from dataclasses import dataclass -from typing import Sequence, Tuple +from typing import Any, Dict, List, Optional, Sequence, Tuple @dataclass @@ -11,12 +11,12 @@ class RemoteRef: name: str revision: str # chosen for dfetch naming - def as_yaml(self): + def as_yaml(self) -> Dict[str, str]: """yaml representation of reference""" yamldata = {"name": self.name, "revision": self.revision} return {k: v for k, v in yamldata.items() if v} - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: if isinstance(other, str): return self.name == other if isinstance(other, RemoteRef): @@ -27,25 +27,25 @@ def __eq__(self, other): class RemoteProjectVersions: """representation of collection of versions for project""" - def __init__(self, vcs=None): - self.tags = [] - self.branches = [] + def __init__(self, vcs: str = ""): + self.tags: List[RemoteRef] = [] + self.branches: List[RemoteRef] = [] self.vcs = vcs - def add_tags(self, tags): + def add_tags(self, tags: Sequence[Tuple[str, str]]) -> None: """add tags""" for hash_val, tag_name in tags: if tag_name not in [tag.name for tag in self.tags]: self.tags += [RemoteRef(tag_name, hash_val)] - def add_branches(self, branches): + def add_branches(self, branches: Sequence[Tuple[str, str]]) -> None: """add branches""" for hash_val, branch_name in branches: if branch_name not in [branch.name for branch in self.branches]: self.branches += [RemoteRef(branch_name, hash_val)] @property - def default(self): + def default(self) -> str: """get default branch""" if not self.vcs or self.vcs == "git": return "main" if "main" in self.branches else "master" @@ -53,7 +53,7 @@ def default(self): return "trunk" raise ValueError("no default version known for repository") - def as_yaml(self): + def as_yaml(self) -> Dict[str, Any]: """get yaml representation""" default = None try: @@ -67,7 +67,7 @@ def as_yaml(self): } return {k: v for k, v in yamldata.items() if v} - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: if isinstance(other, RemoteProjectVersions): return other.branches == self.branches and other.tags == self.tags return False @@ -77,7 +77,13 @@ class RemoteProject: """representation of remote repository project""" def __init__( - self, name, url, repo_path, src, vcs, versions=None + self, + name: str, + url: str, + repo_path: str, + src: str, + vcs: str, + versions: Optional[RemoteProjectVersions] = None, ): # pylint:disable=too-many-arguments,too-many-positional-arguments self.name = name self.url = url @@ -87,15 +93,15 @@ def __init__( self.versions = versions if versions else RemoteProjectVersions() def add_versions( - self, branches=Sequence[Tuple[str, str]], tags=Sequence[Tuple[str, str]] - ): + self, branches: Sequence[Tuple[str, str]], tags: Sequence[Tuple[str, str]] + ) -> None: """add branches and tags""" if not hasattr(self, "versions"): self.versions = RemoteProjectVersions(self.vcs) self.versions.add_branches(branches) self.versions.add_tags(tags) - def as_yaml(self): + def as_yaml(self) -> Dict[str, Any]: """get yaml representation""" yamldata = { "name": self.name, @@ -110,9 +116,9 @@ def as_yaml(self): return {k: v for k, v in yamldata.items() if v} @classmethod - def from_yaml(cls, yaml_data): + def from_yaml(cls, yaml_data: Dict[str, Any]) -> "RemoteProject": """build project from yaml representation""" - src = None if "src" not in yaml_data else yaml_data["src"] + src = "" if "src" not in yaml_data else yaml_data["src"] versions = None if "versions" not in yaml_data else yaml_data["versions"] parsed = cls( yaml_data["name"], @@ -129,7 +135,7 @@ def from_yaml(cls, yaml_data): parsed.add_versions(branches=branches, tags=tags) return parsed - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: if isinstance(other, str): return other == self.name if isinstance(other, RemoteProject): diff --git a/pyproject.toml b/pyproject.toml index 737c01c..11397ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ development = [ "pytest==8.3.4", "pytest-cov==6.0.0", "vcrpy==7.0.0", + "types-PyYaml==6.0.12" ] gui = [ "thefuzz==0.22.1", @@ -44,3 +45,12 @@ DfetchHub-cli = "dfetch_hub.project.cli:main_cli" [tool.pylint.format] max-line-length = "88" + +[tool.mypy] +files = "dfetch_hub" +ignore_missing_imports = true +strict = true + +[[tool.mypy.overrides]] +module = "dfetch_hub.example_gui.gui" +disable_error_code = ["attr-defined"] From 4b48e14748375d8c0e859675a9cca396a66284c8 Mon Sep 17 00:00:00 2001 From: Ben Spoor <37540691+ben-edna@users.noreply.github.com> Date: Wed, 9 Apr 2025 20:32:56 +0000 Subject: [PATCH 15/19] Use yaml safe_load --- dfetch_hub/project/project_parser.py | 2 +- dfetch_hub/project/project_sources.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dfetch_hub/project/project_parser.py b/dfetch_hub/project/project_parser.py index 17dff0d..ef100e8 100644 --- a/dfetch_hub/project/project_parser.py +++ b/dfetch_hub/project/project_parser.py @@ -40,7 +40,7 @@ def from_yaml(cls, yaml_file: str) -> "ProjectParser": """create parser from yaml file""" with open(yaml_file, "r", encoding="utf-8") as yamlf: instance = cls() - yaml_data = yaml.load(yamlf.read(), Loader=yaml.Loader) + yaml_data = yaml.safe_load(yamlf.read()) for project in yaml_data["projects"]: parsed_project = RemoteProject.from_yaml(project) instance.add_project(parsed_project) diff --git a/dfetch_hub/project/project_sources.py b/dfetch_hub/project/project_sources.py index 4f91d72..aaea14d 100644 --- a/dfetch_hub/project/project_sources.py +++ b/dfetch_hub/project/project_sources.py @@ -76,7 +76,7 @@ def from_yaml(cls, yaml_data: Union[str, bytes]) -> "SourceList": instance = cls() - parsed_yaml: Optional[Dict[str, Any]] = yaml.load(yaml_data, Loader=yaml.Loader) + parsed_yaml: Optional[Dict[str, Any]] = yaml.safe_load(yaml_data) if not parsed_yaml: raise RuntimeError("file should have data") assert parsed_yaml["source-list"], "file should have list of sources" From ec93263d8e095a4759a3c93cc41b92025907a80a Mon Sep 17 00:00:00 2001 From: Ben Spoor <37540691+ben-edna@users.noreply.github.com> Date: Wed, 9 Apr 2025 20:38:32 +0000 Subject: [PATCH 16/19] Use append instead of += [stuff] --- dfetch_hub/example_gui/gui.py | 2 +- dfetch_hub/project/export.py | 2 +- dfetch_hub/project/project_finder.py | 6 +++--- dfetch_hub/project/project_parser.py | 9 ++++----- dfetch_hub/project/project_sources.py | 7 ++----- dfetch_hub/project/remote_datasource.py | 4 ++-- 6 files changed, 13 insertions(+), 17 deletions(-) diff --git a/dfetch_hub/example_gui/gui.py b/dfetch_hub/example_gui/gui.py index ff3b5e5..239e64e 100644 --- a/dfetch_hub/example_gui/gui.py +++ b/dfetch_hub/example_gui/gui.py @@ -139,7 +139,7 @@ def update_autocomplete(value: str) -> None: fuzz.ratio(value, project.src), ) if ratio > 30 or url > 20 or repo_path > 20 or src > 20: - sorted_list += [(ratio, project)] + sorted_list.append((ratio, project)) sorted_list.sort(key=lambda i: i[0], reverse=True) for ratio, project in sorted_list: add_project_to_page(project) diff --git a/dfetch_hub/project/export.py b/dfetch_hub/project/export.py index 47c4879..e17c5b1 100644 --- a/dfetch_hub/project/export.py +++ b/dfetch_hub/project/export.py @@ -34,7 +34,7 @@ def __init__(self, entries: Optional[List[Entry]] = None): self._entries = [] def add_entry(self, entry: Entry) -> None: - self._entries += [entry] + self._entries.append(entry) @property def entries(self) -> List[Entry]: diff --git a/dfetch_hub/project/project_finder.py b/dfetch_hub/project/project_finder.py index 7ec390f..868a179 100644 --- a/dfetch_hub/project/project_finder.py +++ b/dfetch_hub/project/project_finder.py @@ -51,7 +51,7 @@ def filter_exclusions(self, paths: Union[List[str], Set[str]]) -> List[str]: if not path_allowed: break if path_allowed and path not in filtered_paths: - filtered_paths += [path] + filtered_paths.append(path) else: filtered_paths = list(paths) return filtered_paths @@ -66,7 +66,7 @@ def add_exclusion(self, exclusion: Optional[str]) -> None: if exclusion: if not self._exclusions: self._exclusions = [] - self._exclusions += [exclusion] + self._exclusions.append(exclusion) print(f"exclusions are {self.exclusions}") def filter_projects(self) -> None: @@ -163,7 +163,7 @@ def _projects_from_paths( project = RemoteProject(name, base_url, repo_path, src, vcs) project.versions.vcs = vcs project.add_versions(branches, tags) - projects += [project] + projects.append(project) return projects diff --git a/dfetch_hub/project/project_parser.py b/dfetch_hub/project/project_parser.py index ef100e8..aaf7543 100644 --- a/dfetch_hub/project/project_parser.py +++ b/dfetch_hub/project/project_parser.py @@ -20,7 +20,7 @@ def __init__(self) -> None: def add_project(self, new_project: RemoteProject) -> None: """add a project""" if new_project not in self._projects: - self._projects += [new_project] + self._projects.append(new_project) def get_projects(self) -> List[RemoteProject]: """get all projects""" @@ -28,10 +28,9 @@ def get_projects(self) -> List[RemoteProject]: def get_projects_as_yaml(self) -> str: """get yaml representation of projects""" - yaml_str = "" - yaml_obj: Dict[str, Any] = {"projects": []} - for project in self._projects: - yaml_obj["projects"] += [project.as_yaml()] + yaml_obj: Dict[str, List[Dict[str, Any]]] = { + "projects": [project.as_yaml() for project in self._projects] + } yaml_str = yaml.dump(yaml_obj) return yaml_str diff --git a/dfetch_hub/project/project_sources.py b/dfetch_hub/project/project_sources.py index aaea14d..3ed0f5a 100644 --- a/dfetch_hub/project/project_sources.py +++ b/dfetch_hub/project/project_sources.py @@ -18,10 +18,7 @@ def __init__(self, args: Union[Namespace, Dict[str, str]]): def add_exclusion(self, exclusion_regex: str) -> None: """add exclusion to project source""" - if not self.exclusions: - self.exclusions = [exclusion_regex] - else: - self.exclusions += [exclusion_regex] + self.exclusions.append(exclusion_regex) def as_yaml(self) -> Dict[str, Any]: """get yaml representation""" @@ -55,7 +52,7 @@ def __init__(self) -> None: def add_remote(self, source: RemoteSource) -> None: """add source""" - self._sources += [source] + self._sources.append(source) def get_remotes(self) -> List[RemoteSource]: """get list of sources""" diff --git a/dfetch_hub/project/remote_datasource.py b/dfetch_hub/project/remote_datasource.py index 948ff50..e0781e5 100644 --- a/dfetch_hub/project/remote_datasource.py +++ b/dfetch_hub/project/remote_datasource.py @@ -36,13 +36,13 @@ def add_tags(self, tags: Sequence[Tuple[str, str]]) -> None: """add tags""" for hash_val, tag_name in tags: if tag_name not in [tag.name for tag in self.tags]: - self.tags += [RemoteRef(tag_name, hash_val)] + self.tags.append(RemoteRef(tag_name, hash_val)) def add_branches(self, branches: Sequence[Tuple[str, str]]) -> None: """add branches""" for hash_val, branch_name in branches: if branch_name not in [branch.name for branch in self.branches]: - self.branches += [RemoteRef(branch_name, hash_val)] + self.branches.append(RemoteRef(branch_name, hash_val)) @property def default(self) -> str: From 45271cb15552c95e49f29f6a936bb3323c0c8cc0 Mon Sep 17 00:00:00 2001 From: Ben Spoor <37540691+ben-edna@users.noreply.github.com> Date: Wed, 9 Apr 2025 20:55:16 +0000 Subject: [PATCH 17/19] Fix pylint issues --- .pre-commit-config.yaml | 2 +- dfetch_hub/project/cli.py | 3 ++- dfetch_hub/project/export.py | 15 +++++++++++++-- dfetch_hub/project/project_parser.py | 3 +-- dfetch_hub/project/project_sources.py | 2 +- pyproject.toml | 2 +- 6 files changed, 19 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d345313..b55fb72 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,5 +14,5 @@ repos: name: lint python files entry: pylint language: system - files: ^src/ + files: ^dfetch_hub/ types: [file, python] \ No newline at end of file diff --git a/dfetch_hub/project/cli.py b/dfetch_hub/project/cli.py index 3dd8fd6..84cab5f 100644 --- a/dfetch_hub/project/cli.py +++ b/dfetch_hub/project/cli.py @@ -1,4 +1,4 @@ -""" """ +"""Commandline interface of dfetch hub.""" import argparse @@ -33,6 +33,7 @@ def main(parser: argparse.ArgumentParser) -> None: def main_cli() -> None: + """Main command line interface.""" arg_parser = argparse.ArgumentParser() arg_parser.add_argument("-u", "--url", required=False, nargs="+") arg_parser.add_argument("-ds", "--dfetch-source", required=False) diff --git a/dfetch_hub/project/export.py b/dfetch_hub/project/export.py index e17c5b1..2eb0fd0 100644 --- a/dfetch_hub/project/export.py +++ b/dfetch_hub/project/export.py @@ -11,6 +11,8 @@ @dataclass class Entry: + """Entry to export.""" + name: str revision: str src: str @@ -20,12 +22,17 @@ class Entry: class Export(ABC): + """Abstract Export interface.""" + + def add_entry(self, entry: Entry) -> None: + """Add entry to export.""" def export(self) -> None: - pass + """Export the projects.""" class DfetchExport(Export): + """Dfetch specific exporter.""" def __init__(self, entries: Optional[List[Entry]] = None): if entries: @@ -34,16 +41,20 @@ def __init__(self, entries: Optional[List[Entry]] = None): self._entries = [] def add_entry(self, entry: Entry) -> None: + """Add entry to export.""" self._entries.append(entry) @property def entries(self) -> List[Entry]: + """All entries in export.""" return self._entries def export(self, path: str = "") -> None: + """Export the DFetch manifest to path.""" remotes: Sequence[Union[RemoteDict, Remote]] = ( [] - ) # TODO: bundle projects with shared path in remotes + ) # Use _create_remotes from import function to bundle projects with shared path in remotes + projects = [ ProjectEntryDict( name=entry.name, diff --git a/dfetch_hub/project/project_parser.py b/dfetch_hub/project/project_parser.py index aaf7543..08233b5 100644 --- a/dfetch_hub/project/project_parser.py +++ b/dfetch_hub/project/project_parser.py @@ -31,8 +31,7 @@ def get_projects_as_yaml(self) -> str: yaml_obj: Dict[str, List[Dict[str, Any]]] = { "projects": [project.as_yaml() for project in self._projects] } - yaml_str = yaml.dump(yaml_obj) - return yaml_str + return str(yaml.dump(yaml_obj)) @classmethod def from_yaml(cls, yaml_file: str) -> "ProjectParser": diff --git a/dfetch_hub/project/project_sources.py b/dfetch_hub/project/project_sources.py index 3ed0f5a..5aa8c10 100644 --- a/dfetch_hub/project/project_sources.py +++ b/dfetch_hub/project/project_sources.py @@ -63,7 +63,7 @@ def as_yaml(self) -> str: versiondata = {"version": self.CURRENT_VERSION} remotes_data = {"remotes": [source.as_yaml() for source in self._sources]} yamldata = {"source-list": [versiondata, remotes_data]} - return yaml.dump(yamldata) + return str(yaml.dump(yamldata)) @classmethod def from_yaml(cls, yaml_data: Union[str, bytes]) -> "SourceList": diff --git a/pyproject.toml b/pyproject.toml index 11397ca..1c95724 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ Issues = "https://github.com/dfetch-org/dfetch-hub/issues" DfetchHub-cli = "dfetch_hub.project.cli:main_cli" [tool.pylint.format] -max-line-length = "88" +max-line-length = "120" [tool.mypy] files = "dfetch_hub" From a40c13d5fad2df07e3a728d8b974c31131b6e21e Mon Sep 17 00:00:00 2001 From: Ben Spoor <37540691+ben-edna@users.noreply.github.com> Date: Wed, 9 Apr 2025 20:58:48 +0000 Subject: [PATCH 18/19] Add author --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 1c95724..c24ddd8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ name = "dfetch_hub" version = "0.0.1" authors = [ { name="p sacharias", email="p.sacharias@gmail.com" }, + { name="ben spoor", email="b.spoor@edna.eu" }, ] description = "Dfetch Hub" readme = "README.md" From d7a2c290bd0f86f25f13e547a8465c88206876a4 Mon Sep 17 00:00:00 2001 From: Ben Spoor <37540691+ben-edna@users.noreply.github.com> Date: Wed, 9 Apr 2025 21:15:29 +0000 Subject: [PATCH 19/19] Improve doc of module --- dfetch_hub/__init__.py | 2 +- dfetch_hub/project/export.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dfetch_hub/__init__.py b/dfetch_hub/__init__.py index 8e13402..bad88b8 100644 --- a/dfetch_hub/__init__.py +++ b/dfetch_hub/__init__.py @@ -1 +1 @@ -# file required to run gui from project path +# This __init__ is required by nicegui to run the example gui from project path diff --git a/dfetch_hub/project/export.py b/dfetch_hub/project/export.py index 2eb0fd0..29cdc9c 100644 --- a/dfetch_hub/project/export.py +++ b/dfetch_hub/project/export.py @@ -1,4 +1,4 @@ -"""export module""" +"""Module that handles the export of the list of projects.""" from abc import ABC from dataclasses import dataclass