diff --git a/.devcontainer/.dfetch_data.yaml b/.devcontainer/.dfetch_data.yaml new file mode 100644 index 0000000..95e5105 --- /dev/null +++ b/.devcontainer/.dfetch_data.yaml @@ -0,0 +1,10 @@ +# This is a generated file by dfetch. Don't edit this, but edit the manifest. +# For more info see https://dfetch.rtfd.io/en/latest/getting_started.html +dfetch: + branch: main + hash: f2a3ac0bd99186f66dbca1f5c37b09f8 + last_fetch: 09/04/2025, 19:11:43 + patch: dfetch-devcontainer.patch + remote_url: https://github.com/dfetch-org/dfetch + revision: 89e8c77621e62b6ef657c4358b350a959c82af16 + tag: '' diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..0a1ea92 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,28 @@ +FROM mcr.microsoft.com/devcontainers/python:1-3.12-bullseye + +# Install dependencies +# pv is required for asciicasts +RUN apt-get update && apt-get install --no-install-recommends -y \ + pv=1.6.6-1+b1 \ + subversion=1.14.1-3+deb11u1 && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /workspaces/dfetch-hub + +# Add a non-root user (dev) +RUN useradd -m dev && chown -R dev:dev /workspaces/dfetch-hub + +USER dev + +ENV PATH="/home/dev/.local/bin:${PATH}" +ENV PYTHONPATH="/home/dev/.local/lib/python3.12" +ENV PYTHONUSERBASE="/home/dev/.local" + +COPY --chown=dev:dev . . + +RUN pip install --no-cache-dir --root-user-action=ignore --upgrade pip==25.0.1 \ + && pip install --no-cache-dir --root-user-action=ignore -e .[development,gui] \ + && pre-commit install --install-hooks + +# Set bash as the default shell +SHELL ["/bin/bash", "-ec"] diff --git a/.devcontainer/LICENSE b/.devcontainer/LICENSE new file mode 100644 index 0000000..f81e253 --- /dev/null +++ b/.devcontainer/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 dfetch-org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..263377c --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,38 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/python +{ + "name": "Python 3", + "build": { + "dockerfile": "Dockerfile", + "context": ".." + }, + "postCreateCommand": "pip install -e .[development,docs,casts]", + "customizations": { + "vscode": { + "extensions": [ + "lextudio.restructuredtext", + "alexkrechik.cucumberautocomplete", + "ms-python.python", + "ms-python.isort", + "ms-python.black-formatter", + "ms-python.debugpy", + "mhutchie.git-graph", + "tamasfe.even-better-toml", + "trond-snekvik.simple-rst", + "jebbs.plantuml", + "jimasp.behave-vsc" + ], + "settings": { + "terminal.integrated.profiles.linux": { + "bash": { + "path": "bash", + "icon": "terminal-bash" + } + }, + "terminal.integrated.defaultProfile.linux": "bash" + } + } + }, + "workspaceFolder": "/workspaces/dfetch-hub", + "remoteUser": "dev" +} diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..f268356 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,24 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" + # Raise pull requests for version updates + # to pip against the `develop` branch + target-branch: "dev" + # Labels on pull requests for version updates only + labels: + - "dependencies" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + - package-ecosystem: "devcontainers" + directory: "/" + schedule: + interval: "daily" + - package-ecosystem: "docker" + directory: "/.devcontainer" + schedule: + interval: "daily" \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..6ef56d7 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,40 @@ +name: Test + +on: + push: + branches: + - main + pull_request: + types: [opened, synchronize, reopened] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + pip install .[gui,development] + + # - run: codespell # Check for typo's + - run: isort --diff dfetch_hub # Checks import order + - run: black --check dfetch_hub # Checks code style + # - run: flake8 dfetch_hub # Checks pep8 conformance + - run: pylint dfetch_hub # Checks pep8 conformance + # - run: ruff check dfetch # Check using ruff + - run: mypy --strict dfetch_hub # Check types + # - run: pyright . # Check types + # - run: doc8 doc # Checks documentation + # - run: pydocstyle dfetch # Checks doc strings + # - run: bandit -r dfetch # Checks security issues + # - run: xenon -b B -m A -a A dfetch # Check code quality + - run: pytest --cov=dfetch_hub test # Run tests + # - run: coverage run --source=dfetch --append -m behave features # Run features tests + # - run: coverage xml -o coverage.xml # Create XML report + # - run: pyroma --directory --min=10 . # Check pyproject diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7085d73 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +__pycache__ +.coverage +.mypy_cache +.pytest_cache +.vscode +build +coverage.xml +dfetch_hub.egg-info +dist +doc/_build +doc/landing-page/_build +example/Tests/ +venv* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..b55fb72 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +repos: +- repo: local + hooks: + - id: isort + name: import sorting + entry: isort + language: python + - id: black + name: black formatter + entry: black + language: system + types: [file, python] + - id: pylint + name: lint python files + entry: pylint + language: system + files: ^dfetch_hub/ + types: [file, python] \ No newline at end of file diff --git a/LICENSE b/LICENSE index 435ccda..7db949e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,21 @@ -MIT License - -Copyright (c) 2025 dfetch-org - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +MIT License + +Copyright (c) 2025 dfetch-org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 16b3be0..cb7677b 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,25 @@ -# dfetch-hub -Explorer for finding new projects to dfetch +# dfetch-hub + +[![Contribute with Codespaces](https://img.shields.io/static/v1?label=Codespaces&message=Open&color=blue)](https://codespaces.new/dfetch-org/dfetch-hub) + +Explorer for finding new projects to dfetch + +## Setup + +```console +pip install -e .[gui] +``` + +## Basic usage + +### CLI + +```console +DfetchHub-cli -u "https://github.com/dfetch-org/dfetch.git" +``` + +### Gui + +```console +python -m "dfetch_hub.example_gui.gui" +``` diff --git a/dfetch-devcontainer.patch b/dfetch-devcontainer.patch new file mode 100644 index 0000000..0ad94ba --- /dev/null +++ b/dfetch-devcontainer.patch @@ -0,0 +1,38 @@ +diff --git a/Dockerfile b/Dockerfile +index b3e8653..4d68823 100644 +--- a/Dockerfile ++++ b/Dockerfile +@@ -7,10 +7,10 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ + subversion=1.14.1-3+deb11u1 && \ + rm -rf /var/lib/apt/lists/* + +-WORKDIR /workspaces/dfetch ++WORKDIR /workspaces/dfetch-hub + + # Add a non-root user (dev) +-RUN useradd -m dev && chown -R dev:dev /workspaces/dfetch ++RUN useradd -m dev && chown -R dev:dev /workspaces/dfetch-hub + + USER dev + +@@ -21,7 +21,7 @@ ENV PYTHONUSERBASE="/home/dev/.local" + COPY --chown=dev:dev . . + + RUN pip install --no-cache-dir --root-user-action=ignore --upgrade pip==25.0.1 \ +- && pip install --no-cache-dir --root-user-action=ignore -e .[development,docs,test,casts] \ ++ && pip install --no-cache-dir --root-user-action=ignore -e .[development,gui] \ + && pre-commit install --install-hooks + + # Set bash as the default shell +diff --git a/devcontainer.json b/devcontainer.json +index 3cc8f4d..263377c 100644 +--- a/devcontainer.json ++++ b/devcontainer.json +@@ -33,6 +33,6 @@ + } + } + }, +- "workspaceFolder": "/workspaces/dfetch", ++ "workspaceFolder": "/workspaces/dfetch-hub", + "remoteUser": "dev" + } diff --git a/dfetch-hub.code-workspace b/dfetch-hub.code-workspace new file mode 100644 index 0000000..5d866c3 --- /dev/null +++ b/dfetch-hub.code-workspace @@ -0,0 +1,142 @@ +{ + "folders": [ + { + "path": "." + } + ], + "settings": { + "[windows]": { + "python.defaultInterpreterPath": "${workspaceFolder}/venv/Scripts/python", + }, + "editor.trimAutoWhitespace": true, + "editor.codeActionsOnSave": { + "source.organizeImports": "explicit" + }, + "isort.check": true, + "restructuredtext.linter.run": "onType", + "restructuredtext.linter.doc8.extraArgs": [ + "--config", + "${workspaceFolder}/pyproject.toml" + ], + "python.testing.pytestArgs": [ + "tests" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "python.testing.autoTestDiscoverOnSaveEnabled": true, + "cucumberautocomplete.steps": [ + "features/steps/*.py" + ], + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter", + "editor.formatOnSave": true, + }, + }, + "launch": { + "version": "0.2.0", + "configurations": [ + { + "name": "DFetch hub gui", + "type": "debugpy", + "request": "launch", + "module": "dfetch_hub.example_gui.gui", + "justMyCode": false, + "args": [ + ] + }, + { + "name": "DFetch hub cli", + "type": "debugpy", + "request": "launch", + "module": "dfetch_hub.project.cli", + "justMyCode": false, + "args": ["-u", "https://github.com/dfetch-org/dfetch.git" + ] + } + ] + }, + "tasks": { + "version": "2.0.0", + "tasks": [ + { + "label": "Build Docs", + "type": "shell", + "linux": { + "command": "make" + }, + "windows": { + "command": "make.bat" + }, + "args": [ + "html" + ], + "options": { + "cwd": "${workspaceFolder}/doc" + }, + "group": { + "kind": "build", + "isDefault": true + }, + "presentation": { + "reveal": "always", + "panel": "shared" + } + }, + { + "label": "Check quality (pre-commit)", + "type": "shell", + "command": "pre-commit", + "args": [ + "run", + "--all-files" + ], + "options": { + "cwd": "${workspaceFolder}" + }, + "group": { + "kind": "test", + "isDefault": false + }, + "presentation": { + "reveal": "always", + "panel": "shared" + } + }, + { + "label": "Run All Unit Tests", + "type": "shell", + "command": "python", + "args": [ + "-m", + "pytest", + "test" + ], + "options": { + "cwd": "${workspaceFolder}" + }, + "group": { + "kind": "test", + "isDefault": false + }, + "presentation": { + "reveal": "always", + "panel": "shared" + } + }, + ] + }, + "extensions": { + "recommendations": [ + "bungcip.better-toml", + "jebbs.plantuml", + "lextudio.restructuredtext", + "ms-python.black-formatter", + "ms-python.debugpy", + "ms-python.isort", + "ms-python.pylint", + "ms-python.python", + "trond-snekvik.simple-rst", + "jimasp.behave-vsc" + ] + } +} \ No newline at end of file diff --git a/dfetch.yaml b/dfetch.yaml new file mode 100644 index 0000000..356ad84 --- /dev/null +++ b/dfetch.yaml @@ -0,0 +1,14 @@ +manifest: + version: 0.0 # DFetch Module syntax version + + remotes: # declare common sources in one place + - name: github + url-base: https://github.com/ + + projects: + - name: dfetch-devcontainer + dst: .devcontainer + repo-path: dfetch-org/dfetch + src: .devcontainer + branch: main + patch: dfetch-devcontainer.patch diff --git a/dfetch_hub/__init__.py b/dfetch_hub/__init__.py new file mode 100644 index 0000000..bad88b8 --- /dev/null +++ b/dfetch_hub/__init__.py @@ -0,0 +1 @@ +# This __init__ is required by nicegui to run the example gui from project path diff --git a/dfetch_hub/example_gui/gui.py b/dfetch_hub/example_gui/gui.py new file mode 100644 index 0000000..239e64e --- /dev/null +++ b/dfetch_hub/example_gui/gui.py @@ -0,0 +1,299 @@ +"""sample of a possible nicegui based gui""" + +from typing import Optional, Sequence + +from nicegui import events, ui +from thefuzz import fuzz + +from dfetch_hub.project.project_finder import GitProjectFinder, ProjectFinder +from dfetch_hub.project.project_sources import RemoteSource, SourceList +from dfetch_hub.project.remote_datasource import RemoteProject, RemoteRef + + +def main() -> None: + """main gui runner""" + ui.context.sl = SourceList() + ui.context.pf = [] + header() + with ui.column().classes("w-1/3 mx-auto mt-10"): + show_sources() + url_input() + sources_input() + ui.run(title="dfetch project viewer", reconnect_timeout=30) + + +def header() -> None: + """main gui header""" + with ui.header().classes("bg-black text-white p-4"): + with ui.row().classes( + "container mx-auto flex justify-between items-center space-x-4" + ): + ui.link("Sources", target="/sources").classes( + "text-white hover:text-gray-400 no-underline" + ) + ui.link("Projects", target="/projects").classes( + "text-white hover:text-gray-400 no-underline" + ) + ui.link("Filters", target="/filters").classes( + "text-white hover:text-gray-400 no-underline" + ) + + +@ui.page("/sources") +def sources_page() -> None: + """page to enter sources to search""" + header() + with ui.column().classes("w-1/3 mx-auto mt-10"): + show_sources() + url_input() + sources_input() + + +def add_projects_to_page() -> None: + """add list of project finder results to page""" + if not ui.context.pf: + ui.context.pf = [] + ui.context.projects = [] + for source in ui.context.sl.get_remotes(): + if source.url not in [pf.url for pf in ui.context.pf]: + print(f"adding source {source.url}") + if hasattr(source, "exclusions"): + pf = GitProjectFinder(source.url, source.exclusions) + else: + pf = GitProjectFinder(source.url) + ui.context.pf += [pf] + for pf in ui.context.pf: + try: + add_project_finder_to_page(pf) + except ValueError as e: + ui.notification(f"{e}") + + +def add_project_finder_to_page( + pf: ProjectFinder, projects: Optional[Sequence[RemoteProject]] = None +) -> None: + """add single project finder result to page""" + if not projects: + projects = pf.list_projects() + ui.context.projects += [ + project for project in projects if project not in ui.context.projects + ] + ui.label(f"{pf.url}").classes("text-xl font-bold mb-4 text-center") + with ui.row().classes("grid grid-cols-2 gap-4 overflow-auto max-h-screen w-2/3"): + for project in projects: + add_project_to_page(project) + + +def add_project_to_page(project: RemoteProject) -> None: + """add single project to page""" + with ui.card().classes( + "bg-black text-white p-6 rounded shadow-lg \ + text-center w-full h-40 flex justify-center items-center" + ): + ui.link(text=project.name, target=f"/project_data/{project.name}").classes( + "text-white hover:text-gray-400 no-underline" + ) + + +@ui.page("/projects/") +def projects_page() -> None: + """projects for source""" + header() + search_input = ui.input(placeholder="Search packages").classes("flex-grow") + search_input.on_value_change(lambda e: update_autocomplete(e.value)) + ui.context.project_col = ui.column().classes("w-2/3 mx-auto mt-10") + with ui.context.project_col: + try: + add_projects_to_page() + except AttributeError as e: + print(e) + ui.navigate.to("/sources") + + +def update_autocomplete(value: str) -> None: + """autocomplete for project search""" + ui.context.project_col.clear() + with ui.context.project_col: + if not value: + print("adding all projects") + add_projects_to_page() + else: + print(f"adding projects matching {value}") + for pf in ui.context.pf: + sorted_list = [] + for project in pf.list_projects(): + url, repo_path, src = ( + fuzz.ratio(value, project.url), + -fuzz.ratio(value, project.repo_path), + -fuzz.ratio(value, project.src), + ) + print( + f"ratios {project.url}{project.repo_path}{project.src}\ +- {fuzz.ratio(value,project.url)}\ +- {fuzz.ratio(value,project.repo_path)}\ +- {fuzz.ratio(value,project.src)}" + ) + ratio = max( + fuzz.ratio(value, project.url), + fuzz.ratio(value, project.repo_path), + fuzz.ratio(value, project.src), + ) + if ratio > 30 or url > 20 or repo_path > 20 or src > 20: + sorted_list.append((ratio, project)) + sorted_list.sort(key=lambda i: i[0], reverse=True) + for ratio, project in sorted_list: + add_project_to_page(project) + + +@ui.page("/project_data/{name}") +def projects_data_page(name: str) -> None: + """data for project""" + header() + with ui.column().classes("w-5/6 items-center mx-auto mt-10"): + try: + found_project = None + for project in ui.context.projects: + if project.name == name: + found_project = project + break + if found_project: + project_representation(found_project) + else: + ui.label( + f"could not find project in \ +{[project.name for project in ui.context.projects]}" + ) + except AttributeError: + ui.label(f"{name} was not found") + + +@ui.page("/filters") +def filters_page() -> None: + """page showing exclusions per source""" + header() + ui.notify("no sources present, redirecting to sources") + if hasattr(ui.context, "pf") and len(ui.context.pf) > 0: + with ui.column(): + for pf in ui.context.pf: + with ui.row(): + ui.label(f"{pf.url}") + filter_in = ui.input(placeholder="enter filter regex") + ui.button( + "add exclusion", + on_click=lambda pf=pf, filter_in=filter_in: add_exclusion( + pf, filter_in.value + ), + ) + if ( + hasattr(pf, "exclusions") + and pf.exclusions + and len(pf.exclusions) > 0 + ): + with ui.column(): + for excl in pf.exclusions: + ui.label(f"{excl}") + ui.button("store", on_click=presist_sources) + else: + ui.navigate.to("/sources") + + +def add_exclusion(pf: ProjectFinder, regex: str) -> None: + """add exclusion for the project finder for a source""" + ui.notify(f"adding exclusion {regex} to projects on url {pf.url}") + project = [ + project for project in ui.context.sl.get_remotes() if project.url == pf.url + ][0] + project.add_exclusion(regex) + pf.add_exclusion(regex) + pf.filter_projects() + + +def presist_sources() -> None: + """persist entered sources to file""" + sl = ui.context.sl + ui.download(sl.as_yaml().encode("utf-8"), filename="sources.yaml") + + +def url_input() -> None: + """url input page""" + url_search_field = ui.input(placeholder="enter url to list packages").classes( + "w-full p-2 text-lg border border-gray-300 rounded" + ) + ui.button( + text="get projects", on_click=lambda a: get_projects(url_search_field.value) + ).classes("bg-black text-white px-4 py-2 rounded hover:bg-gray-800 mt-4") + + +def sources_input() -> None: + """input sources file""" + ui.upload( + on_upload=lambda e: handle_upload(e) # pylint:disable = unnecessary-lambda + ).props("accept=.yaml").classes("max-w-full") + + +def handle_upload(file: events.UploadEventArguments) -> None: + """handle upload of sources file""" + ui.context.sl = SourceList.from_yaml(file.content.read()) + ui.notify(f"uploaded {file.name}") + + +def get_projects(url: str) -> None: + """handling of project search""" + if url and len(url) > 5: # what is min valid url len? + name = url.split("/")[-1] + ui.context.sl.add_remote(RemoteSource({"name": name, "url-base": url})) + ui.navigate.to("/projects/") + + +def project_representation(project: RemoteProject) -> None: + """project representation""" + ui.label(project.name).classes("text-h5 text-black mb-5") + + with ui.row().classes("justify-between m-20"): + # Empty space in column 1 (1-2) + with ui.column().classes("w-full sm:w-1/12"): + pass # No content here (empty) + + # Column 1 (2-4), spanning 3 parts + with ui.column().classes("w-full sm:w-3/12"): + ui.link( + project.url, target=f"http://{project.url}/{project.repo_path}" + ).classes("text-body1 text-black") + ui.label(f"Source - {project.src if project.src else "/"}").classes( + "text-body2 text-black" + ) + ui.label(f"vcs - {project.vcs}") + + # Column 2 (4-7), spanning 3 parts + with ui.column().classes("w-full sm:w-3/12"): + ui.label("branches").classes("text-h5 text-black mb-3") + for branch in project.versions.branches: + revision_representation(branch) + + # Column 3 (7-10), spanning 3 parts + with ui.column().classes("w-full sm:w-3/12"): + ui.label("tags").classes("text-h5 text-black mb-3") + for tag in project.versions.tags: + revision_representation(tag) + + # Empty space in column 3 (10-12) + with ui.column().classes("w-full sm:w-1/12"): + pass # No content here (empty) + + +def revision_representation(rev: RemoteRef) -> None: + """revision representation""" + ui.label(f"revision {rev.name} - {rev.revision}").classes( + "text-body2 text-black mb-2" + ) + + +def show_sources() -> None: + """show sources in source view""" + if hasattr(ui.context, "pf"): + for pf in ui.context.pf: + ui.label(f"{pf.url}") + + +if __name__ in ("__main__", "__mp_main__"): + main() diff --git a/dfetch_hub/project/cli.py b/dfetch_hub/project/cli.py new file mode 100644 index 0000000..84cab5f --- /dev/null +++ b/dfetch_hub/project/cli.py @@ -0,0 +1,50 @@ +"""Commandline interface of dfetch hub.""" + +import argparse + +from dfetch_hub.project.input_parser import InputParser +from dfetch_hub.project.project_finder import GitProjectFinder +from dfetch_hub.project.project_parser import ProjectParser +from dfetch_hub.project.project_sources import SourceList + +# from project.cli_disp import CliDisp + + +def main(parser: argparse.ArgumentParser) -> None: + """main command line interface for program""" + args = parser.parse_args() + if not args.url and not args.dfetch_source: + parser.print_help() + raise ValueError("no url or dfetch manifest found") + input_args_parser = InputParser(args) + url_list = input_args_parser.get_urls() + if args.persist_sources: + sources_list = SourceList.from_input_parser(input_args_parser) + with open("sources.yaml", "w", encoding="utf-8") as sources_file: + sources_file.write(sources_list.as_yaml()) + project_parser = ProjectParser() + for url in url_list: + gpf = GitProjectFinder(url, args.project_exclude_pattern) + projects = gpf.list_projects() + for project in projects: + project_parser.add_project(project) + with open("projects.yaml", "w", encoding="utf-8") as datasource: + datasource.write(project_parser.get_projects_as_yaml()) + + +def main_cli() -> None: + """Main command line interface.""" + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument("-u", "--url", required=False, nargs="+") + arg_parser.add_argument("-ds", "--dfetch-source", required=False) + arg_parser.add_argument( + "-pep", "--project-exclude-pattern", required=False, nargs="+" + ) + arg_parser.add_argument( + "-ps", "--persist-sources", required=False, action="store_true" + ) + main(arg_parser) + + +if __name__ == "__main__": + main_cli() diff --git a/dfetch_hub/project/export.py b/dfetch_hub/project/export.py new file mode 100644 index 0000000..29cdc9c --- /dev/null +++ b/dfetch_hub/project/export.py @@ -0,0 +1,74 @@ +"""Module that handles the export of the list of projects.""" + +from abc import ABC +from dataclasses import dataclass +from typing import List, Optional, Sequence, Union + +from dfetch.manifest.manifest import Manifest, ManifestDict +from dfetch.manifest.project import ProjectEntryDict +from dfetch.manifest.remote import Remote, RemoteDict + + +@dataclass +class Entry: + """Entry to export.""" + + name: str + revision: str + src: str + url: str + repo_path: str + vcs: str = "git" + + +class Export(ABC): + """Abstract Export interface.""" + + def add_entry(self, entry: Entry) -> None: + """Add entry to export.""" + + def export(self) -> None: + """Export the projects.""" + + +class DfetchExport(Export): + """Dfetch specific exporter.""" + + def __init__(self, entries: Optional[List[Entry]] = None): + if entries: + self._entries = entries + else: + self._entries = [] + + def add_entry(self, entry: Entry) -> None: + """Add entry to export.""" + self._entries.append(entry) + + @property + def entries(self) -> List[Entry]: + """All entries in export.""" + return self._entries + + def export(self, path: str = "") -> None: + """Export the DFetch manifest to path.""" + remotes: Sequence[Union[RemoteDict, Remote]] = ( + [] + ) # Use _create_remotes from import function to bundle projects with shared path in remotes + + projects = [ + ProjectEntryDict( + name=entry.name, + revision=entry.revision, + src=entry.src, + url=entry.url, + repo_path=entry.repo_path, + vcs=entry.vcs, + ) + for entry in self._entries + ] + as_dict = ManifestDict( + version=Manifest.CURRENT_VERSION, remotes=remotes, projects=projects + ) + if not path: + path = "dfetch.yaml" + Manifest(as_dict).dump(path) diff --git a/dfetch_hub/project/input_parser.py b/dfetch_hub/project/input_parser.py new file mode 100644 index 0000000..740ee06 --- /dev/null +++ b/dfetch_hub/project/input_parser.py @@ -0,0 +1,25 @@ +"""input parser module""" + +from argparse import Namespace +from typing import Sequence + +from dfetch.manifest.manifest import Manifest + + +class InputParser: # pylint:disable=too-few-public-methods + """parser for url or dfetch file input""" + + def __init__(self, args: Namespace): + self.args = args + + def get_urls(self) -> Sequence[str]: + """get urls for input""" + if self.args.url: + if isinstance(self.args.url, list): + return self.args.url + return [self.args.url] + return self._parse_dfetch_remotes(self.args.dfetch_source) + + def _parse_dfetch_remotes(self, dfetch_path: str) -> Sequence[str]: + manifest = Manifest.from_file(dfetch_path) + return [project.remote_url for project in manifest.projects] diff --git a/dfetch_hub/project/project_finder.py b/dfetch_hub/project/project_finder.py new file mode 100644 index 0000000..868a179 --- /dev/null +++ b/dfetch_hub/project/project_finder.py @@ -0,0 +1,177 @@ +"""project finder module""" + +import logging +import os +import re +import sys +from abc import abstractmethod +from contextlib import chdir +from typing import List, Optional, Sequence, Set, Tuple, Union + +from dfetch.util.cmdline import SubprocessCommandError, run_on_cmdline + +from dfetch_hub.project.remote_datasource import RemoteProject + +WORKDIR = "tmp" + + +class ProjectFinder: + """class to find projects in repositories""" + + def __init__(self, url: str, exclusions: Optional[List[str]] = None): + self._url = url + self._logger = logging.getLogger() + self._projects: list[RemoteProject] = [] + self._exclusions: List[str] = exclusions or [] + + @property + def url(self) -> str: + """repo url""" + return self._url + + @abstractmethod + def list_projects(self) -> list[RemoteProject]: + """list all projects in a repo""" + raise AssertionError("abstractmethod") + + def filter_exclusions(self, paths: Union[List[str], Set[str]]) -> List[str]: + """filter exclusions from list of projects""" + filtered_paths = [] + for path in paths: + path_allowed = True + if self._exclusions: + for exclusion in self._exclusions: + try: + re.compile(exclusion) + except re.error as exc: + raise ValueError( + f"regex of exclusion is invalid, {exclusion}" + ) from exc + path_allowed = path_allowed and not re.search(exclusion, path) + if not path_allowed: + break + if path_allowed and path not in filtered_paths: + filtered_paths.append(path) + else: + filtered_paths = list(paths) + return filtered_paths + + @property + def exclusions(self) -> Optional[List[str]]: + """get exclusion for project finder""" + return self._exclusions + + def add_exclusion(self, exclusion: Optional[str]) -> None: + """add an exclusion regex""" + if exclusion: + if not self._exclusions: + self._exclusions = [] + self._exclusions.append(exclusion) + print(f"exclusions are {self.exclusions}") + + def filter_projects(self) -> None: + """filter projects on exclusions""" + for project in self._projects: + path = f"{project.url, project.repo_path, project.src}" + path_allowed = True + if self._exclusions: + for exclusion in self._exclusions: + try: + re.compile(exclusion) + except re.error as exc: + raise ValueError( + f"regex of exclusion is invalid, {exclusion}" + ) from exc + path_allowed = path_allowed and not re.search(exclusion, path) + if not path_allowed: + break + if not path_allowed: + self._projects.remove(project) + + +class GitProjectFinder(ProjectFinder): + """git implementation of project finder""" + + def list_projects(self) -> List[RemoteProject]: + """list all git projects in a git repo""" + if not self._projects: + if os.path.exists(WORKDIR) and os.path.isdir(WORKDIR): + if sys.platform == "win32": + os.system(f"rmdir /S /Q {WORKDIR}") + elif sys.platform == "linux": + os.system(f"rm -rf {WORKDIR}") + try: + result = run_on_cmdline( + self._logger, f"git clone --no-checkout {self._url} {WORKDIR}" + ) + with chdir(WORKDIR): + result = run_on_cmdline(self._logger, "git status") + # More matching (specific types, add interface) + # keep matched on (e.g. project x matched on ...) + res = re.findall( + r"\sdeleted:\s+(.*(?:README|LICENSE|CHANGELOG|Readme|readme|License|license).*)", # pylint:disable=line-too-long + result.stdout.decode("utf-8"), + ) + result = run_on_cmdline( + self._logger, "git for-each-ref refs/remotes/origin" + ) + branches = re.findall( + r"([a-f0-9]*)\scommit\s.*/origin/(.*)", + result.stdout.decode("utf-8"), + ) + result = run_on_cmdline(self._logger, "git for-each-ref refs/tags") + tags = re.findall( + r"([a-f0-9]*)\s(?:(?:commit)|(?:tag))\s*refs/tags/(.*)", + result.stdout.decode("utf-8"), + ) + except SubprocessCommandError as exc: + raise ValueError( + f"could not find repository at url {self._url}" + ) from exc + finally: + if sys.platform == "win32": + os.system(f"rmdir /S /Q {WORKDIR}") + elif sys.platform == "linux": + os.system(f"rm -rf {WORKDIR}") + paths = set() + for path in res: + if "/" in path: + paths.add(f"{path.rsplit("/", maxsplit=1)[0].strip(" ")}") + else: + paths.add("") + filtered_paths = self.filter_exclusions(paths) + self._projects = self._projects_from_paths(filtered_paths, branches, tags) + return self._projects + + def _projects_from_paths( + self, + paths: Sequence[str], + branches: Sequence[Tuple[str, str]], + tags: Sequence[Tuple[str, str]], + ) -> List[RemoteProject]: + projects = [] + for path in paths: + if "/" in path: + name = path.rsplit("/", maxsplit=1)[1] + elif len(path) > 1: + name = path + else: + name = self.url.rsplit("/", maxsplit=1)[1] + base_url, repo_path = _base_url(self.url) + src = path + vcs = "git" + project = RemoteProject(name, base_url, repo_path, src, vcs) + project.versions.vcs = vcs + project.add_versions(branches, tags) + projects.append(project) + return projects + + +def _base_url(url: str) -> Tuple[str, str]: + if "://" in url: + url = url.split("://", maxsplit=1)[1] + if "/" in url: + url, repo_path = url.split("/", maxsplit=1) + else: + repo_path = "" + return url, repo_path diff --git a/dfetch_hub/project/project_parser.py b/dfetch_hub/project/project_parser.py new file mode 100644 index 0000000..08233b5 --- /dev/null +++ b/dfetch_hub/project/project_parser.py @@ -0,0 +1,45 @@ +"""project parser module""" + +from typing import Any, Dict, List + +import yaml + +from dfetch_hub.project.remote_datasource import RemoteProject + + +class ProjectParser: + """class that parses python projects + + - used on projects found by project finder + - parsed into sources which can be stored and monitored + """ + + def __init__(self) -> None: + self._projects: List[RemoteProject] = [] + + def add_project(self, new_project: RemoteProject) -> None: + """add a project""" + if new_project not in self._projects: + self._projects.append(new_project) + + def get_projects(self) -> List[RemoteProject]: + """get all projects""" + return self._projects + + def get_projects_as_yaml(self) -> str: + """get yaml representation of projects""" + yaml_obj: Dict[str, List[Dict[str, Any]]] = { + "projects": [project.as_yaml() for project in self._projects] + } + return str(yaml.dump(yaml_obj)) + + @classmethod + def from_yaml(cls, yaml_file: str) -> "ProjectParser": + """create parser from yaml file""" + with open(yaml_file, "r", encoding="utf-8") as yamlf: + instance = cls() + yaml_data = yaml.safe_load(yamlf.read()) + for project in yaml_data["projects"]: + parsed_project = RemoteProject.from_yaml(project) + instance.add_project(parsed_project) + return instance diff --git a/dfetch_hub/project/project_sources.py b/dfetch_hub/project/project_sources.py new file mode 100644 index 0000000..5aa8c10 --- /dev/null +++ b/dfetch_hub/project/project_sources.py @@ -0,0 +1,113 @@ +"""project sources module""" + +from argparse import Namespace +from typing import Any, Dict, List, Optional, Union + +import yaml +from dfetch.manifest.remote import Remote + +from dfetch_hub.project.input_parser import InputParser + + +class RemoteSource(Remote): # type: ignore + """class representing source for projects""" + + def __init__(self, args: Union[Namespace, Dict[str, str]]): + super().__init__(args) + self.exclusions: List[str] = [] + + def add_exclusion(self, exclusion_regex: str) -> None: + """add exclusion to project source""" + self.exclusions.append(exclusion_regex) + + def as_yaml(self) -> Dict[str, Any]: + """get yaml representation""" + yaml_data = super().as_yaml() + yaml_data["exclusions"] = self.exclusions + return {k: v for k, v in yaml_data.items() if v} + + def __eq__(self, other: Any) -> bool: + if isinstance(other, RemoteSource): + if hasattr(self, "exclusions"): + if not hasattr(other, "exclusions"): + return False + return ( + self.name == other.name + and self.url == other.url + and self.exclusions == other.exclusions + ) + if hasattr(other, "exclusions"): + return False + return bool(self.name == other.name and self.url == other.url) + return False + + +class SourceList: + """class representing a sequence of project sources""" + + CURRENT_VERSION = "0.0" + + def __init__(self) -> None: + self._sources: List[RemoteSource] = [] + + def add_remote(self, source: RemoteSource) -> None: + """add source""" + self._sources.append(source) + + def get_remotes(self) -> List[RemoteSource]: + """get list of sources""" + return self._sources + + def as_yaml(self) -> str: + """yaml representation""" + versiondata = {"version": self.CURRENT_VERSION} + remotes_data = {"remotes": [source.as_yaml() for source in self._sources]} + yamldata = {"source-list": [versiondata, remotes_data]} + return str(yaml.dump(yamldata)) + + @classmethod + def from_yaml(cls, yaml_data: Union[str, bytes]) -> "SourceList": + """load from sources files""" + if not yaml_data: + raise ValueError("failed to load data from file") + + instance = cls() + + parsed_yaml: Optional[Dict[str, Any]] = yaml.safe_load(yaml_data) + if not parsed_yaml: + raise RuntimeError("file should have data") + assert parsed_yaml["source-list"], "file should have list of sources" + version = [i["version"] for i in parsed_yaml["source-list"] if "version" in i][ + 0 + ] + remotes = [i["remotes"] for i in parsed_yaml["source-list"] if "remotes" in i][ + 0 + ] + if version != cls.CURRENT_VERSION: + raise ValueError("invalid version") + + for source in remotes: + src = RemoteSource({"name": source["name"], "url-base": source["url-base"]}) + if "exclusions" in source: + for excl in source["exclusions"]: + src.add_exclusion(excl) + instance.add_remote(src) + return instance + + @classmethod + def from_input_parser(cls, parser: InputParser) -> "SourceList": + """generate instance from parser""" + instance = cls() + for url in parser.get_urls(): + name = url.split("/")[-1] + src = RemoteSource({"name": name, "url-base": url}) + instance.add_remote(src) + return instance + + def __eq__(self, other: Any) -> bool: + if isinstance(other, SourceList): + return ( + self._sources == other._sources + and self.CURRENT_VERSION == other.CURRENT_VERSION + ) + return False diff --git a/dfetch_hub/project/remote_datasource.py b/dfetch_hub/project/remote_datasource.py new file mode 100644 index 0000000..e0781e5 --- /dev/null +++ b/dfetch_hub/project/remote_datasource.py @@ -0,0 +1,147 @@ +"""remote datasource module""" + +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Sequence, Tuple + + +@dataclass +class RemoteRef: + """representation of a single remote reference""" + + name: str + revision: str # chosen for dfetch naming + + def as_yaml(self) -> Dict[str, str]: + """yaml representation of reference""" + yamldata = {"name": self.name, "revision": self.revision} + return {k: v for k, v in yamldata.items() if v} + + def __eq__(self, other: Any) -> bool: + if isinstance(other, str): + return self.name == other + if isinstance(other, RemoteRef): + return self.name == other.name and self.revision == other.revision + return False + + +class RemoteProjectVersions: + """representation of collection of versions for project""" + + def __init__(self, vcs: str = ""): + self.tags: List[RemoteRef] = [] + self.branches: List[RemoteRef] = [] + self.vcs = vcs + + def add_tags(self, tags: Sequence[Tuple[str, str]]) -> None: + """add tags""" + for hash_val, tag_name in tags: + if tag_name not in [tag.name for tag in self.tags]: + self.tags.append(RemoteRef(tag_name, hash_val)) + + def add_branches(self, branches: Sequence[Tuple[str, str]]) -> None: + """add branches""" + for hash_val, branch_name in branches: + if branch_name not in [branch.name for branch in self.branches]: + self.branches.append(RemoteRef(branch_name, hash_val)) + + @property + def default(self) -> str: + """get default branch""" + if not self.vcs or self.vcs == "git": + return "main" if "main" in self.branches else "master" + if self.vcs == "svn": + return "trunk" + raise ValueError("no default version known for repository") + + def as_yaml(self) -> Dict[str, Any]: + """get yaml representation""" + default = None + try: + default = self.default + except ValueError: + pass + yamldata = { + "default": default, + "tags": [tag.as_yaml() for tag in self.tags], + "branches": [branch.as_yaml() for branch in self.branches], + } + return {k: v for k, v in yamldata.items() if v} + + def __eq__(self, other: Any) -> bool: + if isinstance(other, RemoteProjectVersions): + return other.branches == self.branches and other.tags == self.tags + return False + + +class RemoteProject: + """representation of remote repository project""" + + def __init__( + self, + name: str, + url: str, + repo_path: str, + src: str, + vcs: str, + versions: Optional[RemoteProjectVersions] = None, + ): # pylint:disable=too-many-arguments,too-many-positional-arguments + self.name = name + self.url = url + self.repo_path = repo_path + self.src = src + self.vcs = vcs + self.versions = versions if versions else RemoteProjectVersions() + + def add_versions( + self, branches: Sequence[Tuple[str, str]], tags: Sequence[Tuple[str, str]] + ) -> None: + """add branches and tags""" + if not hasattr(self, "versions"): + self.versions = RemoteProjectVersions(self.vcs) + self.versions.add_branches(branches) + self.versions.add_tags(tags) + + def as_yaml(self) -> Dict[str, Any]: + """get yaml representation""" + yamldata = { + "name": self.name, + "versions": ( + None if not hasattr(self, "versions") else self.versions.as_yaml() + ), + "src": self.src, + "url": self.url, + "repo-path": self.repo_path, + "vcs": None if not hasattr(self, "vcs") else self.vcs, + } + return {k: v for k, v in yamldata.items() if v} + + @classmethod + def from_yaml(cls, yaml_data: Dict[str, Any]) -> "RemoteProject": + """build project from yaml representation""" + src = "" if "src" not in yaml_data else yaml_data["src"] + versions = None if "versions" not in yaml_data else yaml_data["versions"] + parsed = cls( + yaml_data["name"], + yaml_data["url"], + yaml_data["repo-path"], + src, + vcs=yaml_data["vcs"], + ) + if versions: + branches = [ + (branch["revision"], branch["name"]) for branch in versions["branches"] + ] + tags = [(tag["revision"], tag["name"]) for tag in versions["tags"]] + parsed.add_versions(branches=branches, tags=tags) + return parsed + + def __eq__(self, other: Any) -> bool: + if isinstance(other, str): + return other == self.name + if isinstance(other, RemoteProject): + return ( + other.name == self.name + and other.url == self.url + and other.repo_path == self.repo_path + ) + return False diff --git a/doc/index.md b/doc/index.md new file mode 100644 index 0000000..755cb35 --- /dev/null +++ b/doc/index.md @@ -0,0 +1,35 @@ +# Remote Project Overview + +- [Remote Project Overview](#remote-project-overview) + - [Idea](#idea) + - [File types](#file-types) + - [Source list](#source-list) + - [Project lists](#project-lists) + - [Output file](#output-file) + +## Idea + +- url to list project + versions +- dfetch.yaml formatter to output selected items + +## File types + +### Source list + +Source list is a type used to list a collection of project sources. +A project source is a place to look for projects. +Optionally a list of exclusions can be added so certain patterns that look like potential projects but are not projects are excluded. + +The source list can be used to share project locations and exclusions between users. + +### Project lists + +The project list is the main overview containing all the projects in the sources. +It is populated by the information gathered by the `ProjectFinder` when it is passed to the `ProjectParser` by looking in the project sources locations and parsing the information there. +The `ProjectParser` is the module that can use a project list as a source of projects and generate entries for dependency fetchers. +The current use case in mind is `dFetch`, but in the future e.g. `git submodule`'s or other dependency fetchers could be used. + +### Output file + +The output of this project is a source file which can be used by other tools to import the selected projects. +Our first use case is to create `dFetch` manifest files, which can then be used to import the selected versions of projects into other projects. diff --git a/doc/program_components.puml b/doc/program_components.puml new file mode 100644 index 0000000..b139e0c --- /dev/null +++ b/doc/program_components.puml @@ -0,0 +1,31 @@ +@startuml program_components +!include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Component.puml + +Person(user, "user", "person that uses project") + +Container(input_parser, "input parser", "parses input to program project sources") +Container(project_finder, "project finder", "searches repositories for projects") +Container(project_parser, "project parser", "parses repositories data to project data") +Container(export, "export", "select data and export") +Container_Ext(dfetch, "fetch dependencies", "update project with requested dependencies") + +System(project_sources, "project sources file", "contains project sources") +System(project_list, "project data list", "contains all projects for given sources") +System(output_file, "dfetch (or other format) outfile", "contains required info to get project") + +Rel(user, input_parser, "add input", "url, dfetch file, ...") + +Rel(input_parser, project_sources, "generates", "persist data") +Rel_R(input_parser, project_finder, "add parsed sources", "add info required to parse projects") + +Rel_U(project_sources, project_finder, "add parsed sources", "add info required to parse projects") + +Rel_R(project_finder, project_parser, "info to create projects") +Rel(project_parser, project_list, "generates", "persist data") + +Rel_R(project_parser, export, "forward project versions") +Rel(export, output_file, "generates", "persist data") + +Rel(output_file, dfetch, "let dfetch use", "dfetch manifest") + +@enduml diff --git a/projects.yaml b/projects.yaml new file mode 100644 index 0000000..cc7d545 --- /dev/null +++ b/projects.yaml @@ -0,0 +1,315 @@ +projects: +- name: dfetch + repo-path: dfetch-org/dfetch + url: github.com + vcs: git + versions: + branches: + - name: HEAD + revision: c037c000e200704f3db1ac6af1aceeb79cb1954c + - name: add-more-checkers + revision: fa464794a55d2bc2e64a029a8ac1152d1e057088 + - name: dependabot/docker/dot-devcontainer/devcontainers/python-3.13-bullseye + revision: 4777980b443154f0c00f32f5e77a1d2a5d8dec61 + - name: dependabot/pip/main/cyclonedx-python-lib-5.0.1 + revision: 4cd13444f74c7817e212f043dc1e3a75bab8e379 + - name: main + revision: c037c000e200704f3db1ac6af1aceeb79cb1954c + - name: prototype-extended-data + revision: 635cf9711577f446b1a7c9d4048bd6a3854eb872 + default: main + tags: + - name: 0.1.0 + revision: 3733a64356b8899b43d4c9145a7c7ca8aa529927 + - name: 0.1.1 + revision: 7ef62b1cf04f1f3c30a57b0138b5d7b50ec076b7 + - name: 0.2.0 + revision: 3cd50945cb736d8aadce5d8194ea746f454b0e55 + - name: 0.3.0 + revision: 74f1bb642800ff6f561d8bcd1aad01fce7e993e4 + - name: 0.4.0 + revision: 4d960baa0725bf805fecd08ad9f60a4f706e4be0 + - name: 0.5.1 + revision: 0422cae6f3d7f46f8667af3fce67b607dca076df + - name: 0.6.0 + revision: ad0a73489e762e83f911deabe18f6764678bb013 + - name: 0.7.0 + revision: 192466458d69530a149311a98ed95efe51dbcf90 + - name: 0.8.0 + revision: 43319554160abb9ebad64614fd1c8d2beaa2296e + - name: 0.9.0 + revision: 069c65f74000ef0f906fc5574023f6d2eaeb6c5b + - name: 0.9.1 + revision: 73a84be9c02d492d6d39bbaaf9df218696f56ab1 + - name: v0.0.1 + revision: b4c04095990e2abc18a925e82c678d50d90f16af + - name: v0.0.2 + revision: 91aa476b8cd63ae2faf4de19d078f95e29b16bd3 + - name: v0.0.3 + revision: cecd0f2752a25f5af7e2568dac0a1b0432a149c6 + - name: v0.0.4 + revision: 9dbca3302925062e42740bccab00051a112b9cf6 + - name: v0.0.5 + revision: 9267fb1125e2efc05c32b4f9250a2cedf395753f + - name: v0.0.6 + revision: 7e6933bded164a917cbb1a510c67d976656263d2 + - name: v0.0.7 + revision: 1e47a197fdf23cd55ba58d4b7b3a2491cb550302 + - name: v0.0.8 + revision: f8a4541891194c07b953ef6af2c251fe40a8987e + - name: v0.0.9 + revision: 422c7d9d9578ccb8d79d8a6b14dac6f6e30bc4f8 + - name: v0.5.0 + revision: fbedf548fc2929250bdcc4252af298658b36b01c +- name: demo-magic + repo-path: dfetch-org/dfetch + src: doc/generate-casts/demo-magic + url: github.com + vcs: git + versions: + branches: + - name: HEAD + revision: c037c000e200704f3db1ac6af1aceeb79cb1954c + - name: add-more-checkers + revision: fa464794a55d2bc2e64a029a8ac1152d1e057088 + - name: dependabot/docker/dot-devcontainer/devcontainers/python-3.13-bullseye + revision: 4777980b443154f0c00f32f5e77a1d2a5d8dec61 + - name: dependabot/pip/main/cyclonedx-python-lib-5.0.1 + revision: 4cd13444f74c7817e212f043dc1e3a75bab8e379 + - name: main + revision: c037c000e200704f3db1ac6af1aceeb79cb1954c + - name: prototype-extended-data + revision: 635cf9711577f446b1a7c9d4048bd6a3854eb872 + default: main + tags: + - name: 0.1.0 + revision: 3733a64356b8899b43d4c9145a7c7ca8aa529927 + - name: 0.1.1 + revision: 7ef62b1cf04f1f3c30a57b0138b5d7b50ec076b7 + - name: 0.2.0 + revision: 3cd50945cb736d8aadce5d8194ea746f454b0e55 + - name: 0.3.0 + revision: 74f1bb642800ff6f561d8bcd1aad01fce7e993e4 + - name: 0.4.0 + revision: 4d960baa0725bf805fecd08ad9f60a4f706e4be0 + - name: 0.5.1 + revision: 0422cae6f3d7f46f8667af3fce67b607dca076df + - name: 0.6.0 + revision: ad0a73489e762e83f911deabe18f6764678bb013 + - name: 0.7.0 + revision: 192466458d69530a149311a98ed95efe51dbcf90 + - name: 0.8.0 + revision: 43319554160abb9ebad64614fd1c8d2beaa2296e + - name: 0.9.0 + revision: 069c65f74000ef0f906fc5574023f6d2eaeb6c5b + - name: 0.9.1 + revision: 73a84be9c02d492d6d39bbaaf9df218696f56ab1 + - name: v0.0.1 + revision: b4c04095990e2abc18a925e82c678d50d90f16af + - name: v0.0.2 + revision: 91aa476b8cd63ae2faf4de19d078f95e29b16bd3 + - name: v0.0.3 + revision: cecd0f2752a25f5af7e2568dac0a1b0432a149c6 + - name: v0.0.4 + revision: 9dbca3302925062e42740bccab00051a112b9cf6 + - name: v0.0.5 + revision: 9267fb1125e2efc05c32b4f9250a2cedf395753f + - name: v0.0.6 + revision: 7e6933bded164a917cbb1a510c67d976656263d2 + - name: v0.0.7 + revision: 1e47a197fdf23cd55ba58d4b7b3a2491cb550302 + - name: v0.0.8 + revision: f8a4541891194c07b953ef6af2c251fe40a8987e + - name: v0.0.9 + revision: 422c7d9d9578ccb8d79d8a6b14dac6f6e30bc4f8 + - name: v0.5.0 + revision: fbedf548fc2929250bdcc4252af298658b36b01c +- name: generate-casts + repo-path: dfetch-org/dfetch + src: doc/generate-casts + url: github.com + vcs: git + versions: + branches: + - name: HEAD + revision: c037c000e200704f3db1ac6af1aceeb79cb1954c + - name: add-more-checkers + revision: fa464794a55d2bc2e64a029a8ac1152d1e057088 + - name: dependabot/docker/dot-devcontainer/devcontainers/python-3.13-bullseye + revision: 4777980b443154f0c00f32f5e77a1d2a5d8dec61 + - name: dependabot/pip/main/cyclonedx-python-lib-5.0.1 + revision: 4cd13444f74c7817e212f043dc1e3a75bab8e379 + - name: main + revision: c037c000e200704f3db1ac6af1aceeb79cb1954c + - name: prototype-extended-data + revision: 635cf9711577f446b1a7c9d4048bd6a3854eb872 + default: main + tags: + - name: 0.1.0 + revision: 3733a64356b8899b43d4c9145a7c7ca8aa529927 + - name: 0.1.1 + revision: 7ef62b1cf04f1f3c30a57b0138b5d7b50ec076b7 + - name: 0.2.0 + revision: 3cd50945cb736d8aadce5d8194ea746f454b0e55 + - name: 0.3.0 + revision: 74f1bb642800ff6f561d8bcd1aad01fce7e993e4 + - name: 0.4.0 + revision: 4d960baa0725bf805fecd08ad9f60a4f706e4be0 + - name: 0.5.1 + revision: 0422cae6f3d7f46f8667af3fce67b607dca076df + - name: 0.6.0 + revision: ad0a73489e762e83f911deabe18f6764678bb013 + - name: 0.7.0 + revision: 192466458d69530a149311a98ed95efe51dbcf90 + - name: 0.8.0 + revision: 43319554160abb9ebad64614fd1c8d2beaa2296e + - name: 0.9.0 + revision: 069c65f74000ef0f906fc5574023f6d2eaeb6c5b + - name: 0.9.1 + revision: 73a84be9c02d492d6d39bbaaf9df218696f56ab1 + - name: v0.0.1 + revision: b4c04095990e2abc18a925e82c678d50d90f16af + - name: v0.0.2 + revision: 91aa476b8cd63ae2faf4de19d078f95e29b16bd3 + - name: v0.0.3 + revision: cecd0f2752a25f5af7e2568dac0a1b0432a149c6 + - name: v0.0.4 + revision: 9dbca3302925062e42740bccab00051a112b9cf6 + - name: v0.0.5 + revision: 9267fb1125e2efc05c32b4f9250a2cedf395753f + - name: v0.0.6 + revision: 7e6933bded164a917cbb1a510c67d976656263d2 + - name: v0.0.7 + revision: 1e47a197fdf23cd55ba58d4b7b3a2491cb550302 + - name: v0.0.8 + revision: f8a4541891194c07b953ef6af2c251fe40a8987e + - name: v0.0.9 + revision: 422c7d9d9578ccb8d79d8a6b14dac6f6e30bc4f8 + - name: v0.5.0 + revision: fbedf548fc2929250bdcc4252af298658b36b01c +- name: plantuml-c4 + repo-path: dfetch-org/dfetch + src: doc/static/uml/styles/plantuml-c4 + url: github.com + vcs: git + versions: + branches: + - name: HEAD + revision: c037c000e200704f3db1ac6af1aceeb79cb1954c + - name: add-more-checkers + revision: fa464794a55d2bc2e64a029a8ac1152d1e057088 + - name: dependabot/docker/dot-devcontainer/devcontainers/python-3.13-bullseye + revision: 4777980b443154f0c00f32f5e77a1d2a5d8dec61 + - name: dependabot/pip/main/cyclonedx-python-lib-5.0.1 + revision: 4cd13444f74c7817e212f043dc1e3a75bab8e379 + - name: main + revision: c037c000e200704f3db1ac6af1aceeb79cb1954c + - name: prototype-extended-data + revision: 635cf9711577f446b1a7c9d4048bd6a3854eb872 + default: main + tags: + - name: 0.1.0 + revision: 3733a64356b8899b43d4c9145a7c7ca8aa529927 + - name: 0.1.1 + revision: 7ef62b1cf04f1f3c30a57b0138b5d7b50ec076b7 + - name: 0.2.0 + revision: 3cd50945cb736d8aadce5d8194ea746f454b0e55 + - name: 0.3.0 + revision: 74f1bb642800ff6f561d8bcd1aad01fce7e993e4 + - name: 0.4.0 + revision: 4d960baa0725bf805fecd08ad9f60a4f706e4be0 + - name: 0.5.1 + revision: 0422cae6f3d7f46f8667af3fce67b607dca076df + - name: 0.6.0 + revision: ad0a73489e762e83f911deabe18f6764678bb013 + - name: 0.7.0 + revision: 192466458d69530a149311a98ed95efe51dbcf90 + - name: 0.8.0 + revision: 43319554160abb9ebad64614fd1c8d2beaa2296e + - name: 0.9.0 + revision: 069c65f74000ef0f906fc5574023f6d2eaeb6c5b + - name: 0.9.1 + revision: 73a84be9c02d492d6d39bbaaf9df218696f56ab1 + - name: v0.0.1 + revision: b4c04095990e2abc18a925e82c678d50d90f16af + - name: v0.0.2 + revision: 91aa476b8cd63ae2faf4de19d078f95e29b16bd3 + - name: v0.0.3 + revision: cecd0f2752a25f5af7e2568dac0a1b0432a149c6 + - name: v0.0.4 + revision: 9dbca3302925062e42740bccab00051a112b9cf6 + - name: v0.0.5 + revision: 9267fb1125e2efc05c32b4f9250a2cedf395753f + - name: v0.0.6 + revision: 7e6933bded164a917cbb1a510c67d976656263d2 + - name: v0.0.7 + revision: 1e47a197fdf23cd55ba58d4b7b3a2491cb550302 + - name: v0.0.8 + revision: f8a4541891194c07b953ef6af2c251fe40a8987e + - name: v0.0.9 + revision: 422c7d9d9578ccb8d79d8a6b14dac6f6e30bc4f8 + - name: v0.5.0 + revision: fbedf548fc2929250bdcc4252af298658b36b01c +- name: features + repo-path: dfetch-org/dfetch + src: features + url: github.com + vcs: git + versions: + branches: + - name: HEAD + revision: c037c000e200704f3db1ac6af1aceeb79cb1954c + - name: add-more-checkers + revision: fa464794a55d2bc2e64a029a8ac1152d1e057088 + - name: dependabot/docker/dot-devcontainer/devcontainers/python-3.13-bullseye + revision: 4777980b443154f0c00f32f5e77a1d2a5d8dec61 + - name: dependabot/pip/main/cyclonedx-python-lib-5.0.1 + revision: 4cd13444f74c7817e212f043dc1e3a75bab8e379 + - name: main + revision: c037c000e200704f3db1ac6af1aceeb79cb1954c + - name: prototype-extended-data + revision: 635cf9711577f446b1a7c9d4048bd6a3854eb872 + default: main + tags: + - name: 0.1.0 + revision: 3733a64356b8899b43d4c9145a7c7ca8aa529927 + - name: 0.1.1 + revision: 7ef62b1cf04f1f3c30a57b0138b5d7b50ec076b7 + - name: 0.2.0 + revision: 3cd50945cb736d8aadce5d8194ea746f454b0e55 + - name: 0.3.0 + revision: 74f1bb642800ff6f561d8bcd1aad01fce7e993e4 + - name: 0.4.0 + revision: 4d960baa0725bf805fecd08ad9f60a4f706e4be0 + - name: 0.5.1 + revision: 0422cae6f3d7f46f8667af3fce67b607dca076df + - name: 0.6.0 + revision: ad0a73489e762e83f911deabe18f6764678bb013 + - name: 0.7.0 + revision: 192466458d69530a149311a98ed95efe51dbcf90 + - name: 0.8.0 + revision: 43319554160abb9ebad64614fd1c8d2beaa2296e + - name: 0.9.0 + revision: 069c65f74000ef0f906fc5574023f6d2eaeb6c5b + - name: 0.9.1 + revision: 73a84be9c02d492d6d39bbaaf9df218696f56ab1 + - name: v0.0.1 + revision: b4c04095990e2abc18a925e82c678d50d90f16af + - name: v0.0.2 + revision: 91aa476b8cd63ae2faf4de19d078f95e29b16bd3 + - name: v0.0.3 + revision: cecd0f2752a25f5af7e2568dac0a1b0432a149c6 + - name: v0.0.4 + revision: 9dbca3302925062e42740bccab00051a112b9cf6 + - name: v0.0.5 + revision: 9267fb1125e2efc05c32b4f9250a2cedf395753f + - name: v0.0.6 + revision: 7e6933bded164a917cbb1a510c67d976656263d2 + - name: v0.0.7 + revision: 1e47a197fdf23cd55ba58d4b7b3a2491cb550302 + - name: v0.0.8 + revision: f8a4541891194c07b953ef6af2c251fe40a8987e + - name: v0.0.9 + revision: 422c7d9d9578ccb8d79d8a6b14dac6f6e30bc4f8 + - name: v0.5.0 + revision: fbedf548fc2929250bdcc4252af298658b36b01c diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c24ddd8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,57 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "dfetch_hub" +version = "0.0.1" +authors = [ + { name="p sacharias", email="p.sacharias@gmail.com" }, + { name="ben spoor", email="b.spoor@edna.eu" }, +] +description = "Dfetch Hub" +readme = "README.md" +requires-python = ">=3.8" +classifiers = [ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", +] +dependencies = [ + "dfetch==0.10.0", + "PyYAML==6.0.2" +] + +[project.optional-dependencies] # https://packaging.python.org/en/latest/guides/writing-pyproject-toml/#writing-pyproject-toml +development = [ + "black==25.1.0", + "isort==6.0.0", + "mypy==1.14.1", + "pre-commit==4.1.0", + "pylint==3.3.4", + "pytest==8.3.4", + "pytest-cov==6.0.0", + "vcrpy==7.0.0", + "types-PyYaml==6.0.12" +] +gui = [ + "thefuzz==0.22.1", + "nicegui==2.11.1" +] +[project.urls] +Homepage = "https://github.com/dfetch-org/dfetch-hub" +Issues = "https://github.com/dfetch-org/dfetch-hub/issues" + +[project.scripts] +DfetchHub-cli = "dfetch_hub.project.cli:main_cli" + +[tool.pylint.format] +max-line-length = "120" + +[tool.mypy] +files = "dfetch_hub" +ignore_missing_imports = true +strict = true + +[[tool.mypy.overrides]] +module = "dfetch_hub.example_gui.gui" +disable_error_code = ["attr-defined"] diff --git a/test/test_cli.py b/test/test_cli.py new file mode 100644 index 0000000..1dd6ffb --- /dev/null +++ b/test/test_cli.py @@ -0,0 +1,36 @@ +"""test cli module""" + +import pytest +from test_common import ParserMock + +from dfetch_hub.project.cli import main + + +@pytest.fixture +def parser_no_args(): + return ParserMock() + + +@pytest.fixture +def parser_url(): + return ParserMock(url="https://github.com/cpputest/") + + +@pytest.fixture +def parser_dfetch(): + return ParserMock(dfetch_source="test/dfetch.yaml") + + +def test_no_parameters(parser_no_args): + with pytest.raises(ValueError): + main(parser_no_args) + + +def test_url_parameter(parser_url): + with pytest.raises(ValueError): + main(parser_url) + + +def test_dfetch_parameter(parser_dfetch): + with pytest.raises(FileNotFoundError): + main(parser_dfetch) diff --git a/test/test_common.py b/test/test_common.py new file mode 100644 index 0000000..29754bf --- /dev/null +++ b/test/test_common.py @@ -0,0 +1,20 @@ +"""common test utilities file""" + + +class Args: + def __init__(self, url=None, dfetch_source=None): + self.url = url + self.dfetch_source = dfetch_source + self.project_exclude_pattern = [] + self.persist_sources = False + + +class ParserMock: + def __init__(self, url=None, dfetch_source=None): + self.args = Args(url, dfetch_source) + + def parse_args(self): + return self.args + + def print_help(self): + print("help") diff --git a/test/test_dfetch_export.py b/test/test_dfetch_export.py new file mode 100644 index 0000000..036bee3 --- /dev/null +++ b/test/test_dfetch_export.py @@ -0,0 +1,66 @@ +"""test dfetch export functionality""" + +import os +from dataclasses import dataclass + +import pytest + +from dfetch_hub.project.export import DfetchExport + + +@dataclass +class EntryMock: + name: str + revision: str + src: str + url: str + repo_path: str + vcs: str = "git" + + +@pytest.fixture +def entry(): + return EntryMock( + "test", "0123456789abcdef", "/src", "http://test.test", "test_path" + ) + + +@pytest.fixture +def entries(): + entry = EntryMock( + "test", "0123456789abcdef", "/src", "http://test.test", "test_path" + ) + entry2 = EntryMock( + "test2", "0123456789abcdef2", "/src2", "http://test.test2", "test_path2" + ) + return [entry, entry2] + + +def test_add_entry(entry): + export = DfetchExport() + export.add_entry(entry) + assert len(export.entries) == 1 + assert entry in export.entries + + +def test_from_entry(entry): + export = DfetchExport([entry]) + assert len(export.entries) == 1 + assert entry in export.entries + + +def test_multiple_entries(entries): + export = DfetchExport() + for entry in entries: + export.add_entry(entry) + assert len(export.entries) == len(entries) + for entry in entries: + assert entry in export.entries + + +def test_yaml_file(entries): + export = DfetchExport() + for entry in entries: + export.add_entry(entry) + export.export("test/testdata/dfetch_export.yaml") + assert os.path.exists("test/testdata/dfetch_export.yaml") diff --git a/test/test_input_parser.py b/test/test_input_parser.py new file mode 100644 index 0000000..2e17d99 --- /dev/null +++ b/test/test_input_parser.py @@ -0,0 +1,39 @@ +"""test input parser functionality""" + +import pytest +from test_common import Args + +from dfetch_hub.project.input_parser import InputParser + + +def test_input_url(): + url = "http://www.github.com" + mock = Args(url) + assert [url] == InputParser(mock).get_urls() + + +def test_input_multiple_url(): + url = ["http://www.github.com", "http://www.example.com"] + mock = Args(url) + assert url == InputParser(mock).get_urls() + + +def test_input_dfetch_single_remote(): + dfetch_file_name = "test/testdata/dfetch00.yaml" + mock = Args(dfetch_source=dfetch_file_name) + assert [ + "https://github.com/cpputest/cpputest.git", + "https://github.com/zserge/jsmn.git", + ] == InputParser(mock).get_urls() + + +def test_input_dfetch_multiple_remotes(): + dfetch_file_name = "test/testdata/dfetch01.yaml" + mock = Args(dfetch_source=dfetch_file_name) + urls = InputParser(mock).get_urls() + assert 3 == len(urls) + assert [ + "https://github.com/cpputest/cpputest.git", + "https://github.com/zserge/jsmn.git", + "https://gitlab.com/ShacharKraus/pyfixed", + ] == urls diff --git a/test/test_project_finder.py b/test/test_project_finder.py new file mode 100644 index 0000000..6f46a81 --- /dev/null +++ b/test/test_project_finder.py @@ -0,0 +1,50 @@ +"""test project finder functionality""" + +import pytest + +from dfetch_hub.project.project_finder import GitProjectFinder + + +def test_find_cpputest_github(): + url = "https://github.com/cpputest/cpputest.git" + gpf = GitProjectFinder(url) + projects = gpf.list_projects() + assert len(projects) == 13 + assert "cpputest.git" in projects + assert "examples" in projects + assert "CppUTest" in projects + + +def test_find_jasmine_github(): + url = "https://github.com/zserge/jsmn.git" + gpf = GitProjectFinder(url) + projects = gpf.list_projects() + assert len(projects) == 1 + assert "jsmn.git" in projects + + +def test_find_pyfixed_gitlab(): + url = "https://gitlab.com/ShacharKraus/pyfixed" + gpf = GitProjectFinder(url) + projects = gpf.list_projects() + assert len(projects) == 1 + assert "pyfixed" in projects + + +def test_find_cpputest_github_exclusion_filer(): + url = "https://github.com/cpputest/cpputest.git" + exclusions = ["platforms.*", ".*examples.*", "scripts"] + gpf = GitProjectFinder(url, exclusions=exclusions) + projects = gpf.list_projects() + assert len(projects) == 3 + assert "cpputest.git" in projects + assert "CppUTest" in projects + assert "Symbian" in projects + + +def test_find_cpputest_github_invalid_regex(): + url = "https://github.com/cpputest/cpputest.git" + exclusions = ["*examples*"] + with pytest.raises(ValueError): + gpf = GitProjectFinder(url, exclusions=exclusions) + projects = gpf.list_projects() diff --git a/test/test_project_parser.py b/test/test_project_parser.py new file mode 100644 index 0000000..eb04e13 --- /dev/null +++ b/test/test_project_parser.py @@ -0,0 +1,120 @@ +"""test project parser functionality""" + +import pytest +import yaml + +from dfetch_hub.project.project_parser import ProjectParser, RemoteProject + + +def get_sample_project(): + name = "CppUTest" + url = "https://github.com/cpputest/" + src = "" + repo_path = "cpputest.git" + vcs = "git" + return RemoteProject(name, url, repo_path, src, vcs) + + +@pytest.fixture +def project(): + return get_sample_project() + + +@pytest.fixture +def project_w_version(): + project = get_sample_project() + tags = [ + ("aabbccddeeff", "1.0.0"), + ("aaaaaaaaaaaa", "1.1.1"), + ("bbbbbbbbbbbb", "best_tag_ever"), + ] + branches = [ + ("aabbccddaabb", "dev"), + ("bbccddeeffaa", "master"), + ("ccddeeffaabb", "some_old_forgotten_branch"), + ] + project.add_versions(branches, tags) + return project + + +@pytest.fixture +def projects_w_version(): + project_1 = get_sample_project() + project_2 = get_sample_project() + tags1 = [ + ("aabbccddeeff", "1.0.0"), + ("aaaaaaaaaaaa", "1.1.1"), + ("bbbbbbbbbbbb", "best_tag_ever"), + ] + tags2 = [ + ("aabbccddeeff", "2.0.0"), + ("bbddeeffaacc", "2.1.1"), + ("0123456789a", "worst_tag_ever"), + ] + branches = [ + ("aabbccddaabb", "dev"), + ("bbccddeeffaa", "master"), + ("ccddeeffaabb", "some_old_forgotten_branch"), + ] + project_1.add_versions(branches, tags1) + project_2.add_versions(branches, tags2) + return (project_1, project_2) + + +def test_add_project(project): + parser = ProjectParser() + parser.add_project(project) + assert len(parser.get_projects()) == 1 + assert parser.get_projects()[0] == project + + +def test_get_as_yaml(project): + parser = ProjectParser() + parser.add_project(project) + yaml_proj = parser.get_projects_as_yaml() + assert yaml.load(yaml_proj, yaml.Loader) + + +def test_from_yaml(project): + parser = ProjectParser() + yaml_file = "test/testdata/versions.yaml" + parser = ProjectParser.from_yaml(yaml_file) + assert len(parser.get_projects()) == 1 + assert parser.get_projects()[0] == project + + +def test_project_version(project_w_version): + parser = ProjectParser() + project = project_w_version + parser.add_project(project) + assert len(parser.get_projects()) == 1 + assert parser.get_projects()[0] == project + assert "1.0.0" in project.versions.tags + assert "dev" in project.versions.branches + + +def test_two_projects_version(projects_w_version): + parser = ProjectParser() + project1, project2 = projects_w_version + parser.add_project(project1) + parser.add_project(project2) + assert len(parser.get_projects()) == 1 + assert parser.get_projects()[0] == project1 + assert "1.0.0" in project1.versions.tags + assert "dev" in project1.versions.branches + assert "2.0.0" not in project1.versions.tags + assert "2.0.0" in project2.versions.tags + assert "dev" in project2.versions.branches + + +def test_versions_from_yaml(projects_w_version): + parser = ProjectParser() + yaml_file = "test/testdata/versions01.yaml" + parser = ProjectParser.from_yaml(yaml_file) + project1, project2 = projects_w_version + project2.name = "project_2" + project_1_from_file = parser.get_projects()[0] + assert len(parser.get_projects()) == 2 + assert project_1_from_file == project1 + assert parser.get_projects()[1] == project2 + assert project_1_from_file.versions == project1.versions diff --git a/test/test_project_sources.py b/test/test_project_sources.py new file mode 100644 index 0000000..6920eb0 --- /dev/null +++ b/test/test_project_sources.py @@ -0,0 +1,73 @@ +"""test project sources functionality""" + +import pytest + +from dfetch_hub.project.project_sources import RemoteSource, SourceList + + +@pytest.fixture +def input_parser(): + class InputParserMock: + def get_urls(self): + return ["https://github.com/cpputest/cpputest.git"] + + return InputParserMock() + + +def test_add_remote(): + sl = SourceList() + ps = RemoteSource( + {"name": "cpputest", "url-base": "https://github.com/cpputest/cpputest.git"} + ) + sl.add_remote(ps) + assert len(sl.get_remotes()) == 1 + assert sl.get_remotes()[0] == ps + + +def test_add_multiple_remotes(): + sl = SourceList() + ps1 = RemoteSource( + {"name": "cpputest", "url-base": "https://github.com/cpputest/cpputest.git"} + ) + ps2 = RemoteSource( + {"name": "other_repo", "url-base": "https://github.com/cpputest/other_repo.git"} + ) + sl.add_remote(ps1) + sl.add_remote(ps2) + assert len(sl.get_remotes()) == 2 + assert sl.get_remotes() == [ps1, ps2] + + +def test_yaml(): + sl = SourceList() + ps1 = RemoteSource( + {"name": "cpputest", "url-base": "https://github.com/cpputest/cpputest.git"} + ) + ps2 = RemoteSource( + {"name": "other_repo", "url-base": "https://github.com/cpputest/other_repo.git"} + ) + sl.add_remote(ps1) + sl.add_remote(ps2) + sl2 = SourceList.from_yaml(sl.as_yaml()) + assert sl2 == sl + + +def test_input_parser(input_parser): + sl = SourceList.from_input_parser(input_parser) + assert len(sl.get_remotes()) == 1 + ps = RemoteSource( + {"name": "cpputest.git", "url-base": "https://github.com/cpputest/cpputest.git"} + ) + assert ps in sl.get_remotes() + + +def test_yaml_with_exclusions(): + sl = SourceList() + ps = RemoteSource( + {"name": "cpputest", "url-base": "https://github.com/cpputest/cpputest.git"} + ) + ps.add_exclusion("test/.*") + ps.add_exclusion("module*") + sl.add_remote(ps) + sl2 = SourceList.from_yaml(sl.as_yaml()) + assert sl2 == sl diff --git a/test/testdata/dfetch00.yaml b/test/testdata/dfetch00.yaml new file mode 100644 index 0000000..195d971 --- /dev/null +++ b/test/testdata/dfetch00.yaml @@ -0,0 +1,16 @@ +manifest: + version: 0.0 # DFetch Module syntax version + + remotes: # declare common sources in one place + - name: github + url-base: https://github.com/ + + + projects: + - name: cpputest + dst: cpputest/src/ # Destination of this project (relative to this file) + repo-path: cpputest/cpputest.git # Use default github remote + tag: v3.4 # tag + + - name: jsmn # without destination, defaults to project name + repo-path: zserge/jsmn.git # only repo-path is enough diff --git a/test/testdata/dfetch01.yaml b/test/testdata/dfetch01.yaml new file mode 100644 index 0000000..b3c15a9 --- /dev/null +++ b/test/testdata/dfetch01.yaml @@ -0,0 +1,23 @@ +manifest: + version: 0.0 # DFetch Module syntax version + + remotes: # declare common sources in one place + - name: github + url-base: https://github.com/ + + - name: gitlab + url-base: https://gitlab.com/ + + projects: + - name: cpputest + dst: cpputest/src/ # Destination of this project (relative to this file) + repo-path: cpputest/cpputest.git # Use default github remote + tag: v3.4 # tag + + - name: jsmn # without destination, defaults to project name + repo-path: zserge/jsmn.git # only repo-path is enough + + - name: pyfixed + dst: pyfixed + repo-path: ShacharKraus/pyfixed + remote: gitlab \ No newline at end of file diff --git a/test/testdata/dfetch_export.yaml b/test/testdata/dfetch_export.yaml new file mode 100644 index 0000000..ef11d2d --- /dev/null +++ b/test/testdata/dfetch_export.yaml @@ -0,0 +1,15 @@ +manifest: + version: '0.0' + + projects: + - name: test + revision: 0123456789abcdef + src: /src + url: http://test.test + vcs: git + + - name: test2 + revision: 0123456789abcdef2 + src: /src2 + url: http://test.test2 + vcs: git diff --git a/test/testdata/sources00.yaml b/test/testdata/sources00.yaml new file mode 100644 index 0000000..abcf872 --- /dev/null +++ b/test/testdata/sources00.yaml @@ -0,0 +1,3 @@ +sources: +- url: https://github.com/cpputest/cpputest.git +- url: https://github.com/cpputest/other_repo.git diff --git a/test/testdata/versions.yaml b/test/testdata/versions.yaml new file mode 100644 index 0000000..2f4d4b1 --- /dev/null +++ b/test/testdata/versions.yaml @@ -0,0 +1,5 @@ +projects: +- name: CppUTest + repo-path: cpputest.git + url: https://github.com/cpputest/ + vcs: git \ No newline at end of file diff --git a/test/testdata/versions01.yaml b/test/testdata/versions01.yaml new file mode 100644 index 0000000..449e8b1 --- /dev/null +++ b/test/testdata/versions01.yaml @@ -0,0 +1,41 @@ +projects: +- name: CppUTest + repo-path: cpputest.git + url: https://github.com/cpputest/ + vcs: git + versions: + branches: + - revision: aabbccddaabb + name: dev + - revision: bbccddeeffaa + name: master + - revision: ccddeeffaabb + name: some_old_forgotten_branch + default: master + tags: + - revision: aabbccddeeff + name: 1.0.0 + - revision: aaaaaaaaaaaa + name: 1.1.1 + - revision: bbbbbbbbbbbb + name: best_tag_ever +- name: project_2 + repo-path: cpputest.git + url: https://github.com/cpputest/ + vcs: git + versions: + branches: + - revision: aabbccddaabb + name: dev + - revision: bbccddeeffaa + name: master + - revision: ccddeeffaabb + name: some_old_forgotten_branch + default: master + tags: + - revision: aabbccddeeff + name: 2.0.0 + - revision: bbddeeffaacc + name: 2.1.1 + - revision: 0123456789a + name: worst_tag_ever