build: add Taskfile with development tasks (#1867)

Added Taskfile with development tasks

This patch adds a Taskfile with various tasks that is useful for
RDFLib development. Details of the available tasks can be seen from
`task help` output and some more information is included in the RDFLib
developers guide.

For more information about Taskfiles see https://taskfile.dev/#/usage.

The GitHub action pipeline was also changed to now use tasks from the
Taskfile which simplifies the pipeline quite a bit.

This patch also adds development container which can be used to run
various development task and which can also be used with development
container enabled tools such as VSCode or GitHub codespaces.

This is all being done in preperation for moving coveralls reporting to
the GitHub actions workflow.

Other changes:
- Removed the `Makefile` and associated `test/Dockerfile` as it is now replaced by a `Taskfile.yml`.

Author: aucampia

.devcontainer.json

@@ -0,0 +1,23 @@
+{
+  "dockerComposeFile": "docker-compose.yml",
+  "service": "devcontainer",
+  "workspaceFolder": "/srv/workspace/",
+  "shutdownAction": "stopCompose",
+  "settings": {
+    "python.linting.mypyEnabled": true,
+    "python.linting.flake8Enabled": true,
+    "python.linting.pylintEnabled": false,
+    "python.formatting.provider": "black"
+  },
+  "extensions": [
+    "ms-python.python",
+    "redhat.vscode-yaml",
+    "redhat.vscode-xml",
+    "stardog-union.stardog-rdf-grammars",
+    "lextudio.restructuredtext",
+    "trond-snekvik.simple-rst",
+    "EditorConfig.EditorConfig",
+    "paulvarache.vscode-taskfile",
+    "stardog-union.vscode-stardog-languages"
+  ]
+}

.github/workflows/validate.yaml

@@ -13,6 +13,8 @@ env:
 
 jobs:
   validate:
+    permissions:
+      contents: read
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -43,11 +45,6 @@ jobs:
         with:
           distribution: "temurin"
           java-version: "17"
-      - name: Setup env
-        shell: bash
-        run: |
-          MATRIX_PYTHON_VERSION=${{ matrix.python-version }}
-          echo "TOX_PYENV=py${MATRIX_PYTHON_VERSION//./}" >> ${GITHUB_ENV}
       - name: Get pip cache dir
         id: pip-cache
         shell: bash
@@ -59,7 +56,7 @@ jobs:
         with:
           path: ${{ steps.pip-cache.outputs.dir }}
           key: ${{ matrix.os }}-pip-${{ matrix.python-version }}-v1-${{
-            hashFiles('**/setup.py', '**/requirements*.txt') }}
+            hashFiles('**/setup.py', '**/*requirements*.txt') }}
           restore-keys: |
             ${{ matrix.os }}-pip-${{ matrix.python-version }}-v1-
       - name: Cache xdg
@@ -69,38 +66,18 @@ jobs:
           key: ${{ matrix.os }}-xdg-v1-${{ hashFiles('**/with-fuseki.sh') }}
           restore-keys: |
             ${{ matrix.os }}-xdg-v1-
-      - name: Install python dependencies
-        shell: bash
-        # Installing tox-gh-actions to get some enhancement to output rendering
-        # in github actions. Eventually we can maybe collapse the tox-envs job
-        # into this one but there are some limitations of tox-gh-actions which
-        # preclude doing that at the moment.
-        run: |
-          python -m pip install tox tox-gh-actions
-      - name: Install system depdendencies for extensive tests
-        if: ${{ matrix.extensive-tests }}
-        shell: bash
-        run: |
-          if [ "${{ matrix.os }}" == "ubuntu-latest" ]
-          then
-              sudo apt-get install -y libdb-dev
-          elif [ "${{ matrix.os }}" == "macos-latest" ]
-          then
-              brew install berkeley-db@4
-              export BERKELEYDB_DIR=$(brew --prefix berkeley-db@4)
-          fi
-      - name: Validate
+      - name: Install Task
+        uses: arduino/setup-task@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Run validation
         shell: bash
         run: |
-          test_harness=()
-          if "${{ matrix.extensive-tests || false }}" && [ "${{ matrix.os }}" != "windows-latest" ]
-          then
-            1>&2 echo "Running with fuseki"
-            test_harness+="./with-fuseki.sh"
-            TOX_PYENV_SUFFIX=-extensive
-          fi
-          TOXENV="${{ matrix.TOXENV }}"
-          TOXENV="${TOX_PYENV}${TOX_PYENV_SUFFIX}${TOXENV:+,${TOXENV}}"
-          export TOXENV
           export TOX_EXTRA_COMMAND="${{ matrix.TOX_EXTRA_COMMAND }}"
-          "${test_harness[@]}" python -m tox
+          task \
+            EXTENSIVE=${{ matrix.extensive-tests }} \
+            TOX_PYTHON_VERSION=${{ matrix.python-version }} \
+            WITH_GITHUB_ACTIONS=1 \
+            install:system-deps \
+            install:tox \
+            tox

Dockerfile.devcontainer

@@ -0,0 +1,72 @@
+FROM mcr.microsoft.com/vscode/devcontainers/base:ubuntu-22.04
+
+ENV DEBIAN_FRONTEND="noninteractive" TZ="Etc/UTC"
+
+RUN \
+    apt-get update && \
+    apt-get install -y \
+        make \
+        curl \
+        git \
+        build-essential \
+        && \
+    true
+
+RUN \
+    apt-get install -y \
+        libdb-dev \
+        libjpeg-dev \
+        && \
+    true
+
+RUN \
+    apt-get install -y \
+        default-jdk \
+        && \
+    true
+
+RUN \
+    apt-get install -y \
+        software-properties-common \
+        && \
+    add-apt-repository ppa:deadsnakes/ppa && \
+    apt-get install -y \
+        python3 \
+        python3-pip \
+        python3-dev \
+        python3-venv \
+        python3.7 \
+        python3.7-dev \
+        python3.7-venv \
+        python3.8 \
+        python3.8-dev \
+        python3.8-venv \
+        python3.9 \
+        python3.9-dev \
+        python3.9-venv \
+        python3.10 \
+        python3.10-dev \
+        python3.10-venv \
+        python3.11 \
+        python3.11-dev \
+        python3.11-venv \
+        python-is-python3 \
+        && \
+    python --version && \
+    true
+
+RUN sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin
+
+COPY requirements.txt requirements.dev.txt requirements.dev-extra.txt /var/tmp/
+COPY docs/sphinx-requirements.txt /var/tmp/docs/sphinx-requirements.txt
+
+RUN \
+    cd /var/tmp/ && \
+    python -m pip install --upgrade \
+        -r requirements.txt \
+        -r requirements.dev.txt \
+        -r requirements.dev-extra.txt \
+        -r docs/sphinx-requirements.txt \
+        tox \
+        && \
+    true

Makefile

@@ -184,22 +184,6 @@ Run the test suite and generate a HTML coverage report with `pytest` and `pytest
 pytest --cov
 ```
 
-### Running the tests in a Docker container
-
-Run the test suite inside a Docker container for cross-platform support. This resolves issues such as installing BerkeleyDB on Windows and avoids the host and port issues on macOS.
-```shell
-make tests
-```
-
-Tip: If the underlying Dockerfile for the test runner changes, use `make build`.
-
-### Running the tests in a Docker container with coverage report
-
-Run the test suite inside a Docker container with HTML coverage report.
-```shell
-make coverage
-```
-
 ### Viewing test coverage
 
 Once tests have produced HTML output of the coverage report, view it by running: