chore: release v1.1.0 (#2739)

Author: mohammad-alisafaee

.dockerignore

@@ -7,4 +7,5 @@
 !entrypoint-svc.sh
 !.git
 !.gitignore
-.git/config
+!Makefile
+.git/config

.github/dependabot.yml

@@ -8,3 +8,4 @@ updates:
       prefix: "build(deps): "
       include: "scope"
     target-branch: "dependabot-updates"
+    open-pull-requests-limit: 1

.github/workflows/test_deploy.yml

@@ -351,6 +351,9 @@ jobs:
         run: |
           sudo apt-get update -y
           sudo apt-get install -y libyaml-0-2 libyaml-dev
+          sudo add-apt-repository "deb http://archive.ubuntu.com/ubuntu jammy main restricted multiverse universe"
+          sudo apt-get update -y
+          sudo apt-get install -y --allow-downgrades -t jammy git-lfs=3.0.2-1
       - uses: actions/cache@master
         id: dependency-cache
         with:
@@ -840,6 +843,19 @@ jobs:
           PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
         run: make publish-lock-package
 
+  update-develop-branch:
+    runs-on: ubuntu-latest
+    if: "startsWith(github.ref, 'refs/tags/')"
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: 'develop'
+      - name: merge changes from master
+        run: |
+          git merge master
+          git push
+
   build-images:
     runs-on: ubuntu-latest
     needs:

CHANGES.rst

@@ -18,6 +18,44 @@
 Changes
 =======
 
+`1.1.0 <https://github.com/SwissDataScienceCenter/renku-python/compare/v1.0.6...v1.1.0>`__ (2022-03-04)
+-------------------------------------------------------------------------------------------------------
+
+Bug Fixes
+~~~~~~~~~
+
+-  **dataset:** unset wasDerivedFrom for imported datasets
+   (`#2686 <https://github.com/SwissDataScienceCenter/renku-python/issues/2686>`__)
+   (`89023d2 <https://github.com/SwissDataScienceCenter/renku-python/commit/89023d266fc0dde237e8e8164f2cde16e41e342c>`__)
+-  **core:** avoid migration failure for invalid dataset names
+   (`#2703 <https://github.com/SwissDataScienceCenter/renku-python/issues/2703>`__)
+   (`ee607ac <https://github.com/SwissDataScienceCenter/renku-python/commit/ee607acbb374b97c526e9d6c87c08eda735fbb2a>`__)
+-  **core:** fix workflow iterate working with int parameters
+   (`#2720 <https://github.com/SwissDataScienceCenter/renku-python/issues/2720>`__)
+   (`2358962 <https://github.com/SwissDataScienceCenter/renku-python/commit/235896295e60f678e40f989ab9a144b51fbf94e8>`__)
+-  **core:** check workflow execute inputs
+   (`#2727 <https://github.com/SwissDataScienceCenter/renku-python/issues/2727>`__)
+   (`0bfceaf <https://github.com/SwissDataScienceCenter/renku-python/commit/0bfceafa4e6b4750439ab0ed20c61b0a6ba03a1f>`__)
+
+Features
+~~~~~~~~
+
+-  **core:** add template command
+   (`#2590 <https://github.com/SwissDataScienceCenter/renku-python/issues/2590>`__)
+   (`4ff9c4f <https://github.com/SwissDataScienceCenter/renku-python/commit/4ff9c4f77462dcf74083de0f6abad88b286bc6b4>`__)
+-  **dataset:** use posix move semantics when adding with destination
+   (`#2612 <https://github.com/SwissDataScienceCenter/renku-python/issues/2612>`__)
+   (`24f843a <https://github.com/SwissDataScienceCenter/renku-python/commit/24f843a485d46f6e9627ec02e661ffa63d8c69c9>`__)
+-  **core:** add dataset entries to renku log
+   (`#2633 <https://github.com/SwissDataScienceCenter/renku-python/issues/2633>`__)
+   (`f92fbac <https://github.com/SwissDataScienceCenter/renku-python/commit/f92fbac86e042077dec5a7425aa2dd2a2a3607c5>`__)
+-  **core:** color edges on a per-node basis
+   (`#2719 <https://github.com/SwissDataScienceCenter/renku-python/issues/2719>`__)
+   (`ffa10fb <https://github.com/SwissDataScienceCenter/renku-python/commit/ffa10fb759e0092d49f29e7c99738e5406cf5481>`__)
+-  **core:** add check for invalid imported datasets
+   (`#2726 <https://github.com/SwissDataScienceCenter/renku-python/issues/2726>`__)
+   (`9223886 <https://github.com/SwissDataScienceCenter/renku-python/commit/9223886a72369394c33e64149c7d440ea06f8515>`__)
+
 `1.0.6 <https://github.com/SwissDataScienceCenter/renku-python/compare/v1.0.5...v1.0.6>`__ (2022-02-15)
 -------------------------------------------------------------------------------------------------------
 

Dockerfile.cli

@@ -1,14 +1,17 @@
-FROM python:3.7-slim as base
+FROM python:3.9-slim as base
 
 RUN apt-get update && \
-    apt-get install -y git make && \
+    apt-get install -y git git-lfs=2.* python3-dev && \
     pip install --no-cache --upgrade pip poetry
 
 FROM base as builder
 
+RUN apt-get install -y build-essential && \
+    apt-get clean
+
 # The python install is done in two steps to avoid re-installing all dependencies every
 # time the code changes
-COPY pyproject.toml poetry.lock README.rst CHANGES.rst /code/renku/
+COPY pyproject.toml poetry.lock README.rst CHANGES.rst Makefile /code/renku/
 WORKDIR /code/renku
 RUN poetry export --without-hashes -f requirements.txt --output /tmp/requirements.txt && \
     pip install -r /tmp/requirements.txt && \
@@ -21,18 +24,24 @@ COPY renku /code/renku/renku
 # renku-python is installed in the image. This is the default for chartpress builds.
 ARG CLEAN_INSTALL
 RUN if [ -n "${CLEAN_INSTALL}" ]; then git reset --hard ; fi
-# manually update version.py
-RUN poetry-dynamic-versioning
 RUN make download-templates
 RUN pip wheel --wheel-dir /wheels . && \
-    pip install --no-index --no-warn-script-location --force --root=/pythonroot/ /wheels/*.whl && \
-    apt-get clean
+    pip install --no-index --no-warn-script-location --force --root=/pythonroot/ /wheels/*.whl
 
 FROM base
 
-RUN apt-get install -y git-lfs && \
-    git lfs install
+RUN addgroup -gid 1000 shuhitsu && \
+    useradd -m -u 1000 -g shuhitsu shuhitsu
+
+RUN git lfs install
 
 COPY --from=builder /pythonroot/ /
 
+# shuhitsu (執筆): The "secretary" of the renga, as it were, who is responsible for
+# writing down renga verses and for the proceedings of the renga.
+# todo: #2705 avoid code duplication between service and CLI dockerfiles
+USER shuhitsu
+
+ENV RENKU_DISABLE_VERSION_CHECK=1
+
 ENTRYPOINT ["renku"]

Dockerfile.svc

@@ -1,14 +1,17 @@
 FROM python:3.7-slim as base
 
 RUN apt-get update && \
-    apt-get install -y git make && \
+    apt-get install -y git git-lfs=2.* python3-dev && \
     pip install --no-cache --upgrade pip poetry
 
 FROM base as builder
 
+RUN apt-get install -y build-essential && \
+    apt-get clean
+
 # The python install is done in two steps to avoid re-installing all dependencies every
 # time the code changes
-COPY pyproject.toml poetry.lock README.rst CHANGES.rst /code/renku/
+COPY pyproject.toml poetry.lock README.rst CHANGES.rst Makefile /code/renku/
 WORKDIR /code/renku
 RUN poetry export --without-hashes -f requirements.txt --output /tmp/requirements.txt && \
     pip install -r /tmp/requirements.txt && \
@@ -21,37 +24,25 @@ COPY renku /code/renku/renku
 # renku-python is installed in the image. This is the default for chartpress builds.
 ARG CLEAN_INSTALL
 RUN if [ -n "${CLEAN_INSTALL}" ]; then git reset --hard ; fi
-# manually update version.py
-RUN poetry-dynamic-versioning
 RUN make download-templates
 RUN pip wheel --wheel-dir /wheels .[service] && \
-    pip install --no-index --no-warn-script-location --force --root=/pythonroot/ /wheels/*.whl && \
-    apt-get clean
+    pip install --no-index --no-warn-script-location --force --root=/pythonroot/ /wheels/*.whl
 
 FROM base
 
-RUN addgroup -gid 2000 debug && \
-    addgroup -gid 1000 shuhitsu && \
+RUN addgroup -gid 1000 shuhitsu && \
     useradd -m -u 1000 -g shuhitsu shuhitsu && \
-    mkdir /svc && chown shuhitsu:shuhitsu /svc
-
-RUN apt-get install -y git-lfs && \
+    mkdir /svc && chown shuhitsu:shuhitsu /svc && \
     git lfs install
 
 COPY --from=builder /pythonroot/ /
-COPY --from=builder /code/renku /code/renku
-
-# Only allow 'debug' group to modify the code
-RUN chown -R root:debug /code/renku && chmod -R 775 /code/renku
 
 # shuhitsu (執筆): The "secretary" of the renga, as it were, who is responsible for
 # writing down renga verses and for the proceedings of the renga.
 USER shuhitsu
 
 ENV RENKU_SVC_NUM_WORKERS 4
 ENV RENKU_SVC_NUM_THREADS 8
+ENV RENKU_DISABLE_VERSION_CHECK=1
 
-WORKDIR /code/renku
-
-COPY entrypoint-svc.sh /code/renku/
-ENTRYPOINT ["./entrypoint-svc.sh"]
+ENTRYPOINT [ "renku", "service"]

README.rst

@@ -322,15 +322,11 @@ Service
 -------
 
 Developing the service and testing its APIs can be done with ``docker compose`` (see
-"Deploying Locally" above). To enable live reloading of the code, set the environment
-variable ``DEBUG_MODE=true`` either in your shell or in the ``.env`` file. Note that in
-this case the local directory is mounted in the docker container and renku is re-installed
-so it may take a few minutes before the container is ready.
+"Deploying Locally" above).
 
 If you have a full RenkuLab deployment at your disposal, you can
 use `telepresence <https://www.telepresence.io/>`__ v1 to develop and debug locally.
-Just run the `start-telepresence.sh` script and follow the instructions. You can also
-attach a remote debugger using the "remote attach" method described later.
+Just run the `start-telepresence.sh` script and follow the instructions.
 Mind that the script doesn't work with telepresence v2.
 
 
@@ -389,41 +385,3 @@ If using Visual Studio Code, you may also want to set the ``Remote Attach`` conf
             }
         ]
     }
-
-
-Kubernetes
-~~~~~~~~~~
-
-To debug a running renku-core service in a Kubernetes cluster, the service has to be deployed with the
- `core.debug` flag set to `true`, like:
- ::
-
-    core:
-      debug: true
-
-Also, if you want to be able to modify the files remotely, you need to change
-the `security context` on the `deployment.yaml` file for the renku-core component
-from `runAsUser: 1000` to `runAsGroup: 2000`.
-
-Then install the `Kubernetes extension <https://github.com/Azure/vscode-kubernetes-tools>`_
-and configure your local kubectl with the credentials needed for your cluster.
-
-Add a `.vscode/settings.json` in the renku-python project root and set the following two values:
-
-::
-
-    {
-        "vs-kubernetes": {
-            "vs-kubernetes.python-autodetect-remote-root": true,
-            "vs-kubernetes.python-remote-root": "/code/renku",
-        }
-    }
-
-You might also need to run the `Kubernetes: Use Namespace` commandlet in VSCode to pick the correct
-Kubernetes namespace.
-
-Once this is done, go to the `Kubernetes` tab in VSCode, right-click on your cluster -> Workloads -> Pods -> *-renku-core-*
-entry (not the *-renku-core-redis-* one) and pick `Debug (attach)`, select `core` and `python` and you should be good to go.
-
-You can also select `Attach Visual Studio Code` in the context menu to open a new instance of VSCode with write access to
-the source code in the remote pod.

docker-compose.yml

@@ -68,7 +68,7 @@ services:
       - "8081:8080"
     volumes:
       - ${PWD}:/code/renku
-    command: "api --debug"
+    command: "api"
     labels:
       - "traefik.http.routers.renku-svc.rule=PathPrefix(`/api/renku`)"
       - "traefik.http.middlewares.stripprefix.stripprefix.prefixes=/api"

docs/reference/commands.rst

@@ -27,6 +27,13 @@ Renku Command Line
 
 .. automodule:: renku.cli.init
 
+.. _cli-template:
+
+``renku template``
+------------------
+
+.. automodule:: renku.cli.template
+
 .. _cli-clone:
 
 ``renku clone``

docs/reference/management.rst

@@ -27,7 +27,28 @@ This API is built on top of Git and Git-LFS.
 Datasets
 --------
 
-.. automodule:: renku.core.management.datasets
+.. automodule:: renku.core.management.dataset.dataset
+   :members:
+
+.. automodule:: renku.core.management.dataset.dataset_add
+   :members:
+
+.. automodule:: renku.core.management.dataset.constant
+   :members:
+
+.. automodule:: renku.core.management.dataset.context
+   :members:
+
+.. automodule:: renku.core.management.dataset.pointer_file
+   :members:
+
+.. automodule:: renku.core.management.dataset.request_model
+   :members:
+
+.. automodule:: renku.core.management.dataset.tag
+   :members:
+
+.. automodule:: renku.core.management.dataset.datasets_provenance
    :members:
 
 Repository