Factor out requirements for iaas/kaas (#918)

* move cleanup.py to iaas dir, where it belongs

Signed-off-by: Matthias Büchse <[email protected]>

* move iaas/kaas-specific requirements to respective subfolder

Signed-off-by: Matthias Büchse <[email protected]>

* adapt pytest workflow to changed requirements handling

Signed-off-by: Matthias Büchse <[email protected]>

* upgrade all requirements files to avoid conflicting pinnings

Signed-off-by: Matthias Büchse <[email protected]>

* upgrade once more

Signed-off-by: Matthias Büchse <[email protected]>

* Bugfix: add constraints to layers to prevent conflicts

thx @martinmo for help!

Signed-off-by: Matthias Büchse <[email protected]>

* update README.txt

Signed-off-by: Matthias Büchse <[email protected]>

* minor adjustment of Dockerfile

Signed-off-by: Matthias Büchse <[email protected]>

---------

Signed-off-by: Matthias Büchse <[email protected]>

Author: mbuechse

.github/workflows/scs-compliance-check-with-application-credential.yml

@@ -34,7 +34,7 @@ jobs:
           EOF
       - name: "Clean up any lingering resources from previous run"
         if: ${{ inputs.layer == 'iaas' && inputs.version == 'v4' }}
-        run: "cd /scs-compliance && ./cleanup.py -c ${{ inputs.cloud }} --prefix _scs- --ipaddr 10.1.0. --debug"
+        run: "cd /scs-compliance && ./iaas/cleanup.py -c ${{ inputs.cloud }} --prefix _scs- --ipaddr 10.1.0. --debug"
       - name: "Run scs-compliance-check"
         run: "cd /scs-compliance && ./scs-compliance-check.py scs-compatible-${{ inputs.layer }}.yaml --version ${{ inputs.version }} -o result.yaml -s ${{ inputs.cloud }} -a os_cloud=${{ inputs.cloud }}"
       - name: "Upload results"

.github/workflows/scs-compliance-check.yml

@@ -34,7 +34,7 @@ jobs:
           EOF
       - name: "Clean up any lingering resources from previous run"
         if: ${{ inputs.layer == 'iaas' && inputs.version == 'v4' }}
-        run: "cd /scs-compliance && ./cleanup.py -c ${{ inputs.cloud }} --prefix _scs- --ipaddr 10.1.0. --debug"
+        run: "cd /scs-compliance && ./iaas/cleanup.py -c ${{ inputs.cloud }} --prefix _scs- --ipaddr 10.1.0. --debug"
       - name: "Run scs-compliance-check"
         run: "cd /scs-compliance && ./scs-compliance-check.py scs-compatible-${{ inputs.layer }}.yaml --version ${{ inputs.version }} -o result.yaml -s ${{ inputs.cloud }} -a os_cloud=${{ inputs.cloud }}"
       - name: "Upload results"

.github/workflows/test-python.yml

@@ -7,6 +7,8 @@ name: Run Python unit and regression tests
       - '**.py'
       - .github/workflows/test-python.yml
       - Tests/requirements.txt
+      - Tests/iaas/requirements.txt
+      - Tests/kaas/requirements.txt
       - Tests/test-requirements.txt
 
 jobs:
@@ -20,9 +22,11 @@ jobs:
           cache: 'pip'
           cache-dependency-path: |
             Tests/requirements.txt
+            Tests/iaas/requirements.txt
+            Tests/kaas/requirements.txt
             Tests/test-requirements.txt
       - run: pip3 install --upgrade pip setuptools
-      - run: pip3 install -r requirements.txt -r test-requirements.txt
+      - run: pip3 install -r requirements.txt -r iaas/requirements.txt -r kaas/requirements.txt -r test-requirements.txt
         working-directory: ./Tests
       - run: pytest --cov
         working-directory: ./Tests

Tests/Dockerfile

@@ -5,7 +5,9 @@ FROM python:3.10
 WORKDIR /scs-compliance
 
 COPY requirements.txt requirements.txt
-RUN pip3 install -r requirements.txt
+COPY iaas/requirements.txt iaas-requirements.txt
+COPY kaas/requirements.txt kaas-requirements.txt
+RUN pip3 install -r requirements.txt -r iaas-requirements.txt -r kaas-requirements.txt
 
 COPY . .
 

Tests/README.md

@@ -21,6 +21,12 @@ You can also request a YAML report using the option `-o OUTPUT.yaml`
 
 ### IaaS checks
 
+Install IaaS-specific requirements:
+
+```shell
+pip install -r iaas/requirements.txt
+```
+
 With a cloud environment configured in your `~/.config/openstack/clouds.yaml`
 and `secure.yaml`, then run
 
@@ -33,6 +39,12 @@ specified in `clouds.yaml`.
 
 ### KaaS checks
 
+Install KaaS-specific requirements:
+
+```shell
+pip install -r kaas/requirements.txt
+```
+
 Given a kubeconfig file `path/to/kubeconfig.yaml`, run
 
 ```shell
@@ -133,13 +145,13 @@ We run the tests on a regular basis in our GitHub workflows.
 
 ### Maintaining the Python dependencies
 
-We list our main Python dependencies in `requirements.in`. Additionally, we list
-[unit tests](#unit-and-regression-tests) dependencies in `test-requirements.in`.
-The `*.in` files are fed to `pip-compile` to produce corresponding `*.txt` files
-that contain an exact, version-pinned listing of *all* dependencies, including
-transitive ones.
+We use **pip-compile** to pin and upgrade our dependencies in a controlled manner.
+
+With `pip-compile`, you list your dependencies with as few version constraints as
+possible in a dedicated file, and then you have pip-compile generate the conventional
+`requirements.txt` with fully pinned dependencies.
 
-`pip-compile` can be installed via `pip install pip-tools`.
+The tool `pip-compile` can be installed via `pip install pip-tools`.
 It needs to be run in two cases:
 
 1. You modified an `*.in` file: run `pip-compile <INFILE>`. For example:
@@ -155,6 +167,15 @@ It needs to be run in two cases:
    pip-compile --upgrade requirements.in
    ```
 
+We use a **layered approach** to allow for selective installation:
+
+- the most basic layer is `requirements.in`,
+- above that we have `iaas/requirements.in` and `kaas/requirements.in`, and
+- at the very top we have `test-requirements.in`.
+
+Whenever you change or recompile one of these layers,
+*all layers above that layer have to be recompiled as well*.
+
 Note: The Python version used for running `pip-compile` should be consistent. The currently
 used version is documented in the header of the `requirements.txt`. It should match the
 version used in the Docker image (see [Dockerfile](Dockerfile)) and in our GitHub

Tests/cleanup.py Tests/iaas/cleanup.pyTests/cleanup.py renamed to Tests/iaas/cleanup.py

@@ -0,0 +1,3 @@
+-c ../requirements.txt
+boto3
+openstacksdk

Tests/iaas/requirements.in

@@ -0,0 +1,110 @@
+#
+# This file is autogenerated by pip-compile with Python 3.12
+# by the following command:
+#
+#    pip-compile requirements.in
+#
+boto3==1.38.20
+    # via -r requirements.in
+botocore==1.38.20
+    # via
+    #   boto3
+    #   s3transfer
+certifi==2025.4.26
+    # via
+    #   -c ../requirements.txt
+    #   requests
+cffi==1.17.1
+    # via
+    #   -c ../requirements.txt
+    #   cryptography
+charset-normalizer==3.4.2
+    # via
+    #   -c ../requirements.txt
+    #   requests
+cryptography==44.0.3
+    # via
+    #   -c ../requirements.txt
+    #   openstacksdk
+decorator==5.2.1
+    # via
+    #   dogpile-cache
+    #   openstacksdk
+dogpile-cache==1.4.0
+    # via openstacksdk
+idna==3.10
+    # via
+    #   -c ../requirements.txt
+    #   requests
+iso8601==2.1.0
+    # via
+    #   keystoneauth1
+    #   openstacksdk
+jmespath==1.0.1
+    # via
+    #   boto3
+    #   botocore
+    #   openstacksdk
+jsonpatch==1.33
+    # via openstacksdk
+jsonpointer==3.0.0
+    # via jsonpatch
+keystoneauth1==5.11.0
+    # via openstacksdk
+openstacksdk==4.5.0
+    # via -r requirements.in
+os-service-types==1.7.0
+    # via
+    #   keystoneauth1
+    #   openstacksdk
+pbr==6.1.1
+    # via
+    #   keystoneauth1
+    #   openstacksdk
+    #   os-service-types
+    #   stevedore
+platformdirs==4.3.8
+    # via openstacksdk
+psutil==7.0.0
+    # via openstacksdk
+pycparser==2.22
+    # via
+    #   -c ../requirements.txt
+    #   cffi
+python-dateutil==2.9.0.post0
+    # via
+    #   -c ../requirements.txt
+    #   botocore
+pyyaml==6.0.2
+    # via
+    #   -c ../requirements.txt
+    #   openstacksdk
+requests==2.32.3
+    # via
+    #   -c ../requirements.txt
+    #   keystoneauth1
+requestsexceptions==1.4.0
+    # via openstacksdk
+s3transfer==0.12.0
+    # via boto3
+six==1.17.0
+    # via
+    #   -c ../requirements.txt
+    #   python-dateutil
+stevedore==5.4.1
+    # via
+    #   dogpile-cache
+    #   keystoneauth1
+typing-extensions==4.13.2
+    # via
+    #   -c ../requirements.txt
+    #   keystoneauth1
+    #   openstacksdk
+urllib3==2.4.0
+    # via
+    #   -c ../requirements.txt
+    #   botocore
+    #   requests
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

Tests/iaas/requirements.txt

@@ -1,4 +1,6 @@
+-c ../requirements.txt
 pytest-kind
 kubernetes
+kubernetes_asyncio
 jinja2
 junitparser

Tests/kaas/requirements.in

@@ -4,63 +4,119 @@
 #
 #    pip-compile requirements.in
 #
-cachetools==5.5.0
+aiohappyeyeballs==2.6.1
+    # via
+    #   -c ../requirements.txt
+    #   aiohttp
+aiohttp==3.11.18
+    # via
+    #   -c ../requirements.txt
+    #   kubernetes-asyncio
+aiosignal==1.3.2
+    # via
+    #   -c ../requirements.txt
+    #   aiohttp
+attrs==25.3.0
+    # via
+    #   -c ../requirements.txt
+    #   aiohttp
+cachetools==5.5.2
     # via google-auth
-certifi==2024.8.30
+certifi==2025.4.26
     # via
+    #   -c ../requirements.txt
     #   kubernetes
+    #   kubernetes-asyncio
     #   requests
-charset-normalizer==3.3.2
-    # via requests
-google-auth==2.34.0
+charset-normalizer==3.4.2
+    # via
+    #   -c ../requirements.txt
+    #   requests
+durationpy==0.10
+    # via kubernetes
+frozenlist==1.6.0
+    # via
+    #   -c ../requirements.txt
+    #   aiohttp
+    #   aiosignal
+google-auth==2.40.1
     # via kubernetes
-idna==3.8
-    # via requests
+idna==3.10
+    # via
+    #   -c ../requirements.txt
+    #   requests
+    #   yarl
 jinja2==3.1.6
     # via -r requirements.in
 junitparser==3.2.0
     # via -r requirements.in
-kubernetes==30.1.0
+kubernetes==32.0.1
+    # via -r requirements.in
+kubernetes-asyncio==32.3.2
     # via -r requirements.in
 markupsafe==3.0.2
     # via jinja2
+multidict==6.4.4
+    # via
+    #   -c ../requirements.txt
+    #   aiohttp
+    #   yarl
 oauthlib==3.2.2
     # via
     #   kubernetes
     #   requests-oauthlib
-pyasn1==0.6.0
+propcache==0.3.1
+    # via
+    #   -c ../requirements.txt
+    #   aiohttp
+    #   yarl
+pyasn1==0.6.1
     # via
     #   pyasn1-modules
     #   rsa
-pyasn1-modules==0.4.0
+pyasn1-modules==0.4.2
     # via google-auth
 pykube-ng==23.6.0
     # via pytest-kind
 pytest-kind==22.11.1
     # via -r requirements.in
 python-dateutil==2.9.0.post0
-    # via kubernetes
+    # via
+    #   -c ../requirements.txt
+    #   kubernetes
+    #   kubernetes-asyncio
 pyyaml==6.0.2
     # via
+    #   -c ../requirements.txt
     #   kubernetes
+    #   kubernetes-asyncio
     #   pykube-ng
 requests==2.32.3
     # via
+    #   -c ../requirements.txt
     #   kubernetes
     #   pykube-ng
     #   requests-oauthlib
 requests-oauthlib==2.0.0
     # via kubernetes
-rsa==4.9
+rsa==4.9.1
     # via google-auth
-six==1.16.0
+six==1.17.0
     # via
+    #   -c ../requirements.txt
     #   kubernetes
+    #   kubernetes-asyncio
     #   python-dateutil
-urllib3==2.2.2
+urllib3==2.4.0
     # via
+    #   -c ../requirements.txt
     #   kubernetes
+    #   kubernetes-asyncio
     #   pykube-ng
     #   requests
 websocket-client==1.8.0
     # via kubernetes
+yarl==1.20.0
+    # via
+    #   -c ../requirements.txt
+    #   aiohttp