Merge branch 'master' into is6880/hash-api-key-secret

Author: giancarloromeo

.env-devel

@@ -83,6 +83,12 @@ DIRECTOR_REGISTRY_CACHING=True
 DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS=null
 DIRECTOR_TRACING=null
 
+DOCKER_API_PROXY_HOST=docker-api-proxy
+DOCKER_API_PROXY_PASSWORD=null
+DOCKER_API_PROXY_PORT=8888
+DOCKER_API_PROXY_SECURE=False
+DOCKER_API_PROXY_USER=null
+
 EFS_USER_ID=8006
 EFS_USER_NAME=efs
 EFS_GROUP_ID=8106
@@ -135,8 +141,11 @@ FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "[email protected]
 
 WEBSERVER_LICENSES={}
 LICENSES_ITIS_VIP_SYNCER_ENABLED=false
-LICENSES_ITIS_VIP_API_URL=https://some-api/{category}
+LICENSES_ITIS_VIP_SYNCER_PERIODICITY=1D00:00:00
+LICENSES_ITIS_VIP_API_URL=https://replace-with-itis-api/{category}
 LICENSES_ITIS_VIP_CATEGORIES='{"HumanWholeBody": "Humans", "HumanBodyRegion": "Humans (Region)", "AnimalWholeBody": "Animal"}'
+LICENSES_SPEAG_PHANTOMS_API_URL=https://replace-with-speag-api/{category}
+LICENSES_SPEAG_PHANTOMS_CATEGORIES='{"ComputationalPhantom": "Phantom of the Opera"}'
 
 # Can use 'docker run -it itisfoundation/invitations:latest simcore-service-invitations generate-dotenv --auto-password'
 INVITATIONS_DEFAULT_PRODUCT=osparc
@@ -241,8 +250,6 @@ SMTP_PROTOCOL=UNENCRYPTED
 SMTP_USERNAME=it_doesnt_matter
 
 # STORAGE ----
-BF_API_KEY=none
-BF_API_SECRET=none
 STORAGE_ENDPOINT=storage:8080
 STORAGE_HOST=storage
 STORAGE_LOGLEVEL=INFO
@@ -277,7 +284,6 @@ WB_GC_GROUPS=0
 WB_GC_INVITATIONS=null
 WB_GC_LOGIN=null
 WB_GC_LOGLEVEL=INFO
-WB_GC_META_MODELING=0
 WB_GC_NOTIFICATIONS=0
 WB_GC_PAYMENTS=null
 WB_GC_PRODUCTS=0
@@ -292,7 +298,6 @@ WB_GC_STUDIES_DISPATCHER=null
 WB_GC_TAGS=0
 WB_GC_TRACING=null
 WB_GC_USERS={}
-WB_GC_VERSION_CONTROL=0
 WB_GC_WALLETS=0
 
 WB_DB_EL_ACTIVITY=null
@@ -309,7 +314,6 @@ WB_DB_EL_GROUPS=0
 WB_DB_EL_INVITATIONS=null
 WB_DB_EL_LOGIN=null
 WB_DB_EL_LOGLEVEL=INFO
-WB_DB_EL_META_MODELING=0
 WB_DB_EL_NOTIFICATIONS=0
 WB_DB_EL_PAYMENTS=null
 WB_DB_EL_PRODUCTS=0
@@ -324,7 +328,6 @@ WB_DB_EL_STUDIES_DISPATCHER=null
 WB_DB_EL_TAGS=0
 WB_DB_EL_TRACING=null
 WB_DB_EL_USERS={}
-WB_DB_EL_VERSION_CONTROL=0
 WB_DB_EL_WALLETS=0
 
 # WEBSERVER ----
@@ -378,7 +381,6 @@ WEBSERVER_GUNICORN_CMD_ARGS=--timeout=180
 WEBSERVER_HOST=webserver
 WEBSERVER_LOGIN={}
 WEBSERVER_LOGLEVEL=INFO
-WEBSERVER_META_MODELING=1
 WEBSERVER_NOTIFICATIONS=1
 WEBSERVER_PAYMENTS={}
 WEBSERVER_PORT=8080
@@ -396,4 +398,3 @@ WEBSERVER_STUDIES_DISPATCHER={}
 WEBSERVER_TAGS=1
 WEBSERVER_TRACING=null
 WEBSERVER_USERS={}
-WEBSERVER_VERSION_CONTROL=1

.github/CODEOWNERS

@@ -4,45 +4,46 @@
 
 
 # files and folders recursively
-.env-*                                    @mrnicegyu11 @YuryHrytsuk
-Makefile                                  @pcrespov @sanderegg
+.env-*                                             @mrnicegyu11 @YuryHrytsuk
+Makefile                                           @pcrespov @sanderegg
 
 
 # NOTE: '/' denotes the root of the repository
-/.github/                                 @sanderegg @pcrespov
-/api/                                     @sanderegg @pcrespov @matusdrobuliak66
-/ci/                                      @sanderegg @pcrespov
-/docs/                                    @pcrespov
-/packages/common-library/                 @giancarloromeo
-/packages/models-library/                 @sanderegg @pcrespov @matusdrobuliak66 @giancarloromeo
-/packages/postgres-database/              @matusdrobuliak66
-/packages/pytest-simcore/                 @pcrespov @sanderegg
-/packages/service-integration/            @pcrespov @sanderegg @GitHK
-/packages/service-library/                @pcrespov
-/packages/settings-library/               @pcrespov @sanderegg
-/requirements/                            @pcrespov @matusdrobuliak66
-/services/agent/                          @GitHK
-/services/api-server/                     @pcrespov
-/services/autoscaling/                    @sanderegg
-/services/catalog/                        @pcrespov @sanderegg
-/services/clusters-keeper/                @sanderegg
-/services/datcore-adapter/                @sanderegg
-/services/director*/                      @sanderegg @pcrespov @GitHK
-/services/docker-compose*.yml             @sanderegg @mrnicegyu11 @YuryHrytsuk
-/services/dynamic-sidecar/                @GitHK
-/services/efs-guardian/                   @matusdrobuliak66
-/services/invitations/                    @pcrespov
-/services/migration/                      @pcrespov
-/services/payments/                       @pcrespov @matusdrobuliak66
-/services/resource-usage-tracker/         @matusdrobuliak66
-/services/static-webserver/               @GitHK
-/services/static-webserver/client/        @jsaq007 @odeimaiz @ignapas
-/services/storage/                        @sanderegg
-/services/web/server/                     @pcrespov @sanderegg @GitHK @matusdrobuliak66
-/tests/e2e-frontend/                      @odeimaiz
-/tests/e2e-playwright/                    @matusdrobuliak66
-/tests/environment-setup/                 @pcrespov
-/tests/performance/                       @pcrespov @sanderegg
-/tests/public-api/                        @pcrespov
-requirements/*                            @pcrespov
-tools/*                                   @pcrespov
+/.github/                                          @sanderegg @pcrespov
+/api/                                              @sanderegg @pcrespov @matusdrobuliak66
+/ci/                                               @sanderegg @pcrespov
+/docs/                                             @pcrespov
+/packages/common-library/                          @giancarloromeo
+/packages/models-library/                          @sanderegg @pcrespov @matusdrobuliak66 @giancarloromeo
+/packages/postgres-database/                       @matusdrobuliak66
+/packages/pytest-simcore/                          @pcrespov @sanderegg
+/packages/service-integration/                     @pcrespov @sanderegg @GitHK
+/packages/service-library/                         @pcrespov
+/packages/settings-library/                        @pcrespov @sanderegg
+/requirements/                                     @pcrespov @matusdrobuliak66
+/services/agent/                                   @GitHK
+/services/api-server/                              @pcrespov
+/services/api-server/tests/unit/pact_broker/       @matusdrobuliak66
+/services/autoscaling/                             @sanderegg
+/services/catalog/                                 @pcrespov @sanderegg
+/services/clusters-keeper/                         @sanderegg
+/services/datcore-adapter/                         @sanderegg
+/services/director*/                               @sanderegg @pcrespov @GitHK
+/services/docker-compose*.yml                      @sanderegg @mrnicegyu11 @YuryHrytsuk
+/services/dynamic-sidecar/                         @GitHK
+/services/efs-guardian/                            @matusdrobuliak66
+/services/invitations/                             @pcrespov
+/services/migration/                               @pcrespov
+/services/payments/                                @pcrespov @matusdrobuliak66
+/services/resource-usage-tracker/                  @matusdrobuliak66
+/services/static-webserver/                        @GitHK
+/services/static-webserver/client/                 @odeimaiz
+/services/storage/                                 @sanderegg
+/services/web/server/                              @pcrespov @sanderegg @GitHK @matusdrobuliak66
+/tests/e2e-frontend/                               @odeimaiz
+/tests/e2e-playwright/                             @matusdrobuliak66
+/tests/environment-setup/                          @pcrespov
+/tests/performance/                                @pcrespov @sanderegg
+/tests/public-api/                                 @pcrespov
+requirements/*                                     @pcrespov
+tools/*                                            @pcrespov

.github/mergify.yml

@@ -22,48 +22,3 @@ queue_rules:
       - check-success=unit-tests
       - check-success=integration-tests
       - check-success=system-tests
-
-
-pull_request_rules:
-  - name: retry flaky tests (when PR is approved)
-    conditions:
-      - label=🤖-automerge
-      - label!=🤖-do-not-merge
-      - base=master
-
-      - -draft # PR is not in draft state
-      - -conflict # No merge conflicts
-      - "#approved-reviews-by>=2" # Requires 2 approving reviews
-      - "#changes-requested-reviews-by=0" # No changes requested
-      - "#review-threads-unresolved=0" # All review threads resolved
-
-      - or:
-          - check-failure=system-test-environment-setup
-          - check-failure=changes
-          - check-failure=build-test-images
-          - check-failure=build-test-images-frontend
-          - check-failure=unit-tests
-          - check-failure=integration-tests
-          - check-failure=system-tests
-    actions:
-      comment:
-        message: "Restarted CI due to failing tests"
-      github_actions:
-        workflow:
-          dispatch:
-            - workflow: ci-testing-deploy.yml
-
-  - name: Automatically add PR to queue if it meets conditions
-    conditions:
-      - "label=🤖-automerge"
-      - "label!=🤖-do-not-merge"
-      - "base=master"
-
-      - "-draft" # PR is not in draft state
-      - "-conflict" # PR has no conflicts
-      - "#approved-reviews-by>=2" # Requires 2 approvals
-      - "#changes-requested-reviews-by=0" # No requested changes
-      - "#review-threads-unresolved=0" # All review threads resolved
-    actions:
-      queue:
-        name: default

.github/workflows/ci-pact-master.yml

@@ -0,0 +1,41 @@
+# This workflow holds jobs which are required to pass before merging into master
+
+name: CI PACT Master
+on:
+  push:
+    branches:
+      - "master"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  pact-tests:
+    timeout-minutes: 10
+    name: "Run PACT tests"
+    runs-on: ubuntu-latest
+    env:
+      # secrets can be set in settings/secrets on github
+      PACT_BROKER_URL: ${{ secrets.PACT_BROKER_URL }}
+      PACT_BROKER_USERNAME: ${{ secrets.PACT_BROKER_USERNAME }}
+      PACT_BROKER_PASSWORD: ${{ secrets.PACT_BROKER_PASSWORD }}
+    steps:
+      - name: setup python environment
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.5.x"
+          enable-cache: false
+      - name: checkout source branch
+        uses: actions/checkout@v4
+      - name: Run pact tests
+        run: |
+          make devenv
+          source .venv/bin/activate
+          cd services/api-server
+          make install-ci
+          make test-pacts

.github/workflows/ci-testing-deploy.yml

@@ -77,6 +77,7 @@ jobs:
       migration: ${{ steps.filter.outputs.migration }}
       payments: ${{ steps.filter.outputs.payments }}
       dynamic-scheduler: ${{ steps.filter.outputs.dynamic-scheduler }}
+      docker-api-proxy: ${{ steps.filter.outputs.docker-api-proxy }}
       resource-usage-tracker: ${{ steps.filter.outputs.resource-usage-tracker }}
       static-webserver: ${{ steps.filter.outputs.static-webserver }}
       storage: ${{ steps.filter.outputs.storage }}
@@ -233,6 +234,9 @@ jobs:
                 - 'services/docker-compose*'
                 - 'scripts/mypy/*'
                 - 'mypy.ini'
+            docker-api-proxy:
+                - 'packages/**'
+                - 'services/docker-api-proxy/**'
             resource-usage-tracker:
               - 'packages/**'
               - 'services/resource-usage-tracker/**'
@@ -2190,6 +2194,71 @@ jobs:
         with:
           flags: integrationtests #optional
 
+
+  integration-test-docker-api-proxy:
+    needs: [changes, build-test-images]
+    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.docker-api-proxy == 'true' || github.event_name == 'push'}}
+    timeout-minutes: 30 # if this timeout gets too small, then split the tests
+    name: "[int] docker-api-proxy"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python: ["3.11"]
+        os: [ubuntu-22.04]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v4
+      - name: setup docker buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+      - name: setup python environment
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - name: expose github runtime for buildx
+        uses: crazy-max/ghaction-github-runtime@v3
+      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
+      - name: download docker images with retry
+        uses: Wandalen/wretry.action@master
+        with:
+          action: actions/download-artifact@v4
+          with: |
+            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend
+            path: /${{ runner.temp }}/build
+          attempt_limit: 5
+          attempt_delay: 1000
+      - name: load docker images
+        run: make load-images local-src=/${{ runner.temp }}/build
+      - name: install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.5.x"
+          enable-cache: false
+          cache-dependency-glob: "**/docker-api-proxy/requirements/ci.txt"
+      - name: show system version
+        run: ./ci/helpers/show_system_versions.bash
+      - name: install
+        run: ./ci/github/integration-testing/docker-api-proxy.bash install
+      - name: test
+        run: ./ci/github/integration-testing/docker-api-proxy.bash test
+      - name: upload failed tests logs
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ github.job }}_docker_logs
+          path: ./services/docker-api-proxy/test_failures
+      - name: cleanup
+        if: ${{ !cancelled() }}
+        run: ./ci/github/integration-testing/docker-api-proxy.bash clean_up
+      - uses: codecov/codecov-action@v5
+        if: ${{ !cancelled() }}
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        with:
+          flags: integrationtests #optional
+
   integration-test-simcore-sdk:
     needs: [changes, build-test-images]
     if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.simcore-sdk == 'true' || github.event_name == 'push' }}
@@ -2262,6 +2331,7 @@ jobs:
         integration-test-director-v2-01,
         integration-test-director-v2-02,
         integration-test-dynamic-sidecar,
+        integration-test-docker-api-proxy,
         integration-test-simcore-sdk,
         integration-test-webserver-01,
         integration-test-webserver-02,

.github/workflows/ci-testing-pull-request.yml

@@ -39,7 +39,7 @@ jobs:
       - name: Check openapi specs are up to date
         run: |
           if ! ./ci/github/helpers/openapi-specs-diff.bash diff \
-            https://raw.githubusercontent.com/${{ github.event.pull_request.head.repo.full_name }}/${{ github.event.after }} \
+            https://raw.githubusercontent.com/${{ github.event.pull_request.head.repo.full_name }}/${{ github.event.pull_request.head.sha }} \
             .; then \
             echo "::error:: OAS are not up to date. Run 'make openapi-specs' to update them"; exit 1; \
           fi
@@ -59,7 +59,7 @@ jobs:
       - name: check api-server backwards compatibility
         run: |
           ./scripts/openapi-diff.bash breaking --fail-on ERR\
-            https://raw.githubusercontent.com/${{ github.event.pull_request.base.repo.full_name }}/${{ github.event.after }}/services/api-server/openapi.json \
+            https://raw.githubusercontent.com/${{ github.event.pull_request.base.repo.full_name }}/refs/heads/${{ github.base_ref }}/services/api-server/openapi.json \
             /specs/services/api-server/openapi.json
 
   all-oas-breaking:
@@ -78,5 +78,5 @@ jobs:
       - name: Check openapi-specs backwards compatibility
         run: |
           ./ci/github/helpers/openapi-specs-diff.bash breaking \
-            https://raw.githubusercontent.com/${{ github.event.pull_request.base.repo.full_name }}/${{ github.event.after }} \
+            https://raw.githubusercontent.com/${{ github.event.pull_request.base.repo.full_name }}/refs/heads/${{ github.base_ref }} \
             .