✨ Add: Tracing via opentelemetry

.env-devel

@@ -307,9 +307,11 @@ SESSION_COOKIE_SECURE=False
 SIMCORE_VCS_RELEASE_TAG=latest
 STUDIES_ACCESS_ANONYMOUS_ALLOWED=0
 STUDIES_DEFAULT_SERVICE_THUMBNAIL=https://via.placeholder.com/170x120.png
-TRACING_ENABLED=1
-TRACING_THRIFT_COMPACT_ENDPOINT=http://jaeger:5775
-TRACING_ZIPKIN_ENDPOINT=http://jaeger:9411
+TRACING_OPENTELEMETRY_COLLECTOR_BATCH_SIZE=2
+TRACING_OPENTELEMETRY_COLLECTOR_PORT=4318
+TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT=http://opentelemetry-collector
+TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE=100
+TRACING_OPENTELEMETRY_COLLECTOR_EXPORTER_ENDPOINT=http://jaeger:4318
 TRAEFIK_SIMCORE_ZONE=internal_simcore_stack
 TWILIO_ACCOUNT_SID=DUMMY
 TWILIO_AUTH_TOKEN=DUMMY

.github/dependabot.yml

@@ -12,9 +12,6 @@ updates:
     assignees:
       - pcrespov
     ignore:
-      - dependency-name: aiozipkin
-        versions:
-          - ">= 1.a, < 2"
       - dependency-name: docker-compose
         versions:
           - 1.28.2
@@ -41,9 +38,6 @@ updates:
     assignees:
       - pcrespov
     ignore:
-      - dependency-name: aiozipkin
-        versions:
-          - ">= 1.a, < 2"
       - dependency-name: openapi-core
         versions:
           - "> 0.12.0, < 1"

Makefile

@@ -310,18 +310,19 @@ TableWidth=140;\
 printf "%24s | %90s | %12s | %12s\n" Name Endpoint User Password;\
 printf "%.$${TableWidth}s\n" "$$separator";\
 printf "$$rows" "oSparc platform" "http://$(get_my_ip).nip.io:9081";\
-printf "$$rows" "oSparc web API doc" "http://$(get_my_ip).nip.io:9081/dev/doc";\
 printf "$$rows" "oSparc public API doc" "http://$(get_my_ip).nip.io:8006/dev/doc";\
-printf "$$rows" "Postgres DB" "http://$(get_my_ip).nip.io:18080/?pgsql=postgres&username="$${POSTGRES_USER}"&db="$${POSTGRES_DB}"&ns=public" $${POSTGRES_USER} $${POSTGRES_PASSWORD};\
-printf "$$rows" "Portainer" "http://$(get_my_ip).nip.io:9000" admin adminadmin;\
-printf "$$rows" "Redis" "http://$(get_my_ip).nip.io:18081";\
+printf "$$rows" "oSparc web API doc" "http://$(get_my_ip).nip.io:9081/dev/doc";\
 printf "$$rows" "Dask Dashboard" "http://$(get_my_ip).nip.io:8787";\
 printf "$$rows" "Docker Registry" "http://$${REGISTRY_URL}/v2/_catalog" $${REGISTRY_USER} $${REGISTRY_PW};\
 printf "$$rows" "Invitations" "http://$(get_my_ip).nip.io:8008/dev/doc" $${INVITATIONS_USERNAME} $${INVITATIONS_PASSWORD};\
+printf "$$rows" "Jaeger" "http://$(get_my_ip).nip.io:16686";\
 printf "$$rows" "Payments" "http://$(get_my_ip).nip.io:8011/dev/doc" $${PAYMENTS_USERNAME} $${PAYMENTS_PASSWORD};\
+printf "$$rows" "Portainer" "http://$(get_my_ip).nip.io:9000" admin adminadmin;\
+printf "$$rows" "Postgres DB" "http://$(get_my_ip).nip.io:18080/?pgsql=postgres&username="$${POSTGRES_USER}"&db="$${POSTGRES_DB}"&ns=public" $${POSTGRES_USER} $${POSTGRES_PASSWORD};\
 printf "$$rows" "Rabbit Dashboard" "http://$(get_my_ip).nip.io:15672" admin adminadmin;\
-printf "$$rows" "Traefik Dashboard" "http://$(get_my_ip).nip.io:8080/dashboard/";\
+printf "$$rows" "Redis" "http://$(get_my_ip).nip.io:18081";\
 printf "$$rows" "Storage S3 Minio" "http://$(get_my_ip).nip.io:9001" 12345678 12345678;\
+printf "$$rows" "Traefik Dashboard" "http://$(get_my_ip).nip.io:8080/dashboard/";\
 
 printf "\n%s\n" "⚠️ if a DNS is not used (as displayed above), the interactive services started via dynamic-sidecar";\
 echo "⚠️ will not be shown. The frontend accesses them via the uuid.services.YOUR_IP.nip.io:9081";

api/tests/requirements.txt

@@ -6,8 +6,6 @@ aiohttp==3.10.5
     #   -r requirements.in
 aiosignal==1.3.1
     # via aiohttp
-async-timeout==4.0.3
-    # via aiohttp
 attrs==24.2.0
     # via
     #   aiohttp
@@ -23,8 +21,6 @@ coverage==7.6.1
     # via
     #   -r requirements.in
     #   pytest-cov
-exceptiongroup==1.2.2
-    # via pytest
 frozenlist==1.4.1
     # via
     #   aiohttp
@@ -118,10 +114,6 @@ six==1.16.0
     #   rfc3339-validator
 termcolor==2.4.0
     # via pytest-sugar
-tomli==2.0.1
-    # via
-    #   coverage
-    #   pytest
 urllib3==2.2.2
     # via
     #   -c ../../requirements/constraints.txt

docs/coding-conventions.md

@@ -136,3 +136,7 @@ Have a look at `ESLint`'s configuration files [.eslintrc.json](.eslintrc.json) a
 [mypy]:https://www.mypy-lang.org/
 [pep257]:https://peps.python.org/pep-0257/
 [pylint]:https://pylint.readthedocs.io/en/latest/
+
+# My first osparc-simcore PR: common pitfalls
+
+- Make sure to run `make mypy` and `make pylint`, as the associated github-actions are required to pass. If you include new dependencies in `requirements/*.in`, make sure to run `make touch && make reqs "upgrade=NAME_OF_YOUR_NEW_DEPENDENCY"`. It is best to do this inside a reproducible environment, for this purpose a shell inside a docker container can be used: Go to `osparc-simcore/requirements/tools` and run `make shell`. Inside the new shell the osparc-simcore repo is placed in `~`. Run `make reqs` from inside this shell.

packages/aws-library/requirements/_base.txt

@@ -56,8 +56,25 @@ botocore==1.34.131
     #   s3transfer
 botocore-stubs==1.35.2
     # via types-aiobotocore
+certifi==2024.7.4
+    # via
+    #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../requirements/constraints.txt
+    #   requests
+charset-normalizer==3.3.2
+    # via requests
 click==8.1.7
     # via typer
+deprecated==1.2.14
+    # via
+    #   opentelemetry-api
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-semantic-conventions
 dnspython==2.6.1
     # via email-validator
 email-validator==2.2.0
@@ -70,11 +87,20 @@ frozenlist==1.4.1
     # via
     #   aiohttp
     #   aiosignal
+googleapis-common-protos==1.65.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+grpcio==1.66.0
+    # via opentelemetry-exporter-otlp-proto-grpc
 idna==3.7
     # via
     #   anyio
     #   email-validator
+    #   requests
     #   yarl
+importlib-metadata==8.0.0
+    # via opentelemetry-api
 jmespath==1.0.1
     # via
     #   boto3
@@ -93,6 +119,45 @@ multidict==6.0.5
     # via
     #   aiohttp
     #   yarl
+opentelemetry-api==1.26.0
+    # via
+    #   -r requirements/../../../packages/service-library/requirements/_base.in
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-requests
+    #   opentelemetry-sdk
+    #   opentelemetry-semantic-conventions
+opentelemetry-exporter-otlp==1.26.0
+    # via -r requirements/../../../packages/service-library/requirements/_base.in
+opentelemetry-exporter-otlp-proto-common==1.26.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-exporter-otlp-proto-grpc==1.26.0
+    # via opentelemetry-exporter-otlp
+opentelemetry-exporter-otlp-proto-http==1.26.0
+    # via opentelemetry-exporter-otlp
+opentelemetry-instrumentation==0.47b0
+    # via opentelemetry-instrumentation-requests
+opentelemetry-instrumentation-requests==0.47b0
+    # via -r requirements/../../../packages/service-library/requirements/_base.in
+opentelemetry-proto==1.26.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-common
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-sdk==1.26.0
+    # via
+    #   -r requirements/../../../packages/service-library/requirements/_base.in
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-semantic-conventions==0.47b0
+    # via
+    #   opentelemetry-instrumentation-requests
+    #   opentelemetry-sdk
+opentelemetry-util-http==0.47b0
+    # via opentelemetry-instrumentation-requests
 orjson==3.10.7
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
@@ -105,6 +170,10 @@ orjson==3.10.7
     #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
 pamqp==3.3.0
     # via aiormq
+protobuf==4.25.4
+    # via
+    #   googleapis-common-protos
+    #   opentelemetry-proto
 pydantic==1.10.17
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
@@ -151,6 +220,8 @@ referencing==0.29.3
     #   -c requirements/../../../packages/service-library/requirements/./constraints.txt
     #   jsonschema
     #   jsonschema-specifications
+requests==2.32.3
+    # via opentelemetry-exporter-otlp-proto-http
 rich==13.7.1
     # via
     #   -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in
@@ -162,6 +233,8 @@ rpds-py==0.20.0
     #   referencing
 s3transfer==0.10.2
     # via boto3
+setuptools==74.0.0
+    # via opentelemetry-instrumentation
 sh==2.0.7
     # via -r requirements/_base.in
 shellingham==1.5.4
@@ -197,6 +270,7 @@ typing-extensions==4.12.2
     # via
     #   aiodebug
     #   faststream
+    #   opentelemetry-sdk
     #   pydantic
     #   typer
     #   types-aiobotocore
@@ -212,10 +286,16 @@ urllib3==2.2.2
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   botocore
+    #   requests
 wrapt==1.16.0
-    # via aiobotocore
+    # via
+    #   aiobotocore
+    #   deprecated
+    #   opentelemetry-instrumentation
 yarl==1.9.4
     # via
     #   aio-pika
     #   aiohttp
     #   aiormq
+zipp==3.20.1
+    # via importlib-metadata

packages/aws-library/requirements/_test.txt

@@ -36,13 +36,16 @@ botocore-stubs==1.35.2
 certifi==2024.7.4
     # via
     #   -c requirements/../../../requirements/constraints.txt
+    #   -c requirements/_base.txt
     #   requests
 cffi==1.17.0
     # via cryptography
 cfn-lint==1.10.3
     # via moto
 charset-normalizer==3.3.2
-    # via requests
+    # via
+    #   -c requirements/_base.txt
+    #   requests
 click==8.1.7
     # via
     #   -c requirements/_base.txt
@@ -214,6 +217,7 @@ regex==2024.7.24
     # via cfn-lint
 requests==2.32.3
     # via
+    #   -c requirements/_base.txt
     #   docker
     #   jsonschema-path
     #   moto
@@ -231,8 +235,10 @@ s3transfer==0.10.2
     # via
     #   -c requirements/_base.txt
     #   boto3
-setuptools==73.0.1
-    # via moto
+setuptools==74.0.0
+    # via
+    #   -c requirements/_base.txt
+    #   moto
 six==1.16.0
     # via
     #   -c requirements/_base.txt

packages/aws-library/requirements/_tools.txt

@@ -68,8 +68,9 @@ pyyaml==6.0.2
     #   pre-commit
 ruff==0.6.1
     # via -r requirements/../../../requirements/devenv.txt
-setuptools==73.0.1
+setuptools==74.0.0
     # via
+    #   -c requirements/_base.txt
     #   -c requirements/_test.txt
     #   pip-tools
 tomlkit==0.13.2

packages/dask-task-models-library/requirements/_test.txt

@@ -4,8 +4,6 @@ coverage==7.6.1
     # via
     #   -r requirements/_test.in
     #   pytest-cov
-exceptiongroup==1.2.2
-    # via pytest
 faker==27.0.0
     # via -r requirements/_test.in
 flexcache==0.3
@@ -67,10 +65,6 @@ six==1.16.0
     #   python-dateutil
 termcolor==2.4.0
     # via pytest-sugar
-tomli==2.0.1
-    # via
-    #   coverage
-    #   pytest
 typing-extensions==4.12.2
     # via
     #   -c requirements/_base.txt

packages/dask-task-models-library/requirements/_tools.txt

@@ -70,22 +70,12 @@ ruff==0.6.1
     # via -r requirements/../../../requirements/devenv.txt
 setuptools==73.0.1
     # via pip-tools
-tomli==2.0.1
-    # via
-    #   -c requirements/_test.txt
-    #   black
-    #   build
-    #   mypy
-    #   pip-tools
-    #   pylint
 tomlkit==0.13.2
     # via pylint
 typing-extensions==4.12.2
     # via
     #   -c requirements/_base.txt
     #   -c requirements/_test.txt
-    #   astroid
-    #   black
     #   mypy
 virtualenv==20.26.3
     # via pre-commit

packages/models-library/requirements/_test.txt

@@ -8,8 +8,6 @@ coverage==7.6.1
     # via
     #   -r requirements/_test.in
     #   pytest-cov
-exceptiongroup==1.2.2
-    # via pytest
 faker==27.0.0
     # via -r requirements/_test.in
 flexcache==0.3
@@ -87,10 +85,6 @@ six==1.16.0
     #   python-dateutil
 termcolor==2.4.0
     # via pytest-sugar
-tomli==2.0.1
-    # via
-    #   coverage
-    #   pytest
 types-jsonschema==4.23.0.20240813
     # via -r requirements/_test.in
 types-pyyaml==6.0.12.20240808

packages/models-library/requirements/_tools.txt

@@ -78,14 +78,6 @@ setuptools==73.0.1
     # via pip-tools
 shellingham==1.5.4
     # via typer
-tomli==2.0.1
-    # via
-    #   -c requirements/_test.txt
-    #   black
-    #   build
-    #   mypy
-    #   pip-tools
-    #   pylint
 tomlkit==0.13.2
     # via pylint
 typer==0.12.4
@@ -94,8 +86,6 @@ typing-extensions==4.12.2
     # via
     #   -c requirements/_base.txt
     #   -c requirements/_test.txt
-    #   astroid
-    #   black
     #   mypy
     #   typer
 virtualenv==20.26.3