chore(release): 🔖 version 0.20.0

Author: kikkomep

.github/workflows/main.yaml

@@ -26,6 +26,11 @@ env:
   TERM: xterm
   # enable Docker push only if the required secrets are defined
   ENABLE_DOCKER_PUSH: ${{ secrets.DOCKERHUB_USER != null && secrets.DOCKERHUB_TOKEN != null }}
+  DOCKER_BUILDKIT: 1
+  # CACHE_FROM: "/tmp/.buildx-cache"
+  CACHE_FROM_TYPE: "gha"
+  # CACHE_TO: "/tmp/.buildx-cache"
+  CACHE_TO_TYPE: "gha,mode=max"
 
 jobs:
   # Verifies pep8, pyflakes and circular complexity
@@ -34,15 +39,25 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.11"]
+        python-version: ["3.12"]
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v4
       - name: Set up Python v${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Cache dependencies
+        uses: actions/cache@v4
+        id: flake8-cache
+        with:
+          path: ${{ env.pythonLocation }}-flake8
+          # key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-ml.txt') }}
+          key: ${{ env.pythonLocation }}-flake8
+          restore-keys:
+            ${{ env.pythonLocation }}-flake8
       - name: Install flake8
+        if: steps.flake8-primes.outputs.cache-hit != 'true'
         run: pip install flake8
       - name: Run checks
         run: flake8 -v .
@@ -53,15 +68,25 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.11"]
+        python-version: ["3.12"]
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v4
       - name: Set up Python v${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Cache dependencies
+        uses: actions/cache@v4
+        id: openapi-spec-validator-cache
+        with:
+          path: ${{ env.pythonLocation }}-openapi-spec-validator
+          # key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-ml.txt') }}
+          key: ${{ env.pythonLocation }}-openapi-spec-validator
+          restore-keys:
+            ${{ env.pythonLocation }}-openapi-spec-validator
       - name: Install OpenAPI Spec Validator
+        if: steps.openapi-spec-validator-cache.outputs.cache-hit != 'true'
         run: pip install openapi-spec-validator
       - name: Run checks
         run: openapi-spec-validator specs/api.yaml
@@ -75,20 +100,14 @@ jobs:
         run: "docker images"
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+      # - name: Set up QEMU
+      #   uses: docker/setup-qemu-action@v2
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
+      - name: List available Docker images
+        run: "docker images"
       - name: Generate certificates
         run: make certs
-      # So now you can use Actions' own caching!
-      - name: Cache Docker layers
-        uses: actions/cache@v3
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
       - name: "Set software version"
         run: |
           git fetch --unshallow
@@ -106,7 +125,7 @@ jobs:
       # Extract Docker metadata and set labels and tags
       - name: Extract Docker metadata
         id: docker_meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.DOCKERHUB_REPO }}
           tags: |
@@ -122,22 +141,21 @@ jobs:
             org.opencontainers.image.version=${{ env.SOFTWARE_VERSION }}.build${{ github.run_number }}
       # Build and tag LifeMonitor Docker image
       - name: Build Docker image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: docker/lifemonitor.Dockerfile
           tags: ${{ steps.docker_meta.outputs.tags }}
           labels: ${{ steps.docker_meta.outputs.labels }}
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache-new
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
           load: true
           build-args: |
             SW_VERSION=${{ env.SOFTWARE_VERSION }}
             BUILD_NUMBER=${{ github.run_number }}
-      - name: Update cache
-        run: |
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+      # List available Docker images
+      - name: List Docker images
+        run: "docker images"
       # Setup testing environment
       - name: Set up testing environment
         # Tag the latest built image as `crs4/lifemonitor:latest`
@@ -146,14 +164,18 @@ jobs:
           docker images ;
           docker tag ${{ env.DOCKERHUB_REPO }}:${{ env.SOFTWARE_VERSION_AS_DOCKER_TAG }} crs4/lifemonitor:latest ;
           docker images ;
-          SKIP_BUILD=1 SW_VERSION=${{ env.SOFTWARE_VERSION }} BUILD_NUMBER=${{ github.run_number }} make start-testing
+          SKIP_BUILD=1 SW_VERSION=${{ env.SOFTWARE_VERSION }} BUILD_NUMBER=${{ github.run_number }} SEEK_FINAL_WAIT_TIME=30 make start-testing ;
+          sleep 30 ;
+          docker compose logs ;
         env:
           TRAVIS_TESTING_SERVICE_URL: ${{ secrets.TRAVIS_TESTING_SERVICE_URL }}
           TRAVIS_TESTING_SERVICE_TOKEN: ${{ secrets.TRAVIS_TESTING_SERVICE_TOKEN }}
           TRAVIS_TESTING_SERVICE_TYPE: travis
           GH_TESTING_SERVICE_URL: ${{ secrets.GH_TESTING_SERVICE_URL }}
           GH_TESTING_SERVICE_TOKEN: ${{ secrets.GH_TESTING_SERVICE_TOKEN }}
           GH_TESTING_SERVICE_TYPE: github
+          CACHE_FROM: type=gha
+          CACHE_TO: type=gha,mode=max
       # Run tests
       - name: Run tests
         run: SKIP_BUILD=1 SKIP_RESET_COMPOSE=1 SW_VERSION=${{ env.SOFTWARE_VERSION }} BUILD_NUMBER=${{ github.run_number }} make run-tests
@@ -169,14 +191,14 @@ jobs:
         run: make down
       # Log in to DockerHub
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         if: ${{ env.ENABLE_DOCKER_PUSH == 'true' }}
         with:
           username: ${{ secrets.DOCKERHUB_USER }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       # Push LifeMonitor Docker image
       - name: Build and push
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         if: ${{ env.ENABLE_DOCKER_PUSH == 'true' }}
         with:
           context: .

Makefile

@@ -92,6 +92,11 @@ ifdef PLATFORMS
 	platforms_opt := $(call get_opts,platforms,$(PLATFORMS))
 endif
 
+# handle seek wait time
+seek_final_wait_time := 0
+ifdef SEEK_FINAL_WAIT_TIME
+	seek_final_wait_time := $(SEEK_FINAL_WAIT_TIME)
+endif
 
 all: images
 
@@ -104,34 +109,38 @@ compose-files: docker-compose.base.yml \
 	docker-compose.monitoring.yml \
 	settings.conf
 
+log_path:
+	mkdir -p /tmp/lifemonitor-logs
+
 prod:
 	$(eval LM_MODE=prod)
 
 dev:
 	$(eval LM_MODE=dev)
 
-certs:
-	@if ! [[ -f "certs/lm.key" && -f "certs/lm.key" && -f "certs/lifemonitor.ca.crt" ]]; then \
-	  printf "\n$(bold)Generating certificates...$(reset)\n" ; \
-	  mkdir -p certs && \
-	  ./utils/certs/gencerts.sh && \
-	  cp utils/certs/data/ca.* certs/ && \
-	  cp utils/certs/data/lm/*.pem certs/ && \
-	  mv certs/ca.pem certs/lifemonitor.ca.crt && \
-	  mv certs/cert.pem certs/lm.crt && \
-	  mv certs/key.pem certs/lm.key && \
-	  chmod 644 certs/*.{key,crt}; \
-	  printf "\n$(done)\n"; \
+certs: 
+	@if ! [[ -f "certs/lm.key" && -f "certs/lifemonitor.ca.crt" ]]; then \
+		printf "\n$(bold)Generating certificates...$(reset)\n" ; \
+		mkdir -p certs && \
+		./utils/certs/gencerts.sh && \
+		cp utils/certs/data/ca.* certs/ && \
+		cp utils/certs/data/lm/*.pem certs/ && \
+		mv certs/ca.pem certs/lifemonitor.ca.crt && \
+		mv certs/cert.pem certs/lm.crt && \
+		mv certs/key.pem certs/lm.key && \
+		chmod 644 certs/*.{key,crt}; \
+		cp -a "$$(pwd)/certs/" $$(pwd)/tests/config/registries/seek/certs ; \
+		printf "\n$(done)\n"; \
 	else \
-	  echo "$(yellow)WARNING: Using existing certificates$(reset)" ; \
+		echo "$(yellow)WARNING: Using existing certificates$(reset)" ; \
 	fi ;\
 	if ! [[ -f "certs/jwt-key" && -f "certs/jwt-key.pub" ]]; then \
-	  printf "\n$(bold)Generating JWT keys...$(reset)\n" ; \
-	  openssl genrsa -out certs/jwt-key 4096 ; \
-	  openssl rsa -in certs/jwt-key -pubout > certs/jwt-key.pub ; \
-	  printf "\n$(done)\n"; \
+		printf "\n$(bold)Generating JWT keys...$(reset)\n" ; \
+		openssl genrsa -out certs/jwt-key 4096 ; \
+		openssl rsa -in certs/jwt-key -pubout > certs/jwt-key.pub ; \
+		printf "\n$(done)\n" ; \
 	else \
-	  echo "$(yellow)WARNING: Using existing JWT keys $(reset)" ; \
+		echo "$(yellow)WARNING: Using existing JWT keys $(reset)" ; \
 	fi
 
 lifemonitor: docker/lifemonitor.Dockerfile certs app.py gunicorn.conf.py ## Build LifeMonitor Docker image
@@ -141,6 +150,7 @@ lifemonitor: docker/lifemonitor.Dockerfile certs app.py gunicorn.conf.py ## Buil
 		printf "\n$(bold)Building LifeMonitor Docker image...$(reset)\n" ; \
 		$(build_kit) docker $(build_cmd) \
 			--build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) \
+			--build-arg PIP_CACHE_DIR=/tmp/.buildx-cache/pip --build-arg NPM_CONFIG_CACHE=/tmp/.buildx-cache/npm \
 			$(cache_from_opt) $(cache_to_opt) \
 			${sw_version_arg} ${build_number_arg} ${tags_opt} ${labels_opt} ${platforms_opt} \
 			-f docker/lifemonitor.Dockerfile -t crs4/lifemonitor . ;\
@@ -186,17 +196,17 @@ permissions: certs
 		settings.conf \
 		tests/config/registries/seek/nginx.conf \
 		tests/config/registries/seek/doorkeeper.rb \
-	&& mkdir -p /tmp/lifemonitor-logs
+	&& mkdir -p /tmp/lifemonitor-logs && chmod a+rw /tmp/lifemonitor-logs
 
 
 aux_images: tests/config/registries/seek/seek.Dockerfile certs
 	@printf "\n$(bold)Building auxiliary Docker images...$(reset)\n" ; \
-	docker build -f tests/config/registries/seek/seek.Dockerfile \
-	       -t crs4/lifemonitor-tests:seek \
-	       tests/config/registries/seek/ ; \
+	# docker build -f tests/config/registries/seek/seek.Dockerfile \
+	#        -t crs4/lifemonitor-tests:seek \
+	#        tests/config/registries/seek/ ; \
 	printf "$(done)\n"
 
-start: images compose-files prod reset_compose permissions ## Start LifeMonitor in a Production environment
+start: images compose-files log_path prod reset_compose permissions ## Start LifeMonitor in a Production environment
 	@printf "\n$(bold)Starting production services...$(reset)\n" ; \
 	base=$$(if [[ -f "docker-compose.yml" ]]; then echo "-f docker-compose.yml"; fi) ; \
 	echo "$$(USER_UID=$$(id -u) USER_GID=$$(id -g) \
@@ -208,7 +218,7 @@ start: images compose-files prod reset_compose permissions ## Start LifeMonitor
 	&& $(docker_compose) -f docker-compose.yml up -d redis db init lm worker ws_server nginx prometheus ;\
 	printf "$(done)\n"
 
-start-dev: images compose-files dev reset_compose permissions ## Start LifeMonitor in a Development environment
+start-dev: images compose-files dev log_path reset_compose permissions ## Start LifeMonitor in a Development environment
 	@printf "\n$(bold)Starting development services...$(reset)\n" ; \
 	base=$$(if [[ -f "docker-compose.yml" ]]; then echo "-f docker-compose.yml"; fi) ; \
 	echo "$$(USER_UID=$$(id -u) USER_GID=$$(id -g) \
@@ -221,9 +231,8 @@ start-dev: images compose-files dev reset_compose permissions ## Start LifeMonit
 	&& $(docker_compose) -f docker-compose.yml up -d redis db dev_proxy github_event_proxy init lm worker ws_server prometheus nginx ;\
 	printf "$(done)\n"
 
-start-testing: compose-files aux_images ro_crates images reset_compose permissions ## Start LifeMonitor in a Testing environment
-	@printf "\n$(bold)Starting testing services...$(reset)\n" ; \
-	base=$$(if [[ -f "docker-compose.yml" ]]; then echo "-f docker-compose.yml"; fi) ; \
+.start-testing: compose-files aux_images ro_crates log_path images reset_compose permissions ## Start LifeMonitor in a Testing environment
+	@base=$$(if [[ -f "docker-compose.yml" ]]; then echo "-f docker-compose.yml"; fi) ; \
 	echo "$$(USER_UID=$$(id -u) USER_GID=$$(id -g) \
 			$(docker_compose) $${base} \
 				-f docker-compose.extra.yml \
@@ -233,12 +242,22 @@ start-testing: compose-files aux_images ro_crates images reset_compose permissio
 				-f docker-compose.test.yml \
 				config)" > docker-compose.yml \
 	&& cp {,.test.}docker-compose.yml \
-	&& $(docker_compose) -f docker-compose.yml up -d db lmtests seek jenkins webserver worker ws_server ;\
-	$(docker_compose) -f ./docker-compose.yml \
-		exec -T lmtests /bin/bash -c "tests/wait-for-seek.sh 600"; \
-	printf "$(done)\n"
+	&& $(docker_compose) -f docker-compose.yml up -d db lmtests seek jenkins webserver worker ws_server \
+	&& $(docker_compose) -f ./docker-compose.yml \
+		exec -T lmtests /bin/bash -c "SEEK_FINAL_WAIT_TIME=$(seek_final_wait_time) tests/wait-for-seek.sh 600" \
+	&& $(docker_compose) exec lmtests pytest tests/test_users.py::test_user1[RegistryType.SEEK] > /dev/null 2>&1 || echo "Testing environment initialized!" \
+	&& $(docker_compose) restart db lmtests \
+	&& printf "$(done)\n" 
+
+start-testing:
+	@printf "\n$(bold)Starting testing services...$(reset)\n " ; \
+	if [ -f /tmp/.lifemonitor-testing-started ]; then \
+		printf "$(yellow)WARNING: Testing environment already started, skipping...$(reset)" ; \
+	else \
+		$(MAKE) .start-testing && touch /tmp/.lifemonitor-testing-started ; \
+	fi
 
-start-maintenance: compose-files aux_images ro_crates images reset_compose permissions ## Start LifeMonitor in a Testing environment
+start-maintenance: compose-files aux_images ro_crates log_path images reset_compose permissions ## Start LifeMonitor in a Testing environment
 	@printf "\n$(bold)Starting testing services...$(reset)\n" ; \
 	base=$$(if [[ -f "docker-compose.yml" ]]; then echo "-f docker-compose.yml"; fi) ; \
 	echo "$$(USER_UID=$$(id -u) USER_GID=$$(id -g) \
@@ -278,8 +297,10 @@ start-aux-services: aux_images ro_crates docker-compose.extra.yml permissions ##
 # 	      && $(docker_compose) up -d jupyter ; \
 # 	printf "$(done)\n"
 
-run-tests: start-testing ## Run all tests in the Testing Environment
-	@printf "\n$(bold)Running tests...$(reset)\n" ; \
+run-tests: ## Run all tests in the Testing Environment
+	@export SKIP_RESET_COMPOSE=1 ; \
+	$(MAKE) start-testing ; \
+	printf "\n$(bold)Running tests...$(reset)\n" ; \
 	$(docker_compose) exec -T lmtests /bin/bash -c "pytest --durations=10 --color=yes tests"
 
 
@@ -319,9 +340,11 @@ stop-testing: compose-files ## Stop all the services in the Testing Environment
 	USER_UID=$$(id -u) USER_GID=$$(id -g) \
 	$(docker_compose) -f docker-compose.extra.yml \
 				   -f docker-compose.base.yml \
+				   -f docker-compose.monitoring.yml \
 				   -f docker-compose.dev.yml \
 				   -f docker-compose.test.yml \
 				   $(log_level_opt) stop db lmtests seek jenkins webserver worker ws_server ; \
+	rm -f  /tmp/.lifemonitor-testing-started ; \
 	printf "$(done)\n"
 
 stop-dev: compose-files ## Stop all services in the Develop Environment
@@ -353,7 +376,7 @@ stop-all: ## Stop all the services
 
 reset_compose:
 	@if [[ $${SKIP_RESET_COMPOSE} -eq 1 ]]; then \
-		echo "$(bold)Skip reset of docker-compose services $(reset)" ;\
+		echo -e "\n$(bold)Skip reset of docker-compose services $(reset)" ;\
 	elif [[ -f "docker-compose.yml" ]]; then \
 		cmp -s docker-compose.yml .$(LM_MODE).docker-compose.yml ; \
 		RETVAL=$$? ; \
@@ -363,6 +386,7 @@ reset_compose:
 			USER_UID=$$(id -u) USER_GID=$$(id -g) \
 			$(docker_compose) down ; \
 			rm docker-compose.yml ; \
+			rm -rf /tmp/.lifemonitor-testing-started ; \
 			printf "$(done)\n" ; \
 		fi \
 	fi
@@ -372,6 +396,7 @@ down: ## Teardown all the services
 	echo "$(bold)Teardown all services...$(reset)" ; \
 	USER_UID=$$(id -u) USER_GID=$$(id -g) \
 	$(docker_compose) down ; \
+	rm -rf /tmp/.lifemonitor-testing-started ; \
 	printf "$(done)\n" ; \
 	else \
 		printf "\n$(yellow)WARNING: nothing to remove. 'docker-compose.yml' file not found!$(reset)\n\n" ; \
@@ -387,8 +412,9 @@ clean: ## Clean up the working environment (i.e., running services, network, vol
 			printf "$(yellow)WARNING: nothing to remove. 'docker-compose.yml' file not found!$(reset)\n" ; \
 	fi
 	@printf "\n$(bold)Removing certs...$(reset) " ; \
-	rm -rf certs
-	rm -rf utils/certs/data
+	rm -rf certs && rm -rf utils/certs/data && rm -rf tests/config/registries/seek/certs \
+	rm -rf /tmp/lifemonitor-logs \
+	rm -rf /tmp/.lifemonitor-testing-started \
 	@printf "$(done)\n"
 	@printf "\n$(bold)Removing temp files...$(reset) " ; \
 	rm -rf {,.prod.,.dev.,.test.}docker-compose.yml
@@ -399,9 +425,9 @@ clean: ## Clean up the working environment (i.e., running services, network, vol
 help: ## Show help
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
 
-.PHONY: all images aux_images certs prod dev \
+.PHONY: all images aux_images prod dev certs \
 		lifemonitor smeeio ro_crates webserver \
-		start start-dev start-testing start-nginx start-aux-services \
+		start start-dev start-testing .start-testing start-nginx start-aux-services \
 		run-tests tests \
 		stop-aux-services stop-nginx stop-testing \
 		stop-dev stop stop-all down reset_compose clean