♻️ Maintenance: improve production build times (ITISFoundation#3367)

Author: sanderegg

.github/CODEOWNERS

@@ -32,7 +32,7 @@ Makefile                                  @pcrespov @sanderegg
 /services/migration/                      @pcrespov
 /services/static-webserver                @GitHK
 /services/storage/                        @mguidon @pcrespov
-/services/web/client/                     @odeimaiz
+/services/static-webserver/client/        @odeimaiz
 /services/web/server/                     @pcrespov @sanderegg @GitHK
 /tests/environment-setup/                 @pcrespov
 /tests/performance/                       @pcrespov

Makefile

@@ -41,12 +41,12 @@ SERVICES_LIST := \
 	storage \
 	webserver
 
-CLIENT_WEB_OUTPUT       := $(CURDIR)/services/web/client/source-output
+CLIENT_WEB_OUTPUT       := $(CURDIR)/services/static-webserver/client/source-output
 
 # version control
 export VCS_URL          := $(shell git config --get remote.origin.url)
 export VCS_REF          := $(shell git rev-parse --short HEAD)
-export VCS_REF_CLIENT   := $(shell git log --pretty=tformat:"%h" -n1 services/web/client)
+export VCS_REF_CLIENT   := $(shell git log --pretty=tformat:"%h" -n1 services/static-webserver/client)
 export VCS_STATUS_CLIENT:= $(if $(shell git status -s),'modified/untracked','clean')
 export BUILD_DATE       := $(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
 
@@ -131,20 +131,8 @@ endef
 
 rebuild: build-nc # alias
 build build-nc: .env ## Builds production images and tags them as 'local/{service-name}:production'. For single target e.g. 'make target=webserver build'
-ifeq ($(target),)
-	# Compiling front-end
-	$(MAKE_C) services/web/client compile
-
-	# Building services
+	# Building service$(if $(target),,s) $(target)
 	$(_docker_compose_build)
-else
-ifeq ($(findstring static-webserver,$(target)),static-webserver)
-	# Compiling front-end
-	$(MAKE_C) services/web/client clean compile
-endif
-	# Building service $(target)
-	$(_docker_compose_build)
-endif
 
 
 build-devel build-devel-nc: .env ## Builds development images and tags them as 'local/{service-name}:development'. For single target e.g. 'make target=webserver build-devel'
@@ -154,7 +142,7 @@ ifeq ($(target),)
 else
 ifeq ($(findstring static-webserver,$(target)),static-webserver)
 	# Compiling front-end
-	$(MAKE_C) services/web/client touch compile-dev
+	$(MAKE_C) services/static-webserver/client touch compile-dev
 endif
 	# Building service $(target)
 	@$(_docker_compose_build)
@@ -256,12 +244,12 @@ show-endpoints:
 
 up-devel: .stack-simcore-development.yml .init-swarm $(CLIENT_WEB_OUTPUT) ## Deploys local development stack, qx-compile+watch and ops stack (pass 'make ops_disabled=1 up-...' to disable)
 	# Start compile+watch front-end container [front-end]
-	@$(MAKE_C) services/web/client down compile-dev flags=--watch
+	@$(MAKE_C) services/static-webserver/client down compile-dev flags=--watch
 	# Deploy stack $(SWARM_STACK_NAME) [back-end]
 	@docker stack deploy --with-registry-auth -c $< $(SWARM_STACK_NAME)
 	@$(MAKE) .deploy-ops
 	@$(_show_endpoints)
-	@$(MAKE_C) services/web/client follow-dev-logs
+	@$(MAKE_C) services/static-webserver/client follow-dev-logs
 
 
 up-prod: .stack-simcore-production.yml .init-swarm ## Deploys local production stack and ops stack (pass 'make ops_disabled=1 up-...' to disable or target=<service-name> to deploy a single service)
@@ -294,7 +282,7 @@ down: ## Stops and removes stack
 		$(shell docker stack ls --format={{.Name}} | tac),\
 		docker stack rm $(stack);)
 	# Removing client containers (if any)
-	-@$(MAKE_C) services/web/client down
+	-@$(MAKE_C) services/static-webserver/client down
 	# Removing generated docker compose configurations, i.e. .stack-*
 	-@rm $(wildcard .stack-*)
 	# Removing local registry if any
@@ -380,7 +368,7 @@ devenv: .venv ## create a python virtual environment with dev tools (e.g. linter
 
 devenv-all: devenv ## sets up extra development tools (everything else besides python)
 	# Upgrading client compiler
-	@$(MAKE_C) services/web/client upgrade
+	@$(MAKE_C) services/static-webserver/client upgrade
 	# Building tools
 	@$(MAKE_C) scripts/json-schema-to-openapi-schema
 
@@ -577,7 +565,7 @@ ifeq ($(target),)
 			$(call show-meta,$(service))\
 		)
 	## Client images:
-	@$(MAKE_C) services/web/client info
+	@$(MAKE_C) services/static-webserver/client info
 else
 	## $(target) images:
 	@$(call show-meta,$(target))
@@ -622,8 +610,8 @@ clean-hooks: ## Uninstalls git pre-commit hooks
 clean: .check-clean ## cleans all unversioned files in project and temp files create by this makefile
 	# Cleaning unversioned
 	@git clean $(_git_clean_args)
-	# Cleaning web/client
-	@$(MAKE_C) services/web/client clean-files
+	# Cleaning static-webserver/client
+	@$(MAKE_C) services/static-webserver/client clean-files
 
 clean-more: ## cleans containers and unused volumes
 	# stops and deletes running containers
@@ -636,7 +624,7 @@ clean-images: ## removes all created images
 	-$(foreach service,$(SERVICES_LIST)\
 		,docker image rm --force $(shell docker images */$(service):* -q);)
 	# Cleaning webclient
-	@$(MAKE_C) services/web/client clean-images
+	@$(MAKE_C) services/static-webserver/client clean-images
 	# Cleaning postgres maintenance
 	@$(MAKE_C) packages/postgres-database/docker clean
 

ci/github/unit-testing/frontend.bash

@@ -7,16 +7,16 @@ IFS=$'\n\t'
 
 install() {
     npm install
-    make -C services/web/client clean
+    make -C services/static-webserver/client clean
     npx eslint --version
-    make -C services/web/client info
+    make -C services/static-webserver/client info
 }
 
 test() {
     echo "# Running Linter"
     npm run linter
 
-    pushd services/web/client
+    pushd services/static-webserver/client
 
     echo "# Building build version"
     make compile
@@ -56,9 +56,9 @@ test() {
     # cp -rp docs/webdocroot/* itisfoundation.github.io
 
     # # add our build
-    # if [ -d services/web/client/build-output ]; then
+    # if [ -d services/static-webserver/client/build-output ]; then
     #   rm -rf itisfoundation.github.io/frontend
-    #   cp -rp services/web/client/build-output itisfoundation.github.io/frontend
+    #   cp -rp services/static-webserver/client/build-output itisfoundation.github.io/frontend
     # fi
 }
 

docs/img/service-interaction.svg

@@ -5,7 +5,6 @@
 
 import json
 from pathlib import Path
-from typing import Dict
 
 import psycopg2.errors
 import pytest
@@ -20,21 +19,26 @@
 @pytest.fixture
 def web_client_resource_folder(osparc_simcore_root_dir: Path) -> Path:
     wcrf_path = (
-        osparc_simcore_root_dir / "services" / "web" / "client" / "source" / "resource"
+        osparc_simcore_root_dir
+        / "services"
+        / "static-webserver"
+        / "client"
+        / "source"
+        / "resource"
     )
     assert wcrf_path.exists()
     return wcrf_path
 
 
 @pytest.fixture
-def classifiers_bundle(web_client_resource_folder: Path) -> Dict:
+def classifiers_bundle(web_client_resource_folder: Path) -> dict:
     bundle_path = web_client_resource_folder / "dev" / "classifiers.json"
     assert bundle_path.exists()
     return json.loads(bundle_path.read_text())
 
 
 async def test_operations_on_group_classifiers(
-    pg_engine: Engine, classifiers_bundle: Dict
+    pg_engine: Engine, classifiers_bundle: dict
 ):
     # NOTE: mostly for TDD
     async with pg_engine.acquire() as conn:
@@ -85,6 +89,8 @@ async def test_operations_on_group_classifiers(
         assert (
             groups_count == 1
         ), "There should be only the Everyone group in the database!"
+        assert isinstance(classifiers_count, int)
+        assert isinstance(groups_count, int)
         assert classifiers_count <= groups_count
         assert classifiers_count == 0
 

docs/img/service-web.svg

@@ -57,7 +57,7 @@ def osparc_simcore_scripts_dir(osparc_simcore_root_dir: Path) -> Path:
 
 @pytest.fixture(scope="session")
 def osparc_simcore_web_client_dir(services_dir: Path) -> Path:
-    wbc_dir = services_dir / "web/client"
+    wbc_dir = services_dir / "static-webserver/client"
     assert wbc_dir.exists()
     return wbc_dir
 

packages/postgres-database/tests/test_classifiers.py

@@ -137,8 +137,14 @@ services:
   static-webserver:
     image: local/static-webserver:${BUILD_TARGET:?build_target_required}
     build:
-      context: ../
-      dockerfile: services/static-webserver/Dockerfile
+      context: ./static-webserver/client
+      dockerfile: tools/qooxdoo-kit/builder/Dockerfile
+      args:
+        - tag=latest
+        - VCS_REF=${VCS_REF}
+        - VCS_REF_CLIENT=${VCS_REF_CLIENT}
+        - VCS_STATUS_CLIENT=${VCS_STATUS_CLIENT}
+        - VCS_URL=${VCS_URL}
       cache_from:
         - local/static-webserver:${BUILD_TARGET:?build_target_required}
         - ${DOCKER_REGISTRY:-itisfoundation}/static-webserver:master-github-latest

packages/pytest-simcore/src/pytest_simcore/repository_paths.py

@@ -15,7 +15,7 @@
 # Nonetheless, notice that minio is a service used from simcore stack. Therefore, disabling ops stack
 # is meaningful ONLY when simcore stack is intended to run with the the actual stacks from osparc-ops
 #
-#  NOTE: this stack cannot be called tools because it collides with default network created in services/web/client/tools/docker-compose.yml
+#  NOTE: this stack cannot be called tools because it collides with default network created in services/static-webserver/client/tools/docker-compose.yml
 #  IMPORTANT: This stack IS NOT used in the deployed version
 version: "3.8"
 

services/docker-compose-build.yml

@@ -63,15 +63,14 @@ services:
 
   static-webserver:
     volumes:
-      - ./web/client/source-output:/static-content
+      - ./static-webserver/client/source-output:/static-content
     environment:
       - SERVER_LOG_LEVEL=info
 
   webserver:
     volumes:
       &webserver-volumes-dev
       - ./web/server:/devel/services/web/server
-      - ./web/client/source-output:/devel/services/web/client
       - ../packages:/devel/packages
     environment:
       &webserver-environment-dev