Makefile

SHELL=/bin/bash -o pipefail
BUILD_PRINT = \e[1;34mSTEP:
END_BUILD_PRINT = \e[0m

CURRENT_UID := $(shell id -u)
export CURRENT_UID
# These are constants used for make targets so we can start prod and staging services on the same machine
ENV_FILE := .env

# include .env files if they exist
-include .env

PROJECT_PATH = $(shell pwd)
AIRFLOW_INFRA_FOLDER ?= ${PROJECT_PATH}/.airflow
SRC_PATH := $(PROJECT_PATH)/src
INFRA_FOLDER_PATH := $(SRC_PATH)/infra
LIBRARIES_PATH = ${PROJECT_PATH}/libraries
RML_MAPPER_PATH = ${LIBRARIES_PATH}/.rmlmapper/rmlmapper.jar
XML_PROCESSOR_PATH = ${LIBRARIES_PATH}/.saxon/saxon-he-10.9.jar
LIMES_ALIGNMENT_PATH = $(LIBRARIES_PATH)/.limes/limes.jar
HOSTNAME = $(shell hostname)
CAROOT = $(shell pwd)/src/infra/traefik/certs

#-----------------------------------------------------------------------------
# Dev commands
#-----------------------------------------------------------------------------
install:
	@ echo -e "$(BUILD_PRINT)Installing the requirements$(END_BUILD_PRINT)"
	@ pip install --upgrade pip
	@ pip install --no-cache-dir -r requirements.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.5/constraints-no-providers-3.10.txt"

install-dev:
	@ echo -e "$(BUILD_PRINT)Installing the dev requirements$(END_BUILD_PRINT)"
	@ pip install --upgrade pip
	@ pip install --no-cache-dir -r requirements.dev.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.5/constraints-no-providers-3.10.txt"

test: test-unit

test-unit:
	@ echo -e "$(BUILD_PRINT)Unit Testing ...$(END_BUILD_PRINT)"
	@ tox -e unit

test-features:
	@ echo -e "$(BUILD_PRINT)Gherkin Features Testing ...$(END_BUILD_PRINT)"
	@ tox -e features

test-e2e:
	@ echo -e "$(BUILD_PRINT)End to End Testing ...$(END_BUILD_PRINT)"
	@ tox -e e2e

test-all-parallel:
	@ echo -e "$(BUILD_PRINT)Complete Testing ...$(END_BUILD_PRINT)"
	@ tox -p

test-all:
	@ echo -e "$(BUILD_PRINT)Complete Testing ...$(END_BUILD_PRINT)"
	@ tox

build-externals:
	@ echo -e "$(BUILD_PRINT)Creating the necessary volumes, networks and folders and setting the special rights"
	@ docker network create proxy-net || true
	@ docker network create common-ext-${ENVIRONMENT} || true


#-----------------------------------------------------------------------------
# SERVER SERVICES
#-----------------------------------------------------------------------------
start-traefik: build-externals
	@ echo -e "$(BUILD_PRINT)Starting the Traefik services $(END_BUILD_PRINT)"
	@ docker-compose -p common --file $(INFRA_FOLDER_PATH)/traefik/docker-compose.yml --env-file ${ENV_FILE} up -d

stop-traefik:
	@ echo -e "$(BUILD_PRINT)Stopping the Traefik services $(END_BUILD_PRINT)"
	@ docker-compose -p common --file $(INFRA_FOLDER_PATH)/traefik/docker-compose.yml --env-file ${ENV_FILE} down

start-portainer: build-externals
	@ echo -e "$(BUILD_PRINT)Starting the Portainer services $(END_BUILD_PRINT)"
	@ docker-compose -p common --file $(INFRA_FOLDER_PATH)/portainer/docker-compose.yml --env-file ${ENV_FILE} up -d

stop-portainer:
	@ echo -e "$(BUILD_PRINT)Stopping the Portainer services $(END_BUILD_PRINT)"
	@ docker-compose -p common --file $(INFRA_FOLDER_PATH)/portainer/docker-compose.yml --env-file ${ENV_FILE} down

start-server-services: | start-traefik start-portainer
stop-server-services: | stop-traefik stop-portainer

#-----------------------------------------------------------------------------
# PROJECT SERVICES
#-----------------------------------------------------------------------------
create-env-airflow:
	@ echo -e "$(BUILD_PRINT) Create Airflow env $(END_BUILD_PRINT)"
	@ echo -e "$(BUILD_PRINT) ${AIRFLOW_INFRA_FOLDER} ${ENVIRONMENT} $(END_BUILD_PRINT)"
	@ mkdir -p ${AIRFLOW_INFRA_FOLDER}/logs ${AIRFLOW_INFRA_FOLDER}/plugins
	@ ln -s -f ${PROJECT_PATH}/.env ${AIRFLOW_INFRA_FOLDER}/.env
	@ ln -s -f -n ${PROJECT_PATH}/src ${AIRFLOW_INFRA_FOLDER}/src
	@ chmod 777 ${AIRFLOW_INFRA_FOLDER}/logs ${AIRFLOW_INFRA_FOLDER}/plugins ${AIRFLOW_INFRA_FOLDER}/.env
	@ cp requirements.txt $(INFRA_FOLDER_PATH)/airflow/
	@ mkdir -p $(INFRA_FOLDER_PATH)/airflow/src
	@ cp -r src/ted_sws src/dags $(INFRA_FOLDER_PATH)/airflow/src
	@ cp -r libraries $(INFRA_FOLDER_PATH)/airflow/


build-airflow: guard-ENVIRONMENT create-env-airflow build-externals
	@ echo -e "$(BUILD_PRINT) Build Airflow services $(END_BUILD_PRINT)"
	@ docker build -t meaningfy/airflow $(INFRA_FOLDER_PATH)/airflow/
	@ docker-compose -p ${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/airflow/docker-compose.yaml --env-file ${ENV_FILE} up -d --force-recreate

#--------------------------------------AIRFLOW_CLUSTER----BEGIN----TARGETS----------------------------------------------

create-env-airflow-cluster:
	@ echo -e "$(BUILD_PRINT) Create Airflow env $(END_BUILD_PRINT)"
	@ echo -e "$(BUILD_PRINT) ${AIRFLOW_INFRA_FOLDER} ${ENVIRONMENT} $(END_BUILD_PRINT)"
	@ mkdir -p ${AIRFLOW_INFRA_FOLDER}/logs ${AIRFLOW_INFRA_FOLDER}/plugins
	@ ln -s -f ${PROJECT_PATH}/.env ${AIRFLOW_INFRA_FOLDER}/.env
	@ ln -s -f -n ${PROJECT_PATH}/src ${AIRFLOW_INFRA_FOLDER}/src
	@ chmod 777 ${AIRFLOW_INFRA_FOLDER}/logs ${AIRFLOW_INFRA_FOLDER}/plugins ${AIRFLOW_INFRA_FOLDER}/.env
	@ cp requirements.txt $(INFRA_FOLDER_PATH)/airflow-cluster/

build-airflow-cluster: guard-ENVIRONMENT create-env-airflow-cluster build-externals
	@ echo -e "$(BUILD_PRINT) Build Airflow Common Image $(END_BUILD_PRINT)"
	@ docker build -t meaningfy/airflow $(INFRA_FOLDER_PATH)/airflow-cluster/

start-airflow-master: build-externals
	@ echo -e "$(BUILD_PRINT)Starting Airflow Master $(END_BUILD_PRINT)"
	@ docker-compose -p ${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/airflow-cluster/docker-compose.yaml --env-file ${ENV_FILE} up -d --force-recreate

start-airflow-worker: build-externals
	@ echo -e "$(BUILD_PRINT)Starting Airflow Worker $(END_BUILD_PRINT)"
	@ docker-compose -p ${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/airflow-cluster/docker-compose-worker.yaml --env-file ${ENV_FILE} up -d

stop-airflow-master:
	@ echo -e "$(BUILD_PRINT)Stopping Airflow Master $(END_BUILD_PRINT)"
	@ docker-compose -p ${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/airflow-cluster/docker-compose.yaml --env-file ${ENV_FILE} down

stop-airflow-worker:
	@ echo -e "$(BUILD_PRINT)Stopping Airflow Worker $(END_BUILD_PRINT)"
	@ docker-compose -p ${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/airflow-cluster/docker-compose-worker.yaml --env-file ${ENV_FILE} down


#---------------------------------------AIRFLOW_CLUSTER----END----TARGETS-----------------------------------------------

start-airflow: build-externals
	@ echo -e "$(BUILD_PRINT)Starting Airflow services $(END_BUILD_PRINT)"
	@ docker-compose -p ${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/airflow/docker-compose.yaml --env-file ${ENV_FILE} up -d

stop-airflow:
	@ echo -e "$(BUILD_PRINT)Stopping Airflow services $(END_BUILD_PRINT)"
	@ docker-compose -p ${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/airflow/docker-compose.yaml --env-file ${ENV_FILE} down

#	------------------------
start-allegro-graph: build-externals
	@ echo -e "$(BUILD_PRINT)Starting Allegro-Graph services $(END_BUILD_PRINT)"
	@ docker-compose -p ${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/allegro-graph/docker-compose.yml --env-file ${ENV_FILE} up -d

stop-allegro-graph:
	@ echo -e "$(BUILD_PRINT)Stopping Allegro-Graph services $(END_BUILD_PRINT)"
	@ docker-compose -p ${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/allegro-graph/docker-compose.yml --env-file ${ENV_FILE} down

#	------------------------
start-fuseki: build-externals
	@ echo -e "$(BUILD_PRINT)Starting Fuseki services $(END_BUILD_PRINT)"
	@ docker-compose -p ${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/fuseki/docker-compose.yml --env-file ${ENV_FILE} up -d

stop-fuseki:
	@ echo -e "$(BUILD_PRINT)Stopping Fuseki services $(END_BUILD_PRINT)"
	@ docker-compose -p ${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/fuseki/docker-compose.yml --env-file ${ENV_FILE} down

#	------------------------
start-sftp: build-externals
	@ echo -e "$(BUILD_PRINT)Starting SFTP services $(END_BUILD_PRINT)"
	@ docker-compose -p ${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/sftp/docker-compose.yml --env-file ${ENV_FILE} up -d

stop-sftp:
	@ echo -e "$(BUILD_PRINT)Stopping SFTP services $(END_BUILD_PRINT)"
	@ docker-compose -p ${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/sftp/docker-compose.yml --env-file ${ENV_FILE} down

#	------------------------
build-elasticsearch: build-externals
	@ echo -e "$(BUILD_PRINT) Build Elasticsearch services $(END_BUILD_PRINT)"
	@ docker-compose -p ${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/elasticsearch/docker-compose.yml --env-file ${ENV_FILE} build --no-cache --force-rm
	@ docker-compose -p ${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/elasticsearch/docker-compose.yml --env-file ${ENV_FILE} up -d --force-recreate

start-elasticsearch: build-externals
	@ echo -e "$(BUILD_PRINT)Starting the Elasticsearch services $(END_BUILD_PRINT)"
	@ docker-compose -p ${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/elasticsearch/docker-compose.yml --env-file ${ENV_FILE} up -d

stop-elasticsearch:
	@ echo -e "$(BUILD_PRINT)Stopping the Elasticsearch services $(END_BUILD_PRINT)"
	@ docker-compose -p ${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/elasticsearch/docker-compose.yml --env-file ${ENV_FILE} down


start-minio: build-externals
	@ echo -e "$(BUILD_PRINT)Starting the Minio services $(END_BUILD_PRINT)"
	@ docker-compose -p ${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/minio/docker-compose.yml --env-file ${ENV_FILE} up -d

stop-minio:
	@ echo -e "$(BUILD_PRINT)Stopping the Minio services $(END_BUILD_PRINT)"
	@ docker-compose -p ${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/minio/docker-compose.yml --env-file ${ENV_FILE} down


start-mongo: build-externals
	@ echo -e "$(BUILD_PRINT)Starting the Mongo services $(END_BUILD_PRINT)"
	@ docker-compose -p ${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/mongo/docker-compose.yml --env-file ${ENV_FILE} up -d

stop-mongo:
	@ echo -e "$(BUILD_PRINT)Stopping the Mongo services $(END_BUILD_PRINT)"
	@ docker-compose -p ${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/mongo/docker-compose.yml --env-file ${ENV_FILE} down

start-metabase: build-externals
	@ echo -e "$(BUILD_PRINT)Starting the Metabase services $(END_BUILD_PRINT)"
	@ docker-compose -p metabase-${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/metabase/docker-compose.yml --env-file ${ENV_FILE} up -d

stop-metabase:
	@ echo -e "$(BUILD_PRINT)Stopping the Metabase services $(END_BUILD_PRINT)"
	@ docker-compose -p metabase-${ENVIRONMENT} --file $(INFRA_FOLDER_PATH)/metabase/docker-compose.yml --env-file ${ENV_FILE} down

init-rml-mapper:
	@ echo -e "RMLMapper folder initialisation!"
	@ mkdir -p ./libraries/.rmlmapper
	@ wget -c https://github.com/RMLio/rmlmapper-java/releases/download/v6.2.2/rmlmapper-6.2.2-r371-all.jar -O ./libraries/.rmlmapper/rmlmapper.jar

init-limes:
	@ echo -e "Limes folder initialisation!"
	@ mkdir -p ./libraries/.limes
	@ wget -c https://github.com/dice-group/LIMES/releases/download/1.7.9/limes.jar -P ./libraries/.limes/

init-saxon:
	@ echo -e "$(BUILD_PRINT)Saxon folder initialization $(END_BUILD_PRINT)"
	@ mkdir -p ./libraries/.saxon
	@ wget -c https://github.com/Saxonica/Saxon-HE/releases/download/SaxonHE10-9/SaxonHE10-9J.zip -P ./libraries/.saxon/
	@ cd ./libraries/.saxon/ && unzip SaxonHE10-9J.zip && rm -rf SaxonHE10-9J.zip

init-rml-mapper-curl:
	@echo "RMLMapper folder initialisation!"
	@mkdir -p ./libraries/.rmlmapper
	@curl -L -o ./libraries/.rmlmapper/rmlmapper.jar https://github.com/RMLio/rmlmapper-java/releases/download/v6.2.2/rmlmapper-6.2.2-r371-all.jar

init-limes-curl:
	@echo "Limes folder initialisation!"
	@mkdir -p ./libraries/.limes
	@curl -L -o ./libraries/.limes/limes.jar https://github.com/dice-group/LIMES/releases/download/1.7.9/limes.jar

init-saxon-curl:
	@echo "$(BUILD_PRINT)Saxon folder initialization $(END_BUILD_PRINT)"
	@mkdir -p ./libraries/.saxon
	@curl -L -o ./libraries/.saxon/SaxonHE10-9J.zip https://github.com/Saxonica/Saxon-HE/releases/download/SaxonHE10-9/SaxonHE10-9J.zip
	@cd ./libraries/.saxon && unzip -o SaxonHE10-9J.zip && rm -f SaxonHE10-9J.zip

start-project-services: | start-airflow start-mongo init-rml-mapper init-limes start-allegro-graph start-metabase
stop-project-services: | stop-airflow stop-mongo stop-allegro-graph stop-metabase

init-libraries:
	@echo "$(BUILD_PRINT)Initializing libraries (downloads only if missing) $(END_BUILD_PRINT)"
	@[ -f ./libraries/.rmlmapper/rmlmapper.jar ] || \
		(mkdir -p ./libraries/.rmlmapper && \
		 curl -L -o ./libraries/.rmlmapper/rmlmapper.jar \
		 https://github.com/RMLio/rmlmapper-java/releases/download/v6.2.2/rmlmapper-6.2.2-r371-all.jar)
	@[ -f ./libraries/.limes/limes.jar ] || \
		(mkdir -p ./libraries/.limes && \
		 curl -L -o ./libraries/.limes/limes.jar \
		 https://github.com/dice-group/LIMES/releases/download/1.7.9/limes.jar)
	@[ -f ./libraries/.saxon/saxon-he-10.9.jar ] || \
		(mkdir -p ./libraries/.saxon && \
		 curl -L -o ./libraries/.saxon/SaxonHE10-9J.zip \
		 https://github.com/Saxonica/Saxon-HE/releases/download/SaxonHE10-9/SaxonHE10-9J.zip && \
		 cd ./libraries/.saxon && unzip -o SaxonHE10-9J.zip && rm -f SaxonHE10-9J.zip)

#-----------------------------------------------------------------------------
# UNIFIED STACK (ted-sws-stack)
#-----------------------------------------------------------------------------
STACK_PATH = $(INFRA_FOLDER_PATH)/ted-sws-stack

start-local-stack: init-libraries
	@echo "$(BUILD_PRINT)Building Airflow image $(END_BUILD_PRINT)"
	@docker build -t tedsws/airflow:local $(STACK_PATH)/airflow
	@echo "$(BUILD_PRINT)Starting TED-SWS local stack $(END_BUILD_PRINT)"
	@docker compose -f $(STACK_PATH)/docker-compose.yml -f $(STACK_PATH)/docker-compose.local.yml --env-file $(STACK_PATH)/.env.local up -d $(SERVICES)

stop-local-stack:
	@echo "$(BUILD_PRINT)Stopping TED-SWS local stack $(END_BUILD_PRINT)"
	@docker compose -f $(STACK_PATH)/docker-compose.yml -f $(STACK_PATH)/docker-compose.local.yml --env-file $(STACK_PATH)/.env.local down

cleanup-local-stack:
	@echo "$(BUILD_PRINT)Cleaning up TED-SWS local stack $(END_BUILD_PRINT)"
	@docker compose -f $(STACK_PATH)/docker-compose.yml -f $(STACK_PATH)/docker-compose.local.yml --env-file $(STACK_PATH)/.env.local down -v --rmi local --remove-orphans
	@docker builder prune -f --filter label=com.docker.compose.project=ted-sws-stack
	@docker rmi tedsws/airflow:local 2>/dev/null || true

start-local-stack-nodata: init-libraries
	@echo "$(BUILD_PRINT)Building Airflow image $(END_BUILD_PRINT)"
	@docker build -t tedsws/airflow:local $(STACK_PATH)/airflow
	@echo "$(BUILD_PRINT)Starting TED-SWS local stack (no persistent data) $(END_BUILD_PRINT)"
	@docker compose -f $(STACK_PATH)/docker-compose.yml -f $(STACK_PATH)/docker-compose.local.yml -f $(STACK_PATH)/docker-compose.local-nodata.yml --env-file $(STACK_PATH)/.env.local up -d $(SERVICES)

#-----------------------------------------------------------------------------
# VAULT SERVICES
#-----------------------------------------------------------------------------
# Testing whether an env variable is set or not
guard-%:
	@ if [ "${${*}}" = "" ]; then \
        echo -e "$(BUILD_PRINT)Environment variable $* not set $(END_BUILD_PRINT)"; \
        exit 1; \
	fi

# Testing that vault is installed
vault-installed: #; @which vault1 > /dev/null
	@ if ! hash vault 2>/dev/null; then \
        echo -e "$(BUILD_PRINT)Vault is not installed, refer to https://www.vaultproject.io/downloads $(END_BUILD_PRINT)"; \
        exit 1; \
	fi

# Get secrets in dotenv format (unified stack - passwords only from Vault)
# Non-secret app configs come from .env.common via env_file in compose overrides
staging-unified-dotenv: guard-VAULT_ADDR guard-VAULT_TOKEN vault-installed
	@ echo -e "$(BUILD_PRINT)Creating unified stack .env.staging from Vault $(END_BUILD_PRINT)"
	@ VAULT_JSON=$$(vault kv get -format="json" ted-staging/ted-sws-deployment-secrets) && \
	  MONGO_PW=$$(echo "$$VAULT_JSON" | jq -r '.data.data.MONGO_ROOT_PASSWORD') && \
	  MINIO_PW=$$(echo "$$VAULT_JSON" | jq -r '.data.data.MINIO_ROOT_PASSWORD') && \
	  { \
	    echo "$$VAULT_JSON" | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\""; \
	    echo "MONGO_DB_AUTH_URL=mongodb://admin:$$MONGO_PW@mongodb:27017/"; \
	    echo "S3_PUBLISH_PASSWORD=$$MINIO_PW"; \
	    echo "ENVIRONMENT=staging"; \
	    echo "SUBDOMAIN=tedsws-staging."; \
	    echo "DOMAIN=meaningfy.ws"; \
	    echo "AIRFLOW_INFRA_FOLDER=/opt/tedsws"; \
	    echo "AIRFLOW__CORE__PARALLELISM=16"; \
	    echo "AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG=8"; \
	    echo "AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG=8"; \
	    echo "AIRFLOW__CELERY__WORKER_CONCURRENCY=8"; \
	  } > $(STACK_PATH)/.env.staging

# Get secrets in dotenv format (old - pulls everything from multiple Vault paths)
staging-dotenv-file: guard-VAULT_ADDR guard-VAULT_TOKEN vault-installed
	@ echo -e "$(BUILD_PRINT)Creating .env.staging file $(END_BUILD_PRINT)"
	@ echo VAULT_ADDR=${VAULT_ADDR} > .env
	@ echo VAULT_TOKEN=${VAULT_TOKEN} >> .env
	@ echo ENVIRONMENT=staging >> .env
	@ echo SUBDOMAIN=staging. >> .env
	@ echo RML_MAPPER_PATH=${RML_MAPPER_PATH} >> .env
	@ echo LIMES_ALIGNMENT_PATH=${LIMES_ALIGNMENT_PATH} >> .env
	@ echo XML_PROCESSOR_PATH=${XML_PROCESSOR_PATH} >> .env
	@ echo AIRFLOW_INFRA_FOLDER=~/airflow-infra/staging >> .env
	@ echo AIRFLOW_WORKER_HOSTNAME=${HOSTNAME} >> .env
	@ vault kv get -format="json" ted-staging/airflow | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
	@ vault kv get -format="json" ted-staging/mongo-db | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
	@ echo 'MONGO_DB_AUTH_URL=mongodb://$${MONGO_INITDB_ROOT_USERNAME}:$${MONGO_INITDB_ROOT_PASSWORD}@mongodb-staging:27017/' >> .env
	@ vault kv get -format="json" ted-staging/metabase | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
	@ vault kv get -format="json" ted-staging/ted-sws | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
	@ vault kv get -format="json" ted-staging/fuseki | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
	@ echo 'FUSEKI_ADMIN_HOST=http://fuseki-staging:3030' >> .env
	@ vault kv get -format="json" ted-staging/github | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
	@ vault kv get -format="json" ted-staging/minio | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
	@ echo 'S3_PUBLISH_HOST=minio:9000' >> .env
	@ echo 'SFTP_PUBLISH_HOST=sftp' >> .env
	@ echo "# Concurrency limits for staging (4-core, 15GB VM)" >> .env
	@ echo AIRFLOW__CORE__PARALLELISM=8 >> .env
	@ echo AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG=4 >> .env
	@ echo AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG=4 >> .env
	@ echo AIRFLOW__CELERY__WORKER_CONCURRENCY=4 >> .env



prod-dotenv-file: guard-VAULT_ADDR guard-VAULT_TOKEN vault-installed
	@ echo -e "$(BUILD_PRINT)Create .env file $(END_BUILD_PRINT)"
	@ echo VAULT_ADDR=${VAULT_ADDR} > .env
	@ echo VAULT_TOKEN=${VAULT_TOKEN} >> .env
	@ echo ENVIRONMENT=prod >> .env
	@ echo SUBDOMAIN= >> .env
	@ echo RML_MAPPER_PATH=${RML_MAPPER_PATH} >> .env
	@ echo LIMES_ALIGNMENT_PATH=${LIMES_ALIGNMENT_PATH} >> .env
	@ echo XML_PROCESSOR_PATH=${XML_PROCESSOR_PATH} >> .env
	@ echo AIRFLOW_INFRA_FOLDER=~/airflow-infra/prod >> .env
	@ echo AIRFLOW_WORKER_HOSTNAME=${HOSTNAME} >> .env
	@ echo AIRFLOW_CELERY_WORKER_CONCURRENCY=32 >> .env
	@ vault kv get -format="json" ted-prod/airflow | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
	@ vault kv get -format="json" ted-prod/mongo-db | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
	@ vault kv get -format="json" ted-prod/metabase | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
	@ vault kv get -format="json" ted-prod/fuseki | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
	@ vault kv get -format="json" ted-prod/ted-sws | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
	@ vault kv get -format="json" ted-prod/github | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
	@ vault kv get -format="json" ted-prod/minio | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env

local-dotenv-file: rml-mapper-path-add-dotenv-file

rml-mapper-path-add-dotenv-file:
	@ echo -e "$(BUILD_PRINT)Add rml-mapper path to local .env file $(END_BUILD_PRINT)"
	@ sed -i '/^RML_MAPPER_PATH/d' .env
	@ echo RML_MAPPER_PATH=${RML_MAPPER_PATH} >> .env

#clean-mongo-db:
#	@ export PYTHONPATH=$(PWD) && python ./tests/clean_mongo_db.py


#build-open-semantic-search:
#	@ echo -e "Build open-semantic-search"
#	@ cd infra && rm -rf open-semantic-search
#	@ cd infra && git clone --recurse-submodules --remote-submodules https://github.com/opensemanticsearch/open-semantic-search.git
#	@ cd infra/open-semantic-search/ && ./build-deb
#	@ echo -e "Patch open-semantic-search configs"
#	@ cat infra/docker-compose-configs/open-semantic-search-compose-patch.yml > infra/open-semantic-search/docker-compose.yml
#	@ cd infra/open-semantic-search/ && docker-compose rm -fsv
#	@ cd infra/open-semantic-search/ && docker-compose build
#
#start-open-semantic-search:
#	@ echo -e "Start open-semantic-search"
#	@ cd infra/open-semantic-search/ && docker-compose up -d
#
#
#stop-open-semantic-search:
#	@ echo -e "Stop open-semantic-search"
#	@ cd infra/open-semantic-search/ && docker-compose down
#
#
#start-silk-service:
#	@ echo -e "Start silk service"
#	@ cd infra/silk/ && docker-compose up -d
#
#stop-silk-service:
#	@ echo -e "Stop silk service"
#	@ cd infra/silk/ && docker-compose down


#-----------------------------------------------------------------------------
# API Service commands
#-----------------------------------------------------------------------------
build-all-apis: build-digest_service-api

start-all-apis: start-digest_service-api

stop-all-apis: stop-digest_service-api

create-env-digest-api:
	@ cp requirements.txt $(INFRA_FOLDER_PATH)/digest_api/digest_service/project_requirements.txt
	@ cp -r $(SRC_PATH)/ted_sws $(INFRA_FOLDER_PATH)/digest_api/

build-digest_service-api: create-env-digest-api
	@ echo -e "$(BUILD_PRINT) Build digest_service API service $(END_BUILD_PRINT)"
	@ docker-compose -p common --file $(INFRA_FOLDER_PATH)/digest_api/docker-compose.yml --env-file ${ENV_FILE} build --no-cache --force-rm
	@ rm -rf $(INFRA_FOLDER_PATH)/digest_api/ted_sws || true
	@ docker-compose -p common --file $(INFRA_FOLDER_PATH)/digest_api/docker-compose.yml --env-file ${ENV_FILE} up -d --force-recreate

start-digest_service-api:
	@ echo -e "$(BUILD_PRINT)Starting digest_service API service $(END_BUILD_PRINT)"
	@ docker-compose -p common --file $(INFRA_FOLDER_PATH)/digest_api/docker-compose.yml --env-file ${ENV_FILE} up -d

stop-digest_service-api:
	@ echo -e "$(BUILD_PRINT)Stopping digest_service API service $(END_BUILD_PRINT)"
	@ docker-compose -p common --file $(INFRA_FOLDER_PATH)/digest_api/docker-compose.yml --env-file ${ENV_FILE} down


dump-mongodb:
	@ echo -e "Start dump data from mongodb."
	@ docker exec -i mongodb-${ENVIRONMENT} /usr/bin/mongodump --username ${ME_CONFIG_MONGODB_ADMINUSERNAME} --password ${ME_CONFIG_MONGODB_ADMINPASSWORD} --authenticationDatabase admin --db aggregates_db --out /mongodb_dump
	@ mv ./mongodb_dump/aggregates_db "./mongodb_dump/aggregates_db_$$(date +"%Y_%m_%d_%H_%M_%S")" 2>/dev/null || true
	@ docker cp mongodb-${ENVIRONMENT}:/mongodb_dump .
	@ docker exec -it mongodb-${ENVIRONMENT} rm -rf mongodb_dump
	@ echo -e "Finish dump data from mongodb."


restore-mongodb:
	@ echo -e "Start restore data in mongodb."
	@ docker cp ./mongodb_dump mongodb-${ENVIRONMENT}:/mongodb_dump
	@ docker exec -i mongodb-${ENVIRONMENT} /usr/bin/mongorestore --username ${ME_CONFIG_MONGODB_ADMINUSERNAME} --password ${ME_CONFIG_MONGODB_ADMINPASSWORD} --authenticationDatabase admin --db aggregates_db /mongodb_dump/aggregates_db
	@ docker exec -it mongodb-${ENVIRONMENT} rm -rf mongodb_dump
	@ echo -e "Finish restore data in mongodb."

install-allure:
	@ echo -e "Start install Allure commandline."
	@ sudo apt -y install npm
	@ sudo npm install allure-commandline@2.9.0
	#TODO: to be added in pyproject
	@ pip install allure-combine==1.0.11

install-mkcert:
	@ mkdir -p .ssl && cd .ssl && rm -rf *
	@ curl -JLO "https://dl.filippo.io/mkcert/latest?for=linux/amd64"
	@ chmod +x mkcert-v*-linux-amd64
	@ sudo mv mkcert-v*-linux-amd64 /usr/local/bin/mkcert
	@ cd ..
	@ sudo apt install ca-certificates

traefik-certs:
	@ cd $(INFRA_FOLDER_PATH)/traefik && mkdir -p certs && cd certs && sudo rm -rf *
	@ CAROOT=${CAROOT} mkcert -install
	@ echo -e "Generating 'minio' certificates ..." && echo ${CAROOT}
	@ sudo echo $(mkcert -CAROOT)
	@ cd $(INFRA_FOLDER_PATH)/traefik/certs && mkcert minio.${SUBDOMAIN}${DOMAIN}
	@ cd $(INFRA_FOLDER_PATH)/traefik/certs && cat minio.${SUBDOMAIN}${DOMAIN}.pem > minio.${SUBDOMAIN}${DOMAIN}-fullchain.pem
	@ cd $(INFRA_FOLDER_PATH)/traefik/certs && cat ${CAROOT}/rootCA.pem >> minio.${SUBDOMAIN}${DOMAIN}-fullchain.pem
	@ sudo rm -rf /usr/share/ca-certificates/minio.${SUBDOMAIN}${DOMAIN}*
	@ sudo cp $(INFRA_FOLDER_PATH)/traefik/certs/minio.${SUBDOMAIN}${DOMAIN}* /usr/share/ca-certificates