diff --git a/localinstall/Containerfile b/localinstall/Containerfile index 2fb255c9..a2e1787a 100644 --- a/localinstall/Containerfile +++ b/localinstall/Containerfile @@ -42,6 +42,12 @@ RUN groupadd -g ${GROUP_ID} kernelci || true && \ RUN usermod -aG sudo kernelci && \ echo "kernelci ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers +# Install secondary dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + gettext \ + jq + USER kernelci WORKDIR /home/kernelci diff --git a/localinstall/config/lava-boards-template.yaml b/localinstall/config/lava-boards-template.yaml new file mode 100644 index 00000000..d60f7221 --- /dev/null +++ b/localinstall/config/lava-boards-template.yaml @@ -0,0 +1,27 @@ +--- +masters: + - name: lava-master-local + host: local + users: + - name: ${LAVA_USER} + password: ${LAVA_PASSWORD} + token: admin-token + superuser: true + staff: true + tokens: + - username: ${LAVA_USER} + token: ${LAVA_TOKEN} + description: ${LAVA_TOKEN_DESCRIPTION} + write_dot_env: True + webinterface_port: 10070 +slaves: + - name: lava-slave-local + host: local + remote_master: lava-master-local + remote_user: ${LAVA_USER} + use_docker: true + +boards: + - name: qemu-01 + type: qemu + diff --git a/localinstall/config/main.cfg b/localinstall/config/main.cfg index 7178c952..e3f8f635 100755 --- a/localinstall/config/main.cfg +++ b/localinstall/config/main.cfg @@ -5,7 +5,15 @@ KCI_API_REPO=https://github.com/kernelci/kernelci-api KCI_API_BRANCH=staging.kernelci.org KCI_PIPELINE_REPO=https://github.com/kernelci/kernelci-pipeline KCI_PIPELINE_BRANCH=staging.kernelci.org +KCI_LAVA_REPO=https://github.com/BayLibre/lava-docker.git +KCI_LAVA_BRANCH=master YOUR_EMAIL=denys.f@collabora.com ADMIN_PASSWORD="Ch4n93m3HpL33z" STORAGE_TOKEN="FKDFLKJFLKDJFLK" +LAVA_USER="admin" +LAVA_PASSWORD="admin" +LAVA_TOKEN="local-kernelci-token" +LAVA_TOKEN_DESCRIPTION="local-kernelci-token-description" +KCI_INSTANCE="local" +KCI_INSTANCE_CALLBACK="http://172.17.0.1:8100" #KCI_CACHE=1 diff --git a/localinstall/scripts/1-rebuild_all.sh b/localinstall/scripts/1-rebuild_all.sh index 9978e96d..2bd21e42 100755 --- a/localinstall/scripts/1-rebuild_all.sh +++ b/localinstall/scripts/1-rebuild_all.sh @@ -38,6 +38,13 @@ if [ ! -d kernelci ]; then git fetch origin git checkout origin/$KCI_PIPELINE_BRANCH cd .. + + echo Clone LAVA repo + git clone $KCI_LAVA_REPO + cd lava-docker + git fetch origin + git checkout origin/$KCI_LAVA_BRANCH + cd .. else cd kernelci fi diff --git a/localinstall/scripts/5-prepare_lava.sh b/localinstall/scripts/5-prepare_lava.sh new file mode 100755 index 00000000..c4a5168c --- /dev/null +++ b/localinstall/scripts/5-prepare_lava.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Export vars for envsubst +set -a +. ./config/main.cfg +set +a + +set -e + +# Set LAVA secrets +envsubst < config/lava-boards-template.yaml > config/out/lava-boards.yaml + +# Copy config files +cp config/out/lava-boards.yaml kernelci/lava-docker/boards.yaml + +cd kernelci/lava-docker + +# generate docker-compose file +./lavalab-gen.sh + +# Build the Docker images +cd output/local +docker compose build diff --git a/localinstall/scripts/6-start_lava.sh b/localinstall/scripts/6-start_lava.sh new file mode 100755 index 00000000..56b760a2 --- /dev/null +++ b/localinstall/scripts/6-start_lava.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +. ./config/main.cfg + +set -e + +cd kernelci/lava-docker + +cd output/local +docker compose down +docker compose up -d + +echo "Waiting for LAVA master to be up" +sleep 15 +# loop until LAVA master is up, try 5 times +i=0 +while [ $i -lt 5 ]; do + if curl -s --fail http://localhost:10070/api/v0.2/system/ > /dev/null; then + echo "LAVA master is up!" + break + else + echo "LAVA master not ready yet, retrying..." + sleep 5 + fi +done + +echo "Waiting for LAVA worker to be up" +sleep 10 +# loop until LAVA worker is up, try 5 times +i=0 +while [ $i -lt 5 ]; do + ANSWER=$(curl -s http://localhost:10070/api/v0.2/workers/) + # must contain "hostname": "lava-slave" and "state": "Online" + if echo "$ANSWER" | jq -e '.results[] | select(.hostname == "lava-slave-local" and .state == "Online")' >/dev/null; then + echo "LAVA worker 'lava-slave-local' is online" + break + else + echo "LAVA worker not ready yet, retrying..." + i=$((i+1)) + sleep 5 + fi +done + diff --git a/localinstall/scripts/6-start_pipeline.sh b/localinstall/scripts/6-start_pipeline.sh deleted file mode 100755 index ad527016..00000000 --- a/localinstall/scripts/6-start_pipeline.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# is docker-compose exists? if not use docker compose -if [ -z "$(which docker-compose)" ]; then - echo "docker-compose is not installed, using docker compose" - DOCKER_COMPOSE="docker compose" -else - DOCKER_COMPOSE="docker-compose" -fi - -. ./config/main.cfg - -set -e - -cd kernelci/kernelci-pipeline -${DOCKER_COMPOSE} down -${DOCKER_COMPOSE} up -d -echo "You can view now logs of containers using docker logs -f or docker-compose logs -f in kernelci/kernelci-pipeline or kernelci/kernelci-api directories" -echo "Also you can do docker ps to see running containers, and in case of ongoing builds, you can view their logs too by docker logs -f " -echo "You can also open API viewer at http://127.0.0.1:8001/viewer" diff --git a/localinstall/scripts/5-prepare_pipeline.sh b/localinstall/scripts/7-prepare_pipeline.sh similarity index 85% rename from localinstall/scripts/5-prepare_pipeline.sh rename to localinstall/scripts/7-prepare_pipeline.sh index 053a89c8..2f198535 100755 --- a/localinstall/scripts/5-prepare_pipeline.sh +++ b/localinstall/scripts/7-prepare_pipeline.sh @@ -71,24 +71,13 @@ if ! grep -q "force = 1" kernelci/kernelci-pipeline/config/kernelci.toml; then sed -i '/\[trigger\]/a force = 1' kernelci/kernelci-pipeline/config/kernelci.toml fi -# remove from pipeline yaml all build_configs: -sed -i '/build_configs:/,$d' kernelci/kernelci-pipeline/config/pipeline.yaml -# add -cat <> kernelci/kernelci-pipeline/config/pipeline.yaml -build_configs: - kernelci_staging-stable: - tree: kernelci - branch: 'staging-stable' -EOF - #create .env -#KCI_STORAGE_CREDENTIALS=L0CALT0KEN -#KCI_API_TOKEN= -#API_TOKEN= API_TOKEN=$(cat config/out/admin-token.txt) echo "KCI_STORAGE_CREDENTIALS=/home/kernelci/data/ssh/id_rsa_tarball" > .env echo "KCI_API_TOKEN=${API_TOKEN}" >> .env echo "API_TOKEN=${API_TOKEN}" >> .env +echo "KCI_INSTANCE=${KCI_INSTANCE}" >> .env +echo "KCI_INSTANCE_CALLBACK=${KCI_INSTANCE_CALLBACK}" >> .env cp .env kernelci/kernelci-pipeline/.docker-env mv .env kernelci/kernelci-pipeline/.env @@ -101,3 +90,15 @@ TOKEN=$(kernelci/kernelci-pipeline/tools/jwt_generator.py --toml kernelci/kernel --email ${YOUR_EMAIL} --permissions checkout,testretry,patchset | grep "JWT token:" | cut -d' ' -f3) echo $TOKEN > config/out/kci-dev-token.txt echo "kci-dev token saved to config/out/kci-dev-token.txt" + +# set LAVA Token +# Check if [runtime.lava-local] section exists, if not add it +if ! grep -q "\[runtime\.lava-local\]" kernelci/kernelci-pipeline/config/kernelci.toml; then + echo -e "[runtime.lava-local]\nruntime_token = \"$LAVA_TOKEN\"\ncallback_token = \"$LAVA_TOKEN\"" >> kernelci/kernelci-pipeline/config/kernelci.toml +else + # Update existing tokens + sed -i '/\[runtime\.lava-local\]/,/callback_token/{ + s/runtime_token = ".*"/runtime_token = "'$LAVA_TOKEN'"/; + s/callback_token = ".*"/callback_token = "'$LAVA_TOKEN'"/; + }' kernelci/kernelci-pipeline/config/kernelci.toml +fi diff --git a/localinstall/scripts/8-start_pipeline.sh b/localinstall/scripts/8-start_pipeline.sh new file mode 100755 index 00000000..9c0fe0f0 --- /dev/null +++ b/localinstall/scripts/8-start_pipeline.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +. ./config/main.cfg + +set -e + +cd kernelci/kernelci-pipeline + +docker compose down +docker compose up -d diff --git a/localinstall/scripts/run.sh b/localinstall/scripts/run.sh index 7b1cffc0..d34846a8 100755 --- a/localinstall/scripts/run.sh +++ b/localinstall/scripts/run.sh @@ -38,15 +38,25 @@ case "$ACTION" in ./scripts/2-prepare_api.sh ./scripts/3-start_api.sh ./scripts/4-set_api_admin.sh - ./scripts/5-prepare_pipeline.sh - ./scripts/6-start_pipeline.sh + ./scripts/5-prepare_lava.sh + ./scripts/6-start_lava.sh + ./scripts/7-prepare_pipeline.sh + ./scripts/8-start_pipeline.sh + echo "You can view now logs of containers using docker logs -f or docker-compose logs -f in each directory" + echo "Also you can do docker ps to see running containers, and in case of ongoing builds, you can view their logs too by docker logs -f " + echo "API viewer available at http://localhost:8001/viewer" + echo "API endpoints available at http://localhost:8001" + echo "Storage is available at http://localhost:8002/" + echo "Pipeline callback available at http://localhost:8100" + echo "LAVA available at http://localhost:10070" touch kernelci/.done ;; start) check_deploy echo "Starting deployment" ./scripts/3-start_api.sh - ./scripts/6-start_pipeline.sh + ./scripts/6-start_lava.sh + ./scripts/8-start_pipeline.sh ;; stop) check_deploy @@ -55,6 +65,8 @@ case "$ACTION" in docker compose down cd ../kernelci-pipeline docker compose down + cd ../lava-docker/output/local + docker compose down ;; *) echo "Error: Invalid action '$ACTION'"