Merge pull request #745 from spack/gitlab-dev-environment

Add GitLab + GitLab Runner to dev environment

Author: mvandenburgh

analytics/README.md

@@ -7,6 +7,8 @@ This is the simplest configuration for developers to start with.
 1. Run `docker-compose run --rm django ./manage.py migrate`
 2. Run `docker-compose run --rm django ./manage.py createsuperuser`
    and follow the prompts to create your own user
+3. Run `dev/init-gitlab.sh` to initialize your GitLab instance root
+   password, register a runner, and bootstrap a `spack` repo.
 
 
 ### Run Application
@@ -40,6 +42,8 @@ but allows developers to run Python code on their native system.
 6. Run `source ./dev/export-env.sh`
 7. Run `./manage.py migrate`
 8. Run `./manage.py createsuperuser` and follow the prompts to create your own user
+9. Run `dev/init-gitlab.sh` to initialize your GitLab instance root
+   password, register a runner, and bootstrap a `spack` repo.
 
 ### Run Application
 1.  Ensure `docker-compose -f ./docker-compose.yml up -d` is still active

analytics/dev/.env.docker-compose

@@ -8,8 +8,8 @@ OPENSEARCH_ENDPOINT=http://elasticsearch:9200
 OPENSEARCH_USERNAME=elastic
 OPENSEARCH_PASSWORD=elastic
 SECRET_KEY=deadbeef
-GITLAB_ENDPOINT="http://fakeurl"
-GITLAB_TOKEN="bar"
+GITLAB_ENDPOINT="http://gitlab/"
+GITLAB_TOKEN="insecure_token"
 GITLAB_DB_USER=gitlab
 GITLAB_DB_HOST=gitlab-db
 GITLAB_DB_PORT=5432

analytics/dev/.env.docker-compose-native

@@ -7,8 +7,8 @@ CELERY_BROKER_URL=redis://localhost:6379/
 OPENSEARCH_ENDPOINT=http://localhost:9200
 OPENSEARCH_USERNAME=elastic
 OPENSEARCH_PASSWORD=elastic
-GITLAB_ENDPOINT=http://fakegitlab
-GITLAB_TOKEN=glpat-fakegitlab
+GITLAB_ENDPOINT=http://localhost:8080/
+GITLAB_TOKEN=insecure_token
 GITLAB_DB_USER=gitlab
 GITLAB_DB_HOST=localhost
 GITLAB_DB_PORT=5433

analytics/dev/init-gitlab.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+set -euo pipefail
+
+# Wait for gitlab to be up
+echo "Waiting for GitLab..."
+until curl -s -f -o /dev/null "http://localhost:8080"
+do
+    sleep 5
+done
+echo "Done."
+
+# Set root user password + create a personal access token for it
+PERSONAL_ACCESS_TOKEN="insecure_token"
+docker compose exec gitlab gitlab-rails runner "  \
+    print 'Updating root user password...'; \
+    user = User.find_by(username: 'root');                  \
+    user.password = user.password_confirmation = 'deadbeef'; \
+    user.save!; \
+    puts ' done'; \
+    print 'Creating personal access token...'; \
+    token = User.find_by_username('root').personal_access_tokens.create( \
+        scopes: [:api], \
+        name: 'Docker admin token', \
+        expires_at: 365.days.from_now, \
+    ); \
+    token.set_token('$PERSONAL_ACCESS_TOKEN'); \
+    token.save!; \
+    puts ' done';"
+
+# Create a new runner via the GitLab API and save its token
+RUNNER_TOKEN=$(docker compose exec gitlab curl \
+    --silent \
+    --request POST \
+    --url "http://localhost/api/v4/user/runners" \
+    --data "runner_type=instance_type" \
+    --data "description=test" \
+    --data "tag_list=test" \
+    --header "PRIVATE-TOKEN: $PERSONAL_ACCESS_TOKEN" | jq -r '.token')
+
+# Register the containerized gitlab runner using the previously acquired token
+docker compose exec gitlab-runner gitlab-runner register \
+  --non-interactive \
+  --description "docker-runner" \
+  --url "http://gitlab/" \
+  --token "$RUNNER_TOKEN" \
+  --executor "docker" \
+  --docker-volumes=/var/run/docker.sock:/var/run/docker.sock \
+  --docker-image=docker:20-dind
+
+# Create a new project
+docker compose exec gitlab curl \
+    --request POST \
+    --url "http://localhost/api/v4/projects" \
+    --header "PRIVATE-TOKEN: $PERSONAL_ACCESS_TOKEN" \
+    --header "Content-Type: application/json" \
+    --data '{"name": "spack", "path": "spack"}'
+
+# Push spack github repo to the new project
+docker compose exec gitlab bash -c " \
+    git clone https://github.com/spack/spack.git && \
+    cd spack && \
+    git remote add gitlab http://root:deadbeef@localhost/root/spack.git && \
+    git push -u gitlab develop"

analytics/docker-compose.yml

@@ -42,6 +42,45 @@ services:
       POSTGRES_USER: gitlab
       POSTGRES_PASSWORD: gitlab
 
+  gitlab:
+    image: zengxs/gitlab:ee
+    ports:
+      - "8080:80"
+      - "8081:443"
+    environment:
+      GITLAB_OMNIBUS_CONFIG: |
+        external_url 'http://gitlab'
+        gitlab_rails['db_host'] = 'gitlab-db'
+        gitlab_rails['db_port'] = '5432'
+        gitlab_rails['db_username'] = 'gitlab'
+        gitlab_rails['db_password'] = 'gitlab'
+    depends_on:
+      - gitlab-db
+
+  gitlab-runner:
+    image: gitlab/gitlab-runner:latest
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    depends_on:
+      - gitlab
+      - dind
+    environment:
+      DOCKER_HOST: tcp://dind:2375
+      # Use the host's network interface instead of the Docker one.
+      # Note, gitlab-runner runs jobs inside docker, but is itself deployed
+      # in a docker container. So the "host" in this case actually refers to the
+      # `gitlab-runner` docker container defined in this docker-compose.yaml.
+      DOCKER_NETWORK_MODE: host
+
+  dind:
+    image: docker:20-dind
+    restart: always
+    privileged: true
+    environment:
+      DOCKER_TLS_CERTDIR: ""
+    command:
+      - --storage-driver=overlay2
+
 volumes:
   postgres:
   gitlab-db: