Merge branch 'main' into chore/auto-libs

Author: seb4stien

.github/workflows/e2e_test.yaml

@@ -2,7 +2,10 @@ name: End-to-End tests
 
 on:
   pull_request:
-  
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
   # test option values defined at test/conftest.py are passed on via repository secret
@@ -13,7 +16,7 @@ jobs:
     secrets: inherit
     with:
       juju-channel: 3.6/stable
-      pre-run-script: scripts/setup-lxd.sh
+      pre-run-script: scripts/setup-integration-tests.sh
       provider: lxd
       test-tox-env: integration-juju3.6
       modules: '["test_e2e"]'

.github/workflows/integration_test.yaml

@@ -3,10 +3,9 @@ name: integration-tests
 on:
   pull_request:
 
-  # Disabled until we get more resources for the actions
-  # schedule:
-  #   # Trigger at 6:00 AM and 6:00 PM UTC
-  #   - cron: "0 6,18 * * *"
+  schedule:
+    # Trigger at 6:00 AM and 6:00 PM UTC
+    - cron: "0 6,18 * * *"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -21,7 +20,7 @@ jobs:
     secrets: inherit
     with:
       juju-channel: 3.1/stable
-      pre-run-script: scripts/setup-lxd.sh
+      pre-run-script: scripts/setup-integration-tests.sh
       provider: lxd
       test-tox-env: integration-juju3.1
       modules: '["test_charm_scheduled_events", "test_debug_ssh", "test_charm_upgrade"]'
@@ -35,7 +34,7 @@ jobs:
     secrets: inherit
     with:
       juju-channel: 3.6/stable
-      pre-run-script: scripts/setup-lxd.sh
+      pre-run-script: scripts/setup-integration-tests.sh
       provider: lxd
       test-tox-env: integration-juju3.6
       modules: '["test_runner_manager_openstack"]'
@@ -48,10 +47,10 @@ jobs:
     secrets: inherit
     with:
       juju-channel: 3.6/stable
-      pre-run-script: scripts/setup-lxd.sh
+      pre-run-script: scripts/setup-integration-tests.sh
       provider: lxd
       test-tox-env: integration-juju3.6
-      modules: '["test_charm_metrics_failure", "test_charm_metrics_success", "test_charm_fork_repo", "test_charm_fork_path_change", "test_charm_no_runner", "test_charm_runner", "test_reactive"]'
+      modules: '["test_charm_metrics_failure", "test_charm_metrics_success", "test_charm_fork_repo", "test_charm_fork_path_change", "test_charm_no_runner", "test_charm_runner", "test_reactive", "test_jobmanager"]'
       extra-arguments: '-m openstack --log-format="%(asctime)s %(levelname)s %(message)s"'
       self-hosted-runner: true
       self-hosted-runner-label: stg-private-endpoint

.github/workflows/test.yaml

@@ -9,3 +9,4 @@ jobs:
     secrets: inherit
     with:
       self-hosted-runner: true
+      self-hosted-runner-label: edge

.github/workflows/test_github_runner_manager.yaml

@@ -9,4 +9,5 @@ jobs:
     secrets: inherit
     with:
       self-hosted-runner: true
+      self-hosted-runner-label: edge
       working-directory: ./github-runner-manager/

.github/workflows/tics.yaml

@@ -0,0 +1,36 @@
+on: [pull_request]
+
+jobs:
+  TICS:
+    name: TICS
+    runs-on: [self-hosted, linux, amd64, tiobe, jammy]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Run Unit Tests (manager app)
+        run: |
+          cd github-runner-manager
+          pip install tox pylint flake8 -r ./requirements.txt
+          tox -e unit,coverage-report
+          rm -rf .tox
+      - name: Run Unit Tests
+        run: |
+          pip install tox pylint flake8 -r ./requirements.txt -r ./github-runner-manager/requirements.txt
+          tox -e unit,coverage-report
+          rm -rf .tox
+      - name: TICS GitHub Action
+        uses: tiobe/tics-github-action@v3
+        env:
+          PYTHONPATH: "/home/ubuntu/actions-runner/_work/github-runner-operator/github-runner-operator/src:/home/ubuntu/actions-runner/_work/github-runner-operator/github-runner-operator/lib:/home/ubuntu/actions-runner/_work/github-runner-operator/github-runner-operator/github-runner-manager/src/github_runner_manager:/home/ubuntu/actions-runner/_work/github-runner-operator/github-runner-operator/github-runner-manager/src"
+        with:
+          mode: qserver
+          project: github-runner-operator
+          branchdir: ${{ github.workspace }}
+          viewerUrl: https://canonical.tiobe.com/tiobeweb/TICS/api/cfg?name=default
+          ticsAuthToken: ${{ secrets.TICSAUTHTOKEN }}
+          installTics: true
+
+concurrency:
+  group: tics
+  cancel-in-progress: false

.gitignore

@@ -4,6 +4,7 @@ __pycache__/
 placeholders/
 *.charm
 build/
+coverage
 .coverage
 
 # development artifacts

.licenserc.yaml

@@ -27,4 +27,5 @@ header:
     - '.woke.yaml'
     - 'lib/**'
     - 'github-runner-manager/**'
+    - 'jobmanager/**'
   comment: on-failure

.woke.yaml

@@ -1,3 +1,6 @@
+ignore_files:
+  # This is autogenerated code
+  - jobmanager/**
 rules:
   # Ignore whitelist - we are using it to ignore pydantic in pyproject.toml and pylintrc
   - name: whitelist

README.md

@@ -54,7 +54,7 @@ juju integrate github-runner-image-builder github-runner
 You can scale the amount of virtual machines using
 
 ```bash
-juju config github-runner virtual-machines=5
+juju config github-runner base-virtual-machines=5
 ```
 
 You can change the reconciliation interval, to e.g. 5 minutes, using
@@ -75,6 +75,60 @@ If you need to flush and replace the runners with a new set of runners, you can
 juju run github-runner/0 flush-runners
 ```
 
+## Overview of the GitHub runner ecosystem
+
+The `github-runner` charm integrates with several other charms that work
+together to facilitate the deployment, management and debugging of
+self-hosted GitHub runners. Below is a high-level overview of the key
+charms and their interactions:
+
+* [GitHub Runner](https://charmhub.io/github-runner-image): The central component that manages self-hosted GitHub runners. It interacts with OpenStack to spawn runner VMs and communicates with GitHub to register and manage runners.
+* [Image Builder](https://charmhub.io/github-runner-image-builder): Responsible for generating images. It builds images on the builder OpenStack project and uploads them to the GitHub Runner OpenStack project.
+* [MongoDB](https://charmhub.io/mongodb): Acts as a message queue to handle reactive runner requests. The [github-runner-webhook-router](https://charmhub.io/github-runner-webhook-router) charm will put events in MongoDB that will be consumed by the github-runner charm. Only for reactive runners.
+* [tmate-ssh-server](https://charmhub.io/tmate-ssh-server): Provides terminal-sharing capabilities to enable debugging of GitHub runners. Optional.
+* [COS lite stack](https://charmhub.io/topics/canonical-observability-stack/editions/lite): Provides observability to the Github runners ecosystem. Optional.
+
+Below is a diagram representing these components and their relationships, excluding the [COS lite stack](https://charmhub.io/topics/canonical-observability-stack/editions/lite):
+
+```mermaid
+C4Container
+title Container diagram for the github-runner Charm System
+ Container_Boundary(c1, "Image Builder") {
+    Container(imagebuilder, "Image Builder", "", "Provides images to all related charms")
+ }
+    System_Ext(osbuilding, "OpenStack", "OpenStack deployment used for building images")
+Container_Boundary(c2, "GitHub Runner"){
+    Container(githubrunner, "GitHub Runner Charm", "", "Manages self-hosted runners")
+}
+Container_Boundary(c3, "monbodb"){
+    Container(mongodb, "MongoDB", "", "Used as a message queue for reactive runner requests")
+}
+Container_Boundary(c4, "tmate-ssh-server"){
+    Container(tmate_ssh, "tmate-ssh-server", "", "Terminal sharing capabilities to debug GitHub runners")
+}
+
+Container_Boundary(c5, "github-runner-webhook-router"){
+    Container(router, "github-runner-webhook-router", "", "Listens to GitHub webhooks")
+}
+
+    Rel(imagebuilder, osbuilding, "builds images")
+    UpdateRelStyle(imagebuilder, osbuilding, $offsetY="-30", $offsetX="10")
+    Rel(imagebuilder, osgithubrunner, "uploads images")
+    UpdateRelStyle(imagebuilder, osgithubrunner, $offsetY="-30", $offsetX="-90")
+    Rel(imagebuilder, githubrunner, "image ids")
+    UpdateRelStyle(imagebuilder, githubrunner, $offsetY="-10", $offsetX="-30")
+    System_Ext(osgithubrunner, "OpenStack", "OpenStack deployment used for spawning runner VMs")
+    System_Ext(github, "GitHub", "GitHub API")
+    Rel(githubrunner, osgithubrunner, "spawns VMs")
+    UpdateRelStyle(githubrunner, osgithubrunner, $offsetY="-30", $offsetX="10")
+    Rel(githubrunner, github, "Manage runners")
+    Rel(githubrunner, imagebuilder, "OpenStack credentials")
+    UpdateRelStyle(githubrunner, imagebuilder, $offsetY="10", $offsetX="-60")
+    Rel(mongodb, githubrunner, "database credentials")
+    Rel(tmate_ssh, githubrunner, "debug-ssh credentials")
+    Rel(router, mongodb, "new runner requests")
+```
+
 
 ## Integrations
 The charm supports [multiple integrations](https://charmhub.io/github-runner/integrations),

actions.yaml

@@ -7,5 +7,3 @@ reconcile-runners:
     description: Remove offline runners and replace any missing runners
 flush-runners:
     description: Clear out all runners and start a new set
-update-dependencies:
-    description: Update GitHub self-hosted runner binary and services used by the charm