Add code for webhook dispatcher

A dispatcher is a new component to manage incoming webhooks from Github.
It accept new requests and manage a queue based on cluster capacity.

A capacity is defined as a number of running pipelines in certain
namespace. This simple metrics will provide a buffer mechanism for time
periods when we receive a bunk releases of operators. The mechanism
keeps pending requests in the database queue and only trigger related
pipeline in case of free capacity.

The solution is made of:
- Rest API
- Postgres database
- Dispatcher
- Capacity manager

JIRA: ISV-6108

Signed-off-by: Ales Raszka <[email protected]>

Author: Allda

ansible/roles/integration_tests/tasks/check_bundle_existence_in_index_image.yml

@@ -45,7 +45,7 @@
           ansible.builtin.shell:
             cmd: |
               set -o pipefail &&
-              export PATH="{{ temp_tools_dir.path }}:$PATH"
+              export PATH="{{ integration_tests_temp_tools_dir.path }}:$PATH"
               while read image_url; do
                 version=$(echo "$image_url" | sed 's/^.*:v//' )
                 # Add stage to registry address

ansible/roles/integration_tests/tasks/check_pipeline_run.yml

@@ -3,59 +3,59 @@
   tags:
     - verify-pipeline
   block:
-    - name: Find the run of {{ pipeline_name }}
+    - name: Find the run of {{ pipeline_name }} #
       kubernetes.core.k8s_info:
         api_version: tekton.dev/v1
         kind: PipelineRun
         namespace: "{{ oc_namespace }}"
         label_selectors:
           - "tekton.dev/pipeline={{ pipeline_name }}"
           - "suffix={{ suffix }}"
-      register: pipeline_run
-      until: pipeline_run.resources | length > 0
+      register: integration_tests_pipeline_run
+      until: integration_tests_pipeline_run.resources | length > 0
       retries: 10
       delay: 5
-      failed_when: pipeline_run.resources | length == 0
+      failed_when: integration_tests_pipeline_run.resources | length == 0
 
     - name: Pipelinerun
       ansible.builtin.debug:
-        var: pipeline_run.resources[0].metadata.name
+        var: integration_tests_pipeline_run.resources[0].metadata.name
 
     - name: "Wait for the run of {{ pipeline_name }}"
       kubernetes.core.k8s_info:
         api_version: tekton.dev/v1
         kind: PipelineRun
         namespace: "{{ oc_namespace }}"
-        name: "{{ pipeline_run.resources[0].metadata.name }}"
-      register: pipeline_run_wait
+        name: "{{ integration_tests_pipeline_run.resources[0].metadata.name }}"
+      register: integration_tests_pipeline_run_wait
       until: >
-        pipeline_run_wait.resources[0] | community.general.json_query("contains(['False', 'True'], status.conditions[?type == 'Succeeded']|[-1].status)")
+        integration_tests_pipeline_run_wait.resources[0] | community.general.json_query("contains(['False', 'True'], status.conditions[?type == 'Succeeded']|[-1].status)")
       retries: 90
       delay: 60
 
     - name: "Get pipelinerun logs - {{ pipeline_name }}"
       ansible.builtin.shell: |
         ./tkn pipeline logs \
           {{ pipeline_name }} \
-          {{ pipeline_run.resources[0].metadata.name }} \
+          {{ integration_tests_pipeline_run.resources[0].metadata.name }} \
           --namespace {{ oc_namespace }}
       args:
         executable: /bin/bash
-        chdir: "{{ temp_tools_dir.path }}"
+        chdir: "{{ integration_tests_temp_tools_dir.path }}"
       no_log: true
-      register: pipeline_run_logs
+      register: integration_tests_pipeline_run_logs
       changed_when: false
 
     - name: "Print pipelinerun logs - {{ pipeline_name }}"
       ansible.builtin.debug:
-        var: pipeline_run_logs.stdout_lines
+        var: integration_tests_pipeline_run_logs.stdout_lines
 
     - name: Verify successful run of {{ pipeline_name }}
       kubernetes.core.k8s_info:
         api_version: tekton.dev/v1
         kind: PipelineRun
         namespace: "{{ oc_namespace }}"
-        name: "{{ pipeline_run.resources[0].metadata.name }}"
-      register: pipeline_run
+        name: "{{ integration_tests_pipeline_run.resources[0].metadata.name }}"
+      register: integration_tests_pipeline_run
       failed_when: >
-        pipeline_run.resources[0].status.conditions[-1].status != "True"
+        integration_tests_pipeline_run.resources[0].status.conditions[-1].status != "True"

ansible/roles/integration_tests/tasks/clean.yml

@@ -17,5 +17,5 @@
     fi
   args:
     executable: /bin/bash
-    chdir: "{{ git_temp_dir.path }}"
+    chdir: "{{ integration_tests_git_temp_dir.path }}"
   changed_when: true

ansible/roles/integration_tests/tasks/clone.yml

@@ -6,11 +6,11 @@
     - name: Create temporary directory
       ansible.builtin.tempfile:
         state: directory
-      register: git_temp_dir
+      register: integration_tests_git_temp_dir
 
     - name: Clone repository
       ansible.builtin.git:
         repo: "{{ integration_tests_git_repo_url }}"
-        dest: "{{ git_temp_dir.path }}"
+        dest: "{{ integration_tests_git_temp_dir.path }}"
         version: "{{ integration_tests_git_base_branch }}"
         refspec: "refs/heads/{{ integration_tests_src_operator_git_branch }}"

ansible/roles/integration_tests/tasks/open_pull_request.yaml

@@ -13,9 +13,9 @@
       body: E2e test for {{ integration_tests_operator_package_name }} ({{ integration_tests_operator_bundle_version }})
       head: "{{ integration_tests_git_bundle_branch }}"
       base: "{{ integration_tests_git_upstream_branch }}"
-  register: pr_response
+  register: integration_tests_pr_response
   no_log: true
 
 - name: Display PR response
   ansible.builtin.debug:
-    var: pr_response
+    var: integration_tests_pr_response

ansible/roles/integration_tests/tasks/test_data.yml

@@ -7,7 +7,7 @@
     - prepare-bundle
   ansible.builtin.template:
     src: ci.yaml.j2
-    dest: "{{ git_temp_dir.path }}/ci.yaml"
+    dest: "{{ integration_tests_git_temp_dir.path }}/ci.yaml"
     mode: "0644"
   changed_when: true
 
@@ -35,7 +35,7 @@
     git push origin "$UPSTREAM_BRANCH"
   args:
     executable: /bin/bash
-    chdir: "{{ git_temp_dir.path }}"
+    chdir: "{{ integration_tests_git_temp_dir.path }}"
   changed_when: true
 
 - name: Generate and push Operator Bundle
@@ -102,7 +102,7 @@
     git push origin "$BUNDLE_BRANCH"
   args:
     executable: /bin/bash
-    chdir: "{{ git_temp_dir.path }}"
+    chdir: "{{ integration_tests_git_temp_dir.path }}"
   changed_when: true
 
 - name: Generate and push Catalog to a PR branch
@@ -153,6 +153,6 @@
     git push origin "$PR_BRANCH"
   args:
     executable: /bin/bash
-    chdir: "{{ git_temp_dir.path }}"
+    chdir: "{{ integration_tests_git_temp_dir.path }}"
   changed_when: true
   when: integration_tests_fbc_catalog

ansible/roles/integration_tests/tasks/tools.yml

@@ -6,31 +6,31 @@
     - name: Create temporary directory
       ansible.builtin.tempfile:
         state: directory
-      register: temp_tools_dir
+      register: integration_tests_temp_tools_dir
 
     - name: Download and extract the tkn binary
       ansible.builtin.unarchive:
         src: https://mirror.openshift.com/pub/openshift-v4/clients/pipelines/1.18.0/tkn-linux-amd64.tar.gz
-        dest: "{{ temp_tools_dir.path }}"
+        dest: "{{ integration_tests_temp_tools_dir.path }}"
         remote_src: true
         include:
           - tkn
 
     - name: Download and extract the opm binary
       ansible.builtin.unarchive:
         src: https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/latest-4.18/opm-linux.tar.gz
-        dest: "{{ temp_tools_dir.path }}"
+        dest: "{{ integration_tests_temp_tools_dir.path }}"
         remote_src: true
         include:
           - opm-rhel8
 
     - name: Rename the opm binary
       ansible.builtin.copy:
-        src: "{{ temp_tools_dir.path }}/opm-rhel8"
-        dest: "{{ temp_tools_dir.path }}/opm"
+        src: "{{ integration_tests_temp_tools_dir.path }}/opm-rhel8"
+        dest: "{{ integration_tests_temp_tools_dir.path }}/opm"
         mode: '0755'
 
     - name: Delete the original file
       ansible.builtin.file:
-        path: "{{ temp_tools_dir.path }}/opm-rhel8"
+        path: "{{ integration_tests_temp_tools_dir.path }}/opm-rhel8"
         state: absent

ansible/roles/operator-pipeline/tasks/index-img-bootstrap-signing.yml

@@ -101,7 +101,7 @@
         name: tekton-pipeline-executor
       subjects:
         - kind: ServiceAccount
-          name: "{{ item}}"
+          name: "{{ item }}"
           namespace: "{{ oc_index_bootstrap_namespace }}"
   with_items: "{{ index_img_bootstrap_service_accounts }}"
 

docker-compose.yml

@@ -0,0 +1,22 @@
+---
+services:
+  db:
+    image: registry.redhat.io/rhel10/postgresql-16@sha256:8515941a42d6bc5d59b753839174b3c0e839315d686ca2fdd6fa65e1cccf3390
+    container_name: webhook-dispatcher-db
+    restart: unless-stopped
+    environment:
+      POSTGRESQL_USER: user
+      POSTGRESQL_PASSWORD: password
+      POSTGRESQL_DATABASE: webhook_dispatcher
+    ports:
+      - "5432:5432"
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${DB_USER} -d ${DB_NAME}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+volumes:
+  pgdata:

docs/webhook-dispatecher.md

@@ -0,0 +1,173 @@
+# Webhook Dispatcher
+
+The Webhook Dispatcher is a microservice that receives GitHub webhooks and processes them for operator pipelines.
+It validates webhooks, stores events in a database, and triggers Tekton pipelines for operator pipelines with
+a capacity management system.
+
+## Overview
+
+The Webhook Dispatcher serves as the entry point for GitHub events in the operator pipelines.
+When developers submit operators via pull requests, the dispatcher:
+
+1. **Receives and validates** GitHub webhooks with signature verification
+2. **Stores webhook events** in a PostgreSQL database
+3. **Manages pipeline capacity** to prevent resource exhaustion
+4. **Triggers Tekton pipelines** for operator validation
+
+## Architecture
+
+```
+┌─────────────────┐    ┌──────────────────┐    ┌─────────────────┐
+│   GitHub        │    │   Webhook        │    │   PostgreSQL    │
+│   Webhooks      │───▶│   Dispatcher     │───▶│   Database      │
+└─────────────────┘    └──────────────────┘    └─────────────────┘
+                                │
+                                ▼
+                       ┌──────────────────┐
+                       │   Tekton         │
+                       │   Pipelines      │
+                       └──────────────────┘
+```
+
+## Features
+
+- **Secure webhook processing** with HMAC-SHA256 signature verification
+- **Multi-repository support** with configurable processing rules
+- **Pipeline capacity management** to prevent resource overload
+- **Background event processing** for improved performance
+- **Health monitoring** and status tracking
+
+
+## Installation
+
+### Prerequisites
+- Python 3.11+
+- PostgreSQL 12+
+- Kubernetes/OpenShift cluster access
+
+### Setup
+
+1. Install dependencies:
+```bash
+pdm install --no-dev
+```
+
+2. Create database:
+```bash
+docker compose up -d
+```
+
+3. Set environment variables:
+```bash
+export DATABASE_URL="postgresql://webhook_user:secure_password@localhost:5432/webhook_dispatcher"
+export GITHUB_WEBHOOK_SECRET="your_github_webhook_secret"
+```
+
+## Configuration
+
+Create a `dispatcher_config.yaml` file:
+
+```yaml
+---
+dispatcher:
+  items:
+    - name: "Community Operators"
+      events:
+        - opened
+        - synchronize
+        - ready_for_review
+      full_repository_name: "redhat-openshift-ecosystem/community-operators"
+      capacity:
+        type: ocp_tekton
+        pipeline_name: "operator-hosted-pipeline"
+        max_capacity: 5
+        namespace: "operator-pipeline-prod"
+      # Tekton pipeline listener URL
+      callback_url: "https://tekton-dashboard.example.com/api/v1/trigger"
+
+security:
+  github_webhook_secret: "${GITHUB_WEBHOOK_SECRET}"
+  verify_signatures: true
+  allowed_github_events:
+    - pull_request
+```
+
+### Configuration Parameters
+
+- **name**: Configuration identifier
+- **events**: GitHub PR events to process
+- **full_repository_name**: Repository in `owner/repo` format
+- **capacity**: Pipeline capacity settings
+- **callback_url**: Tekton pipeline trigger URL
+- **github_webhook_secret**: Webhook signature verification secret
+
+## Usage
+
+### Running the Service
+
+Production mode:
+```bash
+export CONFIG_FILE="/path/to/dispatcher_config.yaml"
+python -m operatorcert.webhook_dispatcher.main
+```
+
+### GitHub Webhook Setup
+
+1. Go to repository **Settings** → **Webhooks**
+2. Add webhook with:
+   - **URL**: `https://your-domain.com/api/v1/webhooks/github-pipeline`
+   - **Content type**: `application/json`
+   - **Secret**: Your webhook secret
+   - **Events**: Pull requests
+
+## API Endpoints
+
+- `POST /api/v1/webhooks/github-pipeline` - Receive GitHub webhooks
+- `GET /api/v1/status/ping` - Health check
+- `GET /api/v1/status/db` - Database health
+- `GET /api/v1/events/status` - Event status with pagination
+
+## Security
+
+- **HMAC-SHA256 signature verification** for all webhook requests
+- **GitHub User-Agent validation** to prevent spoofing
+- **Event type filtering** based on configuration
+- **TLS encryption** required for production deployments
+
+## Monitoring
+
+### Health Checks
+```bash
+curl http://localhost:5000/api/v1/status/ping
+curl http://localhost:5000/api/v1/status/db
+```
+
+### Event Status
+```bash
+curl "http://localhost:5000/api/v1/events/status?page_size=20"
+```
+
+## Troubleshooting
+
+### Common Issues
+
+**Signature verification failures**:
+- Verify `GITHUB_WEBHOOK_SECRET` matches GitHub configuration
+- Ensure webhook uses `application/json` content type
+
+**Database connection errors**:
+- Check `DATABASE_URL` format
+- Verify PostgreSQL is running and accessible
+
+**Event processing delays**:
+- Check dispatcher thread logs
+- Monitor pipeline capacity usage
+- Review database performance
+
+### Debug Mode
+```bash
+export LOG_LEVEL=DEBUG
+python -m operatorcert.webhook_dispatcher.main --verbose
+```
+
+For support, refer to the [operator-pipelines repository](https://github.com/redhat-openshift-ecosystem/operator-pipelines).