Add a webhook processing server

Author: QuLogic

.github/workflows/build-image.yml

@@ -2,6 +2,35 @@ name: Linting
 on: [pull_request]
 
 jobs:
+  flake8:
+    name: flake8
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python 3
+        uses: actions/setup-python@v1
+
+      - name: Install flake8
+        run: pip3 install flake8
+
+      - name: Set up reviewdog
+        run: |
+          mkdir -p "$HOME/bin"
+          curl -sfL \
+            https://github.com/reviewdog/reviewdog/raw/master/install.sh | \
+              sh -s -- -b "$HOME/bin"
+          echo "$HOME/bin" >> $GITHUB_PATH
+
+      - name: Run flake8
+        env:
+          REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -o pipefail
+          flake8 | \
+            reviewdog -f=pep8 -name=flake8 \
+              -tee -reporter=github-check -filter-mode nofilter
+
   caddyfmt:
     name: caddyfmt
     runs-on: ubuntu-latest

.github/workflows/lint.yml

@@ -0,0 +1,27 @@
+---
+
+name: Tests
+on: [push, pull_request]
+
+jobs:
+  webhook:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10"]
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          pip install -r dev-requirements.txt
+          git config --global user.name "GitHub CI Bot"
+          git config --global user.email "[email protected]"
+
+      - name: Run tests
+        run: pytest

.github/workflows/tests.yml

@@ -1 +1,2 @@
+/__pycache__
 /sites/

.gitignore

@@ -14,7 +14,34 @@
 
 # Set this variable in the environment when running in production.
 {$SITE_ADDRESS::2015} {
-	root * .
+	root * {$SITE_DIR:.}
+
+	# Setup a webhook
+	handle /gh/* {
+		# https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#delivery-headers
+		@valid_webhook {
+			method POST
+			header User-Agent GitHub-Hookshot/*
+			header X-GitHub-Event ping
+			header X-GitHub-Event push
+			header X-GitHub-Delivery *
+			header X-Hub-Signature-256 *
+		}
+
+		handle @valid_webhook {
+			reverse_proxy * localhost:1234 {
+				# Don't leak out internal problems.
+				@error status 4xx 5xx
+				handle_response @error {
+					error 400
+				}
+			}
+		}
+
+		handle {
+			error 400
+		}
+	}
 
 	import subproject basemap
 	import subproject cheatsheets

Caddyfile

@@ -0,0 +1,4 @@
+-r requirements.txt
+pytest
+pytest-aiohttp
+pytest-asyncio

dev-requirements.txt

@@ -0,0 +1,4 @@
+[pytest]
+norecursedirs =
+    sites
+asyncio_mode = auto

pytest.ini

@@ -0,0 +1 @@
+aiohttp

requirements.txt

@@ -0,0 +1,173 @@
+"""GitHub Documentation WebHook handler tests."""
+
+import functools
+from pathlib import Path
+import subprocess
+from unittest import mock
+
+from aiohttp import web
+import pytest
+
+import webhook
+from webhook import create_app, update_repo, verify_signature
+
+
+run_check = functools.partial(subprocess.run, check=True)
+
+
+async def test_signature(monkeypatch):
+    """Test that signature verification fails and works as expected."""
+    with pytest.raises(web.HTTPBadRequest,
+                       match='Secret for non-existent was not set'):
+        await verify_signature(b'unused', 'unused', 'non-existent', 'unused')
+
+    monkeypatch.setenv('WEBHOOK_REPO_SECRET', 'abcdef')
+    with pytest.raises(web.HTTPBadRequest, match='signature was invalid'):
+        await verify_signature(b'unused', 'sha256=incorrect', 'repo', 'unused')
+
+    # Signature found by passing data to `openssl dgst -sha256 -hmac $SECRET`.
+    monkeypatch.setenv('WEBHOOK_REPO_SECRET', 'abcdef')
+    await verify_signature(
+        b'{"data": "foo"}',
+        'sha256='
+        'ebf35862e15f2ac2aa1339ebefff9a88b8270fc8a33dec8f756a398036d86329',
+        'repo',
+        'unused')
+
+
+async def test_update_repo_error():
+    """Test that updating a repository fails as expected."""
+    with pytest.raises(web.HTTPServerError,
+                       match='non-existent does not exist'):
+        await update_repo(Path('non-existent'), 'unused',
+                          'matplotlib/non-existent')
+
+
+async def test_update_repo_empty(tmp_path_factory):
+    """Test that updating an empty repository works as expected."""
+    repo1 = tmp_path_factory.mktemp('repo1')
+    run_check(['git', 'init', '-b', 'main', repo1])
+    await update_repo(repo1, 'unused', 'matplotlib/repo1')
+
+
+async def test_update_repo(tmp_path_factory):
+    """Test that updating a repository works as expected."""
+    # Set up a source repository.
+    src = tmp_path_factory.mktemp('src')
+    run_check(['git', 'init', '-b', 'main', src])
+    (src / 'readme.txt').write_text('Test repo information')
+    run_check(['git', 'add', 'readme.txt'], cwd=src)
+    run_check(['git', 'commit', '-m', 'Initial commit'], cwd=src)
+
+    # Make second from the first.
+    dest = tmp_path_factory.mktemp('dest')
+    run_check(['git', 'clone', src, dest])
+
+    # Make a second commit in source repository.
+    (src / 'install.txt').write_text('There is no installation.')
+    run_check(['git', 'add', 'install.txt'], cwd=src)
+    run_check(['git', 'commit', '-m', 'Add install information'], cwd=src)
+    src_stdout = run_check(['git', 'show-ref', '--head', 'HEAD'], cwd=src,
+                           capture_output=True).stdout.splitlines()
+    src_commit = next(
+        (line for line in src_stdout if line.split()[-1] == 'HEAD'), '')
+
+    # Now this should correctly update the first repository.
+    await update_repo(dest, 'unused', 'matplotlib/dest')
+    dest_stdout = run_check(['git', 'show-ref', '--head', 'HEAD'], cwd=dest,
+                            capture_output=True).stdout.splitlines()
+    dest_commit = next(
+        (line for line in dest_stdout if line.split()[-1] == 'HEAD'), '')
+    assert dest_commit == src_commit
+
+
+async def test_github_webhook_errors(aiohttp_client, monkeypatch):
+    """Test invalid inputs to webhook."""
+    client = await aiohttp_client(create_app())
+
+    # Only /gh/<repo-name> exists.
+    resp = await client.get('/')
+    assert resp.status == 404
+    resp = await client.get('/gh')
+    assert resp.status == 404
+
+    # Not allowed if missing correct headers.
+    resp = await client.get('/gh/non-existent-repo')
+    assert resp.status == 405
+    resp = await client.post('/gh/non-existent-repo')
+    assert resp.status == 400
+    assert 'No delivery' in await resp.text()
+    resp = await client.post('/gh/non-existent-repo',
+                             headers={'X-GitHub-Delivery': 'foo'})
+    assert resp.status == 400
+    assert 'No signature' in await resp.text()
+
+    monkeypatch.setattr(webhook, 'verify_signature',
+                        mock.Mock(verify_signature, return_value=True))
+
+    # Data should be JSON.
+    resp = await client.post(
+        '/gh/non-existent-repo',
+        headers={'X-GitHub-Delivery': 'foo', 'X-Hub-Signature-256': 'unused'},
+        data='}{')
+    assert resp.status == 400
+    assert 'Invalid data input' in await resp.text()
+
+    # Some data fields are required.
+    resp = await client.post(
+        '/gh/non-existent-repo',
+        headers={'X-GitHub-Delivery': 'foo', 'X-Hub-Signature-256': 'unused'},
+        data='{}')
+    assert resp.status == 400
+    assert 'Missing required fields' in await resp.text()
+
+    resp = await client.post(
+        '/gh/non-existent-repo',
+        headers={'X-GitHub-Delivery': 'foo', 'X-Hub-Signature-256': 'unused'},
+        data='{"action": "ping", "sender": "QuLogic", "organization": "foo",'
+             ' "repository": "foo"}')
+    assert resp.status == 400
+    assert 'incorrect organization' in await resp.text()
+
+    resp = await client.post(
+        '/gh/non-existent-repo',
+        headers={'X-GitHub-Delivery': 'foo', 'X-Hub-Signature-256': 'unused'},
+        data='{"action": "ping", "sender": "QuLogic", '
+             '"organization": "matplotlib", "repository": "foo"}')
+    assert resp.status == 400
+    assert 'incorrect repository' in await resp.text()
+
+
+async def test_github_webhook_valid(aiohttp_client, monkeypatch):
+    """Test valid input to webhook."""
+    client = await aiohttp_client(create_app())
+
+    # Do no actual work, since that's tested above.
+    monkeypatch.setattr(webhook, 'verify_signature',
+                        mock.Mock(verify_signature, return_value=True))
+    monkeypatch.setenv('SITE_DIR', 'non-existent-site-dir')
+    ur_mock = mock.Mock(update_repo, return_value=None)
+    monkeypatch.setattr(webhook, 'update_repo', ur_mock)
+
+    # Ping event just returns success.
+    resp = await client.post(
+        '/gh/non-existent-repo',
+        headers={'X-GitHub-Delivery': 'foo', 'X-Hub-Signature-256': 'unused'},
+        data='{"action": "ping", "sender": "QuLogic", "hook_id": "foo",'
+             ' "zen": "Beautiful is better than ugly.",'
+             ' "organization": "matplotlib",'
+             ' "repository": "non-existent-repo"}')
+    assert resp.status == 200
+    ur_mock.assert_not_called()
+
+    # Push event should run an update.
+    resp = await client.post(
+        '/gh/non-existent-repo',
+        headers={'X-GitHub-Delivery': 'foo', 'X-Hub-Signature-256': 'unused'},
+        data='{"action": "push", "sender": "QuLogic",'
+             ' "organization": "matplotlib",'
+             ' "repository": "non-existent-repo"}')
+    assert resp.status == 200
+    ur_mock.assert_called_once_with(
+        Path('non-existent-site-dir/non-existent-repo'), 'foo',
+        'matplotlib/non-existent-repo')