Moving from quick hack to public release, first commit

Author: alexbarcelo

.gitignore

@@ -0,0 +1,9 @@
+# Deployment virtualenv
+/venv
+
+# Compiled Python
+*.pyc
+__pycache__
+build/
+dist/
+webhooks_git_automata.egg-info/

LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2018 The Python Packaging Authority
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,50 @@
+
+Webhook receiver for Git deployments
+====================================
+
+This project started as a dirty & quick hack to perform some deployment actions 
+triggered by a Git webhook.
+
+I looked a little bit at other projects and didn't found any one that suited my 
+needs, so I started this project... and then it grew a little bit and become
+something more versatile than the quick hack originally intended.
+
+This project is aimed at DevOps or sysadmin that have git repositories, typically
+with a VIP-branch, and automatic deployment. You may want to have different branches
+for stage and production or set up push permissions differently to the different 
+branches. This utility, when a webhook is received, will update the local Git repository
+and perform the commands in the settings.
+
+Quickstart
+----------
+
+ - Create and activate a Virtual Environment: 
+```
+virtualenv --python=/usr/bin/python3 /path/to/venv
+source /path/to/venv/bin/activate
+```
+ - Install the package: `pip install webhooks_git_automata`
+ - Create a `settings.yaml` 
+ - Set up a service (e.g. a systemd _service_ file) that does something along: 
+```
+/path/to/venv/bin/wh-gitd /path/to/settings.yaml
+```
+
+Settings
+--------
+
+
+Implementation details
+----------------------
+
+This project contains a minimal Flask server that answers the POST webhooks sent 
+by a Git server like GitLab, GitHub or Gogs. The server is started through the 
+Flask's `app.run` method.
+
+Not a lot of traffic is expected, but you may want to set up a reverse proxy in front
+of the Flask server, or add some fancier method like a WSGI or uWSGI or similar layer.
+
+Typical git servers expect the webhook to send a quick reply (in general, HTTP 
+connections are intended to be short lived) so there is a worker/tasks approach. There
+is a very simple implementation base on `Threading` and a shared `Queue`. More complex
+implementations may be added in the future (pull requests welcome).

setup.py

@@ -0,0 +1,27 @@
+import setuptools
+
+with open("README.md", "r") as fh:
+    long_description = fh.read()
+
+setuptools.setup(
+    name="webhooks_git_automata",
+    version="0.0.1",
+    author="Alex Barcelo",
+    author_email="[email protected]",
+    description="Webhook receiver for Git deployments",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    install_requires=["Flask", "PyYAML"],
+    url="https://github.com/alexbarcelo/webhooks-git-automata",
+    packages=setuptools.find_packages(),
+    entry_points={
+        'console_scripts': [
+            'wh-gitd = webhooks_git_automata.webhooks:main_func',
+        ],
+    },
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
+)

webhooks_git_automata/__init__.py

@@ -0,0 +1,76 @@
+import os.path
+import subprocess
+from logging import getLogger, DEBUG
+
+logger = getLogger("webhooks.actions")
+
+
+class MetaAutomata(type):
+    """Metaclass required to provide a `__getitem__` method to Automata class.
+
+    When Automata[<repo>, <branch>] is called, the corresponding Automaton
+    instance is returned ready to be used by the worker.
+    """
+    actions = dict()
+
+    def __getitem__(cls, item):
+        """Given a (repo, branch) tuple, return its automaton."""
+        return cls.actions[item]
+
+    def load_config(cls, actions):
+        """Given its configuration section, create and store the automaton instances."""
+        for name, automaton in actions.items():
+            repo = automaton.pop("repo", name)
+            branch = automaton.pop("branch", "master")
+
+            cls.actions[repo, branch] = cls(automaton, repo, branch)
+
+
+class Automata(object, metaclass=MetaAutomata):
+    """Manage the set of automaton instances.
+
+    With the method helpers of its metaclass MetaAutomata, manage a set of
+    automaton (one per repository and branch).
+    """
+    def __init__(self, action, repo, branch):
+        self._action = action
+        self._repo = repo
+        self._branch = branch
+
+    def pull_sources(self):
+        os.chdir(self._action["repodir"])
+        submodules = self._action.get("submodules", True)
+        logger.info("Proceeding to pull changes (fetch + checkout)")
+
+        git_fetch = ["/usr/bin/git", "fetch"]
+        if submodules:
+            git_fetch.append("--recurse-submodules=yes")
+
+        git_checkout = ["/usr/bin/git", "checkout", "-f", "origin/%s" % self._branch]
+
+        git_submodule_update = ["/usr/bin/git", "submodule", "update"]
+
+        ret = subprocess.call(git_fetch)
+        if ret != 0:
+            raise RuntimeError("git fetch process failed with exitcode %d" % ret)
+
+        ret = subprocess.call(git_checkout)
+        if ret != 0:
+            raise RuntimeError("git checkout process failed with exitcode %d" % ret)
+
+        if submodules:
+            ret = subprocess.call(git_submodule_update)
+            if ret != 0:
+                raise RuntimeError("git submodule update failed with exitcode %d" % ret)
+
+    def perform_commands(self):
+        os.chdir(self._action["workdir"])
+        for command in self._action["commands"]:
+            ret = subprocess.call(command)
+            if ret != 0:
+                logger.error("Could not perform the following command: `%s`",
+                             " ".join(command))
+                raise RuntimeError("A subprocess command failed")
+            logger.debug("Command `%s` executed", " ".join(command))
+
+        logger.info("All commands executed")

webhooks_git_automata/automata.py

@@ -0,0 +1,92 @@
+from logging import getLogger
+import hmac
+
+from flask import request
+from werkzeug.exceptions import Forbidden, BadRequest
+
+from .worker import event_queue
+
+logger = getLogger(__name__)
+
+
+def _schedule_data_from_request():
+    """Given an ongoing request, schedule it to the worker.
+
+    This can be called from an ongoing request, and it will get the data (the
+    JSON payload) and put it in the worker event_queue for later processing.
+    """
+    data = request.get_json(force=True, cache=False)
+    if not data:
+        raise BadRequest("Unconsistent JSON received")
+    event_queue.put(data)
+
+
+def webhook_github(secret):
+    """GitHub implementation of the webhook (requires secret)."""
+    logger.debug("Received a POST from: %s",
+                 request.remote_addr)
+
+    agent = request.headers.get("User-Agent")
+
+    if not agent.startswith("GitHub-Hookshot"):
+        raise Forbidden("Only GitHub is allowed to POST to us")
+
+    signature = request.headers.get("X-Hub-Signature")
+    event_type = request.headers.get("X-GitHub-Event")
+
+    if not signature or not event_type:
+        raise Forbidden("Webhooks must include the signature "
+                        "(add `secret` to the webhook settings in GitHub)")
+
+    if not secret:
+        raise SystemError("No `secret` configured, refusing to accept petition")
+
+    try:
+        digest = hmac.digest(secret, request.get_data(), "sha1")
+    except AttributeError:  # Python < 3.7, using older & slower approach
+        h = hmac.new(secret, request.get_data(), "sha1")
+        digest = h.digest()
+
+    # Strip the first five characters: 'sha1=xxxxxxx' and get it in binary form
+    sent_digest = bytes.fromhex(signature[5:])
+    if digest != sent_digest:
+        logger.debug("Expected signature %s, received %s" % (digest, sent_digest))
+        raise Forbidden("The HMAC digest do not match. You are not allowed.")
+
+    if event_type == "ping":
+        logger.info("Ignoring ping event")
+    else:
+        _schedule_data_from_request()
+
+    return ""
+
+
+def webhook_gogs(secret):
+    """Gogs implementation of the webhook (requires secret)."""
+    logger.debug("Received a POST from: %s",
+                 request.remote_addr)
+
+    data = request.get_json(force=True, cache=False)
+    if not data:
+        raise BadRequest("Unconsistent JSON received")
+
+    try:
+        if data["secret"] != secret:
+            logger.debug("Received secret: %s, which did not match the expected", data["secret"])
+            raise Forbidden("Invalid `secret` provided in HTTP method payload")
+    except KeyError:
+        raise Forbidden("The request should include the `secret`. "
+                        "Have you configured correctly the gogs sever webhook?")
+
+    event_queue.put(data)
+    return ""
+
+
+def webhook_plain():
+    """Trivial implementation."""
+    logger.debug("Received a POST from: %s",
+                 request.remote_addr)
+
+    _schedule_data_from_request()
+
+    return ""

webhooks_git_automata/providers.py

@@ -0,0 +1,44 @@
+import logging
+
+import yaml
+
+from .automata import Automata
+
+logger = logging.getLogger(__name__)
+
+
+class Configuration(object):
+    @classmethod
+    def load(cls, path):
+        with open(path, 'r') as f:
+            cls._settings = yaml.load(f)
+
+        # Logging settings
+        logging.basicConfig(level=cls._settings.get("log_level", "INFO"))
+        logger.info("Initialized with configuration at %s", path)
+        logger.debug("Debug level is activated")
+
+        # Provider settings
+        provider = cls._settings.get("provider", "plain")
+
+        if isinstance(provider, dict):
+            # If it is a dictionary, the provider is the key name
+            cls.provider = provider.pop("name")
+            # and all other entries are their options
+            cls.provider_options = provider
+        else:
+            # typically "plain", which has no extra configuration
+            cls.provider = provider
+            cls.provider_options = dict()
+
+        # URL path for webhooks
+        cls.webhook_url_path = cls._settings.get("webhook_url_path", "/webhook")
+
+        # IP to listen to
+        cls.listen_ip = cls._settings.get("listen_ip", "127.0.0.1")
+
+        # Port to listen to
+        cls.listen_port = cls._settings.get("listen_port", 5000)
+
+        # Repositories required entry, (managed by the `Actions` class)
+        Automata.load_config(cls._settings["repositories"])

webhooks_git_automata/settings.py

@@ -0,0 +1,41 @@
+import sys
+import logging
+from functools import partial
+from threading import Thread
+
+from flask import Flask
+
+from .settings import Configuration
+from .worker import async_worker
+from . import providers
+
+
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger("webhooks")
+app = Flask(__name__)
+
+
+def main_func():
+    if len(sys.argv) != 2:
+        raise RuntimeError("You should provide the path to the settings YAML file as first argument")
+
+    Configuration.load(sys.argv[1])
+
+    # Future work: check the settings and allow multiple worker implementations (e.g. Celery)
+    Thread(target=async_worker,
+           name="webhook worker").start()
+
+    try:
+        webhook_func = getattr(providers, "webhook_%s" % Configuration.provider)
+    except AttributeError:
+        raise RuntimeError("Could not prepare the webhook for provider %s. "
+                           "Maybe it is not supported?" % Configuration.provider)
+
+    webhook = partial(
+        webhook_func,
+        **Configuration.provider_options
+    )
+
+    app.add_url_rule('/webhook', 'webhook', webhook, methods=['POST'])
+    app.run(host=Configuration.listen_ip,
+            port=Configuration.listen_port)