Webhook request debouncing

Resolves #27

Author: brandonkelly

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Added
 - Added a settings page in the control panel.
+- Webhooks have a new “Debounce Key Format” setting, which can be used to debouncing similar webhook requests. ([#27](https://github.com/craftcms/webhooks/issues/27))
 - Webhook URLs can now be set to environment variables or Twig code. ([#18](https://github.com/craftcms/webhooks/issues/18))
 - Webhooks can now send PUT requests. ([#21](https://github.com/craftcms/webhooks/issues/21))
 

README.md

@@ -136,6 +136,14 @@ class ArticleFilter implements FilterInterface
 }
 ```
 
+#### Debouncing Webhooks
+
+You can prevent multiple similar webhooks from being sent by setting a “Debounce Key Format” on your webhook. This is a Twig template that defines a “debounce key” for the webhook. If two webhooks generate the same debounce key, only the second one will actually be sent.  
+
+An `event` variable will be available to it that references the event that was triggered.
+
+For example, if your webhook is for an entry (`craft\elements\Entry`), then you could set the Debounce Key Format to `{{ event.sender.id }}` to prevent multiple webhook requests from being queued up at the same time.
+
 #### Sending Custom Headers
 
 You can send custom headers along with webhook requests using the Custom Headers setting.

src/Plugin.php

@@ -20,6 +20,7 @@
 use craft\webhooks\filters\RevisionFilter;
 use GuzzleHttp\Exception\RequestException;
 use GuzzleHttp\RequestOptions;
+use yii\base\Application;
 use yii\base\Arrayable;
 use yii\base\Event;
 use yii\base\Exception;
@@ -78,7 +79,14 @@ class Plugin extends \craft\base\Plugin
     /**
      * @inheritdoc
      */
-    public $schemaVersion = '2.2.0';
+    public $schemaVersion = '2.3.0';
+
+    /**
+     * @var SendRequestJob[] The request jobs that should be queued up at the end of the web request.
+     * @see request()
+     * @since 2.3.0
+     */
+    private $_pendingJobs = [];
 
     /**
      * @inheritdoc
@@ -180,7 +188,15 @@ public function init()
                 $url = $view->renderString($url, [
                     'event' => $e,
                 ]);
-                $this->request($webhook->method, $url, $headers, $body, $webhook->id);
+
+                if ($webhook->debounceKeyFormat) {
+                    $debounceKey = Craft::parseEnv($webhook->debounceKeyFormat);
+                    $debounceKey = $webhook->id . ':' . $view->renderString($debounceKey, [
+                        'event' => $e,
+                    ]);
+                }
+
+                $this->request($webhook->method, $url, $headers, $body, $webhook->id, $debounceKey ?? null);
             });
         }
 
@@ -225,14 +241,45 @@ public function getCpNavItem()
      * @param array|null $headers
      * @param string|null $body
      * @param int|null $webhookId
+     * @param string|null $debounceKey
      * @throws \yii\db\Exception
      */
-    public function request(string $method, string $url, array $headers = null, string $body = null, int $webhookId = null)
+    public function request(string $method, string $url, array $headers = null, string $body = null, int $webhookId = null, string $debounceKey = null)
     {
         $db = Craft::$app->getDb();
+
+        if ($debounceKey !== null) {
+            // See if there is an existing pending request with the same key
+            $requestId = (new Query())
+                ->select(['id'])
+                ->from(['{{%webhookrequests}}'])
+                ->where([
+                    'debounceKey' => $debounceKey,
+                    'status' => self::STATUS_PENDING,
+                ])
+                ->scalar();
+
+            // If so and we get a lock on it, update that request instead of creating a new one
+            if ($requestId && $this->_lockRequest($requestId)) {
+                $db->createCommand()
+                    ->update('{{%webhookrequests}}', [
+                        'method' => $method,
+                        'url' => $url,
+                        'requestHeaders' => $headers ? Json::encode($headers) : null,
+                        'requestBody' => $body,
+                        'dateCreated' => Db::prepareDateForDb(new \DateTime()),
+                    ], [
+                        'id' => $requestId,
+                    ], [], false);
+                $this->_unlockRequest($requestId);
+                return;
+            }
+        }
+
         $db->createCommand()
             ->insert('{{%webhookrequests}}', [
                 'webhookId' => $webhookId,
+                'debounceKey' => $debounceKey,
                 'status' => self::STATUS_PENDING,
                 'method' => $method,
                 'url' => $url,
@@ -243,10 +290,27 @@ public function request(string $method, string $url, array $headers = null, stri
             ], false)
             ->execute();
 
-        Craft::$app->getQueue()->push(new SendRequestJob([
+        $this->_pendingJobs[] = new SendRequestJob([
             'requestId' => $db->getLastInsertID('{{%webhookrequests}}'),
             'webhookId' => $webhookId,
-        ]));
+        ]);
+
+        if (count($this->_pendingJobs) === 1) {
+            Craft::$app->on(Application::EVENT_AFTER_REQUEST, [$this, 'pushPendingJobs'], null, false);
+        }
+    }
+
+    /**
+     * Pushes any pending jobs to the queue.
+     *
+     * @since 2.3.0
+     */
+    public function pushPendingJobs()
+    {
+        $queue = Craft::$app->getQueue();
+        while ($job = array_shift($this->_pendingJobs)) {
+            $queue->push($job);
+        }
     }
 
     /**
@@ -287,9 +351,7 @@ public function getRequestData(int $requestId): array
     public function sendRequest(int $requestId): bool
     {
         // Acquire a lock on the request
-        $lockName = 'webhook.' . $requestId;
-        $mutex = Craft::$app->getMutex();
-        if (!$mutex->acquire($lockName)) {
+        if (!$this->_lockRequest($requestId, 1)) {
             throw new Exception('Could not acquire a lock for the webhook request ' . $requestId);
         }
 
@@ -338,7 +400,7 @@ public function sendRequest(int $requestId): bool
             ->execute();
 
         // Release the lock
-        $mutex->release($lockName);
+        $this->_unlockRequest($requestId);
 
         return $success;
     }
@@ -426,4 +488,14 @@ public function getAllFilters(): array
 
         return $event->types;
     }
+
+    private function _lockRequest(int $requestId, int $timeout = 0): bool
+    {
+        return Craft::$app->getMutex()->acquire("webhook:$requestId", $timeout);
+    }
+
+    private function _unlockRequest(int $requestId): bool
+    {
+        return Craft::$app->getMutex()->release("webhook:$requestId");
+    }
 }

src/Webhook.php

@@ -52,6 +52,11 @@ class Webhook extends Model
      */
     public $filters = [];
 
+    /**
+     * @var string|null
+     */
+    public $debounceKeyFormat;
+
     /**
      * @var string
      */
@@ -151,6 +156,7 @@ function(string $attribute, array $params = null, Validator $validator) {
                     }
                 }
             ],
+            [['debounceKeyFormat'], 'string'],
             [
                 ['filters'],
                 function() {

src/WebhookManager.php

@@ -198,6 +198,7 @@ public function saveWebhook(Webhook $webhook, bool $runValidation = true): bool
             'class' => $webhook->class,
             'event' => $webhook->event,
             'filters' => $webhook->filters ? Json::encode($webhook->filters) : null,
+            'debounceKeyFormat' => $webhook->debounceKeyFormat,
             'method' => $webhook->method,
             'url' => $webhook->url,
             'headers' => $webhook->headers ? Json::encode($webhook->headers) : null,
@@ -247,6 +248,7 @@ private function _createWebhookQuery(): Query
                 'class',
                 'event',
                 'filters',
+                'debounceKeyFormat',
                 'method',
                 'url',
                 'headers',

src/migrations/Install.php

@@ -34,6 +34,7 @@ public function safeUp()
             'class' => $this->string()->notNull(),
             'event' => $this->string()->notNull(),
             'filters' => $this->text(),
+            'debounceKeyFormat' => $this->string(),
             'method' => $this->string(10)->notNull(),
             'url' => $this->string()->notNull(),
             'headers' => $this->text(),
@@ -51,6 +52,7 @@ public function safeUp()
         $this->createTable('{{%webhookrequests}}', [
             'id' => $this->primaryKey(),
             'webhookId' => $this->integer(),
+            'debounceKey' => $this->string(),
             'status' => $this->string()->notNull(),
             'attempts' => $this->tinyInteger(),
             'method' => $this->string(),
@@ -69,6 +71,7 @@ public function safeUp()
         $this->createIndex(null, '{{%webhooks}}', ['enabled']);
         $this->createIndex(null, '{{%webhooks}}', ['groupId', 'name']);
         $this->createIndex(null, '{{%webhooks}}', ['name'], true);
+        $this->createIndex(null, '{{%webhookrequests}}', ['debounceKey', 'status']);
         $this->addForeignKey(null, '{{%webhooks}}', ['groupId'], '{{%webhookgroups}}', ['id'], 'SET NULL');
         $this->addForeignKey(null, '{{%webhookrequests}}', ['webhookId'], '{{%webhooks}}', ['id'], 'SET NULL');
     }

src/migrations/m200617_212615_debounce_keys.php

@@ -0,0 +1,31 @@
+<?php
+
+namespace craft\webhooks\migrations;
+
+use Craft;
+use craft\db\Migration;
+
+/**
+ * m200617_212615_debounce_keys migration.
+ */
+class m200617_212615_debounce_keys extends Migration
+{
+    /**
+     * @inheritdoc
+     */
+    public function safeUp()
+    {
+        $this->addColumn('{{%webhooks}}', 'debounceKeyFormat', $this->string()->after('filters'));
+        $this->addColumn('{{%webhookrequests}}', 'debounceKey', $this->string()->after('webhookId'));
+        $this->createIndex(null, '{{%webhookrequests}}', ['debounceKey', 'status']);
+    }
+
+    /**
+     * @inheritdoc
+     */
+    public function safeDown()
+    {
+        echo "m200617_212615_debounce_keys cannot be reverted.\n";
+        return false;
+    }
+}

src/templates/_manage/edit.html

@@ -126,6 +126,16 @@
         </div>
     {% endset %}
 
+    {{ forms.textField({
+        label: 'Debounce Key Format'|t('webhooks'),
+        instructions: 'Template that defines the webhook’s “debounce key” format. If two webhooks generate the same debounce key, only the latter will be sent.'|t('webhooks') ~ ' ' ~ 'Can be set to an environment variable (`$VARIABLE_NAME`) or Twig code.'|t('webhooks'),
+        id: 'debounce-key-format',
+        class: 'code',
+        name: 'debounceKeyFormat',
+        value: webhook.debounceKeyFormat,
+        errors: webhook.getErrors('debounceKeyFormat')
+    }) }}
+
     {{ forms.field({
         label: 'Request Method & URL'|t('webhooks'),
         required: true,