GitHub sources usability improvements (#13895)

* Docs link updates for sources

* Adding docs info alerts

* Improvements for common polling interface

* Adjustments to polling

* Improving polling sources

* New Repository improvements

* Common interface for PR notifications sources

* Reusing base for new security alert

* "New mention" rewrite + adjustments

* Behavior adjustments to notification sources

* Adjustments for mentions

* Version bumps

* version bump

Author: GTFalcao

components/github/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/github",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "Pipedream Github Components",
   "main": "github.app.mjs",
   "keywords": [

components/github/sources/common/common-flex.mjs

@@ -17,18 +17,24 @@ export default {
     db: "$.service.db",
   },
   async additionalProps() {
+    const getDocsInfo = (docsLink) => ({
+      type: "alert",
+      alertType: "info",
+      content: `[See the GitHub documentation](${docsLink}) for more information on the event format.`,
+    });
     if (await this.checkAdminPermission()) {
       return {
         http: {
           type: "$.interface.http",
         },
         ...this.getHttpAdditionalProps(),
+        docsInfo: getDocsInfo(this.getHttpDocsLink()),
       };
     } else {
       return {
         info: {
           type: "alert",
-          alertType: "info",
+          alertType: "warning",
           content: "Admin rights on the repo are required in order to register webhooks. In order to continue setting up your source, configure a polling interval below to check for new events.",
         },
         timer: {
@@ -38,6 +44,7 @@ export default {
           },
         },
         ...this.getTimerAdditionalProps(),
+        docsInfo: getDocsInfo(this.getTimerDocsLink()),
       };
     }
   },
@@ -196,6 +203,12 @@ export default {
 
       this._setSavedItems(savedItems);
     },
+    getHttpDocsLink() {
+      return "https://docs.github.com/en/webhooks/webhook-events-and-payloads";
+    },
+    getTimerDocsLink() {
+      return "https://docs.github.com/en/rest?apiVersion=2022-11-28";
+    },
   },
   hooks: {
     async activate() {

components/github/sources/common/common-polling-pr-notifications.mjs

@@ -0,0 +1,41 @@
+import common from "./common-polling.mjs";
+
+export default {
+  ...common,
+  methods: {
+    ...common.methods,
+    async getAndProcessData(maxEmits = 0) {
+      const savedIds = this._getSavedIds();
+      const items = await this.getItems();
+
+      const urlData = new Map();
+      let amountEmits = 0;
+
+      const promises = items?.
+        filter?.((item) => !savedIds.includes(this.getItemId(item)))
+        .map((item) => (async () => {
+          if (item?.subject?.notification !== null) {
+            const url = item.subject.url;
+            if (!urlData.has(url)) {
+              urlData.set(url, await this.github.getFromUrl({
+                url: item.subject.url,
+              }));
+            }
+            const pullRequest = urlData.get(url);
+            if (!maxEmits || (amountEmits < maxEmits)) {
+              this.$emit(pullRequest, {
+                id: pullRequest.id,
+                ...this.getItemMetadata(pullRequest),
+              });
+              amountEmits++;
+            }
+          }
+          savedIds.push(this.getItemId(item));
+        })());
+
+      if (promises?.length) await Promise.allSettled(promises);
+
+      this._setSavedIds(savedIds);
+    },
+  },
+};

components/github/sources/common/common-polling.mjs

@@ -12,4 +12,39 @@ export default {
     },
     db: "$.service.db",
   },
+  methods: {
+    _getSavedIds() {
+      return this.db.get("savedIds") || [];
+    },
+    _setSavedIds(value) {
+      this.db.set("savedIds", value);
+    },
+    getItemId(item) {
+      return item.id;
+    },
+    async getAndProcessData(maxEmits = 0) {
+      const savedIds = this._getSavedIds();
+      const items = await this.getItems();
+
+      items?.filter?.((item) => !savedIds.includes(this.getItemId(item))).forEach((item, index) => {
+        if ((!maxEmits) || (index < maxEmits)) {
+          this.$emit(item, {
+            id: this.getItemId(item),
+            ...this.getItemMetadata(item),
+          });
+        }
+        savedIds.push(this.getItemId(item));
+      });
+
+      this._setSavedIds(savedIds);
+    },
+  },
+  hooks: {
+    async deploy() {
+      await this.getAndProcessData(5);
+    },
+  },
+  async run() {
+    await this.getAndProcessData();
+  },
 };

components/github/sources/new-branch/new-branch.mjs

@@ -3,15 +3,12 @@ import {
   getSampleTimerEvent, getSampleWebhookEvent,
 } from "./common-sample-events.mjs";
 
-const DOCS_LINK =
-  "https://docs.github.com/en/webhooks/webhook-events-and-payloads#create";
-
 export default {
   ...common,
   key: "github-new-branch",
-  name: "New Branch",
-  description: `Emit new event when a branch is created [See the documentation](${DOCS_LINK})`,
-  version: "1.0.4",
+  name: "New Branch Created",
+  description: "Emit new event when a branch is created.",
+  version: "1.0.5",
   type: "source",
   dedupe: "unique",
   methods: {
@@ -35,5 +32,11 @@ export default {
     getPollingData(args) {
       return this.github.getBranches(args);
     },
+    getHttpDocsLink() {
+      return "https://docs.github.com/en/webhooks/webhook-events-and-payloads#create";
+    },
+    getTimerDocsLink() {
+      return "https://docs.github.com/en/rest/branches/branches?apiVersion=2022-11-28#list-branches";
+    },
   },
 };

components/github/sources/new-collaborator/new-collaborator.mjs

@@ -3,15 +3,12 @@ import {
   getSampleTimerEvent, getSampleWebhookEvent,
 } from "./common-sample-events.mjs";
 
-const DOCS_LINK =
-  "https://docs.github.com/en/webhooks/webhook-events-and-payloads#member";
-
 export default {
   ...common,
   key: "github-new-collaborator",
   name: "New Collaborator",
-  description: `Emit new event when a collaborator is added [See the documentation](${DOCS_LINK})`,
-  version: "1.0.4",
+  description: "Emit new event when a collaborator is added",
+  version: "1.0.5",
   type: "source",
   dedupe: "unique",
   methods: {
@@ -35,5 +32,11 @@ export default {
     getPollingData(args) {
       return this.github.getRepositoryLatestCollaborators(args);
     },
+    getHttpDocsLink() {
+      return "https://docs.github.com/en/webhooks/webhook-events-and-payloads#member";
+    },
+    getTimerDocsLink() {
+      return "https://docs.github.com/en/rest/collaborators/collaborators?apiVersion=2022-11-28#list-repository-collaborators";
+    },
   },
 };

components/github/sources/new-commit-comment/new-commit-comment.mjs

@@ -3,17 +3,22 @@ import {
   getSampleTimerEvent, getSampleWebhookEvent,
 } from "./common-sample-events.mjs";
 
-const DOCS_LINK =
-  "https://docs.github.com/en/webhooks/webhook-events-and-payloads#commit_comment";
-
 export default {
   ...common,
   key: "github-new-commit-comment",
   name: "New Commit Comment",
-  description: `Emit new event when a commit comment is created [See the documentation](${DOCS_LINK})`,
-  version: "1.0.4",
+  description: "Emit new event when a commit comment is created",
+  version: "1.0.5",
   type: "source",
   dedupe: "unique",
+  props: {
+    eventTypeInfo: {
+      type: "alert",
+      alertType: "info",
+      content: "**Note:** commit comments are not the same as pull request comments. [See the GitHub documentation](https://docs.github.com/en/rest/guides/working-with-comments?apiVersion=2022-11-28) for more information.",
+    },
+    ...common.props,
+  },
   methods: {
     ...common.methods,
     getSampleTimerEvent,
@@ -35,5 +40,11 @@ export default {
     getPollingData(args) {
       return this.github.getRepositoryLatestCommitComments(args);
     },
+    getHttpDocsLink() {
+      return "https://docs.github.com/en/webhooks/webhook-events-and-payloads#commit_comment";
+    },
+    getTimerDocsLink() {
+      return "https://docs.github.com/en/rest/commits/comments?apiVersion=2022-11-28#list-commit-comments-for-a-repository";
+    },
   },
 };

components/github/sources/new-commit/new-commit.mjs

@@ -3,18 +3,20 @@ import {
   getSampleTimerEvent, getSampleWebhookEvent,
 } from "./common-sample-events.mjs";
 
-const DOCS_LINK =
-  "https://docs.github.com/en/webhooks/webhook-events-and-payloads#push";
-
 export default {
   ...common,
   key: "github-new-commit",
   name: "New Commit",
-  description: `Emit new event when commits are pushed to a branch [See the documentation](${DOCS_LINK})`,
-  version: "1.0.5",
+  description: "Emit new event when commits are pushed to a branch",
+  version: "1.0.6",
   type: "source",
   dedupe: "unique",
   props: {
+    eventTypeInfo: {
+      type: "alert",
+      alertType: "info",
+      content: "**Note:** one event is emitted for each individual commit, even if they are received at the same time.",
+    },
     ...common.props,
     branch: {
       propDefinition: [
@@ -95,5 +97,11 @@ export default {
       this._setSavedItems(savedItems);
       this._setLastTimestamp(Date.now());
     },
+    getHttpDocsLink() {
+      return "https://docs.github.com/en/webhooks/webhook-events-and-payloads#push";
+    },
+    getTimerDocsLink() {
+      return "https://docs.github.com/en/rest/commits/commits?apiVersion=2022-11-28#list-commits";
+    },
   },
 };

components/github/sources/new-discussion/new-discussion.mjs

@@ -3,15 +3,12 @@ import {
   getSampleTimerEvent, getSampleWebhookEvent,
 } from "./common-sample-events.mjs";
 
-const DOCS_LINK =
-  "https://docs.github.com/en/webhooks/webhook-events-and-payloads#discussion";
-
 export default {
   ...common,
   key: "github-new-discussion",
   name: "New Discussion",
-  description: `Emit new event when a discussion is created [See the documentation](${DOCS_LINK})`,
-  version: "1.0.4",
+  description: "Emit new event when a discussion is created",
+  version: "1.0.5",
   type: "source",
   dedupe: "unique",
   methods: {
@@ -42,5 +39,11 @@ export default {
         return dateA - dateB;
       });
     },
+    getHttpDocsLink() {
+      return "https://docs.github.com/en/webhooks/webhook-events-and-payloads#discussion";
+    },
+    getTimerDocsLink() {
+      return "https://docs.github.com/en/graphql/reference/objects#discussion";
+    },
   },
 };

components/github/sources/new-fork/new-fork.mjs

@@ -3,15 +3,12 @@ import {
   getSampleTimerEvent, getSampleWebhookEvent,
 } from "./common-sample-events.mjs";
 
-const DOCS_LINK =
-  "https://docs.github.com/en/webhooks/webhook-events-and-payloads#fork";
-
 export default {
   ...common,
   key: "github-new-fork",
   name: "New Fork",
-  description: `Emit new event when a repository is forked [See the documentation](${DOCS_LINK})`,
-  version: "1.0.4",
+  description: "Emit new event when a repository is forked",
+  version: "1.0.5",
   type: "source",
   dedupe: "unique",
   methods: {
@@ -35,5 +32,11 @@ export default {
     getPollingData(args) {
       return this.github.getRepositoryForks(args);
     },
+    getHttpDocsLink() {
+      return "https://docs.github.com/en/webhooks/webhook-events-and-payloads#fork";
+    },
+    getTimerDocsLink() {
+      return "https://docs.github.com/en/rest/repos/forks?apiVersion=2022-11-28#list-forks";
+    },
   },
 };