wip

Author: michelle0927

components/youtube_data_api/actions/add-playlist-items/add-playlist-items.mjs

@@ -18,7 +18,7 @@ export default {
     },
     videoIds: {
       type: "string[]",
-      label: "Video Ids",
+      label: "Video IDs",
       description: "Array of identifiers of the videos to be added to the playlist",
     },
     onBehalfOfContentOwner: {

components/youtube_data_api/actions/delete-playlist-items/delete-playlist-items.mjs

@@ -18,7 +18,7 @@ export default {
     },
     videoIds: {
       type: "string[]",
-      label: "Video Ids",
+      label: "Video IDs",
       description: "Array of identifiers of the videos to be removed from the playlist",
       async options() {
         const { data } = await this.youtubeDataApi.getPlaylistItems({

components/youtube_data_api/actions/list-videos/list-videos.mjs

@@ -41,7 +41,7 @@ export default {
       };
       if (this.regionCode && this.regionCode.length === 2) {
         dynamicProps.videoCategoryId = {
-          label: "Video Category Id",
+          label: "Video Category ID",
           description: "The videoCategoryId parameter identifies the video category for which the chart should be retrieved. By default, charts are not restricted to a particular category.",
           type: "string",
           optional: true,

components/youtube_data_api/common-app.mjs

@@ -1,5 +1,5 @@
 import youtube from "@googleapis/youtube";
-import { toArray } from "./utils.mjs";
+import { toArray } from "./common/utils.mjs";
 import { promisify } from "util";
 const pause = promisify((delay, fn) => setTimeout(fn, delay));
 
@@ -12,12 +12,12 @@ export default {
       reloadProps: true,
     },
     playlistId: {
-      label: "Playlist Id",
+      label: "Playlist ID",
       description: "The playlistId parameter specifies a unique YouTube playlist ID.",
       type: "string",
     },
     channelId: {
-      label: "Channel Id",
+      label: "Channel ID",
       description: "The channelId parameter specifies a unique YouTube channel ID.",
       type: "string",
     },
@@ -56,7 +56,7 @@ export default {
     maxResults: {
       type: "integer",
       label: "Maximum Results",
-      description: "The maximum number of results in a channel to return. Should be divisible by 5 (ex. 5, 10, 15).",
+      description: "The maximum number of items that should be returned in the result set. Acceptable values are 0 to 50, inclusive.",
       default: 20,
     },
     title: {
@@ -113,7 +113,7 @@ export default {
     },
     userOwnedPlaylist: {
       type: "string",
-      label: "Playlist Id",
+      label: "Playlist ID",
       description: "Add items to the selected playlist",
       async options({ prevContext }) {
         const { pageToken } = prevContext;
@@ -140,19 +140,25 @@ export default {
     },
     userOwnedVideo: {
       type: "string",
-      label: "Video Id",
+      label: "Video ID",
       description: "Select the video to update",
-      async options({ prevContext }) {
+      async options({
+        prevContext, channelId,
+      }) {
         const { pageToken } = prevContext;
         const params = {
           part: [
             "id",
             "snippet",
           ],
           type: "video",
-          forMine: true,
           pageToken,
         };
+        if (channelId) {
+          params.channelId = channelId;
+        } else {
+          params.forMine = true;
+        }
         const { data } = await this.getVideos(params);
         const options = data.items?.map((item) => ({
           label: item.snippet.title,
@@ -168,7 +174,7 @@ export default {
     },
     userOwnedChannel: {
       type: "string",
-      label: "Channel Id",
+      label: "Channel ID",
       description: "Select the channel to update",
       async options({ prevContext }) {
         const { pageToken } = prevContext;

components/youtube_data_api/utils.mjs renamed to components/youtube_data_api/common/utils.mjs

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/youtube_data_api",
-  "version": "0.6.1",
+  "version": "0.6.2",
   "description": "Pipedream Youtube Components",
   "main": "youtube_data_api.app.mjs",
   "keywords": [
@@ -11,6 +11,7 @@
   "author": "Pipedream <[email protected]> (https://pipedream.com/)",
   "dependencies": {
     "@googleapis/youtube": "^6.0.0",
+    "@pipedream/platform": "^3.0.3",
     "got": "^12.5.1",
     "util": "^0.12.4"
   },

components/youtube_data_api/package.json

@@ -20,8 +20,7 @@ export default {
         ...this.getParams(),
       };
       const publishedAfter = await this.paginateVideos(params);
-      if (publishedAfter) this._setPublishedAfter(publishedAfter);
-      else this._setPublishedAfter(new Date());
+      this._setPublishedAfter(publishedAfter || new Date());
     },
   },
   methods: {
@@ -49,6 +48,9 @@ export default {
     getParams() {
       return {};
     },
+    isRelevant() {
+      return true;
+    },
     generateMeta(video) {
       const {
         id,
@@ -64,58 +66,54 @@ export default {
       const meta = this.generateMeta(result);
       this.$emit(result, meta);
     },
-    /**
-     * Paginate through results from getVideos(), emit each video, and return the
-     * most recent date that a video was published.
-     *
-     * @param {object} params - The parameters to pass into getVideos().
-     * @returns a string of the most recent date that a video was published
-     */
     async paginateVideos(params) {
-      let count = 0;
-      let totalResults = 1;
+      const videos = await this.paginate({
+        fn: this.youtubeDataApi.getVideos,
+        params,
+      });
+      videos.forEach((video) => this.emitEvent(video));
+      return videos[0].snippet.publishedAt;
+    },
+    async paginatePlaylistItems(params, publishedAfter = null) {
+      const videos = await this.paginate({
+        fn: this.youtubeDataApi.getPlaylistItems,
+        params,
+        publishedAfter,
+      });
       let lastPublished;
-      while (
-        count < totalResults &&
-        (!params.maxResults || count < params.maxResults)
-      ) {
-        const results = (await this.youtubeDataApi.getVideos(params)).data;
-        totalResults = results.pageInfo.totalResults;
-        for (const video of results.items) {
-          if (!lastPublished) lastPublished = video.snippet.publishedAt;
+      for (const video of videos) {
+        if (!lastPublished || Date.parse(video.snippet.publishedAt) > Date.parse(lastPublished)) {
+          lastPublished = video.snippet.publishedAt;
           this.emitEvent(video);
-          count++;
         }
-        if (!results.items || results.items.length < 1) break;
-        if (!results.nextPageToken) break;
-        else params.pageToken = results.nextPageToken;
       }
       return lastPublished;
     },
-    async paginatePlaylistItems(params, publishedAfter = null) {
-      let totalResults = 1;
-      let count = 0;
-      let countEmitted = 0;
-      let lastPublished;
-
-      while (count < totalResults && countEmitted < params.maxResults) {
-        const results = (await this.youtubeDataApi.getPlaylistItems(params)).data;
-        totalResults = results.pageInfo.totalResults;
-        for (const video of results.items) {
-          if (this.isRelevant(video, publishedAfter)) {
-            if (
-              !lastPublished ||
-              Date.parse(video.snippet.publishedAt) > Date.parse(lastPublished)
-            )
-              lastPublished = video.snippet.publishedAt;
-            this.emitEvent(video);
-            countEmitted++;
-          }
+    async paginate({
+      fn, params, publishedAfter,
+    }) {
+      let totalResults, done, count = 0;;
+      const results = [];
+      do {
+        const {
+          data: {
+            pageInfo, items, nextPageToken,
+          },
+        } = await fn(params);
+        for (const item of items) {
           count++;
+          if (this.isRelevant(item, publishedAfter)) {
+            results.push(item);
+            if ((params.maxResults && results.length >= params.maxResults) || !nextPageToken) {
+              done = true;
+              break;
+            }
+          }
         }
-        params.pageToken = results.nextPageToken;
-      }
-      return lastPublished;
+        params.pageToken = nextPageToken;
+        totalResults = pageInfo.totalResults;
+      } while ((count < totalResults) && !done);
+      return results;
     },
   },
   async run() {

components/youtube_data_api/sources/common.mjs renamed to components/youtube_data_api/sources/common/common.mjs

@@ -1,10 +1,15 @@
-import common from "../common.mjs";
+import common from "../common/common.mjs";
+import { ConfigurationError } from "@pipedream/platform";
 
 export default {
   ...common,
   hooks: {
     ...common.hooks,
-    deploy() {},
+    deploy() {
+      if (!this.channelId && !this.videoId) {
+        throw new ConfigurationError("Must enter one of `channelId` or `videoId`");
+      }
+    },
   },
   methods: {
     ...common.methods,
@@ -22,8 +27,12 @@ export default {
 
     const params = {
       part: "id,replies,snippet",
-      videoId: this.videoId,
     };
+    if (this.videoId) {
+      params.videoId = this.videoId;
+    } else {
+      params.allThreadsRelatedToChannelId = this.channelId;
+    }
 
     const items = [];
     while (true) {
@@ -47,12 +56,9 @@ export default {
     for (const comment of comments) {
       const publishedAt = Date.parse(comment.snippet.publishedAt);
       if (!publishedAfter || publishedAt > publishedAfter) {
-        const meta = this.generateMeta(comment);
-        this.$emit(comment, meta);
-      }
-      if (publishedAt > maxPublishedAfter) {
-        maxPublishedAfter = publishedAt;
+        this.emitEvent(comment);
       }
+      maxPublishedAfter = Math.max(maxPublishedAfter, publishedAt);
     }
 
     this._setPublishedAfter(maxPublishedAfter);