16762 components scrapecreators

components/scrapecreators/actions/fetch-creator-profile/fetch-creator-profile.mjs

@@ -0,0 +1,45 @@
+import app from "../../scrapecreators.app.mjs";
+
+export default {
+  key: "scrapecreators-fetch-creator-profile",
+  name: "Fetch Creator Profile",
+  description: "Fetch a creator profile based on platform and handle or unique ID. [See the documentation](https://docs.scrapecreators.com/introduction)",
+  version: "0.0.1",
+  type: "action",
+  props: {
+    app,
+    platform: {
+      propDefinition: [
+        app,
+        "platform",
+      ],
+    },
+    profileId: {
+      propDefinition: [
+        app,
+        "profileId",
+      ],
+    },
+  },
+  async run({ $ }) {
+    const summary = `Successfully fetched creator profile for **${this.profileId}**`;
+    try {
+      const response = await this.app.fetchCreatorProfile({
+        $,
+        platform: this.platform,
+        profileId: this.profileId,
+      });
+
+      $.export("$summary", summary);
+      return response;
+    } catch ({ response }) {
+      if (response.data?.success === true) {
+        $.export("$summary", summary);
+        return {
+          message: response.data.message,
+        };
+      }
+      throw new Error(response.data.message);
+    }
+  },
+};

components/scrapecreators/actions/search-creators/search-creators.mjs

@@ -0,0 +1,50 @@
+import { SEARCH_PLATFORMS } from "../../common/constants.mjs";
+import app from "../../scrapecreators.app.mjs";
+
+export default {
+  key: "scrapecreators-search-creators",
+  name: "Search Creators",
+  description: "Search for creators based on platform and query. [See the documentation](https://docs.scrapecreators.com/api-reference/introduction)",
+  version: "0.0.1",
+  type: "action",
+  props: {
+    app,
+    platform: {
+      propDefinition: [
+        app,
+        "platform",
+      ],
+      options: SEARCH_PLATFORMS,
+    },
+    query: {
+      type: "string",
+      label: "Query",
+      description: "The query to search for creators on",
+    },
+    limit: {
+      type: "integer",
+      label: "Limit",
+      description: "The maximum number of creators to return",
+      optional: true,
+    },
+  },
+  async run({ $ }) {
+    const results = this.app.paginate({
+      $,
+      fn: this.app.searchCreators,
+      maxResults: this.limit,
+      platform: this.platform,
+      params: {
+        query: this.query,
+      },
+    });
+
+    const data = [];
+    for await (const item of results) {
+      data.push(item);
+    }
+
+    $.export("$summary", `Successfully searched for **${this.query}**`);
+    return data;
+  },
+};

components/scrapecreators/common/constants.mjs

@@ -0,0 +1,42 @@
+export const PATH_PLATFORMS = {
+  "channel": [
+    "youtube",
+  ],
+  "user/boards": [
+    "pinterest",
+  ],
+  "empty": [
+    "linktree",
+    "komi",
+    "pillar",
+    "linkbio",
+  ],
+};
+
+export const URL_PLATFORMS = [
+  "linkedin",
+  "facebook",
+  ...PATH_PLATFORMS["empty"],
+];
+
+export const SEARCH_PLATFORMS = [
+  "tiktok",
+  "threads",
+];
+
+export const HANDLE_PLATFORMS = [
+  "instagram",
+  "twitter",
+  "truthsocial",
+  "bluesky",
+  "twitch",
+  "snapchat",
+  ...SEARCH_PLATFORMS,
+  ...PATH_PLATFORMS["user/boards"],
+  ...PATH_PLATFORMS["channel"],
+];
+
+export const PLATFORMS = [
+  ...URL_PLATFORMS,
+  ...HANDLE_PLATFORMS,
+];

components/scrapecreators/common/utils.mjs

@@ -0,0 +1,46 @@
+export function getObjectDiff(obj1, obj2) {
+  const diff = {};
+
+  // Check for differences in obj1's properties
+  for (const key in obj1) {
+    if (Object.prototype.hasOwnProperty.call(obj1, key)) {
+      if (!Object.prototype.hasOwnProperty.call(obj2, key)) {
+        diff[key] = {
+          oldValue: obj1[key],
+          newValue: undefined,
+          status: "deleted",
+        };
+      } else if (typeof obj1[key] === "object" && obj1[key] !== null &&
+                 typeof obj2[key] === "object" && obj2[key] !== null) {
+        const nestedDiff = getObjectDiff(obj1[key], obj2[key]);
+        if (Object.keys(nestedDiff).length > 0) {
+          diff[key] = {
+            status: "modified",
+            changes: nestedDiff,
+          };
+        }
+      } else if (obj1[key] !== obj2[key]) {
+        diff[key] = {
+          oldValue: obj1[key],
+          newValue: obj2[key],
+          status: "modified",
+        };
+      }
+    }
+  }
+
+  // Check for properties added in obj2
+  for (const key in obj2) {
+    if (Object.prototype.hasOwnProperty.call(obj2, key)) {
+      if (!Object.prototype.hasOwnProperty.call(obj1, key)) {
+        diff[key] = {
+          oldValue: undefined,
+          newValue: obj2[key],
+          status: "added",
+        };
+      }
+    }
+  }
+
+  return diff;
+}

components/scrapecreators/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/scrapecreators",
-  "version": "0.0.1",
+  "version": "0.1.0",
   "description": "Pipedream ScrapeCreators Components",
   "main": "scrapecreators.app.mjs",
   "keywords": [
@@ -11,5 +11,8 @@
   "author": "Pipedream <[email protected]> (https://pipedream.com/)",
   "publishConfig": {
     "access": "public"
+  },
+  "dependencies": {
+    "@pipedream/platform": "^3.1.0"
   }
-}
+}

components/scrapecreators/scrapecreators.app.mjs

@@ -1,11 +1,116 @@
+import { axios } from "@pipedream/platform";
+import {
+  PATH_PLATFORMS, PLATFORMS, URL_PLATFORMS,
+} from "./common/constants.mjs";
+
 export default {
   type: "app",
   app: "scrapecreators",
-  propDefinitions: {},
+  propDefinitions: {
+    platform: {
+      type: "string",
+      label: "Platform",
+      description: "The platform to search for creators on",
+      options: PLATFORMS,
+    },
+    profileId: {
+      type: "string",
+      label: "Profile",
+      description: "The handle, URL or unique ID of the creator profile",
+    },
+  },
   methods: {
-    // this.$auth contains connected account data
-    authKeys() {
-      console.log(Object.keys(this.$auth));
+    _baseUrl() {
+      return "https://api.scrapecreators.com/v1";
+    },
+    _headers() {
+      return {
+        "x-api-key": `${this.$auth.api_key}`,
+      };
+    },
+    _makeRequest({
+      $ = this, path, ...opts
+    }) {
+      return axios($, {
+        url: this._baseUrl() + path,
+        headers: this._headers(),
+        ...opts,
+      });
+    },
+    searchCreators({
+      platform, ...opts
+    }) {
+      return this._makeRequest({
+        path: `/${platform}/search/users`,
+        ...opts,
+      });
+    },
+    fetchCreatorProfile({
+      platform, profileId, ...opts
+    }) {
+      const params = this.parseParams(platform, profileId);
+      const path = this.parsePath(platform);
+
+      return this._makeRequest({
+        path: `/${platform}${path}`,
+        params,
+        ...opts,
+      });
+    },
+    parseParams(platform, profileId) {
+      const params = {};
+      const urlPlatforms = URL_PLATFORMS;
+
+      if (urlPlatforms.includes(platform)) {
+        params.url = profileId;
+      } else {
+        params.handle = profileId;
+      }
+      return params;
+    },
+    parsePath(platform) {
+      const path = Object.entries(PATH_PLATFORMS).find(([
+        key,
+        value,
+      ]) => value.includes(platform)
+        ? key
+        : null);
+
+      return path
+        ? path[0] === "empty"
+          ? ""
+          : `/${path[0]}`
+        : "/profile";
+    },
+    async *paginate({
+      fn, params = {}, platform, maxResults = null, ...opts
+    }) {
+      let hasMore = false;
+      let count = 0;
+      let newCursor;
+
+      do {
+        params.cursor = newCursor;
+        const {
+          cursor,
+          users,
+        } = await fn({
+          platform,
+          params,
+          ...opts,
+        });
+        for (const d of users) {
+          yield d;
+
+          if (maxResults && ++count === maxResults) {
+            return count;
+          }
+        }
+
+        newCursor = cursor;
+        hasMore = users.length;
+
+      } while (hasMore);
     },
   },
 };

components/scrapecreators/sources/common/base.mjs

@@ -0,0 +1,58 @@
+import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
+import scrapecreators from "../../scrapecreators.app.mjs";
+
+export default {
+  props: {
+    scrapecreators,
+    db: "$.service.db",
+    timer: {
+      type: "$.interface.timer",
+      default: {
+        intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
+      },
+    },
+  },
+  methods: {
+    _getLastId() {
+      return this.db.get("lastId") || 0;
+    },
+    _setLastId(lastId) {
+      this.db.set("lastId", lastId);
+    },
+    async emitEvent(maxResults = false) {
+      const lastId = this._getLastId();
+      const fieldId = this.getFieldId();
+      const fn = this.getFunction();
+      const { value: response } = await fn();
+
+      let responseArray = [];
+      for (const item of response) {
+        if (item[fieldId] === lastId) break;
+        responseArray.push(item);
+      }
+
+      if (responseArray.length) {
+        if (maxResults && (responseArray.length > maxResults)) {
+          responseArray.length = maxResults;
+        }
+        this._setLastId(responseArray[0][fieldId]);
+      }
+
+      for (const item of responseArray.reverse()) {
+        this.$emit(item, {
+          id: item[fieldId],
+          summary: this.getSummary(item),
+          ts: Date.parse(new Date()),
+        });
+      }
+    },
+  },
+  hooks: {
+    async deploy() {
+      await this.emitEvent(25);
+    },
+  },
+  async run() {
+    await this.emitEvent();
+  },
+};