diff --git a/components/scrapecreators/actions/fetch-creator-profile/fetch-creator-profile.mjs b/components/scrapecreators/actions/fetch-creator-profile/fetch-creator-profile.mjs new file mode 100644 index 0000000000000..1b63e33c5c78f --- /dev/null +++ b/components/scrapecreators/actions/fetch-creator-profile/fetch-creator-profile.mjs @@ -0,0 +1,45 @@ +import app from "../../scrapecreators.app.mjs"; + +export default { + key: "scrapecreators-fetch-creator-profile", + name: "Fetch Creator Profile", + description: "Fetch a creator profile based on platform and handle or unique ID. [See the documentation](https://docs.scrapecreators.com/introduction)", + version: "0.0.1", + type: "action", + props: { + app, + platform: { + propDefinition: [ + app, + "platform", + ], + }, + profileId: { + propDefinition: [ + app, + "profileId", + ], + }, + }, + async run({ $ }) { + const summary = `Successfully fetched creator profile for **${this.profileId}**`; + try { + const response = await this.app.fetchCreatorProfile({ + $, + platform: this.platform, + profileId: this.profileId, + }); + + $.export("$summary", summary); + return response; + } catch ({ response }) { + if (response.data?.success === true) { + $.export("$summary", summary); + return { + message: response.data.message, + }; + } + throw new Error(response.data.message); + } + }, +}; diff --git a/components/scrapecreators/actions/search-creators/search-creators.mjs b/components/scrapecreators/actions/search-creators/search-creators.mjs new file mode 100644 index 0000000000000..3d4e08122982a --- /dev/null +++ b/components/scrapecreators/actions/search-creators/search-creators.mjs @@ -0,0 +1,50 @@ +import { SEARCH_PLATFORMS } from "../../common/constants.mjs"; +import app from "../../scrapecreators.app.mjs"; + +export default { + key: "scrapecreators-search-creators", + name: "Search Creators", + description: "Search for creators based on platform and query. [See the documentation](https://docs.scrapecreators.com/api-reference/introduction)", + version: "0.0.1", + type: "action", + props: { + app, + platform: { + propDefinition: [ + app, + "platform", + ], + options: SEARCH_PLATFORMS, + }, + query: { + type: "string", + label: "Query", + description: "The query to search for creators on", + }, + limit: { + type: "integer", + label: "Limit", + description: "The maximum number of creators to return", + optional: true, + }, + }, + async run({ $ }) { + const results = this.app.paginate({ + $, + fn: this.app.searchCreators, + maxResults: this.limit, + platform: this.platform, + params: { + query: this.query, + }, + }); + + const data = []; + for await (const item of results) { + data.push(item); + } + + $.export("$summary", `Successfully searched for **${this.query}**`); + return data; + }, +}; diff --git a/components/scrapecreators/common/constants.mjs b/components/scrapecreators/common/constants.mjs new file mode 100644 index 0000000000000..ed34039d5c40b --- /dev/null +++ b/components/scrapecreators/common/constants.mjs @@ -0,0 +1,42 @@ +export const PATH_PLATFORMS = { + "channel": [ + "youtube", + ], + "user/boards": [ + "pinterest", + ], + "empty": [ + "linktree", + "komi", + "pillar", + "linkbio", + ], +}; + +export const URL_PLATFORMS = [ + "linkedin", + "facebook", + ...PATH_PLATFORMS["empty"], +]; + +export const SEARCH_PLATFORMS = [ + "tiktok", + "threads", +]; + +export const HANDLE_PLATFORMS = [ + "instagram", + "twitter", + "truthsocial", + "bluesky", + "twitch", + "snapchat", + ...SEARCH_PLATFORMS, + ...PATH_PLATFORMS["user/boards"], + ...PATH_PLATFORMS["channel"], +]; + +export const PLATFORMS = [ + ...URL_PLATFORMS, + ...HANDLE_PLATFORMS, +]; diff --git a/components/scrapecreators/common/utils.mjs b/components/scrapecreators/common/utils.mjs new file mode 100644 index 0000000000000..2737fcc619c7a --- /dev/null +++ b/components/scrapecreators/common/utils.mjs @@ -0,0 +1,46 @@ +export function getObjectDiff(obj1, obj2) { + const diff = {}; + + // Check for differences in obj1's properties + for (const key in obj1) { + if (Object.prototype.hasOwnProperty.call(obj1, key)) { + if (!Object.prototype.hasOwnProperty.call(obj2, key)) { + diff[key] = { + oldValue: obj1[key], + newValue: undefined, + status: "deleted", + }; + } else if (typeof obj1[key] === "object" && obj1[key] !== null && + typeof obj2[key] === "object" && obj2[key] !== null) { + const nestedDiff = getObjectDiff(obj1[key], obj2[key]); + if (Object.keys(nestedDiff).length > 0) { + diff[key] = { + status: "modified", + changes: nestedDiff, + }; + } + } else if (obj1[key] !== obj2[key]) { + diff[key] = { + oldValue: obj1[key], + newValue: obj2[key], + status: "modified", + }; + } + } + } + + // Check for properties added in obj2 + for (const key in obj2) { + if (Object.prototype.hasOwnProperty.call(obj2, key)) { + if (!Object.prototype.hasOwnProperty.call(obj1, key)) { + diff[key] = { + oldValue: undefined, + newValue: obj2[key], + status: "added", + }; + } + } + } + + return diff; +} diff --git a/components/scrapecreators/package.json b/components/scrapecreators/package.json index 0ac788418d447..4991b191e56ee 100644 --- a/components/scrapecreators/package.json +++ b/components/scrapecreators/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/scrapecreators", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream ScrapeCreators Components", "main": "scrapecreators.app.mjs", "keywords": [ @@ -11,5 +11,8 @@ "author": "Pipedream (https://pipedream.com/)", "publishConfig": { "access": "public" + }, + "dependencies": { + "@pipedream/platform": "^3.1.0" } -} \ No newline at end of file +} diff --git a/components/scrapecreators/scrapecreators.app.mjs b/components/scrapecreators/scrapecreators.app.mjs index b4ae4ca84cff8..5458915f561d5 100644 --- a/components/scrapecreators/scrapecreators.app.mjs +++ b/components/scrapecreators/scrapecreators.app.mjs @@ -1,11 +1,116 @@ +import { axios } from "@pipedream/platform"; +import { + PATH_PLATFORMS, PLATFORMS, URL_PLATFORMS, +} from "./common/constants.mjs"; + export default { type: "app", app: "scrapecreators", - propDefinitions: {}, + propDefinitions: { + platform: { + type: "string", + label: "Platform", + description: "The platform to search for creators on", + options: PLATFORMS, + }, + profileId: { + type: "string", + label: "Profile", + description: "The handle, URL or unique ID of the creator profile", + }, + }, methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); + _baseUrl() { + return "https://api.scrapecreators.com/v1"; + }, + _headers() { + return { + "x-api-key": `${this.$auth.api_key}`, + }; + }, + _makeRequest({ + $ = this, path, ...opts + }) { + return axios($, { + url: this._baseUrl() + path, + headers: this._headers(), + ...opts, + }); + }, + searchCreators({ + platform, ...opts + }) { + return this._makeRequest({ + path: `/${platform}/search/users`, + ...opts, + }); + }, + fetchCreatorProfile({ + platform, profileId, ...opts + }) { + const params = this.parseParams(platform, profileId); + const path = this.parsePath(platform); + + return this._makeRequest({ + path: `/${platform}${path}`, + params, + ...opts, + }); + }, + parseParams(platform, profileId) { + const params = {}; + const urlPlatforms = URL_PLATFORMS; + + if (urlPlatforms.includes(platform)) { + params.url = profileId; + } else { + params.handle = profileId; + } + return params; + }, + parsePath(platform) { + const path = Object.entries(PATH_PLATFORMS).find(([ + key, + value, + ]) => value.includes(platform) + ? key + : null); + + return path + ? path[0] === "empty" + ? "" + : `/${path[0]}` + : "/profile"; + }, + async *paginate({ + fn, params = {}, platform, maxResults = null, ...opts + }) { + let hasMore = false; + let count = 0; + let newCursor; + + do { + params.cursor = newCursor; + const { + cursor, + users, + } = await fn({ + platform, + params, + ...opts, + }); + for (const d of users) { + yield d; + + if (maxResults && ++count === maxResults) { + return count; + } + } + + newCursor = cursor; + hasMore = users.length; + + } while (hasMore); }, }, }; diff --git a/components/scrapecreators/sources/common/base.mjs b/components/scrapecreators/sources/common/base.mjs new file mode 100644 index 0000000000000..39f66576bcdf8 --- /dev/null +++ b/components/scrapecreators/sources/common/base.mjs @@ -0,0 +1,58 @@ +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; +import scrapecreators from "../../scrapecreators.app.mjs"; + +export default { + props: { + scrapecreators, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + }, + methods: { + _getLastId() { + return this.db.get("lastId") || 0; + }, + _setLastId(lastId) { + this.db.set("lastId", lastId); + }, + async emitEvent(maxResults = false) { + const lastId = this._getLastId(); + const fieldId = this.getFieldId(); + const fn = this.getFunction(); + const { value: response } = await fn(); + + let responseArray = []; + for (const item of response) { + if (item[fieldId] === lastId) break; + responseArray.push(item); + } + + if (responseArray.length) { + if (maxResults && (responseArray.length > maxResults)) { + responseArray.length = maxResults; + } + this._setLastId(responseArray[0][fieldId]); + } + + for (const item of responseArray.reverse()) { + this.$emit(item, { + id: item[fieldId], + summary: this.getSummary(item), + ts: Date.parse(new Date()), + }); + } + }, + }, + hooks: { + async deploy() { + await this.emitEvent(25); + }, + }, + async run() { + await this.emitEvent(); + }, +}; diff --git a/components/scrapecreators/sources/new-profile-update/new-profile-update.mjs b/components/scrapecreators/sources/new-profile-update/new-profile-update.mjs new file mode 100644 index 0000000000000..bfd32a1d67466 --- /dev/null +++ b/components/scrapecreators/sources/new-profile-update/new-profile-update.mjs @@ -0,0 +1,76 @@ +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; +import { getObjectDiff } from "../../common/utils.mjs"; +import app from "../../scrapecreators.app.mjs"; +import sampleEmit from "./test-event.mjs"; + +export default { + key: "scrapecreators-new-profile-update", + name: "New Profile Update", + description: "Emit new event when a new profile is updated. [See the documentation](https://docs.scrapecreators.com/introduction)", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + app, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + platform: { + propDefinition: [ + app, + "platform", + ], + }, + profileId: { + propDefinition: [ + app, + "profileId", + ], + }, + }, + methods: { + _getLastProfile() { + return this.db.get("lastProfile") || {}; + }, + _setLastProfile(lastProfile) { + this.db.set("lastProfile", lastProfile); + }, + async emitEvent() { + const lastProfile = this._getLastProfile(); + const data = await this.app.fetchCreatorProfile({ + platform: this.platform, + profileId: this.profileId, + }); + + const profile = data?.data?.user || data?.data || data; + + const diff = getObjectDiff(lastProfile, profile); + + if (Object.keys(diff).length > 0) { + this._setLastProfile(profile); + this.$emit({ + profile, + diff, + }, { + id: Date.now(), + summary: `New profile update for ${this.profileId}`, + ts: Date.parse(new Date()), + }); + } + + }, + }, + hooks: { + async deploy() { + await this.emitEvent(); + }, + }, + async run() { + await this.emitEvent(); + }, + sampleEmit, +}; diff --git a/components/scrapecreators/sources/new-profile-update/test-event.mjs b/components/scrapecreators/sources/new-profile-update/test-event.mjs new file mode 100644 index 0000000000000..3a3e204aa2b49 --- /dev/null +++ b/components/scrapecreators/sources/new-profile-update/test-event.mjs @@ -0,0 +1,74 @@ +export default { + "profile": { + "channelId": "UC6amticrzGCc5s9xNxMRnpw", + "channel": "http://www.youtube.com/@channel", + "handle": "@channel", + "name": "Channel Name", + "avatar": { + "image": { + "sources": [ + { + "url": "https://yt3.googleusercontent.com/ytc/1zE46aRMzAIdro_lTyMW3QejiA5z_R2HgM28XFV9fQZtTFPVjn=s72-c-k-c0x00ffffff-no-rj", + "width": 72, + "height": 72 + }, + { + "url": "https://yt3.googleusercontent.com/ytc/1zE46aRMzAIdro_lTyMW3QejiA5z_R2HgM28XFV9fQZtTFPVjn=s120-c-k-c0x00ffffff-no-rj", + "width": 120, + "height": 120 + }, + { + "url": "https://yt3.googleusercontent.com/ytc/1zE46aRMzAIdro_lTyMW3QejiA5z_R2HgM28XFV9fQZtTFPVjn=s160-c-k-c0x00ffffff-no-rj", + "width": 160, + "height": 160 + } + ], + "processor": { + "borderImageProcessor": { + "circular": true + } + } + }, + "avatarImageSize": "AVATAR_SIZE_XL", + "loggingDirectives": { + "trackingParams": "C4aDoQ6PCNIhMOEFr5I-jwMVc9s_BB0kTSWa", + "visibility": { + "types": "12" + } + } + }, + "description": "Channel Description", + "subscriberCount": 293, + "subscriberCountText": "293 subscribers", + "videoCountText": "125 videos", + "videoCount": 125, + "viewCountText": "19,353 views", + "viewCount": 19353, + "joinedDateText": "Joined Apr 8, 2025", + "tags": "Channel, Channel Name, Channel Description", + "email": null, + "site_da_barba": "https://channel.com.br", + "facebook_da_barba": "https://facebook.com/channel", + "links": [ + "https://channel.com.br", + "https://facebook.com/channel" + ] + }, + "diff": { + "avatar": { + "status": "modified", + "changes": { + "loggingDirectives": { + "status": "modified", + "changes": { + "trackingParams": { + "oldValue": "CCOhMIoQ6ENIsrXepJP-jwMVXd4_BB0aXTFi", + "newValue": "CCOhMIoQ6ENI4aDFr5P-jwMVc9s_BB0kTSWa", + "status": "modified" + } + } + } + } + } + } +} \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a9ec2f9a787b2..40ab706d80838 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12521,7 +12521,11 @@ importers: specifier: ^1.4.1 version: 1.6.6 - components/scrapecreators: {} + components/scrapecreators: + dependencies: + '@pipedream/platform': + specifier: ^3.1.0 + version: 3.1.0 components/scrapegraphai: dependencies: @@ -30988,22 +30992,22 @@ packages: superagent@3.8.1: resolution: {integrity: sha512-VMBFLYgFuRdfeNQSMLbxGSLfmXL/xc+OO+BZp41Za/NRDBet/BNbkRJrYzCUu0u4GU0i/ml2dtT8b9qgkw9z6Q==} engines: {node: '>= 4.0'} - deprecated: Please upgrade to superagent v10.2.2+, see release notes at https://github.com/forwardemail/superagent/releases/tag/v10.2.2 - maintenance is supported by Forward Email @ https://forwardemail.net + deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net superagent@4.1.0: resolution: {integrity: sha512-FT3QLMasz0YyCd4uIi5HNe+3t/onxMyEho7C3PSqmti3Twgy2rXT4fmkTz6wRL6bTF4uzPcfkUCa8u4JWHw8Ag==} engines: {node: '>= 6.0'} - deprecated: Please upgrade to superagent v10.2.2+, see release notes at https://github.com/forwardemail/superagent/releases/tag/v10.2.2 - maintenance is supported by Forward Email @ https://forwardemail.net + deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net superagent@5.3.1: resolution: {integrity: sha512-wjJ/MoTid2/RuGCOFtlacyGNxN9QLMgcpYLDQlWFIhhdJ93kNscFonGvrpAHSCVjRVj++DGCglocF7Aej1KHvQ==} engines: {node: '>= 7.0.0'} - deprecated: Please upgrade to superagent v10.2.2+, see release notes at https://github.com/forwardemail/superagent/releases/tag/v10.2.2 - maintenance is supported by Forward Email @ https://forwardemail.net + deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net superagent@7.1.6: resolution: {integrity: sha512-gZkVCQR1gy/oUXr+kxJMLDjla434KmSOKbx5iGD30Ql+AkJQ/YlPKECJy2nhqOsHLjGHzoDTXNSjhnvWhzKk7g==} engines: {node: '>=6.4.0 <13 || >=14'} - deprecated: Please upgrade to superagent v10.2.2+, see release notes at https://github.com/forwardemail/superagent/releases/tag/v10.2.2 - maintenance is supported by Forward Email @ https://forwardemail.net + deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net supports-color@10.0.0: resolution: {integrity: sha512-HRVVSbCCMbj7/kdWF9Q+bbckjBHLtHMEoJWlkmYzzdwhYMkjkOwubLM6t7NbWKjgKamGDrWL1++KrjUO1t9oAQ==}