Skip to content

Commit 74caeee

Browse files
luancazarineluancazarine
committed
Update ScrapeCreators component: bump version to 0.1.0, add new actions for fetching creator profiles and searching creators, and introduce constants and utility functions for improved functionality.
1 parent 1f8bd63 commit 74caeee

File tree

9 files changed

+505
-6
lines changed

9 files changed

+505
-6
lines changed

‎components/scrapecreators/actions/fetch-creator-profile/fetch-creator-profile.mjs‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
import app from "../../scrapecreators.app.mjs";
2+
3+
export default {
4+
key: "scrapecreators-fetch-creator-profile",
5+
name: "Fetch Creator Profile",
6+
description: "Fetch a creator profile based on platform and handle or unique ID. [See the documentation](https://docs.scrapecreators.com/introduction)",
7+
version: "0.0.1",
8+
type: "action",
9+
props: {
10+
app,
11+
platform: {
12+
propDefinition: [
13+
app,
14+
"platform",
15+
],
16+
},
17+
profileId: {
18+
propDefinition: [
19+
app,
20+
"profileId",
21+
],
22+
},
23+
},
24+
async run({ $ }) {
25+
const summary = `Successfully fetched creator profile for **${this.profileId}**`;
26+
try {
27+
const response = await this.app.fetchCreatorProfile({
28+
$,
29+
platform: this.platform,
30+
profileId: this.profileId,
31+
});
32+
33+
$.export("$summary", summary);
34+
return response;
35+
} catch ({ response }) {
36+
if (response.data?.success === true) {
37+
$.export("$summary", summary);
38+
return {
39+
message: response.data.message,
40+
};
41+
}
42+
throw new Error(response.data.message);
43+
}
44+
},
45+
};

‎components/scrapecreators/actions/search-creators/search-creators.mjs‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
import { SEARCH_PLATFORMS } from "../../common/constants.mjs";
2+
import app from "../../scrapecreators.app.mjs";
3+
4+
export default {
5+
key: "scrapecreators-search-creators",
6+
name: "Search Creators",
7+
description: "Search for creators based on platformand query. [See the documentation](https://docs.scrapecreators.com/api-reference/introduction)",
8+
version: "0.0.1",
9+
type: "action",
10+
props: {
11+
app,
12+
platform: {
13+
propDefinition: [
14+
app,
15+
"platform",
16+
],
17+
options: SEARCH_PLATFORMS,
18+
},
19+
query: {
20+
type: "string",
21+
label: "Query",
22+
description: "The query to search for creators on",
23+
},
24+
limit: {
25+
type: "integer",
26+
label: "Limit",
27+
description: "The maximum number of creators to return",
28+
optional: true,
29+
},
30+
},
31+
async run({ $ }) {
32+
const results = this.app.paginate({
33+
$,
34+
fn: this.app.searchCreators,
35+
maxResults: this.limit,
36+
platform: this.platform,
37+
params: {
38+
query: this.query,
39+
},
40+
});
41+
42+
const data = [];
43+
for await (const item of results) {
44+
data.push(item);
45+
}
46+
47+
$.export("$summary", `Successfully searched for **${this.query}**`);
48+
return data;
49+
},
50+
};

‎components/scrapecreators/common/constants.mjs‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
export const PATH_PLATFORMS = {
2+
"channel": [
3+
"youtube",
4+
],
5+
"user/boards": [
6+
"pinterest",
7+
],
8+
"empty": [
9+
"linktree",
10+
"komi",
11+
"pillar",
12+
"linkbio",
13+
],
14+
};
15+
16+
export const URL_PLATFORMS = [
17+
"linkedin",
18+
"facebook",
19+
...PATH_PLATFORMS["empty"],
20+
];
21+
22+
export const SEARCH_PLATFORMS = [
23+
"tiktok",
24+
"threads",
25+
];
26+
27+
export const HANDLE_PLATFORMS = [
28+
"instagram",
29+
"twitter",
30+
"truthsocial",
31+
"bluesky",
32+
"twitch",
33+
"snapchat",
34+
...SEARCH_PLATFORMS,
35+
...PATH_PLATFORMS["user/boards"],
36+
...PATH_PLATFORMS["channel"],
37+
];
38+
39+
export const PLATFORMS = [
40+
...URL_PLATFORMS,
41+
...HANDLE_PLATFORMS,
42+
];

‎components/scrapecreators/common/utils.mjs‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
export function getObjectDiff(obj1, obj2) {
2+
const diff = {};
3+
4+
// Check for differences in obj1's properties
5+
for (const key in obj1) {
6+
if (Object.prototype.hasOwnProperty.call(obj1, key)) {
7+
if (!Object.prototype.hasOwnProperty.call(obj2, key)) {
8+
diff[key] = {
9+
oldValue: obj1[key],
10+
newValue: undefined,
11+
status: "deleted",
12+
};
13+
} else if (typeof obj1[key] === "object" && obj1[key] !== null &&
14+
typeof obj2[key] === "object" && obj2[key] !== null) {
15+
const nestedDiff = getObjectDiff(obj1[key], obj2[key]);
16+
if (Object.keys(nestedDiff).length > 0) {
17+
diff[key] = {
18+
status: "modified",
19+
changes: nestedDiff,
20+
};
21+
}
22+
} else if (obj1[key] !== obj2[key]) {
23+
diff[key] = {
24+
oldValue: obj1[key],
25+
newValue: obj2[key],
26+
status: "modified",
27+
};
28+
}
29+
}
30+
}
31+
32+
// Check for properties added in obj2
33+
for (const key in obj2) {
34+
if (Object.prototype.hasOwnProperty.call(obj2, key)) {
35+
if (!Object.prototype.hasOwnProperty.call(obj1, key)) {
36+
diff[key] = {
37+
oldValue: undefined,
38+
newValue: obj2[key],
39+
status: "added",
40+
};
41+
}
42+
}
43+
}
44+
45+
return diff;
46+
}

‎components/scrapecreators/package.json‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@pipedream/scrapecreators",
3-
"version": "0.0.1",
3+
"version": "0.1.0",
44
"description": "Pipedream ScrapeCreators Components",
55
"main": "scrapecreators.app.mjs",
66
"keywords": [
@@ -11,5 +11,8 @@
1111
"author": "Pipedream <[email protected]> (https://pipedream.com/)",
1212
"publishConfig": {
1313
"access": "public"
14+
},
15+
"dependencies": {
16+
"@pipedream/platform": "^3.1.0"
1417
}
15-
}
18+
}

‎components/scrapecreators/scrapecreators.app.mjs‎

Lines changed: 109 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,116 @@
1+
import { axios } from "@pipedream/platform";
2+
import {
3+
PATH_PLATFORMS, PLATFORMS, URL_PLATFORMS,
4+
} from "./common/constants.mjs";
5+
16
export default {
27
type: "app",
38
app: "scrapecreators",
4-
propDefinitions: {},
9+
propDefinitions: {
10+
platform: {
11+
type: "string",
12+
label: "Platform",
13+
description: "The platform to search for creators on",
14+
options: PLATFORMS,
15+
},
16+
profileId: {
17+
type: "string",
18+
label: "Profile",
19+
description: "The handle, URL or unique ID of the creator profile",
20+
},
21+
},
522
methods: {
6-
// this.$auth contains connected account data
7-
authKeys() {
8-
console.log(Object.keys(this.$auth));
23+
_baseUrl() {
24+
return "https://api.scrapecreators.com/v1";
25+
},
26+
_headers() {
27+
return {
28+
"x-api-key": `${this.$auth.api_key}`,
29+
};
30+
},
31+
_makeRequest({
32+
$ = this, path, ...opts
33+
}) {
34+
return axios($, {
35+
url: this._baseUrl() + path,
36+
headers: this._headers(),
37+
...opts,
38+
});
39+
},
40+
searchCreators({
41+
platform, ...opts
42+
}) {
43+
return this._makeRequest({
44+
path: `/${platform}/search/users`,
45+
...opts,
46+
});
47+
},
48+
fetchCreatorProfile({
49+
platform, profileId, ...opts
50+
}) {
51+
const params = this.parseParams(platform, profileId);
52+
const path = this.parsePath(platform);
53+
54+
return this._makeRequest({
55+
path: `/${platform}${path}`,
56+
params,
57+
...opts,
58+
});
59+
},
60+
parseParams(platform, profileId) {
61+
const params = {};
62+
const urlPlatforms = URL_PLATFORMS;
63+
64+
if (urlPlatforms.includes(platform)) {
65+
params.url = profileId;
66+
} else {
67+
params.handle = profileId;
68+
}
69+
return params;
70+
},
71+
parsePath(platform) {
72+
const path = Object.entries(PATH_PLATFORMS).find(([
73+
key,
74+
value,
75+
]) => value.includes(platform)
76+
? key
77+
: null);
78+
79+
return path
80+
? path[0] === "empty"
81+
? ""
82+
: `/${path[0]}`
83+
: "/profile";
84+
},
85+
async *paginate({
86+
fn, params = {}, platform, maxResults = null, ...opts
87+
}) {
88+
let hasMore = false;
89+
let count = 0;
90+
let newCursor;
91+
92+
do {
93+
params.cursor = newCursor;
94+
const {
95+
cursor,
96+
users,
97+
} = await fn({
98+
platform,
99+
params,
100+
...opts,
101+
});
102+
for (const d of users) {
103+
yield d;
104+
105+
if (maxResults && ++count === maxResults) {
106+
return count;
107+
}
108+
}
109+
110+
newCursor = cursor;
111+
hasMore = users.length;
112+
113+
} while (hasMore);
9114
},
10115
},
11116
};

‎components/scrapecreators/sources/common/base.mjs‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
2+
import scrapecreators from "../../scrapecreators.app.mjs";
3+
4+
export default {
5+
props: {
6+
scrapecreators,
7+
db: "$.service.db",
8+
timer: {
9+
type: "$.interface.timer",
10+
default: {
11+
intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
12+
},
13+
},
14+
},
15+
methods: {
16+
_getLastId() {
17+
return this.db.get("lastId") || 0;
18+
},
19+
_setLastId(lastId) {
20+
this.db.set("lastId", lastId);
21+
},
22+
async emitEvent(maxResults = false) {
23+
const lastId = this._getLastId();
24+
const fieldId = this.getFieldId();
25+
const fn = this.getFunction();
26+
const { value: response } = await fn();
27+
28+
let responseArray = [];
29+
for (const item of response) {
30+
if (item[fieldId] === lastId) break;
31+
responseArray.push(item);
32+
}
33+
34+
if (responseArray.length) {
35+
if (maxResults && (responseArray.length > maxResults)) {
36+
responseArray.length = maxResults;
37+
}
38+
this._setLastId(responseArray[0][fieldId]);
39+
}
40+
41+
for (const item of responseArray.reverse()) {
42+
this.$emit(item, {
43+
id: item[fieldId],
44+
summary: this.getSummary(item),
45+
ts: Date.parse(new Date()),
46+
});
47+
}
48+
},
49+
},
50+
hooks: {
51+
async deploy() {
52+
await this.emitEvent(25);
53+
},
54+
},
55+
async run() {
56+
await this.emitEvent();
57+
},
58+
};

0 commit comments

Comments
 (0)