Skip to content

Commit bbf523c

Browse files
ptrumpisptrumpis
committed
Update crawler code to latest version
1 parent e9d6d72 commit bbf523c

File tree

1 file changed

+133
-59
lines changed

1 file changed

+133
-59
lines changed

‎src/lib/crawler.js‎

Lines changed: 133 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
/*
22
Snap Lens Web Crawler
3-
(c) 2023 by Patrick Trumpis
3+
(c) 2023-2025 by Patrick Trumpis
44
Original code copy from:
55
https://github.com/ptrumpis/snap-lens-web-crawler
66
*/
@@ -9,15 +9,25 @@ import fetch from 'node-fetch';
99

1010
export default class SnapLensWebCrawler {
1111
SCRIPT_SELECTOR = '#__NEXT_DATA__';
12-
constructor() {
12+
constructor(connectionTimeoutMs = 9000, headers = null) {
1313
this.json = {};
14+
this.connectionTimeoutMs = connectionTimeoutMs;
15+
this.headers = headers || {
16+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36'
17+
};
1418
}
1519

16-
async getLensByHash(hash) {
20+
async getLensByHash(hash, rawOutput = false) {
1721
try {
1822
const body = await this._loadUrl('https://lens.snapchat.com/' + hash);
1923
const $ = cheerio.load(body);
2024
const json = JSON.parse($(this.SCRIPT_SELECTOR).text());
25+
26+
// debugging
27+
if (rawOutput) {
28+
return json;
29+
}
30+
2131
if (json && json?.props?.pageProps?.lensDisplayInfo) {
2232
return this._lensInfoToLens(json.props.pageProps.lensDisplayInfo);
2333
}
@@ -27,18 +37,24 @@ export default class SnapLensWebCrawler {
2737
return null;
2838
}
2939

30-
async getLensesByCreator(obfuscatedSlug, offset = 0, limit = 100) {
40+
async getLensesByCreator(obfuscatedSlug, offset = 0, limit = 100, rawOutput = false) {
3141
let lenses = [];
3242
try {
3343
limit = Math.min(100, limit);
3444
const jsonString = await this._loadUrl('https://lensstudio.snapchat.com/v1/creator/lenses/?limit=' + limit + '&offset=' + offset + '&order=1&slug=' + obfuscatedSlug);
3545
if (jsonString) {
3646
const json = JSON.parse(jsonString);
47+
48+
// debugging
49+
if (rawOutput) {
50+
return json;
51+
}
52+
3753
if (json && json.lensesList) {
3854
for (let i = 0; i < json.lensesList.length; i++) {
3955
const item = json.lensesList[i];
4056
if (item.lensId && item.deeplinkUrl && item.name && item.creatorName) {
41-
lenses.push(this._creatorItemToLens(item, obfuscatedSlug));
57+
lenses.push(this._lensItemToLens(item, obfuscatedSlug));
4258
}
4359
}
4460
}
@@ -49,24 +65,31 @@ export default class SnapLensWebCrawler {
4965
return lenses;
5066
}
5167

52-
async searchLenses(search) {
68+
async searchLenses(search, rawOutput = false) {
5369
const slug = search.replace(/\W+/g, '-');
5470
let lenses = [];
5571
try {
5672
const body = await this._loadUrl('https://www.snapchat.com/explore/' + slug);
5773
const $ = cheerio.load(body);
5874
const json = JSON.parse($(this.SCRIPT_SELECTOR).text());
5975

76+
// debugging
77+
if (rawOutput) {
78+
return json;
79+
}
80+
6081
if (json && json?.props?.pageProps?.initialApolloState) {
82+
// original data structure
6183
const results = json.props.pageProps.initialApolloState;
6284
for (const key in results) {
6385
if (key != 'ROOT_QUERY') {
6486
if (results[key].id && results[key].deeplinkUrl && results[key].lensName) {
65-
lenses.push(this._searchItemToLens(results[key]));
87+
lenses.push(this._lensItemToLens(results[key]));
6688
}
6789
}
6890
}
6991
} else if (json && json?.props?.pageProps?.encodedSearchResponse) {
92+
// new data structure introduced in summer 2024
7093
const searchResult = JSON.parse(json.props.pageProps.encodedSearchResponse);
7194
let results = [];
7295
for (const index in searchResult.sections) {
@@ -80,7 +103,7 @@ export default class SnapLensWebCrawler {
80103
if (results[index]?.result?.lens) {
81104
let lens = results[index].result.lens;
82105
if (lens.lensId && lens.deeplinkUrl && lens.name) {
83-
lenses.push(this._searchItemToLens(lens));
106+
lenses.push(this._lensItemToLens(lens));
84107
}
85108
}
86109
}
@@ -91,13 +114,42 @@ export default class SnapLensWebCrawler {
91114
return lenses;
92115
}
93116

94-
async getTopLenses() {
117+
async getUserProfileLenses(userName, rawOutput = false) {
118+
let lenses = [];
119+
try {
120+
const body = await this._loadUrl('https://www.snapchat.com/add/' + userName);
121+
const $ = cheerio.load(body);
122+
const json = JSON.parse($(this.SCRIPT_SELECTOR).text());
123+
124+
// debugging
125+
if (rawOutput) {
126+
return json;
127+
}
128+
129+
if (json && json?.props?.pageProps?.lenses) {
130+
const results = json.props.pageProps.lenses;
131+
for (const index in results) {
132+
lenses.push(this._lensInfoToLens(results[index], userName));
133+
}
134+
}
135+
} catch (e) {
136+
console.error(e);
137+
}
138+
return lenses;
139+
}
140+
141+
async getTopLenses(rawOutput = false) {
95142
let lenses = [];
96143
try {
97144
const body = await this._loadUrl('https://www.snapchat.com/lens');
98145
const $ = cheerio.load(body);
99146
const json = JSON.parse($(this.SCRIPT_SELECTOR).text());
100147

148+
// debugging
149+
if (rawOutput) {
150+
return json;
151+
}
152+
101153
if (json && json?.props?.pageProps?.topLenses) {
102154
const results = json.props.pageProps.topLenses;
103155
for (const index in results) {
@@ -111,80 +163,85 @@ export default class SnapLensWebCrawler {
111163
}
112164

113165
async _loadUrl(url) {
166+
const controller = new AbortController();
167+
const timeout = setTimeout(() => {
168+
controller.abort();
169+
}, this.connectionTimeoutMs);
170+
114171
try {
115-
const response = await fetch(url);
172+
const response = await fetch(url, { signal: controller.signal, headers: this.headers });
173+
if (response.status !== 200) {
174+
console.warn("Unexpected HTTP status", response.status);
175+
}
116176
return await response.text();
117177
} catch (e) {
118-
console.error(e);
178+
console.error('Request failed:', e);
179+
} finally {
180+
clearTimeout(timeout);
119181
}
120-
return null;
121-
}
122182

123-
_creatorItemToLens(item, obfuscatedSlug = '') {
124-
const uuid = this._extractUuidFromDeeplink(item.deeplinkUrl);
125-
return {
126-
unlockable_id: item.lensId,
127-
uuid: uuid,
128-
snapcode_url: item.snapcodeUrl,
129-
user_display_name: item.creatorName,
130-
lens_name: item.name,
131-
lens_tags: "",
132-
lens_status: "Live",
133-
deeplink: item.deeplinkUrl,
134-
icon_url: item.iconUrl,
135-
thumbnail_media_url: item.thumbnailUrl || "",
136-
thumbnail_media_poster_url: item.thumbnailUrl || "",
137-
standard_media_url: item.previewVideoUrl || "",
138-
standard_media_poster_url: "",
139-
obfuscated_user_slug: obfuscatedSlug,
140-
image_sequence: {
141-
url_pattern: item.thumbnailSequence?.urlPattern || "",
142-
size: item.thumbnailSequence?.numThumbnails || 0,
143-
frame_interval_ms: item.thumbnailSequence?.animationIntervalMs || 0
144-
}
145-
};
183+
return null;
146184
}
147185

148-
_searchItemToLens(searchItem) {
149-
const uuid = this._extractUuidFromDeeplink(searchItem.deeplinkUrl);
186+
// creator and search lens formatting
187+
_lensItemToLens(lensItem, obfuscatedSlug = '') {
188+
const uuid = this._extractUuidFromDeeplink(lensItem.deeplinkUrl);
150189
let result = {
151-
unlockable_id: searchItem.id || searchItem.lensId,
190+
unlockable_id: lensItem.id || lensItem.lensId,
152191
uuid: uuid,
153-
snapcode_url: "https://app.snapchat.com/web/deeplink/snapcode?data=" + uuid + "&version=1&type=png",
154-
user_display_name: searchItem.creator?.title || "",
155-
lens_name: searchItem.lensName || searchItem.name ||"",
192+
snapcode_url: lensItem.snapcodeUrl || this._snapcodeUrl(uuid),
193+
194+
lens_name: lensItem.lensName || lensItem.name || "",
195+
lens_creator_search_tags: [],
156196
lens_tags: "",
157197
lens_status: "Live",
158-
deeplink: searchItem.deeplinkUrl || "",
159-
icon_url: searchItem.iconUrl || "",
160-
thumbnail_media_url: searchItem.previewImageUrl || "",
161-
thumbnail_media_poster_url: searchItem.previewImageUrl || "",
162-
standard_media_url: "",
198+
199+
user_display_name: lensItem.creator?.title || lensItem.creatorName || "",
200+
user_name: "",
201+
user_profile_url: "",
202+
user_id: lensItem.creatorUserId || "",
203+
user_profile_id: lensItem.creatorProfileId || "",
204+
205+
deeplink: lensItem.deeplinkUrl || "",
206+
icon_url: lensItem.iconUrl || "",
207+
thumbnail_media_url: lensItem.thumbnailUrl || lensItem.previewImageUrl || "",
208+
thumbnail_media_poster_url: lensItem.thumbnailUrl || lensItem.previewImageUrl || "",
209+
standard_media_url: lensItem.previewVideoUrl || "",
163210
standard_media_poster_url: "",
164-
obfuscated_user_slug: "",
211+
obfuscated_user_slug: obfuscatedSlug || "",
165212
image_sequence: {}
166213
};
167-
if (searchItem.thumbnailSequence) {
214+
215+
if (lensItem.thumbnailSequence) {
168216
result.image_sequence = {
169-
url_pattern: searchItem.thumbnailSequence?.urlPattern || "",
170-
size: searchItem.thumbnailSequence?.numThumbnails || 0,
171-
frame_interval_ms: searchItem.thumbnailSequence?.animationIntervalMs || 0
217+
url_pattern: lensItem.thumbnailSequence?.urlPattern || "",
218+
size: lensItem.thumbnailSequence?.numThumbnails || 0,
219+
frame_interval_ms: lensItem.thumbnailSequence?.animationIntervalMs || 0
172220
}
173221
}
174222
return result;
175223
}
176224

177-
_lensInfoToLens(lensInfo) {
178-
const uuid = lensInfo.scannableUuid || "";
225+
// top lenses, user profile lenses and single lens formatting
226+
_lensInfoToLens(lensInfo, userName = '') {
227+
const uuid = lensInfo.scannableUuid || this._extractUuidFromDeeplink(lensInfo.unlockUrl);
179228
return {
180229
//lens
181230
unlockable_id: lensInfo.lensId,
182231
uuid: uuid,
183-
snapcode_url: "https://app.snapchat.com/web/deeplink/snapcode?data=" + uuid + "&version=1&type=png",
184-
user_display_name: lensInfo.lensCreatorDisplayName || "",
232+
snapcode_url: this._snapcodeUrl(uuid),
233+
185234
lens_name: lensInfo.lensName || "",
235+
lens_creator_search_tags: lensInfo.lensCreatorSearchTags || [],
186236
lens_tags: "",
187237
lens_status: "Live",
238+
239+
user_display_name: lensInfo.lensCreatorDisplayName || "",
240+
user_name: lensInfo.lensCreatorUsername || userName || "",
241+
user_profile_url: lensInfo.userProfileUrl || this._profileUrl(lensInfo.lensCreatorUsername || userName),
242+
user_id: "",
243+
user_profile_id: "",
244+
188245
deeplink: lensInfo.unlockUrl || "",
189246
icon_url: lensInfo.iconUrl || "",
190247
thumbnail_media_url: lensInfo.lensPreviewImageUrl || "",
@@ -193,17 +250,34 @@ export default class SnapLensWebCrawler {
193250
standard_media_poster_url: "",
194251
obfuscated_user_slug: "",
195252
image_sequence: {},
253+
196254
//unlock
197255
lens_id: lensInfo.lensId,
198256
lens_url: lensInfo.lensResource?.archiveLink || "",
199257
signature: lensInfo.lensResource?.signature || "",
258+
sha256: lensInfo.lensResource?.checkSum || "",
200259
hint_id: "",
201-
additional_hint_ids: {}
260+
additional_hint_ids: {},
261+
last_updated: lensInfo.lensResource?.lastUpdated || "",
202262
};
203263
}
204264

265+
_profileUrl(username) {
266+
if (typeof username === 'string' && username) {
267+
return "https://www.snapchat.com/add/" + username;
268+
}
269+
return '';
270+
}
271+
272+
_snapcodeUrl(uuid) {
273+
if (typeof uuid === 'string' && uuid) {
274+
return "https://app.snapchat.com/web/deeplink/snapcode?data=" + uuid + "&version=1&type=png";
275+
}
276+
return '';
277+
}
278+
205279
_extractUuidFromDeeplink(deeplink) {
206-
if (typeof deeplink === "string" && deeplink.startsWith("https://www.snapchat.com/unlock/?")) {
280+
if (typeof deeplink === "string" && deeplink && (deeplink.startsWith("https://www.snapchat.com/unlock/?") || deeplink.startsWith("https://snapchat.com/unlock/?"))) {
207281
let deeplinkURL = new URL(deeplink);
208282
const regexExp = /^[a-f0-9]{32}$/gi;
209283
if (regexExp.test(deeplinkURL.searchParams.get('uuid'))) {
@@ -212,4 +286,4 @@ export default class SnapLensWebCrawler {
212286
}
213287
return '';
214288
}
215-
}
289+
}

0 commit comments

Comments
 (0)