@@ -58,7 +58,7 @@ const firstPage = await browser.newPage();
58
58
await firstPage .goto (REPOSITORIES_URL );
59
59
60
60
const lastPageElement = firstPage .locator (' a[aria-label*="Page "]:nth-last-child(2)' );
61
- const lastPageLabel = await lastPageElement .getAttribute (' aria-label' )
61
+ const lastPageLabel = await lastPageElement .getAttribute (' aria-label' );
62
62
const lastPageNumber = Number (lastPageLabel .replace (/ \D / g , ' ' ));
63
63
console .log (lastPageNumber);
64
64
@@ -81,7 +81,7 @@ await firstPage.goto(REPOSITORIES_URL);
81
81
82
82
const lastPageLabel = await firstPage .$eval (
83
83
' a[aria-label*="Page "]:nth-last-child(2)' ,
84
- (element ) => element .getAttribute (' aria-label' )
84
+ (element ) => element .getAttribute (' aria-label' ),
85
85
);
86
86
const lastPageNumber = Number (lastPageLabel .replace (/ \D / g , ' ' ));
87
87
console .log (lastPageNumber);
@@ -132,7 +132,7 @@ const firstPage = await browser.newPage();
132
132
await firstPage .goto (REPOSITORIES_URL );
133
133
134
134
const lastPageElement = firstPage .locator (' a[aria-label*="Page "]:nth-last-child(2)' );
135
- const lastPageLabel = await lastPageElement .getAttribute (' aria-label' )
135
+ const lastPageLabel = await lastPageElement .getAttribute (' aria-label' );
136
136
const lastPageNumber = Number (lastPageLabel .replace (/ \D / g , ' ' ));
137
137
138
138
// Push all results from the first page to the repositories array
@@ -175,7 +175,7 @@ await firstPage.goto(REPOSITORIES_URL);
175
175
176
176
const lastPageLabel = await firstPage .$eval (
177
177
' a[aria-label*="Page "]:nth-last-child(2)' ,
178
- (element ) => element .getAttribute (' aria-label' )
178
+ (element ) => element .getAttribute (' aria-label' ),
179
179
);
180
180
const lastPageNumber = Number (lastPageLabel .replace (/ \D / g , ' ' ));
181
181
@@ -217,14 +217,14 @@ If we click around the pagination links, we can observe that all the URLs follow
217
217
That means we could construct URL for each page if we had an array of numbers with the same range as the pages. If ` lastPageNumber ` is ` 4 ` , the following code creates ` [0, 1, 2, 3, 4] ` :
218
218
219
219
``` js
220
- const array = Array (lastPageNumber + 1 ) // getting an array of certain size
221
- const numbers = [... array .keys ()]; // getting the keys (the actual numbers) as another array
220
+ const array = Array (lastPageNumber + 1 ); // getting an array of certain size
221
+ const numbers = [... array .keys ()]; // getting the keys (the actual numbers) as another array
222
222
```
223
223
224
224
Page ` 0 ` doesn't exist though and we've already scraped page ` 1 ` , so we need one more step to remove those:
225
225
226
226
``` js
227
- const pageNumbers = numbers .slice (2 ); // removes the first two numbers
227
+ const pageNumbers = numbers .slice (2 ); // removes the first two numbers
228
228
```
229
229
230
230
To have our code examples shorter, we'll squash the above to a single line of code:
@@ -237,23 +237,22 @@ Now let's scrape repositories for each of these numbers. We'll create promises f
237
237
238
238
``` js
239
239
const pageNumbers = [... Array (lastPageNumber + 1 ).keys ()].slice (2 );
240
- const promises = pageNumbers .map ((pageNumber ) =>
241
- (async () => {
242
- const paginatedPage = await browser .newPage ();
240
+ const promises = pageNumbers .map ((pageNumber ) => (async () => {
241
+ const paginatedPage = await browser .newPage ();
243
242
244
- // Construct the URL by setting the ?page=... parameter to value of pageNumber
245
- const url = new URL (REPOSITORIES_URL );
246
- url .searchParams .set (' page' , pageNumber);
243
+ // Construct the URL by setting the ?page=... parameter to value of pageNumber
244
+ const url = new URL (REPOSITORIES_URL );
245
+ url .searchParams .set (' page' , pageNumber);
247
246
248
- // Scrape the page
249
- await paginatedPage .goto (url .href );
250
- const results = await scrapeRepos (paginatedPage)
247
+ // Scrape the page
248
+ await paginatedPage .goto (url .href );
249
+ const results = await scrapeRepos (paginatedPage);
251
250
252
- // Push results to the repositories array
253
- repositories .push (... results);
251
+ // Push results to the repositories array
252
+ repositories .push (... results);
254
253
255
- await paginatedPage .close ();
256
- })()
254
+ await paginatedPage .close ();
255
+ })(),
257
256
);
258
257
await Promise .all (promises);
259
258
@@ -302,7 +301,7 @@ const firstPage = await browser.newPage();
302
301
await firstPage .goto (REPOSITORIES_URL );
303
302
304
303
const lastPageElement = firstPage .locator (' a[aria-label*="Page "]:nth-last-child(2)' );
305
- const lastPageLabel = await lastPageElement .getAttribute (' aria-label' )
304
+ const lastPageLabel = await lastPageElement .getAttribute (' aria-label' );
306
305
const lastPageNumber = Number (lastPageLabel .replace (/ \D / g , ' ' ));
307
306
308
307
// Push all results from the first page to the repositories array
@@ -311,23 +310,22 @@ repositories.push(...(await scrapeRepos(firstPage)));
311
310
await firstPage .close ();
312
311
313
312
const pageNumbers = [... Array (lastPageNumber + 1 ).keys ()].slice (2 );
314
- const promises = pageNumbers .map ((pageNumber ) =>
315
- (async () => {
316
- const paginatedPage = await browser .newPage ();
313
+ const promises = pageNumbers .map ((pageNumber ) => (async () => {
314
+ const paginatedPage = await browser .newPage ();
317
315
318
- // Construct the URL by setting the ?page=... parameter to value of pageNumber
319
- const url = new URL (REPOSITORIES_URL );
320
- url .searchParams .set (' page' , pageNumber);
316
+ // Construct the URL by setting the ?page=... parameter to value of pageNumber
317
+ const url = new URL (REPOSITORIES_URL );
318
+ url .searchParams .set (' page' , pageNumber);
321
319
322
- // Scrape the page
323
- await paginatedPage .goto (url .href );
324
- const results = await scrapeRepos (paginatedPage)
320
+ // Scrape the page
321
+ await paginatedPage .goto (url .href );
322
+ const results = await scrapeRepos (paginatedPage);
325
323
326
- // Push results to the repositories array
327
- repositories .push (... results);
324
+ // Push results to the repositories array
325
+ repositories .push (... results);
328
326
329
- await paginatedPage .close ();
330
- })()
327
+ await paginatedPage .close ();
328
+ })(),
331
329
);
332
330
await Promise .all (promises);
333
331
@@ -369,7 +367,7 @@ await firstPage.goto(REPOSITORIES_URL);
369
367
370
368
const lastPageLabel = await firstPage .$eval (
371
369
' a[aria-label*="Page "]:nth-last-child(2)' ,
372
- (element ) => element .getAttribute (' aria-label' )
370
+ (element ) => element .getAttribute (' aria-label' ),
373
371
);
374
372
const lastPageNumber = Number (lastPageLabel .replace (/ \D / g , ' ' ));
375
373
@@ -379,23 +377,22 @@ repositories.push(...(await scrapeRepos(page)));
379
377
await firstPage .close ();
380
378
381
379
const pageNumbers = [... Array (lastPageNumber + 1 ).keys ()].slice (2 );
382
- const promises = pageNumbers .map ((pageNumber ) =>
383
- (async () => {
384
- const paginatedPage = await browser .newPage ();
380
+ const promises = pageNumbers .map ((pageNumber ) => (async () => {
381
+ const paginatedPage = await browser .newPage ();
385
382
386
- // Construct the URL by setting the ?page=... parameter to value of pageNumber
387
- const url = new URL (REPOSITORIES_URL );
388
- url .searchParams .set (' page' , pageNumber);
383
+ // Construct the URL by setting the ?page=... parameter to value of pageNumber
384
+ const url = new URL (REPOSITORIES_URL );
385
+ url .searchParams .set (' page' , pageNumber);
389
386
390
- // Scrape the page
391
- await paginatedPage .goto (url .href );
392
- const results = await scrapeRepos (paginatedPage)
387
+ // Scrape the page
388
+ await paginatedPage .goto (url .href );
389
+ const results = await scrapeRepos (paginatedPage);
393
390
394
- // Push results to the repositories array
395
- repositories .push (... results);
391
+ // Push results to the repositories array
392
+ repositories .push (... results);
396
393
397
- await paginatedPage .close ();
398
- })()
394
+ await paginatedPage .close ();
395
+ })(),
399
396
);
400
397
await Promise .all (promises);
401
398
0 commit comments