refactor(scraper): fix, optimize, refactor (#207)

* refactor(registrationScraper): general cleanup

- replaced export {} syntax with marking each function as export at
  declaration
- exported all interfaces
- refactored all scrap*() arrow functions to proper functions with
  proper return typing
- added some new interfaces for return types of scrap*() functions
- made all scrap*() functions throw improved errors instead of returning
  undefined

* refactor(scraper): major scraper command refactor

- split the main `run()` function into multiple task functions
- task functions may share data between them using properties of the
  command object
- tasks should batch updates and run them all in a few queries by using
  the `*Many()` method variants of lucid models
- implemented simple async semaphores for ratelimiting
- the number of running parallel fetch and DB tasks is limited and can
  be adjusted using commandline flags
- it actually works now (on my machine)

* fix(scraper): eliminate duplicate rows using a constraint

* refactor(scraper): rewrite the archive task in raw SQL

results in a ~40% speedup in that task (~25s -> ~15s)

* ci: bump node version to 22

i need that Set.difference in my scraper

* fix(migrations): keep the lowest ID duplicate group instead of highest

* fix(scraper): reorder group objects after fetching

today's session of pointless debugging was brought to you by today's
sponsor, adonis!
do you want your code to absolutely explode every time you attempt to do
a bulk SQL action?
do you despise the common-sense assumptions, such as the bulk fetch
function returning items in the same order as in the list you provided?
do you like wasting hours sitting in the debugger, inventing new
debugging techniques, such as setting a conditional breakpoint on
`Math.random() < 0.001`?
then adonis is perfect for you!
rewrite your web project in adonis today!
use promo code `mini_bomba` to get 50% more pointless debugging for your
first rewrite and a free database implosion on your first tests in
production!

* fix(frontend): fix connection to api on dev mode

* feat(scraper): vacuum & analyze tables after scrape

* refactor(scraper): minor code cleanup as suggested in code review

- replace () => {return {...};} with () => ({...})
- remove commented-out code
- move utils to their own files in /app/utils
  - create '#utils/' subpath imports for /app/utils

---------

Co-authored-by: QamarQ <kamilm@you2.pl>

Author: mini-bomba

backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:20.12.2-alpine3.18 AS base
+FROM node:22.14.0-alpine3.21 AS base
 
 # All deps stage
 FROM base AS deps

backend/app/scrap-registrations/scrap_registrations.ts

@@ -1,8 +1,6 @@
 import * as cheerio from "cheerio";
 
-import logger from "@adonisjs/core/services/logger";
-
-interface GroupDetails {
+export interface ScrapedGroupSummary {
   name: string;
   type: string;
   group: string;
@@ -12,27 +10,27 @@ interface GroupDetails {
   endTime: string;
   lecturer: string;
 }
-interface Group {
+export interface ScrapedGroup {
   url: string;
-  groups: GroupDetails[];
+  groups: ScrapedGroupSummary[];
 }
 
-interface Course {
+export interface ScrapedCourse {
   name: string;
   courseCode: string;
   url: string;
-  groups: Group[];
+  groups: ScrapedGroup[];
 }
-interface Registration {
+export interface ScrapedRegistration {
   name: string;
   url: string;
-  courses: Course[];
+  courses: ScrapedCourse[];
 }
 
-interface Department {
+export interface ScrapedDepartment {
   name: string;
   url: string;
-  registrations: Registration[];
+  registrations: ScrapedRegistration[];
 }
 
 const DEPARTMENTS_URL =
@@ -57,12 +55,13 @@ async function fetchData(url: string, options = {}, timeout = 10000) {
   }
 }
 
-const scrapDepartments = async () => {
-  const departments: Department[] = [];
+export async function scrapDepartments(): Promise<ScrapedDepartment[]> {
+  const departments: ScrapedDepartment[] = [];
   const response = await fetchData(DEPARTMENTS_URL);
   if (!response.ok) {
-    logger.info("Something went wrong in fetching departments");
-    return;
+    throw new Error(
+      `Got response code ${response.status} ${response.statusText} while fetching departments`,
+    );
   }
   const body = await response.text();
   const $ = cheerio.load(body);
@@ -71,7 +70,7 @@ const scrapDepartments = async () => {
     .find(".autostrong")
     .children("tr");
   departmentsBlock.each((_, element) => {
-    const newDepartment: Department = {
+    const newDepartment: ScrapedDepartment = {
       name: "",
       url: "",
       registrations: [],
@@ -87,15 +86,18 @@ const scrapDepartments = async () => {
     departments.push(newDepartment);
   });
   return departments;
-};
+}
 
-const scrapRegistrations = async (departmentUrl: string) => {
+export async function scrapRegistrations(
+  departmentUrl: string,
+): Promise<ScrapedRegistration[]> {
   const registrationsNames: string[] = [];
   const registrationsUrls: string[] = [];
   const response = await fetchData(departmentUrl);
   if (!response.ok) {
-    logger.info("Something went wrong in fetching registrations");
-    return;
+    throw new Error(
+      `Got reponse code ${response.status} ${response.statusText} while fetching registrations`,
+    );
   }
   const body = await response.text();
   const $ = cheerio.load(body);
@@ -115,16 +117,21 @@ const scrapRegistrations = async (departmentUrl: string) => {
     }
   });
   return registrationsNames.map((name, index) => {
-    return { name, url: registrationsUrls[index], courses: [] as Course[] };
+    return {
+      name,
+      url: registrationsUrls[index],
+      courses: [],
+    };
   });
-};
+}
 
-const scrapCourses = async (registrationUrl: string) => {
+export async function scrapCourses(registrationUrl: string): Promise<string[]> {
   const coursesUrls: string[] = [];
   const response = await fetchData(registrationUrl);
   if (!response.ok) {
-    logger.info("Something went wrong in fetching courses");
-    return;
+    throw new Error(
+      `Got response code ${response.status} ${response.statusText} while fetching courses`,
+    );
   }
 
   const body = await response.text();
@@ -142,16 +149,25 @@ const scrapCourses = async (registrationUrl: string) => {
     }
   });
   return coursesUrls;
-};
+}
+
+export interface ScrapedCourseDetails {
+  courseName: string;
+  courseCode: string;
+  urls: string[];
+}
 
-const scrapCourseNameGroupsUrls = async (courseUrl: string) => {
+export async function scrapCourseNameGroupsUrls(
+  courseUrl: string,
+): Promise<ScrapedCourseDetails> {
   let courseName = "";
   const urls: string[] = [];
   let courseCode = "";
   const response = await fetchData(courseUrl);
   if (!response.ok) {
-    logger.info("Something went wrong in fetching groups");
-    return;
+    throw new Error(
+      `Got response code ${response.status} ${response.statusText} while fetching course details`,
+    );
   }
 
   const body = await response.text();
@@ -186,14 +202,15 @@ const scrapCourseNameGroupsUrls = async (courseUrl: string) => {
     }
   });
   return { courseName, urls, courseCode };
-};
+}
 
-const scrapGroupsUrls = async (groupUrl: string) => {
+export async function scrapGroupsUrls(groupUrl: string): Promise<string[]> {
   const groupsUrls: string[] = [];
   const response = await fetchData(groupUrl);
   if (!response.ok) {
-    logger.info("Something went wrong in fetching groups");
-    return;
+    throw new Error(
+      `Got response code ${response.status} ${response.statusText} while fetching group URLs`,
+    );
   }
 
   const body = await response.text();
@@ -207,13 +224,28 @@ const scrapGroupsUrls = async (groupUrl: string) => {
     }
   });
   return groupsUrls;
-};
+}
 
-const scrapGroupDetails = async (groupUrl: string) => {
+export interface ScrapedGroupDetails {
+  name: string;
+  type: string;
+  group: string;
+  week: string;
+  days: string[];
+  startTimeEndTimes: { startTime: string; endTime: string }[];
+  lecturer: string;
+  spotsOccupied: number;
+  spotsTotal: number;
+}
+
+export async function scrapGroupDetails(
+  groupUrl: string,
+): Promise<ScrapedGroupDetails> {
   const response = await fetchData(groupUrl);
   if (!response.ok) {
-    logger.info("Something went wrong in fetching groups");
-    return;
+    throw new Error(
+      `Got response code ${response.status} ${response.statusText} while fetching group details`,
+    );
   }
 
   const body = await response.text();
@@ -296,7 +328,7 @@ const scrapGroupDetails = async (groupUrl: string) => {
     spotsOccupied: Number.isNaN(spotsOccupiedNumber) ? 0 : spotsOccupiedNumber,
     spotsTotal: Number.isNaN(spotsTotalNumber) ? 0 : spotsTotalNumber,
   };
-};
+}
 
 const getStartEndTime = (time: string) => {
   if (time.includes("brak danych")) {
@@ -365,12 +397,3 @@ const checkDay = (day: string) => {
   }
   return "unknown";
 };
-
-export {
-  scrapDepartments,
-  scrapRegistrations,
-  scrapCourses,
-  scrapCourseNameGroupsUrls,
-  scrapGroupsUrls,
-  scrapGroupDetails,
-};

backend/app/utils/arrays.ts

@@ -0,0 +1,20 @@
+export function chunkArray<T>(array: T[], chunkSize: number): T[][] {
+  const result = [];
+  const input = Array.from(array);
+  while (input.length > 0) {
+    result.push(input.splice(0, chunkSize));
+  }
+  return result;
+}
+
+export function zip<T1, T2>(a1: T1[], a2: T2[]): [T1, T2][] {
+  const array1 = Array.from(a1);
+  const array2 = Array.from(a2);
+  const result: [T1, T2][] = [];
+  while (array1.length > 0 && array2.length > 0) {
+    const el1 = array1.shift() as T1;
+    const el2 = array2.shift() as T2;
+    result.push([el1, el2]);
+  }
+  return result;
+}

backend/app/utils/semaphore.ts

@@ -0,0 +1,49 @@
+export class Semaphore {
+  capacity: number;
+  #currentTasks: number;
+  #waitingTasks: (() => void)[];
+
+  constructor(capacity: number) {
+    this.capacity = capacity;
+    this.#currentTasks = 0;
+    this.#waitingTasks = [];
+  }
+
+  public get currentTasks(): number {
+    return this.#currentTasks;
+  }
+
+  public async runTask<T>(task: () => Promise<T>): Promise<T> {
+    // acquire the semaphore
+    await this.acquire();
+    try {
+      // execute the task
+      return await task();
+    } finally {
+      // don't forget to release
+      this.release();
+    }
+  }
+
+  private acquire(): Promise<void> {
+    // if we're under capacity, bump the count and resolve immediately
+    if (this.capacity > this.#currentTasks) {
+      this.#currentTasks += 1;
+      return Promise.resolve();
+    }
+    // otherwise add ourselves to the queue
+    return new Promise((resolve) => this.#waitingTasks.push(resolve));
+  }
+
+  private release() {
+    // try waking up the next task
+    const nextTask = this.#waitingTasks.shift();
+    if (nextTask === undefined) {
+      // no task in queue, decrement task count
+      this.#currentTasks -= 1;
+    } else {
+      // wake up the task
+      nextTask();
+    }
+  }
+}