refactor(scraper): fix, optimize, refactor

backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:20.12.2-alpine3.18 AS base
+FROM node:22.14.0-alpine3.21 AS base
 
 # All deps stage
 FROM base AS deps

backend/app/scrap-registrations/scrap_registrations.ts

@@ -1,8 +1,6 @@
 import * as cheerio from "cheerio";
 
-import logger from "@adonisjs/core/services/logger";
-
-interface GroupDetails {
+export interface ScrapedGroupSummary {
   name: string;
   type: string;
   group: string;
@@ -12,27 +10,27 @@ interface GroupDetails {
   endTime: string;
   lecturer: string;
 }
-interface Group {
+export interface ScrapedGroup {
   url: string;
-  groups: GroupDetails[];
+  groups: ScrapedGroupSummary[];
 }
 
-interface Course {
+export interface ScrapedCourse {
   name: string;
   courseCode: string;
   url: string;
-  groups: Group[];
+  groups: ScrapedGroup[];
 }
-interface Registration {
+export interface ScrapedRegistration {
   name: string;
   url: string;
-  courses: Course[];
+  courses: ScrapedCourse[];
 }
 
-interface Department {
+export interface ScrapedDepartment {
   name: string;
   url: string;
-  registrations: Registration[];
+  registrations: ScrapedRegistration[];
 }
 
 const DEPARTMENTS_URL =
@@ -57,12 +55,13 @@ async function fetchData(url: string, options = {}, timeout = 10000) {
   }
 }
 
-const scrapDepartments = async () => {
-  const departments: Department[] = [];
+export async function scrapDepartments(): Promise<ScrapedDepartment[]> {
+  const departments: ScrapedDepartment[] = [];
   const response = await fetchData(DEPARTMENTS_URL);
   if (!response.ok) {
-    logger.info("Something went wrong in fetching departments");
-    return;
+    throw new Error(
+      `Got response code ${response.status} ${response.statusText} while fetching departments`,
+    );
   }
   const body = await response.text();
   const $ = cheerio.load(body);
@@ -71,7 +70,7 @@ const scrapDepartments = async () => {
     .find(".autostrong")
     .children("tr");
   departmentsBlock.each((_, element) => {
-    const newDepartment: Department = {
+    const newDepartment: ScrapedDepartment = {
       name: "",
       url: "",
       registrations: [],
@@ -87,15 +86,18 @@ const scrapDepartments = async () => {
     departments.push(newDepartment);
   });
   return departments;
-};
+}
 
-const scrapRegistrations = async (departmentUrl: string) => {
+export async function scrapRegistrations(
+  departmentUrl: string,
+): Promise<ScrapedRegistration[]> {
   const registrationsNames: string[] = [];
   const registrationsUrls: string[] = [];
   const response = await fetchData(departmentUrl);
   if (!response.ok) {
-    logger.info("Something went wrong in fetching registrations");
-    return;
+    throw new Error(
+      `Got reponse code ${response.status} ${response.statusText} while fetching registrations`,
+    );
   }
   const body = await response.text();
   const $ = cheerio.load(body);
@@ -115,16 +117,21 @@ const scrapRegistrations = async (departmentUrl: string) => {
     }
   });
   return registrationsNames.map((name, index) => {
-    return { name, url: registrationsUrls[index], courses: [] as Course[] };
+    return {
+      name,
+      url: registrationsUrls[index],
+      courses: [],
+    };
   });
-};
+}
 
-const scrapCourses = async (registrationUrl: string) => {
+export async function scrapCourses(registrationUrl: string): Promise<string[]> {
   const coursesUrls: string[] = [];
   const response = await fetchData(registrationUrl);
   if (!response.ok) {
-    logger.info("Something went wrong in fetching courses");
-    return;
+    throw new Error(
+      `Got response code ${response.status} ${response.statusText} while fetching courses`,
+    );
   }
 
   const body = await response.text();
@@ -142,16 +149,25 @@ const scrapCourses = async (registrationUrl: string) => {
     }
   });
   return coursesUrls;
-};
+}
+
+export interface ScrapedCourseDetails {
+  courseName: string;
+  courseCode: string;
+  urls: string[];
+}
 
-const scrapCourseNameGroupsUrls = async (courseUrl: string) => {
+export async function scrapCourseNameGroupsUrls(
+  courseUrl: string,
+): Promise<ScrapedCourseDetails> {
   let courseName = "";
   const urls: string[] = [];
   let courseCode = "";
   const response = await fetchData(courseUrl);
   if (!response.ok) {
-    logger.info("Something went wrong in fetching groups");
-    return;
+    throw new Error(
+      `Got response code ${response.status} ${response.statusText} while fetching course details`,
+    );
   }
 
   const body = await response.text();
@@ -186,14 +202,15 @@ const scrapCourseNameGroupsUrls = async (courseUrl: string) => {
     }
   });
   return { courseName, urls, courseCode };
-};
+}
 
-const scrapGroupsUrls = async (groupUrl: string) => {
+export async function scrapGroupsUrls(groupUrl: string): Promise<string[]> {
   const groupsUrls: string[] = [];
   const response = await fetchData(groupUrl);
   if (!response.ok) {
-    logger.info("Something went wrong in fetching groups");
-    return;
+    throw new Error(
+      `Got response code ${response.status} ${response.statusText} while fetching group URLs`,
+    );
   }
 
   const body = await response.text();
@@ -207,13 +224,28 @@ const scrapGroupsUrls = async (groupUrl: string) => {
     }
   });
   return groupsUrls;
-};
+}
 
-const scrapGroupDetails = async (groupUrl: string) => {
+export interface ScrapedGroupDetails {
+  name: string;
+  type: string;
+  group: string;
+  week: string;
+  days: string[];
+  startTimeEndTimes: { startTime: string; endTime: string }[];
+  lecturer: string;
+  spotsOccupied: number;
+  spotsTotal: number;
+}
+
+export async function scrapGroupDetails(
+  groupUrl: string,
+): Promise<ScrapedGroupDetails> {
   const response = await fetchData(groupUrl);
   if (!response.ok) {
-    logger.info("Something went wrong in fetching groups");
-    return;
+    throw new Error(
+      `Got response code ${response.status} ${response.statusText} while fetching group details`,
+    );
   }
 
   const body = await response.text();
@@ -296,7 +328,7 @@ const scrapGroupDetails = async (groupUrl: string) => {
     spotsOccupied: Number.isNaN(spotsOccupiedNumber) ? 0 : spotsOccupiedNumber,
     spotsTotal: Number.isNaN(spotsTotalNumber) ? 0 : spotsTotalNumber,
   };
-};
+}
 
 const getStartEndTime = (time: string) => {
   if (time.includes("brak danych")) {
@@ -365,12 +397,3 @@ const checkDay = (day: string) => {
   }
   return "unknown";
 };
-
-export {
-  scrapDepartments,
-  scrapRegistrations,
-  scrapCourses,
-  scrapCourseNameGroupsUrls,
-  scrapGroupsUrls,
-  scrapGroupDetails,
-};

backend/app/utils/arrays.ts

@@ -0,0 +1,20 @@
+export function chunkArray<T>(array: T[], chunkSize: number): T[][] {
+  const result = [];
+  const input = Array.from(array);
+  while (input.length > 0) {
+    result.push(input.splice(0, chunkSize));
+  }
+  return result;
+}
+
+export function zip<T1, T2>(a1: T1[], a2: T2[]): [T1, T2][] {
+  const array1 = Array.from(a1);
+  const array2 = Array.from(a2);
+  const result: [T1, T2][] = [];
+  while (array1.length > 0 && array2.length > 0) {
+    const el1 = array1.shift() as T1;
+    const el2 = array2.shift() as T2;
+    result.push([el1, el2]);
+  }
+  return result;
+}

backend/app/utils/semaphore.ts

@@ -0,0 +1,49 @@
+export class Semaphore {
+  capacity: number;
+  #currentTasks: number;
+  #waitingTasks: (() => void)[];
+
+  constructor(capacity: number) {
+    this.capacity = capacity;
+    this.#currentTasks = 0;
+    this.#waitingTasks = [];
+  }
+
+  public get currentTasks(): number {
+    return this.#currentTasks;
+  }
+
+  public async runTask<T>(task: () => Promise<T>): Promise<T> {
+    // acquire the semaphore
+    await this.acquire();
+    try {
+      // execute the task
+      return await task();
+    } finally {
+      // don't forget to release
+      this.release();
+    }
+  }
+
+  private acquire(): Promise<void> {
+    // if we're under capacity, bump the count and resolve immediately
+    if (this.capacity > this.#currentTasks) {
+      this.#currentTasks += 1;
+      return Promise.resolve();
+    }
+    // otherwise add ourselves to the queue
+    return new Promise((resolve) => this.#waitingTasks.push(resolve));
+  }
+
+  private release() {
+    // try waking up the next task
+    const nextTask = this.#waitingTasks.shift();
+    if (nextTask === undefined) {
+      // no task in queue, decrement task count
+      this.#currentTasks -= 1;
+    } else {
+      // wake up the task
+      nextTask();
+    }
+  }
+}