Decode HTML entities in module description (#3139)

* yarn add he

* add decode function

* Use decode function

* Add test

* test less verbose

* check null or undefined

* remove compatible operator in package.json

Author: dlqs

scrapers/nus-v2/package.json

@@ -20,6 +20,7 @@
     "@types/bunyan": "1.8.6",
     "@types/deep-diff": "1.0.0",
     "@types/fs-extra": "9.0.6",
+    "@types/he": "1.1.1",
     "@types/lodash": "4.14.167",
     "@types/mock-fs": "4.13.0",
     "@types/node": "14.14.19",
@@ -53,6 +54,7 @@
     "date-fns": "2.16.1",
     "deep-diff": "1.0.2",
     "fs-extra": "9.0.1",
+    "he": "1.2.0",
     "joi": "17.3.0",
     "lodash": "4.17.20",
     "nusmoderator": "3.0.0",

scrapers/nus-v2/src/tasks/GetSemesterData.test.ts

@@ -151,6 +151,23 @@ describe(cleanModuleInfo, () => {
       moduleCode: 'EL5102',
     });
   });
+
+  test('should decode HTML entities in description', () => {
+    expect(
+      cleanModuleInfo({
+        acadYear: '2020/2021',
+        description: 'These concepts pertain to the structure of "ultimate reality"...',
+        title: 'Metaphysics',
+        department: 'Philosophy',
+        faculty: 'Arts and Social Science',
+        moduleCredit: '4',
+        moduleCode: 'PH2213',
+      }),
+    ).toHaveProperty(
+      'description',
+      'These concepts pertain to the structure of "ultimate reality"...',
+    );
+  });
 });
 
 describe(parseWorkload, () => {

scrapers/nus-v2/src/tasks/GetSemesterData.ts

@@ -21,7 +21,7 @@ import GetSemesterTimetable from './GetSemesterTimetable';
 import GetSemesterModules from './GetSemesterModules';
 import { fromTermCode } from '../utils/api';
 import { validateSemester } from '../services/validation';
-import { removeEmptyValues, titleize, trimValues } from '../utils/data';
+import { removeEmptyValues, titleize, trimValues, decodeHTMLEntities } from '../utils/data';
 import { difference } from '../utils/set';
 import { Logger } from '../services/logger';
 
@@ -84,6 +84,7 @@ export function mapAttributes(
  * - Remove empty fields and fields with text like 'nil'
  * - Trim whitespace from module title, description and other text fields
  * - Properly capitalize ALL CAPS title
+ * - Decode HTML entities in description such as '&224;' to 'à'
  */
 export function cleanModuleInfo(module: SemesterModule) {
   let cleanedModule = module;
@@ -93,6 +94,10 @@ export function cleanModuleInfo(module: SemesterModule) {
     cleanedModule.title = titleize(cleanedModule.title);
   }
 
+  if (cleanedModule.description != null) {
+    cleanedModule.description = decodeHTMLEntities(cleanedModule.description);
+  }
+
   // Remove empty values like 'nil' and empty strings for keys that allow them
   // to be nullable
   cleanedModule = removeEmptyValues(cleanedModule, [

scrapers/nus-v2/src/utils/data.test.ts

@@ -9,6 +9,7 @@ import {
   removeEmptyValues,
   titleize,
   trimValues,
+  decodeHTMLEntities,
   ZWSP,
 } from './data';
 
@@ -35,6 +36,14 @@ describe(titleize, () => {
   });
 });
 
+describe(decodeHTMLEntities, () => {
+  test('should decode HTML entities', () => {
+    expect(
+      decodeHTMLEntities('& Schrödinger cried, "Oh là là!"'),
+    ).toEqual('& Schrödinger cried, "Oh là là!"');
+  });
+});
+
 describe(trimValues, () => {
   test('should remove whitespace around the given values', () => {
     expect(

scrapers/nus-v2/src/utils/data.ts

@@ -3,6 +3,7 @@
  */
 
 import { uniq, trim, groupBy, values } from 'lodash';
+import { decode } from 'he';
 import { VenueLesson } from '../types/venues';
 import {
   DayText,
@@ -46,6 +47,10 @@ export function titleize(string: string) {
   return capitalized;
 }
 
+export function decodeHTMLEntities(string: string) {
+  return decode(string);
+}
+
 /**
  * Remove keys with empty values, null or strings like 'nil', 'none'
  * Mutates the input object

scrapers/nus-v2/yarn.lock

@@ -683,6 +683,11 @@
   dependencies:
     "@types/node" "*"
 
+"@types/[email protected]":
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/@types/he/-/he-1.1.1.tgz#19e14033c4ee8f1a702c74dcc6182664839ac2b7"
+  integrity sha512-jpzrsR1ns0n3kyWt92QfOUQhIuJGQ9+QGa7M62rO6toe98woQjnsnzjdMtsQXCdvjjmqjS2ZBCC7xKw0cdzU+Q==
+
 "@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0", "@types/istanbul-lib-coverage@^2.0.1":
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.3.tgz#4ba8ddb720221f432e443bd5f9117fd22cfd4762"
@@ -2333,6 +2338,11 @@ has@^1.0.3:
   dependencies:
     function-bind "^1.1.1"
 
+[email protected]:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f"
+  integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==
+
 hosted-git-info@^2.1.4:
   version "2.8.8"
   resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.8.tgz#7539bd4bc1e0e0a895815a2e0262420b12858488"