improve CSP check, flag implementations and run check on rebuild

krausest · krausest · commit 24d0bde39078 · 2024-03-06T21:00:42.000+01:00
diff --git a/cli/rebuild-check-single.js b/cli/rebuild-check-single.js
@@ -37,6 +37,9 @@ export function rebuildCheckSingle(frameworks) {
     const keyedCmd = `npm run isKeyed -- --headless true ${frameworkNames}`;
     runCommand(keyedCmd, "webdriver-ts");
 
+    const cspCmd = `npm run checkCSP -- --headless true ${frameworkNames}`;
+    runCommand(cspCmd, "webdriver-ts");
+
     console.log("rebuild-check-single.js finished");
     console.log("All checks are fine!");
     console.log(`======> Please rerun the benchmark: npm run bench ${frameworkNames}`);
diff --git a/frameworks/keyed/anansi/package.json b/frameworks/keyed/anansi/package.json
@@ -6,7 +6,7 @@
   "js-framework-benchmark": {
     "frameworkVersion": "0.14.0",
     "frameworkHomeURL": "https://saru-tora.github.io/anansi/",
-    "issues": []
+    "issues": [1139]
   },
   "scripts": {
     "dev": "exit 0",
diff --git a/frameworks/keyed/bobril/package.json b/frameworks/keyed/bobril/package.json
@@ -5,7 +5,8 @@
   "main": "src/app.js",
   "js-framework-benchmark": {
     "frameworkVersionFromPackage": "bobril",
-    "frameworkHomeURL": "https://bobril.com/"
+    "frameworkHomeURL": "https://bobril.com/",
+    "issues": [1139]
   },
   "scripts": {
     "dev": "bb",
diff --git a/frameworks/keyed/cample/package.json b/frameworks/keyed/cample/package.json
@@ -5,7 +5,8 @@
   "main": "dist/main.js",
   "js-framework-benchmark": {
     "frameworkVersionFromPackage": "cample",
-    "frameworkHomeURL": "https://camplejs.github.io"
+    "frameworkHomeURL": "https://camplejs.github.io",
+    "issues": [1139]
   },
   "scripts": {
     "build-prod": "webpack --mode=production",
diff --git a/frameworks/keyed/doohtml-dom/package.json b/frameworks/keyed/doohtml-dom/package.json
@@ -6,7 +6,7 @@
     "js-framework-benchmark": {
       "frameworkVersion": "",
       "frameworkHomeURL": "https://doohtml.com",
-      "issues": [772, 1139]
+      "issues": [772]
     },
     "scripts": {
       "dev": "exit 0",
diff --git a/frameworks/keyed/doohtml/package.json b/frameworks/keyed/doohtml/package.json
@@ -9,7 +9,7 @@
       "useShadowRoot": true,
       "shadowRootName": "doo-main",
       "buttonsInShadowRoot": false,
-      "issues": [772, 1139]
+      "issues": [772]
     },
     "scripts": {
       "dev": "exit 0",
diff --git a/frameworks/keyed/goui/package.json b/frameworks/keyed/goui/package.json
@@ -8,6 +8,7 @@
   "js-framework-benchmark": {
     "customURL": "/build",
     "frameworkVersion": "0.1.2",
-    "frameworkHomeURL": "https://github.com/twharmon/goui"
+    "frameworkHomeURL": "https://github.com/twharmon/goui",
+    "issues": [1139]
   }
 }
diff --git a/frameworks/keyed/marko/package.json b/frameworks/keyed/marko/package.json
@@ -4,7 +4,8 @@
   "js-framework-benchmark": {
     "frameworkVersionFromPackage": "marko",
     "frameworkHomeURL": "https://markojs.com/",
-    "customURL": "/dist"
+    "customURL": "/dist",
+    "issues": [1139]
   },
   "version": "1.0.0",
   "type": "module",
diff --git a/frameworks/keyed/openui5/package.json b/frameworks/keyed/openui5/package.json
@@ -4,7 +4,8 @@
   "description": "UI5 Benchmark",
   "js-framework-benchmark": {
     "frameworkVersion": "1.120.0",
-    "frameworkHomeURL": "https://openui5.org/"
+    "frameworkHomeURL": "https://openui5.org/",
+    "issues": [1139]
   },
   "scripts": {
     "build-dev": "ui5 serve -o index.html",
diff --git a/frameworks/keyed/owl/package.json b/frameworks/keyed/owl/package.json
@@ -4,7 +4,8 @@
   "description": "Benchmark for owl framework",
   "js-framework-benchmark": {
     "frameworkVersionFromPackage": "@odoo/owl",
-    "frameworkHomeURL": "https://odoo.github.io/owl/"
+    "frameworkHomeURL": "https://odoo.github.io/owl/",
+    "issues": [1139]
   },
   "scripts": {
     "dev": "webpack --mode development --watch",
diff --git a/frameworks/keyed/qwik/package.json b/frameworks/keyed/qwik/package.json
@@ -9,7 +9,8 @@
   "js-framework-benchmark": {
     "frameworkVersionFromPackage": "@builder.io/qwik",
     "frameworkHomeURL": "https://qwik.builder.io/",
-    "customURL": "/dist"
+    "customURL": "/dist",
+    "issues": [1139]
   },
   "scripts": {
     "build-dev": "vite --mode ssr",
diff --git a/frameworks/non-keyed/openui5/package.json b/frameworks/non-keyed/openui5/package.json
@@ -4,7 +4,8 @@
   "description": "UI5 Benchmark",
   "js-framework-benchmark": {
     "frameworkVersion": "1.120.0",
-    "frameworkHomeURL": "https://openui5.org/"
+    "frameworkHomeURL": "https://openui5.org/",
+    "issues": [1139]
   },
   "scripts": {
     "build-dev": "ui5 serve -o index.html",
diff --git a/frameworks/non-keyed/quel/package.json b/frameworks/non-keyed/quel/package.json
@@ -6,8 +6,7 @@
   "js-framework-benchmark": {
     "frameworkVersion": "0.9.51",
     "frameworkHomeURL": "https://github.com/mogera551/quel",
-    "issues": [
-    ]
+    "issues": [1139]
   },
   "scripts": {
     "build-dev": "echo 0",
diff --git a/frameworks/non-keyed/sprae/package.json b/frameworks/non-keyed/sprae/package.json
@@ -4,7 +4,8 @@
   "description": "Benchmark for sprae",
   "js-framework-benchmark": {
     "frameworkVersionFromPackage": "sprae",
-    "frameworkHomeURL": "https://github.com/dy/sprae"
+    "frameworkHomeURL": "https://github.com/dy/sprae",
+    "issues": [1139]
   },
   "keywords": [
     "vue"
diff --git a/server/src/csp/cspControllers.js b/server/src/csp/cspControllers.js
@@ -26,7 +26,8 @@ export function addCSP(request, reply) {
 
   const uri = body["csp-report"]["document-uri"];
   const frameworkRegEx = /((non-)?keyed\/.*?\/)/;
-  const framework = uri.match(frameworkRegEx)[0];
+  let framework = uri.match(frameworkRegEx)[0];
+  framework = framework.substring(0, framework.length - 1);
 
   if (!violations.includes(framework)) {
     violations.push(framework);
diff --git a/webdriver-ts/src/isCSPCompliant.ts b/webdriver-ts/src/isCSPCompliant.ts
@@ -52,58 +52,50 @@ async function runBench(
 
   let allCorrect = true;
 
+  console.log("*** headless", benchmarkOptions.headless);
+
   let browser = await startBrowser(benchmarkOptions);
   let page = await browser.newPage();
   try {
-    await page.goto(`http://${benchmarkOptions.host}:${benchmarkOptions.port}/csp/enable`);
-  } finally {
-    await page.close();
-    await browser.close();
-  }
-
-  console.log("*** headless", benchmarkOptions.headless);
-
-  for (let i = 0; i < runFrameworks.length; i++) {
-    let browser = await startBrowser(benchmarkOptions);
-    let page = await browser.newPage();
-    try {
+    for (let i = 0; i < runFrameworks.length; i++) {
+      let cspCheckSucessful = true;
       let framework: FrameworkData = runFrameworks[i];
+      let frameworkPath = (framework.keyed ? "keyed" : "non-keyed") + "/" + framework.name
+      console.log(`checking ${frameworkPath}`);
+      await page.goto(`http://${benchmarkOptions.host}:${benchmarkOptions.port}/csp/enable`);
 
       await page.goto(`http://${benchmarkOptions.host}:${benchmarkOptions.port}/${framework.uri}/index.html`, {
         waitUntil: "networkidle",
       });
       try {
         await checkElementExists(page, "#add");
       } catch (error) {
+        cspCheckSucessful = false;
         console.log(`CSP test failed for ${runFrameworks[i].fullNameWithKeyedAndVersion} - during load`);
       }
-      await clickElement(page, "#add");
-      try {
-        await checkElementContainsText(page, "tbody>tr:nth-of-type(1000)>td:nth-of-type(1)", "1000");
-      } catch (error) {
-        console.log(`CSP test failed for ${runFrameworks[i].fullNameWithKeyedAndVersion} - when clicking`);
+      if (cspCheckSucessful) {
+        try {
+          await clickElement(page, "#add");
+          await checkElementContainsText(page, "tbody>tr:nth-of-type(1000)>td:nth-of-type(1)", "1000");
+        } catch (error) {
+          cspCheckSucessful = false;
+          console.log(`CSP test failed for ${runFrameworks[i].fullNameWithKeyedAndVersion} - when clicking`);
+        }
       }
-    } catch (error) {
-      //console.log("ERROR running " + runFrameworks[i].fullNameWithKeyedAndVersion, e);
-      allCorrect = false;
-    } finally {
-      try {
-        await page.close();
-        await browser.close();
-      } catch (error) {
-        console.log("error calling driver.quit - ignoring this exception");
+      await page.goto(`http://${benchmarkOptions.host}:${benchmarkOptions.port}/csp`);
+      const extractedText = await page.$eval("*", (el: any) => el.innerText);
+      let failed: string[] = JSON.parse(extractedText);
+      if (failed.includes(frameworkPath)) {
+        cspCheckSucessful = false;
+        console.log(`CSP test failed for ${runFrameworks[i].fullNameWithKeyedAndVersion} - due to reporting`);
       }
+      if (!cspCheckSucessful != framework.issues.includes((1139))) {
+        const hint = cspCheckSucessful ? "The flag 1139 should be removed" : "The flag 1139 should be added";
+        console.log(`ERROR: CSP is incorrectly categorized for ${runFrameworks[i].fullNameWithKeyedAndVersion} . ${hint}`);
+        cspCheckSucessful = false;
+      }
+      allCorrect = allCorrect && cspCheckSucessful;
     }
-  }
-
-  browser = await startBrowser(benchmarkOptions);
-  page = await browser.newPage();
-  try {
-    await page.goto(`http://${benchmarkOptions.host}:${benchmarkOptions.port}/csp`);
-    const extractedText = await page.$eval("*", (el: any) => el.innerText);
-    console.log(extractedText);
-    let failed = JSON.parse(extractedText);
-    console.log("CSP check failed for the following frameworks:\n", failed.join("\n"));
   } finally {
     await page.close();
     await browser.close();

Original file line number	Diff line number	Diff line change
`@@ -8,6 +8,7 @@`
`8`	`8`	`"js-framework-benchmark": {`
`9`	`9`	`"customURL": "/build",`
`10`	`10`	`"frameworkVersion": "0.1.2",`
`11`		`- "frameworkHomeURL": "https://github.com/twharmon/goui"`
	`11`	`+ "frameworkHomeURL": "https://github.com/twharmon/goui",`
	`12`	`+ "issues": [1139]`
`12`	`13`	`}`
`13`	`14`	`}`