Merge branch 'main' into web-editor-junior-account

Author: Renat0Ribeir0

.lycheeignore

@@ -6,13 +6,26 @@ https://support.arduino.cc/hc/*
 
 # Returns 404 for some reason
 https://create.arduino.cc/iot/*
-https://cloud.arduino.cc/home/registration
 https://cloud.arduino.cc/home/resources
 https://cloud.arduino.cc/plans
+https://app.arduino.cc/resources
+https://app.arduino.cc/devices
+https://app.arduino.cc/things
+https://app.arduino.cc/dashboards
+https://app.arduino.cc/templates
+https://app.arduino.cc/registration
+https://app.arduino.cc/space
+https://app.arduino.cc/plan-usage
 
 # "Network error: Forbidden" despite working redirect
 https://www.arduino.cc/en/Guide/Troubleshooting
 https://www.aftership.com/
 
 # Returns 403 (but is accessible in browser)
 https://www.digikey.se/product-detail/en/jst-sales-america-inc/PHR-2/455-1165-ND/608607
+
+# Returns [TIMEOUT]
+
+https://www.st.com/resource/en/datasheet/lsm9ds1.pdf
+https://www.st.com/en/development-tools/stlink-v3set.html
+https://www.st.com/en/development-tools/stm32cubeprog.html

_deploy/package-lock.json

@@ -9,22 +9,27 @@
   "author": "Sebastian Wikström",
   "license": "ISC",
   "dependencies": {
-    "algoliasearch": "^4.13.1",
+    "algoliasearch": "^4.23.2",
     "cli-color": "^2.0.3",
     "columnify": "^1.6.0",
     "commander": "^9.5.0",
     "dom-serializer": "^2.0.0",
     "domutils": "^3.0.1",
+    "fast-diff": "^1.3.0",
     "fast-glob": "^3.2.11",
     "form-data": "^4.0.0",
     "front-matter": "^4.0.2",
     "highlight.js": "^11.5.1",
     "html-minifier": "^4.0.0",
     "html-to-text": "^8.2.1",
     "htmlparser2": "^8.0.1",
-    "markdown-it": "^13.0.1",
+    "markdown-it": "^14.0.0",
+    "markdown-it-anchor": "^8.6.7",
+    "markdown-it-attrs": "^4.1.6",
     "markdown-it-footnote": "^3.0.3",
+    "markdown-it-github-alerts": "^0.3.0",
     "node-fetch": "^2.6.7",
-    "node-zendesk": "^2.2.0"
+    "node-zendesk": "^2.2.0",
+    "uslug": "^1.0.4"
   }
 }

_deploy/package.json

@@ -12,7 +12,7 @@ if (ZENDESK_USER && ZENDESK_PASS) {
     console.log('Zendesk credentials not found.');
     zendeskApiLimit = 200;
 }
-console.log(`API requests per minute: ${zendeskApiLimit}\n`);
+console.log(`API requests per minute: ${zendeskApiLimit}`);
 
 const AlgoliaID = process.env.ALGOLIA_APPLICATION_ID;
 const AlgoliaSecret = process.env.ALGOLIA_INDEXER_KEY;
@@ -32,6 +32,8 @@ program
     .option('--cache-read [path]', 'read cached data', false)
     .option('--cache-save [path]', 'save cached data', false)
     .option('--html-save', 'save rendered HTML to disk', false)
+    .option('--skip-algolia', 'skip all Algolia actions', false)
+    .option('--html-diff', 'print rendered HTML diff', false)
     .option('-w, --wait <delay>', 'delay in seconds before fetching data')
     .option('-u, --syncIndex', 'check the entire search index for changes')
 program.parse();
@@ -43,8 +45,11 @@ const deployChanges = program.opts().deploy;
 const verbose = program.opts().verbose;
 const cacheRead = program.opts().cacheRead;
 const cacheSave = program.opts().cacheSave;
+const htmlSave = program.opts().htmlSave;
+const htmlDiff = program.opts().htmlDiff; // TODO
 const wait = program.opts().wait;
 const syncIndex = program.opts().syncIndex;
+var skipAlgolia = program.opts().skipAlgolia;
 
 // Set up Zendesk client
 import { createClient as createZendeskClient } from 'node-zendesk';
@@ -62,8 +67,23 @@ const client = createZendeskClient({
 
 // Algolia
 import algoliasearch from 'algoliasearch';
-const algoliaIndex = algoliasearch(AlgoliaID, AlgoliaSecret)
-    .initIndex(AlgoliaIndexName);
+let algoliaIndex;
+if (!skipAlgolia) {
+    algoliaIndex = algoliasearch(AlgoliaID, AlgoliaSecret)
+        .initIndex(AlgoliaIndexName);
+    try {
+        var algoliaExists = await algoliaIndex.exists();
+        if (algoliaExists) {
+            console.log('Algolia index exists.');
+        }
+    } catch (error) {
+        console.log('Algolia index does not exist, and will not be updated!');
+        skipAlgolia = true;
+    }
+}
+
+// Empty line
+console.log();
 
 // HTML
 import * as htmlparser2 from "htmlparser2";
@@ -73,8 +93,11 @@ import { convert } from 'html-to-text';
 
 // Markdown
 import hljs from 'highlight.js'; // https://highlightjs.org/
-import markdownItFootnotes from 'markdown-it-footnote';
 import MarkdownIt from 'markdown-it';
+import markdownItFootnotes from 'markdown-it-footnote';
+import markdownItAnchor from 'markdown-it-anchor';
+import markdownItAttrs from 'markdown-it-attrs';
+import markdownItGitHubAlerts from 'markdown-it-github-alerts';
 const md = new MarkdownIt({
         html: true,
         smartquotes: true,
@@ -91,6 +114,23 @@ const md = new MarkdownIt({
             return ''; // use external default escaping
         }
     })
+    .use(markdownItGitHubAlerts, {
+        classPrefix: 'callout',
+        icons: {
+            note: '<span class="callout-icon callout-icon-note"></span>',
+            tip: '<span class="callout-icon callout-icon-tip"></span>',
+            important: '<span class="callout-icon callout-icon-important"></span>',
+            warning: '<span class="callout-icon callout-icon-warning"></span>',
+            caution: '<span class="callout-icon callout-icon-caution"></span>'
+        }
+    })
+    .use(markdownItAnchor, {
+        tabIndex: false
+    })
+    .use(markdownItAttrs, {
+        allowedAttributes: ['id', 'class'],
+        slugify: uslug
+    })
     .use(markdownItFootnotes);
 import fm from 'front-matter';
 
@@ -105,6 +145,8 @@ import fs from 'fs';
 const fsPromises = fs.promises;
 import fetch from 'node-fetch';
 import FormData from 'form-data';
+import uslug from 'uslug';
+import diff from 'fast-diff';
 
 /* Run main function */
 main();
@@ -193,16 +235,10 @@ async function main() {
             throw error;
         }
 
-        /*
-        console.log(clc.underline(`\nWaiting for 60 second(s)...`));
-        await delay(60 * 1000);
-        console.log('Done.\n');
-        */
-
         console.log(clc.underline('\nFetching article attachments...'));
         try {
             await Promise.all([
-                exTime(getAllAttachmentsSync(localArticles)).then(result => {
+                exTime(getAllAttachmentsSync(zendeskArticles)).then(result => {
                     console.log(`Fetched ${result.data.length} article attachment lists in ${result.exTime} ms.`);
                     return result.data;
                 })
@@ -565,7 +601,7 @@ async function deploy(zendeskSections, articles) {
         }
 
         // Update Algolia
-        if (translationUpdates || articleUpdates) {
+        if ((translationUpdates || articleUpdates) && !skipAlgolia) {
             var sectionName = zendeskSections.find(s => s.id == a.zd.section_id).name;
             var contentClearText = convert(a.zd.body, {
                 selectors: [
@@ -596,7 +632,9 @@ async function deploy(zendeskSections, articles) {
                 }).wait();
             } catch (error) {
                 console.error("Couldn't save object in Algolia");
-                console.error(error);
+                if (verbose) {
+                    console.error(error);
+                }
             }
         }
     }));
@@ -627,7 +665,9 @@ async function deploy(zendeskSections, articles) {
         } catch (error) {
             console.error(`[${error.statusCode}] Archiving article "${article.zd.title}" (${article.zd.html_url})`);
         }
-        await algoliaIndex.deleteObject(article.zd.url);
+        if (!skipAlgolia) {
+            await algoliaIndex.deleteObject(article.zd.url);
+        }
     }
 }
 
@@ -658,7 +698,17 @@ function hasChanges(article) {
     }
 
     // Render body and compare
-    if (!compareHTML(makeHTML(article.md.body, attachmentReplacements, true), article.zd.body)) {
+    let localHTML = makeHTML(article.md.body, attachmentReplacements, true);
+    let zdHTML = article.zd.body;
+    if (htmlSave) {
+        let htmlFilePath = article.md.filepath.concat('.html')
+        fs.writeFileSync(root + '/' + htmlFilePath, localHTML, function (err) {
+            if (err) {
+                return console.log(err);
+            }
+        });
+    }
+    if (!compareHTML(localHTML, zdHTML)) {
         return true;
     }
 
@@ -720,7 +770,7 @@ function getAttachmentReplacements(article) {
             if (zdAttachment) { // Match found
                 attachmentReplacements.push({
                     "src": src,
-                    "target": zdAttachment.content_url
+                    "target": zdAttachment.content_url // .replace('/' + zdAttachment.file_name, '')
                 });
                 // Remove processed
                 zdAttachment.used = true;
@@ -904,17 +954,25 @@ function getAllAttachments(localArticles) {
     return Promise.all(attachment_promises);
 }
 
-async function getAllAttachmentsSync(localArticles) {
+async function getAllAttachmentsSync(zendeskArticles) {
     var attachmentLists = [];
-    for (const localArticle of localArticles) {
-        const id = localArticle.attributes.id;
-        const draft = localArticle.attributes.draft; // Will fail for drafts unless authenticated
+    for (const zendeskArticle of zendeskArticles) {
+        const id = zendeskArticle.id;
+        const draft = zendeskArticle.draft; // Will fail for drafts unless authenticated
         if (id) {
+            if (verbose) {
+                console.log('Fetching attachments for article ' + id);
+            }
             var result = await client.articleattachments.list(id);
-            attachmentLists.push({
-                "article_id": id,
-                "attachments": result.article_attachments
-            })
+            if (!result.article_attachments) {
+                console.log(`Warning: No article attachment array for article with ID ${id}`);
+                console.log(result);
+            } else {
+                attachmentLists.push({
+                    "article_id": id,
+                    "attachments": result.article_attachments
+                })
+            }
         }
     }
     return attachmentLists;
@@ -1056,3 +1114,23 @@ async function deleteOrphanedSearchObjects(articles) {
     const deleteResult = await algoliaIndex.deleteObjects(removeTheseObjectIDs)
     console.log(`Deleted ${deleteResult.objectIDs.length} objects.`);
 }
+
+function printHtmlDiff(oldHtml, newHtml) {
+    let htmlDiff = diff(oldHtml, newHtml);
+    for (var s of htmlDiff) {
+        switch (s[0]) {
+            case diff.EQUAL:
+              console.log(s[1]);
+              break;
+            case diff.INSERT:
+                console.log(clc.bgGreen(s[1]))
+                break;
+            case diff.DELETE:
+              console.log(clc.bgRed(s[1]));
+              break;
+            default:
+              throw new Error("This shouldn't happen");
+          }
+          
+    }
+}

_deploy/zendesk.mjs

@@ -9,6 +9,10 @@ var glob = require("glob");
 const path = require('path');
 // TODO: https://stackoverflow.com/a/2186565
 const filePaths = glob.sync('content/**/*');
+let normPaths = [];
+for (var filePath of filePaths) {
+  normPaths.push(path.normalize(filePath));
+}
 
 module.exports = {
   "names": [ "HC002", "no-missing-images" ],
@@ -19,7 +23,7 @@ module.exports = {
       var dir = path.dirname(params.name);
       var imgPath = path.join(dir, token.attrs[0][1]);
 
-      if (!filePaths.includes(imgPath)) {
+      if (!normPaths.includes(path.normalize(imgPath))) {
         addError(onError, token.lineNumber, null, imgPath);
       }
     });

_linter/markdownlint/rules/hc002.js

@@ -24,4 +24,5 @@ The Log4j library was removed in Arduino IDE 1.8.19.
 | 2021-12-14  | [1.8.17](https://github.com/arduino/Arduino/releases/tag/1.8.17)                | Upgraded Log4j to 2.15.0, resolving [CVE-2021-44228](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-44228)                                                                                                                                                                         |
 | 2021-09-06  | [1.8.16](https://github.com/arduino/Arduino/releases/tag/1.8.16)                | Using Log4j 2.12.0                                                                                                                                                                                                                                                                          |
 
-> **Note:** The record creation date may reflect when the CVE ID was allocated or reserved, and does not necessarily indicate when this vulnerability was discovered, shared with the affected vendor, publicly disclosed, or updated in CVE.
+> [!NOTE]
+> The record creation date may reflect when the CVE ID was allocated or reserved, and does not necessarily indicate when this vulnerability was discovered, shared with the affected vendor, publicly disclosed, or updated in CVE.

content/About Arduino/Arduino Security Bulletins/ASEC-21-001-Vulnerabilities-in-Apache-Log4j.md

@@ -35,7 +35,7 @@ The identified vulnerabilities may allow an attacker, with local access to the v
 
 ## Action Required
 
-All users are advised to update the Arduino Create Agent to version 1.3.3 or later. An update is automatically initiated when visiting the Arduino Web Editor or when setting up a new device via the Arduino IoT Cloud. Alternatively, a manual update can be performed by downloading the new version of the software [here](https://github.com/arduino/arduino-create-agent/releases).
+All users are advised to update the Arduino Create Agent to version 1.3.3 or later. An update is automatically initiated when visiting the Arduino Cloud Editor or when setting up a new device in Arduino Cloud. Alternatively, a manual update can be performed by downloading the new version of the software [here](https://github.com/arduino/arduino-create-agent/releases).
 
 ## Additional information
 

content/About Arduino/Arduino Security Bulletins/ASEC-23-001-Vulnerabilities-in-Arduino-Create-Agent-1-3-2.md

@@ -0,0 +1,36 @@
+---
+title: ASEC-23-002 - Vulnerabilities in Arduino Create Agent 1.3.5
+id: 11832917802652
+---
+
+Bulletin ID: ASEC-23-002  
+Date: Dec 13, 2023  
+Product/Component: Arduino Create Agent  
+Affected versions: <= 1.3.5  
+Fixed version: 1.3.6
+
+## Summary
+
+This security bulletin provides information on a series of security vulnerabilities that have been identified in the Arduino Create Agent version 1.3.5 and below.
+
+Details on the security vulnerabilities and related advisories can be found below.
+
+### Medium risk
+
+* [CVE-2023-49296](https://www.cve.org/CVERecord?id=CVE-2023-49296): Reflected Cross-Site Scripting (CWE-35), CVSS v3.1 Base Score 7.3 (CVSS:3.1 AV:N/AC:L/PR:N/UI:R/S:U/C:L/I:L/A:L)
+
+## Impact
+
+The identified vulnerabilities may allow an attacker to persuade a victim into clicking on a malicious link and perform a Reflected Cross-Site Scripting attack on the web interface of the create agent, which would allow the attacker to execute arbitrary browser client side code.
+
+## Action Required
+
+All users are advised to update the Arduino Create Agent to version 1.3.6 or later. An update is automatically initiated when visiting the Arduino Cloud Editor or when setting up a new device via Arduino Cloud. Alternatively, a manual update can be performed by downloading the new version of the software [here](https://github.com/arduino/arduino-create-agent/releases).
+
+## Additional information
+
+* [Security Advisory - Reflected Cross-Site Scripting](https://github.com/arduino/arduino-create-agent/security/advisories/GHSA-j5hc-wx84-844h) (CWE-79)
+
+## Contact
+
+If you encounter any issues or have questions regarding this security update, please contact our security team at [[email protected]](mailto:[email protected]).