✨ fix extra whitespace problems in bad-linebreaks run

Author: ctcpip

.eslintrc.js

@@ -12,7 +12,8 @@ module.exports = {
     "arrow-spacing": "error",
     "brace-style": ["error", "1tbs"],
     "keyword-spacing": "error",
-    "object-curly-spacing": ["error", "always"]
+    "object-curly-spacing": ["error", "always"],
+    "semi": ["error", "always"],
   },
   overrides: [
     {

scripts/bad-linebreaks-test.mjs

@@ -1,31 +1,34 @@
 import { strict as assert } from 'node:assert';
-import { findBadLinebreaks } from './bad-linebreaks.mjs';
+import { findBadStuff } from './bad-linebreaks.mjs';
 import fs from 'fs';
 import crypto from 'crypto';
 
 const afterMD = './scripts/test-samples/bad-linebreaks-sample-after.md';
 const beforeMD = './scripts/test-samples/bad-linebreaks-sample-before.md';
 
 // verify hash values to detect file tampering
-const knownAfterHash = '5e03eeb87149ca238ab55d422913c0f8';
-const knownBeforeHash = '1f2c5d3956a9aceb29dd2b872c20a021';
+const knownAfterHash = 'c2b5b7cc30cf5d4ce28274848eeba743';
+const knownBeforeHash = 'c9cf57714ec19de2aeea68d45536b119';
 const afterHash = await getHashSlingingSlasher(afterMD);
 const beforeHash = await getHashSlingingSlasher(beforeMD);
 assert.strictEqual(afterHash, knownAfterHash);
 assert.strictEqual(beforeHash, knownBeforeHash);
 
 let fixed, totalMatches;
 
-({ fixed, totalMatches } = findBadLinebreaks(beforeMD, true));
-assert.strictEqual(totalMatches, 12);
+({ fixed, totalMatches } = findBadStuff(beforeMD, true));
+assert.strictEqual(totalMatches.badLinebreaks, 12);
+assert.strictEqual(totalMatches.extraWhitespace, 28);
 assert.strictEqual(fixed, fs.readFileSync(afterMD, 'utf8').toString());
 
-({ fixed, totalMatches } = findBadLinebreaks(afterMD, true));
-assert.strictEqual(totalMatches, 0);
+({ fixed, totalMatches } = findBadStuff(afterMD, true));
+assert.strictEqual(totalMatches.badLinebreaks, 0);
+assert.strictEqual(totalMatches.extraWhitespace, 0);
 assert.strictEqual(fixed, fs.readFileSync(afterMD, 'utf8').toString());
 
-({ fixed, totalMatches } = findBadLinebreaks(beforeMD));
-assert.strictEqual(totalMatches, 12);
+({ fixed, totalMatches } = findBadStuff(beforeMD));
+assert.strictEqual(totalMatches.badLinebreaks, 12);
+assert.strictEqual(totalMatches.extraWhitespace, 28);
 
 function getHashSlingingSlasher(file) {  // 💀
   return new Promise((res, rej) => {

scripts/bad-linebreaks.mjs

@@ -7,54 +7,84 @@ import { glob } from 'glob';
 // import attributes when?
 const mdlintConfig = JSON.parse(fs.readFileSync('.markdownlint-cli2.jsonc', 'utf8').toString());
 
-const re = /(?<=[\w\d ])\n(?=[\w\d ])/g;
+const reBadLinebreaks = /(?<=[\w\d ])\n(?=[\w\d ])/g;
+const reExtraWhitespace = /^ +| (?= )| +$/gm;
 
-export function findBadLinebreaks(file, fix = false) {
+export function findBadStuff(file, fix = false) {
 
   let contents = fs.readFileSync(file, 'utf8').toString();
 
-  const tokens = marked.lexer(contents, { ...marked.getDefaults(), })
+  const tokens = marked.lexer(contents, { ...marked.getDefaults(), });
 
   const o = [];
-  let totalMatches = 0;
+  const totalMatches = { badLinebreaks: 0, extraWhitespace: 0 };
 
   for (let i = 0; i < tokens.length; i++) {
 
     const t = tokens[i];
+    let tokenContent = t.raw;
 
     if (t.type === 'paragraph') {
+      tokenContent = findBadLinebreaks(tokenContent, totalMatches, fix, file);
+      tokenContent = findExtraWhitespace(tokenContent, totalMatches, fix, file);
+    }
 
-      const matches = Array.from(t.raw.matchAll(re));
-      totalMatches += matches.length;
+    // we don't need to build this array if `fix` is `false`, but this keeps complexity down
+    o.push(tokenContent);
 
-      if (fix) {
-        if (matches.length > 0) {
+  }
 
-          let fixedContent = t.raw;
+  return {
+    fixed: o.join(''),
+    totalMatches,
+  };
 
-          for (const m of matches) {
-            fixedContent = `${fixedContent.slice(0, m.index)} ${fixedContent.slice(m.index + 1)}`;
-          }
+}
 
-          o.push(fixedContent);
+function findBadLinebreaks(tokenContent, totalMatches, fix, file) {
 
-        } else {
-          o.push(t.raw);
-        }
-      } else if (matches.length > 0) {
-        console.log(`${file}\nfound paragraph with ${matches.length} erroneous linebreak(s):\n${t.raw}\n`);
+  const matches = Array.from(tokenContent.matchAll(reBadLinebreaks));
+  totalMatches.badLinebreaks += matches.length;
+
+  if (fix) {
+
+    if (matches.length > 0) {
+
+      let fixedContent = tokenContent;
+
+      for (const m of matches) {
+        fixedContent = `${fixedContent.slice(0, m.index)} ${fixedContent.slice(m.index + 1)}`;
       }
 
-    } else if (fix) {
-      o.push(t.raw);
+      return fixedContent;
+
     }
 
+  } else if (matches.length > 0) {
+    console.error(`${file}\nfound paragraph with ${matches.length} erroneous linebreak(s):\n${tokenContent}\n`);
   }
 
-  return {
-    fixed: o.join(''),
-    totalMatches,
-  };
+  return tokenContent;
+
+}
+
+function findExtraWhitespace(tokenContent, totalMatches, fix, file) {
+
+  const matches = Array.from(tokenContent.matchAll(reExtraWhitespace));
+  const extraWhitespaceCharacters = matches.join('').length;
+  totalMatches.extraWhitespace += extraWhitespaceCharacters;
+
+  if (fix) {
+
+    if (matches.length > 0) {
+      return tokenContent.replace(reExtraWhitespace, '');
+    }
+
+  } else if (matches.length > 0) {
+    console.error(`${file}\nfound paragraph with ${extraWhitespaceCharacters} extra whitespace character(s):\n${tokenContent}\n`);
+  }
+
+  return tokenContent;
 
 }
 
@@ -69,17 +99,26 @@ export async function processFiles(fix) {
 
   for (const f of files) {
 
-    const { fixed, totalMatches } = findBadLinebreaks(f, fix);
+    const { fixed, totalMatches } = findBadStuff(f, fix);
 
     if (fix) {
       fs.writeFileSync(f, fixed);
     }
 
-    if (totalMatches > 0) {
+    if (totalMatches.badLinebreaks > 0 || totalMatches.extraWhitespace > 0) {
 
       if (fix) {
-        console.log(`fixed ${totalMatches} erroneous linebreaks in ${f}`);
+
+        if (totalMatches.badLinebreaks > 0) {
+          console.log(`fixed ${totalMatches.badLinebreaks} erroneous linebreak(s) in ${f}`);
+        }
+
+        if (totalMatches.extraWhitespace > 0) {
+          console.log(`fixed ${totalMatches.extraWhitespace} extra whitespace character(s) in ${f}`);
+        }
+
       } else {
+        console.error('fix bad linebreaks and whitespace by running `npm run bad-linebreaks:fix` or `npm run fix:all`');
         process.exitCode = 1;
       }
 

scripts/test-samples/bad-linebreaks-sample-after.md

@@ -11,7 +11,7 @@ AAA: Lorem.
 
 BBB: This is a sentence.
 
-CCC: This  is  a  sentence with trailing spaces. 
+CCC: This is a sentence with trailing spaces.
 
 CCC: This is a sentence.
 DDD: This is also a sentence.
@@ -40,9 +40,9 @@ let biscuits = "delicious";
 
 ## story time!
 
-True! nervous, very, very dreadfully nervous I had been and am; but why will you say that I am mad? The disease had sharpened my senses,  not destroyed,  not dulled them. Above all was the sense of  hearing acute. I heard all things in the heaven and in the earth. I heard many things in hell. How then am I mad? Hearken! and observe how healthily, how calmly I can tell you the whole story.
+True! nervous, very, very dreadfully nervous I had been and am; but why will you say that I am mad? The disease had sharpened my senses, not destroyed, not dulled them. Above all was the sense of hearing acute. I heard all things in the heaven and in the earth. I heard many things in hell. How then am I mad? Hearken! and observe how healthily, how calmly I can tell you the whole story.
 
-It is impossible to say how first the idea entered my brain, but, once conceived, it haunted me day and night. Object there was none. Passion there was none. I loved the old man. He had never wronged me. He had never given me insult. For his gold I had no desire. I think it was his eye! Yes, it was this! One of his eyes resembled that of a vulture—a pale blue eye with a film over it. Whenever it fell upon me my blood ran cold, and so by degrees, very gradually, I   made up my mind to take the life of the old man, and thus rid myself of the eye forever.
+It is impossible to say how first the idea entered my brain, but, once conceived, it haunted me day and night. Object there was none. Passion there was none. I loved the old man. He had never wronged me. He had never given me insult. For his gold I had no desire. I think it was his eye! Yes, it was this! One of his eyes resembled that of a vulture—a pale blue eye with a film over it. Whenever it fell upon me my blood ran cold, and so by degrees, very gradually, I made up my mind to take the life of the old man, and thus rid myself of the eye forever.
 
 Now this is the point. You fancy me mad. Madmen know nothing. But you should have seen me. You should have seen how wisely I proceeded; with what caution, with what foresight, with what dissimulation, I went to work! I was never kinder to the old man than during the whole week before I killed him. And every night about midnight I turned the latch of his door and opened it—oh, so gently! And then when I had made an opening sufficient for my head I put in a dark lantern all closed, closed so that no light shone out, and then I thrust in my head. Oh, you would have laughed to see how cunningly I thrust it in! I moved it slowly, very, very slowly, so that I might not disturb the old man's sleep. It took me an hour to place my whole head within the opening so far that I could see him as he lay upon his bed. Ha! would a madman have been so wise as this? And then when my head was well in the room I undid the lantern cautiously—oh, so cautiously—cautiously (for the hinges creaked), I undid it just so much that a single thin ray fell upon the vulture eye. And this I did for seven long nights, every night just at midnight, but I found the eye always closed, and so it was impossible to do the work, for it was not the old man who vexed me, but his Evil Eye. And every morning, when the day broke, I went boldly into the chamber and spoke courageously to him, calling him by name in a hearty tone, and inquiring how he had passed the night. So you see he would have been a very profound old man, indeed, to suspect that every night, just at twelve, I looked in upon him while he slept.
 
@@ -61,8 +61,8 @@ It was open, wide, wide open, and I grew furious as I gazed upon it. I saw it wi
 And now have I not told you that what you mistake for madness is but over-acuteness of the senses? now, I say, there came to my ears a low, dull, quick sound, such as a watch makes when enveloped in cotton. I knew that sound well, too. It was the beating of the old man's heart. It increased my fury, as the beating of a drum stimulates the soldier into courage.
 
 But even yet I refrained and kept still. I scarcely breathed. I held the lantern motionless. I tried how steadily I could maintain the ray upon the eye. Meantime the hellish tattoo of the heart increased. It grew quicker and quicker, and louder and louder, every instant. The old man's terror must have been extreme! It grew louder, I say, louder every moment!—do you mark me well? I have told you that I am nervous: so I am. And now at the dead hour of the night, amid the dreadful silence of that old house, so strange a noise as this excited me to uncontrollable terror. Yet, for some minutes longer I refrained and stood still. But the beating grew louder, louder! I thought the heart must burst. And now a new anxiety seized me—the sound would be heard by a neighbor!
- The old man's hour had come! With a loud yell, I threw open the lantern and leaped into the room. He shrieked once—once only. In an instant I dragged him to the floor, and pulled the heavy bed over him. I then smiled gaily, to find the deed so far done. But for many minutes the heart beat on with a muffled sound. This, however,   did not vex me; it would not be heard through the wall. At length it ceased. The old man was dead. I removed the bed and examined the corpse.
-  Yes, he was stone,    stone dead. I placed my hand    upon the heart and held it there many minutes.
+The old man's hour had come! With a loud yell, I threw open the lantern and leaped into the room. He shrieked once—once only. In an instant I dragged him to the floor, and pulled the heavy bed over him. I then smiled gaily, to find the deed so far done. But for many minutes the heart beat on with a muffled sound. This, however, did not vex me; it would not be heard through the wall. At length it ceased. The old man was dead. I removed the bed and examined the corpse.
+Yes, he was stone, stone dead. I placed my hand upon the heart and held it there many minutes.
 There was no pulsation. He was stone dead. His eye would trouble me no more.
 
 If still you think me mad, you will think so no longer when I describe the wise precautions I took for the concealment of the body. The night waned, and I worked hastily, but in silence.
@@ -74,7 +74,7 @@ When I had made an end of these labors, it was four o'clock—still dark as midn
 I smiled—for what had I to fear? I bade the gentlemen welcome. The shriek, I said, was my own in a dream. The old man, I mentioned, was absent in the country. I took my visitors all over the house. I bade them search—search well. I led them, at length, to his chamber.
 I showed them his treasures, secure, undisturbed. In the enthusiasm of my confidence, I brought chairs into the room, and desired them here to rest from their fatigues, while I myself, in the wild audacity of my perfect triumph, placed my own seat upon the very spot beneath which reposed the corpse of the victim.
 
-The officers were satisfied. My manner had convinced them. I was singularly at ease. They sat, and while I answered cheerily, they chatted of familiar things. But, ere long, I felt myself getting pale and wished them gone. My head ached, and I fancied a        ringing in my ears; but still they sat, and still chatted. The ringing became more distinct;—it continued and became more distinct. I talked more freely to get rid of the feeling, but it continued and gained definitiveness—until, at length, I found that the noise was not within my ears.
+The officers were satisfied. My manner had convinced them. I was singularly at ease. They sat, and while I answered cheerily, they chatted of familiar things. But, ere long, I felt myself getting pale and wished them gone. My head ached, and I fancied a ringing in my ears; but still they sat, and still chatted. The ringing became more distinct;—it continued and became more distinct. I talked more freely to get rid of the feeling, but it continued and gained definitiveness—until, at length, I found that the noise was not within my ears.
 
 No doubt I now grew very pale; but I talked more fluently and with a heightened voice. Yet the sound increased—and what could I do? It was a low, dull, quick sound—much such a sound as a watch makes when enveloped in cotton. I gasped for breath—and yet the officers heard it not. I talked more quickly—more vehemently; but the noise steadily increased. I arose and argued about trifles, in a high key and with violent gesticulations; but the noise steadily increased. Why would they not be gone? I paced the floor to and fro with heavy strides, as if excited to fury by the observations of the men—but the noise steadily increased. O God! what could I do? I foamed—I raved—I swore! I swung the chair upon which I had been sitting, and grated it upon the boards, but the noise arose over all and continually increased. It grew louder—louder—louder! And still the men chatted pleasantly, and smiled. Was it possible they heard not? Almighty God!—no, no! They heard!—they suspected!—they knew!—they were making a mockery of my horror!—this I thought, and this I think. But anything was better than this agony! Anything was more tolerable than this derision! I could bear those hypocritical smiles no longer! I felt that I must scream or die!—and now—again!—hark! louder! louder! louder! louder!
 

scripts/test-samples/bad-linebreaks-sample-before.md

@@ -1,6 +1,6 @@
 # Ye Olde Meeting Notes
 
-this file has deliberate linting problems... do not fix.
+ this file has deliberate linting problems... do not fix.
 
 | ye   | olde | table |
 |------|------|-------|