Merge branch 'master' into develop

Author: rhansen

CHANGELOG.md

@@ -29,6 +29,28 @@
     generator function.
   * `newOp()`: Deprecated in favor of the new `Op` class.
 
+# 1.8.16
+
+### Security fixes
+
+If you cannot upgrade to v1.8.16 for some reason, you are encouraged to try
+cherry-picking the fixes to the version you are running:
+
+```shell
+git cherry-pick b7065eb9a0ec..77bcb507b30e
+```
+
+* Maliciously crafted `.etherpad` files can no longer overwrite arbitrary
+  non-pad database records when imported.
+* Imported `.etherpad` files are now subject to numerous consistency checks
+  before any records are written to the database. This should help avoid
+  denial-of-service attacks via imports of malformed `.etherpad` files.
+
+### Notable enhancements and fixes
+
+* Fixed several `.etherpad` import bugs.
+* Improved support for large `.etherpad` imports.
+
 # 1.8.15
 
 ### Security fixes

doc/api/hooks_server-side.md

@@ -175,14 +175,15 @@ Things in context:
 
 This hook gets called when a new pad was created.
 
-## padLoad
-Called from: src/node/db/Pad.js
+## `padLoad`
 
-Things in context:
+Called from: `src/node/db/PadManager.js`
 
-1. pad - the pad instance
+Called when a pad is loaded, including after new pad creation.
+
+Context properties:
 
-This hook gets called when a pad was loaded. If a new pad was created and loaded this event will be emitted too.
+* `pad`: The Pad object.
 
 ## padUpdate
 Called from: src/node/db/Pad.js

src/bin/checkAllPads.js

@@ -10,79 +10,18 @@ process.on('unhandledRejection', (err) => { throw err; });
 if (process.argv.length !== 2) throw new Error('Use: node src/bin/checkAllPads.js');
 
 (async () => {
-  // initialize the database
-  require('../node/utils/Settings');
   const db = require('../node/db/DB');
   await db.init();
-
-  // load modules
-  const Changeset = require('../static/js/Changeset');
   const padManager = require('../node/db/PadManager');
-
-  let revTestedCount = 0;
-
-  // get all pads
-  const res = await padManager.listAllPads();
-  for (const padId of res.padIDs) {
+  await Promise.all((await padManager.listAllPads()).padIDs.map(async (padId) => {
     const pad = await padManager.getPad(padId);
-
-    // check if the pad has a pool
-    if (pad.pool == null) {
-      console.error(`[${pad.id}] Missing attribute pool`);
-      continue;
-    }
-    // create an array with key kevisions
-    // key revisions always save the full pad atext
-    const head = pad.getHeadRevisionNumber();
-    const keyRevisions = [];
-    for (let rev = 0; rev < head; rev += 100) {
-      keyRevisions.push(rev);
-    }
-
-    // run through all key revisions
-    for (const keyRev of keyRevisions) {
-      // create an array of revisions we need till the next keyRevision or the End
-      const revisionsNeeded = [];
-      for (let rev = keyRev; rev <= keyRev + 100 && rev <= head; rev++) {
-        revisionsNeeded.push(rev);
-      }
-
-      // this array will hold all revision changesets
-      const revisions = [];
-
-      // run through all needed revisions and get them from the database
-      for (const revNum of revisionsNeeded) {
-        const revision = await db.get(`pad:${pad.id}:revs:${revNum}`);
-        revisions[revNum] = revision;
-      }
-
-      // check if the revision exists
-      if (revisions[keyRev] == null) {
-        console.error(`[${pad.id}] Missing revision ${keyRev}`);
-        continue;
-      }
-
-      // check if there is a atext in the keyRevisions
-      let {meta: {atext} = {}} = revisions[keyRev];
-      if (atext == null) {
-        console.error(`[${pad.id}] Missing atext in revision ${keyRev}`);
-        continue;
-      }
-
-      const apool = pad.pool;
-      for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) {
-        try {
-          const cs = revisions[rev].changeset;
-          atext = Changeset.applyToAText(cs, atext, apool);
-          revTestedCount++;
-        } catch (e) {
-          console.error(`[${pad.id}] Bad changeset at revision ${rev} - ${e.message}`);
-        }
-      }
+    try {
+      await pad.check();
+    } catch (err) {
+      console.error(`Error in pad ${padId}: ${err.stack || err}`);
+      return;
     }
-  }
-  if (revTestedCount === 0) {
-    throw new Error('No revisions tested');
-  }
-  console.log(`Finished: Tested ${revTestedCount} revisions`);
+    console.log(`Pad ${padId}: OK`);
+  }));
+  console.log('Finished.');
 })();

src/bin/checkPad.js

@@ -8,75 +8,13 @@
 process.on('unhandledRejection', (err) => { throw err; });
 
 if (process.argv.length !== 3) throw new Error('Use: node src/bin/checkPad.js $PADID');
-
-// get the padID
 const padId = process.argv[2];
-let checkRevisionCount = 0;
-
 (async () => {
-  // initialize database
-  require('../node/utils/Settings');
   const db = require('../node/db/DB');
   await db.init();
-
-  // load modules
-  const Changeset = require('../static/js/Changeset');
   const padManager = require('../node/db/PadManager');
-
-  const exists = await padManager.doesPadExists(padId);
-  if (!exists) throw new Error('Pad does not exist');
-
-  // get the pad
+  if (!await padManager.doesPadExists(padId)) throw new Error('Pad does not exist');
   const pad = await padManager.getPad(padId);
-
-  // create an array with key revisions
-  // key revisions always save the full pad atext
-  const head = pad.getHeadRevisionNumber();
-  const keyRevisions = [];
-  for (let rev = 0; rev < head; rev += 100) {
-    keyRevisions.push(rev);
-  }
-
-  // run through all key revisions
-  for (let keyRev of keyRevisions) {
-    keyRev = parseInt(keyRev);
-    // create an array of revisions we need till the next keyRevision or the End
-    const revisionsNeeded = [];
-    for (let rev = keyRev; rev <= keyRev + 100 && rev <= head; rev++) {
-      revisionsNeeded.push(rev);
-    }
-
-    // this array will hold all revision changesets
-    const revisions = [];
-
-    // run through all needed revisions and get them from the database
-    for (const revNum of revisionsNeeded) {
-      const revision = await db.get(`pad:${padId}:revs:${revNum}`);
-      revisions[revNum] = revision;
-    }
-
-    // check if the pad has a pool
-    if (pad.pool == null) throw new Error('Attribute pool is missing');
-
-    // check if there is an atext in the keyRevisions
-    let {meta: {atext} = {}} = revisions[keyRev] || {};
-    if (atext == null) {
-      console.error(`No atext in key revision ${keyRev}`);
-      continue;
-    }
-
-    const apool = pad.pool;
-
-    for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) {
-      checkRevisionCount++;
-      try {
-        const cs = revisions[rev].changeset;
-        atext = Changeset.applyToAText(cs, atext, apool);
-      } catch (e) {
-        console.error(`Bad changeset at revision ${rev} - ${e.message}`);
-        continue;
-      }
-    }
-    console.log(`Finished: Checked ${checkRevisionCount} revisions`);
-  }
+  await pad.check();
+  console.log('Finished.');
 })();