Merge branch 'release/v5.0.0'

Author: joachimvh

.travis.yml

@@ -31,6 +31,8 @@ script:
 after_success:
   - snyk monitor
 
+cache: npm
+
 notifications:
   email:
   - [email protected]

CHANGELOG.md

@@ -1,15 +1,63 @@
 # History
 
+## 5.0.0
+
+- Node versions greater than 8 are supported.
+- Changes to vocabulary use:
+    - `solid:inbox` is deprecated in favour of `ldp:inbox`.
+    - `acl:defaultForNew` is deprecated in favour of `ack:default`.
+- Terms of Service may be added and enforced for new registrations,
+  but is disabled by default.    	
+- DELETE operations on a resource now require that the user has write permissions on
+  the file's container
+- Improved support for logout ensures users can use different
+  identities.
+- The profile container is now public readable by default.
+- Access Control: 
+    - The Access Control List system has undergone extensive
+      changes. Security has been tightened, and some unsafe practices that
+      where web apps was authorized access in the past are now not
+      permitted. 
+    - The browser-reported `Origin` header will now be checked by
+      default, and the ACL system can be used to restrict access
+      to applications for added security.
+- Logging is now verbose by default so the `-v` option has been
+  removed and a `--quiet` option has been added to mute the log.
+- Command line options are now kebab-cased rather than camelCased,
+  config options may be both.
+- Many smaller fixes.
+
+#### 5.0.0 Upgrade Notes
+
+- As of v5.0.0, all Turtle files need an extension. (**Intervention needed when updating from < 5.0.0!**)
+    - **How to upgrade?**
+        1. Stop the server.
+        2. Update node-solid-server to 5.0.0.
+        3. Make a backup of your `data/` and `config/` folders.
+        4. Invoke `solid migrate-legacy-resources -v`.
+           This makes the files in your `data/` and `config/` folders
+           automatically compatible with the new system.
+           You only need to do this once.
+           Different data folders can be migrated as well with the `-p` option:
+           `solid migrate-legacy-resources -p my/custom/data/folder -v`
+        5. You can now start the server again as usual.
+    - **Why?**
+    Before version 5.0.0, `https://pod.example/profile/card`
+    would map to `file:///solid/profile/card`, with the _assumption_
+    that it uses content-type `text/turtle`.
+    Now, this URL will map to `file:///solid/profile/card$.ttl` instead,
+    which makes the content-type automatically detectable.
+    This fixes many of the old Content-Type-related bugs.
+    _More information: https://www.w3.org/DesignIssues/HTTPFilenameMapping.html_
+
 ## 4.4.0
 
 - Introduce a quota system. Delete the /settings/serverSide.ttl in the
   user's POD to disable, or edit to fit your resource constraints.
 
-
 #### Changelog is incomplete for much of the 4.x series
 
-
-## 4.1.0	
+## 4.1.0
 
 - Add support for Group Access Control Lists.
 - Fix `Vary` header.

README.md

@@ -167,7 +167,7 @@ $ solid start --help
 
     --root [value]                Root folder to serve (default: './data')
     --port [value]                SSL port to use
-    --serverUri [value]           Solid server uri (default: 'https://localhost:8443')
+    --server-uri [value]          Solid server uri (default: 'https://localhost:8443')
     --webid                       Enable WebID authentication and access control (uses HTTPS)
     --mount [value]               Serve on a specific URL path (default: '/')
     --config-path [value]
@@ -182,7 +182,7 @@ $ solid start --help
     --idp [value]                 Obsolete; use --multiuser
     --no-live                     Disable live support through WebSockets
     --proxy [value]               Obsolete; use --corsProxy
-    --corsProxy [value]           Serve the CORS proxy on this path
+    --cors-proxy [value]          Serve the CORS proxy on this path
     --suppress-data-browser       Suppress provision of a data browser
     --data-browser-path [value]   An HTML file which is sent to allow users to browse the data (eg using mashlib.js)
     --suffix-acl [value]          Suffix for acl files (default: '.acl')
@@ -191,18 +191,21 @@ $ solid start --help
     --error-pages [value]         Folder from which to look for custom error pages files (files must be named <error-code>.html -- eg. 500.html)
     --force-user [value]          Force a WebID to always be logged in (useful when offline)
     --strict-origin               Enforce same origin policy in the ACL
-    --useEmail                    Do you want to set up an email service?
+    --use-email                   Do you want to set up an email service?
     --email-host [value]          Host of your email service
     --email-port [value]          Port of your email service
     --email-auth-user [value]     User of your email service
     --email-auth-pass [value]     Password of your email service
-    --useApiApps                  Do you want to load your default apps on /api/apps?
+    --use-api-apps                Do you want to load your default apps on /api/apps?
     --api-apps [value]            Path to the folder to mount on /api/apps
     --redirect-http-from [value]  HTTP port or ','-separated ports to redirect to the solid server port (e.g. "80,8080").
     --server-name [value]         A name for your server (not required, but will be presented on your server's frontpage)
     --server-description [value]  A description of your server (not required)
     --server-logo [value]         A logo that represents you, your brand, or your server (not required)
-    -v, --verbose                 Print the logs to console
+    --enforce-toc                 Do you want to enforce Terms & Conditions for your service?
+    --toc-uri [value]             URI to your Terms & Conditions
+    --support-email [value]       The support email you provide for your users (not required)
+    -q, --quiet                   Do not print the logs to console
     -h, --help                    output usage information
  ```
 
@@ -226,8 +229,8 @@ docker run -p 8443:8443 --name solid node-solid-server
 ```
 
 This will enable you to login to solid on https://localhost:8443 and then create a new account
-but not yet use that account. After a new account is made you will need to create an entry for
-it in your local (/etc/)hosts file in line with the account and subdomain i.e.
+but not yet use that account. After a new account is made you will need to create an entry for 
+it in your local (/etc/)hosts file in line with the account and subdomain i.e. 
 
 127.0.0.1	newsoliduser.localhost
 
@@ -391,14 +394,13 @@ blacklist profanities by default.
 
 ## Quota
 
-By default, a file `serverSide.ttl` will be installed to new PODs. Its
-current function is to set a quota for disk usage of just 25 MB, which
-is what we can be sure the current prototype can tolerate under
-load. This file is not writeable to users, but as server administrator
-you can remove it if you don't want to impose a quota. It is currently
-adviceable to remove it rather than set a large quota, because the
-current implementation will impair write performance if there is a lot
-of data.
+By default, a file `serverSide.ttl.inactive` will be installed to new
+PODs. If you rename it to `serverSide.ttl`, it will currently set a
+quota for disk usage.  This file is not writeable to users, only
+server administrators who are authorized on the backend can modify
+it. It is currently adviceable to remove it or set it inactive rather
+than set a large quota, because the current implementation will impair
+write performance if there is a lot of data.
 
 ## Contribute to Solid
 

bin/lib/cli-utils.js

@@ -0,0 +1,70 @@
+const fs = require('fs-extra')
+const { cyan, bold } = require('colorette')
+const { URL } = require('url')
+const LDP = require('../../lib/ldp')
+const AccountManager = require('../../lib/models/account-manager')
+const SolidHost = require('../../lib/models/solid-host')
+
+module.exports.getAccountManager = getAccountManager
+module.exports.loadAccounts = loadAccounts
+module.exports.loadConfig = loadConfig
+module.exports.loadUsernames = loadUsernames
+
+/**
+ * Returns an instance of AccountManager
+ *
+ * @param {Object} config
+ * @param {Object} [options]
+ * @returns {AccountManager}
+ */
+function getAccountManager (config, options = {}) {
+  const ldp = options.ldp || new LDP(config)
+  const host = options.host || SolidHost.from({ port: config.port, serverUri: config.serverUri })
+  return AccountManager.from({
+    host,
+    store: ldp,
+    multiuser: config.multiuser
+  })
+}
+
+function loadConfig (program, options) {
+  let argv = {
+    ...options,
+    version: program.version()
+  }
+  let configFile = argv['configFile'] || './config.json'
+
+  try {
+    const file = fs.readFileSync(configFile)
+
+    // Use flags with priority over config file
+    const config = JSON.parse(file)
+    argv = { ...config, ...argv }
+  } catch (err) {
+    // No file exists, not a problem
+    console.log(cyan(bold('TIP')), 'create a config.json: `$ solid init`')
+  }
+
+  return argv
+}
+
+/**
+ *
+ * @param root
+ * @param [serverUri] If not set, hostname must be set
+ * @param [hostname] If not set, serverUri must be set
+ * @returns {*}
+ */
+function loadAccounts ({ root, serverUri, hostname }) {
+  const files = fs.readdirSync(root)
+  hostname = hostname || new URL(serverUri).hostname
+  const isUserDirectory = new RegExp(`.${hostname}$`)
+  return files
+    .filter(file => isUserDirectory.test(file))
+}
+
+function loadUsernames ({ root, serverUri }) {
+  const hostname = new URL(serverUri).hostname
+  return loadAccounts({ root, hostname })
+    .map(userDirectory => userDirectory.substr(0, userDirectory.length - hostname.length - 1))
+}

bin/lib/cli.js

@@ -2,6 +2,8 @@ const program = require('commander')
 const loadInit = require('./init')
 const loadStart = require('./start')
 const loadInvalidUsernames = require('./invalidUsernames')
+const loadMigrateLegacyResources = require('./migrateLegacyResources')
+const loadUpdateIndex = require('./updateIndex')
 const { spawnSync } = require('child_process')
 const path = require('path')
 
@@ -11,6 +13,8 @@ module.exports = function startCli (server) {
   loadInit(program)
   loadStart(program, server)
   loadInvalidUsernames(program)
+  loadMigrateLegacyResources(program)
+  loadUpdateIndex(program)
 
   program.parse(process.argv)
   if (program.args.length === 0) program.help()

bin/lib/common.js

@@ -1,17 +1,14 @@
 const fs = require('fs-extra')
 const Handlebars = require('handlebars')
 const path = require('path')
-const { URL } = require('url')
 
-const { loadConfig } = require('./common')
+const { getAccountManager, loadConfig, loadUsernames } = require('./cli-utils')
 const { isValidUsername } = require('../../lib/common/user-utils')
 const blacklistService = require('../../lib/services/blacklist-service')
 const { initConfigDir, initTemplateDirs } = require('../../lib/server-config')
 const { fromServerConfig } = require('../../lib/models/oidc-manager')
 
-const AccountManager = require('../../lib/models/account-manager')
 const EmailService = require('../../lib/services/email-service')
-const LDP = require('../../lib/ldp')
 const SolidHost = require('../../lib/models/solid-host')
 
 module.exports = function (program) {
@@ -28,7 +25,7 @@ module.exports = function (program) {
 
       const invalidUsernames = getInvalidUsernames(config)
       const host = SolidHost.from({ port: config.port, serverUri: config.serverUri })
-      const accountManager = getAccountManager(config, host)
+      const accountManager = getAccountManager(config, { host })
 
       if (options.notify) {
         return notifyUsers(invalidUsernames, accountManager, config)
@@ -96,23 +93,9 @@ async function deleteUsers (usernames, accountManager, config, host) {
   console.info(`Deleted ${deletingUsers.length} users succeeded`)
 }
 
-function getAccountManager (config, host) {
-  const ldp = new LDP(config)
-  return AccountManager.from({
-    host,
-    store: ldp,
-    multiuser: config.multiuser
-  })
-}
-
 function getInvalidUsernames (config) {
-  const files = fs.readdirSync(config.root)
-  const hostname = new URL(config.serverUri).hostname
-  const isUserDirectory = new RegExp(`.${hostname}$`)
-  return files
-    .filter(file => isUserDirectory.test(file))
-    .map(userDirectory => userDirectory.substr(0, userDirectory.length - hostname.length - 1))
-    .filter(username => !isValidUsername(username) || !blacklistService.validate(username))
+  const usernames = loadUsernames(config)
+  return usernames.filter(username => !isValidUsername(username) || !blacklistService.validate(username))
 }
 
 function listUsernames (usernames) {

bin/lib/invalidUsernames.js

@@ -0,0 +1,69 @@
+const fs = require('fs')
+const Path = require('path')
+const promisify = require('util').promisify
+const readdir = promisify(fs.readdir)
+const lstat = promisify(fs.lstat)
+const rename = promisify(fs.rename)
+
+/* Converts the old (pre-5.0.0) extensionless files to $-based files _with_ extensions
+ * to make them work in the new resource mapper (post-5.0.0).
+ * By default, all extensionless files (that used to be interpreted as Turtle) will now receive a '$.ttl' suffix. */
+/* https://www.w3.org/DesignIssues/HTTPFilenameMapping.html */
+
+module.exports = function (program) {
+  program
+    .command('migrate-legacy-resources')
+    .option('-p, --path <path>', 'Path to the data folder, defaults to \'data/\'')
+    .option('-s, --suffix <path>', 'The suffix to add to extensionless files, defaults to \'$.ttl\'')
+    .option('-v, --verbose', 'Path to the data folder')
+    .description('Migrate the data folder from node-solid-server 4 to node-solid-server 5')
+    .action(async (opts) => {
+      const verbose = opts.verbose
+      const suffix = opts.suffix || '$.ttl'
+      let paths = opts.path ? [ opts.path ] : [ 'data', 'config/templates' ]
+      paths = paths.map(path => path.startsWith(Path.sep) ? path : Path.join(process.cwd(), path))
+      try {
+        for (const path of paths) {
+          if (verbose) {
+            console.log(`Migrating files in ${path}`)
+          }
+          await migrate(path, suffix, verbose)
+        }
+      } catch (err) {
+        console.error(err)
+      }
+    })
+}
+
+async function migrate (path, suffix, verbose) {
+  const files = await readdir(path)
+  for (const file of files) {
+    const fullFilePath = Path.join(path, file)
+    const stat = await lstat(fullFilePath)
+    if (stat.isFile()) {
+      if (shouldMigrateFile(file)) {
+        const newFullFilePath = getNewFileName(fullFilePath, suffix)
+        if (verbose) {
+          console.log(`${fullFilePath}\n  => ${newFullFilePath}`)
+        }
+        await rename(fullFilePath, newFullFilePath)
+      }
+    } else {
+      if (shouldMigrateFolder(file)) {
+        await migrate(fullFilePath, suffix, verbose)
+      }
+    }
+  }
+}
+
+function getNewFileName (fullFilePath, suffix) {
+  return fullFilePath + suffix
+}
+
+function shouldMigrateFile (filename) {
+  return filename.indexOf('.') < 0
+}
+
+function shouldMigrateFolder (foldername) {
+  return foldername[0] !== '.'
+}