Merge pull request #911 from solid/feature/check-quota

Code to check quota on backend

Author: kjetilk

default-templates/new-account/settings/serverSide.ttl

@@ -0,0 +1,13 @@
+@prefix dct: <http://purl.org/dc/terms/>.
+@prefix pim: <http://www.w3.org/ns/pim/space#>.
+@prefix solid: <http://www.w3.org/ns/solid/terms#>.
+@prefix unit: <http://www.w3.invalid/ns#>.
+
+<>
+  a pim:ConfigurationFile;
+
+  dct:description "Administrative settings for the POD that the user can only read." .
+
+</>
+    solid:storageQuota "25000000" .
+    

default-templates/new-account/settings/serverSide.ttl.acl

@@ -0,0 +1,13 @@
+@prefix acl: <http://www.w3.org/ns/auth/acl#>.
+@prefix foaf: <http://xmlns.com/foaf/0.1/>.
+
+<#owner>
+    a acl:Authorization;
+
+    acl:agent
+        <{{webId}}>;
+
+    acl:accessTo <./serverSide.ttl>;
+
+    acl:mode acl:Read .
+    

lib/debug.js

@@ -13,3 +13,4 @@ exports.container = debug('solid:container')
 exports.accounts = debug('solid:accounts')
 exports.email = debug('solid:email')
 exports.ldp = debug('solid:ldp')
+exports.fs = debug('solid:fs')

lib/handlers/patch.js

@@ -10,6 +10,7 @@ const utils = require('../utils.js')
 const error = require('../http-error')
 const $rdf = require('rdflib')
 const crypto = require('crypto')
+const overQuota = require('../utils').overQuota
 
 const DEFAULT_TARGET_TYPE = 'text/turtle'
 
@@ -53,7 +54,7 @@ function patchHandler (req, res, next) {
   ])
   // Patch the graph and write it back to the file
   .then(([graph, patchObject]) => applyPatch(patchObject, graph, target))
-  .then(graph => writeGraph(graph, target))
+    .then(graph => writeGraph(graph, target, root, ldp.serverUri))
   // Send the result to the client
   .then(result => { res.send(result) })
   .then(next, next)
@@ -137,19 +138,27 @@ function applyPatch (patchObject, graph, target) {
 }
 
 // Writes the RDF graph to the given resource
-function writeGraph (graph, resource) {
+function writeGraph (graph, resource, root, serverUri) {
   debug('PATCH -- Writing patched file')
   return new Promise((resolve, reject) => {
     const resourceSym = graph.sym(resource.uri)
     const serialized = $rdf.serialize(resourceSym, graph, resource.uri, resource.contentType)
 
-    fs.writeFile(resource.file, serialized, {encoding: 'utf8'}, function (err) {
-      if (err) {
-        return reject(error(500, `Failed to write file after patch: ${err}`))
+    // First check if we are above quota
+    overQuota(root, serverUri).then((isOverQuota) => {
+      if (isOverQuota) {
+        return reject(error(413,
+          'User has exceeded their storage quota'))
       }
-      debug('PATCH -- applied successfully')
-      resolve('Patch applied successfully.\n')
-    })
+
+      fs.writeFile(resource.file, serialized, {encoding: 'utf8'}, function (err) {
+        if (err) {
+          return reject(error(500, `Failed to write file after patch: ${err}`))
+        }
+        debug('PATCH -- applied successfully')
+        resolve('Patch applied successfully.\n')
+      })
+    }).catch(() => reject(error(500, 'Error finding user quota')))
   })
 }
 

lib/ldp.js

@@ -10,6 +10,7 @@ const utils = require('./utils')
 const error = require('./http-error')
 const stringToStream = require('./utils').stringToStream
 const serialize = require('./utils').serialize
+const overQuota = require('./utils').overQuota
 const extend = require('extend')
 const rimraf = require('rimraf')
 const ldpContainer = require('./ldp-container')
@@ -266,24 +267,33 @@ class LDP {
       return callback(error(409,
         'PUT not supported on containers, use POST instead'))
     }
-    // First, create the enclosing directory, if necessary
-    const dirName = path.dirname(filePath)
-    mkdirp(dirName, (err) => {
-      if (err) {
-        debug.handlers('PUT -- Error creating directory: ' + err)
-        return callback(error(err,
-          'Failed to create the path to the new resource'))
+
+    // First check if we are above quota
+    overQuota(root, this.serverUri).then((isOverQuota) => {
+      if (isOverQuota) {
+        return callback(error(413,
+          'User has exceeded their storage quota'))
       }
-      // Directory created, now write the file
-      const file = stream.pipe(fs.createWriteStream(filePath))
-      file.on('error', function () {
-        callback(error(500, 'Error writing data'))
-      })
-      file.on('finish', function () {
-        debug.handlers('PUT -- Wrote data to: ' + filePath)
-        callback(null)
+
+      // Second, create the enclosing directory, if necessary
+      const dirName = path.dirname(filePath)
+      mkdirp(dirName, (err) => {
+        if (err) {
+          debug.handlers('PUT -- Error creating directory: ' + err)
+          return callback(error(err,
+            'Failed to create the path to the new resource'))
+        }
+        // Directory created, now write the file
+        const file = stream.pipe(fs.createWriteStream(filePath))
+        file.on('error', function () {
+          callback(error(500, 'Error writing data'))
+        })
+        file.on('finish', function () {
+          debug.handlers('PUT -- Wrote data to: ' + filePath)
+          callback(null)
+        })
       })
-    })
+    }).catch(() => callback(error(500, 'Error finding user quota')))
   }
 
   exists (hostname, path, callback) {

lib/utils.js

@@ -13,12 +13,18 @@ module.exports.debrack = debrack
 module.exports.stripLineEndings = stripLineEndings
 module.exports.fullUrlForReq = fullUrlForReq
 module.exports.routeResolvedFile = routeResolvedFile
+module.exports.getQuota = getQuota
+module.exports.overQuota = overQuota
 
 const fs = require('fs-extra')
 const path = require('path')
+const util = require('util')
 const $rdf = require('rdflib')
 const from = require('from2')
 const url = require('url')
+const debug = require('./debug').fs
+const getSize = require('get-folder-size')
+var ns = require('solid-namespace')($rdf)
 
 /**
  * Returns a fully qualified URL from an Express.js Request object.
@@ -239,3 +245,60 @@ function routeResolvedFile (router, path, file, appendFileName = true) {
   const fullFile = require.resolve(file)
   router.get(fullPath, (req, res) => res.sendFile(fullFile))
 }
+
+/**
+ * Returns the number of bytes that the user owning the requested POD
+ * may store or Infinity if no limit
+ */
+
+async function getQuota (root, serverUri) {
+  const filename = path.join(root, 'settings/serverSide.ttl')
+  var prefs
+  try {
+    prefs = await _asyncReadfile(filename)
+  } catch (error) {
+    debug('Setting no quota. While reading serverSide.ttl, got ' + error)
+    return Infinity
+  }
+  var graph = $rdf.graph()
+  const storageUri = serverUri + '/'
+  try {
+    $rdf.parse(prefs, graph, storageUri, 'text/turtle')
+  } catch (error) {
+    throw new Error('Failed to parse serverSide.ttl, got ' + error)
+  }
+  return Number(graph.anyValue($rdf.sym(storageUri), ns.solid('storageQuota'))) || Infinity
+}
+
+/**
+ * Returns true of the user has already exceeded their quota, i.e. it
+ * will check if new requests should be rejected, which means they
+ * could PUT a large file and get away with it.
+ */
+
+async function overQuota (root, serverUri) {
+  let quota = await getQuota(root, serverUri)
+  if (quota === Infinity) {
+    return false
+  }
+  // TODO: cache this value?
+  var size = await actualSize(root)
+  return (size > quota)
+}
+
+/**
+ * Returns the number of bytes that is occupied by the actual files in
+ * the file system. IMPORTANT NOTE: Since it traverses the directory
+ * to find the actual file sizes, this does a costly operation, but
+ * neglible for the small quotas we currently allow. If the quotas
+ * grow bigger, this will significantly reduce write performance, and
+ * so it needs to be rewritten.
+ */
+
+function actualSize (root) {
+  return util.promisify(getSize)(root)
+}
+
+function _asyncReadfile (filename) {
+  return util.promisify(fs.readFile)(filename, 'utf-8')
+}