chore: add support to read text files larger than 16MB

Author: abose

src/extensibility/ExtensionManager.js

@@ -70,7 +70,7 @@ define(function (require, exports, module) {
         // never rejects
         return new Promise((resolve) => {
             const registryFile = FileSystem.getFileForPath(REGISTRY_CACHE_PATH);
-            FileUtils.readAsText(registryFile)
+            FileUtils.readAsText(registryFile, true, {ignoreFileSizeLimits: true})
                 .done(resolve)
                 .fail(function (err) {
                     console.error(`Registry cache not found ${REGISTRY_CACHE_PATH}`, err);

src/file/FileUtils.js

@@ -70,22 +70,32 @@ define(function (require, exports, module) {
      * Asynchronously reads a file as UTF-8 encoded text.
      * @param {!File} file File to read
      * @param {boolean?} bypassCache - an optional argument, if specified will read from disc instead of using cache.
+     * @param {object} [options]
+     * @param {boolean} [options.ignoreFileSizeLimits]. Will read larger files than 16MB limit. will bypassCache +
+     *          won't cache if enabled.
+     * @param {boolean} [options.doNotCache] will not cache if enabled. Auto-enabled if ignoreFileSizeLimits = true
      * @return {$.Promise} a jQuery promise that will be resolved with the
      *  file's text content plus its timestamp, or rejected with a FileSystemError string
      *  constant if the file can not be read.
      */
-    function readAsText(file, bypassCache) {
+    function readAsText(file, bypassCache, options = {}) {
         const result = new $.Deferred();
+        let doNotCache = options.doNotCache;
+        if(options.ignoreFileSizeLimits) {
+            bypassCache = true;
+            doNotCache = true;
+        }
 
-        file.read({bypassCache: bypassCache}, function (err, data, _encoding, stat) {
-            if(!err && typeof data !== "string"){
-                result.reject(FileSystemError.UNSUPPORTED_ENCODING);
-            } else if (!err) {
-                result.resolve(data, stat.mtime);
-            } else {
-                result.reject(err);
-            }
-        });
+        file.read({ bypassCache: bypassCache, ignoreFileSizeLimits: options.ignoreFileSizeLimits, doNotCache},
+            function (err, data, _encoding, stat) {
+                if(!err && typeof data !== "string"){
+                    result.reject(FileSystemError.UNSUPPORTED_ENCODING);
+                } else if (!err) {
+                    result.resolve(data, stat.mtime);
+                } else {
+                    result.reject(err);
+                }
+            });
 
         return result.promise();
     }

src/filesystem/File.js

@@ -94,8 +94,12 @@ define(function (require, exports, module) {
     /**
      * Read a file.
      *
-     * @param {Object=} options properties \{encoding: 'one of format supported here:
-     * https://developer.mozilla.org/en-US/docs/Web/API/TextDecoder/encoding'}
+     * @param {Object} options
+     * @param {string} [options.encoding] 'one of format supported here:
+     *        https://developer.mozilla.org/en-US/docs/Web/API/TextDecoder/encoding'
+     * @param {boolean} [options.ignoreFileSizeLimits] by default max file size that can be read is 16MB.
+     * @param {boolean} [options.doNotCache] will not cache if enabled. Auto-enabled if ignoreFileSizeLimits = true
+     *
      * @param {function (?string, string=, FileSystemStats=)} callback Callback that is passed the
      *              FileSystemError string or the file's contents and its stats.
      */
@@ -106,6 +110,9 @@ define(function (require, exports, module) {
             options.encoding = this._encoding;
         }
         options.encoding = options.encoding || this._encoding || "utf8";
+        if(options.ignoreFileSizeLimits) {
+            options.doNotCache = true;
+        }
 
         // We don't need to check isWatched() here because contents are only saved
         // for watched files. Note that we need to explicitly test this._contents

src/filesystem/impls/appshell/AppshellFileSystem.js

@@ -551,7 +551,7 @@ define(function (require, exports, module) {
      * If both calls fail, the error from the read call is passed back.
      *
      * @param {string} path
-     * @param {{encoding: string=, stat: FileSystemStats=}} options
+     * @param {{encoding: string=, stat: FileSystemStats=, ignoreFileSizeLimits: boolean=}} options
      * @param {function(?string, string=, FileSystemStats=)} callback
      */
     function readFile(path, options, callback) {
@@ -562,7 +562,7 @@ define(function (require, exports, module) {
         // callback to be executed when the call to stat completes
         //  or immediately if a stat object was passed as an argument
         function doReadFile(stat) {
-            if (stat.size > (FileUtils.MAX_FILE_SIZE)) {
+            if (!options.ignoreFileSizeLimits && stat.size > (FileUtils.MAX_FILE_SIZE)) {
                 callback(FileSystemError.EXCEEDS_MAX_FILE_SIZE);
             } else {
                 appshell.fs.readFile(path, encoding, function (_err, _data, encoding, preserveBOM) {