build: cache consistency in web distributions

abose · abose · commit 2bd38f45f9b8 · 2024-02-29T22:40:00.000+05:30
diff --git a/gulpfile.js/index.js b/gulpfile.js/index.js
@@ -386,7 +386,7 @@ function _getFileDetails(path) {
         {encoding: null});
     return {
         sizeBytes: data.length,
-        hash: crypto.createHash("md5").update(data).digest("hex")
+        hash: crypto.createHash("sha256").update(data).digest("hex")
     };
 }
 
diff --git a/package.json b/package.json
@@ -71,8 +71,9 @@
         "_patchVersionBump": "gulp patchVersionBump",
         "_majorVersionBump": "gulp majorVersionBump",
         "serve": "http-server . -p 8000 -c-1",
-        "serveWithWebCache": "http-server ./src -p 8000 -c-1",
-        "serveExternal": "npm run _releaseWebCache && http-server . -p 8000 -a 0.0.0.0 --log-ip true -c-1",
+        "_serveWithWebCacheHelp": "echo !!!Make sure to npm run release:dev/stageing/prod before testing the cache!!!",
+        "serveWithWebCache": "npm run _releaseWebCache && npm run _serveWithWebCacheHelp && http-server ./dist -p 8000 -c-1",
+        "serveExternal": "http-server . -p 8000 -a 0.0.0.0 --log-ip true -c-1",
         "createJSDocs": "gulp createJSDocs && git add docs",
         "_translateStrings": "gulp translateStrings",
         "_minify": "r.js -o require.min.config.js && echo this is untested see https://stackoverflow.com/questions/14337970/minifying-requirejs-javascript-codebase-to-a-single-file"
diff --git a/src/assets/default-project/en/index.html b/src/assets/default-project/en/index.html
@@ -11,8 +11,8 @@
         <div id="mainDiv">
             <img id="logo" alt="logo" src="images/phoenix-logo.svg" />
             <div id="MainText">
-                <h1>Phoenix</h1>
-                <span>Modern, Open-source, IDE For The Web <br /></span><br />
+                <h1>Phoenix Code</h1>
+                <span>Code Creatively: Visual Editing Tailored for Developers<br /></span><br />
             </div>
             <div class="video-container">
                 <a
diff --git a/src/extensionsIntegrated/Phoenix/newly-added-features.js b/src/extensionsIntegrated/Phoenix/newly-added-features.js
@@ -54,17 +54,6 @@ define(function (require, exports, module) {
         }, 3000);
     }
 
-    function _cacheUpdatedCB(err) {
-        if(err) {
-            Metrics.countEvent(Metrics.EVENT_TYPE.PLATFORM, "cache", "errorRefresh");
-            return;
-        }
-        Metrics.countEvent(Metrics.EVENT_TYPE.PLATFORM, "cache",
-            `${window.Phoenix.cache.updatePendingReloadReason}.done`);
-        Metrics.countEvent(Metrics.EVENT_TYPE.PLATFORM, "cache", `updateCount`,
-            window.Phoenix.cache.updatedFilesCount||0);
-    }
-
     async function _readMarkdownTextFile() {
         try{
             let markdownFile = FileSystem.getFileForPath(_getUpdateMarkdownLocalPath());
@@ -93,11 +82,5 @@ define(function (require, exports, module) {
         if(!Phoenix.firstBoot && !window.testEnvironment){
             _showNewUpdatesIfPresent();
         }
-        if(!Phoenix.browser.isTauri && window.refreshServiceWorkerCache) {
-            // window.refreshServiceWorkerCache is only present if the page loads service worker. Not available
-            // in integ test windows as they don't load service workers.
-            Metrics.countEvent(Metrics.EVENT_TYPE.PLATFORM, "cache", "doRefresh");
-            window.refreshServiceWorkerCache(_cacheUpdatedCB);
-        }
     };
 });
diff --git a/src/index.html b/src/index.html
@@ -52,6 +52,13 @@
     <script>window.PHOENIX_APP_CACHE_VERSION="LATEST";</script>
 <!--    The app cache version is set by the pipeline version update script. Do not update manually.-->
     <script>
+        if(["dev.phcode.dev", "staging.phcode.dev"].includes(location.hostname)
+            && localStorage.getItem("devDomainsEnabled") !== "true"){
+            alert("Hello explorer, you have reached a development version of phcode.dev that is not for general use and could be very unstable." +
+                `\n\nIf you know what you are doing and what to visit the dev site, please go to https://${location.hostname}/devEnable.html to enable dev site and visit again.` +
+                "\n\nYou will now be redirected to phcode.dev.");
+            window.location = "https://phcode.dev";
+        }
         if(location.href.startsWith("tauri://") || location.href.startsWith('https://tauri.localhost')){
             const errorMessage = `You should use custom protocol phtauri:// instead of tauri protocol ${location.href} .`;
             alert(errorMessage);
@@ -335,90 +342,90 @@
             window.PhoenixBaseURL = baseUrl;
         }());
     </script>
+    <script src="phoenix/virtualfs.js"></script>
     <script>
-        if(["dev.phcode.dev", "staging.phcode.dev"].includes(location.hostname)
-            && localStorage.getItem("devDomainsEnabled") !== "true"){
-            alert("Hello explorer, you have reached a development version of phcode.dev that is not for general use and could be very unstable." +
-                `\n\nIf you know what you are doing and what to visit the dev site, please go to https://${location.hostname}/devEnable.html to enable dev site and visit again.` +
-                "\n\nYou will now be redirected to phcode.dev.");
-            window.location = "https://phcode.dev";
-        }
         /*Bootstrap cache check. since index.html is always loaded network first, the cache reset code will always
         be guaranteed to be hit on app update.
         Note that this cannot be moved to a separate file and should be in index.html due to above reason.
         This is a nuke cache option. Though mostly safe, Use it wisely to prevent slow startup when resetting.
         */
-        const urlParams = new URLSearchParams(window.location.search || "");
-        const LESS_REFRESH_SCHEDULED_KEY = "lessRefreshScheduled";
-        const shouldRefreshLess = window.localStorage.getItem(LESS_REFRESH_SCHEDULED_KEY) === 'yes';
-        const CACHE_NAME_EVERYTHING = "everything";
-        function _resetCacheIfNeeded() {
-            if(Phoenix.browser.isTauri) {
-                // no web cache in desktop builds
-                return;
-            }
-            const cacheKey = "browserCacheVersionKey";
-            const newCacheVersion = "V16"; // just increment this number to V2, v3 etc. to force clear the cached content.
-            const lastClearedVersion = window.localStorage.getItem(cacheKey);
-            if(lastClearedVersion === null){
-                // First load, no cache, return.
-                localStorage.setItem(cacheKey, newCacheVersion);
+        async function _resetCacheIfNeeded() {
+            if(Phoenix.browser.isTauri || Phoenix.isTestWindow) {
+                // no web cache in desktop builds ot test windows
                 return;
             }
             if(!window.caches){
-                console.error("CacheStorage Reset: API not supported by browser.");
+                console.error("CacheStorage: API not supported by browser.");
             }
-            if(lastClearedVersion !== newCacheVersion) {
-                window.cacheRefreshInProgress = true;
-                let openedCache;
-                console.log("CacheStorage Reset: Reset cache in progress..");
-                caches.open(CACHE_NAME_EVERYTHING)
-                    .then(cache => {
-                        openedCache = cache;
-                        return cache.keys();
-                    })
-                    .then(keys => {
-                        let promises = []
-                        for(let key of keys){
-                            promises.push(openedCache.delete(key));
+
+            const LESS_REFRESH_SCHEDULED_KEY = "lessRefreshScheduled";
+
+            function _readFileSafe(readPath) {
+                return new Promise(resolve =>{
+                    fs.readFile(readPath, "utf8", (err, text)=>{
+                        if(err){
+                            console.error("Could not read", readPath)
+                            resolve(null);
+                            return;
                         }
-                        console.log(`CacheStorage Reset: Deleting ${promises.length} cache entries from ${CACHE_NAME_EVERYTHING}`);
-                        return Promise.all(promises);
-                    }).then(() => {
-                        console.log("CacheStorage Reset: Cache successfully reset");
-                        let totalWaitTime = 0, waitTime = 100, maxWaitTime = 3000;
-                        let intervalTimer = setInterval(()=>{
-                            // wait for less to get loaded. less caches css in local storage in production urls
-                            // and might not load new css classes if we don't reset. less doesn't cache in localhost.
-                            totalWaitTime += waitTime;
-                            if(window.less && less.refresh){
-                                localStorage.setItem(cacheKey, newCacheVersion);
-                                localStorage.setItem(LESS_REFRESH_SCHEDULED_KEY, "yes");
-                                less.refresh(true).finally(()=>{
-                                    localStorage.setItem(LESS_REFRESH_SCHEDULED_KEY, "no");
-                                    location.reload();
-                                });
-                                clearInterval(intervalTimer);
-                            } else if(totalWaitTime > maxWaitTime){
-                                // ignore less refresh, the app load itself is critical. less refresh we will try later
-                                localStorage.setItem(cacheKey, newCacheVersion);
-                                localStorage.setItem(LESS_REFRESH_SCHEDULED_KEY, "yes");
-                                location.reload();
-                            }
-                        }, waitTime);
-                    }).catch( e => {
-                        console.error("Error while resetting cache", e);
-                        window.cacheClearError = e;
-                        window.cacheRefreshInProgress = false;
-                        // try our luck loading phoenix as cache reset failed
-                        if(window._loadPhoenixAfterSplashScreen) {
-                            window._loadPhoenixAfterSplashScreen();
+                        resolve(text);
+                    });
+                });
+            }
+
+            function _writeFileSafe(writePath, text) {
+                return new Promise(resolve =>{
+                    fs.writeFile(writePath, text, "utf8", (err)=>{
+                        if(err){
+                            console.error("Could not written", writePath)
                         }
-                        // throw again so that bugsnag can report if initialised. else bugsnag
-                        // will report the error when it comes online via window.cacheClearError
-                        throw e;
+                        resolve(err);
                     });
+                });
+            }
+
+            const WEB_CACHE_FILE_PATH = "/webCacheVersion.txt";
+            /**
+             * Clears legacy caches from the browser storage. This function is intended to be used
+             * when migrating to a new cache management strategy. Safe to be deleted after sep-2024
+             * @return {Promise<void>}
+             */
+            async function clearLegacyCache() {
+                // as we switched to new versioned cache management, we will be clearing all the legacy cache.
+                const legacyCacheName = "browserCacheVersionKey";
+                const lastClearedVersion = window.localStorage.getItem(legacyCacheName);
+                if(lastClearedVersion){
+                    // legacy cache detected
+                    console.log("Legacy cache detected, resetting...");
+                    localStorage.removeItem(legacyCacheName);
+                    await window.caches.delete(legacyCacheName);
+                    localStorage.setItem(LESS_REFRESH_SCHEDULED_KEY, "yes");
+                    if(window.fs){
+                        await _writeFileSafe(WEB_CACHE_FILE_PATH, PHOENIX_APP_CACHE_VERSION);
+                        location.reload();
+                    } else {
+                        location.reload();
+                    }
+                }
             }
+            await clearLegacyCache();
+
+            const currentCacheName = await _readFileSafe(WEB_CACHE_FILE_PATH);
+            if(!currentCacheName) {
+                // fresh install, no cache
+                console.log("No cache detected, ignore if first boot or safari ITP.");
+                await _writeFileSafe(WEB_CACHE_FILE_PATH, PHOENIX_APP_CACHE_VERSION);
+                // on next reload, the new cache will be active and populated by service worker
+            } else if(currentCacheName !== PHOENIX_APP_CACHE_VERSION) {
+                // This is an upgrade we could in the future use the cacheManifest.json to increase update speed
+                // by only selectively updating the changed files.
+                console.log("Old version cache detected", currentCacheName, "purging and setting", PHOENIX_APP_CACHE_VERSION);
+                await window.caches.delete(currentCacheName);
+                await _writeFileSafe(WEB_CACHE_FILE_PATH, PHOENIX_APP_CACHE_VERSION);
+                localStorage.setItem(LESS_REFRESH_SCHEDULED_KEY, "yes");
+            }
+
+            const shouldRefreshLess = window.localStorage.getItem(LESS_REFRESH_SCHEDULED_KEY) === 'yes';
             if(shouldRefreshLess){
                 let lessRefreshInterval = setInterval(()=>{
                     // wait for less to get loaded. less caches css in local storage in production urls
@@ -475,11 +482,6 @@
                 document.body.classList.add('dark'); // This will later be overridden by the theme manager as required.
             }
             _removeSplashScreenIfNeeded();
-            if(window.cacheRefreshInProgress){
-                // We should not load anything while the cache is inconsistent.
-                // A page reload will be scheduled on successful clear.
-                return;
-            }
             var loadJS = function(url, implementationCode, location, dataMainValue){
                 //url is URL of external file, implementationCode is the code
                 //to be called from the file, location is the location to
@@ -505,7 +507,7 @@
                     });
             } else {
                 const interval = setInterval(()=>{
-                    if(PhStore){
+                    if(window.PhStore){
                         clearInterval(interval);
                         PhStore.storageReadyPromise
                             .finally(()=>{
@@ -528,7 +530,6 @@
     https://developer.mozilla.org/en-US/docs/Web/HTML/Element/script#attr-defer
     -->
     <!-- Import the phoenix browser virtual file system -->
-    <script src="phoenix/virtualfs.js" defer></script>
     <script src="phoenix/shell.js" type="module" defer></script>
     <script src="phoenix/virtual-server-loader.js" type="module" defer></script>
 <!--    node loader should come only after fs libs are loaded as we init fs libs with node fs urls-->
diff --git a/src/main.js b/src/main.js
@@ -154,8 +154,8 @@ define(function (require) {
         try{
             require(["brackets"]);
         } catch (err) {
-            // try a cache refresh (not a full reset). this will happen in the service worker in the background
-            window.refreshServiceWorkerCache && window.refreshServiceWorkerCache();
+            // try a cache reset
+            window._resetCacheIfNeeded && window._resetCacheIfNeeded();
             // metrics api might not be available here as we were seeing no metrics raised. Only bugsnag there.
             window.logger && window.logger.reportError(err,
                 'Critical error when loading brackets. Trying to reload again.');
diff --git a/src/phoenix/virtual-server-loader.js b/src/phoenix/virtual-server-loader.js
@@ -98,55 +98,15 @@ if (!window.__TAURI__ && _isServiceWorkerLoaderPage() && 'serviceWorker' in navi
         updateViaCache: 'none'
     });
 
-    let isServerReady = false;
-
     window.Phoenix.cache = {};
 
-    // refreshServiceWorkerCache should be done after app load to prevent mixed js script content load. Ie,
-    // if we do the cache reset now, some scripts loaded may be from cache and some from the new version.
-    window.refreshServiceWorkerCache = function (doneCB) {
-        if(!isServerReady){
-            setTimeout(()=>{
-                window.refreshServiceWorkerCache(doneCB);
-            }, 100);
-            return;
-        }
-        logger.leaveTrail(`Service worker loader: triggering REFRESH_CACHE`);
-        wb.messageSW({
-            type: 'REFRESH_CACHE'
-        }).then(({updatedFilesCount})=>{
-            logger.leaveTrail(`Service worker loader: refresh cache updatedFilesCount: `+ updatedFilesCount);
-            window.Phoenix.cache.updatePendingReloadReason = "refreshCache";
-            window.Phoenix.cache.updatedFilesCount = updatedFilesCount || 0;
-            if(updatedFilesCount){
-                let lessRefreshInterval = setInterval(()=>{
-                    // wait for less to get loaded. less caches css in local storage in production urls
-                    // and might not load new css classes if we don't reset. less doesn't cache in localhost.
-                    if(window.less && window.less.refresh){
-                        window.less.refresh(true);
-                        clearInterval(lessRefreshInterval);
-                    }
-                }, 500);
-            }
-            if(doneCB) {
-                doneCB();
-            }
-        }).catch(err=>{
-            console.error("Service worker loader: Error while triggering refresh cache", err);
-            if(doneCB) {
-                doneCB("REFRESH_CACHE Error");
-            }
-        });
-    };
-
     // Hoist service worker comm to window for everyone be able to communicate with the sw.
     window.messageSW = function (params) {
         return wb.messageSW(params);
     };
 
     function serverReady() {
         console.log('Service worker loader: Server ready.');
-        isServerReady = true;
         wb.messageSW({
             type: 'INIT_PHOENIX_CONFIG',
             debugMode: window.logger.logToConsolePref === 'true',
diff --git a/src/virtual-server-main.js b/src/virtual-server-main.js

Original file line number	Diff line number	Diff line change
`@@ -386,7 +386,7 @@ function _getFileDetails(path) {`
`386`	`386`	`{encoding: null});`
`387`	`387`	`return {`
`388`	`388`	`sizeBytes: data.length,`
`389`		`- hash: crypto.createHash("md5").update(data).digest("hex")`
	`389`	`+ hash: crypto.createHash("sha256").update(data).digest("hex")`
`390`	`390`	`};`
`391`	`391`	`}`
`392`	`392`