🐛 FIX: Remove disk cache when version not found (#404)

Avoid package sync delay

Author: fengmk2

.autod.conf.js

@@ -1,6 +1,6 @@
 This software is licensed under the MIT License.
 
-Copyright (c) 2016 - present cnpm and other contributors
+Copyright (c) 2016-present cnpm and other contributors
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

LICENSE.txt

@@ -2,7 +2,8 @@
 
 const debug = require('debug')('npminstall:download:npm');
 const bytes = require('bytes');
-const fs = require('mz/fs');
+const { createWriteStream, createReadStream, rmSync } = require('fs');
+const fs = require('fs/promises');
 const path = require('path');
 const crypto = require('crypto');
 const tar = require('tar');
@@ -80,6 +81,8 @@ async function resolve(pkg, options) {
   let fixScripts;
 
   if (!realPkgVersion) {
+    // remove disk cache
+    await removeCacheInfo(pkg.name, options);
     throw new Error(`[${pkg.displayName}] Can\'t find package ${pkg.name}@${pkg.rawSpec}`);
   }
 
@@ -113,6 +116,8 @@ async function resolve(pkg, options) {
 
   const realPkg = packageMeta.versions[realPkgVersion];
   if (!realPkg) {
+    // remove disk cache
+    await removeCacheInfo(pkg.name, options);
     throw new Error(`[${pkg.displayName}] Can\'t find package ${pkg.name}\'s version: ${realPkgVersion}`);
   }
 
@@ -131,48 +136,69 @@ async function resolve(pkg, options) {
   return realPkg;
 }
 
-function getScope(name) {
+function _getScope(name) {
   if (name[0] === '@') return name.slice(0, name.indexOf('/'));
 }
 
-async function getFullPackageMeta(name, globalOptions) {
+async function _getCacheInfo(fullname, globalOptions) {
   // check name has scope
   let registry = globalOptions.registry;
-  const scope = getScope(name);
+  const scope = _getScope(fullname);
   if (scope) {
     registry = config.get(scope + ':registry') || globalOptions.registry;
   }
-
-  const pkgUrl = utils.formatPackageUrl(registry, name);
+  const info = {
+    pkgUrl: utils.formatPackageUrl(registry, fullname),
+    cacheFile: '',
+    cache: null,
+  };
   if (!globalOptions.cacheDir) {
-    const result = await _fetchFullPackageMeta(pkgUrl, globalOptions);
-    return result.data;
+    return info;
   }
-  const hash = utility.md5(pkgUrl);
+  const hash = utility.md5(info.pkgUrl);
   const parentDir = path.join(globalOptions.cacheDir, 'manifests', hash[0], hash[1], hash[2]);
   // { etag, age, headers, manifests }
-  const cacheFile = path.join(parentDir, `${hash}.json`);
-  const exists = await fs.exists(cacheFile);
+  info.cacheFile = path.join(parentDir, `${hash}.json`);
+  const exists = await utils.exists(info.cacheFile);
   // cache not exists
   if (!exists) {
     await utils.mkdirp(parentDir);
-    return await _fetchFullPackageMetaWithCache(pkgUrl, globalOptions, cacheFile);
+    return info;
   }
   // cache exists
-  const cacheContent = await fs.readFile(cacheFile);
-  let cache;
+  const cacheContent = await fs.readFile(info.cacheFile);
   try {
-    cache = JSON.parse(cacheContent);
+    info.cache = JSON.parse(cacheContent);
   } catch (_) {
-    globalOptions.console.warn('[npminstall:download:npm] Ignore invalid cache file %s', cacheFile);
+    globalOptions.console.warn('[npminstall:download:npm] Ignore invalid cache file %s', info.cacheFile);
+  }
+  return info;
+}
+
+async function removeCacheInfo(fullname, globalOptions) {
+  const info = await _getCacheInfo(fullname, globalOptions);
+  if (info.cache) {
+    await fs.rm(info.cacheFile, { force: true });
+  }
+}
+
+async function getFullPackageMeta(fullname, globalOptions) {
+  const info = await _getCacheInfo(fullname, globalOptions);
+  if (!info.cacheFile) {
+    const result = await _fetchFullPackageMeta(info.pkgUrl, globalOptions);
+    return result.data;
+  }
+  // cache file not exists
+  if (!info.cache) {
+    return await _fetchFullPackageMetaWithCache(info.pkgUrl, globalOptions, info.cacheFile);
   }
   // check is expired or not
-  if (cache && cache.expired > Date.now()) {
+  if (info.cache.expired > Date.now()) {
     globalOptions.totalCacheJSONCount += 1;
-    return cache.manifests;
+    return info.cache.manifests;
   }
   // use etag to request
-  return await _fetchFullPackageMetaWithCache(pkgUrl, globalOptions, cacheFile, cache);
+  return await _fetchFullPackageMetaWithCache(info.pkgUrl, globalOptions, info.cacheFile, info.cache);
 }
 
 async function _fetchFullPackageMetaWithCache(pkgUrl, globalOptions, cacheFile, cache) {
@@ -384,7 +410,7 @@ async function download(pkg, options) {
         // ignore https protocol check on: node_modules/node-pre-gyp/lib/util/versioning.js
         if (/node-pre-gyp install/.test(pkg.scripts.install)) {
           const versioningFile = path.join(ungzipDir, 'node_modules/node-pre-gyp/lib/util/versioning.js');
-          if (await fs.exists(versioningFile)) {
+          if (await utils.exists(versioningFile)) {
             let content = await fs.readFile(versioningFile, 'utf-8');
             content = content.replace('if (protocol === \'http:\') {',
               'if (false && protocol === \'http:\') { // hack by npminstall');
@@ -426,7 +452,7 @@ async function download(pkg, options) {
         options.console.info('%s download from binary mirror: %j, targetPlatform: %s',
           chalk.gray(`${pkg.name}@${pkg.version}`), binaryMirror, targetPlatform);
         const downloadFile = path.join(ungzipDir, 'lib/tasks/download.js');
-        if (await fs.exists(downloadFile)) {
+        if (await utils.exists(downloadFile)) {
           let content = await fs.readFile(downloadFile, 'utf-8');
           // return version ? prepend('desktop/' + version) : prepend('desktop');
           const afterContent = 'return "' + binaryMirror.host + '/" + version + "/' + targetPlatform + '/cypress.zip"; // hack by npminstall\n';
@@ -446,7 +472,7 @@ async function download(pkg, options) {
   if (pkg.name === 'node-pre-gyp') {
     // ignore https protocol check on: lib/util/versioning.js
     const versioningFile = path.join(ungzipDir, 'lib/util/versioning.js');
-    if (await fs.exists(versioningFile)) {
+    if (await utils.exists(versioningFile)) {
       let content = await fs.readFile(versioningFile, 'utf-8');
       content = content.replace('if (protocol === \'http:\') {',
         'if (false && protocol === \'http:\') { // hack by npminstall');
@@ -527,7 +553,7 @@ async function getTarballStream(tarballUrl, pkg, options) {
   paths.push(pkg.name);
   const parentDir = path.join(...paths);
   const tarballFile = path.join(parentDir, `${pkg.version}-${pkg.dist.shasum}.tgz`);
-  let exists = await fs.exists(tarballFile);
+  let exists = await utils.exists(tarballFile);
   if (!exists) {
     // try to remove expired tmp dirs
     const dirs = [];
@@ -552,25 +578,25 @@ async function getTarballStream(tarballUrl, pkg, options) {
       timeout: options.streamingTimeout || options.timeout,
       followRedirect: true,
       formatRedirectUrl,
-      writeStream: fs.createWriteStream(tmpFile),
+      writeStream: createWriteStream(tmpFile),
     }, options);
 
     if (result.status !== 200) {
       throw new Error(`Download ${tarballUrl} status: ${result.status} error, should be 200`);
     }
     // make sure tarball file is not exists again
-    exists = await fs.exists(tarballFile);
+    exists = await utils.exists(tarballFile);
     if (!exists) {
       try {
         await fs.rename(tmpFile, tarballFile);
       } catch (err) {
         if (err.code === 'EPERM') {
           // Error: EPERM: operation not permitted, rename
-          exists = await fs.exists(tarballFile);
+          exists = await utils.exists(tarballFile);
           if (exists) {
             // parallel execution case same file exists, ignore rename error
             // clean tmpFile
-            await fs.unlink(tmpFile);
+            await fs.rm(tmpFile, { force: true });
           } else {
             // rename error
             throw err;
@@ -582,15 +608,15 @@ async function getTarballStream(tarballUrl, pkg, options) {
       }
     } else {
       // clean tmpFile
-      await fs.unlink(tmpFile);
+      await fs.rm(tmpFile, { force: true });
     }
     const stat = await fs.stat(tarballFile);
     debug('[%s@%s] saved %s %s => %s',
       pkg.name, pkg.version, bytes(stat.size), tarballUrl, tarballFile);
     options.totalTarballSize += stat.size;
   }
 
-  const stream = fs.createReadStream(tarballFile);
+  const stream = createReadStream(tarballFile);
   stream.tarballFile = tarballFile;
   stream.tarballUrl = tarballUrl;
   return stream;
@@ -752,12 +778,12 @@ function checkShasumAndUngzip(ungzipDir, readstream, pkg, useTarFormat, options)
         // make sure readstream will be destroy
         destroy(readstream);
         err.message += ` (${pkg.name}@${pkg.version})`;
-        if (readstream.tarballFile && fs.existsSync(readstream.tarballFile)) {
+        if (readstream.tarballFile && utils.existsSync(readstream.tarballFile)) {
           err.message += ` (${readstream.tarballFile})`;
           debug('[%s@%s] remove tarball file: %s, because %s',
             pkg.name, pkg.version, readstream.tarballFile, err);
           // remove tarball cache file
-          fs.unlinkSync(readstream.tarballFile);
+          rmSync(readstream.tarballFile, { force: true });
         }
         return reject(err);
       }
@@ -792,8 +818,8 @@ function checkShasumAndUngzip(ungzipDir, readstream, pkg, useTarFormat, options)
   });
 }
 
-async function replaceHostInFile(pkg, filepath, binaryMirror, options) {
-  const exists = await fs.exists(filepath);
+async function replaceHostInFile(pkg, filepath, binaryMirror, globalOptions) {
+  const exists = await utils.exists(filepath);
   if (!exists) {
     return;
   }
@@ -827,7 +853,7 @@ async function replaceHostInFile(pkg, filepath, binaryMirror, options) {
   }
   debug('%s: \n%s', filepath, content);
   await fs.writeFile(filepath, content);
-  options.console.info('%s download from mirrors: %j, changed file: %s',
+  globalOptions.console.info('%s download from mirrors: %j, changed file: %s',
     chalk.gray(`${pkg.name}@${pkg.version}`),
     replaceHostMap,
     filepath);

lib/download/npm.js

@@ -1,7 +1,8 @@
 'use strict';
 
 const debug = require('debug')('npminstall:utils');
-const fs = require('mz/fs');
+const fs = require('fs/promises');
+const { accessSync } = require('fs');
 const path = require('path');
 const cp = require('child_process');
 const urlparse = require('url').parse;
@@ -21,6 +22,24 @@ const url = require('url');
 const config = require('./config');
 const get = require('./get');
 
+exports.exists = async filepath => {
+  try {
+    await fs.access(filepath);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+exports.existsSync = filepath => {
+  try {
+    accessSync(filepath);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
 exports.hasOwnProp = (target, key) => target.hasOwnProperty(key);
 /**
  *
@@ -44,7 +63,7 @@ exports.pruneJSON = async (filepath, depName) => {
 };
 
 exports.readJSON = async filepath => {
-  if (!(await fs.exists(filepath))) {
+  if (!(await exports.exists(filepath))) {
     return {};
   }
   const content = await fs.readFile(filepath, 'utf8');
@@ -118,7 +137,7 @@ exports.forceSymlink = async (src, dest, type) => {
 
   const destDir = path.dirname(dest);
   // check if destDir is not exist
-  if (!(await fs.exists(destDir))) {
+  if (!(await exports.exists(destDir))) {
     await mkdirp(destDir);
   }
 
@@ -354,7 +373,7 @@ exports.copyInstall = async (src, options) => {
   // 4. if already installed, return with exists = true
   // 5. if not installed, copy and return with exists = false
   const pkgpath = path.join(src, 'package.json');
-  if (!(await fs.exists(pkgpath))) {
+  if (!(await exports.exists(pkgpath))) {
     throw new Error(`package.json missed(${pkgpath})`);
   }
   const realPkg = await exports.readPackageJSON(src);

lib/utils.js

@@ -20,8 +20,7 @@
     "test-cov": "egg-bin cov -t 2000000",
     "test-local": "npm_china=true local=true egg-bin test -t 2000000",
     "lint": "eslint . --fix",
-    "ci": "npm run lint && npm run test-cov",
-    "autod": "autod"
+    "ci": "npm run lint && npm run test-cov"
   },
   "dependencies": {
     "agentkeepalive": "^4.0.2",
@@ -57,21 +56,18 @@
     "uuid": "^3.3.2"
   },
   "devDependencies": {
-    "autod": "3",
-    "coffee": "^5.2.1",
-    "egg-bin": "^4.13.1",
-    "eslint": "7",
-    "eslint-config-egg": "7",
-    "eslint-plugin-jsdoc": "^4.1.1",
-    "git-contributor": "^1.0.10",
-    "http-proxy": "^1.18.1",
-    "mm": "^2.5.0"
+    "coffee": "5",
+    "egg-bin": "5",
+    "eslint": "8",
+    "eslint-config-egg": "12",
+    "git-contributor": "1",
+    "http-proxy": "1",
+    "mm": "3"
   },
   "homepage": "https://github.com/cnpm/npminstall",
   "repository": {
     "type": "git",
-    "url": "git://github.com/cnpm/npminstall.git",
-    "web": "https://github.com/cnpm/npminstall"
+    "url": "git://github.com/cnpm/npminstall.git"
   },
   "bugs": {
     "url": "https://github.com/cnpm/npminstall/issues"