check unverified packages when reading the directory

Author: ricardo-devis-agullo

src/registry/domain/components-cache/components-list.ts

@@ -1,8 +1,9 @@
 import semver from 'semver';
 import pLimit from 'p-limit';
 import getUnixUTCTimestamp from 'oc-get-unix-utc-timestamp';
-import { ComponentsList, Config } from '../../../types';
 import { StorageAdapter } from 'oc-storage-adapters-utils';
+import eventsHandler from '../events-handler';
+import { ComponentsList, Config } from '../../../types';
 
 export default function componentsList(conf: Config, cdn: StorageAdapter) {
   const filePath = (): string =>
@@ -11,17 +12,64 @@ export default function componentsList(conf: Config, cdn: StorageAdapter) {
   const componentsList = {
     getFromJson: (): Promise<ComponentsList> => cdn.getJson(filePath(), true),
 
-    getFromDirectories: async (): Promise<ComponentsList> => {
+    getFromDirectories: async (
+      jsonList: ComponentsList | null
+    ): Promise<ComponentsList> => {
       const componentsInfo: Record<string, string[]> = {};
 
+      const checkVersion = (
+        componentName: string,
+        componentVersion: string
+      ) => {
+        return cdn
+          .getJson(
+            // Check integrity of the package by checking existence of package.json
+            // OC will upload always the package.json last when publishing
+            `${conf.storage.options.componentsDir}/${componentName}/${componentVersion}/package.json`
+          )
+          .then(() => true)
+          .catch(() => false);
+      };
+
       const getVersionsForComponent = async (
         componentName: string
       ): Promise<string[]> => {
-        const versions = await cdn.listSubDirectories(
+        const allVersions = await cdn.listSubDirectories(
           `${conf.storage.options.componentsDir}/${componentName}`
         );
+        const unCheckedVersions = allVersions.filter(
+          version => !jsonList?.components[componentName]?.includes(version)
+        );
+        const limit = pLimit(cdn.maxConcurrentRequests);
+        const invalidVersions = (
+          await Promise.all(
+            unCheckedVersions.map(unCheckedVersion =>
+              limit(async () => {
+                const isValid = await checkVersion(
+                  componentName,
+                  unCheckedVersion
+                );
+
+                return isValid ? null : unCheckedVersion;
+              })
+            )
+          )
+        ).filter((x): x is string => typeof x === 'string');
+
+        if (invalidVersions.length > 0) {
+          eventsHandler.fire('error', {
+            code: 'CORRUPTED_VERSION',
+            message: `Couldn't validate the integrity of the component ${componentName} on the following versions: ${invalidVersions.join(
+              ', '
+            )}.`
+          });
+        }
+
+        const validVersions = allVersions.filter(
+          version => !invalidVersions.includes(version)
+        );
 
-        return versions.sort(semver.compare);
+        return validVersions.sort(semver.compare);
       };
 
       try {
@@ -55,8 +103,8 @@ export default function componentsList(conf: Config, cdn: StorageAdapter) {
       }
     },
 
-    async refresh(): Promise<ComponentsList> {
-      const components = await componentsList.getFromDirectories();
+    async refresh(cachedList: ComponentsList): Promise<ComponentsList> {
+      const components = await componentsList.getFromDirectories(cachedList);
       await componentsList.save(components);
 
       return components;

src/registry/domain/components-cache/index.ts

@@ -11,8 +11,8 @@ export default function componentsCache(conf: Config, cdn: StorageAdapter) {
 
   const componentsList = getComponentsList(conf, cdn);
 
-  const poll = () =>
-    setTimeout(async () => {
+  const poll = () => {
+    return setTimeout(async () => {
       try {
         const data = await componentsList.getFromJson();
 
@@ -29,6 +29,7 @@ export default function componentsCache(conf: Config, cdn: StorageAdapter) {
       }
       refreshLoop = poll();
     }, conf.pollingInterval * 1000);
+  };
 
   const cacheDataAndStartPolling = (data: ComponentsList) => {
     cachedComponentsList = data;
@@ -55,12 +56,12 @@ export default function componentsCache(conf: Config, cdn: StorageAdapter) {
     },
 
     async load(): Promise<ComponentsList> {
-      const dirComponents = await componentsList
-        .getFromDirectories()
-        .catch(err => throwError('components_list_get', err));
       const jsonComponents = await componentsList
         .getFromJson()
         .catch(() => null);
+      const dirComponents = await componentsList
+        .getFromDirectories(jsonComponents)
+        .catch(err => throwError('components_list_get', err));
 
       if (
         !jsonComponents ||
@@ -78,7 +79,7 @@ export default function componentsCache(conf: Config, cdn: StorageAdapter) {
     async refresh(): Promise<ComponentsList> {
       clearTimeout(refreshLoop);
       try {
-        const components = await componentsList.refresh();
+        const components = await componentsList.refresh(cachedComponentsList);
         cacheDataAndStartPolling(components);
 
         return components;

test/unit/registry-domain-components-cache.js

@@ -62,7 +62,8 @@ describe('registry : domain : components-cache', () => {
     describe('when initialising the cache', () => {
       before(done => {
         mockedCdn.getJson = sinon.stub();
-        mockedCdn.getJson.rejects('not_found');
+        mockedCdn.getJson.resolves({});
+        mockedCdn.getJson.onFirstCall(0).rejects('not_found');
         mockedCdn.listSubDirectories = sinon.stub();
         mockedCdn.listSubDirectories.onCall(0).resolves(['hello-world']);
         mockedCdn.listSubDirectories.onCall(1).resolves(['1.0.0', '1.0.2']);
@@ -72,11 +73,17 @@ describe('registry : domain : components-cache', () => {
         componentsCache.load().finally(done);
       });
 
-      it('should try fetching the components.json', () => {
-        expect(mockedCdn.getJson.calledOnce).to.be.true;
+      it('should try fetching the components.json and check components', () => {
+        expect(mockedCdn.getJson.calledThrice).to.be.true;
         expect(mockedCdn.getJson.args[0][0]).to.be.equal(
           'component/components.json'
         );
+        expect(mockedCdn.getJson.args[1][0]).to.be.equal(
+          'component/hello-world/1.0.0/package.json'
+        );
+        expect(mockedCdn.getJson.args[2][0]).to.be.equal(
+          'component/hello-world/1.0.2/package.json'
+        );
       });
 
       it('should scan for directories to fetch components and versions', () => {
@@ -118,10 +125,13 @@ describe('registry : domain : components-cache', () => {
       });
 
       it('should fetch the components.json', () => {
-        expect(mockedCdn.getJson.calledOnce).to.be.true;
+        expect(mockedCdn.getJson.calledTwice).to.be.true;
         expect(mockedCdn.getJson.args[0][0]).to.be.equal(
           'component/components.json'
         );
+        expect(mockedCdn.getJson.args[1][0]).to.be.equal(
+          'component/hello-world/2.0.0/package.json'
+        );
       });
 
       it('should scan for directories to fetch components and versions', () => {

tsconfig.json

@@ -13,7 +13,7 @@
     // "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
 
     /* Language and Environment */
-    "target": "ES2018" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
+    "target": "ES2020" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
     // "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
     // "jsx": "preserve",                                /* Specify what JSX code is generated. */
     // "experimentalDecorators": true,                   /* Enable experimental support for TC39 stage 2 draft decorators. */