merge master

Author: ricardo-devis-agullo

.github/workflows/node.js.yml

@@ -5,13 +5,12 @@ name: Node.js CI
 
 on:
   push:
-    branches: [ master ]
+    branches: [master]
   pull_request:
-    branches: [ master ]
+    branches: [master]
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     strategy:
@@ -20,11 +19,11 @@ jobs:
         # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v2
-      with:
-        node-version: ${{ matrix.node-version }}
-        cache: 'npm'
-    - run: npm install
-    - run: npm test
+      - uses: actions/checkout@v2
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v2
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: 'npm'
+      - run: npm install
+      - run: npm test

.gitignore

@@ -21,6 +21,7 @@ results
 
 npm-debug.log
 node_modules
+.yarn
 # yarn.lock
 package-lock.json
 

.npmignore

@@ -5,5 +5,6 @@
 .travis.yml
 appveyor.yml
 Gruntfile.js
-tasks
-test
+/tasks
+/test
+/src

.nvmrc

@@ -1 +1 @@
-14
+16

.yarnrc.yml

@@ -0,0 +1 @@
+nodeLinker: node-modules

CHANGELOG.md

@@ -1,5 +1,23 @@
 ## Change Log
 
+### v0.49.23
+
+- [#1320](https://github.com/opencomponents/oc/pull/1320) make npmignore applied only to root folders
+
+### v0.49.22
+
+- [#1319](https://github.com/opencomponents/oc/pull/1319) Check unverified packages after start/publishing
+
+### v0.49.21
+
+- [#1303](https://github.com/opencomponents/oc/pull/1303) add ability to retreieve environment variables from an env file
+- [#1312](https://github.com/opencomponents/oc/pull/1312) Upgrade project default Node.js version to 16
+- [#1317](https://github.com/opencomponents/oc/pull/1317) Bump loader-utils from 1.4.0 to 1.4.2
+
+### v0.49.19
+
+- [#1315](https://github.com/opencomponents/oc/pull/1315) include the entire console api in the production stub
+
 ### v0.49.16
 
 - [#1313](https://github.com/opencomponents/oc/pull/1313) remove some any types

package.json

@@ -1,21 +1,19 @@
 {
   "name": "oc",
-  "version": "0.49.18",
+  "version": "0.49.23",
   "description": "A framework for developing and distributing html components",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
-  "bin": {
-    "oc": "./dist/oc-cli.js"
-  },
+  "bin": "./dist/oc-cli.js",
   "scripts": {
     "lint": "eslint src",
     "prebuild": "rimraf dist",
     "build": "npm run lint && tsc && node tasks/build.js",
     "git-stage-and-push": "node tasks/git-stage-and-push.js",
     "npm-publish": "node tasks/npm-publish.js",
-    "publish-major": "npm run version-major && npm run test-silent && npm run git-stage-and-push && npm run npm-publish",
-    "publish-minor": "npm run version-minor && npm run test-silent && npm run git-stage-and-push && npm run npm-publish",
-    "publish-patch": "npm run version-patch && npm run test-silent && npm run git-stage-and-push && npm run npm-publish",
+    "publish-major": "npm run test-silent && npm run version-major && node tasks/build.js && npm run git-stage-and-push && npm run npm-publish",
+    "publish-minor": "npm run test-silent && npm run version-minor && node tasks/build.js && npm run git-stage-and-push && npm run npm-publish",
+    "publish-patch": "npm run test-silent && npm run version-patch && node tasks/build.js && npm run git-stage-and-push && npm run npm-publish",
     "test": "npm run build && node tasks/mochaTest.js",
     "test-silent": "npm run build && node tasks/mochaTest.js --silent",
     "version-major": "node tasks/version.js --type=\"major\"",
@@ -43,45 +41,45 @@
   ],
   "devDependencies": {
     "@types/accept-language-parser": "1.5.3",
-    "@types/async": "3.2.15",
+    "@types/async": "3.2.16",
     "@types/cross-spawn": "6.0.2",
     "@types/errorhandler": "1.5.0",
-    "@types/express": "4.17.14",
+    "@types/express": "4.17.15",
     "@types/fs-extra": "9.0.13",
     "@types/got": "^9.6.12",
     "@types/livereload": "0.9.2",
-    "@types/lodash": "4.14.186",
+    "@types/lodash": "4.14.191",
     "@types/morgan": "1.9.3",
     "@types/multer": "1.4.7",
-    "@types/node": "18.8.4",
+    "@types/node": "18.11.16",
     "@types/parse-author": "2.0.1",
     "@types/read": "0.0.29",
     "@types/response-time": "2.3.5",
-    "@types/semver": "7.3.12",
+    "@types/semver": "7.3.13",
     "@types/targz": "1.0.1",
-    "@types/yargs": "17.0.13",
-    "@typescript-eslint/eslint-plugin": "5.40.0",
-    "@typescript-eslint/parser": "5.40.0",
-    "chai": "4.3.6",
+    "@types/yargs": "17.0.17",
+    "@typescript-eslint/eslint-plugin": "5.46.1",
+    "@typescript-eslint/parser": "5.46.1",
+    "chai": "4.3.7",
     "chalk": "4.1.2",
-    "eslint": "8.25.0",
+    "eslint": "8.30.0",
     "eslint-config-prettier": "8.5.0",
     "eslint-plugin-prettier": "4.2.1",
     "glob": "7.2.0",
-    "husky": "7.0.4",
+    "husky": "8.0.2",
     "injectr": "0.5.1",
     "minimist": "1.2.7",
     "mocha": "9.1.3",
     "node-emoji": "1.11.0",
     "p-limit": "^3.1.0",
-    "prettier": "2.7.1",
+    "prettier": "2.8.1",
     "rimraf": "3.0.2",
     "semver-sort": "1.0.0",
     "simple-git": "2.48.0",
     "sinon": "12.0.1",
     "ts-node": "10.9.1",
     "type-fest": "2.8.0",
-    "typescript": "4.8.4"
+    "typescript": "4.9.4"
   },
   "dependencies": {
     "accept-language-parser": "1.5.0",
@@ -92,10 +90,11 @@
     "colors": "1.4.0",
     "cross-spawn": "7.0.3",
     "dependency-graph": "0.11.0",
+    "dotenv": "16.0.3",
     "errorhandler": "1.5.1",
     "express": "4.18.2",
     "form-data": "4.0.0",
-    "fs-extra": "10.1.0",
+    "fs-extra": "11.1.0",
     "getport": "0.1.0",
     "got": "11.8.5",
     "livereload": "0.9.3",
@@ -127,6 +126,6 @@
     "targz": "1.0.1",
     "try-require": "1.2.1",
     "universalify": "^2.0.0",
-    "yargs": "17.6.0"
+    "yargs": "17.6.2"
   }
 }

src/components/oc-client/package.json

@@ -1,7 +1,7 @@
 {
   "name": "oc-client",
   "description": "The OpenComponents client-side javascript client",
-  "version": "0.49.18",
+  "version": "0.49.23",
   "repository": "https://github.com/opencomponents/oc/tree/master/components/oc-client",
   "author": "Matteo Figus <[email protected]>",
   "oc": {

src/registry/domain/components-cache/components-list.ts

@@ -1,8 +1,9 @@
 import semver from 'semver';
 import pLimit from 'p-limit';
 import getUnixUTCTimestamp from 'oc-get-unix-utc-timestamp';
-import { ComponentsList, Config } from '../../../types';
 import { StorageAdapter } from 'oc-storage-adapters-utils';
+import eventsHandler from '../events-handler';
+import { ComponentsList, Config } from '../../../types';
 
 export default function componentsList(conf: Config, cdn: StorageAdapter) {
   const filePath = (): string =>
@@ -11,17 +12,64 @@ export default function componentsList(conf: Config, cdn: StorageAdapter) {
   const componentsList = {
     getFromJson: (): Promise<ComponentsList> => cdn.getJson(filePath(), true),
 
-    getFromDirectories: async (): Promise<ComponentsList> => {
+    getFromDirectories: async (
+      jsonList: ComponentsList | null
+    ): Promise<ComponentsList> => {
       const componentsInfo: Record<string, string[]> = {};
 
+      const validateComponentVersion = (
+        componentName: string,
+        componentVersion: string
+      ) => {
+        return cdn
+          .getJson(
+            // Check integrity of the package by checking existence of package.json
+            // OC will upload always the package.json last when publishing
+            `${conf.storage.options.componentsDir}/${componentName}/${componentVersion}/package.json`
+          )
+          .then(() => true)
+          .catch(() => false);
+      };
+
       const getVersionsForComponent = async (
         componentName: string
       ): Promise<string[]> => {
-        const versions = await cdn.listSubDirectories(
+        const allVersions = await cdn.listSubDirectories(
           `${conf.storage.options.componentsDir}/${componentName}`
         );
+        const unCheckedVersions = allVersions.filter(
+          version => !jsonList?.components[componentName]?.includes(version)
+        );
+        const limit = pLimit(cdn.maxConcurrentRequests);
+        const invalidVersions = (
+          await Promise.all(
+            unCheckedVersions.map(unCheckedVersion =>
+              limit(async () => {
+                const isValid = await validateComponentVersion(
+                  componentName,
+                  unCheckedVersion
+                );
+
+                return isValid ? null : unCheckedVersion;
+              })
+            )
+          )
+        ).filter((x): x is string => typeof x === 'string');
+
+        if (invalidVersions.length > 0) {
+          eventsHandler.fire('error', {
+            code: 'corrupted_version',
+            message: `Couldn't validate the integrity of the component ${componentName} on the following versions: ${invalidVersions.join(
+              ', '
+            )}.`
+          });
+        }
+
+        const validVersions = allVersions.filter(
+          version => !invalidVersions.includes(version)
+        );
 
-        return versions.sort(semver.compare);
+        return validVersions.sort(semver.compare);
       };
 
       try {
@@ -55,8 +103,8 @@ export default function componentsList(conf: Config, cdn: StorageAdapter) {
       }
     },
 
-    async refresh(): Promise<ComponentsList> {
-      const components = await componentsList.getFromDirectories();
+    async refresh(cachedList: ComponentsList): Promise<ComponentsList> {
+      const components = await componentsList.getFromDirectories(cachedList);
       await componentsList.save(components);
 
       return components;

src/registry/domain/components-cache/index.ts

@@ -3,16 +3,16 @@ import getComponentsList from './components-list';
 import eventsHandler from '../events-handler';
 import getUnixUTCTimestamp from 'oc-get-unix-utc-timestamp';
 import { ComponentsList, Config } from '../../../types';
-import { StorageAdapter } from 'oc-storage-adapters-utils';
+import { StorageAdapter, strings } from 'oc-storage-adapters-utils';
 
 export default function componentsCache(conf: Config, cdn: StorageAdapter) {
   let cachedComponentsList: ComponentsList;
   let refreshLoop: NodeJS.Timeout;
 
   const componentsList = getComponentsList(conf, cdn);
 
-  const poll = () =>
-    setTimeout(async () => {
+  const poll = () => {
+    return setTimeout(async () => {
       try {
         const data = await componentsList.getFromJson();
 
@@ -29,6 +29,7 @@ export default function componentsCache(conf: Config, cdn: StorageAdapter) {
       }
       refreshLoop = poll();
     }, conf.pollingInterval * 1000);
+  };
 
   const cacheDataAndStartPolling = (data: ComponentsList) => {
     cachedComponentsList = data;
@@ -55,12 +56,15 @@ export default function componentsCache(conf: Config, cdn: StorageAdapter) {
     },
 
     async load(): Promise<ComponentsList> {
+      const jsonComponents = await componentsList.getFromJson().catch(err => {
+        if (err?.code === strings.errors.STORAGE.FILE_NOT_FOUND_CODE)
+          return null;
+
+        return Promise.reject(err);
+      });
       const dirComponents = await componentsList
-        .getFromDirectories()
+        .getFromDirectories(jsonComponents)
         .catch(err => throwError('components_list_get', err));
-      const jsonComponents = await componentsList
-        .getFromJson()
-        .catch(() => null);
 
       if (
         !jsonComponents ||
@@ -78,7 +82,8 @@ export default function componentsCache(conf: Config, cdn: StorageAdapter) {
     async refresh(): Promise<ComponentsList> {
       clearTimeout(refreshLoop);
       try {
-        const components = await componentsList.refresh();
+        // Passing components that we know are fine, so it doesn't refresh invalid components
+        const components = await componentsList.refresh(cachedComponentsList);
         cacheDataAndStartPolling(components);
 
         return components;