diff --git a/.depcheckrc b/.depcheckrc index 2d575ec..2d129a8 100644 --- a/.depcheckrc +++ b/.depcheckrc @@ -1,3 +1,4 @@ ignores: - 'reservoir' - '@types/reservoir' + - 'json-schema' diff --git a/package-lock.json b/package-lock.json index 06c945e..31ab965 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,9 +15,11 @@ "mongodb-schema": "bin/mongodb-schema" }, "devDependencies": { + "@types/json-schema": "^7.0.15", "@types/mocha": "^10.0.1", "@types/node": "^18.11.18", "@types/reservoir": "^0.1.0", + "@types/sinon": "^17.0.3", "@typescript-eslint/eslint-plugin": "^5.47.1", "@typescript-eslint/parser": "^5.47.1", "bson": "^6.7.0", @@ -34,6 +36,7 @@ "mocha": "^10.2.0", "mongodb": "^6.6.1", "nyc": "^15.1.0", + "sinon": "^19.0.2", "ts-node": "^10.9.1", "typescript": "^4.9.4" }, @@ -1362,6 +1365,55 @@ "node": ">= 8" } }, + "node_modules/@sinonjs/commons": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", + "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "type-detect": "4.0.8" + } + }, + "node_modules/@sinonjs/fake-timers": { + "version": "13.0.5", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-13.0.5.tgz", + "integrity": "sha512-36/hTbH2uaWuGVERyC6da9YwGWnzUZXuPro/F2LfsdOsLnCojz/iSH8MxUt/FD2S5XBSVPhmArFUXcpCQ2Hkiw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.1" + } + }, + "node_modules/@sinonjs/samsam": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-8.0.2.tgz", + "integrity": "sha512-v46t/fwnhejRSFTGqbpn9u+LQ9xJDse10gNnPgAcxgdoCDMXj/G2asWAC/8Qs+BAZDicX+MNZouXT1A7c83kVw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.1", + "lodash.get": "^4.4.2", + "type-detect": "^4.1.0" + } + }, + "node_modules/@sinonjs/samsam/node_modules/type-detect": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.1.0.tgz", + "integrity": "sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/@sinonjs/text-encoding": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.3.tgz", + "integrity": "sha512-DE427ROAphMQzU4ENbliGYrBSYPXF+TtLg9S8vzeA+OF4ZKzoDdzfL8sxuMUGS/lgRhM6j1URSk9ghf7Xo1tyA==", + "dev": true, + "license": "(Unlicense OR Apache-2.0)" + }, "node_modules/@smithy/abort-controller": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-1.0.1.tgz", @@ -2284,10 +2336,11 @@ "dev": true }, "node_modules/@types/json-schema": { - "version": "7.0.11", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz", - "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==", - "dev": true + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT" }, "node_modules/@types/json5": { "version": "0.0.29", @@ -2331,6 +2384,23 @@ "integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==", "dev": true }, + "node_modules/@types/sinon": { + "version": "17.0.3", + "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-17.0.3.tgz", + "integrity": "sha512-j3uovdn8ewky9kRBG19bOwaZbexJu/XjtkHyjvUgt4xfPFz18dcORIMqnYh66Fx3Powhcr85NT5+er3+oViapw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/sinonjs__fake-timers": "*" + } + }, + "node_modules/@types/sinonjs__fake-timers": { + "version": "8.1.5", + "resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.5.tgz", + "integrity": "sha512-mQkU2jY8jJEF7YHjHvsQO8+3ughTL1mcnn96igfhONmR+fUPSKIkefQYpSe8bsly2Ep7oQbn/6VG5/9/0qcArQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/webidl-conversions": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz", @@ -5419,6 +5489,13 @@ "node": ">=0.6.0" } }, + "node_modules/just-extend": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-6.2.0.tgz", + "integrity": "sha512-cYofQu2Xpom82S6qD778jBDpwvvy39s1l/hrYij2u9AMdQcGRpaBu6kY4mVhuno5kJVi1DAz4aiphA2WI1/OAw==", + "dev": true, + "license": "MIT" + }, "node_modules/lcov-parse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/lcov-parse/-/lcov-parse-1.0.0.tgz", @@ -5471,6 +5548,13 @@ "integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=", "dev": true }, + "node_modules/lodash.get": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==", + "dev": true, + "license": "MIT" + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -5994,6 +6078,20 @@ "integrity": "sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g==", "dev": true }, + "node_modules/nise": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/nise/-/nise-6.1.1.tgz", + "integrity": "sha512-aMSAzLVY7LyeM60gvBS423nBmIPP+Wy7St7hsb+8/fc1HmeoHJfLO8CKse4u3BtOZvQLJghYPI2i/1WZrEj5/g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.1", + "@sinonjs/fake-timers": "^13.0.1", + "@sinonjs/text-encoding": "^0.7.3", + "just-extend": "^6.2.0", + "path-to-regexp": "^8.1.0" + } + }, "node_modules/node-preload": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/node-preload/-/node-preload-0.2.1.tgz", @@ -6401,6 +6499,16 @@ "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, + "node_modules/path-to-regexp": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.2.0.tgz", + "integrity": "sha512-TdrF7fW9Rphjq4RjrW0Kp2AW0Ahwu9sRGTkS6bvDi0SCwZlEZYmcfDbEsTz8RVk0EHIS/Vd1bv3JhG+1xZuAyQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16" + } + }, "node_modules/path-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", @@ -6909,6 +7017,48 @@ "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", "dev": true }, + "node_modules/sinon": { + "version": "19.0.2", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-19.0.2.tgz", + "integrity": "sha512-euuToqM+PjO4UgXeLETsfQiuoyPXlqFezr6YZDFwHR3t4qaX0fZUe1MfPMznTL5f8BWrVS89KduLdMUsxFCO6g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.1", + "@sinonjs/fake-timers": "^13.0.2", + "@sinonjs/samsam": "^8.0.1", + "diff": "^7.0.0", + "nise": "^6.1.1", + "supports-color": "^7.2.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/sinon" + } + }, + "node_modules/sinon/node_modules/diff": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-7.0.0.tgz", + "integrity": "sha512-PJWHUb1RFevKCwaFA9RlG5tCd+FO5iRh9A8HEtkmBH2Li03iJriB6m6JIN4rGz3K3JLawI7/veA1xzRKP6ISBw==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/sinon/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", @@ -7343,6 +7493,16 @@ "node": ">= 0.8.0" } }, + "node_modules/type-detect": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/type-fest": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", @@ -8931,6 +9091,49 @@ "fastq": "^1.6.0" } }, + "@sinonjs/commons": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", + "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", + "dev": true, + "requires": { + "type-detect": "4.0.8" + } + }, + "@sinonjs/fake-timers": { + "version": "13.0.5", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-13.0.5.tgz", + "integrity": "sha512-36/hTbH2uaWuGVERyC6da9YwGWnzUZXuPro/F2LfsdOsLnCojz/iSH8MxUt/FD2S5XBSVPhmArFUXcpCQ2Hkiw==", + "dev": true, + "requires": { + "@sinonjs/commons": "^3.0.1" + } + }, + "@sinonjs/samsam": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-8.0.2.tgz", + "integrity": "sha512-v46t/fwnhejRSFTGqbpn9u+LQ9xJDse10gNnPgAcxgdoCDMXj/G2asWAC/8Qs+BAZDicX+MNZouXT1A7c83kVw==", + "dev": true, + "requires": { + "@sinonjs/commons": "^3.0.1", + "lodash.get": "^4.4.2", + "type-detect": "^4.1.0" + }, + "dependencies": { + "type-detect": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.1.0.tgz", + "integrity": "sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==", + "dev": true + } + } + }, + "@sinonjs/text-encoding": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.3.tgz", + "integrity": "sha512-DE427ROAphMQzU4ENbliGYrBSYPXF+TtLg9S8vzeA+OF4ZKzoDdzfL8sxuMUGS/lgRhM6j1URSk9ghf7Xo1tyA==", + "dev": true + }, "@smithy/abort-controller": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-1.0.1.tgz", @@ -9825,9 +10028,9 @@ "dev": true }, "@types/json-schema": { - "version": "7.0.11", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz", - "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==", + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", "dev": true }, "@types/json5": { @@ -9872,6 +10075,21 @@ "integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==", "dev": true }, + "@types/sinon": { + "version": "17.0.3", + "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-17.0.3.tgz", + "integrity": "sha512-j3uovdn8ewky9kRBG19bOwaZbexJu/XjtkHyjvUgt4xfPFz18dcORIMqnYh66Fx3Powhcr85NT5+er3+oViapw==", + "dev": true, + "requires": { + "@types/sinonjs__fake-timers": "*" + } + }, + "@types/sinonjs__fake-timers": { + "version": "8.1.5", + "resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.5.tgz", + "integrity": "sha512-mQkU2jY8jJEF7YHjHvsQO8+3ughTL1mcnn96igfhONmR+fUPSKIkefQYpSe8bsly2Ep7oQbn/6VG5/9/0qcArQ==", + "dev": true + }, "@types/webidl-conversions": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz", @@ -12148,6 +12366,12 @@ "verror": "1.10.0" } }, + "just-extend": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-6.2.0.tgz", + "integrity": "sha512-cYofQu2Xpom82S6qD778jBDpwvvy39s1l/hrYij2u9AMdQcGRpaBu6kY4mVhuno5kJVi1DAz4aiphA2WI1/OAw==", + "dev": true + }, "lcov-parse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/lcov-parse/-/lcov-parse-1.0.0.tgz", @@ -12191,6 +12415,12 @@ "integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=", "dev": true }, + "lodash.get": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==", + "dev": true + }, "lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -12557,6 +12787,19 @@ "integrity": "sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g==", "dev": true }, + "nise": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/nise/-/nise-6.1.1.tgz", + "integrity": "sha512-aMSAzLVY7LyeM60gvBS423nBmIPP+Wy7St7hsb+8/fc1HmeoHJfLO8CKse4u3BtOZvQLJghYPI2i/1WZrEj5/g==", + "dev": true, + "requires": { + "@sinonjs/commons": "^3.0.1", + "@sinonjs/fake-timers": "^13.0.1", + "@sinonjs/text-encoding": "^0.7.3", + "just-extend": "^6.2.0", + "path-to-regexp": "^8.1.0" + } + }, "node-preload": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/node-preload/-/node-preload-0.2.1.tgz", @@ -12869,6 +13112,12 @@ "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, + "path-to-regexp": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.2.0.tgz", + "integrity": "sha512-TdrF7fW9Rphjq4RjrW0Kp2AW0Ahwu9sRGTkS6bvDi0SCwZlEZYmcfDbEsTz8RVk0EHIS/Vd1bv3JhG+1xZuAyQ==", + "dev": true + }, "path-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", @@ -13229,6 +13478,37 @@ "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", "dev": true }, + "sinon": { + "version": "19.0.2", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-19.0.2.tgz", + "integrity": "sha512-euuToqM+PjO4UgXeLETsfQiuoyPXlqFezr6YZDFwHR3t4qaX0fZUe1MfPMznTL5f8BWrVS89KduLdMUsxFCO6g==", + "dev": true, + "requires": { + "@sinonjs/commons": "^3.0.1", + "@sinonjs/fake-timers": "^13.0.2", + "@sinonjs/samsam": "^8.0.1", + "diff": "^7.0.0", + "nise": "^6.1.1", + "supports-color": "^7.2.0" + }, + "dependencies": { + "diff": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-7.0.0.tgz", + "integrity": "sha512-PJWHUb1RFevKCwaFA9RlG5tCd+FO5iRh9A8HEtkmBH2Li03iJriB6m6JIN4rGz3K3JLawI7/veA1xzRKP6ISBw==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, "slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", @@ -13554,6 +13834,12 @@ "prelude-ls": "^1.2.1" } }, + "type-detect": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", + "dev": true + }, "type-fest": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", diff --git a/package.json b/package.json index 0097cff..e6c4a66 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ ".esm-wrapper.mjs" ], "scripts": { - "test": "nyc mocha --timeout 5000 --colors -r ts-node/register test/*.ts", + "test": "nyc mocha --timeout 5000 --colors -r ts-node/register src/**/*.test.ts", "test-example-parse-from-file": "ts-node examples/parse-from-file.ts", "test-example-parse-schema": "ts-node examples/parse-schema.ts", "test-time": "ts-node ./test/time-testing.ts", @@ -53,9 +53,11 @@ "reservoir": "^0.1.2" }, "devDependencies": { + "@types/json-schema": "^7.0.15", "@types/mocha": "^10.0.1", "@types/node": "^18.11.18", "@types/reservoir": "^0.1.0", + "@types/sinon": "^17.0.3", "@typescript-eslint/eslint-plugin": "^5.47.1", "@typescript-eslint/parser": "^5.47.1", "bson": "^6.7.0", @@ -72,6 +74,7 @@ "mocha": "^10.2.0", "mongodb": "^6.6.1", "nyc": "^15.1.0", + "sinon": "^19.0.2", "ts-node": "^10.9.1", "typescript": "^4.9.4" }, diff --git a/src/index.ts b/src/index.ts index b2f4a5f..3dfc6ed 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,5 @@ -import { SchemaAnalyzer } from './schema-analyzer'; +import { InternalSchemaBasedAccessor, SchemaAccessor } from './schema-accessor'; +import { getCompletedSchemaAnalyzer, SchemaAnalyzer } from './schema-analyzer'; import type { ArraySchemaType, BaseSchemaType, @@ -6,7 +7,7 @@ import type { DocumentSchemaType, PrimitiveSchemaType, SchemaType, - Schema, + Schema as InternalSchema, SchemaField, SchemaParseOptions, SimplifiedSchemaBaseType, @@ -17,31 +18,17 @@ import type { SimplifiedSchema } from './schema-analyzer'; import * as schemaStats from './stats'; +import { AnyIterable, StandardJSONSchema, MongoDBJSONSchema, ExtendedJSONSchema } from './types'; -type AnyIterable = Iterable | AsyncIterable; - -function verifyStreamSource( - source: AnyIterable -): AnyIterable { - if (!(Symbol.iterator in source) && !(Symbol.asyncIterator in source)) { - throw new Error( - 'Unknown input type for `docs`. Must be an array, ' + - 'stream or MongoDB Cursor.' - ); - } - - return source; -} - -async function getCompletedSchemaAnalyzer( +/** + * Analyze documents - schema can be retrieved in different formats. + */ +async function analyzeDocuments( source: AnyIterable, options?: SchemaParseOptions -): Promise { - const analyzer = new SchemaAnalyzer(options); - for await (const doc of verifyStreamSource(source)) { - analyzer.analyzeDoc(doc); - } - return analyzer; +): Promise { + const internalSchema = (await getCompletedSchemaAnalyzer(source, options)).getResult(); + return new InternalSchemaBasedAccessor(internalSchema); } /** @@ -51,7 +38,7 @@ async function getCompletedSchemaAnalyzer( async function parseSchema( source: AnyIterable, options?: SchemaParseOptions -): Promise { +): Promise { return (await getCompletedSchemaAnalyzer(source, options)).getResult(); } @@ -78,7 +65,8 @@ export type { DocumentSchemaType, PrimitiveSchemaType, SchemaType, - Schema, + InternalSchema as Schema, + InternalSchema, SchemaField, SchemaParseOptions, SimplifiedSchemaBaseType, @@ -86,11 +74,15 @@ export type { SimplifiedSchemaDocumentType, SimplifiedSchemaType, SimplifiedSchemaField, - SimplifiedSchema + SimplifiedSchema, + StandardJSONSchema, + MongoDBJSONSchema, + ExtendedJSONSchema }; export { parseSchema, + analyzeDocuments, getSchemaPaths, getSimplifiedSchema, SchemaAnalyzer, diff --git a/src/schema-accessor.ts b/src/schema-accessor.ts new file mode 100644 index 0000000..530d8bd --- /dev/null +++ b/src/schema-accessor.ts @@ -0,0 +1,50 @@ +import { Schema as InternalSchema } from './schema-analyzer'; +import convertors from './schema-convertors'; +import { ExtendedJSONSchema, MongoDBJSONSchema, StandardJSONSchema } from './types'; + +export interface SchemaAccessor { + getStandardJsonSchema: () => Promise; + getMongoDBJsonSchema: () => Promise; + getExtendedJsonSchema: () => Promise; + getInternalSchema: () => Promise; +} + +type Options = { + signal?: AbortSignal; +} + +/** + * Accessor for different schema formats. + * Internal schema is provided at initialization, + * the others are converted lazily and memoized. + * Conversion can be aborted. + */ +export class InternalSchemaBasedAccessor implements SchemaAccessor { + private internalSchema: InternalSchema; + private standardJSONSchema?: StandardJSONSchema; + private mongodbJSONSchema?: MongoDBJSONSchema; + private extendedJSONSchema?: ExtendedJSONSchema; + + constructor(internalSchema: InternalSchema) { + this.internalSchema = internalSchema; + } + + async getInternalSchema(options?: Options): Promise { + return this.internalSchema; + } + + async getStandardJsonSchema(options: Options = {}): Promise { + if (this.standardJSONSchema) return this.standardJSONSchema; + return this.standardJSONSchema = await convertors.internalSchemaToStandard(this.internalSchema, options); + } + + async getMongoDBJsonSchema(options: Options = {}): Promise { + if (this.mongodbJSONSchema) return this.mongodbJSONSchema; + return this.mongodbJSONSchema = await convertors.internalSchemaToMongoDB(this.internalSchema, options); + } + + async getExtendedJsonSchema(options: Options = {}): Promise { + if (this.extendedJSONSchema) return this.extendedJSONSchema; + return this.extendedJSONSchema = await convertors.internalSchemaToExtended(this.internalSchema, options); + } +} diff --git a/src/schema-analyzer.ts b/src/schema-analyzer.ts index 29817bf..0d41ad4 100644 --- a/src/schema-analyzer.ts +++ b/src/schema-analyzer.ts @@ -17,6 +17,7 @@ import { } from 'bson'; import semanticTypes from './semantic-types'; +import { AnyIterable } from './types'; type TypeCastMap = { Array: unknown[]; @@ -163,6 +164,7 @@ type SemanticTypeMap = { export type SchemaParseOptions = { semanticTypes?: boolean | SemanticTypeMap; storeValues?: boolean; + signal?: AbortSignal; }; /** @@ -585,3 +587,28 @@ export class SchemaAnalyzer { return simplifiedSchema(this.schemaAnalysisRoot.fields); } } + +export function verifyStreamSource( + source: AnyIterable +): AnyIterable { + if (!(Symbol.iterator in source) && !(Symbol.asyncIterator in source)) { + throw new Error( + 'Unknown input type for `docs`. Must be an array, ' + + 'stream or MongoDB Cursor.' + ); + } + + return source; +} + +export async function getCompletedSchemaAnalyzer( + source: AnyIterable, + options?: SchemaParseOptions +): Promise { + const analyzer = new SchemaAnalyzer(options); + for await (const doc of verifyStreamSource(source)) { + if (options?.signal?.aborted) throw options.signal.aborted; + analyzer.analyzeDoc(doc); + } + return analyzer; +} diff --git a/src/schema-convertors/index.ts b/src/schema-convertors/index.ts new file mode 100644 index 0000000..c4b4769 --- /dev/null +++ b/src/schema-convertors/index.ts @@ -0,0 +1,27 @@ +import internalSchemaToMongoDB from './internalToMongoDB'; +import { Schema as InternalSchema } from '../schema-analyzer'; +import { ExtendedJSONSchema, StandardJSONSchema } from '../types'; + +function internalSchemaToStandard( + internalSchema: InternalSchema, + options: { + signal?: AbortSignal +}): StandardJSONSchema { + // TODO: COMPASS-8700 + return {} as StandardJSONSchema; +} + +function internalSchemaToExtended( + internalSchema: InternalSchema, + options: { + signal?: AbortSignal +}): ExtendedJSONSchema { + // TODO: COMPASS-8702 + return {} as ExtendedJSONSchema; +} + +export default { + internalSchemaToStandard, + internalSchemaToMongoDB, + internalSchemaToExtended +}; diff --git a/src/schema-convertors/internalToMongoDB.test.ts b/src/schema-convertors/internalToMongoDB.test.ts new file mode 100644 index 0000000..148837a --- /dev/null +++ b/src/schema-convertors/internalToMongoDB.test.ts @@ -0,0 +1,1659 @@ +import assert from 'assert'; +import internalSchemaToStandard from './internalToMongoDB'; +import parseSchema from '..'; +import { allBSONTypesDoc } from '../../test/all-bson-types-fixture'; + +describe.only('internalSchemaToStandard', function() { + describe('Converts: ', function() { + it('get me analyzed thing', async function() { + // const internal = await parseSchema([allBSONTypesDoc]); + // console.log(JSON.stringify(internal)); + const internal = { + count: 1, + fields: [ + { + name: '_id', + path: [ + '_id' + ], + count: 1, + type: 'ObjectId', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'ObjectId', + path: [ + '_id' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + '642d766b7300158b1f22e972' + ], + bsonType: 'ObjectId' + } + ] + }, + { + name: 'array', + path: [ + 'array' + ], + count: 1, + type: 'Array', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: [ + 'array' + ], + count: 1, + probability: 0.8, + bsonType: 'Array', + types: [ + { + name: 'Number', + path: [ + 'array' + ], + count: 3, + probability: 0.8, + unique: 3, + hasDuplicates: false, + values: [ + 1, + 2, + 3 + ], + bsonType: 'Number' + } + ], + totalCount: 3, + lengths: [ + 3 + ], + averageLength: 3 + } + ] + }, + { + name: 'binaries', + path: [ + 'binaries' + ], + count: 1, + type: 'Document', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Document', + path: [ + 'binaries' + ], + count: 1, + probability: 0.8, + bsonType: 'Document', + fields: [ + { + name: 'binaryOld', + path: [ + 'binaries', + 'binaryOld' + ], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: [ + 'binaries', + 'binaryOld' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + '//8=' + ], + bsonType: 'Binary' + } + ] + }, + { + name: 'compressedTimeSeries', + path: [ + 'binaries', + 'compressedTimeSeries' + ], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: [ + 'binaries', + 'compressedTimeSeries' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 'c//SZESzTGmQ6OfR38A11A==' + ], + bsonType: 'Binary' + } + ] + }, + { + name: 'custom', + path: [ + 'binaries', + 'custom' + ], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: [ + 'binaries', + 'custom' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + '//8=' + ], + bsonType: 'Binary' + } + ] + }, + { + name: 'encrypted', + path: [ + 'binaries', + 'encrypted' + ], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: [ + 'binaries', + 'encrypted' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 'c//SZESzTGmQ6OfR38A11A==' + ], + bsonType: 'Binary' + } + ] + }, + { + name: 'functionData', + path: [ + 'binaries', + 'functionData' + ], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: [ + 'binaries', + 'functionData' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + '//8=' + ], + bsonType: 'Binary' + } + ] + }, + { + name: 'generic', + path: [ + 'binaries', + 'generic' + ], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: [ + 'binaries', + 'generic' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 'AQID' + ], + bsonType: 'Binary' + } + ] + }, + { + name: 'md5', + path: [ + 'binaries', + 'md5' + ], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: [ + 'binaries', + 'md5' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 'c//SZESzTGmQ6OfR38A11A==' + ], + bsonType: 'Binary' + } + ] + }, + { + name: 'uuid', + path: [ + 'binaries', + 'uuid' + ], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: [ + 'binaries', + 'uuid' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 'aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa' + ], + bsonType: 'Binary' + } + ] + }, + { + name: 'uuidOld', + path: [ + 'binaries', + 'uuidOld' + ], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: [ + 'binaries', + 'uuidOld' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 'c//SZESzTGmQ6OfR38A11A==' + ], + bsonType: 'Binary' + } + ] + } + ] + } + ] + }, + { + name: 'binData', + path: [ + 'binData' + ], + count: 1, + type: 'Binary', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Binary', + path: [ + 'binData' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 'AQID' + ], + bsonType: 'Binary' + } + ] + }, + { + name: 'boolean', + path: [ + 'boolean' + ], + count: 1, + type: 'Boolean', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Boolean', + path: [ + 'boolean' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + true + ], + bsonType: 'Boolean' + } + ] + }, + { + name: 'date', + path: [ + 'date' + ], + count: 1, + type: 'Date', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Date', + path: [ + 'date' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + '2023-04-05T13:25:08.445Z' + ], + bsonType: 'Date' + } + ] + }, + { + name: 'dbRef', + path: [ + 'dbRef' + ], + count: 1, + type: 'DBRef', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'DBRef', + path: [ + 'dbRef' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + $ref: 'namespace', + $id: '642d76b4b7ebfab15d3c4a78' + } + ], + bsonType: 'DBRef' + } + ] + }, + { + name: 'decimal', + path: [ + 'decimal' + ], + count: 1, + type: 'Decimal128', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Decimal128', + path: [ + 'decimal' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + $numberDecimal: '5.477284286264328586719275128128001E-4088' + } + ], + bsonType: 'Decimal128' + } + ] + }, + { + name: 'double', + path: [ + 'double' + ], + count: 1, + type: 'Double', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Double', + path: [ + 'double' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 1.2 + ], + bsonType: 'Double' + } + ] + }, + { + name: 'int', + path: [ + 'int' + ], + count: 1, + type: 'Int32', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Int32', + path: [ + 'int' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 12345 + ], + bsonType: 'Int32' + } + ] + }, + { + name: 'javascript', + path: [ + 'javascript' + ], + count: 1, + type: 'Code', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Code', + path: [ + 'javascript' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + code: 'function() {}' + } + ], + bsonType: 'Code' + } + ] + }, + { + name: 'javascriptWithScope', + path: [ + 'javascriptWithScope' + ], + count: 1, + type: 'Code', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Code', + path: [ + 'javascriptWithScope' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + code: 'function() {}', + scope: { + foo: 1, + bar: 'a' + } + } + ], + bsonType: 'Code' + } + ] + }, + { + name: 'long', + path: [ + 'long' + ], + count: 1, + type: 'Long', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Long', + path: [ + 'long' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + low: -1395630315, + high: 28744523, + unsigned: false + } + ], + bsonType: 'Long' + } + ] + }, + { + name: 'maxKey', + path: [ + 'maxKey' + ], + count: 1, + type: 'MaxKey', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'MaxKey', + path: [ + 'maxKey' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + {} + ], + bsonType: 'MaxKey' + } + ] + }, + { + name: 'minKey', + path: [ + 'minKey' + ], + count: 1, + type: 'MinKey', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'MinKey', + path: [ + 'minKey' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + {} + ], + bsonType: 'MinKey' + } + ] + }, + { + name: 'null', + path: [ + 'null' + ], + count: 1, + type: 'Null', + probability: 0.8, + hasDuplicates: true, + types: [ + { + name: 'Null', + path: [ + 'null' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: true, + bsonType: 'Null' + } + ] + }, + { + name: 'object', + path: [ + 'object' + ], + count: 1, + type: 'Document', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Document', + path: [ + 'object' + ], + count: 1, + probability: 0.8, + bsonType: 'Document', + fields: [ + { + name: 'key', + path: [ + 'object', + 'key' + ], + count: 1, + type: 'String', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'String', + path: [ + 'object', + 'key' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 'value' + ], + bsonType: 'String' + } + ] + } + ] + } + ] + }, + { + name: 'objectId', + path: [ + 'objectId' + ], + count: 1, + type: 'ObjectId', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'ObjectId', + path: [ + 'objectId' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + '642d766c7300158b1f22e975' + ], + bsonType: 'ObjectId' + } + ] + }, + { + name: 'regex', + path: [ + 'regex' + ], + count: 1, + type: 'BSONRegExp', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'BSONRegExp', + path: [ + 'regex' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + pattern: 'pattern', + options: 'i' + } + ], + bsonType: 'BSONRegExp' + } + ] + }, + { + name: 'string', + path: [ + 'string' + ], + count: 1, + type: 'String', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'String', + path: [ + 'string' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 'Hello, world!' + ], + bsonType: 'String' + } + ] + }, + { + name: 'symbol', + path: [ + 'symbol' + ], + count: 1, + type: 'BSONSymbol', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'BSONSymbol', + path: [ + 'symbol' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + 'symbol' + ], + bsonType: 'BSONSymbol' + } + ] + }, + { + name: 'timestamp', + path: [ + 'timestamp' + ], + count: 1, + type: 'Timestamp', + probability: 0.8, + hasDuplicates: false, + types: [ + { + name: 'Timestamp', + path: [ + 'timestamp' + ], + count: 1, + probability: 0.8, + unique: 1, + hasDuplicates: false, + values: [ + { + $timestamp: '7218556297505931265' + } + ], + bsonType: 'Timestamp' + } + ] + } + ] + }; + const standard = internalSchemaToStandard(internal); + assert.deepStrictEqual(standard, { + bsonType: 'object', + required: [], + properties: { + _id: { + bsonType: 'objectid' + }, + array: { + bsonType: 'array', + items: { + bsonType: 'number' + } + }, + binData: { + bsonType: 'binary' + }, + binaries: { + bsonType: 'object', + properties: { + binaryOld: { + bsonType: 'binary' + }, + compressedTimeSeries: { + bsonType: 'binary' + }, + custom: { + bsonType: 'binary' + }, + encrypted: { + bsonType: 'binary' + }, + functionData: { + bsonType: 'binary' + }, + generic: { + bsonType: 'binary' + }, + md5: { + bsonType: 'binary' + }, + uuid: { + bsonType: 'binary' + }, + uuidOld: { + bsonType: 'binary' + } + }, + required: [] + }, + boolean: { + bsonType: 'boolean' + }, + date: { + bsonType: 'date' + }, + dbRef: { + bsonType: 'dbref' + }, + decimal: { + bsonType: 'decimal128' + }, + double: { + bsonType: 'double' + }, + int: { + bsonType: 'int32' + }, + javascript: { + bsonType: 'code' + }, + javascriptWithScope: { + bsonType: 'code' + }, + long: { + bsonType: 'long' + }, + maxKey: { + bsonType: 'maxkey' + }, + minKey: { + bsonType: 'minkey' + }, + null: { + bsonType: 'null' + }, + object: { + bsonType: 'object', + properties: { + key: { + bsonType: 'string' + } + }, + required: [] + }, + objectId: { + bsonType: 'objectid' + }, + regex: { + bsonType: 'bsonregexp' + }, + string: { + bsonType: 'string' + }, + symbol: { + bsonType: 'bsonsymbol' + }, + timestamp: { + bsonType: 'timestamp' + } + } + }); + }); + + it('nested document/object', function() { + const internal = { + count: 2, + fields: [ + { + name: 'author', + path: [ + 'author' + ], + count: 1, + type: [ + 'Document', + 'Undefined' + ], + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'Document', + path: [ + 'author' + ], + count: 1, + probability: 0.5, + bsonType: 'Document', + fields: [ + { + name: 'name', + path: [ + 'author', + 'name' + ], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: [ + 'author', + 'name' + ], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: [ + 'Peter Sonder' + ], + bsonType: 'String' + } + ] + }, + { + name: 'rating', + path: [ + 'author', + 'rating' + ], + count: 1, + type: 'Double', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'Double', + path: [ + 'author', + 'rating' + ], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: [ + 1.3 + ], + bsonType: 'Double' + } + ] + } + ] + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: [ + 'author' + ], + count: 1, + probability: 0.5 + } + ] + } + ] + }; + const standard = internalSchemaToStandard(internal); + assert.deepStrictEqual(standard, { + bsonType: 'object', + required: ['author'], + properties: { + author: { + bsonType: 'object', + required: ['name', 'rating'], + properties: { + name: { + bsonType: 'string' + }, + rating: { + bsonType: 'double' + } + } + } + } + }); + }); + + describe('arrays', function() { + it('array - single type', function() { + const internal = { + count: 2, + fields: [ + { + name: 'genres', + path: [ + 'genres' + ], + count: 1, + type: [ + 'array', + 'Undefined' + ], + probability: 0.5, + hasDuplicates: false, + types: [ + { + name: 'array', + path: [ + 'genres' + ], + count: 1, + probability: 0.5, + bsonType: 'Array', + types: [ + { + name: 'String', + path: [ + 'genres' + ], + count: 2, + probability: 1, + unique: 2, + hasDuplicates: false, + values: [ + 'crimi', + 'comedy' + ], + bsonType: 'String' + } + ], + totalCount: 2, + lengths: [ + 2 + ], + averageLength: 2 + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: [ + 'genres' + ], + count: 1, + probability: 0.5 + } + ] + } + ] + }; + const standard = internalSchemaToStandard(internal); + assert.deepStrictEqual(standard, { + bsonType: 'object', + required: [], + properties: { + genres: { + bsonType: 'array', + items: { + bsonType: 'string' + } + } + } + }); + }); + + it('array - complex mixed type', function() { + const internal = { + count: 2, + fields: [ + { + name: 'genres', + path: [ + 'genres' + ], + count: 1, + type: [ + 'Array', + 'Undefined' + ], + probability: 0.5, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: [ + 'genres' + ], + count: 1, + probability: 0.5, + bsonType: 'Array', + types: [ + { + name: 'String', + path: [ + 'genres' + ], + count: 2, + probability: 0.6666666666666666, + unique: 2, + hasDuplicates: false, + values: [ + 'crimi', + 'comedy' + ], + bsonType: 'String' + }, + { + name: 'Document', + path: [ + 'genres' + ], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Document', + fields: [ + { + name: 'long', + path: [ + 'genres', + 'long' + ], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: [ + 'genres', + 'long' + ], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: [ + 'science fiction' + ], + bsonType: 'String' + } + ] + }, + { + name: 'short', + path: [ + 'genres', + 'short' + ], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: [ + 'genres', + 'short' + ], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: [ + 'scifi' + ], + bsonType: 'String' + } + ] + } + ] + } + ], + totalCount: 3, + lengths: [ + 3 + ], + averageLength: 3 + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: [ + 'genres' + ], + count: 1, + probability: 0.5 + } + ] + } + ] + }; + const standard = internalSchemaToStandard(internal); + assert.deepStrictEqual(standard, { + bsonType: 'object', + required: [], + properties: { + genres: { + bsonType: 'array', + items: { + anyOf: [ + { + bsonType: 'string' + }, + { + bsonType: 'object', + required: ['long', 'short'], + properties: { + long: { + bsonType: 'string' + }, + short: { + bsonType: 'string' + } + } + } + ] + } + } + } + }); + }); + + it('array - simple mixed type', function() { + const internal = { + count: 2, + fields: [ + { + name: 'arrayMixedType', + path: [ + 'arrayMixedType' + ], + count: 1, + type: 'Array', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: [ + 'arrayMixedType' + ], + count: 1, + probability: 1, + bsonType: 'Array', + types: [ + { + name: 'int32', + path: [ + 'arrayMixedType' + ], + count: 2, + probability: 0.6666666666666666, + unique: 2, + hasDuplicates: false, + values: [ + 1, + 3 + ], + bsonType: 'Int32' + }, + { + name: 'String', + path: [ + 'arrayMixedType' + ], + count: 1, + probability: 0.3333333333333333, + unique: 1, + hasDuplicates: false, + values: [ + '2' + ], + bsonType: 'string' + } + ], + totalCount: 3, + lengths: [ + 3 + ], + averageLength: 3 + } + ] + } + ] + }; + const standard = internalSchemaToStandard(internal); + assert.deepStrictEqual(standard, { + bsonType: 'object', + required: ['arrayMixedType'], + properties: { + arrayMixedType: { + bsonType: 'array', + items: { + bsonType: ['int32', 'string'] + } + } + } + }); + }); + }); + + describe('mixed types', function() { + it('simple mixed type', function() { + const internal = { + count: 2, + fields: [ + { + name: 'mixedType', + path: [ + 'mixedType' + ], + count: 2, + type: [ + 'Int32', + 'String', + 'Undefined' + ], + probability: 0.6666666666666666, + hasDuplicates: false, + types: [ + { + name: 'Int32', + path: [ + 'mixedType' + ], + count: 1, + probability: 0.3333333333333333, + unique: 1, + hasDuplicates: false, + values: [ + 1 + ], + bsonType: 'Int32' + }, + { + name: 'String', + path: [ + 'mixedType' + ], + count: 1, + probability: 0.3333333333333333, + unique: 1, + hasDuplicates: false, + values: [ + 'abc' + ], + bsonType: 'String' + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: [ + 'mixedType' + ], + count: 1, + probability: 0.3333333333333333 + } + ] + } + ] + }; + const standard = internalSchemaToStandard(internal); + assert.deepStrictEqual(standard, { + bsonType: 'object', + required: [], + properties: { + mixedType: { + bsonType: ['int32', 'string'] + } + } + }); + }); + + it('complex mixed type', function() { + const internal = { + count: 2, + fields: [ + { + name: 'mixedComplexType', + path: [ + 'mixedComplexType' + ], + count: 2, + type: [ + 'Array', + 'Document', + 'Undefined' + ], + probability: 0.6666666666666666, + hasDuplicates: false, + types: [ + { + name: 'Array', + path: [ + 'mixedComplexType' + ], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Array', + types: [ + { + name: 'Int32', + path: [ + 'mixedComplexType' + ], + count: 3, + probability: 1, + unique: 3, + hasDuplicates: false, + values: [ + 1, + 2, + 3 + ], + bsonType: 'Int32' + } + ], + totalCount: 3, + lengths: [ + 3 + ], + averageLength: 3 + }, + { + name: 'Document', + path: [ + 'mixedComplexType' + ], + count: 1, + probability: 0.3333333333333333, + bsonType: 'Document', + fields: [ + { + name: 'a', + path: [ + 'mixedComplexType', + 'a' + ], + count: 1, + type: 'String', + probability: 1, + hasDuplicates: false, + types: [ + { + name: 'String', + path: [ + 'mixedComplexType', + 'a' + ], + count: 1, + probability: 1, + unique: 1, + hasDuplicates: false, + values: [ + 'bc' + ], + bsonType: 'String' + } + ] + } + ] + }, + { + name: 'Undefined', + bsonType: 'Undefined', + unique: 1, + hasDuplicates: false, + path: [ + 'mixedComplexType' + ], + count: 1, + probability: 0.3333333333333333 + } + ] + } + ] + }; + const standard = internalSchemaToStandard(internal); + assert.deepStrictEqual(standard, { + bsonType: 'object', + required: [], + properties: { + mixedComplexType: { + anyOf: [ + { + bsonType: 'array', + items: { + bsonType: 'int32' + } + }, + { + bsonType: 'object', + required: ['a'], + properties: { + a: { + bsonType: 'string' + } + } + } + ] + } + } + }); + }); + }); + }); +}); diff --git a/src/schema-convertors/internalToMongoDB.ts b/src/schema-convertors/internalToMongoDB.ts new file mode 100644 index 0000000..d120243 --- /dev/null +++ b/src/schema-convertors/internalToMongoDB.ts @@ -0,0 +1,68 @@ +import { ArraySchemaType, DocumentSchemaType, Schema as InternalSchema, SchemaType } from '../schema-analyzer'; +import { MongoDBJSONSchema } from '../types'; + +const internalTypeToBsonType = (type: string) => type === 'Document' ? 'object' : type.toLowerCase(); + +function parseType(type: SchemaType, signal?: AbortSignal): MongoDBJSONSchema { + if (signal?.aborted) throw new Error('Operation aborted'); + const schema: MongoDBJSONSchema = { + bsonType: internalTypeToBsonType(type.bsonType) + }; + switch (type.bsonType) { + case 'Array': + schema.items = parseTypes((type as ArraySchemaType).types); + break; + case 'Document': + Object.assign(schema, + parseFields((type as DocumentSchemaType).fields, signal) + ); + break; + } + + return schema; +} + +function parseTypes(types: SchemaType[], signal?: AbortSignal): MongoDBJSONSchema { + if (signal?.aborted) throw new Error('Operation aborted'); + const definedTypes = types.filter(type => type.bsonType.toLowerCase() !== 'undefined'); + const isSingleType = definedTypes.length === 1; + if (isSingleType) { + return parseType(definedTypes[0], signal); + } + const parsedTypes = definedTypes.map(type => parseType(type, signal)); + if (definedTypes.some(type => ['Document', 'Array'].includes(type.bsonType))) { + return { + anyOf: parsedTypes + }; + } + return { + bsonType: definedTypes.map((type) => internalTypeToBsonType(type.bsonType)) + }; +} + +function parseFields(fields: DocumentSchemaType['fields'], signal?: AbortSignal): { + required: MongoDBJSONSchema['required'], + properties: MongoDBJSONSchema['properties'], +} { + const required = []; + const properties: MongoDBJSONSchema['properties'] = {}; + for (const field of fields) { + if (signal?.aborted) throw new Error('Operation aborted'); + if (field.probability === 1) required.push(field.name); + properties[field.name] = parseTypes(field.types, signal); + } + + return { required, properties }; +} + +export default function internalSchemaToMongodb( + internalSchema: InternalSchema, + options: { + signal?: AbortSignal +} = {}): MongoDBJSONSchema { + const schema: MongoDBJSONSchema = { + bsonType: 'object', + ...parseFields(internalSchema.fields, options.signal) + }; + return schema; +} diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..2b6c0c3 --- /dev/null +++ b/src/types.ts @@ -0,0 +1,22 @@ +import { type JSONSchema4 } from 'json-schema'; + +export type StandardJSONSchema = JSONSchema4; + +export type MongoDBJSONSchema = Pick & { + bsonType?: string | string[]; + properties?: Record; + items?: MongoDBJSONSchema | MongoDBJSONSchema[]; + anyOf?: MongoDBJSONSchema[]; +} + +export type ExtendedJSONSchema = StandardJSONSchema & { + ['x-bsonType']: string; + ['x-metadata']: { + hasDuplicates: boolean; + probability: number; + count: number; + }; + ['x-sampleValues']: any[]; +} + +export type AnyIterable = Iterable | AsyncIterable; diff --git a/test/analyze-documents.test.ts b/test/analyze-documents.test.ts new file mode 100644 index 0000000..3fa8101 --- /dev/null +++ b/test/analyze-documents.test.ts @@ -0,0 +1,25 @@ +import { analyzeDocuments } from '../src'; +import convertors from '../src/schema-convertors'; +import sinon from 'sinon'; +import assert from 'assert'; + +describe('analyzeDocuments', function() { + const docs = [{}]; + + it('Converts lazily', async function() { + const convertSpy = sinon.spy(convertors, 'internalSchemaToStandard'); + const analyzeResults = await analyzeDocuments(docs); + assert.strictEqual(convertSpy.called, false); + await analyzeResults.getStandardJsonSchema(); + assert.strictEqual(convertSpy.calledOnce, true); + }); + + it('Only converts the same format once', async function() { + const convertSpy = sinon.spy(convertors, 'internalSchemaToExtended'); + const analyzeResults = await analyzeDocuments(docs); + await analyzeResults.getExtendedJsonSchema(); + await analyzeResults.getExtendedJsonSchema(); + await analyzeResults.getExtendedJsonSchema(); + assert.strictEqual(convertSpy.calledOnce, true); + }); +});