diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 9fb507a8dc1..ba02babecb0 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -41,7 +41,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -52,7 +52,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v3 + uses: github/codeql-action/autobuild@v4 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -66,4 +66,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index af8647ba00a..c2270622ebe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,24 @@ Dependencies: - Remove SQLite +## Version 25.03.24 +Fixes: +- [jobs] Fix condition for scheduling alert job + +Enterprise Fixes: +- [compliance-hub] Fixed query patterns +- [data-manager] Fixed bug preventing transformation of events ending in a dot +- [data-manager] Fixed segment data deletion +- [license] Stop sending metric after license expired +- [users] Fix add/remove user to profile group +- [users] Remove link to profile group page after removing user from group + +Dependencies +- Bump @faker-js/faker from 10.0.0 to 10.1.0 in /ui-tests +- Bump countly-sdk-nodejs from 24.10.2 to 24.10.3 +- Bump lint-staged from 16.2.3 to 16.2.4 +- Bump puppeteer from 24.23.0 to 24.24.1 + ## Version 25.03.23 Fixes: - [events] Entries in the event list are now sorted alphabetically diff --git a/api/parts/data/usage.js b/api/parts/data/usage.js index 6d0c45a9cdf..aac6d0fcaf2 100644 --- a/api/parts/data/usage.js +++ b/api/parts/data/usage.js @@ -1125,7 +1125,7 @@ plugins.register("/sdk/user_properties", async function(ob) { userProps.av_major = null; userProps.av_minor = null; userProps.av_patch = null; - userProps.av_rel = null; + userProps.av_prerel = null; userProps.av_build = null; } } diff --git a/api/parts/jobs/job.js b/api/parts/jobs/job.js index 1bf39c2e6a1..46577397cc8 100644 --- a/api/parts/jobs/job.js +++ b/api/parts/jobs/job.js @@ -254,32 +254,33 @@ class Job extends EventEmitter { this._json.next = next.getTime(); } - if (this.name !== "alerts:monitor") { - //check if any job already scheduled or running - let query = { - status: {"$in": [STATUS.SCHEDULED, STATUS.RUNNING]}, - name: this.name, - }; - if (this._id) { - query._id = {$ne: this._id}; - } - var self = this; - return new Promise((resolve, reject) => { - Job.findMany(this.db(), query).then(existing => { - if (existing && existing.length) { - log.d('Job already scheduled or running: %j', existing); - this._json.status = STATUS.CANCELLED; //set this as cancelled now as we have other scheduled - } - else { - self._save().then(resolve, reject); - } - }); - }); + //check if any job already scheduled or running + let query = { + status: {"$in": [STATUS.SCHEDULED, STATUS.RUNNING]}, + name: this.name, + }; + + if (this.name === 'alerts:monitor' && this.data && Object.keys(this.data).length) { + query.data = this.data; } - else { - return this._save(); + + if (this._id) { + query._id = {$ne: this._id}; } + + var self = this; + return new Promise((resolve, reject) => { + Job.findMany(this.db(), query).then(existing => { + if (existing && existing.length) { + log.d('Job already scheduled or running: %j', existing); + this._json.status = STATUS.CANCELLED; //set this as cancelled now as we have other scheduled + } + else { + self._save().then(resolve, reject); + } + }); + }); } /** @@ -1142,4 +1143,4 @@ module.exports = { STATUS: STATUS, STATUS_MAP: STATUS_MAP, debounce: debounce -}; \ No newline at end of file +}; diff --git a/package-lock.json b/package-lock.json index c42fcea6ea7..68499dce819 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2081,17 +2081,17 @@ } }, "node_modules/@puppeteer/browsers": { - "version": "2.10.10", - "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.10.10.tgz", - "integrity": "sha512-3ZG500+ZeLql8rE0hjfhkycJjDj0pI/btEh3L9IkWUYcOrgP0xCNRq3HbtbqOPbvDhFaAWD88pDFtlLv8ns8gA==", + "version": "2.10.12", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.10.12.tgz", + "integrity": "sha512-mP9iLFZwH+FapKJLeA7/fLqOlSUwYpMwjR1P5J23qd4e7qGJwecJccJqHYrjw33jmIZYV4dtiTHPD/J+1e7cEw==", "license": "Apache-2.0", "dependencies": { "debug": "^4.4.3", "extract-zip": "^2.0.1", "progress": "^2.0.3", "proxy-agent": "^6.5.0", - "semver": "^7.7.2", - "tar-fs": "^3.1.0", + "semver": "^7.7.3", + "tar-fs": "^3.1.1", "yargs": "^17.7.2" }, "bin": { @@ -2823,15 +2823,23 @@ "license": "MIT" }, "node_modules/bare-events": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.7.0.tgz", - "integrity": "sha512-b3N5eTW1g7vXkw+0CXh/HazGTcO5KYuu/RCNaJbDMPI6LHDi+7qe8EmxKUVe1sUbY2KZOVZFyj62x0OEz9qyAA==", - "license": "Apache-2.0" + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.8.0.tgz", + "integrity": "sha512-AOhh6Bg5QmFIXdViHbMc2tLDsBIRxdkIaIddPslJF9Z5De3APBScuqGP2uThXnIpqFrgoxMNC6km7uXNIMLHXA==", + "license": "Apache-2.0", + "peerDependencies": { + "bare-abort-controller": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + } + } }, "node_modules/bare-fs": { - "version": "4.4.7", - "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.4.7.tgz", - "integrity": "sha512-huJQxUWc2d1T+6dxnC/FoYpBgEHzJp33mYZqFtQqTTPPyP9xPvmjC16VpR4wTte4ZKd5VxkFAcfDYi51iwWMcg==", + "version": "4.4.10", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.4.10.tgz", + "integrity": "sha512-arqVF+xX/rJHwrONZaSPhlzleT2gXwVs9rsAe1p1mIVwWZI2A76/raio+KwwxfWMO8oV9Wo90EaUkS2QwVmy4w==", "license": "Apache-2.0", "optional": true, "dependencies": { @@ -2896,9 +2904,9 @@ } }, "node_modules/bare-url": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.2.2.tgz", - "integrity": "sha512-g+ueNGKkrjMazDG3elZO1pNs3HY5+mMmOet1jtKyhOaCnkLzitxf26z7hoAEkDNgdNmnc1KIlt/dw6Po6xZMpA==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.3.0.tgz", + "integrity": "sha512-c+RCqMSZbkz97Mw1LWR0gcOqwK82oyYKfLoHJ8k13ybi1+I80ffdDzUy0TdAburdrR/kI0/VuN8YgEnJqX+Nyw==", "license": "Apache-2.0", "optional": true, "dependencies": { @@ -3828,9 +3836,9 @@ "link": true }, "node_modules/countly-sdk-nodejs": { - "version": "24.10.2", - "resolved": "https://registry.npmjs.org/countly-sdk-nodejs/-/countly-sdk-nodejs-24.10.2.tgz", - "integrity": "sha512-ags7l6aXy31ZYgXHgE/GAH/12PhYp9Mzn2vtQHSxhKl0BT6AdlSu3XEYeFmbk8ouwHN8AnGB7YeqOiw5Z3TrxQ==", + "version": "24.10.3", + "resolved": "https://registry.npmjs.org/countly-sdk-nodejs/-/countly-sdk-nodejs-24.10.3.tgz", + "integrity": "sha512-Xf5P6AuyGR63s91ZHAPZbz5vq6xnP0hSUq4tA5qVvZcGJMhzzeOtG1sadY5oXYFfbvKolhrzcL3QlstkUfqz0A==", "license": "MIT" }, "node_modules/countly-sdk-web": { @@ -7802,16 +7810,16 @@ "license": "MIT" }, "node_modules/lint-staged": { - "version": "16.2.3", - "resolved": "https://registry.npmjs.org/lint-staged/-/lint-staged-16.2.3.tgz", - "integrity": "sha512-1OnJEESB9zZqsp61XHH2fvpS1es3hRCxMplF/AJUDa8Ho8VrscYDIuxGrj3m8KPXbcWZ8fT9XTMUhEQmOVKpKw==", + "version": "16.2.4", + "resolved": "https://registry.npmjs.org/lint-staged/-/lint-staged-16.2.4.tgz", + "integrity": "sha512-Pkyr/wd90oAyXk98i/2KwfkIhoYQUMtss769FIT9hFM5ogYZwrk+GRE46yKXSg2ZGhcJ1p38Gf5gmI5Ohjg2yg==", "dev": true, "license": "MIT", "dependencies": { "commander": "^14.0.1", "listr2": "^9.0.4", "micromatch": "^4.0.8", - "nano-spawn": "^1.0.3", + "nano-spawn": "^2.0.0", "pidtree": "^0.6.0", "string-argv": "^0.3.2", "yaml": "^2.8.1" @@ -8895,9 +8903,9 @@ "optional": true }, "node_modules/nano-spawn": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/nano-spawn/-/nano-spawn-1.0.3.tgz", - "integrity": "sha512-jtpsQDetTnvS2Ts1fiRdci5rx0VYws5jGyC+4IYOTnIQ/wwdf6JdomlHBwqC3bJYOvaKu0C2GSZ1A60anrYpaA==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/nano-spawn/-/nano-spawn-2.0.0.tgz", + "integrity": "sha512-tacvGzUY5o2D8CBh2rrwxyNojUsZNU2zjNTzKQrkgGJQTbGAfArVWXSKMBokBeeg6C7OLRGUEyoFlYbfeWQIqw==", "dev": true, "license": "MIT", "engines": { @@ -10096,17 +10104,17 @@ } }, "node_modules/puppeteer": { - "version": "24.23.0", - "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-24.23.0.tgz", - "integrity": "sha512-BVR1Lg8sJGKXY79JARdIssFWK2F6e1j+RyuJP66w4CUmpaXjENicmA3nNpUXA8lcTdDjAndtP+oNdni3T/qQqA==", + "version": "24.25.0", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-24.25.0.tgz", + "integrity": "sha512-P3rUaom2w/Ubrnz3v3kSbxGkN7SpbtQeGRPb7iO86Bv/dAz2WUmGQBHr37W/Rp1fbAocMvu0rHFbCIJvjiNhGw==", "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { - "@puppeteer/browsers": "2.10.10", + "@puppeteer/browsers": "2.10.12", "chromium-bidi": "9.1.0", "cosmiconfig": "^9.0.0", "devtools-protocol": "0.0.1508733", - "puppeteer-core": "24.23.0", + "puppeteer-core": "24.25.0", "typed-query-selector": "^2.12.0" }, "bin": { @@ -10117,17 +10125,17 @@ } }, "node_modules/puppeteer-core": { - "version": "24.23.0", - "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.23.0.tgz", - "integrity": "sha512-yl25C59gb14sOdIiSnJ08XiPP+O2RjuyZmEG+RjYmCXO7au0jcLf7fRiyii96dXGUBW7Zwei/mVKfxMx/POeFw==", + "version": "24.25.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.25.0.tgz", + "integrity": "sha512-8Xs6q3Ut+C8y7sAaqjIhzv1QykGWG4gc2mEZ2mYE7siZFuRp4xQVehOf8uQKSQAkeL7jXUs3mknEeiqnRqUKvQ==", "license": "Apache-2.0", "dependencies": { - "@puppeteer/browsers": "2.10.10", + "@puppeteer/browsers": "2.10.12", "chromium-bidi": "9.1.0", "debug": "^4.4.3", "devtools-protocol": "0.0.1508733", "typed-query-selector": "^2.12.0", - "webdriver-bidi-protocol": "0.3.6", + "webdriver-bidi-protocol": "0.3.7", "ws": "^8.18.3" }, "engines": { @@ -11905,9 +11913,9 @@ } }, "node_modules/webdriver-bidi-protocol": { - "version": "0.3.6", - "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.3.6.tgz", - "integrity": "sha512-mlGndEOA9yK9YAbvtxaPTqdi/kaCWYYfwrZvGzcmkr/3lWM+tQj53BxtpVd6qbC6+E5OnHXgCcAhre6AkXzxjA==", + "version": "0.3.7", + "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.3.7.tgz", + "integrity": "sha512-wIx5Gu/LLTeexxilpk8WxU2cpGAKlfbWRO5h+my6EMD1k5PYqM1qQO1MHUFf4f3KRnhBvpbZU7VkizAgeSEf7g==", "license": "Apache-2.0" }, "node_modules/webidl-conversions": { diff --git a/plugins/compliance-hub/api/api.js b/plugins/compliance-hub/api/api.js index 0d466cdbe94..9f1c05b5803 100644 --- a/plugins/compliance-hub/api/api.js +++ b/plugins/compliance-hub/api/api.js @@ -16,6 +16,13 @@ const FEATURE_NAME = 'compliance_hub'; plugins.internalDrillEvents.push("[CLY]_consent"); + plugins.register("/master", function() { + common.db.collection('consent_history').ensureIndex({app_id: 1, device_id: 1}, function() {}); + common.db.collection('consent_history').ensureIndex({app_id: 1, uid: 1}, function() {}); + common.db.collection('consent_history').ensureIndex({app_id: 1, type: 1}, function() {}); + common.db.collection('consent_history').ensureIndex({app_id: 1, ts: 1}, function() {}); + }); + //write api call plugins.register("/sdk/user_properties", function(ob) { var params = ob.params; @@ -162,7 +169,8 @@ const FEATURE_NAME = 'compliance_hub'; query = {}; } } - common.db.collection("consent_history").count(query, function(err, total) { + query.app_id = params.app_id.toString(); + common.db.collection("consent_history").countDocuments(query, function(err, total) { if (err) { common.returnMessage(params, 400, err); } @@ -338,7 +346,7 @@ const FEATURE_NAME = 'compliance_hub'; var newUid = ob.newUser.uid; if (oldUid !== newUid) { return new Promise(function(resolve, reject) { - common.db.collection('consent_history').update({uid: oldUid}, {'$set': {uid: newUid}}, {multi: true}, function(err) { + common.db.collection('consent_history').update({uid: oldUid}, {'$set': {app_id: ob.app_id, uid: newUid}}, {multi: true}, function(err) { if (err) { reject(err); return; @@ -364,7 +372,7 @@ const FEATURE_NAME = 'compliance_hub'; plugins.register("/i/apps/delete", function(ob) { var appId = ob.appId; common.db.collection('consents').remove({'_id': {$regex: appId + ".*"}}, function() {}); - common.db.collection('consent_history').drop(function() {}); + common.db.collection('consent_history').deleteMany({app_id: appId}, function() {}); if (common.drillDb) { common.drillDb.collection("drill_events" + crypto.createHash('sha1').update("[CLY]_consent" + appId).digest('hex')).drop(function() {}); } @@ -373,7 +381,7 @@ const FEATURE_NAME = 'compliance_hub'; plugins.register("/i/apps/reset", function(ob) { var appId = ob.appId; common.db.collection('consents').remove({'_id': {$regex: appId + ".*"}}, function() {}); - common.db.collection('consent_history').drop(function() {}); + common.db.collection('consent_history').deleteMany({app_id: appId}, function() {}); if (common.drillDb) { common.drillDb.collection("drill_events" + crypto.createHash('sha1').update("[CLY]_consent" + appId).digest('hex')).drop(function() {}); } @@ -382,6 +390,7 @@ const FEATURE_NAME = 'compliance_hub'; plugins.register("/i/apps/clear_all", function(ob) { var appId = ob.appId; common.db.collection('consents').remove({'_id': {$regex: appId + ".*"}}, function() {}); + common.db.collection('consent_history').deleteMany({app_id: appId}, function() {}); if (common.drillDb) { common.drillDb.collection("drill_events" + crypto.createHash('sha1').update("[CLY]_consent" + appId).digest('hex')).drop(function() {}); } @@ -391,6 +400,7 @@ const FEATURE_NAME = 'compliance_hub'; var appId = ob.appId; var ids = ob.ids; common.db.collection('consents').remove({$and: [{'_id': {$regex: appId + ".*"}}, {'_id': {$nin: ids}}]}, function() {}); + common.db.collection('consent_history').deleteMany({app_id: appId, ts: {$lt: ob.moment.valueOf()}}, function() {}); if (common.drillDb) { common.drillDb.collection("drill_events" + crypto.createHash('sha1').update("[CLY]_consent" + appId).digest('hex')).remove({ts: {$lt: ob.moment.valueOf()}}, function() {}); } diff --git a/plugins/compliance-hub/install.js b/plugins/compliance-hub/install.js index 9fbea495bed..e69de29bb2d 100644 --- a/plugins/compliance-hub/install.js +++ b/plugins/compliance-hub/install.js @@ -1,32 +0,0 @@ -var pluginManager = require('../pluginManager.js'), - async = require('async'); - -console.log("Installing compliance-hub plugin"); -pluginManager.dbConnection().then((countlyDb) => { - countlyDb.collection('apps').find({}).toArray(function(err, apps) { - - if (!apps || err) { - console.log("No apps to upgrade"); - countlyDb.close(); - return; - } - function upgrade(app, done) { - console.log("Adding compliance-hub indexes to " + app.name); - var cnt = 0; - function cb() { - cnt++; - if (cnt == 4) { - done(); - } - } - countlyDb.collection('consent_history').ensureIndex({device_id: 1}, cb); - countlyDb.collection('consent_history').ensureIndex({uid: 1}, cb); - countlyDb.collection('consent_history').ensureIndex({type: 1}, cb); - countlyDb.collection('consent_history').ensureIndex({ts: 1}, cb); - } - async.forEach(apps, upgrade, function() { - console.log("Compliance hub plugin installation finished"); - countlyDb.close(); - }); - }); -}); \ No newline at end of file diff --git a/ui-tests/package-lock.json b/ui-tests/package-lock.json index ae822a5d290..ba44c9a2686 100644 --- a/ui-tests/package-lock.json +++ b/ui-tests/package-lock.json @@ -66,9 +66,9 @@ } }, "node_modules/@faker-js/faker": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/@faker-js/faker/-/faker-10.0.0.tgz", - "integrity": "sha512-UollFEUkVXutsaP+Vndjxar40Gs5JL2HeLcl8xO1QAjJgOdhc3OmBFWyEylS+RddWaaBiAzH+5/17PLQJwDiLw==", + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/@faker-js/faker/-/faker-10.1.0.tgz", + "integrity": "sha512-C3mrr3b5dRVlKPJdfrAXS8+dq+rq8Qm5SNRazca0JKgw1HQERFmrVb0towvMmw5uu8hHKNiQasMaR/tydf3Zsg==", "funding": [ { "type": "opencollective",