From 25.03.24

.github/workflows/codeql-analysis.yml

@@ -41,7 +41,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
+      uses: github/codeql-action/init@v4
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -52,7 +52,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v3
+      uses: github/codeql-action/autobuild@v4
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -66,4 +66,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      uses: github/codeql-action/analyze@v4

CHANGELOG.md

@@ -1,3 +1,21 @@
+## Version 25.03.24
+Fixes:
+- [jobs] Fix condition for scheduling alert job
+
+Enterprise Fixes:
+- [compliance-hub] Fixed query patterns
+- [data-manager] Fixed bug preventing transformation of events ending in a dot
+- [data-manager] Fixed segment data deletion
+- [license] Stop sending metric after license expired
+- [users] Fix add/remove user to profile group
+- [users] Remove link to profile group page after removing user from group
+
+Dependencies
+- Bump @faker-js/faker from 10.0.0 to 10.1.0 in /ui-tests
+- Bump countly-sdk-nodejs from 24.10.2 to 24.10.3
+- Bump lint-staged from 16.2.3 to 16.2.4
+- Bump puppeteer from 24.23.0 to 24.24.1
+
 ## Version 25.03.23
 Fixes:
 - [events] Entries in the event list are now sorted alphabetically

api/api.js

@@ -16,6 +16,7 @@ const {WriteBatcher, ReadBatcher, InsertBatcher} = require('./parts/data/batcher
 const pack = require('../package.json');
 const versionInfo = require('../frontend/express/version.info.js');
 const moment = require("moment");
+const tracker = require('./parts/mgmt/tracker.js');
 
 var t = ["countly:", "api"];
 common.processRequest = processRequest;
@@ -38,6 +39,9 @@ else {
 process.title = t.join(' ');
 
 plugins.connectToAllDatabases().then(function() {
+    plugins.loadConfigs(common.db, function() {
+        tracker.enable();
+    });
     common.writeBatcher = new WriteBatcher(common.db);
     common.readBatcher = new ReadBatcher(common.db);
     common.insertBatcher = new InsertBatcher(common.db);
@@ -142,6 +146,41 @@ plugins.connectToAllDatabases().then(function() {
         require('./utils/log.js').ipcHandler(msg);
     });
 
+    /**
+    * Set tracking config
+    */
+    plugins.setConfigs("tracking", {
+        self_tracking_app: "",
+        self_tracking_url: "",
+        self_tracking_app_key: "",
+        self_tracking_id_policy: "_id",
+        self_tracking_sessions: true,
+        self_tracking_events: true,
+        self_tracking_views: true,
+        self_tracking_feedback: true,
+        self_tracking_user_details: true,
+        server_sessions: true,
+        server_events: true,
+        server_crashes: true,
+        server_views: true,
+        server_feedback: true,
+        server_user_details: true,
+        /*user_sessions: true,
+        user_events: true,
+        user_crashes: true,
+        user_views: true,
+        user_feedback: true,
+        user_details: true*/
+    });
+
+    /*plugins.setUserConfigs("tracking", {
+        user_sessions: false,
+        user_events: false,
+        user_crashes: false,
+        user_views: false,
+        user_feedback: false
+    });*/
+
     /**
     * Initialize Plugins
     */
@@ -309,7 +348,7 @@ plugins.connectToAllDatabases().then(function() {
         // Allow configs to load & scanner to find all jobs classes
         setTimeout(() => {
             jobs.job('api:topEvents').replace().schedule('at 00:01 am ' + 'every 1 day');
-            jobs.job('api:ping').replace().schedule('every 1 day');
+            jobs.job('api:ping').replace().schedule('at 00:01 am ' + 'every 1 day');
             jobs.job('api:clear').replace().schedule('every 1 day');
             jobs.job('api:clearTokens').replace().schedule('every 1 day');
             jobs.job('api:clearAutoTasks').replace().schedule('every 1 day');

api/jobs/ping.js

@@ -1,12 +1,8 @@
 'use strict';
 
 const job = require('../parts/jobs/job.js'),
-    log = require('../utils/log.js')('job:ping'),
-    countlyConfig = require("../../frontend/express/config.js"),
-    versionInfo = require('../../frontend/express/version.info'),
     plugins = require('../../plugins/pluginManager.js'),
-    request = require('countly-request')(plugins.getConfig("security"));
-
+    tracker = require('../parts/mgmt/tracker.js');
 
 /** Class for the job of pinging servers **/
 class PingJob extends job.Job {
@@ -16,86 +12,114 @@ class PingJob extends job.Job {
      * @param {done} done callback
      */
     run(db, done) {
-        request({strictSSL: false, uri: (process.env.COUNTLY_CONFIG_PROTOCOL || "http") + "://" + (process.env.COUNTLY_CONFIG_HOSTNAME || "localhost") + (countlyConfig.path || "") + "/configs"}, function() {});
-        var countlyConfigOrig = JSON.parse(JSON.stringify(countlyConfig));
-        var url = "https://count.ly/configurations/ce/tracking";
-        if (versionInfo.type !== "777a2bf527a18e0fffe22fb5b3e322e68d9c07a6") {
-            url = "https://count.ly/configurations/ee/tracking";
-        }
-        plugins.loadConfigs(db, function() {
+        plugins.loadConfigs(db, async function() {
             const offlineMode = plugins.getConfig("api").offline_mode;
-            const { countly_tracking } = plugins.getConfig('frontend');
             if (!offlineMode) {
-                request(url, function(err, response, body) {
-                    if (typeof body === "string") {
-                        try {
-                            body = JSON.parse(body);
-                        }
-                        catch (ex) {
-                            body = null;
-                        }
+                var server = tracker.getBulkServer();
+                var user = tracker.getBulkUser(server);
+                if (!user) {
+                    return done();
+                }
+
+                try {
+                    var custom = await tracker.getAllData();
+                    if (Object.keys(custom).length) {
+                        user.user_details({"custom": custom });
                     }
-                    if (body) {
-                        if (countlyConfigOrig.web.use_intercom && typeof body.intercom !== "undefined") {
-                            countlyConfig.web.use_intercom = body.intercom;
-                        }
-                        if (typeof countlyConfigOrig.web.track === "undefined" && typeof body.stats !== "undefined") {
-                            if (body.stats) {
-                                countlyConfig.web.track = null;
-                            }
-                            else {
-                                countlyConfig.web.track = "none";
-                            }
-                        }
+                }
+                catch (ex) {
+                    console.log("Error collecting server data:", ex);
+                }
+                var days = 90;
+                var current_sync = Date.now();
+
+                // Atomically retrieve old last_sync value and set new one
+                var syncResult = await db.collection("plugins").findOneAndUpdate(
+                    {_id: "version"},
+                    {$set: {last_sync: current_sync}},
+                    {
+                        upsert: true,
+                        returnDocument: 'before',
+                        projection: {last_sync: 1}
                     }
-                    log.d(err, body, countlyConfigOrig, countlyConfig);
-                    if (countly_tracking) {
-                        db.collection("members").findOne({global_admin: true}, function(err2, member) {
-                            if (!err2 && member) {
-                                var date = new Date();
-                                let domain = plugins.getConfig('api').domain;
+                );
 
-                                try {
-                                    // try to extract hostname from full domain url
-                                    const urlObj = new URL(domain);
-                                    domain = urlObj.hostname;
-                                }
-                                catch (_) {
-                                    // do nothing, domain from config will be used as is
-                                }
+                var last_sync = syncResult.value ? syncResult.value.last_sync : null;
+                if (last_sync) {
+                    days = Math.floor((new Date().getTime() - last_sync) / (1000 * 60 * 60 * 24));
+                }
 
-                                request({
-                                    uri: "https://stats.count.ly/i",
-                                    method: "GET",
-                                    timeout: 4E3,
-                                    qs: {
-                                        device_id: domain,
-                                        app_key: "e70ec21cbe19e799472dfaee0adb9223516d238f",
-                                        timestamp: Math.floor(date.getTime() / 1000),
-                                        hour: date.getHours(),
-                                        dow: date.getDay(),
-                                        no_meta: true,
-                                        events: JSON.stringify([
-                                            {
-                                                key: "PING",
-                                                count: 1
-                                            }
-                                        ])
+                if (days > 0) {
+                    //calculate seconds timestamp of days before today
+                    var startTs = Math.round((new Date().getTime() - (30 * 24 * 60 * 60 * 1000)) / 1000);
+
+                    //sync server events - use aggregation pipeline to group by day and action on MongoDB side
+                    var aggregationPipeline = [
+                        // Match documents with timestamp greater than startTs and valid action
+                        {
+                            $match: {
+                                ts: { $gt: startTs }
+                            }
+                        },
+                        // Add calculated fields for day grouping
+                        {
+                            $addFields: {
+                                // Convert timestamp to date and set to noon (12:00:00)
+                                dayDate: {
+                                    $dateFromParts: {
+                                        year: { $year: { $toDate: { $multiply: ["$ts", 1000] } } },
+                                        month: { $month: { $toDate: { $multiply: ["$ts", 1000] } } },
+                                        day: { $dayOfMonth: { $toDate: { $multiply: ["$ts", 1000] } } },
+                                        hour: 12,
+                                        minute: 0,
+                                        second: 0
                                     }
-                                }, function(a/*, c, b*/) {
-                                    log.d('Done running ping job: %j', a);
-                                    done();
-                                });
+                                }
+                            }
+                        },
+                        // Convert back to timestamp in seconds
+                        {
+                            $addFields: {
+                                noonTimestamp: {
+                                    $divide: [{ $toLong: "$dayDate" }, 1000]
+                                }
                             }
-                            else {
-                                done();
+                        },
+                        // Group by day and action
+                        {
+                            $group: {
+                                _id: {
+                                    day: "$noonTimestamp",
+                                    action: "$a"
+                                },
+                                count: { $sum: 1 }
                             }
-                        });
+                        },
+                        // Project to final format
+                        {
+                            $project: {
+                                _id: 0,
+                                action: "$_id.action",
+                                timestamp: "$_id.day",
+                                count: 1
+                            }
+                        }
+                    ];
+
+                    var cursor = db.collection("systemlogs").aggregate(aggregationPipeline);
+
+                    while (cursor && await cursor.hasNext()) {
+                        let eventData = await cursor.next();
+                        user.add_event({key: eventData.action, count: eventData.count, timestamp: eventData.timestamp});
                     }
-                    else {
+                    server.start(function() {
+                        server.stop();
                         done();
-                    }
-                });
+                    });
+                }
+                else {
+                    done();
+                }
             }
             else {
                 done();
@@ -104,4 +128,4 @@ class PingJob extends job.Job {
     }
 }
 
-module.exports = PingJob;
+module.exports = PingJob;

api/parts/data/usage.js

@@ -1125,7 +1125,7 @@ plugins.register("/sdk/user_properties", async function(ob) {
                 userProps.av_major = null;
                 userProps.av_minor = null;
                 userProps.av_patch = null;
-                userProps.av_rel = null;
+                userProps.av_prerel = null;
                 userProps.av_build = null;
             }
         }

api/parts/jobs/job.js

@@ -254,32 +254,33 @@ class Job extends EventEmitter {
 
             this._json.next = next.getTime();
         }
-        if (this.name !== "alerts:monitor") {
-            //check if any job already scheduled or running
-            let query = {
-                status: {"$in": [STATUS.SCHEDULED, STATUS.RUNNING]},
-                name: this.name,
-            };
-            if (this._id) {
-                query._id = {$ne: this._id};
-            }
-            var self = this;
-            return new Promise((resolve, reject) => {
-                Job.findMany(this.db(), query).then(existing => {
-                    if (existing && existing.length) {
-                        log.d('Job already scheduled or running: %j', existing);
-                        this._json.status = STATUS.CANCELLED; //set this as cancelled now as we have other scheduled
-                    }
-                    else {
-                        self._save().then(resolve, reject);
-                    }
 
-                });
-            });
+        //check if any job already scheduled or running
+        let query = {
+            status: {"$in": [STATUS.SCHEDULED, STATUS.RUNNING]},
+            name: this.name,
+        };
+
+        if (this.name === 'alerts:monitor' && this.data && Object.keys(this.data).length) {
+            query.data = this.data;
         }
-        else {
-            return this._save();
+
+        if (this._id) {
+            query._id = {$ne: this._id};
         }
+
+        var self = this;
+        return new Promise((resolve, reject) => {
+            Job.findMany(this.db(), query).then(existing => {
+                if (existing && existing.length) {
+                    log.d('Job already scheduled or running: %j', existing);
+                    this._json.status = STATUS.CANCELLED; //set this as cancelled now as we have other scheduled
+                }
+                else {
+                    self._save().then(resolve, reject);
+                }
+            });
+        });
     }
 
     /**
@@ -1142,4 +1143,4 @@ module.exports = {
     STATUS: STATUS,
     STATUS_MAP: STATUS_MAP,
     debounce: debounce
-};
+};