Merge pull request #6150 from Countly/release_24_10_to_master

Update master from Release.24.10

Author: Cookiezaurs

CHANGELOG.md

@@ -1,14 +1,27 @@
-## Version 25.03.x
-
+## Version 25.03.4
 Features:
 - Add ability to allow multiple CORS per app for web apps
 - Add app id and name as view segment for self-tracking
+- [dashboards] Added the option to set a refresh rate for dashboards, allowing data to update more frequently for selected dashboards
 
 Enterprise Features:
 - [license] Update locking conditions for expired license and over limit usage
 - [license] Enable force locking with remote config
 - [license] Update dashboard lock with redirection to home page for non admin user
 
+Fixes:
+- [core] Allow downloading data also from other databases in dbviewer
+- [crashes] Fix unescaped SDK logs
+- [crash_symbolication] Symbolication server api end point test fix
+- [star-rating] Added missing columns to Rating Widgets table edit
+- [ui] Fix alignment of drawers title and close icon
+- [push] Fixed push notifications title and content text and variables combination
+- [reports] Correctly match event for email report if event key contains '.'
+
+Enterprise Fixes:
+- [cohorts] Fixed issue with combining multiple cohorts
+- [drill] Do not recheck old collections on app_user data deletion if querying from old collections is disabled
+
 Dependencies:
 - Bump body-parser from 1.20.3 to 2.2.0
 - Bump moment-timezone from 0.5.47 to 0.5.48

api/jobs/topEvents.js

@@ -138,14 +138,19 @@ class TopEventsJob extends job.Job {
      * The errors of all functions will be caught here.
      */
     async getAllApps() {
+        log.d("Fetching all apps");
         try {
             const getAllApps = await new Promise((res, rej) => common.db.collection("apps").find({}, { _id: 1, timezone: 1 }).toArray((err, apps) => err ? rej(err) : res(apps)));
-            await Promise.all(getAllApps.map((app) => this.getAppEvents(app)));
+            for (var z = 0; z < getAllApps.length; z++) {
+                //Calculating for each app serially.
+                await this.getAppEvents(getAllApps[z]);
+            }
         }
         catch (error) {
             log.e("TopEvents Job has a error: ", error);
             throw error;
         }
+        log.d("Finished processing");
     }
 
     /**
@@ -178,6 +183,7 @@ class TopEventsJob extends job.Job {
      * @param {Object} app - saveAppEvents object
      */
     async getAppEvents(app) {
+        log.d(app._id + ": Fetching app events");
         const getEvents = await new Promise((res, rej) => common.db.collection("events").findOne({ _id: app._id }, (errorEvents, result) => errorEvents ? rej(errorEvents) : res(result)));
         if (getEvents && 'list' in getEvents) {
             const eventMap = this.eventsFilter(getEvents.list);
@@ -199,6 +205,7 @@ class TopEventsJob extends job.Job {
                     let totalDuration = 0;
                     let prevTotalDuration = 0;
                     for (const event of eventMap) {
+                        log.d("    getting event data for event: " + event + " (" + period + ")");
                         const collectionNameEvents = this.eventsCollentions({ event, id: app._id });
                         await this.getEventsCount({ collectionNameEvents, ob, data, event });
                         totalCount += data[event].data.count.total;
@@ -208,11 +215,16 @@ class TopEventsJob extends job.Job {
                         totalDuration += data[event].data.duration.total;
                         prevTotalDuration += data[event].data.duration["prev-total"];
                     }
+                    log.d("    getting session count (" + period + ")");
                     await this.getSessionCount({ ob, sessionData, usersData, usersCollectionName });
+                    log.d("    saving data (" + period + ")");
                     await this.saveAppEvents({ app, data, sessionData, usersData, period, totalCount, prevTotalCount, totalSum, prevTotalSum, totalDuration, prevTotalDuration });
                 }
             }
         }
+        else {
+            log.d("    No events found for app");
+        }
 
     }
 

api/utils/taskmanager.js

@@ -984,7 +984,13 @@ taskmanager.rerunTask = function(options, callback) {
         });
     }
 
-    options.db.collection("long_tasks").findOne({_id: options.id}, function(err, res) {
+    var qq = {_id: options.id};
+    if (options.additionalQuery) {
+        qq = options.additionalQuery;
+        qq._id = options.id;
+    }
+    log.d("Fetching from long_tasks to rerun: " + JSON.stringify(qq));
+    options.db.collection("long_tasks").findOne(qq, function(err, res) {
         if (!err && res && res.request) {
             var reqData = {};
             try {

frontend/express/public/javascripts/countly/vue/components/drawer.js

@@ -62,6 +62,7 @@
                         'is-open': this.isOpened,
                         'has-sidecars': this.hasSidecars
                     };
+                    // NOTE: currentScreenMode variable seems to be not defined it should be defined or removed
                     classes["cly-vue-drawer--" + this.currentScreenMode + "-screen"] = true;
                     if (this.currentScreenMode === 'half') {
                         classes["cly-vue-drawer--half-screen-" + this.size] = true;

frontend/express/public/javascripts/countly/vue/templates/drawer.html

@@ -20,25 +20,35 @@
                 </div>
                 <div class="cly-vue-drawer__header" :class="{ 'is-full-screen': currentScreenMode === 'full'}">
                     <div class="cly-vue-drawer__title">
-                        <div v-if="currentScreenMode !== 'full'">
-                            <div v-if="hasBackLink" class="cly-vue-drawer__title-container bu-is-flex bu-is-justify-content-space-between">
-                                <div class="bu-is-flex bu-is-flex-direction-column bu-is-justify-content-center bu-p-1 bu-ml-5 bu-mt-5">
-                                    <a style="cursor: pointer;" @click="doClose" :data-test-id="testId + '-drawer-back-link'">
-                                        <span class="text-medium bu-is-capitalized"><i class="fas fa-arrow-left bu-pr-3"></i>{{i18n('plugins.back')}}</span>
-                                    </a>
-                                </div>
-                                <div class="bu-is-flex" :style="hasBackLink.style" :class="{ 'bu-is-justify-content-flex-start': hasBackLink, 'bu-is-justify-content-center': !hasBackLink }">
-                                    <h3 :data-test-id="testId + '-header-title'" class="cly-vue-drawer__title-header">{{title}}</h3>
-                                </div>
-                                <span :data-test-id="testId + '-close-button'" class="cly-vue-drawer__close-button bu-p-1 bu-mr-5" v-on:click="doClose">
-                                    <i class="ion-ios-close-empty"></i>
-                                </span>
-                            </div>
-                            <div v-else class="cly-vue-drawer__title-container bu-is-flex bu-is-justify-content-space-between">
-                                <h3 :data-test-id="testId + '-header-title'" class="cly-vue-drawer__title-header">{{title}}</h3>
-                                <span :data-test-id="testId + '-close-button'" class="cly-vue-drawer__close-button" v-on:click="doClose">
-                                    <i class="ion-ios-close-empty"></i>
+                        <div
+                            v-if="currentScreenMode !== 'full'"
+                            class="cly-vue-drawer__title-container bu-is-flex bu-is-justify-content-space-between"
+                            :class="{ 'bu-is-align-items-center': !hasBackLink }"
+                        >
+                            <a
+                                v-if="hasBackLink"
+                                :data-test-id="testId + '-drawer-back-link'"
+                                style="cursor: pointer;"
+                                @click="doClose"
+                            >
+                                <span class="text-medium bu-is-capitalized">
+                                    <i class="fas fa-arrow-left bu-pr-3" />
+                                    {{ i18n('plugins.back') }}
                                 </span>
+                            </a>
+                            <h3
+                                class="cly-vue-drawer__title-header"
+                                :data-test-id="testId + '-header-title'"
+                                :style="hasBackLink.style"
+                            >
+                                {{ title }}
+                            </h3>
+                            <div
+                                class="cly-vue-drawer__close-button"
+                                :data-test-id="testId + '-close-button'"
+                                @click="doClose"
+                            >
+                                <i class="ion-ios-close-empty" />
                             </div>
                         </div>
                         <div v-if="isMultiStep" class="bu-columns bu-is-gapless bu-is-mobile cly-vue-drawer__subtitle">
@@ -51,7 +61,7 @@ <h3 :data-test-id="testId + '-header-title'" class="cly-vue-drawer__title-header
                                                     <span class="index text-small" :data-test-id="testId + '-current-step-index-' + (i + 1)" :class="{'color-white': i === currentStepIndex}">{{i + 1}}</span>
                                                     <span class="done-icon text-small color-white bu-pt-0" :data-test-id="testId + '-current-step-index-img-container' + (i + 1)">
                                                         <img :data-test-id="testId + '-step-' + (i + 1)" src="images/icons/check-icon.svg">
-                                                    </span>                                                     
+                                                    </span>
                                                 </div>
                                                 <div :data-test-id="testId + '-' + (currentContent && currentContent.name ? currentContent.name.toLowerCase().replaceAll(/[\s&]+/g, '-') + '-label' : 'content-label')" class="cly-vue-drawer__step-title text-small font-weight-bold color-cool-gray-40">{{currentContent.name}}</div>
                                             </div>

frontend/express/public/stylesheets/vue/clyvue.scss

@@ -1899,31 +1899,29 @@
             box-sizing: border-box;
         }
 
+        // .cly-vue-drawer__title-container
         &__title-container {
-            width: 100%; 
-            margin-bottom: 24px;
-        }
-
-        &__title-header {
-            margin-top: 24px; 
-            margin-left: 32px;
+            margin: 24px 32px;
         }
 
         &__subtitle {
             margin-left: 32px !important;
             margin-bottom: 24px !important;
         }
 
+        // .cly-vue-drawer__close-button
         &__close-button {
+            display: flex;
+            align-items: center;
+            justify-content: center;
+
             font-size: 30px;
             color: #81868D;
             transition: color 1s;
             cursor: pointer;
-            //width: 30px;
-            height: 15px;
-            //margin-block: auto;
-            margin-right: 32px;
-            margin-top: 24px;
+            height: 20px;
+
+            // .cly-vue-drawer__close-button:hover
             &:hover {
                 color: #333;
                 transition: color 1s;

plugins/crashes/frontend/public/javascripts/countly.models.js

@@ -808,6 +808,9 @@ function transformAppVersion(inpVersion) {
                             }
 
                             crashgroupJson.data.forEach(function(crash, crashIndex) {
+                                // unescape logs
+                                crashgroupJson.data[crashIndex].logs = countlyCommon.unescapeHtml(crash.logs);
+
                                 if (crash.uid in userIds) {
                                     userIds[crash.uid].push(crashIndex);
                                 }
@@ -1327,4 +1330,4 @@ function transformAppVersion(inpVersion) {
         return resultQuery;
     };
 
-}(window.countlyCrashes = window.countlyCrashes || {}));
+}(window.countlyCrashes = window.countlyCrashes || {}));

plugins/dashboards/api/api.js

@@ -22,6 +22,12 @@ plugins.setConfigs("dashboards", {
 
 (function() {
 
+    plugins.register("/master", function() {
+        setTimeout(() => {
+            require('../../../api/parts/jobs').job('dashboards:refreshDashboards').replace().schedule('every 5 minutes');
+        }, 1000);
+    });
+
     /**
      * @api {get} /o/dashboards Get dashboard
      * @apiName GetDashboard
@@ -162,6 +168,11 @@ plugins.setConfigs("dashboards", {
                                 }
                             }
 
+                            if (dashboard.refreshRate) {
+                                dashboard.refreshRate = dashboard.refreshRate / 60; //Convert to minutes
+                                dashboard.use_refresh_rate = true;
+                            }
+
                             if (canSeeDashboardShares(params.member, dashboard)) {
                                 parallelTasks.push(fetchSharedUsersInfo.bind(null, dashboard));
                             }
@@ -597,6 +608,19 @@ plugins.setConfigs("dashboards", {
                 shareWith = params.qstring.share_with || "",
                 copyDashId = params.qstring.copy_dash_id;
 
+            var refreshRate = 0;
+            if (params.qstring.use_refresh_rate && params.qstring.use_refresh_rate !== "false" && params.qstring.refreshRate > 0) {
+                try {
+                    refreshRate = parseInt(params.qstring.refreshRate, 10);
+                    refreshRate = Math.max(5, refreshRate) * 60; //Convert to seconds
+                }
+                catch (ex) {
+                    refreshRate = 0;
+                    log.e("passed unexpected refresh rate");
+                }
+
+            }
+
             try {
                 sharedEmailEdit = JSON.parse(sharedEmailEdit);
             }
@@ -711,6 +735,9 @@ plugins.setConfigs("dashboards", {
                     theme: theme,
                     created_at: new Date().getTime()
                 };
+                if (refreshRate > 0) {
+                    dashData.refreshRate = refreshRate;
+                }
 
                 var widgets = dataObj.newWidgetIds;
                 if (widgets && widgets.length) {
@@ -846,6 +873,19 @@ plugins.setConfigs("dashboards", {
                 send_email_invitation = params.qstring.send_email_invitation,
                 memberId = params.member._id + "";
 
+            var refreshRate = 0;
+            if (params.qstring.use_refresh_rate && params.qstring.use_refresh_rate !== "false" && params.qstring.refreshRate > 0) {
+                try {
+                    refreshRate = parseInt(params.qstring.refreshRate, 10);
+                    refreshRate = Math.max(5, refreshRate) * 60; //Convert to seconds
+                }
+                catch (ex) {
+                    refreshRate = 0;
+                    log.e("passed unexpected refresh rate");
+                }
+            }
+
+
             if (!dashboardId || dashboardId.length !== 24) {
                 common.returnMessage(params, 400, 'Invalid parameter: dashboard_id');
                 return true;
@@ -972,12 +1012,19 @@ plugins.setConfigs("dashboards", {
                                 changedFields.shared_user_groups_view = sharedUserGroupView;
                             }
                         }
+                        var unset = {shared_with_view: "", shared_with_edit: ""};
+                        if (refreshRate && refreshRate > 0 && refreshRate !== dashboard.refreshRate) {
+                            changedFields.refreshRate = refreshRate;
+                        }
+                        else if (refreshRate === 0) {
+                            unset.refreshRate = ""; //unset refresh rate for dashboard
+                        }
 
                         common.db.collection("dashboards").update(
                             filterCond,
                             {
                                 $set: changedFields,
-                                $unset: {shared_with_view: "", shared_with_edit: ""}
+                                $unset: unset
                             },
                             async function(e, res) {
                                 if (!e && res) {

plugins/dashboards/api/jobs/refreshDashboards.js

@@ -0,0 +1,67 @@
+const job = require('../../../../api/parts/jobs/job.js');
+const log = require('../../../../api/utils/log.js')('job:dashboards:refreshDashboards');
+const pluginManager = require('../../../pluginManager.js');
+var customDashboards = require('./../parts/dashboards.js');
+
+
+/** class RefreshDashboardsJob */
+class RefreshDashboardsJob extends job.Job {
+    /** function run
+     * @param {object} countlyDb - db connection object
+     * @param {function} doneJob - function to call when finishing Job
+     * @param {function} progressJob - fnction to call while running job
+    */
+    run(countlyDb, doneJob, progressJob) {
+        var total = 0;
+        var current = 0;
+        var bookmark = '';
+        log.d('Starting dashboards refresh job');
+
+        /**
+         * check job status periodically
+         */
+        function ping() {
+            log.d('Pinging dashboards refresh job');
+            if (pingTimeout) {
+                progressJob(total, current, bookmark);
+                pingTimeout = setTimeout(ping, 10000);
+            }
+        }
+        var pingTimeout = setTimeout(ping, 10000);
+
+        /**
+         * end job
+         * @returns {varies} job done
+         */
+        function endJob() {
+            log.d('Ending dashboards refresh job');
+            clearTimeout(pingTimeout);
+            pingTimeout = 0;
+            return doneJob();
+        }
+
+        pluginManager.loadConfigs(countlyDb, async() => {
+            //Fetch all sashboards.
+            //Check for the ones that have set refresh rate.
+            //Trigger regeneration for those dashboards.
+            try {
+                var dashboards = await countlyDb.collection('dashboards').find({}).toArray();
+                for (var z = 0; z < dashboards.length; z++) {
+                    if (dashboards[z].refreshRate && dashboards[z].refreshRate > 0) {
+                        if (dashboards[z].refreshRate < 300) {
+                            dashboards[z].refreshRate = 300;
+                        }
+                        log.d('Refreshing dashboard: ' + dashboards[z]._id);
+                        await customDashboards.refreshDashboard(countlyDb, dashboards[z]);
+                    }
+                }
+            }
+            catch (error) {
+                log.e('Error while refreshing dashboards: ' + error);
+            }
+            return endJob();
+        });
+    }
+}
+
+module.exports = RefreshDashboardsJob;