+
+
+## License
+This project is licensed under **AGPL-3.0** with modified Section 7., see the [LICENSE](LICENSE) file for more details.
+
+## 💚 Thanks
+
+This project is tested with BrowserStack.
diff --git a/api/api.js b/api/api.js
index 843deb94f1b..efb4f061c65 100644
--- a/api/api.js
+++ b/api/api.js
@@ -16,6 +16,7 @@ const {WriteBatcher, ReadBatcher, InsertBatcher} = require('./parts/data/batcher
const pack = require('../package.json');
const versionInfo = require('../frontend/express/version.info.js');
const moment = require("moment");
+const tracker = require('./parts/mgmt/tracker.js');
var t = ["countly:", "api"];
common.processRequest = processRequest;
@@ -38,6 +39,9 @@ else {
process.title = t.join(' ');
plugins.connectToAllDatabases().then(function() {
+ plugins.loadConfigs(common.db, function() {
+ tracker.enable();
+ });
common.writeBatcher = new WriteBatcher(common.db);
common.readBatcher = new ReadBatcher(common.db);
common.insertBatcher = new InsertBatcher(common.db);
@@ -142,6 +146,41 @@ plugins.connectToAllDatabases().then(function() {
require('./utils/log.js').ipcHandler(msg);
});
+ /**
+ * Set tracking config
+ */
+ plugins.setConfigs("tracking", {
+ self_tracking_app: "",
+ self_tracking_url: "",
+ self_tracking_app_key: "",
+ self_tracking_id_policy: "_id",
+ self_tracking_sessions: true,
+ self_tracking_events: true,
+ self_tracking_views: true,
+ self_tracking_feedback: true,
+ self_tracking_user_details: true,
+ server_sessions: true,
+ server_events: true,
+ server_crashes: true,
+ server_views: true,
+ server_feedback: true,
+ server_user_details: true,
+ /*user_sessions: true,
+ user_events: true,
+ user_crashes: true,
+ user_views: true,
+ user_feedback: true,
+ user_details: true*/
+ });
+
+ /*plugins.setUserConfigs("tracking", {
+ user_sessions: false,
+ user_events: false,
+ user_crashes: false,
+ user_views: false,
+ user_feedback: false
+ });*/
+
/**
* Initialize Plugins
*/
@@ -309,7 +348,7 @@ plugins.connectToAllDatabases().then(function() {
// Allow configs to load & scanner to find all jobs classes
setTimeout(() => {
jobs.job('api:topEvents').replace().schedule('at 00:01 am ' + 'every 1 day');
- jobs.job('api:ping').replace().schedule('every 1 day');
+ jobs.job('api:ping').replace().schedule('at 00:01 am ' + 'every 1 day');
jobs.job('api:clear').replace().schedule('every 1 day');
jobs.job('api:clearTokens').replace().schedule('every 1 day');
jobs.job('api:clearAutoTasks').replace().schedule('every 1 day');
diff --git a/api/jobs/ping.js b/api/jobs/ping.js
index 984e82c566c..efc4e78ceea 100644
--- a/api/jobs/ping.js
+++ b/api/jobs/ping.js
@@ -1,12 +1,8 @@
'use strict';
const job = require('../parts/jobs/job.js'),
- log = require('../utils/log.js')('job:ping'),
- countlyConfig = require("../../frontend/express/config.js"),
- versionInfo = require('../../frontend/express/version.info'),
plugins = require('../../plugins/pluginManager.js'),
- request = require('countly-request')(plugins.getConfig("security"));
-
+ tracker = require('../parts/mgmt/tracker.js');
/** Class for the job of pinging servers **/
class PingJob extends job.Job {
@@ -16,86 +12,114 @@ class PingJob extends job.Job {
* @param {done} done callback
*/
run(db, done) {
- request({strictSSL: false, uri: (process.env.COUNTLY_CONFIG_PROTOCOL || "http") + "://" + (process.env.COUNTLY_CONFIG_HOSTNAME || "localhost") + (countlyConfig.path || "") + "/configs"}, function() {});
- var countlyConfigOrig = JSON.parse(JSON.stringify(countlyConfig));
- var url = "https://count.ly/configurations/ce/tracking";
- if (versionInfo.type !== "777a2bf527a18e0fffe22fb5b3e322e68d9c07a6") {
- url = "https://count.ly/configurations/ee/tracking";
- }
- plugins.loadConfigs(db, function() {
+ plugins.loadConfigs(db, async function() {
const offlineMode = plugins.getConfig("api").offline_mode;
- const { countly_tracking } = plugins.getConfig('frontend');
if (!offlineMode) {
- request(url, function(err, response, body) {
- if (typeof body === "string") {
- try {
- body = JSON.parse(body);
- }
- catch (ex) {
- body = null;
- }
+ var server = tracker.getBulkServer();
+ var user = tracker.getBulkUser(server);
+ if (!user) {
+ return done();
+ }
+
+ try {
+ var custom = await tracker.getAllData();
+ if (Object.keys(custom).length) {
+ user.user_details({"custom": custom });
}
- if (body) {
- if (countlyConfigOrig.web.use_intercom && typeof body.intercom !== "undefined") {
- countlyConfig.web.use_intercom = body.intercom;
- }
- if (typeof countlyConfigOrig.web.track === "undefined" && typeof body.stats !== "undefined") {
- if (body.stats) {
- countlyConfig.web.track = null;
- }
- else {
- countlyConfig.web.track = "none";
- }
- }
+ }
+ catch (ex) {
+ console.log("Error collecting server data:", ex);
+ }
+ var days = 90;
+ var current_sync = Date.now();
+
+ // Atomically retrieve old last_sync value and set new one
+ var syncResult = await db.collection("plugins").findOneAndUpdate(
+ {_id: "version"},
+ {$set: {last_sync: current_sync}},
+ {
+ upsert: true,
+ returnDocument: 'before',
+ projection: {last_sync: 1}
}
- log.d(err, body, countlyConfigOrig, countlyConfig);
- if (countly_tracking) {
- db.collection("members").findOne({global_admin: true}, function(err2, member) {
- if (!err2 && member) {
- var date = new Date();
- let domain = plugins.getConfig('api').domain;
+ );
- try {
- // try to extract hostname from full domain url
- const urlObj = new URL(domain);
- domain = urlObj.hostname;
- }
- catch (_) {
- // do nothing, domain from config will be used as is
- }
+ var last_sync = syncResult.value ? syncResult.value.last_sync : null;
+ if (last_sync) {
+ days = Math.floor((new Date().getTime() - last_sync) / (1000 * 60 * 60 * 24));
+ }
- request({
- uri: "https://stats.count.ly/i",
- method: "GET",
- timeout: 4E3,
- qs: {
- device_id: domain,
- app_key: "e70ec21cbe19e799472dfaee0adb9223516d238f",
- timestamp: Math.floor(date.getTime() / 1000),
- hour: date.getHours(),
- dow: date.getDay(),
- no_meta: true,
- events: JSON.stringify([
- {
- key: "PING",
- count: 1
- }
- ])
+ if (days > 0) {
+ //calculate seconds timestamp of days before today
+ var startTs = Math.round((new Date().getTime() - (30 * 24 * 60 * 60 * 1000)) / 1000);
+
+ //sync server events - use aggregation pipeline to group by day and action on MongoDB side
+ var aggregationPipeline = [
+ // Match documents with timestamp greater than startTs and valid action
+ {
+ $match: {
+ ts: { $gt: startTs }
+ }
+ },
+ // Add calculated fields for day grouping
+ {
+ $addFields: {
+ // Convert timestamp to date and set to noon (12:00:00)
+ dayDate: {
+ $dateFromParts: {
+ year: { $year: { $toDate: { $multiply: ["$ts", 1000] } } },
+ month: { $month: { $toDate: { $multiply: ["$ts", 1000] } } },
+ day: { $dayOfMonth: { $toDate: { $multiply: ["$ts", 1000] } } },
+ hour: 12,
+ minute: 0,
+ second: 0
}
- }, function(a/*, c, b*/) {
- log.d('Done running ping job: %j', a);
- done();
- });
+ }
+ }
+ },
+ // Convert back to timestamp in seconds
+ {
+ $addFields: {
+ noonTimestamp: {
+ $divide: [{ $toLong: "$dayDate" }, 1000]
+ }
}
- else {
- done();
+ },
+ // Group by day and action
+ {
+ $group: {
+ _id: {
+ day: "$noonTimestamp",
+ action: "$a"
+ },
+ count: { $sum: 1 }
}
- });
+ },
+ // Project to final format
+ {
+ $project: {
+ _id: 0,
+ action: "$_id.action",
+ timestamp: "$_id.day",
+ count: 1
+ }
+ }
+ ];
+
+ var cursor = db.collection("systemlogs").aggregate(aggregationPipeline);
+
+ while (cursor && await cursor.hasNext()) {
+ let eventData = await cursor.next();
+ user.add_event({key: eventData.action, count: eventData.count, timestamp: eventData.timestamp});
}
- else {
+ server.start(function() {
+ server.stop();
done();
- }
- });
+ });
+ }
+ else {
+ done();
+ }
}
else {
done();
@@ -104,4 +128,4 @@ class PingJob extends job.Job {
}
}
-module.exports = PingJob;
+module.exports = PingJob;
\ No newline at end of file
diff --git a/api/parts/data/usage.js b/api/parts/data/usage.js
index 0b03e0599de..93fc8b978eb 100644
--- a/api/parts/data/usage.js
+++ b/api/parts/data/usage.js
@@ -1030,7 +1030,7 @@ plugins.register("/sdk/user_properties", async function(ob) {
if (plugins.getConfig('api', params.app && params.app.plugins, true).city_data === true && !userProps.loc && typeof data.lat !== "undefined" && typeof data.lon !== "undefined") {
// only override lat/lon if no recent gps location exists in user document
- if (!params.app_user.loc || (params.app_user.loc.gps && params.time.mstimestamp - params.app_user.loc.date > 7 * 24 * 3600)) {
+ if (!params.app_user.loc || !params.app_user.loc.gps || params.time.mstimestamp - params.app_user.loc.date > 7 * 24 * 3600) {
userProps.loc = {
gps: false,
geo: {
@@ -1061,7 +1061,7 @@ plugins.register("/sdk/user_properties", async function(ob) {
if (plugins.getConfig('api', params.app && params.app.plugins, true).city_data === true && !userProps.loc && data.ll && typeof data.ll[0] !== "undefined" && typeof data.ll[1] !== "undefined") {
// only override lat/lon if no recent gps location exists in user document
- if (!params.app_user.loc || (params.app_user.loc.gps && params.time.mstimestamp - params.app_user.loc.date > 7 * 24 * 3600)) {
+ if (!params.app_user.loc || !params.app_user.loc.gps || params.time.mstimestamp - params.app_user.loc.date > 7 * 24 * 3600) {
userProps.loc = {
gps: false,
geo: {
@@ -1125,7 +1125,7 @@ plugins.register("/sdk/user_properties", async function(ob) {
userProps.av_major = null;
userProps.av_minor = null;
userProps.av_patch = null;
- userProps.av_rel = null;
+ userProps.av_prerel = null;
userProps.av_build = null;
}
}
diff --git a/api/parts/jobs/job.js b/api/parts/jobs/job.js
index 1bf39c2e6a1..46577397cc8 100644
--- a/api/parts/jobs/job.js
+++ b/api/parts/jobs/job.js
@@ -254,32 +254,33 @@ class Job extends EventEmitter {
this._json.next = next.getTime();
}
- if (this.name !== "alerts:monitor") {
- //check if any job already scheduled or running
- let query = {
- status: {"$in": [STATUS.SCHEDULED, STATUS.RUNNING]},
- name: this.name,
- };
- if (this._id) {
- query._id = {$ne: this._id};
- }
- var self = this;
- return new Promise((resolve, reject) => {
- Job.findMany(this.db(), query).then(existing => {
- if (existing && existing.length) {
- log.d('Job already scheduled or running: %j', existing);
- this._json.status = STATUS.CANCELLED; //set this as cancelled now as we have other scheduled
- }
- else {
- self._save().then(resolve, reject);
- }
- });
- });
+ //check if any job already scheduled or running
+ let query = {
+ status: {"$in": [STATUS.SCHEDULED, STATUS.RUNNING]},
+ name: this.name,
+ };
+
+ if (this.name === 'alerts:monitor' && this.data && Object.keys(this.data).length) {
+ query.data = this.data;
}
- else {
- return this._save();
+
+ if (this._id) {
+ query._id = {$ne: this._id};
}
+
+ var self = this;
+ return new Promise((resolve, reject) => {
+ Job.findMany(this.db(), query).then(existing => {
+ if (existing && existing.length) {
+ log.d('Job already scheduled or running: %j', existing);
+ this._json.status = STATUS.CANCELLED; //set this as cancelled now as we have other scheduled
+ }
+ else {
+ self._save().then(resolve, reject);
+ }
+ });
+ });
}
/**
@@ -1142,4 +1143,4 @@ module.exports = {
STATUS: STATUS,
STATUS_MAP: STATUS_MAP,
debounce: debounce
-};
\ No newline at end of file
+};
diff --git a/api/parts/mgmt/mail.js b/api/parts/mgmt/mail.js
index c5330bcfc0b..19e01d517fd 100644
--- a/api/parts/mgmt/mail.js
+++ b/api/parts/mgmt/mail.js
@@ -11,8 +11,38 @@ var mail = {},
versionInfo = require('../../../frontend/express/version.info'),
authorize = require('../../utils/authorizer'),
config = require('../../config'),
+ log = require('../../utils/log.js')('mail'),
+ util = require('node:util'),
ip = require('./ip.js');
+const smtpLogger = {};
+
+// Set up logger wrapper
+for (let level of ['trace', 'debug', 'info', 'warn', 'error', 'fatal']) {
+ smtpLogger[level] = (data, message, ...args) => {
+ if (args && args.length) {
+ message = util.format(message, ...args);
+ }
+ if (level === 'error' || level === 'fatal') {
+ log.e(message, data || '');
+ }
+ else if (level === 'warn') {
+ log.w(message, data || '');
+ }
+ else if (level === 'info') {
+ log.i(message, data || '');
+ }
+ else {
+ log.d(message, data || '');
+ }
+ };
+}
+
+if (config.mail && config.mail.config) {
+ config.mail.config.logger = smtpLogger;
+ config.mail.config.debug = true;
+}
+
if (config.mail && config.mail.transport && config.mail.transport !== "nodemailer-smtp-transport") {
mail.smtpTransport = nodemailer.createTransport(require(config.mail.transport)(config.mail.config));
}
@@ -23,7 +53,8 @@ else {
mail.smtpTransport = nodemailer.createTransport({
sendmail: true,
newline: 'unix',
- path: '/usr/sbin/sendmail'
+ path: '/usr/sbin/sendmail',
+ logger: smtpLogger
});
}
diff --git a/api/parts/mgmt/tracker.js b/api/parts/mgmt/tracker.js
index d878ab7188f..d75dce31e1a 100644
--- a/api/parts/mgmt/tracker.js
+++ b/api/parts/mgmt/tracker.js
@@ -8,176 +8,158 @@ var tracker = {},
stats = require('../data/stats.js'),
common = require('../../utils/common.js'),
logger = require('../../utils/log.js'),
+ countlyFs = require('../../utils/countlyFs.js'),
//log = logger("tracker:server"),
Countly = require('countly-sdk-nodejs'),
- countlyConfig = require("../../../frontend/express/config.js"),
- versionInfo = require('../../../frontend/express/version.info'),
- ip = require('./ip.js'),
- cluster = require('cluster'),
- os = require('os'),
fs = require('fs'),
- asyncjs = require('async'),
- trial = "79199134e635edb05fc137e8cd202bb8640fb0eb",
- app = "e70ec21cbe19e799472dfaee0adb9223516d238f",
- server = "e0693b48a5513cb60c112c21aede3cab809d52d0",
+ path = require('path'),
+ https = require('https'),
+ http = require('http'),
+ FormData = require('form-data'),
+ { Readable } = require('node:stream'),
+ versionInfo = require('../../../frontend/express/version.info'),
+ server = "9c28c347849f2c03caf1b091ec7be8def435e85e",
+ user = "fa6e9ae7b410cb6d756e8088c5f3936bf1fab5f3",
url = "https://stats.count.ly",
- plugins = require('../../../plugins/pluginManager.js'),
- request = require('countly-request')(plugins.getConfig("security")),
- offlineMode = plugins.getConfig("api").offline_mode,
- domain = plugins.getConfig("api").domain;
-
+ plugins = require('../../../plugins/pluginManager.js');
-//update configs
-var cache = {};
-if (countlyConfig.web && countlyConfig.web.track === "all") {
- countlyConfig.web.track = null;
-}
-var countlyConfigOrig = JSON.parse(JSON.stringify(countlyConfig));
-var url_check = "https://count.ly/configurations/ce/tracking";
-if (versionInfo.type !== "777a2bf527a18e0fffe22fb5b3e322e68d9c07a6") {
- url_check = "https://count.ly/configurations/ee/tracking";
-}
+var IS_FLEX = false;
-if (!offlineMode) {
- request(url_check, function(err, response, body) {
- if (typeof body === "string") {
- try {
- body = JSON.parse(body);
- }
- catch (ex) {
- body = null;
- }
+if (fs.existsSync(path.resolve('/opt/deployment_env.json'))) {
+ var deploymentConf = fs.readFileSync('/opt/deployment_env.json', 'utf8');
+ try {
+ if (JSON.parse(deploymentConf).DEPLOYMENT_ID) {
+ IS_FLEX = true;
}
- if (body) {
- if (countlyConfigOrig.web.use_intercom && typeof body.intercom !== "undefined") {
- countlyConfig.web.use_intercom = body.intercom;
- }
- if (typeof countlyConfigOrig.web.track === "undefined" && typeof body.stats !== "undefined") {
- if (body.stats) {
- countlyConfig.web.track = null;
- }
- else {
- countlyConfig.web.track = "none";
- }
- }
- if (typeof countlyConfigOrig.web.server_track === "undefined" && typeof body.server !== "undefined") {
- if (body.server) {
- countlyConfig.web.server_track = null;
- }
- else {
- countlyConfig.web.server_track = "none";
- }
- }
- }
- });
+ }
+ catch (e) {
+ IS_FLEX = false;
+ }
}
+//update configs
var isEnabled = false;
/**
* Enable tracking for this server
**/
tracker.enable = function() {
+ if (isEnabled) {
+ return;
+ }
+
var config = {
- app_key: (versionInfo.trial) ? trial : server,
+ app_key: server,
url: url,
app_version: versionInfo.version,
storage_path: "../../../.sdk/",
interval: 10000,
fail_timeout: 600,
- session_update: 120,
+ session_update: 60 * 60 * 12,
remote_config: true,
debug: (logger.getLevel("tracker:server") === "debug")
};
+
+ var domain = plugins.getConfig("api").domain;
+
//set static device id if domain is defined
if (domain) {
config.device_id = stripTrailingSlash((domain + "").split("://").pop());
}
- Countly.init(config);
- //change device id if is it not domain
- if (domain && Countly.get_device_id() !== domain) {
- Countly.change_id(stripTrailingSlash((domain + "").split("://").pop()), true);
- }
- else if (!domain) {
- checkDomain();
- }
+ if (config.device_id && config.device_id !== "localhost") {
+ Countly.init(config);
- isEnabled = true;
- if (countlyConfig.web.track !== "none" && countlyConfig.web.server_track !== "none") {
- Countly.track_errors();
- }
- if (cluster.isMaster) {
- setTimeout(function() {
- if (countlyConfig.web.track !== "none" && countlyConfig.web.server_track !== "none") {
- Countly.begin_session(true);
- setTimeout(function() {
- collectServerStats();
- collectServerData();
- }, 20000);
- }
- }, 1000);
- //report app start trace
- if (Countly.report_app_start) {
- Countly.report_app_start();
+ //change device id if is it not domain
+ if (Countly.get_device_id() !== domain) {
+ Countly.change_id(stripTrailingSlash((domain + "").split("://").pop()), true);
+ }
+
+ isEnabled = true;
+ Countly.user_details({"name": config.device_id });
+ if (plugins.getConfig("white-labeling") && (plugins.getConfig("white-labeling").favicon || plugins.getConfig("white-labeling").stopleftlogo || plugins.getConfig("white-labeling").prelogo)) {
+ var id = plugins.getConfig("white-labeling").favicon || plugins.getConfig("white-labeling").stopleftlogo || plugins.getConfig("white-labeling").prelogo;
+ countlyFs.gridfs.getDataById("white-labeling", id, function(errWhitelabel, data) {
+ if (!errWhitelabel && data) {
+ tracker.uploadBase64FileFromGridFS(data).catch(() => {});
+ }
+ });
}
+ else {
+ Countly.user_details({"picture": "./images/favicon.png" });
+ }
+ if (plugins.getConfig("tracking").server_sessions) {
+ Countly.begin_session(true);
+ }
+ if (plugins.getConfig("tracking").server_crashes) {
+ Countly.track_errors();
+ }
+ setTimeout(function() {
+ tracker.getAllData().then((custom) => {
+ if (Object.keys(custom).length) {
+ Countly.user_details({"custom": custom });
+ }
+ });
+ }, 20000);
}
};
/**
-* Enable tracking for dashboard process
-**/
-tracker.enableDashboard = function() {
- var config = {
- app_key: (versionInfo.trial) ? trial : server,
- url: url,
- app_version: versionInfo.version,
- storage_path: "../../../.sdk/",
- interval: 60000,
- fail_timeout: 600,
- session_update: 120,
- debug: (logger.getLevel("tracker:server") === "debug")
- };
- //set static device id if domain is defined
- if (domain) {
- config.device_id = stripTrailingSlash((domain + "").split("://").pop());
- }
- Countly.init(config);
+ * Get bulk server instance
+ * @returns {Object} Countly Bulk instance
+ */
+tracker.getBulkServer = function() {
+ return new Countly.Bulk({
+ app_key: server,
+ url: url
+ });
+};
- //change device id if is it not domain
- if (domain && Countly.get_device_id() !== domain) {
- Countly.change_id(stripTrailingSlash((domain + "").split("://").pop()), true);
- }
- else if (!domain) {
- checkDomain();
- }
- isEnabled = true;
- if (countlyConfig.web.track !== "none" && countlyConfig.web.server_track !== "none") {
- Countly.track_errors();
+/**
+ * Get bulk user instance
+ * @param {Object} serverInstance - Countly Bulk server instance
+ * @returns {Object} Countly Bulk User instance
+ */
+tracker.getBulkUser = function(serverInstance) {
+ var domain = stripTrailingSlash((plugins.getConfig("api").domain + "").split("://").pop());
+ if (domain && domain !== "localhost") {
+ return serverInstance.add_user({device_id: domain});
}
};
-
/**
* Report server level event
* @param {object} event - event object
**/
tracker.reportEvent = function(event) {
- if (isEnabled && countlyConfig.web.track !== "none" && countlyConfig.web.server_track !== "none") {
+ if (isEnabled && plugins.getConfig("tracking").server_events) {
Countly.add_event(event);
}
};
+/**
+* Report server level event in bulk
+* @param {Array} events - array of event objects
+**/
+tracker.reportEventBulk = function(events) {
+ if (isEnabled && plugins.getConfig("tracking").server_events) {
+ Countly.request({
+ app_key: server,
+ device_id: Countly.get_device_id(),
+ events: JSON.stringify(events)
+ });
+ }
+};
+
/**
* Report user level event
* @param {string} id - id of the device
* @param {object} event - event object
-* @param {string} level - tracking level
**/
-tracker.reportUserEvent = function(id, event, level) {
- if (isEnabled && countlyConfig.web.track !== "none" && (!level || countlyConfig.web.track === level) && countlyConfig.web.server_track !== "none") {
+tracker.reportUserEvent = function(id, event) {
+ if (isEnabled && plugins.getConfig("tracking").user_events) {
Countly.request({
- app_key: app,
+ app_key: user,
device_id: id,
events: JSON.stringify([event])
});
@@ -186,11 +168,10 @@ tracker.reportUserEvent = function(id, event, level) {
/**
* Check if tracking enabled
-* @param {boolean|string} level - level of tracking
* @returns {boolean} if enabled
**/
-tracker.isEnabled = function(level) {
- return (isEnabled && countlyConfig.web.track !== "none" && (!level || countlyConfig.web.track === level) && countlyConfig.web.server_track !== "none");
+tracker.isEnabled = function() {
+ return isEnabled;
};
/**
@@ -203,164 +184,403 @@ tracker.getSDK = function() {
/**
* Get server stats
+* @returns {Promise