Skip to content

New tracking #6812

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ar2rsawseen merged 4 commits into from
Oct 16, 2025
Merged

New tracking #6812

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7ae7a23
New tracking
ar2rsawseen Oct 16, 2025
36f832a
Fix tests for server stats
ar2rsawseen Oct 16, 2025
8a43684
Fixe frontend
ar2rsawseen Oct 16, 2025
5600a2a
Fixes
ar2rsawseen Oct 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 26 additions & 1 deletion api/api.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ const {WriteBatcher, ReadBatcher, InsertBatcher} = require('./parts/data/batcher
const pack = require('../package.json');
const versionInfo = require('../frontend/express/version.info.js');
const moment = require("moment");
const tracker = require('./parts/mgmt/tracker.js');

var t = ["countly:", "api"];
common.processRequest = processRequest;
Expand All @@ -36,6 +37,9 @@ else {
process.title = t.join(' ');

plugins.connectToAllDatabases().then(function() {
plugins.loadConfigs(common.db, function() {
tracker.enable();
});
common.writeBatcher = new WriteBatcher(common.db);
common.readBatcher = new ReadBatcher(common.db);
common.insertBatcher = new InsertBatcher(common.db);
Expand Down Expand Up @@ -120,6 +124,27 @@ plugins.connectToAllDatabases().then(function() {
proxy_type: "https"
});

/**
* Set tracking config
*/
plugins.setConfigs("tracking", {
self_tracking_app: "",
self_tracking_url: "",
self_tracking_app_key: "",
self_tracking_id_policy: "_id",
self_tracking_sessions: true,
self_tracking_events: true,
self_tracking_views: true,
self_tracking_feedback: true,
self_tracking_user_details: true,
server_sessions: true,
server_events: true,
server_crashes: true,
server_views: true,
server_feedback: true,
server_user_details: true
});

/**
* Set Plugins Logs Config
*/
Expand Down Expand Up @@ -307,7 +332,7 @@ plugins.connectToAllDatabases().then(function() {
// Allow configs to load & scanner to find all jobs classes
setTimeout(() => {
jobs.job('api:topEvents').replace().schedule('at 00:01 am ' + 'every 1 day');
jobs.job('api:ping').replace().schedule('every 1 day');
jobs.job('api:ping').replace().schedule('at 00:01 am ' + 'every 1 day');
jobs.job('api:clear').replace().schedule('every 1 day');
jobs.job('api:clearTokens').replace().schedule('every 1 day');
jobs.job('api:clearAutoTasks').replace().schedule('every 1 day');
Expand Down
174 changes: 99 additions & 75 deletions api/jobs/ping.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,8 @@
'use strict';

const job = require('../parts/jobs/job.js'),
log = require('../utils/log.js')('job:ping'),
countlyConfig = require("../../frontend/express/config.js"),
versionInfo = require('../../frontend/express/version.info'),
plugins = require('../../plugins/pluginManager.js'),
request = require('countly-request')(plugins.getConfig("security"));

tracker = require('../parts/mgmt/tracker.js');

/** Class for the job of pinging servers **/
class PingJob extends job.Job {
Expand All @@ -16,86 +12,114 @@ class PingJob extends job.Job {
* @param {done} done callback
*/
run(db, done) {
request({strictSSL: false, uri: (process.env.COUNTLY_CONFIG_PROTOCOL || "http") + "://" + (process.env.COUNTLY_CONFIG_HOSTNAME || "localhost") + (countlyConfig.path || "") + "/configs"}, function() {});
var countlyConfigOrig = JSON.parse(JSON.stringify(countlyConfig));
var url = "https://count.ly/configurations/ce/tracking";
if (versionInfo.type !== "777a2bf527a18e0fffe22fb5b3e322e68d9c07a6") {
url = "https://count.ly/configurations/ee/tracking";
}
plugins.loadConfigs(db, function() {
plugins.loadConfigs(db, async function() {
const offlineMode = plugins.getConfig("api").offline_mode;
const { countly_tracking } = plugins.getConfig('frontend');
if (!offlineMode) {
request(url, function(err, response, body) {
if (typeof body === "string") {
try {
body = JSON.parse(body);
}
catch (ex) {
body = null;
}
var server = tracker.getBulkServer();
var user = tracker.getBulkUser(server);
if (!user) {
return done();
}

try {
var custom = await tracker.getAllData();
if (Object.keys(custom).length) {
user.user_details({"custom": custom });
}
if (body) {
if (countlyConfigOrig.web.use_intercom && typeof body.intercom !== "undefined") {
countlyConfig.web.use_intercom = body.intercom;
}
if (typeof countlyConfigOrig.web.track === "undefined" && typeof body.stats !== "undefined") {
if (body.stats) {
countlyConfig.web.track = null;
}
else {
countlyConfig.web.track = "none";
}
}
}
catch (ex) {
console.log("Error collecting server data:", ex);
}
var days = 90;
var current_sync = Date.now();

// Atomically retrieve old last_sync value and set new one
var syncResult = await db.collection("plugins").findOneAndUpdate(
{_id: "version"},
{$set: {last_sync: current_sync}},
{
upsert: true,
returnDocument: 'before',
projection: {last_sync: 1}
}
log.d(err, body, countlyConfigOrig, countlyConfig);
if (countly_tracking) {
db.collection("members").findOne({global_admin: true}, function(err2, member) {
if (!err2 && member) {
var date = new Date();
let domain = plugins.getConfig('api').domain;
);

try {
// try to extract hostname from full domain url
const urlObj = new URL(domain);
domain = urlObj.hostname;
}
catch (_) {
// do nothing, domain from config will be used as is
}
var last_sync = syncResult.value ? syncResult.value.last_sync : null;
if (last_sync) {
days = Math.floor((new Date().getTime() - last_sync) / (1000 * 60 * 60 * 24));
}

request({
uri: "https://stats.count.ly/i",
method: "GET",
timeout: 4E3,
qs: {
device_id: domain,
app_key: "e70ec21cbe19e799472dfaee0adb9223516d238f",
timestamp: Math.floor(date.getTime() / 1000),
hour: date.getHours(),
dow: date.getDay(),
no_meta: true,
events: JSON.stringify([
{
key: "PING",
count: 1
}
])
if (days > 0) {
//calculate seconds timestamp of days before today
var startTs = Math.round((new Date().getTime() - (30 * 24 * 60 * 60 * 1000)) / 1000);

//sync server events - use aggregation pipeline to group by day and action on MongoDB side
var aggregationPipeline = [
// Match documents with timestamp greater than startTs and valid action
{
$match: {
ts: { $gt: startTs }
}
},
// Add calculated fields for day grouping
{
$addFields: {
// Convert timestamp to date and set to noon (12:00:00)
dayDate: {
$dateFromParts: {
year: { $year: { $toDate: { $multiply: ["$ts", 1000] } } },
month: { $month: { $toDate: { $multiply: ["$ts", 1000] } } },
day: { $dayOfMonth: { $toDate: { $multiply: ["$ts", 1000] } } },
hour: 12,
minute: 0,
second: 0
}
}, function(a/*, c, b*/) {
log.d('Done running ping job: %j', a);
done();
});
}
}
},
// Convert back to timestamp in seconds
{
$addFields: {
noonTimestamp: {
$divide: [{ $toLong: "$dayDate" }, 1000]
}
}
else {
done();
},
// Group by day and action
{
$group: {
_id: {
day: "$noonTimestamp",
action: "$a"
},
count: { $sum: 1 }
}
});
},
// Project to final format
{
$project: {
_id: 0,
action: "$_id.action",
timestamp: "$_id.day",
count: 1
}
}
];

var cursor = db.collection("systemlogs").aggregate(aggregationPipeline);

while (cursor && await cursor.hasNext()) {
let eventData = await cursor.next();
user.add_event({key: eventData.action, count: eventData.count, timestamp: eventData.timestamp});
}
else {
server.start(function() {
server.stop();
done();
}
});
});
}
else {
done();
}
}
else {
done();
Expand All @@ -104,4 +128,4 @@ class PingJob extends job.Job {
}
}

module.exports = PingJob;
module.exports = PingJob;
Loading
Loading