Merge pull request #6898 from Countly/ar2rsawseen/newarchitecture

Merging next into new arch

Author: ar2rsawseen

.github/workflows/codeql-analysis.yml

@@ -41,7 +41,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
+      uses: github/codeql-action/init@v4
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -52,7 +52,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v3
+      uses: github/codeql-action/autobuild@v4
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -66,4 +66,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      uses: github/codeql-action/analyze@v4

.github/workflows/main.yml

@@ -280,6 +280,13 @@ jobs:
           wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb -O /tmp/chrome.deb
           apt install -y /tmp/chrome.deb
 
+      - name: Install Sharp dependencies for image processing
+        shell: bash
+        run: |
+          export DEBIAN_FRONTEND=noninteractive
+          apt-get update -y
+          apt-get install -y libvips-dev
+
       - name: Copy code
         shell: bash
         run: |
@@ -338,8 +345,8 @@ jobs:
         working-directory: /opt/countly/ui-tests/cypress
         run: |
           ARTIFACT_ARCHIVE_NAME="$(date '+%Y%m%d-%H.%M')_${GITHUB_REPOSITORY#*/}_CI#${{ github.run_number }}_${{ matrix.test_type }}.tar.gz"
-          mkdir -p screenshots videos
-          tar zcvf "$ARTIFACT_ARCHIVE_NAME" screenshots videos
+          mkdir -p screenshots videos downloads
+          tar zcvf "$ARTIFACT_ARCHIVE_NAME" screenshots videos downloads
           curl -o /tmp/uploader.log -u "${{ secrets.BOX_UPLOAD_AUTH }}" ${{ secrets.BOX_UPLOAD_PATH }} -T "$ARTIFACT_ARCHIVE_NAME"
 
   ui-test-sdk:

CHANGELOG.md

@@ -2,6 +2,69 @@
 Dependencies:
 - Remove SQLite
 
+## Version 25.03.xx
+Fixes:
+- [push] Fixed timeout setting
+- [security] Fixed injection possibility on res.expose
+
+Enterprise Fixes:
+- [groups] Add logs for user updates
+- [surveys] Change question map log to debug log
+
+Enterprise Fixes:
+- [data-manager] Fixed bug when merging events with ampersand symbol in the name
+
+Dependencies:
+- Bump axios from 1.12.2 to 1.13.1 in /plugins/cognito
+- Bump csvtojson from 1.1.12 to 2.0.14
+- Bump eslint-plugin-vue from 10.5.0 to 10.5.1
+- Bump express-rate-limit from 8.1.0 to 8.2.0
+- Bump get-random-values from 4.0.0 to 4.1.0
+- Bump lint-staged from 16.2.4 to 16.2.6
+- Bump mockttp from 4.2.0 to 4.2.1 in /plugins/crash_symbolication
+- Bump nodemailer from 7.0.9 to 7.0.10
+- Bump puppeteer from 24.25.0 to 24.27.0
+- Bump vite from 7.1.10 to 7.1.12
+
+## Version 25.03.25
+Fixes:
+- [crashes] Fixed resolving audit log recording
+- [location] Fixed updating none gps coordinate location after gps location was used
+
+Enterprise Fixes:
+- [ab-testing] Add script for fixing variant cohort
+- [groups] Fix user permission update after updating user group permission
+## Version 25.03.24
+Fixes:
+- [jobs] Fix condition for scheduling alert job
+
+Enterprise Fixes:
+- [compliance-hub] Fixed query patterns
+- [data-manager] Fixed bug preventing transformation of events ending in a dot
+- [data-manager] Fixed segment data deletion
+- [license] Stop sending metric after license expired
+- [users] Fix add/remove user to profile group
+- [users] Remove link to profile group page after removing user from group
+
+Dependencies
+- Bump @faker-js/faker from 10.0.0 to 10.1.0 in /ui-tests
+- Bump countly-sdk-nodejs from 24.10.2 to 24.10.3
+- Bump lint-staged from 16.2.3 to 16.2.4
+- Bump puppeteer from 24.23.0 to 24.24.1
+
+## Version 25.03.23
+Fixes:
+- [events] Entries in the event list are now sorted alphabetically
+- [mail] Add smtp debug option for mail module
+
+Enterprise Fixes:
+- [block] Fixed filter for consent events
+- [drill] [survey] Fix survey answer in drill
+- [funnels] Show notification if funnel results are from cache/task manager
+- [revenue] Card in revenue page are now correctly indentified
+- [users] Add survey section to user feedback page
+- [users] Fixed uploading user profile pictures
+
 ## Version 25.03.22
 Fixes:
 - [alerts] Fix: Migrate alerts to the new events model

Dockerfile-core

@@ -1,4 +1,4 @@
-FROM phusion/baseimage:focal-1.2.0
+FROM phusion/baseimage:jammy-1.0.4
 
 ARG COUNTLY_PLUGINS=mobile,web,desktop,plugins,density,locale,browser,sources,views,logger,systemlogs,populator,reports,crashes,push,star-rating,slipping-away-users,compare,server-stats,dbviewer,times-of-day,compliance-hub,alerts,onboarding,consolidate,remote-config,hooks,dashboards,sdk,data-manager,guides
 # Countly Enterprise:
@@ -40,7 +40,7 @@ RUN useradd -r -M -U -d /opt/countly -s /bin/false countly && \
         libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils \
         # push / nghttp2
         gcc g++ make binutils autoconf automake autotools-dev libtool pkg-config zlib1g-dev libcunit1-dev libssl-dev libxml2-dev libev-dev \
-        libevent-dev libjansson-dev libjemalloc-dev cython python3-dev python-setuptools && \
+        libevent-dev libjansson-dev libjemalloc-dev python3-dev python-setuptools && \
     # node
     wget -qO- https://deb.nodesource.com/setup_20.x | bash - && \
     # data_migration (mongo clients)
@@ -85,7 +85,7 @@ RUN useradd -r -M -U -d /opt/countly -s /bin/false countly && \
     # cleanup
     npm remove -y --no-save mocha nyc should supertest && \
     apt-get remove -y build-essential libkrb5-dev wget \
-        gcc g++ make binutils autoconf automake autotools-dev libtool pkg-config zlib1g-dev libcunit1-dev libssl-dev libxml2-dev libev-dev libevent-dev libjansson-dev libjemalloc-dev cython python3-dev python-setuptools && \
+        gcc g++ make binutils autoconf automake autotools-dev libtool pkg-config zlib1g-dev libcunit1-dev libssl-dev libxml2-dev libev-dev libevent-dev libjansson-dev libjemalloc-dev python3-dev python-setuptools && \
     apt-get install -y gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /tmp/.??* /var/tmp/* /var/tmp/.??* ~/.npm ~/.cache && \

api/parts/data/usage.js

@@ -1018,7 +1018,7 @@ plugins.register("/sdk/user_properties", async function(ob) {
 
                 if (plugins.getConfig('api', params.app && params.app.plugins, true).city_data === true && !userProps.loc && typeof data.lat !== "undefined" && typeof data.lon !== "undefined") {
                     // only override lat/lon if no recent gps location exists in user document
-                    if (!params.app_user.loc || (params.app_user.loc.gps && params.time.mstimestamp - params.app_user.loc.date > 7 * 24 * 3600)) {
+                    if (!params.app_user.loc || !params.app_user.loc.gps || params.time.mstimestamp - params.app_user.loc.date > 7 * 24 * 3600) {
                         userProps.loc = {
                             gps: false,
                             geo: {
@@ -1049,7 +1049,7 @@ plugins.register("/sdk/user_properties", async function(ob) {
 
                     if (plugins.getConfig('api', params.app && params.app.plugins, true).city_data === true && !userProps.loc && data.ll && typeof data.ll[0] !== "undefined" && typeof data.ll[1] !== "undefined") {
                         // only override lat/lon if no recent gps location exists in user document
-                        if (!params.app_user.loc || (params.app_user.loc.gps && params.time.mstimestamp - params.app_user.loc.date > 7 * 24 * 3600)) {
+                        if (!params.app_user.loc || !params.app_user.loc.gps || params.time.mstimestamp - params.app_user.loc.date > 7 * 24 * 3600) {
                             userProps.loc = {
                                 gps: false,
                                 geo: {
@@ -1113,7 +1113,7 @@ plugins.register("/sdk/user_properties", async function(ob) {
                 userProps.av_major = null;
                 userProps.av_minor = null;
                 userProps.av_patch = null;
-                userProps.av_rel = null;
+                userProps.av_prerel = null;
                 userProps.av_build = null;
             }
         }

api/parts/jobs/job.js

@@ -254,32 +254,33 @@ class Job extends EventEmitter {
 
             this._json.next = next.getTime();
         }
-        if (this.name !== "alerts:monitor") {
-            //check if any job already scheduled or running
-            let query = {
-                status: {"$in": [STATUS.SCHEDULED, STATUS.RUNNING]},
-                name: this.name,
-            };
-            if (this._id) {
-                query._id = {$ne: this._id};
-            }
-            var self = this;
-            return new Promise((resolve, reject) => {
-                Job.findMany(this.db(), query).then(existing => {
-                    if (existing && existing.length) {
-                        log.d('Job already scheduled or running: %j', existing);
-                        this._json.status = STATUS.CANCELLED; //set this as cancelled now as we have other scheduled
-                    }
-                    else {
-                        self._save().then(resolve, reject);
-                    }
 
-                });
-            });
+        //check if any job already scheduled or running
+        let query = {
+            status: {"$in": [STATUS.SCHEDULED, STATUS.RUNNING]},
+            name: this.name,
+        };
+
+        if (this.name === 'alerts:monitor' && this.data && Object.keys(this.data).length) {
+            query.data = this.data;
         }
-        else {
-            return this._save();
+
+        if (this._id) {
+            query._id = {$ne: this._id};
         }
+
+        var self = this;
+        return new Promise((resolve, reject) => {
+            Job.findMany(this.db(), query).then(existing => {
+                if (existing && existing.length) {
+                    log.d('Job already scheduled or running: %j', existing);
+                    this._json.status = STATUS.CANCELLED; //set this as cancelled now as we have other scheduled
+                }
+                else {
+                    self._save().then(resolve, reject);
+                }
+            });
+        });
     }
 
     /**
@@ -1142,4 +1143,4 @@ module.exports = {
     STATUS: STATUS,
     STATUS_MAP: STATUS_MAP,
     debounce: debounce
-};
+};

api/utils/pdf.js

@@ -42,7 +42,7 @@ exports.renderPDF = async function(html, callback, options = null, puppeteerArgs
         }
         const updatedTimeout = 240000;
         const page = await browser.newPage();
-
+        await page.setBypassCSP(true);
         page.on('console', (msg) => {
             log.d("Headless chrome page log", msg.text());
         });
@@ -74,6 +74,16 @@ exports.renderPDF = async function(html, callback, options = null, puppeteerArgs
             await page.setContent(html);
         }
 
+        const contentHeight = await page.evaluate(() => {
+            /*global document*/
+            return document.body.scrollHeight;
+        });
+
+        options.width = '210mm'; // A4 width, for example
+        options.height = `${contentHeight}px`; // full content height
+        options.printBackground = true;
+        options.preferCSSPageSize = true;
+
         await page.pdf(options).then(callback, function(error) {
             log.d('pdf generation error', error);
         });

api/utils/render.js

@@ -68,7 +68,7 @@ exports.renderView = function(options, cb) {
                     XDG_CONFIG_HOME: pathModule.resolve(__dirname, "../../.cache/chrome/tmp/.chromium"),
                     XDG_CACHE_HOME: pathModule.resolve(__dirname, "../../.cache/chrome/tmp/.chromium")
                 },
-                args: ['--no-sandbox', '--disable-setuid-sandbox', '--ignore-certificate-errors'],
+                args: ['--no-sandbox', '--disable-setuid-sandbox', '--ignore-certificate-errors', '--disable-web-security'],
                 ignoreHTTPSErrors: true,
                 userDataDir: pathModule.resolve(__dirname, "../../dump/chrome/" + Date.now())
             };
@@ -82,6 +82,7 @@ exports.renderView = function(options, cb) {
             try {
                 log.d('Started rendering images');
                 var page = await browser.newPage();
+                await page.setBypassCSP(true);
 
                 page.on('console', (msg) => {
                     log.d("Headless chrome page log", msg.text());

bin/docker/modify.sh

@@ -17,12 +17,26 @@ if [ "${COUNTLY_CONTAINER}" != "frontend" ]; then
 	# Run ab-testing models compilation if it's there
 	if [ -d /opt/countly/plugins/ab-testing ]; then
 		if [ "${ID}" == "debian" ] || [ "${ID}" == "ubuntu" ]; then
-			apt-get install -y python3-pip
+			echo "Debian noninteractive"
+            export DEBIAN_FRONTEND=noninteractive
+            export TZ=Etc/UTC
+
+           apt-get -y update
+           apt-get install -y software-properties-common build-essential python3-dev libncurses*-dev  libsqlite3-dev libreadline6-dev libgdbm-dev zlib1g-dev libbz2-dev sqlite3 tk-dev zip libssl-dev libncurses5-dev liblzma-dev lsb-core lsb-release
+
+           export LC_ALL="en_US.UTF-8"
+           export LC_CTYPE="en_US.UTF-8"
+           export -n CC
+           export -n CXX
+           add-apt-repository -y ppa:deadsnakes/ppa
+           apt -y install python3.12 python3.12-dev
+           curl -sSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+           python3.12 get-pip.py
 		else
 			yum install -y python36 python36-libs python36-devel python36-pip
 		fi
+		ln -s /usr/bin/python3 /usr/bin/python
 		# shellcheck disable=SC1091
-		python3.8 -m pip install -r /opt/countly/plugins/ab-testing/api/bayesian/requirements.txt
-		cd /opt/countly/plugins/ab-testing/api/bayesian && python3.8 model.py
+		python3.12 -m pip install -r "/opt/countly/plugins/ab-testing/api/bayesian/requirements_docker.txt" && sudo python3.12 "/opt/countly/plugins/ab-testing/api/bayesian/models/cmdstanpy_model.py"
 	fi
 fi