Fix flaky segment editor test (#23905)

* trying to fix failing test for segment editor

* putting back commented out code for faster testing

* removed unused code

* put back touched code. removing unnecessary timeout and log

* adding check that list is closed before the loop starts

* adding a console log to see if the check is actually working and adding a test screenshot to see what happens

* changing how we check if segment is checked if it is expanded or not

* moved the test a little bit on top to prevent issues caused by side effect from previous tests

* moved the new test to its own suite to test if this works w/ a new test state

* adding missing parameters

* trying to add back the test but making sure to close the panel at the end of the last test

* removing the test from old test suite and putting it on its own suite so that there are no issues with test state

* removed console log

* trying out resetting the compared segments before testing the limits

* removing test solution because it did not work

Author: chippison

plugins/SegmentEditor/tests/UI/SegmentSelectorCompare_spec.js

@@ -0,0 +1,70 @@
+/*!
+ * Matomo - free/libre analytics platform
+ *
+ * SegmentEditor Compare tests.
+ *
+ * @link    https://matomo.org
+ * @license https://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+describe('SegmentComparison', () => {
+  var generalParams = 'idSite=1&period=year&date=2012-08-09';
+  it('should not allow comparing segments more than the limit set', async function() {
+    const configLimit = 2;
+    const maxSegments = configLimit + 1;
+    testEnvironment.overrideConfig('General', 'data_comparison_segment_limit', configLimit);
+    await testEnvironment.save();
+    const dashUrl = '?module=CoreHome&action=index&' + generalParams + '#?' + generalParams + '&category=Dashboard_Dashboard&subcategory=1'
+    // Need to reload here since overrideConfig above does not really
+    // reflect well when the config is used in javascript
+    // start a fresh navigation so the new config is injected
+    await page.goto('about:blank');
+    await page.waitForNetworkIdle();
+    await page.goto(dashUrl);
+    await page.waitForNetworkIdle();
+    // We grab the max limit message
+    const maxLimitMessage = await page.evaluate(
+      (limit) => _pk_translate('General_MaximumNumberOfSegmentsComparedIs', [limit]),
+      maxSegments);
+
+    // We check that the title attribute is still not the max limit message
+    let title = await page.$eval('.segmentationContainer .segmentList li:last-child', (el) => el.getAttribute('title'));
+    expect(title).to.not.equal(maxLimitMessage);
+
+    await page.waitForSelector('.segmentationContainer');
+    // We check that the number of <li> elements is greater than the limit we set
+    const liElemLength = await page.$$eval('.segmentListContainer .segmentList li', (e) => e.length);
+    expect(liElemLength).to.be.greaterThan(maxSegments);
+
+    // We check that the number of segments compared is 1 at the start
+    let comparedCount = await page.$$eval(
+      '.segmentListContainer .segmentList li.comparedSegment',
+      (nodes) => nodes.length,
+    );
+    expect(comparedCount).to.equal(1);
+
+    // Making sure that the list is closed initially before the loop starts
+    const segmentListIsExpanded = await page.evaluate(() => !!document.querySelector('.segmentEditorPanel.expanded'));
+    if (segmentListIsExpanded) {
+      await page.click('.segmentationContainer .title');
+      await page.waitForTimeout(100);
+    }
+
+    // We want to click all the segments so that we can check that it stops at the limit
+    for (let i=0; i<liElemLength; i++) {
+      await page.click('.segmentationContainer .title');
+      await page.waitForTimeout(100);
+      const elements = await page.$$('.segmentListContainer .segmentList li button.compareSegment');
+      if (!elements[i]) break;
+      await elements[i].click();
+      await page.waitForTimeout(100);
+    }
+
+    // We check that the number of segments compared is now equal to the limit we set
+    comparedCount = await page.$$eval(
+      '.segmentListContainer .segmentList li.comparedSegment',
+      (nodes) => nodes.length,
+    );
+    expect(comparedCount).to.equal(maxSegments);
+  });
+});

plugins/SegmentEditor/tests/UI/SegmentSelectorEditor_spec.js

@@ -404,54 +404,4 @@ describe("SegmentSelectorEditorTest", function () {
         await page.click('.segmentationContainer .title');
         expect(await page.screenshotSelector(selectorsToCapture)).to.matchImage('enabled_create_realtime_segments_saved');
     });
-
-    it('should not allow comparing segments more than the limit set', async function() {
-      const configLimit = 2;
-      const maxSegments = configLimit + 1;
-      testEnvironment.overrideConfig('General', 'data_comparison_segment_limit', configLimit);
-      testEnvironment.save();
-      // Need to reload here since overrideConfig above does not really
-      // reflect well when the config is used in javascript
-      await page.reload();
-      await page.waitForNetworkIdle();
-      // We grab the max limit message
-      const maxLimitMessage = await page.evaluate(
-        (limit) => _pk_translate('General_MaximumNumberOfSegmentsComparedIs', [limit]),
-      maxSegments);
-
-      // We check that the title attribute is still not the max limit message
-      let title = await page.$eval('.segmentationContainer .segmentList li:last-child', (el) => el.getAttribute('title'));
-      expect(title).to.not.equal(maxLimitMessage);
-
-      await page.waitForSelector('.segmentationContainer');
-      // We check that the number of <li> elements is greater than the limit we set
-      const liElemLength = await page.$$eval('.segmentListContainer .segmentList li', (e) => e.length);
-      expect(liElemLength).to.be.greaterThan(maxSegments);
-
-      // We check that the number of segments compared is 1 at the start
-      let comparedCount = await page.$$eval(
-        '.segmentListContainer .segmentList li.comparedSegment',
-        (nodes) => nodes.length,
-      );
-      expect(comparedCount).to.equal(1);
-
-      // We want to click all the segments so that we can check that it stops at the limit
-      for (let i=0; i<liElemLength; i++) {
-        await page.click('.segmentationContainer .title');
-        const segments = await page.$$('.segmentListContainer .segmentList li span.compareSegment');
-        await segments[i].click();
-        await page.waitForTimeout(50);
-      }
-
-      // We check that the number of segments compared is now equal to the limit we set
-      comparedCount = await page.$$eval(
-        '.segmentListContainer .segmentList li.comparedSegment',
-        (nodes) => nodes.length,
-      );
-      expect(comparedCount).to.equal(maxSegments);
-
-      // Last we check that the title attribute has changed to reflect the limit has been reached
-      title = await page.$eval('.segmentationContainer .segmentList li:last-child', (el) => el.getAttribute('title'));
-      expect(title).to.equal(maxLimitMessage);
-    });
 });