better hooks handling by submitting where test.failed occur

lib/helper/Playwright.js

@@ -523,12 +523,17 @@ class Playwright extends Helper {
         this.currentRunningTest.artifacts.har = fileName
         contextOptions.recordHar = this.options.recordHar
       }
+
+      // load pre-saved cookies
+      if (test.opts.cookies) contextOptions.storageState = { cookies: test.opts.cookies }
+
       if (this.storageState) contextOptions.storageState = this.storageState
       if (this.options.userAgent) contextOptions.userAgent = this.options.userAgent
       if (this.options.locale) contextOptions.locale = this.options.locale
       if (this.options.colorScheme) contextOptions.colorScheme = this.options.colorScheme
       this.contextOptions = contextOptions
       if (!this.browserContext || !restartsSession()) {
+        this.debugSection('New Session', JSON.stringify(this.contextOptions))
         this.browserContext = await this.browser.newContext(this.contextOptions) // Adding the HTTPSError ignore in the context so that we can ignore those errors
       }
     }
@@ -938,7 +943,8 @@ class Playwright extends Helper {
       throw new Error('Cannot open pages inside an Electron container')
     }
     if (!/^\w+\:(\/\/|.+)/.test(url)) {
-      url = this.options.url + (url.startsWith('/') ? url : `/${url}`)
+      url = this.options.url + (!this.options.url.endsWith('/') && url.startsWith('/') ? url : `/${url}`)
+      this.debug(`Changed URL to base url + relative path: ${url}`)
     }
 
     if (this.options.basicAuth && this.isAuthenticated !== true) {

lib/listener/store.js

@@ -2,11 +2,19 @@ const event = require('../event')
 const store = require('../store')
 
 module.exports = function () {
+  event.dispatcher.on(event.suite.before, suite => {
+    store.currentSuite = suite
+  })
+
+  event.dispatcher.on(event.suite.after, () => {
+    store.currentSuite = null
+  })
+
   event.dispatcher.on(event.test.before, test => {
     store.currentTest = test
   })
 
-  event.dispatcher.on(event.test.finished, test => {
+  event.dispatcher.on(event.test.finished, () => {
     store.currentTest = null
   })
 }

lib/mocha/asyncWrapper.js

@@ -19,10 +19,10 @@ const injectHook = function (inject, suite) {
   return recorder.promise()
 }
 
-function suiteTestFailedHookError(suite, err) {
+function suiteTestFailedHookError(suite, err, hookName) {
   suite.eachTest(test => {
     test.err = err
-    event.emit(event.test.failed, test, err)
+    event.emit(event.test.failed, test, err, ucfirst(hookName))
   })
 }
 
@@ -120,7 +120,7 @@ module.exports.injected = function (fn, suite, hookName) {
     const errHandler = err => {
       recorder.session.start('teardown')
       recorder.cleanAsyncErr()
-      if (hookName == 'before' || hookName == 'beforeSuite') suiteTestFailedHookError(suite, err)
+      if (hookName == 'before' || hookName == 'beforeSuite') suiteTestFailedHookError(suite, err, hookName)
       if (hookName === 'after') event.emit(event.test.after, suite)
       if (hookName === 'afterSuite') event.emit(event.suite.after, suite)
       recorder.add(() => doneFn(err))

lib/plugin/analyze.js

@@ -12,8 +12,8 @@ const { ansiRegExp, base64EncodeFile, markdownToAnsi } = require('../utils')
 const MAX_DATA_LENGTH = 5000
 
 const defaultConfig = {
-  clusterize: 2,
-  analyze: 3,
+  clusterize: 5,
+  analyze: 2,
   vision: false,
   categories: [
     'Browser connection error / browser crash',
@@ -64,17 +64,18 @@ const defaultConfig = {
         If you identify that all tests in the group have the same tag, add this tag to the group report, otherwise ignore TAG section.
         If you identify that all tests in the group have the same suite, add this suite to the group report, otherwise ignore SUITE section.
         Pick different emojis for each group.
-        Do not include group into report if it has only one test in affected tests section.
+        Order groups by the number of tests in the group.
+        If group has one test, skip that group.
 
         Provide list of groups in following format:
 
         _______________________________
 
-        ## Group <group_number>
+        ## Group <group_number> <emoji>
 
+        * SUMMARY <summary_of_errors>
         * CATEGORY <category_of_failure>
         * ERROR <error_message_1>, <error_message_2>, ...
-        * SUMMARY <summary_of_errors>
         * STEP <step_of_failure> (use CodeceptJS format I.click(), I.see(), etc; if all failures happend on the same step)
         * SUITE <suite_title>, <suite_title> (if SUITE is present, and if all tests in the group have the same suite or suites)
         * TAG <tag> (if TAG is present, and if all tests in the group have the same tag)
@@ -126,14 +127,16 @@ const defaultConfig = {
         Do not get to details, be concise.
         If there is failed step, just write it in STEPS section.
         If you have suggestions for the test, write them in SUMMARY section.
+        Do not be too technical in SUMMARY section.
         Inside SUMMARY write exact values, if you have suggestions, explain which information you used to suggest.
         Be concise, each section should not take more than one sentence.
 
         Response format:
 
+        * SUMMARY <explanation_of_failure>
+        * ERROR <error_message_1>, <error_message_2>, ...
         * CATEGORY <category_of_failure>
         * STEPS <step_of_failure>
-        * SUMMARY <explanation_of_failure>
 
         Do not add any other sections or explanations. Only CATEGORY, SUMMARY, STEPS.
         ${config.vision ? 'Also a screenshot of the page is attached to the prompt.' : ''}
@@ -153,11 +156,6 @@ const defaultConfig = {
         })
       }
 
-      messages.push({
-        role: 'assistant',
-        content: `## `,
-      })
-
       return messages
     },
   },

lib/plugin/autoLogin.js

@@ -1,8 +1,12 @@
 const fs = require('fs')
 const path = require('path')
 const { fileExists } = require('../utils')
+const FuncStep = require('../step/func')
+const Section = require('../step/section')
+const recordStep = require('../step/record')
 const container = require('../container')
 const store = require('../store')
+const event = require('../event')
 const recorder = require('../recorder')
 const { debug } = require('../output')
 const isAsyncFunction = require('../utils').isAsyncFunction
@@ -276,8 +280,42 @@ module.exports = function (config) {
   }
 
   const loginFunction = async name => {
-    const userSession = config.users[name]
     const I = container.support('I')
+    const test = store.currentTest
+
+    // we are in BeforeSuite hook
+    if (!test) {
+      enableAuthBeforeEachTest(name)
+      return
+    }
+
+    if (config.saveToFile && !store[`${name}_session`]) {
+      // loading from file
+      for (const name in config.users) {
+        const fileName = path.join(global.output_dir, `${name}_session.json`)
+        if (!fileExists(fileName)) continue
+        const data = fs.readFileSync(fileName).toString()
+        try {
+          store[`${name}_session`] = JSON.parse(data)
+        } catch (err) {
+          throw new Error(`Could not load session from ${fileName}\n${err}`)
+        }
+        debug(`Loaded user session for ${name}`)
+      }
+    }
+
+    if (isPlaywrightSession() && test?.opts?.cookies) {
+      if (test.opts.user == name) {
+        debug(`Cookies already loaded for ${name}`)
+        // alreadyLoggedIn(name);
+        return
+      } else {
+        debug(`Cookies already loaded for ${test.opts.user}, but not for ${name}`)
+        await I.deleteCookie()
+      }
+    }
+
+    const userSession = config.users[name]
     const cookies = store[`${name}_session`]
     const shouldAwait = isAsyncFunction(userSession.login) || isAsyncFunction(userSession.restore) || isAsyncFunction(userSession.check)
 
@@ -332,8 +370,62 @@ module.exports = function (config) {
     return recorder.promise()
   }
 
+  function enableAuthBeforeEachTest(name) {
+    const suite = store.currentSuite
+    if (!suite) return
+
+    debug(`enabling auth as ${name} for each test of suite ${suite.title}`)
+
+    // we are setting test opts so they can be picked up by Playwright if it starts browser for this test
+    suite.eachTest(test => {
+      // preload from store
+      if (store[`${name}_session`]) {
+        test.opts.cookies = store[`${name}_session`]
+        test.opts.user = name
+        return
+      }
+
+      if (!config.saveToFile) return
+      const cookieFile = path.join(global.output_dir, `${name}_session.json`)
+
+      if (!fileExists(cookieFile)) {
+        return
+      }
+
+      const context = fs.readFileSync(cookieFile).toString()
+      test.opts.cookies = JSON.parse(context)
+      test.opts.user = name
+    })
+
+    function runLoginFunctionForTest(test) {
+      if (!suite.tests.includes(test)) return
+      // let's call this function to ensure that authorization happened
+      // if no cookies, it will login and save them
+      loginFunction(name)
+    }
+
+    // we are in BeforeSuite hook
+    event.dispatcher.on(event.test.started, runLoginFunctionForTest)
+    event.dispatcher.on(event.suite.after, () => {
+      event.dispatcher.off(event.test.started, runLoginFunctionForTest)
+    })
+  }
+
   // adding this to DI container
   const support = {}
   support[config.inject] = loginFunction
   container.append({ support })
+
+  return loginFunction
+}
+
+function isPlaywrightSession() {
+  return !!container.helpers('Playwright')
+}
+
+function alreadyLoggedIn(name) {
+  const step = new FuncStep('am logged in as')
+  step.actor = 'I'
+  step.setCallable(() => {})
+  return recordStep(step, [name])
 }

lib/plugin/screenshotOnFail.js

@@ -72,11 +72,12 @@ module.exports = function (config) {
     return
   }
 
-  event.dispatcher.on(event.test.failed, test => {
-    if (test.ctx?._runnable.title.includes('hook: ')) {
-      output.plugin('screenshotOnFail', 'BeforeSuite/AfterSuite do not have any access to the browser, hence it could not take screenshot.')
+  event.dispatcher.on(event.test.failed, (test, _err, hookName) => {
+    if (hookName == 'BeforeSuite' || hookName == 'AfterSuite') {
+      // no browser here
       return
     }
+
     recorder.add(
       'screenshot of failed test',
       async () => {

lib/plugin/stepByStepReport.js

@@ -121,12 +121,13 @@ module.exports = function (config) {
     deleteDir(dir)
   })
 
-  event.dispatcher.on(event.test.failed, (test, err) => {
-    if (test.ctx._runnable.title.includes('hook: ')) {
-      output.plugin('stepByStepReport', 'BeforeSuite/AfterSuite do not have any access to the browser, hence it could not take screenshot.')
+  event.dispatcher.on(event.test.failed, (test, _err, hookName) => {
+    if (hookName == 'BeforeSuite' || hookName == 'AfterSuite') {
+      // no browser here
       return
     }
-    persist(test, err)
+
+    persist(test)
   })
 
   event.dispatcher.on(event.all.result, () => {

lib/step.js

@@ -14,7 +14,13 @@ const Step = require('./step/helper')
  */
 const MetaStep = require('./step/meta')
 
+/**
+ * Step used to execute a single function
+ */
+const FuncStep = require('./step/func')
+
 module.exports = Step
 module.exports.MetaStep = MetaStep
 module.exports.BaseStep = BaseStep
 module.exports.StepConfig = StepConfig
+module.exports.FuncStep = FuncStep

lib/store.js

@@ -15,6 +15,8 @@ const store = {
   currentTest: null,
   /** @type {any} */
   currentStep: null,
+  /** @type {CodeceptJS.Suite | null} */
+  currentSuite: null,
 }
 
 module.exports = store

package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeceptjs",
-  "version": "3.7.0-beta.1",
+  "version": "3.7.0-beta.8",
   "description": "Supercharged End 2 End Testing Framework for NodeJS",
   "keywords": [
     "acceptance",