diff --git a/docs/docs/static/conversationsuite.json b/docs/docs/static/conversationsuite.json index 5161b3e..b19438f 100644 --- a/docs/docs/static/conversationsuite.json +++ b/docs/docs/static/conversationsuite.json @@ -26,6 +26,9 @@ "environmentName": { "type": "string" }, + "newSessionPerTest": { + "type": "boolean" + }, "tests": { "items": { "$ref": "#/$defs/TestFile" diff --git a/docs/docs/static/swagger.json b/docs/docs/static/swagger.json index 101a243..5ff6f22 100644 --- a/docs/docs/static/swagger.json +++ b/docs/docs/static/swagger.json @@ -298,6 +298,10 @@ "type": "string", "example": "Example Suite" }, + "new_session_per_test": { + "type": "boolean", + "example": false + }, "tests": { "type": "array", "items": { diff --git a/docs/docs/static/swagger.yaml b/docs/docs/static/swagger.yaml index bc60611..251725b 100644 --- a/docs/docs/static/swagger.yaml +++ b/docs/docs/static/swagger.yaml @@ -98,6 +98,9 @@ definitions: name: example: Example Suite type: string + new_session_per_test: + example: false + type: boolean tests: items: $ref: '#/definitions/TestRequest' diff --git a/docs/docs/test-platform/test-suites.md b/docs/docs/test-platform/test-suites.md index 43985d7..f4a06dc 100644 --- a/docs/docs/test-platform/test-suites.md +++ b/docs/docs/test-platform/test-suites.md @@ -12,6 +12,7 @@ A Test Suite is a JSON-formatted definition that contains: - **Test Configuration**: API keys, environment settings, and Voiceflow project details - **Test Cases**: Individual test scenarios with user inputs and expected agent responses - **Validation Rules**: Criteria for determining test success or failure +- **Session Management**: Control whether tests share the same user session or use isolated sessions ## Creating Test Suites @@ -76,6 +77,37 @@ Individual test case files referenced in the suite. You can find more details on For more details on the JSON structure, refer to the [Test Suite JSON Schema](https://docs.voiceflow.com/reference/post_api-v1-tests-execute#/). +#### Session Management + +Test suites support session management through the `new_session_per_test` field: + +```json +{ + "name": "Example Suite", + "description": "Test suite with isolated sessions", + "environment_name": "production", + "new_session_per_test": true, + "tests": [...] +} +``` + +**Session Behavior:** + +- **`false` (default)**: All tests share the same user session + - Variables persist across tests + - Conversation context carries over between tests + - Tests execute sequentially with shared state + +- **`true`**: Each test gets a fresh user session + - New user ID generated for each test + - Complete isolation between tests + - No shared context or variables + +**When to use isolated sessions:** +- Testing independent scenarios that shouldn't affect each other +- Validating conversation flows from a fresh start +- Ensuring tests don't have hidden dependencies on execution order + ## API Integration Test suites integrate with the Voiceflow API for execution: diff --git a/docs/docs/tests/suites.md b/docs/docs/tests/suites.md index 75e7eb6..ffe3c54 100644 --- a/docs/docs/tests/suites.md +++ b/docs/docs/tests/suites.md @@ -13,6 +13,9 @@ name: Example Suite description: Suite used as an example # Environment name of your Voiceflow agent. It could be development, or production. environmentName: development +# Optional: Create a new user session for each test (default: false) +# When enabled, each test will run with a fresh user session instead of sharing one session across all tests +newSessionPerTest: false # You can have multiple tests defined in separated files tests: # ID of the test. @@ -23,6 +26,20 @@ tests: It has the same structure as the NLU Profiler suite. +### Session Management + +By default, all tests within a suite share the same user session (user ID). This means that: + +- Variables set in one test persist to the next test +- The conversation context carries over between tests +- Tests are executed sequentially with the same user state + +If you want each test to start with a fresh session, set `newSessionPerTest: true`. This will: + +- Generate a new user ID for each test +- Clear all conversation context between tests +- Ensure tests are completely isolated from each other + ## JSON Schema `voiceflow-cli` also has a [jsonschema](http://json-schema.org/draft/2020-12/json-schema-validation.html) file, which you can use to have better diff --git a/examples/suite.yaml b/examples/suite.yaml index 7b16180..a8fea6e 100644 --- a/examples/suite.yaml +++ b/examples/suite.yaml @@ -1,6 +1,9 @@ name: Example Conversation Profiler Suite description: Suite used as an example environmentName: production +# Optional: Create a new user session for each test (default: false) +# When enabled, each test will run with a fresh user session +newSessionPerTest: true tests: - id: traditional_test file: ./tests/test_1.yaml diff --git a/internal/types/tests/suiteTypes.go b/internal/types/tests/suiteTypes.go index 39eeda7..7425e6a 100644 --- a/internal/types/tests/suiteTypes.go +++ b/internal/types/tests/suiteTypes.go @@ -1,11 +1,12 @@ package tests type Suite struct { - Name string `yaml:"name" json:"name"` - Description string `yaml:"description" json:"description"` - EnvironmentName string `yaml:"environmentName" json:"environmentName"` - Tests []TestFile `yaml:"tests" json:"tests"` - OpenAIConfig *OpenAIConfig `yaml:"openAIConfig,omitempty" json:"openAIConfig,omitempty"` + Name string `yaml:"name" json:"name"` + Description string `yaml:"description" json:"description"` + EnvironmentName string `yaml:"environmentName" json:"environmentName"` + NewSessionPerTest bool `yaml:"newSessionPerTest,omitempty" json:"newSessionPerTest,omitempty"` + Tests []TestFile `yaml:"tests" json:"tests"` + OpenAIConfig *OpenAIConfig `yaml:"openAIConfig,omitempty" json:"openAIConfig,omitempty"` } type TestFile struct { diff --git a/pkg/test/execute.go b/pkg/test/execute.go index 7bedc49..5a10312 100644 --- a/pkg/test/execute.go +++ b/pkg/test/execute.go @@ -14,6 +14,7 @@ type HTTPSuiteRequest struct { Name string `json:"name"` Description string `json:"description"` EnvironmentName string `json:"environment_name"` + NewSessionPerTest bool `json:"new_session_per_test,omitempty"` // Optional flag to create a new user session for each test (default: false) Tests []HTTPTestRequest `json:"tests"` ApiKey string `json:"api_key,omitempty"` // Optional token to override global.VoiceflowAPIKey VoiceflowSubdomain string `json:"voiceflow_subdomain,omitempty"` // Optional subdomain to override global.VoiceflowSubdomain @@ -68,7 +69,7 @@ func ExecuteFromHTTPRequest(suiteReq HTTPSuiteRequest) *ExecuteSuiteResult { // executeHTTPSuite executes a suite from HTTP request data func executeHTTPSuite(suiteReq HTTPSuiteRequest, logCollector *LogCollector) error { // Define the user ID - userID := uuid.New().String() + userID := "test-" + uuid.New().String() if suiteReq.VoiceflowSubdomain != "" { logCollector.AddLog("Using Voiceflow subdomain: " + suiteReq.VoiceflowSubdomain) @@ -77,11 +78,20 @@ func executeHTTPSuite(suiteReq HTTPSuiteRequest, logCollector *LogCollector) err logCollector.AddLog("Suite: " + suiteReq.Name) logCollector.AddLog("Description: " + suiteReq.Description) logCollector.AddLog("Environment: " + suiteReq.EnvironmentName) - logCollector.AddLog("User ID: " + userID) + if suiteReq.NewSessionPerTest { + logCollector.AddLog("New session per test: enabled") + } else { + logCollector.AddLog("User ID: " + userID) + } logCollector.AddLog("Running Tests:") // Execute each test directly from the request data for _, testReq := range suiteReq.Tests { + // Create a new user ID for each test if newSessionPerTest is enabled + if suiteReq.NewSessionPerTest { + userID = "test-" + uuid.New().String() + logCollector.AddLog("User ID for test " + testReq.ID + ": " + userID) + } logCollector.AddLog("Running Test ID: " + testReq.ID) err := runTest(suiteReq.EnvironmentName, userID, testReq.Test, suiteReq.ApiKey, suiteReq.VoiceflowSubdomain, logCollector, suiteReq.OpenAIConfig) if err != nil { @@ -97,7 +107,7 @@ func executeHTTPSuite(suiteReq HTTPSuiteRequest, logCollector *LogCollector) err func ExecuteSuite(suitesPath string) error { // Define the user ID - userID := uuid.New().String() + userID := "test-" + uuid.New().String() // Load all suites from the path suites, err := utils.LoadSuitesFromPath(suitesPath) @@ -108,10 +118,19 @@ func ExecuteSuite(suitesPath string) error { // Iterate over each suite and its tests for _, suite := range suites { - global.Log.Infof("Suite: %s\nDescription: %s\nEnvironment: %s\nUser ID: %s", suite.Name, suite.Description, suite.EnvironmentName, userID) + if suite.NewSessionPerTest { + global.Log.Infof("Suite: %s\nDescription: %s\nEnvironment: %s\nNew session per test: enabled", suite.Name, suite.Description, suite.EnvironmentName) + } else { + global.Log.Infof("Suite: %s\nDescription: %s\nEnvironment: %s\nUser ID: %s", suite.Name, suite.Description, suite.EnvironmentName, userID) + } global.Log.Infof("Running Tests:") for _, testFile := range suite.Tests { + // Create a new user ID for each test if newSessionPerTest is enabled + if suite.NewSessionPerTest { + userID = "test-" + uuid.New().String() + global.Log.Infof("User ID for test %s: %s", testFile.ID, userID) + } test, err := utils.LoadTestFromPath(testFile.File) if err != nil { global.Log.Errorf("Error loading test: %v", err) diff --git a/server/docs/docs.go b/server/docs/docs.go index d7c56a8..55d2a11 100644 --- a/server/docs/docs.go +++ b/server/docs/docs.go @@ -304,6 +304,10 @@ const docTemplate = `{ "type": "string", "example": "Example Suite" }, + "new_session_per_test": { + "type": "boolean", + "example": false + }, "tests": { "type": "array", "items": { diff --git a/server/handlers/handlers.go b/server/handlers/handlers.go index 7542874..0508b73 100644 --- a/server/handlers/handlers.go +++ b/server/handlers/handlers.go @@ -23,10 +23,11 @@ type TestExecutionRequest struct { // TestSuiteRequest represents a test suite configuration for HTTP requests type TestSuiteRequest struct { - Name string `json:"name" binding:"required" example:"Example Suite"` - Description string `json:"description" example:"Suite used as an example"` - EnvironmentName string `json:"environment_name" binding:"required" example:"production"` - Tests []TestRequest `json:"tests" binding:"required,dive"` + Name string `json:"name" binding:"required" example:"Example Suite"` + Description string `json:"description" example:"Suite used as an example"` + EnvironmentName string `json:"environment_name" binding:"required" example:"production"` + NewSessionPerTest bool `json:"new_session_per_test" example:"false"` + Tests []TestRequest `json:"tests" binding:"required,dive"` } // @name TestSuiteRequest // TestRequest represents a test configuration for HTTP requests @@ -187,6 +188,7 @@ func ExecuteTestSuite(c *gin.Context) { Name: req.Suite.Name, Description: req.Suite.Description, EnvironmentName: req.Suite.EnvironmentName, + NewSessionPerTest: req.Suite.NewSessionPerTest, Tests: make([]test.HTTPTestRequest, len(req.Suite.Tests)), ApiKey: req.ApiKey, // Pass the optional ApiKey VoiceflowSubdomain: req.VoiceflowSubdomain, // Pass the optional VoiceflowSubdomain