From a4195495bf54bf8c3043f0a35fa05280c50516ac Mon Sep 17 00:00:00 2001 From: David Rajnoha Date: Fri, 31 Oct 2025 14:48:55 +0100 Subject: [PATCH 1/2] chore(cypress): Added a submodule referencing the incidents detection testing documenation --- .gitmodules | 3 +++ web/cypress/fixtures/incidents/test-docs | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 web/cypress/fixtures/incidents/test-docs diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..94447ed2 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "web/cypress/fixtures/incidents/test-docs"] + path = web/cypress/fixtures/incidents/test-docs + url = git@gitlab.cee.redhat.com:drajnoha/cat-test-docs.git diff --git a/web/cypress/fixtures/incidents/test-docs b/web/cypress/fixtures/incidents/test-docs new file mode 160000 index 00000000..c601bc0e --- /dev/null +++ b/web/cypress/fixtures/incidents/test-docs @@ -0,0 +1 @@ +Subproject commit c601bc0eb518a12aed3c0cbded6d6f81c4dc3bd0 From f264cbfc8d20f95dbfeb8cce48b4e7a9016581ff Mon Sep 17 00:00:00 2001 From: David Rajnoha Date: Fri, 31 Oct 2025 14:50:34 +0100 Subject: [PATCH 2/2] feat(cypress): Add cursor commands for incidents test generation --- .claude/commands/fixture-schema-reference.md | 1 + .claude/commands/generate-incident-fixture.md | 1 + .claude/commands/generate-regression-test.md | 1 + .claude/commands/refactor-regression-test.md | 1 + .../commands/validate-incident-fixtures.md | 1 + .cursor/commands/generate-regression-test.md | 643 ++++++++++++++++++ .cursor/commands/refactor-regression-test.md | 426 ++++++++++++ .../rules/incidents-testing-guidelines.mdc | 558 +++++++++++++++ .gitmodules | 3 - docs/incident_detection/tests/0.overview.md | 74 ++ .../tests/1.filtering_flows.md | 59 ++ .../tests/2.ui_display_flows.md | 117 ++++ .../tests/3.api_calls_data_loading_flows.md | 94 +++ .../tests/4.redux_state_and_effects_flows.md | 120 ++++ .../tests/5.customization.md | 12 + .../tests/6.table_interactions.md | 10 + .../tests/Uncategorized.testing_flows_ui.md | 15 + web/cypress/README.md | 6 + web/cypress/fixtures/incidents/test-docs | 1 - 19 files changed, 2139 insertions(+), 4 deletions(-) create mode 120000 .claude/commands/fixture-schema-reference.md create mode 120000 .claude/commands/generate-incident-fixture.md create mode 120000 .claude/commands/generate-regression-test.md create mode 120000 .claude/commands/refactor-regression-test.md create mode 120000 .claude/commands/validate-incident-fixtures.md create mode 100644 .cursor/commands/generate-regression-test.md create mode 100644 .cursor/commands/refactor-regression-test.md create mode 100644 .cursor/rules/incidents-testing-guidelines.mdc delete mode 100644 .gitmodules create mode 100644 docs/incident_detection/tests/0.overview.md create mode 100644 docs/incident_detection/tests/1.filtering_flows.md create mode 100644 docs/incident_detection/tests/2.ui_display_flows.md create mode 100644 docs/incident_detection/tests/3.api_calls_data_loading_flows.md create mode 100644 docs/incident_detection/tests/4.redux_state_and_effects_flows.md create mode 100644 docs/incident_detection/tests/5.customization.md create mode 100644 docs/incident_detection/tests/6.table_interactions.md create mode 100644 docs/incident_detection/tests/Uncategorized.testing_flows_ui.md delete mode 160000 web/cypress/fixtures/incidents/test-docs diff --git a/.claude/commands/fixture-schema-reference.md b/.claude/commands/fixture-schema-reference.md new file mode 120000 index 00000000..97edc3ca --- /dev/null +++ b/.claude/commands/fixture-schema-reference.md @@ -0,0 +1 @@ +../../.cursor/commands/fixture-schema-reference.md \ No newline at end of file diff --git a/.claude/commands/generate-incident-fixture.md b/.claude/commands/generate-incident-fixture.md new file mode 120000 index 00000000..9f30f373 --- /dev/null +++ b/.claude/commands/generate-incident-fixture.md @@ -0,0 +1 @@ +../../.cursor/commands/generate-incident-fixture.md \ No newline at end of file diff --git a/.claude/commands/generate-regression-test.md b/.claude/commands/generate-regression-test.md new file mode 120000 index 00000000..46668558 --- /dev/null +++ b/.claude/commands/generate-regression-test.md @@ -0,0 +1 @@ +../../.cursor/commands/generate-regression-test.md \ No newline at end of file diff --git a/.claude/commands/refactor-regression-test.md b/.claude/commands/refactor-regression-test.md new file mode 120000 index 00000000..b13ef9d0 --- /dev/null +++ b/.claude/commands/refactor-regression-test.md @@ -0,0 +1 @@ +../../.cursor/commands/refactor-regression-test.md \ No newline at end of file diff --git a/.claude/commands/validate-incident-fixtures.md b/.claude/commands/validate-incident-fixtures.md new file mode 120000 index 00000000..c41caae9 --- /dev/null +++ b/.claude/commands/validate-incident-fixtures.md @@ -0,0 +1 @@ +../../.cursor/commands/validate-incident-fixtures.md \ No newline at end of file diff --git a/.cursor/commands/generate-regression-test.md b/.cursor/commands/generate-regression-test.md new file mode 100644 index 00000000..12a81907 --- /dev/null +++ b/.cursor/commands/generate-regression-test.md @@ -0,0 +1,643 @@ +--- +description: Generate automated regression test from test documentation +--- + +# Generate Regression Test + +Generate automated regression tests from test documentation in [`docs/incident_detection/tests/`](../../docs/incident_detection/tests/), following the style of existing tests in `@incidents/` and using `@incidents-page.ts` Page Object Model. + + +## Process + +### 1. Parse Test Documentation + +**Input**: Section number (e.g., "Section 2.1", "1.2", "3") + +**Actions**: +- Read test flow files from [`docs/incident_detection/tests/`](../../docs/incident_detection/tests/) (e.g., `1.filtering_flows.md`, `2.ui_display_flows.md`) +- Locate the specified section by number +- Extract: + - Section title and description + - Test prerequisites and data requirements + - Test cases with expected behaviors + - Known bug references (e.g., "Verifies: OU-XXX") + - Any notes about testability (e.g., "WARNING Not possible to test on Injected Data") + +### 2. Analyze Test Requirements + +**Extract from documentation**: +- **Test data needs**: What incidents, alerts, severities are required +- **Test actions**: User interactions (clicks, hovers, filters, selections) +- **Assertions**: Expected outcomes (visibility, counts, content, positions) +- **Edge cases**: Special scenarios to verify + +**Design test flows following Cypress e2e best practices**: +- **Think user journeys**: How would a real user interact with this feature? +- **Combine related actions**: Don't split filtering, verification, and interaction into separate tests +- **Prefer comprehensive flows**: Each `it()` should test a complete, realistic workflow +- **Avoid unit test mindset**: Don't create many tiny isolated tests + +**Map to existing patterns**: +- Identify which `incidentsPage` elements/methods are needed +- Identify any missing page object functionality +- Determine fixture requirements + +### 3. Check/Create Fixtures + +**Fixture location**: `web/cypress/fixtures/incident-scenarios/` + +**Naming convention**: `XX-descriptive-name.yaml` (e.g., `13-tooltip-positioning-scenarios.yaml`) + +**Process**: +1. Check if appropriate fixture exists for the test requirements +2. If missing, prompt user: + ``` + Fixture not found for this test scenario. + + Required test data: + - [List incidents, alerts, severities needed] + + Should I create a fixture using the generate-incident-fixture command? + ``` +3. If user approves, delegate to `generate-incident-fixture` command +4. **Preference**: Use single scenario per test file for focused regression testing +5. Validate created fixture against schema + +**Reference**: See `.cursor/commands/generate-incident-fixture.md` for fixture creation + +### 4. Generate Test File + +**File location**: `web/cypress/e2e/incidents/regression/` + +**Naming convention**: `XX.reg_.cy.ts` +- Use next available number (check existing files) +- Convert section title to kebab-case +- Examples: `05.reg_tooltip_positioning.cy.ts`, `06.reg_silence_matching.cy.ts` + +**File structure**: +```typescript +/* +[Brief description of what this test verifies] + +[Additional context about the bug or behavior being tested] + +Verifies: OU-XXX +*/ + +import { incidentsPage } from '../../../views/incidents-page'; + +const MCP = { + namespace: 'openshift-cluster-observability-operator', + packageName: 'cluster-observability-operator', + operatorName: 'Cluster Observability Operator', + config: { + kind: 'UIPlugin', + name: 'monitoring', + }, +}; + +const MP = { + namespace: 'openshift-monitoring', + operatorName: 'Cluster Monitoring Operator', +}; + +describe('Regression: [Section Name]', () => { + + before(() => { + cy.beforeBlockCOO(MCP, MP); + }); + + beforeEach(() => { + cy.log('Navigate to Observe → Incidents'); + incidentsPage.goTo(); + cy.log('[Brief description of scenario being loaded]'); + cy.mockIncidentFixture('incident-scenarios/XX-scenario-name.yaml'); + }); + + it('1. [First test case description]', () => { + cy.log('1.1 [First step description]'); + incidentsPage.clearAllFilters(); + + incidentsPage.elements.incidentsChartContainer().should('be.visible'); + incidentsPage.elements.incidentsChartBarsGroups().should('have.length', N); + cy.pause(); // Manual verification point + + cy.log('1.2 [Second step description]'); + // More test steps with assertions + cy.pause(); // Manual verification point + + // Another test case... + }); +}); +``` + +### 5. Implement Automated Assertions + +Convert manual verification steps from documentation to automated assertions. + +**IMPORTANT - E2E Test Flow Design**: +- **Combine related steps**: Group filtering, verification, interaction, and results checking in one test +- **Test complete workflows**: Each `it()` should tell a complete story of user interaction +- **Multiple assertions per test**: Don't split every assertion into a separate test +- **Realistic user journeys**: Simulate how users actually use the feature, with multiple steps +- Tests can be 50-100+ lines if they represent a complete, realistic user workflow + +**IMPORTANT - Two-Phase Approach**: +- **Initial test generation**: Include `cy.pause()` statements after key setup steps for manual verification +- **Purpose**: Allow user to manually verify behavior before adding complex assertions +- **User workflow**: User will manually delete `cy.pause()` statements once verified +- **Follow-up edits**: Do NOT reintroduce `cy.pause()` if user has already removed them + +**When to include cy.pause()**: +- Include in newly generated test files +- Include when adding new test cases to existing files +- Do NOT include if editing existing test cases that already have assertions + +**Common assertion patterns**: + +#### Visibility and Existence +```typescript +incidentsPage.elements.incidentsChartContainer().should('be.visible'); +incidentsPage.elements.incidentsTable().should('not.exist'); +``` + +#### Counts and Length +```typescript +incidentsPage.elements.incidentsChartBarsGroups().should('have.length', 12); +incidentsPage.elements.incidentsDetailsTableRows().should('have.length.greaterThan', 0); +``` + +#### Text Content +```typescript +incidentsPage.elements.daysSelectToggle().should('contain.text', '7 days'); +incidentsPage.elements.incidentsTableComponentCell(0) + .invoke('text') + .should('contain', 'monitoring'); +``` + +#### Conditional Waiting +```typescript +cy.waitUntil( + () => incidentsPage.elements.incidentsChartBarsGroups().then($groups => $groups.length === 12), + { + timeout: 10000, + interval: 500, + errorMsg: 'All 12 incidents should load within 10 seconds' + } +); +``` + +#### Position and Layout Checks +```typescript +incidentsPage.elements.incidentsChartBarsVisiblePaths() + .first() + .then(($element) => { + const rect = $element[0].getBoundingClientRect(); + expect(rect.width).to.be.greaterThan(5); + expect(rect.height).to.be.greaterThan(0); + }); +``` + +#### Tooltip Interactions +```typescript +incidentsPage.elements.incidentsChartBarsVisiblePaths() + .first() + .trigger('mouseover', { force: true }); + +cy.get('[role="tooltip"]').should('be.visible'); +cy.get('[role="tooltip"]').should('contain.text', 'Expected content'); +``` + +#### Filter Chips +```typescript +incidentsPage.elements.severityFilterChip().should('be.visible'); +incidentsPage.elements.severityFilterChip() + .should('contain.text', 'Critical'); +``` + +### 6. Page Object Usage + +**Priority order**: +1. Use existing `incidentsPage.elements.*` selectors +2. Use existing `incidentsPage.*` methods +3. Suggest adding new elements/methods to page object +4. Custom selectors only as last resort + +**When missing functionality is identified**: + +Prompt user: +``` +The following elements/methods are needed but not present in incidents-page.ts: + +Elements needed: +- tooltipContainer: () => cy.get('[role="tooltip"]') +- tooltipContent: () => cy.get('[role="tooltip"] .pf-c-tooltip__content') + +Methods needed: +- hoverOverIncidentBar: (index: number) => { + cy.log('incidentsPage.hoverOverIncidentBar'); + incidentsPage.elements.incidentsChartBarsVisiblePaths() + .eq(index) + .trigger('mouseover', { force: true }); + } + +Should I add these to incidents-page.ts? +``` + +**Page Object Patterns**: + +*Element selector*: +```typescript +elements: { + simpleElement: () => cy.byTestID(DataTestIDs.Component.Element), + + parameterizedElement: (param: string) => + cy.byTestID(`${DataTestIDs.Component.Element}-${param.toLowerCase()}`), + + compositeSelector: () => + incidentsPage.elements.toolbar().contains('span', 'Category').parent(), +} +``` + +*Action method*: +```typescript +actionName: (param: Type) => { + cy.log('incidentsPage.actionName'); + incidentsPage.elements.something().click(); + incidentsPage.elements.result().should('be.visible'); +} +``` + +*Query method returning Chainable*: +```typescript +getData: (): Cypress.Chainable => { + cy.log('incidentsPage.getData'); + return incidentsPage.elements.container() + .invoke('text') + .then((text) => { + return cy.wrap(processData(text)); + }); +} +``` + +#### 6.5. Type Safety Guidelines + +**Use specific types, avoid `any`**: + +```typescript +// Good +const verifyOpacity = ( + selector: Cypress.Chainable>, + expectedOpacity: number +) => { ... } + +// Avoid +const verifyProperty = (selector: any, value: any) => { ... } +``` + +**Common Cypress patterns**: +- DOM elements: `Cypress.Chainable>` +- Data returns: `Cypress.Chainable` +- Actions: `Cypress.Chainable` or omit return type +- Constrained strings: `'critical' | 'warning' | 'info'` + +**Always specify return types** when suggesting page object methods. + +### 7. Error & Ambiguity Handling + +Handle common failure scenarios gracefully and make reasonable decisions when requirements are unclear. + +#### 7.1. Ambiguous Test Documentation + +**When**: Test documentation is unclear, incomplete, or contradictory. + +**Actions**: +1. Check similar test sections and existing regression tests for patterns +2. Make reasonable assumptions based on common UI testing patterns +3. Document assumptions in test comments with TODO markers +4. Prompt user: + ``` + Found ambiguities: [list specific unclear points] + Proceeding with assumptions: [list assumptions] + Test will include TODO comments for review. Continue? + ``` + +**Example**: +```typescript +// TODO: Documentation unclear on severity filter - assuming 'Critical' based on similar tests +incidentsPage.toggleFilter('Critical'); +``` + +#### 7.3. Fixture Not Found or Multiple Fixtures Match + +**Scenario A - No fixture exists**: +1. Search for similar fixtures in `web/cypress/fixtures/incident-scenarios/` +2. Prompt with options: + ``` + No fixture found. Required: [list requirements] + + Options: + 1. Create new fixture (recommended) + 2. Modify existing: [list closest matches] + 3. Use cy.mockIncidents([]) for empty state + ``` + +**Scenario B - Multiple fixtures match**: +1. Rank by specificity (incident count, severities, components match) +2. Prompt with comparison: + ``` + Multiple fixtures match: + 1. 05-severity-filtering.yaml (90% match) ← Recommended + ✓ Required severities, ✓ Components, ✓ Count + 2. 07-comprehensive.yaml (75% match) + ✓ Severities, ~ Components, ⚠ More incidents than needed + ``` + +#### 7.6. Conflicting Guidance Between Documentation and Existing Tests + +**When**: Documentation describes behavior differently than existing tests implement. + +**Actions**: +``` +Conflict detected: + +Documentation (Section X): [description] +Existing test (file.cy.ts): [different implementation] + +Which to follow? +1. Documentation (may indicate bug in existing test) +2. Existing test (documentation may be outdated) +3. Investigate further +``` + +#### 7.7. Page Object File Not Found or Outdated + +**When**: `incidents-page.ts` not found or structure differs significantly. + +**Actions**: +1. Search likely locations: `web/cypress/views/`, `web/cypress/support/page-objects/` +2. If different structure, attempt to adapt +3. If not found: + ``` + incidents-page.ts not found. + + Options: + 1. Provide correct path + 2. Use custom selectors (not recommended) + 3. Cannot proceed without page object + ``` + +#### 7.8. Missing DataTestIDs in Page Object + +**When**: Element needs DataTestID that doesn't exist in page object. + +**Actions**: +``` +DataTestID not found: [name] + +Fallback options: +1. Text-based: cy.contains('[data-test-id*="chip"]', 'Critical') +2. Role-based: cy.get('[role="listitem"]').contains('Critical') +3. Add DataTestID to component (recommended) ← Recommended + +Which approach? +``` + +#### 7.9. Test Requirements Exceed Fixture Capabilities + +**When**: Test needs scenarios impossible with fixtures (exact timing, external services, animations). + +**Actions**: +``` +Requirement may not be fully testable with fixtures: +"[exact requirement]" + +Issue: [explain limitation] + +Approaches: +1. Test relative behavior (testable with fixtures) ← Recommended +2. Use cy.clock() for timing control (if applicable) +3. Mark as integration test requiring real backend +4. Document: "WARNING: Not possible to test on injected data" +``` + +#### 7.10. General Fallback Strategy + +**For any unexpected situation**: + +1. **Don't fail silently** - Always inform user +2. **Provide context** - Explain what went wrong and impact +3. **Offer 2-3 options** with recommendation +4. **Document workarounds** in comments + +**Template**: +``` +[Issue] - [Why it matters] + +Options: +1. [Recommended approach] ← Recommended +2. [Alternative] +3. [Fallback] + +Proceeding with option 1 will: [actions] +Continue? (y/n/specify) +``` + +### 8. Refactoring +**Note on Refactoring**: Initial test generation focuses on functionality and coverage. After manual verification, use the `/refactor-regression-test` command to clean up duplications and improve readability by extracting helper functions. + + +### 9. Validation Before Output + +**Automated checks (AI should verify):** +- [ ] File naming matches `XX.reg_.cy.ts` +- [ ] Standard MCP/MP configuration blocks present +- [ ] Uses `cy.beforeBlockCOO(MCP, MP)` in `before()` hook +- [ ] Uses `incidentsPage.goTo()` in `beforeEach()` +- [ ] Uses `cy.mockIncidentFixture()` with valid fixture path +- [ ] No emojis in cy.log() statements +- [ ] File header includes purpose and issue reference +- [ ] **For new tests**: Includes `cy.pause()` after key verification points + +**Human judgment (AI provides evidence):** +- [ ] **Tests follow e2e philosophy**: Each `it()` covers a complete user flow + Evidence: List test structure, count of `it()` blocks, steps per test +- [ ] **Test reads like a story**: Implementation details hidden in helpers + Evidence: Show helper functions extracted, test body readability + +**For complete detailed checklist**, see `incidents-testing-guidelines.mdc` +## Example Usage + +### Example 1: Generate Tooltip Positioning Test + +**User Input**: "Generate regression test for Section 2.1: Tooltip Positioning" + +**AI Actions**: +1. Parse Section 2.1 from testing_flows_ui.md +2. Identify requirements: + - Test tooltip positioning for incidents at different chart positions + - Verify tooltip content for multi-component incidents + - Test tooltip positioning in alerts chart +3. **Design comprehensive flow**: Combine all tooltip testing into realistic user journeys + - Flow 1: User hovers over multiple bars to inspect incidents (bottom, middle, top positions) + - Flow 2: User explores multi-component incident details via tooltip +4. Check for fixture - not found +5. Prompt: "Fixture needed with 14 incidents at varying Y positions. Create?" +6. Generate `05.reg_tooltip_positioning.cy.ts` with **comprehensive multi-step tests**: + +```typescript +/* +Regression test for Charts UI tooltip positioning (Section 2.1) + +Verifies that tooltips appear correctly positioned without overlapping +bars or going off-screen, regardless of bar position in chart. +Tests both incidents chart and alerts chart tooltip behavior. + +Verifies: OU-XXX +*/ + +import { incidentsPage } from '../../../views/incidents-page'; + +const MCP = { + namespace: 'openshift-cluster-observability-operator', + packageName: 'cluster-observability-operator', + operatorName: 'Cluster Observability Operator', + config: { + kind: 'UIPlugin', + name: 'monitoring', + }, +}; + +const MP = { + namespace: 'openshift-monitoring', + operatorName: 'Cluster Monitoring Operator', +}; + +describe('Regression: Tooltip Positioning', () => { + + before(() => { + cy.beforeBlockCOO(MCP, MP); + }); + + beforeEach(() => { + cy.log('Navigate to Observe → Incidents'); + incidentsPage.goTo(); + cy.log('Loading tooltip positioning test scenarios'); + cy.mockIncidentFixture('incident-scenarios/13-tooltip-positioning-scenarios.yaml'); + }); + + it('1. Complete tooltip interaction flow - positioning, content, and navigation', () => { + cy.log('1.1 Verify all incidents loaded'); + incidentsPage.clearAllFilters(); + incidentsPage.setDays('7 days'); + incidentsPage.elements.incidentsChartContainer().should('be.visible'); + incidentsPage.elements.incidentsChartBarsGroups().should('have.length', 14); + cy.pause(); // Verify incidents loaded correctly + + cy.log('1.2 Test bottom bar tooltip positioning'); + incidentsPage.elements.incidentsChartBarsVisiblePaths() + .first() + .trigger('mouseover', { force: true }); + + cy.get('[role="tooltip"]').should('be.visible'); + cy.get('[role="tooltip"]').then(($tooltip) => { + const tooltipRect = $tooltip[0].getBoundingClientRect(); + expect(tooltipRect.top).to.be.greaterThan(0); + expect(tooltipRect.left).to.be.greaterThan(0); + }); + cy.pause(); // Verify bottom tooltip positioning + + cy.log('1.3 Test middle bar tooltip and verify content'); + incidentsPage.elements.incidentsChartBarsVisiblePaths() + .eq(7) + .trigger('mouseover', { force: true }); + + cy.get('[role="tooltip"]').should('be.visible'); + cy.get('[role="tooltip"]').should('contain.text', 'Incident'); + cy.pause(); // Verify middle tooltip + + cy.log('1.4 Test top bar tooltip positioning'); + incidentsPage.elements.incidentsChartBarsVisiblePaths() + .last() + .trigger('mouseover', { force: true }); + + cy.get('[role="tooltip"]').should('be.visible'); + cy.get('[role="tooltip"]').then(($tooltip) => { + const tooltipRect = $tooltip[0].getBoundingClientRect(); + const viewportHeight = Cypress.$(window).height(); + expect(tooltipRect.bottom).to.be.lessThan(viewportHeight); + }); + cy.pause(); // Verify top tooltip positioning + + cy.log('1.5 Hover over multi-component incident and verify content'); + incidentsPage.elements.incidentsChartBarsVisiblePaths() + .eq(3) + .trigger('mouseover', { force: true }); + + cy.get('[role="tooltip"]') + .should('be.visible') + .should('contain.text', 'network') + .should('contain.text', 'compute') + .should('contain.text', 'storage'); + cy.pause(); // Verify multi-component tooltip content + + cy.log('1.6 Click incident bar and verify details panel opens'); + incidentsPage.elements.incidentsChartBarsVisiblePaths().eq(3).click(); + incidentsPage.elements.incidentsDetailsPanel().should('be.visible'); + incidentsPage.elements.incidentsDetailsTableRows() + .should('have.length.greaterThan', 0); + cy.pause(); // Verify details panel + + cy.log('Verified: Complete tooltip interaction and navigation workflow'); + }); +}); +``` + +6. Suggest page object additions if needed (e.g., tooltip helpers) + +### Example 2: Generate Filtering Test + +**User Input**: "Generate regression test for Section 1: Filtering Bugs" + +**AI Actions**: +1. Parse Section 1 from testing_flows_ui.md +2. Note: Fixture `7-comprehensive-filtering-test-scenarios.yaml` already exists +3. **Design comprehensive flow**: Instead of separate tests for each filter type, create complete filtering workflows + - Flow 1: User applies multiple filters in sequence, verifies each step, then clears all + - Flow 2: User changes time range while filters are active, verifies data updates +4. Generate `01.reg_filtering.cy.ts` with **comprehensive multi-step tests** +5. Each test should have 5-8 steps covering realistic filter combinations and transitions + +## Output Format + +Provide: +1. **Test file path and name**: Full path to generated test file +2. **Test file content**: Complete TypeScript test file +3. **Fixture status**: + - If existing: "Using fixture: incident-scenarios/X-name.yaml" + - If new: "Created fixture: incident-scenarios/X-name.yaml" + YAML content +4. **Page object changes**: If any elements/methods need to be added, list them with implementation +5. **Validation status**: Confirm all checklist items passed + +## Notes + +- **Follow Cypress e2e/integration testing philosophy**: Tests should cover complete user flows, not isolated units +- **Prefer comprehensive flows**: Generate 1-3 longer tests per file rather than 10+ tiny tests +- **Think user journeys**: Combine related actions (filtering → verification → interaction → results) in single tests +- Tests can be 50-100+ lines if they represent realistic, complete workflows +- Tests should be runnable immediately without manual intervention +- Each test should be independent and self-contained (not dependent on execution order) +- Follow workspace rules: no emojis in logs, sparse comments +- Prefer single scenario per test file for focused regression testing +- Reference the bug tracking number (e.g., "Verifies: OU-XXX") if available in documentation +- If documentation mentions "WARNING Not possible to test", note this in test comments and implement as far as possible +- Use `cy.waitUntil()` for dynamic loading scenarios instead of fixed waits when possible + +## Workflow + +**Recommended workflow**: +1. Use this command to generate initial test from documentation +2. Manually verify the test works (using `cy.pause()` points) +3. Once verified, use `/refactor-regression-test` to clean up and improve code quality + + diff --git a/.cursor/commands/refactor-regression-test.md b/.cursor/commands/refactor-regression-test.md new file mode 100644 index 00000000..f865a7ed --- /dev/null +++ b/.cursor/commands/refactor-regression-test.md @@ -0,0 +1,426 @@ +--- +description: Refactor and clean up existing regression test for improved readability and maintainability +--- + +# Refactor Regression Test + +Refactor an existing regression test to improve code quality, eliminate duplication, and enhance readability. This command should be run after initial test generation and manual verification. + +## Purpose + +After generating and verifying a regression test works correctly, this command: +- Extracts repetitive patterns into helper functions +- Improves test readability (makes `it()` blocks read like user stories) +- Consolidates similar assertions +- Suggests page object additions for reusable functionality +- Ensures compliance with e2e testing best practices + +## Process + +### 1. Analyze Existing Test + +**Input**: Path to test file (e.g., `web/cypress/e2e/incidents/regression/05.reg_tooltip_positioning.cy.ts`) + +**Read and analyze**: +- Test structure and flow +- Repetitive assertion patterns +- Complex multi-step verifications +- Inline calculations or data parsing +- Custom selectors that could be page object methods +- Overall readability of `it()` blocks + +### 2. Identify Refactoring Opportunities + +**Look for**: + +#### Repeated Assertion Patterns +```typescript +// Example: Repeated opacity checks +incidentsPage.elements.alertsChartBarsPaths().eq(0).then(($bar) => { + const opacity = parseFloat($bar.css('opacity') || '1'); + expect(opacity).to.equal(0.3); +}); + +incidentsPage.elements.alertsChartBarsPaths().eq(1).then(($bar) => { + const opacity = parseFloat($bar.css('opacity') || '1'); + expect(opacity).to.equal(1.0); +}); +``` + +#### Complex Multi-Step Verifications +```typescript +// Example: Complex tooltip verification repeated multiple times +incidentsPage.elements.incidentsChartBarsVisiblePaths().eq(0).trigger('mouseover'); +cy.get('[role="tooltip"]').should('be.visible'); +cy.get('[role="tooltip"]').should('contain.text', 'Expected text'); +cy.get('[role="tooltip"]').then(($tooltip) => { + const rect = $tooltip[0].getBoundingClientRect(); + expect(rect.top).to.be.greaterThan(0); +}); +``` + +#### Inline Calculations +```typescript +// Example: Calculations within test body +incidentsPage.elements.component().invoke('text').then((text) => { + const cleaned = text.trim().toLowerCase(); + const parts = cleaned.split(','); + expect(parts).to.have.length(3); +}); +``` + +#### Custom Selectors Used Multiple Times +```typescript +// Example: Direct selector usage instead of page object +cy.get('[role="tooltip"]').should('be.visible'); +cy.get('[role="tooltip"]').should('contain.text', 'Text'); +// Repeated many times in the test +``` + +### 3. Create Helper Functions + +**Guidelines**: +- Place helper functions within the test file (inside or outside `describe()` block) +- Use descriptive names that explain what they verify +- Keep helpers focused on a single responsibility +- Preserve type safety with TypeScript types + +**Helper Function Patterns**: + +#### Simple Assertion Helper +```typescript +const verifyElementProperty = ( + selector: Cypress.Chainable>, + property: string, + expectedValue: any +) => { + selector.then(($el) => { + const value = $el.css(property); + expect(parseFloat(value || '0')).to.equal(expectedValue); + }); +}; +``` + +#### Multi-Step Verification Helper +```typescript +const verifyTooltipContent = (expectedTexts: string[], shouldBeSilenced: boolean = false) => { + const tooltip = cy.get('[role="tooltip"]').should('be.visible'); + expectedTexts.forEach(text => tooltip.should('contain.text', text)); + tooltip.should(shouldBeSilenced ? 'contain.text' : 'not.contain.text', '(silenced)'); +}; +``` + +#### Interaction + Verification Helper +```typescript +const hoverAndVerifyTooltipPosition = (barIndex: number, expectedPosition: 'top' | 'bottom') => { + incidentsPage.elements.incidentsChartBarsVisiblePaths() + .eq(barIndex) + .trigger('mouseover', { force: true }); + + cy.get('[role="tooltip"]').should('be.visible').then(($tooltip) => { + const rect = $tooltip[0].getBoundingClientRect(); + if (expectedPosition === 'top') { + expect(rect.bottom).to.be.lessThan(Cypress.$(window).height()); + } else { + expect(rect.top).to.be.greaterThan(0); + } + }); +}; +``` + +#### Data Processing Helper +```typescript +const parseComponentList = (text: string): string[] => { + return text.trim().split(',').map(s => s.trim()).filter(s => s.length > 0); +}; +``` + +### 4. Refactor Test Body + +**Goal**: The `it()` block should read like a user story, with implementation details hidden in helpers. + +**Before**: +```typescript +it('1. Verify alert opacity and tooltips', () => { + cy.log('1.1 Check first alert opacity'); + incidentsPage.elements.alertsChartBarsPaths().eq(0).then(($bar) => { + const opacity = parseFloat($bar.css('opacity') || '1'); + expect(opacity).to.equal(0.3); + }); + + cy.log('1.2 Check first alert tooltip'); + incidentsPage.elements.alertsChartBarsPaths().eq(0).trigger('mouseover'); + cy.get('[role="tooltip"]').should('be.visible'); + cy.get('[role="tooltip"]').should('contain.text', 'Alert 1'); + cy.get('[role="tooltip"]').should('contain.text', '(silenced)'); + + cy.log('1.3 Check second alert opacity'); + incidentsPage.elements.alertsChartBarsPaths().eq(1).then(($bar) => { + const opacity = parseFloat($bar.css('opacity') || '1'); + expect(opacity).to.equal(1.0); + }); + + cy.log('1.4 Check second alert tooltip'); + incidentsPage.elements.alertsChartBarsPaths().eq(1).trigger('mouseover'); + cy.get('[role="tooltip"]').should('be.visible'); + cy.get('[role="tooltip"]').should('contain.text', 'Alert 2'); + cy.get('[role="tooltip"]').should('not.contain.text', '(silenced)'); +}); +``` + +**After**: +```typescript +const verifyAlertOpacity = (alertIndex: number, expectedOpacity: number) => { + incidentsPage.elements.alertsChartBarsPaths() + .eq(alertIndex) + .then(($bar) => { + const opacity = parseFloat($bar.css('opacity') || '1'); + expect(opacity).to.equal(expectedOpacity); + }); +}; + +const verifyAlertTooltip = (alertIndex: number, expectedTexts: string[], shouldBeSilenced: boolean) => { + incidentsPage.elements.alertsChartBarsPaths().eq(alertIndex).trigger('mouseover'); + const tooltip = cy.get('[role="tooltip"]').should('be.visible'); + expectedTexts.forEach(text => tooltip.should('contain.text', text)); + tooltip.should(shouldBeSilenced ? 'contain.text' : 'not.contain.text', '(silenced)'); +}; + +it('1. Verify alert opacity and tooltips', () => { + cy.log('1.1 Verify silenced alert has reduced opacity and indicator'); + verifyAlertOpacity(0, 0.3); + verifyAlertTooltip(0, ['Alert 1'], true); + + cy.log('1.2 Verify non-silenced alert has full opacity without indicator'); + verifyAlertOpacity(1, 1.0); + verifyAlertTooltip(1, ['Alert 2'], false); + + cy.log('Verified: Alert silence indicators work correctly'); +}); +``` + +### 5. Suggest Page Object Additions + +**When to suggest page object additions**: +- Helper functionality could be reused across multiple test files +- Custom selectors are used repeatedly (e.g., `cy.get('[role="tooltip"]')`) +- Complex interactions that represent common user actions + +**Format suggestion**: +``` +The following functionality could be added to incidents-page.ts for reusability: + +Elements: +- tooltip: () => cy.get('[role="tooltip"]') +- tooltipContent: () => cy.get('[role="tooltip"] .pf-c-tooltip__content') + +Methods: +- hoverOverIncidentBar: (index: number) => { + cy.log('incidentsPage.hoverOverIncidentBar'); + incidentsPage.elements.incidentsChartBarsVisiblePaths() + .eq(index) + .trigger('mouseover', { force: true }); + } + +- verifyTooltipVisible: () => { + cy.log('incidentsPage.verifyTooltipVisible'); + incidentsPage.elements.tooltip().should('be.visible'); + } + +Should I add these to incidents-page.ts? +``` + +### 6. Remove cy.pause() Statements + +**Important**: Only remove `cy.pause()` statements if user explicitly requests it or confirms. + +**When to remove**: +- User says "remove pauses" +- User says "cleanup test" or "finalize test" +- Test has been verified and is working correctly + +**When NOT to remove**: +- User just generated the test (they need to verify first) +- User hasn't confirmed the test works +- Not explicitly requested + +**Process**: +1. Identify all `cy.pause()` statements +2. Check if they're still needed for manual verification +3. If removing, preserve the surrounding assertions +4. Update `cy.log()` messages to reflect completed verification + +**Example removal**: +```typescript +// Before +cy.log('1.1 Verify incidents loaded'); +incidentsPage.elements.incidentsChartBarsGroups().should('have.length', 12); +cy.pause(); // Manual verification point + +// After +cy.log('1.1 Verify incidents loaded'); +incidentsPage.elements.incidentsChartBarsGroups().should('have.length', 12); +``` + +### 7. Ensure E2E Best Practices + +**Verify the refactored test follows**: +- [ ] Tests cover complete user flows, not isolated actions +- [ ] Each `it()` block represents a realistic user journey +- [ ] Test body is readable as a story (implementation details in helpers) +- [ ] Appropriate test count (1-3 comprehensive tests preferred) +- [ ] Tests are independent and self-contained +- [ ] Helper functions have descriptive names +- [ ] No duplicate code patterns +- [ ] Follows workspace rules (no emojis, sparse comments) + +### 8. Output Refactored Test + +**Provide**: +1. **Complete refactored test file**: Full content with helpers and cleaned-up test body +2. **Summary of changes**: + - List of helper functions added + - Number of duplications eliminated + - Readability improvements + - Lines of code change (before/after) +3. **Page object suggestions**: If any (with implementation) +4. **Validation status**: Confirm best practices checklist + +## Example Transformations + +### Example 1: Tooltip Verification + +**Before** (repetitive): +```typescript +it('1. Verify tooltips', () => { + cy.log('1.1 Bottom bar tooltip'); + incidentsPage.elements.incidentsChartBarsVisiblePaths() + .first() + .trigger('mouseover', { force: true }); + cy.get('[role="tooltip"]').should('be.visible'); + cy.get('[role="tooltip"]').then(($tooltip) => { + const rect = $tooltip[0].getBoundingClientRect(); + expect(rect.top).to.be.greaterThan(0); + }); + + cy.log('1.2 Top bar tooltip'); + incidentsPage.elements.incidentsChartBarsVisiblePaths() + .last() + .trigger('mouseover', { force: true }); + cy.get('[role="tooltip"]').should('be.visible'); + cy.get('[role="tooltip"]').then(($tooltip) => { + const rect = $tooltip[0].getBoundingClientRect(); + const viewportHeight = Cypress.$(window).height(); + expect(rect.bottom).to.be.lessThan(viewportHeight); + }); +}); +``` + +**After** (clean): +```typescript +const verifyTooltipPosition = (barIndex: number, position: 'top' | 'bottom') => { + incidentsPage.elements.incidentsChartBarsVisiblePaths() + .eq(barIndex) + .trigger('mouseover', { force: true }); + + cy.get('[role="tooltip"]').should('be.visible').then(($tooltip) => { + const rect = $tooltip[0].getBoundingClientRect(); + if (position === 'bottom') { + expect(rect.top).to.be.greaterThan(0); + } else { + const viewportHeight = Cypress.$(window).height(); + expect(rect.bottom).to.be.lessThan(viewportHeight); + } + }); +}; + +it('1. Verify tooltips', () => { + cy.log('1.1 Verify bottom and top bar tooltip positioning'); + verifyTooltipPosition(0, 'bottom'); + verifyTooltipPosition(-1, 'top'); + cy.log('Verified: Tooltips positioned correctly at all chart positions'); +}); +``` + +### Example 2: Opacity and Tooltip Combined + +**Before** (verbose): +```typescript +it('1. Alert silence indicators', () => { + cy.log('1.1 Check silenced alert'); + incidentsPage.elements.alertsChartBarsPaths().eq(0).then(($bar) => { + const opacity = parseFloat($bar.css('opacity') || '1'); + expect(opacity).to.equal(0.3); + }); + incidentsPage.elements.alertsChartBarsPaths().eq(0).trigger('mouseover'); + cy.get('[role="tooltip"]').should('be.visible'); + cy.get('[role="tooltip"]').should('contain.text', '(silenced)'); + + cy.log('1.2 Check non-silenced alert'); + incidentsPage.elements.alertsChartBarsPaths().eq(1).then(($bar) => { + const opacity = parseFloat($bar.css('opacity') || '1'); + expect(opacity).to.equal(1.0); + }); + incidentsPage.elements.alertsChartBarsPaths().eq(1).trigger('mouseover'); + cy.get('[role="tooltip"]').should('be.visible'); + cy.get('[role="tooltip"]').should('not.contain.text', '(silenced)'); +}); +``` + +**After** (concise): +```typescript +const verifyAlertSilenceIndicator = ( + alertIndex: number, + isSilenced: boolean, + alertName: string +) => { + const expectedOpacity = isSilenced ? 0.3 : 1.0; + + incidentsPage.elements.alertsChartBarsPaths() + .eq(alertIndex) + .then(($bar) => { + const opacity = parseFloat($bar.css('opacity') || '1'); + expect(opacity).to.equal(expectedOpacity); + }); + + incidentsPage.elements.alertsChartBarsPaths() + .eq(alertIndex) + .trigger('mouseover'); + + const tooltip = cy.get('[role="tooltip"]').should('be.visible'); + tooltip.should('contain.text', alertName); + tooltip.should(isSilenced ? 'contain.text' : 'not.contain.text', '(silenced)'); +}; + +it('1. Alert silence indicators', () => { + cy.log('1.1 Verify silence indicators on silenced and non-silenced alerts'); + verifyAlertSilenceIndicator(0, true, 'SilencedAlert'); + verifyAlertSilenceIndicator(1, false, 'ActiveAlert'); + cy.log('Verified: Silence indicators work correctly'); +}); +``` + +## Validation Checklist + +Before outputting refactored test: +- [ ] Helper functions eliminate all significant code duplication +- [ ] Helper functions have descriptive, clear names +- [ ] Test body (`it()` blocks) reads like a user story +- [ ] Complex logic is extracted to helpers +- [ ] Tests still follow e2e philosophy (complete flows) +- [ ] Tests remain independent and self-contained +- [ ] No obvious comments (sparse comments rule) +- [ ] No emojis in logs +- [ ] `cy.pause()` removed only if explicitly requested +- [ ] Page object suggestions identified (if applicable) +- [ ] Code is more maintainable than before + +## Notes + +- **Focus on readability**: The primary goal is making tests easier to understand and maintain +- **Preserve test behavior**: Refactoring should not change what the test verifies +- **Don't over-abstract**: Only extract patterns that appear 2+ times +- **Keep helpers simple**: Each helper should have a single, clear purpose +- **Test-specific vs. reusable**: Keep test-specific helpers in test file, suggest page object additions for reusable functionality +- **Respect user's verification process**: Don't remove `cy.pause()` unless explicitly asked + diff --git a/.cursor/rules/incidents-testing-guidelines.mdc b/.cursor/rules/incidents-testing-guidelines.mdc new file mode 100644 index 00000000..8b8f01eb --- /dev/null +++ b/.cursor/rules/incidents-testing-guidelines.mdc @@ -0,0 +1,558 @@ +--- +alwaysApply: true +description: "Development guidelines for Incidents page Cypress tests" +globs: + - "web/cypress/e2e/incidents/**/*.cy.ts" + - "web/cypress/views/incidents-page.ts" + - "web/cypress/fixtures/incident-scenarios/**/*.yaml" +--- + +# Incidents Testing Development Guidelines + +Guidelines for developing and maintaining Cypress tests for the Incidents page, including regression tests, page object patterns, and fixture management. + +## Page Object Model (POM) Usage + +### Priority Order for Selectors +1. **First**: Use existing `incidentsPage.elements.*` selectors +2. **Second**: Use existing `incidentsPage.*` methods +3. **Third**: Suggest additions to page object (ask user before implementing) +4. **Last Resort**: Use custom selectors (with explanation) + +### When to Suggest Page Object Additions +If a test needs an element or action not in `incidents-page.ts`: +- List the missing elements/methods +- Show proposed implementation following existing patterns +- Ask: "Should I add these to incidents-page.ts?" +- Wait for user approval before modifying page object + +### Page Object Element Patterns +```typescript +// Simple element selector +elementName: () => cy.byTestID(DataTestIDs.Component.Element) + +// Parameterized element selector +elementWithParam: (param: string) => + cy.byTestID(`${DataTestIDs.Component.Element}-${param.toLowerCase()}`) + +// Composite selector (when data-test not available) +compositeSelector: () => + incidentsPage.elements.toolbar().contains('span', 'Category').parent() +``` + +### Page Object Method Patterns +```typescript +// Action method +actionName: (param: Type) => { + cy.log('incidentsPage.actionName'); + incidentsPage.elements.something().click(); +} + +// Query method returning Chainable +getData: (): Cypress.Chainable => { + cy.log('incidentsPage.getData'); + return incidentsPage.elements.container() + .invoke('text') + .then((text) => cy.wrap(processData(text))); +} +``` + +## Test Structure and Organization + +### E2E Testing Philosophy + +**Why fewer, longer tests?** +- Faster overall execution (less setup/teardown overhead) +- Better reflects real user behavior +- Easier to understand user journeys +- Reduces test interdependencies +- More valuable failure signals + + +**Follow Cypress best practices for e2e/integration testing**: +- Tests should cover **complete user flows**, not isolated units +- Prefer **fewer, longer tests** over many small tests +- Each `it()` block should test a **realistic user journey** with multiple steps +- Unlike unit tests, e2e tests should combine related actions and verifications +- Test how users actually interact with the application end-to-end + +**Examples of good e2e test flows**: +- User navigates → filters data → verifies results → changes filter → verifies new results +- User loads page → interacts with chart → opens details → verifies content → performs actions +- Complete workflow from initial state through multiple user interactions to final verification + +**Avoid**: +- Splitting every small action into a separate `it()` block +- Testing each button click or filter in isolation +- Creating many tiny tests that don't reflect real usage + +### File Naming Convention +- Location: `web/cypress/e2e/incidents/regression/` +- Pattern: `XX.reg_.cy.ts` +- Examples: `05.reg_tooltip_positioning.cy.ts`, `01.reg_filtering.cy.ts` + +### Test File Structure +```typescript +/* +Brief description of what this test verifies. +Additional context about the bug or behavior. +Verifies: OU-XXX +*/ + +import { incidentsPage } from '../../../views/incidents-page'; + +const MCP = { + namespace: 'openshift-cluster-observability-operator', + packageName: 'cluster-observability-operator', + operatorName: 'Cluster Observability Operator', + config: { + kind: 'UIPlugin', + name: 'monitoring', + }, +}; + +const MP = { + namespace: 'openshift-monitoring', + operatorName: 'Cluster Monitoring Operator', +}; + +describe('Regression:
', () => { + before(() => { + cy.beforeBlockCOO(MCP, MP); + }); + + beforeEach(() => { + cy.log('Navigate to Observe → Incidents'); + incidentsPage.goTo(); + cy.log('Brief description of scenario'); + cy.mockIncidentFixture('incident-scenarios/XX-name.yaml'); + }); + + it('1. Test case description', () => { + cy.log('1.1 Step description'); + incidentsPage.clearAllFilters(); + // Test assertions + cy.log('Verified: Expected outcome'); + }); +}); +``` + +### Required Elements +- File header comment with purpose and issue reference (e.g., "Verifies: OU-XXX") +- Import `incidentsPage` from relative path +- Standard MCP/MP configuration blocks +- Use `cy.beforeBlockCOO(MCP, MP)` in `before()` hook +- Use `incidentsPage.goTo()` in `beforeEach()` +- Use `cy.mockIncidentFixture()` for test data + +## Test Assertions + +### Two-Phase Approach for cy.pause() +- **For new test files**: Include `cy.pause()` statements after key setup/verification steps for manual verification +- **For new test cases**: Include `cy.pause()` when adding to existing files +- **For existing test cases**: DO NOT reintroduce `cy.pause()` if the user has already removed them +- **Rule**: Check if test already has assertions without pauses - preserve that state +- **Purpose**: Initial pauses allow manual verification, then user deletes them once confident + +### Use Automated Assertions +- **NO** `VERIFY:` comments in place of assertions +- Convert all manual verification steps to automated assertions +- Include `cy.pause()` only in initial test generation for manual verification + +### Common Assertion Patterns + +#### Visibility and Existence +```typescript +incidentsPage.elements.incidentsChartContainer().should('be.visible'); +incidentsPage.elements.incidentsTable().should('not.exist'); +``` + +#### Counts and Length +```typescript +incidentsPage.elements.incidentsChartBarsGroups().should('have.length', 12); +incidentsPage.elements.incidentsDetailsTableRows() + .should('have.length.greaterThan', 0); +``` + +#### Text Content +```typescript +incidentsPage.elements.daysSelectToggle().should('contain.text', '7 days'); +incidentsPage.elements.incidentsTableComponentCell(0) + .invoke('text') + .should('contain', 'monitoring'); +``` + +#### Conditional Waiting +```typescript +cy.waitUntil( + () => incidentsPage.elements.incidentsChartBarsGroups() + .then($groups => $groups.length === 12), + { + timeout: 10000, + interval: 500, + errorMsg: 'All 12 incidents should load within 10 seconds' + } +); +``` + +#### Position and Layout +```typescript +incidentsPage.elements.incidentsChartBarsVisiblePaths() + .first() + .then(($element) => { + const rect = $element[0].getBoundingClientRect(); + expect(rect.width).to.be.greaterThan(5); + }); +``` + +#### Tooltip Interactions +```typescript +incidentsPage.elements.incidentsChartBarsVisiblePaths() + .first() + .trigger('mouseover', { force: true }); + +cy.get('[role="tooltip"]').should('be.visible'); +cy.get('[role="tooltip"]').should('contain.text', 'Expected content'); +``` + +### Descriptive Logging +Use `cy.log()` for test flow clarity: +```typescript +cy.log('1.1 Verify all incidents loaded'); +incidentsPage.elements.incidentsChartBarsGroups().should('have.length', 12); +cy.pause(); // Manual verification point (for new tests) +cy.log('Verified: 12 incidents shown'); +``` + +### cy.pause() Best Practices +```typescript +// NEW test - include pauses for manual verification +it('1. New test case', () => { + cy.log('1.1 Setup'); + incidentsPage.clearAllFilters(); + incidentsPage.elements.incidentsChartContainer().should('be.visible'); + cy.pause(); // Manual verification + + cy.log('1.2 Verify behavior'); + // more assertions + cy.pause(); // Manual verification +}); + +// EXISTING test without pauses - DO NOT reintroduce them +it('2. Existing test', () => { + cy.log('2.1 Setup'); + incidentsPage.clearAllFilters(); + incidentsPage.elements.incidentsChartContainer().should('be.visible'); + // No pause - user has already verified and removed it + + cy.log('2.2 Verify behavior'); + // assertions without pause +}); +``` + +## Fixture Management + +### Fixture Location and Naming +- Location: `web/cypress/fixtures/incident-scenarios/` +- Pattern: `XX-descriptive-name.yaml` +- Examples: `13-tooltip-positioning-scenarios.yaml` + +### Fixture Usage in Tests +```typescript +// Preferred: Single scenario per test file +cy.mockIncidentFixture('incident-scenarios/13-tooltip-positioning-scenarios.yaml'); + +// For empty state +cy.mockIncidents([]); +``` + +### Creating Fixtures +- Use `generate-incident-fixture` command for new fixtures +- Follow schema from `web/cypress/support/incidents_prometheus_query_mocks/schema/fixture-schema.json` +- Validate fixtures before committing +- Prefer single scenario per test file for focused regression testing + +### Fixture Schema Compliance +- Use relative durations (e.g., `"2h"`, `"30m"`) not absolute timestamps +- Use valid component names: `monitoring`, `storage`, `network`, `compute`, `api-server`, `etcd`, `version`, `Others` +- Use valid layers: `core`, `Others` +- Use valid severities: `critical`, `warning`, `info` +- Include descriptive scenario name and description + +## Code Quality Standards + +### Test Organization +- **Prefer comprehensive flows**: Each `it()` should test a complete user journey +- **Combine related steps**: Don't split filtering, verification, interaction into separate tests +- **Think e2e, not unit**: Test realistic multi-step workflows, not isolated actions +- **Limit `it()` blocks**: Prefer 1-3 comprehensive tests over 10+ tiny tests per `describe()` +- Tests can be longer (50-100+ lines) if they test a complete, realistic workflow + +### Test Clarity +- Use numbered test cases with descriptive names +- Use sub-step numbering in logs (e.g., "1.1", "1.2", "1.3", etc.) +- Group related assertions logically +- Each test should tell a complete story of user interaction + +### Refactoring for Readability +- **Extract helper functions for duplicated logic**: If you repeat the same action/assertion pattern, create a helper function within the test file +- **Keep `it()` blocks readable as a story**: The test body should read like steps a user takes, not implementation details +- **Move complex logic to helpers or page object**: Multi-step verifications, calculations, or repetitive patterns belong in functions +- **Helper function placement**: + - Test-specific helpers: Define within the test file (inside or outside `it()`) + - Reusable helpers: Add to `incidents-page.ts` page object + +### Naming Conventions +- Test descriptions: Clear, specific, behavior-focused +- Variables: Descriptive, follow TypeScript conventions +- Constants: UPPER_CASE for configuration values + +### Imports +```typescript +// Always import from relative path +import { incidentsPage } from '../../../views/incidents-page'; + +// Import types when needed +import { IncidentDefinition } from '../../support/incidents_prometheus_query_mocks'; +``` + +### Comments +- Follow sparse comments rule (explain "why", not "what") +- Add file header comments explaining test purpose +- Document workarounds or known issues +- Reference issue numbers (e.g., "Verifies: OU-XXX") + +### Logging +- Follow no-emojis rule (no emojis in cy.log() statements) +- Use clear, descriptive text +- Log test flow for debugging + +## Testing Best Practices + +### Test Independence +- Tests should be self-contained +- Should not depend on execution order +- Use `beforeEach()` to set up clean state + +### Performance +- Use `cy.waitUntil()` for dynamic loading instead of fixed `cy.wait()` +- Only wait when necessary +- Use appropriate timeouts + +### Maintainability +- Keep tests DRY (use page object methods) +- Follow established patterns +- Make changes to page object for reusable functionality +- Update page object when UI changes + +### Documentation References +- Reference test documentation in [`docs/incident_detection/tests/`](../../docs/incident_detection/tests/) +- Link to TESTING_CHECKLIST.md sections when applicable +- Reference bug tracking issues in test files + +## Validation Checklist + +Before submitting tests, verify: +- [ ] **Tests follow e2e philosophy**: Each `it()` covers a complete user flow, not isolated actions +- [ ] **Appropriate test count**: Prefer 1-3 comprehensive tests over many small tests +- [ ] **Tests are independent**: Each test is self-contained and doesn't depend on others +- [ ] **Test reads like a story**: Refactor duplications into helper functions, keep `it()` body readable +- [ ] Test file follows naming convention +- [ ] Uses existing page object elements/methods (or suggests additions) +- [ ] **For new tests**: Include `cy.pause()` after key verification points +- [ ] **For existing tests**: Preserve pause/no-pause state (don't reintroduce) +- [ ] NO `VERIFY:` comments in place of assertions +- [ ] Fixture validated against schema +- [ ] Standard imports and setup blocks +- [ ] Descriptive test names and log statements +- [ ] No emojis in logs +- [ ] Minimal, value-adding comments +- [ ] File header includes purpose and issue reference + +## Examples + +### Good Test Example (Comprehensive E2E Flow) +```typescript +it('1. Complete filtering workflow - severity, component, and time range', () => { + cy.log('1.1 Verify initial state with all incidents'); + incidentsPage.clearAllFilters(); + incidentsPage.elements.incidentsChartBarsGroups().should('have.length', 12); + incidentsPage.elements.incidentsChartContainer().should('be.visible'); + cy.pause(); // Manual verification point + + cy.log('1.2 Apply Critical severity filter and verify results'); + incidentsPage.toggleFilter('Critical'); + incidentsPage.elements.severityFilterChip().should('be.visible'); + incidentsPage.elements.incidentsChartBarsGroups().should('have.length', 7); + cy.pause(); // Manual verification point + + cy.log('1.3 Add component filter and verify combined filtering'); + incidentsPage.selectComponent('monitoring'); + incidentsPage.elements.componentFilterChip().should('be.visible'); + incidentsPage.elements.incidentsChartBarsGroups().should('have.length', 4); + cy.pause(); // Manual verification point + + cy.log('1.4 Change time range and verify filtered data updates'); + incidentsPage.setDays('30 days'); + incidentsPage.elements.daysSelectToggle().should('contain.text', '30 days'); + incidentsPage.elements.incidentsChartBarsGroups().should('have.length.greaterThan', 4); + cy.pause(); // Manual verification point + + cy.log('1.5 Click incident bar and verify details panel'); + incidentsPage.elements.incidentsChartBarsVisiblePaths().first().click(); + incidentsPage.elements.incidentsDetailsPanel().should('be.visible'); + incidentsPage.elements.incidentsDetailsTableRows().should('have.length.greaterThan', 0); + cy.pause(); // Manual verification point + + cy.log('1.6 Clear all filters and verify return to initial state'); + incidentsPage.clearAllFilters(); + incidentsPage.elements.severityFilterChip().should('not.exist'); + incidentsPage.elements.componentFilterChip().should('not.exist'); + incidentsPage.elements.incidentsChartBarsGroups().should('have.length', 12); + cy.log('Verified: Complete filtering workflow'); +}); +``` + +### Good Test Example (After User Removed Pauses) +```typescript +it('1. Complete filtering workflow - severity, component, and time range', () => { + cy.log('1.1 Verify initial state with all incidents'); + incidentsPage.clearAllFilters(); + incidentsPage.elements.incidentsChartBarsGroups().should('have.length', 12); + incidentsPage.elements.incidentsChartContainer().should('be.visible'); + + cy.log('1.2 Apply Critical severity filter and verify results'); + incidentsPage.toggleFilter('Critical'); + incidentsPage.elements.severityFilterChip().should('be.visible'); + incidentsPage.elements.incidentsChartBarsGroups().should('have.length', 7); + + cy.log('1.3 Add component filter and verify combined filtering'); + incidentsPage.selectComponent('monitoring'); + incidentsPage.elements.componentFilterChip().should('be.visible'); + incidentsPage.elements.incidentsChartBarsGroups().should('have.length', 4); + + cy.log('1.4 Change time range and verify filtered data updates'); + incidentsPage.setDays('30 days'); + incidentsPage.elements.daysSelectToggle().should('contain.text', '30 days'); + incidentsPage.elements.incidentsChartBarsGroups().should('have.length.greaterThan', 4); + + cy.log('1.5 Click incident bar and verify details panel'); + incidentsPage.elements.incidentsChartBarsVisiblePaths().first().click(); + incidentsPage.elements.incidentsDetailsPanel().should('be.visible'); + incidentsPage.elements.incidentsDetailsTableRows().should('have.length.greaterThan', 0); + + cy.log('1.6 Clear all filters and verify return to initial state'); + incidentsPage.clearAllFilters(); + incidentsPage.elements.severityFilterChip().should('not.exist'); + incidentsPage.elements.componentFilterChip().should('not.exist'); + incidentsPage.elements.incidentsChartBarsGroups().should('have.length', 12); + cy.log('Verified: Complete filtering workflow'); +}); +``` + +### Good Test Example (With Helper Functions) +```typescript +it('1. Silence matching verification flow - opacity and tooltip indicators', () => { + const verifyAlertOpacity = (alertIndex: number, expectedOpacity: number) => { + incidentsPage.elements.alertsChartBarsPaths() + .eq(alertIndex) + .then(($bar) => { + const opacity = parseFloat($bar.css('opacity') || '1'); + expect(opacity).to.equal(expectedOpacity); + }); + }; + + const verifyAlertTooltip = (alertIndex: number, expectedTexts: string[], shouldBeSilenced: boolean) => { + incidentsPage.hoverOverAlertBar(alertIndex); + const tooltip = incidentsPage.elements.alertsChartTooltip().should('be.visible'); + expectedTexts.forEach(text => tooltip.should('contain.text', text)); + tooltip.should(shouldBeSilenced ? 'contain.text' : 'not.contain.text', '(silenced)'); + }; + + cy.log('1.1 Select silenced alert incident'); + incidentsPage.elements.incidentsChartBar('PAIR-2-storage-SILENCED').click(); + incidentsPage.elements.alertsChartContainer().should('be.visible'); + + cy.log('1.2 Verify silenced alert has reduced opacity and tooltip indicator'); + verifyAlertOpacity(0, 0.3); + verifyAlertTooltip(0, ['SyntheticSharedFiring002'], true); + + cy.log('2.1 Select non-silenced alert with same name'); + incidentsPage.elements.incidentsChartBar('PAIR-2-network-UNSILENCED').click(); + + cy.log('2.2 Verify non-silenced alert has full opacity without indicator'); + verifyAlertOpacity(0, 1.0); + verifyAlertTooltip(0, ['SyntheticSharedFiring002'], false); + + cy.log('Verified: Silence matching uses alertname + namespace + severity'); +}); +``` + +**Why this is good**: +- Helper functions extract repeated verification patterns +- Test body reads like a user story (select → verify → select → verify) +- Complex opacity/tooltip logic hidden in helpers +- Easy to understand the test flow at a glance + +### Bad Test Examples + +#### Example 1: Too Many Small Tests (Unit Test Mindset) +```typescript +describe('Regression: Filtering', () => { + it('1. Should clear filters', () => { + incidentsPage.clearAllFilters(); + incidentsPage.elements.severityFilterChip().should('not.exist'); + }); + + it('2. Should apply Critical filter', () => { + incidentsPage.toggleFilter('Critical'); + incidentsPage.elements.severityFilterChip().should('be.visible'); + }); + + it('3. Should show correct count after filtering', () => { + incidentsPage.elements.incidentsChartBarsGroups().should('have.length', 7); + }); + + it('4. Should apply component filter', () => { + incidentsPage.selectComponent('monitoring'); + }); + + it('5. Should show combined filter results', () => { + incidentsPage.elements.incidentsChartBarsGroups().should('have.length', 4); + }); +}); +``` + +**Issues**: +- Splitting a single user flow into many tiny tests +- Tests depend on each other (not independent) +- Doesn't reflect how users actually interact with the app +- Unit test mindset applied to e2e testing +- More setup overhead, slower test execution + +#### Example 2: Poor Code Quality +```typescript +it('Test filtering', () => { + // Clear the filters + cy.get('[data-test="toolbar"]').find('button').contains('Clear').click(); + cy.pause(); // VERIFY: Filters are cleared + + // Select critical + cy.get('[data-test="filters-select-toggle"]').click(); + cy.pause(); // VERIFY: Menu opened +}); +``` + +**Issues**: +- Not using page object methods +- Using custom selectors instead of page object +- Using cy.pause() WITHOUT assertions (should have both) +- Obvious comments that don't add value +- No descriptive logging +- No verification of expected outcomes +- Too short, doesn't test a complete flow + +**Important**: +- `cy.pause()` should be used WITH assertions for manual verification, not INSTEAD of assertions +- Tests should cover complete user journeys, not isolated button clicks +- Use page object methods for maintainability diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 94447ed2..00000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "web/cypress/fixtures/incidents/test-docs"] - path = web/cypress/fixtures/incidents/test-docs - url = git@gitlab.cee.redhat.com:drajnoha/cat-test-docs.git diff --git a/docs/incident_detection/tests/0.overview.md b/docs/incident_detection/tests/0.overview.md new file mode 100644 index 00000000..fb418a60 --- /dev/null +++ b/docs/incident_detection/tests/0.overview.md @@ -0,0 +1,74 @@ +# Incidents Page Testing Checklist + +This checklist covers regression testing for **known bug areas** in the Incidents page. Focus on sections 1-4 for critical regressions. Section 5 contains optional tests for stable areas. + +**Incident Detection Known Bug Areas**: +- **Filtering**: Incidents were filtered by alert severity instead of incident's own severity history +- **Charts Display**: Tooltip positioning, bar sorting, and short alert visibility issues +- **API calls**: Alert resolution timing, silence matching logic (name+namespace+severity not just name) +- **Redux State Management**: Initialization race conditions, selection persistence, stale data, dropdown states + + + + +## How to Use This Checklist + +1. **Set up test data first**: + - Copy the CSV from the "Complete Test Data" section below + - Save it to a file and use with the simulation script from the `cluster-health-analyzer` repository. +2. **Identify relevant areas**: Based on the scope of the change, decide on areas that need to be targeted. +3. **Run the tests**: Follow the test cases in order, checking expected vs actual behavior +4. **Reference specific incidents**: Tests reference incidents by letter (A, B, C, etc.) + +**Time notation**: All times are positive values in minutes. The simulation script adjusts them to be relative to "now". + +--- + +## Complete Test Data - CSV Format + +**Use this complete CSV with your simulation script to set up all test data at once:** + +```csv +start,end,alertname,namespace,severity,silenced,labels +0,180,AlertA_Info,A-openshift-logging,info,false,{"component": "logging"} +240,360,AlertB_Warning,B-openshift-storage,warning,false,{"component": "storage"} +420,420,AlertC_ShortDuration,C-openshift-apiserver,warning,false,{"component": "api-server"} +480,780,AlertD_Info,D-openshift-monitoring,info,false,{"component": "monitoring"} +540,780,AlertD_Warning,D-openshift-monitoring,warning,false,{"component": "monitoring"} +600,780,AlertD_Critical,D-openshift-monitoring,critical,false,{"component": "monitoring"} +840,1080,AlertE_Etcd,-Eopenshift-etcd,critical,false,{"component": "etcd"} +840,1080,AlertE_KubeAPI,E-openshift-kube-apiserver,critical,false,{"component": "kube-apiserver"} +840,1080,AlertE_Controller_Very_Very_Very_Very_Long_Name_Alert,E-openshift-kube-controller,critical,false,{"component": "kube-controller"} +1140,1260,AlertF_KubePodCrashLooping,F-openshift-monitoring,warning,false,{"component": "monitoring"} +1200,1380,AlertF_HighMemoryUsage,F-openshift-monitoring,critical,false,{"component": "monitoring"} +1440,1500,AlertG_APIServerLatency,G-openshift-kube-apiserver,warning,false,{"component": "kube-apiserver"} +1560,1740,AlertH_Critical,H-openshift-network,critical,false,{"component": "network"} +1800,1980,AlertI_KubePodNotReady,I-openshift-operators,warning,true,{"component": "operators"} +2040,2220,AlertJ_KubePodNotReady,J-openshift-storage,warning,false,{"component": "storage"} +``` + +**What this creates** (incidents named A-J in chronological order): +- **Incident A** (0-180 min / 3 hrs): Info only, resolved - logging component +- **Incident B** (240-360 / 2 hrs): Warning only, resolved - storage component +- **Incident C** (420): Single data point, short duration - api-server component +- **Incident D** (480-780 / 5 hrs): Multi-severity transition (Info→Warning→Critical), firing - monitoring component +- **Incident E** (840-1080 / 4 hrs): Multi-component (3 alerts), resolved - etcd/kube-apiserver/kube-controller +- **Incident F** (1140-1380 / 4 hrs): Resolution testing (2 overlapping alerts) - monitoring component +- **Incident G** (1440-1500 / 1 hr): Short duration alert, resolved - kube-apiserver component +- **Incident H** (1560-1740 / 3 hrs): Critical, firing - network component +- **Incident I** (1800-1980 / 3 hrs): Silenced alert - operators component +- **Incident J** (2040-2220 / 3 hrs): NOT silenced (different namespace) - storage component + +**Timeline** (37 hours total, NO overlaps between incidents, 60 min gaps): +``` +0──────180──240───360──420──480──────780──840────1080──1140────1380──1440─1500──1560───1740──1800───1980──2040───2220 + A (3hr) │ B(2hr) │ C │ D (5hr multi) │ E (4hr x3) │ F (4hr test) │ G(1hr) │ H(3hr) │ I(3hr) │ J(3hr) + 60min gap 60 60min gap 60min gap 60min gap 60 60min 60min 60min +``` + +**Format notes**: +- All times in minutes, positive values (simulation script adjusts to relative times) +- Alert names A-J match chronological order (A fires first, J fires last) +- NO overlaps between different incidents +- Alerts within same incident DO overlap (D has 3 overlapping alerts, E has 3 simultaneous, F has 2 overlapping) +- Long durations (1-5 hours) and large gaps (1 hour) ensure no unintended grouping diff --git a/docs/incident_detection/tests/1.filtering_flows.md b/docs/incident_detection/tests/1.filtering_flows.md new file mode 100644 index 00000000..1d68391e --- /dev/null +++ b/docs/incident_detection/tests/1.filtering_flows.md @@ -0,0 +1,59 @@ +## 1. CRITICAL: Filtering Bugs + +**Automation Status**: AUTOMATED in `01.reg_filtering.cy.ts` + +### Prerequisites: Test Data Setup for Filtering Tests + +**CSV Format** - Use this with your simulation script (these create incidents A, B, D, H): + +```csv +start,end,alertname,namespace,severity,silenced,labels +0,180,AlertA_Info,openshift-logging,info,false,{"component": "logging"} +240,360,AlertB_Warning,openshift-storage,warning,false,{"component": "storage"} +480,780,AlertD_Info,openshift-monitoring,info,false,{"component": "monitoring"} +540,780,AlertD_Warning,openshift-monitoring,warning,false,{"component": "monitoring"} +600,780,AlertD_Critical,openshift-monitoring,critical,false,{"component": "monitoring"} +1560,1740,AlertH_Critical,openshift-network,critical,false,{"component": "network"} +``` + +**Quick Reference**: +| Incident | Component | Severity History | State | Time Range | +|----------|-----------|------------------|-------|------------| +| A | logging | Info | Resolved | 0-180 | +| B | storage | Warning | Resolved | 240-360 | +| D | monitoring | Info→Warning→Critical | Firing | 480-780 | +| H | network | Critical | Firing | 1560-1740 | + +### 1.1 Incident Severity Filtering (Not Alert Severity) +**BUG**: Incidents were being filtered by underlying alert severities instead of the incident's own severity history. + +- [ ] **Filter by "Critical"**: + +- [ ] **Filter by "Warning"**: + +- [ ] **Filter by "Informative"**: + +- [ ] **Multiple Severity Filters (e.g., Critical + Warning)**: + +For each, ensure that the correct incidents are shown. The particular numbers may be slightly off if additional alerts are firing in the cluster. + +### 1.2 Resolved Incident Filter Not Working +**BUG**: "Resolved" state filter wasn't working correctly. + +- [ ] **Filter by "Resolved"**: +- [ ] **Filter by "Firing"**: +- [ ] **Verify Resolution Logic**: + - Firing: `currentTime - lastTimestamp <= 10 minutes` + - Resolved: `currentTime - lastTimestamp > 10 minutes` + - Check Incidents that are resolved have last activity > 10 min ago + +### 1.3 Combined Filtering (AND Logic Between Categories) +- [ ] **Critical + Resolved**: +- [ ] **Warning + Resolved**: +- [ ] **Critical + Firing**: + + +- [ ] **Filter Persistence on URL**: Apply filters, refresh page + - Apply: Warning + Resolved + - Check URL: `?days=7+days&severity=Warning&state=Resolved` + - Refresh page → verify Incident B still shown \ No newline at end of file diff --git a/docs/incident_detection/tests/2.ui_display_flows.md b/docs/incident_detection/tests/2.ui_display_flows.md new file mode 100644 index 00000000..c4f45cb3 --- /dev/null +++ b/docs/incident_detection/tests/2.ui_display_flows.md @@ -0,0 +1,117 @@ +## 2. CRITICAL: Charts – UI Bugs + +**Automation Status**: AUTOMATED in `02.reg_ui_charts_comprehensive.cy.ts` apart from 2.3.1 and 2.4 +- Uses fixture: `incident-scenarios/12-charts-ui-comprehensive.yaml` +- Covers: Tooltip positioning, bar sorting & visibility, date/time display +- Verifies: Incidents chart, alerts chart, multi-component tooltips, long alert names + +### Prerequisites: Test Data Setup for Chart Tests + +**CSV Format** - Add these to the incidents from Section 1 (these create incidents C, D, E): + +```csv +start,end,alertname,namespace,severity,silenced,labels +420,420,AlertC_ShortDuration,openshift-apiserver,warning,false,{"component": "api-server"} +480,780,AlertD_Info,openshift-monitoring,info,false,{"component": "monitoring"} +540,780,AlertD_Warning,openshift-monitoring,warning,false,{"component": "monitoring"} +600,780,AlertD_Critical,openshift-monitoring,critical,false,{"component": "monitoring"} +840,1080,AlertE_Etcd,openshift-etcd,critical,false,{"component": "etcd"} +840,1080,AlertE_KubeAPI,openshift-kube-apiserver,critical,false,{"component": "kube-apiserver"} +840,1080,AlertE_Controller_Very_Very_Very_Very_Long_Name_Alert,openshift-kube-controller,critical,false,{"component": "kube-controller"} +``` + +**Quick Reference** (Charts Test Data): +| Incident | Components | Duration | Severity | Time Range | Use For Testing | +|----------|------------|----------|----------|------------|-----------------| +| C | api-server | 0 min | Warning | 420 | Short duration visibility | +| D | monitoring | 5 hrs | Multi-severity (Info→Warning→Critical) | 480-780 | Multi-severity segments | +| E | etcd, kube-apiserver, kube-controller | 4 hrs | Critical | 840-1080 | Multi-component tooltip, long names | + +### 2.1 Tooltip Positioning Issues +**BUG**: Tooltips were overlapping bars or going off-screen. +**Automation Status**: AUTOMATED + +- [ ] **Tooltip Positioning - Incidents Chart**: + - Hover over Incident A (oldest, at bottom) + - Hover over Incident H (one of newest, near top) + - Hover over Incident D (middle position) + - Verify tooltip appears directly above/on each bar without overlap + +- [ ] **Tooltip Content - Multi-Component**: Hover over Incident E + - Verify shows: "Component(s): etcd, kube-apiserver, kube-controller" + - Verify comma-separated list format + - Test with long alert name (Controller_Very_Very_Very_Very_Long_Name_Alert) + +- [ ] **Tooltip Content - Firing vs Resolved**: + - Hover over Incident D (firing): End should show "---" + - Hover over Incident A (resolved): End should show actual end time + +- [ ] **Tooltip Positioning - Alerts Chart** + - Select Incident E (multi-component) + - Hover over all 3 alerts in the incident + - Verify Tooltip appears directly above each bar + - Verify alert name length does not influence the behaviour (test long controller name) + +### 2.2 Bar Sorting & Visibility Issues +**BUGS**: Bars not sorted by start date, filtered bars not leaving space, short alerts not visible. +**Automation Status**: AUTOMATED + +- [ ] **Bar Sorting by Start Date**: Check incidents chart Y-axis order + - Expected order (oldest at bottom): + 1. Incident A (0-180) - bottom + 2. Incident B (240-360) + 3. Incident C (420) + 4. Incident D (480-780) + 5. Incident E (840-1080) + 6. Incident F (1140-1380) + 7. Incident G (1440-1500) + 8. Incident H (1560-1740) + 9. Incident I (1800-1980) + 10. Incident J (2040-2220) - top + - Verify this order maintained after applying filters + +- [ ] **Filtered Bars Leave Empty Space**: Apply "Critical" filter + - Expected visible: D, E, H (Critical incidents) + - Expected hidden but space preserved: A, B, C, F, G, I, J + - Verify gaps appear where non-Critical incidents would be + - Verify Y-axis still shows all 10 positions + +- [ ] **Short Duration Incidents Visible**: Check Incident C (single point at 420) + - Verify bar IS visible despite 0-minute duration + - Hover to confirm tooltip shows Incident C + - Verify bar has minimum visible width + +### 2.3 Date/Time Display Issues +**BUGS**: Start/End times not displaying correctly, date format not respecting language. +**Automation Status**: AUTOMATED + +- [ ] **Start/End Times Correct**: Check specific incidents + - Incident D (firing): Start = T-480min, End = "---" + - Incident A (resolved): Start = T-0min, End = T-180min (both shown) + - Incident C (short, resolved): Start = T-420min, End = T-420min + - Verify tooltip shows these times correctly + +- [ ] **Multi-Severity Segments**: Check Incident D (Info→Warning→Critical) + - Verify each severity segment shows correct time range: + - Info segment: T-480min to T-540min + - Warning segment: T-540min to T-600min + - Critical segment: T-600min to "---" + +- [ ] **Date Format Respects Language**: + - Set browser/app language to English + - Check Incident A tooltip: should show "Jan 15, 2025, 3:45 PM" format + - Switch to Chinese (if available) and verify format changes + - Verify `dateTimeFormatter(i18n.language)` respects setting + +### 2.3.1 (Not Automated) +- [ ] **Date Format Changed Immediately** (xfail): + - Change the app language to Spanish + - Check the "last updated date" field + - Verify that the format changes without the need to reload the page + +### 2.4 Silences labels (Not Automated) +- Verify that information about silences is contained in the alert name + as `NetworkLatencyHigh (silenced)` instead of the additional `silenced=true` + field + + diff --git a/docs/incident_detection/tests/3.api_calls_data_loading_flows.md b/docs/incident_detection/tests/3.api_calls_data_loading_flows.md new file mode 100644 index 00000000..1ae58d35 --- /dev/null +++ b/docs/incident_detection/tests/3.api_calls_data_loading_flows.md @@ -0,0 +1,94 @@ +## 3. CRITICAL: Data Loading – API Call Bugs + +**Automation Status**: PARTIALLY AUTOMATED (Sections 3.1 and 3.2) + +### Prerequisites: Test Data Setup for Data Loading Tests + +**CSV Format** - These alerts test resolution, short duration, and silence logic (creates incidents F, G, I, J): + +```csv +start,end,alertname,namespace,severity,silenced,labels +1140,1260,AlertF_KubePodCrashLooping,openshift-monitoring,warning,false,{"component": "monitoring"} +1200,1380,AlertF_HighMemoryUsage,openshift-monitoring,critical,false,{"component": "monitoring"} +1440,1500,AlertG_APIServerLatency,openshift-kube-apiserver,warning,false,{"component": "kube-apiserver"} +1800,1980,AlertI_KubePodNotReady,openshift-operators,warning,true,{"component": "operators"} +2040,2220,AlertJ_KubePodNotReady,openshift-storage,warning,false,{"component": "storage"} +``` + +**Silence Matching Logic**: +- Silence determined by `silenced` field in CSV (becomes label in `cluster_health_components_map` metric) +- Incidents I and J have same `alertname` but different `namespace` and different `silenced` values +- Tests that silence matching uses: `alertname` + `namespace` + `severity` (NOT just alert name) + +**Quick Reference** (Alerts & Silences): +| Alert | alertname | namespace | severity | Time Range | Expected State | silenced | Use For Testing | +|-------|-----------|-----------|----------|------------|----------------|----------|-----------------| +| F1 | AlertF_KubePodCrashLooping | openshift-monitoring | warning | 1140-1260 | Resolved | false | Time-based resolution | +| F2 | AlertF_HighMemoryUsage | openshift-monitoring | critical | 1200-1380 | Resolved | false | Time-based resolution | +| G | AlertG_APIServerLatency | openshift-kube-apiserver | warning | 1440-1500 | Resolved | false | Short duration (1 hr) | +| I | AlertI_KubePodNotReady | openshift-operators | warning | 1800-1980 | Resolved | **true** | Silence matching | +| J | AlertJ_KubePodNotReady | openshift-storage | warning | 2040-2220 | Resolved | **false** | Different namespace = not silenced | + + +### 3.1 Short Incidents Not Visible +**BUG**: Incidents with duration < 5 minutes weren't showing up. +**Automation Status**: AUTOMATED in `02.reg_ui_charts_comprehensive.cy.ts` (Section 3.1) +- Uses fixture: `incident-scenarios/12-charts-ui-comprehensive.yaml` (includes very short duration incidents) +- Tests: 5-minute, 9-minute, and recently resolved (2 min ago) incidents +- Verifies: Bar visibility, dimensions, transparency, selectability, and alert loading + +- [x] **Short Incident C**: Check `api-server` incident (0 min duration, single point) - AUTOMATED + - Verify appears in incidents chart (has visible bar despite 0 duration) + - Select it and verify alert loads + +- [x] **Short Incident G**: Check `kube-apiserver` incident (60 min duration) - AUTOMATED + - Verify appears in incidents chart with visible bar + - Select it and verify AlertG_APIServerLatency appears + - Verify no minimum duration threshold filters it out + +### 3.2 Silences Not Applied Correctly +**BUG**: Silences were being matched by name only, not by name + namespace + severity. +**Automation Status**: AUTOMATED in `03.reg_api_calls.cy.ts` +- Uses fixture: `incident-scenarios/9-silenced-alerts-mixed-scenario.yaml` +- Verifies: Opacity (0.3 for silenced, 1.0 for non-silenced) +- Verifies: Tooltip "(silenced)" indicator +- Tests: Same alert name with different namespaces + +- [ ] **Incident I IS Silenced**: Check `AlertI_KubePodNotReady` in `openshift-operators` + - Silence matches: name=`KubePodNotReady` + namespace=`openshift-operators` + severity=`warning` + - Expected: Alert marked as `silenced = true` + - Verify alert bar has opacity: 0.3 (reduced) + - Verify tooltip shows: "AlertI_KubePodNotReady (silenced)" + +- [ ] **Incident J NOT Silenced**: Check `AlertJ_KubePodNotReady` in `openshift-storage` + - Same alert name as Incident I, but DIFFERENT namespace + - Silence does NOT match (namespace mismatch) + - Expected: Alert marked as `silenced = false` + - Verify alert bar has opacity: 1.0 (full) + - Verify tooltip shows: "AlertJ_KubePodNotReady" (no silenced suffix) + +- [ ] **Silence Matching Logic**: Verify implementation + - Check that matching uses: `alertname` + `namespace` + `severity` + - NOT just `alertname` alone + - Silence source: `cluster_health_components_map` metric (NOT Alertmanager API) + + ### 3.3 Alerts Marked as Resolved After Time +**BUG**: Alerts not being marked as resolved when they should be. +**Automation Status**: NOT AUTOMATED (requires live firing alerts) +- **WARNING Not possible to test on Injected Data, requires continously firing alert** + - Trigger a real firing alert (Pod CrashLooping...) + - Verify that the alert is firing + - Wait for 10 minutes without refreshing incidents + - Toggle the days filter to retrigger the alert queries + - Verify it is not marked as resolved + - Verify the latest query end time param is within the last 5 minutes + + + ### 3.4 Data Integrity + **NEW, NOT AUTOMATED, TODO COO 1.4** +- [ ] Incident grouping by `group_id` works correctly +- [ ] Values deduplicated across multiple time range queries +- [ ] Component lists combined for same group_id +- [ ] Watchdog alerts filtered out + + diff --git a/docs/incident_detection/tests/4.redux_state_and_effects_flows.md b/docs/incident_detection/tests/4.redux_state_and_effects_flows.md new file mode 100644 index 00000000..9f793b5b --- /dev/null +++ b/docs/incident_detection/tests/4.redux_state_and_effects_flows.md @@ -0,0 +1,120 @@ +## 4. CRITICAL: Effects / Redux State Management Bugs + +**Automation Status**: PARTIALLY AUTOMATED (4.5, 4.6, dropdown closure) + +### Prerequisites: Test Data Setup for State Management Tests + +Use the complete set of incidents (A-J). These tests focus on how the UI responds to state changes rather than specific data values. + +### 4.1 Basic Element Rendering +**Automation Status**: AUTOMATED +- Covered by `01.incidents.cy.ts` (tests 2, 3) and `04.reg_redux_effects.cy.ts` (test 3) +- Tests days filter changes and severity filter updates +- Verifies chart updates immediately without page reload + +- [x] **Incidents Refresh on Days Change**: AUTOMATED in `01.incidents.cy.ts` test 2 + - Start with "Last 7 days" (showing all 10 incidents A-J) + - Change to "Last 1 day" (should show only recent incidents) + - Verify incidents chart updates, loading spinner shows, new data displayed + +- [x] **Filtered Data Updates on Filter Change**: AUTOMATED in `01.incidents.cy.ts` test 3 and `04.reg_redux_effects.cy.ts` test 3 + - Apply "Critical" filter → verify only D, E, H shown + - Add "Warning" filter → verify B, F, G, I, J also appear + - Verify chart updates immediately (no page reload) + +### 4.2 Selected Incident Does Not Survive State Changes +**BUG**: Selected incident was being lost when changing filters or toggling graphs. +**Automation Status**: PARTIALLY AUTOMATED (filter changes covered, graph toggle not covered) +- Covered by `04.reg_redux_effects.cy.ts` test 3 +- Tests incident ID filter persistence when non-matching severity filter applied +- Graph toggle test not automated + +- [x] **Selection Survives Severity Filter**: AUTOMATED in `04.reg_redux_effects.cy.ts` test 3 + - Select Incident D (has Info, Warning, Critical in history) + - Apply "Warning" filter (D matches because it had Warning) + - Verify: Incident D still selected, URL has `?groupId=D`, alerts still shown + +- [x] **Selection Lost When Filtered Out (but ID filter persists)**: AUTOMATED in `04.reg_redux_effects.cy.ts` test 3 + - Select Incident A (Info only) + - Apply "Critical" filter (A doesn't match) + - Verify: Incident A disappears, but Incident ID filter chip remains, appropriate state + +- [ ] **Selection Survives Graph Toggle**: NOT AUTOMATED + - Select Incident H + - Click "Hide graph" + - Verify: URL still has `?groupId=H`, table shows H's alert + - Click "Show graph" → verify chart renders correctly + +### 4.3 Stale Alerts Displayed on Incident Reselection +**BUG**: When switching between incidents, stale alerts from previous incident shown briefly. +**Automation Status**: INDIRECTLY COVERED by `01.incidents.cy.ts` (test 5: Traverse Incident Table) +- The `findIncidentWithAlert` method would fail if stale alerts from previous selections are displayed +- Not explicitly tested with dedicated assertions, but functionality breaks if bug exists + +- [ ] **Incident Switching**: + - Select Incident D (with 3 alerts: Info, Warning, Critical) + - Deselect Incident D + - Immediately select Incident E (3 different component alerts) + - Verify: NO brief flash of D's alerts; loading state shown immediately + - Verify: Only E's alerts displayed after fetch completes + +- [ ] **Deselect Incident**: + - Select Incident F + - Click on Incident F again to deselect + - Verify: Alerts chart shows "select an incident" empty state + - Verify: No stale alerts remain + +### 4.4 Incident Dropdown Staying Open After Page Refresh +**BUG**: Dropdowns remained open after page refresh (Incidents Display / not F5). +**Automation Status**: AUTOMATED in `04.reg_redux_effects.cy.ts` (Test 2: Dropdown closure on deselection) + +- [ ] **Dropdown State After Refresh**: + - Select a particular Incident + - ~~Toggle filters that cause deselection of the incident and page reload~~ + NOTE: This won't happen, as the page will not be reloaded in new update + - Verify: Dropdown is closed after reload + - Verify: Dropdown does not jump to 0,0 coordinates + - Verify: Filter state restored from URL but dropdown collapsed + + ### 4.5 Dropdown Staying open after deselection + **BUG**: Deselection of incident causes reposition of the dropdown + **Automation Status**: AUTOMATED in `04.reg_redux_effects.cy.ts` (Test 2: Dropdown closure on deselection) + - Select a particular incident + - Open the left dropdown menu (Severity, State, ID) + - Deselect the incident by clicking on the bar, the site data should reload + - Verify: The dropdown should not reposition to 0.0 and should be closed + - Verify: Do the same also with the right toolbar. + + + +- [ ] **Dropdowns Auto-Close After Selection**: + - Open "Days" dropdown → select "3 days" → verify closes + - Open "Incident ID" filter → select an incident → verify closes + +### 4.5 Adding filter when incident selected does not remove the incident filter +**BUG:** When incident-id was filtered and additional filter (severity) applied, then if the filter was not matching the selected issue, the id filter was removed. +**Automation Status**: AUTOMATED in `04.reg_redux_effects.cy.ts` (Test 3: Filter state preservation) +- [ ] Select a "critical" incident by id +- [ ] Apply waring filter. +- [ ] Verify incident is filtered out +- [ ] Verify the filters "warning", and "incident id" are applied. + +### 4.6 Incidents Not Loaded Initially +**BUG**: Old Redux state was being used for effects fired at the beginning of page load. When the page loaded, only several issues were displayed. +**Automation Status**: AUTOMATED in `04.reg_redux_effects.cy.ts` (Test 1: Fresh load verification) + +**NOTE:** Hard to replicate, requires fresh browser instance + +### 4.7 Cached end time for prometheus query +**BUG**: End Time parameter for the prometheus query request uses the time of the initial load of the page instead of the current time, which causes firing alerts to be marked as resolved. +**Automation Status**: NOT AUTOMATED (requires live firing alerts) +**NOTE**: The issue is conceptually very similiar to 3.3, but is caused by the redux state caching, so it belongs to this section. +- **WARNING Not possible to test on Injected Data, requires continously firing alert, mocked (firing) data might be applicable though.** + - Trigger a real firing alert (Pod CrashLooping...) + - Verify that the alert is firing + - Wait for 10 minutes without refreshing incidents + - Refresh the days filter. + - Verify that the end time in the query to prometheus is updated to the current time value. + - Verify + + diff --git a/docs/incident_detection/tests/5.customization.md b/docs/incident_detection/tests/5.customization.md new file mode 100644 index 00000000..9d710ebe --- /dev/null +++ b/docs/incident_detection/tests/5.customization.md @@ -0,0 +1,12 @@ +# 5. Customization and Visual Bugs +**Automation Status**: NOT AUTOMATED + +### 5.1 Theme & Visual Polish +- [ ] Light/dark theme switching works correctly +- [ ] Chart legend displays properly with correct colors +- [ ] Chart responsiveness on window resize +- [ ] Chart height adapts to incident count (< 5: 300px, >= 5: count * 60px) + +### 5.2 Internationalization +- [ ] Translation keys used for all user-facing strings +- [ ] Multiple language support works (if available) diff --git a/docs/incident_detection/tests/6.table_interactions.md b/docs/incident_detection/tests/6.table_interactions.md new file mode 100644 index 00000000..dbffef8d --- /dev/null +++ b/docs/incident_detection/tests/6.table_interactions.md @@ -0,0 +1,10 @@ +### 6.1 Table Interactions +**Automation Status**: INDIRECTLY COVERED by existing tests +- Indirectly tested by `01.incidents.cy.ts` (test 5: Traverse Incident Table) and `02.reg_ui_charts_comprehensive.cy.ts` +- Table expansion, row interactions, and data display are exercised during incident selection and traversal + +- [ ] Expand/collapse all rows button works +- [ ] Individual row expansion works +- [ ] Expanded rows collapse when alert data changes +- [ ] Table sorting by start date (earliest at top) +- [ ] Severity badges show correct counts diff --git a/docs/incident_detection/tests/Uncategorized.testing_flows_ui.md b/docs/incident_detection/tests/Uncategorized.testing_flows_ui.md new file mode 100644 index 00000000..65d3d4d3 --- /dev/null +++ b/docs/incident_detection/tests/Uncategorized.testing_flows_ui.md @@ -0,0 +1,15 @@ +## Uncategorized Flows: Additional Testing + +**Automation Status**: NOT AUTOMATED + +These areas are generally stable but can be tested if you have time or suspect related issues. + + +### 5.4 URL State Management +- [ ] Browser back/forward buttons work correctly +- [ ] Direct URL access with filters loads correctly +- [ ] URL updates without page reload (`history.replaceState`) + + +--- + diff --git a/web/cypress/README.md b/web/cypress/README.md index 0eea7c06..bfa7f69a 100644 --- a/web/cypress/README.md +++ b/web/cypress/README.md @@ -428,4 +428,10 @@ cypress/ --- +### Incident Detection Test Documentation + +Test documentation for the Incidents feature is available at [`docs/incident_detection/tests/`](../../docs/incident_detection/tests/) in the repository root. + +--- + *For questions about test architecture, creating tests, or testing workflows, refer to [CYPRESS_TESTING_GUIDE.md](CYPRESS_TESTING_GUIDE.md)* diff --git a/web/cypress/fixtures/incidents/test-docs b/web/cypress/fixtures/incidents/test-docs deleted file mode 160000 index c601bc0e..00000000 --- a/web/cypress/fixtures/incidents/test-docs +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c601bc0eb518a12aed3c0cbded6d6f81c4dc3bd0