Skip to content

Commit 812b7bc

Browse files
VeikkosuhonenVeikkosuhonen
committed
test: Epic test matrix for different roles and course vs non-course chat
1 parent 01070c5 commit 812b7bc

File tree

11 files changed

+214
-146
lines changed

11 files changed

+214
-146
lines changed

‎e2e/chat.spec.ts‎

Lines changed: 134 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,148 @@
11
import { expect } from '@playwright/test'
2-
import { acceptDisclaimer, closeSendPreference, sendChatMessage } from './utils/test-helpers'
3-
import { teacherTest as test } from './fixtures'
2+
import { acceptDisclaimer, closeSendPreference, sendChatMessage, useMockModel } from './utils/test-helpers'
3+
import { TestByRole } from './fixtures'
44

5-
test.describe('Chat v2 Conversation tests', () => {
6-
test.beforeEach(async ({ page }) => {
7-
await page.goto('/v2')
8-
await acceptDisclaimer(page)
9-
})
5+
// Matrix of tests
6+
const testMatrix: { role: keyof typeof TestByRole; courses: (string | undefined)[] }[] = [
7+
{ role: 'student', courses: ['test-course'] },
8+
{ role: 'teacher', courses: ['test-course', undefined] },
9+
{ role: 'admin', courses: ['test-course', undefined] },
10+
]
1011

11-
test('Chat v2 mock response works', async ({ page }) => {
12-
await page.getByTestId('model-selector').first().click()
13-
await page.getByRole('option', { name: 'mock' }).click()
12+
testMatrix.forEach((testConfig) => {
13+
const test = TestByRole[testConfig.role]
1414

15-
await sendChatMessage(page, 'testinen morjens')
15+
testConfig.courses.forEach((course) => {
16+
test.describe(`${course ? `Course` : 'General'} chat test for ${testConfig.role}`, () => {
17+
test.beforeEach(async ({ page }) => {
18+
await page.goto(`/v2/${course || ''}`)
19+
})
1620

17-
await closeSendPreference(page)
21+
test('Disclaimer is visible', async ({ page }) => {
22+
await expect(page.getByTestId('submit-accept-disclaimer')).toBeVisible()
23+
})
1824

19-
await expect(page.getByTestId('user-message')).toContainText('testinen morjens')
20-
await expect(page.getByTestId('assistant-message')).toContainText('You are calling mock endpoint for streaming mock data')
21-
})
25+
test('Disclaimer is not visible after accepting and reloading', async ({ page }) => {
26+
await acceptDisclaimer(page)
27+
await page.reload()
28+
await expect(page.getByTestId('submit-accept-disclaimer')).not.toBeVisible()
29+
})
30+
31+
test('Disclaimer (help) can be opened manually', async ({ page }) => {
32+
await acceptDisclaimer(page)
33+
await page.getByTestId('help-button').click()
34+
await expect(page.getByTestId('submit-accept-disclaimer')).toBeVisible()
35+
})
36+
37+
test('Settings can be opened and closed', async ({ page }) => {
38+
await acceptDisclaimer(page)
39+
await page.getByTestId('settings-button').click()
40+
await expect(page.locator('#close-settings')).toBeVisible()
41+
42+
await page.keyboard.press('Escape')
43+
await expect(page.locator('#close-settings')).not.toBeVisible()
44+
})
45+
46+
test('Can select model', async ({ page }) => {
47+
await acceptDisclaimer(page)
48+
await useMockModel(page)
49+
})
50+
51+
test('Chat v2 mock response works', async ({ page }) => {
52+
await acceptDisclaimer(page)
53+
await useMockModel(page)
54+
55+
await sendChatMessage(page, 'testinen morjens')
56+
57+
await closeSendPreference(page)
58+
59+
await expect(page.getByTestId('user-message')).toContainText('testinen morjens')
60+
await expect(page.getByTestId('assistant-message')).toContainText('You are calling mock endpoint for streaming mock data')
61+
})
62+
63+
test('Can empty conversation', async ({ page }) => {
64+
await acceptDisclaimer(page)
65+
await useMockModel(page)
66+
67+
await sendChatMessage(page, 'tää tyhjennetään')
68+
69+
await closeSendPreference(page)
70+
71+
await expect(page.getByTestId('user-message')).toContainText('tää tyhjennetään')
72+
await expect(page.getByTestId('assistant-message')).toContainText('OVER', { timeout: 6000 })
73+
74+
page.on('dialog', (dialog) => dialog.accept())
75+
await page.getByTestId('empty-conversation-button').click()
76+
77+
await expect(page.getByTestId('user-message')).not.toBeVisible()
78+
await expect(page.getByTestId('assistant-message')).not.toBeVisible()
79+
})
80+
81+
test('Custom system prompt can be changed', async ({ page }) => {
82+
await acceptDisclaimer(page)
83+
await useMockModel(page)
84+
await page.getByTestId('settings-button').click()
85+
86+
const systemPrompt = 'mocktest this is the system prompt'
87+
await page.getByTestId('assistant-instructions-input').fill(systemPrompt)
88+
await page.getByTestId('settings-ok-button').click()
89+
90+
await sendChatMessage(page, 'I can send anything now, the model just echoes the system prompt since it begins with mocktest.')
91+
await closeSendPreference(page)
92+
93+
await expect(page.getByTestId('assistant-message')).toContainText(systemPrompt)
94+
})
95+
96+
test('Default temperature is 0.5 and can adjust temperature', async ({ page }) => {
97+
await acceptDisclaimer(page)
98+
await useMockModel(page)
99+
100+
await sendChatMessage(page, 'temperature')
101+
await closeSendPreference(page)
102+
await expect(page.getByTestId('assistant-message').first()).toContainText('Temperature: 0.5')
103+
104+
await page.getByTestId('settings-button').click()
105+
106+
const slider = page.getByRole('slider').first()
107+
// Move right 6 times
108+
for (let i = 0; i < 6; i++) {
109+
await slider.press('ArrowRight')
110+
}
111+
112+
// Close settings
113+
await page.getByTestId('settings-ok-button').click()
114+
115+
await sendChatMessage(page, 'temperature')
116+
await expect(page.getByTestId('assistant-message').last()).toContainText('Temperature: 1')
117+
})
118+
119+
if (course)
120+
test('Course chat RAG feature', async ({ page }) => {
121+
await acceptDisclaimer(page)
122+
await useMockModel(page)
123+
124+
const ragName = `rag-${test.info().workerIndex}-${testConfig.role}`
125+
await page.locator('#rag-index-selector').first().click()
126+
await page.getByRole('menuitem', { name: ragName, exact: true }).click()
22127

23-
test('Can empty conversation', async ({ page }) => {
24-
await page.getByTestId('model-selector').first().click()
25-
await page.getByRole('option', { name: 'mock' }).click()
128+
await sendChatMessage(page, 'rag')
129+
await closeSendPreference(page)
26130

27-
await sendChatMessage(page, 'tää tyhjennetään')
131+
// Shows file search loading indicator
132+
await expect(page.getByTestId('tool-call-message')).toBeVisible()
28133

29-
await closeSendPreference(page)
134+
// Responds with RAG mock document text
135+
await expect(page.getByTestId('assistant-message')).toContainText('This is the first mock document')
30136

31-
await expect(page.getByTestId('user-message')).toContainText('tää tyhjennetään')
32-
await expect(page.getByTestId('assistant-message')).toContainText('OVER', { timeout: 6000 })
137+
// Source button is visible
138+
await expect(page.getByTestId('file-search-sources')).toBeVisible()
33139

34-
page.on('dialog', (dialog) => dialog.accept())
35-
await page.getByTestId('empty-conversation-button').click()
140+
// Sources drawer has been opened and title is visible
141+
await expect(page.getByTestId('sources-header')).toBeVisible()
36142

37-
await expect(page.getByTestId('user-message')).not.toBeVisible()
38-
await expect(page.getByTestId('assistant-message')).not.toBeVisible()
143+
// Three source items should be visible
144+
await expect(page.getByTestId('sources-truncated-item')).toHaveCount(3)
145+
})
146+
})
39147
})
40148
})

‎e2e/chatFeatures.spec.ts‎

Lines changed: 0 additions & 34 deletions
This file was deleted.

‎e2e/courseChatRag.spec.ts‎

Lines changed: 0 additions & 45 deletions
This file was deleted.

‎e2e/fixtures.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,3 +93,9 @@ export const studentTest = base.extend<{ forEachTest: void }>({
9393
{ auto: true },
9494
], // automatically starts for every test.
9595
})
96+
97+
export const TestByRole = {
98+
student: studentTest,
99+
teacher: teacherTest,
100+
admin: adminTest,
101+
}

‎e2e/utils/test-helpers.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,8 @@ export const sendChatMessage = async (page: Page, message: string) => {
1515
const sendBtn = page.getByTestId('send-chat-message')
1616
await sendBtn.click()
1717
}
18+
19+
export const useMockModel = async (page: Page) => {
20+
await page.getByTestId('model-selector').first().click()
21+
await page.getByRole('option', { name: 'mock' }).click()
22+
}

‎playwright.config.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ export default defineConfig({
2020
/* Retry on CI only */
2121
retries: inCI ? 1 : 0,
2222
/* Global timeout for each test */
23-
timeout: inCI ? 15_000 : 12_000,
23+
timeout: inCI ? 20_000 : 15_000,
2424
expect: {
2525
timeout: inCI ? 4_500 : 3000,
2626
},

‎src/client/components/ChatV2/SettingsModal.tsx‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,7 @@ export const SettingsModal: React.FC<SettingsModalProps> = ({
248248
value={modelTemperature}
249249
marks
250250
valueLabelDisplay="auto"
251+
name="Temperature"
251252
onChange={(_event, value) => setModelTemperature(typeof value === 'number' ? value : modelTemperature)}
252253
/>
253254
<Box sx={{ display: 'flex', justifyContent: 'space-between' }}>
@@ -282,7 +283,9 @@ export const SettingsModal: React.FC<SettingsModalProps> = ({
282283
>
283284
{t('settings:resetDefault')}
284285
</OutlineButtonBlack>{' '}
285-
<BlueButton onClick={handleClose}>OK</BlueButton>
286+
<BlueButton onClick={handleClose} data-testid="settings-ok-button">
287+
OK
288+
</BlueButton>
286289
</Box>
287290
<SaveMyPromptModal
288291
isOpen={myPromptModalOpen}

‎src/server/routes/ai/v3.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,7 @@ router.post('/stream', upload.single('file'), async (r, res) => {
140140
chatMessages: options.chatMessages,
141141
systemMessage: options.systemMessage,
142142
model: options.model,
143+
temperature: options.modelTemperature,
143144
tools,
144145
writeEvent: async (event: ChatEvent) => {
145146
await new Promise((resolve) => {

0 commit comments

Comments
 (0)