Skip to content

Commit a5cf374

Browse files
testlabautotestlabauto
authored
Get Pyrefly tests running (#10937)
Get pyrefly tests running again, for now in a manually kicked off workflow. Only include pyrefly disable settings if ALLOW_PYREFLY is not 'true'. ### QA Notes @:web
1 parent 08eeef7 commit a5cf374

File tree

9 files changed

+80
-46
lines changed

9 files changed

+80
-46
lines changed

‎.github/workflows/test-e2e-pyrefly.yml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,13 +47,13 @@ jobs:
4747
aws-role-to-assume: ${{ secrets.QA_AWS_RO_ROLE }}
4848
aws-region: ${{ secrets.QA_AWS_REGION }}
4949

50-
- name: Send Results to GH Summary
51-
uses: ./.github/actions/gen-report-dir
52-
5350
- name: Alter AppArmor Restrictions for Playwright
5451
run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
5552

56-
- name: Run Playwright Tests
53+
- name: Send Results to GH Summary
54+
uses: ./.github/actions/gen-report-dir
55+
56+
- name: Run Playwright Tests - Electron
5757
shell: bash
5858
env:
5959
POSITRON_PY_VER_SEL: "3.10.12"
@@ -67,7 +67,7 @@ jobs:
6767
USE_KEY: true
6868
GH_SUMMARY_REPORT: true
6969
run: |
70-
SKIP_BOOTSTRAP=true npx playwright test --project e2e-elctron test/e2e/tests/pyrefly --workers 1 --reporter=json
70+
ALLOW_PYREFLY=true npx playwright test test/e2e/tests/pyrefly --project=e2e-electron --workers=1 --reporter=html
7171
7272
- name: Upload Playwright Report to S3
7373
if: ${{ success() || failure() }}

‎playwright.config.ts‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,13 @@ const githubSummaryReport = process.env.GH_SUMMARY_REPORT === 'true' ? [['@midle
1919
*/
2020
const projectName = process.env.PW_PROJECT_NAME || 'default';
2121

22+
const baseIgnore = [
23+
'example.test.ts',
24+
'**/workbench/**',
25+
'**/inspect-ai/**',
26+
'**/remote-ssh/**',
27+
];
28+
2229
export default defineConfig<ExtendedTestOptions>({
2330
captureGitInfo: { commit: true, diff: true },
2431
globalSetup: './test/e2e/tests/_global.setup.ts',
@@ -27,13 +34,9 @@ export default defineConfig<ExtendedTestOptions>({
2734
shardingMode: 'duration-round-robin',
2835
// @ts-expect-error shardingMode and lastRunFile added by playwright patch
2936
lastRunFile: `./blob-report/.last-run-${projectName}.json`,
30-
testIgnore: [
31-
'example.test.ts',
32-
'**/workbench/**',
33-
'**/inspect-ai/**',
34-
'**/remote-ssh/**',
35-
'**/pyrefly/**'
36-
],
37+
testIgnore: process.env.ALLOW_PYREFLY === 'true'
38+
? baseIgnore
39+
: [...baseIgnore, '**/pyrefly/**'],
3740
fullyParallel: false, // Run individual tests w/in a spec in parallel
3841
forbidOnly: !!process.env.CI,
3942
retries: process.env.CI ? 1 : 0,

‎test/e2e/fixtures/settings.json‎

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,15 @@
11
{
2-
"python.locator": "native",
3-
"interpreters.startupBehavior": "manual",
4-
"positron.r.kernel.logLevel": "trace",
5-
"python.languageServerLogLevel": "debug",
6-
"remote.autoForwardPortsFallback": 0,
7-
"files.simpleDialog.enable": true,
8-
"positron.importSettings.enable": false,
9-
"positron.assistant.enable": true,
10-
"positron.assistant.testModels": true,
11-
"positron.assistant.enabledProviders": ["openai-api"],
12-
"posit.workbench.showWorkbenchFlaskHint": false,
13-
"extensions.allowed": {
14-
"meta.pyrefly": false,
15-
"*": true
16-
}
17-
}
2+
"python.locator": "native",
3+
"interpreters.startupBehavior": "manual",
4+
"positron.r.kernel.logLevel": "trace",
5+
"python.languageServerLogLevel": "debug",
6+
"remote.autoForwardPortsFallback": 0,
7+
"files.simpleDialog.enable": true,
8+
"positron.importSettings.enable": false,
9+
"positron.assistant.enable": true,
10+
"positron.assistant.testModels": true,
11+
"positron.assistant.enabledProviders": [
12+
"openai-api"
13+
],
14+
"posit.workbench.showWorkbenchFlaskHint": false
15+
}

‎test/e2e/fixtures/settingsSkipPyrefly.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"extensions.allowed": {
3+
"meta.pyrefly": false,
4+
"*": true
5+
}
6+
}

‎test/e2e/fixtures/test-setup/shared-utils.ts‎

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -45,20 +45,40 @@ export async function captureScreenshotOnError(app: Application, logsPath: strin
4545
*/
4646
export async function copyUserSettings(userDir: string): Promise<string> {
4747
const settingsFileName = 'settings.json';
48+
const fixturesDir = path.join(ROOT_PATH, 'test/e2e/fixtures');
49+
const settingsFile = path.join(fixturesDir, settingsFileName);
4850

49-
if (fs.existsSync('/.dockerenv')) {
50-
const fixturesDir = path.join(ROOT_PATH, 'test/e2e/fixtures');
51-
const settingsFile = path.join(fixturesDir, 'settings.json');
51+
// Start from the current settings.json in fixtures
52+
let mergedSettings = JSON.parse(fs.readFileSync(settingsFile, 'utf8'));
5253

53-
const mergedSettings = {
54-
...JSON.parse(fs.readFileSync(settingsFile, 'utf8')),
55-
...JSON.parse(fs.readFileSync(path.join(fixturesDir, 'settingsDocker.json'), 'utf8')),
56-
};
54+
// 1. Merge Docker-specific overrides when running in Docker
55+
if (fs.existsSync('/.dockerenv')) {
56+
const dockerSettingsFile = path.join(fixturesDir, 'settingsDocker.json');
57+
if (fs.existsSync(dockerSettingsFile)) {
58+
const dockerSettings = JSON.parse(fs.readFileSync(dockerSettingsFile, 'utf8'));
59+
mergedSettings = {
60+
...mergedSettings,
61+
...dockerSettings,
62+
};
63+
}
64+
}
5765

58-
// Overwrite file
59-
fs.writeFileSync(settingsFile, JSON.stringify(mergedSettings, null, 2));
66+
// 2. Merge skip-pyrefly settings if ALLOW_PYREFLY is not explicitly 'true'
67+
if (process.env.ALLOW_PYREFLY !== 'true') {
68+
const skipPyreflyFile = path.join(fixturesDir, 'settingsSkipPyrefly.json');
69+
if (fs.existsSync(skipPyreflyFile)) {
70+
const skipPyreflySettings = JSON.parse(fs.readFileSync(skipPyreflyFile, 'utf8'));
71+
mergedSettings = {
72+
...mergedSettings,
73+
...skipPyreflySettings,
74+
};
75+
}
6076
}
6177

78+
// Overwrite fixtures/settings.json with the merged result
79+
fs.writeFileSync(settingsFile, JSON.stringify(mergedSettings, null, 2));
80+
81+
// Let existing helper copy settings.json into the user dir
6282
await copyFixtureFile(settingsFileName, userDir);
6383
return userDir;
6484
}

‎test/e2e/infra/test-runner/test-tags.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ export enum TestTags {
4848
PLOTS = '@:plots',
4949
PROBLEMS = '@:problems',
5050
PUBLISHER = '@:publisher',
51+
PYREFLY = '@:pyrefly',
5152
REFERENCES = '@:references',
5253
R_MARKDOWN = '@:r-markdown',
5354
R_PKG_DEVELOPMENT = '@:r-pkg-development',

‎test/e2e/tests/pyrefly/pyrefly-diagnostics.test.ts‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,13 @@ test.use({
1010
});
1111

1212
test.describe('Diagnostics', {
13-
tag: [tags.SESSIONS, tags.PROBLEMS, tags.WEB, tags.WIN, tags.SOFT_FAIL],
13+
tag: [tags.PYREFLY, tags.WEB],
1414
}, () => {
1515

16+
test.beforeAll(async function ({ settings }) {
17+
await settings.set({ 'python.pyrefly.displayTypeErrors': 'force-on' });
18+
});
19+
1620
test.afterEach(async function ({ runCommand }) {
1721
await runCommand('workbench.action.closeAllEditors');
1822
});
@@ -45,9 +49,11 @@ test.describe('Diagnostics', {
4549

4650
// Python Alt Session - verify warning since pkg not installed
4751
await sessions.start('pythonAlt');
48-
await problems.expectDiagnosticsToBe({ badgeCount: 1, warningCount: 1, errorCount: 0 });
49-
await problems.expectWarningText('Import "termcolor" could not be resolved');
50-
await problems.expectSquigglyCountToBe('warning', 1);
52+
await problems.expectDiagnosticsToBe({ badgeCount: 1, warningCount: 0, errorCount: 1 });
53+
await problems.expectWarningText('Could not find import of `termcolor`');
54+
55+
// does pyrefly use squiggly correctly?
56+
// await problems.expectSquigglyCountToBe('warning', 1);
5157

5258
// Python Session 1 - restart session and verify no problems
5359
await sessions.select(pySession.id);

‎test/e2e/tests/pyrefly/pyrefly-outline.test.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,13 @@ test.use({
1414
suiteId: __filename
1515
});
1616

17-
test.describe('Outline', { tag: [tags.WEB, tags.WIN, tags.OUTLINE] }, () => {
17+
test.describe('Outline', { tag: [tags.WEB, tags.PYREFLY] }, () => {
1818

1919
test.afterAll(async function ({ hotKeys }) {
2020
await hotKeys.closeAllEditors();
2121
});
2222

23-
test.describe('Outline: Sessions', { tag: [tags.SESSIONS, tags.ARK] }, () => {
23+
test.describe('Outline: Sessions', () => {
2424

2525
test.beforeAll(async function ({ app, openFile, hotKeys }) {
2626
const { outline } = app.workbench;
@@ -80,7 +80,7 @@ test.describe('Outline', { tag: [tags.WEB, tags.WIN, tags.OUTLINE] }, () => {
8080
await verifyPythonOutline(outline);
8181
});
8282

83-
test('Verify outline after reload with Python in foreground and R in background', {
83+
test.skip('Verify outline after reload with Python in foreground and R in background', {
8484
annotation: [{ type: 'issue', description: 'https://github.com/posit-dev/positron/issues/7052' }],
8585
}, async function ({ app, runCommand, sessions }) {
8686
const { outline, editor } = app.workbench;
@@ -110,7 +110,7 @@ test.describe('Outline', { tag: [tags.WEB, tags.WIN, tags.OUTLINE] }, () => {
110110
await verifyROutline(outline);
111111
});
112112

113-
test('Verify outline after reload with R in foreground and Python in background', {
113+
test.skip('Verify outline after reload with R in foreground and Python in background', {
114114
annotation: [{ type: 'issue', description: 'https://github.com/posit-dev/positron/issues/7052' }],
115115
},
116116
async function ({ app, runCommand, sessions }) {

‎test/e2e/tests/pyrefly/pyrefly-references.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ test.use({
1313
});
1414

1515
test.describe('References', {
16-
tag: [tags.REFERENCES, tags.WEB, tags.WIN]
16+
tag: [tags.PYREFLY, tags.WEB]
1717
}, () => {
1818

1919
test.afterEach(async ({ app, runCommand }) => {

0 commit comments

Comments
 (0)