Skip to content

Commit c83d074

Browse files
mattseddonmattseddon
authored
Attempt to fix flaky integration test (simplify and split) (#3201)
* Attempt to fix flaky integration test (timing issue) * Split flaky test
1 parent 3028042 commit c83d074

File tree

2 files changed

+107
-75
lines changed

2 files changed

+107
-75
lines changed

‎extension/src/test/suite/extension.test.ts‎

Lines changed: 75 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,8 @@ import { join, resolve } from 'path'
22
import { afterEach, beforeEach, describe, it, suite } from 'mocha'
33
import { expect } from 'chai'
44
import { stub, restore, spy, match } from 'sinon'
5-
import { window, commands, workspace } from 'vscode'
6-
import {
7-
closeAllEditors,
8-
mockDisposable,
9-
mockDuration,
10-
quickPickInitialized,
11-
selectQuickPickItem
12-
} from './util'
5+
import { commands, workspace } from 'vscode'
6+
import { closeAllEditors, mockDisposable, mockDuration } from './util'
137
import { mockHasCheckpoints } from './experiments/util'
148
import { Disposable } from '../../extension'
159
import * as Python from '../../extensions/python'
@@ -49,41 +43,10 @@ suite('Extension Test Suite', () => {
4943
])
5044
})
5145

46+
// eslint-disable-next-line sonarjs/cognitive-complexity
5247
describe('dvc.setupWorkspace', () => {
5348
it('should initialize the extension when the cli is usable', async () => {
5449
stub(Python, 'isPythonExtensionInstalled').returns(true)
55-
const selectVirtualEnvWithPython = async (path: string) => {
56-
const mockShowQuickPick = stub(window, 'showQuickPick')
57-
58-
const venvQuickPickActive = quickPickInitialized(mockShowQuickPick, 0)
59-
60-
const setupWorkspaceWizard = commands.executeCommand(
61-
RegisteredCommands.EXTENSION_SETUP_WORKSPACE
62-
)
63-
64-
const mockSelectPythonInterpreter = stub(
65-
Python,
66-
'selectPythonInterpreter'
67-
)
68-
const executeCommandCalled = new Promise(resolve =>
69-
mockSelectPythonInterpreter.callsFake(() => {
70-
void setConfigValue(ConfigKey.PYTHON_PATH, path)
71-
resolve(undefined)
72-
})
73-
)
74-
75-
await venvQuickPickActive
76-
77-
await selectQuickPickItem(1)
78-
79-
await executeCommandCalled
80-
81-
mockSelectPythonInterpreter.restore()
82-
83-
mockShowQuickPick.restore()
84-
85-
return setupWorkspaceWizard
86-
}
8750

8851
const mockCreateFileSystemWatcher = stub(
8952
workspace,
@@ -98,6 +61,10 @@ suite('Extension Test Suite', () => {
9861
onDidDelete: () => mockDisposable
9962
})
10063

64+
const mockGetDvcRoots = stub(Setup.prototype, 'getRoots').returns([
65+
dvcDemoPath
66+
])
67+
10168
const mockCanRunCli = stub(DvcReader.prototype, 'version')
10269
.onFirstCall()
10370
.resolves(MIN_CLI_VERSION)
@@ -110,9 +77,13 @@ suite('Extension Test Suite', () => {
11077
new Promise(resolve => {
11178
mockDisposer.resetBehavior()
11279
mockDisposer.resetHistory()
113-
mockDisposer.callsFake((...args) => {
80+
mockDisposer.callsFake((disposables, untrack) => {
11481
resolve(undefined)
115-
return mockDisposer.wrappedMethod(...args)
82+
for (const repository of Object.values(disposables)) {
83+
untrack(repository)
84+
disposable.track(repository)
85+
}
86+
return {}
11687
})
11788
})
11889

@@ -128,41 +99,58 @@ suite('Extension Test Suite', () => {
12899
)
129100

130101
mockHasCheckpoints(expShowFixture)
131-
const mockExpShow = stub(DvcReader.prototype, 'expShow').resolves(
132-
expShowFixture
133-
)
102+
const mockExpShow = stub(DvcReader.prototype, 'expShow')
103+
const mockDataStatus = stub(DvcReader.prototype, 'dataStatus')
104+
const mockPlotsDiff = stub(DvcReader.prototype, 'plotsDiff')
134105

135106
stub(DvcReader.prototype, 'root').resolves('.')
136107

137-
const mockDataStatus = stub(DvcReader.prototype, 'dataStatus').resolves({
138-
committed: {
139-
added: [],
140-
deleted: [],
141-
modified: [],
142-
renamed: []
143-
},
144-
not_in_cache: [],
145-
unchanged: [
146-
join('data', 'MNIST', 'raw', 't10k-images-idx3-ubyte'),
147-
join('data', 'MNIST', 'raw', 't10k-images-idx3-ubyte.gz'),
148-
join('data', 'MNIST', 'raw', 't10k-labels-idx1-ubyte'),
149-
join('data', 'MNIST', 'raw', 't10k-labels-idx1-ubyte.gz'),
150-
join('data', 'MNIST', 'raw', 'train-images-idx3-ubyte'),
151-
join('data', 'MNIST', 'raw', 'train-images-idx3-ubyte.gz'),
152-
join('data', 'MNIST', 'raw', 'train-labels-idx1-ubyte'),
153-
join('data', 'MNIST', 'raw', 'train-labels-idx1-ubyte.gz'),
154-
join('logs', 'acc.tsv'),
155-
join('logs', 'loss.tsv')
156-
],
157-
uncommitted: {
158-
added: [],
159-
deleted: [],
160-
modified: ['model.pt', join('data', 'MNIST', 'raw'), 'logs'],
161-
renamed: []
162-
}
108+
const dataStatusCalled = new Promise(resolve => {
109+
mockDataStatus.callsFake(() => {
110+
resolve(undefined)
111+
return Promise.resolve({
112+
committed: {
113+
added: [],
114+
deleted: [],
115+
modified: [],
116+
renamed: []
117+
},
118+
not_in_cache: [],
119+
unchanged: [
120+
join('data', 'MNIST', 'raw', 't10k-images-idx3-ubyte'),
121+
join('data', 'MNIST', 'raw', 't10k-images-idx3-ubyte.gz'),
122+
join('data', 'MNIST', 'raw', 't10k-labels-idx1-ubyte'),
123+
join('data', 'MNIST', 'raw', 't10k-labels-idx1-ubyte.gz'),
124+
join('data', 'MNIST', 'raw', 'train-images-idx3-ubyte'),
125+
join('data', 'MNIST', 'raw', 'train-images-idx3-ubyte.gz'),
126+
join('data', 'MNIST', 'raw', 'train-labels-idx1-ubyte'),
127+
join('data', 'MNIST', 'raw', 'train-labels-idx1-ubyte.gz'),
128+
join('logs', 'acc.tsv'),
129+
join('logs', 'loss.tsv')
130+
],
131+
uncommitted: {
132+
added: [],
133+
deleted: [],
134+
modified: ['model.pt', join('data', 'MNIST', 'raw'), 'logs'],
135+
renamed: []
136+
}
137+
})
138+
})
163139
})
164140

165-
stub(DvcReader.prototype, 'plotsDiff').resolves(plotsDiffFixture)
141+
const expShowCalled = new Promise(resolve => {
142+
mockExpShow.callsFake(() => {
143+
resolve(undefined)
144+
return Promise.resolve(expShowFixture)
145+
})
146+
})
147+
148+
const plotsDiffCalled = new Promise(resolve => {
149+
mockPlotsDiff.callsFake(() => {
150+
resolve(undefined)
151+
return Promise.resolve(plotsDiffFixture)
152+
})
153+
})
166154

167155
const mockWorkspaceExperimentsReady = stub(
168156
WorkspaceExperiments.prototype,
@@ -181,9 +169,15 @@ suite('Extension Test Suite', () => {
181169

182170
const mockPath = resolve('path', 'to', 'venv')
183171

184-
await selectVirtualEnvWithPython(resolve('path', 'to', 'venv'))
172+
void (await setConfigValue(ConfigKey.PYTHON_PATH, mockPath))
185173

186-
await Promise.all([firstDisposal, correctTelemetryEventSent])
174+
await Promise.all([
175+
firstDisposal,
176+
correctTelemetryEventSent,
177+
dataStatusCalled,
178+
expShowCalled,
179+
plotsDiffCalled
180+
])
187181

188182
expect(
189183
await workspace.getConfiguration().get(ConfigKey.PYTHON_PATH)
@@ -197,6 +191,9 @@ suite('Extension Test Suite', () => {
197191
.been.called
198192
expect(mockExpShow, 'should have updated the experiments data').to.have
199193
.been.called
194+
expect(mockPlotsDiff, 'should have updated the plots data').to.have.been
195+
.called
196+
expect(mockGetDvcRoots).to.have.been.called
200197

201198
expect(
202199
mockSendTelemetryEvent,
@@ -231,7 +228,10 @@ suite('Extension Test Suite', () => {
231228
await workspaceExperimentsAreReady
232229
const secondDisposal = disposalEvent()
233230

234-
await selectVirtualEnvWithPython(resolve('path', 'to', 'virtualenv'))
231+
void (await setConfigValue(
232+
ConfigKey.PYTHON_PATH,
233+
resolve('path', 'to', 'virtualenv')
234+
))
235235

236236
await secondDisposal
237237

‎extension/src/test/suite/setup/index.test.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import * as Telemetry from '../../../telemetry'
3131
import { StopWatch } from '../../../util/time'
3232
import { MIN_CLI_VERSION } from '../../../cli/dvc/contract'
3333
import { run } from '../../../setup/runner'
34+
import * as Python from '../../../extensions/python'
3435

3536
suite('Setup Test Suite', () => {
3637
const disposable = Disposable.fn()
@@ -382,6 +383,37 @@ suite('Setup Test Suite', () => {
382383
)
383384
})
384385

386+
it("should prompt show the Python extension's select python interpreter command if the extension is installed and the user chooses to use a virtual environment", async () => {
387+
stub(Python, 'isPythonExtensionInstalled').returns(true)
388+
const mockShowQuickPick = stub(window, 'showQuickPick')
389+
390+
const venvQuickPickActive = quickPickInitialized(mockShowQuickPick, 0)
391+
392+
const setupWorkspaceWizard = commands.executeCommand(
393+
RegisteredCommands.EXTENSION_SETUP_WORKSPACE
394+
)
395+
396+
const mockSelectPythonInterpreter = stub(
397+
Python,
398+
'selectPythonInterpreter'
399+
)
400+
const executeCommandCalled = new Promise(resolve =>
401+
mockSelectPythonInterpreter.callsFake(() => {
402+
resolve(undefined)
403+
})
404+
)
405+
406+
await venvQuickPickActive
407+
408+
await Promise.all([
409+
selectQuickPickItem(1),
410+
executeCommandCalled,
411+
setupWorkspaceWizard
412+
])
413+
414+
expect(mockSelectPythonInterpreter).to.be.calledOnce
415+
})
416+
385417
it('should set dvc.pythonPath to the picked value when the user selects to pick a Python interpreter', async () => {
386418
const { config, setup, mockExecuteCommand, mockVersion } =
387419
buildSetup(disposable)

0 commit comments

Comments
 (0)