Skip to content

Commit 4b3fcb6

Browse files
author
Kartik Raj
authored
Added workspace virtualenv filesystem watcher (microsoft#14780)
* Added workspace virtual env watcher * Fix tests * Remove option * Mege the two tests together * Skip * Try this out * Let's try this fix * Clean up * Fix lint errors
1 parent abfb8c3 commit 4b3fcb6

File tree

6 files changed

+213
-14
lines changed

6 files changed

+213
-14
lines changed

src/client/common/platform/fileSystemWatcher.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ function watchLocationUsingChokidar(
7676
'**/lib/**',
7777
'**/includes/**'
7878
], // https://github.com/microsoft/vscode/issues/23954
79-
followSymlinks: false
79+
followSymlinks: true
8080
};
8181
traceVerbose(`Start watching: ${baseDir} with pattern ${pattern} using chokidar`);
8282
let watcher: chokidar.FSWatcher | null = chokidar.watch(pattern, watcherOpts);

src/client/pythonEnvironments/base/locators/lowLevel/fsWatchingLocator.ts

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,11 @@ export abstract class FSWatchingLocator extends Locator {
2929
/**
3030
* Glob which represents basename of the executable to watch.
3131
*/
32-
executableBaseGlob?: string,
32+
executableBaseGlob?: string;
3333
/**
3434
* Time to wait before handling an environment-created event.
3535
*/
36-
delayOnCreated?: number, // milliseconds
36+
delayOnCreated?: number; // milliseconds
3737
} = {},
3838
) {
3939
super();
@@ -72,7 +72,9 @@ export abstract class FSWatchingLocator extends Locator {
7272
await sleep(this.opts.delayOnCreated);
7373
}
7474
}
75-
const kind = await this.getKind(executable);
75+
// Fetching kind after deletion normally fails because the file structure around the
76+
// executable is no longer available, so ignore the errors.
77+
const kind = await this.getKind(executable).catch(() => undefined);
7678
this.emitter.fire({ type, kind });
7779
},
7880
this.opts.executableBaseGlob,

src/client/pythonEnvironments/base/locators/lowLevel/workspaceVirtualEnvLocator.ts

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ import { isPipenvEnvironment } from '../../../discovery/locators/services/pipEnv
1515
import { isVenvEnvironment, isVirtualenvEnvironment } from '../../../discovery/locators/services/virtualEnvironmentIdentifier';
1616
import { PythonEnvInfo, PythonEnvKind } from '../../info';
1717
import { buildEnvInfo } from '../../info/env';
18-
import { IPythonEnvsIterator, Locator } from '../../locator';
18+
import { IDisposableLocator, IPythonEnvsIterator } from '../../locator';
19+
import { FSWatchingLocator } from './fsWatchingLocator';
1920

2021
/**
2122
* Default number of levels of sub-directories to recurse when looking for interpreters.
@@ -75,9 +76,13 @@ async function buildSimpleVirtualEnvInfo(executablePath: string, kind: PythonEnv
7576
/**
7677
* Finds and resolves virtual environments created in workspace roots.
7778
*/
78-
export class WorkspaceVirtualEnvironmentLocator extends Locator {
79+
class WorkspaceVirtualEnvironmentLocator extends FSWatchingLocator {
7980
public constructor(private readonly root: string) {
80-
super();
81+
super(() => getWorkspaceVirtualEnvDirs(this.root), getVirtualEnvKind, {
82+
// Note detecting kind of virtual env depends on the file structure around the
83+
// executable, so we need to wait before attempting to detect it.
84+
delayOnCreated: 1000,
85+
});
8186
}
8287

8388
public iterEnvs(): IPythonEnvsIterator {
@@ -128,3 +133,9 @@ export class WorkspaceVirtualEnvironmentLocator extends Locator {
128133
return undefined;
129134
}
130135
}
136+
137+
export async function createWorkspaceVirtualEnvLocator(root: string): Promise<IDisposableLocator> {
138+
const locator = new WorkspaceVirtualEnvironmentLocator(root);
139+
await locator.initialize();
140+
return locator;
141+
}

src/client/pythonEnvironments/index.ts

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ import {
1111
IDisposableLocator, IPythonEnvsIterator, PythonLocatorQuery
1212
} from './base/locator';
1313
import { CachingLocator } from './base/locators/composite/cachingLocator';
14-
import { WorkspaceVirtualEnvironmentLocator } from './base/locators/lowLevel/workspaceVirtualEnvLocator';
1514
import { PythonEnvsChangedEvent } from './base/watcher';
1615
import { getGlobalPersistentStore, initializeExternalDependencies as initializeLegacyExternalDependencies } from './common/externalDependencies';
1716
import { ExtensionLocators, WorkspaceLocators } from './discovery/locators';
@@ -101,7 +100,6 @@ async function initLocators(): Promise<ExtensionLocators> {
101100

102101
const workspaceLocators = new WorkspaceLocators([
103102
// Add an ILocator factory func here for each kind of workspace-rooted locator.
104-
(root: vscode.Uri) => [new WorkspaceVirtualEnvironmentLocator(root.fsPath)],
105103
]);
106104

107105
// Any non-workspace locator activation goes here.
Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
// eslint-disable-next-line max-classes-per-file
5+
import { assert } from 'chai';
6+
import * as fs from 'fs-extra';
7+
import * as path from 'path';
8+
import { traceWarning } from '../../../../../client/common/logger';
9+
import { FileChangeType } from '../../../../../client/common/platform/fileSystemWatcher';
10+
import { createDeferred, Deferred, sleep } from '../../../../../client/common/utils/async';
11+
import { getOSType, OSType } from '../../../../../client/common/utils/platform';
12+
import { IDisposableLocator } from '../../../../../client/pythonEnvironments/base/locator';
13+
import { createWorkspaceVirtualEnvLocator } from '../../../../../client/pythonEnvironments/base/locators/lowLevel/workspaceVirtualEnvLocator';
14+
import { getEnvs } from '../../../../../client/pythonEnvironments/base/locatorUtils';
15+
import { PythonEnvsChangedEvent } from '../../../../../client/pythonEnvironments/base/watcher';
16+
import { getInterpreterPathFromDir } from '../../../../../client/pythonEnvironments/common/commonUtils';
17+
import { arePathsSame } from '../../../../../client/pythonEnvironments/common/externalDependencies';
18+
import { deleteFiles, PYTHON_PATH } from '../../../../common';
19+
import { TEST_TIMEOUT } from '../../../../constants';
20+
import { TEST_LAYOUT_ROOT } from '../../../common/commonTestConstants';
21+
import { run } from '../../../discovery/locators/envTestUtils';
22+
23+
class WorkspaceVenvs {
24+
constructor(private readonly root: string, private readonly prefix = '.virtual') { }
25+
26+
public async create(name: string): Promise<string> {
27+
const envName = this.resolve(name);
28+
const argv = [PYTHON_PATH.fileToCommandArgument(), '-m', 'virtualenv', envName];
29+
try {
30+
await run(argv, { cwd: this.root });
31+
} catch (err) {
32+
throw new Error(`Failed to create Env ${path.basename(envName)} Error: ${err}`);
33+
}
34+
const dirToLookInto = path.join(this.root, envName);
35+
const filename = await getInterpreterPathFromDir(dirToLookInto);
36+
if (!filename) {
37+
throw new Error(`No environment to update exists in ${dirToLookInto}`);
38+
}
39+
return filename;
40+
}
41+
42+
/**
43+
* Creates a dummy environment by creating a fake executable.
44+
* @param name environment suffix name to create
45+
*/
46+
public async createDummyEnv(name: string): Promise<string> {
47+
const envName = this.resolve(name);
48+
const filepath = path.join(this.root, envName, getOSType() === OSType.Windows ? 'python.exe' : 'python');
49+
try {
50+
await fs.createFile(filepath);
51+
} catch (err) {
52+
throw new Error(`Failed to create python executable ${filepath}, Error: ${err}`);
53+
}
54+
return filepath;
55+
}
56+
57+
// eslint-disable-next-line class-methods-use-this
58+
public async update(filename: string): Promise<void> {
59+
try {
60+
await fs.writeFile(filename, 'Environment has been updated');
61+
} catch (err) {
62+
throw new Error(`Failed to update Workspace virtualenv executable ${filename}, Error: ${err}`);
63+
}
64+
}
65+
66+
// eslint-disable-next-line class-methods-use-this
67+
public async delete(filename: string): Promise<void> {
68+
try {
69+
await fs.remove(filename);
70+
} catch (err) {
71+
traceWarning(`Failed to clean up ${filename}`);
72+
}
73+
}
74+
75+
public async cleanUp() {
76+
const globPattern = path.join(this.root, `${this.prefix}*`);
77+
await deleteFiles(globPattern);
78+
}
79+
80+
private resolve(name: string): string {
81+
// Ensure env is random to avoid conflicts in tests (corrupting test data)
82+
const now = new Date().getTime().toString().substr(-8);
83+
return `${this.prefix}${name}${now}`;
84+
}
85+
}
86+
87+
suite('WorkspaceVirtualEnvironment Locator', async () => {
88+
const testWorkspaceFolder = path.join(TEST_LAYOUT_ROOT, 'workspace', 'folder1');
89+
const workspaceVenvs = new WorkspaceVenvs(testWorkspaceFolder);
90+
let locator: IDisposableLocator;
91+
92+
async function waitForChangeToBeDetected(deferred: Deferred<void>) {
93+
const timeout = setTimeout(
94+
() => {
95+
clearTimeout(timeout);
96+
deferred.reject(new Error('Environment not detected'));
97+
},
98+
TEST_TIMEOUT,
99+
);
100+
await deferred.promise;
101+
}
102+
103+
async function isLocated(executable: string): Promise<boolean> {
104+
const items = await getEnvs(locator.iterEnvs());
105+
return items.some((item) => arePathsSame(item.executable.filename, executable));
106+
}
107+
108+
suiteSetup(async () => workspaceVenvs.cleanUp());
109+
110+
async function setupLocator(onChanged: (e: PythonEnvsChangedEvent) => Promise<void>) {
111+
locator = await createWorkspaceVirtualEnvLocator(testWorkspaceFolder);
112+
// Wait for watchers to get ready
113+
await sleep(1000);
114+
locator.onChanged(onChanged);
115+
}
116+
117+
teardown(async () => {
118+
await workspaceVenvs.cleanUp();
119+
locator.dispose();
120+
});
121+
122+
test('Detect a new environment', async () => {
123+
let actualEvent: PythonEnvsChangedEvent;
124+
const deferred = createDeferred<void>();
125+
await setupLocator(async (e) => {
126+
actualEvent = e;
127+
deferred.resolve();
128+
});
129+
130+
const executable = await workspaceVenvs.create('one');
131+
await waitForChangeToBeDetected(deferred);
132+
const isFound = await isLocated(executable);
133+
134+
assert.ok(isFound);
135+
// Detecting kind of virtual env depends on the file structure around the executable, so we need to wait before
136+
// attempting to verify it. Omitting that check as we can never deterministically say when it's ready to check.
137+
assert.deepEqual(actualEvent!.type, FileChangeType.Created, 'Wrong event emitted');
138+
});
139+
140+
test('Detect when an environment has been deleted', async () => {
141+
let actualEvent: PythonEnvsChangedEvent;
142+
const deferred = createDeferred<void>();
143+
const executable = await workspaceVenvs.create('one');
144+
// Wait before the change event has been sent. If both operations occur almost simultaneously no event is sent.
145+
await sleep(100);
146+
await setupLocator(async (e) => {
147+
actualEvent = e;
148+
deferred.resolve();
149+
});
150+
151+
// VSCode API has a limitation where it fails to fire event when environment folder is deleted directly:
152+
// https://github.com/microsoft/vscode/issues/110923
153+
// Using chokidar directly in tests work, but it has permission issues on Windows that you cannot delete a
154+
// folder if it has a subfolder that is being watched inside: https://github.com/paulmillr/chokidar/issues/422
155+
// Hence we test directly deleting the executable, and not the whole folder using `workspaceVenvs.cleanUp()`.
156+
await workspaceVenvs.delete(executable);
157+
await waitForChangeToBeDetected(deferred);
158+
const isFound = await isLocated(executable);
159+
160+
assert.notOk(isFound);
161+
assert.deepEqual(actualEvent!.type, FileChangeType.Deleted, 'Wrong event emitted');
162+
});
163+
164+
test('Detect when an environment has been updated', async () => {
165+
let actualEvent: PythonEnvsChangedEvent;
166+
const deferred = createDeferred<void>();
167+
// Create a dummy environment so we can update its executable later. We can't choose a real environment here.
168+
// Executables inside real environments can be symlinks, so writing on them can result in the real executable
169+
// being updated instead of the symlink.
170+
const executable = await workspaceVenvs.createDummyEnv('one');
171+
// Wait before the change event has been sent. If both operations occur almost simultaneously no event is sent.
172+
await sleep(100);
173+
await setupLocator(async (e) => {
174+
actualEvent = e;
175+
deferred.resolve();
176+
});
177+
178+
await workspaceVenvs.update(executable);
179+
await waitForChangeToBeDetected(deferred);
180+
const isFound = await isLocated(executable);
181+
182+
assert.ok(isFound);
183+
// Detecting kind of virtual env depends on the file structure around the executable, so we need to wait before
184+
// attempting to verify it. Omitting that check as we can never deterministically say when it's ready to check.
185+
assert.deepEqual(actualEvent!.type, FileChangeType.Changed, 'Wrong event emitted');
186+
});
187+
});

src/test/pythonEnvironments/base/locators/lowLevel/workspaceVirtualEnvLocator.unit.test.ts

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,17 +10,18 @@ import {
1010
PythonEnvKind,
1111
PythonReleaseLevel,
1212
PythonVersion,
13-
UNKNOWN_PYTHON_VERSION
13+
UNKNOWN_PYTHON_VERSION,
1414
} from '../../../../../client/pythonEnvironments/base/info';
15-
import { WorkspaceVirtualEnvironmentLocator } from '../../../../../client/pythonEnvironments/base/locators/lowLevel/workspaceVirtualEnvLocator';
15+
import { IDisposableLocator } from '../../../../../client/pythonEnvironments/base/locator';
16+
import { createWorkspaceVirtualEnvLocator } from '../../../../../client/pythonEnvironments/base/locators/lowLevel/workspaceVirtualEnvLocator';
1617
import { getEnvs } from '../../../../../client/pythonEnvironments/base/locatorUtils';
1718
import { TEST_LAYOUT_ROOT } from '../../../common/commonTestConstants';
1819
import { assertEnvEqual, assertEnvsEqual } from '../../../discovery/locators/envTestUtils';
1920

2021
suite('WorkspaceVirtualEnvironment Locator', () => {
2122
const testWorkspaceFolder = path.join(TEST_LAYOUT_ROOT, 'workspace', 'folder1');
2223
let getOSTypeStub: sinon.SinonStub;
23-
let locator: WorkspaceVirtualEnvironmentLocator;
24+
let locator: IDisposableLocator;
2425

2526
function createExpectedEnvInfo(
2627
interpreterPath: string,
@@ -53,10 +54,10 @@ suite('WorkspaceVirtualEnvironment Locator', () => {
5354
assert.deepStrictEqual(actualPaths, expectedPaths);
5455
}
5556

56-
setup(() => {
57+
setup(async () => {
5758
getOSTypeStub = sinon.stub(platformUtils, 'getOSType');
5859
getOSTypeStub.returns(platformUtils.OSType.Linux);
59-
locator = new WorkspaceVirtualEnvironmentLocator(testWorkspaceFolder);
60+
locator = await createWorkspaceVirtualEnvLocator(testWorkspaceFolder);
6061
});
6162
teardown(() => {
6263
getOSTypeStub.restore();

0 commit comments

Comments
 (0)