Properly parse multi-digit Python "versions". (#14878)

The main focus of the PR is the helpers needed to fix the situation where the version for Anaconda is getting reported as "38.0.0".  There isn't much information about the original failure, so more changes may be needed.

Author: ericsnowcurrently

src/client/pythonEnvironments/base/info/pythonVersion.ts

@@ -1,16 +1,21 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-import { cloneDeep } from 'lodash';
 import * as path from 'path';
-import { PythonReleaseLevel, PythonVersion, UNKNOWN_PYTHON_VERSION } from '.';
 import { traceError } from '../../../common/logger';
 import {
     EMPTY_VERSION,
     isVersionInfoEmpty,
     parseBasicVersionInfo,
 } from '../../../common/utils/version';
 
+import {
+    PythonReleaseLevel,
+    PythonVersion,
+    PythonVersionRelease,
+    UNKNOWN_PYTHON_VERSION,
+} from '.';
+
 export function getPythonVersionFromPath(exe:string): PythonVersion {
     let version = UNKNOWN_PYTHON_VERSION;
     try {
@@ -23,84 +28,100 @@ export function getPythonVersionFromPath(exe:string): PythonVersion {
 
 /**
  * Convert the given string into the corresponding Python version object.
+ *
  * Example:
  *   3.9.0
  *   3.9.0a1
  *   3.9.0b2
  *   3.9.0rc1
- *
- * Does not parse:
+ *   3.9.0-beta2
+ *   3.9.0.beta.2
  *   3.9.0.final.0
+ *   39
  */
 export function parseVersion(versionStr: string): PythonVersion {
-    const parsed = parseBasicVersionInfo<PythonVersion>(versionStr);
-    if (!parsed) {
-        if (versionStr === '') {
-            return EMPTY_VERSION as PythonVersion;
-        }
-        throw Error(`invalid version ${versionStr}`);
+    const [version, after] = parseBasicVersion(versionStr);
+    if (version.micro === -1) {
+        return version;
     }
-    const { version, after } = parsed;
-    const match = after.match(/^(a|b|rc)(\d+)$/);
+    const [release] = parseRelease(after);
+    version.release = release;
+    return version;
+}
+
+export function parseRelease(text: string): [PythonVersionRelease | undefined, string] {
+    let after: string;
+
+    let alpha: string | undefined;
+    let beta: string | undefined;
+    let rc: string | undefined;
+    let fin: string | undefined;
+    let serialStr: string;
+
+    let match = text.match(/^(?:-?final|\.final(?:\.0)?)(.*)$/);
     if (match) {
-        const [, levelStr, serialStr] = match;
-        let level: PythonReleaseLevel;
-        if (levelStr === 'a') {
-            level = PythonReleaseLevel.Alpha;
-        } else if (levelStr === 'b') {
-            level = PythonReleaseLevel.Beta;
-        } else if (levelStr === 'rc') {
-            level = PythonReleaseLevel.Candidate;
-        } else {
-            throw Error('unreachable!');
+        [, after] = match;
+        fin = 'final';
+        serialStr = '0';
+    } else {
+        for (const regex of [
+            /^(?:(a)|(b)|(rc))([1-9]\d*)(.*)$/,
+            /^-(?:(?:(alpha)|(beta)|(candidate))([1-9]\d*))(.*)$/,
+            /^\.(?:(?:(alpha)|(beta)|(candidate))\.([1-9]\d*))(.*)$/,
+        ]) {
+            match = text.match(regex);
+            if (match) {
+                [, alpha, beta, rc, serialStr, after] = match;
+                break;
+            }
         }
-        version.release = {
-            level,
-            serial: parseInt(serialStr, 10),
-        };
     }
-    return version;
+
+    let level: PythonReleaseLevel;
+    if (fin) {
+        level = PythonReleaseLevel.Final;
+    } else if (rc) {
+        level = PythonReleaseLevel.Candidate;
+    } else if (beta) {
+        level = PythonReleaseLevel.Beta;
+    } else if (alpha) {
+        level = PythonReleaseLevel.Alpha;
+    } else {
+        // We didn't find release info.
+        return [undefined, text];
+    }
+    const serial = parseInt(serialStr!, 10);
+    return [{ level, serial }, after!];
 }
 
 /**
  * Convert the given string into the corresponding Python version object.
- * Example:
- *   3.9.0.final.0
- *   3.9.0.alpha.1
- *   3.9.0.beta.2
- *   3.9.0.candidate.1
- *
- * Does not parse:
- *   3.9.0
- *   3.9.0a1
- *   3.9.0b2
- *   3.9.0rc1
  */
-export function parseVersionInfo(versionInfoStr: string): PythonVersion {
-    const parts = versionInfoStr.split('.');
-    const version = cloneDeep(UNKNOWN_PYTHON_VERSION);
-    if (parts.length >= 2) {
-        version.major = parseInt(parts[0], 10);
-        version.minor = parseInt(parts[1], 10);
-    }
-
-    if (parts.length >= 3) {
-        version.micro = parseInt(parts[2], 10);
-    }
-
-    if (parts.length >= 4 && version.release) {
-        const levels = ['alpha', 'beta', 'candidate', 'final'];
-        const level = parts[3].toLowerCase();
-        if (levels.includes(level)) {
-            version.release.level = level as PythonReleaseLevel;
+export function parseBasicVersion(versionStr: string): [PythonVersion, string] {
+    // We set a prefix (which will be ignored) to make sure "plain"
+    // versions are fully parsed.
+    const parsed = parseBasicVersionInfo<PythonVersion>(`ignored-${versionStr}`);
+    if (!parsed) {
+        if (versionStr === '') {
+            return [getEmptyVersion(), ''];
         }
+        throw Error(`invalid version ${versionStr}`);
     }
+    // We ignore any "before" text.
+    const { version, after } = parsed;
+    version.release = undefined;
 
-    if (parts.length >= 5 && version.release) {
-        version.release.serial = parseInt(parts[4], 10);
+    if (version.minor === -1) {
+        // We trust that the major version is always single-digit.
+        if (version.major > 9) {
+            const numdigits = version.major.toString().length - 1;
+            const factor = 10 ** numdigits;
+            version.minor = version.major % factor;
+            version.major = Math.floor(version.major / factor);
+        }
     }
 
-    return version;
+    return [version, after];
 }
 
 /**

src/client/pythonEnvironments/discovery/locators/services/virtualEnvironmentIdentifier.ts

@@ -5,11 +5,15 @@ import * as fsapi from 'fs-extra';
 import * as path from 'path';
 import '../../../../common/extensions';
 import {
-    getEnvironmentVariable, getOSType, getUserHomeDir, OSType
+    getEnvironmentVariable, getOSType, getUserHomeDir, OSType,
 } from '../../../../common/utils/platform';
 import { PythonVersion, UNKNOWN_PYTHON_VERSION } from '../../../base/info';
 import { comparePythonVersionSpecificity } from '../../../base/info/env';
-import { parseVersion, parseVersionInfo } from '../../../base/info/pythonVersion';
+import {
+    parseBasicVersion,
+    parseRelease,
+    parseVersion,
+} from '../../../base/info/pythonVersion';
 import { pathExists, readFile } from '../../../common/externalDependencies';
 
 function getPyvenvConfigPathsFrom(interpreterPath:string): string[] {
@@ -178,3 +182,32 @@ export async function getPythonVersionFromPyvenvCfg(interpreterPath:string): Pro
 
     return version;
 }
+
+/**
+ * Convert the given string into the corresponding Python version object.
+ * Example:
+ *   3.9.0.final.0
+ *   3.9.0.alpha.1
+ *   3.9.0.beta.2
+ *   3.9.0.candidate.1
+ *
+ * Does not parse:
+ *   3.9.0
+ *   3.9.0a1
+ *   3.9.0b2
+ *   3.9.0rc1
+ */
+function parseVersionInfo(versionInfoStr: string): PythonVersion {
+    let version: PythonVersion;
+    let after: string;
+    try {
+        [version, after] = parseBasicVersion(versionInfoStr);
+    } catch {
+        // XXX Use getEmptyVersion().
+        return UNKNOWN_PYTHON_VERSION;
+    }
+    if (version.micro !== -1 && after.startsWith('.')) {
+        [version.release] = parseRelease(after);
+    }
+    return version;
+}

src/test/pythonEnvironments/base/info/pythonVersion.unit.test.ts

@@ -0,0 +1,152 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import * as assert from 'assert';
+
+import { PythonReleaseLevel, PythonVersion } from '../../../../client/pythonEnvironments/base/info';
+import {
+    getEmptyVersion,
+    parseVersion,
+} from '../../../../client/pythonEnvironments/base/info/pythonVersion';
+
+export function ver(
+    major: number,
+    minor: number | undefined,
+    micro: number | undefined,
+    level?: string,
+    serial?: number,
+): PythonVersion {
+    const version: PythonVersion = {
+        major,
+        minor: minor === undefined ? -1 : minor,
+        micro: micro === undefined ? -1 : micro,
+        release: undefined,
+    };
+    if (level !== undefined) {
+        version.release = {
+            serial: serial!,
+            level: level as PythonReleaseLevel,
+        };
+    }
+    return version;
+}
+
+const VERSION_STRINGS: [string, PythonVersion][] = [
+    ['0.9.2b2', ver(0, 9, 2, 'beta', 2)],
+    ['3.3.1', ver(3, 3, 1)], // final
+    ['3.9.0rc1', ver(3, 9, 0, 'candidate', 1)],
+    ['2.7.11a3', ver(2, 7, 11, 'alpha', 3)],
+];
+
+suite('pyenvs info - parseVersion', () => {
+    suite('full versions (short)', () => {
+        VERSION_STRINGS.forEach((data) => {
+            const [text, expected] = data;
+            test(`conversion works for '${text}'`, () => {
+                const result = parseVersion(text);
+
+                assert.deepEqual(result, expected);
+            });
+        });
+    });
+
+    suite('full versions (long)', () => {
+        [
+            ['0.9.2-beta2', ver(0, 9, 2, 'beta', 2)],
+            ['3.3.1-final', ver(3, 3, 1, 'final', 0)],
+            ['3.3.1-final0', ver(3, 3, 1, 'final', 0)],
+            ['3.9.0-candidate1', ver(3, 9, 0, 'candidate', 1)],
+            ['2.7.11-alpha3', ver(2, 7, 11, 'alpha', 3)],
+            ['0.9.2.beta.2', ver(0, 9, 2, 'beta', 2)],
+            ['3.3.1.final.0', ver(3, 3, 1, 'final', 0)],
+            ['3.9.0.candidate.1', ver(3, 9, 0, 'candidate', 1)],
+            ['2.7.11.alpha.3', ver(2, 7, 11, 'alpha', 3)],
+        ].forEach((data) => {
+            const [text, expected] = data as [string, PythonVersion];
+            test(`conversion works for '${text}'`, () => {
+                const result = parseVersion(text);
+
+                assert.deepEqual(result, expected);
+            });
+        });
+    });
+
+    suite('partial versions', () => {
+        [
+            ['3.7.1', ver(3, 7, 1)],
+            ['3.7', ver(3, 7, -1)],
+            ['3', ver(3, -1, -1)],
+            ['37', ver(3, 7, -1)], // not 37
+            ['371', ver(3, 71, -1)], // not 3.7.1
+            ['3102', ver(3, 102, -1)], // not 3.10.2
+            ['2.7', ver(2, 7, -1)],
+            ['2', ver(2, -1, -1)], // not 2.7
+            ['27', ver(2, 7, -1)],
+        ].forEach((data) => {
+            const [text, expected] = data as [string, PythonVersion];
+            test(`conversion works for '${text}'`, () => {
+                const result = parseVersion(text);
+
+                assert.deepEqual(result, expected);
+            });
+        });
+    });
+
+    suite('other forms', () => {
+        [
+            // prefixes
+            ['python3', ver(3, -1, -1)],
+            ['python3.8', ver(3, 8, -1)],
+            ['python3.8.1', ver(3, 8, 1)],
+            ['python3.8.1b2', ver(3, 8, 1, 'beta', 2)],
+            ['python-3', ver(3, -1, -1)],
+            // release ignored (missing micro)
+            ['python3.8b2', ver(3, 8, -1)],
+            ['python38b2', ver(3, 8, -1)],
+            ['python381b2', ver(3, 81, -1)], // not 3.8.1
+            // suffixes
+            ['python3.exe', ver(3, -1, -1)],
+            ['python3.8.exe', ver(3, 8, -1)],
+            ['python3.8.1.exe', ver(3, 8, 1)],
+            ['python3.8.1b2.exe', ver(3, 8, 1, 'beta', 2)],
+            ['3.8.1.build123.revDEADBEEF', ver(3, 8, 1)],
+            ['3.8.1b2.build123.revDEADBEEF', ver(3, 8, 1, 'beta', 2)],
+            // dirnames
+            ['/x/y/z/python38/bin/python', ver(3, 8, -1)],
+            ['/x/y/z/python/38/bin/python', ver(3, 8, -1)],
+            ['/x/y/z/python/38/bin/python', ver(3, 8, -1)],
+        ].forEach((data) => {
+            const [text, expected] = data as [string, PythonVersion];
+            test(`conversion works for '${text}'`, () => {
+                const result = parseVersion(text);
+
+                assert.deepEqual(result, expected);
+            });
+        });
+    });
+
+    test('empty string results in empty version', () => {
+        const expected = getEmptyVersion();
+
+        const result = parseVersion('');
+
+        assert.deepEqual(result, expected);
+    });
+
+    suite('bogus input', () => {
+        [
+            // errant dots
+            'py.3.7',
+            'py3.7.',
+            'python.3',
+            // no version
+            'spam',
+            'python.exe',
+            'python',
+        ].forEach((text) => {
+            test(`conversion does not work for '${text}'`, () => {
+                assert.throws(() => parseVersion(text));
+            });
+        });
+    });
+});