From f2a93f1a6ba7952df18e9727150cea4a7754e521 Mon Sep 17 00:00:00 2001 From: Nils Werner Date: Mon, 18 May 2020 00:10:02 -0400 Subject: [PATCH 1/6] Failed test --- package-lock.json | 27 ++++++++ src/docstring/index.ts | 1 + src/docstring/parse_docstring.ts | 11 ++++ src/test/docstring/parse_docstring.spec.ts | 72 ++++++++++++++++++++++ 4 files changed, 111 insertions(+) create mode 100644 src/docstring/parse_docstring.ts create mode 100644 src/test/docstring/parse_docstring.spec.ts diff --git a/package-lock.json b/package-lock.json index 4a1270f..8c4d915 100644 --- a/package-lock.json +++ b/package-lock.json @@ -177,6 +177,11 @@ "wrap-ansi": "^5.1.0" } }, + "clone": { + "version": "0.1.19", + "resolved": "https://registry.npmjs.org/clone/-/clone-0.1.19.tgz", + "integrity": "sha1-YT+2hjmyaklKxTJT4Vsaa9iK2oU=" + }, "color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -783,6 +788,28 @@ "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", "dev": true }, + "reverse-mustache": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/reverse-mustache/-/reverse-mustache-1.9.0.tgz", + "integrity": "sha1-bRwp8ssPXTF5avk4B1ClHrtQIwY=", + "requires": { + "clone": "~0.1.11", + "mustache": "~2.3.0", + "pathval": "~0.1.1" + }, + "dependencies": { + "mustache": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-2.3.2.tgz", + "integrity": "sha512-KpMNwdQsYz3O/SBS1qJ/o3sqUJ5wSb8gb0pul8CO0S56b9Y2ALm8zCfsjPXsqGFfoNBkDwZuZIAjhsZI03gYVQ==" + }, + "pathval": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-0.1.1.tgz", + "integrity": "sha1-CPkRzcqczllCiA2ngXvAtyO2bYI=" + } + } + }, "rimraf": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", diff --git a/src/docstring/index.ts b/src/docstring/index.ts index ad7bd54..06e4383 100644 --- a/src/docstring/index.ts +++ b/src/docstring/index.ts @@ -1,2 +1,3 @@ export { DocstringFactory } from "./docstring_factory"; export { getTemplate } from "./get_template"; +export { parseDocstring } from "./parse_docstring"; diff --git a/src/docstring/parse_docstring.ts b/src/docstring/parse_docstring.ts new file mode 100644 index 0000000..b72de1f --- /dev/null +++ b/src/docstring/parse_docstring.ts @@ -0,0 +1,11 @@ +// import { reverseMustache } from "reverse-mustache"; +const reverseMustache = require("reverse-mustache"); + +export function parseDocstring(docstring: string, template: string) { + const data = reverseMustache({ + template: template, + content: docstring, + }); + + console.log(data); +} diff --git a/src/test/docstring/parse_docstring.spec.ts b/src/test/docstring/parse_docstring.spec.ts new file mode 100644 index 0000000..382f031 --- /dev/null +++ b/src/test/docstring/parse_docstring.spec.ts @@ -0,0 +1,72 @@ +import chai = require("chai"); +import "mocha"; + +import { parseDocstring, getTemplate } from "../../docstring"; + +chai.config.truncateThreshold = 0; +const expect = chai.expect; + +it.only("should return the string containing the google mustache template", () => { + const template = getTemplate("google"); + parseDocstring(googleDocstring, template); + + // parseDocstring("world", "{{#place}}{{name}}{{/place}}"); + parseDocstring(testString, testTemplate); +}); + +// describe("getTemplate()", () => { +// context("when asked for google template", () => { +// it("should return the string containing the google mustache template", () => { +// const result = getTemplate("google"); + +// expect(result).to.contain("Google Docstring Template"); +// }); +// }); + +// context("when asked for sphinx template", () => { +// it("should return the string containing the sphinx mustache template", () => { +// const result = getTemplate("sphinx"); + +// expect(result).to.contain("Sphinx Docstring Template"); +// }); +// }); + +// context("when asked for sphinx template", () => { +// it("should return the string containing the numpy mustache template", () => { +// const result = getTemplate("numpy"); + +// expect(result).to.contain("Numpy Docstring Template"); +// }); +// }); + +// context("when asked for anything else", () => { +// it("should return the string containing the default mustache template", () => { +// const result = getTemplate("blah"); +// const result2 = getTemplate("default"); + +// expect(result).to.contain("Default Docstring Template"); +// expect(result2).to.contain("Default Docstring Template"); +// }); +// }); +// }); + +const googleDocstring = ` +[summary] + +Args: + a (int): [description] + b (str): [description] + c (list, optional): [description]. Defaults to [1,2]. + +Raises: + EnvironmentError: [description] + ArithmeticError: [description] + +Returns: + [type]: [description] +`; + +const testTemplate = `{{#args}}{{var}} +{{/args}}`; + +const testString = `a a`; From fa481309cd006329f966743f9000559819a6f32e Mon Sep 17 00:00:00 2001 From: Nils Werner Date: Mon, 18 May 2020 09:48:30 -0400 Subject: [PATCH 2/6] WIP --- package-lock.json | 18 ++++++++ src/docstring/parse_docstring.ts | 10 ++--- src/test/docstring/parse_docstring.spec.ts | 51 +++++++--------------- 3 files changed, 38 insertions(+), 41 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8c4d915..2978b6e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,11 @@ "integrity": "sha512-t7uW6eFafjO+qJ3BIV2gGUyZs27egcNRkUdalkud+Qa3+kg//f129iuOFivHDXQ+vnU3fDXuwgv0cqMCbcE8sw==", "dev": true }, + "@types/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-mIenTfsIe586/yzsyfql69KRnA75S8SVXQbTLpDejRrjH0QSJcpu3AUOi/Vjnt9IOsXKxPhJfGpQUNMueIU1fQ==" + }, "@types/mocha": { "version": "5.2.7", "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-5.2.7.tgz", @@ -485,6 +490,11 @@ "integrity": "sha512-USlDT524woQ08aoZFzh3/Z6ch9Y/EWXEHQ/AaRN0SkKq4t2Jw2R2339tSXmwuVoY7LLlBCbOIlx2myP/L5zk0g==", "dev": true }, + "is-equal": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/is-equal/-/is-equal-0.1.0.tgz", + "integrity": "sha1-/Pbg/cncqaTUdouuc7jNqls/q24=" + }, "is-fullwidth-code-point": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", @@ -531,6 +541,14 @@ "esprima": "^4.0.0" } }, + "jsdiff": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/jsdiff/-/jsdiff-1.1.1.tgz", + "integrity": "sha1-6qbnwW259kuXnSj5hvocpFeyIuU=", + "requires": { + "is-equal": "~0.1.0" + } + }, "locate-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", diff --git a/src/docstring/parse_docstring.ts b/src/docstring/parse_docstring.ts index b72de1f..01c70bd 100644 --- a/src/docstring/parse_docstring.ts +++ b/src/docstring/parse_docstring.ts @@ -1,11 +1,9 @@ // import { reverseMustache } from "reverse-mustache"; -const reverseMustache = require("reverse-mustache"); +// const reverseMustache = require("reverse-mustache"); +import { diffLines } from "Diff"; -export function parseDocstring(docstring: string, template: string) { - const data = reverseMustache({ - template: template, - content: docstring, - }); +export function parseDocstring(oldDocstring: string, newDocstring: string) { + const data = diffLines(oldDocstring, newDocstring); console.log(data); } diff --git a/src/test/docstring/parse_docstring.spec.ts b/src/test/docstring/parse_docstring.spec.ts index 382f031..5a111b7 100644 --- a/src/test/docstring/parse_docstring.spec.ts +++ b/src/test/docstring/parse_docstring.spec.ts @@ -7,48 +7,29 @@ chai.config.truncateThreshold = 0; const expect = chai.expect; it.only("should return the string containing the google mustache template", () => { - const template = getTemplate("google"); - parseDocstring(googleDocstring, template); + // const template = getTemplate("google"); + // parseDocstring(googleDocstring, template); // parseDocstring("world", "{{#place}}{{name}}{{/place}}"); - parseDocstring(testString, testTemplate); + parseDocstring(oldDocstring, newDocstring); }); -// describe("getTemplate()", () => { -// context("when asked for google template", () => { -// it("should return the string containing the google mustache template", () => { -// const result = getTemplate("google"); - -// expect(result).to.contain("Google Docstring Template"); -// }); -// }); - -// context("when asked for sphinx template", () => { -// it("should return the string containing the sphinx mustache template", () => { -// const result = getTemplate("sphinx"); - -// expect(result).to.contain("Sphinx Docstring Template"); -// }); -// }); - -// context("when asked for sphinx template", () => { -// it("should return the string containing the numpy mustache template", () => { -// const result = getTemplate("numpy"); +const oldDocstring = ` +[summary] -// expect(result).to.contain("Numpy Docstring Template"); -// }); -// }); +Args: + a (int): [description] + b (str): abcdefg hijk +`; -// context("when asked for anything else", () => { -// it("should return the string containing the default mustache template", () => { -// const result = getTemplate("blah"); -// const result2 = getTemplate("default"); +const newDocstring = ` +[summary] -// expect(result).to.contain("Default Docstring Template"); -// expect(result2).to.contain("Default Docstring Template"); -// }); -// }); -// }); +Args: + a (int): [description] + c (list, optional): [description]. Defaults to [1,2]. + d (str): abcdefg hijk +`; const googleDocstring = ` [summary] From cdfc140f63ae15c30f2ffbe252d470272df0804e Mon Sep 17 00:00:00 2001 From: Nils Werner Date: Tue, 2 Jun 2020 23:25:44 -0400 Subject: [PATCH 3/6] wip --- src/docstring/parse_docstring.ts | 204 ++++++++++++++++++++- src/test/docstring/parse_docstring.spec.ts | 105 ++++++++--- 2 files changed, 280 insertions(+), 29 deletions(-) diff --git a/src/docstring/parse_docstring.ts b/src/docstring/parse_docstring.ts index 01c70bd..42634f7 100644 --- a/src/docstring/parse_docstring.ts +++ b/src/docstring/parse_docstring.ts @@ -1,9 +1,207 @@ // import { reverseMustache } from "reverse-mustache"; // const reverseMustache = require("reverse-mustache"); import { diffLines } from "Diff"; +import { Argument, DocstringParts } from "../docstring_parts"; -export function parseDocstring(oldDocstring: string, newDocstring: string) { - const data = diffLines(oldDocstring, newDocstring); +// Need to deal with 1. multiline regex 2. indentation 3. description 4. check crlf +// 5. sections in different orders 6. A section is not there - console.log(data); +export function parseDocstring(oldDocstring: string, template: string) { + const docstringLines = oldDocstring.split("\n"); + const templateLines = template.split("\n"); + + const argRegex = getBlockRegex(template, "args"); + const kwargRegex = getBlockRegex(template, "kwargs"); + const exceptionRegex = getBlockRegex(template, "exceptions"); + const returnRegex = getBlockRegex(template, "returns"); + const yieldRegex = getBlockRegex(template, "yields"); + + console.log(yieldRegex); + + const argRegexLineCount = argRegex.toString().split("\\n").length; + const kwargRegexLineCount = kwargRegex.toString().split("\\n").length; + const exceptionRegexLineCount = exceptionRegex.toString().split("\\n").length; + const returnRegexLineCount = returnRegex.toString().split("\\n").length; + const yieldRegexLineCount = yieldRegex.toString().split("\\n").length; + + const docstringParts = emptyDocstringParts(); + + let j = 0; + for (let i = 0; i < templateLines.length; i++) { + // Advance docstring and template if lines match + if (templateLines[i] == docstringLines[j]) { + j++; + continue; + } + + // Start parsing args if the start arg block is reached + if (/{{#args}}/.test(templateLines[i])) { + // Keep parsing arg lines until lines no longer match arg regex or they match kwarg pattern + while (j <= docstringLines.length) { + const lines = docstringLines.slice(j, j + argRegexLineCount).join("\n"); + + if (!argRegex.test(lines) || kwargRegex.test(lines)) { + break; + } + + const match = lines.match(argRegex); + docstringParts.args.push({ + var: match.groups.var, + type: match.groups.type, + }); + + j += argRegexLineCount; + } + + // Advance to end of arg block + while (!/{{\/args}}/.test(templateLines[i]) && i < templateLines.length) { + i++; + } + } + + // Start parsing kwargs if the start kwarg block is reached + if (/{{#kwargs}}/.test(templateLines[i])) { + while (j <= docstringLines.length) { + const lines = docstringLines.slice(j, j + kwargRegexLineCount).join("\n"); + + if (!kwargRegex.test(lines)) { + break; + } + + const match = lines.match(kwargRegex); + docstringParts.kwargs.push({ + var: match.groups.var, + type: match.groups.type, + default: match.groups.default, + }); + + j += kwargRegexLineCount; + } + + // Advance to end of kwarg block + while (!/{{\/kwargs}}/.test(templateLines[i]) && i < templateLines.length) { + i++; + } + } + + if (/{{#exceptions}}/.test(templateLines[i])) { + while (j <= docstringLines.length) { + const lines = docstringLines.slice(j, j + exceptionRegexLineCount).join("\n"); + + if (!exceptionRegex.test(lines)) { + break; + } + + const match = lines.match(exceptionRegex); + docstringParts.exceptions.push({ + type: match.groups.type, + }); + + j += exceptionRegexLineCount; + } + + // Advance to end of kwarg block + while (!/{{\/exceptions}}/.test(templateLines[i]) && i < templateLines.length) { + i++; + } + } + + if (/{{#returns}}/.test(templateLines[i])) { + while (j <= docstringLines.length) { + const lines = docstringLines.slice(j, j + returnRegexLineCount).join("\n"); + console.log(lines); + if (!returnRegex.test(lines)) { + break; + } + + const match = lines.match(returnRegex); + docstringParts.returns = { + type: match.groups.type, + }; + + j += returnRegexLineCount; + } + + // Advance to end of kwarg block + while (!/{{\/returns}}/.test(templateLines[i]) && i < templateLines.length) { + i++; + } + } + + if (/{{#yields}}/.test(templateLines[i])) { + while (j <= docstringLines.length) { + const lines = docstringLines.slice(j, j + yieldRegexLineCount).join("\n"); + console.log(lines); + if (!yieldRegex.test(lines)) { + break; + } + + const match = lines.match(yieldRegex); + docstringParts.yields = { + type: match.groups.type, + }; + + j += yieldRegexLineCount; + } + + // Advance to end of kwarg block + while (!/{{\/yields}}/.test(templateLines[i]) && i < templateLines.length) { + i++; + } + } + } + + console.log(docstringParts); +} + +function getBlockRegex(template: string, block: string): RegExp { + const blockStartTag = `{{#${block}}}`; + const blockEndTag = `{{/${block}}}`; + const blockRegex = new RegExp(blockStartTag + "(.*)" + blockEndTag, "s"); + + const match = template.match(blockRegex); + let pattern = match[1]; + + // Escape all characters that can be misidentified in regex + pattern = escapeRegExp(pattern); + + // Replace all tags with named regex capture groups + pattern = replaceTags(pattern, "{{var}}", "\\w+", "var"); + pattern = replaceTags(pattern, "{{typePlaceholder}}", "[\\w\\[\\], ]+", "type"); + pattern = replaceTags(pattern, "{{descriptionPlaceholder}}", ".*", "description"); + pattern = replaceTags(pattern, "{{&default}}", ".+", "default"); + pattern = replaceTags(pattern, "{{type}}", "\\w+", "type"); + + pattern = pattern.trim(); + pattern = pattern.replace(/\n/g, "\\n\\s*"); + + return new RegExp(pattern.trim()); +} + +function replaceTags(str: string, tag: string, pattern: string, captureGroupName: string): string { + tag = escapeRegExp(tag); + str = str.replace(tag, `(?<${captureGroupName}>${pattern})`); + + // Replace remaining matches with non capturing pattern + while (new RegExp(escapeRegExp(tag)).test(str)) { + str = str.replace(tag, pattern); + } + + return str; +} + +function escapeRegExp(str: string) { + return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); // $& means the whole matched string +} + +function emptyDocstringParts(): DocstringParts { + return { + name: "", + args: [], + kwargs: [], + decorators: [], + exceptions: [], + yields: null, + returns: null, + }; } diff --git a/src/test/docstring/parse_docstring.spec.ts b/src/test/docstring/parse_docstring.spec.ts index 5a111b7..ba871b6 100644 --- a/src/test/docstring/parse_docstring.spec.ts +++ b/src/test/docstring/parse_docstring.spec.ts @@ -6,48 +6,101 @@ import { parseDocstring, getTemplate } from "../../docstring"; chai.config.truncateThreshold = 0; const expect = chai.expect; -it.only("should return the string containing the google mustache template", () => { +it.only("google", () => { // const template = getTemplate("google"); // parseDocstring(googleDocstring, template); // parseDocstring("world", "{{#place}}{{name}}{{/place}}"); - parseDocstring(oldDocstring, newDocstring); + parseDocstring(fullGoogleDocstring, googleTemplate); }); -const oldDocstring = ` -[summary] +it.only("sphinx", () => { + // const template = getTemplate("google"); + // parseDocstring(googleDocstring, template); + + // parseDocstring("world", "{{#place}}{{name}}{{/place}}"); + parseDocstring(fullSphinxDocstring, sphinxTemplate); +}); +const fullGoogleDocstring = ` Args: - a (int): [description] - b (str): abcdefg hijk -`; + arg1 ([type]): An argument. It is named arg1 + arg2 (Dict[str, int]): This is also an argument => a good one! + kwarg1 (int, optional): a kwarg this time. A really good one. Defaults to 1. -const newDocstring = ` -[summary] +Raises: + FileExistsError: Oh nej! + KeyError: bad things! -Args: - a (int): [description] - c (list, optional): [description]. Defaults to [1,2]. - d (str): abcdefg hijk -`; +Returns: + [type]: [description] -const googleDocstring = ` -[summary] +Yields: + [type]: [description] +"""`; +const googleTemplate = ` +{{#parametersExist}} Args: - a (int): [description] - b (str): [description] - c (list, optional): [description]. Defaults to [1,2]. +{{#args}} + {{var}} ({{typePlaceholder}}): {{descriptionPlaceholder}} +{{/args}} +{{#kwargs}} + {{var}} ({{typePlaceholder}}, optional): {{descriptionPlaceholder}}. Defaults to {{&default}}. +{{/kwargs}} +{{/parametersExist}} +{{#exceptionsExist}} Raises: - EnvironmentError: [description] - ArithmeticError: [description] +{{#exceptions}} + {{type}}: {{descriptionPlaceholder}} +{{/exceptions}} +{{/exceptionsExist}} +{{#returnsExist}} Returns: - [type]: [description] -`; +{{#returns}} + {{typePlaceholder}}: {{descriptionPlaceholder}} +{{/returns}} +{{/returnsExist}} + +{{#yieldsExist}} +Yields: +{{#yields}} + {{typePlaceholder}}: {{descriptionPlaceholder}} +{{/yields}} +{{/yieldsExist}}`; -const testTemplate = `{{#args}}{{var}} -{{/args}}`; +const fullSphinxDocstring = ` +:param arg1: [description] +:type arg1: [type] +:param arg2: [description] +:type arg2: [type] +:param kwarg1: [description], defaults to 1 +:type kwarg1: int, optional +:raises FileExistsError: [description] +:return: [description] +:rtype: [type] +:yield: [description] +:rtype: [type]`; -const testString = `a a`; +const sphinxTemplate = ` +{{#args}} +:param {{var}}: {{descriptionPlaceholder}} +:type {{var}}: {{typePlaceholder}} +{{/args}} +{{#kwargs}} +:param {{var}}: {{descriptionPlaceholder}}, defaults to {{&default}} +:type {{var}}: {{typePlaceholder}}, optional +{{/kwargs}} +{{#exceptions}} +:raises {{type}}: {{descriptionPlaceholder}} +{{/exceptions}} +{{#returns}} +:return: {{descriptionPlaceholder}} +:rtype: {{typePlaceholder}} +{{/returns}} +{{#yields}} +:yield: {{descriptionPlaceholder}} +:rtype: {{typePlaceholder}} +{{/yields}}`; From 3885c4f65575fdf66bb93ba4c8597733411c5b26 Mon Sep 17 00:00:00 2001 From: Nils Werner Date: Tue, 17 Nov 2020 21:22:34 -0500 Subject: [PATCH 4/6] wip --- .gitignore | 1 + package.json | 4 + src/constants.ts | 1 + src/docstring/parse_docstring.ts | 50 +++--- src/extension.ts | 18 +- src/generate_docstring.ts | 14 +- src/parse/docstring/get_docstring.ts | 13 ++ src/parse/docstring/get_docstring_range.ts | 61 +++++++ src/parse/docstring/normalize_docstring.ts | 42 +++++ src/parse/index.ts | 1 + src/parse/valid_docstring_prefix.ts | 2 +- src/test/docstring/parse_docstring.spec.ts | 33 +++- .../docstring/get_docstring_test.spec.ts | 164 ++++++++++++++++++ .../normalize_docstring_test.spec.ts | 84 +++++++++ 14 files changed, 462 insertions(+), 26 deletions(-) create mode 100644 src/parse/docstring/get_docstring.ts create mode 100644 src/parse/docstring/get_docstring_range.ts create mode 100644 src/parse/docstring/normalize_docstring.ts create mode 100644 src/test/parse/docstring/get_docstring_test.spec.ts create mode 100644 src/test/parse/docstring/normalize_docstring_test.spec.ts diff --git a/.gitignore b/.gitignore index 59b4bd4..c54d3d5 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ test_* .vscode/settings.json *.psd todo +scratch diff --git a/package.json b/package.json index f97f30b..6ec3bf7 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,10 @@ { "command": "autoDocstring.generateDocstring", "title": "Generate Docstring" + }, + { + "command": "autoDocstring.updateDocstring", + "title": "Update Docstring [beta]" } ], "keybindings": [ diff --git a/src/constants.ts b/src/constants.ts index 95a23c8..f44ec57 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -5,3 +5,4 @@ export const extensionRoot: ExtensionRoot = { path: "" }; export const debug = false; export const extensionID = "autoDocstring"; export const generateDocstringCommand = "autoDocstring.generateDocstring"; +export const updateDocstringCommand = "autoDocstring.updateDocstring"; diff --git a/src/docstring/parse_docstring.ts b/src/docstring/parse_docstring.ts index 42634f7..81c1a4f 100644 --- a/src/docstring/parse_docstring.ts +++ b/src/docstring/parse_docstring.ts @@ -10,11 +10,11 @@ export function parseDocstring(oldDocstring: string, template: string) { const docstringLines = oldDocstring.split("\n"); const templateLines = template.split("\n"); - const argRegex = getBlockRegex(template, "args"); - const kwargRegex = getBlockRegex(template, "kwargs"); - const exceptionRegex = getBlockRegex(template, "exceptions"); - const returnRegex = getBlockRegex(template, "returns"); - const yieldRegex = getBlockRegex(template, "yields"); + const argRegex = getTemplateBlockRegex(template, "args"); + const kwargRegex = getTemplateBlockRegex(template, "kwargs"); + const exceptionRegex = getTemplateBlockRegex(template, "exceptions"); + const returnRegex = getTemplateBlockRegex(template, "returns"); + const yieldRegex = getTemplateBlockRegex(template, "yields"); console.log(yieldRegex); @@ -151,10 +151,31 @@ export function parseDocstring(oldDocstring: string, template: string) { } } - console.log(docstringParts); + return docstringParts; } -function getBlockRegex(template: string, block: string): RegExp { +/** Gets the docstring template section for the given block type */ +function getTemplateBlockRegex(template: string, blockName: string): RegExp { + const block = getTemplateBlock(template, blockName); + + // Escape all characters that can be misidentified in regex + let blockRegex = escapeRegExp(block); + + // Replace all tags with named regex capture groups + blockRegex = replaceTags(blockRegex, "{{var}}", "\\w+", "var"); + blockRegex = replaceTags(blockRegex, "{{typePlaceholder}}", "[\\w\\[\\], ]+", "type"); + blockRegex = replaceTags(blockRegex, "{{descriptionPlaceholder}}", ".*", "description"); + blockRegex = replaceTags(blockRegex, "{{&default}}", ".+", "default"); + blockRegex = replaceTags(blockRegex, "{{type}}", "\\w+", "type"); + + // Escape newlines and handle possible indentation + blockRegex = blockRegex.trim(); + blockRegex = blockRegex.replace(/\n/g, "\\n\\s*"); + + return new RegExp(blockRegex.trim()); +} + +function getTemplateBlock(template: string, block: string): string { const blockStartTag = `{{#${block}}}`; const blockEndTag = `{{/${block}}}`; const blockRegex = new RegExp(blockStartTag + "(.*)" + blockEndTag, "s"); @@ -162,20 +183,7 @@ function getBlockRegex(template: string, block: string): RegExp { const match = template.match(blockRegex); let pattern = match[1]; - // Escape all characters that can be misidentified in regex - pattern = escapeRegExp(pattern); - - // Replace all tags with named regex capture groups - pattern = replaceTags(pattern, "{{var}}", "\\w+", "var"); - pattern = replaceTags(pattern, "{{typePlaceholder}}", "[\\w\\[\\], ]+", "type"); - pattern = replaceTags(pattern, "{{descriptionPlaceholder}}", ".*", "description"); - pattern = replaceTags(pattern, "{{&default}}", ".+", "default"); - pattern = replaceTags(pattern, "{{type}}", "\\w+", "type"); - - pattern = pattern.trim(); - pattern = pattern.replace(/\n/g, "\\n\\s*"); - - return new RegExp(pattern.trim()); + return pattern; } function replaceTags(str: string, tag: string, pattern: string, captureGroupName: string): string { diff --git a/src/extension.ts b/src/extension.ts index bea8bfc..51b3cf7 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -2,7 +2,12 @@ import * as vs from "vscode"; import { AutoDocstring } from "./generate_docstring"; import { docstringIsClosed, validDocstringPrefix } from "./parse"; -import { extensionRoot, generateDocstringCommand, extensionID } from "./constants"; +import { + extensionRoot, + generateDocstringCommand, + extensionID, + updateDocstringCommand, +} from "./constants"; import { getStackTrace } from "./telemetry"; import { logInfo, logError } from "./logger"; @@ -21,6 +26,17 @@ export function activate(context: vs.ExtensionContext): void { } }), + vs.commands.registerCommand(updateDocstringCommand, () => { + const editor = vs.window.activeTextEditor; + const autoDocstring = new AutoDocstring(editor); + + try { + return autoDocstring.updateDocstring(); + } catch (error) { + logError(error + "\n\t" + getStackTrace(error)); + } + }), + vs.languages.registerCompletionItemProvider( "python", { diff --git a/src/generate_docstring.ts b/src/generate_docstring.ts index b2cf775..5ea69d7 100644 --- a/src/generate_docstring.ts +++ b/src/generate_docstring.ts @@ -2,7 +2,7 @@ import * as path from "path"; import * as vs from "vscode"; import { DocstringFactory } from "./docstring/docstring_factory"; import { getCustomTemplate, getTemplate } from "./docstring/get_template"; -import { getDocstringIndentation, getDefaultIndentation, parse } from "./parse"; +import { getDocstringIndentation, getDefaultIndentation, parse, getDocstring } from "./parse"; import { extensionID } from "./constants"; import { logInfo } from "./logger"; @@ -42,6 +42,18 @@ export class AutoDocstring { return success; } + public updateDocstring() { + const position = this.editor.selection.active; + const document = this.editor.document.getText(); + logInfo(`Updating Docstring at line: ${position.line}`); + + let existingDocstring = getDocstring(document, position.line); + + // existingDocstring = normalize; + + logInfo(existingDocstring); + } + private generateDocstringSnippet(document: string, position: vs.Position): vs.SnippetString { const config = this.getConfig(); diff --git a/src/parse/docstring/get_docstring.ts b/src/parse/docstring/get_docstring.ts new file mode 100644 index 0000000..73193f1 --- /dev/null +++ b/src/parse/docstring/get_docstring.ts @@ -0,0 +1,13 @@ +export function getDocstring(document: string, lineNum: number): string { + const lines = document.split("\n"); + + const startIndex = getDocstringStartIndex(lines, lineNum); + const endIndex = getDocstringEndIndex(lines, lineNum); + + if (startIndex == undefined || endIndex == undefined) { + return ""; + } + + const docstringLines = lines.slice(startIndex, endIndex + 1); + return docstringLines.join("\n"); +} diff --git a/src/parse/docstring/get_docstring_range.ts b/src/parse/docstring/get_docstring_range.ts new file mode 100644 index 0000000..20d3d40 --- /dev/null +++ b/src/parse/docstring/get_docstring_range.ts @@ -0,0 +1,61 @@ +export function getDocstringRange(document: string, lineNum: number): ; + +export function getDocstringStartIndex(lines: string[], lineNum: number) { + const docstringStartPattern = /^\s*("""|''')/; + + if (lineNum == 0) { + return 0; + } + + // If the starting line contains only docstring quotes and the previous + // line is not the function definition we can assume that we are at the + // end quotes of the docstring and should shift our position back 1 + if (isOnlyDocstringQuotes(lines[lineNum]) && !isFunctionDefinition(lines[lineNum - 1])) { + lineNum -= 1; + } + + while (lineNum >= 0) { + const line = lines[lineNum]; + if (docstringStartPattern.test(line)) { + return lineNum; + } + lineNum--; + } + + return undefined; +} + +function getDocstringEndIndex(lines: string[], lineNum: number) { + const docstringEndPattern = /("""|''')\s*$/; + + if (lineNum >= lines.length - 1) { + return lineNum; + } + + // If the starting line contains only docstring quotes and the previous + // line is the function definition we can assume that we are at the + // start quotes of the docstring and should shift our position forward 1 + if (isOnlyDocstringQuotes(lines[lineNum]) && isFunctionDefinition(lines[lineNum - 1])) { + lineNum += 1; + } + + while (lineNum < lines.length) { + const line = lines[lineNum]; + if (docstringEndPattern.test(line)) { + return lineNum; + } + lineNum++; + } + + return undefined; +} + +/** Line contains only a docstring opening or closing quotes */ +function isOnlyDocstringQuotes(line: string): boolean { + return /^\s*("""|''')\s*$/.test(line); +} + +/** Line contains the last part of a function definition */ +function isFunctionDefinition(line: string): boolean { + return /(:|(?=[#'"\n]))/.test(line); +} diff --git a/src/parse/docstring/normalize_docstring.ts b/src/parse/docstring/normalize_docstring.ts new file mode 100644 index 0000000..8e55b27 --- /dev/null +++ b/src/parse/docstring/normalize_docstring.ts @@ -0,0 +1,42 @@ +import { blankLine } from "../utilities"; + +export function normalizeDocstring(docstring: string): string { + let lines = docstring.split("\n"); + + lines = normalizeDocstringIndentation(lines); + lines = removeDocstringQuotes(lines); + + return lines.join("\n"); +} + +/** Removes docstring block indentation while keeping relative internal + * documentation consistent + */ +function normalizeDocstringIndentation(lines: string[]): string[] { + const indentationPattern = /^\s*/; + + let minimumIndentation = " ".repeat(50); + for (const line of lines) { + if (blankLine(line)) { + continue; + } + + const match = indentationPattern.exec(line); + + if (match[0].length < minimumIndentation.length) { + minimumIndentation = match[0]; + } + } + + const minimumIndentationPattern = new RegExp("^" + minimumIndentation); + lines = lines.map((line) => line.replace(minimumIndentationPattern, "")); + + return lines; +} + +/** Remove opening and closing docstring quotes */ +function removeDocstringQuotes(lines: string[]): string[] { + lines = lines.map((line) => line.replace(/^\s*("""|''')/, "")); + lines = lines.map((line) => line.replace(/("""|''')\s*$/, "")); + return lines; +} diff --git a/src/parse/index.ts b/src/parse/index.ts index 915625c..0543d9f 100644 --- a/src/parse/index.ts +++ b/src/parse/index.ts @@ -9,3 +9,4 @@ export { parse } from "./parse"; export { parseParameters } from "./parse_parameters"; export { tokenizeDefinition } from "./tokenize_definition"; export { getDefaultIndentation } from "./utilities"; +export { getDocstring } from "./docstring/get_docstring"; diff --git a/src/parse/valid_docstring_prefix.ts b/src/parse/valid_docstring_prefix.ts index d955fa0..cc1a98e 100644 --- a/src/parse/valid_docstring_prefix.ts +++ b/src/parse/valid_docstring_prefix.ts @@ -8,7 +8,7 @@ export function validDocstringPrefix( charPosition: number, quoteStyle: string, ): boolean { - const lines = document.split(/\r?\n/); + const lines = document.split("\n"); const line = lines[linePosition]; const prefix = line.slice(0, charPosition + 1); diff --git a/src/test/docstring/parse_docstring.spec.ts b/src/test/docstring/parse_docstring.spec.ts index ba871b6..96b643b 100644 --- a/src/test/docstring/parse_docstring.spec.ts +++ b/src/test/docstring/parse_docstring.spec.ts @@ -6,7 +6,7 @@ import { parseDocstring, getTemplate } from "../../docstring"; chai.config.truncateThreshold = 0; const expect = chai.expect; -it.only("google", () => { +it("Full google docstring", () => { // const template = getTemplate("google"); // parseDocstring(googleDocstring, template); @@ -14,7 +14,21 @@ it.only("google", () => { parseDocstring(fullGoogleDocstring, googleTemplate); }); -it.only("sphinx", () => { +it("parses a google docstring with no args", () => { + const docstring = parseDocstring(noArgsGoogleDocstring, googleTemplate); + + expect(docstring).to.eql({ + name: "", + args: [], + kwargs: [{ var: "kwarg1", type: "int", default: "1" }], + decorators: [], + exceptions: [{ type: "FileExistsError" }, { type: "KeyError" }], + yields: { type: "[type]" }, + returns: { type: "[type]" }, + }); +}); + +it("sphinx", () => { // const template = getTemplate("google"); // parseDocstring(googleDocstring, template); @@ -39,6 +53,21 @@ Yields: [type]: [description] """`; +const noArgsGoogleDocstring = ` +Args: + kwarg1 (int, optional): a kwarg this time. A really good one. Defaults to 1. + +Raises: + FileExistsError: Oh nej! + KeyError: bad things! + +Returns: + [type]: [description] + +Yields: + [type]: [description] +"""`; + const googleTemplate = ` {{#parametersExist}} Args: diff --git a/src/test/parse/docstring/get_docstring_test.spec.ts b/src/test/parse/docstring/get_docstring_test.spec.ts new file mode 100644 index 0000000..bad79f3 --- /dev/null +++ b/src/test/parse/docstring/get_docstring_test.spec.ts @@ -0,0 +1,164 @@ +import chai = require("chai"); +import "mocha"; + +import { getDocstring } from "../../../parse/docstring/get_docstring"; + +chai.config.truncateThreshold = 0; +const expect = chai.expect; + +describe.only("getDocstring()", () => { + it("should return the lines with the docstring the linePosition is focused on with quotes removed", () => { + const result = getDocstring(oneLineDocstring, 4); + + expect(result).to.equal(' """A docstring"""'); + }); + + it("should preserve the relative indentation", () => { + const result = getDocstring(largeDocstring, 4); + + expect(result).to.equal( + [ + ' """[summary]', + "", + " Args:", + " a ([type]): [description]", + " b (int, optional): [description]. Defaults to 1.", + "", + " Returns:", + " Tuple[int, d]: [description]", + ' """', + ].join("\n"), + ); + }); + + it("should work from linePosition at the beginning or end of the docstring", () => { + for (const line of [4, 5, 6, 7, 8]) { + const result = getDocstring(smallDocstring, line); + + expect(result).to.equal( + [ + ' """', + " [summary]", + "", + " [description]", + " [more description]", + ' """', + ].join("\n"), + "Line: " + line, + ); + } + }); + + it("should handle single quotes", () => { + const result = getDocstring(singleQuoteDocstring, 4); + + expect(result).to.equal(" '''A docstring'''"); + }); + + it("should return an empty string if a start and end could not be found", () => { + const result = getDocstring(noDocstring, 4); + + expect(result).to.equal(""); + }); +}); + +const oneLineDocstring = ` + return 3 + +def basic_function(param1, param2 = abc): + """A docstring""" + print("HELLO WORLD") + try: + something() + except Error: + raise SomethingWentWrong + return 3 + +def something_else(): +`; + +const largeDocstring = ` +Something Else + +def gap_function(): + """[summary] + + Args: + a ([type]): [description] + b (int, optional): [description]. Defaults to 1. + + Returns: + Tuple[int, d]: [description] + """ + print('HELLO WORLD') + + print('HELLO AGAIN') + +Something Else +`; +const smallDocstring = ` +Something Else + +def gap_function(): + """ + [summary] + + [description] + [more description] + """ + print('HELLO WORLD') + + print('HELLO AGAIN') + +Something Else +`; + +const indentedDocstring = ` +Something Else + + def gap_function(): + """[summary] + + Args: + a ([type]): [description] + b (int, optional): [description]. Defaults to 1. + + Returns: + Tuple[int, d]: [description] + """ + print('HELLO WORLD') + + print('HELLO AGAIN') + +Something Else +`; + +const singleQuoteDocstring = ` + return 3 + +def basic_function(param1, param2 = abc): + '''A docstring''' + print("HELLO WORLD") + try: + something() + except Error: + raise SomethingWentWrong + return 3 + +def something_else(): +`; + +const noDocstring = ` + return 3 + +def basic_function(param1, param2 = abc): + + print("HELLO WORLD") + try: + something() + except Error: + raise SomethingWentWrong + return 3 + +def something_else(): +`; diff --git a/src/test/parse/docstring/normalize_docstring_test.spec.ts b/src/test/parse/docstring/normalize_docstring_test.spec.ts new file mode 100644 index 0000000..baa7720 --- /dev/null +++ b/src/test/parse/docstring/normalize_docstring_test.spec.ts @@ -0,0 +1,84 @@ +import chai = require("chai"); +import "mocha"; + +import { normalizeDocstring } from "../../../parse/docstring/normalize_docstring"; + +chai.config.truncateThreshold = 0; +const expect = chai.expect; + +describe.only("normalizeDocstring()", () => { + it("should remove indentation and quotes of a one line docstring", () => { + const result = normalizeDocstring(oneLineDocstring); + + expect(result).to.equal("A docstring"); + }); + + it("should preserve the relative indentation", () => { + const result = normalizeDocstring(largeDocstring); + + expect(result).to.equal( + [ + "[summary]", + "", + "Args:", + " a ([type]): [description]", + " b (int, optional): [description]. Defaults to 1.", + "", + "Returns:", + " Tuple[int, d]: [description]", + "", + ].join("\n"), + ); + }); + + it("should remove the minimum indentation", () => { + const result = normalizeDocstring(indentedDocstring); + + expect(result).to.equal( + [ + "[summary]", + "", + "Args:", + " a ([type]): [description]", + " b (int, optional): [description]. Defaults to 1.", + "", + "Returns:", + " Tuple[int, d]: [description]", + "", + ].join("\n"), + ); + }); + + it("should handle single quotes", () => { + const result = normalizeDocstring(singleQuoteDocstring); + + expect(result).to.equal("A docstring"); + }); +}); + +const oneLineDocstring = `\ + """A docstring"""`; + +const largeDocstring = `\ + """[summary] + + Args: + a ([type]): [description] + b (int, optional): [description]. Defaults to 1. + + Returns: + Tuple[int, d]: [description] + """`; + +const indentedDocstring = `\ + """[summary] + + Args: + a ([type]): [description] + b (int, optional): [description]. Defaults to 1. + + Returns: + Tuple[int, d]: [description] + """`; + +const singleQuoteDocstring = `'''A docstring'''`; From 45ac7ec87f2f2024ca2930dd5033a4597d26d9ad Mon Sep 17 00:00:00 2001 From: Nils Werner Date: Tue, 15 Feb 2022 22:38:38 -0500 Subject: [PATCH 5/6] wip --- src/docstring/parse_docstring.ts | 9 ++- src/parse/docstring/get_docstring.ts | 2 + src/parse/docstring/get_docstring_range.ts | 4 +- src/test/docstring/parse_docstring.spec.ts | 79 +++++++++++-------- .../docstring/get_docstring_test.spec.ts | 2 +- .../normalize_docstring_test.spec.ts | 2 +- 6 files changed, 58 insertions(+), 40 deletions(-) diff --git a/src/docstring/parse_docstring.ts b/src/docstring/parse_docstring.ts index 81c1a4f..07bdb85 100644 --- a/src/docstring/parse_docstring.ts +++ b/src/docstring/parse_docstring.ts @@ -3,8 +3,13 @@ import { diffLines } from "Diff"; import { Argument, DocstringParts } from "../docstring_parts"; -// Need to deal with 1. multiline regex 2. indentation 3. description 4. check crlf -// 5. sections in different orders 6. A section is not there +// Need to deal with +// 1. multiline regex +// 2. indentation +// 3. description +// 4. check crlf +// 5. sections in different orders +// 6. A section is not there export function parseDocstring(oldDocstring: string, template: string) { const docstringLines = oldDocstring.split("\n"); diff --git a/src/parse/docstring/get_docstring.ts b/src/parse/docstring/get_docstring.ts index 73193f1..9ac8be9 100644 --- a/src/parse/docstring/get_docstring.ts +++ b/src/parse/docstring/get_docstring.ts @@ -1,3 +1,5 @@ +import { getDocstringStartIndex, getDocstringEndIndex } from "./get_docstring_range"; + export function getDocstring(document: string, lineNum: number): string { const lines = document.split("\n"); diff --git a/src/parse/docstring/get_docstring_range.ts b/src/parse/docstring/get_docstring_range.ts index 20d3d40..647f94a 100644 --- a/src/parse/docstring/get_docstring_range.ts +++ b/src/parse/docstring/get_docstring_range.ts @@ -1,4 +1,4 @@ -export function getDocstringRange(document: string, lineNum: number): ; +// export function getDocstringRange(document: string, lineNum: number): ; export function getDocstringStartIndex(lines: string[], lineNum: number) { const docstringStartPattern = /^\s*("""|''')/; @@ -25,7 +25,7 @@ export function getDocstringStartIndex(lines: string[], lineNum: number) { return undefined; } -function getDocstringEndIndex(lines: string[], lineNum: number) { +export function getDocstringEndIndex(lines: string[], lineNum: number) { const docstringEndPattern = /("""|''')\s*$/; if (lineNum >= lines.length - 1) { diff --git a/src/test/docstring/parse_docstring.spec.ts b/src/test/docstring/parse_docstring.spec.ts index 96b643b..dcf6706 100644 --- a/src/test/docstring/parse_docstring.spec.ts +++ b/src/test/docstring/parse_docstring.spec.ts @@ -1,20 +1,63 @@ import chai = require("chai"); import "mocha"; +import dedent from "ts-dedent"; import { parseDocstring, getTemplate } from "../../docstring"; chai.config.truncateThreshold = 0; const expect = chai.expect; -it("Full google docstring", () => { +it.only("Full google docstring", () => { + const fullGoogleDocstring = dedent` + Args: + arg1 ([type]): An argument. It is named arg1 + arg2 (Dict[str, int]): This is also an argument => a good one! + kwarg1 (int, optional): a kwarg this time. A really good one. Defaults to 1. + + Raises: + FileExistsError: Oh nej! + KeyError: bad things! + + Returns: + [type]: [description] + + Yields: + [type]: [description] + """`; + // const template = getTemplate("google"); // parseDocstring(googleDocstring, template); // parseDocstring("world", "{{#place}}{{name}}{{/place}}"); - parseDocstring(fullGoogleDocstring, googleTemplate); + const docstring = parseDocstring(fullGoogleDocstring, googleTemplate); + + expect(docstring).to.eql({ + name: "", + args: [{ var: "arg1", type: "[type]" }, { var: "arg2" }], + kwargs: [{ var: "kwarg1", type: "int", default: "1" }], + decorators: [], + exceptions: [{ type: "FileExistsError" }, { type: "KeyError" }], + yields: { type: "[type]" }, + returns: { type: "[type]" }, + }); }); it("parses a google docstring with no args", () => { + const noArgsGoogleDocstring = dedent` + Args: + kwarg1 (int, optional): a kwarg this time. A really good one. Defaults to 1. + + Raises: + FileExistsError: Oh nej! + KeyError: bad things! + + Returns: + [type]: [description] + + Yields: + [type]: [description] + """`; + const docstring = parseDocstring(noArgsGoogleDocstring, googleTemplate); expect(docstring).to.eql({ @@ -36,38 +79,6 @@ it("sphinx", () => { parseDocstring(fullSphinxDocstring, sphinxTemplate); }); -const fullGoogleDocstring = ` -Args: - arg1 ([type]): An argument. It is named arg1 - arg2 (Dict[str, int]): This is also an argument => a good one! - kwarg1 (int, optional): a kwarg this time. A really good one. Defaults to 1. - -Raises: - FileExistsError: Oh nej! - KeyError: bad things! - -Returns: - [type]: [description] - -Yields: - [type]: [description] -"""`; - -const noArgsGoogleDocstring = ` -Args: - kwarg1 (int, optional): a kwarg this time. A really good one. Defaults to 1. - -Raises: - FileExistsError: Oh nej! - KeyError: bad things! - -Returns: - [type]: [description] - -Yields: - [type]: [description] -"""`; - const googleTemplate = ` {{#parametersExist}} Args: diff --git a/src/test/parse/docstring/get_docstring_test.spec.ts b/src/test/parse/docstring/get_docstring_test.spec.ts index bad79f3..5fc043f 100644 --- a/src/test/parse/docstring/get_docstring_test.spec.ts +++ b/src/test/parse/docstring/get_docstring_test.spec.ts @@ -6,7 +6,7 @@ import { getDocstring } from "../../../parse/docstring/get_docstring"; chai.config.truncateThreshold = 0; const expect = chai.expect; -describe.only("getDocstring()", () => { +describe("getDocstring()", () => { it("should return the lines with the docstring the linePosition is focused on with quotes removed", () => { const result = getDocstring(oneLineDocstring, 4); diff --git a/src/test/parse/docstring/normalize_docstring_test.spec.ts b/src/test/parse/docstring/normalize_docstring_test.spec.ts index baa7720..c28165e 100644 --- a/src/test/parse/docstring/normalize_docstring_test.spec.ts +++ b/src/test/parse/docstring/normalize_docstring_test.spec.ts @@ -6,7 +6,7 @@ import { normalizeDocstring } from "../../../parse/docstring/normalize_docstring chai.config.truncateThreshold = 0; const expect = chai.expect; -describe.only("normalizeDocstring()", () => { +describe("normalizeDocstring()", () => { it("should remove indentation and quotes of a one line docstring", () => { const result = normalizeDocstring(oneLineDocstring); From b647638d9016930807fb42d804bff648ea9659a0 Mon Sep 17 00:00:00 2001 From: Some Call Me Tim? Date: Fri, 14 Mar 2025 13:18:53 -0600 Subject: [PATCH 6/6] Add step-by-step guide for Python setup in VSCode --- .vscode/Below is a step-by-step guide on how to .py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .vscode/Below is a step-by-step guide on how to .py diff --git a/.vscode/Below is a step-by-step guide on how to .py b/.vscode/Below is a step-by-step guide on how to .py new file mode 100644 index 0000000..e69de29