Skip to content
This repository was archived by the owner on Jul 8, 2025. It is now read-only.

Commit e34e974

Browse files
arajkumararajkumar
authored
fix: Split collector into separate module according to file type (#175)
1 parent db68ce1 commit e34e974

File tree

13 files changed

+354
-375
lines changed

13 files changed

+354
-375
lines changed

‎src/collector.ts‎

Lines changed: 70 additions & 213 deletions
Original file line numberDiff line numberDiff line change
@@ -3,230 +3,87 @@
33
* Licensed under the Apache-2.0 License. See License.txt in the project root for license information.
44
* ------------------------------------------------------------------------------------------ */
55
'use strict';
6-
import { Stream } from 'stream';
7-
import * as jsonAst from 'json-to-ast';
8-
import { IPosition, IKeyValueEntry, KeyValueEntry, Variant, ValueType, IDependency, IPositionedString, IDependencyCollector, Dependency } from './types';
9-
import { stream_from_string, getGoLangImportsCmd } from './utils';
10-
import { config } from './config';
11-
import { exec } from 'child_process';
12-
import { parse, DocumentCstNode } from "@xml-tools/parser";
13-
import { buildAst, accept, XMLElement, XMLDocument } from "@xml-tools/ast";
146

15-
/* Please note :: There was issue with semverRegex usage in the code. During run time, it extracts
16-
* version with 'v' prefix, but this is not be behavior of semver in CLI and test environment.
17-
* At the moment, using regex directly to extract version information without 'v' prefix. */
18-
//import semverRegex = require('semver-regex');
19-
function semVerRegExp(line: string): RegExpExecArray {
20-
const regExp = /(?<=^v?|\sv?)(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-(?:0|[1-9]\d*|[\da-z-]*[a-z-][\da-z-]*)(?:\.(?:0|[1-9]\d*|[\da-z-]*[a-z-][\da-z-]*))*)?(?:\+[\da-z-]+(?:\.[\da-z-]+)*)?(?=$|\s)/ig
21-
return regExp.exec(line);
7+
/* Determine what is the value */
8+
enum ValueType {
9+
Invalid,
10+
String,
11+
Integer,
12+
Float,
13+
Array,
14+
Object,
15+
Boolean,
16+
Null
17+
};
18+
19+
/* Value variant */
20+
interface IVariant {
21+
type: ValueType;
22+
object: any;
2223
}
2324

24-
class NaivePyParser {
25-
constructor(contents: string) {
26-
this.dependencies = NaivePyParser.parseDependencies(contents);
27-
}
28-
29-
dependencies: Array<IDependency>;
30-
31-
static parseDependencies(contents:string): Array<IDependency> {
32-
const requirements = contents.split("\n");
33-
return requirements.reduce((dependencies, req, index) => {
34-
// skip any text after #
35-
if (req.includes('#')) {
36-
req = req.split('#')[0];
37-
}
38-
const parsedRequirement: Array<string> = req.split(/[==,>=,<=]+/);
39-
const pkgName:string = (parsedRequirement[0] || '').trim();
40-
// skip empty lines
41-
if (pkgName.length > 0) {
42-
const version = (parsedRequirement[1] || '').trim();
43-
const entry: IKeyValueEntry = new KeyValueEntry(pkgName, { line: 0, column: 0 });
44-
entry.value = new Variant(ValueType.String, version);
45-
entry.value_position = { line: index + 1, column: req.indexOf(version) + 1 };
46-
dependencies.push(new Dependency(entry));
47-
}
48-
return dependencies;
49-
}, []);
50-
}
51-
52-
parse(): Array<IDependency> {
53-
return this.dependencies;
54-
}
25+
/* Line and column inside the JSON file */
26+
interface IPosition {
27+
line: number;
28+
column: number;
29+
};
30+
31+
/* Key/Value entry with positions */
32+
interface IKeyValueEntry {
33+
key: string;
34+
value: IVariant;
35+
key_position: IPosition;
36+
value_position: IPosition;
37+
};
38+
39+
class KeyValueEntry implements IKeyValueEntry {
40+
key: string;
41+
value: IVariant;
42+
key_position: IPosition;
43+
value_position: IPosition;
44+
45+
constructor(k: string, pos: IPosition) {
46+
this.key = k;
47+
this.key_position = pos;
48+
}
5549
}
5650

57-
/* Process entries found in the txt files and collect all dependency
58-
* related information */
59-
class ReqDependencyCollector implements IDependencyCollector {
60-
constructor(public classes: Array<string> = ["dependencies"]) {}
61-
62-
async collect(contents: string): Promise<Array<IDependency>> {
63-
let parser = new NaivePyParser(contents);
64-
return parser.parse();
65-
}
66-
51+
class Variant implements IVariant {
52+
constructor(public type: ValueType, public object: any) {}
6753
}
6854

69-
class NaiveGomodParser {
70-
constructor(contents: string, goImports: Set<string>) {
71-
this.dependencies = NaiveGomodParser.parseDependencies(contents, goImports);
72-
}
73-
74-
dependencies: Array<IDependency>;
75-
76-
static getReplaceMap(line: string, index: number): any{
77-
// split the replace statements by '=>'
78-
const parts: Array<string> = line.replace('replace', '').replace('(', '').replace(')', '').trim().split('=>');
79-
const replaceWithVersion = semVerRegExp(parts[1]);
80-
81-
// Skip lines without final version string
82-
if (replaceWithVersion && replaceWithVersion.length > 0) {
83-
const replaceTo: Array<string> = (parts[0] || '').trim().split(' ');
84-
const replaceToVersion = semVerRegExp(replaceTo[1]);
85-
const replaceWith: Array<string> = (parts[1] || '').trim().split(' ');
86-
const replaceWithIndex = line.lastIndexOf(parts[1]);
87-
const replaceEntry: IKeyValueEntry = new KeyValueEntry(replaceWith[0].trim(), { line: 0, column: 0 });
88-
replaceEntry.value = new Variant(ValueType.String, 'v' + replaceWithVersion[0]);
89-
replaceEntry.value_position = { line: index + 1, column: (replaceWithIndex + replaceWithVersion.index) };
90-
const replaceDependency = new Dependency(replaceEntry);
91-
const isReplaceToVersion: boolean = replaceToVersion && replaceToVersion.length > 0;
92-
return {key: replaceTo[0].trim() + (isReplaceToVersion ? ('@v' + replaceToVersion[0]) : ''), value: replaceDependency};
93-
}
94-
return null;
95-
}
96-
97-
static applyReplaceMap(dep: IDependency, replaceMap: Map<string, IDependency>): IDependency {
98-
let replaceDependency = replaceMap.get(dep.name.value + "@" + dep.version.value);
99-
if (replaceDependency === undefined) {
100-
replaceDependency = replaceMap.get(dep.name.value);
101-
if(replaceDependency === undefined) {
102-
return dep;
103-
}
104-
}
105-
return replaceDependency;
106-
}
107-
108-
static parseDependencies(contents:string, goImports: Set<string>): Array<IDependency> {
109-
let replaceMap = new Map<string, IDependency>();
110-
let goModDeps = contents.split("\n").reduce((dependencies, line, index) => {
111-
// skip any text after '//'
112-
if (line.includes("//")) {
113-
line = line.split("//")[0];
114-
}
115-
if (line.includes("=>")) {
116-
let replaceEntry = NaiveGomodParser.getReplaceMap(line, index);
117-
if (replaceEntry) {
118-
replaceMap.set(replaceEntry.key, replaceEntry.value);
119-
}
120-
} else {
121-
// Not using semver directly, look at comment on import statement.
122-
//const version = semverRegex().exec(line)
123-
const version = semVerRegExp(line);
124-
// Skip lines without version string
125-
if (version && version.length > 0) {
126-
const parts: Array<string> = line.replace('require', '').replace('(', '').replace(')', '').trim().split(' ');
127-
const pkgName: string = (parts[0] || '').trim();
128-
// Ignore line starting with replace clause and empty package
129-
if (pkgName.length > 0) {
130-
const entry: IKeyValueEntry = new KeyValueEntry(pkgName, { line: 0, column: 0 });
131-
entry.value = new Variant(ValueType.String, 'v' + version[0]);
132-
entry.value_position = { line: index + 1, column: version.index };
133-
// Push all direct and indirect modules present in go.mod (manifest)
134-
dependencies.push(new Dependency(entry));
135-
}
136-
}
137-
}
138-
return dependencies;
139-
}, []);
140-
141-
let goPackageDeps = [];
142-
143-
goImports.forEach(importStatement => {
144-
let exactMatchDep: Dependency = null;
145-
let moduleMatchDep: Dependency = null;
146-
goModDeps.forEach(goModDep => {
147-
if (importStatement == goModDep.name.value) {
148-
// Software stack uses the module
149-
exactMatchDep = goModDep;
150-
} else if (importStatement.startsWith(goModDep.name.value + "/")) {
151-
// Find longest module name that matches the import statement
152-
if (moduleMatchDep == null) {
153-
moduleMatchDep = goModDep;
154-
} else if (moduleMatchDep.name.value.length < goModDep.name.value.length) {
155-
moduleMatchDep = goModDep;
156-
}
157-
}
158-
});
159-
160-
if (exactMatchDep == null && moduleMatchDep != null) {
161-
// Software stack uses a package from the module
162-
let replaceDependency = NaiveGomodParser.applyReplaceMap(moduleMatchDep, replaceMap);
163-
if (replaceDependency !== moduleMatchDep) {
164-
importStatement = importStatement.replace(moduleMatchDep.name.value, replaceDependency.name.value);
165-
}
166-
const entry: IKeyValueEntry = new KeyValueEntry(importStatement + '@' + replaceDependency.name.value, replaceDependency.name.position);
167-
entry.value = new Variant(ValueType.String, replaceDependency.version.value);
168-
entry.value_position = replaceDependency.version.position;
169-
goPackageDeps.push(new Dependency(entry));
170-
}
171-
});
172-
173-
goModDeps = goModDeps.map(goModDep => NaiveGomodParser.applyReplaceMap(goModDep, replaceMap));
174-
175-
// Return modules present in go.mod and packages used in imports.
176-
return [...goModDeps, ...goPackageDeps];
177-
}
178-
179-
parse(): Array<IDependency> {
180-
return this.dependencies;
181-
}
55+
/* String value with position */
56+
interface IPositionedString {
57+
value: string;
58+
position: IPosition;
18259
}
18360

184-
/* Process entries found in the go.mod file and collect all dependency
185-
* related information */
186-
class GomodDependencyCollector implements IDependencyCollector {
187-
constructor(private manifestFile: string, public classes: Array<string> = ["dependencies"]) {
188-
this.manifestFile = manifestFile;
189-
}
190-
191-
async collect(contents: string): Promise<Array<IDependency>> {
192-
let promiseExec = new Promise<Set<string>>((resolve, reject) => {
193-
const vscodeRootpath = this.manifestFile.replace("file://", "").replace("/go.mod", "")
194-
exec(getGoLangImportsCmd(),
195-
{ shell: process.env["SHELL"], windowsHide: true, cwd: vscodeRootpath, maxBuffer: 1024 * 1200 }, (error, stdout, stderr) => {
196-
if (error) {
197-
console.error(`Command failed, environment SHELL: [${process.env["SHELL"]}] PATH: [${process.env["PATH"]}] CWD: [${process.env["CWD"]}]`)
198-
if (error.code == 127) { // Invalid command, go executable not found
199-
reject(`Unable to locate '${config.golang_executable}'`);
200-
} else {
201-
reject(`Unable to execute '${config.golang_executable} list' command, run '${config.golang_executable} mod tidy' to know more`);
202-
}
203-
} else {
204-
resolve(new Set(stdout.toString().split("\n")));
205-
}
206-
});
207-
});
208-
const goImports: Set<string> = await promiseExec;
209-
let parser = new NaiveGomodParser(contents, goImports);
210-
return parser.parse();
211-
}
212-
61+
/* Dependency specification */
62+
interface IDependency {
63+
name: IPositionedString;
64+
version: IPositionedString;
21365
}
21466

215-
class PackageJsonCollector implements IDependencyCollector {
216-
constructor(public classes: Array<string> = ["dependencies"]) {}
67+
/* Dependency collector interface */
68+
interface IDependencyCollector {
69+
classes: Array<string>;
70+
collect(contents: string): Promise<Array<IDependency>>;
71+
}
21772

218-
async collect(contents: string): Promise<Array<IDependency>> {
219-
const ast = jsonAst(contents);
220-
return ast.children.
221-
filter(c => this.classes.includes(c.key.value)).
222-
flatMap(c => c.value.children).
223-
map(c => {
224-
let entry: IKeyValueEntry = new KeyValueEntry(c.key.value, {line: c.key.loc.start.line, column: c.key.loc.start.column + 1});
225-
entry.value = new Variant(ValueType.String, c.value.value);
226-
entry.value_position = {line: c.value.loc.start.line, column: c.value.loc.start.column + 1};
227-
return new Dependency(entry);
228-
});
229-
}
73+
/* Dependency class that can be created from `IKeyValueEntry` */
74+
class Dependency implements IDependency {
75+
name: IPositionedString;
76+
version: IPositionedString;
77+
constructor(dependency: IKeyValueEntry) {
78+
this.name = {
79+
value: dependency.key,
80+
position: dependency.key_position
81+
};
82+
this.version = {
83+
value: dependency.value.object,
84+
position: dependency.value_position
85+
};
86+
}
23087
}
23188

232-
export { IDependencyCollector, PackageJsonCollector, ReqDependencyCollector, GomodDependencyCollector, IPositionedString, IDependency };
89+
export { IPosition, IKeyValueEntry, KeyValueEntry, Variant, ValueType, IDependency, IPositionedString, IDependencyCollector, Dependency };

0 commit comments

Comments
 (0)