Merge pull request #108 from IBM/feature/abstract_fs

Abstract file systems

Author: worksofliam

cli/src/builders/make/folderSettings.ts

@@ -1,7 +1,8 @@
 import * as path from "path";
-import { getFiles } from "../../utils";
 import { readFileSync } from "fs";
 import { warningOut } from "../../cli";
+import { ReadFileSystem } from "../../readFileSystem";
+import { getFiles } from "../../utils";
 
 export interface FolderOptions {
 	version?: "0.0.1",

cli/src/index.ts

@@ -10,8 +10,9 @@ import { BobProject } from "./builders/bob";
 import { ImpactMarkdown } from "./builders/imd";
 import { allExtensions, referencesFileName } from "./extensions";
 import { getBranchLibraryName, getDefaultCompiles } from "./builders/environment";
-import { getFiles, renameFiles, replaceIncludes } from './utils';
+import { renameFiles, replaceIncludes } from './utils';
 import { iProject } from './builders/iProject';
+import { ReadFileSystem } from './readFileSystem';
 
 const isCli = process.argv.length >= 2 && (process.argv[1].endsWith(`so`) || process.argv[1].endsWith(`index.js`));
 
@@ -153,7 +154,8 @@ async function main() {
 		process.exit(0);
 	}
 
-	const targets = new Targets(cwd);
+	const fs = new ReadFileSystem();
+	const targets = new Targets(cwd, fs);
 
 	targets.setSuggestions({
 		includes: cliSettings.fixIncludes,
@@ -165,14 +167,14 @@ async function main() {
 	let files: string[];
 
 	try {
-		files = getFiles(cwd, scanGlob);
+		files = await fs.getFiles(cwd, scanGlob);
 	} catch (e) {
 		error(e.message || e);
 		process.exit(1);
 	}
 
 	const referenceFile = path.join(cwd, referencesFileName);
-	if (existsSync(referenceFile)) {
+	if (await fs.exists(referenceFile)) {
 		infoOut(`Found reference file: ${referenceFile}`);
 		targets.handleRefsFile(referenceFile);
 	}
@@ -206,7 +208,7 @@ async function main() {
 
 	if (cliSettings.lookupFiles && cliSettings.buildFile === `none`) {
 		for (const value of cliSettings.lookupFiles) {
-			listDeps(cwd, targets, value);
+			await listDeps(cwd, targets, value);
 		}
 	}
 
@@ -277,7 +279,7 @@ function initProject(cwd) {
 /**
  * @param query Can be object (ABCD.PGM) or relative path
  */
-function listDeps(cwd: string, targets: Targets, query: string) {
+async function listDeps(cwd: string, targets: Targets, query: string) {
 	const fullPath = path.join(cwd, query);
 
 	let [name, type] = query.split(`.`);
@@ -288,7 +290,7 @@ function listDeps(cwd: string, targets: Targets, query: string) {
 	let theObject = targets.getResolvedObjects().find(o => o.systemName === name && o.type === type);
 
 	if (!theObject) {
-		theObject = targets.resolvePathToObject(fullPath);
+		theObject = await targets.resolvePathToObject(fullPath);
 	}
 
 	const allDeps = targets.getTargets();

cli/src/readFileSystem.ts

@@ -0,0 +1,23 @@
+import fs from 'fs/promises';
+import ffs from 'fs';
+import glob from "glob";
+import path from 'path';
+import os from 'os';
+import { getFiles } from './utils';
+import { scanGlob } from './extensions';
+
+export class ReadFileSystem {
+  constructor() {}
+
+  async getFiles(cwd: string, globPath = scanGlob): Promise<string[]> {
+    return getFiles(cwd, globPath);
+  }
+
+  readFile(filePath: string): Promise<string> {
+    return fs.readFile(filePath, { encoding: `utf8` });
+  }
+
+  async exists(filePath: string): Promise<boolean> {
+    return ffs.existsSync(filePath);
+  }
+}

cli/src/targets.ts

@@ -1,7 +1,5 @@
 import glob from 'glob';
 import path from 'path';
-import fs from 'fs/promises';
-import fss from 'fs';
 import Cache from "vscode-rpgle/language/models/cache";
 import { IncludeStatement } from "vscode-rpgle/language/parserTypes";
 import { infoOut, warningOut } from './cli';
@@ -16,6 +14,7 @@ import { Logger } from './logger';
 import { asPosix, getReferenceObjectsFrom, getSystemNameFromPath, toLocalPath } from './utils';
 import { extCanBeProgram, getObjectType } from './builders/environment';
 import { isSqlFunction } from './languages/sql';
+import { ReadFileSystem } from './readFileSystem';
 
 export type ObjectType = "PGM" | "SRVPGM" | "MODULE" | "FILE" | "BNDDIR" | "DTAARA" | "CMD" | "MENU" | "DTAQ";
 
@@ -109,7 +108,7 @@ export class Targets {
 
 	public logger: Logger;
 
-	constructor(private cwd: string) {
+	constructor(private cwd: string, private fs: ReadFileSystem) {
 		this.rpgParser = setupParser(this);
 		this.logger = new Logger();
 	}
@@ -138,7 +137,17 @@ export class Targets {
 		this.resolvedObjects[localPath] = ileObject;
 	}
 
-	public resolvePathToObject(localPath: string, newText?: string) {
+	public async loadProject(withRef?: string) {
+		if (withRef) {
+			await this.handleRefsFile(path.join(this.cwd, withRef));
+		}
+
+		const initialFiles = await this.fs.getFiles(this.cwd);
+		await this.loadObjectsFromPaths(initialFiles);
+		await Promise.allSettled(initialFiles.map(f => this.parseFile(f)));
+	}
+
+	public async resolvePathToObject(localPath: string, newText?: string) {
 		if (this.resolvedObjects[localPath]) {
 			if (newText) this.resolvedObjects[localPath].text = newText;
 			return this.resolvedObjects[localPath];
@@ -163,7 +172,7 @@ export class Targets {
 
 		// If this file is an SQL file, we need to look to see if it has a long name as we need to resolve all names here
 		if (sqlExtensions.includes(extension.toLowerCase())) {
-			const ref = this.sqlObjectDataFromPath(localPath);
+			const ref = await this.sqlObjectDataFromPath(localPath);
 			if (ref) {
 				if (ref.object.system) theObject.systemName = ref.object.system.toUpperCase();
 				if (ref.object.name) theObject.longName = ref.object.name;
@@ -190,17 +199,17 @@ export class Targets {
 	 * @param filePath Fully qualified path to the file. Assumed to exist.
 	 */
 	public async handleRefsFile(filePath: string) {
-		const content = await fs.readFile(filePath, { encoding: `utf-8` });
+		const content = await this.fs.readFile(filePath);
 
 		const pseudoObjects = getReferenceObjectsFrom(content);
 
-		pseudoObjects.forEach(ileObject => {
+		for (const ileObject of pseudoObjects) {
 			if (!this.searchForObject(ileObject)) {
 				const key = `${ileObject.systemName}.${ileObject.type}`;
 				ileObject.reference = true;
 				this.resolvedObjects[key] = ileObject;
 			}
-		});
+		};
 	}
 
 	public isReferenceObject(ileObject: ILEObject, remove?: boolean) {
@@ -330,7 +339,7 @@ export class Targets {
 
 	public loadObjectsFromPaths(paths: string[]) {
 		// optimiseFileList(paths); //Ensure we load SQL files first
-		paths.forEach(p => this.resolvePathToObject(p));
+		return Promise.all(paths.map(p => this.resolvePathToObject(p)));
 	}
 
 	public async parseFile(filePath: string) {
@@ -348,7 +357,7 @@ export class Targets {
 			const ext = pathDetail.ext.substring(1).toLowerCase();
 
 			try {
-				const content = await fs.readFile(filePath, { encoding: `utf-8` });
+				const content = await this.fs.readFile(filePath);
 				const eol = content.indexOf(`\r\n`) >= 0 ? `\r\n` : `\n`;
 
 				// Really only applied to rpg
@@ -380,7 +389,8 @@ export class Targets {
 					);
 
 					if (rpgDocs) {
-						this.createRpgTarget(filePath, rpgDocs, options);
+						const ileObject = await this.resolvePathToObject(filePath, options.text);
+						this.createRpgTarget(ileObject, filePath, rpgDocs, options);
 					}
 
 				}
@@ -391,13 +401,15 @@ export class Targets {
 					const module = new Module();
 					module.parseStatements(tokens);
 
-					this.createClTarget(filePath, module, options);
+					const ileObject = await this.resolvePathToObject(filePath);
+					this.createClTarget(ileObject, filePath, module, options);
 				}
 				else if (ddsExtension.includes(ext)) {
 					const ddsFile = new dds();
 					ddsFile.parse(content.split(eol));
 
-					this.createDdsFileTarget(filePath, ddsFile, options);
+					const ileObject = await this.resolvePathToObject(filePath, options.text);
+					this.createDdsFileTarget(ileObject, filePath, ddsFile, options);
 				}
 				else if (sqlExtensions.includes(ext)) {
 					const sqlDoc = new Document(content);
@@ -410,7 +422,8 @@ export class Targets {
 					const module = new Module();
 					module.parseStatements(tokens);
 
-					this.createSrvPgmTarget(filePath, module, options);
+					const ileObject = await this.resolvePathToObject(filePath, options.text);
+					this.createSrvPgmTarget(ileObject, filePath, module, options);
 				}
 				else if (cmdExtensions.includes(ext)) {
 					this.createCmdTarget(filePath, options);
@@ -442,8 +455,7 @@ export class Targets {
 		// Since cmd source doesn't explicity contains deps, we resolve later on
 	}
 
-	private createSrvPgmTarget(localPath: string, module: Module, options: FileOptions = {}) {
-		const ileObject = this.resolvePathToObject(localPath, options.text);
+	private createSrvPgmTarget(ileObject: ILEObject, localPath: string, module: Module, options: FileOptions = {}) {
 		const target: ILEObjectTarget = {
 			...ileObject,
 			deps: [],
@@ -515,8 +527,7 @@ export class Targets {
 	/**
 	 * Handles all DDS types: pf, lf, dspf
 	 */
-	private createDdsFileTarget(localPath: string, dds: dds, options: FileOptions = {}) {
-		const ileObject = this.resolvePathToObject(localPath, options.text);
+	private createDdsFileTarget(ileObject: ILEObject, localPath: string, dds: dds, options: FileOptions = {}) {
 		const target: ILEObjectTarget = {
 			...ileObject,
 			deps: []
@@ -610,10 +621,8 @@ export class Targets {
 		this.addNewTarget(target);
 	}
 
-	private createClTarget(localPath: string, module: Module, options: FileOptions = {}) {
+	private createClTarget(ileObject: ILEObject, localPath: string, module: Module, options: FileOptions = {}) {
 		const pathDetail = path.parse(localPath);
-		const sourceName = pathDetail.base;
-		const ileObject = this.resolvePathToObject(localPath);
 		const target: ILEObjectTarget = {
 			...ileObject,
 			deps: []
@@ -966,10 +975,8 @@ export class Targets {
 		}
 	}
 
-	private createRpgTarget(localPath: string, cache: Cache, options: FileOptions = {}) {
+	private createRpgTarget(ileObject: ILEObject, localPath: string, cache: Cache, options: FileOptions = {}) {
 		const pathDetail = path.parse(localPath);
-		const ileObject = this.resolvePathToObject(localPath, options.text);
-
 		// define internal imports
 		ileObject.imports = cache.procedures
 			.filter((proc: any) => proc.keyword[`EXTPROC`])
@@ -1380,9 +1387,9 @@ export class Targets {
 					this.createOrAppend(bindingDirectoryTarget, target);
 
 					// Make sure we can resolve to this service program
-					target.exports.forEach(e => {
+					for (const e of target.exports) {
 						this.resolvedExports[e.toUpperCase()] = target;
-					});
+					}
 				} else {
 					// This service program target doesn't have any deps... so, it's not used?
 					this.removeObject(target);
@@ -1408,7 +1415,7 @@ export class Targets {
 				// Remove any service program deps so we can resolve them cleanly
 				currentTarget.deps = currentTarget.deps.filter(d => ![`SRVPGM`].includes(d.type));
 
-				currentTarget.imports.forEach(importName => {
+				for (const importName of currentTarget.imports) {
 					// Find if this import resolves to another object
 					const possibleSrvPgmDep = this.resolvedExports[importName.toUpperCase()];
 					// We can't add a module as a dependency at this step.
@@ -1430,7 +1437,7 @@ export class Targets {
 							}
 						}
 					}
-				});
+				};
 
 				// If the program or module has imports that we ca resolve, then we add them as deps
 				if (newImports.length > 0) {
@@ -1619,11 +1626,11 @@ export class Targets {
 	 * Sadly the long name is not typically part of the path name, so we need to
 	 * find the name inside of the source code.
 	 */
-	sqlObjectDataFromPath(fullPath: string): ObjectRef | undefined {
+	async sqlObjectDataFromPath(fullPath: string): Promise<ObjectRef> {
 		const relativePath = this.getRelative(fullPath);
 
-		if (fss.existsSync(fullPath)) {
-			const content = fss.readFileSync(fullPath, { encoding: `utf-8` });
+		if (await this.fs.exists(fullPath)) {
+			const content = await this.fs.readFile(fullPath);
 			const document = new Document(content);
 
 			const groups = document.getStatementGroups();

cli/src/utils.ts

@@ -7,9 +7,9 @@ import * as path from "path";
 import * as os from "os"
 import { ILEObject, ObjectType } from "./targets";
 import { CommandParameters } from "./builders/environment";
+import { ReadFileSystem } from "./readFileSystem";
 
 export function getSystemNameFromPath(inputName: string) {
-
 	let baseName = inputName.includes(`-`) ? inputName.split(`-`)[0] : inputName;
 
 	// If the name is of valid length, return it
@@ -45,6 +45,9 @@ export function getSystemNameFromPath(inputName: string) {
 	return systemName.toUpperCase();
 }
 
+/**
+ * @deprecated Use {@link ReadFileSystem.getFiles} instead
+ */
 export function getFiles(cwd: string, globPath: string): string[] {
 	let paths: string[] = glob.sync(globPath, {
 		cwd,