[Plugin] Vignette, Test-Info and general meta-file-plugin support (#2142)

* refactor: refine project discover configurability

* feat: meta loader and files reset fixes

+ improved project output

* feat: one file may have more than one role!

* feat: vignette plugin, allow file plugins to continue

* feat: test info plugin (closes #2106)

* refactor: clean up

Author: EagleoutIce

src/project/context/flowr-analyzer-files-context.ts

@@ -41,14 +41,16 @@ export type RoleBasedFiles = {
 	[FileRole.News]:        FlowrNewsFile[];
 	[FileRole.Namespace]:   FlowrNamespaceFile[];
     /* currently no special support */
-    [FileRole.Source]:   FlowrFileProvider[];
-    [FileRole.Data]:     FlowrFileProvider[];
-    [FileRole.Other]:    FlowrFileProvider[];
+	[FileRole.Vignette]:    FlowrFileProvider[];
+	[FileRole.Test]:        FlowrFileProvider[];
+	[FileRole.Source]:      FlowrFileProvider[];
+	[FileRole.Data]:        FlowrFileProvider[];
+	[FileRole.Other]:       FlowrFileProvider[];
 }
 
-function wrapFile(file: string | FlowrFileProvider | RParseRequestFromFile, role?: FileRole): FlowrFileProvider {
+function wrapFile(file: string | FlowrFileProvider | RParseRequestFromFile, roles?: readonly FileRole[]): FlowrFileProvider {
 	if(typeof file === 'string') {
-		return new FlowrTextFile(file, role);
+		return new FlowrTextFile(file, roles);
 	} else if('request' in file) {
 		return FlowrFile.fromRequest(file);
 	} else {
@@ -130,14 +132,7 @@ export class FlowrAnalyzerFilesContext extends AbstractFlowrAnalyzerContext<RPro
 	private readonly consideredFiles: string[] = [];
 
 	/* files that are part of the analysis, e.g. source files */
-	private byRole:        RoleBasedFiles = {
-		[FileRole.Description]: [],
-		[FileRole.News]:        [],
-		[FileRole.Namespace]:   [],
-		[FileRole.Source]:      [],
-		[FileRole.Data]:        [],
-		[FileRole.Other]:       []
-	} satisfies Record<FileRole, FlowrFileProvider[]>;
+	private byRole: RoleBasedFiles = Object.fromEntries<FlowrFileProvider[]>(Object.values(FileRole).map(k => [k, []])) as RoleBasedFiles;
 
 	constructor(
 		loadingOrder: FlowrAnalyzerLoadingOrderContext,
@@ -152,6 +147,9 @@ export class FlowrAnalyzerFilesContext extends AbstractFlowrAnalyzerContext<RPro
 	public reset(): void {
 		this.loadingOrder.reset();
 		this.files = new Map<FilePath, FlowrFileProvider>();
+		this.consideredFiles.length = 0;
+		this.inlineFiles.length = 0;
+		this.byRole = Object.fromEntries<FlowrFileProvider[]>(Object.values(FileRole).map(k => [k, []])) as RoleBasedFiles;
 	}
 
 	/**
@@ -191,7 +189,7 @@ export class FlowrAnalyzerFilesContext extends AbstractFlowrAnalyzerContext<RPro
 			if(isParseRequest(req)) {
 				this.addRequest(req);
 			} else {
-				this.addFile(req, req.role);
+				this.addFile(req, req.roles);
 			}
 		}
 	}
@@ -209,8 +207,8 @@ export class FlowrAnalyzerFilesContext extends AbstractFlowrAnalyzerContext<RPro
 	 * Add a file to the context. If the file has a special role, it will be added to the corresponding list of special files.
 	 * This method also applies any registered {@link FlowrAnalyzerFilePlugin}s to the file before adding it to the context.
 	 */
-	public addFile(file: string | FlowrFileProvider | RParseRequestFromFile, role?: FileRole) {
-		const f = this.fileLoadPlugins(wrapFile(file, role));
+	public addFile(file: string | FlowrFileProvider | RParseRequestFromFile, roles?: readonly FileRole[]) {
+		const f = this.fileLoadPlugins(wrapFile(file, roles));
 
 		if(f.path() === FlowrFile.INLINE_PATH) {
 			this.inlineFiles.push(f);
@@ -220,8 +218,10 @@ export class FlowrAnalyzerFilesContext extends AbstractFlowrAnalyzerContext<RPro
 			this.files.set(f.path(), f);
 		}
 
-		if(f.role) {
-			this.byRole[f.role].push(f as never);
+		if(f.roles) {
+			for(const r of f.roles) {
+				this.byRole[r].push(f as never);
+			}
 		}
 
 		return f;
@@ -266,8 +266,16 @@ export class FlowrAnalyzerFilesContext extends AbstractFlowrAnalyzerContext<RPro
 		for(const loader of this.fileLoaders) {
 			if(loader.applies(f.path())) {
 				fileLog.debug(`Applying file loader ${loader.name} to file ${f.path()}`);
-				fFinal = loader.processor(this.ctx, f);
-				break;
+				const res = loader.processor(this.ctx, fFinal);
+				if(Array.isArray(res)) {
+					fFinal = res[0];
+					if(!res[1]) {
+						break;
+					}
+				} else {
+					fFinal = res;
+					break;
+				}
 			}
 		}
 		return fFinal;

src/project/context/flowr-file.ts

@@ -1,6 +1,5 @@
 import type { PathLike } from 'fs';
 import fs from 'fs';
-import { guard } from '../../util/assert';
 import type { RParseRequest } from '../../r-bridge/retriever';
 
 /**
@@ -20,7 +19,11 @@ export enum FileRole {
 	/** The `NAMESPACE` file in R packages, currently not specially supported. */
 	Namespace   = 'namespace',
 	/** The `NEWS` file in R packages */
-	News = 'news',
+	News        = 'news',
+	/** Vignette files, e.g., R Markdown files in the `vignettes/` folder */
+	Vignette    = 'vignette',
+	/** Test source files, e.g., files in the `tests/` folder */
+	Test        = 'test',
 	/** Data files, e.g., `R/sysdata.rda`, currently not specially supported. */
 	Data        = 'data',
 	/**
@@ -49,11 +52,11 @@ export type StringableContent = { toString(): string };
  */
 export interface FlowrFileProvider<Content extends { toString(): string } = { toString(): string }> {
 	/**
-	 * The role of this file, if any, in general your file should _not_ decide for itself what role it has in the project context,
+	 * The role(s) of this file, if any, in general your file should _not_ decide for itself what role it has in the project context,
 	 * this is for the loaders plugins to decide (cf. {@link PluginType}) as they can, e.g., respect ignore files, updated mappings, etc.
 	 * However, they will 1) set this role as soon as they decide on it (using {@link assignRole}) and 2) try to respect an already assigned role (however, user configurations may override this).
 	 */
-    role?: FileRole;
+    roles?: readonly FileRole[];
 
 	/**
 	 * The path to the file, this is used for identification and logging purposes.
@@ -82,14 +85,18 @@ export interface FlowrFileProvider<Content extends { toString(): string } = { to
  * See {@link FlowrTextFile} for a text-file specific implementation and {@link FlowrInlineTextFile} for inline text files.
  */
 export abstract class FlowrFile<Content extends StringableContent = StringableContent> implements FlowrFileProvider<Content> {
-	private contentCache:  Content | undefined;
-	protected filePath:    PathLike;
-	public readonly role?: FileRole;
+	private contentCache: Content | undefined;
+	protected filePath:   PathLike;
+	private _roles?:      FileRole[];
 	public static readonly INLINE_PATH = '@inline';
 
-	public constructor(filePath: PathLike, role?: FileRole) {
+	public constructor(filePath: PathLike, roles?: readonly FileRole[]) {
 		this.filePath = filePath;
-		this.role     = role;
+		this._roles     = roles ? Array.from(roles) : undefined;
+	}
+
+	public get roles(): readonly FileRole[] | undefined {
+		return this._roles;
 	}
 
 	public path(): string {
@@ -106,8 +113,14 @@ export abstract class FlowrFile<Content extends StringableContent = StringableCo
 	protected abstract loadContent(): Content;
 
 	public assignRole(role: FileRole): void {
-		guard(this.role === undefined || this.role === role, `File ${this.filePath.toString()} already has a role assigned: ${this.role}`);
-		(this as { role?: FileRole }).role = role;
+		if(this._roles === undefined) {
+			this._roles = [role];
+		} else if(this._roles.includes(role)) {
+			// already assigned
+			return;
+		} else {
+			this._roles.push(role);
+		}
 	}
 
 	/**

src/project/plugins/file-plugins/files/flowr-description-file.ts

@@ -20,7 +20,7 @@ export class FlowrDescriptionFile extends FlowrFile<DeepReadonly<DCF>> {
 	 * and handle role assignments.
 	 */
 	constructor(file: FlowrFileProvider) {
-		super(file.path(), file.role);
+		super(file.path(), file.roles);
 		this.wrapped = file;
 	}
 

src/project/plugins/file-plugins/files/flowr-jupyter-file.ts

@@ -15,7 +15,7 @@ export class FlowrJupyterFile extends FlowrFile<string> {
 	 * @param file - the file to load as R Markdown
 	 */
 	constructor(file: FlowrFileProvider<string>) {
-		super(file.path(), FileRole.Source);
+		super(file.path(), file.roles ? [...file.roles, FileRole.Source] : [FileRole.Source]);
 		this.wrapped = file;
 	}
 

src/project/plugins/file-plugins/files/flowr-namespace-file.ts

@@ -24,7 +24,7 @@ export class FlowrNamespaceFile extends FlowrFile<NamespaceFormat> {
 	 * and handle role assignments.
 	 */
 	constructor(file: FlowrFileProvider) {
-		super(file.path(), file.role);
+		super(file.path(), file.roles);
 		this.wrapped = file;
 	}
 

src/project/plugins/file-plugins/files/flowr-news-file.ts

@@ -20,7 +20,7 @@ export class FlowrNewsFile extends FlowrFile<NewsChunk[]> {
 	 * and handle role assignments.
 	 */
 	constructor(file: FlowrFileProvider) {
-		super(file.path(), file.role);
+		super(file.path(), file.roles);
 		this.wrapped = file;
 	}
 

src/project/plugins/file-plugins/files/flowr-rmarkdown-file.ts

@@ -17,7 +17,7 @@ export class FlowrRMarkdownFile extends FlowrFile<string> {
 	 * @param file - the file to load as R Markdown
 	 */
 	constructor(file: FlowrFileProvider<string>) {
-		super(file.path(), FileRole.Source);
+		super(file.path(), file.roles ? [...file.roles, FileRole.Source] : [FileRole.Source]);
 		this.wrapped = file;
 	}
 

src/project/plugins/file-plugins/flowr-analyzer-file-plugin.ts

@@ -12,10 +12,13 @@ import type { FlowrAnalyzerContext } from '../../context/flowr-analyzer-context'
  *
  * It is up to the construction to ensure that no two file plugins {@link applies} to the same file, otherwise, the loading order
  * of these plugins will determine which plugin gets to process the file.
+ * On transforming a file, your plugin can indicate whether other plugins should still get to process the file,
+ * by returning a tuple of `[transformedFile, <boolean>]` where a boolean `true` indicates that other plugins should still get to process the file.
+ * One example of a plugin doing this is the {@link FlowrAnalyzerMetaVignetteFilesPlugin}.
  *
  * See {@link DefaultFlowrAnalyzerFilePlugin} for the no-op default implementation.
  */
-export abstract class FlowrAnalyzerFilePlugin extends FlowrAnalyzerPlugin<FlowrFileProvider, FlowrFileProvider> {
+export abstract class FlowrAnalyzerFilePlugin extends FlowrAnalyzerPlugin<FlowrFileProvider, FlowrFileProvider | [file: FlowrFileProvider, cont: boolean]> {
 	public readonly type = PluginType.FileLoad;
 
 	/**

…/flowr-analyzer-namespace-file-plugin.ts …flowr-analyzer-namespace-files-plugin.tssrc/project/plugins/file-plugins/flowr-analyzer-namespace-file-plugin.ts renamed to src/project/plugins/file-plugins/flowr-analyzer-namespace-files-plugin.ts src/project/plugins/file-plugins/flowr-analyzer-namespace-file-plugin.ts renamed to src/project/plugins/file-plugins/flowr-analyzer-namespace-files-plugin.ts

@@ -12,7 +12,7 @@ const NamespaceFilePattern = /^NAMESPACE(\.txt)?$/i;
 /**
  * This plugin provides support for R `NAMESPACE` files.
  */
-export class FlowrAnalyzerNamespaceFilePlugin extends FlowrAnalyzerFilePlugin {
+export class FlowrAnalyzerNamespaceFilesPlugin extends FlowrAnalyzerFilePlugin {
 	public readonly name = 'flowr-analyzer-namespace-file-plugin';
 	public readonly description = 'This plugin provides support for NAMESPACE files and extracts their content into the NAMESPACEFormat.';
 	public readonly version = new SemVer('0.1.0');

src/project/plugins/file-plugins/flowr-analyzer-test-file-plugin.ts

@@ -0,0 +1,44 @@
+import { FlowrAnalyzerFilePlugin } from './flowr-analyzer-file-plugin';
+import { SemVer } from 'semver';
+import type { PathLike } from 'fs';
+import type { FlowrAnalyzerContext } from '../../context/flowr-analyzer-context';
+import type { FlowrFileProvider } from '../../context/flowr-file';
+import { FileRole } from '../../context/flowr-file';
+import {
+	platformDirname
+} from '../../../dataflow/internal/process/functions/call/built-in/built-in-source';
+
+const TestPathPattern = /tests?/i;
+
+/**
+ * This plugin provides supports for the identification of test files.
+ * If you use multiple plugins, this should be included *before* other plugins.
+ */
+export class FlowrAnalyzerMetaTestFilesPlugin extends FlowrAnalyzerFilePlugin {
+	public readonly name = 'flowr-analyzer-test-files-plugin';
+	public readonly description = 'This plugin provides support for loading test files.';
+	public readonly version = new SemVer('0.1.0');
+	private readonly pathPattern: RegExp;
+
+	/**
+	 * Creates a new instance of the TEST file plugin.
+	 * @param pathPattern - The pathPattern to identify TEST files, see {@link TestPathPattern} for the default pathPattern.
+	 */
+	constructor(pathPattern: RegExp = TestPathPattern) {
+		super();
+		this.pathPattern = pathPattern;
+	}
+
+	public applies(file: PathLike): boolean {
+		return this.pathPattern.test(platformDirname(file.toString()));
+	}
+
+	/**
+	 * Processes the given file, assigning it the {@link FileRole.Test} role.
+	 * Given that the file may still need to be processed by other plugins, this method returns the `true` flag for that purpose.
+	 */
+	public process(_ctx: FlowrAnalyzerContext, file: FlowrFileProvider): [FlowrFileProvider, true] {
+		file.assignRole(FileRole.Test);
+		return [file, true];
+	}
+}