[DFG] support recall (#2287)

* feat: basic recall support #totalrecall

* test: ensure argument linkage!

* refactor: drop debug trace

* doc: env closure

* refactor: drop outdated todo

Author: EagleoutIce

src/control-flow/semantic-cfg-guided-visitor.ts

@@ -308,6 +308,8 @@ export class SemanticCfgGuidedVisitor<
 				return this.onReturnCall({ call });
 			case BuiltInProcName.Unnamed:
 				return this.onUnnamedCall({ call });
+			case BuiltInProcName.Recall:
+				return this.onRecallCall({ call });
 			case BuiltInProcName.Default:
 			case BuiltInProcName.Function:
 			case BuiltInProcName.FunctionDefinition:
@@ -687,4 +689,10 @@ export class SemanticCfgGuidedVisitor<
 	 * @protected
 	 */
 	protected onReturnCall(_data: { call: DataflowGraphVertexFunctionCall }) {}
+
+	/**
+	 * This event triggers for every call to `Recall`, which is used to recall the function closure (usually in recursive functions).
+	 * @protected
+	 */
+	protected onRecallCall(_data: { call: DataflowGraphVertexFunctionCall }) {}
 }

src/dataflow/environments/built-in.ts

@@ -53,6 +53,7 @@ import { processTryCatch } from '../internal/process/functions/call/built-in/bui
 import { processRegisterHook } from '../internal/process/functions/call/built-in/built-in-register-hook';
 import { processLocal } from '../internal/process/functions/call/built-in/built-in-local';
 import { processS3Dispatch } from '../internal/process/functions/call/built-in/built-in-s-three-dispatch';
+import { processRecall } from '../internal/process/functions/call/built-in/built-in-recall';
 
 export type BuiltIn = `built-in:${string}`;
 
@@ -230,6 +231,10 @@ export enum BuiltInProcName {
 	Pipe                = 'builtin:pipe',
 	/** for `quote`, and other substituting calls, see {@link processQuote} */
 	Quote               = 'builtin:quote',
+	/**
+	 * for `recall` calls, see {@link processRecall}
+	 */
+	Recall              = 'builtin:recall',
 	/** for `on.exìt` and other hooks, see {@link processRegisterHook} */
 	RegisterHook        = 'builtin:register-hook',
 	/** for `repeat` loops, see {@link processRepeatLoop} */
@@ -280,6 +285,7 @@ export const BuiltInProcessorMapper = {
 	[BuiltInProcName.Local]:              processLocal,
 	[BuiltInProcName.Pipe]:               processPipe,
 	[BuiltInProcName.Quote]:              processQuote,
+	[BuiltInProcName.Recall]:             processRecall,
 	[BuiltInProcName.RegisterHook]:       processRegisterHook,
 	[BuiltInProcName.RepeatLoop]:         processRepeatLoop,
 	[BuiltInProcName.Replacement]:        processReplacementFunction,

src/dataflow/environments/default-builtin-config.ts

@@ -282,6 +282,7 @@ export const DefaultBuiltinConfig = [
 	{ type: 'function', names: ['interference'],                               processor: BuiltInProcName.Apply,               config: { unquoteFunction: true, nameOfFunctionArgument: 'propensity_integrand', libFn: true },     assumePrimitive: false },
 	{ type: 'function', names: ['ddply'],                                      processor: BuiltInProcName.Apply,               config: { unquoteFunction: true, indexOfFunction: 2, nameOfFunctionArgument: '.fun', libFn: true }, assumePrimitive: false },
 	{ type: 'function', names: ['list'],                                       processor: BuiltInProcName.List,                config: {},                                                                            assumePrimitive: true  },
+	{ type: 'function', names: ['Recall'],                                     processor: BuiltInProcName.Recall,              config: { libFn: true },                                                              assumePrimitive: false },
 	{ type: 'function', names: ['c'],                                          processor: BuiltInProcName.Vector,              config: {},                                                                            assumePrimitive: true, evalHandler: 'built-in:c'  },
 	{
 		type:      'function',

src/dataflow/environments/environment.ts

@@ -42,7 +42,11 @@ let environmentIdCounter = 1; // Zero is reserved for built-in environment
 
 /** @see REnvironmentInformation */
 export class Environment implements IEnvironment {
-	readonly id;
+	readonly id: number;
+	/** Optional name for namespaced/non-anonymous environments, please only set if you know what you are doing */
+	n?:          string;
+	/** if created by a closure, the node id of that closure */
+	private c?:  NodeId;
 	parent:      Environment;
 	memory:      BuiltInMemory;
 	cache?:      BuiltInMemory;
@@ -58,6 +62,19 @@ export class Environment implements IEnvironment {
 		}
 	}
 
+	/** please only use if you know what you are doing */
+	public setClosureNodeId(nodeId: NodeId) {
+		this.c = nodeId;
+	}
+
+	/**
+	 * Provides the closure linked to this environment.
+	 * This is of importance if, for example, if you want to know the function definition associated with this environment.
+	 */
+	public get closure(): NodeId | undefined {
+		return this.c;
+	}
+
 	/**
 	 * Create a deep clone of this environment.
 	 * @param recurseParents     - Whether to also clone parent environments
@@ -69,6 +86,8 @@ export class Environment implements IEnvironment {
 
 		const parent = recurseParents ? this.parent.clone(recurseParents) : this.parent;
 		const clone = new Environment(parent, this.builtInEnv);
+		clone.c = this.c;
+		clone.n = this.n;
 		clone.memory = new Map(
 			this.memory.entries()
 				.map(([k, v]) => [k,
@@ -176,6 +195,8 @@ export class Environment implements IEnvironment {
 		}
 
 		const out = new Environment(this.parent.overwrite(other.parent, applyCds));
+		out.c = this.c;
+		out.n = this.n;
 		out.memory = map;
 		return out;
 	}

src/dataflow/eval/resolve/resolve.ts

@@ -25,6 +25,7 @@ import type { ReadOnlyFlowrAnalyzerContext } from '../../../project/context/flow
  * aliases and vectors (in case of a vector).
  * @param a       - Ast node to resolve
  * @param resolve - Variable resolve mode
+ * @param ctx     - Analyzer context
  * @param env     - Environment to use
  * @param graph   - Dataflow Graph to use
  * @param map     - Idmap of Dataflow Graph

src/dataflow/graph/call-graph.ts

@@ -61,7 +61,7 @@ export function getSubCallGraph(graph: CallGraph, entryPoints: Set<NodeId>): Cal
 
 
 /**
- *
+ * Determines whether there is a path from `from` to `to` in the given graph (via any edge type, only respecting direction)
  */
 export function reaches(from: NodeId, to: NodeId, graph: DataflowGraph, knownReachability: DefaultMap<NodeId, Set<NodeId>> = new DefaultMap(() => new Set())): boolean {
 	const visited: Set<NodeId> = new Set();
@@ -209,10 +209,12 @@ function processCall(vtx: Required<DataflowGraphVertexFunctionCall>, from: NodeI
 	// for each call, resolve the targets
 	const tars = getAllFunctionCallTargets(vid, graph, vtx.environment);
 	let addedTarget = false;
+	let addedBiTarget = false;
 	for(const tar of tars) {
 		if(isBuiltIn(tar)) {
 			result.addEdge(vid, tar, EdgeType.Calls);
 			addedTarget = true;
+			addedBiTarget = true;
 			continue;
 		}
 		const targetVtx = graph.getVertex(tar);
@@ -222,7 +224,7 @@ function processCall(vtx: Required<DataflowGraphVertexFunctionCall>, from: NodeI
 		addedTarget = true;
 		processFunctionDefinition(targetVtx, vid, graph, result, state);
 	}
-	if(vtx.origin !== 'unnamed') {
+	if(!addedBiTarget && vtx.origin !== 'unnamed') {
 		for(const origs of vtx.origin) {
 			if(origs.startsWith('builtin:')) {
 				addedTarget = true;

src/dataflow/internal/linker.ts

@@ -22,6 +22,7 @@ import type { REnvironmentInformation } from '../environments/environment';
 import { findByPrefixIfUnique } from '../../util/prefix';
 import type { ExitPoint } from '../info';
 import { doesExitPointPropagateCalls } from '../info';
+import { UnnamedFunctionCallPrefix } from './process/functions/call/unnamed-call-handling';
 
 export type NameIdMap = DefaultMap<string, IdentifierReference[]>
 
@@ -361,7 +362,7 @@ export function linkFunctionCalls(
  * convenience function returning all known call targets, as well as the name source which defines them
  */
 export function getAllFunctionCallTargets(call: NodeId, graph: DataflowGraph, environment?: REnvironmentInformation): NodeId[] {
-	let found: NodeId[] = [];
+	const found: Set<NodeId> = new Set();
 	const callVertex = graph.get(call, true);
 	if(callVertex === undefined) {
 		return [];
@@ -373,23 +374,31 @@ export function getAllFunctionCallTargets(call: NodeId, graph: DataflowGraph, en
 		return [];
 	}
 
-	if(info.name !== undefined && (environment !== undefined || info.environment !== undefined)) {
-		const functionCallDefs = resolveByName(
-			info.name, environment ?? info.environment as REnvironmentInformation, info.origin.includes(BuiltInProcName.S3Dispatch) ? ReferenceType.S3MethodPrefix : ReferenceType.Function
-		)?.map(d => d.nodeId) ?? [];
+	if(environment !== undefined || info.environment !== undefined) {
+		let functionCallDefs: NodeId[] = [];
+		if(info.name !== undefined && !info.name.startsWith(UnnamedFunctionCallPrefix)) {
+			functionCallDefs = resolveByName(
+				info.name, environment ?? info.environment as REnvironmentInformation, info.origin.includes(BuiltInProcName.S3Dispatch) ? ReferenceType.S3MethodPrefix : ReferenceType.Function
+			)?.map(d => d.nodeId) ?? [];
+		}
 		for(const [target, outgoingEdge] of outgoingEdges.entries()) {
 			if(edgeIncludesType(outgoingEdge.types, EdgeType.Calls)) {
 				functionCallDefs.push(target);
 			}
 		}
+
 		const [functionCallTargets, builtInTargets] = getAllLinkedFunctionDefinitions(new Set(functionCallDefs), graph);
 		for(const target of functionCallTargets) {
-			found.push(target.id);
+			found.add(target.id);
+		}
+		for(const arr of [builtInTargets, functionCallDefs]) {
+			for(const target of arr) {
+				found.add(target);
+			}
 		}
-		found = found.concat(Array.from(builtInTargets), functionCallDefs);
 	}
 
-	return found;
+	return Array.from(found);
 }
 
 const LinkedFnFollowBits = EdgeType.Reads | EdgeType.DefinedBy | EdgeType.DefinedByOnCall;

src/dataflow/internal/process/functions/call/built-in/built-in-function-definition.ts

@@ -57,7 +57,7 @@ export function processFunctionDefinition<OtherInfo>(
 
 	const originalEnvironment = data.environment;
 	// within a function def we do not pass on the outer binds as they could be overwritten when called
-	data = prepareFunctionEnvironment(data);
+	data = prepareFunctionEnvironment(data, rootId);
 
 	const subgraph = new DataflowGraph(data.completeAst.idMap);
 
@@ -342,10 +342,13 @@ export function updateNestedFunctionCalls(
 	}
 }
 
-function prepareFunctionEnvironment<OtherInfo>(data: DataflowProcessorInformation<OtherInfo & ParentInformation>) {
+function prepareFunctionEnvironment<OtherInfo>(data: DataflowProcessorInformation<OtherInfo & ParentInformation>, rootId: NodeId) {
 	let env = data.ctx.env.makeCleanEnv();
 	for(let i = 0; i < data.environment.level + 1 /* add another env */; i++) {
 		env = pushLocalEnvironment(env);
+		if(i === data.environment.level) {
+			env.current.setClosureNodeId(rootId);
+		}
 	}
 	return { ...data, environment: env };
 }

src/dataflow/internal/process/functions/call/built-in/built-in-recall.ts

@@ -0,0 +1,57 @@
+import type { DataflowProcessorInformation } from '../../../../../processor';
+import type { DataflowInformation } from '../../../../../info';
+import { processKnownFunctionCall } from '../known-call-handling';
+import type { ParentInformation } from '../../../../../../r-bridge/lang-4.x/ast/model/processing/decorate';
+import type { RFunctionArgument } from '../../../../../../r-bridge/lang-4.x/ast/model/nodes/r-function-call';
+import type { RSymbol } from '../../../../../../r-bridge/lang-4.x/ast/model/nodes/r-symbol';
+import type { NodeId } from '../../../../../../r-bridge/lang-4.x/ast/model/processing/node-id';
+import { BuiltInProcName } from '../../../../../environments/built-in';
+import { log } from '../../../../../../util/log';
+import { EdgeType } from '../../../../../graph/edge';
+import { isFunctionCallVertex } from '../../../../../graph/vertex';
+import { UnnamedFunctionCallPrefix } from '../unnamed-call-handling';
+import type { REnvironmentInformation } from '../../../../../environments/environment';
+
+
+/**
+ * Processes a built-in 'Recall' function call, linking
+ * the recall to the enveloping function closure.
+ */
+export function processRecall<OtherInfo>(
+	name: RSymbol<OtherInfo & ParentInformation>,
+	args: readonly RFunctionArgument<OtherInfo & ParentInformation>[],
+	rootId: NodeId,
+	data: DataflowProcessorInformation<OtherInfo & ParentInformation>,
+): DataflowInformation {
+	const { information } = processKnownFunctionCall({
+		name,
+		args,
+		rootId,
+		data,
+		origin: BuiltInProcName.Recall
+	});
+
+	let cur = data.environment.current;
+	let closure: NodeId | undefined;
+	while(cur) {
+		if(cur.closure) {
+			closure = cur.closure;
+			break;
+		}
+		cur = cur.parent;
+	}
+
+	if(closure) {
+		information.graph.addEdge(rootId, closure, EdgeType.Calls);
+		// also kill the name of the recall function
+		const r = information.graph.getVertex(rootId);
+		if(isFunctionCallVertex(r)){
+			(r as { name: string }).name = UnnamedFunctionCallPrefix + rootId + '-' + r.name;
+			(r as { environment: REnvironmentInformation }).environment = information.environment;
+		}
+	} else {
+		log.warn('No enclosing function closure found for recall at node', rootId);
+	}
+
+	return information;
+}

src/dataflow/internal/process/functions/call/known-call-handling.ts

@@ -18,9 +18,13 @@ import { handleUnknownSideEffect } from '../../../../graph/unknown-side-effect';
 import { BuiltInProcName } from '../../../../environments/built-in';
 
 export interface ProcessKnownFunctionCallInput<OtherInfo> extends ForceArguments {
+	/** The name of the function being called. */
 	readonly name:                  RSymbol<OtherInfo & ParentInformation>
+	/** The arguments to the function call. */
 	readonly args:                  readonly (RNode<OtherInfo & ParentInformation> | RFunctionArgument<OtherInfo & ParentInformation>)[]
+	/** The node ID to use for the function call vertex. */
 	readonly rootId:                NodeId
+	/** The dataflow processor information at the point of the function call. */
 	readonly data:                  DataflowProcessorInformation<OtherInfo & ParentInformation>
 	/** should arguments be processed from right to left? This does not affect the order recorded in the call but of the environments */
 	readonly reverseOrder?:         boolean