Skip to content

Commit da25019

Browse files
spotandjakespotandjake
committed
feat: Split compiler out from cli
1 parent 1f28a63 commit da25019

12 files changed

Lines changed: 266 additions & 147 deletions

File tree

‎TODO.md‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ These are things directly related to deliverables or causing codegen issues that
2828
* Documentation
2929
* Split README docs into separate files under `./docs/`
3030
* Ensure all the docs are to date.
31-
* Document the behavior of `<module>.Main` in `./docs/compiler_walkthrough/README.md`
3231

3332
### Less Critical
3433
* Add a statement for defining wasm imports

‎decaf/Compiler.cs‎

Lines changed: 233 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,233 @@
1+
using System.Reflection;
2+
using System.IO;
3+
4+
using Antlr4.Runtime;
5+
6+
using Decaf.Frontend;
7+
using Decaf.MiddleEnd.TypeChecker;
8+
using Decaf.MiddleEnd.Optimizations;
9+
using Decaf.MiddleEnd;
10+
using Decaf.Backend;
11+
12+
using ParseTree = Decaf.IR.ParseTree;
13+
using TypedTree = Decaf.IR.TypedTree;
14+
using AnfTree = Decaf.IR.AnfTree;
15+
using Wasm = Decaf.WasmBuilder;
16+
17+
namespace Decaf.Compiler {
18+
public static class Compiler {
19+
// --- Generic ---
20+
21+
/// <summary>
22+
/// This method bundles the runtime code into the user program.
23+
///
24+
/// This is done in a rather naive way we simply take the runtime code, and then append the parsed runtime modules to the
25+
/// user-defined modules.
26+
/// This is not the most efficient way to do this, but it is simple and works for our purposes.
27+
/// One benefit of appending the parsed runtime modules over doing a string append is that we get better error reporting and handling
28+
/// as locations are right and malformed runtime code will be caught while parsing the runtime,
29+
/// so user errors are scoped to their source.
30+
/// </summary>
31+
/// <param name="program">The program to bundle the runtime code into.</param>
32+
/// <returns>The bundled program.</returns>
33+
private static ParseTree.ProgramNode BundleRuntime(ParseTree.ProgramNode program) {
34+
// Get the embedded runtime resource (This is basically a hack to include the runtime code in the compiled assembly)
35+
var assembly = Assembly.GetExecutingAssembly();
36+
using var stream = assembly.GetManifestResourceStream("decaf.Runtime.decaf");
37+
using var reader = new BinaryReader(stream);
38+
byte[] data = reader.ReadBytes((int)stream.Length);
39+
var runtimeSource = System.Text.Encoding.UTF8.GetString(data);
40+
// Process with the front end
41+
var runtimeProgram = FrontEnd(runtimeSource, "$internal$/Runtime.decaf", false);
42+
// Bundle the runtime into the program
43+
return new ParseTree.ProgramNode(
44+
program.Position,
45+
// Put the runtime modules before the user-defined modules to ensure that the runtime modules are
46+
// available to the user-defined modules
47+
[.. runtimeProgram.Modules, .. program.Modules]
48+
);
49+
}
50+
51+
// --- Entry points ---
52+
53+
/// <summary>
54+
/// This is the main entry point for the compiler, it takes the raw source code and runs the
55+
/// entire compilation pipeline on it, returning the compiled wasm module.
56+
///
57+
/// This method runs the entire compilation pipeline on the given source code, this includes:
58+
/// - Front end (Lexing, Parsing, Semantic Analysis)
59+
/// - Middle end (Type checking, Lowering to ANF, Optimizations)
60+
/// - Back end (Lowering to wasm)
61+
/// </summary>
62+
/// <param name="source">The raw source code to compile.</param>
63+
/// <param name="inputFileName">The name of the file that contained the source code.</param>
64+
/// <returns>The compiled wasm module.</returns>
65+
#nullable enable
66+
public static Wasm.WasmModule CompileString(string source, string? inputFileName) {
67+
// Front end
68+
var frontEndProgram = FrontEnd(source, inputFileName);
69+
// Middle end
70+
var middleEndProgram = MiddleEnd(frontEndProgram);
71+
// Back end
72+
var wasmModule = Backend(middleEndProgram);
73+
// Return the compiled wasm module
74+
return wasmModule;
75+
}
76+
#nullable restore
77+
// --- Front end ---
78+
79+
/// <summary>
80+
/// This method runs the entire front end pipeline on the given source code, this includes:
81+
/// - Lexing
82+
/// - Parsing
83+
/// - Bundling the runtime code
84+
/// - Scope checking
85+
/// - Semantic checking
86+
/// </summary>
87+
/// <param name="source">The raw source code to compile.</param>
88+
/// <param name="inputFileName">The name of the file that contained the source code.</param>
89+
/// <returns>The program after front end processing.</returns>
90+
#nullable enable
91+
public static ParseTree.ProgramNode FrontEnd(string source, string? inputFileName, bool bundleRuntime = true) {
92+
// Lex the program
93+
var lexer = LexSource(source, inputFileName);
94+
// Parse the program
95+
var tokenStream = new CommonTokenStream(lexer);
96+
var program = ParseSource(tokenStream);
97+
// Bundle the runtime
98+
var bundledProgram = bundleRuntime ? BundleRuntime(program) : program;
99+
// Check semantics - NOTE: we can't do semantic checks before bundling
100+
var checkedProgram = bundleRuntime ? CheckSemantics(bundledProgram) : program;
101+
// Return the program after front end processing
102+
return checkedProgram;
103+
}
104+
#nullable restore
105+
/// <summary>
106+
/// This method runs the lexer on the given source code.
107+
/// </summary>
108+
/// <param name="source">The raw source code to lex.</param>
109+
/// <param name="inputFileName">The name of the file that contained the source code.</param>
110+
/// <returns>The lexer instance.</returns>
111+
#nullable enable
112+
public static DecafLexer LexSource(string source, string? inputFileName) {
113+
// Create Input Stream
114+
var inputStream = new AntlrInputStream(source) {
115+
name = inputFileName ?? "<unknown file>"
116+
};
117+
// Create Lexer Instance
118+
var lexer = new DecafLexer(inputStream);
119+
// Setup our custom error handler for better error reporting
120+
lexer.RemoveErrorListeners();
121+
lexer.AddErrorListener(LexerErrorListener.Instance);
122+
return lexer;
123+
}
124+
#nullable restore
125+
/// <summary>This method runs the parser on the given token stream.</summary>
126+
/// <param name="tokenStream">The token stream to parse.</param>
127+
/// <returns>The parsed program.</returns>
128+
public static ParseTree.ProgramNode ParseSource(CommonTokenStream tokenStream) {
129+
// Initialize the parser
130+
var parser = new DecafParser(tokenStream);
131+
// Setup our custom error handler for better error reporting
132+
parser.RemoveErrorListeners();
133+
parser.AddErrorListener(ParserErrorListener.Instance);
134+
// Convert the ANTLR parse tree to our own parse tree representation
135+
var program = ParseTreeMapper.MapProgramContext(parser.program());
136+
return program;
137+
}
138+
/// <summary>
139+
/// This method runs the semantic analysis phase on the given program, which includes:
140+
/// - Scope Validation
141+
/// - Semantic Validation
142+
/// </summary>
143+
/// <param name="program">The parsed program to check.</param>
144+
/// <returns>The checked program.</returns>
145+
public static ParseTree.ProgramNode CheckSemantics(ParseTree.ProgramNode program) {
146+
// Validate program scoping semantics
147+
ScopeChecker.CheckProgramNode(program);
148+
// Validate program general semantics
149+
SemanticChecker.CheckProgramNode(program);
150+
return program;
151+
}
152+
// --- Middle end ---
153+
154+
/// <summary>
155+
/// This method runs the entire middle end pipeline on the given program, this includes:
156+
/// - Type checking
157+
/// - Lowering to ANF
158+
/// - Optimizations
159+
///
160+
/// NOTE: This method expects the input program to have already been processed by the front end,
161+
/// if scoping or semantics have not been validated then this may not work correctly and may throw
162+
/// unexpected exceptions, or produce malformed outputs.
163+
/// </summary>
164+
/// <param name="program">The parsed program to process.</param>
165+
/// <returns>The processed program.</returns>
166+
public static AnfTree.ProgramNode MiddleEnd(ParseTree.ProgramNode program) {
167+
// Type check the program
168+
var typedProgram = TypeChecker.TypeProgramNode(program);
169+
// Lower to ANF
170+
var anfProgram = AnfMapper.FromProgramNode(typedProgram);
171+
// Run optimizations
172+
var optimizedProgram = Optimizer.Optimize(anfProgram);
173+
// Return the program after middle end processing
174+
return optimizedProgram;
175+
}
176+
/// <summary>
177+
/// This method runs the type checker on the given program, which includes:
178+
/// - Validating that all expressions are well-typed
179+
/// - Annotating the parse tree with type information
180+
/// </summary>
181+
/// <param name="program">The parsed program to check.</param>
182+
/// <returns>The type-checked program.</returns>
183+
public static TypedTree.ProgramNode TypeCheck(ParseTree.ProgramNode program) {
184+
// Type check the program
185+
return TypeChecker.TypeProgramNode(program);
186+
}
187+
/// <summary>
188+
/// This method lowers the given program to ANF, which includes:
189+
/// - Converting all expressions to ANF form
190+
/// </summary>
191+
/// <param name="program">The type-checked program to lower.</param>
192+
/// <returns>The program in ANF form.</returns>
193+
public static AnfTree.ProgramNode LowerToAnf(TypedTree.ProgramNode program) {
194+
// Map the typed tree to the anf tree
195+
return AnfMapper.FromProgramNode(program);
196+
}
197+
/// <summary>
198+
/// This method runs the optimizer on the given program, which includes:
199+
/// - Running various optimization passes on the program to improve performance and reduce code size.
200+
/// </summary>
201+
/// <param name="program">The anf program to optimize.</param>
202+
/// <returns>The optimized program.</returns>
203+
public static AnfTree.ProgramNode OptimizeAnf(AnfTree.ProgramNode program) {
204+
// Run optimizations
205+
return Optimizer.Optimize(program);
206+
}
207+
// --- Back end ---
208+
209+
/// <summary>
210+
/// This method runs the entire back end pipeline on the given program, this includes:
211+
/// - Lowering to wasm
212+
/// </summary>
213+
/// <param name="program">The anf program to compile.</param>
214+
/// <returns>The compiled wasm module.</returns>
215+
public static Wasm.WasmModule Backend(AnfTree.ProgramNode program) {
216+
// Lower the program to wasm
217+
var wasmModule = Codegen.CompileProgram(program);
218+
// Return the program after back end processing
219+
return wasmModule;
220+
}
221+
/// <summary>
222+
/// This method lowers the given program to wasm, which includes:
223+
/// - Converting all ANF instructions to their corresponding wasm instructions
224+
/// </summary>
225+
/// <param name="program">The anf program to lower.</param>
226+
/// <returns>The compiled wasm module.</returns>
227+
public static Wasm.WasmModule LowerToWasm(AnfTree.ProgramNode program) {
228+
// Lower the program to wasm
229+
return Codegen.CompileProgram(program);
230+
}
231+
}
232+
233+
}

‎decaf/Main.cs‎

Lines changed: 2 additions & 115 deletions
Original file line numberDiff line numberDiff line change
@@ -1,126 +1,13 @@
11
// TODO: Clean up this file
22
// TODO: Switch the library we are using for the cli
3-
// TODO: Create a new Compiler.cs file that contains the compiler itself
43

54
using System;
65
using System.Collections.Generic;
7-
using System.Reflection;
86
using System.IO;
9-
using Antlr4.Runtime;
107
using CommandLine;
118

12-
using ParseTree = Decaf.IR.ParseTree;
13-
using TypedTree = Decaf.IR.TypedTree;
14-
using AnfTree = Decaf.IR.AnfTree;
15-
using Decaf.Frontend;
16-
using Decaf.MiddleEnd.TypeChecker;
17-
using Decaf.Backend;
9+
using Decaf.Compiler;
1810
using Decaf.Utils.Errors;
19-
using Decaf.WasmBuilder;
20-
21-
namespace Compiler {
22-
public class Compiler {
23-
/// <summary>
24-
/// This method bundles the runtime code into the user program.
25-
///
26-
/// This is done in a rather naive way we simply take the runtime code, and then append the parsed runtime modules to the
27-
/// user-defined modules.
28-
/// This is not the most efficient way to do this, but it is simple and works for our purposes.
29-
/// One benefit of appending the parsed runtime modules over doing a string append is that we get better error reporting and handling
30-
/// as locations are right and malformed runtime code will be caught while parsing the runtime,
31-
/// so user errors are scoped to their source.
32-
/// </summary>
33-
/// <param name="program">The program to bundle the runtime code into.</param>
34-
/// <returns>The bundled program.</returns>
35-
private static ParseTree.ProgramNode BundleRuntime(ParseTree.ProgramNode program) {
36-
// Get the embedded runtime resource (This is basically a hack to include the runtime code in the compiled assembly)
37-
var assembly = Assembly.GetExecutingAssembly();
38-
using var stream = assembly.GetManifestResourceStream("decaf.Runtime.decaf");
39-
using var reader = new BinaryReader(stream);
40-
byte[] data = reader.ReadBytes((int)stream.Length);
41-
// Lex the runtime
42-
var runtimeSource = System.Text.Encoding.UTF8.GetString(data);
43-
var lexer = LexString(runtimeSource, "$internal$/Runtime.decaf");
44-
var tokenStream = new CommonTokenStream(lexer);
45-
// Parse the runtime
46-
var runtimeProgram = ParseTokenStream(tokenStream);
47-
// Bundle the runtime into the program
48-
return new ParseTree.ProgramNode(
49-
program.Position,
50-
// Put the runtime modules before the user-defined modules to ensure that the runtime modules are
51-
// available to the user-defined modules
52-
[.. runtimeProgram.Modules, .. program.Modules]
53-
);
54-
}
55-
#nullable enable
56-
public static DecafLexer LexString(string source, string? inputFileName) {
57-
#nullable disable
58-
// Create Input Stream
59-
AntlrInputStream inputStream = new AntlrInputStream(source) {
60-
name = inputFileName ?? "<unknown file>"
61-
};
62-
// Create Lexer Instance
63-
DecafLexer lexer = new DecafLexer(inputStream);
64-
lexer.RemoveErrorListeners();
65-
lexer.AddErrorListener(LexerErrorListener.Instance);
66-
return lexer;
67-
}
68-
public static ParseTree.ProgramNode ParseTokenStream(CommonTokenStream tokenStream) {
69-
DecafParser parser = new DecafParser(tokenStream);
70-
parser.RemoveErrorListeners();
71-
parser.AddErrorListener(ParserErrorListener.Instance);
72-
ParseTree.ProgramNode program = ParseTreeMapper.MapProgramContext(parser.program());
73-
return program;
74-
}
75-
public static ParseTree.ProgramNode SemanticAnalysis(ParseTree.ProgramNode program) {
76-
ScopeChecker.CheckProgramNode(program);
77-
SemanticChecker.CheckProgramNode(program);
78-
return program;
79-
}
80-
public static TypedTree.ProgramNode TypeChecking(ParseTree.ProgramNode program) {
81-
return TypeChecker.TypeProgramNode(program);
82-
}
83-
public static AnfTree.ProgramNode AnfMapping(TypedTree.ProgramNode program) {
84-
var anfProgram = AnfMapper.FromProgramNode(program);
85-
// Run optimizations
86-
var optimizedProgram = Decaf.Backend.Optimizations.Optimizer.Optimize(anfProgram);
87-
return optimizedProgram;
88-
}
89-
public static WasmModule Codegen(AnfTree.ProgramNode program) {
90-
return Decaf.Backend.Codegen.CompileProgram(program);
91-
}
92-
#nullable enable
93-
public static WasmModule CompileString(string source, string? inputFileName) {
94-
#nullable disable
95-
// Lexing
96-
var lexer = LexString(source, inputFileName);
97-
var tokenStream = new CommonTokenStream(lexer);
98-
// NOTE: For debugging lexer token stream
99-
// while (true) {
100-
// IToken token = lexer.NextToken();
101-
// if (token.Type == TokenConstants.EOF)
102-
// break;
103-
// Console.WriteLine(
104-
// $"Token Type: {DecafLexer.ruleNames[token.Type - 1]}, Text: '{token.Text}'"
105-
// );
106-
// }
107-
// Parsing
108-
var parsedProgram = ParseTokenStream(tokenStream);
109-
// Include the runtime code
110-
var bundledProgram = BundleRuntime(parsedProgram);
111-
// Semantic Analysis
112-
var scopedProgram = SemanticAnalysis(bundledProgram);
113-
// TypeChecking
114-
var TypeCheckingProgram = TypeChecking(scopedProgram);
115-
// Anf Conversion
116-
var anfProgram = AnfMapping(TypeCheckingProgram);
117-
// Code Generation
118-
var wasmModule = Codegen(anfProgram);
119-
// Return the wasm module
120-
return wasmModule;
121-
}
122-
}
123-
}
12411

12512
namespace CLI {
12613
public class Program {
@@ -154,7 +41,7 @@ static void RunOptions(Options opts) {
15441
string source = File.ReadAllText(absPath);
15542
// Compile
15643
try {
157-
var wasmModule = Compiler.Compiler.CompileString(source, relPath);
44+
var wasmModule = Compiler.CompileString(source, relPath);
15845
Console.WriteLine(wasmModule.ToWat());
15946
// TODO: Write output to opts.output if specified in the format specified
16047
// string json = JsonSerializer.Serialize(wasmModule, new JsonSerializerOptions { Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, WriteIndented = true });

‎decaf/MiddleEnd/AnfMapper.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
using Decaf.Utils;
99

1010
// The purpose of this file is to map from the typed tree to the ANF tree.
11-
namespace Decaf.Backend {
11+
namespace Decaf.MiddleEnd {
1212
// NOTE: One downside of the recursive approach is theoretically we could blow the stack if we have very nested expressions.
1313
// However if this were to ever become an issue we would switch to a work queue based approach.
1414
public static class AnfMapper {
@@ -137,7 +137,7 @@ private static (List<AnfTree.InstructionNode.BindNode>, AnfTree.InstructionNode)
137137
// NOTE: if falseBranch is null then falseBinds should be 0, but we check both just to be safe
138138
var falseBranch =
139139
falseBinds.Count > 0 && _falseBranch != null ?
140-
new AnfTree.InstructionNode.BlockNode(node.TrueBranch.Position, [.. trueBinds, _falseBranch]) :
140+
new AnfTree.InstructionNode.BlockNode(node.FalseBranch.Position, [.. falseBinds, _falseBranch]) :
141141
_falseBranch;
142142
// Emit the anf instruction
143143
return (

0 commit comments

Comments
 (0)