introduce optimizer

Author: kleinesfilmroellchen

sof-rs/src/cli.rs

@@ -32,12 +32,13 @@ impl Args {
 	/// - if --interactive, always open REPL
 	/// - otherwise, only open REPL if neither input file nor inline command was used.
 	pub fn should_open_repl(&self) -> bool {
-		self.interactive || (!self.input.is_some() && !self.command.is_some())
+		self.interactive || (self.input.is_none() && self.command.is_none())
 	}
 
 	/// Configure the logger with the options that control logging behavior.
 	pub fn configure_env_logger(&self, builder: &mut env_logger::Builder) {
 		let mut should_apply_default = true;
+		#[allow(clippy::match_wildcard_for_single_variants)]
 		for log_level in self.debug_options.iter().filter_map(|f| match f {
 			DebugOption::LogLevel(log_level) => Some(log_level),
 			_ => None,
@@ -74,7 +75,7 @@ pub enum LogLevel {
 }
 
 impl LogLevel {
-	pub fn configure_env_logger(&self, builder: &mut env_logger::Builder) {
+	pub fn configure_env_logger(self, builder: &mut env_logger::Builder) {
 		match self {
 			Self::Off => builder.filter_level(log::LevelFilter::Off),
 			Self::SelfDebug => builder.filter_module("sof", log::LevelFilter::Debug),

sof-rs/src/main.rs

@@ -21,6 +21,7 @@ mod cli;
 mod error;
 mod identifier;
 mod lib;
+mod optimizer;
 mod parser;
 mod runtime;
 mod token;
@@ -146,15 +147,11 @@ fn main() -> miette::Result<()> {
 
 	// Finalize.
 
-	if let Some(snapshot_filename) = args
-		.debug_options
-		.iter()
-		.filter_map(|d| match d {
-			DebugOption::ExportSnapshot { target_filename } => Some(target_filename),
-			_ => None,
-		})
-		.next()
-	{
+	#[allow(clippy::match_wildcard_for_single_variants)]
+	if let Some(snapshot_filename) = args.debug_options.iter().find_map(|d| match d {
+		DebugOption::ExportSnapshot { target_filename } => Some(target_filename),
+		_ => None,
+	}) {
 		info!("Output snapshot to {}", snapshot_filename.display());
 	}
 
@@ -168,8 +165,9 @@ fn run_code_on_arena(
 	library_path: impl Into<PathBuf>,
 ) -> miette::Result<()> {
 	let result = parser::lexer::lex(code)?;
-	let parsed = Arc::new(parser::parse(result)?);
+	let mut parsed = Arc::new(parser::parse(result)?);
 	debug!("parsed code as {parsed:#?}");
+	optimizer::run_passes(&mut parsed);
 	run_on_arena(arena, parsed, path, library_path)?;
 	Ok(())
 }

sof-rs/src/optimizer.rs

@@ -0,0 +1,75 @@
+//! Program optimization.
+
+use std::sync::Arc;
+
+use log::{debug, trace};
+
+use crate::runtime::stackable::TokenVec;
+use crate::token::Token;
+
+/// Single optimization pass that can be executed on a token list.
+pub type Pass = fn(tokens: &mut Vec<Token>);
+
+pub static DEFAULT_PASSES: [Pass; 1] = [passes::combine_literal_pushes];
+
+pub fn run_passes(tokens: &mut TokenVec) {
+	debug!("running optimizer on {} tokens…", tokens.len());
+	// TODO: super wasteful
+	let mut work_tokens = (**tokens).clone();
+	for pass in DEFAULT_PASSES {
+		pass(&mut work_tokens);
+	}
+	trace!("after optimizer: {work_tokens:#?}");
+	*tokens = Arc::new(work_tokens);
+}
+
+mod passes {
+	use log::{debug, trace};
+	use miette::SourceSpan;
+	use smallvec::SmallVec;
+
+	use crate::token::{InnerToken, Token};
+
+	/// Optimization pass which combines multiple literal tokens into a single token which pushes multiple literals at
+	/// once.
+	pub fn combine_literal_pushes(tokens: &mut Vec<Token>) {
+		debug!("combine_literal_pushes");
+
+		let mut start_span = SourceSpan::new(0.into(), 0);
+		let mut end_span = SourceSpan::new(0.into(), 0);
+		let mut start_index = 0;
+		let mut previous_literal_tokens = smallvec::SmallVec::new();
+
+		let mut idx = 0;
+		while idx < tokens.len() {
+			let current = &tokens[idx];
+			if let InnerToken::Literals(literals) = &current.inner {
+				// Set start span via first token.
+				if previous_literal_tokens.is_empty() {
+					start_span = current.span;
+					start_index = idx;
+				}
+				end_span = current.span;
+				previous_literal_tokens.extend(literals.clone());
+			} else {
+				if idx > start_index + 1 && !previous_literal_tokens.is_empty() {
+					// End the literal list
+					trace!("combining literals in range [{start_index}; {}]", idx - 1);
+					let previous_lit_tokens = previous_literal_tokens;
+					previous_literal_tokens = SmallVec::new();
+					tokens.splice(start_index .. idx, [Token {
+						inner: InnerToken::Literals(previous_lit_tokens),
+						span:  SourceSpan::new(
+							start_span.offset().into(),
+							end_span.offset() + end_span.len() - start_span.offset(),
+						),
+					}]);
+					idx = start_index + 1;
+					continue;
+				}
+				previous_literal_tokens = SmallVec::new();
+			}
+			idx += 1;
+		}
+	}
+}

sof-rs/src/parser/mod.rs

@@ -1,9 +1,10 @@
 use std::borrow::Borrow;
 
 use miette::SourceSpan;
+use smallvec::smallvec;
 
 use crate::error::Error;
-use crate::token::{Command, InnerToken, Token};
+use crate::token::{Command, InnerToken, Literal, Token};
 
 pub mod lexer;
 
@@ -18,14 +19,19 @@ where
 		let span = t.borrow().span;
 		match token {
 			lexer::RawToken::Keyword(lexer::Keyword::ListStart) =>
-				output.push(Token { inner: InnerToken::ListStart, span }),
-			lexer::RawToken::Keyword(lexer::Keyword::Curry) => output.push(Token { inner: InnerToken::Curry, span }),
-			lexer::RawToken::Decimal(decimal) => output.push(Token { inner: InnerToken::Decimal(*decimal), span }),
-			lexer::RawToken::Integer(int) => output.push(Token { inner: InnerToken::Integer(*int), span }),
-			lexer::RawToken::String(string) => output.push(Token { inner: InnerToken::String(string.clone()), span }),
-			lexer::RawToken::Boolean(boolean) => output.push(Token { inner: InnerToken::Boolean(*boolean), span }),
-			lexer::RawToken::Identifier(identifier) =>
-				output.push(Token { inner: InnerToken::Identifier(identifier.clone()), span }),
+				output.push(Token { inner: InnerToken::Literals(smallvec![Literal::ListStart]), span }),
+			lexer::RawToken::Keyword(lexer::Keyword::Curry) =>
+				output.push(Token { inner: InnerToken::Literals(smallvec![Literal::Curry]), span }),
+			lexer::RawToken::Decimal(decimal) =>
+				output.push(Token { inner: InnerToken::Literals(smallvec![Literal::Decimal(*decimal)]), span }),
+			lexer::RawToken::Integer(int) =>
+				output.push(Token { inner: InnerToken::Literals(smallvec![Literal::Integer(*int)]), span }),
+			lexer::RawToken::String(string) =>
+				output.push(Token { inner: InnerToken::Literals(smallvec![Literal::String(string.clone())]), span }),
+			lexer::RawToken::Boolean(boolean) =>
+				output.push(Token { inner: InnerToken::Literals(smallvec![Literal::Boolean(*boolean)]), span }),
+			lexer::RawToken::Identifier(identifier) => output
+				.push(Token { inner: InnerToken::Literals(smallvec![Literal::Identifier(identifier.clone())]), span }),
 			lexer::RawToken::Keyword(lexer::Keyword::CodeBlockStart) => {
 				let mut depth = 1usize;
 				let mut inner_tokens = Vec::new();

sof-rs/src/runtime.rs

@@ -85,6 +85,15 @@ impl<'gc> Stack<'gc> {
 		self.main.push(value);
 	}
 
+	pub fn push_n(&mut self, values: impl ExactSizeIterator<Item = Stackable<'gc>>) {
+		let needed_size = values.len();
+		self.main.reserve(needed_size);
+		for value in values {
+			// FIXME: could be some kind of unchecked push since reserve guarantees space
+			self.main.push(value);
+		}
+	}
+
 	pub fn push_nametable(&mut self, nametable: GcRefLock<'gc, Nametable<'gc>>) {
 		self.push(Stackable::Nametable(nametable));
 		self.top_nametable = self.main.len() - 1;

sof-rs/src/runtime/interpreter.rs

@@ -11,17 +11,17 @@ use miette::SourceSpan;
 use smallvec::{SmallVec, smallvec};
 
 use crate::arc_iter::ArcVecIter;
-use crate::call_builtin_function;
 use crate::error::Error;
 use crate::identifier::Identifier;
 use crate::lib::DEFAULT_REGISTRY;
 use crate::runtime::list::List;
 use crate::runtime::module::ModuleRegistry;
 use crate::runtime::nametable::{Nametable, NametableType};
-use crate::runtime::stackable::{BuiltinType, Function, TokenVec};
+use crate::runtime::stackable::{BuiltinType, CodeBlock, Function, TokenVec};
 use crate::runtime::util::{SwitchCase, SwitchCases, UtilityData};
 use crate::runtime::{Stack, StackArena, Stackable};
-use crate::token::{Command, InnerToken, Token};
+use crate::token::{Command, InnerToken, Literal, Token};
+use crate::{call_builtin_function, optimizer};
 
 #[derive(Default, Clone, Copy)]
 #[non_exhaustive]
@@ -32,18 +32,22 @@ pub struct Metrics {
 	pub call_count:  usize,
 }
 
-pub fn run(tokens: TokenVec, file_path: impl Into<PathBuf>, library_path: &Path) -> Result<Metrics, Error> {
+pub fn run(mut tokens: TokenVec, file_path: impl Into<PathBuf>, library_path: &Path) -> Result<Metrics, Error> {
 	let mut arena: StackArena = new_arena(library_path);
+	optimizer::run_passes(&mut tokens);
 	run_on_arena(&mut arena, tokens, file_path, library_path)
 }
 
 /// Shortest token sequence that runs the preamble:
 /// `"preamble" use`
 static RUN_PREAMBLE: LazyLock<[Token; 2]> = LazyLock::new(|| {
-	[Token { inner: InnerToken::String("preamble".into()), span: SourceSpan::new(0.into(), 0) }, Token {
-		inner: InnerToken::Command(Command::Use),
-		span:  SourceSpan::new(0.into(), 0),
-	}]
+	[
+		Token {
+			inner: InnerToken::Literals(smallvec![Literal::String("preamble".into())]),
+			span:  SourceSpan::new(0.into(), 0),
+		},
+		Token { inner: InnerToken::Command(Command::Use), span: SourceSpan::new(0.into(), 0) },
+	]
 });
 
 /// Constructs a new arena and runs the preamble on it.
@@ -867,8 +871,12 @@ fn execute_token<'a>(token: &Token, mc: &Mutation<'a>, stack: &mut Stack<'a>) ->
 			no_action()
 		},
 		InnerToken::Command(cmd) => todo!("{cmd:?} is unimplemented"),
-		literal => {
-			stack.push(literal.as_stackable(mc));
+		InnerToken::Literals(literals) => {
+			stack.push_n(literals.into_iter().map(Literal::as_stackable));
+			no_action()
+		},
+		InnerToken::CodeBlock(cb) => {
+			stack.push(Stackable::CodeBlock(Gc::new(mc, CodeBlock { code: cb.clone() })));
 			no_action()
 		},
 	}

sof-rs/src/token.rs

@@ -2,12 +2,11 @@ use std::cmp::Ordering;
 use std::fmt::{Debug, Display};
 
 use flexstr::SharedStr;
-use gc_arena::{Gc, Mutation};
 use miette::SourceSpan;
 
 use crate::identifier::Identifier;
 use crate::parser::lexer;
-use crate::runtime::stackable::{CodeBlock, Stackable, TokenVec};
+use crate::runtime::stackable::{Stackable, TokenVec};
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum InnerToken {
@@ -17,19 +16,25 @@ pub enum InnerToken {
 	/// Special token only used internally to support switch cases.
 	SwitchBody,
 
-	// literals
+	Literals(smallvec::SmallVec<[Literal; 3]>),
+
+	CodeBlock(TokenVec),
+}
+
+/// Nonrecursive literals.
+#[derive(Debug, PartialEq, Clone)]
+pub enum Literal {
 	Integer(i64),
 	Decimal(f64),
 	Identifier(Identifier),
 	String(SharedStr),
 	Boolean(bool),
-	CodeBlock(TokenVec),
 	ListStart,
 	Curry,
 }
 
-impl InnerToken {
-	pub fn as_stackable<'gc>(&self, mc: &Mutation<'gc>) -> Stackable<'gc> {
+impl Literal {
+	pub fn as_stackable<'gc>(&self) -> Stackable<'gc> {
 		match self {
 			Self::Integer(int) => Stackable::Integer(*int),
 			Self::Decimal(decimal) => Stackable::Decimal(*decimal),
@@ -38,8 +43,6 @@ impl InnerToken {
 			Self::Boolean(boolean) => Stackable::Boolean(*boolean),
 			Self::ListStart => Stackable::ListStart,
 			Self::Curry => Stackable::Curry,
-			Self::CodeBlock(code) => Stackable::CodeBlock(Gc::new(mc, CodeBlock { code: code.clone() })),
-			_ => unreachable!(),
 		}
 	}
 }
@@ -239,13 +242,7 @@ impl Debug for Token {
 		match &self.inner {
 			InnerToken::Command(arg0) => f.debug_tuple("Command").field(arg0).finish(),
 			InnerToken::CodeBlock(arg0) => f.debug_tuple("CodeBlock").field(arg0).finish(),
-			InnerToken::Integer(arg0) => f.debug_tuple("Integer").field(arg0).finish(),
-			InnerToken::Decimal(arg0) => f.debug_tuple("Decimal").field(arg0).finish(),
-			InnerToken::Identifier(arg0) => f.debug_tuple("Identifier").field(arg0).finish(),
-			InnerToken::String(arg0) => f.debug_tuple("String").field(arg0).finish(),
-			InnerToken::Boolean(arg0) => f.debug_tuple("Boolean").field(arg0).finish(),
-			InnerToken::ListStart => f.debug_tuple("ListStart").finish(),
-			InnerToken::Curry => f.debug_tuple("Curry").finish(),
+			InnerToken::Literals(arg0) => f.debug_list().entries(arg0).finish(),
 			InnerToken::WhileBody => f.debug_tuple("WhileBody").finish(),
 			InnerToken::SwitchBody => f.debug_tuple("SwitchBody").finish(),
 		}?;