Implement basic generator functions with function* syntax, yield expressions, and iterator protocol

Author: ssrlive

examples/js.rs

@@ -71,6 +71,8 @@ fn print_eval_result(result: &Value) {
         Value::Set(_) => println!("[object Set]"),
         Value::WeakMap(_) => println!("[object WeakMap]"),
         Value::WeakSet(_) => println!("[object WeakSet]"),
+        Value::GeneratorFunction(_, _, _) => println!("[GeneratorFunction]"),
+        Value::Generator(_) => println!("[object Generator]"),
     }
 }
 

src/core.rs

@@ -156,6 +156,7 @@ pub enum Expr {
     Call(Box<Expr>, Vec<Expr>),
     Function(Vec<String>, Vec<Statement>),                // parameters, body
     AsyncFunction(Vec<String>, Vec<Statement>),           // parameters, body for async functions
+    GeneratorFunction(Vec<String>, Vec<Statement>),       // parameters, body for generator functions
     ArrowFunction(Vec<String>, Vec<Statement>),           // parameters, body
     AsyncArrowFunction(Vec<String>, Vec<Statement>),      // parameters, body for async arrow functions
     Object(Vec<(String, Expr)>),                          // object literal: key-value pairs
@@ -167,6 +168,8 @@ pub enum Expr {
     OptionalCall(Box<Expr>, Vec<Expr>),                   // optional call: obj?.method(args)
     OptionalIndex(Box<Expr>, Box<Expr>),                  // optional bracket access: obj?.[expr]
     Await(Box<Expr>),                                     // await expression
+    Yield(Option<Box<Expr>>),                             // yield expression (optional value)
+    YieldStar(Box<Expr>),                                 // yield* expression (delegation)
     This,                                                 // this keyword
     New(Box<Expr>, Vec<Expr>),                            // new expression: new Constructor(args)
     Super,                                                // super keyword

src/core/eval.rs

@@ -1740,6 +1740,7 @@ pub fn evaluate_expr(env: &JSObjectDataPtr, expr: &Expr) -> Result<Value, JSErro
             }
         },
         Expr::Function(params, body) => Ok(Value::Closure(params.clone(), body.clone(), env.clone())),
+        Expr::GeneratorFunction(params, body) => Ok(Value::GeneratorFunction(params.clone(), body.clone(), env.clone())),
         Expr::ArrowFunction(params, body) => Ok(Value::Closure(params.clone(), body.clone(), env.clone())),
         Expr::AsyncArrowFunction(params, body) => Ok(Value::AsyncClosure(params.clone(), body.clone(), env.clone())),
         Expr::Object(properties) => evaluate_object(env, properties),
@@ -1805,6 +1806,14 @@ pub fn evaluate_expr(env: &JSObjectDataPtr, expr: &Expr) -> Result<Value, JSErro
                 _ => Err(raise_eval_error!("await can only be used with promises")),
             }
         }
+        Expr::Yield(_expr) => {
+            // Yield expressions are only valid in generator functions
+            Err(raise_eval_error!("yield expression is only valid in generator functions"))
+        }
+        Expr::YieldStar(_expr) => {
+            // Yield* expressions are only valid in generator functions
+            Err(raise_eval_error!("yield* expression is only valid in generator functions"))
+        }
         Expr::Value(value) => Ok(value.clone()),
         Expr::Regex(pattern, flags) => {
             // Build temporary Expr list to reuse the existing RegExp constructor
@@ -2724,7 +2733,7 @@ fn evaluate_typeof(env: &JSObjectDataPtr, expr: &Expr) -> Result<Value, JSError>
         Value::BigInt(_) => "bigint",
         Value::Object(_) => "object",
         Value::Function(_) => "function",
-        Value::Closure(_, _, _) | Value::AsyncClosure(_, _, _) => "function",
+        Value::Closure(_, _, _) | Value::AsyncClosure(_, _, _) | Value::GeneratorFunction(_, _, _) => "function",
         Value::ClassDefinition(_) => "function",
         Value::Getter(_, _) => "function",
         Value::Setter(_, _, _) => "function",
@@ -2735,6 +2744,7 @@ fn evaluate_typeof(env: &JSObjectDataPtr, expr: &Expr) -> Result<Value, JSError>
         Value::Set(_) => "object",
         Value::WeakMap(_) => "object",
         Value::WeakSet(_) => "object",
+        Value::Generator(_) => "object",
     };
     Ok(Value::String(utf8_to_utf16(type_str)))
 }
@@ -3788,6 +3798,7 @@ fn evaluate_call(env: &JSObjectDataPtr, func_expr: &Expr, args: &[Expr]) -> Resu
             (Value::Set(set), method) => crate::js_set::handle_set_instance_method(&set, method, args, env),
             (Value::WeakMap(weakmap), method) => crate::js_weakmap::handle_weakmap_instance_method(&weakmap, method, args, env),
             (Value::WeakSet(weakset), method) => crate::js_weakset::handle_weakset_instance_method(&weakset, method, args, env),
+            (Value::Generator(generator), method) => crate::js_generator::handle_generator_instance_method(&generator, method, args, env),
             (Value::Object(obj_map), method) => {
                 // Object prototype methods are supplied on `Object.prototype`.
                 // Lookups will find user-defined (own) methods before inherited
@@ -4013,6 +4024,10 @@ fn evaluate_call(env: &JSObjectDataPtr, func_expr: &Expr, args: &[Expr]) -> Resu
         let func_val = evaluate_expr(env, func_expr)?;
         match func_val {
             Value::Function(func_name) => crate::js_function::handle_global_function(&func_name, args, env),
+            Value::GeneratorFunction(params, body, captured_env) => {
+                // Generator function call - return a generator object
+                crate::js_generator::handle_generator_function_call(&params, &body, args, &captured_env)
+            }
             Value::Closure(params, body, captured_env) => {
                 // Function call
                 // Collect all arguments, expanding spreads

src/core/ffi.rs

@@ -943,10 +943,12 @@ pub unsafe fn JS_Eval(_ctx: *mut JSContext, input: *const i8, input_len: usize,
             Ok(Value::Promise(_)) => JS_UNDEFINED,            // For now
             Ok(Value::Symbol(_)) => JS_UNDEFINED,             // For now
             Ok(Value::BigInt(_)) => JS_UNDEFINED,
-            Ok(Value::Map(_)) => JS_UNDEFINED,     // For now
-            Ok(Value::Set(_)) => JS_UNDEFINED,     // For now
-            Ok(Value::WeakMap(_)) => JS_UNDEFINED, // For now
-            Ok(Value::WeakSet(_)) => JS_UNDEFINED, // For now
+            Ok(Value::Map(_)) => JS_UNDEFINED,                     // For now
+            Ok(Value::Set(_)) => JS_UNDEFINED,                     // For now
+            Ok(Value::WeakMap(_)) => JS_UNDEFINED,                 // For now
+            Ok(Value::WeakSet(_)) => JS_UNDEFINED,                 // For now
+            Ok(Value::GeneratorFunction(_, _, _)) => JS_UNDEFINED, // For now
+            Ok(Value::Generator(_)) => JS_UNDEFINED,               // For now
             Err(_) => JS_UNDEFINED,
         }
     }

src/core/parser.rs

@@ -407,6 +407,24 @@ fn parse_primary(tokens: &mut Vec<Token>) -> Result<Expr, JSError> {
             let inner = parse_primary(tokens)?;
             Expr::Await(Box::new(inner))
         }
+        Token::Yield => {
+            // yield can be followed by an optional expression
+            if tokens.is_empty()
+                || matches!(
+                    tokens[0],
+                    Token::Semicolon | Token::Comma | Token::RParen | Token::RBracket | Token::RBrace | Token::Colon
+                )
+            {
+                Expr::Yield(None)
+            } else {
+                let inner = parse_assignment(tokens)?;
+                Expr::Yield(Some(Box::new(inner)))
+            }
+        }
+        Token::YieldStar => {
+            let inner = parse_assignment(tokens)?;
+            Expr::YieldStar(Box::new(inner))
+        }
         Token::LogicalNot => {
             let inner = parse_primary(tokens)?;
             Expr::LogicalNot(Box::new(inner))
@@ -1076,7 +1094,8 @@ fn parse_primary(tokens: &mut Vec<Token>) -> Result<Expr, JSError> {
             }
             Expr::Array(elements)
         }
-        Token::Function => {
+        Token::Function | Token::FunctionStar => {
+            let is_generator = matches!(tokens[0], Token::FunctionStar);
             // Parse function expression
             if !tokens.is_empty() && matches!(tokens[0], Token::LParen) {
                 tokens.remove(0); // consume (
@@ -1114,7 +1133,11 @@ fn parse_primary(tokens: &mut Vec<Token>) -> Result<Expr, JSError> {
                     return Err(raise_parse_error!());
                 }
                 tokens.remove(0); // consume }
-                Expr::Function(params, body)
+                if is_generator {
+                    Expr::GeneratorFunction(params, body)
+                } else {
+                    Expr::Function(params, body)
+                }
             } else {
                 return Err(raise_parse_error!());
             }

src/core/statement.rs

@@ -442,8 +442,9 @@ pub fn parse_statement(tokens: &mut Vec<Token>) -> Result<Statement, JSError> {
         }
         return Err(raise_parse_error!());
     }
-    if !tokens.is_empty() && matches!(tokens[0], Token::Function) {
-        tokens.remove(0); // consume function
+    if !tokens.is_empty() && (matches!(tokens[0], Token::Function) || matches!(tokens[0], Token::FunctionStar)) {
+        let is_generator = matches!(tokens[0], Token::FunctionStar);
+        tokens.remove(0); // consume function or function*
         log::trace!(
             "parse_statement: entered Function branch; next tokens: {:?}",
             tokens.iter().take(12).collect::<Vec<_>>()
@@ -500,7 +501,11 @@ pub fn parse_statement(tokens: &mut Vec<Token>) -> Result<Statement, JSError> {
                     return Err(raise_parse_error!());
                 }
                 tokens.remove(0); // consume }
-                return Ok(Statement::Let(name, Some(Expr::Function(params, body))));
+                if is_generator {
+                    return Ok(Statement::Let(name, Some(Expr::GeneratorFunction(params, body))));
+                } else {
+                    return Ok(Statement::Let(name, Some(Expr::Function(params, body))));
+                }
             }
         }
     }

src/core/token.rs

@@ -90,6 +90,9 @@ pub enum Token {
     Decrement,
     Async,
     Await,
+    Yield,
+    YieldStar,
+    FunctionStar,
     LineTerminator,
     /// Exponentiation assignment (`**=`)
     PowAssign,
@@ -143,6 +146,8 @@ impl Token {
             Token::False => Some("false".to_string()),
             Token::Async => Some("async".to_string()),
             Token::Await => Some("await".to_string()),
+            Token::Yield => Some("yield".to_string()),
+            Token::FunctionStar => Some("function*".to_string()),
             _ => None,
         }
     }
@@ -622,7 +627,15 @@ pub fn tokenize(expr: &str) -> Result<Vec<Token>, JSError> {
                     "catch" => tokens.push(Token::Catch),
                     "finally" => tokens.push(Token::Finally),
                     "throw" => tokens.push(Token::Throw),
-                    "function" => tokens.push(Token::Function),
+                    "function" => {
+                        // Check if followed by '*'
+                        if i < chars.len() && chars[i] == '*' {
+                            tokens.push(Token::FunctionStar);
+                            i += 1; // consume '*'
+                        } else {
+                            tokens.push(Token::Function);
+                        }
+                    }
                     "return" => tokens.push(Token::Return),
                     "if" => tokens.push(Token::If),
                     "else" => tokens.push(Token::Else),
@@ -638,6 +651,15 @@ pub fn tokenize(expr: &str) -> Result<Vec<Token>, JSError> {
                     "false" => tokens.push(Token::False),
                     "async" => tokens.push(Token::Async),
                     "await" => tokens.push(Token::Await),
+                    "yield" => {
+                        // Check if followed by '*'
+                        if i < chars.len() && chars[i] == '*' {
+                            tokens.push(Token::YieldStar);
+                            i += 1; // consume '*'
+                        } else {
+                            tokens.push(Token::Yield);
+                        }
+                    }
                     _ => tokens.push(Token::Identifier(ident)),
                 }
             }

src/core/value.rs

@@ -29,6 +29,22 @@ pub struct JSWeakSet {
     pub values: Vec<std::rc::Weak<RefCell<JSObjectData>>>, // weak values
 }
 
+#[derive(Clone, Debug)]
+pub struct JSGenerator {
+    pub params: Vec<String>,
+    pub body: Vec<Statement>,
+    pub env: JSObjectDataPtr, // captured environment
+    pub state: GeneratorState,
+}
+
+#[derive(Clone, Debug)]
+pub enum GeneratorState {
+    NotStarted,
+    Running { pc: usize, stack: Vec<Value> },   // program counter and value stack
+    Suspended { pc: usize, stack: Vec<Value> }, // suspended at yield
+    Completed,
+}
+
 pub type JSObjectDataPtr = Rc<RefCell<JSObjectData>>;
 
 #[derive(Clone, Default)]
@@ -99,25 +115,27 @@ pub enum Value {
     String(Vec<u16>), // UTF-16 code units
     Boolean(bool),
     Undefined,
-    Object(JSObjectDataPtr),                                    // Object with properties
-    Function(String),                                           // Function name
-    Closure(Vec<String>, Vec<Statement>, JSObjectDataPtr),      // parameters, body, captured environment
-    AsyncClosure(Vec<String>, Vec<Statement>, JSObjectDataPtr), // parameters, body, captured environment
-    ClassDefinition(Rc<ClassDefinition>),                       // Class definition
-    Getter(Vec<Statement>, JSObjectDataPtr),                    // getter body, captured environment
-    Setter(Vec<String>, Vec<Statement>, JSObjectDataPtr),       // setter parameter, body, captured environment
+    Object(JSObjectDataPtr),                                         // Object with properties
+    Function(String),                                                // Function name
+    Closure(Vec<String>, Vec<Statement>, JSObjectDataPtr),           // parameters, body, captured environment
+    AsyncClosure(Vec<String>, Vec<Statement>, JSObjectDataPtr),      // parameters, body, captured environment
+    GeneratorFunction(Vec<String>, Vec<Statement>, JSObjectDataPtr), // parameters, body, captured environment
+    ClassDefinition(Rc<ClassDefinition>),                            // Class definition
+    Getter(Vec<Statement>, JSObjectDataPtr),                         // getter body, captured environment
+    Setter(Vec<String>, Vec<Statement>, JSObjectDataPtr),            // setter parameter, body, captured environment
     Property {
         // Property descriptor with getter/setter/value
         value: Option<Rc<RefCell<Value>>>,
         getter: Option<(Vec<Statement>, JSObjectDataPtr)>,
         setter: Option<(Vec<String>, Vec<Statement>, JSObjectDataPtr)>,
     },
-    Promise(Rc<RefCell<JSPromise>>), // Promise object
-    Symbol(Rc<SymbolData>),          // Symbol primitive with description
-    Map(Rc<RefCell<JSMap>>),         // Map object
-    Set(Rc<RefCell<JSSet>>),         // Set object
-    WeakMap(Rc<RefCell<JSWeakMap>>), // WeakMap object
-    WeakSet(Rc<RefCell<JSWeakSet>>), // WeakSet object
+    Promise(Rc<RefCell<JSPromise>>),     // Promise object
+    Symbol(Rc<SymbolData>),              // Symbol primitive with description
+    Map(Rc<RefCell<JSMap>>),             // Map object
+    Set(Rc<RefCell<JSSet>>),             // Set object
+    WeakMap(Rc<RefCell<JSWeakMap>>),     // WeakMap object
+    WeakSet(Rc<RefCell<JSWeakSet>>),     // WeakSet object
+    Generator(Rc<RefCell<JSGenerator>>), // Generator object
 }
 
 impl std::fmt::Debug for Value {
@@ -132,6 +150,7 @@ impl std::fmt::Debug for Value {
             Value::Function(name) => write!(f, "Function({})", name),
             Value::Closure(_, _, _) => write!(f, "Closure"),
             Value::AsyncClosure(_, _, _) => write!(f, "AsyncClosure"),
+            Value::GeneratorFunction(_, _, _) => write!(f, "GeneratorFunction"),
             Value::ClassDefinition(_) => write!(f, "ClassDefinition"),
             Value::Getter(_, _) => write!(f, "Getter"),
             Value::Setter(_, _, _) => write!(f, "Setter"),
@@ -142,6 +161,7 @@ impl std::fmt::Debug for Value {
             Value::Set(s) => write!(f, "Set({:p})", Rc::as_ptr(s)),
             Value::WeakMap(wm) => write!(f, "WeakMap({:p})", Rc::as_ptr(wm)),
             Value::WeakSet(ws) => write!(f, "WeakSet({:p})", Rc::as_ptr(ws)),
+            Value::Generator(g) => write!(f, "Generator({:p})", Rc::as_ptr(g)),
         }
     }
 }
@@ -171,6 +191,7 @@ pub fn is_truthy(val: &Value) -> bool {
         Value::Function(_) => true,
         Value::Closure(_, _, _) => true,
         Value::AsyncClosure(_, _, _) => true,
+        Value::GeneratorFunction(_, _, _) => true,
         Value::ClassDefinition(_) => true,
         Value::Getter(_, _) => true,
         Value::Setter(_, _, _) => true,
@@ -181,6 +202,7 @@ pub fn is_truthy(val: &Value) -> bool {
         Value::Set(_) => true,
         Value::WeakMap(_) => true,
         Value::WeakSet(_) => true,
+        Value::Generator(_) => true,
     }
 }
 
@@ -210,6 +232,7 @@ pub fn value_to_string(val: &Value) -> String {
         Value::Function(name) => format!("function {}", name),
         Value::Closure(_, _, _) => "function".to_string(),
         Value::AsyncClosure(_, _, _) => "function".to_string(),
+        Value::GeneratorFunction(_, _, _) => "function".to_string(),
         Value::ClassDefinition(_) => "class".to_string(),
         Value::Getter(_, _) => "getter".to_string(),
         Value::Setter(_, _, _) => "setter".to_string(),
@@ -223,6 +246,7 @@ pub fn value_to_string(val: &Value) -> String {
         Value::Set(_) => "[object Set]".to_string(),
         Value::WeakMap(_) => "[object WeakMap]".to_string(),
         Value::WeakSet(_) => "[object WeakSet]".to_string(),
+        Value::Generator(_) => "[object Generator]".to_string(),
     }
 }
 
@@ -310,7 +334,7 @@ pub fn value_to_sort_string(val: &Value) -> String {
         Value::Undefined => "undefined".to_string(),
         Value::Object(_) => "[object Object]".to_string(),
         Value::Function(name) => format!("[function {}]", name),
-        Value::Closure(_, _, _) | Value::AsyncClosure(_, _, _) => "[function]".to_string(),
+        Value::Closure(_, _, _) | Value::AsyncClosure(_, _, _) | Value::GeneratorFunction(_, _, _) => "[function]".to_string(),
         Value::ClassDefinition(_) => "[class]".to_string(),
         Value::Getter(_, _) => "[getter]".to_string(),
         Value::Setter(_, _, _) => "[setter]".to_string(),
@@ -321,6 +345,7 @@ pub fn value_to_sort_string(val: &Value) -> String {
         Value::Set(_) => "[object Set]".to_string(),
         Value::WeakMap(_) => "[object WeakMap]".to_string(),
         Value::WeakSet(_) => "[object WeakSet]".to_string(),
+        Value::Generator(_) => "[object Generator]".to_string(),
     }
 }
 

src/js_class.rs

@@ -672,6 +672,8 @@ pub(crate) fn handle_string_constructor(args: &[Expr], env: &JSObjectDataPtr) ->
             Value::Set(_) => utf8_to_utf16("[object Set]"),
             Value::WeakMap(_) => utf8_to_utf16("[object WeakMap]"),
             Value::WeakSet(_) => utf8_to_utf16("[object WeakSet]"),
+            Value::GeneratorFunction(_, _, _) => utf8_to_utf16("[GeneratorFunction]"),
+            Value::Generator(_) => utf8_to_utf16("[object Generator]"),
         }
     };
 

src/js_console.rs

@@ -109,6 +109,8 @@ pub fn handle_console_method(method: &str, args: &[Expr], env: &JSObjectDataPtr)
                     Value::Set(_) => print!("[object Set]"),
                     Value::WeakMap(_) => print!("[object WeakMap]"),
                     Value::WeakSet(_) => print!("[object WeakSet]"),
+                    Value::GeneratorFunction(_, _, _) => print!("[GeneratorFunction]"),
+                    Value::Generator(_) => print!("[object Generator]"),
                 }
                 if i < count - 1 {
                     print!(" ");