Merge branch 'main' into complete-and-easy

Author: ysthakur

.github/workflows/ci.yml

@@ -16,10 +16,10 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        # Pinning to Ubuntu 20.04 because building on newer Ubuntu versions causes linux-gnu
+        # Pinning to Ubuntu 24.04 because building on newer Ubuntu versions causes linux-gnu
         # builds to link against a too-new-for-many-Linux-installs glibc version. Consider
-        # revisiting this when 20.04 is closer to EOL (April 2025)
-        platform: [macos-latest, ubuntu-20.04]
+        # revisiting this when 24.04 is closer to EOL
+        platform: [macos-latest, ubuntu-24.04]
         feature: [default]
         include:
           - feature: default
@@ -49,7 +49,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        platform: [macos-latest, ubuntu-20.04]
+        platform: [macos-latest, ubuntu-24.04]
         feature: [default]
         include:
           - feature: default

src/compiler.rs

@@ -1,5 +1,5 @@
 use crate::errors::SourceError;
-use crate::parser::{AstNode, Block, NodeId};
+use crate::parser::{AstNode, Block, NodeId, Pipeline};
 use crate::protocol::Command;
 use crate::resolver::{DeclId, Frame, NameBindings, ScopeId, TypeDecl, TypeDeclId, VarId, Variable};
 use crate::typechecker::{TypeId, Types};
@@ -44,7 +44,8 @@ pub struct Compiler {
     pub ast_nodes: Vec<AstNode>,
     pub node_types: Vec<TypeId>,
     // node_lifetimes: Vec<AllocationLifetime>,
-    pub blocks: Vec<Block>, // Blocks, indexed by BlockId
+    pub blocks: Vec<Block>,       // Blocks, indexed by BlockId
+    pub pipelines: Vec<Pipeline>, // Pipelines, indexed by PipelineId
     pub source: Vec<u8>,
     pub file_offsets: Vec<(String, usize, usize)>, // fname, start, end
 
@@ -92,6 +93,7 @@ impl Compiler {
             ast_nodes: vec![],
             node_types: vec![],
             blocks: vec![],
+            pipelines: vec![],
             source: vec![],
             file_offsets: vec![],
 

src/parser.rs

@@ -15,6 +15,9 @@ pub struct NodeId(pub usize);
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub struct BlockId(pub usize);
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct PipelineId(pub usize);
+
 #[derive(Debug, Clone)]
 pub struct Block {
     pub nodes: Vec<NodeId>,
@@ -26,6 +29,32 @@ impl Block {
     }
 }
 
+// Pipeline just contains a list of expressions
+//
+// It's not allowed if there is only one element in pipeline, in that
+// case, it's just an expression.
+//
+// Making such restriction can reduce indirect access on expression, which
+// can improve performance in parse time.
+#[derive(Debug, Clone, PartialEq)]
+pub struct Pipeline {
+    pub nodes: Vec<NodeId>,
+}
+
+impl Pipeline {
+    pub fn new(nodes: Vec<NodeId>) -> Self {
+        debug_assert!(
+            nodes.len() > 1,
+            "a pipeline must contain at least 2 nodes, or else it's actually an expression"
+        );
+        Self { nodes }
+    }
+
+    pub fn get_expressions(&self) -> &Vec<NodeId> {
+        &self.nodes
+    }
+}
+
 #[derive(Debug, Clone, PartialEq)]
 pub enum BlockContext {
     /// This block is a whole block of code not wrapped in curlies (e.g., a file)
@@ -54,6 +83,19 @@ pub enum BarewordContext {
     Call,
 }
 
+enum AssignmentOrExpression {
+    Assignment(NodeId),
+    Expression(NodeId),
+}
+
+impl AssignmentOrExpression {
+    fn get_node_id(&self) -> NodeId {
+        match self {
+            AssignmentOrExpression::Assignment(i) | AssignmentOrExpression::Expression(i) => *i,
+        }
+    }
+}
+
 // TODO: All nodes with Vec<...> should be moved to their own ID (like BlockId) to allow Copy trait
 #[derive(Debug, PartialEq, Clone)]
 pub enum AstNode {
@@ -196,6 +238,7 @@ pub enum AstNode {
         field: NodeId,
     },
     Block(BlockId),
+    Pipeline(PipelineId),
     If {
         condition: NodeId,
         then_block: NodeId,
@@ -261,17 +304,57 @@ impl Parser {
         self.compiler
     }
 
-    pub fn expression_or_assignment(&mut self) -> NodeId {
+    pub fn expression(&mut self) -> NodeId {
         let _span = span!();
-        self.math_expression(true)
+        self.math_expression(false).get_node_id()
     }
 
-    pub fn expression(&mut self) -> NodeId {
+    fn pipeline(&mut self, first_element: NodeId, span_start: usize) -> NodeId {
+        let mut expressions = vec![first_element];
+        while self.is_pipe() {
+            self.pipe();
+            // maybe a new time
+            if self.is_newline() {
+                self.tokens.advance()
+            }
+            expressions.push(self.expression());
+        }
+        self.compiler.pipelines.push(Pipeline::new(expressions));
+        let span_end = self.position();
+        self.create_node(
+            AstNode::Pipeline(PipelineId(self.compiler.pipelines.len() - 1)),
+            span_start,
+            span_end,
+        )
+    }
+    pub fn pipeline_or_expression_or_assignment(&mut self) -> NodeId {
+        // get the first expression
         let _span = span!();
-        self.math_expression(false)
+        let span_start = self.position();
+        let first = self.math_expression(true);
+        let first_id = first.get_node_id();
+        if let AssignmentOrExpression::Assignment(_) = &first {
+            return first_id;
+        }
+        // pipeline with one element is an expression actually
+        if !self.is_pipe() {
+            return first_id;
+        }
+        self.pipeline(first_id, span_start)
     }
 
-    pub fn math_expression(&mut self, allow_assignment: bool) -> NodeId {
+    pub fn pipeline_or_expression(&mut self) -> NodeId {
+        let _span = span!();
+        let span_start = self.position();
+        let first_id = self.expression();
+        // pipeline with one element is an expression actually.
+        if !self.is_pipe() {
+            return first_id;
+        }
+        self.pipeline(first_id, span_start)
+    }
+
+    fn math_expression(&mut self, allow_assignment: bool) -> AssignmentOrExpression {
         let _span = span!();
         let mut expr_stack = Vec::<(NodeId, NodeId)>::new();
 
@@ -281,9 +364,9 @@ impl Parser {
 
         // Check for special forms
         if self.is_keyword(b"if") {
-            return self.if_expression();
+            return AssignmentOrExpression::Expression(self.if_expression());
         } else if self.is_keyword(b"match") {
-            return self.match_expression();
+            return AssignmentOrExpression::Expression(self.match_expression());
         }
         // TODO
         // } else if self.is_keyword(b"where") {
@@ -298,18 +381,18 @@ impl Parser {
             }
             let op = self.operator();
 
-            let rhs = self.expression();
+            let rhs = self.pipeline_or_expression();
             let span_end = self.get_span_end(rhs);
 
-            return self.create_node(
+            return AssignmentOrExpression::Assignment(self.create_node(
                 AstNode::BinaryOp {
                     lhs: leftmost,
                     op,
                     rhs,
                 },
                 span_start,
                 span_end,
-            );
+            ));
         }
 
         while self.has_tokens() {
@@ -380,7 +463,7 @@ impl Parser {
             );
         }
 
-        leftmost
+        AssignmentOrExpression::Expression(leftmost)
     }
 
     pub fn simple_expression(&mut self, bareword_context: BarewordContext) -> NodeId {
@@ -1153,7 +1236,7 @@ impl Parser {
 
         self.equals();
 
-        let initializer = self.expression();
+        let initializer = self.pipeline_or_expression();
 
         let span_end = self.get_span_end(initializer);
 
@@ -1190,7 +1273,7 @@ impl Parser {
 
         self.equals();
 
-        let initializer = self.expression();
+        let initializer = self.pipeline_or_expression();
 
         let span_end = self.get_span_end(initializer);
 
@@ -1259,19 +1342,19 @@ impl Parser {
                 code_body.push(self.alias_statement());
             } else {
                 let exp_span_start = self.position();
-                let expression = self.expression_or_assignment();
-                let exp_span_end = self.get_span_end(expression);
+                let pipeline = self.pipeline_or_expression_or_assignment();
+                let exp_span_end = self.get_span_end(pipeline);
 
                 if self.is_semicolon() {
                     // This is a statement, not an expression
                     self.tokens.advance();
                     code_body.push(self.create_node(
-                        AstNode::Statement(expression),
+                        AstNode::Statement(pipeline),
                         exp_span_start,
                         exp_span_end,
                     ))
                 } else {
-                    code_body.push(expression);
+                    code_body.push(pipeline);
                 }
             }
         }

src/resolver.rs

@@ -2,7 +2,7 @@ use crate::protocol::{Command, Declaration};
 use crate::{
     compiler::Compiler,
     errors::{Severity, SourceError},
-    parser::{AstNode, BlockId, NodeId},
+    parser::{AstNode, BlockId, NodeId, PipelineId},
 };
 use std::collections::HashMap;
 
@@ -427,13 +427,22 @@ impl<'a> Resolver<'a> {
                 self.resolve_node(in_ty);
                 self.resolve_node(out_ty);
             }
+            AstNode::Pipeline(pipeline_id) => self.resolve_pipeline(pipeline_id),
             AstNode::Param { .. } => (/* seems unused for now */),
             AstNode::NamedValue { .. } => (/* seems unused for now */),
             // All remaining matches do not contain NodeId => there is nothing to resolve
             _ => (),
         }
     }
 
+    pub fn resolve_pipeline(&mut self, pipeline_id: PipelineId) {
+        let pipeline = &self.compiler.pipelines[pipeline_id.0];
+
+        for exp in pipeline.get_expressions() {
+            self.resolve_node(*exp)
+        }
+    }
+
     pub fn resolve_variable(&mut self, unbound_node_id: NodeId) {
         let var_name = trim_var_name(self.compiler.get_span_contents(unbound_node_id));
 

src/snapshots/new_nu_parser__test__node_output@let_.nu.snap

@@ -9,18 +9,31 @@ snapshot_kind: text
 1: Int (8 to 11) "123"
 2: Let { variable_name: NodeId(0), ty: None, initializer: NodeId(1), is_mutable: false } (0 to 11)
 3: Variable (13 to 15) "$x"
-4: Block(BlockId(0)) (0 to 15)
+4: Variable (21 to 22) "x"
+5: Int (25 to 28) "123"
+6: Int (31 to 34) "456"
+7: Pipeline(PipelineId(0)) (25 to 34)
+8: Let { variable_name: NodeId(4), ty: None, initializer: NodeId(7), is_mutable: false } (17 to 34)
+9: Variable (36 to 38) "$x"
+10: Block(BlockId(0)) (0 to 39)
 ==== SCOPE ====
-0: Frame Scope, node_id: NodeId(4)
-  variables: [ x: NodeId(0) ]
+0: Frame Scope, node_id: NodeId(10)
+  variables: [ x: NodeId(4) ]
 ==== TYPES ====
 0: int
 1: int
 2: ()
 3: int
 4: int
+5: int
+6: int
+7: int
+8: ()
+9: int
+10: int
 ==== IR ====
 register_count: 0
 file_count: 0
 ==== IR ERRORS ====
 Error (NodeId 2): node Let { variable_name: NodeId(0), ty: None, initializer: NodeId(1), is_mutable: false } not suported yet
+

src/snapshots/new_nu_parser__test__node_output@mut_.nu.snap

@@ -2,6 +2,7 @@
 source: src/test.rs
 expression: evaluate_example(path)
 input_file: tests/mut_.nu
+snapshot_kind: text
 ---
 ==== COMPILER ====
 0: Variable (4 to 5) "x"
@@ -16,10 +17,24 @@ input_file: tests/mut_.nu
 9: Int (27 to 30) "456"
 10: BinaryOp { lhs: NodeId(7), op: NodeId(8), rhs: NodeId(9) } (23 to 30)
 11: BinaryOp { lhs: NodeId(5), op: NodeId(6), rhs: NodeId(10) } (18 to 30)
-12: Block(BlockId(0)) (0 to 30)
+12: Variable (36 to 37) "y"
+13: Name (39 to 42) "int"
+14: Type { name: NodeId(13), args: None, optional: false } (39 to 42)
+15: Int (45 to 48) "123"
+16: Int (51 to 54) "344"
+17: Pipeline(PipelineId(0)) (45 to 54)
+18: Let { variable_name: NodeId(12), ty: Some(NodeId(14)), initializer: NodeId(17), is_mutable: true } (32 to 54)
+19: Variable (56 to 58) "$y"
+20: Assignment (59 to 60)
+21: Int (61 to 62) "3"
+22: Plus (63 to 64)
+23: Int (65 to 66) "1"
+24: BinaryOp { lhs: NodeId(21), op: NodeId(22), rhs: NodeId(23) } (61 to 66)
+25: BinaryOp { lhs: NodeId(19), op: NodeId(20), rhs: NodeId(24) } (56 to 66)
+26: Block(BlockId(0)) (0 to 68)
 ==== SCOPE ====
-0: Frame Scope, node_id: NodeId(12)
-  variables: [ x: NodeId(0) ]
+0: Frame Scope, node_id: NodeId(26)
+  variables: [ x: NodeId(0), y: NodeId(12) ]
 ==== TYPES ====
 0: int
 1: unknown
@@ -33,9 +48,24 @@ input_file: tests/mut_.nu
 9: int
 10: int
 11: ()
-12: ()
+12: int
+13: unknown
+14: int
+15: int
+16: int
+17: int
+18: ()
+19: int
+20: forbidden
+21: int
+22: forbidden
+23: int
+24: int
+25: ()
+26: ()
 ==== IR ====
 register_count: 0
 file_count: 0
 ==== IR ERRORS ====
 Error (NodeId 4): node Let { variable_name: NodeId(0), ty: Some(NodeId(2)), initializer: NodeId(3), is_mutable: true } not suported yet
+