More thorough support for newlines across the grammar

src/main/kotlin/com/intellij/plugin/powershell/ide/editor/PowerShellFoldingBuilder.kt

@@ -86,7 +86,7 @@ class PowerShellFoldingBuilder : CustomFoldingBuilder(), DumbAware {
 
 fun PsiElement.getNextSiblingNonWhiteSpace(): PsiElement? {
   var next = this.nextSibling
-  while (next != null && (next is PsiWhiteSpace || next.node.elementType == NLS)) {
+  while (next != null && (next is PsiWhiteSpace || next.node.elementType == NEWLINE)) {
     next = next.nextSibling
   }
   return next

src/main/kotlin/com/intellij/plugin/powershell/ide/editor/formatting/PSFormattingUtils.kt

@@ -50,7 +50,7 @@ internal fun isSwitchStatementContext(node: ASTNode): Boolean {
 
 //internal fun isFirstInChainedCall(node: ASTNode): Boolean {
 //  if (node.treeParent?.elementType !== PowerShellTypes.INVOCATION_EXPRESSION) return false
-//  return findSiblingSkipping(node, arrayOf(PowerShellTypes.NLS, TokenType.WHITE_SPACE, PowerShellTypes.DOT, PowerShellTypes.COLON2), false)?.elementType !== PowerShellTypes.INVOCATION_EXPRESSION
+//  return findSiblingSkipping(node, arrayOf(PowerShellTypes.NEWLINE, TokenType.WHITE_SPACE, PowerShellTypes.DOT, PowerShellTypes.COLON2), false)?.elementType !== PowerShellTypes.INVOCATION_EXPRESSION
 //}
 
 internal fun isSubExpressionContext(node: ASTNode): Boolean {
@@ -231,16 +231,16 @@ internal fun isParenthesizedExpressionContext(node: ASTNode): Boolean {
   return node.treeParent?.elementType === PowerShellTypes.PARENTHESIZED_EXPRESSION
 }
 
-internal fun isFollowedByAttribute(childNode: ASTNode) = findSiblingSkipping(childNode, arrayOf(PowerShellTypes.NLS, TokenType.WHITE_SPACE), false)?.elementType === PowerShellTypes.ATTRIBUTE
+internal fun isFollowedByAttribute(childNode: ASTNode) = findSiblingSkipping(childNode, arrayOf(PowerShellTypes.NEWLINE, TokenType.WHITE_SPACE), false)?.elementType === PowerShellTypes.ATTRIBUTE
 
 internal fun isRhsBinaryExpressionContext(node: ASTNode): Boolean {
   if (!isBinaryExpressionContext(node)) return false
-  val type = findSiblingSkipping(node, arrayOf(PowerShellTypes.NLS, TokenType.WHITE_SPACE), false)?.elementType
+  val type = findSiblingSkipping(node, arrayOf(PowerShellTypes.NEWLINE, TokenType.WHITE_SPACE), false)?.elementType
   return node.psi is PowerShellExpression && PowerShellTokenTypeSets.OPERATORS.contains(type)
 }
 
 internal fun isFirstNodeInForParameter(node: ASTNode): Boolean {
-  val prevIEType = findSiblingSkipping(node, arrayOf(PowerShellTypes.NLS, TokenType.WHITE_SPACE), false)?.elementType
+  val prevIEType = findSiblingSkipping(node, arrayOf(PowerShellTypes.NEWLINE, TokenType.WHITE_SPACE), false)?.elementType
   return node.treeParent.elementType === PowerShellTypes.FOR_CLAUSE && (prevIEType === PowerShellTypes.SEMI || prevIEType === PowerShellTypes.LP)
 }
 
@@ -258,12 +258,12 @@ internal fun isArrayElement(node: ASTNode): Boolean {
 
 internal fun isInvocationExpressionQualifier(node: ASTNode): Boolean {
   if (node.treeParent?.elementType !== PowerShellTypes.INVOCATION_EXPRESSION) return false
-  val prevElement = findSiblingSkipping(node, arrayOf(PowerShellTypes.NLS, TokenType.WHITE_SPACE), false)?.elementType
+  val prevElement = findSiblingSkipping(node, arrayOf(PowerShellTypes.NEWLINE, TokenType.WHITE_SPACE), false)?.elementType
   return (prevElement === PowerShellTypes.DOT || prevElement === PowerShellTypes.COLON2) && (node.elementType === PowerShellTypes.REFERENCE_IDENTIFIER || node.psi is PowerShellExpression)
 }
 
 internal fun isFirstNodeInParameter(node: ASTNode, checkFirstParameter: Boolean = false): Boolean {
-  val prevSibling = findSiblingSkipping(node, arrayOf(PowerShellTypes.NLS, TokenType.WHITE_SPACE), false)
+  val prevSibling = findSiblingSkipping(node, arrayOf(PowerShellTypes.NEWLINE, TokenType.WHITE_SPACE), false)
   return prevSibling != null && (prevSibling.elementType === PowerShellTypes.COMMA || (checkFirstParameter && prevSibling.elementType === PowerShellTypes.LP))
 }
 

src/main/kotlin/com/intellij/plugin/powershell/ide/editor/formatting/PowerShellBlockImpl.kt

@@ -132,7 +132,7 @@ open class PowerShellBlockImpl(node: ASTNode, wrap: Wrap?, alignment: Alignment?
         val type = child.elementType
         if (type === INVOCATION_EXPRESSION) {
           collectNodes(nodes, child)
-        } else if (type !== NLS) {
+        } else if (type !== NEWLINE) {
           nodes.add(child)
         }
       }
@@ -225,7 +225,7 @@ open class PowerShellBlockImpl(node: ASTNode, wrap: Wrap?, alignment: Alignment?
   }
 
   private fun createWrap(childNode: ASTNode): Wrap {
-    if ((isFunctionParameter(childNode) || isBlockParameter(childNode)) && findSiblingSkipping(childNode, arrayOf(NLS, WHITE_SPACE), false)?.elementType != LP) {
+    if ((isFunctionParameter(childNode) || isBlockParameter(childNode)) && findSiblingSkipping(childNode, arrayOf(NEWLINE, WHITE_SPACE), false)?.elementType != LP) {
       val firstNode = isFirstNodeInParameter(childNode, true)
       val wrapValue = if (isFunctionParameter(childNode)) myCommonSettings.METHOD_PARAMETERS_WRAP else myPSSettings.BLOCK_PARAMETER_CLAUSE_WRAP
       val paramWrap = Wrap.createWrap(WrappingUtil.getWrapType(wrapValue), firstNode)
@@ -246,7 +246,7 @@ open class PowerShellBlockImpl(node: ASTNode, wrap: Wrap?, alignment: Alignment?
       return paramWrap
     }
 
-    if (isCallArgument(childNode) && findSiblingSkipping(childNode, arrayOf(NLS, WHITE_SPACE), false)?.elementType != LP) {
+    if (isCallArgument(childNode) && findSiblingSkipping(childNode, arrayOf(NEWLINE, WHITE_SPACE), false)?.elementType != LP) {
       return Wrap.createWrap(WrappingUtil.getWrapType(myCommonSettings.CALL_PARAMETERS_WRAP), isFirstNodeInParameter(childNode, true))
     }
 
@@ -373,7 +373,7 @@ class PowerShellSpacingProcessor(private val myCommonSettings: CommonCodeStyleSe
 
     if (LCURLY === type1 || RCURLY === type2) {
       val braceNode = if (LCURLY === type1) node1 else node2
-      val blockBody = findSiblingSkipping(braceNode, arrayOf(NLS, WHITE_SPACE), braceNode === node1)
+      val blockBody = findSiblingSkipping(braceNode, arrayOf(NEWLINE, WHITE_SPACE), braceNode === node1)
       val blockBodyToken = blockBody?.elementType
       if (blockBodyToken === IDENTIFIER || blockBodyToken === SIMPLE_ID) return null
 
@@ -403,7 +403,7 @@ class PowerShellSpacingProcessor(private val myCommonSettings: CommonCodeStyleSe
 
     if (LP === type1) {
       if (isArgumentListContext(node1)) {
-        val treeNext = findSiblingSkipping(node1, arrayOf(NLS, WHITE_SPACE))
+        val treeNext = findSiblingSkipping(node1, arrayOf(NEWLINE, WHITE_SPACE))
         val lfAfterLparen = myCommonSettings.CALL_PARAMETERS_WRAP != DO_NOT_WRAP && myCommonSettings.CALL_PARAMETERS_LPAREN_ON_NEXT_LINE
         if (treeNext?.elementType === RP) {
           val lfAfterRparen = myCommonSettings.CALL_PARAMETERS_WRAP != DO_NOT_WRAP && myCommonSettings.CALL_PARAMETERS_RPAREN_ON_NEXT_LINE
@@ -413,7 +413,7 @@ class PowerShellSpacingProcessor(private val myCommonSettings: CommonCodeStyleSe
         return createSpacing(myCommonSettings.SPACE_WITHIN_METHOD_CALL_PARENTHESES, lfAfterLparen)
       }
       if (isParameterClauseContext(node1) || isBlockParameterClauseContext(node1)) {
-        val treeNext = findSiblingSkipping(node1, arrayOf(NLS, WHITE_SPACE))
+        val treeNext = findSiblingSkipping(node1, arrayOf(NEWLINE, WHITE_SPACE))
         val methodParam = isParameterClauseContext(node1)
         val wrapValue = if (methodParam) myCommonSettings.METHOD_PARAMETERS_WRAP else myPowerShellSettings.BLOCK_PARAMETER_CLAUSE_WRAP
         val lParenNextLine = if (methodParam) myCommonSettings.METHOD_PARAMETERS_LPAREN_ON_NEXT_LINE else myPowerShellSettings.BLOCK_PARAMETERS_LPAREN_ON_NEXT_LINE
@@ -447,14 +447,14 @@ class PowerShellSpacingProcessor(private val myCommonSettings: CommonCodeStyleSe
     }
     if (RP === type2) {
       if (isArgumentListContext(node2)) {
-        val treePrev = findSiblingSkipping(node2, arrayOf(NLS, WHITE_SPACE), false)
+        val treePrev = findSiblingSkipping(node2, arrayOf(NEWLINE, WHITE_SPACE), false)
         val lfAfterRparen = myCommonSettings.CALL_PARAMETERS_WRAP != DO_NOT_WRAP && myCommonSettings.CALL_PARAMETERS_RPAREN_ON_NEXT_LINE
         if (treePrev?.elementType === LP) return createSpacing(myCommonSettings.SPACE_WITHIN_EMPTY_METHOD_CALL_PARENTHESES, lfAfterRparen)
 
         return createSpacing(myCommonSettings.SPACE_WITHIN_METHOD_CALL_PARENTHESES, lfAfterRparen)
       }
       if (isParameterClauseContext(node2) || isBlockParameterClauseContext(node1)) {
-        val treePrev = findSiblingSkipping(node2, arrayOf(NLS, WHITE_SPACE), false)
+        val treePrev = findSiblingSkipping(node2, arrayOf(NEWLINE, WHITE_SPACE), false)
         val methodParam = isParameterClauseContext(node2)
         val wrapValue = if (methodParam) myCommonSettings.METHOD_PARAMETERS_WRAP else myPowerShellSettings.BLOCK_PARAMETER_CLAUSE_WRAP
         val rParenNextLine = if (methodParam) myCommonSettings.METHOD_PARAMETERS_RPAREN_ON_NEXT_LINE else myPowerShellSettings.BLOCK_PARAMETERS_RPAREN_ON_NEXT_LINE
@@ -641,7 +641,7 @@ class PowerShellSpacingProcessor(private val myCommonSettings: CommonCodeStyleSe
 
     if (type2 == SEMI) return simpleSpacing(myCommonSettings.SPACE_BEFORE_SEMICOLON)
 
-    if ((PowerShellTokenTypeSets.KEYWORDS.contains(type1) && !isIdentifier(node1.treeParent)/*|| HANDLER_PARAMETER_LABEL === type1*/) && NLS !== type2) {
+    if ((PowerShellTokenTypeSets.KEYWORDS.contains(type1) && !isIdentifier(node1.treeParent)/*|| HANDLER_PARAMETER_LABEL === type1*/) && NEWLINE !== type2) {
       return Spacing.createSpacing(1, 1, 0, true, 0)
     }
 

src/main/kotlin/com/intellij/plugin/powershell/psi/PowerShellPsiUtil.kt

@@ -16,7 +16,7 @@ internal fun findSiblingSkipping(node: ASTNode, toSkip: Array<IElementType>, for
 }
 
 internal fun findSiblingSkippingWS(node: ASTNode, forward: Boolean = true): ASTNode? {
-  return findSiblingSkipping(node, arrayOf(PowerShellTypes.NLS, TokenType.WHITE_SPACE), forward)
+  return findSiblingSkipping(node, arrayOf(PowerShellTypes.NEWLINE, TokenType.WHITE_SPACE), forward)
 }
 
 fun isTargetVariableContext(node: ASTNode): Boolean {
@@ -29,4 +29,4 @@ fun isFunctionDeclarationContext(node: ASTNode): Boolean {
 
 fun isPathExpressionContext(node: ASTNode): Boolean {
   return node.treeParent.elementType == PATH_EXPRESSION
-}
+}

src/main/kotlin/com/intellij/plugin/powershell/psi/impl/PowerShellPsiImplUtil.kt

@@ -14,7 +14,7 @@ private fun isWhiteSpace(node: ASTNode): Boolean {
 }
 
 fun ASTNode.isWhiteSpaceOrNls(): Boolean {
-  return isWhiteSpace(this) || elementType === PowerShellTypes.NLS || elementType === PowerShellTypes.LF
+  return isWhiteSpace(this) || elementType === PowerShellTypes.NEWLINE || elementType === PowerShellTypes.LF
 }
 
 val PowerShellAssignmentExpression.targetVariables: List<PowerShellTargetVariableExpression>

src/main/resources/PowerShell.bnf

@@ -26,10 +26,8 @@
     OP_BNOT='-bnot'
     EXCL_MARK='!'
 
-  // NewLines and spaces //
-    NLS ='regexp:[\ \t\f]*(\r|\n|\r\n)((\r|\n|\r\n)|([\ \t\f])*)*'
-    LF = 'regexp:(\r|\n|\r\n)+'
-
+  // A single newline
+  NEWLINE = 'regexp:(\r|\n|\r\n)'
 
     LP='('
     RP=')'
@@ -381,7 +379,7 @@ cast_expression ::= type_literal_expression unary_expression
 
 //private attributed_expression ::= type_literal variable
 
-left pipeline_tail ::= ( PIPE nls? command_call_expression )+
+left pipeline_tail ::= ( NEWLINE? PIPE nls? command_call_expression )+
 {elementType=pipeline}
 
 command_call_expression ::= command_invocation_operator ( (command_module command_name_expr command_element* ) | command_name_expr command_element* ) | command_name_expression command_element*
@@ -793,9 +791,9 @@ if_statement ::= 'if' nls? LP nls? pipeline nls? RP nls? statement_block (nls? e
 elseif_clause ::= 'elseif' nls? LP nls? pipeline nls? RP nls? statement_block {pin=1}
 else_clause ::= 'else' nls? statement_block {pin=1}
 
-private sep ::= SEMI |  NLS | new_line_char | comment //COMMENT //hide psi
+private sep ::= SEMI | (NEWLINE space*) | new_line_char | comment //COMMENT //hide psi
 private statement_end ::= sep | RCURLY | RP | <<eof>>//ex: parethnisized_expression $AllNodes.Where{$_.Role -eq "WebServer"}
-private nls ::= (comment? NLS comment?)+ //NLS
+private nls ::= (comment? NEWLINE space* comment?)+
 private new_line_char ::= LF
 
 comment ::= !is_id requires_comment | !is_id SINGLE_LINE_COMMENT | DELIMITED_COMMENT

src/main/resources/_PowerShellLexer.flex

@@ -159,7 +159,6 @@ OP_BOR={DASH}[bB]{OR}
 EXCL_MARK="!"
 
 NL=(\r|\n|\r\n)
-NLS={WHITE_SPACE}*{NL}({NL}|{WHITE_SPACE}*)*
 WS_OR_NL={WHITE_SPACE}|{NL}
 CH_DQ=(\"|“|”|„)
 CH_SQ=(\'|‘|’|‚|‛)
@@ -302,12 +301,12 @@ BRACED_VAR_START={DS}{LCURLY}
 <VERBATIM_ARGUMENT> {
   {VERBATIM_ARG_INPUT}                                         { return VERBATIM_ARG_INPUT; }
   "|"                                                          { popState(); return PIPE; }
-  {NLS}                                                        { popState(); return NLS; }
+  {NL}                                                         { popState(); return NEWLINE; }
 }
 <FUNCTION_ID> {
   {SIMPLE_ID}                                                  { popState(); return SIMPLE_ID; }
   {WHITE_SPACE}                                                { return WHITE_SPACE; }
-  {NLS}                                                        { return NLS; }
+  {NL}                                                         { return NEWLINE; }
   {GENERIC_ID_PART_TOKENS}                                     { popState(); return GENERIC_ID_PART; }
   [^]                                                          { popState(); yypushback(yylength()); }
 }
@@ -420,7 +419,7 @@ BRACED_VAR_START={DS}{LCURLY}
   {OP_OR}/[^a-zA-Z]+{PARAMETER_CHAR}*      { return OP_OR; }
   {OP_XOR}/[^a-zA-Z]+{PARAMETER_CHAR}*     { return OP_XOR; }
   {EXCL_MARK}                      { return EXCL_MARK; }
-  {NLS}                            { return NLS; }
+  {NL}                             { return NEWLINE; }
   {CH_DQ}                          { pushState(STRING); return DQ_OPEN; }
   {VERBATIM_STRING}                { return VERBATIM_STRING; }
   {EXPANDABLE_HERE_STRING_START}   { pushState(HERE_STRING); return EXPANDABLE_HERE_STRING_START; }

src/test/kotlin/com/intellij/plugin/powershell/lang/PowerShellParserTest.kt

@@ -40,4 +40,5 @@ class PowerShellParserTest : ParsingTestCase("parser", "ps1", PowerShellParserDe
   fun testEnumDeclaration() { doTest(true) }
   fun testParamBlock() { doTest(true) }
   fun testComment() { doTest(true) }
+  fun testNewLineThenPipeOperator() { doTest(true) }
 }

src/test/resources/testData/parser/NewLineThenPipeOperator.ps1

@@ -0,0 +1,6 @@
+1, 2, 3
+| Foreach-Object { $_ }
+
+1, 2, 3
+
+| Foreach-Object { $_ }

src/test/resources/testData/parser/NewLineThenPipeOperator.txt

@@ -0,0 +1,64 @@
+FILE(0,65)
+  PowerShellPipelineTailImplGen(PIPELINE)(0,31)
+    PowerShellArrayLiteralExpressionImplGen(ARRAY_LITERAL_EXPRESSION)(0,7)
+      PowerShellIntegerLiteralExpressionImplGen(INTEGER_LITERAL_EXPRESSION)(0,1)
+        PsiElement(DEC_INTEGER)('1')(0,1)
+      PsiElement(,)(',')(1,2)
+      PsiWhiteSpace(' ')(2,3)
+      PowerShellIntegerLiteralExpressionImplGen(INTEGER_LITERAL_EXPRESSION)(3,4)
+        PsiElement(DEC_INTEGER)('2')(3,4)
+      PsiElement(,)(',')(4,5)
+      PsiWhiteSpace(' ')(5,6)
+      PowerShellIntegerLiteralExpressionImplGen(INTEGER_LITERAL_EXPRESSION)(6,7)
+        PsiElement(DEC_INTEGER)('3')(6,7)
+    PsiElement(NLS)('\n')(7,8)
+    PsiElement(|)('|')(8,9)
+    PsiWhiteSpace(' ')(9,10)
+    PowerShellCommandCallExpressionImplGen(COMMAND_CALL_EXPRESSION)(10,31)
+      PowerShellCommandNameImplGen(COMMAND_NAME)(10,24)
+        PowerShellIdentifierImplGen(IDENTIFIER)(10,24)
+          PsiElement(GENERIC_ID_PART)('Foreach-Object')(10,24)
+      PsiWhiteSpace(' ')(24,25)
+      PowerShellCommandArgumentImplGen(COMMAND_ARGUMENT)(25,31)
+        PowerShellScriptBlockExpressionImplGen(SCRIPT_BLOCK_EXPRESSION)(25,31)
+          PsiElement({)('{')(25,26)
+          PsiWhiteSpace(' ')(26,27)
+          PowerShellBlockBodyImplGen(BLOCK_BODY)(27,29)
+            PowerShellTargetVariableExpressionImplGen(TARGET_VARIABLE_EXPRESSION)(27,29)
+              PsiElement($)('$')(27,28)
+              PowerShellIdentifierImplGen(IDENTIFIER)(28,29)
+                PsiElement(SIMPLE_ID)('_')(28,29)
+          PsiWhiteSpace(' ')(29,30)
+          PsiElement(})('}')(30,31)
+  PsiElement(NLS)('\n\n')(31,33)
+  PowerShellPipelineTailImplGen(PIPELINE)(33,65)
+    PowerShellArrayLiteralExpressionImplGen(ARRAY_LITERAL_EXPRESSION)(33,40)
+      PowerShellIntegerLiteralExpressionImplGen(INTEGER_LITERAL_EXPRESSION)(33,34)
+        PsiElement(DEC_INTEGER)('1')(33,34)
+      PsiElement(,)(',')(34,35)
+      PsiWhiteSpace(' ')(35,36)
+      PowerShellIntegerLiteralExpressionImplGen(INTEGER_LITERAL_EXPRESSION)(36,37)
+        PsiElement(DEC_INTEGER)('2')(36,37)
+      PsiElement(,)(',')(37,38)
+      PsiWhiteSpace(' ')(38,39)
+      PowerShellIntegerLiteralExpressionImplGen(INTEGER_LITERAL_EXPRESSION)(39,40)
+        PsiElement(DEC_INTEGER)('3')(39,40)
+    PsiElement(NLS)('\n\n')(40,42)
+    PsiElement(|)('|')(42,43)
+    PsiWhiteSpace(' ')(43,44)
+    PowerShellCommandCallExpressionImplGen(COMMAND_CALL_EXPRESSION)(44,65)
+      PowerShellCommandNameImplGen(COMMAND_NAME)(44,58)
+        PowerShellIdentifierImplGen(IDENTIFIER)(44,58)
+          PsiElement(GENERIC_ID_PART)('Foreach-Object')(44,58)
+      PsiWhiteSpace(' ')(58,59)
+      PowerShellCommandArgumentImplGen(COMMAND_ARGUMENT)(59,65)
+        PowerShellScriptBlockExpressionImplGen(SCRIPT_BLOCK_EXPRESSION)(59,65)
+          PsiElement({)('{')(59,60)
+          PsiWhiteSpace(' ')(60,61)
+          PowerShellBlockBodyImplGen(BLOCK_BODY)(61,63)
+            PowerShellTargetVariableExpressionImplGen(TARGET_VARIABLE_EXPRESSION)(61,63)
+              PsiElement($)('$')(61,62)
+              PowerShellIdentifierImplGen(IDENTIFIER)(62,63)
+                PsiElement(SIMPLE_ID)('_')(62,63)
+          PsiWhiteSpace(' ')(63,64)
+          PsiElement(})('}')(64,65)