Merge pull request #65 from mindsdb/fix-kw-spaces

Expect spaces between keywords

Author: ea-rus

mindsdb_sql_parser/lexer.py

@@ -127,8 +127,8 @@ class MindsDBLexer(Lexer):
     ANOMALY = r'\bANOMALY\b'
     DETECTION = r'\bDETECTION\b'
 
-    KNOWLEDGE_BASE = r'\bKNOWLEDGE[_|\s]BASE\b'
-    KNOWLEDGE_BASES = r'\bKNOWLEDGE[_|\s]BASES\b'
+    KNOWLEDGE_BASE = r'\bKNOWLEDGE([_]|[\s]+)BASE\b'
+    KNOWLEDGE_BASES = r'\bKNOWLEDGE([_]|[\s]+)BASES\b'
     SKILL = r'\bSKILL\b'
     AGENT = r'\bAGENT\b'
 
@@ -168,7 +168,7 @@ class MindsDBLexer(Lexer):
     STATUS = r'\bSTATUS\b'
     GLOBAL = r'\bGLOBAL\b'
     PROCEDURE = r'\bPROCEDURE\b'
-    PRIMARY_KEY = r'\bPRIMARY[\s]KEY\b'
+    PRIMARY_KEY = r'\bPRIMARY[\s]+KEY\b'
     DEFAULT = r'\bDEFAULT\b'
     FUNCTION = r'\bFUNCTION\b'
     INDEX = r'\bINDEX\b'
@@ -215,8 +215,8 @@ class MindsDBLexer(Lexer):
     ON = r'\bON\b'
     ASC = r'\bASC\b'
     DESC = r'\bDESC\b'
-    NULLS_FIRST = r'\bNULLS FIRST\b'
-    NULLS_LAST = r'\bNULLS LAST\b'
+    NULLS_FIRST = r'\bNULLS[\s]+FIRST\b'
+    NULLS_LAST = r'\bNULLS[\s]+LAST\b'
     WITH = r'\bWITH\b'
     SELECT = r'\bSELECT\b'
     DISTINCT = r'\bDISTINCT\b'
@@ -225,15 +225,15 @@ class MindsDBLexer(Lexer):
     WHERE = r'\bWHERE\b'
     LIMIT = r'\bLIMIT\b'
     OFFSET = r'\bOFFSET\b'
-    GROUP_BY = r'\bGROUP BY\b'
+    GROUP_BY = r'\bGROUP[\s]+BY\b'
     HAVING = r'\bHAVING\b'
-    ORDER_BY = r'\bORDER BY\b'
+    ORDER_BY = r'\bORDER[\s]+BY\b'
     STAR = r'\*'
     FOR = r'\bFOR\b'
     UPDATE = r'\bUPDATE\b'
     # FETCH FIRST and FETCH NEXT are the same
-    FETCH_FIRST = r'\bFETCH[\s](FIRST|NEXT)\b'
-    ROWS_ONLY = r'\bROWS[\s]ONLY\b'
+    FETCH_FIRST = r'\bFETCH[\s]+(FIRST|NEXT)\b'
+    ROWS_ONLY = r'\bROWS[\s]+ONLY\b'
     ROLLUP = r'\bROLLUP\b'
 
     JOIN = r'\bJOIN\b'
@@ -330,7 +330,7 @@ class MindsDBLexer(Lexer):
     INTERVAL = r'\bINTERVAL\b'
     WINDOW = r'\bWINDOW\b'
     OVER = r'\bOVER\b'
-    PARTITION_BY = r'\bPARTITION BY\b'
+    PARTITION_BY = r'\bPARTITION[\s]+BY\b'
 
     # Data types
     NULL = r'\bNULL\b'

mindsdb_sql_parser/parser.py

@@ -1,3 +1,5 @@
+import re
+
 from sly import Parser
 from mindsdb_sql_parser.ast import *
 from mindsdb_sql_parser.ast.drop import DropDatabase, DropView
@@ -559,7 +561,8 @@ def show(self, p):
     def show(self, p):
         modes = getattr(p, 'show_modifier_list', None)
         return Show(
-            category=p.show_category,
+            # replace multiple spaces
+            category=re.sub(r"[\s]+", " ", p.show_category),
             modes=modes
         )
 
@@ -1195,12 +1198,12 @@ def ordering_terms(self, p):
 
     @_('ordering_term NULLS_FIRST')
     def ordering_term(self, p):
-        p.ordering_term.nulls = p.NULLS_FIRST
+        p.ordering_term.nulls = 'NULLS FIRST'
         return p.ordering_term
 
     @_('ordering_term NULLS_LAST')
     def ordering_term(self, p):
-        p.ordering_term.nulls = p.NULLS_LAST
+        p.ordering_term.nulls = 'NULLS LAST'
         return p.ordering_term
 
     @_('ordering_term DESC')

tests/test_base_sql/test_create.py

@@ -113,7 +113,7 @@ def test_create(self):
 
         sql = f'''
          CREATE TABLE mydb.Persons(
-            PersonID INT PRIMARY KEY,
+            PersonID INT PRIMARY   KEY,
             name TEXT NULL   
          )
         '''

tests/test_base_sql/test_select_structure.py

@@ -294,7 +294,7 @@ def test_select_group_by(self):
         ast = parse_sql(sql)
         assert str(ast).lower() == sql.lower()
 
-        sql = f'SELECT column FROM tab WHERE column != 1 GROUP BY column1, column2'
+        sql = f'SELECT column FROM tab WHERE column != 1 GROUP  BY column1, column2'
         ast = parse_sql(sql)
 
         assert isinstance(ast, Select)
@@ -314,7 +314,7 @@ def test_select_group_by(self):
         assert isinstance(ast.group_by[1], Identifier)
         assert ast.group_by[1].parts[0] == 'column2'
 
-        assert str(ast).lower() == sql.lower()
+        assert str(ast).lower() == sql.lower().replace("  ", " ")
 
     def test_select_group_by_elaborate(self):
         query = """SELECT column1, column2, sum(column3) AS total FROM t1 GROUP BY column1, column2"""
@@ -356,7 +356,7 @@ def test_select_group_by_having_elaborate(self):
         assert str(parse_sql(sql)) == sql
 
     def test_select_order_by_elaborate(self):
-        sql = """SELECT * FROM t1 ORDER BY column1 ASC, column2, column3 DESC NULLS FIRST"""
+        sql = """SELECT * FROM t1 ORDER  BY column1 ASC, column2, column3 DESC NULLS  FIRST"""
         ast = parse_sql(sql)
         expected_ast = Select(targets=[Star()],
                                from_table=Identifier(parts=['t1']),
@@ -367,18 +367,18 @@ def test_select_order_by_elaborate(self):
                                            nulls='NULLS FIRST')],
                                )
 
-        assert str(ast).lower() == sql.lower()
+        assert str(ast).lower() == sql.replace("  ", " ").lower()
         assert ast.to_tree() == expected_ast.to_tree()
         assert str(ast) == str(expected_ast)
 
     def test_select_aliases_order_by(self):
-        sql = "select max(name) as `max(name)` from tbl order by `max(name)`"
+        sql = "select max(name) as `max(name)` from tbl order by `max(name)` NULLS  LAST"
 
         ast = parse_sql(sql)
 
         expected_ast = Select(targets=[Function('max', args=[Identifier('name')], alias=Identifier('max(name)'))],
                               from_table=Identifier('tbl'),
-                              order_by=[OrderBy(Identifier('max(name)'))])
+                              order_by=[OrderBy(Identifier('max(name)'), nulls='NULLS LAST')])
 
         assert ast.to_tree() == expected_ast.to_tree()
 
@@ -829,7 +829,7 @@ def test_window_function(self):
         assert ast.to_tree() == expected_ast.to_tree()
 
         # no order by
-        query = "select SUM(col0) OVER (PARTITION BY col1) from table1 "
+        query = "select SUM(col0) OVER (PARTITION  BY col1) from table1 "
         expected_ast = Select(
             targets=[
                 WindowFunction(

tests/test_mindsdb/test_knowledgebase.py

@@ -52,7 +52,7 @@ def test_create_knowledge_base(self):
 
         # the order of MODEL and STORAGE should not matter
         sql = """
-            CREATE KNOWLEDGE_BASE my_knowledge_base
+            CREATE KNOWLEDGE   BASE my_knowledge_base
             USING
                 STORAGE = my_vector_database.some_table,
                 MODEL = mindsdb.my_embedding_model
@@ -224,7 +224,7 @@ def test_show_knowledge_base(self):
 
         # without underscore shall also work
         sql = """
-            SHOW KNOWLEDGE BASES
+            SHOW KNOWLEDGE \n BASES
         """
         ast = parse_sql(sql)
         expected_ast = Show(

tests/test_mindsdb/test_oracle.py

@@ -29,7 +29,7 @@ def test_left_outer_join_render_skip(self):
 
     def test_limit(self):
         for keyword in ('NEXT', 'FIRST'):
-            sql = f"SELECT * FROM customer FETCH {keyword} 10 ROWS ONLY"
+            sql = f"SELECT * FROM customer FETCH  {keyword} 10 ROWS ONLY"
 
             ast = parse_sql(sql)
             expected_ast = Select(