Merge pull request #13 from mindsdb/fix-oracle

Update for oracle TPC DS/H benchmarks

Author: ea-rus

mindsdb_sql_parser/__about__.py

@@ -1,6 +1,6 @@
 __title__ = 'mindsdb_sql_parser'
 __package_name__ = 'mindsdb_sql_parser'
-__version__ = '0.5.0'
+__version__ = '0.6.0'
 __description__ = "Mindsdb SQL parser"
 __email__ = "[email protected]"
 __author__ = 'MindsDB Inc'

mindsdb_sql_parser/ast/create.py

@@ -9,13 +9,14 @@
 
 
 class TableColumn():
-    def __init__(self, name, type='integer', length=None, default=None,
+    def __init__(self, name, type='integer', length=None, length2=None, default=None,
                  is_primary_key=False, nullable=None):
         self.name = name
         self.type = type
         self.is_primary_key = is_primary_key
         self.default = default
         self.length = length
+        self.length2 = length2
         self.nullable = nullable
 
     def __eq__(self, other):
@@ -97,7 +98,10 @@ def get_string(self, *args, **kwargs):
                 else:
                     type = str(col.type)
                 if col.length is not None:
-                    type = f'{type}({col.length})'
+                    len_str = str(col.length)
+                    if col.length2 is not None:
+                        len_str += f", {col.length2}"
+                    type = f'{type}({len_str})'
                 col_str = f'{col.name} {type}'
                 if col.nullable is True:
                     col_str += ' NULL'

mindsdb_sql_parser/ast/select/identifier.py

@@ -39,7 +39,7 @@ def get_reserved_words():
 
 
 class Identifier(ASTNode):
-    def __init__(self, path_str=None, parts=None, *args, **kwargs):
+    def __init__(self, path_str=None, parts=None, is_outer=False, *args, **kwargs):
         super().__init__(*args, **kwargs)
         assert path_str or parts, "Either path_str or parts must be provided for an Identifier"
         assert not (path_str and parts), "Provide either path_str or parts, but not both"
@@ -54,6 +54,8 @@ def __init__(self, path_str=None, parts=None, *args, **kwargs):
         self.parts = parts
         # parts which were quoted
         self.is_quoted: List[bool] = is_quoted
+        # used to define type of implicit join in oracle
+        self.is_outer: bool = is_outer
 
     @classmethod
     def from_path_str(self, value, *args, **kwargs):

mindsdb_sql_parser/lexer.py

@@ -53,10 +53,12 @@ class MindsDBLexer(Lexer):
         GROUP_BY, HAVING, ORDER_BY,
         STAR, FOR, UPDATE,
 
-        JOIN, INNER, OUTER, CROSS, LEFT, RIGHT, ON, ASOF, LATERAL,
+        JOIN, INNER, OUTER, CROSS, LEFT, RIGHT, ON, ASOF, LATERAL, IS_OUTER,
 
         UNION, ALL, INTERSECT, EXCEPT,
 
+        FETCH_FIRST, ROWS_ONLY,
+
         # CASE
         CASE, ELSE, END, THEN, WHEN,
 
@@ -165,7 +167,7 @@ class MindsDBLexer(Lexer):
     STATUS = r'\bSTATUS\b'
     GLOBAL = r'\bGLOBAL\b'
     PROCEDURE = r'\bPROCEDURE\b'
-    PRIMARY_KEY = r'\bPRIMARY[_|\s]KEY\b'
+    PRIMARY_KEY = r'\bPRIMARY[\s]KEY\b'
     DEFAULT = r'\bDEFAULT\b'
     FUNCTION = r'\bFUNCTION\b'
     INDEX = r'\bINDEX\b'
@@ -228,6 +230,9 @@ class MindsDBLexer(Lexer):
     STAR = r'\*'
     FOR = r'\bFOR\b'
     UPDATE = r'\bUPDATE\b'
+    # FETCH FIRST and FETCH NEXT are the same
+    FETCH_FIRST = r'\bFETCH[\s](FIRST|NEXT)\b'
+    ROWS_ONLY = r'\bROWS[\s]ONLY\b'
 
     JOIN = r'\bJOIN\b'
     INNER = r'\bINNER\b'
@@ -237,6 +242,7 @@ class MindsDBLexer(Lexer):
     RIGHT = r'\bRIGHT\b'
     ASOF = r'\bASOF\b'
     LATERAL = r'\bLATERAL\b'
+    IS_OUTER = r'\(\+\)'
 
     # UNION
 

mindsdb_sql_parser/parser.py

@@ -713,6 +713,7 @@ def drop_table(self, p):
        'id id DEFAULT id',
        'id id PRIMARY_KEY',
        'id id LPAREN INTEGER RPAREN',
+       'id id LPAREN INTEGER COMMA INTEGER RPAREN',
        'id id LPAREN INTEGER RPAREN DEFAULT id',
        'PRIMARY_KEY LPAREN column_list RPAREN',
        )
@@ -730,10 +731,18 @@ def table_column(self, p):
         elif hasattr(p, 'PRIMARY_KEY'):
             is_primary_key = True
 
+        length, length2 = None, None
+        if hasattr(p, 'INTEGER'):
+            length = p.INTEGER
+        elif hasattr(p, 'INTEGER0'):
+            length = p.INTEGER0
+            length2 = p.INTEGER1
+
         return TableColumn(
             name=p[0],
             type=p[1],
-            length=getattr(p, 'INTEGER', None),
+            length=length,
+            length2=length2,
             default=default,
             is_primary_key=is_primary_key
         )
@@ -1100,6 +1109,7 @@ def select(self, p):
         return select
 
     @_('select LIMIT constant')
+    @_('select FETCH_FIRST constant ROWS_ONLY')
     def select(self, p):
         select = p.select
         ensure_select_keyword_order(select, 'LIMIT')
@@ -1765,6 +1775,12 @@ def identifier(self, p):
     def string(self, p):
         return p[0]
 
+    @_('identifier IS_OUTER')
+    def identifier(self, p):
+        value = p[0]
+        value.is_outer = True
+        return value
+
     @_('id', 'dquote_string')
     def identifier(self, p):
         value = p[0]

tests/test_base_sql/test_create.py

@@ -137,6 +137,8 @@ def test_create(self):
             location_id INT,
             num INT,
             name TEXT,
+            balance decimal(5,2),
+            links NUMBER(10, 0) NOT NULL,
             PRIMARY KEY (location_id, num)  
          )
         '''
@@ -148,6 +150,8 @@ def test_create(self):
                 TableColumn(name='location_id', type='INT', is_primary_key=True),
                 TableColumn(name='num', type='INT', is_primary_key=True),
                 TableColumn(name='name', type='TEXT'),
+                TableColumn(name='balance', type='decimal', length=5, length2=2),
+                TableColumn(name='links', type='NUMBER', nullable=False, length=10, length2=0),
             ]
         )
 

tests/test_mindsdb/test_oracle.py

@@ -0,0 +1,42 @@
+from mindsdb_sql_parser import parse_sql
+from mindsdb_sql_parser.ast import *
+from mindsdb_sql_parser.utils import JoinType
+
+
+class TestOracle:
+    def test_left_outer_join_render_skip(self):
+        sql = "SELECT * FROM customer, orders "\
+              "WHERE c_custkey = o_custkey(+)"
+
+        ast = parse_sql(sql)
+        expected_ast = Select(
+            targets=[Star()],
+            from_table=Join(left=Identifier('customer'),
+                            right=Identifier('orders'),
+                            join_type=JoinType.INNER_JOIN,
+                            implicit=True),
+            where=BinaryOperation('=', args=[
+                Identifier('c_custkey'), Identifier('o_custkey', is_outer=True)
+            ]),
+        )
+
+        # don't render (+)
+        assert str(ast).lower() == sql.replace('(+)', '').lower()
+        assert str(ast) == str(expected_ast)
+        assert ast.to_tree() == expected_ast.to_tree()
+
+        assert ast.where.args[1].is_outer is True
+
+    def test_limit(self):
+        for keyword in ('NEXT', 'FIRST'):
+            sql = f"SELECT * FROM customer FETCH {keyword} 10 ROWS ONLY"
+
+            ast = parse_sql(sql)
+            expected_ast = Select(
+                targets=[Star()],
+                from_table=Identifier('customer'),
+                limit=Constant(10)
+            )
+
+            assert str(ast) == str(expected_ast)
+            assert ast.to_tree() == expected_ast.to_tree()