Merge branch 'mindsdb:main' into issue-fix-parameterized-queries

Author: pankaj-bind

.github/workflows/pypi.yml

@@ -9,11 +9,13 @@ jobs:
   deploy_to_pypi:
     name: Publish to PyPI
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
     if: github.actor != 'mindsdbadmin'
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v5.1.0
+        uses: actions/setup-python@v5.6.0
         with:
           python-version: ${{ vars.CI_PYTHON_VERSION }}
       - name: Install dependencies

.github/workflows/test.yml

@@ -8,14 +8,16 @@ on:
 jobs:
   test:
     runs-on: ${{ matrix.os }}
+    permissions:
+      contents: read
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
-        python-version: [3.8,3.9,'3.10']
+        python-version: ["3.9","3.10","3.11","3.12","3.13"]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5.6.0
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
@@ -35,11 +37,11 @@ jobs:
     permissions:
       pull-requests: write
     steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v5.6.0
         with:
-          python-version: 3.8
+          python-version: 3.9
 
       - name: Install dependencies
         run: |

mindsdb_sql_parser/__about__.py

@@ -1,6 +1,6 @@
 __title__ = 'mindsdb_sql_parser'
 __package_name__ = 'mindsdb_sql_parser'
-__version__ = '0.10.0'
+__version__ = '0.10.1'
 __description__ = "Mindsdb SQL parser"
 __email__ = "[email protected]"
 __author__ = 'MindsDB Inc'

mindsdb_sql_parser/ast/select/union.py

@@ -9,11 +9,13 @@ def __init__(self,
                  left,
                  right,
                  unique=True,
+                 distinct_key=False,
                  *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.left = left
         self.right = right
         self.unique = unique
+        self.distinct_key = distinct_key
 
         if self.alias:
             self.parentheses = True
@@ -26,7 +28,7 @@ def to_tree(self, *args, level=0, **kwargs):
         right_str = f'\n{ind1}right=\n{self.right.to_tree(level=level + 2)},'
 
         cls_name = self.__class__.__name__
-        out_str = f'{ind}{cls_name}(unique={repr(self.unique)},' \
+        out_str = f'{ind}{cls_name}(unique={repr(self.unique)}, distinct_key={repr(self.distinct_key)}' \
                   f'{left_str}' \
                   f'{right_str}' \
                   f'\n{ind})'
@@ -38,6 +40,8 @@ def get_string(self, *args, **kwargs):
         keyword = self.operation
         if not self.unique:
             keyword += ' ALL'
+        if self.distinct_key:
+            keyword += ' DISTINCT'
         out_str = f"""{left_str}\n{keyword}\n{right_str}"""
 
         return out_str

mindsdb_sql_parser/parser.py

@@ -1061,25 +1061,35 @@ def database_engine(self, p):
     @_('select UNION select',
        'union UNION select',
        'select UNION ALL select',
-       'union UNION ALL select')
+       'union UNION ALL select',
+       'select UNION DISTINCT select',
+       'union UNION DISTINCT select')
     def union(self, p):
         unique = not hasattr(p, 'ALL')
-        return Union(left=p[0], right=p[2] if unique else p[3], unique=unique)
+        distinct_key = hasattr(p, 'DISTINCT')
+        return Union(left=p[0], right=p[-1], unique=unique, distinct_key=distinct_key)
 
     @_('select INTERSECT select',
        'union INTERSECT select',
        'select INTERSECT ALL select',
-       'union INTERSECT ALL select')
+       'union INTERSECT ALL select',
+       'select INTERSECT DISTINCT select',
+       'union INTERSECT DISTINCT select')
     def union(self, p):
         unique = not hasattr(p, 'ALL')
-        return Intersect(left=p[0], right=p[2] if unique else p[3], unique=unique)
+        distinct_key = hasattr(p, 'DISTINCT')
+        return Intersect(left=p[0], right=p[-1], unique=unique, distinct_key=distinct_key)
+
     @_('select EXCEPT select',
        'union EXCEPT select',
        'select EXCEPT ALL select',
-       'union EXCEPT ALL select')
+       'union EXCEPT ALL select',
+       'select EXCEPT DISTINCT select',
+       'union EXCEPT DISTINCT select')
     def union(self, p):
         unique = not hasattr(p, 'ALL')
-        return Except(left=p[0], right=p[2] if unique else p[3], unique=unique)
+        distinct_key = hasattr(p, 'DISTINCT')
+        return Except(left=p[0], right=p[-1], unique=unique, distinct_key=distinct_key)
 
     # tableau
     @_('LPAREN select RPAREN')

setup.py

@@ -32,7 +32,7 @@ def run(self):
         "Programming Language :: Python :: 3",
         "Operating System :: OS Independent",
     ],
-    python_requires=">=3.6",
+    python_requires=">=3.9",
     cmdclass={
         'build': Build
     },

tests/test_base_sql/test_union.py

@@ -12,21 +12,26 @@ def test_single_select_error(self):
 
     def test_union_base(self):
         for keyword, cls in {'union': Union, 'intersect': Intersect, 'except': Except}.items():
-            sql = f"""SELECT col1 FROM tab1 
-            {keyword} 
-            SELECT col1 FROM tab2"""
+            for rule in ['', 'distinct']:
+                sql = f"""SELECT col1 FROM tab1 
+                {keyword} {rule}
+                SELECT col1 FROM tab2"""
 
-            ast = parse_sql(sql)
-            expected_ast = cls(unique=True,
-                                 left=Select(targets=[Identifier('col1')],
-                                             from_table=Identifier(parts=['tab1']),
-                                             ),
-                                 right=Select(targets=[Identifier('col1')],
-                                             from_table=Identifier(parts=['tab2']),
-                                             ),
-                                 )
-            assert ast.to_tree() == expected_ast.to_tree()
-            assert str(ast) == str(expected_ast)
+                ast = parse_sql(sql)
+                expected_ast = cls(
+                    unique=True,
+                    distinct_key=rule == 'distinct',
+                    left=Select(
+                        targets=[Identifier('col1')],
+                        from_table=Identifier(parts=['tab1']),
+                    ),
+                    right=Select(
+                        targets=[Identifier('col1')],
+                        from_table=Identifier(parts=['tab2']),
+                    ),
+                )
+                assert ast.to_tree() == expected_ast.to_tree()
+                assert str(ast) == str(expected_ast)
 
     def test_union_all(self):
         for keyword, cls in {'union': Union, 'intersect': Intersect, 'except': Except}.items():