rebuild and retest

Author: JohnMount

build/lib/data_algebra/SparkSQL.py

@@ -12,8 +12,24 @@
 except ImportError:
     have_Spark = False
 
+
+def _sparksql_mean_expr(dbmodel, expression):
+    return (
+        "AVG(" + dbmodel.expr_to_sql(expression.args[0], want_inline_parens=False) + ")"
+    )
+
+
+def _sparksql_size_expr(dbmodel, expression):
+    return "SUM(1)"
+
+
+
 # map from op-name to special SQL formatting code
-SparkSQL_formatters = {"___": lambda dbmodel, expression: expression.to_python()}
+SparkSQL_formatters = {
+    "___": lambda dbmodel, expression: expression.to_python(),
+    "mean": _sparksql_mean_expr,
+    "size": _sparksql_size_expr,
+}
 
 
 class SparkConnection:
@@ -43,14 +59,6 @@ def __init__(self):
             sql_formatters=SparkSQL_formatters,
         )
 
-    def quote_identifier(self, identifier):
-        if not isinstance(identifier, str):
-            raise TypeError("expected identifier to be a str")
-        if self.identifier_quote in identifier:
-            raise ValueError('did not expect " in identifier')
-        # TODO: see if we can get rid of the tolower conversion
-        return self.identifier_quote + identifier.lower() + self.identifier_quote
-
     # noinspection PyMethodMayBeStatic
     def execute(self, conn, q):
         assert isinstance(conn, SparkConnection)

coverage.txt

@@ -79,7 +79,7 @@ data_algebra/MySQL.py                     33     11    67%
 data_algebra/OrderedSet.py                52      7    87%
 data_algebra/PostgreSQL.py                22      7    68%
 data_algebra/SQLite.py                    72      6    92%
-data_algebra/SparkSQL.py                  64     41    36%
+data_algebra/SparkSQL.py                  62     39    37%
 data_algebra/__init__.py                   6      0   100%
 data_algebra/arrow.py                    173     39    77%
 data_algebra/bigquery_user_fns.py         83     58    30%
@@ -104,7 +104,7 @@ data_algebra/test_util.py                181     27    85%
 data_algebra/user_fn.py                   84     27    68%
 data_algebra/util.py                      28      4    86%
 ----------------------------------------------------------
-TOTAL                                   4464    957    79%
+TOTAL                                   4462    955    79%
 
 
-============================= 162 passed in 25.63s =============================
+============================= 162 passed in 26.33s =============================

dist/data_algebra-0.7.2-py3-none-any.whl

@@ -760,7 +760,7 @@ <h5>Inherited Members</h5>
     *,
     db_model=BigQueryModel,
     conn,
-    fns={&#39;as_int64&#39;: &lt;function as_int64 at 0x7fc20913c700&gt;, &#39;as_str&#39;: &lt;function as_str at 0x7fc20913c040&gt;, &#39;trimstr&#39;: &lt;function trimstr at 0x7fc20913c0d0&gt;, &#39;coalesce_0&#39;: &lt;function coalesce_0 at 0x7fc20913c310&gt;, &#39;coalesce&#39;: &lt;function coalesce at 0x7fc20913c550&gt;, &#39;datetime_to_date&#39;: &lt;function datetime_to_date at 0x7fc1b84184c0&gt;, &#39;parse_datetime&#39;: &lt;function parse_datetime at 0x7fc1b841e160&gt;, &#39;parse_date&#39;: &lt;function parse_date at 0x7fc1b841e1f0&gt;, &#39;format_datetime&#39;: &lt;function format_datetime at 0x7fc1b841e280&gt;, &#39;format_date&#39;: &lt;function format_date at 0x7fc1b841e310&gt;, &#39;dayofweek&#39;: &lt;function dayofweek at 0x7fc1b841e3a0&gt;, &#39;dayofyear&#39;: &lt;function dayofyear at 0x7fc1b841e430&gt;, &#39;dayofmonth&#39;: &lt;function dayofmonth at 0x7fc1b841e550&gt;, &#39;weekofyear&#39;: &lt;function weekofyear at 0x7fc1b841e4c0&gt;, &#39;month&#39;: &lt;function month at 0x7fc1b841e5e0&gt;, &#39;quarter&#39;: &lt;function quarter at 0x7fc1b841e670&gt;, &#39;year&#39;: &lt;function year at 0x7fc1b841e700&gt;, &#39;timestamp_diff&#39;: &lt;function timestamp_diff at 0x7fc1b841e790&gt;, &#39;date_diff&#39;: &lt;function date_diff at 0x7fc1b841e820&gt;, &#39;base_Sunday&#39;: &lt;function base_Sunday at 0x7fc1b841e8b0&gt;}
+    fns={&#39;as_int64&#39;: &lt;function as_int64 at 0x7f9478aac700&gt;, &#39;as_str&#39;: &lt;function as_str at 0x7f9478aac040&gt;, &#39;trimstr&#39;: &lt;function trimstr at 0x7f9478aac0d0&gt;, &#39;coalesce_0&#39;: &lt;function coalesce_0 at 0x7f9478aac310&gt;, &#39;coalesce&#39;: &lt;function coalesce at 0x7f9478aac550&gt;, &#39;datetime_to_date&#39;: &lt;function datetime_to_date at 0x7f9478cff4c0&gt;, &#39;parse_datetime&#39;: &lt;function parse_datetime at 0x7f9478d05160&gt;, &#39;parse_date&#39;: &lt;function parse_date at 0x7f9478d051f0&gt;, &#39;format_datetime&#39;: &lt;function format_datetime at 0x7f9478d05280&gt;, &#39;format_date&#39;: &lt;function format_date at 0x7f9478d05310&gt;, &#39;dayofweek&#39;: &lt;function dayofweek at 0x7f9478d053a0&gt;, &#39;dayofyear&#39;: &lt;function dayofyear at 0x7f9478d05430&gt;, &#39;dayofmonth&#39;: &lt;function dayofmonth at 0x7f9478d05550&gt;, &#39;weekofyear&#39;: &lt;function weekofyear at 0x7f9478d054c0&gt;, &#39;month&#39;: &lt;function month at 0x7f9478d055e0&gt;, &#39;quarter&#39;: &lt;function quarter at 0x7f9478d05670&gt;, &#39;year&#39;: &lt;function year at 0x7f9478d05700&gt;, &#39;timestamp_diff&#39;: &lt;function timestamp_diff at 0x7f9478d05790&gt;, &#39;date_diff&#39;: &lt;function date_diff at 0x7f9478d05820&gt;, &#39;base_Sunday&#39;: &lt;function base_Sunday at 0x7f9478d058b0&gt;}
 )</span>
     </div>
 

dist/data_algebra-0.7.2.tar.gz

@@ -48,9 +48,6 @@ <h2>API Documentation</h2>
                         <li>
                                 <a class="function" href="#SparkSQLModel.__init__">SparkSQLModel</a>
                         </li>
-                        <li>
-                                <a class="function" href="#SparkSQLModel.quote_identifier">quote_identifier</a>
-                        </li>
                         <li>
                                 <a class="function" href="#SparkSQLModel.execute">execute</a>
                         </li>
@@ -98,8 +95,24 @@ <h1 class="modulename">
 <span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
     <span class="n">have_Spark</span> <span class="o">=</span> <span class="kc">False</span>
 
+
+<span class="k">def</span> <span class="nf">_sparksql_mean_expr</span><span class="p">(</span><span class="n">dbmodel</span><span class="p">,</span> <span class="n">expression</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">(</span>
+        <span class="s2">&quot;AVG(&quot;</span> <span class="o">+</span> <span class="n">dbmodel</span><span class="o">.</span><span class="n">expr_to_sql</span><span class="p">(</span><span class="n">expression</span><span class="o">.</span><span class="n">args</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">want_inline_parens</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span> <span class="o">+</span> <span class="s2">&quot;)&quot;</span>
+    <span class="p">)</span>
+
+
+<span class="k">def</span> <span class="nf">_sparksql_size_expr</span><span class="p">(</span><span class="n">dbmodel</span><span class="p">,</span> <span class="n">expression</span><span class="p">):</span>
+    <span class="k">return</span> <span class="s2">&quot;SUM(1)&quot;</span>
+
+
+
 <span class="c1"># map from op-name to special SQL formatting code</span>
-<span class="n">SparkSQL_formatters</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;___&quot;</span><span class="p">:</span> <span class="k">lambda</span> <span class="n">dbmodel</span><span class="p">,</span> <span class="n">expression</span><span class="p">:</span> <span class="n">expression</span><span class="o">.</span><span class="n">to_python</span><span class="p">()}</span>
+<span class="n">SparkSQL_formatters</span> <span class="o">=</span> <span class="p">{</span>
+    <span class="s2">&quot;___&quot;</span><span class="p">:</span> <span class="k">lambda</span> <span class="n">dbmodel</span><span class="p">,</span> <span class="n">expression</span><span class="p">:</span> <span class="n">expression</span><span class="o">.</span><span class="n">to_python</span><span class="p">(),</span>
+    <span class="s2">&quot;mean&quot;</span><span class="p">:</span> <span class="n">_sparksql_mean_expr</span><span class="p">,</span>
+    <span class="s2">&quot;size&quot;</span><span class="p">:</span> <span class="n">_sparksql_size_expr</span><span class="p">,</span>
+<span class="p">}</span>
 
 
 <span class="k">class</span> <span class="nc">SparkConnection</span><span class="p">:</span>
@@ -129,14 +142,6 @@ <h1 class="modulename">
             <span class="n">sql_formatters</span><span class="o">=</span><span class="n">SparkSQL_formatters</span><span class="p">,</span>
         <span class="p">)</span>
 
-    <span class="k">def</span> <span class="nf">quote_identifier</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">identifier</span><span class="p">):</span>
-        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">identifier</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
-            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">&quot;expected identifier to be a str&quot;</span><span class="p">)</span>
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">identifier_quote</span> <span class="ow">in</span> <span class="n">identifier</span><span class="p">:</span>
-            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;did not expect &quot; in identifier&#39;</span><span class="p">)</span>
-        <span class="c1"># TODO: see if we can get rid of the tolower conversion</span>
-        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">identifier_quote</span> <span class="o">+</span> <span class="n">identifier</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">identifier_quote</span>
-
     <span class="c1"># noinspection PyMethodMayBeStatic</span>
     <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn</span><span class="p">,</span> <span class="n">q</span><span class="p">):</span>
         <span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">conn</span><span class="p">,</span> <span class="n">SparkConnection</span><span class="p">)</span>
@@ -294,14 +299,6 @@ <h1 class="modulename">
             <span class="n">sql_formatters</span><span class="o">=</span><span class="n">SparkSQL_formatters</span><span class="p">,</span>
         <span class="p">)</span>
 
-    <span class="k">def</span> <span class="nf">quote_identifier</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">identifier</span><span class="p">):</span>
-        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">identifier</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
-            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">&quot;expected identifier to be a str&quot;</span><span class="p">)</span>
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">identifier_quote</span> <span class="ow">in</span> <span class="n">identifier</span><span class="p">:</span>
-            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;did not expect &quot; in identifier&#39;</span><span class="p">)</span>
-        <span class="c1"># TODO: see if we can get rid of the tolower conversion</span>
-        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">identifier_quote</span> <span class="o">+</span> <span class="n">identifier</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">identifier_quote</span>
-
     <span class="c1"># noinspection PyMethodMayBeStatic</span>
     <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">conn</span><span class="p">,</span> <span class="n">q</span><span class="p">):</span>
         <span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">conn</span><span class="p">,</span> <span class="n">SparkConnection</span><span class="p">)</span>
@@ -366,30 +363,6 @@ <h1 class="modulename">
 
 
 
-                            </div>
-                            <div id="SparkSQLModel.quote_identifier" class="classattr">
-                                        <div class="attr function"><a class="headerlink" href="#SparkSQLModel.quote_identifier">#&nbsp;&nbsp</a>
-
-        
-            <span class="def">def</span>
-            <span class="name">quote_identifier</span><span class="signature">(self, identifier)</span>:
-    </div>
-
-                <details>
-            <summary>View Source</summary>
-            <div class="codehilite"><pre><span></span>    <span class="k">def</span> <span class="nf">quote_identifier</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">identifier</span><span class="p">):</span>
-        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">identifier</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
-            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">&quot;expected identifier to be a str&quot;</span><span class="p">)</span>
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">identifier_quote</span> <span class="ow">in</span> <span class="n">identifier</span><span class="p">:</span>
-            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;did not expect &quot; in identifier&#39;</span><span class="p">)</span>
-        <span class="c1"># TODO: see if we can get rid of the tolower conversion</span>
-        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">identifier_quote</span> <span class="o">+</span> <span class="n">identifier</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">identifier_quote</span>
-</pre></div>
-
-        </details>
-
-    
-
                             </div>
                             <div id="SparkSQLModel.execute" class="classattr">
                                         <div class="attr function"><a class="headerlink" href="#SparkSQLModel.execute">#&nbsp;&nbsp</a>
@@ -506,6 +479,7 @@ <h5>Inherited Members</h5>
                 <dd id="SparkSQLModel.drop_table" class="function"><a href="db_model.html#DBModel.drop_table">drop_table</a></dd>
                 <dd id="SparkSQLModel.read_table" class="function"><a href="db_model.html#DBModel.read_table">read_table</a></dd>
                 <dd id="SparkSQLModel.read" class="function"><a href="db_model.html#DBModel.read">read</a></dd>
+                <dd id="SparkSQLModel.quote_identifier" class="function"><a href="db_model.html#DBModel.quote_identifier">quote_identifier</a></dd>
                 <dd id="SparkSQLModel.quote_table_name" class="function"><a href="db_model.html#DBModel.quote_table_name">quote_table_name</a></dd>
                 <dd id="SparkSQLModel.quote_string" class="function"><a href="db_model.html#DBModel.quote_string">quote_string</a></dd>
                 <dd id="SparkSQLModel.quote_literal" class="function"><a href="db_model.html#DBModel.quote_literal">quote_literal</a></dd>