rebuild and retest

Author: JohnMount

build/lib/data_algebra/util.py

@@ -75,16 +75,21 @@ def guess_carried_scalar_type(col):
     :param col: column or scalar to inspect
     :return: type of first non-None entry, if any , else type(None)
     """
+    # check for scalars first
     ct = map_type_to_canonical(type(col))
     if ct in {str, int, float, bool, type(None), numpy.int64, numpy.float64,
               datetime.datetime, datetime.date, datetime.timedelta}:
         return ct
+    # look at a list or Series
+    if isinstance(col, data_algebra.default_data_model.pd.core.series.Series):
+        col = col.values
     if len(col) < 1:
         return type(None)
-    idx = col.notna().idxmax()
-    if idx is None:
-        return map_type_to_canonical(type(col[0]))
-    return map_type_to_canonical(type(col[idx]))
+    good_idx = numpy.where(numpy.logical_not(data_algebra.default_data_model.pd.isna(col)))[0]
+    test_idx = 0
+    if len(good_idx) > 0:
+        test_idx = good_idx[0]
+    return map_type_to_canonical(type(col[test_idx]))
 
 
 def guess_column_types(d, *, columns=None):
@@ -106,7 +111,7 @@ def guess_column_types(d, *, columns=None):
     res = dict()
     for c in columns:
         gt = guess_carried_scalar_type(d[c])
-        if (gt is None) or (not isinstance(gt, type)) or str(gt).endswith('.Series\'>'):
+        if (gt is None) or (not isinstance(gt, type)) or gt == data_algebra.default_data_model.pd.core.series.Series:
             # pandas.concat() poisons types with Series, don't allow that
             return dict()
         res[c] = gt

coverage.txt

@@ -116,9 +116,9 @@ data_algebra/pandas_model.py              18      3    83%
 data_algebra/parse_by_lark.py            143     26    82%
 data_algebra/python3_lark.py               1      0   100%
 data_algebra/test_util.py                202     37    82%
-data_algebra/util.py                      86      6    93%
+data_algebra/util.py                      89      6    93%
 ----------------------------------------------------------
-TOTAL                                   4811    853    82%
+TOTAL                                   4814    853    82%
 
 
-============================= 210 passed in 16.73s =============================
+============================= 210 passed in 16.61s =============================

dist/data_algebra-0.8.0-py3-none-any.whl

@@ -145,16 +145,21 @@ <h1 class="modulename">
 <span class="sd">    :param col: column or scalar to inspect</span>
 <span class="sd">    :return: type of first non-None entry, if any , else type(None)</span>
 <span class="sd">    &quot;&quot;&quot;</span>
+    <span class="c1"># check for scalars first</span>
     <span class="n">ct</span> <span class="o">=</span> <span class="n">map_type_to_canonical</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">col</span><span class="p">))</span>
     <span class="k">if</span> <span class="n">ct</span> <span class="ow">in</span> <span class="p">{</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">float</span><span class="p">,</span> <span class="nb">bool</span><span class="p">,</span> <span class="nb">type</span><span class="p">(</span><span class="kc">None</span><span class="p">),</span> <span class="n">numpy</span><span class="o">.</span><span class="n">int64</span><span class="p">,</span> <span class="n">numpy</span><span class="o">.</span><span class="n">float64</span><span class="p">,</span>
               <span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="p">,</span> <span class="n">datetime</span><span class="o">.</span><span class="n">date</span><span class="p">,</span> <span class="n">datetime</span><span class="o">.</span><span class="n">timedelta</span><span class="p">}:</span>
         <span class="k">return</span> <span class="n">ct</span>
+    <span class="c1"># look at a list or Series</span>
+    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="n">data_algebra</span><span class="o">.</span><span class="n">default_data_model</span><span class="o">.</span><span class="n">pd</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">series</span><span class="o">.</span><span class="n">Series</span><span class="p">):</span>
+        <span class="n">col</span> <span class="o">=</span> <span class="n">col</span><span class="o">.</span><span class="n">values</span>
     <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">col</span><span class="p">)</span> <span class="o">&lt;</span> <span class="mi">1</span><span class="p">:</span>
         <span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="kc">None</span><span class="p">)</span>
-    <span class="n">idx</span> <span class="o">=</span> <span class="n">col</span><span class="o">.</span><span class="n">notna</span><span class="p">()</span><span class="o">.</span><span class="n">idxmax</span><span class="p">()</span>
-    <span class="k">if</span> <span class="n">idx</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="k">return</span> <span class="n">map_type_to_canonical</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">col</span><span class="p">[</span><span class="mi">0</span><span class="p">]))</span>
-    <span class="k">return</span> <span class="n">map_type_to_canonical</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">col</span><span class="p">[</span><span class="n">idx</span><span class="p">]))</span>
+    <span class="n">good_idx</span> <span class="o">=</span> <span class="n">numpy</span><span class="o">.</span><span class="n">where</span><span class="p">(</span><span class="n">numpy</span><span class="o">.</span><span class="n">logical_not</span><span class="p">(</span><span class="n">data_algebra</span><span class="o">.</span><span class="n">default_data_model</span><span class="o">.</span><span class="n">pd</span><span class="o">.</span><span class="n">isna</span><span class="p">(</span><span class="n">col</span><span class="p">)))[</span><span class="mi">0</span><span class="p">]</span>
+    <span class="n">test_idx</span> <span class="o">=</span> <span class="mi">0</span>
+    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">good_idx</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+        <span class="n">test_idx</span> <span class="o">=</span> <span class="n">good_idx</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+    <span class="k">return</span> <span class="n">map_type_to_canonical</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">col</span><span class="p">[</span><span class="n">test_idx</span><span class="p">]))</span>
 
 
 <span class="k">def</span> <span class="nf">guess_column_types</span><span class="p">(</span><span class="n">d</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
@@ -176,7 +181,7 @@ <h1 class="modulename">
     <span class="n">res</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
     <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">columns</span><span class="p">:</span>
         <span class="n">gt</span> <span class="o">=</span> <span class="n">guess_carried_scalar_type</span><span class="p">(</span><span class="n">d</span><span class="p">[</span><span class="n">c</span><span class="p">])</span>
-        <span class="k">if</span> <span class="p">(</span><span class="n">gt</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">)</span> <span class="ow">or</span> <span class="p">(</span><span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">gt</span><span class="p">,</span> <span class="nb">type</span><span class="p">))</span> <span class="ow">or</span> <span class="nb">str</span><span class="p">(</span><span class="n">gt</span><span class="p">)</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s1">&#39;.Series</span><span class="se">\&#39;</span><span class="s1">&gt;&#39;</span><span class="p">):</span>
+        <span class="k">if</span> <span class="p">(</span><span class="n">gt</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">)</span> <span class="ow">or</span> <span class="p">(</span><span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">gt</span><span class="p">,</span> <span class="nb">type</span><span class="p">))</span> <span class="ow">or</span> <span class="n">gt</span> <span class="o">==</span> <span class="n">data_algebra</span><span class="o">.</span><span class="n">default_data_model</span><span class="o">.</span><span class="n">pd</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">series</span><span class="o">.</span><span class="n">Series</span><span class="p">:</span>
             <span class="c1"># pandas.concat() poisons types with Series, don&#39;t allow that</span>
             <span class="k">return</span> <span class="nb">dict</span><span class="p">()</span>
         <span class="n">res</span><span class="p">[</span><span class="n">c</span><span class="p">]</span> <span class="o">=</span> <span class="n">gt</span>
@@ -337,16 +342,21 @@ <h1 class="modulename">
 <span class="sd">    :param col: column or scalar to inspect</span>
 <span class="sd">    :return: type of first non-None entry, if any , else type(None)</span>
 <span class="sd">    &quot;&quot;&quot;</span>
+    <span class="c1"># check for scalars first</span>
     <span class="n">ct</span> <span class="o">=</span> <span class="n">map_type_to_canonical</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">col</span><span class="p">))</span>
     <span class="k">if</span> <span class="n">ct</span> <span class="ow">in</span> <span class="p">{</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">float</span><span class="p">,</span> <span class="nb">bool</span><span class="p">,</span> <span class="nb">type</span><span class="p">(</span><span class="kc">None</span><span class="p">),</span> <span class="n">numpy</span><span class="o">.</span><span class="n">int64</span><span class="p">,</span> <span class="n">numpy</span><span class="o">.</span><span class="n">float64</span><span class="p">,</span>
               <span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="p">,</span> <span class="n">datetime</span><span class="o">.</span><span class="n">date</span><span class="p">,</span> <span class="n">datetime</span><span class="o">.</span><span class="n">timedelta</span><span class="p">}:</span>
         <span class="k">return</span> <span class="n">ct</span>
+    <span class="c1"># look at a list or Series</span>
+    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="n">data_algebra</span><span class="o">.</span><span class="n">default_data_model</span><span class="o">.</span><span class="n">pd</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">series</span><span class="o">.</span><span class="n">Series</span><span class="p">):</span>
+        <span class="n">col</span> <span class="o">=</span> <span class="n">col</span><span class="o">.</span><span class="n">values</span>
     <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">col</span><span class="p">)</span> <span class="o">&lt;</span> <span class="mi">1</span><span class="p">:</span>
         <span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="kc">None</span><span class="p">)</span>
-    <span class="n">idx</span> <span class="o">=</span> <span class="n">col</span><span class="o">.</span><span class="n">notna</span><span class="p">()</span><span class="o">.</span><span class="n">idxmax</span><span class="p">()</span>
-    <span class="k">if</span> <span class="n">idx</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
-        <span class="k">return</span> <span class="n">map_type_to_canonical</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">col</span><span class="p">[</span><span class="mi">0</span><span class="p">]))</span>
-    <span class="k">return</span> <span class="n">map_type_to_canonical</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">col</span><span class="p">[</span><span class="n">idx</span><span class="p">]))</span>
+    <span class="n">good_idx</span> <span class="o">=</span> <span class="n">numpy</span><span class="o">.</span><span class="n">where</span><span class="p">(</span><span class="n">numpy</span><span class="o">.</span><span class="n">logical_not</span><span class="p">(</span><span class="n">data_algebra</span><span class="o">.</span><span class="n">default_data_model</span><span class="o">.</span><span class="n">pd</span><span class="o">.</span><span class="n">isna</span><span class="p">(</span><span class="n">col</span><span class="p">)))[</span><span class="mi">0</span><span class="p">]</span>
+    <span class="n">test_idx</span> <span class="o">=</span> <span class="mi">0</span>
+    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">good_idx</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+        <span class="n">test_idx</span> <span class="o">=</span> <span class="n">good_idx</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+    <span class="k">return</span> <span class="n">map_type_to_canonical</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">col</span><span class="p">[</span><span class="n">test_idx</span><span class="p">]))</span>
 </pre></div>
 
         </details>
@@ -388,7 +398,7 @@ <h1 class="modulename">
     <span class="n">res</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
     <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">columns</span><span class="p">:</span>
         <span class="n">gt</span> <span class="o">=</span> <span class="n">guess_carried_scalar_type</span><span class="p">(</span><span class="n">d</span><span class="p">[</span><span class="n">c</span><span class="p">])</span>
-        <span class="k">if</span> <span class="p">(</span><span class="n">gt</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">)</span> <span class="ow">or</span> <span class="p">(</span><span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">gt</span><span class="p">,</span> <span class="nb">type</span><span class="p">))</span> <span class="ow">or</span> <span class="nb">str</span><span class="p">(</span><span class="n">gt</span><span class="p">)</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s1">&#39;.Series</span><span class="se">\&#39;</span><span class="s1">&gt;&#39;</span><span class="p">):</span>
+        <span class="k">if</span> <span class="p">(</span><span class="n">gt</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">)</span> <span class="ow">or</span> <span class="p">(</span><span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">gt</span><span class="p">,</span> <span class="nb">type</span><span class="p">))</span> <span class="ow">or</span> <span class="n">gt</span> <span class="o">==</span> <span class="n">data_algebra</span><span class="o">.</span><span class="n">default_data_model</span><span class="o">.</span><span class="n">pd</span><span class="o">.</span><span class="n">core</span><span class="o">.</span><span class="n">series</span><span class="o">.</span><span class="n">Series</span><span class="p">:</span>
             <span class="c1"># pandas.concat() poisons types with Series, don&#39;t allow that</span>
             <span class="k">return</span> <span class="nb">dict</span><span class="p">()</span>
         <span class="n">res</span><span class="p">[</span><span class="n">c</span><span class="p">]</span> <span class="o">=</span> <span class="n">gt</span>