add null/bad tests

Author: JohnMount

build/lib/data_algebra/SQLite.py

@@ -1,11 +1,32 @@
+import math
 import pandas
+import numpy
+import numbers
 
 import data_algebra.util
 import data_algebra.data_ops
 import data_algebra.db_model
 
+
 # map from op-name to special SQL formatting code
-SQLite_formatters = {"___": lambda dbmodel, expression: expression.to_python()}
+
+def _sqlite_is_bad_expr(dbmodel, expression):
+    return "is_bad(" + dbmodel.expr_to_sql(expression.args[0], want_inline_parens=False) + ")"
+
+
+SQLite_formatters = {
+    "is_bad": _sqlite_is_bad_expr,
+}
+
+
+def _check_scalar_bad(x):
+    if x is None:
+        return 1
+    if not isinstance(x, numbers.Number):
+        return 0
+    if numpy.isinf(x) or numpy.isnan(x):
+        return 1
+    return 0
 
 
 class SQLiteModel(data_algebra.db_model.DBModel):
@@ -19,6 +40,11 @@ def __init__(self):
             sql_formatters=SQLite_formatters,
         )
 
+    def prepare_connection(self, conn):
+        # https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.create_function
+        conn.create_function("exp", 1, math.exp)
+        conn.create_function("is_bad", 1, _check_scalar_bad)
+
     def quote_identifier(self, identifier):
         if not isinstance(identifier, str):
             raise Exception("expected identifier to be a str")

build/lib/data_algebra/db_model.py

@@ -10,6 +10,24 @@
 import data_algebra.cdata
 
 
+# map from op-name to special SQL formatting code
+
+def _db_is_null_expr(dbmodel, expression):
+    return "(" + dbmodel.expr_to_sql(expression.args[0], want_inline_parens=False) + " IS NULL)"
+
+
+def _db_is_bad_expr(dbmodel, expression):
+    subexpr = dbmodel.expr_to_sql(expression.args[0], want_inline_parens=True)
+    return "(" + subexpr + " IS NULL OR " + subexpr + " >= Infinity OR " + subexpr + " <= -Infinity " +\
+           " OR (" + subexpr + " != 0 AND " + subexpr + " == -" + subexpr + "))"
+
+
+db_expr_formatters = {
+    "is_null": _db_is_null_expr,
+    "is_bad": _db_is_bad_expr,
+}
+
+
 class DBModel:
     """A model of how SQL should be generated for a given database.
        """
@@ -31,11 +49,17 @@ def __init__(
         self.string_quote = string_quote
         if sql_formatters is None:
             sql_formatters = {}
-        self.sql_formatters = sql_formatters
+        self.sql_formatters = sql_formatters.copy()
+        for k in db_expr_formatters.keys():
+            if k not in self.sql_formatters.keys():
+                self.sql_formatters[k] = db_expr_formatters[k]
         if op_replacements is None:
             op_replacements = {"==": "="}
         self.op_replacements = op_replacements
 
+    def prepare_connection(self, conn):
+        pass
+
     def quote_identifier(self, identifier):
         if not isinstance(identifier, str):
             raise Exception("expected identifier to be a str")

coverage.txt

@@ -20,21 +20,21 @@ tests/test_sqlite.py .                                                   [100%]
 Name                           Stmts   Miss  Cover
 --------------------------------------------------
 data_algebra/PostgreSQL.py        19      4    79%
-data_algebra/SQLite.py            28      5    82%
+data_algebra/SQLite.py            44      6    86%
 data_algebra/__init__.py           5      0   100%
 data_algebra/cdata.py             49      5    90%
 data_algebra/cdata_impl.py        94     16    83%
 data_algebra/data_ops.py         692    124    82%
 data_algebra/data_pipe.py        158     33    79%
-data_algebra/db_model.py         339     84    75%
+data_algebra/db_model.py         350     86    75%
 data_algebra/env.py               54     12    78%
 data_algebra/expr_rep.py         294     89    70%
 data_algebra/pending_eval.py      34     34     0%
 data_algebra/pipe.py              65     19    71%
 data_algebra/util.py              69      6    91%
 data_algebra/yaml.py              73      9    88%
 --------------------------------------------------
-TOTAL                           1973    440    78%
+TOTAL                           2000    443    78%
 
 
-========================== 14 passed in 2.22 seconds ===========================
+========================== 14 passed in 2.42 seconds ===========================

data_algebra/SQLite.py

@@ -1,15 +1,34 @@
+import math
 import pandas
+import numpy
+import numbers
 
 import data_algebra.util
 import data_algebra.data_ops
 import data_algebra.db_model
 
 
 # map from op-name to special SQL formatting code
+
+def _sqlite_is_bad_expr(dbmodel, expression):
+    return "is_bad(" + dbmodel.expr_to_sql(expression.args[0], want_inline_parens=False) + ")"
+
+
 SQLite_formatters = {
+    "is_bad": _sqlite_is_bad_expr,
 }
 
 
+def _check_scalar_bad(x):
+    if x is None:
+        return 1
+    if not isinstance(x, numbers.Number):
+        return 0
+    if numpy.isinf(x) or numpy.isnan(x):
+        return 1
+    return 0
+
+
 class SQLiteModel(data_algebra.db_model.DBModel):
     """A model of how SQL should be generated for SQLite"""
 
@@ -21,6 +40,11 @@ def __init__(self):
             sql_formatters=SQLite_formatters,
         )
 
+    def prepare_connection(self, conn):
+        # https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.create_function
+        conn.create_function("exp", 1, math.exp)
+        conn.create_function("is_bad", 1, _check_scalar_bad)
+
     def quote_identifier(self, identifier):
         if not isinstance(identifier, str):
             raise Exception("expected identifier to be a str")

data_algebra/db_model.py

@@ -57,6 +57,9 @@ def __init__(
             op_replacements = {"==": "="}
         self.op_replacements = op_replacements
 
+    def prepare_connection(self, conn):
+        pass
+
     def quote_identifier(self, identifier):
         if not isinstance(identifier, str):
             raise Exception("expected identifier to be a str")

dist/data_algebra-0.1.4-py3-none-any.whl

@@ -1,25 +1,26 @@
 
 import math
 import numpy
+import sqlite3
 
 import data_algebra.util
 from data_algebra.data_ops import *
 import data_algebra.SQLite
 
 def test_null_bad():
-    ops =  TableDescription("d", ["x"]).extend({
+    ops = TableDescription("d", ["x"]).extend({
         "x_is_null": "x.is_null()",
         "x_is_bad": "x.is_bad()"
     })
 
     d = pandas.DataFrame({
-        'x': [1, numpy.nan, math.inf, -math.inf, None, 2]
+        'x': [1, numpy.nan, math.inf, -math.inf, None, 0]
     })
 
     d2 = ops.transform(d)
 
     expect = pandas.DataFrame({
-        'x': [1, numpy.nan, math.inf, -math.inf, None, 2],
+        'x': [1, numpy.nan, math.inf, -math.inf, None, 0],
         'x_is_null': [False, True, False, False, True, False],
         'x_is_bad': [False, True, True, True, True, False]
     })
@@ -31,3 +32,21 @@ def test_null_bad():
 
     sql = ops.to_sql(db_model, pretty=True)
     assert isinstance(sql, str)
+
+    conn = sqlite3.connect(":memory:")
+    db_model.prepare_connection(conn)
+
+    db_model.insert_table(conn, d, 'd')
+
+    res = db_model.read_query(conn, sql)
+
+    conn.close()
+
+    expectr = pandas.DataFrame({
+        'x': [1, numpy.nan, math.inf, -math.inf, None, 0],
+        'x_is_null': [0, 1, 0, 0, 1, 0],
+        'x_is_bad': [0, 1, 1, 1, 1, 0]
+    })
+
+    assert all(res['x_is_null'] == expectr['x_is_null'])
+    assert all(res['x_is_bad'] == expectr['x_is_bad'])

dist/data_algebra-0.1.4.tar.gz

@@ -96,8 +96,7 @@ def test_scoring_example():
     sql_s = ops.to_sql(db_model_p, pretty=True)
 
     conn = sqlite3.connect(":memory:")
-    # https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.create_function
-    conn.create_function("exp", 1, math.exp)
+    db_model_s.prepare_connection(conn)
 
     db_model_s.insert_table(conn, d_local, "d")
     back = db_model_s.read_table(conn, "d")