Removed convert skip_false and --skip-false, closes #542

Author: simonw

docs/cli-reference.rst

@@ -653,7 +653,6 @@ See :ref:`cli_convert`.
       --output-type [integer|float|blob|text]
                                       Column type to use for the output column
       --drop                          Drop original column afterwards
-      --no-skip-false                 Don't skip falsey values
       -s, --silent                    Don't show a progress bar
       --pdb                           Open pdb debugger on first error
       -h, --help                      Show this message and exit.

docs/cli.rst

@@ -1711,8 +1711,6 @@ You can include named parameters in your where clause and populate them using on
 
 The ``--dry-run`` option will output a preview of the conversion against the first ten rows, without modifying the database.
 
-By default any rows with a falsey value for the column - such as ``0`` or ``null`` - will be skipped. Use the ``--no-skip-false`` option to disable this behaviour.
-
 .. _cli_convert_import:
 
 Importing additional modules

docs/python-api.rst

@@ -1009,8 +1009,6 @@ This will add the new column, if it does not already exist. You can pass ``outpu
 
 If you want to drop the original column after saving the results in a separate output column, pass ``drop=True``.
 
-By default any rows with a falsey value for the column - such as ``0`` or ``None`` - will be skipped. Pass ``skip_false=False`` to disable this behaviour.
-
 You can create multiple new columns from a single input column by passing ``multi=True`` and a conversion function that returns a Python dictionary. This example creates new ``upper`` and ``lower`` columns populated from the single ``title`` column:
 
 .. code-block:: python

sqlite_utils/cli.py

@@ -2952,7 +2952,6 @@ def _generate_convert_help():
     type=click.Choice(["integer", "float", "blob", "text"]),
 )
 @click.option("--drop", is_flag=True, help="Drop original column afterwards")
-@click.option("--no-skip-false", is_flag=True, help="Don't skip falsey values")
 @click.option("-s", "--silent", is_flag=True, help="Don't show a progress bar")
 @click.option("pdb_", "--pdb", is_flag=True, help="Open pdb debugger on first error")
 def convert(
@@ -2968,7 +2967,6 @@ def convert(
     output,
     output_type,
     drop,
-    no_skip_false,
     silent,
     pdb_,
 ):
@@ -3045,7 +3043,6 @@ def wrapped_fn(value):
                 output=output,
                 output_type=output_type,
                 drop=drop,
-                skip_false=not no_skip_false,
                 multi=multi,
                 show_progress=not silent,
             )

sqlite_utils/db.py

@@ -2909,7 +2909,6 @@ def convert(
         where: Optional[str] = None,
         where_args: Optional[Union[Iterable, dict]] = None,
         show_progress: bool = False,
-        skip_false: bool = True,
     ):
         """
         Apply conversion function ``fn`` to every value in the specified columns.
@@ -2952,8 +2951,6 @@ def convert(
 
             def convert_value(v):
                 bar.update(1)
-                if skip_false and not v:
-                    return v
                 return jsonify_if_needed(fn(v))
 
             fn_name = fn.__name__

sqlite_utils/recipes.py

@@ -14,6 +14,8 @@ def parsedate(value, dayfirst=False, yearfirst=False, errors=None):
     - errors=r.IGNORE to ignore values that cannot be parsed
     - errors=r.SET_NULL to set values that cannot be parsed to null
     """
+    if not value:
+        return value
     try:
         return (
             parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst)
@@ -38,6 +40,8 @@ def parsedatetime(value, dayfirst=False, yearfirst=False, errors=None):
     - errors=r.IGNORE to ignore values that cannot be parsed
     - errors=r.SET_NULL to set values that cannot be parsed to null
     """
+    if not value:
+        return value
     try:
         return parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).isoformat()
     except parser.ParserError:

tests/test_cli_convert.py

@@ -80,7 +80,7 @@ def test_convert_import(test_db_and_path):
             db_path,
             "example",
             "dt",
-            "return re.sub('O..', 'OXX', value)",
+            "return re.sub('O..', 'OXX', value) if value else value",
             "--import",
             "re",
         ],
@@ -223,7 +223,7 @@ def test_convert_output_column(test_db_and_path, drop):
         db_path,
         "example",
         "dt",
-        "value.replace('October', 'Spooktober')",
+        "value.replace('October', 'Spooktober') if value else value",
         "--output",
         "newcol",
     ]
@@ -628,14 +628,8 @@ def test_convert_initialization_pattern(fresh_db_and_path):
     ]
 
 
-@pytest.mark.parametrize(
-    "no_skip_false,expected",
-    (
-        (True, 1),
-        (False, 0),
-    ),
-)
-def test_convert_no_skip_false(fresh_db_and_path, no_skip_false, expected):
+def test_convert_handles_falsey_values(fresh_db_and_path):
+    # Falsey values like 0 should be converted (issue #527)
     db, db_path = fresh_db_and_path
     args = [
         "convert",
@@ -644,12 +638,10 @@ def test_convert_no_skip_false(fresh_db_and_path, no_skip_false, expected):
         "x",
         "-",
     ]
-    if no_skip_false:
-        args.append("--no-skip-false")
     db["t"].insert_all([{"x": 0}, {"x": 1}])
     assert db["t"].get(1)["x"] == 0
     assert db["t"].get(2)["x"] == 1
     result = CliRunner().invoke(cli.cli, args, input="value + 1")
     assert result.exit_code == 0, result.output
-    assert db["t"].get(1)["x"] == expected
+    assert db["t"].get(1)["x"] == 1
     assert db["t"].get(2)["x"] == 2

tests/test_convert.py

@@ -50,12 +50,13 @@ def test_convert_where(fresh_db, where, where_args):
     assert list(table.rows) == [{"id": 1, "title": "One"}, {"id": 2, "title": "TWO"}]
 
 
-def test_convert_skip_false(fresh_db):
+def test_convert_handles_falsey_values(fresh_db):
+    # Falsey values like 0 should be converted (issue #527)
     table = fresh_db["table"]
     table.insert_all([{"x": 0}, {"x": 1}])
     assert table.get(1)["x"] == 0
     assert table.get(2)["x"] == 1
-    table.convert("x", lambda x: x + 1, skip_false=False)
+    table.convert("x", lambda x: x + 1)
     assert table.get(1)["x"] == 1
     assert table.get(2)["x"] == 2