sqlite-utils convert --pdb option, closes #581

Author: simonw

docs/cli-reference.rst

@@ -648,6 +648,7 @@ See :ref:`cli_convert`.
       --drop                          Drop original column afterwards
       --no-skip-false                 Don't skip falsey values
       -s, --silent                    Don't show a progress bar
+      --pdb                           Open pdb debugger on first error
       -h, --help                      Show this message and exit.
 
 

docs/cli.rst

@@ -1732,6 +1732,44 @@ This supports nested imports as well, for example to use `ElementTree <https://d
         'xml.etree.ElementTree.fromstring(value).attrib["title"]' \
         --import=xml.etree.ElementTree
 
+.. _cli_convert_debugger:
+
+Using the debugger
+------------------
+
+If an error occurs while running your conversion operation you may see a message like this::
+
+    user-defined function raised exception
+
+Add the ``--pdb`` option to catch the error and open the Python debugger at that point. The conversion operation will exit after you type ``q`` in the debugger.
+
+Here's an example debugging session. First, create a ``articles`` table with invalid XML in the ``content`` column:
+
+.. code-block:: bash
+
+    echo '{"content": "This is not XML"}' | sqlite-utils insert content.db articles -
+
+Now run the conversion with the ``--pdb`` option:
+
+.. code-block:: bash
+
+    sqlite-utils convert content.db articles content \
+        'xml.etree.ElementTree.fromstring(value).attrib["title"]' \
+        --import=xml.etree.ElementTree \
+        --pdb
+
+When the error occurs the debugger will open::
+
+    Exception raised, dropping into pdb...: syntax error: line 1, column 0
+    > .../python3.11/xml/etree/ElementTree.py(1338)XML()
+    -> parser.feed(text)
+    (Pdb) args
+    text = 'This is not XML'
+    parser = <xml.etree.ElementTree.XMLParser object at 0x102c405e0>
+    (Pdb) q
+
+``args`` here shows the arguments to the current function in the stack. The Python `pdb documentation <https://docs.python.org/3/library/pdb.html#debugger-commands>`__ has full details on the other available commands.
+
 .. _cli_convert_complex:
 
 Defining a convert() function

sqlite_utils/cli.py

@@ -16,6 +16,7 @@
 import itertools
 import json
 import os
+import pdb
 import sys
 import csv as csv_std
 import tabulate
@@ -2901,6 +2902,7 @@ def _generate_convert_help():
 @click.option("--drop", is_flag=True, help="Drop original column afterwards")
 @click.option("--no-skip-false", is_flag=True, help="Don't skip falsey values")
 @click.option("-s", "--silent", is_flag=True, help="Don't show a progress bar")
+@click.option("pdb_", "--pdb", is_flag=True, help="Open pdb debugger on first error")
 def convert(
     db_path,
     table,
@@ -2916,6 +2918,7 @@ def convert(
     drop,
     no_skip_false,
     silent,
+    pdb_,
 ):
     sqlite3.enable_callback_tracebacks(True)
     db = sqlite_utils.Database(db_path)
@@ -2968,6 +2971,19 @@ def preview(v):
         )
         click.echo("Would affect {} row{}".format(count, "" if count == 1 else "s"))
     else:
+        # Wrap fn with a thing that will catch errors and optionally drop into pdb
+        if pdb_:
+            fn_ = fn
+
+            def wrapped_fn(value):
+                try:
+                    return fn_(value)
+                except Exception as ex:
+                    print("\nException raised, dropping into pdb...:", ex)
+                    pdb.post_mortem(ex.__traceback__)
+                    sys.exit(1)
+
+            fn = wrapped_fn
         try:
             db[table].convert(
                 columns,