sqlite-utils memory --flatten, closes #332

Author: simonw

docs/cli.rst

@@ -758,7 +758,7 @@ This also means you pipe ``sqlite-utils`` together to easily create a new SQLite
 Flattening nested JSON objects
 ------------------------------
 
-``sqlite-utils insert`` expects incoming data to consist of an array of JSON objects, where the top-level keys of each object will become columns in the created database table.
+``sqlite-utils insert`` and ``sqlite-utils memory`` both expect incoming JSON data to consist of an array of JSON objects, where the top-level keys of each object will become columns in the created database table.
 
 If your data is nested you can use the ``--flatten`` option to create columns that are derived from the nested data.
 

sqlite_utils/cli.py

@@ -1190,6 +1190,7 @@ def query(
     multiple=True,
     help="Additional databases to attach - specify alias and filepath",
 )
+@click.option("--flatten", is_flag=True, help="Flatten nested JSON objects")
 @output_options
 @click.option("-r", "--raw", is_flag=True, help="Raw output, first column of first row")
 @click.option(
@@ -1226,6 +1227,7 @@ def memory(
     paths,
     sql,
     attach,
+    flatten,
     nl,
     arrays,
     csv,
@@ -1300,6 +1302,8 @@ def memory(
         if format_used in (Format.CSV, Format.TSV) and not no_detect_types:
             tracker = TypeTracker()
             rows = tracker.wrap(rows)
+        if flatten:
+            rows = (dict(_flatten(row)) for row in rows)
         db[csv_table].insert_all(rows, alter=True)
         if tracker is not None:
             db[csv_table].transform(types=tracker.types)

tests/test_cli_memory.py

@@ -222,6 +222,30 @@ def test_memory_no_detect_types(option):
     ]
 
 
+def test_memory_flatten():
+    result = CliRunner().invoke(
+        cli.cli,
+        ["memory", "-", "select * from stdin", "--flatten"],
+        input=json.dumps(
+            {
+                "httpRequest": {
+                    "latency": "0.112114537s",
+                    "requestMethod": "GET",
+                },
+                "insertId": "6111722f000b5b4c4d4071e2",
+            }
+        ),
+    )
+    assert result.exit_code == 0, result.output
+    assert json.loads(result.output.strip()) == [
+        {
+            "httpRequest_latency": "0.112114537s",
+            "httpRequest_requestMethod": "GET",
+            "insertId": "6111722f000b5b4c4d4071e2",
+        }
+    ]
+
+
 def test_memory_analyze():
     result = CliRunner().invoke(
         cli.cli,