Add flat transaction & scheduled transaction views

README.md

@@ -54,9 +54,17 @@ full_refresh = False
 asyncio.run(sync(token, db, full_refresh))
 ```
 
-## SQL
+## Relations
 
-The schema is defined in [create-tables.sql](sqlite_export_for_ynab/ddl/create-tables.sql). It is very similar to [YNAB's OpenAPI Spec](https://api.ynab.com/papi/open_api_spec.yaml) however some objects are pulled out into their own tables (ex: subtransactions, loan account periodic values) and foreign keys are added as needed (ex: budget ID, transaction ID). You can query the DB with typical SQLite tools.
+The relations are defined in [create-relations.sql](sqlite_export_for_ynab/ddl/create-relations.sql). They are 1:1 with [YNAB's OpenAPI Spec](https://api.ynab.com/papi/open_api_spec.yaml) (ex: transactions, accounts, etc) with some additions:
+
+1. Some objects are pulled out into their own tables so they can be more cleanly modeled in SQLite (ex: subtransactions, loan account periodic values).
+1. Foreign keys are added as needed (ex: budget ID, transaction ID) so data across budgets remains separate.
+1. Two new views called `flat_transactions` and `scheduled_flat_transactions` allow you to query split and non-split transactions easily, without needing to also query `subtransactions` and `scheduled_subtransactions` respectively.
+
+## Querying
+
+You can issue queries with typical SQLite tools. *`sqlite-export-for-ynab` deliberately does not implement a SQL REPL.*
 
 ### Sample Queries
 
@@ -76,7 +84,7 @@ WITH
                     SUM(t.amount) ASC
             ) AS rnk
         FROM
-            transactions t
+            flat_transactions t
             JOIN payees p ON t.payee_id = p.id
             JOIN budgets b ON t.budget_id = b.id
         WHERE
@@ -105,8 +113,8 @@ To get payees with no transactions:
 
 ```sql
 SELECT DISTINCT
-    b.name,
-    p.name
+    b.name as budget,
+    p.name as payee
 FROM
     budgets b
     JOIN payees p ON b.id = p.budget_id
@@ -116,7 +124,7 @@ FROM
             payee_id,
             MAX(NOT deleted) AS has_active_transaction
         FROM
-            transactions
+            flat_transactions
         GROUP BY
             budget_id,
             payee_id
@@ -130,7 +138,7 @@ FROM
             payee_id,
             MAX(NOT deleted) AS has_active_transaction
         FROM
-            scheduled_transactions
+            scheduled_flat_transactions
         GROUP BY
             budget_id,
             payee_id

sqlite_export_for_ynab/_main.py

@@ -39,8 +39,9 @@
     | Literal["scheduled_transactions"]
     | Literal["scheduled_subtransactions"]
 )
-_ALL_TABLES = frozenset(
-    ("budgets",) + tuple(lit.__args__[0] for lit in _EntryTable.__args__)
+_ALL_RELATIONS = frozenset(
+    ("budgets", "flat_transactions", "scheduled_flat_transactions")
+    + tuple(lit.__args__[0] for lit in _EntryTable.__args__)
 )
 
 _ENV_TOKEN = "YNAB_PERSONAL_ACCESS_TOKEN"
@@ -105,15 +106,15 @@ async def sync(token: str, db: Path, full_refresh: bool) -> None:
         cur = con.cursor()
 
         if full_refresh:
-            print("Dropping tables...")
-            cur.executescript(contents("drop-tables.sql"))
+            print("Dropping relations...")
+            cur.executescript(contents("drop-relations.sql"))
             con.commit()
             print("Done")
 
-        tables = get_tables(cur)
-        if tables != _ALL_TABLES:
-            print("Recreating tables...")
-            cur.executescript(contents("create-tables.sql"))
+        relations = get_relations(cur)
+        if relations != _ALL_RELATIONS:
+            print("Recreating relations...")
+            cur.executescript(contents("create-relations.sql"))
             con.commit()
             print("Done")
 
@@ -180,11 +181,11 @@ def contents(filename: str) -> str:
     return (resources.files(ddl) / filename).read_text()
 
 
-def get_tables(cur: sqlite3.Cursor) -> set[str]:
+def get_relations(cur: sqlite3.Cursor) -> set[str]:
     return {
         t["name"]
         for t in cur.execute(
-            "SELECT name FROM sqlite_master WHERE type='table'"
+            "SELECT name FROM sqlite_master WHERE type='table' OR type='view'"
         ).fetchall()
     }
 

sqlite_export_for_ynab/ddl/create-tables.sql → sqlite_export_for_ynab/ddl/create-relations.sql

@@ -133,6 +133,52 @@ CREATE TABLE IF NOT EXISTS subtransactions (
     FOREIGN KEY (transaction_id) REFERENCES transaction_id (id)
 );
 
+CREATE VIEW IF NOT EXISTS flat_transactions AS
+SELECT
+    t.id AS transaction_id,
+    st.id AS subtransaction_id,
+    t.budget_id,
+    t.account_id,
+    t.account_name,
+    t.approved,
+    t.cleared,
+    t.date,
+    t.debt_transaction_type,
+    t.flag_color,
+    t.flag_name,
+    t.import_id,
+    t.import_payee_name,
+    t.import_payee_name_original,
+    t.matched_transaction_id,
+    COALESCE(st.id, t.id) AS id,
+    COALESCE(st.amount, t.amount) AS amount,
+    CASE
+        WHEN
+            COALESCE(st.transfer_account_id, t.transfer_account_id) IS null
+            THEN COALESCE(st.category_id, t.category_id)
+    END AS category_id,
+    CASE
+        WHEN
+            COALESCE(st.transfer_account_id, t.transfer_account_id) IS null
+            THEN COALESCE(st.category_name, t.category_name)
+    END AS category_name,
+    COALESCE(st.deleted, t.deleted) AS deleted,
+    COALESCE(st.memo, t.memo) AS memo,
+    COALESCE(st.payee_id, t.payee_id) AS payee_id,
+    COALESCE(st.payee_name, t.payee_name) AS payee_name,
+    COALESCE(st.transfer_account_id, t.transfer_account_id)
+        AS transfer_account_id,
+    COALESCE(
+        st.transfer_transaction_id,
+        t.transfer_transaction_id
+    ) AS transfer_transaction_id
+FROM
+    transactions AS t
+LEFT JOIN subtransactions AS st ON (
+    t.budget_id = st.budget_id
+    AND t.id = st.transaction_id
+);
+
 CREATE TABLE IF NOT EXISTS scheduled_transactions (
     id TEXT PRIMARY KEY,
     budget_id TEXT,
@@ -174,3 +220,51 @@ CREATE TABLE IF NOT EXISTS scheduled_subtransactions (
     FOREIGN KEY (payee_id) REFERENCES payees (id),
     FOREIGN KEY (scheduled_transaction_id) REFERENCES transaction_id (id)
 );
+
+CREATE VIEW IF NOT EXISTS scheduled_flat_transactions AS
+SELECT
+    t.id AS transaction_id,
+    st.id AS subtransaction_id,
+    t.budget_id,
+    t.account_id,
+    t.account_name,
+    t.date_first,
+    t.date_next,
+    t.flag_color,
+    t.flag_name,
+    p.name AS payee_name,
+    COALESCE(st.id, t.id) AS id,
+    COALESCE(st.amount, t.amount) AS amount,
+    CASE
+        WHEN
+            COALESCE(st.transfer_account_id, t.transfer_account_id) IS null
+            THEN COALESCE(st.category_id, t.category_id)
+    END AS category_id,
+    CASE
+        WHEN
+            COALESCE(st.transfer_account_id, t.transfer_account_id) IS null
+            THEN c.name
+    END AS category_name,
+    COALESCE(st.deleted, t.deleted) AS deleted,
+    COALESCE(st.memo, t.memo) AS memo,
+    COALESCE(st.payee_id, t.payee_id) AS payee_id,
+    COALESCE(st.transfer_account_id, t.transfer_account_id)
+        AS transfer_account_id
+FROM
+    scheduled_transactions AS t
+LEFT JOIN scheduled_subtransactions AS st
+    ON (
+        t.budget_id = st.budget_id
+        AND t.id = st.scheduled_transaction_id
+    )
+    -- work around missing category name from scheduled subtransaction response
+LEFT JOIN categories AS c
+    ON (
+        t.budget_id = c.budget_id
+        AND COALESCE(st.category_id, t.category_id) = c.id
+    )
+    -- work around missing payee name from scheduled subtransaction response
+LEFT JOIN payees AS p ON (
+    t.budget_id = p.budget_id
+    AND COALESCE(st.payee_id, t.payee_id) = p.name
+);

sqlite_export_for_ynab/ddl/drop-tables.sql → sqlite_export_for_ynab/ddl/drop-relations.sql

@@ -14,6 +14,10 @@ DROP TABLE IF EXISTS transactions;
 
 DROP TABLE IF EXISTS subtransactions;
 
+DROP VIEW IF EXISTS flat_transactions;
+
 DROP TABLE IF EXISTS scheduled_transactions;
 
 DROP TABLE IF EXISTS scheduled_subtransactions;
+
+DROP VIEW IF EXISTS scheduled_flat_transactions;

testing/fixtures.py

@@ -187,7 +187,7 @@
 def cur():
     with sqlite3.connect(":memory:") as con:
         cursor = con.cursor()
-        cursor.executescript(contents("create-tables.sql"))
+        cursor.executescript(contents("create-relations.sql"))
         cursor.row_factory = lambda c, row: dict(
             zip([name for name, *_ in c.description], row, strict=True)
         )

tests/_main_test.py

@@ -12,11 +12,11 @@
 from tqdm import tqdm
 
 from sqlite_export_for_ynab import default_db_path
-from sqlite_export_for_ynab._main import _ALL_TABLES
+from sqlite_export_for_ynab._main import _ALL_RELATIONS
 from sqlite_export_for_ynab._main import _ENV_TOKEN
 from sqlite_export_for_ynab._main import contents
 from sqlite_export_for_ynab._main import get_last_knowledge_of_server
-from sqlite_export_for_ynab._main import get_tables
+from sqlite_export_for_ynab._main import get_relations
 from sqlite_export_for_ynab._main import insert_accounts
 from sqlite_export_for_ynab._main import insert_budgets
 from sqlite_export_for_ynab._main import insert_category_groups
@@ -85,8 +85,8 @@ def test_default_db_path(monkeypatch, xdg_data_home, expected_prefix):
 
 
 @pytest.mark.usefixtures(cur.__name__)
-def test_get_tables(cur):
-    assert get_tables(cur) == _ALL_TABLES
+def test_get_relations(cur):
+    assert get_relations(cur) == _ALL_RELATIONS
 
 
 @pytest.mark.usefixtures(cur.__name__)
@@ -265,6 +265,33 @@ def test_insert_transactions(cur):
         },
     ]
 
+    cur.execute("SELECT * FROM flat_transactions ORDER BY amount")
+    assert [strip_nones(d) for d in cur.fetchall()] == [
+        {
+            "transaction_id": TRANSACTION_ID_2,
+            "budget_id": BUDGET_ID_1,
+            "date": "2024-02-01",
+            "id": TRANSACTION_ID_2,
+            "amount": -15000,
+        },
+        {
+            "transaction_id": TRANSACTION_ID_1,
+            "subtransaction_id": SUBTRANSACTION_ID_1,
+            "budget_id": BUDGET_ID_1,
+            "date": "2024-01-01",
+            "id": SUBTRANSACTION_ID_1,
+            "amount": -7000,
+        },
+        {
+            "transaction_id": TRANSACTION_ID_1,
+            "subtransaction_id": SUBTRANSACTION_ID_2,
+            "budget_id": BUDGET_ID_1,
+            "date": "2024-01-01",
+            "id": SUBTRANSACTION_ID_2,
+            "amount": -3000,
+        },
+    ]
+
 
 @pytest.mark.usefixtures(cur.__name__)
 def test_insert_scheduled_transactions(cur):
@@ -303,6 +330,30 @@ def test_insert_scheduled_transactions(cur):
         },
     ]
 
+    cur.execute("SELECT * FROM scheduled_flat_transactions ORDER BY amount")
+    assert [strip_nones(d) for d in cur.fetchall()] == [
+        {
+            "transaction_id": SCHEDULED_TRANSACTION_ID_2,
+            "budget_id": BUDGET_ID_1,
+            "id": SCHEDULED_TRANSACTION_ID_2,
+            "amount": -11000,
+        },
+        {
+            "transaction_id": SCHEDULED_TRANSACTION_ID_1,
+            "subtransaction_id": SCHEDULED_SUBTRANSACTION_ID_1,
+            "budget_id": BUDGET_ID_1,
+            "id": SCHEDULED_SUBTRANSACTION_ID_1,
+            "amount": -8000,
+        },
+        {
+            "transaction_id": SCHEDULED_TRANSACTION_ID_1,
+            "subtransaction_id": SCHEDULED_SUBTRANSACTION_ID_2,
+            "budget_id": BUDGET_ID_1,
+            "id": SCHEDULED_SUBTRANSACTION_ID_2,
+            "amount": -4000,
+        },
+    ]
+
 
 @pytest.mark.asyncio
 @pytest.mark.usefixtures(mock_aioresponses.__name__)
@@ -387,7 +438,7 @@ async def test_sync_no_data(tmp_path, mock_aioresponses):
     # create the db and tables to exercise all code branches
     db = tmp_path / "db.sqlite"
     with sqlite3.connect(db) as con:
-        con.executescript(contents("create-tables.sql"))
+        con.executescript(contents("create-relations.sql"))
 
     await sync(TOKEN, db, False)