diff --git a/README.md b/README.md index b301d62..efc89ab 100644 --- a/README.md +++ b/README.md @@ -54,9 +54,17 @@ full_refresh = False asyncio.run(sync(token, db, full_refresh)) ``` -## SQL +## Relations -The schema is defined in [create-tables.sql](sqlite_export_for_ynab/ddl/create-tables.sql). It is very similar to [YNAB's OpenAPI Spec](https://api.ynab.com/papi/open_api_spec.yaml) however some objects are pulled out into their own tables (ex: subtransactions, loan account periodic values) and foreign keys are added as needed (ex: budget ID, transaction ID). You can query the DB with typical SQLite tools. +The relations are defined in [create-relations.sql](sqlite_export_for_ynab/ddl/create-relations.sql). They are 1:1 with [YNAB's OpenAPI Spec](https://api.ynab.com/papi/open_api_spec.yaml) (ex: transactions, accounts, etc) with some additions: + +1. Some objects are pulled out into their own tables so they can be more cleanly modeled in SQLite (ex: subtransactions, loan account periodic values). +1. Foreign keys are added as needed (ex: budget ID, transaction ID) so data across budgets remains separate. +1. Two new views called `flat_transactions` and `scheduled_flat_transactions` allow you to query split and non-split transactions easily, without needing to also query `subtransactions` and `scheduled_subtransactions` respectively. + +## Querying + +You can issue queries with typical SQLite tools. *`sqlite-export-for-ynab` deliberately does not implement a SQL REPL.* ### Sample Queries @@ -76,7 +84,7 @@ WITH SUM(t.amount) ASC ) AS rnk FROM - transactions t + flat_transactions t JOIN payees p ON t.payee_id = p.id JOIN budgets b ON t.budget_id = b.id WHERE @@ -105,8 +113,8 @@ To get payees with no transactions: ```sql SELECT DISTINCT - b.name, - p.name + b.name as budget, + p.name as payee FROM budgets b JOIN payees p ON b.id = p.budget_id @@ -116,7 +124,7 @@ FROM payee_id, MAX(NOT deleted) AS has_active_transaction FROM - transactions + flat_transactions GROUP BY budget_id, payee_id @@ -130,7 +138,7 @@ FROM payee_id, MAX(NOT deleted) AS has_active_transaction FROM - scheduled_transactions + scheduled_flat_transactions GROUP BY budget_id, payee_id diff --git a/sqlite_export_for_ynab/_main.py b/sqlite_export_for_ynab/_main.py index 71decaa..0640203 100644 --- a/sqlite_export_for_ynab/_main.py +++ b/sqlite_export_for_ynab/_main.py @@ -39,8 +39,9 @@ | Literal["scheduled_transactions"] | Literal["scheduled_subtransactions"] ) -_ALL_TABLES = frozenset( - ("budgets",) + tuple(lit.__args__[0] for lit in _EntryTable.__args__) +_ALL_RELATIONS = frozenset( + ("budgets", "flat_transactions", "scheduled_flat_transactions") + + tuple(lit.__args__[0] for lit in _EntryTable.__args__) ) _ENV_TOKEN = "YNAB_PERSONAL_ACCESS_TOKEN" @@ -105,15 +106,15 @@ async def sync(token: str, db: Path, full_refresh: bool) -> None: cur = con.cursor() if full_refresh: - print("Dropping tables...") - cur.executescript(contents("drop-tables.sql")) + print("Dropping relations...") + cur.executescript(contents("drop-relations.sql")) con.commit() print("Done") - tables = get_tables(cur) - if tables != _ALL_TABLES: - print("Recreating tables...") - cur.executescript(contents("create-tables.sql")) + relations = get_relations(cur) + if relations != _ALL_RELATIONS: + print("Recreating relations...") + cur.executescript(contents("create-relations.sql")) con.commit() print("Done") @@ -180,11 +181,11 @@ def contents(filename: str) -> str: return (resources.files(ddl) / filename).read_text() -def get_tables(cur: sqlite3.Cursor) -> set[str]: +def get_relations(cur: sqlite3.Cursor) -> set[str]: return { t["name"] for t in cur.execute( - "SELECT name FROM sqlite_master WHERE type='table'" + "SELECT name FROM sqlite_master WHERE type='table' OR type='view'" ).fetchall() } diff --git a/sqlite_export_for_ynab/ddl/create-tables.sql b/sqlite_export_for_ynab/ddl/create-relations.sql similarity index 62% rename from sqlite_export_for_ynab/ddl/create-tables.sql rename to sqlite_export_for_ynab/ddl/create-relations.sql index be87f4b..d559dd7 100644 --- a/sqlite_export_for_ynab/ddl/create-tables.sql +++ b/sqlite_export_for_ynab/ddl/create-relations.sql @@ -133,6 +133,52 @@ CREATE TABLE IF NOT EXISTS subtransactions ( FOREIGN KEY (transaction_id) REFERENCES transaction_id (id) ); +CREATE VIEW IF NOT EXISTS flat_transactions AS +SELECT + t.id AS transaction_id, + st.id AS subtransaction_id, + t.budget_id, + t.account_id, + t.account_name, + t.approved, + t.cleared, + t.date, + t.debt_transaction_type, + t.flag_color, + t.flag_name, + t.import_id, + t.import_payee_name, + t.import_payee_name_original, + t.matched_transaction_id, + COALESCE(st.id, t.id) AS id, + COALESCE(st.amount, t.amount) AS amount, + CASE + WHEN + COALESCE(st.transfer_account_id, t.transfer_account_id) IS null + THEN COALESCE(st.category_id, t.category_id) + END AS category_id, + CASE + WHEN + COALESCE(st.transfer_account_id, t.transfer_account_id) IS null + THEN COALESCE(st.category_name, t.category_name) + END AS category_name, + COALESCE(st.deleted, t.deleted) AS deleted, + COALESCE(st.memo, t.memo) AS memo, + COALESCE(st.payee_id, t.payee_id) AS payee_id, + COALESCE(st.payee_name, t.payee_name) AS payee_name, + COALESCE(st.transfer_account_id, t.transfer_account_id) + AS transfer_account_id, + COALESCE( + st.transfer_transaction_id, + t.transfer_transaction_id + ) AS transfer_transaction_id +FROM + transactions AS t +LEFT JOIN subtransactions AS st ON ( + t.budget_id = st.budget_id + AND t.id = st.transaction_id +); + CREATE TABLE IF NOT EXISTS scheduled_transactions ( id TEXT PRIMARY KEY, budget_id TEXT, @@ -174,3 +220,51 @@ CREATE TABLE IF NOT EXISTS scheduled_subtransactions ( FOREIGN KEY (payee_id) REFERENCES payees (id), FOREIGN KEY (scheduled_transaction_id) REFERENCES transaction_id (id) ); + +CREATE VIEW IF NOT EXISTS scheduled_flat_transactions AS +SELECT + t.id AS transaction_id, + st.id AS subtransaction_id, + t.budget_id, + t.account_id, + t.account_name, + t.date_first, + t.date_next, + t.flag_color, + t.flag_name, + p.name AS payee_name, + COALESCE(st.id, t.id) AS id, + COALESCE(st.amount, t.amount) AS amount, + CASE + WHEN + COALESCE(st.transfer_account_id, t.transfer_account_id) IS null + THEN COALESCE(st.category_id, t.category_id) + END AS category_id, + CASE + WHEN + COALESCE(st.transfer_account_id, t.transfer_account_id) IS null + THEN c.name + END AS category_name, + COALESCE(st.deleted, t.deleted) AS deleted, + COALESCE(st.memo, t.memo) AS memo, + COALESCE(st.payee_id, t.payee_id) AS payee_id, + COALESCE(st.transfer_account_id, t.transfer_account_id) + AS transfer_account_id +FROM + scheduled_transactions AS t +LEFT JOIN scheduled_subtransactions AS st + ON ( + t.budget_id = st.budget_id + AND t.id = st.scheduled_transaction_id + ) + -- work around missing category name from scheduled subtransaction response +LEFT JOIN categories AS c + ON ( + t.budget_id = c.budget_id + AND COALESCE(st.category_id, t.category_id) = c.id + ) + -- work around missing payee name from scheduled subtransaction response +LEFT JOIN payees AS p ON ( + t.budget_id = p.budget_id + AND COALESCE(st.payee_id, t.payee_id) = p.name +); diff --git a/sqlite_export_for_ynab/ddl/drop-tables.sql b/sqlite_export_for_ynab/ddl/drop-relations.sql similarity index 80% rename from sqlite_export_for_ynab/ddl/drop-tables.sql rename to sqlite_export_for_ynab/ddl/drop-relations.sql index cebecacc..3b6d657 100644 --- a/sqlite_export_for_ynab/ddl/drop-tables.sql +++ b/sqlite_export_for_ynab/ddl/drop-relations.sql @@ -14,6 +14,10 @@ DROP TABLE IF EXISTS transactions; DROP TABLE IF EXISTS subtransactions; +DROP VIEW IF EXISTS flat_transactions; + DROP TABLE IF EXISTS scheduled_transactions; DROP TABLE IF EXISTS scheduled_subtransactions; + +DROP VIEW IF EXISTS scheduled_flat_transactions; diff --git a/testing/fixtures.py b/testing/fixtures.py index f0dc501..a6ca02b 100644 --- a/testing/fixtures.py +++ b/testing/fixtures.py @@ -187,7 +187,7 @@ def cur(): with sqlite3.connect(":memory:") as con: cursor = con.cursor() - cursor.executescript(contents("create-tables.sql")) + cursor.executescript(contents("create-relations.sql")) cursor.row_factory = lambda c, row: dict( zip([name for name, *_ in c.description], row, strict=True) ) diff --git a/tests/_main_test.py b/tests/_main_test.py index 7fbbcbe..a0c1f9f 100644 --- a/tests/_main_test.py +++ b/tests/_main_test.py @@ -12,11 +12,11 @@ from tqdm import tqdm from sqlite_export_for_ynab import default_db_path -from sqlite_export_for_ynab._main import _ALL_TABLES +from sqlite_export_for_ynab._main import _ALL_RELATIONS from sqlite_export_for_ynab._main import _ENV_TOKEN from sqlite_export_for_ynab._main import contents from sqlite_export_for_ynab._main import get_last_knowledge_of_server -from sqlite_export_for_ynab._main import get_tables +from sqlite_export_for_ynab._main import get_relations from sqlite_export_for_ynab._main import insert_accounts from sqlite_export_for_ynab._main import insert_budgets from sqlite_export_for_ynab._main import insert_category_groups @@ -85,8 +85,8 @@ def test_default_db_path(monkeypatch, xdg_data_home, expected_prefix): @pytest.mark.usefixtures(cur.__name__) -def test_get_tables(cur): - assert get_tables(cur) == _ALL_TABLES +def test_get_relations(cur): + assert get_relations(cur) == _ALL_RELATIONS @pytest.mark.usefixtures(cur.__name__) @@ -265,6 +265,33 @@ def test_insert_transactions(cur): }, ] + cur.execute("SELECT * FROM flat_transactions ORDER BY amount") + assert [strip_nones(d) for d in cur.fetchall()] == [ + { + "transaction_id": TRANSACTION_ID_2, + "budget_id": BUDGET_ID_1, + "date": "2024-02-01", + "id": TRANSACTION_ID_2, + "amount": -15000, + }, + { + "transaction_id": TRANSACTION_ID_1, + "subtransaction_id": SUBTRANSACTION_ID_1, + "budget_id": BUDGET_ID_1, + "date": "2024-01-01", + "id": SUBTRANSACTION_ID_1, + "amount": -7000, + }, + { + "transaction_id": TRANSACTION_ID_1, + "subtransaction_id": SUBTRANSACTION_ID_2, + "budget_id": BUDGET_ID_1, + "date": "2024-01-01", + "id": SUBTRANSACTION_ID_2, + "amount": -3000, + }, + ] + @pytest.mark.usefixtures(cur.__name__) def test_insert_scheduled_transactions(cur): @@ -303,6 +330,30 @@ def test_insert_scheduled_transactions(cur): }, ] + cur.execute("SELECT * FROM scheduled_flat_transactions ORDER BY amount") + assert [strip_nones(d) for d in cur.fetchall()] == [ + { + "transaction_id": SCHEDULED_TRANSACTION_ID_2, + "budget_id": BUDGET_ID_1, + "id": SCHEDULED_TRANSACTION_ID_2, + "amount": -11000, + }, + { + "transaction_id": SCHEDULED_TRANSACTION_ID_1, + "subtransaction_id": SCHEDULED_SUBTRANSACTION_ID_1, + "budget_id": BUDGET_ID_1, + "id": SCHEDULED_SUBTRANSACTION_ID_1, + "amount": -8000, + }, + { + "transaction_id": SCHEDULED_TRANSACTION_ID_1, + "subtransaction_id": SCHEDULED_SUBTRANSACTION_ID_2, + "budget_id": BUDGET_ID_1, + "id": SCHEDULED_SUBTRANSACTION_ID_2, + "amount": -4000, + }, + ] + @pytest.mark.asyncio @pytest.mark.usefixtures(mock_aioresponses.__name__) @@ -387,7 +438,7 @@ async def test_sync_no_data(tmp_path, mock_aioresponses): # create the db and tables to exercise all code branches db = tmp_path / "db.sqlite" with sqlite3.connect(db) as con: - con.executescript(contents("create-tables.sql")) + con.executescript(contents("create-relations.sql")) await sync(TOKEN, db, False)