Skip to content

Commit f7aa910

Browse files
darynaishchenkodarynaishchenko
committed
updated unit test
1 parent 11039b9 commit f7aa910

File tree

1 file changed

+158
-9
lines changed

1 file changed

+158
-9
lines changed

‎unit_tests/sources/declarative/incremental/test_concurrent_perpartitioncursor.py‎

Lines changed: 158 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1461,7 +1461,7 @@ def run_incremental_parent_state_test(
14611461
},
14621462
),
14631463
(
1464-
"test_incremental_parent_state_records_without_cursor",
1464+
"test_incremental_parent_state_one_record_without_cursor",
14651465
SUBSTREAM_MANIFEST,
14661466
[
14671467
# Fetch the first page of posts
@@ -1499,14 +1499,7 @@ def run_incremental_parent_state_test(
14991499
# Fetch the first page of votes for comment 10 of post 1 (vote without cursor field)
15001500
(
15011501
f"https://api.example.com/community/posts/1/comments/10/votes?per_page=100&start_time={INITIAL_STATE_PARTITION_10_CURSOR}",
1502-
{
1503-
"votes": [
1504-
{
1505-
"id": 100,
1506-
"comment_id": 10,
1507-
}
1508-
],
1509-
},
1502+
{"votes": [{"id": 100, "comment_id": 10,}],},
15101503
),
15111504
# Fetch the first page of votes for comment 11 of post 1
15121505
(
@@ -1628,6 +1621,162 @@ def run_incremental_parent_state_test(
16281621
],
16291622
},
16301623
),
1624+
(
1625+
"test_incremental_parent_state_all_records_without_cursor",
1626+
SUBSTREAM_MANIFEST,
1627+
[
1628+
# Fetch the first page of posts
1629+
(
1630+
f"https://api.example.com/community/posts?per_page=100&start_time={PARENT_POSTS_CURSOR}",
1631+
{
1632+
"posts": [
1633+
{"id": 1, "updated_at": POST_1_UPDATED_AT},
1634+
]
1635+
},
1636+
),
1637+
# Fetch the first page of comments for post 1
1638+
(
1639+
"https://api.example.com/community/posts/1/comments?per_page=100",
1640+
{
1641+
"comments": [
1642+
{
1643+
"id": 9,
1644+
"post_id": 1,
1645+
"updated_at": COMMENT_9_OLDEST,
1646+
},
1647+
{
1648+
"id": 10,
1649+
"post_id": 1,
1650+
"updated_at": COMMENT_10_UPDATED_AT,
1651+
},
1652+
{
1653+
"id": 11,
1654+
"post_id": 1,
1655+
"updated_at": COMMENT_11_UPDATED_AT,
1656+
},
1657+
]
1658+
},
1659+
),
1660+
# Fetch the first page of votes for comment 10 of post 1 (vote without cursor field)
1661+
(
1662+
f"https://api.example.com/community/posts/1/comments/10/votes?per_page=100&start_time={INITIAL_STATE_PARTITION_10_CURSOR}",
1663+
{"votes": [{"id": 100, "comment_id": 10,}],},
1664+
),
1665+
# Fetch the first page of votes for comment 11 of post 1 (vote without cursor field)
1666+
(
1667+
f"https://api.example.com/community/posts/1/comments/11/votes"
1668+
f"?per_page=100&start_time={INITIAL_STATE_PARTITION_11_CURSOR}",
1669+
{"votes": [{"id": 111, "comment_id": 11}]},
1670+
),
1671+
# Fetch the first page of votes for comment 12 of post 1
1672+
(
1673+
f"https://api.example.com/community/posts/1/comments/12/votes?per_page=100&start_time={LOOKBACK_DATE}",
1674+
{"votes": []},
1675+
),
1676+
# Fetch the first page of comments for post 2
1677+
(
1678+
"https://api.example.com/community/posts/2/comments?per_page=100",
1679+
{
1680+
"comments": [{"id": 20, "post_id": 2, "updated_at": COMMENT_20_UPDATED_AT}],
1681+
"next_page": "https://api.example.com/community/posts/2/comments?per_page=100&page=2",
1682+
},
1683+
),
1684+
# Requests with intermediate states
1685+
# Fetch votes for comment 10 of post 1
1686+
(
1687+
f"https://api.example.com/community/posts/1/comments/10/votes?per_page=100&start_time={VOTE_100_CREATED_AT}",
1688+
{
1689+
"votes": [{"id": 100, "comment_id": 10, "created_at": VOTE_100_CREATED_AT}],
1690+
},
1691+
),
1692+
# Fetch votes for comment 11 of post 1
1693+
(
1694+
f"https://api.example.com/community/posts/1/comments/11/votes?per_page=100&start_time={VOTE_111_CREATED_AT}",
1695+
{
1696+
"votes": [{"id": 111, "comment_id": 11, "created_at": VOTE_111_CREATED_AT}],
1697+
},
1698+
),
1699+
# Fetch votes for comment 12 of post 1
1700+
(
1701+
f"https://api.example.com/community/posts/1/comments/12/votes?per_page=100&start_time={VOTE_111_CREATED_AT}",
1702+
{
1703+
"votes": [],
1704+
},
1705+
),
1706+
],
1707+
# Expected records
1708+
[
1709+
{"comment_id": 10, "comment_updated_at": COMMENT_10_UPDATED_AT, "id": 100},
1710+
{"comment_id": 11, "comment_updated_at": COMMENT_11_UPDATED_AT, "id": 111,},
1711+
],
1712+
# Number of intermediate states - 6 as number of parent partitions
1713+
2,
1714+
# Initial state
1715+
{
1716+
"parent_state": {
1717+
"post_comments": {
1718+
"states": [
1719+
{
1720+
"partition": {"id": 1, "parent_slice": {}},
1721+
"cursor": {"updated_at": PARENT_COMMENT_CURSOR_PARTITION_1},
1722+
}
1723+
],
1724+
"parent_state": {"posts": {"updated_at": PARENT_POSTS_CURSOR}},
1725+
}
1726+
},
1727+
"state": {"created_at": INITIAL_GLOBAL_CURSOR},
1728+
"states": [
1729+
{
1730+
"partition": {
1731+
"id": 10,
1732+
"parent_slice": {"id": 1, "parent_slice": {}},
1733+
},
1734+
"cursor": {"created_at": INITIAL_STATE_PARTITION_10_CURSOR},
1735+
},
1736+
{
1737+
"partition": {
1738+
"id": 11,
1739+
"parent_slice": {"id": 1, "parent_slice": {}},
1740+
},
1741+
"cursor": {"created_at": INITIAL_STATE_PARTITION_11_CURSOR},
1742+
},
1743+
],
1744+
"lookback_window": 86400,
1745+
},
1746+
# Expected state
1747+
{
1748+
"state": {"created_at": INITIAL_STATE_PARTITION_11_CURSOR},
1749+
"parent_state": {
1750+
"post_comments": {
1751+
"use_global_cursor": False,
1752+
"state": {"updated_at": COMMENT_10_UPDATED_AT}, # 10 is the "latest"
1753+
"parent_state": {
1754+
"posts": {"updated_at": POST_1_UPDATED_AT}
1755+
}, # post 1 is the latest
1756+
"lookback_window": 1,
1757+
"states": [
1758+
{
1759+
"partition": {"id": 1, "parent_slice": {}},
1760+
"cursor": {"updated_at": COMMENT_10_UPDATED_AT},
1761+
},
1762+
],
1763+
}
1764+
},
1765+
"lookback_window": 1,
1766+
"use_global_cursor": False,
1767+
"states": [
1768+
{
1769+
"partition": {"id": 10, "parent_slice": {"id": 1, "parent_slice": {}}},
1770+
# initial state because record doesn't have a cursor field
1771+
"cursor": {"created_at": INITIAL_STATE_PARTITION_10_CURSOR},
1772+
},
1773+
{
1774+
"partition": {"id": 11, "parent_slice": {"id": 1, "parent_slice": {}}},
1775+
"cursor": {"created_at": INITIAL_STATE_PARTITION_11_CURSOR},
1776+
},
1777+
],
1778+
},
1779+
),
16311780
],
16321781
)
16331782
def test_incremental_parent_state(

0 commit comments

Comments
 (0)