Skip to content

Commit 2aec1df

Browse files
darynaishchenkodarynaishchenko
committed
added unit test
1 parent 683d974 commit 2aec1df

File tree

1 file changed

+172
-0
lines changed

1 file changed

+172
-0
lines changed

‎unit_tests/sources/declarative/incremental/test_concurrent_perpartitioncursor.py‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1460,6 +1460,178 @@ def run_incremental_parent_state_test(
14601460
],
14611461
},
14621462
),
1463+
(
1464+
"test_incremental_parent_state_records_without_cursor",
1465+
SUBSTREAM_MANIFEST,
1466+
[
1467+
# Fetch the first page of posts
1468+
(
1469+
f"https://api.example.com/community/posts?per_page=100&start_time={PARENT_POSTS_CURSOR}",
1470+
{
1471+
"posts": [
1472+
{"id": 1, "updated_at": POST_1_UPDATED_AT},
1473+
]
1474+
},
1475+
),
1476+
# Fetch the first page of comments for post 1
1477+
(
1478+
"https://api.example.com/community/posts/1/comments?per_page=100",
1479+
{
1480+
"comments": [
1481+
{
1482+
"id": 9,
1483+
"post_id": 1,
1484+
"updated_at": COMMENT_9_OLDEST,
1485+
},
1486+
{
1487+
"id": 10,
1488+
"post_id": 1,
1489+
"updated_at": COMMENT_10_UPDATED_AT,
1490+
},
1491+
{
1492+
"id": 11,
1493+
"post_id": 1,
1494+
"updated_at": COMMENT_11_UPDATED_AT,
1495+
},
1496+
]
1497+
},
1498+
),
1499+
# Fetch the first page of votes for comment 10 of post 1 (vote without cursor field)
1500+
(
1501+
f"https://api.example.com/community/posts/1/comments/10/votes?per_page=100&start_time={INITIAL_STATE_PARTITION_10_CURSOR}",
1502+
{
1503+
"votes": [
1504+
{
1505+
"id": 100,
1506+
"comment_id": 10,
1507+
}
1508+
],
1509+
},
1510+
),
1511+
# Fetch the first page of votes for comment 11 of post 1
1512+
(
1513+
f"https://api.example.com/community/posts/1/comments/11/votes"
1514+
f"?per_page=100&start_time={INITIAL_STATE_PARTITION_11_CURSOR}",
1515+
{"votes": [{"id": 111, "comment_id": 11, "created_at": VOTE_111_CREATED_AT}]},
1516+
),
1517+
# Fetch the first page of votes for comment 12 of post 1
1518+
(
1519+
f"https://api.example.com/community/posts/1/comments/12/votes?per_page=100&start_time={LOOKBACK_DATE}",
1520+
{"votes": []},
1521+
),
1522+
# Fetch the first page of comments for post 2
1523+
(
1524+
"https://api.example.com/community/posts/2/comments?per_page=100",
1525+
{
1526+
"comments": [{"id": 20, "post_id": 2, "updated_at": COMMENT_20_UPDATED_AT}],
1527+
"next_page": "https://api.example.com/community/posts/2/comments?per_page=100&page=2",
1528+
},
1529+
),
1530+
# Requests with intermediate states
1531+
# Fetch votes for comment 10 of post 1
1532+
(
1533+
f"https://api.example.com/community/posts/1/comments/10/votes?per_page=100&start_time={VOTE_100_CREATED_AT}",
1534+
{
1535+
"votes": [{"id": 100, "comment_id": 10, "created_at": VOTE_100_CREATED_AT}],
1536+
},
1537+
),
1538+
# Fetch votes for comment 11 of post 1
1539+
(
1540+
f"https://api.example.com/community/posts/1/comments/11/votes?per_page=100&start_time={VOTE_111_CREATED_AT}",
1541+
{
1542+
"votes": [{"id": 111, "comment_id": 11, "created_at": VOTE_111_CREATED_AT}],
1543+
},
1544+
),
1545+
# Fetch votes for comment 12 of post 1
1546+
(
1547+
f"https://api.example.com/community/posts/1/comments/12/votes?per_page=100&start_time={VOTE_111_CREATED_AT}",
1548+
{
1549+
"votes": [],
1550+
},
1551+
),
1552+
],
1553+
# Expected records
1554+
[
1555+
{
1556+
"comment_id": 10,
1557+
"comment_updated_at": COMMENT_10_UPDATED_AT,
1558+
"id": 100
1559+
},
1560+
{
1561+
"comment_id": 11,
1562+
"comment_updated_at": COMMENT_11_UPDATED_AT,
1563+
"created_at": "2024-01-13T00:00:00Z",
1564+
"id": 111
1565+
}
1566+
],
1567+
# Number of intermediate states - 6 as number of parent partitions
1568+
2,
1569+
# Initial state
1570+
{
1571+
"parent_state": {
1572+
"post_comments": {
1573+
"states": [
1574+
{
1575+
"partition": {"id": 1, "parent_slice": {}},
1576+
"cursor": {"updated_at": PARENT_COMMENT_CURSOR_PARTITION_1},
1577+
}
1578+
],
1579+
"parent_state": {"posts": {"updated_at": PARENT_POSTS_CURSOR}},
1580+
}
1581+
},
1582+
"state": {"created_at": INITIAL_GLOBAL_CURSOR},
1583+
"states": [
1584+
{
1585+
"partition": {
1586+
"id": 10,
1587+
"parent_slice": {"id": 1, "parent_slice": {}},
1588+
},
1589+
"cursor": {"created_at": INITIAL_STATE_PARTITION_10_CURSOR},
1590+
},
1591+
{
1592+
"partition": {
1593+
"id": 11,
1594+
"parent_slice": {"id": 1, "parent_slice": {}},
1595+
},
1596+
"cursor": {"created_at": INITIAL_STATE_PARTITION_11_CURSOR},
1597+
},
1598+
],
1599+
"lookback_window": 86400,
1600+
},
1601+
# Expected state
1602+
{
1603+
"state": {"created_at": VOTE_111_CREATED_AT},
1604+
"parent_state": {
1605+
"post_comments": {
1606+
"use_global_cursor": False,
1607+
"state": {"updated_at": COMMENT_10_UPDATED_AT}, # 10 is the "latest"
1608+
"parent_state": {
1609+
"posts": {"updated_at": POST_1_UPDATED_AT}
1610+
}, # post 1 is the latest
1611+
"lookback_window": 1,
1612+
"states": [
1613+
{
1614+
"partition": {"id": 1, "parent_slice": {}},
1615+
"cursor": {"updated_at": COMMENT_10_UPDATED_AT},
1616+
},
1617+
],
1618+
}
1619+
},
1620+
"lookback_window": 1,
1621+
"use_global_cursor": False,
1622+
"states": [
1623+
{
1624+
"partition": {"id": 10, "parent_slice": {"id": 1, "parent_slice": {}}},
1625+
# initial state because record doesn't have a cursor field
1626+
"cursor": {"created_at": INITIAL_STATE_PARTITION_10_CURSOR},
1627+
},
1628+
{
1629+
"partition": {"id": 11, "parent_slice": {"id": 1, "parent_slice": {}}},
1630+
"cursor": {"created_at": VOTE_111_CREATED_AT},
1631+
},
1632+
],
1633+
},
1634+
),
14631635
],
14641636
)
14651637
def test_incremental_parent_state(

0 commit comments

Comments
 (0)