Skip to content

Commit 62808d8

Browse files
ziaddevvziaddevv
committed
added tests for regenerating failed/expired zim files
1 parent abe7346 commit 62808d8

File tree

1 file changed

+227
-0
lines changed

1 file changed

+227
-0
lines changed

‎wp1/logic/builder_test.py‎

Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1793,3 +1793,230 @@ def test_get_builder_module_class_missing_builder_class(self, mock_import):
17931793
logic_builder.get_builder_module_class("wp1.selection.models.simple")
17941794

17951795
self.assertIn("Builder class not found in module", str(cm.exception))
1796+
1797+
@patch("wp1.logic.builder.zimfarm.request_zimfarm_task")
1798+
@patch(
1799+
"wp1.logic.builder.utcnow", return_value=datetime.datetime(2025, 1, 2, 0, 0, 0)
1800+
)
1801+
def test_regenerate_zim_updates_old_task_when_selection_version_changed(
1802+
self, mock_utcnow, mock_request_zimfarm_task
1803+
):
1804+
"""
1805+
Ensure the existing zim_task is updated (not dupplicated) when the selection version changes.
1806+
"""
1807+
self._insert_builder()
1808+
1809+
with self.wp10db.cursor() as cursor:
1810+
cursor.execute(
1811+
"""INSERT INTO selections
1812+
(s_id, s_builder_id, s_updated_at, s_content_type, s_version, s_object_key, s_article_count)
1813+
VALUES (%s, %s, '20250102000000', 'text/tab-separated-values', 1, 'old.tsv', 100)""",
1814+
(1, self.builder.b_id),
1815+
)
1816+
self.wp10db.commit()
1817+
1818+
zim_schedule_id = b"schedule-123"
1819+
self._insert_zim_schedule(zim_schedule_id, self.builder.b_id)
1820+
1821+
with self.wp10db.cursor() as cursor:
1822+
cursor.execute(
1823+
"""INSERT INTO zim_tasks
1824+
(z_selection_id, z_zim_schedule_id, z_status, z_task_id)
1825+
VALUES (%s, %s, 'FAILED', 'old-task-id')""",
1826+
(1, zim_schedule_id),
1827+
)
1828+
self.wp10db.commit()
1829+
1830+
with self.wp10db.cursor() as cursor:
1831+
cursor.execute(
1832+
"""INSERT INTO selections
1833+
(s_id, s_builder_id, s_updated_at, s_content_type, s_version, s_object_key, s_article_count)
1834+
VALUES (%s, %s, '20250103000000', 'text/tab-separated-values', 2, 'new.tsv', 100)""",
1835+
(2, self.builder.b_id),
1836+
)
1837+
cursor.execute(
1838+
"UPDATE builders SET b_current_version = 2 WHERE b_id = %s",
1839+
(self.builder.b_id,),
1840+
)
1841+
self.wp10db.commit()
1842+
1843+
mock_request_zimfarm_task.return_value = "new-task-id"
1844+
1845+
redis = MagicMock()
1846+
result = logic_builder.request_zim_file_task_for_builder(
1847+
redis, self.wp10db, self.builder, zim_schedule_id=zim_schedule_id
1848+
)
1849+
1850+
with self.wp10db.cursor() as cursor:
1851+
cursor.execute("SELECT COUNT(*) as count FROM zim_tasks")
1852+
count = cursor.fetchone()["count"]
1853+
self.assertEqual(1, count)
1854+
1855+
cursor.execute("SELECT z_selection_id FROM zim_tasks")
1856+
row = cursor.fetchone()
1857+
self.assertEqual(b"2", row["z_selection_id"])
1858+
1859+
@patch("wp1.logic.builder.zimfarm.request_zimfarm_task")
1860+
@patch(
1861+
"wp1.logic.builder.utcnow", return_value=datetime.datetime(2025, 1, 2, 0, 0, 0)
1862+
)
1863+
def test_regenerate_zim_saves_new_task_id(
1864+
self, mock_utcnow, mock_request_zimfarm_task
1865+
):
1866+
"""
1867+
test that new task_id from Zimfarm is saved correctly.
1868+
"""
1869+
self._insert_builder()
1870+
self._insert_selection(
1871+
1, "text/tab-separated-values", builder_id=self.builder.b_id
1872+
)
1873+
1874+
zim_schedule_id = b"schedule-123"
1875+
self._insert_zim_schedule(zim_schedule_id, self.builder.b_id)
1876+
1877+
with self.wp10db.cursor() as cursor:
1878+
cursor.execute(
1879+
"""INSERT INTO zim_tasks
1880+
(z_selection_id, z_zim_schedule_id, z_status, z_task_id)
1881+
VALUES (%s, %s, 'FAILED', 'task_v1')""",
1882+
(1, zim_schedule_id),
1883+
)
1884+
self.wp10db.commit()
1885+
1886+
mock_request_zimfarm_task.return_value = "task_v2"
1887+
1888+
redis = MagicMock()
1889+
result = logic_builder.request_zim_file_task_for_builder(
1890+
redis, self.wp10db, self.builder, zim_schedule_id=zim_schedule_id
1891+
)
1892+
1893+
self.assertIsNotNone(result)
1894+
self.assertEqual(b"task_v2", result.z_task_id)
1895+
self.assertEqual(b"REQUESTED", result.z_status)
1896+
1897+
with self.wp10db.cursor() as cursor:
1898+
cursor.execute("SELECT z_task_id FROM zim_tasks WHERE z_selection_id = 1")
1899+
row = cursor.fetchone()
1900+
self.assertEqual(b"task_v2", row["z_task_id"])
1901+
1902+
@patch("wp1.logic.builder.zimfarm.request_zimfarm_task")
1903+
@patch(
1904+
"wp1.logic.builder.utcnow", return_value=datetime.datetime(2025, 1, 2, 0, 0, 0)
1905+
)
1906+
def test_regenerate_zim_updates_b_selection_zim_version(
1907+
self, mock_utcnow, mock_request_zimfarm_task
1908+
):
1909+
"""
1910+
test that b_selection_zim_version is updated for downloads to work.
1911+
"""
1912+
self._insert_builder(zim_version=1) # startts with version 1
1913+
1914+
# selection v1
1915+
with self.wp10db.cursor() as cursor:
1916+
cursor.execute(
1917+
"""INSERT INTO selections
1918+
(s_id, s_builder_id, s_updated_at, s_content_type, s_version, s_object_key, s_article_count)
1919+
VALUES (%s, %s, '20250102000000', 'text/tab-separated-values', 1, 'old.tsv', 100)""",
1920+
(1, self.builder.b_id),
1921+
)
1922+
self.wp10db.commit()
1923+
1924+
zim_schedule_id = b"schedule-123"
1925+
self._insert_zim_schedule(zim_schedule_id, self.builder.b_id)
1926+
1927+
with self.wp10db.cursor() as cursor:
1928+
cursor.execute(
1929+
"""INSERT INTO zim_tasks
1930+
(z_selection_id, z_zim_schedule_id, z_status, z_task_id)
1931+
VALUES (%s, %s, 'FAILED', 'old-task')""",
1932+
(1, zim_schedule_id),
1933+
)
1934+
self.wp10db.commit()
1935+
1936+
# Selection v2
1937+
with self.wp10db.cursor() as cursor:
1938+
cursor.execute(
1939+
"""INSERT INTO selections
1940+
(s_id, s_builder_id, s_updated_at, s_content_type, s_version, s_object_key, s_article_count)
1941+
VALUES (%s, %s, '20250103000000', 'text/tab-separated-values', 2, 'new.tsv', 100)""",
1942+
(2, self.builder.b_id),
1943+
)
1944+
cursor.execute(
1945+
"UPDATE builders SET b_current_version = 2 WHERE b_id = %s",
1946+
(self.builder.b_id,),
1947+
)
1948+
self.wp10db.commit()
1949+
1950+
with self.wp10db.cursor() as cursor:
1951+
cursor.execute(
1952+
"SELECT b_selection_zim_version FROM builders WHERE b_id = %s",
1953+
(self.builder.b_id,),
1954+
)
1955+
version_before = cursor.fetchone()["b_selection_zim_version"]
1956+
self.assertEqual(1, version_before)
1957+
1958+
mock_request_zimfarm_task.return_value = "task_v2"
1959+
1960+
redis = MagicMock()
1961+
result = logic_builder.request_zim_file_task_for_builder(
1962+
redis, self.wp10db, self.builder, zim_schedule_id=zim_schedule_id
1963+
)
1964+
1965+
with self.wp10db.cursor() as cursor:
1966+
cursor.execute(
1967+
"SELECT b_selection_zim_version FROM builders WHERE b_id = %s",
1968+
(self.builder.b_id,),
1969+
)
1970+
version_after = cursor.fetchone()["b_selection_zim_version"]
1971+
self.assertEqual(2, version_after)
1972+
1973+
@patch("wp1.logic.builder.zimfarm.zim_file_url_for_task_id")
1974+
def test_download_url_after_regeneration(self, mock_zim_file_url):
1975+
"""
1976+
test that download URL works after regenerating a failed ZIM.
1977+
"""
1978+
self._insert_builder()
1979+
1980+
with self.wp10db.cursor() as cursor:
1981+
cursor.execute(
1982+
"""INSERT INTO selections
1983+
(s_id, s_builder_id, s_updated_at, s_content_type, s_version, s_object_key, s_article_count)
1984+
VALUES (%s, %s, '20230101000000', 'text/tab-separated-values', 1, 'old.tsv', 100)""",
1985+
(1, self.builder.b_id),
1986+
)
1987+
cursor.execute(
1988+
"""INSERT INTO zim_tasks
1989+
(z_selection_id, z_status, z_task_id)
1990+
VALUES (%s, 'FAILED', 'old-failed-task')""",
1991+
(1,),
1992+
)
1993+
self.wp10db.commit()
1994+
1995+
with self.wp10db.cursor() as cursor:
1996+
cursor.execute(
1997+
"""INSERT INTO selections
1998+
(s_id, s_builder_id, s_updated_at, s_content_type, s_version, s_object_key, s_article_count)
1999+
VALUES (%s, %s, '20230102000000', 'text/tab-separated-values', 2, 'new.tsv', 100)""",
2000+
(2, self.builder.b_id),
2001+
)
2002+
cursor.execute(
2003+
"""INSERT INTO zim_tasks
2004+
(z_selection_id, z_status, z_task_id)
2005+
VALUES (%s, 'FILE_READY', 'new-successful-task')""",
2006+
(2,),
2007+
)
2008+
cursor.execute(
2009+
"""UPDATE builders SET
2010+
b_current_version = 2,
2011+
b_selection_zim_version = 2
2012+
WHERE b_id = %s""",
2013+
(self.builder.b_id,),
2014+
)
2015+
self.wp10db.commit()
2016+
2017+
mock_zim_file_url.return_value = "https://download.kiwix.org/zim/new-file.zim"
2018+
2019+
url = logic_builder.latest_zim_file_url_for(self.wp10db, self.builder.b_id)
2020+
2021+
self.assertIsNotNone(url)
2022+
mock_zim_file_url.assert_called_once_with(b"new-successful-task")

0 commit comments

Comments
 (0)