Add tests for sh.hs.frontier/jobsmeta

Author: vshlapakov

tests/hubstorage/conftest.py

@@ -63,12 +63,20 @@ def setup_test(hsclient, hsproject, hscollection, request, vcr_instance):
         _set_testbotgroup(hsproject)
         _remove_all_jobs(hsproject)
         _clean_collection(hscollection)
+        _delete_frontier_slot(hsproject)
         yield
         _remove_all_jobs(hsproject)
+        _clean_collection(hscollection)
+        _delete_frontier_slot(hsproject)
         _unset_testbotgroup(hsproject)
 
 # ----------------------------------------------------------------------------
 
+def _delete_frontier_slot(hsproject):
+    frontier = hsproject.frontier
+    frontier.delete_slot(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT)
+
+
 def _clean_collection(hscollection):
     for item in hscollection.iter_values():
         hscollection.delete(item['_key'])

tests/hubstorage/test_frontier.py

@@ -1,128 +1,114 @@
 """
 Test Frontier
 """
-from .hstestcase import HSTestCase
+from .conftest import TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT
 
 
-class FrontierTest(HSTestCase):
+def _get_urls(batch):
+    return [r[0] for r in batch['requests']]
 
-    def setUp(self):
-        self._delete_slot()
 
-    def tearDown(self):
-        self._delete_slot()
+def test_add_read(hsproject):
+    frontier = hsproject.frontier
 
-    def _delete_slot(self):
-        frontier = self.project.frontier
-        frontier.delete_slot(self.frontier, self.slot)
+    fps = [{'fp': '/'}]
+    frontier.add(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT, fps)
+    fps = [{'fp': '/index.html'}, {'fp': '/index2.html'}]
+    frontier.add(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT, fps)
+    frontier.flush()
 
-    def _remove_all_ids(self):
-        frontier = self.project.frontier
-        ids = [batch['id'] for batch in frontier.read(self.frontier, self.slot)]
-        frontier.delete(self.frontier, self.slot, ids)
+    urls = [_get_urls(batch) for batch
+            in frontier.read(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT)]
+    expected_urls = [[u'/', u'/index.html', u'/index2.html']]
+    assert urls == expected_urls
 
-    def _get_urls(self, batch):
-        return [r[0] for r in batch['requests']]
 
-    def test_add_read(self):
-        frontier = self.project.frontier
+def test_add_multiple_chunks(hsproject):
+    frontier = hsproject.frontier
+    old_count = frontier.newcount
 
-        fps = [{'fp': '/'}]
-        frontier.add(self.frontier, self.slot, fps)
-        fps = [{'fp': '/index.html'}, {'fp': '/index2.html'}]
-        frontier.add(self.frontier, self.slot, fps)
-        frontier.flush()
+    batch_size = 50
+    fps1 = [{'fp': '/index_%s.html' % fp} for fp in range(0, batch_size)]
+    frontier.add(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT, fps1)
 
-        urls = [self._get_urls(batch) for batch
-                in frontier.read(self.frontier, self.slot)]
-        expected_urls = [[u'/', u'/index.html', u'/index2.html']]
-        self.assertEqual(urls, expected_urls)
+    fps2 = [{'fp': '/index_%s.html' % fp} for fp in range(batch_size, batch_size * 2)]
+    frontier.add(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT, fps2)
 
-    def test_add_multiple_chunks(self):
-        frontier = self.project.frontier
-        old_count = frontier.newcount
+    fps3 = [{'fp': '/index_%s.html' % fp} for fp in range(batch_size * 2, batch_size * 3)]
+    frontier.add(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT, fps3)
+    frontier.flush()
 
-        batch_size = 50
-        fps1 = [{'fp': '/index_%s.html' % fp} for fp in range(0, batch_size)]
-        frontier.add(self.frontier, self.slot, fps1)
+    assert frontier.newcount == 150 + old_count
 
-        fps2 = [{'fp': '/index_%s.html' % fp} for fp in range(batch_size, batch_size * 2)]
-        frontier.add(self.frontier, self.slot, fps2)
+    # insert repeated fingerprints
+    fps4 = [{'fp': '/index_%s.html' % fp} for fp in range(0, batch_size)]
+    frontier.add(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT, fps3)
+    frontier.flush()
 
-        fps3 = [{'fp': '/index_%s.html' % fp} for fp in range(batch_size * 2, batch_size * 3)]
-        frontier.add(self.frontier, self.slot, fps3)
-        frontier.flush()
+    # new count is the same
+    assert frontier.newcount == 150 + old_count
 
-        self.assertEqual(frontier.newcount, 150 + old_count)
+    # get first 100
+    batches = list(frontier.read(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT, mincount=100))
+    urls = [_get_urls(batch) for batch in batches]
+    expected_urls = [[fp['fp'] for fp in fps1 + fps2]]
+    assert urls == expected_urls
 
-        # insert repeated fingerprints
-        fps4 = [{'fp': '/index_%s.html' % fp} for fp in range(0, batch_size)]
-        frontier.add(self.frontier, self.slot, fps3)
-        frontier.flush()
+    # delete first 100
+    ids = [batch['id'] for batch in batches]
+    frontier.delete(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT, ids)
 
-        # new count is the same
-        self.assertEqual(frontier.newcount, 150 + old_count)
+    # get remaining 50
+    batches = list(frontier.read(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT))
+    urls = [_get_urls(batch) for batch in batches]
+    expected_urls = [[fp['fp'] for fp in fps3]]
+    assert urls == expected_urls
 
-        # get first 100
-        batches = list(frontier.read(self.frontier, self.slot, mincount=100))
-        urls = [self._get_urls(batch) for batch in batches]
-        expected_urls = [[fp['fp'] for fp in fps1 + fps2]]
-        self.assertEqual(urls, expected_urls)
 
-        # delete first 100
-        ids = [batch['id'] for batch in batches]
-        frontier.delete(self.frontier, self.slot, ids)
+def test_add_big_chunk(hsproject):
+    frontier = hsproject.frontier
 
-        # get remaining 50
-        batches = list(frontier.read(self.frontier, self.slot))
-        urls = [self._get_urls(batch) for batch in batches]
-        expected_urls = [[fp['fp'] for fp in fps3]]
-        self.assertEqual(urls, expected_urls)
+    batch_size = 300
+    fps1 = [{'fp': '/index_%s.html' % fp} for fp in range(0, batch_size)]
+    frontier.add(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT, fps1)
+    frontier.flush()
 
-    def test_add_big_chunk(self):
-        frontier = self.project.frontier
+    # get first 100
+    batches = list(frontier.read(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT, mincount=100))
+    urls = [_get_urls(batch) for batch in batches]
+    expected_urls = [[fp['fp'] for fp in fps1[:100]]]
+    assert urls == expected_urls
 
-        batch_size = 300
-        fps1 = [{'fp': '/index_%s.html' % fp} for fp in range(0, batch_size)]
-        frontier.add(self.frontier, self.slot, fps1)
-        frontier.flush()
+    # delete first 100
+    ids = [batch['id'] for batch in batches]
+    frontier.delete(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT, ids)
 
-        # get first 100
-        batches = list(frontier.read(self.frontier, self.slot, mincount=100))
-        urls = [self._get_urls(batch) for batch in batches]
-        expected_urls = [[fp['fp'] for fp in fps1[:100]]]
-        self.assertEqual(urls, expected_urls)
+    # get next 100
+    batches = list(frontier.read(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT, mincount=100))
+    urls = [_get_urls(batch) for batch in batches]
+    expected_urls = [[fp['fp'] for fp in fps1[100:200]]]
+    assert urls == expected_urls
 
-        # delete first 100
-        ids = [batch['id'] for batch in batches]
-        frontier.delete(self.frontier, self.slot, ids)
+    # delete next 100
+    ids = [batch['id'] for batch in batches]
+    frontier.delete(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT, ids)
 
-        # get next 100
-        batches = list(frontier.read(self.frontier, self.slot, mincount=100))
-        urls = [self._get_urls(batch) for batch in batches]
-        expected_urls = [[fp['fp'] for fp in fps1[100:200]]]
-        self.assertEqual(urls, expected_urls)
+    # get next 100
+    batches = list(frontier.read(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT, mincount=100))
+    urls = [_get_urls(batch) for batch in batches]
+    expected_urls = [[fp['fp'] for fp in fps1[200:300]]]
+    assert urls == expected_urls
 
-        # delete next 100
-        ids = [batch['id'] for batch in batches]
-        frontier.delete(self.frontier, self.slot, ids)
-
-        # get next 100
-        batches = list(frontier.read(self.frontier, self.slot, mincount=100))
-        urls = [self._get_urls(batch) for batch in batches]
-        expected_urls = [[fp['fp'] for fp in fps1[200:300]]]
-        self.assertEqual(urls, expected_urls)
-
-    def test_add_extra_params(self):
-        frontier = self.project.frontier
-
-        qdata = {"a": 1, "b": 2, "c": 3}
-        fps = [{'fp': '/', "qdata": qdata}]
-        frontier.add(self.frontier, self.slot, fps)
-        frontier.flush()
-
-        expected_request = [[u'/', {u'a': 1, u'c': 3, u'b': 2}]]
-        batches = list(frontier.read(self.frontier, self.slot))
-        request = batches[0]['requests']
-        self.assertEqual(request, expected_request)
 
+def test_add_extra_params(hsproject):
+    frontier = hsproject.frontier
+
+    qdata = {"a": 1, "b": 2, "c": 3}
+    fps = [{'fp': '/', "qdata": qdata}]
+    frontier.add(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT, fps)
+    frontier.flush()
+
+    expected_request = [[u'/', {u'a': 1, u'c': 3, u'b': 2}]]
+    batches = list(frontier.read(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT))
+    request = batches[0]['requests']
+    assert request == expected_request