Cassette for session setup, clean test bg logic

Author: vshlapakov

tests/hubstorage/conftest.py

@@ -10,13 +10,15 @@
 TEST_SPIDER_NAME = 'hs-test-spider'
 TEST_FRONTIER_NAME = 'test'
 TEST_FRONTIER_SLOT = 'site.com'
-TEST_BOTGROUPS = ['python-hubstorage-test', 'g1']
+TEST_BOTGROUP = 'python-hubstorage-test'
 TEST_COLLECTION_NAME = "test_collection_123"
 TEST_AUTH = os.getenv('HS_AUTH', 'f' * 32)
 TEST_ENDPOINT = os.getenv('HS_ENDPOINT', 'http://storage.vm.scrapinghub.com')
 
 VCR_CASSETES_DIR = 'tests/hubstorage/cassetes'
 
+my_vcr = vcr.VCR(cassette_library_dir=VCR_CASSETES_DIR, record_mode='once')
+
 
 @pytest.fixture(scope='session')
 def hsclient():
@@ -28,44 +30,44 @@ def hsproject(hsclient):
     return hsclient.get_project(TEST_PROJECT_ID)
 
 
-@pytest.fixture
+#@my_vcr.use_cassette()
+@pytest.fixture(scope='session')
 def hsspiderid(hsproject):
-    # it's important that scope for the fixture is per test ('function'):
-    # all the current per-session fixtures don't do any external requests,
-    # it allows to use vcrpy inside a per-test fixture
     return str(hsproject.ids.spider(TEST_SPIDER_NAME, create=1))
 
 
-@pytest.fixture
-def hscollection(hsproject):
-    return hsproject.collections.new_store(TEST_COLLECTION_NAME)
-
 
+#@my_vcr.use_cassette()
 @pytest.fixture(autouse=True, scope='session')
-def setup_session(hsclient):
+def setup_session(hsclient, hsproject, hscollection):
+    set_testbotgroup(hsproject)
+    remove_all_jobs(hsproject)
     yield
+    remove_all_jobs(hsproject)
+    unset_testbotgroup(hsproject)
     hsclient.close()
 
 
-@pytest.fixture
-def vcr_instance(scope='session'):
-    return vcr.VCR(cassette_library_dir=VCR_CASSETES_DIR, record_mode='once')
-
-
 @pytest.fixture(autouse=True)
-def setup_test(hsclient, hsproject, hscollection, request, vcr_instance):
+def setup_vcrpy_per_test(request, hsproject):
     # generates names like "test_module/test_function.yaml"
     # vcrpy creates the cassetes automatically under VCR_CASSETES_DIR
     cassette_name = '{}/{}.yaml'.format(
         request.function.__module__.split('.')[-1],
         request.function.__name__
     )
-    with vcr_instance.use_cassette(cassette_name):
-        _set_testbotgroup(hsproject)
-        clean_environment(hsproject, hscollection)
-        yield
-        clean_environment(hsproject, hscollection)
-        _unset_testbotgroup(hsproject)
+    #with my_vcr.use_cassette(cassette_name):
+    yield
+
+
+#@my_vcr.use_cassette()
+@pytest.fixture(scope='session')
+def hscollection(hsproject):
+    collection = get_test_collection(hsproject)
+    clean_collection(collection)
+    yield collection
+    clean_collection(collection)
+
 
 # ----------------------------------------------------------------------------
 
@@ -81,17 +83,7 @@ def start_job(hsproject, **startparams):
 # Clean environment section
 
 
-def clean_environment(hsproject, hscollection):
-    _remove_all_jobs(hsproject)
-    # drop all items in test collection
-    for item in hscollection.iter_values():
-        hscollection.delete(item['_key'])
-    # delete frontier slot
-    frontier = hsproject.frontier
-    frontier.delete_slot(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT)
-
-
-def _remove_all_jobs(hsproject):
+def remove_all_jobs(hsproject):
     for k in list(hsproject.settings.keys()):
         if k != 'botgroups':
             del hsproject.settings[k]
@@ -114,23 +106,32 @@ def _remove_job(hsproject, jobkey):
     assert jobkey.startswith(TEST_PROJECT_ID), jobkey
     hsproject.jobs.apidelete(jobkey.partition('/')[2])
 
+# Collection helpers section
+
+
+def get_test_collection(project):
+    return project.collections.new_store(TEST_COLLECTION_NAME)
+
+
+def clean_collection(collection):
+    for item in collection.iter_values():
+        collection.delete(item['_key'])
+
 
 # Botgroups helpers section
 
 
-def _set_testbotgroup(hsproject):
-    hsproject.settings.apipost(jl={'botgroups': [TEST_BOTGROUPS[0]]})
+def set_testbotgroup(hsproject):
+    hsproject.settings.apipost(jl={'botgroups': [TEST_BOTGROUP]})
     # Additional step to populate JobQ's botgroups table
-    for botgroup in TEST_BOTGROUPS:
-        url = urlpathjoin(TEST_ENDPOINT, 'botgroups', botgroup, 'max_running')
-        requests.post(url, auth=hsproject.auth, data='null')
+    url = urlpathjoin(TEST_ENDPOINT, 'botgroups', TEST_BOTGROUP, 'max_running')
+    requests.post(url, auth=hsproject.auth, data='null')
     hsproject.settings.expire()
 
 
-def _unset_testbotgroup(hsproject):
+def unset_testbotgroup(hsproject):
     hsproject.settings.apidelete('botgroups')
     hsproject.settings.expire()
     # Additional step to delete botgroups in JobQ
-    for botgroup in TEST_BOTGROUPS:
-        url = urlpathjoin(TEST_ENDPOINT, 'botgroups', botgroup)
-        requests.delete(url, auth=hsproject.auth)
+    url = urlpathjoin(TEST_ENDPOINT, 'botgroups', TEST_BOTGROUP)
+    requests.delete(url, auth=hsproject.auth)

tests/hubstorage/test_collections.py

@@ -17,13 +17,12 @@ def _mkitem():
                 field3=3, field4={'v4k': 'v4v'})
 
 
-def test_simple_count(hsproject):
+def test_simple_count(hsproject, hscollection):
     test_item = dict(_mkitem())
     test_item['_key'] = 'a'
 
-    collection = hsproject.collections.new_store(TEST_COLLECTION_NAME)
-    collection.set(test_item)
-    assert collection.count() == 1
+    hscollection.set(test_item)
+    assert hscollection.count() == 1
 
 
 def post_get_delete_test(hsproject):
@@ -51,42 +50,40 @@ def post_get_delete_test(hsproject):
             col.get(test_key)
 
 
-def post_scan_test(hsproject):
-    col = hsproject.collections.new_store(TEST_COLLECTION_NAME)
-
+def post_scan_test(hsproject, hscollection):
     # populate with 20 items
     test_item = _mkitem()
     last_key = None
-    with closing(col.create_writer()) as writer:
+    with closing(hscollection.create_writer()) as writer:
         for i in range(20):
             test_item['_key'] = last_key = "post_scan_test%d" % i
             test_item['counter'] = i
             writer.write(test_item)
 
     # check last value is as expected
-    returned_item = col.get(last_key)
+    returned_item = hscollection.get(last_key)
     del test_item['_key']
     assert test_item == returned_item
 
     # get all values starting with 1
-    result = list(col.get(prefix='post_scan_test1'))
+    result = list(hscollection.get(prefix='post_scan_test1'))
     # 1 & 10-19 = 11 items
     assert len(result) == 11
 
     # combining with normal filters
-    result = list(col.get(filter='["counter", ">", [5]]',
+    result = list(hscollection.get(filter='["counter", ">", [5]]',
                           prefix='post_scan_test1'))
     # 10-19
     assert len(result) == 10
 
     # bulk delete
-    col.delete('post_scan_test%d' % i for i in range(20))
+    hscollection.delete('post_scan_test%d' % i for i in range(20))
 
     # test items removed (check first and last)
     with pytest.raises(KeyError):
-        col.get('post_scan_test0')
+        hscollection.get('post_scan_test0')
     with pytest.raises(KeyError):
-        col.get(last_key)
+        hscollection.get(last_key)
 
 
 def test_errors(hscollection):
@@ -102,10 +99,9 @@ def test_errors(hscollection):
             hscollection.set(arg)
 
 
-def test_data_download(hsproject):
-    col = hsproject.collections.new_store(TEST_COLLECTION_NAME)
+def test_data_download(hsproject, hscollection):
     items = []
-    with closing(col.create_writer()) as writer:
+    with closing(hscollection.create_writer()) as writer:
         for i in range(20):
             test_item = _mkitem()
             test_item['_key'] = "test_data_download%d" % i
@@ -114,12 +110,12 @@ def test_data_download(hsproject):
             items.append(test_item)
 
     # check parameters are passed correctly
-    downloaded = list(col.iter_values(prefix='test_data_download1'))
+    downloaded = list(hscollection.iter_values(prefix='test_data_download1'))
     assert len(downloaded) == 11
 
     # simulate network timeouts and download data
     with failing_downloader(hsproject.collections):
-        downloaded = list(col.iter_values(start='test_data_download1'))
+        downloaded = list(hscollection.iter_values(start='test_data_download1'))
         assert len(downloaded) == 19
 
 

tests/hubstorage/test_frontier.py

@@ -1,9 +1,16 @@
 """
 Test Frontier
 """
+import pytest
 from .conftest import TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT
 
 
+@pytest.fixture(autouse=True)
+def delete_frontier_slot(hsproject):
+    frontier = hsproject.frontier
+    frontier.delete_slot(TEST_FRONTIER_NAME, TEST_FRONTIER_SLOT)
+
+
 def _get_urls(batch):
     return [r[0] for r in batch['requests']]
 

tests/hubstorage/test_jobq.py

@@ -11,6 +11,14 @@
 
 from .conftest import TEST_PROJECT_ID, TEST_SPIDER_NAME
 from .conftest import hsspiderid
+from .conftest import remove_all_jobs
+
+
+@pytest.fixture(autouse=True)
+def clean_jobs(hsproject):
+    remove_all_jobs(hsproject)
+    yield
+    remove_all_jobs(hsproject)
 
 
 def _keys(lst):

tests/hubstorage/test_jobsmeta.py

@@ -3,8 +3,17 @@
 
 System tests for operations on stored job metadata
 """
+import pytest
 from .conftest import TEST_SPIDER_NAME
 from .conftest import start_job
+from .conftest import remove_all_jobs
+
+
+@pytest.fixture(autouse=True)
+def clean_jobs(hsproject):
+    remove_all_jobs(hsproject)
+    yield
+    remove_all_jobs(hsproject)
 
 
 def _assertMetadata(meta1, meta2):

tests/hubstorage/test_project.py

@@ -1,10 +1,8 @@
 """
 Test Project
 """
-import json
-from random import randint, random
-
 import six
+import json
 import pytest
 from six.moves import range
 from requests.exceptions import HTTPError
@@ -14,9 +12,18 @@
 from .conftest import TEST_PROJECT_ID, TEST_SPIDER_NAME
 from .conftest import hsspiderid
 from .conftest import start_job
+from .conftest import set_testbotgroup
+from .conftest import remove_all_jobs
 from .testutil import failing_downloader
 
 
+@pytest.fixture(autouse=True)
+def clean_jobs(hsproject):
+    remove_all_jobs(hsproject)
+    yield
+    remove_all_jobs(hsproject)
+
+
 def test_projectid(hsclient):
     p1 = hsclient.get_project(int(TEST_PROJECT_ID))
     p2 = hsclient.get_project(str(TEST_PROJECT_ID))
@@ -157,7 +164,13 @@ def test_broad(hsproject, hsspiderid):
     job.purged()
 
 
-def test_settings(hsproject):
+@pytest.fixture
+def restore_botgroup(hsproject):
+    yield
+    set_testbotgroup(hsproject)
+
+
+def test_settings(hsproject, restore_botgroup):
     settings = dict(hsproject.settings)
     settings.pop('botgroups', None)  # ignore testsuite botgroups
     assert settings == {}

tests/hubstorage/test_system.py

@@ -10,10 +10,17 @@
 from .conftest import TEST_ENDPOINT, TEST_SPIDER_NAME
 from .conftest import TEST_PROJECT_ID, TEST_AUTH
 from .conftest import start_job
+from .conftest import remove_all_jobs
 
 
 MAGICN = 1211
 
+@pytest.fixture(autouse=True)
+def clean_jobs(hsproject):
+    remove_all_jobs(hsproject)
+    yield
+    remove_all_jobs(hsproject)
+
 
 @pytest.fixture
 def panelclient():