Add local testing infrastructure (#149)

* initial commit. This will likely need to be updated to work in ci

* pull redis host from environment, use requirements file

* formatting

* downgrade pytest so it still supports python 2.7

Author: AlecRosenbaum

.circleci/config.yml

@@ -12,11 +12,14 @@ workflows:
 
 defaults: &defaults
   working_directory: ~/code
+  environment:
+    # circleci exposes services on localhost
+    REDIS_HOST: localhost
   steps:
   - checkout
   - run:
       name: Install dependencies
-      command: sudo pip install -r requirements.txt freezefrog psutil pytest
+      command: sudo pip install -r requirements.txt -r requirements-test.txt
   - run:
       name: Test
       command: pytest

Dockerfile

@@ -0,0 +1,6 @@
+FROM circleci/python:3.6
+
+WORKDIR /src
+COPY requirements.txt .
+COPY requirements-test.txt .
+RUN pip install --user -r requirements.txt -r requirements-test.txt

README.rst

@@ -718,3 +718,19 @@ reported to Rollbar. Here is a custom worker launch script:
   tiger.log.addHandler(rollbar_handler)
 
   tiger.run_worker_with_args(sys.argv[1:])
+
+
+Running The Test Suite
+----------------------
+
+Tests can be run locally using the provided docker compose file. After installing docker, tests should be runnable with:
+
+.. code :: bash
+
+  docker-compose run --rm tasktiger pytest
+
+Tests can be more granularly run using normal pytest flags. For example:
+
+.. code :: bash
+
+  docker-compose run --rm tasktiger pytest tests/test_base.py::TestCase

docker-compose.yml

@@ -0,0 +1,16 @@
+version: "3.7"
+services:
+  redis:
+    image: redis:4.0.6
+    expose:
+      - 6379
+  tasktiger:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    environment:
+      REDIS_HOST: redis
+    volumes:
+      - .:/src
+    depends_on:
+      - redis

requirements-test.txt

@@ -0,0 +1,3 @@
+freezefrog==0.3.2
+psutil==5.6.3
+pytest==4.6.5

tests/config.py

@@ -1,5 +1,10 @@
+import os
+
 # How much to delay scheduled tasks for testing purposes.
 DELAY = 0.2
 
 # Redis database number which will be wiped and used for the tests
-TEST_DB = 7
+TEST_DB = int(os.environ.get('REDIS_DB', 7))
+
+# Redis hostname
+REDIS_HOST = os.environ.get('REDIS_HOST', 'localhost')

tests/tasks.py

@@ -7,7 +7,7 @@
 from tasktiger import RetryException
 from tasktiger.retry import fixed
 
-from .config import DELAY, TEST_DB
+from .config import DELAY, TEST_DB, REDIS_HOST
 from .utils import get_tiger
 
 
@@ -52,28 +52,28 @@ def long_task_killed():
 @tiger.task(hard_timeout=DELAY * 2)
 def long_task_ok():
     # Signal task has started
-    conn = redis.Redis(db=TEST_DB, decode_responses=True)
+    conn = redis.Redis(host=REDIS_HOST, db=TEST_DB, decode_responses=True)
     conn.lpush(LONG_TASK_SIGNAL_KEY, '1')
 
     time.sleep(DELAY)
 
 
 def wait_for_long_task():
     """Waits for a long task to start."""
-    conn = redis.Redis(db=TEST_DB, decode_responses=True)
+    conn = redis.Redis(host=REDIS_HOST, db=TEST_DB, decode_responses=True)
     result = conn.blpop(LONG_TASK_SIGNAL_KEY, int(ceil(DELAY * 3)))
     assert result[1] == '1'
 
 
 @tiger.task(unique=True)
 def unique_task(value=None):
-    conn = redis.Redis(db=TEST_DB, decode_responses=True)
+    conn = redis.Redis(host=REDIS_HOST, db=TEST_DB, decode_responses=True)
     conn.lpush('unique_task', value)
 
 
 @tiger.task(lock=True)
 def locked_task(key, other=None):
-    conn = redis.Redis(db=TEST_DB, decode_responses=True)
+    conn = redis.Redis(host=REDIS_HOST, db=TEST_DB, decode_responses=True)
     data = conn.getset(key, 1)
     if data is not None:
         raise Exception('task failed, key already set')
@@ -83,7 +83,7 @@ def locked_task(key, other=None):
 
 @tiger.task(queue='batch', batch=True)
 def batch_task(params):
-    conn = redis.Redis(db=TEST_DB, decode_responses=True)
+    conn = redis.Redis(host=REDIS_HOST, db=TEST_DB, decode_responses=True)
     try:
         conn.rpush('batch_task', json.dumps(params))
     except Exception:
@@ -94,7 +94,7 @@ def batch_task(params):
 
 @tiger.task(queue='batch')
 def non_batch_task(arg):
-    conn = redis.Redis(db=TEST_DB, decode_responses=True)
+    conn = redis.Redis(host=REDIS_HOST, db=TEST_DB, decode_responses=True)
     conn.rpush('batch_task', arg)
     if arg == 10:
         raise Exception('exception')
@@ -109,7 +109,7 @@ def retry_task_2():
 
 
 def verify_current_task():
-    conn = redis.Redis(db=TEST_DB, decode_responses=True)
+    conn = redis.Redis(host=REDIS_HOST, db=TEST_DB, decode_responses=True)
 
     try:
         tiger.current_tasks
@@ -121,7 +121,7 @@ def verify_current_task():
 
 @tiger.task(batch=True, queue='batch')
 def verify_current_tasks(tasks):
-    conn = redis.Redis(db=TEST_DB, decode_responses=True)
+    conn = redis.Redis(host=REDIS_HOST, db=TEST_DB, decode_responses=True)
 
     try:
         tasks = tiger.current_task

tests/tasks_periodic.py

@@ -4,7 +4,7 @@
 
 from tasktiger.schedule import periodic
 
-from .config import TEST_DB
+from .config import TEST_DB, REDIS_HOST
 from .utils import get_tiger
 
 tiger = get_tiger()
@@ -13,7 +13,7 @@
 @tiger.task(schedule=periodic(seconds=1), queue='periodic')
 def periodic_task():
     """Periodic task."""
-    conn = redis.Redis(db=TEST_DB, decode_responses=True)
+    conn = redis.Redis(host=REDIS_HOST, db=TEST_DB, decode_responses=True)
     conn.incr('period_count', 1)
 
 

tests/test_context_manager.py

@@ -5,7 +5,7 @@
 
 from .tasks import exception_task, simple_task
 from .test_base import BaseTestCase
-from .config import TEST_DB
+from .config import TEST_DB, REDIS_HOST
 
 
 class ContextManagerTester(object):
@@ -17,7 +17,9 @@ class ContextManagerTester(object):
 
     def __init__(self, name):
         self.name = name
-        self.conn = redis.Redis(db=TEST_DB, decode_responses=True)
+        self.conn = redis.Redis(
+            host=REDIS_HOST, db=TEST_DB, decode_responses=True
+        )
         self.conn.set('cm:{}:enter'.format(self.name), 0)
         self.conn.set('cm:{}:exit'.format(self.name), 0)
 

tests/utils.py

@@ -3,7 +3,7 @@
 import structlog
 from tasktiger import TaskTiger, Worker, fixed
 
-from .config import DELAY, TEST_DB
+from .config import DELAY, TEST_DB, REDIS_HOST
 
 TEST_TIGER_CONFIG = {
     # We need this 0 here so we don't pick up scheduled tasks when
@@ -51,7 +51,7 @@ def setup_structlog():
 
 
 def get_redis():
-    return redis.Redis(db=TEST_DB, decode_responses=True)
+    return redis.Redis(host=REDIS_HOST, db=TEST_DB, decode_responses=True)
 
 
 def get_tiger():