Added docker fixtures for future testcases

Author: terricain

.travis.yml

@@ -9,6 +9,9 @@ env:
         - PYTHONASYNCIODEBUG=1
         - PYTHONASYNCIODEBUG=
 
+services:
+    - docker
+
 matrix:
    include:
         - python: 3.6

requirements-dev.txt

@@ -5,6 +5,8 @@ ipython==6.2.1
 pytest==3.4.2
 pytest-cov==2.5.1
 pytest-sugar==0.9.1
+PyMySQL>=0.7.5
+docker==3.1.4
 sphinx==1.7.1
 sphinxcontrib-asyncio==0.2.0
 sqlalchemy==1.2.5

tests/conftest.py

@@ -1,9 +1,16 @@
 import asyncio
 import gc
 import os
+import ssl
+import socket
 import sys
+import time
+import uuid
+
+from docker import APIClient
 
 import aiomysql
+import pymysql
 import pytest
 
 
@@ -14,11 +21,36 @@
     uvloop = None
 
 
+@pytest.fixture(scope='session')
+def unused_port():
+    def f():
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            s.bind(('127.0.0.1', 0))
+            return s.getsockname()[1]
+    return f
+
+
 def pytest_generate_tests(metafunc):
     if 'loop_type' in metafunc.fixturenames:
         loop_type = ['asyncio', 'uvloop'] if uvloop else ['asyncio']
         metafunc.parametrize("loop_type", loop_type)
 
+    # if 'mysql_tag' in metafunc.fixturenames:
+    #     tags = set(metafunc.config.option.mysql_tag)
+    #     if not tags:
+    #         tags = ['5.7']
+    #     elif 'all' in tags:
+    #         tags = ['5.6', '5.7', '8.0']
+    #     else:
+    #         tags = list(tags)
+    #     metafunc.parametrize("mysql_tag", tags, scope='session')
+
+
+# This is here unless someone fixes the generate_tests bit
+@pytest.yield_fixture(scope='session')
+def mysql_tag():
+    return '5.6'
+
 
 @pytest.yield_fixture
 def loop(request, loop_type):
@@ -77,6 +109,15 @@ def pytest_ignore_collect(path, config):
             return True
 
 
+def pytest_addoption(parser):
+    parser.addoption("--mysql_tag", action="append", default=[],
+                     help=("MySQL server versions. "
+                           "May be used several times. "
+                           "Available values: 5.6, 5.7, 8.0, all"))
+    parser.addoption("--no-pull", action="store_true", default=False,
+                     help="Don't perform docker images pulling")
+
+
 @pytest.fixture
 def mysql_params():
     params = {"host": os.environ.get('MYSQL_HOST', 'localhost'),
@@ -164,3 +205,108 @@ def _register_table(table_name):
         # TODO: probably this is not safe code
         sql = "DROP TABLE IF EXISTS {};".format(t)
         loop.run_until_complete(cursor.execute(sql))
+
+
+@pytest.fixture(scope='session')
+def session_id():
+    """Unique session identifier, random string."""
+    return str(uuid.uuid4())
+
+
+@pytest.fixture(scope='session')
+def docker():
+    return APIClient(version='auto')
+
+
+@pytest.fixture(scope='session')
+def mysql_server(unused_port, docker, session_id, mysql_tag, request):
+    if not request.config.option.no_pull:
+        docker.pull('mysql:{}'.format(mysql_tag))
+
+    # bound IPs do not work on OSX
+    host = "127.0.0.1"
+    host_port = unused_port()
+
+    # As TLS is optional, might as well always configure it
+    ssl_directory = os.path.join(os.path.dirname(__file__),
+                                 'ssl_resources', 'ssl')
+    ca_file = os.path.join(ssl_directory, 'ca.pem')
+    tls_cnf = os.path.join(os.path.dirname(__file__),
+                           'ssl_resources', 'tls.cnf')
+
+    ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
+    ctx.check_hostname = False
+    ctx.load_verify_locations(cafile=ca_file)
+    # ctx.verify_mode = ssl.CERT_NONE
+
+    container_args = dict(
+        image='mysql:{}'.format(mysql_tag),
+        name='aiomysql-test-server-{}-{}'.format(mysql_tag, session_id),
+        ports=[3306],
+        detach=True,
+        host_config=docker.create_host_config(
+            port_bindings={3306: (host, host_port)},
+            binds={
+                ssl_directory: {'bind': '/etc/mysql/ssl', 'mode': 'ro'},
+                tls_cnf: {'bind': '/etc/mysql/conf.d/tls.cnf', 'mode': 'ro'},
+            }
+        ),
+        environment={'MYSQL_ROOT_PASSWORD': 'rootpw'}
+    )
+
+    container = docker.create_container(**container_args)
+
+    try:
+        docker.start(container=container['Id'])
+
+        # MySQL restarts at least 4 times in the container before its ready
+        time.sleep(10)
+
+        server_params = {
+            'host': host,
+            'port': host_port,
+            'user': 'root',
+            'password': 'rootpw',
+            'ssl': ctx
+        }
+        delay = 0.001
+        for i in range(100):
+            try:
+                connection = pymysql.connect(
+                    db='mysql',
+                    charset='utf8mb4',
+                    cursorclass=pymysql.cursors.DictCursor,
+                    **server_params)
+
+                with connection.cursor() as cursor:
+                    cursor.execute("SHOW VARIABLES LIKE '%ssl%';")
+
+                    result = cursor.fetchall()
+                    result = {item['Variable_name']:
+                              item['Value'] for item in result}
+
+                    assert result['have_ssl'] == "YES", \
+                        "SSL Not Enabled on docker'd MySQL"
+
+                    cursor.execute("SHOW STATUS LIKE '%Ssl_version%'")
+
+                    result = cursor.fetchone()
+                    # As we connected with TLS, it should start with that :D
+                    assert result['Value'].startswith('TLS'), \
+                        "Not connected to the database with TLS"
+
+                break
+            except Exception as err:
+                time.sleep(delay)
+                delay *= 2
+        else:
+            pytest.fail("Cannot start MySQL server")
+
+        container['host'] = host
+        container['port'] = host_port
+        container['conn_params'] = server_params
+
+        yield container
+    finally:
+        docker.kill(container=container['Id'])
+        docker.remove_container(container['Id'])

tests/ssl_resources/README.md

@@ -0,0 +1,20 @@
+# MySQL TLS Stuff
+
+This folder contains some resources to be mounted into a mysql container to support TLS
+ 
+Most of the instructions were taken from here https://dev.mysql.com/doc/refman/5.7/en/creating-ssl-files-using-openssl.html
+
+# Generating certificates
+```bash
+openssl genrsa 2048 > ca-key.pem
+openssl req -new -x509 -nodes -days 3600 -key ca-key.pem -out ca.pem
+
+openssl req -newkey rsa:2048 -days 3600 -nodes -keyout server-key.pem -out server-req.pem
+openssl rsa -in server-key.pem -out server-key.pem
+openssl x509 -req -in server-req.pem -days 3600 -CA ca.pem -CAkey ca-key.pem -set_serial 01 -out server-cert.pem
+```
+The current files under `ssl/` have the default values provided by openssl.
+
+# MySQL Config
+MySQL imports all `.cnf` files under `/etc/mysql/conf.d` so a `tls.cnf` is placed in there referencing the SSL CA 
+cert and server cert and key. The entire `ssl/` directory should be mounted on the container to `/etc/mysql/ssl/`

tests/ssl_resources/ssl/ca-key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA4QGo8oVPe8HCIqfJXYVxZbHAZDVlMK/e6ypQM463B8lNg6Ce
+dd0QfAVIs3wuXtRTVD6ELVFDrmcg/eBjM00Idh2GcI0ojDAPcECBvvk0SWYrUBFn
+bou5R1tPwCb8yY601dNExteT61egVhTQd8Vz6NjN9tkMqrpnZ3pn86QqnFWuDOvJ
+5YkOAHOjXipkb8ba+Y4Jl50N6KZH2vrHwdWiO/DYQJLuvoOtYI6a+AXNH+ZSNvid
+Vz9nb3taUQcNxUlYtTByPfnU9wo/dzDFpNKvC5yScPvh2AmJUrEKNEYkFY/YbqLc
+iCXOoBfh1EHqmR2EASjanr+LgwTJfXoX3bQeQQIDAQABAoIBAQDLQ3igPhXjstHy
+BKlANwCN4dnvrNzQ8s/qmbsCGHb4Lb48nqkHyMDPiOZ4XkJ1oFH21NMLLVJ7Buci
+8cYr3fc63MlKe/qZSgFoYp3TK8U0WXvfRRmvH8Is2CxfZdkPLD/ouoZzKuSRwgMy
+QHNi/5kKTHEkAkgTI3muXUHzM+baeknisEqXqCib1yY4FfX2Vnip1dTbbj1gEbGu
+vgFM67uXsoKeQ8ykTF5ZUDIphP+tFWgLMZA2L3iTgRSZJztWwxUMa9NZ8erxy25h
+HPntC5OELndKEbO+su7wCMkw8w2cNA1V6yyQlwSoIRD1fhit/v4hOmvofTpoikXH
+DaCJjgwBAoGBAP84Wq8nrp+79BWRX/qsivERtkGnfjFIDbWBhj70az/CgCV8OWWu
+td6VEEl/Gk7IHsrmOySF0EMYYeMoc5WQKMZbb+1x2m7ovxEwS/UGwNGdtZswyOJw
+y0LKI6dJPQUiZm/O2m0w1zs6/CvvIfu9QOUTxyvwub4lM8UW/gDy79URAoGBAOGx
+q83cTMHKY32dNU/IVaMpw9OxojAWpWXYOEqyV7hM2+gw/lrdBKRyWwB3l3smwet1
+FvKINCZ0bTIRbz/UsNtXp8lISTvw5bQhGsQEYx6ncBeBeHN50zSVoR2xBfqu3pQ4
+G5V/UI82hba7QUDXkuMJ2T7dcZixLk1vp3y+LvYxAoGBANkGYc7J7qs0F6Xzfeta
+p7fA+PuxYxSjEc1DfBWyoDSSv4egr+owe8Tveu8UnxlZAR5GUwqGo4c6h5qzvj3z
+XUj3XiFKjJV9Y2RJbn3IpVRaSKDUBi7P/XgpDdJl6/aevv7aplDtlEhwqxjs+zfn
+QfTKMbbCuB/h4Lj7CTljW+ARAoGAchB7hgVK/b4t3jRv1yymq1nWUM077RXk7b4D
+ZS0RTGH72jO4uW9ugzYQbAIFGwaRh1CcEmNoB+9bqKxLD3WNFK4ObJoN+S9cyFba
+0iptdfalnhufJq1xYugkj38CSJnMgBiDSGEZ8+dYWOv2pLDO2dQGadE9MjCJ+DTv
+7wmnbmECgYBnkVNH0wOvdMuT0vovm18zqtH04PmQGg5JXgQjtpn/6iC9BjRaF2Nz
+Jj9arX00KxIO5vkzbyt5ht5fzk4dpXoG5ozOqnqbj6WNEfgJDeyCkqo1gZOMZSQ/
+YPjKGL7rhXbE/FsNEH90nG0NpIAk1ibtD9sYn6LBBYJcuauegOgE7g==
+-----END RSA PRIVATE KEY-----

tests/ssl_resources/ssl/ca.pem

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDXTCCAkWgAwIBAgIJAP2JtVGC0ZPKMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV
+BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
+aWRnaXRzIFB0eSBMdGQwHhcNMTgwMzI5MjEzMzA1WhcNMjgwMjA1MjEzMzA1WjBF
+MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50
+ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEA4QGo8oVPe8HCIqfJXYVxZbHAZDVlMK/e6ypQM463B8lNg6Cedd0QfAVI
+s3wuXtRTVD6ELVFDrmcg/eBjM00Idh2GcI0ojDAPcECBvvk0SWYrUBFnbou5R1tP
+wCb8yY601dNExteT61egVhTQd8Vz6NjN9tkMqrpnZ3pn86QqnFWuDOvJ5YkOAHOj
+Xipkb8ba+Y4Jl50N6KZH2vrHwdWiO/DYQJLuvoOtYI6a+AXNH+ZSNvidVz9nb3ta
+UQcNxUlYtTByPfnU9wo/dzDFpNKvC5yScPvh2AmJUrEKNEYkFY/YbqLciCXOoBfh
+1EHqmR2EASjanr+LgwTJfXoX3bQeQQIDAQABo1AwTjAdBgNVHQ4EFgQUyNN4RMiv
+zhCNNvMta2kOoiw/SXswHwYDVR0jBBgwFoAUyNN4RMivzhCNNvMta2kOoiw/SXsw
+DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAyAiIqfKoemjnJmoMqcrg
+N/7QhC9P79rzwKQOEFkCFWJFuGiOD6GhPPCu/9ssrl5vBEwDLdl0V4AeGq8DeKV+
+SNON1o6Y6uUAiX5uYK5Asv0avQUu7SS+uvE3YhTELjbvp4vqdLCUjBqq4KZoEA+F
+4hXCltPVdOItztzAd7hgktYrkJeDA1M7sZHTv26HaO6vJ0trUdb4tvqzShzMCvN/
+2s9ZJAVBZL77Px40yUPiK6cjpq5fGcUen0zBumymBRFOb8ykvq7azUdjk6sz65Vb
+Q3kgsKGBHjVOPfXf00YWvgG3NePX3FsBEHtYbBDSD6k+aXQ1WfOUB8mtdwjyEnzD
+7w==
+-----END CERTIFICATE-----

tests/ssl_resources/ssl/server-cert.pem

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIC/jCCAeYCAQEwDQYJKoZIhvcNAQELBQAwRTELMAkGA1UEBhMCQVUxEzARBgNV
+BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0
+ZDAeFw0xODAzMjkyMTMzMTFaFw0yODAyMDUyMTMzMTFaMEUxCzAJBgNVBAYTAkFV
+MRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRz
+IFB0eSBMdGQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCfUvCGVUNU
+reTRbcuBQQ2w9+VM3TPMNr3gdZV1X1yg3TtEJc/kBYnSPfhz3S0Rar+Y8pdxCLvr
+lGJeBaevaXwzj8tW85S/3mv5BFtG71x5bB/dueG4wgEaEEhuXJIQHs89nzGgggCb
+hZCuDp2lqwptX2U/Pr9YJDbSuko3pp0EvMi7qfnNyzrZY9gLqUkTd1QH+bNWQW5G
+DMK9fbDXvGme/xyD4OLLEHy7Cc+KyGBzSOFqLNVpmppC4M5jf9aTM2s57fEaDE+t
+pT4E8LKciXYKAW03n+pfoECf1JneslGltmhbvANGLuA4lfOr/I6v5f2QsNS8ISV/
+d7+YR+O9k5oVAgMBAAEwDQYJKoZIhvcNAQELBQADggEBACUWEAWfv3EOy8JmUbWA
+jEytJIh/N55hfknninjOBPMv1U1BRixJwXVKlwA8o+8JiacriObWeje2rDUOt6zY
+U5DnySQbTYJcJZ9jprqU7VXST7D9NvA0ueLclWTZcqIr/josyhK+l1YbezFYBf41
+JQ4PVzkNz9Of4e022qONnlEX0MbtFlcyPEK4yWyXLAhidPAV9QcOCy85vob0+3EE
+hmRVVzcTv4Pbzgpee0ZORqozSLzZ3N6RvDyYIczqaytcbyvaQ7GuykE7XvIK5hz0
+EM8pwsvxSY1z2yNIw38M8ZOYk18LsEGkf/TyT6eQqymMMD9Qy8rOTsOLfY5eQCf7
+pKQ=
+-----END CERTIFICATE-----

tests/ssl_resources/ssl/server-key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAn1LwhlVDVK3k0W3LgUENsPflTN0zzDa94HWVdV9coN07RCXP
+5AWJ0j34c90tEWq/mPKXcQi765RiXgWnr2l8M4/LVvOUv95r+QRbRu9ceWwf3bnh
+uMIBGhBIblySEB7PPZ8xoIIAm4WQrg6dpasKbV9lPz6/WCQ20rpKN6adBLzIu6n5
+zcs62WPYC6lJE3dUB/mzVkFuRgzCvX2w17xpnv8cg+DiyxB8uwnPishgc0jhaizV
+aZqaQuDOY3/WkzNrOe3xGgxPraU+BPCynIl2CgFtN5/qX6BAn9SZ3rJRpbZoW7wD
+Ri7gOJXzq/yOr+X9kLDUvCElf3e/mEfjvZOaFQIDAQABAoIBAFNubFP8LEEYut1M
+4Kez+EZ22hXRNEG5XN9A095d7LS0hUefgWkH2W9GUmfiJ6qaOvEOAG4Jw9aOoqBX
+18LMu2SI5VOIRJnhEKubM21HBSb0jw9eOqy0sz0Bz9wzD63vZFkBl0xVJ5pJbEUp
+lDZgBhrWPL/MzQiMFkVtllXkIw+KNRIokV0HJn0VNUm+ORaDO0TnTAiqL7Wv553+
+lWvGaeI4NpMZOPtlgqym1neQmllkeB07pSEtTopO0iINwuTuDUU7IMleN4eOomfh
+GwirEHUban8BDL5djckS0GrkUq9EuJbvjEikHAFuNwz7D1Sn3LsDYygD6pKWoXxh
+8Ng/AZUCgYEAy1ma5xKvRZZ6QUFssZjtwbruxhj0j3lPA3t9VGlp4SQb3g6FgeOV
+6dyJLuuFfSLT9ps+k8D1Er+v85OqO0IM8TR12rPjWMzSjBz/xHb+0uRbEoPwArfn
+wcPj3NN6M+tcZH//djogrpphN7u+BudBG6YWzUOQjLwdStM52s8hFD8CgYEAyJM2
+M5AWPYL57CI1lqzN135aS6OFyG4N2+rtEWEGAmoeP+NoSGFmQPLTWBdT4ZOGl59/
+fSBrWRKted6/H9frjZuSdsGXFMb7e71DDyYZq7tJbYEExc0a6BePINLCwLCIMKHj
+PcoPGVsdQXfZK17+qzACwDNbEis/J3H7xcPv7KsCgYBO+SG7k/oV4HbiWPJJlsbf
+ciXBMXfpMIeLJq5p1faUxV09RA59f1F9XXS5kCZrjtca8ve+kjWbbm56/mIiWWiF
+VIZgxXQJzKIIYErEliIo7R6hdjQEGkAbdGROIqNW/pUHQt6Hn9OJe9M9vd/y9mTG
+xB4e4ZqFzZjisl3JqJ+EKQKBgFyJboxDgb9HWj7TWZ32g9FT/hy/iM172PEJZe6K
+sNcUVnhrVoVuSlrUrSULPivogEQb1hnIhz5FG7wKRGtQluByUhRwJF/1nbjtDK9E
+iLtuYOYgjC8l/a/ujp46Hpf/2hV12v1655RvMQQvYwZbgWtBb0N1biLnyO9N6zbG
+uz6ZAoGAXW3mhN9zbN6EgsIlLu+wCdPuC9Vs968gCT612E8Ijul1kiSODG3rbMoG
+2FbjqZLahyX8vWVhX/m4xDqO7DXTwi81polfFuxbc/PimOLI4DKq62lWLMBDmUba
+X8Bxal3FXLvdcNEplcXadqmxJeXMYnYsfC+MCQhe6im0bGyDOAg=
+-----END RSA PRIVATE KEY-----

tests/ssl_resources/ssl/server-req.pem

@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIICijCCAXICAQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUx
+ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDCCASIwDQYJKoZIhvcN
+AQEBBQADggEPADCCAQoCggEBAJ9S8IZVQ1St5NFty4FBDbD35UzdM8w2veB1lXVf
+XKDdO0Qlz+QFidI9+HPdLRFqv5jyl3EIu+uUYl4Fp69pfDOPy1bzlL/ea/kEW0bv
+XHlsH9254bjCARoQSG5ckhAezz2fMaCCAJuFkK4OnaWrCm1fZT8+v1gkNtK6Sjem
+nQS8yLup+c3LOtlj2AupSRN3VAf5s1ZBbkYMwr19sNe8aZ7/HIPg4ssQfLsJz4rI
+YHNI4Wos1WmamkLgzmN/1pMzaznt8RoMT62lPgTwspyJdgoBbTef6l+gQJ/Umd6y
+UaW2aFu8A0Yu4DiV86v8jq/l/ZCw1LwhJX93v5hH472TmhUCAwEAAaAAMA0GCSqG
+SIb3DQEBCwUAA4IBAQCWRJYZTj9Wxt5Wf7IG7K6V7cZNfiJBkAyKbt7Ny60lB3/C
+xS3hmq4mFKpsE2OXtNCHjs1/fhodgR4aMkORTN7Zin5tbGoc3bRwfY92ktbZx4z0
+W3EOP4peih6xGtrTYdcr/Ww/M72fhDvLopr55n091U2fismXcjLD5BJWx7T/rGX1
+Luwu+S90E+fA5vqYhOgfT87OcopAGzia1xDFnsX62OSpj36EwPcB6tS1IjIZz87a
+lugDmd39+gyM0rJ0bpe0HH8PmgKOAc/F5SZCCWJChmYcvXNcURaLEReCg0RHOvsD
+ZEYmB2Ki+jwHO6DoQbNz3Erq8rl4GWjorYz4GDUH
+-----END CERTIFICATE REQUEST-----

tests/ssl_resources/tls.cnf

@@ -0,0 +1,4 @@
+[mysqld]
+ssl-ca=/etc/mysql/ssl/ca.pem
+ssl-cert=/etc/mysql/ssl/server-cert.pem
+ssl-key=/etc/mysql/ssl/server-key.pem