Merge pull request #685 from wwwjfy/mysql-support

mysql support

Author: wwwjfy

.codacy.yml

@@ -1,3 +1,4 @@
 exclude_paths:
   - 'tests/**'
+  - 'mysql_tests/**'
   - 'docs/**'

.coveragerc

@@ -1,6 +1,8 @@
 [run]
 source =
     ./src
+omit =
+    ./src/gino/aiocontextvars.py
 [report]
 exclude_lines =
     pragma: no cover

.github/workflows/test.yml

@@ -66,21 +66,127 @@ jobs:
         DB_HOST: localhost
         DB_USER: gino
       run: |
-        $HOME/.poetry/bin/poetry run pytest --cov=src --cov-fail-under=95 --cov-report xml
+        $HOME/.poetry/bin/poetry run pytest tests/
+  test-mysql:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ '3.5', '3.6', '3.7', '3.8' ]
+        deps-version: [ 'lowest', 'highest' ]
+    services:
+      mysql:
+        image: mysql:5
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: 1
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+    steps:
+    - name: Checkout source code
+      uses: actions/checkout@v1
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: virtualenv cache
+      uses: actions/cache@preview
+      with:
+        path: ~/.cache/pypoetry/virtualenvs
+        key: ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.deps-version }}-venv-${{ hashFiles(format('{0}{1}', github.workspace, '/poetry.lock')) }}
+        restore-keys: |
+          ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.deps-version }}-venv-
+    - name: Poetry cache
+      uses: actions/cache@preview
+      with:
+        path: ~/.poetry
+        key: ${{ runner.os }}-${{ matrix.python-version }}-dotpoetry
+        restore-keys: |
+          ${{ runner.os }}-${{ matrix.python-version }}-dotpoetry-
+    - name: Install Python dependencies
+      run: |
+        curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
+        $HOME/.poetry/bin/poetry install --no-interaction
+    - name: Use lowest dependencies versions
+      if: matrix.deps-version == 'lowest'
+      run: |
+          $HOME/.poetry/bin/poetry run pip install asyncpg==0.18 SQLAlchemy==1.3
+    - name: List installed packages
+      run: |
+          $HOME/.poetry/bin/poetry run pip list
+    - name: Test with pytest
+      env:
+        MYSQL_DB_HOST: 127.0.0.1
+        MYSQL_DB_USER: root
+      run: |
+        $HOME/.poetry/bin/poetry run pytest mysql_tests/
+  summary:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: fantix/postgres-ssl:12.1
+        env:
+          POSTGRES_USER: gino
+        ports:
+        - 5432:5432
+        # needed because the postgres container does not provide a healthcheck
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+      mysql:
+        image: mysql:5
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: 1
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+    steps:
+    - name: Checkout source code
+      uses: actions/checkout@v1
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+    - name: virtualenv cache
+      uses: actions/cache@preview
+      with:
+        path: ~/.cache/pypoetry/virtualenvs
+        key: ${{ runner.os }}-3.8-highest-venv-${{ hashFiles(format('{0}{1}', github.workspace, '/poetry.lock')) }}
+        restore-keys: |
+          ${{ runner.os }}-3.8-highest-venv-
+    - name: Poetry cache
+      uses: actions/cache@preview
+      with:
+        path: ~/.poetry
+        key: ${{ runner.os }}-3.8-dotpoetry
+        restore-keys: |
+          ${{ runner.os }}-3.8-dotpoetry-
+    - name: Install Python dependencies
+      run: |
+        curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
+        $HOME/.poetry/bin/poetry install --no-interaction
+    - name: List installed packages
+      run: |
+          $HOME/.poetry/bin/poetry run pip list
+    - name: Test with pytest
+      env:
+        DB_HOST: localhost
+        DB_USER: gino
+        MYSQL_DB_HOST: 127.0.0.1
+        MYSQL_DB_USER: root
+      run: |
+        $HOME/.poetry/bin/poetry run pytest --cov=src --cov-fail-under=95 --cov-report xml tests/ mysql_tests/
     - name: Check code format with black
-      if: matrix.python-version >= '3.6'
       run: |
           $HOME/.poetry/bin/poetry run black --check src
     - name: Submit coverage report
-      if: matrix.python-version == '3.8' && matrix.postgres-version == '12.1' && matrix.deps-version == 'highest' && github.ref == 'refs/heads/master'
+      if: github.ref == 'refs/heads/master'
       env:
         CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_TOKEN }}
       run: |
         pip install codacy-coverage
         python-codacy-coverage -r coverage.xml
+
   release:
     runs-on: ubuntu-latest
-    needs: test
+    needs: summary
     strategy:
       matrix:
         python-version: [ '3.8' ]

mysql_tests/__init__.py

@@ -0,0 +1,59 @@
+import ssl
+
+import aiomysql
+import pytest
+import sqlalchemy
+from async_generator import yield_, async_generator
+
+import gino
+from .models import db, DB_ARGS, MYSQL_URL, random_name
+
+ECHO = False
+
+
+@pytest.fixture(scope="module")
+def sa_engine():
+    rv = sqlalchemy.create_engine(MYSQL_URL, echo=ECHO)
+    db.create_all(rv)
+    yield rv
+    db.drop_all(rv)
+    rv.dispose()
+
+
+@pytest.fixture
+@async_generator
+async def engine(sa_engine):
+    e = await gino.create_engine(MYSQL_URL, echo=ECHO, minsize=10)
+    await yield_(e)
+    await e.close()
+    sa_engine.execute("DELETE FROM gino_user_settings")
+    sa_engine.execute("DELETE FROM gino_users")
+
+
+# noinspection PyUnusedLocal,PyShadowingNames
+@pytest.fixture
+@async_generator
+async def bind(sa_engine):
+    async with db.with_bind(MYSQL_URL, echo=ECHO, minsize=10) as e:
+        await yield_(e)
+    sa_engine.execute("DELETE FROM gino_user_settings")
+    sa_engine.execute("DELETE FROM gino_users")
+
+
+# noinspection PyUnusedLocal,PyShadowingNames
+@pytest.fixture
+@async_generator
+async def aiomysql_pool(sa_engine):
+    async with aiomysql.create_pool(**DB_ARGS) as rv:
+        await yield_(rv)
+        async with rv.acquire() as conn:
+            await conn.query("DELETE FROM gino_user_settings")
+            await conn.query("DELETE FROM gino_users")
+
+
+@pytest.fixture
+def ssl_ctx():
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+    return ctx

mysql_tests/conftest.py

@@ -0,0 +1,157 @@
+import os
+import enum
+import random
+import string
+from datetime import datetime
+
+import aiomysql
+import asyncpg
+import pytest
+
+from gino import Gino
+from gino.dialects.aiomysql import JSON
+
+DB_ARGS = dict(
+    host=os.getenv("MYSQL_DB_HOST", "localhost"),
+    port=os.getenv("MYSQL_DB_PORT", 3306),
+    user=os.getenv("MYSQL_DB_USER", "root"),
+    password=os.getenv("MYSQL_DB_PASS", ""),
+    db=os.getenv("MYSQL_DB_NAME", "mysql"),
+)
+MYSQL_URL = "mysql://{user}:{password}@{host}:{port}/{db}".format(**DB_ARGS)
+db = Gino()
+
+
+@pytest.fixture
+def random_name(length=8) -> str:
+    return _random_name(length)
+
+
+def _random_name(length=8):
+    return "".join(random.choice(string.ascii_letters) for _ in range(length))
+
+
+class UserType(enum.Enum):
+    USER = "USER"
+
+
+class User(db.Model):
+    __tablename__ = "gino_users"
+
+    id = db.Column(db.BigInteger(), primary_key=True)
+    nickname = db.Column("name", db.Unicode(255), default=_random_name)
+    profile = db.Column("props", JSON(), nullable=False, default="{}")
+    type = db.Column(db.Enum(UserType), nullable=False, default=UserType.USER)
+    realname = db.StringProperty()
+    age = db.IntegerProperty(default=18)
+    balance = db.IntegerProperty(default=0)
+    birthday = db.DateTimeProperty(default=lambda i: datetime.utcfromtimestamp(0))
+    team_id = db.Column(db.ForeignKey("gino_teams.id"))
+
+    @balance.after_get
+    def balance(self, val):
+        if val is None:
+            return 0.0
+        return float(val)
+
+    def __repr__(self):
+        return "{}<{}>".format(self.nickname, self.id)
+
+
+class Friendship(db.Model):
+    __tablename__ = "gino_friendship"
+
+    my_id = db.Column(db.BigInteger(), primary_key=True)
+    friend_id = db.Column(db.BigInteger(), primary_key=True)
+
+    def __repr__(self):
+        return "Friends<{}, {}>".format(self.my_id, self.friend_id)
+
+
+class Relation(db.Model):
+    __tablename__ = "gino_relation"
+
+    name = db.Column(db.VARCHAR(255), primary_key=True)
+
+
+class Team(db.Model):
+    __tablename__ = "gino_teams"
+
+    id = db.Column(db.BigInteger(), primary_key=True)
+    name = db.Column(db.Unicode(255), default=_random_name)
+    parent_id = db.Column(db.ForeignKey("gino_teams.id", ondelete='CASCADE'))
+    company_id = db.Column(db.ForeignKey("gino_companies.id"))
+
+    def __init__(self, **kw):
+        super().__init__(**kw)
+        self._members = set()
+
+    @property
+    def members(self):
+        return self._members
+
+    @members.setter
+    def add_member(self, user):
+        self._members.add(user)
+
+
+class TeamWithDefaultCompany(Team):
+    company = Team(name="DEFAULT")
+
+
+class TeamWithoutMembersSetter(Team):
+    def add_member(self, user):
+        self._members.add(user)
+
+
+class Company(db.Model):
+    __tablename__ = "gino_companies"
+
+    id = db.Column(db.BigInteger(), primary_key=True)
+    name = db.Column(db.Unicode(255), default=_random_name)
+    logo = db.Column(db.LargeBinary())
+
+    def __init__(self, **kw):
+        super().__init__(**kw)
+        self._teams = set()
+
+    @property
+    def teams(self):
+        return self._teams
+
+    @teams.setter
+    def add_team(self, team):
+        self._teams.add(team)
+
+
+class CompanyWithoutTeamsSetter(Company):
+    def add_team(self, team):
+        self._teams.add(team)
+
+
+class UserSetting(db.Model):
+    __tablename__ = "gino_user_settings"
+
+    # No constraints defined on columns
+    id = db.Column(db.BigInteger())
+    user_id = db.Column(db.BigInteger())
+    setting = db.Column(db.VARCHAR(255))
+    value = db.Column(db.Text())
+    col1 = db.Column(db.Integer, default=1)
+    col2 = db.Column(db.Integer, default=2)
+
+    # Define indexes and constraints inline
+    id_pkey = db.PrimaryKeyConstraint("id")
+    user_id_fk = db.ForeignKeyConstraint(["user_id"], ["gino_users.id"])
+    user_id_setting_unique = db.UniqueConstraint("user_id", "setting")
+    col1_check = db.CheckConstraint("col1 >= 1 AND col1 <= 5")
+    col2_idx = db.Index("col2_idx", "col2")
+
+
+def qsize(engine):
+    if isinstance(engine.raw_pool, aiomysql.pool.Pool):
+        return engine.raw_pool.freesize
+    if isinstance(engine.raw_pool, asyncpg.pool.Pool):
+        # noinspection PyProtectedMember
+        return engine.raw_pool._queue.qsize()
+    raise Exception('Unknown pool')