Simplify tox.ini, base testing on galaxy-compose

.github/workflows/ci.yaml

@@ -24,7 +24,4 @@ jobs:
           python -m pip install --upgrade pip
           pip install tox coveralls
       - name: Test with tox
-        run: |
-          TOXENV=$(echo $TOXENV | sed 's/\.//') tox
-        env:
-          TOXENV: py${{ matrix.python-version }}-${{ matrix.tox-action }}
+        run: tox -e ${{ matrix.tox-action }}

.travis.yml

@@ -6,9 +6,9 @@ env:
 matrix:
   include:
     - python: 3.6
-      env: TOX_ENV=py36-lint
+      env: TOX_ENV=lint
     - python: 3.6
-      env: TOX_ENV=py36-pytest
+      env: TOX_ENV=pytest
       install:
       - pip install tox
       - pip install codacy-coverage

dev-requirements.txt

@@ -4,6 +4,7 @@ coverage
 pytest
 pytest-cov
 docker
+jinja2
 
 #Building Docs
 sphinx_rtd_theme

tests/docker_for_galaxy.py

@@ -1,59 +1,139 @@
+import os
+import random
+import tempfile
 from collections import namedtuple
+from pathlib import Path
+from typing import Generator, Optional, Union
 
 import docker
+import jinja2
 import pytest
+import requests
 from bioblend.galaxy import GalaxyInstance
+from docker.models.containers import Container
 
-from ephemeris.sleep import galaxy_wait
-
-
-# It needs to work well with dev. Alternatively we can pin this to 'master' or another stable branch.
-# Preferably a branch that updates with each stable release
-GALAXY_IMAGE = "bgruening/galaxy-stable:20.05"
+# Latest will be the latest stable version
+GALAXY_IMAGE = "lumc/galaxy-compose:latest"
 GALAXY_ADMIN_KEY = "fakekey"
 GALAXY_ADMIN_PASSWORD = "password"
 GALAXY_ADMIN_USER = "admin@galaxy.org"
 
+POSTGRES_IMAGE = "postgres:latest"  # Use the latest stable version
+NGINX_IMAGE = "nginx:stable"  # Use the latest stable version
+NGINX_TEMPLATE = Path(__file__).parent / "files" / "galaxy.nginx.j2"
+CREATE_GALAXY_USER_PY = Path(__file__).parent / "files" / "create_galaxy_user.py"
+
 client = docker.from_env()
 
-GalaxyContainer = namedtuple('GalaxyContainer', ['url', 'container', 'attributes', 'gi'])
+GalaxyContainer = namedtuple('GalaxyContainer',
+                             ['url', 'container', 'attributes', 'gi'])
+
+
+def template_to_temp(template: Union[os.PathLike, str], **kwargs):
+    template_string = Path(template).read_text()
+    templated = jinja2.Template(template_string)
+    with tempfile.NamedTemporaryFile('wt', delete=False) as temp_file:
+        temp_file.write(templated.render(**kwargs))
+    return temp_file.name
+
+
+def get_container_url(container, port: str) -> str:
+    container_attributes = client.containers.get(container.id).attrs
+    network_settings = container_attributes.get('NetworkSettings')
+    ports = network_settings.get('Ports')
+    host_port = ports.get(port)[0].get('HostPort')
+    return f"http://localhost:{host_port}"
+
+
+class GalaxyService:
+
+    def __init__(self,
+                 api_key: Optional[str] = None):
+        self.client = docker.from_env()
+
+        # Setup unique IDS for this network.
+        self.id = hex(random.randint(0, 2**32-1)).lstrip("0x")
+        self.network_name = f"galaxy_{self.id}"
+        self.postgres_name = f"ephemeris_db_{self.id}"
+        self.galaxy_web_name = f"galaxy_web_{self.id}"
+
+        # Create a fitting nginx config
+        self.nginx_config = template_to_temp(
+            NGINX_TEMPLATE, galaxy_web=self.galaxy_web_name)
+
+        # Setup containers. This is similar to using docker-compose, but
+        # docker-compose does NOT have a good python API.
+        self.network = self.client.networks.create(self.network_name)
+        self.postgres_container: Container = self.client.containers.run(
+            POSTGRES_IMAGE, detach=True, network=self.network_name,
+            name=self.postgres_name,
+            environment=dict(
+                POSTGRES_USER="dbuser",
+                POSTGRES_ROOT_PASSWORD="secret",
+                POSTGRES_PASSWORD="secret",
+                POSTGRES_DB="galaxydb"
+            ))
+        self.galaxy_container: Container = self.client.containers.run(
+            GALAXY_IMAGE, detach=True, network=self.network_name,
+            name=self.galaxy_web_name,
+            volumes=[f"{str(CREATE_GALAXY_USER_PY)}:/usr/local/bin/create_galaxy_user.py"],
+            environment=dict(
+                GALAXY_CONFIG_DATABASE_CONNECTION=f"postgresql://{self.postgres_name}/galaxydb?client_encoding=utf8",
+                GALAXY_CONFIG_ADMIN_USERS=GALAXY_ADMIN_USER,
+                GALAXY_CONFIG_JOB_CONFIG_FILE="job_conf.xml.sample_basic",  # Use basic job conf xml not kubernetes one.
+                PGUSER="dbuser",
+                PGPASSWORD="secret"
+            ),
+            ports={'8080/tcp': None}
+        )
+        self.nginx_container: Container = self.client.containers.run(
+            NGINX_IMAGE, detach=True, network=self.network_name,
+            ports={'80/tcp': None},
+            volumes=[f"{str(self.nginx_config)}:/etc/nginx/conf.d/default.conf"])
+
+        # Create admin api_key user in galaxy.
+        requests.get(get_container_url(self.galaxy_container, "8080/tcp"), timeout=5000)
+        self.api_key = api_key or GALAXY_ADMIN_KEY
+        result = self.galaxy_container.exec_run([
+            "/galaxy/venv/bin/python",
+            "/usr/local/bin/create_galaxy_user.py",
+            "--user", GALAXY_ADMIN_USER,
+            "--username", "admin",
+            "--key", self.api_key,
+            "--password", GALAXY_ADMIN_PASSWORD,
+            "-c", "/galaxy/config/galaxy.yml"
+        ], workdir="/galaxy/server")
+        if result.exit_code != 0:
+            raise RuntimeError(f"Error when creating API Key: {result.output}")
+
+        # Create api endpoint.
+        self.url = get_container_url(self.nginx_container, '80/tcp')
+        self.api = GalaxyInstance(self.url, self.api_key)
+
+    def stop(self, **kwargs):
+        self.galaxy_container.stop(**kwargs)
+        self.nginx_container.stop(**kwargs)
+        self.postgres_container.stop(**kwargs)
+
+    def remove(self, **kwargs):
+        self.galaxy_container.remove(**kwargs)
+        self.postgres_container.remove(**kwargs)
+        self.nginx_container.remove(**kwargs)
+        os.remove(self.nginx_config)
+
+    def restart_galaxy(self):
+        self.galaxy_container.restart()
 
 
 # Class scope is chosen here so we can group tests on the same galaxy in a class.
 @pytest.fixture(scope="class")
-def start_container(**kwargs):
+def galaxy_service(**kwargs) -> Generator[GalaxyService, None, None]:
     """Starts a docker container with the galaxy image. Returns a named tuple with the url, a GalaxyInstance object,
     the container attributes, and the container itself."""
     # We start a container from the galaxy image. We detach it. Port 80 is exposed to the host at a random port.
     # The random port is because we need mac compatibility. On GNU/linux a better option would be not to expose it
     # and use the internal ip address instead.
     # But alas, the trappings of a proprietary BSD kernel compel us to do ugly workarounds.
-    key = kwargs.get("api_key", GALAXY_ADMIN_KEY)
-    ensure_admin = kwargs.get("ensure_admin", True)
-
-    container = client.containers.run(GALAXY_IMAGE, detach=True, ports={'80/tcp': None}, **kwargs)
-    container_id = container.attrs.get('Id')
-    print(container_id)
-
-    # This seems weird as we also can just get container.attrs but for some reason
-    # the network settings are not loaded in container.attrs. With the get request
-    # these attributes are loaded
-    container_attributes = client.containers.get(container_id).attrs
-
-    # Venturing into deep nested dictionaries.
-    exposed_port = container_attributes.get('NetworkSettings').get('Ports').get('80/tcp')[0].get('HostPort')
-
-    container_url = "http://localhost:{0}".format(exposed_port)
-    assert key
-    ready = galaxy_wait(container_url,
-                        timeout=180,
-                        api_key=key,
-                        ensure_admin=ensure_admin)
-    if not ready:
-        raise Exception("Failed to wait on Galaxy to start.")
-    gi = GalaxyInstance(container_url, key=key)
-    yield GalaxyContainer(url=container_url,
-                          container=container,
-                          attributes=container_attributes,
-                          gi=gi)
-    container.remove(force=True)
+    galaxy_service = GalaxyService(**kwargs)
+    yield galaxy_service
+    galaxy_service.remove(force=True)

tests/files/create_galaxy_user.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+import argparse
+import sys
+
+sys.path.insert(1, '/galaxy/server')
+sys.path.insert(1, '/galaxy/server/lib')
+
+from galaxy.model import APIKeys, User  # noqa: E402, import must be after sys.path.insert
+from galaxy.model.mapping import init  # noqa: E402
+from galaxy.model.orm.scripts import get_config  # noqa: E402
+
+
+def add_user(sa_session, security_agent, email, password, key=None,
+             username="admin"):
+    """
+        Add Galaxy User.
+        From John https://gist.github.com/jmchilton/4475646
+    """
+    query = sa_session.query(User).filter_by(email=email)
+    if query.count() > 0:
+        return query.first()
+    else:
+        User.use_pbkdf2 = False
+        user = User(email)
+        user.username = username
+        user.set_password_cleartext(password)
+        sa_session.add(user)
+        sa_session.flush()
+
+        security_agent.create_private_user_role(user)
+        if not user.default_permissions:
+            security_agent.user_set_default_permissions(user, history=True,
+                                                        dataset=True)
+
+        if key is not None:
+            api_key = APIKeys()
+            api_key.user_id = user.id
+            api_key.key = key
+            sa_session.add(api_key)
+            sa_session.flush()
+        return user
+
+
+if __name__ == "__main__":
+    db_url = get_config(sys.argv, use_argparse=False)['db_url']
+
+    parser = argparse.ArgumentParser(description='Create Galaxy Admin User.')
+
+    parser.add_argument("--user", required=True,
+                        help="Username, it should be an email address.")
+    parser.add_argument("--password", required=True,
+                        help="Password.")
+    parser.add_argument("--key", help="API-Key.")
+    parser.add_argument("--username", default="admin",
+                        help="The public username. Public names must be at "
+                             "least three characters in length and contain "
+                             "only lower-case letters, numbers, and the '-' "
+                             "character.")
+    parser.add_argument('args', nargs=argparse.REMAINDER)
+
+    options = parser.parse_args()
+
+    mapping = init('/tmp/', db_url)
+    sa_session = mapping.context
+    security_agent = mapping.security_agent
+
+    add_user(sa_session, security_agent, options.user, options.password,
+             key=options.key, username=options.username)

tests/files/galaxy.nginx.j2

@@ -0,0 +1,26 @@
+server {
+    listen 80 default_server;
+    listen [::]:80 default_server;
+    server_name _;
+
+    # compress responses whenever possible
+    gzip on;
+    gzip_http_version 1.1;
+    gzip_vary on;
+    gzip_comp_level 6;
+    gzip_proxied any;
+    gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
+    gzip_buffers 16 8k;
+
+    location / {
+        include uwsgi_params;
+        # allow up to 3 minutes for Galaxy to respond to slow requests before timing out
+        uwsgi_read_timeout 180;
+        # maximum file upload size
+        uwsgi_pass uwsgi://{{galaxy_web}}:3031;
+        uwsgi_param Host $host;
+        uwsgi_param X-Real-IP $remote_addr;
+        uwsgi_param X-Forwarded-For $proxy_add_x_forwarded_for;
+        uwsgi_param X-Forwarded-Proto $http_x_forwarded_proto;
+    }
+}

tests/test_run_data_managers.py

@@ -7,7 +7,7 @@
 
 import pytest
 import yaml
-from docker_for_galaxy import GALAXY_ADMIN_KEY, GALAXY_ADMIN_PASSWORD, GALAXY_ADMIN_USER, start_container  # noqa: F401 prevent unused error
+from docker_for_galaxy import GALAXY_ADMIN_KEY, GALAXY_ADMIN_PASSWORD, GALAXY_ADMIN_USER, galaxy_service  # noqa: F401 prevent unused error
 
 from ephemeris import run_data_managers
 from ephemeris.run_data_managers import DataManagers
@@ -20,9 +20,8 @@
 class TestRunDataManagers(object):
     """This class tests run-data-managers"""
 
-    def test_install_data_managers(self, start_container):  # noqa: F811 Prevent start_container unused warning.
+    def test_install_data_managers(self, galaxy_service):  # noqa: F811
         """Install the data_managers on galaxy"""
-        container = start_container
         data_managers = [
             dict(name="data_manager_fetch_genome_dbkeys_all_fasta",
                  owner="devteam"),
@@ -31,16 +30,15 @@ def test_install_data_managers(self, start_container):  # noqa: F811 Prevent sta
             dict(name="data_manager_bwa_mem_index_builder",
                  owner="devteam")
         ]
-        irm = InstallRepositoryManager(container.gi)
+        irm = InstallRepositoryManager(galaxy_service.api)
         irm.install_repositories(data_managers)
         # Galaxy is restarted because otherwise data tables are not watched.
-        container.container.exec_run("supervisorctl restart galaxy:")
+        galaxy_service.restart_galaxy()
         time.sleep(10)  # give time for the services to go down
-        galaxy_wait(container.url)
+        galaxy_wait(galaxy_service.url)
 
-    def test_run_data_managers(self, start_container):  # noqa: F811 Prevent start_container unused warning.
+    def test_run_data_managers(self, galaxy_service):  # noqa: F811 Prevent start_container unused warning.
         """Tests an installation using the command line"""
-        container = start_container
         argv = ["run-data-managers"]
         if AUTH_BY == "user":
             argv.extend([
@@ -50,24 +48,22 @@ def test_run_data_managers(self, start_container):  # noqa: F811 Prevent start_c
         else:
             argv.extend(["-a", GALAXY_ADMIN_KEY])
         argv.extend([
-            "-g", container.url,
+            "-g", galaxy_service.url,
             "--config", "tests/run_data_managers.yaml.test"
         ])
         sys.argv = argv
         run_data_managers.main()
 
-    def test_run_data_managers_installation_skipped(self, start_container):  # noqa: F811 Prevent start_container unused warning.
-        container = start_container
+    def test_run_data_managers_installation_skipped(self, galaxy_service):  # noqa: F811
         with open("tests/run_data_managers.yaml.test") as config_file:
             configuration = yaml.safe_load(config_file)
-        dm = DataManagers(container.gi, configuration)
+        dm = DataManagers(galaxy_service.api, configuration)
         install_results = dm.run()
         assert (len(install_results.successful_jobs) == 0)
         assert (len(install_results.skipped_jobs) == 9)
         assert (len(install_results.failed_jobs) == 0)
 
-    def test_run_data_managers_installation_fail(self, start_container, caplog):  # noqa: F811 Prevent start_container unused warning.
-        container = start_container
+    def test_run_data_managers_installation_fail(self, galaxy_service, caplog):  # noqa: F811
         configuration = dict(
             data_managers=[
                 dict(
@@ -89,7 +85,7 @@ def test_run_data_managers_installation_fail(self, start_container, caplog):  #
                 )
             ]
         )
-        dm = DataManagers(container.gi, configuration)
+        dm = DataManagers(galaxy_service.api, configuration)
         with pytest.raises(RuntimeError):
             dm.run()
         assert ("HTTP Error 404" in caplog.text)