Merge pull request #939 from cortex-lab/deploy

Deployment improvements for Alyx servers

Author: oliche

.flake8

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       postgres:
-        image: postgres:16
+        image: postgres:17
         env:
           POSTGRES_PASSWORD: postgres
           POSTGRES_USER: postgres
@@ -34,7 +34,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: '3.12'
+          python-version: '3.13'
 
       - name: Checkout branch
         uses: actions/checkout@v3
@@ -46,49 +46,27 @@ jobs:
           pip install coverage coveralls pyarrow pandas  # for one_cache tests
         env:
           PIP_USE_MIRRORS: true
-
+      - name: Ruff
+        run: |
+          ruff check ./alyx
       - name: Run tests
         run: |
-          sudo touch /var/log/alyx.log; sudo chmod 666 /var/log/alyx.log
-          sudo touch /var/log/alyx_json.log; sudo chmod 666 /var/log/alyx_json.log
+          sudo mkdir /var/log/alyx
+          sudo touch /var/log/alyx/django.log
+          sudo chmod 666 /var/log/alyx/django.log
           cd alyx
-          cp alyx/settings_ci.py alyx/settings.py
+          cp ./alyx/environment_template.env ./alyx/.env
+          cp ../deploy/docker/settings-deploy.py  alyx/settings.py
+          cp ../deploy/docker/settings_lab-deploy.py  alyx/settings_lab.py
           python manage.py collectstatic --noinput --link
           coverage run manage.py test -n
           coveralls --service=github
         env:
           DJANGO_SETTINGS_MODULE: alyx.settings
           PYTHONPATH: $HOME/builds/cortexlab/alyx
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Flake
-        run: |
-          cd alyx
-          flake8 .
-
-      - name: Generate new requirements_frozen.txt if needed
-        # Only runs when branch pushed to directly OR when a PR is merged
-        if: ${{ github.event_name == 'push' }}
-        run: |
-          pip freeze > requirements_frozen_temp.txt
-          if diff requirements_frozen.txt requirements_frozen_temp.txt > /dev/null; then
-            echo "requirements_frozen.txt unchanged"
-            rm requirements_frozen_temp.txt
-            echo "GIT_PUSH_NEEDED=false" >> "$GITHUB_ENV"
-          else
-            echo "requirements_frozen.txt is different, git push needed"
-            mv requirements_frozen_temp.txt requirements_frozen.txt
-            echo "GIT_PUSH_NEEDED=true" >> "$GITHUB_ENV"
-          fi
-
-      - name: Setup git/commit/push for requirements_frozen.txt if needed
-        # Only runs when requirements_frozen.txt is updated
-        if: env.GIT_PUSH_NEEDED == 'true'
-        run: |
-          git config user.name github-actions
-          git config user.email github-actions@github.com
-          git add requirements_frozen.txt
-          git commit -m "GitHub Actions generated requirements_frozen.txt"
-          git push
+          APACHE_LOG_DIR: /var/log/alyx
+          POSTGRES_HOST: localhost
 
       # Docker steps only run when master branch pushed to directly OR when a PR is merged
       - name: Set Docker conditional value if needed

.github/workflows/main.yml

@@ -1,3 +1,23 @@
+# environment files for deployments
+.env*
+# all of the iblreports sym links
+alyx/data/management/commands/*ibl*
+alyx/data/management/commands/sync_patcher.py
+alyx/ibl_reports
+alyx/templates/ibl_reports
+# setting files
+alyx/alyx/settings_secret.py
+alyx/alyx/settings_lab.py
+alyx/alyx/settings.py
+alyx/static/*/*
+scripts/deployment_examples/docker-apache/settings*
+# this is a custom lab settings to be provided by the user rebuilding the head docker
+deploy/docker/settings_lab-user.py
+
+# package if installed
+alyx/alyx.egg-info
+
+# other
 *.pyc
 .DS_Store
 *~
@@ -24,14 +44,6 @@ build
 .cache/
 alyx/alyx_full.sql.gz
 .vscode/
-alyx/data/management/commands/*ibl*
-alyx/ibl_reports
-alyx/templates/ibl_reports
-alyx/alyx/settings_secret.py
-alyx/alyx/settings_lab.py
-alyx/alyx/settings.py
-alyx/static/*/*
-scripts/deployment_examples/docker-apache/settings*
 
 alyx/.idea/
 

.gitignore

@@ -7,13 +7,9 @@ version: 2
 
 # Set the version of Python and other tools you might need
 build:
-  os: ubuntu-20.04
+  os: ubuntu-24.04
   tools:
-    python: "3.9"
-    # You can also specify other tool versions:
-    # nodejs: "16"
-    # rust: "1.55"
-    # golang: "1.17"
+    python: "3.12"
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:

.readthedocs.yaml

@@ -5,30 +5,45 @@
 
 Database for experimental neuroscience laboratories
 
-Documentation: [Installation and getting started](http://alyx.readthedocs.io), [Alyx usage guide](https://docs.google.com/document/d/1cx3XLZiZRh3lUzhhR_p65BggEqTKpXHUDkUDagvf9Kc/edit?usp=sharing)
+[Documentation](https://alyx.readthedocs.io)
+
+[Alyx Experimenter Guide](https://docs.google.com/document/d/1cx3XLZiZRh3lUzhhR_p65BggEqTKpXHUDkUDagvf9Kc/edit?usp=sharing)
 
 
 ## Installation
-Alyx has only been tested on Ubuntu (16.04 / 18.04 / 20.04), the latest is recommended. There are no guarantees that 
-this setup will work on other systems. Assumptions made are that you have sudo permissions under an account named
 
-[The getting started](docs/gettingstarted.md) section of the documentation details the steps for 
+[The getting started](https://alyx.readthedocs.io/en/latest/gettingstarted.html) section of the documentation details the steps for 
 -   installing the Python/Django environment
--   serving a local database
+-   running the app with a development server
 -   registering local data
 -   accessing local data using [ONE](https://one.internationalbrainlab.org)
 
+More complex deployments scenarios using web servers and Cloud applications are in the [how-to guides section of the documtentaiton](docs/how-to-guides)
+
 ## Contribution
 
 * Development happens on the **dev** branch
 * alyx is sync with the **master** branch
 * alyx-dev is sync with the **dev** branch
-* Migrations files are provided by the repository
-* Continuous integration is setup, to run tests locally:
+* Migrations files are always provided by the repository
+
+### Running tests
+
+Continuous integration is setup. But before submitting a PR or commit,the tests can run locally.
     - `./manage.py test -n` test without migrations (faster)
     - `./manage.py test` test with migrations (recommended if model changes)
-    - NB: When running tests ensure `DEBUG = True` in the settings.py file (specifically `SECURE_SSL_REDIRECT = True` causes REST tests to fail)
 
-```shell
-./manage.py test -n
+### Documentation contribution guide
+
+#### Dependencies
+```
+pip install myst-parser sphinx_rtd_theme sphinx-autobuild
 ```
+
+#### Build documentation locally
+
+From the root of the repository.
+````shell
+sphinx-autobuild -b html ./docs ./docs/_build/ --port 8700
+````
+https://www.scan.co.uk/products/3xs-evolve-studio-pro-intel-core-ultra-9-285k-64gb-ddr5-16gb-nvidia-rtx-5070-ti-super-1tb-ssd-2tb-ss

README.md

@@ -1,6 +1,6 @@
 import base64
 import json
-import structlog
+import logging
 
 from django import forms
 from django.conf import settings
@@ -27,7 +27,7 @@
 from experiments.models import ProbeInsertion, FOV
 from jobs.models import Task
 
-logger = structlog.get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 # Filters

alyx/actions/admin.py

@@ -1,7 +1,7 @@
 from datetime import timedelta
 from math import inf
 
-import structlog
+import logging
 from one.alf.spec import QC
 
 from django.conf import settings
@@ -13,7 +13,7 @@
 from misc.models import Lab, LabLocation, LabMember, Note
 
 
-logger = structlog.get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 def _default_water_type():
@@ -387,7 +387,7 @@ class OtherAction(BaseAction):
 
 def delay_since_last_notification(notification_type, title, subject):
     """Return the delay since the last notification corresponding to the given
-    type, title, subject, in seconds, wheter it was actually sent or not."""
+    type, title, subject, in seconds, whether it was actually sent or not."""
     last_notif = Notification.objects.filter(
         notification_type=notification_type,
         title=title,

alyx/actions/models.py

@@ -1,12 +1,12 @@
-import structlog
+import logging
 from textwrap import dedent
 
 from django.utils import timezone
 
 from actions.models import create_notification
 
 
-logger = structlog.get_logger(__name__)
+logger = logging.getLogger(__name__)
 
 
 def responsible_user_changed(subject, old_user, new_user):

alyx/actions/notifications.py

@@ -219,8 +219,8 @@ def test_notif_water_1(self):
     def test_notif_water_2(self):
         # If the last water admin was on June 3 at 12pm, the notification
         # should be created after June 4 at 11am.
-        l = ((9, False), (10, False), (11, True), (12, True))
-        for (h, r) in l:
+        teupeul = ((9, False), (10, False), (11, True), (12, True))
+        for (h, r) in teupeul:
             date = timezone.datetime(2018, 6, 4, h, 0, 0)
             check_water_administration(self.subject, date=date)
             notif = Notification.objects.last()

alyx/actions/tests.py

@@ -548,15 +548,15 @@ def test_list_retrieve_water_restrictions(self):
     def test_list_retrieve_lab_locations(self):
         # test list
         url = reverse("location-list")
-        l = self.ar(self.client.get(url))
-        self.assertTrue(len(l) > 0)
-        self.assertEqual(set(l[0].keys()), {"name", "json", "lab"})
+        reponse = self.ar(self.client.get(url))
+        self.assertTrue(len(reponse) > 0)
+        self.assertEqual(set(reponse[0].keys()), {"name", "json", "lab"})
         # test detail
-        url = reverse("location-detail", args=[l[0]["name"]])
+        url = reverse("location-detail", args=[reponse[0]["name"]])
         d = self.ar(self.client.get(url))
-        self.assertEqual(d, l[0])
+        self.assertEqual(d, reponse[0])
         # test patch
-        url = reverse("location-detail", args=[l[0]["name"]])
+        url = reverse("location-detail", args=[reponse[0]["name"]])
         json_dict = {
             "string": "look at me! I'm a Json field",
             "integer": 15,