Add option to hide industrial visits from staff

gfrn · gfrn · commit 5a216b6a5365 · 2025-04-24T15:07:37.000+01:00
diff --git a/Dockerfile b/Dockerfile
@@ -3,7 +3,7 @@
 # The devcontainer should use the build target and run as root with podman
 # or docker with user namespaces.
 #
-FROM docker.io/library/python:3.13.2-slim-bullseye as build
+FROM docker.io/library/python:3.13.2-slim-bookworm as build
 
 # Add any system dependencies for the developer/build environment here
 RUN apt-get update && apt-get upgrade -y && \
@@ -40,7 +40,7 @@ RUN pip install --upgrade pip && \
     # and replace with a comment to avoid a zero length asset upload later
     sed -i '/file:/s/^/# Requirements for /' lockfiles/requirements.txt
 
-FROM docker.io/library/python:3.13.2-slim-bullseye as runtime
+FROM docker.io/library/python:3.13.2-slim-bookworm as runtime
 
 # Add apt-get system dependecies for runtime here if needed
 RUN apt-get update && apt-get install -y libmariadb-dev
diff --git a/README.rst b/README.rst
@@ -35,6 +35,7 @@ As for the JSON configuration file, details are as follows:
     - port: Mesasge queue port 
     - queue: Queue name
     - vhost: Message queue virtual host
+    - consumer_queue: Queue to consume messages from (for notifications)
 - ispyb
     - pool: Connection pool size
     - overflow: Connection pool overflow max size
@@ -43,6 +44,7 @@ As for the JSON configuration file, details are as follows:
     - smtp_server: SMTP server host
     - smtp_port: SMTP port to be used for emailing reports
     - active_session_cutoff: Time, in weeks, to be used as a threshold for determining if a session is active or not, following the end of the first processing pipeline. For example, a session would be considered inactive if there were no new actions for the past 5 weeks, by default.
+    - users_only_on_industrial: Hide industrial session details from staff, only display it to users/staff directly listed as part of that visit
 
 ==========
 Deployment
diff --git a/config.json b/config.json
@@ -29,7 +29,8 @@
     "host": "127.0.0.1",
     "port": 5672,
     "queue": "processing_recipe",
-    "vhost": "/"
+    "vhost": "/",
+    "consumer_queue": "pato_notification"
   },
   "ispyb": { "pool": 3, "overflow": 6 },
   "facility": {
@@ -38,7 +39,8 @@
     "smtp_server": "smtp.ac.uk",
     "active_session_cutoff": 5,
     "sample_handling_url": "https://ebic-sample-handling.diamond.ac.uk",
-    "frontend_url": "http://localhost:8080"
+    "frontend_url": "http://localhost:8080",
+    "users_only_on_industrial": false
   },
   "enable_cors": false
 }
diff --git a/database/data.sql b/database/data.sql
diff --git a/pyproject.toml b/pyproject.toml
@@ -16,16 +16,16 @@ description = "PATO's backend"
 dependencies = [
     "python-multipart~=0.0.9",
     "pika~=1.3.2",
-    "SQLAlchemy~=2.0.34",
-    "fastapi~=0.115.5",
-    "uvicorn[standard]~=0.30.1",
+    "SQLAlchemy~=2.0.40",
+    "fastapi~=0.115.12",
+    "uvicorn[standard]~=0.34.2",
     "requests~=2.32.3",
-    "mysqlclient~=2.2.4",
+    "mysqlclient~=2.2.7",
     "mysql-connector-python~=8.2.0",
-    "pydantic[email]~=2.9.2",
-    "fpdf2~=2.8.2",
+    "pydantic[email]~=2.11.3",
+    "fpdf2~=2.8.3",
     "types-requests",
-    "lims-utils~=0.4.3"
+    "lims-utils~=0.4.4"
 ]
 dynamic = ["version"]
 license.file = "LICENSE"
@@ -100,7 +100,7 @@ allowlist_externals =
     # sphinx-build
     # sphinx-autobuild
 commands =
-    pytest: pytest tests --cov=pato --cov-report term --cov-report xml:cov.xml {posargs} -s
+    pytest: pytest tests --cov=pato --cov-report term --cov-report xml:cov.xml {posargs}
     mypy: mypy src tests {posargs}
     pre-commit: pre-commit run --all-files {posargs}
 setenv = 
diff --git a/src/pato/auth/micro.py b/src/pato/auth/micro.py
@@ -42,11 +42,12 @@ def __init__(
         super().__init__(**user)
 
 
-def _check_perms(data_id: str | int, endpoint: str, token: str, options: str = ""):
+def _check_perms(data_id: str | int, endpoint: str, token: str, options: dict[str, str | int | bool] = {}):
     response = requests.get(
         "".join(
-            [Config.auth.endpoint, "permission/", endpoint, "/", str(data_id), options]
+            [Config.auth.endpoint, "permission/", endpoint, "/", str(data_id)]
         ),
+        params={**options, "usersOnly": True},
         headers={"Authorization": f"Bearer {token}"},
     )
 
@@ -117,7 +118,7 @@ def data_collection_group(
                 detail="Data collection group not found",
             )
 
-        _check_perms(session_id, "session", token.credentials, "?useId=true")
+        _check_perms(session_id, "session", token.credentials, {"useId": True})
 
         return groupId
 
@@ -140,7 +141,7 @@ def grid_square(
                 detail="Grid square not found",
             )
 
-        _check_perms(session_id, "session", token.credentials, "?useId=true")
+        _check_perms(session_id, "session", token.credentials, {"useId": True})
 
         return gridSquareId
 
@@ -164,6 +165,6 @@ def foil_holes(
                 detail="Foil hole not found",
             )
 
-        _check_perms(session_id, "session", token.credentials, "?useId=true")
+        _check_perms(session_id, "session", token.credentials, {"useId": True})
 
         return foilHoleId
diff --git a/src/pato/crud/autoproc.py b/src/pato/crud/autoproc.py
@@ -27,7 +27,7 @@
     ItemList,
     TomogramResponse,
 )
-from ..utils.database import db, paginate, unravel
+from ..utils.database import db, unravel
 from ..utils.generic import validate_path
 
 
@@ -57,7 +57,7 @@ def get_motion_correction(limit: int, page: int, autoProcId: int) -> Paged[FullM
         .order_by(MotionCorrection.imageNumber)
     )
 
-    return paginate(query, limit, page)
+    return db.paginate(query, limit, page)
 
 
 def get_ctf(autoProcId: int):
@@ -94,7 +94,7 @@ def get_particle_picker(autoProcId: int, filterNull: bool, limit: int, page: int
         .order_by(MotionCorrection.imageNumber)
     )
 
-    return paginate(query, limit, page)
+    return db.paginate(query, limit, page)
 
 
 _2d_ordering: dict[str, UnaryExpression] = {
@@ -162,7 +162,7 @@ def get_classification(
     if excludeUnselected:
         query = query.filter(ParticleClassification.selected != 0)
 
-    return paginate(query, limit, page)
+    return db.paginate(query, limit, page)
 
 
 @validate_path
diff --git a/src/pato/crud/collections.py b/src/pato/crud/collections.py
@@ -41,7 +41,7 @@
     ProcessingJobResponse,
     TomogramFullResponse,
 )
-from ..utils.database import db, paginate
+from ..utils.database import db
 from ..utils.generic import check_session_active, parse_count
 from ..utils.pika import PikaPublisher
 
@@ -102,7 +102,7 @@ def get_tomograms(
         .order_by(ProcessingJob.processingJobId.desc())
     )
 
-    return paginate(query, limit, page, slow_count=False)
+    return db.paginate(query, limit, page, slow_count=False)
 
 
 def get_motion_correction(limit: int, page: int, collectionId: int) -> Paged[FullMovie]:
@@ -115,7 +115,7 @@ def get_motion_correction(limit: int, page: int, collectionId: int) -> Paged[Ful
         .group_by(Movie.movieId)
     )
 
-    return paginate(query, limit, page, slow_count=True)
+    return db.paginate(query, limit, page, slow_count=True)
 
 
 def initiate_reprocessing_tomogram(
@@ -310,7 +310,7 @@ def get_processing_jobs(
         )
     )
 
-    return paginate(query, limit, page, slow_count=False)
+    return db.paginate(query, limit, page, slow_count=False)
 
 
 def _with_ctf_joins(query: Select, collectionId: int):
diff --git a/src/pato/crud/proposals.py b/src/pato/crud/proposals.py
@@ -6,7 +6,7 @@
 
 from ..models.response import ProposalResponse
 from ..utils.auth import check_proposal
-from ..utils.database import paginate
+from ..utils.database import db
 
 
 def get_proposals(
@@ -31,4 +31,4 @@ def get_proposals(
         .order_by(BLSession.startDate.desc())
     )
 
-    return paginate(check_proposal(query, user), limit, page)
+    return db.paginate(check_proposal(query, user), limit, page)
diff --git a/src/pato/crud/sessions.py b/src/pato/crud/sessions.py
@@ -14,7 +14,7 @@
 from ..models.response import SessionAllowsReprocessing, SessionResponse
 from ..utils.auth import check_session
 from ..utils.config import Config
-from ..utils.database import db, paginate, unravel
+from ..utils.database import db, unravel
 from ..utils.generic import ProposalReference, check_session_active, parse_proposal
 
 HDF5_FILE_SIGNATURE = b"\x89\x48\x44\x46\x0d\x0a\x1a\x0a"
@@ -137,7 +137,7 @@ def get_sessions(
             isouter=True,
         ).group_by(BLSession.visit_number, BLSession.proposalId)
 
-    return paginate(query, limit, page)
+    return db.paginate(query, limit, page)
 
 
 def get_session(proposalReference: ProposalReference):
diff --git a/src/pato/crud/tomograms.py b/src/pato/crud/tomograms.py
@@ -19,7 +19,7 @@
     FullMovieWithTilt,
     ItemList,
 )
-from ..utils.database import db, paginate
+from ..utils.database import db
 from ..utils.generic import MovieType, parse_json_file, validate_path
 
 
@@ -123,7 +123,7 @@ def get_motion_correction(
         .order_by(TiltImageAlignment.refinedTiltAngle)
     )
 
-    motion = dict(paginate(query, limit, page))
+    motion = dict(db.paginate(query, limit, page))
 
     raw_total = db.session.execute(
         select(
diff --git a/src/pato/utils/auth.py b/src/pato/utils/auth.py
@@ -1,6 +1,6 @@
 from lims_utils.auth import GenericUser
 from lims_utils.tables import BLSession, Proposal, ProposalHasPerson, SessionHasPerson
-from sqlalchemy import Select, or_
+from sqlalchemy import Select, and_, or_
 
 from ..auth import is_admin
 from .config import Config
@@ -15,23 +15,55 @@ def get_allowed_beamlines(perms: list[int]) -> set[str]:
 
     return allowed_beamlines
 
+def _get_staff_perms(user: GenericUser):
+    """Get the appropriate set of beamline-related permissions based on an user's user groups
+
+    Args:
+        user: User to check permissions for
+
+    Returns:
+        Additional SQL conditions based on user permissions, or None if user has no extra permissions"""
+    allowed_beamlines = get_allowed_beamlines(user.permissions)
+    user_is_admin = is_admin(user.permissions)
+
+    if allowed_beamlines or user_is_admin:
+        if Config.facility.users_only_on_industrial:
+            # We only want to restrict industrial visits to listed users
+            return and_(
+                Proposal.proposalCode.not_in(["ic", "il", "in", "sw"]),
+                or_(
+                    BLSession.beamLineName.in_(allowed_beamlines),
+                    user_is_admin,
+                ),
+            )
+
+        return BLSession.beamLineName.in_(allowed_beamlines)
 
 def check_session(query: Select, user: GenericUser):
-    if is_admin(user.permissions):
+    """Include filters in provided query to ensure that only sessions the user has permission to view
+    are returned.
+
+    Args:
+        query: Original query
+        user: User to check against
+
+    Returns
+        Modified query"""
+    if not Config.facility.users_only_on_industrial and is_admin(user.permissions):
         return query
 
-    allowed_beamlines = get_allowed_beamlines(user.permissions)
     or_expressions = [SessionHasPerson.personId == user.id]
 
-    if allowed_beamlines:
-        or_expressions.append(BLSession.beamLineName.in_(allowed_beamlines))
+    staff_perms = _get_staff_perms(user)
+    if staff_perms is not None:
+        or_expressions.append(staff_perms)
 
     return (
         query.distinct()
         .join(
             SessionHasPerson,
             SessionHasPerson.sessionId == BLSession.sessionId,
-            isouter=(len(allowed_beamlines) > 0),
+            isouter=(staff_perms is not None),
         )
         .filter(
             or_(*or_expressions),
@@ -40,21 +72,30 @@ def check_session(query: Select, user: GenericUser):
 
 
 def check_proposal(query: Select, user: GenericUser):
-    if is_admin(user.permissions):
+    """Include filters in provided query to ensure that only proposals the user has permission to view
+    are returned.
+
+    Args:
+        query: Original query
+        user: User to check against
+
+    Returns
+        Modified query"""
+    if not Config.facility.users_only_on_industrial and is_admin(user.permissions):
         return query
 
-    allowed_beamlines = get_allowed_beamlines(user.permissions)
     or_expressions = [ProposalHasPerson.personId == user.id]
 
-    if allowed_beamlines:
-        or_expressions.append(BLSession.beamLineName.in_(allowed_beamlines))
+    staff_perms = _get_staff_perms(user)
+    if staff_perms is not None:
+        or_expressions.append(staff_perms)
 
     return (
         query.distinct()
         .join(
             ProposalHasPerson,
             ProposalHasPerson.proposalId == Proposal.proposalId,
-            isouter=(len(allowed_beamlines) > 0),
+            isouter=(staff_perms is not None),
         )
         .filter(
             or_(*or_expressions),
diff --git a/src/pato/utils/config.py b/src/pato/utils/config.py
@@ -23,6 +23,7 @@ class Facility:
     active_session_cutoff: int = 5
     sample_handling_url: str = "https://ebic-sample-handling.diamond.ac.uk"
     frontend_url: str = "http://localhost/"
+    users_only_on_industrial: bool = False
 
 
 @dataclass
diff --git a/src/pato/utils/database.py b/src/pato/utils/database.py
diff --git a/tests/proposals/test_proposal.py b/tests/proposals/test_proposal.py
diff --git a/tests/sessions/test_sessions.py b/tests/sessions/test_sessions.py
diff --git a/tests/tomograms/test_motion.py b/tests/tomograms/test_motion.py
diff --git a/tests/users.py b/tests/users.py

Original file line number	Diff line number	Diff line change
`@@ -41,7 +41,7 @@`
`41`	`41`	`ProcessingJobResponse,`
`42`	`42`	`TomogramFullResponse,`
`43`	`43`	`)`
`44`		`-from ..utils.database import db, paginate`
	`44`	`+from ..utils.database import db`
`45`	`45`	`from ..utils.generic import check_session_active, parse_count`
`46`	`46`	`from ..utils.pika import PikaPublisher`
`47`	`47`
`@@ -102,7 +102,7 @@ def get_tomograms(`
`102`	`102`	`.order_by(ProcessingJob.processingJobId.desc())`
`103`	`103`	`)`
`104`	`104`
`105`		`- return paginate(query, limit, page, slow_count=False)`
	`105`	`+ return db.paginate(query, limit, page, slow_count=False)`
`106`	`106`
`107`	`107`
`108`	`108`	`def get_motion_correction(limit: int, page: int, collectionId: int) -> Paged[FullMovie]:`
`@@ -115,7 +115,7 @@ def get_motion_correction(limit: int, page: int, collectionId: int) -> Paged[Ful`
`115`	`115`	`.group_by(Movie.movieId)`
`116`	`116`	`)`
`117`	`117`
`118`		`- return paginate(query, limit, page, slow_count=True)`
	`118`	`+ return db.paginate(query, limit, page, slow_count=True)`
`119`	`119`
`120`	`120`
`121`	`121`	`def initiate_reprocessing_tomogram(`
`@@ -310,7 +310,7 @@ def get_processing_jobs(`
`310`	`310`	`)`
`311`	`311`	`)`
`312`	`312`
`313`		`- return paginate(query, limit, page, slow_count=False)`
	`313`	`+ return db.paginate(query, limit, page, slow_count=False)`
`314`	`314`
`315`	`315`
`316`	`316`	`def _with_ctf_joins(query: Select, collectionId: int):`