diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 37cbcd6b14..34a3aebf7d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,4 +42,4 @@ repos: hooks: - id: codespell args: ["--ignore-words", ".codespellignore"] - files: fractal_server/ + files: ^(fractal_server/|docs/) diff --git a/docs/configuration.md b/docs/configuration.md index 130c2a39eb..f8df3e6258 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,12 +1,46 @@ -To configure the Fractal Server one must define some environment variables. +To configure the Fractal Server one must define some configuration variables. + Some of them are required, and the server will not start unless they are set. Some are optional and sensible defaults are provided. -> The required variables are the following -> -> ``` -> JWT_SECRET_KEY -> POSTGRES_DB -> ``` +## How to set configuration + +There are two possibilities for setting the configuration variables: + +- defining them as *environment variables*, in the same environment as Fractal Server: + ```sh + export VAR1=value1 + export VAR2=value2 + ``` +- write them inside a file called `.fractal_server.env`, located in your current working directory: + ``` + VAR1=value1 + VAR2=value2 + ``` + +If the same variable is defined both in the environment and inside the env file, the value defined in the environment takes priority. + +Once the variables have been defined in one of these ways, they will be read automatically by the Fractal Server during the start-up phase. + +## How to get configuration + +Admins can retrieve the settings values through the appropriate endpoints [`GET /api/settings/`](../openapi.md/#operations-default-view_settings_api_settings_app__get). + +## Minimal working example + +This is a minimal working example of a `.fractal_server.env`, with all the required configuration variables: +```txt +POSTGRES_DB=fractal_test +JWT_SECRET_KEY=jwt_secret_key +``` +These are the only variables that must be mandatorily set by the user. All others, if not specified, will assume their default value. + + + +--- -::: fractal_server.config +::: fractal_server.config._main +::: fractal_server.config._database +::: fractal_server.config._data +::: fractal_server.config._email +::: fractal_server.config._oauth diff --git a/docs/development.md b/docs/development.md index 41ed4f2d77..b7f1fe3a3d 100644 --- a/docs/development.md +++ b/docs/development.md @@ -30,7 +30,7 @@ poetry install --with dev --with docs ``` will initialise a Python virtual environment and install Fractal Server and all its dependencies, including optional dependencies. Note that to run commands from within this environment you should prepend them with `poetry run` (as in `poetry run fractalctl set-db`). -## Update database schema during development +## Update database schema Whenever the models are modified (either in [`app/models`](reference/fractal_server/app/models/index.md) or in diff --git a/docs/install_and_deploy.md b/docs/install_and_deploy.md index ef0d6dfafe..5d8162b82d 100644 --- a/docs/install_and_deploy.md +++ b/docs/install_and_deploy.md @@ -1,62 +1,115 @@ -Fractal Server is the core ingredient of more deployments of the [Fractal framework](https://fractal-analytics-platform.github.io/), which also includes several other Fractal components (e.g. a web client) and also relies on external resources being availble (e.g. a PostgreSQL database and a SLURM cluster). +Fractal Server is the core ingredient of more deployments of the [Fractal framework](https://fractal-analytics-platform.github.io/), which also includes several other Fractal components (e.g. a web client) and also relies on external resources being available (e.g. a PostgreSQL database and a SLURM cluster). -Here we do not describe the full procedure for a full-fledged Fractal deployment in detail. Some examples of typical deployment are available as container-based demos at https://github.com/fractal-analytics-platform/fractal-containers/tree/main/examples. +Here we just describe the basic procedure for a local deployment. +Any production deployments will require greater attention and detail. +Some examples of typical deployment are available as container-based demos at https://github.com/fractal-analytics-platform/fractal-containers/tree/main/examples. -## How to install -> ⚠️ The minimum supported Python version for fractal-server is 3.11. +## Prerequisites + +The following will assume that: + +- You are using a Python version greater or equal than 3.11 +- You are working within an isolated Python environment, for example with `venv`: + ``` + python3 -m venv venv + venv/bin/activate + ``` + +- You have configured the required environment variables (see [configuration page](configuration.md)). + + If you choose to declare the environment variables using the `.fractal_server.env` file, that file must be placed in the current working directory; + +- You have access to a dedicated Postgres database (see the [database page](internals/database_interface.md)). + +## How to install -Fractal Server is hosted on [the PyPI index](https://pypi.org/project/fractal-server), and it can be installed with `pip` via +Fractal Server is hosted on [PyPI](https://pypi.org/project/fractal-server), and can be installed with `pip`: ``` pip install fractal-server ``` -For details on how to install Fractal Server in a development environment, see the [Development](development.md) page. +Fractal Server is also available as a [Conda package](https://anaconda.org/conda-forge/fractal-server), but the PyPI version is the recommended one. + +For details on how to install Fractal Server in a development environment see the [Development](development.md) page. + ## How to deploy -Here we describe the basic steps for running Fractal Server. +Installing `fractal-server` will automatically install `fractalctl`, its companion command-line utility that provides the basic commands for deploying Fractal Server. -### 1. Set up configuration variables +### 1. Set up the database -For this command to work properly, a set of variables need to be specified, -either as enviromnent variables or in a file like `.fractal_server.env`. -An example of such file is +> This only requires [DatabaseSettings](http://localhost:8000/configuration/#fractal_server.config._database.DatabaseSettings). + +Use the command ``` -JWT_SECRET_KEY=XXX -FRACTAL_RUNNER_BACKEND=local -POSTGRES_DB=fractal-database-name +fractalctl set-db ``` +to apply the schema migrations to the database. -> ⚠️ **`JWT_SECRET_KEY=XXX` must be replaced with a more secure string, that -> should not be disclosed.** ⚠️ +### 2. Initialize the database -More details (including default values) are available in the [Configuration](configuration.md) page. +> This only requires [DatabaseSettings](http://localhost:8000/configuration/#fractal_server.config._database.DatabaseSettings) and, possibly, [FRACTAL_DEFAULT_GROUP_NAME](http://localhost:8000/configuration/#fractal_server.config._main.Settings.FRACTAL_DEFAULT_GROUP_NAME). + +With the command +``` +fractalctl init-db-data +``` +you can do multiple things, depending on the environment variables set and on the flag provided: + - create the default user group, by settings `FRACTAL_DEFAULT_GROUP_NAME=All`; + - create the first admin user, by providing the `--admin-*` flags; + - create the first couple resource/profile and associate users to them, providing `--resource` and `--profile`. -### 2. Set up the database -After creating a PostgreSQL database for `fractal-server`, and after setting the proper `fractal-server` configuration variables (see the [database page](internals/database_interface.md)), the command + +**Help message** ``` -fractalctl set-db +usage: fractalctl init-db-data [-h] [--resource RESOURCE] [--profile PROFILE] [--admin-email ADMIN_EMAIL] [--admin-pwd ADMIN_PWD] [--admin-project-dir ADMIN_PROJECT_DIR] + +Populate database with initial data. + +options: + -h, --help show this help message and exit + --resource RESOURCE Either `default` or path to the JSON file of the first resource. + --profile PROFILE Either `default` or path to the JSON file of the first profile. + --admin-email ADMIN_EMAIL + Email of the first admin user. + --admin-pwd ADMIN_PWD + Password for the first admin user. + --admin-project-dir ADMIN_PROJECT_DIR + Project_dir for the first admin user. + ``` -applies the appropriate schema migrations. ### 3. Start the server -In the environment where Fractal Server is installed, you can run it via [`gunicorn`](https://gunicorn.org) with a command like +Finally, we use the command +``` +fractalctl start +``` +to start the server using [Uvicorn](https://uvicorn.dev/). + +**Help message** + ``` -gunicorn fractal_server.main:app \ - --workers 2 \ - --bind "0.0.0.0:8000" \ - --access-logfile - \ - --error-logfile - \ - --worker-class fractal_server.gunicorn_fractal.FractalWorker \ - --logger-class fractal_server.gunicorn_fractal.FractalGunicornLogger +usage: fractalctl start [-h] [--host HOST] [-p PORT] [--reload] + +Start the server (with uvicorn) + +options: + -h, --help show this help message and exit + --host HOST bind socket to this host (default: 127.0.0.1) + -p PORT, --port PORT bind socket to this port (default: 8000) + --reload enable auto-reload ``` + +### 4. Test the server + To verify that the server is up, you can use the `/api/alive/` endpoint - as in -```console -$ curl http://localhost:8000/api/alive/ -{"alive":true,"version":"2.15.6"} +``` +curl http://localhost:8000/api/alive/ +{"alive":true,"version":"2.17.0"} ``` diff --git a/docs/internals/logs.md b/docs/internals/logs.md index 855d4aa4f0..98bde6473a 100644 --- a/docs/internals/logs.md +++ b/docs/internals/logs.md @@ -15,8 +15,8 @@ a logger created with are defined as follows: * The minimum logging level for logs to appear in the console is set by - [`FRACTAL_LOGGING_LEVEL`](../configuration.md/#fractal_server.config.Settings.FRACTAL_LOGGING_LEVEL); -* The `FileHandler` logger handlers are alwasy set at the `DEBUG` level, that + [`FRACTAL_LOGGING_LEVEL`](../configuration/#fractal_server.config._main.Settings.FRACTAL_LOGGING_LEVEL); +* The `FileHandler` logger handlers are always set at the `DEBUG` level, that is, they write all log records. This means that the `FRACTAL_LOGGING_LEVEL` offers a quick way to switch to diff --git a/docs/internals/runners/slurm.md b/docs/internals/runners/slurm.md index 337fdca032..696f59e610 100644 --- a/docs/internals/runners/slurm.md +++ b/docs/internals/runners/slurm.md @@ -77,7 +77,7 @@ The logic for handling the batching parameters (that is, how many tasks can be c ### `sudo`-based impersonation -The user who runs `fractal-server` must have sufficient priviliges for running some commands via `sudo -u` to impersonate other users of the SLURM cluster without any password. The required commands include `sbatch`, `scancel`, `cat`, `ls` and `mkdir`. An example of how to achieve this is to add this block to the `sudoers` file: +The user who runs `fractal-server` must have sufficient privileges for running some commands via `sudo -u` to impersonate other users of the SLURM cluster without any password. The required commands include `sbatch`, `scancel`, `cat`, `ls` and `mkdir`. An example of how to achieve this is to add this block to the `sudoers` file: ``` Runas_Alias FRACTAL_IMPERSONATE_USERS = fractal, user1, user2, user3 Cmnd_Alias FRACTAL_CMD = /usr/bin/sbatch, /usr/bin/scancel, /usr/bin/cat, /usr/bin/ls, /usr/bin/mkdir diff --git a/docs/internals/users.md b/docs/internals/users.md index 534a8147ad..3214ade446 100644 --- a/docs/internals/users.md +++ b/docs/internals/users.md @@ -20,10 +20,6 @@ A Fractal user corresponds to an instance of the [`UserOAuth`](../reference/frac Most attributes are [the default ones from `fastapi-users`](https://fastapi-users.github.io/fastapi-users/latest/configuration/schemas/). In the startup phase, `fractal-server` creates a default user, who also has the superuser privileges that are necessary for managing other users. -The credentials for this user are defined via the environment variables -[`FRACTAL_ADMIN_DEFAULT_EMAIL`](../configuration.md/#fractal_server.config.Settings.FRACTAL_DEFAULT_ADMIN_EMAIL) (default: `admin@example.org`) -and -[`FRACTAL_ADMIN_DEFAULT_PASSWORD`](../configuration.md/#fractal_server.config.Settings.FRACTAL_DEFAULT_ADMIN_PASSWORD) (default: `1234`). > **⚠️ You should always modify the password of the default user after it's created;** > this can be done with API calls to the `PATCH /auth/users/{id}` endpoint of the [`fractal-server` API](../openapi.md), e.g. through the `curl` command or the [Fractal command-line client](https://fractal-analytics-platform.github.io/fractal-client/reference/fractal/user/#user-edit). @@ -208,7 +204,7 @@ We can now review how `fractal-server` handles these steps: - **Steps 9 → 10**
* The callback endpoint uses the Access Token to obtain the user's email address and an account identifier from the Resource Server (which, depending on the client, may coincide with the Authorization Server). -After that, the callback endpoint performs some extra operations, which are not stricly part of the `OAuth2` protocol: +After that, the callback endpoint performs some extra operations, which are not strictly part of the `OAuth2` protocol: - It checks that `state` is still valid; - If a user with the given email address doesn't already exist, it creates one with a random password; @@ -355,7 +351,7 @@ The endpoints to manage users can be found under the route `/auth/`. On top of t 🔐 *Restricted to superusers*. -New users can be registred by a superuser at [`/auth/register`](https://fastapi-users.github.io/fastapi-users/latest/usage/routes/#register-router): +New users can be registered by a superuser at [`/auth/register`](https://fastapi-users.github.io/fastapi-users/latest/usage/routes/#register-router): ```console $ curl \ @@ -383,7 +379,7 @@ Here we provided `email` and `password`, which are the only required fields of ` 🔐 *Restricted to superusers*. -The route `/auth/userlist` returns the list of all registred users: +The route `/auth/userlist` returns the list of all registered users: ```console $ curl \ @@ -417,7 +413,7 @@ $ curl \ ### GET `/auth/current-user/` -At `/auth/current-user/`, authenticated users can get informations about themself: +At `/auth/current-user/`, authenticated users can get information about themself: ``` curl \ -X GET \ diff --git a/fractal_server/__main__.py b/fractal_server/__main__.py index e4f4f2e35e..09e7a37429 100644 --- a/fractal_server/__main__.py +++ b/fractal_server/__main__.py @@ -49,9 +49,7 @@ # fractalctl set-db set_db_parser = subparsers.add_parser( "set-db", - description=( - "Initialise/upgrade database schemas and create first group&user." - ), + description="Initialise/upgrade database schemas.", ) # fractalctl init-db-data diff --git a/fractal_server/app/schemas/v2/accounting.py b/fractal_server/app/schemas/v2/accounting.py index 8fa51b71dd..bafae79968 100644 --- a/fractal_server/app/schemas/v2/accounting.py +++ b/fractal_server/app/schemas/v2/accounting.py @@ -6,6 +6,17 @@ class AccountingRecordRead(BaseModel): + """ + AccountingRecordRead + + Attributes: + id: + user_id: + timestamp: + num_tasks: + num_new_images: + """ + id: int user_id: int timestamp: AwareDatetime diff --git a/fractal_server/app/schemas/v2/dataset.py b/fractal_server/app/schemas/v2/dataset.py index 46a11fbe03..8c40873368 100644 --- a/fractal_server/app/schemas/v2/dataset.py +++ b/fractal_server/app/schemas/v2/dataset.py @@ -8,22 +8,38 @@ from fractal_server.app.schemas.v2.project import ProjectReadV2 from fractal_server.images import SingleImage -from fractal_server.types import AttributeFilters from fractal_server.types import NonEmptyStr from fractal_server.types import ZarrDirStr class DatasetCreateV2(BaseModel): + """ + DatasetCreateV2 + + Attributes: + name: + zarr_dir: + """ + model_config = ConfigDict(extra="forbid") name: NonEmptyStr - zarr_dir: ZarrDirStr | None = None - attribute_filters: AttributeFilters = Field(default_factory=dict) - class DatasetReadV2(BaseModel): + """ + DatasetReadV2 + + Attributes: + id: + name: + project_id: + project: + timestamp_created: + zarr_dir: + """ + id: int name: str @@ -40,6 +56,14 @@ def serialize_datetime(v: datetime) -> str: class DatasetUpdateV2(BaseModel): + """ + DatasetUpdateV2 + + Attributes: + name: + zarr_dir: + """ + model_config = ConfigDict(extra="forbid") name: NonEmptyStr = None diff --git a/fractal_server/app/schemas/v2/manifest.py b/fractal_server/app/schemas/v2/manifest.py index 38fe89d25d..da0e668b24 100644 --- a/fractal_server/app/schemas/v2/manifest.py +++ b/fractal_server/app/schemas/v2/manifest.py @@ -119,9 +119,9 @@ class ManifestV2(BaseModel): manifests as the schema evolves. This is for instance used by Fractal to determine which subclass of the present base class needs be used to read and validate the input. - task_list : list[TaskManifestType] + task_list: The list of tasks, represented as specified by subclasses of the - _TaskManifestBase (a.k.a. TaskManifestType) + `_TaskManifestBase` (a.k.a. `TaskManifestType`) has_args_schemas: `True` if the manifest includes JSON Schemas for the arguments of each task. diff --git a/fractal_server/app/schemas/v2/resource.py b/fractal_server/app/schemas/v2/resource.py index 52f87f4e89..31f1473a40 100644 --- a/fractal_server/app/schemas/v2/resource.py +++ b/fractal_server/app/schemas/v2/resource.py @@ -21,6 +21,15 @@ class ResourceType(StrEnum): + """ + Enum for the possible resource types. + + Attributes: + SLURM_SUDO: + SLURM_SSH: + LOCAL: + """ + SLURM_SUDO = "slurm_sudo" SLURM_SSH = "slurm_ssh" LOCAL = "local" @@ -68,6 +77,22 @@ def _pixi_slurm_config(self) -> Self: class ValidResourceLocal(_ValidResourceBase): + """ + Valid local resource + + Attributes: + type: + name: + tasks_python_config: + tasks_pixi_config: + tasks_local_dir: + jobs_local_dir: + jobs_runner_config: + jobs_poll_interval: + jobs_slurm_python_worker: + host: + """ + type: Literal[ResourceType.LOCAL] jobs_runner_config: JobRunnerConfigLocal jobs_slurm_python_worker: None = None @@ -75,6 +100,22 @@ class ValidResourceLocal(_ValidResourceBase): class ValidResourceSlurmSudo(_ValidResourceBase): + """ + Valid SLURM-sudo resource. + + Attributes: + type: + name: + tasks_python_config: + tasks_pixi_config: + tasks_local_dir: + jobs_local_dir: + jobs_runner_config: + jobs_poll_interval: + jobs_slurm_python_worker: + host: + """ + type: Literal[ResourceType.SLURM_SUDO] jobs_slurm_python_worker: AbsolutePathStr jobs_runner_config: JobRunnerConfigSLURM @@ -82,6 +123,22 @@ class ValidResourceSlurmSudo(_ValidResourceBase): class ValidResourceSlurmSSH(_ValidResourceBase): + """ + Valid SLURM-SSH resource. + + Attributes: + type: + name: + tasks_python_config: + tasks_pixi_config: + tasks_local_dir: + jobs_local_dir: + jobs_runner_config: + jobs_poll_interval: + jobs_slurm_python_worker: + host: + """ + type: Literal[ResourceType.SLURM_SSH] host: NonEmptyStr jobs_slurm_python_worker: AbsolutePathStr @@ -101,6 +158,9 @@ def get_discriminator_value(v: Any) -> str: | Annotated[ValidResourceSlurmSSH, Tag(ResourceType.SLURM_SSH)], Discriminator(get_discriminator_value), ] +""" +ResourceCreate +""" class ResourceRead(BaseModel): @@ -131,6 +191,9 @@ def cast_serialize_resource(_data: ResourceCreate) -> dict[str, Any]: We use `@validate_call` because `ResourceCreate` is a `Union` type and it cannot be instantiated directly. + Args: + _data: + Return: Serialized version of a valid resource object. """ diff --git a/fractal_server/config/_data.py b/fractal_server/config/_data.py index 0372b97f7d..be2c560d22 100644 --- a/fractal_server/config/_data.py +++ b/fractal_server/config/_data.py @@ -18,37 +18,45 @@ class DataAuthScheme(StrEnum): class DataSettings(BaseSettings): """ Settings for the `fractal-data` integration. - """ - model_config = SettingsConfigDict(**SETTINGS_CONFIG_DICT) + Attributes: + FRACTAL_DATA_AUTH_SCHEME: + Defines how the list of allowed viewer paths is built. - FRACTAL_DATA_AUTH_SCHEME: DataAuthScheme = "none" - """ - Defines how the list of allowed viewer paths is built. + This variable affects the + `GET /auth/current-user/allowed-viewer-paths/` response, which is + then consumed by + [fractal-data](https://github.com/fractal-analytics-platform/fractal-data). - This variable affects the `GET /auth/current-user/allowed-viewer-paths/` - response, which is then consumed by - [fractal-data](https://github.com/fractal-analytics-platform/fractal-data). + Options: + + FRACTAL_DATA_BASE_FOLDER: + Base path to Zarr files that will be served by + fractal-vizarr-viewer. + This variable is required and used only when + `FRACTAL_DATA_AUTHORIZATION_SCHEME` is set to `"users-folders"`. + """ - Options: + model_config = SettingsConfigDict(**SETTINGS_CONFIG_DICT) - - "viewer-paths": The list of allowed viewer paths will include the user's - `project_dir` along with any path defined in user groups' `viewer_paths` - attributes. - - "users-folders": The list will consist of the user's `project_dir` and a - user-specific folder. The user folder is constructed by concatenating - the base folder `FRACTAL_DATA_BASE_FOLDER` with the user's profile - `username`. - - "none": An empty list will be returned, indicating no access to - viewer paths. Useful when vizarr viewer is not used. - """ + FRACTAL_DATA_AUTH_SCHEME: DataAuthScheme = "none" FRACTAL_DATA_BASE_FOLDER: AbsolutePathStr | None = None - """ - Base path to Zarr files that will be served by fractal-vizarr-viewer; - This variable is required and used only when - FRACTAL_DATA_AUTHORIZATION_SCHEME is set to "users-folders". - """ @model_validator(mode="after") def check(self: Self) -> Self: diff --git a/fractal_server/config/_database.py b/fractal_server/config/_database.py index 8defe65f92..c3f15f743a 100644 --- a/fractal_server/config/_database.py +++ b/fractal_server/config/_database.py @@ -11,34 +11,30 @@ class DatabaseSettings(BaseSettings): """ Minimal set of configurations needed for operating on the database (e.g for schema migrations). + + Attributes: + DB_ECHO: + If `True`, make database operations verbose. + POSTGRES_USER: + User to use when connecting to the PostgreSQL database. + POSTGRES_PASSWORD: + Password to use when connecting to the PostgreSQL database. + POSTGRES_HOST: + URL to the PostgreSQL server or path to a UNIX domain socket. + POSTGRES_PORT: + Port number to use when connecting to the PostgreSQL server. + POSTGRES_DB: + Name of the PostgreSQL database to connect to. """ model_config = SettingsConfigDict(**SETTINGS_CONFIG_DICT) DB_ECHO: bool = False - """ - If `True`, make database operations verbose. - """ POSTGRES_USER: NonEmptyStr | None = None - """ - User to use when connecting to the PostgreSQL database. - """ POSTGRES_PASSWORD: SecretStr | None = None - """ - Password to use when connecting to the PostgreSQL database. - """ POSTGRES_HOST: NonEmptyStr | None = "localhost" - """ - URL to the PostgreSQL server or path to a UNIX domain socket. - """ POSTGRES_PORT: NonEmptyStr | None = "5432" - """ - Port number to use when connecting to the PostgreSQL server. - """ POSTGRES_DB: NonEmptyStr - """ - Name of the PostgreSQL database to connect to. - """ @property def DATABASE_URL(self) -> URL: diff --git a/fractal_server/config/_email.py b/fractal_server/config/_email.py index 4db019b608..b1ff67a1ae 100644 --- a/fractal_server/config/_email.py +++ b/fractal_server/config/_email.py @@ -17,14 +17,14 @@ class PublicEmailSettings(BaseModel): Schema for `EmailSettings.public`, namely the ready-to-use settings. Attributes: - sender: Sender email address - recipients: List of recipients email address - smtp_server: SMTP server address - port: SMTP server port - password: Sender password - instance_name: Name of SMTP server instance - use_starttls: Whether to use the security protocol - use_login: Whether to use login + sender: Sender email address. + recipients: List of recipients email address. + smtp_server: SMTP server address. + port: SMTP server port. + password: Sender password. + instance_name: Name of SMTP server instance. + use_starttls: Whether to use the security protocol. + use_login: Whether to use login. """ sender: EmailStr @@ -40,50 +40,42 @@ class PublicEmailSettings(BaseModel): class EmailSettings(BaseSettings): """ Class with settings for email-sending feature. + + Attributes: + FRACTAL_EMAIL_SENDER: + Address of the OAuth-signup email sender. + FRACTAL_EMAIL_PASSWORD: + Password for the OAuth-signup email sender. + FRACTAL_EMAIL_SMTP_SERVER: + SMTP server for the OAuth-signup emails. + FRACTAL_EMAIL_SMTP_PORT: + SMTP server port for the OAuth-signup emails. + FRACTAL_EMAIL_INSTANCE_NAME: + Fractal instance name, to be included in the OAuth-signup emails. + FRACTAL_EMAIL_RECIPIENTS: + Comma-separated list of recipients of the OAuth-signup emails. + FRACTAL_EMAIL_USE_STARTTLS: + Whether to use StartTLS when using the SMTP server. + FRACTAL_EMAIL_USE_LOGIN: + Whether to use login when using the SMTP server. + If 'true', FRACTAL_EMAIL_PASSWORD must be provided. """ model_config = SettingsConfigDict(**SETTINGS_CONFIG_DICT) FRACTAL_EMAIL_SENDER: EmailStr | None = None - """ - Address of the OAuth-signup email sender. - """ FRACTAL_EMAIL_PASSWORD: SecretStr | None = None - """ - Password for the OAuth-signup email sender. - """ FRACTAL_EMAIL_SMTP_SERVER: str | None = None - """ - SMTP server for the OAuth-signup emails. - """ FRACTAL_EMAIL_SMTP_PORT: int | None = None - """ - SMTP server port for the OAuth-signup emails. - """ FRACTAL_EMAIL_INSTANCE_NAME: str | None = None - """ - Fractal instance name, to be included in the OAuth-signup emails. - """ FRACTAL_EMAIL_RECIPIENTS: str | None = None - """ - Comma-separated list of recipients of the OAuth-signup emails. - """ FRACTAL_EMAIL_USE_STARTTLS: Literal["true", "false"] = "true" - """ - Whether to use StartTLS when using the SMTP server. - Accepted values: 'true', 'false'. - """ FRACTAL_EMAIL_USE_LOGIN: Literal["true", "false"] = "true" - """ - Whether to use login when using the SMTP server. - If 'true', FRACTAL_EMAIL_PASSWORD must be provided. - Accepted values: 'true', 'false'. - """ public: PublicEmailSettings | None = None """ - The validated field which is actually used in `fractal-server - (automatically populated upon creation). + The validated field which is actually used in `fractal-server`, + automatically populated upon creation. """ @model_validator(mode="after") diff --git a/fractal_server/config/_main.py b/fractal_server/config/_main.py index dfb60c6e51..4ffaef308f 100644 --- a/fractal_server/config/_main.py +++ b/fractal_server/config/_main.py @@ -11,68 +11,49 @@ class Settings(BaseSettings): """ - Contains all the configuration variables for Fractal Server - - The attributes of this class are set from the environment. + Contains the general configuration variables for Fractal Server. + + Attributes: + JWT_EXPIRE_SECONDS: + JWT token lifetime, in seconds. + JWT_SECRET_KEY: + JWT secret.
+ ⚠️ Set this variable to a secure string, and do not disclose it. + COOKIE_EXPIRE_SECONDS: + Cookie token lifetime, in seconds. + FRACTAL_RUNNER_BACKEND: + Select which runner backend to use. + FRACTAL_LOGGING_LEVEL: + Logging-level threshold for logging + Only logs of with this level (or higher) will appear in the console + logs. + FRACTAL_API_MAX_JOB_LIST_LENGTH: + Number of ids that can be stored in the `jobsV2` attribute of + `app.state`. + FRACTAL_GRACEFUL_SHUTDOWN_TIME: + Waiting time for the shutdown phase of executors, in seconds. + FRACTAL_HELP_URL: + The URL of an instance-specific Fractal help page. + FRACTAL_DEFAULT_GROUP_NAME: + Name of the default user group. + + If set to `"All"`, then the user group with that name is a special + user group (e.g. it cannot be deleted, and new users are + automatically added to it). If set to `None` (the default value), + then user groups are all equivalent, independently on their name. """ model_config = SettingsConfigDict(**SETTINGS_CONFIG_DICT) JWT_EXPIRE_SECONDS: int = 180 - """ - JWT token lifetime, in seconds. - """ - JWT_SECRET_KEY: SecretStr - """ - JWT secret - - ⚠️ **IMPORTANT**: set this variable to a secure string, and do not disclose - it. - """ - COOKIE_EXPIRE_SECONDS: int = 86400 - """ - Cookie token lifetime, in seconds. - """ - # Note: we do not use ResourceType here to avoid circular imports FRACTAL_RUNNER_BACKEND: Literal[ "local", "slurm_ssh", "slurm_sudo" ] = "local" - """ - Select which runner backend to use. - """ - FRACTAL_LOGGING_LEVEL: int = logging.INFO - """ - Logging-level threshold for logging - - Only logs of with this level (or higher) will appear in the console logs. - """ - FRACTAL_API_MAX_JOB_LIST_LENGTH: int = 25 - """ - Number of ids that can be stored in the `jobsV2` attribute of - `app.state`. - """ - FRACTAL_GRACEFUL_SHUTDOWN_TIME: int = 30 - """ - Waiting time for the shutdown phase of executors - """ - FRACTAL_HELP_URL: HttpUrl | None = None - """ - The URL of an instance-specific Fractal help page. - """ - FRACTAL_DEFAULT_GROUP_NAME: Literal["All"] | None = None - """ - Name of the default user group. - - If set to `"All"`, then the user group with that name is a special user - group (e.g. it cannot be deleted, and new users are automatically added - to it). If set to `None` (the default value), then user groups are all - equivalent, independently on their name. - """ diff --git a/fractal_server/config/_oauth.py b/fractal_server/config/_oauth.py index fdb48485d8..09d067c4ca 100644 --- a/fractal_server/config/_oauth.py +++ b/fractal_server/config/_oauth.py @@ -15,6 +15,19 @@ class OAuthSettings(BaseSettings): """ Minimal set of configurations needed for operating on the database (e.g for schema migrations). + + Attributes: + OAUTH_CLIENT_NAME: The name of the client. + OAUTH_CLIENT_ID: ID of client. + OAUTH_CLIENT_SECRET: + Secret to authorise against the identity provider. + OAUTH_OIDC_CONFIG_ENDPOINT: + OpenID configuration endpoint, for autodiscovery of relevant + endpoints. + OAUTH_REDIRECT_URL: + String to be used as `redirect_url` argument in + `fastapi_users.get_oauth_router`, and then in + `httpx_oauth.integrations.fastapi.OAuth2AuthorizeCallback`. """ model_config = SettingsConfigDict(**SETTINGS_CONFIG_DICT) @@ -26,27 +39,10 @@ class OAuthSettings(BaseSettings): ] | None ) = None - """ - The name of the client. - """ OAUTH_CLIENT_ID: SecretStr | None = None - """ - ID of client. - """ OAUTH_CLIENT_SECRET: SecretStr | None = None - """ - Secret to authorise against the identity provider. - """ OAUTH_OIDC_CONFIG_ENDPOINT: SecretStr | None = None - """ - OpenID configuration endpoint, for autodiscovery of relevant endpoints. - """ OAUTH_REDIRECT_URL: str | None = None - """ - String to be used as `redirect_url` argument in - `fastapi_users.get_oauth_router`, and then in - `httpx_oauth.integrations.fastapi.OAuth2AuthorizeCallback` - """ @model_validator(mode="after") def check_configuration(self: Self) -> Self: diff --git a/fractal_server/json_schemas/manifest_v2.json b/fractal_server/json_schemas/manifest_v2.json index b60fbf3a02..68e068f2d6 100644 --- a/fractal_server/json_schemas/manifest_v2.json +++ b/fractal_server/json_schemas/manifest_v2.json @@ -168,7 +168,7 @@ "type": "string" } }, - "description": "Packages containing tasks are required to include a special file\n`__FRACTAL_MANIFEST__.json` in order to be discovered and used by Fractal.\n\nThis model class and the model classes it depends on provide the base\nschema to read, write and validate manifests.\n\nAttributes:\n manifest_version:\n A version string that provides indication for compatibility between\n manifests as the schema evolves. This is for instance used by\n Fractal to determine which subclass of the present base class needs\n be used to read and validate the input.\n task_list : list[TaskManifestType]\n The list of tasks, represented as specified by subclasses of the\n _TaskManifestBase (a.k.a. TaskManifestType)\n has_args_schemas:\n `True` if the manifest includes JSON Schemas for the arguments of\n each task.\n args_schema_version:\n Label of how `args_schema`s were generated (e.g. `pydantic_v1`).", + "description": "Packages containing tasks are required to include a special file\n`__FRACTAL_MANIFEST__.json` in order to be discovered and used by Fractal.\n\nThis model class and the model classes it depends on provide the base\nschema to read, write and validate manifests.\n\nAttributes:\n manifest_version:\n A version string that provides indication for compatibility between\n manifests as the schema evolves. This is for instance used by\n Fractal to determine which subclass of the present base class needs\n be used to read and validate the input.\n task_list:\n The list of tasks, represented as specified by subclasses of the\n `_TaskManifestBase` (a.k.a. `TaskManifestType`)\n has_args_schemas:\n `True` if the manifest includes JSON Schemas for the arguments of\n each task.\n args_schema_version:\n Label of how `args_schema`s were generated (e.g. `pydantic_v1`).", "properties": { "manifest_version": { "const": "2", diff --git a/fractal_server/runner/config/_slurm.py b/fractal_server/runner/config/_slurm.py index 07083b1f41..6d85badfb6 100644 --- a/fractal_server/runner/config/_slurm.py +++ b/fractal_server/runner/config/_slurm.py @@ -14,6 +14,9 @@ MemMBType = Annotated[ PositiveInt | NonEmptyStr, AfterValidator(slurm_mem_to_MB) ] +""" +Memory expressed in MB. +""" class _SlurmConfigSet(BaseModel): @@ -24,7 +27,6 @@ class _SlurmConfigSet(BaseModel): partition: cpus_per_task: mem: - See `_parse_mem_value` for details on allowed values. constraint: gres: time: @@ -32,6 +34,7 @@ class _SlurmConfigSet(BaseModel): nodelist: account: extra_lines: + gpus: """ model_config = ConfigDict(extra="forbid") @@ -58,9 +61,7 @@ class _BatchingConfigSet(BaseModel): target_cpus_per_job: max_cpus_per_job: target_mem_per_job: - (see `_parse_mem_value` for details on allowed values) max_mem_per_job: - (see `_parse_mem_value` for details on allowed values) target_num_jobs: max_num_jobs: """ diff --git a/fractal_server/runner/executors/slurm_common/get_slurm_config.py b/fractal_server/runner/executors/slurm_common/get_slurm_config.py index 35590c1012..19d6c6d6db 100644 --- a/fractal_server/runner/executors/slurm_common/get_slurm_config.py +++ b/fractal_server/runner/executors/slurm_common/get_slurm_config.py @@ -33,7 +33,6 @@ def _get_slurm_config_internal( which_type: Whether we should look at the non-parallel or parallel part of `wftask`. - tot_tasks: Not used here, only present as a common interface. Returns: A ready-to-use `SlurmConfig` object. @@ -162,6 +161,15 @@ def get_slurm_config( which_type: Literal["non_parallel", "parallel"], tot_tasks: int = 1, ) -> SlurmConfig: + """ + Get `SlurmConfig` object. + + Args: + shared_config: + wftask: + which_type: + tot_tasks: + """ config = _get_slurm_config_internal( shared_config=shared_config, wftask=wftask, diff --git a/fractal_server/runner/v2/submit_workflow.py b/fractal_server/runner/v2/submit_workflow.py index 562120e08d..6860eb728f 100644 --- a/fractal_server/runner/v2/submit_workflow.py +++ b/fractal_server/runner/v2/submit_workflow.py @@ -119,7 +119,7 @@ def submit_workflow( Computational resource to be used for this job (e.g. a SLURM cluster). profile: - Computational profile to be used for this job. + Computational profile to be used for this job. fractal_ssh: SSH object, for when `resource.type = "slurm_ssh"`. """ # Declare runner backend and set `process_workflow` function diff --git a/fractal_server/ssh/_fabric.py b/fractal_server/ssh/_fabric.py index 8fef467660..d732d81c8e 100644 --- a/fractal_server/ssh/_fabric.py +++ b/fractal_server/ssh/_fabric.py @@ -374,8 +374,6 @@ def run_command( Args: cmd: Command to be run allow_char: Forbidden chars to allow for this command - max_attempts: - base_interval: lock_timeout: Returns: diff --git a/fractal_server/tasks/v2/ssh/collect.py b/fractal_server/tasks/v2/ssh/collect.py index b934af3a0e..d8160e673f 100644 --- a/fractal_server/tasks/v2/ssh/collect.py +++ b/fractal_server/tasks/v2/ssh/collect.py @@ -55,8 +55,8 @@ def collect_ssh( Args: task_group_id: task_group_activity_id: - ssh_config: resource: + profile: wheel_file: """ diff --git a/fractal_server/tasks/v2/ssh/collect_pixi.py b/fractal_server/tasks/v2/ssh/collect_pixi.py index a8ba21f320..a897f9192a 100644 --- a/fractal_server/tasks/v2/ssh/collect_pixi.py +++ b/fractal_server/tasks/v2/ssh/collect_pixi.py @@ -52,8 +52,9 @@ def collect_ssh_pixi( Args: task_group_id: task_group_activity_id: - ssh_config: tar_gz_file: + resource: + profile: """ LOGGER_NAME = f"{__name__}.ID{task_group_activity_id}" diff --git a/fractal_server/tasks/v2/ssh/deactivate.py b/fractal_server/tasks/v2/ssh/deactivate.py index 65c53abadd..5738eb60e9 100644 --- a/fractal_server/tasks/v2/ssh/deactivate.py +++ b/fractal_server/tasks/v2/ssh/deactivate.py @@ -42,7 +42,8 @@ def deactivate_ssh( Args: task_group_id: task_group_activity_id: - ssh_config: + resource: + profile: """ LOGGER_NAME = f"{__name__}.ID{task_group_activity_id}" diff --git a/fractal_server/tasks/v2/ssh/delete.py b/fractal_server/tasks/v2/ssh/delete.py index 502c2daf85..d2ada31f42 100644 --- a/fractal_server/tasks/v2/ssh/delete.py +++ b/fractal_server/tasks/v2/ssh/delete.py @@ -35,7 +35,8 @@ def delete_ssh( Args: task_group_id: task_group_activity_id: - ssh_config: + resource: + profile: """ LOGGER_NAME = f"{__name__}.ID{task_group_activity_id}" diff --git a/fractal_server/tasks/v2/ssh/reactivate.py b/fractal_server/tasks/v2/ssh/reactivate.py index 243434364a..14882cf87f 100644 --- a/fractal_server/tasks/v2/ssh/reactivate.py +++ b/fractal_server/tasks/v2/ssh/reactivate.py @@ -40,9 +40,10 @@ def reactivate_ssh( handled. Args: - task_group_id: task_group_activity_id: - ssh_config: + task_group_id: + resource: + profile: """ LOGGER_NAME = f"{__name__}.ID{task_group_activity_id}" diff --git a/fractal_server/types/__init__.py b/fractal_server/types/__init__.py index d8983b8f77..75f2fc4a38 100644 --- a/fractal_server/types/__init__.py +++ b/fractal_server/types/__init__.py @@ -18,69 +18,143 @@ str, StringConstraints(min_length=1, strip_whitespace=True), ] +""" +A non-empty string, with no leading/trailing whitespaces. +""" + AbsolutePathStr = Annotated[ NonEmptyStr, AfterValidator(val_absolute_path), ] +""" +String representing an absolute path. +""" + + HttpUrlStr = Annotated[ NonEmptyStr, AfterValidator(val_http_url), ] +""" +String representing an URL. +""" + + ZarrUrlStr = Annotated[ NonEmptyStr, AfterValidator(normalize_url), ] +""" +String representing a zarr URL/path. +""" + + ZarrDirStr = Annotated[ NonEmptyStr, AfterValidator(normalize_url), ] +""" +String representing a `zarr_dir` path. +""" DictStrAny = Annotated[ dict[str, Any], AfterValidator(valdict_keys), ] +""" +Dictionary where keys are strings with no leading/trailing whitespaces. +""" + + DictStrStr = Annotated[ dict[str, NonEmptyStr], AfterValidator(valdict_keys), ] +""" +Dictionary where keys are strings with no leading/trailing whitespaces and +values are non-empty strings. +""" ListUniqueNonEmptyString = Annotated[ list[NonEmptyStr], AfterValidator(val_unique_list), ] +""" +List of unique non-empty-string items. +""" + + ListUniqueNonNegativeInt = Annotated[ list[NonNegativeInt], AfterValidator(val_unique_list), ] +""" +List of unique non-negative-integer items. +""" + + ListUniqueAbsolutePathStr = Annotated[ list[AbsolutePathStr], AfterValidator(val_unique_list), ] +""" +List of unique absolute-path-string items. +""" WorkflowTaskArgument = Annotated[ DictStrAny, AfterValidator(validate_wft_args), ] +""" +Dictionary with no keys from a given forbid-list. +""" ImageAttributeValue = Union[int, float, str, bool] +""" +Possible values for image attributes. +""" + ImageAttributes = Annotated[ dict[str, ImageAttributeValue], AfterValidator(valdict_keys), ] +""" +Image-attributes dictionary. +""" + + ImageAttributesWithNone = Annotated[ dict[str, ImageAttributeValue | None], AfterValidator(valdict_keys), ] +""" +Image-attributes dictionary, including `None` attributes. +""" + + AttributeFilters = Annotated[ dict[str, list[ImageAttributeValue]], AfterValidator(validate_attribute_filters), ] +""" +Image-attributes filters. +""" + + TypeFilters = Annotated[ dict[str, bool], AfterValidator(valdict_keys), ] +""" +Image-type filters. +""" + + ImageTypes = Annotated[ dict[str, bool], AfterValidator(valdict_keys), ] +""" +Image types. +""" diff --git a/mkdocs.yml b/mkdocs.yml index 34ef099c56..9734f8fe80 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -74,13 +74,17 @@ plugins: default_handler: python handlers: python: + inventories: + - https://docs.pydantic.dev/latest/objects.inv options: show_signature_annotations: false - docstring_section_style: "table" + docstring_section_style: "spacy" docstring_style: "google" show_source: true filters: [] show_root_full_path: false + signature_crossrefs: true + show_attribute_values: true - render_swagger extra_css: diff --git a/poetry.lock b/poetry.lock index fe4ce2bd54..6ec1969cd1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -546,14 +546,14 @@ files = [ [[package]] name = "click" -version = "8.3.0" +version = "8.2.1" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.10" groups = ["main", "dev", "docs"] files = [ - {file = "click-8.3.0-py3-none-any.whl", hash = "sha256:9b9f285302c6e3064f4330c05f05b81945b2a39544279343e6e7c5f27a9baddc"}, - {file = "click-8.3.0.tar.gz", hash = "sha256:e7b8232224eba16f4ebe410c25ced9f7875cb5f3263ffc93cc3e8da705e229c4"}, + {file = "click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b"}, + {file = "click-8.2.1.tar.gz", hash = "sha256:27c491cc05d968d271d5a1db13e3b5a184636d9d930f148c50b038f0d0646202"}, ] [package.dependencies] @@ -2885,4 +2885,4 @@ dev = ["pytest", "setuptools"] [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.14" -content-hash = "1c81a5c33f55ae5b939734657cd2adacab48b2d9824e0caf9ac0e96a29e08c75" +content-hash = "f7e55202fcf381b4f3a5ecde17b79e8b440c14a28193556be26fbf4c33792a90" diff --git a/pyproject.toml b/pyproject.toml index 6fcc279567..a1cb2809f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,7 @@ mkdocs-literate-nav="0.6.2" mkdocs-section-index="0.3.10" mkdocs-render-swagger-plugin="0.1.2" pyyaml=">=6.0.0" +click = "8.2.1" [tool.pytest.ini_options] asyncio_mode = "auto"