CI Fixups

.devcontainer/Dockerfile

@@ -1,7 +1,11 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
-FROM mcr.microsoft.com/vscode/devcontainers/base AS base
+# TODO: FIXME: Temporarily pin to bookworm until latest (trixie) base image is fully supported.
+# See Also:
+# - <https://github.com/microsoft/MLOS/issues/1002>
+# - .devcontainer/devcontainer.json
+FROM mcr.microsoft.com/vscode/devcontainers/base:bookworm AS base
 
 # Add some additional packages for the devcontainer terminal environment.
 USER root
@@ -80,6 +84,8 @@ RUN echo "Setup miniconda" \
     && /opt/conda/bin/conda init \
     && /opt/conda/bin/conda config --set channel_priority strict \
     && /opt/conda/bin/conda info \
+    && /opt/conda/bin/conda config --show \
+    && /opt/conda/bin/conda tos accept --override-channels --channel defaults \
     && /opt/conda/bin/conda update -v -y -n base -c conda-forge -c defaults --all \
     && /opt/conda/bin/conda list -n base \
     && /opt/conda/bin/conda install -v -y -n base -c conda-forge -c defaults conda-libmamba-solver \

.devcontainer/devcontainer.json

@@ -31,7 +31,13 @@
     "features": {
         "ghcr.io/devcontainers/features/azure-cli:1": {},
         //"ghcr.io/devcontainers/features/conda:1": {},
-        "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {},
+        "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {
+            // Note: this may be required when moving to Debian Trixie base image.
+            // See Also:
+            // - <https://github.com/microsoft/MLOS/issues/1002>
+            // - .devcontainer/Dockerfile
+            //"moby": false
+        },
         "ghcr.io/devcontainers/features/github-cli:1": {},
         "ghcr.io/stuartleeks/dev-container-features/azure-cli-persistence:0": {},
         "ghcr.io/stuartleeks/dev-container-features/shell-history:0": {}

.devcontainer/scripts/run-devcontainer.sh

@@ -50,5 +50,6 @@ docker run -it --rm \
     --env http_proxy="${http_proxy:-}" \
     --env https_proxy="${https_proxy:-}" \
     --env no_proxy="${no_proxy:-}" \
+    --add-host host.docker.internal:host-gateway \
     mlos-devcontainer \
     $*

.github/workflows/devcontainer.yml

@@ -78,7 +78,7 @@ jobs:
         echo "NO_CACHE=true" >> $GITHUB_ENV
 
     - name: Run pytest with debug logging enabled for nightly runs
-      if: github.event_name == 'schedule'
+      #if: github.event_name == 'schedule'
       run: |
         echo "PYTEST_EXTRA_OPTIONS=--log-level=DEBUG" >> $GITHUB_ENV
 

.vscode/settings.json

@@ -176,5 +176,7 @@
     "githubPullRequests.experimental.chat": true,
     "github.copilot.chat.codesearch.enabled": true,
     "github.copilot.chat.copilotDebugCommand.enabled": true,
-    "github.copilot.chat.reviewSelection.enabled": true
+    "github.copilot.chat.reviewSelection.enabled": true,
+    "python-envs.defaultEnvManager": "ms-python.python:conda",
+    "python-envs.defaultPackageManager": "ms-python.python:conda"
 }

conda-envs/mlos-3.10.yml

@@ -20,6 +20,9 @@ dependencies:
   - python=3.10
   # See comments in mlos.yml.
   #- gcc_linux-64
+  # Install an SMAC requirement pre-compiled from conda-forge.
+  # See Also: https://github.com/microsoft/MLOS/issues/1001
+  - pyrfr>=0.9.0
   - pip:
     - bump2version
     - check-jsonschema

conda-envs/mlos-3.11.yml

@@ -20,6 +20,9 @@ dependencies:
   - python=3.11
   # See comments in mlos.yml.
   #- gcc_linux-64
+  # Install an SMAC requirement pre-compiled from conda-forge.
+  # See Also: https://github.com/microsoft/MLOS/issues/1001
+  - pyrfr>=0.9.0
   - pip:
     - bump2version
     - check-jsonschema

conda-envs/mlos-3.12.yml

@@ -22,6 +22,9 @@ dependencies:
   - python=3.12
   # See comments in mlos.yml.
   #- gcc_linux-64
+  # Install an SMAC requirement pre-compiled from conda-forge.
+  # See Also: https://github.com/microsoft/MLOS/issues/1001
+  - pyrfr>=0.9.0
   - pip:
     - bump2version
     - check-jsonschema

conda-envs/mlos-3.13.yml

@@ -22,6 +22,9 @@ dependencies:
   - python=3.13
   # See comments in mlos.yml.
   #- gcc_linux-64
+  # Install an SMAC requirement pre-compiled from conda-forge.
+  # See Also: https://github.com/microsoft/MLOS/issues/1001
+  - pyrfr>=0.9.0
   - pip:
     - bump2version
     - check-jsonschema

conda-envs/mlos.yml

@@ -18,6 +18,9 @@ dependencies:
   # FIXME: Temporarily avoid broken libpq that's missing client headers.
   - libpq<17.0
   - python
+  # Install an SMAC requirement pre-compiled from conda-forge.
+  # See Also: https://github.com/microsoft/MLOS/issues/1001
+  - pyrfr>=0.9.0
   - pip:
     - bump2version
     - check-jsonschema
@@ -28,7 +31,7 @@ dependencies:
     - pylint==3.3.7
     - tomlkit
     - mypy==1.15.0
-    - pyright==1.1.400
+    - pyright==1.1.405
     - pandas-stubs
     - types-beautifulsoup4
     - types-colorama

mlos_bench/mlos_bench/optimizers/mlos_core_optimizer.py

@@ -285,7 +285,7 @@ def bulk_register(
 
         if status is not None:
             # Select only the completed trials, set scores for failed trials to +inf.
-            df_status = pd.Series(status)
+            df_status = pd.Series(list(status), dtype=object)
             # TODO: Be more flexible with values used for failed trials (not just +inf).
             # Issue: https://github.com/microsoft/MLOS/issues/523
             df_scores[df_status != Status.SUCCEEDED] = float("inf")

mlos_bench/mlos_bench/storage/sql/alembic/versions/b61aa446e724_support_fractional_seconds_with_mysql.py

@@ -34,8 +34,8 @@ def _mysql_datetime(*, with_fsp: bool = False) -> mysql.DATETIME:
     See <https://github.com/sqlalchemy/sqlalchemy/pull/12164> for details.
     """
     if with_fsp:
-        return mysql.DATETIME(fsp=6)  # type: ignore[no-untyped-call]
-    return mysql.DATETIME()  # type: ignore[no-untyped-call]
+        return mysql.DATETIME(fsp=6)
+    return mysql.DATETIME()
 
 
 def upgrade() -> None:

mlos_bench/mlos_bench/storage/sql/schema.py

@@ -56,7 +56,7 @@ def _mysql_datetime_with_fsp() -> mysql.DATETIME:
     Split out to allow single mypy ignore.
     See <https://github.com/sqlalchemy/sqlalchemy/pull/12164> for details.
     """
-    return mysql.DATETIME(fsp=6)  # type: ignore[no-untyped-call]
+    return mysql.DATETIME(fsp=6)
 
 
 class _DDL:

mlos_bench/mlos_bench/tests/__init__.py

@@ -13,7 +13,7 @@
 import shutil
 import socket
 from datetime import tzinfo
-from logging import debug, warning
+from logging import warning
 from subprocess import run
 
 import pytest
@@ -52,8 +52,10 @@
     if cmd.returncode != 0 or not any(
         line for line in stdout.splitlines() if "Platform" in line and "linux" in line
     ):
-        debug("Docker is available but missing support for targeting linux platform.")
+        warning("Docker is available but missing buildx support for targeting linux platform.")
         DOCKER = None
+if not DOCKER:
+    warning("Docker is not available on this system. Some tests will be skipped.")
 requires_docker = pytest.mark.skipif(
     not DOCKER,
     reason="Docker with Linux support is not available on this system.",
@@ -62,6 +64,8 @@
 # A decorator for tests that require ssh.
 # Use with @requires_ssh above a test_...() function.
 SSH = shutil.which("ssh")
+if not SSH:
+    warning("ssh is not available on this system.  Some tests will be skipped.")
 requires_ssh = pytest.mark.skipif(not SSH, reason="ssh is not available on this system.")
 
 # A common seed to use to avoid tracking down race conditions and intermingling
@@ -112,7 +116,7 @@ def wait_docker_service_healthy(
     docker_services: DockerServices,
     project_name: str,
     service_name: str,
-    timeout: float = 30.0,
+    timeout: float = 60.0,
 ) -> None:
     """Wait until a docker service is healthy."""
     docker_services.wait_until_responsive(
@@ -126,7 +130,7 @@ def wait_docker_service_socket(docker_services: DockerServices, hostname: str, p
     """Wait until a docker service is ready."""
     docker_services.wait_until_responsive(
         check=lambda: check_socket(hostname, port),
-        timeout=30.0,
+        timeout=60.0,
         pause=0.5,
     )
 

mlos_bench/mlos_bench/tests/conftest.py

@@ -104,6 +104,8 @@ def docker_compose_file(pytestconfig: pytest.Config) -> list[str]:
         Path to the docker-compose file.
     """
     _ = pytestconfig  # unused
+    # TODO: move this closer to the necessary submodules so that different
+    # docker tests can run independently.
     return [
         os.path.join(os.path.dirname(__file__), "services", "remote", "ssh", "docker-compose.yml"),
         os.path.join(os.path.dirname(__file__), "storage", "sql", "docker-compose.yml"),

mlos_viz/mlos_viz/base.py

@@ -30,7 +30,7 @@ def _get_kwarg_defaults(target: Callable, **kwargs: Any) -> dict[str, Any]:
     Note: this only works with non-positional kwargs (e.g., those after a * arg).
     """
     target_kwargs = {}
-    for kword in target.__kwdefaults__:  # or {} # intentionally omitted for now
+    for kword in target.__kwdefaults__ or {}:
         if kword in kwargs:
             target_kwargs[kword] = kwargs[kword]
     return target_kwargs
@@ -59,6 +59,13 @@ def ignore_plotter_warnings() -> None:
         ),
     )
 
+    warnings.filterwarnings(
+        "ignore",
+        module="matplotlib",
+        category=DeprecationWarning,
+        message="'mode' parameter is deprecated and will be removed in Pillow 13",
+    )
+
 
 def _add_groupby_desc_column(
     results_df: pandas.DataFrame,

mlos_viz/mlos_viz/dabl.py

@@ -101,3 +101,10 @@ def ignore_plotter_warnings() -> None:
         module="dabl",
         message="The legendHandles attribute was deprecated in Matplotlib 3.7 and will be removed",
     )
+
+    warnings.filterwarnings(
+        "ignore",
+        module="matplotlib",
+        category=DeprecationWarning,
+        message="'mode' parameter is deprecated and will be removed in Pillow 13",
+    )