diff --git a/docs/how-tos/version-specific-prebuilt.rst b/docs/how-tos/version-specific-prebuilt.rst new file mode 100644 index 00000000..a2343037 --- /dev/null +++ b/docs/how-tos/version-specific-prebuilt.rst @@ -0,0 +1,81 @@ +Version-Specific Prebuilt Settings +=================================== + +When working with multiple collections that share the same variant but need +different versions of a package, you may want some versions to use prebuilt +wheels while others are built from source. + +Version-specific prebuilt settings allow you to configure ``pre_built`` and +``wheel_server_url`` on a per-version basis within a variant, providing +fine-grained control over which package versions use prebuilt wheels. + +Configuration +------------- + +Add version-specific settings under the ``versions`` key within a variant: + +.. code-block:: yaml + + # overrides/settings/torchvision.yaml + variants: + tpu-ubi9: + # Default behavior for unlisted versions + pre_built: false + + # Version-specific overrides + versions: + # Use prebuilt wheel for this version + "0.24.0.dev20250730": + pre_built: true + wheel_server_url: https://gitlab.com/api/v4/projects/12345/packages/pypi/simple + + # Build from source for this version + "0.23.0": + pre_built: false + +Available Settings +------------------ + +Within each version-specific block, you can configure: + +``pre_built`` + Boolean indicating whether to use prebuilt wheels for this version. + +``wheel_server_url`` + URL to download prebuilt wheels from for this version. + +``env`` + Environment variables specific to this version. + +``annotations`` + Version-specific annotations. + +Precedence Rules +---------------- + +Version-specific settings override variant-wide settings. If both are defined, +environment variables are merged with version-specific values taking precedence +for conflicting keys. + +Example Use Case +---------------- + +Consider two TPU collections using different ``torchvision`` versions: + +**Global Collection** (``collections/accelerated/tpu-ubi9/requirements.txt``): + +.. code-block:: text + + torchvision==0.24.0.dev20250730 + +**Torch-2.8.0 Collection** (``collections/torch-2.8.0/tpu-ubi9/requirements.txt``): + +.. code-block:: text + + torchvision==0.23.0 + +With the configuration above: + +- Global collection downloads prebuilt ``torchvision==0.24.0.dev20250730`` wheels +- Torch-2.8.0 collection builds ``torchvision==0.23.0`` from source +- Both use the same variant (``tpu-ubi9``) with different build methods diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py index 970b1381..52a06615 100644 --- a/src/fromager/bootstrapper.py +++ b/src/fromager/bootstrapper.py @@ -88,11 +88,28 @@ def resolve_version( return self._resolved_requirements[req_str] pbi = self.ctx.package_build_info(req) - if pbi.pre_built: - source_url, resolved_version = self._resolve_prebuilt_with_history( - req=req, - req_type=req_type, - ) + + # Check if package has version-specific settings (any version-specific config) + variant_info = pbi._ps.variants.get(pbi._variant) + has_version_specific_prebuilt = ( + variant_info and variant_info.versions and len(variant_info.versions) > 0 + ) + + if pbi.pre_built or has_version_specific_prebuilt: + try: + source_url, resolved_version = self._resolve_prebuilt_with_history( + req=req, + req_type=req_type, + ) + except Exception as e: + # Version-specific prebuilt resolution failed, fall back to source + logger.debug( + f"{req.name}: prebuilt resolution failed, falling back to source: {e}" + ) + source_url, resolved_version = self._resolve_source_with_history( + req=req, + req_type=req_type, + ) else: source_url, resolved_version = self._resolve_source_with_history( req=req, @@ -185,7 +202,8 @@ def bootstrap(self, req: Requirement, req_type: RequirementType) -> Version: source_url_type = sources.get_source_type(self.ctx, req) - if pbi.pre_built: + # Use version-aware prebuilt check now that we have resolved_version + if pbi.is_pre_built(resolved_version): wheel_filename, unpack_dir = self._download_prebuilt( req=req, req_type=req_type, @@ -826,6 +844,19 @@ def _resolve_prebuilt_with_history( req: Requirement, req_type: RequirementType, ) -> tuple[str, Version]: + # Try version-specific resolution FIRST (highest priority) + # This allows version-specific wheel_server_url settings to override + # any cached resolutions from previous bootstraps or current graph + try: + wheel_url, resolved_version = ( + self._resolve_prebuilt_with_version_specific_urls(req, req_type) + ) + return (wheel_url, resolved_version) + except Exception: + # No version-specific settings matched, fall back to cached resolution + pass + + # Fall back to cached resolution from graph cached_resolution = self._resolve_from_graph( req=req, req_type=req_type, @@ -835,15 +866,77 @@ def _resolve_prebuilt_with_history( if cached_resolution and not req.url: wheel_url, resolved_version = cached_resolution logger.debug(f"resolved from previous bootstrap to {resolved_version}") - else: - servers = wheels.get_wheel_server_urls( - self.ctx, req, cache_wheel_server_url=resolver.PYPI_SERVER_URL - ) - wheel_url, resolved_version = wheels.resolve_prebuilt_wheel( - ctx=self.ctx, req=req, wheel_server_urls=servers, req_type=req_type - ) + return (wheel_url, resolved_version) + + # Fall back to regular prebuilt wheel resolution (no version-specific or cached) + servers = wheels.get_wheel_server_urls( + self.ctx, req, cache_wheel_server_url=resolver.PYPI_SERVER_URL + ) + wheel_url, resolved_version = wheels.resolve_prebuilt_wheel( + ctx=self.ctx, req=req, wheel_server_urls=servers, req_type=req_type + ) return (wheel_url, resolved_version) + def _resolve_prebuilt_with_version_specific_urls( + self, + req: Requirement, + req_type: RequirementType, + ) -> tuple[str, Version]: + """Resolve prebuilt wheel using version-specific wheel server URLs if configured.""" + pbi = self.ctx.package_build_info(req) + + # Check if there are version-specific settings + variant_info = pbi._ps.variants.get(pbi._variant) + if not variant_info or not variant_info.versions: + raise ValueError("No version-specific settings configured") + + # Get the constraint for this package + constraint = self.ctx.constraints.get_constraint(req.name) + + # Try to resolve using version-specific wheel server URLs + for version_str, version_settings in variant_info.versions.items(): + # Only process versions that have both wheel_server_url and pre_built=True + if not (version_settings.wheel_server_url and version_settings.pre_built): + continue + + # Only try this version if it satisfies the requirement specifier AND constraint + try: + version_obj = Version(version_str) + + # Check requirement specifier (if present) + if req.specifier and version_obj not in req.specifier: + continue + + # Check constraint (if present) - this is the version from constraints.txt + if constraint and version_obj not in constraint.specifier: + continue + + except Exception: + continue # Skip invalid version strings + + # Create a constraint for this specific version + version_req = Requirement(f"{req.name}=={version_str}") + + try: + # Try to resolve this specific version from the version-specific server + wheel_url, resolved_version = wheels.resolve_prebuilt_wheel( + ctx=self.ctx, + req=version_req, + wheel_server_urls=[version_settings.wheel_server_url], + req_type=req_type, + ) + logger.info( + f"{req.name}: using version-specific prebuilt wheel " + f"{resolved_version} from {version_settings.wheel_server_url}" + ) + return (wheel_url, resolved_version) + except Exception: + # Version not found on this server, try next + continue + + # No matching version-specific prebuilt settings found + raise ValueError("No matching version-specific prebuilt settings found") + def _resolve_from_graph( self, req: Requirement, @@ -949,7 +1042,8 @@ def _add_to_graph( req=req, req_version=req_version, download_url=download_url, - pre_built=pbi.pre_built, + # Use version-aware prebuilt check for dependency graph + pre_built=pbi.is_pre_built(req_version), ) self.ctx.write_to_graph_to_file() diff --git a/src/fromager/commands/build.py b/src/fromager/commands/build.py index 1d7bc063..1a15e149 100644 --- a/src/fromager/commands/build.py +++ b/src/fromager/commands/build.py @@ -345,10 +345,14 @@ def _build( logger.info("starting processing") pbi = wkctx.package_build_info(req) - prebuilt = pbi.pre_built + # Use version-aware prebuilt check now that we have resolved_version + prebuilt = pbi.is_pre_built(resolved_version) wheel_server_urls = wheels.get_wheel_server_urls( - wkctx, req, cache_wheel_server_url=cache_wheel_server_url + wkctx, + req, + cache_wheel_server_url=cache_wheel_server_url, + version=resolved_version, ) # See if we can reuse an existing wheel. diff --git a/src/fromager/packagesettings.py b/src/fromager/packagesettings.py index 709e317f..b568b030 100644 --- a/src/fromager/packagesettings.py +++ b/src/fromager/packagesettings.py @@ -313,6 +313,32 @@ def validate_update_build_requires(cls, v: list[str]) -> list[str]: return v +class VersionSpecificSettings(pydantic.BaseModel): + """Version-specific variant settings + + :: + + pre_built: True + wheel_server_url: https://pypi.org/simple/ + env: + VERSION_SPECIFIC_VAR: "value" + """ + + model_config = MODEL_CONFIG + + annotations: RawAnnotations | None = None + """Version-specific annotations (override variant annotations)""" + + env: EnvVars = Field(default_factory=dict) + """Version-specific env vars (override variant env vars)""" + + wheel_server_url: str | None = None + """Version-specific package index for pre-built wheel""" + + pre_built: bool | None = None + """Version-specific pre-built setting (None = use variant default)""" + + class VariantInfo(pydantic.BaseModel): """Variant information for a package @@ -323,6 +349,12 @@ class VariantInfo(pydantic.BaseModel): VAR2: "2.0 wheel_server_url: https://pypi.org/simple/ pre_build: False + versions: + "1.0.0": + pre_built: True + wheel_server_url: https://custom.server/simple/ + "2.0.0": + pre_built: False """ model_config = MODEL_CONFIG @@ -343,6 +375,9 @@ class VariantInfo(pydantic.BaseModel): pre_built: bool = False """Use pre-built wheel from index server?""" + versions: Mapping[str, VersionSpecificSettings] = Field(default_factory=dict) + """Version-specific settings that override variant defaults""" + class GitOptions(pydantic.BaseModel): """Git repository cloning options @@ -732,6 +767,33 @@ def pre_built(self) -> bool: return vi.pre_built return False + def is_pre_built(self, version: Version | str | None = None) -> bool: + """Check if a specific version uses pre-built wheels + + This method provides version-aware lookup with proper precedence: + 1. Version-specific setting (if version provided and setting exists) + 2. Variant-wide setting (fallback) + + Args: + version: Package version to check. If None, uses variant-wide setting only. + + Returns: + True if the version should use pre-built wheels + """ + vi = self._ps.variants.get(self.variant) + if vi is None: + return False + + # Priority 1: Version-specific setting (if version provided) + if version is not None: + version_str = str(version) if not isinstance(version, str) else version + version_settings = vi.versions.get(version_str) + if version_settings is not None and version_settings.pre_built is not None: + return version_settings.pre_built + + # Priority 2: Variant-wide setting (backward compatibility) + return vi.pre_built + @property def wheel_server_url(self) -> str | None: """Alternative package index for pre-build wheel""" @@ -740,6 +802,38 @@ def wheel_server_url(self) -> str | None: return str(vi.wheel_server_url) return None + def get_wheel_server_url(self, version: Version | str | None = None) -> str | None: + """Get wheel server URL for a specific version + + This method provides version-aware lookup with proper precedence: + 1. Version-specific wheel_server_url (if version provided and setting exists) + 2. Variant-wide wheel_server_url (fallback) + + Args: + version: Package version to check. If None, uses variant-wide setting only. + + Returns: + Wheel server URL for the version or None + """ + vi = self._ps.variants.get(self.variant) + if vi is None: + return None + + # Priority 1: Version-specific setting (if version provided) + if version is not None: + version_str = str(version) if not isinstance(version, str) else version + version_settings = vi.versions.get(version_str) + if ( + version_settings is not None + and version_settings.wheel_server_url is not None + ): + return str(version_settings.wheel_server_url) + + # Priority 2: Variant-wide setting (backward compatibility) + if vi.wheel_server_url is not None: + return str(vi.wheel_server_url) + return None + @property def override_module_name(self) -> str: """Override module name from package name""" @@ -850,15 +944,23 @@ def get_extra_environ( *, template_env: dict[str, str] | None = None, build_env: build_environment.BuildEnvironment | None = None, + version: Version | str | None = None, ) -> dict[str, str]: - """Get extra environment variables for a variant + """Get extra environment variables for a variant and optionally a specific version + Environment variable precedence (later entries override earlier ones): 1. parallel jobs: ``MAKEFLAGS``, ``MAX_JOBS``, ``CMAKE_BUILD_PARALLEL_LEVEL`` 2. PATH and VIRTUAL_ENV from ``build_env`` (if given) 3. package's env settings 4. package variant's env settings + 5. version-specific env settings (if version provided) `template_env` defaults to `os.environ`. + + Args: + template_env: Base environment for template substitution + build_env: Build environment for PATH/VIRTUAL_ENV + version: Package version for version-specific env vars """ if template_env is None: template_env = os.environ.copy() @@ -884,11 +986,19 @@ def get_extra_environ( extra_environ.update(venv_environ) # chain entries so variant entries can reference general entries + # Priority order: package env -> variant env -> version-specific env entries = list(self._ps.env.items()) vi = self._ps.variants.get(self.variant) if vi is not None: entries.extend(vi.env.items()) + # Add version-specific env vars if version provided + if version is not None: + version_str = str(version) if not isinstance(version, str) else version + version_settings = vi.versions.get(version_str) + if version_settings is not None: + entries.extend(version_settings.env.items()) + for key, value in entries: value = substitute_template(value, template_env) extra_environ[key] = value @@ -1124,12 +1234,17 @@ def package_build_info(self, package: str | Package) -> PackageBuildInfo: return pbi def list_pre_built(self) -> set[Package]: - """List packages marked as pre-built""" - return set( - name - for name in self._package_settings - if self.package_build_info(name).pre_built - ) + """List packages marked as pre-built (only includes packages that are definitely prebuilt)""" + pre_built_packages = set() + + for name in self._package_settings: + pbi = self.package_build_info(name) + # Only include if variant-wide pre_built is True + # Version-specific packages with mixed settings are handled at resolution time + if pbi.pre_built: + pre_built_packages.add(name) + + return pre_built_packages def list_overrides(self) -> set[Package]: """List packages with overrides diff --git a/src/fromager/wheels.py b/src/fromager/wheels.py index dc99f767..aedc275a 100644 --- a/src/fromager/wheels.py +++ b/src/fromager/wheels.py @@ -450,13 +450,21 @@ def _download_wheel_check( def get_wheel_server_urls( - ctx: context.WorkContext, req: Requirement, *, cache_wheel_server_url: str | None + ctx: context.WorkContext, + req: Requirement, + *, + cache_wheel_server_url: str | None, + version: Version | str | None = None, ) -> list[str]: pbi = ctx.package_build_info(req) wheel_server_urls: list[str] = [] - if pbi.wheel_server_url: + # Use version-aware wheel server URL lookup if version is available + wheel_server_url = ( + pbi.get_wheel_server_url(version) if version else pbi.wheel_server_url + ) + if wheel_server_url: # use only the wheel server from settings if it is defined. Do not fallback to other URLs - wheel_server_urls.append(pbi.wheel_server_url) + wheel_server_urls.append(wheel_server_url) else: if ctx.wheel_server_url: # local wheel server diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index dd6c278d..1bd92fbc 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -87,6 +87,7 @@ "env": {"EGG": "spam ${EGG}", "EGG_AGAIN": "$EGG"}, "wheel_server_url": "https://wheel.test/simple", "pre_built": False, + "versions": {}, }, "rocm": { "annotations": { @@ -95,12 +96,14 @@ "env": {"SPAM": ""}, "wheel_server_url": None, "pre_built": True, + "versions": {}, }, "cuda": { "annotations": None, "env": {}, "wheel_server_url": None, "pre_built": False, + "versions": {}, }, }, } @@ -182,6 +185,7 @@ "env": {}, "pre_built": True, "wheel_server_url": None, + "versions": {}, }, }, } diff --git a/tests/test_version_specific_prebuilt.py b/tests/test_version_specific_prebuilt.py new file mode 100644 index 00000000..c8b19bf5 --- /dev/null +++ b/tests/test_version_specific_prebuilt.py @@ -0,0 +1,183 @@ +"""Test version-specific prebuilt settings functionality.""" + +import pathlib +import typing + +from fromager.packagesettings import ( + PackageBuildInfo, + PackageSettings, + Settings, + Variant, +) + + +class MockSettings: + """Mock settings object for testing PackageBuildInfo.""" + + def __init__(self, variant: str = "tpu-ubi9", max_jobs: int | None = None) -> None: + self._variant = Variant(variant) + self._patches_dir: pathlib.Path | None = None + self._max_jobs = max_jobs + + @property + def variant(self) -> Variant: + return self._variant + + @property + def patches_dir(self) -> pathlib.Path | None: + return self._patches_dir + + @property + def max_jobs(self) -> int | None: + return self._max_jobs + + def variant_changelog(self) -> list[str]: + return [] + + +def test_version_specific_prebuilt_settings() -> None: + """Test that version-specific prebuilt settings override variant defaults.""" + package_data = { + "variants": { + "tpu-ubi9": { + "pre_built": False, + "wheel_server_url": "https://default.example.com/simple/", + "versions": { + "2.9.0.dev20250730": { + "pre_built": True, + "wheel_server_url": "https://prebuilt.example.com/simple/", + }, + "0.24.0.dev20250730": { + "pre_built": True, + }, + "2.8.0": { + "pre_built": False, + }, + }, + } + } + } + + ps = PackageSettings.from_mapping( + package="test-package", + parsed=package_data, + source="test", + has_config=True, + ) + + pbi = PackageBuildInfo(typing.cast(Settings, MockSettings()), ps) + + # Version with prebuilt and custom URL + assert pbi.is_pre_built("2.9.0.dev20250730") is True + assert ( + pbi.get_wheel_server_url("2.9.0.dev20250730") + == "https://prebuilt.example.com/simple/" + ) + + # Version with prebuilt but no custom URL (uses variant default) + assert pbi.is_pre_built("0.24.0.dev20250730") is True + assert ( + pbi.get_wheel_server_url("0.24.0.dev20250730") + == "https://default.example.com/simple/" + ) + + # Version explicitly set to build from source + assert pbi.is_pre_built("2.8.0") is False + assert pbi.get_wheel_server_url("2.8.0") == "https://default.example.com/simple/" + + # Unknown version uses variant default + assert pbi.is_pre_built("1.0.0") is False + assert pbi.get_wheel_server_url("1.0.0") == "https://default.example.com/simple/" + + # No version specified uses variant default + assert pbi.is_pre_built() is False + assert pbi.get_wheel_server_url() == "https://default.example.com/simple/" + + # Legacy property access + assert pbi.pre_built is False + assert pbi.wheel_server_url == "https://default.example.com/simple/" + + +def test_version_specific_env_vars() -> None: + """Test that version-specific environment variables work correctly.""" + package_data = { + "env": { + "GLOBAL_VAR": "global_value", + }, + "variants": { + "cuda-ubi9": { + "env": { + "VARIANT_VAR": "variant_value", + "OVERRIDE_ME": "variant_override", + }, + "versions": { + "2.0.0": { + "env": { + "VERSION_VAR": "version_value", + "OVERRIDE_ME": "version_override", + } + } + }, + } + }, + } + + ps = PackageSettings.from_mapping( + package="test-package", + parsed=package_data, + source="test", + has_config=True, + ) + + pbi = PackageBuildInfo( + typing.cast(Settings, MockSettings(variant="cuda-ubi9", max_jobs=1)), ps + ) + + # With version: should include all levels with version taking precedence + env_with_version = pbi.get_extra_environ( + template_env={}, build_env=None, version="2.0.0" + ) + assert env_with_version["GLOBAL_VAR"] == "global_value" + assert env_with_version["VARIANT_VAR"] == "variant_value" + assert env_with_version["VERSION_VAR"] == "version_value" + assert env_with_version["OVERRIDE_ME"] == "version_override" + + # Without version: should not include version-specific vars + env_without_version = pbi.get_extra_environ( + template_env={}, + build_env=None, + ) + assert env_without_version["GLOBAL_VAR"] == "global_value" + assert env_without_version["VARIANT_VAR"] == "variant_value" + assert "VERSION_VAR" not in env_without_version + assert env_without_version["OVERRIDE_ME"] == "variant_override" + + +def test_backward_compatibility() -> None: + """Test that existing configurations without version-specific settings still work.""" + package_data = { + "variants": { + "cpu-ubi9": { + "pre_built": True, + "wheel_server_url": "https://legacy.example.com/simple/", + } + } + } + + ps = PackageSettings.from_mapping( + package="legacy-package", + parsed=package_data, + source="test", + has_config=True, + ) + + pbi = PackageBuildInfo(typing.cast(Settings, MockSettings(variant="cpu-ubi9")), ps) + + # All methods should return the variant-wide setting + assert pbi.pre_built is True + assert pbi.is_pre_built() is True + assert pbi.is_pre_built("1.0.0") is True + + assert pbi.wheel_server_url == "https://legacy.example.com/simple/" + assert pbi.get_wheel_server_url() == "https://legacy.example.com/simple/" + assert pbi.get_wheel_server_url("1.0.0") == "https://legacy.example.com/simple/"