Add support for passing extra parameters to package type handlers

Signed-off-by: Matthew Ballance <matt.ballance@gmail.com>

Author: mballance

docs/source/package_types.rst

@@ -366,6 +366,49 @@ Python packages are installed into the project's virtual environment.
     - name: requests
       src: pypi  # Automatically type: python
 
+**Type-specific parameters (** ``with:`` **):**
+
+When ``type: python`` is specified you can pass additional parameters under a
+``with:`` key to control how the package is installed:
+
+``editable`` *(boolean, default: true)*
+    Install the source package in editable mode (``pip install -e``).
+    Set to ``false`` to install as a regular (non-editable) package.
+
+``extras`` *(string or list of strings)*
+    `PEP 508 extras <https://peps.python.org/pep-0508/#extras>`_ to request
+    when installing (e.g. ``[tests]``, ``[litellm, openai]``).
+
+.. code-block:: yaml
+
+    # Non-editable install
+    - name: my-lib
+      url: https://github.com/org/my-lib.git
+      type: python
+      with:
+        editable: false
+
+    # Source package with extras
+    - name: my-lib
+      url: https://github.com/org/my-lib.git
+      type: python
+      with:
+        extras: [tests, docs]
+
+    # Non-editable with extras
+    - name: my-lib
+      url: https://github.com/org/my-lib.git
+      type: python
+      with:
+        extras: [tests]
+        editable: false
+
+.. note::
+
+   ``with:`` is only valid when ``type:`` is also specified on the same entry.
+   Unknown keys inside ``with:`` are a validation error.
+   Parameter validation happens at YAML-read time, before any packages are fetched.
+
 Raw (``raw``)
 -------------
 
@@ -393,6 +436,12 @@ Raw packages are placed in ``packages/<name>/`` but not processed further.
       url: https://example.com/vectors.tar.gz
       type: raw
 
+.. note::
+
+   ``type: raw`` accepts no ``with:`` parameters. This type is also useful to
+   suppress Python auto-detection on a package that happens to contain a
+   ``pyproject.toml`` but should not be pip-installed.
+
 Auto-Detection
 ==============
 

docs/source/python_packages.rst

@@ -183,6 +183,78 @@ Binary Packages
 
     import requests  # Uses installed package
 
+Controlling Installation with ``with:``
+========================================
+
+When ``type: python`` is declared on a package entry, an optional ``with:``
+block lets you pass type-specific installation parameters.
+
+``editable`` — Non-editable Source Installs
+--------------------------------------------
+
+By default, source packages (git, dir) are installed in *editable* mode
+(``pip install -e``).  Set ``editable: false`` to install them as regular,
+non-editable packages instead.  This is useful for dependencies that are
+stable releases you don't intend to modify.
+
+.. code-block:: yaml
+
+    deps:
+      # Default: editable install
+      - name: my-lib
+        url: https://github.com/org/my-lib.git
+        type: python
+
+      # Non-editable install
+      - name: stable-lib
+        url: https://github.com/org/stable-lib.git
+        type: python
+        with:
+          editable: false
+
+``extras`` — PEP 508 Extras
+-----------------------------
+
+Use ``extras`` to request `PEP 508 optional dependency groups
+<https://peps.python.org/pep-0508/#extras>`_ when installing a source package.
+The value may be a single string or a list.
+
+.. code-block:: yaml
+
+    deps:
+      # Single extra
+      - name: my-lib
+        url: https://github.com/org/my-lib.git
+        type: python
+        with:
+          extras: tests
+
+      # Multiple extras
+      - name: my-lib
+        url: https://github.com/org/my-lib.git
+        type: python
+        with:
+          extras: [tests, docs]
+
+      # Non-editable with extras
+      - name: my-lib
+        url: https://github.com/org/my-lib.git
+        type: python
+        with:
+          extras: [tests]
+          editable: false
+
+.. note::
+
+   ``with:`` is only valid when ``type:`` is also specified on the same entry.
+   Unknown keys inside ``with:`` are reported as errors at YAML-read time,
+   before any packages are fetched.
+
+   For PyPI packages (``src: pypi``), extras can also be specified using the
+   top-level ``extras:`` field directly on the package entry — that form is
+   preserved for backward compatibility and is the concise form for the common
+   case.
+
 PyPI Package Configuration
 ==========================
 

src/ivpm/handlers/package_handler_python.py

@@ -29,6 +29,7 @@
 from typing import Dict, List, Set
 from ..project_ops_info import ProjectUpdateInfo, ProjectBuildInfo
 from ..utils import note, fatal, get_venv_python
+from ..pkg_content_type import PythonTypeData
 
 from ..package import Package
 from .package_handler import PackageHandler
@@ -49,6 +50,10 @@ def process_pkg(self, pkg: Package):
         if pkg.src_type == "pypi":
             self.pypi_pkg_s.add(pkg.name)
             add = True
+        elif isinstance(pkg.type_data, PythonTypeData):
+            # Explicit type: python via 'with:' mechanism
+            self.src_pkg_s.add(pkg.name)
+            add = True
         elif pkg.pkg_type is not None and pkg.pkg_type == PackageHandlerPython.name:
             self.src_pkg_s.add(pkg.name)
             add = True
@@ -408,17 +413,30 @@ def _write_requirements_txt(self,
             for pkg in python_pkgs:
 
                 if hasattr(pkg, "url"):
-                    # Editable package
-                    # fp.write("-e file://%s/%s#egg=%s\n" % (
-                    #     packages_dir.replace("\\","/"), 
-                    #     pkg.name, 
-                    #     pkg.name))
-                    fp.write("-e %s/%s\n" % (
-                        packages_dir.replace("\\","/"), 
-                        pkg.name))
+                    # Source package (git, dir, http, etc.)
+                    # Determine editability: type_data takes priority, then default True
+                    editable = True
+                    if isinstance(pkg.type_data, PythonTypeData) and pkg.type_data.editable is not None:
+                        editable = pkg.type_data.editable
+
+                    # Extras from type_data (for source packages declared with type: python + with: extras:)
+                    extras = None
+                    if isinstance(pkg.type_data, PythonTypeData):
+                        extras = pkg.type_data.extras
+                    extras_str = "[%s]" % ",".join(extras) if extras else ""
+
+                    pkg_path = "%s/%s" % (packages_dir.replace("\\","/"), pkg.name)
+                    if editable:
+                        fp.write("-e %s%s\n" % (pkg_path, extras_str))
+                    else:
+                        fp.write("%s%s\n" % (pkg_path, extras_str))
                 else:
                     # PyPi package — build PEP 508 specifier: name[extras]version
-                    extras = getattr(pkg, "extras", None)
+                    # Extras: prefer type_data if present, fall back to pkg.extras (PackagePyPi)
+                    if isinstance(pkg.type_data, PythonTypeData) and pkg.type_data.extras is not None:
+                        extras = pkg.type_data.extras
+                    else:
+                        extras = getattr(pkg, "extras", None)
                     extras_str = "[%s]" % ",".join(extras) if extras else ""
                     if pkg.version is not None:
                         if pkg.version[0] in ['<','>','=']:

src/ivpm/ivpm_yaml_reader.py

@@ -119,6 +119,7 @@ def read_dep_sets(self, info : 'ProjInfo', dep_sets):
 
     def read_deps(self, ret : PackagesInfo, deps, default_dep_set):
         from .pkg_types.pkg_type_rgy import PkgTypeRgy
+        from .pkg_content_type_rgy import PkgContentTypeRgy
 
         for d in deps:
             si = d.srcinfo
@@ -167,6 +168,20 @@ def read_deps(self, ret : PackagesInfo, deps, default_dep_set):
                 raise Exception("Package %s has unknown type %s" % (d["name"], src))
             pkg = PkgTypeRgy.inst().mkPackage(src, str(d["name"]), d, si)
 
+            # Resolve content type and validate 'with:' parameters
+            ct_rgy = PkgContentTypeRgy.inst()
+            if "type" in d.keys():
+                type_name = str(d["type"])
+                if not ct_rgy.has(type_name):
+                    fatal("Package '%s': unknown type '%s' @ %s ; known types: %s" % (
+                        pkg.name, type_name, getlocstr(d["type"]),
+                        ", ".join(ct_rgy.names())))
+                with_opts = d["with"] if "with" in d.keys() else {}
+                pkg.type_data = ct_rgy.get(type_name).create_data(with_opts, si)
+            elif "with" in d.keys():
+                fatal("Package '%s': 'with:' is specified but 'type:' is not @ %s" % (
+                    pkg.name, getlocstr(d["with"])))
+
             # Unless specified, load the same dep-set from sub-packages
             if pkg.dep_set is None:
                 if default_dep_set is not None:

src/ivpm/package.py

@@ -24,7 +24,7 @@
 import os
 import dataclasses as dc
 from enum import Enum, auto
-from typing import Dict, List, Set
+from typing import Dict, List, Set, Optional
 from .project_ops_info import ProjectUpdateInfo
 from .utils import fatal, getlocstr
 
@@ -97,6 +97,9 @@ class Package(object):
     path : str = None
     pkg_type : PackageType = None
     src_type : str = None
+    # type_data holds validated, type-specific parameters from the 'with:' YAML key.
+    # Set by IvpmYamlReader when 'type:' is present; None when type is auto-detected.
+    type_data : Optional['TypeData'] = None
 
     process_deps : bool = True
     setup_deps : Set[str] = dc.field(default_factory=set)
@@ -148,14 +151,16 @@ def process_options(self, opts, si):
             else:
                 fatal("Unknown value for 'deps': %s" % opts["deps"])
 
-        # Determine the package type (eg Python, Raw)
+        # Set pkg_type from 'type:' for backward compatibility.
+        # The authoritative typed data is stored in type_data by IvpmYamlReader
+        # after calling PkgContentTypeRgy. We still set pkg_type here so that
+        # code that has not yet migrated to type_data continues to work.
         if "type" in opts.keys():
             type_s = opts["type"]
-            if not type_s in Spec2PackageType.keys():
-                fatal("unknown package type %s @ %s ; Supported types 'raw', 'python'" % (
-                    type_s, getlocstr(opts["type"])))
-                    
-            self.pkg_type = Spec2PackageType[type_s]
+            if type_s in Spec2PackageType.keys():
+                self.pkg_type = Spec2PackageType[type_s]
+            # Unknown type strings are tolerated here; the YAML reader validates them
+            # via PkgContentTypeRgy and emits a proper error with source location.
 
     @staticmethod
     def mk(name, opts, si) -> 'Package':