Add 'uses' support

Signed-off-by: Matthew Ballance <matt.ballance@gmail.com>

Author: mballance

docs/source/dependency_sets.rst

@@ -91,6 +91,91 @@ While you can use any names, IVPM recognizes these standard names:
     Development dependencies. Includes everything from ``default`` plus 
     development tools. This is no longer special - any name can be used.
 
+Dep-Set Inheritance with ``uses``
+----------------------------------
+
+The ``uses`` field lets a dep-set inherit all packages from another dep-set
+defined in the same ``ivpm.yaml``.  This avoids duplicating shared entries
+across multiple sets.
+
+**Merge rules**
+
+- Every package from the *base* dep-set is copied into the *child* dep-set.
+- If the same package name appears in both, the **child's definition wins**.
+- Inheritance is resolved at parse time, so there is no runtime overhead.
+- Chains of any depth are supported (``a`` uses ``b`` uses ``c`` …).
+- Cycles are detected and raise an error.
+- Definition order does not matter; the base may be defined after the child.
+
+**Basic example** — ``default-dev`` extends ``default``:
+
+.. code-block:: yaml
+
+    package:
+      name: my-project
+      default-dep-set: default-dev
+
+      dep-sets:
+        - name: default
+          deps:
+            - name: runtime-lib
+              url: https://github.com/org/runtime-lib.git
+
+        - name: default-dev
+          uses: default          # inherit runtime-lib from above
+          deps:
+            - name: pytest
+              src: pypi
+            - name: test-framework
+              url: https://github.com/org/test-framework.git
+
+Running ``ivpm update -d default-dev`` installs ``runtime-lib``, ``pytest``,
+and ``test-framework``.  Running ``ivpm update -d default`` installs only
+``runtime-lib``.
+
+**Overriding an inherited package** — pin a different version in the child:
+
+.. code-block:: yaml
+
+    dep-sets:
+      - name: default
+        deps:
+          - name: mylib
+            url: https://github.com/org/mylib.git
+            branch: v1.0
+
+      - name: default-dev
+        uses: default
+        deps:
+          - name: mylib
+            url: https://github.com/org/mylib.git
+            branch: dev   # overrides the v1.0 branch from 'default'
+          - name: pytest
+            src: pypi
+
+**Multi-level inheritance**:
+
+.. code-block:: yaml
+
+    dep-sets:
+      - name: base
+        deps:
+          - name: core-lib
+            src: pypi
+
+      - name: dev
+        uses: base
+        deps:
+          - name: pytest
+            src: pypi
+
+      - name: ci
+        uses: dev
+        deps:
+          - name: coverage
+            src: pypi
+          # inherits core-lib (from base via dev) and pytest (from dev)
+
 Using Dependency Sets
 =====================
 

src/ivpm/handlers/package_handler_python.py

@@ -30,6 +30,7 @@
 from ..project_ops_info import ProjectUpdateInfo, ProjectBuildInfo
 from ..utils import note, fatal, get_venv_python
 from ..pkg_content_type import PythonTypeData
+from ..package import get_type_data
 
 from ..package import Package
 from .package_handler import PackageHandler
@@ -50,8 +51,8 @@ def process_pkg(self, pkg: Package):
         if pkg.src_type == "pypi":
             self.pypi_pkg_s.add(pkg.name)
             add = True
-        elif isinstance(pkg.type_data, PythonTypeData):
-            # Explicit type: python via 'with:' mechanism
+        elif get_type_data(pkg, PythonTypeData) is not None:
+            # Explicit type: python
             self.src_pkg_s.add(pkg.name)
             add = True
         elif pkg.pkg_type is not None and pkg.pkg_type == PackageHandlerPython.name:
@@ -416,13 +417,14 @@ def _write_requirements_txt(self,
                     # Source package (git, dir, http, etc.)
                     # Determine editability: type_data takes priority, then default True
                     editable = True
-                    if isinstance(pkg.type_data, PythonTypeData) and pkg.type_data.editable is not None:
-                        editable = pkg.type_data.editable
+                    td = get_type_data(pkg, PythonTypeData)
+                    if td is not None and td.editable is not None:
+                        editable = td.editable
 
-                    # Extras from type_data (for source packages declared with type: python + with: extras:)
+                    # Extras from type_data
                     extras = None
-                    if isinstance(pkg.type_data, PythonTypeData):
-                        extras = pkg.type_data.extras
+                    if td is not None:
+                        extras = td.extras
                     extras_str = "[%s]" % ",".join(extras) if extras else ""
 
                     pkg_path = "%s/%s" % (packages_dir.replace("\\","/"), pkg.name)
@@ -433,8 +435,9 @@ def _write_requirements_txt(self,
                 else:
                     # PyPi package — build PEP 508 specifier: name[extras]version
                     # Extras: prefer type_data if present, fall back to pkg.extras (PackagePyPi)
-                    if isinstance(pkg.type_data, PythonTypeData) and pkg.type_data.extras is not None:
-                        extras = pkg.type_data.extras
+                    td = get_type_data(pkg, PythonTypeData)
+                    if td is not None and td.extras is not None:
+                        extras = td.extras
                     else:
                         extras = getattr(pkg, "extras", None)
                     extras_str = "[%s]" % ",".join(extras) if extras else ""

src/ivpm/ivpm_yaml_reader.py

@@ -15,6 +15,7 @@
 from .utils import fatal, getlocstr, warning
 from ivpm.package import Package, PackageType, SourceType
 from ivpm.packages_info import PackagesInfo
+from ivpm.pkg_content_type import parse_type_field
 
 
 class IvpmYamlReader(object):
@@ -50,6 +51,9 @@ def read(self, fp, name) -> 'ProjInfo':
         else:
             ret.version = None
 
+        if "type" in pkg.keys():
+            ret.self_types = parse_type_field(pkg["type"])
+
         # Specify where sub-packages are stored. Defaults to 'packages'        
         if "deps-dir" in pkg.keys():
             ret.deps_dir = pkg["deps-dir"]
@@ -105,16 +109,64 @@ def read_dep_sets(self, info : 'ProjInfo', dep_sets):
             ds = PackagesInfo(ds_name)
             default_dep_set = None
 
+            if "uses" in ds_ent.keys():
+                ds.uses = str(ds_ent["uses"])
+
             if "default-dep-set" in ds_ent.keys():
                 default_dep_set = ds_ent["default-dep-set"]
 
-            
             deps = ds_ent["deps"]
 
             if not isinstance(deps, list):
                 raise Exception("deps is not a list")
             self.read_deps(ds, ds_ent["deps"], default_dep_set)
             info.set_dep_set(ds.name, ds)
+
+        self._resolve_dep_set_inheritance(info)
+
+    def _resolve_dep_set_inheritance(self, info: 'ProjInfo'):
+        """
+        Merge inherited packages for every dep-set that declares a 'uses' base.
+        The current dep-set's packages win on name collision.
+        Detects cycles and unknown base names.
+        """
+        dep_set_m = info.dep_set_m
+        resolved = set()
+
+        def resolve(name, visiting):
+            if name in resolved:
+                return
+            ds = dep_set_m[name]
+            if ds.uses is None:
+                resolved.add(name)
+                return
+            if name in visiting:
+                cycle = " -> ".join(list(visiting) + [name])
+                raise Exception(
+                    "Cyclic dep-set inheritance detected: %s" % cycle)
+            base_name = ds.uses
+            if base_name not in dep_set_m:
+                raise Exception(
+                    "dep-set '%s' references unknown base dep-set '%s'"
+                    % (name, base_name))
+            visiting.add(name)
+            resolve(base_name, visiting)
+            visiting.discard(name)
+
+            base_ds = dep_set_m[base_name]
+            # Start with base packages, then let current overwrite
+            merged_pkgs = base_ds.packages.copy()
+            merged_pkgs.update(ds.packages)
+            ds.packages = merged_pkgs
+
+            merged_opts = base_ds.options.copy()
+            merged_opts.update(ds.options)
+            ds.options = merged_opts
+
+            resolved.add(name)
+
+        for ds_name in list(dep_set_m.keys()):
+            resolve(ds_name, set())
 
 
     def read_deps(self, ret : PackagesInfo, deps, default_dep_set):
@@ -168,19 +220,21 @@ def read_deps(self, ret : PackagesInfo, deps, default_dep_set):
                 raise Exception("Package %s has unknown type %s" % (d["name"], src))
             pkg = PkgTypeRgy.inst().mkPackage(src, str(d["name"]), d, si)
 
-            # Resolve content type and validate 'with:' parameters
+            # Resolve content type from 'type:' field (string, dict, or list form).
+            # 'with:' is no longer supported; options are now inline in the type dict.
             ct_rgy = PkgContentTypeRgy.inst()
+            if "with" in d.keys():
+                fatal("Package '%s': 'with:' is no longer supported; "
+                      "use inline options instead, e.g. type: { python: { editable: false } } @ %s" % (
+                          pkg.name, getlocstr(d["with"])))
             if "type" in d.keys():
-                type_name = str(d["type"])
-                if not ct_rgy.has(type_name):
-                    fatal("Package '%s': unknown type '%s' @ %s ; known types: %s" % (
-                        pkg.name, type_name, getlocstr(d["type"]),
-                        ", ".join(ct_rgy.names())))
-                with_opts = d["with"] if "with" in d.keys() else {}
-                pkg.type_data = ct_rgy.get(type_name).create_data(with_opts, si)
-            elif "with" in d.keys():
-                fatal("Package '%s': 'with:' is specified but 'type:' is not @ %s" % (
-                    pkg.name, getlocstr(d["with"])))
+                raw = parse_type_field(d["type"])
+                for type_name, opts in raw:
+                    if not ct_rgy.has(type_name):
+                        fatal("Package '%s': unknown type '%s' @ %s ; known types: %s" % (
+                            pkg.name, type_name, getlocstr(d["type"]),
+                            ", ".join(ct_rgy.names())))
+                    pkg.type_data.append(ct_rgy.get(type_name).create_data(opts, si))
 
             # Unless specified, load the same dep-set from sub-packages
             if pkg.dep_set is None:

src/ivpm/package.py

@@ -24,7 +24,7 @@
 import os
 import dataclasses as dc
 from enum import Enum, auto
-from typing import Dict, List, Set, Optional
+from typing import Dict, List, Set, Optional, Tuple
 from .project_ops_info import ProjectUpdateInfo
 from .utils import fatal, getlocstr
 
@@ -97,9 +97,12 @@ class Package(object):
     path : str = None
     pkg_type : PackageType = None
     src_type : str = None
-    # type_data holds validated, type-specific parameters from the 'with:' YAML key.
-    # Set by IvpmYamlReader when 'type:' is present; None when type is auto-detected.
-    type_data : Optional['TypeData'] = None
+    # type_data holds the list of validated TypeData objects produced from the 'type:' field.
+    # Set by IvpmYamlReader; empty list means no explicit type (auto-detection may still apply).
+    type_data : List['TypeData'] = dc.field(default_factory=list)
+    # self_types holds the raw (type_name, opts) pairs read from the package's own ivpm.yaml.
+    # Populated during dep resolution by package_updater; empty until then.
+    self_types : List[Tuple[str, dict]] = dc.field(default_factory=list)
 
     process_deps : bool = True
     setup_deps : Set[str] = dc.field(default_factory=set)
@@ -152,19 +155,24 @@ def process_options(self, opts, si):
                 fatal("Unknown value for 'deps': %s" % opts["deps"])
 
         # Set pkg_type from 'type:' for backward compatibility.
-        # The authoritative typed data is stored in type_data by IvpmYamlReader
-        # after calling PkgContentTypeRgy. We still set pkg_type here so that
-        # code that has not yet migrated to type_data continues to work.
+        # The authoritative typed data is stored in type_data by IvpmYamlReader.
         if "type" in opts.keys():
-            type_s = opts["type"]
-            if type_s in Spec2PackageType.keys():
-                self.pkg_type = Spec2PackageType[type_s]
-            # Unknown type strings are tolerated here; the YAML reader validates them
-            # via PkgContentTypeRgy and emits a proper error with source location.
+            from .pkg_content_type import parse_type_field
+            pairs = parse_type_field(opts["type"])
+            if pairs:
+                first_name = pairs[0][0]
+                if first_name in Spec2PackageType.keys():
+                    self.pkg_type = Spec2PackageType[first_name]
 
     @staticmethod
     def mk(name, opts, si) -> 'Package':
         raise NotImplementedError()
-    
 
 
+def get_type_data(pkg: 'Package', cls):
+    """Return the first TypeData entry in pkg.type_data that is an instance of cls, or None."""
+    for td in pkg.type_data:
+        if isinstance(td, cls):
+            return td
+    return None
+

src/ivpm/package_updater.py

@@ -212,6 +212,17 @@ def _update_pkg(self, pkg : Package) -> Tuple[Package, ProjInfo]:
         try:
             pkg.proj_info = pkg.update(self.update_info)
 
+            # Merge self-declared types from the dep's own ivpm.yaml into pkg.type_data.
+            # Caller-specified types take priority; self-declared ones are appended only
+            # if their type name is not already present.
+            if pkg.proj_info is not None and pkg.proj_info.self_types:
+                from .pkg_content_type_rgy import PkgContentTypeRgy
+                ct_rgy = PkgContentTypeRgy.inst()
+                caller_names = {td.type_name for td in pkg.type_data}
+                for type_name, opts in pkg.proj_info.self_types:
+                    if type_name not in caller_names and ct_rgy.has(type_name):
+                        pkg.type_data.append(ct_rgy.get(type_name).create_data(opts, None))
+
             # Notify the package handlers after the source is 
             # loaded so they can take further action if required 
             self.pkg_handler.process_pkg(pkg)

src/ivpm/packages_info.py

@@ -19,7 +19,7 @@
 #*     Author: mballance
 #*
 #****************************************************************************
-from typing import Dict, List, Set
+from typing import Dict, List, Optional, Set
 from ivpm.package import Package
 
 class PackagesInfo():
@@ -30,6 +30,9 @@ class PackagesInfo():
 
     def __init__(self, name):
         self.name = name
+        # Name of a dep-set in the same file to inherit packages from.
+        # Populated during parsing; resolved (merged) before use.
+        self.uses : Optional[str] = None
         self.packages : Dict[str,Package] = {}
 
         # Map of package name to set of packages
@@ -63,6 +66,7 @@ def __setitem__(self, key, value):
 
     def copy(self) -> 'PackagesInfo':
         ret = PackagesInfo(self.name)
+        ret.uses     = self.uses
         ret.packages = self.packages.copy()
         ret.options  = self.options.copy()