add sub-package (#1036)

Author: wzh1994

.github/workflows/main.yml

@@ -367,11 +367,16 @@ jobs:
 
       - name: Build
         run: |
-          pip install build
+          pip install build tomlkit
+          cp pyproject.toml pyproject.toml.backup
+          python scripts/generate_toml_optional_deps.py --split
           python -m build --config-setting=skbuild.wheel.cmake=false
+          mv pyproject.toml.backup pyproject.toml
 
       - name: Test installing built package
-        run: pip install dist/lazyllm*.whl
+        run: |
+          pip install dist/lazyllm*.whl
+          pip install "$(ls dist/lazyllm*.whl)[online-advanced]"
 
       - name: Test import
         working-directory: /tmp

.github/workflows/publish_release.yml

@@ -155,9 +155,12 @@ jobs:
 
       - name: Build default wheel
         run: |
-          pip install build
+          pip install build tomlkit
+          cp pyproject.toml pyproject.toml.backup
+          python scripts/generate_toml_optional_deps.py --split
           python -m build --config-setting=skbuild.wheel.cmake=false
           ls -la dist/
+          mv pyproject.toml.backup pyproject.toml
 
       - name: Upload source_code
         uses: actions/upload-artifact@v4
@@ -211,9 +214,12 @@ jobs:
 
       - name: Build wheel
         run: |
-          pip install cibuildwheel
+          pip install cibuildwheel tomlkit
+          cp pyproject.toml pyproject.toml.backup
+          python scripts/generate_toml_optional_deps.py --split
           cibuildwheel --output-dir repaired_wheel
           ls -la repaired_wheel/
+          mv pyproject.toml.backup pyproject.toml
         env:
           CIBW_BUILD_VERBOSITY: 1
 

scripts/generate_toml_optional_deps.py

@@ -4,6 +4,7 @@
 '''
 from pathlib import Path
 import tomlkit
+import sys
 
 def expand_caret(version: str) -> str:
     '''
@@ -45,6 +46,13 @@ def normalize_version(version: str) -> str:
     return version
 
 
+def make_subgroup(project, name, deps):
+    arr = tomlkit.array()
+    arr.multiline(True)
+    for dep in sorted(deps):
+        arr.append(dep)
+    project['optional-dependencies'][name] = arr
+
 def main():
     path = Path('pyproject.toml')
     if not path.exists():
@@ -57,23 +65,24 @@ def main():
 
     poetry_optional_deps = doc['tool']['poetry']['dependencies']
 
-    proj_optional_deps = []
+    split = 'split' in sys.argv[1:] or '--split' in sys.argv[1:]
+    proj_optional_deps = {} if split else []
 
     for name, spec in poetry_optional_deps.items():
         version = normalize_version(spec['version'])
-        proj_optional_deps.append(f'{name}{version}')
+        if split:
+            proj_optional_deps[name] = f'{name}{version}'
+        else:
+            proj_optional_deps.append(f'{name}{version}')
 
-    project = doc['project']
     project['optional-dependencies'] = tomlkit.table()
 
-    arr = tomlkit.array()
-    arr.multiline(True)
-    for dep in sorted(proj_optional_deps):
-        arr.append(dep)
-
-    project = doc['project']
-    project['optional-dependencies'] = tomlkit.table()
-    project['optional-dependencies']['optional'] = arr
+    if split:
+        extras = doc['tool']['poetry']['extras']
+        for k, vs in extras.items():
+            make_subgroup(project, k, [proj_optional_deps[v] for v in vs])
+    else:
+        make_subgroup(project, 'optional', proj_optional_deps)
 
     # remove poetry sections
     doc['tool'].pop('poetry', None)