project.py: Add new 'joblib' dependency to parallelize update

The projects from manifests were updated one by one.
The parallel aproach has potential to fully utilize
CPU and network connection to speed up the update.

Signed-off-by: Robert Gałat <[email protected]>

Author: RobertGalatNordic

setup.py

@@ -46,6 +46,7 @@
         'pykwalify',
         'setuptools',
         'packaging',
+        'joblib'
     ],
     python_requires='>=3.8',
     entry_points={'console_scripts': ('west = west.app.main:main',)},

src/west/app/project.py

@@ -30,6 +30,12 @@
 from west.manifest import QUAL_MANIFEST_REV_BRANCH as QUAL_MANIFEST_REV
 from west.manifest import QUAL_REFS_WEST as QUAL_REFS
 
+JOBLIB_PRESENT = True
+try:
+    from joblib import Parallel, delayed
+except ModuleNotFoundError:
+    JOBLIB_PRESENT = False
+
 #
 # Project-related or multi-repo commands, like "init", "update",
 # "diff", etc.
@@ -883,6 +889,12 @@ def do_add_parser(self, parser_adder):
         parser.add_argument('--stats', action='store_true',
                             help='''print performance statistics for
                             update operations''')
+        if JOBLIB_PRESENT:
+            parser.add_argument('-j', '--jobs', nargs='?', const=-1,
+                                default=1, type=int, action='store',
+                                help='''Use multiple jobs to paralelize update process.
+                                Pass -1 to use all avaliable jobs.
+                                ''')
 
         group = parser.add_argument_group(
             title='local project clone caches',
@@ -1018,18 +1030,26 @@ def update_all(self):
             import_flags=ImportFlag.FORCE_PROJECTS)
 
         failed = []
-        for project in self.manifest.projects:
+
+        def project_update(project):
             if (isinstance(project, ManifestProject) or
                     project.name in self.updated):
-                continue
+                return
             try:
                 if not self.project_is_active(project):
                     self.dbg(f'{project.name}: skipping inactive project')
-                    continue
-                self.update(project)
+                    return
                 self.updated.add(project.name)
+                self.update(project)
             except subprocess.CalledProcessError:
                 failed.append(project)
+
+        if not JOBLIB_PRESENT or self.args.jobs == 1:
+            for project in self.manifest.projects:
+                project_update(project)
+        else:
+            Parallel(n_jobs=self.args.jobs, require='sharedmem')(
+                delayed(project_update)(project) for project in self.manifest.projects)
         self._handle_failed(self.args, failed)
 
     def update_importer(self, project, path):
@@ -1090,13 +1110,22 @@ def update_some(self):
             projects = self._projects(self.args.projects)
 
         failed = []
-        for project in projects:
+
+        def project_update_some(project):
             if isinstance(project, ManifestProject):
-                continue
+                return
             try:
                 self.update(project)
             except subprocess.CalledProcessError:
                 failed.append(project)
+
+        if not JOBLIB_PRESENT or self.args.jobs == 1:
+            for project in projects:
+                project_update_some(project)
+        else:
+            Parallel(n_jobs=self.args.jobs, require='sharedmem')(
+                delayed(project_update_some)(project) for project in projects)
+
         self._handle_failed(self.args, failed)
 
     def toplevel_projects(self):

tests/test_project.py

@@ -416,14 +416,14 @@ def test_grep(west_init_tmpdir):
 
     assert re.search('west-commands', cmd('grep -- -- -commands'))
 
-
-def test_update_projects(west_init_tmpdir):
+@pytest.mark.parametrize("options", ["", "-j 1", "-j 2", "-j"])
+def test_update_projects(options, west_init_tmpdir):
     # Test the 'west update' command. It calls through to the same backend
     # functions that are used for automatic updates and 'west init'
     # reinitialization.
 
     # create local repositories
-    cmd('update')
+    cmd('update ' + options)
 
     # Add commits to the local repos.
     ur = update_helper(west_init_tmpdir)
@@ -613,7 +613,8 @@ def test_update_head_0(west_init_tmpdir):
     assert modified_files.strip() == "M CODEOWNERS", \
            'local zephyr change not preserved'
 
-def test_update_some_with_imports(repos_tmpdir):
+@pytest.mark.parametrize("options", ["", "-j 1", "-j 2", "-j -1"])
+def test_update_some_with_imports(options, repos_tmpdir):
     # 'west update project1 project2' should work fine even when
     # imports are used, as long as the relevant projects are all
     # defined in the manifest repository.
@@ -654,19 +655,19 @@ def test_update_some_with_imports(repos_tmpdir):
     # Updating unknown projects should fail as always.
 
     with pytest.raises(subprocess.CalledProcessError):
-        cmd('update unknown-project', cwd=ws)
+        cmd(f'update {options} unknown-project', cwd=ws)
 
     # Updating a list of projects when some are resolved via project
     # imports must fail.
 
     with pytest.raises(subprocess.CalledProcessError):
-        cmd('update Kconfiglib net-tools', cwd=ws)
+        cmd(f'update {options} Kconfiglib net-tools', cwd=ws)
 
     # Updates of projects defined in the manifest repository or all
     # projects must succeed, and behave the same as if no imports
     # existed.
 
-    cmd('update net-tools', cwd=ws)
+    cmd(f'update {options} net-tools', cwd=ws)
     with pytest.raises(ManifestImportFailed):
         Manifest.from_topdir(topdir=ws)
     manifest = Manifest.from_topdir(topdir=ws,
@@ -677,10 +678,10 @@ def test_update_some_with_imports(repos_tmpdir):
     assert net_tools_project.is_cloned()
     assert not zephyr_project.is_cloned()
 
-    cmd('update zephyr', cwd=ws)
+    cmd(f'update {options} zephyr', cwd=ws)
     assert zephyr_project.is_cloned()
 
-    cmd('update', cwd=ws)
+    cmd(f'update {options}', cwd=ws)
     manifest = Manifest.from_topdir(topdir=ws)
     assert manifest.get_projects(['Kconfiglib'])[0].is_cloned()