|
1 | 1 | #!/usr/bin/env python |
2 | | -# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD |
| 2 | +# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD |
3 | 3 | # SPDX-License-Identifier: Apache-2.0 |
4 | 4 |
|
5 | 5 | # internal use only for CI |
6 | 6 | # download archive of one commit instead of cloning entire submodule repo |
7 | 7 |
|
8 | 8 | import argparse |
9 | | -import os |
10 | | -import re |
11 | | -import shutil |
12 | 9 | import subprocess |
| 10 | +import sys |
13 | 11 | import time |
14 | | -from typing import Any, List |
15 | | - |
16 | | -import gitlab_api |
17 | | - |
18 | | -SUBMODULE_PATTERN = re.compile(r"\[submodule \"([^\"]+)\"]") |
19 | | -PATH_PATTERN = re.compile(r'path\s+=\s+(\S+)') |
20 | | -URL_PATTERN = re.compile(r'url\s+=\s+(\S+)') |
21 | | - |
22 | | -SUBMODULE_ARCHIVE_TEMP_FOLDER = 'submodule_archive' |
23 | | -# need to match the one defined in CI yaml files for caching purpose |
24 | | -SUBMODULE_ARCHIVE_CACHE_DIR = '.cache/submodule_archives' |
25 | | - |
26 | | - |
27 | | -class SubModule(object): |
28 | | - # We don't need to support recursive submodule clone now |
29 | | - |
30 | | - GIT_LS_TREE_OUTPUT_PATTERN = re.compile(r'\d+\s+commit\s+([0-9a-f]+)\s+') |
31 | | - |
32 | | - def __init__(self, gitlab_inst: gitlab_api.Gitlab, path: str, url: str) -> None: |
33 | | - self.path = path |
34 | | - self.url = url |
35 | | - self.gitlab_inst = gitlab_inst |
36 | | - self.project_id = self._get_project_id(url) |
37 | | - self.commit_id = self._get_commit_id(path) |
38 | | - |
39 | | - def _get_commit_id(self, path: str) -> str: |
40 | | - output = subprocess.check_output(['git', 'ls-tree', 'HEAD', path]).decode() |
41 | | - # example output: 160000 commit d88a262fbdf35e5abb372280eb08008749c3faa0 components/esp_wifi/lib |
42 | | - match = self.GIT_LS_TREE_OUTPUT_PATTERN.search(output) |
43 | | - return match.group(1) if match is not None else '' |
44 | | - |
45 | | - def _get_project_id(self, url: str) -> Any: |
46 | | - base_name = os.path.basename(url) |
47 | | - project_id = self.gitlab_inst.get_project_id(os.path.splitext(base_name)[0], # remove .git |
48 | | - namespace='espressif') |
49 | | - return project_id |
50 | | - |
51 | | - def download_archive(self) -> None: |
52 | | - print('Update submodule: {}: {}'.format(self.path, self.commit_id)) |
53 | | - path_name = self.gitlab_inst.download_archive(self.commit_id, SUBMODULE_ARCHIVE_TEMP_FOLDER, |
54 | | - self.project_id, SUBMODULE_ARCHIVE_CACHE_DIR) |
55 | | - renamed_path = os.path.join(os.path.dirname(path_name), os.path.basename(self.path)) |
56 | | - os.rename(path_name, renamed_path) |
57 | | - shutil.rmtree(self.path, ignore_errors=True) |
58 | | - shutil.move(renamed_path, os.path.dirname(self.path)) |
59 | | - |
60 | | - |
61 | | -def update_submodule(git_module_file: str, submodules_to_update: List) -> None: |
62 | | - gitlab_inst = gitlab_api.Gitlab() |
63 | | - submodules = [] |
64 | | - with open(git_module_file, 'r') as f: |
65 | | - data = f.read() |
66 | | - match = SUBMODULE_PATTERN.search(data) |
67 | | - if match is not None: |
68 | | - while True: |
69 | | - next_match = SUBMODULE_PATTERN.search(data, pos=match.end()) |
70 | | - if next_match: |
71 | | - end_pos = next_match.start() |
72 | | - else: |
73 | | - end_pos = len(data) |
74 | | - path_match = PATH_PATTERN.search(data, pos=match.end(), endpos=end_pos) |
75 | | - url_match = URL_PATTERN.search(data, pos=match.end(), endpos=end_pos) |
76 | | - path = path_match.group(1) if path_match is not None else '' |
77 | | - url = url_match.group(1) if url_match is not None else '' |
78 | | - |
79 | | - filter_result = True |
80 | | - if submodules_to_update: |
81 | | - if path not in submodules_to_update: |
82 | | - filter_result = False |
83 | | - if filter_result: |
84 | | - submodules.append(SubModule(gitlab_inst, path, url)) |
85 | | - |
86 | | - match = next_match |
87 | | - if not match: |
88 | | - break |
89 | | - |
90 | | - shutil.rmtree(SUBMODULE_ARCHIVE_TEMP_FOLDER, ignore_errors=True) |
91 | | - |
92 | | - for submodule in submodules: |
93 | | - submodule.download_archive() |
94 | | - |
95 | 12 |
|
96 | 13 | if __name__ == '__main__': |
97 | | - start_time = time.time() |
98 | 14 | parser = argparse.ArgumentParser() |
99 | 15 | parser.add_argument('--repo_path', '-p', default='.', help='repo path') |
100 | | - parser.add_argument('--submodule', '-s', default='all', |
101 | | - help='Submodules to update. By default update all submodules. ' |
102 | | - 'For multiple submodules, separate them with `;`. ' |
103 | | - '`all` and `none` are special values that indicates we fetch all / none submodules') |
| 16 | + parser.add_argument( |
| 17 | + '--submodule', |
| 18 | + '-s', |
| 19 | + default='all', |
| 20 | + help='Submodules to update. By default update all submodules. ' |
| 21 | + 'For multiple submodules, separate them with `;`. ' |
| 22 | + '`all` and `none` are special values that indicates we fetch all / none submodules', |
| 23 | + ) |
104 | 24 | args = parser.parse_args() |
105 | | - if args.submodule == 'none': |
106 | | - print("don't need to update submodules") |
107 | | - exit(0) |
108 | | - if args.submodule == 'all': |
109 | | - _submodules = [] |
110 | | - else: |
111 | | - _submodules = args.submodule.split(';') |
112 | | - update_submodule(os.path.join(args.repo_path, '.gitmodules'), _submodules) |
| 25 | + |
| 26 | + print('This script is deprecated, please use the following git command with gitlab cache `.git/modules` instead.') |
| 27 | + print('Calling `git submodule update --init --depth=1` ...') |
| 28 | + |
| 29 | + start_time = time.time() |
| 30 | + subprocess.check_call( |
| 31 | + ['git', 'submodule', 'update', '--init', '--depth=1'], stdout=sys.stdout, stderr=sys.stderr, cwd=args.repo_path |
| 32 | + ) |
113 | 33 | print('total time spent on update submodule: {:.02f}s'.format(time.time() - start_time)) |
0 commit comments