Skip to content

Commit 6e3568d

Browse files
philschmidjulien-c
philschmid
and
julien-c
authored
added stderr and exceptions (#19)
* added stderr and exceptions * added log to push and check for commit error * fixed quality * doc improvements + small tweaks * [git_add] actually take input pattern * less brittle test if local_dir is an actual git repo Co-authored-by: Julien Chaumond <[email protected]>
1 parent d7855db commit 6e3568d

File tree

1 file changed

+183
-74
lines changed

1 file changed

+183
-74
lines changed

‎src/huggingface_hub/repository.py‎

Lines changed: 183 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -30,21 +30,21 @@ def __init__(
3030
Instantiate a local clone of a git repo.
3131
3232
If specifying a `clone_from`:
33-
will clone an existing remote repository
33+
will clone an existing remote repository, for instance one
3434
that was previously created using ``HfApi().create_repo(token=huggingface_token, name=repo_name)``.
3535
``Repository`` uses the local git credentials by default, but if required, the ``huggingface_token``
36-
as well as the git ``user`` and the ``email`` can be specified.
37-
``Repository`` will then override them.
36+
as well as the git ``user`` and the ``email`` can be explicitly specified.
3837
If `clone_from` is used, and the repository is being instantiated into a non-empty directory,
3938
e.g. a directory with your trained model files, it will automatically merge them.
4039
4140
Args:
4241
local_dir (``str``):
43-
path (e.g. ``'my_trained_model/'``) to the local directory, where the ``Repository`` will be either initalized.
42+
path (e.g. ``'my_trained_model/'``) to the local directory, where the ``Repository`` will be initalized.
4443
clone_from (``str``, optional):
4544
repository url (e.g. ``'https://huggingface.co/philschmid/playground-tests'``).
4645
use_auth_token (``str`` or ``bool``, `optional`, defaults ``None``):
47-
huggingface_token can be extract from ``HfApi().login(username, password)`` and is used to authenticate against the hub.
46+
huggingface_token can be extract from ``HfApi().login(username, password)`` and is used to authenticate against the hub
47+
(useful from Google Colab for instance).
4848
git_user (``str``, `optional`, defaults ``None``):
4949
will override the ``git config user.name`` for committing and pushing files to the hub.
5050
git_email (``str``, `optional`, defaults ``None``):
@@ -59,15 +59,9 @@ def __init__(
5959
if clone_from is not None:
6060
self.clone_from(repo_url=clone_from, use_auth_token=use_auth_token)
6161
else:
62-
try:
63-
remotes = subprocess.check_output(
64-
["git", "remote", "-v"],
65-
encoding="utf-8",
66-
cwd=self.local_dir,
67-
)
68-
logger.debug("[Repository] has remotes")
69-
logger.debug(remotes)
70-
except subprocess.CalledProcessError:
62+
if os.path.isdir(os.path.join(self.local_dir, ".git")):
63+
logger.debug("[Repository] is a valid git repo")
64+
else:
7165
logger.error(
7266
"If not specifying `clone_from`, you need to pass Repository a valid git clone."
7367
)
@@ -84,19 +78,26 @@ def check_git_versions(self):
8478
print git and git-lfs versions, raises if they aren't installed.
8579
"""
8680
try:
87-
git_version = subprocess.check_output(
88-
["git", "--version"], encoding="utf-8"
89-
).strip()
81+
git_version = subprocess.run(
82+
["git", "--version"],
83+
stderr=subprocess.PIPE,
84+
stdout=subprocess.PIPE,
85+
check=True,
86+
encoding="utf-8",
87+
).stdout.strip()
9088
except FileNotFoundError:
9189
raise EnvironmentError(
9290
"Looks like you do not have git installed, please install."
9391
)
9492

9593
try:
96-
lfs_version = subprocess.check_output(
94+
lfs_version = subprocess.run(
9795
["git-lfs", "--version"],
9896
encoding="utf-8",
99-
).strip()
97+
check=True,
98+
stderr=subprocess.PIPE,
99+
stdout=subprocess.PIPE,
100+
).stdout.strip()
100101
except FileNotFoundError:
101102
raise EnvironmentError(
102103
"Looks like you do not have git-lfs installed, please install."
@@ -126,77 +127,143 @@ def clone_from(self, repo_url: str, use_auth_token: Union[bool, str, None] = Non
126127
repo_url = repo_url.replace(
127128
"https://", f"https://user:{huggingface_token}@"
128129
)
129-
130-
subprocess.run("git lfs install".split(), check=True)
131-
132-
# checks if repository is initialized in a empty repository or in one with files
133-
if len(os.listdir(self.local_dir)) == 0:
134-
subprocess.run(
135-
["git", "clone", repo_url, "."], check=True, cwd=self.local_dir
136-
)
137-
else:
138-
logger.warning(
139-
"[Repository] local_dir is not empty, so let's try to pull the remote over a non-empty folder."
140-
)
141-
subprocess.run("git init".split(), check=True, cwd=self.local_dir)
130+
try:
142131
subprocess.run(
143-
["git", "remote", "add", "origin", repo_url],
132+
"git lfs install".split(),
133+
stderr=subprocess.PIPE,
134+
stdout=subprocess.PIPE,
144135
check=True,
145-
cwd=self.local_dir,
146-
)
147-
subprocess.run("git fetch".split(), check=True, cwd=self.local_dir)
148-
subprocess.run(
149-
"git reset origin/main".split(), check=True, cwd=self.local_dir
150-
)
151-
# TODO(check if we really want the --force flag)
152-
subprocess.run(
153-
"git checkout origin/main -ft".split(), check=True, cwd=self.local_dir
136+
encoding="utf-8",
154137
)
155138

139+
# checks if repository is initialized in a empty repository or in one with files
140+
if len(os.listdir(self.local_dir)) == 0:
141+
subprocess.run(
142+
["git", "clone", repo_url, "."],
143+
stderr=subprocess.PIPE,
144+
stdout=subprocess.PIPE,
145+
check=True,
146+
encoding="utf-8",
147+
cwd=self.local_dir,
148+
)
149+
else:
150+
logger.warning(
151+
"[Repository] local_dir is not empty, so let's try to pull the remote over a non-empty folder."
152+
)
153+
subprocess.run(
154+
"git init".split(),
155+
stderr=subprocess.PIPE,
156+
stdout=subprocess.PIPE,
157+
check=True,
158+
encoding="utf-8",
159+
cwd=self.local_dir,
160+
)
161+
subprocess.run(
162+
["git", "remote", "add", "origin", repo_url],
163+
stderr=subprocess.PIPE,
164+
stdout=subprocess.PIPE,
165+
check=True,
166+
encoding="utf-8",
167+
cwd=self.local_dir,
168+
)
169+
subprocess.run(
170+
"git fetch".split(),
171+
stderr=subprocess.PIPE,
172+
stdout=subprocess.PIPE,
173+
check=True,
174+
encoding="utf-8",
175+
cwd=self.local_dir,
176+
)
177+
subprocess.run(
178+
"git reset origin/main".split(),
179+
stderr=subprocess.PIPE,
180+
stdout=subprocess.PIPE,
181+
encoding="utf-8",
182+
check=True,
183+
cwd=self.local_dir,
184+
)
185+
# TODO(check if we really want the --force flag)
186+
subprocess.run(
187+
"git checkout origin/main -ft".split(),
188+
stderr=subprocess.PIPE,
189+
stdout=subprocess.PIPE,
190+
encoding="utf-8",
191+
check=True,
192+
cwd=self.local_dir,
193+
)
194+
except subprocess.CalledProcessError as exc:
195+
raise EnvironmentError(exc.stderr)
196+
156197
def git_config_username_and_email(
157198
self, git_user: Optional[str] = None, git_email: Optional[str] = None
158199
):
159200
"""
160201
sets git user name and email (only in the current repo)
161202
"""
162-
if git_user is not None:
163-
subprocess.run(
164-
f"git config user.name {git_user}".split(),
165-
check=True,
166-
cwd=self.local_dir,
167-
)
168-
if git_email is not None:
169-
subprocess.run(
170-
f"git config user.email {git_email}".split(),
171-
check=True,
172-
cwd=self.local_dir,
173-
)
203+
try:
204+
if git_user is not None:
205+
subprocess.run(
206+
["git", "config", "user.name", git_user],
207+
stderr=subprocess.PIPE,
208+
stdout=subprocess.PIPE,
209+
check=True,
210+
encoding="utf-8",
211+
cwd=self.local_dir,
212+
)
213+
if git_email is not None:
214+
subprocess.run(
215+
["git", "config", "user.email", git_email],
216+
stderr=subprocess.PIPE,
217+
stdout=subprocess.PIPE,
218+
check=True,
219+
encoding="utf-8",
220+
cwd=self.local_dir,
221+
)
222+
except subprocess.CalledProcessError as exc:
223+
raise EnvironmentError(exc.stderr)
174224

175225
def lfs_track(self, patterns: Union[str, List[str]]):
176226
"""
177227
Tell git-lfs to track those files.
178228
"""
179229
if isinstance(patterns, str):
180230
patterns = [patterns]
181-
for pattern in patterns:
182-
subprocess.run(
183-
["git", "lfs", "track", pattern], check=True, cwd=self.local_dir
184-
)
231+
try:
232+
for pattern in patterns:
233+
subprocess.run(
234+
["git", "lfs", "track", pattern],
235+
stderr=subprocess.PIPE,
236+
stdout=subprocess.PIPE,
237+
check=True,
238+
encoding="utf-8",
239+
cwd=self.local_dir,
240+
)
241+
except subprocess.CalledProcessError as exc:
242+
raise EnvironmentError(exc.stderr)
185243

186244
def lfs_enable_largefiles(self):
187245
"""
188246
HF-specific. This enables upload support of files >5GB.
189247
"""
190-
subprocess.run(
191-
"git config lfs.customtransfer.multipart.path huggingface-cli".split(),
192-
check=True,
193-
cwd=self.local_dir,
194-
)
195-
subprocess.run(
196-
f"git config lfs.customtransfer.multipart.args {LFS_MULTIPART_UPLOAD_COMMAND}".split(),
197-
check=True,
198-
cwd=self.local_dir,
199-
)
248+
try:
249+
subprocess.run(
250+
"git config lfs.customtransfer.multipart.path huggingface-cli".split(),
251+
stderr=subprocess.PIPE,
252+
stdout=subprocess.PIPE,
253+
check=True,
254+
encoding="utf-8",
255+
cwd=self.local_dir,
256+
)
257+
subprocess.run(
258+
f"git config lfs.customtransfer.multipart.args {LFS_MULTIPART_UPLOAD_COMMAND}".split(),
259+
stderr=subprocess.PIPE,
260+
stdout=subprocess.PIPE,
261+
check=True,
262+
encoding="utf-8",
263+
cwd=self.local_dir,
264+
)
265+
except subprocess.CalledProcessError as exc:
266+
raise EnvironmentError(exc.stderr)
200267

201268
def git_pull(self, rebase: Optional[bool] = False):
202269
"""
@@ -205,27 +272,69 @@ def git_pull(self, rebase: Optional[bool] = False):
205272
args = "git pull".split()
206273
if rebase:
207274
args.append("--rebase")
208-
subprocess.run(args, check=True, cwd=self.local_dir)
275+
try:
276+
subprocess.run(
277+
args,
278+
stderr=subprocess.PIPE,
279+
stdout=subprocess.PIPE,
280+
check=True,
281+
encoding="utf-8",
282+
cwd=self.local_dir,
283+
)
284+
except subprocess.CalledProcessError as exc:
285+
raise EnvironmentError(exc.stderr)
209286

210287
def git_add(self, pattern="."):
211288
"""
212289
git add
213290
"""
214-
subprocess.run("git add .".split(), check=True, cwd=self.local_dir)
291+
try:
292+
subprocess.run(
293+
["git", "add", pattern],
294+
stderr=subprocess.PIPE,
295+
stdout=subprocess.PIPE,
296+
check=True,
297+
encoding="utf-8",
298+
cwd=self.local_dir,
299+
)
300+
except subprocess.CalledProcessError as exc:
301+
raise EnvironmentError(exc.stderr)
215302

216303
def git_commit(self, commit_message="commit files to HF hub"):
217304
"""
218305
git commit
219306
"""
220-
subprocess.run(
221-
["git", "commit", "-m", commit_message], check=True, cwd=self.local_dir
222-
)
307+
try:
308+
subprocess.run(
309+
["git", "commit", "-m", commit_message],
310+
stderr=subprocess.PIPE,
311+
stdout=subprocess.PIPE,
312+
check=True,
313+
encoding="utf-8",
314+
cwd=self.local_dir,
315+
)
316+
except subprocess.CalledProcessError as exc:
317+
if len(exc.stderr) > 0:
318+
raise EnvironmentError(exc.stderr)
319+
else:
320+
raise EnvironmentError(exc.stdout)
223321

224322
def git_push(self):
225323
"""
226324
git push
227325
"""
228-
subprocess.run("git push".split(), check=True, cwd=self.local_dir)
326+
try:
327+
result = subprocess.run(
328+
"git push".split(),
329+
stderr=subprocess.PIPE,
330+
stdout=subprocess.PIPE,
331+
check=True,
332+
encoding="utf-8",
333+
cwd=self.local_dir,
334+
)
335+
logger.info(result.stdout)
336+
except subprocess.CalledProcessError as exc:
337+
raise EnvironmentError(exc.stderr)
229338

230339
def push_to_hub(self, commit_message="commit files to HF hub"):
231340
"""

0 commit comments

Comments
 (0)