Skip to content

Commit e248619

Browse files
gitstergitster
committed
Merge branch 'rc/http-push'
* rc/http-push: (22 commits) http*: add helper methods for fetching objects (loose) http*: add helper methods for fetching packs http: use new http API in fetch_index() http*: add http_get_info_packs http-push.c::fetch_symref(): use the new http API http-push.c::remote_exists(): use the new http API http.c::http_fetch_ref(): use the new http API transport.c::get_refs_via_curl(): use the new http API http.c: new functions for the http API http: create function end_url_with_slash http*: move common variables and macros to http.[ch] transport.c::get_refs_via_curl(): do not leak refs_url Don't expect verify_pack() callers to set pack_size http-push: do not SEGV after fetching a bad pack idx file http*: copy string returned by sha1_to_hex http-walker: verify remote packs http-push, http-walker: style fixes t5550-http-fetch: test fetching of packed objects http-push: fix missing "#ifdef USE_CURL_MULTI" around "is_running_queue" http-push: send out fetch requests on queue ...
2 parents cec3f98 + 5424bc5 commit e248619

File tree

8 files changed

+904
-993
lines changed

8 files changed

+904
-993
lines changed

‎http-push.c

Lines changed: 93 additions & 492 deletions
Large diffs are not rendered by default.

‎http-walker.c

Lines changed: 87 additions & 452 deletions
Large diffs are not rendered by default.

‎http.c

Lines changed: 577 additions & 24 deletions
Large diffs are not rendered by default.

‎http.h

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ extern void http_cleanup(void);
9393

9494
extern int data_received;
9595
extern int active_requests;
96+
extern int http_is_verbose;
9697

9798
extern char curl_errorstr[CURL_ERROR_SIZE];
9899

@@ -109,6 +110,90 @@ static inline int missing__target(int code, int result)
109110

110111
#define missing_target(a) missing__target((a)->http_code, (a)->curl_result)
111112

113+
/* Helpers for modifying and creating URLs */
114+
extern void append_remote_object_url(struct strbuf *buf, const char *url,
115+
const char *hex,
116+
int only_two_digit_prefix);
117+
extern char *get_remote_object_url(const char *url, const char *hex,
118+
int only_two_digit_prefix);
119+
120+
/* Options for http_request_*() */
121+
#define HTTP_NO_CACHE 1
122+
123+
/* Return values for http_request_*() */
124+
#define HTTP_OK 0
125+
#define HTTP_MISSING_TARGET 1
126+
#define HTTP_ERROR 2
127+
#define HTTP_START_FAILED 3
128+
129+
/*
130+
* Requests an url and stores the result in a strbuf.
131+
*
132+
* If the result pointer is NULL, a HTTP HEAD request is made instead of GET.
133+
*/
134+
int http_get_strbuf(const char *url, struct strbuf *result, int options);
135+
136+
/*
137+
* Downloads an url and stores the result in the given file.
138+
*
139+
* If a previous interrupted download is detected (i.e. a previous temporary
140+
* file is still around) the download is resumed.
141+
*/
142+
int http_get_file(const char *url, const char *filename, int options);
143+
144+
/*
145+
* Prints an error message using error() containing url and curl_errorstr,
146+
* and returns ret.
147+
*/
148+
int http_error(const char *url, int ret);
149+
112150
extern int http_fetch_ref(const char *base, struct ref *ref);
113151

152+
/* Helpers for fetching packs */
153+
extern int http_get_info_packs(const char *base_url,
154+
struct packed_git **packs_head);
155+
156+
struct http_pack_request
157+
{
158+
char *url;
159+
struct packed_git *target;
160+
struct packed_git **lst;
161+
FILE *packfile;
162+
char filename[PATH_MAX];
163+
char tmpfile[PATH_MAX];
164+
struct curl_slist *range_header;
165+
struct active_request_slot *slot;
166+
};
167+
168+
extern struct http_pack_request *new_http_pack_request(
169+
struct packed_git *target, const char *base_url);
170+
extern int finish_http_pack_request(struct http_pack_request *preq);
171+
extern void release_http_pack_request(struct http_pack_request *preq);
172+
173+
/* Helpers for fetching object */
174+
struct http_object_request
175+
{
176+
char *url;
177+
char filename[PATH_MAX];
178+
char tmpfile[PATH_MAX];
179+
int localfile;
180+
CURLcode curl_result;
181+
char errorstr[CURL_ERROR_SIZE];
182+
long http_code;
183+
unsigned char sha1[20];
184+
unsigned char real_sha1[20];
185+
git_SHA_CTX c;
186+
z_stream stream;
187+
int zret;
188+
int rename;
189+
struct active_request_slot *slot;
190+
};
191+
192+
extern struct http_object_request *new_http_object_request(
193+
const char *base_url, unsigned char *sha1);
194+
extern void process_http_object_request(struct http_object_request *freq);
195+
extern int finish_http_object_request(struct http_object_request *freq);
196+
extern void abort_http_object_request(struct http_object_request *freq);
197+
extern void release_http_object_request(struct http_object_request *freq);
198+
114199
#endif /* HTTP_H */

‎pack-check.c

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ static int verify_packfile(struct packed_git *p,
4949
const unsigned char *index_base = p->index_data;
5050
git_SHA_CTX ctx;
5151
unsigned char sha1[20], *pack_sig;
52-
off_t offset = 0, pack_sig_ofs = p->pack_size - 20;
52+
off_t offset = 0, pack_sig_ofs = 0;
5353
uint32_t nr_objects, i;
5454
int err = 0;
5555
struct idx_entry *entries;
@@ -61,14 +61,16 @@ static int verify_packfile(struct packed_git *p,
6161
*/
6262

6363
git_SHA1_Init(&ctx);
64-
while (offset < pack_sig_ofs) {
64+
do {
6565
unsigned int remaining;
6666
unsigned char *in = use_pack(p, w_curs, offset, &remaining);
6767
offset += remaining;
68+
if (!pack_sig_ofs)
69+
pack_sig_ofs = p->pack_size - 20;
6870
if (offset > pack_sig_ofs)
6971
remaining -= (unsigned int)(offset - pack_sig_ofs);
7072
git_SHA1_Update(&ctx, in, remaining);
71-
}
73+
} while (offset < pack_sig_ofs);
7274
git_SHA1_Final(sha1, &ctx);
7375
pack_sig = use_pack(p, w_curs, pack_sig_ofs, NULL);
7476
if (hashcmp(sha1, pack_sig))

‎t/t5540-http-push.sh

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,42 @@ test_expect_success ' push to remote repository with unpacked refs' '
6767
test $HEAD = $(git rev-parse --verify HEAD))
6868
'
6969

70+
test_expect_success 'http-push fetches unpacked objects' '
71+
cp -R "$HTTPD_DOCUMENT_ROOT_PATH"/test_repo.git \
72+
"$HTTPD_DOCUMENT_ROOT_PATH"/test_repo_unpacked.git &&
73+
74+
git clone $HTTPD_URL/test_repo_unpacked.git \
75+
"$ROOT_PATH"/fetch_unpacked &&
76+
77+
# By reset, we force git to retrieve the object
78+
(cd "$ROOT_PATH"/fetch_unpacked &&
79+
git reset --hard HEAD^ &&
80+
git remote rm origin &&
81+
git reflog expire --expire=0 --all &&
82+
git prune &&
83+
git push -f -v $HTTPD_URL/test_repo_unpacked.git master)
84+
'
85+
86+
test_expect_success 'http-push fetches packed objects' '
87+
cp -R "$HTTPD_DOCUMENT_ROOT_PATH"/test_repo.git \
88+
"$HTTPD_DOCUMENT_ROOT_PATH"/test_repo_packed.git &&
89+
90+
git clone $HTTPD_URL/test_repo_packed.git \
91+
"$ROOT_PATH"/test_repo_clone_packed &&
92+
93+
(cd "$HTTPD_DOCUMENT_ROOT_PATH"/test_repo_packed.git &&
94+
git --bare repack &&
95+
git --bare prune-packed) &&
96+
97+
# By reset, we force git to retrieve the packed object
98+
(cd "$ROOT_PATH"/test_repo_clone_packed &&
99+
git reset --hard HEAD^ &&
100+
git remote rm origin &&
101+
git reflog expire --expire=0 --all &&
102+
git prune &&
103+
git push -f -v $HTTPD_URL/test_repo_packed.git master)
104+
'
105+
70106
test_expect_success 'create and delete remote branch' '
71107
cd "$ROOT_PATH"/test_repo_clone &&
72108
git checkout -b dev &&

‎t/t5550-http-fetch.sh

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,5 +53,13 @@ test_expect_success 'http remote detects correct HEAD' '
5353
)
5454
'
5555

56+
test_expect_success 'fetch packed objects' '
57+
cp -R "$HTTPD_DOCUMENT_ROOT_PATH"/repo.git "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git &&
58+
cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git &&
59+
git --bare repack &&
60+
git --bare prune-packed &&
61+
git clone $HTTPD_URL/repo_pack.git
62+
'
63+
5664
stop_httpd
5765
test_done

‎transport.c

Lines changed: 13 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -439,9 +439,7 @@ static struct ref *get_refs_via_curl(struct transport *transport, int for_push)
439439
char *ref_name;
440440
char *refs_url;
441441
int i = 0;
442-
443-
struct active_request_slot *slot;
444-
struct slot_results results;
442+
int http_ret;
445443

446444
struct ref *refs = NULL;
447445
struct ref *ref = NULL;
@@ -461,25 +459,16 @@ static struct ref *get_refs_via_curl(struct transport *transport, int for_push)
461459
refs_url = xmalloc(strlen(transport->url) + 11);
462460
sprintf(refs_url, "%s/info/refs", transport->url);
463461

464-
slot = get_active_slot();
465-
slot->results = &results;
466-
curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
467-
curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
468-
curl_easy_setopt(slot->curl, CURLOPT_URL, refs_url);
469-
curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
470-
471-
if (start_active_slot(slot)) {
472-
run_active_slot(slot);
473-
if (results.curl_result != CURLE_OK) {
474-
strbuf_release(&buffer);
475-
if (missing_target(&results))
476-
die("%s not found: did you run git update-server-info on the server?", refs_url);
477-
else
478-
die("%s download error - %s", refs_url, curl_errorstr);
479-
}
480-
} else {
481-
strbuf_release(&buffer);
482-
die("Unable to start HTTP request");
462+
http_ret = http_get_strbuf(refs_url, &buffer, HTTP_NO_CACHE);
463+
switch (http_ret) {
464+
case HTTP_OK:
465+
break;
466+
case HTTP_MISSING_TARGET:
467+
die("%s not found: did you run git update-server-info on the"
468+
" server?", refs_url);
469+
default:
470+
http_error(refs_url, http_ret);
471+
die("HTTP request failed");
483472
}
484473

485474
data = buffer.buf;
@@ -519,6 +508,8 @@ static struct ref *get_refs_via_curl(struct transport *transport, int for_push)
519508
free(ref);
520509
}
521510

511+
strbuf_release(&buffer);
512+
free(refs_url);
522513
return refs;
523514
}
524515

0 commit comments

Comments
 (0)