From c6b69bdbc1b0b914aa0d1e59a29305fce82d6f06 Mon Sep 17 00:00:00 2001 From: Petr Baudis Date: Thu, 27 Jul 2006 23:56:14 +0200 Subject: [PATCH 1/4] Make pull() take some implicit data as explicit arguments Currently it's a bit weird that pull() takes a single argument describing the commit but takes the write_ref from a global variable. This makes it take that as a parameter as well, which might be nicer for the libification in the future, but especially it will make for nicer code when we implement pull()ing multiple commits at once. Signed-off-by: Petr Baudis Signed-off-by: Junio C Hamano --- fetch.c | 6 ++---- fetch.h | 11 ++++------- http-fetch.c | 4 ++-- local-fetch.c | 4 ++-- ssh-fetch.c | 4 ++-- 5 files changed, 12 insertions(+), 17 deletions(-) diff --git a/fetch.c b/fetch.c index 989d7a4788..3255cc6250 100644 --- a/fetch.c +++ b/fetch.c @@ -8,9 +8,6 @@ #include "blob.h" #include "refs.h" -const char *write_ref = NULL; -const char *write_ref_log_details = NULL; - int get_tree = 0; int get_history = 0; int get_all = 0; @@ -213,7 +210,8 @@ static int mark_complete(const char *path, const unsigned char *sha1) return 0; } -int pull(char *target) +int pull(char *target, const char *write_ref, + const char *write_ref_log_details) { struct ref_lock *lock = NULL; unsigned char sha1[20]; diff --git a/fetch.h b/fetch.h index 841bb1af9c..7bda355bc1 100644 --- a/fetch.h +++ b/fetch.h @@ -22,12 +22,6 @@ extern void prefetch(unsigned char *sha1); */ extern int fetch_ref(char *ref, unsigned char *sha1); -/* If set, the ref filename to write the target value to. */ -extern const char *write_ref; - -/* If set additional text will appear in the ref log. */ -extern const char *write_ref_log_details; - /* Set to fetch the target tree. */ extern int get_tree; @@ -46,6 +40,9 @@ extern int get_recover; /* Report what we got under get_verbosely */ extern void pull_say(const char *, const char *); -extern int pull(char *target); +/* If write_ref is set, the ref filename to write the target value to. */ +/* If write_ref_log_details is set, additional text will appear in the ref log. */ +extern int pull(char *target, const char *write_ref, + const char *write_ref_log_details); #endif /* PULL_H */ diff --git a/http-fetch.c b/http-fetch.c index dc286b79f6..963d439f82 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -1216,6 +1216,7 @@ int fetch_ref(char *ref, unsigned char *sha1) int main(int argc, char **argv) { + const char *write_ref = NULL; char *commit_id; char *url; char *path; @@ -1250,7 +1251,6 @@ int main(int argc, char **argv) } commit_id = argv[arg]; url = argv[arg + 1]; - write_ref_log_details = url; http_init(); @@ -1268,7 +1268,7 @@ int main(int argc, char **argv) alt->path_len = strlen(path); } - if (pull(commit_id)) + if (pull(commit_id, write_ref, url)) rc = 1; http_cleanup(); diff --git a/local-fetch.c b/local-fetch.c index a05ac16cd0..452b83396c 100644 --- a/local-fetch.c +++ b/local-fetch.c @@ -204,6 +204,7 @@ static const char local_pull_usage[] = */ int main(int argc, char **argv) { + const char *write_ref = NULL; char *commit_id; int arg = 1; @@ -240,9 +241,8 @@ int main(int argc, char **argv) usage(local_pull_usage); commit_id = argv[arg]; path = argv[arg + 1]; - write_ref_log_details = path; - if (pull(commit_id)) + if (pull(commit_id, write_ref, path)) return 1; return 0; diff --git a/ssh-fetch.c b/ssh-fetch.c index a8a6cfbb30..aef3aa4a05 100644 --- a/ssh-fetch.c +++ b/ssh-fetch.c @@ -123,6 +123,7 @@ static const char ssh_fetch_usage[] = " [-c] [-t] [-a] [-v] [--recover] [-w ref] commit-id url"; int main(int argc, char **argv) { + const char *write_ref = NULL; char *commit_id; char *url; int arg = 1; @@ -159,7 +160,6 @@ int main(int argc, char **argv) } commit_id = argv[arg]; url = argv[arg + 1]; - write_ref_log_details = url; if (setup_connection(&fd_in, &fd_out, prog, url, arg, argv + 1)) return 1; @@ -167,7 +167,7 @@ int main(int argc, char **argv) if (get_version()) return 1; - if (pull(commit_id)) + if (pull(commit_id, write_ref, url)) return 1; return 0; From 4211e4d10cd98b1aeed97bdb6cdebb9411956bb5 Mon Sep 17 00:00:00 2001 From: Petr Baudis Date: Thu, 27 Jul 2006 23:56:17 +0200 Subject: [PATCH 2/4] Make pull() support fetching multiple targets at once pull() now takes an array of arguments instead of just one of each kind. Currently, no users use the new capability, but that'll change. Signed-off-by: Petr Baudis Signed-off-by: Junio C Hamano --- fetch.c | 80 +++++++++++++++++++++++++++++---------------------- fetch.h | 2 +- http-fetch.c | 2 +- local-fetch.c | 2 +- ssh-fetch.c | 2 +- 5 files changed, 50 insertions(+), 38 deletions(-) diff --git a/fetch.c b/fetch.c index 3255cc6250..281df61e7b 100644 --- a/fetch.c +++ b/fetch.c @@ -210,55 +210,67 @@ static int mark_complete(const char *path, const unsigned char *sha1) return 0; } -int pull(char *target, const char *write_ref, +int pull(int targets, char **target, const char **write_ref, const char *write_ref_log_details) { - struct ref_lock *lock = NULL; - unsigned char sha1[20]; + struct ref_lock **lock = xcalloc(targets, sizeof(struct ref_lock *)); + unsigned char *sha1 = xmalloc(targets * 20); char *msg; int ret; + int i; save_commit_buffer = 0; track_object_refs = 0; - if (write_ref) { - lock = lock_ref_sha1(write_ref, NULL, 0); - if (!lock) { - error("Can't lock ref %s", write_ref); - return -1; + + for (i = 0; i < targets; i++) { + if (!write_ref[i]) + continue; + + lock[i] = lock_ref_sha1(write_ref[i], NULL, 0); + if (!lock[i]) { + error("Can't lock ref %s", write_ref[i]); + goto unlock_and_fail; } } if (!get_recover) for_each_ref(mark_complete); - if (interpret_target(target, sha1)) { - error("Could not interpret %s as something to pull", target); - if (lock) - unlock_ref(lock); - return -1; - } - if (process(lookup_unknown_object(sha1))) { - if (lock) - unlock_ref(lock); - return -1; - } - if (loop()) { - if (lock) - unlock_ref(lock); - return -1; + for (i = 0; i < targets; i++) { + if (interpret_target(target[i], &sha1[20 * i])) { + error("Could not interpret %s as something to pull", target[i]); + goto unlock_and_fail; + } + if (process(lookup_unknown_object(&sha1[20 * i]))) + goto unlock_and_fail; } - if (write_ref) { - if (write_ref_log_details) { - msg = xmalloc(strlen(write_ref_log_details) + 12); - sprintf(msg, "fetch from %s", write_ref_log_details); - } - else - msg = NULL; - ret = write_ref_sha1(lock, sha1, msg ? msg : "fetch (unknown)"); - if (msg) - free(msg); - return ret; + if (loop()) + goto unlock_and_fail; + + if (write_ref_log_details) { + msg = xmalloc(strlen(write_ref_log_details) + 12); + sprintf(msg, "fetch from %s", write_ref_log_details); + } else { + msg = NULL; } + for (i = 0; i < targets; i++) { + if (!write_ref[i]) + continue; + ret = write_ref_sha1(lock[i], &sha1[20 * i], msg ? msg : "fetch (unknown)"); + lock[i] = NULL; + if (ret) + goto unlock_and_fail; + } + if (msg) + free(msg); + return 0; + + +unlock_and_fail: + for (i = 0; i < targets; i++) + if (lock[i]) + unlock_ref(lock[i]); + return -1; } diff --git a/fetch.h b/fetch.h index 7bda355bc1..75e48af780 100644 --- a/fetch.h +++ b/fetch.h @@ -42,7 +42,7 @@ extern void pull_say(const char *, const char *); /* If write_ref is set, the ref filename to write the target value to. */ /* If write_ref_log_details is set, additional text will appear in the ref log. */ -extern int pull(char *target, const char *write_ref, +extern int pull(int targets, char **target, const char **write_ref, const char *write_ref_log_details); #endif /* PULL_H */ diff --git a/http-fetch.c b/http-fetch.c index 963d439f82..bc67db1add 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -1268,7 +1268,7 @@ int main(int argc, char **argv) alt->path_len = strlen(path); } - if (pull(commit_id, write_ref, url)) + if (pull(1, &commit_id, &write_ref, url)) rc = 1; http_cleanup(); diff --git a/local-fetch.c b/local-fetch.c index 452b83396c..1be73904f1 100644 --- a/local-fetch.c +++ b/local-fetch.c @@ -242,7 +242,7 @@ int main(int argc, char **argv) commit_id = argv[arg]; path = argv[arg + 1]; - if (pull(commit_id, write_ref, path)) + if (pull(1, &commit_id, &write_ref, path)) return 1; return 0; diff --git a/ssh-fetch.c b/ssh-fetch.c index aef3aa4a05..6e16568f88 100644 --- a/ssh-fetch.c +++ b/ssh-fetch.c @@ -167,7 +167,7 @@ int main(int argc, char **argv) if (get_version()) return 1; - if (pull(commit_id, write_ref, url)) + if (pull(1, &commit_id, &write_ref, url)) return 1; return 0; From 8e87ca661513d9f4b737295b234e93767cd2ee0c Mon Sep 17 00:00:00 2001 From: Petr Baudis Date: Thu, 27 Jul 2006 23:56:19 +0200 Subject: [PATCH 3/4] Teach git-local-fetch the --stdin switch This makes it possible to fetch many commits (refs) at once, greatly speeding up cg-clone. Signed-off-by: Petr Baudis Signed-off-by: Junio C Hamano --- Documentation/git-local-fetch.txt | 6 +++++ fetch.c | 40 +++++++++++++++++++++++++++++++ fetch.h | 6 +++++ local-fetch.c | 32 +++++++++++++++++-------- 4 files changed, 74 insertions(+), 10 deletions(-) diff --git a/Documentation/git-local-fetch.txt b/Documentation/git-local-fetch.txt index 87abec1c4e..2fbdfe086a 100644 --- a/Documentation/git-local-fetch.txt +++ b/Documentation/git-local-fetch.txt @@ -29,6 +29,12 @@ OPTIONS Writes the commit-id into the filename under $GIT_DIR/refs/ on the local end after the transfer is complete. +--stdin:: + Instead of a commit id on the commandline (which is not expected in this + case), 'git-local-fetch' expects lines on stdin in the format + + ['\t'] + Author ------ Written by Junio C Hamano diff --git a/fetch.c b/fetch.c index 281df61e7b..2151c7b78c 100644 --- a/fetch.c +++ b/fetch.c @@ -7,6 +7,7 @@ #include "tag.h" #include "blob.h" #include "refs.h" +#include "strbuf.h" int get_tree = 0; int get_history = 0; @@ -210,6 +211,45 @@ static int mark_complete(const char *path, const unsigned char *sha1) return 0; } +int pull_targets_stdin(char ***target, const char ***write_ref) +{ + int targets = 0, targets_alloc = 0; + struct strbuf buf; + *target = NULL; *write_ref = NULL; + strbuf_init(&buf); + while (1) { + char *rf_one = NULL; + char *tg_one; + + read_line(&buf, stdin, '\n'); + if (buf.eof) + break; + tg_one = buf.buf; + rf_one = strchr(tg_one, '\t'); + if (rf_one) + *rf_one++ = 0; + + if (targets >= targets_alloc) { + targets_alloc = targets_alloc ? targets_alloc * 2 : 64; + *target = xrealloc(*target, targets_alloc * sizeof(**target)); + *write_ref = xrealloc(*write_ref, targets_alloc * sizeof(**write_ref)); + } + (*target)[targets] = strdup(tg_one); + (*write_ref)[targets] = rf_one ? strdup(rf_one) : NULL; + targets++; + } + return targets; +} + +void pull_targets_free(int targets, char **target, const char **write_ref) +{ + while (targets--) { + free(target[targets]); + if (write_ref[targets]) + free((char *) write_ref[targets]); + } +} + int pull(int targets, char **target, const char **write_ref, const char *write_ref_log_details) { diff --git a/fetch.h b/fetch.h index 75e48af780..be48c6f190 100644 --- a/fetch.h +++ b/fetch.h @@ -40,6 +40,12 @@ extern int get_recover; /* Report what we got under get_verbosely */ extern void pull_say(const char *, const char *); +/* Load pull targets from stdin */ +extern int pull_targets_stdin(char ***target, const char ***write_ref); + +/* Free up loaded targets */ +extern void pull_targets_free(int targets, char **target, const char **write_ref); + /* If write_ref is set, the ref filename to write the target value to. */ /* If write_ref_log_details is set, additional text will appear in the ref log. */ extern int pull(int targets, char **target, const char **write_ref, diff --git a/local-fetch.c b/local-fetch.c index 1be73904f1..b216bdd557 100644 --- a/local-fetch.c +++ b/local-fetch.c @@ -8,8 +8,9 @@ static int use_link = 0; static int use_symlink = 0; static int use_filecopy = 1; +static int commits_on_stdin = 0; -static char *path; /* "Remote" git repository */ +static const char *path; /* "Remote" git repository */ void prefetch(unsigned char *sha1) { @@ -194,7 +195,7 @@ int fetch_ref(char *ref, unsigned char *sha1) } static const char local_pull_usage[] = -"git-local-fetch [-c] [-t] [-a] [-v] [-w filename] [--recover] [-l] [-s] [-n] commit-id path"; +"git-local-fetch [-c] [-t] [-a] [-v] [-w filename] [--recover] [-l] [-s] [-n] [--stdin] commit-id path"; /* * By default we only use file copy. @@ -202,10 +203,11 @@ static const char local_pull_usage[] = * If -s is specified, then a symlink is attempted. * If -n is _not_ specified, then a regular file-to-file copy is done. */ -int main(int argc, char **argv) +int main(int argc, const char **argv) { - const char *write_ref = NULL; - char *commit_id; + int commits; + const char **write_ref = NULL; + char **commit_id; int arg = 1; setup_git_directory(); @@ -230,20 +232,30 @@ int main(int argc, char **argv) else if (argv[arg][1] == 'v') get_verbosely = 1; else if (argv[arg][1] == 'w') - write_ref = argv[++arg]; + write_ref = &argv[++arg]; else if (!strcmp(argv[arg], "--recover")) get_recover = 1; + else if (!strcmp(argv[arg], "--stdin")) + commits_on_stdin = 1; else usage(local_pull_usage); arg++; } - if (argc < arg + 2) + if (argc < arg + 2 - commits_on_stdin) usage(local_pull_usage); - commit_id = argv[arg]; - path = argv[arg + 1]; + if (commits_on_stdin) { + commits = pull_targets_stdin(&commit_id, &write_ref); + } else { + commit_id = (char **) &argv[arg++]; + commits = 1; + } + path = argv[arg]; - if (pull(1, &commit_id, &write_ref, path)) + if (pull(commits, commit_id, write_ref, path)) return 1; + if (commits_on_stdin) + pull_targets_free(commits, commit_id, write_ref); + return 0; } From 8e29f6a07e321a24f1d4f579817f5a8a43cdac05 Mon Sep 17 00:00:00 2001 From: Petr Baudis Date: Thu, 27 Jul 2006 23:56:22 +0200 Subject: [PATCH 4/4] Teach git-http-fetch the --stdin switch Speeds up things quite a lot when fetching tags with Cogito. Signed-off-by: Petr Baudis Signed-off-by: Junio C Hamano --- Documentation/git-http-fetch.txt | 8 +++++- http-fetch.c | 45 ++++++++++++++++++++------------ 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/Documentation/git-http-fetch.txt b/Documentation/git-http-fetch.txt index bc1a132891..3d508094af 100644 --- a/Documentation/git-http-fetch.txt +++ b/Documentation/git-http-fetch.txt @@ -8,7 +8,7 @@ git-http-fetch - downloads a remote git repository via HTTP SYNOPSIS -------- -'git-http-fetch' [-c] [-t] [-a] [-d] [-v] [-w filename] [--recover] +'git-http-fetch' [-c] [-t] [-a] [-d] [-v] [-w filename] [--recover] [--stdin] DESCRIPTION ----------- @@ -33,6 +33,12 @@ commit-id:: Writes the commit-id into the filename under $GIT_DIR/refs/ on the local end after the transfer is complete. +--stdin:: + Instead of a commit id on the commandline (which is not expected in this + case), 'git-http-fetch' expects lines on stdin in the format + + ['\t'] + Author ------ Written by Linus Torvalds diff --git a/http-fetch.c b/http-fetch.c index bc67db1add..1aad39b4d8 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -36,6 +36,8 @@ enum XML_Status { #define PREV_BUF_SIZE 4096 #define RANGE_HEADER_SIZE 30 +static int commits_on_stdin = 0; + static int got_alternates = -1; static int corrupt_object_found = 0; @@ -43,7 +45,7 @@ static struct curl_slist *no_pragma_header; struct alt_base { - char *base; + const char *base; int path_len; int got_indices; struct packed_git *packs; @@ -81,7 +83,7 @@ struct object_request }; struct alternates_request { - char *base; + const char *base; char *url; struct buffer *buffer; struct active_request_slot *slot; @@ -142,7 +144,7 @@ static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb, return size; } -static void fetch_alternates(char *base); +static void fetch_alternates(const char *base); static void process_object_response(void *callback_data); @@ -507,7 +509,7 @@ static void process_alternates_response(void *callback_data) (struct alternates_request *)callback_data; struct active_request_slot *slot = alt_req->slot; struct alt_base *tail = alt; - char *base = alt_req->base; + const char *base = alt_req->base; static const char null_byte = '\0'; char *data; int i = 0; @@ -612,7 +614,7 @@ static void process_alternates_response(void *callback_data) got_alternates = 1; } -static void fetch_alternates(char *base) +static void fetch_alternates(const char *base) { struct buffer buffer; char *url; @@ -1185,7 +1187,7 @@ int fetch_ref(char *ref, unsigned char *sha1) char *url; char hex[42]; struct buffer buffer; - char *base = alt->base; + const char *base = alt->base; struct active_request_slot *slot; struct slot_results results; buffer.size = 41; @@ -1214,11 +1216,12 @@ int fetch_ref(char *ref, unsigned char *sha1) return 0; } -int main(int argc, char **argv) +int main(int argc, const char **argv) { - const char *write_ref = NULL; - char *commit_id; - char *url; + int commits; + const char **write_ref = NULL; + char **commit_id; + const char *url; char *path; int arg = 1; int rc = 0; @@ -1238,19 +1241,26 @@ int main(int argc, char **argv) } else if (argv[arg][1] == 'v') { get_verbosely = 1; } else if (argv[arg][1] == 'w') { - write_ref = argv[arg + 1]; + write_ref = &argv[arg + 1]; arg++; } else if (!strcmp(argv[arg], "--recover")) { get_recover = 1; + } else if (!strcmp(argv[arg], "--stdin")) { + commits_on_stdin = 1; } arg++; } - if (argc < arg + 2) { - usage("git-http-fetch [-c] [-t] [-a] [-v] [--recover] [-w ref] commit-id url"); + if (argc < arg + 2 - commits_on_stdin) { + usage("git-http-fetch [-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdin] commit-id url"); return 1; } - commit_id = argv[arg]; - url = argv[arg + 1]; + if (commits_on_stdin) { + commits = pull_targets_stdin(&commit_id, &write_ref); + } else { + commit_id = (char **) &argv[arg++]; + commits = 1; + } + url = argv[arg]; http_init(); @@ -1268,13 +1278,16 @@ int main(int argc, char **argv) alt->path_len = strlen(path); } - if (pull(1, &commit_id, &write_ref, url)) + if (pull(commits, commit_id, write_ref, url)) rc = 1; http_cleanup(); curl_slist_free_all(no_pragma_header); + if (commits_on_stdin) + pull_targets_free(commits, commit_id, write_ref); + if (corrupt_object_found) { fprintf(stderr, "Some loose object were found to be corrupt, but they might be just\n"