From bd0b42aed3084bf66557485fd7d87e975a4f6d4e Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Thu, 10 Jan 2019 11:36:45 -0800 Subject: [PATCH 1/2] fetch-pack: do not take shallow lock unnecessarily When fetching using protocol v2, the remote may send a "shallow-info" section if the client is shallow. If so, Git as the client currently takes the shallow file lock, even if the "shallow-info" section is empty. This is not a problem except that Git does not support taking the shallow file lock after modifying the shallow file, because is_repository_shallow() stores information that is never cleared. And this take-after-modify occurs when Git does a tag-following fetch from a shallow repository on a transport that does not support tag following (since in this case, 2 fetches are performed). To solve this issue, take the shallow file lock (and perform all other shallow processing) only if the "shallow-info" section is non-empty; otherwise, behave as if it were empty. A full solution (probably, ensuring that any action of committing shallow file locks also includes clearing the information stored by is_repository_shallow()) would solve the issue without need for this patch, but this patch is independently useful (as an optimization to prevent writing a file in an unnecessary case), hence why I wrote it. I have included a NEEDSWORK outlining the full solution. Signed-off-by: Jonathan Tan Signed-off-by: Junio C Hamano --- fetch-pack.c | 11 +++++++++-- shallow.c | 7 +++++++ t/t5702-protocol-v2.sh | 18 ++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/fetch-pack.c b/fetch-pack.c index dd6700bda9..5885623ece 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -1232,6 +1232,8 @@ static int process_acks(struct fetch_negotiator *negotiator, static void receive_shallow_info(struct fetch_pack_args *args, struct packet_reader *reader) { + int line_received = 0; + process_section_header(reader, "shallow-info", 0); while (packet_reader_read(reader) == PACKET_READ_NORMAL) { const char *arg; @@ -1241,6 +1243,7 @@ static void receive_shallow_info(struct fetch_pack_args *args, if (get_oid_hex(arg, &oid)) die(_("invalid shallow line: %s"), reader->line); register_shallow(the_repository, &oid); + line_received = 1; continue; } if (skip_prefix(reader->line, "unshallow ", &arg)) { @@ -1253,6 +1256,7 @@ static void receive_shallow_info(struct fetch_pack_args *args, die(_("error in object: %s"), reader->line); if (unregister_shallow(&oid)) die(_("no shallow found: %s"), reader->line); + line_received = 1; continue; } die(_("expected shallow/unshallow, got %s"), reader->line); @@ -1262,8 +1266,11 @@ static void receive_shallow_info(struct fetch_pack_args *args, reader->status != PACKET_READ_DELIM) die(_("error processing shallow info: %d"), reader->status); - setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL); - args->deepen = 1; + if (line_received) { + setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, + NULL); + args->deepen = 1; + } } static void receive_wanted_refs(struct packet_reader *reader, diff --git a/shallow.c b/shallow.c index 02fdbfc554..ce45297940 100644 --- a/shallow.c +++ b/shallow.c @@ -43,6 +43,13 @@ int register_shallow(struct repository *r, const struct object_id *oid) int is_repository_shallow(struct repository *r) { + /* + * NEEDSWORK: This function updates + * r->parsed_objects->{is_shallow,shallow_stat} as a side effect but + * there is no corresponding function to clear them when the shallow + * file is updated. + */ + FILE *fp; char buf[1024]; const char *path = r->parsed_objects->alternate_shallow_file; diff --git a/t/t5702-protocol-v2.sh b/t/t5702-protocol-v2.sh index 0f2b09ebb8..fd164d414d 100755 --- a/t/t5702-protocol-v2.sh +++ b/t/t5702-protocol-v2.sh @@ -471,6 +471,24 @@ test_expect_success 'upload-pack respects client shallows' ' grep "fetch< version 2" trace ' +test_expect_success 'ensure that multiple fetches in same process from a shallow repo works' ' + rm -rf server client trace && + + test_create_repo server && + test_commit -C server one && + test_commit -C server two && + test_commit -C server three && + git clone --shallow-exclude two "file://$(pwd)/server" client && + + git -C server tag -a -m "an annotated tag" twotag two && + + # Triggers tag following (thus, 2 fetches in one process) + GIT_TRACE_PACKET="$(pwd)/trace" git -C client -c protocol.version=2 \ + fetch --shallow-exclude one origin && + # Ensure that protocol v2 is used + grep "fetch< version 2" trace +' + # Test protocol v2 with 'http://' transport # . "$TEST_DIRECTORY"/lib-httpd.sh From 5056cf4a62338c013a723e37cbe1f525f1dcc29d Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Tue, 18 Dec 2018 13:24:35 -0800 Subject: [PATCH 2/2] upload-pack: teach deepen-relative in protocol v2 Commit 685fbd3291 ("fetch-pack: perform a fetch using v2", 2018-03-15) attempted to teach Git deepen-relative in protocol v2 (among other things), but it didn't work: (1) fetch-pack.c needs to emit "deepen-relative". (2) upload-pack.c needs to ensure that the correct deepen_relative variable is passed to deepen() (there are two - the static variable and the one in struct upload_pack_data). (3) Before deepen() computes the list of reachable shallows, it first needs to mark all "our" refs as OUR_REF. v2 currently does not do this, because unlike v0, it is not needed otherwise. Fix all this and include a test demonstrating that it works now. For (2), the static variable deepen_relative is also eliminated, removing a source of confusion. Signed-off-by: Jonathan Tan Reviewed-by: Josh Steadmon Signed-off-by: Junio C Hamano --- fetch-pack.c | 2 ++ t/t5702-protocol-v2.sh | 29 +++++++++++++++++++++++++++++ upload-pack.c | 17 +++++++++++++++-- 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/fetch-pack.c b/fetch-pack.c index 5885623ece..577faa6229 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -1007,6 +1007,8 @@ static void add_shallow_requests(struct strbuf *req_buf, packet_buf_write(req_buf, "deepen-not %s", s->string); } } + if (args->deepen_relative) + packet_buf_write(req_buf, "deepen-relative\n"); } static void add_wants(int no_dependents, const struct ref *wants, struct strbuf *req_buf) diff --git a/t/t5702-protocol-v2.sh b/t/t5702-protocol-v2.sh index fd164d414d..a738c0c4ce 100755 --- a/t/t5702-protocol-v2.sh +++ b/t/t5702-protocol-v2.sh @@ -489,6 +489,35 @@ test_expect_success 'ensure that multiple fetches in same process from a shallow grep "fetch< version 2" trace ' +test_expect_success 'deepen-relative' ' + rm -rf server client trace && + + test_create_repo server && + test_commit -C server one && + test_commit -C server two && + test_commit -C server three && + git clone --depth 1 "file://$(pwd)/server" client && + test_commit -C server four && + + # Sanity check that only "three" is downloaded + git -C client log --pretty=tformat:%s master >actual && + echo three >expected && + test_cmp expected actual && + + GIT_TRACE_PACKET="$(pwd)/trace" git -C client -c protocol.version=2 \ + fetch --deepen=1 origin && + # Ensure that protocol v2 is used + grep "fetch< version 2" trace && + + git -C client log --pretty=tformat:%s origin/master >actual && + cat >expected <<-\EOF && + four + three + two + EOF + test_cmp expected actual +' + # Test protocol v2 with 'http://' transport # . "$TEST_DIRECTORY"/lib-httpd.sh diff --git a/upload-pack.c b/upload-pack.c index 5e81f1ff24..9df27b55a0 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -43,7 +43,6 @@ static timestamp_t oldest_have; -static int deepen_relative; static int multi_ack; static int no_done; static int use_thin_pack, use_ofs_delta, use_include_tag; @@ -662,6 +661,9 @@ static void send_unshallow(const struct object_array *shallows, } } +static int check_ref(const char *refname_full, const struct object_id *oid, + int flag, void *cb_data); + static void deepen(int depth, int deepen_relative, struct object_array *shallows, struct object_array *want_obj) { @@ -676,6 +678,13 @@ static void deepen(int depth, int deepen_relative, struct object_array reachable_shallows = OBJECT_ARRAY_INIT; struct commit_list *result; + /* + * Checking for reachable shallows requires that our refs be + * marked with OUR_REF. + */ + head_ref_namespaced(check_ref, NULL); + for_each_namespaced_ref(check_ref, NULL); + get_reachable_list(shallows, &reachable_shallows); result = get_shallow_commits(&reachable_shallows, depth + 1, @@ -712,6 +721,7 @@ static void deepen_by_rev_list(int ac, const char **av, static int send_shallow_list(int depth, int deepen_rev_list, timestamp_t deepen_since, struct string_list *deepen_not, + int deepen_relative, struct object_array *shallows, struct object_array *want_obj) { @@ -834,6 +844,7 @@ static void receive_needs(struct object_array *want_obj) int has_non_tip = 0; timestamp_t deepen_since = 0; int deepen_rev_list = 0; + int deepen_relative = 0; shallow_nr = 0; for (;;) { @@ -925,7 +936,8 @@ static void receive_needs(struct object_array *want_obj) return; if (send_shallow_list(depth, deepen_rev_list, deepen_since, - &deepen_not, &shallows, want_obj)) + &deepen_not, deepen_relative, &shallows, + want_obj)) packet_flush(1); object_array_clear(&shallows); } @@ -1398,6 +1410,7 @@ static void send_shallow_info(struct upload_pack_data *data, if (!send_shallow_list(data->depth, data->deepen_rev_list, data->deepen_since, &data->deepen_not, + data->deepen_relative, &data->shallows, want_obj) && is_repository_shallow(the_repository)) deepen(INFINITE_DEPTH, data->deepen_relative, &data->shallows,