From d1185aa6fab2c12016e92ac9b9e448984cdc6c5d Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Mon, 27 Apr 2020 17:01:08 -0700 Subject: [PATCH 1/3] fetch-pack: return enum from process_acks() process_acks() returns 0, 1, or 2, depending on whether "ready" was received and if not, whether at least one commit was found to be common. Replace these magic numbers with a documented enum. Signed-off-by: Jonathan Tan Reviewed-by: Jonathan Nieder Signed-off-by: Junio C Hamano --- fetch-pack.c | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/fetch-pack.c b/fetch-pack.c index 1734a573b0..66cd258c38 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -1268,9 +1268,29 @@ static int process_section_header(struct packet_reader *reader, return ret; } -static int process_acks(struct fetch_negotiator *negotiator, - struct packet_reader *reader, - struct oidset *common) +enum common_found { + /* + * No commit was found to be possessed by both the client and the + * server, and "ready" was not received. + */ + NO_COMMON_FOUND, + + /* + * At least one commit was found to be possessed by both the client and + * the server, and "ready" was not received. + */ + COMMON_FOUND, + + /* + * "ready" was received, indicating that the server is ready to send + * the packfile without any further negotiation. + */ + READY +}; + +static enum common_found process_acks(struct fetch_negotiator *negotiator, + struct packet_reader *reader, + struct oidset *common) { /* received */ int received_ready = 0; @@ -1319,8 +1339,8 @@ static int process_acks(struct fetch_negotiator *negotiator, if (!received_ready && reader->status != PACKET_READ_FLUSH) die(_("expected no other sections to be sent after no 'ready'")); - /* return 0 if no common, 1 if there are common, or 2 if ready */ - return received_ready ? 2 : (received_ack ? 1 : 0); + return received_ready ? READY : + (received_ack ? COMMON_FOUND : NO_COMMON_FOUND); } static void receive_shallow_info(struct fetch_pack_args *args, @@ -1508,13 +1528,13 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, case FETCH_PROCESS_ACKS: /* Process ACKs/NAKs */ switch (process_acks(negotiator, &reader, &common)) { - case 2: + case READY: state = FETCH_GET_PACK; break; - case 1: + case COMMON_FOUND: in_vain = 0; /* fallthrough */ - default: + case NO_COMMON_FOUND: state = FETCH_SEND_REQUEST; break; } From 4fa3f00abb55c3334cce71e201a5ff2c70f8561f Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Mon, 27 Apr 2020 17:01:09 -0700 Subject: [PATCH 2/3] fetch-pack: in protocol v2, in_vain only after ACK When fetching, Git stops negotiation when it has sent at least MAX_IN_VAIN (which is 256) "have" lines without having any of them ACK-ed. But this is supposed to trigger only after the first ACK, as pack-protocol.txt says: However, the 256 limit *only* turns on in the canonical client implementation if we have received at least one "ACK %s continue" during a prior round. This helps to ensure that at least one common ancestor is found before we give up entirely. The code path for protocol v0 observes this, but not protocol v2, resulting in shorter negotiation rounds but significantly larger packfiles. Teach the code path for protocol v2 to check this criterion only after at least one ACK was received. Signed-off-by: Jonathan Tan Reviewed-by: Jonathan Nieder Signed-off-by: Junio C Hamano --- fetch-pack.c | 13 +++++++++---- t/t5500-fetch-pack.sh | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/fetch-pack.c b/fetch-pack.c index 66cd258c38..3891f8bb86 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -1143,6 +1143,7 @@ static void add_common(struct strbuf *req_buf, struct oidset *common) } static int add_haves(struct fetch_negotiator *negotiator, + int seen_ack, struct strbuf *req_buf, int *haves_to_send, int *in_vain) { @@ -1157,7 +1158,7 @@ static int add_haves(struct fetch_negotiator *negotiator, } *in_vain += haves_added; - if (!haves_added || *in_vain >= MAX_IN_VAIN) { + if (!haves_added || (seen_ack && *in_vain >= MAX_IN_VAIN)) { /* Send Done */ packet_buf_write(req_buf, "done\n"); ret = 1; @@ -1173,7 +1174,7 @@ static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, struct fetch_pack_args *args, const struct ref *wants, struct oidset *common, int *haves_to_send, int *in_vain, - int sideband_all) + int sideband_all, int seen_ack) { int ret = 0; struct strbuf req_buf = STRBUF_INIT; @@ -1230,7 +1231,8 @@ static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, add_common(&req_buf, common); /* Add initial haves */ - ret = add_haves(negotiator, &req_buf, haves_to_send, in_vain); + ret = add_haves(negotiator, seen_ack, &req_buf, + haves_to_send, in_vain); } /* Send request */ @@ -1464,6 +1466,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, int haves_to_send = INITIAL_FLUSH; struct fetch_negotiator negotiator_alloc; struct fetch_negotiator *negotiator; + int seen_ack = 0; if (args->no_dependents) { negotiator = NULL; @@ -1520,7 +1523,8 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, if (send_fetch_request(negotiator, fd[1], args, ref, &common, &haves_to_send, &in_vain, - reader.use_sideband)) + reader.use_sideband, + seen_ack)) state = FETCH_GET_PACK; else state = FETCH_PROCESS_ACKS; @@ -1533,6 +1537,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, break; case COMMON_FOUND: in_vain = 0; + seen_ack = 1; /* fallthrough */ case NO_COMMON_FOUND: state = FETCH_SEND_REQUEST; diff --git a/t/t5500-fetch-pack.sh b/t/t5500-fetch-pack.sh index 6b97923964..95ed08db1b 100755 --- a/t/t5500-fetch-pack.sh +++ b/t/t5500-fetch-pack.sh @@ -385,6 +385,24 @@ test_expect_success 'clone shallow with packed refs' ' test_cmp count8.expected count8.actual ' +test_expect_success 'in_vain not triggered before first ACK' ' + rm -rf myserver myclient trace && + git init myserver && + test_commit -C myserver foo && + git clone "file://$(pwd)/myserver" myclient && + + # MAX_IN_VAIN is 256. Because of batching, the client will send 496 + # (16+32+64+128+256) commits, not 256, before giving up. So create 496 + # irrelevant commits. + test_commit_bulk -C myclient 496 && + + # The new commit that the client wants to fetch. + test_commit -C myserver bar && + + GIT_TRACE_PACKET="$(pwd)/trace" git -C myclient fetch --progress origin && + test_i18ngrep "Total 3 " trace +' + test_expect_success 'fetch in shallow repo unreachable shallow objects' ' ( git clone --bare --branch B --single-branch "file://$(pwd)/." no-reflog && From 2f0a093dd640e0dad0b261dae2427f2541b5426c Mon Sep 17 00:00:00 2001 From: Jonathan Tan Date: Mon, 27 Apr 2020 17:01:10 -0700 Subject: [PATCH 3/3] fetch-pack: in protocol v2, reset in_vain upon ACK In the function process_acks() in fetch-pack.c, the variable received_ack is meant to track that an ACK was received, but it was never set. This results in negotiation terminating prematurely through the in_vain counter, when the counter should have been reset upon every ACK. Therefore, reset the in_vain counter upon every ACK. Helped-by: Jonathan Nieder Signed-off-by: Jonathan Tan Reviewed-by: Jonathan Nieder Signed-off-by: Junio C Hamano --- fetch-pack.c | 1 + t/t5500-fetch-pack.sh | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/fetch-pack.c b/fetch-pack.c index 3891f8bb86..10314df25b 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -1307,6 +1307,7 @@ static enum common_found process_acks(struct fetch_negotiator *negotiator, if (skip_prefix(reader->line, "ACK ", &arg)) { struct object_id oid; + received_ack = 1; if (!get_oid_hex(arg, &oid)) { struct commit *commit; oidset_insert(common, &oid); diff --git a/t/t5500-fetch-pack.sh b/t/t5500-fetch-pack.sh index 95ed08db1b..fd39aad93f 100755 --- a/t/t5500-fetch-pack.sh +++ b/t/t5500-fetch-pack.sh @@ -403,6 +403,36 @@ test_expect_success 'in_vain not triggered before first ACK' ' test_i18ngrep "Total 3 " trace ' +test_expect_success 'in_vain resetted upon ACK' ' + rm -rf myserver myclient trace && + git init myserver && + + # Linked list of commits on master. The first is common; the rest are + # not. + test_commit -C myserver first_master_commit && + git clone "file://$(pwd)/myserver" myclient && + test_commit_bulk -C myclient 255 && + + # Another linked list of commits on anotherbranch with no connection to + # master. The first is common; the rest are not. + git -C myserver checkout --orphan anotherbranch && + test_commit -C myserver first_anotherbranch_commit && + git -C myclient fetch origin anotherbranch:refs/heads/anotherbranch && + git -C myclient checkout anotherbranch && + test_commit_bulk -C myclient 255 && + + # The new commit that the client wants to fetch. + git -C myserver checkout master && + test_commit -C myserver to_fetch && + + # The client will send (as "have"s) all 256 commits in anotherbranch + # first. The 256th commit is common between the client and the server, + # and should reset in_vain. This allows negotiation to continue until + # the client reports that first_anotherbranch_commit is common. + GIT_TRACE_PACKET="$(pwd)/trace" git -C myclient fetch --progress origin master && + test_i18ngrep "Total 3 " trace +' + test_expect_success 'fetch in shallow repo unreachable shallow objects' ' ( git clone --bare --branch B --single-branch "file://$(pwd)/." no-reflog &&