From 5a3743da326b2e041c65ac12ba0d04a1fed45b58 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Tue, 8 Jun 2021 16:11:39 +0000 Subject: [PATCH 1/4] merge-ort: replace string_list_df_name_compare with faster alternative MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gathering accumulated times from trace2 output on the mega-renames testcase, I saw the following timings (where I'm only showing a few lines to highlight the portions of interest): 10.120 : label:incore_nonrecursive 4.462 : ..label:process_entries 3.143 : ....label:process_entries setup 2.988 : ......label:plist special sort 1.305 : ....label:processing 2.604 : ..label:collect_merge_info 2.018 : ..label:merge_start 1.018 : ..label:renames In the above output, note that the 4.462 seconds for process_entries was split as 3.143 seconds for "process_entries setup" and 1.305 seconds for "processing" (and a little time for other stuff removed from the highlight). Most of the "process_entries setup" time was spent on "plist special sort" which corresponds to the following code: trace2_region_enter("merge", "plist special sort", opt->repo); plist.cmp = string_list_df_name_compare; string_list_sort(&plist); trace2_region_leave("merge", "plist special sort", opt->repo); In other words, in a merge strategy that would be invoked by passing "-sort" to either rebase or merge, sorting an array takes more time than anything else. Serves me right for naming my merge strategy this way. Rewrite the comparison function in a way that does not require finding out the lengths of the strings when comparing them. While at it, tweak the code for our specific case -- no need to handle a variety of modes, for example. The combination of these changes reduced the time spent in "plist special sort" by ~25% in the mega-renames case. For the testcases mentioned in commit 557ac0350d ("merge-ort: begin performance work; instrument with trace2_region_* calls", 2020-10-28), this change improves the performance as follows: Before After no-renames: 5.622 s ± 0.059 s 5.235 s ± 0.042 s mega-renames: 10.127 s ± 0.073 s 9.419 s ± 0.107 s just-one-mega: 500.3 ms ± 3.8 ms 480.1 ms ± 3.9 ms Signed-off-by: Elijah Newren Reviewed-by: Derrick Stolee Signed-off-by: Junio C Hamano --- merge-ort.c | 67 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/merge-ort.c b/merge-ort.c index ed21dc8724..be80143f4d 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -2746,31 +2746,58 @@ simple_cleanup: /*** Function Grouping: functions related to process_entries() ***/ -static int string_list_df_name_compare(const char *one, const char *two) +static int sort_dirs_next_to_their_children(const char *one, const char *two) { - int onelen = strlen(one); - int twolen = strlen(two); + unsigned char c1, c2; + /* - * Here we only care that entries for D/F conflicts are - * adjacent, in particular with the file of the D/F conflict - * appearing before files below the corresponding directory. - * The order of the rest of the list is irrelevant for us. + * Here we only care that entries for directories appear adjacent + * to and before files underneath the directory. We can achieve + * that by pretending to add a trailing slash to every file and + * then sorting. In other words, we do not want the natural + * sorting of + * foo + * foo.txt + * foo/bar + * Instead, we want "foo" to sort as though it were "foo/", so that + * we instead get + * foo.txt + * foo + * foo/bar + * To achieve this, we basically implement our own strcmp, except that + * if we get to the end of either string instead of comparing NUL to + * another character, we compare '/' to it. * - * To achieve this, we sort with df_name_compare and provide - * the mode S_IFDIR so that D/F conflicts will sort correctly. - * We use the mode S_IFDIR for everything else for simplicity, - * since in other cases any changes in their order due to - * sorting cause no problems for us. + * If this unusual "sort as though '/' were appended" perplexes + * you, perhaps it will help to note that this is not the final + * sort. write_tree() will sort again without the trailing slash + * magic, but just on paths immediately under a given tree. + * + * The reason to not use df_name_compare directly was that it was + * just too expensive (we don't have the string lengths handy), so + * it was reimplemented. */ - int cmp = df_name_compare(one, onelen, S_IFDIR, - two, twolen, S_IFDIR); + /* - * Now that 'foo' and 'foo/bar' compare equal, we have to make sure - * that 'foo' comes before 'foo/bar'. + * NOTE: This function will never be called with two equal strings, + * because it is used to sort the keys of a strmap, and strmaps have + * unique keys by construction. That simplifies our c1==c2 handling + * below. */ - if (cmp) - return cmp; - return onelen - twolen; + + while (*one && (*one == *two)) { + one++; + two++; + } + + c1 = *one ? *one : '/'; + c2 = *two ? *two : '/'; + + if (c1 == c2) { + /* Getting here means one is a leading directory of the other */ + return (*one) ? 1 : -1; + } else + return c1 - c2; } static int read_oid_strbuf(struct merge_options *opt, @@ -3489,7 +3516,7 @@ static void process_entries(struct merge_options *opt, trace2_region_leave("merge", "plist copy", opt->repo); trace2_region_enter("merge", "plist special sort", opt->repo); - plist.cmp = string_list_df_name_compare; + plist.cmp = sort_dirs_next_to_their_children; string_list_sort(&plist); trace2_region_leave("merge", "plist special sort", opt->repo); From 61bf4490afffb946787edec63b932551b3089d27 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Tue, 8 Jun 2021 16:11:40 +0000 Subject: [PATCH 2/4] diffcore-rename: avoid unnecessary strdup'ing in break_idx The keys of break_idx are strings from the diff_filepairs of diff_queued_diff. break_idx is only used in location_rename_dst(), and that usage is always before any free'ing of the pairs (and thus the strings in the pairs). As such, there is no need to strdup these keys; we can just reuse the existing strings as-is. The merge logic doesn't make use of break detection, so this does not affect the performance of any of my testcases. It was just a minor unrelated optimization noted in passing while looking at the code. Signed-off-by: Elijah Newren Reviewed-by: Derrick Stolee Signed-off-by: Junio C Hamano --- diffcore-rename.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diffcore-rename.c b/diffcore-rename.c index dfbe65c917..9c0bc5afb4 100644 --- a/diffcore-rename.c +++ b/diffcore-rename.c @@ -54,7 +54,7 @@ static void register_rename_src(struct diff_filepair *p) if (p->broken_pair) { if (!break_idx) { break_idx = xmalloc(sizeof(*break_idx)); - strintmap_init(break_idx, -1); + strintmap_init_with_options(break_idx, -1, NULL, 0); } strintmap_set(break_idx, p->one->path, rename_dst_nr); } From 356da0f98b2c6631b72e69d8638517dd849d28f8 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Tue, 8 Jun 2021 16:11:41 +0000 Subject: [PATCH 3/4] Fix various issues found in comments A random hodge-podge of incorrect or out-of-date comments that I found: * t6423 had a comment that has referred to the wrong test for years; fix it to refer to the right one. * diffcore-rename had a FIXME comment meant to remind myself to investigate if I could make another code change. I later investigated and removed the FIXME, but while cherry-picking the patch to submit upstream I missed the later update. Remove the comment now. * merge-ort had the early part of a comment for a function; I had meant to include the more involved description when I updated the function. Update the comment now. Signed-off-by: Elijah Newren Reviewed-by: Derrick Stolee Signed-off-by: Junio C Hamano --- diffcore-rename.c | 2 +- merge-ort.c | 8 +++++--- t/t6423-merge-rename-directories.sh | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/diffcore-rename.c b/diffcore-rename.c index 9c0bc5afb4..eb77e5c4f0 100644 --- a/diffcore-rename.c +++ b/diffcore-rename.c @@ -1544,7 +1544,7 @@ void diffcore_rename_extended(struct diff_options *options, /* all the usual ones need to be kept */ diff_q(&outq, p); else - /* no need to keep unmodified pairs; FIXME: remove earlier? */ + /* no need to keep unmodified pairs */ pair_to_free = p; if (pair_to_free) diff --git a/merge-ort.c b/merge-ort.c index be80143f4d..a3bb3e1132 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -2533,7 +2533,7 @@ static int compare_pairs(const void *a_, const void *b_) return strcmp(a->one->path, b->one->path); } -/* Call diffcore_rename() to compute which files have changed on given side */ +/* Call diffcore_rename() to update deleted/added pairs into rename pairs */ static void detect_regular_renames(struct merge_options *opt, unsigned side_index) { @@ -2586,8 +2586,10 @@ static void detect_regular_renames(struct merge_options *opt, } /* - * Get information of all renames which occurred in 'side_pairs', discarding - * non-renames. + * Get information of all renames which occurred in 'side_pairs', making use + * of any implicit directory renames in side_dir_renames (also making use of + * implicit directory renames rename_exclusions as needed by + * check_for_directory_rename()). Add all (updated) renames into result. */ static int collect_renames(struct merge_options *opt, struct diff_queue_struct *result, diff --git a/t/t6423-merge-rename-directories.sh b/t/t6423-merge-rename-directories.sh index 3037c5c9bf..85d947dd27 100755 --- a/t/t6423-merge-rename-directories.sh +++ b/t/t6423-merge-rename-directories.sh @@ -454,7 +454,7 @@ test_expect_success '1f: Split a directory into two other directories' ' # the directory renamed, but the files within it. (see 1b) # # If renames split a directory into two or more others, the directory -# with the most renames, "wins" (see 1c). However, see the testcases +# with the most renames, "wins" (see 1f). However, see the testcases # in section 2, plus testcases 3a and 4a. ########################################################################### From ef68c3d800b9c4516f1a13038e047610882cd26f Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Tue, 8 Jun 2021 16:11:42 +0000 Subject: [PATCH 4/4] merge-ort: miscellaneous touch-ups Add some notes in the code about invariants with match_mask when adding pairs. Also add a comment that seems to have been left out in my work of pushing these changes upstream. Signed-off-by: Elijah Newren Reviewed-by: Derrick Stolee Signed-off-by: Junio C Hamano --- merge-ort.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/merge-ort.c b/merge-ort.c index a3bb3e1132..53571742a0 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -764,6 +764,7 @@ static void add_pair(struct merge_options *opt, int names_idx = is_add ? side : 0; if (is_add) { + assert(match_mask == 0 || match_mask == 6); if (strset_contains(&renames->cached_target_names[side], pathname)) return; @@ -771,6 +772,8 @@ static void add_pair(struct merge_options *opt, unsigned content_relevant = (match_mask == 0); unsigned location_relevant = (dir_rename_mask == 0x07); + assert(match_mask == 0 || match_mask == 3 || match_mask == 5); + /* * If pathname is found in cached_irrelevant[side] due to * previous pick but for this commit content is relevant, @@ -3485,6 +3488,8 @@ static void process_entry(struct merge_options *opt, */ if (!ci->merged.clean) strmap_put(&opt->priv->conflicted, path, ci); + + /* Record metadata for ci->merged in dir_metadata */ record_entry_for_tree(dir_metadata, path, &ci->merged); }