From e3959cc78c968d8f029daa48d4aadcb486da0629 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Wed, 27 May 2026 15:55:50 -0400 Subject: [PATCH 1/8] pack-bitmap: pass object position to `fill_bitmap_tree()` In the following commit, callers of `fill_bitmap_tree()` will be required to check the bit corresponding to their tree before calling that function. That change will reduce the overhead of setting up and tearing down stack frames for trees whose bits are already set. To prepare for that change, have callers pass in the tree's bit position in `fill_bitmap_tree()`, which will make the next commit easier to read. In the meantime, this change has a surprising and measurable benefit during bitmap generation, particularly on very large repositories. When processing sub-trees within `fill_bitmap_tree()`, the preimage of this patch did the following: while (tree_entry(&desc, entry)) { switch (object_type(entry.mode)) { case OBJ_TREE: if (fill_bitmap_tree(writer, bitmap, lookup_tree(writer->repo, &entry.oid)) < 0) { /* ... */ } /* ... */ } } , first performing the object lookup via `lookup_tree()`, and then locating its bit position within the recursive call. This patch effectively reorders those two calls so that we first discover the sub-tree's bit position, *then* load its tree. By reordering these two operations, we spend fewer CPU cycles per instruction, likely due to improved CPU dependency/cache/pipeline behavior. Comparing the results of: running `perf stat` before and after this commit, we have: +--------------+-------------+-------------+-------------------+ | | HEAD^ | HEAD | Delta | +--------------+-------------+-------------+-------------------+ | elapsed | 612.5 s | 582.4 s | -30.1 s (-4.9%) | | cycles | 2,857.3 B | 2,713.3 B | -144.0 B (-5.0%) | | instructions | 2,413.2 B | 2,415.5 B | +2.3 B (+0.1%) | | CPI | 1.184 | 1.123 | -0.061 (-5.1%) | +--------------+-------------+-------------+-------------------+ In a large repository with ~4.8M commit, and ~37.1M tree objects this change improves timing from ~612.5 seconds down to ~582.4 seconds, or a ~4.9% improvement. More importantly, the number of CPU cycles spent dropped off significantly as a result of this commit, lowering our cycles-per-instruction ratio by about ~5.1%. Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- pack-bitmap-write.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/pack-bitmap-write.c b/pack-bitmap-write.c index 1c8070f99c..2d5ff8fd40 100644 --- a/pack-bitmap-write.c +++ b/pack-bitmap-write.c @@ -456,10 +456,10 @@ static void bitmap_builder_clear(struct bitmap_builder *bb) static int fill_bitmap_tree(struct bitmap_writer *writer, struct bitmap *bitmap, - struct tree *tree) + struct tree *tree, + uint32_t pos) { int found; - uint32_t pos; struct tree_desc desc; struct name_entry entry; @@ -467,9 +467,6 @@ static int fill_bitmap_tree(struct bitmap_writer *writer, * If our bit is already set, then there is nothing to do. Both this * tree and all of its children will be set. */ - pos = find_object_pos(writer, &tree->object.oid, &found); - if (!found) - return -1; if (bitmap_get(bitmap, pos)) return 0; bitmap_set(bitmap, pos); @@ -482,8 +479,12 @@ static int fill_bitmap_tree(struct bitmap_writer *writer, while (tree_entry(&desc, &entry)) { switch (object_type(entry.mode)) { case OBJ_TREE: + pos = find_object_pos(writer, &entry.oid, &found); + if (!found) + return -1; if (fill_bitmap_tree(writer, bitmap, - lookup_tree(writer->repo, &entry.oid)) < 0) + lookup_tree(writer->repo, + &entry.oid), pos) < 0) return -1; break; case OBJ_BLOB: @@ -575,8 +576,14 @@ static int fill_bitmap_commit(struct bitmap_writer *writer, } while (tree_queue->nr) { - if (fill_bitmap_tree(writer, ent->bitmap, - prio_queue_get(tree_queue)) < 0) + struct tree *t = prio_queue_get(tree_queue); + int found; + + pos = find_object_pos(writer, &t->object.oid, &found); + if (!found) + return -1; + + if (fill_bitmap_tree(writer, ent->bitmap, t, pos) < 0) return -1; } return 0; From 1760c372589af09ff0b986c57bfe0b9101275674 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Wed, 27 May 2026 15:55:53 -0400 Subject: [PATCH 2/8] pack-bitmap: check subtree bits before recursing In the previous commit, we adjusted the callers of `fill_bitmap_tree()` to pass in the bit position of the tree they wish to fill. This commit makes use of that information at the call site to avoid setting up a stack frame for fill_bitmap_tree() entirely whenever a tree's bit position is already set. Since this is such a hot path, the avoided cost of setting up and tearing down stack frames for each noop'd call to `fill_bitmap_tree()` is significant: +--------------+-------------+-------------+-------------------+ | | HEAD^ | HEAD | Delta | +--------------+-------------+-------------+-------------------+ | elapsed | 582.4 s | 562.8 s | -19.6 s (-3.4%) | | cycles | 2,713.3 B | 2,621.3 B | -92.0 B (-3.4%) | | instructions | 2,415.5 B | 2,348.9 B | -66.6 B (-2.8%) | | CPI | 1.123 | 1.116 | -0.007 (-0.7%) | +--------------+-------------+-------------+-------------------+ In the same repository as in the previous commit, our timings dropped from ~582.4 seconds down to ~562.77 seconds. While the cycles-per-instruction ratio is basically unchanged, we execute significantly fewer instructions, and correspondingly fewer cycles. Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- pack-bitmap-write.c | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/pack-bitmap-write.c b/pack-bitmap-write.c index 2d5ff8fd40..7261039702 100644 --- a/pack-bitmap-write.c +++ b/pack-bitmap-write.c @@ -463,12 +463,6 @@ static int fill_bitmap_tree(struct bitmap_writer *writer, struct tree_desc desc; struct name_entry entry; - /* - * If our bit is already set, then there is nothing to do. Both this - * tree and all of its children will be set. - */ - if (bitmap_get(bitmap, pos)) - return 0; bitmap_set(bitmap, pos); if (repo_parse_tree(writer->repo, tree) < 0) @@ -482,6 +476,15 @@ static int fill_bitmap_tree(struct bitmap_writer *writer, pos = find_object_pos(writer, &entry.oid, &found); if (!found) return -1; + if (bitmap_get(bitmap, pos)) { + /* + * If our bit is already set, then there + * is nothing to do. Both this tree and + * all of its children will be set. + */ + break; + } + if (fill_bitmap_tree(writer, bitmap, lookup_tree(writer->repo, &entry.oid), pos) < 0) @@ -582,6 +585,14 @@ static int fill_bitmap_commit(struct bitmap_writer *writer, pos = find_object_pos(writer, &t->object.oid, &found); if (!found) return -1; + if (bitmap_get(ent->bitmap, pos)) { + /* + * If our bit is already set, then there is + * nothing to do. Both this tree and all of its + * children will be set. + */ + continue; + } if (fill_bitmap_tree(writer, ent->bitmap, t, pos) < 0) return -1; From 3ea5fe8482e44fe8636b2725edffcadc81b22161 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Wed, 27 May 2026 15:55:56 -0400 Subject: [PATCH 3/8] pack-bitmap: reuse stored selected bitmaps When `fill_bitmap_commit()` reaches an ancestor that was selected for its own bitmap and processed earlier, its object closure is already stored in `writer->bitmaps` as an EWAH bitmap. As a result, walking through that commit's tree and parents again is redundant. Teach `fill_bitmap_commit()` to notice that case. For non-root commits in the walk, look for a stored selected bitmap and OR it into the bitmap being built. If one exists, skip the commit, its tree, and its parents. Building bitmaps from scratch on the same test repository from the previous commits yields a significant speed-up: +------------------+-------------+-------------+---------------------+ | | HEAD^ | HEAD | Delta | +------------------+-------------+-------------+---------------------+ | elapsed | 562.8 s | 324.8 s | -237.9 s (-42.3%) | | cycles | 2,621.3 B | 1,508.6 B | -1,112.7 B (-42.4%) | | instructions | 2,348.9 B | 1,436.6 B | -912.3 B (-38.8%) | | CPI | 1.116 | 1.050 | -0.066 (-5.9%) | +------------------+-------------+-------------+---------------------+ In our testing repository, there are 1,261 commits selected for bitmap coverage, and 1,382 maximal commits induced as a result of that. Of the 1,382 calls made to `fill_bitmap_commit()` (one per maximal commit), 131 of them can be short-circuited at some point during their traversal as a consequence of this change. In large repositories where the cost of filling the bitmap for any individual commit is large, being able to short-circuit even ~9.5% of the calls to `fill_bitmap_commit()` results in a significant savings. Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- pack-bitmap-write.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/pack-bitmap-write.c b/pack-bitmap-write.c index 7261039702..651ad46746 100644 --- a/pack-bitmap-write.c +++ b/pack-bitmap-write.c @@ -509,6 +509,9 @@ static int fill_bitmap_tree(struct bitmap_writer *writer, static int reused_bitmaps_nr; static int reused_pseudo_merge_bitmaps_nr; +static int fill_bitmap_commit_calls_nr; +static int fill_bitmap_commit_found_ancestor_nr; + static int fill_bitmap_commit(struct bitmap_writer *writer, struct bb_commit *ent, struct commit *commit, @@ -519,6 +522,9 @@ static int fill_bitmap_commit(struct bitmap_writer *writer, { int found; uint32_t pos; + + fill_bitmap_commit_calls_nr++; + if (!ent->bitmap) ent->bitmap = bitmap_new(); @@ -553,6 +559,28 @@ static int fill_bitmap_commit(struct bitmap_writer *writer, bitmap_free(remapped); } + /* + * If we encounter an ancestor for which we have already + * computed a bitmap during this build (i.e. a regular + * selected commit processed earlier in topo order), we can + * short-circuit the walk: its stored bitmap already covers + * the commit itself, its tree, and all of its ancestors. + */ + if (c != commit) { + khiter_t hash_pos = kh_get_oid_map(writer->bitmaps, + c->object.oid); + if (hash_pos != kh_end(writer->bitmaps)) { + struct bitmapped_commit *stored = + kh_value(writer->bitmaps, hash_pos); + if (stored && stored->bitmap) { + fill_bitmap_commit_found_ancestor_nr++; + bitmap_or_ewah(ent->bitmap, + stored->bitmap); + continue; + } + } + } + /* * Mark ourselves and queue our tree. The commit * walk ensures we cover all parents. @@ -692,6 +720,12 @@ int bitmap_writer_build(struct bitmap_writer *writer) trace2_data_intmax("pack-bitmap-write", writer->repo, "building_bitmaps_pseudo_merge_reused", reused_pseudo_merge_bitmaps_nr); + trace2_data_intmax("pack-bitmap-write", writer->repo, + "fill_bitmap_commit_calls_nr", + fill_bitmap_commit_calls_nr); + trace2_data_intmax("pack-bitmap-write", writer->repo, + "fill_bitmap_commit_found_ancestor_nr", + fill_bitmap_commit_found_ancestor_nr); stop_progress(&writer->progress); From ece3465d44157157a03eb7cd5de955e552e7831c Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Wed, 27 May 2026 15:55:59 -0400 Subject: [PATCH 4/8] pack-bitmap: consolidate `find_object_pos()` success path Both sides of `find_object_pos()` report success in the same way by setting the optional `found` out-parameter and return the resolved bitmap position. Prepare for adding more bookkeeping around object-position lookups by storing the result in a local `pos` variable and sharing the success return path between the packlist and MIDX cases. Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- pack-bitmap-write.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pack-bitmap-write.c b/pack-bitmap-write.c index 651ad46746..42ed22feac 100644 --- a/pack-bitmap-write.c +++ b/pack-bitmap-write.c @@ -217,6 +217,7 @@ static uint32_t find_object_pos(struct bitmap_writer *writer, const struct object_id *oid, int *found) { struct object_entry *entry; + uint32_t pos; entry = packlist_find(writer->to_pack, oid); if (entry) { @@ -224,23 +225,22 @@ static uint32_t find_object_pos(struct bitmap_writer *writer, if (writer->midx) base_objects = writer->midx->num_objects + writer->midx->num_objects_in_base; - - if (found) - *found = 1; - return oe_in_pack_pos(writer->to_pack, entry) + base_objects; + pos = oe_in_pack_pos(writer->to_pack, entry) + base_objects; } else if (writer->midx) { - uint32_t at, pos; + uint32_t at; if (!bsearch_midx(oid, writer->midx, &at)) goto missing; if (midx_to_pack_pos(writer->midx, at, &pos) < 0) goto missing; - - if (found) - *found = 1; - return pos; + } else { + goto missing; } + if (found) + *found = 1; + return pos; + missing: if (found) *found = 0; From c720bbcc53f223236220c7a879f0a0e73e5d3739 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Wed, 27 May 2026 15:56:02 -0400 Subject: [PATCH 5/8] pack-bitmap: cache object positions during fill The previous commits removed some redundant work from bitmap generation by avoiding unnecessary tree recursion and by reusing selected bitmaps that have already been computed. Even with those changes in place, there is still an extremely hot path from `fill_bitmap_commit()` and `fill_bitmap_tree()` to translate object IDs into their corresponding bit positions in order to generate their bitmaps. In a small repository, this overhead is not significant. However, in a very large repository (e.g., the one that we have been using as a benchmark over the past several commits with ~57M total objects), the overhead of locating object bit positions (often repeatedly) adds up significantly. Combat this by adding a small, direct-mapped cache to the bitmap writer which maps object IDs to their corresponding bit positions. Size the cache according to the number of objects being written, with fixed lower and upper bounds so small repositories do not pay for a large table and large repositories can avoid most repeated packlist and MIDX lookups. On my machine with (a somewhat outdated) GCC 15.2.0, each entry in the cache is 40 bytes wide: $ pahole -C bitmap_pos_cache_entry pack-bitmap-write.o struct bitmap_pos_cache_entry { struct object_id oid; /* 0 36 */ uint32_t pos; /* 36 4 */ /* size: 40, cachelines: 1, members: 2 */ /* last cacheline: 40 bytes */ }; , and we will allocate up to 2^21 entries for a maximum total of 80 MiB of cache overhead. In our example repository from above and in earlier commits, this results in a ~9.4% reduction in runtime relative to the previous commit: +------------------+-------------+-------------+---------------------+ | | HEAD^ | HEAD | Delta | +------------------+-------------+-------------+---------------------+ | elapsed | 324.8 s | 294.1 s | -30.7 s (-9.4%) | | cycles | 1,508.6 B | 1,365.5 B | -143.0 B (-9.5%) | | instructions | 1,436.6 B | 1,389.8 B | -46.9 B (-3.3%) | | CPI | 1.050 | 0.983 | -0.068 (-6.4%) | +------------------+-------------+-------------+---------------------+ When generating bitmaps on this repository (to produce the above timings), the cache grew to its maximum size of 80 MiB, and resulted in 1.024B cache hits and 59.957M cache misses. Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- pack-bitmap-write.c | 88 ++++++++++++++++++++++++++++++++++++++++++++- pack-bitmap.h | 7 ++++ 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/pack-bitmap-write.c b/pack-bitmap-write.c index 42ed22feac..4b6fb07edd 100644 --- a/pack-bitmap-write.c +++ b/pack-bitmap-write.c @@ -89,6 +89,7 @@ void bitmap_writer_free(struct bitmap_writer *writer) ewah_free(writer->tags); kh_destroy_oid_map(writer->bitmaps); + free(writer->pos_cache); kh_foreach_value(writer->pseudo_merge_commits, idx, free_pseudo_merge_commit_idx(idx)); @@ -213,15 +214,92 @@ void bitmap_writer_push_commit(struct bitmap_writer *writer, writer->selected_nr++; } +struct bitmap_pos_cache_entry { + struct object_id oid; + uint32_t pos; +}; + +#define BITMAP_POS_MIN_CACHE_SIZE (1U << 10) +#define BITMAP_POS_MAX_CACHE_SIZE (1U << 21) +#define BITMAP_POS_CACHE_VALID (1U << 31) + +static void bitmap_writer_init_pos_cache(struct bitmap_writer *writer) +{ + if (writer->pos_cache) + return; + + writer->pos_cache_nr = BITMAP_POS_MIN_CACHE_SIZE; + + while (writer->pos_cache_nr < writer->to_pack->nr_objects && + writer->pos_cache_nr < BITMAP_POS_MAX_CACHE_SIZE) + writer->pos_cache_nr <<= 1; + + CALLOC_ARRAY(writer->pos_cache, writer->pos_cache_nr); +} + +static size_t bitmap_writer_pos_cache_slot(struct bitmap_writer *writer, + const struct object_id *oid) +{ + return oidhash(oid) & (writer->pos_cache_nr - 1); +} + +static bool bitmap_writer_pos_cache_valid(struct bitmap_writer *writer, + size_t slot) +{ + return !!(writer->pos_cache[slot].pos & BITMAP_POS_CACHE_VALID); +} + +static int find_cached_object_pos(struct bitmap_writer *writer, + const struct object_id *oid, uint32_t *pos) +{ + size_t slot = bitmap_writer_pos_cache_slot(writer, oid); + + if (bitmap_writer_pos_cache_valid(writer, slot) && + oideq(&writer->pos_cache[slot].oid, oid)) { + writer->pos_cache_hits++; + *pos = writer->pos_cache[slot].pos & ~BITMAP_POS_CACHE_VALID; + return 1; + } + + writer->pos_cache_misses++; + return 0; +} + +static uint32_t store_cached_object_pos(struct bitmap_writer *writer, + const struct object_id *oid, + uint32_t pos) +{ + size_t slot; + + if (pos & BITMAP_POS_CACHE_VALID) + return pos; /* too large to cache */ + + slot = bitmap_writer_pos_cache_slot(writer, oid); + + oidcpy(&writer->pos_cache[slot].oid, oid); + writer->pos_cache[slot].pos = pos | BITMAP_POS_CACHE_VALID; + + return pos; +} + static uint32_t find_object_pos(struct bitmap_writer *writer, const struct object_id *oid, int *found) { struct object_entry *entry; uint32_t pos; + bitmap_writer_init_pos_cache(writer); + + if (find_cached_object_pos(writer, oid, &pos)) { + if (found) + *found = 1; + return pos; + } + entry = packlist_find(writer->to_pack, oid); if (entry) { uint32_t base_objects = 0; + if (writer->midx) base_objects = writer->midx->num_objects + writer->midx->num_objects_in_base; @@ -239,7 +317,7 @@ static uint32_t find_object_pos(struct bitmap_writer *writer, if (found) *found = 1; - return pos; + return store_cached_object_pos(writer, oid, pos); missing: if (found) @@ -662,6 +740,10 @@ int bitmap_writer_build(struct bitmap_writer *writer) writer->progress = start_progress(writer->repo, "Building bitmaps", writer->selected_nr); + + writer->pos_cache_hits = 0; + writer->pos_cache_misses = 0; + trace2_region_enter("pack-bitmap-write", "building_bitmaps_total", writer->repo); @@ -726,6 +808,10 @@ int bitmap_writer_build(struct bitmap_writer *writer) trace2_data_intmax("pack-bitmap-write", writer->repo, "fill_bitmap_commit_found_ancestor_nr", fill_bitmap_commit_found_ancestor_nr); + trace2_data_intmax("pack-bitmap-write", writer->repo, + "bitmap_pos_cache_hits", writer->pos_cache_hits); + trace2_data_intmax("pack-bitmap-write", writer->repo, + "bitmap_pos_cache_misses", writer->pos_cache_misses); stop_progress(&writer->progress); diff --git a/pack-bitmap.h b/pack-bitmap.h index a95e1c2d11..19a8655457 100644 --- a/pack-bitmap.h +++ b/pack-bitmap.h @@ -132,6 +132,8 @@ int bitmap_has_oid_in_uninteresting(struct bitmap_index *, const struct object_i off_t get_disk_usage_from_bitmap(struct bitmap_index *, struct rev_info *); +struct bitmap_pos_cache_entry; + struct bitmap_writer { struct repository *repo; struct ewah_bitmap *commits; @@ -143,6 +145,11 @@ struct bitmap_writer { struct packing_data *to_pack; struct multi_pack_index *midx; /* if appending to a MIDX chain */ + struct bitmap_pos_cache_entry *pos_cache; + size_t pos_cache_nr; + uint64_t pos_cache_hits; + uint64_t pos_cache_misses; + struct bitmapped_commit *selected; unsigned int selected_nr, selected_alloc; From dcccd997462e2130bcc35f933285ff087454275e Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Wed, 27 May 2026 15:56:05 -0400 Subject: [PATCH 6/8] pack-bitmap: sort bitmaps before XORing Reachability bitmaps may be stored as XORs against nearby bitmaps, up to 10 away. However, when callers provide selected commits in an arbitrary order, the writer may miss good ancestor/descendant pairs and produce much larger bitmap files without changing query coverage. Sort the selected bitmaps in date order (from oldest to newest) before computing XOR offsets, leaving pseudo-merge bitmaps alone (which we will deal with separately in following commits). On our same testing repository from previous commits, this change shrunk our selection of 1,261 bitmaps from ~635.46 MiB to 176.4 MiB for a ~72.24% reduction in the on-disk size of our *.bitmap file. The time to generate the smaller bitmap file decreased by ~3.69 seconds, though this is likely mostly noise. Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- pack-bitmap-write.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pack-bitmap-write.c b/pack-bitmap-write.c index 4b6fb07edd..66282ea14b 100644 --- a/pack-bitmap-write.c +++ b/pack-bitmap-write.c @@ -327,11 +327,40 @@ missing: return 0; } +static int bitmapped_commit_date_cmp(const void *_a, const void *_b) +{ + const struct bitmapped_commit *a = _a; + const struct bitmapped_commit *b = _b; + + if (a->commit->date < b->commit->date) + return -1; + if (a->commit->date > b->commit->date) + return 1; + return 0; +} + static void compute_xor_offsets(struct bitmap_writer *writer) { static const int MAX_XOR_OFFSET_SEARCH = 10; int i, next = 0; + int nr = bitmap_writer_nr_selected_commits(writer); + + if (nr > 1) { + QSORT(writer->selected, nr, bitmapped_commit_date_cmp); + + for (i = 0; i < nr; i++) { + struct bitmapped_commit *stored = &writer->selected[i]; + khiter_t hash_pos = kh_get_oid_map(writer->bitmaps, + stored->commit->object.oid); + + if (hash_pos == kh_end(writer->bitmaps)) + BUG("selected commit missing from bitmap map: %s", + oid_to_hex(&stored->commit->object.oid)); + + kh_value(writer->bitmaps, hash_pos) = stored; + } + } while (next < writer->selected_nr) { struct bitmapped_commit *stored = &writer->selected[next]; From b04d26607de35b88cf9c62ca11931d4f8cc4ac05 Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Wed, 27 May 2026 15:56:08 -0400 Subject: [PATCH 7/8] pack-bitmap: remember pseudo-merge parents write_pseudo_merges() currently builds an array of temporary bitmaps for the parent set of each pseudo-merge, then serializes those bitmaps later while writing the extension. Move those parent bitmaps onto the corresponding bitmapped_commit entries instead. This keeps the on-disk output unchanged, but gives the parent bitmap the same lifetime and access pattern that later changes will use when pseudo-merge object bitmaps are built before the write step. Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- pack-bitmap-write.c | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/pack-bitmap-write.c b/pack-bitmap-write.c index 66282ea14b..8200aed610 100644 --- a/pack-bitmap-write.c +++ b/pack-bitmap-write.c @@ -32,6 +32,7 @@ struct bitmapped_commit { struct commit *commit; struct ewah_bitmap *bitmap; struct ewah_bitmap *write_as; + struct ewah_bitmap *pseudo_merge_parents; int flags; int xor_offset; uint32_t commit_pos; @@ -102,6 +103,7 @@ void bitmap_writer_free(struct bitmap_writer *writer) if (bc->write_as != bc->bitmap) ewah_free(bc->write_as); ewah_free(bc->bitmap); + ewah_free(bc->pseudo_merge_parents); } free(writer->selected); } @@ -210,6 +212,7 @@ void bitmap_writer_push_commit(struct bitmap_writer *writer, writer->selected[writer->selected_nr].write_as = NULL; writer->selected[writer->selected_nr].flags = 0; writer->selected[writer->selected_nr].pseudo_merge = pseudo_merge; + writer->selected[writer->selected_nr].pseudo_merge_parents = NULL; writer->selected_nr++; } @@ -1004,42 +1007,47 @@ static void write_pseudo_merges(struct bitmap_writer *writer, struct hashfile *f) { struct oid_array commits = OID_ARRAY_INIT; - struct bitmap **commits_bitmap = NULL; off_t *pseudo_merge_ofs = NULL; off_t start, table_start, next_ext; uint32_t base = bitmap_writer_nr_selected_commits(writer); size_t i, j = 0; - CALLOC_ARRAY(commits_bitmap, writer->pseudo_merges_nr); CALLOC_ARRAY(pseudo_merge_ofs, writer->pseudo_merges_nr); for (i = 0; i < writer->pseudo_merges_nr; i++) { struct bitmapped_commit *merge = &writer->selected[base + i]; struct commit_list *p; + struct bitmap *parents = bitmap_new(); if (!merge->pseudo_merge) BUG("found non-pseudo merge commit at %"PRIuMAX, (uintmax_t)i); - commits_bitmap[i] = bitmap_new(); - for (p = merge->commit->parents; p; p = p->next) - bitmap_set(commits_bitmap[i], + bitmap_set(parents, find_object_pos(writer, &p->item->object.oid, NULL)); + + merge->pseudo_merge_parents = bitmap_to_ewah(parents); + bitmap_free(parents); } start = hashfile_total(f); for (i = 0; i < writer->pseudo_merges_nr; i++) { - struct ewah_bitmap *commits_ewah = bitmap_to_ewah(commits_bitmap[i]); + struct bitmapped_commit *merge = &writer->selected[base + i]; + + if (!merge->pseudo_merge) + BUG("found non-pseudo merge commit at %"PRIuMAX, (uintmax_t)i); + + if (!merge->pseudo_merge_parents) + BUG("missing pseudo-merge parents bitmap for commit %s", + oid_to_hex(&merge->commit->object.oid)); pseudo_merge_ofs[i] = hashfile_total(f); - dump_bitmap(f, commits_ewah); + dump_bitmap(f, merge->pseudo_merge_parents); dump_bitmap(f, writer->selected[base+i].write_as); - - ewah_free(commits_ewah); } next_ext = st_add(hashfile_total(f), @@ -1122,12 +1130,8 @@ static void write_pseudo_merges(struct bitmap_writer *writer, hashwrite_be64(f, table_start - start); hashwrite_be64(f, hashfile_total(f) - start + sizeof(uint64_t)); - for (i = 0; i < writer->pseudo_merges_nr; i++) - bitmap_free(commits_bitmap[i]); - oid_array_clear(&commits); free(pseudo_merge_ofs); - free(commits_bitmap); } static int table_cmp(const void *_va, const void *_vb, void *_data) From 49633dc88c14008f9a405f215b60994362b36d6c Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Wed, 27 May 2026 15:56:11 -0400 Subject: [PATCH 8/8] pack-bitmap: build pseudo-merge bitmaps after regular bitmaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When generating bitmaps, `bitmap_builder_init()` starts with an initial selection of commits to receive bitmap coverage, and then determines a set of "maximal" commits based on its input. Commit 089f751360f (pack-bitmap-write: build fewer intermediate bitmaps, 2020-12-08) has extensive details, but the gist is as follows: Each selected commit starts with one commit_mask bit in its "commit mask" bitmap. Then, we walk the first-parent history in topological order and OR each commit's mask into its (first) parent. Whenever that OR results in the parent having more bits set, the child is deemed to be non-maximal, and the frontier is pushed further back along the first parent history. That approach works extremely well for ordinary selected commits, whose first-parent histories often describe real sharing between the bitmaps we are going to write. It struggles, however, to efficiently generate pseudo-merge bitmaps. Unlike ordinary commits for which the above algorithm is designed, pseudo-merges don't represent any "real" commit in history, just a grouping of non-bitmapped reference tips. In that sense, their first parent is just a part of a larger set, and treating them like ordinary selected commits imposes a significant slow-down when generating bitmaps with pseudo-merges enabled. Consider partitioning all non-bitmapped reference tips into eight individual pseudo-merges via the following configuration: [bitmapPseudoMerge "all"] pattern=refs/ threshold=now stableSize=10000000 maxMerges=8 , the cost of generating a bitmap from scratch rises significantly: +------------------+-----------------+---------------+---------------------+ | | no pseudo-merge | pseudo-merges | Delta | | | | (HEAD^) | | +------------------+-----------------+---------------+---------------------+ | elapsed | 294.1 s | 575.0 s | +280.9 s (+95.5%) | | cycles | 1,365.5 B | 2,686.9 B | +1,321.4 B (+96.8%) | | instructions | 1,389.8 B | 2,546.6 B | +1,156.8 B (+83.2%) | | CPI | 0.983 | 1.055 | +0.073 (+7.4%) | +------------------+-----------------+---------------+---------------------+ This is a particularly poor trade-off, because the time saved by these pseudo-merges during, e.g., $ git rev-list --count --all --objects --use-bitmap-index is only: $ hyperfine -L v true,false -n 'pseudo-merges: {v}' ' GIT_TEST_USE_PSEUDO_MERGES={v} git.compile rev-list --count \ --objects --all --use-bitmap-index ' Benchmark 1: pseudo-merges: true Time (mean ± σ): 2.613 s ± 0.012 s [User: 2.308 s, System: 0.305 s] Range (min … max): 2.594 s … 2.633 s 10 runs Benchmark 2: pseudo-merges: false Time (mean ± σ): 52.205 s ± 0.170 s [User: 51.500 s, System: 0.697 s] Range (min … max): 51.956 s … 52.458 s 10 runs Summary pseudo-merges: true ran 19.98 ± 0.11 times faster than pseudo-merges: false In other words, we pay a nearly ~5 minute penalty to generate pseudo-merge bitmaps, but only save ~50 seconds during traversal. The problem stems from injecting pseudo-merges into the bitmap builder as if they were normal commits. The maximal commit selection algorithm was simply not designed for that case, and performs predictably poorly. The only reason we reused the maximal commit selection routine for pseudo-merges alongside regular non-pseudo-merge commits is because we represent them both as commit objects (where the pseudo-merge commits just represent a made-up commit as opposed to one that actually exists in a repository's object store). Instead, build the regular selected commit bitmaps first, considering only non-pseudo-merge commits in `bitmap_builder_init()`. Once those bitmaps have been stored, build each pseudo-merge bitmap separately and attach its parent and object bitmaps to the corresponding pseudo-merge entry before writing the extension. This keeps the regular bitmap build shaped like the no-pseudo-merge case. The later pseudo-merge fill can still stop at stored selected ancestor bitmaps, so it does not have to rewalk each pseudo-merge closure from scratch. When an existing bitmap has the same pseudo-merge parent set, reuse and remap that whole pseudo-merge bitmap before falling back to fill_bitmap_commit(). This preserves the benefit of stable pseudo-merges while keeping the on-disk format and reader behavior unchanged. As a result, the overhead cost for generating pseudo-merges in the above configuration is much smaller: +------------------+-----------------+---------------+-------------------+ | | no pseudo-merge | pseudo-merges | Delta | | | | (HEAD) | | +------------------+-----------------+---------------+-------------------+ | elapsed | 294.1 s | 328.4 s | +34.3 s (+11.7%) | | cycles | 1,365.5 B | 1,529.3 B | +163.7 B (+12.0%) | | instructions | 1,389.8 B | 1,552.8 B | +163.0 B (+11.7%) | | CPI | 0.983 | 0.985 | +0.002 (+0.2%) | +------------------+-----------------+---------------+-------------------+ Recall that at the start of this series, generating reachability bitmaps took 612.5 seconds *without* pseudo-merges. With this commit, it is still ~46.38% *faster* to generate reachability bitmaps *with* pseudo-merges than it was to generate bitmaps wihtout them at the beginning of this series. The changes to implement this are mostly straightforward. We exclude pseudo-merge commits from the existing bitmap generation, and walk over them in a separate pass, by either reusing an existing on-disk pseudo-merge, or passing the pseudo-merge commit itself back to the existing routine in `fill_bitmap_commit()`. (Note that the routine to build pseudo-merge bitmaps is the same both before and after this change, the difference is only that we do not let psuedo-merges participate in determining the set of maximal commits.) The only wrinkle is that `fill_bitmap_commit()` must be taught to not expect that all tree objects have been parsed, which is the case for any portion of history reachable by one or more pseudo-merge(s), but not by any non-pseudo-merge commit selected for bitmapping. Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- pack-bitmap-write.c | 210 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 174 insertions(+), 36 deletions(-) diff --git a/pack-bitmap-write.c b/pack-bitmap-write.c index 8200aed610..1bcb3f98a4 100644 --- a/pack-bitmap-write.c +++ b/pack-bitmap-write.c @@ -446,13 +446,17 @@ static void bitmap_builder_init(struct bitmap_builder *bb, revs.topo_order = 1; revs.first_parent_only = 1; - for (i = 0; i < writer->selected_nr; i++) { + for (i = 0; i < bitmap_writer_nr_selected_commits(writer); i++) { struct bitmapped_commit *bc = &writer->selected[i]; struct bb_commit *ent = bb_data_at(&bb->data, bc->commit); + if (bc->pseudo_merge) + BUG("unexpected pseudo-merge at %"PRIuMAX, + (uintmax_t)i); + ent->selected = 1; ent->maximal = 1; - ent->pseudo_merge = bc->pseudo_merge; + ent->pseudo_merge = 0; ent->idx = i; ent->commit_mask = bitmap_new(); @@ -618,6 +622,8 @@ static int fill_bitmap_tree(struct bitmap_writer *writer, static int reused_bitmaps_nr; static int reused_pseudo_merge_bitmaps_nr; +static int pseudo_merge_bitmap_nr; +static int pseudo_merge_bitmap_parents; static int fill_bitmap_commit_calls_nr; static int fill_bitmap_commit_found_ancestor_nr; @@ -631,8 +637,12 @@ static int fill_bitmap_commit(struct bitmap_writer *writer, const uint32_t *mapping) { int found; + int from_pseudo_merge = commit->object.flags & BITMAP_PSEUDO_MERGE; uint32_t pos; + if (ent->pseudo_merge) + BUG("unexpected pseudo-merge commit in fill_bitmap_commit()"); + fill_bitmap_commit_calls_nr++; if (!ent->bitmap) @@ -648,10 +658,7 @@ static int fill_bitmap_commit(struct bitmap_writer *writer, struct ewah_bitmap *old; struct bitmap *remapped = bitmap_new(); - if (commit->object.flags & BITMAP_PSEUDO_MERGE) - old = pseudo_merge_bitmap_for_commit(old_bitmap, c); - else - old = bitmap_for_commit(old_bitmap, c); + old = bitmap_for_commit(old_bitmap, c); /* * If this commit has an old bitmap, then translate that * bitmap and add its bits to this one. No need to walk @@ -660,10 +667,7 @@ static int fill_bitmap_commit(struct bitmap_writer *writer, if (old && !rebuild_bitmap(mapping, old, remapped)) { bitmap_or(ent->bitmap, remapped); bitmap_free(remapped); - if (commit->object.flags & BITMAP_PSEUDO_MERGE) - reused_pseudo_merge_bitmaps_nr++; - else - reused_bitmaps_nr++; + reused_bitmaps_nr++; continue; } bitmap_free(remapped); @@ -696,12 +700,32 @@ static int fill_bitmap_commit(struct bitmap_writer *writer, * walk ensures we cover all parents. */ if (!(c->object.flags & BITMAP_PSEUDO_MERGE)) { + struct tree *tree; + + if (from_pseudo_merge && !c->object.parsed) { + /* + * Commits reachable from selected + * non-pseudo-merges are already parsed + * by the regular bitmap build. + * + * However, pseudo-merge fills can also + * reach commits that were not covered + * there, so parse any such leftovers + * before reading their tree or parents. + */ + if (repo_parse_commit(writer->repo, c)) + return -1; + } + pos = find_object_pos(writer, &c->object.oid, &found); if (!found) return -1; bitmap_set(ent->bitmap, pos); - prio_queue_put(tree_queue, - repo_get_commit_tree(writer->repo, c)); + + tree = repo_get_commit_tree(writer->repo, c); + if (!tree) + return -1; + prio_queue_put(tree_queue, tree); } for (p = c->parents; p; p = p->next) { @@ -738,6 +762,137 @@ static int fill_bitmap_commit(struct bitmap_writer *writer, return 0; } +static int reuse_pseudo_merge_bitmap(struct bitmap_index *old_bitmap, + const uint32_t *mapping, + struct commit *merge, + struct ewah_bitmap **out) +{ + struct ewah_bitmap *old; + struct bitmap *remapped; + + if (!old_bitmap || !mapping) + return 0; + + old = pseudo_merge_bitmap_for_commit(old_bitmap, merge); + if (!old) + return 0; + + remapped = bitmap_new(); + if (rebuild_bitmap(mapping, old, remapped) < 0) { + bitmap_free(remapped); + return 0; + } + + *out = bitmap_to_ewah(remapped); + bitmap_free(remapped); + reused_pseudo_merge_bitmaps_nr++; + return 1; +} + +static int build_pseudo_merge_bitmap(struct bitmap_writer *writer, + struct bitmap_index *old_bitmap, + const uint32_t *mapping, + struct commit *merge, + struct ewah_bitmap **out) +{ + struct bb_commit ent = { 0 }; + struct prio_queue queue = { NULL }; + struct prio_queue tree_queue = { NULL }; + unsigned parents = commit_list_count(merge->parents); + int ret; + + ent.bitmap = bitmap_new(); + + pseudo_merge_bitmap_nr++; + pseudo_merge_bitmap_parents += parents; + + if (reuse_pseudo_merge_bitmap(old_bitmap, mapping, merge, out)) { + ret = 0; + goto done; + } + + ret = fill_bitmap_commit(writer, &ent, merge, &queue, &tree_queue, + old_bitmap, mapping); + + if (!ret) + *out = bitmap_to_ewah(ent.bitmap); + +done: + bitmap_free(ent.bitmap); + clear_prio_queue(&queue); + clear_prio_queue(&tree_queue); + + return ret; +} + +static int build_pseudo_merge_bitmaps(struct bitmap_writer *writer, + struct bitmap_index *old_bitmap, + const uint32_t *mapping, + int *nr_stored) +{ + size_t i = bitmap_writer_nr_selected_commits(writer); + int ret = 0; + + if (!writer->pseudo_merges_nr) + return 0; + + trace2_region_enter("pack-bitmap-write", "building_pseudo_merge_bitmaps", + writer->repo); + + for (; i < writer->selected_nr; i++) { + struct bitmapped_commit *merge = &writer->selected[i]; + struct commit_list *p; + struct bitmap *parents = bitmap_new(); + struct ewah_bitmap *objects = NULL; + + if (!merge->pseudo_merge) + BUG("found non-pseudo merge commit at %"PRIuMAX, + (uintmax_t)i); + + for (p = merge->commit->parents; p; p = p->next) { + int found; + uint32_t pos = find_object_pos(writer, + &p->item->object.oid, + &found); + if (!found) { + bitmap_free(parents); + ret = -1; + goto done; + } + bitmap_set(parents, pos); + } + + merge->pseudo_merge_parents = bitmap_to_ewah(parents); + bitmap_free(parents); + + if (build_pseudo_merge_bitmap(writer, old_bitmap, mapping, + merge->commit, &objects) < 0) { + ret = -1; + goto done; + } + merge->bitmap = objects; + + (*nr_stored)++; + display_progress(writer->progress, *nr_stored); + } + +done: + trace2_region_leave("pack-bitmap-write", "building_pseudo_merge_bitmaps", + writer->repo); + + trace2_data_intmax("pack-bitmap-write", writer->repo, + "pseudo_merge_bitmap_nr", + pseudo_merge_bitmap_nr); + trace2_data_intmax("pack-bitmap-write", writer->repo, + "building_bitmaps_pseudo_merge_reused", + reused_pseudo_merge_bitmaps_nr); + trace2_data_intmax("pack-bitmap-write", writer->repo, + "pseudo_merge_bitmap_parents", + pseudo_merge_bitmap_parents); + + return ret; +} + static void store_selected(struct bitmap_writer *writer, struct bb_commit *ent, struct commit *commit) { @@ -821,6 +976,10 @@ int bitmap_writer_build(struct bitmap_writer *writer) bitmap_free(ent->bitmap); ent->bitmap = NULL; } + if (closed && + build_pseudo_merge_bitmaps(writer, old_bitmap, mapping, + &nr_stored) < 0) + closed = 0; clear_prio_queue(&queue); clear_prio_queue(&tree_queue); bitmap_builder_clear(&bb); @@ -831,9 +990,6 @@ int bitmap_writer_build(struct bitmap_writer *writer) writer->repo); trace2_data_intmax("pack-bitmap-write", writer->repo, "building_bitmaps_reused", reused_bitmaps_nr); - trace2_data_intmax("pack-bitmap-write", writer->repo, - "building_bitmaps_pseudo_merge_reused", - reused_pseudo_merge_bitmaps_nr); trace2_data_intmax("pack-bitmap-write", writer->repo, "fill_bitmap_commit_calls_nr", fill_bitmap_commit_calls_nr); @@ -1015,23 +1171,6 @@ static void write_pseudo_merges(struct bitmap_writer *writer, CALLOC_ARRAY(pseudo_merge_ofs, writer->pseudo_merges_nr); - for (i = 0; i < writer->pseudo_merges_nr; i++) { - struct bitmapped_commit *merge = &writer->selected[base + i]; - struct commit_list *p; - struct bitmap *parents = bitmap_new(); - - if (!merge->pseudo_merge) - BUG("found non-pseudo merge commit at %"PRIuMAX, (uintmax_t)i); - - for (p = merge->commit->parents; p; p = p->next) - bitmap_set(parents, - find_object_pos(writer, &p->item->object.oid, - NULL)); - - merge->pseudo_merge_parents = bitmap_to_ewah(parents); - bitmap_free(parents); - } - start = hashfile_total(f); for (i = 0; i < writer->pseudo_merges_nr; i++) { @@ -1040,14 +1179,13 @@ static void write_pseudo_merges(struct bitmap_writer *writer, if (!merge->pseudo_merge) BUG("found non-pseudo merge commit at %"PRIuMAX, (uintmax_t)i); - if (!merge->pseudo_merge_parents) - BUG("missing pseudo-merge parents bitmap for commit %s", + if (!merge->pseudo_merge_parents || !merge->bitmap) + BUG("missing pseudo-merge bitmap for commit %s", oid_to_hex(&merge->commit->object.oid)); pseudo_merge_ofs[i] = hashfile_total(f); - dump_bitmap(f, merge->pseudo_merge_parents); - dump_bitmap(f, writer->selected[base+i].write_as); + dump_bitmap(f, merge->bitmap); } next_ext = st_add(hashfile_total(f),