kernel
/
git
mirror of https://git.kernel.org/pub/scm/git/git.git
1
0
Fork 0
Code Releases Activity

Merge branch 'ds/midx-write-fixes' 

Fixes multiple crashes around midx write-out codepaths.

* ds/midx-write-fixes:
  midx-write: simplify error cases
  midx-write: reenable signed comparison errors
  midx-write: use uint32_t for preferred_pack_idx
  midx-write: use cleanup when incremental midx fails
  midx-write: put failing response value back
  midx-write: only load initialized packs
main
Junio C Hamano 2025-09-15 08:52:06 -07:00
parent 13d1e86888 c25651aefd
commit e18e761bef
2 changed files with 84 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

131
midx-write.c
View File

@ -1,5 +1,3 @@
#define DISABLE_SIGN_COMPARE_WARNINGS

#include "git-compat-util.h" #include "git-compat-util.h"
#include "abspath.h" #include "abspath.h"
#include "config.h" #include "config.h"
@ -24,6 +22,7 @@
#define BITMAP_POS_UNKNOWN (~((uint32_t)0)) #define BITMAP_POS_UNKNOWN (~((uint32_t)0))
#define MIDX_CHUNK_FANOUT_SIZE (sizeof(uint32_t) * 256) #define MIDX_CHUNK_FANOUT_SIZE (sizeof(uint32_t) * 256)
#define MIDX_CHUNK_LARGE_OFFSET_WIDTH (sizeof(uint64_t)) #define MIDX_CHUNK_LARGE_OFFSET_WIDTH (sizeof(uint64_t))
#define NO_PREFERRED_PACK (~((uint32_t)0))


extern int midx_checksum_valid(struct multi_pack_index *m); extern int midx_checksum_valid(struct multi_pack_index *m);
extern void clear_midx_files_ext(struct odb_source *source, const char *ext, extern void clear_midx_files_ext(struct odb_source *source, const char *ext,
@ -104,7 +103,7 @@ struct write_midx_context {
unsigned large_offsets_needed:1; unsigned large_offsets_needed:1;
uint32_t num_large_offsets; uint32_t num_large_offsets;


int preferred_pack_idx; uint32_t preferred_pack_idx;


int incremental; int incremental;
uint32_t num_multi_pack_indexes_before; uint32_t num_multi_pack_indexes_before;
@ -261,7 +260,7 @@ static void midx_fanout_sort(struct midx_fanout *fanout)
static void midx_fanout_add_midx_fanout(struct midx_fanout *fanout, static void midx_fanout_add_midx_fanout(struct midx_fanout *fanout,
struct multi_pack_index *m, struct multi_pack_index *m,
uint32_t cur_fanout, uint32_t cur_fanout,
int preferred_pack) uint32_t preferred_pack)
{ {
uint32_t start = m->num_objects_in_base, end; uint32_t start = m->num_objects_in_base, end;
uint32_t cur_object; uint32_t cur_object;
@ -275,7 +274,7 @@ static void midx_fanout_add_midx_fanout(struct midx_fanout *fanout,
end = m->num_objects_in_base + ntohl(m->chunk_oid_fanout[cur_fanout]); end = m->num_objects_in_base + ntohl(m->chunk_oid_fanout[cur_fanout]);


for (cur_object = start; cur_object < end; cur_object++) { for (cur_object = start; cur_object < end; cur_object++) {
if ((preferred_pack > -1) && if ((preferred_pack != NO_PREFERRED_PACK) &&
(preferred_pack == nth_midxed_pack_int_id(m, cur_object))) { (preferred_pack == nth_midxed_pack_int_id(m, cur_object))) {
/* /*
* Objects from preferred packs are added * Objects from preferred packs are added
@ -365,7 +364,8 @@ static void compute_sorted_entries(struct write_midx_context *ctx,
preferred, cur_fanout); preferred, cur_fanout);
} }


if (-1 < ctx->preferred_pack_idx && ctx->preferred_pack_idx < start_pack) if (ctx->preferred_pack_idx != NO_PREFERRED_PACK &&
ctx->preferred_pack_idx < start_pack)
midx_fanout_add_pack_fanout(&fanout, ctx->info, midx_fanout_add_pack_fanout(&fanout, ctx->info,
ctx->preferred_pack_idx, 1, ctx->preferred_pack_idx, 1,
cur_fanout); cur_fanout);
@ -841,7 +841,7 @@ static int write_midx_bitmap(struct write_midx_context *ctx,
uint32_t commits_nr, uint32_t commits_nr,
unsigned flags) unsigned flags)
{ {
int ret, i; int ret;
uint16_t options = 0; uint16_t options = 0;
struct bitmap_writer writer; struct bitmap_writer writer;
struct pack_idx_entry **index; struct pack_idx_entry **index;
@ -868,7 +868,7 @@ static int write_midx_bitmap(struct write_midx_context *ctx,
* this order). * this order).
*/ */
ALLOC_ARRAY(index, pdata->nr_objects); ALLOC_ARRAY(index, pdata->nr_objects);
for (i = 0; i < pdata->nr_objects; i++) for (uint32_t i = 0; i < pdata->nr_objects; i++)
index[i] = &pdata->objects[i].idx; index[i] = &pdata->objects[i].idx;


bitmap_writer_init(&writer, ctx->repo, pdata, bitmap_writer_init(&writer, ctx->repo, pdata,
@ -889,7 +889,7 @@ static int write_midx_bitmap(struct write_midx_context *ctx,
* happens between bitmap_writer_build_type_index() and * happens between bitmap_writer_build_type_index() and
* bitmap_writer_finish(). * bitmap_writer_finish().
*/ */
for (i = 0; i < pdata->nr_objects; i++) for (uint32_t i = 0; i < pdata->nr_objects; i++)
index[ctx->pack_order[i]] = &pdata->objects[i].idx; index[ctx->pack_order[i]] = &pdata->objects[i].idx;


bitmap_writer_select_commits(&writer, commits, commits_nr); bitmap_writer_select_commits(&writer, commits, commits_nr);
@ -910,8 +910,7 @@ cleanup:
return ret; return ret;
} }


static int fill_packs_from_midx(struct write_midx_context *ctx, static int fill_packs_from_midx(struct write_midx_context *ctx)
const char *preferred_pack_name, uint32_t flags)
{ {
struct multi_pack_index *m; struct multi_pack_index *m;


@ -919,29 +918,10 @@ static int fill_packs_from_midx(struct write_midx_context *ctx,
uint32_t i; uint32_t i;


for (i = 0; i < m->num_packs; i++) { for (i = 0; i < m->num_packs; i++) {
if (prepare_midx_pack(m, m->num_packs_in_base + i))
return error(_("could not load pack"));

ALLOC_GROW(ctx->info, ctx->nr + 1, ctx->alloc); ALLOC_GROW(ctx->info, ctx->nr + 1, ctx->alloc);

/*
* If generating a reverse index, need to have
* packed_git's loaded to compare their
* mtimes and object count.
*
* If a preferred pack is specified, need to
* have packed_git's loaded to ensure the chosen
* preferred pack has a non-zero object count.
*/
if (flags & MIDX_WRITE_REV_INDEX ||
preferred_pack_name) {
if (prepare_midx_pack(m, m->num_packs_in_base + i)) {
error(_("could not load pack"));
return 1;
}

if (open_pack_index(m->packs[i]))
die(_("could not open index for %s"),
m->packs[i]->pack_name);
}

fill_pack_info(&ctx->info[ctx->nr++], m->packs[i], fill_pack_info(&ctx->info[ctx->nr++], m->packs[i],
m->pack_names[i], m->pack_names[i],
m->num_packs_in_base + i); m->num_packs_in_base + i);
@ -1045,15 +1025,17 @@ static int write_midx_internal(struct odb_source *source,
struct repository *r = source->odb->repo; struct repository *r = source->odb->repo;
struct strbuf midx_name = STRBUF_INIT; struct strbuf midx_name = STRBUF_INIT;
unsigned char midx_hash[GIT_MAX_RAWSZ]; unsigned char midx_hash[GIT_MAX_RAWSZ];
uint32_t i, start_pack; uint32_t start_pack;
struct hashfile *f = NULL; struct hashfile *f = NULL;
struct lock_file lk; struct lock_file lk;
struct tempfile *incr; struct tempfile *incr;
struct write_midx_context ctx = { 0 }; struct write_midx_context ctx = {
.preferred_pack_idx = NO_PREFERRED_PACK,
};
int bitmapped_packs_concat_len = 0; int bitmapped_packs_concat_len = 0;
int pack_name_concat_len = 0; int pack_name_concat_len = 0;
int dropped_packs = 0; int dropped_packs = 0;
int result = 0; int result = -1;
const char **keep_hashes = NULL; const char **keep_hashes = NULL;
struct chunkfile *cf; struct chunkfile *cf;


@ -1107,14 +1089,12 @@ static int write_midx_internal(struct odb_source *source,
error(_("could not load reverse index for MIDX %s"), error(_("could not load reverse index for MIDX %s"),
hash_to_hex_algop(get_midx_checksum(m), hash_to_hex_algop(get_midx_checksum(m),
m->source->odb->repo->hash_algo)); m->source->odb->repo->hash_algo));
result = 1;
goto cleanup; goto cleanup;
} }
ctx.num_multi_pack_indexes_before++; ctx.num_multi_pack_indexes_before++;
m = m->base_midx; m = m->base_midx;
} }
} else if (ctx.m && fill_packs_from_midx(&ctx, preferred_pack_name, } else if (ctx.m && fill_packs_from_midx(&ctx)) {
flags) < 0) {
goto cleanup; goto cleanup;
} }


@ -1150,17 +1130,20 @@ static int write_midx_internal(struct odb_source *source,
*/ */
if (!want_bitmap) if (!want_bitmap)
clear_midx_files_ext(source, "bitmap", NULL); clear_midx_files_ext(source, "bitmap", NULL);
result = 0;
goto cleanup; goto cleanup;
} }
} }


if (ctx.incremental && !ctx.nr) if (ctx.incremental && !ctx.nr) {
result = 0;
goto cleanup; /* nothing to do */ goto cleanup; /* nothing to do */
}


if (preferred_pack_name) { if (preferred_pack_name) {
ctx.preferred_pack_idx = -1; ctx.preferred_pack_idx = NO_PREFERRED_PACK;


for (i = 0; i < ctx.nr; i++) { for (size_t i = 0; i < ctx.nr; i++) {
if (!cmp_idx_or_pack_name(preferred_pack_name, if (!cmp_idx_or_pack_name(preferred_pack_name,
ctx.info[i].pack_name)) { ctx.info[i].pack_name)) {
ctx.preferred_pack_idx = i; ctx.preferred_pack_idx = i;
@ -1168,14 +1151,21 @@ static int write_midx_internal(struct odb_source *source,
} }
} }


if (ctx.preferred_pack_idx == -1) if (ctx.preferred_pack_idx == NO_PREFERRED_PACK)
warning(_("unknown preferred pack: '%s'"), warning(_("unknown preferred pack: '%s'"),
preferred_pack_name); preferred_pack_name);
} else if (ctx.nr && } else if (ctx.nr &&
(flags & (MIDX_WRITE_REV_INDEX | MIDX_WRITE_BITMAP))) { (flags & (MIDX_WRITE_REV_INDEX | MIDX_WRITE_BITMAP))) {
struct packed_git *oldest = ctx.info[ctx.preferred_pack_idx].p; struct packed_git *oldest = ctx.info[0].p;
ctx.preferred_pack_idx = 0; ctx.preferred_pack_idx = 0;


/*
* Attempt opening the pack index to populate num_objects.
* Ignore failiures as they can be expected and are not
* fatal during this selection time.
*/
open_pack_index(oldest);

if (packs_to_drop && packs_to_drop->nr) if (packs_to_drop && packs_to_drop->nr)
BUG("cannot write a MIDX bitmap during expiration"); BUG("cannot write a MIDX bitmap during expiration");


@ -1185,11 +1175,12 @@ static int write_midx_internal(struct odb_source *source,
* pack-order has all of its objects selected from that pack * pack-order has all of its objects selected from that pack
* (and not another pack containing a duplicate) * (and not another pack containing a duplicate)
*/ */
for (i = 1; i < ctx.nr; i++) { for (size_t i = 1; i < ctx.nr; i++) {
struct packed_git *p = ctx.info[i].p; struct packed_git *p = ctx.info[i].p;


if (!oldest->num_objects || p->mtime < oldest->mtime) { if (!oldest->num_objects || p->mtime < oldest->mtime) {
oldest = p; oldest = p;
open_pack_index(oldest);
ctx.preferred_pack_idx = i; ctx.preferred_pack_idx = i;
} }
} }
@ -1201,22 +1192,26 @@ static int write_midx_internal(struct odb_source *source,
* objects to resolve, so the preferred value doesn't * objects to resolve, so the preferred value doesn't
* matter. * matter.
*/ */
ctx.preferred_pack_idx = -1; ctx.preferred_pack_idx = NO_PREFERRED_PACK;
} }
} else { } else {
/* /*
* otherwise don't mark any pack as preferred to avoid * otherwise don't mark any pack as preferred to avoid
* interfering with expiration logic below * interfering with expiration logic below
*/ */
ctx.preferred_pack_idx = -1; ctx.preferred_pack_idx = NO_PREFERRED_PACK;
} }


if (ctx.preferred_pack_idx > -1) { if (ctx.preferred_pack_idx != NO_PREFERRED_PACK) {
struct packed_git *preferred = ctx.info[ctx.preferred_pack_idx].p; struct packed_git *preferred = ctx.info[ctx.preferred_pack_idx].p;

if (open_pack_index(preferred))
die(_("failed to open preferred pack %s"),
ctx.info[ctx.preferred_pack_idx].pack_name);

if (!preferred->num_objects) { if (!preferred->num_objects) {
error(_("cannot select preferred pack %s with no objects"), error(_("cannot select preferred pack %s with no objects"),
preferred->pack_name); preferred->pack_name);
result = 1;
goto cleanup; goto cleanup;
} }
} }
@ -1224,7 +1219,7 @@ static int write_midx_internal(struct odb_source *source,
compute_sorted_entries(&ctx, start_pack); compute_sorted_entries(&ctx, start_pack);


ctx.large_offsets_needed = 0; ctx.large_offsets_needed = 0;
for (i = 0; i < ctx.entries_nr; i++) { for (size_t i = 0; i < ctx.entries_nr; i++) {
if (ctx.entries[i].offset > 0x7fffffff) if (ctx.entries[i].offset > 0x7fffffff)
ctx.num_large_offsets++; ctx.num_large_offsets++;
if (ctx.entries[i].offset > 0xffffffff) if (ctx.entries[i].offset > 0xffffffff)
@ -1234,10 +1229,10 @@ static int write_midx_internal(struct odb_source *source,
QSORT(ctx.info, ctx.nr, pack_info_compare); QSORT(ctx.info, ctx.nr, pack_info_compare);


if (packs_to_drop && packs_to_drop->nr) { if (packs_to_drop && packs_to_drop->nr) {
int drop_index = 0; size_t drop_index = 0;
int missing_drops = 0; int missing_drops = 0;


for (i = 0; i < ctx.nr && drop_index < packs_to_drop->nr; i++) { for (size_t i = 0; i < ctx.nr && drop_index < packs_to_drop->nr; i++) {
int cmp = strcmp(ctx.info[i].pack_name, int cmp = strcmp(ctx.info[i].pack_name,
packs_to_drop->items[drop_index].string); packs_to_drop->items[drop_index].string);


@ -1255,11 +1250,9 @@ static int write_midx_internal(struct odb_source *source,
} }
} }


if (missing_drops) { if (missing_drops)
result = 1;
goto cleanup; goto cleanup;
} }
}


/* /*
* pack_perm stores a permutation between pack-int-ids from the * pack_perm stores a permutation between pack-int-ids from the
@ -1268,7 +1261,7 @@ static int write_midx_internal(struct odb_source *source,
* pack_perm[old_id] = new_id * pack_perm[old_id] = new_id
*/ */
ALLOC_ARRAY(ctx.pack_perm, ctx.nr); ALLOC_ARRAY(ctx.pack_perm, ctx.nr);
for (i = 0; i < ctx.nr; i++) { for (size_t i = 0; i < ctx.nr; i++) {
if (ctx.info[i].expired) { if (ctx.info[i].expired) {
dropped_packs++; dropped_packs++;
ctx.pack_perm[ctx.info[i].orig_pack_int_id] = PACK_EXPIRED; ctx.pack_perm[ctx.info[i].orig_pack_int_id] = PACK_EXPIRED;
@ -1277,7 +1270,7 @@ static int write_midx_internal(struct odb_source *source,
} }
} }


for (i = 0; i < ctx.nr; i++) { for (size_t i = 0; i < ctx.nr; i++) {
if (ctx.info[i].expired) if (ctx.info[i].expired)
continue; continue;
pack_name_concat_len += strlen(ctx.info[i].pack_name) + 1; pack_name_concat_len += strlen(ctx.info[i].pack_name) + 1;
@ -1304,7 +1297,6 @@ static int write_midx_internal(struct odb_source *source,


if (ctx.nr - dropped_packs == 0) { if (ctx.nr - dropped_packs == 0) {
error(_("no pack files to index.")); error(_("no pack files to index."));
result = 1;
goto cleanup; goto cleanup;
} }


@ -1324,13 +1316,13 @@ static int write_midx_internal(struct odb_source *source,
incr = mks_tempfile_m(midx_name.buf, 0444); incr = mks_tempfile_m(midx_name.buf, 0444);
if (!incr) { if (!incr) {
error(_("unable to create temporary MIDX layer")); error(_("unable to create temporary MIDX layer"));
return -1; goto cleanup;
} }


if (adjust_shared_perm(r, get_tempfile_path(incr))) { if (adjust_shared_perm(r, get_tempfile_path(incr))) {
error(_("unable to adjust shared permissions for '%s'"), error(_("unable to adjust shared permissions for '%s'"),
get_tempfile_path(incr)); get_tempfile_path(incr));
return -1; goto cleanup;
} }


f = hashfd(r->hash_algo, get_tempfile_fd(incr), f = hashfd(r->hash_algo, get_tempfile_fd(incr),
@ -1407,7 +1399,6 @@ static int write_midx_internal(struct odb_source *source,
midx_hash, &pdata, commits, commits_nr, midx_hash, &pdata, commits, commits_nr,
flags) < 0) { flags) < 0) {
error(_("could not write multi-pack bitmap")); error(_("could not write multi-pack bitmap"));
result = 1;
clear_packing_data(&pdata); clear_packing_data(&pdata);
free(commits); free(commits);
goto cleanup; goto cleanup;
@ -1421,6 +1412,9 @@ static int write_midx_internal(struct odb_source *source,
* have been freed in the previous if block. * have been freed in the previous if block.
*/ */


if (ctx.num_multi_pack_indexes_before == UINT32_MAX)
die(_("too many multi-pack-indexes"));

CALLOC_ARRAY(keep_hashes, ctx.num_multi_pack_indexes_before + 1); CALLOC_ARRAY(keep_hashes, ctx.num_multi_pack_indexes_before + 1);


if (ctx.incremental) { if (ctx.incremental) {
@ -1430,18 +1424,18 @@ static int write_midx_internal(struct odb_source *source,


if (!chainf) { if (!chainf) {
error_errno(_("unable to open multi-pack-index chain file")); error_errno(_("unable to open multi-pack-index chain file"));
return -1; goto cleanup;
} }


if (link_midx_to_chain(ctx.base_midx) < 0) if (link_midx_to_chain(ctx.base_midx) < 0)
return -1; goto cleanup;


get_split_midx_filename_ext(source, &final_midx_name, get_split_midx_filename_ext(source, &final_midx_name,
midx_hash, MIDX_EXT_MIDX); midx_hash, MIDX_EXT_MIDX);


if (rename_tempfile(&incr, final_midx_name.buf) < 0) { if (rename_tempfile(&incr, final_midx_name.buf) < 0) {
error_errno(_("unable to rename new multi-pack-index layer")); error_errno(_("unable to rename new multi-pack-index layer"));
return -1; goto cleanup;
} }


strbuf_release(&final_midx_name); strbuf_release(&final_midx_name);
@ -1449,7 +1443,7 @@ static int write_midx_internal(struct odb_source *source,
keep_hashes[ctx.num_multi_pack_indexes_before] = keep_hashes[ctx.num_multi_pack_indexes_before] =
xstrdup(hash_to_hex_algop(midx_hash, r->hash_algo)); xstrdup(hash_to_hex_algop(midx_hash, r->hash_algo));


for (i = 0; i < ctx.num_multi_pack_indexes_before; i++) { for (uint32_t i = 0; i < ctx.num_multi_pack_indexes_before; i++) {
uint32_t j = ctx.num_multi_pack_indexes_before - i - 1; uint32_t j = ctx.num_multi_pack_indexes_before - i - 1;


keep_hashes[j] = xstrdup(hash_to_hex_algop(get_midx_checksum(m), keep_hashes[j] = xstrdup(hash_to_hex_algop(get_midx_checksum(m),
@ -1457,7 +1451,7 @@ static int write_midx_internal(struct odb_source *source,
m = m->base_midx; m = m->base_midx;
} }


for (i = 0; i < ctx.num_multi_pack_indexes_before + 1; i++) for (uint32_t i = 0; i <= ctx.num_multi_pack_indexes_before; i++)
fprintf(get_lock_file_fp(&lk), "%s\n", keep_hashes[i]); fprintf(get_lock_file_fp(&lk), "%s\n", keep_hashes[i]);
} else { } else {
keep_hashes[ctx.num_multi_pack_indexes_before] = keep_hashes[ctx.num_multi_pack_indexes_before] =
@ -1473,9 +1467,10 @@ static int write_midx_internal(struct odb_source *source,
clear_midx_files(source, keep_hashes, clear_midx_files(source, keep_hashes,
ctx.num_multi_pack_indexes_before + 1, ctx.num_multi_pack_indexes_before + 1,
ctx.incremental); ctx.incremental);
result = 0;


cleanup: cleanup:
for (i = 0; i < ctx.nr; i++) { for (size_t i = 0; i < ctx.nr; i++) {
if (ctx.info[i].p) { if (ctx.info[i].p) {
close_pack(ctx.info[i].p); close_pack(ctx.info[i].p);
free(ctx.info[i].p); free(ctx.info[i].p);
@ -1488,7 +1483,7 @@ cleanup:
free(ctx.pack_perm); free(ctx.pack_perm);
free(ctx.pack_order); free(ctx.pack_order);
if (keep_hashes) { if (keep_hashes) {
for (i = 0; i < ctx.num_multi_pack_indexes_before + 1; i++) for (uint32_t i = 0; i <= ctx.num_multi_pack_indexes_before; i++)
free((char *)keep_hashes[i]); free((char *)keep_hashes[i]);
free(keep_hashes); free(keep_hashes);
} }

22
t/t5319-multi-pack-index.sh
View File

@ -989,6 +989,23 @@ test_expect_success 'repack --batch-size=0 repacks everything' '
) )
' '


test_expect_success EXPENSIVE 'repack/expire with many packs' '
cp -r dup many &&
(
cd many &&

for i in $(test_seq 1 100)
do
test_commit extra$i &&
git maintenance run --task=loose-objects || return 1
done &&

git multi-pack-index write &&
git multi-pack-index repack &&
git multi-pack-index expire
)
'

test_expect_success 'repack --batch-size=<large> repacks everything' ' test_expect_success 'repack --batch-size=<large> repacks everything' '
( (
cd dup2 && cd dup2 &&
@ -1083,7 +1100,10 @@ test_expect_success 'load reverse index when missing .idx, .pack' '
mv $idx.bak $idx && mv $idx.bak $idx &&


mv $pack $pack.bak && mv $pack $pack.bak &&
git cat-file --batch-check="%(objectsize:disk)" <tip git cat-file --batch-check="%(objectsize:disk)" <tip &&

test_must_fail git multi-pack-index write 2>err &&
test_grep "could not load pack" err
) )
' '