From 56ee7ff15655fa1c9a1368ddd717fc84556359bc Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 13 Sep 2018 11:02:13 -0700 Subject: [PATCH 01/11] multi-pack-index: add 'verify' verb The multi-pack-index builtin writes multi-pack-index files, and uses a 'write' verb to do so. Add a 'verify' verb that checks this file matches the contents of the pack-indexes it replaces. The current implementation is a no-op, but will be extended in small increments in later commits. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- Documentation/git-multi-pack-index.txt | 10 ++++++++++ builtin/multi-pack-index.c | 4 +++- midx.c | 13 +++++++++++++ midx.h | 1 + t/t5319-multi-pack-index.sh | 8 ++++++++ 5 files changed, 35 insertions(+), 1 deletion(-) diff --git a/Documentation/git-multi-pack-index.txt b/Documentation/git-multi-pack-index.txt index 1f97e79912..f7778a2c85 100644 --- a/Documentation/git-multi-pack-index.txt +++ b/Documentation/git-multi-pack-index.txt @@ -27,6 +27,10 @@ write:: When given as the verb, write a new MIDX file to `/packs/multi-pack-index`. +verify:: + When given as the verb, verify the contents of the MIDX file + at `/packs/multi-pack-index`. + EXAMPLES -------- @@ -43,6 +47,12 @@ $ git multi-pack-index write $ git multi-pack-index --object-dir write ----------------------------------------------- +* Verify the MIDX file for the packfiles in the current .git folder. ++ +----------------------------------------------- +$ git multi-pack-index verify +----------------------------------------------- + SEE ALSO -------- diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index 2633efd95d..fca70f8e4f 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -5,7 +5,7 @@ #include "midx.h" static char const * const builtin_multi_pack_index_usage[] = { - N_("git multi-pack-index [--object-dir=] write"), + N_("git multi-pack-index [--object-dir=] (write|verify)"), NULL }; @@ -42,6 +42,8 @@ int cmd_multi_pack_index(int argc, const char **argv, if (!strcmp(argv[0], "write")) return write_midx_file(opts.object_dir); + if (!strcmp(argv[0], "verify")) + return verify_midx_file(opts.object_dir); die(_("unrecognized verb: %s"), argv[0]); } diff --git a/midx.c b/midx.c index f3e8dbc108..8b6c855d48 100644 --- a/midx.c +++ b/midx.c @@ -928,3 +928,16 @@ void clear_midx_file(const char *object_dir) free(midx); } + +static int verify_midx_error; + +int verify_midx_file(const char *object_dir) +{ + struct multi_pack_index *m = load_multi_pack_index(object_dir, 1); + verify_midx_error = 0; + + if (!m) + return 0; + + return verify_midx_error; +} diff --git a/midx.h b/midx.h index a210f1af2a..ce80b91c68 100644 --- a/midx.h +++ b/midx.h @@ -43,5 +43,6 @@ int prepare_multi_pack_index_one(struct repository *r, const char *object_dir, i int write_midx_file(const char *object_dir); void clear_midx_file(const char *object_dir); +int verify_midx_file(const char *object_dir); #endif diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index 6f56b38674..1c4e0e6d31 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -150,6 +150,10 @@ test_expect_success 'write midx with twelve packs' ' compare_results_with_midx "twelve packs" +test_expect_success 'verify multi-pack-index success' ' + git multi-pack-index verify --object-dir=$objdir +' + test_expect_success 'repack removes multi-pack-index' ' test_path_is_file $objdir/pack/multi-pack-index && git repack -adf && @@ -214,4 +218,8 @@ test_expect_success 'force some 64-bit offsets with pack-objects' ' midx_read_expect 1 63 5 objects64 " large-offsets" ' +test_expect_success 'verify multi-pack-index with 64-bit offsets' ' + git multi-pack-index verify --object-dir=objects64 +' + test_done From 53ad0407444ac4da835dbe9cf85c272b4065f3b4 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 13 Sep 2018 11:02:15 -0700 Subject: [PATCH 02/11] multi-pack-index: verify bad header When verifying if a multi-pack-index file is valid, we want the command to fail to signal an invalid file. Previously, we wrote an error to stderr and continued as if we had no multi-pack-index. Now, die() instead of error(). Add tests that check corrupted headers in a few ways: * Bad signature * Bad file version * Bad hash version * Truncated hash count * Extended hash count Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- midx.c | 18 +++++---------- t/t5319-multi-pack-index.sh | 46 ++++++++++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/midx.c b/midx.c index 8b6c855d48..7199b8392b 100644 --- a/midx.c +++ b/midx.c @@ -76,24 +76,18 @@ struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local m->local = local; m->signature = get_be32(m->data); - if (m->signature != MIDX_SIGNATURE) { - error(_("multi-pack-index signature 0x%08x does not match signature 0x%08x"), + if (m->signature != MIDX_SIGNATURE) + die(_("multi-pack-index signature 0x%08x does not match signature 0x%08x"), m->signature, MIDX_SIGNATURE); - goto cleanup_fail; - } m->version = m->data[MIDX_BYTE_FILE_VERSION]; - if (m->version != MIDX_VERSION) { - error(_("multi-pack-index version %d not recognized"), + if (m->version != MIDX_VERSION) + die(_("multi-pack-index version %d not recognized"), m->version); - goto cleanup_fail; - } hash_version = m->data[MIDX_BYTE_HASH_VERSION]; - if (hash_version != MIDX_HASH_VERSION) { - error(_("hash version %u does not match"), hash_version); - goto cleanup_fail; - } + if (hash_version != MIDX_HASH_VERSION) + die(_("hash version %u does not match"), hash_version); m->hash_len = MIDX_HASH_LEN; m->num_chunks = m->data[MIDX_BYTE_NUM_CHUNKS]; diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index 1c4e0e6d31..e04b5f43a2 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -154,6 +154,51 @@ test_expect_success 'verify multi-pack-index success' ' git multi-pack-index verify --object-dir=$objdir ' +# usage: corrupt_midx_and_verify +corrupt_midx_and_verify() { + POS=$1 && + DATA="${2:-\0}" && + OBJDIR=$3 && + GREPSTR="$4" && + FILE=$OBJDIR/pack/multi-pack-index && + chmod a+w $FILE && + test_when_finished mv midx-backup $FILE && + cp $FILE midx-backup && + printf "$DATA" | dd of="$FILE" bs=1 seek="$POS" conv=notrunc && + test_must_fail git multi-pack-index verify --object-dir=$OBJDIR 2>test_err && + grep -v "^+" test_err >err && + test_i18ngrep "$GREPSTR" err +} + +test_expect_success 'verify bad signature' ' + corrupt_midx_and_verify 0 "\00" $objdir \ + "multi-pack-index signature" +' + +MIDX_BYTE_VERSION=4 +MIDX_BYTE_OID_VERSION=5 +MIDX_BYTE_CHUNK_COUNT=6 + +test_expect_success 'verify bad version' ' + corrupt_midx_and_verify $MIDX_BYTE_VERSION "\00" $objdir \ + "multi-pack-index version" +' + +test_expect_success 'verify bad OID version' ' + corrupt_midx_and_verify $MIDX_BYTE_OID_VERSION "\02" $objdir \ + "hash version" +' + +test_expect_success 'verify truncated chunk count' ' + corrupt_midx_and_verify $MIDX_BYTE_CHUNK_COUNT "\01" $objdir \ + "missing required" +' + +test_expect_success 'verify extended chunk count' ' + corrupt_midx_and_verify $MIDX_BYTE_CHUNK_COUNT "\07" $objdir \ + "terminating multi-pack-index chunk id appears earlier than expected" +' + test_expect_success 'repack removes multi-pack-index' ' test_path_is_file $objdir/pack/multi-pack-index && git repack -adf && @@ -191,7 +236,6 @@ test_expect_success 'multi-pack-index in an alternate' ' compare_results_with_midx "with alternate (remote midx)" - # usage: corrupt_data [] corrupt_data () { file=$1 From d3f8e211700c6c38f905a7d7ab1338df4784c79d Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 13 Sep 2018 11:02:16 -0700 Subject: [PATCH 03/11] multi-pack-index: verify corrupt chunk lookup table Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- midx.c | 3 +++ t/t5319-multi-pack-index.sh | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/midx.c b/midx.c index 7199b8392b..9e43216d09 100644 --- a/midx.c +++ b/midx.c @@ -100,6 +100,9 @@ struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local uint64_t chunk_offset = get_be64(m->data + MIDX_HEADER_SIZE + 4 + MIDX_CHUNKLOOKUP_WIDTH * i); + if (chunk_offset >= m->data_len) + die(_("invalid chunk offset (too large)")); + switch (chunk_id) { case MIDX_CHUNKID_PACKNAMES: m->chunk_pack_names = m->data + chunk_offset; diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index e04b5f43a2..c54b6e7188 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -178,6 +178,9 @@ test_expect_success 'verify bad signature' ' MIDX_BYTE_VERSION=4 MIDX_BYTE_OID_VERSION=5 MIDX_BYTE_CHUNK_COUNT=6 +MIDX_HEADER_SIZE=12 +MIDX_BYTE_CHUNK_ID=$MIDX_HEADER_SIZE +MIDX_BYTE_CHUNK_OFFSET=$(($MIDX_HEADER_SIZE + 4)) test_expect_success 'verify bad version' ' corrupt_midx_and_verify $MIDX_BYTE_VERSION "\00" $objdir \ @@ -199,6 +202,16 @@ test_expect_success 'verify extended chunk count' ' "terminating multi-pack-index chunk id appears earlier than expected" ' +test_expect_success 'verify missing required chunk' ' + corrupt_midx_and_verify $MIDX_BYTE_CHUNK_ID "\01" $objdir \ + "missing required" +' + +test_expect_success 'verify invalid chunk offset' ' + corrupt_midx_and_verify $MIDX_BYTE_CHUNK_OFFSET "\01" $objdir \ + "invalid chunk offset (too large)" +' + test_expect_success 'repack removes multi-pack-index' ' test_path_is_file $objdir/pack/multi-pack-index && git repack -adf && From 8e72a3c321eb70caad036e45664de9ea10839b93 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 13 Sep 2018 11:02:18 -0700 Subject: [PATCH 04/11] multi-pack-index: verify packname order The final check we make while loading a multi-pack-index is that the packfile names are in lexicographical order. Make this error be a die() instead. In order to test this condition, we need multiple packfiles. Earlier in t5319-multi-pack-index.sh, we tested the interaction with 'git repack' but this limits us to one packfile in our object dir. Move these repack tests until after the 'verify' tests. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- midx.c | 6 ++---- t/t5319-multi-pack-index.sh | 10 ++++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/midx.c b/midx.c index 9e43216d09..939e8fa391 100644 --- a/midx.c +++ b/midx.c @@ -157,12 +157,10 @@ struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local cur_pack_name += strlen(cur_pack_name) + 1; - if (i && strcmp(m->pack_names[i], m->pack_names[i - 1]) <= 0) { - error(_("multi-pack-index pack names out of order: '%s' before '%s'"), + if (i && strcmp(m->pack_names[i], m->pack_names[i - 1]) <= 0) + die(_("multi-pack-index pack names out of order: '%s' before '%s'"), m->pack_names[i - 1], m->pack_names[i]); - goto cleanup_fail; - } } return m; diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index c54b6e7188..01a3cd6b00 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -181,6 +181,11 @@ MIDX_BYTE_CHUNK_COUNT=6 MIDX_HEADER_SIZE=12 MIDX_BYTE_CHUNK_ID=$MIDX_HEADER_SIZE MIDX_BYTE_CHUNK_OFFSET=$(($MIDX_HEADER_SIZE + 4)) +MIDX_NUM_CHUNKS=5 +MIDX_CHUNK_LOOKUP_WIDTH=12 +MIDX_OFFSET_PACKNAMES=$(($MIDX_HEADER_SIZE + \ + $MIDX_NUM_CHUNKS * $MIDX_CHUNK_LOOKUP_WIDTH)) +MIDX_BYTE_PACKNAME_ORDER=$(($MIDX_OFFSET_PACKNAMES + 2)) test_expect_success 'verify bad version' ' corrupt_midx_and_verify $MIDX_BYTE_VERSION "\00" $objdir \ @@ -212,6 +217,11 @@ test_expect_success 'verify invalid chunk offset' ' "invalid chunk offset (too large)" ' +test_expect_success 'verify packnames out of order' ' + corrupt_midx_and_verify $MIDX_BYTE_PACKNAME_ORDER "z" $objdir \ + "pack names out of order" +' + test_expect_success 'repack removes multi-pack-index' ' test_path_is_file $objdir/pack/multi-pack-index && git repack -adf && From d4bf1d88b9020d3921ac4e4d8a8fd6ca5015b243 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 13 Sep 2018 11:02:19 -0700 Subject: [PATCH 05/11] multi-pack-index: verify missing pack Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- midx.c | 16 ++++++++++++++++ t/t5319-multi-pack-index.sh | 5 +++++ 2 files changed, 21 insertions(+) diff --git a/midx.c b/midx.c index 939e8fa391..01f4d694d2 100644 --- a/midx.c +++ b/midx.c @@ -926,13 +926,29 @@ void clear_midx_file(const char *object_dir) static int verify_midx_error; +static void midx_report(const char *fmt, ...) +{ + va_list ap; + verify_midx_error = 1; + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + fprintf(stderr, "\n"); + va_end(ap); +} + int verify_midx_file(const char *object_dir) { + uint32_t i; struct multi_pack_index *m = load_multi_pack_index(object_dir, 1); verify_midx_error = 0; if (!m) return 0; + for (i = 0; i < m->num_packs; i++) { + if (prepare_midx_pack(m, i)) + midx_report("failed to load pack in position %d", i); + } + return verify_midx_error; } diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index 01a3cd6b00..0a566afb05 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -222,6 +222,11 @@ test_expect_success 'verify packnames out of order' ' "pack names out of order" ' +test_expect_success 'verify packnames out of order' ' + corrupt_midx_and_verify $MIDX_BYTE_PACKNAME_ORDER "a" $objdir \ + "failed to load pack" +' + test_expect_success 'repack removes multi-pack-index' ' test_path_is_file $objdir/pack/multi-pack-index && git repack -adf && From 2f23d3f3f92dddfdb524203d0993e052b7d9e20e Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 13 Sep 2018 11:02:20 -0700 Subject: [PATCH 06/11] multi-pack-index: verify oid fanout order Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- midx.c | 9 +++++++++ t/t5319-multi-pack-index.sh | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/midx.c b/midx.c index 01f4d694d2..7dab13a736 100644 --- a/midx.c +++ b/midx.c @@ -950,5 +950,14 @@ int verify_midx_file(const char *object_dir) midx_report("failed to load pack in position %d", i); } + for (i = 0; i < 255; i++) { + uint32_t oid_fanout1 = ntohl(m->chunk_oid_fanout[i]); + uint32_t oid_fanout2 = ntohl(m->chunk_oid_fanout[i + 1]); + + if (oid_fanout1 > oid_fanout2) + midx_report(_("oid fanout out of order: fanout[%d] = %"PRIx32" > %"PRIx32" = fanout[%d]"), + i, oid_fanout1, oid_fanout2, i + 1); + } + return verify_midx_error; } diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index 0a566afb05..47a54e138d 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -186,6 +186,9 @@ MIDX_CHUNK_LOOKUP_WIDTH=12 MIDX_OFFSET_PACKNAMES=$(($MIDX_HEADER_SIZE + \ $MIDX_NUM_CHUNKS * $MIDX_CHUNK_LOOKUP_WIDTH)) MIDX_BYTE_PACKNAME_ORDER=$(($MIDX_OFFSET_PACKNAMES + 2)) +MIDX_OFFSET_OID_FANOUT=$(($MIDX_OFFSET_PACKNAMES + 652)) +MIDX_OID_FANOUT_WIDTH=4 +MIDX_BYTE_OID_FANOUT_ORDER=$((MIDX_OFFSET_OID_FANOUT + 250 * $MIDX_OID_FANOUT_WIDTH + 1)) test_expect_success 'verify bad version' ' corrupt_midx_and_verify $MIDX_BYTE_VERSION "\00" $objdir \ @@ -227,6 +230,11 @@ test_expect_success 'verify packnames out of order' ' "failed to load pack" ' +test_expect_success 'verify oid fanout out of order' ' + corrupt_midx_and_verify $MIDX_BYTE_OID_FANOUT_ORDER "\01" $objdir \ + "oid fanout out of order" +' + test_expect_success 'repack removes multi-pack-index' ' test_path_is_file $objdir/pack/multi-pack-index && git repack -adf && From 55c5648d804cea734f402c015ec9d2005373804e Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 13 Sep 2018 11:02:22 -0700 Subject: [PATCH 07/11] multi-pack-index: verify oid lookup order Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- midx.c | 11 +++++++++++ t/t5319-multi-pack-index.sh | 8 ++++++++ 2 files changed, 19 insertions(+) diff --git a/midx.c b/midx.c index 7dab13a736..c13c6f9d72 100644 --- a/midx.c +++ b/midx.c @@ -959,5 +959,16 @@ int verify_midx_file(const char *object_dir) i, oid_fanout1, oid_fanout2, i + 1); } + for (i = 0; i < m->num_objects - 1; i++) { + struct object_id oid1, oid2; + + nth_midxed_object_oid(&oid1, m, i); + nth_midxed_object_oid(&oid2, m, i + 1); + + if (oidcmp(&oid1, &oid2) >= 0) + midx_report(_("oid lookup out of order: oid[%d] = %s >= %s = oid[%d]"), + i, oid_to_hex(&oid1), oid_to_hex(&oid2), i + 1); + } + return verify_midx_error; } diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index 47a54e138d..a968b9a959 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -175,6 +175,7 @@ test_expect_success 'verify bad signature' ' "multi-pack-index signature" ' +HASH_LEN=20 MIDX_BYTE_VERSION=4 MIDX_BYTE_OID_VERSION=5 MIDX_BYTE_CHUNK_COUNT=6 @@ -189,6 +190,8 @@ MIDX_BYTE_PACKNAME_ORDER=$(($MIDX_OFFSET_PACKNAMES + 2)) MIDX_OFFSET_OID_FANOUT=$(($MIDX_OFFSET_PACKNAMES + 652)) MIDX_OID_FANOUT_WIDTH=4 MIDX_BYTE_OID_FANOUT_ORDER=$((MIDX_OFFSET_OID_FANOUT + 250 * $MIDX_OID_FANOUT_WIDTH + 1)) +MIDX_OFFSET_OID_LOOKUP=$(($MIDX_OFFSET_OID_FANOUT + 256 * $MIDX_OID_FANOUT_WIDTH)) +MIDX_BYTE_OID_LOOKUP=$(($MIDX_OFFSET_OID_LOOKUP + 16 * $HASH_LEN)) test_expect_success 'verify bad version' ' corrupt_midx_and_verify $MIDX_BYTE_VERSION "\00" $objdir \ @@ -235,6 +238,11 @@ test_expect_success 'verify oid fanout out of order' ' "oid fanout out of order" ' +test_expect_success 'verify oid lookup out of order' ' + corrupt_midx_and_verify $MIDX_BYTE_OID_LOOKUP "\00" $objdir \ + "oid lookup out of order" +' + test_expect_success 'repack removes multi-pack-index' ' test_path_is_file $objdir/pack/multi-pack-index && git repack -adf && From d8ac9ee109ade79783ea398200219960d352f503 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 13 Sep 2018 11:02:23 -0700 Subject: [PATCH 08/11] multi-pack-index: fix 32-bit vs 64-bit size check When loading a 64-bit offset, we intend to check that off_t can store the resulting offset. However, the condition accidentally checks the 32-bit offset to see if it is smaller than a 64-bit value. Fix it, and this will be covered by a test in the 'git multi-pack-index verify' command in a later commit. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- midx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/midx.c b/midx.c index c13c6f9d72..db9c49bb2b 100644 --- a/midx.c +++ b/midx.c @@ -236,7 +236,7 @@ static off_t nth_midxed_offset(struct multi_pack_index *m, uint32_t pos) offset32 = get_be32(offset_data + sizeof(uint32_t)); if (m->chunk_large_offsets && offset32 & MIDX_LARGE_OFFSET_NEEDED) { - if (sizeof(offset32) < sizeof(uint64_t)) + if (sizeof(off_t) < sizeof(uint64_t)) die(_("multi-pack-index stores a 64-bit offset, but off_t is too small")); offset32 ^= MIDX_LARGE_OFFSET_NEEDED; From cc6af73c029da23f7b2c98c60e4fba1ca2db215b Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 13 Sep 2018 11:02:25 -0700 Subject: [PATCH 09/11] multi-pack-index: verify object offsets The 'git multi-pack-index verify' command must verify the object offsets stored in the multi-pack-index are correct. There are two ways the offset chunk can be incorrect: the pack-int-id and the object offset. Replace the BUG() statement with a die() statement, now that we may hit a bad pack-int-id during a 'verify' command on a corrupt multi-pack-index, and it is covered by a test. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- midx.c | 29 ++++++++++++++++++++++++++++- t/t5319-multi-pack-index.sh | 27 +++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/midx.c b/midx.c index db9c49bb2b..ef4a6969df 100644 --- a/midx.c +++ b/midx.c @@ -197,7 +197,8 @@ int prepare_midx_pack(struct multi_pack_index *m, uint32_t pack_int_id) struct strbuf pack_name = STRBUF_INIT; if (pack_int_id >= m->num_packs) - BUG("bad pack-int-id"); + die(_("bad pack-int-id: %u (%u total packs"), + pack_int_id, m->num_packs); if (m->packs[pack_int_id]) return 0; @@ -970,5 +971,31 @@ int verify_midx_file(const char *object_dir) i, oid_to_hex(&oid1), oid_to_hex(&oid2), i + 1); } + for (i = 0; i < m->num_objects; i++) { + struct object_id oid; + struct pack_entry e; + off_t m_offset, p_offset; + + nth_midxed_object_oid(&oid, m, i); + if (!fill_midx_entry(&oid, &e, m)) { + midx_report(_("failed to load pack entry for oid[%d] = %s"), + i, oid_to_hex(&oid)); + continue; + } + + if (open_pack_index(e.p)) { + midx_report(_("failed to load pack-index for packfile %s"), + e.p->pack_name); + break; + } + + m_offset = e.offset; + p_offset = find_pack_entry_one(oid.hash, e.p); + + if (m_offset != p_offset) + midx_report(_("incorrect object offset for oid[%d] = %s: %"PRIx64" != %"PRIx64), + i, oid_to_hex(&oid), m_offset, p_offset); + } + return verify_midx_error; } diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index a968b9a959..828c240389 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -176,6 +176,7 @@ test_expect_success 'verify bad signature' ' ' HASH_LEN=20 +NUM_OBJECTS=74 MIDX_BYTE_VERSION=4 MIDX_BYTE_OID_VERSION=5 MIDX_BYTE_CHUNK_COUNT=6 @@ -192,6 +193,10 @@ MIDX_OID_FANOUT_WIDTH=4 MIDX_BYTE_OID_FANOUT_ORDER=$((MIDX_OFFSET_OID_FANOUT + 250 * $MIDX_OID_FANOUT_WIDTH + 1)) MIDX_OFFSET_OID_LOOKUP=$(($MIDX_OFFSET_OID_FANOUT + 256 * $MIDX_OID_FANOUT_WIDTH)) MIDX_BYTE_OID_LOOKUP=$(($MIDX_OFFSET_OID_LOOKUP + 16 * $HASH_LEN)) +MIDX_OFFSET_OBJECT_OFFSETS=$(($MIDX_OFFSET_OID_LOOKUP + $NUM_OBJECTS * $HASH_LEN)) +MIDX_OFFSET_WIDTH=8 +MIDX_BYTE_PACK_INT_ID=$(($MIDX_OFFSET_OBJECT_OFFSETS + 16 * $MIDX_OFFSET_WIDTH + 2)) +MIDX_BYTE_OFFSET=$(($MIDX_OFFSET_OBJECT_OFFSETS + 16 * $MIDX_OFFSET_WIDTH + 6)) test_expect_success 'verify bad version' ' corrupt_midx_and_verify $MIDX_BYTE_VERSION "\00" $objdir \ @@ -243,6 +248,16 @@ test_expect_success 'verify oid lookup out of order' ' "oid lookup out of order" ' +test_expect_success 'verify incorrect pack-int-id' ' + corrupt_midx_and_verify $MIDX_BYTE_PACK_INT_ID "\07" $objdir \ + "bad pack-int-id" +' + +test_expect_success 'verify incorrect offset' ' + corrupt_midx_and_verify $MIDX_BYTE_OFFSET "\07" $objdir \ + "incorrect object offset" +' + test_expect_success 'repack removes multi-pack-index' ' test_path_is_file $objdir/pack/multi-pack-index && git repack -adf && @@ -310,4 +325,16 @@ test_expect_success 'verify multi-pack-index with 64-bit offsets' ' git multi-pack-index verify --object-dir=objects64 ' +NUM_OBJECTS=63 +MIDX_OFFSET_OID_FANOUT=$((MIDX_OFFSET_PACKNAMES + 54)) +MIDX_OFFSET_OID_LOOKUP=$((MIDX_OFFSET_OID_FANOUT + 256 * $MIDX_OID_FANOUT_WIDTH)) +MIDX_OFFSET_OBJECT_OFFSETS=$(($MIDX_OFFSET_OID_LOOKUP + $NUM_OBJECTS * $HASH_LEN)) +MIDX_OFFSET_LARGE_OFFSETS=$(($MIDX_OFFSET_OBJECT_OFFSETS + $NUM_OBJECTS * $MIDX_OFFSET_WIDTH)) +MIDX_BYTE_LARGE_OFFSET=$(($MIDX_OFFSET_LARGE_OFFSETS + 3)) + +test_expect_success 'verify incorrect 64-bit offset' ' + corrupt_midx_and_verify $MIDX_BYTE_LARGE_OFFSET "\07" objects64 \ + "incorrect object offset" +' + test_done From 144d70333e826ea2480d60eb2298013eeef0cabf Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 13 Sep 2018 11:02:26 -0700 Subject: [PATCH 10/11] multi-pack-index: report progress during 'verify' When verifying a multi-pack-index, the only action that takes significant time is checking the object offsets. For example, to verify a multi-pack-index containing 6.2 million objects in the Linux kernel repository takes 1.3 seconds on my machine. 99% of that time is spent looking up object offsets in each of the packfiles and comparing them to the multi-pack-index offset. Add a progress indicator for that section of the 'verify' verb. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- midx.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/midx.c b/midx.c index ef4a6969df..713d6f9dde 100644 --- a/midx.c +++ b/midx.c @@ -7,6 +7,7 @@ #include "object-store.h" #include "sha1-lookup.h" #include "midx.h" +#include "progress.h" #define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */ #define MIDX_VERSION 1 @@ -940,6 +941,7 @@ static void midx_report(const char *fmt, ...) int verify_midx_file(const char *object_dir) { uint32_t i; + struct progress *progress = NULL; struct multi_pack_index *m = load_multi_pack_index(object_dir, 1); verify_midx_error = 0; @@ -971,6 +973,7 @@ int verify_midx_file(const char *object_dir) i, oid_to_hex(&oid1), oid_to_hex(&oid2), i + 1); } + progress = start_progress(_("Verifying object offsets"), m->num_objects); for (i = 0; i < m->num_objects; i++) { struct object_id oid; struct pack_entry e; @@ -995,7 +998,10 @@ int verify_midx_file(const char *object_dir) if (m_offset != p_offset) midx_report(_("incorrect object offset for oid[%d] = %s: %"PRIx64" != %"PRIx64), i, oid_to_hex(&oid), m_offset, p_offset); + + display_progress(progress, i + 1); } + stop_progress(&progress); return verify_midx_error; } From 66ec0390e75406aa9f9295577052b9ec06d3a169 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 13 Sep 2018 11:02:27 -0700 Subject: [PATCH 11/11] fsck: verify multi-pack-index When core.multiPackIndex is true, we may have a multi-pack-index in our object directory. Add calls to 'git multi-pack-index verify' at the end of 'git fsck' if so. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- builtin/fsck.c | 18 ++++++++++++++++++ t/t5319-multi-pack-index.sh | 13 ++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/builtin/fsck.c b/builtin/fsck.c index 63c8578cc1..06eb421720 100644 --- a/builtin/fsck.c +++ b/builtin/fsck.c @@ -848,5 +848,23 @@ int cmd_fsck(int argc, const char **argv, const char *prefix) } } + if (!git_config_get_bool("core.multipackindex", &i) && i) { + struct child_process midx_verify = CHILD_PROCESS_INIT; + const char *midx_argv[] = { "multi-pack-index", "verify", NULL, NULL, NULL }; + + midx_verify.argv = midx_argv; + midx_verify.git_cmd = 1; + if (run_command(&midx_verify)) + errors_found |= ERROR_COMMIT_GRAPH; + + prepare_alt_odb(the_repository); + for (alt = the_repository->objects->alt_odb_list; alt; alt = alt->next) { + midx_argv[2] = "--object-dir"; + midx_argv[3] = alt->path; + if (run_command(&midx_verify)) + errors_found |= ERROR_COMMIT_GRAPH; + } + } + return errors_found; } diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index 828c240389..bd8e841b81 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -160,12 +160,17 @@ corrupt_midx_and_verify() { DATA="${2:-\0}" && OBJDIR=$3 && GREPSTR="$4" && + COMMAND="$5" && + if test -z "$COMMAND" + then + COMMAND="git multi-pack-index verify --object-dir=$OBJDIR" + fi && FILE=$OBJDIR/pack/multi-pack-index && chmod a+w $FILE && test_when_finished mv midx-backup $FILE && cp $FILE midx-backup && printf "$DATA" | dd of="$FILE" bs=1 seek="$POS" conv=notrunc && - test_must_fail git multi-pack-index verify --object-dir=$OBJDIR 2>test_err && + test_must_fail $COMMAND 2>test_err && grep -v "^+" test_err >err && test_i18ngrep "$GREPSTR" err } @@ -258,6 +263,12 @@ test_expect_success 'verify incorrect offset' ' "incorrect object offset" ' +test_expect_success 'git-fsck incorrect offset' ' + corrupt_midx_and_verify $MIDX_BYTE_OFFSET "\07" $objdir \ + "incorrect object offset" \ + "git -c core.multipackindex=true fsck" +' + test_expect_success 'repack removes multi-pack-index' ' test_path_is_file $objdir/pack/multi-pack-index && git repack -adf &&