Merge branch 'ac/bitmap-lookup-table'
The pack bitmap file gained a bitmap-lookup table to speed up locating the necessary bitmap for a given commit. * ac/bitmap-lookup-table: pack-bitmap-write: drop unused pack_idx_entry parameters bitmap-lookup-table: add performance tests for lookup table pack-bitmap: prepare to read lookup table extension pack-bitmap-write: learn pack.writeBitmapLookupTable and add tests pack-bitmap-write.c: write lookup table extension bitmap: move `get commit positions` code to `bitmap_writer_finish` Documentation/technical: describe bitmap lookup table extensionmaint
Junio C Hamano
commit
3fe0121479
|
|
@ -164,6 +164,13 @@ When writing a multi-pack reachability bitmap, no new namehashes are
|
||||||
computed; instead, any namehashes stored in an existing bitmap are
|
computed; instead, any namehashes stored in an existing bitmap are
|
||||||
permuted into their appropriate location when writing a new bitmap.
|
permuted into their appropriate location when writing a new bitmap.
|
||||||
|
|
||||||
|
pack.writeBitmapLookupTable::
|
||||||
|
When true, Git will include a "lookup table" section in the
|
||||||
|
bitmap index (if one is written). This table is used to defer
|
||||||
|
loading individual bitmaps as late as possible. This can be
|
||||||
|
beneficial in repositories that have relatively large bitmap
|
||||||
|
indexes. Defaults to false.
|
||||||
|
|
||||||
pack.writeReverseIndex::
|
pack.writeReverseIndex::
|
||||||
When true, git will write a corresponding .rev file (see:
|
When true, git will write a corresponding .rev file (see:
|
||||||
linkgit:gitformat-pack[5])
|
linkgit:gitformat-pack[5])
|
||||||
|
|
|
||||||
|
|
@ -72,6 +72,17 @@ MIDXs, both the bit-cache and rev-cache extensions are required.
|
||||||
pack/MIDX. The format and meaning of the name-hash is
|
pack/MIDX. The format and meaning of the name-hash is
|
||||||
described below.
|
described below.
|
||||||
|
|
||||||
|
** {empty}
|
||||||
|
BITMAP_OPT_LOOKUP_TABLE (0x10): :::
|
||||||
|
If present, the end of the bitmap file contains a table
|
||||||
|
containing a list of `N` <commit_pos, offset, xor_row>
|
||||||
|
triplets. The format and meaning of the table is described
|
||||||
|
below.
|
||||||
|
+
|
||||||
|
NOTE: Unlike the xor_offset used to compress an individual bitmap,
|
||||||
|
`xor_row` stores an *absolute* index into the lookup table, not a location
|
||||||
|
relative to the current entry.
|
||||||
|
|
||||||
4-byte entry count (network byte order): ::
|
4-byte entry count (network byte order): ::
|
||||||
The total count of entries (bitmapped commits) in this bitmap index.
|
The total count of entries (bitmapped commits) in this bitmap index.
|
||||||
|
|
||||||
|
|
@ -216,3 +227,31 @@ Note that this hashing scheme is tied to the BITMAP_OPT_HASH_CACHE flag.
|
||||||
If implementations want to choose a different hashing scheme, they are
|
If implementations want to choose a different hashing scheme, they are
|
||||||
free to do so, but MUST allocate a new header flag (because comparing
|
free to do so, but MUST allocate a new header flag (because comparing
|
||||||
hashes made under two different schemes would be pointless).
|
hashes made under two different schemes would be pointless).
|
||||||
|
|
||||||
|
Commit lookup table
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
If the BITMAP_OPT_LOOKUP_TABLE flag is set, the last `N * (4 + 8 + 4)`
|
||||||
|
bytes (preceding the name-hash cache and trailing hash) of the `.bitmap`
|
||||||
|
file contains a lookup table specifying the information needed to get
|
||||||
|
the desired bitmap from the entries without parsing previous unnecessary
|
||||||
|
bitmaps.
|
||||||
|
|
||||||
|
For a `.bitmap` containing `nr_entries` reachability bitmaps, the table
|
||||||
|
contains a list of `nr_entries` <commit_pos, offset, xor_row> triplets
|
||||||
|
(sorted in the ascending order of `commit_pos`). The content of i'th
|
||||||
|
triplet is -
|
||||||
|
|
||||||
|
* {empty}
|
||||||
|
commit_pos (4 byte integer, network byte order): ::
|
||||||
|
It stores the object position of a commit (in the midx or pack
|
||||||
|
index).
|
||||||
|
|
||||||
|
* {empty}
|
||||||
|
offset (8 byte integer, network byte order): ::
|
||||||
|
The offset from which that commit's bitmap can be read.
|
||||||
|
|
||||||
|
* {empty}
|
||||||
|
xor_row (4 byte integer, network byte order): ::
|
||||||
|
The position of the triplet whose bitmap is used to compress
|
||||||
|
this one, or `0xffffffff` if no such bitmap exists.
|
||||||
|
|
|
||||||
|
|
@ -87,6 +87,13 @@ static int git_multi_pack_index_write_config(const char *var, const char *value,
|
||||||
opts.flags &= ~MIDX_WRITE_BITMAP_HASH_CACHE;
|
opts.flags &= ~MIDX_WRITE_BITMAP_HASH_CACHE;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!strcmp(var, "pack.writebitmaplookuptable")) {
|
||||||
|
if (git_config_bool(var, value))
|
||||||
|
opts.flags |= MIDX_WRITE_BITMAP_LOOKUP_TABLE;
|
||||||
|
else
|
||||||
|
opts.flags &= ~MIDX_WRITE_BITMAP_LOOKUP_TABLE;
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* We should never make a fall-back call to 'git_default_config', since
|
* We should never make a fall-back call to 'git_default_config', since
|
||||||
* this was already called in 'cmd_multi_pack_index()'.
|
* this was already called in 'cmd_multi_pack_index()'.
|
||||||
|
|
|
||||||
|
|
@ -3148,6 +3148,14 @@ static int git_pack_config(const char *k, const char *v, void *cb)
|
||||||
else
|
else
|
||||||
write_bitmap_options &= ~BITMAP_OPT_HASH_CACHE;
|
write_bitmap_options &= ~BITMAP_OPT_HASH_CACHE;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!strcmp(k, "pack.writebitmaplookuptable")) {
|
||||||
|
if (git_config_bool(k, v))
|
||||||
|
write_bitmap_options |= BITMAP_OPT_LOOKUP_TABLE;
|
||||||
|
else
|
||||||
|
write_bitmap_options &= ~BITMAP_OPT_LOOKUP_TABLE;
|
||||||
|
}
|
||||||
|
|
||||||
if (!strcmp(k, "pack.usebitmaps")) {
|
if (!strcmp(k, "pack.usebitmaps")) {
|
||||||
use_bitmap_index_default = git_config_bool(k, v);
|
use_bitmap_index_default = git_config_bool(k, v);
|
||||||
return 0;
|
return 0;
|
||||||
|
|
|
||||||
3
midx.c
3
midx.c
|
|
@ -1119,6 +1119,9 @@ static int write_midx_bitmap(const char *midx_name,
|
||||||
if (flags & MIDX_WRITE_BITMAP_HASH_CACHE)
|
if (flags & MIDX_WRITE_BITMAP_HASH_CACHE)
|
||||||
options |= BITMAP_OPT_HASH_CACHE;
|
options |= BITMAP_OPT_HASH_CACHE;
|
||||||
|
|
||||||
|
if (flags & MIDX_WRITE_BITMAP_LOOKUP_TABLE)
|
||||||
|
options |= BITMAP_OPT_LOOKUP_TABLE;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Build the MIDX-order index based on pdata.objects (which is already
|
* Build the MIDX-order index based on pdata.objects (which is already
|
||||||
* in MIDX order; c.f., 'midx_pack_order_cmp()' for the definition of
|
* in MIDX order; c.f., 'midx_pack_order_cmp()' for the definition of
|
||||||
|
|
|
||||||
1
midx.h
1
midx.h
|
|
@ -47,6 +47,7 @@ struct multi_pack_index {
|
||||||
#define MIDX_WRITE_REV_INDEX (1 << 1)
|
#define MIDX_WRITE_REV_INDEX (1 << 1)
|
||||||
#define MIDX_WRITE_BITMAP (1 << 2)
|
#define MIDX_WRITE_BITMAP (1 << 2)
|
||||||
#define MIDX_WRITE_BITMAP_HASH_CACHE (1 << 3)
|
#define MIDX_WRITE_BITMAP_HASH_CACHE (1 << 3)
|
||||||
|
#define MIDX_WRITE_BITMAP_LOOKUP_TABLE (1 << 4)
|
||||||
|
|
||||||
const unsigned char *get_midx_checksum(struct multi_pack_index *m);
|
const unsigned char *get_midx_checksum(struct multi_pack_index *m);
|
||||||
void get_midx_filename(struct strbuf *out, const char *object_dir);
|
void get_midx_filename(struct strbuf *out, const char *object_dir);
|
||||||
|
|
|
||||||
|
|
@ -649,21 +649,18 @@ static const struct object_id *oid_access(size_t pos, const void *table)
|
||||||
}
|
}
|
||||||
|
|
||||||
static void write_selected_commits_v1(struct hashfile *f,
|
static void write_selected_commits_v1(struct hashfile *f,
|
||||||
struct pack_idx_entry **index,
|
uint32_t *commit_positions,
|
||||||
uint32_t index_nr)
|
off_t *offsets)
|
||||||
{
|
{
|
||||||
int i;
|
int i;
|
||||||
|
|
||||||
for (i = 0; i < writer.selected_nr; ++i) {
|
for (i = 0; i < writer.selected_nr; ++i) {
|
||||||
struct bitmapped_commit *stored = &writer.selected[i];
|
struct bitmapped_commit *stored = &writer.selected[i];
|
||||||
|
|
||||||
int commit_pos =
|
if (offsets)
|
||||||
oid_pos(&stored->commit->object.oid, index, index_nr, oid_access);
|
offsets[i] = hashfile_total(f);
|
||||||
|
|
||||||
if (commit_pos < 0)
|
hashwrite_be32(f, commit_positions[i]);
|
||||||
BUG("trying to write commit not in index");
|
|
||||||
|
|
||||||
hashwrite_be32(f, commit_pos);
|
|
||||||
hashwrite_u8(f, stored->xor_offset);
|
hashwrite_u8(f, stored->xor_offset);
|
||||||
hashwrite_u8(f, stored->flags);
|
hashwrite_u8(f, stored->flags);
|
||||||
|
|
||||||
|
|
@ -671,6 +668,79 @@ static void write_selected_commits_v1(struct hashfile *f,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int table_cmp(const void *_va, const void *_vb, void *_data)
|
||||||
|
{
|
||||||
|
uint32_t *commit_positions = _data;
|
||||||
|
uint32_t a = commit_positions[*(uint32_t *)_va];
|
||||||
|
uint32_t b = commit_positions[*(uint32_t *)_vb];
|
||||||
|
|
||||||
|
if (a > b)
|
||||||
|
return 1;
|
||||||
|
else if (a < b)
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void write_lookup_table(struct hashfile *f,
|
||||||
|
uint32_t *commit_positions,
|
||||||
|
off_t *offsets)
|
||||||
|
{
|
||||||
|
uint32_t i;
|
||||||
|
uint32_t *table, *table_inv;
|
||||||
|
|
||||||
|
ALLOC_ARRAY(table, writer.selected_nr);
|
||||||
|
ALLOC_ARRAY(table_inv, writer.selected_nr);
|
||||||
|
|
||||||
|
for (i = 0; i < writer.selected_nr; i++)
|
||||||
|
table[i] = i;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* At the end of this sort table[j] = i means that the i'th
|
||||||
|
* bitmap corresponds to j'th bitmapped commit (among the selected
|
||||||
|
* commits) in lex order of OIDs.
|
||||||
|
*/
|
||||||
|
QSORT_S(table, writer.selected_nr, table_cmp, commit_positions);
|
||||||
|
|
||||||
|
/* table_inv helps us discover that relationship (i'th bitmap
|
||||||
|
* to j'th commit by j = table_inv[i])
|
||||||
|
*/
|
||||||
|
for (i = 0; i < writer.selected_nr; i++)
|
||||||
|
table_inv[table[i]] = i;
|
||||||
|
|
||||||
|
trace2_region_enter("pack-bitmap-write", "writing_lookup_table", the_repository);
|
||||||
|
for (i = 0; i < writer.selected_nr; i++) {
|
||||||
|
struct bitmapped_commit *selected = &writer.selected[table[i]];
|
||||||
|
uint32_t xor_offset = selected->xor_offset;
|
||||||
|
uint32_t xor_row;
|
||||||
|
|
||||||
|
if (xor_offset) {
|
||||||
|
/*
|
||||||
|
* xor_index stores the index (in the bitmap entries)
|
||||||
|
* of the corresponding xor bitmap. But we need to convert
|
||||||
|
* this index into lookup table's index. So, table_inv[xor_index]
|
||||||
|
* gives us the index position w.r.t. the lookup table.
|
||||||
|
*
|
||||||
|
* If "k = table[i] - xor_offset" then the xor base is the k'th
|
||||||
|
* bitmap. `table_inv[k]` gives us the position of that bitmap
|
||||||
|
* in the lookup table.
|
||||||
|
*/
|
||||||
|
uint32_t xor_index = table[i] - xor_offset;
|
||||||
|
xor_row = table_inv[xor_index];
|
||||||
|
} else {
|
||||||
|
xor_row = 0xffffffff;
|
||||||
|
}
|
||||||
|
|
||||||
|
hashwrite_be32(f, commit_positions[table[i]]);
|
||||||
|
hashwrite_be64(f, (uint64_t)offsets[table[i]]);
|
||||||
|
hashwrite_be32(f, xor_row);
|
||||||
|
}
|
||||||
|
trace2_region_leave("pack-bitmap-write", "writing_lookup_table", the_repository);
|
||||||
|
|
||||||
|
free(table);
|
||||||
|
free(table_inv);
|
||||||
|
}
|
||||||
|
|
||||||
static void write_hash_cache(struct hashfile *f,
|
static void write_hash_cache(struct hashfile *f,
|
||||||
struct pack_idx_entry **index,
|
struct pack_idx_entry **index,
|
||||||
uint32_t index_nr)
|
uint32_t index_nr)
|
||||||
|
|
@ -697,6 +767,9 @@ void bitmap_writer_finish(struct pack_idx_entry **index,
|
||||||
static uint16_t flags = BITMAP_OPT_FULL_DAG;
|
static uint16_t flags = BITMAP_OPT_FULL_DAG;
|
||||||
struct strbuf tmp_file = STRBUF_INIT;
|
struct strbuf tmp_file = STRBUF_INIT;
|
||||||
struct hashfile *f;
|
struct hashfile *f;
|
||||||
|
uint32_t *commit_positions = NULL;
|
||||||
|
off_t *offsets = NULL;
|
||||||
|
uint32_t i;
|
||||||
|
|
||||||
struct bitmap_disk_header header;
|
struct bitmap_disk_header header;
|
||||||
|
|
||||||
|
|
@ -715,7 +788,26 @@ void bitmap_writer_finish(struct pack_idx_entry **index,
|
||||||
dump_bitmap(f, writer.trees);
|
dump_bitmap(f, writer.trees);
|
||||||
dump_bitmap(f, writer.blobs);
|
dump_bitmap(f, writer.blobs);
|
||||||
dump_bitmap(f, writer.tags);
|
dump_bitmap(f, writer.tags);
|
||||||
write_selected_commits_v1(f, index, index_nr);
|
|
||||||
|
if (options & BITMAP_OPT_LOOKUP_TABLE)
|
||||||
|
CALLOC_ARRAY(offsets, index_nr);
|
||||||
|
|
||||||
|
ALLOC_ARRAY(commit_positions, writer.selected_nr);
|
||||||
|
|
||||||
|
for (i = 0; i < writer.selected_nr; i++) {
|
||||||
|
struct bitmapped_commit *stored = &writer.selected[i];
|
||||||
|
int commit_pos = oid_pos(&stored->commit->object.oid, index, index_nr, oid_access);
|
||||||
|
|
||||||
|
if (commit_pos < 0)
|
||||||
|
BUG(_("trying to write commit not in index"));
|
||||||
|
|
||||||
|
commit_positions[i] = commit_pos;
|
||||||
|
}
|
||||||
|
|
||||||
|
write_selected_commits_v1(f, commit_positions, offsets);
|
||||||
|
|
||||||
|
if (options & BITMAP_OPT_LOOKUP_TABLE)
|
||||||
|
write_lookup_table(f, commit_positions, offsets);
|
||||||
|
|
||||||
if (options & BITMAP_OPT_HASH_CACHE)
|
if (options & BITMAP_OPT_HASH_CACHE)
|
||||||
write_hash_cache(f, index, index_nr);
|
write_hash_cache(f, index, index_nr);
|
||||||
|
|
@ -730,4 +822,6 @@ void bitmap_writer_finish(struct pack_idx_entry **index,
|
||||||
die_errno("unable to rename temporary bitmap file to '%s'", filename);
|
die_errno("unable to rename temporary bitmap file to '%s'", filename);
|
||||||
|
|
||||||
strbuf_release(&tmp_file);
|
strbuf_release(&tmp_file);
|
||||||
|
free(commit_positions);
|
||||||
|
free(offsets);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
290
pack-bitmap.c
290
pack-bitmap.c
|
|
@ -83,6 +83,12 @@ struct bitmap_index {
|
||||||
/* The checksum of the packfile or MIDX; points into map. */
|
/* The checksum of the packfile or MIDX; points into map. */
|
||||||
const unsigned char *checksum;
|
const unsigned char *checksum;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* If not NULL, this point into the commit table extension
|
||||||
|
* (within the memory mapped region `map`).
|
||||||
|
*/
|
||||||
|
unsigned char *table_lookup;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Extended index.
|
* Extended index.
|
||||||
*
|
*
|
||||||
|
|
@ -186,6 +192,16 @@ static int load_bitmap_header(struct bitmap_index *index)
|
||||||
index->hashes = (void *)(index_end - cache_size);
|
index->hashes = (void *)(index_end - cache_size);
|
||||||
index_end -= cache_size;
|
index_end -= cache_size;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (flags & BITMAP_OPT_LOOKUP_TABLE) {
|
||||||
|
size_t table_size = st_mult(ntohl(header->entry_count),
|
||||||
|
BITMAP_LOOKUP_TABLE_TRIPLET_WIDTH);
|
||||||
|
if (table_size > index_end - index->map - header_size)
|
||||||
|
return error(_("corrupted bitmap index file (too short to fit lookup table)"));
|
||||||
|
if (git_env_bool("GIT_TEST_READ_COMMIT_TABLE", 1))
|
||||||
|
index->table_lookup = (void *)(index_end - table_size);
|
||||||
|
index_end -= table_size;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
index->entry_count = ntohl(header->entry_count);
|
index->entry_count = ntohl(header->entry_count);
|
||||||
|
|
@ -212,9 +228,11 @@ static struct stored_bitmap *store_bitmap(struct bitmap_index *index,
|
||||||
|
|
||||||
hash_pos = kh_put_oid_map(index->bitmaps, stored->oid, &ret);
|
hash_pos = kh_put_oid_map(index->bitmaps, stored->oid, &ret);
|
||||||
|
|
||||||
/* a 0 return code means the insertion succeeded with no changes,
|
/*
|
||||||
* because the SHA1 already existed on the map. this is bad, there
|
* A 0 return code means the insertion succeeded with no changes,
|
||||||
* shouldn't be duplicated commits in the index */
|
* because the SHA1 already existed on the map. This is bad, there
|
||||||
|
* shouldn't be duplicated commits in the index.
|
||||||
|
*/
|
||||||
if (ret == 0) {
|
if (ret == 0) {
|
||||||
error(_("duplicate entry in bitmap index: '%s'"), oid_to_hex(oid));
|
error(_("duplicate entry in bitmap index: '%s'"), oid_to_hex(oid));
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
@ -482,7 +500,7 @@ static int load_bitmap(struct bitmap_index *bitmap_git)
|
||||||
!(bitmap_git->tags = read_bitmap_1(bitmap_git)))
|
!(bitmap_git->tags = read_bitmap_1(bitmap_git)))
|
||||||
goto failed;
|
goto failed;
|
||||||
|
|
||||||
if (load_bitmap_entries_v1(bitmap_git) < 0)
|
if (!bitmap_git->table_lookup && load_bitmap_entries_v1(bitmap_git) < 0)
|
||||||
goto failed;
|
goto failed;
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
|
|
@ -570,13 +588,256 @@ struct include_data {
|
||||||
struct bitmap *seen;
|
struct bitmap *seen;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
struct bitmap_lookup_table_triplet {
|
||||||
|
uint32_t commit_pos;
|
||||||
|
uint64_t offset;
|
||||||
|
uint32_t xor_row;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct bitmap_lookup_table_xor_item {
|
||||||
|
struct object_id oid;
|
||||||
|
uint64_t offset;
|
||||||
|
};
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Given a `triplet` struct pointer and pointer `p`, this
|
||||||
|
* function reads the triplet beginning at `p` into the struct.
|
||||||
|
* Note that this function assumes that there is enough memory
|
||||||
|
* left for filling the `triplet` struct from `p`.
|
||||||
|
*/
|
||||||
|
static int bitmap_lookup_table_get_triplet_by_pointer(struct bitmap_lookup_table_triplet *triplet,
|
||||||
|
const unsigned char *p)
|
||||||
|
{
|
||||||
|
if (!triplet)
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
triplet->commit_pos = get_be32(p);
|
||||||
|
p += sizeof(uint32_t);
|
||||||
|
triplet->offset = get_be64(p);
|
||||||
|
p += sizeof(uint64_t);
|
||||||
|
triplet->xor_row = get_be32(p);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This function gets the raw triplet from `row`'th row in the
|
||||||
|
* lookup table and fills that data to the `triplet`.
|
||||||
|
*/
|
||||||
|
static int bitmap_lookup_table_get_triplet(struct bitmap_index *bitmap_git,
|
||||||
|
uint32_t pos,
|
||||||
|
struct bitmap_lookup_table_triplet *triplet)
|
||||||
|
{
|
||||||
|
unsigned char *p = NULL;
|
||||||
|
if (pos >= bitmap_git->entry_count)
|
||||||
|
return error(_("corrupt bitmap lookup table: triplet position out of index"));
|
||||||
|
|
||||||
|
p = bitmap_git->table_lookup + st_mult(pos, BITMAP_LOOKUP_TABLE_TRIPLET_WIDTH);
|
||||||
|
|
||||||
|
return bitmap_lookup_table_get_triplet_by_pointer(triplet, p);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Searches for a matching triplet. `commit_pos` is a pointer
|
||||||
|
* to the wanted commit position value. `table_entry` points to
|
||||||
|
* a triplet in lookup table. The first 4 bytes of each
|
||||||
|
* triplet (pointed by `table_entry`) are compared with `*commit_pos`.
|
||||||
|
*/
|
||||||
|
static int triplet_cmp(const void *commit_pos, const void *table_entry)
|
||||||
|
{
|
||||||
|
|
||||||
|
uint32_t a = *(uint32_t *)commit_pos;
|
||||||
|
uint32_t b = get_be32(table_entry);
|
||||||
|
if (a > b)
|
||||||
|
return 1;
|
||||||
|
else if (a < b)
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static uint32_t bitmap_bsearch_pos(struct bitmap_index *bitmap_git,
|
||||||
|
struct object_id *oid,
|
||||||
|
uint32_t *result)
|
||||||
|
{
|
||||||
|
int found;
|
||||||
|
|
||||||
|
if (bitmap_is_midx(bitmap_git))
|
||||||
|
found = bsearch_midx(oid, bitmap_git->midx, result);
|
||||||
|
else
|
||||||
|
found = bsearch_pack(oid, bitmap_git->pack, result);
|
||||||
|
|
||||||
|
return found;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* `bsearch_triplet_by_pos` function searches for the raw triplet
|
||||||
|
* having commit position same as `commit_pos` and fills `triplet`
|
||||||
|
* object from the raw triplet. Returns 1 on success and 0 on
|
||||||
|
* failure.
|
||||||
|
*/
|
||||||
|
static int bitmap_bsearch_triplet_by_pos(uint32_t commit_pos,
|
||||||
|
struct bitmap_index *bitmap_git,
|
||||||
|
struct bitmap_lookup_table_triplet *triplet)
|
||||||
|
{
|
||||||
|
unsigned char *p = bsearch(&commit_pos, bitmap_git->table_lookup, bitmap_git->entry_count,
|
||||||
|
BITMAP_LOOKUP_TABLE_TRIPLET_WIDTH, triplet_cmp);
|
||||||
|
|
||||||
|
if (!p)
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
return bitmap_lookup_table_get_triplet_by_pointer(triplet, p);
|
||||||
|
}
|
||||||
|
|
||||||
|
static struct stored_bitmap *lazy_bitmap_for_commit(struct bitmap_index *bitmap_git,
|
||||||
|
struct commit *commit)
|
||||||
|
{
|
||||||
|
uint32_t commit_pos, xor_row;
|
||||||
|
uint64_t offset;
|
||||||
|
int flags;
|
||||||
|
struct bitmap_lookup_table_triplet triplet;
|
||||||
|
struct object_id *oid = &commit->object.oid;
|
||||||
|
struct ewah_bitmap *bitmap;
|
||||||
|
struct stored_bitmap *xor_bitmap = NULL;
|
||||||
|
const int bitmap_header_size = 6;
|
||||||
|
static struct bitmap_lookup_table_xor_item *xor_items = NULL;
|
||||||
|
static size_t xor_items_nr = 0, xor_items_alloc = 0;
|
||||||
|
static int is_corrupt = 0;
|
||||||
|
int xor_flags;
|
||||||
|
khiter_t hash_pos;
|
||||||
|
struct bitmap_lookup_table_xor_item *xor_item;
|
||||||
|
|
||||||
|
if (is_corrupt)
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
if (!bitmap_bsearch_pos(bitmap_git, oid, &commit_pos))
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
if (bitmap_bsearch_triplet_by_pos(commit_pos, bitmap_git, &triplet) < 0)
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
xor_items_nr = 0;
|
||||||
|
offset = triplet.offset;
|
||||||
|
xor_row = triplet.xor_row;
|
||||||
|
|
||||||
|
while (xor_row != 0xffffffff) {
|
||||||
|
ALLOC_GROW(xor_items, xor_items_nr + 1, xor_items_alloc);
|
||||||
|
|
||||||
|
if (xor_items_nr + 1 >= bitmap_git->entry_count) {
|
||||||
|
error(_("corrupt bitmap lookup table: xor chain exceed entry count"));
|
||||||
|
goto corrupt;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (bitmap_lookup_table_get_triplet(bitmap_git, xor_row, &triplet) < 0)
|
||||||
|
goto corrupt;
|
||||||
|
|
||||||
|
xor_item = &xor_items[xor_items_nr];
|
||||||
|
xor_item->offset = triplet.offset;
|
||||||
|
|
||||||
|
if (nth_bitmap_object_oid(bitmap_git, &xor_item->oid, triplet.commit_pos) < 0) {
|
||||||
|
error(_("corrupt bitmap lookup table: commit index %u out of range"),
|
||||||
|
triplet.commit_pos);
|
||||||
|
goto corrupt;
|
||||||
|
}
|
||||||
|
|
||||||
|
hash_pos = kh_get_oid_map(bitmap_git->bitmaps, xor_item->oid);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* If desired bitmap is already stored, we don't need
|
||||||
|
* to iterate further. Because we know that bitmaps
|
||||||
|
* that are needed to be parsed to parse this bitmap
|
||||||
|
* has already been stored. So, assign this stored bitmap
|
||||||
|
* to the xor_bitmap.
|
||||||
|
*/
|
||||||
|
if (hash_pos < kh_end(bitmap_git->bitmaps) &&
|
||||||
|
(xor_bitmap = kh_value(bitmap_git->bitmaps, hash_pos)))
|
||||||
|
break;
|
||||||
|
xor_items_nr++;
|
||||||
|
xor_row = triplet.xor_row;
|
||||||
|
}
|
||||||
|
|
||||||
|
while (xor_items_nr) {
|
||||||
|
xor_item = &xor_items[xor_items_nr - 1];
|
||||||
|
bitmap_git->map_pos = xor_item->offset;
|
||||||
|
if (bitmap_git->map_size - bitmap_git->map_pos < bitmap_header_size) {
|
||||||
|
error(_("corrupt ewah bitmap: truncated header for bitmap of commit \"%s\""),
|
||||||
|
oid_to_hex(&xor_item->oid));
|
||||||
|
goto corrupt;
|
||||||
|
}
|
||||||
|
|
||||||
|
bitmap_git->map_pos += sizeof(uint32_t) + sizeof(uint8_t);
|
||||||
|
xor_flags = read_u8(bitmap_git->map, &bitmap_git->map_pos);
|
||||||
|
bitmap = read_bitmap_1(bitmap_git);
|
||||||
|
|
||||||
|
if (!bitmap)
|
||||||
|
goto corrupt;
|
||||||
|
|
||||||
|
xor_bitmap = store_bitmap(bitmap_git, bitmap, &xor_item->oid, xor_bitmap, xor_flags);
|
||||||
|
xor_items_nr--;
|
||||||
|
}
|
||||||
|
|
||||||
|
bitmap_git->map_pos = offset;
|
||||||
|
if (bitmap_git->map_size - bitmap_git->map_pos < bitmap_header_size) {
|
||||||
|
error(_("corrupt ewah bitmap: truncated header for bitmap of commit \"%s\""),
|
||||||
|
oid_to_hex(oid));
|
||||||
|
goto corrupt;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Don't bother reading the commit's index position or its xor
|
||||||
|
* offset:
|
||||||
|
*
|
||||||
|
* - The commit's index position is irrelevant to us, since
|
||||||
|
* load_bitmap_entries_v1 only uses it to learn the object
|
||||||
|
* id which is used to compute the hashmap's key. We already
|
||||||
|
* have an object id, so no need to look it up again.
|
||||||
|
*
|
||||||
|
* - The xor_offset is unusable for us, since it specifies how
|
||||||
|
* many entries previous to ours we should look at. This
|
||||||
|
* makes sense when reading the bitmaps sequentially (as in
|
||||||
|
* load_bitmap_entries_v1()), since we can keep track of
|
||||||
|
* each bitmap as we read them.
|
||||||
|
*
|
||||||
|
* But it can't work for us, since the bitmap's don't have a
|
||||||
|
* fixed size. So we learn the position of the xor'd bitmap
|
||||||
|
* from the commit table (and resolve it to a bitmap in the
|
||||||
|
* above if-statement).
|
||||||
|
*
|
||||||
|
* Instead, we can skip ahead and immediately read the flags and
|
||||||
|
* ewah bitmap.
|
||||||
|
*/
|
||||||
|
bitmap_git->map_pos += sizeof(uint32_t) + sizeof(uint8_t);
|
||||||
|
flags = read_u8(bitmap_git->map, &bitmap_git->map_pos);
|
||||||
|
bitmap = read_bitmap_1(bitmap_git);
|
||||||
|
|
||||||
|
if (!bitmap)
|
||||||
|
goto corrupt;
|
||||||
|
|
||||||
|
return store_bitmap(bitmap_git, bitmap, oid, xor_bitmap, flags);
|
||||||
|
|
||||||
|
corrupt:
|
||||||
|
free(xor_items);
|
||||||
|
is_corrupt = 1;
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
struct ewah_bitmap *bitmap_for_commit(struct bitmap_index *bitmap_git,
|
struct ewah_bitmap *bitmap_for_commit(struct bitmap_index *bitmap_git,
|
||||||
struct commit *commit)
|
struct commit *commit)
|
||||||
{
|
{
|
||||||
khiter_t hash_pos = kh_get_oid_map(bitmap_git->bitmaps,
|
khiter_t hash_pos = kh_get_oid_map(bitmap_git->bitmaps,
|
||||||
commit->object.oid);
|
commit->object.oid);
|
||||||
if (hash_pos >= kh_end(bitmap_git->bitmaps))
|
if (hash_pos >= kh_end(bitmap_git->bitmaps)) {
|
||||||
return NULL;
|
struct stored_bitmap *bitmap = NULL;
|
||||||
|
if (!bitmap_git->table_lookup)
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
trace2_region_enter("pack-bitmap", "reading_lookup_table", the_repository);
|
||||||
|
/* NEEDSWORK: cache misses aren't recorded */
|
||||||
|
bitmap = lazy_bitmap_for_commit(bitmap_git, commit);
|
||||||
|
trace2_region_leave("pack-bitmap", "reading_lookup_table", the_repository);
|
||||||
|
if (!bitmap)
|
||||||
|
return NULL;
|
||||||
|
return lookup_stored_bitmap(bitmap);
|
||||||
|
}
|
||||||
return lookup_stored_bitmap(kh_value(bitmap_git->bitmaps, hash_pos));
|
return lookup_stored_bitmap(kh_value(bitmap_git->bitmaps, hash_pos));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1712,8 +1973,10 @@ void test_bitmap_walk(struct rev_info *revs)
|
||||||
if (revs->pending.nr != 1)
|
if (revs->pending.nr != 1)
|
||||||
die(_("you must specify exactly one commit to test"));
|
die(_("you must specify exactly one commit to test"));
|
||||||
|
|
||||||
fprintf_ln(stderr, "Bitmap v%d test (%d entries loaded)",
|
fprintf_ln(stderr, "Bitmap v%d test (%d entries%s)",
|
||||||
bitmap_git->version, bitmap_git->entry_count);
|
bitmap_git->version,
|
||||||
|
bitmap_git->entry_count,
|
||||||
|
bitmap_git->table_lookup ? "" : " loaded");
|
||||||
|
|
||||||
root = revs->pending.objects[0].item;
|
root = revs->pending.objects[0].item;
|
||||||
bm = bitmap_for_commit(bitmap_git, (struct commit *)root);
|
bm = bitmap_for_commit(bitmap_git, (struct commit *)root);
|
||||||
|
|
@ -1766,13 +2029,22 @@ void test_bitmap_walk(struct rev_info *revs)
|
||||||
|
|
||||||
int test_bitmap_commits(struct repository *r)
|
int test_bitmap_commits(struct repository *r)
|
||||||
{
|
{
|
||||||
struct bitmap_index *bitmap_git = prepare_bitmap_git(r);
|
|
||||||
struct object_id oid;
|
struct object_id oid;
|
||||||
MAYBE_UNUSED void *value;
|
MAYBE_UNUSED void *value;
|
||||||
|
struct bitmap_index *bitmap_git = prepare_bitmap_git(r);
|
||||||
|
|
||||||
if (!bitmap_git)
|
if (!bitmap_git)
|
||||||
die(_("failed to load bitmap indexes"));
|
die(_("failed to load bitmap indexes"));
|
||||||
|
|
||||||
|
/*
|
||||||
|
* As this function is only used to print bitmap selected
|
||||||
|
* commits, we don't have to read the commit table.
|
||||||
|
*/
|
||||||
|
if (bitmap_git->table_lookup) {
|
||||||
|
if (load_bitmap_entries_v1(bitmap_git) < 0)
|
||||||
|
die(_("failed to load bitmap indexes"));
|
||||||
|
}
|
||||||
|
|
||||||
kh_foreach(bitmap_git->bitmaps, oid, value, {
|
kh_foreach(bitmap_git->bitmaps, oid, value, {
|
||||||
printf_ln("%s", oid_to_hex(&oid));
|
printf_ln("%s", oid_to_hex(&oid));
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -23,9 +23,19 @@ struct bitmap_disk_header {
|
||||||
|
|
||||||
#define NEEDS_BITMAP (1u<<22)
|
#define NEEDS_BITMAP (1u<<22)
|
||||||
|
|
||||||
|
/*
|
||||||
|
* The width in bytes of a single triplet in the lookup table
|
||||||
|
* extension:
|
||||||
|
* (commit_pos, offset, xor_row)
|
||||||
|
*
|
||||||
|
* whose fields ar 32-, 64-, 32- bits wide, respectively.
|
||||||
|
*/
|
||||||
|
#define BITMAP_LOOKUP_TABLE_TRIPLET_WIDTH (16)
|
||||||
|
|
||||||
enum pack_bitmap_opts {
|
enum pack_bitmap_opts {
|
||||||
BITMAP_OPT_FULL_DAG = 1,
|
BITMAP_OPT_FULL_DAG = 0x1,
|
||||||
BITMAP_OPT_HASH_CACHE = 4,
|
BITMAP_OPT_HASH_CACHE = 0x4,
|
||||||
|
BITMAP_OPT_LOOKUP_TABLE = 0x10,
|
||||||
};
|
};
|
||||||
|
|
||||||
enum pack_bitmap_flags {
|
enum pack_bitmap_flags {
|
||||||
|
|
|
||||||
|
|
@ -67,3 +67,34 @@ test_partial_bitmap () {
|
||||||
--filter=tree:0 >/dev/null
|
--filter=tree:0 >/dev/null
|
||||||
'
|
'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
test_pack_bitmap () {
|
||||||
|
test_perf "repack to disk" '
|
||||||
|
git repack -ad
|
||||||
|
'
|
||||||
|
|
||||||
|
test_full_bitmap
|
||||||
|
|
||||||
|
test_expect_success "create partial bitmap state" '
|
||||||
|
# pick a commit to represent the repo tip in the past
|
||||||
|
cutoff=$(git rev-list HEAD~100 -1) &&
|
||||||
|
orig_tip=$(git rev-parse HEAD) &&
|
||||||
|
|
||||||
|
# now kill off all of the refs and pretend we had
|
||||||
|
# just the one tip
|
||||||
|
rm -rf .git/logs .git/refs/* .git/packed-refs &&
|
||||||
|
git update-ref HEAD $cutoff &&
|
||||||
|
|
||||||
|
# and then repack, which will leave us with a nice
|
||||||
|
# big bitmap pack of the "old" history, and all of
|
||||||
|
# the new history will be loose, as if it had been pushed
|
||||||
|
# up incrementally and exploded via unpack-objects
|
||||||
|
git repack -Ad &&
|
||||||
|
|
||||||
|
# and now restore our original tip, as if the pushes
|
||||||
|
# had happened
|
||||||
|
git update-ref HEAD $orig_tip
|
||||||
|
'
|
||||||
|
|
||||||
|
test_partial_bitmap
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -4,51 +4,37 @@ test_description='Tests pack performance using bitmaps'
|
||||||
. ./perf-lib.sh
|
. ./perf-lib.sh
|
||||||
. "${TEST_DIRECTORY}/perf/lib-bitmap.sh"
|
. "${TEST_DIRECTORY}/perf/lib-bitmap.sh"
|
||||||
|
|
||||||
test_perf_large_repo
|
test_lookup_pack_bitmap () {
|
||||||
|
test_expect_success 'start the test from scratch' '
|
||||||
|
rm -rf * .git
|
||||||
|
'
|
||||||
|
|
||||||
# note that we do everything through config,
|
test_perf_large_repo
|
||||||
# since we want to be able to compare bitmap-aware
|
|
||||||
# git versus non-bitmap git
|
|
||||||
#
|
|
||||||
# We intentionally use the deprecated pack.writebitmaps
|
|
||||||
# config so that we can test against older versions of git.
|
|
||||||
test_expect_success 'setup bitmap config' '
|
|
||||||
git config pack.writebitmaps true
|
|
||||||
'
|
|
||||||
|
|
||||||
# we need to create the tag up front such that it is covered by the repack and
|
# note that we do everything through config,
|
||||||
# thus by generated bitmaps.
|
# since we want to be able to compare bitmap-aware
|
||||||
test_expect_success 'create tags' '
|
# git versus non-bitmap git
|
||||||
git tag --message="tag pointing to HEAD" perf-tag HEAD
|
#
|
||||||
'
|
# We intentionally use the deprecated pack.writebitmaps
|
||||||
|
# config so that we can test against older versions of git.
|
||||||
|
test_expect_success 'setup bitmap config' '
|
||||||
|
git config pack.writebitmaps true
|
||||||
|
'
|
||||||
|
|
||||||
test_perf 'repack to disk' '
|
# we need to create the tag up front such that it is covered by the repack and
|
||||||
git repack -ad
|
# thus by generated bitmaps.
|
||||||
'
|
test_expect_success 'create tags' '
|
||||||
|
git tag --message="tag pointing to HEAD" perf-tag HEAD
|
||||||
|
'
|
||||||
|
|
||||||
test_full_bitmap
|
test_perf "enable lookup table: $1" '
|
||||||
|
git config pack.writeBitmapLookupTable '"$1"'
|
||||||
|
'
|
||||||
|
|
||||||
test_expect_success 'create partial bitmap state' '
|
test_pack_bitmap
|
||||||
# pick a commit to represent the repo tip in the past
|
}
|
||||||
cutoff=$(git rev-list HEAD~100 -1) &&
|
|
||||||
orig_tip=$(git rev-parse HEAD) &&
|
|
||||||
|
|
||||||
# now kill off all of the refs and pretend we had
|
test_lookup_pack_bitmap false
|
||||||
# just the one tip
|
test_lookup_pack_bitmap true
|
||||||
rm -rf .git/logs .git/refs/* .git/packed-refs &&
|
|
||||||
git update-ref HEAD $cutoff &&
|
|
||||||
|
|
||||||
# and then repack, which will leave us with a nice
|
|
||||||
# big bitmap pack of the "old" history, and all of
|
|
||||||
# the new history will be loose, as if it had been pushed
|
|
||||||
# up incrementally and exploded via unpack-objects
|
|
||||||
git repack -Ad &&
|
|
||||||
|
|
||||||
# and now restore our original tip, as if the pushes
|
|
||||||
# had happened
|
|
||||||
git update-ref HEAD $orig_tip
|
|
||||||
'
|
|
||||||
|
|
||||||
test_partial_bitmap
|
|
||||||
|
|
||||||
test_done
|
test_done
|
||||||
|
|
|
||||||
|
|
@ -3,42 +3,52 @@
|
||||||
test_description='performance of fetches from bitmapped packs'
|
test_description='performance of fetches from bitmapped packs'
|
||||||
. ./perf-lib.sh
|
. ./perf-lib.sh
|
||||||
|
|
||||||
test_perf_default_repo
|
test_fetch_bitmaps () {
|
||||||
|
test_expect_success 'setup test directory' '
|
||||||
test_expect_success 'create bitmapped server repo' '
|
rm -fr * .git
|
||||||
git config pack.writebitmaps true &&
|
|
||||||
git repack -ad
|
|
||||||
'
|
|
||||||
|
|
||||||
# simulate a fetch from a repository that last fetched N days ago, for
|
|
||||||
# various values of N. We do so by following the first-parent chain,
|
|
||||||
# and assume the first entry in the chain that is N days older than the current
|
|
||||||
# HEAD is where the HEAD would have been then.
|
|
||||||
for days in 1 2 4 8 16 32 64 128; do
|
|
||||||
title=$(printf '%10s' "($days days)")
|
|
||||||
test_expect_success "setup revs from $days days ago" '
|
|
||||||
now=$(git log -1 --format=%ct HEAD) &&
|
|
||||||
then=$(($now - ($days * 86400))) &&
|
|
||||||
tip=$(git rev-list -1 --first-parent --until=$then HEAD) &&
|
|
||||||
{
|
|
||||||
echo HEAD &&
|
|
||||||
echo ^$tip
|
|
||||||
} >revs
|
|
||||||
'
|
'
|
||||||
|
|
||||||
test_perf "server $title" '
|
test_perf_default_repo
|
||||||
git pack-objects --stdout --revs \
|
|
||||||
--thin --delta-base-offset \
|
test_expect_success 'create bitmapped server repo' '
|
||||||
<revs >tmp.pack
|
git config pack.writebitmaps true &&
|
||||||
|
git config pack.writeBitmapLookupTable '"$1"' &&
|
||||||
|
git repack -ad
|
||||||
'
|
'
|
||||||
|
|
||||||
test_size "size $title" '
|
# simulate a fetch from a repository that last fetched N days ago, for
|
||||||
wc -c <tmp.pack
|
# various values of N. We do so by following the first-parent chain,
|
||||||
'
|
# and assume the first entry in the chain that is N days older than the current
|
||||||
|
# HEAD is where the HEAD would have been then.
|
||||||
|
for days in 1 2 4 8 16 32 64 128; do
|
||||||
|
title=$(printf '%10s' "($days days)")
|
||||||
|
test_expect_success "setup revs from $days days ago" '
|
||||||
|
now=$(git log -1 --format=%ct HEAD) &&
|
||||||
|
then=$(($now - ($days * 86400))) &&
|
||||||
|
tip=$(git rev-list -1 --first-parent --until=$then HEAD) &&
|
||||||
|
{
|
||||||
|
echo HEAD &&
|
||||||
|
echo ^$tip
|
||||||
|
} >revs
|
||||||
|
'
|
||||||
|
|
||||||
test_perf "client $title" '
|
test_perf "server $title (lookup=$1)" '
|
||||||
git index-pack --stdin --fix-thin <tmp.pack
|
git pack-objects --stdout --revs \
|
||||||
'
|
--thin --delta-base-offset \
|
||||||
done
|
<revs >tmp.pack
|
||||||
|
'
|
||||||
|
|
||||||
|
test_size "size $title" '
|
||||||
|
wc -c <tmp.pack
|
||||||
|
'
|
||||||
|
|
||||||
|
test_perf "client $title (lookup=$1)" '
|
||||||
|
git index-pack --stdin --fix-thin <tmp.pack
|
||||||
|
'
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
test_fetch_bitmaps true
|
||||||
|
test_fetch_bitmaps false
|
||||||
|
|
||||||
test_done
|
test_done
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,35 @@
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
test_description='Tests pack performance using bitmaps (rev index enabled)'
|
||||||
|
. ./perf-lib.sh
|
||||||
|
. "${TEST_DIRECTORY}/perf/lib-bitmap.sh"
|
||||||
|
|
||||||
|
test_lookup_pack_bitmap () {
|
||||||
|
test_expect_success 'start the test from scratch' '
|
||||||
|
rm -rf * .git
|
||||||
|
'
|
||||||
|
|
||||||
|
test_perf_large_repo
|
||||||
|
|
||||||
|
test_expect_success 'setup bitmap config' '
|
||||||
|
git config pack.writebitmaps true &&
|
||||||
|
git config pack.writeReverseIndex true
|
||||||
|
'
|
||||||
|
|
||||||
|
# we need to create the tag up front such that it is covered by the repack and
|
||||||
|
# thus by generated bitmaps.
|
||||||
|
test_expect_success 'create tags' '
|
||||||
|
git tag --message="tag pointing to HEAD" perf-tag HEAD
|
||||||
|
'
|
||||||
|
|
||||||
|
test_perf "enable lookup table: $1" '
|
||||||
|
git config pack.writeBitmapLookupTable '"$1"'
|
||||||
|
'
|
||||||
|
|
||||||
|
test_pack_bitmap
|
||||||
|
}
|
||||||
|
|
||||||
|
test_lookup_pack_bitmap false
|
||||||
|
test_lookup_pack_bitmap true
|
||||||
|
|
||||||
|
test_done
|
||||||
|
|
@ -4,49 +4,64 @@ test_description='Tests performance using midx bitmaps'
|
||||||
. ./perf-lib.sh
|
. ./perf-lib.sh
|
||||||
. "${TEST_DIRECTORY}/perf/lib-bitmap.sh"
|
. "${TEST_DIRECTORY}/perf/lib-bitmap.sh"
|
||||||
|
|
||||||
test_perf_large_repo
|
test_bitmap () {
|
||||||
|
local enabled="$1"
|
||||||
|
|
||||||
# we need to create the tag up front such that it is covered by the repack and
|
test_expect_success "remove existing repo (lookup=$enabled)" '
|
||||||
# thus by generated bitmaps.
|
rm -fr * .git
|
||||||
test_expect_success 'create tags' '
|
'
|
||||||
git tag --message="tag pointing to HEAD" perf-tag HEAD
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'start with bitmapped pack' '
|
test_perf_large_repo
|
||||||
git repack -adb
|
|
||||||
'
|
|
||||||
|
|
||||||
test_perf 'setup multi-pack index' '
|
# we need to create the tag up front such that it is covered by the repack and
|
||||||
git multi-pack-index write --bitmap
|
# thus by generated bitmaps.
|
||||||
'
|
test_expect_success 'create tags' '
|
||||||
|
git tag --message="tag pointing to HEAD" perf-tag HEAD
|
||||||
|
'
|
||||||
|
|
||||||
test_expect_success 'drop pack bitmap' '
|
test_expect_success "use lookup table: $enabled" '
|
||||||
rm -f .git/objects/pack/pack-*.bitmap
|
git config pack.writeBitmapLookupTable '"$enabled"'
|
||||||
'
|
'
|
||||||
|
|
||||||
test_full_bitmap
|
test_expect_success "start with bitmapped pack (lookup=$enabled)" '
|
||||||
|
git repack -adb
|
||||||
|
'
|
||||||
|
|
||||||
test_expect_success 'create partial bitmap state' '
|
test_perf "setup multi-pack index (lookup=$enabled)" '
|
||||||
# pick a commit to represent the repo tip in the past
|
git multi-pack-index write --bitmap
|
||||||
cutoff=$(git rev-list HEAD~100 -1) &&
|
'
|
||||||
orig_tip=$(git rev-parse HEAD) &&
|
|
||||||
|
|
||||||
# now pretend we have just one tip
|
test_expect_success "drop pack bitmap (lookup=$enabled)" '
|
||||||
rm -rf .git/logs .git/refs/* .git/packed-refs &&
|
rm -f .git/objects/pack/pack-*.bitmap
|
||||||
git update-ref HEAD $cutoff &&
|
'
|
||||||
|
|
||||||
# and then repack, which will leave us with a nice
|
test_full_bitmap
|
||||||
# big bitmap pack of the "old" history, and all of
|
|
||||||
# the new history will be loose, as if it had been pushed
|
|
||||||
# up incrementally and exploded via unpack-objects
|
|
||||||
git repack -Ad &&
|
|
||||||
git multi-pack-index write --bitmap &&
|
|
||||||
|
|
||||||
# and now restore our original tip, as if the pushes
|
test_expect_success "create partial bitmap state (lookup=$enabled)" '
|
||||||
# had happened
|
# pick a commit to represent the repo tip in the past
|
||||||
git update-ref HEAD $orig_tip
|
cutoff=$(git rev-list HEAD~100 -1) &&
|
||||||
'
|
orig_tip=$(git rev-parse HEAD) &&
|
||||||
|
|
||||||
test_partial_bitmap
|
# now pretend we have just one tip
|
||||||
|
rm -rf .git/logs .git/refs/* .git/packed-refs &&
|
||||||
|
git update-ref HEAD $cutoff &&
|
||||||
|
|
||||||
|
# and then repack, which will leave us with a nice
|
||||||
|
# big bitmap pack of the "old" history, and all of
|
||||||
|
# the new history will be loose, as if it had been pushed
|
||||||
|
# up incrementally and exploded via unpack-objects
|
||||||
|
git repack -Ad &&
|
||||||
|
git multi-pack-index write --bitmap &&
|
||||||
|
|
||||||
|
# and now restore our original tip, as if the pushes
|
||||||
|
# had happened
|
||||||
|
git update-ref HEAD $orig_tip
|
||||||
|
'
|
||||||
|
|
||||||
|
test_partial_bitmap
|
||||||
|
}
|
||||||
|
|
||||||
|
test_bitmap false
|
||||||
|
test_bitmap true
|
||||||
|
|
||||||
test_done
|
test_done
|
||||||
|
|
|
||||||
|
|
@ -26,22 +26,415 @@ has_any () {
|
||||||
grep -Ff "$1" "$2"
|
grep -Ff "$1" "$2"
|
||||||
}
|
}
|
||||||
|
|
||||||
setup_bitmap_history
|
test_bitmap_cases () {
|
||||||
|
writeLookupTable=false
|
||||||
|
for i in "$@"
|
||||||
|
do
|
||||||
|
case "$i" in
|
||||||
|
"pack.writeBitmapLookupTable") writeLookupTable=true;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
test_expect_success 'setup writing bitmaps during repack' '
|
test_expect_success 'setup test repository' '
|
||||||
git config repack.writeBitmaps true
|
rm -fr * .git &&
|
||||||
'
|
git init &&
|
||||||
|
git config pack.writeBitmapLookupTable '"$writeLookupTable"'
|
||||||
|
'
|
||||||
|
setup_bitmap_history
|
||||||
|
|
||||||
test_expect_success 'full repack creates bitmaps' '
|
test_expect_success 'setup writing bitmaps during repack' '
|
||||||
GIT_TRACE2_EVENT="$(pwd)/trace" \
|
git config repack.writeBitmaps true
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'full repack creates bitmaps' '
|
||||||
|
GIT_TRACE2_EVENT="$(pwd)/trace" \
|
||||||
|
git repack -ad &&
|
||||||
|
ls .git/objects/pack/ | grep bitmap >output &&
|
||||||
|
test_line_count = 1 output &&
|
||||||
|
grep "\"key\":\"num_selected_commits\",\"value\":\"106\"" trace &&
|
||||||
|
grep "\"key\":\"num_maximal_commits\",\"value\":\"107\"" trace
|
||||||
|
'
|
||||||
|
|
||||||
|
basic_bitmap_tests
|
||||||
|
|
||||||
|
test_expect_success 'pack-objects respects --local (non-local loose)' '
|
||||||
|
git init --bare alt.git &&
|
||||||
|
echo $(pwd)/alt.git/objects >.git/objects/info/alternates &&
|
||||||
|
echo content1 >file1 &&
|
||||||
|
# non-local loose object which is not present in bitmapped pack
|
||||||
|
altblob=$(GIT_DIR=alt.git git hash-object -w file1) &&
|
||||||
|
# non-local loose object which is also present in bitmapped pack
|
||||||
|
git cat-file blob $blob | GIT_DIR=alt.git git hash-object -w --stdin &&
|
||||||
|
git add file1 &&
|
||||||
|
test_tick &&
|
||||||
|
git commit -m commit_file1 &&
|
||||||
|
echo HEAD | git pack-objects --local --stdout --revs >1.pack &&
|
||||||
|
git index-pack 1.pack &&
|
||||||
|
list_packed_objects 1.idx >1.objects &&
|
||||||
|
printf "%s\n" "$altblob" "$blob" >nonlocal-loose &&
|
||||||
|
! has_any nonlocal-loose 1.objects
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'pack-objects respects --honor-pack-keep (local non-bitmapped pack)' '
|
||||||
|
echo content2 >file2 &&
|
||||||
|
blob2=$(git hash-object -w file2) &&
|
||||||
|
git add file2 &&
|
||||||
|
test_tick &&
|
||||||
|
git commit -m commit_file2 &&
|
||||||
|
printf "%s\n" "$blob2" "$bitmaptip" >keepobjects &&
|
||||||
|
pack2=$(git pack-objects pack2 <keepobjects) &&
|
||||||
|
mv pack2-$pack2.* .git/objects/pack/ &&
|
||||||
|
>.git/objects/pack/pack2-$pack2.keep &&
|
||||||
|
rm $(objpath $blob2) &&
|
||||||
|
echo HEAD | git pack-objects --honor-pack-keep --stdout --revs >2a.pack &&
|
||||||
|
git index-pack 2a.pack &&
|
||||||
|
list_packed_objects 2a.idx >2a.objects &&
|
||||||
|
! has_any keepobjects 2a.objects
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'pack-objects respects --local (non-local pack)' '
|
||||||
|
mv .git/objects/pack/pack2-$pack2.* alt.git/objects/pack/ &&
|
||||||
|
echo HEAD | git pack-objects --local --stdout --revs >2b.pack &&
|
||||||
|
git index-pack 2b.pack &&
|
||||||
|
list_packed_objects 2b.idx >2b.objects &&
|
||||||
|
! has_any keepobjects 2b.objects
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'pack-objects respects --honor-pack-keep (local bitmapped pack)' '
|
||||||
|
ls .git/objects/pack/ | grep bitmap >output &&
|
||||||
|
test_line_count = 1 output &&
|
||||||
|
packbitmap=$(basename $(cat output) .bitmap) &&
|
||||||
|
list_packed_objects .git/objects/pack/$packbitmap.idx >packbitmap.objects &&
|
||||||
|
test_when_finished "rm -f .git/objects/pack/$packbitmap.keep" &&
|
||||||
|
>.git/objects/pack/$packbitmap.keep &&
|
||||||
|
echo HEAD | git pack-objects --honor-pack-keep --stdout --revs >3a.pack &&
|
||||||
|
git index-pack 3a.pack &&
|
||||||
|
list_packed_objects 3a.idx >3a.objects &&
|
||||||
|
! has_any packbitmap.objects 3a.objects
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'pack-objects respects --local (non-local bitmapped pack)' '
|
||||||
|
mv .git/objects/pack/$packbitmap.* alt.git/objects/pack/ &&
|
||||||
|
rm -f .git/objects/pack/multi-pack-index &&
|
||||||
|
test_when_finished "mv alt.git/objects/pack/$packbitmap.* .git/objects/pack/" &&
|
||||||
|
echo HEAD | git pack-objects --local --stdout --revs >3b.pack &&
|
||||||
|
git index-pack 3b.pack &&
|
||||||
|
list_packed_objects 3b.idx >3b.objects &&
|
||||||
|
! has_any packbitmap.objects 3b.objects
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'pack-objects to file can use bitmap' '
|
||||||
|
# make sure we still have 1 bitmap index from previous tests
|
||||||
|
ls .git/objects/pack/ | grep bitmap >output &&
|
||||||
|
test_line_count = 1 output &&
|
||||||
|
# verify equivalent packs are generated with/without using bitmap index
|
||||||
|
packasha1=$(git pack-objects --no-use-bitmap-index --all packa </dev/null) &&
|
||||||
|
packbsha1=$(git pack-objects --use-bitmap-index --all packb </dev/null) &&
|
||||||
|
list_packed_objects packa-$packasha1.idx >packa.objects &&
|
||||||
|
list_packed_objects packb-$packbsha1.idx >packb.objects &&
|
||||||
|
test_cmp packa.objects packb.objects
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'full repack, reusing previous bitmaps' '
|
||||||
git repack -ad &&
|
git repack -ad &&
|
||||||
ls .git/objects/pack/ | grep bitmap >output &&
|
ls .git/objects/pack/ | grep bitmap >output &&
|
||||||
test_line_count = 1 output &&
|
test_line_count = 1 output
|
||||||
grep "\"key\":\"num_selected_commits\",\"value\":\"106\"" trace &&
|
'
|
||||||
grep "\"key\":\"num_maximal_commits\",\"value\":\"107\"" trace
|
|
||||||
'
|
|
||||||
|
|
||||||
basic_bitmap_tests
|
test_expect_success 'fetch (full bitmap)' '
|
||||||
|
git --git-dir=clone.git fetch origin second:second &&
|
||||||
|
git rev-parse HEAD >expect &&
|
||||||
|
git --git-dir=clone.git rev-parse HEAD >actual &&
|
||||||
|
test_cmp expect actual
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'create objects for missing-HAVE tests' '
|
||||||
|
blob=$(echo "missing have" | git hash-object -w --stdin) &&
|
||||||
|
tree=$(printf "100644 blob $blob\tfile\n" | git mktree) &&
|
||||||
|
parent=$(echo parent | git commit-tree $tree) &&
|
||||||
|
commit=$(echo commit | git commit-tree $tree -p $parent) &&
|
||||||
|
cat >revs <<-EOF
|
||||||
|
HEAD
|
||||||
|
^HEAD^
|
||||||
|
^$commit
|
||||||
|
EOF
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'pack-objects respects --incremental' '
|
||||||
|
cat >revs2 <<-EOF &&
|
||||||
|
HEAD
|
||||||
|
$commit
|
||||||
|
EOF
|
||||||
|
git pack-objects --incremental --stdout --revs <revs2 >4.pack &&
|
||||||
|
git index-pack 4.pack &&
|
||||||
|
list_packed_objects 4.idx >4.objects &&
|
||||||
|
test_line_count = 4 4.objects &&
|
||||||
|
git rev-list --objects $commit >revlist &&
|
||||||
|
cut -d" " -f1 revlist |sort >objects &&
|
||||||
|
test_cmp 4.objects objects
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'pack with missing blob' '
|
||||||
|
rm $(objpath $blob) &&
|
||||||
|
git pack-objects --stdout --revs <revs >/dev/null
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'pack with missing tree' '
|
||||||
|
rm $(objpath $tree) &&
|
||||||
|
git pack-objects --stdout --revs <revs >/dev/null
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'pack with missing parent' '
|
||||||
|
rm $(objpath $parent) &&
|
||||||
|
git pack-objects --stdout --revs <revs >/dev/null
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success JGIT,SHA1 'we can read jgit bitmaps' '
|
||||||
|
git clone --bare . compat-jgit.git &&
|
||||||
|
(
|
||||||
|
cd compat-jgit.git &&
|
||||||
|
rm -f objects/pack/*.bitmap &&
|
||||||
|
jgit gc &&
|
||||||
|
git rev-list --test-bitmap HEAD
|
||||||
|
)
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success JGIT,SHA1 'jgit can read our bitmaps' '
|
||||||
|
git clone --bare . compat-us.git &&
|
||||||
|
(
|
||||||
|
cd compat-us.git &&
|
||||||
|
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
|
||||||
|
git repack -adb &&
|
||||||
|
# jgit gc will barf if it does not like our bitmaps
|
||||||
|
jgit gc
|
||||||
|
)
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'splitting packs does not generate bogus bitmaps' '
|
||||||
|
test-tool genrandom foo $((1024 * 1024)) >rand &&
|
||||||
|
git add rand &&
|
||||||
|
git commit -m "commit with big file" &&
|
||||||
|
git -c pack.packSizeLimit=500k repack -adb &&
|
||||||
|
git init --bare no-bitmaps.git &&
|
||||||
|
git -C no-bitmaps.git fetch .. HEAD
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'set up reusable pack' '
|
||||||
|
rm -f .git/objects/pack/*.keep &&
|
||||||
|
git repack -adb &&
|
||||||
|
reusable_pack () {
|
||||||
|
git for-each-ref --format="%(objectname)" |
|
||||||
|
git pack-objects --delta-base-offset --revs --stdout "$@"
|
||||||
|
}
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'pack reuse respects --honor-pack-keep' '
|
||||||
|
test_when_finished "rm -f .git/objects/pack/*.keep" &&
|
||||||
|
for i in .git/objects/pack/*.pack
|
||||||
|
do
|
||||||
|
>${i%.pack}.keep || return 1
|
||||||
|
done &&
|
||||||
|
reusable_pack --honor-pack-keep >empty.pack &&
|
||||||
|
git index-pack empty.pack &&
|
||||||
|
git show-index <empty.idx >actual &&
|
||||||
|
test_must_be_empty actual
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'pack reuse respects --local' '
|
||||||
|
mv .git/objects/pack/* alt.git/objects/pack/ &&
|
||||||
|
test_when_finished "mv alt.git/objects/pack/* .git/objects/pack/" &&
|
||||||
|
reusable_pack --local >empty.pack &&
|
||||||
|
git index-pack empty.pack &&
|
||||||
|
git show-index <empty.idx >actual &&
|
||||||
|
test_must_be_empty actual
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'pack reuse respects --incremental' '
|
||||||
|
reusable_pack --incremental >empty.pack &&
|
||||||
|
git index-pack empty.pack &&
|
||||||
|
git show-index <empty.idx >actual &&
|
||||||
|
test_must_be_empty actual
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'truncated bitmap fails gracefully (ewah)' '
|
||||||
|
test_config pack.writebitmaphashcache false &&
|
||||||
|
test_config pack.writebitmaplookuptable false &&
|
||||||
|
git repack -ad &&
|
||||||
|
git rev-list --use-bitmap-index --count --all >expect &&
|
||||||
|
bitmap=$(ls .git/objects/pack/*.bitmap) &&
|
||||||
|
test_when_finished "rm -f $bitmap" &&
|
||||||
|
test_copy_bytes 256 <$bitmap >$bitmap.tmp &&
|
||||||
|
mv -f $bitmap.tmp $bitmap &&
|
||||||
|
git rev-list --use-bitmap-index --count --all >actual 2>stderr &&
|
||||||
|
test_cmp expect actual &&
|
||||||
|
test_i18ngrep corrupt.ewah.bitmap stderr
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'truncated bitmap fails gracefully (cache)' '
|
||||||
|
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
|
||||||
|
git repack -ad &&
|
||||||
|
git rev-list --use-bitmap-index --count --all >expect &&
|
||||||
|
bitmap=$(ls .git/objects/pack/*.bitmap) &&
|
||||||
|
test_when_finished "rm -f $bitmap" &&
|
||||||
|
test_copy_bytes 512 <$bitmap >$bitmap.tmp &&
|
||||||
|
mv -f $bitmap.tmp $bitmap &&
|
||||||
|
git rev-list --use-bitmap-index --count --all >actual 2>stderr &&
|
||||||
|
test_cmp expect actual &&
|
||||||
|
test_i18ngrep corrupted.bitmap.index stderr
|
||||||
|
'
|
||||||
|
|
||||||
|
# Create a state of history with these properties:
|
||||||
|
#
|
||||||
|
# - refs that allow a client to fetch some new history, while sharing some old
|
||||||
|
# history with the server; we use branches delta-reuse-old and
|
||||||
|
# delta-reuse-new here
|
||||||
|
#
|
||||||
|
# - the new history contains an object that is stored on the server as a delta
|
||||||
|
# against a base that is in the old history
|
||||||
|
#
|
||||||
|
# - the base object is not immediately reachable from the tip of the old
|
||||||
|
# history; finding it would involve digging down through history we know the
|
||||||
|
# other side has
|
||||||
|
#
|
||||||
|
# This should result in a state where fetching from old->new would not
|
||||||
|
# traditionally reuse the on-disk delta (because we'd have to dig to realize
|
||||||
|
# that the client has it), but we will do so if bitmaps can tell us cheaply
|
||||||
|
# that the other side has it.
|
||||||
|
test_expect_success 'set up thin delta-reuse parent' '
|
||||||
|
# This first commit contains the buried base object.
|
||||||
|
test-tool genrandom delta 16384 >file &&
|
||||||
|
git add file &&
|
||||||
|
git commit -m "delta base" &&
|
||||||
|
base=$(git rev-parse --verify HEAD:file) &&
|
||||||
|
|
||||||
|
# These intermediate commits bury the base back in history.
|
||||||
|
# This becomes the "old" state.
|
||||||
|
for i in 1 2 3 4 5
|
||||||
|
do
|
||||||
|
echo $i >file &&
|
||||||
|
git commit -am "intermediate $i" || return 1
|
||||||
|
done &&
|
||||||
|
git branch delta-reuse-old &&
|
||||||
|
|
||||||
|
# And now our new history has a delta against the buried base. Note
|
||||||
|
# that this must be smaller than the original file, since pack-objects
|
||||||
|
# prefers to create deltas from smaller objects to larger.
|
||||||
|
test-tool genrandom delta 16300 >file &&
|
||||||
|
git commit -am "delta result" &&
|
||||||
|
delta=$(git rev-parse --verify HEAD:file) &&
|
||||||
|
git branch delta-reuse-new &&
|
||||||
|
|
||||||
|
# Repack with bitmaps and double check that we have the expected delta
|
||||||
|
# relationship.
|
||||||
|
git repack -adb &&
|
||||||
|
have_delta $delta $base
|
||||||
|
'
|
||||||
|
|
||||||
|
# Now we can sanity-check the non-bitmap behavior (that the server is not able
|
||||||
|
# to reuse the delta). This isn't strictly something we care about, so this
|
||||||
|
# test could be scrapped in the future. But it makes sure that the next test is
|
||||||
|
# actually triggering the feature we want.
|
||||||
|
#
|
||||||
|
# Note that our tools for working with on-the-wire "thin" packs are limited. So
|
||||||
|
# we actually perform the fetch, retain the resulting pack, and inspect the
|
||||||
|
# result.
|
||||||
|
test_expect_success 'fetch without bitmaps ignores delta against old base' '
|
||||||
|
test_config pack.usebitmaps false &&
|
||||||
|
test_when_finished "rm -rf client.git" &&
|
||||||
|
git init --bare client.git &&
|
||||||
|
(
|
||||||
|
cd client.git &&
|
||||||
|
git config transfer.unpackLimit 1 &&
|
||||||
|
git fetch .. delta-reuse-old:delta-reuse-old &&
|
||||||
|
git fetch .. delta-reuse-new:delta-reuse-new &&
|
||||||
|
have_delta $delta $ZERO_OID
|
||||||
|
)
|
||||||
|
'
|
||||||
|
|
||||||
|
# And do the same for the bitmap case, where we do expect to find the delta.
|
||||||
|
test_expect_success 'fetch with bitmaps can reuse old base' '
|
||||||
|
test_config pack.usebitmaps true &&
|
||||||
|
test_when_finished "rm -rf client.git" &&
|
||||||
|
git init --bare client.git &&
|
||||||
|
(
|
||||||
|
cd client.git &&
|
||||||
|
git config transfer.unpackLimit 1 &&
|
||||||
|
git fetch .. delta-reuse-old:delta-reuse-old &&
|
||||||
|
git fetch .. delta-reuse-new:delta-reuse-new &&
|
||||||
|
have_delta $delta $base
|
||||||
|
)
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'pack.preferBitmapTips' '
|
||||||
|
git init repo &&
|
||||||
|
test_when_finished "rm -fr repo" &&
|
||||||
|
(
|
||||||
|
cd repo &&
|
||||||
|
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
|
||||||
|
|
||||||
|
# create enough commits that not all are receive bitmap
|
||||||
|
# coverage even if they are all at the tip of some reference.
|
||||||
|
test_commit_bulk --message="%s" 103 &&
|
||||||
|
|
||||||
|
git rev-list HEAD >commits.raw &&
|
||||||
|
sort <commits.raw >commits &&
|
||||||
|
|
||||||
|
git log --format="create refs/tags/%s %H" HEAD >refs &&
|
||||||
|
git update-ref --stdin <refs &&
|
||||||
|
|
||||||
|
git repack -adb &&
|
||||||
|
test-tool bitmap list-commits | sort >bitmaps &&
|
||||||
|
|
||||||
|
# remember which commits did not receive bitmaps
|
||||||
|
comm -13 bitmaps commits >before &&
|
||||||
|
test_file_not_empty before &&
|
||||||
|
|
||||||
|
# mark the commits which did not receive bitmaps as preferred,
|
||||||
|
# and generate the bitmap again
|
||||||
|
perl -pe "s{^}{create refs/tags/include/$. }" <before |
|
||||||
|
git update-ref --stdin &&
|
||||||
|
git -c pack.preferBitmapTips=refs/tags/include repack -adb &&
|
||||||
|
|
||||||
|
# finally, check that the commit(s) without bitmap coverage
|
||||||
|
# are not the same ones as before
|
||||||
|
test-tool bitmap list-commits | sort >bitmaps &&
|
||||||
|
comm -13 bitmaps commits >after &&
|
||||||
|
|
||||||
|
! test_cmp before after
|
||||||
|
)
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'complains about multiple pack bitmaps' '
|
||||||
|
rm -fr repo &&
|
||||||
|
git init repo &&
|
||||||
|
test_when_finished "rm -fr repo" &&
|
||||||
|
(
|
||||||
|
cd repo &&
|
||||||
|
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
|
||||||
|
|
||||||
|
test_commit base &&
|
||||||
|
|
||||||
|
git repack -adb &&
|
||||||
|
bitmap="$(ls .git/objects/pack/pack-*.bitmap)" &&
|
||||||
|
mv "$bitmap" "$bitmap.bak" &&
|
||||||
|
|
||||||
|
test_commit other &&
|
||||||
|
git repack -ab &&
|
||||||
|
|
||||||
|
mv "$bitmap.bak" "$bitmap" &&
|
||||||
|
|
||||||
|
find .git/objects/pack -type f -name "*.pack" >packs &&
|
||||||
|
find .git/objects/pack -type f -name "*.bitmap" >bitmaps &&
|
||||||
|
test_line_count = 2 packs &&
|
||||||
|
test_line_count = 2 bitmaps &&
|
||||||
|
|
||||||
|
git rev-list --use-bitmap-index HEAD 2>err &&
|
||||||
|
grep "ignoring extra bitmap file" err
|
||||||
|
)
|
||||||
|
'
|
||||||
|
}
|
||||||
|
|
||||||
|
test_bitmap_cases
|
||||||
|
|
||||||
test_expect_success 'incremental repack fails when bitmaps are requested' '
|
test_expect_success 'incremental repack fails when bitmaps are requested' '
|
||||||
test_commit more-1 &&
|
test_commit more-1 &&
|
||||||
|
|
@ -54,219 +447,24 @@ test_expect_success 'incremental repack can disable bitmaps' '
|
||||||
git repack -d --no-write-bitmap-index
|
git repack -d --no-write-bitmap-index
|
||||||
'
|
'
|
||||||
|
|
||||||
test_expect_success 'pack-objects respects --local (non-local loose)' '
|
test_bitmap_cases "pack.writeBitmapLookupTable"
|
||||||
git init --bare alt.git &&
|
|
||||||
echo $(pwd)/alt.git/objects >.git/objects/info/alternates &&
|
test_expect_success 'verify writing bitmap lookup table when enabled' '
|
||||||
echo content1 >file1 &&
|
GIT_TRACE2_EVENT="$(pwd)/trace2" \
|
||||||
# non-local loose object which is not present in bitmapped pack
|
git repack -ad &&
|
||||||
altblob=$(GIT_DIR=alt.git git hash-object -w file1) &&
|
grep "\"label\":\"writing_lookup_table\"" trace2
|
||||||
# non-local loose object which is also present in bitmapped pack
|
|
||||||
git cat-file blob $blob | GIT_DIR=alt.git git hash-object -w --stdin &&
|
|
||||||
git add file1 &&
|
|
||||||
test_tick &&
|
|
||||||
git commit -m commit_file1 &&
|
|
||||||
echo HEAD | git pack-objects --local --stdout --revs >1.pack &&
|
|
||||||
git index-pack 1.pack &&
|
|
||||||
list_packed_objects 1.idx >1.objects &&
|
|
||||||
printf "%s\n" "$altblob" "$blob" >nonlocal-loose &&
|
|
||||||
! has_any nonlocal-loose 1.objects
|
|
||||||
'
|
'
|
||||||
|
|
||||||
test_expect_success 'pack-objects respects --honor-pack-keep (local non-bitmapped pack)' '
|
test_expect_success 'lookup table is actually used to traverse objects' '
|
||||||
echo content2 >file2 &&
|
|
||||||
blob2=$(git hash-object -w file2) &&
|
|
||||||
git add file2 &&
|
|
||||||
test_tick &&
|
|
||||||
git commit -m commit_file2 &&
|
|
||||||
printf "%s\n" "$blob2" "$bitmaptip" >keepobjects &&
|
|
||||||
pack2=$(git pack-objects pack2 <keepobjects) &&
|
|
||||||
mv pack2-$pack2.* .git/objects/pack/ &&
|
|
||||||
>.git/objects/pack/pack2-$pack2.keep &&
|
|
||||||
rm $(objpath $blob2) &&
|
|
||||||
echo HEAD | git pack-objects --honor-pack-keep --stdout --revs >2a.pack &&
|
|
||||||
git index-pack 2a.pack &&
|
|
||||||
list_packed_objects 2a.idx >2a.objects &&
|
|
||||||
! has_any keepobjects 2a.objects
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'pack-objects respects --local (non-local pack)' '
|
|
||||||
mv .git/objects/pack/pack2-$pack2.* alt.git/objects/pack/ &&
|
|
||||||
echo HEAD | git pack-objects --local --stdout --revs >2b.pack &&
|
|
||||||
git index-pack 2b.pack &&
|
|
||||||
list_packed_objects 2b.idx >2b.objects &&
|
|
||||||
! has_any keepobjects 2b.objects
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'pack-objects respects --honor-pack-keep (local bitmapped pack)' '
|
|
||||||
ls .git/objects/pack/ | grep bitmap >output &&
|
|
||||||
test_line_count = 1 output &&
|
|
||||||
packbitmap=$(basename $(cat output) .bitmap) &&
|
|
||||||
list_packed_objects .git/objects/pack/$packbitmap.idx >packbitmap.objects &&
|
|
||||||
test_when_finished "rm -f .git/objects/pack/$packbitmap.keep" &&
|
|
||||||
>.git/objects/pack/$packbitmap.keep &&
|
|
||||||
echo HEAD | git pack-objects --honor-pack-keep --stdout --revs >3a.pack &&
|
|
||||||
git index-pack 3a.pack &&
|
|
||||||
list_packed_objects 3a.idx >3a.objects &&
|
|
||||||
! has_any packbitmap.objects 3a.objects
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'pack-objects respects --local (non-local bitmapped pack)' '
|
|
||||||
mv .git/objects/pack/$packbitmap.* alt.git/objects/pack/ &&
|
|
||||||
rm -f .git/objects/pack/multi-pack-index &&
|
|
||||||
test_when_finished "mv alt.git/objects/pack/$packbitmap.* .git/objects/pack/" &&
|
|
||||||
echo HEAD | git pack-objects --local --stdout --revs >3b.pack &&
|
|
||||||
git index-pack 3b.pack &&
|
|
||||||
list_packed_objects 3b.idx >3b.objects &&
|
|
||||||
! has_any packbitmap.objects 3b.objects
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'pack-objects to file can use bitmap' '
|
|
||||||
# make sure we still have 1 bitmap index from previous tests
|
|
||||||
ls .git/objects/pack/ | grep bitmap >output &&
|
|
||||||
test_line_count = 1 output &&
|
|
||||||
# verify equivalent packs are generated with/without using bitmap index
|
|
||||||
packasha1=$(git pack-objects --no-use-bitmap-index --all packa </dev/null) &&
|
|
||||||
packbsha1=$(git pack-objects --use-bitmap-index --all packb </dev/null) &&
|
|
||||||
list_packed_objects packa-$packasha1.idx >packa.objects &&
|
|
||||||
list_packed_objects packb-$packbsha1.idx >packb.objects &&
|
|
||||||
test_cmp packa.objects packb.objects
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'full repack, reusing previous bitmaps' '
|
|
||||||
git repack -ad &&
|
|
||||||
ls .git/objects/pack/ | grep bitmap >output &&
|
|
||||||
test_line_count = 1 output
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'fetch (full bitmap)' '
|
|
||||||
git --git-dir=clone.git fetch origin second:second &&
|
|
||||||
git rev-parse HEAD >expect &&
|
|
||||||
git --git-dir=clone.git rev-parse HEAD >actual &&
|
|
||||||
test_cmp expect actual
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'create objects for missing-HAVE tests' '
|
|
||||||
blob=$(echo "missing have" | git hash-object -w --stdin) &&
|
|
||||||
tree=$(printf "100644 blob $blob\tfile\n" | git mktree) &&
|
|
||||||
parent=$(echo parent | git commit-tree $tree) &&
|
|
||||||
commit=$(echo commit | git commit-tree $tree -p $parent) &&
|
|
||||||
cat >revs <<-EOF
|
|
||||||
HEAD
|
|
||||||
^HEAD^
|
|
||||||
^$commit
|
|
||||||
EOF
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'pack-objects respects --incremental' '
|
|
||||||
cat >revs2 <<-EOF &&
|
|
||||||
HEAD
|
|
||||||
$commit
|
|
||||||
EOF
|
|
||||||
git pack-objects --incremental --stdout --revs <revs2 >4.pack &&
|
|
||||||
git index-pack 4.pack &&
|
|
||||||
list_packed_objects 4.idx >4.objects &&
|
|
||||||
test_line_count = 4 4.objects &&
|
|
||||||
git rev-list --objects $commit >revlist &&
|
|
||||||
cut -d" " -f1 revlist |sort >objects &&
|
|
||||||
test_cmp 4.objects objects
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'pack with missing blob' '
|
|
||||||
rm $(objpath $blob) &&
|
|
||||||
git pack-objects --stdout --revs <revs >/dev/null
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'pack with missing tree' '
|
|
||||||
rm $(objpath $tree) &&
|
|
||||||
git pack-objects --stdout --revs <revs >/dev/null
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'pack with missing parent' '
|
|
||||||
rm $(objpath $parent) &&
|
|
||||||
git pack-objects --stdout --revs <revs >/dev/null
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success JGIT,SHA1 'we can read jgit bitmaps' '
|
|
||||||
git clone --bare . compat-jgit.git &&
|
|
||||||
(
|
|
||||||
cd compat-jgit.git &&
|
|
||||||
rm -f objects/pack/*.bitmap &&
|
|
||||||
jgit gc &&
|
|
||||||
git rev-list --test-bitmap HEAD
|
|
||||||
)
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success JGIT,SHA1 'jgit can read our bitmaps' '
|
|
||||||
git clone --bare . compat-us.git &&
|
|
||||||
(
|
|
||||||
cd compat-us.git &&
|
|
||||||
git repack -adb &&
|
|
||||||
# jgit gc will barf if it does not like our bitmaps
|
|
||||||
jgit gc
|
|
||||||
)
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'splitting packs does not generate bogus bitmaps' '
|
|
||||||
test-tool genrandom foo $((1024 * 1024)) >rand &&
|
|
||||||
git add rand &&
|
|
||||||
git commit -m "commit with big file" &&
|
|
||||||
git -c pack.packSizeLimit=500k repack -adb &&
|
|
||||||
git init --bare no-bitmaps.git &&
|
|
||||||
git -C no-bitmaps.git fetch .. HEAD
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'set up reusable pack' '
|
|
||||||
rm -f .git/objects/pack/*.keep &&
|
|
||||||
git repack -adb &&
|
git repack -adb &&
|
||||||
reusable_pack () {
|
GIT_TRACE2_EVENT="$(pwd)/trace3" \
|
||||||
git for-each-ref --format="%(objectname)" |
|
git rev-list --use-bitmap-index --count --all &&
|
||||||
git pack-objects --delta-base-offset --revs --stdout "$@"
|
grep "\"label\":\"reading_lookup_table\"" trace3
|
||||||
}
|
|
||||||
'
|
'
|
||||||
|
|
||||||
test_expect_success 'pack reuse respects --honor-pack-keep' '
|
test_expect_success 'truncated bitmap fails gracefully (lookup table)' '
|
||||||
test_when_finished "rm -f .git/objects/pack/*.keep" &&
|
|
||||||
for i in .git/objects/pack/*.pack
|
|
||||||
do
|
|
||||||
>${i%.pack}.keep || return 1
|
|
||||||
done &&
|
|
||||||
reusable_pack --honor-pack-keep >empty.pack &&
|
|
||||||
git index-pack empty.pack &&
|
|
||||||
git show-index <empty.idx >actual &&
|
|
||||||
test_must_be_empty actual
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'pack reuse respects --local' '
|
|
||||||
mv .git/objects/pack/* alt.git/objects/pack/ &&
|
|
||||||
test_when_finished "mv alt.git/objects/pack/* .git/objects/pack/" &&
|
|
||||||
reusable_pack --local >empty.pack &&
|
|
||||||
git index-pack empty.pack &&
|
|
||||||
git show-index <empty.idx >actual &&
|
|
||||||
test_must_be_empty actual
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'pack reuse respects --incremental' '
|
|
||||||
reusable_pack --incremental >empty.pack &&
|
|
||||||
git index-pack empty.pack &&
|
|
||||||
git show-index <empty.idx >actual &&
|
|
||||||
test_must_be_empty actual
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'truncated bitmap fails gracefully (ewah)' '
|
|
||||||
test_config pack.writebitmaphashcache false &&
|
test_config pack.writebitmaphashcache false &&
|
||||||
git repack -ad &&
|
git repack -adb &&
|
||||||
git rev-list --use-bitmap-index --count --all >expect &&
|
|
||||||
bitmap=$(ls .git/objects/pack/*.bitmap) &&
|
|
||||||
test_when_finished "rm -f $bitmap" &&
|
|
||||||
test_copy_bytes 256 <$bitmap >$bitmap.tmp &&
|
|
||||||
mv -f $bitmap.tmp $bitmap &&
|
|
||||||
git rev-list --use-bitmap-index --count --all >actual 2>stderr &&
|
|
||||||
test_cmp expect actual &&
|
|
||||||
test_i18ngrep corrupt.ewah.bitmap stderr
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'truncated bitmap fails gracefully (cache)' '
|
|
||||||
git repack -ad &&
|
|
||||||
git rev-list --use-bitmap-index --count --all >expect &&
|
git rev-list --use-bitmap-index --count --all >expect &&
|
||||||
bitmap=$(ls .git/objects/pack/*.bitmap) &&
|
bitmap=$(ls .git/objects/pack/*.bitmap) &&
|
||||||
test_when_finished "rm -f $bitmap" &&
|
test_when_finished "rm -f $bitmap" &&
|
||||||
|
|
@ -277,152 +475,4 @@ test_expect_success 'truncated bitmap fails gracefully (cache)' '
|
||||||
test_i18ngrep corrupted.bitmap.index stderr
|
test_i18ngrep corrupted.bitmap.index stderr
|
||||||
'
|
'
|
||||||
|
|
||||||
# Create a state of history with these properties:
|
|
||||||
#
|
|
||||||
# - refs that allow a client to fetch some new history, while sharing some old
|
|
||||||
# history with the server; we use branches delta-reuse-old and
|
|
||||||
# delta-reuse-new here
|
|
||||||
#
|
|
||||||
# - the new history contains an object that is stored on the server as a delta
|
|
||||||
# against a base that is in the old history
|
|
||||||
#
|
|
||||||
# - the base object is not immediately reachable from the tip of the old
|
|
||||||
# history; finding it would involve digging down through history we know the
|
|
||||||
# other side has
|
|
||||||
#
|
|
||||||
# This should result in a state where fetching from old->new would not
|
|
||||||
# traditionally reuse the on-disk delta (because we'd have to dig to realize
|
|
||||||
# that the client has it), but we will do so if bitmaps can tell us cheaply
|
|
||||||
# that the other side has it.
|
|
||||||
test_expect_success 'set up thin delta-reuse parent' '
|
|
||||||
# This first commit contains the buried base object.
|
|
||||||
test-tool genrandom delta 16384 >file &&
|
|
||||||
git add file &&
|
|
||||||
git commit -m "delta base" &&
|
|
||||||
base=$(git rev-parse --verify HEAD:file) &&
|
|
||||||
|
|
||||||
# These intermediate commits bury the base back in history.
|
|
||||||
# This becomes the "old" state.
|
|
||||||
for i in 1 2 3 4 5
|
|
||||||
do
|
|
||||||
echo $i >file &&
|
|
||||||
git commit -am "intermediate $i" || return 1
|
|
||||||
done &&
|
|
||||||
git branch delta-reuse-old &&
|
|
||||||
|
|
||||||
# And now our new history has a delta against the buried base. Note
|
|
||||||
# that this must be smaller than the original file, since pack-objects
|
|
||||||
# prefers to create deltas from smaller objects to larger.
|
|
||||||
test-tool genrandom delta 16300 >file &&
|
|
||||||
git commit -am "delta result" &&
|
|
||||||
delta=$(git rev-parse --verify HEAD:file) &&
|
|
||||||
git branch delta-reuse-new &&
|
|
||||||
|
|
||||||
# Repack with bitmaps and double check that we have the expected delta
|
|
||||||
# relationship.
|
|
||||||
git repack -adb &&
|
|
||||||
have_delta $delta $base
|
|
||||||
'
|
|
||||||
|
|
||||||
# Now we can sanity-check the non-bitmap behavior (that the server is not able
|
|
||||||
# to reuse the delta). This isn't strictly something we care about, so this
|
|
||||||
# test could be scrapped in the future. But it makes sure that the next test is
|
|
||||||
# actually triggering the feature we want.
|
|
||||||
#
|
|
||||||
# Note that our tools for working with on-the-wire "thin" packs are limited. So
|
|
||||||
# we actually perform the fetch, retain the resulting pack, and inspect the
|
|
||||||
# result.
|
|
||||||
test_expect_success 'fetch without bitmaps ignores delta against old base' '
|
|
||||||
test_config pack.usebitmaps false &&
|
|
||||||
test_when_finished "rm -rf client.git" &&
|
|
||||||
git init --bare client.git &&
|
|
||||||
(
|
|
||||||
cd client.git &&
|
|
||||||
git config transfer.unpackLimit 1 &&
|
|
||||||
git fetch .. delta-reuse-old:delta-reuse-old &&
|
|
||||||
git fetch .. delta-reuse-new:delta-reuse-new &&
|
|
||||||
have_delta $delta $ZERO_OID
|
|
||||||
)
|
|
||||||
'
|
|
||||||
|
|
||||||
# And do the same for the bitmap case, where we do expect to find the delta.
|
|
||||||
test_expect_success 'fetch with bitmaps can reuse old base' '
|
|
||||||
test_config pack.usebitmaps true &&
|
|
||||||
test_when_finished "rm -rf client.git" &&
|
|
||||||
git init --bare client.git &&
|
|
||||||
(
|
|
||||||
cd client.git &&
|
|
||||||
git config transfer.unpackLimit 1 &&
|
|
||||||
git fetch .. delta-reuse-old:delta-reuse-old &&
|
|
||||||
git fetch .. delta-reuse-new:delta-reuse-new &&
|
|
||||||
have_delta $delta $base
|
|
||||||
)
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'pack.preferBitmapTips' '
|
|
||||||
git init repo &&
|
|
||||||
test_when_finished "rm -fr repo" &&
|
|
||||||
(
|
|
||||||
cd repo &&
|
|
||||||
|
|
||||||
# create enough commits that not all are receive bitmap
|
|
||||||
# coverage even if they are all at the tip of some reference.
|
|
||||||
test_commit_bulk --message="%s" 103 &&
|
|
||||||
|
|
||||||
git rev-list HEAD >commits.raw &&
|
|
||||||
sort <commits.raw >commits &&
|
|
||||||
|
|
||||||
git log --format="create refs/tags/%s %H" HEAD >refs &&
|
|
||||||
git update-ref --stdin <refs &&
|
|
||||||
|
|
||||||
git repack -adb &&
|
|
||||||
test-tool bitmap list-commits | sort >bitmaps &&
|
|
||||||
|
|
||||||
# remember which commits did not receive bitmaps
|
|
||||||
comm -13 bitmaps commits >before &&
|
|
||||||
test_file_not_empty before &&
|
|
||||||
|
|
||||||
# mark the commits which did not receive bitmaps as preferred,
|
|
||||||
# and generate the bitmap again
|
|
||||||
perl -pe "s{^}{create refs/tags/include/$. }" <before |
|
|
||||||
git update-ref --stdin &&
|
|
||||||
git -c pack.preferBitmapTips=refs/tags/include repack -adb &&
|
|
||||||
|
|
||||||
# finally, check that the commit(s) without bitmap coverage
|
|
||||||
# are not the same ones as before
|
|
||||||
test-tool bitmap list-commits | sort >bitmaps &&
|
|
||||||
comm -13 bitmaps commits >after &&
|
|
||||||
|
|
||||||
! test_cmp before after
|
|
||||||
)
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'complains about multiple pack bitmaps' '
|
|
||||||
rm -fr repo &&
|
|
||||||
git init repo &&
|
|
||||||
test_when_finished "rm -fr repo" &&
|
|
||||||
(
|
|
||||||
cd repo &&
|
|
||||||
|
|
||||||
test_commit base &&
|
|
||||||
|
|
||||||
git repack -adb &&
|
|
||||||
bitmap="$(ls .git/objects/pack/pack-*.bitmap)" &&
|
|
||||||
mv "$bitmap" "$bitmap.bak" &&
|
|
||||||
|
|
||||||
test_commit other &&
|
|
||||||
git repack -ab &&
|
|
||||||
|
|
||||||
mv "$bitmap.bak" "$bitmap" &&
|
|
||||||
|
|
||||||
find .git/objects/pack -type f -name "*.pack" >packs &&
|
|
||||||
find .git/objects/pack -type f -name "*.bitmap" >bitmaps &&
|
|
||||||
test_line_count = 2 packs &&
|
|
||||||
test_line_count = 2 bitmaps &&
|
|
||||||
|
|
||||||
git rev-list --use-bitmap-index HEAD 2>err &&
|
|
||||||
grep "ignoring extra bitmap file" err
|
|
||||||
)
|
|
||||||
'
|
|
||||||
|
|
||||||
test_done
|
test_done
|
||||||
|
|
|
||||||
|
|
@ -17,23 +17,40 @@ test_description='check bitmap operation with shallow repositories'
|
||||||
# the tree for A. But in a shallow one, we've grafted away
|
# the tree for A. But in a shallow one, we've grafted away
|
||||||
# A, and fetching A to B requires that the other side send
|
# A, and fetching A to B requires that the other side send
|
||||||
# us the tree for file=1.
|
# us the tree for file=1.
|
||||||
test_expect_success 'setup shallow repo' '
|
test_shallow_bitmaps () {
|
||||||
echo 1 >file &&
|
writeLookupTable=false
|
||||||
git add file &&
|
|
||||||
git commit -m orig &&
|
|
||||||
echo 2 >file &&
|
|
||||||
git commit -a -m update &&
|
|
||||||
git clone --no-local --bare --depth=1 . shallow.git &&
|
|
||||||
echo 1 >file &&
|
|
||||||
git commit -a -m repeat
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'turn on bitmaps in the parent' '
|
for i in "$@"
|
||||||
git repack -adb
|
do
|
||||||
'
|
case $i in
|
||||||
|
"pack.writeBitmapLookupTable") writeLookupTable=true;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
test_expect_success 'shallow fetch from bitmapped repo' '
|
test_expect_success 'setup shallow repo' '
|
||||||
(cd shallow.git && git fetch)
|
rm -rf * .git &&
|
||||||
'
|
git init &&
|
||||||
|
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
|
||||||
|
echo 1 >file &&
|
||||||
|
git add file &&
|
||||||
|
git commit -m orig &&
|
||||||
|
echo 2 >file &&
|
||||||
|
git commit -a -m update &&
|
||||||
|
git clone --no-local --bare --depth=1 . shallow.git &&
|
||||||
|
echo 1 >file &&
|
||||||
|
git commit -a -m repeat
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'turn on bitmaps in the parent' '
|
||||||
|
git repack -adb
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'shallow fetch from bitmapped repo' '
|
||||||
|
(cd shallow.git && git fetch)
|
||||||
|
'
|
||||||
|
}
|
||||||
|
|
||||||
|
test_shallow_bitmaps
|
||||||
|
test_shallow_bitmaps "pack.writeBitmapLookupTable"
|
||||||
|
|
||||||
test_done
|
test_done
|
||||||
|
|
|
||||||
|
|
@ -15,17 +15,24 @@ GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=0
|
||||||
sane_unset GIT_TEST_MIDX_WRITE_REV
|
sane_unset GIT_TEST_MIDX_WRITE_REV
|
||||||
sane_unset GIT_TEST_MIDX_READ_RIDX
|
sane_unset GIT_TEST_MIDX_READ_RIDX
|
||||||
|
|
||||||
midx_bitmap_core
|
|
||||||
|
|
||||||
bitmap_reuse_tests() {
|
bitmap_reuse_tests() {
|
||||||
from=$1
|
from=$1
|
||||||
to=$2
|
to=$2
|
||||||
|
writeLookupTable=false
|
||||||
|
|
||||||
|
for i in $3-${$#}
|
||||||
|
do
|
||||||
|
case $i in
|
||||||
|
"pack.writeBitmapLookupTable") writeLookupTable=true;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
test_expect_success "setup pack reuse tests ($from -> $to)" '
|
test_expect_success "setup pack reuse tests ($from -> $to)" '
|
||||||
rm -fr repo &&
|
rm -fr repo &&
|
||||||
git init repo &&
|
git init repo &&
|
||||||
(
|
(
|
||||||
cd repo &&
|
cd repo &&
|
||||||
|
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
|
||||||
test_commit_bulk 16 &&
|
test_commit_bulk 16 &&
|
||||||
git tag old-tip &&
|
git tag old-tip &&
|
||||||
|
|
||||||
|
|
@ -43,6 +50,7 @@ bitmap_reuse_tests() {
|
||||||
test_expect_success "build bitmap from existing ($from -> $to)" '
|
test_expect_success "build bitmap from existing ($from -> $to)" '
|
||||||
(
|
(
|
||||||
cd repo &&
|
cd repo &&
|
||||||
|
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
|
||||||
test_commit_bulk --id=further 16 &&
|
test_commit_bulk --id=further 16 &&
|
||||||
git tag new-tip &&
|
git tag new-tip &&
|
||||||
|
|
||||||
|
|
@ -59,6 +67,7 @@ bitmap_reuse_tests() {
|
||||||
test_expect_success "verify resulting bitmaps ($from -> $to)" '
|
test_expect_success "verify resulting bitmaps ($from -> $to)" '
|
||||||
(
|
(
|
||||||
cd repo &&
|
cd repo &&
|
||||||
|
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
|
||||||
git for-each-ref &&
|
git for-each-ref &&
|
||||||
git rev-list --test-bitmap refs/tags/old-tip &&
|
git rev-list --test-bitmap refs/tags/old-tip &&
|
||||||
git rev-list --test-bitmap refs/tags/new-tip
|
git rev-list --test-bitmap refs/tags/new-tip
|
||||||
|
|
@ -66,244 +75,294 @@ bitmap_reuse_tests() {
|
||||||
'
|
'
|
||||||
}
|
}
|
||||||
|
|
||||||
bitmap_reuse_tests 'pack' 'MIDX'
|
test_midx_bitmap_cases () {
|
||||||
bitmap_reuse_tests 'MIDX' 'pack'
|
writeLookupTable=false
|
||||||
bitmap_reuse_tests 'MIDX' 'MIDX'
|
writeBitmapLookupTable=
|
||||||
|
|
||||||
test_expect_success 'missing object closure fails gracefully' '
|
for i in "$@"
|
||||||
rm -fr repo &&
|
do
|
||||||
git init repo &&
|
case $i in
|
||||||
test_when_finished "rm -fr repo" &&
|
"pack.writeBitmapLookupTable")
|
||||||
(
|
writeLookupTable=true
|
||||||
cd repo &&
|
writeBitmapLookupTable="$i"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
test_commit loose &&
|
test_expect_success 'setup test_repository' '
|
||||||
test_commit packed &&
|
rm -rf * .git &&
|
||||||
|
git init &&
|
||||||
|
git config pack.writeBitmapLookupTable '"$writeLookupTable"'
|
||||||
|
'
|
||||||
|
|
||||||
# Do not pass "--revs"; we want a pack without the "loose"
|
midx_bitmap_core
|
||||||
# commit.
|
|
||||||
git pack-objects $objdir/pack/pack <<-EOF &&
|
|
||||||
$(git rev-parse packed)
|
|
||||||
EOF
|
|
||||||
|
|
||||||
test_must_fail git multi-pack-index write --bitmap 2>err &&
|
bitmap_reuse_tests 'pack' 'MIDX' "$writeBitmapLookupTable"
|
||||||
grep "doesn.t have full closure" err &&
|
bitmap_reuse_tests 'MIDX' 'pack' "$writeBitmapLookupTable"
|
||||||
test_path_is_missing $midx
|
bitmap_reuse_tests 'MIDX' 'MIDX' "$writeBitmapLookupTable"
|
||||||
)
|
|
||||||
'
|
|
||||||
|
|
||||||
midx_bitmap_partial_tests
|
|
||||||
|
|
||||||
test_expect_success 'removing a MIDX clears stale bitmaps' '
|
|
||||||
rm -fr repo &&
|
|
||||||
git init repo &&
|
|
||||||
test_when_finished "rm -fr repo" &&
|
|
||||||
(
|
|
||||||
cd repo &&
|
|
||||||
test_commit base &&
|
|
||||||
git repack &&
|
|
||||||
git multi-pack-index write --bitmap &&
|
|
||||||
|
|
||||||
# Write a MIDX and bitmap; remove the MIDX but leave the bitmap.
|
|
||||||
stale_bitmap=$midx-$(midx_checksum $objdir).bitmap &&
|
|
||||||
rm $midx &&
|
|
||||||
|
|
||||||
# Then write a new MIDX.
|
|
||||||
test_commit new &&
|
|
||||||
git repack &&
|
|
||||||
git multi-pack-index write --bitmap &&
|
|
||||||
|
|
||||||
test_path_is_file $midx &&
|
|
||||||
test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
|
|
||||||
test_path_is_missing $stale_bitmap
|
|
||||||
)
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'pack.preferBitmapTips' '
|
|
||||||
git init repo &&
|
|
||||||
test_when_finished "rm -fr repo" &&
|
|
||||||
(
|
|
||||||
cd repo &&
|
|
||||||
|
|
||||||
test_commit_bulk --message="%s" 103 &&
|
|
||||||
|
|
||||||
git log --format="%H" >commits.raw &&
|
|
||||||
sort <commits.raw >commits &&
|
|
||||||
|
|
||||||
git log --format="create refs/tags/%s %H" HEAD >refs &&
|
|
||||||
git update-ref --stdin <refs &&
|
|
||||||
|
|
||||||
git multi-pack-index write --bitmap &&
|
|
||||||
test_path_is_file $midx &&
|
|
||||||
test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
|
|
||||||
|
|
||||||
test-tool bitmap list-commits | sort >bitmaps &&
|
|
||||||
comm -13 bitmaps commits >before &&
|
|
||||||
test_line_count = 1 before &&
|
|
||||||
|
|
||||||
perl -ne "printf(\"create refs/tags/include/%d \", $.); print" \
|
|
||||||
<before | git update-ref --stdin &&
|
|
||||||
|
|
||||||
rm -fr $midx-$(midx_checksum $objdir).bitmap &&
|
|
||||||
rm -fr $midx &&
|
|
||||||
|
|
||||||
git -c pack.preferBitmapTips=refs/tags/include \
|
|
||||||
multi-pack-index write --bitmap &&
|
|
||||||
test-tool bitmap list-commits | sort >bitmaps &&
|
|
||||||
comm -13 bitmaps commits >after &&
|
|
||||||
|
|
||||||
! test_cmp before after
|
|
||||||
)
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'writing a bitmap with --refs-snapshot' '
|
|
||||||
git init repo &&
|
|
||||||
test_when_finished "rm -fr repo" &&
|
|
||||||
(
|
|
||||||
cd repo &&
|
|
||||||
|
|
||||||
test_commit one &&
|
|
||||||
test_commit two &&
|
|
||||||
|
|
||||||
git rev-parse one >snapshot &&
|
|
||||||
|
|
||||||
git repack -ad &&
|
|
||||||
|
|
||||||
# First, write a MIDX which see both refs/tags/one and
|
|
||||||
# refs/tags/two (causing both of those commits to receive
|
|
||||||
# bitmaps).
|
|
||||||
git multi-pack-index write --bitmap &&
|
|
||||||
|
|
||||||
test_path_is_file $midx &&
|
|
||||||
test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
|
|
||||||
|
|
||||||
test-tool bitmap list-commits | sort >bitmaps &&
|
|
||||||
grep "$(git rev-parse one)" bitmaps &&
|
|
||||||
grep "$(git rev-parse two)" bitmaps &&
|
|
||||||
|
|
||||||
rm -fr $midx-$(midx_checksum $objdir).bitmap &&
|
|
||||||
rm -fr $midx &&
|
|
||||||
|
|
||||||
# Then again, but with a refs snapshot which only sees
|
|
||||||
# refs/tags/one.
|
|
||||||
git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
|
|
||||||
|
|
||||||
test_path_is_file $midx &&
|
|
||||||
test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
|
|
||||||
|
|
||||||
test-tool bitmap list-commits | sort >bitmaps &&
|
|
||||||
grep "$(git rev-parse one)" bitmaps &&
|
|
||||||
! grep "$(git rev-parse two)" bitmaps
|
|
||||||
)
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'write a bitmap with --refs-snapshot (preferred tips)' '
|
|
||||||
git init repo &&
|
|
||||||
test_when_finished "rm -fr repo" &&
|
|
||||||
(
|
|
||||||
cd repo &&
|
|
||||||
|
|
||||||
test_commit_bulk --message="%s" 103 &&
|
|
||||||
|
|
||||||
git log --format="%H" >commits.raw &&
|
|
||||||
sort <commits.raw >commits &&
|
|
||||||
|
|
||||||
git log --format="create refs/tags/%s %H" HEAD >refs &&
|
|
||||||
git update-ref --stdin <refs &&
|
|
||||||
|
|
||||||
git multi-pack-index write --bitmap &&
|
|
||||||
test_path_is_file $midx &&
|
|
||||||
test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
|
|
||||||
|
|
||||||
test-tool bitmap list-commits | sort >bitmaps &&
|
|
||||||
comm -13 bitmaps commits >before &&
|
|
||||||
test_line_count = 1 before &&
|
|
||||||
|
|
||||||
|
test_expect_success 'missing object closure fails gracefully' '
|
||||||
|
rm -fr repo &&
|
||||||
|
git init repo &&
|
||||||
|
test_when_finished "rm -fr repo" &&
|
||||||
(
|
(
|
||||||
grep -vf before commits.raw &&
|
cd repo &&
|
||||||
# mark missing commits as preferred
|
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
|
||||||
sed "s/^/+/" before
|
|
||||||
) >snapshot &&
|
|
||||||
|
|
||||||
rm -fr $midx-$(midx_checksum $objdir).bitmap &&
|
test_commit loose &&
|
||||||
rm -fr $midx &&
|
test_commit packed &&
|
||||||
|
|
||||||
git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
|
# Do not pass "--revs"; we want a pack without the "loose"
|
||||||
test-tool bitmap list-commits | sort >bitmaps &&
|
# commit.
|
||||||
comm -13 bitmaps commits >after &&
|
git pack-objects $objdir/pack/pack <<-EOF &&
|
||||||
|
$(git rev-parse packed)
|
||||||
|
EOF
|
||||||
|
|
||||||
! test_cmp before after
|
test_must_fail git multi-pack-index write --bitmap 2>err &&
|
||||||
)
|
grep "doesn.t have full closure" err &&
|
||||||
'
|
test_path_is_missing $midx
|
||||||
|
)
|
||||||
|
'
|
||||||
|
|
||||||
test_expect_success 'hash-cache values are propagated from pack bitmaps' '
|
midx_bitmap_partial_tests
|
||||||
|
|
||||||
|
test_expect_success 'removing a MIDX clears stale bitmaps' '
|
||||||
|
rm -fr repo &&
|
||||||
|
git init repo &&
|
||||||
|
test_when_finished "rm -fr repo" &&
|
||||||
|
(
|
||||||
|
cd repo &&
|
||||||
|
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
|
||||||
|
test_commit base &&
|
||||||
|
git repack &&
|
||||||
|
git multi-pack-index write --bitmap &&
|
||||||
|
|
||||||
|
# Write a MIDX and bitmap; remove the MIDX but leave the bitmap.
|
||||||
|
stale_bitmap=$midx-$(midx_checksum $objdir).bitmap &&
|
||||||
|
rm $midx &&
|
||||||
|
|
||||||
|
# Then write a new MIDX.
|
||||||
|
test_commit new &&
|
||||||
|
git repack &&
|
||||||
|
git multi-pack-index write --bitmap &&
|
||||||
|
|
||||||
|
test_path_is_file $midx &&
|
||||||
|
test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
|
||||||
|
test_path_is_missing $stale_bitmap
|
||||||
|
)
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'pack.preferBitmapTips' '
|
||||||
|
git init repo &&
|
||||||
|
test_when_finished "rm -fr repo" &&
|
||||||
|
(
|
||||||
|
cd repo &&
|
||||||
|
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
|
||||||
|
|
||||||
|
test_commit_bulk --message="%s" 103 &&
|
||||||
|
|
||||||
|
git log --format="%H" >commits.raw &&
|
||||||
|
sort <commits.raw >commits &&
|
||||||
|
|
||||||
|
git log --format="create refs/tags/%s %H" HEAD >refs &&
|
||||||
|
git update-ref --stdin <refs &&
|
||||||
|
|
||||||
|
git multi-pack-index write --bitmap &&
|
||||||
|
test_path_is_file $midx &&
|
||||||
|
test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
|
||||||
|
|
||||||
|
test-tool bitmap list-commits | sort >bitmaps &&
|
||||||
|
comm -13 bitmaps commits >before &&
|
||||||
|
test_line_count = 1 before &&
|
||||||
|
|
||||||
|
perl -ne "printf(\"create refs/tags/include/%d \", $.); print" \
|
||||||
|
<before | git update-ref --stdin &&
|
||||||
|
|
||||||
|
rm -fr $midx-$(midx_checksum $objdir).bitmap &&
|
||||||
|
rm -fr $midx &&
|
||||||
|
|
||||||
|
git -c pack.preferBitmapTips=refs/tags/include \
|
||||||
|
multi-pack-index write --bitmap &&
|
||||||
|
test-tool bitmap list-commits | sort >bitmaps &&
|
||||||
|
comm -13 bitmaps commits >after &&
|
||||||
|
|
||||||
|
! test_cmp before after
|
||||||
|
)
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'writing a bitmap with --refs-snapshot' '
|
||||||
|
git init repo &&
|
||||||
|
test_when_finished "rm -fr repo" &&
|
||||||
|
(
|
||||||
|
cd repo &&
|
||||||
|
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
|
||||||
|
|
||||||
|
test_commit one &&
|
||||||
|
test_commit two &&
|
||||||
|
|
||||||
|
git rev-parse one >snapshot &&
|
||||||
|
|
||||||
|
git repack -ad &&
|
||||||
|
|
||||||
|
# First, write a MIDX which see both refs/tags/one and
|
||||||
|
# refs/tags/two (causing both of those commits to receive
|
||||||
|
# bitmaps).
|
||||||
|
git multi-pack-index write --bitmap &&
|
||||||
|
|
||||||
|
test_path_is_file $midx &&
|
||||||
|
test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
|
||||||
|
|
||||||
|
test-tool bitmap list-commits | sort >bitmaps &&
|
||||||
|
grep "$(git rev-parse one)" bitmaps &&
|
||||||
|
grep "$(git rev-parse two)" bitmaps &&
|
||||||
|
|
||||||
|
rm -fr $midx-$(midx_checksum $objdir).bitmap &&
|
||||||
|
rm -fr $midx &&
|
||||||
|
|
||||||
|
# Then again, but with a refs snapshot which only sees
|
||||||
|
# refs/tags/one.
|
||||||
|
git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
|
||||||
|
|
||||||
|
test_path_is_file $midx &&
|
||||||
|
test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
|
||||||
|
|
||||||
|
test-tool bitmap list-commits | sort >bitmaps &&
|
||||||
|
grep "$(git rev-parse one)" bitmaps &&
|
||||||
|
! grep "$(git rev-parse two)" bitmaps
|
||||||
|
)
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'write a bitmap with --refs-snapshot (preferred tips)' '
|
||||||
|
git init repo &&
|
||||||
|
test_when_finished "rm -fr repo" &&
|
||||||
|
(
|
||||||
|
cd repo &&
|
||||||
|
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
|
||||||
|
|
||||||
|
test_commit_bulk --message="%s" 103 &&
|
||||||
|
|
||||||
|
git log --format="%H" >commits.raw &&
|
||||||
|
sort <commits.raw >commits &&
|
||||||
|
|
||||||
|
git log --format="create refs/tags/%s %H" HEAD >refs &&
|
||||||
|
git update-ref --stdin <refs &&
|
||||||
|
|
||||||
|
git multi-pack-index write --bitmap &&
|
||||||
|
test_path_is_file $midx &&
|
||||||
|
test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
|
||||||
|
|
||||||
|
test-tool bitmap list-commits | sort >bitmaps &&
|
||||||
|
comm -13 bitmaps commits >before &&
|
||||||
|
test_line_count = 1 before &&
|
||||||
|
|
||||||
|
(
|
||||||
|
grep -vf before commits.raw &&
|
||||||
|
# mark missing commits as preferred
|
||||||
|
sed "s/^/+/" before
|
||||||
|
) >snapshot &&
|
||||||
|
|
||||||
|
rm -fr $midx-$(midx_checksum $objdir).bitmap &&
|
||||||
|
rm -fr $midx &&
|
||||||
|
|
||||||
|
git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
|
||||||
|
test-tool bitmap list-commits | sort >bitmaps &&
|
||||||
|
comm -13 bitmaps commits >after &&
|
||||||
|
|
||||||
|
! test_cmp before after
|
||||||
|
)
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'hash-cache values are propagated from pack bitmaps' '
|
||||||
|
rm -fr repo &&
|
||||||
|
git init repo &&
|
||||||
|
test_when_finished "rm -fr repo" &&
|
||||||
|
(
|
||||||
|
cd repo &&
|
||||||
|
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
|
||||||
|
|
||||||
|
test_commit base &&
|
||||||
|
test_commit base2 &&
|
||||||
|
git repack -adb &&
|
||||||
|
|
||||||
|
test-tool bitmap dump-hashes >pack.raw &&
|
||||||
|
test_file_not_empty pack.raw &&
|
||||||
|
sort pack.raw >pack.hashes &&
|
||||||
|
|
||||||
|
test_commit new &&
|
||||||
|
git repack &&
|
||||||
|
git multi-pack-index write --bitmap &&
|
||||||
|
|
||||||
|
test-tool bitmap dump-hashes >midx.raw &&
|
||||||
|
sort midx.raw >midx.hashes &&
|
||||||
|
|
||||||
|
# ensure that every namehash in the pack bitmap can be found in
|
||||||
|
# the midx bitmap (i.e., that there are no oid-namehash pairs
|
||||||
|
# unique to the pack bitmap).
|
||||||
|
comm -23 pack.hashes midx.hashes >dropped.hashes &&
|
||||||
|
test_must_be_empty dropped.hashes
|
||||||
|
)
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'no .bitmap is written without any objects' '
|
||||||
|
rm -fr repo &&
|
||||||
|
git init repo &&
|
||||||
|
test_when_finished "rm -fr repo" &&
|
||||||
|
(
|
||||||
|
cd repo &&
|
||||||
|
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
|
||||||
|
|
||||||
|
empty="$(git pack-objects $objdir/pack/pack </dev/null)" &&
|
||||||
|
cat >packs <<-EOF &&
|
||||||
|
pack-$empty.idx
|
||||||
|
EOF
|
||||||
|
|
||||||
|
git multi-pack-index write --bitmap --stdin-packs \
|
||||||
|
<packs 2>err &&
|
||||||
|
|
||||||
|
grep "bitmap without any objects" err &&
|
||||||
|
|
||||||
|
test_path_is_file $midx &&
|
||||||
|
test_path_is_missing $midx-$(midx_checksum $objdir).bitmap
|
||||||
|
)
|
||||||
|
'
|
||||||
|
|
||||||
|
test_expect_success 'graceful fallback when missing reverse index' '
|
||||||
|
rm -fr repo &&
|
||||||
|
git init repo &&
|
||||||
|
test_when_finished "rm -fr repo" &&
|
||||||
|
(
|
||||||
|
cd repo &&
|
||||||
|
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
|
||||||
|
|
||||||
|
test_commit base &&
|
||||||
|
|
||||||
|
# write a pack and MIDX bitmap containing base
|
||||||
|
git repack -adb &&
|
||||||
|
git multi-pack-index write --bitmap &&
|
||||||
|
|
||||||
|
GIT_TEST_MIDX_READ_RIDX=0 \
|
||||||
|
git rev-list --use-bitmap-index HEAD 2>err &&
|
||||||
|
! grep "ignoring extra bitmap file" err
|
||||||
|
)
|
||||||
|
'
|
||||||
|
}
|
||||||
|
|
||||||
|
test_midx_bitmap_cases
|
||||||
|
|
||||||
|
test_midx_bitmap_cases "pack.writeBitmapLookupTable"
|
||||||
|
|
||||||
|
test_expect_success 'multi-pack-index write writes lookup table if enabled' '
|
||||||
rm -fr repo &&
|
rm -fr repo &&
|
||||||
git init repo &&
|
git init repo &&
|
||||||
test_when_finished "rm -fr repo" &&
|
test_when_finished "rm -fr repo" &&
|
||||||
(
|
(
|
||||||
cd repo &&
|
cd repo &&
|
||||||
|
|
||||||
test_commit base &&
|
test_commit base &&
|
||||||
test_commit base2 &&
|
git config pack.writeBitmapLookupTable true &&
|
||||||
git repack -adb &&
|
git repack -ad &&
|
||||||
|
GIT_TRACE2_EVENT="$(pwd)/trace" \
|
||||||
test-tool bitmap dump-hashes >pack.raw &&
|
git multi-pack-index write --bitmap &&
|
||||||
test_file_not_empty pack.raw &&
|
grep "\"label\":\"writing_lookup_table\"" trace
|
||||||
sort pack.raw >pack.hashes &&
|
|
||||||
|
|
||||||
test_commit new &&
|
|
||||||
git repack &&
|
|
||||||
git multi-pack-index write --bitmap &&
|
|
||||||
|
|
||||||
test-tool bitmap dump-hashes >midx.raw &&
|
|
||||||
sort midx.raw >midx.hashes &&
|
|
||||||
|
|
||||||
# ensure that every namehash in the pack bitmap can be found in
|
|
||||||
# the midx bitmap (i.e., that there are no oid-namehash pairs
|
|
||||||
# unique to the pack bitmap).
|
|
||||||
comm -23 pack.hashes midx.hashes >dropped.hashes &&
|
|
||||||
test_must_be_empty dropped.hashes
|
|
||||||
)
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'no .bitmap is written without any objects' '
|
|
||||||
rm -fr repo &&
|
|
||||||
git init repo &&
|
|
||||||
test_when_finished "rm -fr repo" &&
|
|
||||||
(
|
|
||||||
cd repo &&
|
|
||||||
|
|
||||||
empty="$(git pack-objects $objdir/pack/pack </dev/null)" &&
|
|
||||||
cat >packs <<-EOF &&
|
|
||||||
pack-$empty.idx
|
|
||||||
EOF
|
|
||||||
|
|
||||||
git multi-pack-index write --bitmap --stdin-packs \
|
|
||||||
<packs 2>err &&
|
|
||||||
|
|
||||||
grep "bitmap without any objects" err &&
|
|
||||||
|
|
||||||
test_path_is_file $midx &&
|
|
||||||
test_path_is_missing $midx-$(midx_checksum $objdir).bitmap
|
|
||||||
)
|
|
||||||
'
|
|
||||||
|
|
||||||
test_expect_success 'graceful fallback when missing reverse index' '
|
|
||||||
rm -fr repo &&
|
|
||||||
git init repo &&
|
|
||||||
test_when_finished "rm -fr repo" &&
|
|
||||||
(
|
|
||||||
cd repo &&
|
|
||||||
|
|
||||||
test_commit base &&
|
|
||||||
|
|
||||||
# write a pack and MIDX bitmap containing base
|
|
||||||
git repack -adb &&
|
|
||||||
git multi-pack-index write --bitmap &&
|
|
||||||
|
|
||||||
GIT_TEST_MIDX_READ_RIDX=0 \
|
|
||||||
git rev-list --use-bitmap-index HEAD 2>err &&
|
|
||||||
! grep "ignoring extra bitmap file" err
|
|
||||||
)
|
)
|
||||||
'
|
'
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,27 @@ GIT_TEST_MIDX_READ_RIDX=0
|
||||||
export GIT_TEST_MIDX_WRITE_REV
|
export GIT_TEST_MIDX_WRITE_REV
|
||||||
export GIT_TEST_MIDX_READ_RIDX
|
export GIT_TEST_MIDX_READ_RIDX
|
||||||
|
|
||||||
midx_bitmap_core rev
|
test_midx_bitmap_rev () {
|
||||||
midx_bitmap_partial_tests rev
|
writeLookupTable=false
|
||||||
|
|
||||||
|
for i in "$@"
|
||||||
|
do
|
||||||
|
case $i in
|
||||||
|
"pack.writeBitmapLookupTable") writeLookupTable=true;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
test_expect_success 'setup bitmap config' '
|
||||||
|
rm -rf * .git &&
|
||||||
|
git init &&
|
||||||
|
git config pack.writeBitmapLookupTable '"$writeLookupTable"'
|
||||||
|
'
|
||||||
|
|
||||||
|
midx_bitmap_core rev
|
||||||
|
midx_bitmap_partial_tests rev
|
||||||
|
}
|
||||||
|
|
||||||
|
test_midx_bitmap_rev
|
||||||
|
test_midx_bitmap_rev "pack.writeBitmapLookupTable"
|
||||||
|
|
||||||
test_done
|
test_done
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue