kernel
/
git
mirror of https://git.kernel.org/pub/scm/git/git.git
1
0
Fork 0
Code Releases Activity

Merge branch 'tb/incremental-midx-part-3.1' into ps/maintenance-geometric 

* tb/incremental-midx-part-3.1: (64 commits)
  builtin/repack.c: clean up unused `#include`s
  repack: move `write_cruft_pack()` out of the builtin
  repack: move `write_filtered_pack()` out of the builtin
  repack: move `pack_kept_objects` to `struct pack_objects_args`
  repack: move `finish_pack_objects_cmd()` out of the builtin
  builtin/repack.c: pass `write_pack_opts` to `finish_pack_objects_cmd()`
  repack: extract `write_pack_opts_is_local()`
  repack: move `find_pack_prefix()` out of the builtin
  builtin/repack.c: use `write_pack_opts` within `write_cruft_pack()`
  builtin/repack.c: introduce `struct write_pack_opts`
  repack: 'write_midx_included_packs' API from the builtin
  builtin/repack.c: inline packs within `write_midx_included_packs()`
  builtin/repack.c: pass `repack_write_midx_opts` to `midx_included_packs`
  builtin/repack.c: inline `remove_redundant_bitmaps()`
  builtin/repack.c: reorder `remove_redundant_bitmaps()`
  repack: keep track of MIDX pack names using existing_packs
  builtin/repack.c: use a string_list for 'midx_pack_names'
  builtin/repack.c: extract opts struct for 'write_midx_included_packs()'
  builtin/repack.c: remove ref snapshotting from builtin
  repack: remove pack_geometry API from the builtin
  ...
main
Junio C Hamano 2025-10-21 11:39:31 -07:00
parent 821f583da6 935ab44a0a
commit 8bca1c5d59
40 changed files with 1858 additions and 1528 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Makefile
View File

@ -1136,6 +1136,12 @@ LIB_OBJS += refs/packed-backend.o
LIB_OBJS += refs/ref-cache.o
LIB_OBJS += refspec.o
LIB_OBJS += remote.o
LIB_OBJS += repack.o
LIB_OBJS += repack-cruft.o
LIB_OBJS += repack-filtered.o
LIB_OBJS += repack-geometry.o
LIB_OBJS += repack-midx.o
LIB_OBJS += repack-promisor.o
LIB_OBJS += replace-object.o
LIB_OBJS += repo-settings.o
LIB_OBJS += repository.o

2
builtin/backfill.c
View File

@ -53,7 +53,7 @@ static void download_batch(struct backfill_context *ctx)
* We likely have a new packfile. Add it to the packed list to
* avoid possible duplicate downloads of the same objects.
*/
reprepare_packed_git(ctx->repo);
odb_reprepare(ctx->repo->objects);
}

static int fill_missing_blobs(const char *path UNUSED,

3
builtin/cat-file.c
View File

@ -852,9 +852,10 @@ static void batch_each_object(struct batch_options *opt,

if (bitmap && !for_each_bitmapped_object(bitmap, &opt->objects_filter,
batch_one_object_bitmapped, &payload)) {
struct packfile_store *packs = the_repository->objects->packfiles;
struct packed_git *pack;

for (pack = get_all_packs(the_repository); pack; pack = pack->next) {
for (pack = packfile_store_get_all_packs(packs); pack; pack = pack->next) {
if (bitmap_index_contains_pack(bitmap, pack) ||
open_pack_index(pack))
continue;

3
builtin/count-objects.c
View File

@ -122,6 +122,7 @@ int cmd_count_objects(int argc,
count_loose, count_cruft, NULL, NULL);

if (verbose) {
struct packfile_store *packs = the_repository->objects->packfiles;
struct packed_git *p;
unsigned long num_pack = 0;
off_t size_pack = 0;
@ -129,7 +130,7 @@ int cmd_count_objects(int argc,
struct strbuf pack_buf = STRBUF_INIT;
struct strbuf garbage_buf = STRBUF_INIT;

for (p = get_all_packs(the_repository); p; p = p->next) {
for (p = packfile_store_get_all_packs(packs); p; p = p->next) {
if (!p->pack_local)
continue;
if (open_pack_index(p))

10
builtin/fast-import.c
View File

@ -897,11 +897,11 @@ static void end_packfile(void)
idx_name = keep_pack(create_index());

/* Register the packfile with core git's machinery. */
new_p = add_packed_git(pack_data->repo, idx_name, strlen(idx_name), 1);
new_p = packfile_store_load_pack(pack_data->repo->objects->packfiles,
idx_name, 1);
if (!new_p)
die("core git rejected index %s", idx_name);
all_packs[pack_id] = new_p;
install_packed_git(the_repository, new_p);
free(idx_name);

/* Print the boundary */
@ -952,6 +952,7 @@ static int store_object(
struct object_id *oidout,
uintmax_t mark)
{
struct packfile_store *packs = the_repository->objects->packfiles;
void *out, *delta;
struct object_entry *e;
unsigned char hdr[96];
@ -975,7 +976,7 @@ static int store_object(
if (e->idx.offset) {
duplicate_count_by_type[type]++;
return 1;
} else if (find_oid_pack(&oid, get_all_packs(the_repository))) {
} else if (find_oid_pack(&oid, packfile_store_get_all_packs(packs))) {
e->type = type;
e->pack_id = MAX_PACK_ID;
e->idx.offset = 1; /* just not zero! */
@ -1092,6 +1093,7 @@ static void truncate_pack(struct hashfile_checkpoint *checkpoint)

static void stream_blob(uintmax_t len, struct object_id *oidout, uintmax_t mark)
{
struct packfile_store *packs = the_repository->objects->packfiles;
size_t in_sz = 64 * 1024, out_sz = 64 * 1024;
unsigned char *in_buf = xmalloc(in_sz);
unsigned char *out_buf = xmalloc(out_sz);
@ -1175,7 +1177,7 @@ static void stream_blob(uintmax_t len, struct object_id *oidout, uintmax_t mark)
duplicate_count_by_type[OBJ_BLOB]++;
truncate_pack(&checkpoint);

} else if (find_oid_pack(&oid, get_all_packs(the_repository))) {
} else if (find_oid_pack(&oid, packfile_store_get_all_packs(packs))) {
e->type = OBJ_BLOB;
e->pack_id = MAX_PACK_ID;
e->idx.offset = 1; /* just not zero! */

11
builtin/fsck.c
View File

@ -867,19 +867,20 @@ static int mark_packed_for_connectivity(const struct object_id *oid,

static int check_pack_rev_indexes(struct repository *r, int show_progress)
{
struct packfile_store *packs = r->objects->packfiles;
struct progress *progress = NULL;
uint32_t pack_count = 0;
int res = 0;

if (show_progress) {
for (struct packed_git *p = get_all_packs(r); p; p = p->next)
for (struct packed_git *p = packfile_store_get_all_packs(packs); p; p = p->next)
pack_count++;
progress = start_delayed_progress(the_repository,
"Verifying reverse pack-indexes", pack_count);
pack_count = 0;
}

for (struct packed_git *p = get_all_packs(r); p; p = p->next) {
for (struct packed_git *p = packfile_store_get_all_packs(packs); p; p = p->next) {
int load_error = load_pack_revindex_from_disk(p);

if (load_error < 0) {
@ -999,6 +1000,8 @@ int cmd_fsck(int argc,
for_each_packed_object(the_repository,
mark_packed_for_connectivity, NULL, 0);
} else {
struct packfile_store *packs = the_repository->objects->packfiles;

odb_prepare_alternates(the_repository->objects);
for (source = the_repository->objects->sources; source; source = source->next)
fsck_source(source);
@ -1009,7 +1012,7 @@ int cmd_fsck(int argc,
struct progress *progress = NULL;

if (show_progress) {
for (p = get_all_packs(the_repository); p;
for (p = packfile_store_get_all_packs(packs); p;
p = p->next) {
if (open_pack_index(p))
continue;
@ -1019,7 +1022,7 @@ int cmd_fsck(int argc,
progress = start_progress(the_repository,
_("Checking objects"), total);
}
for (p = get_all_packs(the_repository); p;
for (p = packfile_store_get_all_packs(packs); p;
p = p->next) {
/* verify gives error messages itself */
if (verify_pack(the_repository,

14
builtin/gc.c
View File

@ -487,9 +487,10 @@ static int too_many_loose_objects(struct gc_config *cfg)
static struct packed_git *find_base_packs(struct string_list *packs,
unsigned long limit)
{
struct packfile_store *packfiles = the_repository->objects->packfiles;
struct packed_git *p, *base = NULL;

for (p = get_all_packs(the_repository); p; p = p->next) {
for (p = packfile_store_get_all_packs(packfiles); p; p = p->next) {
if (!p->pack_local || p->is_cruft)
continue;
if (limit) {
@ -508,13 +509,14 @@ static struct packed_git *find_base_packs(struct string_list *packs,

static int too_many_packs(struct gc_config *cfg)
{
struct packfile_store *packs = the_repository->objects->packfiles;
struct packed_git *p;
int cnt;

if (cfg->gc_auto_pack_limit <= 0)
return 0;

for (cnt = 0, p = get_all_packs(the_repository); p; p = p->next) {
for (cnt = 0, p = packfile_store_get_all_packs(packs); p; p = p->next) {
if (!p->pack_local)
continue;
if (p->pack_keep)
@ -1042,7 +1044,7 @@ int cmd_gc(int argc,
die(FAILED_RUN, "rerere");

report_garbage = report_pack_garbage;
reprepare_packed_git(the_repository);
odb_reprepare(the_repository->objects);
if (pack_garbage.nr > 0) {
close_object_store(the_repository->objects);
clean_pack_garbage();
@ -1423,7 +1425,7 @@ static int incremental_repack_auto_condition(struct gc_config *cfg UNUSED)
if (incremental_repack_auto_limit < 0)
return 1;

for (p = get_packed_git(the_repository);
for (p = packfile_store_get_packs(the_repository->objects->packfiles);
count < incremental_repack_auto_limit && p;
p = p->next) {
if (!p->multi_pack_index)
@ -1491,8 +1493,8 @@ static off_t get_auto_pack_size(void)
struct packed_git *p;
struct repository *r = the_repository;

reprepare_packed_git(r);
for (p = get_all_packs(r); p; p = p->next) {
odb_reprepare(r->objects);
for (p = packfile_store_get_all_packs(r->objects->packfiles); p; p = p->next) {
if (p->pack_size > max_size) {
second_largest_size = max_size;
max_size = p->pack_size;

2
builtin/grep.c
View File

@ -1214,7 +1214,7 @@ int cmd_grep(int argc,
if (recurse_submodules)
repo_read_gitmodules(the_repository, 1);
if (startup_info->have_repository)
(void)get_packed_git(the_repository);
(void)packfile_store_get_packs(the_repository->objects->packfiles);

start_threads(&opt);
} else {

10
builtin/index-pack.c
View File

@ -1640,13 +1640,9 @@ static void final(const char *final_pack_name, const char *curr_pack_name,
rename_tmp_packfile(&final_index_name, curr_index_name, &index_name,
hash, "idx", 1);

if (do_fsck_object) {
struct packed_git *p;
p = add_packed_git(the_repository, final_index_name,
strlen(final_index_name), 0);
if (p)
install_packed_git(the_repository, p);
}
if (do_fsck_object)
packfile_store_load_pack(the_repository->objects->packfiles,
final_index_name, 0);

if (!from_stdin) {
printf("%s\n", hash_to_hex(hash));

32
builtin/pack-objects.c
View File

@ -1748,12 +1748,12 @@ static int want_object_in_pack_mtime(const struct object_id *oid,
}
}

list_for_each(pos, get_packed_git_mru(the_repository)) {
list_for_each(pos, packfile_store_get_packs_mru(the_repository->objects->packfiles)) {
struct packed_git *p = list_entry(pos, struct packed_git, mru);
want = want_object_in_pack_one(p, oid, exclude, found_pack, found_offset, found_mtime);
if (!exclude && want > 0)
list_move(&p->mru,
get_packed_git_mru(the_repository));
packfile_store_get_packs_mru(the_repository->objects->packfiles));
if (want != -1)
return want;
}
@ -3831,6 +3831,7 @@ static int pack_mtime_cmp(const void *_a, const void *_b)

static void read_packs_list_from_stdin(struct rev_info *revs)
{
struct packfile_store *packs = the_repository->objects->packfiles;
struct strbuf buf = STRBUF_INIT;
struct string_list include_packs = STRING_LIST_INIT_DUP;
struct string_list exclude_packs = STRING_LIST_INIT_DUP;
@ -3855,7 +3856,7 @@ static void read_packs_list_from_stdin(struct rev_info *revs)
string_list_sort(&exclude_packs);
string_list_remove_duplicates(&exclude_packs, 0);

for (p = get_all_packs(the_repository); p; p = p->next) {
for (p = packfile_store_get_all_packs(packs); p; p = p->next) {
const char *pack_name = pack_basename(p);

if ((item = string_list_lookup(&include_packs, pack_name)))
@ -4076,6 +4077,7 @@ static void enumerate_cruft_objects(void)

static void enumerate_and_traverse_cruft_objects(struct string_list *fresh_packs)
{
struct packfile_store *packs = the_repository->objects->packfiles;
struct packed_git *p;
struct rev_info revs;
int ret;
@ -4105,7 +4107,7 @@ static void enumerate_and_traverse_cruft_objects(struct string_list *fresh_packs
* Re-mark only the fresh packs as kept so that objects in
* unknown packs do not halt the reachability traversal early.
*/
for (p = get_all_packs(the_repository); p; p = p->next)
for (p = packfile_store_get_all_packs(packs); p; p = p->next)
p->pack_keep_in_core = 0;
mark_pack_kept_in_core(fresh_packs, 1);

@ -4122,6 +4124,7 @@ static void enumerate_and_traverse_cruft_objects(struct string_list *fresh_packs

static void read_cruft_objects(void)
{
struct packfile_store *packs = the_repository->objects->packfiles;
struct strbuf buf = STRBUF_INIT;
struct string_list discard_packs = STRING_LIST_INIT_DUP;
struct string_list fresh_packs = STRING_LIST_INIT_DUP;
@ -4142,7 +4145,7 @@ static void read_cruft_objects(void)
string_list_sort(&discard_packs);
string_list_sort(&fresh_packs);

for (p = get_all_packs(the_repository); p; p = p->next) {
for (p = packfile_store_get_all_packs(packs); p; p = p->next) {
const char *pack_name = pack_basename(p);
struct string_list_item *item;

@ -4390,11 +4393,12 @@ static void add_unreachable_loose_objects(struct rev_info *revs)

static int has_sha1_pack_kept_or_nonlocal(const struct object_id *oid)
{
struct packfile_store *packs = the_repository->objects->packfiles;
static struct packed_git *last_found = (void *)1;
struct packed_git *p;

p = (last_found != (void *)1) ? last_found :
get_all_packs(the_repository);
packfile_store_get_all_packs(packs);

while (p) {
if ((!p->pack_local || p->pack_keep ||
@ -4404,7 +4408,7 @@ static int has_sha1_pack_kept_or_nonlocal(const struct object_id *oid)
return 1;
}
if (p == last_found)
p = get_all_packs(the_repository);
p = packfile_store_get_all_packs(packs);
else
p = p->next;
if (p == last_found)
@ -4436,12 +4440,13 @@ static int loosened_object_can_be_discarded(const struct object_id *oid,

static void loosen_unused_packed_objects(void)
{
struct packfile_store *packs = the_repository->objects->packfiles;
struct packed_git *p;
uint32_t i;
uint32_t loosened_objects_nr = 0;
struct object_id oid;

for (p = get_all_packs(the_repository); p; p = p->next) {
for (p = packfile_store_get_all_packs(packs); p; p = p->next) {
if (!p->pack_local || p->pack_keep || p->pack_keep_in_core)
continue;

@ -4742,12 +4747,13 @@ static void get_object_list(struct rev_info *revs, struct strvec *argv)

static void add_extra_kept_packs(const struct string_list *names)
{
struct packfile_store *packs = the_repository->objects->packfiles;
struct packed_git *p;

if (!names->nr)
return;

for (p = get_all_packs(the_repository); p; p = p->next) {
for (p = packfile_store_get_all_packs(packs); p; p = p->next) {
const char *name = basename(p->pack_name);
int i;

@ -5185,8 +5191,10 @@ int cmd_pack_objects(int argc,

add_extra_kept_packs(&keep_pack_list);
if (ignore_packed_keep_on_disk) {
struct packfile_store *packs = the_repository->objects->packfiles;
struct packed_git *p;
for (p = get_all_packs(the_repository); p; p = p->next)

for (p = packfile_store_get_all_packs(packs); p; p = p->next)
if (p->pack_local && p->pack_keep)
break;
if (!p) /* no keep-able packs found */
@ -5198,8 +5206,10 @@ int cmd_pack_objects(int argc,
* want to unset "local" based on looking at packs, as
* it also covers non-local objects
*/
struct packfile_store *packs = the_repository->objects->packfiles;
struct packed_git *p;
for (p = get_all_packs(the_repository); p; p = p->next) {

for (p = packfile_store_get_all_packs(packs); p; p = p->next) {
if (!p->pack_local) {
have_non_local_packs = 1;
break;

6
builtin/pack-redundant.c
View File

@ -566,7 +566,8 @@ static struct pack_list * add_pack(struct packed_git *p)

static struct pack_list * add_pack_file(const char *filename)
{
struct packed_git *p = get_all_packs(the_repository);
struct packfile_store *packs = the_repository->objects->packfiles;
struct packed_git *p = packfile_store_get_all_packs(packs);

if (strlen(filename) < 40)
die("Bad pack filename: %s", filename);
@ -581,7 +582,8 @@ static struct pack_list * add_pack_file(const char *filename)

static void load_all(void)
{
struct packed_git *p = get_all_packs(the_repository);
struct packfile_store *packs = the_repository->objects->packfiles;
struct packed_git *p = packfile_store_get_all_packs(packs);

while (p) {
add_pack(p);

2
builtin/receive-pack.c
View File

@ -2389,7 +2389,7 @@ static const char *unpack(int err_fd, struct shallow_info *si)
status = finish_command(&child);
if (status)
return "index-pack abnormal exit";
reprepare_packed_git(the_repository);
odb_reprepare(the_repository->objects);
}
return NULL;
}

1349
builtin/repack.c
View File

File diff suppressed because it is too large Load Diff

2
bulk-checkin.c
View File

@ -95,7 +95,7 @@ clear_exit:

strbuf_release(&packname);
/* Make objects we just wrote available to ourselves */
reprepare_packed_git(repo);
odb_reprepare(repo->objects);
}

/*

5
connected.c
View File

@ -72,11 +72,12 @@ int check_connected(oid_iterate_fn fn, void *cb_data,
* Before checking for promisor packs, be sure we have the
* latest pack-files loaded into memory.
*/
reprepare_packed_git(the_repository);
odb_reprepare(the_repository->objects);
do {
struct packfile_store *packs = the_repository->objects->packfiles;
struct packed_git *p;

for (p = get_all_packs(the_repository); p; p = p->next) {
for (p = packfile_store_get_all_packs(packs); p; p = p->next) {
if (!p->pack_promisor)
continue;
if (find_pack_entry_one(oid, p))

4
fetch-pack.c
View File

@ -1983,7 +1983,7 @@ static void update_shallow(struct fetch_pack_args *args,
* remote is shallow, but this is a clone, there are
* no objects in repo to worry about. Accept any
* shallow points that exist in the pack (iow in repo
* after get_pack() and reprepare_packed_git())
* after get_pack() and odb_reprepare())
*/
struct oid_array extra = OID_ARRAY_INIT;
struct object_id *oid = si->shallow->oid;
@ -2108,7 +2108,7 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
&si, pack_lockfiles);
}
reprepare_packed_git(the_repository);
odb_reprepare(the_repository->objects);

if (!args->cloning && args->deepen) {
struct check_connected_options opt = CHECK_CONNECTED_INIT;

5
http-backend.c
View File

@ -603,18 +603,19 @@ static void get_head(struct strbuf *hdr, char *arg UNUSED)
static void get_info_packs(struct strbuf *hdr, char *arg UNUSED)
{
size_t objdirlen = strlen(repo_get_object_directory(the_repository));
struct packfile_store *packs = the_repository->objects->packfiles;
struct strbuf buf = STRBUF_INIT;
struct packed_git *p;
size_t cnt = 0;

select_getanyfile(hdr);
for (p = get_all_packs(the_repository); p; p = p->next) {
for (p = packfile_store_get_all_packs(packs); p; p = p->next) {
if (p->pack_local)
cnt++;
}

strbuf_grow(&buf, cnt * 53 + 2);
for (p = get_all_packs(the_repository); p; p = p->next) {
for (p = packfile_store_get_all_packs(packs); p; p = p->next) {
if (p->pack_local)
strbuf_addf(&buf, "P %s\n", p->pack_name + objdirlen + 6);
}

5
http.c
View File

@ -2416,6 +2416,7 @@ static char *fetch_pack_index(unsigned char *hash, const char *base_url)
static int fetch_and_setup_pack_index(struct packed_git **packs_head,
unsigned char *sha1, const char *base_url)
{
struct packfile_store *packs = the_repository->objects->packfiles;
struct packed_git *new_pack, *p;
char *tmp_idx = NULL;
int ret;
@ -2424,7 +2425,7 @@ static int fetch_and_setup_pack_index(struct packed_git **packs_head,
* If we already have the pack locally, no need to fetch its index or
* even add it to list; we already have all of its objects.
*/
for (p = get_all_packs(the_repository); p; p = p->next) {
for (p = packfile_store_get_all_packs(packs); p; p = p->next) {
if (hasheq(p->hash, sha1, the_repository->hash_algo))
return 0;
}
@ -2549,7 +2550,7 @@ void http_install_packfile(struct packed_git *p,
lst = &((*lst)->next);
*lst = (*lst)->next;

install_packed_git(the_repository, p);
packfile_store_add_pack(the_repository->objects->packfiles, p);
}

struct http_pack_request *new_http_pack_request(

2
http.h
View File

@ -210,7 +210,7 @@ int finish_http_pack_request(struct http_pack_request *preq);
void release_http_pack_request(struct http_pack_request *preq);

/*
* Remove p from the given list, and invoke install_packed_git() on it.
* Remove p from the given list, and invoke packfile_store_add_pack() on it.
*
* This is a convenience function for users that have obtained a list of packs
* from http_get_info_packs() and have chosen a specific pack to fetch.

6
meson.build
View File

@ -462,6 +462,12 @@ libgit_sources = [
'reftable/tree.c',
'reftable/writer.c',
'remote.c',
'repack.c',
'repack-cruft.c',
'repack-filtered.c',
'repack-geometry.c',
'repack-midx.c',
'repack-promisor.c',
'replace-object.c',
'repo-settings.c',
'repository.c',

29
midx.c
View File

@ -93,6 +93,12 @@ static int midx_read_object_offsets(const unsigned char *chunk_start,
return 0;
}

struct multi_pack_index *get_multi_pack_index(struct odb_source *source)
{
packfile_store_prepare(source->odb->packfiles);
return source->midx;
}

static struct multi_pack_index *load_multi_pack_index_one(struct odb_source *source,
const char *midx_name)
{
@ -443,7 +449,6 @@ int prepare_midx_pack(struct multi_pack_index *m,
{
struct repository *r = m->source->odb->repo;
struct strbuf pack_name = STRBUF_INIT;
struct strbuf key = STRBUF_INIT;
struct packed_git *p;

pack_int_id = midx_for_pack(&m, pack_int_id);
@ -455,25 +460,11 @@ int prepare_midx_pack(struct multi_pack_index *m,

strbuf_addf(&pack_name, "%s/pack/%s", m->source->path,
m->pack_names[pack_int_id]);

/* pack_map holds the ".pack" name, but we have the .idx */
strbuf_addbuf(&key, &pack_name);
strbuf_strip_suffix(&key, ".idx");
strbuf_addstr(&key, ".pack");
p = hashmap_get_entry_from_hash(&r->objects->pack_map,
strhash(key.buf), key.buf,
struct packed_git, packmap_ent);
if (!p) {
p = add_packed_git(r, pack_name.buf, pack_name.len,
m->source->local);
if (p) {
install_packed_git(r, p);
list_add_tail(&p->mru, &r->objects->packed_git_mru);
}
}

p = packfile_store_load_pack(r->objects->packfiles,
pack_name.buf, m->source->local);
if (p)
list_add_tail(&p->mru, &r->objects->packfiles->mru);
strbuf_release(&pack_name);
strbuf_release(&key);

if (!p) {
m->packs[pack_int_id] = MIDX_PACK_ERROR;

1
midx.h
View File

@ -94,6 +94,7 @@ void get_midx_chain_filename(struct odb_source *source, struct strbuf *out);
void get_split_midx_filename_ext(struct odb_source *source, struct strbuf *buf,
const unsigned char *hash, const char *ext);

struct multi_pack_index *get_multi_pack_index(struct odb_source *source);
struct multi_pack_index *load_multi_pack_index(struct odb_source *source);
int prepare_midx_pack(struct multi_pack_index *m, uint32_t pack_int_id);
struct packed_git *nth_midxed_pack(struct multi_pack_index *m,

6
object-name.c
View File

@ -213,7 +213,7 @@ static void find_short_packed_object(struct disambiguate_state *ds)
unique_in_midx(m, ds);
}

for (p = get_packed_git(ds->repo); p && !ds->ambiguous;
for (p = packfile_store_get_packs(ds->repo->objects->packfiles); p && !ds->ambiguous;
p = p->next)
unique_in_pack(p, ds);
}
@ -596,7 +596,7 @@ static enum get_oid_result get_short_oid(struct repository *r,
* or migrated from loose to packed.
*/
if (status == MISSING_OBJECT) {
reprepare_packed_git(r);
odb_reprepare(r->objects);
find_short_object_filename(&ds);
find_short_packed_object(&ds);
status = finish_object_disambiguation(&ds, oid);
@ -805,7 +805,7 @@ static void find_abbrev_len_packed(struct min_abbrev_data *mad)
find_abbrev_len_for_midx(m, mad);
}

for (p = get_packed_git(mad->repo); p; p = p->next)
for (p = packfile_store_get_packs(mad->repo->objects->packfiles); p; p = p->next)
find_abbrev_len_for_pack(p, mad);
}


44
odb.c
View File

@ -694,7 +694,7 @@ static int do_oid_object_info_extended(struct object_database *odb,

/* Not a loose object; someone else may have just packed it. */
if (!(flags & OBJECT_INFO_QUICK)) {
reprepare_packed_git(odb->repo);
odb_reprepare(odb->repo->objects);
if (find_pack_entry(odb->repo, real, &e))
break;
}
@ -996,8 +996,7 @@ struct object_database *odb_new(struct repository *repo)

memset(o, 0, sizeof(*o));
o->repo = repo;
INIT_LIST_HEAD(&o->packed_git_mru);
hashmap_init(&o->pack_map, pack_map_entry_cmp, NULL, 0);
o->packfiles = packfile_store_new(o);
pthread_mutex_init(&o->replace_mutex, NULL);
string_list_init_dup(&o->submodule_source_paths);
return o;
@ -1035,19 +1034,34 @@ void odb_clear(struct object_database *o)
free((char *) o->cached_objects[i].value.buf);
FREE_AND_NULL(o->cached_objects);

INIT_LIST_HEAD(&o->packed_git_mru);
close_object_store(o);
packfile_store_free(o->packfiles);
o->packfiles = NULL;

/*
* `close_object_store()` only closes the packfiles, but doesn't free
* them. We thus have to do this manually.
*/
for (struct packed_git *p = o->packed_git, *next; p; p = next) {
next = p->next;
free(p);
}
o->packed_git = NULL;

hashmap_clear(&o->pack_map);
string_list_clear(&o->submodule_source_paths, 0);
}

void odb_reprepare(struct object_database *o)
{
struct odb_source *source;

obj_read_lock();

/*
* Reprepare alt odbs, in case the alternates file was modified
* during the course of this process. This only _adds_ odbs to
* the linked list, so existing odbs will continue to exist for
* the lifetime of the process.
*/
o->loaded_alternates = 0;
odb_prepare_alternates(o);

for (source = o->sources; source; source = source->next)
odb_clear_loose_cache(source);

o->approximate_object_count_valid = 0;

packfile_store_reprepare(o->packfiles);

obj_read_unlock();
}

36
odb.h
View File

@ -3,7 +3,6 @@

#include "hashmap.h"
#include "object.h"
#include "list.h"
#include "oidset.h"
#include "oidmap.h"
#include "string-list.h"
@ -91,6 +90,7 @@ struct odb_source {
};

struct packed_git;
struct packfile_store;
struct cached_object_entry;
struct odb_transaction;

@ -139,20 +139,8 @@ struct object_database {
struct commit_graph *commit_graph;
unsigned commit_graph_attempted : 1; /* if loading has been attempted */

/*
* private data
*
* should only be accessed directly by packfile.c
*/

struct packed_git *packed_git;
/* A most-recently-used ordered version of the packed_git list. */
struct list_head packed_git_mru;

struct {
struct packed_git **packs;
unsigned flags;
} kept_pack_cache;
/* Should only be accessed directly by packfile.c and midx.c. */
struct packfile_store *packfiles;

/*
* This is meant to hold a *small* number of objects that you would
@ -163,12 +151,6 @@ struct object_database {
struct cached_object_entry *cached_objects;
size_t cached_object_nr, cached_object_alloc;

/*
* A map of packfiles to packed_git structs for tracking which
* packs have been loaded already.
*/
struct hashmap pack_map;

/*
* A fast, rough count of the number of objects in the repository.
* These two fields are not meant for direct access. Use
@ -177,12 +159,6 @@ struct object_database {
unsigned long approximate_object_count;
unsigned approximate_object_count_valid : 1;

/*
* Whether packed_git has already been populated with this repository's
* packs.
*/
unsigned packed_git_initialized : 1;

/*
* Submodule source paths that will be added as additional sources to
* allow lookup of submodule objects via the main object database.
@ -193,6 +169,12 @@ struct object_database {
struct object_database *odb_new(struct repository *repo);
void odb_clear(struct object_database *o);

/*
* Clear caches, reload alternates and then reload object sources so that new
* objects may become accessible.
*/
void odb_reprepare(struct object_database *o);

/*
* Find source by its object directory path. Returns a `NULL` pointer in case
* the source could not be found.

4
pack-bitmap.c
View File

@ -664,7 +664,7 @@ static int open_pack_bitmap(struct repository *r,
struct packed_git *p;
int ret = -1;

for (p = get_all_packs(r); p; p = p->next) {
for (p = packfile_store_get_all_packs(r->objects->packfiles); p; p = p->next) {
if (open_pack_bitmap_1(bitmap_git, p) == 0) {
ret = 0;
/*
@ -3362,7 +3362,7 @@ int verify_bitmap_files(struct repository *r)
free(midx_bitmap_name);
}

for (struct packed_git *p = get_all_packs(r);
for (struct packed_git *p = packfile_store_get_all_packs(r->objects->packfiles);
p; p = p->next) {
char *pack_bitmap_name = pack_bitmap_filename(p);
res |= verify_bitmap_file(r->hash_algo, pack_bitmap_name);

3
pack-objects.c
View File

@ -86,6 +86,7 @@ struct object_entry *packlist_find(struct packing_data *pdata,

static void prepare_in_pack_by_idx(struct packing_data *pdata)
{
struct packfile_store *packs = pdata->repo->objects->packfiles;
struct packed_git **mapping, *p;
int cnt = 0, nr = 1U << OE_IN_PACK_BITS;

@ -95,7 +96,7 @@ static void prepare_in_pack_by_idx(struct packing_data *pdata)
* (i.e. in_pack_idx also zero) should return NULL.
*/
mapping[cnt++] = NULL;
for (p = get_all_packs(pdata->repo); p; p = p->next, cnt++) {
for (p = packfile_store_get_all_packs(packs); p; p = p->next, cnt++) {
if (cnt == nr) {
free(mapping);
return;

283
packfile.c
View File

@ -278,7 +278,7 @@ static int unuse_one_window(struct packed_git *current)

if (current)
scan_windows(current, &lru_p, &lru_w, &lru_l);
for (p = current->repo->objects->packed_git; p; p = p->next)
for (p = current->repo->objects->packfiles->packs; p; p = p->next)
scan_windows(p, &lru_p, &lru_w, &lru_l);
if (lru_p) {
munmap(lru_w->base, lru_w->len);
@ -362,13 +362,8 @@ void close_pack(struct packed_git *p)
void close_object_store(struct object_database *o)
{
struct odb_source *source;
struct packed_git *p;

for (p = o->packed_git; p; p = p->next)
if (p->do_not_close)
BUG("want to close pack marked 'do-not-close'");
else
close_pack(p);
packfile_store_close(o->packfiles);

for (source = o->sources; source; source = source->next) {
if (source->midx)
@ -468,7 +463,7 @@ static int close_one_pack(struct repository *r)
struct pack_window *mru_w = NULL;
int accept_windows_inuse = 1;

for (p = r->objects->packed_git; p; p = p->next) {
for (p = r->objects->packfiles->packs; p; p = p->next) {
if (p->pack_fd == -1)
continue;
find_lru_pack(p, &lru_p, &mru_w, &accept_windows_inuse);
@ -784,16 +779,44 @@ struct packed_git *add_packed_git(struct repository *r, const char *path,
return p;
}

void install_packed_git(struct repository *r, struct packed_git *pack)
void packfile_store_add_pack(struct packfile_store *store,
struct packed_git *pack)
{
if (pack->pack_fd != -1)
pack_open_fds++;

pack->next = r->objects->packed_git;
r->objects->packed_git = pack;
pack->next = store->packs;
store->packs = pack;

hashmap_entry_init(&pack->packmap_ent, strhash(pack->pack_name));
hashmap_add(&r->objects->pack_map, &pack->packmap_ent);
hashmap_add(&store->map, &pack->packmap_ent);
}

struct packed_git *packfile_store_load_pack(struct packfile_store *store,
const char *idx_path, int local)
{
struct strbuf key = STRBUF_INIT;
struct packed_git *p;

/*
* We're being called with the path to the index file, but `pack_map`
* holds the path to the packfile itself.
*/
strbuf_addstr(&key, idx_path);
strbuf_strip_suffix(&key, ".idx");
strbuf_addstr(&key, ".pack");

p = hashmap_get_entry_from_hash(&store->map, strhash(key.buf), key.buf,
struct packed_git, packmap_ent);
if (!p) {
p = add_packed_git(store->odb->repo, idx_path,
strlen(idx_path), local);
if (p)
packfile_store_add_pack(store, p);
}

strbuf_release(&key);
return p;
}

void (*report_garbage)(unsigned seen_bits, const char *path);
@ -895,23 +918,14 @@ static void prepare_pack(const char *full_name, size_t full_name_len,
const char *file_name, void *_data)
{
struct prepare_pack_data *data = (struct prepare_pack_data *)_data;
struct packed_git *p;
size_t base_len = full_name_len;

if (strip_suffix_mem(full_name, &base_len, ".idx") &&
!(data->m && midx_contains_pack(data->m, file_name))) {
struct hashmap_entry hent;
char *pack_name = xstrfmt("%.*s.pack", (int)base_len, full_name);
unsigned int hash = strhash(pack_name);
hashmap_entry_init(&hent, hash);

/* Don't reopen a pack we already have. */
if (!hashmap_get(&data->r->objects->pack_map, &hent, pack_name)) {
p = add_packed_git(data->r, full_name, full_name_len, data->local);
if (p)
install_packed_git(data->r, p);
}
free(pack_name);
char *trimmed_path = xstrndup(full_name, full_name_len);
packfile_store_load_pack(data->r->objects->packfiles,
trimmed_path, data->local);
free(trimmed_path);
}

if (!report_garbage)
@ -951,40 +965,6 @@ static void prepare_packed_git_one(struct odb_source *source)
string_list_clear(data.garbage, 0);
}

static void prepare_packed_git(struct repository *r);
/*
* Give a fast, rough count of the number of objects in the repository. This
* ignores loose objects completely. If you have a lot of them, then either
* you should repack because your performance will be awful, or they are
* all unreachable objects about to be pruned, in which case they're not really
* interesting as a measure of repo size in the first place.
*/
unsigned long repo_approximate_object_count(struct repository *r)
{
if (!r->objects->approximate_object_count_valid) {
struct odb_source *source;
unsigned long count = 0;
struct packed_git *p;

prepare_packed_git(r);

for (source = r->objects->sources; source; source = source->next) {
struct multi_pack_index *m = get_multi_pack_index(source);
if (m)
count += m->num_objects;
}

for (p = r->objects->packed_git; p; p = p->next) {
if (open_pack_index(p))
continue;
count += p->num_objects;
}
r->objects->approximate_object_count = count;
r->objects->approximate_object_count_valid = 1;
}
return r->objects->approximate_object_count;
}

DEFINE_LIST_SORT(static, sort_packs, struct packed_git, next);

static int sort_pack(const struct packed_git *a, const struct packed_git *b)
@ -1013,80 +993,51 @@ static int sort_pack(const struct packed_git *a, const struct packed_git *b)
return -1;
}

static void rearrange_packed_git(struct repository *r)
{
sort_packs(&r->objects->packed_git, sort_pack);
}

static void prepare_packed_git_mru(struct repository *r)
static void packfile_store_prepare_mru(struct packfile_store *store)
{
struct packed_git *p;

INIT_LIST_HEAD(&r->objects->packed_git_mru);
INIT_LIST_HEAD(&store->mru);

for (p = r->objects->packed_git; p; p = p->next)
list_add_tail(&p->mru, &r->objects->packed_git_mru);
for (p = store->packs; p; p = p->next)
list_add_tail(&p->mru, &store->mru);
}

static void prepare_packed_git(struct repository *r)
void packfile_store_prepare(struct packfile_store *store)
{
struct odb_source *source;

if (r->objects->packed_git_initialized)
if (store->initialized)
return;

odb_prepare_alternates(r->objects);
for (source = r->objects->sources; source; source = source->next) {
odb_prepare_alternates(store->odb);
for (source = store->odb->sources; source; source = source->next) {
prepare_multi_pack_index_one(source);
prepare_packed_git_one(source);
}
rearrange_packed_git(r);
sort_packs(&store->packs, sort_pack);

prepare_packed_git_mru(r);
r->objects->packed_git_initialized = 1;
packfile_store_prepare_mru(store);
store->initialized = true;
}

void reprepare_packed_git(struct repository *r)
void packfile_store_reprepare(struct packfile_store *store)
{
struct odb_source *source;

obj_read_lock();

/*
* Reprepare alt odbs, in case the alternates file was modified
* during the course of this process. This only _adds_ odbs to
* the linked list, so existing odbs will continue to exist for
* the lifetime of the process.
*/
r->objects->loaded_alternates = 0;
odb_prepare_alternates(r->objects);

for (source = r->objects->sources; source; source = source->next)
odb_clear_loose_cache(source);

r->objects->approximate_object_count_valid = 0;
r->objects->packed_git_initialized = 0;
prepare_packed_git(r);
obj_read_unlock();
store->initialized = false;
packfile_store_prepare(store);
}

struct packed_git *get_packed_git(struct repository *r)
struct packed_git *packfile_store_get_packs(struct packfile_store *store)
{
prepare_packed_git(r);
return r->objects->packed_git;
packfile_store_prepare(store);
return store->packs;
}

struct multi_pack_index *get_multi_pack_index(struct odb_source *source)
struct packed_git *packfile_store_get_all_packs(struct packfile_store *store)
{
prepare_packed_git(source->odb->repo);
return source->midx;
}
packfile_store_prepare(store);

struct packed_git *get_all_packs(struct repository *r)
{
prepare_packed_git(r);

for (struct odb_source *source = r->objects->sources; source; source = source->next) {
for (struct odb_source *source = store->odb->sources; source; source = source->next) {
struct multi_pack_index *m = source->midx;
if (!m)
continue;
@ -1094,13 +1045,46 @@ struct packed_git *get_all_packs(struct repository *r)
prepare_midx_pack(m, i);
}

return r->objects->packed_git;
return store->packs;
}

struct list_head *get_packed_git_mru(struct repository *r)
struct list_head *packfile_store_get_packs_mru(struct packfile_store *store)
{
prepare_packed_git(r);
return &r->objects->packed_git_mru;
packfile_store_prepare(store);
return &store->mru;
}

/*
* Give a fast, rough count of the number of objects in the repository. This
* ignores loose objects completely. If you have a lot of them, then either
* you should repack because your performance will be awful, or they are
* all unreachable objects about to be pruned, in which case they're not really
* interesting as a measure of repo size in the first place.
*/
unsigned long repo_approximate_object_count(struct repository *r)
{
if (!r->objects->approximate_object_count_valid) {
struct odb_source *source;
unsigned long count = 0;
struct packed_git *p;

packfile_store_prepare(r->objects->packfiles);

for (source = r->objects->sources; source; source = source->next) {
struct multi_pack_index *m = get_multi_pack_index(source);
if (m)
count += m->num_objects;
}

for (p = r->objects->packfiles->packs; p; p = p->next) {
if (open_pack_index(p))
continue;
count += p->num_objects;
}
r->objects->approximate_object_count = count;
r->objects->approximate_object_count_valid = 1;
}
return r->objects->approximate_object_count;
}

unsigned long unpack_object_header_buffer(const unsigned char *buf,
@ -1155,7 +1139,7 @@ unsigned long get_size_from_delta(struct packed_git *p,
*
* Other worrying sections could be the call to close_pack_fd(),
* which can close packs even with in-use windows, and to
* reprepare_packed_git(). Regarding the former, mmap doc says:
* odb_reprepare(). Regarding the former, mmap doc says:
* "closing the file descriptor does not unmap the region". And
* for the latter, it won't re-open already available packs.
*/
@ -1219,7 +1203,7 @@ const struct packed_git *has_packed_and_bad(struct repository *r,
{
struct packed_git *p;

for (p = r->objects->packed_git; p; p = p->next)
for (p = r->objects->packfiles->packs; p; p = p->next)
if (oidset_contains(&p->bad_objects, oid))
return p;
return NULL;
@ -2074,19 +2058,19 @@ int find_pack_entry(struct repository *r, const struct object_id *oid, struct pa
{
struct list_head *pos;

prepare_packed_git(r);
packfile_store_prepare(r->objects->packfiles);

for (struct odb_source *source = r->objects->sources; source; source = source->next)
if (source->midx && fill_midx_entry(source->midx, oid, e))
return 1;

if (!r->objects->packed_git)
if (!r->objects->packfiles->packs)
return 0;

list_for_each(pos, &r->objects->packed_git_mru) {
list_for_each(pos, &r->objects->packfiles->mru) {
struct packed_git *p = list_entry(pos, struct packed_git, mru);
if (!p->multi_pack_index && fill_pack_entry(oid, e, p)) {
list_move(&p->mru, &r->objects->packed_git_mru);
list_move(&p->mru, &r->objects->packfiles->mru);
return 1;
}
}
@ -2096,19 +2080,19 @@ int find_pack_entry(struct repository *r, const struct object_id *oid, struct pa
static void maybe_invalidate_kept_pack_cache(struct repository *r,
unsigned flags)
{
if (!r->objects->kept_pack_cache.packs)
if (!r->objects->packfiles->kept_cache.packs)
return;
if (r->objects->kept_pack_cache.flags == flags)
if (r->objects->packfiles->kept_cache.flags == flags)
return;
FREE_AND_NULL(r->objects->kept_pack_cache.packs);
r->objects->kept_pack_cache.flags = 0;
FREE_AND_NULL(r->objects->packfiles->kept_cache.packs);
r->objects->packfiles->kept_cache.flags = 0;
}

struct packed_git **kept_pack_cache(struct repository *r, unsigned flags)
{
maybe_invalidate_kept_pack_cache(r, flags);

if (!r->objects->kept_pack_cache.packs) {
if (!r->objects->packfiles->kept_cache.packs) {
struct packed_git **packs = NULL;
size_t nr = 0, alloc = 0;
struct packed_git *p;
@ -2121,7 +2105,7 @@ struct packed_git **kept_pack_cache(struct repository *r, unsigned flags)
* covers, one kept and one not kept, but the midx returns only
* the non-kept version.
*/
for (p = get_all_packs(r); p; p = p->next) {
for (p = packfile_store_get_all_packs(r->objects->packfiles); p; p = p->next) {
if ((p->pack_keep && (flags & ON_DISK_KEEP_PACKS)) ||
(p->pack_keep_in_core && (flags & IN_CORE_KEEP_PACKS))) {
ALLOC_GROW(packs, nr + 1, alloc);
@ -2131,11 +2115,11 @@ struct packed_git **kept_pack_cache(struct repository *r, unsigned flags)
ALLOC_GROW(packs, nr + 1, alloc);
packs[nr] = NULL;

r->objects->kept_pack_cache.packs = packs;
r->objects->kept_pack_cache.flags = flags;
r->objects->packfiles->kept_cache.packs = packs;
r->objects->packfiles->kept_cache.flags = flags;
}

return r->objects->kept_pack_cache.packs;
return r->objects->packfiles->kept_cache.packs;
}

int find_kept_pack_entry(struct repository *r,
@ -2218,7 +2202,7 @@ int for_each_packed_object(struct repository *repo, each_packed_object_fn cb,
int r = 0;
int pack_errors = 0;

for (p = get_all_packs(repo); p; p = p->next) {
for (p = packfile_store_get_all_packs(repo->objects->packfiles); p; p = p->next) {
if ((flags & FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local)
continue;
if ((flags & FOR_EACH_OBJECT_PROMISOR_ONLY) &&
@ -2332,3 +2316,46 @@ int parse_pack_header_option(const char *in, unsigned char *out, unsigned int *l
*len = hdr - out;
return 0;
}

static int pack_map_entry_cmp(const void *cmp_data UNUSED,
const struct hashmap_entry *entry,
const struct hashmap_entry *entry2,
const void *keydata)
{
const char *key = keydata;
const struct packed_git *pg1, *pg2;

pg1 = container_of(entry, const struct packed_git, packmap_ent);
pg2 = container_of(entry2, const struct packed_git, packmap_ent);

return strcmp(pg1->pack_name, key ? key : pg2->pack_name);
}

struct packfile_store *packfile_store_new(struct object_database *odb)
{
struct packfile_store *store;
CALLOC_ARRAY(store, 1);
store->odb = odb;
INIT_LIST_HEAD(&store->mru);
hashmap_init(&store->map, pack_map_entry_cmp, NULL, 0);
return store;
}

void packfile_store_free(struct packfile_store *store)
{
for (struct packed_git *p = store->packs, *next; p; p = next) {
next = p->next;
free(p);
}
hashmap_clear(&store->map);
free(store);
}

void packfile_store_close(struct packfile_store *store)
{
for (struct packed_git *p = store->packs; p; p = p->next) {
if (p->do_not_close)
BUG("want to close pack marked 'do-not-close'");
close_pack(p);
}
}

125
packfile.h
View File

@ -52,19 +52,114 @@ struct packed_git {
char pack_name[FLEX_ARRAY]; /* more */
};

static inline int pack_map_entry_cmp(const void *cmp_data UNUSED,
const struct hashmap_entry *entry,
const struct hashmap_entry *entry2,
const void *keydata)
{
const char *key = keydata;
const struct packed_git *pg1, *pg2;
/*
* A store that manages packfiles for a given object database.
*/
struct packfile_store {
struct object_database *odb;

pg1 = container_of(entry, const struct packed_git, packmap_ent);
pg2 = container_of(entry2, const struct packed_git, packmap_ent);
/*
* The list of packfiles in the order in which they are being added to
* the store.
*/
struct packed_git *packs;

return strcmp(pg1->pack_name, key ? key : pg2->pack_name);
}
/*
* Cache of packfiles which are marked as "kept", either because there
* is an on-disk ".keep" file or because they are marked as "kept" in
* memory.
*
* Should not be accessed directly, but via `kept_pack_cache()`. The
* list of packs gets invalidated when the stored flags and the flags
* passed to `kept_pack_cache()` mismatch.
*/
struct {
struct packed_git **packs;
unsigned flags;
} kept_cache;

/* A most-recently-used ordered version of the packs list. */
struct list_head mru;

/*
* A map of packfile names to packed_git structs for tracking which
* packs have been loaded already.
*/
struct hashmap map;

/*
* Whether packfiles have already been populated with this store's
* packs.
*/
bool initialized;
};

/*
* Allocate and initialize a new empty packfile store for the given object
* database.
*/
struct packfile_store *packfile_store_new(struct object_database *odb);

/*
* Free the packfile store and all its associated state. All packfiles
* tracked by the store will be closed.
*/
void packfile_store_free(struct packfile_store *store);

/*
* Close all packfiles associated with this store. The packfiles won't be
* free'd, so they can be re-opened at a later point in time.
*/
void packfile_store_close(struct packfile_store *store);

/*
* Prepare the packfile store by loading packfiles and multi-pack indices for
* all alternates. This becomes a no-op if the store is already prepared.
*
* It shouldn't typically be necessary to call this function directly, as
* functions that access the store know to prepare it.
*/
void packfile_store_prepare(struct packfile_store *store);

/*
* Clear the packfile caches and try to look up any new packfiles that have
* appeared since last preparing the packfiles store.
*
* This function must be called under the `odb_read_lock()`.
*/
void packfile_store_reprepare(struct packfile_store *store);

/*
* Add the pack to the store so that contained objects become accessible via
* the store. This moves ownership into the store.
*/
void packfile_store_add_pack(struct packfile_store *store,
struct packed_git *pack);

/*
* Get packs managed by the given store. Does not load the MIDX or any packs
* referenced by it.
*/
struct packed_git *packfile_store_get_packs(struct packfile_store *store);

/*
* Get all packs managed by the given store, including packfiles that are
* referenced by multi-pack indices.
*/
struct packed_git *packfile_store_get_all_packs(struct packfile_store *store);

/*
* Get all packs in most-recently-used order.
*/
struct list_head *packfile_store_get_packs_mru(struct packfile_store *store);

/*
* Open the packfile and add it to the store if it isn't yet known. Returns
* either the newly opened packfile or the preexisting packfile. Returns a
* `NULL` pointer in case the packfile could not be opened.
*/
struct packed_git *packfile_store_load_pack(struct packfile_store *store,
const char *idx_path, int local);

struct pack_window {
struct pack_window *next;
@ -142,14 +237,6 @@ int for_each_packed_object(struct repository *repo, each_packed_object_fn cb,
#define PACKDIR_FILE_GARBAGE 4
extern void (*report_garbage)(unsigned seen_bits, const char *path);

void reprepare_packed_git(struct repository *r);
void install_packed_git(struct repository *r, struct packed_git *pack);

struct packed_git *get_packed_git(struct repository *r);
struct list_head *get_packed_git_mru(struct repository *r);
struct multi_pack_index *get_multi_pack_index(struct odb_source *source);
struct packed_git *get_all_packs(struct repository *r);

/*
* Give a rough count of objects in the repository. This sacrifices accuracy
* for speed.

99
repack-cruft.c Normal file
View File

@ -0,0 +1,99 @@
#include "git-compat-util.h"
#include "repack.h"
#include "packfile.h"
#include "repository.h"
#include "run-command.h"

static void combine_small_cruft_packs(FILE *in, off_t combine_cruft_below_size,
struct existing_packs *existing)
{
struct packfile_store *packs = existing->repo->objects->packfiles;
struct packed_git *p;
struct strbuf buf = STRBUF_INIT;
size_t i;

for (p = packfile_store_get_all_packs(packs); p; p = p->next) {
if (!(p->is_cruft && p->pack_local))
continue;

strbuf_reset(&buf);
strbuf_addstr(&buf, pack_basename(p));
strbuf_strip_suffix(&buf, ".pack");

if (!string_list_has_string(&existing->cruft_packs, buf.buf))
continue;

if (p->pack_size < combine_cruft_below_size) {
fprintf(in, "-%s\n", pack_basename(p));
} else {
existing_packs_retain_cruft(existing, p);
fprintf(in, "%s\n", pack_basename(p));
}
}

for (i = 0; i < existing->non_kept_packs.nr; i++)
fprintf(in, "-%s.pack\n",
existing->non_kept_packs.items[i].string);

strbuf_release(&buf);
}

int write_cruft_pack(const struct write_pack_opts *opts,
const char *cruft_expiration,
unsigned long combine_cruft_below_size,
struct string_list *names,
struct existing_packs *existing)
{
struct child_process cmd = CHILD_PROCESS_INIT;
struct string_list_item *item;
FILE *in;
int ret;
const char *pack_prefix = write_pack_opts_pack_prefix(opts);

prepare_pack_objects(&cmd, opts->po_args, opts->destination);

strvec_push(&cmd.args, "--cruft");
if (cruft_expiration)
strvec_pushf(&cmd.args, "--cruft-expiration=%s",
cruft_expiration);

strvec_push(&cmd.args, "--non-empty");

cmd.in = -1;

ret = start_command(&cmd);
if (ret)
return ret;

/*
* names has a confusing double use: it both provides the list
* of just-written new packs, and accepts the name of the cruft
* pack we are writing.
*
* By the time it is read here, it contains only the pack(s)
* that were just written, which is exactly the set of packs we
* want to consider kept.
*
* If `--expire-to` is given, the double-use served by `names`
* ensures that the pack written to `--expire-to` excludes any
* objects contained in the cruft pack.
*/
in = xfdopen(cmd.in, "w");
for_each_string_list_item(item, names)
fprintf(in, "%s-%s.pack\n", pack_prefix, item->string);
if (combine_cruft_below_size && !cruft_expiration) {
combine_small_cruft_packs(in, combine_cruft_below_size,
existing);
} else {
for_each_string_list_item(item, &existing->non_kept_packs)
fprintf(in, "-%s.pack\n", item->string);
for_each_string_list_item(item, &existing->cruft_packs)
fprintf(in, "-%s.pack\n", item->string);
}
for_each_string_list_item(item, &existing->kept_packs)
fprintf(in, "%s.pack\n", item->string);
fclose(in);

return finish_pack_objects_cmd(existing->repo->hash_algo, opts, &cmd,
names);
}

51
repack-filtered.c Normal file
View File

@ -0,0 +1,51 @@
#include "git-compat-util.h"
#include "repack.h"
#include "repository.h"
#include "run-command.h"
#include "string-list.h"

int write_filtered_pack(const struct write_pack_opts *opts,
struct existing_packs *existing,
struct string_list *names)
{
struct child_process cmd = CHILD_PROCESS_INIT;
struct string_list_item *item;
FILE *in;
int ret;
const char *caret;
const char *pack_prefix = write_pack_opts_pack_prefix(opts);

prepare_pack_objects(&cmd, opts->po_args, opts->destination);

strvec_push(&cmd.args, "--stdin-packs");

for_each_string_list_item(item, &existing->kept_packs)
strvec_pushf(&cmd.args, "--keep-pack=%s", item->string);

cmd.in = -1;

ret = start_command(&cmd);
if (ret)
return ret;

/*
* Here 'names' contains only the pack(s) that were just
* written, which is exactly the packs we want to keep. Also
* 'existing_kept_packs' already contains the packs in
* 'keep_pack_list'.
*/
in = xfdopen(cmd.in, "w");
for_each_string_list_item(item, names)
fprintf(in, "^%s-%s.pack\n", pack_prefix, item->string);
for_each_string_list_item(item, &existing->non_kept_packs)
fprintf(in, "%s.pack\n", item->string);
for_each_string_list_item(item, &existing->cruft_packs)
fprintf(in, "%s.pack\n", item->string);
caret = opts->po_args->pack_kept_objects ? "" : "^";
for_each_string_list_item(item, &existing->kept_packs)
fprintf(in, "%s%s.pack\n", caret, item->string);
fclose(in);

return finish_pack_objects_cmd(existing->repo->hash_algo, opts, &cmd,
names);
}

233
repack-geometry.c Normal file
View File

@ -0,0 +1,233 @@
#define DISABLE_SIGN_COMPARE_WARNINGS

#include "git-compat-util.h"
#include "repack.h"
#include "repository.h"
#include "hex.h"
#include "packfile.h"

static uint32_t pack_geometry_weight(struct packed_git *p)
{
if (open_pack_index(p))
die(_("cannot open index for %s"), p->pack_name);
return p->num_objects;
}

static int pack_geometry_cmp(const void *va, const void *vb)
{
uint32_t aw = pack_geometry_weight(*(struct packed_git **)va),
bw = pack_geometry_weight(*(struct packed_git **)vb);

if (aw < bw)
return -1;
if (aw > bw)
return 1;
return 0;
}

void pack_geometry_init(struct pack_geometry *geometry,
struct existing_packs *existing,
const struct pack_objects_args *args)
{
struct packfile_store *packs = existing->repo->objects->packfiles;
struct packed_git *p;
struct strbuf buf = STRBUF_INIT;

for (p = packfile_store_get_all_packs(packs); p; p = p->next) {
if (args->local && !p->pack_local)
/*
* When asked to only repack local packfiles we skip
* over any packfiles that are borrowed from alternate
* object directories.
*/
continue;

if (!args->pack_kept_objects) {
/*
* Any pack that has its pack_keep bit set will
* appear in existing->kept_packs below, but
* this saves us from doing a more expensive
* check.
*/
if (p->pack_keep)
continue;

/*
* The pack may be kept via the --keep-pack
* option; check 'existing->kept_packs' to
* determine whether to ignore it.
*/
strbuf_reset(&buf);
strbuf_addstr(&buf, pack_basename(p));
strbuf_strip_suffix(&buf, ".pack");

if (string_list_has_string(&existing->kept_packs, buf.buf))
continue;
}
if (p->is_cruft)
continue;

ALLOC_GROW(geometry->pack,
geometry->pack_nr + 1,
geometry->pack_alloc);

geometry->pack[geometry->pack_nr] = p;
geometry->pack_nr++;
}

QSORT(geometry->pack, geometry->pack_nr, pack_geometry_cmp);
strbuf_release(&buf);
}

void pack_geometry_split(struct pack_geometry *geometry)
{
uint32_t i;
uint32_t split;
off_t total_size = 0;

if (!geometry->pack_nr) {
geometry->split = geometry->pack_nr;
return;
}

/*
* First, count the number of packs (in descending order of size) which
* already form a geometric progression.
*/
for (i = geometry->pack_nr - 1; i > 0; i--) {
struct packed_git *ours = geometry->pack[i];
struct packed_git *prev = geometry->pack[i - 1];

if (unsigned_mult_overflows(geometry->split_factor,
pack_geometry_weight(prev)))
die(_("pack %s too large to consider in geometric "
"progression"),
prev->pack_name);

if (pack_geometry_weight(ours) <
geometry->split_factor * pack_geometry_weight(prev))
break;
}

split = i;

if (split) {
/*
* Move the split one to the right, since the top element in the
* last-compared pair can't be in the progression. Only do this
* when we split in the middle of the array (otherwise if we got
* to the end, then the split is in the right place).
*/
split++;
}

/*
* Then, anything to the left of 'split' must be in a new pack. But,
* creating that new pack may cause packs in the heavy half to no longer
* form a geometric progression.
*
* Compute an expected size of the new pack, and then determine how many
* packs in the heavy half need to be joined into it (if any) to restore
* the geometric progression.
*/
for (i = 0; i < split; i++) {
struct packed_git *p = geometry->pack[i];

if (unsigned_add_overflows(total_size, pack_geometry_weight(p)))
die(_("pack %s too large to roll up"), p->pack_name);
total_size += pack_geometry_weight(p);
}
for (i = split; i < geometry->pack_nr; i++) {
struct packed_git *ours = geometry->pack[i];

if (unsigned_mult_overflows(geometry->split_factor,
total_size))
die(_("pack %s too large to roll up"), ours->pack_name);

if (pack_geometry_weight(ours) <
geometry->split_factor * total_size) {
if (unsigned_add_overflows(total_size,
pack_geometry_weight(ours)))
die(_("pack %s too large to roll up"),
ours->pack_name);

split++;
total_size += pack_geometry_weight(ours);
} else
break;
}

geometry->split = split;
}

struct packed_git *pack_geometry_preferred_pack(struct pack_geometry *geometry)
{
uint32_t i;

if (!geometry) {
/*
* No geometry means either an all-into-one repack (in which
* case there is only one pack left and it is the largest) or an
* incremental one.
*
* If repacking incrementally, then we could check the size of
* all packs to determine which should be preferred, but leave
* this for later.
*/
return NULL;
}
if (geometry->split == geometry->pack_nr)
return NULL;

/*
* The preferred pack is the largest pack above the split line. In
* other words, it is the largest pack that does not get rolled up in
* the geometric repack.
*/
for (i = geometry->pack_nr; i > geometry->split; i--)
/*
* A pack that is not local would never be included in a
* multi-pack index. We thus skip over any non-local packs.
*/
if (geometry->pack[i - 1]->pack_local)
return geometry->pack[i - 1];

return NULL;
}

void pack_geometry_remove_redundant(struct pack_geometry *geometry,
struct string_list *names,
struct existing_packs *existing,
const char *packdir)
{
const struct git_hash_algo *algop = existing->repo->hash_algo;
struct strbuf buf = STRBUF_INIT;
uint32_t i;

for (i = 0; i < geometry->split; i++) {
struct packed_git *p = geometry->pack[i];
if (string_list_has_string(names, hash_to_hex_algop(p->hash,
algop)))
continue;

strbuf_reset(&buf);
strbuf_addstr(&buf, pack_basename(p));
strbuf_strip_suffix(&buf, ".pack");

if ((p->pack_keep) ||
(string_list_has_string(&existing->kept_packs, buf.buf)))
continue;

repack_remove_redundant_pack(existing->repo, packdir, buf.buf);
}

strbuf_release(&buf);
}

void pack_geometry_release(struct pack_geometry *geometry)
{
if (!geometry)
return;

free(geometry->pack);
}

372
repack-midx.c Normal file
View File

@ -0,0 +1,372 @@
#include "git-compat-util.h"
#include "repack.h"
#include "hash.h"
#include "hex.h"
#include "odb.h"
#include "oidset.h"
#include "pack-bitmap.h"
#include "refs.h"
#include "run-command.h"
#include "tempfile.h"

struct midx_snapshot_ref_data {
struct repository *repo;
struct tempfile *f;
struct oidset seen;
int preferred;
};

static int midx_snapshot_ref_one(const char *refname UNUSED,
const char *referent UNUSED,
const struct object_id *oid,
int flag UNUSED, void *_data)
{
struct midx_snapshot_ref_data *data = _data;
struct object_id peeled;

if (!peel_iterated_oid(data->repo, oid, &peeled))
oid = &peeled;

if (oidset_insert(&data->seen, oid))
return 0; /* already seen */

if (odb_read_object_info(data->repo->objects, oid, NULL) != OBJ_COMMIT)
return 0;

fprintf(data->f->fp, "%s%s\n", data->preferred ? "+" : "",
oid_to_hex(oid));

return 0;
}

void midx_snapshot_refs(struct repository *repo, struct tempfile *f)
{
struct midx_snapshot_ref_data data;
const struct string_list *preferred = bitmap_preferred_tips(repo);

data.repo = repo;
data.f = f;
data.preferred = 0;
oidset_init(&data.seen, 0);

if (!fdopen_tempfile(f, "w"))
die(_("could not open tempfile %s for writing"),
get_tempfile_path(f));

if (preferred) {
struct string_list_item *item;

data.preferred = 1;
for_each_string_list_item(item, preferred)
refs_for_each_ref_in(get_main_ref_store(repo),
item->string,
midx_snapshot_ref_one, &data);
data.preferred = 0;
}

refs_for_each_ref(get_main_ref_store(repo),
midx_snapshot_ref_one, &data);

if (close_tempfile_gently(f)) {
int save_errno = errno;
delete_tempfile(&f);
errno = save_errno;
die_errno(_("could not close refs snapshot tempfile"));
}

oidset_clear(&data.seen);
}

static int midx_has_unknown_packs(struct string_list *include,
struct pack_geometry *geometry,
struct existing_packs *existing)
{
struct string_list_item *item;

string_list_sort(include);

for_each_string_list_item(item, &existing->midx_packs) {
const char *pack_name = item->string;

/*
* Determine whether or not each MIDX'd pack from the existing
* MIDX (if any) is represented in the new MIDX. For each pack
* in the MIDX, it must either be:
*
* - In the "include" list of packs to be included in the new
* MIDX. Note this function is called before the include
* list is populated with any cruft pack(s).
*
* - Below the geometric split line (if using pack geometry),
* indicating that the pack won't be included in the new
* MIDX, but its contents were rolled up as part of the
* geometric repack.
*
* - In the existing non-kept packs list (if not using pack
* geometry), and marked as non-deleted.
*/
if (string_list_has_string(include, pack_name)) {
continue;
} else if (geometry) {
struct strbuf buf = STRBUF_INIT;
uint32_t j;

for (j = 0; j < geometry->split; j++) {
strbuf_reset(&buf);
strbuf_addstr(&buf, pack_basename(geometry->pack[j]));
strbuf_strip_suffix(&buf, ".pack");
strbuf_addstr(&buf, ".idx");

if (!strcmp(pack_name, buf.buf)) {
strbuf_release(&buf);
break;
}
}

strbuf_release(&buf);

if (j < geometry->split)
continue;
} else {
struct string_list_item *item;

item = string_list_lookup(&existing->non_kept_packs,
pack_name);
if (item && !existing_pack_is_marked_for_deletion(item))
continue;
}

/*
* If we got to this point, the MIDX includes some pack that we
* don't know about.
*/
return 1;
}

return 0;
}

static void midx_included_packs(struct string_list *include,
struct repack_write_midx_opts *opts)
{
struct existing_packs *existing = opts->existing;
struct pack_geometry *geometry = opts->geometry;
struct string_list *names = opts->names;
struct string_list_item *item;
struct strbuf buf = STRBUF_INIT;

for_each_string_list_item(item, &existing->kept_packs) {
strbuf_reset(&buf);
strbuf_addf(&buf, "%s.idx", item->string);
string_list_insert(include, buf.buf);
}

for_each_string_list_item(item, names) {
strbuf_reset(&buf);
strbuf_addf(&buf, "pack-%s.idx", item->string);
string_list_insert(include, buf.buf);
}

if (geometry->split_factor) {
uint32_t i;

for (i = geometry->split; i < geometry->pack_nr; i++) {
struct packed_git *p = geometry->pack[i];

/*
* The multi-pack index never refers to packfiles part
* of an alternate object database, so we skip these.
* While git-multi-pack-index(1) would silently ignore
* them anyway, this allows us to skip executing the
* command completely when we have only non-local
* packfiles.
*/
if (!p->pack_local)
continue;

strbuf_reset(&buf);
strbuf_addstr(&buf, pack_basename(p));
strbuf_strip_suffix(&buf, ".pack");
strbuf_addstr(&buf, ".idx");

string_list_insert(include, buf.buf);
}
} else {
for_each_string_list_item(item, &existing->non_kept_packs) {
if (existing_pack_is_marked_for_deletion(item))
continue;

strbuf_reset(&buf);
strbuf_addf(&buf, "%s.idx", item->string);
string_list_insert(include, buf.buf);
}
}

if (opts->midx_must_contain_cruft ||
midx_has_unknown_packs(include, geometry, existing)) {
/*
* If there are one or more unknown pack(s) present (see
* midx_has_unknown_packs() for what makes a pack
* "unknown") in the MIDX before the repack, keep them
* as they may be required to form a reachability
* closure if the MIDX is bitmapped.
*
* For example, a cruft pack can be required to form a
* reachability closure if the MIDX is bitmapped and one
* or more of the bitmap's selected commits reaches a
* once-cruft object that was later made reachable.
*/
for_each_string_list_item(item, &existing->cruft_packs) {
/*
* When doing a --geometric repack, there is no
* need to check for deleted packs, since we're
* by definition not doing an ALL_INTO_ONE
* repack (hence no packs will be deleted).
* Otherwise we must check for and exclude any
* packs which are enqueued for deletion.
*
* So we could omit the conditional below in the
* --geometric case, but doing so is unnecessary
* since no packs are marked as pending
* deletion (since we only call
* `existing_packs_mark_for_deletion()` when
* doing an all-into-one repack).
*/
if (existing_pack_is_marked_for_deletion(item))
continue;

strbuf_reset(&buf);
strbuf_addf(&buf, "%s.idx", item->string);
string_list_insert(include, buf.buf);
}
} else {
/*
* Modern versions of Git (with the appropriate
* configuration setting) will write new copies of
* once-cruft objects when doing a --geometric repack.
*
* If the MIDX has no cruft pack, new packs written
* during a --geometric repack will not rely on the
* cruft pack to form a reachability closure, so we can
* avoid including them in the MIDX in that case.
*/
;
}

strbuf_release(&buf);
}

static void remove_redundant_bitmaps(struct string_list *include,
const char *packdir)
{
struct strbuf path = STRBUF_INIT;
struct string_list_item *item;
size_t packdir_len;

strbuf_addstr(&path, packdir);
strbuf_addch(&path, '/');
packdir_len = path.len;

/*
* Remove any pack bitmaps corresponding to packs which are now
* included in the MIDX.
*/
for_each_string_list_item(item, include) {
strbuf_addstr(&path, item->string);
strbuf_strip_suffix(&path, ".idx");
strbuf_addstr(&path, ".bitmap");

if (unlink(path.buf) && errno != ENOENT)
warning_errno(_("could not remove stale bitmap: %s"),
path.buf);

strbuf_setlen(&path, packdir_len);
}
strbuf_release(&path);
}

int write_midx_included_packs(struct repack_write_midx_opts *opts)
{
struct child_process cmd = CHILD_PROCESS_INIT;
struct string_list include = STRING_LIST_INIT_DUP;
struct string_list_item *item;
struct packed_git *preferred = pack_geometry_preferred_pack(opts->geometry);
FILE *in;
int ret = 0;

midx_included_packs(&include, opts);
if (!include.nr)
goto done;

cmd.in = -1;
cmd.git_cmd = 1;

strvec_push(&cmd.args, "multi-pack-index");
strvec_pushl(&cmd.args, "write", "--stdin-packs", NULL);

if (opts->show_progress)
strvec_push(&cmd.args, "--progress");
else
strvec_push(&cmd.args, "--no-progress");

if (opts->write_bitmaps)
strvec_push(&cmd.args, "--bitmap");

if (preferred)
strvec_pushf(&cmd.args, "--preferred-pack=%s",
pack_basename(preferred));
else if (opts->names->nr) {
/* The largest pack was repacked, meaning that either
* one or two packs exist depending on whether the
* repository has a cruft pack or not.
*
* Select the non-cruft one as preferred to encourage
* pack-reuse among packs containing reachable objects
* over unreachable ones.
*
* (Note we could write multiple packs here if
* `--max-pack-size` was given, but any one of them
* will suffice, so pick the first one.)
*/
for_each_string_list_item(item, opts->names) {
struct generated_pack *pack = item->util;
if (generated_pack_has_ext(pack, ".mtimes"))
continue;

strvec_pushf(&cmd.args, "--preferred-pack=pack-%s.pack",
item->string);
break;
}
} else {
/*
* No packs were kept, and no packs were written. The
* only thing remaining are .keep packs (unless
* --pack-kept-objects was given).
*
* Set the `--preferred-pack` arbitrarily here.
*/
;
}

if (opts->refs_snapshot)
strvec_pushf(&cmd.args, "--refs-snapshot=%s",
opts->refs_snapshot);

ret = start_command(&cmd);
if (ret)
goto done;

in = xfdopen(cmd.in, "w");
for_each_string_list_item(item, &include)
fprintf(in, "%s\n", item->string);
fclose(in);

ret = finish_command(&cmd);
done:
if (!ret && opts->write_bitmaps)
remove_redundant_bitmaps(&include, opts->packdir);

string_list_clear(&include, 0);

return ret;
}

102
repack-promisor.c Normal file
View File

@ -0,0 +1,102 @@
#include "git-compat-util.h"
#include "repack.h"
#include "hex.h"
#include "pack.h"
#include "packfile.h"
#include "path.h"
#include "repository.h"
#include "run-command.h"

struct write_oid_context {
struct child_process *cmd;
const struct git_hash_algo *algop;
};

/*
* Write oid to the given struct child_process's stdin, starting it first if
* necessary.
*/
static int write_oid(const struct object_id *oid,
struct packed_git *pack UNUSED,
uint32_t pos UNUSED, void *data)
{
struct write_oid_context *ctx = data;
struct child_process *cmd = ctx->cmd;

if (cmd->in == -1) {
if (start_command(cmd))
die(_("could not start pack-objects to repack promisor objects"));
}

if (write_in_full(cmd->in, oid_to_hex(oid), ctx->algop->hexsz) < 0 ||
write_in_full(cmd->in, "\n", 1) < 0)
die(_("failed to feed promisor objects to pack-objects"));
return 0;
}

void repack_promisor_objects(struct repository *repo,
const struct pack_objects_args *args,
struct string_list *names, const char *packtmp)
{
struct write_oid_context ctx;
struct child_process cmd = CHILD_PROCESS_INIT;
FILE *out;
struct strbuf line = STRBUF_INIT;

prepare_pack_objects(&cmd, args, packtmp);
cmd.in = -1;

/*
* NEEDSWORK: Giving pack-objects only the OIDs without any ordering
* hints may result in suboptimal deltas in the resulting pack. See if
* the OIDs can be sent with fake paths such that pack-objects can use a
* {type -> existing pack order} ordering when computing deltas instead
* of a {type -> size} ordering, which may produce better deltas.
*/
ctx.cmd = &cmd;
ctx.algop = repo->hash_algo;
for_each_packed_object(repo, write_oid, &ctx,
FOR_EACH_OBJECT_PROMISOR_ONLY);

if (cmd.in == -1) {
/* No packed objects; cmd was never started */
child_process_clear(&cmd);
return;
}

close(cmd.in);

out = xfdopen(cmd.out, "r");
while (strbuf_getline_lf(&line, out) != EOF) {
struct string_list_item *item;
char *promisor_name;

if (line.len != repo->hash_algo->hexsz)
die(_("repack: Expecting full hex object ID lines only from pack-objects."));
item = string_list_append(names, line.buf);

/*
* pack-objects creates the .pack and .idx files, but not the
* .promisor file. Create the .promisor file, which is empty.
*
* NEEDSWORK: fetch-pack sometimes generates non-empty
* .promisor files containing the ref names and associated
* hashes at the point of generation of the corresponding
* packfile, but this would not preserve their contents. Maybe
* concatenate the contents of all .promisor files instead of
* just creating a new empty file.
*/
promisor_name = mkpathdup("%s-%s.promisor", packtmp,
line.buf);
write_promisor_file(promisor_name, NULL, 0);

item->util = generated_pack_populate(item->string, packtmp);

free(promisor_name);
}

fclose(out);
if (finish_command(&cmd))
die(_("could not finish pack-objects to repack promisor objects"));
strbuf_release(&line);
}

360
repack.c Normal file
View File

@ -0,0 +1,360 @@
#include "git-compat-util.h"
#include "dir.h"
#include "midx.h"
#include "odb.h"
#include "packfile.h"
#include "path.h"
#include "repack.h"
#include "repository.h"
#include "run-command.h"
#include "tempfile.h"

void prepare_pack_objects(struct child_process *cmd,
const struct pack_objects_args *args,
const char *out)
{
strvec_push(&cmd->args, "pack-objects");
if (args->window)
strvec_pushf(&cmd->args, "--window=%s", args->window);
if (args->window_memory)
strvec_pushf(&cmd->args, "--window-memory=%s", args->window_memory);
if (args->depth)
strvec_pushf(&cmd->args, "--depth=%s", args->depth);
if (args->threads)
strvec_pushf(&cmd->args, "--threads=%s", args->threads);
if (args->max_pack_size)
strvec_pushf(&cmd->args, "--max-pack-size=%lu", args->max_pack_size);
if (args->no_reuse_delta)
strvec_pushf(&cmd->args, "--no-reuse-delta");
if (args->no_reuse_object)
strvec_pushf(&cmd->args, "--no-reuse-object");
if (args->name_hash_version)
strvec_pushf(&cmd->args, "--name-hash-version=%d", args->name_hash_version);
if (args->path_walk)
strvec_pushf(&cmd->args, "--path-walk");
if (args->local)
strvec_push(&cmd->args, "--local");
if (args->quiet)
strvec_push(&cmd->args, "--quiet");
if (args->delta_base_offset)
strvec_push(&cmd->args, "--delta-base-offset");
if (!args->pack_kept_objects)
strvec_push(&cmd->args, "--honor-pack-keep");
strvec_push(&cmd->args, out);
cmd->git_cmd = 1;
cmd->out = -1;
}

void pack_objects_args_release(struct pack_objects_args *args)
{
free(args->window);
free(args->window_memory);
free(args->depth);
free(args->threads);
list_objects_filter_release(&args->filter_options);
}

void repack_remove_redundant_pack(struct repository *repo, const char *dir_name,
const char *base_name)
{
struct strbuf buf = STRBUF_INIT;
struct odb_source *source = repo->objects->sources;
struct multi_pack_index *m = get_multi_pack_index(source);
strbuf_addf(&buf, "%s.pack", base_name);
if (m && source->local && midx_contains_pack(m, buf.buf))
clear_midx_file(repo);
strbuf_insertf(&buf, 0, "%s/", dir_name);
unlink_pack_path(buf.buf, 1);
strbuf_release(&buf);
}

const char *write_pack_opts_pack_prefix(const struct write_pack_opts *opts)
{
const char *pack_prefix;
if (!skip_prefix(opts->packtmp, opts->packdir, &pack_prefix))
die(_("pack prefix %s does not begin with objdir %s"),
opts->packtmp, opts->packdir);
if (*pack_prefix == '/')
pack_prefix++;
return pack_prefix;
}

bool write_pack_opts_is_local(const struct write_pack_opts *opts)
{
return starts_with(opts->destination, opts->packdir);
}

int finish_pack_objects_cmd(const struct git_hash_algo *algop,
const struct write_pack_opts *opts,
struct child_process *cmd,
struct string_list *names)
{
FILE *out;
bool local = write_pack_opts_is_local(opts);
struct strbuf line = STRBUF_INIT;

out = xfdopen(cmd->out, "r");
while (strbuf_getline_lf(&line, out) != EOF) {
struct string_list_item *item;

if (line.len != algop->hexsz)
die(_("repack: Expecting full hex object ID lines only "
"from pack-objects."));
/*
* Avoid putting packs written outside of the repository in the
* list of names.
*/
if (local) {
item = string_list_append(names, line.buf);
item->util = generated_pack_populate(line.buf,
opts->packtmp);
}
}
fclose(out);

strbuf_release(&line);

return finish_command(cmd);
}

#define DELETE_PACK 1
#define RETAIN_PACK 2

void existing_packs_collect(struct existing_packs *existing,
const struct string_list *extra_keep)
{
struct packfile_store *packs = existing->repo->objects->packfiles;
struct packed_git *p;
struct strbuf buf = STRBUF_INIT;

for (p = packfile_store_get_all_packs(packs); p; p = p->next) {
size_t i;
const char *base;

if (p->multi_pack_index)
string_list_append(&existing->midx_packs,
pack_basename(p));
if (!p->pack_local)
continue;

base = pack_basename(p);

for (i = 0; i < extra_keep->nr; i++)
if (!fspathcmp(base, extra_keep->items[i].string))
break;

strbuf_reset(&buf);
strbuf_addstr(&buf, base);
strbuf_strip_suffix(&buf, ".pack");

if ((extra_keep->nr > 0 && i < extra_keep->nr) || p->pack_keep)
string_list_append(&existing->kept_packs, buf.buf);
else if (p->is_cruft)
string_list_append(&existing->cruft_packs, buf.buf);
else
string_list_append(&existing->non_kept_packs, buf.buf);
}

string_list_sort(&existing->kept_packs);
string_list_sort(&existing->non_kept_packs);
string_list_sort(&existing->cruft_packs);
string_list_sort(&existing->midx_packs);
strbuf_release(&buf);
}

int existing_packs_has_non_kept(const struct existing_packs *existing)
{
return existing->non_kept_packs.nr || existing->cruft_packs.nr;
}

static void existing_pack_mark_for_deletion(struct string_list_item *item)
{
item->util = (void*)((uintptr_t)item->util | DELETE_PACK);
}

static void existing_pack_unmark_for_deletion(struct string_list_item *item)
{
item->util = (void*)((uintptr_t)item->util & ~DELETE_PACK);
}

int existing_pack_is_marked_for_deletion(struct string_list_item *item)
{
return (uintptr_t)item->util & DELETE_PACK;
}

static void existing_packs_mark_retained(struct string_list_item *item)
{
item->util = (void*)((uintptr_t)item->util | RETAIN_PACK);
}

static int existing_pack_is_retained(struct string_list_item *item)
{
return (uintptr_t)item->util & RETAIN_PACK;
}

static void existing_packs_mark_for_deletion_1(const struct git_hash_algo *algop,
struct string_list *names,
struct string_list *list)
{
struct string_list_item *item;
const size_t hexsz = algop->hexsz;

for_each_string_list_item(item, list) {
char *sha1;
size_t len = strlen(item->string);
if (len < hexsz)
continue;
sha1 = item->string + len - hexsz;

if (existing_pack_is_retained(item)) {
existing_pack_unmark_for_deletion(item);
} else if (!string_list_has_string(names, sha1)) {
/*
* Mark this pack for deletion, which ensures
* that this pack won't be included in a MIDX
* (if `--write-midx` was given) and that we
* will actually delete this pack (if `-d` was
* given).
*/
existing_pack_mark_for_deletion(item);
}
}
}

void existing_packs_retain_cruft(struct existing_packs *existing,
struct packed_git *cruft)
{
struct strbuf buf = STRBUF_INIT;
struct string_list_item *item;

strbuf_addstr(&buf, pack_basename(cruft));
strbuf_strip_suffix(&buf, ".pack");

item = string_list_lookup(&existing->cruft_packs, buf.buf);
if (!item)
BUG("could not find cruft pack '%s'", pack_basename(cruft));

existing_packs_mark_retained(item);
strbuf_release(&buf);
}

void existing_packs_mark_for_deletion(struct existing_packs *existing,
struct string_list *names)

{
const struct git_hash_algo *algop = existing->repo->hash_algo;
existing_packs_mark_for_deletion_1(algop, names,
&existing->non_kept_packs);
existing_packs_mark_for_deletion_1(algop, names,
&existing->cruft_packs);
}

static void remove_redundant_packs_1(struct repository *repo,
struct string_list *packs,
const char *packdir)
{
struct string_list_item *item;
for_each_string_list_item(item, packs) {
if (!existing_pack_is_marked_for_deletion(item))
continue;
repack_remove_redundant_pack(repo, packdir, item->string);
}
}

void existing_packs_remove_redundant(struct existing_packs *existing,
const char *packdir)
{
remove_redundant_packs_1(existing->repo, &existing->non_kept_packs,
packdir);
remove_redundant_packs_1(existing->repo, &existing->cruft_packs,
packdir);
}

void existing_packs_release(struct existing_packs *existing)
{
string_list_clear(&existing->kept_packs, 0);
string_list_clear(&existing->non_kept_packs, 0);
string_list_clear(&existing->cruft_packs, 0);
string_list_clear(&existing->midx_packs, 0);
}

static struct {
const char *name;
unsigned optional:1;
} exts[] = {
{".pack"},
{".rev", 1},
{".mtimes", 1},
{".bitmap", 1},
{".promisor", 1},
{".idx"},
};

struct generated_pack {
struct tempfile *tempfiles[ARRAY_SIZE(exts)];
};

struct generated_pack *generated_pack_populate(const char *name,
const char *packtmp)
{
struct stat statbuf;
struct strbuf path = STRBUF_INIT;
struct generated_pack *pack = xcalloc(1, sizeof(*pack));
size_t i;

for (i = 0; i < ARRAY_SIZE(exts); i++) {
strbuf_reset(&path);
strbuf_addf(&path, "%s-%s%s", packtmp, name, exts[i].name);

if (stat(path.buf, &statbuf))
continue;

pack->tempfiles[i] = register_tempfile(path.buf);
}

strbuf_release(&path);
return pack;
}

int generated_pack_has_ext(const struct generated_pack *pack, const char *ext)
{
size_t i;
for (i = 0; i < ARRAY_SIZE(exts); i++) {
if (strcmp(exts[i].name, ext))
continue;
return !!pack->tempfiles[i];
}
BUG("unknown pack extension: '%s'", ext);
}

void generated_pack_install(struct generated_pack *pack, const char *name,
const char *packdir, const char *packtmp)
{
size_t ext;
for (ext = 0; ext < ARRAY_SIZE(exts); ext++) {
char *fname;

fname = mkpathdup("%s/pack-%s%s", packdir, name,
exts[ext].name);

if (pack->tempfiles[ext]) {
const char *fname_old = get_tempfile_path(pack->tempfiles[ext]);
struct stat statbuffer;

if (!stat(fname_old, &statbuffer)) {
statbuffer.st_mode &= ~(S_IWUSR | S_IWGRP | S_IWOTH);
chmod(fname_old, statbuffer.st_mode);
}

if (rename_tempfile(&pack->tempfiles[ext], fname))
die_errno(_("renaming pack to '%s' failed"),
fname);
} else if (!exts[ext].optional)
die(_("pack-objects did not write a '%s' file for pack %s-%s"),
exts[ext].name, packtmp, name);
else if (unlink(fname) < 0 && errno != ENOENT)
die_errno(_("could not unlink: %s"), fname);

free(fname);
}
}

146
repack.h Normal file
View File

@ -0,0 +1,146 @@
#ifndef REPACK_H
#define REPACK_H

#include "list-objects-filter-options.h"
#include "string-list.h"

struct pack_objects_args {
char *window;
char *window_memory;
char *depth;
char *threads;
unsigned long max_pack_size;
int no_reuse_delta;
int no_reuse_object;
int quiet;
int local;
int name_hash_version;
int path_walk;
int delta_base_offset;
int pack_kept_objects;
struct list_objects_filter_options filter_options;
};

#define PACK_OBJECTS_ARGS_INIT { \
.delta_base_offset = 1, \
.pack_kept_objects = -1, \
}

struct child_process;

void prepare_pack_objects(struct child_process *cmd,
const struct pack_objects_args *args,
const char *out);
void pack_objects_args_release(struct pack_objects_args *args);

void repack_remove_redundant_pack(struct repository *repo, const char *dir_name,
const char *base_name);

struct write_pack_opts {
struct pack_objects_args *po_args;
const char *destination;
const char *packdir;
const char *packtmp;
};

const char *write_pack_opts_pack_prefix(const struct write_pack_opts *opts);
bool write_pack_opts_is_local(const struct write_pack_opts *opts);

int finish_pack_objects_cmd(const struct git_hash_algo *algop,
const struct write_pack_opts *opts,
struct child_process *cmd,
struct string_list *names);

struct repository;
struct packed_git;

struct existing_packs {
struct repository *repo;
struct string_list kept_packs;
struct string_list non_kept_packs;
struct string_list cruft_packs;
struct string_list midx_packs;
};

#define EXISTING_PACKS_INIT { \
.kept_packs = STRING_LIST_INIT_DUP, \
.non_kept_packs = STRING_LIST_INIT_DUP, \
.cruft_packs = STRING_LIST_INIT_DUP, \
}

/*
* Adds all packs hex strings (pack-$HASH) to either packs->non_kept
* or packs->kept based on whether each pack has a corresponding
* .keep file or not. Packs without a .keep file are not to be kept
* if we are going to pack everything into one file.
*/
void existing_packs_collect(struct existing_packs *existing,
const struct string_list *extra_keep);
int existing_packs_has_non_kept(const struct existing_packs *existing);
int existing_pack_is_marked_for_deletion(struct string_list_item *item);
void existing_packs_retain_cruft(struct existing_packs *existing,
struct packed_git *cruft);
void existing_packs_mark_for_deletion(struct existing_packs *existing,
struct string_list *names);
void existing_packs_remove_redundant(struct existing_packs *existing,
const char *packdir);
void existing_packs_release(struct existing_packs *existing);

struct generated_pack;

struct generated_pack *generated_pack_populate(const char *name,
const char *packtmp);
int generated_pack_has_ext(const struct generated_pack *pack, const char *ext);
void generated_pack_install(struct generated_pack *pack, const char *name,
const char *packdir, const char *packtmp);

void repack_promisor_objects(struct repository *repo,
const struct pack_objects_args *args,
struct string_list *names, const char *packtmp);

struct pack_geometry {
struct packed_git **pack;
uint32_t pack_nr, pack_alloc;
uint32_t split;

int split_factor;
};

void pack_geometry_init(struct pack_geometry *geometry,
struct existing_packs *existing,
const struct pack_objects_args *args);
void pack_geometry_split(struct pack_geometry *geometry);
struct packed_git *pack_geometry_preferred_pack(struct pack_geometry *geometry);
void pack_geometry_remove_redundant(struct pack_geometry *geometry,
struct string_list *names,
struct existing_packs *existing,
const char *packdir);
void pack_geometry_release(struct pack_geometry *geometry);

struct tempfile;

struct repack_write_midx_opts {
struct existing_packs *existing;
struct pack_geometry *geometry;
struct string_list *names;
const char *refs_snapshot;
const char *packdir;
int show_progress;
int write_bitmaps;
int midx_must_contain_cruft;
};

void midx_snapshot_refs(struct repository *repo, struct tempfile *f);
int write_midx_included_packs(struct repack_write_midx_opts *opts);

int write_filtered_pack(const struct write_pack_opts *opts,
struct existing_packs *existing,
struct string_list *names);

int write_cruft_pack(const struct write_pack_opts *opts,
const char *cruft_expiration,
unsigned long combine_cruft_below_size,
struct string_list *names,
struct existing_packs *existing);

#endif /* REPACK_H */

3
server-info.c
View File

@ -287,12 +287,13 @@ static int compare_info(const void *a_, const void *b_)

static void init_pack_info(struct repository *r, const char *infofile, int force)
{
struct packfile_store *packs = r->objects->packfiles;
struct packed_git *p;
int stale;
int i;
size_t alloc = 0;

for (p = get_all_packs(r); p; p = p->next) {
for (p = packfile_store_get_all_packs(packs); p; p = p->next) {
/* we ignore things on alternate path since they are
* not available to the pullers in general.
*/

2
t/helper/test-find-pack.c
View File

@ -39,7 +39,7 @@ int cmd__find_pack(int argc, const char **argv)
if (repo_get_oid(the_repository, argv[0], &oid))
die("cannot parse %s as an object name", argv[0]);

for (p = get_all_packs(the_repository); p; p = p->next)
for (p = packfile_store_get_all_packs(the_repository->objects->packfiles); p; p = p->next)
if (find_pack_entry_one(&oid, p)) {
printf("%s\n", p->pack_name);
actual_count++;

2
t/helper/test-pack-mtimes.c
View File

@ -37,7 +37,7 @@ int cmd__pack_mtimes(int argc, const char **argv)
if (argc != 2)
usage(pack_mtimes_usage);

for (p = get_all_packs(the_repository); p; p = p->next) {
for (p = packfile_store_get_all_packs(the_repository->objects->packfiles); p; p = p->next) {
strbuf_addstr(&buf, basename(p->pack_name));
strbuf_strip_suffix(&buf, ".pack");
strbuf_addstr(&buf, ".mtimes");

2
transport-helper.c
View File

@ -450,7 +450,7 @@ static int fetch_with_fetch(struct transport *transport,
}
strbuf_release(&buf);

reprepare_packed_git(the_repository);
odb_reprepare(the_repository->objects);
return 0;
}