kernel
/
git
mirror of https://git.kernel.org/pub/scm/git/git.git
1
0
Fork 0
Code Releases Activity

Merge branch 'ps/odb-pluggable-housekeeping' into jch 

Object database housekeeping in 'git gc' and 'git maintenance' has
been refactored to be pluggable.  The files-backend-specific logic,
including incremental and geometric repacking as well as object
pruning, has been moved out of the command implementation and into the
files object database source, enabling future alternative object
database backends to implement their own housekeeping services.

* ps/odb-pluggable-housekeeping:
  odb: make optimizations pluggable
  builtin/gc: fix signedness issues in ODB-related functionality
  builtin/gc: refactor ODB optimizations to operate on "files" source
  builtin/gc: introduce `odb_optimize_required()`
  builtin/gc: move geometric repacking into `odb_optimize()`
  builtin/gc: introduce object database optimization options
  builtin/gc: inline config values specific to the "files" backend
  builtin/gc: make repack arguments self-contained
  builtin/gc: extract object database optimizations into separate function
  builtin/gc: move worktree and rerere tasks before object optimizations
  odb: run "pre-auto-gc" hook for all maintenance tasks
  t7900: simplify how we check for maintenance tasks
jch
Junio C Hamano 2026-07-24 16:11:57 -07:00
parent 0417aabc00 56f3fc9520
commit e6a9693655
7 changed files with 892 additions and 554 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

534
builtin/gc.c
View File

@ -30,16 +30,11 @@
#include "commit-graph.h"
#include "packfile.h"
#include "object-file.h"
#include "pack.h"
#include "pack-objects.h"
#include "odb.h"
#include "path.h"
#include "reflog.h"
#include "repack.h"
#include "rerere.h"
#include "revision.h"
#include "blob.h"
#include "tree.h"
#include "promisor-remote.h"
#include "refs.h"
#include "remote.h"
#include "exec-cmd.h"
@ -130,22 +125,11 @@ struct gc_config {
unsigned long max_cruft_size;
int aggressive_depth;
int aggressive_window;
int gc_auto_threshold;
int gc_auto_pack_limit;
int detach_auto;
char *gc_log_expire;
char *prune_expire;
char *prune_worktrees_expire;
char *repack_filter;
char *repack_filter_to;
char *repack_expire_to;
unsigned long big_pack_threshold;
unsigned long max_delta_cache_size;
/*
* Remove this member from gc_config once repo_settings is passed
* through the callchain.
*/
size_t delta_base_cache_limit;
};

#define GC_CONFIG_INIT { \
@ -154,14 +138,10 @@ struct gc_config {
.cruft_packs = 1, \
.aggressive_depth = 50, \
.aggressive_window = 250, \
.gc_auto_threshold = 6700, \
.gc_auto_pack_limit = 50, \
.detach_auto = 1, \
.gc_log_expire = xstrdup("1.day.ago"), \
.prune_expire = xstrdup("2.weeks.ago"), \
.prune_worktrees_expire = xstrdup("3.months.ago"), \
.max_delta_cache_size = DEFAULT_DELTA_CACHE_SIZE, \
.delta_base_cache_limit = DEFAULT_DELTA_BASE_CACHE_LIMIT, \
}

static void gc_config_release(struct gc_config *cfg)
@ -169,15 +149,12 @@ static void gc_config_release(struct gc_config *cfg)
free(cfg->gc_log_expire);
free(cfg->prune_expire);
free(cfg->prune_worktrees_expire);
free(cfg->repack_filter);
free(cfg->repack_filter_to);
}

static void gc_config(struct gc_config *cfg)
{
const char *value;
char *owned = NULL;
unsigned long ulongval;

if (!repo_config_get_value(the_repository, "gc.packrefs", &value)) {
if (value && !strcmp(value, "notbare"))
@ -192,8 +169,6 @@ static void gc_config(struct gc_config *cfg)

repo_config_get_int(the_repository, "gc.aggressivewindow", &cfg->aggressive_window);
repo_config_get_int(the_repository, "gc.aggressivedepth", &cfg->aggressive_depth);
repo_config_get_int(the_repository, "gc.auto", &cfg->gc_auto_threshold);
repo_config_get_int(the_repository, "gc.autopacklimit", &cfg->gc_auto_pack_limit);
repo_config_get_bool(the_repository, "gc.autodetach", &cfg->detach_auto);
repo_config_get_bool(the_repository, "gc.cruftpacks", &cfg->cruft_packs);
repo_config_get_ulong(the_repository, "gc.maxcruftsize", &cfg->max_cruft_size);
@ -213,22 +188,6 @@ static void gc_config(struct gc_config *cfg)
cfg->gc_log_expire = owned;
}

repo_config_get_ulong(the_repository, "gc.bigpackthreshold", &cfg->big_pack_threshold);
repo_config_get_ulong(the_repository, "pack.deltacachesize", &cfg->max_delta_cache_size);

if (!repo_config_get_ulong(the_repository, "core.deltabasecachelimit", &ulongval))
cfg->delta_base_cache_limit = ulongval;

if (!repo_config_get_string(the_repository, "gc.repackfilter", &owned)) {
free(cfg->repack_filter);
cfg->repack_filter = owned;
}

if (!repo_config_get_string(the_repository, "gc.repackfilterto", &owned)) {
free(cfg->repack_filter_to);
cfg->repack_filter_to = owned;
}

repo_config(the_repository, git_default_config, NULL);
}

@ -464,255 +423,13 @@ out:
return should_gc;
}

static int too_many_loose_objects(int limit)
{
struct odb_source_files *files = odb_source_files_downcast(the_repository->objects->sources);
/*
* This is weird, but stems from legacy behaviour: the GC auto
* threshold was always essentially interpreted as if it was rounded up
* to the next multiple 256 of, so we retain this behaviour for now.
*/
int auto_threshold = DIV_ROUND_UP(limit, 256) * 256;
unsigned long loose_count;

if (odb_source_count_objects(&files->loose->base, ODB_COUNT_OBJECTS_APPROXIMATE,
&loose_count) < 0)
return 0;

return loose_count > auto_threshold;
}

static struct packed_git *find_base_packs(struct string_list *packs,
unsigned long limit)
{
struct packed_git *p, *base = NULL;

repo_for_each_pack(the_repository, p) {
if (!p->pack_local || p->is_cruft)
continue;
if (limit) {
if (p->pack_size >= limit)
string_list_append(packs, p->pack_name);
} else if (!base || base->pack_size < p->pack_size) {
base = p;
}
}

if (base)
string_list_append(packs, base->pack_name);

return base;
}

static int too_many_packs(struct gc_config *cfg)
{
struct packed_git *p;
int cnt = 0;

if (cfg->gc_auto_pack_limit <= 0)
return 0;

repo_for_each_pack(the_repository, p) {
if (!p->pack_local)
continue;
if (p->pack_keep)
continue;
/*
* Perhaps check the size of the pack and count only
* very small ones here?
*/
cnt++;
}
return cfg->gc_auto_pack_limit < cnt;
}

static uint64_t total_ram(void)
{
#if defined(HAVE_SYSINFO)
struct sysinfo si;

if (!sysinfo(&si)) {
uint64_t total = si.totalram;

if (si.mem_unit > 1)
total *= (uint64_t)si.mem_unit;
return total;
}
#elif defined(HAVE_BSD_SYSCTL) && (defined(HW_MEMSIZE) || defined(HW_PHYSMEM) || defined(HW_PHYSMEM64))
uint64_t physical_memory;
int mib[2];
size_t length;

mib[0] = CTL_HW;
# if defined(HW_MEMSIZE)
mib[1] = HW_MEMSIZE;
# elif defined(HW_PHYSMEM64)
mib[1] = HW_PHYSMEM64;
# else
mib[1] = HW_PHYSMEM;
# endif
length = sizeof(physical_memory);
if (!sysctl(mib, 2, &physical_memory, &length, NULL, 0)) {
if (length == 4) {
uint32_t mem;

if (!sysctl(mib, 2, &mem, &length, NULL, 0))
physical_memory = mem;
}
return physical_memory;
}
#elif defined(GIT_WINDOWS_NATIVE)
MEMORYSTATUSEX memInfo;

memInfo.dwLength = sizeof(MEMORYSTATUSEX);
if (GlobalMemoryStatusEx(&memInfo))
return memInfo.ullTotalPhys;
#endif
return 0;
}

static uint64_t estimate_repack_memory(struct gc_config *cfg,
struct packed_git *pack)
{
unsigned long nr_objects;
size_t os_cache, heap;

if (odb_count_objects(the_repository->objects,
ODB_COUNT_OBJECTS_APPROXIMATE, &nr_objects) < 0)
return 0;

if (!pack || !nr_objects)
return 0;

/*
* First we have to scan through at least one pack.
* Assume enough room in OS file cache to keep the entire pack
* or we may accidentally evict data of other processes from
* the cache.
*/
os_cache = pack->pack_size + pack->index_size;
/* then pack-objects needs lots more for book keeping */
heap = sizeof(struct object_entry) * nr_objects;
/*
* internal rev-list --all --objects takes up some memory too,
* let's say half of it is for blobs
*/
heap += sizeof(struct blob) * nr_objects / 2;
/*
* and the other half is for trees (commits and tags are
* usually insignificant)
*/
heap += sizeof(struct tree) * nr_objects / 2;
/* and then obj_hash[], underestimated in fact */
heap += sizeof(struct object *) * nr_objects;
/* revindex is used also */
heap += (sizeof(off_t) + sizeof(uint32_t)) * nr_objects;
/*
* read_sha1_file() (either at delta calculation phase, or
* writing phase) also fills up the delta base cache
*/
heap += cfg->delta_base_cache_limit;
/* and of course pack-objects has its own delta cache */
heap += cfg->max_delta_cache_size;

return os_cache + heap;
}

static int keep_one_pack(struct string_list_item *item, void *data)
{
struct strvec *args = data;
strvec_pushf(args, "--keep-pack=%s", basename(item->string));
return 0;
}

static void add_repack_all_option(struct gc_config *cfg,
struct string_list *keep_pack,
struct strvec *args)
{
if (cfg->prune_expire && !strcmp(cfg->prune_expire, "now")
&& !(cfg->cruft_packs && cfg->repack_expire_to))
strvec_push(args, "-a");
else if (cfg->cruft_packs) {
strvec_push(args, "--cruft");
if (cfg->prune_expire)
strvec_pushf(args, "--cruft-expiration=%s", cfg->prune_expire);
if (cfg->max_cruft_size)
strvec_pushf(args, "--max-cruft-size=%lu",
cfg->max_cruft_size);
if (cfg->repack_expire_to)
strvec_pushf(args, "--expire-to=%s", cfg->repack_expire_to);
} else {
strvec_push(args, "-A");
if (cfg->prune_expire)
strvec_pushf(args, "--unpack-unreachable=%s", cfg->prune_expire);
}

if (keep_pack)
for_each_string_list(keep_pack, keep_one_pack, args);

if (cfg->repack_filter && *cfg->repack_filter)
strvec_pushf(args, "--filter=%s", cfg->repack_filter);
if (cfg->repack_filter_to && *cfg->repack_filter_to)
strvec_pushf(args, "--filter-to=%s", cfg->repack_filter_to);
}

static void add_repack_incremental_option(struct strvec *args)
{
strvec_push(args, "--no-write-bitmap-index");
}

static int need_to_gc(struct gc_config *cfg, struct strvec *repack_args)
{
/*
* Setting gc.auto to 0 or negative can disable the
* automatic gc.
*/
if (cfg->gc_auto_threshold <= 0)
return 0;

/*
* If there are too many loose objects, but not too many
* packs, we run "repack -d -l". If there are too many packs,
* we run "repack -A -d -l". Otherwise we tell the caller
* there is no need.
*/
if (too_many_packs(cfg)) {
struct string_list keep_pack = STRING_LIST_INIT_NODUP;

if (cfg->big_pack_threshold) {
find_base_packs(&keep_pack, cfg->big_pack_threshold);
if (keep_pack.nr >= cfg->gc_auto_pack_limit) {
cfg->big_pack_threshold = 0;
string_list_clear(&keep_pack, 0);
find_base_packs(&keep_pack, 0);
}
} else {
struct packed_git *p = find_base_packs(&keep_pack, 0);
uint64_t mem_have, mem_want;

mem_have = total_ram();
mem_want = estimate_repack_memory(cfg, p);

/*
* Only allow 1/2 of memory for pack-objects, leave
* the rest for the OS and other processes in the
* system.
*/
if (!mem_have || mem_want < mem_have / 2)
string_list_clear(&keep_pack, 0);
}

add_repack_all_option(cfg, &keep_pack, repack_args);
string_list_clear(&keep_pack, 0);
} else if (too_many_loose_objects(cfg->gc_auto_threshold))
add_repack_incremental_option(repack_args);
else
return 0;

if (run_hooks(the_repository, "pre-auto-gc"))
return 0;
return 1;
}
#define OPTIMIZE_FIELDS_FROM_GC_CONFIG(cfg, aggressive) \
.prune_expire = (cfg)->prune_expire, \
.expire_to = (cfg)->repack_expire_to, \
.cruft_packs = (cfg)->cruft_packs, \
.max_cruft_size = (cfg)->max_cruft_size, \
.window = (aggressive) ? (cfg)->aggressive_window : 0, \
.depth = (aggressive) ? (cfg)->aggressive_depth : 0

/* return NULL on success, else hostname running the gc */
static const char *lock_repo_for_gc(int force, pid_t* ret_pid)
@ -841,6 +558,27 @@ static int gc_foreground_tasks(struct maintenance_run_opts *opts,
return 0;
}

static int maintenance_task_odb(struct maintenance_run_opts *opts,
struct gc_config *cfg,
int keep_largest_pack,
int aggressive)
{
struct odb_optimize_options odb_opts = {
.strategy = ODB_OPTIMIZE_INCREMENTAL,
.keep_largest_pack = keep_largest_pack,
OPTIMIZE_FIELDS_FROM_GC_CONFIG(cfg, aggressive),
};

if (opts->auto_flag)
odb_opts.flags |= ODB_OPTIMIZE_AUTO;
if (!opts->quiet)
odb_opts.flags |= ODB_OPTIMIZE_VERBOSE;
if (aggressive)
odb_opts.flags |= ODB_OPTIMIZE_NO_REUSE_DELTAS;

return odb_optimize(the_repository->objects, &odb_opts);
}

int cmd_gc(int argc,
const char **argv,
const char *prefix,
@ -854,7 +592,6 @@ int cmd_gc(int argc,
int keep_largest_pack = -1;
int skip_foreground_tasks = 0;
timestamp_t dummy;
struct strvec repack_args = STRVEC_INIT;
struct maintenance_run_opts opts = MAINTENANCE_RUN_OPTS_INIT;
struct gc_config cfg = GC_CONFIG_INIT;
const char *prune_expire_sentinel = "sentinel";
@ -894,8 +631,6 @@ int cmd_gc(int argc,
show_usage_with_options_if_asked(argc, argv,
builtin_gc_usage, builtin_gc_options);

strvec_pushl(&repack_args, "repack", "-d", "-l", NULL);

gc_config(&cfg);

if (parse_expiry_date(cfg.gc_log_expire, &gc_log_expire_time))
@ -916,24 +651,20 @@ int cmd_gc(int argc,
if (cfg.prune_expire && parse_expiry_date(cfg.prune_expire, &dummy))
die(_("failed to parse prune expiry value %s"), cfg.prune_expire);

if (aggressive) {
strvec_push(&repack_args, "-f");
if (cfg.aggressive_depth > 0)
strvec_pushf(&repack_args, "--depth=%d", cfg.aggressive_depth);
if (cfg.aggressive_window > 0)
strvec_pushf(&repack_args, "--window=%d", cfg.aggressive_window);
}
if (opts.quiet)
strvec_push(&repack_args, "-q");

if (opts.auto_flag) {
struct odb_optimize_options optimize_opts = {
.strategy = ODB_OPTIMIZE_INCREMENTAL,
OPTIMIZE_FIELDS_FROM_GC_CONFIG(&cfg, 0),
};

if (cfg.detach_auto && opts.detach < 0)
opts.detach = 1;

/*
* Auto-gc should be least intrusive as possible.
*/
if (!need_to_gc(&cfg, &repack_args)) {
if (!odb_optimize_required(the_repository->objects, &optimize_opts) ||
run_hooks(the_repository, "pre-auto-gc")) {
ret = 0;
goto out;
}
@ -945,18 +676,6 @@ int cmd_gc(int argc,
fprintf(stderr, _("Auto packing the repository for optimum performance.\n"));
fprintf(stderr, _("See \"git help gc\" for manual housekeeping.\n"));
}
} else {
struct string_list keep_pack = STRING_LIST_INIT_NODUP;

if (keep_largest_pack != -1) {
if (keep_largest_pack)
find_base_packs(&keep_pack, 0);
} else if (cfg.big_pack_threshold) {
find_base_packs(&keep_pack, cfg.big_pack_threshold);
}

add_repack_all_option(&cfg, &keep_pack, &repack_args);
string_list_clear(&keep_pack, 0);
}

if (opts.detach > 0) {
@ -1012,33 +731,6 @@ int cmd_gc(int argc,
if (opts.detach <= 0 && !skip_foreground_tasks)
gc_foreground_tasks(&opts, &cfg);

if (!the_repository->repository_format_precious_objects) {
struct child_process repack_cmd = CHILD_PROCESS_INIT;

repack_cmd.git_cmd = 1;
repack_cmd.odb_to_close = the_repository->objects;
strvec_pushv(&repack_cmd.args, repack_args.v);
if (run_command(&repack_cmd))
die(FAILED_RUN, repack_args.v[0]);

if (cfg.prune_expire) {
struct child_process prune_cmd = CHILD_PROCESS_INIT;

strvec_pushl(&prune_cmd.args, "prune", "--expire", NULL);
/* run `git prune` even if using cruft packs */
strvec_push(&prune_cmd.args, cfg.prune_expire);
if (opts.quiet)
strvec_push(&prune_cmd.args, "--no-progress");
if (repo_has_promisor_remote(the_repository))
strvec_push(&prune_cmd.args,
"--exclude-promisor-objects");
prune_cmd.git_cmd = 1;

if (run_command(&prune_cmd))
die(FAILED_RUN, prune_cmd.args.v[0]);
}
}

if (cfg.prune_worktrees_expire &&
maintenance_task_worktree_prune(&opts, &cfg))
die(FAILED_RUN, "worktree");
@ -1046,6 +738,9 @@ int cmd_gc(int argc,
if (maintenance_task_rerere_gc(&opts, &cfg))
die(FAILED_RUN, "rerere");

if (maintenance_task_odb(&opts, &cfg, keep_largest_pack, aggressive))
die(NULL);

report_garbage = report_pack_garbage;
odb_reprepare(the_repository->objects);
if (pack_garbage.nr > 0) {
@ -1058,10 +753,6 @@ int cmd_gc(int argc,
!opts.quiet && !daemonized ? COMMIT_GRAPH_WRITE_PROGRESS : 0,
NULL);

if (opts.auto_flag && too_many_loose_objects(cfg.gc_auto_threshold))
warning(_("There are too many unreachable loose objects; "
"run 'git prune' to remove them."));

if (!daemonized) {
char *path = repo_git_path(the_repository, "gc.log");
unlink(path);
@ -1070,7 +761,6 @@ int cmd_gc(int argc,

out:
maintenance_run_opts_release(&opts);
strvec_clear(&repack_args);
gc_config_release(&cfg);
return 0;
}
@ -1273,15 +963,11 @@ static int maintenance_task_gc_background(struct maintenance_run_opts *opts,

static int gc_condition(struct gc_config *cfg)
{
/*
* Note that it's fine to drop the repack arguments here, as we execute
* git-gc(1) as a separate child process anyway. So it knows to compute
* these arguments again.
*/
struct strvec repack_args = STRVEC_INIT;
int ret = need_to_gc(cfg, &repack_args);
strvec_clear(&repack_args);
return ret;
struct odb_optimize_options opts = {
.strategy = ODB_OPTIMIZE_INCREMENTAL,
OPTIMIZE_FIELDS_FROM_GC_CONFIG(cfg, 0),
};
return odb_optimize_required(the_repository->objects, &opts);
}

static int prune_packed(struct maintenance_run_opts *opts)
@ -1570,104 +1256,24 @@ static int maintenance_task_incremental_repack(struct maintenance_run_opts *opts
static int maintenance_task_geometric_repack(struct maintenance_run_opts *opts,
struct gc_config *cfg)
{
struct pack_geometry geometry = {
.split_factor = 2,
struct odb_optimize_options odb_opts = {
.strategy = ODB_OPTIMIZE_GEOMETRIC,
OPTIMIZE_FIELDS_FROM_GC_CONFIG(cfg, 0),
};
struct pack_objects_args po_args = {
.local = 1,
};
struct existing_packs existing_packs = EXISTING_PACKS_INIT;
struct string_list kept_packs = STRING_LIST_INIT_DUP;
struct child_process child = CHILD_PROCESS_INIT;
int ret;

repo_config_get_int(the_repository, "maintenance.geometric-repack.splitFactor",
&geometry.split_factor);
if (!opts->quiet)
odb_opts.flags |= ODB_OPTIMIZE_VERBOSE;

existing_packs.repo = the_repository;
existing_packs_collect(&existing_packs, &kept_packs);
pack_geometry_init(&geometry, &existing_packs, &po_args);
pack_geometry_split(&geometry);

child.git_cmd = 1;
child.odb_to_close = the_repository->objects;

strvec_pushl(&child.args, "repack", "-d", "-l", NULL);
if (geometry.split < geometry.pack_nr)
strvec_pushf(&child.args, "--geometric=%d",
geometry.split_factor);
else
add_repack_all_option(cfg, NULL, &child.args);
if (opts->quiet)
strvec_push(&child.args, "--quiet");
if (the_repository->settings.core_multi_pack_index)
strvec_push(&child.args, "--write-midx");

if (run_command(&child)) {
ret = error(_("failed to perform geometric repack"));
goto out;
}

ret = 0;

out:
existing_packs_release(&existing_packs);
pack_geometry_release(&geometry);
return ret;
return odb_optimize(the_repository->objects, &odb_opts);
}

static int geometric_repack_auto_condition(struct gc_config *cfg UNUSED)
static int geometric_repack_auto_condition(struct gc_config *cfg)
{
struct pack_geometry geometry = {
.split_factor = 2,
struct odb_optimize_options opts = {
.strategy = ODB_OPTIMIZE_GEOMETRIC,
OPTIMIZE_FIELDS_FROM_GC_CONFIG(cfg, 0),
};
struct pack_objects_args po_args = {
.local = 1,
};
struct existing_packs existing_packs = EXISTING_PACKS_INIT;
struct string_list kept_packs = STRING_LIST_INIT_DUP;
int auto_value = 100;
int ret;

repo_config_get_int(the_repository, "maintenance.geometric-repack.auto",
&auto_value);
if (!auto_value)
return 0;
if (auto_value < 0)
return 1;

repo_config_get_int(the_repository, "maintenance.geometric-repack.splitFactor",
&geometry.split_factor);

existing_packs.repo = the_repository;
existing_packs_collect(&existing_packs, &kept_packs);
pack_geometry_init(&geometry, &existing_packs, &po_args);
pack_geometry_split(&geometry);

/*
* When we'd merge at least two packs with one another we always
* perform the repack.
*/
if (geometry.split) {
ret = 1;
goto out;
}

/*
* Otherwise, we estimate the number of loose objects to determine
* whether we want to create a new packfile or not.
*/
if (too_many_loose_objects(auto_value)) {
ret = 1;
goto out;
}

ret = 0;

out:
existing_packs_release(&existing_packs);
pack_geometry_release(&geometry);
return ret;
return odb_optimize_required(the_repository->objects, &opts);
}

typedef int (*maintenance_task_fn)(struct maintenance_run_opts *opts,
@ -1755,11 +1361,18 @@ enum task_phase {
TASK_PHASE_BACKGROUND,
};

enum auto_gc_hook_result {
AUTO_GC_HOOK_UNDECIDED = 0,
AUTO_GC_HOOK_RUN = 1,
AUTO_GC_HOOK_SKIP = 2,
};

static int maybe_run_task(const struct maintenance_task *task,
struct repository *repo,
struct maintenance_run_opts *opts,
struct gc_config *cfg,
enum task_phase phase)
enum task_phase phase,
enum auto_gc_hook_result *auto_gc_hook_result)
{
int foreground = (phase == TASK_PHASE_FOREGROUND);
maintenance_task_fn fn = foreground ? task->foreground : task->background;
@ -1768,9 +1381,19 @@ static int maybe_run_task(const struct maintenance_task *task,

if (!fn)
return 0;
if (opts->auto_flag &&
(!task->auto_condition || !task->auto_condition(cfg)))
return 0;
if (opts->auto_flag) {
if (*auto_gc_hook_result == AUTO_GC_HOOK_SKIP)
return 0;

if (!task->auto_condition || !task->auto_condition(cfg))
return 0;

if (*auto_gc_hook_result == AUTO_GC_HOOK_UNDECIDED)
*auto_gc_hook_result = run_hooks(repo, "pre-auto-gc") ?
AUTO_GC_HOOK_SKIP : AUTO_GC_HOOK_RUN;
if (*auto_gc_hook_result == AUTO_GC_HOOK_SKIP)
return 0;
}

trace2_region_enter(region, task->name, repo);
if (fn(opts, cfg)) {
@ -1789,6 +1412,7 @@ static int maintenance_run_tasks(struct maintenance_run_opts *opts,
struct lock_file lk;
struct repository *r = the_repository;
char *lock_path = xstrfmt("%s/maintenance", r->objects->sources->path);
enum auto_gc_hook_result auto_gc_hook_result = AUTO_GC_HOOK_UNDECIDED;

if (repo_hold_lock_file_for_update(r, &lk, lock_path, LOCK_NO_DEREF) < 0) {
/*
@ -1808,7 +1432,7 @@ static int maintenance_run_tasks(struct maintenance_run_opts *opts,

for (size_t i = 0; i < opts->tasks_nr; i++)
if (maybe_run_task(&tasks[opts->tasks[i]], r, opts, cfg,
TASK_PHASE_FOREGROUND))
TASK_PHASE_FOREGROUND, &auto_gc_hook_result))
result = 1;

/* Failure to daemonize is ok, we'll continue in foreground. */
@ -1820,7 +1444,7 @@ static int maintenance_run_tasks(struct maintenance_run_opts *opts,

for (size_t i = 0; i < opts->tasks_nr; i++)
if (maybe_run_task(&tasks[opts->tasks[i]], r, opts, cfg,
TASK_PHASE_BACKGROUND))
TASK_PHASE_BACKGROUND, &auto_gc_hook_result))
result = 1;

rollback_lock_file(&lk);

12
odb.c
View File

@ -1034,6 +1034,18 @@ int odb_write_object_stream(struct object_database *odb,
return odb_source_write_object_stream(odb->sources, stream, len, oid);
}

int odb_optimize(struct object_database *odb,
const struct odb_optimize_options *opts)
{
return odb_source_optimize(odb->sources, opts);
}

bool odb_optimize_required(struct object_database *odb,
const struct odb_optimize_options *opts)
{
return odb_source_optimize_required(odb->sources, opts);
}

struct object_database *odb_new(struct repository *repo,
const char *primary_source,
const char *secondary_sources)

45
odb.h
View File

@ -118,6 +118,51 @@ struct object_database *odb_new(struct repository *repo,
/* Free the object database and release all resources. */
void odb_free(struct object_database *o);

enum odb_optimize_strategy {
ODB_OPTIMIZE_INCREMENTAL,
ODB_OPTIMIZE_GEOMETRIC,
};

enum odb_optimize_flags {
/* Enable verbose logging and progress reporting. */
ODB_OPTIMIZE_VERBOSE = (1 << 0),

/* Perform auto-maintenance, only optimizing objects as required. */
ODB_OPTIMIZE_AUTO = (1 << 1),

/* Recompute existing deltas. */
ODB_OPTIMIZE_NO_REUSE_DELTAS = (1 << 2),
};

struct odb_optimize_options {
enum odb_optimize_strategy strategy;
enum odb_optimize_flags flags;
const char *prune_expire;
const char *expire_to;
int depth;
int window;

/* Backend-specific options. */
int keep_largest_pack;
int cruft_packs;
unsigned long max_cruft_size;
};

/*
* Optimize the object database. Returns 0 on success, a negative error code
* otherwise.
*/
int odb_optimize(struct object_database *odb,
const struct odb_optimize_options *opts);

/*
* Check whether optimization of the object database is required given the
* provided options. Returns true if optimization should be performed, false
* otherwise.
*/
bool odb_optimize_required(struct object_database *odb,
const struct odb_optimize_options *opts);

/*
* Close the object database and all of its sources so that any held resources
* will be released. The database can still be used after closing it, in which

470
odb/source-files.c
View File

@ -1,6 +1,8 @@
#include "git-compat-util.h"
#include "abspath.h"
#include "blob.h"
#include "chdir-notify.h"
#include "config.h"
#include "gettext.h"
#include "lockfile.h"
#include "object-file.h"
@ -8,8 +10,16 @@
#include "odb/source.h"
#include "odb/source-files.h"
#include "odb/source-loose.h"
#include "pack-objects.h"
#include "packfile.h"
#include "path.h"
#include "promisor-remote.h"
#include "repack.h"
#include "run-command.h"
#include "strbuf.h"
#include "string-list.h"
#include "strvec.h"
#include "tree.h"
#include "write-or-die.h"

static void odb_source_files_reparent(const char *name UNUSED,
@ -261,6 +271,464 @@ out:
return ret;
}

static int too_many_loose_objects(struct odb_source_files *files, int limit)
{
unsigned long loose_count;

if (limit <= 0)
return 0;

if (odb_source_count_objects(&files->loose->base, ODB_COUNT_OBJECTS_APPROXIMATE,
&loose_count) < 0)
return 0;

/*
* This is weird, but stems from legacy behaviour: the GC auto
* threshold was always essentially interpreted as if it was rounded up
* to the next multiple 256 of, so we retain this behaviour for now.
*/
return loose_count > (DIV_ROUND_UP(((unsigned long) limit), 256) * 256);
}

static struct packed_git *find_base_packs(struct odb_source_files *files,
struct string_list *packs,
unsigned long limit)
{
struct packfile_list_entry *e;
struct packed_git *base = NULL;

for (e = packfile_store_get_packs(files->packed); e; e = e->next) {
if (e->pack->is_cruft)
continue;
if (limit) {
if ((uintmax_t) e->pack->pack_size >= limit)
string_list_append(packs, e->pack->pack_name);
} else if (!base || base->pack_size < e->pack->pack_size) {
base = e->pack;
}
}

if (base)
string_list_append(packs, base->pack_name);

return base;
}

static int too_many_packs(struct odb_source_files *files, int gc_auto_pack_limit)
{
struct packfile_list_entry *e;
int cnt = 0;

if (gc_auto_pack_limit <= 0)
return 0;

for (e = packfile_store_get_packs(files->packed); e; e = e->next) {
if (e->pack->pack_keep)
continue;
/*
* Perhaps check the size of the pack and count only
* very small ones here?
*/
cnt++;
}
return gc_auto_pack_limit < cnt;
}

static uint64_t total_ram(void)
{
#if defined(HAVE_SYSINFO)
struct sysinfo si;

if (!sysinfo(&si)) {
uint64_t total = si.totalram;

if (si.mem_unit > 1)
total *= (uint64_t)si.mem_unit;
return total;
}
#elif defined(HAVE_BSD_SYSCTL) && (defined(HW_MEMSIZE) || defined(HW_PHYSMEM) || defined(HW_PHYSMEM64))
uint64_t physical_memory;
int mib[2];
size_t length;

mib[0] = CTL_HW;
# if defined(HW_MEMSIZE)
mib[1] = HW_MEMSIZE;
# elif defined(HW_PHYSMEM64)
mib[1] = HW_PHYSMEM64;
# else
mib[1] = HW_PHYSMEM;
# endif
length = sizeof(physical_memory);
if (!sysctl(mib, 2, &physical_memory, &length, NULL, 0)) {
if (length == 4) {
uint32_t mem;

if (!sysctl(mib, 2, &mem, &length, NULL, 0))
physical_memory = mem;
}
return physical_memory;
}
#elif defined(GIT_WINDOWS_NATIVE)
MEMORYSTATUSEX memInfo;

memInfo.dwLength = sizeof(MEMORYSTATUSEX);
if (GlobalMemoryStatusEx(&memInfo))
return memInfo.ullTotalPhys;
#endif
return 0;
}

static uint64_t estimate_repack_memory(struct odb_source_files *files,
struct packed_git *pack)
{
unsigned long max_delta_cache_size = DEFAULT_DELTA_CACHE_SIZE;
unsigned long delta_base_cache_limit = DEFAULT_DELTA_BASE_CACHE_LIMIT;
unsigned long nr_objects;
size_t os_cache, heap;

if (odb_source_count_objects(&files->base, ODB_COUNT_OBJECTS_APPROXIMATE,
&nr_objects) < 0)
return 0;

if (!pack || !nr_objects)
return 0;

repo_config_get_ulong(files->base.odb->repo, "pack.deltacachesize",
&max_delta_cache_size);
repo_config_get_ulong(files->base.odb->repo, "core.deltabasecachelimit",
&delta_base_cache_limit);

/*
* First we have to scan through at least one pack.
* Assume enough room in OS file cache to keep the entire pack
* or we may accidentally evict data of other processes from
* the cache.
*/
os_cache = pack->pack_size + pack->index_size;
/* then pack-objects needs lots more for book keeping */
heap = sizeof(struct object_entry) * nr_objects;
/*
* internal rev-list --all --objects takes up some memory too,
* let's say half of it is for blobs
*/
heap += sizeof(struct blob) * nr_objects / 2;
/*
* and the other half is for trees (commits and tags are
* usually insignificant)
*/
heap += sizeof(struct tree) * nr_objects / 2;
/* and then obj_hash[], underestimated in fact */
heap += sizeof(struct object *) * nr_objects;
/* revindex is used also */
heap += (sizeof(off_t) + sizeof(uint32_t)) * nr_objects;
/*
* read_sha1_file() (either at delta calculation phase, or
* writing phase) also fills up the delta base cache
*/
heap += delta_base_cache_limit;
/* and of course pack-objects has its own delta cache */
heap += max_delta_cache_size;

return os_cache + heap;
}

static int keep_one_pack(struct string_list_item *item, void *data)
{
struct strvec *args = data;
strvec_pushf(args, "--keep-pack=%s", basename(item->string));
return 0;
}

static void add_repack_all_option(struct repository *repo,
const struct odb_optimize_options *opts,
struct string_list *keep_pack,
struct strvec *args)
{
char *repack_filter = NULL;
char *repack_filter_to = NULL;

repo_config_get_string(repo, "gc.repackfilter", &repack_filter);
repo_config_get_string(repo, "gc.repackfilterto", &repack_filter_to);

if (opts->prune_expire && !strcmp(opts->prune_expire, "now") &&
!(opts->cruft_packs && opts->expire_to))
strvec_push(args, "-a");
else if (opts->cruft_packs) {
strvec_push(args, "--cruft");
if (opts->prune_expire)
strvec_pushf(args, "--cruft-expiration=%s", opts->prune_expire);
if (opts->max_cruft_size)
strvec_pushf(args, "--max-cruft-size=%lu",
opts->max_cruft_size);
if (opts->expire_to)
strvec_pushf(args, "--expire-to=%s", opts->expire_to);
} else {
strvec_push(args, "-A");
if (opts->prune_expire)
strvec_pushf(args, "--unpack-unreachable=%s", opts->prune_expire);
}

if (keep_pack)
for_each_string_list(keep_pack, keep_one_pack, args);

if (repack_filter && *repack_filter)
strvec_pushf(args, "--filter=%s", repack_filter);
if (repack_filter_to && *repack_filter_to)
strvec_pushf(args, "--filter-to=%s", repack_filter_to);

free(repack_filter);
free(repack_filter_to);
}

static void add_repack_incremental_option(struct strvec *args)
{
strvec_push(args, "--no-write-bitmap-index");
}

bool odb_source_files_optimize_required(struct odb_source *source,
const struct odb_optimize_options *opts)
{
struct odb_source_files *files = odb_source_files_downcast(source);
struct repository *repo = source->odb->repo;

switch (opts->strategy) {
case ODB_OPTIMIZE_INCREMENTAL: {
int gc_auto_threshold = 6700;
int gc_auto_pack_limit = 50;

repo_config_get_int(repo, "gc.auto", &gc_auto_threshold);
repo_config_get_int(repo, "gc.autopacklimit", &gc_auto_pack_limit);

/*
* Setting gc.auto to 0 or negative can disable the
* automatic gc.
*/
if (gc_auto_threshold <= 0)
return false;
if (!too_many_packs(files, gc_auto_pack_limit) &&
!too_many_loose_objects(files, gc_auto_threshold))
return false;

return true;
}
case ODB_OPTIMIZE_GEOMETRIC: {
struct pack_geometry geometry = {
.split_factor = 2,
};
struct pack_objects_args po_args = {
.local = 1,
};
struct existing_packs existing_packs = EXISTING_PACKS_INIT;
struct string_list kept_packs = STRING_LIST_INIT_DUP;
int auto_value = 100;
bool ret;

repo_config_get_int(repo, "maintenance.geometric-repack.auto",
&auto_value);
if (!auto_value)
return false;
if (auto_value < 0)
return true;

repo_config_get_int(repo, "maintenance.geometric-repack.splitFactor",
&geometry.split_factor);

existing_packs.repo = repo;
existing_packs_collect(&existing_packs, &kept_packs);
pack_geometry_init(&geometry, &existing_packs, &po_args);
pack_geometry_split(&geometry);

/*
* When we'd merge at least two packs with one another we always
* perform the repack.
*/
if (geometry.split) {
ret = true;
goto out;
}

/*
* Otherwise, we estimate the number of loose objects to determine
* whether we want to create a new packfile or not.
*/
if (too_many_loose_objects(files, auto_value)) {
ret = true;
goto out;
}

ret = false;

out:
existing_packs_release(&existing_packs);
pack_geometry_release(&geometry);
return ret;
}
default:
BUG("unknown maintenance strategy '%d'", opts->strategy);
}
}

int odb_source_files_optimize(struct odb_source *source,
const struct odb_optimize_options *opts)
{
struct odb_source_files *files = odb_source_files_downcast(source);
struct repository *repo = source->odb->repo;
struct child_process repack_cmd = CHILD_PROCESS_INIT;
unsigned long big_pack_threshold = 0;
int gc_auto_threshold = 6700;
int gc_auto_pack_limit = 50;
int ret;

repo_config_get_int(repo, "gc.auto", &gc_auto_threshold);
repo_config_get_int(repo, "gc.autopacklimit", &gc_auto_pack_limit);
repo_config_get_ulong(repo, "gc.bigpackthreshold", &big_pack_threshold);

if (repo->repository_format_precious_objects)
return 0;

repack_cmd.git_cmd = 1;
repack_cmd.odb_to_close = repo->objects;

strvec_pushl(&repack_cmd.args, "repack", "-d", "-l", NULL);
if (opts->flags & ODB_OPTIMIZE_NO_REUSE_DELTAS)
strvec_push(&repack_cmd.args, "-f");
if (opts->depth > 0)
strvec_pushf(&repack_cmd.args, "--depth=%d", opts->depth);
if (opts->window > 0)
strvec_pushf(&repack_cmd.args, "--window=%d", opts->window);
if (!(opts->flags & ODB_OPTIMIZE_VERBOSE))
strvec_push(&repack_cmd.args, "-q");

/*
* There's three cases we need to consider:
*
* - If we're invoked without `--auto` we'll need to perform a full
* repack.
*
* - If we're invoked with `--auto` and there's too many packs, then
* we perform a full repack, as well.
*
* - Otherwise we perform an incremental repack.
*/
switch (opts->strategy) {
case ODB_OPTIMIZE_INCREMENTAL:
if (!(opts->flags & ODB_OPTIMIZE_AUTO)) {
struct string_list keep_pack = STRING_LIST_INIT_NODUP;

if (opts->keep_largest_pack != -1) {
if (opts->keep_largest_pack)
find_base_packs(files, &keep_pack, 0);
} else if (big_pack_threshold) {
find_base_packs(files, &keep_pack, big_pack_threshold);
}

add_repack_all_option(repo, opts, &keep_pack, &repack_cmd.args);
string_list_clear(&keep_pack, 0);
} else {
if (too_many_packs(files, gc_auto_pack_limit)) {
struct string_list keep_pack = STRING_LIST_INIT_NODUP;

if (big_pack_threshold) {
find_base_packs(files, &keep_pack, big_pack_threshold);
if (keep_pack.nr >= (unsigned long) gc_auto_pack_limit) {
string_list_clear(&keep_pack, 0);
find_base_packs(files, &keep_pack, 0);
}
} else {
struct packed_git *p = find_base_packs(files, &keep_pack, 0);
uint64_t mem_have, mem_want;

mem_have = total_ram();
mem_want = estimate_repack_memory(files, p);

/*
* Only allow 1/2 of memory for pack-objects, leave
* the rest for the OS and other processes in the
* system.
*/
if (!mem_have || mem_want < mem_have / 2)
string_list_clear(&keep_pack, 0);
}

add_repack_all_option(repo, opts, &keep_pack, &repack_cmd.args);
string_list_clear(&keep_pack, 0);
} else {
add_repack_incremental_option(&repack_cmd.args);
}
}

break;
case ODB_OPTIMIZE_GEOMETRIC: {
struct pack_geometry geometry = {
.split_factor = 2,
};
struct pack_objects_args po_args = {
.local = 1,
};
struct existing_packs existing_packs = EXISTING_PACKS_INIT;
struct string_list kept_packs = STRING_LIST_INIT_DUP;

repo_config_get_int(repo, "maintenance.geometric-repack.splitFactor",
&geometry.split_factor);

existing_packs.repo = repo;
existing_packs_collect(&existing_packs, &kept_packs);
pack_geometry_init(&geometry, &existing_packs, &po_args);
pack_geometry_split(&geometry);

if (geometry.split < geometry.pack_nr) {
strvec_pushf(&repack_cmd.args, "--geometric=%d",
geometry.split_factor);
} else {
add_repack_all_option(repo, opts, NULL, &repack_cmd.args);
}
if (repo->settings.core_multi_pack_index)
strvec_push(&repack_cmd.args, "--write-midx");

existing_packs_release(&existing_packs);
pack_geometry_release(&geometry);
break;
}
default:
die("unknown maintenance strategy '%d'", opts->strategy);
}

if (run_command(&repack_cmd)) {
ret = error("failed to run %s", repack_cmd.args.v[0]);
goto out;
}

/* Geometric repacking uses cruft packs, so we don't have to prune separately. */
if (opts->strategy != ODB_OPTIMIZE_GEOMETRIC && opts->prune_expire) {
struct child_process prune_cmd = CHILD_PROCESS_INIT;

strvec_pushl(&prune_cmd.args, "prune", "--expire", NULL);
/* run `git prune` even if using cruft packs */
strvec_push(&prune_cmd.args, opts->prune_expire);
if (!(opts->flags & ODB_OPTIMIZE_VERBOSE))
strvec_push(&prune_cmd.args, "--no-progress");
if (repo_has_promisor_remote(repo))
strvec_push(&prune_cmd.args,
"--exclude-promisor-objects");
prune_cmd.git_cmd = 1;

if (run_command(&prune_cmd)) {
ret = error("failed to run %s", prune_cmd.args.v[0]);
goto out;
}
}

if (opts->flags & ODB_OPTIMIZE_AUTO && too_many_loose_objects(files, gc_auto_threshold))
warning(_("There are too many unreachable loose objects; "
"run 'git prune' to remove them."));

ret = 0;

out:
return ret;
}

struct odb_source_files *odb_source_files_new(struct object_database *odb,
const char *path,
bool local)
@ -286,6 +754,8 @@ struct odb_source_files *odb_source_files_new(struct object_database *odb,
files->base.begin_transaction = odb_source_files_begin_transaction;
files->base.read_alternates = odb_source_files_read_alternates;
files->base.write_alternate = odb_source_files_write_alternate;
files->base.optimize = odb_source_files_optimize;
files->base.optimize_required = odb_source_files_optimize_required;

/*
* Ideally, we would only ever store absolute paths in the source. This

15
odb/source-files.h
View File

@ -21,6 +21,21 @@ struct odb_source_files *odb_source_files_new(struct object_database *odb,
const char *path,
bool local);

/*
* Optimize the files object database source by repacking loose objects and
* packfiles as needed. Returns 0 on success, a negative error code otherwise.
*/
int odb_source_files_optimize(struct odb_source *source,
const struct odb_optimize_options *opts);

/*
* Check whether optimization of the files object database source is required
* given the provided options. Returns true if optimization should be
* performed, false otherwise.
*/
bool odb_source_files_optimize_required(struct odb_source *source,
const struct odb_optimize_options *opts);

/*
* Cast the given object database source to the files backend. This will cause
* a BUG in case the source doesn't use this backend.

36
odb/source.h
View File

@ -263,6 +263,21 @@ struct odb_source {
*/
int (*write_alternate)(struct odb_source *source,
const char *alternate);

/*
* This callback is expected to optimize the object database source.
* Returns 0 on success, a negative error code otherwise.
*/
int (*optimize)(struct odb_source *source,
const struct odb_optimize_options *opts);

/*
* This callback is expected to check whether optimization of the
* object database source is required given the provided options.
* Returns true if optimization should be performed, false otherwise.
*/
bool (*optimize_required)(struct odb_source *source,
const struct odb_optimize_options *opts);
};

/*
@ -484,4 +499,25 @@ static inline int odb_source_begin_transaction(struct odb_source *source,
return source->begin_transaction(source, out, flags);
}

/*
* Optimize the object database source. Returns 0 on success, a negative error
* code otherwise.
*/
static inline int odb_source_optimize(struct odb_source *source,
const struct odb_optimize_options *opts)
{
return source->optimize(source, opts);
}

/*
* Check whether optimization of the object database source is required given
* the provided options. Returns true if optimization should be performed,
* false otherwise.
*/
static inline bool odb_source_optimize_required(struct odb_source *source,
const struct odb_optimize_options *opts)
{
return source->optimize_required(source, opts);
}

#endif

334
t/t7900-maintenance.sh
View File

@ -23,6 +23,12 @@ test_xmllint () {
fi
}

test_maintenance_tasks () {
cat >expect &&
sed -ne "s/.*\"region_enter\".*\"category\":\"maintenance\([^\"]*\)\".*\"label\":\"\([^\"][^\"]*\)\".*/\2\1/p" "$1" >actual &&
test_cmp expect actual
}

test_lazy_prereq SYSTEMD_ANALYZE '
systemd-analyze verify /lib/systemd/system/basic.target
'
@ -180,8 +186,9 @@ test_expect_success 'maintenance.<task>.enabled' '
git config maintenance.gc.enabled false &&
git config maintenance.commit-graph.enabled true &&
GIT_TRACE2_EVENT="$(pwd)/run-config.txt" git maintenance run 2>err &&
test_subcommand ! git gc --quiet <run-config.txt &&
test_subcommand git commit-graph write --split --reachable --no-progress <run-config.txt
test_maintenance_tasks run-config.txt <<-\EOF
commit-graph
EOF
'

test_expect_success 'run --task=<task>' '
@ -189,16 +196,20 @@ test_expect_success 'run --task=<task>' '
git maintenance run --task=commit-graph 2>/dev/null &&
GIT_TRACE2_EVENT="$(pwd)/run-gc.txt" \
git maintenance run --task=gc 2>/dev/null &&
GIT_TRACE2_EVENT="$(pwd)/run-commit-graph.txt" \
git maintenance run --task=commit-graph 2>/dev/null &&
GIT_TRACE2_EVENT="$(pwd)/run-both.txt" \
git maintenance run --task=commit-graph --task=gc 2>/dev/null &&
test_subcommand ! git gc --quiet --no-detach --skip-foreground-tasks <run-commit-graph.txt &&
test_subcommand git gc --quiet --no-detach --skip-foreground-tasks <run-gc.txt &&
test_subcommand git gc --quiet --no-detach --skip-foreground-tasks <run-both.txt &&
test_subcommand git commit-graph write --split --reachable --no-progress <run-commit-graph.txt &&
test_subcommand ! git commit-graph write --split --reachable --no-progress <run-gc.txt &&
test_subcommand git commit-graph write --split --reachable --no-progress <run-both.txt
test_maintenance_tasks run-commit-graph.txt <<-\EOF &&
commit-graph
EOF
test_maintenance_tasks run-gc.txt <<-\EOF &&
gc foreground
gc
EOF
test_maintenance_tasks run-both.txt <<-\EOF
gc foreground
commit-graph
gc
EOF
'

test_expect_success 'core.commitGraph=false prevents write process' '
@ -235,12 +246,19 @@ test_expect_success 'commit-graph auto condition' '
GIT_TRACE2_EVENT="$(pwd)/cg-two-satisfied.txt" \
git -c maintenance.commit-graph.auto=2 $COMMAND &&

COMMIT_GRAPH_WRITE="git commit-graph write --split --reachable --no-progress" &&
test_subcommand ! $COMMIT_GRAPH_WRITE <cg-no.txt &&
test_subcommand $COMMIT_GRAPH_WRITE <cg-negative-means-yes.txt &&
test_subcommand ! $COMMIT_GRAPH_WRITE <cg-zero-means-no.txt &&
test_subcommand $COMMIT_GRAPH_WRITE <cg-one-satisfied.txt &&
test_subcommand $COMMIT_GRAPH_WRITE <cg-two-satisfied.txt
test_maintenance_tasks cg-no.txt <<-\EOF &&
EOF
test_maintenance_tasks cg-negative-means-yes.txt <<-\EOF &&
commit-graph
EOF
test_maintenance_tasks cg-zero-means-no.txt <<-\EOF &&
EOF
test_maintenance_tasks cg-one-satisfied.txt <<-\EOF &&
commit-graph
EOF
test_maintenance_tasks cg-two-satisfied.txt <<-\EOF
commit-graph
EOF
'

test_expect_success 'commit-graph auto condition with merges' '
@ -562,8 +580,8 @@ run_and_verify_geometric_pack () {
rm -f "trace2.txt" &&
GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \
git maintenance run --task=geometric-repack 2>/dev/null &&
test_subcommand git repack -d -l --geometric=2 \
--quiet --write-midx <trace2.txt &&
test_subcommand git repack -d -l -q --geometric=2 \
--write-midx <trace2.txt &&

# Verify that the number of packfiles matches our expectation.
ls -l .git/objects/pack/*.pack >packfiles &&
@ -594,8 +612,8 @@ test_expect_success 'geometric repacking task' '
# The initial repack causes an all-into-one repack.
GIT_TRACE2_EVENT="$(pwd)/initial-repack.txt" \
git maintenance run --task=geometric-repack 2>/dev/null &&
test_subcommand git repack -d -l --cruft --cruft-expiration=2.weeks.ago \
--quiet --write-midx <initial-repack.txt &&
test_subcommand git repack -d -l -q --cruft --cruft-expiration=2.weeks.ago \
--write-midx <initial-repack.txt &&

# Repacking should now cause a no-op geometric repack because
# no packfiles need to be combined.
@ -615,8 +633,8 @@ test_expect_success 'geometric repacking task' '
# an all-into-one-repack.
GIT_TRACE2_EVENT="$(pwd)/all-into-one-repack.txt" \
git maintenance run --task=geometric-repack 2>/dev/null &&
test_subcommand git repack -d -l --cruft --cruft-expiration=2.weeks.ago \
--quiet --write-midx <all-into-one-repack.txt &&
test_subcommand git repack -d -l -q --cruft --cruft-expiration=2.weeks.ago \
--write-midx <all-into-one-repack.txt &&

# The geometric repack soaks up unreachable objects.
echo blob-1 | git hash-object -w --stdin -t blob &&
@ -650,8 +668,8 @@ test_expect_success 'geometric repacking task' '
run_and_verify_geometric_pack 3 &&
GIT_TRACE2_EVENT="$(pwd)/cruft-repack.txt" \
git maintenance run --task=geometric-repack 2>/dev/null &&
test_subcommand git repack -d -l --cruft --cruft-expiration=2.weeks.ago \
--quiet --write-midx <cruft-repack.txt &&
test_subcommand git repack -d -l -q --cruft --cruft-expiration=2.weeks.ago \
--write-midx <cruft-repack.txt &&
ls .git/objects/pack/*.pack >packs &&
test_line_count = 2 packs &&
ls .git/objects/pack/*.mtimes >cruft &&
@ -742,7 +760,133 @@ test_expect_success 'geometric repacking honors configured split factor' '

test_geometric_repack_needed false splitFactor=2 &&
test_geometric_repack_needed true splitFactor=3 &&
test_subcommand git repack -d -l --geometric=3 --quiet --write-midx <trace2.txt
test_subcommand git repack -d -l -q --geometric=3 --write-midx <trace2.txt
)
'

test_expect_success 'pre-auto-gc hook runs exactly once' '
test_when_finished "rm -rf repo" &&
git init repo &&
(
cd repo &&
write_script .git/hooks/pre-auto-gc <<-\EOF &&
echo hook >>hook.log
EOF

# Satisfy the auto condition for multiple tasks, both in the
# foreground and in the background phase.
git config set maintenance.reflog-expire.auto -1 &&
git config set maintenance.geometric-repack.auto -1 &&
git config set maintenance.rerere-gc.auto -1 &&

GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \
git maintenance run --auto 2>/dev/null &&

# The successful hook does not inhibit any of the tasks...
test_maintenance_tasks trace2.txt <<-\EOF &&
reflog-expire foreground
geometric-repack
rerere-gc
EOF
# ... but it must only have been executed a single time.
test_line_count = 1 hook.log
)
'

test_expect_success 'pre-auto-gc hook can inhibit geometric strategy' '
test_when_finished "rm -rf repo" &&
git init repo &&
(
cd repo &&
write_script .git/hooks/pre-auto-gc <<-\EOF &&
echo hook >>hook.log
exit 1
EOF

git config set maintenance.reflog-expire.auto -1 &&
git config set maintenance.geometric-repack.auto -1 &&
git config set maintenance.rerere-gc.auto -1 &&

# Maintenance would be required...
git maintenance is-needed --auto &&

GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \
git maintenance run --auto 2>/dev/null &&

# ... but the failing hook inhibits all tasks. The hook itself
# is expected to be the only child process being spawned, and
# it must only run a single time.
test_grep "child_start.*pre-auto-gc" trace2.txt &&
test_maintenance_tasks trace2.txt <<-\EOF &&
EOF
test_line_count = 1 hook.log
)
'

test_expect_success 'pre-auto-gc hook can inhibit gc strategy' '
test_when_finished "rm -rf repo" &&
git init repo &&
(
cd repo &&
write_script .git/hooks/pre-auto-gc <<-\EOF &&
echo hook >>hook.log
exit 1
EOF

git config set maintenance.strategy gc &&
git config set maintenance.auto false &&
git config set gc.auto 3 &&

test_oid_init &&

# We need to create two objects whose hashes start with 17
# since this is what the gc task counts.
test_commit "$(test_oid blob17_1)" &&
test_commit "$(test_oid blob17_2)" &&

# Maintenance would be required...
git maintenance is-needed --auto &&

GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \
git maintenance run --auto 2>/dev/null &&

# ... but the failing hook inhibits all tasks. The hook itself
# is expected to be the only child process being spawned, and
# it must only run a single time.
test_grep "child_start.*pre-auto-gc" trace2.txt &&
test_maintenance_tasks trace2.txt <<-\EOF &&
EOF
test_subcommand_flex ! git trace2 &&
test_line_count = 1 hook.log
)
'

test_expect_success 'pre-auto-gc hook does not run when no maintenance is needed' '
test_when_finished "rm -rf repo" &&
git init repo &&
(
cd repo &&
write_script .git/hooks/pre-auto-gc <<-\EOF &&
echo hook >>hook.log
EOF
test_must_fail git maintenance is-needed --auto &&
git maintenance run --auto 2>/dev/null &&
test_path_is_missing hook.log
)
'

test_expect_success 'pre-auto-gc hook does not run without --auto' '
test_when_finished "rm -rf repo" &&
git init repo &&
test_hook -C repo pre-auto-gc <<-\EOF &&
echo hook >>hook.log
EOF
(
cd repo &&
GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \
git maintenance run 2>/dev/null &&
test_grep "\[\"git\",\"repack\"," trace2.txt &&
test_path_is_missing hook.log
)
'

@ -916,24 +1060,28 @@ test_expect_success '--schedule inheritance weekly -> daily -> hourly' '

GIT_TRACE2_EVENT="$(pwd)/hourly.txt" \
git maintenance run --schedule=hourly 2>/dev/null &&
test_subcommand git prune-packed --quiet <hourly.txt &&
test_subcommand ! git commit-graph write --split --reachable \
--no-progress <hourly.txt &&
test_subcommand ! git multi-pack-index write --no-progress <hourly.txt &&
test_maintenance_tasks hourly.txt <<-\EOF &&
prefetch
loose-objects
EOF

GIT_TRACE2_EVENT="$(pwd)/daily.txt" \
git maintenance run --schedule=daily 2>/dev/null &&
test_subcommand git prune-packed --quiet <daily.txt &&
test_subcommand git commit-graph write --split --reachable \
--no-progress <daily.txt &&
test_subcommand ! git multi-pack-index write --no-progress <daily.txt &&
test_maintenance_tasks daily.txt <<-\EOF &&
prefetch
loose-objects
commit-graph
EOF

GIT_TRACE2_EVENT="$(pwd)/weekly.txt" \
git maintenance run --schedule=weekly 2>/dev/null &&
test_subcommand git prune-packed --quiet <weekly.txt &&
test_subcommand git commit-graph write --split --reachable \
--no-progress <weekly.txt &&
test_subcommand git multi-pack-index write --no-progress <weekly.txt
test_maintenance_tasks weekly.txt <<-\EOF
pack-refs foreground
prefetch
loose-objects
incremental-repack
commit-graph
EOF
'

test_expect_success 'maintenance.strategy inheritance' '
@ -952,29 +1100,25 @@ test_expect_success 'maintenance.strategy inheritance' '
GIT_TRACE2_EVENT="$(pwd)/incremental-weekly.txt" \
git maintenance run --schedule=weekly --quiet &&

test_subcommand git commit-graph write --split --reachable \
--no-progress <incremental-hourly.txt &&
test_subcommand ! git prune-packed --quiet <incremental-hourly.txt &&
test_subcommand ! git multi-pack-index write --no-progress \
<incremental-hourly.txt &&
test_subcommand ! git pack-refs --all --prune \
<incremental-hourly.txt &&
test_maintenance_tasks incremental-hourly.txt <<-\EOF &&
prefetch
commit-graph
EOF

test_subcommand git commit-graph write --split --reachable \
--no-progress <incremental-daily.txt &&
test_subcommand git prune-packed --quiet <incremental-daily.txt &&
test_subcommand git multi-pack-index write --no-progress \
<incremental-daily.txt &&
test_subcommand ! git pack-refs --all --prune \
<incremental-daily.txt &&
test_maintenance_tasks incremental-daily.txt <<-\EOF &&
prefetch
loose-objects
incremental-repack
commit-graph
EOF

test_subcommand git commit-graph write --split --reachable \
--no-progress <incremental-weekly.txt &&
test_subcommand git prune-packed --quiet <incremental-weekly.txt &&
test_subcommand git multi-pack-index write --no-progress \
<incremental-weekly.txt &&
test_subcommand git pack-refs --all --prune \
<incremental-weekly.txt &&
test_maintenance_tasks incremental-weekly.txt <<-\EOF &&
pack-refs foreground
prefetch
loose-objects
incremental-repack
commit-graph
EOF

# Modify defaults
git config maintenance.commit-graph.schedule daily &&
@ -986,30 +1130,26 @@ test_expect_success 'maintenance.strategy inheritance' '
GIT_TRACE2_EVENT="$(pwd)/modified-daily.txt" \
git maintenance run --schedule=daily --quiet &&

test_subcommand ! git commit-graph write --split --reachable \
--no-progress <modified-hourly.txt &&
test_subcommand git prune-packed --quiet <modified-hourly.txt &&
test_subcommand ! git multi-pack-index write --no-progress \
<modified-hourly.txt &&
test_maintenance_tasks modified-hourly.txt <<-\EOF &&
prefetch
loose-objects
EOF

test_subcommand git commit-graph write --split --reachable \
--no-progress <modified-daily.txt &&
test_subcommand git prune-packed --quiet <modified-daily.txt &&
test_subcommand ! git multi-pack-index write --no-progress \
<modified-daily.txt
test_maintenance_tasks modified-daily.txt <<-\EOF
prefetch
loose-objects
commit-graph
EOF
'

test_strategy () {
STRATEGY="$1"
shift

cat >expect &&
rm -f trace2.txt &&
GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \
git -c maintenance.strategy=$STRATEGY maintenance run --quiet "$@" &&
sed -n 's/{"event":"child_start","sid":"[^/"]*",.*,"argv":\["\(.*\)\"]}/\1/p' <trace2.txt |
sed 's/","/ /g' >actual
test_cmp expect actual
test_maintenance_tasks trace2.txt
}

test_expect_success 'maintenance.strategy is respected' '
@ -1023,48 +1163,44 @@ test_expect_success 'maintenance.strategy is respected' '
test_grep "unknown maintenance strategy: .unknown." err &&

test_strategy incremental <<-\EOF &&
git pack-refs --all --prune
git reflog expire --all
git gc --quiet --no-detach --skip-foreground-tasks
gc foreground
gc
EOF

test_strategy incremental --schedule=weekly <<-\EOF &&
git pack-refs --all --prune
git prune-packed --quiet
git multi-pack-index write --no-progress
git multi-pack-index expire --no-progress
git multi-pack-index repack --no-progress --batch-size=1
git commit-graph write --split --reachable --no-progress
pack-refs foreground
prefetch
loose-objects
incremental-repack
commit-graph
EOF

test_strategy gc <<-\EOF &&
git pack-refs --all --prune
git reflog expire --all
git gc --quiet --no-detach --skip-foreground-tasks
gc foreground
gc
EOF

test_strategy gc --schedule=weekly <<-\EOF &&
git pack-refs --all --prune
git reflog expire --all
git gc --quiet --no-detach --skip-foreground-tasks
gc foreground
gc
EOF

test_strategy geometric <<-\EOF &&
git pack-refs --all --prune
git reflog expire --all
git repack -d -l --geometric=2 --quiet --write-midx
git commit-graph write --split --reachable --no-progress
git worktree prune --expire 3.months.ago
git rerere gc
pack-refs foreground
reflog-expire foreground
geometric-repack
commit-graph
worktree-prune
rerere-gc
EOF

test_strategy geometric --schedule=weekly <<-\EOF
git pack-refs --all --prune
git reflog expire --all
git repack -d -l --geometric=2 --quiet --write-midx
git commit-graph write --split --reachable --no-progress
git worktree prune --expire 3.months.ago
git rerere gc
pack-refs foreground
reflog-expire foreground
geometric-repack
commit-graph
worktree-prune
rerere-gc
EOF
)
'