Merge branch 'tb/incremental-midx-part-3.1' into seen
* tb/incremental-midx-part-3.1: (50 commits) SQUASH??? play well with other topics by preemptively including "repository.h" builtin/repack.c: clean up unused `#include`s repack: move `write_cruft_pack()` out of the builtin repack: move `write_filtered_pack()` out of the builtin repack: move `pack_kept_objects` to `struct pack_objects_args` repack: move `finish_pack_objects_cmd()` out of the builtin builtin/repack.c: pass `write_pack_opts` to `finish_pack_objects_cmd()` repack: extract `write_pack_opts_is_local()` repack: move `find_pack_prefix()` out of the builtin builtin/repack.c: use `write_pack_opts` within `write_cruft_pack()` builtin/repack.c: introduce `struct write_pack_opts` repack: 'write_midx_included_packs' API from the builtin builtin/repack.c: inline packs within `write_midx_included_packs()` builtin/repack.c: pass `repack_write_midx_opts` to `midx_included_packs` builtin/repack.c: inline `remove_redundant_bitmaps()` builtin/repack.c: reorder `remove_redundant_bitmaps()` repack: keep track of MIDX pack names using existing_packs builtin/repack.c: use a string_list for 'midx_pack_names' builtin/repack.c: extract opts struct for 'write_midx_included_packs()' builtin/repack.c: remove ref snapshotting from builtin ...
Junio C Hamano
commit
eaa03a89bf
6
Makefile
6
Makefile
|
|
@ -1249,6 +1249,12 @@ LIB_OBJS += refs/packed-backend.o
|
||||||
LIB_OBJS += refs/ref-cache.o
|
LIB_OBJS += refs/ref-cache.o
|
||||||
LIB_OBJS += refspec.o
|
LIB_OBJS += refspec.o
|
||||||
LIB_OBJS += remote.o
|
LIB_OBJS += remote.o
|
||||||
|
LIB_OBJS += repack.o
|
||||||
|
LIB_OBJS += repack-cruft.o
|
||||||
|
LIB_OBJS += repack-filtered.o
|
||||||
|
LIB_OBJS += repack-geometry.o
|
||||||
|
LIB_OBJS += repack-midx.o
|
||||||
|
LIB_OBJS += repack-promisor.o
|
||||||
LIB_OBJS += replace-object.o
|
LIB_OBJS += replace-object.o
|
||||||
LIB_OBJS += replay.o
|
LIB_OBJS += replay.o
|
||||||
LIB_OBJS += repo-settings.o
|
LIB_OBJS += repo-settings.o
|
||||||
|
|
|
||||||
1348
builtin/repack.c
1348
builtin/repack.c
File diff suppressed because it is too large
Load Diff
|
|
@ -463,6 +463,12 @@ libgit_sources = [
|
||||||
'reftable/tree.c',
|
'reftable/tree.c',
|
||||||
'reftable/writer.c',
|
'reftable/writer.c',
|
||||||
'remote.c',
|
'remote.c',
|
||||||
|
'repack.c',
|
||||||
|
'repack-cruft.c',
|
||||||
|
'repack-filtered.c',
|
||||||
|
'repack-geometry.c',
|
||||||
|
'repack-midx.c',
|
||||||
|
'repack-promisor.c',
|
||||||
'replace-object.c',
|
'replace-object.c',
|
||||||
'replay.c',
|
'replay.c',
|
||||||
'repo-settings.c',
|
'repo-settings.c',
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,99 @@
|
||||||
|
#include "git-compat-util.h"
|
||||||
|
#include "repack.h"
|
||||||
|
#include "packfile.h"
|
||||||
|
#include "repository.h"
|
||||||
|
#include "run-command.h"
|
||||||
|
|
||||||
|
static void combine_small_cruft_packs(FILE *in, off_t combine_cruft_below_size,
|
||||||
|
struct existing_packs *existing)
|
||||||
|
{
|
||||||
|
struct packfile_store *packs = existing->repo->objects->packfiles;
|
||||||
|
struct packed_git *p;
|
||||||
|
struct strbuf buf = STRBUF_INIT;
|
||||||
|
size_t i;
|
||||||
|
|
||||||
|
for (p = packfile_store_get_all_packs(packs); p; p = p->next) {
|
||||||
|
if (!(p->is_cruft && p->pack_local))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
strbuf_reset(&buf);
|
||||||
|
strbuf_addstr(&buf, pack_basename(p));
|
||||||
|
strbuf_strip_suffix(&buf, ".pack");
|
||||||
|
|
||||||
|
if (!string_list_has_string(&existing->cruft_packs, buf.buf))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
if (p->pack_size < combine_cruft_below_size) {
|
||||||
|
fprintf(in, "-%s\n", pack_basename(p));
|
||||||
|
} else {
|
||||||
|
existing_packs_retain_cruft(existing, p);
|
||||||
|
fprintf(in, "%s\n", pack_basename(p));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (i = 0; i < existing->non_kept_packs.nr; i++)
|
||||||
|
fprintf(in, "-%s.pack\n",
|
||||||
|
existing->non_kept_packs.items[i].string);
|
||||||
|
|
||||||
|
strbuf_release(&buf);
|
||||||
|
}
|
||||||
|
|
||||||
|
int write_cruft_pack(struct write_pack_opts *opts,
|
||||||
|
const char *cruft_expiration,
|
||||||
|
unsigned long combine_cruft_below_size,
|
||||||
|
struct string_list *names,
|
||||||
|
struct existing_packs *existing)
|
||||||
|
{
|
||||||
|
struct child_process cmd = CHILD_PROCESS_INIT;
|
||||||
|
struct string_list_item *item;
|
||||||
|
FILE *in;
|
||||||
|
int ret;
|
||||||
|
const char *pack_prefix = write_pack_opts_pack_prefix(opts);
|
||||||
|
|
||||||
|
prepare_pack_objects(&cmd, opts->po_args, opts->destination);
|
||||||
|
|
||||||
|
strvec_push(&cmd.args, "--cruft");
|
||||||
|
if (cruft_expiration)
|
||||||
|
strvec_pushf(&cmd.args, "--cruft-expiration=%s",
|
||||||
|
cruft_expiration);
|
||||||
|
|
||||||
|
strvec_push(&cmd.args, "--non-empty");
|
||||||
|
|
||||||
|
cmd.in = -1;
|
||||||
|
|
||||||
|
ret = start_command(&cmd);
|
||||||
|
if (ret)
|
||||||
|
return ret;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* names has a confusing double use: it both provides the list
|
||||||
|
* of just-written new packs, and accepts the name of the cruft
|
||||||
|
* pack we are writing.
|
||||||
|
*
|
||||||
|
* By the time it is read here, it contains only the pack(s)
|
||||||
|
* that were just written, which is exactly the set of packs we
|
||||||
|
* want to consider kept.
|
||||||
|
*
|
||||||
|
* If `--expire-to` is given, the double-use served by `names`
|
||||||
|
* ensures that the pack written to `--expire-to` excludes any
|
||||||
|
* objects contained in the cruft pack.
|
||||||
|
*/
|
||||||
|
in = xfdopen(cmd.in, "w");
|
||||||
|
for_each_string_list_item(item, names)
|
||||||
|
fprintf(in, "%s-%s.pack\n", pack_prefix, item->string);
|
||||||
|
if (combine_cruft_below_size && !cruft_expiration) {
|
||||||
|
combine_small_cruft_packs(in, combine_cruft_below_size,
|
||||||
|
existing);
|
||||||
|
} else {
|
||||||
|
for_each_string_list_item(item, &existing->non_kept_packs)
|
||||||
|
fprintf(in, "-%s.pack\n", item->string);
|
||||||
|
for_each_string_list_item(item, &existing->cruft_packs)
|
||||||
|
fprintf(in, "-%s.pack\n", item->string);
|
||||||
|
}
|
||||||
|
for_each_string_list_item(item, &existing->kept_packs)
|
||||||
|
fprintf(in, "%s.pack\n", item->string);
|
||||||
|
fclose(in);
|
||||||
|
|
||||||
|
return finish_pack_objects_cmd(existing->repo->hash_algo, opts, &cmd,
|
||||||
|
names);
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,51 @@
|
||||||
|
#include "git-compat-util.h"
|
||||||
|
#include "repack.h"
|
||||||
|
#include "repository.h"
|
||||||
|
#include "run-command.h"
|
||||||
|
#include "string-list.h"
|
||||||
|
|
||||||
|
int write_filtered_pack(struct write_pack_opts *opts,
|
||||||
|
struct existing_packs *existing,
|
||||||
|
struct string_list *names)
|
||||||
|
{
|
||||||
|
struct child_process cmd = CHILD_PROCESS_INIT;
|
||||||
|
struct string_list_item *item;
|
||||||
|
FILE *in;
|
||||||
|
int ret;
|
||||||
|
const char *caret;
|
||||||
|
const char *pack_prefix = write_pack_opts_pack_prefix(opts);
|
||||||
|
|
||||||
|
prepare_pack_objects(&cmd, opts->po_args, opts->destination);
|
||||||
|
|
||||||
|
strvec_push(&cmd.args, "--stdin-packs");
|
||||||
|
|
||||||
|
for_each_string_list_item(item, &existing->kept_packs)
|
||||||
|
strvec_pushf(&cmd.args, "--keep-pack=%s", item->string);
|
||||||
|
|
||||||
|
cmd.in = -1;
|
||||||
|
|
||||||
|
ret = start_command(&cmd);
|
||||||
|
if (ret)
|
||||||
|
return ret;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Here 'names' contains only the pack(s) that were just
|
||||||
|
* written, which is exactly the packs we want to keep. Also
|
||||||
|
* 'existing_kept_packs' already contains the packs in
|
||||||
|
* 'keep_pack_list'.
|
||||||
|
*/
|
||||||
|
in = xfdopen(cmd.in, "w");
|
||||||
|
for_each_string_list_item(item, names)
|
||||||
|
fprintf(in, "^%s-%s.pack\n", pack_prefix, item->string);
|
||||||
|
for_each_string_list_item(item, &existing->non_kept_packs)
|
||||||
|
fprintf(in, "%s.pack\n", item->string);
|
||||||
|
for_each_string_list_item(item, &existing->cruft_packs)
|
||||||
|
fprintf(in, "%s.pack\n", item->string);
|
||||||
|
caret = opts->po_args->pack_kept_objects ? "" : "^";
|
||||||
|
for_each_string_list_item(item, &existing->kept_packs)
|
||||||
|
fprintf(in, "%s%s.pack\n", caret, item->string);
|
||||||
|
fclose(in);
|
||||||
|
|
||||||
|
return finish_pack_objects_cmd(existing->repo->hash_algo, opts, &cmd,
|
||||||
|
names);
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,233 @@
|
||||||
|
#define DISABLE_SIGN_COMPARE_WARNINGS
|
||||||
|
|
||||||
|
#include "git-compat-util.h"
|
||||||
|
#include <repository.h>
|
||||||
|
#include "repack.h"
|
||||||
|
#include "hex.h"
|
||||||
|
#include "packfile.h"
|
||||||
|
|
||||||
|
static uint32_t pack_geometry_weight(struct packed_git *p)
|
||||||
|
{
|
||||||
|
if (open_pack_index(p))
|
||||||
|
die(_("cannot open index for %s"), p->pack_name);
|
||||||
|
return p->num_objects;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int pack_geometry_cmp(const void *va, const void *vb)
|
||||||
|
{
|
||||||
|
uint32_t aw = pack_geometry_weight(*(struct packed_git **)va),
|
||||||
|
bw = pack_geometry_weight(*(struct packed_git **)vb);
|
||||||
|
|
||||||
|
if (aw < bw)
|
||||||
|
return -1;
|
||||||
|
if (aw > bw)
|
||||||
|
return 1;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
void pack_geometry_init(struct pack_geometry *geometry,
|
||||||
|
struct existing_packs *existing,
|
||||||
|
const struct pack_objects_args *args)
|
||||||
|
{
|
||||||
|
struct packfile_store *packs = existing->repo->objects->packfiles;
|
||||||
|
struct packed_git *p;
|
||||||
|
struct strbuf buf = STRBUF_INIT;
|
||||||
|
|
||||||
|
for (p = packfile_store_get_all_packs(packs); p; p = p->next) {
|
||||||
|
if (args->local && !p->pack_local)
|
||||||
|
/*
|
||||||
|
* When asked to only repack local packfiles we skip
|
||||||
|
* over any packfiles that are borrowed from alternate
|
||||||
|
* object directories.
|
||||||
|
*/
|
||||||
|
continue;
|
||||||
|
|
||||||
|
if (!args->pack_kept_objects) {
|
||||||
|
/*
|
||||||
|
* Any pack that has its pack_keep bit set will
|
||||||
|
* appear in existing->kept_packs below, but
|
||||||
|
* this saves us from doing a more expensive
|
||||||
|
* check.
|
||||||
|
*/
|
||||||
|
if (p->pack_keep)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* The pack may be kept via the --keep-pack
|
||||||
|
* option; check 'existing->kept_packs' to
|
||||||
|
* determine whether to ignore it.
|
||||||
|
*/
|
||||||
|
strbuf_reset(&buf);
|
||||||
|
strbuf_addstr(&buf, pack_basename(p));
|
||||||
|
strbuf_strip_suffix(&buf, ".pack");
|
||||||
|
|
||||||
|
if (string_list_has_string(&existing->kept_packs, buf.buf))
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (p->is_cruft)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
ALLOC_GROW(geometry->pack,
|
||||||
|
geometry->pack_nr + 1,
|
||||||
|
geometry->pack_alloc);
|
||||||
|
|
||||||
|
geometry->pack[geometry->pack_nr] = p;
|
||||||
|
geometry->pack_nr++;
|
||||||
|
}
|
||||||
|
|
||||||
|
QSORT(geometry->pack, geometry->pack_nr, pack_geometry_cmp);
|
||||||
|
strbuf_release(&buf);
|
||||||
|
}
|
||||||
|
|
||||||
|
void pack_geometry_split(struct pack_geometry *geometry)
|
||||||
|
{
|
||||||
|
uint32_t i;
|
||||||
|
uint32_t split;
|
||||||
|
off_t total_size = 0;
|
||||||
|
|
||||||
|
if (!geometry->pack_nr) {
|
||||||
|
geometry->split = geometry->pack_nr;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* First, count the number of packs (in descending order of size) which
|
||||||
|
* already form a geometric progression.
|
||||||
|
*/
|
||||||
|
for (i = geometry->pack_nr - 1; i > 0; i--) {
|
||||||
|
struct packed_git *ours = geometry->pack[i];
|
||||||
|
struct packed_git *prev = geometry->pack[i - 1];
|
||||||
|
|
||||||
|
if (unsigned_mult_overflows(geometry->split_factor,
|
||||||
|
pack_geometry_weight(prev)))
|
||||||
|
die(_("pack %s too large to consider in geometric "
|
||||||
|
"progression"),
|
||||||
|
prev->pack_name);
|
||||||
|
|
||||||
|
if (pack_geometry_weight(ours) <
|
||||||
|
geometry->split_factor * pack_geometry_weight(prev))
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
split = i;
|
||||||
|
|
||||||
|
if (split) {
|
||||||
|
/*
|
||||||
|
* Move the split one to the right, since the top element in the
|
||||||
|
* last-compared pair can't be in the progression. Only do this
|
||||||
|
* when we split in the middle of the array (otherwise if we got
|
||||||
|
* to the end, then the split is in the right place).
|
||||||
|
*/
|
||||||
|
split++;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Then, anything to the left of 'split' must be in a new pack. But,
|
||||||
|
* creating that new pack may cause packs in the heavy half to no longer
|
||||||
|
* form a geometric progression.
|
||||||
|
*
|
||||||
|
* Compute an expected size of the new pack, and then determine how many
|
||||||
|
* packs in the heavy half need to be joined into it (if any) to restore
|
||||||
|
* the geometric progression.
|
||||||
|
*/
|
||||||
|
for (i = 0; i < split; i++) {
|
||||||
|
struct packed_git *p = geometry->pack[i];
|
||||||
|
|
||||||
|
if (unsigned_add_overflows(total_size, pack_geometry_weight(p)))
|
||||||
|
die(_("pack %s too large to roll up"), p->pack_name);
|
||||||
|
total_size += pack_geometry_weight(p);
|
||||||
|
}
|
||||||
|
for (i = split; i < geometry->pack_nr; i++) {
|
||||||
|
struct packed_git *ours = geometry->pack[i];
|
||||||
|
|
||||||
|
if (unsigned_mult_overflows(geometry->split_factor,
|
||||||
|
total_size))
|
||||||
|
die(_("pack %s too large to roll up"), ours->pack_name);
|
||||||
|
|
||||||
|
if (pack_geometry_weight(ours) <
|
||||||
|
geometry->split_factor * total_size) {
|
||||||
|
if (unsigned_add_overflows(total_size,
|
||||||
|
pack_geometry_weight(ours)))
|
||||||
|
die(_("pack %s too large to roll up"),
|
||||||
|
ours->pack_name);
|
||||||
|
|
||||||
|
split++;
|
||||||
|
total_size += pack_geometry_weight(ours);
|
||||||
|
} else
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
geometry->split = split;
|
||||||
|
}
|
||||||
|
|
||||||
|
struct packed_git *pack_geometry_preferred_pack(struct pack_geometry *geometry)
|
||||||
|
{
|
||||||
|
uint32_t i;
|
||||||
|
|
||||||
|
if (!geometry) {
|
||||||
|
/*
|
||||||
|
* No geometry means either an all-into-one repack (in which
|
||||||
|
* case there is only one pack left and it is the largest) or an
|
||||||
|
* incremental one.
|
||||||
|
*
|
||||||
|
* If repacking incrementally, then we could check the size of
|
||||||
|
* all packs to determine which should be preferred, but leave
|
||||||
|
* this for later.
|
||||||
|
*/
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (geometry->split == geometry->pack_nr)
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* The preferred pack is the largest pack above the split line. In
|
||||||
|
* other words, it is the largest pack that does not get rolled up in
|
||||||
|
* the geometric repack.
|
||||||
|
*/
|
||||||
|
for (i = geometry->pack_nr; i > geometry->split; i--)
|
||||||
|
/*
|
||||||
|
* A pack that is not local would never be included in a
|
||||||
|
* multi-pack index. We thus skip over any non-local packs.
|
||||||
|
*/
|
||||||
|
if (geometry->pack[i - 1]->pack_local)
|
||||||
|
return geometry->pack[i - 1];
|
||||||
|
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
void pack_geometry_remove_redundant(struct pack_geometry *geometry,
|
||||||
|
struct string_list *names,
|
||||||
|
struct existing_packs *existing,
|
||||||
|
const char *packdir)
|
||||||
|
{
|
||||||
|
const struct git_hash_algo *algop = existing->repo->hash_algo;
|
||||||
|
struct strbuf buf = STRBUF_INIT;
|
||||||
|
uint32_t i;
|
||||||
|
|
||||||
|
for (i = 0; i < geometry->split; i++) {
|
||||||
|
struct packed_git *p = geometry->pack[i];
|
||||||
|
if (string_list_has_string(names, hash_to_hex_algop(p->hash,
|
||||||
|
algop)))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
strbuf_reset(&buf);
|
||||||
|
strbuf_addstr(&buf, pack_basename(p));
|
||||||
|
strbuf_strip_suffix(&buf, ".pack");
|
||||||
|
|
||||||
|
if ((p->pack_keep) ||
|
||||||
|
(string_list_has_string(&existing->kept_packs, buf.buf)))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
repack_remove_redundant_pack(existing->repo, packdir, buf.buf);
|
||||||
|
}
|
||||||
|
|
||||||
|
strbuf_release(&buf);
|
||||||
|
}
|
||||||
|
|
||||||
|
void pack_geometry_release(struct pack_geometry *geometry)
|
||||||
|
{
|
||||||
|
if (!geometry)
|
||||||
|
return;
|
||||||
|
|
||||||
|
free(geometry->pack);
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,372 @@
|
||||||
|
#include "git-compat-util.h"
|
||||||
|
#include "repack.h"
|
||||||
|
#include "hash.h"
|
||||||
|
#include "hex.h"
|
||||||
|
#include "odb.h"
|
||||||
|
#include "oidset.h"
|
||||||
|
#include "pack-bitmap.h"
|
||||||
|
#include "refs.h"
|
||||||
|
#include "run-command.h"
|
||||||
|
#include "tempfile.h"
|
||||||
|
|
||||||
|
struct midx_snapshot_ref_data {
|
||||||
|
struct repository *repo;
|
||||||
|
struct tempfile *f;
|
||||||
|
struct oidset seen;
|
||||||
|
int preferred;
|
||||||
|
};
|
||||||
|
|
||||||
|
static int midx_snapshot_ref_one(const char *refname UNUSED,
|
||||||
|
const char *referent UNUSED,
|
||||||
|
const struct object_id *oid,
|
||||||
|
int flag UNUSED, void *_data)
|
||||||
|
{
|
||||||
|
struct midx_snapshot_ref_data *data = _data;
|
||||||
|
struct object_id peeled;
|
||||||
|
|
||||||
|
if (!peel_iterated_oid(data->repo, oid, &peeled))
|
||||||
|
oid = &peeled;
|
||||||
|
|
||||||
|
if (oidset_insert(&data->seen, oid))
|
||||||
|
return 0; /* already seen */
|
||||||
|
|
||||||
|
if (odb_read_object_info(data->repo->objects, oid, NULL) != OBJ_COMMIT)
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
fprintf(data->f->fp, "%s%s\n", data->preferred ? "+" : "",
|
||||||
|
oid_to_hex(oid));
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
void midx_snapshot_refs(struct repository *repo, struct tempfile *f)
|
||||||
|
{
|
||||||
|
struct midx_snapshot_ref_data data;
|
||||||
|
const struct string_list *preferred = bitmap_preferred_tips(repo);
|
||||||
|
|
||||||
|
data.repo = repo;
|
||||||
|
data.f = f;
|
||||||
|
data.preferred = 0;
|
||||||
|
oidset_init(&data.seen, 0);
|
||||||
|
|
||||||
|
if (!fdopen_tempfile(f, "w"))
|
||||||
|
die(_("could not open tempfile %s for writing"),
|
||||||
|
get_tempfile_path(f));
|
||||||
|
|
||||||
|
if (preferred) {
|
||||||
|
struct string_list_item *item;
|
||||||
|
|
||||||
|
data.preferred = 1;
|
||||||
|
for_each_string_list_item(item, preferred)
|
||||||
|
refs_for_each_ref_in(get_main_ref_store(repo),
|
||||||
|
item->string,
|
||||||
|
midx_snapshot_ref_one, &data);
|
||||||
|
data.preferred = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
refs_for_each_ref(get_main_ref_store(repo),
|
||||||
|
midx_snapshot_ref_one, &data);
|
||||||
|
|
||||||
|
if (close_tempfile_gently(f)) {
|
||||||
|
int save_errno = errno;
|
||||||
|
delete_tempfile(&f);
|
||||||
|
errno = save_errno;
|
||||||
|
die_errno(_("could not close refs snapshot tempfile"));
|
||||||
|
}
|
||||||
|
|
||||||
|
oidset_clear(&data.seen);
|
||||||
|
}
|
||||||
|
|
||||||
|
static int midx_has_unknown_packs(struct string_list *include,
|
||||||
|
struct pack_geometry *geometry,
|
||||||
|
struct existing_packs *existing)
|
||||||
|
{
|
||||||
|
struct string_list_item *item;
|
||||||
|
|
||||||
|
string_list_sort(include);
|
||||||
|
|
||||||
|
for_each_string_list_item(item, &existing->midx_packs) {
|
||||||
|
const char *pack_name = item->string;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Determine whether or not each MIDX'd pack from the existing
|
||||||
|
* MIDX (if any) is represented in the new MIDX. For each pack
|
||||||
|
* in the MIDX, it must either be:
|
||||||
|
*
|
||||||
|
* - In the "include" list of packs to be included in the new
|
||||||
|
* MIDX. Note this function is called before the include
|
||||||
|
* list is populated with any cruft pack(s).
|
||||||
|
*
|
||||||
|
* - Below the geometric split line (if using pack geometry),
|
||||||
|
* indicating that the pack won't be included in the new
|
||||||
|
* MIDX, but its contents were rolled up as part of the
|
||||||
|
* geometric repack.
|
||||||
|
*
|
||||||
|
* - In the existing non-kept packs list (if not using pack
|
||||||
|
* geometry), and marked as non-deleted.
|
||||||
|
*/
|
||||||
|
if (string_list_has_string(include, pack_name)) {
|
||||||
|
continue;
|
||||||
|
} else if (geometry) {
|
||||||
|
struct strbuf buf = STRBUF_INIT;
|
||||||
|
uint32_t j;
|
||||||
|
|
||||||
|
for (j = 0; j < geometry->split; j++) {
|
||||||
|
strbuf_reset(&buf);
|
||||||
|
strbuf_addstr(&buf, pack_basename(geometry->pack[j]));
|
||||||
|
strbuf_strip_suffix(&buf, ".pack");
|
||||||
|
strbuf_addstr(&buf, ".idx");
|
||||||
|
|
||||||
|
if (!strcmp(pack_name, buf.buf)) {
|
||||||
|
strbuf_release(&buf);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
strbuf_release(&buf);
|
||||||
|
|
||||||
|
if (j < geometry->split)
|
||||||
|
continue;
|
||||||
|
} else {
|
||||||
|
struct string_list_item *item;
|
||||||
|
|
||||||
|
item = string_list_lookup(&existing->non_kept_packs,
|
||||||
|
pack_name);
|
||||||
|
if (item && !existing_pack_is_marked_for_deletion(item))
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* If we got to this point, the MIDX includes some pack that we
|
||||||
|
* don't know about.
|
||||||
|
*/
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void midx_included_packs(struct string_list *include,
|
||||||
|
struct repack_write_midx_opts *opts)
|
||||||
|
{
|
||||||
|
struct existing_packs *existing = opts->existing;
|
||||||
|
struct pack_geometry *geometry = opts->geometry;
|
||||||
|
struct string_list *names = opts->names;
|
||||||
|
struct string_list_item *item;
|
||||||
|
struct strbuf buf = STRBUF_INIT;
|
||||||
|
|
||||||
|
for_each_string_list_item(item, &existing->kept_packs) {
|
||||||
|
strbuf_reset(&buf);
|
||||||
|
strbuf_addf(&buf, "%s.idx", item->string);
|
||||||
|
string_list_insert(include, buf.buf);
|
||||||
|
}
|
||||||
|
|
||||||
|
for_each_string_list_item(item, names) {
|
||||||
|
strbuf_reset(&buf);
|
||||||
|
strbuf_addf(&buf, "pack-%s.idx", item->string);
|
||||||
|
string_list_insert(include, buf.buf);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (geometry->split_factor) {
|
||||||
|
uint32_t i;
|
||||||
|
|
||||||
|
for (i = geometry->split; i < geometry->pack_nr; i++) {
|
||||||
|
struct packed_git *p = geometry->pack[i];
|
||||||
|
|
||||||
|
/*
|
||||||
|
* The multi-pack index never refers to packfiles part
|
||||||
|
* of an alternate object database, so we skip these.
|
||||||
|
* While git-multi-pack-index(1) would silently ignore
|
||||||
|
* them anyway, this allows us to skip executing the
|
||||||
|
* command completely when we have only non-local
|
||||||
|
* packfiles.
|
||||||
|
*/
|
||||||
|
if (!p->pack_local)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
strbuf_reset(&buf);
|
||||||
|
strbuf_addstr(&buf, pack_basename(p));
|
||||||
|
strbuf_strip_suffix(&buf, ".pack");
|
||||||
|
strbuf_addstr(&buf, ".idx");
|
||||||
|
|
||||||
|
string_list_insert(include, buf.buf);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for_each_string_list_item(item, &existing->non_kept_packs) {
|
||||||
|
if (existing_pack_is_marked_for_deletion(item))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
strbuf_reset(&buf);
|
||||||
|
strbuf_addf(&buf, "%s.idx", item->string);
|
||||||
|
string_list_insert(include, buf.buf);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (opts->midx_must_contain_cruft ||
|
||||||
|
midx_has_unknown_packs(include, geometry, existing)) {
|
||||||
|
/*
|
||||||
|
* If there are one or more unknown pack(s) present (see
|
||||||
|
* midx_has_unknown_packs() for what makes a pack
|
||||||
|
* "unknown") in the MIDX before the repack, keep them
|
||||||
|
* as they may be required to form a reachability
|
||||||
|
* closure if the MIDX is bitmapped.
|
||||||
|
*
|
||||||
|
* For example, a cruft pack can be required to form a
|
||||||
|
* reachability closure if the MIDX is bitmapped and one
|
||||||
|
* or more of the bitmap's selected commits reaches a
|
||||||
|
* once-cruft object that was later made reachable.
|
||||||
|
*/
|
||||||
|
for_each_string_list_item(item, &existing->cruft_packs) {
|
||||||
|
/*
|
||||||
|
* When doing a --geometric repack, there is no
|
||||||
|
* need to check for deleted packs, since we're
|
||||||
|
* by definition not doing an ALL_INTO_ONE
|
||||||
|
* repack (hence no packs will be deleted).
|
||||||
|
* Otherwise we must check for and exclude any
|
||||||
|
* packs which are enqueued for deletion.
|
||||||
|
*
|
||||||
|
* So we could omit the conditional below in the
|
||||||
|
* --geometric case, but doing so is unnecessary
|
||||||
|
* since no packs are marked as pending
|
||||||
|
* deletion (since we only call
|
||||||
|
* `existing_packs_mark_for_deletion()` when
|
||||||
|
* doing an all-into-one repack).
|
||||||
|
*/
|
||||||
|
if (existing_pack_is_marked_for_deletion(item))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
strbuf_reset(&buf);
|
||||||
|
strbuf_addf(&buf, "%s.idx", item->string);
|
||||||
|
string_list_insert(include, buf.buf);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
/*
|
||||||
|
* Modern versions of Git (with the appropriate
|
||||||
|
* configuration setting) will write new copies of
|
||||||
|
* once-cruft objects when doing a --geometric repack.
|
||||||
|
*
|
||||||
|
* If the MIDX has no cruft pack, new packs written
|
||||||
|
* during a --geometric repack will not rely on the
|
||||||
|
* cruft pack to form a reachability closure, so we can
|
||||||
|
* avoid including them in the MIDX in that case.
|
||||||
|
*/
|
||||||
|
;
|
||||||
|
}
|
||||||
|
|
||||||
|
strbuf_release(&buf);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void remove_redundant_bitmaps(struct string_list *include,
|
||||||
|
const char *packdir)
|
||||||
|
{
|
||||||
|
struct strbuf path = STRBUF_INIT;
|
||||||
|
struct string_list_item *item;
|
||||||
|
size_t packdir_len;
|
||||||
|
|
||||||
|
strbuf_addstr(&path, packdir);
|
||||||
|
strbuf_addch(&path, '/');
|
||||||
|
packdir_len = path.len;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Remove any pack bitmaps corresponding to packs which are now
|
||||||
|
* included in the MIDX.
|
||||||
|
*/
|
||||||
|
for_each_string_list_item(item, include) {
|
||||||
|
strbuf_addstr(&path, item->string);
|
||||||
|
strbuf_strip_suffix(&path, ".idx");
|
||||||
|
strbuf_addstr(&path, ".bitmap");
|
||||||
|
|
||||||
|
if (unlink(path.buf) && errno != ENOENT)
|
||||||
|
warning_errno(_("could not remove stale bitmap: %s"),
|
||||||
|
path.buf);
|
||||||
|
|
||||||
|
strbuf_setlen(&path, packdir_len);
|
||||||
|
}
|
||||||
|
strbuf_release(&path);
|
||||||
|
}
|
||||||
|
|
||||||
|
int write_midx_included_packs(struct repack_write_midx_opts *opts)
|
||||||
|
{
|
||||||
|
struct child_process cmd = CHILD_PROCESS_INIT;
|
||||||
|
struct string_list include = STRING_LIST_INIT_DUP;
|
||||||
|
struct string_list_item *item;
|
||||||
|
struct packed_git *preferred = pack_geometry_preferred_pack(opts->geometry);
|
||||||
|
FILE *in;
|
||||||
|
int ret = 0;
|
||||||
|
|
||||||
|
midx_included_packs(&include, opts);
|
||||||
|
if (!include.nr)
|
||||||
|
goto done;
|
||||||
|
|
||||||
|
cmd.in = -1;
|
||||||
|
cmd.git_cmd = 1;
|
||||||
|
|
||||||
|
strvec_push(&cmd.args, "multi-pack-index");
|
||||||
|
strvec_pushl(&cmd.args, "write", "--stdin-packs", NULL);
|
||||||
|
|
||||||
|
if (opts->show_progress)
|
||||||
|
strvec_push(&cmd.args, "--progress");
|
||||||
|
else
|
||||||
|
strvec_push(&cmd.args, "--no-progress");
|
||||||
|
|
||||||
|
if (opts->write_bitmaps)
|
||||||
|
strvec_push(&cmd.args, "--bitmap");
|
||||||
|
|
||||||
|
if (preferred)
|
||||||
|
strvec_pushf(&cmd.args, "--preferred-pack=%s",
|
||||||
|
pack_basename(preferred));
|
||||||
|
else if (opts->names->nr) {
|
||||||
|
/* The largest pack was repacked, meaning that either
|
||||||
|
* one or two packs exist depending on whether the
|
||||||
|
* repository has a cruft pack or not.
|
||||||
|
*
|
||||||
|
* Select the non-cruft one as preferred to encourage
|
||||||
|
* pack-reuse among packs containing reachable objects
|
||||||
|
* over unreachable ones.
|
||||||
|
*
|
||||||
|
* (Note we could write multiple packs here if
|
||||||
|
* `--max-pack-size` was given, but any one of them
|
||||||
|
* will suffice, so pick the first one.)
|
||||||
|
*/
|
||||||
|
for_each_string_list_item(item, opts->names) {
|
||||||
|
struct generated_pack *pack = item->util;
|
||||||
|
if (generated_pack_has_ext(pack, ".mtimes"))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
strvec_pushf(&cmd.args, "--preferred-pack=pack-%s.pack",
|
||||||
|
item->string);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
/*
|
||||||
|
* No packs were kept, and no packs were written. The
|
||||||
|
* only thing remaining are .keep packs (unless
|
||||||
|
* --pack-kept-objects was given).
|
||||||
|
*
|
||||||
|
* Set the `--preferred-pack` arbitrarily here.
|
||||||
|
*/
|
||||||
|
;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (opts->refs_snapshot)
|
||||||
|
strvec_pushf(&cmd.args, "--refs-snapshot=%s",
|
||||||
|
opts->refs_snapshot);
|
||||||
|
|
||||||
|
ret = start_command(&cmd);
|
||||||
|
if (ret)
|
||||||
|
goto done;
|
||||||
|
|
||||||
|
in = xfdopen(cmd.in, "w");
|
||||||
|
for_each_string_list_item(item, &include)
|
||||||
|
fprintf(in, "%s\n", item->string);
|
||||||
|
fclose(in);
|
||||||
|
|
||||||
|
ret = finish_command(&cmd);
|
||||||
|
done:
|
||||||
|
if (!ret && opts->write_bitmaps)
|
||||||
|
remove_redundant_bitmaps(&include, opts->packdir);
|
||||||
|
|
||||||
|
string_list_clear(&include, 0);
|
||||||
|
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,102 @@
|
||||||
|
#include "git-compat-util.h"
|
||||||
|
#include "repack.h"
|
||||||
|
#include "run-command.h"
|
||||||
|
#include "hex.h"
|
||||||
|
#include "repository.h"
|
||||||
|
#include "packfile.h"
|
||||||
|
#include "path.h"
|
||||||
|
#include "pack.h"
|
||||||
|
|
||||||
|
struct write_oid_context {
|
||||||
|
struct child_process *cmd;
|
||||||
|
const struct git_hash_algo *algop;
|
||||||
|
};
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Write oid to the given struct child_process's stdin, starting it first if
|
||||||
|
* necessary.
|
||||||
|
*/
|
||||||
|
static int write_oid(const struct object_id *oid,
|
||||||
|
struct packed_git *pack UNUSED,
|
||||||
|
uint32_t pos UNUSED, void *data)
|
||||||
|
{
|
||||||
|
struct write_oid_context *ctx = data;
|
||||||
|
struct child_process *cmd = ctx->cmd;
|
||||||
|
|
||||||
|
if (cmd->in == -1) {
|
||||||
|
if (start_command(cmd))
|
||||||
|
die(_("could not start pack-objects to repack promisor objects"));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (write_in_full(cmd->in, oid_to_hex(oid), ctx->algop->hexsz) < 0 ||
|
||||||
|
write_in_full(cmd->in, "\n", 1) < 0)
|
||||||
|
die(_("failed to feed promisor objects to pack-objects"));
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
void repack_promisor_objects(struct repository *repo,
|
||||||
|
const struct pack_objects_args *args,
|
||||||
|
struct string_list *names, const char *packtmp)
|
||||||
|
{
|
||||||
|
struct write_oid_context ctx;
|
||||||
|
struct child_process cmd = CHILD_PROCESS_INIT;
|
||||||
|
FILE *out;
|
||||||
|
struct strbuf line = STRBUF_INIT;
|
||||||
|
|
||||||
|
prepare_pack_objects(&cmd, args, packtmp);
|
||||||
|
cmd.in = -1;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* NEEDSWORK: Giving pack-objects only the OIDs without any ordering
|
||||||
|
* hints may result in suboptimal deltas in the resulting pack. See if
|
||||||
|
* the OIDs can be sent with fake paths such that pack-objects can use a
|
||||||
|
* {type -> existing pack order} ordering when computing deltas instead
|
||||||
|
* of a {type -> size} ordering, which may produce better deltas.
|
||||||
|
*/
|
||||||
|
ctx.cmd = &cmd;
|
||||||
|
ctx.algop = repo->hash_algo;
|
||||||
|
for_each_packed_object(repo, write_oid, &ctx,
|
||||||
|
FOR_EACH_OBJECT_PROMISOR_ONLY);
|
||||||
|
|
||||||
|
if (cmd.in == -1) {
|
||||||
|
/* No packed objects; cmd was never started */
|
||||||
|
child_process_clear(&cmd);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
close(cmd.in);
|
||||||
|
|
||||||
|
out = xfdopen(cmd.out, "r");
|
||||||
|
while (strbuf_getline_lf(&line, out) != EOF) {
|
||||||
|
struct string_list_item *item;
|
||||||
|
char *promisor_name;
|
||||||
|
|
||||||
|
if (line.len != repo->hash_algo->hexsz)
|
||||||
|
die(_("repack: Expecting full hex object ID lines only from pack-objects."));
|
||||||
|
item = string_list_append(names, line.buf);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* pack-objects creates the .pack and .idx files, but not the
|
||||||
|
* .promisor file. Create the .promisor file, which is empty.
|
||||||
|
*
|
||||||
|
* NEEDSWORK: fetch-pack sometimes generates non-empty
|
||||||
|
* .promisor files containing the ref names and associated
|
||||||
|
* hashes at the point of generation of the corresponding
|
||||||
|
* packfile, but this would not preserve their contents. Maybe
|
||||||
|
* concatenate the contents of all .promisor files instead of
|
||||||
|
* just creating a new empty file.
|
||||||
|
*/
|
||||||
|
promisor_name = mkpathdup("%s-%s.promisor", packtmp,
|
||||||
|
line.buf);
|
||||||
|
write_promisor_file(promisor_name, NULL, 0);
|
||||||
|
|
||||||
|
item->util = generated_pack_populate(item->string, packtmp);
|
||||||
|
|
||||||
|
free(promisor_name);
|
||||||
|
}
|
||||||
|
|
||||||
|
fclose(out);
|
||||||
|
if (finish_command(&cmd))
|
||||||
|
die(_("could not finish pack-objects to repack promisor objects"));
|
||||||
|
strbuf_release(&line);
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,361 @@
|
||||||
|
#include "git-compat-util.h"
|
||||||
|
#include "dir.h"
|
||||||
|
#include "midx.h"
|
||||||
|
#include "odb.h"
|
||||||
|
#include "packfile.h"
|
||||||
|
#include "path.h"
|
||||||
|
#include "repack.h"
|
||||||
|
#include "repository.h"
|
||||||
|
#include "run-command.h"
|
||||||
|
#include "tempfile.h"
|
||||||
|
|
||||||
|
void prepare_pack_objects(struct child_process *cmd,
|
||||||
|
const struct pack_objects_args *args,
|
||||||
|
const char *out)
|
||||||
|
{
|
||||||
|
strvec_push(&cmd->args, "pack-objects");
|
||||||
|
if (args->window)
|
||||||
|
strvec_pushf(&cmd->args, "--window=%s", args->window);
|
||||||
|
if (args->window_memory)
|
||||||
|
strvec_pushf(&cmd->args, "--window-memory=%s", args->window_memory);
|
||||||
|
if (args->depth)
|
||||||
|
strvec_pushf(&cmd->args, "--depth=%s", args->depth);
|
||||||
|
if (args->threads)
|
||||||
|
strvec_pushf(&cmd->args, "--threads=%s", args->threads);
|
||||||
|
if (args->max_pack_size)
|
||||||
|
strvec_pushf(&cmd->args, "--max-pack-size=%lu", args->max_pack_size);
|
||||||
|
if (args->no_reuse_delta)
|
||||||
|
strvec_pushf(&cmd->args, "--no-reuse-delta");
|
||||||
|
if (args->no_reuse_object)
|
||||||
|
strvec_pushf(&cmd->args, "--no-reuse-object");
|
||||||
|
if (args->name_hash_version)
|
||||||
|
strvec_pushf(&cmd->args, "--name-hash-version=%d", args->name_hash_version);
|
||||||
|
if (args->path_walk)
|
||||||
|
strvec_pushf(&cmd->args, "--path-walk");
|
||||||
|
if (args->local)
|
||||||
|
strvec_push(&cmd->args, "--local");
|
||||||
|
if (args->quiet)
|
||||||
|
strvec_push(&cmd->args, "--quiet");
|
||||||
|
if (args->delta_base_offset)
|
||||||
|
strvec_push(&cmd->args, "--delta-base-offset");
|
||||||
|
if (!args->pack_kept_objects)
|
||||||
|
strvec_push(&cmd->args, "--honor-pack-keep");
|
||||||
|
strvec_push(&cmd->args, out);
|
||||||
|
cmd->git_cmd = 1;
|
||||||
|
cmd->out = -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
void pack_objects_args_release(struct pack_objects_args *args)
|
||||||
|
{
|
||||||
|
free(args->window);
|
||||||
|
free(args->window_memory);
|
||||||
|
free(args->depth);
|
||||||
|
free(args->threads);
|
||||||
|
list_objects_filter_release(&args->filter_options);
|
||||||
|
}
|
||||||
|
|
||||||
|
void repack_remove_redundant_pack(struct repository *repo, const char *dir_name,
|
||||||
|
const char *base_name)
|
||||||
|
{
|
||||||
|
struct strbuf buf = STRBUF_INIT;
|
||||||
|
struct odb_source *source = repo->objects->sources;
|
||||||
|
struct multi_pack_index *m = get_multi_pack_index(source);
|
||||||
|
strbuf_addf(&buf, "%s.pack", base_name);
|
||||||
|
if (m && source->local && midx_contains_pack(m, buf.buf))
|
||||||
|
clear_midx_file(repo);
|
||||||
|
strbuf_insertf(&buf, 0, "%s/", dir_name);
|
||||||
|
unlink_pack_path(buf.buf, 1);
|
||||||
|
strbuf_release(&buf);
|
||||||
|
}
|
||||||
|
|
||||||
|
const char *write_pack_opts_pack_prefix(struct write_pack_opts *opts)
|
||||||
|
{
|
||||||
|
const char *pack_prefix;
|
||||||
|
if (!skip_prefix(opts->packtmp, opts->packdir, &pack_prefix))
|
||||||
|
die(_("pack prefix %s does not begin with objdir %s"),
|
||||||
|
opts->packtmp, opts->packdir);
|
||||||
|
if (*pack_prefix == '/')
|
||||||
|
pack_prefix++;
|
||||||
|
return pack_prefix;
|
||||||
|
}
|
||||||
|
|
||||||
|
int write_pack_opts_is_local(struct write_pack_opts *opts)
|
||||||
|
{
|
||||||
|
const char *scratch;
|
||||||
|
return skip_prefix(opts->destination, opts->packdir, &scratch);
|
||||||
|
}
|
||||||
|
|
||||||
|
int finish_pack_objects_cmd(const struct git_hash_algo *algop,
|
||||||
|
struct write_pack_opts *opts,
|
||||||
|
struct child_process *cmd,
|
||||||
|
struct string_list *names)
|
||||||
|
{
|
||||||
|
FILE *out;
|
||||||
|
int local = write_pack_opts_is_local(opts);
|
||||||
|
struct strbuf line = STRBUF_INIT;
|
||||||
|
|
||||||
|
out = xfdopen(cmd->out, "r");
|
||||||
|
while (strbuf_getline_lf(&line, out) != EOF) {
|
||||||
|
struct string_list_item *item;
|
||||||
|
|
||||||
|
if (line.len != algop->hexsz)
|
||||||
|
die(_("repack: Expecting full hex object ID lines only "
|
||||||
|
"from pack-objects."));
|
||||||
|
/*
|
||||||
|
* Avoid putting packs written outside of the repository in the
|
||||||
|
* list of names.
|
||||||
|
*/
|
||||||
|
if (local) {
|
||||||
|
item = string_list_append(names, line.buf);
|
||||||
|
item->util = generated_pack_populate(line.buf,
|
||||||
|
opts->packtmp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fclose(out);
|
||||||
|
|
||||||
|
strbuf_release(&line);
|
||||||
|
|
||||||
|
return finish_command(cmd);
|
||||||
|
}
|
||||||
|
|
||||||
|
#define DELETE_PACK 1
|
||||||
|
#define RETAIN_PACK 2
|
||||||
|
|
||||||
|
void existing_packs_collect(struct existing_packs *existing,
|
||||||
|
const struct string_list *extra_keep)
|
||||||
|
{
|
||||||
|
struct packfile_store *packs = existing->repo->objects->packfiles;
|
||||||
|
struct packed_git *p;
|
||||||
|
struct strbuf buf = STRBUF_INIT;
|
||||||
|
|
||||||
|
for (p = packfile_store_get_all_packs(packs); p; p = p->next) {
|
||||||
|
size_t i;
|
||||||
|
const char *base;
|
||||||
|
|
||||||
|
if (p->multi_pack_index)
|
||||||
|
string_list_append(&existing->midx_packs,
|
||||||
|
pack_basename(p));
|
||||||
|
if (!p->pack_local)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
base = pack_basename(p);
|
||||||
|
|
||||||
|
for (i = 0; i < extra_keep->nr; i++)
|
||||||
|
if (!fspathcmp(base, extra_keep->items[i].string))
|
||||||
|
break;
|
||||||
|
|
||||||
|
strbuf_reset(&buf);
|
||||||
|
strbuf_addstr(&buf, base);
|
||||||
|
strbuf_strip_suffix(&buf, ".pack");
|
||||||
|
|
||||||
|
if ((extra_keep->nr > 0 && i < extra_keep->nr) || p->pack_keep)
|
||||||
|
string_list_append(&existing->kept_packs, buf.buf);
|
||||||
|
else if (p->is_cruft)
|
||||||
|
string_list_append(&existing->cruft_packs, buf.buf);
|
||||||
|
else
|
||||||
|
string_list_append(&existing->non_kept_packs, buf.buf);
|
||||||
|
}
|
||||||
|
|
||||||
|
string_list_sort(&existing->kept_packs);
|
||||||
|
string_list_sort(&existing->non_kept_packs);
|
||||||
|
string_list_sort(&existing->cruft_packs);
|
||||||
|
string_list_sort(&existing->midx_packs);
|
||||||
|
strbuf_release(&buf);
|
||||||
|
}
|
||||||
|
|
||||||
|
int existing_packs_has_non_kept(const struct existing_packs *existing)
|
||||||
|
{
|
||||||
|
return existing->non_kept_packs.nr || existing->cruft_packs.nr;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void existing_pack_mark_for_deletion(struct string_list_item *item)
|
||||||
|
{
|
||||||
|
item->util = (void*)((uintptr_t)item->util | DELETE_PACK);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void existing_pack_unmark_for_deletion(struct string_list_item *item)
|
||||||
|
{
|
||||||
|
item->util = (void*)((uintptr_t)item->util & ~DELETE_PACK);
|
||||||
|
}
|
||||||
|
|
||||||
|
int existing_pack_is_marked_for_deletion(struct string_list_item *item)
|
||||||
|
{
|
||||||
|
return (uintptr_t)item->util & DELETE_PACK;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void existing_packs_mark_retained(struct string_list_item *item)
|
||||||
|
{
|
||||||
|
item->util = (void*)((uintptr_t)item->util | RETAIN_PACK);
|
||||||
|
}
|
||||||
|
|
||||||
|
static int existing_pack_is_retained(struct string_list_item *item)
|
||||||
|
{
|
||||||
|
return (uintptr_t)item->util & RETAIN_PACK;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void existing_packs_mark_for_deletion_1(const struct git_hash_algo *algop,
|
||||||
|
struct string_list *names,
|
||||||
|
struct string_list *list)
|
||||||
|
{
|
||||||
|
struct string_list_item *item;
|
||||||
|
const size_t hexsz = algop->hexsz;
|
||||||
|
|
||||||
|
for_each_string_list_item(item, list) {
|
||||||
|
char *sha1;
|
||||||
|
size_t len = strlen(item->string);
|
||||||
|
if (len < hexsz)
|
||||||
|
continue;
|
||||||
|
sha1 = item->string + len - hexsz;
|
||||||
|
|
||||||
|
if (existing_pack_is_retained(item)) {
|
||||||
|
existing_pack_unmark_for_deletion(item);
|
||||||
|
} else if (!string_list_has_string(names, sha1)) {
|
||||||
|
/*
|
||||||
|
* Mark this pack for deletion, which ensures
|
||||||
|
* that this pack won't be included in a MIDX
|
||||||
|
* (if `--write-midx` was given) and that we
|
||||||
|
* will actually delete this pack (if `-d` was
|
||||||
|
* given).
|
||||||
|
*/
|
||||||
|
existing_pack_mark_for_deletion(item);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void existing_packs_retain_cruft(struct existing_packs *existing,
|
||||||
|
struct packed_git *cruft)
|
||||||
|
{
|
||||||
|
struct strbuf buf = STRBUF_INIT;
|
||||||
|
struct string_list_item *item;
|
||||||
|
|
||||||
|
strbuf_addstr(&buf, pack_basename(cruft));
|
||||||
|
strbuf_strip_suffix(&buf, ".pack");
|
||||||
|
|
||||||
|
item = string_list_lookup(&existing->cruft_packs, buf.buf);
|
||||||
|
if (!item)
|
||||||
|
BUG("could not find cruft pack '%s'", pack_basename(cruft));
|
||||||
|
|
||||||
|
existing_packs_mark_retained(item);
|
||||||
|
strbuf_release(&buf);
|
||||||
|
}
|
||||||
|
|
||||||
|
void existing_packs_mark_for_deletion(struct existing_packs *existing,
|
||||||
|
struct string_list *names)
|
||||||
|
|
||||||
|
{
|
||||||
|
const struct git_hash_algo *algop = existing->repo->hash_algo;
|
||||||
|
existing_packs_mark_for_deletion_1(algop, names,
|
||||||
|
&existing->non_kept_packs);
|
||||||
|
existing_packs_mark_for_deletion_1(algop, names,
|
||||||
|
&existing->cruft_packs);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void remove_redundant_packs_1(struct repository *repo,
|
||||||
|
struct string_list *packs,
|
||||||
|
const char *packdir)
|
||||||
|
{
|
||||||
|
struct string_list_item *item;
|
||||||
|
for_each_string_list_item(item, packs) {
|
||||||
|
if (!existing_pack_is_marked_for_deletion(item))
|
||||||
|
continue;
|
||||||
|
repack_remove_redundant_pack(repo, packdir, item->string);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void existing_packs_remove_redundant(struct existing_packs *existing,
|
||||||
|
const char *packdir)
|
||||||
|
{
|
||||||
|
remove_redundant_packs_1(existing->repo, &existing->non_kept_packs,
|
||||||
|
packdir);
|
||||||
|
remove_redundant_packs_1(existing->repo, &existing->cruft_packs,
|
||||||
|
packdir);
|
||||||
|
}
|
||||||
|
|
||||||
|
void existing_packs_release(struct existing_packs *existing)
|
||||||
|
{
|
||||||
|
string_list_clear(&existing->kept_packs, 0);
|
||||||
|
string_list_clear(&existing->non_kept_packs, 0);
|
||||||
|
string_list_clear(&existing->cruft_packs, 0);
|
||||||
|
string_list_clear(&existing->midx_packs, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
static struct {
|
||||||
|
const char *name;
|
||||||
|
unsigned optional:1;
|
||||||
|
} exts[] = {
|
||||||
|
{".pack"},
|
||||||
|
{".rev", 1},
|
||||||
|
{".mtimes", 1},
|
||||||
|
{".bitmap", 1},
|
||||||
|
{".promisor", 1},
|
||||||
|
{".idx"},
|
||||||
|
};
|
||||||
|
|
||||||
|
struct generated_pack {
|
||||||
|
struct tempfile *tempfiles[ARRAY_SIZE(exts)];
|
||||||
|
};
|
||||||
|
|
||||||
|
struct generated_pack *generated_pack_populate(const char *name,
|
||||||
|
const char *packtmp)
|
||||||
|
{
|
||||||
|
struct stat statbuf;
|
||||||
|
struct strbuf path = STRBUF_INIT;
|
||||||
|
struct generated_pack *pack = xcalloc(1, sizeof(*pack));
|
||||||
|
size_t i;
|
||||||
|
|
||||||
|
for (i = 0; i < ARRAY_SIZE(exts); i++) {
|
||||||
|
strbuf_reset(&path);
|
||||||
|
strbuf_addf(&path, "%s-%s%s", packtmp, name, exts[i].name);
|
||||||
|
|
||||||
|
if (stat(path.buf, &statbuf))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
pack->tempfiles[i] = register_tempfile(path.buf);
|
||||||
|
}
|
||||||
|
|
||||||
|
strbuf_release(&path);
|
||||||
|
return pack;
|
||||||
|
}
|
||||||
|
|
||||||
|
int generated_pack_has_ext(const struct generated_pack *pack, const char *ext)
|
||||||
|
{
|
||||||
|
size_t i;
|
||||||
|
for (i = 0; i < ARRAY_SIZE(exts); i++) {
|
||||||
|
if (strcmp(exts[i].name, ext))
|
||||||
|
continue;
|
||||||
|
return !!pack->tempfiles[i];
|
||||||
|
}
|
||||||
|
BUG("unknown pack extension: '%s'", ext);
|
||||||
|
}
|
||||||
|
|
||||||
|
void generated_pack_install(struct generated_pack *pack, const char *name,
|
||||||
|
const char *packdir, const char *packtmp)
|
||||||
|
{
|
||||||
|
size_t ext;
|
||||||
|
for (ext = 0; ext < ARRAY_SIZE(exts); ext++) {
|
||||||
|
char *fname;
|
||||||
|
|
||||||
|
fname = mkpathdup("%s/pack-%s%s", packdir, name,
|
||||||
|
exts[ext].name);
|
||||||
|
|
||||||
|
if (pack->tempfiles[ext]) {
|
||||||
|
const char *fname_old = get_tempfile_path(pack->tempfiles[ext]);
|
||||||
|
struct stat statbuffer;
|
||||||
|
|
||||||
|
if (!stat(fname_old, &statbuffer)) {
|
||||||
|
statbuffer.st_mode &= ~(S_IWUSR | S_IWGRP | S_IWOTH);
|
||||||
|
chmod(fname_old, statbuffer.st_mode);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (rename_tempfile(&pack->tempfiles[ext], fname))
|
||||||
|
die_errno(_("renaming pack to '%s' failed"),
|
||||||
|
fname);
|
||||||
|
} else if (!exts[ext].optional)
|
||||||
|
die(_("pack-objects did not write a '%s' file for pack %s-%s"),
|
||||||
|
exts[ext].name, packtmp, name);
|
||||||
|
else if (unlink(fname) < 0 && errno != ENOENT)
|
||||||
|
die_errno(_("could not unlink: %s"), fname);
|
||||||
|
|
||||||
|
free(fname);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,146 @@
|
||||||
|
#ifndef REPACK_H
|
||||||
|
#define REPACK_H
|
||||||
|
|
||||||
|
#include "list-objects-filter-options.h"
|
||||||
|
#include "string-list.h"
|
||||||
|
|
||||||
|
struct pack_objects_args {
|
||||||
|
char *window;
|
||||||
|
char *window_memory;
|
||||||
|
char *depth;
|
||||||
|
char *threads;
|
||||||
|
unsigned long max_pack_size;
|
||||||
|
int no_reuse_delta;
|
||||||
|
int no_reuse_object;
|
||||||
|
int quiet;
|
||||||
|
int local;
|
||||||
|
int name_hash_version;
|
||||||
|
int path_walk;
|
||||||
|
int delta_base_offset;
|
||||||
|
int pack_kept_objects;
|
||||||
|
struct list_objects_filter_options filter_options;
|
||||||
|
};
|
||||||
|
|
||||||
|
#define PACK_OBJECTS_ARGS_INIT { \
|
||||||
|
.delta_base_offset = 1, \
|
||||||
|
.pack_kept_objects = -1, \
|
||||||
|
}
|
||||||
|
|
||||||
|
struct child_process;
|
||||||
|
|
||||||
|
void prepare_pack_objects(struct child_process *cmd,
|
||||||
|
const struct pack_objects_args *args,
|
||||||
|
const char *out);
|
||||||
|
void pack_objects_args_release(struct pack_objects_args *args);
|
||||||
|
|
||||||
|
void repack_remove_redundant_pack(struct repository *repo, const char *dir_name,
|
||||||
|
const char *base_name);
|
||||||
|
|
||||||
|
struct write_pack_opts {
|
||||||
|
struct pack_objects_args *po_args;
|
||||||
|
const char *destination;
|
||||||
|
const char *packdir;
|
||||||
|
const char *packtmp;
|
||||||
|
};
|
||||||
|
|
||||||
|
const char *write_pack_opts_pack_prefix(struct write_pack_opts *opts);
|
||||||
|
int write_pack_opts_is_local(struct write_pack_opts *opts);
|
||||||
|
|
||||||
|
int finish_pack_objects_cmd(const struct git_hash_algo *algop,
|
||||||
|
struct write_pack_opts *opts,
|
||||||
|
struct child_process *cmd,
|
||||||
|
struct string_list *names);
|
||||||
|
|
||||||
|
struct repository;
|
||||||
|
struct packed_git;
|
||||||
|
|
||||||
|
struct existing_packs {
|
||||||
|
struct repository *repo;
|
||||||
|
struct string_list kept_packs;
|
||||||
|
struct string_list non_kept_packs;
|
||||||
|
struct string_list cruft_packs;
|
||||||
|
struct string_list midx_packs;
|
||||||
|
};
|
||||||
|
|
||||||
|
#define EXISTING_PACKS_INIT { \
|
||||||
|
.kept_packs = STRING_LIST_INIT_DUP, \
|
||||||
|
.non_kept_packs = STRING_LIST_INIT_DUP, \
|
||||||
|
.cruft_packs = STRING_LIST_INIT_DUP, \
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Adds all packs hex strings (pack-$HASH) to either packs->non_kept
|
||||||
|
* or packs->kept based on whether each pack has a corresponding
|
||||||
|
* .keep file or not. Packs without a .keep file are not to be kept
|
||||||
|
* if we are going to pack everything into one file.
|
||||||
|
*/
|
||||||
|
void existing_packs_collect(struct existing_packs *existing,
|
||||||
|
const struct string_list *extra_keep);
|
||||||
|
int existing_packs_has_non_kept(const struct existing_packs *existing);
|
||||||
|
int existing_pack_is_marked_for_deletion(struct string_list_item *item);
|
||||||
|
void existing_packs_retain_cruft(struct existing_packs *existing,
|
||||||
|
struct packed_git *cruft);
|
||||||
|
void existing_packs_mark_for_deletion(struct existing_packs *existing,
|
||||||
|
struct string_list *names);
|
||||||
|
void existing_packs_remove_redundant(struct existing_packs *existing,
|
||||||
|
const char *packdir);
|
||||||
|
void existing_packs_release(struct existing_packs *existing);
|
||||||
|
|
||||||
|
struct generated_pack;
|
||||||
|
|
||||||
|
struct generated_pack *generated_pack_populate(const char *name,
|
||||||
|
const char *packtmp);
|
||||||
|
int generated_pack_has_ext(const struct generated_pack *pack, const char *ext);
|
||||||
|
void generated_pack_install(struct generated_pack *pack, const char *name,
|
||||||
|
const char *packdir, const char *packtmp);
|
||||||
|
|
||||||
|
void repack_promisor_objects(struct repository *repo,
|
||||||
|
const struct pack_objects_args *args,
|
||||||
|
struct string_list *names, const char *packtmp);
|
||||||
|
|
||||||
|
struct pack_geometry {
|
||||||
|
struct packed_git **pack;
|
||||||
|
uint32_t pack_nr, pack_alloc;
|
||||||
|
uint32_t split;
|
||||||
|
|
||||||
|
int split_factor;
|
||||||
|
};
|
||||||
|
|
||||||
|
void pack_geometry_init(struct pack_geometry *geometry,
|
||||||
|
struct existing_packs *existing,
|
||||||
|
const struct pack_objects_args *args);
|
||||||
|
void pack_geometry_split(struct pack_geometry *geometry);
|
||||||
|
struct packed_git *pack_geometry_preferred_pack(struct pack_geometry *geometry);
|
||||||
|
void pack_geometry_remove_redundant(struct pack_geometry *geometry,
|
||||||
|
struct string_list *names,
|
||||||
|
struct existing_packs *existing,
|
||||||
|
const char *packdir);
|
||||||
|
void pack_geometry_release(struct pack_geometry *geometry);
|
||||||
|
|
||||||
|
struct tempfile;
|
||||||
|
|
||||||
|
struct repack_write_midx_opts {
|
||||||
|
struct existing_packs *existing;
|
||||||
|
struct pack_geometry *geometry;
|
||||||
|
struct string_list *names;
|
||||||
|
const char *refs_snapshot;
|
||||||
|
const char *packdir;
|
||||||
|
int show_progress;
|
||||||
|
int write_bitmaps;
|
||||||
|
int midx_must_contain_cruft;
|
||||||
|
};
|
||||||
|
|
||||||
|
void midx_snapshot_refs(struct repository *repo, struct tempfile *f);
|
||||||
|
int write_midx_included_packs(struct repack_write_midx_opts *opts);
|
||||||
|
|
||||||
|
int write_filtered_pack(struct write_pack_opts *opts,
|
||||||
|
struct existing_packs *existing,
|
||||||
|
struct string_list *names);
|
||||||
|
|
||||||
|
int write_cruft_pack(struct write_pack_opts *opts,
|
||||||
|
const char *cruft_expiration,
|
||||||
|
unsigned long combine_cruft_below_size,
|
||||||
|
struct string_list *names,
|
||||||
|
struct existing_packs *existing);
|
||||||
|
|
||||||
|
#endif /* REPACK_H */
|
||||||
Loading…
Reference in New Issue