Merge branch 'jt/receive-pack-pluggable-writes'
The 'git receive-pack' command has been updated to use a new ODB transaction interface for writing incoming packfiles, making it more backend-agnostic. * jt/receive-pack-pluggable-writes: odb/transaction: add transaction interface to write packfiles odb: return temporary ODB source when set builtin/receive-pack: explicitly pass packfile fd builtin/receive-pack: report unpack errors via strbuf builtin/receive-pack: lift global state out of unpack() builtin/receive-pack: read unpack limit config lazily builtin/receive-pack: pass shallow file explicitly odb/transaction: add transaction finalize interface builtin/receive-pack: properly clean up keep filesmain
Junio C Hamano
commit
26e1e47b47
|
|
@ -466,7 +466,7 @@ int cmd_add(int argc,
|
|||
char *seen = NULL;
|
||||
char *ps_matched = NULL;
|
||||
struct lock_file lock_file = LOCK_INIT;
|
||||
struct odb_transaction *transaction;
|
||||
struct odb_transaction *transaction = NULL;
|
||||
|
||||
repo_config(repo, add_config, NULL);
|
||||
|
||||
|
|
@ -680,7 +680,7 @@ int cmd_add(int argc,
|
|||
|
||||
if (chmod_arg && pathspec.nr)
|
||||
exit_status |= chmod_pathspec(repo, &pathspec, chmod_arg[0], show_only);
|
||||
odb_transaction_commit(transaction);
|
||||
odb_transaction_commit_and_finalize_or_die(transaction);
|
||||
|
||||
finish:
|
||||
if (write_locked_index(repo->index, &lock_file,
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@
|
|||
#include "gpg-interface.h"
|
||||
#include "hex.h"
|
||||
#include "hook.h"
|
||||
#include "lockfile.h"
|
||||
#include "object.h"
|
||||
#include "object-file.h"
|
||||
#include "object-name.h"
|
||||
|
|
@ -23,7 +22,6 @@
|
|||
#include "oid-array.h"
|
||||
#include "oidset.h"
|
||||
#include "pack.h"
|
||||
#include "packfile.h"
|
||||
#include "parse-options.h"
|
||||
#include "pkt-line.h"
|
||||
#include "protocol.h"
|
||||
|
|
@ -62,12 +60,9 @@ static enum deny_action deny_delete_current = DENY_UNCONFIGURED;
|
|||
static int receive_fsck_objects = -1;
|
||||
static int transfer_fsck_objects = -1;
|
||||
static struct strbuf fsck_msg_types = STRBUF_INIT;
|
||||
static int receive_unpack_limit = -1;
|
||||
static int transfer_unpack_limit = -1;
|
||||
static int advertise_atomic_push = 1;
|
||||
static int advertise_push_options;
|
||||
static int advertise_sid;
|
||||
static int unpack_limit = 100;
|
||||
static off_t max_input_size;
|
||||
static int report_status;
|
||||
static int report_status_v2;
|
||||
|
|
@ -86,7 +81,6 @@ static const char *head_name;
|
|||
static void *head_name_to_free;
|
||||
static int sent_capabilities;
|
||||
static int shallow_update;
|
||||
static const char *alt_shallow_file;
|
||||
static struct strbuf push_cert = STRBUF_INIT;
|
||||
static struct object_id push_cert_oid;
|
||||
static struct signature_check sigcheck;
|
||||
|
|
@ -158,16 +152,6 @@ static int receive_pack_config(const char *var, const char *value,
|
|||
return 0;
|
||||
}
|
||||
|
||||
if (strcmp(var, "receive.unpacklimit") == 0) {
|
||||
receive_unpack_limit = git_config_int(var, value, ctx->kvi);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (strcmp(var, "transfer.unpacklimit") == 0) {
|
||||
transfer_unpack_limit = git_config_int(var, value, ctx->kvi);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (strcmp(var, "receive.fsck.skiplist") == 0) {
|
||||
char *path;
|
||||
|
||||
|
|
@ -2029,7 +2013,7 @@ cleanup:
|
|||
}
|
||||
|
||||
static void execute_commands(struct command *commands,
|
||||
const char *unpacker_error,
|
||||
int unpacker_error,
|
||||
struct shallow_info *si,
|
||||
struct odb_transaction *transaction,
|
||||
const struct string_list *push_options)
|
||||
|
|
@ -2306,150 +2290,44 @@ static void read_push_options(struct packet_reader *reader,
|
|||
}
|
||||
}
|
||||
|
||||
static const char *parse_pack_header(struct pack_header *hdr)
|
||||
static int unpack_with_sideband(struct odb_transaction *transaction,
|
||||
const char *shallow_file,
|
||||
struct strbuf *err_msg)
|
||||
{
|
||||
switch (read_pack_header(0, hdr)) {
|
||||
case PH_ERROR_EOF:
|
||||
return "eof before pack header was fully read";
|
||||
|
||||
case PH_ERROR_PACK_SIGNATURE:
|
||||
return "protocol error (pack signature mismatch detected)";
|
||||
|
||||
case PH_ERROR_PROTOCOL:
|
||||
return "protocol error (pack version unsupported)";
|
||||
|
||||
default:
|
||||
return "unknown error in parse_pack_header";
|
||||
|
||||
case 0:
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static struct tempfile *pack_lockfile;
|
||||
|
||||
static void push_header_arg(struct strvec *args, struct pack_header *hdr)
|
||||
{
|
||||
strvec_pushf(args, "--pack_header=%"PRIu32",%"PRIu32,
|
||||
ntohl(hdr->hdr_version), ntohl(hdr->hdr_entries));
|
||||
}
|
||||
|
||||
static const char *unpack(int err_fd, struct shallow_info *si,
|
||||
struct odb_transaction *transaction)
|
||||
{
|
||||
struct pack_header hdr;
|
||||
const char *hdr_err;
|
||||
int status;
|
||||
struct child_process child = CHILD_PROCESS_INIT;
|
||||
int fsck_objects = (receive_fsck_objects >= 0
|
||||
struct odb_transaction_write_pack_opts opts = {
|
||||
.fsck_objects = (receive_fsck_objects >= 0
|
||||
? receive_fsck_objects
|
||||
: transfer_fsck_objects >= 0
|
||||
? transfer_fsck_objects
|
||||
: 0);
|
||||
|
||||
hdr_err = parse_pack_header(&hdr);
|
||||
if (hdr_err) {
|
||||
if (err_fd > 0)
|
||||
close(err_fd);
|
||||
return hdr_err;
|
||||
}
|
||||
|
||||
if (si->nr_ours || si->nr_theirs) {
|
||||
alt_shallow_file = setup_temporary_shallow(si->shallow);
|
||||
strvec_push(&child.args, "--shallow-file");
|
||||
strvec_push(&child.args, alt_shallow_file);
|
||||
}
|
||||
|
||||
odb_transaction_env(transaction, &child.env);
|
||||
|
||||
if (ntohl(hdr.hdr_entries) < unpack_limit) {
|
||||
strvec_push(&child.args, "unpack-objects");
|
||||
push_header_arg(&child.args, &hdr);
|
||||
if (quiet)
|
||||
strvec_push(&child.args, "-q");
|
||||
if (fsck_objects)
|
||||
strvec_pushf(&child.args, "--strict%s",
|
||||
fsck_msg_types.buf);
|
||||
if (max_input_size)
|
||||
strvec_pushf(&child.args, "--max-input-size=%"PRIuMAX,
|
||||
(uintmax_t)max_input_size);
|
||||
child.no_stdout = 1;
|
||||
child.err = err_fd;
|
||||
child.git_cmd = 1;
|
||||
status = run_command(&child);
|
||||
if (status)
|
||||
return "unpack-objects abnormal exit";
|
||||
} else {
|
||||
char hostname[HOST_NAME_MAX + 1];
|
||||
char *lockfile;
|
||||
|
||||
strvec_pushl(&child.args, "index-pack", "--stdin", NULL);
|
||||
push_header_arg(&child.args, &hdr);
|
||||
|
||||
if (xgethostname(hostname, sizeof(hostname)))
|
||||
xsnprintf(hostname, sizeof(hostname), "localhost");
|
||||
strvec_pushf(&child.args,
|
||||
"--keep=receive-pack %"PRIuMAX" on %s",
|
||||
(uintmax_t)getpid(),
|
||||
hostname);
|
||||
|
||||
if (!quiet && err_fd)
|
||||
strvec_push(&child.args, "--show-resolving-progress");
|
||||
if (use_sideband)
|
||||
strvec_push(&child.args, "--report-end-of-input");
|
||||
if (fsck_objects)
|
||||
strvec_pushf(&child.args, "--strict%s",
|
||||
fsck_msg_types.buf);
|
||||
if (!reject_thin)
|
||||
strvec_push(&child.args, "--fix-thin");
|
||||
if (max_input_size)
|
||||
strvec_pushf(&child.args, "--max-input-size=%"PRIuMAX,
|
||||
(uintmax_t)max_input_size);
|
||||
child.out = -1;
|
||||
child.err = err_fd;
|
||||
child.git_cmd = 1;
|
||||
status = start_command(&child);
|
||||
if (status)
|
||||
return "index-pack fork failed";
|
||||
|
||||
lockfile = index_pack_lockfile(the_repository, child.out, NULL);
|
||||
if (lockfile) {
|
||||
pack_lockfile = register_tempfile(lockfile);
|
||||
free(lockfile);
|
||||
}
|
||||
close(child.out);
|
||||
|
||||
status = finish_command(&child);
|
||||
if (status)
|
||||
return "index-pack abnormal exit";
|
||||
odb_reprepare(the_repository->objects);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static const char *unpack_with_sideband(struct shallow_info *si,
|
||||
struct odb_transaction *transaction)
|
||||
{
|
||||
: 0),
|
||||
.fsck_msg_types = fsck_msg_types.buf,
|
||||
.max_input_size = max_input_size,
|
||||
.shallow_file = shallow_file,
|
||||
.reject_thin = reject_thin,
|
||||
.quiet = quiet,
|
||||
};
|
||||
struct async muxer;
|
||||
const char *ret;
|
||||
int ret;
|
||||
|
||||
if (!use_sideband)
|
||||
return unpack(0, si, transaction);
|
||||
return odb_transaction_write_pack(transaction, 0, err_msg, &opts);
|
||||
|
||||
use_keepalive = KEEPALIVE_AFTER_NUL;
|
||||
memset(&muxer, 0, sizeof(muxer));
|
||||
muxer.proc = copy_to_sideband;
|
||||
muxer.in = -1;
|
||||
if (start_async(&muxer))
|
||||
return NULL;
|
||||
return 0;
|
||||
|
||||
ret = unpack(muxer.in, si, transaction);
|
||||
opts.err_fd = muxer.in;
|
||||
ret = odb_transaction_write_pack(transaction, 0, err_msg, &opts);
|
||||
|
||||
finish_async(&muxer);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void prepare_shallow_update(struct shallow_info *si)
|
||||
static void prepare_shallow_update(struct shallow_info *si,
|
||||
const char *shallow_file)
|
||||
{
|
||||
int i, j, k, bitmap_size = DIV_ROUND_UP(si->ref->nr, 32);
|
||||
|
||||
|
|
@ -2489,12 +2367,13 @@ static void prepare_shallow_update(struct shallow_info *si)
|
|||
* command. check_connected() will be done with
|
||||
* true .git/shallow though.
|
||||
*/
|
||||
setenv(GIT_SHALLOW_FILE_ENVIRONMENT, alt_shallow_file, 1);
|
||||
setenv(GIT_SHALLOW_FILE_ENVIRONMENT, shallow_file, 1);
|
||||
}
|
||||
|
||||
static void update_shallow_info(struct command *commands,
|
||||
struct shallow_info *si,
|
||||
struct oid_array *ref)
|
||||
struct oid_array *ref,
|
||||
const char *shallow_file)
|
||||
{
|
||||
struct command *cmd;
|
||||
int *ref_status;
|
||||
|
|
@ -2513,7 +2392,7 @@ static void update_shallow_info(struct command *commands,
|
|||
si->ref = ref;
|
||||
|
||||
if (shallow_update) {
|
||||
prepare_shallow_update(si);
|
||||
prepare_shallow_update(si, shallow_file);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -2530,13 +2409,13 @@ static void update_shallow_info(struct command *commands,
|
|||
free(ref_status);
|
||||
}
|
||||
|
||||
static void report(struct command *commands, const char *unpack_status)
|
||||
static void report(struct command *commands, const struct strbuf *unpack_status)
|
||||
{
|
||||
struct command *cmd;
|
||||
struct strbuf buf = STRBUF_INIT;
|
||||
|
||||
packet_buf_write(&buf, "unpack %s\n",
|
||||
unpack_status ? unpack_status : "ok");
|
||||
unpack_status->len ? unpack_status->buf : "ok");
|
||||
for (cmd = commands; cmd; cmd = cmd->next) {
|
||||
if (!cmd->error_string)
|
||||
packet_buf_write(&buf, "ok %s\n",
|
||||
|
|
@ -2554,14 +2433,14 @@ static void report(struct command *commands, const char *unpack_status)
|
|||
strbuf_release(&buf);
|
||||
}
|
||||
|
||||
static void report_v2(struct command *commands, const char *unpack_status)
|
||||
static void report_v2(struct command *commands, const struct strbuf *unpack_status)
|
||||
{
|
||||
struct command *cmd;
|
||||
struct strbuf buf = STRBUF_INIT;
|
||||
struct ref_push_report *report;
|
||||
|
||||
packet_buf_write(&buf, "unpack %s\n",
|
||||
unpack_status ? unpack_status : "ok");
|
||||
unpack_status->len ? unpack_status->buf : "ok");
|
||||
for (cmd = commands; cmd; cmd = cmd->next) {
|
||||
int count = 0;
|
||||
|
||||
|
|
@ -2652,11 +2531,6 @@ int cmd_receive_pack(int argc,
|
|||
if (cert_nonce_seed)
|
||||
push_cert_nonce = prepare_push_cert_nonce(service_dir, time(NULL));
|
||||
|
||||
if (0 <= receive_unpack_limit)
|
||||
unpack_limit = receive_unpack_limit;
|
||||
else if (0 <= transfer_unpack_limit)
|
||||
unpack_limit = transfer_unpack_limit;
|
||||
|
||||
switch (determine_protocol_version_server()) {
|
||||
case protocol_v2:
|
||||
/*
|
||||
|
|
@ -2690,8 +2564,8 @@ int cmd_receive_pack(int argc,
|
|||
PACKET_READ_DIE_ON_ERR_PACKET);
|
||||
|
||||
if ((commands = read_head_info(&reader, &shallow))) {
|
||||
const char *unpack_status = NULL;
|
||||
struct string_list push_options = STRING_LIST_INIT_DUP;
|
||||
struct strbuf unpack_status = STRBUF_INIT;
|
||||
|
||||
if (use_push_options)
|
||||
read_push_options(&reader, &push_options);
|
||||
|
|
@ -2705,21 +2579,27 @@ int cmd_receive_pack(int argc,
|
|||
if (!si.nr_ours && !si.nr_theirs)
|
||||
shallow_update = 0;
|
||||
if (!delete_only(commands)) {
|
||||
const char *alt_shallow_file = NULL;
|
||||
|
||||
if (si.nr_ours || si.nr_theirs)
|
||||
alt_shallow_file = setup_temporary_shallow(si.shallow);
|
||||
|
||||
if (odb_transaction_begin(the_repository->objects, &transaction, ODB_TRANSACTION_RECEIVE))
|
||||
unpack_status = "unable to start object transaction";
|
||||
strbuf_addstr(&unpack_status, "unable to start object transaction");
|
||||
else
|
||||
unpack_status = unpack_with_sideband(&si, transaction);
|
||||
update_shallow_info(commands, &si, &ref);
|
||||
unpack_with_sideband(transaction, alt_shallow_file, &unpack_status);
|
||||
|
||||
update_shallow_info(commands, &si, &ref, alt_shallow_file);
|
||||
}
|
||||
use_keepalive = KEEPALIVE_ALWAYS;
|
||||
execute_commands(commands, unpack_status, &si, transaction,
|
||||
execute_commands(commands, !!unpack_status.len, &si, transaction,
|
||||
&push_options);
|
||||
delete_tempfile(&pack_lockfile);
|
||||
odb_transaction_finalize(transaction);
|
||||
sigchain_push(SIGPIPE, SIG_IGN);
|
||||
if (report_status_v2)
|
||||
report_v2(commands, unpack_status);
|
||||
report_v2(commands, &unpack_status);
|
||||
else if (report_status)
|
||||
report(commands, unpack_status);
|
||||
report(commands, &unpack_status);
|
||||
sigchain_pop(SIGPIPE);
|
||||
run_receive_hook(commands, "post-receive", 1, NULL,
|
||||
&push_options);
|
||||
|
|
@ -2744,6 +2624,7 @@ int cmd_receive_pack(int argc,
|
|||
if (auto_update_server_info)
|
||||
update_server_info(the_repository, 0);
|
||||
clear_shallow_info(&si);
|
||||
strbuf_release(&unpack_status);
|
||||
}
|
||||
if (use_sideband)
|
||||
packet_flush(1);
|
||||
|
|
|
|||
|
|
@ -607,7 +607,7 @@ static void unpack_all(void)
|
|||
unpack_one(i);
|
||||
display_progress(progress, i + 1);
|
||||
}
|
||||
odb_transaction_commit(transaction);
|
||||
odb_transaction_commit_and_finalize_or_die(transaction);
|
||||
stop_progress(&progress);
|
||||
|
||||
if (delta_list)
|
||||
|
|
|
|||
|
|
@ -1156,7 +1156,7 @@ int cmd_update_index(int argc,
|
|||
* a transaction.
|
||||
*/
|
||||
if (transaction && verbose) {
|
||||
odb_transaction_commit(transaction);
|
||||
odb_transaction_commit_and_finalize_or_die(transaction);
|
||||
transaction = NULL;
|
||||
}
|
||||
|
||||
|
|
@ -1224,7 +1224,7 @@ int cmd_update_index(int argc,
|
|||
/*
|
||||
* By now we have added all of the new objects
|
||||
*/
|
||||
odb_transaction_commit(transaction);
|
||||
odb_transaction_commit_and_finalize_or_die(transaction);
|
||||
|
||||
if (split_index > 0) {
|
||||
if (repo_config_get_split_index(the_repository) == 0)
|
||||
|
|
|
|||
|
|
@ -538,7 +538,7 @@ int cache_tree_update(struct index_state *istate, int flags)
|
|||
i = update_one(istate->cache_tree, istate->cache, istate->cache_nr,
|
||||
"", 0, &skip, flags);
|
||||
if (!inflight)
|
||||
odb_transaction_commit(transaction);
|
||||
odb_transaction_commit_and_finalize_or_die(transaction);
|
||||
trace2_region_leave("cache_tree", "update", istate->repo);
|
||||
trace_performance_leave("cache_tree_update");
|
||||
if (i < 0)
|
||||
|
|
|
|||
|
|
@ -1075,7 +1075,7 @@ static int get_pack(struct fetch_pack_args *args,
|
|||
die(_("fetch-pack: unable to fork off %s"), cmd_name);
|
||||
if (do_keep && (pack_lockfiles || fsck_objects)) {
|
||||
int is_well_formed;
|
||||
char *pack_lockfile = index_pack_lockfile(the_repository,
|
||||
char *pack_lockfile = index_pack_lockfile(the_repository->objects->sources,
|
||||
cmd.out,
|
||||
&is_well_formed);
|
||||
|
||||
|
|
|
|||
183
object-file.c
183
object-file.c
|
|
@ -10,6 +10,7 @@
|
|||
#define USE_THE_REPOSITORY_VARIABLE
|
||||
|
||||
#include "git-compat-util.h"
|
||||
#include "config.h"
|
||||
#include "convert.h"
|
||||
#include "dir.h"
|
||||
#include "environment.h"
|
||||
|
|
@ -26,6 +27,7 @@
|
|||
#include "packfile.h"
|
||||
#include "path.h"
|
||||
#include "read-cache-ll.h"
|
||||
#include "run-command.h"
|
||||
#include "setup.h"
|
||||
#include "strvec.h"
|
||||
#include "tempfile.h"
|
||||
|
|
@ -483,10 +485,16 @@ struct transaction_packfile {
|
|||
|
||||
struct odb_transaction_files {
|
||||
struct odb_transaction base;
|
||||
enum odb_transaction_flags flags;
|
||||
|
||||
struct tmp_objdir *objdir;
|
||||
struct odb_source *quarantine;
|
||||
struct transaction_packfile packfile;
|
||||
const char *prefix;
|
||||
|
||||
struct tempfile **pack_lockfiles;
|
||||
size_t pack_lockfiles_nr;
|
||||
size_t pack_lockfiles_alloc;
|
||||
};
|
||||
|
||||
int odb_transaction_files_prepare(struct odb_transaction *base)
|
||||
|
|
@ -507,7 +515,7 @@ int odb_transaction_files_prepare(struct odb_transaction *base)
|
|||
if (!transaction->objdir)
|
||||
return error(_("unable to create temporary object directory"));
|
||||
|
||||
tmp_objdir_replace_primary_odb(transaction->objdir, 0);
|
||||
transaction->quarantine = tmp_objdir_replace_primary_odb(transaction->objdir, 0);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -965,7 +973,7 @@ int index_fd(struct index_state *istate, struct object_id *oid,
|
|||
ret = odb_transaction_write_object_stream(transaction,
|
||||
stream, oid);
|
||||
if (!inflight)
|
||||
odb_transaction_commit(transaction);
|
||||
odb_transaction_commit_and_finalize_or_die(transaction);
|
||||
} else {
|
||||
ret = hash_stream(stream, the_repository->hash_algo, oid);
|
||||
}
|
||||
|
|
@ -1288,6 +1296,174 @@ static int odb_transaction_files_commit(struct odb_transaction *base)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static const char *parse_pack_header(struct pack_header *hdr, int pack_fd)
|
||||
{
|
||||
switch (read_pack_header(pack_fd, hdr)) {
|
||||
case PH_ERROR_EOF:
|
||||
return "eof before pack header was fully read";
|
||||
|
||||
case PH_ERROR_PACK_SIGNATURE:
|
||||
return "protocol error (pack signature mismatch detected)";
|
||||
|
||||
case PH_ERROR_PROTOCOL:
|
||||
return "protocol error (pack version unsupported)";
|
||||
|
||||
default:
|
||||
return "unknown error in parse_pack_header";
|
||||
|
||||
case 0:
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static void push_header_arg(struct strvec *args, struct pack_header *hdr)
|
||||
{
|
||||
strvec_pushf(args, "--pack_header=%"PRIu32",%"PRIu32,
|
||||
ntohl(hdr->hdr_version), ntohl(hdr->hdr_entries));
|
||||
}
|
||||
|
||||
static unsigned int get_unpack_limit(struct repository *repo,
|
||||
enum odb_transaction_flags flags)
|
||||
{
|
||||
unsigned int limit = 0;
|
||||
|
||||
if (flags & ODB_TRANSACTION_RECEIVE) {
|
||||
limit = 100;
|
||||
repo_config_get_uint(repo, "transfer.unpacklimit", &limit);
|
||||
repo_config_get_uint(repo, "receive.unpacklimit", &limit);
|
||||
}
|
||||
|
||||
return limit;
|
||||
}
|
||||
|
||||
static int odb_transaction_files_write_pack(struct odb_transaction *base,
|
||||
int pack_fd, struct strbuf *err_msg,
|
||||
const struct odb_transaction_write_pack_opts *opts)
|
||||
{
|
||||
struct odb_transaction_files *transaction =
|
||||
container_of(base, struct odb_transaction_files, base);
|
||||
struct repository *repo = base->source->odb->repo;
|
||||
struct child_process child = CHILD_PROCESS_INIT;
|
||||
struct pack_header hdr;
|
||||
const char *hdr_err;
|
||||
int err_fd = opts->err_fd;
|
||||
int status;
|
||||
|
||||
hdr_err = parse_pack_header(&hdr, pack_fd);
|
||||
if (hdr_err) {
|
||||
if (err_fd > 0)
|
||||
close(err_fd);
|
||||
strbuf_addstr(err_msg, hdr_err);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (opts->shallow_file) {
|
||||
strvec_push(&child.args, "--shallow-file");
|
||||
strvec_push(&child.args, opts->shallow_file);
|
||||
}
|
||||
|
||||
odb_transaction_env(base, &child.env);
|
||||
|
||||
if (ntohl(hdr.hdr_entries) < get_unpack_limit(repo, transaction->flags)) {
|
||||
strvec_push(&child.args, "unpack-objects");
|
||||
push_header_arg(&child.args, &hdr);
|
||||
if (opts->quiet)
|
||||
strvec_push(&child.args, "-q");
|
||||
if (opts->fsck_objects)
|
||||
strvec_pushf(&child.args, "--strict%s",
|
||||
opts->fsck_msg_types);
|
||||
if (opts->max_input_size)
|
||||
strvec_pushf(&child.args, "--max-input-size=%"PRIuMAX,
|
||||
(uintmax_t)opts->max_input_size);
|
||||
child.no_stdout = 1;
|
||||
child.in = pack_fd;
|
||||
child.err = err_fd;
|
||||
child.git_cmd = 1;
|
||||
status = run_command(&child);
|
||||
if (status) {
|
||||
strbuf_addstr(err_msg, "unpack-objects abnormal exit");
|
||||
return -1;
|
||||
}
|
||||
} else {
|
||||
char hostname[HOST_NAME_MAX + 1];
|
||||
char *lockfile;
|
||||
|
||||
strvec_pushl(&child.args, "index-pack", "--stdin", NULL);
|
||||
push_header_arg(&child.args, &hdr);
|
||||
|
||||
if (xgethostname(hostname, sizeof(hostname)))
|
||||
xsnprintf(hostname, sizeof(hostname), "localhost");
|
||||
strvec_pushf(&child.args,
|
||||
"--keep=receive-pack %"PRIuMAX" on %s",
|
||||
(uintmax_t)getpid(),
|
||||
hostname);
|
||||
|
||||
if (!opts->quiet && err_fd)
|
||||
strvec_push(&child.args, "--show-resolving-progress");
|
||||
if (err_fd)
|
||||
strvec_push(&child.args, "--report-end-of-input");
|
||||
if (opts->fsck_objects)
|
||||
strvec_pushf(&child.args, "--strict%s",
|
||||
opts->fsck_msg_types);
|
||||
if (!opts->reject_thin)
|
||||
strvec_push(&child.args, "--fix-thin");
|
||||
if (opts->max_input_size)
|
||||
strvec_pushf(&child.args, "--max-input-size=%"PRIuMAX,
|
||||
(uintmax_t)opts->max_input_size);
|
||||
child.out = -1;
|
||||
child.in = pack_fd;
|
||||
child.err = err_fd;
|
||||
child.git_cmd = 1;
|
||||
status = start_command(&child);
|
||||
if (status) {
|
||||
strbuf_addstr(err_msg, "index-pack fork failed");
|
||||
return -1;
|
||||
}
|
||||
|
||||
/*
|
||||
* The lockfile filepath is expected to be the final location of
|
||||
* the ".keep" file after being migrated to the main ODB source.
|
||||
* This ensures the lockfile can be found and removed later
|
||||
* after the ODB transaction has been committed.
|
||||
*/
|
||||
lockfile = index_pack_lockfile(base->source, child.out, NULL);
|
||||
if (lockfile) {
|
||||
ALLOC_GROW(transaction->pack_lockfiles,
|
||||
transaction->pack_lockfiles_nr + 1,
|
||||
transaction->pack_lockfiles_alloc);
|
||||
transaction->pack_lockfiles[transaction->pack_lockfiles_nr++] =
|
||||
register_tempfile(lockfile);
|
||||
free(lockfile);
|
||||
}
|
||||
close(child.out);
|
||||
|
||||
status = finish_command(&child);
|
||||
if (status) {
|
||||
strbuf_addstr(err_msg, "index-pack abnormal exit");
|
||||
return -1;
|
||||
}
|
||||
|
||||
odb_source_prepare(transaction->quarantine,
|
||||
ODB_PREPARE_FLUSH_CACHES);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int odb_transaction_files_finalize(struct odb_transaction *base)
|
||||
{
|
||||
struct odb_transaction_files *transaction =
|
||||
container_of(base, struct odb_transaction_files, base);
|
||||
int ret = 0;
|
||||
|
||||
for (size_t i = 0; i < transaction->pack_lockfiles_nr; i++)
|
||||
ret |= delete_tempfile(&transaction->pack_lockfiles[i]);
|
||||
|
||||
free(transaction->pack_lockfiles);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int odb_transaction_files_env(struct odb_transaction *base,
|
||||
struct strvec *env)
|
||||
{
|
||||
|
|
@ -1311,8 +1487,11 @@ int odb_transaction_files_begin(struct odb_source *source,
|
|||
transaction = xcalloc(1, sizeof(*transaction));
|
||||
transaction->base.source = source;
|
||||
transaction->base.commit = odb_transaction_files_commit;
|
||||
transaction->base.finalize = odb_transaction_files_finalize;
|
||||
transaction->base.write_object_stream = odb_transaction_files_write_object_stream;
|
||||
transaction->base.write_pack = odb_transaction_files_write_pack;
|
||||
transaction->base.env = odb_transaction_files_env;
|
||||
transaction->flags = flags;
|
||||
|
||||
transaction->prefix = "bulk-fsync";
|
||||
if (flags & ODB_TRANSACTION_RECEIVE) {
|
||||
|
|
|
|||
9
odb.c
9
odb.c
|
|
@ -254,7 +254,8 @@ struct odb_source *odb_add_to_alternates_memory(struct object_database *odb,
|
|||
}
|
||||
|
||||
struct odb_source *odb_set_temporary_primary_source(struct object_database *odb,
|
||||
const char *dir, int will_destroy)
|
||||
const char *dir, int will_destroy,
|
||||
struct odb_source **prev_source)
|
||||
{
|
||||
struct odb_source *source;
|
||||
|
||||
|
|
@ -272,7 +273,11 @@ struct odb_source *odb_set_temporary_primary_source(struct object_database *odb,
|
|||
source->will_destroy = will_destroy;
|
||||
source->next = odb->sources;
|
||||
odb->sources = source;
|
||||
return source->next;
|
||||
|
||||
if (prev_source)
|
||||
*prev_source = source->next;
|
||||
|
||||
return source;
|
||||
}
|
||||
|
||||
void odb_restore_primary_source(struct object_database *odb,
|
||||
|
|
|
|||
6
odb.h
6
odb.h
|
|
@ -216,10 +216,12 @@ struct odb_source *odb_find_source_or_die(struct object_database *odb, const cha
|
|||
|
||||
/*
|
||||
* Replace the current writable object directory with the specified temporary
|
||||
* object directory; returns the former primary source.
|
||||
* object directory and return the newly installed primary source. The former
|
||||
* primary source is reported via `prev_source` when non-NULL.
|
||||
*/
|
||||
struct odb_source *odb_set_temporary_primary_source(struct object_database *odb,
|
||||
const char *dir, int will_destroy);
|
||||
const char *dir, int will_destroy,
|
||||
struct odb_source **prev_source);
|
||||
|
||||
/*
|
||||
* Restore the primary source that was previously replaced by
|
||||
|
|
|
|||
|
|
@ -33,6 +33,20 @@ int odb_transaction_commit(struct odb_transaction *transaction)
|
|||
|
||||
ret = transaction->commit(transaction);
|
||||
transaction->source->odb->transaction = NULL;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int odb_transaction_finalize(struct odb_transaction *transaction)
|
||||
{
|
||||
int ret = 0;
|
||||
|
||||
if (!transaction)
|
||||
return 0;
|
||||
|
||||
if (transaction->finalize)
|
||||
ret = transaction->finalize(transaction);
|
||||
|
||||
free(transaction);
|
||||
|
||||
return ret;
|
||||
|
|
@ -45,6 +59,13 @@ int odb_transaction_write_object_stream(struct odb_transaction *transaction,
|
|||
return transaction->write_object_stream(transaction, stream, oid);
|
||||
}
|
||||
|
||||
int odb_transaction_write_pack(struct odb_transaction *transaction, int pack_fd,
|
||||
struct strbuf *err_msg,
|
||||
const struct odb_transaction_write_pack_opts *opts)
|
||||
{
|
||||
return transaction->write_pack(transaction, pack_fd, err_msg, opts);
|
||||
}
|
||||
|
||||
int odb_transaction_env(struct odb_transaction *transaction, struct strvec *env)
|
||||
{
|
||||
if (!transaction)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,50 @@
|
|||
#include "gettext.h"
|
||||
#include "odb.h"
|
||||
|
||||
/*
|
||||
* Options controlling how odb_transaction_write_pack() ingests a packfile.
|
||||
*/
|
||||
struct odb_transaction_write_pack_opts {
|
||||
/*
|
||||
* Optional fsck severity configuration to apply when incoming objects
|
||||
* are verified.
|
||||
*/
|
||||
const char *fsck_msg_types;
|
||||
|
||||
/*
|
||||
* Path to an alternative shallow file describing the shallow boundaries
|
||||
* to honor while ingesting the pack.
|
||||
*/
|
||||
const char *shallow_file;
|
||||
|
||||
/*
|
||||
* The max size in bytes of the incoming packfile allowed. No limit is
|
||||
* enforced when set to 0.
|
||||
*/
|
||||
off_t max_input_size;
|
||||
|
||||
/*
|
||||
* Whether the validity of incoming objects should be verified.
|
||||
*/
|
||||
int fsck_objects;
|
||||
|
||||
/*
|
||||
* Whether to reject an incoming packfile if it is "thin".
|
||||
*/
|
||||
int reject_thin;
|
||||
|
||||
/*
|
||||
* Optional file descriptor for reporting progress and errors. Set to 0
|
||||
* for none.
|
||||
*/
|
||||
int err_fd;
|
||||
|
||||
/*
|
||||
* Suppresses progress reporting.
|
||||
*/
|
||||
int quiet;
|
||||
};
|
||||
|
||||
/*
|
||||
* A transaction may be started for an object database prior to writing new
|
||||
* objects via odb_transaction_begin(). These objects are not committed until
|
||||
|
|
@ -22,6 +66,13 @@ struct odb_transaction {
|
|||
*/
|
||||
int (*commit)(struct odb_transaction *transaction);
|
||||
|
||||
/*
|
||||
* Optional ODB source specific callback invoked when the transaction
|
||||
* needs to perform any deferred cleanup after objects have been
|
||||
* committed. Returns 0 on success, a negative error code otherwise.
|
||||
*/
|
||||
int (*finalize)(struct odb_transaction *transaction);
|
||||
|
||||
/*
|
||||
* This callback is expected to write the given object stream into
|
||||
* the ODB transaction.
|
||||
|
|
@ -33,6 +84,15 @@ struct odb_transaction {
|
|||
int (*write_object_stream)(struct odb_transaction *transaction,
|
||||
struct odb_stream *stream,
|
||||
struct object_id *oid);
|
||||
/*
|
||||
* This callback is expected to ingest the packfile readable via
|
||||
* `pack_fd` into the transaction. Returns 0 on success, a negative
|
||||
* error code otherwise. On failure, a human-readable description is
|
||||
* appended to `err_msg`.
|
||||
*/
|
||||
int (*write_pack)(struct odb_transaction *transaction, int pack_fd,
|
||||
struct strbuf *err_msg,
|
||||
const struct odb_transaction_write_pack_opts *opts);
|
||||
|
||||
/*
|
||||
* This callback is expected to populate the provided strvec with the
|
||||
|
|
@ -75,6 +135,22 @@ static inline void odb_transaction_begin_or_die(struct object_database *odb,
|
|||
*/
|
||||
int odb_transaction_commit(struct odb_transaction *transaction);
|
||||
|
||||
/*
|
||||
* Finalizes an ODB transaction, performing any deferred cleanup and freeing it.
|
||||
* Must be called for every successfully started transaction. Note that, if the
|
||||
* specified transaction is NULL, the function is a no-op. Returns 0 on success,
|
||||
* a negative error code otherwise.
|
||||
*/
|
||||
int odb_transaction_finalize(struct odb_transaction *transaction);
|
||||
|
||||
static inline void odb_transaction_commit_and_finalize_or_die(struct odb_transaction *transaction)
|
||||
{
|
||||
if (odb_transaction_commit(transaction))
|
||||
die(_("failed to commit ODB transaction"));
|
||||
if (odb_transaction_finalize(transaction))
|
||||
die(_("failed to finalize ODB transaction"));
|
||||
}
|
||||
|
||||
/*
|
||||
* Writes the object in the provided stream into the transaction. The resulting
|
||||
* object ID is written into the out pointer. Returns 0 on success, a negative
|
||||
|
|
@ -84,6 +160,15 @@ int odb_transaction_write_object_stream(struct odb_transaction *transaction,
|
|||
struct odb_stream *stream,
|
||||
struct object_id *oid);
|
||||
|
||||
/*
|
||||
* Ingests the packfile readable via `pack_fd` into the transaction. Returns 0
|
||||
* on success, a negative error code otherwise. On failure, a human-readable
|
||||
* description is appended to `err_msg`.
|
||||
*/
|
||||
int odb_transaction_write_pack(struct odb_transaction *transaction, int pack_fd,
|
||||
struct strbuf *err_msg,
|
||||
const struct odb_transaction_write_pack_opts *opts);
|
||||
|
||||
/*
|
||||
* Populates the provided strvec with the environment variables that a child
|
||||
* process should inherit so that its object writes participate in the
|
||||
|
|
|
|||
|
|
@ -469,10 +469,11 @@ void fixup_pack_header_footer(const struct git_hash_algo *hash_algo,
|
|||
fsync_component_or_die(FSYNC_COMPONENT_PACK, pack_fd, pack_name);
|
||||
}
|
||||
|
||||
char *index_pack_lockfile(struct repository *r, int ip_out, int *is_well_formed)
|
||||
char *index_pack_lockfile(struct odb_source *source, int ip_out,
|
||||
int *is_well_formed)
|
||||
{
|
||||
char packname[GIT_MAX_HEXSZ + 6];
|
||||
const int len = r->hash_algo->hexsz + 6;
|
||||
const int len = source->odb->repo->hash_algo->hexsz + 6;
|
||||
|
||||
/*
|
||||
* The first thing we expect from index-pack's output
|
||||
|
|
@ -489,7 +490,7 @@ char *index_pack_lockfile(struct repository *r, int ip_out, int *is_well_formed)
|
|||
packname[len-1] = 0;
|
||||
if (skip_prefix(packname, "keep\t", &name))
|
||||
return xstrfmt("%s/pack/pack-%s.keep",
|
||||
repo_get_object_directory(r), name);
|
||||
source->path, name);
|
||||
return NULL;
|
||||
}
|
||||
if (is_well_formed)
|
||||
|
|
|
|||
4
pack.h
4
pack.h
|
|
@ -7,6 +7,7 @@
|
|||
struct packed_git;
|
||||
struct pack_window;
|
||||
struct repository;
|
||||
struct odb_source;
|
||||
|
||||
/*
|
||||
* Packed object header
|
||||
|
|
@ -105,7 +106,8 @@ off_t write_pack_header(struct hashfile *f, uint32_t);
|
|||
void fixup_pack_header_footer(const struct git_hash_algo *, int,
|
||||
unsigned char *, const char *, uint32_t,
|
||||
unsigned char *, off_t);
|
||||
char *index_pack_lockfile(struct repository *r, int fd, int *is_well_formed);
|
||||
char *index_pack_lockfile(struct odb_source *source, int fd,
|
||||
int *is_well_formed);
|
||||
|
||||
struct ref;
|
||||
|
||||
|
|
|
|||
|
|
@ -4062,7 +4062,7 @@ int add_files_to_cache(struct repository *repo, const char *prefix,
|
|||
odb_transaction_begin_or_die(repo->objects, &transaction, 0);
|
||||
run_diff_files(&rev, DIFF_RACY_IS_MODIFIED);
|
||||
if (!inflight)
|
||||
odb_transaction_commit(transaction);
|
||||
odb_transaction_commit_and_finalize_or_die(transaction);
|
||||
|
||||
release_revisions(&rev);
|
||||
return !!data.add_errors;
|
||||
|
|
|
|||
|
|
@ -70,4 +70,35 @@ test_expect_success 'updating a ref from quarantine is forbidden' '
|
|||
git -C update.git fsck
|
||||
'
|
||||
|
||||
test_expect_success '.keep file is removed after push' '
|
||||
test_when_finished rm -rf keep.git &&
|
||||
git init --bare keep.git &&
|
||||
|
||||
git -C keep.git config set receive.unpackLimit 0 &&
|
||||
|
||||
# While incoming objects are still quarantined, validate that the
|
||||
# ".keep" lockfile is present in the quarantine directory.
|
||||
test_hook -C keep.git pre-receive <<-\EOF &&
|
||||
keep="$(ls "$GIT_QUARANTINE_PATH"/pack/pack-*.keep)" &&
|
||||
test -f "$keep"
|
||||
EOF
|
||||
|
||||
# After quarantined objects are migrated, validate that the ".keep"
|
||||
# lockfile is migrated and present in the main ODB.
|
||||
test_hook -C keep.git reference-transaction <<-\EOF &&
|
||||
keep="$(ls objects/pack/pack-*.keep)" &&
|
||||
test -f "$keep"
|
||||
EOF
|
||||
|
||||
test_commit foo &&
|
||||
git push keep.git HEAD &&
|
||||
|
||||
# Once the operation is complete, validate that the ".keep" lockfile has
|
||||
# been removed.
|
||||
pack="$(ls keep.git/objects/pack/pack-*.pack)" &&
|
||||
keep="${pack%.pack}.keep" &&
|
||||
test_path_is_file "$pack" &&
|
||||
test_path_is_missing "$keep"
|
||||
'
|
||||
|
||||
test_done
|
||||
|
|
|
|||
|
|
@ -326,11 +326,13 @@ void tmp_objdir_add_as_alternate(const struct tmp_objdir *t)
|
|||
odb_add_to_alternates_memory(t->repo->objects, t->path.buf);
|
||||
}
|
||||
|
||||
void tmp_objdir_replace_primary_odb(struct tmp_objdir *t, int will_destroy)
|
||||
struct odb_source *tmp_objdir_replace_primary_odb(struct tmp_objdir *t,
|
||||
int will_destroy)
|
||||
{
|
||||
if (t->prev_source)
|
||||
BUG("the primary object database is already replaced");
|
||||
t->prev_source = odb_set_temporary_primary_source(t->repo->objects,
|
||||
t->path.buf, will_destroy);
|
||||
t->will_destroy = will_destroy;
|
||||
|
||||
return odb_set_temporary_primary_source(t->repo->objects, t->path.buf,
|
||||
will_destroy, &t->prev_source);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -64,8 +64,10 @@ void tmp_objdir_add_as_alternate(const struct tmp_objdir *);
|
|||
/*
|
||||
* Replaces the writable object store in the current process with the temporary
|
||||
* object directory and makes the former main object store an alternate.
|
||||
* If will_destroy is nonzero, the object directory may not be migrated.
|
||||
* If will_destroy is nonzero, the object directory may not be migrated. Returns
|
||||
* the newly installed primary source.
|
||||
*/
|
||||
void tmp_objdir_replace_primary_odb(struct tmp_objdir *, int will_destroy);
|
||||
struct odb_source *tmp_objdir_replace_primary_odb(struct tmp_objdir *,
|
||||
int will_destroy);
|
||||
|
||||
#endif /* TMP_OBJDIR_H */
|
||||
|
|
|
|||
Loading…
Reference in New Issue