diff --git a/builtin/add.c b/builtin/add.c index c859f66519..60ffbede2b 100644 --- a/builtin/add.c +++ b/builtin/add.c @@ -581,7 +581,7 @@ int cmd_add(int argc, string_list_clear(&only_match_skip_worktree, 0); } - transaction = odb_transaction_begin(repo->objects); + odb_transaction_begin_or_die(repo->objects, &transaction, 0); ps_matched = xcalloc(pathspec.nr, 1); if (add_renormalize) diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index 19eb6a1b61..ee8e03e2ab 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -112,8 +112,6 @@ static enum { } use_keepalive; static int keepalive_in_sec = 5; -static struct tmp_objdir *tmp_objdir; - static struct proc_receive_ref { unsigned int want_add:1, want_delete:1, @@ -959,8 +957,8 @@ static int run_receive_hook(struct command *commands, strvec_push(&opt.env, "GIT_PUSH_OPTION_COUNT"); } - if (tmp_objdir) - strvec_pushv(&opt.env, tmp_objdir_env(tmp_objdir)); + if (the_repository->objects->transaction) + strvec_pushv(&opt.env, odb_transaction_env(the_repository->objects->transaction)); prepare_push_cert_sha1(&opt); @@ -1363,7 +1361,7 @@ static int update_shallow_ref(struct command *cmd, struct shallow_info *si) !delayed_reachability_test(si, i)) oid_array_append(&extra, &si->shallow->oid[i]); - opt.env = tmp_objdir_env(tmp_objdir); + opt.env = odb_transaction_env(the_repository->objects->transaction); setup_alternate_shallow(&shallow_lock, &opt.shallow_file, &extra); if (check_connected(command_singleton_iterator, cmd, &opt)) { rollback_shallow_file(the_repository, &shallow_lock); @@ -1802,7 +1800,7 @@ static void set_connectivity_errors(struct command *commands, /* to be checked in update_shallow_ref() */ continue; - opt.env = tmp_objdir_env(tmp_objdir); + opt.env = odb_transaction_env(the_repository->objects->transaction); if (!check_connected(command_singleton_iterator, &singleton, &opt)) continue; @@ -2057,7 +2055,7 @@ static void execute_commands(struct command *commands, data.si = si; opt.err_fd = err_fd; opt.progress = err_fd && !quiet; - opt.env = tmp_objdir_env(tmp_objdir); + opt.env = odb_transaction_env(the_repository->objects->transaction); opt.exclude_hidden_refs_section = "receive"; if (check_connected(iterate_receive_command_list, &data, &opt)) @@ -2106,14 +2104,13 @@ static void execute_commands(struct command *commands, * Now we'll start writing out refs, which means the objects need * to be in their final positions so that other processes can see them. */ - if (tmp_objdir_migrate(tmp_objdir) < 0) { + if (odb_transaction_commit(the_repository->objects->transaction)) { for (cmd = commands; cmd; cmd = cmd->next) { if (!cmd->error_string) cmd->error_string = "unable to migrate objects to permanent storage"; } return; } - tmp_objdir = NULL; check_aliased_updates(commands); @@ -2326,7 +2323,8 @@ static void push_header_arg(struct strvec *args, struct pack_header *hdr) ntohl(hdr->hdr_version), ntohl(hdr->hdr_entries)); } -static const char *unpack(int err_fd, struct shallow_info *si) +static const char *unpack(int err_fd, struct shallow_info *si, + struct odb_transaction *transaction) { struct pack_header hdr; const char *hdr_err; @@ -2351,20 +2349,7 @@ static const char *unpack(int err_fd, struct shallow_info *si) strvec_push(&child.args, alt_shallow_file); } - tmp_objdir = tmp_objdir_create(the_repository, "incoming"); - if (!tmp_objdir) { - if (err_fd > 0) - close(err_fd); - return "unable to create temporary object directory"; - } - strvec_pushv(&child.env, tmp_objdir_env(tmp_objdir)); - - /* - * Normally we just pass the tmp_objdir environment to the child - * processes that do the heavy lifting, but we may need to see these - * objects ourselves to set up shallow information. - */ - tmp_objdir_add_as_alternate(tmp_objdir); + strvec_pushv(&child.env, odb_transaction_env(transaction)); if (ntohl(hdr.hdr_entries) < unpack_limit) { strvec_push(&child.args, "unpack-objects"); @@ -2431,13 +2416,14 @@ static const char *unpack(int err_fd, struct shallow_info *si) return NULL; } -static const char *unpack_with_sideband(struct shallow_info *si) +static const char *unpack_with_sideband(struct shallow_info *si, + struct odb_transaction *transaction) { struct async muxer; const char *ret; if (!use_sideband) - return unpack(0, si); + return unpack(0, si, transaction); use_keepalive = KEEPALIVE_AFTER_NUL; memset(&muxer, 0, sizeof(muxer)); @@ -2446,7 +2432,7 @@ static const char *unpack_with_sideband(struct shallow_info *si) if (start_async(&muxer)) return NULL; - ret = unpack(muxer.in, si); + ret = unpack(muxer.in, si, transaction); finish_async(&muxer); return ret; @@ -2623,6 +2609,7 @@ int cmd_receive_pack(int argc, struct oid_array ref = OID_ARRAY_INIT; struct shallow_info si; struct packet_reader reader; + struct odb_transaction *transaction = NULL; struct option options[] = { OPT__QUIET(&quiet, N_("quiet")), @@ -2707,7 +2694,10 @@ int cmd_receive_pack(int argc, if (!si.nr_ours && !si.nr_theirs) shallow_update = 0; if (!delete_only(commands)) { - unpack_status = unpack_with_sideband(&si); + if (odb_transaction_begin(the_repository->objects, &transaction, ODB_TRANSACTION_RECEIVE)) + unpack_status = "unable to start ODB transaction"; + else + unpack_status = unpack_with_sideband(&si, transaction); update_shallow_info(commands, &si, &ref); } use_keepalive = KEEPALIVE_ALWAYS; diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c index f3849bb654..c3d0fc7507 100644 --- a/builtin/unpack-objects.c +++ b/builtin/unpack-objects.c @@ -598,7 +598,7 @@ static void unpack_all(void) progress = start_progress(the_repository, _("Unpacking objects"), nr_objects); CALLOC_ARRAY(obj_list, nr_objects); - transaction = odb_transaction_begin(the_repository->objects); + odb_transaction_begin_or_die(the_repository->objects, &transaction, 0); for (i = 0; i < nr_objects; i++) { unpack_one(i); display_progress(progress, i + 1); diff --git a/builtin/update-index.c b/builtin/update-index.c index f43d150eb3..b65213670a 100644 --- a/builtin/update-index.c +++ b/builtin/update-index.c @@ -1124,7 +1124,7 @@ int cmd_update_index(int argc, * Allow the object layer to optimize adding multiple objects in * a batch. */ - transaction = odb_transaction_begin(the_repository->objects); + odb_transaction_begin_or_die(the_repository->objects, &transaction, 0); while (ctx.argc) { if (parseopt_state != PARSE_OPT_DONE) parseopt_state = parse_options_step(&ctx, options, diff --git a/cache-tree.c b/cache-tree.c index 9204eb3ac7..d7be52c11e 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -532,7 +532,7 @@ int cache_tree_update(struct index_state *istate, int flags) trace_performance_enter(); trace2_region_enter("cache_tree", "update", istate->repo); - transaction = odb_transaction_begin(the_repository->objects); + odb_transaction_begin_or_die(the_repository->objects, &transaction, 0); i = update_one(istate->cache_tree, istate->cache, istate->cache_nr, "", 0, &skip, flags); odb_transaction_commit(transaction); diff --git a/object-file.c b/object-file.c index 07f019a0f6..c3707f62a7 100644 --- a/object-file.c +++ b/object-file.c @@ -497,9 +497,10 @@ struct odb_transaction_files { struct tmp_objdir *objdir; struct transaction_packfile packfile; + const char *prefix; }; -static void prepare_loose_object_transaction(struct odb_transaction *base) +static int odb_transaction_files_prepare(struct odb_transaction *base) { struct odb_transaction_files *transaction = container_of_or_null(base, struct odb_transaction_files, base); @@ -511,11 +512,15 @@ static void prepare_loose_object_transaction(struct odb_transaction *base) * added at the time they call odb_transaction_files_begin. */ if (!transaction || transaction->objdir) - return; + return 0; - transaction->objdir = tmp_objdir_create(base->source->odb->repo, "bulk-fsync"); - if (transaction->objdir) - tmp_objdir_replace_primary_odb(transaction->objdir, 0); + transaction->objdir = tmp_objdir_create(base->source->odb->repo, transaction->prefix); + if (!transaction->objdir) + return -1; + + tmp_objdir_replace_primary_odb(transaction->objdir, 0); + + return 0; } static void fsync_loose_object_transaction(struct odb_transaction *base, @@ -542,13 +547,13 @@ static void fsync_loose_object_transaction(struct odb_transaction *base, /* * Cleanup after batch-mode fsync_object_files. */ -static void flush_loose_object_transaction(struct odb_transaction_files *transaction) +static int flush_loose_object_transaction(struct odb_transaction_files *transaction) { struct strbuf temp_path = STRBUF_INIT; struct tempfile *temp; if (!transaction->objdir) - return; + return 0; /* * Issue a full hardware flush against a temporary file to ensure @@ -570,8 +575,12 @@ static void flush_loose_object_transaction(struct odb_transaction_files *transac * Make the object files visible in the primary ODB after their data is * fully durable. */ - tmp_objdir_migrate(transaction->objdir); + if (tmp_objdir_migrate(transaction->objdir)) + return -1; + transaction->objdir = NULL; + + return 0; } /* Finalize a file on disk, and close it. */ @@ -761,7 +770,7 @@ int write_loose_object(struct odb_source_loose *loose, static struct strbuf filename = STRBUF_INIT; if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT)) - prepare_loose_object_transaction(loose->base.odb->transaction); + odb_transaction_files_prepare(loose->base.odb->transaction); odb_loose_path(loose, &filename, oid); @@ -825,7 +834,7 @@ int odb_source_loose_write_stream(struct odb_source_loose *loose, int hdrlen; if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT)) - prepare_loose_object_transaction(loose->base.odb->transaction); + odb_transaction_files_prepare(loose->base.odb->transaction); /* Since oid is not determined, save tmp file to odb path. */ strbuf_addf(&filename, "%s/", loose->base.path); @@ -1381,8 +1390,9 @@ int index_fd(struct index_state *istate, struct object_id *oid, if (flags & INDEX_WRITE_OBJECT) { struct object_database *odb = the_repository->objects; - struct odb_transaction *transaction = odb_transaction_begin(odb); + struct odb_transaction *transaction; + odb_transaction_begin_or_die(odb, &transaction, 0); ret = odb_transaction_write_object_stream(odb->transaction, &stream, xsize_t(st->st_size), @@ -1670,29 +1680,62 @@ out: return ret; } -static void odb_transaction_files_commit(struct odb_transaction *base) +static int odb_transaction_files_commit(struct odb_transaction *base) { struct odb_transaction_files *transaction = container_of(base, struct odb_transaction_files, base); - flush_loose_object_transaction(transaction); + if (flush_loose_object_transaction(transaction)) + return -1; flush_packfile_transaction(transaction); + + return 0; } -struct odb_transaction *odb_transaction_files_begin(struct odb_source *source) +static const char **odb_transaction_files_env(struct odb_transaction *base) +{ + struct odb_transaction_files *transaction = + container_of(base, struct odb_transaction_files, base); + + odb_transaction_files_prepare(&transaction->base); + + return tmp_objdir_env(transaction->objdir); +} + +int odb_transaction_files_begin(struct odb_source *source, + struct odb_transaction **out, + enum odb_transaction_flags flags) { struct odb_transaction_files *transaction; struct object_database *odb = source->odb; - if (odb->transaction) - return NULL; + if (odb->transaction) { + *out = NULL; + return 0; + } transaction = xcalloc(1, sizeof(*transaction)); transaction->base.source = source; transaction->base.commit = odb_transaction_files_commit; transaction->base.write_object_stream = odb_transaction_files_write_object_stream; + transaction->base.env = odb_transaction_files_env; - return &transaction->base; + transaction->prefix = "bulk-fsync"; + if (flags & ODB_TRANSACTION_RECEIVE) { + /* + * ODB transactions for git-receive-pack(1) eagerly create a + * temporary directory and use a different prefix. + */ + transaction->prefix = "incoming"; + if (odb_transaction_files_prepare(&transaction->base)) { + free(transaction); + return -1; + } + } + + *out = &transaction->base; + + return 0; } void free_object_info_contents(struct object_info *object_info) diff --git a/object-file.h b/object-file.h index 4c87cd160b..4e10b442b2 100644 --- a/object-file.h +++ b/object-file.h @@ -5,6 +5,7 @@ #include "object.h" #include "odb.h" #include "odb/source-loose.h" +#include "odb/transaction.h" /* The maximum size for an object header. */ #define MAX_HEADER_LEN 32 @@ -195,8 +196,10 @@ struct odb_transaction; * Tell the object database to optimize for adding * multiple objects. odb_transaction_files_commit must be called * to make new objects visible. If a transaction is already - * pending, NULL is returned. + * pending, out is set to NULL. */ -struct odb_transaction *odb_transaction_files_begin(struct odb_source *source); +int odb_transaction_files_begin(struct odb_source *source, + struct odb_transaction **out, + enum odb_transaction_flags flags); #endif /* OBJECT_FILE_H */ diff --git a/odb/source-files.c b/odb/source-files.c index 6c8e935c75..4138758511 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -181,13 +181,10 @@ static int odb_source_files_write_object_stream(struct odb_source *source, } static int odb_source_files_begin_transaction(struct odb_source *source, - struct odb_transaction **out) + struct odb_transaction **out, + enum odb_transaction_flags flags) { - struct odb_transaction *tx = odb_transaction_files_begin(source); - if (!tx) - return -1; - *out = tx; - return 0; + return odb_transaction_files_begin(source, out, flags); } static int odb_source_files_read_alternates(struct odb_source *source, diff --git a/odb/source-inmemory.c b/odb/source-inmemory.c index 8d402201be..814b5da94c 100644 --- a/odb/source-inmemory.c +++ b/odb/source-inmemory.c @@ -305,7 +305,8 @@ static int odb_source_inmemory_freshen_object(struct odb_source *source, } static int odb_source_inmemory_begin_transaction(struct odb_source *source UNUSED, - struct odb_transaction **out UNUSED) + struct odb_transaction **out UNUSED, + enum odb_transaction_flags flags UNUSED) { return error("in-memory source does not support transactions"); } diff --git a/odb/source-loose.c b/odb/source-loose.c index deeaf4db63..213b66fbb7 100644 --- a/odb/source-loose.c +++ b/odb/source-loose.c @@ -638,7 +638,8 @@ static int odb_source_loose_write_object_stream(struct odb_source *source, } static int odb_source_loose_begin_transaction(struct odb_source *source UNUSED, - struct odb_transaction **out UNUSED) + struct odb_transaction **out UNUSED, + enum odb_transaction_flags flags UNUSED) { /* TODO: this is a known omission that we'll want to address eventually. */ return error("loose source does not support transactions"); diff --git a/odb/source-packed.c b/odb/source-packed.c index 647f23cdb9..8d9ce197cc 100644 --- a/odb/source-packed.c +++ b/odb/source-packed.c @@ -545,7 +545,8 @@ static int odb_source_packed_write_object_stream(struct odb_source *source UNUSE } static int odb_source_packed_begin_transaction(struct odb_source *source UNUSED, - struct odb_transaction **out UNUSED) + struct odb_transaction **out UNUSED, + enum odb_transaction_flags flags UNUSED) { return error("packed backend cannot begin transactions"); } diff --git a/odb/source.h b/odb/source.h index efeff7ed12..cd63dba91f 100644 --- a/odb/source.h +++ b/odb/source.h @@ -3,6 +3,7 @@ #include "object.h" #include "odb.h" +#include "odb/transaction.h" enum odb_source_type { /* @@ -232,7 +233,8 @@ struct odb_source { * negative error code otherwise. */ int (*begin_transaction)(struct odb_source *source, - struct odb_transaction **out); + struct odb_transaction **out, + enum odb_transaction_flags flags); /* * This callback is expected to read the list of alternate object @@ -472,9 +474,10 @@ static inline int odb_source_write_alternate(struct odb_source *source, * Returns 0 on success, a negative error code otherwise. */ static inline int odb_source_begin_transaction(struct odb_source *source, - struct odb_transaction **out) + struct odb_transaction **out, + enum odb_transaction_flags flags) { - return source->begin_transaction(source, out); + return source->begin_transaction(source, out, flags); } #endif diff --git a/odb/transaction.c b/odb/transaction.c index b16e07aebf..34c212020c 100644 --- a/odb/transaction.c +++ b/odb/transaction.c @@ -2,29 +2,43 @@ #include "odb/source.h" #include "odb/transaction.h" -struct odb_transaction *odb_transaction_begin(struct object_database *odb) +int odb_transaction_begin(struct object_database *odb, + struct odb_transaction **out, + enum odb_transaction_flags flags) { - if (odb->transaction) - return NULL; + int ret; - odb_source_begin_transaction(odb->sources, &odb->transaction); + if (odb->transaction) { + *out = NULL; + return 0; + } - return odb->transaction; + ret = odb_source_begin_transaction(odb->sources, out, flags); + odb->transaction = *out; + + return ret; } -void odb_transaction_commit(struct odb_transaction *transaction) +int odb_transaction_commit(struct odb_transaction *transaction) { + int ret; + if (!transaction) - return; + return 0; /* * Ensure the transaction ending matches the pending transaction. */ ASSERT(transaction == transaction->source->odb->transaction); - transaction->commit(transaction); + ret = transaction->commit(transaction); + if (ret) + return ret; + transaction->source->odb->transaction = NULL; free(transaction); + + return 0; } int odb_transaction_write_object_stream(struct odb_transaction *transaction, @@ -33,3 +47,11 @@ int odb_transaction_write_object_stream(struct odb_transaction *transaction, { return transaction->write_object_stream(transaction, stream, len, oid); } + +const char **odb_transaction_env(struct odb_transaction *transaction) +{ + if (!transaction) + return NULL; + + return transaction->env(transaction); +} diff --git a/odb/transaction.h b/odb/transaction.h index 854fda06f5..78392ff13d 100644 --- a/odb/transaction.h +++ b/odb/transaction.h @@ -1,8 +1,9 @@ #ifndef ODB_TRANSACTION_H #define ODB_TRANSACTION_H +#include "git-compat-util.h" +#include "gettext.h" #include "odb.h" -#include "odb/source.h" /* * A transaction may be started for an object database prior to writing new @@ -17,7 +18,7 @@ struct odb_transaction { struct odb_source *source; /* The ODB source specific callback invoked to commit a transaction. */ - void (*commit)(struct odb_transaction *transaction); + int (*commit)(struct odb_transaction *transaction); /* * This callback is expected to write the given object stream into @@ -30,20 +31,45 @@ struct odb_transaction { int (*write_object_stream)(struct odb_transaction *transaction, struct odb_write_stream *stream, size_t len, struct object_id *oid); + + /* + * This callback is expected to return a NULL-terminated array of + * environment variables that a child process should inherit so + * that its object writes participate in the transaction. The + * returned array is owned by the backend and remains valid until + * the transaction ends. May return NULL when the backend does not + * need to expose any state to child processes. + */ + const char **(*env)(struct odb_transaction *transaction); +}; + +enum odb_transaction_flags { + ODB_TRANSACTION_RECEIVE = (1 << 0), }; /* - * Starts an ODB transaction. Subsequent objects are written to the transaction - * and not committed until odb_transaction_commit() is invoked on the - * transaction. If the ODB already has a pending transaction, NULL is returned. + * Starts an ODB transaction and returns it via `out`. Subsequent objects are + * written to the transaction and not committed until odb_transaction_commit() + * is invoked on the transaction. Returns 0 on success and a negative value on + * error. If the ODB already has a pending transaction, `out` is set to NULL. */ -struct odb_transaction *odb_transaction_begin(struct object_database *odb); +int odb_transaction_begin(struct object_database *odb, + struct odb_transaction **out, + enum odb_transaction_flags flags); + +static inline void odb_transaction_begin_or_die(struct object_database *odb, + struct odb_transaction **out, + enum odb_transaction_flags flags) +{ + if (odb_transaction_begin(odb, out, flags)) + die(_("failed to start ODB transaction")); +} /* * Commits an ODB transaction making the written objects visible. If the * specified transaction is NULL, the function is a no-op. */ -void odb_transaction_commit(struct odb_transaction *transaction); +int odb_transaction_commit(struct odb_transaction *transaction); /* * Writes the object in the provided stream into the transaction. The resulting @@ -54,4 +80,13 @@ int odb_transaction_write_object_stream(struct odb_transaction *transaction, struct odb_write_stream *stream, size_t len, struct object_id *oid); +/* + * Returns a NULL-terminated array of environment variables that a child + * process should inherit so that its object writes participate in the + * transaction, suitable for passing via child_process.env. Returns NULL if + * the transaction is NULL or the backend does not expose any state to child + * processes. + */ +const char **odb_transaction_env(struct odb_transaction *transaction); + #endif diff --git a/read-cache.c b/read-cache.c index acfa940139..63313c8771 100644 --- a/read-cache.c +++ b/read-cache.c @@ -4056,7 +4056,7 @@ int add_files_to_cache(struct repository *repo, const char *prefix, * This function is invoked from commands other than 'add', which * may not have their own transaction active. */ - transaction = odb_transaction_begin(repo->objects); + odb_transaction_begin_or_die(repo->objects, &transaction, 0); run_diff_files(&rev, DIFF_RACY_IS_MODIFIED); odb_transaction_commit(transaction);