diff --git a/odb.c b/odb.c index caf1d0f542..cd9d5b48bc 100644 --- a/odb.c +++ b/odb.c @@ -1046,6 +1046,27 @@ bool odb_optimize_required(struct object_database *odb, return odb_source_optimize_required(odb->sources, opts); } +void odb_generate_pack_options_release(struct odb_generate_pack_options *opts) +{ + oid_array_clear(&opts->wants); + oid_array_clear(&opts->haves); + oid_array_clear(&opts->shallows); +} + +int odb_generate_pack(struct object_database *odb, + struct odb_pack_generator **out, + const struct odb_generate_pack_options *opts) +{ + if (!odb->sources->generate_pack) + return error(_("primary object source does not support generating packfiles")); + return odb_source_generate_pack(odb->sources, out, opts); +} + +int odb_pack_generator_finish(struct odb_pack_generator *generator) +{ + return generator->finish(generator); +} + struct object_database *odb_new(struct repository *repo, const char *primary_source, const char *secondary_sources) diff --git a/odb.h b/odb.h index fca67e8253..fc1442f243 100644 --- a/odb.h +++ b/odb.h @@ -2,6 +2,7 @@ #define ODB_H #include "object.h" +#include "oid-array.h" #include "oidset.h" #include "oidmap.h" #include "string-list.h" @@ -677,6 +678,157 @@ int odb_write_object_stream(struct object_database *odb, struct odb_write_stream *stream, size_t len, struct object_id *oid); +/* + * Options for generating a packfile via `odb_generate_pack()`. + */ +struct odb_generate_pack_options { + /* Tips of the object graph that shall be packed. */ + struct oid_array wants; + + /* + * Boundary of the object graph. Objects reachable from any of these + * tips are expected to already be available to whoever consumes the + * pack and shall thus not be packed. + */ + struct oid_array haves; + + /* + * The shallow boundary that shall be used when computing object + * reachability. When set, any shallow information of the repository + * itself shall be ignored in favor of these objects. + */ + struct oid_array shallows; + + /* + * Pre-expanded object filter specification that limits the set of + * objects that shall be packed. May be `NULL` in case no filter shall + * be applied. + */ + const char *filter_spec; + + /* + * Protocols that may be used to offload objects via packfile URIs. + * May be `NULL` in case packfile URIs shall not be used. + */ + const struct string_list *uri_protocols; + + /* + * Hook command that shall be executed instead of the internal + * machinery to generate the pack. It is up to the specific backend + * whether or not this hook is supported. May be `NULL` in case no + * hook shall be executed. + */ + const char *pack_objects_hook; + + /* + * File descriptor that the generated pack shall be written to. If set + * to `-1`, a pipe will be created and exposed via the pack generator's + * `out` field. If set to `0`, the pack will be written to the standard + * output stream. Otherwise, the provided descriptor will be written to + * and is consumed by the generator. + */ + int pack_fd; + + /* + * File descriptor that progress output shall be written to. The same + * semantics as for `pack_fd` apply, except that `0` will cause the + * generator to write to stderr instead of stdout. + */ + int progress_fd; + + /* Whether to print progress or not. */ + enum { + /* Don't print progress output. */ + ODB_GENERATE_PACK_PROGRESS_NONE, + + /* + * Print progress while computing the packfile, but stop + * printing progress once starting to write it. + */ + ODB_GENERATE_PACK_PROGRESS_STANDARD, + + /* + * Similar to STANDARD, but also print progress when writing + * the packfile. + */ + ODB_GENERATE_PACK_PROGRESS_VERBOSE, + } progress; + + /* Allow the pack to contain deltas against unpacked objects. */ + unsigned thin:1; + + /* Use offset deltas instead of reference deltas. */ + unsigned ofs_delta:1; + + /* Include unasked-for annotated tags of packed objects. */ + unsigned include_tag:1; + + /* The generated pack is destined for a shallow consumer. */ + unsigned shallow:1; + + /* Allow objects that may be missing due to a promisor remote. */ + unsigned missing_allow_promisor:1; + + /* Do not use bitmap indices when computing reachability. */ + unsigned disable_bitmaps:1; +}; + +#define ODB_GENERATE_PACK_OPTIONS_INIT { \ + .wants = OID_ARRAY_INIT, \ + .haves = OID_ARRAY_INIT, \ + .shallows = OID_ARRAY_INIT, \ + .pack_fd = -1, \ +} + +/* Release resources associated with the options. */ +void odb_generate_pack_options_release(struct odb_generate_pack_options *opts); + +/* + * A handle for an ongoing packfile generation as started via + * `odb_generate_pack()`. + */ +struct odb_pack_generator { + /* + * File descriptor from which the generated pack can be read. Only set + * when the pack generation was started with `pack_fd == -1`. The + * caller is responsible for closing the descriptor. + */ + int out; + + /* + * File descriptor from which progress output can be read. Only set + * when the pack generation was started with `progress_fd == -1`. The + * caller is responsible for closing the descriptor. + */ + int err; + + /* + * Callback function to finish this generator. This callback is + * expected to wait for the packfile generation to complete and to then + * free the generator itself. + */ + int (*finish)(struct odb_pack_generator *); +}; + +/* + * Start generating a packfile from the object database with the given + * options. The pack is generated asynchronously; the caller is expected to + * consume the file descriptors exposed via the pack generator and to then + * wait for completion via `odb_pack_generator_finish()`. + * + * Returns 0 on success and populates the `out` pointer with the pack + * generator. Returns a negative error code otherwise. + */ +int odb_generate_pack(struct object_database *odb, + struct odb_pack_generator **out, + const struct odb_generate_pack_options *opts); + +/* + * Wait for the packfile generation to complete and free the pack generator. + * Returns 0 on success, a negative error code otherwise. + */ +int odb_pack_generator_finish(struct odb_pack_generator *generator); + void parse_alternates(const char *string, int sep, const char *relative_base, diff --git a/odb/source-files.c b/odb/source-files.c index 5a68af7d84..a33e01fbed 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -4,6 +4,7 @@ #include "chdir-notify.h" #include "config.h" #include "gettext.h" +#include "hex.h" #include "lockfile.h" #include "object-file.h" #include "odb.h" @@ -729,6 +730,153 @@ out: return ret; } +struct odb_pack_generator_files { + struct odb_pack_generator base; + struct child_process cp; +}; + +static int odb_pack_generator_files_finish(struct odb_pack_generator *_generator) +{ + struct odb_pack_generator_files *generator = + (struct odb_pack_generator_files *)_generator; + int ret; + + ret = finish_command(&generator->cp); + free(generator); + + if (ret) { + /* + * On failure, pack-objects is expected to have written a + * useful error message to its standard error stream already. + * Death by signal is worth mentioning, though, with the + * exception of SIGPIPE: that is a normal occurrence when the + * consumer of the pack hangs up. + */ + if (ret > 128 && ret - 128 == SIGPIPE) + return -1; + if (ret > 128) + error(_("pack-objects died of signal %d"), ret - 128); + return -1; + } + + return 0; +} + +static int odb_source_files_generate_pack(struct odb_source *source UNUSED, + struct odb_pack_generator **out, + const struct odb_generate_pack_options *opts) +{ + struct odb_pack_generator_files *generator; + struct child_process *cp; + FILE *in; + + CALLOC_ARRAY(generator, 1); + child_process_init(&generator->cp); + cp = &generator->cp; + + /* + * The hook is expected to spawn "$hook git pack-objects " + * and to behave like git-pack-objects(1) would have. This can for + * example be used to serve precomputed packfiles. + */ + if (opts->pack_objects_hook) { + strvec_push(&cp->args, opts->pack_objects_hook); + strvec_push(&cp->args, "git"); + cp->use_shell = 1; + } else { + cp->git_cmd = 1; + } + + /* + * The caller-provided shallow boundary overrides any shallow state + * that the repository itself may have, so the shallow file needs to + * be neutralized. + */ + if (opts->shallows.nr) { + strvec_push(&cp->args, "--shallow-file"); + strvec_push(&cp->args, ""); + } + strvec_push(&cp->args, "pack-objects"); + strvec_push(&cp->args, "--revs"); + strvec_push(&cp->args, "--stdout"); + if (opts->thin) + strvec_push(&cp->args, "--thin"); + if (opts->shallow) + strvec_push(&cp->args, "--shallow"); + if (opts->ofs_delta) + strvec_push(&cp->args, "--delta-base-offset"); + if (opts->include_tag) + strvec_push(&cp->args, "--include-tag"); + if (opts->missing_allow_promisor) + strvec_push(&cp->args, "--missing=allow-promisor"); + if (opts->disable_bitmaps) + strvec_push(&cp->args, "--no-use-bitmap-index"); + switch (opts->progress) { + case ODB_GENERATE_PACK_PROGRESS_NONE: + strvec_push(&cp->args, "--quiet"); + break; + case ODB_GENERATE_PACK_PROGRESS_STANDARD: + strvec_push(&cp->args, "--progress"); + break; + case ODB_GENERATE_PACK_PROGRESS_VERBOSE: + strvec_push(&cp->args, "--all-progress"); + break; + default: + BUG("unknown progress option %d", opts->progress); + } + if (opts->filter_spec) + strvec_pushf(&cp->args, "--filter=%s", opts->filter_spec); + if (opts->uri_protocols) + for (size_t i = 0; i < opts->uri_protocols->nr; i++) + strvec_pushf(&cp->args, "--uri-protocol=%s", + opts->uri_protocols->items[i].string); + + cp->in = -1; + cp->out = opts->pack_fd; + cp->err = opts->progress_fd; + cp->clean_on_exit = 1; + + if (start_command(cp)) { + free(generator); + return error(_("could not spawn pack-objects")); + } + + /* + * Feed the objects to pack-objects. This is safe to do synchronously + * because pack-objects consumes all of its standard input before it + * starts to generate the pack. + */ + in = xfdopen(cp->in, "w"); + for (size_t i = 0; i < opts->shallows.nr; i++) + fprintf(in, "--shallow %s\n", oid_to_hex(&opts->shallows.oid[i])); + for (size_t i = 0; i < opts->wants.nr; i++) + fprintf(in, "%s\n", oid_to_hex(&opts->wants.oid[i])); + fprintf(in, "--not\n"); + for (size_t i = 0; i < opts->haves.nr; i++) + fprintf(in, "%s\n", oid_to_hex(&opts->haves.oid[i])); + fprintf(in, "\n"); + fflush(in); + if (ferror(in)) { + error(_("error writing to pack-objects")); + fclose(in); + if (opts->pack_fd < 0) + close(cp->out); + if (opts->progress_fd < 0) + close(cp->err); + finish_command(cp); + free(generator); + return -1; + } + fclose(in); + + generator->base.out = opts->pack_fd < 0 ? cp->out : -1; + generator->base.err = opts->progress_fd < 0 ? cp->err : -1; + generator->base.finish = odb_pack_generator_files_finish; + + *out = &generator->base; + return 0; +} + struct odb_source_files *odb_source_files_new(struct object_database *odb, const char *path, bool local) @@ -756,6 +904,7 @@ struct odb_source_files *odb_source_files_new(struct object_database *odb, files->base.write_alternate = odb_source_files_write_alternate; files->base.optimize = odb_source_files_optimize; files->base.optimize_required = odb_source_files_optimize_required; + files->base.generate_pack = odb_source_files_generate_pack; /* * Ideally, we would only ever store absolute paths in the source. This diff --git a/odb/source.h b/odb/source.h index d69f8e2d1c..e2129766fc 100644 --- a/odb/source.h +++ b/odb/source.h @@ -278,6 +278,23 @@ struct odb_source { */ bool (*optimize_required)(struct odb_source *source, const struct odb_optimize_options *opts); + + /* + * This callback is expected to start generating a packfile with the + * given options. The pack shall be generated asynchronously so that + * the caller can consume the pack data and progress output while the + * pack is being generated. + * + * This callback is optional. Sources that cannot generate packfiles + * shall leave it unset. + * + * The callback is expected to return 0 on success and populate the + * `out` pointer with the pack generator, a negative error code + * otherwise. + */ + int (*generate_pack)(struct odb_source *source, + struct odb_pack_generator **out, + const struct odb_generate_pack_options *opts); }; /* @@ -520,4 +537,20 @@ static inline bool odb_source_optimize_required(struct odb_source *source, return source->optimize_required(source, opts); } +/* + * Start generating a packfile from the given source with the given options. + * The pack is generated asynchronously; the caller is expected to consume the + * file descriptors exposed via the pack generator and to then wait for + * completion via `odb_pack_generator_finish()`. + * + * Returns 0 on success and populates the `out` pointer with the pack + * generator, a negative error code otherwise. + */ +static inline int odb_source_generate_pack(struct odb_source *source, + struct odb_pack_generator **out, + const struct odb_generate_pack_options *opts) +{ + return source->generate_pack(source, out, opts); +} + #endif