diff --git a/Documentation/git-cat-file.adoc b/Documentation/git-cat-file.adoc index 86b9181599..ac3b528c6f 100644 --- a/Documentation/git-cat-file.adoc +++ b/Documentation/git-cat-file.adoc @@ -169,6 +169,13 @@ info :: Print object info for object reference ``. This corresponds to the output of `--batch-check`. +remote-object-info ...:: + Print object info for object references `` at specified + `` without downloading objects from the remote. + Raise an error when the `object-info` capability is not supported by the remote. + Raise an error when no object references are provided. + This command may be combined with `--buffer`. + flush:: Used with `--buffer` to execute all preceding commands that were issued since the beginning or since the last flush was issued. When `--buffer` @@ -301,7 +308,8 @@ one per line, and print information based on the command given. With `--batch-command`, the `info` command followed by an object will print information about the object the same way `--batch-check` would, and the `contents` command followed by an object prints contents in the same way -`--batch` would. +`--batch` would. The `remote-object-info` command followed by a remote and +object IDs prints object info from the remote without downloading the objects. You can specify the information shown for each object by using a custom ``. The `` is copied literally to stdout for each @@ -340,8 +348,15 @@ newline. The available atoms are: after that first run of whitespace (i.e., the "rest" of the line) are output in place of the `%(rest)` atom. +The command `remote-object-info` only supports the `%(objectname)` and +`%(objectsize)` placeholders. See `CAVEATS` below for more information. + If no format is specified, the default format is `%(objectname) -%(objecttype) %(objectsize)`. +%(objecttype) %(objectsize)`, except for `remote-object-info` commands which +use `%(objectname) %(objectsize)` because `%(objecttype)` is not supported yet. + +WARNING: When "%(objecttype)" is supported, the default format WILL be unified, +so DO NOT RELY on the current default format to stay the same!!! If `--batch` is specified, or if `--batch-command` is used with the `contents` command, the object information is followed by the object contents (consisting @@ -438,6 +453,10 @@ scripting purposes. CAVEATS ------- +Note that only `%(objectname)` and `%(objectsize)` are currently +supported by the `remote-object-info` command. Using any other placeholder in +the format string will return an empty string in its position. + Note that the sizes of objects on disk are reported accurately, but care should be taken in drawing conclusions about which refs or objects are responsible for disk usage. The size of a packed non-delta object may be diff --git a/Documentation/gitprotocol-v2.adoc b/Documentation/gitprotocol-v2.adoc index 2beb70595f..7bf62014c3 100644 --- a/Documentation/gitprotocol-v2.adoc +++ b/Documentation/gitprotocol-v2.adoc @@ -568,21 +568,26 @@ An `object-info` request takes the following arguments: oid Indicates to the server an object which the client wants to obtain - information for. + information for. They must be full OIDs. -The response of `object-info` is a list of the requested object ids -and associated requested information, each separated by a single space. +The response of `object-info` consists of one pkt-line per requested attribute, +echoing the attributes the server will report, followed by one pkt-line per +requested object id with its information, each field separated by a single +space. output = info flush-pkt - info = PKT-LINE(attrs) LF) - *PKT-LINE(obj-info LF) - - attrs = attr | attrs SP attrs + info = *PKT-LINE(attr LF) + *PKT-LINE(obj-info LF) attr = "size" - obj-info = obj-id SP obj-size + obj-size = 1*DIGIT + + obj-info = obj-id [SP [obj-size]] + +If the server does not recognize the OID, the response will be ` SP` +regardless of the number of attributes requested. bundle-uri ~~~~~~~~~~ diff --git a/Makefile b/Makefile index 98e995e4be..fac3e8879c 100644 --- a/Makefile +++ b/Makefile @@ -1169,6 +1169,7 @@ LIB_OBJS += ewah/ewah_io.o LIB_OBJS += ewah/ewah_rlw.o LIB_OBJS += exec-cmd.o LIB_OBJS += fetch-negotiator.o +LIB_OBJS += fetch-object-info.o LIB_OBJS += fetch-pack.o LIB_OBJS += fmt-merge-msg.o LIB_OBJS += fsck.o diff --git a/builtin/cat-file.c b/builtin/cat-file.c index 1458dd76d6..884b6d5ad3 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -28,6 +28,22 @@ #include "promisor-remote.h" #include "mailmap.h" #include "write-or-die.h" +#include "alias.h" +#include "remote.h" +#include "transport.h" + +/* + * Maximum length for a remote URL. While no universal standard exists, + * 8K is assumed to be a reasonable limit. + */ +#define MAX_REMOTE_URL_LEN (8 * 1024) + +/* Maximum number of objects allowed in a single remote-object-info request. */ +#define MAX_ALLOWED_OBJ_LIMIT 10000 + +/* Maximum input size permitted for the remote-object-info command. */ +#define MAX_REMOTE_OBJ_INFO_LINE \ + (MAX_REMOTE_URL_LEN + MAX_ALLOWED_OBJ_LIMIT * (GIT_MAX_HEXSZ + 1)) enum batch_mode { BATCH_MODE_CONTENTS, @@ -316,8 +332,24 @@ struct expand_data { * optimized out. */ unsigned skip_object_info : 1; + + /* + * Flags about when an object info is being fetched from remote. + */ + unsigned is_remote:1; + + /* + * List of atoms (i.e. "objectsize") that the server supports. Built + * from the server's object-info advertised capabilities. + */ + struct string_list remote_allowed_atoms; }; -#define EXPAND_DATA_INIT { .mode = S_IFINVALID } + +#define EXPAND_DATA_INIT { \ + .mode = S_IFINVALID, \ + .type = OBJ_BAD, \ + .remote_allowed_atoms = STRING_LIST_INIT_NODUP, \ +} static int is_atom(const char *atom, const char *s, int slen) { @@ -328,14 +360,26 @@ static int is_atom(const char *atom, const char *s, int slen) static int expand_atom(struct strbuf *sb, const char *atom, int len, struct expand_data *data) { + if (data->is_remote) { + size_t i; + for (i = 0; i < data->remote_allowed_atoms.nr; i++) + if (is_atom(data->remote_allowed_atoms.items[i].string, + atom, len)) + break; + if (i == data->remote_allowed_atoms.nr) + return 1; + } + if (is_atom("objectname", atom, len)) { if (!data->mark_query) strbuf_add_oid_hex(sb, &data->oid); } else if (is_atom("objecttype", atom, len)) { - if (data->mark_query) + if (data->mark_query) { data->info.typep = &data->type; - else - strbuf_addstr(sb, type_name(data->type)); + } else { + const char *t = type_name(data->type); + strbuf_addstr(sb, t ? t : ""); + } } else if (is_atom("objectsize", atom, len)) { if (data->mark_query) data->info.sizep = &data->size; @@ -635,6 +679,62 @@ out: object_context_release(&ctx); } +static int get_remote_info(int argc, + const char **argv, + struct object_info **remote_object_info, + struct oid_array *object_info_oids, + struct string_list *object_info_options) +{ + int retval = 0; + struct remote *remote = NULL; + struct object_id oid; + struct transport *gtransport; + + remote = remote_get(argv[0]); + if (!remote) + die(_("must supply valid remote when using remote-object-info")); + + oid_array_clear(object_info_oids); + for (size_t i = 1; i < argc; i++) { + if (get_oid_hex(argv[i], &oid)) { + size_t len = strlen(argv[i]); + + if (len < the_hash_algo->hexsz && len >= 4) { + size_t j; + for (j = 0; j < len; j++) + if (!isxdigit(argv[i][j])) + break; + if (j == len) + die(_("remote-object-info does not support " + "short oids, %d characters required"), + (int)the_hash_algo->hexsz); + } + die(_("not a valid object name '%s'"), argv[i]); + } + oid_array_append(object_info_oids, &oid); + } + + if (!object_info_oids->nr) + die(_("remote-object-info requires objects")); + + gtransport = transport_get(remote, NULL); + + if (!gtransport->smart_options) { + retval = -1; + goto cleanup; + } + + CALLOC_ARRAY(*remote_object_info, object_info_oids->nr); + gtransport->smart_options->object_info_oids = object_info_oids; + + gtransport->smart_options->object_info_options = object_info_options; + gtransport->smart_options->object_info_data = *remote_object_info; + retval = transport_fetch_object_info(gtransport); +cleanup: + transport_disconnect(gtransport); + return retval; +} + struct object_cb_data { struct batch_options *opt; struct expand_data *expand; @@ -716,18 +816,115 @@ static void parse_cmd_mailmap(struct batch_options *opt UNUSED, load_mailmap(); } +struct protocol_placeholder_entry { + const char *option; + const char *atom; +}; + +static const struct protocol_placeholder_entry remote_atom_map[] = { + {"size", "objectsize"}, + {"type", "objecttype"}, + /* + * Add new protocol options here. Even if the server doesn't support + * them the allow_list will drop them if the server doesn't advertise + * them. + */ +}; + +static void parse_cmd_remote_object_info(struct batch_options *opt, + const char *line, struct strbuf *output, + struct expand_data *data) +{ + int count; + const char **argv; + char *line_to_split; + struct object_info *remote_object_info = NULL; + struct oid_array object_info_oids = OID_ARRAY_INIT; + struct string_list object_info_options = STRING_LIST_INIT_NODUP; + const char *saved_format = opt->format; + + if (strlen(line) >= MAX_REMOTE_OBJ_INFO_LINE) + die(_("remote-object-info command too long")); + /* + * TODO: Use the default format once %(objecttype) is supported. + */ + if (!opt->format) + opt->format = "%(objectname) %(objectsize)"; + + line_to_split = xstrdup(line); + count = split_cmdline(line_to_split, &argv); + if (count < 0) + die(_("remote-object-info: failed to parse command line: %s"), + split_cmdline_strerror(count)); + if (count - 1 > MAX_ALLOWED_OBJ_LIMIT) + die(_("remote-object-info supports at most %d objects"), + MAX_ALLOWED_OBJ_LIMIT); + + if (data->info.sizep) + string_list_append(&object_info_options, "size"); + if (data->info.typep) + string_list_append(&object_info_options, "type"); + + if (get_remote_info(count, argv, &remote_object_info, + &object_info_oids, &object_info_options)) + die(_("failed to get object info from the remote: %s"), argv[0]); + + string_list_clear(&data->remote_allowed_atoms, 0); + string_list_append(&data->remote_allowed_atoms, "objectname"); + for (size_t i = 0; i < ARRAY_SIZE(remote_atom_map); i++) + if (unsorted_string_list_has_string(&object_info_options, remote_atom_map[i].option)) + string_list_append(&data->remote_allowed_atoms, + remote_atom_map[i].atom); + + data->skip_object_info = 1; + for (size_t i = 0; i < object_info_oids.nr; i++) { + data->oid = object_info_oids.oid[i]; + + if (remote_object_info[i].unrecognized) { + report_object_status(opt, oid_to_hex(&data->oid), + &data->oid, "missing"); + continue; + } + + /* + * When reaching here, it means remote-object-info can retrieve + * information from server without downloading them. + */ + if (remote_object_info[i].sizep) { + data->size = *remote_object_info[i].sizep; + } + + if (remote_object_info[i].typep) { + data->type = *remote_object_info[i].typep; + } + + opt->batch_mode = BATCH_MODE_INFO; + data->is_remote = 1; + batch_object_write(argv[i + 1], output, opt, data, NULL, 0); + data->is_remote = 0; + } + data->skip_object_info = 0; + opt->format = saved_format; + + for (size_t i = 0; i < object_info_oids.nr; i++) + free_object_info_contents(&remote_object_info[i]); + string_list_clear(&object_info_options, 0); + free(line_to_split); + free(argv); + free(remote_object_info); + oid_array_clear(&object_info_oids); +} + static void dispatch_calls(struct batch_options *opt, struct strbuf *output, struct expand_data *data, struct queued_cmd *cmd, - int nr) + size_t nr) { - int i; - if (!opt->buffer_output) die(_("flush is only for --buffer mode")); - for (i = 0; i < nr; i++) + for (size_t i = 0; i < nr; i++) cmd[i].fn(opt, cmd[i].line, output, data); fflush(stdout); @@ -735,9 +932,7 @@ static void dispatch_calls(struct batch_options *opt, static void free_cmds(struct queued_cmd *cmd, size_t *nr) { - size_t i; - - for (i = 0; i < *nr; i++) + for (size_t i = 0; i < *nr; i++) FREE_AND_NULL(cmd[i].line); *nr = 0; @@ -750,9 +945,10 @@ static const struct parse_cmd { unsigned takes_args; } commands[] = { { "contents", parse_cmd_contents, 1 }, - { "info", parse_cmd_info, 1 }, { "flush", NULL, 0 }, + { "info", parse_cmd_info, 1 }, { "mailmap", parse_cmd_mailmap, 1 }, + { "remote-object-info", parse_cmd_remote_object_info, 1 }, }; static void batch_objects_command(struct batch_options *opt, @@ -764,7 +960,6 @@ static void batch_objects_command(struct batch_options *opt, size_t alloc = 0, nr = 0; while (strbuf_getdelim_strip_crlf(&input, stdin, opt->input_delim) != EOF) { - int i; const struct parse_cmd *cmd = NULL; const char *p = NULL, *cmd_end; struct queued_cmd call = {0}; @@ -774,7 +969,7 @@ static void batch_objects_command(struct batch_options *opt, if (isspace(*input.buf)) die(_("whitespace before command: '%s'"), input.buf); - for (i = 0; i < ARRAY_SIZE(commands); i++) { + for (size_t i = 0; i < ARRAY_SIZE(commands); i++) { if (!skip_prefix(input.buf, commands[i].name, &cmd_end)) continue; @@ -972,6 +1167,7 @@ static int batch_objects(struct batch_options *opt) cleanup: strbuf_release(&input); strbuf_release(&output); + string_list_clear(&data.remote_allowed_atoms, 0); cfg->warn_on_object_refname_ambiguity = save_warning; return retval; } diff --git a/connect.c b/connect.c index 1d74c1eda2..feb0420b0f 100644 --- a/connect.c +++ b/connect.c @@ -709,6 +709,40 @@ int server_supports(const char *feature) return !!server_feature_value(feature, NULL); } +void write_command_and_capabilities(struct strbuf *req_buf, const char *command, + const struct string_list *server_options) +{ + const char *hash_name; + int advertise_sid = 0; + + repo_config_get_bool(the_repository, "transfer.advertisesid", &advertise_sid); + + ensure_server_supports_v2(command); + packet_buf_write(req_buf, "command=%s", command); + if (server_supports_v2("agent")) + packet_buf_write(req_buf, "agent=%s", git_user_agent_sanitized()); + if (advertise_sid && server_supports_v2("session-id")) + packet_buf_write(req_buf, "session-id=%s", trace2_session_id()); + if (server_options && server_options->nr) { + ensure_server_supports_v2("server-option"); + for (size_t i = 0; i < server_options->nr; i++) + packet_buf_write(req_buf, "server-option=%s", + server_options->items[i].string); + } + + if (server_feature_v2("object-format", &hash_name)) { + const unsigned int hash_algo = hash_algo_by_name(hash_name); + if (hash_algo_by_ptr(the_hash_algo) != hash_algo) + die(_("mismatched algorithms: client %s; server %s"), + the_hash_algo->name, hash_name); + packet_buf_write(req_buf, "object-format=%s", the_hash_algo->name); + } else if (hash_algo_by_ptr(the_hash_algo) != GIT_HASH_SHA1_LEGACY) { + die(_("the server does not support algorithm '%s'"), + the_hash_algo->name); + } + packet_buf_delim(req_buf); +} + static const char *url_scheme_name(enum url_scheme scheme) { switch (scheme) { diff --git a/connect.h b/connect.h index aa482a37fb..957e5fe2b9 100644 --- a/connect.h +++ b/connect.h @@ -34,4 +34,12 @@ void check_stateless_delimiter(int stateless_rpc, struct packet_reader *reader, const char *error); +struct string_list; +/* + * Write a protocol v2 command request, along with the capability + * advertisements, into req_buf. + */ +void write_command_and_capabilities(struct strbuf *req_buf, const char *command, + const struct string_list *server_options); + #endif diff --git a/fetch-object-info.c b/fetch-object-info.c new file mode 100644 index 0000000000..ba7e179c44 --- /dev/null +++ b/fetch-object-info.c @@ -0,0 +1,156 @@ +#include "git-compat-util.h" +#include "gettext.h" +#include "hex.h" +#include "pkt-line.h" +#include "connect.h" +#include "oid-array.h" +#include "odb.h" +#include "fetch-object-info.h" +#include "string-list.h" + +/* Sends object-info command and its arguments into the request buffer. */ +static void send_object_info_request(const int fd_out, struct object_info_args *args) +{ + struct strbuf req_buf = STRBUF_INIT; + + write_command_and_capabilities(&req_buf, "object-info", args->server_options); + + if (unsorted_string_list_has_string(args->object_info_options, "size")) + packet_buf_write(&req_buf, "size"); + else if (args->object_info_options->nr) + BUG("only size should be in object_info_options"); + + if (args->oids) + for (size_t i = 0; i < args->oids->nr; i++) + packet_buf_write(&req_buf, "oid %s", oid_to_hex(&args->oids->oid[i])); + + packet_buf_flush(&req_buf); + if (write_in_full(fd_out, req_buf.buf, req_buf.len) < 0) + die_errno(_("unable to write request to remote")); + + strbuf_release(&req_buf); +} + +static int parse_object_size(const char *s, size_t *res) +{ + uintmax_t uim; + + if (!s[0] || s[strspn(s, "0123456789")]) + return -1; + errno = 0; + uim = strtoumax(s, NULL, 10); + if (errno || uim > SIZE_MAX) + return -1; + *res = uim; + return 0; +} + +int fetch_object_info(const enum protocol_version version, struct object_info_args *args, + struct packet_reader *reader, struct object_info *object_info_data, + const int stateless_rpc, const int fd_out) +{ + int size_index = -1; + + switch (version) { + case protocol_v2: + if (!server_supports_v2("object-info")) + die(_("object-info capability is not enabled on the server")); + /* + * When removing an element from the list it gets swapped by the + * last element, iterate backwards to prevent elements skipping + * evaluation. + * + * object_info_options->nr can be safely casted without overflow + * because the number of options is a small known number (the + * supported placeholders which currently are size and type). + */ + for (int i = (int)args->object_info_options->nr - 1; i >= 0; i--) + if (!server_supports_feature("object-info", + args->object_info_options->items[i].string, 0)) + unsorted_string_list_delete_item(args->object_info_options, i, 0); + + /* + * Even if no options are left, we still send the oid so we get + * at least an existence check. + */ + send_object_info_request(fd_out, args); + break; + case protocol_v1: + case protocol_v0: + die(_("object-info requires protocol v2")); + case protocol_unknown_version: + BUG("unknown protocol version"); + } + + for (size_t i = 0; i < args->object_info_options->nr; i++) { + if (packet_reader_read(reader) != PACKET_READ_NORMAL) { + check_stateless_delimiter(stateless_rpc, reader, + "stateless delimiter expected"); + return -1; + } + + if (!unsorted_string_list_has_string(args->object_info_options, reader->line)) + return -1; + + if (!strcmp(reader->line, "size")) { + /* + * i is the number of supported options which currently + * is only size. No risk of overflow. + */ + size_index = (int)i; + for (size_t j = 0; j < args->oids->nr; j++) + object_info_data[j].sizep = + xcalloc(1, sizeof(*object_info_data[j].sizep)); + } else { + BUG("only size is supported"); + } + } + + for (size_t i = 0; + packet_reader_read(reader) == PACKET_READ_NORMAL && + i < args->oids->nr; + i++) { + struct string_list object_info_values = STRING_LIST_INIT_DUP; + + string_list_split(&object_info_values, reader->line, " ", -1); + + if (strcmp(object_info_values.items[0].string, + oid_to_hex(&args->oids->oid[i]))) + die(_("object-info: expected OID: %s, got %s"), + oid_to_hex(&args->oids->oid[i]), + object_info_values.items[0].string); + + /* + * If the response is two elements but the second one is an + * empty string, that means that the OID is unrecognized by the + * server. + */ + if (object_info_values.nr >= 2 && + !strcmp(object_info_values.items[1].string, "")) { + object_info_data[i].unrecognized = 1; + string_list_clear(&object_info_values, 0); + continue; + } + + /* + * Because we filter the options to be only the supported by + * the server we expect the server to answer with the same + * number of attributes requested. + */ + if (args->object_info_options->nr + 1 != object_info_values.nr) + die("object-info: unexpected number of attributes: %s", + reader->line); + + if (size_index >= 0 && + parse_object_size(object_info_values.items[size_index + 1].string, + object_info_data[i].sizep)) + die("object-info: ref %s has invalid size %s", + object_info_values.items[0].string, + object_info_values.items[size_index + 1].string); + + string_list_clear(&object_info_values, 0); + } + check_stateless_delimiter(stateless_rpc, reader, "stateless delimiter expected"); + + return 0; +} diff --git a/fetch-object-info.h b/fetch-object-info.h new file mode 100644 index 0000000000..269cebb3f7 --- /dev/null +++ b/fetch-object-info.h @@ -0,0 +1,25 @@ +#ifndef FETCH_OBJECT_INFO_H +#define FETCH_OBJECT_INFO_H + +#include "pkt-line.h" +#include "protocol.h" + +struct object_info_args { + struct string_list *object_info_options; + const struct string_list *server_options; + struct oid_array *oids; +}; + +struct object_info; +/* + * Sends git-cat-file object-info command into the request buf and read the + * results from packets. + * + * Modifies args->object_info_options, on return it contains only the supported + * options by the server. + */ +int fetch_object_info(enum protocol_version version, struct object_info_args *args, + struct packet_reader *reader, struct object_info *object_info_data, + int stateless_rpc, int fd_out); + +#endif /* FETCH_OBJECT_INFO_H */ diff --git a/fetch-pack.c b/fetch-pack.c index 29c41132ee..922a9b2581 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -49,7 +49,6 @@ static int fetch_fsck_objects = -1; static int transfer_fsck_objects = -1; static int agent_supported; static int server_supports_filtering; -static int advertise_sid; static struct shallow_lock shallow_lock; static const char *alternate_shallow_file; static struct strbuf fsck_msg_types = STRBUF_INIT; @@ -363,6 +362,9 @@ static int find_common(struct fetch_negotiator *negotiator, size_t state_len = 0; struct packet_reader reader; struct oidset negotiation_include_oids = OIDSET_INIT; + int advertise_sid = 0; + + repo_config_get_bool(the_repository, "transfer.advertisesid", &advertise_sid); if (args->stateless_rpc && multi_ack == 1) die(_("the option '%s' requires '%s'"), "--stateless-rpc", "multi_ack_detailed"); @@ -414,7 +416,7 @@ static int find_common(struct fetch_negotiator *negotiator, if (deepen_not_ok) strbuf_addstr(&c, " deepen-not"); if (agent_supported) strbuf_addf(&c, " agent=%s", git_user_agent_sanitized()); - if (advertise_sid) + if (advertise_sid && server_supports("session-id")) strbuf_addf(&c, " session-id=%s", trace2_session_id()); if (args->filter_options.choice) strbuf_addstr(&c, " filter"); @@ -1160,9 +1162,6 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, (int)agent_len, agent_feature); } - if (!server_supports("session-id")) - advertise_sid = 0; - if (server_supports("shallow")) print_verbose(args, _("Server supports %s"), "shallow"); else if (args->depth > 0 || is_repository_shallow(r)) @@ -1376,38 +1375,6 @@ static int add_haves(struct fetch_negotiator *negotiator, return haves_added; } -static void write_fetch_command_and_capabilities(struct strbuf *req_buf, - const struct string_list *server_options) -{ - const char *hash_name; - - ensure_server_supports_v2("fetch"); - packet_buf_write(req_buf, "command=fetch"); - if (server_supports_v2("agent")) - packet_buf_write(req_buf, "agent=%s", git_user_agent_sanitized()); - if (advertise_sid && server_supports_v2("session-id")) - packet_buf_write(req_buf, "session-id=%s", trace2_session_id()); - if (server_options && server_options->nr) { - int i; - ensure_server_supports_v2("server-option"); - for (i = 0; i < server_options->nr; i++) - packet_buf_write(req_buf, "server-option=%s", - server_options->items[i].string); - } - - if (server_feature_v2("object-format", &hash_name)) { - int hash_algo = hash_algo_by_name(hash_name); - if (hash_algo_by_ptr(the_hash_algo) != hash_algo) - die(_("mismatched algorithms: client %s; server %s"), - the_hash_algo->name, hash_name); - packet_buf_write(req_buf, "object-format=%s", the_hash_algo->name); - } else if (hash_algo_by_ptr(the_hash_algo) != GIT_HASH_SHA1_LEGACY) { - die(_("the server does not support algorithm '%s'"), - the_hash_algo->name); - } - packet_buf_delim(req_buf); -} - static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, struct fetch_pack_args *args, const struct ref *wants, struct oidset *common, @@ -1419,7 +1386,7 @@ static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, int done_sent = 0; struct strbuf req_buf = STRBUF_INIT; - write_fetch_command_and_capabilities(&req_buf, args->server_options); + write_command_and_capabilities(&req_buf, "fetch", args->server_options); if (args->use_thin_pack) packet_buf_write(&req_buf, "thin-pack"); @@ -1768,18 +1735,18 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, reader.me = "fetch-pack"; } + /* v2 supports these by default */ + allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1; + use_sideband = 2; + if (args->depth > 0 || args->deepen_since || args->deepen_not) + args->deepen = 1; + while (state != FETCH_DONE) { switch (state) { case FETCH_CHECK_LOCAL: sort_ref_list(&ref, ref_compare_name); QSORT(sought, nr_sought, cmp_ref_by_name); - /* v2 supports these by default */ - allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1; - use_sideband = 2; - if (args->depth > 0 || args->deepen_since || args->deepen_not) - args->deepen = 1; - /* Filter 'ref' by 'sought' and those that aren't local */ mark_complete_and_common_ref(negotiator, args, &ref); filter_refs(args, &ref, sought, nr_sought); @@ -1999,7 +1966,6 @@ static void fetch_pack_config(void) repo_config_get_bool(the_repository, "repack.usedeltabaseoffset", &prefer_ofs_delta); repo_config_get_bool(the_repository, "fetch.fsckobjects", &fetch_fsck_objects); repo_config_get_bool(the_repository, "transfer.fsckobjects", &transfer_fsck_objects); - repo_config_get_bool(the_repository, "transfer.advertisesid", &advertise_sid); if (!uri_protocols.nr) { char *str; @@ -2287,7 +2253,7 @@ void negotiate_using_fetch(const struct oid_array *negotiation_restrict_tips, the_repository, "%d", negotiation_round); strbuf_reset(&req_buf); - write_fetch_command_and_capabilities(&req_buf, server_options); + write_command_and_capabilities(&req_buf, "fetch", server_options); packet_buf_write(&req_buf, "wait-for-done"); diff --git a/meson.build b/meson.build index f7c40ea079..7073d5844d 100644 --- a/meson.build +++ b/meson.build @@ -374,6 +374,7 @@ libgit_sources = [ 'ewah/ewah_rlw.c', 'exec-cmd.c', 'fetch-negotiator.c', + 'fetch-object-info.c', 'fetch-pack.c', 'fmt-merge-msg.c', 'fsck.c', diff --git a/object-file.c b/object-file.c index 0567b9d5f1..6f48e5e418 100644 --- a/object-file.c +++ b/object-file.c @@ -1741,3 +1741,13 @@ int odb_transaction_files_begin(struct odb_source *source, return 0; } + +void free_object_info_contents(struct object_info *object_info) +{ + if (!object_info) + return; + free(object_info->typep); + free(object_info->sizep); + free(object_info->disk_sizep); + free(object_info->delta_base_oid); +} diff --git a/odb.h b/odb.h index 7995bed97b..3f7c483656 100644 --- a/odb.h +++ b/odb.h @@ -340,6 +340,12 @@ struct object_info { * or multiple times in the same source. */ struct odb_source_info *source_infop; + + /* + * object-info protocol specific. Set by the protocol when the remote + * does not recognize the requested object. + */ + unsigned int unrecognized:1; }; /* @@ -629,4 +635,7 @@ void parse_alternates(const char *string, const char *relative_base, struct strvec *out); +/* Free pointers inside of object_info, but not object_info itself */ +void free_object_info_contents(struct object_info *object_info); + #endif /* ODB_H */ diff --git a/protocol-caps.c b/protocol-caps.c index 8858ea4489..02261be14d 100644 --- a/protocol-caps.c +++ b/protocol-caps.c @@ -30,6 +30,32 @@ static int parse_oid(const char *line, struct string_list *oid_str_list) return 1; } +/* + * odb_read_object_info_extended() wrapper. Similar to odb_read_object_info() + * but uses the flags: + * + * - OBJECT_INFO_SKIP_FETCH_OBJECT so a server won't fetch an object when a + * object-info request asks for an OID that it doesn't have. + * + * - OBJECT_INFO_QUICK to avoid re-scanning packs when the object is not found. + */ +static enum object_type get_object_info(struct object_database *odb, + const struct object_id *oid, + size_t *sizep) +{ + enum object_type type; + struct object_info oi = OBJECT_INFO_INIT; + + oi.typep = &type; + oi.sizep = sizep; + if (odb_read_object_info_extended(odb, oid, &oi, + OBJECT_INFO_LOOKUP_REPLACE | + OBJECT_INFO_SKIP_FETCH_OBJECT | + OBJECT_INFO_QUICK) < 0) + return OBJ_BAD; + return type; +} + /* * Validates and send requested info back to the client. Any errors detected * are returned as they are detected. @@ -62,15 +88,22 @@ static void send_info(struct repository *r, struct packet_writer *writer, strbuf_addstr(&send_buffer, oid_str); - if (info->size) { - if (odb_read_object_info(r->objects, &oid, &object_size) < 0) { - strbuf_addstr(&send_buffer, " "); - } else { - strbuf_addf(&send_buffer, " %"PRIuMAX, - (uintmax_t)object_size); - } + /* + * Check the existence of the object first. + * If an object is not recognized by the server append SP to + * the response. + */ + if (get_object_info(r->objects, &oid, &object_size) <= OBJ_NONE) { + strbuf_addstr(&send_buffer, " "); + goto write; } + if (info->size) { + strbuf_addf(&send_buffer, " %"PRIuMAX, + (uintmax_t)object_size); + } + +write: packet_writer_write(writer, "%s", send_buffer.buf); strbuf_reset(&send_buffer); } diff --git a/serve.c b/serve.c index 49a6e39b1d..2b07d922b3 100644 --- a/serve.c +++ b/serve.c @@ -89,7 +89,7 @@ static void session_id_receive(struct repository *r UNUSED, trace2_data_string("transfer", NULL, "client-sid", client_sid); } -static int object_info_advertise(struct repository *r, struct strbuf *value UNUSED) +static int object_info_advertise(struct repository *r, struct strbuf *value) { if (advertise_object_info == -1 && repo_config_get_bool(r, "transfer.advertiseobjectinfo", @@ -97,6 +97,9 @@ static int object_info_advertise(struct repository *r, struct strbuf *value UNUS /* disabled by default */ advertise_object_info = 0; } + /* Currently only size is supported */ + if (value && advertise_object_info) + strbuf_addstr(value, "size"); return advertise_object_info; } diff --git a/t/lib-cat-file.sh b/t/lib-cat-file.sh new file mode 100644 index 0000000000..7c2e877016 --- /dev/null +++ b/t/lib-cat-file.sh @@ -0,0 +1,16 @@ +# Library of git-cat-file related test functions. + +# Print a string without a trailing newline. +echo_without_newline () { + printf '%s' "$*" +} + +# Print a string without newlines and replace them with a NUL character (\0). +echo_without_newline_nul () { + echo_without_newline "$@" | tr '\n' '\0' +} + +# Calculate the length of a string. +strlen () { + echo_without_newline "$1" | wc -c | sed -e 's/^ *//' +} diff --git a/t/meson.build b/t/meson.build index 797c5a75d1..78b8ea54ad 100644 --- a/t/meson.build +++ b/t/meson.build @@ -171,6 +171,7 @@ integration_tests = [ 't1014-read-tree-confusing.sh', 't1015-read-index-unmerged.sh', 't1016-compatObjectFormat.sh', + 't1017-cat-file-remote-object-info.sh', 't1020-subdirectory.sh', 't1022-read-tree-partial-clone.sh', 't1050-large.sh', diff --git a/t/t1006-cat-file.sh b/t/t1006-cat-file.sh index 762c77c351..0c1dc0fb40 100755 --- a/t/t1006-cat-file.sh +++ b/t/t1006-cat-file.sh @@ -3,7 +3,8 @@ test_description='git cat-file' . ./test-lib.sh -. "$TEST_DIRECTORY/lib-loose.sh" +. "$TEST_DIRECTORY"/lib-loose.sh +. "$TEST_DIRECTORY"/lib-cat-file.sh test_cmdmode_usage () { test_expect_code 129 "$@" 2>err && @@ -99,18 +100,6 @@ do ' done -echo_without_newline () { - printf '%s' "$*" -} - -echo_without_newline_nul () { - echo_without_newline "$@" | tr '\n' '\0' -} - -strlen () { - echo_without_newline "$1" | wc -c | sed -e 's/^ *//' -} - run_tests () { type=$1 object_name="$2" diff --git a/t/t1017-cat-file-remote-object-info.sh b/t/t1017-cat-file-remote-object-info.sh new file mode 100755 index 0000000000..116862f9d0 --- /dev/null +++ b/t/t1017-cat-file-remote-object-info.sh @@ -0,0 +1,747 @@ +#!/bin/sh + +test_description='git cat-file --batch-command with remote-object-info command' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-cat-file.sh + +hello_content="Hello World" +hello_size=$(strlen "$hello_content") +hello_oid=$(echo_without_newline "$hello_content" | git hash-object --stdin) +hello_short_oid=$(git rev-parse --short "$hello_oid") + +unstored_content="Hello Git" +unstored_oid=$(echo_without_newline "$unstored_content" | git hash-object --stdin) + +# This is how we get 13: +# 13 = + + + , where +# file mode is 100644, which is 6 characters; +# file name is hello, which is 5 characters +# a space is 1 character and a null is 1 character +tree_size=$(($(test_oid rawsz) + 13)) + +commit_message="Initial commit" + +# This is how we get 137: +# 137 = + + + +# + + +# + + +# + +# +# An easier way to calculate is: 1. use `git cat-file commit | wc -c`, +# to get 177, 2. then deduct 40 hex characters to get 137 +commit_size=$(($(test_oid hexsz) + 137)) + +tag_header_without_oid="type blob +tag hellotag +tagger $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL>" +tag_header_without_timestamp="object $hello_oid +$tag_header_without_oid" +tag_description="This is a tag" +tag_content="$tag_header_without_timestamp 0 +0000 + +$tag_description" + +tag_oid=$(echo_without_newline "$tag_content" | git hash-object -t tag --stdin -w) +tag_size=$(strlen "$tag_content") + +set_transport_variables () { + hello_oid=$(echo_without_newline "$hello_content" | git hash-object --stdin) + tree_oid=$(git -C "$1" write-tree) + commit_oid=$(echo_without_newline "$commit_message" | git -C "$1" commit-tree $tree_oid) + tag_oid=$(echo_without_newline "$tag_content" | git -C "$1" hash-object -t tag --stdin -w) + tag_size=$(strlen "$tag_content") +} + +# This section tests --batch-command with remote-object-info command +# Since "%(objecttype)" is currently not supported by the command remote-object-info , +# the filters are set to "%(objectname) %(objectsize)" in some test cases. + +# Test --batch-command remote-object-info with 'git://' transport with +# transfer.advertiseobjectinfo set to true, i.e. server has object-info capability +. "$TEST_DIRECTORY"/lib-git-daemon.sh +start_git_daemon --export-all +daemon_parent=$GIT_DAEMON_DOCUMENT_ROOT_PATH/parent + +test_expect_success 'create repo to be served by git-daemon' ' + git init "$daemon_parent" && + echo_without_newline "$hello_content" > $daemon_parent/hello && + git -C "$daemon_parent" update-index --add hello && + git -C "$daemon_parent" config transfer.advertiseobjectinfo true && + git clone "$GIT_DAEMON_URL/parent" -n "$daemon_parent/daemon_client_empty" +' + +test_expect_success 'batch-command remote-object-info git://' ' + ( + set_transport_variables "$daemon_parent" && + cd "$daemon_parent/daemon_client_empty" && + + # These results prove remote-object-info can get object info from the remote + echo "$hello_oid $hello_size" >expect && + echo "$tree_oid $tree_size" >>expect && + echo "$commit_oid $commit_size" >>expect && + echo "$tag_oid $tag_size" >>expect && + + # These results prove remote-object-info did not download objects from the remote + echo "$hello_oid missing" >>expect && + echo "$tree_oid missing" >>expect && + echo "$commit_oid missing" >>expect && + echo "$tag_oid missing" >>expect && + + git cat-file --batch-command="%(objectname) %(objectsize)" >actual <<-EOF && + remote-object-info "$GIT_DAEMON_URL/parent" $hello_oid + remote-object-info "$GIT_DAEMON_URL/parent" $tree_oid + remote-object-info "$GIT_DAEMON_URL/parent" $commit_oid + remote-object-info "$GIT_DAEMON_URL/parent" $tag_oid + info $hello_oid + info $tree_oid + info $commit_oid + info $tag_oid + EOF + test_cmp expect actual + ) +' + +test_expect_success 'batch-command remote-object-info git:// multiple sha1 per line' ' + ( + set_transport_variables "$daemon_parent" && + cd "$daemon_parent/daemon_client_empty" && + + # These results prove remote-object-info can get object info from the remote + echo "$hello_oid $hello_size" >expect && + echo "$tree_oid $tree_size" >>expect && + echo "$commit_oid $commit_size" >>expect && + echo "$tag_oid $tag_size" >>expect && + + # These results prove remote-object-info did not download objects from the remote + echo "$hello_oid missing" >>expect && + echo "$tree_oid missing" >>expect && + echo "$commit_oid missing" >>expect && + echo "$tag_oid missing" >>expect && + + git cat-file --batch-command="%(objectname) %(objectsize)" >actual <<-EOF && + remote-object-info "$GIT_DAEMON_URL/parent" $hello_oid $tree_oid $commit_oid $tag_oid + info $hello_oid + info $tree_oid + info $commit_oid + info $tag_oid + EOF + test_cmp expect actual + ) +' + +test_expect_success 'batch-command remote-object-info git:// default filter' ' + ( + set_transport_variables "$daemon_parent" && + cd "$daemon_parent/daemon_client_empty" && + + echo "$hello_oid $hello_size" >expect && + echo "$tree_oid $tree_size" >>expect && + echo "$commit_oid $commit_size" >>expect && + echo "$tag_oid $tag_size" >>expect && + + git cat-file --batch-command >actual <<-EOF && + remote-object-info "$GIT_DAEMON_URL/parent" $hello_oid $tree_oid + remote-object-info "$GIT_DAEMON_URL/parent" $commit_oid $tag_oid + EOF + test_cmp expect actual + ) +' + +test_expect_success 'remote-object-info does not change the default format of info' ' + ( + set_transport_variables "$daemon_parent" && + cd "$daemon_parent/daemon_client_empty" && + + local_content="local object" && + local_oid=$(echo_without_newline "$local_content" | git hash-object -w --stdin) && + local_size=$(strlen "$local_content") && + + echo "$local_oid blob $local_size" >expect && + echo "$hello_oid $hello_size" >>expect && + echo "$local_oid blob $local_size" >>expect && + + git cat-file --batch-command >actual <<-EOF && + info $local_oid + remote-object-info "$GIT_DAEMON_URL/parent" $hello_oid + info $local_oid + EOF + test_cmp expect actual + ) +' + +test_expect_success 'batch-command --buffer remote-object-info git://' ' + ( + set_transport_variables "$daemon_parent" && + cd "$daemon_parent/daemon_client_empty" && + + # These results prove remote-object-info can get object info from the remote + echo "$hello_oid $hello_size" >expect && + echo "$tree_oid $tree_size" >>expect && + echo "$commit_oid $commit_size" >>expect && + echo "$tag_oid $tag_size" >>expect && + + # These results prove remote-object-info did not download objects from the remote + echo "$hello_oid missing" >>expect && + echo "$tree_oid missing" >>expect && + echo "$commit_oid missing" >>expect && + echo "$tag_oid missing" >>expect && + + git cat-file --batch-command="%(objectname) %(objectsize)" --buffer >actual <<-EOF && + remote-object-info "$GIT_DAEMON_URL/parent" $hello_oid $tree_oid + remote-object-info "$GIT_DAEMON_URL/parent" $commit_oid $tag_oid + info $hello_oid + info $tree_oid + info $commit_oid + info $tag_oid + flush + EOF + test_cmp expect actual + ) +' + +test_expect_success 'batch-command -Z remote-object-info git:// default filter' ' + ( + set_transport_variables "$daemon_parent" && + cd "$daemon_parent/daemon_client_empty" && + + printf "%s\0" "$hello_oid $hello_size" >expect && + printf "%s\0" "$tree_oid $tree_size" >>expect && + printf "%s\0" "$commit_oid $commit_size" >>expect && + printf "%s\0" "$tag_oid $tag_size" >>expect && + + printf "%s\0" "$hello_oid missing" >>expect && + printf "%s\0" "$tree_oid missing" >>expect && + printf "%s\0" "$commit_oid missing" >>expect && + printf "%s\0" "$tag_oid missing" >>expect && + + batch_input="remote-object-info $GIT_DAEMON_URL/parent $hello_oid $tree_oid +remote-object-info $GIT_DAEMON_URL/parent $commit_oid $tag_oid +info $hello_oid +info $tree_oid +info $commit_oid +info $tag_oid +" && + echo_without_newline_nul "$batch_input" >commands_null_delimited && + + git cat-file --batch-command -Z < commands_null_delimited >actual && + test_cmp expect actual + ) +' + +test_expect_success 'remote-object-info does not support short oids' ' + ( + set_transport_variables "$daemon_parent" && + cd "$daemon_parent/daemon_client_empty" && + + test_must_fail git cat-file --batch-command 2>err <<-EOF && + remote-object-info $GIT_DAEMON_URL/parent $hello_short_oid + EOF + test_grep "does not support short oids" err + ) +' + +test_expect_success 'remote-object-info does not die on missing oid like info' ' + ( + set_transport_variables "$daemon_parent" && + cd "$daemon_parent/daemon_client_empty" && + + git cat-file --batch-command >local <<-EOF && + info $unstored_oid + EOF + git cat-file --batch-command >remote <<-EOF && + remote-object-info $GIT_DAEMON_URL/parent $unstored_oid + EOF + test_cmp local remote + ) +' + +# This tests depends on %(objecttype) not being supported yet, once supported +# it needs to be updated. +test_expect_success 'unsupported placeholder on remote returns empty string' ' + ( + set_transport_variables "$daemon_parent" && + cd "$daemon_parent/daemon_client_empty" && + + echo "" >expect && + git cat-file --batch-command="%(objecttype)" >actual <<-EOF && + remote-object-info "$GIT_DAEMON_URL/parent" $hello_oid + EOF + test_cmp expect actual + ) +' + +test_expect_success 'requesting only objectname echoes back' ' + ( + set_transport_variables "$daemon_parent" && + cd "$daemon_parent/daemon_client_empty" && + + echo $hello_oid >expect && + git cat-file --batch-command="%(objectname)" >actual <<-EOF && + remote-object-info "$GIT_DAEMON_URL/parent" $hello_oid + EOF + test_cmp expect actual + ) +' + +test_expect_success 'objectname goes through existence check' ' + ( + set_transport_variables "$daemon_parent" && + cd "$daemon_parent/daemon_client_empty" && + + echo "$unstored_oid missing" >expect && + + git cat-file --batch-command="%(objectname)" >actual <<-EOF && + remote-object-info "$GIT_DAEMON_URL/parent" $unstored_oid + EOF + + test_cmp expect actual + ) +' + +# Test --batch-command remote-object-info with 'git://' and +# transfer.advertiseobjectinfo set to false, i.e. server does not have object-info capability +test_expect_success 'batch-command remote-object-info git:// fails when transfer.advertiseobjectinfo=false' ' + ( + git -C "$daemon_parent" config transfer.advertiseobjectinfo false && + set_transport_variables "$daemon_parent" && + + test_must_fail git cat-file --batch-command="%(objectname) %(objectsize)" 2>err <<-EOF && + remote-object-info $GIT_DAEMON_URL/parent $hello_oid $tree_oid $commit_oid $tag_oid + EOF + test_grep "object-info capability is not enabled on the server" err && + + # revert server state back + git -C "$daemon_parent" config transfer.advertiseobjectinfo true + + ) +' + +stop_git_daemon + +# Test --batch-command remote-object-info with 'file://' transport with +# transfer.advertiseobjectinfo set to true, i.e. server has object-info capability +# shellcheck disable=SC2016 +test_expect_success 'create repo to be served by file:// transport' ' + git init server && + git -C server config protocol.version 2 && + git -C server config transfer.advertiseobjectinfo true && + echo_without_newline "$hello_content" > server/hello && + git -C server update-index --add hello && + git clone -n "file://$(pwd)/server" file_client_empty +' + +test_expect_success 'batch-command remote-object-info file://' ' + ( + set_transport_variables "server" && + server_path="$(pwd)/server" && + cd file_client_empty && + + # These results prove remote-object-info can get object info from the remote + echo "$hello_oid $hello_size" >expect && + echo "$tree_oid $tree_size" >>expect && + echo "$commit_oid $commit_size" >>expect && + echo "$tag_oid $tag_size" >>expect && + + # These results prove remote-object-info did not download objects from the remote + echo "$hello_oid missing" >>expect && + echo "$tree_oid missing" >>expect && + echo "$commit_oid missing" >>expect && + echo "$tag_oid missing" >>expect && + + git cat-file --batch-command="%(objectname) %(objectsize)" >actual <<-EOF && + remote-object-info "file://${server_path}" $hello_oid + remote-object-info "file://${server_path}" $tree_oid + remote-object-info "file://${server_path}" $commit_oid + remote-object-info "file://${server_path}" $tag_oid + info $hello_oid + info $tree_oid + info $commit_oid + info $tag_oid + EOF + test_cmp expect actual + ) +' + +test_expect_success 'batch-command remote-object-info file:// multiple sha1 per line' ' + ( + set_transport_variables "server" && + server_path="$(pwd)/server" && + cd file_client_empty && + + # These results prove remote-object-info can get object info from the remote + echo "$hello_oid $hello_size" >expect && + echo "$tree_oid $tree_size" >>expect && + echo "$commit_oid $commit_size" >>expect && + echo "$tag_oid $tag_size" >>expect && + + # These results prove remote-object-info did not download objects from the remote + echo "$hello_oid missing" >>expect && + echo "$tree_oid missing" >>expect && + echo "$commit_oid missing" >>expect && + echo "$tag_oid missing" >>expect && + + + git cat-file --batch-command="%(objectname) %(objectsize)" >actual <<-EOF && + remote-object-info "file://${server_path}" $hello_oid $tree_oid $commit_oid $tag_oid + info $hello_oid + info $tree_oid + info $commit_oid + info $tag_oid + EOF + test_cmp expect actual + ) +' + +test_expect_success 'batch-command --buffer remote-object-info file://' ' + ( + set_transport_variables "server" && + server_path="$(pwd)/server" && + cd file_client_empty && + + # These results prove remote-object-info can get object info from the remote + echo "$hello_oid $hello_size" >expect && + echo "$tree_oid $tree_size" >>expect && + echo "$commit_oid $commit_size" >>expect && + echo "$tag_oid $tag_size" >>expect && + + # These results prove remote-object-info did not download objects from the remote + echo "$hello_oid missing" >>expect && + echo "$tree_oid missing" >>expect && + echo "$commit_oid missing" >>expect && + echo "$tag_oid missing" >>expect && + + git cat-file --batch-command="%(objectname) %(objectsize)" --buffer >actual <<-EOF && + remote-object-info "file://${server_path}" $hello_oid $tree_oid + remote-object-info "file://${server_path}" $commit_oid $tag_oid + info $hello_oid + info $tree_oid + info $commit_oid + info $tag_oid + flush + EOF + test_cmp expect actual + ) +' + +test_expect_success 'batch-command remote-object-info file:// default filter' ' + ( + set_transport_variables "server" && + server_path="$(pwd)/server" && + cd file_client_empty && + + echo "$hello_oid $hello_size" >expect && + echo "$tree_oid $tree_size" >>expect && + echo "$commit_oid $commit_size" >>expect && + echo "$tag_oid $tag_size" >>expect && + + git cat-file --batch-command >actual <<-EOF && + remote-object-info "file://${server_path}" $hello_oid $tree_oid + remote-object-info "file://${server_path}" $commit_oid $tag_oid + EOF + test_cmp expect actual + ) +' + +test_expect_success 'batch-command -Z remote-object-info file:// default filter' ' + ( + set_transport_variables "server" && + server_path="$(pwd)/server" && + cd file_client_empty && + + printf "%s\0" "$hello_oid $hello_size" >expect && + printf "%s\0" "$tree_oid $tree_size" >>expect && + printf "%s\0" "$commit_oid $commit_size" >>expect && + printf "%s\0" "$tag_oid $tag_size" >>expect && + + printf "%s\0" "$hello_oid missing" >>expect && + printf "%s\0" "$tree_oid missing" >>expect && + printf "%s\0" "$commit_oid missing" >>expect && + printf "%s\0" "$tag_oid missing" >>expect && + + batch_input="remote-object-info \"file://${server_path}\" $hello_oid $tree_oid +remote-object-info \"file://${server_path}\" $commit_oid $tag_oid +info $hello_oid +info $tree_oid +info $commit_oid +info $tag_oid +" && + echo_without_newline_nul "$batch_input" >commands_null_delimited && + + git cat-file --batch-command -Z < commands_null_delimited >actual && + test_cmp expect actual + ) +' + +# Test --batch-command remote-object-info with 'file://' and +# transfer.advertiseobjectinfo set to false, i.e. server does not have object-info capability +test_expect_success 'batch-command remote-object-info file:// fails when transfer.advertiseobjectinfo=false' ' + ( + set_transport_variables "server" && + server_path="$(pwd)/server" && + git -C "${server_path}" config transfer.advertiseobjectinfo false && + + test_must_fail git cat-file --batch-command="%(objectname) %(objectsize)" 2>err <<-EOF && + remote-object-info "file://${server_path}" $hello_oid $tree_oid $commit_oid $tag_oid + EOF + test_grep "object-info capability is not enabled on the server" err && + + # revert server state back + git -C "${server_path}" config transfer.advertiseobjectinfo true + ) +' + +# Test --batch-command remote-object-info with 'http://' transport with +# transfer.advertiseobjectinfo set to true, i.e. server has object-info capability + +. "$TEST_DIRECTORY"/lib-httpd.sh +start_httpd + +test_expect_success 'create repo to be served by http:// transport' ' + git init "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" config http.receivepack true && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" config transfer.advertiseobjectinfo true && + echo_without_newline "$hello_content" > $HTTPD_DOCUMENT_ROOT_PATH/http_parent/hello && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" update-index --add hello && + git clone "$HTTPD_URL/smart/http_parent" -n "$HTTPD_DOCUMENT_ROOT_PATH/http_client_empty" +' + +test_expect_success 'batch-command remote-object-info http://' ' + ( + set_transport_variables "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + cd "$HTTPD_DOCUMENT_ROOT_PATH/http_client_empty" && + + # These results prove remote-object-info can get object info from the remote + echo "$hello_oid $hello_size" >expect && + echo "$tree_oid $tree_size" >>expect && + echo "$commit_oid $commit_size" >>expect && + echo "$tag_oid $tag_size" >>expect && + + # These results prove remote-object-info did not download objects from the remote + echo "$hello_oid missing" >>expect && + echo "$tree_oid missing" >>expect && + echo "$commit_oid missing" >>expect && + echo "$tag_oid missing" >>expect && + + git cat-file --batch-command="%(objectname) %(objectsize)" >actual <<-EOF && + remote-object-info "$HTTPD_URL/smart/http_parent" $hello_oid + remote-object-info "$HTTPD_URL/smart/http_parent" $tree_oid + remote-object-info "$HTTPD_URL/smart/http_parent" $commit_oid + remote-object-info "$HTTPD_URL/smart/http_parent" $tag_oid + info $hello_oid + info $tree_oid + info $commit_oid + info $tag_oid + EOF + test_cmp expect actual + ) +' + +test_expect_success 'batch-command remote-object-info http:// one line' ' + ( + set_transport_variables "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + cd "$HTTPD_DOCUMENT_ROOT_PATH/http_client_empty" && + + # These results prove remote-object-info can get object info from the remote + echo "$hello_oid $hello_size" >expect && + echo "$tree_oid $tree_size" >>expect && + echo "$commit_oid $commit_size" >>expect && + echo "$tag_oid $tag_size" >>expect && + + # These results prove remote-object-info did not download objects from the remote + echo "$hello_oid missing" >>expect && + echo "$tree_oid missing" >>expect && + echo "$commit_oid missing" >>expect && + echo "$tag_oid missing" >>expect && + + git cat-file --batch-command="%(objectname) %(objectsize)" >actual <<-EOF && + remote-object-info "$HTTPD_URL/smart/http_parent" $hello_oid $tree_oid $commit_oid $tag_oid + info $hello_oid + info $tree_oid + info $commit_oid + info $tag_oid + EOF + test_cmp expect actual + ) +' + +test_expect_success 'batch-command --buffer remote-object-info http://' ' + ( + set_transport_variables "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + cd "$HTTPD_DOCUMENT_ROOT_PATH/http_client_empty" && + + # These results prove remote-object-info can get object info from the remote + echo "$hello_oid $hello_size" >expect && + echo "$tree_oid $tree_size" >>expect && + echo "$commit_oid $commit_size" >>expect && + echo "$tag_oid $tag_size" >>expect && + + # These results prove remote-object-info did not download objects from the remote + echo "$hello_oid missing" >>expect && + echo "$tree_oid missing" >>expect && + echo "$commit_oid missing" >>expect && + echo "$tag_oid missing" >>expect && + + git cat-file --batch-command="%(objectname) %(objectsize)" --buffer >actual <<-EOF && + remote-object-info "$HTTPD_URL/smart/http_parent" $hello_oid $tree_oid + remote-object-info "$HTTPD_URL/smart/http_parent" $commit_oid $tag_oid + info $hello_oid + info $tree_oid + info $commit_oid + info $tag_oid + flush + EOF + test_cmp expect actual + ) +' + +test_expect_success 'batch-command remote-object-info http:// default filter' ' + ( + set_transport_variables "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + cd "$HTTPD_DOCUMENT_ROOT_PATH/http_client_empty" && + + echo "$hello_oid $hello_size" >expect && + echo "$tree_oid $tree_size" >>expect && + echo "$commit_oid $commit_size" >>expect && + echo "$tag_oid $tag_size" >>expect && + + git cat-file --batch-command >actual <<-EOF && + remote-object-info "$HTTPD_URL/smart/http_parent" $hello_oid $tree_oid + remote-object-info "$HTTPD_URL/smart/http_parent" $commit_oid $tag_oid + EOF + test_cmp expect actual + ) +' + +test_expect_success 'batch-command -Z remote-object-info http:// default filter' ' + ( + set_transport_variables "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + cd "$HTTPD_DOCUMENT_ROOT_PATH/http_client_empty" && + + printf "%s\0" "$hello_oid $hello_size" >expect && + printf "%s\0" "$tree_oid $tree_size" >>expect && + printf "%s\0" "$commit_oid $commit_size" >>expect && + printf "%s\0" "$tag_oid $tag_size" >>expect && + + batch_input="remote-object-info $HTTPD_URL/smart/http_parent $hello_oid $tree_oid +remote-object-info $HTTPD_URL/smart/http_parent $commit_oid $tag_oid +" && + echo_without_newline_nul "$batch_input" >commands_null_delimited && + + git cat-file --batch-command -Z < commands_null_delimited >actual && + test_cmp expect actual + ) +' + +test_expect_success 'remote-object-info fails on unsupported filter option (objectsize:disk)' ' + ( + set_transport_variables "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + cd "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + + echo "$hello_oid " >expect && + + git cat-file --batch-command="%(objectname) %(objectsize:disk)" >actual <<-EOF && + remote-object-info "$HTTPD_URL/smart/http_parent" $hello_oid + EOF + test_cmp expect actual + ) +' + +test_expect_success 'remote-object-info fails on unsupported filter option (deltabase)' ' + ( + set_transport_variables "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + cd "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + + echo "" >expect && + + git cat-file --batch-command="%(deltabase)" >actual <<-EOF && + remote-object-info "$HTTPD_URL/smart/http_parent" $hello_oid + EOF + test_cmp expect actual + ) +' + +test_expect_success 'remote-object-info fails on server with legacy protocol' ' + ( + set_transport_variables "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + cd "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + + test_must_fail git -c protocol.version=0 cat-file --batch-command="%(objectname) %(objectsize)" 2>err <<-EOF && + remote-object-info "$HTTPD_URL/smart/http_parent" $hello_oid + EOF + test_grep "object-info requires protocol v2" err + ) +' + +test_expect_success 'remote-object-info fails on server with legacy protocol with default filter' ' + ( + set_transport_variables "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + cd "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + + test_must_fail git -c protocol.version=0 cat-file --batch-command 2>err <<-EOF && + remote-object-info "$HTTPD_URL/smart/http_parent" $hello_oid + EOF + test_grep "object-info requires protocol v2" err + ) +' + +test_expect_success 'remote-object-info fails on malformed OID' ' + ( + set_transport_variables "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + cd "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + malformed_object_id="this_id_is_not_valid" && + + test_must_fail git cat-file --batch-command="%(objectname) %(objectsize)" 2>err <<-EOF && + remote-object-info "$HTTPD_URL/smart/http_parent" $malformed_object_id + EOF + test_grep "not a valid object name '$malformed_object_id'" err + ) +' + +test_expect_success 'remote-object-info fails on malformed OID with default filter' ' + ( + set_transport_variables "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + cd "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + malformed_object_id="this_id_is_not_valid" && + + test_must_fail git cat-file --batch-command 2>err <<-EOF && + remote-object-info "$HTTPD_URL/smart/http_parent" $malformed_object_id + EOF + test_grep "not a valid object name '$malformed_object_id'" err + ) +' + +test_expect_success 'remote-object-info fails on not providing OID' ' + ( + set_transport_variables "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + cd "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + + test_must_fail git cat-file --batch-command="%(objectname) %(objectsize)" 2>err <<-EOF && + remote-object-info "$HTTPD_URL/smart/http_parent" + EOF + test_grep "remote-object-info requires objects" err + ) +' + + +# Test --batch-command remote-object-info with 'http://' transport and +# transfer.advertiseobjectinfo set to false, i.e. server does not have object-info capability +test_expect_success 'batch-command remote-object-info http:// fails when transfer.advertiseobjectinfo=false ' ' + ( + set_transport_variables "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" config transfer.advertiseobjectinfo false && + + test_must_fail git cat-file --batch-command="%(objectname) %(objectsize)" 2>err <<-EOF && + remote-object-info "$HTTPD_URL/smart/http_parent" $hello_oid $tree_oid $commit_oid $tag_oid + EOF + test_grep "object-info capability is not enabled on the server" err && + + # revert server state back + git -C "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" config transfer.advertiseobjectinfo true + ) +' + +# DO NOT add non-httpd-specific tests here, because the last part of this +# test script is only executed when httpd is available and enabled. + +test_done diff --git a/t/t5701-git-serve.sh b/t/t5701-git-serve.sh index 108eb30945..9a575aa098 100755 --- a/t/t5701-git-serve.sh +++ b/t/t5701-git-serve.sh @@ -7,6 +7,8 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME . ./test-lib.sh +unknown_oid=$(printf "test" | git hash-object --stdin) + test_expect_success 'setup to generate files with expected content' ' printf "agent=git/%s" "$(git version | cut -d" " -f3)" >agent_capability && @@ -364,6 +366,67 @@ test_expect_success 'basics of object-info' ' test_cmp expect actual ' +test_expect_success 'bare OID request' ' + test_config transfer.advertiseObjectInfo true && + + test-tool pkt-line pack >in <<-EOF && + command=object-info + object-format=$(test_oid algo) + 0001 + oid $(git rev-parse two:two.t) + 0000 + EOF + + cat >expect <<-EOF && + $(git rev-parse two:two.t) + 0000 + EOF + + test-tool serve-v2 --stateless-rpc out && + test-tool pkt-line unpack actual && + test_cmp expect actual +' + +test_expect_success 'object-info with bare unrecognized OID' ' + test_config transfer.advertiseObjectInfo true && + + test-tool pkt-line pack >in <<-EOF && + command=object-info + object-format=$(test_oid algo) + 0001 + oid $unknown_oid + 0000 + EOF + + printf "%s \n" "$unknown_oid" >expect && + printf "0000\n" >>expect && + + test-tool serve-v2 --stateless-rpc out && + test-tool pkt-line unpack actual && + test_cmp expect actual +' + +test_expect_success 'object-info with size for unrecognized OID' ' + test_config transfer.advertiseObjectInfo true && + + test-tool pkt-line pack >in <<-EOF && + command=object-info + object-format=$(test_oid algo) + 0001 + size + oid $unknown_oid + 0000 + EOF + + printf "size\n" >expect && + printf "%s \n" "$unknown_oid" >>expect && + printf "0000\n" >>expect && + + test-tool serve-v2 --stateless-rpc out && + test-tool pkt-line unpack actual && + test_cmp expect actual +' + test_expect_success 'test capability advertisement with uploadpack.advertiseBundleURIs' ' test_config uploadpack.advertiseBundleURIs true && diff --git a/transport-helper.c b/transport-helper.c index 8a25707b03..b109fbd8c3 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -266,9 +266,9 @@ static int disconnect_helper(struct transport *transport) close(data->helper->out); fclose(data->out); res = finish_command(data->helper); - FREE_AND_NULL(data->name); FREE_AND_NULL(data->helper); } + FREE_AND_NULL(data->name); return res; } @@ -784,6 +784,15 @@ static int fetch_refs(struct transport *transport, return -1; } +static int fetch_object_info_helper(struct transport *transport) +{ + get_helper(transport); + if (process_connect(transport, 0)) + return transport->vtable->fetch_object_info(transport); + + die(_("object-info requires protocol v2")); +} + struct push_update_ref_state { struct ref *hint; struct ref_push_report *report; @@ -1330,6 +1339,7 @@ static struct transport_vtable vtable = { .get_refs_list = get_refs_list, .get_bundle_uri = get_bundle_uri, .fetch_refs = fetch_refs, + .fetch_object_info = fetch_object_info_helper, .push_refs = push_refs, .connect = connect_helper, .disconnect = release_helper diff --git a/transport-internal.h b/transport-internal.h index 051f3ab0dc..60db0bedcd 100644 --- a/transport-internal.h +++ b/transport-internal.h @@ -45,6 +45,14 @@ struct transport_vtable { **/ int (*fetch_refs)(struct transport *transport, int refs_nr, struct ref **refs); + /* + * Fetch object info (only size currently) from remote without + * downloading the objects. + * + * Uses object-info capability of v2 protocol. + */ + int (*fetch_object_info)(struct transport *transport); + /** * Push the objects and refs. Send the necessary objects, and * then, for any refs where peer_ref is set and diff --git a/transport.c b/transport.c index fc144f0aed..f0a6a45547 100644 --- a/transport.c +++ b/transport.c @@ -9,6 +9,7 @@ #include "hook.h" #include "pkt-line.h" #include "fetch-pack.h" +#include "fetch-object-info.h" #include "remote.h" #include "connect.h" #include "send-pack.h" @@ -432,6 +433,47 @@ static int get_bundle_uri(struct transport *transport) transport->bundles, stateless_rpc); } +static int fetch_object_info_via_pack(struct transport *transport) +{ + int ret = 0; + struct git_transport_data *data = transport->data; + struct packet_reader reader; + struct object_info_args args = { 0 }; + + args.server_options = transport->server_options; + args.oids = transport->smart_options->object_info_oids; + args.object_info_options = transport->smart_options->object_info_options; + + connect_setup(transport, 0); + packet_reader_init(&reader, data->fd[0], NULL, 0, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_GENTLE_ON_EOF | + PACKET_READ_DIE_ON_ERR_PACKET); + + data->version = discover_version(&reader); + transport->hash_algo = reader.hash_algo; + + ret = fetch_object_info(data->version, &args, &reader, + data->options.object_info_data, + transport->stateless_rpc, data->fd[1]); + + close(data->fd[0]); + if (data->fd[1] >= 0) + close(data->fd[1]); + if (finish_connect(data->conn)) + ret = -1; + data->conn = NULL; + + return ret; +} + +int transport_fetch_object_info(struct transport *transport) +{ + if (!transport->vtable->fetch_object_info) + die(_("remote does not support object-info")); + return transport->vtable->fetch_object_info(transport); +} + static int fetch_refs_via_pack(struct transport *transport, int nr_heads, struct ref **to_fetch) { @@ -1004,6 +1046,7 @@ static struct transport_vtable taken_over_vtable = { .get_refs_list = get_refs_via_connect, .get_bundle_uri = get_bundle_uri, .fetch_refs = fetch_refs_via_pack, + .fetch_object_info = fetch_object_info_via_pack, .push_refs = git_transport_push, .disconnect = disconnect_git }; @@ -1169,6 +1212,7 @@ static struct transport_vtable builtin_smart_vtable = { .get_refs_list = get_refs_via_connect, .get_bundle_uri = get_bundle_uri, .fetch_refs = fetch_refs_via_pack, + .fetch_object_info = fetch_object_info_via_pack, .push_refs = git_transport_push, .connect = connect_git, .disconnect = disconnect_git diff --git a/transport.h b/transport.h index 7e5867cffa..a7869d18e0 100644 --- a/transport.h +++ b/transport.h @@ -55,6 +55,10 @@ struct git_transport_options { * common commits to this oidset instead of fetching any packfiles. */ struct oidset *acked_commits; + + struct oid_array *object_info_oids; + struct object_info *object_info_data; + struct string_list *object_info_options; }; enum transport_family { @@ -309,6 +313,11 @@ int transport_get_remote_bundle_uri(struct transport *transport); const struct git_hash_algo *transport_get_hash_algo(struct transport *transport); int transport_fetch_refs(struct transport *transport, struct ref *refs); +/* + * Fetch the object info from remote + */ +int transport_fetch_object_info(struct transport *transport); + /* * If this flag is set, unlocking will avoid to call non-async-signal-safe * functions. This will necessarily leave behind some data structures which