From db6bfb9fe8098b8795d8e6bb8c6d85fd39f2cdaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Fri, 2 Jul 2021 11:57:30 +0200 Subject: [PATCH 1/3] bundle cmd: stop leaking memory from parse_options_cmd_bundle() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix a memory leak from the prefix_filename() function introduced with its use in 3b754eedd5 (bundle: use prefix_filename with bundle path, 2017-03-20). As noted in that commit the leak was intentional as a part of being sloppy about freeing resources just before we exit, I'm changing this because I'll be fixing other memory leaks in the bundle API (including the library version) in subsequent commits. It's easier to reason about those fixes if valgrind runs cleanly at the end without any leaks whatsoever. An earlier version of this change[1] went out of its way to not leak memory on the die() codepaths here, but doing so will only avoid reports of potential leaks under heap-only leak trackers such as valgrind, not the SANITIZE=leak mode. Avoiding those leaks as well might be useful to enable us to run cleanly under the likes of valgrind in the future. But for now the relative verbosity of the resulting code, and the fact that we don't have some valgrind or SANITIZE=leak mode as part of our CI (it's only run ad-hoc, see [2]), means we're not worrying about that for now. 1. https://lore.kernel.org/git/87v95vdxrc.fsf@evledraar.gmail.com/ 2. https://lore.kernel.org/git/87czsv2idy.fsf@evledraar.gmail.com/ Signed-off-by: Ævar Arnfjörð Bjarmason Acked-by: Jeff King Signed-off-by: Junio C Hamano --- builtin/bundle.c | 62 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/builtin/bundle.c b/builtin/bundle.c index ea6948110b..15e2bd6196 100644 --- a/builtin/bundle.c +++ b/builtin/bundle.c @@ -46,7 +46,7 @@ static int parse_options_cmd_bundle(int argc, const char* prefix, const char * const usagestr[], const struct option options[], - const char **bundle_file) { + char **bundle_file) { int newargc; newargc = parse_options(argc, argv, NULL, options, usagestr, PARSE_OPT_STOP_AT_NON_OPTION); @@ -61,7 +61,7 @@ static int cmd_bundle_create(int argc, const char **argv, const char *prefix) { int progress = isatty(STDERR_FILENO); struct strvec pack_opts; int version = -1; - + int ret; struct option options[] = { OPT_SET_INT('q', "quiet", &progress, N_("do not show progress meter"), 0), @@ -76,7 +76,7 @@ static int cmd_bundle_create(int argc, const char **argv, const char *prefix) { N_("specify bundle format version")), OPT_END() }; - const char* bundle_file; + char *bundle_file; argc = parse_options_cmd_bundle(argc, argv, prefix, builtin_bundle_create_usage, options, &bundle_file); @@ -94,75 +94,95 @@ static int cmd_bundle_create(int argc, const char **argv, const char *prefix) { if (!startup_info->have_repository) die(_("Need a repository to create a bundle.")); - return !!create_bundle(the_repository, bundle_file, argc, argv, &pack_opts, version); + ret = !!create_bundle(the_repository, bundle_file, argc, argv, &pack_opts, version); + free(bundle_file); + return ret; } static int cmd_bundle_verify(int argc, const char **argv, const char *prefix) { struct bundle_header header; int bundle_fd = -1; int quiet = 0; - + int ret; struct option options[] = { OPT_BOOL('q', "quiet", &quiet, N_("do not show bundle details")), OPT_END() }; - const char* bundle_file; + char *bundle_file; argc = parse_options_cmd_bundle(argc, argv, prefix, builtin_bundle_verify_usage, options, &bundle_file); /* bundle internals use argv[1] as further parameters */ memset(&header, 0, sizeof(header)); - if ((bundle_fd = read_bundle_header(bundle_file, &header)) < 0) - return 1; + if ((bundle_fd = read_bundle_header(bundle_file, &header)) < 0) { + ret = 1; + goto cleanup; + } close(bundle_fd); - if (verify_bundle(the_repository, &header, !quiet)) - return 1; + if (verify_bundle(the_repository, &header, !quiet)) { + ret = 1; + goto cleanup; + } + fprintf(stderr, _("%s is okay\n"), bundle_file); - return 0; + ret = 0; +cleanup: + free(bundle_file); + return ret; } static int cmd_bundle_list_heads(int argc, const char **argv, const char *prefix) { struct bundle_header header; int bundle_fd = -1; - + int ret; struct option options[] = { OPT_END() }; - const char* bundle_file; + char *bundle_file; argc = parse_options_cmd_bundle(argc, argv, prefix, builtin_bundle_list_heads_usage, options, &bundle_file); /* bundle internals use argv[1] as further parameters */ memset(&header, 0, sizeof(header)); - if ((bundle_fd = read_bundle_header(bundle_file, &header)) < 0) - return 1; + if ((bundle_fd = read_bundle_header(bundle_file, &header)) < 0) { + ret = 1; + goto cleanup; + } close(bundle_fd); - return !!list_bundle_refs(&header, argc, argv); + ret = !!list_bundle_refs(&header, argc, argv); +cleanup: + free(bundle_file); + return ret; } static int cmd_bundle_unbundle(int argc, const char **argv, const char *prefix) { struct bundle_header header; int bundle_fd = -1; - + int ret; struct option options[] = { OPT_END() }; - const char* bundle_file; + char *bundle_file; argc = parse_options_cmd_bundle(argc, argv, prefix, builtin_bundle_unbundle_usage, options, &bundle_file); /* bundle internals use argv[1] as further parameters */ memset(&header, 0, sizeof(header)); - if ((bundle_fd = read_bundle_header(bundle_file, &header)) < 0) - return 1; + if ((bundle_fd = read_bundle_header(bundle_file, &header)) < 0) { + ret = 1; + goto cleanup; + } if (!startup_info->have_repository) die(_("Need a repository to unbundle.")); - return !!unbundle(the_repository, &header, bundle_fd, 0) || + ret = !!unbundle(the_repository, &header, bundle_fd, 0) || list_bundle_refs(&header, argc, argv); +cleanup: + free(bundle_file); + return ret; } int cmd_bundle(int argc, const char **argv, const char *prefix) From 15e7c7dca66ab7b020316696f54433f76e5e1084 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Fri, 2 Jul 2021 11:57:31 +0200 Subject: [PATCH 2/3] bundle.c: use a temporary variable for OIDs and names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In preparation for moving away from accessing the OID and name via the "oid" and "name" slots in a subsequent commit, change the code that accesses it to use named variables. This makes the subsequent change smaller. Signed-off-by: Ævar Arnfjörð Bjarmason Acked-by: Jeff King Signed-off-by: Junio C Hamano --- bundle.c | 26 ++++++++++++++++++-------- transport.c | 6 ++++-- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/bundle.c b/bundle.c index 693d619551..7210e5e710 100644 --- a/bundle.c +++ b/bundle.c @@ -156,6 +156,9 @@ static int list_refs(struct ref_list *r, int argc, const char **argv) int i; for (i = 0; i < r->nr; i++) { + struct object_id *oid; + const char *name; + if (argc > 1) { int j; for (j = 1; j < argc; j++) @@ -164,8 +167,10 @@ static int list_refs(struct ref_list *r, int argc, const char **argv) if (j == argc) continue; } - printf("%s %s\n", oid_to_hex(&r->list[i].oid), - r->list[i].name); + + oid = &r->list[i].oid; + name = r->list[i].name; + printf("%s %s\n", oid_to_hex(oid), name); } return 0; } @@ -194,15 +199,17 @@ int verify_bundle(struct repository *r, repo_init_revisions(r, &revs, NULL); for (i = 0; i < p->nr; i++) { struct ref_list_entry *e = p->list + i; - struct object *o = parse_object(r, &e->oid); + const char *name = e->name; + struct object_id *oid = &e->oid; + struct object *o = parse_object(r, oid); if (o) { o->flags |= PREREQ_MARK; - add_pending_object(&revs, o, e->name); + add_pending_object(&revs, o, name); continue; } if (++ret == 1) error("%s", message); - error("%s %s", oid_to_hex(&e->oid), e->name); + error("%s %s", oid_to_hex(oid), name); } if (revs.pending.nr != p->nr) return ret; @@ -219,19 +226,22 @@ int verify_bundle(struct repository *r, for (i = 0; i < p->nr; i++) { struct ref_list_entry *e = p->list + i; - struct object *o = parse_object(r, &e->oid); + const char *name = e->name; + struct object_id *oid = &e->oid; + struct object *o = parse_object(r, oid); assert(o); /* otherwise we'd have returned early */ if (o->flags & SHOWN) continue; if (++ret == 1) error("%s", message); - error("%s %s", oid_to_hex(&e->oid), e->name); + error("%s %s", oid_to_hex(oid), name); } /* Clean up objects used, as they will be reused. */ for (i = 0; i < p->nr; i++) { struct ref_list_entry *e = p->list + i; - commit = lookup_commit_reference_gently(r, &e->oid, 1); + struct object_id *oid = &e->oid; + commit = lookup_commit_reference_gently(r, oid, 1); if (commit) clear_commit_marks(commit, ALL_REV_FLAGS); } diff --git a/transport.c b/transport.c index 50f5830eb6..95c1138e9a 100644 --- a/transport.c +++ b/transport.c @@ -148,8 +148,10 @@ static struct ref *get_refs_from_bundle(struct transport *transport, for (i = 0; i < data->header.references.nr; i++) { struct ref_list_entry *e = data->header.references.list + i; - struct ref *ref = alloc_ref(e->name); - oidcpy(&ref->old_oid, &e->oid); + const char *name = e->name; + struct ref *ref = alloc_ref(name); + struct object_id *oid = &e->oid; + oidcpy(&ref->old_oid, oid); ref->next = result; result = ref; } From 10b635b77311badcbb5045b7421e6826c4536613 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Fri, 2 Jul 2021 11:57:32 +0200 Subject: [PATCH 3/3] bundle: remove "ref_list" in favor of string-list.c API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move away from the "struct ref_list" in bundle.c in favor of the almost identical string-list.c API. That API fits this use-case perfectly, but did not exist in its current form when this code was added in 2e0afafebd (Add git-bundle: move objects and references by archive, 2007-02-22), with hindsight we could have used the path-list API, which later got renamed to string-list. See 8fd2cb4069 (Extract helper bits from c-merge-recursive work, 2006-07-25) We need to change "name" to "string" and "oid" to "util" to make this conversion, but other than that the APIs are pretty much identical for what bundle.c made use of. Let's also replace the memset(..,0,...) pattern with a more idiomatic "INIT" macro, and finally add a *_release() function so to free the allocated memory. Before this the add_to_ref_list() would leak memory, now e.g. "bundle list-heads" reports no memory leaks at all under valgrind. In the bundle_header_init() function we're using a clever trick to memcpy() what we'd get from the corresponding BUNDLE_HEADER_INIT. There is a concurrent series to make use of that pattern more generally, see [1]. 1. https://lore.kernel.org/git/cover-0.5-00000000000-20210701T104855Z-avarab@gmail.com/ Signed-off-by: Ævar Arnfjörð Bjarmason Acked-by: Jeff King Signed-off-by: Junio C Hamano --- builtin/bundle.c | 12 +++++------ bundle.c | 52 ++++++++++++++++++++++++++---------------------- bundle.h | 21 +++++++++---------- transport.c | 8 +++++--- 4 files changed, 50 insertions(+), 43 deletions(-) diff --git a/builtin/bundle.c b/builtin/bundle.c index 15e2bd6196..053a51bea1 100644 --- a/builtin/bundle.c +++ b/builtin/bundle.c @@ -100,7 +100,7 @@ static int cmd_bundle_create(int argc, const char **argv, const char *prefix) { } static int cmd_bundle_verify(int argc, const char **argv, const char *prefix) { - struct bundle_header header; + struct bundle_header header = BUNDLE_HEADER_INIT; int bundle_fd = -1; int quiet = 0; int ret; @@ -115,7 +115,6 @@ static int cmd_bundle_verify(int argc, const char **argv, const char *prefix) { builtin_bundle_verify_usage, options, &bundle_file); /* bundle internals use argv[1] as further parameters */ - memset(&header, 0, sizeof(header)); if ((bundle_fd = read_bundle_header(bundle_file, &header)) < 0) { ret = 1; goto cleanup; @@ -130,11 +129,12 @@ static int cmd_bundle_verify(int argc, const char **argv, const char *prefix) { ret = 0; cleanup: free(bundle_file); + bundle_header_release(&header); return ret; } static int cmd_bundle_list_heads(int argc, const char **argv, const char *prefix) { - struct bundle_header header; + struct bundle_header header = BUNDLE_HEADER_INIT; int bundle_fd = -1; int ret; struct option options[] = { @@ -146,7 +146,6 @@ static int cmd_bundle_list_heads(int argc, const char **argv, const char *prefix builtin_bundle_list_heads_usage, options, &bundle_file); /* bundle internals use argv[1] as further parameters */ - memset(&header, 0, sizeof(header)); if ((bundle_fd = read_bundle_header(bundle_file, &header)) < 0) { ret = 1; goto cleanup; @@ -155,11 +154,12 @@ static int cmd_bundle_list_heads(int argc, const char **argv, const char *prefix ret = !!list_bundle_refs(&header, argc, argv); cleanup: free(bundle_file); + bundle_header_release(&header); return ret; } static int cmd_bundle_unbundle(int argc, const char **argv, const char *prefix) { - struct bundle_header header; + struct bundle_header header = BUNDLE_HEADER_INIT; int bundle_fd = -1; int ret; struct option options[] = { @@ -171,7 +171,6 @@ static int cmd_bundle_unbundle(int argc, const char **argv, const char *prefix) builtin_bundle_unbundle_usage, options, &bundle_file); /* bundle internals use argv[1] as further parameters */ - memset(&header, 0, sizeof(header)); if ((bundle_fd = read_bundle_header(bundle_file, &header)) < 0) { ret = 1; goto cleanup; @@ -180,6 +179,7 @@ static int cmd_bundle_unbundle(int argc, const char **argv, const char *prefix) die(_("Need a repository to unbundle.")); ret = !!unbundle(the_repository, &header, bundle_fd, 0) || list_bundle_refs(&header, argc, argv); + bundle_header_release(&header); cleanup: free(bundle_file); return ret; diff --git a/bundle.c b/bundle.c index 7210e5e710..ab63f40226 100644 --- a/bundle.c +++ b/bundle.c @@ -23,13 +23,16 @@ static struct { { 3, v3_bundle_signature }, }; -static void add_to_ref_list(const struct object_id *oid, const char *name, - struct ref_list *list) +void bundle_header_init(struct bundle_header *header) { - ALLOC_GROW(list->list, list->nr + 1, list->alloc); - oidcpy(&list->list[list->nr].oid, oid); - list->list[list->nr].name = xstrdup(name); - list->nr++; + struct bundle_header blank = BUNDLE_HEADER_INIT; + memcpy(header, &blank, sizeof(*header)); +} + +void bundle_header_release(struct bundle_header *header) +{ + string_list_clear(&header->prerequisites, 1); + string_list_clear(&header->references, 1); } static int parse_capability(struct bundle_header *header, const char *capability) @@ -112,10 +115,11 @@ static int parse_bundle_header(int fd, struct bundle_header *header, status = -1; break; } else { + struct object_id *dup = oiddup(&oid); if (is_prereq) - add_to_ref_list(&oid, "", &header->prerequisites); + string_list_append(&header->prerequisites, "")->util = dup; else - add_to_ref_list(&oid, p + 1, &header->references); + string_list_append(&header->references, p + 1)->util = dup; } } @@ -139,19 +143,19 @@ int read_bundle_header(const char *path, struct bundle_header *header) int is_bundle(const char *path, int quiet) { - struct bundle_header header; + struct bundle_header header = BUNDLE_HEADER_INIT; int fd = open(path, O_RDONLY); if (fd < 0) return 0; - memset(&header, 0, sizeof(header)); fd = parse_bundle_header(fd, &header, quiet ? NULL : path); if (fd >= 0) close(fd); + bundle_header_release(&header); return (fd >= 0); } -static int list_refs(struct ref_list *r, int argc, const char **argv) +static int list_refs(struct string_list *r, int argc, const char **argv) { int i; @@ -162,14 +166,14 @@ static int list_refs(struct ref_list *r, int argc, const char **argv) if (argc > 1) { int j; for (j = 1; j < argc; j++) - if (!strcmp(r->list[i].name, argv[j])) + if (!strcmp(r->items[i].string, argv[j])) break; if (j == argc) continue; } - oid = &r->list[i].oid; - name = r->list[i].name; + oid = r->items[i].util; + name = r->items[i].string; printf("%s %s\n", oid_to_hex(oid), name); } return 0; @@ -186,7 +190,7 @@ int verify_bundle(struct repository *r, * Do fast check, then if any prereqs are missing then go line by line * to be verbose about the errors */ - struct ref_list *p = &header->prerequisites; + struct string_list *p = &header->prerequisites; struct rev_info revs; const char *argv[] = {NULL, "--all", NULL}; struct commit *commit; @@ -198,9 +202,9 @@ int verify_bundle(struct repository *r, repo_init_revisions(r, &revs, NULL); for (i = 0; i < p->nr; i++) { - struct ref_list_entry *e = p->list + i; - const char *name = e->name; - struct object_id *oid = &e->oid; + struct string_list_item *e = p->items + i; + const char *name = e->string; + struct object_id *oid = e->util; struct object *o = parse_object(r, oid); if (o) { o->flags |= PREREQ_MARK; @@ -225,9 +229,9 @@ int verify_bundle(struct repository *r, i--; for (i = 0; i < p->nr; i++) { - struct ref_list_entry *e = p->list + i; - const char *name = e->name; - struct object_id *oid = &e->oid; + struct string_list_item *e = p->items + i; + const char *name = e->string; + const struct object_id *oid = e->util; struct object *o = parse_object(r, oid); assert(o); /* otherwise we'd have returned early */ if (o->flags & SHOWN) @@ -239,15 +243,15 @@ int verify_bundle(struct repository *r, /* Clean up objects used, as they will be reused. */ for (i = 0; i < p->nr; i++) { - struct ref_list_entry *e = p->list + i; - struct object_id *oid = &e->oid; + struct string_list_item *e = p->items + i; + struct object_id *oid = e->util; commit = lookup_commit_reference_gently(r, oid, 1); if (commit) clear_commit_marks(commit, ALL_REV_FLAGS); } if (verbose) { - struct ref_list *r; + struct string_list *r; r = &header->references; printf_ln(Q_("The bundle contains this ref:", diff --git a/bundle.h b/bundle.h index f9e2d1c8ef..1927d8cd6a 100644 --- a/bundle.h +++ b/bundle.h @@ -3,22 +3,23 @@ #include "strvec.h" #include "cache.h" - -struct ref_list { - unsigned int nr, alloc; - struct ref_list_entry { - struct object_id oid; - char *name; - } *list; -}; +#include "string-list.h" struct bundle_header { unsigned version; - struct ref_list prerequisites; - struct ref_list references; + struct string_list prerequisites; + struct string_list references; const struct git_hash_algo *hash_algo; }; +#define BUNDLE_HEADER_INIT \ +{ \ + .prerequisites = STRING_LIST_INIT_DUP, \ + .references = STRING_LIST_INIT_DUP, \ +} +void bundle_header_init(struct bundle_header *header); +void bundle_header_release(struct bundle_header *header); + int is_bundle(const char *path, int quiet); int read_bundle_header(const char *path, struct bundle_header *header); int create_bundle(struct repository *r, const char *path, diff --git a/transport.c b/transport.c index 95c1138e9a..745ffa2247 100644 --- a/transport.c +++ b/transport.c @@ -147,10 +147,10 @@ static struct ref *get_refs_from_bundle(struct transport *transport, transport->hash_algo = data->header.hash_algo; for (i = 0; i < data->header.references.nr; i++) { - struct ref_list_entry *e = data->header.references.list + i; - const char *name = e->name; + struct string_list_item *e = data->header.references.items + i; + const char *name = e->string; struct ref *ref = alloc_ref(name); - struct object_id *oid = &e->oid; + struct object_id *oid = e->util; oidcpy(&ref->old_oid, oid); ref->next = result; result = ref; @@ -177,6 +177,7 @@ static int close_bundle(struct transport *transport) struct bundle_transport_data *data = transport->data; if (data->fd > 0) close(data->fd); + bundle_header_release(&data->header); free(data); return 0; } @@ -1083,6 +1084,7 @@ struct transport *transport_get(struct remote *remote, const char *url) die(_("git-over-rsync is no longer supported")); } else if (url_is_local_not_ssh(url) && is_file(url) && is_bundle(url, 1)) { struct bundle_transport_data *data = xcalloc(1, sizeof(*data)); + bundle_header_init(&data->header); transport_check_allowed("file"); ret->data = data; ret->vtable = &bundle_vtable;