diff --git a/Documentation/technical/api-run-command.txt b/Documentation/technical/api-run-command.txt index b26c28133c..8994859c81 100644 --- a/Documentation/technical/api-run-command.txt +++ b/Documentation/technical/api-run-command.txt @@ -64,8 +64,8 @@ The functions above do the following: `start_async`:: Run a function asynchronously. Takes a pointer to a `struct - async` that specifies the details and returns a pipe FD - from which the caller reads. See below for details. + async` that specifies the details and returns a set of pipe FDs + for communication with the function. See below for details. `finish_async`:: @@ -135,7 +135,7 @@ stderr as follows: .in: The FD must be readable; it becomes child's stdin. .out: The FD must be writable; it becomes child's stdout. - .err > 0 is not supported. + .err: The FD must be writable; it becomes child's stderr. The specified FD is closed by start_command(), even if it fails to run the sub-process! @@ -180,17 +180,47 @@ The caller: struct async variable; 2. initializes .proc and .data; 3. calls start_async(); -4. processes the data by reading from the fd in .out; -5. closes .out; +4. processes communicates with proc through .in and .out; +5. closes .in and .out; 6. calls finish_async(). +The members .in, .out are used to provide a set of fd's for +communication between the caller and the callee as follows: + +. Specify 0 to have no file descriptor passed. The callee will + receive -1 in the corresponding argument. + +. Specify < 0 to have a pipe allocated; start_async() replaces + with the pipe FD in the following way: + + .in: Returns the writable pipe end into which the caller + writes; the readable end of the pipe becomes the function's + in argument. + + .out: Returns the readable pipe end from which the caller + reads; the writable end of the pipe becomes the function's + out argument. + + The caller of start_async() must close the returned FDs after it + has completed reading from/writing from them. + +. Specify a file descriptor > 0 to be used by the function: + + .in: The FD must be readable; it becomes the function's in. + .out: The FD must be writable; it becomes the function's out. + + The specified FD is closed by start_async(), even if it fails to + run the function. + The function pointer in .proc has the following signature: - int proc(int fd, void *data); + int proc(int in, int out, void *data); -. fd specifies a writable file descriptor to which the function must - write the data that it produces. The function *must* close this - descriptor before it returns. +. in, out specifies a set of file descriptors to which the function + must read/write the data that it needs/produces. The function + *must* close these descriptors before it returns. A descriptor + may be -1 if the caller did not configure a descriptor for that + direction. . data is the value that the caller has specified in the .data member of struct async. @@ -205,8 +235,8 @@ because this facility is implemented by a pipe to a forked process on UNIX, but by a thread in the same address space on Windows: . It cannot change the program's state (global variables, environment, - etc.) in a way that the caller notices; in other words, .out is the - only communication channel to the caller. + etc.) in a way that the caller notices; in other words, .in and .out + are the only communication channels to the caller. . It must not change the program's state that the caller of the facility also uses. diff --git a/builtin-fetch-pack.c b/builtin-fetch-pack.c index 8ed4a6feaa..dbd8b7bcc8 100644 --- a/builtin-fetch-pack.c +++ b/builtin-fetch-pack.c @@ -586,12 +586,12 @@ static int everything_local(struct ref **refs, int nr_match, char **match) return retval; } -static int sideband_demux(int fd, void *data) +static int sideband_demux(int in, int out, void *data) { int *xd = data; - int ret = recv_sideband("fetch-pack", xd[0], fd); - close(fd); + int ret = recv_sideband("fetch-pack", xd[0], out); + close(out); return ret; } @@ -613,6 +613,7 @@ static int get_pack(int xd[2], char **pack_lockfile) */ demux.proc = sideband_demux; demux.data = xd; + demux.out = -1; if (start_async(&demux)) die("fetch-pack: unable to fork off sideband" " demultiplexer"); diff --git a/builtin-fsck.c b/builtin-fsck.c index 0e5faae381..0929c7f245 100644 --- a/builtin-fsck.c +++ b/builtin-fsck.c @@ -578,7 +578,7 @@ static struct option fsck_opts[] = { OPT_BOOLEAN(0, "root", &show_root, "report root nodes"), OPT_BOOLEAN(0, "cache", &keep_cache_objects, "make index objects head nodes"), OPT_BOOLEAN(0, "reflogs", &include_reflogs, "make reflogs head nodes (default)"), - OPT_BOOLEAN(0, "full", &check_full, "also consider alternate objects"), + OPT_BOOLEAN(0, "full", &check_full, "also consider packs and alternate objects"), OPT_BOOLEAN(0, "strict", &check_strict, "enable more strict checking"), OPT_BOOLEAN(0, "lost-found", &write_lost_and_found, "write dangling objects in .git/lost-found"), diff --git a/builtin-receive-pack.c b/builtin-receive-pack.c index 4320c93e70..b704fbd5b5 100644 --- a/builtin-receive-pack.c +++ b/builtin-receive-pack.c @@ -2,6 +2,7 @@ #include "pack.h" #include "refs.h" #include "pkt-line.h" +#include "sideband.h" #include "run-command.h" #include "exec_cmd.h" #include "commit.h" @@ -27,11 +28,12 @@ static int receive_unpack_limit = -1; static int transfer_unpack_limit = -1; static int unpack_limit = 100; static int report_status; +static int use_sideband; static int prefer_ofs_delta = 1; static int auto_update_server_info; static int auto_gc = 1; static const char *head_name; -static char *capabilities_to_send; +static int sent_capabilities; static enum deny_action parse_deny_action(const char *var, const char *value) { @@ -105,19 +107,21 @@ static int receive_pack_config(const char *var, const char *value, void *cb) static int show_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data) { - if (!capabilities_to_send) + if (sent_capabilities) packet_write(1, "%s %s\n", sha1_to_hex(sha1), path); else - packet_write(1, "%s %s%c%s\n", - sha1_to_hex(sha1), path, 0, capabilities_to_send); - capabilities_to_send = NULL; + packet_write(1, "%s %s%c%s%s\n", + sha1_to_hex(sha1), path, 0, + " report-status delete-refs side-band-64k", + prefer_ofs_delta ? " ofs-delta" : ""); + sent_capabilities = 1; return 0; } static void write_head_info(void) { for_each_ref(show_ref, NULL); - if (capabilities_to_send) + if (!sent_capabilities) show_ref("capabilities^{}", null_sha1, 0, NULL); } @@ -135,11 +139,25 @@ static struct command *commands; static const char pre_receive_hook[] = "hooks/pre-receive"; static const char post_receive_hook[] = "hooks/post-receive"; +static int copy_to_sideband(int in, int out, void *arg) +{ + char data[128]; + while (1) { + ssize_t sz = xread(in, data, sizeof(data)); + if (sz <= 0) + break; + send_sideband(1, 2, data, sz, use_sideband); + } + close(in); + return 0; +} + static int run_receive_hook(const char *hook_name) { static char buf[sizeof(commands->old_sha1) * 2 + PATH_MAX + 4]; struct command *cmd; struct child_process proc; + struct async muxer; const char *argv[2]; int have_input = 0, code; @@ -159,9 +177,23 @@ static int run_receive_hook(const char *hook_name) proc.in = -1; proc.stdout_to_stderr = 1; + if (use_sideband) { + memset(&muxer, 0, sizeof(muxer)); + muxer.proc = copy_to_sideband; + muxer.in = -1; + code = start_async(&muxer); + if (code) + return code; + proc.err = muxer.in; + } + code = start_command(&proc); - if (code) + if (code) { + if (use_sideband) + finish_async(&muxer); return code; + } + for (cmd = commands; cmd; cmd = cmd->next) { if (!cmd->error_string) { size_t n = snprintf(buf, sizeof(buf), "%s %s %s\n", @@ -173,6 +205,8 @@ static int run_receive_hook(const char *hook_name) } } close(proc.in); + if (use_sideband) + finish_async(&muxer); return finish_command(&proc); } @@ -180,6 +214,8 @@ static int run_update_hook(struct command *cmd) { static const char update_hook[] = "hooks/update"; const char *argv[5]; + struct child_process proc; + int code; if (access(update_hook, X_OK) < 0) return 0; @@ -190,8 +226,18 @@ static int run_update_hook(struct command *cmd) argv[3] = sha1_to_hex(cmd->new_sha1); argv[4] = NULL; - return run_command_v_opt(argv, RUN_COMMAND_NO_STDIN | - RUN_COMMAND_STDOUT_TO_STDERR); + memset(&proc, 0, sizeof(proc)); + proc.no_stdin = 1; + proc.stdout_to_stderr = 1; + proc.err = use_sideband ? -1 : 0; + proc.argv = argv; + + code = start_command(&proc); + if (code) + return code; + if (use_sideband) + copy_to_sideband(proc.err, -1, NULL); + return finish_command(&proc); } static int is_ref_checked_out(const char *ref) @@ -368,8 +414,9 @@ static char update_post_hook[] = "hooks/post-update"; static void run_update_post_hook(struct command *cmd) { struct command *cmd_p; - int argc, status; + int argc; const char **argv; + struct child_process proc; for (argc = 0, cmd_p = cmd; cmd_p; cmd_p = cmd_p->next) { if (cmd_p->error_string) @@ -391,8 +438,18 @@ static void run_update_post_hook(struct command *cmd) argc++; } argv[argc] = NULL; - status = run_command_v_opt(argv, RUN_COMMAND_NO_STDIN - | RUN_COMMAND_STDOUT_TO_STDERR); + + memset(&proc, 0, sizeof(proc)); + proc.no_stdin = 1; + proc.stdout_to_stderr = 1; + proc.err = use_sideband ? -1 : 0; + proc.argv = argv; + + if (!start_command(&proc)) { + if (use_sideband) + copy_to_sideband(proc.err, -1, NULL); + finish_command(&proc); + } } static void execute_commands(const char *unpacker_error) @@ -452,6 +509,8 @@ static void read_head_info(void) if (reflen + 82 < len) { if (strstr(refname + reflen + 1, "report-status")) report_status = 1; + if (strstr(refname + reflen + 1, "side-band-64k")) + use_sideband = LARGE_PACKET_MAX; } cmd = xmalloc(sizeof(struct command) + len - 80); hashcpy(cmd->old_sha1, old_sha1); @@ -551,17 +610,25 @@ static const char *unpack(void) static void report(const char *unpack_status) { struct command *cmd; - packet_write(1, "unpack %s\n", - unpack_status ? unpack_status : "ok"); + struct strbuf buf = STRBUF_INIT; + + packet_buf_write(&buf, "unpack %s\n", + unpack_status ? unpack_status : "ok"); for (cmd = commands; cmd; cmd = cmd->next) { if (!cmd->error_string) - packet_write(1, "ok %s\n", - cmd->ref_name); + packet_buf_write(&buf, "ok %s\n", + cmd->ref_name); else - packet_write(1, "ng %s %s\n", - cmd->ref_name, cmd->error_string); + packet_buf_write(&buf, "ng %s %s\n", + cmd->ref_name, cmd->error_string); } - packet_flush(1); + packet_buf_flush(&buf); + + if (use_sideband) + send_sideband(1, 1, buf.buf, buf.len, use_sideband); + else + safe_write(1, buf.buf, buf.len); + strbuf_release(&buf); } static int delete_only(struct command *cmd) @@ -658,10 +725,6 @@ int cmd_receive_pack(int argc, const char **argv, const char *prefix) else if (0 <= receive_unpack_limit) unpack_limit = receive_unpack_limit; - capabilities_to_send = (prefer_ofs_delta) ? - " report-status delete-refs ofs-delta " : - " report-status delete-refs "; - if (advertise_refs || !stateless_rpc) { add_alternate_refs(); write_head_info(); @@ -695,5 +758,7 @@ int cmd_receive_pack(int argc, const char **argv, const char *prefix) if (auto_update_server_info) update_server_info(0); } + if (use_sideband) + packet_flush(1); return 0; } diff --git a/builtin-send-pack.c b/builtin-send-pack.c index 76c72065de..2183a47052 100644 --- a/builtin-send-pack.c +++ b/builtin-send-pack.c @@ -372,6 +372,14 @@ static void print_helper_status(struct ref *ref) strbuf_release(&buf); } +static int sideband_demux(int in, int out, void *data) +{ + int *fd = data; + int ret = recv_sideband("send-pack", fd[0], out); + close(out); + return ret; +} + int send_pack(struct send_pack_args *args, int fd[], struct child_process *conn, struct ref *remote_refs, @@ -382,18 +390,22 @@ int send_pack(struct send_pack_args *args, struct strbuf req_buf = STRBUF_INIT; struct ref *ref; int new_refs; - int ask_for_status_report = 0; int allow_deleting_refs = 0; - int expect_status_report = 0; + int status_report = 0; + int use_sideband = 0; + unsigned cmds_sent = 0; int ret; + struct async demux; /* Does the other end support the reporting? */ if (server_supports("report-status")) - ask_for_status_report = 1; + status_report = 1; if (server_supports("delete-refs")) allow_deleting_refs = 1; if (server_supports("ofs-delta")) args->use_ofs_delta = 1; + if (server_supports("side-band-64k")) + use_sideband = 1; if (!remote_refs) { fprintf(stderr, "No refs in common and none specified; doing nothing.\n" @@ -426,28 +438,30 @@ int send_pack(struct send_pack_args *args, if (!ref->deletion) new_refs++; - if (!args->dry_run) { + if (args->dry_run) { + ref->status = REF_STATUS_OK; + } else { char *old_hex = sha1_to_hex(ref->old_sha1); char *new_hex = sha1_to_hex(ref->new_sha1); - if (ask_for_status_report) { - packet_buf_write(&req_buf, "%s %s %s%c%s", + if (!cmds_sent && (status_report || use_sideband)) { + packet_buf_write(&req_buf, "%s %s %s%c%s%s", old_hex, new_hex, ref->name, 0, - "report-status"); - ask_for_status_report = 0; - expect_status_report = 1; + status_report ? " report-status" : "", + use_sideband ? " side-band-64k" : ""); } else packet_buf_write(&req_buf, "%s %s %s", old_hex, new_hex, ref->name); + ref->status = status_report ? + REF_STATUS_EXPECTING_REPORT : + REF_STATUS_OK; + cmds_sent++; } - ref->status = expect_status_report ? - REF_STATUS_EXPECTING_REPORT : - REF_STATUS_OK; } if (args->stateless_rpc) { - if (!args->dry_run) { + if (!args->dry_run && cmds_sent) { packet_buf_flush(&req_buf); send_sideband(out, -1, req_buf.buf, req_buf.len, LARGE_PACKET_MAX); } @@ -457,23 +471,43 @@ int send_pack(struct send_pack_args *args, } strbuf_release(&req_buf); - if (new_refs && !args->dry_run) { + if (use_sideband && cmds_sent) { + memset(&demux, 0, sizeof(demux)); + demux.proc = sideband_demux; + demux.data = fd; + demux.out = -1; + if (start_async(&demux)) + die("receive-pack: unable to fork off sideband demultiplexer"); + in = demux.out; + } + + if (new_refs && cmds_sent) { if (pack_objects(out, remote_refs, extra_have, args) < 0) { for (ref = remote_refs; ref; ref = ref->next) ref->status = REF_STATUS_NONE; + if (use_sideband) + finish_async(&demux); return -1; } } - if (args->stateless_rpc && !args->dry_run) + if (args->stateless_rpc && cmds_sent) packet_flush(out); - if (expect_status_report) + if (status_report && cmds_sent) ret = receive_status(in, remote_refs); else ret = 0; if (args->stateless_rpc) packet_flush(out); + if (use_sideband && cmds_sent) { + if (finish_async(&demux)) { + error("error in sideband demultiplexer"); + ret = -1; + } + close(demux.out); + } + if (ret < 0) return ret; for (ref = remote_refs; ref; ref = ref->next) { diff --git a/convert.c b/convert.c index 27acce58bc..4f8fcb7bbb 100644 --- a/convert.c +++ b/convert.c @@ -241,7 +241,7 @@ struct filter_params { const char *cmd; }; -static int filter_buffer(int fd, void *data) +static int filter_buffer(int in, int out, void *data) { /* * Spawn cmd and feed the buffer contents through its stdin. @@ -255,7 +255,7 @@ static int filter_buffer(int fd, void *data) child_process.argv = argv; child_process.use_shell = 1; child_process.in = -1; - child_process.out = fd; + child_process.out = out; if (start_command(&child_process)) return error("cannot fork to run external filter %s", params->cmd); @@ -292,6 +292,7 @@ static int apply_filter(const char *path, const char *src, size_t len, memset(&async, 0, sizeof(async)); async.proc = filter_buffer; async.data = ¶ms; + async.out = -1; params.src = src; params.size = len; params.cmd = cmd; diff --git a/remote-curl.c b/remote-curl.c index a904164e42..d388120851 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -184,13 +184,13 @@ static struct discovery* discover_refs(const char *service) return last; } -static int write_discovery(int fd, void *data) +static int write_discovery(int in, int out, void *data) { struct discovery *heads = data; int err = 0; - if (write_in_full(fd, heads->buf, heads->len) != heads->len) + if (write_in_full(out, heads->buf, heads->len) != heads->len) err = 1; - close(fd); + close(out); return err; } @@ -202,6 +202,7 @@ static struct ref *parse_git_refs(struct discovery *heads) memset(&async, 0, sizeof(async)); async.proc = write_discovery; async.data = heads; + async.out = -1; if (start_async(&async)) die("cannot start thread to parse advertised refs"); diff --git a/run-command.c b/run-command.c index 2feb493951..0cd7f02ffe 100644 --- a/run-command.c +++ b/run-command.c @@ -233,6 +233,9 @@ fail_pipe: else if (need_err) { dup2(fderr[1], 2); close_pair(fderr); + } else if (cmd->err > 1) { + dup2(cmd->err, 2); + close(cmd->err); } if (cmd->no_stdout) @@ -325,6 +328,8 @@ fail_pipe: fherr = open("/dev/null", O_RDWR); else if (need_err) fherr = dup(fderr[1]); + else if (cmd->err > 2) + fherr = dup(cmd->err); if (cmd->no_stdout) fhout = open("/dev/null", O_RDWR); @@ -394,6 +399,8 @@ fail_pipe: if (need_err) close(fderr[1]); + else if (cmd->err) + close(cmd->err); return 0; } @@ -444,17 +451,51 @@ int run_command_v_opt_cd_env(const char **argv, int opt, const char *dir, const static unsigned __stdcall run_thread(void *data) { struct async *async = data; - return async->proc(async->fd_for_proc, async->data); + return async->proc(async->proc_in, async->proc_out, async->data); } #endif int start_async(struct async *async) { - int pipe_out[2]; + int need_in, need_out; + int fdin[2], fdout[2]; + int proc_in, proc_out; - if (pipe(pipe_out) < 0) - return error("cannot create pipe: %s", strerror(errno)); - async->out = pipe_out[0]; + need_in = async->in < 0; + if (need_in) { + if (pipe(fdin) < 0) { + if (async->out > 0) + close(async->out); + return error("cannot create pipe: %s", strerror(errno)); + } + async->in = fdin[1]; + } + + need_out = async->out < 0; + if (need_out) { + if (pipe(fdout) < 0) { + if (need_in) + close_pair(fdin); + else if (async->in) + close(async->in); + return error("cannot create pipe: %s", strerror(errno)); + } + async->out = fdout[0]; + } + + if (need_in) + proc_in = fdin[0]; + else if (async->in) + proc_in = async->in; + else + proc_in = -1; + + if (need_out) + proc_out = fdout[1]; + else if (async->out) + proc_out = async->out; + else + proc_out = -1; #ifndef WIN32 /* Flush stdio before fork() to avoid cloning buffers */ @@ -463,24 +504,47 @@ int start_async(struct async *async) async->pid = fork(); if (async->pid < 0) { error("fork (async) failed: %s", strerror(errno)); - close_pair(pipe_out); - return -1; + goto error; } if (!async->pid) { - close(pipe_out[0]); - exit(!!async->proc(pipe_out[1], async->data)); + if (need_in) + close(fdin[1]); + if (need_out) + close(fdout[0]); + exit(!!async->proc(proc_in, proc_out, async->data)); } - close(pipe_out[1]); + + if (need_in) + close(fdin[0]); + else if (async->in) + close(async->in); + + if (need_out) + close(fdout[1]); + else if (async->out) + close(async->out); #else - async->fd_for_proc = pipe_out[1]; + async->proc_in = proc_in; + async->proc_out = proc_out; async->tid = (HANDLE) _beginthreadex(NULL, 0, run_thread, async, 0, NULL); if (!async->tid) { error("cannot create thread: %s", strerror(errno)); - close_pair(pipe_out); - return -1; + goto error; } #endif return 0; + +error: + if (need_in) + close_pair(fdin); + else if (async->in) + close(async->in); + + if (need_out) + close_pair(fdout); + else if (async->out) + close(async->out); + return -1; } int finish_async(struct async *async) diff --git a/run-command.h b/run-command.h index 967ba8cc09..94619f52d9 100644 --- a/run-command.h +++ b/run-command.h @@ -18,7 +18,7 @@ struct child_process { * - Specify > 0 to set a channel to a particular FD as follows: * .in: a readable FD, becomes child's stdin * .out: a writable FD, becomes child's stdout/stderr - * .err > 0 not supported + * .err: a writable FD, becomes child's stderr * The specified FD is closed by start_command(), even in case * of errors! */ @@ -66,17 +66,20 @@ int run_command_v_opt_cd_env(const char **argv, int opt, const char *dir, const */ struct async { /* - * proc writes to fd and closes it; + * proc reads from in; closes it before return + * proc writes to out; closes it before return * returns 0 on success, non-zero on failure */ - int (*proc)(int fd, void *data); + int (*proc)(int in, int out, void *data); void *data; + int in; /* caller writes here and closes it */ int out; /* caller reads from here and closes it */ #ifndef WIN32 pid_t pid; #else HANDLE tid; - int fd_for_proc; + int proc_in; + int proc_out; #endif }; diff --git a/t/t5401-update-hooks.sh b/t/t5401-update-hooks.sh index 325714e529..0686390d2f 100755 --- a/t/t5401-update-hooks.sh +++ b/t/t5401-update-hooks.sh @@ -119,19 +119,19 @@ test_expect_success 'send-pack produced no output' ' ' cat <expect -STDOUT pre-receive -STDERR pre-receive -STDOUT update refs/heads/master -STDERR update refs/heads/master -STDOUT update refs/heads/tofail -STDERR update refs/heads/tofail -STDOUT post-receive -STDERR post-receive -STDOUT post-update -STDERR post-update +remote: STDOUT pre-receive +remote: STDERR pre-receive +remote: STDOUT update refs/heads/master +remote: STDERR update refs/heads/master +remote: STDOUT update refs/heads/tofail +remote: STDERR update refs/heads/tofail +remote: STDOUT post-receive +remote: STDERR post-receive +remote: STDOUT post-update +remote: STDERR post-update EOF test_expect_success 'send-pack stderr contains hook messages' ' - grep ^STD send.err >actual && + grep ^remote: send.err | sed "s/ *\$//" >actual && test_cmp - actual object.sha1)); } -static int do_rev_list(int fd, void *create_full_pack) +static int do_rev_list(int in, int out, void *create_full_pack) { int i; struct rev_info revs; - pack_pipe = xfdopen(fd, "w"); + pack_pipe = xfdopen(out, "w"); init_revisions(&revs, NULL); revs.tag_objects = 1; revs.tree_objects = 1; @@ -162,8 +162,9 @@ static void create_pack_file(void) int arg = 0; if (shallow_nr) { + memset(&rev_list, 0, sizeof(rev_list)); rev_list.proc = do_rev_list; - rev_list.data = 0; + rev_list.out = -1; if (start_async(&rev_list)) die("git upload-pack: unable to fork git-rev-list"); argv[arg++] = "pack-objects";