Merge branch 'jc/sideband' into jc/archive

* jc/sideband:
  Prepare larger packet buffer for upload-pack protocol.
  Move sideband server side support into reusable form.
  Move sideband client side support into reusable form.
  get_sha1_hex() micro-optimization

Makefile

@@ -233,7 +233,7 @@ XDIFF_LIB=xdiff/lib.a
 
 LIB_H = \
 	archive.h blob.h cache.h commit.h csum-file.h delta.h \
-	diff.h object.h pack.h pkt-line.h quote.h refs.h \
+	diff.h object.h pack.h pkt-line.h quote.h refs.h sideband.h \
 	run-command.h strbuf.h tag.h tree.h git-compat-util.h revision.h \
 	tree-walk.h log-tree.h dir.h path-list.h unpack-trees.h builtin.h
 
@@ -245,7 +245,7 @@ DIFF_OBJS = \
 LIB_OBJS = \
 	blob.o commit.o connect.o csum-file.o cache-tree.o base85.o \
 	date.o diff-delta.o entry.o exec_cmd.o ident.o lockfile.o \
-	object.o pack-check.o patch-delta.o path.o pkt-line.o \
+	object.o pack-check.o patch-delta.o path.o pkt-line.o sideband.o \
 	quote.o read-cache.o refs.o run-command.o dir.o object-refs.o \
 	server-info.o setup.o sha1_file.o sha1_name.o strbuf.o \
 	tag.o tree.o usage.o config.o environment.o ctype.o copy.o \

fetch-clone.c

@@ -1,6 +1,7 @@
 #include "cache.h"
 #include "exec_cmd.h"
 #include "pkt-line.h"
+#include "sideband.h"
 #include <sys/wait.h>
 #include <sys/time.h>
 
@@ -114,36 +115,13 @@ static pid_t setup_sideband(int sideband, const char *me, int fd[2], int xd[2])
 		die("%s: unable to fork off sideband demultiplexer", me);
 	if (!side_pid) {
 		/* subprocess */
+		char buf[LARGE_PACKET_MAX];
+
 		close(fd[0]);
 		if (xd[0] != xd[1])
 			close(xd[1]);
-		while (1) {
+		if (recv_sideband(me, xd[0], fd[1], 2, buf, sizeof(buf)))
-			char buf[1024];
-			int len = packet_read_line(xd[0], buf, sizeof(buf));
-			if (len == 0)
-				break;
-			if (len < 1)
-				die("%s: protocol error: no band designator",
-				    me);
-			len--;
-			switch (buf[0] & 0xFF) {
-			case 3:
-				safe_write(2, "remote: ", 8);
-				safe_write(2, buf+1, len);
-				safe_write(2, "\n", 1);
 			exit(1);
-			case 2:
-				safe_write(2, "remote: ", 8);
-				safe_write(2, buf+1, len);
-				continue;
-			case 1:
-				safe_write(fd[1], buf+1, len);
-				continue;
-			default:
-				die("%s: protocol error: bad band #%d",
-				    me, (buf[0] & 0xFF));
-			}
-		}
 		exit(0);
 	}
 	close(xd[0]);

fetch-pack.c

@@ -166,10 +166,11 @@ static int find_common(int fd[2], unsigned char *result_sha1,
 		}
 
 		if (!fetching)
-			packet_write(fd[1], "want %s%s%s%s\n",
+			packet_write(fd[1], "want %s%s%s%s%s\n",
 				     sha1_to_hex(remote),
 				     (multi_ack ? " multi_ack" : ""),
-				     (use_sideband ? " side-band" : ""),
+				     (use_sideband == 2 ? " side-band-64k" : ""),
+				     (use_sideband == 1 ? " side-band" : ""),
 				     (use_thin_pack ? " thin-pack" : ""));
 		else
 			packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
@@ -426,7 +427,12 @@ static int fetch_pack(int fd[2], int nr_match, char **match)
 			fprintf(stderr, "Server supports multi_ack\n");
 		multi_ack = 1;
 	}
-	if (server_supports("side-band")) {
+	if (server_supports("side-band-64k")) {
+		if (verbose)
+			fprintf(stderr, "Server supports side-band-64k\n");
+		use_sideband = 2;
+	}
+	else if (server_supports("side-band")) {
 		if (verbose)
 			fprintf(stderr, "Server supports side-band\n");
 		use_sideband = 1;

sha1_file.c

@@ -26,15 +26,43 @@ const unsigned char null_sha1[20];
 
 static unsigned int sha1_file_open_flag = O_NOATIME;
 
-static unsigned hexval(char c)
+static inline unsigned int hexval(unsigned int c)
-{
+{
-	if (c >= '0' && c <= '9')
+	static signed char val[256] = {
-		return c - '0';
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* 00-07 */
-	if (c >= 'a' && c <= 'f')
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* 08-0f */
-		return c - 'a' + 10;
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* 10-17 */
-	if (c >= 'A' && c <= 'F')
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* 18-1f */
-		return c - 'A' + 10;
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* 20-27 */
-	return ~0;
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* 28-2f */
+		  0,  1,  2,  3,  4,  5,  6,  7,		/* 30-37 */
+		  8,  9, -1, -1, -1, -1, -1, -1,		/* 38-3f */
+		 -1, 10, 11, 12, 13, 14, 15, -1,		/* 40-47 */
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* 48-4f */
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* 50-57 */
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* 58-5f */
+		 -1, 10, 11, 12, 13, 14, 15, -1,		/* 60-67 */
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* 68-67 */
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* 70-77 */
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* 78-7f */
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* 80-87 */
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* 88-8f */
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* 90-97 */
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* 98-9f */
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* a0-a7 */
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* a8-af */
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* b0-b7 */
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* b8-bf */
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* c0-c7 */
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* c8-cf */
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* d0-d7 */
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* d8-df */
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* e0-e7 */
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* e8-ef */
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* f0-f7 */
+		 -1, -1, -1, -1, -1, -1, -1, -1,		/* f8-ff */
+	};
+	return val[c];
 }
 
 int get_sha1_hex(const char *hex, unsigned char *sha1)

sideband.c

@@ -0,0 +1,74 @@
+#include "pkt-line.h"
+#include "sideband.h"
+
+/*
+ * Receive multiplexed output stream over git native protocol.
+ * in_stream is the input stream from the remote, which carries data
+ * in pkt_line format with band designator.  Demultiplex it into out
+ * and err and return error appropriately.  Band #1 carries the
+ * primary payload.  Things coming over band #2 is not necessarily
+ * error; they are usually informative message on the standard error
+ * stream, aka "verbose").  A message over band #3 is a signal that
+ * the remote died unexpectedly.  A flush() concludes the stream.
+ */
+int recv_sideband(const char *me, int in_stream, int out, int err, char *buf, int bufsz)
+{
+	while (1) {
+		int len = packet_read_line(in_stream, buf, bufsz);
+		if (len == 0)
+			break;
+		if (len < 1) {
+			len = sprintf(buf, "%s: protocol error: no band designator\n", me);
+			safe_write(err, buf, len);
+			return SIDEBAND_PROTOCOL_ERROR;
+		}
+		len--;
+		switch (buf[0] & 0xFF) {
+		case 3:
+			safe_write(err, "remote: ", 8);
+			safe_write(err, buf+1, len);
+			safe_write(err, "\n", 1);
+			return SIDEBAND_REMOTE_ERROR;
+		case 2:
+			safe_write(err, "remote: ", 8);
+			safe_write(err, buf+1, len);
+			continue;
+		case 1:
+			safe_write(out, buf+1, len);
+			continue;
+		default:
+			len = sprintf(buf + 1,
+				      "%s: protocol error: bad band #%d\n",
+				      me, buf[0] & 0xFF);
+			safe_write(err, buf+1, len);
+			return SIDEBAND_PROTOCOL_ERROR;
+		}
+	}
+	return 0;
+}
+
+/*
+ * fd is connected to the remote side; send the sideband data
+ * over multiplexed packet stream.
+ */
+ssize_t send_sideband(int fd, int band, const char *data, ssize_t sz, int packet_max)
+{
+	ssize_t ssz = sz;
+	const char *p = data;
+
+	while (sz) {
+		unsigned n;
+		char hdr[5];
+
+		n = sz;
+		if (packet_max - 5 < n)
+			n = packet_max - 5;
+		sprintf(hdr, "%04x", n + 5);
+		hdr[4] = band;
+		safe_write(fd, hdr, 5);
+		safe_write(fd, p, n);
+		p += n;
+		sz -= n;
+	}
+	return ssz;
+}

sideband.h

@@ -0,0 +1,13 @@
+#ifndef SIDEBAND_H
+#define SIDEBAND_H
+
+#define SIDEBAND_PROTOCOL_ERROR -2
+#define SIDEBAND_REMOTE_ERROR -1
+
+#define DEFAULT_PACKET_MAX 1000
+#define LARGE_PACKET_MAX 65520
+
+int recv_sideband(const char *me, int in_stream, int out, int err, char *, int);
+ssize_t send_sideband(int fd, int band, const char *data, ssize_t sz, int packet_max);
+
+#endif

upload-pack.c

@@ -4,6 +4,7 @@
 #include "cache.h"
 #include "refs.h"
 #include "pkt-line.h"
+#include "sideband.h"
 #include "tag.h"
 #include "object.h"
 #include "commit.h"
@@ -19,6 +20,9 @@ static int use_thin_pack;
 static struct object_array have_obj;
 static struct object_array want_obj;
 static unsigned int timeout;
+/* 0 for no sideband,
+ * otherwise maximum packet size (up to 65520 bytes).
+ */
 static int use_sideband;
 
 static void reset_timeout(void)
@@ -33,19 +37,10 @@ static int strip(char *line, int len)
 	return len;
 }
 
-#define PACKET_MAX 1000
 static ssize_t send_client_data(int fd, const char *data, ssize_t sz)
 {
-	ssize_t ssz;
+	if (use_sideband)
-	const char *p;
+		return send_sideband(1, fd, data, sz, use_sideband);
-
-	if (!data) {
-		if (!use_sideband)
-			return 0;
-		packet_flush(1);
-	}
-
-	if (!use_sideband) {
 	if (fd == 3)
 		/* emergency quit */
 		fd = 2;
@@ -55,24 +50,6 @@ static ssize_t send_client_data(int fd, const char *data, ssize_t sz)
 	}
 	return safe_write(fd, data, sz);
 }
-	p = data;
-	ssz = sz;
-	while (sz) {
-		unsigned n;
-		char hdr[5];
-
-		n = sz;
-		if (PACKET_MAX - 5 < n)
-			n = PACKET_MAX - 5;
-		sprintf(hdr, "%04x", n + 5);
-		hdr[4] = fd;
-		safe_write(1, hdr, 5);
-		safe_write(1, p, n);
-		p += n;
-		sz -= n;
-	}
-	return ssz;
-}
 
 static void create_pack_file(void)
 {
@@ -308,7 +285,8 @@ static void create_pack_file(void)
 				goto fail;
 			fprintf(stderr, "flushed.\n");
 		}
-		send_client_data(1, NULL, 0);
+		if (use_sideband)
+			packet_flush(1);
 		return;
 	}
  fail:
@@ -413,8 +391,10 @@ static void receive_needs(void)
 			multi_ack = 1;
 		if (strstr(line+45, "thin-pack"))
 			use_thin_pack = 1;
-		if (strstr(line+45, "side-band"))
+		if (strstr(line+45, "side-band-64k"))
-			use_sideband = 1;
+			use_sideband = LARGE_PACKET_MAX;
+		else if (strstr(line+45, "side-band"))
+			use_sideband = DEFAULT_PACKET_MAX;
 
 		/* We have sent all our refs already, and the other end
 		 * should have chosen out of them; otherwise they are
@@ -436,7 +416,7 @@ static void receive_needs(void)
 
 static int send_ref(const char *refname, const unsigned char *sha1)
 {
-	static const char *capabilities = "multi_ack thin-pack side-band";
+	static const char *capabilities = "multi_ack thin-pack side-band side-band-64k";
 	struct object *o = parse_object(sha1);
 
 	if (!o)