Merge branch 'np/progress'

* np/progress:
  Show total transferred as part of throughput progress
  make sure throughput display gets updated even if progress doesn't move
  return the prune-packed progress display to the inner loop
  add throughput display to git-push
  add some copyright notice to the progress display code
  add throughput display to index-pack
  add throughput to progress display
  relax usage of the progress API
  make struct progress an opaque type
  prune-packed: don't call display_progress() for every file
  Stop displaying "Pack pack-$ID created." during git-gc
  Teach prune-packed to use the standard progress meter
  Change 'Deltifying objects' to 'Compressing objects'
  fix for more minor memory leaks
  fix const issues with some functions
  pack-objects.c: fix some global variable abuse and memory leaks
  pack-objects: no delta possible with only one object in the list
  cope with multiple line breaks within sideband progress messages
  more compact progress display

builtin-pack-objects.c

@@ -65,8 +65,6 @@ static int no_reuse_delta, no_reuse_object, keep_unreachable;
 static int local;
 static int incremental;
 static int allow_ofs_delta;
-static const char *pack_tmp_name, *idx_tmp_name;
-static char tmpname[PATH_MAX];
 static const char *base_name;
 static int progress = 1;
 static int window = 10;
@@ -75,7 +73,7 @@ static int depth = 50;
 static int delta_search_threads = 1;
 static int pack_to_stdout;
 static int num_preferred_base;
-static struct progress progress_state;
+static struct progress *progress_state;
 static int pack_compression_level = Z_DEFAULT_COMPRESSION;
 static int pack_compression_seen;
 
@@ -587,12 +585,6 @@ static off_t write_one(struct sha1file *f,
 	return offset + size;
 }
 
-static int open_object_dir_tmp(const char *path)
-{
-    snprintf(tmpname, sizeof(tmpname), "%s/%s", get_object_directory(), path);
-    return xmkstemp(tmpname);
-}
-
 /* forward declaration for write_pack_file */
 static int adjust_perm(const char *path, mode_t mode);
 
@@ -606,16 +598,21 @@ static void write_pack_file(void)
 	uint32_t nr_remaining = nr_result;
 
 	if (do_progress)
-		start_progress(&progress_state, "Writing %u objects...", "", nr_result);
+		progress_state = start_progress("Writing objects", nr_result);
 	written_list = xmalloc(nr_objects * sizeof(struct object_entry *));
 
 	do {
 		unsigned char sha1[20];
+		char *pack_tmp_name = NULL;
 
 		if (pack_to_stdout) {
-			f = sha1fd(1, "<stdout>");
+			f = sha1fd_throughput(1, "<stdout>", progress_state);
 		} else {
-			int fd = open_object_dir_tmp("tmp_pack_XXXXXX");
+			char tmpname[PATH_MAX];
+			int fd;
+			snprintf(tmpname, sizeof(tmpname),
+				 "%s/tmp_pack_XXXXXX", get_object_directory());
+			fd = xmkstemp(tmpname);
 			pack_tmp_name = xstrdup(tmpname);
 			f = sha1fd(fd, pack_tmp_name);
 		}
@@ -632,8 +629,7 @@ static void write_pack_file(void)
 			if (!offset_one)
 				break;
 			offset = offset_one;
-			if (do_progress)
-				display_progress(&progress_state, written);
+			display_progress(progress_state, written);
 		}
 
 		/*
@@ -643,19 +639,21 @@ static void write_pack_file(void)
 		if (pack_to_stdout || nr_written == nr_remaining) {
 			sha1close(f, sha1, 1);
 		} else {
-			sha1close(f, sha1, 0);
-			fixup_pack_header_footer(f->fd, sha1, pack_tmp_name, nr_written);
-			close(f->fd);
+			int fd = sha1close(f, NULL, 0);
+			fixup_pack_header_footer(fd, sha1, pack_tmp_name, nr_written);
+			close(fd);
 		}
 
 		if (!pack_to_stdout) {
 			mode_t mode = umask(0);
+			char *idx_tmp_name, tmpname[PATH_MAX];
 
 			umask(mode);
 			mode = 0444 & ~mode;
 
 			idx_tmp_name = write_idx_file(NULL,
-				(struct pack_idx_entry **) written_list, nr_written, sha1);
+					(struct pack_idx_entry **) written_list,
+					nr_written, sha1);
 			snprintf(tmpname, sizeof(tmpname), "%s-%s.pack",
 				 base_name, sha1_to_hex(sha1));
 			if (adjust_perm(pack_tmp_name, mode))
@@ -672,6 +670,8 @@ static void write_pack_file(void)
 			if (rename(idx_tmp_name, tmpname))
 				die("unable to rename temporary index file: %s",
 				    strerror(errno));
+			free(idx_tmp_name);
+			free(pack_tmp_name);
 			puts(sha1_to_hex(sha1));
 		}
 
@@ -683,8 +683,7 @@ static void write_pack_file(void)
 	} while (nr_remaining && i < nr_objects);
 
 	free(written_list);
-	if (do_progress)
-		stop_progress(&progress_state);
+	stop_progress(&progress_state);
 	if (written != nr_result)
 		die("wrote %u objects while expecting %u", written, nr_result);
 	/*
@@ -852,8 +851,7 @@ static int add_object_entry(const unsigned char *sha1, enum object_type type,
 	else
 		object_ix[-1 - ix] = nr_objects;
 
-	if (progress)
-		display_progress(&progress_state, nr_objects);
+	display_progress(progress_state, nr_objects);
 
 	if (name && no_try_delta(name))
 		entry->no_try_delta = 1;
@@ -1516,8 +1514,7 @@ static void find_deltas(struct object_entry **list, unsigned list_size,
 
 		progress_lock();
 		(*processed)++;
-		if (progress)
-			display_progress(&progress_state, *processed);
+		display_progress(progress_state, *processed);
 		progress_unlock();
 
 		/*
@@ -1714,16 +1711,14 @@ static void prepare_pack(int window, int depth)
 		delta_list[n++] = entry;
 	}
 
-	if (nr_deltas) {
+	if (nr_deltas && n > 1) {
 		unsigned nr_done = 0;
 		if (progress)
-			start_progress(&progress_state,
-				       "Deltifying %u objects...", "",
-				       nr_deltas);
+			progress_state = start_progress("Compressing objects",
+							nr_deltas);
 		qsort(delta_list, n, sizeof(*delta_list), type_size_sort);
 		ll_find_deltas(delta_list, n, window+1, depth, &nr_done);
-		if (progress)
-			stop_progress(&progress_state);
+		stop_progress(&progress_state);
 		if (nr_done != nr_deltas)
 			die("inconsistency with delta count");
 	}
@@ -2135,23 +2130,17 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
 	prepare_packed_git();
 
 	if (progress)
-		start_progress(&progress_state, "Generating pack...",
-			       "Counting objects: ", 0);
+		progress_state = start_progress("Counting objects", 0);
 	if (!use_internal_rev_list)
 		read_object_list_from_stdin();
 	else {
 		rp_av[rp_ac] = NULL;
 		get_object_list(rp_ac, rp_av);
 	}
-	if (progress) {
-		stop_progress(&progress_state);
-		fprintf(stderr, "Done counting %u objects.\n", nr_objects);
-	}
+	stop_progress(&progress_state);
 
 	if (non_empty && !nr_result)
 		return 0;
-	if (progress && (nr_objects != nr_result))
-		fprintf(stderr, "Result has %u objects.\n", nr_result);
 	if (nr_result)
 		prepare_pack(window, depth);
 	write_pack_file();

builtin-prune-packed.c

@@ -1,5 +1,6 @@
 #include "builtin.h"
 #include "cache.h"
+#include "progress.h"
 
 static const char prune_packed_usage[] =
 "git-prune-packed [-n] [-q]";
@@ -7,6 +8,8 @@ static const char prune_packed_usage[] =
 #define DRY_RUN 01
 #define VERBOSE 02
 
+static struct progress *progress;
+
 static void prune_dir(int i, DIR *dir, char *pathname, int len, int opts)
 {
 	struct dirent *de;
@@ -27,6 +30,7 @@ static void prune_dir(int i, DIR *dir, char *pathname, int len, int opts)
 			printf("rm -f %s\n", pathname);
 		else if (unlink(pathname) < 0)
 			error("unable to unlink %s", pathname);
+		display_progress(progress, i + 1);
 	}
 	pathname[len] = 0;
 	rmdir(pathname);
@@ -39,6 +43,10 @@ void prune_packed_objects(int opts)
 	const char *dir = get_object_directory();
 	int len = strlen(dir);
 
+	if (opts == VERBOSE)
+		progress = start_progress_delay("Removing duplicate objects",
+			256, 95, 2);
+
 	if (len > PATH_MAX - 42)
 		die("impossible object directory");
 	memcpy(pathname, dir, len);
@@ -49,16 +57,12 @@ void prune_packed_objects(int opts)
 
 		sprintf(pathname + len, "%02x/", i);
 		d = opendir(pathname);
-		if (opts == VERBOSE && (d || i == 255))
-			fprintf(stderr, "Removing unused objects %d%%...\015",
-				((i+1) * 100) / 256);
 		if (!d)
 			continue;
 		prune_dir(i, d, pathname, len + 3, opts);
 		closedir(d);
 	}
-	if (opts == VERBOSE)
-		fprintf(stderr, "\nDone.\n");
+	stop_progress(&progress);
 }
 
 int cmd_prune_packed(int argc, const char **argv, const char *prefix)

builtin-unpack-objects.c

@@ -311,7 +311,7 @@ static void unpack_one(unsigned nr)
 static void unpack_all(void)
 {
 	int i;
-	struct progress progress;
+	struct progress *progress = NULL;
 	struct pack_header *hdr = fill(sizeof(struct pack_header));
 	unsigned nr_objects = ntohl(hdr->hdr_entries);
 
@@ -322,15 +322,13 @@ static void unpack_all(void)
 	use(sizeof(struct pack_header));
 
 	if (!quiet)
-		start_progress(&progress, "Unpacking %u objects...", "", nr_objects);
+		progress = start_progress("Unpacking objects", nr_objects);
 	obj_list = xmalloc(nr_objects * sizeof(*obj_list));
 	for (i = 0; i < nr_objects; i++) {
 		unpack_one(i);
-		if (!quiet)
-			display_progress(&progress, i + 1);
+		display_progress(progress, i + 1);
 	}
-	if (!quiet)
-		stop_progress(&progress);
+	stop_progress(&progress);
 
 	if (delta_list)
 		die("unresolved deltas left after unpacking");

csum-file.c

@@ -8,6 +8,7 @@
  * able to verify hasn't been messed with afterwards.
  */
 #include "cache.h"
+#include "progress.h"
 #include "csum-file.h"
 
 static void sha1flush(struct sha1file *f, unsigned int count)
@@ -17,6 +18,7 @@ static void sha1flush(struct sha1file *f, unsigned int count)
 	for (;;) {
 		int ret = xwrite(f->fd, buf, count);
 		if (ret > 0) {
+			display_throughput(f->tp, ret);
 			buf = (char *) buf + ret;
 			count -= ret;
 			if (count)
@@ -31,22 +33,27 @@ static void sha1flush(struct sha1file *f, unsigned int count)
 
 int sha1close(struct sha1file *f, unsigned char *result, int final)
 {
+	int fd;
 	unsigned offset = f->offset;
 	if (offset) {
 		SHA1_Update(&f->ctx, f->buffer, offset);
 		sha1flush(f, offset);
 		f->offset = 0;
 	}
-	if (!final)
-		return 0;	/* only want to flush (no checksum write, no close) */
-	SHA1_Final(f->buffer, &f->ctx);
-	if (result)
-		hashcpy(result, f->buffer);
-	sha1flush(f, 20);
-	if (close(f->fd))
-		die("%s: sha1 file error on close (%s)", f->name, strerror(errno));
+	if (final) {
+		/* write checksum and close fd */
+		SHA1_Final(f->buffer, &f->ctx);
+		if (result)
+			hashcpy(result, f->buffer);
+		sha1flush(f, 20);
+		if (close(f->fd))
+			die("%s: sha1 file error on close (%s)",
+			    f->name, strerror(errno));
+		fd = 0;
+	} else
+		fd = f->fd;
 	free(f);
-	return 0;
+	return fd;
 }
 
 int sha1write(struct sha1file *f, void *buf, unsigned int count)
@@ -74,6 +81,11 @@ int sha1write(struct sha1file *f, void *buf, unsigned int count)
 }
 
 struct sha1file *sha1fd(int fd, const char *name)
+{
+	return sha1fd_throughput(fd, name, NULL);
+}
+
+struct sha1file *sha1fd_throughput(int fd, const char *name, struct progress *tp)
 {
 	struct sha1file *f;
 	unsigned len;
@@ -89,6 +101,7 @@ struct sha1file *sha1fd(int fd, const char *name)
 	f->fd = fd;
 	f->error = 0;
 	f->offset = 0;
+	f->tp = tp;
 	f->do_crc = 0;
 	SHA1_Init(&f->ctx);
 	return f;

csum-file.h

@@ -1,11 +1,14 @@
 #ifndef CSUM_FILE_H
 #define CSUM_FILE_H
 
+struct progress;
+
 /* A SHA1-protected file */
 struct sha1file {
 	int fd, error;
 	unsigned int offset, namelen;
 	SHA_CTX ctx;
+	struct progress *tp;
 	char name[PATH_MAX];
 	int do_crc;
 	uint32_t crc32;
@@ -13,6 +16,7 @@ struct sha1file {
 };
 
 extern struct sha1file *sha1fd(int fd, const char *name);
+extern struct sha1file *sha1fd_throughput(int fd, const char *name, struct progress *tp);
 extern int sha1close(struct sha1file *, unsigned char *, int);
 extern int sha1write(struct sha1file *, void *, unsigned int);
 extern void crc32_begin(struct sha1file *);

git-repack.sh

@@ -83,9 +83,6 @@ for name in $names ; do
 	fullbases="$fullbases pack-$name"
 	chmod a-w "$PACKTMP-$name.pack"
 	chmod a-w "$PACKTMP-$name.idx"
-	if test "$quiet" != '-q'; then
-	    echo "Pack pack-$name created."
-	fi
 	mkdir -p "$PACKDIR" || exit
 
 	for sfx in pack idx

index-pack.c

@@ -46,7 +46,7 @@ static int nr_resolved_deltas;
 static int from_stdin;
 static int verbose;
 
-static struct progress progress;
+static struct progress *progress;
 
 /* We always read in 4kB chunks. */
 static unsigned char input_buffer[4096];
@@ -87,6 +87,8 @@ static void *fill(int min)
 				die("early EOF");
 			die("read error on input: %s", strerror(errno));
 		}
+		if (from_stdin)
+			display_throughput(progress, ret);
 		input_len += ret;
 	} while (input_len < min);
 	return input_buffer;
@@ -106,7 +108,7 @@ static void use(int bytes)
 	consumed_bytes += bytes;
 }
 
-static const char *open_pack_file(const char *pack_name)
+static char *open_pack_file(char *pack_name)
 {
 	if (from_stdin) {
 		input_fd = 0;
@@ -406,7 +408,9 @@ static void parse_pack_objects(unsigned char *sha1)
 	 * - remember base (SHA1 or offset) for all deltas.
 	 */
 	if (verbose)
-		start_progress(&progress, "Indexing %u objects...", "", nr_objects);
+		progress = start_progress(
+				from_stdin ? "Receiving objects" : "Indexing objects",
+				nr_objects);
 	for (i = 0; i < nr_objects; i++) {
 		struct object_entry *obj = &objects[i];
 		data = unpack_raw_entry(obj, &delta->base);
@@ -418,12 +422,10 @@ static void parse_pack_objects(unsigned char *sha1)
 		} else
 			sha1_object(data, obj->size, obj->type, obj->idx.sha1);
 		free(data);
-		if (verbose)
-			display_progress(&progress, i+1);
+		display_progress(progress, i+1);
 	}
 	objects[i].idx.offset = consumed_bytes;
-	if (verbose)
-		stop_progress(&progress);
+	stop_progress(&progress);
 
 	/* Check pack integrity */
 	flush();
@@ -455,7 +457,7 @@ static void parse_pack_objects(unsigned char *sha1)
 	 *   for some more deltas.
 	 */
 	if (verbose)
-		start_progress(&progress, "Resolving %u deltas...", "", nr_deltas);
+		progress = start_progress("Resolving deltas", nr_deltas);
 	for (i = 0; i < nr_objects; i++) {
 		struct object_entry *obj = &objects[i];
 		union delta_base base;
@@ -486,8 +488,7 @@ static void parse_pack_objects(unsigned char *sha1)
 						      obj->size, obj->type);
 			}
 		free(data);
-		if (verbose)
-			display_progress(&progress, nr_resolved_deltas);
+		display_progress(progress, nr_resolved_deltas);
 	}
 }
 
@@ -594,8 +595,7 @@ static void fix_unresolved_deltas(int nr_unresolved)
 			die("local object %s is corrupt", sha1_to_hex(d->base.sha1));
 		append_obj_to_pack(d->base.sha1, data, size, type);
 		free(data);
-		if (verbose)
-			display_progress(&progress, nr_resolved_deltas);
+		display_progress(progress, nr_resolved_deltas);
 	}
 	free(sorted_by_pos);
 }
@@ -686,15 +686,15 @@ static void final(const char *final_pack_name, const char *curr_pack_name,
 int main(int argc, char **argv)
 {
 	int i, fix_thin_pack = 0;
-	const char *curr_pack, *pack_name = NULL;
-	const char *curr_index, *index_name = NULL;
+	char *curr_pack, *pack_name = NULL;
+	char *curr_index, *index_name = NULL;
 	const char *keep_name = NULL, *keep_msg = NULL;
 	char *index_name_buf = NULL, *keep_name_buf = NULL;
 	struct pack_idx_entry **idx_objects;
 	unsigned char sha1[20];
 
 	for (i = 1; i < argc; i++) {
-		const char *arg = argv[i];
+		char *arg = argv[i];
 
 		if (*arg == '-') {
 			if (!strcmp(arg, "--stdin")) {
@@ -774,8 +774,7 @@ int main(int argc, char **argv)
 	deltas = xmalloc(nr_objects * sizeof(struct delta_entry));
 	parse_pack_objects(sha1);
 	if (nr_deltas == nr_resolved_deltas) {
-		if (verbose)
-			stop_progress(&progress);
+		stop_progress(&progress);
 		/* Flush remaining pack final 20-byte SHA1. */
 		flush();
 	} else {
@@ -788,11 +787,10 @@ int main(int argc, char **argv)
 					   (nr_objects + nr_unresolved + 1)
 					   * sizeof(*objects));
 			fix_unresolved_deltas(nr_unresolved);
-			if (verbose) {
-				stop_progress(&progress);
+			stop_progress(&progress);
+			if (verbose)
 				fprintf(stderr, "%d objects were added to complete this thin pack.\n",
 					nr_objects - nr_objects_initial);
-			}
 			fixup_pack_header_footer(output_fd, sha1,
 				curr_pack, nr_objects);
 		}
@@ -815,6 +813,10 @@ int main(int argc, char **argv)
 	free(objects);
 	free(index_name_buf);
 	free(keep_name_buf);
+	if (pack_name == NULL)
+		free(curr_pack);
+	if (index_name == NULL)
+		free(curr_index);
 
 	return 0;
 }

pack-write.c

@@ -17,7 +17,8 @@ static int sha1_compare(const void *_a, const void *_b)
  * the SHA1 hash of sorted object names. The objects array passed in
  * will be sorted by SHA1 on exit.
  */
-const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects, int nr_objects, unsigned char *sha1)
+char *write_idx_file(char *index_name, struct pack_idx_entry **objects,
+		     int nr_objects, unsigned char *sha1)
 {
 	struct sha1file *f;
 	struct pack_idx_entry **sorted_by_sha, **list, **last;

pack.h

@@ -55,7 +55,7 @@ struct pack_idx_entry {
 	off_t offset;
 };
 
-extern const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects, int nr_objects, unsigned char *sha1);
+extern char *write_idx_file(char *index_name, struct pack_idx_entry **objects, int nr_objects, unsigned char *sha1);
 
 extern int verify_pack(struct packed_git *, int);
 extern void fixup_pack_header_footer(int, unsigned char *, const char *, uint32_t);

progress.c

@@ -1,6 +1,40 @@
+/*
+ * Simple text-based progress display module for GIT
+ *
+ * Copyright (c) 2007 by Nicolas Pitre <nico@cam.org>
+ *
+ * This code is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
 #include "git-compat-util.h"
 #include "progress.h"
 
+#define TP_IDX_MAX      8
+
+struct throughput {
+	struct timeval prev_tv;
+	off_t total;
+	unsigned long count;
+	unsigned long avg_bytes;
+	unsigned long last_bytes[TP_IDX_MAX];
+	unsigned int avg_misecs;
+	unsigned int last_misecs[TP_IDX_MAX];
+	unsigned int idx;
+	char display[32];
+};
+
+struct progress {
+	const char *title;
+	int last_value;
+	unsigned total;
+	unsigned last_percent;
+	unsigned delay;
+	unsigned delayed_percent_treshold;
+	struct throughput *throughput;
+};
+
 static volatile sig_atomic_t progress_update;
 
 static void progress_interval(int signum)
@@ -35,10 +69,11 @@ static void clear_progress_signal(void)
 	progress_update = 0;
 }
 
-int display_progress(struct progress *progress, unsigned n)
+static int display(struct progress *progress, unsigned n, int done)
 {
+	char *eol, *tp;
+
 	if (progress->delay) {
-		char buf[80];
 		if (!progress_update || --progress->delay)
 			return 0;
 		if (progress->total) {
@@ -51,60 +86,151 @@ int display_progress(struct progress *progress, unsigned n)
 				return 0;
 			}
 		}
-		if (snprintf(buf, sizeof(buf),
-			     progress->delayed_title, progress->total))
-			fprintf(stderr, "%s\n", buf);
 	}
+
+	progress->last_value = n;
+	tp = (progress->throughput) ? progress->throughput->display : "";
+	eol = done ? ", done.   \n" : "   \r";
 	if (progress->total) {
 		unsigned percent = n * 100 / progress->total;
 		if (percent != progress->last_percent || progress_update) {
 			progress->last_percent = percent;
-			fprintf(stderr, "%s%4u%% (%u/%u) done\r",
-				progress->prefix, percent, n, progress->total);
+			fprintf(stderr, "%s: %3u%% (%u/%u)%s%s",
+				progress->title, percent, n,
+				progress->total, tp, eol);
 			progress_update = 0;
-			progress->need_lf = 1;
 			return 1;
 		}
 	} else if (progress_update) {
-		fprintf(stderr, "%s%u\r", progress->prefix, n);
+		fprintf(stderr, "%s: %u%s%s", progress->title, n, tp, eol);
 		progress_update = 0;
-		progress->need_lf = 1;
 		return 1;
 	}
+
 	return 0;
 }
 
-void start_progress(struct progress *progress, const char *title,
-		    const char *prefix, unsigned total)
+void display_throughput(struct progress *progress, unsigned long n)
 {
-	char buf[80];
-	progress->prefix = prefix;
-	progress->total = total;
-	progress->last_percent = -1;
-	progress->delay = 0;
-	progress->need_lf = 0;
-	if (snprintf(buf, sizeof(buf), title, total))
-		fprintf(stderr, "%s\n", buf);
-	set_progress_signal();
+	struct throughput *tp;
+	struct timeval tv;
+	unsigned int misecs;
+
+	if (!progress)
+		return;
+	tp = progress->throughput;
+
+	gettimeofday(&tv, NULL);
+
+	if (!tp) {
+		progress->throughput = tp = calloc(1, sizeof(*tp));
+		if (tp)
+			tp->prev_tv = tv;
+		return;
+	}
+
+	tp->total += n;
+	tp->count += n;
+
+	/*
+	 * We have x = bytes and y = microsecs.  We want z = KiB/s:
+	 *
+	 *	z = (x / 1024) / (y / 1000000)
+	 *	z = x / y * 1000000 / 1024
+	 *	z = x / (y * 1024 / 1000000)
+	 *	z = x / y'
+	 *
+	 * To simplify things we'll keep track of misecs, or 1024th of a sec
+	 * obtained with:
+	 *
+	 *	y' = y * 1024 / 1000000
+	 *	y' = y / (1000000 / 1024)
+	 *	y' = y / 977
+	 */
+	misecs = (tv.tv_sec - tp->prev_tv.tv_sec) * 1024;
+	misecs += (int)(tv.tv_usec - tp->prev_tv.tv_usec) / 977;
+
+	if (misecs > 512) {
+		int l = sizeof(tp->display);
+		tp->prev_tv = tv;
+		tp->avg_bytes += tp->count;
+		tp->avg_misecs += misecs;
+
+		if (tp->total > 1 << 30) {
+			l -= snprintf(tp->display, l, ", %u.%2.2u GiB",
+				      (int)(tp->total >> 30),
+				      (int)(tp->total & ((1 << 30) - 1)) / 10737419);
+		} else if (tp->total > 1 << 20) {
+			l -= snprintf(tp->display, l, ", %u.%2.2u MiB",
+				      (int)(tp->total >> 20),
+				      ((int)(tp->total & ((1 << 20) - 1))
+				       * 100) >> 20);
+		} else if (tp->total > 1 << 10) {
+			l -= snprintf(tp->display, l, ", %u.%2.2u KiB",
+				      (int)(tp->total >> 10),
+				      ((int)(tp->total & ((1 << 10) - 1))
+				       * 100) >> 10);
+		} else {
+			l -= snprintf(tp->display, l, ", %u bytes",
+				      (int)tp->total);
+		}
+		snprintf(tp->display + sizeof(tp->display) - l, l,
+			 " | %lu KiB/s", tp->avg_bytes / tp->avg_misecs);
+
+		tp->avg_bytes -= tp->last_bytes[tp->idx];
+		tp->avg_misecs -= tp->last_misecs[tp->idx];
+		tp->last_bytes[tp->idx] = tp->count;
+		tp->last_misecs[tp->idx] = misecs;
+		tp->idx = (tp->idx + 1) % TP_IDX_MAX;
+		tp->count = 0;
+
+		if (progress->last_value != -1 && progress_update)
+			display(progress, progress->last_value, 0);
+	}
+}
+
+int display_progress(struct progress *progress, unsigned n)
+{
+	return progress ? display(progress, n, 0) : 0;
 }
 
-void start_progress_delay(struct progress *progress, const char *title,
-			  const char *prefix, unsigned total,
-			  unsigned percent_treshold, unsigned delay)
+struct progress *start_progress_delay(const char *title, unsigned total,
+				       unsigned percent_treshold, unsigned delay)
 {
-	progress->prefix = prefix;
+	struct progress *progress = malloc(sizeof(*progress));
+	if (!progress) {
+		/* unlikely, but here's a good fallback */
+		fprintf(stderr, "%s...\n", title);
+		return NULL;
+	}
+	progress->title = title;
 	progress->total = total;
+	progress->last_value = -1;
 	progress->last_percent = -1;
 	progress->delayed_percent_treshold = percent_treshold;
-	progress->delayed_title = title;
 	progress->delay = delay;
-	progress->need_lf = 0;
+	progress->throughput = NULL;
 	set_progress_signal();
+	return progress;
 }
 
-void stop_progress(struct progress *progress)
+struct progress *start_progress(const char *title, unsigned total)
 {
+	return start_progress_delay(title, total, 0, 0);
+}
+
+void stop_progress(struct progress **p_progress)
+{
+	struct progress *progress = *p_progress;
+	if (!progress)
+		return;
+	*p_progress = NULL;
+	if (progress->last_value != -1) {
+		/* Force the last update */
+		progress_update = 1;
+		display(progress, progress->last_value, 1);
+	}
 	clear_progress_signal();
-	if (progress->need_lf)
-		fputc('\n', stderr);
+	free(progress->throughput);
+	free(progress);
 }

progress.h

@@ -1,22 +1,13 @@
 #ifndef PROGRESS_H
 #define PROGRESS_H
 
-struct progress {
-	const char *prefix;
-	unsigned total;
-	unsigned last_percent;
-	unsigned delay;
-	unsigned delayed_percent_treshold;
-	const char *delayed_title;
-	int need_lf;
-};
+struct progress;
 
+void display_throughput(struct progress *progress, unsigned long n);
 int display_progress(struct progress *progress, unsigned n);
-void start_progress(struct progress *progress, const char *title,
-		    const char *prefix, unsigned total);
-void start_progress_delay(struct progress *progress, const char *title,
-			  const char *prefix, unsigned total,
-			  unsigned percent_treshold, unsigned delay);
-void stop_progress(struct progress *progress);
+struct progress *start_progress(const char *title, unsigned total);
+struct progress *start_progress_delay(const char *title, unsigned total,
+				       unsigned percent_treshold, unsigned delay);
+void stop_progress(struct progress **progress);
 
 #endif

sideband.c

@@ -17,7 +17,7 @@ int recv_sideband(const char *me, int in_stream, int out, int err)
 	strcpy(buf, "remote:");
 	while (1) {
 		int band, len;
-		len	= packet_read_line(in_stream, buf+7, LARGE_PACKET_MAX);
+		len = packet_read_line(in_stream, buf+7, LARGE_PACKET_MAX);
 		if (len == 0)
 			break;
 		if (len < 1) {
@@ -35,7 +35,22 @@ int recv_sideband(const char *me, int in_stream, int out, int err)
 			return SIDEBAND_REMOTE_ERROR;
 		case 2:
 			buf[7] = ' ';
-			safe_write(err, buf, 8+len);
+			len += 8;
+			while (1) {
+				int brk = 8;
+				while (brk < len) {
+					brk++;
+					if (buf[brk-1] == '\n' ||
+					    buf[brk-1] == '\r')
+						break;
+				}
+				safe_write(err, buf, brk);
+				if (brk < len) {
+					memmove(buf + 8, buf + brk, len - brk);
+					len = len - brk + 8;
+				} else
+					break;
+			}
 			continue;
 		case 1:
 			safe_write(out, buf+8, len);

unpack-trees.c

@@ -297,7 +297,7 @@ static void check_updates(struct cache_entry **src, int nr,
 {
 	unsigned short mask = htons(CE_UPDATE);
 	unsigned cnt = 0, total = 0;
-	struct progress progress;
+	struct progress *progress = NULL;
 	char last_symlink[PATH_MAX];
 
 	if (o->update && o->verbose_update) {
@@ -307,8 +307,8 @@ static void check_updates(struct cache_entry **src, int nr,
 				total++;
 		}
 
-		start_progress_delay(&progress, "Checking %u files out...",
-				     "", total, 50, 2);
+		progress = start_progress_delay("Checking out files",
+						total, 50, 2);
 		cnt = 0;
 	}
 
@@ -316,9 +316,8 @@ static void check_updates(struct cache_entry **src, int nr,
 	while (nr--) {
 		struct cache_entry *ce = *src++;
 
-		if (total)
-			if (!ce->ce_mode || ce->ce_flags & mask)
-				display_progress(&progress, ++cnt);
+		if (!ce->ce_mode || ce->ce_flags & mask)
+			display_progress(progress, ++cnt);
 		if (!ce->ce_mode) {
 			if (o->update)
 				unlink_entry(ce->name, last_symlink);
@@ -332,8 +331,7 @@ static void check_updates(struct cache_entry **src, int nr,
 			}
 		}
 	}
-	if (total)
-		stop_progress(&progress);;
+	stop_progress(&progress);
 }
 
 int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options *o)