odb/streaming: track write stream size in the structure

When passing around a `struct odb_write_stream` we typically also have
to pass the number of bytes that the stream will yield. This is required
because the object header itself contains that size, and consequently we
cannot write the header without that information.

Move this information into the stream itself so that it becomes self-
describing. In addition to that, this also brings the `struct
odb_write_stream` a bit closer to the `struct odb_read_stream` so that
we can eventually merge both stream types.

Signed-off-by: Patrick Steinhardt <ps@pks.im>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

builtin/unpack-objects.c

@@ -392,13 +392,14 @@ static void stream_blob(unsigned long size, unsigned nr)
 	struct odb_write_stream in_stream = {
 		.read = feed_input_zstream,
 		.data = &data,
+		.size = size,
 	};
 	struct obj_info *info = &obj_list[nr];
 
 	data.zstream = &zstream;
 	git_inflate_init(&zstream);
 
-	if (odb_write_object_stream(the_repository->objects, &in_stream, size, &info->oid))
+	if (odb_write_object_stream(the_repository->objects, &in_stream, &info->oid))
 		die(_("failed to write object in stream"));
 
 	if (data.status != Z_STREAM_END)

object-file.c

@@ -704,7 +704,7 @@ static void prepare_packfile_transaction(struct odb_transaction_files *transacti
 
 static int hash_blob_stream(struct odb_write_stream *stream,
 			    const struct git_hash_algo *hash_algo,
-			    struct object_id *result_oid, size_t size)
+			    struct object_id *result_oid)
 {
 	unsigned char buf[16384];
 	struct git_hash_ctx ctx;
@@ -712,7 +712,7 @@ static int hash_blob_stream(struct odb_write_stream *stream,
 	size_t bytes_hashed = 0;
 
 	header_len = format_object_header((char *)buf, sizeof(buf),
-					  OBJ_BLOB, size);
+					  OBJ_BLOB, stream->size);
 	git_hash_init(&ctx, hash_algo);
 	git_hash_update(&ctx, buf, header_len);
 
@@ -727,7 +727,7 @@ static int hash_blob_stream(struct odb_write_stream *stream,
 		bytes_hashed += read_result;
 	}
 
-	if (bytes_hashed != size)
+	if (bytes_hashed != stream->size)
 		return -1;
 
 	git_hash_final_oid(result_oid, &ctx);
@@ -740,7 +740,7 @@ static int hash_blob_stream(struct odb_write_stream *stream,
  * packfile in state while updating the hash in ctx.
  */
 static void stream_blob_to_pack(struct transaction_packfile *state,
-				struct git_hash_ctx *ctx, size_t size,
+				struct git_hash_ctx *ctx,
 				struct odb_write_stream *stream)
 {
 	git_zstream s;
@@ -753,7 +753,7 @@ static void stream_blob_to_pack(struct transaction_packfile *state,
 
 	git_deflate_init(&s, cfg->pack_compression_level);
 
-	hdrlen = encode_in_pack_object_header(obuf, sizeof(obuf), OBJ_BLOB, size);
+	hdrlen = encode_in_pack_object_header(obuf, sizeof(obuf), OBJ_BLOB, stream->size);
 	s.next_out = obuf + hdrlen;
 	s.avail_out = sizeof(obuf) - hdrlen;
 
@@ -793,9 +793,9 @@ static void stream_blob_to_pack(struct transaction_packfile *state,
 		}
 	}
 
-	if (bytes_read != size)
+	if (bytes_read != stream->size)
 		die("read %" PRIuMAX " bytes of blob data, but expected %" PRIuMAX " bytes",
-		    (uintmax_t)bytes_read, (uintmax_t)size);
+		    (uintmax_t)bytes_read, (uintmax_t)stream->size);
 
 	git_deflate_end(&s);
 }
@@ -870,7 +870,6 @@ clear_exit:
  */
 static int odb_transaction_files_write_object_stream(struct odb_transaction *base,
 						     struct odb_write_stream *stream,
-						     size_t size,
 						     struct object_id *result_oid)
 {
 	struct odb_transaction_files *transaction = container_of(base,
@@ -884,7 +883,7 @@ static int odb_transaction_files_write_object_stream(struct odb_transaction *bas
 	struct pack_idx_entry *idx;
 
 	header_len = format_object_header((char *)obuf, sizeof(obuf),
-					  OBJ_BLOB, size);
+					  OBJ_BLOB, stream->size);
 	git_hash_init(&ctx, transaction->base.source->odb->repo->hash_algo);
 	git_hash_update(&ctx, obuf, header_len);
 
@@ -899,7 +898,7 @@ static int odb_transaction_files_write_object_stream(struct odb_transaction *bas
 	 * to zlib compression and is sufficient for this check.
 	 */
 	if (state->nr_written && pack_size_limit_cfg &&
-	    pack_size_limit_cfg < state->offset + size)
+	    pack_size_limit_cfg < state->offset + stream->size)
 		flush_packfile_transaction(transaction);
 
 	CALLOC_ARRAY(idx, 1);
@@ -909,7 +908,7 @@ static int odb_transaction_files_write_object_stream(struct odb_transaction *bas
 	hashfile_checkpoint(state->f, &checkpoint);
 	idx->offset = state->offset;
 	crc32_begin(state->f);
-	stream_blob_to_pack(state, &ctx, size, stream);
+	stream_blob_to_pack(state, &ctx, stream);
 	git_hash_final_oid(result_oid, &ctx);
 
 	idx->crc32 = crc32_end(state->f);
@@ -962,14 +961,12 @@ int index_fd(struct index_state *istate, struct object_id *oid,
 				odb_transaction_begin_or_die(odb, &transaction, 0);
 			ret = odb_transaction_write_object_stream(transaction,
 								  &stream,
-								  xsize_t(st->st_size),
 								  oid);
 			if (!inflight)
 				odb_transaction_commit(transaction);
 		} else {
 			ret = hash_blob_stream(&stream,
-					       the_repository->hash_algo, oid,
-					       xsize_t(st->st_size));
+					       the_repository->hash_algo, oid);
 		}
 
 		odb_write_stream_release(&stream);

odb.c

@@ -1028,10 +1028,10 @@ int odb_write_object_ext(struct object_database *odb,
 }
 
 int odb_write_object_stream(struct object_database *odb,
-			    struct odb_write_stream *stream, size_t len,
+			    struct odb_write_stream *stream,
 			    struct object_id *oid)
 {
-	return odb_source_write_object_stream(odb->sources, stream, len, oid);
+	return odb_source_write_object_stream(odb->sources, stream, oid);
 }
 
 struct object_database *odb_new(struct repository *repo,

odb.h

@@ -629,7 +629,7 @@ static inline int odb_write_object(struct object_database *odb,
 struct odb_write_stream;
 
 int odb_write_object_stream(struct object_database *odb,
-			    struct odb_write_stream *stream, size_t len,
+			    struct odb_write_stream *stream,
 			    struct object_id *oid);
 
 void parse_alternates(const char *string,

odb/source-files.c

@@ -175,11 +175,10 @@ static int odb_source_files_write_object(struct odb_source *source,
 
 static int odb_source_files_write_object_stream(struct odb_source *source,
 						struct odb_write_stream *stream,
-						size_t len,
 						struct object_id *oid)
 {
 	struct odb_source_files *files = odb_source_files_downcast(source);
-	return odb_source_write_object_stream(&files->loose->base, stream, len, oid);
+	return odb_source_write_object_stream(&files->loose->base, stream, oid);
 }
 
 static int odb_source_files_begin_transaction(struct odb_source *source,

odb/source-inmemory.c

@@ -257,7 +257,6 @@ static int odb_source_inmemory_write_object(struct odb_source *source,
 
 static int odb_source_inmemory_write_object_stream(struct odb_source *source,
 						   struct odb_write_stream *stream,
-						   size_t len,
 						   struct object_id *oid)
 {
 	char buf[16384];
@@ -265,12 +264,12 @@ static int odb_source_inmemory_write_object_stream(struct odb_source *source,
 	char *data;
 	int ret;
 
-	CALLOC_ARRAY(data, len);
+	CALLOC_ARRAY(data, stream->size);
 	while (!stream->is_finished) {
 		ssize_t bytes_read;
 
 		bytes_read = odb_write_stream_read(stream, buf, sizeof(buf));
-		if (total_read + bytes_read > len) {
+		if (total_read + bytes_read > stream->size) {
 			ret = error("object stream yielded more bytes than expected");
 			goto out;
 		}
@@ -279,15 +278,15 @@ static int odb_source_inmemory_write_object_stream(struct odb_source *source,
 		total_read += bytes_read;
 	}
 
-	if (total_read != len) {
+	if (total_read != stream->size) {
 		ret = error("object stream yielded less bytes than expected");
 		goto out;
 	}
 
 	hash_object_file(source->odb->repo->hash_algo, data, total_read, OBJ_BLOB, oid);
 
-	ret = odb_source_inmemory_write_object(source, data, len, OBJ_BLOB, oid,
-					       NULL, NULL, 0);
+	ret = odb_source_inmemory_write_object(source, data, stream->size,
+					       OBJ_BLOB, oid, NULL, NULL, 0);
 	if (ret < 0)
 		goto out;
 

odb/source-loose.c

@@ -846,7 +846,6 @@ static int odb_source_loose_write_object(struct odb_source *source,
 
 static int odb_source_loose_write_object_stream(struct odb_source *source,
 						struct odb_write_stream *in_stream,
-						size_t len,
 						struct object_id *oid)
 {
 	struct odb_source_loose *loose = odb_source_loose_downcast(source);
@@ -868,7 +867,7 @@ static int odb_source_loose_write_object_stream(struct odb_source *source,
 
 	/* Since oid is not determined, save tmp file to odb path. */
 	strbuf_addf(&filename, "%s/", loose->base.path);
-	hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len);
+	hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, in_stream->size);
 
 	/*
 	 * Common steps for write_loose_object and stream_loose_object to
@@ -916,9 +915,9 @@ static int odb_source_loose_write_object_stream(struct odb_source *source,
 		 */
 	} while (ret == Z_OK || ret == Z_BUF_ERROR);
 
-	if (stream.total_in != len + hdrlen)
+	if (stream.total_in != in_stream->size + hdrlen)
 		die(_("write stream object %"PRIuMAX" != %"PRIuMAX), (uintmax_t)stream.total_in,
-		    (uintmax_t)len + hdrlen);
+		    (uintmax_t)in_stream->size + hdrlen);
 
 	/*
 	 * Common steps for write_loose_object and stream_loose_object to

odb/source-packed.c

@@ -610,7 +610,6 @@ static int odb_source_packed_write_object(struct odb_source *source UNUSED,
 
 static int odb_source_packed_write_object_stream(struct odb_source *source UNUSED,
 						 struct odb_write_stream *stream UNUSED,
-						 size_t len UNUSED,
 						 struct object_id *oid UNUSED)
 {
 	return error("packed backend cannot write object streams");

odb/source.h

@@ -221,7 +221,7 @@ struct odb_source {
 	 * otherwise.
 	 */
 	int (*write_object_stream)(struct odb_source *source,
-				   struct odb_write_stream *stream, size_t len,
+				   struct odb_write_stream *stream,
 				   struct object_id *oid);
 
 	/*
@@ -437,10 +437,9 @@ static inline int odb_source_write_object(struct odb_source *source,
  */
 static inline int odb_source_write_object_stream(struct odb_source *source,
 						 struct odb_write_stream *stream,
-						 size_t len,
 						 struct object_id *oid)
 {
-	return source->write_object_stream(source, stream, len, oid);
+	return source->write_object_stream(source, stream, oid);
 }
 
 /*

odb/streaming.c

@@ -336,5 +336,6 @@ void odb_write_stream_from_fd(struct odb_write_stream *stream, int fd,
 
 	stream->data = data;
 	stream->read = read_object_fd;
+	stream->size = size;
 	stream->is_finished = 0;
 }

odb/streaming.h

@@ -55,6 +55,7 @@ ssize_t odb_read_stream_read(struct odb_read_stream *stream, void *buf, size_t l
 struct odb_write_stream {
 	ssize_t (*read)(struct odb_write_stream *, unsigned char *, size_t);
 	void *data;
+	size_t size;
 	int is_finished;
 };
 

odb/transaction.c

@@ -40,9 +40,9 @@ int odb_transaction_commit(struct odb_transaction *transaction)
 
 int odb_transaction_write_object_stream(struct odb_transaction *transaction,
 					struct odb_write_stream *stream,
-					size_t len, struct object_id *oid)
+					struct object_id *oid)
 {
-	return transaction->write_object_stream(transaction, stream, len, oid);
+	return transaction->write_object_stream(transaction, stream, oid);
 }
 
 int odb_transaction_env(struct odb_transaction *transaction, struct strvec *env)

odb/transaction.h

@@ -31,7 +31,7 @@ struct odb_transaction {
 	 * otherwise.
 	 */
 	int (*write_object_stream)(struct odb_transaction *transaction,
-				   struct odb_write_stream *stream, size_t len,
+				   struct odb_write_stream *stream,
 				   struct object_id *oid);
 
 	/*
@@ -82,7 +82,7 @@ int odb_transaction_commit(struct odb_transaction *transaction);
  */
 int odb_transaction_write_object_stream(struct odb_transaction *transaction,
 					struct odb_write_stream *stream,
-					size_t len, struct object_id *oid);
+					struct object_id *oid);
 
 /*
  * Populates the provided strvec with the environment variables that a child

t/unit-tests/u-odb-inmemory.c

@@ -269,7 +269,6 @@ struct membuf_write_stream {
 	struct odb_write_stream base;
 	const char *buf;
 	size_t offset;
-	size_t size;
 };
 
 static ssize_t membuf_write_stream_read(struct odb_write_stream *stream,
@@ -280,13 +279,13 @@ static ssize_t membuf_write_stream_read(struct odb_write_stream *stream,
 
 	if (chunk_size > len)
 		chunk_size = len;
-	if (chunk_size > s->size - s->offset)
-		chunk_size = s->size - s->offset;
+	if (chunk_size > s->base.size - s->offset)
+		chunk_size = s->base.size - s->offset;
 
 	memcpy(buf, s->buf + s->offset, chunk_size);
 
 	s->offset += chunk_size;
-	if (s->offset == s->size)
+	if (s->offset == s->base.size)
 		s->base.is_finished = 1;
 
 	return chunk_size;
@@ -298,13 +297,13 @@ void test_odb_inmemory__write_object_stream(void)
 	const char data[] = "foobar";
 	struct membuf_write_stream stream = {
 		.base.read = membuf_write_stream_read,
+		.base.size = strlen(data),
 		.buf = data,
-		.size = strlen(data),
 	};
 	struct object_id written_oid;
 
 	cl_must_pass(odb_source_write_object_stream(&source->base, &stream.base,
-						    strlen(data), &written_oid));
+						    &written_oid));
 	cl_assert_equal_s(oid_to_hex(&written_oid), FOOBAR_OID);
 	cl_assert_object_info(source, &written_oid, OBJ_BLOB, "foobar");