Merge branch 'ps/object-file-cleanup' into ps/object-store-cleanup

* ps/object-file-cleanup:
  object-store: merge "object-store-ll.h" and "object-store.h"
  object-store: remove global array of cached objects
  object: split out functions relating to object store subsystem
  object-file: drop `index_blob_stream()`
  object-file: split up concerns of `HASH_*` flags
  object-file: split out functions relating to object store subsystem
  object-file: move `xmmap()` into "wrapper.c"
  object-file: move `git_open_cloexec()` to "compat/open.c"
  object-file: move `safe_create_leading_directories()` into "path.c"
  object-file: move `mkdir_in_gitdir()` into "path.c"

Makefile

@@ -994,6 +994,7 @@ LIB_OBJS += common-exit.o
 LIB_OBJS += common-init.o
 LIB_OBJS += compat/nonblock.o
 LIB_OBJS += compat/obstack.o
+LIB_OBJS += compat/open.o
 LIB_OBJS += compat/terminal.o
 LIB_OBJS += compiler-tricks/not-constant.o
 LIB_OBJS += config.o
@@ -1084,6 +1085,7 @@ LIB_OBJS += notes.o
 LIB_OBJS += object-file-convert.o
 LIB_OBJS += object-file.o
 LIB_OBJS += object-name.o
+LIB_OBJS += object-store.o
 LIB_OBJS += object.o
 LIB_OBJS += oid-array.o
 LIB_OBJS += oidmap.o
@@ -1811,7 +1813,6 @@ ifdef FREAD_READS_DIRECTORIES
 endif
 ifdef OPEN_RETURNS_EINTR
 	COMPAT_CFLAGS += -DOPEN_RETURNS_EINTR
-	COMPAT_OBJS += compat/open.o
 endif
 ifdef NO_SYMLINK_HEAD
 	BASIC_CFLAGS += -DNO_SYMLINK_HEAD

apply.c

@@ -14,7 +14,7 @@
 #include "abspath.h"
 #include "base85.h"
 #include "config.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "delta.h"
 #include "diff.h"
 #include "dir.h"

archive-tar.c

@@ -11,7 +11,7 @@
 #include "hex.h"
 #include "tar.h"
 #include "archive.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "strbuf.h"
 #include "streaming.h"
 #include "run-command.h"

archive-zip.c

@@ -12,7 +12,7 @@
 #include "hex.h"
 #include "streaming.h"
 #include "utf8.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "strbuf.h"
 #include "userdiff.h"
 #include "write-or-die.h"

archive.c

@@ -14,7 +14,7 @@
 #include "pretty.h"
 #include "setup.h"
 #include "refs.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "commit.h"
 #include "tree.h"
 #include "tree-walk.h"

attr.c

@@ -22,7 +22,7 @@
 #include "read-cache-ll.h"
 #include "refs.h"
 #include "revision.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "setup.h"
 #include "thread-utils.h"
 #include "tree-walk.h"

bisect.c

@@ -20,7 +20,7 @@
 #include "commit-slab.h"
 #include "commit-reach.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "path.h"
 #include "dir.h"
 

blame.c

@@ -3,7 +3,7 @@
 
 #include "git-compat-util.h"
 #include "refs.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "cache-tree.h"
 #include "mergesort.h"
 #include "commit.h"
@@ -277,7 +277,7 @@ static struct commit *fake_working_tree_commit(struct repository *r,
 	convert_to_git(r->index, path, buf.buf, buf.len, &buf, 0);
 	origin->file.ptr = buf.buf;
 	origin->file.size = buf.len;
-	pretend_object_file(buf.buf, buf.len, OBJ_BLOB, &origin->blob_oid);
+	pretend_object_file(the_repository, buf.buf, buf.len, OBJ_BLOB, &origin->blob_oid);
 
 	/*
 	 * Read the current index, replace the path entry with

builtin/backfill.c

@@ -13,7 +13,7 @@
 #include "tree.h"
 #include "tree-walk.h"
 #include "object.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "oid-array.h"
 #include "oidset.h"
 #include "promisor-remote.h"

builtin/blame.c

@@ -28,7 +28,7 @@
 #include "line-log.h"
 #include "progress.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "pager.h"
 #include "blame.h"
 #include "refs.h"

builtin/bugreport.c

@@ -4,13 +4,13 @@
 #include "editor.h"
 #include "gettext.h"
 #include "parse-options.h"
+#include "path.h"
 #include "strbuf.h"
 #include "help.h"
 #include "compat/compiler.h"
 #include "hook.h"
 #include "hook-list.h"
 #include "diagnose.h"
-#include "object-file.h"
 #include "setup.h"
 #include "version.h"
 
@@ -141,7 +141,7 @@ int cmd_bugreport(int argc,
 	}
 	strbuf_addstr(&report_path, ".txt");
 
-	switch (safe_create_leading_directories(report_path.buf)) {
+	switch (safe_create_leading_directories(the_repository, report_path.buf)) {
 	case SCLD_OK:
 	case SCLD_EXISTS:
 		break;

builtin/cat-file.c

@@ -24,7 +24,7 @@
 #include "pack-bitmap.h"
 #include "object-file.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "replace-object.h"
 #include "promisor-remote.h"
 #include "mailmap.h"

builtin/checkout.c

@@ -18,8 +18,9 @@
 #include "lockfile.h"
 #include "mem-pool.h"
 #include "merge-ort-wrappers.h"
+#include "object-file.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "parse-options.h"
 #include "path.h"
 #include "preload-index.h"

builtin/clone.c

@@ -25,7 +25,7 @@
 #include "refs.h"
 #include "refspec.h"
 #include "object-file.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "tree.h"
 #include "tree-walk.h"
 #include "unpack-trees.h"
@@ -1092,7 +1092,7 @@ int cmd_clone(int argc,
 	sigchain_push_common(remove_junk_on_signal);
 
 	if (!option_bare) {
-		if (safe_create_leading_directories_const(work_tree) < 0)
+		if (safe_create_leading_directories_const(the_repository, work_tree) < 0)
 			die_errno(_("could not create leading directories of '%s'"),
 				  work_tree);
 		if (dest_exists)
@@ -1113,7 +1113,7 @@ int cmd_clone(int argc,
 			junk_git_dir_flags |= REMOVE_DIR_KEEP_TOPLEVEL;
 		junk_git_dir = git_dir;
 	}
-	if (safe_create_leading_directories_const(git_dir) < 0)
+	if (safe_create_leading_directories_const(the_repository, git_dir) < 0)
 		die(_("could not create leading directories of '%s'"), git_dir);
 
 	if (0 <= option_verbosity) {

builtin/commit-graph.c

@@ -6,7 +6,7 @@
 #include "hex.h"
 #include "parse-options.h"
 #include "commit-graph.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "progress.h"
 #include "replace-object.h"
 #include "strbuf.h"

builtin/commit-tree.c

@@ -9,7 +9,7 @@
 #include "gettext.h"
 #include "hex.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 
 #include "commit.h"
 #include "parse-options.h"

builtin/count-objects.c

@@ -12,7 +12,7 @@
 #include "parse-options.h"
 #include "quote.h"
 #include "packfile.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 
 static unsigned long garbage;
 static off_t size_garbage;

builtin/credential-cache--daemon.c

@@ -2,8 +2,8 @@
 #include "builtin.h"
 #include "abspath.h"
 #include "gettext.h"
-#include "object-file.h"
 #include "parse-options.h"
+#include "path.h"
 
 #ifndef NO_UNIX_SOCKETS
 
@@ -271,7 +271,7 @@ static void init_socket_directory(const char *path)
 		 * condition in which somebody can chdir to it, sleep, then try to open
 		 * our protected socket.
 		 */
-		if (safe_create_leading_directories_const(dir) < 0)
+		if (safe_create_leading_directories_const(the_repository, dir) < 0)
 			die_errno("unable to create directories for '%s'", dir);
 		if (mkdir(dir, 0700) < 0)
 			die_errno("unable to mkdir '%s'", dir);

builtin/describe.c

@@ -19,7 +19,7 @@
 #include "setup.h"
 #include "strvec.h"
 #include "run-command.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "list-objects.h"
 #include "commit-slab.h"
 #include "wildmatch.h"

builtin/diagnose.c

@@ -3,8 +3,8 @@
 #include "builtin.h"
 #include "abspath.h"
 #include "gettext.h"
-#include "object-file.h"
 #include "parse-options.h"
+#include "path.h"
 #include "diagnose.h"
 
 static const char * const diagnose_usage[] = {
@@ -50,7 +50,7 @@ int cmd_diagnose(int argc,
 	strbuf_addftime(&zip_path, option_suffix, localtime_r(&now, &tm), 0, 0);
 	strbuf_addstr(&zip_path, ".zip");
 
-	switch (safe_create_leading_directories(zip_path.buf)) {
+	switch (safe_create_leading_directories(the_repository, zip_path.buf)) {
 	case SCLD_OK:
 	case SCLD_EXISTS:
 		break;

builtin/difftool.c

@@ -22,6 +22,7 @@
 #include "gettext.h"
 #include "hex.h"
 #include "parse-options.h"
+#include "path.h"
 #include "read-cache-ll.h"
 #include "repository.h"
 #include "sparse-index.h"
@@ -29,7 +30,7 @@
 #include "strbuf.h"
 #include "lockfile.h"
 #include "object-file.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "dir.h"
 #include "entry.h"
 #include "setup.h"
@@ -271,9 +272,9 @@ static void changed_files(struct repository *repo,
 	strbuf_release(&buf);
 }
 
-static int ensure_leading_directories(char *path)
+static int ensure_leading_directories(struct repository *repo, char *path)
 {
-	switch (safe_create_leading_directories(path)) {
+	switch (safe_create_leading_directories(repo, path)) {
 		case SCLD_OK:
 		case SCLD_EXISTS:
 			return 0;
@@ -341,11 +342,12 @@ static int checkout_path(unsigned mode, struct object_id *oid,
 	return ret;
 }
 
-static void write_file_in_directory(struct strbuf *dir, size_t dir_len,
+static void write_file_in_directory(struct repository *repo,
+				    struct strbuf *dir, size_t dir_len,
 				    const char *path, const char *content)
 {
 	add_path(dir, dir_len, path);
-	ensure_leading_directories(dir->buf);
+	ensure_leading_directories(repo, dir->buf);
 	unlink(dir->buf);
 	write_file(dir->buf, "%s", content);
 }
@@ -356,14 +358,15 @@ static void write_file_in_directory(struct strbuf *dir, size_t dir_len,
  * as text files, resulting in behavior that is analogous to what "git diff"
  * displays for symlink and submodule diffs.
  */
-static void write_standin_files(struct pair_entry *entry,
+static void write_standin_files(struct repository *repo,
+				struct pair_entry *entry,
 				struct strbuf *ldir, size_t ldir_len,
 				struct strbuf *rdir, size_t rdir_len)
 {
 	if (*entry->left)
-		write_file_in_directory(ldir, ldir_len, entry->path, entry->left);
+		write_file_in_directory(repo, ldir, ldir_len, entry->path, entry->left);
 	if (*entry->right)
-		write_file_in_directory(rdir, rdir_len, entry->path, entry->right);
+		write_file_in_directory(repo, rdir, rdir_len, entry->path, entry->right);
 }
 
 static int run_dir_diff(struct repository *repo,
@@ -533,7 +536,7 @@ static int run_dir_diff(struct repository *repo,
 						ADD_CACHE_JUST_APPEND);
 
 				add_path(&rdir, rdir_len, dst_path);
-				if (ensure_leading_directories(rdir.buf)) {
+				if (ensure_leading_directories(repo, rdir.buf)) {
 					ret = error("could not create "
 						    "directory for '%s'",
 						    dst_path);
@@ -576,7 +579,7 @@ static int run_dir_diff(struct repository *repo,
 	 */
 	hashmap_for_each_entry(&submodules, &iter, entry,
 				entry /* member name */) {
-		write_standin_files(entry, &ldir, ldir_len, &rdir, rdir_len);
+		write_standin_files(repo, entry, &ldir, ldir_len, &rdir, rdir_len);
 	}
 
 	/*
@@ -587,7 +590,7 @@ static int run_dir_diff(struct repository *repo,
 	hashmap_for_each_entry(&symlinks2, &iter, entry,
 				entry /* member name */) {
 
-		write_standin_files(entry, &ldir, ldir_len, &rdir, rdir_len);
+		write_standin_files(repo, entry, &ldir, ldir_len, &rdir, rdir_len);
 	}
 
 	strbuf_setlen(&ldir, ldir_len);

builtin/fast-export.c

@@ -14,7 +14,7 @@
 #include "refs.h"
 #include "refspec.h"
 #include "object-file.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "commit.h"
 #include "object.h"
 #include "tag.h"

builtin/fast-import.c

@@ -24,7 +24,7 @@
 #include "packfile.h"
 #include "object-file.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "mem-pool.h"
 #include "commit-reach.h"
 #include "khash.h"
@@ -1720,7 +1720,7 @@ static void dump_marks(void)
 	if (!export_marks_file || (import_marks_file && !import_marks_file_done))
 		return;
 
-	if (safe_create_leading_directories_const(export_marks_file)) {
+	if (safe_create_leading_directories_const(the_repository, export_marks_file)) {
 		failure |= error_errno("unable to create leading directories of %s",
 				       export_marks_file);
 		return;

builtin/fetch.c

@@ -14,7 +14,7 @@
 #include "refs.h"
 #include "refspec.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "oidset.h"
 #include "oid-array.h"
 #include "commit.h"

builtin/fsck.c

@@ -17,7 +17,7 @@
 #include "packfile.h"
 #include "object-file.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "path.h"
 #include "read-cache-ll.h"
 #include "replace-object.h"
@@ -332,7 +332,7 @@ static void check_unreachable_object(struct object *obj)
 				describe_object(&obj->oid));
 			FILE *f;
 
-			if (safe_create_leading_directories_const(filename)) {
+			if (safe_create_leading_directories_const(the_repository, filename)) {
 				error(_("could not create lost-found"));
 				free(filename);
 				return;

builtin/gc.c

@@ -28,8 +28,7 @@
 #include "commit.h"
 #include "commit-graph.h"
 #include "packfile.h"
-#include "object-file.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "pack.h"
 #include "pack-objects.h"
 #include "path.h"
@@ -2169,7 +2168,7 @@ static int launchctl_schedule_plist(const char *exec_path, enum schedule_priorit
 	}
 	strbuf_addstr(&plist, "</array>\n</dict>\n</plist>\n");
 
-	if (safe_create_leading_directories(filename))
+	if (safe_create_leading_directories(the_repository, filename))
 		die(_("failed to create directories for '%s'"), filename);
 
 	if ((long)lock_file_timeout_ms < 0 &&
@@ -2635,7 +2634,7 @@ static int systemd_timer_write_timer_file(enum schedule_priority schedule,
 
 	filename = xdg_config_home_systemd(local_timer_name);
 
-	if (safe_create_leading_directories(filename)) {
+	if (safe_create_leading_directories(the_repository, filename)) {
 		error(_("failed to create directories for '%s'"), filename);
 		goto error;
 	}
@@ -2708,7 +2707,7 @@ static int systemd_timer_write_service_template(const char *exec_path)
 	char *local_service_name = xstrfmt(SYSTEMD_UNIT_FORMAT, "", "service");
 
 	filename = xdg_config_home_systemd(local_service_name);
-	if (safe_create_leading_directories(filename)) {
+	if (safe_create_leading_directories(the_repository, filename)) {
 		error(_("failed to create directories for '%s'"), filename);
 		goto error;
 	}

builtin/grep.c

@@ -26,7 +26,7 @@
 #include "submodule-config.h"
 #include "object-file.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "packfile.h"
 #include "pager.h"
 #include "path.h"

builtin/hash-object.c

@@ -11,7 +11,7 @@
 #include "gettext.h"
 #include "hex.h"
 #include "object-file.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "blob.h"
 #include "quote.h"
 #include "parse-options.h"
@@ -19,6 +19,11 @@
 #include "strbuf.h"
 #include "write-or-die.h"
 
+enum {
+	HASH_OBJECT_CHECK = (1 << 0),
+	HASH_OBJECT_WRITE = (1 << 1),
+};
+
 /*
  * This is to create corrupt objects for debugging and as such it
  * needs to bypass the data conversion performed by, and the type
@@ -33,7 +38,7 @@ static int hash_literally(struct object_id *oid, int fd, const char *type, unsig
 		ret = -1;
 	else
 		ret = write_object_file_literally(buf.buf, buf.len, type, oid,
-						 flags);
+						  (flags & HASH_OBJECT_WRITE) ? WRITE_OBJECT_FILE_PERSIST : 0);
 	close(fd);
 	strbuf_release(&buf);
 	return ret;
@@ -42,15 +47,21 @@ static int hash_literally(struct object_id *oid, int fd, const char *type, unsig
 static void hash_fd(int fd, const char *type, const char *path, unsigned flags,
 		    int literally)
 {
+	unsigned int index_flags = 0;
 	struct stat st;
 	struct object_id oid;
 
+	if (flags & HASH_OBJECT_WRITE)
+		index_flags |= INDEX_WRITE_OBJECT;
+	if (flags & HASH_OBJECT_CHECK)
+		index_flags |= INDEX_FORMAT_CHECK;
+
 	if (fstat(fd, &st) < 0 ||
 	    (literally
 	     ? hash_literally(&oid, fd, type, flags)
 	     : index_fd(the_repository->index, &oid, fd, &st,
-			type_from_string(type), path, flags)))
-		die((flags & HASH_WRITE_OBJECT)
+			type_from_string(type), path, index_flags)))
+		die((flags & HASH_OBJECT_WRITE)
 		    ? "Unable to add %s to database"
 		    : "Unable to hash %s", path);
 	printf("%s\n", oid_to_hex(&oid));
@@ -102,13 +113,13 @@ int cmd_hash_object(int argc,
 	int no_filters = 0;
 	int literally = 0;
 	int nongit = 0;
-	unsigned flags = HASH_FORMAT_CHECK;
+	unsigned flags = HASH_OBJECT_CHECK;
 	const char *vpath = NULL;
 	char *vpath_free = NULL;
 	const struct option hash_object_options[] = {
 		OPT_STRING('t', NULL, &type, N_("type"), N_("object type")),
 		OPT_BIT('w', NULL, &flags, N_("write the object into the object database"),
-			HASH_WRITE_OBJECT),
+			HASH_OBJECT_WRITE),
 		OPT_COUNTUP( 0 , "stdin", &hashstdin, N_("read the object from stdin")),
 		OPT_BOOL( 0 , "stdin-paths", &stdin_paths, N_("read file names from stdin")),
 		OPT_BOOL( 0 , "no-filters", &no_filters, N_("store file as is without filters")),
@@ -122,7 +133,7 @@ int cmd_hash_object(int argc,
 	argc = parse_options(argc, argv, prefix, hash_object_options,
 			     hash_object_usage, 0);
 
-	if (flags & HASH_WRITE_OBJECT)
+	if (flags & HASH_OBJECT_WRITE)
 		prefix = setup_git_directory();
 	else
 		prefix = setup_git_directory_gently(&nongit);

builtin/index-pack.c

@@ -21,7 +21,7 @@
 #include "packfile.h"
 #include "pack-revindex.h"
 #include "object-file.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "oid-array.h"
 #include "oidset.h"
 #include "path.h"

builtin/init-db.c

@@ -8,7 +8,6 @@
 #include "abspath.h"
 #include "environment.h"
 #include "gettext.h"
-#include "object-file.h"
 #include "parse-options.h"
 #include "path.h"
 #include "refs.h"
@@ -134,7 +133,7 @@ int cmd_init_db(int argc,
 				 */
 				saved = repo_settings_get_shared_repository(the_repository);
 				repo_settings_set_shared_repository(the_repository, 0);
-				switch (safe_create_leading_directories_const(argv[0])) {
+				switch (safe_create_leading_directories_const(the_repository, argv[0])) {
 				case SCLD_OK:
 				case SCLD_PERMS:
 					break;

builtin/log.c

@@ -14,9 +14,8 @@
 #include "gettext.h"
 #include "hex.h"
 #include "refs.h"
-#include "object-file.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "pager.h"
 #include "color.h"
 #include "commit.h"
@@ -29,6 +28,7 @@
 #include "tag.h"
 #include "reflog-walk.h"
 #include "patch-ids.h"
+#include "path.h"
 #include "shortlog.h"
 #include "remote.h"
 #include "string-list.h"
@@ -2311,7 +2311,7 @@ int cmd_format_patch(int argc,
 		 */
 		saved = repo_settings_get_shared_repository(the_repository);
 		repo_settings_set_shared_repository(the_repository, 0);
-		switch (safe_create_leading_directories_const(output_directory)) {
+		switch (safe_create_leading_directories_const(the_repository, output_directory)) {
 		case SCLD_OK:
 		case SCLD_EXISTS:
 			break;

builtin/ls-tree.c

@@ -10,7 +10,7 @@
 #include "gettext.h"
 #include "hex.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "tree.h"
 #include "path.h"
 #include "quote.h"

builtin/merge-file.c

@@ -5,6 +5,7 @@
 #include "abspath.h"
 #include "diff.h"
 #include "hex.h"
+#include "object-file.h"
 #include "object-name.h"
 #include "object-store.h"
 #include "config.h"

builtin/merge-tree.c

@@ -10,7 +10,7 @@
 #include "commit-reach.h"
 #include "merge-ort.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "parse-options.h"
 #include "blob.h"
 #include "merge-blobs.h"

builtin/mktag.c

@@ -6,7 +6,7 @@
 #include "strbuf.h"
 #include "replace-object.h"
 #include "object-file.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "fsck.h"
 #include "config.h"
 

builtin/mktree.c

@@ -11,7 +11,8 @@
 #include "strbuf.h"
 #include "tree.h"
 #include "parse-options.h"
-#include "object-store-ll.h"
+#include "object-file.h"
+#include "object-store.h"
 
 static struct treeent {
 	unsigned mode;

builtin/multi-pack-index.c

@@ -7,7 +7,7 @@
 #include "midx.h"
 #include "strbuf.h"
 #include "trace2.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "replace-object.h"
 #include "repository.h"
 

builtin/mv.c

@@ -15,6 +15,7 @@
 #include "gettext.h"
 #include "name-hash.h"
 #include "object-file.h"
+#include "path.h"
 #include "pathspec.h"
 #include "lockfile.h"
 #include "dir.h"
@@ -555,7 +556,7 @@ remove_entry:
 					 */
 					char *dst_dup = xstrdup(dst);
 					string_list_append(&dirty_paths, dst);
-					safe_create_leading_directories(dst_dup);
+					safe_create_leading_directories(the_repository, dst_dup);
 					FREE_AND_NULL(dst_dup);
 					rename(src, dst);
 				}

builtin/notes.c

@@ -14,8 +14,9 @@
 #include "gettext.h"
 #include "hex.h"
 #include "notes.h"
+#include "object-file.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "path.h"
 
 #include "pretty.h"

builtin/pack-objects.c

@@ -32,7 +32,7 @@
 #include "list.h"
 #include "packfile.h"
 #include "object-file.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "replace-object.h"
 #include "dir.h"
 #include "midx.h"

builtin/pack-redundant.c

@@ -13,7 +13,7 @@
 #include "hex.h"
 
 #include "packfile.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "strbuf.h"
 
 #define BLKSIZE 512

builtin/prune.c

@@ -17,7 +17,7 @@
 #include "replace-object.h"
 #include "object-file.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "shallow.h"
 
 static const char * const prune_usage[] = {

builtin/rebase.c

@@ -267,7 +267,8 @@ static int init_basic_state(struct replay_opts *opts, const char *head_name,
 {
 	FILE *interactive;
 
-	if (!is_directory(merge_dir()) && mkdir_in_gitdir(merge_dir()))
+	if (!is_directory(merge_dir()) &&
+	    safe_create_dir_in_gitdir(the_repository, merge_dir()))
 		return error_errno(_("could not create temporary %s"), merge_dir());
 
 	refs_delete_reflog(get_main_ref_store(the_repository), "REBASE_HEAD");

builtin/receive-pack.c

@@ -31,8 +31,9 @@
 #include "tmp-objdir.h"
 #include "oidset.h"
 #include "packfile.h"
+#include "object-file.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "path.h"
 #include "protocol.h"
 #include "commit-reach.h"

builtin/remote.c

@@ -14,7 +14,7 @@
 #include "rebase.h"
 #include "refs.h"
 #include "refspec.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "strvec.h"
 #include "commit-reach.h"
 #include "progress.h"

builtin/repack.c

@@ -17,7 +17,7 @@
 #include "midx.h"
 #include "packfile.h"
 #include "prune-packed.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "promisor-remote.h"
 #include "shallow.h"
 #include "pack.h"

builtin/replace.c

@@ -19,7 +19,7 @@
 #include "run-command.h"
 #include "object-file.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "replace-object.h"
 #include "tag.h"
 #include "wildmatch.h"
@@ -305,7 +305,7 @@ static int import_object(struct object_id *oid, enum object_type type,
 		strbuf_release(&result);
 	} else {
 		struct stat st;
-		int flags = HASH_FORMAT_CHECK | HASH_WRITE_OBJECT;
+		int flags = INDEX_FORMAT_CHECK | INDEX_WRITE_OBJECT;
 
 		if (fstat(fd, &st) < 0) {
 			error_errno(_("unable to fstat %s"), filename);

builtin/rev-list.c

@@ -14,7 +14,7 @@
 #include "object.h"
 #include "object-name.h"
 #include "object-file.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "pack-bitmap.h"
 #include "parse-options.h"
 #include "log-tree.h"

builtin/show-ref.c

@@ -5,7 +5,7 @@
 #include "hex.h"
 #include "refs/refs-internal.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "object.h"
 #include "string-list.h"
 #include "parse-options.h"

builtin/sparse-checkout.c

@@ -9,6 +9,7 @@
 #include "object-file.h"
 #include "object-name.h"
 #include "parse-options.h"
+#include "path.h"
 #include "pathspec.h"
 #include "strbuf.h"
 #include "string-list.h"
@@ -335,7 +336,7 @@ static int write_patterns_and_update(struct pattern_list *pl)
 
 	sparse_filename = get_sparse_checkout_filename();
 
-	if (safe_create_leading_directories(sparse_filename))
+	if (safe_create_leading_directories(the_repository, sparse_filename))
 		die(_("failed to create directory for sparse-checkout file"));
 
 	hold_lock_file_for_update(&lk, sparse_filename, LOCK_DIE_ON_ERROR);
@@ -491,7 +492,7 @@ static int sparse_checkout_init(int argc, const char **argv, const char *prefix,
 		FILE *fp;
 
 		/* assume we are in a fresh repo, but update the sparse-checkout file */
-		if (safe_create_leading_directories(sparse_filename))
+		if (safe_create_leading_directories(the_repository, sparse_filename))
 			die(_("unable to create leading directories of %s"),
 			    sparse_filename);
 		fp = xfopen(sparse_filename, "w");

builtin/submodule--helper.c

@@ -28,7 +28,7 @@
 #include "diff.h"
 #include "object-file.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "advice.h"
 #include "branch.h"
 #include "list-objects-filter-options.h"
@@ -1739,7 +1739,7 @@ static int clone_submodule(const struct module_clone_data *clone_data,
 		    !is_empty_dir(clone_data_path))
 			die(_("directory not empty: '%s'"), clone_data_path);
 
-		if (safe_create_leading_directories_const(sm_gitdir) < 0)
+		if (safe_create_leading_directories_const(the_repository, sm_gitdir) < 0)
 			die(_("could not create directory '%s'"), sm_gitdir);
 
 		prepare_possible_alternates(clone_data->name, reference);
@@ -1800,7 +1800,7 @@ static int clone_submodule(const struct module_clone_data *clone_data,
 		if (clone_data->require_init && !stat(clone_data_path, &st) &&
 		    !is_empty_dir(clone_data_path))
 			die(_("directory not empty: '%s'"), clone_data_path);
-		if (safe_create_leading_directories_const(clone_data_path) < 0)
+		if (safe_create_leading_directories_const(the_repository, clone_data_path) < 0)
 			die(_("could not create directory '%s'"), clone_data_path);
 		path = xstrfmt("%s/index", sm_gitdir);
 		unlink_or_warn(path);

builtin/tag.c

@@ -17,8 +17,9 @@
 #include "gettext.h"
 #include "hex.h"
 #include "refs.h"
+#include "object-file.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "path.h"
 #include "tag.h"
 #include "parse-options.h"

builtin/unpack-file.c

@@ -2,8 +2,9 @@
 #include "builtin.h"
 #include "config.h"
 #include "hex.h"
+#include "object-file.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 
 static char *create_temp_file(struct object_id *oid)
 {

builtin/unpack-objects.c

@@ -8,7 +8,8 @@
 #include "gettext.h"
 #include "git-zlib.h"
 #include "hex.h"
-#include "object-store-ll.h"
+#include "object-file.h"
+#include "object-store.h"
 #include "object.h"
 #include "delta.h"
 #include "pack.h"

builtin/update-index.c

@@ -304,7 +304,7 @@ static int add_one_path(const struct cache_entry *old, const char *path, int len
 	ce->ce_mode = ce_mode_from_stat(old, st->st_mode);
 
 	if (index_path(the_repository->index, &ce->oid, path, st,
-		       info_only ? 0 : HASH_WRITE_OBJECT)) {
+		       info_only ? 0 : INDEX_WRITE_OBJECT)) {
 		discard_cache_entry(ce);
 		return -1;
 	}

builtin/worktree.c

@@ -348,7 +348,7 @@ static void copy_sparse_checkout(const char *worktree_git_dir)
 	char *to_file = xstrfmt("%s/info/sparse-checkout", worktree_git_dir);
 
 	if (file_exists(from_file)) {
-		if (safe_create_leading_directories(to_file) ||
+		if (safe_create_leading_directories(the_repository, to_file) ||
 			copy_file(to_file, from_file, 0666))
 			error(_("failed to copy '%s' to '%s'; sparse-checkout may not work correctly"),
 				from_file, to_file);
@@ -367,7 +367,7 @@ static void copy_filtered_worktree_config(const char *worktree_git_dir)
 		struct config_set cs = { { 0 } };
 		int bare;
 
-		if (safe_create_leading_directories(to_file) ||
+		if (safe_create_leading_directories(the_repository, to_file) ||
 			copy_file(to_file, from_file, 0666)) {
 			error(_("failed to copy worktree config from '%s' to '%s'"),
 				from_file, to_file);
@@ -466,7 +466,7 @@ static int add_worktree(const char *path, const char *refname,
 	name = sb_name.buf;
 	repo_git_path_replace(the_repository, &sb_repo, "worktrees/%s", name);
 	len = sb_repo.len;
-	if (safe_create_leading_directories_const(sb_repo.buf))
+	if (safe_create_leading_directories_const(the_repository, sb_repo.buf))
 		die_errno(_("could not create leading directories of '%s'"),
 			  sb_repo.buf);
 
@@ -498,7 +498,7 @@ static int add_worktree(const char *path, const char *refname,
 		write_file(sb.buf, _("initializing"));
 
 	strbuf_addf(&sb_git, "%s/.git", path);
-	if (safe_create_leading_directories_const(sb_git.buf))
+	if (safe_create_leading_directories_const(the_repository, sb_git.buf))
 		die_errno(_("could not create leading directories of '%s'"),
 			  sb_git.buf);
 	junk_work_tree = xstrdup(path);

bulk-checkin.c

@@ -17,7 +17,7 @@
 #include "tmp-objdir.h"
 #include "packfile.h"
 #include "object-file.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 
 static int odb_transaction_nesting;
 
@@ -167,7 +167,7 @@ static int stream_blob_to_pack(struct bulk_checkin_packfile *state,
 	unsigned char obuf[16384];
 	unsigned hdrlen;
 	int status = Z_OK;
-	int write_object = (flags & HASH_WRITE_OBJECT);
+	int write_object = (flags & INDEX_WRITE_OBJECT);
 	off_t offset = 0;
 
 	git_deflate_init(&s, pack_compression_level);
@@ -237,7 +237,7 @@ static int stream_blob_to_pack(struct bulk_checkin_packfile *state,
 static void prepare_to_stream(struct bulk_checkin_packfile *state,
 			      unsigned flags)
 {
-	if (!(flags & HASH_WRITE_OBJECT) || state->f)
+	if (!(flags & INDEX_WRITE_OBJECT) || state->f)
 		return;
 
 	state->f = create_tmp_packfile(the_repository, &state->pack_tmp_name);
@@ -271,7 +271,7 @@ static int deflate_blob_to_pack(struct bulk_checkin_packfile *state,
 	git_hash_update(&ctx, obuf, header_len);
 
 	/* Note: idx is non-NULL when we are writing */
-	if ((flags & HASH_WRITE_OBJECT) != 0) {
+	if ((flags & INDEX_WRITE_OBJECT) != 0) {
 		CALLOC_ARRAY(idx, 1);
 
 		prepare_to_stream(state, flags);

bulk-checkin.h

@@ -9,6 +9,21 @@
 void prepare_loose_object_bulk_checkin(void);
 void fsync_loose_object_bulk_checkin(int fd, const char *filename);
 
+/*
+ * This creates one packfile per large blob unless bulk-checkin
+ * machinery is "plugged".
+ *
+ * This also bypasses the usual "convert-to-git" dance, and that is on
+ * purpose. We could write a streaming version of the converting
+ * functions and insert that before feeding the data to fast-import
+ * (or equivalent in-core API described above). However, that is
+ * somewhat complicated, as we do not know the size of the filter
+ * result, which we need to know beforehand when writing a git object.
+ * Since the primary motivation for trying to stream from the working
+ * tree file and to avoid mmaping it in core is to deal with large
+ * binary blobs, they generally do not want to get any conversion, and
+ * callers should avoid this code path when filters are requested.
+ */
 int index_blob_bulk_checkin(struct object_id *oid,
 			    int fd, size_t size,
 			    const char *path, unsigned flags);

bundle-uri.c

@@ -14,7 +14,7 @@
 #include "fetch-pack.h"
 #include "remote.h"
 #include "trace2.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 
 static struct {
 	enum bundle_list_heuristic heuristic;

bundle.c

@@ -7,7 +7,7 @@
 #include "environment.h"
 #include "gettext.h"
 #include "hex.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "repository.h"
 #include "object.h"
 #include "commit.h"

cache-tree.c

@@ -10,7 +10,7 @@
 #include "cache-tree.h"
 #include "bulk-checkin.h"
 #include "object-file.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "read-cache-ll.h"
 #include "replace-object.h"
 #include "repository.h"
@@ -452,7 +452,7 @@ static int update_one(struct cache_tree *it,
 				 OBJ_TREE, &it->oid);
 	} else if (write_object_file_flags(buffer.buf, buffer.len, OBJ_TREE,
 					   &it->oid, NULL, flags & WRITE_TREE_SILENT
-					   ? HASH_SILENT : 0)) {
+					   ? WRITE_OBJECT_FILE_SILENT : 0)) {
 		strbuf_release(&buffer);
 		return -1;
 	}

combine-diff.c

@@ -2,7 +2,7 @@
 #define DISABLE_SIGN_COMPARE_WARNINGS
 
 #include "git-compat-util.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "commit.h"
 #include "convert.h"
 #include "diff.h"

commit-graph.c

@@ -13,8 +13,7 @@
 #include "refs.h"
 #include "hash-lookup.h"
 #include "commit-graph.h"
-#include "object-file.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "oid-array.h"
 #include "path.h"
 #include "alloc.h"
@@ -2065,7 +2064,7 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx)
 		ctx->graph_name = get_commit_graph_filename(ctx->odb);
 	}
 
-	if (safe_create_leading_directories(ctx->graph_name)) {
+	if (safe_create_leading_directories(the_repository, ctx->graph_name)) {
 		error(_("unable to create leading directories of %s"),
 			ctx->graph_name);
 		return -1;

commit-graph.h

@@ -1,7 +1,7 @@
 #ifndef COMMIT_GRAPH_H
 #define COMMIT_GRAPH_H
 
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "oidset.h"
 
 #define GIT_TEST_COMMIT_GRAPH "GIT_TEST_COMMIT_GRAPH"

commit.c

@@ -9,7 +9,7 @@
 #include "hex.h"
 #include "repository.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "utf8.h"
 #include "diff.h"
 #include "revision.h"
@@ -29,6 +29,7 @@
 #include "tree.h"
 #include "hook.h"
 #include "parse.h"
+#include "object-file.h"
 #include "object-file-convert.h"
 
 static struct commit_extra_header *read_commit_extra_header_lines(const char *buf, size_t len, const char **);

compat/open.c

@@ -1,5 +1,6 @@
 #include "git-compat-util.h"
 
+#ifdef OPEN_RETURNS_EINTR
 #undef open
 int git_open_with_retry(const char *path, int flags, ...)
 {
@@ -23,3 +24,31 @@ int git_open_with_retry(const char *path, int flags, ...)
 
 	return ret;
 }
+#endif
+
+int git_open_cloexec(const char *name, int flags)
+{
+	int fd;
+	static int o_cloexec = O_CLOEXEC;
+
+	fd = open(name, flags | o_cloexec);
+	if ((o_cloexec & O_CLOEXEC) && fd < 0 && errno == EINVAL) {
+		/* Try again w/o O_CLOEXEC: the kernel might not support it */
+		o_cloexec &= ~O_CLOEXEC;
+		fd = open(name, flags | o_cloexec);
+	}
+
+#if defined(F_GETFD) && defined(F_SETFD) && defined(FD_CLOEXEC)
+	{
+		static int fd_cloexec = FD_CLOEXEC;
+
+		if (!o_cloexec && 0 <= fd && fd_cloexec) {
+			/* Opened w/o O_CLOEXEC?  try with fcntl(2) to add it */
+			int flags = fcntl(fd, F_GETFD);
+			if (fcntl(fd, F_SETFD, flags | fd_cloexec))
+				fd_cloexec = 0;
+		}
+	}
+#endif
+	return fd;
+}

config.c

@@ -31,7 +31,7 @@
 #include "hashmap.h"
 #include "string-list.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "pager.h"
 #include "path.h"
 #include "utf8.h"

connected.c

@@ -3,7 +3,7 @@
 #include "git-compat-util.h"
 #include "gettext.h"
 #include "hex.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "run-command.h"
 #include "sigchain.h"
 #include "connected.h"

convert.c

@@ -8,7 +8,7 @@
 #include "copy.h"
 #include "gettext.h"
 #include "hex.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "attr.h"
 #include "run-command.h"
 #include "quote.h"

diagnose.c

@@ -7,7 +7,7 @@
 #include "gettext.h"
 #include "hex.h"
 #include "strvec.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "packfile.h"
 #include "parse-options.h"
 #include "repository.h"

diff.c

@@ -23,7 +23,7 @@
 #include "color.h"
 #include "run-command.h"
 #include "utf8.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "userdiff.h"
 #include "submodule.h"
 #include "hashmap.h"

diffcore-rename.c

@@ -8,7 +8,7 @@
 #include "git-compat-util.h"
 #include "diff.h"
 #include "diffcore.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "hashmap.h"
 #include "mem-pool.h"
 #include "oid-array.h"

dir.c

@@ -17,8 +17,7 @@
 #include "environment.h"
 #include "gettext.h"
 #include "name-hash.h"
-#include "object-file.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "path.h"
 #include "refs.h"
 #include "repository.h"
@@ -4063,12 +4062,12 @@ void connect_work_tree_and_git_dir(const char *work_tree_,
 
 	/* Prepare .git file */
 	strbuf_addf(&gitfile_sb, "%s/.git", work_tree_);
-	if (safe_create_leading_directories_const(gitfile_sb.buf))
+	if (safe_create_leading_directories_const(the_repository, gitfile_sb.buf))
 		die(_("could not create directories for %s"), gitfile_sb.buf);
 
 	/* Prepare config file */
 	strbuf_addf(&cfg_sb, "%s/config", git_dir_);
-	if (safe_create_leading_directories_const(cfg_sb.buf))
+	if (safe_create_leading_directories_const(the_repository, cfg_sb.buf))
 		die(_("could not create directories for %s"), cfg_sb.buf);
 
 	git_dir = real_pathdup(git_dir_, 1);

entry.c

@@ -1,7 +1,7 @@
 #define USE_THE_REPOSITORY_VARIABLE
 
 #include "git-compat-util.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "dir.h"
 #include "environment.h"
 #include "gettext.h"

fetch-pack.c

@@ -24,7 +24,7 @@
 #include "oid-array.h"
 #include "oidset.h"
 #include "packfile.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "path.h"
 #include "connected.h"
 #include "fetch-negotiator.h"

fmt-merge-msg.c

@@ -6,7 +6,7 @@
 #include "environment.h"
 #include "refs.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "diff.h"
 #include "diff-merges.h"
 #include "hex.h"

fsck.c

@@ -4,7 +4,7 @@
 #include "date.h"
 #include "dir.h"
 #include "hex.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "path.h"
 #include "repository.h"
 #include "object.h"

git-compat-util.h

@@ -598,6 +598,9 @@ static inline bool strip_suffix(const char *str, const char *suffix,
 #define DEFAULT_PACKED_GIT_LIMIT \
 	((1024L * 1024L) * (size_t)(sizeof(void*) >= 8 ? (32 * 1024L * 1024L) : 256))
 
+int git_open_cloexec(const char *name, int flags);
+#define git_open(name) git_open_cloexec(name, O_RDONLY)
+
 static inline size_t st_add(size_t a, size_t b)
 {
 	if (unsigned_add_overflows(a, b))

grep.c

@@ -5,7 +5,7 @@
 #include "gettext.h"
 #include "grep.h"
 #include "hex.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "pretty.h"
 #include "userdiff.h"
 #include "xdiff-interface.h"

http-backend.c

@@ -18,7 +18,7 @@
 #include "url.h"
 #include "strvec.h"
 #include "packfile.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "protocol.h"
 #include "date.h"
 #include "write-or-die.h"

http-push.c

@@ -19,7 +19,8 @@
 #include "tree-walk.h"
 #include "url.h"
 #include "packfile.h"
-#include "object-store-ll.h"
+#include "object-file.h"
+#include "object-store.h"
 #include "commit-reach.h"
 
 #ifdef EXPAT_NEEDS_XMLPARSE_H

http-walker.c

@@ -9,7 +9,7 @@
 #include "list.h"
 #include "transport.h"
 #include "packfile.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 
 struct alt_base {
 	char *base;

http.c

@@ -19,7 +19,7 @@
 #include "packfile.h"
 #include "string-list.h"
 #include "object-file.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "tempfile.h"
 
 static struct trace_key trace_curl = TRACE_KEY_INIT(CURL);

list-objects-filter.c

@@ -12,7 +12,7 @@
 #include "oidmap.h"
 #include "oidset.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 
 /* Remember to update object flag allocation in object.h */
 /*

list-objects.c

@@ -14,7 +14,7 @@
 #include "list-objects-filter.h"
 #include "list-objects-filter-options.h"
 #include "packfile.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "trace.h"
 #include "environment.h"
 

log-tree.c

@@ -9,7 +9,7 @@
 #include "environment.h"
 #include "hex.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "repository.h"
 #include "tmp-objdir.h"
 #include "commit.h"

mailmap.c

@@ -6,7 +6,7 @@
 #include "string-list.h"
 #include "mailmap.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "setup.h"
 
 char *git_mailmap_file;

match-trees.c

@@ -6,7 +6,8 @@
 #include "strbuf.h"
 #include "tree.h"
 #include "tree-walk.h"
-#include "object-store-ll.h"
+#include "object-file.h"
+#include "object-store.h"
 #include "repository.h"
 
 static int score_missing(unsigned mode)

merge-blobs.c

@@ -4,7 +4,7 @@
 #include "merge-ll.h"
 #include "blob.h"
 #include "merge-blobs.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 
 static int fill_mmfile_blob(mmfile_t *f, struct blob *obj)
 {

merge-ort.c

@@ -37,8 +37,9 @@
 #include "merge-ll.h"
 #include "match-trees.h"
 #include "mem-pool.h"
+#include "object-file.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "oid-array.h"
 #include "path.h"
 #include "promisor-remote.h"

meson.build

@@ -296,6 +296,7 @@ libgit_sources = [
   'common-init.c',
   'compat/nonblock.c',
   'compat/obstack.c',
+  'compat/open.c',
   'compat/terminal.c',
   'compiler-tricks/not-constant.c',
   'config.c',
@@ -386,6 +387,7 @@ libgit_sources = [
   'object-file-convert.c',
   'object-file.c',
   'object-name.c',
+  'object-store.c',
   'object.c',
   'oid-array.c',
   'oidmap.c',

midx-write.c

@@ -1098,7 +1098,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir,
 			    object_dir);
 	else
 		get_midx_filename(r->hash_algo, &midx_name, object_dir);
-	if (safe_create_leading_directories(midx_name.buf))
+	if (safe_create_leading_directories(r, midx_name.buf))
 		die_errno(_("unable to create leading directories of %s"),
 			  midx_name.buf);
 

midx.c

@@ -5,7 +5,6 @@
 #include "dir.h"
 #include "hex.h"
 #include "packfile.h"
-#include "object-file.h"
 #include "hash-lookup.h"
 #include "midx.h"
 #include "progress.h"

notes-cache.c

@@ -2,7 +2,8 @@
 
 #include "git-compat-util.h"
 #include "notes-cache.h"
-#include "object-store-ll.h"
+#include "object-file.h"
+#include "object-store.h"
 #include "pretty.h"
 #include "repository.h"
 #include "commit.h"

notes-merge.c

@@ -8,7 +8,7 @@
 #include "refs.h"
 #include "object-file.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "path.h"
 #include "repository.h"
 #include "diff.h"
@@ -296,7 +296,7 @@ static void check_notes_merge_worktree(struct notes_merge_options *o)
 				    "(%s exists)."), repo_git_path_replace(the_repository, &buf, "NOTES_MERGE_*"));
 		}
 
-		if (safe_create_leading_directories_const(repo_git_path_replace(the_repository, &buf,
+		if (safe_create_leading_directories_const(the_repository, repo_git_path_replace(the_repository, &buf,
 				NOTES_MERGE_WORKTREE "/.test")))
 			die_errno("unable to create directory %s",
 				  repo_git_path_replace(the_repository, &buf, NOTES_MERGE_WORKTREE));
@@ -314,7 +314,7 @@ static void write_buf_to_worktree(const struct object_id *obj,
 {
 	int fd;
 	char *path = repo_git_path(the_repository, NOTES_MERGE_WORKTREE "/%s", oid_to_hex(obj));
-	if (safe_create_leading_directories_const(path))
+	if (safe_create_leading_directories_const(the_repository, path))
 		die_errno("unable to create directory for '%s'", path);
 
 	fd = xopen(path, O_WRONLY | O_EXCL | O_CREAT, 0666);
@@ -729,7 +729,7 @@ int notes_merge_commit(struct notes_merge_options *o,
 		/* write file as blob, and add to partial_tree */
 		if (stat(path.buf, &st))
 			die_errno("Failed to stat '%s'", path.buf);
-		if (index_path(o->repo->index, &blob_oid, path.buf, &st, HASH_WRITE_OBJECT))
+		if (index_path(o->repo->index, &blob_oid, path.buf, &st, INDEX_WRITE_OBJECT))
 			die("Failed to write blob object from '%s'", path.buf);
 		if (add_note(partial_tree, &obj_oid, &blob_oid, NULL))
 			die("Failed to add resolved note '%s' to notes tree",

notes.c

@@ -6,8 +6,9 @@
 #include "environment.h"
 #include "hex.h"
 #include "notes.h"
+#include "object-file.h"
 #include "object-name.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "utf8.h"
 #include "strbuf.h"
 #include "tree-walk.h"

object-file.h

@@ -14,50 +14,37 @@ struct index_state;
  */
 extern int fetch_if_missing;
 
-#define HASH_WRITE_OBJECT 1
-#define HASH_FORMAT_CHECK 2
-#define HASH_RENORMALIZE  4
-#define HASH_SILENT 8
+enum {
+	INDEX_WRITE_OBJECT = (1 << 0),
+	INDEX_FORMAT_CHECK = (1 << 1),
+	INDEX_RENORMALIZE  = (1 << 2),
+};
+
 int index_fd(struct index_state *istate, struct object_id *oid, int fd, struct stat *st, enum object_type type, const char *path, unsigned flags);
 int index_path(struct index_state *istate, struct object_id *oid, const char *path, struct stat *st, unsigned flags);
 
+struct object_directory;
+
+const char *odb_loose_path(struct object_directory *odb,
+			   struct strbuf *buf,
+			   const struct object_id *oid);
+
 /*
- * Create the directory containing the named path, using care to be
- * somewhat safe against races. Return one of the scld_error values to
- * indicate success/failure. On error, set errno to describe the
- * problem.
- *
- * SCLD_VANISHED indicates that one of the ancestor directories of the
- * path existed at one point during the function call and then
- * suddenly vanished, probably because another process pruned the
- * directory while we were working.  To be robust against this kind of
- * race, callers might want to try invoking the function again when it
- * returns SCLD_VANISHED.
- *
- * safe_create_leading_directories() temporarily changes path while it
- * is working but restores it before returning.
- * safe_create_leading_directories_const() doesn't modify path, even
- * temporarily. Both these variants adjust the permissions of the
- * created directories to honor core.sharedRepository, so they are best
- * suited for files inside the git dir. For working tree files, use
- * safe_create_leading_directories_no_share() instead, as it ignores
- * the core.sharedRepository setting.
+ * Return true iff an alternate object database has a loose object
+ * with the specified name.  This function does not respect replace
+ * references.
  */
-enum scld_error {
-	SCLD_OK = 0,
-	SCLD_FAILED = -1,
-	SCLD_PERMS = -2,
-	SCLD_EXISTS = -3,
-	SCLD_VANISHED = -4
-};
-enum scld_error safe_create_leading_directories(char *path);
-enum scld_error safe_create_leading_directories_const(const char *path);
-enum scld_error safe_create_leading_directories_no_share(char *path);
+int has_loose_object_nonlocal(const struct object_id *);
 
-int mkdir_in_gitdir(const char *path);
+int has_loose_object(const struct object_id *);
 
-int git_open_cloexec(const char *name, int flags);
-#define git_open(name) git_open_cloexec(name, O_RDONLY)
+/**
+ * format_object_header() is a thin wrapper around s xsnprintf() that
+ * writes the initial "<type> <obj-len>" part of the loose object
+ * header. It returns the size that snprintf() returns + 1.
+ */
+int format_object_header(char *str, size_t size, enum object_type type,
+			 size_t objsize);
 
 /**
  * unpack_loose_header() initializes the data stream needed to unpack
@@ -99,6 +86,44 @@ enum unpack_loose_header_result unpack_loose_header(git_zstream *stream,
 struct object_info;
 int parse_loose_header(const char *hdr, struct object_info *oi);
 
+enum {
+	/*
+	 * By default, `write_object_file_literally()` does not actually write
+	 * anything into the object store, but only computes the object ID.
+	 * This flag changes that so that the object will be written as a loose
+	 * object and persisted.
+	 */
+	WRITE_OBJECT_FILE_PERSIST = (1 << 0),
+
+	/*
+	 * Do not print an error in case something gose wrong.
+	 */
+	WRITE_OBJECT_FILE_SILENT = (1 << 1),
+};
+
+int write_object_file_flags(const void *buf, unsigned long len,
+			    enum object_type type, struct object_id *oid,
+			    struct object_id *comapt_oid_in, unsigned flags);
+static inline int write_object_file(const void *buf, unsigned long len,
+				    enum object_type type, struct object_id *oid)
+{
+	return write_object_file_flags(buf, len, type, oid, NULL, 0);
+}
+
+struct input_stream {
+	const void *(*read)(struct input_stream *, unsigned long *len);
+	void *data;
+	int is_finished;
+};
+
+int write_object_file_literally(const void *buf, unsigned long len,
+				const char *type, struct object_id *oid,
+				unsigned flags);
+int stream_loose_object(struct input_stream *in_stream, size_t len,
+			struct object_id *oid);
+
+int force_object_loose(const struct object_id *oid, time_t mtime);
+
 /**
  * With in-core object data in "buf", rehash it to make sure the
  * object name actually matches "oid" to detect object corruption.
@@ -117,6 +142,10 @@ int check_object_signature(struct repository *r, const struct object_id *oid,
  */
 int stream_object_signature(struct repository *r, const struct object_id *oid);
 
+int loose_object_info(struct repository *r,
+		      const struct object_id *oid,
+		      struct object_info *oi, int flags);
+
 enum finalize_object_file_flags {
 	FOF_SKIP_COLLISION_CHECK = 1,
 };
@@ -128,10 +157,18 @@ int finalize_object_file_flags(const char *tmpfile, const char *filename,
 /* Helper to check and "touch" a file */
 int check_and_freshen_file(const char *fn, int freshen);
 
-void *read_object_with_reference(struct repository *r,
-				 const struct object_id *oid,
-				 enum object_type required_type,
-				 unsigned long *size,
-				 struct object_id *oid_ret);
+/*
+ * Open the loose object at path, check its hash, and return the contents,
+ * use the "oi" argument to assert things about the object, or e.g. populate its
+ * type, and size. If the object is a blob, then "contents" may return NULL,
+ * to allow streaming of large blobs.
+ *
+ * Returns 0 on success, negative on error (details may be written to stderr).
+ */
+int read_loose_object(const char *path,
+		      const struct object_id *expected_oid,
+		      struct object_id *real_oid,
+		      void **contents,
+		      struct object_info *oi);
 
 #endif /* OBJECT_FILE_H */

object-name.c

@@ -19,7 +19,7 @@
 #include "oidtree.h"
 #include "packfile.h"
 #include "pretty.h"
-#include "object-store-ll.h"
+#include "object-store.h"
 #include "read-cache-ll.h"
 #include "repo-settings.h"
 #include "repository.h"