diff --git a/compat/mingw-posix.h b/compat/mingw-posix.h
index 8dddfa818d..88e0cf9292 100644
--- a/compat/mingw-posix.h
+++ b/compat/mingw-posix.h
@@ -201,8 +201,12 @@ int uname(struct utsname *buf);
  * replacements of existing functions
  */
 
-int mingw_unlink(const char *pathname);
-#define unlink mingw_unlink
+int mingw_unlink(const char *pathname, int handle_in_use_error);
+#ifdef MINGW_DONT_HANDLE_IN_USE_ERROR
+# define unlink(path) mingw_unlink(path, 0)
+#else
+# define unlink(path) mingw_unlink(path, 1)
+#endif
 
 int mingw_rmdir(const char *path);
 #define rmdir mingw_rmdir
diff --git a/compat/mingw.c b/compat/mingw.c
index 305a999f1f..6ea2e7a35b 100644
--- a/compat/mingw.c
+++ b/compat/mingw.c
@@ -302,7 +302,7 @@ static wchar_t *normalize_ntpath(wchar_t *wbuf)
 	return wbuf;
 }
 
-int mingw_unlink(const char *pathname)
+int mingw_unlink(const char *pathname, int handle_in_use_error)
 {
 	int ret, tries = 0;
 	wchar_t wpathname[MAX_PATH];
@@ -317,6 +317,9 @@ int mingw_unlink(const char *pathname)
 	while ((ret = _wunlink(wpathname)) == -1 && tries < ARRAY_SIZE(delay)) {
 		if (!is_file_in_use_error(GetLastError()))
 			break;
+		if (!handle_in_use_error)
+			return ret;
+
 		/*
 		 * We assume that some other process had the source or
 		 * destination file open at the wrong moment and retry.
diff --git a/reftable/system.h b/reftable/system.h
index 10055fbff2..072d9daea0 100644
--- a/reftable/system.h
+++ b/reftable/system.h
@@ -11,6 +11,7 @@ https://developers.google.com/open-source/licenses/bsd
 
 /* This header glues the reftable library to the rest of Git */
 
+#define MINGW_DONT_HANDLE_IN_USE_ERROR
 #include "compat/posix.h"
 #include "compat/zlib-compat.h"