diff --git a/dir.c b/dir.c
index 325fb56ad3..532bcb65b5 100644
--- a/dir.c
+++ b/dir.c
@@ -1192,7 +1192,7 @@ int remove_dir_recursively(struct strbuf *path, int flag)
 
 	dir = opendir(path->buf);
 	if (!dir)
-		return -1;
+		return rmdir(path->buf);
 	if (path->buf[original_len - 1] != '/')
 		strbuf_addch(path, '/');
 
diff --git a/t/t7300-clean.sh b/t/t7300-clean.sh
index 7e1be44402..800b5368a5 100755
--- a/t/t7300-clean.sh
+++ b/t/t7300-clean.sh
@@ -453,4 +453,11 @@ test_expect_success 'git clean -e' '
 	)
 '
 
+test_expect_success SANITY 'git clean -d with an unreadable empty directory' '
+	mkdir foo &&
+	chmod a= foo &&
+	git clean -dfx foo &&
+	! test -d foo
+'
+
 test_done