diff --git a/Documentation/git-ls-files.adoc b/Documentation/git-ls-files.adoc index 2b175388e1..11323b4ba9 100644 --- a/Documentation/git-ls-files.adoc +++ b/Documentation/git-ls-files.adoc @@ -127,6 +127,10 @@ OPTIONS --exclude-standard:: Add the standard Git exclusions: .git/info/exclude, .gitignore in each directory, and the user's global exclusion file. ++ +With `--others`, an enabled untracked cache can be updated in the index +after a complete traversal. Use `--no-optional-locks` (see linkgit:git[1]) +to disable this optional index update. --error-unmatch:: If any does not appear in the index, treat this as an diff --git a/builtin/ls-files.c b/builtin/ls-files.c index b044520f9e..e74ffa344d 100644 --- a/builtin/ls-files.c +++ b/builtin/ls-files.c @@ -15,6 +15,7 @@ #include "quote.h" #include "dir.h" #include "gettext.h" +#include "lockfile.h" #include "object-name.h" #include "strbuf.h" #include "parse-options.h" @@ -597,6 +598,8 @@ int cmd_ls_files(int argc, struct dir_struct dir = DIR_INIT; struct pattern_list *pl; struct string_list exclude_list = STRING_LIST_INIT_NODUP; + struct lock_file index_lock = LOCK_INIT; + int index_fd = -1; struct option builtin_ls_files_options[] = { /* Think twice before adding "--nul" synonym to this */ OPT_SET_INT('z', NULL, &line_terminator, @@ -678,14 +681,13 @@ int cmd_ls_files(int argc, prefix_len = strlen(prefix); repo_config(repo, git_default_config, NULL); - if (repo_read_index(repo) < 0) - die("index file corrupt"); - argc = parse_options(argc, argv, prefix, builtin_ls_files_options, ls_files_usage, 0); - pl = add_pattern_list(&dir, EXC_CMDL, "--exclude option"); - for (i = 0; i < exclude_list.nr; i++) { - add_pattern(exclude_list.items[i].string, "", 0, pl, --exclude_args); + /* Even an empty command-line exclude list would disable the cache. */ + if (exclude_list.nr) { + pl = add_pattern_list(&dir, EXC_CMDL, "--exclude option"); + for (i = 0; i < exclude_list.nr; i++) + add_pattern(exclude_list.items[i].string, "", 0, pl, --exclude_args); } if (format && (show_stage || show_others || show_killed || @@ -747,6 +749,25 @@ int cmd_ls_files(int argc, max_prefix = common_prefix(&pathspec); max_prefix_len = get_common_prefix_len(max_prefix); + /* + * Do not save an index pruned by a pathspec or changed by --with-tree. + * With index.skipHash, the checksum cannot detect concurrent index + * changes. Lock the index before reading it. + */ + if (show_others && !dir.flags && dir.exclude_per_dir && + !max_prefix && !with_tree && use_optional_locks() && + !dir.internal.unmanaged_exclude_files && !exclude_list.nr && + repo->settings.core_untracked_cache != UNTRACKED_CACHE_REMOVE) + index_fd = repo_hold_locked_index(repo, &index_lock, 0); + if (repo_read_index(repo) < 0) + die("index file corrupt"); + if (show_others && !dir.flags && dir.exclude_per_dir && !with_tree) + dir.untracked = repo->index->untracked; + if (index_fd >= 0 && !dir.untracked) { + rollback_lock_file(&index_lock); + index_fd = -1; + } + prune_index(repo->index, max_prefix, max_prefix_len); /* Treat unmatching pathspec elements as errors */ @@ -784,6 +805,13 @@ int cmd_ls_files(int argc, ret = 1; } + if (index_fd >= 0) { + if (!ret && dir.untracked) + repo_update_index_if_able(repo, &index_lock); + else + rollback_lock_file(&index_lock); + } + string_list_clear(&exclude_list, 0); dir_clear(&dir); free(max_prefix); diff --git a/dir.c b/dir.c index 425fe2af67..949f06c8f7 100644 --- a/dir.c +++ b/dir.c @@ -1209,7 +1209,8 @@ static int add_patterns(const char *fname, const char *base, int baselen, close(fd); if (oid_stat) { int pos; - if (oid_stat->valid && + /* Racy stat checks need the index timestamp. */ + if (istate && oid_stat->valid && !match_stat_data_racy(istate, &oid_stat->stat, &st)) ; /* no content change, oid_stat->oid still good */ else if (istate && @@ -1321,18 +1322,15 @@ struct pattern_list *add_pattern_list(struct dir_struct *dir, } /* - * Used to set up core.excludesfile and .git/info/exclude lists. + * Only the standard exclude files have object IDs saved in the untracked + * cache. Other files have no oid_stat and must disable use of the cache. */ static void add_patterns_from_file_1(struct dir_struct *dir, const char *fname, struct oid_stat *oid_stat) { struct pattern_list *pl; - /* - * catch setup_standard_excludes() that's called before - * dir->untracked is assigned. That function behaves - * differently when dir->untracked is non-NULL. - */ - if (!dir->untracked) + + if (!oid_stat) dir->internal.unmanaged_exclude_files++; pl = add_pattern_list(dir, EXC_FILE, fname); if (add_patterns(fname, "", 0, pl, NULL, 0, oid_stat) < 0) @@ -1341,7 +1339,6 @@ static void add_patterns_from_file_1(struct dir_struct *dir, const char *fname, void add_patterns_from_file(struct dir_struct *dir, const char *fname) { - dir->internal.unmanaged_exclude_files++; /* see validate_untracked_cache() */ add_patterns_from_file_1(dir, fname, NULL); } @@ -3009,10 +3006,7 @@ static struct untracked_cache_dir *validate_untracked_cache(struct dir_struct *d /* * We only support $GIT_COMMON_DIR/info/exclude and core.excludesfile - * as the global ignore rule files. Any other additions - * (e.g. from command line) invalidate the cache. This - * condition also catches running setup_standard_excludes() - * before setting dir->untracked! + * as the global ignore rule files. Other exclude files bypass the cache. */ if (dir->internal.unmanaged_exclude_files) return NULL; @@ -3511,17 +3505,21 @@ void setup_standard_excludes(struct dir_struct *dir) dir->exclude_per_dir = ".gitignore"; + /* + * Option parsing may precede reading the index. Record the object IDs + * even before the untracked cache is available for validation. + */ /* core.excludesfile defaulting to $XDG_CONFIG_HOME/git/ignore */ if (excludes_file && !access_or_warn(excludes_file, R_OK, 0)) add_patterns_from_file_1(dir, excludes_file, - dir->untracked ? &dir->internal.ss_excludes_file : NULL); + &dir->internal.ss_excludes_file); /* per repository user preference */ if (startup_info->have_repository) { const char *path = git_path_info_exclude(); if (!access_or_warn(path, R_OK, 0)) add_patterns_from_file_1(dir, path, - dir->untracked ? &dir->internal.ss_info_exclude : NULL); + &dir->internal.ss_info_exclude); } } diff --git a/t/perf/p3010-ls-files.sh b/t/perf/p3010-ls-files.sh index ae14449432..bdf398906f 100755 --- a/t/perf/p3010-ls-files.sh +++ b/t/perf/p3010-ls-files.sh @@ -28,4 +28,19 @@ test_perf 'ls-files --modified with pathspec' ' -- "$pathspec" >/dev/null ' +test_perf 'ls-files --others with pathspec and no untracked cache' ' + git -c core.fsmonitor=false -c core.untrackedCache=false \ + ls-files --cached --others --exclude-standard -- "$pathspec" >/dev/null +' + +test_expect_success 'populate the untracked cache with ls-files' ' + git config core.untrackedCache true && + git -c core.fsmonitor=false ls-files --others --exclude-standard >/dev/null +' + +test_perf 'ls-files --others with pathspec and untracked cache' ' + git -c core.fsmonitor=false ls-files --cached --others \ + --exclude-standard -- "$pathspec" >/dev/null +' + test_done diff --git a/t/t3001-ls-files-others-exclude.sh b/t/t3001-ls-files-others-exclude.sh index 29a0a25b30..5ca80932ac 100755 --- a/t/t3001-ls-files-others-exclude.sh +++ b/t/t3001-ls-files-others-exclude.sh @@ -306,4 +306,24 @@ test_expect_success 'ls-files with "**" patterns and no slashes' ' test_must_be_empty actual ' +test_expect_success 'ls-files preserves repeated and ordered exclude options' ' + test_create_repo exclude-options && + ( + cd exclude-options && + touch keep info-hidden global-hidden && + echo info-hidden >.git/info/exclude && + echo global-hidden >.git/global-ignore && + git config core.excludesFile .git/global-ignore && + echo "!info-hidden" >.git/custom-ignore && + echo keep >../expect && + git ls-files --others --exclude-from=.git/custom-ignore \ + --exclude-standard --exclude-standard >../actual && + test_cmp ../expect ../actual && + git ls-files --others --exclude-standard \ + --exclude-from=.git/custom-ignore >../actual && + printf "%s\n" info-hidden keep >../expect && + test_cmp ../expect ../actual + ) +' + test_done diff --git a/t/t7063-status-untracked-cache.sh b/t/t7063-status-untracked-cache.sh index 2efbb93b71..70eed91dd7 100755 --- a/t/t7063-status-untracked-cache.sh +++ b/t/t7063-status-untracked-cache.sh @@ -1016,6 +1016,9 @@ test_expect_success 'untracked cache handles nested repository changes' ' echo "?? nested/" >../expect && test_cmp ../expect ../actual && rm -rf nested/.git && + git ls-files --others --exclude-standard >../actual && + echo nested/file >../expect && + test_cmp ../expect ../actual && git status -uall --porcelain >../actual && echo "?? nested/file" >../expect && test_cmp ../expect ../actual && @@ -1048,4 +1051,98 @@ test_expect_success 'filtered status retains complete untracked listings' ' ) ' +test_expect_success 'ls-files expands cached directories and filters wildcard results' ' + test_create_repo ls-files-cache && + ( + cd ls-files-cache && + git config core.untrackedCache true && + mkdir tracked untracked empty ignored-only && + touch tracked/pyproject.toml untracked/pyproject.toml && + touch untracked/other ignored-only/file && + echo ignored-only/file >.gitignore && + echo "*.toml selected" >.gitattributes && + git add .gitignore .gitattributes tracked && + git init nested.git && + test-tool chmtime =-300 . tracked untracked empty ignored-only && + git status -unormal --porcelain >/dev/null && + # Complete the partial listing, then reuse it without opening directories. + for opened in 1 0 + do + : >"$TRASH_DIRECTORY/ls-files.trace" && + GIT_TRACE2_PERF="$TRASH_DIRECTORY/ls-files.trace" \ + git ls-files --cached --others --exclude-standard -z \ + -- "**/pyproject.toml" >../actual && + printf "%s\0" untracked/pyproject.toml tracked/pyproject.toml \ + >../expect && + test_cmp ../expect ../actual && + test_grep "read_directo.*opendir:$opened\$" \ + "$TRASH_DIRECTORY/ls-files.trace" && + test_grep "read_directo.*gitignore-invalidation:0\$" \ + "$TRASH_DIRECTORY/ls-files.trace" || return 1 + done && + for pathspec in "*.git/" ":(glob)**/*.toml" \ + ":(exclude)untracked/" ":(attr:selected)**/*.toml" + do + GIT_DISABLE_UNTRACKED_CACHE=1 git ls-files --others \ + --exclude-standard -- "$pathspec" >../expect && + git ls-files --others --exclude-standard \ + -- "$pathspec" >../actual && + test_cmp ../expect ../actual || return 1 + done + ) +' + +test_expect_success 'ls-files cache is reused after status -unormal' ' + test_create_repo persistent-cache && + ( + cd persistent-cache && + mkdir tracked untracked && + touch tracked/a tracked/b untracked/a untracked/b && + git add tracked && + git commit -m initial && + git config core.untrackedCache true && + test-tool chmtime =-300 . tracked untracked && + git ls-files --others --exclude-standard >../actual && + printf "%s\n" untracked/a untracked/b >../expect && + test_cmp ../expect ../actual && + git status --porcelain >../actual && + echo "?? untracked/" >../status-expect && + test_cmp ../status-expect ../actual && + GIT_TRACE2_PERF="$TRASH_DIRECTORY/persistent.trace" \ + git ls-files --others --exclude-standard >../actual && + test_cmp ../expect ../actual && + test_grep "read_directo.*opendir:0\$" "$TRASH_DIRECTORY/persistent.trace" + ) +' + +test_expect_success 'ls-files respects optional locks and a busy index lock' ' + test_when_finished "rm -f persistent-cache/.git/index.lock" && + ( + cd persistent-cache && + touch untracked/new && + cp .git/index ../saved-index && + git --no-optional-locks ls-files --others --exclude-standard >../actual && + test_cmp_bin ../saved-index .git/index && + printf "%s\n" untracked/a untracked/b untracked/new >../expect && + test_cmp ../expect ../actual && + touch .git/index.lock && + git ls-files --others --exclude-standard >../actual && + test_cmp ../expect ../actual && + test_cmp_bin ../saved-index .git/index + ) +' + +test_expect_success 'ls-files does not write the index with a pathspec prefix or --with-tree' ' + ( + cd persistent-cache && + cp .git/index ../saved-index && + git ls-files --cached --others --exclude-standard -- tracked/a >../actual && + test_cmp_bin ../saved-index .git/index && + git rm --cached tracked/b && + cp .git/index ../saved-index && + git ls-files --cached --others --exclude-standard --with-tree=HEAD >../actual && + test_cmp_bin ../saved-index .git/index + ) +' + test_done diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh index 93973ed25a..8feca101ab 100755 --- a/t/t7519-status-fsmonitor.sh +++ b/t/t7519-status-fsmonitor.sh @@ -477,4 +477,48 @@ test_expect_success 'status succeeds with sparse index' ' ) ' +test_expect_success UNTRACKED_CACHE 'ls-files saves the fsmonitor token with index.skipHash' ' + test_create_repo ls-files-fsmonitor && + ( + # index.skipHash gives shared indexes a null OID, so they cannot + # be reloaded. + sane_unset GIT_TEST_SPLIT_INDEX && + cd ls-files-fsmonitor && + touch tracked && + git add tracked && + git commit -m initial && + git config core.untrackedCache true && + git config index.skipHash true && + git config core.fsmonitor .git/hooks/fsmonitor-test && + test_hook --setup fsmonitor-test <<-\EOF && + printf "initial-token\0/\0" + EOF + mkdir untracked && + touch untracked/first && + git status --porcelain >/dev/null && + test_hook --clobber fsmonitor-test <<-\EOF && + test -f .git/index.lock || exit 1 + printf "%s\n" "$2" >.git/query-token + printf "next-token\0" + if test "$2" != next-token + then + printf "untracked/second\0" + fi + EOF + touch untracked/second && + git ls-files --others --exclude-standard >../actual && + printf "%s\n" untracked/first untracked/second >../expect && + test_cmp ../expect ../actual && + echo initial-token >../token-expect && + test_cmp ../token-expect .git/query-token && + GIT_TRACE2_PERF="$TRASH_DIRECTORY/ls-files-fsmonitor.trace" \ + git ls-files --others --exclude-standard >../actual && + test_cmp ../expect ../actual && + echo next-token >../token-expect && + test_cmp ../token-expect .git/query-token && + test_grep "read_directo.*opendir:0\$" \ + "$TRASH_DIRECTORY/ls-files-fsmonitor.trace" + ) +' + test_done