dir.c: git-status --ignored: don't scan the work tree twice

'git-status --ignored' still scans the work tree twice to collect
untracked and ignored files, respectively.

fill_directory / read_directory already supports collecting untracked and
ignored files in a single directory scan. However, the DIR_COLLECT_IGNORED
flag to enable this has some git-add specific side-effects (e.g. it
doesn't recurse into ignored directories, so listing ignored files with
--untracked=all doesn't work).

The DIR_SHOW_IGNORED flag doesn't list untracked files and returns ignored
files in dir_struct.entries[] (instead of dir_struct.ignored[] as
DIR_COLLECT_IGNORED). DIR_SHOW_IGNORED is used all throughout git.

We don't want to break the existing API, so lets introduce a new flag
DIR_SHOW_IGNORED_TOO that lists untracked as well as ignored files similar
to DIR_COLLECT_FILES, but will recurse into sub-directories based on the
other flags as DIR_SHOW_IGNORED does.

In dir.c::read_directory_recursive, add ignored files to either
dir_struct.entries[] or dir_struct.ignored[] based on the flags. Also move
the DIR_COLLECT_IGNORED case here so that filling result lists is in a
common place.

In wt-status.c::wt_status_collect_untracked, use the new flag and read
results from dir_struct.ignored[]. Remove the extra fill_directory call.

builtin/check-ignore.c doesn't call fill_directory, setting the git-add
specific DIR_COLLECT_IGNORED flag has no effect here. Remove for clarity.

Update API documentation to reflect the changes.

Performance: with this patch, 'git-status --ignored' is typically as fast
as 'git-status'.

Signed-off-by: Karsten Blees <blees@dcon.de>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

Documentation/technical/api-directory-listing.txt

@@ -22,12 +22,23 @@ The notable options are:
 
 `flags`::
 
-	A bit-field of options:
+	A bit-field of options (the `*IGNORED*` flags are mutually exclusive):
 
 `DIR_SHOW_IGNORED`:::
 
-	The traversal is for finding just ignored files, not unignored
-	files.
+	Return just ignored files in `entries[]`, not untracked files.
+
+`DIR_SHOW_IGNORED_TOO`:::
+
+	Similar to `DIR_SHOW_IGNORED`, but return ignored files in `ignored[]`
+	in addition to untracked files in `entries[]`.
+
+`DIR_COLLECT_IGNORED`:::
+
+	Special mode for git-add. Return ignored files in `ignored[]` and
+	untracked files in `entries[]`. Only returns ignored files that match
+	pathspec exactly (no wildcards). Does not recurse into ignored
+	directories.
 
 `DIR_SHOW_OTHER_DIRECTORIES`:::
 
@@ -57,6 +68,14 @@ The result of the enumeration is left in these fields:
 
 	Internal use; keeps track of allocation of `entries[]` array.
 
+`ignored[]`::
+
+	An array of `struct dir_entry`, used for ignored paths with the
+	`DIR_SHOW_IGNORED_TOO` and `DIR_COLLECT_IGNORED` flags.
+
+`ignored_nr`::
+
+	The number of members in `ignored[]` array.
 
 Calling sequence
 ----------------

builtin/check-ignore.c

@@ -66,7 +66,6 @@ static int check_ignore(const char *prefix, const char **pathspec)
 		die(_("index file corrupt"));
 
 	memset(&dir, 0, sizeof(dir));
-	dir.flags |= DIR_COLLECT_IGNORED;
 	setup_standard_excludes(&dir);
 
 	if (!pathspec || !*pathspec) {

dir.c

@@ -1183,15 +1183,12 @@ static enum path_treatment treat_one_path(struct dir_struct *dir,
 		return path_none;
 
 	exclude = is_excluded(dir, path->buf, &dtype);
-	if (exclude && (dir->flags & DIR_COLLECT_IGNORED)
-	    && exclude_matches_pathspec(path->buf, path->len, simplify))
-		dir_add_ignored(dir, path->buf, path->len);
 
 	/*
 	 * Excluded? If we don't explicitly want to show
 	 * ignored files, ignore it
 	 */
-	if (exclude && !(dir->flags & DIR_SHOW_IGNORED))
+	if (exclude && !(dir->flags & (DIR_SHOW_IGNORED|DIR_SHOW_IGNORED_TOO)))
 		return path_excluded;
 
 	switch (dtype) {
@@ -1280,6 +1277,11 @@ static enum path_treatment read_directory_recursive(struct dir_struct *dir,
 		case path_excluded:
 			if (dir->flags & DIR_SHOW_IGNORED)
 				dir_add_name(dir, path.buf, path.len);
+			else if ((dir->flags & DIR_SHOW_IGNORED_TOO) ||
+				((dir->flags & DIR_COLLECT_IGNORED) &&
+				exclude_matches_pathspec(path.buf, path.len,
+					simplify)))
+				dir_add_ignored(dir, path.buf, path.len);
 			break;
 
 		case path_untracked:

dir.h

@@ -79,7 +79,8 @@ struct dir_struct {
 		DIR_SHOW_OTHER_DIRECTORIES = 1<<1,
 		DIR_HIDE_EMPTY_DIRECTORIES = 1<<2,
 		DIR_NO_GITLINKS = 1<<3,
-		DIR_COLLECT_IGNORED = 1<<4
+		DIR_COLLECT_IGNORED = 1<<4,
+		DIR_SHOW_IGNORED_TOO = 1<<5
 	} flags;
 	struct dir_entry **entries;
 	struct dir_entry **ignored;

wt-status.c

@@ -511,9 +511,12 @@ static void wt_status_collect_untracked(struct wt_status *s)
 	if (s->show_untracked_files != SHOW_ALL_UNTRACKED_FILES)
 		dir.flags |=
 			DIR_SHOW_OTHER_DIRECTORIES | DIR_HIDE_EMPTY_DIRECTORIES;
+	if (s->show_ignored_files)
+		dir.flags |= DIR_SHOW_IGNORED_TOO;
 	setup_standard_excludes(&dir);
 
 	fill_directory(&dir, s->pathspec);
+
 	for (i = 0; i < dir.nr; i++) {
 		struct dir_entry *ent = dir.entries[i];
 		if (cache_name_is_other(ent->name, ent->len) &&
@@ -522,22 +525,17 @@ static void wt_status_collect_untracked(struct wt_status *s)
 		free(ent);
 	}
 
-	if (s->show_ignored_files) {
-		dir.nr = 0;
-		dir.flags = DIR_SHOW_IGNORED;
-		if (s->show_untracked_files != SHOW_ALL_UNTRACKED_FILES)
-			dir.flags |= DIR_SHOW_OTHER_DIRECTORIES | DIR_HIDE_EMPTY_DIRECTORIES;
-		fill_directory(&dir, s->pathspec);
-		for (i = 0; i < dir.nr; i++) {
-			struct dir_entry *ent = dir.entries[i];
+	for (i = 0; i < dir.ignored_nr; i++) {
+		struct dir_entry *ent = dir.ignored[i];
 		if (cache_name_is_other(ent->name, ent->len) &&
 		    match_pathspec(s->pathspec, ent->name, ent->len, 0, NULL))
 			string_list_insert(&s->ignored, ent->name);
 		free(ent);
 	}
-	}
 
 	free(dir.entries);
+	free(dir.ignored);
+	clear_directory(&dir);
 
 	if (advice_status_u_option) {
 		struct timeval t_end;