From b6b468b2bfaa8015568efad8e0a78b342e1ac640 Mon Sep 17 00:00:00 2001 From: Victor Leschuk Date: Tue, 15 Dec 2015 18:31:39 +0300 Subject: [PATCH 1/3] grep: allow threading even on a single-core machine Earlier we disabled threading when online_cpus() said "1", but on a filesystem with long latency (or in a cold cache situation), using multiple threads to drive I/O in parallel would improve performance even on a single-core machines. Signed-off-by: Victor Leschuk Signed-off-by: Junio C Hamano --- builtin/grep.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/grep.c b/builtin/grep.c index d04f4400d9..734d6ceaae 100644 --- a/builtin/grep.c +++ b/builtin/grep.c @@ -832,7 +832,7 @@ int cmd_grep(int argc, const char **argv, const char *prefix) } #ifndef NO_PTHREADS - if (list.nr || cached || online_cpus() == 1) + if (list.nr || cached) use_threads = 0; #else use_threads = 0; From 044b1f3cb4ff09551f6c514ed02edafa6c4987b0 Mon Sep 17 00:00:00 2001 From: Victor Leschuk Date: Tue, 15 Dec 2015 18:31:39 +0300 Subject: [PATCH 2/3] grep: slight refactoring to the code that disables threading When show-in-pager option is used, threading is unconditionally disabled, but this happened much earlier than the code that determines the use of threading based on the operand (i.e. we do not thread search in the object database). Consolidate the code to disable threading to just one place. Signed-off-by: Victor Leschuk Signed-off-by: Junio C Hamano --- builtin/grep.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/builtin/grep.c b/builtin/grep.c index 734d6ceaae..a892967467 100644 --- a/builtin/grep.c +++ b/builtin/grep.c @@ -801,7 +801,6 @@ int cmd_grep(int argc, const char **argv, const char *prefix) opt.output_priv = &path_list; opt.output = append_path; string_list_append(&path_list, show_in_pager); - use_threads = 0; } if (!opt.pattern_list) @@ -832,7 +831,7 @@ int cmd_grep(int argc, const char **argv, const char *prefix) } #ifndef NO_PTHREADS - if (list.nr || cached) + if (list.nr || cached || show_in_pager) use_threads = 0; #else use_threads = 0; From 89f09dd34e613394c749f69ddd74adc90cb034ca Mon Sep 17 00:00:00 2001 From: Victor Leschuk Date: Tue, 15 Dec 2015 18:31:39 +0300 Subject: [PATCH 3/3] grep: add --threads= option and grep.threads configuration "git grep" can now be configured (or told from the command line) how many threads to use when searching in the working tree files. Signed-off-by: Victor Leschuk Signed-off-by: Junio C Hamano --- Documentation/config.txt | 4 +++ Documentation/git-grep.txt | 9 ++++++ builtin/grep.c | 43 ++++++++++++++++++-------- contrib/completion/git-completion.bash | 1 + 4 files changed, 44 insertions(+), 13 deletions(-) diff --git a/Documentation/config.txt b/Documentation/config.txt index 391a0c3c85..ff2179901e 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -1447,6 +1447,10 @@ grep.extendedRegexp:: option is ignored when the 'grep.patternType' option is set to a value other than 'default'. +grep.threads:: + Number of grep worker threads to use. + See `grep.threads` in linkgit:git-grep[1] for more information. + gpg.program:: Use this custom program instead of "gpg" found on $PATH when making or verifying a PGP signature. The program must support the diff --git a/Documentation/git-grep.txt b/Documentation/git-grep.txt index 4a44d6da13..35aada1447 100644 --- a/Documentation/git-grep.txt +++ b/Documentation/git-grep.txt @@ -23,6 +23,7 @@ SYNOPSIS [--break] [--heading] [-p | --show-function] [-A ] [-B ] [-C ] [-W | --function-context] + [--threads ] [-f ] [-e] [--and|--or|--not|(|)|-e ...] [ [--[no-]exclude-standard] [--cached | --no-index | --untracked] | ...] @@ -53,6 +54,10 @@ grep.extendedRegexp:: option is ignored when the 'grep.patternType' option is set to a value other than 'default'. +grep.threads:: + Number of grep worker threads to use. If unset (or set to 0), + 8 threads are used by default (for now). + grep.fullName:: If set to true, enable '--full-name' option by default. @@ -227,6 +232,10 @@ OPTIONS effectively showing the whole function in which the match was found. +--threads :: + Number of grep worker threads to use. + See `grep.threads` in 'CONFIGURATION' for more information. + -f :: Read patterns from , one per line. diff --git a/builtin/grep.c b/builtin/grep.c index a892967467..97d9ec3308 100644 --- a/builtin/grep.c +++ b/builtin/grep.c @@ -24,11 +24,11 @@ static char const * const grep_usage[] = { NULL }; -static int use_threads = 1; +#define GREP_NUM_THREADS_DEFAULT 8 +static int num_threads; #ifndef NO_PTHREADS -#define THREADS 8 -static pthread_t threads[THREADS]; +static pthread_t *threads; /* We use one producer thread and THREADS consumer * threads. The producer adds struct work_items to 'todo' and the @@ -63,13 +63,13 @@ static pthread_mutex_t grep_mutex; static inline void grep_lock(void) { - if (use_threads) + if (num_threads) pthread_mutex_lock(&grep_mutex); } static inline void grep_unlock(void) { - if (use_threads) + if (num_threads) pthread_mutex_unlock(&grep_mutex); } @@ -206,7 +206,8 @@ static void start_threads(struct grep_opt *opt) strbuf_init(&todo[i].out, 0); } - for (i = 0; i < ARRAY_SIZE(threads); i++) { + threads = xcalloc(num_threads, sizeof(*threads)); + for (i = 0; i < num_threads; i++) { int err; struct grep_opt *o = grep_opt_dup(opt); o->output = strbuf_out; @@ -238,12 +239,14 @@ static int wait_all(void) pthread_cond_broadcast(&cond_add); grep_unlock(); - for (i = 0; i < ARRAY_SIZE(threads); i++) { + for (i = 0; i < num_threads; i++) { void *h; pthread_join(threads[i], &h); hit |= (int) (intptr_t) h; } + free(threads); + pthread_mutex_destroy(&grep_mutex); pthread_mutex_destroy(&grep_read_mutex); pthread_mutex_destroy(&grep_attr_mutex); @@ -267,6 +270,14 @@ static int grep_cmd_config(const char *var, const char *value, void *cb) int st = grep_config(var, value, cb); if (git_color_default_config(var, value, cb) < 0) st = -1; + + if (!strcmp(var, "grep.threads")) { + num_threads = git_config_int(var, value); + if (num_threads < 0) + die(_("invalid number of threads specified (%d) for %s"), + num_threads, var); + } + return st; } @@ -294,7 +305,7 @@ static int grep_sha1(struct grep_opt *opt, const unsigned char *sha1, } #ifndef NO_PTHREADS - if (use_threads) { + if (num_threads) { add_work(opt, GREP_SOURCE_SHA1, pathbuf.buf, path, sha1); strbuf_release(&pathbuf); return 0; @@ -323,7 +334,7 @@ static int grep_file(struct grep_opt *opt, const char *filename) strbuf_addstr(&buf, filename); #ifndef NO_PTHREADS - if (use_threads) { + if (num_threads) { add_work(opt, GREP_SOURCE_FILE, buf.buf, filename, filename); strbuf_release(&buf); return 0; @@ -702,6 +713,8 @@ int cmd_grep(int argc, const char **argv, const char *prefix) N_("show context lines before matches")), OPT_INTEGER('A', "after-context", &opt.post_context, N_("show context lines after matches")), + OPT_INTEGER(0, "threads", &num_threads, + N_("use worker threads")), OPT_NUMBER_CALLBACK(&opt, N_("shortcut for -C NUM"), context_callback), OPT_BOOL('p', "show-function", &opt.funcname, @@ -832,13 +845,17 @@ int cmd_grep(int argc, const char **argv, const char *prefix) #ifndef NO_PTHREADS if (list.nr || cached || show_in_pager) - use_threads = 0; + num_threads = 0; + else if (num_threads == 0) + num_threads = GREP_NUM_THREADS_DEFAULT; + else if (num_threads < 0) + die(_("invalid number of threads specified (%d)"), num_threads); #else - use_threads = 0; + num_threads = 0; #endif #ifndef NO_PTHREADS - if (use_threads) { + if (num_threads) { if (!(opt.name_only || opt.unmatch_name_only || opt.count) && (opt.pre_context || opt.post_context || opt.file_break || opt.funcbody)) @@ -908,7 +925,7 @@ int cmd_grep(int argc, const char **argv, const char *prefix) hit = grep_objects(&opt, &pathspec, &list); } - if (use_threads) + if (num_threads) hit |= wait_all(); if (hit && show_in_pager) run_pager(&opt, prefix); diff --git a/contrib/completion/git-completion.bash b/contrib/completion/git-completion.bash index 482ca84b45..390d9c04c0 100644 --- a/contrib/completion/git-completion.bash +++ b/contrib/completion/git-completion.bash @@ -1310,6 +1310,7 @@ _git_grep () --full-name --line-number --extended-regexp --basic-regexp --fixed-strings --perl-regexp + --threads --files-with-matches --name-only --files-without-match --max-depth