Merge branch 'tb/pack-bitmap-lookup-tables' into jch

Enable lookup tables extension in pack bitmap (and midx bitmap) by
default.

Comments?

* tb/pack-bitmap-lookup-tables:
  t/perf/lib-bitmap.sh: avoid test_perf during setup
  t/perf: avoid testing bitmaps without lookup table
  p5312: removed duplicate performance test script
  pack-bitmap: write lookup table extension by default

Documentation/config/pack.adoc

@@ -199,7 +199,7 @@ pack.writeBitmapLookupTable::
 	bitmap index (if one is written). This table is used to defer
 	loading individual bitmaps as late as possible. This can be
 	beneficial in repositories that have relatively large bitmap
-	indexes. Defaults to false.
+	indexes. Defaults to true.
 
 pack.readReverseIndex::
 	When true, git will read any .rev file(s) that may be available

builtin/multi-pack-index.c

@@ -142,6 +142,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv,
 	int ret;
 
 	opts.flags |= MIDX_WRITE_BITMAP_HASH_CACHE;
+	opts.flags |= MIDX_WRITE_BITMAP_LOOKUP_TABLE;
 
 	git_config(git_multi_pack_index_write_config, NULL);
 

builtin/pack-objects.c

@@ -250,7 +250,7 @@ static enum {
 	WRITE_BITMAP_QUIET,
 	WRITE_BITMAP_TRUE,
 } write_bitmap_index;
-static uint16_t write_bitmap_options = BITMAP_OPT_HASH_CACHE;
+static uint16_t write_bitmap_options = BITMAP_OPT_HASH_CACHE | BITMAP_OPT_LOOKUP_TABLE;
 
 static int exclude_promisor_objects;
 static int exclude_promisor_objects_best_effort;

t/meson.build

@@ -1131,7 +1131,6 @@ benchmarks = [
   'perf/p5304-prune.sh',
   'perf/p5310-pack-bitmaps.sh',
   'perf/p5311-pack-bitmaps-fetch.sh',
-  'perf/p5312-pack-bitmaps-revs.sh',
   'perf/p5313-pack-objects.sh',
   'perf/p5314-name-hash.sh',
   'perf/p5326-multi-pack-bitmaps.sh',

t/perf/lib-bitmap.sh

@@ -69,7 +69,7 @@ test_partial_bitmap () {
 }
 
 test_pack_bitmap () {
-	test_perf "repack to disk" '
+	test_expect_success "repack to disk" '
 		git repack -ad
 	'
 

t/perf/p5310-pack-bitmaps.sh

@@ -4,37 +4,28 @@ test_description='Tests pack performance using bitmaps'
 . ./perf-lib.sh
 . "${TEST_DIRECTORY}/perf/lib-bitmap.sh"
 
-test_lookup_pack_bitmap () {
+test_expect_success 'start the test from scratch' '
-	test_expect_success 'start the test from scratch' '
 	rm -rf * .git
-	'
+'
 
-	test_perf_large_repo
+test_perf_large_repo
 
-	# note that we do everything through config,
+# note that we do everything through config,
-	# since we want to be able to compare bitmap-aware
+# since we want to be able to compare bitmap-aware
-	# git versus non-bitmap git
+# git versus non-bitmap git
-	#
+#
-	# We intentionally use the deprecated pack.writebitmaps
+# We intentionally use the deprecated pack.writebitmaps
-	# config so that we can test against older versions of git.
+# config so that we can test against older versions of git.
-	test_expect_success 'setup bitmap config' '
+test_expect_success 'setup bitmap config' '
 	git config pack.writebitmaps true
-	'
+'
 
-	# we need to create the tag up front such that it is covered by the repack and
+# we need to create the tag up front such that it is covered by the repack and
-	# thus by generated bitmaps.
+# thus by generated bitmaps.
-	test_expect_success 'create tags' '
+test_expect_success 'create tags' '
 	git tag --message="tag pointing to HEAD" perf-tag HEAD
-	'
+'
 
-	test_perf "enable lookup table: $1" '
+test_pack_bitmap
-		git config pack.writeBitmapLookupTable '"$1"'
-	'
-
-	test_pack_bitmap
-}
-
-test_lookup_pack_bitmap false
-test_lookup_pack_bitmap true
 
 test_done

t/perf/p5311-pack-bitmaps-fetch.sh

@@ -3,24 +3,22 @@
 test_description='performance of fetches from bitmapped packs'
 . ./perf-lib.sh
 
-test_fetch_bitmaps () {
+test_expect_success 'setup test directory' '
-	test_expect_success 'setup test directory' '
 	rm -fr * .git
-	'
+'
 
-	test_perf_default_repo
+test_perf_default_repo
 
-	test_expect_success 'create bitmapped server repo' '
+test_expect_success 'create bitmapped server repo' '
 	git config pack.writebitmaps true &&
-		git config pack.writeBitmapLookupTable '"$1"' &&
 	git repack -ad
-	'
+'
 
-	# simulate a fetch from a repository that last fetched N days ago, for
+# simulate a fetch from a repository that last fetched N days ago, for
-	# various values of N. We do so by following the first-parent chain,
+# various values of N. We do so by following the first-parent chain,
-	# and assume the first entry in the chain that is N days older than the current
+# and assume the first entry in the chain that is N days older than the current
-	# HEAD is where the HEAD would have been then.
+# HEAD is where the HEAD would have been then.
-	for days in 1 2 4 8 16 32 64 128; do
+for days in 1 2 4 8 16 32 64 128; do
 	title=$(printf '%10s' "($days days)")
 	test_expect_success "setup revs from $days days ago" '
 		now=$(git log -1 --format=%ct HEAD) &&
@@ -32,7 +30,7 @@ test_fetch_bitmaps () {
 		} >revs
 	'
 
-		test_perf "server $title (lookup=$1)" '
+	test_perf "server $title" '
 		git pack-objects --stdout --revs \
 				--thin --delta-base-offset \
 				<revs >tmp.pack
@@ -42,13 +40,9 @@ test_fetch_bitmaps () {
 		test_file_size tmp.pack
 	'
 
-		test_perf "client $title (lookup=$1)" '
+	test_perf "client $title" '
 		git index-pack --stdin --fix-thin <tmp.pack
 	'
-	done
+done
-}
-
-test_fetch_bitmaps true
-test_fetch_bitmaps false
 
 test_done

t/perf/p5312-pack-bitmaps-revs.sh

@@ -1,34 +0,0 @@
-#!/bin/sh
-
-test_description='Tests pack performance using bitmaps (rev index enabled)'
-. ./perf-lib.sh
-. "${TEST_DIRECTORY}/perf/lib-bitmap.sh"
-
-test_lookup_pack_bitmap () {
-	test_expect_success 'start the test from scratch' '
-		rm -rf * .git
-	'
-
-	test_perf_large_repo
-
-	test_expect_success 'setup bitmap config' '
-		git config pack.writebitmaps true
-	'
-
-	# we need to create the tag up front such that it is covered by the repack and
-	# thus by generated bitmaps.
-	test_expect_success 'create tags' '
-		git tag --message="tag pointing to HEAD" perf-tag HEAD
-	'
-
-	test_perf "enable lookup table: $1" '
-		git config pack.writeBitmapLookupTable '"$1"'
-	'
-
-	test_pack_bitmap
-}
-
-test_lookup_pack_bitmap false
-test_lookup_pack_bitmap true
-
-test_done

t/perf/p5326-multi-pack-bitmaps.sh

@@ -4,40 +4,33 @@ test_description='Tests performance using midx bitmaps'
 . ./perf-lib.sh
 . "${TEST_DIRECTORY}/perf/lib-bitmap.sh"
 
-test_bitmap () {
+test_expect_success "remove existing repo" '
-	local enabled="$1"
-
-	test_expect_success "remove existing repo (lookup=$enabled)" '
 	rm -fr * .git
-	'
+'
 
-	test_perf_large_repo
+test_perf_large_repo
 
-	# we need to create the tag up front such that it is covered by the repack and
+# we need to create the tag up front such that it is covered by the repack and
-	# thus by generated bitmaps.
+# thus by generated bitmaps.
-	test_expect_success 'create tags' '
+test_expect_success 'create tags' '
 	git tag --message="tag pointing to HEAD" perf-tag HEAD
-	'
+'
 
-	test_expect_success "use lookup table: $enabled" '
+test_expect_success "start with bitmapped pack" '
-		git config pack.writeBitmapLookupTable '"$enabled"'
-	'
-
-	test_expect_success "start with bitmapped pack (lookup=$enabled)" '
 	git repack -adb
-	'
+'
 
-	test_perf "setup multi-pack index (lookup=$enabled)" '
+test_perf "setup multi-pack index" '
 	git multi-pack-index write --bitmap
-	'
+'
 
-	test_expect_success "drop pack bitmap (lookup=$enabled)" '
+test_expect_success "drop pack bitmap" '
 	rm -f .git/objects/pack/pack-*.bitmap
-	'
+'
 
-	test_full_bitmap
+test_full_bitmap
 
-	test_expect_success "create partial bitmap state (lookup=$enabled)" '
+test_expect_success "create partial bitmap state" '
 	# pick a commit to represent the repo tip in the past
 	cutoff=$(git rev-list HEAD~100 -1) &&
 	orig_tip=$(git rev-parse HEAD) &&
@@ -56,12 +49,8 @@ test_bitmap () {
 	# and now restore our original tip, as if the pushes
 	# had happened
 	git update-ref HEAD $orig_tip
-	'
+'
 
-	test_partial_bitmap
+test_partial_bitmap
-}
-
-test_bitmap false
-test_bitmap true
 
 test_done

t/perf/p5333-pseudo-merge-bitmaps.sh

@@ -11,7 +11,6 @@ test_expect_success 'setup' '
 		-c bitmapPseudoMerge.all.threshold=now \
 		-c bitmapPseudoMerge.all.stableThreshold=never \
 		-c bitmapPseudoMerge.all.maxMerges=64 \
-		-c pack.writeBitmapLookupTable=true \
 		repack -adb
 '