Merge branch 'vd/scalar-to-main'

Hoist the remainder of "scalar" out of contrib/ to the main part of
the codebase.

* vd/scalar-to-main:
  Documentation/technical: include Scalar technical doc
  t/perf: add 'GIT_PERF_USE_SCALAR' run option
  t/perf: add Scalar performance tests
  scalar-clone: add test coverage
  scalar: add to 'git help -a' command list
  scalar: implement the `help` subcommand
  git help: special-case `scalar`
  scalar: include in standard Git build & installation
  scalar: fix command documentation section header

t/perf/README

@@ -95,6 +95,10 @@ You can set the following variables (also in your config.mak):
 	Git (e.g., performance of index-pack as the number of threads
 	changes). These can be enabled with GIT_PERF_EXTRA.
 
+    GIT_PERF_USE_SCALAR
+	Boolean indicating whether to register test repo(s) with Scalar
+	before executing tests.
+
 You can also pass the options taken by ordinary git tests; the most
 useful one is:
 

t/perf/p9210-scalar.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+test_description='test scalar performance'
+. ./perf-lib.sh
+
+test_perf_large_repo "$TRASH_DIRECTORY/to-clone"
+
+test_expect_success 'enable server-side partial clone' '
+	git -C to-clone config uploadpack.allowFilter true &&
+	git -C to-clone config uploadpack.allowAnySHA1InWant true
+'
+
+test_perf 'scalar clone' '
+	rm -rf scalar-clone &&
+	scalar clone "file://$(pwd)/to-clone" scalar-clone
+'
+
+test_perf 'git clone' '
+	rm -rf git-clone &&
+	git clone "file://$(pwd)/to-clone" git-clone
+'
+
+test_compare_perf () {
+	command=$1
+	shift
+	args=$*
+	test_perf "$command $args (scalar)" "
+		$command -C scalar-clone/src $args
+	"
+
+	test_perf "$command $args (non-scalar)" "
+		$command -C git-clone $args
+	"
+}
+
+test_compare_perf git status
+test_compare_perf test_commit --append --no-tag A
+
+test_done

t/perf/perf-lib.sh

@@ -49,6 +49,9 @@ export TEST_DIRECTORY TRASH_DIRECTORY GIT_BUILD_DIR GIT_TEST_CMP
 MODERN_GIT=$GIT_BUILD_DIR/bin-wrappers/git
 export MODERN_GIT
 
+MODERN_SCALAR=$GIT_BUILD_DIR/bin-wrappers/scalar
+export MODERN_SCALAR
+
 perf_results_dir=$TEST_RESULTS_DIR
 test -n "$GIT_PERF_SUBSECTION" && perf_results_dir="$perf_results_dir/$GIT_PERF_SUBSECTION"
 mkdir -p "$perf_results_dir"
@@ -120,6 +123,10 @@ test_perf_create_repo_from () {
 			# status" due to a locked index. Since we have
 			# a copy it's fine to remove the lock.
 			rm .git/index.lock
+		fi &&
+		if test_bool_env GIT_PERF_USE_SCALAR false
+		then
+			"$MODERN_SCALAR" register
 		fi
 	) || error "failed to copy repository '$source' to '$repo'"
 }
@@ -130,7 +137,11 @@ test_perf_fresh_repo () {
 	"$MODERN_GIT" init -q "$repo" &&
 	(
 		cd "$repo" &&
-		test_perf_do_repo_symlink_config_
+		test_perf_do_repo_symlink_config_ &&
+		if test_bool_env GIT_PERF_USE_SCALAR false
+		then
+			"$MODERN_SCALAR" register
+		fi
 	)
 }
 

t/perf/run

@@ -171,6 +171,9 @@ run_subsection () {
 	get_var_from_env_or_config "GIT_PERF_MAKE_COMMAND" "perf" "makeCommand"
 	get_var_from_env_or_config "GIT_PERF_MAKE_OPTS" "perf" "makeOpts"
 
+	get_var_from_env_or_config "GIT_PERF_USE_SCALAR" "perf" "useScalar" "--bool"
+	export GIT_PERF_USE_SCALAR
+
 	get_var_from_env_or_config "GIT_PERF_REPO_NAME" "perf" "repoName"
 	export GIT_PERF_REPO_NAME
 

t/t9210-scalar.sh

@@ -0,0 +1,210 @@
+#!/bin/sh
+
+test_description='test the `scalar` command'
+
+. ./test-lib.sh
+
+GIT_TEST_MAINT_SCHEDULER="crontab:test-tool crontab cron.txt,launchctl:true,schtasks:true"
+export GIT_TEST_MAINT_SCHEDULER
+
+test_expect_success 'scalar shows a usage' '
+	test_expect_code 129 scalar -h
+'
+
+test_expect_success 'scalar invoked on enlistment root' '
+	test_when_finished rm -rf test src deeper &&
+
+	for enlistment_root in test src deeper/test
+	do
+		git init ${enlistment_root}/src &&
+
+		# Register
+		scalar register ${enlistment_root} &&
+		scalar list >out &&
+		grep "$(pwd)/${enlistment_root}/src\$" out &&
+
+		# Delete (including enlistment root)
+		scalar delete $enlistment_root &&
+		test_path_is_missing $enlistment_root &&
+		scalar list >out &&
+		! grep "^$(pwd)/${enlistment_root}/src\$" out || return 1
+	done
+'
+
+test_expect_success 'scalar invoked on enlistment src repo' '
+	test_when_finished rm -rf test src deeper &&
+
+	for enlistment_root in test src deeper/test
+	do
+		git init ${enlistment_root}/src &&
+
+		# Register
+		scalar register ${enlistment_root}/src &&
+		scalar list >out &&
+		grep "$(pwd)/${enlistment_root}/src\$" out &&
+
+		# Delete (will not include enlistment root)
+		scalar delete ${enlistment_root}/src &&
+		test_path_is_dir $enlistment_root &&
+		scalar list >out &&
+		! grep "^$(pwd)/${enlistment_root}/src\$" out || return 1
+	done
+'
+
+test_expect_success 'scalar invoked when enlistment root and repo are the same' '
+	test_when_finished rm -rf test src deeper &&
+
+	for enlistment_root in test src deeper/test
+	do
+		git init ${enlistment_root} &&
+
+		# Register
+		scalar register ${enlistment_root} &&
+		scalar list >out &&
+		grep "$(pwd)/${enlistment_root}\$" out &&
+
+		# Delete (will not include enlistment root)
+		scalar delete ${enlistment_root} &&
+		test_path_is_missing $enlistment_root &&
+		scalar list >out &&
+		! grep "^$(pwd)/${enlistment_root}\$" out &&
+
+		# Make sure we did not accidentally delete the trash dir
+		test_path_is_dir "$TRASH_DIRECTORY" || return 1
+	done
+'
+
+test_expect_success 'scalar repo search respects GIT_CEILING_DIRECTORIES' '
+	test_when_finished rm -rf test &&
+
+	git init test/src &&
+	mkdir -p test/src/deep &&
+	GIT_CEILING_DIRECTORIES="$(pwd)/test/src" &&
+	! scalar register test/src/deep 2>err &&
+	grep "not a git repository" err
+'
+
+test_expect_success 'scalar enlistments need a worktree' '
+	test_when_finished rm -rf bare test &&
+
+	git init --bare bare/src &&
+	! scalar register bare/src 2>err &&
+	grep "Scalar enlistments require a worktree" err &&
+
+	git init test/src &&
+	! scalar register test/src/.git 2>err &&
+	grep "Scalar enlistments require a worktree" err
+'
+
+test_expect_success FSMONITOR_DAEMON 'scalar register starts fsmon daemon' '
+	git init test/src &&
+	test_must_fail git -C test/src fsmonitor--daemon status &&
+	scalar register test/src &&
+	git -C test/src fsmonitor--daemon status &&
+	test_cmp_config -C test/src true core.fsmonitor
+'
+
+test_expect_success 'scalar unregister' '
+	git init vanish/src &&
+	scalar register vanish/src &&
+	git config --get --global --fixed-value \
+		maintenance.repo "$(pwd)/vanish/src" &&
+	scalar list >scalar.repos &&
+	grep -F "$(pwd)/vanish/src" scalar.repos &&
+	rm -rf vanish/src/.git &&
+	scalar unregister vanish &&
+	test_must_fail git config --get --global --fixed-value \
+		maintenance.repo "$(pwd)/vanish/src" &&
+	scalar list >scalar.repos &&
+	! grep -F "$(pwd)/vanish/src" scalar.repos
+'
+
+test_expect_success 'set up repository to clone' '
+	test_commit first &&
+	test_commit second &&
+	test_commit third &&
+	git switch -c parallel first &&
+	mkdir -p 1/2 &&
+	test_commit 1/2/3 &&
+	git config uploadPack.allowFilter true &&
+	git config uploadPack.allowAnySHA1InWant true
+'
+
+test_expect_success 'scalar clone' '
+	second=$(git rev-parse --verify second:second.t) &&
+	scalar clone "file://$(pwd)" cloned --single-branch &&
+	(
+		cd cloned/src &&
+
+		git config --get --global --fixed-value maintenance.repo \
+			"$(pwd)" &&
+
+		git for-each-ref --format="%(refname)" refs/remotes/origin/ >actual &&
+		echo "refs/remotes/origin/parallel" >expect &&
+		test_cmp expect actual &&
+
+		test_path_is_missing 1/2 &&
+		test_must_fail git rev-list --missing=print $second &&
+		git rev-list $second &&
+		git cat-file blob $second >actual &&
+		echo "second" >expect &&
+		test_cmp expect actual
+	)
+'
+
+test_expect_success 'scalar reconfigure' '
+	git init one/src &&
+	scalar register one &&
+	git -C one/src config core.preloadIndex false &&
+	scalar reconfigure one &&
+	test true = "$(git -C one/src config core.preloadIndex)" &&
+	git -C one/src config core.preloadIndex false &&
+	scalar reconfigure -a &&
+	test true = "$(git -C one/src config core.preloadIndex)"
+'
+
+test_expect_success 'scalar delete without enlistment shows a usage' '
+	test_expect_code 129 scalar delete
+'
+
+test_expect_success 'scalar delete with enlistment' '
+	scalar delete cloned &&
+	test_path_is_missing cloned
+'
+
+test_expect_success 'scalar supports -c/-C' '
+	test_when_finished "scalar delete sub" &&
+	git init sub &&
+	scalar -C sub -c status.aheadBehind=bogus register &&
+	test -z "$(git -C sub config --local status.aheadBehind)" &&
+	test true = "$(git -C sub config core.preloadIndex)"
+'
+
+test_expect_success '`scalar [...] <dir>` errors out when dir is missing' '
+	! scalar run config cloned 2>err &&
+	grep "cloned. does not exist" err
+'
+
+SQ="'"
+test_expect_success UNZIP 'scalar diagnose' '
+	scalar clone "file://$(pwd)" cloned --single-branch &&
+	git repack &&
+	echo "$(pwd)/.git/objects/" >>cloned/src/.git/objects/info/alternates &&
+	test_commit -C cloned/src loose &&
+	scalar diagnose cloned >out 2>err &&
+	grep "Available space" out &&
+	sed -n "s/.*$SQ\\(.*\\.zip\\)$SQ.*/\\1/p" <err >zip_path &&
+	zip_path=$(cat zip_path) &&
+	test -n "$zip_path" &&
+	"$GIT_UNZIP" -v "$zip_path" &&
+	folder=${zip_path%.zip} &&
+	test_path_is_missing "$folder" &&
+	"$GIT_UNZIP" -p "$zip_path" diagnostics.log >out &&
+	test_file_not_empty out &&
+	"$GIT_UNZIP" -p "$zip_path" packs-local.txt >out &&
+	grep "$(pwd)/.git/objects" out &&
+	"$GIT_UNZIP" -p "$zip_path" objects-local.txt >out &&
+	grep "^Total: [1-9]" out
+'
+
+test_done

t/t9211-scalar-clone.sh

@@ -0,0 +1,151 @@
+#!/bin/sh
+
+test_description='test the `scalar clone` subcommand'
+
+. ./test-lib.sh
+
+GIT_TEST_MAINT_SCHEDULER="crontab:test-tool crontab cron.txt,launchctl:true,schtasks:true"
+export GIT_TEST_MAINT_SCHEDULER
+
+test_expect_success 'set up repository to clone' '
+	rm -rf .git &&
+	git init to-clone &&
+	(
+		cd to-clone &&
+		git branch -m base &&
+
+		test_commit first &&
+		test_commit second &&
+		test_commit third &&
+
+		git switch -c parallel first &&
+		mkdir -p 1/2 &&
+		test_commit 1/2/3 &&
+
+		git switch base &&
+
+		# By default, permit
+		git config uploadpack.allowfilter true &&
+		git config uploadpack.allowanysha1inwant true
+	)
+'
+
+cleanup_clone () {
+	rm -rf "$1"
+}
+
+test_expect_success 'creates content in enlistment root' '
+	enlistment=cloned &&
+
+	scalar clone "file://$(pwd)/to-clone" $enlistment &&
+	ls -A $enlistment >enlistment-root &&
+	test_line_count = 1 enlistment-root &&
+	test_path_is_dir $enlistment/src &&
+	test_path_is_dir $enlistment/src/.git &&
+
+	cleanup_clone $enlistment
+'
+
+test_expect_success 'with spaces' '
+	enlistment="cloned with space" &&
+
+	scalar clone "file://$(pwd)/to-clone" "$enlistment" &&
+	test_path_is_dir "$enlistment" &&
+	test_path_is_dir "$enlistment/src" &&
+	test_path_is_dir "$enlistment/src/.git" &&
+
+	cleanup_clone "$enlistment"
+'
+
+test_expect_success 'partial clone if supported by server' '
+	enlistment=partial-clone &&
+
+	scalar clone "file://$(pwd)/to-clone" $enlistment &&
+
+	(
+		cd $enlistment/src &&
+
+		# Two promisor packs: one for refs, the other for blobs
+		ls .git/objects/pack/pack-*.promisor >promisorlist &&
+		test_line_count = 2 promisorlist
+	) &&
+
+	cleanup_clone $enlistment
+'
+
+test_expect_success 'fall back on full clone if partial unsupported' '
+	enlistment=no-partial-support &&
+
+	test_config -C to-clone uploadpack.allowfilter false &&
+	test_config -C to-clone uploadpack.allowanysha1inwant false &&
+
+	scalar clone "file://$(pwd)/to-clone" $enlistment 2>err &&
+	grep "filtering not recognized by server, ignoring" err &&
+
+	(
+		cd $enlistment/src &&
+
+		# Still get a refs promisor file, but none for blobs
+		ls .git/objects/pack/pack-*.promisor >promisorlist &&
+		test_line_count = 1 promisorlist
+	) &&
+
+	cleanup_clone $enlistment
+'
+
+test_expect_success 'initializes sparse-checkout by default' '
+	enlistment=sparse &&
+
+	scalar clone "file://$(pwd)/to-clone" $enlistment &&
+	(
+		cd $enlistment/src &&
+		test_cmp_config true core.sparseCheckout &&
+		test_cmp_config true core.sparseCheckoutCone
+	) &&
+
+	cleanup_clone $enlistment
+'
+
+test_expect_success '--full-clone does not create sparse-checkout' '
+	enlistment=full-clone &&
+
+	scalar clone --full-clone "file://$(pwd)/to-clone" $enlistment &&
+	(
+		cd $enlistment/src &&
+		test_cmp_config "" --default "" core.sparseCheckout &&
+		test_cmp_config "" --default "" core.sparseCheckoutCone
+	) &&
+
+	cleanup_clone $enlistment
+'
+
+test_expect_success '--single-branch clones HEAD only' '
+	enlistment=single-branch &&
+
+	scalar clone --single-branch "file://$(pwd)/to-clone" $enlistment &&
+	(
+		cd $enlistment/src &&
+		git for-each-ref refs/remotes/origin >out &&
+		test_line_count = 1 out &&
+		grep "refs/remotes/origin/base" out
+	) &&
+
+	cleanup_clone $enlistment
+'
+
+test_expect_success '--no-single-branch clones all branches' '
+	enlistment=no-single-branch &&
+
+	scalar clone --no-single-branch "file://$(pwd)/to-clone" $enlistment &&
+	(
+		cd $enlistment/src &&
+		git for-each-ref refs/remotes/origin >out &&
+		test_line_count = 2 out &&
+		grep "refs/remotes/origin/base" out &&
+		grep "refs/remotes/origin/parallel" out
+	) &&
+
+	cleanup_clone $enlistment
+'
+
+test_done