From 7c01878eeb15e8dd75f0262bdfb3249c85a30a4a Mon Sep 17 00:00:00 2001 From: Taylor Blau Date: Mon, 5 Feb 2024 17:50:19 -0500 Subject: [PATCH 1/2] t5332-multi-pack-reuse.sh: extract pack-objects helper functions Most of the tests in t5332 perform some setup before repeating a common refrain that looks like: : >trace2.txt && GIT_TRACE2_EVENT="$PWD/trace2.txt" \ git pack-objects --stdout --revs --all >/dev/null && test_pack_reused $objects_nr Signed-off-by: Junio C Hamano --- t/t5332-multi-pack-reuse.sh | 71 +++++++++++++++---------------------- 1 file changed, 29 insertions(+), 42 deletions(-) diff --git a/t/t5332-multi-pack-reuse.sh b/t/t5332-multi-pack-reuse.sh index 2ba788b042..d516062f84 100755 --- a/t/t5332-multi-pack-reuse.sh +++ b/t/t5332-multi-pack-reuse.sh @@ -23,6 +23,27 @@ pack_position () { grep "$1" objects | cut -d" " -f1 } +# test_pack_objects_reused_all +test_pack_objects_reused_all () { + : >trace2.txt && + GIT_TRACE2_EVENT="$PWD/trace2.txt" \ + git pack-objects --stdout --revs --all --delta-base-offset \ + >/dev/null && + + test_pack_reused "$1" +test_pack_objects_reused () { + : >trace2.txt && + GIT_TRACE2_EVENT="$PWD/trace2.txt" \ + git pack-objects --stdout --revs >/dev/null && + + test_pack_reused "$1" trace2.txt && - GIT_TRACE2_EVENT="$PWD/trace2.txt" \ - git pack-objects --stdout --revs --all >/dev/null && - - test_pack_reused 3 trace2.txt && - GIT_TRACE2_EVENT="$PWD/trace2.txt" \ - git pack-objects --stdout --revs /dev/null && - - test_pack_reused 6 trace2.txt && - GIT_TRACE2_EVENT="$PWD/trace2.txt" \ - git pack-objects --stdout --revs --all >/dev/null && - - test_pack_reused 9 trace2.txt && - GIT_TRACE2_EVENT="$PWD/trace2.txt" \ - git pack-objects --stdout --delta-base-offset --revs /dev/null && - - test_pack_reused 3 trace2.txt && - GIT_TRACE2_EVENT="$PWD/trace2.txt" \ - git pack-objects --stdout --delta-base-offset --revs /dev/null && - - test_pack_reused 3 trace2.txt && - GIT_TRACE2_EVENT="$PWD/trace2.txt" \ - git pack-objects --stdout --delta-base-offset --revs /dev/null && - # We can only reuse the 3 objects corresponding to "other" from # the latest pack. # @@ -175,8 +168,7 @@ test_expect_success 'omit delta with uninteresting base (same pack)' ' # The remaining objects from the other pack are similarly not # reused because their objects are on the uninteresting side of # the query. - test_pack_reused 3 trace2.txt && - GIT_TRACE2_EVENT="$PWD/trace2.txt" \ - git pack-objects --stdout --delta-base-offset --all >/dev/null && - packs_nr="$(find $packdir -type f -name "pack-*.pack" | wc -l)" && objects_nr="$(git rev-list --count --all --objects)" && - test_pack_reused $(($objects_nr - 1)) Date: Mon, 5 Feb 2024 17:50:23 -0500 Subject: [PATCH 2/2] pack-objects: enable multi-pack reuse via `feature.experimental` Now that multi-pack reuse is supported, enable it via the feature.experimental configuration in addition to the classic `pack.allowPackReuse`. This will allow more users to experiment with the new behavior who might not otherwise be aware of the existing `pack.allowPackReuse` configuration option. The enum with values NO_PACK_REUSE, SINGLE_PACK_REUSE, and MULTI_PACK_REUSE is defined statically in builtin/pack-objects.c's compilation unit. We could hoist that enum into a scope visible from the repository_settings struct, and then use that enum value in pack-objects. Instead, define a single int that indicates what pack-objects's default value should be to avoid additional unnecessary code movement. Though `feature.experimental` implies `pack.allowPackReuse=multi`, this can still be overridden by explicitly setting the latter configuration to either "single" or "false". Tests covering all of these cases are showin t5332. Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- Documentation/config/feature.txt | 3 +++ builtin/pack-objects.c | 2 ++ repo-settings.c | 1 + repository.h | 1 + t/t5332-multi-pack-reuse.sh | 16 ++++++++++++++++ 5 files changed, 23 insertions(+) diff --git a/Documentation/config/feature.txt b/Documentation/config/feature.txt index bf9546fca4..f061b64b74 100644 --- a/Documentation/config/feature.txt +++ b/Documentation/config/feature.txt @@ -17,6 +17,9 @@ skipping more commits at a time, reducing the number of round trips. + * `pack.useBitmapBoundaryTraversal=true` may improve bitmap traversal times by walking fewer objects. ++ +* `pack.allowPackReuse=multi` may improve the time it takes to create a pack by +reusing objects from multiple packs instead of just one. feature.manyFiles:: Enable config options that optimize for repos with many files in the diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index d8c2128a97..329aeac804 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -4396,6 +4396,8 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) prepare_repo_settings(the_repository); if (sparse < 0) sparse = the_repository->settings.pack_use_sparse; + if (the_repository->settings.pack_use_multi_pack_reuse) + allow_pack_reuse = MULTI_PACK_REUSE; } reset_pack_idx_option(&pack_idx_opts); diff --git a/repo-settings.c b/repo-settings.c index 30cd478762..a0b590bc6c 100644 --- a/repo-settings.c +++ b/repo-settings.c @@ -43,6 +43,7 @@ void prepare_repo_settings(struct repository *r) if (experimental) { r->settings.fetch_negotiation_algorithm = FETCH_NEGOTIATION_SKIPPING; r->settings.pack_use_bitmap_boundary_traversal = 1; + r->settings.pack_use_multi_pack_reuse = 1; } if (manyfiles) { r->settings.index_version = 4; diff --git a/repository.h b/repository.h index 7a250a6605..21949c5a17 100644 --- a/repository.h +++ b/repository.h @@ -39,6 +39,7 @@ struct repo_settings { int sparse_index; int pack_read_reverse_index; int pack_use_bitmap_boundary_traversal; + int pack_use_multi_pack_reuse; /* * Does this repository have core.useReplaceRefs=true (on by diff --git a/t/t5332-multi-pack-reuse.sh b/t/t5332-multi-pack-reuse.sh index d516062f84..b925a81d37 100755 --- a/t/t5332-multi-pack-reuse.sh +++ b/t/t5332-multi-pack-reuse.sh @@ -58,6 +58,22 @@ test_expect_success 'preferred pack is reused for single-pack reuse' ' test_pack_objects_reused_all 3 1 ' +test_expect_success 'multi-pack reuse is disabled by default' ' + test_pack_objects_reused_all 3 1 +' + +test_expect_success 'feature.experimental implies multi-pack reuse' ' + test_config feature.experimental true && + + test_pack_objects_reused_all 6 2 +' + +test_expect_success 'multi-pack reuse can be disabled with feature.experimental' ' + test_config feature.experimental true && + test_config pack.allowPackReuse single && + + test_pack_objects_reused_all 3 1 +' test_expect_success 'enable multi-pack reuse' ' git config pack.allowPackReuse multi