Merge branch 'ps/fix-geom-repack-with-alternates'

Geometric repacking ("git repack --geometric=<n>") in a repository
that borrows from an alternate object database had various corner
case bugs, which have been corrected.

* ps/fix-geom-repack-with-alternates:
  repack: disable writing bitmaps when doing a local repack
  repack: honor `-l` when calculating pack geometry
  t/helper: allow chmtime to print verbosely without modifying mtime
  pack-objects: extend test coverage of `--stdin-packs` with alternates
  pack-objects: fix error when same packfile is included and excluded
  pack-objects: fix error when packing same pack twice
  pack-objects: split out `--stdin-packs` tests into separate file
  repack: fix generating multi-pack-index with only non-local packs
  repack: fix trying to use preferred pack in alternates
  midx: fix segfault with no packs and invalid preferred pack
This commit is contained in:
Junio C Hamano
2023-04-25 13:56:20 -07:00
11 changed files with 504 additions and 151 deletions

View File

@ -94,7 +94,7 @@ int cmd__chmtime(int argc, const char **argv)
if (timespec_arg(argv[i], &set_time, &set_eq)) {
++i;
} else {
if (get == 0) {
if (get == 0 && verbose == 0) {
fprintf(stderr, "Not a base-10 integer: %s\n", argv[i] + 1);
goto usage;
}

View File

@ -589,141 +589,6 @@ test_expect_success 'prefetch objects' '
test_line_count = 1 donelines
'
test_expect_success 'setup for --stdin-packs tests' '
git init stdin-packs &&
(
cd stdin-packs &&
test_commit A &&
test_commit B &&
test_commit C &&
for id in A B C
do
git pack-objects .git/objects/pack/pack-$id \
--incremental --revs <<-EOF || exit 1
refs/tags/$id
EOF
done &&
ls -la .git/objects/pack
)
'
test_expect_success '--stdin-packs with excluded packs' '
(
cd stdin-packs &&
PACK_A="$(basename .git/objects/pack/pack-A-*.pack)" &&
PACK_B="$(basename .git/objects/pack/pack-B-*.pack)" &&
PACK_C="$(basename .git/objects/pack/pack-C-*.pack)" &&
git pack-objects test --stdin-packs <<-EOF &&
$PACK_A
^$PACK_B
$PACK_C
EOF
(
git show-index <$(ls .git/objects/pack/pack-A-*.idx) &&
git show-index <$(ls .git/objects/pack/pack-C-*.idx)
) >expect.raw &&
git show-index <$(ls test-*.idx) >actual.raw &&
cut -d" " -f2 <expect.raw | sort >expect &&
cut -d" " -f2 <actual.raw | sort >actual &&
test_cmp expect actual
)
'
test_expect_success '--stdin-packs is incompatible with --filter' '
(
cd stdin-packs &&
test_must_fail git pack-objects --stdin-packs --stdout \
--filter=blob:none </dev/null 2>err &&
test_i18ngrep "cannot use --filter with --stdin-packs" err
)
'
test_expect_success '--stdin-packs is incompatible with --revs' '
(
cd stdin-packs &&
test_must_fail git pack-objects --stdin-packs --revs out \
</dev/null 2>err &&
test_i18ngrep "cannot use internal rev list with --stdin-packs" err
)
'
test_expect_success '--stdin-packs with loose objects' '
(
cd stdin-packs &&
PACK_A="$(basename .git/objects/pack/pack-A-*.pack)" &&
PACK_B="$(basename .git/objects/pack/pack-B-*.pack)" &&
PACK_C="$(basename .git/objects/pack/pack-C-*.pack)" &&
test_commit D && # loose
git pack-objects test2 --stdin-packs --unpacked <<-EOF &&
$PACK_A
^$PACK_B
$PACK_C
EOF
(
git show-index <$(ls .git/objects/pack/pack-A-*.idx) &&
git show-index <$(ls .git/objects/pack/pack-C-*.idx) &&
git rev-list --objects --no-object-names \
refs/tags/C..refs/tags/D
) >expect.raw &&
ls -la . &&
git show-index <$(ls test2-*.idx) >actual.raw &&
cut -d" " -f2 <expect.raw | sort >expect &&
cut -d" " -f2 <actual.raw | sort >actual &&
test_cmp expect actual
)
'
test_expect_success '--stdin-packs with broken links' '
(
cd stdin-packs &&
# make an unreachable object with a bogus parent
git cat-file -p HEAD >commit &&
sed "s/$(git rev-parse HEAD^)/$(test_oid zero)/" <commit |
git hash-object -w -t commit --stdin >in &&
git pack-objects .git/objects/pack/pack-D <in &&
PACK_A="$(basename .git/objects/pack/pack-A-*.pack)" &&
PACK_B="$(basename .git/objects/pack/pack-B-*.pack)" &&
PACK_C="$(basename .git/objects/pack/pack-C-*.pack)" &&
PACK_D="$(basename .git/objects/pack/pack-D-*.pack)" &&
git pack-objects test3 --stdin-packs --unpacked <<-EOF &&
$PACK_A
^$PACK_B
$PACK_C
$PACK_D
EOF
(
git show-index <$(ls .git/objects/pack/pack-A-*.idx) &&
git show-index <$(ls .git/objects/pack/pack-C-*.idx) &&
git show-index <$(ls .git/objects/pack/pack-D-*.idx) &&
git rev-list --objects --no-object-names \
refs/tags/C..refs/tags/D
) >expect.raw &&
git show-index <$(ls test3-*.idx) >actual.raw &&
cut -d" " -f2 <expect.raw | sort >expect &&
cut -d" " -f2 <actual.raw | sort >actual &&
test_cmp expect actual
)
'
test_expect_success 'negative window clamps to 0' '
git pack-objects --progress --window=-1 neg-window <obj-list 2>stderr &&
check_deltas stderr = 0

View File

@ -183,6 +183,18 @@ test_expect_success 'write midx with --stdin-packs' '
compare_results_with_midx "mixed mode (one pack + extra)"
test_expect_success 'write with no objects and preferred pack' '
test_when_finished "rm -rf empty" &&
git init empty &&
test_must_fail git -C empty multi-pack-index write \
--stdin-packs --preferred-pack=does-not-exist </dev/null 2>err &&
cat >expect <<-EOF &&
warning: unknown preferred pack: ${SQ}does-not-exist${SQ}
error: no pack files to index.
EOF
test_cmp expect err
'
test_expect_success 'write progress off for redirected stderr' '
git multi-pack-index --object-dir=$objdir write 2>err &&
test_line_count = 0 err

240
t/t5331-pack-objects-stdin.sh Executable file
View File

@ -0,0 +1,240 @@
#!/bin/sh
test_description='pack-objects --stdin'
GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
TEST_PASSES_SANITIZE_LEAK=true
. ./test-lib.sh
packed_objects () {
git show-index <"$1" >tmp-object-list &&
cut -d' ' -f2 tmp-object-list | sort &&
rm tmp-object-list
}
test_expect_success 'setup for --stdin-packs tests' '
git init stdin-packs &&
(
cd stdin-packs &&
test_commit A &&
test_commit B &&
test_commit C &&
for id in A B C
do
git pack-objects .git/objects/pack/pack-$id \
--incremental --revs <<-EOF || exit 1
refs/tags/$id
EOF
done &&
ls -la .git/objects/pack
)
'
test_expect_success '--stdin-packs with excluded packs' '
(
cd stdin-packs &&
PACK_A="$(basename .git/objects/pack/pack-A-*.pack)" &&
PACK_B="$(basename .git/objects/pack/pack-B-*.pack)" &&
PACK_C="$(basename .git/objects/pack/pack-C-*.pack)" &&
git pack-objects test --stdin-packs <<-EOF &&
$PACK_A
^$PACK_B
$PACK_C
EOF
(
git show-index <$(ls .git/objects/pack/pack-A-*.idx) &&
git show-index <$(ls .git/objects/pack/pack-C-*.idx)
) >expect.raw &&
git show-index <$(ls test-*.idx) >actual.raw &&
cut -d" " -f2 <expect.raw | sort >expect &&
cut -d" " -f2 <actual.raw | sort >actual &&
test_cmp expect actual
)
'
test_expect_success '--stdin-packs is incompatible with --filter' '
(
cd stdin-packs &&
test_must_fail git pack-objects --stdin-packs --stdout \
--filter=blob:none </dev/null 2>err &&
test_i18ngrep "cannot use --filter with --stdin-packs" err
)
'
test_expect_success '--stdin-packs is incompatible with --revs' '
(
cd stdin-packs &&
test_must_fail git pack-objects --stdin-packs --revs out \
</dev/null 2>err &&
test_i18ngrep "cannot use internal rev list with --stdin-packs" err
)
'
test_expect_success '--stdin-packs with loose objects' '
(
cd stdin-packs &&
PACK_A="$(basename .git/objects/pack/pack-A-*.pack)" &&
PACK_B="$(basename .git/objects/pack/pack-B-*.pack)" &&
PACK_C="$(basename .git/objects/pack/pack-C-*.pack)" &&
test_commit D && # loose
git pack-objects test2 --stdin-packs --unpacked <<-EOF &&
$PACK_A
^$PACK_B
$PACK_C
EOF
(
git show-index <$(ls .git/objects/pack/pack-A-*.idx) &&
git show-index <$(ls .git/objects/pack/pack-C-*.idx) &&
git rev-list --objects --no-object-names \
refs/tags/C..refs/tags/D
) >expect.raw &&
ls -la . &&
git show-index <$(ls test2-*.idx) >actual.raw &&
cut -d" " -f2 <expect.raw | sort >expect &&
cut -d" " -f2 <actual.raw | sort >actual &&
test_cmp expect actual
)
'
test_expect_success '--stdin-packs with broken links' '
(
cd stdin-packs &&
# make an unreachable object with a bogus parent
git cat-file -p HEAD >commit &&
sed "s/$(git rev-parse HEAD^)/$(test_oid zero)/" <commit |
git hash-object -w -t commit --stdin >in &&
git pack-objects .git/objects/pack/pack-D <in &&
PACK_A="$(basename .git/objects/pack/pack-A-*.pack)" &&
PACK_B="$(basename .git/objects/pack/pack-B-*.pack)" &&
PACK_C="$(basename .git/objects/pack/pack-C-*.pack)" &&
PACK_D="$(basename .git/objects/pack/pack-D-*.pack)" &&
git pack-objects test3 --stdin-packs --unpacked <<-EOF &&
$PACK_A
^$PACK_B
$PACK_C
$PACK_D
EOF
(
git show-index <$(ls .git/objects/pack/pack-A-*.idx) &&
git show-index <$(ls .git/objects/pack/pack-C-*.idx) &&
git show-index <$(ls .git/objects/pack/pack-D-*.idx) &&
git rev-list --objects --no-object-names \
refs/tags/C..refs/tags/D
) >expect.raw &&
git show-index <$(ls test3-*.idx) >actual.raw &&
cut -d" " -f2 <expect.raw | sort >expect &&
cut -d" " -f2 <actual.raw | sort >actual &&
test_cmp expect actual
)
'
test_expect_success 'pack-objects --stdin with duplicate packfile' '
test_when_finished "rm -fr repo" &&
git init repo &&
(
cd repo &&
test_commit "commit" &&
git repack -ad &&
{
basename .git/objects/pack/pack-*.pack &&
basename .git/objects/pack/pack-*.pack
} >packfiles &&
git pack-objects --stdin-packs generated-pack <packfiles &&
packed_objects .git/objects/pack/pack-*.idx >expect &&
packed_objects generated-pack-*.idx >actual &&
test_cmp expect actual
)
'
test_expect_success 'pack-objects --stdin with same packfile excluded and included' '
test_when_finished "rm -fr repo" &&
git init repo &&
(
cd repo &&
test_commit "commit" &&
git repack -ad &&
{
basename .git/objects/pack/pack-*.pack &&
printf "^%s\n" "$(basename .git/objects/pack/pack-*.pack)"
} >packfiles &&
git pack-objects --stdin-packs generated-pack <packfiles &&
packed_objects generated-pack-*.idx >packed-objects &&
test_must_be_empty packed-objects
)
'
test_expect_success 'pack-objects --stdin with packfiles from alternate object database' '
test_when_finished "rm -fr shared member" &&
# Set up a shared repository with a single packfile.
git init shared &&
test_commit -C shared "shared-objects" &&
git -C shared repack -ad &&
basename shared/.git/objects/pack/pack-*.pack >packfile &&
# Set up a repository that is connected to the shared repository. This
# repository has no objects on its own, but we still expect to be able
# to pack objects from its alternate.
git clone --shared shared member &&
git -C member pack-objects --stdin-packs generated-pack <packfile &&
test_cmp shared/.git/objects/pack/pack-*.pack member/generated-pack-*.pack
'
test_expect_success 'pack-objects --stdin with packfiles from main and alternate object database' '
test_when_finished "rm -fr shared member" &&
# Set up a shared repository with a single packfile.
git init shared &&
test_commit -C shared "shared-commit" &&
git -C shared repack -ad &&
# Set up a repository that is connected to the shared repository. This
# repository has a second packfile so that we can verify that it is
# possible to write packs that include packfiles from different object
# databases.
git clone --shared shared member &&
test_commit -C member "local-commit" &&
git -C member repack -dl &&
{
basename shared/.git/objects/pack/pack-*.pack &&
basename member/.git/objects/pack/pack-*.pack
} >packfiles &&
{
packed_objects shared/.git/objects/pack/pack-*.idx &&
packed_objects member/.git/objects/pack/pack-*.idx
} | sort >expected-objects &&
git -C member pack-objects --stdin-packs generated-pack <packfiles &&
packed_objects member/generated-pack-*.idx >actual-objects &&
test_cmp expected-objects actual-objects
'
test_done

View File

@ -106,6 +106,23 @@ test_expect_success SYMLINKS '--local keeps packs when alternate is objectdir '
test_cmp expect actual
'
test_expect_success '--local disables writing bitmaps when connected to alternate ODB' '
test_when_finished "rm -rf shared member" &&
git init shared &&
git clone --shared shared member &&
(
cd member &&
test_commit "object" &&
GIT_TEST_MULTI_PACK_INDEX=0 git repack -Adl --write-bitmap-index 2>err &&
cat >expect <<-EOF &&
warning: disabling bitmap writing, as some objects are not being packed
EOF
test_cmp expect err &&
test_path_is_missing .git/objects/pack-*.bitmap
)
'
test_expect_success 'packed obs in alt ODB are repacked even when local repo is packless' '
mkdir alt_objects/pack &&
mv .git/objects/pack/* alt_objects/pack &&

View File

@ -10,6 +10,12 @@ objdir=.git/objects
packdir=$objdir/pack
midx=$objdir/pack/multi-pack-index
packed_objects () {
git show-index <"$1" >tmp-object-list &&
cut -d' ' -f2 tmp-object-list | sort &&
rm tmp-object-list
}
test_expect_success '--geometric with no packs' '
git init geometric &&
test_when_finished "rm -fr geometric" &&
@ -281,4 +287,162 @@ test_expect_success '--geometric with pack.packSizeLimit' '
)
'
test_expect_success '--geometric --write-midx with packfiles in main and alternate ODB' '
test_when_finished "rm -fr shared member" &&
# Create a shared repository that will serve as the alternate object
# database for the member linked to it. It has got some objects on its
# own that are packed into a single packfile.
git init shared &&
test_commit -C shared common-object &&
git -C shared repack -ad &&
# We create member so that its alternates file points to the shared
# repository. We then create a commit in it so that git-repack(1) has
# something to repack.
# of the shared object database.
git clone --shared shared member &&
test_commit -C member unique-object &&
git -C member repack --geometric=2 --write-midx 2>err &&
test_must_be_empty err &&
# We should see that a new packfile was generated.
find shared/.git/objects/pack -type f -name "*.pack" >packs &&
test_line_count = 1 packs &&
# We should also see a multi-pack-index. This multi-pack-index should
# never refer to any packfiles in the alternate object database.
test_path_is_file member/.git/objects/pack/multi-pack-index &&
test-tool read-midx member/.git/objects >packs.midx &&
grep "^pack-.*\.idx$" packs.midx | sort >actual &&
basename member/.git/objects/pack/pack-*.idx >expect &&
test_cmp expect actual
'
test_expect_success '--geometric --with-midx with no local objects' '
test_when_finished "rm -fr shared member" &&
# Create a repository with a single packfile that acts as alternate
# object database.
git init shared &&
test_commit -C shared "shared-objects" &&
git -C shared repack -ad &&
# Create a second repository linked to the first one and perform a
# geometric repack on it.
git clone --shared shared member &&
git -C member repack --geometric 2 --write-midx 2>err &&
test_must_be_empty err &&
# Assert that we wrote neither a new packfile nor a multi-pack-index.
# We should not have a packfile because the single packfile in the
# alternate object database does not invalidate the geometric sequence.
# And we should not have a multi-pack-index because these only index
# local packfiles, and there are none.
test_dir_is_empty member/$packdir
'
test_expect_success '--geometric with same pack in main and alternate ODB' '
test_when_finished "rm -fr shared member" &&
# Create a repository with a single packfile that acts as alternate
# object database.
git init shared &&
test_commit -C shared "shared-objects" &&
git -C shared repack -ad &&
# We create the member repository as an exact copy so that it has the
# same packfile.
cp -r shared member &&
test-tool path-utils real_path shared/.git/objects >member/.git/objects/info/alternates &&
find shared/.git/objects -type f >expected-files &&
# Verify that we can repack objects as expected without observing any
# error. Having the same packfile in both ODBs used to cause an error
# in git-pack-objects(1).
git -C member repack --geometric 2 2>err &&
test_must_be_empty err &&
# Nothing should have changed.
find shared/.git/objects -type f >actual-files &&
test_cmp expected-files actual-files
'
test_expect_success '--geometric -l with non-intact geometric sequence across ODBs' '
test_when_finished "rm -fr shared member" &&
git init shared &&
test_commit_bulk -C shared --start=1 1 &&
git clone --shared shared member &&
test_commit_bulk -C member --start=2 1 &&
# Verify that our assumptions actually hold: both generated packfiles
# should have three objects and should be non-equal.
packed_objects shared/.git/objects/pack/pack-*.idx >shared-objects &&
packed_objects member/.git/objects/pack/pack-*.idx >member-objects &&
test_line_count = 3 shared-objects &&
test_line_count = 3 member-objects &&
! test_cmp shared-objects member-objects &&
# Perform the geometric repack. With `-l`, we should only see the local
# packfile and thus arrive at the conclusion that the geometric
# sequence is intact. We thus expect no changes.
#
# Note that we are tweaking mtimes of the packfiles so that we can
# verify they did not change. This is done in order to detect the case
# where we do repack objects, but the resulting packfile is the same.
test-tool chmtime --verbose =0 member/.git/objects/pack/* >expected-member-packs &&
git -C member repack --geometric=2 -l -d &&
test-tool chmtime --verbose member/.git/objects/pack/* >actual-member-packs &&
test_cmp expected-member-packs actual-member-packs &&
{
packed_objects shared/.git/objects/pack/pack-*.idx &&
packed_objects member/.git/objects/pack/pack-*.idx
} | sort >expected-objects &&
# On the other hand, when doing a non-local geometric repack we should
# see both packfiles and thus repack them. We expect that the shared
# object database was not changed.
test-tool chmtime --verbose =0 shared/.git/objects/pack/* >expected-shared-packs &&
git -C member repack --geometric=2 -d &&
test-tool chmtime --verbose shared/.git/objects/pack/* >actual-shared-packs &&
test_cmp expected-shared-packs actual-shared-packs &&
# Furthermore, we expect that the member repository now has a single
# packfile that contains the combined shared and non-shared objects.
ls member/.git/objects/pack/pack-*.idx >actual &&
test_line_count = 1 actual &&
packed_objects member/.git/objects/pack/pack-*.idx >actual-objects &&
test_line_count = 6 actual-objects &&
test_cmp expected-objects actual-objects
'
test_expect_success '--geometric -l disables writing bitmaps with non-local packfiles' '
test_when_finished "rm -fr shared member" &&
git init shared &&
test_commit_bulk -C shared --start=1 1 &&
git clone --shared shared member &&
test_commit_bulk -C member --start=2 1 &&
# When performing a geometric repack with `-l` while connected to an
# alternate object database that has a packfile we do not have full
# coverage of objects. As a result, we expect that writing the bitmap
# will be disabled.
git -C member repack -l --geometric=2 --write-midx --write-bitmap-index 2>err &&
cat >expect <<-EOF &&
warning: disabling bitmap writing, as some objects are not being packed
EOF
test_cmp expect err &&
test_path_is_missing member/.git/objects/pack/multi-pack-index-*.bitmap &&
# On the other hand, when we repack without `-l`, we should see that
# the bitmap gets created.
git -C member repack --geometric=2 --write-midx --write-bitmap-index 2>err &&
test_must_be_empty err &&
test_path_is_file member/.git/objects/pack/multi-pack-index-*.bitmap
'
test_done