Merge branch 'ac/bitmap-lookup-table'
The pack bitmap file gained a bitmap-lookup table to speed up locating the necessary bitmap for a given commit. * ac/bitmap-lookup-table: pack-bitmap-write: drop unused pack_idx_entry parameters bitmap-lookup-table: add performance tests for lookup table pack-bitmap: prepare to read lookup table extension pack-bitmap-write: learn pack.writeBitmapLookupTable and add tests pack-bitmap-write.c: write lookup table extension bitmap: move `get commit positions` code to `bitmap_writer_finish` Documentation/technical: describe bitmap lookup table extension
This commit is contained in:
Junio C Hamano
@ -67,3 +67,34 @@ test_partial_bitmap () {
|
||||
--filter=tree:0 >/dev/null
|
||||
'
|
||||
}
|
||||
|
||||
test_pack_bitmap () {
|
||||
test_perf "repack to disk" '
|
||||
git repack -ad
|
||||
'
|
||||
|
||||
test_full_bitmap
|
||||
|
||||
test_expect_success "create partial bitmap state" '
|
||||
# pick a commit to represent the repo tip in the past
|
||||
cutoff=$(git rev-list HEAD~100 -1) &&
|
||||
orig_tip=$(git rev-parse HEAD) &&
|
||||
|
||||
# now kill off all of the refs and pretend we had
|
||||
# just the one tip
|
||||
rm -rf .git/logs .git/refs/* .git/packed-refs &&
|
||||
git update-ref HEAD $cutoff &&
|
||||
|
||||
# and then repack, which will leave us with a nice
|
||||
# big bitmap pack of the "old" history, and all of
|
||||
# the new history will be loose, as if it had been pushed
|
||||
# up incrementally and exploded via unpack-objects
|
||||
git repack -Ad &&
|
||||
|
||||
# and now restore our original tip, as if the pushes
|
||||
# had happened
|
||||
git update-ref HEAD $orig_tip
|
||||
'
|
||||
|
||||
test_partial_bitmap
|
||||
}
|
||||
|
||||
@ -4,51 +4,37 @@ test_description='Tests pack performance using bitmaps'
|
||||
. ./perf-lib.sh
|
||||
. "${TEST_DIRECTORY}/perf/lib-bitmap.sh"
|
||||
|
||||
test_perf_large_repo
|
||||
test_lookup_pack_bitmap () {
|
||||
test_expect_success 'start the test from scratch' '
|
||||
rm -rf * .git
|
||||
'
|
||||
|
||||
# note that we do everything through config,
|
||||
# since we want to be able to compare bitmap-aware
|
||||
# git versus non-bitmap git
|
||||
#
|
||||
# We intentionally use the deprecated pack.writebitmaps
|
||||
# config so that we can test against older versions of git.
|
||||
test_expect_success 'setup bitmap config' '
|
||||
git config pack.writebitmaps true
|
||||
'
|
||||
test_perf_large_repo
|
||||
|
||||
# we need to create the tag up front such that it is covered by the repack and
|
||||
# thus by generated bitmaps.
|
||||
test_expect_success 'create tags' '
|
||||
git tag --message="tag pointing to HEAD" perf-tag HEAD
|
||||
'
|
||||
# note that we do everything through config,
|
||||
# since we want to be able to compare bitmap-aware
|
||||
# git versus non-bitmap git
|
||||
#
|
||||
# We intentionally use the deprecated pack.writebitmaps
|
||||
# config so that we can test against older versions of git.
|
||||
test_expect_success 'setup bitmap config' '
|
||||
git config pack.writebitmaps true
|
||||
'
|
||||
|
||||
test_perf 'repack to disk' '
|
||||
git repack -ad
|
||||
'
|
||||
# we need to create the tag up front such that it is covered by the repack and
|
||||
# thus by generated bitmaps.
|
||||
test_expect_success 'create tags' '
|
||||
git tag --message="tag pointing to HEAD" perf-tag HEAD
|
||||
'
|
||||
|
||||
test_full_bitmap
|
||||
test_perf "enable lookup table: $1" '
|
||||
git config pack.writeBitmapLookupTable '"$1"'
|
||||
'
|
||||
|
||||
test_expect_success 'create partial bitmap state' '
|
||||
# pick a commit to represent the repo tip in the past
|
||||
cutoff=$(git rev-list HEAD~100 -1) &&
|
||||
orig_tip=$(git rev-parse HEAD) &&
|
||||
test_pack_bitmap
|
||||
}
|
||||
|
||||
# now kill off all of the refs and pretend we had
|
||||
# just the one tip
|
||||
rm -rf .git/logs .git/refs/* .git/packed-refs &&
|
||||
git update-ref HEAD $cutoff &&
|
||||
|
||||
# and then repack, which will leave us with a nice
|
||||
# big bitmap pack of the "old" history, and all of
|
||||
# the new history will be loose, as if it had been pushed
|
||||
# up incrementally and exploded via unpack-objects
|
||||
git repack -Ad &&
|
||||
|
||||
# and now restore our original tip, as if the pushes
|
||||
# had happened
|
||||
git update-ref HEAD $orig_tip
|
||||
'
|
||||
|
||||
test_partial_bitmap
|
||||
test_lookup_pack_bitmap false
|
||||
test_lookup_pack_bitmap true
|
||||
|
||||
test_done
|
||||
|
||||
@ -3,42 +3,52 @@
|
||||
test_description='performance of fetches from bitmapped packs'
|
||||
. ./perf-lib.sh
|
||||
|
||||
test_perf_default_repo
|
||||
|
||||
test_expect_success 'create bitmapped server repo' '
|
||||
git config pack.writebitmaps true &&
|
||||
git repack -ad
|
||||
'
|
||||
|
||||
# simulate a fetch from a repository that last fetched N days ago, for
|
||||
# various values of N. We do so by following the first-parent chain,
|
||||
# and assume the first entry in the chain that is N days older than the current
|
||||
# HEAD is where the HEAD would have been then.
|
||||
for days in 1 2 4 8 16 32 64 128; do
|
||||
title=$(printf '%10s' "($days days)")
|
||||
test_expect_success "setup revs from $days days ago" '
|
||||
now=$(git log -1 --format=%ct HEAD) &&
|
||||
then=$(($now - ($days * 86400))) &&
|
||||
tip=$(git rev-list -1 --first-parent --until=$then HEAD) &&
|
||||
{
|
||||
echo HEAD &&
|
||||
echo ^$tip
|
||||
} >revs
|
||||
test_fetch_bitmaps () {
|
||||
test_expect_success 'setup test directory' '
|
||||
rm -fr * .git
|
||||
'
|
||||
|
||||
test_perf "server $title" '
|
||||
git pack-objects --stdout --revs \
|
||||
--thin --delta-base-offset \
|
||||
<revs >tmp.pack
|
||||
test_perf_default_repo
|
||||
|
||||
test_expect_success 'create bitmapped server repo' '
|
||||
git config pack.writebitmaps true &&
|
||||
git config pack.writeBitmapLookupTable '"$1"' &&
|
||||
git repack -ad
|
||||
'
|
||||
|
||||
test_size "size $title" '
|
||||
wc -c <tmp.pack
|
||||
'
|
||||
# simulate a fetch from a repository that last fetched N days ago, for
|
||||
# various values of N. We do so by following the first-parent chain,
|
||||
# and assume the first entry in the chain that is N days older than the current
|
||||
# HEAD is where the HEAD would have been then.
|
||||
for days in 1 2 4 8 16 32 64 128; do
|
||||
title=$(printf '%10s' "($days days)")
|
||||
test_expect_success "setup revs from $days days ago" '
|
||||
now=$(git log -1 --format=%ct HEAD) &&
|
||||
then=$(($now - ($days * 86400))) &&
|
||||
tip=$(git rev-list -1 --first-parent --until=$then HEAD) &&
|
||||
{
|
||||
echo HEAD &&
|
||||
echo ^$tip
|
||||
} >revs
|
||||
'
|
||||
|
||||
test_perf "client $title" '
|
||||
git index-pack --stdin --fix-thin <tmp.pack
|
||||
'
|
||||
done
|
||||
test_perf "server $title (lookup=$1)" '
|
||||
git pack-objects --stdout --revs \
|
||||
--thin --delta-base-offset \
|
||||
<revs >tmp.pack
|
||||
'
|
||||
|
||||
test_size "size $title" '
|
||||
wc -c <tmp.pack
|
||||
'
|
||||
|
||||
test_perf "client $title (lookup=$1)" '
|
||||
git index-pack --stdin --fix-thin <tmp.pack
|
||||
'
|
||||
done
|
||||
}
|
||||
|
||||
test_fetch_bitmaps true
|
||||
test_fetch_bitmaps false
|
||||
|
||||
test_done
|
||||
|
||||
35
t/perf/p5312-pack-bitmaps-revs.sh
Executable file
35
t/perf/p5312-pack-bitmaps-revs.sh
Executable file
@ -0,0 +1,35 @@
|
||||
#!/bin/sh
|
||||
|
||||
test_description='Tests pack performance using bitmaps (rev index enabled)'
|
||||
. ./perf-lib.sh
|
||||
. "${TEST_DIRECTORY}/perf/lib-bitmap.sh"
|
||||
|
||||
test_lookup_pack_bitmap () {
|
||||
test_expect_success 'start the test from scratch' '
|
||||
rm -rf * .git
|
||||
'
|
||||
|
||||
test_perf_large_repo
|
||||
|
||||
test_expect_success 'setup bitmap config' '
|
||||
git config pack.writebitmaps true &&
|
||||
git config pack.writeReverseIndex true
|
||||
'
|
||||
|
||||
# we need to create the tag up front such that it is covered by the repack and
|
||||
# thus by generated bitmaps.
|
||||
test_expect_success 'create tags' '
|
||||
git tag --message="tag pointing to HEAD" perf-tag HEAD
|
||||
'
|
||||
|
||||
test_perf "enable lookup table: $1" '
|
||||
git config pack.writeBitmapLookupTable '"$1"'
|
||||
'
|
||||
|
||||
test_pack_bitmap
|
||||
}
|
||||
|
||||
test_lookup_pack_bitmap false
|
||||
test_lookup_pack_bitmap true
|
||||
|
||||
test_done
|
||||
@ -4,49 +4,64 @@ test_description='Tests performance using midx bitmaps'
|
||||
. ./perf-lib.sh
|
||||
. "${TEST_DIRECTORY}/perf/lib-bitmap.sh"
|
||||
|
||||
test_perf_large_repo
|
||||
test_bitmap () {
|
||||
local enabled="$1"
|
||||
|
||||
# we need to create the tag up front such that it is covered by the repack and
|
||||
# thus by generated bitmaps.
|
||||
test_expect_success 'create tags' '
|
||||
git tag --message="tag pointing to HEAD" perf-tag HEAD
|
||||
'
|
||||
test_expect_success "remove existing repo (lookup=$enabled)" '
|
||||
rm -fr * .git
|
||||
'
|
||||
|
||||
test_expect_success 'start with bitmapped pack' '
|
||||
git repack -adb
|
||||
'
|
||||
test_perf_large_repo
|
||||
|
||||
test_perf 'setup multi-pack index' '
|
||||
git multi-pack-index write --bitmap
|
||||
'
|
||||
# we need to create the tag up front such that it is covered by the repack and
|
||||
# thus by generated bitmaps.
|
||||
test_expect_success 'create tags' '
|
||||
git tag --message="tag pointing to HEAD" perf-tag HEAD
|
||||
'
|
||||
|
||||
test_expect_success 'drop pack bitmap' '
|
||||
rm -f .git/objects/pack/pack-*.bitmap
|
||||
'
|
||||
test_expect_success "use lookup table: $enabled" '
|
||||
git config pack.writeBitmapLookupTable '"$enabled"'
|
||||
'
|
||||
|
||||
test_full_bitmap
|
||||
test_expect_success "start with bitmapped pack (lookup=$enabled)" '
|
||||
git repack -adb
|
||||
'
|
||||
|
||||
test_expect_success 'create partial bitmap state' '
|
||||
# pick a commit to represent the repo tip in the past
|
||||
cutoff=$(git rev-list HEAD~100 -1) &&
|
||||
orig_tip=$(git rev-parse HEAD) &&
|
||||
test_perf "setup multi-pack index (lookup=$enabled)" '
|
||||
git multi-pack-index write --bitmap
|
||||
'
|
||||
|
||||
# now pretend we have just one tip
|
||||
rm -rf .git/logs .git/refs/* .git/packed-refs &&
|
||||
git update-ref HEAD $cutoff &&
|
||||
test_expect_success "drop pack bitmap (lookup=$enabled)" '
|
||||
rm -f .git/objects/pack/pack-*.bitmap
|
||||
'
|
||||
|
||||
# and then repack, which will leave us with a nice
|
||||
# big bitmap pack of the "old" history, and all of
|
||||
# the new history will be loose, as if it had been pushed
|
||||
# up incrementally and exploded via unpack-objects
|
||||
git repack -Ad &&
|
||||
git multi-pack-index write --bitmap &&
|
||||
test_full_bitmap
|
||||
|
||||
# and now restore our original tip, as if the pushes
|
||||
# had happened
|
||||
git update-ref HEAD $orig_tip
|
||||
'
|
||||
test_expect_success "create partial bitmap state (lookup=$enabled)" '
|
||||
# pick a commit to represent the repo tip in the past
|
||||
cutoff=$(git rev-list HEAD~100 -1) &&
|
||||
orig_tip=$(git rev-parse HEAD) &&
|
||||
|
||||
test_partial_bitmap
|
||||
# now pretend we have just one tip
|
||||
rm -rf .git/logs .git/refs/* .git/packed-refs &&
|
||||
git update-ref HEAD $cutoff &&
|
||||
|
||||
# and then repack, which will leave us with a nice
|
||||
# big bitmap pack of the "old" history, and all of
|
||||
# the new history will be loose, as if it had been pushed
|
||||
# up incrementally and exploded via unpack-objects
|
||||
git repack -Ad &&
|
||||
git multi-pack-index write --bitmap &&
|
||||
|
||||
# and now restore our original tip, as if the pushes
|
||||
# had happened
|
||||
git update-ref HEAD $orig_tip
|
||||
'
|
||||
|
||||
test_partial_bitmap
|
||||
}
|
||||
|
||||
test_bitmap false
|
||||
test_bitmap true
|
||||
|
||||
test_done
|
||||
|
||||
Reference in New Issue
Block a user