Merge branch 'jk/pack-bitmap'
Borrow the bitmap index into packfiles from JGit to speed up enumeration of objects involved in a commit range without having to fully traverse the history. * jk/pack-bitmap: (26 commits) ewah: unconditionally ntohll ewah data ewah: support platforms that require aligned reads read-cache: use get_be32 instead of hand-rolled ntoh_l block-sha1: factor out get_be and put_be wrappers do not discard revindex when re-preparing packfiles pack-bitmap: implement optional name_hash cache t/perf: add tests for pack bitmaps t: add basic bitmap functionality tests count-objects: recognize .bitmap in garbage-checking repack: consider bitmaps when performing repacks repack: handle optional files created by pack-objects repack: turn exts array into array-of-struct repack: stop using magic number for ARRAY_SIZE(exts) pack-objects: implement bitmap writing rev-list: add bitmap mode to speed up object lists pack-objects: use bitmaps when packing objects pack-objects: split add_object_entry pack-bitmap: add support for bitmap indexes documentation: add documentation for the bitmap format ewah: compressed bitmap implementation ...
This commit is contained in:
Junio C Hamano
57
t/perf/p5310-pack-bitmaps.sh
Executable file
57
t/perf/p5310-pack-bitmaps.sh
Executable file
@ -0,0 +1,57 @@
|
||||
#!/bin/sh
|
||||
|
||||
test_description='Tests pack performance using bitmaps'
|
||||
. ./perf-lib.sh
|
||||
|
||||
test_perf_large_repo
|
||||
|
||||
# note that we do everything through config,
|
||||
# since we want to be able to compare bitmap-aware
|
||||
# git versus non-bitmap git
|
||||
test_expect_success 'setup bitmap config' '
|
||||
git config pack.writebitmaps true &&
|
||||
git config pack.writebitmaphashcache true
|
||||
'
|
||||
|
||||
test_perf 'repack to disk' '
|
||||
git repack -ad
|
||||
'
|
||||
|
||||
test_perf 'simulated clone' '
|
||||
git pack-objects --stdout --all </dev/null >/dev/null
|
||||
'
|
||||
|
||||
test_perf 'simulated fetch' '
|
||||
have=$(git rev-list HEAD~100 -1) &&
|
||||
{
|
||||
echo HEAD &&
|
||||
echo ^$have
|
||||
} | git pack-objects --revs --stdout >/dev/null
|
||||
'
|
||||
|
||||
test_expect_success 'create partial bitmap state' '
|
||||
# pick a commit to represent the repo tip in the past
|
||||
cutoff=$(git rev-list HEAD~100 -1) &&
|
||||
orig_tip=$(git rev-parse HEAD) &&
|
||||
|
||||
# now kill off all of the refs and pretend we had
|
||||
# just the one tip
|
||||
rm -rf .git/logs .git/refs/* .git/packed-refs
|
||||
git update-ref HEAD $cutoff
|
||||
|
||||
# and then repack, which will leave us with a nice
|
||||
# big bitmap pack of the "old" history, and all of
|
||||
# the new history will be loose, as if it had been pushed
|
||||
# up incrementally and exploded via unpack-objects
|
||||
git repack -Ad
|
||||
|
||||
# and now restore our original tip, as if the pushes
|
||||
# had happened
|
||||
git update-ref HEAD $orig_tip
|
||||
'
|
||||
|
||||
test_perf 'partial bitmap' '
|
||||
git pack-objects --stdout --all </dev/null >/dev/null
|
||||
'
|
||||
|
||||
test_done
|
||||
139
t/t5310-pack-bitmaps.sh
Executable file
139
t/t5310-pack-bitmaps.sh
Executable file
@ -0,0 +1,139 @@
|
||||
#!/bin/sh
|
||||
|
||||
test_description='exercise basic bitmap functionality'
|
||||
. ./test-lib.sh
|
||||
|
||||
test_expect_success 'setup repo with moderate-sized history' '
|
||||
for i in $(test_seq 1 10); do
|
||||
test_commit $i
|
||||
done &&
|
||||
git checkout -b other HEAD~5 &&
|
||||
for i in $(test_seq 1 10); do
|
||||
test_commit side-$i
|
||||
done &&
|
||||
git checkout master &&
|
||||
blob=$(echo tagged-blob | git hash-object -w --stdin) &&
|
||||
git tag tagged-blob $blob &&
|
||||
git config pack.writebitmaps true &&
|
||||
git config pack.writebitmaphashcache true
|
||||
'
|
||||
|
||||
test_expect_success 'full repack creates bitmaps' '
|
||||
git repack -ad &&
|
||||
ls .git/objects/pack/ | grep bitmap >output &&
|
||||
test_line_count = 1 output
|
||||
'
|
||||
|
||||
test_expect_success 'rev-list --test-bitmap verifies bitmaps' '
|
||||
git rev-list --test-bitmap HEAD
|
||||
'
|
||||
|
||||
rev_list_tests() {
|
||||
state=$1
|
||||
|
||||
test_expect_success "counting commits via bitmap ($state)" '
|
||||
git rev-list --count HEAD >expect &&
|
||||
git rev-list --use-bitmap-index --count HEAD >actual &&
|
||||
test_cmp expect actual
|
||||
'
|
||||
|
||||
test_expect_success "counting partial commits via bitmap ($state)" '
|
||||
git rev-list --count HEAD~5..HEAD >expect &&
|
||||
git rev-list --use-bitmap-index --count HEAD~5..HEAD >actual &&
|
||||
test_cmp expect actual
|
||||
'
|
||||
|
||||
test_expect_success "counting non-linear history ($state)" '
|
||||
git rev-list --count other...master >expect &&
|
||||
git rev-list --use-bitmap-index --count other...master >actual &&
|
||||
test_cmp expect actual
|
||||
'
|
||||
|
||||
test_expect_success "enumerate --objects ($state)" '
|
||||
git rev-list --objects --use-bitmap-index HEAD >tmp &&
|
||||
cut -d" " -f1 <tmp >tmp2 &&
|
||||
sort <tmp2 >actual &&
|
||||
git rev-list --objects HEAD >tmp &&
|
||||
cut -d" " -f1 <tmp >tmp2 &&
|
||||
sort <tmp2 >expect &&
|
||||
test_cmp expect actual
|
||||
'
|
||||
|
||||
test_expect_success "bitmap --objects handles non-commit objects ($state)" '
|
||||
git rev-list --objects --use-bitmap-index HEAD tagged-blob >actual &&
|
||||
grep $blob actual
|
||||
'
|
||||
}
|
||||
|
||||
rev_list_tests 'full bitmap'
|
||||
|
||||
test_expect_success 'clone from bitmapped repository' '
|
||||
git clone --no-local --bare . clone.git &&
|
||||
git rev-parse HEAD >expect &&
|
||||
git --git-dir=clone.git rev-parse HEAD >actual &&
|
||||
test_cmp expect actual
|
||||
'
|
||||
|
||||
test_expect_success 'setup further non-bitmapped commits' '
|
||||
for i in $(test_seq 1 10); do
|
||||
test_commit further-$i
|
||||
done
|
||||
'
|
||||
|
||||
rev_list_tests 'partial bitmap'
|
||||
|
||||
test_expect_success 'fetch (partial bitmap)' '
|
||||
git --git-dir=clone.git fetch origin master:master &&
|
||||
git rev-parse HEAD >expect &&
|
||||
git --git-dir=clone.git rev-parse HEAD >actual &&
|
||||
test_cmp expect actual
|
||||
'
|
||||
|
||||
test_expect_success 'incremental repack cannot create bitmaps' '
|
||||
test_commit more-1 &&
|
||||
test_must_fail git repack -d
|
||||
'
|
||||
|
||||
test_expect_success 'incremental repack can disable bitmaps' '
|
||||
test_commit more-2 &&
|
||||
git repack -d --no-write-bitmap-index
|
||||
'
|
||||
|
||||
test_expect_success 'full repack, reusing previous bitmaps' '
|
||||
git repack -ad &&
|
||||
ls .git/objects/pack/ | grep bitmap >output &&
|
||||
test_line_count = 1 output
|
||||
'
|
||||
|
||||
test_expect_success 'fetch (full bitmap)' '
|
||||
git --git-dir=clone.git fetch origin master:master &&
|
||||
git rev-parse HEAD >expect &&
|
||||
git --git-dir=clone.git rev-parse HEAD >actual &&
|
||||
test_cmp expect actual
|
||||
'
|
||||
|
||||
test_lazy_prereq JGIT '
|
||||
type jgit
|
||||
'
|
||||
|
||||
test_expect_success JGIT 'we can read jgit bitmaps' '
|
||||
git clone . compat-jgit &&
|
||||
(
|
||||
cd compat-jgit &&
|
||||
rm -f .git/objects/pack/*.bitmap &&
|
||||
jgit gc &&
|
||||
git rev-list --test-bitmap HEAD
|
||||
)
|
||||
'
|
||||
|
||||
test_expect_success JGIT 'jgit can read our bitmaps' '
|
||||
git clone . compat-us &&
|
||||
(
|
||||
cd compat-us &&
|
||||
git repack -adb &&
|
||||
# jgit gc will barf if it does not like our bitmaps
|
||||
jgit gc
|
||||
)
|
||||
'
|
||||
|
||||
test_done
|
||||
Reference in New Issue
Block a user