Merge branch 'tb/bloom-improvements'
"git commit-graph write" learned to limit the number of bloom filters that are computed from scratch with the --max-new-filters option. * tb/bloom-improvements: commit-graph: introduce 'commitGraph.maxNewFilters' builtin/commit-graph.c: introduce '--max-new-filters=<n>' commit-graph: rename 'split_commit_graph_opts' bloom: encode out-of-bounds filters as non-empty bloom/diff: properly short-circuit on max_changes bloom: use provided 'struct bloom_filter_settings' bloom: split 'get_bloom_filter()' in two commit-graph.c: store maximum changed paths commit-graph: respect 'commitGraph.readChangedPaths' t/helper/test-read-graph.c: prepare repo settings commit-graph: pass a 'struct repository *' in more places t4216: use an '&&'-chain commit-graph: introduce 'get_bloom_filter_settings()'
This commit is contained in:
Junio C Hamano
@ -30,6 +30,7 @@ test_expect_success 'setup test - repo, commits, commit graph, log outputs' '
|
||||
rm file_to_be_deleted &&
|
||||
git add . &&
|
||||
git commit -m "file removed" &&
|
||||
git commit --allow-empty -m "empty" &&
|
||||
git commit-graph write --reachable --changed-paths &&
|
||||
|
||||
test_oid_cache <<-EOF
|
||||
@ -37,6 +38,7 @@ test_expect_success 'setup test - repo, commits, commit graph, log outputs' '
|
||||
oid_version sha256:2
|
||||
EOF
|
||||
'
|
||||
|
||||
graph_read_expect () {
|
||||
NUM_CHUNKS=5
|
||||
cat >expect <<- EOF
|
||||
@ -49,7 +51,7 @@ graph_read_expect () {
|
||||
}
|
||||
|
||||
test_expect_success 'commit-graph write wrote out the bloom chunks' '
|
||||
graph_read_expect 15
|
||||
graph_read_expect 16
|
||||
'
|
||||
|
||||
# Turn off any inherited trace2 settings for this test.
|
||||
@ -58,14 +60,14 @@ sane_unset GIT_TRACE2_PERF_BRIEF
|
||||
sane_unset GIT_TRACE2_CONFIG_PARAMS
|
||||
|
||||
setup () {
|
||||
rm "$TRASH_DIRECTORY/trace.perf"
|
||||
rm -f "$TRASH_DIRECTORY/trace.perf" &&
|
||||
git -c core.commitGraph=false log --pretty="format:%s" $1 >log_wo_bloom &&
|
||||
GIT_TRACE2_PERF="$TRASH_DIRECTORY/trace.perf" git -c core.commitGraph=true log --pretty="format:%s" $1 >log_w_bloom
|
||||
}
|
||||
|
||||
test_bloom_filters_used () {
|
||||
log_args=$1
|
||||
bloom_trace_prefix="statistics:{\"filter_not_present\":0,\"maybe\""
|
||||
bloom_trace_prefix="statistics:{\"filter_not_present\":${2:-0},\"maybe\""
|
||||
setup "$log_args" &&
|
||||
grep -q "$bloom_trace_prefix" "$TRASH_DIRECTORY/trace.perf" &&
|
||||
test_cmp log_wo_bloom log_w_bloom &&
|
||||
@ -95,7 +97,9 @@ do
|
||||
"--ancestry-path side..master"
|
||||
do
|
||||
test_expect_success "git log option: $option for path: $path" '
|
||||
test_bloom_filters_used "$option -- $path"
|
||||
test_bloom_filters_used "$option -- $path" &&
|
||||
test_config commitgraph.readChangedPaths false &&
|
||||
test_bloom_filters_not_used "$option -- $path"
|
||||
'
|
||||
done
|
||||
done
|
||||
@ -139,8 +143,11 @@ test_expect_success 'setup - add commit-graph to the chain without Bloom filters
|
||||
test_line_count = 2 .git/objects/info/commit-graphs/commit-graph-chain
|
||||
'
|
||||
|
||||
test_expect_success 'Do not use Bloom filters if the latest graph does not have Bloom filters.' '
|
||||
test_bloom_filters_not_used "-- A/B"
|
||||
test_expect_success 'use Bloom filters even if the latest graph does not have Bloom filters' '
|
||||
# Ensure that the number of empty filters is equal to the number of
|
||||
# filters in the latest graph layer to prove that they are loaded (and
|
||||
# ignored).
|
||||
test_bloom_filters_used "-- A/B" 3
|
||||
'
|
||||
|
||||
test_expect_success 'setup - add commit-graph to the chain with Bloom filters' '
|
||||
@ -151,7 +158,7 @@ test_expect_success 'setup - add commit-graph to the chain with Bloom filters' '
|
||||
|
||||
test_bloom_filters_used_when_some_filters_are_missing () {
|
||||
log_args=$1
|
||||
bloom_trace_prefix="statistics:{\"filter_not_present\":3,\"maybe\":6,\"definitely_not\":8"
|
||||
bloom_trace_prefix="statistics:{\"filter_not_present\":3,\"maybe\":6,\"definitely_not\":9"
|
||||
setup "$log_args" &&
|
||||
grep -q "$bloom_trace_prefix" "$TRASH_DIRECTORY/trace.perf" &&
|
||||
test_cmp log_wo_bloom log_w_bloom
|
||||
@ -169,31 +176,230 @@ test_expect_success 'persist filter settings' '
|
||||
GIT_TEST_BLOOM_SETTINGS_NUM_HASHES=9 \
|
||||
GIT_TEST_BLOOM_SETTINGS_BITS_PER_ENTRY=15 \
|
||||
git commit-graph write --reachable --changed-paths &&
|
||||
grep "{\"hash_version\":1,\"num_hashes\":9,\"bits_per_entry\":15}" trace2.txt &&
|
||||
grep "{\"hash_version\":1,\"num_hashes\":9,\"bits_per_entry\":15,\"max_changed_paths\":512" trace2.txt &&
|
||||
GIT_TRACE2_EVENT="$(pwd)/trace2-auto.txt" \
|
||||
GIT_TRACE2_EVENT_NESTING=5 \
|
||||
git commit-graph write --reachable --changed-paths &&
|
||||
grep "{\"hash_version\":1,\"num_hashes\":9,\"bits_per_entry\":15}" trace2-auto.txt
|
||||
grep "{\"hash_version\":1,\"num_hashes\":9,\"bits_per_entry\":15,\"max_changed_paths\":512" trace2-auto.txt
|
||||
'
|
||||
|
||||
test_max_changed_paths () {
|
||||
grep "\"max_changed_paths\":$1" $2
|
||||
}
|
||||
|
||||
test_filter_not_computed () {
|
||||
grep "\"key\":\"filter-not-computed\",\"value\":\"$1\"" $2
|
||||
}
|
||||
|
||||
test_filter_computed () {
|
||||
grep "\"key\":\"filter-computed\",\"value\":\"$1\"" $2
|
||||
}
|
||||
|
||||
test_filter_trunc_empty () {
|
||||
grep "\"key\":\"filter-trunc-empty\",\"value\":\"$1\"" $2
|
||||
}
|
||||
|
||||
test_filter_trunc_large () {
|
||||
grep "\"key\":\"filter-trunc-large\",\"value\":\"$1\"" $2
|
||||
}
|
||||
|
||||
test_expect_success 'correctly report changes over limit' '
|
||||
git init 513changes &&
|
||||
git init limits &&
|
||||
(
|
||||
cd 513changes &&
|
||||
for i in $(test_seq 1 513)
|
||||
cd limits &&
|
||||
mkdir d &&
|
||||
mkdir d/e &&
|
||||
|
||||
for i in $(test_seq 1 2)
|
||||
do
|
||||
echo $i >file$i.txt || return 1
|
||||
printf $i >d/file$i.txt &&
|
||||
printf $i >d/e/file$i.txt || return 1
|
||||
done &&
|
||||
git add . &&
|
||||
|
||||
mkdir mode &&
|
||||
printf bash >mode/script.sh &&
|
||||
|
||||
mkdir foo &&
|
||||
touch foo/bar &&
|
||||
touch foo.txt &&
|
||||
|
||||
git add d foo foo.txt mode &&
|
||||
git commit -m "files" &&
|
||||
git commit-graph write --reachable --changed-paths &&
|
||||
for i in $(test_seq 1 513)
|
||||
|
||||
# Commit has 7 file and 4 directory adds
|
||||
GIT_TEST_BLOOM_SETTINGS_MAX_CHANGED_PATHS=10 \
|
||||
GIT_TRACE2_EVENT="$(pwd)/trace" \
|
||||
git commit-graph write --reachable --changed-paths &&
|
||||
test_max_changed_paths 10 trace &&
|
||||
test_filter_computed 1 trace &&
|
||||
test_filter_trunc_large 1 trace &&
|
||||
|
||||
for path in $(git ls-tree -r --name-only HEAD)
|
||||
do
|
||||
git -c core.commitGraph=false log -- file$i.txt >expect &&
|
||||
git log -- file$i.txt >actual &&
|
||||
git -c commitGraph.readChangedPaths=false log \
|
||||
-- $path >expect &&
|
||||
git log -- $path >actual &&
|
||||
test_cmp expect actual || return 1
|
||||
done &&
|
||||
|
||||
# Make a variety of path changes
|
||||
printf new1 >d/e/file1.txt &&
|
||||
printf new2 >d/file2.txt &&
|
||||
rm d/e/file2.txt &&
|
||||
rm -r foo &&
|
||||
printf text >foo &&
|
||||
mkdir f &&
|
||||
printf new1 >f/file1.txt &&
|
||||
|
||||
# including a mode-only change (counts as modified)
|
||||
git update-index --chmod=+x mode/script.sh &&
|
||||
|
||||
git add foo d f &&
|
||||
git commit -m "complicated" &&
|
||||
|
||||
# start from scratch and rebuild
|
||||
rm -f .git/objects/info/commit-graph &&
|
||||
GIT_TEST_BLOOM_SETTINGS_MAX_CHANGED_PATHS=10 \
|
||||
GIT_TRACE2_EVENT="$(pwd)/trace-edit" \
|
||||
git commit-graph write --reachable --changed-paths &&
|
||||
test_max_changed_paths 10 trace-edit &&
|
||||
test_filter_computed 2 trace-edit &&
|
||||
test_filter_trunc_large 2 trace-edit &&
|
||||
|
||||
for path in $(git ls-tree -r --name-only HEAD)
|
||||
do
|
||||
git -c commitGraph.readChangedPaths=false log \
|
||||
-- $path >expect &&
|
||||
git log -- $path >actual &&
|
||||
test_cmp expect actual || return 1
|
||||
done &&
|
||||
|
||||
# start from scratch and rebuild
|
||||
rm -f .git/objects/info/commit-graph &&
|
||||
GIT_TEST_BLOOM_SETTINGS_MAX_CHANGED_PATHS=11 \
|
||||
GIT_TRACE2_EVENT="$(pwd)/trace-update" \
|
||||
git commit-graph write --reachable --changed-paths &&
|
||||
test_max_changed_paths 11 trace-update &&
|
||||
test_filter_computed 2 trace-update &&
|
||||
test_filter_trunc_large 0 trace-update &&
|
||||
|
||||
for path in $(git ls-tree -r --name-only HEAD)
|
||||
do
|
||||
git -c commitGraph.readChangedPaths=false log \
|
||||
-- $path >expect &&
|
||||
git log -- $path >actual &&
|
||||
test_cmp expect actual || return 1
|
||||
done
|
||||
)
|
||||
'
|
||||
|
||||
test_expect_success 'correctly report commits with no changed paths' '
|
||||
git init empty &&
|
||||
test_when_finished "rm -fr empty" &&
|
||||
(
|
||||
cd empty &&
|
||||
|
||||
git commit --allow-empty -m "initial commit" &&
|
||||
|
||||
GIT_TRACE2_EVENT="$(pwd)/trace.event" \
|
||||
git commit-graph write --reachable --changed-paths &&
|
||||
test_filter_computed 1 trace.event &&
|
||||
test_filter_not_computed 0 trace.event &&
|
||||
test_filter_trunc_empty 1 trace.event &&
|
||||
test_filter_trunc_large 0 trace.event
|
||||
)
|
||||
'
|
||||
|
||||
test_expect_success 'Bloom generation is limited by --max-new-filters' '
|
||||
(
|
||||
cd limits &&
|
||||
test_commit c2 filter &&
|
||||
test_commit c3 filter &&
|
||||
test_commit c4 no-filter &&
|
||||
|
||||
rm -f trace.event &&
|
||||
GIT_TRACE2_EVENT="$(pwd)/trace.event" \
|
||||
git commit-graph write --reachable --split=replace \
|
||||
--changed-paths --max-new-filters=2 &&
|
||||
|
||||
test_filter_computed 2 trace.event &&
|
||||
test_filter_not_computed 3 trace.event &&
|
||||
test_filter_trunc_empty 0 trace.event &&
|
||||
test_filter_trunc_large 0 trace.event
|
||||
)
|
||||
'
|
||||
|
||||
test_expect_success 'Bloom generation backfills previously-skipped filters' '
|
||||
# Check specifying commitGraph.maxNewFilters over "git config" works.
|
||||
test_config -C limits commitGraph.maxNewFilters 1 &&
|
||||
(
|
||||
cd limits &&
|
||||
|
||||
rm -f trace.event &&
|
||||
GIT_TRACE2_EVENT="$(pwd)/trace.event" \
|
||||
git commit-graph write --reachable --changed-paths \
|
||||
--split=replace &&
|
||||
test_filter_computed 1 trace.event &&
|
||||
test_filter_not_computed 4 trace.event &&
|
||||
test_filter_trunc_empty 0 trace.event &&
|
||||
test_filter_trunc_large 0 trace.event
|
||||
)
|
||||
'
|
||||
|
||||
test_expect_success '--max-new-filters overrides configuration' '
|
||||
git init override &&
|
||||
test_when_finished "rm -fr override" &&
|
||||
test_config -C override commitGraph.maxNewFilters 2 &&
|
||||
(
|
||||
cd override &&
|
||||
test_commit one &&
|
||||
test_commit two &&
|
||||
|
||||
rm -f trace.event &&
|
||||
GIT_TRACE2_EVENT="$(pwd)/trace.event" \
|
||||
git commit-graph write --reachable --changed-paths \
|
||||
--max-new-filters=1 &&
|
||||
test_filter_computed 1 trace.event &&
|
||||
test_filter_not_computed 1 trace.event &&
|
||||
test_filter_trunc_empty 0 trace.event &&
|
||||
test_filter_trunc_large 0 trace.event
|
||||
)
|
||||
'
|
||||
|
||||
test_expect_success 'Bloom generation backfills empty commits' '
|
||||
git init empty &&
|
||||
test_when_finished "rm -fr empty" &&
|
||||
(
|
||||
cd empty &&
|
||||
for i in $(test_seq 1 6)
|
||||
do
|
||||
git commit --allow-empty -m "$i"
|
||||
done &&
|
||||
|
||||
# Generate Bloom filters for empty commits 1-6, two at a time.
|
||||
for i in $(test_seq 1 3)
|
||||
do
|
||||
rm -f trace.event &&
|
||||
GIT_TRACE2_EVENT="$(pwd)/trace.event" \
|
||||
git commit-graph write --reachable \
|
||||
--changed-paths --max-new-filters=2 &&
|
||||
test_filter_computed 2 trace.event &&
|
||||
test_filter_not_computed 4 trace.event &&
|
||||
test_filter_trunc_empty 2 trace.event &&
|
||||
test_filter_trunc_large 0 trace.event
|
||||
done &&
|
||||
|
||||
# Finally, make sure that once all commits have filters, that
|
||||
# none are subsequently recomputed.
|
||||
rm -f trace.event &&
|
||||
GIT_TRACE2_EVENT="$(pwd)/trace.event" \
|
||||
git commit-graph write --reachable \
|
||||
--changed-paths --max-new-filters=2 &&
|
||||
test_filter_computed 0 trace.event &&
|
||||
test_filter_not_computed 6 trace.event &&
|
||||
test_filter_trunc_empty 0 trace.event &&
|
||||
test_filter_trunc_large 0 trace.event
|
||||
)
|
||||
'
|
||||
|
||||
test_done
|
||||
|
||||
Reference in New Issue
Block a user