ba5a81d52b
The murmur3 implementation in bloom.c has a bug when converting series
of 4 bytes into network-order integers when char is signed (which is
controllable by a compiler option, and the default signedness of char is
platform-specific). When a string contains characters with the high bit
set, this bug causes results that, although internally consistent within
Git, does not accord with other implementations of murmur3 (thus,
the changed path filters wouldn't be readable by other off-the-shelf
implementatios of murmur3) and even with Git binaries that were compiled
with different signedness of char. This bug affects both how Git writes
changed path filters to disk and how Git interprets changed path filters
on disk.
Therefore, introduce a new version (2) of changed path filters that
corrects this problem. The existing version (1) is still supported and
is still the default, but users should migrate away from it as soon
as possible.
Because this bug only manifests with characters that have the high bit
set, it may be possible that some (or all) commits in a given repo would
have the same changed path filter both before and after this fix is
applied. However, in order to determine whether this is the case, the
changed paths would first have to be computed, at which point it is not
much more expensive to just compute a new changed path filter.
So this patch does not include any mechanism to "salvage" changed path
filters from repositories. There is also no "mixed" mode - for each
invocation of Git, reading and writing changed path filters are done
with the same version number; this version number may be explicitly
stated (typically if the user knows which version they need) or
automatically determined from the version of the existing changed path
filters in the repository.
There is a change in write_commit_graph(). graph_read_bloom_data()
makes it possible for chunk_bloom_data to be non-NULL but
bloom_filter_settings to be NULL, which causes a segfault later on. I
produced such a segfault while developing this patch, but couldn't find
a way to reproduce it neither after this complete patch (or before),
but in any case it seemed like a good thing to include that might help
future patch authors.
The value in t0095 was obtained from another murmur3 implementation
using the following Go source code:
package main
import "fmt"
import "github.com/spaolacci/murmur3"
func main() {
fmt.Printf("%x\n", murmur3.Sum32([]byte("Hello world!")))
fmt.Printf("%x\n", murmur3.Sum32([]byte{0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}))
}
Signed-off-by: Jonathan Tan <jonathantanmy@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
Signed-off-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
Signed-off-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
108 lines
2.6 KiB
C
108 lines
2.6 KiB
C
#include "test-tool.h"
|
|
#include "bloom.h"
|
|
#include "hex.h"
|
|
#include "commit.h"
|
|
#include "repository.h"
|
|
#include "setup.h"
|
|
|
|
static struct bloom_filter_settings settings = DEFAULT_BLOOM_FILTER_SETTINGS;
|
|
|
|
static void add_string_to_filter(const char *data, struct bloom_filter *filter) {
|
|
struct bloom_key key;
|
|
int i;
|
|
|
|
fill_bloom_key(data, strlen(data), &key, &settings);
|
|
printf("Hashes:");
|
|
for (i = 0; i < settings.num_hashes; i++){
|
|
printf("0x%08x|", key.hashes[i]);
|
|
}
|
|
printf("\n");
|
|
add_key_to_filter(&key, filter, &settings);
|
|
clear_bloom_key(&key);
|
|
}
|
|
|
|
static void print_bloom_filter(struct bloom_filter *filter) {
|
|
int i;
|
|
|
|
if (!filter) {
|
|
printf("No filter.\n");
|
|
return;
|
|
}
|
|
printf("Filter_Length:%d\n", (int)filter->len);
|
|
printf("Filter_Data:");
|
|
for (i = 0; i < filter->len; i++) {
|
|
printf("%02x|", filter->data[i]);
|
|
}
|
|
printf("\n");
|
|
}
|
|
|
|
static void get_bloom_filter_for_commit(const struct object_id *commit_oid)
|
|
{
|
|
struct commit *c;
|
|
struct bloom_filter *filter;
|
|
c = lookup_commit(the_repository, commit_oid);
|
|
filter = get_or_compute_bloom_filter(the_repository, c, 1,
|
|
&settings,
|
|
NULL);
|
|
print_bloom_filter(filter);
|
|
}
|
|
|
|
static const char *bloom_usage = "\n"
|
|
" test-tool bloom get_murmur3 <string>\n"
|
|
" test-tool bloom get_murmur3_seven_highbit\n"
|
|
" test-tool bloom generate_filter <string> [<string>...]\n"
|
|
" test-tool bloom get_filter_for_commit <commit-hex>\n";
|
|
|
|
int cmd__bloom(int argc, const char **argv)
|
|
{
|
|
setup_git_directory();
|
|
|
|
if (argc < 2)
|
|
usage(bloom_usage);
|
|
|
|
if (!strcmp(argv[1], "get_murmur3")) {
|
|
uint32_t hashed;
|
|
if (argc < 3)
|
|
usage(bloom_usage);
|
|
hashed = murmur3_seeded_v2(0, argv[2], strlen(argv[2]));
|
|
printf("Murmur3 Hash with seed=0:0x%08x\n", hashed);
|
|
}
|
|
|
|
if (!strcmp(argv[1], "get_murmur3_seven_highbit")) {
|
|
uint32_t hashed;
|
|
hashed = murmur3_seeded_v2(0, "\x99\xaa\xbb\xcc\xdd\xee\xff", 7);
|
|
printf("Murmur3 Hash with seed=0:0x%08x\n", hashed);
|
|
}
|
|
|
|
if (!strcmp(argv[1], "generate_filter")) {
|
|
struct bloom_filter filter;
|
|
int i = 2;
|
|
filter.len = (settings.bits_per_entry + BITS_PER_WORD - 1) / BITS_PER_WORD;
|
|
CALLOC_ARRAY(filter.data, filter.len);
|
|
|
|
if (argc - 1 < i)
|
|
usage(bloom_usage);
|
|
|
|
while (argv[i]) {
|
|
add_string_to_filter(argv[i], &filter);
|
|
i++;
|
|
}
|
|
|
|
print_bloom_filter(&filter);
|
|
free(filter.data);
|
|
}
|
|
|
|
if (!strcmp(argv[1], "get_filter_for_commit")) {
|
|
struct object_id oid;
|
|
const char *end;
|
|
if (argc < 3)
|
|
usage(bloom_usage);
|
|
if (parse_oid_hex(argv[2], &oid, &end))
|
|
die("cannot parse oid '%s'", argv[2]);
|
|
init_bloom_filters();
|
|
get_bloom_filter_for_commit(&oid);
|
|
}
|
|
|
|
return 0;
|
|
}
|