mirrors/git
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'ew/many-alternate-optim'

Browse Source 
Optimization for repositories with many alternate object store.

* ew/many-alternate-optim:
  oidtree: a crit-bit tree for odb_loose_cache
  oidcpy_with_padding: constify `src' arg
  make object_directory.loose_objects_subdir_seen a bitmap
  avoid strlen via strbuf_addstr in link_alt_odb_entry
  speed up alt_odb_usable() with many alternates
This commit is contained in:
Junio C Hamano
2021-07-28 13:17:57 -07:00
parent 14793a4e37 92d8ed8ac1
commit e5cc59c77c
16 changed files with 535 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
object-file.c
View File

@ -517,9 +517,9 @@ const char *loose_object_path(struct repository *r, struct strbuf *buf,
*/
static int alt_odb_usable(struct raw_object_store *o,
struct strbuf *path,
const char *normalized_objdir)
const char *normalized_objdir, khiter_t *pos)
{
struct object_directory *odb;
int r;
/* Detect cases where alternate disappeared */
if (!is_directory(path->buf)) {
@ -533,14 +533,20 @@ static int alt_odb_usable(struct raw_object_store *o,
* Prevent the common mistake of listing the same
* thing twice, or object directory itself.
*/
for (odb = o->odb; odb; odb = odb->next) {
if (!fspathcmp(path->buf, odb->path))
return 0;
}
if (!fspathcmp(path->buf, normalized_objdir))
return 0;
if (!o->odb_by_path) {
khiter_t p;
return 1;
o->odb_by_path = kh_init_odb_path_map();
assert(!o->odb->next);
p = kh_put_odb_path_map(o->odb_by_path, o->odb->path, &r);
assert(r == 1); /* never used */
kh_value(o->odb_by_path, p) = o->odb;
}
if (fspatheq(path->buf, normalized_objdir))
return 0;
*pos = kh_put_odb_path_map(o->odb_by_path, path->buf, &r);
/* r: 0 = exists, 1 = never used, 2 = deleted */
return r == 0 ? 0 : 1;
}
/*
@ -561,17 +567,18 @@ static int alt_odb_usable(struct raw_object_store *o,
static void read_info_alternates(struct repository *r,
const char *relative_base,
int depth);
static int link_alt_odb_entry(struct repository *r, const char *entry,
static int link_alt_odb_entry(struct repository *r, const struct strbuf *entry,
const char *relative_base, int depth, const char *normalized_objdir)
{
struct object_directory *ent;
struct strbuf pathbuf = STRBUF_INIT;
khiter_t pos;
if (!is_absolute_path(entry) && relative_base) {
if (!is_absolute_path(entry->buf) && relative_base) {
strbuf_realpath(&pathbuf, relative_base, 1);
strbuf_addch(&pathbuf, '/');
}
strbuf_addstr(&pathbuf, entry);
strbuf_addbuf(&pathbuf, entry);
if (strbuf_normalize_path(&pathbuf) < 0 && relative_base) {
error(_("unable to normalize alternate object path: %s"),
@ -587,23 +594,25 @@ static int link_alt_odb_entry(struct repository *r, const char *entry,
while (pathbuf.len && pathbuf.buf[pathbuf.len - 1] == '/')
strbuf_setlen(&pathbuf, pathbuf.len - 1);
if (!alt_odb_usable(r->objects, &pathbuf, normalized_objdir)) {
if (!alt_odb_usable(r->objects, &pathbuf, normalized_objdir, &pos)) {
strbuf_release(&pathbuf);
return -1;
}
CALLOC_ARRAY(ent, 1);
ent->path = xstrdup(pathbuf.buf);
/* pathbuf.buf is already in r->objects->odb_by_path */
ent->path = strbuf_detach(&pathbuf, NULL);
/* add the alternate entry */
*r->objects->odb_tail = ent;
r->objects->odb_tail = &(ent->next);
ent->next = NULL;
assert(r->objects->odb_by_path);
kh_value(r->objects->odb_by_path, pos) = ent;
/* recursively add alternates */
read_info_alternates(r, pathbuf.buf, depth + 1);
read_info_alternates(r, ent->path, depth + 1);
strbuf_release(&pathbuf);
return 0;
}
@ -660,7 +669,7 @@ static void link_alt_odb_entries(struct repository *r, const char *alt,
alt = parse_alt_odb_entry(alt, sep, &entry);
if (!entry.len)
continue;
link_alt_odb_entry(r, entry.buf,
link_alt_odb_entry(r, &entry,
relative_base, depth, objdirbuf.buf);
}
strbuf_release(&entry);
@ -1178,7 +1187,7 @@ static int quick_has_loose(struct repository *r,
prepare_alt_odb(r);
for (odb = r->objects->odb; odb; odb = odb->next) {
if (oid_array_lookup(odb_loose_cache(odb, oid), oid) >= 0)
if (oidtree_contains(odb_loose_cache(odb, oid), oid))
return 1;
}
return 0;
@ -2454,39 +2463,45 @@ int for_each_loose_object(each_loose_object_fn cb, void *data,
static int append_loose_object(const struct object_id *oid, const char *path,
void *data)
{
oid_array_append(data, oid);
oidtree_insert(data, oid);
return 0;
}
struct oid_array *odb_loose_cache(struct object_directory *odb,
struct oidtree *odb_loose_cache(struct object_directory *odb,
const struct object_id *oid)
{
int subdir_nr = oid->hash[0];
struct strbuf buf = STRBUF_INIT;
size_t word_bits = bitsizeof(odb->loose_objects_subdir_seen[0]);
size_t word_index = subdir_nr / word_bits;
size_t mask = 1 << (subdir_nr % word_bits);
uint32_t *bitmap;
if (subdir_nr < 0 ||
subdir_nr >= ARRAY_SIZE(odb->loose_objects_subdir_seen))
subdir_nr >= bitsizeof(odb->loose_objects_subdir_seen))
BUG("subdir_nr out of range");
if (odb->loose_objects_subdir_seen[subdir_nr])
return &odb->loose_objects_cache[subdir_nr];
bitmap = &odb->loose_objects_subdir_seen[word_index];
if (*bitmap & mask)
return odb->loose_objects_cache;
if (!odb->loose_objects_cache) {
ALLOC_ARRAY(odb->loose_objects_cache, 1);
oidtree_init(odb->loose_objects_cache);
}
strbuf_addstr(&buf, odb->path);
for_each_file_in_obj_subdir(subdir_nr, &buf,
append_loose_object,
NULL, NULL,
&odb->loose_objects_cache[subdir_nr]);
odb->loose_objects_subdir_seen[subdir_nr] = 1;
odb->loose_objects_cache);
*bitmap |= mask;
strbuf_release(&buf);
return &odb->loose_objects_cache[subdir_nr];
return odb->loose_objects_cache;
}
void odb_clear_loose_cache(struct object_directory *odb)
{
int i;
for (i = 0; i < ARRAY_SIZE(odb->loose_objects_cache); i++)
oid_array_clear(&odb->loose_objects_cache[i]);
oidtree_clear(odb->loose_objects_cache);
FREE_AND_NULL(odb->loose_objects_cache);
memset(&odb->loose_objects_subdir_seen, 0,
sizeof(odb->loose_objects_subdir_seen));
}