Sync 'ds/multi-pack-index' to v2.19.0-rc0
* ds/multi-pack-index: (23 commits) midx: clear midx on repack packfile: skip loading index if in multi-pack-index midx: prevent duplicate packfile loads midx: use midx in approximate_object_count midx: use existing midx when writing new one midx: use midx in abbreviation calculations midx: read objects from multi-pack-index config: create core.multiPackIndex setting midx: write object offsets midx: write object id fanout chunk midx: write object ids in a chunk midx: sort and deduplicate objects from packfiles midx: read pack names into array multi-pack-index: write pack names in chunk multi-pack-index: read packfile list packfile: generalize pack directory list t5319: expand test data multi-pack-index: load into memory midx: write header information to lockfile multi-pack-index: add 'write' verb ...
This commit is contained in:
Junio C Hamano
169
packfile.c
169
packfile.c
@ -15,6 +15,7 @@
|
||||
#include "tree-walk.h"
|
||||
#include "tree.h"
|
||||
#include "object-store.h"
|
||||
#include "midx.h"
|
||||
|
||||
char *odb_pack_name(struct strbuf *buf,
|
||||
const unsigned char *sha1,
|
||||
@ -196,6 +197,23 @@ int open_pack_index(struct packed_git *p)
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint32_t get_pack_fanout(struct packed_git *p, uint32_t value)
|
||||
{
|
||||
const uint32_t *level1_ofs = p->index_data;
|
||||
|
||||
if (!level1_ofs) {
|
||||
if (open_pack_index(p))
|
||||
return 0;
|
||||
level1_ofs = p->index_data;
|
||||
}
|
||||
|
||||
if (p->index_version > 1) {
|
||||
level1_ofs += 2;
|
||||
}
|
||||
|
||||
return ntohl(level1_ofs[value]);
|
||||
}
|
||||
|
||||
static struct packed_git *alloc_packed_git(int extra)
|
||||
{
|
||||
struct packed_git *p = xmalloc(st_add(sizeof(*p), extra));
|
||||
@ -451,8 +469,19 @@ static int open_packed_git_1(struct packed_git *p)
|
||||
ssize_t read_result;
|
||||
const unsigned hashsz = the_hash_algo->rawsz;
|
||||
|
||||
if (!p->index_data && open_pack_index(p))
|
||||
return error("packfile %s index unavailable", p->pack_name);
|
||||
if (!p->index_data) {
|
||||
struct multi_pack_index *m;
|
||||
const char *pack_name = strrchr(p->pack_name, '/');
|
||||
|
||||
for (m = the_repository->objects->multi_pack_index;
|
||||
m; m = m->next) {
|
||||
if (midx_contains_pack(m, pack_name))
|
||||
break;
|
||||
}
|
||||
|
||||
if (!m && open_pack_index(p))
|
||||
return error("packfile %s index unavailable", p->pack_name);
|
||||
}
|
||||
|
||||
if (!pack_max_fds) {
|
||||
unsigned int max_fds = get_max_fd_limit();
|
||||
@ -503,6 +532,10 @@ static int open_packed_git_1(struct packed_git *p)
|
||||
" supported (try upgrading GIT to a newer version)",
|
||||
p->pack_name, ntohl(hdr.hdr_version));
|
||||
|
||||
/* Skip index checking if in multi-pack-index */
|
||||
if (!p->index_data)
|
||||
return 0;
|
||||
|
||||
/* Verify the pack matches its index. */
|
||||
if (p->num_objects != ntohl(hdr.hdr_entries))
|
||||
return error("packfile %s claims to have %"PRIu32" objects"
|
||||
@ -738,13 +771,14 @@ static void report_pack_garbage(struct string_list *list)
|
||||
report_helper(list, seen_bits, first, list->nr);
|
||||
}
|
||||
|
||||
static void prepare_packed_git_one(struct repository *r, char *objdir, int local)
|
||||
void for_each_file_in_pack_dir(const char *objdir,
|
||||
each_file_in_pack_dir_fn fn,
|
||||
void *data)
|
||||
{
|
||||
struct strbuf path = STRBUF_INIT;
|
||||
size_t dirnamelen;
|
||||
DIR *dir;
|
||||
struct dirent *de;
|
||||
struct string_list garbage = STRING_LIST_INIT_DUP;
|
||||
|
||||
strbuf_addstr(&path, objdir);
|
||||
strbuf_addstr(&path, "/pack");
|
||||
@ -759,51 +793,84 @@ static void prepare_packed_git_one(struct repository *r, char *objdir, int local
|
||||
strbuf_addch(&path, '/');
|
||||
dirnamelen = path.len;
|
||||
while ((de = readdir(dir)) != NULL) {
|
||||
struct packed_git *p;
|
||||
size_t base_len;
|
||||
|
||||
if (is_dot_or_dotdot(de->d_name))
|
||||
continue;
|
||||
|
||||
strbuf_setlen(&path, dirnamelen);
|
||||
strbuf_addstr(&path, de->d_name);
|
||||
|
||||
base_len = path.len;
|
||||
if (strip_suffix_mem(path.buf, &base_len, ".idx")) {
|
||||
/* Don't reopen a pack we already have. */
|
||||
for (p = r->objects->packed_git; p;
|
||||
p = p->next) {
|
||||
size_t len;
|
||||
if (strip_suffix(p->pack_name, ".pack", &len) &&
|
||||
len == base_len &&
|
||||
!memcmp(p->pack_name, path.buf, len))
|
||||
break;
|
||||
}
|
||||
if (p == NULL &&
|
||||
/*
|
||||
* See if it really is a valid .idx file with
|
||||
* corresponding .pack file that we can map.
|
||||
*/
|
||||
(p = add_packed_git(path.buf, path.len, local)) != NULL)
|
||||
install_packed_git(r, p);
|
||||
fn(path.buf, path.len, de->d_name, data);
|
||||
}
|
||||
|
||||
closedir(dir);
|
||||
strbuf_release(&path);
|
||||
}
|
||||
|
||||
struct prepare_pack_data {
|
||||
struct repository *r;
|
||||
struct string_list *garbage;
|
||||
int local;
|
||||
struct multi_pack_index *m;
|
||||
};
|
||||
|
||||
static void prepare_pack(const char *full_name, size_t full_name_len,
|
||||
const char *file_name, void *_data)
|
||||
{
|
||||
struct prepare_pack_data *data = (struct prepare_pack_data *)_data;
|
||||
struct packed_git *p;
|
||||
size_t base_len = full_name_len;
|
||||
|
||||
if (strip_suffix_mem(full_name, &base_len, ".idx")) {
|
||||
if (data->m && midx_contains_pack(data->m, file_name))
|
||||
return;
|
||||
/* Don't reopen a pack we already have. */
|
||||
for (p = data->r->objects->packed_git; p; p = p->next) {
|
||||
size_t len;
|
||||
if (strip_suffix(p->pack_name, ".pack", &len) &&
|
||||
len == base_len &&
|
||||
!memcmp(p->pack_name, full_name, len))
|
||||
break;
|
||||
}
|
||||
|
||||
if (!report_garbage)
|
||||
continue;
|
||||
|
||||
if (ends_with(de->d_name, ".idx") ||
|
||||
ends_with(de->d_name, ".pack") ||
|
||||
ends_with(de->d_name, ".bitmap") ||
|
||||
ends_with(de->d_name, ".keep") ||
|
||||
ends_with(de->d_name, ".promisor"))
|
||||
string_list_append(&garbage, path.buf);
|
||||
else
|
||||
report_garbage(PACKDIR_FILE_GARBAGE, path.buf);
|
||||
if (!p) {
|
||||
p = add_packed_git(full_name, full_name_len, data->local);
|
||||
if (p)
|
||||
install_packed_git(data->r, p);
|
||||
}
|
||||
}
|
||||
closedir(dir);
|
||||
report_pack_garbage(&garbage);
|
||||
string_list_clear(&garbage, 0);
|
||||
strbuf_release(&path);
|
||||
|
||||
if (!report_garbage)
|
||||
return;
|
||||
|
||||
if (ends_with(file_name, ".idx") ||
|
||||
ends_with(file_name, ".pack") ||
|
||||
ends_with(file_name, ".bitmap") ||
|
||||
ends_with(file_name, ".keep") ||
|
||||
ends_with(file_name, ".promisor"))
|
||||
string_list_append(data->garbage, full_name);
|
||||
else
|
||||
report_garbage(PACKDIR_FILE_GARBAGE, full_name);
|
||||
}
|
||||
|
||||
static void prepare_packed_git_one(struct repository *r, char *objdir, int local)
|
||||
{
|
||||
struct prepare_pack_data data;
|
||||
struct string_list garbage = STRING_LIST_INIT_DUP;
|
||||
|
||||
data.m = r->objects->multi_pack_index;
|
||||
|
||||
/* look for the multi-pack-index for this object directory */
|
||||
while (data.m && strcmp(data.m->object_dir, objdir))
|
||||
data.m = data.m->next;
|
||||
|
||||
data.r = r;
|
||||
data.garbage = &garbage;
|
||||
data.local = local;
|
||||
|
||||
for_each_file_in_pack_dir(objdir, prepare_pack, &data);
|
||||
|
||||
report_pack_garbage(data.garbage);
|
||||
string_list_clear(data.garbage, 0);
|
||||
}
|
||||
|
||||
static void prepare_packed_git(struct repository *r);
|
||||
@ -818,10 +885,13 @@ unsigned long approximate_object_count(void)
|
||||
{
|
||||
if (!the_repository->objects->approximate_object_count_valid) {
|
||||
unsigned long count;
|
||||
struct multi_pack_index *m;
|
||||
struct packed_git *p;
|
||||
|
||||
prepare_packed_git(the_repository);
|
||||
count = 0;
|
||||
for (m = get_multi_pack_index(the_repository); m; m = m->next)
|
||||
count += m->num_objects;
|
||||
for (p = the_repository->objects->packed_git; p; p = p->next) {
|
||||
if (open_pack_index(p))
|
||||
continue;
|
||||
@ -893,10 +963,13 @@ static void prepare_packed_git(struct repository *r)
|
||||
|
||||
if (r->objects->packed_git_initialized)
|
||||
return;
|
||||
prepare_multi_pack_index_one(r, r->objects->objectdir);
|
||||
prepare_packed_git_one(r, r->objects->objectdir, 1);
|
||||
prepare_alt_odb(r);
|
||||
for (alt = r->objects->alt_odb_list; alt; alt = alt->next)
|
||||
for (alt = r->objects->alt_odb_list; alt; alt = alt->next) {
|
||||
prepare_multi_pack_index_one(r, alt->path);
|
||||
prepare_packed_git_one(r, alt->path, 0);
|
||||
}
|
||||
rearrange_packed_git(r);
|
||||
prepare_packed_git_mru(r);
|
||||
r->objects->packed_git_initialized = 1;
|
||||
@ -915,6 +988,12 @@ struct packed_git *get_packed_git(struct repository *r)
|
||||
return r->objects->packed_git;
|
||||
}
|
||||
|
||||
struct multi_pack_index *get_multi_pack_index(struct repository *r)
|
||||
{
|
||||
prepare_packed_git(r);
|
||||
return r->objects->multi_pack_index;
|
||||
}
|
||||
|
||||
struct list_head *get_packed_git_mru(struct repository *r)
|
||||
{
|
||||
prepare_packed_git(r);
|
||||
@ -1856,11 +1935,17 @@ static int fill_pack_entry(const struct object_id *oid,
|
||||
int find_pack_entry(struct repository *r, const struct object_id *oid, struct pack_entry *e)
|
||||
{
|
||||
struct list_head *pos;
|
||||
struct multi_pack_index *m;
|
||||
|
||||
prepare_packed_git(r);
|
||||
if (!r->objects->packed_git)
|
||||
if (!r->objects->packed_git && !r->objects->multi_pack_index)
|
||||
return 0;
|
||||
|
||||
for (m = r->objects->multi_pack_index; m; m = m->next) {
|
||||
if (fill_midx_entry(oid, e, m))
|
||||
return 1;
|
||||
}
|
||||
|
||||
list_for_each(pos, &r->objects->packed_git_mru) {
|
||||
struct packed_git *p = list_entry(pos, struct packed_git, mru);
|
||||
if (fill_pack_entry(oid, e, p)) {
|
||||
|
||||
Reference in New Issue
Block a user