mirrors/git
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'ds/multi-pack-index'

Browse Source 
When there are too many packfiles in a repository (which is not
recommended), looking up an object in these would require
consulting many pack .idx files; a new mechanism to have a single
file that consolidates all of these .idx files is introduced.

* ds/multi-pack-index: (32 commits)
  pack-objects: consider packs in multi-pack-index
  midx: test a few commands that use get_all_packs
  treewide: use get_all_packs
  packfile: add all_packs list
  midx: fix bug that skips midx with alternates
  midx: stop reporting garbage
  midx: mark bad packed objects
  multi-pack-index: store local property
  multi-pack-index: provide more helpful usage info
  midx: clear midx on repack
  packfile: skip loading index if in multi-pack-index
  midx: prevent duplicate packfile loads
  midx: use midx in approximate_object_count
  midx: use existing midx when writing new one
  midx: use midx in abbreviation calculations
  midx: read objects from multi-pack-index
  config: create core.multiPackIndex setting
  midx: write object offsets
  midx: write object id fanout chunk
  midx: write object ids in a chunk
  ...
This commit is contained in:
Junio C Hamano
2018-09-17 13:53:50 -07:00
parent 7dc341cedb 6a22d52126
commit 49f210fd52
31 changed files with 1859 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
builtin/pack-objects.c
View File

@ -31,6 +31,7 @@
#include "packfile.h"
#include "object-store.h"
#include "dir.h"
#include "midx.h"
#define IN_PACK(obj) oe_in_pack(&to_pack, obj)
#define SIZE(obj) oe_size(&to_pack, obj)
@ -1040,6 +1041,7 @@ static int want_object_in_pack(const struct object_id *oid,
{
int want;
struct list_head *pos;
struct multi_pack_index *m;
if (!exclude && local && has_loose_object_nonlocal(oid))
return 0;
@ -1054,6 +1056,32 @@ static int want_object_in_pack(const struct object_id *oid,
if (want != -1)
return want;
}
for (m = get_multi_pack_index(the_repository); m; m = m->next) {
struct pack_entry e;
if (fill_midx_entry(oid, &e, m)) {
struct packed_git *p = e.p;
off_t offset;
if (p == *found_pack)
offset = *found_offset;
else
offset = find_pack_entry_one(oid->hash, p);
if (offset) {
if (!*found_pack) {
if (!is_pack_valid(p))
continue;
*found_offset = offset;
*found_pack = p;
}
want = want_found_object(exclude, p);
if (want != -1)
return want;
}
}
}
list_for_each(pos, get_packed_git_mru(the_repository)) {
struct packed_git *p = list_entry(pos, struct packed_git, mru);
off_t offset;
@ -2806,7 +2834,7 @@ static void add_objects_in_unpacked_packs(struct rev_info *revs)
memset(&in_pack, 0, sizeof(in_pack));
for (p = get_packed_git(the_repository); p; p = p->next) {
for (p = get_all_packs(the_repository); p; p = p->next) {
struct object_id oid;
struct object *o;
@ -2870,7 +2898,7 @@ static int has_sha1_pack_kept_or_nonlocal(const struct object_id *oid)
struct packed_git *p;
p = (last_found != (void *)1) ? last_found :
get_packed_git(the_repository);
get_all_packs(the_repository);
while (p) {
if ((!p->pack_local || p->pack_keep ||
@ -2880,7 +2908,7 @@ static int has_sha1_pack_kept_or_nonlocal(const struct object_id *oid)
return 1;
}
if (p == last_found)
p = get_packed_git(the_repository);
p = get_all_packs(the_repository);
else
p = p->next;
if (p == last_found)
@ -2916,7 +2944,7 @@ static void loosen_unused_packed_objects(struct rev_info *revs)
uint32_t i;
struct object_id oid;
for (p = get_packed_git(the_repository); p; p = p->next) {
for (p = get_all_packs(the_repository); p; p = p->next) {
if (!p->pack_local || p->pack_keep || p->pack_keep_in_core)
continue;
@ -3063,7 +3091,7 @@ static void add_extra_kept_packs(const struct string_list *names)
if (!names->nr)
return;
for (p = get_packed_git(the_repository); p; p = p->next) {
for (p = get_all_packs(the_repository); p; p = p->next) {
const char *name = basename(p->pack_name);
int i;
@ -3336,7 +3364,7 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
add_extra_kept_packs(&keep_pack_list);
if (ignore_packed_keep_on_disk) {
struct packed_git *p;
for (p = get_packed_git(the_repository); p; p = p->next)
for (p = get_all_packs(the_repository); p; p = p->next)
if (p->pack_local && p->pack_keep)
break;
if (!p) /* no keep-able packs found */
@ -3349,7 +3377,7 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
* it also covers non-local objects
*/
struct packed_git *p;
for (p = get_packed_git(the_repository); p; p = p->next) {
for (p = get_all_packs(the_repository); p; p = p->next) {
if (!p->pack_local) {
have_non_local_packs = 1;
break;