Merge branch 'jk/pack-bitmap'
Borrow the bitmap index into packfiles from JGit to speed up enumeration of objects involved in a commit range without having to fully traverse the history. * jk/pack-bitmap: (26 commits) ewah: unconditionally ntohll ewah data ewah: support platforms that require aligned reads read-cache: use get_be32 instead of hand-rolled ntoh_l block-sha1: factor out get_be and put_be wrappers do not discard revindex when re-preparing packfiles pack-bitmap: implement optional name_hash cache t/perf: add tests for pack bitmaps t: add basic bitmap functionality tests count-objects: recognize .bitmap in garbage-checking repack: consider bitmaps when performing repacks repack: handle optional files created by pack-objects repack: turn exts array into array-of-struct repack: stop using magic number for ARRAY_SIZE(exts) pack-objects: implement bitmap writing rev-list: add bitmap mode to speed up object lists pack-objects: use bitmaps when packing objects pack-objects: split add_object_entry pack-bitmap: add support for bitmap indexes documentation: add documentation for the bitmap format ewah: compressed bitmap implementation ...
This commit is contained in:
@ -14,10 +14,12 @@
|
||||
#include "diff.h"
|
||||
#include "revision.h"
|
||||
#include "list-objects.h"
|
||||
#include "pack-objects.h"
|
||||
#include "progress.h"
|
||||
#include "refs.h"
|
||||
#include "streaming.h"
|
||||
#include "thread-utils.h"
|
||||
#include "pack-bitmap.h"
|
||||
|
||||
static const char *pack_usage[] = {
|
||||
N_("git pack-objects --stdout [options...] [< ref-list | < object-list]"),
|
||||
@ -25,42 +27,15 @@ static const char *pack_usage[] = {
|
||||
NULL
|
||||
};
|
||||
|
||||
struct object_entry {
|
||||
struct pack_idx_entry idx;
|
||||
unsigned long size; /* uncompressed size */
|
||||
struct packed_git *in_pack; /* already in pack */
|
||||
off_t in_pack_offset;
|
||||
struct object_entry *delta; /* delta base object */
|
||||
struct object_entry *delta_child; /* deltified objects who bases me */
|
||||
struct object_entry *delta_sibling; /* other deltified objects who
|
||||
* uses the same base as me
|
||||
*/
|
||||
void *delta_data; /* cached delta (uncompressed) */
|
||||
unsigned long delta_size; /* delta data size (uncompressed) */
|
||||
unsigned long z_delta_size; /* delta data size (compressed) */
|
||||
enum object_type type;
|
||||
enum object_type in_pack_type; /* could be delta */
|
||||
uint32_t hash; /* name hint hash */
|
||||
unsigned char in_pack_header_size;
|
||||
unsigned preferred_base:1; /*
|
||||
* we do not pack this, but is available
|
||||
* to be used as the base object to delta
|
||||
* objects against.
|
||||
*/
|
||||
unsigned no_try_delta:1;
|
||||
unsigned tagged:1; /* near the very tip of refs */
|
||||
unsigned filled:1; /* assigned write-order */
|
||||
};
|
||||
|
||||
/*
|
||||
* Objects we are going to pack are collected in objects array (dynamically
|
||||
* expanded). nr_objects & nr_alloc controls this array. They are stored
|
||||
* in the order we see -- typically rev-list --objects order that gives us
|
||||
* nice "minimum seek" order.
|
||||
* Objects we are going to pack are collected in the `to_pack` structure.
|
||||
* It contains an array (dynamically expanded) of the object data, and a map
|
||||
* that can resolve SHA1s to their position in the array.
|
||||
*/
|
||||
static struct object_entry *objects;
|
||||
static struct packing_data to_pack;
|
||||
|
||||
static struct pack_idx_entry **written_list;
|
||||
static uint32_t nr_objects, nr_alloc, nr_result, nr_written;
|
||||
static uint32_t nr_result, nr_written;
|
||||
|
||||
static int non_empty;
|
||||
static int reuse_delta = 1, reuse_object = 1;
|
||||
@ -83,27 +58,43 @@ static struct progress *progress_state;
|
||||
static int pack_compression_level = Z_DEFAULT_COMPRESSION;
|
||||
static int pack_compression_seen;
|
||||
|
||||
static struct packed_git *reuse_packfile;
|
||||
static uint32_t reuse_packfile_objects;
|
||||
static off_t reuse_packfile_offset;
|
||||
|
||||
static int use_bitmap_index = 1;
|
||||
static int write_bitmap_index;
|
||||
static uint16_t write_bitmap_options;
|
||||
|
||||
static unsigned long delta_cache_size = 0;
|
||||
static unsigned long max_delta_cache_size = 256 * 1024 * 1024;
|
||||
static unsigned long cache_max_small_delta_size = 1000;
|
||||
|
||||
static unsigned long window_memory_limit = 0;
|
||||
|
||||
/*
|
||||
* The object names in objects array are hashed with this hashtable,
|
||||
* to help looking up the entry by object name.
|
||||
* This hashtable is built after all the objects are seen.
|
||||
*/
|
||||
static int *object_ix;
|
||||
static int object_ix_hashsz;
|
||||
static struct object_entry *locate_object_entry(const unsigned char *sha1);
|
||||
|
||||
/*
|
||||
* stats
|
||||
*/
|
||||
static uint32_t written, written_delta;
|
||||
static uint32_t reused, reused_delta;
|
||||
|
||||
/*
|
||||
* Indexed commits
|
||||
*/
|
||||
static struct commit **indexed_commits;
|
||||
static unsigned int indexed_commits_nr;
|
||||
static unsigned int indexed_commits_alloc;
|
||||
|
||||
static void index_commit_for_bitmap(struct commit *commit)
|
||||
{
|
||||
if (indexed_commits_nr >= indexed_commits_alloc) {
|
||||
indexed_commits_alloc = (indexed_commits_alloc + 32) * 2;
|
||||
indexed_commits = xrealloc(indexed_commits,
|
||||
indexed_commits_alloc * sizeof(struct commit *));
|
||||
}
|
||||
|
||||
indexed_commits[indexed_commits_nr++] = commit;
|
||||
}
|
||||
|
||||
static void *get_delta(struct object_entry *entry)
|
||||
{
|
||||
@ -553,12 +544,12 @@ static int mark_tagged(const char *path, const unsigned char *sha1, int flag,
|
||||
void *cb_data)
|
||||
{
|
||||
unsigned char peeled[20];
|
||||
struct object_entry *entry = locate_object_entry(sha1);
|
||||
struct object_entry *entry = packlist_find(&to_pack, sha1, NULL);
|
||||
|
||||
if (entry)
|
||||
entry->tagged = 1;
|
||||
if (!peel_ref(path, peeled)) {
|
||||
entry = locate_object_entry(peeled);
|
||||
entry = packlist_find(&to_pack, peeled, NULL);
|
||||
if (entry)
|
||||
entry->tagged = 1;
|
||||
}
|
||||
@ -633,9 +624,10 @@ static struct object_entry **compute_write_order(void)
|
||||
{
|
||||
unsigned int i, wo_end, last_untagged;
|
||||
|
||||
struct object_entry **wo = xmalloc(nr_objects * sizeof(*wo));
|
||||
struct object_entry **wo = xmalloc(to_pack.nr_objects * sizeof(*wo));
|
||||
struct object_entry *objects = to_pack.objects;
|
||||
|
||||
for (i = 0; i < nr_objects; i++) {
|
||||
for (i = 0; i < to_pack.nr_objects; i++) {
|
||||
objects[i].tagged = 0;
|
||||
objects[i].filled = 0;
|
||||
objects[i].delta_child = NULL;
|
||||
@ -647,7 +639,7 @@ static struct object_entry **compute_write_order(void)
|
||||
* Make sure delta_sibling is sorted in the original
|
||||
* recency order.
|
||||
*/
|
||||
for (i = nr_objects; i > 0;) {
|
||||
for (i = to_pack.nr_objects; i > 0;) {
|
||||
struct object_entry *e = &objects[--i];
|
||||
if (!e->delta)
|
||||
continue;
|
||||
@ -665,7 +657,7 @@ static struct object_entry **compute_write_order(void)
|
||||
* Give the objects in the original recency order until
|
||||
* we see a tagged tip.
|
||||
*/
|
||||
for (i = wo_end = 0; i < nr_objects; i++) {
|
||||
for (i = wo_end = 0; i < to_pack.nr_objects; i++) {
|
||||
if (objects[i].tagged)
|
||||
break;
|
||||
add_to_write_order(wo, &wo_end, &objects[i]);
|
||||
@ -675,7 +667,7 @@ static struct object_entry **compute_write_order(void)
|
||||
/*
|
||||
* Then fill all the tagged tips.
|
||||
*/
|
||||
for (; i < nr_objects; i++) {
|
||||
for (; i < to_pack.nr_objects; i++) {
|
||||
if (objects[i].tagged)
|
||||
add_to_write_order(wo, &wo_end, &objects[i]);
|
||||
}
|
||||
@ -683,7 +675,7 @@ static struct object_entry **compute_write_order(void)
|
||||
/*
|
||||
* And then all remaining commits and tags.
|
||||
*/
|
||||
for (i = last_untagged; i < nr_objects; i++) {
|
||||
for (i = last_untagged; i < to_pack.nr_objects; i++) {
|
||||
if (objects[i].type != OBJ_COMMIT &&
|
||||
objects[i].type != OBJ_TAG)
|
||||
continue;
|
||||
@ -693,7 +685,7 @@ static struct object_entry **compute_write_order(void)
|
||||
/*
|
||||
* And then all the trees.
|
||||
*/
|
||||
for (i = last_untagged; i < nr_objects; i++) {
|
||||
for (i = last_untagged; i < to_pack.nr_objects; i++) {
|
||||
if (objects[i].type != OBJ_TREE)
|
||||
continue;
|
||||
add_to_write_order(wo, &wo_end, &objects[i]);
|
||||
@ -702,17 +694,57 @@ static struct object_entry **compute_write_order(void)
|
||||
/*
|
||||
* Finally all the rest in really tight order
|
||||
*/
|
||||
for (i = last_untagged; i < nr_objects; i++) {
|
||||
for (i = last_untagged; i < to_pack.nr_objects; i++) {
|
||||
if (!objects[i].filled)
|
||||
add_family_to_write_order(wo, &wo_end, &objects[i]);
|
||||
}
|
||||
|
||||
if (wo_end != nr_objects)
|
||||
die("ordered %u objects, expected %"PRIu32, wo_end, nr_objects);
|
||||
if (wo_end != to_pack.nr_objects)
|
||||
die("ordered %u objects, expected %"PRIu32, wo_end, to_pack.nr_objects);
|
||||
|
||||
return wo;
|
||||
}
|
||||
|
||||
static off_t write_reused_pack(struct sha1file *f)
|
||||
{
|
||||
unsigned char buffer[8192];
|
||||
off_t to_write;
|
||||
int fd;
|
||||
|
||||
if (!is_pack_valid(reuse_packfile))
|
||||
die("packfile is invalid: %s", reuse_packfile->pack_name);
|
||||
|
||||
fd = git_open_noatime(reuse_packfile->pack_name);
|
||||
if (fd < 0)
|
||||
die_errno("unable to open packfile for reuse: %s",
|
||||
reuse_packfile->pack_name);
|
||||
|
||||
if (lseek(fd, sizeof(struct pack_header), SEEK_SET) == -1)
|
||||
die_errno("unable to seek in reused packfile");
|
||||
|
||||
if (reuse_packfile_offset < 0)
|
||||
reuse_packfile_offset = reuse_packfile->pack_size - 20;
|
||||
|
||||
to_write = reuse_packfile_offset - sizeof(struct pack_header);
|
||||
|
||||
while (to_write) {
|
||||
int read_pack = xread(fd, buffer, sizeof(buffer));
|
||||
|
||||
if (read_pack <= 0)
|
||||
die_errno("unable to read from reused packfile");
|
||||
|
||||
if (read_pack > to_write)
|
||||
read_pack = to_write;
|
||||
|
||||
sha1write(f, buffer, read_pack);
|
||||
to_write -= read_pack;
|
||||
}
|
||||
|
||||
close(fd);
|
||||
written += reuse_packfile_objects;
|
||||
return reuse_packfile_offset - sizeof(struct pack_header);
|
||||
}
|
||||
|
||||
static void write_pack_file(void)
|
||||
{
|
||||
uint32_t i = 0, j;
|
||||
@ -724,7 +756,7 @@ static void write_pack_file(void)
|
||||
|
||||
if (progress > pack_to_stdout)
|
||||
progress_state = start_progress("Writing objects", nr_result);
|
||||
written_list = xmalloc(nr_objects * sizeof(*written_list));
|
||||
written_list = xmalloc(to_pack.nr_objects * sizeof(*written_list));
|
||||
write_order = compute_write_order();
|
||||
|
||||
do {
|
||||
@ -737,8 +769,17 @@ static void write_pack_file(void)
|
||||
f = create_tmp_packfile(&pack_tmp_name);
|
||||
|
||||
offset = write_pack_header(f, nr_remaining);
|
||||
|
||||
if (reuse_packfile) {
|
||||
off_t packfile_size;
|
||||
assert(pack_to_stdout);
|
||||
|
||||
packfile_size = write_reused_pack(f);
|
||||
offset += packfile_size;
|
||||
}
|
||||
|
||||
nr_written = 0;
|
||||
for (; i < nr_objects; i++) {
|
||||
for (; i < to_pack.nr_objects; i++) {
|
||||
struct object_entry *e = write_order[i];
|
||||
if (write_one(f, e, &offset) == WRITE_ONE_BREAK)
|
||||
break;
|
||||
@ -789,9 +830,31 @@ static void write_pack_file(void)
|
||||
if (sizeof(tmpname) <= strlen(base_name) + 50)
|
||||
die("pack base name '%s' too long", base_name);
|
||||
snprintf(tmpname, sizeof(tmpname), "%s-", base_name);
|
||||
|
||||
if (write_bitmap_index) {
|
||||
bitmap_writer_set_checksum(sha1);
|
||||
bitmap_writer_build_type_index(written_list, nr_written);
|
||||
}
|
||||
|
||||
finish_tmp_packfile(tmpname, pack_tmp_name,
|
||||
written_list, nr_written,
|
||||
&pack_idx_opts, sha1);
|
||||
|
||||
if (write_bitmap_index) {
|
||||
char *end_of_name_prefix = strrchr(tmpname, 0);
|
||||
sprintf(end_of_name_prefix, "%s.bitmap", sha1_to_hex(sha1));
|
||||
|
||||
stop_progress(&progress_state);
|
||||
|
||||
bitmap_writer_show_progress(progress);
|
||||
bitmap_writer_reuse_bitmaps(&to_pack);
|
||||
bitmap_writer_select_commits(indexed_commits, indexed_commits_nr, -1);
|
||||
bitmap_writer_build(&to_pack);
|
||||
bitmap_writer_finish(written_list, nr_written,
|
||||
tmpname, write_bitmap_options);
|
||||
write_bitmap_index = 0;
|
||||
}
|
||||
|
||||
free(pack_tmp_name);
|
||||
puts(sha1_to_hex(sha1));
|
||||
}
|
||||
@ -801,7 +864,7 @@ static void write_pack_file(void)
|
||||
written_list[j]->offset = (off_t)-1;
|
||||
}
|
||||
nr_remaining -= nr_written;
|
||||
} while (nr_remaining && i < nr_objects);
|
||||
} while (nr_remaining && i < to_pack.nr_objects);
|
||||
|
||||
free(written_list);
|
||||
free(write_order);
|
||||
@ -811,73 +874,6 @@ static void write_pack_file(void)
|
||||
written, nr_result);
|
||||
}
|
||||
|
||||
static int locate_object_entry_hash(const unsigned char *sha1)
|
||||
{
|
||||
int i;
|
||||
unsigned int ui;
|
||||
memcpy(&ui, sha1, sizeof(unsigned int));
|
||||
i = ui % object_ix_hashsz;
|
||||
while (0 < object_ix[i]) {
|
||||
if (!hashcmp(sha1, objects[object_ix[i] - 1].idx.sha1))
|
||||
return i;
|
||||
if (++i == object_ix_hashsz)
|
||||
i = 0;
|
||||
}
|
||||
return -1 - i;
|
||||
}
|
||||
|
||||
static struct object_entry *locate_object_entry(const unsigned char *sha1)
|
||||
{
|
||||
int i;
|
||||
|
||||
if (!object_ix_hashsz)
|
||||
return NULL;
|
||||
|
||||
i = locate_object_entry_hash(sha1);
|
||||
if (0 <= i)
|
||||
return &objects[object_ix[i]-1];
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void rehash_objects(void)
|
||||
{
|
||||
uint32_t i;
|
||||
struct object_entry *oe;
|
||||
|
||||
object_ix_hashsz = nr_objects * 3;
|
||||
if (object_ix_hashsz < 1024)
|
||||
object_ix_hashsz = 1024;
|
||||
object_ix = xrealloc(object_ix, sizeof(int) * object_ix_hashsz);
|
||||
memset(object_ix, 0, sizeof(int) * object_ix_hashsz);
|
||||
for (i = 0, oe = objects; i < nr_objects; i++, oe++) {
|
||||
int ix = locate_object_entry_hash(oe->idx.sha1);
|
||||
if (0 <= ix)
|
||||
continue;
|
||||
ix = -1 - ix;
|
||||
object_ix[ix] = i + 1;
|
||||
}
|
||||
}
|
||||
|
||||
static uint32_t name_hash(const char *name)
|
||||
{
|
||||
uint32_t c, hash = 0;
|
||||
|
||||
if (!name)
|
||||
return 0;
|
||||
|
||||
/*
|
||||
* This effectively just creates a sortable number from the
|
||||
* last sixteen non-whitespace characters. Last characters
|
||||
* count "most", so things that end in ".c" sort together.
|
||||
*/
|
||||
while ((c = *name++) != 0) {
|
||||
if (isspace(c))
|
||||
continue;
|
||||
hash = (hash >> 2) + (c << 24);
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
|
||||
static void setup_delta_attr_check(struct git_attr_check *check)
|
||||
{
|
||||
static struct git_attr *attr_delta;
|
||||
@ -900,42 +896,69 @@ static int no_try_delta(const char *path)
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int add_object_entry(const unsigned char *sha1, enum object_type type,
|
||||
const char *name, int exclude)
|
||||
/*
|
||||
* When adding an object, check whether we have already added it
|
||||
* to our packing list. If so, we can skip. However, if we are
|
||||
* being asked to excludei t, but the previous mention was to include
|
||||
* it, make sure to adjust its flags and tweak our numbers accordingly.
|
||||
*
|
||||
* As an optimization, we pass out the index position where we would have
|
||||
* found the item, since that saves us from having to look it up again a
|
||||
* few lines later when we want to add the new entry.
|
||||
*/
|
||||
static int have_duplicate_entry(const unsigned char *sha1,
|
||||
int exclude,
|
||||
uint32_t *index_pos)
|
||||
{
|
||||
struct object_entry *entry;
|
||||
struct packed_git *p, *found_pack = NULL;
|
||||
off_t found_offset = 0;
|
||||
int ix;
|
||||
uint32_t hash = name_hash(name);
|
||||
|
||||
ix = nr_objects ? locate_object_entry_hash(sha1) : -1;
|
||||
if (ix >= 0) {
|
||||
if (exclude) {
|
||||
entry = objects + object_ix[ix] - 1;
|
||||
if (!entry->preferred_base)
|
||||
nr_result--;
|
||||
entry->preferred_base = 1;
|
||||
}
|
||||
entry = packlist_find(&to_pack, sha1, index_pos);
|
||||
if (!entry)
|
||||
return 0;
|
||||
|
||||
if (exclude) {
|
||||
if (!entry->preferred_base)
|
||||
nr_result--;
|
||||
entry->preferred_base = 1;
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
/*
|
||||
* Check whether we want the object in the pack (e.g., we do not want
|
||||
* objects found in non-local stores if the "--local" option was used).
|
||||
*
|
||||
* As a side effect of this check, we will find the packed version of this
|
||||
* object, if any. We therefore pass out the pack information to avoid having
|
||||
* to look it up again later.
|
||||
*/
|
||||
static int want_object_in_pack(const unsigned char *sha1,
|
||||
int exclude,
|
||||
struct packed_git **found_pack,
|
||||
off_t *found_offset)
|
||||
{
|
||||
struct packed_git *p;
|
||||
|
||||
if (!exclude && local && has_loose_object_nonlocal(sha1))
|
||||
return 0;
|
||||
|
||||
*found_pack = NULL;
|
||||
*found_offset = 0;
|
||||
|
||||
for (p = packed_git; p; p = p->next) {
|
||||
off_t offset = find_pack_entry_one(sha1, p);
|
||||
if (offset) {
|
||||
if (!found_pack) {
|
||||
if (!*found_pack) {
|
||||
if (!is_pack_valid(p)) {
|
||||
warning("packfile %s cannot be accessed", p->pack_name);
|
||||
continue;
|
||||
}
|
||||
found_offset = offset;
|
||||
found_pack = p;
|
||||
*found_offset = offset;
|
||||
*found_pack = p;
|
||||
}
|
||||
if (exclude)
|
||||
break;
|
||||
return 1;
|
||||
if (incremental)
|
||||
return 0;
|
||||
if (local && !p->pack_local)
|
||||
@ -945,14 +968,21 @@ static int add_object_entry(const unsigned char *sha1, enum object_type type,
|
||||
}
|
||||
}
|
||||
|
||||
if (nr_objects >= nr_alloc) {
|
||||
nr_alloc = (nr_alloc + 1024) * 3 / 2;
|
||||
objects = xrealloc(objects, nr_alloc * sizeof(*entry));
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
entry = objects + nr_objects++;
|
||||
memset(entry, 0, sizeof(*entry));
|
||||
hashcpy(entry->idx.sha1, sha1);
|
||||
static void create_object_entry(const unsigned char *sha1,
|
||||
enum object_type type,
|
||||
uint32_t hash,
|
||||
int exclude,
|
||||
int no_try_delta,
|
||||
uint32_t index_pos,
|
||||
struct packed_git *found_pack,
|
||||
off_t found_offset)
|
||||
{
|
||||
struct object_entry *entry;
|
||||
|
||||
entry = packlist_alloc(&to_pack, sha1, index_pos);
|
||||
entry->hash = hash;
|
||||
if (type)
|
||||
entry->type = type;
|
||||
@ -965,16 +995,43 @@ static int add_object_entry(const unsigned char *sha1, enum object_type type,
|
||||
entry->in_pack_offset = found_offset;
|
||||
}
|
||||
|
||||
if (object_ix_hashsz * 3 <= nr_objects * 4)
|
||||
rehash_objects();
|
||||
else
|
||||
object_ix[-1 - ix] = nr_objects;
|
||||
entry->no_try_delta = no_try_delta;
|
||||
}
|
||||
|
||||
display_progress(progress_state, nr_objects);
|
||||
static int add_object_entry(const unsigned char *sha1, enum object_type type,
|
||||
const char *name, int exclude)
|
||||
{
|
||||
struct packed_git *found_pack;
|
||||
off_t found_offset;
|
||||
uint32_t index_pos;
|
||||
|
||||
if (name && no_try_delta(name))
|
||||
entry->no_try_delta = 1;
|
||||
if (have_duplicate_entry(sha1, exclude, &index_pos))
|
||||
return 0;
|
||||
|
||||
if (!want_object_in_pack(sha1, exclude, &found_pack, &found_offset))
|
||||
return 0;
|
||||
|
||||
create_object_entry(sha1, type, pack_name_hash(name),
|
||||
exclude, name && no_try_delta(name),
|
||||
index_pos, found_pack, found_offset);
|
||||
|
||||
display_progress(progress_state, to_pack.nr_objects);
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int add_object_entry_from_bitmap(const unsigned char *sha1,
|
||||
enum object_type type,
|
||||
int flags, uint32_t name_hash,
|
||||
struct packed_git *pack, off_t offset)
|
||||
{
|
||||
uint32_t index_pos;
|
||||
|
||||
if (have_duplicate_entry(sha1, 0, &index_pos))
|
||||
return 0;
|
||||
|
||||
create_object_entry(sha1, type, name_hash, 0, 0, index_pos, pack, offset);
|
||||
|
||||
display_progress(progress_state, to_pack.nr_objects);
|
||||
return 1;
|
||||
}
|
||||
|
||||
@ -1175,7 +1232,7 @@ static void add_preferred_base_object(const char *name)
|
||||
{
|
||||
struct pbase_tree *it;
|
||||
int cmplen;
|
||||
unsigned hash = name_hash(name);
|
||||
unsigned hash = pack_name_hash(name);
|
||||
|
||||
if (!num_preferred_base || check_pbase_path(hash))
|
||||
return;
|
||||
@ -1327,7 +1384,7 @@ static void check_object(struct object_entry *entry)
|
||||
break;
|
||||
}
|
||||
|
||||
if (base_ref && (base_entry = locate_object_entry(base_ref))) {
|
||||
if (base_ref && (base_entry = packlist_find(&to_pack, base_ref, NULL))) {
|
||||
/*
|
||||
* If base_ref was set above that means we wish to
|
||||
* reuse delta data, and we even found that base
|
||||
@ -1401,12 +1458,12 @@ static void get_object_details(void)
|
||||
uint32_t i;
|
||||
struct object_entry **sorted_by_offset;
|
||||
|
||||
sorted_by_offset = xcalloc(nr_objects, sizeof(struct object_entry *));
|
||||
for (i = 0; i < nr_objects; i++)
|
||||
sorted_by_offset[i] = objects + i;
|
||||
qsort(sorted_by_offset, nr_objects, sizeof(*sorted_by_offset), pack_offset_sort);
|
||||
sorted_by_offset = xcalloc(to_pack.nr_objects, sizeof(struct object_entry *));
|
||||
for (i = 0; i < to_pack.nr_objects; i++)
|
||||
sorted_by_offset[i] = to_pack.objects + i;
|
||||
qsort(sorted_by_offset, to_pack.nr_objects, sizeof(*sorted_by_offset), pack_offset_sort);
|
||||
|
||||
for (i = 0; i < nr_objects; i++) {
|
||||
for (i = 0; i < to_pack.nr_objects; i++) {
|
||||
struct object_entry *entry = sorted_by_offset[i];
|
||||
check_object(entry);
|
||||
if (big_file_threshold < entry->size)
|
||||
@ -2032,7 +2089,7 @@ static int add_ref_tag(const char *path, const unsigned char *sha1, int flag, vo
|
||||
|
||||
if (starts_with(path, "refs/tags/") && /* is a tag? */
|
||||
!peel_ref(path, peeled) && /* peelable? */
|
||||
locate_object_entry(peeled)) /* object packed? */
|
||||
packlist_find(&to_pack, peeled, NULL)) /* object packed? */
|
||||
add_object_entry(sha1, OBJ_TAG, NULL, 0);
|
||||
return 0;
|
||||
}
|
||||
@ -2055,14 +2112,14 @@ static void prepare_pack(int window, int depth)
|
||||
if (!pack_to_stdout)
|
||||
do_check_packed_object_crc = 1;
|
||||
|
||||
if (!nr_objects || !window || !depth)
|
||||
if (!to_pack.nr_objects || !window || !depth)
|
||||
return;
|
||||
|
||||
delta_list = xmalloc(nr_objects * sizeof(*delta_list));
|
||||
delta_list = xmalloc(to_pack.nr_objects * sizeof(*delta_list));
|
||||
nr_deltas = n = 0;
|
||||
|
||||
for (i = 0; i < nr_objects; i++) {
|
||||
struct object_entry *entry = objects + i;
|
||||
for (i = 0; i < to_pack.nr_objects; i++) {
|
||||
struct object_entry *entry = to_pack.objects + i;
|
||||
|
||||
if (entry->delta)
|
||||
/* This happens if we decided to reuse existing
|
||||
@ -2140,6 +2197,20 @@ static int git_pack_config(const char *k, const char *v, void *cb)
|
||||
cache_max_small_delta_size = git_config_int(k, v);
|
||||
return 0;
|
||||
}
|
||||
if (!strcmp(k, "pack.writebitmaps")) {
|
||||
write_bitmap_index = git_config_bool(k, v);
|
||||
return 0;
|
||||
}
|
||||
if (!strcmp(k, "pack.writebitmaphashcache")) {
|
||||
if (git_config_bool(k, v))
|
||||
write_bitmap_options |= BITMAP_OPT_HASH_CACHE;
|
||||
else
|
||||
write_bitmap_options &= ~BITMAP_OPT_HASH_CACHE;
|
||||
}
|
||||
if (!strcmp(k, "pack.usebitmaps")) {
|
||||
use_bitmap_index = git_config_bool(k, v);
|
||||
return 0;
|
||||
}
|
||||
if (!strcmp(k, "pack.threads")) {
|
||||
delta_search_threads = git_config_int(k, v);
|
||||
if (delta_search_threads < 0)
|
||||
@ -2198,6 +2269,9 @@ static void show_commit(struct commit *commit, void *data)
|
||||
{
|
||||
add_object_entry(commit->object.sha1, OBJ_COMMIT, NULL, 0);
|
||||
commit->object.flags |= OBJECT_ADDED;
|
||||
|
||||
if (write_bitmap_index)
|
||||
index_commit_for_bitmap(commit);
|
||||
}
|
||||
|
||||
static void show_object(struct object *obj,
|
||||
@ -2340,7 +2414,7 @@ static void loosen_unused_packed_objects(struct rev_info *revs)
|
||||
|
||||
for (i = 0; i < p->num_objects; i++) {
|
||||
sha1 = nth_packed_object_sha1(p, i);
|
||||
if (!locate_object_entry(sha1) &&
|
||||
if (!packlist_find(&to_pack, sha1, NULL) &&
|
||||
!has_sha1_pack_kept_or_nonlocal(sha1))
|
||||
if (force_object_loose(sha1, p->mtime))
|
||||
die("unable to force loose object");
|
||||
@ -2348,6 +2422,29 @@ static void loosen_unused_packed_objects(struct rev_info *revs)
|
||||
}
|
||||
}
|
||||
|
||||
static int get_object_list_from_bitmap(struct rev_info *revs)
|
||||
{
|
||||
if (prepare_bitmap_walk(revs) < 0)
|
||||
return -1;
|
||||
|
||||
if (!reuse_partial_packfile_from_bitmap(
|
||||
&reuse_packfile,
|
||||
&reuse_packfile_objects,
|
||||
&reuse_packfile_offset)) {
|
||||
assert(reuse_packfile_objects);
|
||||
nr_result += reuse_packfile_objects;
|
||||
|
||||
if (progress) {
|
||||
fprintf(stderr, "Reusing existing pack: %d, done.\n",
|
||||
reuse_packfile_objects);
|
||||
fflush(stderr);
|
||||
}
|
||||
}
|
||||
|
||||
traverse_bitmap_commit_list(&add_object_entry_from_bitmap);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void get_object_list(int ac, const char **av)
|
||||
{
|
||||
struct rev_info revs;
|
||||
@ -2367,6 +2464,7 @@ static void get_object_list(int ac, const char **av)
|
||||
if (*line == '-') {
|
||||
if (!strcmp(line, "--not")) {
|
||||
flags ^= UNINTERESTING;
|
||||
write_bitmap_index = 0;
|
||||
continue;
|
||||
}
|
||||
die("not a rev '%s'", line);
|
||||
@ -2375,6 +2473,9 @@ static void get_object_list(int ac, const char **av)
|
||||
die("bad revision '%s'", line);
|
||||
}
|
||||
|
||||
if (use_bitmap_index && !get_object_list_from_bitmap(&revs))
|
||||
return;
|
||||
|
||||
if (prepare_revision_walk(&revs))
|
||||
die("revision walk setup failed");
|
||||
mark_edges_uninteresting(&revs, show_edge);
|
||||
@ -2504,6 +2605,10 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
|
||||
N_("pack compression level")),
|
||||
OPT_SET_INT(0, "keep-true-parents", &grafts_replace_parents,
|
||||
N_("do not hide commits by grafts"), 0),
|
||||
OPT_BOOL(0, "use-bitmap-index", &use_bitmap_index,
|
||||
N_("use a bitmap index if available to speed up counting objects")),
|
||||
OPT_BOOL(0, "write-bitmap-index", &write_bitmap_index,
|
||||
N_("write a bitmap index together with the pack index")),
|
||||
OPT_END(),
|
||||
};
|
||||
|
||||
@ -2570,6 +2675,12 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
|
||||
if (keep_unreachable && unpack_unreachable)
|
||||
die("--keep-unreachable and --unpack-unreachable are incompatible.");
|
||||
|
||||
if (!use_internal_rev_list || !pack_to_stdout || is_repository_shallow())
|
||||
use_bitmap_index = 0;
|
||||
|
||||
if (pack_to_stdout || !rev_list_all)
|
||||
write_bitmap_index = 0;
|
||||
|
||||
if (progress && all_progress_implied)
|
||||
progress = 2;
|
||||
|
||||
|
Reference in New Issue
Block a user