mirrors/git
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

pack-objects: add delta-islands support

Browse Source 
Implement support for delta islands in git pack-objects
and document how delta islands work in
"Documentation/git-pack-objects.txt" and Documentation/config.txt.

This allows users to setup delta islands in their config and
get the benefit of less disk usage while cloning and fetching
is still quite fast and not much more CPU intensive.

Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
This commit is contained in:
Jeff King
2018-08-16 08:13:09 +02:00
committed by Junio C Hamano
parent f64ba53ee7
commit 28b8a73080
3 changed files with 161 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
builtin/pack-objects.c
View File

@ -24,6 +24,7 @@
#include "streaming.h"
#include "thread-utils.h"
#include "pack-bitmap.h"
#include "delta-islands.h"
#include "reachable.h"
#include "sha1-array.h"
#include "argv-array.h"
@ -59,6 +60,7 @@ static struct packing_data to_pack;
static struct pack_idx_entry **written_list;
static uint32_t nr_result, nr_written, nr_seen;
static uint32_t write_layer;
static int non_empty;
static int reuse_delta = 1, reuse_object = 1;
@ -93,6 +95,8 @@ static uint16_t write_bitmap_options;
static int exclude_promisor_objects;
static int use_delta_islands;
static unsigned long delta_cache_size = 0;
static unsigned long max_delta_cache_size = DEFAULT_DELTA_CACHE_SIZE;
static unsigned long cache_max_small_delta_size = 1000;
@ -607,7 +611,7 @@ static inline void add_to_write_order(struct object_entry **wo,
unsigned int *endp,
struct object_entry *e)
{
if (e->filled)
if (e->filled || e->layer != write_layer)
return;
wo[(*endp)++] = e;
e->filled = 1;
@ -710,13 +714,14 @@ static void compute_layer_order(struct object_entry **wo, unsigned int *wo_end)
* Finally all the rest in really tight order
*/
for (i = last_untagged; i < to_pack.nr_objects; i++) {
if (!objects[i].filled)
if (!objects[i].filled && objects[i].layer == write_layer)
add_family_to_write_order(wo, wo_end, &objects[i]);
}
}
static struct object_entry **compute_write_order(void)
{
uint32_t max_layers = 1;
unsigned int i, wo_end;
struct object_entry **wo;
@ -748,14 +753,14 @@ static struct object_entry **compute_write_order(void)
*/
for_each_tag_ref(mark_tagged, NULL);
/*
* Give the objects in the original recency order until
* we see a tagged tip.
*/
if (use_delta_islands)
max_layers = compute_pack_layers(&to_pack);
ALLOC_ARRAY(wo, to_pack.nr_objects);
wo_end = 0;
compute_layer_order(wo, &wo_end);
for (; write_layer < max_layers; ++write_layer)
compute_layer_order(wo, &wo_end);
if (wo_end != to_pack.nr_objects)
die("ordered %u objects, expected %"PRIu32, wo_end, to_pack.nr_objects);
@ -1514,7 +1519,8 @@ static void check_object(struct object_entry *entry)
break;
}
if (base_ref && (base_entry = packlist_find(&to_pack, base_ref, NULL))) {
if (base_ref && (base_entry = packlist_find(&to_pack, base_ref, NULL)) &&
in_same_island(&entry->idx.oid, &base_entry->idx.oid)) {
/*
* If base_ref was set above that means we wish to
* reuse delta data, and we even found that base
@ -1830,6 +1836,11 @@ static int type_size_sort(const void *_a, const void *_b)
return -1;
if (a->preferred_base < b->preferred_base)
return 1;
if (use_delta_islands) {
int island_cmp = island_delta_cmp(&a->idx.oid, &b->idx.oid);
if (island_cmp)
return island_cmp;
}
if (a_size > b_size)
return -1;
if (a_size < b_size)
@ -1978,6 +1989,9 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
if (trg_size < src_size / 32)
return 0;
if (!in_same_island(&trg->entry->idx.oid, &src->entry->idx.oid))
return 0;
/* Load data if not already done */
if (!trg->data) {
read_lock();
@ -2516,6 +2530,9 @@ static void prepare_pack(int window, int depth)
uint32_t i, nr_deltas;
unsigned n;
if (use_delta_islands)
resolve_tree_islands(progress, &to_pack);
get_object_details();
/*
@ -2679,6 +2696,9 @@ static void show_commit(struct commit *commit, void *data)
if (write_bitmap_index)
index_commit_for_bitmap(commit);
if (use_delta_islands)
propagate_island_marks(commit);
}
static void show_object(struct object *obj, const char *name, void *data)
@ -2686,6 +2706,19 @@ static void show_object(struct object *obj, const char *name, void *data)
add_preferred_base_object(name);
add_object_entry(&obj->oid, obj->type, name, 0);
obj->flags |= OBJECT_ADDED;
if (use_delta_islands) {
const char *p;
unsigned depth = 0;
struct object_entry *ent;
for (p = strchr(name, '/'); p; p = strchr(p + 1, '/'))
depth++;
ent = packlist_find(&to_pack, obj->oid.hash, NULL);
if (ent && depth > ent->tree_depth)
ent->tree_depth = depth;
}
}
static void show_object__ma_allow_any(struct object *obj, const char *name, void *data)
@ -3013,6 +3046,9 @@ static void get_object_list(int ac, const char **av)
if (use_bitmap_index && !get_object_list_from_bitmap(&revs))
return;
if (use_delta_islands)
load_delta_islands();
if (prepare_revision_walk(&revs))
die("revision walk setup failed");
mark_edges_uninteresting(&revs, show_edge);
@ -3192,6 +3228,8 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
option_parse_missing_action },
OPT_BOOL(0, "exclude-promisor-objects", &exclude_promisor_objects,
N_("do not pack objects in promisor packfiles")),
OPT_BOOL(0, "delta-islands", &use_delta_islands,
N_("respect islands during delta compression")),
OPT_END(),
};
@ -3318,6 +3356,9 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
if (pack_to_stdout || !rev_list_all)
write_bitmap_index = 0;
if (use_delta_islands)
argv_array_push(&rp, "--topo-order");
if (progress && all_progress_implied)
progress = 2;