Merge branch 'tb/repack-existing-packs-cleanup'
The code to keep track of existing packs in the repository while repacking has been refactored. * tb/repack-existing-packs-cleanup: builtin/repack.c: extract common cruft pack loop builtin/repack.c: avoid directly inspecting "util" builtin/repack.c: store existing cruft packs separately builtin/repack.c: extract `has_existing_non_kept_packs()` builtin/repack.c: extract redundant pack cleanup for existing packs builtin/repack.c: extract redundant pack cleanup for --geometric builtin/repack.c: extract marking packs for deletion builtin/repack.c: extract structure to store existing packs
This commit is contained in:
Junio C Hamano
commit
5c0f9933ec
293
builtin/repack.c
293
builtin/repack.c
@ -27,7 +27,6 @@
|
||||
#define PACK_CRUFT 4
|
||||
|
||||
#define DELETE_PACK 1
|
||||
#define CRUFT_PACK 2
|
||||
|
||||
static int pack_everything;
|
||||
static int delta_base_offset = 1;
|
||||
@ -95,14 +94,106 @@ static int repack_config(const char *var, const char *value,
|
||||
return git_default_config(var, value, ctx, cb);
|
||||
}
|
||||
|
||||
struct existing_packs {
|
||||
struct string_list kept_packs;
|
||||
struct string_list non_kept_packs;
|
||||
struct string_list cruft_packs;
|
||||
};
|
||||
|
||||
#define EXISTING_PACKS_INIT { \
|
||||
.kept_packs = STRING_LIST_INIT_DUP, \
|
||||
.non_kept_packs = STRING_LIST_INIT_DUP, \
|
||||
.cruft_packs = STRING_LIST_INIT_DUP, \
|
||||
}
|
||||
|
||||
static int has_existing_non_kept_packs(const struct existing_packs *existing)
|
||||
{
|
||||
return existing->non_kept_packs.nr || existing->cruft_packs.nr;
|
||||
}
|
||||
|
||||
static void pack_mark_for_deletion(struct string_list_item *item)
|
||||
{
|
||||
item->util = (void*)((uintptr_t)item->util | DELETE_PACK);
|
||||
}
|
||||
|
||||
static int pack_is_marked_for_deletion(struct string_list_item *item)
|
||||
{
|
||||
return (uintptr_t)item->util & DELETE_PACK;
|
||||
}
|
||||
|
||||
static void mark_packs_for_deletion_1(struct string_list *names,
|
||||
struct string_list *list)
|
||||
{
|
||||
struct string_list_item *item;
|
||||
const int hexsz = the_hash_algo->hexsz;
|
||||
|
||||
for_each_string_list_item(item, list) {
|
||||
char *sha1;
|
||||
size_t len = strlen(item->string);
|
||||
if (len < hexsz)
|
||||
continue;
|
||||
sha1 = item->string + len - hexsz;
|
||||
/*
|
||||
* Mark this pack for deletion, which ensures that this
|
||||
* pack won't be included in a MIDX (if `--write-midx`
|
||||
* was given) and that we will actually delete this pack
|
||||
* (if `-d` was given).
|
||||
*/
|
||||
if (!string_list_has_string(names, sha1))
|
||||
pack_mark_for_deletion(item);
|
||||
}
|
||||
}
|
||||
|
||||
static void mark_packs_for_deletion(struct existing_packs *existing,
|
||||
struct string_list *names)
|
||||
|
||||
{
|
||||
mark_packs_for_deletion_1(names, &existing->non_kept_packs);
|
||||
mark_packs_for_deletion_1(names, &existing->cruft_packs);
|
||||
}
|
||||
|
||||
static void remove_redundant_pack(const char *dir_name, const char *base_name)
|
||||
{
|
||||
struct strbuf buf = STRBUF_INIT;
|
||||
struct multi_pack_index *m = get_local_multi_pack_index(the_repository);
|
||||
strbuf_addf(&buf, "%s.pack", base_name);
|
||||
if (m && midx_contains_pack(m, buf.buf))
|
||||
clear_midx_file(the_repository);
|
||||
strbuf_insertf(&buf, 0, "%s/", dir_name);
|
||||
unlink_pack_path(buf.buf, 1);
|
||||
strbuf_release(&buf);
|
||||
}
|
||||
|
||||
static void remove_redundant_packs_1(struct string_list *packs)
|
||||
{
|
||||
struct string_list_item *item;
|
||||
for_each_string_list_item(item, packs) {
|
||||
if (!pack_is_marked_for_deletion(item))
|
||||
continue;
|
||||
remove_redundant_pack(packdir, item->string);
|
||||
}
|
||||
}
|
||||
|
||||
static void remove_redundant_existing_packs(struct existing_packs *existing)
|
||||
{
|
||||
remove_redundant_packs_1(&existing->non_kept_packs);
|
||||
remove_redundant_packs_1(&existing->cruft_packs);
|
||||
}
|
||||
|
||||
static void existing_packs_release(struct existing_packs *existing)
|
||||
{
|
||||
string_list_clear(&existing->kept_packs, 0);
|
||||
string_list_clear(&existing->non_kept_packs, 0);
|
||||
string_list_clear(&existing->cruft_packs, 0);
|
||||
}
|
||||
|
||||
/*
|
||||
* Adds all packs hex strings (pack-$HASH) to either fname_nonkept_list
|
||||
* or fname_kept_list based on whether each pack has a corresponding
|
||||
* Adds all packs hex strings (pack-$HASH) to either packs->non_kept
|
||||
* or packs->kept based on whether each pack has a corresponding
|
||||
* .keep file or not. Packs without a .keep file are not to be kept
|
||||
* if we are going to pack everything into one file.
|
||||
*/
|
||||
static void collect_pack_filenames(struct string_list *fname_nonkept_list,
|
||||
struct string_list *fname_kept_list,
|
||||
static void collect_pack_filenames(struct existing_packs *existing,
|
||||
const struct string_list *extra_keep)
|
||||
{
|
||||
struct packed_git *p;
|
||||
@ -126,28 +217,14 @@ static void collect_pack_filenames(struct string_list *fname_nonkept_list,
|
||||
strbuf_strip_suffix(&buf, ".pack");
|
||||
|
||||
if ((extra_keep->nr > 0 && i < extra_keep->nr) || p->pack_keep)
|
||||
string_list_append(fname_kept_list, buf.buf);
|
||||
else {
|
||||
struct string_list_item *item;
|
||||
item = string_list_append(fname_nonkept_list, buf.buf);
|
||||
if (p->is_cruft)
|
||||
item->util = (void*)(uintptr_t)CRUFT_PACK;
|
||||
}
|
||||
string_list_append(&existing->kept_packs, buf.buf);
|
||||
else if (p->is_cruft)
|
||||
string_list_append(&existing->cruft_packs, buf.buf);
|
||||
else
|
||||
string_list_append(&existing->non_kept_packs, buf.buf);
|
||||
}
|
||||
|
||||
string_list_sort(fname_kept_list);
|
||||
strbuf_release(&buf);
|
||||
}
|
||||
|
||||
static void remove_redundant_pack(const char *dir_name, const char *base_name)
|
||||
{
|
||||
struct strbuf buf = STRBUF_INIT;
|
||||
struct multi_pack_index *m = get_local_multi_pack_index(the_repository);
|
||||
strbuf_addf(&buf, "%s.pack", base_name);
|
||||
if (m && midx_contains_pack(m, buf.buf))
|
||||
clear_midx_file(the_repository);
|
||||
strbuf_insertf(&buf, 0, "%s/", dir_name);
|
||||
unlink_pack_path(buf.buf, 1);
|
||||
string_list_sort(&existing->kept_packs);
|
||||
strbuf_release(&buf);
|
||||
}
|
||||
|
||||
@ -327,7 +404,7 @@ static int geometry_cmp(const void *va, const void *vb)
|
||||
}
|
||||
|
||||
static void init_pack_geometry(struct pack_geometry *geometry,
|
||||
struct string_list *existing_kept_packs,
|
||||
struct existing_packs *existing,
|
||||
const struct pack_objects_args *args)
|
||||
{
|
||||
struct packed_git *p;
|
||||
@ -344,23 +421,24 @@ static void init_pack_geometry(struct pack_geometry *geometry,
|
||||
|
||||
if (!pack_kept_objects) {
|
||||
/*
|
||||
* Any pack that has its pack_keep bit set will appear
|
||||
* in existing_kept_packs below, but this saves us from
|
||||
* doing a more expensive check.
|
||||
* Any pack that has its pack_keep bit set will
|
||||
* appear in existing->kept_packs below, but
|
||||
* this saves us from doing a more expensive
|
||||
* check.
|
||||
*/
|
||||
if (p->pack_keep)
|
||||
continue;
|
||||
|
||||
/*
|
||||
* The pack may be kept via the --keep-pack option;
|
||||
* check 'existing_kept_packs' to determine whether to
|
||||
* ignore it.
|
||||
* The pack may be kept via the --keep-pack
|
||||
* option; check 'existing->kept_packs' to
|
||||
* determine whether to ignore it.
|
||||
*/
|
||||
strbuf_reset(&buf);
|
||||
strbuf_addstr(&buf, pack_basename(p));
|
||||
strbuf_strip_suffix(&buf, ".pack");
|
||||
|
||||
if (string_list_has_string(existing_kept_packs, buf.buf))
|
||||
if (string_list_has_string(&existing->kept_packs, buf.buf))
|
||||
continue;
|
||||
}
|
||||
if (p->is_cruft)
|
||||
@ -494,6 +572,32 @@ static struct packed_git *get_preferred_pack(struct pack_geometry *geometry)
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void geometry_remove_redundant_packs(struct pack_geometry *geometry,
|
||||
struct string_list *names,
|
||||
struct existing_packs *existing)
|
||||
{
|
||||
struct strbuf buf = STRBUF_INIT;
|
||||
uint32_t i;
|
||||
|
||||
for (i = 0; i < geometry->split; i++) {
|
||||
struct packed_git *p = geometry->pack[i];
|
||||
if (string_list_has_string(names, hash_to_hex(p->hash)))
|
||||
continue;
|
||||
|
||||
strbuf_reset(&buf);
|
||||
strbuf_addstr(&buf, pack_basename(p));
|
||||
strbuf_strip_suffix(&buf, ".pack");
|
||||
|
||||
if ((p->pack_keep) ||
|
||||
(string_list_has_string(&existing->kept_packs, buf.buf)))
|
||||
continue;
|
||||
|
||||
remove_redundant_pack(packdir, buf.buf);
|
||||
}
|
||||
|
||||
strbuf_release(&buf);
|
||||
}
|
||||
|
||||
static void free_pack_geometry(struct pack_geometry *geometry)
|
||||
{
|
||||
if (!geometry)
|
||||
@ -565,14 +669,13 @@ static void midx_snapshot_refs(struct tempfile *f)
|
||||
}
|
||||
|
||||
static void midx_included_packs(struct string_list *include,
|
||||
struct string_list *existing_nonkept_packs,
|
||||
struct string_list *existing_kept_packs,
|
||||
struct existing_packs *existing,
|
||||
struct string_list *names,
|
||||
struct pack_geometry *geometry)
|
||||
{
|
||||
struct string_list_item *item;
|
||||
|
||||
for_each_string_list_item(item, existing_kept_packs)
|
||||
for_each_string_list_item(item, &existing->kept_packs)
|
||||
string_list_insert(include, xstrfmt("%s.idx", item->string));
|
||||
for_each_string_list_item(item, names)
|
||||
string_list_insert(include, xstrfmt("pack-%s.idx", item->string));
|
||||
@ -599,24 +702,32 @@ static void midx_included_packs(struct string_list *include,
|
||||
|
||||
string_list_insert(include, strbuf_detach(&buf, NULL));
|
||||
}
|
||||
|
||||
for_each_string_list_item(item, existing_nonkept_packs) {
|
||||
if (!((uintptr_t)item->util & CRUFT_PACK)) {
|
||||
/*
|
||||
* no need to check DELETE_PACK, since we're not
|
||||
* doing an ALL_INTO_ONE repack
|
||||
*/
|
||||
continue;
|
||||
}
|
||||
string_list_insert(include, xstrfmt("%s.idx", item->string));
|
||||
}
|
||||
} else {
|
||||
for_each_string_list_item(item, existing_nonkept_packs) {
|
||||
if ((uintptr_t)item->util & DELETE_PACK)
|
||||
for_each_string_list_item(item, &existing->non_kept_packs) {
|
||||
if (pack_is_marked_for_deletion(item))
|
||||
continue;
|
||||
string_list_insert(include, xstrfmt("%s.idx", item->string));
|
||||
}
|
||||
}
|
||||
|
||||
for_each_string_list_item(item, &existing->cruft_packs) {
|
||||
/*
|
||||
* When doing a --geometric repack, there is no need to check
|
||||
* for deleted packs, since we're by definition not doing an
|
||||
* ALL_INTO_ONE repack (hence no packs will be deleted).
|
||||
* Otherwise we must check for and exclude any packs which are
|
||||
* enqueued for deletion.
|
||||
*
|
||||
* So we could omit the conditional below in the --geometric
|
||||
* case, but doing so is unnecessary since no packs are marked
|
||||
* as pending deletion (since we only call
|
||||
* `mark_packs_for_deletion()` when doing an all-into-one
|
||||
* repack).
|
||||
*/
|
||||
if (pack_is_marked_for_deletion(item))
|
||||
continue;
|
||||
string_list_insert(include, xstrfmt("%s.idx", item->string));
|
||||
}
|
||||
}
|
||||
|
||||
static int write_midx_included_packs(struct string_list *include,
|
||||
@ -700,8 +811,7 @@ static int write_cruft_pack(const struct pack_objects_args *args,
|
||||
const char *pack_prefix,
|
||||
const char *cruft_expiration,
|
||||
struct string_list *names,
|
||||
struct string_list *existing_packs,
|
||||
struct string_list *existing_kept_packs)
|
||||
struct existing_packs *existing)
|
||||
{
|
||||
struct child_process cmd = CHILD_PROCESS_INIT;
|
||||
struct strbuf line = STRBUF_INIT;
|
||||
@ -743,9 +853,11 @@ static int write_cruft_pack(const struct pack_objects_args *args,
|
||||
in = xfdopen(cmd.in, "w");
|
||||
for_each_string_list_item(item, names)
|
||||
fprintf(in, "%s-%s.pack\n", pack_prefix, item->string);
|
||||
for_each_string_list_item(item, existing_packs)
|
||||
for_each_string_list_item(item, &existing->non_kept_packs)
|
||||
fprintf(in, "-%s.pack\n", item->string);
|
||||
for_each_string_list_item(item, existing_kept_packs)
|
||||
for_each_string_list_item(item, &existing->cruft_packs)
|
||||
fprintf(in, "-%s.pack\n", item->string);
|
||||
for_each_string_list_item(item, &existing->kept_packs)
|
||||
fprintf(in, "%s.pack\n", item->string);
|
||||
fclose(in);
|
||||
|
||||
@ -777,8 +889,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
|
||||
struct child_process cmd = CHILD_PROCESS_INIT;
|
||||
struct string_list_item *item;
|
||||
struct string_list names = STRING_LIST_INIT_DUP;
|
||||
struct string_list existing_nonkept_packs = STRING_LIST_INIT_DUP;
|
||||
struct string_list existing_kept_packs = STRING_LIST_INIT_DUP;
|
||||
struct existing_packs existing = EXISTING_PACKS_INIT;
|
||||
struct pack_geometry geometry = { 0 };
|
||||
struct strbuf line = STRBUF_INIT;
|
||||
struct tempfile *refs_snapshot = NULL;
|
||||
@ -914,13 +1025,12 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
|
||||
packtmp_name = xstrfmt(".tmp-%d-pack", (int)getpid());
|
||||
packtmp = mkpathdup("%s/%s", packdir, packtmp_name);
|
||||
|
||||
collect_pack_filenames(&existing_nonkept_packs, &existing_kept_packs,
|
||||
&keep_pack_list);
|
||||
collect_pack_filenames(&existing, &keep_pack_list);
|
||||
|
||||
if (geometry.split_factor) {
|
||||
if (pack_everything)
|
||||
die(_("options '%s' and '%s' cannot be used together"), "--geometric", "-A/-a");
|
||||
init_pack_geometry(&geometry, &existing_kept_packs, &po_args);
|
||||
init_pack_geometry(&geometry, &existing, &po_args);
|
||||
split_pack_geometry(&geometry);
|
||||
}
|
||||
|
||||
@ -964,7 +1074,8 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
|
||||
if (pack_everything & ALL_INTO_ONE) {
|
||||
repack_promisor_objects(&po_args, &names);
|
||||
|
||||
if (existing_nonkept_packs.nr && delete_redundant &&
|
||||
if (has_existing_non_kept_packs(&existing) &&
|
||||
delete_redundant &&
|
||||
!(pack_everything & PACK_CRUFT)) {
|
||||
for_each_string_list_item(item, &names) {
|
||||
strvec_pushf(&cmd.args, "--keep-pack=%s-%s.pack",
|
||||
@ -1055,8 +1166,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
|
||||
|
||||
ret = write_cruft_pack(&cruft_po_args, packtmp, pack_prefix,
|
||||
cruft_expiration, &names,
|
||||
&existing_nonkept_packs,
|
||||
&existing_kept_packs);
|
||||
&existing);
|
||||
if (ret)
|
||||
goto cleanup;
|
||||
|
||||
@ -1087,8 +1197,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
|
||||
pack_prefix,
|
||||
NULL,
|
||||
&names,
|
||||
&existing_nonkept_packs,
|
||||
&existing_kept_packs);
|
||||
&existing);
|
||||
if (ret)
|
||||
goto cleanup;
|
||||
}
|
||||
@ -1132,29 +1241,12 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
|
||||
}
|
||||
/* End of pack replacement. */
|
||||
|
||||
if (delete_redundant && pack_everything & ALL_INTO_ONE) {
|
||||
const int hexsz = the_hash_algo->hexsz;
|
||||
for_each_string_list_item(item, &existing_nonkept_packs) {
|
||||
char *sha1;
|
||||
size_t len = strlen(item->string);
|
||||
if (len < hexsz)
|
||||
continue;
|
||||
sha1 = item->string + len - hexsz;
|
||||
/*
|
||||
* Mark this pack for deletion, which ensures that this
|
||||
* pack won't be included in a MIDX (if `--write-midx`
|
||||
* was given) and that we will actually delete this pack
|
||||
* (if `-d` was given).
|
||||
*/
|
||||
if (!string_list_has_string(&names, sha1))
|
||||
item->util = (void*)(uintptr_t)((size_t)item->util | DELETE_PACK);
|
||||
}
|
||||
}
|
||||
if (delete_redundant && pack_everything & ALL_INTO_ONE)
|
||||
mark_packs_for_deletion(&existing, &names);
|
||||
|
||||
if (write_midx) {
|
||||
struct string_list include = STRING_LIST_INIT_NODUP;
|
||||
midx_included_packs(&include, &existing_nonkept_packs,
|
||||
&existing_kept_packs, &names, &geometry);
|
||||
midx_included_packs(&include, &existing, &names, &geometry);
|
||||
|
||||
ret = write_midx_included_packs(&include, &geometry,
|
||||
refs_snapshot ? get_tempfile_path(refs_snapshot) : NULL,
|
||||
@ -1173,35 +1265,11 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
|
||||
|
||||
if (delete_redundant) {
|
||||
int opts = 0;
|
||||
for_each_string_list_item(item, &existing_nonkept_packs) {
|
||||
if (!((uintptr_t)item->util & DELETE_PACK))
|
||||
continue;
|
||||
remove_redundant_pack(packdir, item->string);
|
||||
}
|
||||
remove_redundant_existing_packs(&existing);
|
||||
|
||||
if (geometry.split_factor) {
|
||||
struct strbuf buf = STRBUF_INIT;
|
||||
|
||||
uint32_t i;
|
||||
for (i = 0; i < geometry.split; i++) {
|
||||
struct packed_git *p = geometry.pack[i];
|
||||
if (string_list_has_string(&names,
|
||||
hash_to_hex(p->hash)))
|
||||
continue;
|
||||
|
||||
strbuf_reset(&buf);
|
||||
strbuf_addstr(&buf, pack_basename(p));
|
||||
strbuf_strip_suffix(&buf, ".pack");
|
||||
|
||||
if ((p->pack_keep) ||
|
||||
(string_list_has_string(&existing_kept_packs,
|
||||
buf.buf)))
|
||||
continue;
|
||||
|
||||
remove_redundant_pack(packdir, buf.buf);
|
||||
}
|
||||
strbuf_release(&buf);
|
||||
}
|
||||
if (geometry.split_factor)
|
||||
geometry_remove_redundant_packs(&geometry, &names,
|
||||
&existing);
|
||||
if (show_progress)
|
||||
opts |= PRUNE_PACKED_VERBOSE;
|
||||
prune_packed_objects(opts);
|
||||
@ -1225,8 +1293,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
|
||||
|
||||
cleanup:
|
||||
string_list_clear(&names, 1);
|
||||
string_list_clear(&existing_nonkept_packs, 0);
|
||||
string_list_clear(&existing_kept_packs, 0);
|
||||
existing_packs_release(&existing);
|
||||
free_pack_geometry(&geometry);
|
||||
|
||||
return ret;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user