mirrors/git
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

git-repack --max-pack-size: split packs as asked by write_{object,one}()

Browse Source 
Rewrite write_pack_file() to break to a new packfile
whenever write_object/write_one request it,  and
correct the header's object count in the previous packfile.
Change write_index_file() to write an index
for just the objects in the most recent packfile.

Signed-off-by: Dana L. How <danahow@gmail.com>
Signed-off-by: Junio C Hamano <junkio@cox.net>
This commit is contained in:
Dana L. How
2007-05-13 12:09:16 -07:00
committed by Junio C Hamano
parent 17b08f2cd0
commit ebe27b137c
1 changed files with 71 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
builtin-pack-objects.c
View File

@ -568,6 +568,7 @@ static off_t write_one(struct sha1file *f,
e->offset = 0; e->offset = 0;
return 0; return 0;
} }
written_list[nr_written++] = e;
/* make sure off_t is sufficiently large not to wrap */ /* make sure off_t is sufficiently large not to wrap */
if (offset > offset + size) if (offset > offset + size)
@ -587,15 +588,20 @@ static int adjust_perm(const char *path, mode_t mode);
static void write_pack_file(void) static void write_pack_file(void)
{ {
uint32_t i; uint32_t i = 0, j;
struct sha1file *f; struct sha1file *f;
off_t offset, last_obj_offset = 0; off_t offset, offset_one, last_obj_offset = 0;
struct pack_header hdr; struct pack_header hdr;
int do_progress = progress; int do_progress = progress >> pack_to_stdout;
uint32_t nr_remaining = nr_result;
if (do_progress)
start_progress(&progress_state, "Writing %u objects...", "", nr_result);
written_list = xmalloc(nr_objects * sizeof(struct object_entry *));
do {
if (pack_to_stdout) { if (pack_to_stdout) {
f = sha1fd(1, "<stdout>"); f = sha1fd(1, "<stdout>");
do_progress >>= 1;
} else { } else {
int fd = open_object_dir_tmp("tmp_pack_XXXXXX"); int fd = open_object_dir_tmp("tmp_pack_XXXXXX");
if (fd < 0) if (fd < 0)
@ -604,28 +610,33 @@ static void write_pack_file(void)
f = sha1fd(fd, pack_tmp_name); f = sha1fd(fd, pack_tmp_name);
} }
if (do_progress)
start_progress(&progress_state, "Writing %u objects...", "", nr_result);
hdr.hdr_signature = htonl(PACK_SIGNATURE); hdr.hdr_signature = htonl(PACK_SIGNATURE);
hdr.hdr_version = htonl(PACK_VERSION); hdr.hdr_version = htonl(PACK_VERSION);
hdr.hdr_entries = htonl(nr_result); hdr.hdr_entries = htonl(nr_remaining);
sha1write(f, &hdr, sizeof(hdr)); sha1write(f, &hdr, sizeof(hdr));
offset = sizeof(hdr); offset = sizeof(hdr);
if (!nr_result) nr_written = 0;
goto done; for (; i < nr_objects; i++) {
for (i = 0; i < nr_objects; i++) {
last_obj_offset = offset; last_obj_offset = offset;
offset = write_one(f, objects + i, offset); offset_one = write_one(f, objects + i, offset);
if (!offset_one)
break;
offset = offset_one;
if (do_progress) if (do_progress)
display_progress(&progress_state, written); display_progress(&progress_state, written);
} }
if (do_progress)
stop_progress(&progress_state); /*
done: * Did we write the wrong # entries in the header?
if (written != nr_result) * If so, rewrite it like in fast-import
die("wrote %u objects while expecting %u", written, nr_result); */
if (pack_to_stdout || nr_written == nr_remaining) {
sha1close(f, pack_file_sha1, 1); sha1close(f, pack_file_sha1, 1);
} else {
sha1close(f, pack_file_sha1, 0);
fixup_pack_header_footer(f->fd, pack_file_sha1, pack_tmp_name, nr_written);
close(f->fd);
}
if (!pack_to_stdout) { if (!pack_to_stdout) {
unsigned char object_list_sha1[20]; unsigned char object_list_sha1[20];
@ -653,6 +664,25 @@ static void write_pack_file(void)
strerror(errno)); strerror(errno));
puts(sha1_to_hex(object_list_sha1)); puts(sha1_to_hex(object_list_sha1));
} }
/* mark written objects as written to previous pack */
for (j = 0; j < nr_written; j++) {
written_list[j]->offset = (off_t)-1;
}
nr_remaining -= nr_written;
} while (nr_remaining && i < nr_objects);
free(written_list);
if (do_progress)
stop_progress(&progress_state);
if (written != nr_result)
die("wrote %u objects while expecting %u", written, nr_result);
for (j = 0; i < nr_objects; i++) {
struct object_entry *e = objects + i;
j += !e->offset && !e->preferred_base;
}
if (j)
die("wrote %u objects as expected but %u unwritten", written, j);
} }
static int sha1_sort(const void *_a, const void *_b) static int sha1_sort(const void *_a, const void *_b)
@ -679,18 +709,11 @@ static void write_index_file(off_t last_obj_offset, unsigned char *sha1)
idx_tmp_name = xstrdup(tmpname); idx_tmp_name = xstrdup(tmpname);
f = sha1fd(fd, idx_tmp_name); f = sha1fd(fd, idx_tmp_name);
if (nr_result) { if (nr_written) {
uint32_t j = 0; sorted_by_sha = written_list;
sorted_by_sha = qsort(sorted_by_sha, nr_written, sizeof(*sorted_by_sha), sha1_sort);
xcalloc(nr_result, sizeof(struct object_entry *));
for (i = 0; i < nr_objects; i++)
if (!objects[i].preferred_base)
sorted_by_sha[j++] = objects + i;
if (j != nr_result)
die("listed %u objects while expecting %u", j, nr_result);
qsort(sorted_by_sha, nr_result, sizeof(*sorted_by_sha), sha1_sort);
list = sorted_by_sha; list = sorted_by_sha;
last = sorted_by_sha + nr_result; last = sorted_by_sha + nr_written;
} else } else
sorted_by_sha = list = last = NULL; sorted_by_sha = list = last = NULL;
@ -728,7 +751,7 @@ static void write_index_file(off_t last_obj_offset, unsigned char *sha1)
/* Write the actual SHA1 entries. */ /* Write the actual SHA1 entries. */
list = sorted_by_sha; list = sorted_by_sha;
for (i = 0; i < nr_result; i++) { for (i = 0; i < nr_written; i++) {
struct object_entry *entry = *list++; struct object_entry *entry = *list++;
if (index_version < 2) { if (index_version < 2) {
uint32_t offset = htonl(entry->offset); uint32_t offset = htonl(entry->offset);
@ -743,7 +766,7 @@ static void write_index_file(off_t last_obj_offset, unsigned char *sha1)
/* write the crc32 table */ /* write the crc32 table */
list = sorted_by_sha; list = sorted_by_sha;
for (i = 0; i < nr_objects; i++) { for (i = 0; i < nr_written; i++) {
struct object_entry *entry = *list++; struct object_entry *entry = *list++;
uint32_t crc32_val = htonl(entry->crc32); uint32_t crc32_val = htonl(entry->crc32);
sha1write(f, &crc32_val, 4); sha1write(f, &crc32_val, 4);
@ -751,7 +774,7 @@ static void write_index_file(off_t last_obj_offset, unsigned char *sha1)
/* write the 32-bit offset table */ /* write the 32-bit offset table */
list = sorted_by_sha; list = sorted_by_sha;
for (i = 0; i < nr_objects; i++) { for (i = 0; i < nr_written; i++) {
struct object_entry *entry = *list++; struct object_entry *entry = *list++;
uint32_t offset = (entry->offset <= index_off32_limit) ? uint32_t offset = (entry->offset <= index_off32_limit) ?
entry->offset : (0x80000000 | nr_large_offset++); entry->offset : (0x80000000 | nr_large_offset++);
@ -776,7 +799,6 @@ static void write_index_file(off_t last_obj_offset, unsigned char *sha1)
sha1write(f, pack_file_sha1, 20); sha1write(f, pack_file_sha1, 20);
sha1close(f, NULL, 1); sha1close(f, NULL, 1);
free(sorted_by_sha);
SHA1_Final(sha1, &ctx); SHA1_Final(sha1, &ctx);
} }