mirrors/git
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'jt/cdn-offload'

Browse Source 
The "fetch/clone" protocol has been updated to allow the server to
instruct the clients to grab pre-packaged packfile(s) in addition
to the packed object data coming over the wire.

* jt/cdn-offload:
  upload-pack: fix a sparse '0 as NULL pointer' warning
  upload-pack: send part of packfile response as uri
  fetch-pack: support more than one pack lockfile
  upload-pack: refactor reading of pack-objects out
  Documentation: add Packfile URIs design doc
  Documentation: order protocol v2 sections
  http-fetch: support fetching packfiles by URL
  http-fetch: refactor into function
  http: refactor finish_http_pack_request()
  http: use --stdin when indexing dumb HTTP pack
This commit is contained in:
Junio C Hamano
2020-06-25 12:27:47 -07:00
parent 10462829e3 cae2ee1055
commit 34e849b05a
19 changed files with 765 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
fetch-pack.c
View File

@ -38,6 +38,7 @@ static int server_supports_filtering;
static struct shallow_lock shallow_lock;
static const char *alternate_shallow_file;
static struct strbuf fsck_msg_types = STRBUF_INIT;
static struct string_list uri_protocols = STRING_LIST_INIT_DUP;
/* Remember to update object flag allocation in object.h */
#define COMPLETE (1U << 0)
@ -794,7 +795,8 @@ static void write_promisor_file(const char *keep_name,
}
static int get_pack(struct fetch_pack_args *args,
int xd[2], char **pack_lockfile,
int xd[2], struct string_list *pack_lockfiles,
int only_packfile,
struct ref **sought, int nr_sought)
{
struct async demux;
@ -838,7 +840,7 @@ static int get_pack(struct fetch_pack_args *args,
}
if (do_keep || args->from_promisor) {
if (pack_lockfile)
if (pack_lockfiles)
cmd.out = -1;
cmd_name = "index-pack";
argv_array_push(&cmd.args, cmd_name);
@ -855,15 +857,22 @@ static int get_pack(struct fetch_pack_args *args,
"--keep=fetch-pack %"PRIuMAX " on %s",
(uintmax_t)getpid(), hostname);
}
if (args->check_self_contained_and_connected)
if (only_packfile && args->check_self_contained_and_connected)
argv_array_push(&cmd.args, "--check-self-contained-and-connected");
else
/*
* We cannot perform any connectivity checks because
* not all packs have been downloaded; let the caller
* have this responsibility.
*/
args->check_self_contained_and_connected = 0;
/*
* If we're obtaining the filename of a lockfile, we'll use
* that filename to write a .promisor file with more
* information below. If not, we need index-pack to do it for
* us.
*/
if (!(do_keep && pack_lockfile) && args->from_promisor)
if (!(do_keep && pack_lockfiles) && args->from_promisor)
argv_array_push(&cmd.args, "--promisor");
}
else {
@ -899,8 +908,9 @@ static int get_pack(struct fetch_pack_args *args,
cmd.git_cmd = 1;
if (start_command(&cmd))
die(_("fetch-pack: unable to fork off %s"), cmd_name);
if (do_keep && pack_lockfile) {
*pack_lockfile = index_pack_lockfile(cmd.out);
if (do_keep && pack_lockfiles) {
string_list_append_nodup(pack_lockfiles,
index_pack_lockfile(cmd.out));
close(cmd.out);
}
@ -922,8 +932,8 @@ static int get_pack(struct fetch_pack_args *args,
* Now that index-pack has succeeded, write the promisor file using the
* obtained .keep filename if necessary
*/
if (do_keep && pack_lockfile && args->from_promisor)
write_promisor_file(*pack_lockfile, sought, nr_sought);
if (do_keep && pack_lockfiles && pack_lockfiles->nr && args->from_promisor)
write_promisor_file(pack_lockfiles->items[0].string, sought, nr_sought);
return 0;
}
@ -940,7 +950,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
const struct ref *orig_ref,
struct ref **sought, int nr_sought,
struct shallow_info *si,
char **pack_lockfile)
struct string_list *pack_lockfiles)
{
struct repository *r = the_repository;
struct ref *ref = copy_ref_list(orig_ref);
@ -1067,7 +1077,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
alternate_shallow_file = setup_temporary_shallow(si->shallow);
else
alternate_shallow_file = NULL;
if (get_pack(args, fd, pack_lockfile, sought, nr_sought))
if (get_pack(args, fd, pack_lockfiles, 1, sought, nr_sought))
die(_("git fetch-pack: fetch failed."));
all_done:
@ -1221,6 +1231,26 @@ static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out,
warning("filtering not recognized by server, ignoring");
}
if (server_supports_feature("fetch", "packfile-uris", 0)) {
int i;
struct strbuf to_send = STRBUF_INIT;
for (i = 0; i < uri_protocols.nr; i++) {
const char *s = uri_protocols.items[i].string;
if (!strcmp(s, "https") || !strcmp(s, "http")) {
if (to_send.len)
strbuf_addch(&to_send, ',');
strbuf_addstr(&to_send, s);
}
}
if (to_send.len) {
packet_buf_write(&req_buf, "packfile-uris %s",
to_send.buf);
strbuf_release(&to_send);
}
}
/* add wants */
add_wants(args->no_dependents, wants, &req_buf);
@ -1443,6 +1473,21 @@ static void receive_wanted_refs(struct packet_reader *reader,
die(_("error processing wanted refs: %d"), reader->status);
}
static void receive_packfile_uris(struct packet_reader *reader,
struct string_list *uris)
{
process_section_header(reader, "packfile-uris", 0);
while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
if (reader->pktlen < the_hash_algo->hexsz ||
reader->line[the_hash_algo->hexsz] != ' ')
die("expected '<hash> <uri>', got: %s\n", reader->line);
string_list_append(uris, reader->line);
}
if (reader->status != PACKET_READ_DELIM)
die("expected DELIM");
}
enum fetch_state {
FETCH_CHECK_LOCAL = 0,
FETCH_SEND_REQUEST,
@ -1464,7 +1509,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
struct ref **sought, int nr_sought,
struct oid_array *shallows,
struct shallow_info *si,
char **pack_lockfile)
struct string_list *pack_lockfiles)
{
struct repository *r = the_repository;
struct ref *ref = copy_ref_list(orig_ref);
@ -1476,6 +1521,8 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
struct fetch_negotiator negotiator_alloc;
struct fetch_negotiator *negotiator;
int seen_ack = 0;
struct string_list packfile_uris = STRING_LIST_INIT_DUP;
int i;
if (args->no_dependents) {
negotiator = NULL;
@ -1569,9 +1616,12 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
if (process_section_header(&reader, "wanted-refs", 1))
receive_wanted_refs(&reader, sought, nr_sought);
/* get the pack */
/* get the pack(s) */
if (process_section_header(&reader, "packfile-uris", 1))
receive_packfile_uris(&reader, &packfile_uris);
process_section_header(&reader, "packfile", 0);
if (get_pack(args, fd, pack_lockfile, sought, nr_sought))
if (get_pack(args, fd, pack_lockfiles,
!packfile_uris.nr, sought, nr_sought))
die(_("git fetch-pack: fetch failed."));
do_check_stateless_delimiter(args, &reader);
@ -1582,8 +1632,55 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
}
}
for (i = 0; i < packfile_uris.nr; i++) {
struct child_process cmd = CHILD_PROCESS_INIT;
char packname[GIT_MAX_HEXSZ + 1];
const char *uri = packfile_uris.items[i].string +
the_hash_algo->hexsz + 1;
argv_array_push(&cmd.args, "http-fetch");
argv_array_pushf(&cmd.args, "--packfile=%.*s",
(int) the_hash_algo->hexsz,
packfile_uris.items[i].string);
argv_array_push(&cmd.args, uri);
cmd.git_cmd = 1;
cmd.no_stdin = 1;
cmd.out = -1;
if (start_command(&cmd))
die("fetch-pack: unable to spawn http-fetch");
if (read_in_full(cmd.out, packname, 5) < 0 ||
memcmp(packname, "keep\t", 5))
die("fetch-pack: expected keep then TAB at start of http-fetch output");
if (read_in_full(cmd.out, packname,
the_hash_algo->hexsz + 1) < 0 ||
packname[the_hash_algo->hexsz] != '\n')
die("fetch-pack: expected hash then LF at end of http-fetch output");
packname[the_hash_algo->hexsz] = '\0';
close(cmd.out);
if (finish_command(&cmd))
die("fetch-pack: unable to finish http-fetch");
if (memcmp(packfile_uris.items[i].string, packname,
the_hash_algo->hexsz))
die("fetch-pack: pack downloaded from %s does not match expected hash %.*s",
uri, (int) the_hash_algo->hexsz,
packfile_uris.items[i].string);
string_list_append_nodup(pack_lockfiles,
xstrfmt("%s/pack/pack-%s.keep",
get_object_directory(),
packname));
}
string_list_clear(&packfile_uris, 0);
if (negotiator)
negotiator->release(negotiator);
oidset_clear(&common);
return ref;
}
@ -1620,6 +1717,14 @@ static void fetch_pack_config(void)
git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
if (!uri_protocols.nr) {
char *str;
if (!git_config_get_string("fetch.uriprotocols", &str) && str) {
string_list_split(&uri_protocols, str, ',', -1);
free(str);
}
}
git_config(fetch_pack_config_cb, NULL);
}
@ -1772,7 +1877,7 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
const struct ref *ref,
struct ref **sought, int nr_sought,
struct oid_array *shallow,
char **pack_lockfile,
struct string_list *pack_lockfiles,
enum protocol_version version)
{
struct ref *ref_cpy;
@ -1807,11 +1912,11 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
memset(&si, 0, sizeof(si));
ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
&shallows_scratch, &si,
pack_lockfile);
pack_lockfiles);
} else {
prepare_shallow_info(&si, shallow);
ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
&si, pack_lockfile);
&si, pack_lockfiles);
}
reprepare_packed_git(the_repository);