mirrors/git
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

upload-pack: send part of packfile response as uri

Browse Source 
Teach upload-pack to send part of its packfile response as URIs.

An administrator may configure a repository with one or more
"uploadpack.blobpackfileuri" lines, each line containing an OID, a pack
hash, and a URI. A client may configure fetch.uriprotocols to be a
comma-separated list of protocols that it is willing to use to fetch
additional packfiles - this list will be sent to the server. Whenever an
object with one of those OIDs would appear in the packfile transmitted
by upload-pack, the server may exclude that object, and instead send the
URI. The client will then download the packs referred to by those URIs
before performing the connectivity check.

Signed-off-by: Jonathan Tan <jonathantanmy@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
This commit is contained in:
Jonathan Tan
2020-06-10 13:57:23 -07:00
committed by Junio C Hamano
parent 9da69a6539
commit dd4b732df7
4 changed files with 343 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
upload-pack.c
View File

@ -83,6 +83,8 @@ struct upload_pack_data {
/* 0 for no sideband, otherwise DEFAULT_PACKET_MAX or LARGE_PACKET_MAX */
int use_sideband;
struct string_list uri_protocols;
struct list_objects_filter_options filter_options;
struct packet_writer writer;
@ -114,6 +116,7 @@ static void upload_pack_data_init(struct upload_pack_data *data)
struct oid_array haves = OID_ARRAY_INIT;
struct object_array shallows = OBJECT_ARRAY_INIT;
struct string_list deepen_not = STRING_LIST_INIT_DUP;
struct string_list uri_protocols = STRING_LIST_INIT_DUP;
memset(data, 0, sizeof(*data));
data->symref = symref;
@ -123,6 +126,7 @@ static void upload_pack_data_init(struct upload_pack_data *data)
data->haves = haves;
data->shallows = shallows;
data->deepen_not = deepen_not;
data->uri_protocols = uri_protocols;
packet_writer_init(&data->writer, 1);
data->keepalive = 5;
@ -176,10 +180,12 @@ static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
struct output_state {
char buffer[8193];
int used;
unsigned packfile_uris_started : 1;
unsigned packfile_started : 1;
};
static int relay_pack_data(int pack_objects_out, struct output_state *os,
int use_sideband)
int use_sideband, int write_packfile_line)
{
/*
* We keep the last byte to ourselves
@ -200,6 +206,37 @@ static int relay_pack_data(int pack_objects_out, struct output_state *os,
}
os->used += readsz;
while (!os->packfile_started) {
char *p;
if (os->used >= 4 && !memcmp(os->buffer, "PACK", 4)) {
os->packfile_started = 1;
if (write_packfile_line) {
if (os->packfile_uris_started)
packet_delim(1);
packet_write_fmt(1, "\1packfile\n");
}
break;
}
if ((p = memchr(os->buffer, '\n', os->used))) {
if (!os->packfile_uris_started) {
os->packfile_uris_started = 1;
if (!write_packfile_line)
BUG("packfile_uris requires sideband-all");
packet_write_fmt(1, "\1packfile-uris\n");
}
*p = '\0';
packet_write_fmt(1, "\1%s\n", os->buffer);
os->used -= p - os->buffer + 1;
memmove(os->buffer, p + 1, os->used);
} else {
/*
* Incomplete line.
*/
return readsz;
}
}
if (os->used > 1) {
send_client_data(1, os->buffer, os->used - 1, use_sideband);
os->buffer[0] = os->buffer[os->used - 1];
@ -212,7 +249,8 @@ static int relay_pack_data(int pack_objects_out, struct output_state *os,
return readsz;
}
static void create_pack_file(struct upload_pack_data *pack_data)
static void create_pack_file(struct upload_pack_data *pack_data,
const struct string_list *uri_protocols)
{
struct child_process pack_objects = CHILD_PROCESS_INIT;
struct output_state output_state = { { 0 } };
@ -262,6 +300,11 @@ static void create_pack_file(struct upload_pack_data *pack_data)
spec);
}
}
if (uri_protocols) {
for (i = 0; i < uri_protocols->nr; i++)
argv_array_pushf(&pack_objects.args, "--uri-protocol=%s",
uri_protocols->items[i].string);
}
pack_objects.in = -1;
pack_objects.out = -1;
@ -353,7 +396,8 @@ static void create_pack_file(struct upload_pack_data *pack_data)
if (0 <= pu && (pfd[pu].revents & (POLLIN|POLLHUP))) {
int result = relay_pack_data(pack_objects.out,
&output_state,
pack_data->use_sideband);
pack_data->use_sideband,
!!uri_protocols);
if (result == 0) {
close(pack_objects.out);
@ -1210,7 +1254,7 @@ void upload_pack(struct upload_pack_options *options)
receive_needs(&data, &reader);
if (data.want_obj.nr) {
get_common_commits(&data, &reader);
create_pack_file(&data);
create_pack_file(&data, 0);
}
}
@ -1363,10 +1407,18 @@ static void process_args(struct packet_reader *request,
continue;
}
if (skip_prefix(arg, "packfile-uris ", &p)) {
string_list_split(&data->uri_protocols, p, ',', -1);
continue;
}
/* ignore unknown lines maybe? */
die("unexpected line: '%s'", arg);
}
if (data->uri_protocols.nr && !data->writer.use_sideband)
string_list_clear(&data->uri_protocols, 0);
if (request->status != PACKET_READ_FLUSH)
die(_("expected flush after fetch arguments"));
}
@ -1553,8 +1605,12 @@ int upload_pack_v2(struct repository *r, struct argv_array *keys,
send_wanted_ref_info(&data);
send_shallow_info(&data);
packet_writer_write(&data.writer, "packfile\n");
create_pack_file(&data);
if (data.uri_protocols.nr) {
create_pack_file(&data, &data.uri_protocols);
} else {
packet_writer_write(&data.writer, "packfile\n");
create_pack_file(&data, NULL);
}
state = FETCH_DONE;
break;
case FETCH_DONE:
@ -1573,6 +1629,7 @@ int upload_pack_advertise(struct repository *r,
int allow_filter_value;
int allow_ref_in_want;
int allow_sideband_all_value;
char *str = NULL;
strbuf_addstr(value, "shallow");
@ -1594,6 +1651,14 @@ int upload_pack_advertise(struct repository *r,
&allow_sideband_all_value) &&
allow_sideband_all_value))
strbuf_addstr(value, " sideband-all");
if (!repo_config_get_string(the_repository,
"uploadpack.blobpackfileuri",
&str) &&
str) {
strbuf_addstr(value, " packfile-uris");
free(str);
}
}
return 1;