mirrors/git
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'jt/cdn-offload' into jch

Browse Source 
The "fetch/clone" protocol has been updated to allow the server to
instruct the clients to grab pre-packaged packfile(s) in addition
to the packed object data coming over the wire.

* jt/cdn-offload:
  upload-pack: fix a sparse '0 as NULL pointer' warning
  upload-pack: send part of packfile response as uri
  fetch-pack: support more than one pack lockfile
  upload-pack: refactor reading of pack-objects out
  Documentation: add Packfile URIs design doc
  Documentation: order protocol v2 sections
  http-fetch: support fetching packfiles by URL
  http-fetch: refactor into function
  http: refactor finish_http_pack_request()
  http: use --stdin when indexing dumb HTTP pack
This commit is contained in:
Junio C Hamano
2020-06-18 16:46:05 -07:00
parent e13bf44c75 cae2ee1055
commit 314f988d9c
19 changed files with 765 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files

156
upload-pack.c
View File

@ -84,6 +84,7 @@ struct upload_pack_data {
/* 0 for no sideband, otherwise DEFAULT_PACKET_MAX or LARGE_PACKET_MAX */
int use_sideband;
struct string_list uri_protocols;
enum allow_uor allow_uor;
struct list_objects_filter_options filter_options;
@ -117,6 +118,7 @@ static void upload_pack_data_init(struct upload_pack_data *data)
struct oid_array haves = OID_ARRAY_INIT;
struct object_array shallows = OBJECT_ARRAY_INIT;
struct string_list deepen_not = STRING_LIST_INIT_DUP;
struct string_list uri_protocols = STRING_LIST_INIT_DUP;
struct object_array extra_edge_obj = OBJECT_ARRAY_INIT;
memset(data, 0, sizeof(*data));
@ -127,6 +129,7 @@ static void upload_pack_data_init(struct upload_pack_data *data)
data->haves = haves;
data->shallows = shallows;
data->deepen_not = deepen_not;
data->uri_protocols = uri_protocols;
data->extra_edge_obj = extra_edge_obj;
packet_writer_init(&data->writer, 1);
@ -179,13 +182,86 @@ static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
return 0;
}
static void create_pack_file(struct upload_pack_data *pack_data)
struct output_state {
char buffer[8193];
int used;
unsigned packfile_uris_started : 1;
unsigned packfile_started : 1;
};
static int relay_pack_data(int pack_objects_out, struct output_state *os,
int use_sideband, int write_packfile_line)
{
/*
* We keep the last byte to ourselves
* in case we detect broken rev-list, so that we
* can leave the stream corrupted. This is
* unfortunate -- unpack-objects would happily
* accept a valid packdata with trailing garbage,
* so appending garbage after we pass all the
* pack data is not good enough to signal
* breakage to downstream.
*/
ssize_t readsz;
readsz = xread(pack_objects_out, os->buffer + os->used,
sizeof(os->buffer) - os->used);
if (readsz < 0) {
return readsz;
}
os->used += readsz;
while (!os->packfile_started) {
char *p;
if (os->used >= 4 && !memcmp(os->buffer, "PACK", 4)) {
os->packfile_started = 1;
if (write_packfile_line) {
if (os->packfile_uris_started)
packet_delim(1);
packet_write_fmt(1, "\1packfile\n");
}
break;
}
if ((p = memchr(os->buffer, '\n', os->used))) {
if (!os->packfile_uris_started) {
os->packfile_uris_started = 1;
if (!write_packfile_line)
BUG("packfile_uris requires sideband-all");
packet_write_fmt(1, "\1packfile-uris\n");
}
*p = '\0';
packet_write_fmt(1, "\1%s\n", os->buffer);
os->used -= p - os->buffer + 1;
memmove(os->buffer, p + 1, os->used);
} else {
/*
* Incomplete line.
*/
return readsz;
}
}
if (os->used > 1) {
send_client_data(1, os->buffer, os->used - 1, use_sideband);
os->buffer[0] = os->buffer[os->used - 1];
os->used = 1;
} else {
send_client_data(1, os->buffer, os->used, use_sideband);
os->used = 0;
}
return readsz;
}
static void create_pack_file(struct upload_pack_data *pack_data,
const struct string_list *uri_protocols)
{
struct child_process pack_objects = CHILD_PROCESS_INIT;
char data[8193], progress[128];
struct output_state output_state = { { 0 } };
char progress[128];
char abort_msg[] = "aborting due to possible repository "
"corruption on the remote side.";
int buffered = -1;
ssize_t sz;
int i;
FILE *pipe_fd;
@ -229,6 +305,11 @@ static void create_pack_file(struct upload_pack_data *pack_data)
spec);
}
}
if (uri_protocols) {
for (i = 0; i < uri_protocols->nr; i++)
argv_array_pushf(&pack_objects.args, "--uri-protocol=%s",
uri_protocols->items[i].string);
}
pack_objects.in = -1;
pack_objects.out = -1;
@ -318,40 +399,17 @@ static void create_pack_file(struct upload_pack_data *pack_data)
continue;
}
if (0 <= pu && (pfd[pu].revents & (POLLIN|POLLHUP))) {
/* Data ready; we keep the last byte to ourselves
* in case we detect broken rev-list, so that we
* can leave the stream corrupted. This is
* unfortunate -- unpack-objects would happily
* accept a valid packdata with trailing garbage,
* so appending garbage after we pass all the
* pack data is not good enough to signal
* breakage to downstream.
*/
char *cp = data;
ssize_t outsz = 0;
if (0 <= buffered) {
*cp++ = buffered;
outsz++;
}
sz = xread(pack_objects.out, cp,
sizeof(data) - outsz);
if (0 < sz)
;
else if (sz == 0) {
int result = relay_pack_data(pack_objects.out,
&output_state,
pack_data->use_sideband,
!!uri_protocols);
if (result == 0) {
close(pack_objects.out);
pack_objects.out = -1;
}
else
} else if (result < 0) {
goto fail;
sz += outsz;
if (1 < sz) {
buffered = data[sz-1] & 0xFF;
sz--;
}
else
buffered = -1;
send_client_data(1, data, sz,
pack_data->use_sideband);
}
/*
@ -376,9 +434,8 @@ static void create_pack_file(struct upload_pack_data *pack_data)
}
/* flush the data */
if (0 <= buffered) {
data[0] = buffered;
send_client_data(1, data, 1,
if (output_state.used > 0) {
send_client_data(1, output_state.buffer, output_state.used,
pack_data->use_sideband);
fprintf(stderr, "flushed.\n");
}
@ -1188,7 +1245,7 @@ void upload_pack(struct upload_pack_options *options)
receive_needs(&data, &reader);
if (data.want_obj.nr) {
get_common_commits(&data, &reader);
create_pack_file(&data);
create_pack_file(&data, NULL);
}
}
@ -1341,10 +1398,18 @@ static void process_args(struct packet_reader *request,
continue;
}
if (skip_prefix(arg, "packfile-uris ", &p)) {
string_list_split(&data->uri_protocols, p, ',', -1);
continue;
}
/* ignore unknown lines maybe? */
die("unexpected line: '%s'", arg);
}
if (data->uri_protocols.nr && !data->writer.use_sideband)
string_list_clear(&data->uri_protocols, 0);
if (request->status != PACKET_READ_FLUSH)
die(_("expected flush after fetch arguments"));
}
@ -1502,8 +1567,12 @@ int upload_pack_v2(struct repository *r, struct argv_array *keys,
send_wanted_ref_info(&data);
send_shallow_info(&data);
packet_writer_write(&data.writer, "packfile\n");
create_pack_file(&data);
if (data.uri_protocols.nr) {
create_pack_file(&data, &data.uri_protocols);
} else {
packet_writer_write(&data.writer, "packfile\n");
create_pack_file(&data, NULL);
}
state = FETCH_DONE;
break;
case FETCH_DONE:
@ -1522,6 +1591,7 @@ int upload_pack_advertise(struct repository *r,
int allow_filter_value;
int allow_ref_in_want;
int allow_sideband_all_value;
char *str = NULL;
strbuf_addstr(value, "shallow");
@ -1543,6 +1613,14 @@ int upload_pack_advertise(struct repository *r,
&allow_sideband_all_value) &&
allow_sideband_all_value))
strbuf_addstr(value, " sideband-all");
if (!repo_config_get_string(the_repository,
"uploadpack.blobpackfileuri",
&str) &&
str) {
strbuf_addstr(value, " packfile-uris");
free(str);
}
}
return 1;