Merge branch 'bw/protocol-v2'
The beginning of the next-gen transfer protocol. * bw/protocol-v2: (35 commits) remote-curl: don't request v2 when pushing remote-curl: implement stateless-connect command http: eliminate "# service" line when using protocol v2 http: don't always add Git-Protocol header http: allow providing extra headers for http requests remote-curl: store the protocol version the server responded with remote-curl: create copy of the service name pkt-line: add packet_buf_write_len function transport-helper: introduce stateless-connect transport-helper: refactor process_connect_service transport-helper: remove name parameter connect: don't request v2 when pushing connect: refactor git_connect to only get the protocol version once fetch-pack: support shallow requests fetch-pack: perform a fetch using v2 upload-pack: introduce fetch server command push: pass ref prefixes when pushing fetch: pass ref prefixes when fetching ls-remote: pass ref prefixes when requesting a remote's refs transport: convert transport_get_remote_refs to take a list of ref prefixes ...
This commit is contained in:
Junio C Hamano
280
remote-curl.c
280
remote-curl.c
@ -1,6 +1,7 @@
|
||||
#include "cache.h"
|
||||
#include "config.h"
|
||||
#include "remote.h"
|
||||
#include "connect.h"
|
||||
#include "strbuf.h"
|
||||
#include "walker.h"
|
||||
#include "http.h"
|
||||
@ -13,6 +14,7 @@
|
||||
#include "credential.h"
|
||||
#include "sha1-array.h"
|
||||
#include "send-pack.h"
|
||||
#include "protocol.h"
|
||||
#include "quote.h"
|
||||
|
||||
static struct remote *remote;
|
||||
@ -184,12 +186,13 @@ static int set_option(const char *name, const char *value)
|
||||
}
|
||||
|
||||
struct discovery {
|
||||
const char *service;
|
||||
char *service;
|
||||
char *buf_alloc;
|
||||
char *buf;
|
||||
size_t len;
|
||||
struct ref *refs;
|
||||
struct oid_array shallow;
|
||||
enum protocol_version version;
|
||||
unsigned proto_git : 1;
|
||||
};
|
||||
static struct discovery *last_discovery;
|
||||
@ -197,8 +200,31 @@ static struct discovery *last_discovery;
|
||||
static struct ref *parse_git_refs(struct discovery *heads, int for_push)
|
||||
{
|
||||
struct ref *list = NULL;
|
||||
get_remote_heads(-1, heads->buf, heads->len, &list,
|
||||
for_push ? REF_NORMAL : 0, NULL, &heads->shallow);
|
||||
struct packet_reader reader;
|
||||
|
||||
packet_reader_init(&reader, -1, heads->buf, heads->len,
|
||||
PACKET_READ_CHOMP_NEWLINE |
|
||||
PACKET_READ_GENTLE_ON_EOF);
|
||||
|
||||
heads->version = discover_version(&reader);
|
||||
switch (heads->version) {
|
||||
case protocol_v2:
|
||||
/*
|
||||
* Do nothing. This isn't a list of refs but rather a
|
||||
* capability advertisement. Client would have run
|
||||
* 'stateless-connect' so we'll dump this capability listing
|
||||
* and let them request the refs themselves.
|
||||
*/
|
||||
break;
|
||||
case protocol_v1:
|
||||
case protocol_v0:
|
||||
get_remote_heads(&reader, &list, for_push ? REF_NORMAL : 0,
|
||||
NULL, &heads->shallow);
|
||||
break;
|
||||
case protocol_unknown_version:
|
||||
BUG("unknown protocol version");
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
@ -259,6 +285,7 @@ static void free_discovery(struct discovery *d)
|
||||
free(d->shallow.oid);
|
||||
free(d->buf_alloc);
|
||||
free_refs(d->refs);
|
||||
free(d->service);
|
||||
free(d);
|
||||
}
|
||||
}
|
||||
@ -290,6 +317,19 @@ static int show_http_message(struct strbuf *type, struct strbuf *charset,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int get_protocol_http_header(enum protocol_version version,
|
||||
struct strbuf *header)
|
||||
{
|
||||
if (version > 0) {
|
||||
strbuf_addf(header, GIT_PROTOCOL_HEADER ": version=%d",
|
||||
version);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static struct discovery *discover_refs(const char *service, int for_push)
|
||||
{
|
||||
struct strbuf exp = STRBUF_INIT;
|
||||
@ -298,9 +338,12 @@ static struct discovery *discover_refs(const char *service, int for_push)
|
||||
struct strbuf buffer = STRBUF_INIT;
|
||||
struct strbuf refs_url = STRBUF_INIT;
|
||||
struct strbuf effective_url = STRBUF_INIT;
|
||||
struct strbuf protocol_header = STRBUF_INIT;
|
||||
struct string_list extra_headers = STRING_LIST_INIT_DUP;
|
||||
struct discovery *last = last_discovery;
|
||||
int http_ret, maybe_smart = 0;
|
||||
struct http_get_options http_options;
|
||||
enum protocol_version version = get_protocol_version_config();
|
||||
|
||||
if (last && !strcmp(service, last->service))
|
||||
return last;
|
||||
@ -317,11 +360,24 @@ static struct discovery *discover_refs(const char *service, int for_push)
|
||||
strbuf_addf(&refs_url, "service=%s", service);
|
||||
}
|
||||
|
||||
/*
|
||||
* NEEDSWORK: If we are trying to use protocol v2 and we are planning
|
||||
* to perform a push, then fallback to v0 since the client doesn't know
|
||||
* how to push yet using v2.
|
||||
*/
|
||||
if (version == protocol_v2 && !strcmp("git-receive-pack", service))
|
||||
version = protocol_v0;
|
||||
|
||||
/* Add the extra Git-Protocol header */
|
||||
if (get_protocol_http_header(version, &protocol_header))
|
||||
string_list_append(&extra_headers, protocol_header.buf);
|
||||
|
||||
memset(&http_options, 0, sizeof(http_options));
|
||||
http_options.content_type = &type;
|
||||
http_options.charset = &charset;
|
||||
http_options.effective_url = &effective_url;
|
||||
http_options.base_url = &url;
|
||||
http_options.extra_headers = &extra_headers;
|
||||
http_options.initial_request = 1;
|
||||
http_options.no_cache = 1;
|
||||
http_options.keep_error = 1;
|
||||
@ -345,7 +401,7 @@ static struct discovery *discover_refs(const char *service, int for_push)
|
||||
warning(_("redirecting to %s"), url.buf);
|
||||
|
||||
last= xcalloc(1, sizeof(*last_discovery));
|
||||
last->service = service;
|
||||
last->service = xstrdup(service);
|
||||
last->buf_alloc = strbuf_detach(&buffer, &last->len);
|
||||
last->buf = last->buf_alloc;
|
||||
|
||||
@ -377,6 +433,9 @@ static struct discovery *discover_refs(const char *service, int for_push)
|
||||
;
|
||||
|
||||
last->proto_git = 1;
|
||||
} else if (maybe_smart &&
|
||||
last->len > 5 && starts_with(last->buf + 4, "version 2")) {
|
||||
last->proto_git = 1;
|
||||
}
|
||||
|
||||
if (last->proto_git)
|
||||
@ -390,6 +449,8 @@ static struct discovery *discover_refs(const char *service, int for_push)
|
||||
strbuf_release(&charset);
|
||||
strbuf_release(&effective_url);
|
||||
strbuf_release(&buffer);
|
||||
strbuf_release(&protocol_header);
|
||||
string_list_clear(&extra_headers, 0);
|
||||
last_discovery = last;
|
||||
return last;
|
||||
}
|
||||
@ -426,6 +487,7 @@ struct rpc_state {
|
||||
char *service_url;
|
||||
char *hdr_content_type;
|
||||
char *hdr_accept;
|
||||
char *protocol_header;
|
||||
char *buf;
|
||||
size_t alloc;
|
||||
size_t len;
|
||||
@ -612,6 +674,10 @@ static int post_rpc(struct rpc_state *rpc)
|
||||
headers = curl_slist_append(headers, needs_100_continue ?
|
||||
"Expect: 100-continue" : "Expect:");
|
||||
|
||||
/* Add the extra Git-Protocol header */
|
||||
if (rpc->protocol_header)
|
||||
headers = curl_slist_append(headers, rpc->protocol_header);
|
||||
|
||||
retry:
|
||||
slot = get_active_slot();
|
||||
|
||||
@ -752,6 +818,11 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads)
|
||||
strbuf_addf(&buf, "Accept: application/x-%s-result", svc);
|
||||
rpc->hdr_accept = strbuf_detach(&buf, NULL);
|
||||
|
||||
if (get_protocol_http_header(heads->version, &buf))
|
||||
rpc->protocol_header = strbuf_detach(&buf, NULL);
|
||||
else
|
||||
rpc->protocol_header = NULL;
|
||||
|
||||
while (!err) {
|
||||
int n = packet_read(rpc->out, NULL, NULL, rpc->buf, rpc->alloc, 0);
|
||||
if (!n)
|
||||
@ -779,6 +850,7 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads)
|
||||
free(rpc->service_url);
|
||||
free(rpc->hdr_content_type);
|
||||
free(rpc->hdr_accept);
|
||||
free(rpc->protocol_header);
|
||||
free(rpc->buf);
|
||||
strbuf_release(&buf);
|
||||
return err;
|
||||
@ -1056,6 +1128,202 @@ static void parse_push(struct strbuf *buf)
|
||||
free(specs);
|
||||
}
|
||||
|
||||
/*
|
||||
* Used to represent the state of a connection to an HTTP server when
|
||||
* communicating using git's wire-protocol version 2.
|
||||
*/
|
||||
struct proxy_state {
|
||||
char *service_name;
|
||||
char *service_url;
|
||||
struct curl_slist *headers;
|
||||
struct strbuf request_buffer;
|
||||
int in;
|
||||
int out;
|
||||
struct packet_reader reader;
|
||||
size_t pos;
|
||||
int seen_flush;
|
||||
};
|
||||
|
||||
static void proxy_state_init(struct proxy_state *p, const char *service_name,
|
||||
enum protocol_version version)
|
||||
{
|
||||
struct strbuf buf = STRBUF_INIT;
|
||||
|
||||
memset(p, 0, sizeof(*p));
|
||||
p->service_name = xstrdup(service_name);
|
||||
|
||||
p->in = 0;
|
||||
p->out = 1;
|
||||
strbuf_init(&p->request_buffer, 0);
|
||||
|
||||
strbuf_addf(&buf, "%s%s", url.buf, p->service_name);
|
||||
p->service_url = strbuf_detach(&buf, NULL);
|
||||
|
||||
p->headers = http_copy_default_headers();
|
||||
|
||||
strbuf_addf(&buf, "Content-Type: application/x-%s-request", p->service_name);
|
||||
p->headers = curl_slist_append(p->headers, buf.buf);
|
||||
strbuf_reset(&buf);
|
||||
|
||||
strbuf_addf(&buf, "Accept: application/x-%s-result", p->service_name);
|
||||
p->headers = curl_slist_append(p->headers, buf.buf);
|
||||
strbuf_reset(&buf);
|
||||
|
||||
p->headers = curl_slist_append(p->headers, "Transfer-Encoding: chunked");
|
||||
|
||||
/* Add the Git-Protocol header */
|
||||
if (get_protocol_http_header(version, &buf))
|
||||
p->headers = curl_slist_append(p->headers, buf.buf);
|
||||
|
||||
packet_reader_init(&p->reader, p->in, NULL, 0,
|
||||
PACKET_READ_GENTLE_ON_EOF);
|
||||
|
||||
strbuf_release(&buf);
|
||||
}
|
||||
|
||||
static void proxy_state_clear(struct proxy_state *p)
|
||||
{
|
||||
free(p->service_name);
|
||||
free(p->service_url);
|
||||
curl_slist_free_all(p->headers);
|
||||
strbuf_release(&p->request_buffer);
|
||||
}
|
||||
|
||||
/*
|
||||
* CURLOPT_READFUNCTION callback function.
|
||||
* Attempts to copy over a single packet-line at a time into the
|
||||
* curl provided buffer.
|
||||
*/
|
||||
static size_t proxy_in(char *buffer, size_t eltsize,
|
||||
size_t nmemb, void *userdata)
|
||||
{
|
||||
size_t max;
|
||||
struct proxy_state *p = userdata;
|
||||
size_t avail = p->request_buffer.len - p->pos;
|
||||
|
||||
|
||||
if (eltsize != 1)
|
||||
BUG("curl read callback called with size = %"PRIuMAX" != 1",
|
||||
(uintmax_t)eltsize);
|
||||
max = nmemb;
|
||||
|
||||
if (!avail) {
|
||||
if (p->seen_flush) {
|
||||
p->seen_flush = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
strbuf_reset(&p->request_buffer);
|
||||
switch (packet_reader_read(&p->reader)) {
|
||||
case PACKET_READ_EOF:
|
||||
die("unexpected EOF when reading from parent process");
|
||||
case PACKET_READ_NORMAL:
|
||||
packet_buf_write_len(&p->request_buffer, p->reader.line,
|
||||
p->reader.pktlen);
|
||||
break;
|
||||
case PACKET_READ_DELIM:
|
||||
packet_buf_delim(&p->request_buffer);
|
||||
break;
|
||||
case PACKET_READ_FLUSH:
|
||||
packet_buf_flush(&p->request_buffer);
|
||||
p->seen_flush = 1;
|
||||
break;
|
||||
}
|
||||
p->pos = 0;
|
||||
avail = p->request_buffer.len;
|
||||
}
|
||||
|
||||
if (max < avail)
|
||||
avail = max;
|
||||
memcpy(buffer, p->request_buffer.buf + p->pos, avail);
|
||||
p->pos += avail;
|
||||
return avail;
|
||||
}
|
||||
|
||||
static size_t proxy_out(char *buffer, size_t eltsize,
|
||||
size_t nmemb, void *userdata)
|
||||
{
|
||||
size_t size;
|
||||
struct proxy_state *p = userdata;
|
||||
|
||||
if (eltsize != 1)
|
||||
BUG("curl read callback called with size = %"PRIuMAX" != 1",
|
||||
(uintmax_t)eltsize);
|
||||
size = nmemb;
|
||||
|
||||
write_or_die(p->out, buffer, size);
|
||||
return size;
|
||||
}
|
||||
|
||||
/* Issues a request to the HTTP server configured in `p` */
|
||||
static int proxy_request(struct proxy_state *p)
|
||||
{
|
||||
struct active_request_slot *slot;
|
||||
|
||||
slot = get_active_slot();
|
||||
|
||||
curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);
|
||||
curl_easy_setopt(slot->curl, CURLOPT_POST, 1);
|
||||
curl_easy_setopt(slot->curl, CURLOPT_URL, p->service_url);
|
||||
curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, p->headers);
|
||||
|
||||
/* Setup function to read request from client */
|
||||
curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, proxy_in);
|
||||
curl_easy_setopt(slot->curl, CURLOPT_READDATA, p);
|
||||
|
||||
/* Setup function to write server response to client */
|
||||
curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, proxy_out);
|
||||
curl_easy_setopt(slot->curl, CURLOPT_WRITEDATA, p);
|
||||
|
||||
if (run_slot(slot, NULL) != HTTP_OK)
|
||||
return -1;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int stateless_connect(const char *service_name)
|
||||
{
|
||||
struct discovery *discover;
|
||||
struct proxy_state p;
|
||||
|
||||
/*
|
||||
* Run the info/refs request and see if the server supports protocol
|
||||
* v2. If and only if the server supports v2 can we successfully
|
||||
* establish a stateless connection, otherwise we need to tell the
|
||||
* client to fallback to using other transport helper functions to
|
||||
* complete their request.
|
||||
*/
|
||||
discover = discover_refs(service_name, 0);
|
||||
if (discover->version != protocol_v2) {
|
||||
printf("fallback\n");
|
||||
fflush(stdout);
|
||||
return -1;
|
||||
} else {
|
||||
/* Stateless Connection established */
|
||||
printf("\n");
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
proxy_state_init(&p, service_name, discover->version);
|
||||
|
||||
/*
|
||||
* Dump the capability listing that we got from the server earlier
|
||||
* during the info/refs request.
|
||||
*/
|
||||
write_or_die(p.out, discover->buf, discover->len);
|
||||
|
||||
/* Peek the next packet line. Until we see EOF keep sending POSTs */
|
||||
while (packet_reader_peek(&p.reader) != PACKET_READ_EOF) {
|
||||
if (proxy_request(&p)) {
|
||||
/* We would have an err here */
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
proxy_state_clear(&p);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int cmd_main(int argc, const char **argv)
|
||||
{
|
||||
struct strbuf buf = STRBUF_INIT;
|
||||
@ -1124,12 +1392,16 @@ int cmd_main(int argc, const char **argv)
|
||||
fflush(stdout);
|
||||
|
||||
} else if (!strcmp(buf.buf, "capabilities")) {
|
||||
printf("stateless-connect\n");
|
||||
printf("fetch\n");
|
||||
printf("option\n");
|
||||
printf("push\n");
|
||||
printf("check-connectivity\n");
|
||||
printf("\n");
|
||||
fflush(stdout);
|
||||
} else if (skip_prefix(buf.buf, "stateless-connect ", &arg)) {
|
||||
if (!stateless_connect(arg))
|
||||
break;
|
||||
} else {
|
||||
error("remote-curl: unknown command '%s' from git", buf.buf);
|
||||
return 1;
|
||||
|
||||
Reference in New Issue
Block a user