Merge branch 'ab/fsck-unexpected-type'

"git fsck" has been taught to report mismatch between expected and
actual types of an object better.

* ab/fsck-unexpected-type:
  fsck: report invalid object type-path combinations
  fsck: don't hard die on invalid object types
  object-file.c: stop dying in parse_loose_header()
  object-file.c: return ULHR_TOO_LONG on "header too long"
  object-file.c: use "enum" return type for unpack_loose_header()
  object-file.c: simplify unpack_loose_short_header()
  object-file.c: make parse_loose_header_extended() public
  object-file.c: return -1, not "status" from unpack_loose_header()
  object-file.c: don't set "typep" when returning non-zero
  cat-file tests: test for current --allow-unknown-type behavior
  cat-file tests: add corrupt loose object test
  cat-file tests: test for missing/bogus object with -t, -s and -p
  cat-file tests: move bogus_* variable declarations earlier
  fsck tests: test for garbage appended to a loose object
  fsck tests: test current hash/type mismatch behavior
  fsck tests: refactor one test to use a sub-repo
  fsck tests: add test for fsck-ing an unknown type
This commit is contained in:
Junio C Hamano
2021-10-25 16:06:56 -07:00
13 changed files with 478 additions and 162 deletions

View File

@ -1016,9 +1016,11 @@ void *xmmap(void *start, size_t length,
* the streaming interface and rehash it to do the same.
*/
int check_object_signature(struct repository *r, const struct object_id *oid,
void *map, unsigned long size, const char *type)
void *map, unsigned long size, const char *type,
struct object_id *real_oidp)
{
struct object_id real_oid;
struct object_id tmp;
struct object_id *real_oid = real_oidp ? real_oidp : &tmp;
enum object_type obj_type;
struct git_istream *st;
git_hash_ctx c;
@ -1026,8 +1028,8 @@ int check_object_signature(struct repository *r, const struct object_id *oid,
int hdrlen;
if (map) {
hash_object_file(r->hash_algo, map, size, type, &real_oid);
return !oideq(oid, &real_oid) ? -1 : 0;
hash_object_file(r->hash_algo, map, size, type, real_oid);
return !oideq(oid, real_oid) ? -1 : 0;
}
st = open_istream(r, oid, &obj_type, &size, NULL);
@ -1052,9 +1054,9 @@ int check_object_signature(struct repository *r, const struct object_id *oid,
break;
r->hash_algo->update_fn(&c, buf, readlen);
}
r->hash_algo->final_oid_fn(&real_oid, &c);
r->hash_algo->final_oid_fn(real_oid, &c);
close_istream(st);
return !oideq(oid, &real_oid) ? -1 : 0;
return !oideq(oid, real_oid) ? -1 : 0;
}
int git_open_cloexec(const char *name, int flags)
@ -1187,11 +1189,14 @@ void *map_loose_object(struct repository *r,
return map_loose_object_1(r, NULL, oid, size);
}
static int unpack_loose_short_header(git_zstream *stream,
unsigned char *map, unsigned long mapsize,
void *buffer, unsigned long bufsiz)
enum unpack_loose_header_result unpack_loose_header(git_zstream *stream,
unsigned char *map,
unsigned long mapsize,
void *buffer,
unsigned long bufsiz,
struct strbuf *header)
{
int ret;
int status;
/* Get the data stream */
memset(stream, 0, sizeof(*stream));
@ -1202,43 +1207,24 @@ static int unpack_loose_short_header(git_zstream *stream,
git_inflate_init(stream);
obj_read_unlock();
ret = git_inflate(stream, 0);
status = git_inflate(stream, 0);
obj_read_lock();
return ret;
}
int unpack_loose_header(git_zstream *stream,
unsigned char *map, unsigned long mapsize,
void *buffer, unsigned long bufsiz)
{
int status = unpack_loose_short_header(stream, map, mapsize,
buffer, bufsiz);
if (status < Z_OK)
return status;
/* Make sure we have the terminating NUL */
if (!memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
return -1;
return 0;
}
static int unpack_loose_header_to_strbuf(git_zstream *stream, unsigned char *map,
unsigned long mapsize, void *buffer,
unsigned long bufsiz, struct strbuf *header)
{
int status;
status = unpack_loose_short_header(stream, map, mapsize, buffer, bufsiz);
if (status < Z_OK)
return -1;
return ULHR_BAD;
/*
* Check if entire header is unpacked in the first iteration.
*/
if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
return 0;
return ULHR_OK;
/*
* We have a header longer than MAX_HEADER_LEN. The "header"
* here is only non-NULL when we run "cat-file
* --allow-unknown-type".
*/
if (!header)
return ULHR_TOO_LONG;
/*
* buffer[0..bufsiz] was not large enough. Copy the partial
@ -1259,7 +1245,7 @@ static int unpack_loose_header_to_strbuf(git_zstream *stream, unsigned char *map
stream->next_out = buffer;
stream->avail_out = bufsiz;
} while (status != Z_STREAM_END);
return -1;
return ULHR_TOO_LONG;
}
static void *unpack_loose_rest(git_zstream *stream,
@ -1317,8 +1303,7 @@ static void *unpack_loose_rest(git_zstream *stream,
* too permissive for what we want to check. So do an anal
* object header parse by hand.
*/
static int parse_loose_header_extended(const char *hdr, struct object_info *oi,
unsigned int flags)
int parse_loose_header(const char *hdr, struct object_info *oi)
{
const char *type_buf = hdr;
unsigned long size;
@ -1340,15 +1325,6 @@ static int parse_loose_header_extended(const char *hdr, struct object_info *oi,
type = type_from_string_gently(type_buf, type_len, 1);
if (oi->type_name)
strbuf_add(oi->type_name, type_buf, type_len);
/*
* Set type to 0 if its an unknown object and
* we're obtaining the type using '--allow-unknown-type'
* option.
*/
if ((flags & OBJECT_INFO_ALLOW_UNKNOWN_TYPE) && (type < 0))
type = 0;
else if (type < 0)
die(_("invalid object type"));
if (oi->typep)
*oi->typep = type;
@ -1375,15 +1351,14 @@ static int parse_loose_header_extended(const char *hdr, struct object_info *oi,
/*
* The length must be followed by a zero byte
*/
return *hdr ? -1 : type;
}
if (*hdr)
return -1;
int parse_loose_header(const char *hdr, unsigned long *sizep)
{
struct object_info oi = OBJECT_INFO_INIT;
oi.sizep = sizep;
return parse_loose_header_extended(hdr, &oi, 0);
/*
* The format is valid, but the type may still be bogus. The
* Caller needs to check its oi->typep.
*/
return 0;
}
static int loose_object_info(struct repository *r,
@ -1397,6 +1372,8 @@ static int loose_object_info(struct repository *r,
char hdr[MAX_HEADER_LEN];
struct strbuf hdrbuf = STRBUF_INIT;
unsigned long size_scratch;
enum object_type type_scratch;
int allow_unknown = flags & OBJECT_INFO_ALLOW_UNKNOWN_TYPE;
if (oi->delta_base_oid)
oidclr(oi->delta_base_oid);
@ -1427,43 +1404,48 @@ static int loose_object_info(struct repository *r,
if (!oi->sizep)
oi->sizep = &size_scratch;
if (!oi->typep)
oi->typep = &type_scratch;
if (oi->disk_sizep)
*oi->disk_sizep = mapsize;
if ((flags & OBJECT_INFO_ALLOW_UNKNOWN_TYPE)) {
if (unpack_loose_header_to_strbuf(&stream, map, mapsize, hdr, sizeof(hdr), &hdrbuf) < 0)
status = error(_("unable to unpack %s header with --allow-unknown-type"),
oid_to_hex(oid));
} else if (unpack_loose_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
switch (unpack_loose_header(&stream, map, mapsize, hdr, sizeof(hdr),
allow_unknown ? &hdrbuf : NULL)) {
case ULHR_OK:
if (parse_loose_header(hdrbuf.len ? hdrbuf.buf : hdr, oi) < 0)
status = error(_("unable to parse %s header"), oid_to_hex(oid));
else if (!allow_unknown && *oi->typep < 0)
die(_("invalid object type"));
if (!oi->contentp)
break;
*oi->contentp = unpack_loose_rest(&stream, hdr, *oi->sizep, oid);
if (*oi->contentp)
goto cleanup;
status = -1;
break;
case ULHR_BAD:
status = error(_("unable to unpack %s header"),
oid_to_hex(oid));
if (status < 0)
; /* Do nothing */
else if (hdrbuf.len) {
if ((status = parse_loose_header_extended(hdrbuf.buf, oi, flags)) < 0)
status = error(_("unable to parse %s header with --allow-unknown-type"),
oid_to_hex(oid));
} else if ((status = parse_loose_header_extended(hdr, oi, flags)) < 0)
status = error(_("unable to parse %s header"), oid_to_hex(oid));
if (status >= 0 && oi->contentp) {
*oi->contentp = unpack_loose_rest(&stream, hdr,
*oi->sizep, oid);
if (!*oi->contentp) {
git_inflate_end(&stream);
status = -1;
}
} else
git_inflate_end(&stream);
break;
case ULHR_TOO_LONG:
status = error(_("header for %s too long, exceeds %d bytes"),
oid_to_hex(oid), MAX_HEADER_LEN);
break;
}
git_inflate_end(&stream);
cleanup:
munmap(map, mapsize);
if (status && oi->typep)
*oi->typep = status;
if (oi->sizep == &size_scratch)
oi->sizep = NULL;
strbuf_release(&hdrbuf);
if (oi->typep == &type_scratch)
oi->typep = NULL;
oi->whence = OI_LOOSE;
return (status < 0) ? status : 0;
return status;
}
int obj_read_use_lock = 0;
@ -2524,17 +2506,16 @@ static int check_stream_oid(git_zstream *stream,
int read_loose_object(const char *path,
const struct object_id *expected_oid,
enum object_type *type,
unsigned long *size,
void **contents)
struct object_id *real_oid,
void **contents,
struct object_info *oi)
{
int ret = -1;
void *map = NULL;
unsigned long mapsize;
git_zstream stream;
char hdr[MAX_HEADER_LEN];
*contents = NULL;
unsigned long *size = oi->sizep;
map = map_loose_object_1(the_repository, path, NULL, &mapsize);
if (!map) {
@ -2542,19 +2523,19 @@ int read_loose_object(const char *path,
goto out;
}
if (unpack_loose_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0) {
if (unpack_loose_header(&stream, map, mapsize, hdr, sizeof(hdr),
NULL) < 0) {
error(_("unable to unpack header of %s"), path);
goto out;
}
*type = parse_loose_header(hdr, size);
if (*type < 0) {
if (parse_loose_header(hdr, oi) < 0) {
error(_("unable to parse header of %s"), path);
git_inflate_end(&stream);
goto out;
}
if (*type == OBJ_BLOB && *size > big_file_threshold) {
if (*oi->typep == OBJ_BLOB && *size > big_file_threshold) {
if (check_stream_oid(&stream, hdr, *size, path, expected_oid) < 0)
goto out;
} else {
@ -2565,10 +2546,7 @@ int read_loose_object(const char *path,
goto out;
}
if (check_object_signature(the_repository, expected_oid,
*contents, *size,
type_name(*type))) {
error(_("hash mismatch for %s (expected %s)"), path,
oid_to_hex(expected_oid));
*contents, *size, oi->type_name->buf, real_oid)) {
free(*contents);
goto out;
}