Git 2.23.4

Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>

Documentation/RelNotes/2.17.5.txt

@@ -0,0 +1,22 @@
+Git v2.17.5 Release Notes
+=========================
+
+This release is to address a security issue: CVE-2020-11008
+
+Fixes since v2.17.4
+-------------------
+
+ * With a crafted URL that contains a newline or empty host, or lacks
+   a scheme, the credential helper machinery can be fooled into
+   providing credential information that is not appropriate for the
+   protocol in use and host being contacted.
+
+   Unlike the vulnerability CVE-2020-5260 fixed in v2.17.4, the
+   credentials are not for a host of the attacker's choosing; instead,
+   they are for some unspecified host (based on how the configured
+   credential helper handles an absent "host" parameter).
+
+   The attack has been made impossible by refusing to work with
+   under-specified credential patterns.
+
+Credit for finding the vulnerability goes to Carlo Arenas.

Documentation/RelNotes/2.17.6.txt

@@ -0,0 +1,16 @@
+Git v2.17.6 Release Notes
+=========================
+
+This release addresses the security issues CVE-2021-21300.
+
+Fixes since v2.17.5
+-------------------
+
+ * CVE-2021-21300:
+   On case-insensitive file systems with support for symbolic links,
+   if Git is configured globally to apply delay-capable clean/smudge
+   filters (such as Git LFS), Git could be fooled into running
+   remote code during a clone.
+
+Credit for finding and fixing this vulnerability goes to Matheus
+Tavares, helped by Johannes Schindelin.

Documentation/RelNotes/2.18.4.txt

@@ -0,0 +1,5 @@
+Git v2.18.4 Release Notes
+=========================
+
+This release merges the security fix that appears in v2.17.5; see
+the release notes for that version for details.

Documentation/RelNotes/2.18.5.txt

@@ -0,0 +1,6 @@
+Git v2.18.5 Release Notes
+=========================
+
+This release merges up the fixes that appear in v2.17.6 to address
+the security issue CVE-2021-21300; see the release notes for that
+version for details.

Documentation/RelNotes/2.19.5.txt

@@ -0,0 +1,5 @@
+Git v2.19.5 Release Notes
+=========================
+
+This release merges the security fix that appears in v2.17.5; see
+the release notes for that version for details.

Documentation/RelNotes/2.19.6.txt

@@ -0,0 +1,6 @@
+Git v2.19.6 Release Notes
+=========================
+
+This release merges up the fixes that appear in v2.17.6 and
+v2.18.5 to address the security issue CVE-2021-21300; see the
+release notes for these versions for details.

Documentation/RelNotes/2.20.4.txt

@@ -0,0 +1,5 @@
+Git v2.20.4 Release Notes
+=========================
+
+This release merges the security fix that appears in v2.17.5; see
+the release notes for that version for details.

Documentation/RelNotes/2.20.5.txt

@@ -0,0 +1,6 @@
+Git v2.20.5 Release Notes
+=========================
+
+This release merges up the fixes that appear in v2.17.6, v2.18.5
+and v2.19.6 to address the security issue CVE-2021-21300; see
+the release notes for these versions for details.

Documentation/RelNotes/2.21.3.txt

@@ -0,0 +1,5 @@
+Git v2.21.3 Release Notes
+=========================
+
+This release merges the security fix that appears in v2.17.5; see
+the release notes for that version for details.

Documentation/RelNotes/2.21.4.txt

@@ -0,0 +1,6 @@
+Git v2.21.4 Release Notes
+=========================
+
+This release merges up the fixes that appear in v2.17.6, v2.18.5,
+v2.19.6 and v2.20.5 to address the security issue CVE-2021-21300;
+see the release notes for these versions for details.

Documentation/RelNotes/2.22.4.txt

@@ -0,0 +1,5 @@
+Git v2.22.4 Release Notes
+=========================
+
+This release merges the security fix that appears in v2.17.5; see
+the release notes for that version for details.

Documentation/RelNotes/2.22.5.txt

@@ -0,0 +1,7 @@
+Git v2.22.5 Release Notes
+=========================
+
+This release merges up the fixes that appear in v2.17.6,
+v2.18.5, v2.19.6, v2.20.5 and v2.21.4 to address the security
+issue CVE-2021-21300; see the release notes for these versions
+for details.

Documentation/RelNotes/2.23.3.txt

@@ -0,0 +1,5 @@
+Git v2.23.3 Release Notes
+=========================
+
+This release merges the security fix that appears in v2.17.5; see
+the release notes for that version for details.

Documentation/RelNotes/2.23.4.txt

@@ -0,0 +1,7 @@
+Git v2.23.4 Release Notes
+=========================
+
+This release merges up the fixes that appear in v2.17.6, v2.18.5,
+v2.19.6, v2.20.5, v2.21.4 and v2.22.5 to address the security
+issue CVE-2021-21300; see the release notes for these versions
+for details.

GIT-VERSION-GEN

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 GVF=GIT-VERSION-FILE
-DEF_VER=v2.23.2
+DEF_VER=v2.23.4
 
 LF='
 '

RelNotes

@@ -1 +1 @@
-Documentation/RelNotes/2.23.2.txt
+Documentation/RelNotes/2.23.4.txt

cache.h

@@ -1631,6 +1631,7 @@ int has_symlink_leading_path(const char *name, int len);
 int threaded_has_symlink_leading_path(struct cache_def *, const char *, int);
 int check_leading_path(const char *name, int len);
 int has_dirs_only_path(const char *name, int len, int prefix_len);
+void invalidate_lstat_cache(void);
 void schedule_dir_for_removal(const char *name, int len);
 void remove_scheduled_dirs(void);
 

compat/mingw.c

@@ -340,6 +340,8 @@ int mingw_rmdir(const char *pathname)
 	       ask_yes_no_if_possible("Deletion of directory '%s' failed. "
 			"Should I try again?", pathname))
 	       ret = _wrmdir(wpathname);
+	if (!ret)
+		invalidate_lstat_cache();
 	return ret;
 }
 

credential.c

@@ -89,6 +89,11 @@ static int proto_is_http(const char *s)
 
 static void credential_apply_config(struct credential *c)
 {
+	if (!c->host)
+		die(_("refusing to work with credential missing host field"));
+	if (!c->protocol)
+		die(_("refusing to work with credential missing protocol field"));
+
 	if (c->configured)
 		return;
 	git_config(credential_config_callback, c);
@@ -191,8 +196,11 @@ int credential_read(struct credential *c, FILE *fp)
 	return 0;
 }
 
-static void credential_write_item(FILE *fp, const char *key, const char *value)
+static void credential_write_item(FILE *fp, const char *key, const char *value,
+				  int required)
 {
+	if (!value && required)
+		BUG("credential value for %s is missing", key);
 	if (!value)
 		return;
 	if (strchr(value, '\n'))
@@ -202,11 +210,11 @@ static void credential_write_item(FILE *fp, const char *key, const char *value)
 
 void credential_write(const struct credential *c, FILE *fp)
 {
-	credential_write_item(fp, "protocol", c->protocol);
-	credential_write_item(fp, "host", c->host);
-	credential_write_item(fp, "path", c->path);
-	credential_write_item(fp, "username", c->username);
-	credential_write_item(fp, "password", c->password);
+	credential_write_item(fp, "protocol", c->protocol, 1);
+	credential_write_item(fp, "host", c->host, 1);
+	credential_write_item(fp, "path", c->path, 0);
+	credential_write_item(fp, "username", c->username, 0);
+	credential_write_item(fp, "password", c->password, 0);
 }
 
 static int run_credential_helper(struct credential *c,
@@ -352,8 +360,11 @@ int credential_from_url_gently(struct credential *c, const char *url,
 	 *   (3) proto://<user>:<pass>@<host>/...
 	 */
 	proto_end = strstr(url, "://");
-	if (!proto_end)
-		return 0;
+	if (!proto_end || proto_end == url) {
+		if (!quiet)
+			warning(_("url has no scheme: %s"), url);
+		return -1;
+	}
 	cp = proto_end + 3;
 	at = strchr(cp, '@');
 	colon = strchr(cp, ':');
@@ -374,10 +385,8 @@ int credential_from_url_gently(struct credential *c, const char *url,
 		host = at + 1;
 	}
 
-	if (proto_end - url > 0)
-		c->protocol = xmemdupz(url, proto_end - url);
-	if (slash - host > 0)
-		c->host = url_decode_mem(host, slash - host);
+	c->protocol = xmemdupz(url, proto_end - url);
+	c->host = url_decode_mem(host, slash - host);
 	/* Trim leading and trailing slashes from path */
 	while (*slash == '/')
 		slash++;
@@ -401,8 +410,6 @@ int credential_from_url_gently(struct credential *c, const char *url,
 
 void credential_from_url(struct credential *c, const char *url)
 {
-	if (credential_from_url_gently(c, url, 0) < 0) {
-		warning(_("skipping credential lookup for url: %s"), url);
-		credential_clear(c);
-	}
+	if (credential_from_url_gently(c, url, 0) < 0)
+		die(_("credential url cannot be parsed: %s"), url);
 }

fsck.c

@@ -9,6 +9,7 @@
 #include "tag.h"
 #include "fsck.h"
 #include "refs.h"
+#include "url.h"
 #include "utf8.h"
 #include "decorate.h"
 #include "oidset.h"
@@ -948,17 +949,147 @@ static int fsck_tag(struct tag *tag, const char *data,
 	return fsck_tag_buffer(tag, data, size, options);
 }
 
+/*
+ * Like builtin/submodule--helper.c's starts_with_dot_slash, but without
+ * relying on the platform-dependent is_dir_sep helper.
+ *
+ * This is for use in checking whether a submodule URL is interpreted as
+ * relative to the current directory on any platform, since \ is a
+ * directory separator on Windows but not on other platforms.
+ */
+static int starts_with_dot_slash(const char *str)
+{
+	return str[0] == '.' && (str[1] == '/' || str[1] == '\\');
+}
+
+/*
+ * Like starts_with_dot_slash, this is a variant of submodule--helper's
+ * helper of the same name with the twist that it accepts backslash as a
+ * directory separator even on non-Windows platforms.
+ */
+static int starts_with_dot_dot_slash(const char *str)
+{
+	return str[0] == '.' && starts_with_dot_slash(str + 1);
+}
+
+static int submodule_url_is_relative(const char *url)
+{
+	return starts_with_dot_slash(url) || starts_with_dot_dot_slash(url);
+}
+
+/*
+ * Count directory components that a relative submodule URL should chop
+ * from the remote_url it is to be resolved against.
+ *
+ * In other words, this counts "../" components at the start of a
+ * submodule URL.
+ *
+ * Returns the number of directory components to chop and writes a
+ * pointer to the next character of url after all leading "./" and
+ * "../" components to out.
+ */
+static int count_leading_dotdots(const char *url, const char **out)
+{
+	int result = 0;
+	while (1) {
+		if (starts_with_dot_dot_slash(url)) {
+			result++;
+			url += strlen("../");
+			continue;
+		}
+		if (starts_with_dot_slash(url)) {
+			url += strlen("./");
+			continue;
+		}
+		*out = url;
+		return result;
+	}
+}
+/*
+ * Check whether a transport is implemented by git-remote-curl.
+ *
+ * If it is, returns 1 and writes the URL that would be passed to
+ * git-remote-curl to the "out" parameter.
+ *
+ * Otherwise, returns 0 and leaves "out" untouched.
+ *
+ * Examples:
+ *   http::https://example.com/repo.git -> 1, https://example.com/repo.git
+ *   https://example.com/repo.git -> 1, https://example.com/repo.git
+ *   git://example.com/repo.git -> 0
+ *
+ * This is for use in checking for previously exploitable bugs that
+ * required a submodule URL to be passed to git-remote-curl.
+ */
+static int url_to_curl_url(const char *url, const char **out)
+{
+	/*
+	 * We don't need to check for case-aliases, "http.exe", and so
+	 * on because in the default configuration, is_transport_allowed
+	 * prevents URLs with those schemes from being cloned
+	 * automatically.
+	 */
+	if (skip_prefix(url, "http::", out) ||
+	    skip_prefix(url, "https::", out) ||
+	    skip_prefix(url, "ftp::", out) ||
+	    skip_prefix(url, "ftps::", out))
+		return 1;
+	if (starts_with(url, "http://") ||
+	    starts_with(url, "https://") ||
+	    starts_with(url, "ftp://") ||
+	    starts_with(url, "ftps://")) {
+		*out = url;
+		return 1;
+	}
+	return 0;
+}
+
 static int check_submodule_url(const char *url)
 {
-	struct credential c = CREDENTIAL_INIT;
-	int ret;
+	const char *curl_url;
 
 	if (looks_like_command_line_option(url))
 		return -1;
 
-	ret = credential_from_url_gently(&c, url, 1);
-	credential_clear(&c);
-	return ret;
+	if (submodule_url_is_relative(url)) {
+		char *decoded;
+		const char *next;
+		int has_nl;
+
+		/*
+		 * This could be appended to an http URL and url-decoded;
+		 * check for malicious characters.
+		 */
+		decoded = url_decode(url);
+		has_nl = !!strchr(decoded, '\n');
+
+		free(decoded);
+		if (has_nl)
+			return -1;
+
+		/*
+		 * URLs which escape their root via "../" can overwrite
+		 * the host field and previous components, resolving to
+		 * URLs like https::example.com/submodule.git and
+		 * https:///example.com/submodule.git that were
+		 * susceptible to CVE-2020-11008.
+		 */
+		if (count_leading_dotdots(url, &next) > 0 &&
+		    (*next == ':' || *next == '/'))
+			return -1;
+	}
+
+	else if (url_to_curl_url(url, &curl_url)) {
+		struct credential c = CREDENTIAL_INIT;
+		int ret = 0;
+		if (credential_from_url_gently(&c, curl_url, 1) ||
+		    !*c.host)
+			ret = -1;
+		credential_clear(&c);
+		return ret;
+	}
+
+	return 0;
 }
 
 struct fsck_gitmodules_data {

git-compat-util.h

@@ -364,6 +364,11 @@ static inline int noop_core_config(const char *var, const char *value, void *cb)
 #define platform_core_config noop_core_config
 #endif
 
+int lstat_cache_aware_rmdir(const char *path);
+#if !defined(__MINGW32__) && !defined(_MSC_VER)
+#define rmdir lstat_cache_aware_rmdir
+#endif
+
 #ifndef has_dos_drive_prefix
 static inline int git_has_dos_drive_prefix(const char *path)
 {

http.c

@@ -558,6 +558,7 @@ static int has_cert_password(void)
 		return 0;
 	if (!cert_auth.password) {
 		cert_auth.protocol = xstrdup("cert");
+		cert_auth.host = xstrdup("");
 		cert_auth.username = xstrdup("");
 		cert_auth.path = xstrdup(ssl_cert);
 		credential_fill(&cert_auth);

run-command.c

@@ -989,6 +989,7 @@ int finish_command(struct child_process *cmd)
 	int ret = wait_or_whine(cmd->pid, cmd->argv[0], 0);
 	trace2_child_exit(cmd, ret);
 	child_process_clear(cmd);
+	invalidate_lstat_cache();
 	return ret;
 }
 
@@ -1289,13 +1290,19 @@ error:
 int finish_async(struct async *async)
 {
 #ifdef NO_PTHREADS
-	return wait_or_whine(async->pid, "child process", 0);
+	int ret = wait_or_whine(async->pid, "child process", 0);
+
+	invalidate_lstat_cache();
+
+	return ret;
 #else
 	void *ret = (void *)(intptr_t)(-1);
 
 	if (pthread_join(async->tid, &ret))
 		error("pthread_join failed");
+	invalidate_lstat_cache();
 	return (int)(intptr_t)ret;
+
 #endif
 }
 

symlinks.c

@@ -267,6 +267,13 @@ int has_dirs_only_path(const char *name, int len, int prefix_len)
  */
 static int threaded_has_dirs_only_path(struct cache_def *cache, const char *name, int len, int prefix_len)
 {
+	/*
+	 * Note: this function is used by the checkout machinery, which also
+	 * takes care to properly reset the cache when it performs an operation
+	 * that would leave the cache outdated. If this function starts caching
+	 * anything else besides FL_DIR, remember to also invalidate the cache
+	 * when creating or deleting paths that might be in the cache.
+	 */
 	return lstat_cache(cache, name, len,
 			   FL_DIR|FL_FULLPATH, prefix_len) &
 		FL_DIR;
@@ -321,3 +328,20 @@ void remove_scheduled_dirs(void)
 {
 	do_remove_scheduled_dirs(0);
 }
+
+void invalidate_lstat_cache(void)
+{
+	reset_lstat_cache(&default_cache);
+}
+
+#undef rmdir
+int lstat_cache_aware_rmdir(const char *path)
+{
+	/* Any change in this function must be made also in `mingw_rmdir()` */
+	int ret = rmdir(path);
+
+	if (!ret)
+		invalidate_lstat_cache();
+
+	return ret;
+}

t/t0021-conversion.sh

@@ -817,4 +817,85 @@ test_expect_success PERL 'invalid file in delayed checkout' '
 	grep "error: external filter .* signaled that .unfiltered. is now available although it has not been delayed earlier" git-stderr.log
 '
 
+for mode in 'case' 'utf-8'
+do
+	case "$mode" in
+	case)	dir='A' symlink='a' mode_prereq='CASE_INSENSITIVE_FS' ;;
+	utf-8)
+		dir=$(printf "\141\314\210") symlink=$(printf "\303\244")
+		mode_prereq='UTF8_NFD_TO_NFC' ;;
+	esac
+
+	test_expect_success PERL,SYMLINKS,$mode_prereq \
+	"delayed checkout with $mode-collision don't write to the wrong place" '
+		test_config_global filter.delay.process \
+			"\"$TEST_ROOT/rot13-filter.pl\" --always-delay delayed.log clean smudge delay" &&
+		test_config_global filter.delay.required true &&
+
+		git init $mode-collision &&
+		(
+			cd $mode-collision &&
+			mkdir target-dir &&
+
+			empty_oid=$(printf "" | git hash-object -w --stdin) &&
+			symlink_oid=$(printf "%s" "$PWD/target-dir" | git hash-object -w --stdin) &&
+			attr_oid=$(echo "$dir/z filter=delay" | git hash-object -w --stdin) &&
+
+			cat >objs <<-EOF &&
+			100644 blob $empty_oid	$dir/x
+			100644 blob $empty_oid	$dir/y
+			100644 blob $empty_oid	$dir/z
+			120000 blob $symlink_oid	$symlink
+			100644 blob $attr_oid	.gitattributes
+			EOF
+
+			git update-index --index-info <objs &&
+			git commit -m "test commit"
+		) &&
+
+		git clone $mode-collision $mode-collision-cloned &&
+		# Make sure z was really delayed
+		grep "IN: smudge $dir/z .* \\[DELAYED\\]" $mode-collision-cloned/delayed.log &&
+
+		# Should not create $dir/z at $symlink/z
+		test_path_is_missing $mode-collision/target-dir/z
+	'
+done
+
+test_expect_success PERL,SYMLINKS,CASE_INSENSITIVE_FS \
+"delayed checkout with submodule collision don't write to the wrong place" '
+	git init collision-with-submodule &&
+	(
+		cd collision-with-submodule &&
+		git config filter.delay.process "\"$TEST_ROOT/rot13-filter.pl\" --always-delay delayed.log clean smudge delay" &&
+		git config filter.delay.required true &&
+
+		# We need Git to treat the submodule "a" and the
+		# leading dir "A" as different paths in the index.
+		git config --local core.ignoreCase false &&
+
+		empty_oid=$(printf "" | git hash-object -w --stdin) &&
+		attr_oid=$(echo "A/B/y filter=delay" | git hash-object -w --stdin) &&
+		cat >objs <<-EOF &&
+		100644 blob $empty_oid	A/B/x
+		100644 blob $empty_oid	A/B/y
+		100644 blob $attr_oid	.gitattributes
+		EOF
+		git update-index --index-info <objs &&
+
+		git init a &&
+		mkdir target-dir &&
+		symlink_oid=$(printf "%s" "$PWD/target-dir" | git -C a hash-object -w --stdin) &&
+		echo "120000 blob $symlink_oid	b" >objs &&
+		git -C a update-index --index-info <objs &&
+		git -C a commit -m sub &&
+		git submodule add ./a &&
+		git commit -m super &&
+
+		git checkout --recurse-submodules . &&
+		grep "IN: smudge A/B/y .* \\[DELAYED\\]" delayed.log &&
+		test_path_is_missing target-dir/y
+	)
+'
+
 test_done

t/t0021/rot13-filter.pl

@@ -2,9 +2,15 @@
 # Example implementation for the Git filter protocol version 2
 # See Documentation/gitattributes.txt, section "Filter Protocol"
 #
-# The first argument defines a debug log file that the script write to.
-# All remaining arguments define a list of supported protocol
-# capabilities ("clean", "smudge", etc).
+# Usage: rot13-filter.pl [--always-delay] <log path> <capabilities>
+#
+# Log path defines a debug log file that the script writes to. The
+# subsequent arguments define a list of supported protocol capabilities
+# ("clean", "smudge", etc).
+#
+# When --always-delay is given all pathnames with the "can-delay" flag
+# that don't appear on the list bellow are delayed with a count of 1
+# (see more below).
 #
 # This implementation supports special test cases:
 # (1) If data with the pathname "clean-write-fail.r" is processed with
@@ -53,6 +59,13 @@ use IO::File;
 use Git::Packet;
 
 my $MAX_PACKET_CONTENT_SIZE = 65516;
+
+my $always_delay = 0;
+if ( $ARGV[0] eq '--always-delay' ) {
+	$always_delay = 1;
+	shift @ARGV;
+}
+
 my $log_file                = shift @ARGV;
 my @capabilities            = @ARGV;
 
@@ -134,6 +147,8 @@ while (1) {
 			if ( $buffer eq "can-delay=1" ) {
 				if ( exists $DELAY{$pathname} and $DELAY{$pathname}{"requested"} == 0 ) {
 					$DELAY{$pathname}{"requested"} = 1;
+				} elsif ( !exists $DELAY{$pathname} and $always_delay ) {
+					$DELAY{$pathname} = { "requested" => 1, "count" => 1 };
 				}
 			} else {
 				die "Unknown message '$buffer'";

t/t0300-credentials.sh

@@ -22,6 +22,11 @@ test_expect_success 'setup helper scripts' '
 	exit 0
 	EOF
 
+	write_script git-credential-quit <<-\EOF &&
+	. ./dump
+	echo quit=1
+	EOF
+
 	write_script git-credential-verbatim <<-\EOF &&
 	user=$1; shift
 	pass=$1; shift
@@ -35,43 +40,71 @@ test_expect_success 'setup helper scripts' '
 
 test_expect_success 'credential_fill invokes helper' '
 	check fill "verbatim foo bar" <<-\EOF
+	protocol=http
+	host=example.com
 	--
+	protocol=http
+	host=example.com
 	username=foo
 	password=bar
 	--
 	verbatim: get
+	verbatim: protocol=http
+	verbatim: host=example.com
 	EOF
 '
 
 test_expect_success 'credential_fill invokes multiple helpers' '
 	check fill useless "verbatim foo bar" <<-\EOF
+	protocol=http
+	host=example.com
 	--
+	protocol=http
+	host=example.com
 	username=foo
 	password=bar
 	--
 	useless: get
+	useless: protocol=http
+	useless: host=example.com
 	verbatim: get
+	verbatim: protocol=http
+	verbatim: host=example.com
 	EOF
 '
 
 test_expect_success 'credential_fill stops when we get a full response' '
 	check fill "verbatim one two" "verbatim three four" <<-\EOF
+	protocol=http
+	host=example.com
 	--
+	protocol=http
+	host=example.com
 	username=one
 	password=two
 	--
 	verbatim: get
+	verbatim: protocol=http
+	verbatim: host=example.com
 	EOF
 '
 
 test_expect_success 'credential_fill continues through partial response' '
 	check fill "verbatim one \"\"" "verbatim two three" <<-\EOF
+	protocol=http
+	host=example.com
 	--
+	protocol=http
+	host=example.com
 	username=two
 	password=three
 	--
 	verbatim: get
+	verbatim: protocol=http
+	verbatim: host=example.com
 	verbatim: get
+	verbatim: protocol=http
+	verbatim: host=example.com
 	verbatim: username=one
 	EOF
 '
@@ -97,14 +130,20 @@ test_expect_success 'credential_fill passes along metadata' '
 
 test_expect_success 'credential_approve calls all helpers' '
 	check approve useless "verbatim one two" <<-\EOF
+	protocol=http
+	host=example.com
 	username=foo
 	password=bar
 	--
 	--
 	useless: store
+	useless: protocol=http
+	useless: host=example.com
 	useless: username=foo
 	useless: password=bar
 	verbatim: store
+	verbatim: protocol=http
+	verbatim: host=example.com
 	verbatim: username=foo
 	verbatim: password=bar
 	EOF
@@ -112,6 +151,8 @@ test_expect_success 'credential_approve calls all helpers' '
 
 test_expect_success 'do not bother storing password-less credential' '
 	check approve useless <<-\EOF
+	protocol=http
+	host=example.com
 	username=foo
 	--
 	--
@@ -121,14 +162,20 @@ test_expect_success 'do not bother storing password-less credential' '
 
 test_expect_success 'credential_reject calls all helpers' '
 	check reject useless "verbatim one two" <<-\EOF
+	protocol=http
+	host=example.com
 	username=foo
 	password=bar
 	--
 	--
 	useless: erase
+	useless: protocol=http
+	useless: host=example.com
 	useless: username=foo
 	useless: password=bar
 	verbatim: erase
+	verbatim: protocol=http
+	verbatim: host=example.com
 	verbatim: username=foo
 	verbatim: password=bar
 	EOF
@@ -136,33 +183,49 @@ test_expect_success 'credential_reject calls all helpers' '
 
 test_expect_success 'usernames can be preserved' '
 	check fill "verbatim \"\" three" <<-\EOF
+	protocol=http
+	host=example.com
 	username=one
 	--
+	protocol=http
+	host=example.com
 	username=one
 	password=three
 	--
 	verbatim: get
+	verbatim: protocol=http
+	verbatim: host=example.com
 	verbatim: username=one
 	EOF
 '
 
 test_expect_success 'usernames can be overridden' '
 	check fill "verbatim two three" <<-\EOF
+	protocol=http
+	host=example.com
 	username=one
 	--
+	protocol=http
+	host=example.com
 	username=two
 	password=three
 	--
 	verbatim: get
+	verbatim: protocol=http
+	verbatim: host=example.com
 	verbatim: username=one
 	EOF
 '
 
 test_expect_success 'do not bother completing already-full credential' '
 	check fill "verbatim three four" <<-\EOF
+	protocol=http
+	host=example.com
 	username=one
 	password=two
 	--
+	protocol=http
+	host=example.com
 	username=one
 	password=two
 	--
@@ -174,23 +237,31 @@ test_expect_success 'do not bother completing already-full credential' '
 # askpass helper is run, we know the internal getpass is working.
 test_expect_success 'empty helper list falls back to internal getpass' '
 	check fill <<-\EOF
+	protocol=http
+	host=example.com
 	--
+	protocol=http
+	host=example.com
 	username=askpass-username
 	password=askpass-password
 	--
-	askpass: Username:
-	askpass: Password:
+	askpass: Username for '\''http://example.com'\'':
+	askpass: Password for '\''http://askpass-username@example.com'\'':
 	EOF
 '
 
 test_expect_success 'internal getpass does not ask for known username' '
 	check fill <<-\EOF
+	protocol=http
+	host=example.com
 	username=foo
 	--
+	protocol=http
+	host=example.com
 	username=foo
 	password=askpass-password
 	--
-	askpass: Password:
+	askpass: Password for '\''http://foo@example.com'\'':
 	EOF
 '
 
@@ -202,7 +273,11 @@ HELPER="!f() {
 test_expect_success 'respect configured credentials' '
 	test_config credential.helper "$HELPER" &&
 	check fill <<-\EOF
+	protocol=http
+	host=example.com
 	--
+	protocol=http
+	host=example.com
 	username=foo
 	password=bar
 	--
@@ -291,35 +366,85 @@ test_expect_success 'http paths can be part of context' '
 
 test_expect_success 'helpers can abort the process' '
 	test_must_fail git \
-		-c credential.helper="!f() { echo quit=1; }; f" \
+		-c credential.helper=quit \
 		-c credential.helper="verbatim foo bar" \
-		credential fill >stdout &&
-	test_must_be_empty stdout
+		credential fill >stdout 2>stderr <<-\EOF &&
+	protocol=http
+	host=example.com
+	EOF
+	test_must_be_empty stdout &&
+	cat >expect <<-\EOF &&
+	quit: get
+	quit: protocol=http
+	quit: host=example.com
+	fatal: credential helper '\''quit'\'' told us to quit
+	EOF
+	test_i18ncmp expect stderr
 '
 
 test_expect_success 'empty helper spec resets helper list' '
 	test_config credential.helper "verbatim file file" &&
 	check fill "" "verbatim cmdline cmdline" <<-\EOF
+	protocol=http
+	host=example.com
 	--
+	protocol=http
+	host=example.com
 	username=cmdline
 	password=cmdline
 	--
 	verbatim: get
+	verbatim: protocol=http
+	verbatim: host=example.com
 	EOF
 '
 
-test_expect_success 'url parser ignores embedded newlines' '
-	check fill <<-EOF
+test_expect_success 'url parser rejects embedded newlines' '
+	test_must_fail git credential fill 2>stderr <<-\EOF &&
 	url=https://one.example.com?%0ahost=two.example.com/
-	--
-	username=askpass-username
-	password=askpass-password
-	--
-	warning: url contains a newline in its host component: https://one.example.com?%0ahost=two.example.com/
-	warning: skipping credential lookup for url: https://one.example.com?%0ahost=two.example.com/
-	askpass: Username:
-	askpass: Password:
 	EOF
+	cat >expect <<-\EOF &&
+	warning: url contains a newline in its host component: https://one.example.com?%0ahost=two.example.com/
+	fatal: credential url cannot be parsed: https://one.example.com?%0ahost=two.example.com/
+	EOF
+	test_i18ncmp expect stderr
+'
+
+test_expect_success 'host-less URLs are parsed as empty host' '
+	check fill "verbatim foo bar" <<-\EOF
+	url=cert:///path/to/cert.pem
+	--
+	protocol=cert
+	host=
+	path=path/to/cert.pem
+	username=foo
+	password=bar
+	--
+	verbatim: get
+	verbatim: protocol=cert
+	verbatim: host=
+	verbatim: path=path/to/cert.pem
+	EOF
+'
+
+test_expect_success 'credential system refuses to work with missing host' '
+	test_must_fail git credential fill 2>stderr <<-\EOF &&
+	protocol=http
+	EOF
+	cat >expect <<-\EOF &&
+	fatal: refusing to work with credential missing host field
+	EOF
+	test_i18ncmp expect stderr
+'
+
+test_expect_success 'credential system refuses to work with missing protocol' '
+	test_must_fail git credential fill 2>stderr <<-\EOF &&
+	host=example.com
+	EOF
+	cat >expect <<-\EOF &&
+	fatal: refusing to work with credential missing protocol field
+	EOF
+	test_i18ncmp expect stderr
 '
 
 test_done

t/t2006-checkout-index-basic.sh

@@ -21,4 +21,50 @@ test_expect_success 'checkout-index -h in broken repository' '
 	test_i18ngrep "[Uu]sage" broken/usage
 '
 
+for mode in 'case' 'utf-8'
+do
+	case "$mode" in
+	case)	dir='A' symlink='a' mode_prereq='CASE_INSENSITIVE_FS' ;;
+	utf-8)
+		dir=$(printf "\141\314\210") symlink=$(printf "\303\244")
+		mode_prereq='UTF8_NFD_TO_NFC' ;;
+	esac
+
+	test_expect_success SYMLINKS,$mode_prereq \
+	"checkout-index with $mode-collision don't write to the wrong place" '
+		git init $mode-collision &&
+		(
+			cd $mode-collision &&
+			mkdir target-dir &&
+
+			empty_obj_hex=$(git hash-object -w --stdin </dev/null) &&
+			symlink_hex=$(printf "%s" "$PWD/target-dir" | git hash-object -w --stdin) &&
+
+			cat >objs <<-EOF &&
+			100644 blob ${empty_obj_hex}	${dir}/x
+			100644 blob ${empty_obj_hex}	${dir}/y
+			100644 blob ${empty_obj_hex}	${dir}/z
+			120000 blob ${symlink_hex}	${symlink}
+			EOF
+
+			git update-index --index-info <objs &&
+
+			# Note: the order is important here to exercise the
+			# case where the file at ${dir} has its type changed by
+			# the time Git tries to check out ${dir}/z.
+			#
+			# Also, we use core.precomposeUnicode=false because we
+			# want Git to treat the UTF-8 paths transparently on
+			# Mac OS, matching what is in the index.
+			#
+			git -c core.precomposeUnicode=false checkout-index -f \
+				${dir}/x ${dir}/y ${symlink} ${dir}/z &&
+
+			# Should not create ${dir}/z at ${symlink}/z
+			test_path_is_missing target-dir/z
+
+		)
+	'
+done
+
 test_done

t/t5550-http-fetch-dumb.sh

@@ -321,11 +321,17 @@ test_expect_success 'git client does not send an empty Accept-Language' '
 '
 
 test_expect_success 'remote-http complains cleanly about malformed urls' '
-	# do not actually issue "list" or other commands, as we do not
-	# want to rely on what curl would actually do with such a broken
-	# URL. This is just about making sure we do not segfault during
-	# initialization.
-	test_must_fail git remote-http http::/example.com/repo.git
+	test_must_fail git remote-http http::/example.com/repo.git 2>stderr &&
+	test_i18ngrep "url has no scheme" stderr
+'
+
+# NEEDSWORK: Writing commands to git-remote-curl can race against the latter
+# erroring out, producing SIGPIPE. Remove "ok=sigpipe" once transport-helper has
+# learned to handle early remote helper failures more cleanly.
+test_expect_success 'remote-http complains cleanly about empty scheme' '
+	test_must_fail ok=sigpipe git ls-remote \
+		http::${HTTPD_URL#http}/dumb/repo.git 2>stderr &&
+	test_i18ngrep "url has no scheme" stderr
 '
 
 test_expect_success 'redirects can be forbidden/allowed' '

t/t7416-submodule-dash-url.sh

@@ -60,6 +60,116 @@ test_expect_success 'trailing backslash is handled correctly' '
 	test_i18ngrep ! "unknown option" err
 '
 
+test_expect_success 'fsck rejects missing URL scheme' '
+	git checkout --orphan missing-scheme &&
+	cat >.gitmodules <<-\EOF &&
+	[submodule "foo"]
+		url = http::one.example.com/foo.git
+	EOF
+	git add .gitmodules &&
+	test_tick &&
+	git commit -m "gitmodules with missing URL scheme" &&
+	test_when_finished "rm -rf dst" &&
+	git init --bare dst &&
+	git -C dst config transfer.fsckObjects true &&
+	test_must_fail git push dst HEAD 2>err &&
+	grep gitmodulesUrl err
+'
+
+test_expect_success 'fsck rejects relative URL resolving to missing scheme' '
+	git checkout --orphan relative-missing-scheme &&
+	cat >.gitmodules <<-\EOF &&
+	[submodule "foo"]
+		url = "..\\../.\\../:one.example.com/foo.git"
+	EOF
+	git add .gitmodules &&
+	test_tick &&
+	git commit -m "gitmodules with relative URL that strips off scheme" &&
+	test_when_finished "rm -rf dst" &&
+	git init --bare dst &&
+	git -C dst config transfer.fsckObjects true &&
+	test_must_fail git push dst HEAD 2>err &&
+	grep gitmodulesUrl err
+'
+
+test_expect_success 'fsck rejects empty URL scheme' '
+	git checkout --orphan empty-scheme &&
+	cat >.gitmodules <<-\EOF &&
+	[submodule "foo"]
+		url = http::://one.example.com/foo.git
+	EOF
+	git add .gitmodules &&
+	test_tick &&
+	git commit -m "gitmodules with empty URL scheme" &&
+	test_when_finished "rm -rf dst" &&
+	git init --bare dst &&
+	git -C dst config transfer.fsckObjects true &&
+	test_must_fail git push dst HEAD 2>err &&
+	grep gitmodulesUrl err
+'
+
+test_expect_success 'fsck rejects relative URL resolving to empty scheme' '
+	git checkout --orphan relative-empty-scheme &&
+	cat >.gitmodules <<-\EOF &&
+	[submodule "foo"]
+		url = ../../../:://one.example.com/foo.git
+	EOF
+	git add .gitmodules &&
+	test_tick &&
+	git commit -m "relative gitmodules URL resolving to empty scheme" &&
+	test_when_finished "rm -rf dst" &&
+	git init --bare dst &&
+	git -C dst config transfer.fsckObjects true &&
+	test_must_fail git push dst HEAD 2>err &&
+	grep gitmodulesUrl err
+'
+
+test_expect_success 'fsck rejects empty hostname' '
+	git checkout --orphan empty-host &&
+	cat >.gitmodules <<-\EOF &&
+	[submodule "foo"]
+		url = http:///one.example.com/foo.git
+	EOF
+	git add .gitmodules &&
+	test_tick &&
+	git commit -m "gitmodules with extra slashes" &&
+	test_when_finished "rm -rf dst" &&
+	git init --bare dst &&
+	git -C dst config transfer.fsckObjects true &&
+	test_must_fail git push dst HEAD 2>err &&
+	grep gitmodulesUrl err
+'
+
+test_expect_success 'fsck rejects relative url that produced empty hostname' '
+	git checkout --orphan messy-relative &&
+	cat >.gitmodules <<-\EOF &&
+	[submodule "foo"]
+		url = ../../..//one.example.com/foo.git
+	EOF
+	git add .gitmodules &&
+	test_tick &&
+	git commit -m "gitmodules abusing relative_path" &&
+	test_when_finished "rm -rf dst" &&
+	git init --bare dst &&
+	git -C dst config transfer.fsckObjects true &&
+	test_must_fail git push dst HEAD 2>err &&
+	grep gitmodulesUrl err
+'
+
+test_expect_success 'fsck permits embedded newline with unrecognized scheme' '
+	git checkout --orphan newscheme &&
+	cat >.gitmodules <<-\EOF &&
+	[submodule "foo"]
+		url = "data://acjbkd%0akajfdickajkd"
+	EOF
+	git add .gitmodules &&
+	git commit -m "gitmodules with unrecognized scheme" &&
+	test_when_finished "rm -rf dst" &&
+	git init --bare dst &&
+	git -C dst config transfer.fsckObjects true &&
+	git push dst HEAD
+'
+
 test_expect_success 'fsck rejects embedded newline in url' '
 	# create an orphan branch to avoid existing .gitmodules objects
 	git checkout --orphan newline &&
@@ -76,4 +186,19 @@ test_expect_success 'fsck rejects embedded newline in url' '
 	grep gitmodulesUrl err
 '
 
+test_expect_success 'fsck rejects embedded newline in relative url' '
+	git checkout --orphan relative-newline &&
+	cat >.gitmodules <<-\EOF &&
+	[submodule "foo"]
+		url = "./%0ahost=two.example.com/foo.git"
+	EOF
+	git add .gitmodules &&
+	git commit -m "relative url with newline" &&
+	test_when_finished "rm -rf dst" &&
+	git init --bare dst &&
+	git -C dst config transfer.fsckObjects true &&
+	test_must_fail git push dst HEAD 2>err &&
+	grep gitmodulesUrl err
+'
+
 test_done

unpack-trees.c

@@ -378,6 +378,9 @@ static int check_updates(struct unpack_trees_options *o)
 
 	progress = get_progress(o);
 
+	/* Start with clean cache to avoid using any possibly outdated info. */
+	invalidate_lstat_cache();
+
 	if (o->update)
 		git_attr_set_direction(GIT_ATTR_CHECKOUT);