From e40622a60b7473b7a5feac4a9330239863e1352d Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Wed, 22 Jan 2025 13:05:44 +0100 Subject: [PATCH 01/11] GIT-VERSION-GEN: simplify computing the dirty marker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GIT-VERSION-GEN script computes the version that Git is being built from. When building from a commit with an unclean worktree it knows to append "-dirty" to that version to indicate that there were custom changes applied and that it isn't the exact same as that commit. The dirtiness check is done manually via git-diff-index(1), which is somewhat puzzling though: we already use git-describe(1) to compute the version, which also knows to compute dirtiness via the "--dirty" flag. But digging back in history explains why: the "-dirty" suffix was added in 31e0b2ca81 (GIT 1.5.4.3, 2008-02-23), and git-describe(1) didn't yet have support for "--dirty" back then. Refactor the script to use git-describe(1). Despite being simpler, it also results in a small speedup: Benchmark 1: git describe --dirty --match "v[0-9]*" Time (mean ± σ): 12.5 ms ± 0.3 ms [User: 6.3 ms, System: 8.8 ms] Range (min … max): 12.0 ms … 13.5 ms 200 runs Benchmark 2: git describe --match "v[0-9]*" HEAD && git update-index -q --refresh && git diff-index --name-only HEAD -- Time (mean ± σ): 17.9 ms ± 1.1 ms [User: 8.8 ms, System: 14.4 ms] Range (min … max): 17.0 ms … 30.6 ms 148 runs Summary git describe --dirty --match "v[0-9]*" ran 1.43 ± 0.09 times faster than git describe --match "v[0-9]*" && git update-index -q --refresh && git diff-index --name-only HEAD -- While the speedup doesn't really matter on Unix-based systems, where filesystem operations are typically fast, they do matter on Windows where the commands take a couple hundred milliseconds. A quick and dirty check on that system shows a speedup from ~800ms to ~400ms. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- GIT-VERSION-GEN | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/GIT-VERSION-GEN b/GIT-VERSION-GEN index f2af817fea..b8b683b933 100755 --- a/GIT-VERSION-GEN +++ b/GIT-VERSION-GEN @@ -39,13 +39,9 @@ then test -d "${GIT_DIR:-.git}" || test -f "$SOURCE_DIR"/.git; } && - VN=$(git -C "$SOURCE_DIR" describe --match "v[0-9]*" HEAD 2>/dev/null) && + VN=$(git -C "$SOURCE_DIR" describe --dirty --match="v[0-9]*" 2>/dev/null) && case "$VN" in *$LF*) (exit 1) ;; - v[0-9]*) - git -C "$SOURCE_DIR" update-index -q --refresh - test -z "$(git -C "$SOURCE_DIR" diff-index --name-only HEAD --)" || - VN="$VN-dirty" ;; esac then VN=$(echo "$VN" | sed -e 's/-/./g'); From f6a2efdc9b24a14f294fdbedfae3df4ce55faa9f Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Wed, 22 Jan 2025 13:05:45 +0100 Subject: [PATCH 02/11] GIT-VERSION-GEN: allow running without input and output files The GIT-VERSION-GEN script requires an input file containing formatting directives to be replaced as well as an output file that will get overwritten in case the file contents have changed. When computing the project version for Meson we don't want to have either though: - We only want to compute the version without anything else, but don't have an input file that would match that exact format. While we could of course introduce a new file just for that usecase, it feels suboptimal to add another file every time we want to have a slightly different format for versioned data. - The computed version needs to be read from stdout so that Meson can wire it up for the project. Extend the script to handle both usecases by recognizing `--format=` as alternative to providing an input path and by writing to stdout in case no output file was given. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- GIT-VERSION-GEN | 44 +++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/GIT-VERSION-GEN b/GIT-VERSION-GEN index b8b683b933..9d201a98fd 100755 --- a/GIT-VERSION-GEN +++ b/GIT-VERSION-GEN @@ -5,21 +5,29 @@ DEF_VER=v2.48.0 LF=' ' -if test "$#" -ne 3 +if test "$#" -lt 2 || test "$#" -gt 3 then - echo >&2 "USAGE: $0 " + echo >&2 "USAGE: $0 (--format=|) []" exit 1 fi SOURCE_DIR="$1" -INPUT="$2" -OUTPUT="$3" -if ! test -f "$INPUT" -then - echo >&2 "Input is not a file: $INPUT" - exit 1 -fi +case "$2" in +--format=*) + INPUT="${2#--format=}" + ;; +*) + if ! test -f "$2" + then + echo >&2 "Input is not a file: $2" + exit 1 + fi + INPUT=$(cat "$2") + ;; +esac + +OUTPUT="$3" # Protect us from reading Git version information outside of the Git directory # in case it is not a repository itself, but embedded in an unrelated @@ -74,19 +82,25 @@ read GIT_MAJOR_VERSION GIT_MINOR_VERSION GIT_MICRO_VERSION GIT_PATCH_LEVEL trail $(echo "$GIT_VERSION" 0 0 0 0 | tr '.a-zA-Z-' ' ') EOF -sed -e "s|@GIT_VERSION@|$GIT_VERSION|" \ +REPLACED=$(printf "%s" "$INPUT" | sed -e "s|@GIT_VERSION@|$GIT_VERSION|" \ -e "s|@GIT_MAJOR_VERSION@|$GIT_MAJOR_VERSION|" \ -e "s|@GIT_MINOR_VERSION@|$GIT_MINOR_VERSION|" \ -e "s|@GIT_MICRO_VERSION@|$GIT_MICRO_VERSION|" \ -e "s|@GIT_PATCH_LEVEL@|$GIT_PATCH_LEVEL|" \ -e "s|@GIT_BUILT_FROM_COMMIT@|$GIT_BUILT_FROM_COMMIT|" \ -e "s|@GIT_USER_AGENT@|$GIT_USER_AGENT|" \ - -e "s|@GIT_DATE@|$GIT_DATE|" \ - "$INPUT" >"$OUTPUT".$$+ + -e "s|@GIT_DATE@|$GIT_DATE|" +) -if ! test -f "$OUTPUT" || ! cmp "$OUTPUT".$$+ "$OUTPUT" >/dev/null +if test -z "$OUTPUT" then - mv "$OUTPUT".$$+ "$OUTPUT" + printf "%s\n" "$REPLACED" else - rm "$OUTPUT".$$+ + printf "%s\n" "$REPLACED" >"$OUTPUT".$$+ + if ! test -f "$OUTPUT" || ! cmp "$OUTPUT".$$+ "$OUTPUT" >/dev/null + then + mv "$OUTPUT".$$+ "$OUTPUT" + else + rm "$OUTPUT".$$+ + fi fi From 6ff99174d152f0e1359c30e30ef08ab75b137037 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Wed, 22 Jan 2025 13:05:46 +0100 Subject: [PATCH 03/11] meson: populate project version via GIT-VERSION-GEN The Git version for Meson is currently wired up manually. It can thus grow (and already has grown) stale quite easily, as having multiple sources of truth is never a good idea. This issue is mostly of cosmetic nature as we don't use the project version anywhere, and instead use the GIT-VERSION-GEN script to propagate the correct version into our build. But it is somewhat puzzling when `meson setup` announces to build an old Git release. There are a couple of alternatives for how to solve this: - We can keep the version undefined, but this makes Meson output "undefined" for the version, as well. - We can use GIT-VERSION-GEN to generate the version for us. At the point of configuring the project we haven't yet figured out host details though, and thus we didn't yet set up the shell environment. While not an issue for Unix-based systems, this would be an issue in Windows, where the shell typically gets provided via Git for Windows and thus requires some special setup. - We can pull the default version out of GIT-VERSION-GEN and move it into its own file. This likely requires some adjustments for scripts that bump the version, but allows Meson to read the version from that file trivially. Pick the second option and use GIT-VERSION-GEN as it gives us the most accurate version. In order to fix the bootstrapping issue on Windows systems we simply set the version to 'unknown' in case no shell was found. As the version is only of cosmetic value this isn't really much of an issue. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- meson.build | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/meson.build b/meson.build index 7361eb2eaa..213998986e 100644 --- a/meson.build +++ b/meson.build @@ -170,7 +170,14 @@ project('git', 'c', meson_version: '>=0.61.0', - version: 'v2.47.GIT', + # The version is only of cosmetic nature, so if we cannot find a shell yet we + # simply don't set up a version at all. This may be the case for example on + # Windows systems, where we first have to bootstrap the host environment. + version: find_program('sh', required: false).found() ? run_command( + 'GIT-VERSION-GEN', meson.current_source_dir(), '--format=@GIT_VERSION@', + capture: true, + check: true, + ).stdout().strip() : 'unknown', ) fs = import('fs') From 53d75bd3e4f1e1d6c721fd90f9754a80caa61fd5 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Wed, 22 Jan 2025 13:05:47 +0100 Subject: [PATCH 04/11] meson: fix dependencies for generated headers We generate a couple of headers from our documentation. These headers are added to the libgit sources, but two of them aren't used by the library, but instead by our builtins. This can cause parallel builds to fail because the builtin object may be compiled before the header was generated. Fix the issue by adding both "config-list.h" and "hook-list.h" to the list of builtin sources. While "command-list.h" is generated similarly, it is used by "help.c" and thus part of the libgit sources indeed. Reported-by: Evan Martin Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- meson.build | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/meson.build b/meson.build index 213998986e..4053024dad 100644 --- a/meson.build +++ b/meson.build @@ -487,6 +487,13 @@ libgit_sources = [ 'xdiff/xutils.c', ] +libgit_sources += custom_target( + input: 'command-list.txt', + output: 'command-list.h', + command: [shell, meson.current_source_dir() + '/generate-cmdlist.sh', meson.current_source_dir(), '@OUTPUT@'], + env: script_environment, +) + builtin_sources = [ 'builtin/add.c', 'builtin/am.c', @@ -614,14 +621,7 @@ builtin_sources = [ 'builtin/write-tree.c', ] -libgit_sources += custom_target( - input: 'command-list.txt', - output: 'command-list.h', - command: [shell, meson.current_source_dir() + '/generate-cmdlist.sh', meson.current_source_dir(), '@OUTPUT@'], - env: script_environment, -) - -libgit_sources += custom_target( +builtin_sources += custom_target( output: 'config-list.h', command: [ shell, @@ -632,7 +632,7 @@ libgit_sources += custom_target( env: script_environment, ) -libgit_sources += custom_target( +builtin_sources += custom_target( input: 'Documentation/githooks.txt', output: 'hook-list.h', command: [ From 5d0cf6bb3a126725f70529f205a32b4720f81183 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Wed, 22 Jan 2025 13:05:48 +0100 Subject: [PATCH 05/11] meson: wire up development environments The Meson build system is able to wire up development environments. The intent is to make build artifacts of the project available. This is typically used to export e.g. paths to linkable libraries, which isn't all that interesting in our context given that we don't have an official library interface. But what we can use this mechanism for is to expose the built Git executables as well as the build directory. This allows users to play around with the built Git version in the devenv, and allows them to execute our test scripts directly with the built distribution. Wire up this feature, which can then be used via `meson devenv` in the build directory. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- meson.build | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/meson.build b/meson.build index 4053024dad..ab4f229436 100644 --- a/meson.build +++ b/meson.build @@ -1939,6 +1939,14 @@ configure_file( configuration: build_options_config, ) +# Development environments can be used via `meson devenv -C `. This +# allows you to execute test scripts directly with the built Git version and +# puts the built version of Git in your PATH. +devenv = environment() +devenv.set('GIT_BUILD_DIR', meson.current_build_dir()) +devenv.prepend('PATH', meson.current_build_dir() / 'bin-wrappers') +meson.add_devenv(devenv) + summary({ 'curl': curl.found(), 'expat': expat.found(), From 88d4bff8c376cae3029b7da94a21c4fd4ac0249e Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Wed, 22 Jan 2025 13:05:49 +0100 Subject: [PATCH 06/11] meson: wire up generation of distribution archive Meson knows to generate distribution archives via `meson dist`. In addition to generating the archive itself, this target also knows to compile and execute tests from that archive, which helps to ensure that the result is an adequate drop-in replacement for the versioned project. While this already works as-is, one omission is that we don't propagate the commit that this is built from into the resulting archive. This can be fixed though by adding a distribution script that propagates the version into the "version" file, which GIT-VERSION-GEN knows to read if present. Use GIT-VERSION-GEN to populate that file. As the script is executed in the build directory, not in the directory where we generate the archive, we have to use a shell to resolve the "MESON_DIST_ROOT" environment variable. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- meson.build | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/meson.build b/meson.build index ab4f229436..a59072edf5 100644 --- a/meson.build +++ b/meson.build @@ -1947,6 +1947,19 @@ devenv.set('GIT_BUILD_DIR', meson.current_build_dir()) devenv.prepend('PATH', meson.current_build_dir() / 'bin-wrappers') meson.add_devenv(devenv) +# Generate the 'version' file in the distribution tarball. This is used via +# `meson dist -C ` to populate the source archive with the Git +# version that the archive is being generated from. +meson.add_dist_script( + shell, + '-c', + '"$1" "$2" "$3" --format="@GIT_VERSION@" "$MESON_DIST_ROOT/version"', + 'GIT-VERSION-GEN', + shell, + meson.current_source_dir() / 'GIT-VERSION-GEN', + meson.current_source_dir(), +) + summary({ 'curl': curl.found(), 'expat': expat.found(), From 28911f7dcad1ccc6ac4f6939036de76bb4f4c09b Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Wed, 22 Jan 2025 13:05:50 +0100 Subject: [PATCH 07/11] meson: wire up fuzzers Meson does not yet know to build our fuzzers. Introduce a new build option "fuzzers" and wire up the fuzzers in case it is enabled. Adapt our CI jobs so that they build the fuzzers by default. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- ci/run-build-and-tests.sh | 3 ++- meson.build | 4 ++++ meson_options.txt | 2 ++ oss-fuzz/meson.build | 20 ++++++++++++++++++++ 4 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 oss-fuzz/meson.build diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh index 76667a1277..6c828c3b75 100755 --- a/ci/run-build-and-tests.sh +++ b/ci/run-build-and-tests.sh @@ -53,7 +53,8 @@ case "$jobname" in *-meson) group "Configure" meson setup build . \ --warnlevel 2 --werror \ - --wrap-mode nofallback + --wrap-mode nofallback \ + -Dfuzzers=true group "Build" meson compile -C build -- if test -n "$run_tests" then diff --git a/meson.build b/meson.build index a59072edf5..052bd80ac4 100644 --- a/meson.build +++ b/meson.build @@ -1906,6 +1906,10 @@ if get_option('tests') subdir('t') endif +if get_option('fuzzers') + subdir('oss-fuzz') +endif + subdir('bin-wrappers') if get_option('docs') != [] subdir('Documentation') diff --git a/meson_options.txt b/meson_options.txt index 89b01bad04..34ba679cf9 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -95,3 +95,5 @@ option('tests', type: 'boolean', value: true, description: 'Enable building tests. This requires Perl, but is separate from the "perl" option such that you can build tests without Perl features enabled.') option('test_output_directory', type: 'string', description: 'Path to the directory used to store test outputs') +option('fuzzers', type: 'boolean', value: false, + description: 'Enable building fuzzers.') diff --git a/oss-fuzz/meson.build b/oss-fuzz/meson.build new file mode 100644 index 0000000000..ed79665501 --- /dev/null +++ b/oss-fuzz/meson.build @@ -0,0 +1,20 @@ +fuzz_programs = [ + 'fuzz-commit-graph.c', + 'fuzz-config.c', + 'fuzz-credential-from-url-gently.c', + 'fuzz-date.c', + 'fuzz-pack-headers.c', + 'fuzz-pack-idx.c', + 'fuzz-parse-attr-line.c', + 'fuzz-url-decode-mem.c', +] + +foreach fuzz_program : fuzz_programs + executable(fs.stem(fuzz_program), + sources: [ + 'dummy-cmd-main.c', + fuzz_program, + ], + dependencies: [libgit, common_main], + ) +endforeach From ef8c3a1b8aa04e09a00dffcfda24daec6908615c Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Wed, 22 Jan 2025 13:05:51 +0100 Subject: [PATCH 08/11] meson: make the CSPRNG backend configurable The CSPRNG backend is not configurable in Meson and isn't quite discoverable, either. Make it configurable and add the actual backend used to the summary. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- meson.build | 28 +++++++++++++++++++++------- meson_options.txt | 2 ++ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/meson.build b/meson.build index 052bd80ac4..22dcd4aeee 100644 --- a/meson.build +++ b/meson.build @@ -1332,6 +1332,7 @@ if not meson.is_cross_build() and fs.exists('/dev/tty') libgit_c_args += '-DHAVE_DEV_TTY' endif +csprng_backend = get_option('csprng_backend') https_backend = get_option('https_backend') sha1_backend = get_option('sha1_backend') sha1_unsafe_backend = get_option('sha1_unsafe_backend') @@ -1343,7 +1344,7 @@ if https_backend == 'auto' and security_framework.found() https_backend = 'CommonCrypto' endif -openssl_required = 'openssl' in [https_backend, sha1_backend, sha1_unsafe_backend, sha256_backend] +openssl_required = 'openssl' in [csprng_backend, https_backend, sha1_backend, sha1_unsafe_backend, sha256_backend] openssl = dependency('openssl', required: openssl_required, default_options: ['default_library=static']) if https_backend == 'auto' and openssl.found() https_backend = 'openssl' @@ -1428,18 +1429,30 @@ else error('Unhandled SHA256 backend ' + sha256_backend) endif -if compiler.has_header_symbol('stdlib.h', 'arc4random_buf') +# Backends are ordered to reflect our preference for more secure and faster +# ones over the ones that are less so. +if csprng_backend in ['auto', 'arc4random'] and compiler.has_header_symbol('stdlib.h', 'arc4random_buf', required: csprng_backend == 'arc4random') libgit_c_args += '-DHAVE_ARC4RANDOM' -elif compiler.has_header_symbol('bsd/stdlib.h', 'arc4random_buf') + csprng_backend = 'arc4random' +elif csprng_backend in ['auto', 'arc4random_bsd'] and compiler.has_header_symbol('bsd/stdlib.h', 'arc4random_buf', required: csprng_backend == 'arc4random_bsd') libgit_c_args += '-DHAVE_ARC4RANDOM_BSD' -elif compiler.has_function('getrandom', prefix: '#include ') + csprng_backend = 'arc4random_bsd' +elif csprng_backend in ['auto', 'getrandom'] and compiler.has_header_symbol('sys/random.h', 'getrandom', required: csprng_backend == 'getrandom') libgit_c_args += '-DHAVE_GETRANDOM' -elif compiler.has_function('getentropy', prefix: '#include ') + csprng_backend = 'getrandom' +elif csprng_backend in ['auto', 'getentropy'] and compiler.has_header_symbol('unistd.h', 'getentropy', required: csprng_backend == 'getentropy') libgit_c_args += '-DHAVE_GETENTROPY' -elif compiler.has_function('RtlGenRandom', prefix: '#include \n#include ') + csprng_backend = 'getentropy' +elif csprng_backend in ['auto', 'rtlgenrandom'] and compiler.has_header_symbol('ntsecapi.h', 'RtlGenRandom', prefix: '#include ', required: csprng_backend == 'rtlgenrandom') libgit_c_args += '-DHAVE_RTLGENRANDOM' -elif openssl.found() + csprng_backend = 'rtlgenrandom' +elif csprng_backend in ['auto', 'openssl'] and openssl.found() libgit_c_args += '-DHAVE_OPENSSL_CSPRNG' + csprng_backend = 'openssl' +elif csprng_backend in ['auto', 'urandom'] + csprng_backend = 'urandom' +else + error('Unsupported CSPRNG backend: ' + csprng_backend) endif if get_option('runtime_prefix') @@ -1977,6 +1990,7 @@ summary({ }, section: 'Auto-detected features') summary({ + 'csprng': csprng_backend, 'https': https_backend, 'sha1': sha1_backend, 'sha1_unsafe': sha1_unsafe_backend, diff --git a/meson_options.txt b/meson_options.txt index 34ba679cf9..5429022f30 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -47,6 +47,8 @@ option('regex', type: 'feature', value: 'auto', description: 'Use the system-provided regex library instead of the bundled one.') # Backends. +option('csprng_backend', type: 'combo', value: 'auto', choices: ['auto', 'arc4random', 'arc4random_bsd', 'getrandom', 'getentropy', 'rtlgenrandom', 'openssl', 'urandom'], + description: 'The backend to use for generating cryptographically-secure pseudo-random numbers.') option('https_backend', type: 'combo', value: 'auto', choices: ['auto', 'openssl', 'CommonCrypto', 'none'], description: 'The HTTPS backend to use when connecting to remotes.') option('sha1_backend', type: 'combo', choices: ['openssl', 'block', 'sha1dc', 'CommonCrypto'], value: 'sha1dc', From 13cb20fc4634029762e80d2858e081b0691ff50f Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Wed, 22 Jan 2025 13:05:52 +0100 Subject: [PATCH 09/11] meson: fix compilation with Visual Studio The Visual Studio compiler defaults to C89 unless explicitly asked to use a different version of the C standard. We don't specify any C standard at all though in our Meson build, and consequently compiling Git fails: ...\git\git-compat-util.h(14): fatal error C1189: #error: "Required C99 support is in a test phase. Please see git-compat-util.h for more details." Fix the issue by specifying the project's C standard. Funny enough, specifying C99 does not work because apparently, `__STDC_VERSION__` is not getting defined in that version at all. Instead, we have to specify C11 as the project's C standard, which is also done in our CMake build instructions. We don't want to generally enforce C11 though, as our requiremets only state that a C99 compiler is required. In fact, we don't even require plain C99, but rather the GNU variant thereof. Meson allows us to handle this case rather easily by specifying "gnu99,c11", which will cause it to fall back to C11 in case GNU C99 is unsupported. This feature has only been introduced with Meson 1.3.0 though, and we support 0.61.0 and newer. In case we use such an oldish version though we fall back to requiring GNU99 unconditionally. This means that Windows essentially requires Meson 1.3.0 and newer when using Visual Studio, but I doubt that this is ever going to be a real problem. Tested-by: M Hickford Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- meson.build | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/meson.build b/meson.build index 22dcd4aeee..af7ca042b9 100644 --- a/meson.build +++ b/meson.build @@ -178,6 +178,14 @@ project('git', 'c', capture: true, check: true, ).stdout().strip() : 'unknown', + default_options: [ + # Git requires C99 with GNU extensions, which of course isn't supported by + # MSVC. Funny enough, C99 doesn't work with MSVC either, as it has only + # learned to define __STDC_VERSION__ with C11 and later. We thus require + # GNU C99 and fall back to C11. Meson only learned to handle the fallback + # with version 1.3.0, so on older versions we use GNU C99 unconditionally. + 'c_std=' + (meson.version().version_compare('>=1.3.0') ? 'gnu99,c11' : 'gnu99'), + ], ) fs = import('fs') From a8179952e13af2902d4806d2e2982bcfeb6339eb Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Wed, 22 Jan 2025 13:05:53 +0100 Subject: [PATCH 10/11] ci: raise error when Meson generates warnings Meson prints warnings in several cases, like for example when using a feature supported by the current version of Meson, but not yet supported by the minimum required version as declared by the project. These warnings will not cause the setup to fail by default, which makes it quite easy to miss them. Improve this by passing `--fatal-meson-warnings` to `meson setup` so that our CI jobs will fail on warnings. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- ci/run-build-and-tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh index 6c828c3b75..964322055f 100755 --- a/ci/run-build-and-tests.sh +++ b/ci/run-build-and-tests.sh @@ -52,6 +52,7 @@ esac case "$jobname" in *-meson) group "Configure" meson setup build . \ + --fatal-meson-warnings \ --warnlevel 2 --werror \ --wrap-mode nofallback \ -Dfuzzers=true From 7304bd2bc390aa6cf27f7a2527d7806dd3fad53e Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Wed, 22 Jan 2025 13:05:54 +0100 Subject: [PATCH 11/11] ci: wire up Visual Studio build with Meson Add a new job to GitHub Actions and GitLab CI that builds and tests Meson-based builds with Visual Studio. A couple notes: - While the build job is mandatory, the test job is marked as "manual" on GitLab so that it doesn't run by default. We already have a bunch of Windows-based jobs, and the computational overhead that these cause is simply out of proportion to run the test suite twice. The same isn't true for GitHub as I could not find a way to make a subset of jobs manually triggered. - We disable Perl. This is because we pick up Perl from Git for Windows, which outputs different paths ("/c/" instead of "C:\") than what we expect in our tests. - We don't use the Git for Windows SDK. Instead, the build only depends on Visual Studio, Meson and Git for Windows. All the other dependencies like curl, pcre2 and zlib get pulled in and compiled automatically by Meson and thus do not have to be provided by the system. - We open-code "ci/run-test-slice.sh". This is because we only have direct access to PowerShell, so we manually implement the logic. There is an upstream pull request for the Meson build system [1] to implement test slicing in Meson directly. - We don't process test artifacts for failed CI jobs. This is done to keep down prerequisites to a minimum. All tests are passing. [1]: https://github.com/mesonbuild/meson/pull/14092 Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- .github/workflows/main.yml | 52 ++++++++++++++++++++++++++++++++++++++ .gitlab-ci.yml | 38 ++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 900be9957a..7f55f8b3a9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -248,6 +248,58 @@ jobs: with: name: failed-tests-windows-vs-${{ matrix.nr }} path: ${{env.FAILED_TEST_ARTIFACTS}} + + windows-meson-build: + name: win+Meson build + needs: ci-config + if: needs.ci-config.outputs.enabled == 'yes' + runs-on: windows-latest + concurrency: + group: windows-meson-build-${{ github.ref }} + cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - name: Set up dependencies + shell: pwsh + run: pip install meson ninja + - name: Setup + shell: pwsh + run: meson setup build -Dperl=disabled + - name: Compile + shell: pwsh + run: meson compile -C build + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: windows-meson-artifacts + path: build + windows-meson-test: + name: win+Meson test + runs-on: windows-latest + needs: [ci-config, windows-meson-build] + strategy: + fail-fast: false + matrix: + nr: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + concurrency: + group: windows-meson-test-${{ matrix.nr }}-${{ github.ref }} + cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - name: Set up dependencies + shell: pwsh + run: pip install meson ninja + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: windows-meson-artifacts + path: build + - name: Test + shell: pwsh + run: meson test -C build --list | Select-Object -Skip 1 | Select-String .* | Group-Object -Property { $_.LineNumber % 10 } | Where-Object Name -EQ ${{ matrix.nr }} | ForEach-Object { meson test -C build --no-rebuild --print-errorlogs $_.Group } + regular: name: ${{matrix.vector.jobname}} (${{matrix.vector.pool}}) needs: ci-config diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9254e01583..4976e18a05 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -149,6 +149,44 @@ test:mingw64: - git-sdk/usr/bin/bash.exe -l -c 'ci/print-test-failures.sh' parallel: 10 +.msvc-meson: + tags: + - saas-windows-medium-amd64 + before_script: + - choco install -y git meson ninja openssl + - Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1 + - refreshenv + # The certificate store for Python on Windows is broken and fails to fetch + # certificates, see https://bugs.python.org/issue36011. This seems to + # mostly be an issue with how the GitLab image is set up as it is a + # non-issue on GitHub Actions. Work around the issue by importing + # cetrificates manually. + - Invoke-WebRequest https://curl.haxx.se/ca/cacert.pem -OutFile cacert.pem + - openssl pkcs12 -export -nokeys -in cacert.pem -out certs.pfx -passout "pass:" + - Import-PfxCertificate -CertStoreLocation Cert:\LocalMachine\Root -FilePath certs.pfx + +build:msvc-meson: + extends: .msvc-meson + stage: build + script: + - meson setup build -Dperl=disabled + - meson compile -C build + artifacts: + paths: + - build + +test:msvc-meson: + extends: .msvc-meson + stage: test + when: manual + timeout: 6h + needs: + - job: "build:msvc-meson" + artifacts: true + script: + - meson test -C build --list | Select-Object -Skip 1 | Select-String .* | Group-Object -Property { $_.LineNumber % $Env:CI_NODE_TOTAL + 1 } | Where-Object Name -EQ $Env:CI_NODE_INDEX | ForEach-Object { meson test -C build --no-rebuild --print-errorlogs $_.Group } + parallel: 10 + test:fuzz-smoke-tests: image: ubuntu:latest stage: test