Merge branch 'jt/subprocess-handshake' into maint

Code cleanup.

* jt/subprocess-handshake:
  sub-process: refactor handshake to common function
  Documentation: migrate sub-process docs to header
  convert: add "status=delayed" to filter process protocol
  convert: refactor capabilities negotiation
  convert: move multiple file filter error handling to separate function
  convert: put the flags field before the flag itself for consistent style
  t0021: write "OUT <size>" only on success
  t0021: make debug log file name configurable
  t0021: keep filter log files on comparison
This commit is contained in:
Junio C Hamano
2017-08-23 14:33:52 -07:00
14 changed files with 810 additions and 293 deletions

View File

@ -28,7 +28,7 @@ file_size () {
}
filter_git () {
rm -f rot13-filter.log &&
rm -f *.log &&
git "$@"
}
@ -42,10 +42,10 @@ test_cmp_count () {
for FILE in "$expect" "$actual"
do
sort "$FILE" | uniq -c |
sed -e "s/^ *[0-9][0-9]*[ ]*IN: /x IN: /" >"$FILE.tmp" &&
mv "$FILE.tmp" "$FILE" || return
sed -e "s/^ *[0-9][0-9]*[ ]*IN: /x IN: /" >"$FILE.tmp"
done &&
test_cmp "$expect" "$actual"
test_cmp "$expect.tmp" "$actual.tmp" &&
rm "$expect.tmp" "$actual.tmp"
}
# Compare two files but exclude all `clean` invocations because Git can
@ -56,10 +56,10 @@ test_cmp_exclude_clean () {
actual=$2
for FILE in "$expect" "$actual"
do
grep -v "IN: clean" "$FILE" >"$FILE.tmp" &&
mv "$FILE.tmp" "$FILE"
grep -v "IN: clean" "$FILE" >"$FILE.tmp"
done &&
test_cmp "$expect" "$actual"
test_cmp "$expect.tmp" "$actual.tmp" &&
rm "$expect.tmp" "$actual.tmp"
}
# Check that the contents of two files are equal and that their rot13 version
@ -342,7 +342,7 @@ test_expect_success 'diff does not reuse worktree files that need cleaning' '
'
test_expect_success PERL 'required process filter should filter data' '
test_config_global filter.protocol.process "rot13-filter.pl clean smudge" &&
test_config_global filter.protocol.process "rot13-filter.pl debug.log clean smudge" &&
test_config_global filter.protocol.required true &&
rm -rf repo &&
mkdir repo &&
@ -375,7 +375,7 @@ test_expect_success PERL 'required process filter should filter data' '
IN: clean testsubdir/test3 '\''sq'\'',\$x=.r $S3 [OK] -- OUT: $S3 . [OK]
STOP
EOF
test_cmp_count expected.log rot13-filter.log &&
test_cmp_count expected.log debug.log &&
git commit -m "test commit 2" &&
rm -f test2.r "testsubdir/test3 '\''sq'\'',\$x=.r" &&
@ -388,7 +388,7 @@ test_expect_success PERL 'required process filter should filter data' '
IN: smudge testsubdir/test3 '\''sq'\'',\$x=.r $S3 [OK] -- OUT: $S3 . [OK]
STOP
EOF
test_cmp_exclude_clean expected.log rot13-filter.log &&
test_cmp_exclude_clean expected.log debug.log &&
filter_git checkout --quiet --no-progress empty-branch &&
cat >expected.log <<-EOF &&
@ -397,7 +397,7 @@ test_expect_success PERL 'required process filter should filter data' '
IN: clean test.r $S [OK] -- OUT: $S . [OK]
STOP
EOF
test_cmp_exclude_clean expected.log rot13-filter.log &&
test_cmp_exclude_clean expected.log debug.log &&
filter_git checkout --quiet --no-progress master &&
cat >expected.log <<-EOF &&
@ -409,7 +409,7 @@ test_expect_success PERL 'required process filter should filter data' '
IN: smudge testsubdir/test3 '\''sq'\'',\$x=.r $S3 [OK] -- OUT: $S3 . [OK]
STOP
EOF
test_cmp_exclude_clean expected.log rot13-filter.log &&
test_cmp_exclude_clean expected.log debug.log &&
test_cmp_committed_rot13 "$TEST_ROOT/test.o" test.r &&
test_cmp_committed_rot13 "$TEST_ROOT/test2.o" test2.r &&
@ -419,7 +419,7 @@ test_expect_success PERL 'required process filter should filter data' '
test_expect_success PERL 'required process filter takes precedence' '
test_config_global filter.protocol.clean false &&
test_config_global filter.protocol.process "rot13-filter.pl clean" &&
test_config_global filter.protocol.process "rot13-filter.pl debug.log clean" &&
test_config_global filter.protocol.required true &&
rm -rf repo &&
mkdir repo &&
@ -439,12 +439,12 @@ test_expect_success PERL 'required process filter takes precedence' '
IN: clean test.r $S [OK] -- OUT: $S . [OK]
STOP
EOF
test_cmp_count expected.log rot13-filter.log
test_cmp_count expected.log debug.log
)
'
test_expect_success PERL 'required process filter should be used only for "clean" operation only' '
test_config_global filter.protocol.process "rot13-filter.pl clean" &&
test_config_global filter.protocol.process "rot13-filter.pl debug.log clean" &&
rm -rf repo &&
mkdir repo &&
(
@ -462,7 +462,7 @@ test_expect_success PERL 'required process filter should be used only for "clean
IN: clean test.r $S [OK] -- OUT: $S . [OK]
STOP
EOF
test_cmp_count expected.log rot13-filter.log &&
test_cmp_count expected.log debug.log &&
rm test.r &&
@ -474,12 +474,12 @@ test_expect_success PERL 'required process filter should be used only for "clean
init handshake complete
STOP
EOF
test_cmp_exclude_clean expected.log rot13-filter.log
test_cmp_exclude_clean expected.log debug.log
)
'
test_expect_success PERL 'required process filter should process multiple packets' '
test_config_global filter.protocol.process "rot13-filter.pl clean smudge" &&
test_config_global filter.protocol.process "rot13-filter.pl debug.log clean smudge" &&
test_config_global filter.protocol.required true &&
rm -rf repo &&
@ -514,7 +514,7 @@ test_expect_success PERL 'required process filter should process multiple packet
IN: clean 3pkt_2+1.file $(($S*2+1)) [OK] -- OUT: $(($S*2+1)) ... [OK]
STOP
EOF
test_cmp_count expected.log rot13-filter.log &&
test_cmp_count expected.log debug.log &&
rm -f *.file &&
@ -529,7 +529,7 @@ test_expect_success PERL 'required process filter should process multiple packet
IN: smudge 3pkt_2+1.file $(($S*2+1)) [OK] -- OUT: $(($S*2+1)) ... [OK]
STOP
EOF
test_cmp_exclude_clean expected.log rot13-filter.log &&
test_cmp_exclude_clean expected.log debug.log &&
for FILE in *.file
do
@ -539,7 +539,7 @@ test_expect_success PERL 'required process filter should process multiple packet
'
test_expect_success PERL 'required process filter with clean error should fail' '
test_config_global filter.protocol.process "rot13-filter.pl clean smudge" &&
test_config_global filter.protocol.process "rot13-filter.pl debug.log clean smudge" &&
test_config_global filter.protocol.required true &&
rm -rf repo &&
mkdir repo &&
@ -558,7 +558,7 @@ test_expect_success PERL 'required process filter with clean error should fail'
'
test_expect_success PERL 'process filter should restart after unexpected write failure' '
test_config_global filter.protocol.process "rot13-filter.pl clean smudge" &&
test_config_global filter.protocol.process "rot13-filter.pl debug.log clean smudge" &&
rm -rf repo &&
mkdir repo &&
(
@ -579,7 +579,7 @@ test_expect_success PERL 'process filter should restart after unexpected write f
git add . &&
rm -f *.r &&
rm -f rot13-filter.log &&
rm -f debug.log &&
git checkout --quiet --no-progress . 2>git-stderr.log &&
grep "smudge write error at" git-stderr.log &&
@ -588,14 +588,14 @@ test_expect_success PERL 'process filter should restart after unexpected write f
cat >expected.log <<-EOF &&
START
init handshake complete
IN: smudge smudge-write-fail.r $SF [OK] -- OUT: $SF [WRITE FAIL]
IN: smudge smudge-write-fail.r $SF [OK] -- [WRITE FAIL]
START
init handshake complete
IN: smudge test.r $S [OK] -- OUT: $S . [OK]
IN: smudge test2.r $S2 [OK] -- OUT: $S2 . [OK]
STOP
EOF
test_cmp_exclude_clean expected.log rot13-filter.log &&
test_cmp_exclude_clean expected.log debug.log &&
test_cmp_committed_rot13 "$TEST_ROOT/test.o" test.r &&
test_cmp_committed_rot13 "$TEST_ROOT/test2.o" test2.r &&
@ -609,7 +609,7 @@ test_expect_success PERL 'process filter should restart after unexpected write f
'
test_expect_success PERL 'process filter should not be restarted if it signals an error' '
test_config_global filter.protocol.process "rot13-filter.pl clean smudge" &&
test_config_global filter.protocol.process "rot13-filter.pl debug.log clean smudge" &&
rm -rf repo &&
mkdir repo &&
(
@ -634,12 +634,12 @@ test_expect_success PERL 'process filter should not be restarted if it signals a
cat >expected.log <<-EOF &&
START
init handshake complete
IN: smudge error.r $SE [OK] -- OUT: 0 [ERROR]
IN: smudge error.r $SE [OK] -- [ERROR]
IN: smudge test.r $S [OK] -- OUT: $S . [OK]
IN: smudge test2.r $S2 [OK] -- OUT: $S2 . [OK]
STOP
EOF
test_cmp_exclude_clean expected.log rot13-filter.log &&
test_cmp_exclude_clean expected.log debug.log &&
test_cmp_committed_rot13 "$TEST_ROOT/test.o" test.r &&
test_cmp_committed_rot13 "$TEST_ROOT/test2.o" test2.r &&
@ -648,7 +648,7 @@ test_expect_success PERL 'process filter should not be restarted if it signals a
'
test_expect_success PERL 'process filter abort stops processing of all further files' '
test_config_global filter.protocol.process "rot13-filter.pl clean smudge" &&
test_config_global filter.protocol.process "rot13-filter.pl debug.log clean smudge" &&
rm -rf repo &&
mkdir repo &&
(
@ -673,10 +673,10 @@ test_expect_success PERL 'process filter abort stops processing of all further f
cat >expected.log <<-EOF &&
START
init handshake complete
IN: smudge abort.r $SA [OK] -- OUT: 0 [ABORT]
IN: smudge abort.r $SA [OK] -- [ABORT]
STOP
EOF
test_cmp_exclude_clean expected.log rot13-filter.log &&
test_cmp_exclude_clean expected.log debug.log &&
test_cmp "$TEST_ROOT/test.o" test.r &&
test_cmp "$TEST_ROOT/test2.o" test2.r &&
@ -697,8 +697,124 @@ test_expect_success PERL 'invalid process filter must fail (and not hang!)' '
cp "$TEST_ROOT/test.o" test.r &&
test_must_fail git add . 2>git-stderr.log &&
grep "does not support filter protocol version" git-stderr.log
grep "expected git-filter-server" git-stderr.log
)
'
test_expect_success PERL 'delayed checkout in process filter' '
test_config_global filter.a.process "rot13-filter.pl a.log clean smudge delay" &&
test_config_global filter.a.required true &&
test_config_global filter.b.process "rot13-filter.pl b.log clean smudge delay" &&
test_config_global filter.b.required true &&
rm -rf repo &&
mkdir repo &&
(
cd repo &&
git init &&
echo "*.a filter=a" >.gitattributes &&
echo "*.b filter=b" >>.gitattributes &&
cp "$TEST_ROOT/test.o" test.a &&
cp "$TEST_ROOT/test.o" test-delay10.a &&
cp "$TEST_ROOT/test.o" test-delay11.a &&
cp "$TEST_ROOT/test.o" test-delay20.a &&
cp "$TEST_ROOT/test.o" test-delay10.b &&
git add . &&
git commit -m "test commit"
) &&
S=$(file_size "$TEST_ROOT/test.o") &&
cat >a.exp <<-EOF &&
START
init handshake complete
IN: smudge test.a $S [OK] -- OUT: $S . [OK]
IN: smudge test-delay10.a $S [OK] -- [DELAYED]
IN: smudge test-delay11.a $S [OK] -- [DELAYED]
IN: smudge test-delay20.a $S [OK] -- [DELAYED]
IN: list_available_blobs test-delay10.a test-delay11.a [OK]
IN: smudge test-delay10.a 0 [OK] -- OUT: $S . [OK]
IN: smudge test-delay11.a 0 [OK] -- OUT: $S . [OK]
IN: list_available_blobs test-delay20.a [OK]
IN: smudge test-delay20.a 0 [OK] -- OUT: $S . [OK]
IN: list_available_blobs [OK]
STOP
EOF
cat >b.exp <<-EOF &&
START
init handshake complete
IN: smudge test-delay10.b $S [OK] -- [DELAYED]
IN: list_available_blobs test-delay10.b [OK]
IN: smudge test-delay10.b 0 [OK] -- OUT: $S . [OK]
IN: list_available_blobs [OK]
STOP
EOF
rm -rf repo-cloned &&
filter_git clone repo repo-cloned &&
test_cmp_count a.exp repo-cloned/a.log &&
test_cmp_count b.exp repo-cloned/b.log &&
(
cd repo-cloned &&
test_cmp_committed_rot13 "$TEST_ROOT/test.o" test.a &&
test_cmp_committed_rot13 "$TEST_ROOT/test.o" test-delay10.a &&
test_cmp_committed_rot13 "$TEST_ROOT/test.o" test-delay11.a &&
test_cmp_committed_rot13 "$TEST_ROOT/test.o" test-delay20.a &&
test_cmp_committed_rot13 "$TEST_ROOT/test.o" test-delay10.b &&
rm *.a *.b &&
filter_git checkout . &&
test_cmp_count ../a.exp a.log &&
test_cmp_count ../b.exp b.log &&
test_cmp_committed_rot13 "$TEST_ROOT/test.o" test.a &&
test_cmp_committed_rot13 "$TEST_ROOT/test.o" test-delay10.a &&
test_cmp_committed_rot13 "$TEST_ROOT/test.o" test-delay11.a &&
test_cmp_committed_rot13 "$TEST_ROOT/test.o" test-delay20.a &&
test_cmp_committed_rot13 "$TEST_ROOT/test.o" test-delay10.b
)
'
test_expect_success PERL 'missing file in delayed checkout' '
test_config_global filter.bug.process "rot13-filter.pl bug.log clean smudge delay" &&
test_config_global filter.bug.required true &&
rm -rf repo &&
mkdir repo &&
(
cd repo &&
git init &&
echo "*.a filter=bug" >.gitattributes &&
cp "$TEST_ROOT/test.o" missing-delay.a
git add . &&
git commit -m "test commit"
) &&
rm -rf repo-cloned &&
test_must_fail git clone repo repo-cloned 2>git-stderr.log &&
cat git-stderr.log &&
grep "error: .missing-delay\.a. was not filtered properly" git-stderr.log
'
test_expect_success PERL 'invalid file in delayed checkout' '
test_config_global filter.bug.process "rot13-filter.pl bug.log clean smudge delay" &&
test_config_global filter.bug.required true &&
rm -rf repo &&
mkdir repo &&
(
cd repo &&
git init &&
echo "*.a filter=bug" >.gitattributes &&
cp "$TEST_ROOT/test.o" invalid-delay.a &&
cp "$TEST_ROOT/test.o" unfiltered
git add . &&
git commit -m "test commit"
) &&
rm -rf repo-cloned &&
test_must_fail git clone repo repo-cloned 2>git-stderr.log &&
grep "error: external filter .* signaled that .unfiltered. is now available although it has not been delayed earlier" git-stderr.log
'
test_done

View File

@ -2,8 +2,9 @@
# Example implementation for the Git filter protocol version 2
# See Documentation/gitattributes.txt, section "Filter Protocol"
#
# The script takes the list of supported protocol capabilities as
# arguments ("clean", "smudge", etc).
# The first argument defines a debug log file that the script write to.
# All remaining arguments define a list of supported protocol
# capabilities ("clean", "smudge", etc).
#
# This implementation supports special test cases:
# (1) If data with the pathname "clean-write-fail.r" is processed with
@ -17,6 +18,16 @@
# operation then the filter signals that it cannot or does not want
# to process the file and any file after that is processed with the
# same command.
# (5) If data with a pathname that is a key in the DELAY hash is
# requested (e.g. "test-delay10.a") then the filter responds with
# a "delay" status and sets the "requested" field in the DELAY hash.
# The filter will signal the availability of this object after
# "count" (field in DELAY hash) "list_available_blobs" commands.
# (6) If data with the pathname "missing-delay.a" is processed that the
# filter will drop the path from the "list_available_blobs" response.
# (7) If data with the pathname "invalid-delay.a" is processed that the
# filter will add the path "unfiltered" which was not delayed before
# to the "list_available_blobs" response.
#
use strict;
@ -24,9 +35,19 @@ use warnings;
use IO::File;
my $MAX_PACKET_CONTENT_SIZE = 65516;
my $log_file = shift @ARGV;
my @capabilities = @ARGV;
open my $debug, ">>", "rot13-filter.log" or die "cannot open log file: $!";
open my $debug, ">>", $log_file or die "cannot open log file: $!";
my %DELAY = (
'test-delay10.a' => { "requested" => 0, "count" => 1 },
'test-delay11.a' => { "requested" => 0, "count" => 1 },
'test-delay20.a' => { "requested" => 0, "count" => 2 },
'test-delay10.b' => { "requested" => 0, "count" => 1 },
'missing-delay.a' => { "requested" => 0, "count" => 1 },
'invalid-delay.a' => { "requested" => 0, "count" => 1 },
);
sub rot13 {
my $str = shift;
@ -64,7 +85,7 @@ sub packet_bin_read {
sub packet_txt_read {
my ( $res, $buf ) = packet_bin_read();
unless ( $buf =~ s/\n$// ) {
unless ( $buf eq '' or $buf =~ s/\n$// ) {
die "A non-binary line MUST be terminated by an LF.";
}
return ( $res, $buf );
@ -99,6 +120,7 @@ packet_flush();
( packet_txt_read() eq ( 0, "capability=clean" ) ) || die "bad capability";
( packet_txt_read() eq ( 0, "capability=smudge" ) ) || die "bad capability";
( packet_txt_read() eq ( 0, "capability=delay" ) ) || die "bad capability";
( packet_bin_read() eq ( 1, "" ) ) || die "bad capability end";
foreach (@capabilities) {
@ -109,88 +131,142 @@ print $debug "init handshake complete\n";
$debug->flush();
while (1) {
my ($command) = packet_txt_read() =~ /^command=(.+)$/;
my ( $command ) = packet_txt_read() =~ /^command=(.+)$/;
print $debug "IN: $command";
$debug->flush();
my ($pathname) = packet_txt_read() =~ /^pathname=(.+)$/;
print $debug " $pathname";
$debug->flush();
if ( $command eq "list_available_blobs" ) {
# Flush
packet_bin_read();
if ( $pathname eq "" ) {
die "bad pathname '$pathname'";
}
# Flush
packet_bin_read();
my $input = "";
{
binmode(STDIN);
my $buffer;
my $done = 0;
while ( !$done ) {
( $done, $buffer ) = packet_bin_read();
$input .= $buffer;
}
print $debug " " . length($input) . " [OK] -- ";
$debug->flush();
}
my $output;
if ( $pathname eq "error.r" or $pathname eq "abort.r" ) {
$output = "";
}
elsif ( $command eq "clean" and grep( /^clean$/, @capabilities ) ) {
$output = rot13($input);
}
elsif ( $command eq "smudge" and grep( /^smudge$/, @capabilities ) ) {
$output = rot13($input);
}
else {
die "bad command '$command'";
}
print $debug "OUT: " . length($output) . " ";
$debug->flush();
if ( $pathname eq "error.r" ) {
print $debug "[ERROR]\n";
$debug->flush();
packet_txt_write("status=error");
packet_flush();
}
elsif ( $pathname eq "abort.r" ) {
print $debug "[ABORT]\n";
$debug->flush();
packet_txt_write("status=abort");
packet_flush();
}
else {
packet_txt_write("status=success");
packet_flush();
if ( $pathname eq "${command}-write-fail.r" ) {
print $debug "[WRITE FAIL]\n";
$debug->flush();
die "${command} write error";
}
while ( length($output) > 0 ) {
my $packet = substr( $output, 0, $MAX_PACKET_CONTENT_SIZE );
packet_bin_write($packet);
# dots represent the number of packets
print $debug ".";
if ( length($output) > $MAX_PACKET_CONTENT_SIZE ) {
$output = substr( $output, $MAX_PACKET_CONTENT_SIZE );
}
else {
$output = "";
foreach my $pathname ( sort keys %DELAY ) {
if ( $DELAY{$pathname}{"requested"} >= 1 ) {
$DELAY{$pathname}{"count"} = $DELAY{$pathname}{"count"} - 1;
if ( $pathname eq "invalid-delay.a" ) {
# Send Git a pathname that was not delayed earlier
packet_txt_write("pathname=unfiltered");
}
if ( $pathname eq "missing-delay.a" ) {
# Do not signal Git that this file is available
} elsif ( $DELAY{$pathname}{"count"} == 0 ) {
print $debug " $pathname";
packet_txt_write("pathname=$pathname");
}
}
}
packet_flush();
print $debug " [OK]\n";
$debug->flush();
packet_txt_write("status=success");
packet_flush();
}
else {
my ( $pathname ) = packet_txt_read() =~ /^pathname=(.+)$/;
print $debug " $pathname";
$debug->flush();
if ( $pathname eq "" ) {
die "bad pathname '$pathname'";
}
# Read until flush
my ( $done, $buffer ) = packet_txt_read();
while ( $buffer ne '' ) {
if ( $buffer eq "can-delay=1" ) {
if ( exists $DELAY{$pathname} and $DELAY{$pathname}{"requested"} == 0 ) {
$DELAY{$pathname}{"requested"} = 1;
}
} else {
die "Unknown message '$buffer'";
}
( $done, $buffer ) = packet_txt_read();
}
my $input = "";
{
binmode(STDIN);
my $buffer;
my $done = 0;
while ( !$done ) {
( $done, $buffer ) = packet_bin_read();
$input .= $buffer;
}
print $debug " " . length($input) . " [OK] -- ";
$debug->flush();
}
my $output;
if ( exists $DELAY{$pathname} and exists $DELAY{$pathname}{"output"} ) {
$output = $DELAY{$pathname}{"output"}
}
elsif ( $pathname eq "error.r" or $pathname eq "abort.r" ) {
$output = "";
}
elsif ( $command eq "clean" and grep( /^clean$/, @capabilities ) ) {
$output = rot13($input);
}
elsif ( $command eq "smudge" and grep( /^smudge$/, @capabilities ) ) {
$output = rot13($input);
}
else {
die "bad command '$command'";
}
if ( $pathname eq "error.r" ) {
print $debug "[ERROR]\n";
$debug->flush();
packet_txt_write("status=error");
packet_flush();
}
elsif ( $pathname eq "abort.r" ) {
print $debug "[ABORT]\n";
$debug->flush();
packet_txt_write("status=abort");
packet_flush();
}
elsif ( $command eq "smudge" and
exists $DELAY{$pathname} and
$DELAY{$pathname}{"requested"} == 1
) {
print $debug "[DELAYED]\n";
$debug->flush();
packet_txt_write("status=delayed");
packet_flush();
$DELAY{$pathname}{"requested"} = 2;
$DELAY{$pathname}{"output"} = $output;
}
else {
packet_txt_write("status=success");
packet_flush();
if ( $pathname eq "${command}-write-fail.r" ) {
print $debug "[WRITE FAIL]\n";
$debug->flush();
die "${command} write error";
}
print $debug "OUT: " . length($output) . " ";
$debug->flush();
while ( length($output) > 0 ) {
my $packet = substr( $output, 0, $MAX_PACKET_CONTENT_SIZE );
packet_bin_write($packet);
# dots represent the number of packets
print $debug ".";
if ( length($output) > $MAX_PACKET_CONTENT_SIZE ) {
$output = substr( $output, $MAX_PACKET_CONTENT_SIZE );
}
else {
$output = "";
}
}
packet_flush();
print $debug " [OK]\n";
$debug->flush();
packet_flush();
}
}
}