version: 3.3.6

Signed-off-by: Gyuho Lee <gyuhox@gmail.com>

.travis.yml

@@ -6,7 +6,7 @@ sudo: required
 services: docker
 
 go:
-- 1.9.5
+- 1.9.6
 
 notifications:
   on_success: never

auth/jwt_test.go

@@ -16,6 +16,7 @@ package auth
 
 import (
 	"context"
+	"fmt"
 	"testing"
 )
 
@@ -92,3 +93,8 @@ func TestJWTBad(t *testing.T) {
 	}
 	opts["priv-key"] = jwtPrivKey
 }
+
+// testJWTOpts is useful for passing to NewTokenProvider which requires a string.
+func testJWTOpts() string {
+	return fmt.Sprintf("%s,pub-key=%s,priv-key=%s,sign-method=RS256", tokenTypeJWT, jwtPubKey, jwtPrivKey)
+}

auth/nop.go

@@ -0,0 +1,35 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package auth
+
+import (
+	"context"
+)
+
+type tokenNop struct{}
+
+func (t *tokenNop) enable()                         {}
+func (t *tokenNop) disable()                        {}
+func (t *tokenNop) invalidateUser(string)           {}
+func (t *tokenNop) genTokenPrefix() (string, error) { return "", nil }
+func (t *tokenNop) info(ctx context.Context, token string, rev uint64) (*AuthInfo, bool) {
+	return nil, false
+}
+func (t *tokenNop) assign(ctx context.Context, username string, revision uint64) (string, error) {
+	return "", ErrAuthFailed
+}
+func newTokenProviderNop() (*tokenNop, error) {
+	return &tokenNop{}, nil
+}

auth/store.go

@@ -73,6 +73,9 @@ const (
 	rootUser = "root"
 	rootRole = "root"
 
+	tokenTypeSimple = "simple"
+	tokenTypeJWT    = "jwt"
+
 	revBytesLen = 8
 )
 
@@ -1050,11 +1053,15 @@ func NewTokenProvider(tokenOpts string, indexWaiter func(uint64) <-chan struct{}
 	}
 
 	switch tokenType {
-	case "simple":
+	case tokenTypeSimple:
 		plog.Warningf("simple token is not cryptographically signed")
 		return newTokenProviderSimple(indexWaiter), nil
-	case "jwt":
+
+	case tokenTypeJWT:
 		return newTokenProviderJWT(typeSpecificOpts)
+
+	case "":
+		return newTokenProviderNop()
 	default:
 		plog.Errorf("unknown token type: %s", tokenType)
 		return nil, ErrInvalidAuthOpts
@@ -1067,7 +1074,7 @@ func (as *authStore) WithRoot(ctx context.Context) context.Context {
 	}
 
 	var ctxForAssign context.Context
-	if ts := as.tokenProvider.(*tokenSimple); ts != nil {
+	if ts, ok := as.tokenProvider.(*tokenSimple); ok && ts != nil {
 		ctx1 := context.WithValue(ctx, AuthenticateParamIndex{}, uint64(0))
 		prefix, err := ts.genTokenPrefix()
 		if err != nil {

auth/store_test.go

@@ -48,7 +48,7 @@ func TestNewAuthStoreRevision(t *testing.T) {
 	b, tPath := backend.NewDefaultTmpBackend()
 	defer os.Remove(tPath)
 
-	tp, err := NewTokenProvider("simple", dummyIndexWaiter)
+	tp, err := NewTokenProvider(tokenTypeSimple, dummyIndexWaiter)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -76,7 +76,7 @@ func TestNewAuthStoreRevision(t *testing.T) {
 func setupAuthStore(t *testing.T) (store *authStore, teardownfunc func(t *testing.T)) {
 	b, tPath := backend.NewDefaultTmpBackend()
 
-	tp, err := NewTokenProvider("simple", dummyIndexWaiter)
+	tp, err := NewTokenProvider(tokenTypeSimple, dummyIndexWaiter)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -513,7 +513,7 @@ func TestAuthInfoFromCtxRace(t *testing.T) {
 	b, tPath := backend.NewDefaultTmpBackend()
 	defer os.Remove(tPath)
 
-	tp, err := NewTokenProvider("simple", dummyIndexWaiter)
+	tp, err := NewTokenProvider(tokenTypeSimple, dummyIndexWaiter)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -579,7 +579,7 @@ func TestRecoverFromSnapshot(t *testing.T) {
 
 	as.Close()
 
-	tp, err := NewTokenProvider("simple", dummyIndexWaiter)
+	tp, err := NewTokenProvider(tokenTypeSimple, dummyIndexWaiter)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -661,7 +661,7 @@ func TestRolesOrder(t *testing.T) {
 	b, tPath := backend.NewDefaultTmpBackend()
 	defer os.Remove(tPath)
 
-	tp, err := NewTokenProvider("simple", dummyIndexWaiter)
+	tp, err := NewTokenProvider(tokenTypeSimple, dummyIndexWaiter)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -702,12 +702,21 @@ func TestRolesOrder(t *testing.T) {
 	}
 }
 
-// TestAuthInfoFromCtxWithRoot ensures "WithRoot" properly embeds token in the context.
-func TestAuthInfoFromCtxWithRoot(t *testing.T) {
+func TestAuthInfoFromCtxWithRootSimple(t *testing.T) {
+	testAuthInfoFromCtxWithRoot(t, tokenTypeSimple)
+}
+
+func TestAuthInfoFromCtxWithRootJWT(t *testing.T) {
+	opts := testJWTOpts()
+	testAuthInfoFromCtxWithRoot(t, opts)
+}
+
+// testAuthInfoFromCtxWithRoot ensures "WithRoot" properly embeds token in the context.
+func testAuthInfoFromCtxWithRoot(t *testing.T, opts string) {
 	b, tPath := backend.NewDefaultTmpBackend()
 	defer os.Remove(tPath)
 
-	tp, err := NewTokenProvider("simple", dummyIndexWaiter)
+	tp, err := NewTokenProvider(opts, dummyIndexWaiter)
 	if err != nil {
 		t.Fatal(err)
 	}

e2e/ctl_v3_watch_test.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The etcd Authors
+// Copyright 2018 The etcd Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,167 +14,13 @@
 
 package e2e
 
-import (
-	"os"
-	"strings"
-	"testing"
-)
-
-func TestCtlV3Watch(t *testing.T)          { testCtl(t, watchTest) }
-func TestCtlV3WatchNoTLS(t *testing.T)     { testCtl(t, watchTest, withCfg(configNoTLS)) }
-func TestCtlV3WatchClientTLS(t *testing.T) { testCtl(t, watchTest, withCfg(configClientTLS)) }
-func TestCtlV3WatchPeerTLS(t *testing.T)   { testCtl(t, watchTest, withCfg(configPeerTLS)) }
-func TestCtlV3WatchTimeout(t *testing.T)   { testCtl(t, watchTest, withDialTimeout(0)) }
-
-func TestCtlV3WatchInteractive(t *testing.T) {
-	testCtl(t, watchTest, withInteractive())
-}
-func TestCtlV3WatchInteractiveNoTLS(t *testing.T) {
-	testCtl(t, watchTest, withInteractive(), withCfg(configNoTLS))
-}
-func TestCtlV3WatchInteractiveClientTLS(t *testing.T) {
-	testCtl(t, watchTest, withInteractive(), withCfg(configClientTLS))
-}
-func TestCtlV3WatchInteractivePeerTLS(t *testing.T) {
-	testCtl(t, watchTest, withInteractive(), withCfg(configPeerTLS))
-}
+import "strings"
 
 type kvExec struct {
 	key, val   string
 	execOutput string
 }
 
-func watchTest(cx ctlCtx) {
-	tests := []struct {
-		puts     []kv
-		envKey   string
-		envRange string
-		args     []string
-
-		wkv []kvExec
-	}{
-		{ // watch 1 key
-			puts: []kv{{"sample", "value"}},
-			args: []string{"sample", "--rev", "1"},
-			wkv:  []kvExec{{key: "sample", val: "value"}},
-		},
-		{ // watch 1 key with env
-			puts:   []kv{{"sample", "value"}},
-			envKey: "sample",
-			args:   []string{"--rev", "1"},
-			wkv:    []kvExec{{key: "sample", val: "value"}},
-		},
-		{ // watch 1 key with "echo watch event received"
-			puts: []kv{{"sample", "value"}},
-			args: []string{"sample", "--rev", "1", "--", "echo", "watch event received"},
-			wkv:  []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}},
-		},
-		{ // watch 1 key with ${ETCD_WATCH_VALUE}
-			puts: []kv{{"sample", "value"}},
-			args: []string{"sample", "--rev", "1", "--", "env"},
-			wkv:  []kvExec{{key: "sample", val: "value", execOutput: `ETCD_WATCH_VALUE="value"`}},
-		},
-		{ // watch 1 key with "echo watch event received", with env
-			puts:   []kv{{"sample", "value"}},
-			envKey: "sample",
-			args:   []string{"--rev", "1", "--", "echo", "watch event received"},
-			wkv:    []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}},
-		},
-		{ // watch 1 key with "echo watch event received"
-			puts: []kv{{"sample", "value"}},
-			args: []string{"--rev", "1", "sample", "--", "echo", "watch event received"},
-			wkv:  []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}},
-		},
-		{ // watch 1 key with "echo \"Hello World!\""
-			puts: []kv{{"sample", "value"}},
-			args: []string{"--rev", "1", "sample", "--", "echo", "\"Hello World!\""},
-			wkv:  []kvExec{{key: "sample", val: "value", execOutput: "Hello World!"}},
-		},
-		{ // watch 1 key with "echo watch event received"
-			puts: []kv{{"sample", "value"}},
-			args: []string{"sample", "samplx", "--rev", "1", "--", "echo", "watch event received"},
-			wkv:  []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}},
-		},
-		{ // watch 1 key with "echo watch event received"
-			puts:     []kv{{"sample", "value"}},
-			envKey:   "sample",
-			envRange: "samplx",
-			args:     []string{"--rev", "1", "--", "echo", "watch event received"},
-			wkv:      []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}},
-		},
-		{ // watch 1 key with "echo watch event received"
-			puts: []kv{{"sample", "value"}},
-			args: []string{"sample", "--rev", "1", "samplx", "--", "echo", "watch event received"},
-			wkv:  []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}},
-		},
-		{ // watch 3 keys by prefix
-			puts: []kv{{"key1", "val1"}, {"key2", "val2"}, {"key3", "val3"}},
-			args: []string{"key", "--rev", "1", "--prefix"},
-			wkv:  []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}, {key: "key3", val: "val3"}},
-		},
-		{ // watch 3 keys by prefix, with env
-			puts:   []kv{{"key1", "val1"}, {"key2", "val2"}, {"key3", "val3"}},
-			envKey: "key",
-			args:   []string{"--rev", "1", "--prefix"},
-			wkv:    []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}, {key: "key3", val: "val3"}},
-		},
-		{ // watch by revision
-			puts: []kv{{"etcd", "revision_1"}, {"etcd", "revision_2"}, {"etcd", "revision_3"}},
-			args: []string{"etcd", "--rev", "2"},
-			wkv:  []kvExec{{key: "etcd", val: "revision_2"}, {key: "etcd", val: "revision_3"}},
-		},
-		{ // watch 3 keys by range
-			puts: []kv{{"key1", "val1"}, {"key3", "val3"}, {"key2", "val2"}},
-			args: []string{"key", "key3", "--rev", "1"},
-			wkv:  []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}},
-		},
-		{ // watch 3 keys by range, with env
-			puts:     []kv{{"key1", "val1"}, {"key3", "val3"}, {"key2", "val2"}},
-			envKey:   "key",
-			envRange: "key3",
-			args:     []string{"--rev", "1"},
-			wkv:      []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}},
-		},
-	}
-
-	for i, tt := range tests {
-		donec := make(chan struct{})
-		go func(i int, puts []kv) {
-			for j := range puts {
-				if err := ctlV3Put(cx, puts[j].key, puts[j].val, ""); err != nil {
-					cx.t.Fatalf("watchTest #%d-%d: ctlV3Put error (%v)", i, j, err)
-				}
-			}
-			close(donec)
-		}(i, tt.puts)
-
-		unsetEnv := func() {}
-		if tt.envKey != "" || tt.envRange != "" {
-			if tt.envKey != "" {
-				os.Setenv("ETCDCTL_WATCH_KEY", tt.envKey)
-				unsetEnv = func() { os.Unsetenv("ETCDCTL_WATCH_KEY") }
-			}
-			if tt.envRange != "" {
-				os.Setenv("ETCDCTL_WATCH_RANGE_END", tt.envRange)
-				unsetEnv = func() { os.Unsetenv("ETCDCTL_WATCH_RANGE_END") }
-			}
-			if tt.envKey != "" && tt.envRange != "" {
-				unsetEnv = func() {
-					os.Unsetenv("ETCDCTL_WATCH_KEY")
-					os.Unsetenv("ETCDCTL_WATCH_RANGE_END")
-				}
-			}
-		}
-		if err := ctlV3Watch(cx, tt.args, tt.wkv...); err != nil {
-			if cx.dialTimeout > 0 && !isGRPCTimedout(err) {
-				cx.t.Errorf("watchTest #%d: ctlV3Watch error (%v)", i, err)
-			}
-		}
-		unsetEnv()
-		<-donec
-	}
-}
-
 func setupWatchArgs(cx ctlCtx, args []string) []string {
 	cmdArgs := append(cx.PrefixArgs(), "watch")
 	if cx.interactive {

etcdctl/ctlv3/command/watch_command.go

@@ -179,118 +179,159 @@ func printWatchCh(c *clientv3.Client, ch clientv3.WatchChan, execArgs []string)
 // "--" characters are invalid arguments for "spf13/cobra" library,
 // so no need to handle such cases.
 func parseWatchArgs(osArgs, commandArgs []string, envKey, envRange string, interactive bool) (watchArgs []string, execArgs []string, err error) {
-	watchArgs = commandArgs
+	rawArgs := make([]string, len(osArgs))
+	copy(rawArgs, osArgs)
+	watchArgs = make([]string, len(commandArgs))
+	copy(watchArgs, commandArgs)
 
-	// remove preceding commands (e.g. "watch foo bar" in interactive mode)
-	idx := 0
-	for idx = range watchArgs {
-		if watchArgs[idx] == "watch" {
+	// remove preceding commands (e.g. ./bin/etcdctl watch)
+	// handle "./bin/etcdctl watch foo -- echo watch event"
+	for idx := range rawArgs {
+		if rawArgs[idx] == "watch" {
+			rawArgs = rawArgs[idx+1:]
 			break
 		}
 	}
-	if idx < len(watchArgs)-1 || envKey != "" {
-		if idx < len(watchArgs)-1 {
-			watchArgs = watchArgs[idx+1:]
-		}
 
-		execIdx, execExist := 0, false
-		for execIdx = range osArgs {
-			v := osArgs[execIdx]
-			if v == "--" && execIdx != len(osArgs)-1 {
+	// remove preceding commands (e.g. "watch foo bar" in interactive mode)
+	// handle "./bin/etcdctl watch foo -- echo watch event"
+	if interactive {
+		if watchArgs[0] != "watch" {
+			// "watch" not found
+			watchPrefix, watchRev, watchPrevKey = false, 0, false
+			return nil, nil, errBadArgsInteractiveWatch
+		}
+		watchArgs = watchArgs[1:]
+	}
+
+	execIdx, execExist := 0, false
+	if !interactive {
+		for execIdx = range rawArgs {
+			if rawArgs[execIdx] == "--" {
 				execExist = true
 				break
 			}
 		}
-
-		if idx == len(watchArgs)-1 && envKey != "" {
-			if len(watchArgs) > 0 && !interactive {
-				// "watch --rev 1 -- echo Hello World" has no conflict
-				if !execExist {
-					// "watch foo" with ETCDCTL_WATCH_KEY=foo
-					// (watchArgs==["foo"])
-					return nil, nil, errBadArgsNumConflictEnv
-				}
-			}
-			// otherwise, watch with no argument and environment key is set
-			// if interactive, first "watch" command string should be removed
-			if interactive {
-				watchArgs = []string{}
-			}
+		if execExist && execIdx == len(rawArgs)-1 {
+			// "watch foo bar --" should error
+			return nil, nil, errBadArgsNumSeparator
 		}
-
-		// "watch foo -- echo hello" with ETCDCTL_WATCH_KEY=foo
-		// (watchArgs==["foo","echo","hello"])
-		if envKey != "" && execExist {
-			widx, oidx := 0, len(osArgs)-1
-			for widx = len(watchArgs) - 1; widx >= 0; widx-- {
-				if watchArgs[widx] == osArgs[oidx] {
-					oidx--
+		// "watch" with no argument should error
+		if !execExist && len(rawArgs) < 1 && envKey == "" {
+			return nil, nil, errBadArgsNum
+		}
+		if execExist && envKey != "" {
+			// "ETCDCTL_WATCH_KEY=foo watch foo -- echo 1" should error
+			// (watchArgs==["foo","echo","1"])
+			widx, ridx := len(watchArgs)-1, len(rawArgs)-1
+			for ; widx >= 0; widx-- {
+				if watchArgs[widx] == rawArgs[ridx] {
+					ridx--
 					continue
 				}
-				if oidx == execIdx { // watchArgs has extra
+				// watchArgs has extra:
+				// ETCDCTL_WATCH_KEY=foo watch foo  --  echo 1
+				// watchArgs:                       foo echo 1
+				if ridx == execIdx {
 					return nil, nil, errBadArgsNumConflictEnv
 				}
 			}
 		}
-	} else if interactive { // "watch" not found
-		return nil, nil, errBadArgsInteractiveWatch
-	}
-	if len(watchArgs) < 1 && envKey == "" {
-		return nil, nil, errBadArgsNum
-	}
+		// check conflicting arguments
+		// e.g. "watch --rev 1 -- echo Hello World" has no conflict
+		if !execExist && len(watchArgs) > 0 && envKey != "" {
+			// "ETCDCTL_WATCH_KEY=foo watch foo" should error
+			// (watchArgs==["foo"])
+			return nil, nil, errBadArgsNumConflictEnv
+		}
+	} else {
+		for execIdx = range watchArgs {
+			if watchArgs[execIdx] == "--" {
+				execExist = true
+				break
+			}
+		}
+		if execExist && execIdx == len(watchArgs)-1 {
+			// "watch foo bar --" should error
+			watchPrefix, watchRev, watchPrevKey = false, 0, false
+			return nil, nil, errBadArgsNumSeparator
+		}
 
-	// remove preceding commands (e.g. ./bin/etcdctl watch)
-	for idx = range osArgs {
-		if osArgs[idx] == "watch" {
-			break
+		flagset := NewWatchCommand().Flags()
+		if err := flagset.Parse(watchArgs); err != nil {
+			watchPrefix, watchRev, watchPrevKey = false, 0, false
+			return nil, nil, err
+		}
+		pArgs := flagset.Args()
+
+		// "watch" with no argument should error
+		if !execExist && envKey == "" && len(pArgs) < 1 {
+			watchPrefix, watchRev, watchPrevKey = false, 0, false
+			return nil, nil, errBadArgsNum
+		}
+		// check conflicting arguments
+		// e.g. "watch --rev 1 -- echo Hello World" has no conflict
+		if !execExist && len(pArgs) > 0 && envKey != "" {
+			// "ETCDCTL_WATCH_KEY=foo watch foo" should error
+			// (watchArgs==["foo"])
+			watchPrefix, watchRev, watchPrevKey = false, 0, false
+			return nil, nil, errBadArgsNumConflictEnv
 		}
 	}
-	if idx < len(osArgs)-1 {
-		osArgs = osArgs[idx+1:]
-	} else if envKey == "" {
-		return nil, nil, errBadArgsNum
-	}
 
-	argsWithSep := osArgs
-	if interactive { // interactive mode pass "--" to the command args
+	argsWithSep := rawArgs
+	if interactive {
+		// interactive mode directly passes "--" to the command args
 		argsWithSep = watchArgs
 	}
-	foundSep := false
+
+	idx, foundSep := 0, false
 	for idx = range argsWithSep {
 		if argsWithSep[idx] == "--" {
 			foundSep = true
 			break
 		}
 	}
+	if foundSep {
+		execArgs = argsWithSep[idx+1:]
+	}
+
 	if interactive {
 		flagset := NewWatchCommand().Flags()
 		if err := flagset.Parse(argsWithSep); err != nil {
 			return nil, nil, err
 		}
 		watchArgs = flagset.Args()
+
+		watchPrefix, err = flagset.GetBool("prefix")
+		if err != nil {
+			return nil, nil, err
+		}
+		watchRev, err = flagset.GetInt64("rev")
+		if err != nil {
+			return nil, nil, err
+		}
+		watchPrevKey, err = flagset.GetBool("prev-kv")
+		if err != nil {
+			return nil, nil, err
+		}
 	}
 
-	// "watch -- echo hello" with ETCDCTL_WATCH_KEY=foo
-	// should be translated to "watch foo -- echo hello"
+	// "ETCDCTL_WATCH_KEY=foo watch -- echo hello"
+	// should translate "watch foo -- echo hello"
 	// (watchArgs=["echo","hello"] should be ["foo","echo","hello"])
 	if envKey != "" {
-		tmp := []string{envKey}
+		ranges := []string{envKey}
 		if envRange != "" {
-			tmp = append(tmp, envRange)
+			ranges = append(ranges, envRange)
 		}
-		watchArgs = append(tmp, watchArgs...)
+		watchArgs = append(ranges, watchArgs...)
 	}
 
 	if !foundSep {
 		return watchArgs, nil, nil
 	}
 
-	if idx == len(argsWithSep)-1 {
-		// "watch foo bar --" should error
-		return nil, nil, errBadArgsNumSeparator
-	}
-	execArgs = argsWithSep[idx+1:]
-
 	// "watch foo bar --rev 1 -- echo hello" or "watch foo --rev 1 bar -- echo hello",
 	// then "watchArgs" is "foo bar echo hello"
 	// so need ignore args after "argsWithSep[idx]", which is "--"

etcdctl/ctlv3/command/watch_command_test.go

@@ -26,6 +26,10 @@ func Test_parseWatchArgs(t *testing.T) {
 		envKey, envRange string
 		interactive      bool
 
+		interactiveWatchPrefix  bool
+		interactiveWatchRev     int64
+		interactiveWatchPrevKey bool
+
 		watchArgs []string
 		execArgs  []string
 		err       error
@@ -145,6 +149,14 @@ func Test_parseWatchArgs(t *testing.T) {
 			execArgs:    []string{"echo", "Hello", "World"},
 			err:         nil,
 		},
+		{
+			osArgs:      []string{"./bin/etcdctl", "watch", "foo", "--", "echo", "watch", "event", "received"},
+			commandArgs: []string{"foo", "echo", "watch", "event", "received"},
+			interactive: false,
+			watchArgs:   []string{"foo"},
+			execArgs:    []string{"echo", "watch", "event", "received"},
+			err:         nil,
+		},
 		{
 			osArgs:      []string{"./bin/etcdctl", "watch", "foo", "--rev", "1", "--", "echo", "Hello", "World"},
 			commandArgs: []string{"foo", "echo", "Hello", "World"},
@@ -153,6 +165,22 @@ func Test_parseWatchArgs(t *testing.T) {
 			execArgs:    []string{"echo", "Hello", "World"},
 			err:         nil,
 		},
+		{
+			osArgs:      []string{"./bin/etcdctl", "watch", "foo", "--rev", "1", "--", "echo", "watch", "event", "received"},
+			commandArgs: []string{"foo", "echo", "watch", "event", "received"},
+			interactive: false,
+			watchArgs:   []string{"foo"},
+			execArgs:    []string{"echo", "watch", "event", "received"},
+			err:         nil,
+		},
+		{
+			osArgs:      []string{"./bin/etcdctl", "watch", "--rev", "1", "foo", "--", "echo", "watch", "event", "received"},
+			commandArgs: []string{"foo", "echo", "watch", "event", "received"},
+			interactive: false,
+			watchArgs:   []string{"foo"},
+			execArgs:    []string{"echo", "watch", "event", "received"},
+			err:         nil,
+		},
 		{
 			osArgs:      []string{"./bin/etcdctl", "watch", "foo", "bar", "--", "echo", "Hello", "World"},
 			commandArgs: []string{"foo", "bar", "echo", "Hello", "World"},
@@ -185,6 +213,30 @@ func Test_parseWatchArgs(t *testing.T) {
 			execArgs:    []string{"echo", "Hello", "World"},
 			err:         nil,
 		},
+		{
+			osArgs:      []string{"./bin/etcdctl", "watch", "foo", "bar", "--rev", "1", "--", "echo", "watch", "event", "received"},
+			commandArgs: []string{"foo", "bar", "echo", "watch", "event", "received"},
+			interactive: false,
+			watchArgs:   []string{"foo", "bar"},
+			execArgs:    []string{"echo", "watch", "event", "received"},
+			err:         nil,
+		},
+		{
+			osArgs:      []string{"./bin/etcdctl", "watch", "foo", "--rev", "1", "bar", "--", "echo", "Hello", "World"},
+			commandArgs: []string{"foo", "bar", "echo", "Hello", "World"},
+			interactive: false,
+			watchArgs:   []string{"foo", "bar"},
+			execArgs:    []string{"echo", "Hello", "World"},
+			err:         nil,
+		},
+		{
+			osArgs:      []string{"./bin/etcdctl", "watch", "--rev", "1", "foo", "bar", "--", "echo", "Hello", "World"},
+			commandArgs: []string{"foo", "bar", "echo", "Hello", "World"},
+			interactive: false,
+			watchArgs:   []string{"foo", "bar"},
+			execArgs:    []string{"echo", "Hello", "World"},
+			err:         nil,
+		},
 		{
 			osArgs:      []string{"./bin/etcdctl", "watch", "--rev", "1", "--", "echo", "Hello", "World"},
 			commandArgs: []string{"echo", "Hello", "World"},
@@ -215,133 +267,269 @@ func Test_parseWatchArgs(t *testing.T) {
 			err:         errBadArgsNumConflictEnv,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"foo", "bar", "--", "echo", "Hello", "World"},
-			interactive: true,
-			watchArgs:   nil,
-			execArgs:    nil,
-			err:         errBadArgsInteractiveWatch,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"foo", "bar", "--", "echo", "Hello", "World"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     0,
+			interactiveWatchPrevKey: false,
+			watchArgs:               nil,
+			execArgs:                nil,
+			err:                     errBadArgsInteractiveWatch,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "foo"},
-			interactive: true,
-			watchArgs:   []string{"foo"},
-			execArgs:    nil,
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "foo"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     0,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo"},
+			execArgs:                nil,
+			err:                     nil,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "foo", "bar"},
-			interactive: true,
-			watchArgs:   []string{"foo", "bar"},
-			execArgs:    nil,
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "foo", "bar"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     0,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo", "bar"},
+			execArgs:                nil,
+			err:                     nil,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch"},
-			envKey:      "foo",
-			envRange:    "bar",
-			interactive: true,
-			watchArgs:   []string{"foo", "bar"},
-			execArgs:    nil,
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch"},
+			envKey:                  "foo",
+			envRange:                "bar",
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     0,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo", "bar"},
+			execArgs:                nil,
+			err:                     nil,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch"},
-			envKey:      "hello world!",
-			envRange:    "bar",
-			interactive: true,
-			watchArgs:   []string{"hello world!", "bar"},
-			execArgs:    nil,
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch"},
+			envKey:                  "hello world!",
+			envRange:                "bar",
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     0,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"hello world!", "bar"},
+			execArgs:                nil,
+			err:                     nil,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "foo", "--rev", "1"},
-			interactive: true,
-			watchArgs:   []string{"foo"},
-			execArgs:    nil,
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "foo", "--rev", "1"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     1,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo"},
+			execArgs:                nil,
+			err:                     nil,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "foo", "--rev", "1", "--", "echo", "Hello", "World"},
-			interactive: true,
-			watchArgs:   []string{"foo"},
-			execArgs:    []string{"echo", "Hello", "World"},
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "foo", "--rev", "1", "--", "echo", "Hello", "World"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     1,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "--rev", "1", "foo", "--", "echo", "Hello", "World"},
-			interactive: true,
-			watchArgs:   []string{"foo"},
-			execArgs:    []string{"echo", "Hello", "World"},
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "--rev", "1", "foo", "--", "echo", "Hello", "World"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     1,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "--", "echo", "Hello", "World"},
-			envKey:      "foo",
-			interactive: true,
-			watchArgs:   []string{"foo"},
-			execArgs:    []string{"echo", "Hello", "World"},
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "--rev", "5", "--prev-kv", "foo", "--", "echo", "Hello", "World"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     5,
+			interactiveWatchPrevKey: true,
+			watchArgs:               []string{"foo"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "--", "echo", "Hello", "World"},
-			envKey:      "foo",
-			envRange:    "bar",
-			interactive: true,
-			watchArgs:   []string{"foo", "bar"},
-			execArgs:    []string{"echo", "Hello", "World"},
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "--rev", "1"},
+			envKey:                  "foo",
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     1,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo"},
+			execArgs:                nil,
+			err:                     nil,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "foo", "bar", "--", "echo", "Hello", "World"},
-			interactive: true,
-			watchArgs:   []string{"foo", "bar"},
-			execArgs:    []string{"echo", "Hello", "World"},
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "--rev", "1"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     0,
+			interactiveWatchPrevKey: false,
+			watchArgs:               nil,
+			execArgs:                nil,
+			err:                     errBadArgsNum,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "--rev", "1", "foo", "bar", "--", "echo", "Hello", "World"},
-			interactive: true,
-			watchArgs:   []string{"foo", "bar"},
-			execArgs:    []string{"echo", "Hello", "World"},
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "--rev", "1", "--prefix"},
+			envKey:                  "foo",
+			interactive:             true,
+			interactiveWatchPrefix:  true,
+			interactiveWatchRev:     1,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo"},
+			execArgs:                nil,
+			err:                     nil,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "--rev", "1", "--", "echo", "Hello", "World"},
-			envKey:      "foo",
-			envRange:    "bar",
-			interactive: true,
-			watchArgs:   []string{"foo", "bar"},
-			execArgs:    []string{"echo", "Hello", "World"},
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "--rev", "100", "--prefix", "--prev-kv"},
+			envKey:                  "foo",
+			interactive:             true,
+			interactiveWatchPrefix:  true,
+			interactiveWatchRev:     100,
+			interactiveWatchPrevKey: true,
+			watchArgs:               []string{"foo"},
+			execArgs:                nil,
+			err:                     nil,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "foo", "--rev", "1", "bar", "--", "echo", "Hello", "World"},
-			interactive: true,
-			watchArgs:   []string{"foo", "bar"},
-			execArgs:    []string{"echo", "Hello", "World"},
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "--rev", "1", "--prefix"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     0,
+			interactiveWatchPrevKey: false,
+			watchArgs:               nil,
+			execArgs:                nil,
+			err:                     errBadArgsNum,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "foo", "bar", "--rev", "1", "--", "echo", "Hello", "World"},
-			interactive: true,
-			watchArgs:   []string{"foo", "bar"},
-			execArgs:    []string{"echo", "Hello", "World"},
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "--", "echo", "Hello", "World"},
+			envKey:                  "foo",
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     0,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
+		},
+		{
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "--", "echo", "Hello", "World"},
+			envKey:                  "foo",
+			envRange:                "bar",
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     0,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo", "bar"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
+		},
+		{
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "foo", "bar", "--", "echo", "Hello", "World"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     0,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo", "bar"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
+		},
+		{
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "--rev", "1", "foo", "bar", "--", "echo", "Hello", "World"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     1,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo", "bar"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
+		},
+		{
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "--rev", "1", "--", "echo", "Hello", "World"},
+			envKey:                  "foo",
+			envRange:                "bar",
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     1,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo", "bar"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
+		},
+		{
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "foo", "--rev", "1", "bar", "--", "echo", "Hello", "World"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     1,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo", "bar"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
+		},
+		{
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "foo", "bar", "--rev", "1", "--", "echo", "Hello", "World"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     1,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo", "bar"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
+		},
+		{
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "foo", "bar", "--rev", "7", "--prefix", "--", "echo", "Hello", "World"},
+			interactive:             true,
+			interactiveWatchPrefix:  true,
+			interactiveWatchRev:     7,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo", "bar"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
+		},
+		{
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "foo", "bar", "--rev", "7", "--prefix", "--prev-kv", "--", "echo", "Hello", "World"},
+			interactive:             true,
+			interactiveWatchPrefix:  true,
+			interactiveWatchRev:     7,
+			interactiveWatchPrevKey: true,
+			watchArgs:               []string{"foo", "bar"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
 		},
 	}
 	for i, ts := range tt {
@@ -355,5 +543,16 @@ func Test_parseWatchArgs(t *testing.T) {
 		if !reflect.DeepEqual(execArgs, ts.execArgs) {
 			t.Fatalf("#%d: execArgs expected %q, got %v", i, ts.execArgs, execArgs)
 		}
+		if ts.interactive {
+			if ts.interactiveWatchPrefix != watchPrefix {
+				t.Fatalf("#%d: interactive watchPrefix expected %v, got %v", i, ts.interactiveWatchPrefix, watchPrefix)
+			}
+			if ts.interactiveWatchRev != watchRev {
+				t.Fatalf("#%d: interactive watchRev expected %d, got %d", i, ts.interactiveWatchRev, watchRev)
+			}
+			if ts.interactiveWatchPrevKey != watchPrevKey {
+				t.Fatalf("#%d: interactive watchPrevKey expected %v, got %v", i, ts.interactiveWatchPrevKey, watchPrevKey)
+			}
+		}
 	}
 }

etcdmain/help.go

@@ -154,6 +154,8 @@ security flags:
 		enable peer client cert authentication.
 	--peer-trusted-ca-file ''
 		path to the peer server TLS trusted CA file.
+	--peer-cert-allowed-cn ''
+		Required CN for client certs connecting to the peer endpoint.
 	--peer-auto-tls 'false'
 		peer TLS using self-generated certificates if --peer-key-file and --peer-cert-file are not provided.
 	--peer-crl-file ''

functional.yaml

@@ -21,13 +21,13 @@ agent-configs:
     key-file: ""
     trusted-ca-file: ""
     listen-peer-urls: ["https://127.0.0.1:1380"]
-    initial-advertise-peer-urls: ["https://127.0.0.1:13800"]
+    initial-advertise-peer-urls: ["https://127.0.0.1:1381"]
     peer-auto-tls: true
     peer-client-cert-auth: false
     peer-cert-file: ""
     peer-key-file: ""
     peer-trusted-ca-file: ""
-    initial-cluster: s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800
+    initial-cluster: s1=https://127.0.0.1:1381,s2=https://127.0.0.1:2381,s3=https://127.0.0.1:3381
     initial-cluster-state: new
     initial-cluster-token: tkn
     snapshot-count: 10000
@@ -70,13 +70,13 @@ agent-configs:
     key-file: ""
     trusted-ca-file: ""
     listen-peer-urls: ["https://127.0.0.1:2380"]
-    initial-advertise-peer-urls: ["https://127.0.0.1:23800"]
+    initial-advertise-peer-urls: ["https://127.0.0.1:2381"]
     peer-auto-tls: true
     peer-client-cert-auth: false
     peer-cert-file: ""
     peer-key-file: ""
     peer-trusted-ca-file: ""
-    initial-cluster: s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800
+    initial-cluster: s1=https://127.0.0.1:1381,s2=https://127.0.0.1:2381,s3=https://127.0.0.1:3381
     initial-cluster-state: new
     initial-cluster-token: tkn
     snapshot-count: 10000
@@ -119,13 +119,13 @@ agent-configs:
     key-file: ""
     trusted-ca-file: ""
     listen-peer-urls: ["https://127.0.0.1:3380"]
-    initial-advertise-peer-urls: ["https://127.0.0.1:33800"]
+    initial-advertise-peer-urls: ["https://127.0.0.1:3381"]
     peer-auto-tls: true
     peer-client-cert-auth: false
     peer-cert-file: ""
     peer-key-file: ""
     peer-trusted-ca-file: ""
-    initial-cluster: s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800
+    initial-cluster: s1=https://127.0.0.1:1381,s2=https://127.0.0.1:2381,s3=https://127.0.0.1:3381
     initial-cluster-state: new
     initial-cluster-token: tkn
     snapshot-count: 10000

functional/tester/case.go

@@ -274,32 +274,39 @@ func (c *caseUntilSnapshot) Inject(clus *Cluster) error {
 	}
 
 	for i := 0; i < retries; i++ {
-		lastRev, _ = clus.maxRev()
+		lastRev, err = clus.maxRev()
 		// If the number of proposals committed is bigger than snapshot count,
 		// a new snapshot should have been created.
-		dicc := lastRev - startRev
-		if dicc > snapshotCount {
+		diff := lastRev - startRev
+		if diff > snapshotCount {
 			clus.lg.Info(
 				"trigger snapshot PASS",
 				zap.Int("retries", i),
 				zap.String("desc", c.Desc()),
-				zap.Int64("committed-entries", dicc),
+				zap.Int64("committed-entries", diff),
 				zap.Int64("etcd-snapshot-count", snapshotCount),
+				zap.Int64("start-revision", startRev),
 				zap.Int64("last-revision", lastRev),
 				zap.Duration("took", time.Since(now)),
 			)
 			return nil
 		}
 
+		dur := time.Second
+		if diff < 0 || err != nil {
+			dur = 3 * time.Second
+		}
 		clus.lg.Info(
 			"trigger snapshot PROGRESS",
 			zap.Int("retries", i),
-			zap.Int64("committed-entries", dicc),
+			zap.Int64("committed-entries", diff),
 			zap.Int64("etcd-snapshot-count", snapshotCount),
+			zap.Int64("start-revision", startRev),
 			zap.Int64("last-revision", lastRev),
 			zap.Duration("took", time.Since(now)),
+			zap.Error(err),
 		)
-		time.Sleep(time.Second)
+		time.Sleep(dur)
 	}
 
 	return fmt.Errorf("cluster too slow: only %d commits in %d retries", lastRev-startRev, retries)

mvcc/watchable_store.go

@@ -192,6 +192,7 @@ func (s *watchableStore) Restore(b backend.Backend) error {
 	}
 
 	for wa := range s.synced.watchers {
+		wa.restore = true
 		s.unsynced.add(wa)
 	}
 	s.synced = newWatcherGroup()
@@ -482,6 +483,14 @@ type watcher struct {
 	// compacted is set when the watcher is removed because of compaction
 	compacted bool
 
+	// restore is true when the watcher is being restored from leader snapshot
+	// which means that this watcher has just been moved from "synced" to "unsynced"
+	// watcher group, possibly with a future revision when it was first added
+	// to the synced watcher
+	// "unsynced" watcher revision must always be <= current revision,
+	// except when the watcher were to be moved from "synced" watcher group
+	restore bool
+
 	// minRev is the minimum revision update the watcher will accept
 	minRev int64
 	id     WatchID

mvcc/watchable_store_test.go

@@ -338,6 +338,62 @@ func TestWatchRestore(t *testing.T) {
 	t.Run("RunSyncWatchLoopBeforeRestore", test(time.Millisecond*120)) // longer than default waitDuration
 }
 
+// TestWatchRestoreSyncedWatcher tests such a case that:
+//   1. watcher is created with a future revision "math.MaxInt64 - 2"
+//   2. watcher with a future revision is added to "synced" watcher group
+//   3. restore/overwrite storage with snapshot of a higher lasat revision
+//   4. restore operation moves "synced" to "unsynced" watcher group
+//   5. choose the watcher from step 1, without panic
+func TestWatchRestoreSyncedWatcher(t *testing.T) {
+	b1, b1Path := backend.NewDefaultTmpBackend()
+	s1 := newWatchableStore(b1, &lease.FakeLessor{}, nil)
+	defer cleanup(s1, b1, b1Path)
+
+	b2, b2Path := backend.NewDefaultTmpBackend()
+	s2 := newWatchableStore(b2, &lease.FakeLessor{}, nil)
+	defer cleanup(s2, b2, b2Path)
+
+	testKey, testValue := []byte("foo"), []byte("bar")
+	rev := s1.Put(testKey, testValue, lease.NoLease)
+	startRev := rev + 2
+
+	// create a watcher with a future revision
+	// add to "synced" watcher group (startRev > s.store.currentRev)
+	w1 := s1.NewWatchStream()
+	w1.Watch(testKey, nil, startRev)
+
+	// make "s2" ends up with a higher last revision
+	s2.Put(testKey, testValue, lease.NoLease)
+	s2.Put(testKey, testValue, lease.NoLease)
+
+	// overwrite storage with higher revisions
+	if err := s1.Restore(b2); err != nil {
+		t.Fatal(err)
+	}
+
+	// wait for next "syncWatchersLoop" iteration
+	// and the unsynced watcher should be chosen
+	time.Sleep(2 * time.Second)
+
+	// trigger events for "startRev"
+	s1.Put(testKey, testValue, lease.NoLease)
+
+	select {
+	case resp := <-w1.Chan():
+		if resp.Revision != startRev {
+			t.Fatalf("resp.Revision expect %d, got %d", startRev, resp.Revision)
+		}
+		if len(resp.Events) != 1 {
+			t.Fatalf("len(resp.Events) expect 1, got %d", len(resp.Events))
+		}
+		if resp.Events[0].Kv.ModRevision != startRev {
+			t.Fatalf("resp.Events[0].Kv.ModRevision expect %d, got %d", startRev, resp.Events[0].Kv.ModRevision)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("failed to receive event in 1 second")
+	}
+}
+
 // TestWatchBatchUnsynced tests batching on unsynced watchers
 func TestWatchBatchUnsynced(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()

mvcc/watcher_group.go

@@ -15,6 +15,7 @@
 package mvcc
 
 import (
+	"fmt"
 	"math"
 
 	"github.com/coreos/etcd/mvcc/mvccpb"
@@ -238,7 +239,15 @@ func (wg *watcherGroup) chooseAll(curRev, compactRev int64) int64 {
 	minRev := int64(math.MaxInt64)
 	for w := range wg.watchers {
 		if w.minRev > curRev {
-			panic("watcher current revision should not exceed current revision")
+			// after network partition, possibly choosing future revision watcher from restore operation
+			// with watch key "proxy-namespace__lostleader" and revision "math.MaxInt64 - 2"
+			// do not panic when such watcher had been moved from "synced" watcher during restore operation
+			if !w.restore {
+				panic(fmt.Errorf("watcher minimum revision %d should not exceed current revision %d", w.minRev, curRev))
+			}
+
+			// mark 'restore' done, since it's chosen
+			w.restore = false
 		}
 		if w.minRev < compactRev {
 			select {

scripts/release

@@ -16,6 +16,10 @@ help() {
   echo ""
   echo "  args:"
   echo "    version: version of etcd to release, e.g. '3.2.18'"
+  echo "  flags:"
+  echo "    --no-upload: skip gs://etcd binary artifact uploads."
+  echo "    --no-docker-push: skip docker image pushes."
+  echo ""
 }
 
 main() {
@@ -28,6 +32,41 @@ main() {
   MINOR_VERSION=$(echo "${VERSION}" | cut -d. -f 1-2)
   BRANCH="release-${MINOR_VERSION}"
 
+  if ! command -v acbuild >/dev/null; then
+    echo "cannot find acbuild"
+    exit 1
+  fi
+
+  if ! command -v docker >/dev/null; then
+    echo "cannot find docker"
+    exit 1
+  fi
+
+  KEYID=$(gpg --list-keys --with-colons| awk -F: '/^pub:/ { print $5 }')
+  if [[ -z "${KEYID}" ]]; then
+    echo "Failed to load gpg key. Is gpg set up correctly for etcd releases?"
+    exit 1
+  fi
+
+  # Expected umask for etcd release artifacts
+  umask 022
+
+  # Set up release directory.
+  local reldir="/tmp/etcd-release-${VERSION}"
+  if [ ! -d "${reldir}/etcd" ]; then
+    mkdir -p "${reldir}"
+    cd "${reldir}"
+    git clone git@github.com:coreos/etcd.git --branch "${BRANCH}"
+  fi
+  cd "${reldir}/etcd"
+
+  # If a release version tag already exists, use it.
+  local remote_tag_exists=$(git ls-remote origin "refs/tags/${RELEASE_VERSION}" | grep -c "${RELEASE_VERSION}")
+  if [ ${remote_tag_exists} -gt 0 ]; then
+    echo "Release version tag exists on remote. Checking out refs/tags/${RELEASE_VERSION}"
+    git checkout -q "tags/${RELEASE_VERSION}"
+  fi
+
   # Check go version.
   local go_version="go$(yq -r ".go[0]" .travis.yml)"
   local current_go_version=$(go version | awk '{ print $3 }')
@@ -36,85 +75,114 @@ main() {
     exit 1
   fi
 
-  # Make a temp directory
-  cd $(mktemp -d)
-  git clone git@github.com:coreos/etcd.git --branch "${BRANCH}"
-  cd etcd
+  # If the release tag does not already exist remotely, create it.
+  if [ ${remote_tag_exists} -eq 0 ]; then
+    # Bump version/version.go to release version.
+    local source_version=$(egrep "\s+Version\s*=" version/version.go | sed -e "s/.*\"\(.*\)\".*/\1/g")
+    if [[ "${source_version}" != "${VERSION}" ]]; then
+      source_minor_version=$(echo "${source_version}" | cut -d. -f 1-2)
+      if [[ "${source_minor_version}" != "${MINOR_VERSION}" ]]; then
+        echo "Wrong etcd minor version in version/version.go. Expected ${MINOR_VERSION} but got ${source_minor_version}. Aborting."
+        exit 1
+      fi
+      echo "Updating version from ${source_version} to ${VERSION} in version/version.go"
+      sed -i "s/${source_version}/${VERSION}/g" version/version.go
+      echo "Building etcd with updated version"
+      ./build
+    fi
 
-  KEYID=$(gpg --list-keys --with-colons| awk -F: '/^pub:/ { print $5 }')
-
-  if [[ -z "${KEYID}" ]]; then
-    echo "Failed to load gpg key. Is gpg set up correctly for etcd releases?"
-    exit 1
-  fi
-
-  # Bump version/version.go to release version.
-  local source_version=$(egrep "\s+Version\s*=" version/version.go | sed -e "s/.*\"\(.*\)\".*/\1/g")
-  if [[ "${source_version}" != "${VERSION}" ]]; then
-    source_minor_version=$(echo "${source_version}" | cut -d. -f 1-2)
-    if [[ "${source_minor_version}" != "${MINOR_VERSION}" ]]; then
-      echo "Wrong etcd minor version in version/version.go. Expected ${MINOR_VERSION} but got ${source_minor_version}. Aborting."
+    local etcd_version=$(bin/etcd --version | grep "etcd Version" | awk '{ print $3 }')
+    if [[ "${etcd_version}" != "${VERSION}" ]]; then
+      echo "Wrong etcd version in version/version.go. Expected ${etcd_version} but got ${VERSION}. Aborting."
       exit 1
     fi
+
+    if [[ ! -z $(git status -s) ]]; then
+      echo "Committing version/version.go update."
+      git add version/version.go
+      git commit -m "version: bump up to ${VERSION}"
+      git diff --staged
+    fi
+
+    # Push the version change if it's not already been pushed.
+    if [ $(git rev-list --count "origin/${BRANCH}..${BRANCH}") -gt 0 ]; then
+      read -p "Push version bump up to ${VERSION} to github.com/coreos/etcd [y/N]? " confirm
+      [[ "${confirm,,}" == "y" ]] || exit 1
+      git push
+    fi
+
+    # Tag release.
+    if [ $(git tag --list | grep -c "${RELEASE_VERSION}") -gt 0 ]; then
+      echo "Skipping tag step. git tag ${RELEASE_VERSION} already exists."
+    else
+      echo "Tagging release..."
+      git tag --local-user "${KEYID}" --sign "${RELEASE_VERSION}" --message "${RELEASE_VERSION}"
+    fi
+
+    # Push the tag change if it's not already been pushed.
+    read -p "Push etcd ${RELEASE_VERSION} tag [y/N]? " confirm
+    [[ "${confirm,,}" == "y" ]] || exit 1
+    git push origin "tags/${RELEASE_VERSION}"
+  fi
+
+  # Build release.
+  # TODO: check the release directory for all required build artifacts.
+  if [ -d release ]; then
+    echo "Skpping release build step. /release directory already exists."
+  else
+    echo "Building release..."
+    # Check for old and new names of the release build script.
+    # TODO: Move the release script into this on as a function?
+    if [ -f ./scripts/release.sh ]; then
+      ./scripts/release.sh "${RELEASE_VERSION}"
+    else
+      ./scripts/build-release.sh "${RELEASE_VERSION}"
+    fi
+  fi
+
+  # Sanity checks.
+  ./release/etcd-${RELEASE_VERSION}-linux-amd64/etcd --version | grep -q "etcd Version: ${VERSION}"
+  ETCDCTL_API=3 ./release/etcd-${RELEASE_VERSION}-linux-amd64/etcdctl version | grep -q "etcdctl version: ${VERSION}"
+
+  # Upload artifacts.
+  if [ "${NO_UPLOAD}" == 1 ]; then
+    echo "Skipping artifact upload to gs://etcd. --no-upload flat is set."
+  else
+    read -p "Upload etcd ${RELEASE_VERSION} release artifacts to gs://etcd [y/N]? " confirm
+    [[ "${confirm,,}" == "y" ]] || exit 1
+    gsutil -m cp ./release/*.zip gs://etcd/${RELEASE_VERSION}/
+    gsutil -m cp ./release/*.tar.gz gs://etcd/${RELEASE_VERSION}/
+    gsutil -m cp ./release/*.aci gs://etcd/${RELEASE_VERSION}/
+    gsutil -m acl ch -u allUsers:R -r gs://etcd/${RELEASE_VERSION}/
+  fi
+  
+    # Push images.
+  if [ "${NO_DOCKER_PUSH}" == 1 ]; then
+    echo "Skipping docker push. --no-docker-push flat is set."
+  else    
+    read -p "Publish etcd ${RELEASE_VERSION} docker images to quay.io [y/N]? " confirm
+    [[ "${confirm,,}" == "y" ]] || exit 1
+    for i in {1..5}; do
+      docker login quay.io && break
+      echo "login failed, retrying"
+    done
+    gcloud docker -- login -u _json_key -p "$(cat /etc/gcp-key-etcd-development.json)" https://gcr.io
+
+    if [ "${MINOR_VERSION}" != "3.1" ]; then
+      for TARGET_ARCH in "-arm64" "-ppc64le" ""; do
+        docker push quay.io/coreos/etcd:${RELEASE_VERSION}${TARGET_ARCH}
+        gcloud docker -- push gcr.io/etcd-development/etcd:${RELEASE_VERSION}${TARGET_ARCH}
+      done
+    fi
+    gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
+
+    docker tag quay.io/coreos/etcd:${RELEASE_VERSION} quay.io/coreos/etcd:v${MINOR_VERSION}
+    docker push quay.io/coreos/etcd:v${MINOR_VERSION}
     
-    echo "Updating version from ${source_version} to ${VERSION} in version/version.go"
-    sed -i.bak "s/${source_version}/${VERSION}/g" version/version.go
-    echo "Building etcd with updated version"
-    ./build
+    gcloud docker -- tag gcr.io/etcd-development/etcd:${RELEASE_VERSION} gcr.io/etcd-development/etcd:v${MINOR_VERSION}
+    gcloud docker -- push gcr.io/etcd-development/etcd:v${MINOR_VERSION}
   fi
 
-  local etcd_version=$(bin/etcd --version | grep "etcd Version" | awk '{ print $3 }')
-  if [[ "${etcd_version}" != "${VERSION}" ]]; then
-    echo "Wrong etcd version in version/version.go. Expected ${etcd_version} but got ${VERSION}. Aborting."
-    exit 1
-  fi
-
-  git add version/version.go
-  git commit -m "version: bump up to ${VERSION}"
-  git diff --staged
-  read -p "Push version bump up to ${VERSION} to github.com/coreos/etcd [y/N]? " confirm
-  [[ "${confirm,,}" == "y" ]] || exit 1
-  git push
-
-  echo "Tagging release..."
-  git tag --local-user "${KEYID}" --sign "${RELEASE_VERSION}" --message "${RELEASE_VERSION}"
-  git tag --list | grep "{VERSION}"
-  read -p "Push etcd ${RELEASE_VERSION} tag [y/N]? " confirm
-  [[ "${confirm,,}" == "y" ]] || exit 1
-  git push origin "tags/${RELEASE_VERSION}"
-
-  echo "Building release..."
-  ./scripts/release.sh "${RELEASE_VERSION}"
-
-  # TODO: validate output of checks
-  ./release/etcd-${RELEASE_VERSION}-linux-amd64/etcd --version
-  ETCDCTL_API=3 ./release/etcd-${RELEASE_VERSION}-linux-amd64/etcdctl version
-  
-  read -p "Upload etcd ${RELEASE_VERSION} release artifacts to gs://etcd [y/N]? " confirm
-  [[ "${confirm,,}" == "y" ]] || exit 1
-
-  gsutil -m cp ./release/*.zip gs://etcd/${RELEASE_VERSION}/
-  gsutil -m cp ./release/*.tar.gz gs://etcd/${RELEASE_VERSION}/
-  gsutil -m cp ./release/*.aci gs://etcd/${RELEASE_VERSION}/
-  gsutil -m acl ch -u allUsers:R -r gs://etcd/${RELEASE_VERSION}/
-
-  read -p "Publish etcd ${RELEASE_VERSION} docker images to quay.io [y/N]? " confirm
-  [[ "${confirm,,}" == "y" ]] || exit 1
-  docker login quay.io
-  gcloud docker -- login -u _json_key -p "$(cat /etc/gcp-key-etcd-development.json)" https://gcr.io
- 
-  for TARGET_ARCH in "-arm64" "-ppc64le" ""; do
-    docker push quay.io/coreos/etcd:${RELEASE_VERSION}${TARGET_ARCH}
-    gcloud docker -- push gcr.io/etcd-development/etcd:${RELEASE_VERSION}${TARGET_ARCH}
-  done
-  gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
-
-  docker tag quay.io/coreos/etcd:${RELEASE_VERSION} quay.io/coreos/etcd:${MINOR_VERSION}
-  docker push quay.io/coreos/etcd:${MINOR_VERSION}
-  
-  gcloud docker -- tag gcr.io/etcd-development/etcd:${RELEASE_VERSION} gcr.io/etcd-development/etcd:${MINOR_VERSION}
-  gcloud docker -- push gcr.io/etcd-development/etcd:${MINOR_VERSION}
-
   # TODO: test
   # docker run --rm --name etcd-gcr-${RELEASE_VERSION} gcr.io/etcd-development/etcd:${RELEASE_VERSION};
   # docker exec etcd-gcr-${RELEASE_VERSION} /bin/sh -c "/usr/local/bin/etcd --version"
@@ -122,25 +190,34 @@ main() {
   # docker exec etcd-gcr-${RELEASE_VERSION} /bin/sh -c "ETCDCTL_API=3 /usr/local/bin/etcdctl put foo bar"
   # docker exec etcd-gcr-${RELEASE_VERSION} /bin/sh -c "ETCDCTL_API=3 /usr/local/bin/etcdctl get foo"
 
-  echo "Updating version from ${VERSION} to ${VERSION}+git in version/version.go"
-  sed -i.bak "s/${VERSION}/${VERSION}+git/g" version/version.go
-  echo "Building etcd with ${VERSION}+git in version/version.go"
-  git add version/version.go
-  git commit -m "version: bump up to ${VERSION}+git"
-  git diff --staged
-  read -p "Push version bump up to ${VERSION}+git to github.com/coreos/etcd [y/N]? " confirm
-  [[ "${confirm,,}" == "y" ]] || exit 1
-  git push
-
+  # Bump version to next development version.
+  git checkout -q "${BRANCH}" # Since we might be on a checkout of the remote version tag.
+  local source_version=$(egrep "\s+Version\s*=" version/version.go | sed -e "s/.*\"\(.*\)\".*/\1/g")
+  if [[ "${source_version}" != "${VERSION}+git" ]]; then
+    echo "Updating version from ${source_version} to ${VERSION}+git in version/version.go"
+    sed -i "s/${source_version}/${VERSION}+git/g" version/version.go
+    echo "Building etcd with ${VERSION}+git in version/version.go"
+    git add version/version.go
+    git commit -m "version: bump up to ${VERSION}+git"
+    git diff --staged
+    read -p "Push version bump up to ${VERSION}+git to github.com/coreos/etcd [y/N]? " confirm
+    [[ "${confirm,,}" == "y" ]] || exit 1
+    git push
+  fi
+  
   # TODO: signing process
+  echo ""
   echo "WARNING: The release has not been signed and published to github. This must be done manually."
-
   echo "WARNING: version/version.go has not been updated to ${RELEASE_VERSION}+git. This must be done manually."
+  echo ""
   echo "Success."
   exit 0
 }
 
 POSITIONAL=()
+NO_UPLOAD=0
+NO_DOCKER_PUSH=0
+
 while test $# -gt 0; do
         case "$1" in
           -h|--help)
@@ -148,6 +225,14 @@ while test $# -gt 0; do
             help
             exit 0
             ;;
+          --no-upload)
+            NO_UPLOAD=1
+            shift
+            ;;
+          --no-docker-push)
+            NO_DOCKER_PUSH=1
+            shift
+            ;;
           *)
             POSITIONAL+=("$1") # save it in an array for later
             shift # past argument

tests/e2e/ctl_v3_watch_cov_test.go

@@ -0,0 +1,134 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build cov
+
+package e2e
+
+import (
+	"os"
+	"testing"
+)
+
+func TestCtlV3Watch(t *testing.T)          { testCtl(t, watchTest) }
+func TestCtlV3WatchNoTLS(t *testing.T)     { testCtl(t, watchTest, withCfg(configNoTLS)) }
+func TestCtlV3WatchClientTLS(t *testing.T) { testCtl(t, watchTest, withCfg(configClientTLS)) }
+func TestCtlV3WatchPeerTLS(t *testing.T)   { testCtl(t, watchTest, withCfg(configPeerTLS)) }
+func TestCtlV3WatchTimeout(t *testing.T)   { testCtl(t, watchTest, withDialTimeout(0)) }
+
+func TestCtlV3WatchInteractive(t *testing.T) {
+	testCtl(t, watchTest, withInteractive())
+}
+func TestCtlV3WatchInteractiveNoTLS(t *testing.T) {
+	testCtl(t, watchTest, withInteractive(), withCfg(configNoTLS))
+}
+func TestCtlV3WatchInteractiveClientTLS(t *testing.T) {
+	testCtl(t, watchTest, withInteractive(), withCfg(configClientTLS))
+}
+func TestCtlV3WatchInteractivePeerTLS(t *testing.T) {
+	testCtl(t, watchTest, withInteractive(), withCfg(configPeerTLS))
+}
+
+func watchTest(cx ctlCtx) {
+	tests := []struct {
+		puts     []kv
+		envKey   string
+		envRange string
+		args     []string
+
+		wkv []kvExec
+	}{
+		{ // watch 1 key
+			puts: []kv{{"sample", "value"}},
+			args: []string{"sample", "--rev", "1"},
+			wkv:  []kvExec{{key: "sample", val: "value"}},
+		},
+		{ // watch 1 key with env
+			puts:   []kv{{"sample", "value"}},
+			envKey: "sample",
+			args:   []string{"--rev", "1"},
+			wkv:    []kvExec{{key: "sample", val: "value"}},
+		},
+
+		// coverage tests get extra arguments:
+		// ./bin/etcdctl_test -test.coverprofile=e2e.1525392462795198897.coverprofile -test.outputdir=../..
+		// do not test watch exec commands
+
+		{ // watch 3 keys by prefix
+			puts: []kv{{"key1", "val1"}, {"key2", "val2"}, {"key3", "val3"}},
+			args: []string{"key", "--rev", "1", "--prefix"},
+			wkv:  []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}, {key: "key3", val: "val3"}},
+		},
+		{ // watch 3 keys by prefix, with env
+			puts:   []kv{{"key1", "val1"}, {"key2", "val2"}, {"key3", "val3"}},
+			envKey: "key",
+			args:   []string{"--rev", "1", "--prefix"},
+			wkv:    []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}, {key: "key3", val: "val3"}},
+		},
+		{ // watch by revision
+			puts: []kv{{"etcd", "revision_1"}, {"etcd", "revision_2"}, {"etcd", "revision_3"}},
+			args: []string{"etcd", "--rev", "2"},
+			wkv:  []kvExec{{key: "etcd", val: "revision_2"}, {key: "etcd", val: "revision_3"}},
+		},
+		{ // watch 3 keys by range
+			puts: []kv{{"key1", "val1"}, {"key3", "val3"}, {"key2", "val2"}},
+			args: []string{"key", "key3", "--rev", "1"},
+			wkv:  []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}},
+		},
+		{ // watch 3 keys by range, with env
+			puts:     []kv{{"key1", "val1"}, {"key3", "val3"}, {"key2", "val2"}},
+			envKey:   "key",
+			envRange: "key3",
+			args:     []string{"--rev", "1"},
+			wkv:      []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}},
+		},
+	}
+
+	for i, tt := range tests {
+		donec := make(chan struct{})
+		go func(i int, puts []kv) {
+			for j := range puts {
+				if err := ctlV3Put(cx, puts[j].key, puts[j].val, ""); err != nil {
+					cx.t.Fatalf("watchTest #%d-%d: ctlV3Put error (%v)", i, j, err)
+				}
+			}
+			close(donec)
+		}(i, tt.puts)
+
+		unsetEnv := func() {}
+		if tt.envKey != "" || tt.envRange != "" {
+			if tt.envKey != "" {
+				os.Setenv("ETCDCTL_WATCH_KEY", tt.envKey)
+				unsetEnv = func() { os.Unsetenv("ETCDCTL_WATCH_KEY") }
+			}
+			if tt.envRange != "" {
+				os.Setenv("ETCDCTL_WATCH_RANGE_END", tt.envRange)
+				unsetEnv = func() { os.Unsetenv("ETCDCTL_WATCH_RANGE_END") }
+			}
+			if tt.envKey != "" && tt.envRange != "" {
+				unsetEnv = func() {
+					os.Unsetenv("ETCDCTL_WATCH_KEY")
+					os.Unsetenv("ETCDCTL_WATCH_RANGE_END")
+				}
+			}
+		}
+		if err := ctlV3Watch(cx, tt.args, tt.wkv...); err != nil {
+			if cx.dialTimeout > 0 && !isGRPCTimedout(err) {
+				cx.t.Errorf("watchTest #%d: ctlV3Watch error (%v)", i, err)
+			}
+		}
+		unsetEnv()
+		<-donec
+	}
+}

tests/e2e/ctl_v3_watch_no_cov_test.go

@@ -0,0 +1,167 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !cov
+
+package e2e
+
+import (
+	"os"
+	"testing"
+)
+
+func TestCtlV3Watch(t *testing.T)          { testCtl(t, watchTest) }
+func TestCtlV3WatchNoTLS(t *testing.T)     { testCtl(t, watchTest, withCfg(configNoTLS)) }
+func TestCtlV3WatchClientTLS(t *testing.T) { testCtl(t, watchTest, withCfg(configClientTLS)) }
+func TestCtlV3WatchPeerTLS(t *testing.T)   { testCtl(t, watchTest, withCfg(configPeerTLS)) }
+func TestCtlV3WatchTimeout(t *testing.T)   { testCtl(t, watchTest, withDialTimeout(0)) }
+
+func TestCtlV3WatchInteractive(t *testing.T) {
+	testCtl(t, watchTest, withInteractive())
+}
+func TestCtlV3WatchInteractiveNoTLS(t *testing.T) {
+	testCtl(t, watchTest, withInteractive(), withCfg(configNoTLS))
+}
+func TestCtlV3WatchInteractiveClientTLS(t *testing.T) {
+	testCtl(t, watchTest, withInteractive(), withCfg(configClientTLS))
+}
+func TestCtlV3WatchInteractivePeerTLS(t *testing.T) {
+	testCtl(t, watchTest, withInteractive(), withCfg(configPeerTLS))
+}
+
+func watchTest(cx ctlCtx) {
+	tests := []struct {
+		puts     []kv
+		envKey   string
+		envRange string
+		args     []string
+
+		wkv []kvExec
+	}{
+		{ // watch 1 key
+			puts: []kv{{"sample", "value"}},
+			args: []string{"sample", "--rev", "1"},
+			wkv:  []kvExec{{key: "sample", val: "value"}},
+		},
+		{ // watch 1 key with env
+			puts:   []kv{{"sample", "value"}},
+			envKey: "sample",
+			args:   []string{"--rev", "1"},
+			wkv:    []kvExec{{key: "sample", val: "value"}},
+		},
+		{ // watch 1 key with ${ETCD_WATCH_VALUE}
+			puts: []kv{{"sample", "value"}},
+			args: []string{"sample", "--rev", "1", "--", "env"},
+			wkv:  []kvExec{{key: "sample", val: "value", execOutput: `ETCD_WATCH_VALUE="value"`}},
+		},
+		{ // watch 1 key with "echo watch event received", with env
+			puts:   []kv{{"sample", "value"}},
+			envKey: "sample",
+			args:   []string{"--rev", "1", "--", "echo", "watch event received"},
+			wkv:    []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}},
+		},
+		{ // watch 1 key with "echo watch event received"
+			puts: []kv{{"sample", "value"}},
+			args: []string{"--rev", "1", "sample", "--", "echo", "watch event received"},
+			wkv:  []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}},
+		},
+		{ // watch 1 key with "echo \"Hello World!\""
+			puts: []kv{{"sample", "value"}},
+			args: []string{"--rev", "1", "sample", "--", "echo", "\"Hello World!\""},
+			wkv:  []kvExec{{key: "sample", val: "value", execOutput: "Hello World!"}},
+		},
+		{ // watch 1 key with "echo watch event received"
+			puts: []kv{{"sample", "value"}},
+			args: []string{"sample", "samplx", "--rev", "1", "--", "echo", "watch event received"},
+			wkv:  []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}},
+		},
+		{ // watch 1 key with "echo watch event received"
+			puts:     []kv{{"sample", "value"}},
+			envKey:   "sample",
+			envRange: "samplx",
+			args:     []string{"--rev", "1", "--", "echo", "watch event received"},
+			wkv:      []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}},
+		},
+		{ // watch 1 key with "echo watch event received"
+			puts: []kv{{"sample", "value"}},
+			args: []string{"sample", "--rev", "1", "samplx", "--", "echo", "watch event received"},
+			wkv:  []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}},
+		},
+		{ // watch 3 keys by prefix
+			puts: []kv{{"key1", "val1"}, {"key2", "val2"}, {"key3", "val3"}},
+			args: []string{"key", "--rev", "1", "--prefix"},
+			wkv:  []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}, {key: "key3", val: "val3"}},
+		},
+		{ // watch 3 keys by prefix, with env
+			puts:   []kv{{"key1", "val1"}, {"key2", "val2"}, {"key3", "val3"}},
+			envKey: "key",
+			args:   []string{"--rev", "1", "--prefix"},
+			wkv:    []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}, {key: "key3", val: "val3"}},
+		},
+		{ // watch by revision
+			puts: []kv{{"etcd", "revision_1"}, {"etcd", "revision_2"}, {"etcd", "revision_3"}},
+			args: []string{"etcd", "--rev", "2"},
+			wkv:  []kvExec{{key: "etcd", val: "revision_2"}, {key: "etcd", val: "revision_3"}},
+		},
+		{ // watch 3 keys by range
+			puts: []kv{{"key1", "val1"}, {"key3", "val3"}, {"key2", "val2"}},
+			args: []string{"key", "key3", "--rev", "1"},
+			wkv:  []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}},
+		},
+		{ // watch 3 keys by range, with env
+			puts:     []kv{{"key1", "val1"}, {"key3", "val3"}, {"key2", "val2"}},
+			envKey:   "key",
+			envRange: "key3",
+			args:     []string{"--rev", "1"},
+			wkv:      []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}},
+		},
+	}
+
+	for i, tt := range tests {
+		donec := make(chan struct{})
+		go func(i int, puts []kv) {
+			for j := range puts {
+				if err := ctlV3Put(cx, puts[j].key, puts[j].val, ""); err != nil {
+					cx.t.Fatalf("watchTest #%d-%d: ctlV3Put error (%v)", i, j, err)
+				}
+			}
+			close(donec)
+		}(i, tt.puts)
+
+		unsetEnv := func() {}
+		if tt.envKey != "" || tt.envRange != "" {
+			if tt.envKey != "" {
+				os.Setenv("ETCDCTL_WATCH_KEY", tt.envKey)
+				unsetEnv = func() { os.Unsetenv("ETCDCTL_WATCH_KEY") }
+			}
+			if tt.envRange != "" {
+				os.Setenv("ETCDCTL_WATCH_RANGE_END", tt.envRange)
+				unsetEnv = func() { os.Unsetenv("ETCDCTL_WATCH_RANGE_END") }
+			}
+			if tt.envKey != "" && tt.envRange != "" {
+				unsetEnv = func() {
+					os.Unsetenv("ETCDCTL_WATCH_KEY")
+					os.Unsetenv("ETCDCTL_WATCH_RANGE_END")
+				}
+			}
+		}
+		if err := ctlV3Watch(cx, tt.args, tt.wkv...); err != nil {
+			if cx.dialTimeout > 0 && !isGRPCTimedout(err) {
+				cx.t.Errorf("watchTest #%d: ctlV3Watch error (%v)", i, err)
+			}
+		}
+		unsetEnv()
+		<-donec
+	}
+}

tests/semaphore.test.bash

@@ -15,7 +15,7 @@ fi
 docker run \
   --rm \
   --volume=`pwd`:/go/src/github.com/coreos/etcd \
-  gcr.io/etcd-development/etcd-test:go1.9.5 \
+  gcr.io/etcd-development/etcd-test:go1.9.6 \
   /bin/bash -c "${TEST_OPTS} ./test 2>&1 | tee test-${TEST_SUFFIX}.log"
 
 ! egrep "(--- FAIL:|panic: test timed out|appears to have leaked)" -B50 -A10 test-${TEST_SUFFIX}.log

version/version.go

@@ -26,7 +26,7 @@ import (
 var (
 	// MinClusterVersion is the min cluster version this etcd binary is compatible with.
 	MinClusterVersion = "3.0.0"
-	Version           = "3.3.4"
+	Version           = "3.3.6"
 	APIVersion        = "unknown"
 
 	// Git SHA Value will be set during build