version: 3.3.8

Signed-off-by: Gyuho Lee <gyuhox@gmail.com>

.gitignore

@@ -1,6 +1,8 @@
 /agent-*
 /coverage
 /covdir
+/docs
+/vendor
 /gopath
 /gopath.proto
 /go-bindata
@@ -16,4 +18,4 @@
 *.test
 hack/tls-setup/certs
 .idea
-*.bak
+*.bak

.travis.yml

@@ -6,7 +6,7 @@ sudo: required
 services: docker
 
 go:
-- 1.9.5
+- 1.9.7
 
 notifications:
   on_success: never

Makefile

@@ -20,12 +20,16 @@ clean:
 	rm -f ./codecov
 	rm -rf ./agent-*
 	rm -rf ./covdir
+	rm -f ./*.coverprofile
 	rm -f ./*.log
 	rm -f ./bin/Dockerfile-release
 	rm -rf ./bin/*.etcd
+	rm -rf ./default.etcd
+	rm -rf ./tests/e2e/default.etcd
 	rm -rf ./gopath
 	rm -rf ./gopath.proto
 	rm -rf ./release
+	rm -f ./snapshot/localhost:*
 	rm -f ./integration/127.0.0.1:* ./integration/localhost:*
 	rm -f ./clientv3/integration/127.0.0.1:* ./clientv3/integration/localhost:*
 	rm -f ./clientv3/ordering/127.0.0.1:* ./clientv3/ordering/localhost:*
@@ -46,7 +50,8 @@ docker-remove:
 
 
 
-GO_VERSION ?= 1.10.1
+# GO_VERSION ?= 1.10.3
+GO_VERSION ?= 1.9.6
 ETCD_VERSION ?= $(shell git rev-parse --short HEAD || echo "GitNotFound")
 
 TEST_SUFFIX = $(shell date +%s | base64 | head -c 15)
@@ -61,16 +66,16 @@ endif
 
 # Example:
 #   GO_VERSION=1.8.7 make build-docker-test
-#   GO_VERSION=1.9.5 make build-docker-test
+#   GO_VERSION=1.9.7 make build-docker-test
 #   make build-docker-test
 #
 #   gcloud docker -- login -u _json_key -p "$(cat /etc/gcp-key-etcd-development.json)" https://gcr.io
 #   GO_VERSION=1.8.7 make push-docker-test
-#   GO_VERSION=1.9.5 make push-docker-test
+#   GO_VERSION=1.9.7 make push-docker-test
 #   make push-docker-test
 #
 #   gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
-#   GO_VERSION=1.9.5 make pull-docker-test
+#   GO_VERSION=1.9.7 make pull-docker-test
 #   make pull-docker-test
 
 build-docker-test:

auth/jwt_test.go

@@ -16,6 +16,7 @@ package auth
 
 import (
 	"context"
+	"fmt"
 	"testing"
 )
 
@@ -92,3 +93,8 @@ func TestJWTBad(t *testing.T) {
 	}
 	opts["priv-key"] = jwtPrivKey
 }
+
+// testJWTOpts is useful for passing to NewTokenProvider which requires a string.
+func testJWTOpts() string {
+	return fmt.Sprintf("%s,pub-key=%s,priv-key=%s,sign-method=RS256", tokenTypeJWT, jwtPubKey, jwtPrivKey)
+}

auth/nop.go

@@ -0,0 +1,35 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package auth
+
+import (
+	"context"
+)
+
+type tokenNop struct{}
+
+func (t *tokenNop) enable()                         {}
+func (t *tokenNop) disable()                        {}
+func (t *tokenNop) invalidateUser(string)           {}
+func (t *tokenNop) genTokenPrefix() (string, error) { return "", nil }
+func (t *tokenNop) info(ctx context.Context, token string, rev uint64) (*AuthInfo, bool) {
+	return nil, false
+}
+func (t *tokenNop) assign(ctx context.Context, username string, revision uint64) (string, error) {
+	return "", ErrAuthFailed
+}
+func newTokenProviderNop() (*tokenNop, error) {
+	return &tokenNop{}, nil
+}

auth/store.go

@@ -73,6 +73,9 @@ const (
 	rootUser = "root"
 	rootRole = "root"
 
+	tokenTypeSimple = "simple"
+	tokenTypeJWT    = "jwt"
+
 	revBytesLen = 8
 )
 
@@ -1050,11 +1053,15 @@ func NewTokenProvider(tokenOpts string, indexWaiter func(uint64) <-chan struct{}
 	}
 
 	switch tokenType {
-	case "simple":
+	case tokenTypeSimple:
 		plog.Warningf("simple token is not cryptographically signed")
 		return newTokenProviderSimple(indexWaiter), nil
-	case "jwt":
+
+	case tokenTypeJWT:
 		return newTokenProviderJWT(typeSpecificOpts)
+
+	case "":
+		return newTokenProviderNop()
 	default:
 		plog.Errorf("unknown token type: %s", tokenType)
 		return nil, ErrInvalidAuthOpts
@@ -1067,7 +1074,7 @@ func (as *authStore) WithRoot(ctx context.Context) context.Context {
 	}
 
 	var ctxForAssign context.Context
-	if ts := as.tokenProvider.(*tokenSimple); ts != nil {
+	if ts, ok := as.tokenProvider.(*tokenSimple); ok && ts != nil {
 		ctx1 := context.WithValue(ctx, AuthenticateParamIndex{}, uint64(0))
 		prefix, err := ts.genTokenPrefix()
 		if err != nil {

auth/store_test.go

@@ -48,7 +48,7 @@ func TestNewAuthStoreRevision(t *testing.T) {
 	b, tPath := backend.NewDefaultTmpBackend()
 	defer os.Remove(tPath)
 
-	tp, err := NewTokenProvider("simple", dummyIndexWaiter)
+	tp, err := NewTokenProvider(tokenTypeSimple, dummyIndexWaiter)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -76,7 +76,7 @@ func TestNewAuthStoreRevision(t *testing.T) {
 func setupAuthStore(t *testing.T) (store *authStore, teardownfunc func(t *testing.T)) {
 	b, tPath := backend.NewDefaultTmpBackend()
 
-	tp, err := NewTokenProvider("simple", dummyIndexWaiter)
+	tp, err := NewTokenProvider(tokenTypeSimple, dummyIndexWaiter)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -513,7 +513,7 @@ func TestAuthInfoFromCtxRace(t *testing.T) {
 	b, tPath := backend.NewDefaultTmpBackend()
 	defer os.Remove(tPath)
 
-	tp, err := NewTokenProvider("simple", dummyIndexWaiter)
+	tp, err := NewTokenProvider(tokenTypeSimple, dummyIndexWaiter)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -579,7 +579,7 @@ func TestRecoverFromSnapshot(t *testing.T) {
 
 	as.Close()
 
-	tp, err := NewTokenProvider("simple", dummyIndexWaiter)
+	tp, err := NewTokenProvider(tokenTypeSimple, dummyIndexWaiter)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -661,7 +661,7 @@ func TestRolesOrder(t *testing.T) {
 	b, tPath := backend.NewDefaultTmpBackend()
 	defer os.Remove(tPath)
 
-	tp, err := NewTokenProvider("simple", dummyIndexWaiter)
+	tp, err := NewTokenProvider(tokenTypeSimple, dummyIndexWaiter)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -702,12 +702,21 @@ func TestRolesOrder(t *testing.T) {
 	}
 }
 
-// TestAuthInfoFromCtxWithRoot ensures "WithRoot" properly embeds token in the context.
-func TestAuthInfoFromCtxWithRoot(t *testing.T) {
+func TestAuthInfoFromCtxWithRootSimple(t *testing.T) {
+	testAuthInfoFromCtxWithRoot(t, tokenTypeSimple)
+}
+
+func TestAuthInfoFromCtxWithRootJWT(t *testing.T) {
+	opts := testJWTOpts()
+	testAuthInfoFromCtxWithRoot(t, opts)
+}
+
+// testAuthInfoFromCtxWithRoot ensures "WithRoot" properly embeds token in the context.
+func testAuthInfoFromCtxWithRoot(t *testing.T, opts string) {
 	b, tPath := backend.NewDefaultTmpBackend()
 	defer os.Remove(tPath)
 
-	tp, err := NewTokenProvider("simple", dummyIndexWaiter)
+	tp, err := NewTokenProvider(opts, dummyIndexWaiter)
 	if err != nil {
 		t.Fatal(err)
 	}

clientv3/client.go

@@ -529,6 +529,20 @@ func isHaltErr(ctx context.Context, err error) bool {
 	return ev.Code() != codes.Unavailable && ev.Code() != codes.Internal
 }
 
+// isUnavailableErr returns true if the given error is an unavailable error
+func isUnavailableErr(ctx context.Context, err error) bool {
+	if ctx != nil && ctx.Err() != nil {
+		return false
+	}
+	if err == nil {
+		return false
+	}
+	ev, _ := status.FromError(err)
+	// Unavailable codes mean the system will be right back.
+	// (e.g., can't connect, lost leader)
+	return ev.Code() == codes.Unavailable
+}
+
 func toErr(ctx context.Context, err error) error {
 	if err == nil {
 		return nil

clientv3/watch.go

@@ -769,10 +769,13 @@ func (w *watchGrpcStream) joinSubstreams() {
 	}
 }
 
+var maxBackoff = 100 * time.Millisecond
+
 // openWatchClient retries opening a watch client until success or halt.
 // manually retry in case "ws==nil && err==nil"
 // TODO: remove FailFast=false
 func (w *watchGrpcStream) openWatchClient() (ws pb.Watch_WatchClient, err error) {
+	backoff := time.Millisecond
 	for {
 		select {
 		case <-w.ctx.Done():
@@ -788,6 +791,17 @@ func (w *watchGrpcStream) openWatchClient() (ws pb.Watch_WatchClient, err error)
 		if isHaltErr(w.ctx, err) {
 			return nil, v3rpc.Error(err)
 		}
+		if isUnavailableErr(w.ctx, err) {
+			// retry, but backoff
+			if backoff < maxBackoff {
+				// 25% backoff factor
+				backoff = backoff + backoff/4
+				if backoff > maxBackoff {
+					backoff = maxBackoff
+				}
+			}
+			time.Sleep(backoff)
+		}
 	}
 	return ws, nil
 }

e2e/cluster_test.go

@@ -112,6 +112,8 @@ type etcdProcessClusterConfig struct {
 	isClientAutoTLS       bool
 	isClientCRL           bool
 
+	cipherSuites []string
+
 	forceNewCluster     bool
 	initialToken        string
 	quotaBackendBytes   int64
@@ -296,6 +298,10 @@ func (cfg *etcdProcessClusterConfig) tlsArgs() (args []string) {
 		args = append(args, "--client-crl-file", crlPath, "--client-cert-auth")
 	}
 
+	if len(cfg.cipherSuites) > 0 {
+		args = append(args, "--cipher-suites", strings.Join(cfg.cipherSuites, ","))
+	}
+
 	return args
 }
 

e2e/ctl_v3_move_leader_test.go

@@ -16,6 +16,7 @@ package e2e
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"os"
 	"testing"
@@ -23,19 +24,42 @@ import (
 
 	"github.com/coreos/etcd/clientv3"
 	"github.com/coreos/etcd/pkg/testutil"
+	"github.com/coreos/etcd/pkg/transport"
 	"github.com/coreos/etcd/pkg/types"
 )
 
-func TestCtlV3MoveLeader(t *testing.T) {
+func TestCtlV3MoveLeaderSecure(t *testing.T) {
+	testCtlV3MoveLeader(t, configTLS)
+}
+
+func TestCtlV3MoveLeaderInsecure(t *testing.T) {
+	testCtlV3MoveLeader(t, configNoTLS)
+}
+
+func testCtlV3MoveLeader(t *testing.T, cfg etcdProcessClusterConfig) {
 	defer testutil.AfterTest(t)
 
-	epc := setupEtcdctlTest(t, &configNoTLS, true)
+	epc := setupEtcdctlTest(t, &cfg, true)
 	defer func() {
 		if errC := epc.Close(); errC != nil {
 			t.Fatalf("error closing etcd processes (%v)", errC)
 		}
 	}()
 
+	var tcfg *tls.Config
+	if cfg.clientTLS == clientTLS {
+		tinfo := transport.TLSInfo{
+			CertFile:      certPath,
+			KeyFile:       privateKeyPath,
+			TrustedCAFile: caPath,
+		}
+		var err error
+		tcfg, err = tinfo.ClientConfig()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
 	var leadIdx int
 	var leaderID uint64
 	var transferee uint64
@@ -43,6 +67,7 @@ func TestCtlV3MoveLeader(t *testing.T) {
 		cli, err := clientv3.New(clientv3.Config{
 			Endpoints:   []string{ep},
 			DialTimeout: 3 * time.Second,
+			TLS:         tcfg,
 		})
 		if err != nil {
 			t.Fatal(err)

e2e/ctl_v3_watch_test.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The etcd Authors
+// Copyright 2018 The etcd Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,167 +14,13 @@
 
 package e2e
 
-import (
-	"os"
-	"strings"
-	"testing"
-)
-
-func TestCtlV3Watch(t *testing.T)          { testCtl(t, watchTest) }
-func TestCtlV3WatchNoTLS(t *testing.T)     { testCtl(t, watchTest, withCfg(configNoTLS)) }
-func TestCtlV3WatchClientTLS(t *testing.T) { testCtl(t, watchTest, withCfg(configClientTLS)) }
-func TestCtlV3WatchPeerTLS(t *testing.T)   { testCtl(t, watchTest, withCfg(configPeerTLS)) }
-func TestCtlV3WatchTimeout(t *testing.T)   { testCtl(t, watchTest, withDialTimeout(0)) }
-
-func TestCtlV3WatchInteractive(t *testing.T) {
-	testCtl(t, watchTest, withInteractive())
-}
-func TestCtlV3WatchInteractiveNoTLS(t *testing.T) {
-	testCtl(t, watchTest, withInteractive(), withCfg(configNoTLS))
-}
-func TestCtlV3WatchInteractiveClientTLS(t *testing.T) {
-	testCtl(t, watchTest, withInteractive(), withCfg(configClientTLS))
-}
-func TestCtlV3WatchInteractivePeerTLS(t *testing.T) {
-	testCtl(t, watchTest, withInteractive(), withCfg(configPeerTLS))
-}
+import "strings"
 
 type kvExec struct {
 	key, val   string
 	execOutput string
 }
 
-func watchTest(cx ctlCtx) {
-	tests := []struct {
-		puts     []kv
-		envKey   string
-		envRange string
-		args     []string
-
-		wkv []kvExec
-	}{
-		{ // watch 1 key
-			puts: []kv{{"sample", "value"}},
-			args: []string{"sample", "--rev", "1"},
-			wkv:  []kvExec{{key: "sample", val: "value"}},
-		},
-		{ // watch 1 key with env
-			puts:   []kv{{"sample", "value"}},
-			envKey: "sample",
-			args:   []string{"--rev", "1"},
-			wkv:    []kvExec{{key: "sample", val: "value"}},
-		},
-		{ // watch 1 key with "echo watch event received"
-			puts: []kv{{"sample", "value"}},
-			args: []string{"sample", "--rev", "1", "--", "echo", "watch event received"},
-			wkv:  []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}},
-		},
-		{ // watch 1 key with ${ETCD_WATCH_VALUE}
-			puts: []kv{{"sample", "value"}},
-			args: []string{"sample", "--rev", "1", "--", "env"},
-			wkv:  []kvExec{{key: "sample", val: "value", execOutput: `ETCD_WATCH_VALUE="value"`}},
-		},
-		{ // watch 1 key with "echo watch event received", with env
-			puts:   []kv{{"sample", "value"}},
-			envKey: "sample",
-			args:   []string{"--rev", "1", "--", "echo", "watch event received"},
-			wkv:    []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}},
-		},
-		{ // watch 1 key with "echo watch event received"
-			puts: []kv{{"sample", "value"}},
-			args: []string{"--rev", "1", "sample", "--", "echo", "watch event received"},
-			wkv:  []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}},
-		},
-		{ // watch 1 key with "echo \"Hello World!\""
-			puts: []kv{{"sample", "value"}},
-			args: []string{"--rev", "1", "sample", "--", "echo", "\"Hello World!\""},
-			wkv:  []kvExec{{key: "sample", val: "value", execOutput: "Hello World!"}},
-		},
-		{ // watch 1 key with "echo watch event received"
-			puts: []kv{{"sample", "value"}},
-			args: []string{"sample", "samplx", "--rev", "1", "--", "echo", "watch event received"},
-			wkv:  []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}},
-		},
-		{ // watch 1 key with "echo watch event received"
-			puts:     []kv{{"sample", "value"}},
-			envKey:   "sample",
-			envRange: "samplx",
-			args:     []string{"--rev", "1", "--", "echo", "watch event received"},
-			wkv:      []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}},
-		},
-		{ // watch 1 key with "echo watch event received"
-			puts: []kv{{"sample", "value"}},
-			args: []string{"sample", "--rev", "1", "samplx", "--", "echo", "watch event received"},
-			wkv:  []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}},
-		},
-		{ // watch 3 keys by prefix
-			puts: []kv{{"key1", "val1"}, {"key2", "val2"}, {"key3", "val3"}},
-			args: []string{"key", "--rev", "1", "--prefix"},
-			wkv:  []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}, {key: "key3", val: "val3"}},
-		},
-		{ // watch 3 keys by prefix, with env
-			puts:   []kv{{"key1", "val1"}, {"key2", "val2"}, {"key3", "val3"}},
-			envKey: "key",
-			args:   []string{"--rev", "1", "--prefix"},
-			wkv:    []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}, {key: "key3", val: "val3"}},
-		},
-		{ // watch by revision
-			puts: []kv{{"etcd", "revision_1"}, {"etcd", "revision_2"}, {"etcd", "revision_3"}},
-			args: []string{"etcd", "--rev", "2"},
-			wkv:  []kvExec{{key: "etcd", val: "revision_2"}, {key: "etcd", val: "revision_3"}},
-		},
-		{ // watch 3 keys by range
-			puts: []kv{{"key1", "val1"}, {"key3", "val3"}, {"key2", "val2"}},
-			args: []string{"key", "key3", "--rev", "1"},
-			wkv:  []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}},
-		},
-		{ // watch 3 keys by range, with env
-			puts:     []kv{{"key1", "val1"}, {"key3", "val3"}, {"key2", "val2"}},
-			envKey:   "key",
-			envRange: "key3",
-			args:     []string{"--rev", "1"},
-			wkv:      []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}},
-		},
-	}
-
-	for i, tt := range tests {
-		donec := make(chan struct{})
-		go func(i int, puts []kv) {
-			for j := range puts {
-				if err := ctlV3Put(cx, puts[j].key, puts[j].val, ""); err != nil {
-					cx.t.Fatalf("watchTest #%d-%d: ctlV3Put error (%v)", i, j, err)
-				}
-			}
-			close(donec)
-		}(i, tt.puts)
-
-		unsetEnv := func() {}
-		if tt.envKey != "" || tt.envRange != "" {
-			if tt.envKey != "" {
-				os.Setenv("ETCDCTL_WATCH_KEY", tt.envKey)
-				unsetEnv = func() { os.Unsetenv("ETCDCTL_WATCH_KEY") }
-			}
-			if tt.envRange != "" {
-				os.Setenv("ETCDCTL_WATCH_RANGE_END", tt.envRange)
-				unsetEnv = func() { os.Unsetenv("ETCDCTL_WATCH_RANGE_END") }
-			}
-			if tt.envKey != "" && tt.envRange != "" {
-				unsetEnv = func() {
-					os.Unsetenv("ETCDCTL_WATCH_KEY")
-					os.Unsetenv("ETCDCTL_WATCH_RANGE_END")
-				}
-			}
-		}
-		if err := ctlV3Watch(cx, tt.args, tt.wkv...); err != nil {
-			if cx.dialTimeout > 0 && !isGRPCTimedout(err) {
-				cx.t.Errorf("watchTest #%d: ctlV3Watch error (%v)", i, err)
-			}
-		}
-		unsetEnv()
-		<-donec
-	}
-}
-
 func setupWatchArgs(cx ctlCtx, args []string) []string {
 	cmdArgs := append(cx.PrefixArgs(), "watch")
 	if cx.interactive {

e2e/v2_curl_test.go

@@ -127,6 +127,8 @@ type cURLReq struct {
 	header   string
 
 	metricsURLScheme string
+
+	ciphers string
 }
 
 // cURLPrefixArgs builds the beginning of a curl command for a given key
@@ -165,6 +167,10 @@ func cURLPrefixArgs(clus *etcdProcessCluster, method string, req cURLReq) []stri
 		cmdArgs = append(cmdArgs, "-H", req.header)
 	}
 
+	if req.ciphers != "" {
+		cmdArgs = append(cmdArgs, "--ciphers", req.ciphers)
+	}
+
 	switch method {
 	case "POST", "PUT":
 		dt := req.value

e2e/v3_curl_test.go

@@ -17,6 +17,7 @@ package e2e
 import (
 	"encoding/base64"
 	"encoding/json"
+	"fmt"
 	"path"
 	"strconv"
 	"testing"
@@ -24,6 +25,7 @@ import (
 	epb "github.com/coreos/etcd/etcdserver/api/v3election/v3electionpb"
 	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
 	"github.com/coreos/etcd/pkg/testutil"
+	"github.com/coreos/etcd/version"
 
 	"github.com/grpc-ecosystem/grpc-gateway/runtime"
 )
@@ -390,3 +392,45 @@ type campaignResponse struct {
 		Lease string `json:"lease,omitempty"`
 	} `json:"leader,omitempty"`
 }
+
+func TestV3CurlCipherSuitesValid(t *testing.T)    { testV3CurlCipherSuites(t, true) }
+func TestV3CurlCipherSuitesMismatch(t *testing.T) { testV3CurlCipherSuites(t, false) }
+func testV3CurlCipherSuites(t *testing.T, valid bool) {
+	cc := configClientTLS
+	cc.clusterSize = 1
+	cc.cipherSuites = []string{
+		"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+		"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
+		"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
+		"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
+		"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305",
+		"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
+	}
+	testFunc := cipherSuiteTestValid
+	if !valid {
+		testFunc = cipherSuiteTestMismatch
+	}
+	testCtl(t, testFunc, withCfg(cc))
+}
+
+func cipherSuiteTestValid(cx ctlCtx) {
+	if err := cURLGet(cx.epc, cURLReq{
+		endpoint:         "/metrics",
+		expected:         fmt.Sprintf(`etcd_server_version{server_version="%s"} 1`, version.Version),
+		metricsURLScheme: cx.cfg.metricsURLScheme,
+		ciphers:          "ECDHE-RSA-AES128-GCM-SHA256", // TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
+	}); err != nil {
+		cx.t.Fatalf("failed get with curl (%v)", err)
+	}
+}
+
+func cipherSuiteTestMismatch(cx ctlCtx) {
+	if err := cURLGet(cx.epc, cURLReq{
+		endpoint:         "/metrics",
+		expected:         "alert handshake failure",
+		metricsURLScheme: cx.cfg.metricsURLScheme,
+		ciphers:          "ECDHE-RSA-DES-CBC3-SHA", // TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA
+	}); err != nil {
+		cx.t.Fatalf("failed get with curl (%v)", err)
+	}
+}

embed/config.go

@@ -31,6 +31,7 @@ import (
 	"github.com/coreos/etcd/pkg/cors"
 	"github.com/coreos/etcd/pkg/netutil"
 	"github.com/coreos/etcd/pkg/srv"
+	"github.com/coreos/etcd/pkg/tlsutil"
 	"github.com/coreos/etcd/pkg/transport"
 	"github.com/coreos/etcd/pkg/types"
 
@@ -183,6 +184,11 @@ type Config struct {
 	PeerTLSInfo   transport.TLSInfo
 	PeerAutoTLS   bool
 
+	// CipherSuites is a list of supported TLS cipher suites between
+	// client/server and peers. If empty, Go auto-populates the list.
+	// Note that cipher suites are prioritized in the given order.
+	CipherSuites []string `json:"cipher-suites"`
+
 	// debug
 
 	Debug                 bool   `json:"debug"`
@@ -426,6 +432,24 @@ func (cfg *configYAML) configFromFile(path string) error {
 	return cfg.Validate()
 }
 
+func updateCipherSuites(tls *transport.TLSInfo, ss []string) error {
+	if len(tls.CipherSuites) > 0 && len(ss) > 0 {
+		return fmt.Errorf("TLSInfo.CipherSuites is already specified (given %v)", ss)
+	}
+	if len(ss) > 0 {
+		cs := make([]uint16, len(ss))
+		for i, s := range ss {
+			var ok bool
+			cs[i], ok = tlsutil.GetCipherSuite(s)
+			if !ok {
+				return fmt.Errorf("unexpected TLS cipher suite %q", s)
+			}
+		}
+		tls.CipherSuites = cs
+	}
+	return nil
+}
+
 // Validate ensures that '*embed.Config' fields are properly configured.
 func (cfg *Config) Validate() error {
 	if err := checkBindURLs(cfg.LPUrls); err != nil {
@@ -562,31 +586,41 @@ func (cfg Config) defaultClientHost() bool {
 }
 
 func (cfg *Config) ClientSelfCert() (err error) {
-	if cfg.ClientAutoTLS && cfg.ClientTLSInfo.Empty() {
-		chosts := make([]string, len(cfg.LCUrls))
-		for i, u := range cfg.LCUrls {
-			chosts[i] = u.Host
-		}
-		cfg.ClientTLSInfo, err = transport.SelfCert(filepath.Join(cfg.Dir, "fixtures", "client"), chosts)
-		return err
-	} else if cfg.ClientAutoTLS {
-		plog.Warningf("ignoring client auto TLS since certs given")
+	if !cfg.ClientAutoTLS {
+		return nil
 	}
-	return nil
+	if !cfg.ClientTLSInfo.Empty() {
+		plog.Warningf("ignoring client auto TLS since certs given")
+		return nil
+	}
+	chosts := make([]string, len(cfg.LCUrls))
+	for i, u := range cfg.LCUrls {
+		chosts[i] = u.Host
+	}
+	cfg.ClientTLSInfo, err = transport.SelfCert(filepath.Join(cfg.Dir, "fixtures", "client"), chosts)
+	if err != nil {
+		return err
+	}
+	return updateCipherSuites(&cfg.ClientTLSInfo, cfg.CipherSuites)
 }
 
 func (cfg *Config) PeerSelfCert() (err error) {
-	if cfg.PeerAutoTLS && cfg.PeerTLSInfo.Empty() {
-		phosts := make([]string, len(cfg.LPUrls))
-		for i, u := range cfg.LPUrls {
-			phosts[i] = u.Host
-		}
-		cfg.PeerTLSInfo, err = transport.SelfCert(filepath.Join(cfg.Dir, "fixtures", "peer"), phosts)
-		return err
-	} else if cfg.PeerAutoTLS {
-		plog.Warningf("ignoring peer auto TLS since certs given")
+	if !cfg.PeerAutoTLS {
+		return nil
 	}
-	return nil
+	if !cfg.PeerTLSInfo.Empty() {
+		plog.Warningf("ignoring peer auto TLS since certs given")
+		return nil
+	}
+	phosts := make([]string, len(cfg.LPUrls))
+	for i, u := range cfg.LPUrls {
+		phosts[i] = u.Host
+	}
+	cfg.PeerTLSInfo, err = transport.SelfCert(filepath.Join(cfg.Dir, "fixtures", "peer"), phosts)
+	if err != nil {
+		return err
+	}
+	return updateCipherSuites(&cfg.PeerTLSInfo, cfg.CipherSuites)
 }
 
 // UpdateDefaultClusterFromName updates cluster advertise URLs with, if available, default host,

embed/etcd.go

@@ -42,7 +42,7 @@ import (
 	"github.com/coreos/etcd/rafthttp"
 
 	"github.com/coreos/pkg/capnslog"
-	"github.com/grpc-ecosystem/go-grpc-prometheus"
+	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/soheilhy/cmux"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/keepalive"
@@ -302,6 +302,9 @@ func stopServers(ctx context.Context, ss *servers) {
 func (e *Etcd) Err() <-chan error { return e.errc }
 
 func startPeerListeners(cfg *Config) (peers []*peerListener, err error) {
+	if err = updateCipherSuites(&cfg.PeerTLSInfo, cfg.CipherSuites); err != nil {
+		return nil, err
+	}
 	if err = cfg.PeerSelfCert(); err != nil {
 		plog.Fatalf("could not get certs (%v)", err)
 	}
@@ -387,6 +390,9 @@ func (e *Etcd) servePeers() (err error) {
 }
 
 func startClientListeners(cfg *Config) (sctxs map[string]*serveCtx, err error) {
+	if err = updateCipherSuites(&cfg.ClientTLSInfo, cfg.CipherSuites); err != nil {
+		return nil, err
+	}
 	if err = cfg.ClientSelfCert(); err != nil {
 		plog.Fatalf("could not get certs (%v)", err)
 	}

etcdctl/ctlv3/command/global.go

@@ -109,11 +109,12 @@ func (*discardValue) Type() string     { return "" }
 
 func clientConfigFromCmd(cmd *cobra.Command) *clientConfig {
 	fs := cmd.InheritedFlags()
-
-	// silence "pkg/flags: unrecognized environment variable ETCDCTL_WATCH_KEY=foo" warnings
-	// silence "pkg/flags: unrecognized environment variable ETCDCTL_WATCH_RANGE_END=bar" warnings
-	fs.AddFlag(&pflag.Flag{Name: "watch-key", Value: &discardValue{}})
-	fs.AddFlag(&pflag.Flag{Name: "watch-range-end", Value: &discardValue{}})
+	if strings.HasPrefix(cmd.Use, "watch") {
+		// silence "pkg/flags: unrecognized environment variable ETCDCTL_WATCH_KEY=foo" warnings
+		// silence "pkg/flags: unrecognized environment variable ETCDCTL_WATCH_RANGE_END=bar" warnings
+		fs.AddFlag(&pflag.Flag{Name: "watch-key", Value: &discardValue{}})
+		fs.AddFlag(&pflag.Flag{Name: "watch-range-end", Value: &discardValue{}})
+	}
 	flags.SetPflagsFromEnv("ETCDCTL", fs)
 
 	debug, err := cmd.Flags().GetBool("debug")

etcdctl/ctlv3/command/move_leader_command.go

@@ -17,7 +17,6 @@ package command
 import (
 	"fmt"
 	"strconv"
-	"time"
 
 	"github.com/coreos/etcd/clientv3"
 	"github.com/spf13/cobra"
@@ -53,16 +52,12 @@ func transferLeadershipCommandFunc(cmd *cobra.Command, args []string) {
 	var leaderCli *clientv3.Client
 	var leaderID uint64
 	for _, ep := range eps {
-		cli, err := clientv3.New(clientv3.Config{
-			Endpoints:   []string{ep},
-			DialTimeout: 3 * time.Second,
-		})
-		if err != nil {
-			ExitWithError(ExitError, err)
-		}
-		resp, err := cli.Status(ctx, ep)
-		if err != nil {
-			ExitWithError(ExitError, err)
+		cfg := clientConfigFromCmd(cmd)
+		cfg.endpoints = []string{ep}
+		cli := cfg.mustClient()
+		resp, serr := cli.Status(ctx, ep)
+		if serr != nil {
+			ExitWithError(ExitError, serr)
 		}
 
 		if resp.Header.GetMemberId() == resp.Leader {

etcdctl/ctlv3/command/watch_command.go

@@ -179,118 +179,159 @@ func printWatchCh(c *clientv3.Client, ch clientv3.WatchChan, execArgs []string)
 // "--" characters are invalid arguments for "spf13/cobra" library,
 // so no need to handle such cases.
 func parseWatchArgs(osArgs, commandArgs []string, envKey, envRange string, interactive bool) (watchArgs []string, execArgs []string, err error) {
-	watchArgs = commandArgs
+	rawArgs := make([]string, len(osArgs))
+	copy(rawArgs, osArgs)
+	watchArgs = make([]string, len(commandArgs))
+	copy(watchArgs, commandArgs)
 
-	// remove preceding commands (e.g. "watch foo bar" in interactive mode)
-	idx := 0
-	for idx = range watchArgs {
-		if watchArgs[idx] == "watch" {
+	// remove preceding commands (e.g. ./bin/etcdctl watch)
+	// handle "./bin/etcdctl watch foo -- echo watch event"
+	for idx := range rawArgs {
+		if rawArgs[idx] == "watch" {
+			rawArgs = rawArgs[idx+1:]
 			break
 		}
 	}
-	if idx < len(watchArgs)-1 || envKey != "" {
-		if idx < len(watchArgs)-1 {
-			watchArgs = watchArgs[idx+1:]
-		}
 
-		execIdx, execExist := 0, false
-		for execIdx = range osArgs {
-			v := osArgs[execIdx]
-			if v == "--" && execIdx != len(osArgs)-1 {
+	// remove preceding commands (e.g. "watch foo bar" in interactive mode)
+	// handle "./bin/etcdctl watch foo -- echo watch event"
+	if interactive {
+		if watchArgs[0] != "watch" {
+			// "watch" not found
+			watchPrefix, watchRev, watchPrevKey = false, 0, false
+			return nil, nil, errBadArgsInteractiveWatch
+		}
+		watchArgs = watchArgs[1:]
+	}
+
+	execIdx, execExist := 0, false
+	if !interactive {
+		for execIdx = range rawArgs {
+			if rawArgs[execIdx] == "--" {
 				execExist = true
 				break
 			}
 		}
-
-		if idx == len(watchArgs)-1 && envKey != "" {
-			if len(watchArgs) > 0 && !interactive {
-				// "watch --rev 1 -- echo Hello World" has no conflict
-				if !execExist {
-					// "watch foo" with ETCDCTL_WATCH_KEY=foo
-					// (watchArgs==["foo"])
-					return nil, nil, errBadArgsNumConflictEnv
-				}
-			}
-			// otherwise, watch with no argument and environment key is set
-			// if interactive, first "watch" command string should be removed
-			if interactive {
-				watchArgs = []string{}
-			}
+		if execExist && execIdx == len(rawArgs)-1 {
+			// "watch foo bar --" should error
+			return nil, nil, errBadArgsNumSeparator
 		}
-
-		// "watch foo -- echo hello" with ETCDCTL_WATCH_KEY=foo
-		// (watchArgs==["foo","echo","hello"])
-		if envKey != "" && execExist {
-			widx, oidx := 0, len(osArgs)-1
-			for widx = len(watchArgs) - 1; widx >= 0; widx-- {
-				if watchArgs[widx] == osArgs[oidx] {
-					oidx--
+		// "watch" with no argument should error
+		if !execExist && len(rawArgs) < 1 && envKey == "" {
+			return nil, nil, errBadArgsNum
+		}
+		if execExist && envKey != "" {
+			// "ETCDCTL_WATCH_KEY=foo watch foo -- echo 1" should error
+			// (watchArgs==["foo","echo","1"])
+			widx, ridx := len(watchArgs)-1, len(rawArgs)-1
+			for ; widx >= 0; widx-- {
+				if watchArgs[widx] == rawArgs[ridx] {
+					ridx--
 					continue
 				}
-				if oidx == execIdx { // watchArgs has extra
+				// watchArgs has extra:
+				// ETCDCTL_WATCH_KEY=foo watch foo  --  echo 1
+				// watchArgs:                       foo echo 1
+				if ridx == execIdx {
 					return nil, nil, errBadArgsNumConflictEnv
 				}
 			}
 		}
-	} else if interactive { // "watch" not found
-		return nil, nil, errBadArgsInteractiveWatch
-	}
-	if len(watchArgs) < 1 && envKey == "" {
-		return nil, nil, errBadArgsNum
-	}
+		// check conflicting arguments
+		// e.g. "watch --rev 1 -- echo Hello World" has no conflict
+		if !execExist && len(watchArgs) > 0 && envKey != "" {
+			// "ETCDCTL_WATCH_KEY=foo watch foo" should error
+			// (watchArgs==["foo"])
+			return nil, nil, errBadArgsNumConflictEnv
+		}
+	} else {
+		for execIdx = range watchArgs {
+			if watchArgs[execIdx] == "--" {
+				execExist = true
+				break
+			}
+		}
+		if execExist && execIdx == len(watchArgs)-1 {
+			// "watch foo bar --" should error
+			watchPrefix, watchRev, watchPrevKey = false, 0, false
+			return nil, nil, errBadArgsNumSeparator
+		}
 
-	// remove preceding commands (e.g. ./bin/etcdctl watch)
-	for idx = range osArgs {
-		if osArgs[idx] == "watch" {
-			break
+		flagset := NewWatchCommand().Flags()
+		if err := flagset.Parse(watchArgs); err != nil {
+			watchPrefix, watchRev, watchPrevKey = false, 0, false
+			return nil, nil, err
+		}
+		pArgs := flagset.Args()
+
+		// "watch" with no argument should error
+		if !execExist && envKey == "" && len(pArgs) < 1 {
+			watchPrefix, watchRev, watchPrevKey = false, 0, false
+			return nil, nil, errBadArgsNum
+		}
+		// check conflicting arguments
+		// e.g. "watch --rev 1 -- echo Hello World" has no conflict
+		if !execExist && len(pArgs) > 0 && envKey != "" {
+			// "ETCDCTL_WATCH_KEY=foo watch foo" should error
+			// (watchArgs==["foo"])
+			watchPrefix, watchRev, watchPrevKey = false, 0, false
+			return nil, nil, errBadArgsNumConflictEnv
 		}
 	}
-	if idx < len(osArgs)-1 {
-		osArgs = osArgs[idx+1:]
-	} else if envKey == "" {
-		return nil, nil, errBadArgsNum
-	}
 
-	argsWithSep := osArgs
-	if interactive { // interactive mode pass "--" to the command args
+	argsWithSep := rawArgs
+	if interactive {
+		// interactive mode directly passes "--" to the command args
 		argsWithSep = watchArgs
 	}
-	foundSep := false
+
+	idx, foundSep := 0, false
 	for idx = range argsWithSep {
 		if argsWithSep[idx] == "--" {
 			foundSep = true
 			break
 		}
 	}
+	if foundSep {
+		execArgs = argsWithSep[idx+1:]
+	}
+
 	if interactive {
 		flagset := NewWatchCommand().Flags()
 		if err := flagset.Parse(argsWithSep); err != nil {
 			return nil, nil, err
 		}
 		watchArgs = flagset.Args()
+
+		watchPrefix, err = flagset.GetBool("prefix")
+		if err != nil {
+			return nil, nil, err
+		}
+		watchRev, err = flagset.GetInt64("rev")
+		if err != nil {
+			return nil, nil, err
+		}
+		watchPrevKey, err = flagset.GetBool("prev-kv")
+		if err != nil {
+			return nil, nil, err
+		}
 	}
 
-	// "watch -- echo hello" with ETCDCTL_WATCH_KEY=foo
-	// should be translated to "watch foo -- echo hello"
+	// "ETCDCTL_WATCH_KEY=foo watch -- echo hello"
+	// should translate "watch foo -- echo hello"
 	// (watchArgs=["echo","hello"] should be ["foo","echo","hello"])
 	if envKey != "" {
-		tmp := []string{envKey}
+		ranges := []string{envKey}
 		if envRange != "" {
-			tmp = append(tmp, envRange)
+			ranges = append(ranges, envRange)
 		}
-		watchArgs = append(tmp, watchArgs...)
+		watchArgs = append(ranges, watchArgs...)
 	}
 
 	if !foundSep {
 		return watchArgs, nil, nil
 	}
 
-	if idx == len(argsWithSep)-1 {
-		// "watch foo bar --" should error
-		return nil, nil, errBadArgsNumSeparator
-	}
-	execArgs = argsWithSep[idx+1:]
-
 	// "watch foo bar --rev 1 -- echo hello" or "watch foo --rev 1 bar -- echo hello",
 	// then "watchArgs" is "foo bar echo hello"
 	// so need ignore args after "argsWithSep[idx]", which is "--"

etcdctl/ctlv3/command/watch_command_test.go

@@ -26,6 +26,10 @@ func Test_parseWatchArgs(t *testing.T) {
 		envKey, envRange string
 		interactive      bool
 
+		interactiveWatchPrefix  bool
+		interactiveWatchRev     int64
+		interactiveWatchPrevKey bool
+
 		watchArgs []string
 		execArgs  []string
 		err       error
@@ -145,6 +149,14 @@ func Test_parseWatchArgs(t *testing.T) {
 			execArgs:    []string{"echo", "Hello", "World"},
 			err:         nil,
 		},
+		{
+			osArgs:      []string{"./bin/etcdctl", "watch", "foo", "--", "echo", "watch", "event", "received"},
+			commandArgs: []string{"foo", "echo", "watch", "event", "received"},
+			interactive: false,
+			watchArgs:   []string{"foo"},
+			execArgs:    []string{"echo", "watch", "event", "received"},
+			err:         nil,
+		},
 		{
 			osArgs:      []string{"./bin/etcdctl", "watch", "foo", "--rev", "1", "--", "echo", "Hello", "World"},
 			commandArgs: []string{"foo", "echo", "Hello", "World"},
@@ -153,6 +165,22 @@ func Test_parseWatchArgs(t *testing.T) {
 			execArgs:    []string{"echo", "Hello", "World"},
 			err:         nil,
 		},
+		{
+			osArgs:      []string{"./bin/etcdctl", "watch", "foo", "--rev", "1", "--", "echo", "watch", "event", "received"},
+			commandArgs: []string{"foo", "echo", "watch", "event", "received"},
+			interactive: false,
+			watchArgs:   []string{"foo"},
+			execArgs:    []string{"echo", "watch", "event", "received"},
+			err:         nil,
+		},
+		{
+			osArgs:      []string{"./bin/etcdctl", "watch", "--rev", "1", "foo", "--", "echo", "watch", "event", "received"},
+			commandArgs: []string{"foo", "echo", "watch", "event", "received"},
+			interactive: false,
+			watchArgs:   []string{"foo"},
+			execArgs:    []string{"echo", "watch", "event", "received"},
+			err:         nil,
+		},
 		{
 			osArgs:      []string{"./bin/etcdctl", "watch", "foo", "bar", "--", "echo", "Hello", "World"},
 			commandArgs: []string{"foo", "bar", "echo", "Hello", "World"},
@@ -185,6 +213,30 @@ func Test_parseWatchArgs(t *testing.T) {
 			execArgs:    []string{"echo", "Hello", "World"},
 			err:         nil,
 		},
+		{
+			osArgs:      []string{"./bin/etcdctl", "watch", "foo", "bar", "--rev", "1", "--", "echo", "watch", "event", "received"},
+			commandArgs: []string{"foo", "bar", "echo", "watch", "event", "received"},
+			interactive: false,
+			watchArgs:   []string{"foo", "bar"},
+			execArgs:    []string{"echo", "watch", "event", "received"},
+			err:         nil,
+		},
+		{
+			osArgs:      []string{"./bin/etcdctl", "watch", "foo", "--rev", "1", "bar", "--", "echo", "Hello", "World"},
+			commandArgs: []string{"foo", "bar", "echo", "Hello", "World"},
+			interactive: false,
+			watchArgs:   []string{"foo", "bar"},
+			execArgs:    []string{"echo", "Hello", "World"},
+			err:         nil,
+		},
+		{
+			osArgs:      []string{"./bin/etcdctl", "watch", "--rev", "1", "foo", "bar", "--", "echo", "Hello", "World"},
+			commandArgs: []string{"foo", "bar", "echo", "Hello", "World"},
+			interactive: false,
+			watchArgs:   []string{"foo", "bar"},
+			execArgs:    []string{"echo", "Hello", "World"},
+			err:         nil,
+		},
 		{
 			osArgs:      []string{"./bin/etcdctl", "watch", "--rev", "1", "--", "echo", "Hello", "World"},
 			commandArgs: []string{"echo", "Hello", "World"},
@@ -215,133 +267,269 @@ func Test_parseWatchArgs(t *testing.T) {
 			err:         errBadArgsNumConflictEnv,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"foo", "bar", "--", "echo", "Hello", "World"},
-			interactive: true,
-			watchArgs:   nil,
-			execArgs:    nil,
-			err:         errBadArgsInteractiveWatch,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"foo", "bar", "--", "echo", "Hello", "World"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     0,
+			interactiveWatchPrevKey: false,
+			watchArgs:               nil,
+			execArgs:                nil,
+			err:                     errBadArgsInteractiveWatch,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "foo"},
-			interactive: true,
-			watchArgs:   []string{"foo"},
-			execArgs:    nil,
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "foo"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     0,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo"},
+			execArgs:                nil,
+			err:                     nil,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "foo", "bar"},
-			interactive: true,
-			watchArgs:   []string{"foo", "bar"},
-			execArgs:    nil,
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "foo", "bar"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     0,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo", "bar"},
+			execArgs:                nil,
+			err:                     nil,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch"},
-			envKey:      "foo",
-			envRange:    "bar",
-			interactive: true,
-			watchArgs:   []string{"foo", "bar"},
-			execArgs:    nil,
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch"},
+			envKey:                  "foo",
+			envRange:                "bar",
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     0,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo", "bar"},
+			execArgs:                nil,
+			err:                     nil,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch"},
-			envKey:      "hello world!",
-			envRange:    "bar",
-			interactive: true,
-			watchArgs:   []string{"hello world!", "bar"},
-			execArgs:    nil,
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch"},
+			envKey:                  "hello world!",
+			envRange:                "bar",
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     0,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"hello world!", "bar"},
+			execArgs:                nil,
+			err:                     nil,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "foo", "--rev", "1"},
-			interactive: true,
-			watchArgs:   []string{"foo"},
-			execArgs:    nil,
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "foo", "--rev", "1"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     1,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo"},
+			execArgs:                nil,
+			err:                     nil,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "foo", "--rev", "1", "--", "echo", "Hello", "World"},
-			interactive: true,
-			watchArgs:   []string{"foo"},
-			execArgs:    []string{"echo", "Hello", "World"},
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "foo", "--rev", "1", "--", "echo", "Hello", "World"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     1,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "--rev", "1", "foo", "--", "echo", "Hello", "World"},
-			interactive: true,
-			watchArgs:   []string{"foo"},
-			execArgs:    []string{"echo", "Hello", "World"},
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "--rev", "1", "foo", "--", "echo", "Hello", "World"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     1,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "--", "echo", "Hello", "World"},
-			envKey:      "foo",
-			interactive: true,
-			watchArgs:   []string{"foo"},
-			execArgs:    []string{"echo", "Hello", "World"},
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "--rev", "5", "--prev-kv", "foo", "--", "echo", "Hello", "World"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     5,
+			interactiveWatchPrevKey: true,
+			watchArgs:               []string{"foo"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "--", "echo", "Hello", "World"},
-			envKey:      "foo",
-			envRange:    "bar",
-			interactive: true,
-			watchArgs:   []string{"foo", "bar"},
-			execArgs:    []string{"echo", "Hello", "World"},
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "--rev", "1"},
+			envKey:                  "foo",
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     1,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo"},
+			execArgs:                nil,
+			err:                     nil,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "foo", "bar", "--", "echo", "Hello", "World"},
-			interactive: true,
-			watchArgs:   []string{"foo", "bar"},
-			execArgs:    []string{"echo", "Hello", "World"},
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "--rev", "1"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     0,
+			interactiveWatchPrevKey: false,
+			watchArgs:               nil,
+			execArgs:                nil,
+			err:                     errBadArgsNum,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "--rev", "1", "foo", "bar", "--", "echo", "Hello", "World"},
-			interactive: true,
-			watchArgs:   []string{"foo", "bar"},
-			execArgs:    []string{"echo", "Hello", "World"},
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "--rev", "1", "--prefix"},
+			envKey:                  "foo",
+			interactive:             true,
+			interactiveWatchPrefix:  true,
+			interactiveWatchRev:     1,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo"},
+			execArgs:                nil,
+			err:                     nil,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "--rev", "1", "--", "echo", "Hello", "World"},
-			envKey:      "foo",
-			envRange:    "bar",
-			interactive: true,
-			watchArgs:   []string{"foo", "bar"},
-			execArgs:    []string{"echo", "Hello", "World"},
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "--rev", "100", "--prefix", "--prev-kv"},
+			envKey:                  "foo",
+			interactive:             true,
+			interactiveWatchPrefix:  true,
+			interactiveWatchRev:     100,
+			interactiveWatchPrevKey: true,
+			watchArgs:               []string{"foo"},
+			execArgs:                nil,
+			err:                     nil,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "foo", "--rev", "1", "bar", "--", "echo", "Hello", "World"},
-			interactive: true,
-			watchArgs:   []string{"foo", "bar"},
-			execArgs:    []string{"echo", "Hello", "World"},
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "--rev", "1", "--prefix"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     0,
+			interactiveWatchPrevKey: false,
+			watchArgs:               nil,
+			execArgs:                nil,
+			err:                     errBadArgsNum,
 		},
 		{
-			osArgs:      []string{"./bin/etcdctl", "watch", "-i"},
-			commandArgs: []string{"watch", "foo", "bar", "--rev", "1", "--", "echo", "Hello", "World"},
-			interactive: true,
-			watchArgs:   []string{"foo", "bar"},
-			execArgs:    []string{"echo", "Hello", "World"},
-			err:         nil,
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "--", "echo", "Hello", "World"},
+			envKey:                  "foo",
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     0,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
+		},
+		{
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "--", "echo", "Hello", "World"},
+			envKey:                  "foo",
+			envRange:                "bar",
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     0,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo", "bar"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
+		},
+		{
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "foo", "bar", "--", "echo", "Hello", "World"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     0,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo", "bar"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
+		},
+		{
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "--rev", "1", "foo", "bar", "--", "echo", "Hello", "World"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     1,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo", "bar"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
+		},
+		{
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "--rev", "1", "--", "echo", "Hello", "World"},
+			envKey:                  "foo",
+			envRange:                "bar",
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     1,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo", "bar"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
+		},
+		{
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "foo", "--rev", "1", "bar", "--", "echo", "Hello", "World"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     1,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo", "bar"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
+		},
+		{
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "foo", "bar", "--rev", "1", "--", "echo", "Hello", "World"},
+			interactive:             true,
+			interactiveWatchPrefix:  false,
+			interactiveWatchRev:     1,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo", "bar"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
+		},
+		{
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "foo", "bar", "--rev", "7", "--prefix", "--", "echo", "Hello", "World"},
+			interactive:             true,
+			interactiveWatchPrefix:  true,
+			interactiveWatchRev:     7,
+			interactiveWatchPrevKey: false,
+			watchArgs:               []string{"foo", "bar"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
+		},
+		{
+			osArgs:                  []string{"./bin/etcdctl", "watch", "-i"},
+			commandArgs:             []string{"watch", "foo", "bar", "--rev", "7", "--prefix", "--prev-kv", "--", "echo", "Hello", "World"},
+			interactive:             true,
+			interactiveWatchPrefix:  true,
+			interactiveWatchRev:     7,
+			interactiveWatchPrevKey: true,
+			watchArgs:               []string{"foo", "bar"},
+			execArgs:                []string{"echo", "Hello", "World"},
+			err:                     nil,
 		},
 	}
 	for i, ts := range tt {
@@ -355,5 +543,16 @@ func Test_parseWatchArgs(t *testing.T) {
 		if !reflect.DeepEqual(execArgs, ts.execArgs) {
 			t.Fatalf("#%d: execArgs expected %q, got %v", i, ts.execArgs, execArgs)
 		}
+		if ts.interactive {
+			if ts.interactiveWatchPrefix != watchPrefix {
+				t.Fatalf("#%d: interactive watchPrefix expected %v, got %v", i, ts.interactiveWatchPrefix, watchPrefix)
+			}
+			if ts.interactiveWatchRev != watchRev {
+				t.Fatalf("#%d: interactive watchRev expected %d, got %d", i, ts.interactiveWatchRev, watchRev)
+			}
+			if ts.interactiveWatchPrevKey != watchPrevKey {
+				t.Fatalf("#%d: interactive watchPrevKey expected %v, got %v", i, ts.interactiveWatchPrevKey, watchPrevKey)
+			}
+		}
 	}
 }

etcdmain/config.go

@@ -190,6 +190,8 @@ func newConfig() *config {
 	fs.StringVar(&cfg.ec.PeerTLSInfo.CRLFile, "peer-crl-file", "", "Path to the peer certificate revocation list file.")
 	fs.StringVar(&cfg.ec.PeerTLSInfo.AllowedCN, "peer-cert-allowed-cn", "", "Allowed CN for inter peer authentication.")
 
+	fs.Var(flags.NewStringsValueV2(""), "cipher-suites", "Comma-separated list of supported TLS cipher suites between client/server and peers (empty will be auto-populated by Go).")
+
 	// logging
 	fs.BoolVar(&cfg.ec.Debug, "debug", false, "Enable debug-level logging for etcd.")
 	fs.StringVar(&cfg.ec.LogPkgLevels, "log-package-levels", "", "Specify a particular log level for each etcd package (eg: 'etcdmain=CRITICAL,etcdserver=DEBUG').")
@@ -275,6 +277,8 @@ func (cfg *config) configFromCmdLine() error {
 		cfg.ec.ListenMetricsUrls = []url.URL(u)
 	}
 
+	cfg.ec.CipherSuites = flags.StringsFromFlagV2(cfg.cf.flagSet, "cipher-suites")
+
 	cfg.ec.ClusterState = cfg.cf.clusterState.String()
 	cfg.cp.Fallback = cfg.cf.fallback.String()
 	cfg.cp.Proxy = cfg.cf.proxy.String()

etcdmain/help.go

@@ -154,10 +154,14 @@ security flags:
 		enable peer client cert authentication.
 	--peer-trusted-ca-file ''
 		path to the peer server TLS trusted CA file.
+	--peer-cert-allowed-cn ''
+		Required CN for client certs connecting to the peer endpoint.
 	--peer-auto-tls 'false'
 		peer TLS using self-generated certificates if --peer-key-file and --peer-cert-file are not provided.
 	--peer-crl-file ''
 		path to the peer certificate revocation list file.
+	--cipher-suites ''
+		comma-separated list of supported TLS cipher suites between client/server and peers (empty will be auto-populated by Go).
 
 logging flags
 

etcdserver/apply.go

@@ -107,9 +107,10 @@ func (s *EtcdServer) newApplierV3() applierV3 {
 }
 
 func (a *applierV3backend) Apply(r *pb.InternalRaftRequest) *applyResult {
-	defer warnOfExpensiveRequest(time.Now(), r)
-
 	ar := &applyResult{}
+	defer func(start time.Time) {
+		warnOfExpensiveRequest(start, &pb.InternalRaftStringer{Request: r}, ar.resp, ar.err)
+	}(time.Now())
 
 	// call into a.s.applyV3.F instead of a.F so upper appliers can check individual calls
 	switch {

etcdserver/apply_v2.go

@@ -107,7 +107,7 @@ func (a *applierV2store) Sync(r *RequestV2) Response {
 // applyV2Request interprets r as a call to store.X and returns a Response interpreted
 // from store.Event
 func (s *EtcdServer) applyV2Request(r *RequestV2) Response {
-	defer warnOfExpensiveRequest(time.Now(), r)
+	defer warnOfExpensiveRequest(time.Now(), r, nil, nil)
 
 	switch r.Method {
 	case "POST":

etcdserver/etcdserverpb/raft_internal_stringer.go

@@ -0,0 +1,183 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package etcdserverpb
+
+import (
+	"fmt"
+	"strings"
+
+	proto "github.com/golang/protobuf/proto"
+)
+
+// InternalRaftStringer implements custom proto Stringer:
+// redact password, replace value fields with value_size fields.
+type InternalRaftStringer struct {
+	Request *InternalRaftRequest
+}
+
+func (as *InternalRaftStringer) String() string {
+	switch {
+	case as.Request.LeaseGrant != nil:
+		return fmt.Sprintf("header:<%s> lease_grant:<ttl:%d-second id:%016x>",
+			as.Request.Header.String(),
+			as.Request.LeaseGrant.TTL,
+			as.Request.LeaseGrant.ID,
+		)
+	case as.Request.LeaseRevoke != nil:
+		return fmt.Sprintf("header:<%s> lease_revoke:<id:%016x>",
+			as.Request.Header.String(),
+			as.Request.LeaseRevoke.ID,
+		)
+	case as.Request.Authenticate != nil:
+		return fmt.Sprintf("header:<%s> authenticate:<name:%s simple_token:%s>",
+			as.Request.Header.String(),
+			as.Request.Authenticate.Name,
+			as.Request.Authenticate.SimpleToken,
+		)
+	case as.Request.AuthUserAdd != nil:
+		return fmt.Sprintf("header:<%s> auth_user_add:<name:%s>",
+			as.Request.Header.String(),
+			as.Request.AuthUserAdd.Name,
+		)
+	case as.Request.AuthUserChangePassword != nil:
+		return fmt.Sprintf("header:<%s> auth_user_change_password:<name:%s>",
+			as.Request.Header.String(),
+			as.Request.AuthUserChangePassword.Name,
+		)
+	case as.Request.Put != nil:
+		return fmt.Sprintf("header:<%s> put:<%s>",
+			as.Request.Header.String(),
+			newLoggablePutRequest(as.Request.Put).String(),
+		)
+	case as.Request.Txn != nil:
+		return fmt.Sprintf("header:<%s> txn:<%s>",
+			as.Request.Header.String(),
+			NewLoggableTxnRequest(as.Request.Txn).String(),
+		)
+	default:
+		// nothing to redact
+	}
+	return as.Request.String()
+}
+
+// txnRequestStringer implements a custom proto String to replace value bytes fields with value size
+// fields in any nested txn and put operations.
+type txnRequestStringer struct {
+	Request *TxnRequest
+}
+
+func NewLoggableTxnRequest(request *TxnRequest) *txnRequestStringer {
+	return &txnRequestStringer{request}
+}
+
+func (as *txnRequestStringer) String() string {
+	var compare []string
+	for _, c := range as.Request.Compare {
+		switch cv := c.TargetUnion.(type) {
+		case *Compare_Value:
+			compare = append(compare, newLoggableValueCompare(c, cv).String())
+		default:
+			// nothing to redact
+			compare = append(compare, c.String())
+		}
+	}
+	var success []string
+	for _, s := range as.Request.Success {
+		success = append(success, newLoggableRequestOp(s).String())
+	}
+	var failure []string
+	for _, f := range as.Request.Failure {
+		failure = append(failure, newLoggableRequestOp(f).String())
+	}
+	return fmt.Sprintf("compare:<%s> success:<%s> failure:<%s>",
+		strings.Join(compare, " "),
+		strings.Join(success, " "),
+		strings.Join(failure, " "),
+	)
+}
+
+// requestOpStringer implements a custom proto String to replace value bytes fields with value
+// size fields in any nested txn and put operations.
+type requestOpStringer struct {
+	Op *RequestOp
+}
+
+func newLoggableRequestOp(op *RequestOp) *requestOpStringer {
+	return &requestOpStringer{op}
+}
+
+func (as *requestOpStringer) String() string {
+	switch op := as.Op.Request.(type) {
+	case *RequestOp_RequestPut:
+		return fmt.Sprintf("request_put:<%s>", newLoggablePutRequest(op.RequestPut).String())
+	case *RequestOp_RequestTxn:
+		return fmt.Sprintf("request_txn:<%s>", NewLoggableTxnRequest(op.RequestTxn).String())
+	default:
+		// nothing to redact
+	}
+	return as.Op.String()
+}
+
+// loggableValueCompare implements a custom proto String for Compare.Value union member types to
+// replace the value bytes field with a value size field.
+// To preserve proto encoding of the key and range_end bytes, a faked out proto type is used here.
+type loggableValueCompare struct {
+	Result    Compare_CompareResult `protobuf:"varint,1,opt,name=result,proto3,enum=etcdserverpb.Compare_CompareResult"`
+	Target    Compare_CompareTarget `protobuf:"varint,2,opt,name=target,proto3,enum=etcdserverpb.Compare_CompareTarget"`
+	Key       []byte                `protobuf:"bytes,3,opt,name=key,proto3"`
+	ValueSize int                   `protobuf:"bytes,7,opt,name=value_size,proto3"`
+	RangeEnd  []byte                `protobuf:"bytes,64,opt,name=range_end,proto3"`
+}
+
+func newLoggableValueCompare(c *Compare, cv *Compare_Value) *loggableValueCompare {
+	return &loggableValueCompare{
+		c.Result,
+		c.Target,
+		c.Key,
+		len(cv.Value),
+		c.RangeEnd,
+	}
+}
+
+func (m *loggableValueCompare) Reset()         { *m = loggableValueCompare{} }
+func (m *loggableValueCompare) String() string { return proto.CompactTextString(m) }
+func (*loggableValueCompare) ProtoMessage()    {}
+
+// loggablePutRequest implements a custom proto String to replace value bytes field with a value
+// size field.
+// To preserve proto encoding of the key bytes, a faked out proto type is used here.
+type loggablePutRequest struct {
+	Key         []byte `protobuf:"bytes,1,opt,name=key,proto3"`
+	ValueSize   int    `protobuf:"varint,2,opt,name=value_size,proto3"`
+	Lease       int64  `protobuf:"varint,3,opt,name=lease,proto3"`
+	PrevKv      bool   `protobuf:"varint,4,opt,name=prev_kv,proto3"`
+	IgnoreValue bool   `protobuf:"varint,5,opt,name=ignore_value,proto3"`
+	IgnoreLease bool   `protobuf:"varint,6,opt,name=ignore_lease,proto3"`
+}
+
+func newLoggablePutRequest(request *PutRequest) *loggablePutRequest {
+	return &loggablePutRequest{
+		request.Key,
+		len(request.Value),
+		request.Lease,
+		request.PrevKv,
+		request.IgnoreValue,
+		request.IgnoreLease,
+	}
+}
+
+func (m *loggablePutRequest) Reset()         { *m = loggablePutRequest{} }
+func (m *loggablePutRequest) String() string { return proto.CompactTextString(m) }
+func (*loggablePutRequest) ProtoMessage()    {}

etcdserver/util.go

@@ -16,11 +16,15 @@ package etcdserver
 
 import (
 	"fmt"
+	"reflect"
+	"strings"
 	"time"
 
+	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
 	"github.com/coreos/etcd/etcdserver/membership"
 	"github.com/coreos/etcd/pkg/types"
 	"github.com/coreos/etcd/rafthttp"
+	"github.com/golang/protobuf/proto"
 )
 
 // isConnectedToQuorumSince checks whether the local member is connected to the
@@ -97,18 +101,54 @@ func (nc *notifier) notify(err error) {
 	close(nc.c)
 }
 
-func warnOfExpensiveRequest(now time.Time, stringer fmt.Stringer) {
-	warnOfExpensiveGenericRequest(now, stringer, "")
+func warnOfExpensiveRequest(now time.Time, reqStringer fmt.Stringer, respMsg proto.Message, err error) {
+	var resp string
+	if !isNil(respMsg) {
+		resp = fmt.Sprintf("size:%d", proto.Size(respMsg))
+	}
+	warnOfExpensiveGenericRequest(now, reqStringer, "", resp, err)
 }
 
-func warnOfExpensiveReadOnlyRangeRequest(now time.Time, stringer fmt.Stringer) {
-	warnOfExpensiveGenericRequest(now, stringer, "read-only range ")
+func warnOfExpensiveReadOnlyTxnRequest(now time.Time, r *pb.TxnRequest, txnResponse *pb.TxnResponse, err error) {
+	reqStringer := pb.NewLoggableTxnRequest(r)
+	var resp string
+	if !isNil(txnResponse) {
+		var resps []string
+		for _, r := range txnResponse.Responses {
+			switch op := r.Response.(type) {
+			case *pb.ResponseOp_ResponseRange:
+				resps = append(resps, fmt.Sprintf("range_response_count:%d", len(op.ResponseRange.Kvs)))
+			default:
+				// only range responses should be in a read only txn request
+			}
+		}
+		resp = fmt.Sprintf("responses:<%s> size:%d", strings.Join(resps, " "), proto.Size(txnResponse))
+	}
+	warnOfExpensiveGenericRequest(now, reqStringer, "read-only range ", resp, err)
 }
 
-func warnOfExpensiveGenericRequest(now time.Time, stringer fmt.Stringer, prefix string) {
+func warnOfExpensiveReadOnlyRangeRequest(now time.Time, reqStringer fmt.Stringer, rangeResponse *pb.RangeResponse, err error) {
+	var resp string
+	if !isNil(rangeResponse) {
+		resp = fmt.Sprintf("range_response_count:%d size:%d", len(rangeResponse.Kvs), proto.Size(rangeResponse))
+	}
+	warnOfExpensiveGenericRequest(now, reqStringer, "read-only range ", resp, err)
+}
+
+func warnOfExpensiveGenericRequest(now time.Time, reqStringer fmt.Stringer, prefix string, resp string, err error) {
 	// TODO: add metrics
 	d := time.Since(now)
 	if d > warnApplyDuration {
-		plog.Warningf("%srequest %q took too long (%v) to execute", prefix, stringer.String(), d)
+		var result string
+		if err != nil {
+			result = fmt.Sprintf("error:%v", err)
+		} else {
+			result = resp
+		}
+		plog.Warningf("%srequest %q with result %q took too long (%v) to execute", prefix, reqStringer.String(), result, d)
 	}
 }
+
+func isNil(msg proto.Message) bool {
+	return msg == nil || reflect.ValueOf(msg).IsNil()
+}

etcdserver/v3_server.go

@@ -84,23 +84,26 @@ type Authenticator interface {
 }
 
 func (s *EtcdServer) Range(ctx context.Context, r *pb.RangeRequest) (*pb.RangeResponse, error) {
-	defer warnOfExpensiveReadOnlyRangeRequest(time.Now(), r)
+	var resp *pb.RangeResponse
+	var err error
+	defer func(start time.Time) {
+		warnOfExpensiveReadOnlyRangeRequest(start, r, resp, err)
+	}(time.Now())
 
 	if !r.Serializable {
-		err := s.linearizableReadNotify(ctx)
+		err = s.linearizableReadNotify(ctx)
 		if err != nil {
 			return nil, err
 		}
 	}
-	var resp *pb.RangeResponse
-	var err error
 	chk := func(ai *auth.AuthInfo) error {
 		return s.authStore.IsRangePermitted(ai, r.Key, r.RangeEnd)
 	}
 
 	get := func() { resp, err = s.applyV3Base.Range(nil, r) }
 	if serr := s.doSerialize(ctx, chk, get); serr != nil {
-		return nil, serr
+		err = serr
+		return nil, err
 	}
 	return resp, err
 }
@@ -135,7 +138,9 @@ func (s *EtcdServer) Txn(ctx context.Context, r *pb.TxnRequest) (*pb.TxnResponse
 			return checkTxnAuth(s.authStore, ai, r)
 		}
 
-		defer warnOfExpensiveReadOnlyRangeRequest(time.Now(), r)
+		defer func(start time.Time) {
+			warnOfExpensiveReadOnlyTxnRequest(start, r, resp, err)
+		}(time.Now())
 
 		get := func() { resp, err = s.applyV3Base.Txn(r) }
 		if serr := s.doSerialize(ctx, chk, get); serr != nil {

functional.yaml

@@ -21,13 +21,13 @@ agent-configs:
     key-file: ""
     trusted-ca-file: ""
     listen-peer-urls: ["https://127.0.0.1:1380"]
-    initial-advertise-peer-urls: ["https://127.0.0.1:13800"]
+    initial-advertise-peer-urls: ["https://127.0.0.1:1381"]
     peer-auto-tls: true
     peer-client-cert-auth: false
     peer-cert-file: ""
     peer-key-file: ""
     peer-trusted-ca-file: ""
-    initial-cluster: s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800
+    initial-cluster: s1=https://127.0.0.1:1381,s2=https://127.0.0.1:2381,s3=https://127.0.0.1:3381
     initial-cluster-state: new
     initial-cluster-token: tkn
     snapshot-count: 10000
@@ -70,13 +70,13 @@ agent-configs:
     key-file: ""
     trusted-ca-file: ""
     listen-peer-urls: ["https://127.0.0.1:2380"]
-    initial-advertise-peer-urls: ["https://127.0.0.1:23800"]
+    initial-advertise-peer-urls: ["https://127.0.0.1:2381"]
     peer-auto-tls: true
     peer-client-cert-auth: false
     peer-cert-file: ""
     peer-key-file: ""
     peer-trusted-ca-file: ""
-    initial-cluster: s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800
+    initial-cluster: s1=https://127.0.0.1:1381,s2=https://127.0.0.1:2381,s3=https://127.0.0.1:3381
     initial-cluster-state: new
     initial-cluster-token: tkn
     snapshot-count: 10000
@@ -119,13 +119,13 @@ agent-configs:
     key-file: ""
     trusted-ca-file: ""
     listen-peer-urls: ["https://127.0.0.1:3380"]
-    initial-advertise-peer-urls: ["https://127.0.0.1:33800"]
+    initial-advertise-peer-urls: ["https://127.0.0.1:3381"]
     peer-auto-tls: true
     peer-client-cert-auth: false
     peer-cert-file: ""
     peer-key-file: ""
     peer-trusted-ca-file: ""
-    initial-cluster: s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800
+    initial-cluster: s1=https://127.0.0.1:1381,s2=https://127.0.0.1:2381,s3=https://127.0.0.1:3381
     initial-cluster-state: new
     initial-cluster-token: tkn
     snapshot-count: 10000

functional/tester/case.go

@@ -274,32 +274,39 @@ func (c *caseUntilSnapshot) Inject(clus *Cluster) error {
 	}
 
 	for i := 0; i < retries; i++ {
-		lastRev, _ = clus.maxRev()
+		lastRev, err = clus.maxRev()
 		// If the number of proposals committed is bigger than snapshot count,
 		// a new snapshot should have been created.
-		dicc := lastRev - startRev
-		if dicc > snapshotCount {
+		diff := lastRev - startRev
+		if diff > snapshotCount {
 			clus.lg.Info(
 				"trigger snapshot PASS",
 				zap.Int("retries", i),
 				zap.String("desc", c.Desc()),
-				zap.Int64("committed-entries", dicc),
+				zap.Int64("committed-entries", diff),
 				zap.Int64("etcd-snapshot-count", snapshotCount),
+				zap.Int64("start-revision", startRev),
 				zap.Int64("last-revision", lastRev),
 				zap.Duration("took", time.Since(now)),
 			)
 			return nil
 		}
 
+		dur := time.Second
+		if diff < 0 || err != nil {
+			dur = 3 * time.Second
+		}
 		clus.lg.Info(
 			"trigger snapshot PROGRESS",
 			zap.Int("retries", i),
-			zap.Int64("committed-entries", dicc),
+			zap.Int64("committed-entries", diff),
 			zap.Int64("etcd-snapshot-count", snapshotCount),
+			zap.Int64("start-revision", startRev),
 			zap.Int64("last-revision", lastRev),
 			zap.Duration("took", time.Since(now)),
+			zap.Error(err),
 		)
-		time.Sleep(time.Second)
+		time.Sleep(dur)
 	}
 
 	return fmt.Errorf("cluster too slow: only %d commits in %d retries", lastRev-startRev, retries)

integration/v3_tls_test.go

@@ -0,0 +1,71 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package integration
+
+import (
+	"context"
+	"crypto/tls"
+	"testing"
+	"time"
+
+	"github.com/coreos/etcd/clientv3"
+	"github.com/coreos/etcd/pkg/testutil"
+)
+
+func TestTLSClientCipherSuitesValid(t *testing.T)    { testTLSCipherSuites(t, true) }
+func TestTLSClientCipherSuitesMismatch(t *testing.T) { testTLSCipherSuites(t, false) }
+
+// testTLSCipherSuites ensures mismatching client-side cipher suite
+// fail TLS handshake with the server.
+func testTLSCipherSuites(t *testing.T, valid bool) {
+	defer testutil.AfterTest(t)
+
+	cipherSuites := []uint16{
+		tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+		tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+		tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+		tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+		tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
+		tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
+	}
+	srvTLS, cliTLS := testTLSInfo, testTLSInfo
+	if valid {
+		srvTLS.CipherSuites, cliTLS.CipherSuites = cipherSuites, cipherSuites
+	} else {
+		srvTLS.CipherSuites, cliTLS.CipherSuites = cipherSuites[:2], cipherSuites[2:]
+	}
+
+	clus := NewClusterV3(t, &ClusterConfig{Size: 1, ClientTLS: &srvTLS})
+	defer clus.Terminate(t)
+
+	cc, err := cliTLS.ClientConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cli, cerr := clientv3.New(clientv3.Config{
+		Endpoints:   []string{clus.Members[0].GRPCAddr()},
+		DialTimeout: time.Second,
+		TLS:         cc,
+	})
+	if cli != nil {
+		cli.Close()
+	}
+	if !valid && cerr != context.DeadlineExceeded {
+		t.Fatalf("expected %v with TLS handshake failure, got %v", context.DeadlineExceeded, cerr)
+	}
+	if valid && cerr != nil {
+		t.Fatalf("expected TLS handshake success, got %v", cerr)
+	}
+}

mvcc/watchable_store.go

@@ -192,6 +192,7 @@ func (s *watchableStore) Restore(b backend.Backend) error {
 	}
 
 	for wa := range s.synced.watchers {
+		wa.restore = true
 		s.unsynced.add(wa)
 	}
 	s.synced = newWatcherGroup()
@@ -482,6 +483,14 @@ type watcher struct {
 	// compacted is set when the watcher is removed because of compaction
 	compacted bool
 
+	// restore is true when the watcher is being restored from leader snapshot
+	// which means that this watcher has just been moved from "synced" to "unsynced"
+	// watcher group, possibly with a future revision when it was first added
+	// to the synced watcher
+	// "unsynced" watcher revision must always be <= current revision,
+	// except when the watcher were to be moved from "synced" watcher group
+	restore bool
+
 	// minRev is the minimum revision update the watcher will accept
 	minRev int64
 	id     WatchID

mvcc/watchable_store_test.go

@@ -338,6 +338,62 @@ func TestWatchRestore(t *testing.T) {
 	t.Run("RunSyncWatchLoopBeforeRestore", test(time.Millisecond*120)) // longer than default waitDuration
 }
 
+// TestWatchRestoreSyncedWatcher tests such a case that:
+//   1. watcher is created with a future revision "math.MaxInt64 - 2"
+//   2. watcher with a future revision is added to "synced" watcher group
+//   3. restore/overwrite storage with snapshot of a higher lasat revision
+//   4. restore operation moves "synced" to "unsynced" watcher group
+//   5. choose the watcher from step 1, without panic
+func TestWatchRestoreSyncedWatcher(t *testing.T) {
+	b1, b1Path := backend.NewDefaultTmpBackend()
+	s1 := newWatchableStore(b1, &lease.FakeLessor{}, nil)
+	defer cleanup(s1, b1, b1Path)
+
+	b2, b2Path := backend.NewDefaultTmpBackend()
+	s2 := newWatchableStore(b2, &lease.FakeLessor{}, nil)
+	defer cleanup(s2, b2, b2Path)
+
+	testKey, testValue := []byte("foo"), []byte("bar")
+	rev := s1.Put(testKey, testValue, lease.NoLease)
+	startRev := rev + 2
+
+	// create a watcher with a future revision
+	// add to "synced" watcher group (startRev > s.store.currentRev)
+	w1 := s1.NewWatchStream()
+	w1.Watch(testKey, nil, startRev)
+
+	// make "s2" ends up with a higher last revision
+	s2.Put(testKey, testValue, lease.NoLease)
+	s2.Put(testKey, testValue, lease.NoLease)
+
+	// overwrite storage with higher revisions
+	if err := s1.Restore(b2); err != nil {
+		t.Fatal(err)
+	}
+
+	// wait for next "syncWatchersLoop" iteration
+	// and the unsynced watcher should be chosen
+	time.Sleep(2 * time.Second)
+
+	// trigger events for "startRev"
+	s1.Put(testKey, testValue, lease.NoLease)
+
+	select {
+	case resp := <-w1.Chan():
+		if resp.Revision != startRev {
+			t.Fatalf("resp.Revision expect %d, got %d", startRev, resp.Revision)
+		}
+		if len(resp.Events) != 1 {
+			t.Fatalf("len(resp.Events) expect 1, got %d", len(resp.Events))
+		}
+		if resp.Events[0].Kv.ModRevision != startRev {
+			t.Fatalf("resp.Events[0].Kv.ModRevision expect %d, got %d", startRev, resp.Events[0].Kv.ModRevision)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("failed to receive event in 1 second")
+	}
+}
+
 // TestWatchBatchUnsynced tests batching on unsynced watchers
 func TestWatchBatchUnsynced(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()

mvcc/watcher_group.go

@@ -15,6 +15,7 @@
 package mvcc
 
 import (
+	"fmt"
 	"math"
 
 	"github.com/coreos/etcd/mvcc/mvccpb"
@@ -238,7 +239,15 @@ func (wg *watcherGroup) chooseAll(curRev, compactRev int64) int64 {
 	minRev := int64(math.MaxInt64)
 	for w := range wg.watchers {
 		if w.minRev > curRev {
-			panic("watcher current revision should not exceed current revision")
+			// after network partition, possibly choosing future revision watcher from restore operation
+			// with watch key "proxy-namespace__lostleader" and revision "math.MaxInt64 - 2"
+			// do not panic when such watcher had been moved from "synced" watcher during restore operation
+			if !w.restore {
+				panic(fmt.Errorf("watcher minimum revision %d should not exceed current revision %d", w.minRev, curRev))
+			}
+
+			// mark 'restore' done, since it's chosen
+			w.restore = false
 		}
 		if w.minRev < compactRev {
 			select {

pkg/flags/strings.go

@@ -14,7 +14,12 @@
 
 package flags
 
-import "errors"
+import (
+	"errors"
+	"flag"
+	"sort"
+	"strings"
+)
 
 // NewStringsFlag creates a new string flag for which any one of the given
 // strings is a valid value, and any other value is an error.
@@ -47,3 +52,34 @@ func (ss *StringsFlag) Set(s string) error {
 func (ss *StringsFlag) String() string {
 	return ss.val
 }
+
+// StringsValueV2 wraps "sort.StringSlice".
+type StringsValueV2 sort.StringSlice
+
+// Set parses a command line set of strings, separated by comma.
+// Implements "flag.Value" interface.
+func (ss *StringsValueV2) Set(s string) error {
+	*ss = strings.Split(s, ",")
+	return nil
+}
+
+// String implements "flag.Value" interface.
+func (ss *StringsValueV2) String() string { return strings.Join(*ss, ",") }
+
+// NewStringsValueV2 implements string slice as "flag.Value" interface.
+// Given value is to be separated by comma.
+func NewStringsValueV2(s string) (ss *StringsValueV2) {
+	if s == "" {
+		return &StringsValueV2{}
+	}
+	ss = new(StringsValueV2)
+	if err := ss.Set(s); err != nil {
+		plog.Panicf("new StringsValueV2 should never fail: %v", err)
+	}
+	return ss
+}
+
+// StringsFromFlagV2 returns a string slice from the flag.
+func StringsFromFlagV2(fs *flag.FlagSet, flagName string) []string {
+	return []string(*fs.Lookup(flagName).Value.(*StringsValueV2))
+}

pkg/tlsutil/cipher_suites.go

@@ -0,0 +1,51 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tlsutil
+
+import "crypto/tls"
+
+// cipher suites implemented by Go
+// https://github.com/golang/go/blob/dev.boringcrypto.go1.10/src/crypto/tls/cipher_suites.go
+var cipherSuites = map[string]uint16{
+	"TLS_RSA_WITH_RC4_128_SHA":                tls.TLS_RSA_WITH_RC4_128_SHA,
+	"TLS_RSA_WITH_3DES_EDE_CBC_SHA":           tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA,
+	"TLS_RSA_WITH_AES_128_CBC_SHA":            tls.TLS_RSA_WITH_AES_128_CBC_SHA,
+	"TLS_RSA_WITH_AES_256_CBC_SHA":            tls.TLS_RSA_WITH_AES_256_CBC_SHA,
+	"TLS_RSA_WITH_AES_128_CBC_SHA256":         tls.TLS_RSA_WITH_AES_128_CBC_SHA256,
+	"TLS_RSA_WITH_AES_128_GCM_SHA256":         tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
+	"TLS_RSA_WITH_AES_256_GCM_SHA384":         tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
+	"TLS_ECDHE_ECDSA_WITH_RC4_128_SHA":        tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA,
+	"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA":    tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+	"TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA":    tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+	"TLS_ECDHE_RSA_WITH_RC4_128_SHA":          tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA,
+	"TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA":     tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,
+	"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA":      tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+	"TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA":      tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+	"TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256": tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,
+	"TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256":   tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,
+	"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256":   tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+	"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256": tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+	"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384":   tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+	"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384": tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+	"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305":    tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
+	"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305":  tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
+}
+
+// GetCipherSuite returns the corresponding cipher suite,
+// and boolean value if it is supported.
+func GetCipherSuite(s string) (uint16, bool) {
+	v, ok := cipherSuites[s]
+	return v, ok
+}

pkg/tlsutil/cipher_suites_test.go

@@ -0,0 +1,42 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tlsutil
+
+import (
+	"go/importer"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestGetCipherSuites(t *testing.T) {
+	pkg, err := importer.For("source", nil).Import("crypto/tls")
+	if err != nil {
+		t.Fatal(err)
+	}
+	cm := make(map[string]uint16)
+	for _, s := range pkg.Scope().Names() {
+		if strings.HasPrefix(s, "TLS_RSA_") || strings.HasPrefix(s, "TLS_ECDHE_") {
+			v, ok := GetCipherSuite(s)
+			if !ok {
+				t.Fatalf("Go implements missing cipher suite %q (%v)", s, v)
+			}
+			cm[s] = v
+		}
+	}
+	if !reflect.DeepEqual(cm, cipherSuites) {
+		t.Fatalf("found unmatched cipher suites %v (Go) != %v", cm, cipherSuites)
+	}
+}

pkg/transport/listener.go

@@ -72,6 +72,11 @@ type TLSInfo struct {
 	// connection will be closed immediately afterwards.
 	HandshakeFailure func(*tls.Conn, error)
 
+	// CipherSuites is a list of supported cipher suites.
+	// If empty, Go auto-populates it by default.
+	// Note that cipher suites are prioritized in the given order.
+	CipherSuites []uint16
+
 	selfCert bool
 
 	// parseFunc exists to simplify testing. Typically, parseFunc
@@ -178,6 +183,10 @@ func (info TLSInfo) baseConfig() (*tls.Config, error) {
 		ServerName: info.ServerName,
 	}
 
+	if len(info.CipherSuites) > 0 {
+		cfg.CipherSuites = info.CipherSuites
+	}
+
 	if info.AllowedCN != "" {
 		cfg.VerifyPeerCertificate = func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
 			for _, chains := range verifiedChains {

pkg/transport/transport_test.go

@@ -0,0 +1,73 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transport
+
+import (
+	"crypto/tls"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+)
+
+// TestNewTransportTLSInvalidCipherSuites expects a client with invalid
+// cipher suites fail to handshake with the server.
+func TestNewTransportTLSInvalidCipherSuites(t *testing.T) {
+	tlsInfo, del, err := createSelfCert()
+	if err != nil {
+		t.Fatalf("unable to create cert: %v", err)
+	}
+	defer del()
+
+	cipherSuites := []uint16{
+		tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+		tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+		tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+		tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+		tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
+		tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
+	}
+
+	// make server and client have unmatched cipher suites
+	srvTLS, cliTLS := *tlsInfo, *tlsInfo
+	srvTLS.CipherSuites, cliTLS.CipherSuites = cipherSuites[:2], cipherSuites[2:]
+
+	ln, err := NewListener("127.0.0.1:0", "https", &srvTLS)
+	if err != nil {
+		t.Fatalf("unexpected NewListener error: %v", err)
+	}
+	defer ln.Close()
+
+	donec := make(chan struct{})
+	go func() {
+		ln.Accept()
+		donec <- struct{}{}
+	}()
+	go func() {
+		tr, err := NewTransport(cliTLS, 3*time.Second)
+		if err != nil {
+			t.Fatalf("unexpected NewTransport error: %v", err)
+		}
+		cli := &http.Client{Transport: tr}
+		_, gerr := cli.Get("https://" + ln.Addr().String())
+		if gerr == nil || !strings.Contains(gerr.Error(), "tls: handshake failure") {
+			t.Fatal("expected client TLS handshake error")
+		}
+		ln.Close()
+		donec <- struct{}{}
+	}()
+	<-donec
+	<-donec
+}

scripts/release

@@ -16,6 +16,10 @@ help() {
   echo ""
   echo "  args:"
   echo "    version: version of etcd to release, e.g. '3.2.18'"
+  echo "  flags:"
+  echo "    --no-upload: skip gs://etcd binary artifact uploads."
+  echo "    --no-docker-push: skip docker image pushes."
+  echo ""
 }
 
 main() {
@@ -28,6 +32,41 @@ main() {
   MINOR_VERSION=$(echo "${VERSION}" | cut -d. -f 1-2)
   BRANCH="release-${MINOR_VERSION}"
 
+  if ! command -v acbuild >/dev/null; then
+    echo "cannot find acbuild"
+    exit 1
+  fi
+
+  if ! command -v docker >/dev/null; then
+    echo "cannot find docker"
+    exit 1
+  fi
+
+  KEYID=$(gpg --list-keys --with-colons| awk -F: '/^pub:/ { print $5 }')
+  if [[ -z "${KEYID}" ]]; then
+    echo "Failed to load gpg key. Is gpg set up correctly for etcd releases?"
+    exit 1
+  fi
+
+  # Expected umask for etcd release artifacts
+  umask 022
+
+  # Set up release directory.
+  local reldir="/tmp/etcd-release-${VERSION}"
+  if [ ! -d "${reldir}/etcd" ]; then
+    mkdir -p "${reldir}"
+    cd "${reldir}"
+    git clone git@github.com:coreos/etcd.git --branch "${BRANCH}"
+  fi
+  cd "${reldir}/etcd"
+
+  # If a release version tag already exists, use it.
+  local remote_tag_exists=$(git ls-remote origin "refs/tags/${RELEASE_VERSION}" | grep -c "${RELEASE_VERSION}")
+  if [ ${remote_tag_exists} -gt 0 ]; then
+    echo "Release version tag exists on remote. Checking out refs/tags/${RELEASE_VERSION}"
+    git checkout -q "tags/${RELEASE_VERSION}"
+  fi
+
   # Check go version.
   local go_version="go$(yq -r ".go[0]" .travis.yml)"
   local current_go_version=$(go version | awk '{ print $3 }')
@@ -36,85 +75,116 @@ main() {
     exit 1
   fi
 
-  # Make a temp directory
-  cd $(mktemp -d)
-  git clone git@github.com:coreos/etcd.git --branch "${BRANCH}"
-  cd etcd
+  # If the release tag does not already exist remotely, create it.
+  if [ ${remote_tag_exists} -eq 0 ]; then
+    # Bump version/version.go to release version.
+    local source_version=$(egrep "\s+Version\s*=" version/version.go | sed -e "s/.*\"\(.*\)\".*/\1/g")
+    if [[ "${source_version}" != "${VERSION}" ]]; then
+      source_minor_version=$(echo "${source_version}" | cut -d. -f 1-2)
+      if [[ "${source_minor_version}" != "${MINOR_VERSION}" ]]; then
+        echo "Wrong etcd minor version in version/version.go. Expected ${MINOR_VERSION} but got ${source_minor_version}. Aborting."
+        exit 1
+      fi
+      echo "Updating version from ${source_version} to ${VERSION} in version/version.go"
+      sed -i "s/${source_version}/${VERSION}/g" version/version.go
+      echo "Building etcd with updated version"
+      ./build
+    fi
 
-  KEYID=$(gpg --list-keys --with-colons| awk -F: '/^pub:/ { print $5 }')
-
-  if [[ -z "${KEYID}" ]]; then
-    echo "Failed to load gpg key. Is gpg set up correctly for etcd releases?"
-    exit 1
-  fi
-
-  # Bump version/version.go to release version.
-  local source_version=$(egrep "\s+Version\s*=" version/version.go | sed -e "s/.*\"\(.*\)\".*/\1/g")
-  if [[ "${source_version}" != "${VERSION}" ]]; then
-    source_minor_version=$(echo "${source_version}" | cut -d. -f 1-2)
-    if [[ "${source_minor_version}" != "${MINOR_VERSION}" ]]; then
-      echo "Wrong etcd minor version in version/version.go. Expected ${MINOR_VERSION} but got ${source_minor_version}. Aborting."
+    local etcd_version=$(bin/etcd --version | grep "etcd Version" | awk '{ print $3 }')
+    if [[ "${etcd_version}" != "${VERSION}" ]]; then
+      echo "Wrong etcd version in version/version.go. Expected ${etcd_version} but got ${VERSION}. Aborting."
       exit 1
     fi
+
+    if [[ ! -z $(git status -s) ]]; then
+      echo "Committing version/version.go update."
+      git add version/version.go
+      git commit -m "version: bump up to ${VERSION}"
+      git diff --staged
+    fi
+
+    # Push the version change if it's not already been pushed.
+    if [ $(git rev-list --count "origin/${BRANCH}..${BRANCH}") -gt 0 ]; then
+      read -p "Push version bump up to ${VERSION} to github.com/coreos/etcd [y/N]? " confirm
+      [[ "${confirm,,}" == "y" ]] || exit 1
+      git push
+    fi
+
+    # Tag release.
+    if [ $(git tag --list | grep -c "${RELEASE_VERSION}") -gt 0 ]; then
+      echo "Skipping tag step. git tag ${RELEASE_VERSION} already exists."
+    else
+      echo "Tagging release..."
+      git tag --local-user "${KEYID}" --sign "${RELEASE_VERSION}" --message "${RELEASE_VERSION}"
+    fi
+
+    # Push the tag change if it's not already been pushed.
+    read -p "Push etcd ${RELEASE_VERSION} tag [y/N]? " confirm
+    [[ "${confirm,,}" == "y" ]] || exit 1
+    git push origin "tags/${RELEASE_VERSION}"
+  fi
+
+  # Build release.
+  # TODO: check the release directory for all required build artifacts.
+  if [ -d release ]; then
+    echo "Skpping release build step. /release directory already exists."
+  else
+    echo "Building release..."
+    # Check for old and new names of the release build script.
+    # TODO: Move the release script into this on as a function?
+    if [ -f ./scripts/release.sh ]; then
+      ./scripts/release.sh "${RELEASE_VERSION}"
+    else
+      ./scripts/build-release.sh "${RELEASE_VERSION}"
+    fi
+  fi
+
+  # Sanity checks.
+  ./release/etcd-${RELEASE_VERSION}-linux-amd64/etcd --version | grep -q "etcd Version: ${VERSION}"
+  ETCDCTL_API=3 ./release/etcd-${RELEASE_VERSION}-linux-amd64/etcdctl version | grep -q "etcdctl version: ${VERSION}"
+
+  # Upload artifacts.
+  if [ "${NO_UPLOAD}" == 1 ]; then
+    echo "Skipping artifact upload to gs://etcd. --no-upload flat is set."
+  else
+    read -p "Upload etcd ${RELEASE_VERSION} release artifacts to gs://etcd [y/N]? " confirm
+    [[ "${confirm,,}" == "y" ]] || exit 1
+    gsutil -m cp ./release/*.zip gs://etcd/${RELEASE_VERSION}/
+    gsutil -m cp ./release/*.tar.gz gs://etcd/${RELEASE_VERSION}/
+    gsutil -m cp ./release/*.aci gs://etcd/${RELEASE_VERSION}/
+    gsutil -m acl ch -u allUsers:R -r gs://etcd/${RELEASE_VERSION}/
+  fi
+  
+    # Push images.
+  if [ "${NO_DOCKER_PUSH}" == 1 ]; then
+    echo "Skipping docker push. --no-docker-push flat is set."
+  else    
+    read -p "Publish etcd ${RELEASE_VERSION} docker images to quay.io [y/N]? " confirm
+    [[ "${confirm,,}" == "y" ]] || exit 1
+    for i in {1..5}; do
+      docker login quay.io && break
+      echo "login failed, retrying"
+    done
+    gcloud docker -- login -u _json_key -p "$(cat /etc/gcp-key-etcd-development.json)" https://gcr.io
+
+    docker push quay.io/coreos/etcd:${RELEASE_VERSION}
+    gcloud docker -- push gcr.io/etcd-development/etcd:${RELEASE_VERSION}
+    if [ "${MINOR_VERSION}" != "3.1" ]; then
+      for TARGET_ARCH in "-arm64" "-ppc64le"; do
+        docker push quay.io/coreos/etcd:${RELEASE_VERSION}${TARGET_ARCH}
+        gcloud docker -- push gcr.io/etcd-development/etcd:${RELEASE_VERSION}${TARGET_ARCH}
+      done
+    fi
+    gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
+
+    docker tag quay.io/coreos/etcd:${RELEASE_VERSION} quay.io/coreos/etcd:v${MINOR_VERSION}
+    docker push quay.io/coreos/etcd:v${MINOR_VERSION}
     
-    echo "Updating version from ${source_version} to ${VERSION} in version/version.go"
-    sed -i.bak "s/${source_version}/${VERSION}/g" version/version.go
-    echo "Building etcd with updated version"
-    ./build
+    gcloud docker -- tag gcr.io/etcd-development/etcd:${RELEASE_VERSION} gcr.io/etcd-development/etcd:v${MINOR_VERSION}
+    gcloud docker -- push gcr.io/etcd-development/etcd:v${MINOR_VERSION}
   fi
 
-  local etcd_version=$(bin/etcd --version | grep "etcd Version" | awk '{ print $3 }')
-  if [[ "${etcd_version}" != "${VERSION}" ]]; then
-    echo "Wrong etcd version in version/version.go. Expected ${etcd_version} but got ${VERSION}. Aborting."
-    exit 1
-  fi
-
-  git add version/version.go
-  git commit -m "version: bump up to ${VERSION}"
-  git diff --staged
-  read -p "Push version bump up to ${VERSION} to github.com/coreos/etcd [y/N]? " confirm
-  [[ "${confirm,,}" == "y" ]] || exit 1
-  git push
-
-  echo "Tagging release..."
-  git tag --local-user "${KEYID}" --sign "${RELEASE_VERSION}" --message "${RELEASE_VERSION}"
-  git tag --list | grep "{VERSION}"
-  read -p "Push etcd ${RELEASE_VERSION} tag [y/N]? " confirm
-  [[ "${confirm,,}" == "y" ]] || exit 1
-  git push origin "tags/${RELEASE_VERSION}"
-
-  echo "Building release..."
-  ./scripts/release.sh "${RELEASE_VERSION}"
-
-  # TODO: validate output of checks
-  ./release/etcd-${RELEASE_VERSION}-linux-amd64/etcd --version
-  ETCDCTL_API=3 ./release/etcd-${RELEASE_VERSION}-linux-amd64/etcdctl version
-  
-  read -p "Upload etcd ${RELEASE_VERSION} release artifacts to gs://etcd [y/N]? " confirm
-  [[ "${confirm,,}" == "y" ]] || exit 1
-
-  gsutil -m cp ./release/*.zip gs://etcd/${RELEASE_VERSION}/
-  gsutil -m cp ./release/*.tar.gz gs://etcd/${RELEASE_VERSION}/
-  gsutil -m cp ./release/*.aci gs://etcd/${RELEASE_VERSION}/
-  gsutil -m acl ch -u allUsers:R -r gs://etcd/${RELEASE_VERSION}/
-
-  read -p "Publish etcd ${RELEASE_VERSION} docker images to quay.io [y/N]? " confirm
-  [[ "${confirm,,}" == "y" ]] || exit 1
-  docker login quay.io
-  gcloud docker -- login -u _json_key -p "$(cat /etc/gcp-key-etcd-development.json)" https://gcr.io
- 
-  for TARGET_ARCH in "-arm64" "-ppc64le" ""; do
-    docker push quay.io/coreos/etcd:${RELEASE_VERSION}${TARGET_ARCH}
-    gcloud docker -- push gcr.io/etcd-development/etcd:${RELEASE_VERSION}${TARGET_ARCH}
-  done
-  gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
-
-  docker tag quay.io/coreos/etcd:${RELEASE_VERSION} quay.io/coreos/etcd:${MINOR_VERSION}
-  docker push quay.io/coreos/etcd:${MINOR_VERSION}
-  
-  gcloud docker -- tag gcr.io/etcd-development/etcd:${RELEASE_VERSION} gcr.io/etcd-development/etcd:${MINOR_VERSION}
-  gcloud docker -- push gcr.io/etcd-development/etcd:${MINOR_VERSION}
-
   # TODO: test
   # docker run --rm --name etcd-gcr-${RELEASE_VERSION} gcr.io/etcd-development/etcd:${RELEASE_VERSION};
   # docker exec etcd-gcr-${RELEASE_VERSION} /bin/sh -c "/usr/local/bin/etcd --version"
@@ -122,25 +192,33 @@ main() {
   # docker exec etcd-gcr-${RELEASE_VERSION} /bin/sh -c "ETCDCTL_API=3 /usr/local/bin/etcdctl put foo bar"
   # docker exec etcd-gcr-${RELEASE_VERSION} /bin/sh -c "ETCDCTL_API=3 /usr/local/bin/etcdctl get foo"
 
-  echo "Updating version from ${VERSION} to ${VERSION}+git in version/version.go"
-  sed -i.bak "s/${VERSION}/${VERSION}+git/g" version/version.go
-  echo "Building etcd with ${VERSION}+git in version/version.go"
-  git add version/version.go
-  git commit -m "version: bump up to ${VERSION}+git"
-  git diff --staged
-  read -p "Push version bump up to ${VERSION}+git to github.com/coreos/etcd [y/N]? " confirm
-  [[ "${confirm,,}" == "y" ]] || exit 1
-  git push
-
+  # Bump version to next development version.
+  git checkout -q "${BRANCH}" # Since we might be on a checkout of the remote version tag.
+  local source_version=$(egrep "\s+Version\s*=" version/version.go | sed -e "s/.*\"\(.*\)\".*/\1/g")
+  if [[ "${source_version}" != "${VERSION}+git" ]]; then
+    echo "Updating version from ${source_version} to ${VERSION}+git in version/version.go"
+    sed -i "s/${source_version}/${VERSION}+git/g" version/version.go
+    echo "Building etcd with ${VERSION}+git in version/version.go"
+    git add version/version.go
+    git commit -m "version: bump up to ${VERSION}+git"
+    git diff --staged
+    read -p "Push version bump up to ${VERSION}+git to github.com/coreos/etcd [y/N]? " confirm
+    [[ "${confirm,,}" == "y" ]] || exit 1
+    git push
+  fi
+  
   # TODO: signing process
+  echo ""
   echo "WARNING: The release has not been signed and published to github. This must be done manually."
-
-  echo "WARNING: version/version.go has not been updated to ${RELEASE_VERSION}+git. This must be done manually."
+  echo ""
   echo "Success."
   exit 0
 }
 
 POSITIONAL=()
+NO_UPLOAD=0
+NO_DOCKER_PUSH=0
+
 while test $# -gt 0; do
         case "$1" in
           -h|--help)
@@ -148,6 +226,14 @@ while test $# -gt 0; do
             help
             exit 0
             ;;
+          --no-upload)
+            NO_UPLOAD=1
+            shift
+            ;;
+          --no-docker-push)
+            NO_DOCKER_PUSH=1
+            shift
+            ;;
           *)
             POSITIONAL+=("$1") # save it in an array for later
             shift # past argument

tests/e2e/ctl_v3_watch_cov_test.go

@@ -0,0 +1,134 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build cov
+
+package e2e
+
+import (
+	"os"
+	"testing"
+)
+
+func TestCtlV3Watch(t *testing.T)          { testCtl(t, watchTest) }
+func TestCtlV3WatchNoTLS(t *testing.T)     { testCtl(t, watchTest, withCfg(configNoTLS)) }
+func TestCtlV3WatchClientTLS(t *testing.T) { testCtl(t, watchTest, withCfg(configClientTLS)) }
+func TestCtlV3WatchPeerTLS(t *testing.T)   { testCtl(t, watchTest, withCfg(configPeerTLS)) }
+func TestCtlV3WatchTimeout(t *testing.T)   { testCtl(t, watchTest, withDialTimeout(0)) }
+
+func TestCtlV3WatchInteractive(t *testing.T) {
+	testCtl(t, watchTest, withInteractive())
+}
+func TestCtlV3WatchInteractiveNoTLS(t *testing.T) {
+	testCtl(t, watchTest, withInteractive(), withCfg(configNoTLS))
+}
+func TestCtlV3WatchInteractiveClientTLS(t *testing.T) {
+	testCtl(t, watchTest, withInteractive(), withCfg(configClientTLS))
+}
+func TestCtlV3WatchInteractivePeerTLS(t *testing.T) {
+	testCtl(t, watchTest, withInteractive(), withCfg(configPeerTLS))
+}
+
+func watchTest(cx ctlCtx) {
+	tests := []struct {
+		puts     []kv
+		envKey   string
+		envRange string
+		args     []string
+
+		wkv []kvExec
+	}{
+		{ // watch 1 key
+			puts: []kv{{"sample", "value"}},
+			args: []string{"sample", "--rev", "1"},
+			wkv:  []kvExec{{key: "sample", val: "value"}},
+		},
+		{ // watch 1 key with env
+			puts:   []kv{{"sample", "value"}},
+			envKey: "sample",
+			args:   []string{"--rev", "1"},
+			wkv:    []kvExec{{key: "sample", val: "value"}},
+		},
+
+		// coverage tests get extra arguments:
+		// ./bin/etcdctl_test -test.coverprofile=e2e.1525392462795198897.coverprofile -test.outputdir=../..
+		// do not test watch exec commands
+
+		{ // watch 3 keys by prefix
+			puts: []kv{{"key1", "val1"}, {"key2", "val2"}, {"key3", "val3"}},
+			args: []string{"key", "--rev", "1", "--prefix"},
+			wkv:  []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}, {key: "key3", val: "val3"}},
+		},
+		{ // watch 3 keys by prefix, with env
+			puts:   []kv{{"key1", "val1"}, {"key2", "val2"}, {"key3", "val3"}},
+			envKey: "key",
+			args:   []string{"--rev", "1", "--prefix"},
+			wkv:    []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}, {key: "key3", val: "val3"}},
+		},
+		{ // watch by revision
+			puts: []kv{{"etcd", "revision_1"}, {"etcd", "revision_2"}, {"etcd", "revision_3"}},
+			args: []string{"etcd", "--rev", "2"},
+			wkv:  []kvExec{{key: "etcd", val: "revision_2"}, {key: "etcd", val: "revision_3"}},
+		},
+		{ // watch 3 keys by range
+			puts: []kv{{"key1", "val1"}, {"key3", "val3"}, {"key2", "val2"}},
+			args: []string{"key", "key3", "--rev", "1"},
+			wkv:  []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}},
+		},
+		{ // watch 3 keys by range, with env
+			puts:     []kv{{"key1", "val1"}, {"key3", "val3"}, {"key2", "val2"}},
+			envKey:   "key",
+			envRange: "key3",
+			args:     []string{"--rev", "1"},
+			wkv:      []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}},
+		},
+	}
+
+	for i, tt := range tests {
+		donec := make(chan struct{})
+		go func(i int, puts []kv) {
+			for j := range puts {
+				if err := ctlV3Put(cx, puts[j].key, puts[j].val, ""); err != nil {
+					cx.t.Fatalf("watchTest #%d-%d: ctlV3Put error (%v)", i, j, err)
+				}
+			}
+			close(donec)
+		}(i, tt.puts)
+
+		unsetEnv := func() {}
+		if tt.envKey != "" || tt.envRange != "" {
+			if tt.envKey != "" {
+				os.Setenv("ETCDCTL_WATCH_KEY", tt.envKey)
+				unsetEnv = func() { os.Unsetenv("ETCDCTL_WATCH_KEY") }
+			}
+			if tt.envRange != "" {
+				os.Setenv("ETCDCTL_WATCH_RANGE_END", tt.envRange)
+				unsetEnv = func() { os.Unsetenv("ETCDCTL_WATCH_RANGE_END") }
+			}
+			if tt.envKey != "" && tt.envRange != "" {
+				unsetEnv = func() {
+					os.Unsetenv("ETCDCTL_WATCH_KEY")
+					os.Unsetenv("ETCDCTL_WATCH_RANGE_END")
+				}
+			}
+		}
+		if err := ctlV3Watch(cx, tt.args, tt.wkv...); err != nil {
+			if cx.dialTimeout > 0 && !isGRPCTimedout(err) {
+				cx.t.Errorf("watchTest #%d: ctlV3Watch error (%v)", i, err)
+			}
+		}
+		unsetEnv()
+		<-donec
+	}
+}

tests/e2e/ctl_v3_watch_no_cov_test.go

@@ -0,0 +1,167 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !cov
+
+package e2e
+
+import (
+	"os"
+	"testing"
+)
+
+func TestCtlV3Watch(t *testing.T)          { testCtl(t, watchTest) }
+func TestCtlV3WatchNoTLS(t *testing.T)     { testCtl(t, watchTest, withCfg(configNoTLS)) }
+func TestCtlV3WatchClientTLS(t *testing.T) { testCtl(t, watchTest, withCfg(configClientTLS)) }
+func TestCtlV3WatchPeerTLS(t *testing.T)   { testCtl(t, watchTest, withCfg(configPeerTLS)) }
+func TestCtlV3WatchTimeout(t *testing.T)   { testCtl(t, watchTest, withDialTimeout(0)) }
+
+func TestCtlV3WatchInteractive(t *testing.T) {
+	testCtl(t, watchTest, withInteractive())
+}
+func TestCtlV3WatchInteractiveNoTLS(t *testing.T) {
+	testCtl(t, watchTest, withInteractive(), withCfg(configNoTLS))
+}
+func TestCtlV3WatchInteractiveClientTLS(t *testing.T) {
+	testCtl(t, watchTest, withInteractive(), withCfg(configClientTLS))
+}
+func TestCtlV3WatchInteractivePeerTLS(t *testing.T) {
+	testCtl(t, watchTest, withInteractive(), withCfg(configPeerTLS))
+}
+
+func watchTest(cx ctlCtx) {
+	tests := []struct {
+		puts     []kv
+		envKey   string
+		envRange string
+		args     []string
+
+		wkv []kvExec
+	}{
+		{ // watch 1 key
+			puts: []kv{{"sample", "value"}},
+			args: []string{"sample", "--rev", "1"},
+			wkv:  []kvExec{{key: "sample", val: "value"}},
+		},
+		{ // watch 1 key with env
+			puts:   []kv{{"sample", "value"}},
+			envKey: "sample",
+			args:   []string{"--rev", "1"},
+			wkv:    []kvExec{{key: "sample", val: "value"}},
+		},
+		{ // watch 1 key with ${ETCD_WATCH_VALUE}
+			puts: []kv{{"sample", "value"}},
+			args: []string{"sample", "--rev", "1", "--", "env"},
+			wkv:  []kvExec{{key: "sample", val: "value", execOutput: `ETCD_WATCH_VALUE="value"`}},
+		},
+		{ // watch 1 key with "echo watch event received", with env
+			puts:   []kv{{"sample", "value"}},
+			envKey: "sample",
+			args:   []string{"--rev", "1", "--", "echo", "watch event received"},
+			wkv:    []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}},
+		},
+		{ // watch 1 key with "echo watch event received"
+			puts: []kv{{"sample", "value"}},
+			args: []string{"--rev", "1", "sample", "--", "echo", "watch event received"},
+			wkv:  []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}},
+		},
+		{ // watch 1 key with "echo \"Hello World!\""
+			puts: []kv{{"sample", "value"}},
+			args: []string{"--rev", "1", "sample", "--", "echo", "\"Hello World!\""},
+			wkv:  []kvExec{{key: "sample", val: "value", execOutput: "Hello World!"}},
+		},
+		{ // watch 1 key with "echo watch event received"
+			puts: []kv{{"sample", "value"}},
+			args: []string{"sample", "samplx", "--rev", "1", "--", "echo", "watch event received"},
+			wkv:  []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}},
+		},
+		{ // watch 1 key with "echo watch event received"
+			puts:     []kv{{"sample", "value"}},
+			envKey:   "sample",
+			envRange: "samplx",
+			args:     []string{"--rev", "1", "--", "echo", "watch event received"},
+			wkv:      []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}},
+		},
+		{ // watch 1 key with "echo watch event received"
+			puts: []kv{{"sample", "value"}},
+			args: []string{"sample", "--rev", "1", "samplx", "--", "echo", "watch event received"},
+			wkv:  []kvExec{{key: "sample", val: "value", execOutput: "watch event received"}},
+		},
+		{ // watch 3 keys by prefix
+			puts: []kv{{"key1", "val1"}, {"key2", "val2"}, {"key3", "val3"}},
+			args: []string{"key", "--rev", "1", "--prefix"},
+			wkv:  []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}, {key: "key3", val: "val3"}},
+		},
+		{ // watch 3 keys by prefix, with env
+			puts:   []kv{{"key1", "val1"}, {"key2", "val2"}, {"key3", "val3"}},
+			envKey: "key",
+			args:   []string{"--rev", "1", "--prefix"},
+			wkv:    []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}, {key: "key3", val: "val3"}},
+		},
+		{ // watch by revision
+			puts: []kv{{"etcd", "revision_1"}, {"etcd", "revision_2"}, {"etcd", "revision_3"}},
+			args: []string{"etcd", "--rev", "2"},
+			wkv:  []kvExec{{key: "etcd", val: "revision_2"}, {key: "etcd", val: "revision_3"}},
+		},
+		{ // watch 3 keys by range
+			puts: []kv{{"key1", "val1"}, {"key3", "val3"}, {"key2", "val2"}},
+			args: []string{"key", "key3", "--rev", "1"},
+			wkv:  []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}},
+		},
+		{ // watch 3 keys by range, with env
+			puts:     []kv{{"key1", "val1"}, {"key3", "val3"}, {"key2", "val2"}},
+			envKey:   "key",
+			envRange: "key3",
+			args:     []string{"--rev", "1"},
+			wkv:      []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}},
+		},
+	}
+
+	for i, tt := range tests {
+		donec := make(chan struct{})
+		go func(i int, puts []kv) {
+			for j := range puts {
+				if err := ctlV3Put(cx, puts[j].key, puts[j].val, ""); err != nil {
+					cx.t.Fatalf("watchTest #%d-%d: ctlV3Put error (%v)", i, j, err)
+				}
+			}
+			close(donec)
+		}(i, tt.puts)
+
+		unsetEnv := func() {}
+		if tt.envKey != "" || tt.envRange != "" {
+			if tt.envKey != "" {
+				os.Setenv("ETCDCTL_WATCH_KEY", tt.envKey)
+				unsetEnv = func() { os.Unsetenv("ETCDCTL_WATCH_KEY") }
+			}
+			if tt.envRange != "" {
+				os.Setenv("ETCDCTL_WATCH_RANGE_END", tt.envRange)
+				unsetEnv = func() { os.Unsetenv("ETCDCTL_WATCH_RANGE_END") }
+			}
+			if tt.envKey != "" && tt.envRange != "" {
+				unsetEnv = func() {
+					os.Unsetenv("ETCDCTL_WATCH_KEY")
+					os.Unsetenv("ETCDCTL_WATCH_RANGE_END")
+				}
+			}
+		}
+		if err := ctlV3Watch(cx, tt.args, tt.wkv...); err != nil {
+			if cx.dialTimeout > 0 && !isGRPCTimedout(err) {
+				cx.t.Errorf("watchTest #%d: ctlV3Watch error (%v)", i, err)
+			}
+		}
+		unsetEnv()
+		<-donec
+	}
+}

tests/semaphore.test.bash

@@ -5,17 +5,34 @@ if ! [[ "$0" =~ "tests/semaphore.test.bash" ]]; then
   exit 255
 fi
 
-TEST_SUFFIX=$(date +%s | base64 | head -c 15)
+<<COMMENT
+# amd64-e2e
+bash tests/semaphore.test.bash
 
-TEST_OPTS="PASSES='build release e2e' MANUAL_VER=v3.3.3"
-if [ "$TEST_ARCH" == "386" ]; then
+# 386-e2e
+TEST_ARCH=386 bash tests/semaphore.test.bash
+
+# grpc-proxy
+TEST_OPTS="PASSES='build grpcproxy'" bash tests/semaphore.test.bash
+
+# coverage
+TEST_OPTS="coverage" bash tests/semaphore.test.bash
+COMMENT
+
+if [ -z "${TEST_OPTS}" ]; then
+	TEST_OPTS="PASSES='build release e2e' MANUAL_VER=v3.3.7"
+fi
+if [ "${TEST_ARCH}" == "386" ]; then
   TEST_OPTS="GOARCH=386 PASSES='build e2e'"
 fi
 
-docker run \
-  --rm \
-  --volume=`pwd`:/go/src/github.com/coreos/etcd \
-  gcr.io/etcd-development/etcd-test:go1.9.5 \
-  /bin/bash -c "${TEST_OPTS} ./test 2>&1 | tee test-${TEST_SUFFIX}.log"
-
-! egrep "(--- FAIL:|panic: test timed out|appears to have leaked)" -B50 -A10 test-${TEST_SUFFIX}.log
+echo "Running tests with" ${TEST_OPTS}
+if [ "${TEST_OPTS}" == "PASSES='build grpcproxy'" ]; then
+  echo "Skip proxy tests for this branch!"
+  exit 0
+elif [ "${TEST_OPTS}" == "coverage" ]; then
+  echo "Skip coverage tests for this branch!"
+  exit 0
+else
+  sudo HOST_TMP_DIR=/tmp TEST_OPTS="${TEST_OPTS}" make docker-test
+fi

version/version.go

@@ -26,7 +26,7 @@ import (
 var (
 	// MinClusterVersion is the min cluster version this etcd binary is compatible with.
 	MinClusterVersion = "3.0.0"
-	Version           = "3.3.4"
+	Version           = "3.3.8"
 	APIVersion        = "unknown"
 
 	// Git SHA Value will be set during build