Merge pull request #10018 from ymmt2005/fix_sample_conf

sample: correct directive name

.dockerignore

@@ -1 +0,0 @@
-.git

.github/ISSUE_TEMPLATE.md

@@ -1,7 +1,2 @@
-# Bug reporting
 
-A good bug report has some very specific qualities, so please read over our short document on [reporting bugs][report_bugs] before submitting a bug report.
-
-To ask a question, go ahead and ignore this.
-
-[report_bugs]: https://github.com/coreos/etcd/blob/master/Documentation/reporting_bugs.md
+Please read https://github.com/coreos/etcd/blob/master/Documentation/reporting_bugs.md.

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,5 +1,2 @@
-# Contributing guidelines
 
-Please read our [contribution workflow][contributing] before submitting a pull request.
-
-[contributing]: https://github.com/coreos/etcd/blob/master/CONTRIBUTING.md#contribution-flow
+Please read https://github.com/coreos/etcd/blob/master/CONTRIBUTING.md#contribution-flow.

.gitignore

@@ -3,20 +3,32 @@
 /covdir
 /gopath
 /gopath.proto
-/go-bindata
 /release
-/machine*
 /bin
-.Dockerfile-test
-.vagrant
 *.etcd
 *.log
 /etcd
-*.swp
 /hack/insta-discovery/.env
+*.coverprofile
 *.test
-tools/functional-tester/docker/bin
-hack/scripts-dev/docker-dns/.Dockerfile
-hack/scripts-dev/docker-dns-srv/.Dockerfile
 hack/tls-setup/certs
 .idea
+
+# TODO: use dep prune
+# https://github.com/golang/dep/issues/120#issuecomment-306518546
+vendor/**/*
+!vendor/**/
+!vendor/**/*.go
+!vendor/**/*.c
+!vendor/**/*.cpp
+!vendor/**/*.s
+!vendor/**/COPYING*
+!vendor/**/PATENTS*
+!vendor/**/NOTICE*
+!vendor/**/Licence*
+!vendor/**/License*
+!vendor/**/LICENCE*
+!vendor/**/LICENSE*
+vendor/**/*_test.go
+
+*.bak

.godir

@@ -1 +0,0 @@
-github.com/coreos/etcd

.semaphore.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-TEST_SUFFIX=$(date +%s | base64 | head -c 15)
-
-TEST_OPTS="RELEASE_TEST=y INTEGRATION=y PASSES='build unit release integration_e2e functional' MANUAL_VER=v3.2.11"
-if [ "$TEST_ARCH" == "386" ]; then
-	TEST_OPTS="GOARCH=386 PASSES='build unit integration_e2e'"
-fi
-
-docker run \
-	--rm \
-	--volume=`pwd`:/go/src/github.com/coreos/etcd \
-	gcr.io/etcd-development/etcd-test:go1.9.2 \
-	/bin/bash -c "${TEST_OPTS} ./test 2>&1 | tee test-${TEST_SUFFIX}.log"
-
-! egrep "(--- FAIL:|panic: test timed out|appears to have leaked)" -B50 -A10 test-${TEST_SUFFIX}.log

.travis.yml

@@ -6,7 +6,7 @@ sudo: required
 services: docker
 
 go:
-- 1.9.2
+- 1.10.3
 - tip
 
 notifications:
@@ -15,74 +15,116 @@ notifications:
 
 env:
   matrix:
-  - TARGET=amd64
-  - TARGET=amd64-go-tip
-  - TARGET=darwin-amd64
-  - TARGET=windows-amd64
-  - TARGET=arm64
-  - TARGET=arm
-  - TARGET=386
-  - TARGET=ppc64le
+  - TARGET=linux-amd64-fmt
+  - TARGET=linux-amd64-integration-1-cpu
+  - TARGET=linux-amd64-integration-2-cpu
+  - TARGET=linux-amd64-integration-4-cpu
+  - TARGET=linux-amd64-functional
+  - TARGET=linux-amd64-unit
+  - TARGET=all-build
+  - TARGET=linux-amd64-grpcproxy
+  - TARGET=linux-amd64-coverage
+  - TARGET=linux-amd64-fmt-unit-go-tip
+  - TARGET=linux-386-unit
 
 matrix:
   fast_finish: true
   allow_failures:
+  - go: 1.10.3
+    env: TARGET=linux-amd64-grpcproxy
+  - go: 1.10.3
+    env: TARGET=linux-amd64-coverage
   - go: tip
-    env: TARGET=amd64-go-tip
+    env: TARGET=linux-amd64-fmt-unit-go-tip
+  - go: 1.10.3
+    env: TARGET=linux-386-unit
   exclude:
-  - go: 1.9.2
-    env: TARGET=amd64-go-tip
   - go: tip
-    env: TARGET=amd64
+    env: TARGET=linux-amd64-fmt
   - go: tip
-    env: TARGET=darwin-amd64
+    env: TARGET=linux-amd64-integration-1-cpu
   - go: tip
-    env: TARGET=windows-amd64
+    env: TARGET=linux-amd64-integration-2-cpu
   - go: tip
-    env: TARGET=arm
+    env: TARGET=linux-amd64-integration-4-cpu
   - go: tip
-    env: TARGET=arm64
+    env: TARGET=linux-amd64-functional
   - go: tip
-    env: TARGET=386
+    env: TARGET=linux-amd64-unit
   - go: tip
-    env: TARGET=ppc64le
+    env: TARGET=all-build
+  - go: tip
+    env: TARGET=linux-amd64-grpcproxy
+  - go: tip
+    env: TARGET=linux-amd64-coverage
+  - go: 1.10.3
+    env: TARGET=linux-amd64-fmt-unit-go-tip
+  - go: tip
+    env: TARGET=linux-386-unit
 
 before_install:
-- docker pull gcr.io/etcd-development/etcd-test:go1.9.2
+- if [[ $TRAVIS_GO_VERSION == 1.* ]]; then docker pull gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION}; fi
 
 install:
-- pushd cmd/etcd && go get -t -v ./... && popd
+- go get -t -d ./...
 
 script:
+ - echo "TRAVIS_GO_VERSION=${TRAVIS_GO_VERSION}"
  - >
     case "${TARGET}" in
-      amd64)
+      linux-amd64-fmt)
         docker run --rm \
-          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go1.9.2 \
-          /bin/bash -c "GOARCH=amd64 ./test"
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
+          /bin/bash -c "GOARCH=amd64 PASSES='fmt bom dep' ./test"
         ;;
-      amd64-go-tip)
-        GOARCH=amd64 ./test
-        ;;
-      darwin-amd64)
+      linux-amd64-integration-1-cpu)
         docker run --rm \
-          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go1.9.2 \
-          /bin/bash -c "GO_BUILD_FLAGS='-a -v' GOOS=darwin GOARCH=amd64 ./build"
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
+          /bin/bash -c "GOARCH=amd64 CPU=1 PASSES='integration' ./test"
         ;;
-      windows-amd64)
+      linux-amd64-integration-2-cpu)
         docker run --rm \
-          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go1.9.2 \
-          /bin/bash -c "GO_BUILD_FLAGS='-a -v' GOOS=windows GOARCH=amd64 ./build"
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
+          /bin/bash -c "GOARCH=amd64 CPU=2 PASSES='integration' ./test"
         ;;
-      386)
+      linux-amd64-integration-4-cpu)
         docker run --rm \
-          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go1.9.2 \
-          /bin/bash -c "GOARCH=386 PASSES='build unit' ./test"
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
+          /bin/bash -c "GOARCH=amd64 CPU=4 PASSES='integration' ./test"
         ;;
-      *)
-        # test building out of gopath
+      linux-amd64-functional)
         docker run --rm \
-          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go1.9.2 \
-          /bin/bash -c "GO_BUILD_FLAGS='-a -v' GOARCH='${TARGET}' ./build"
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
+          /bin/bash -c "./build && GOARCH=amd64 PASSES='functional' ./test"
+        ;;
+      linux-amd64-unit)
+        docker run --rm \
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
+          /bin/bash -c "GOARCH=amd64 PASSES='unit' ./test"
+        ;;
+      all-build)
+        docker run --rm \
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
+          /bin/bash -c "GOARCH=amd64 PASSES='build' ./test \
+            && GOARCH=386 PASSES='build' ./test \
+            && GO_BUILD_FLAGS='-v' GOOS=darwin GOARCH=amd64 ./build \
+            && GO_BUILD_FLAGS='-v' GOOS=windows GOARCH=amd64 ./build \
+            && GO_BUILD_FLAGS='-v' GOARCH=arm ./build \
+            && GO_BUILD_FLAGS='-v' GOARCH=arm64 ./build \
+            && GO_BUILD_FLAGS='-v' GOARCH=ppc64le ./build"
+        ;;
+      linux-amd64-grpcproxy)
+        sudo HOST_TMP_DIR=/tmp TEST_OPTS="PASSES='build grpcproxy'" make docker-test
+        ;;
+      linux-amd64-coverage)
+        sudo HOST_TMP_DIR=/tmp make docker-test-coverage
+        ;;
+      linux-amd64-fmt-unit-go-tip)
+        GOARCH=amd64 PASSES='fmt unit' ./test
+        ;;
+      linux-386-unit)
+        docker run --rm \
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
+          /bin/bash -c "GOARCH=386 PASSES='unit' ./test"
         ;;
     esac

.words

@@ -8,7 +8,10 @@ MiB
 ResourceExhausted
 RPC
 RPCs
-TODO
+
+WithRequireLeader
+InfoLevel
+args
 backoff
 blackhole
 blackholed
@@ -17,27 +20,86 @@ cancelation
 cluster_proxy
 defragment
 defragmenting
+deleter
+dev
+/dev/null
+dev/null
+errClientDisconnected
 etcd
 gRPC
 goroutine
 goroutines
 healthcheck
+hostname
 iff
 inflight
 keepalive
 keepalives
 keyspace
 linearization
+liveness
+linearized
 localhost
 mutex
 prefetching
 protobuf
 prometheus
+rafthttp
 repin
+rpc
 serializable
+statusError
 teardown
 too_many_pings
+transactional
 uncontended
 unprefixed
 unlisting
+nondeterministically
+atomics
+transferee
+Balancer
+lexicographically
+lexically
+accessors
+unbuffered
+nils
+reconnection
+mutators
+ConsistentIndexGetter
+OutputWALDir
+WAL
+consistentIndex
+todo
+saveWALAndSnap
 
+subconns
+nop
+SubConns
+DNS
+passthrough
+ccBalancerWrapper
+rebalanced
+addrConns
+subConn
+TestBalancerDoNotBlockOnClose
+middleware
+clusterName
+jitter
+FIXME
+retriable
+github
+retriable
+jitter
+WithBackoff
+BackoffLinearWithJitter
+jitter
+WithMax
+ServerStreams
+BidiStreams
+transientFailure
+BackoffFunc
+CallOptions
+
+__lostleader
+ErrConnClosing

CHANGELOG-2.3.md

@@ -0,0 +1,10 @@
+
+
+## [v2.3.8](https://github.com/coreos/etcd/releases/tag/v2.3.8) (2017-02-17)
+
+See [code changes](https://github.com/coreos/etcd/compare/v2.3.7...v2.3.8).
+
+### Go
+
+- Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7).
+

CHANGELOG-3.0.md

@@ -0,0 +1,203 @@
+
+
+## [v3.0.16](https://github.com/coreos/etcd/releases/tag/v3.0.16) (2016-11-13)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.0.15...v3.0.16) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md).**
+
+### Go
+
+- Compile with [*Go 1.6.4*](https://golang.org/doc/devel/release.html#go1.6).
+
+
+## [v3.0.15](https://github.com/coreos/etcd/releases/tag/v3.0.15) (2016-11-11)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.0.14...v3.0.15) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md).**
+
+### Fixed
+
+- Fix cancel watch request with wrong range end.
+
+### Go
+
+- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6).
+
+
+## [v3.0.14](https://github.com/coreos/etcd/releases/tag/v3.0.14) (2016-11-04)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.0.13...v3.0.14) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md).**
+
+### Added
+
+- v3 `etcdctl migrate` command now supports `--no-ttl` flag to discard keys on transform.
+
+### Go
+
+- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6).
+
+
+## [v3.0.13](https://github.com/coreos/etcd/releases/tag/v3.0.13) (2016-10-24)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.0.12...v3.0.13) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md).**
+
+### Go
+
+- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6).
+
+
+## [v3.0.12](https://github.com/coreos/etcd/releases/tag/v3.0.12) (2016-10-07)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.0.11...v3.0.12) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md).**
+
+### Go
+
+- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6).
+
+
+## [v3.0.11](https://github.com/coreos/etcd/releases/tag/v3.0.11) (2016-10-07)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.0.10...v3.0.11) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md).**
+
+### Added
+
+- Server returns previous key-value (optional)
+  - `clientv3.WithPrevKV` option
+  - v3 etcdctl `put,watch,del --prev-kv` flag
+
+### Go
+
+- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6).
+
+
+## [v3.0.10](https://github.com/coreos/etcd/releases/tag/v3.0.10) (2016-09-23)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.0.9...v3.0.10) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md).**
+
+### Go
+
+- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6).
+
+
+## [v3.0.9](https://github.com/coreos/etcd/releases/tag/v3.0.9) (2016-09-15)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.0.8...v3.0.9) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md).**
+
+### Added
+
+- Warn on domain names on listen URLs (v3.2 will reject domain names).
+
+### Go
+
+- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6).
+
+
+## [v3.0.8](https://github.com/coreos/etcd/releases/tag/v3.0.8) (2016-09-09)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.0.7...v3.0.8) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md).**
+
+### Other
+
+- Allow only IP addresses in listen URLs (domain names are rejected).
+
+### Go
+
+- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6).
+
+
+## [v3.0.7](https://github.com/coreos/etcd/releases/tag/v3.0.7) (2016-08-31)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.0.6...v3.0.7) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md).**
+
+### Other
+
+- SRV records only allow A records (RFC 2052).
+
+### Go
+
+- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6).
+
+
+## [v3.0.6](https://github.com/coreos/etcd/releases/tag/v3.0.6) (2016-08-19)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.0.5...v3.0.6) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md).**
+
+### Go
+
+- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6).
+
+
+## [v3.0.5](https://github.com/coreos/etcd/releases/tag/v3.0.5) (2016-08-19)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.0.4...v3.0.5) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md).**
+
+### Other
+
+- SRV records (e.g., infra1.example.com) must match the discovery domain (i.e., example.com) if no custom certificate authority is given.
+
+### Go
+
+- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6).
+
+
+## [v3.0.4](https://github.com/coreos/etcd/releases/tag/v3.0.4) (2016-07-27)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.0.3...v3.0.4) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md).**
+
+### Added
+
+- v2 `etcdctl ls` command now supports `--output=json`.
+- Add /var/lib/etcd directory to etcd official Docker image.
+
+### Other
+
+- v2 auth can now use common name from TLS certificate when `--client-cert-auth` is enabled.
+
+### Go
+
+- Compile with [*Go 1.6.3*](https://golang.org/doc/devel/release.html#go1.6).
+
+
+## [v3.0.3](https://github.com/coreos/etcd/releases/tag/v3.0.3) (2016-07-15)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.0.2...v3.0.3) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md).**
+
+### Other
+
+- Revert Dockerfile to use `CMD`, instead of `ENTRYPOINT`, to support `etcdctl` run.
+  - Docker commands for v3.0.2 won't work without specifying executable binary paths.
+- v3 etcdctl default endpoints are now `127.0.0.1:2379`.
+
+### Go
+
+- Compile with [*Go 1.6.2*](https://golang.org/doc/devel/release.html#go1.6).
+
+
+## [v3.0.2](https://github.com/coreos/etcd/releases/tag/v3.0.2) (2016-07-08)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.0.1...v3.0.2) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md).**
+
+### Other
+
+- Dockerfile uses `ENTRYPOINT`, instead of `CMD`, to run etcd without binary path specified.
+
+### Go
+
+- Compile with [*Go 1.6.2*](https://golang.org/doc/devel/release.html#go1.6).
+
+
+## [v3.0.1](https://github.com/coreos/etcd/releases/tag/v3.0.1) (2016-07-01)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.0.0...v3.0.1) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md).**
+
+### Go
+
+- Compile with [*Go 1.6.2*](https://golang.org/doc/devel/release.html#go1.6).
+
+
+## [v3.0.0](https://github.com/coreos/etcd/releases/tag/v3.0.0) (2016-06-30)
+
+See [code changes](https://github.com/coreos/etcd/compare/v2.3.0...v3.0.0) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md).**
+
+### Go
+
+- Compile with [*Go 1.6.2*](https://golang.org/doc/devel/release.html#go1.6).
+

CHANGELOG-3.1.md

@@ -0,0 +1,405 @@
+
+
+Previous change logs can be found at [CHANGELOG-3.0](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.0.md).
+
+
+## [v3.1.19](https://github.com/coreos/etcd/releases/tag/v3.1.19) (2018-07-24)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.1.18...v3.1.19) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md).**
+
+### Improved
+
+- Improve [Raft Read Index timeout warning messages](https://github.com/coreos/etcd/pull/9897).
+
+### Metrics, Monitoring
+
+See [List of metrics](https://etcd.readthedocs.io/en/latest/operate.html#v3-1) for all metrics per release.
+
+Note that any `etcd_debugging_*` metrics are experimental and subject to change.
+
+- Add [`etcd_server_go_version`](https://github.com/coreos/etcd/pull/9957) Prometheus metric.
+- Add [`etcd_server_slow_read_indexes_total`](https://github.com/coreos/etcd/pull/9897) Prometheus metric.
+- Add [`etcd_server_quota_backend_bytes`](https://github.com/coreos/etcd/pull/9820) Prometheus metric.
+  - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_mvcc_db_total_size_in_use_in_bytes`.
+  - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB.
+  - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB.
+  - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete.
+  - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation.
+- Add [`etcd_mvcc_db_total_size_in_bytes`](https://github.com/coreos/etcd/pull/9819) Prometheus metric.
+  - In addition to [`etcd_debugging_mvcc_db_total_size_in_bytes`](https://github.com/coreos/etcd/pull/9819).
+- Add [`etcd_mvcc_db_total_size_in_use_in_bytes`](https://github.com/coreos/etcd/pull/9256) Prometheus metric.
+  - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_mvcc_db_total_size_in_use_in_bytes`.
+  - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB.
+  - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB.
+  - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete.
+  - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation.
+
+### client v3
+
+- Fix [lease keepalive interval updates when response queue is full](https://github.com/coreos/etcd/pull/9952).
+  - If `<-chan *clientv3LeaseKeepAliveResponse` from `clientv3.Lease.KeepAlive` was never consumed or channel is full, client was [sending keepalive request every 500ms](https://github.com/coreos/etcd/issues/9911) instead of expected rate of every "TTL / 3" duration.
+
+### Go
+
+- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.1.18](https://github.com/coreos/etcd/releases/tag/v3.1.18) (2018-06-15)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.1.17...v3.1.18) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md).**
+
+### Metrics, Monitoring
+
+See [List of metrics](https://etcd.readthedocs.io/en/latest/operate.html#v3-1) for all metrics per release.
+
+Note that any `etcd_debugging_*` metrics are experimental and subject to change.
+
+- Add [`etcd_server_version`](https://github.com/coreos/etcd/pull/8960) Prometheus metric.
+  - To replace [Kubernetes `etcd-version-monitor`](https://github.com/coreos/etcd/issues/8948).
+
+### Go
+
+- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.1.17](https://github.com/coreos/etcd/releases/tag/v3.1.17) (2018-06-06)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.1.16...v3.1.17) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md).**
+
+### etcd server
+
+- Fix [v3 snapshot recovery](https://github.com/coreos/etcd/issues/7628).
+  - A follower receives a leader snapshot to be persisted as a `[SNAPSHOT-INDEX].snap.db` file on disk.
+  - Now, server [ensures that the incoming snapshot be persisted on disk before loading it](https://github.com/coreos/etcd/pull/7876).
+  - Otherwise, index mismatch happens and triggers server-side panic (e.g. newer WAL entry with outdated snapshot index).
+
+### Go
+
+- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.1.16](https://github.com/coreos/etcd/releases/tag/v3.1.16) (2018-05-31)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.1.15...v3.1.16) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md).**
+
+### etcd server
+
+- Fix [`mvcc` server panic from restore operation](https://github.com/coreos/etcd/pull/9775).
+  - Let's assume that a watcher had been requested with a future revision X and sent to node A that became network-partitioned thereafter. Meanwhile, cluster makes progress. Then when the partition gets removed, the leader sends a snapshot to node A. Previously if the snapshot's latest revision is still lower than the watch revision X,  **etcd server panicked** during snapshot restore operation.
+  - Now, this server-side panic has been fixed.
+
+### Go
+
+- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.1.15](https://github.com/coreos/etcd/releases/tag/v3.1.15) (2018-05-09)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.1.14...v3.1.15) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md).**
+
+### etcd server
+
+- Purge old [`*.snap.db` snapshot files](https://github.com/coreos/etcd/pull/7967).
+  - Previously, etcd did not respect `--max-snapshots` flag to purge old `*.snap.db` files.
+  - Now, etcd purges old `*.snap.db` files to keep maximum `--max-snapshots` number of files on disk.
+
+### Go
+
+- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.1.14](https://github.com/coreos/etcd/releases/tag/v3.1.14) (2018-04-24)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.1.13...v3.1.14) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md).**
+
+### Metrics, Monitoring
+
+See [List of metrics](https://etcd.readthedocs.io/en/latest/operate.html#v3-1) for all metrics per release.
+
+Note that any `etcd_debugging_*` metrics are experimental and subject to change.
+
+- Add [`etcd_server_is_leader`](https://github.com/coreos/etcd/pull/9587) Prometheus metric.
+
+### etcd server
+
+- Add [`--initial-election-tick-advance`](https://github.com/coreos/etcd/pull/9591) flag to configure initial election tick fast-forward.
+  - By default, `--initial-election-tick-advance=true`, then local member fast-forwards election ticks to speed up "initial" leader election trigger.
+  - This benefits the case of larger election ticks. For instance, cross datacenter deployment may require longer election timeout of 10-second. If true, local node does not need wait up to 10-second. Instead, forwards its election ticks to 8-second, and have only 2-second left before leader election.
+  - Major assumptions are that: cluster has no active leader thus advancing ticks enables faster leader election. Or cluster already has an established leader, and rejoining follower is likely to receive heartbeats from the leader after tick advance and before election timeout.
+  - However, when network from leader to rejoining follower is congested, and the follower does not receive leader heartbeat within left election ticks, disruptive election has to happen thus affecting cluster availabilities.
+  - Now, this can be disabled by setting `--initial-election-tick-advance=false`.
+  - Disabling this would slow down initial bootstrap process for cross datacenter deployments. Make tradeoffs by configuring `--initial-election-tick-advance` at the cost of slow initial bootstrap.
+  - If single-node, it advances ticks regardless.
+  - Address [disruptive rejoining follower node](https://github.com/coreos/etcd/issues/9333).
+
+### Go
+
+- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.1.13](https://github.com/coreos/etcd/releases/tag/v3.1.13) (2018-03-29)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.1.12...v3.1.13) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md).**
+
+### Improved
+
+- Adjust [election timeout on server restart](https://github.com/coreos/etcd/pull/9415) to reduce [disruptive rejoining servers](https://github.com/coreos/etcd/issues/9333).
+  - Previously, etcd fast-forwards election ticks on server start, with only one tick left for leader election. This is to speed up start phase, without having to wait until all election ticks elapse. Advancing election ticks is useful for cross datacenter deployments with larger election timeouts. However, it was affecting cluster availability if the last tick elapses before leader contacts the restarted node.
+  - Now, when etcd restarts, it adjusts election ticks with more than one tick left, thus more time for leader to prevent disruptive restart.
+
+### Metrics, Monitoring
+
+See [List of metrics](https://etcd.readthedocs.io/en/latest/operate.html#v3-1) for all metrics per release.
+
+Note that any `etcd_debugging_*` metrics are experimental and subject to change.
+
+- Add missing [`etcd_network_peer_sent_failures_total` count](https://github.com/coreos/etcd/pull/9437).
+
+### Go
+
+- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.1.12](https://github.com/coreos/etcd/releases/tag/v3.1.12) (2018-03-08)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.1.11...v3.1.12) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md).**
+
+### etcd server
+
+- Fix [`mvcc` "unsynced" watcher restore operation](https://github.com/coreos/etcd/pull/9297).
+  - "unsynced" watcher is watcher that needs to be in sync with events that have happened.
+  - That is, "unsynced" watcher is the slow watcher that was requested on old revision.
+  - "unsynced" watcher restore operation was not correctly populating its underlying watcher group.
+  - Which possibly causes [missing events from "unsynced" watchers](https://github.com/coreos/etcd/issues/9086).
+  - A node gets network partitioned with a watcher on a future revision, and falls behind receiving a leader snapshot after partition gets removed. When applying this snapshot, etcd watch storage moves current synced watchers to unsynced since sync watchers might have become stale during network partition. And reset synced watcher group to restart watcher routines. Previously, there was a bug when moving from synced watcher group to unsynced, thus client would miss events when the watcher was requested to the network-partitioned node.
+
+### Go
+
+- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.1.11](https://github.com/coreos/etcd/releases/tag/v3.1.11) (2017-11-28)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.1.10...v3.1.11) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md).**
+
+### etcd server
+
+- [#8411](https://github.com/coreos/etcd/issues/8411),[#8806](https://github.com/coreos/etcd/pull/8806) backport "mvcc: sending events after restore"
+- [#8009](https://github.com/coreos/etcd/issues/8009),[#8902](https://github.com/coreos/etcd/pull/8902) backport coreos/bbolt v1.3.1-coreos.5
+
+### Go
+
+- Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.1.10](https://github.com/coreos/etcd/releases/tag/v3.1.10) (2017-07-14)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.1.9...v3.1.10) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md).**
+
+### Added
+
+- Tag docker images with minor versions.
+  - e.g. `docker pull quay.io/coreos/etcd:v3.1` to fetch latest v3.1 versions.
+
+### Go
+
+- Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8).
+  - Fix panic on `net/http.CloseNotify`
+
+
+## [v3.1.9](https://github.com/coreos/etcd/releases/tag/v3.1.9) (2017-06-09)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.1.8...v3.1.9) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md).**
+
+### etcd server
+
+- Allow v2 snapshot over 512MB.
+
+### Go
+
+- Compile with [*Go 1.7.6*](https://golang.org/doc/devel/release.html#go1.7).
+
+
+## [v3.1.8](https://github.com/coreos/etcd/releases/tag/v3.1.8) (2017-05-19)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.1.7...v3.1.8) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md).**
+
+### Go
+
+- Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7).
+
+
+## [v3.1.7](https://github.com/coreos/etcd/releases/tag/v3.1.7) (2017-04-28)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.1.6...v3.1.7) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md).**
+
+### Go
+
+- Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7).
+
+
+## [v3.1.6](https://github.com/coreos/etcd/releases/tag/v3.1.6) (2017-04-19)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.1.5...v3.1.6) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md).**
+
+### etcd server
+
+- Fill in Auth API response header.
+- Remove auth check in Status API.
+
+### Go
+
+- Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7).
+
+
+## [v3.1.5](https://github.com/coreos/etcd/releases/tag/v3.1.5) (2017-03-27)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.1.4...v3.1.5) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md).**
+
+### etcd server
+
+- Fix raft memory leak issue.
+- Fix Windows file path issues.
+
+### Other
+
+- Add `/etc/nsswitch.conf` file to alpine-based Docker image.
+
+### Go
+
+- Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7).
+
+
+## [v3.1.4](https://github.com/coreos/etcd/releases/tag/v3.1.4) (2017-03-22)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.1.3...v3.1.4) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md).**
+
+### Go
+
+- Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7).
+
+
+## [v3.1.3](https://github.com/coreos/etcd/releases/tag/v3.1.3) (2017-03-10)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.1.2...v3.1.3) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md).**
+
+### etcd gateway
+
+- Fix `etcd gateway` schema handling in DNS discovery.
+- Fix sd_notify behaviors in `gateway`, `grpc-proxy`.
+
+### gRPC Proxy
+
+- Fix sd_notify behaviors in `gateway`, `grpc-proxy`.
+
+### Other
+
+- Use machine default host when advertise URLs are default values(`localhost:2379,2380`) AND if listen URL is `0.0.0.0`.
+
+### Go
+
+- Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7).
+
+
+## [v3.1.2](https://github.com/coreos/etcd/releases/tag/v3.1.2) (2017-02-24)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.1.1...v3.1.2) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md).**
+
+### etcd gateway
+
+- Fix `etcd gateway` with multiple endpoints.
+
+### Other
+
+- Use IPv4 default host, by default (when IPv4 and IPv6 are available).
+
+### Go
+
+- Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7).
+
+
+## [v3.1.1](https://github.com/coreos/etcd/releases/tag/v3.1.1) (2017-02-17)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.1.0...v3.1.1) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md).**
+
+### Go
+
+- Compile with [*Go 1.7.5*](https://golang.org/doc/devel/release.html#go1.7).
+
+
+## [v3.1.0](https://github.com/coreos/etcd/releases/tag/v3.1.0) (2017-01-20)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.0.0...v3.1.0) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md).**
+
+### Improved
+
+- Faster linearizable reads (implements Raft [read-index](https://github.com/coreos/etcd/pull/6212)).
+- v3 authentication API is now stable.
+
+### Breaking Changes
+
+- Deprecated following gRPC metrics in favor of [go-grpc-prometheus](https://github.com/grpc-ecosystem/go-grpc-prometheus).
+  - `etcd_grpc_requests_total`
+  - `etcd_grpc_requests_failed_total`
+  - `etcd_grpc_active_streams`
+  - `etcd_grpc_unary_requests_duration_seconds`
+
+### Dependency
+
+- Upgrade [`github.com/ugorji/go/codec`](https://github.com/ugorji/go) to [**`ugorji/go@9c7f9b7`**](https://github.com/ugorji/go/commit/9c7f9b7a2bc3a520f7c7b30b34b7f85f47fe27b6), and [regenerate v2 `client`](https://github.com/coreos/etcd/pull/6945).
+
+### Security, Authentication
+
+See [security doc](https://github.com/coreos/etcd/blob/master/Documentation/op-guide/security.md) for more details.
+
+- SRV records (e.g., infra1.example.com) must match the discovery domain (i.e., example.com) if no custom certificate authority is given.
+  - `TLSConfig.ServerName` is ignored with user-provided certificates for backwards compatibility; to be deprecated.
+  - For example, `etcd --discovery-srv=example.com` will only authenticate peers/clients when the provided certs have root domain `example.com` as an entry in Subject Alternative Name (SAN) field.
+
+### etcd server
+
+- Automatic leadership transfer when leader steps down.
+- etcd flags
+  - `--strict-reconfig-check` flag is set by default.
+  - Add `--log-output` flag.
+  - Add `--metrics` flag.
+- etcd uses default route IP if advertise URL is not given.
+- Cluster rejects removing members if quorum will be lost.
+- Discovery now has upper limit for waiting on retries.
+- Warn on binding listeners through domain names; to be deprecated.
+- v3.0 and v3.1 with `--auto-compaction-retention=10` run periodic compaction on v3 key-value store for every 10-hour.
+  - Compactor only supports periodic compaction.
+  - Compactor records latest revisions every 5-minute, until it reaches the first compaction period (e.g. 10-hour).
+  - In order to retain key-value history of last compaction period, it uses the last revision that was fetched before compaction period, from the revision records that were collected every 5-minute.
+  - When `--auto-compaction-retention=10`, compactor uses revision 100 for compact revision where revision 100 is the latest revision fetched from 10 hours ago.
+  - If compaction succeeds or requested revision has already been compacted, it resets period timer and starts over with new historical revision records (e.g. restart revision collect and compact for the next 10-hour period).
+  - If compaction fails, it retries in 5 minutes.
+
+### client v3
+
+- Add `SetEndpoints` method; update endpoints at runtime.
+- Add `Sync` method; auto-update endpoints at runtime.
+- Add `Lease TimeToLive` API; fetch lease information.
+- replace Config.Logger field with global logger.
+- Get API responses are sorted in ascending order by default.
+
+### etcdctl v3
+
+- Add `lease timetolive` command.
+- Add `--print-value-only` flag to get command.
+- Add `--dest-prefix` flag to make-mirror command.
+- `get` command responses are sorted in ascending order by default.
+
+### gRPC Proxy
+
+- Experimental gRPC proxy feature.
+
+### Other
+
+- `recipes` now conform to sessions defined in `clientv3/concurrency`.
+- ACI has symlinks to `/usr/local/bin/etcd*`.
+
+### Go
+
+- Compile with [*Go 1.7.4*](https://golang.org/doc/devel/release.html#go1.7).
+

CHANGELOG-3.2.md

@@ -0,0 +1,665 @@
+
+
+Previous change logs can be found at [CHANGELOG-3.1](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.1.md).
+
+## [v3.2.24](https://github.com/coreos/etcd/releases/tag/v3.2.24) (2018-07-24)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.23...v3.2.24) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### Improved
+
+- Improve [Raft Read Index timeout warning messages](https://github.com/coreos/etcd/pull/9897).
+
+### Metrics, Monitoring
+
+See [List of metrics](https://etcd.readthedocs.io/en/latest/operate.html#v3-2) for all metrics per release.
+
+Note that any `etcd_debugging_*` metrics are experimental and subject to change.
+
+- Add [`etcd_server_go_version`](https://github.com/coreos/etcd/pull/9957) Prometheus metric.
+- Add [`etcd_server_heartbeat_send_failures_total`](https://github.com/coreos/etcd/pull/9942) Prometheus metric.
+- Add [`etcd_server_slow_apply_total`](https://github.com/coreos/etcd/pull/9942) Prometheus metric.
+- Add [`etcd_disk_backend_defrag_duration_seconds`](https://github.com/coreos/etcd/pull/9942) Prometheus metric.
+- Add [`etcd_mvcc_hash_duration_seconds`](https://github.com/coreos/etcd/pull/9942) Prometheus metric.
+- Add [`etcd_server_slow_read_indexes_total`](https://github.com/coreos/etcd/pull/9897) Prometheus metric.
+- Add [`etcd_server_quota_backend_bytes`](https://github.com/coreos/etcd/pull/9820) Prometheus metric.
+  - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_mvcc_db_total_size_in_use_in_bytes`.
+  - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB.
+  - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB.
+  - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete.
+  - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation.
+- Add [`etcd_mvcc_db_total_size_in_bytes`](https://github.com/coreos/etcd/pull/9819) Prometheus metric.
+  - In addition to [`etcd_debugging_mvcc_db_total_size_in_bytes`](https://github.com/coreos/etcd/pull/9819).
+- Add [`etcd_mvcc_db_total_size_in_use_in_bytes`](https://github.com/coreos/etcd/pull/9256) Prometheus metric.
+  - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_server_quota_backend_bytes`.
+  - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB.
+  - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB.
+  - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete.
+  - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation.
+
+### gRPC Proxy
+
+- Add [flags for specifying TLS for connecting to proxy](https://github.com/coreos/etcd/pull/9894):
+  - Add `grpc-proxy start --cert-file`, `grpc-proxy start --key-file` and `grpc-proxy start --trusted-ca-file` flags.
+- Add [`grpc-proxy start --metrics-addr` flag for specifying a separate metrics listen address](https://github.com/coreos/etcd/pull/9894).
+
+### client v3
+
+- Fix [lease keepalive interval updates when response queue is full](https://github.com/coreos/etcd/pull/9952).
+  - If `<-chan *clientv3LeaseKeepAliveResponse` from `clientv3.Lease.KeepAlive` was never consumed or channel is full, client was [sending keepalive request every 500ms](https://github.com/coreos/etcd/issues/9911) instead of expected rate of every "TTL / 3" duration.
+
+### Go
+
+- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.23](https://github.com/coreos/etcd/releases/tag/v3.2.23) (2018-06-15)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.22...v3.2.23) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### Improved
+
+- Improve [slow request apply warning log](https://github.com/coreos/etcd/pull/9288).
+  - e.g. `read-only range request "key:\"/a\" range_end:\"/b\" " with result "range_response_count:3 size:96" took too long (97.966µs) to execute`.
+  - Redact [request value field](https://github.com/coreos/etcd/pull/9822).
+  - Provide [response size](https://github.com/coreos/etcd/pull/9826).
+- Add [backoff on watch retries on transient errors](https://github.com/coreos/etcd/pull/9840).
+
+### Metrics, Monitoring
+
+See [List of metrics](https://etcd.readthedocs.io/en/latest/operate.html#v3-2) for all metrics per release.
+
+Note that any `etcd_debugging_*` metrics are experimental and subject to change.
+
+- Add [`etcd_server_version`](https://github.com/coreos/etcd/pull/8960) Prometheus metric.
+  - To replace [Kubernetes `etcd-version-monitor`](https://github.com/coreos/etcd/issues/8948).
+
+### Go
+
+- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.22](https://github.com/coreos/etcd/releases/tag/v3.2.22) (2018-06-06)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.21...v3.2.22) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### Security, Authentication
+
+- Support TLS cipher suite whitelisting.
+  - To block [weak cipher suites](https://github.com/coreos/etcd/issues/8320).
+  - TLS handshake fails when client hello is requested with invalid cipher suites.
+  - Add [`etcd --cipher-suites`](https://github.com/coreos/etcd/pull/9801) flag.
+  - If empty, Go auto-populates the list.
+
+### Go
+
+- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.21](https://github.com/coreos/etcd/releases/tag/v3.2.21) (2018-05-31)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.20...v3.2.21) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### etcd server
+
+- Fix [auth storage panic when simple token provider is disabled](https://github.com/coreos/etcd/pull/8695).
+- Fix [`mvcc` server panic from restore operation](https://github.com/coreos/etcd/pull/9775).
+  - Let's assume that a watcher had been requested with a future revision X and sent to node A that became network-partitioned thereafter. Meanwhile, cluster makes progress. Then when the partition gets removed, the leader sends a snapshot to node A. Previously if the snapshot's latest revision is still lower than the watch revision X,  **etcd server panicked** during snapshot restore operation.
+  - Now, this server-side panic has been fixed.
+
+### Go
+
+- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.20](https://github.com/coreos/etcd/releases/tag/v3.2.20) (2018-05-09)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.19...v3.2.20) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### etcd server
+
+- Purge old [`*.snap.db` snapshot files](https://github.com/coreos/etcd/pull/7967).
+  - Previously, etcd did not respect `--max-snapshots` flag to purge old `*.snap.db` files.
+  - Now, etcd purges old `*.snap.db` files to keep maximum `--max-snapshots` number of files on disk.
+
+### Go
+
+- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.19](https://github.com/coreos/etcd/releases/tag/v3.2.19) (2018-04-24)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.18...v3.2.19) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### Metrics, Monitoring
+
+See [List of metrics](https://etcd.readthedocs.io/en/latest/operate.html#v3-2) for all metrics per release.
+
+Note that any `etcd_debugging_*` metrics are experimental and subject to change.
+
+- Fix [`etcd_debugging_server_lease_expired_total`](https://github.com/coreos/etcd/pull/9557) Prometheus metric.
+- Fix [race conditions in v2 server stat collecting](https://github.com/coreos/etcd/pull/9562).
+- Add [`etcd_server_is_leader`](https://github.com/coreos/etcd/pull/9587) Prometheus metric.
+
+### Security, Authentication
+
+- Fix [TLS reload](https://github.com/coreos/etcd/pull/9570) when [certificate SAN field only includes IP addresses but no domain names](https://github.com/coreos/etcd/issues/9541).
+  - In Go, server calls `(*tls.Config).GetCertificate` for TLS reload if and only if server's `(*tls.Config).Certificates` field is not empty, or `(*tls.ClientHelloInfo).ServerName` is not empty with a valid SNI from the client. Previously, etcd always populates `(*tls.Config).Certificates` on the initial client TLS handshake, as non-empty. Thus, client was always expected to supply a matching SNI in order to pass the TLS verification and to trigger `(*tls.Config).GetCertificate` to reload TLS assets.
+  - However, a certificate whose SAN field does [not include any domain names but only IP addresses](https://github.com/coreos/etcd/issues/9541) would request `*tls.ClientHelloInfo` with an empty `ServerName` field, thus failing to trigger the TLS reload on initial TLS handshake; this becomes a problem when expired certificates need to be replaced online.
+  - Now, `(*tls.Config).Certificates` is created empty on initial TLS client handshake, first to trigger `(*tls.Config).GetCertificate`, and then to populate rest of the certificates on every new TLS connection, even when client SNI is empty (e.g. cert only includes IPs).
+
+### etcd server
+
+- Add [`etcd --initial-election-tick-advance`](https://github.com/coreos/etcd/pull/9591) flag to configure initial election tick fast-forward.
+  - By default, `etcd --initial-election-tick-advance=true`, then local member fast-forwards election ticks to speed up "initial" leader election trigger.
+  - This benefits the case of larger election ticks. For instance, cross datacenter deployment may require longer election timeout of 10-second. If true, local node does not need wait up to 10-second. Instead, forwards its election ticks to 8-second, and have only 2-second left before leader election.
+  - Major assumptions are that: cluster has no active leader thus advancing ticks enables faster leader election. Or cluster already has an established leader, and rejoining follower is likely to receive heartbeats from the leader after tick advance and before election timeout.
+  - However, when network from leader to rejoining follower is congested, and the follower does not receive leader heartbeat within left election ticks, disruptive election has to happen thus affecting cluster availabilities.
+  - Now, this can be disabled by setting `--initial-election-tick-advance=false`.
+  - Disabling this would slow down initial bootstrap process for cross datacenter deployments. Make tradeoffs by configuring `--initial-election-tick-advance` at the cost of slow initial bootstrap.
+  - If single-node, it advances ticks regardless.
+  - Address [disruptive rejoining follower node](https://github.com/coreos/etcd/issues/9333).
+
+### Go
+
+- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.18](https://github.com/coreos/etcd/releases/tag/v3.2.18) (2018-03-29)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.17...v3.2.18) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### Improved
+
+- Adjust [election timeout on server restart](https://github.com/coreos/etcd/pull/9415) to reduce [disruptive rejoining servers](https://github.com/coreos/etcd/issues/9333).
+  - Previously, etcd fast-forwards election ticks on server start, with only one tick left for leader election. This is to speed up start phase, without having to wait until all election ticks elapse. Advancing election ticks is useful for cross datacenter deployments with larger election timeouts. However, it was affecting cluster availability if the last tick elapses before leader contacts the restarted node.
+  - Now, when etcd restarts, it adjusts election ticks with more than one tick left, thus more time for leader to prevent disruptive restart.
+
+### Metrics, Monitoring
+
+See [List of metrics](https://etcd.readthedocs.io/en/latest/operate.html#v3-2) for all metrics per release.
+
+Note that any `etcd_debugging_*` metrics are experimental and subject to change.
+
+- Add missing [`etcd_network_peer_sent_failures_total` count](https://github.com/coreos/etcd/pull/9437).
+
+### Go
+
+- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.17](https://github.com/coreos/etcd/releases/tag/v3.2.17) (2018-03-08)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.16...v3.2.17) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### etcd server
+
+- Fix [server panic on invalid Election Proclaim/Resign HTTP(S) requests](https://github.com/coreos/etcd/pull/9379).
+  - Previously, wrong-formatted HTTP requests to Election API could trigger panic in etcd server.
+  - e.g. `curl -L http://localhost:2379/v3/election/proclaim -X POST -d '{"value":""}'`, `curl -L http://localhost:2379/v3/election/resign -X POST -d '{"value":""}'`.
+- Prevent [overflow by large `TTL` values for `Lease` `Grant`](https://github.com/coreos/etcd/pull/9399).
+  - `TTL` parameter to `Grant` request is unit of second.
+  - Leases with too large `TTL` values exceeding `math.MaxInt64` [expire in unexpected ways](https://github.com/coreos/etcd/issues/9374).
+  - Server now returns `rpctypes.ErrLeaseTTLTooLarge` to client, when the requested `TTL` is larger than *9,000,000,000 seconds* (which is >285 years).
+  - Again, etcd `Lease` is meant for short-periodic keepalives or sessions, in the range of seconds or minutes. Not for hours or days!
+- Enable etcd server [`raft.Config.CheckQuorum` when starting with `ForceNewCluster`](https://github.com/coreos/etcd/pull/9347).
+
+### Proxy v2
+
+- Fix [v2 proxy leaky HTTP requests](https://github.com/coreos/etcd/pull/9336).
+
+### Go
+
+- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.16](https://github.com/coreos/etcd/releases/tag/v3.2.16) (2018-02-12)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.15...v3.2.16) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### etcd server
+
+- Fix [`mvcc` "unsynced" watcher restore operation](https://github.com/coreos/etcd/pull/9297).
+  - "unsynced" watcher is watcher that needs to be in sync with events that have happened.
+  - That is, "unsynced" watcher is the slow watcher that was requested on old revision.
+  - "unsynced" watcher restore operation was not correctly populating its underlying watcher group.
+  - Which possibly causes [missing events from "unsynced" watchers](https://github.com/coreos/etcd/issues/9086).
+  - A node gets network partitioned with a watcher on a future revision, and falls behind receiving a leader snapshot after partition gets removed. When applying this snapshot, etcd watch storage moves current synced watchers to unsynced since sync watchers might have become stale during network partition. And reset synced watcher group to restart watcher routines. Previously, there was a bug when moving from synced watcher group to unsynced, thus client would miss events when the watcher was requested to the network-partitioned node.
+
+### Go
+
+- Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.15](https://github.com/coreos/etcd/releases/tag/v3.2.15) (2018-01-22)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.14...v3.2.15) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### etcd server
+
+- Prevent [server panic from member update/add](https://github.com/coreos/etcd/pull/9174) with [wrong scheme URLs](https://github.com/coreos/etcd/issues/9173).
+- Log [user context cancel errors on stream APIs in debug level with TLS](https://github.com/coreos/etcd/pull/9178).
+
+### Go
+
+- Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.14](https://github.com/coreos/etcd/releases/tag/v3.2.14) (2018-01-11)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.13...v3.2.14) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### Improved
+
+- Log [user context cancel errors on stream APIs in debug level](https://github.com/coreos/etcd/pull/9105).
+
+### etcd server
+
+- Fix [`mvcc/backend.defragdb` nil-pointer dereference on create bucket failure](https://github.com/coreos/etcd/pull/9119).
+
+### Go
+
+- Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.13](https://github.com/coreos/etcd/releases/tag/v3.2.13) (2018-01-02)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.12...v3.2.13) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### etcd server
+
+- Remove [verbose error messages on stream cancel and gRPC info-level logs](https://github.com/coreos/etcd/pull/9080) in server-side.
+- Fix [gRPC server panic on `GracefulStop` TLS-enabled server](https://github.com/coreos/etcd/pull/8987).
+
+### Go
+
+- Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.12](https://github.com/coreos/etcd/releases/tag/v3.2.12) (2017-12-20)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.11...v3.2.12) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### Dependency
+
+- Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases/tag) from [**`v1.7.4`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.4) to [**`v1.7.5`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.5).
+- Upgrade [`github.com/grpc-ecosystem/grpc-gateway`](https://github.com/grpc-ecosystem/grpc-gateway/releases) from [**`v1.3`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.3) to [**`v1.3.0`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.3.0).
+
+### etcd server
+
+- Fix [error message of `Revision` compactor](https://github.com/coreos/etcd/pull/8999) in server-side.
+
+### client v3
+
+- Add [`MaxCallSendMsgSize` and `MaxCallRecvMsgSize`](https://github.com/coreos/etcd/pull/9047) fields to [`clientv3.Config`](https://godoc.org/github.com/coreos/etcd/clientv3#Config).
+  - Fix [exceeded response size limit error in client-side](https://github.com/coreos/etcd/issues/9043).
+  - Address [kubernetes#51099](https://github.com/kubernetes/kubernetes/issues/51099).
+    - In previous versions(v3.2.10, v3.2.11), client response size was limited to only 4 MiB.
+  - `MaxCallSendMsgSize` default value is 2 MiB, if not configured.
+  - `MaxCallRecvMsgSize` default value is `math.MaxInt32`, if not configured.
+
+### Go
+
+- Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.11](https://github.com/coreos/etcd/releases/tag/v3.2.11) (2017-12-05)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.10...v3.2.11) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### Dependency
+
+- Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases/tag) from [**`v1.7.3`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.3) to [**`v1.7.4`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.4).
+
+### Security, Authentication
+
+See [security doc](https://github.com/coreos/etcd/blob/master/Documentation/op-guide/security.md) for more details.
+
+- Log [more details on TLS handshake failures](https://github.com/coreos/etcd/pull/8952/files).
+
+### client v3
+
+- Fix racey grpc-go's server handler transport `WriteStatus` call to prevent [TLS-enabled etcd server crash](https://github.com/coreos/etcd/issues/8904).
+- Add [gRPC RPC failure warnings](https://github.com/coreos/etcd/pull/8939) to help debug such issues in the future.
+
+### Documentation
+
+- Remove `--listen-metrics-urls` flag in monitoring document (non-released in `v3.2.x`, planned for `v3.3.x`).
+
+### Go
+
+- Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.10](https://github.com/coreos/etcd/releases/tag/v3.2.10) (2017-11-16)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.9...v3.2.10) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### Dependency
+
+- Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases/tag) from [**`v1.2.1`**](https://github.com/grpc/grpc-go/releases/tag/v1.2.1) to [**`v1.7.3`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.3).
+- Upgrade [`github.com/grpc-ecosystem/grpc-gateway`](https://github.com/grpc-ecosystem/grpc-gateway/releases) from [**`v1.2.0`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.2.0) to [**`v1.3`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.3).
+
+### Security, Authentication
+
+See [security doc](https://github.com/coreos/etcd/blob/master/Documentation/op-guide/security.md) for more details.
+
+- Revert [discovery SRV auth `ServerName` with `*.{ROOT_DOMAIN}`](https://github.com/coreos/etcd/pull/8651) to support non-wildcard subject alternative names in the certs (see [issue #8445](https://github.com/coreos/etcd/issues/8445) for more contexts).
+  - For instance, `etcd --discovery-srv=etcd.local` will only authenticate peers/clients when the provided certs have root domain `etcd.local` (**not `*.etcd.local`**) as an entry in Subject Alternative Name (SAN) field.
+
+### etcd server
+
+- Replace backend key-value database `boltdb/bolt` with [`coreos/bbolt`](https://github.com/coreos/bbolt/releases) to address [backend database size issue](https://github.com/coreos/etcd/issues/8009).
+
+### client v3
+
+- Rewrite balancer to handle [network partitions](https://github.com/coreos/etcd/issues/8711).
+
+### Go
+
+- Compile with [*Go 1.8.5*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.9](https://github.com/coreos/etcd/releases/tag/v3.2.9) (2017-10-06)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.8...v3.2.9) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### Security, Authentication
+
+See [security doc](https://github.com/coreos/etcd/blob/master/Documentation/op-guide/security.md) for more details.
+
+- Update `golang.org/x/crypto/bcrypt` (see [golang/crypto@6c586e1](https://github.com/golang/crypto/commit/6c586e17d90a7d08bbbc4069984180dce3b04117)).
+- Fix discovery SRV bootstrapping to [authenticate `ServerName` with `*.{ROOT_DOMAIN}`](https://github.com/coreos/etcd/pull/8651), in order to support sub-domain wildcard matching (see [issue #8445](https://github.com/coreos/etcd/issues/8445) for more contexts).
+  - For instance, `etcd --discovery-srv=etcd.local` will only authenticate peers/clients when the provided certs have root domain `*.etcd.local` as an entry in Subject Alternative Name (SAN) field.
+
+### Go
+
+- Compile with [*Go 1.8.4*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.8](https://github.com/coreos/etcd/releases/tag/v3.2.8) (2017-09-29)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.7...v3.2.8) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### client v2
+
+- Fix v2 client failover to next endpoint on mutable operation.
+
+### gRPC Proxy
+
+- Handle [`KeysOnly` flag](https://github.com/coreos/etcd/pull/8552).
+
+### Go
+
+- Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.7](https://github.com/coreos/etcd/releases/tag/v3.2.7) (2017-09-01)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.6...v3.2.7) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### Security, Authentication
+
+- Fix [server-side auth so concurrent auth operations do not return old revision error](https://github.com/coreos/etcd/pull/8306).
+
+### client v3
+
+- Fix [`concurrency/stm` Put with serializable snapshot](https://github.com/coreos/etcd/pull/8439).
+  - Use store revision from first fetch to resolve write conflicts instead of modified revision.
+
+### Go
+
+- Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.6](https://github.com/coreos/etcd/releases/tag/v3.2.6) (2017-08-21)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.5...v3.2.6) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### etcd server
+
+- Fix watch restore from snapshot.
+- Fix multiple URLs for `--listen-peer-urls` flag.
+- Add `--enable-pprof` flag to etcd configuration file format.
+
+### Metrics, Monitoring
+
+See [List of metrics](https://etcd.readthedocs.io/en/latest/operate.html#v3-2) for all metrics per release.
+
+Note that any `etcd_debugging_*` metrics are experimental and subject to change.
+
+- Fix `etcd_debugging_mvcc_keys_total` inconsistency.
+
+### Go
+
+- Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.5](https://github.com/coreos/etcd/releases/tag/v3.2.5) (2017-08-04)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.4...v3.2.5) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### etcdctl v3
+
+- Return non-zero exit code on unhealthy `endpoint health`.
+
+### Security, Authentication
+
+See [security doc](https://github.com/coreos/etcd/blob/master/Documentation/op-guide/security.md) for more details.
+
+- [Server supports reverse-lookup on wildcard DNS `SAN`](https://github.com/coreos/etcd/pull/8281). For instance, if peer cert contains only DNS names (no IP addresses) in Subject Alternative Name (SAN) field, server first reverse-lookups the remote IP address to get a list of names mapping to that address (e.g. `nslookup IPADDR`). Then accepts the connection if those names have a matching name with peer cert's DNS names (either by exact or wildcard match). If none is matched, server forward-lookups each DNS entry in peer cert (e.g. look up `example.default.svc` when the entry is `*.example.default.svc`), and accepts connection only when the host's resolved addresses have the matching IP address with the peer's remote IP address. For example, peer B's CSR (with `cfssl`) SAN field is `["*.example.default.svc", "*.example.default.svc.cluster.local"]` when peer B's remote IP address is `10.138.0.2`. When peer B tries to join the cluster, peer A reverse-lookup the IP `10.138.0.2` to get the list of host names. And either exact or wildcard match the host names with peer B's cert DNS names in Subject Alternative Name (SAN) field. If none of reverse/forward lookups worked, it returns an error `"tls: "10.138.0.2" does not match any of DNSNames ["*.example.default.svc","*.example.default.svc.cluster.local"]`. See [issue#8268](https://github.com/coreos/etcd/issues/8268) for more detail.
+
+### Metrics, Monitoring
+
+See [List of metrics](https://etcd.readthedocs.io/en/latest/operate.html#v3-2) for all metrics per release.
+
+Note that any `etcd_debugging_*` metrics are experimental and subject to change.
+
+- Fix unreachable `/metrics` endpoint when `--enable-v2=false`.
+
+### gRPC Proxy
+
+- Handle [`PrevKv` flag](https://github.com/coreos/etcd/pull/8366).
+
+### Other
+
+- Add container registry `gcr.io/etcd-development/etcd`.
+
+### Go
+
+- Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.4](https://github.com/coreos/etcd/releases/tag/v3.2.4) (2017-07-19)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.3...v3.2.4) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### etcd server
+
+- Do not block on active client stream when stopping server
+
+### gRPC proxy
+
+- Fix gRPC proxy Snapshot RPC error handling
+
+### Go
+
+- Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.3](https://github.com/coreos/etcd/releases/tag/v3.2.3) (2017-07-14)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.2...v3.2.3) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### client v3
+
+- Let clients establish unlimited streams
+
+### Other
+
+- Tag docker images with minor versions
+  - e.g. `docker pull quay.io/coreos/etcd:v3.2` to fetch latest v3.2 versions
+
+### Go
+
+- Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.2](https://github.com/coreos/etcd/releases/tag/v3.2.2) (2017-07-07)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.1...v3.2.2) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### Improved
+
+- Rate-limit lease revoke on expiration.
+- Extend leases on promote to avoid queueing effect on lease expiration.
+
+### Security, Authentication
+
+See [security doc](https://github.com/coreos/etcd/blob/master/Documentation/op-guide/security.md) for more details.
+
+- [Server accepts connections if IP matches, without checking DNS entries](https://github.com/coreos/etcd/pull/8223). For instance, if peer cert contains IP addresses and DNS names in Subject Alternative Name (SAN) field, and the remote IP address matches one of those IP addresses, server just accepts connection without further checking the DNS names. For example, peer B's CSR (with `cfssl`) SAN field is `["invalid.domain", "10.138.0.2"]` when peer B's remote IP address is `10.138.0.2` and `invalid.domain` is a invalid host. When peer B tries to join the cluster, peer A successfully authenticates B, since Subject Alternative Name (SAN) field has a valid matching IP address. See [issue#8206](https://github.com/coreos/etcd/issues/8206) for more detail.
+
+### etcd server
+
+- Accept connection with matched IP SAN but no DNS match.
+  - Don't check DNS entries in certs if there's a matching IP.
+
+### gRPC gateway
+
+- Use user-provided listen address to connect to gRPC gateway.
+  - `net.Listener` rewrites IPv4 0.0.0.0 to IPv6 [::], breaking IPv6 disabled hosts.
+  - Only v3.2.0, v3.2.1 are affected.
+
+### Go
+
+- Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.1](https://github.com/coreos/etcd/releases/tag/v3.2.1) (2017-06-23)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.0...v3.2.1) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### etcd server
+
+- Fix backend database in-memory index corruption issue on restore (only 3.2.0 is affected).
+
+### gRPC gateway
+
+- Fix Txn marshaling.
+
+### Metrics, Monitoring
+
+See [List of metrics](https://etcd.readthedocs.io/en/latest/operate.html#v3-2) for all metrics per release.
+
+Note that any `etcd_debugging_*` metrics are experimental and subject to change.
+
+- Fix backend database size debugging metrics.
+
+### Go
+
+- Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8).
+
+
+## [v3.2.0](https://github.com/coreos/etcd/releases/tag/v3.2.0) (2017-06-09)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.1.0...v3.2.0) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md).**
+
+### Improved
+
+- Improve backend read concurrency.
+
+### Breaking Changes
+
+- Increased [`--snapshot-count` default value from 10,000 to 100,000](https://github.com/coreos/etcd/pull/7160).
+  - Higher snapshot count means it holds Raft entries in memory for longer before discarding old entries.
+  - It is a trade-off between less frequent snapshotting and [higher memory usage](https://github.com/kubernetes/kubernetes/issues/60589#issuecomment-371977156).
+  - User lower `--snapshot-count` value for lower memory usage.
+  - User higher `--snapshot-count` value for better availabilities of slow followers (less frequent snapshots from leader).
+- `clientv3.Lease.TimeToLive` returns `LeaseTimeToLiveResponse.TTL == -1` on lease not found.
+- `clientv3.NewFromConfigFile` is moved to `clientv3/yaml.NewConfig`.
+- `embed.Etcd.Peers` field is now `[]*peerListener`.
+- Rejects domains names for `--listen-peer-urls` and `--listen-client-urls` (3.1 only prints out warnings), since [domain name is invalid for network interface binding](https://github.com/coreos/etcd/issues/6336).
+
+### Dependency
+
+- Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) from [**`v1.0.4`**](https://github.com/grpc/grpc-go/releases/tag/v1.0.4) to [**`v1.2.1`**](https://github.com/grpc/grpc-go/releases/tag/v1.2.1).
+- Upgrade [`github.com/grpc-ecosystem/grpc-gateway`](https://github.com/grpc-ecosystem/grpc-gateway/releases) to [**`v1.2.0`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.2.0).
+
+### Metrics, Monitoring
+
+See [List of metrics](https://etcd.readthedocs.io/en/latest/operate.html#v3-2) for all metrics per release.
+
+Note that any `etcd_debugging_*` metrics are experimental and subject to change.
+
+- Add [`etcd_disk_backend_snapshot_duration_seconds`](https://github.com/coreos/etcd/pull/7892)
+- Add `etcd_debugging_server_lease_expired_total` metrics.
+
+### Security, Authentication
+
+See [security doc](https://github.com/coreos/etcd/blob/master/Documentation/op-guide/security.md) for more details.
+
+- [TLS certificates get reloaded on every client connection](https://github.com/coreos/etcd/pull/7829). This is useful when replacing expiry certs without stopping etcd servers; it can be done by overwriting old certs with new ones. Refreshing certs for every connection should not have too much overhead, but can be improved in the future, with caching layer. Example tests can be found [here](https://github.com/coreos/etcd/blob/b041ce5d514a4b4aaeefbffb008f0c7570a18986/integration/v3_grpc_test.go#L1601-L1757).
+- [Server denies incoming peer certs with wrong IP `SAN`](https://github.com/coreos/etcd/pull/7687). For instance, if peer cert contains any IP addresses in Subject Alternative Name (SAN) field, server authenticates a peer only when the remote IP address matches one of those IP addresses. This is to prevent unauthorized endpoints from joining the cluster.  For example, peer B's CSR (with `cfssl`) SAN field is `["*.example.default.svc", "*.example.default.svc.cluster.local", "10.138.0.27"]` when peer B's actual IP address is `10.138.0.2`, not `10.138.0.27`. When peer B tries to join the cluster, peer A will reject B with the error `x509: certificate is valid for 10.138.0.27, not 10.138.0.2`, because B's remote IP address does not match the one in Subject Alternative Name (SAN) field.
+- [Server resolves TLS `DNSNames` when checking `SAN`](https://github.com/coreos/etcd/pull/7767). For instance, if peer cert contains only DNS names (no IP addresses) in Subject Alternative Name (SAN) field, server authenticates a peer only when forward-lookups (`dig b.com`) on those DNS names have matching IP with the remote IP address.  For example, peer B's CSR (with `cfssl`) SAN field is `["b.com"]` when peer B's remote IP address is `10.138.0.2`. When peer B tries to join the cluster, peer A looks up the incoming host `b.com` to get the list of IP addresses (e.g. `dig b.com`). And rejects B if the list does not contain the IP `10.138.0.2`, with the error `tls: 10.138.0.2 does not match any of DNSNames ["b.com"]`.
+- Auth support JWT token.
+
+### etcd server
+
+- RPCs
+  - Add Election, Lock service.
+- Native client `etcdserver/api/v3client`
+  - client "embedded" in the server.
+- Logging, monitoring
+  - Server warns large snapshot operations.
+- Add `etcd --enable-v2` flag to enable v2 API server.
+  - `etcd --enable-v2=true` by default.
+- Add `etcd --auth-token` flag.
+- v3.2 compactor runs [every hour](https://github.com/coreos/etcd/pull/7875).
+  - Compactor only supports periodic compaction.
+  - Compactor continues to record latest revisions every 5-minute.
+  - For every hour, it uses the last revision that was fetched before compaction period, from the revision records that were collected every 5-minute.
+  - That is, for every hour, compactor discards historical data created before compaction period.
+  - The retention window of compaction period moves to next hour.
+  - For instance, when hourly writes are 100 and `--auto-compaction-retention=10`, v3.1 compacts revision 1000, 2000, and 3000 for every 10-hour, while v3.2 compacts revision 1000, 1100, and 1200 for every 1-hour.
+  - If compaction succeeds or requested revision has already been compacted, it resets period timer and removes used compacted revision from historical revision records (e.g. start next revision collect and compaction from previously collected revisions).
+  - If compaction fails, it retries in 5 minutes.
+- Allow snapshot over 512MB.
+
+### client v3
+
+- STM prefetching.
+- Add namespace feature.
+- Add `ErrOldCluster` with server version checking.
+- Translate `WithPrefix()` into `WithFromKey()` for empty key.
+
+### etcdctl v3
+
+- Add `check perf` command.
+- Add `etcdctl --from-key` flag to role grant-permission command.
+- `lock` command takes an optional command to execute.
+
+### gRPC Proxy
+
+- Proxy endpoint discovery.
+- Namespaces.
+- Coalesce lease requests.
+
+### etcd gateway
+
+- Support [DNS SRV priority](https://github.com/coreos/etcd/pull/7882) for [smart proxy routing](https://github.com/coreos/etcd/issues/4378).
+
+### Other
+
+- v3 client
+  - concurrency package's elections updated to match RPC interfaces.
+  - let client dial endpoints not in the balancer.
+- Release
+  - Annotate acbuild with supports-systemd-notify.
+  - Add `nsswitch.conf` to Docker container image.
+  - Add ppc64le, arm64(experimental) builds.
+
+### Go
+
+- Compile with [*Go 1.8.3*](https://golang.org/doc/devel/release.html#go1.8).
+

CHANGELOG-3.3.md

@@ -0,0 +1,521 @@
+
+
+Previous change logs can be found at [CHANGELOG-3.2](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.2.md).
+
+
+## [v3.3.9](https://github.com/coreos/etcd/releases/tag/v3.3.9) (2018-07-24)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.3.8...v3.3.9) and [v3.3 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md).**
+
+### Improved
+
+- Improve [Raft Read Index timeout warning messages](https://github.com/coreos/etcd/pull/9897).
+
+### Security, Authentication
+
+- Compile with [*Go 1.10.3*](https://golang.org/doc/devel/release.html#go1.10) to support [crypto/x509 "Name Constraints"](https://github.com/coreos/etcd/issues/9912).
+
+### Metrics, Monitoring
+
+See [List of metrics](https://etcd.readthedocs.io/en/latest/operate.html#v3-3) for all metrics per release.
+
+Note that any `etcd_debugging_*` metrics are experimental and subject to change.
+
+- Add [`etcd_server_go_version`](https://github.com/coreos/etcd/pull/9957) Prometheus metric.
+- Add [`etcd_server_heartbeat_send_failures_total`](https://github.com/coreos/etcd/pull/9940) Prometheus metric.
+- Add [`etcd_server_slow_apply_total`](https://github.com/coreos/etcd/pull/9940) Prometheus metric.
+- Add [`etcd_disk_backend_defrag_duration_seconds`](https://github.com/coreos/etcd/pull/9940) Prometheus metric.
+- Add [`etcd_mvcc_hash_duration_seconds`](https://github.com/coreos/etcd/pull/9940) Prometheus metric.
+- Add [`etcd_mvcc_hash_rev_duration_seconds`](https://github.com/coreos/etcd/pull/9940) Prometheus metric.
+- Add [`etcd_server_slow_read_indexes_total`](https://github.com/coreos/etcd/pull/9897) Prometheus metric.
+- Add [`etcd_server_quota_backend_bytes`](https://github.com/coreos/etcd/pull/9820) Prometheus metric.
+  - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_mvcc_db_total_size_in_use_in_bytes`.
+  - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB.
+  - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB.
+  - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete.
+  - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation.
+- Add [`etcd_mvcc_db_total_size_in_bytes`](https://github.com/coreos/etcd/pull/9819) Prometheus metric.
+  - In addition to [`etcd_debugging_mvcc_db_total_size_in_bytes`](https://github.com/coreos/etcd/pull/9819).
+- Add [`etcd_mvcc_db_total_size_in_use_in_bytes`](https://github.com/coreos/etcd/pull/9256) Prometheus metric.
+  - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_mvcc_db_total_size_in_use_in_bytes`.
+  - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB.
+  - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB.
+  - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete.
+  - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation.
+
+### client v3
+
+- Fix [lease keepalive interval updates when response queue is full](https://github.com/coreos/etcd/pull/9952).
+  - If `<-chan *clientv3LeaseKeepAliveResponse` from `clientv3.Lease.KeepAlive` was never consumed or channel is full, client was [sending keepalive request every 500ms](https://github.com/coreos/etcd/issues/9911) instead of expected rate of every "TTL / 3" duration.
+
+### Go
+
+- Compile with [*Go 1.10.3*](https://golang.org/doc/devel/release.html#go1.10).
+
+
+## [v3.3.8](https://github.com/coreos/etcd/releases/tag/v3.3.8) (2018-06-15)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.3.7...v3.3.8) and [v3.3 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md).**
+
+### Improved
+
+- Improve [slow request apply warning log](https://github.com/coreos/etcd/pull/9288).
+  - e.g. `read-only range request "key:\"/a\" range_end:\"/b\" " with result "range_response_count:3 size:96" took too long (97.966µs) to execute`.
+  - Redact [request value field](https://github.com/coreos/etcd/pull/9822).
+  - Provide [response size](https://github.com/coreos/etcd/pull/9826).
+- Add [backoff on watch retries on transient errors](https://github.com/coreos/etcd/pull/9840).
+
+### Go
+
+- Compile with [*Go 1.9.7*](https://golang.org/doc/devel/release.html#go1.9).
+
+
+## [v3.3.7](https://github.com/coreos/etcd/releases/tag/v3.3.7) (2018-06-06)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.3.6...v3.3.7) and [v3.3 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md).**
+
+### Security, Authentication
+
+- Support TLS cipher suite whitelisting.
+  - To block [weak cipher suites](https://github.com/coreos/etcd/issues/8320).
+  - TLS handshake fails when client hello is requested with invalid cipher suites.
+  - Add [`etcd --cipher-suites`](https://github.com/coreos/etcd/pull/9801) flag.
+  - If empty, Go auto-populates the list.
+
+### etcdctl v3
+
+- Fix [`etcdctl move-leader` command for TLS-enabled endpoints](https://github.com/coreos/etcd/pull/9807).
+
+### Go
+
+- Compile with [*Go 1.9.6*](https://golang.org/doc/devel/release.html#go1.9).
+
+
+## [v3.3.6](https://github.com/coreos/etcd/releases/tag/v3.3.6) (2018-05-31)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.3.5...v3.3.6) and [v3.3 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md).**
+
+### etcd server
+
+- Allow [empty auth token](https://github.com/coreos/etcd/pull/9369).
+  - Previously, when auth token is an empty string, it returns [`failed to initialize the etcd server: auth: invalid auth options` error](https://github.com/coreos/etcd/issues/9349).
+- Fix [auth storage panic on server lease revoke routine with JWT token](https://github.com/coreos/etcd/issues/9695).
+- Fix [`mvcc` server panic from restore operation](https://github.com/coreos/etcd/pull/9775).
+  - Let's assume that a watcher had been requested with a future revision X and sent to node A that became network-partitioned thereafter. Meanwhile, cluster makes progress. Then when the partition gets removed, the leader sends a snapshot to node A. Previously if the snapshot's latest revision is still lower than the watch revision X,  **etcd server panicked** during snapshot restore operation.
+  - Now, this server-side panic has been fixed.
+
+### Go
+
+- Compile with [*Go 1.9.6*](https://golang.org/doc/devel/release.html#go1.9).
+
+
+## [v3.3.5](https://github.com/coreos/etcd/releases/tag/v3.3.5) (2018-05-09)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.3.4...v3.3.5) and [v3.3 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md).**
+
+### etcdctl v3
+
+- Fix [`etcdctl watch [key] [range_end] -- [exec-command…]`](https://github.com/coreos/etcd/pull/9688) parsing.
+  - Previously,  `ETCDCTL_API=3 ./bin/etcdctl watch foo -- echo watch event received` panicked.
+
+### Go
+
+- Compile with [*Go 1.9.6*](https://golang.org/doc/devel/release.html#go1.9).
+
+
+## [v3.3.4](https://github.com/coreos/etcd/releases/tag/v3.3.4) (2018-04-24)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.3.3...v3.3.4) and [v3.3 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md).**
+
+### Metrics, Monitoring
+
+See [List of metrics](https://etcd.readthedocs.io/en/latest/operate.html#v3-3) for all metrics per release.
+
+Note that any `etcd_debugging_*` metrics are experimental and subject to change.
+
+- Add [`etcd_server_is_leader`](https://github.com/coreos/etcd/pull/9587) Prometheus metric.
+- Fix [`etcd_debugging_server_lease_expired_total`](https://github.com/coreos/etcd/pull/9557) Prometheus metric.
+- Fix [race conditions in v2 server stat collecting](https://github.com/coreos/etcd/pull/9562).
+
+### Security, Authentication
+
+- Fix [TLS reload](https://github.com/coreos/etcd/pull/9570) when [certificate SAN field only includes IP addresses but no domain names](https://github.com/coreos/etcd/issues/9541).
+  - In Go, server calls `(*tls.Config).GetCertificate` for TLS reload if and only if server's `(*tls.Config).Certificates` field is not empty, or `(*tls.ClientHelloInfo).ServerName` is not empty with a valid SNI from the client. Previously, etcd always populates `(*tls.Config).Certificates` on the initial client TLS handshake, as non-empty. Thus, client was always expected to supply a matching SNI in order to pass the TLS verification and to trigger `(*tls.Config).GetCertificate` to reload TLS assets.
+  - However, a certificate whose SAN field does [not include any domain names but only IP addresses](https://github.com/coreos/etcd/issues/9541) would request `*tls.ClientHelloInfo` with an empty `ServerName` field, thus failing to trigger the TLS reload on initial TLS handshake; this becomes a problem when expired certificates need to be replaced online.
+  - Now, `(*tls.Config).Certificates` is created empty on initial TLS client handshake, first to trigger `(*tls.Config).GetCertificate`, and then to populate rest of the certificates on every new TLS connection, even when client SNI is empty (e.g. cert only includes IPs).
+
+### etcd server
+
+- Add [`etcd --initial-election-tick-advance`](https://github.com/coreos/etcd/pull/9591) flag to configure initial election tick fast-forward.
+  - By default, `etcd --initial-election-tick-advance=true`, then local member fast-forwards election ticks to speed up "initial" leader election trigger.
+  - This benefits the case of larger election ticks. For instance, cross datacenter deployment may require longer election timeout of 10-second. If true, local node does not need wait up to 10-second. Instead, forwards its election ticks to 8-second, and have only 2-second left before leader election.
+  - Major assumptions are that: cluster has no active leader thus advancing ticks enables faster leader election. Or cluster already has an established leader, and rejoining follower is likely to receive heartbeats from the leader after tick advance and before election timeout.
+  - However, when network from leader to rejoining follower is congested, and the follower does not receive leader heartbeat within left election ticks, disruptive election has to happen thus affecting cluster availabilities.
+  - Now, this can be disabled by setting `--initial-election-tick-advance=false`.
+  - Disabling this would slow down initial bootstrap process for cross datacenter deployments. Make tradeoffs by configuring `etcd --initial-election-tick-advance` at the cost of slow initial bootstrap.
+  - If single-node, it advances ticks regardless.
+  - Address [disruptive rejoining follower node](https://github.com/coreos/etcd/issues/9333).
+
+### Package `embed`
+
+- Add [`embed.Config.InitialElectionTickAdvance`](https://github.com/coreos/etcd/pull/9591) to enable/disable initial election tick fast-forward.
+  - `embed.NewConfig()` would return `*embed.Config` with `InitialElectionTickAdvance` as true by default.
+
+### Go
+
+- Compile with [*Go 1.9.5*](https://golang.org/doc/devel/release.html#go1.9).
+
+
+## [v3.3.3](https://github.com/coreos/etcd/releases/tag/v3.3.3) (2018-03-29)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.3.2...v3.3.3) and [v3.3 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md).**
+
+### Improved
+
+- Adjust [election timeout on server restart](https://github.com/coreos/etcd/pull/9415) to reduce [disruptive rejoining servers](https://github.com/coreos/etcd/issues/9333).
+  - Previously, etcd fast-forwards election ticks on server start, with only one tick left for leader election. This is to speed up start phase, without having to wait until all election ticks elapse. Advancing election ticks is useful for cross datacenter deployments with larger election timeouts. However, it was affecting cluster availability if the last tick elapses before leader contacts the restarted node.
+  - Now, when etcd restarts, it adjusts election ticks with more than one tick left, thus more time for leader to prevent disruptive restart.
+- Adjust [periodic compaction retention window](https://github.com/coreos/etcd/pull/9485).
+  - e.g. `etcd --auto-compaction-mode=revision --auto-compaction-retention=1000` automatically `Compact` on `"latest revision" - 1000` every 5-minute (when latest revision is 30000, compact on revision 29000).
+  - e.g. Previously, `etcd --auto-compaction-mode=periodic --auto-compaction-retention=72h` automatically `Compact` with 72-hour retention windown for every 7.2-hour. **Now, `Compact` happens, for every 1-hour but still with 72-hour retention window.**
+  - e.g. Previously, `etcd --auto-compaction-mode=periodic --auto-compaction-retention=30m` automatically `Compact` with 30-minute retention windown for every 3-minute. **Now, `Compact` happens, for every 30-minute but still with 30-minute retention window.**
+  - Periodic compactor keeps recording latest revisions for every compaction period when given period is less than 1-hour, or for every 1-hour when given compaction period is greater than 1-hour (e.g. 1-hour when `etcd --auto-compaction-mode=periodic --auto-compaction-retention=24h`).
+  - For every compaction period or 1-hour, compactor uses the last revision that was fetched before compaction period, to discard historical data.
+  - The retention window of compaction period moves for every given compaction period or hour.
+  - For instance, when hourly writes are 100 and `etcd --auto-compaction-mode=periodic --auto-compaction-retention=24h`, `v3.2.x`, `v3.3.0`, `v3.3.1`, and `v3.3.2` compact revision 2400, 2640, and 2880 for every 2.4-hour, while `v3.3.3` *or later* compacts revision 2400, 2500, 2600 for every 1-hour.
+  - Futhermore, when `etcd --auto-compaction-mode=periodic --auto-compaction-retention=30m` and writes per minute are about 1000, `v3.3.0`, `v3.3.1`, and `v3.3.2` compact revision 30000, 33000, and 36000, for every 3-minute, while `v3.3.3` *or later* compacts revision 30000, 60000, and 90000, for every 30-minute.
+
+### Metrics, Monitoring
+
+See [List of metrics](https://etcd.readthedocs.io/en/latest/operate.html#v3-3) for all metrics per release.
+
+Note that any `etcd_debugging_*` metrics are experimental and subject to change.
+
+- Add missing [`etcd_network_peer_sent_failures_total` count](https://github.com/coreos/etcd/pull/9437).
+
+### Go
+
+- Compile with [*Go 1.9.5*](https://golang.org/doc/devel/release.html#go1.9).
+
+
+## [v3.3.2](https://github.com/coreos/etcd/releases/tag/v3.3.2) (2018-03-08)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.3.1...v3.3.2) and [v3.3 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md).**
+
+### etcd server
+
+- Fix [server panic on invalid Election Proclaim/Resign HTTP(S) requests](https://github.com/coreos/etcd/pull/9379).
+  - Previously, wrong-formatted HTTP requests to Election API could trigger panic in etcd server.
+  - e.g. `curl -L http://localhost:2379/v3/election/proclaim -X POST -d '{"value":""}'`, `curl -L http://localhost:2379/v3/election/resign -X POST -d '{"value":""}'`.
+- Fix [revision-based compaction retention parsing](https://github.com/coreos/etcd/pull/9339).
+  - Previously, `etcd --auto-compaction-mode revision --auto-compaction-retention 1` was [translated to revision retention 3600000000000](https://github.com/coreos/etcd/issues/9337).
+  - Now, `etcd --auto-compaction-mode revision --auto-compaction-retention 1` is correctly parsed as revision retention 1.
+- Prevent [overflow by large `TTL` values for `Lease` `Grant`](https://github.com/coreos/etcd/pull/9399).
+  - `TTL` parameter to `Grant` request is unit of second.
+  - Leases with too large `TTL` values exceeding `math.MaxInt64` [expire in unexpected ways](https://github.com/coreos/etcd/issues/9374).
+  - Server now returns `rpctypes.ErrLeaseTTLTooLarge` to client, when the requested `TTL` is larger than *9,000,000,000 seconds* (which is >285 years).
+  - Again, etcd `Lease` is meant for short-periodic keepalives or sessions, in the range of seconds or minutes. Not for hours or days!
+- Enable etcd server [`raft.Config.CheckQuorum` when starting with `ForceNewCluster`](https://github.com/coreos/etcd/pull/9347).
+
+### Proxy v2
+
+- Fix [v2 proxy leaky HTTP requests](https://github.com/coreos/etcd/pull/9336).
+
+### Go
+
+- Compile with [*Go 1.9.4*](https://golang.org/doc/devel/release.html#go1.9).
+
+
+## [v3.3.1](https://github.com/coreos/etcd/releases/tag/v3.3.1) (2018-02-12)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.3.0...v3.3.1) and [v3.3 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md).**
+
+### Improved
+
+- Add [warnings on requests taking too long](https://github.com/coreos/etcd/pull/9288).
+  - e.g. `etcdserver: read-only range request "key:\"\\000\" range_end:\"\\000\" " took too long [3.389041388s] to execute`
+
+### etcd server
+
+- Fix [`mvcc` "unsynced" watcher restore operation](https://github.com/coreos/etcd/pull/9281).
+  - "unsynced" watcher is watcher that needs to be in sync with events that have happened.
+  - That is, "unsynced" watcher is the slow watcher that was requested on old revision.
+  - "unsynced" watcher restore operation was not correctly populating its underlying watcher group.
+  - Which possibly causes [missing events from "unsynced" watchers](https://github.com/coreos/etcd/issues/9086).
+  - A node gets network partitioned with a watcher on a future revision, and falls behind receiving a leader snapshot after partition gets removed. When applying this snapshot, etcd watch storage moves current synced watchers to unsynced since sync watchers might have become stale during network partition. And reset synced watcher group to restart watcher routines. Previously, there was a bug when moving from synced watcher group to unsynced, thus client would miss events when the watcher was requested to the network-partitioned node.
+
+### Go
+
+- Compile with [*Go 1.9.4*](https://golang.org/doc/devel/release.html#go1.9).
+
+
+## [v3.3.0](https://github.com/coreos/etcd/releases/tag/v3.3.0) (2018-02-01)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.2.0...v3.3.0) and [v3.3 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md).**
+
+- [v3.3.0-rc.4](https://github.com/coreos/etcd/releases/tag/v3.3.0-rc.4) (2018-01-22), see [code changes](https://github.com/coreos/etcd/compare/v3.3.0-rc.3...v3.3.0-rc.4).
+- [v3.3.0-rc.3](https://github.com/coreos/etcd/releases/tag/v3.3.0-rc.3) (2018-01-17), see [code changes](https://github.com/coreos/etcd/compare/v3.3.0-rc.2...v3.3.0-rc.3).
+- [v3.3.0-rc.2](https://github.com/coreos/etcd/releases/tag/v3.3.0-rc.2) (2018-01-11), see [code changes](https://github.com/coreos/etcd/compare/v3.3.0-rc.1...v3.3.0-rc.2).
+- [v3.3.0-rc.1](https://github.com/coreos/etcd/releases/tag/v3.3.0-rc.1) (2018-01-02), see [code changes](https://github.com/coreos/etcd/compare/v3.3.0-rc.0...v3.3.0-rc.1).
+- [v3.3.0-rc.0](https://github.com/coreos/etcd/releases/tag/v3.3.0-rc.0) (2017-12-20), see [code changes](https://github.com/coreos/etcd/compare/v3.2.0...v3.3.0-rc.0).
+
+### Improved
+
+- Use [`coreos/bbolt`](https://github.com/coreos/bbolt/releases) to replace [`boltdb/bolt`](https://github.com/boltdb/bolt#project-status).
+  - Fix [etcd database size grows until `mvcc: database space exceeded`](https://github.com/coreos/etcd/issues/8009).
+- [Support database size larger than 8GiB](https://github.com/coreos/etcd/pull/7525) (8GiB is now a suggested maximum size for normal environments)
+- [Reduce memory allocation](https://github.com/coreos/etcd/pull/8428) on [Range operations](https://github.com/coreos/etcd/pull/8475).
+- [Rate limit](https://github.com/coreos/etcd/pull/8099) and [randomize](https://github.com/coreos/etcd/pull/8101) lease revoke on restart or leader elections.
+  - Prevent [spikes in Raft proposal rate](https://github.com/coreos/etcd/issues/8096).
+- Support `clientv3` balancer failover under [network faults/partitions](https://github.com/coreos/etcd/issues/8711).
+- Better warning on [mismatched `etcd --initial-cluster`](https://github.com/coreos/etcd/pull/8083) flag.
+  - etcd compares `etcd --initial-advertise-peer-urls` against corresponding `etcd --initial-cluster` URLs with forward-lookup.
+  - If resolved IP addresses of `etcd --initial-advertise-peer-urls` and `etcd --initial-cluster` do not match (e.g. [due to DNS error](https://github.com/coreos/etcd/pull/9210)), etcd will exit with errors.
+    - v3.2 error: `etcd --initial-cluster must include s1=https://s1.test:2380 given --initial-advertise-peer-urls=https://s1.test:2380`.
+    - v3.3 error: `failed to resolve https://s1.test:2380 to match --initial-cluster=s1=https://s1.test:2380 (failed to resolve "https://s1.test:2380" (error ...))`.
+
+### Breaking Changes
+
+- Require [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) [**`v1.7.4`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.4) or [**`v1.7.5`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.5).
+  - Deprecate [`metadata.Incoming/OutgoingContext`](https://github.com/coreos/etcd/pull/7896).
+  - Deprecate `grpclog.Logger`, upgrade to [`grpclog.LoggerV2`](https://github.com/coreos/etcd/pull/8533).
+  - Deprecate [`grpc.ErrClientConnTimeout`](https://github.com/coreos/etcd/pull/8505) errors in `clientv3`.
+  - Use [`MaxRecvMsgSize` and `MaxSendMsgSize`](https://github.com/coreos/etcd/pull/8437) to limit message size, in etcd server.
+- Translate [gRPC status error in v3 client `Snapshot` API](https://github.com/coreos/etcd/pull/9038).
+- v3 `etcdctl` [`lease timetolive LEASE_ID`](https://github.com/coreos/etcd/issues/9028) on expired lease now prints [`"lease LEASE_ID already expired"`](https://github.com/coreos/etcd/pull/9047).
+  - <=3.2 prints `"lease LEASE_ID granted with TTL(0s), remaining(-1s)"`.
+- Replace [gRPC gateway](https://github.com/grpc-ecosystem/grpc-gateway) endpoint `/v3alpha` with [`/v3beta`](https://github.com/coreos/etcd/pull/8880).
+  - To deprecate [`/v3alpha`](https://github.com/coreos/etcd/issues/8125) in v3.4.
+  - In v3.3, `curl -L http://localhost:2379/v3alpha/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` still works as a fallback to `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'`, but `curl -L http://localhost:2379/v3alpha/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` won't work in v3.4. Use `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` instead.
+- Change `etcd --auto-compaction-retention` flag to [accept string values](https://github.com/coreos/etcd/pull/8563) with [finer granularity](https://github.com/coreos/etcd/issues/8503).
+  - Now that `etcd --auto-compaction-retention` accepts string values, etcd configuration YAML file `auto-compaction-retention` field must be changed to `string` type.
+  - Previously, `--config-file etcd.config.yaml` can have `auto-compaction-retention: 24` field, now must be `auto-compaction-retention: "24"` or `auto-compaction-retention: "24h"`.
+  - If configured as `etcd --auto-compaction-mode periodic --auto-compaction-retention "24h"`, the time duration value for `etcd --auto-compaction-retention` flag must be valid for [`time.ParseDuration`](https://golang.org/pkg/time/#ParseDuration) function in Go.
+
+### Dependency
+
+- Upgrade [`boltdb/bolt`](https://github.com/boltdb/bolt#project-status) from [**`v1.3.0`**](https://github.com/boltdb/bolt/releases/tag/v1.3.0) to [`coreos/bbolt`](https://github.com/coreos/bbolt/releases) [**`v1.3.1-coreos.6`**](https://github.com/coreos/bbolt/releases/tag/v1.3.1-coreos.6).
+- Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) from [**`v1.2.1`**](https://github.com/grpc/grpc-go/releases/tag/v1.2.1) to [**`v1.7.5`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.5).
+- Upgrade [`github.com/ugorji/go/codec`](https://github.com/ugorji/go) to [**`v1.1`**](https://github.com/ugorji/go/releases/tag/v1.1), and [regenerate v2 `client`](https://github.com/coreos/etcd/pull/8721).
+- Upgrade [`github.com/ugorji/go/codec`](https://github.com/ugorji/go) to [**`ugorji/go@54210f4e0`**](https://github.com/ugorji/go/commit/54210f4e076c57f351166f0ed60e67d3fca57a36), and [regenerate v2 `client`](https://github.com/coreos/etcd/pull/8574).
+- Upgrade [`github.com/grpc-ecosystem/grpc-gateway`](https://github.com/grpc-ecosystem/grpc-gateway/releases) from [**`v1.2.2`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.2.2) to [**`v1.3.0`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.3.0).
+- Upgrade [`golang.org/x/crypto/bcrypt`](https://github.com/golang/crypto) to [**`golang/crypto@6c586e17d`**](https://github.com/golang/crypto/commit/6c586e17d90a7d08bbbc4069984180dce3b04117).
+
+### Metrics, Monitoring
+
+See [List of metrics](https://etcd.readthedocs.io/en/latest/operate.html#v3-3) for all metrics per release.
+
+Note that any `etcd_debugging_*` metrics are experimental and subject to change.
+
+- Add [`etcd --listen-metrics-urls`](https://github.com/coreos/etcd/pull/8242) flag for additional `/metrics` and `/health` endpoints.
+  - Useful for [bypassing critical APIs when monitoring etcd](https://github.com/coreos/etcd/issues/8060).
+- Add [`etcd_server_version`](https://github.com/coreos/etcd/pull/8960) Prometheus metric.
+  - To replace [Kubernetes `etcd-version-monitor`](https://github.com/coreos/etcd/issues/8948).
+- Add [`etcd_debugging_mvcc_db_compaction_keys_total`](https://github.com/coreos/etcd/pull/8280) Prometheus metric.
+- Add [`etcd_debugging_server_lease_expired_total`](https://github.com/coreos/etcd/pull/8064) Prometheus metric.
+  - To improve [lease revoke monitoring](https://github.com/coreos/etcd/issues/8050).
+- Document [Prometheus 2.0 rules](https://github.com/coreos/etcd/pull/8879).
+- Initialize gRPC server [metrics with zero values](https://github.com/coreos/etcd/pull/8878).
+- Fix [range/put/delete operation metrics](https://github.com/coreos/etcd/pull/8054) with transaction.
+  - `etcd_debugging_mvcc_range_total`
+  - `etcd_debugging_mvcc_put_total`
+  - `etcd_debugging_mvcc_delete_total`
+  - `etcd_debugging_mvcc_txn_total`
+- Fix [`etcd_debugging_mvcc_keys_total`](https://github.com/coreos/etcd/pull/8390) on restore.
+- Fix [`etcd_debugging_mvcc_db_total_size_in_bytes`](https://github.com/coreos/etcd/pull/8120) on restore.
+  - Also change to [`prometheus.NewGaugeFunc`](https://github.com/coreos/etcd/pull/8150).
+
+### Security, Authentication
+
+See [security doc](https://github.com/coreos/etcd/blob/master/Documentation/op-guide/security.md) for more details.
+
+- Add [CRL based connection rejection](https://github.com/coreos/etcd/pull/8124) to manage [revoked certs](https://github.com/coreos/etcd/issues/4034).
+- Document [TLS authentication changes](https://github.com/coreos/etcd/pull/8895).
+  - [Server accepts connections if IP matches, without checking DNS entries](https://github.com/coreos/etcd/pull/8223). For instance, if peer cert contains IP addresses and DNS names in Subject Alternative Name (SAN) field, and the remote IP address matches one of those IP addresses, server just accepts connection without further checking the DNS names.
+  - [Server supports reverse-lookup on wildcard DNS `SAN`](https://github.com/coreos/etcd/pull/8281). For instance, if peer cert contains only DNS names (no IP addresses) in Subject Alternative Name (SAN) field, server first reverse-lookups the remote IP address to get a list of names mapping to that address (e.g. `nslookup IPADDR`). Then accepts the connection if those names have a matching name with peer cert's DNS names (either by exact or wildcard match). If none is matched, server forward-lookups each DNS entry in peer cert (e.g. look up `example.default.svc` when the entry is `*.example.default.svc`), and accepts connection only when the host's resolved addresses have the matching IP address with the peer's remote IP address.
+- Add [`etcd --peer-cert-allowed-cn`](https://github.com/coreos/etcd/pull/8616) flag.
+  - To support [CommonName(CN) based auth](https://github.com/coreos/etcd/issues/8262) for inter peer connection.
+- [Swap priority](https://github.com/coreos/etcd/pull/8594) of cert CommonName(CN) and username + password.
+  - To address ["username and password specified in the request should take priority over CN in the cert"](https://github.com/coreos/etcd/issues/8584).
+- Protect [lease revoke with auth](https://github.com/coreos/etcd/pull/8031).
+- Provide user's role on [auth permission error](https://github.com/coreos/etcd/pull/8164).
+- Fix [auth store panic with disabled token](https://github.com/coreos/etcd/pull/8695).
+
+### etcd server
+
+- Add [`etcd --experimental-initial-corrupt-check`](https://github.com/coreos/etcd/pull/8554) flag to [check cluster database hashes before serving client/peer traffic](https://github.com/coreos/etcd/issues/8313).
+  - `etcd --experimental-initial-corrupt-check=false` by default.
+  - v3.4 will enable `--initial-corrupt-check=true` by default.
+- Add [`etcd --experimental-corrupt-check-time`](https://github.com/coreos/etcd/pull/8420) flag to [raise corrupt alarm monitoring](https://github.com/coreos/etcd/issues/7125).
+  - `etcd --experimental-corrupt-check-time=0s` disabled by default.
+- Add [`etcd --experimental-enable-v2v3`](https://github.com/coreos/etcd/pull/8407) flag to [emulate v2 API with v3](https://github.com/coreos/etcd/issues/6925).
+  - `etcd --experimental-enable-v2v3=false` by default.
+- Add [`etcd --max-txn-ops`](https://github.com/coreos/etcd/pull/7976) flag to [configure maximum number operations in transaction](https://github.com/coreos/etcd/issues/7826).
+- Add [`etcd --max-request-bytes`](https://github.com/coreos/etcd/pull/7968) flag to [configure maximum client request size](https://github.com/coreos/etcd/issues/7923).
+  - If not configured, it defaults to 1.5 MiB.
+- Add [`etcd --client-crl-file`, `--peer-crl-file`](https://github.com/coreos/etcd/pull/8124) flags for [Certificate revocation list](https://github.com/coreos/etcd/issues/4034).
+- Add [`etcd --peer-cert-allowed-cn`](https://github.com/coreos/etcd/pull/8616) flag to support [CN-based auth for inter-peer connection](https://github.com/coreos/etcd/issues/8262).
+- Add [`etcd --listen-metrics-urls`](https://github.com/coreos/etcd/pull/8242) flag for additional `/metrics` and `/health` endpoints.
+  - Support [additional (non) TLS `/metrics` endpoints for a TLS-enabled cluster](https://github.com/coreos/etcd/pull/8282).
+  - e.g. `etcd --listen-metrics-urls=https://localhost:2378,http://localhost:9379` to serve `/metrics` and `/health` on secure port 2378 and insecure port 9379.
+  - Useful for [bypassing critical APIs when monitoring etcd](https://github.com/coreos/etcd/issues/8060).
+- Add [`etcd --auto-compaction-mode`](https://github.com/coreos/etcd/pull/8123) flag to [support revision-based compaction](https://github.com/coreos/etcd/issues/8098).
+- Change `etcd --auto-compaction-retention` flag to [accept string values](https://github.com/coreos/etcd/pull/8563) with [finer granularity](https://github.com/coreos/etcd/issues/8503).
+  - Now that `etcd --auto-compaction-retention` accepts string values, etcd configuration YAML file `auto-compaction-retention` field must be changed to `string` type.
+  - Previously, `etcd --config-file etcd.config.yaml` can have `auto-compaction-retention: 24` field, now must be `auto-compaction-retention: "24"` or `auto-compaction-retention: "24h"`.
+  - If configured as `--auto-compaction-mode periodic --auto-compaction-retention "24h"`, the time duration value for `etcd --auto-compaction-retention` flag must be valid for [`time.ParseDuration`](https://golang.org/pkg/time/#ParseDuration) function in Go.
+  - e.g. `etcd --auto-compaction-mode=revision --auto-compaction-retention=1000` automatically `Compact` on `"latest revision" - 1000` every 5-minute (when latest revision is 30000, compact on revision 29000).
+  - e.g. `etcd --auto-compaction-mode=periodic --auto-compaction-retention=72h` automatically `Compact` with 72-hour retention windown, for every 7.2-hour.
+  - e.g. `etcd --auto-compaction-mode=periodic --auto-compaction-retention=30m` automatically `Compact` with 30-minute retention windown, for every 3-minute.
+  - Periodic compactor continues to record latest revisions for every 1/10 of given compaction period (e.g. 1-hour when `etcd --auto-compaction-mode=periodic --auto-compaction-retention=10h`).
+  - For every 1/10 of given compaction period, compactor uses the last revision that was fetched before compaction period, to discard historical data.
+  - The retention window of compaction period moves for every 1/10 of given compaction period.
+  - For instance, when hourly writes are 100 and `--auto-compaction-retention=10`, v3.1 compacts revision 1000, 2000, and 3000 for every 10-hour, while v3.2.x, v3.3.0, v3.3.1, and v3.3.2 compact revision 1000, 1100, and 1200 for every 1-hour. Futhermore, when writes per minute are 1000, v3.3.0, v3.3.1, and v3.3.2 with `--auto-compaction-mode=periodic --auto-compaction-retention=30m` compact revision 30000, 33000, and 36000, for every 3-minute with more finer granularity.
+  - Whether compaction succeeds or not, this process repeats for every 1/10 of given compaction period. If compaction succeeds, it just removes compacted revision from historical revision records.
+- Add [`etcd --grpc-keepalive-min-time`, `etcd --grpc-keepalive-interval`, `etcd --grpc-keepalive-timeout`](https://github.com/coreos/etcd/pull/8535) flags to configure server-side keepalive policies.
+- Serve [`/health` endpoint as unhealthy](https://github.com/coreos/etcd/pull/8272) when [alarm (e.g. `NOSPACE`) is raised or there's no leader](https://github.com/coreos/etcd/issues/8207).
+  - Define [`etcdhttp.Health`](https://godoc.org/github.com/coreos/etcd/etcdserver/api/etcdhttp#Health) struct with JSON encoder.
+  - Note that `"health"` field is [`string` type, not `bool`](https://github.com/coreos/etcd/pull/9143).
+    - e.g. `{"health":"false"}`, `{"health":"true"}`
+  - [Remove `"errors"` field](https://github.com/coreos/etcd/pull/9162) since `v3.3.0-rc.3` (did exist only in `v3.3.0-rc.0`, `v3.3.0-rc.1`, `v3.3.0-rc.2`).
+- Move [logging setup to embed package](https://github.com/coreos/etcd/pull/8810)
+  - Disable gRPC server info-level logs by default (can be enabled with `etcd --debug` flag).
+- Use [monotonic time in Go 1.9](https://github.com/coreos/etcd/pull/8507) for `lease` package.
+- Warn on [empty hosts in advertise URLs](https://github.com/coreos/etcd/pull/8384).
+  - Address [advertise client URLs accepts empty hosts](https://github.com/coreos/etcd/issues/8379).
+  - etcd v3.4 will exit on this error.
+    - e.g. `etcd --advertise-client-urls=http://:2379`.
+- Warn on [shadowed environment variables](https://github.com/coreos/etcd/pull/8385).
+  - Address [error on shadowed environment variables](https://github.com/coreos/etcd/issues/8380).
+  - etcd v3.4 will exit on this error.
+
+### API
+
+- Support [ranges in transaction comparisons](https://github.com/coreos/etcd/pull/8025) for [disconnected linearized reads](https://github.com/coreos/etcd/issues/7924).
+- Add [nested transactions](https://github.com/coreos/etcd/pull/8102) to extend [proxy use cases](https://github.com/coreos/etcd/issues/7857).
+- Add [lease comparison target in transaction](https://github.com/coreos/etcd/pull/8324).
+- Add [lease list](https://github.com/coreos/etcd/pull/8358).
+- Add [hash by revision](https://github.com/coreos/etcd/pull/8263) for [better corruption checking against boltdb](https://github.com/coreos/etcd/issues/8016).
+
+### client v3
+
+- Add [health balancer](https://github.com/coreos/etcd/pull/8545) to fix [watch API hangs](https://github.com/coreos/etcd/issues/7247), improve [endpoint switch under network faults](https://github.com/coreos/etcd/issues/7941).
+- [Refactor balancer](https://github.com/coreos/etcd/pull/8840) and add [client-side keepalive pings](https://github.com/coreos/etcd/pull/8199) to handle [network partitions](https://github.com/coreos/etcd/issues/8711).
+- Add [`MaxCallSendMsgSize` and `MaxCallRecvMsgSize`](https://github.com/coreos/etcd/pull/9047) fields to [`clientv3.Config`](https://godoc.org/github.com/coreos/etcd/clientv3#Config).
+  - Fix [exceeded response size limit error in client-side](https://github.com/coreos/etcd/issues/9043).
+  - Address [kubernetes#51099](https://github.com/kubernetes/kubernetes/issues/51099).
+    - In previous versions(v3.2.10, v3.2.11), client response size was limited to only 4 MiB.
+  - `MaxCallSendMsgSize` default value is 2 MiB, if not configured.
+  - `MaxCallRecvMsgSize` default value is `math.MaxInt32`, if not configured.
+- Accept [`Compare_LEASE`](https://github.com/coreos/etcd/pull/8324) in [`clientv3.Compare`](https://godoc.org/github.com/coreos/etcd/clientv3#Compare).
+- Add [`LeaseValue` helper](https://github.com/coreos/etcd/pull/8488) to `Cmp` `LeaseID` values in `Txn`.
+- Add [`MoveLeader`](https://github.com/coreos/etcd/pull/8153) to `Maintenance`.
+- Add [`HashKV`](https://github.com/coreos/etcd/pull/8351) to `Maintenance`.
+- Add [`Leases`](https://github.com/coreos/etcd/pull/8358) to `Lease`.
+- Add [`clientv3/ordering`](https://github.com/coreos/etcd/pull/8092) for enforce [ordering in serialized requests](https://github.com/coreos/etcd/issues/7623).
+- Fix ["put at-most-once" violation](https://github.com/coreos/etcd/pull/8335).
+- Fix [`WatchResponse.Canceled`](https://github.com/coreos/etcd/pull/8283) on [compacted watch request](https://github.com/coreos/etcd/issues/8231).
+- Fix [`concurrency/stm` `Put` with serializable snapshot](https://github.com/coreos/etcd/pull/8439).
+  - Use store revision from first fetch to resolve write conflicts instead of modified revision.
+
+### etcdctl v3
+
+- Add [`etcdctl --discovery-srv`](https://github.com/coreos/etcd/pull/8462) flag.
+- Add [`etcdctl --keepalive-time`, `--keepalive-timeout`](https://github.com/coreos/etcd/pull/8663) flags.
+- Add [`etcdctl lease list`](https://github.com/coreos/etcd/pull/8358) command.
+- Add [`etcdctl lease keep-alive --once`](https://github.com/coreos/etcd/pull/8775) flag.
+- Make [`lease timetolive LEASE_ID`](https://github.com/coreos/etcd/issues/9028) on expired lease print [`lease LEASE_ID already expired`](https://github.com/coreos/etcd/pull/9047).
+  - <=3.2 prints `lease LEASE_ID granted with TTL(0s), remaining(-1s)`.
+- Add [`etcdctl snapshot restore --wal-dir`](https://github.com/coreos/etcd/pull/9124) flag.
+- Add [`etcdctl defrag --data-dir`](https://github.com/coreos/etcd/pull/8367) flag.
+- Add [`etcdctl move-leader`](https://github.com/coreos/etcd/pull/8153) command.
+- Add [`etcdctl endpoint hashkv`](https://github.com/coreos/etcd/pull/8351) command.
+- Add [`etcdctl endpoint --cluster`](https://github.com/coreos/etcd/pull/8143) flag, equivalent to [v2 `etcdctl cluster-health`](https://github.com/coreos/etcd/issues/8117).
+- Make `etcdctl endpoint health` command terminate with [non-zero exit code on unhealthy status](https://github.com/coreos/etcd/pull/8342).
+- Add [`etcdctl lock --ttl`](https://github.com/coreos/etcd/pull/8370) flag.
+- Support [`etcdctl watch [key] [range_end] -- [exec-command…]`](https://github.com/coreos/etcd/pull/8919), equivalent to [v2 `etcdctl exec-watch`](https://github.com/coreos/etcd/issues/8814).
+  - Make `etcdctl watch -- [exec-command]` set environmental variables [`ETCD_WATCH_REVISION`, `ETCD_WATCH_EVENT_TYPE`, `ETCD_WATCH_KEY`, `ETCD_WATCH_VALUE`](https://github.com/coreos/etcd/pull/9142) for each event.
+- Support [`etcdctl watch` with environmental variables `ETCDCTL_WATCH_KEY` and `ETCDCTL_WATCH_RANGE_END`](https://github.com/coreos/etcd/pull/9142).
+- Enable [`clientv3.WithRequireLeader(context.Context)` for `watch`](https://github.com/coreos/etcd/pull/8672) command.
+- Print [`"del"` instead of `"delete"`](https://github.com/coreos/etcd/pull/8297) in `txn` interactive mode.
+- Print [`ETCD_INITIAL_ADVERTISE_PEER_URLS` in `member add`](https://github.com/coreos/etcd/pull/8332).
+
+### etcdctl v3
+
+- Handle [empty key permission](https://github.com/coreos/etcd/pull/8514) in `etcdctl`.
+
+### etcdctl v2
+
+- Add [`etcdctl backup --with-v3`](https://github.com/coreos/etcd/pull/8479) flag.
+
+### gRPC Proxy
+
+- Add [`grpc-proxy start --experimental-leasing-prefix`](https://github.com/coreos/etcd/pull/8341) flag.
+  - For disconnected linearized reads.
+  - Based on [V system leasing](https://github.com/coreos/etcd/issues/6065).
+  - See ["Disconnected consistent reads with etcd" blog post](https://coreos.com/blog/coreos-labs-disconnected-consistent-reads-with-etcd).
+- Add [`grpc-proxy start --experimental-serializable-ordering`](https://github.com/coreos/etcd/pull/8315) flag.
+  - To ensure serializable reads have monotonically increasing store revisions across endpoints.
+- Add [`grpc-proxy start --metrics-addr`](https://github.com/coreos/etcd/pull/8242) flag for an additional `/metrics` endpoint.
+  - Set `--metrics-addr=http://[HOST]:9379` to serve `/metrics` in insecure port 9379.
+- Serve [`/health` endpoint in grpc-proxy](https://github.com/coreos/etcd/pull/8322).
+- Add [`grpc-proxy start --debug`](https://github.com/coreos/etcd/pull/8994) flag.
+- Add [`grpc-proxy start --max-send-bytes`](https://github.com/coreos/etcd/pull/9250) flag to [configure maximum client request size](https://github.com/coreos/etcd/issues/7923).
+- Add [`grpc-proxy start --max-recv-bytes`](https://github.com/coreos/etcd/pull/9250) flag to [configure maximum client request size](https://github.com/coreos/etcd/issues/7923).
+- Fix [Snapshot API error handling](https://github.com/coreos/etcd/commit/dbd16d52fbf81e5fd806d21ff5e9148d5bf203ab).
+- Fix [KV API `PrevKv` flag handling](https://github.com/coreos/etcd/pull/8366).
+- Fix [KV API `KeysOnly` flag handling](https://github.com/coreos/etcd/pull/8552).
+
+### gRPC gateway
+
+- Replace [gRPC gateway](https://github.com/grpc-ecosystem/grpc-gateway) endpoint `/v3alpha` with [`/v3beta`](https://github.com/coreos/etcd/pull/8880).
+  - To deprecate [`/v3alpha`](https://github.com/coreos/etcd/issues/8125) in v3.4.
+  - In v3.3, `curl -L http://localhost:2379/v3alpha/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` still works as a fallback to `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'`, but `curl -L http://localhost:2379/v3alpha/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` won't work in v3.4. Use `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` instead.
+- Support ["authorization" token](https://github.com/coreos/etcd/pull/7999).
+- Support [websocket for bi-directional streams](https://github.com/coreos/etcd/pull/8257).
+  - Fix [`Watch` API with gRPC gateway](https://github.com/coreos/etcd/issues/8237).
+- Upgrade gRPC gateway to [v1.3.0](https://github.com/coreos/etcd/issues/8838).
+
+### etcd server
+
+- Fix [backend database in-memory index corruption](https://github.com/coreos/etcd/pull/8127) issue on restore (only 3.2.0 is affected).
+- Fix [watch restore from snapshot](https://github.com/coreos/etcd/pull/8427).
+- Fix [`mvcc/backend.defragdb` nil-pointer dereference on create bucket failure](https://github.com/coreos/etcd/pull/9119).
+- Fix [server crash](https://github.com/coreos/etcd/pull/8010) on [invalid transaction request from gRPC gateway](https://github.com/coreos/etcd/issues/7889).
+- Prevent [server panic from member update/add](https://github.com/coreos/etcd/pull/9174) with [wrong scheme URLs](https://github.com/coreos/etcd/issues/9173).
+- Make [peer dial timeout longer](https://github.com/coreos/etcd/pull/8599).
+  - See [coreos/etcd-operator#1300](https://github.com/coreos/etcd-operator/issues/1300) for more detail.
+- Make server [wait up to request time-out](https://github.com/coreos/etcd/pull/8267) with [pending RPCs](https://github.com/coreos/etcd/issues/8224).
+- Fix [`grpc.Server` panic on `GracefulStop`](https://github.com/coreos/etcd/pull/8987) with [TLS-enabled server](https://github.com/coreos/etcd/issues/8916).
+- Fix ["multiple peer URLs cannot start" issue](https://github.com/coreos/etcd/issues/8383).
+- Fix server-side auth so [concurrent auth operations do not return old revision error](https://github.com/coreos/etcd/pull/8442).
+- Handle [WAL renaming failure on Windows](https://github.com/coreos/etcd/pull/8286).
+- Upgrade [`coreos/go-systemd`](https://github.com/coreos/go-systemd/releases) to `v15` (see https://github.com/coreos/go-systemd/releases/tag/v15).
+- [Put back `/v2/machines`](https://github.com/coreos/etcd/pull/8062) endpoint for python-etcd wrapper.
+
+### client v2
+
+- [Fail-over v2 client](https://github.com/coreos/etcd/pull/8519) to next endpoint on [oneshot failure](https://github.com/coreos/etcd/issues/8515).
+
+### Package `raft`
+
+- Add [non-voting member](https://github.com/coreos/etcd/pull/8751).
+  - To implement [Raft thesis 4.2.1 Catching up new servers](https://github.com/coreos/etcd/issues/8568).
+  - `Learner` node does not vote or promote itself.
+
+### Other
+
+- Support previous two minor versions (see our [new release policy](https://github.com/coreos/etcd/pull/8805)).
+- `v3.3.x` is the last release cycle that supports `ACI`.
+  - [AppC was officially suspended](https://github.com/appc/spec#-disclaimer-), as of late 2016.
+  - [`acbuild`](https://github.com/containers/build#this-project-is-currently-unmaintained) is not maintained anymore.
+  - `*.aci` files won't be available from etcd v3.4 release.
+- Add container registry [`gcr.io/etcd-development/etcd`](https://gcr.io/etcd-development/etcd).
+  - [quay.io/coreos/etcd](https://quay.io/coreos/etcd) is still supported as secondary.
+
+### Go
+
+- Require [*Go 1.9+*](https://github.com/coreos/etcd/issues/6174).
+- Compile with [*Go 1.9.3*](https://golang.org/doc/devel/release.html#go1.9).
+- Deprecate [`golang.org/x/net/context`](https://github.com/coreos/etcd/pull/8511).
+

CHANGELOG-3.4.md

@@ -0,0 +1,424 @@
+
+
+Previous change logs can be found at [CHANGELOG-3.3](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.3.md).
+
+
+## v3.4.0 (TBD 2018-09)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.3.0...v3.4.0) and [v3.4 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_4.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.4 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_4.md).**
+
+### Improved
+
+- Rewrite [client balancer](https://github.com/coreos/etcd/pull/9860) with [new gRPC balancer interface](https://github.com/coreos/etcd/issues/9106).
+- Add [backoff on watch retries on transient errors](https://github.com/coreos/etcd/pull/9840).
+- Add [jitter to watch progress notify](https://github.com/coreos/etcd/pull/9278) to prevent [spikes in `etcd_network_client_grpc_sent_bytes_total`](https://github.com/coreos/etcd/issues/9246).
+- Improve [slow request apply warning log](https://github.com/coreos/etcd/pull/9288).
+  - e.g. `read-only range request "key:\"/a\" range_end:\"/b\" " with result "range_response_count:3 size:96" took too long (97.966µs) to execute`.
+  - Redact [request value field](https://github.com/coreos/etcd/pull/9822).
+  - Provide [response size](https://github.com/coreos/etcd/pull/9826).
+- Improve [TLS setup error logging](https://github.com/coreos/etcd/pull/9518) to help debug [TLS-enabled cluster configuring issues](https://github.com/coreos/etcd/issues/9400).
+- Improve [long-running concurrent read transactions under light write workloads](https://github.com/coreos/etcd/pull/9296).
+  - Previously, periodic commit on pending writes blocks incoming read transactions, even if there is no pending write.
+  - Now, periodic commit operation does not block concurrent read transactions, thus improves long-running read transaction performance.
+- Improve [Raft Read Index timeout warning messages](https://github.com/coreos/etcd/pull/9897).
+- Adjust [election timeout on server restart](https://github.com/coreos/etcd/pull/9415) to reduce [disruptive rejoining servers](https://github.com/coreos/etcd/issues/9333).
+  - Previously, etcd fast-forwards election ticks on server start, with only one tick left for leader election. This is to speed up start phase, without having to wait until all election ticks elapse. Advancing election ticks is useful for cross datacenter deployments with larger election timeouts. However, it was affecting cluster availability if the last tick elapses before leader contacts the restarted node.
+  - Now, when etcd restarts, it adjusts election ticks with more than one tick left, thus more time for leader to prevent disruptive restart.
+- Add [Raft Pre-Vote feature](https://github.com/coreos/etcd/pull/9352) to reduce [disruptive rejoining servers](https://github.com/coreos/etcd/issues/9333).
+  - For instance, a flaky(or rejoining) member may drop in and out, and start campaign. This member will end up with a higher term, and ignore all incoming messages with lower term. In this case, a new leader eventually need to get elected, thus disruptive to cluster availability. Raft implements Pre-Vote phase to prevent this kind of disruptions. If enabled, Raft runs an additional phase of election to check if pre-candidate can get enough votes to win an election.
+- Adjust [periodic compaction retention window](https://github.com/coreos/etcd/pull/9485).
+  - e.g. `etcd --auto-compaction-mode=revision --auto-compaction-retention=1000` automatically `Compact` on `"latest revision" - 1000` every 5-minute (when latest revision is 30000, compact on revision 29000).
+  - e.g. Previously, `etcd --auto-compaction-mode=periodic --auto-compaction-retention=24h` automatically `Compact` with 24-hour retention windown for every 2.4-hour. Now, `Compact` happens for every 1-hour.
+  - e.g. Previously, `etcd --auto-compaction-mode=periodic --auto-compaction-retention=30m` automatically `Compact` with 30-minute retention windown for every 3-minute. Now, `Compact` happens for every 30-minute.
+  - Periodic compactor keeps recording latest revisions for every compaction period when given period is less than 1-hour, or for every 1-hour when given compaction period is greater than 1-hour (e.g. 1-hour when `etcd --auto-compaction-mode=periodic --auto-compaction-retention=24h`).
+  - For every compaction period or 1-hour, compactor uses the last revision that was fetched before compaction period, to discard historical data.
+  - The retention window of compaction period moves for every given compaction period or hour.
+  - For instance, when hourly writes are 100 and `etcd --auto-compaction-mode=periodic --auto-compaction-retention=24h`, `v3.2.x`, `v3.3.0`, `v3.3.1`, and `v3.3.2` compact revision 2400, 2640, and 2880 for every 2.4-hour, while `v3.3.3` *or later* compacts revision 2400, 2500, 2600 for every 1-hour.
+  - Futhermore, when `etcd --auto-compaction-mode=periodic --auto-compaction-retention=30m` and writes per minute are about 1000, `v3.3.0`, `v3.3.1`, and `v3.3.2` compact revision 30000, 33000, and 36000, for every 3-minute, while `v3.3.3` *or later* compacts revision 30000, 60000, and 90000, for every 30-minute.
+- Improve [lease expire/revoke operation performance](https://github.com/coreos/etcd/pull/9418), address [lease scalability issue](https://github.com/coreos/etcd/issues/9496).
+- Make [Lease `Lookup` non-blocking with concurrent `Grant`/`Revoke`](https://github.com/coreos/etcd/pull/9229).
+- Make etcd server return `raft.ErrProposalDropped` on internal Raft proposal drop in [v3 applier](https://github.com/coreos/etcd/pull/9549) and [v2 applier](https://github.com/coreos/etcd/pull/9558).
+  - e.g. a node is removed from cluster, or [`raftpb.MsgProp` arrives at current leader while there is an ongoing leadership transfer](https://github.com/coreos/etcd/issues/8975).
+- Add [`snapshot`](https://github.com/coreos/etcd/pull/9118) package for easier snapshot workflow (see [`godoc.org/github.com/etcd/clientv3/snapshot`](https://godoc.org/github.com/coreos/etcd/clientv3/snapshot) for more).
+- Improve [functional tester](https://github.com/coreos/etcd/tree/master/functional) coverage: [proxy layer to run network fault tests in CI](https://github.com/coreos/etcd/pull/9081), [TLS is enabled both for server and client](https://github.com/coreos/etcd/pull/9534), [liveness mode](https://github.com/coreos/etcd/issues/9230), [shuffle test sequence](https://github.com/coreos/etcd/issues/9381), [membership reconfiguration failure cases](https://github.com/coreos/etcd/pull/9564), [disastrous quorum loss and snapshot recover from a seed member](https://github.com/coreos/etcd/pull/9565), [embedded etcd](https://github.com/coreos/etcd/pull/9572).
+- Improve [index compaction blocking](https://github.com/coreos/etcd/pull/9511) by using a copy on write clone to avoid holding the lock for the traversal of the entire index.
+- Update [JWT methods](https://github.com/coreos/etcd/pull/9883) to allow for use of any supported signature method/algorithm.
+- Add [Lease checkpointing](https://github.com/coreos/etcd/pull/9924) to persist remaining TTLs to the consensus log periodically so that long lived leases progress toward expiry in the presence of leader elections and server restarts.
+
+### Breaking Changes
+
+- Make [`ETCDCTL_API=3 etcdctl` default](https://github.com/coreos/etcd/issues/9600).
+  - Now, `etcdctl set foo bar` must be `ETCDCTL_API=2 etcdctl set foo bar`.
+  - Now, `ETCDCTL_API=3 etcdctl put foo bar` could be just `etcdctl put foo bar`.
+- **Remove `etcd --ca-file` flag**, instead [use `etcd --trusted-ca-file`](https://github.com/coreos/etcd/pull/9470) (`etcd --ca-file` flag has been marked deprecated since v2.1).
+- **Remove `etcd --peer-ca-file` flag**, instead [use `etcd --peer-trusted-ca-file`](https://github.com/coreos/etcd/pull/9470) (`etcd --peer-ca-file` flag has been marked deprecated since v2.1).
+- **Remove `pkg/transport.TLSInfo.CAFile` field**, instead [use `pkg/transport.TLSInfo.TrustedCAFile`](https://github.com/coreos/etcd/pull/9470) (`CAFile` field has been marked deprecated since v2.1).
+- Deprecate `latest` [release container](https://console.cloud.google.com/gcr/images/etcd-development/GLOBAL/etcd) tag.
+  - **`docker pull gcr.io/etcd-development/etcd:latest` would not be up-to-date**.
+- Deprecate [minor](https://semver.org/) version [release container](https://console.cloud.google.com/gcr/images/etcd-development/GLOBAL/etcd) tags.
+  - `docker pull gcr.io/etcd-development/etcd:v3.3` would still work.
+  - **`docker pull gcr.io/etcd-development/etcd:v3.4` would not work**.
+  - Use **`docker pull gcr.io/etcd-development/etcd:v3.4.x`** instead, with the exact patch version.
+- Drop [ACIs from official release](https://github.com/coreos/etcd/pull/9059).
+  - [AppC was officially suspended](https://github.com/appc/spec#-disclaimer-), as of late 2016.
+  - [`acbuild`](https://github.com/containers/build#this-project-is-currently-unmaintained) is not maintained anymore.
+  - `*.aci` files are not available from `v3.4` release.
+- Exit on [empty hosts in advertise URLs](https://github.com/coreos/etcd/pull/8786).
+  - Address [advertise client URLs accepts empty hosts](https://github.com/coreos/etcd/issues/8379).
+  - e.g. exit with error on `--advertise-client-urls=http://:2379`.
+  - e.g. exit with error on `--initial-advertise-peer-urls=http://:2380`.
+- Exit on [shadowed environment variables](https://github.com/coreos/etcd/pull/9382).
+  - Address [error on shadowed environment variables](https://github.com/coreos/etcd/issues/8380).
+  - e.g. exit with error on `ETCD_NAME=abc etcd --name=def`.
+  - e.g. exit with error on `ETCD_INITIAL_CLUSTER_TOKEN=abc etcd --initial-cluster-token=def`.
+  - e.g. exit with error on `ETCDCTL_ENDPOINTS=abc.com ETCDCTL_API=3 etcdctl endpoint health --endpoints=def.com`.
+- Change [`etcdserverpb.AuthRoleRevokePermissionRequest/key,range_end` fields type from `string` to `bytes`](https://github.com/coreos/etcd/pull/9433).
+- Rename [`etcd_debugging_mvcc_db_total_size_in_bytes` Prometheus metric to `etcd_mvcc_db_total_size_in_bytes`](https://github.com/coreos/etcd/pull/9819).
+- Rename `etcdserver.ServerConfig.SnapCount` field to `etcdserver.ServerConfig.SnapshotCount`, to be consistent with the flag name `etcd --snapshot-count`.
+- Rename `embed.Config.SnapCount` field to [`embed.Config.SnapshotCount`](https://github.com/coreos/etcd/pull/9745), to be consistent with the flag name `etcd --snapshot-count`.
+- Change [`embed.Config.CorsInfo` in `*cors.CORSInfo` type to `embed.Config.CORS` in `map[string]struct{}` type](https://github.com/coreos/etcd/pull/9490).
+- Remove [`embed.Config.SetupLogging`](https://github.com/coreos/etcd/pull/9572).
+  - Now logger is set up automatically based on [`embed.Config.Logger`, `embed.Config.LogOutputs`, `embed.Config.Debug` fields](https://github.com/coreos/etcd/pull/9572).
+- Rename [`etcd --log-output` to `etcd --log-outputs`](https://github.com/coreos/etcd/pull/9624) to support multiple log outputs.
+  - **`etcd --log-output`** will be deprecated in v3.5.
+- Rename [**`embed.Config.LogOutput`** to **`embed.Config.LogOutputs`**](https://github.com/coreos/etcd/pull/9624) to support multiple log outputs.
+- Change [**`embed.Config.LogOutputs`** type from `string` to `[]string`](https://github.com/coreos/etcd/pull/9579) to support multiple log outputs.
+  - Now that `etcd --log-outputs` accepts multiple writers, etcd configuration YAML file `log-outputs` field must be changed to `[]string` type.
+  - Previously, `etcd --config-file etcd.config.yaml` can have `log-outputs: default` field, now must be `log-outputs: [default]`.
+- Change v3 `etcdctl snapshot` exit codes with [`snapshot` package](https://github.com/coreos/etcd/pull/9118/commits/df689f4280e1cce4b9d61300be13ca604d41670a).
+  - Exit on error with exit code 1 (no more exit code 5 or 6 on `snapshot save/restore` commands).
+- Migrate dependency management tool from `glide` to [`golang/dep`](https://github.com/coreos/etcd/pull/9155).
+  - <= 3.3 puts `vendor` directory under `cmd/vendor` directory to [prevent conflicting transitive dependencies](https://github.com/coreos/etcd/issues/4913).
+  - 3.4 moves `cmd/vendor` directory to `vendor` at repository root.
+  - Remove recursive symlinks in `cmd` directory.
+  - Now `go get/install/build` on `etcd` packages (e.g. `clientv3`, `tools/benchmark`) enforce builds with etcd `vendor` directory.
+- Replace [gRPC gateway](https://github.com/grpc-ecosystem/grpc-gateway) endpoint `/v3beta` with [`/v3`](https://github.com/coreos/etcd/pull/9298).
+  - Deprecated [`/v3alpha`](https://github.com/coreos/etcd/pull/9298).
+  - To deprecate [`/v3beta`](https://github.com/coreos/etcd/issues/9189) in v3.5.
+  - In v3.4, `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` still works as a fallback to `curl -L http://localhost:2379/v3/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'`, but `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` won't work in v3.5. Use `curl -L http://localhost:2379/v3/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` instead.
+- Change [`wal` package function signatures](https://github.com/coreos/etcd/pull/9572) to support [structured logger and logging to file](https://github.com/coreos/etcd/issues/9438) in server-side.
+  - Previously, `Open(dirpath string, snap walpb.Snapshot) (*WAL, error)`, now `Open(lg *zap.Logger, dirpath string, snap walpb.Snapshot) (*WAL, error)`.
+  - Previously, `OpenForRead(dirpath string, snap walpb.Snapshot) (*WAL, error)`, now `OpenForRead(lg *zap.Logger, dirpath string, snap walpb.Snapshot) (*WAL, error)`.
+  - Previously, `Repair(dirpath string) bool`, now `Repair(lg *zap.Logger, dirpath string) bool`.
+  - Previously, `Create(dirpath string, metadata []byte) (*WAL, error)`, now `Create(lg *zap.Logger, dirpath string, metadata []byte) (*WAL, error)`.
+- Remove [`pkg/cors` package](https://github.com/coreos/etcd/pull/9490).
+- Change [`etcd --experimental-enable-v2v3`](TODO) flag to `etcd --enable-v2v3`; v2 storage emulation is now stable.
+- Move internal packages to `etcdserver`.
+  - `"github.com/coreos/etcd/alarm"` to `"github.com/coreos/etcd/etcdserver/api/v3alarm"`.
+  - `"github.com/coreos/etcd/compactor"` to `"github.com/coreos/etcd/etcdserver/api/v3compactor"`.
+  - `"github.com/coreos/etcd/discovery"` to `"github.com/coreos/etcd/etcdserver/api/v2discovery"`.
+  - `"github.com/coreos/etcd/etcdserver/auth"` to `"github.com/coreos/etcd/etcdserver/api/v2auth"`.
+  - `"github.com/coreos/etcd/etcdserver/membership"` to `"github.com/coreos/etcd/etcdserver/api/membership"`.
+  - `"github.com/coreos/etcd/etcdserver/stats"` to `"github.com/coreos/etcd/etcdserver/api/v2stats"`.
+  - `"github.com/coreos/etcd/error"` to `"github.com/coreos/etcd/etcdserver/api/v2error"`.
+  - `"github.com/coreos/etcd/rafthttp"` to `"github.com/coreos/etcd/etcdserver/api/rafthttp"`.
+  - `"github.com/coreos/etcd/snap"` to `"github.com/coreos/etcd/etcdserver/api/snap"`.
+  - `"github.com/coreos/etcd/store"` to `"github.com/coreos/etcd/etcdserver/api/v2store"`.
+- Change [snapshot file permissions](https://github.com/coreos/etcd/pull/9977): On Linux, the snapshot file changes from readable by all (mode 0644) to readable by the user only (mode 0600).
+
+### Dependency
+
+- Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) from [**`v1.7.5`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.5) to [**`v1.13.0`**](https://github.com/grpc/grpc-go/releases/tag/v1.13.0).
+- Upgrade [`github.com/golang/protobuf`](https://github.com/golang/protobuf/releases) from [**`golang/protobuf@1e59b77b5`**](https://github.com/golang/protobuf/commit/1e59b77b52bf8e4b449a57e6f79f21226d571845) to [**`v1.1.0`**](https://github.com/golang/protobuf/releases/tag/v1.1.0).
+- Upgrade [`golang.org/x/crypto`](https://github.com/golang/crypto) from [**`crypto@9419663f5`**](https://github.com/golang/crypto/commit/9419663f5a44be8b34ca85f08abc5fe1be11f8a3) to [**`crypto@8ac0e0d97`**](https://github.com/golang/crypto/commit/8ac0e0d97ce45cd83d1d7243c060cb8461dda5e9).
+- Upgrade [`golang.org/x/net`](https://github.com/golang/net) from [**`net@66aacef3d`**](https://github.com/golang/net/commit/66aacef3dd8a676686c7ae3716979581e8b03c47) to [**`net@db08ff08e`**](https://github.com/golang/net/commit/db08ff08e8622530d9ed3a0e8ac279f6d4c02196).
+- Upgrade [`golang.org/x/sys`](https://github.com/golang/sys) from [**`sys@ebfc5b463`**](https://github.com/golang/sys/commit/ebfc5b4631820b793c9010c87fd8fef0f39eb082) to [**`sys@56ede360e`**](https://github.com/golang/sys/commit/56ede360ec1c541828fb88741b3f1049406d28f5).
+- Upgrade [`golang.org/x/text`](https://github.com/golang/text) from [**`text@b19bf474d`**](https://github.com/golang/text/commit/b19bf474d317b857955b12035d2c5acb57ce8b01) to [**`text@f21a4dfb5`**](https://github.com/golang/text/commit/f21a4dfb5e38f5895301dc265a8def02365cc3d0).
+- Upgrade [`golang.org/x/time`](https://github.com/golang/time) from [**`time@c06e80d93`**](https://github.com/golang/time/commit/c06e80d9300e4443158a03817b8a8cb37d230320) to [**`time@fbb02b229`**](https://github.com/golang/time/commit/fbb02b2291d28baffd63558aa44b4b56f178d650).
+- Upgrade [`github.com/golang/protobuf`](https://github.com/golang/protobuf/releases) from [**`golang/protobuf@1e59b77b5`**](https://github.com/golang/protobuf/commit/1e59b77b52bf8e4b449a57e6f79f21226d571845) to [**`v1.1.0`**](https://github.com/golang/protobuf/releases/tag/v1.1.0).
+- Upgrade [`gopkg.in/yaml.v2`](https://github.com/go-yaml/yaml/releases) from [**`yaml@cd8b52f82`**](https://github.com/go-yaml/yaml/commit/cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b) to [**`yaml@5420a8b67`**](https://github.com/go-yaml/yaml/commit/5420a8b6744d3b0345ab293f6fcba19c978f1183).
+- Upgrade [`github.com/dgrijalva/jwt-go`](https://github.com/dgrijalva/jwt-go/releases) from [**`v3.0.0`**](https://github.com/dgrijalva/jwt-go/releases/tag/v3.0.0) to [**`v3.2.0`**](https://github.com/dgrijalva/jwt-go/releases/tag/v3.2.0).
+- Upgrade [`github.com/ugorji/go/codec`](https://github.com/ugorji/go/releases) to [**`v1.1.1`**](https://github.com/ugorji/go/releases/tag/v1.1.1), and [regenerate v2 `client`](https://github.com/coreos/etcd/pull/9494).
+- Upgrade [`github.com/soheilhy/cmux`](https://github.com/soheilhy/cmux/releases) from [**`v0.1.3`**](https://github.com/soheilhy/cmux/releases/tag/v0.1.3) to [**`v0.1.4`**](https://github.com/soheilhy/cmux/releases/tag/v0.1.4).
+- Upgrade [`github.com/google/btree`](https://github.com/google/btree/releases) from [**`google/btree@925471ac9`**](https://github.com/google/btree/commit/925471ac9e2131377a91e1595defec898166fe49) to [**`google/btree@e89373fe6`**](https://github.com/google/btree/commit/e89373fe6b4a7413d7acd6da1725b83ef713e6e4).
+- Upgrade [`github.com/spf13/cobra`](https://github.com/spf13/cobra/releases) from [**`spf13/cobra@1c44ec8d3`**](https://github.com/spf13/cobra/commit/1c44ec8d3f1552cac48999f9306da23c4d8a288b) to [**`v0.0.3`**](https://github.com/spf13/cobra/releases/tag/v0.0.3).
+- Upgrade [`github.com/spf13/pflag`](https://github.com/spf13/pflag/releases) from [**`v1.0.0`**](https://github.com/spf13/pflag/releases/tag/v1.0.0) to [**`spf13/pflag@1ce0cc6db`**](https://github.com/spf13/pflag/commit/1ce0cc6db4029d97571db82f85092fccedb572ce).
+- Upgrade [`github.com/coreos/go-systemd`](https://github.com/coreos/go-systemd/releases) from [**`v15`**](https://github.com/coreos/go-systemd/releases/tag/v15) to [**`v17`**](https://github.com/coreos/go-systemd/releases/tag/v17).
+- Upgrade [`github.com/prometheus/client_golang`](https://github.com/prometheus/client_golang/releases) from [**``prometheus/client_golang@5cec1d042``**](https://github.com/prometheus/client_golang/commit/5cec1d0429b02e4323e042eb04dafdb079ddf568) to [**`v0.8.0`**](https://github.com/prometheus/client_golang/releases/tag/v0.8.0).
+- Upgrade [`github.com/grpc-ecosystem/go-grpc-prometheus`](https://github.com/grpc-ecosystem/go-grpc-prometheus/releases) from [**``grpc-ecosystem/go-grpc-prometheus@0dafe0d49``**](https://github.com/grpc-ecosystem/go-grpc-prometheus/commit/0dafe0d496ea71181bf2dd039e7e3f44b6bd11a7) to [**`v1.2.0`**](https://github.com/grpc-ecosystem/go-grpc-prometheus/releases/tag/v1.2.0).
+- Upgrade [`github.com/grpc-ecosystem/grpc-gateway`](https://github.com/grpc-ecosystem/grpc-gateway/releases) from [**`v1.3.1`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.3.1) to [**`v1.4.1`**](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.4.1).
+
+### Metrics, Monitoring
+
+See [List of metrics](https://etcd.readthedocs.io/en/latest/operate.html#latest) for all metrics per release.
+
+Note that any `etcd_debugging_*` metrics are experimental and subject to change.
+
+- Add [`etcd_network_active_peers`](https://github.com/coreos/etcd/pull/9762) Prometheus metric.
+  - Let's say `"7339c4e5e833c029"` server `/metrics` returns `etcd_network_active_peers{Local="7339c4e5e833c029",Remote="729934363faa4a24"} 1` and `etcd_network_active_peers{Local="7339c4e5e833c029",Remote="b548c2511513015"} 1`. This indicates that the local node `"7339c4e5e833c029"` currently has two active remote peers `"729934363faa4a24"` and `"b548c2511513015"` in a 3-node cluster. If the node `"b548c2511513015"` is down, the local node `"7339c4e5e833c029"` will show `etcd_network_active_peers{Local="7339c4e5e833c029",Remote="729934363faa4a24"} 1` and `etcd_network_active_peers{Local="7339c4e5e833c029",Remote="b548c2511513015"} 0`.
+- Add [`etcd_network_disconnected_peers_total`](https://github.com/coreos/etcd/pull/9762) Prometheus metric.
+  - If a remote peer `"b548c2511513015"` is down, the local node `"7339c4e5e833c029"` server `/metrics` would return `etcd_network_disconnected_peers_total{Local="7339c4e5e833c029",Remote="b548c2511513015"} 1`, while active peer metrics will show `etcd_network_active_peers{Local="7339c4e5e833c029",Remote="729934363faa4a24"} 1` and `etcd_network_active_peers{Local="7339c4e5e833c029",Remote="b548c2511513015"} 0`.
+- Add [`etcd_network_server_stream_failures_total`](https://github.com/coreos/etcd/pull/9760) Prometheus metric.
+  - e.g. `etcd_network_server_stream_failures_total{API="lease-keepalive",Type="receive"} 1`
+  - e.g. `etcd_network_server_stream_failures_total{API="watch",Type="receive"} 1`
+- Increase [`etcd_network_peer_round_trip_time_seconds`](https://github.com/coreos/etcd/pull/9762) Prometheus metric histogram upper-bound.
+  - Previously, highest bucket only collects requests taking 0.8192 seconds or more.
+  - Now, highest buckets collect 0.8192 seconds, 1.6384 seconds, and 3.2768 seconds or more.
+- Add [`etcd_server_is_leader`](https://github.com/coreos/etcd/pull/9587) Prometheus metric.
+- Add [`etcd_server_version`](https://github.com/coreos/etcd/pull/8960) Prometheus metric.
+  - To replace [Kubernetes `etcd-version-monitor`](https://github.com/coreos/etcd/issues/8948).
+- Add [`etcd_server_go_version`](https://github.com/coreos/etcd/pull/9957) Prometheus metric.
+- Add [`etcd_server_heartbeat_send_failures_total`](https://github.com/coreos/etcd/pull/9761) Prometheus metric.
+- Add [`etcd_server_slow_apply_total`](https://github.com/coreos/etcd/pull/9761) Prometheus metric.
+- Add [`etcd_server_slow_read_indexes_total`](https://github.com/coreos/etcd/pull/9897) Prometheus metric.
+- Add [`etcd_server_quota_backend_bytes`](https://github.com/coreos/etcd/pull/9820) Prometheus metric.
+  - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_mvcc_db_total_size_in_use_in_bytes`.
+  - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB.
+  - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB.
+  - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete.
+  - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation.
+- Add [`etcd_mvcc_db_total_size_in_bytes`](https://github.com/coreos/etcd/pull/9819) Prometheus metric.
+  - Renamed from [`etcd_debugging_mvcc_db_total_size_in_bytes`](https://github.com/coreos/etcd/pull/9819).
+- Add [`etcd_mvcc_db_total_size_in_use_in_bytes`](https://github.com/coreos/etcd/pull/9256) Prometheus metric.
+  - Use it with `etcd_mvcc_db_total_size_in_bytes` and `etcd_mvcc_db_total_size_in_use_in_bytes`.
+  - `etcd_server_quota_backend_bytes 2.147483648e+09` means current quota size is 2 GB.
+  - `etcd_mvcc_db_total_size_in_bytes 20480` means current physically allocated DB size is 20 KB.
+  - `etcd_mvcc_db_total_size_in_use_in_bytes 16384` means future DB size if defragment operation is complete.
+  - `etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes` is the number of bytes that can be saved on disk with defragment operation.
+- Add [`etcd_snap_fsync_duration_seconds`](https://github.com/coreos/etcd/pull/9762) Prometheus metric.
+- Add [`etcd_disk_backend_defrag_duration_seconds`](https://github.com/coreos/etcd/pull/9761) Prometheus metric.
+- Add [`etcd_mvcc_hash_duration_seconds`](https://github.com/coreos/etcd/pull/9761) Prometheus metric.
+- Add [`etcd_mvcc_hash_rev_duration_seconds`](https://github.com/coreos/etcd/pull/9761) Prometheus metric.
+- Add [`etcd_debugging_disk_backend_commit_rebalance_duration_seconds`](https://github.com/coreos/etcd/pull/9834) Prometheus metric.
+- Add [`etcd_debugging_disk_backend_commit_spill_duration_seconds`](https://github.com/coreos/etcd/pull/9834) Prometheus metric.
+- Add [`etcd_debugging_disk_backend_commit_write_duration_seconds`](https://github.com/coreos/etcd/pull/9834) Prometheus metric.
+- Add [`etcd_debugging_lease_granted_total`](https://github.com/coreos/etcd/pull/9778) Prometheus metric.
+- Add [`etcd_debugging_lease_revoked_total`](https://github.com/coreos/etcd/pull/9778) Prometheus metric.
+- Add [`etcd_debugging_lease_renewed_total`](https://github.com/coreos/etcd/pull/9778) Prometheus metric.
+- Add [`etcd_debugging_lease_ttl_total`](https://github.com/coreos/etcd/pull/9778) Prometheus metric.
+- Increase [`etcd_debugging_mvcc_index_compaction_pause_duration_milliseconds`](https://github.com/coreos/etcd/pull/9762) Prometheus metric histogram upper-bound.
+  - Previously, highest bucket only collects requests taking 1.024 seconds or more.
+  - Now, highest buckets collect 1.024 seconds, 2.048 seconds, and 4.096 seconds or more.
+- Fix missing [`etcd_network_peer_sent_failures_total`](https://github.com/coreos/etcd/pull/9437) Prometheus metric count.
+- Fix [`etcd_debugging_server_lease_expired_total`](https://github.com/coreos/etcd/pull/9557) Prometheus metric.
+- Fix [race conditions in v2 server stat collecting](https://github.com/coreos/etcd/pull/9562).
+
+### Security, Authentication
+
+See [security doc](https://github.com/coreos/etcd/blob/master/Documentation/op-guide/security.md) for more details.
+
+- Support TLS cipher suite whitelisting.
+  - To block [weak cipher suites](https://github.com/coreos/etcd/issues/8320).
+  - TLS handshake fails when client hello is requested with invalid cipher suites.
+  - Add [`etcd --cipher-suites`](https://github.com/coreos/etcd/pull/9801) flag.
+  - If empty, Go auto-populates the list.
+- Add [`etcd --host-whitelist`](https://github.com/coreos/etcd/pull/9372) flag, [`etcdserver.Config.HostWhitelist`](https://github.com/coreos/etcd/pull/9372), and [`embed.Config.HostWhitelist`](https://github.com/coreos/etcd/pull/9372), to prevent ["DNS Rebinding"](https://en.wikipedia.org/wiki/DNS_rebinding) attack.
+  - Any website can simply create an authorized DNS name, and direct DNS to `"localhost"` (or any other address). Then, all HTTP endpoints of etcd server listening on `"localhost"` becomes accessible, thus vulnerable to [DNS rebinding attacks (CVE-2018-5702)](https://bugs.chromium.org/p/project-zero/issues/detail?id=1447#c2).
+  - Client origin enforce policy works as follow:
+    - If client connection is secure via HTTPS, allow any hostnames..
+    - If client connection is not secure and `"HostWhitelist"` is not empty, only allow HTTP requests whose Host field is listed in whitelist.
+  - By default, `"HostWhitelist"` is `"*"`, which means insecure server allows all client HTTP requests.
+  - Note that the client origin policy is enforced whether authentication is enabled or not, for tighter controls.
+  - When specifying hostnames, loopback addresses are not added automatically. To allow loopback interfaces, add them to whitelist manually (e.g. `"localhost"`, `"127.0.0.1"`, etc.).
+  - e.g. `etcd --host-whitelist example.com`, then the server will reject all HTTP requests whose Host field is not `example.com` (also rejects requests to `"localhost"`).
+- Support [`etcd --cors`](https://github.com/coreos/etcd/pull/9490) in v3 HTTP requests (gRPC gateway).
+- Support [`ttl` field for `etcd` Authentication JWT token](https://github.com/coreos/etcd/pull/8302).
+  - e.g. `etcd --auth-token jwt,pub-key=<pub key path>,priv-key=<priv key path>,sign-method=<sign method>,ttl=5m`.
+- Allow empty token provider in [`etcdserver.ServerConfig.AuthToken`](https://github.com/coreos/etcd/pull/9369).
+- Fix [TLS reload](https://github.com/coreos/etcd/pull/9570) when [certificate SAN field only includes IP addresses but no domain names](https://github.com/coreos/etcd/issues/9541).
+  - In Go, server calls `(*tls.Config).GetCertificate` for TLS reload if and only if server's `(*tls.Config).Certificates` field is not empty, or `(*tls.ClientHelloInfo).ServerName` is not empty with a valid SNI from the client. Previously, etcd always populates `(*tls.Config).Certificates` on the initial client TLS handshake, as non-empty. Thus, client was always expected to supply a matching SNI in order to pass the TLS verification and to trigger `(*tls.Config).GetCertificate` to reload TLS assets.
+  - However, a certificate whose SAN field does [not include any domain names but only IP addresses](https://github.com/coreos/etcd/issues/9541) would request `*tls.ClientHelloInfo` with an empty `ServerName` field, thus failing to trigger the TLS reload on initial TLS handshake; this becomes a problem when expired certificates need to be replaced online.
+  - Now, `(*tls.Config).Certificates` is created empty on initial TLS client handshake, first to trigger `(*tls.Config).GetCertificate`, and then to populate rest of the certificates on every new TLS connection, even when client SNI is empty (e.g. cert only includes IPs).
+
+### etcd server
+
+- Add [`etcd --initial-election-tick-advance`](https://github.com/coreos/etcd/pull/9591) flag to configure initial election tick fast-forward.
+  - By default, `etcd --initial-election-tick-advance=true`, then local member fast-forwards election ticks to speed up "initial" leader election trigger.
+  - This benefits the case of larger election ticks. For instance, cross datacenter deployment may require longer election timeout of 10-second. If true, local node does not need wait up to 10-second. Instead, forwards its election ticks to 8-second, and have only 2-second left before leader election.
+  - Major assumptions are that: cluster has no active leader thus advancing ticks enables faster leader election. Or cluster already has an established leader, and rejoining follower is likely to receive heartbeats from the leader after tick advance and before election timeout.
+  - However, when network from leader to rejoining follower is congested, and the follower does not receive leader heartbeat within left election ticks, disruptive election has to happen thus affecting cluster availabilities.
+  - Now, this can be disabled by setting `etcd --initial-election-tick-advance=false`.
+  - Disabling this would slow down initial bootstrap process for cross datacenter deployments. Make tradeoffs by configuring `etcd --initial-election-tick-advance` at the cost of slow initial bootstrap.
+  - If single-node, it advances ticks regardless.
+  - Address [disruptive rejoining follower node](https://github.com/coreos/etcd/issues/9333).
+- Add [`etcd --pre-vote`](https://github.com/coreos/etcd/pull/9352) flag to enable to run an additional Raft election phase.
+  - For instance, a flaky(or rejoining) member may drop in and out, and start campaign. This member will end up with a higher term, and ignore all incoming messages with lower term. In this case, a new leader eventually need to get elected, thus disruptive to cluster availability. Raft implements Pre-Vote phase to prevent this kind of disruptions. If enabled, Raft runs an additional phase of election to check if pre-candidate can get enough votes to win an election.
+  - `etcd --pre-vote=false` by default.
+  - v3.5 will enable `etcd --pre-vote=true` by default.
+- [`etcd --initial-corrupt-check`](TODO) flag is now stable (`etcd --experimental-initial-corrupt-check`haisbeen  deprecated).
+  - `etcd --initial-corrupt-check=true` by default, to check cluster database hashes before serving client/peer traffic.
+- [`etcd --corrupt-check-time`](TODO) flag is now stable (`etcd --experimental-corrupt-check-time`haisbeen  deprecated).
+  - `etcd --corrupt-check-time=12h` by default, to check cluster database hashes for every 12-hour.
+- [`etcd --enable-v2v3`](TODO) flag is now stable.
+  - `etcd --experimental-enable-v2v3` has been deprecated.
+  - Added [more v2v3 integration tests](https://github.com/coreos/etcd/pull/9634).
+  - `etcd --enable-v2=true --enable-v2v3=''` by default, to enable v2 API server that is backed by **v2 store**.
+  - `etcd --enable-v2=true --enable-v2v3=/aaa` to enable v2 API server that is backed by **v3 storage**.
+  - `etcd --enable-v2=false --enable-v2v3=''` to disable v2 API server.
+  - `etcd --enable-v2=false --enable-v2v3=/aaa` to disable v2 API server. TODO: error?
+  - Automatically [create parent directory if it does not exist](https://github.com/coreos/etcd/pull/9626) (fix [issue#9609](https://github.com/coreos/etcd/issues/9609)).
+  - v4.0 will configure `etcd --enable-v2=true --enable-v2v3=/aaa` to enable v2 API server that is backed by **v3 storage**.
+- Add [`etcd --discovery-srv-name`](https://github.com/coreos/etcd/pull/8690) flag to support custom DNS SRV name with discovery.
+  - If not given, etcd queries `_etcd-server-ssl._tcp.[YOUR_HOST]` and `_etcd-server._tcp.[YOUR_HOST]`.
+  - If `etcd --discovery-srv-name="foo"`, then query `_etcd-server-ssl-foo._tcp.[YOUR_HOST]` and `_etcd-server-foo._tcp.[YOUR_HOST]`.
+  - Useful for operating multiple etcd clusters under the same domain.
+- Support TLS cipher suite whitelisting.
+  - To block [weak cipher suites](https://github.com/coreos/etcd/issues/8320).
+  - TLS handshake fails when client hello is requested with invalid cipher suites.
+  - Add [`etcd --cipher-suites`](https://github.com/coreos/etcd/pull/9801) flag.
+  - If empty, Go auto-populates the list.
+- Support [`etcd --cors`](https://github.com/coreos/etcd/pull/9490) in v3 HTTP requests (gRPC gateway).
+- Rename [`etcd --log-output` to `etcd --log-outputs`](https://github.com/coreos/etcd/pull/9624) to support multiple log outputs.
+  - **`etcd --log-output` will be deprecated in v3.5**.
+- Add [`etcd --logger`](https://github.com/coreos/etcd/pull/9572) flag to support [structured logger and multiple log outputs](https://github.com/coreos/etcd/issues/9438) in server-side.
+  - **`etcd --logger=capnslog` will be deprecated in v3.5**.
+  - Main motivation is to promote automated etcd monitoring, rather than looking back server logs when it starts breaking. Future development will make etcd log as few as possible, and make etcd easier to monitor with metrics and alerts.
+  - `etcd --logger=capnslog --log-outputs=default` is the default setting and same as previous etcd server logging format.
+  - `etcd --logger=zap --log-outputs=default` is not supported when `etcd --logger=zap`.
+    - Instead, use `etcd --logger=zap --log-outputs=stderr`.
+    - Or, use `etcd --logger=zap --log-outputs=systemd/journal` to send logs to the local systemd journal.
+    - Previously, if etcd parent process ID (PPID) is 1 (e.g. run with systemd), `etcd --logger=capnslog --log-outputs=default` redirects server logs to local systemd journal. And if write to journald fails, it writes to `os.Stderr` as a fallback.
+    - However, even with PPID 1, it can fail to dial systemd journal (e.g. run embedded etcd with Docker container). Then, [every single log write will fail](https://github.com/coreos/etcd/pull/9729) and fall back to `os.Stderr`, which is inefficient.
+    - To avoid this problem, systemd journal logging must be configured manually.
+  - `etcd --logger=zap --log-outputs=stderr` will log server operations in [JSON-encoded format](https://godoc.org/go.uber.org/zap#NewProductionEncoderConfig) and writes logs to `os.Stderr`. Use this to override journald log redirects.
+  - `etcd --logger=zap --log-outputs=stdout` will log server operations in [JSON-encoded format](https://godoc.org/go.uber.org/zap#NewProductionEncoderConfig) and writes logs to `os.Stdout` Use this to override journald log redirects.
+  - `etcd --logger=zap --log-outputs=a.log` will log server operations in [JSON-encoded format](https://godoc.org/go.uber.org/zap#NewProductionEncoderConfig) and writes logs to the specified file `a.log`.
+  - `etcd --logger=zap --log-outputs=a.log,b.log,c.log,stdout` [writes server logs to multiple files `a.log`, `b.log` and `c.log` at the same time](https://github.com/coreos/etcd/pull/9579) and outputs to `os.Stderr`, in [JSON-encoded format](https://godoc.org/go.uber.org/zap#NewProductionEncoderConfig).
+  - `etcd --logger=zap --log-outputs=/dev/null` will discard all server logs.
+- Fix [`mvcc` "unsynced" watcher restore operation](https://github.com/coreos/etcd/pull/9281).
+  - "unsynced" watcher is watcher that needs to be in sync with events that have happened.
+  - That is, "unsynced" watcher is the slow watcher that was requested on old revision.
+  - "unsynced" watcher restore operation was not correctly populating its underlying watcher group.
+  - Which possibly causes [missing events from "unsynced" watchers](https://github.com/coreos/etcd/issues/9086).
+  - A node gets network partitioned with a watcher on a future revision, and falls behind receiving a leader snapshot after partition gets removed. When applying this snapshot, etcd watch storage moves current synced watchers to unsynced since sync watchers might have become stale during network partition. And reset synced watcher group to restart watcher routines. Previously, there was a bug when moving from synced watcher group to unsynced, thus client would miss events when the watcher was requested to the network-partitioned node.
+- Fix [`mvcc` server panic from restore operation](https://github.com/coreos/etcd/pull/9775).
+  - Let's assume that a watcher had been requested with a future revision X and sent to node A that became network-partitioned thereafter. Meanwhile, cluster makes progress. Then when the partition gets removed, the leader sends a snapshot to node A. Previously if the snapshot's latest revision is still lower than the watch revision X,  **etcd server panicked** during snapshot restore operation.
+  - Now, this server-side panic has been fixed.
+- Fix [server panic on invalid Election Proclaim/Resign HTTP(S) requests](https://github.com/coreos/etcd/pull/9379).
+  - Previously, wrong-formatted HTTP requests to Election API could trigger panic in etcd server.
+  - e.g. `curl -L http://localhost:2379/v3/election/proclaim -X POST -d '{"value":""}'`, `curl -L http://localhost:2379/v3/election/resign -X POST -d '{"value":""}'`.
+- Fix [revision-based compaction retention parsing](https://github.com/coreos/etcd/pull/9339).
+  - Previously, `etcd --auto-compaction-mode revision --auto-compaction-retention 1` was [translated to revision retention 3600000000000](https://github.com/coreos/etcd/issues/9337).
+  - Now, `etcd --auto-compaction-mode revision --auto-compaction-retention 1` is correctly parsed as revision retention 1.
+- Prevent [overflow by large `TTL` values for `Lease` `Grant`](https://github.com/coreos/etcd/pull/9399).
+  - `TTL` parameter to `Grant` request is unit of second.
+  - Leases with too large `TTL` values exceeding `math.MaxInt64` [expire in unexpected ways](https://github.com/coreos/etcd/issues/9374).
+  - Server now returns `rpctypes.ErrLeaseTTLTooLarge` to client, when the requested `TTL` is larger than *9,000,000,000 seconds* (which is >285 years).
+  - Again, etcd `Lease` is meant for short-periodic keepalives or sessions, in the range of seconds or minutes. Not for hours or days!
+- Enable etcd server [`raft.Config.CheckQuorum` when starting with `ForceNewCluster`](https://github.com/coreos/etcd/pull/9347).
+- Allow [non-WAL files in `etcd --wal-dir` directory](https://github.com/coreos/etcd/pull/9743).
+  - Previously, existing files such as [`lost+found`](https://github.com/coreos/etcd/issues/7287) in WAL directory prevent etcd server boot.
+  - Now, WAL directory that contains only `lost+found` or a file that's not suffixed with `.wal` is considered non-initialized.
+
+### API
+
+- Add [`snapshot`](https://github.com/coreos/etcd/pull/9118) package for snapshot restore/save operations (see [`godoc.org/github.com/etcd/clientv3/snapshot`](https://godoc.org/github.com/coreos/etcd/clientv3/snapshot) for more).
+- Add [`watch_id` field to `etcdserverpb.WatchCreateRequest`](https://github.com/coreos/etcd/pull/9065) to allow user-provided watch ID to `mvcc`.
+  - Corresponding `watch_id` is returned via `etcdserverpb.WatchResponse`, if any.
+- Add [`fragment` field to `etcdserverpb.WatchCreateRequest`](https://github.com/coreos/etcd/pull/9291) to request etcd server to [split watch events](https://github.com/coreos/etcd/issues/9294) when the total size of events exceeds `etcd --max-request-bytes` flag value plus gRPC-overhead 512 bytes.
+  - The default server-side request bytes limit is `embed.DefaultMaxRequestBytes` which is 1.5 MiB plus gRPC-overhead 512 bytes.
+  - If watch response events exceed this server-side request limit and watch request is created with `fragment` field `true`, the server will split watch events into a set of chunks, each of which is a subset of watch events below server-side request limit.
+  - Useful when client-side has limited bandwidths.
+  - For example, watch response contains 10 events, where each event is 1 MiB. And server `etcd --max-request-bytes` flag value is 1 MiB. Then, server will send 10 separate fragmented events to the client.
+  - For example, watch response contains 5 events, where each event is 2 MiB. And server `etcd --max-request-bytes` flag value is 1 MiB and `clientv3.Config.MaxCallRecvMsgSize` is 1 MiB. Then, server will try to send 5 separate fragmented events to the client, and the client will error with `"code = ResourceExhausted desc = grpc: received message larger than max (...)"`.
+  - Client must implement fragmented watch event merge (which `clientv3` does in etcd v3.4).
+- Add [`raftAppliedIndex` field to `etcdserverpb.StatusResponse`](https://github.com/coreos/etcd/pull/9176) for current Raft applied index.
+- Add [`errors` field to `etcdserverpb.StatusResponse`](https://github.com/coreos/etcd/pull/9206) for server-side error.
+  - e.g. `"etcdserver: no leader", "NOSPACE", "CORRUPT"`
+- Add [`dbSizeInUse` field to `etcdserverpb.StatusResponse`](https://github.com/coreos/etcd/pull/9256) for actual DB size after compaction.
+- Add [`WatchRequest.WatchProgressRequest`](https://github.com/coreos/etcd/pull/9869).
+  - To manually trigger broadcasting watch progress event (empty watch response with latest header) to all associated watch streams.
+  - Think of it as `WithProgressNotify` that can be triggered manually.
+
+Note: **v3.5 will deprecate `etcd --log-package-levels` flag for `capnslog`**; `etcd --logger=zap --log-outputs=stderr` will the default. **v3.5 will deprecate `[CLIENT-URL]/config/local/log` endpoint.**
+
+### Package `embed`
+
+- Add [`embed.Config.CipherSuites`](https://github.com/coreos/etcd/pull/9801) to specify a list of supported cipher suites for TLS handshake between client/server  and peers.
+  - If empty, Go auto-populates the list.
+  - Both `embed.Config.ClientTLSInfo.CipherSuites` and `embed.Config.CipherSuites` cannot be non-empty at the same time.
+  - If not empty, specify either `embed.Config.ClientTLSInfo.CipherSuites` or `embed.Config.CipherSuites`.
+- Add [`embed.Config.InitialElectionTickAdvance`](https://github.com/coreos/etcd/pull/9591) to enable/disable initial election tick fast-forward.
+  - `embed.NewConfig()` would return `*embed.Config` with `InitialElectionTickAdvance` as true by default.
+- Define [`embed.CompactorModePeriodic`](https://godoc.org/github.com/coreos/etcd/embed#pkg-variables) for `compactor.ModePeriodic`.
+- Define [`embed.CompactorModeRevision`](https://godoc.org/github.com/coreos/etcd/embed#pkg-variables) for `compactor.ModeRevision`.
+- Change [`embed.Config.CorsInfo` in `*cors.CORSInfo` type to `embed.Config.CORS` in `map[string]struct{}` type](https://github.com/coreos/etcd/pull/9490).
+- Remove [`embed.Config.SetupLogging`](https://github.com/coreos/etcd/pull/9572).
+  - Now logger is set up automatically based on [`embed.Config.Logger`, `embed.Config.LogOutputs`, `embed.Config.Debug` fields](https://github.com/coreos/etcd/pull/9572).
+- Add [`embed.Config.Logger`](https://github.com/coreos/etcd/pull/9518) to support [structured logger `zap`](https://github.com/uber-go/zap) in server-side.
+- Rename `embed.Config.SnapCount` field to [`embed.Config.SnapshotCount`](https://github.com/coreos/etcd/pull/9745), to be consistent with the flag name `etcd --snapshot-count`.
+- Rename [**`embed.Config.LogOutput`** to **`embed.Config.LogOutputs`**](https://github.com/coreos/etcd/pull/9624) to support multiple log outputs.
+- Change [**`embed.Config.LogOutputs`** type from `string` to `[]string`](https://github.com/coreos/etcd/pull/9579) to support multiple log outputs.
+
+### Package `integration`
+
+- Add [`CLUSTER_DEBUG` to enable test cluster logging](https://github.com/coreos/etcd/pull/9678).
+  - Deprecated `capnslog` in integration tests.
+
+### client v3
+
+- Add [`WithFragment` `OpOption`](https://github.com/coreos/etcd/pull/9291) to support [watch events fragmentation](https://github.com/coreos/etcd/issues/9294) when the total size of events exceeds `etcd --max-request-bytes` flag value plus gRPC-overhead 512 bytes.
+  - Watch fragmentation is disabled by default.
+  - The default server-side request bytes limit is `embed.DefaultMaxRequestBytes` which is 1.5 MiB plus gRPC-overhead 512 bytes.
+  - If watch response events exceed this server-side request limit and watch request is created with `fragment` field `true`, the server will split watch events into a set of chunks, each of which is a subset of watch events below server-side request limit.
+  - Useful when client-side has limited bandwidths.
+  - For example, watch response contains 10 events, where each event is 1 MiB. And server `etcd --max-request-bytes` flag value is 1 MiB. Then, server will send 10 separate fragmented events to the client.
+  - For example, watch response contains 5 events, where each event is 2 MiB. And server `etcd --max-request-bytes` flag value is 1 MiB and `clientv3.Config.MaxCallRecvMsgSize` is 1 MiB. Then, server will try to send 5 separate fragmented events to the client, and the client will error with `"code = ResourceExhausted desc = grpc: received message larger than max (...)"`.
+- Add [`Watcher.RequestProgress` method](https://github.com/coreos/etcd/pull/9869).
+  - To manually trigger broadcasting watch progress event (empty watch response with latest header) to all associated watch streams.
+  - Think of it as `WithProgressNotify` that can be triggered manually.
+- Fix [lease keepalive interval updates when response queue is full](https://github.com/coreos/etcd/pull/9952).
+  - If `<-chan *clientv3LeaseKeepAliveResponse` from `clientv3.Lease.KeepAlive` was never consumed or channel is full, client was [sending keepalive request every 500ms](https://github.com/coreos/etcd/issues/9911) instead of expected rate of every "TTL / 3" duration.
+- Change [snapshot file permissions](https://github.com/coreos/etcd/pull/9977): On Linux, the snapshot file changes from readable by all (mode 0644) to readable by the user only (mode 0600).
+
+### etcdctl v3
+
+- Make [`ETCDCTL_API=3 etcdctl` default](https://github.com/coreos/etcd/issues/9600).
+  - Now, `etcdctl set foo bar` must be `ETCDCTL_API=2 etcdctl set foo bar`.
+  - Now, `ETCDCTL_API=3 etcdctl put foo bar` could be just `etcdctl put foo bar`.
+- Add [`etcdctl --password`](https://github.com/coreos/etcd/pull/9730) flag.
+  - To support [`:` character in user name](https://github.com/coreos/etcd/issues/9691).
+  - e.g. `etcdctl --user user --password password get foo`
+- Add [`etcdctl user add --new-user-password`](https://github.com/coreos/etcd/pull/9730) flag.
+- Add [`etcdctl check datascale`](https://github.com/coreos/etcd/pull/9185) command.
+- Add [`etcdctl check datascale --auto-compact, --auto-defrag`](https://github.com/coreos/etcd/pull/9351) flags.
+- Add [`etcdctl check perf --auto-compact, --auto-defrag`](https://github.com/coreos/etcd/pull/9330) flags.
+- Add [`etcdctl defrag --cluster`](https://github.com/coreos/etcd/pull/9390) flag.
+- Add ["raft applied index" field to `endpoint status`](https://github.com/coreos/etcd/pull/9176).
+- Add ["errors" field to `endpoint status`](https://github.com/coreos/etcd/pull/9206).
+- Add [`etcdctl endpoint health --write-out` support](https://github.com/coreos/etcd/pull/9540).
+  - Previously, [`etcdctl endpoint health --write-out json` did not work](https://github.com/coreos/etcd/issues/9532).
+- Fix [`etcdctl watch [key] [range_end] -- [exec-command…]`](https://github.com/coreos/etcd/pull/9688) parsing.
+  - Previously,  `ETCDCTL_API=3 etcdctl watch foo -- echo watch event received` panicked.
+- Fix [`etcdctl move-leader` command for TLS-enabled endpoints](https://github.com/coreos/etcd/pull/9807).
+- Add [`progress` command to `etcdctl watch --interactive`](https://github.com/coreos/etcd/pull/9869).
+  - To manually trigger broadcasting watch progress event (empty watch response with latest header) to all associated watch streams.
+  - Think of it as `WithProgressNotify` that can be triggered manually.
+
+### gRPC proxy
+
+- Fix [etcd server panic from restore operation](https://github.com/coreos/etcd/pull/9775).
+  - Let's assume that a watcher had been requested with a future revision X and sent to node A that became network-partitioned thereafter. Meanwhile, cluster makes progress. Then when the partition gets removed, the leader sends a snapshot to node A. Previously if the snapshot's latest revision is still lower than the watch revision X,  **etcd server panicked** during snapshot restore operation.
+  - Especially, gRPC proxy was affected, since it detects a leader loss with a key `"proxy-namespace__lostleader"` and a watch revision `"int64(math.MaxInt64 - 2)"`.
+  - Now, this server-side panic has been fixed.
+
+### gRPC gateway
+
+- Replace [gRPC gateway](https://github.com/grpc-ecosystem/grpc-gateway) endpoint `/v3beta` with [`/v3`](https://github.com/coreos/etcd/pull/9298).
+  - Deprecated [`/v3alpha`](https://github.com/coreos/etcd/pull/9298).
+  - To deprecate [`/v3beta`](https://github.com/coreos/etcd/issues/9189) in v3.5.
+  - In v3.4, `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` still works as a fallback to `curl -L http://localhost:2379/v3/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'`, but `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` won't work in v3.5. Use `curl -L http://localhost:2379/v3/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` instead.
+- Add API endpoints [`/{v3beta,v3}/lease/leases, /{v3beta,v3}/lease/revoke, /{v3beta,v3}/lease/timetolive`](https://github.com/coreos/etcd/pull/9450).
+  - To deprecate [`/{v3beta,v3}/kv/lease/leases, /{v3beta,v3}/kv/lease/revoke, /{v3beta,v3}/kv/lease/timetolive`](https://github.com/coreos/etcd/issues/9430) in v3.5.
+- Support [`etcd --cors`](https://github.com/coreos/etcd/pull/9490) in v3 HTTP requests (gRPC gateway).
+
+### Package `raft`
+
+- Fix [deadlock during PreVote migration process](https://github.com/coreos/etcd/pull/8525).
+- Add [`raft.ErrProposalDropped`](https://github.com/coreos/etcd/pull/9067).
+  - Now [`(r *raft) Step` returns `raft.ErrProposalDropped`](https://github.com/coreos/etcd/pull/9137) if a proposal has been ignored.
+  - e.g. a node is removed from cluster, or [`raftpb.MsgProp` arrives at current leader while there is an ongoing leadership transfer](https://github.com/coreos/etcd/issues/8975).
+- Improve [Raft `becomeLeader` and `stepLeader`](https://github.com/coreos/etcd/pull/9073) by keeping track of latest `pb.EntryConfChange` index.
+  - Previously record `pendingConf` boolean field scanning the entire tail of the log, which can delay hearbeat send.
+- Fix [missing learner nodes on `(n *node) ApplyConfChange`](https://github.com/coreos/etcd/pull/9116).
+
+### Tooling
+
+- Add [`etcd-dump-logs --entry-type`](https://github.com/coreos/etcd/pull/9628) flag to support WAL log filtering by entry type.
+- Add [`etcd-dump-logs --stream-decoder`](https://github.com/coreos/etcd/pull/9790) flag to support custom decoder.
+
+### Go
+
+- Require *Go 1.10+*.
+- Compile with [*Go 1.10.3*](https://golang.org/doc/devel/release.html#go1.10).
+

CHANGELOG-3.5.md

@@ -0,0 +1,43 @@
+
+
+Previous change logs can be found at [CHANGELOG-3.4](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.4.md).
+
+
+## v3.5.0 (TBD 2018-12)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.4.0...v3.5.0) and [v3.5 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_5.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.5 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_5.md).**
+
+### Breaking Changes
+
+- [gRPC gateway](https://github.com/grpc-ecosystem/grpc-gateway) only supports [`/v3`](TODO) endpoint.
+  - Deprecated [`/v3beta`](https://github.com/coreos/etcd/pull/9298).
+  - `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` does work in v3.5. Use `curl -L http://localhost:2379/v3/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` instead.
+- **`etcd --log-output` flag has been deprecated.** Use **`etcd --log-outputs`** instead.
+- **`etcd --logger=zap --log-outputs=stderr`** is now the default.
+- **`etcd --logger=capnslog` flag has been deprecated.**
+- **`etcd --logger=zap --log-outputs=default` flag value is not supported.**.
+  - Instead, use `etcd --logger=zap --log-outputs=stderr`.
+  - Or, use `etcd --logger=zap --log-outputs=systemd/journal` to send logs to the local systemd journal.
+  - Previously, if etcd parent process ID (PPID) is 1 (e.g. run with systemd), `etcd --logger=capnslog --log-outputs=default` redirects server logs to local systemd journal. And if write to journald fails, it writes to `os.Stderr` as a fallback.
+  - However, even with PPID 1, it can fail to dial systemd journal (e.g. run embedded etcd with Docker container). Then, [every single log write will fail](https://github.com/coreos/etcd/pull/9729) and fall back to `os.Stderr`, which is inefficient.
+  - To avoid this problem, systemd journal logging must be configured manually.
+- **`etcd --log-outputs=stderr`** is now the default.
+- **`etcd --log-package-levels` flag for `capnslog` has been deprecated.** Now, **`etcd --logger=zap --log-outputs=stderr`** is the default.
+- **`[CLIENT-URL]/config/local/log` endpoint has been deprecated, as is `etcd --log-package-levels` flag.**
+  - `curl http://127.0.0.1:2379/config/local/log -XPUT -d '{"Level":"DEBUG"}'` won't work.
+  - Please use `etcd --logger=zap --log-outputs=stderr` instead.
+- Deprecated `etcd_debugging_mvcc_db_total_size_in_bytes` Prometheus metric. Instead, use `etcd_mvcc_db_total_size_in_bytes`.
+
+### Metrics, Monitoring
+
+See [List of metrics](https://etcd.readthedocs.io/en/latest/operate.html#latest) for all metrics per release.
+
+Note that any `etcd_debugging_*` metrics are experimental and subject to change.
+
+- Deprecated `etcd_debugging_mvcc_db_total_size_in_bytes` Prometheus metric. Instead, use `etcd_mvcc_db_total_size_in_bytes`.
+
+### gRPC gateway
+
+- [gRPC gateway](https://github.com/grpc-ecosystem/grpc-gateway) only supports [`/v3`](TODO) endpoint.
+  - Deprecated [`/v3beta`](https://github.com/coreos/etcd/pull/9298).
+  - `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` does work in v3.5. Use `curl -L http://localhost:2379/v3/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` instead.

CHANGELOG-4.0.md

@@ -0,0 +1,28 @@
+
+
+Previous change logs can be found at [CHANGELOG-3.x](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.x.md).
+
+
+Planned breaking changes.
+
+
+## v4.0.0 (TBD)
+
+See [code changes](https://github.com/coreos/etcd/compare/v3.3.0...v4.0.0) and [v4.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_4_0.md) for any breaking changes. **Again, before running upgrades from any previous release, please make sure to read change logs below and [v4.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_4_0.md).**
+
+### Breaking Changes
+
+- [Secure etcd by default](https://github.com/coreos/etcd/issues/9475)?
+- Change `/health` endpoint output.
+  - Previously, `{"health":"true"}`.
+  - Now, `{"health":true}`.
+  - Breaks [Kubernetes `kubectl get componentstatuses` command](https://github.com/kubernetes/kubernetes/issues/58240).
+- Deprecate [`etcd --proxy*`](TODO) flags; **no more v2 proxy**.
+- Deprecate [v2 storage backend](https://github.com/coreos/etcd/issues/9232); **no more v2 store**.
+  - v2 API is still supported via [v2 emulation](TODO).
+- Deprecate [`etcdctl backup`](TODO) command.
+- `clientv3.Client.KeepAlive(ctx context.Context, id LeaseID) (<-chan *LeaseKeepAliveResponse, error)` is now [`clientv4.Client.KeepAlive(ctx context.Context, id LeaseID) <-chan *LeaseKeepAliveResponse`](TODO).
+  - Similar to `Watch`, [`KeepAlive` does not return errors](https://github.com/coreos/etcd/issues/7488).
+  - If there's an unknown server error, kill all open channels and create a new stream on the next `KeepAlive` call.
+- Rename `github.com/coreos/client` to `github.com/coreos/clientv2`.
+

CHANGELOG.md

@@ -1,750 +0,0 @@
-## [v3.3.0](https://github.com/coreos/etcd/releases/tag/v3.3.0) (2018-01-??)
-
-**v3.3.0 is not yet released; expected to be released in January 2018.**
-
-## [v3.3.0-rc.0](https://github.com/coreos/etcd/releases/tag/v3.3.0-rc.0) (2017-12-20)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.2.0...v3.3.0) and [v3.3 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md) for any breaking changes.
-
-### Improved
-
-- Use [`coreos/bbolt`](https://github.com/coreos/bbolt/releases) to replace [`boltdb/bolt`](https://github.com/boltdb/bolt#project-status).
-  - Fix [etcd database size grows until `mvcc: database space exceeded`](https://github.com/coreos/etcd/issues/8009).
-- [Reduce memory allocation](https://github.com/coreos/etcd/pull/8428) on [Range operations](https://github.com/coreos/etcd/pull/8475).
-- [Rate limit](https://github.com/coreos/etcd/pull/8099) and [randomize](https://github.com/coreos/etcd/pull/8101) lease revoke on restart or leader elections.
-  - Prevent [spikes in Raft proposal rate](https://github.com/coreos/etcd/issues/8096).
-- Support `clientv3` balancer failover under [network faults/partitions](https://github.com/coreos/etcd/issues/8711).
-- Better warning on [mismatched `--initial-cluster`](https://github.com/coreos/etcd/pull/8083) flag.
-
-### Changed(Breaking Changes)
-
-- Require [Go 1.9+](https://github.com/coreos/etcd/issues/6174).
-  - Compile with *Go 1.9.2*.
-  - Deprecate [`golang.org/x/net/context`](https://github.com/coreos/etcd/pull/8511).
-- Require [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) [**`v1.7.4`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.4) or [**`v1.7.5+`**](https://github.com/grpc/grpc-go/releases/tag/v1.7.5):
-  - Deprecate [`metadata.Incoming/OutgoingContext`](https://github.com/coreos/etcd/pull/7896).
-  - Deprecate `grpclog.Logger`, upgrade to [`grpclog.LoggerV2`](https://github.com/coreos/etcd/pull/8533).
-  - Deprecate [`grpc.ErrClientConnTimeout`](https://github.com/coreos/etcd/pull/8505) errors in `clientv3`.
-  - Use [`MaxRecvMsgSize` and `MaxSendMsgSize`](https://github.com/coreos/etcd/pull/8437) to limit message size, in etcd server.
-- Upgrade [`github.com/grpc-ecosystem/grpc-gateway`](https://github.com/grpc-ecosystem/grpc-gateway/releases) `v1.2.2` to `v1.3.0`.
-- Translate [gRPC status error in v3 client `Snapshot` API](https://github.com/coreos/etcd/pull/9038).
-- Upgrade [`github.com/ugorji/go/codec`](https://github.com/ugorji/go) for v2 `client`.
-  - [Regenerated](https://github.com/coreos/etcd/pull/8721) v2 `client` source code with latest `ugorji/go/codec`.
-- Fix [`/health` endpoint JSON output](https://github.com/coreos/etcd/pull/8312).
-- v3 `etcdctl` [`lease timetolive LEASE_ID`](https://github.com/coreos/etcd/issues/9028) on expired lease now prints [`lease LEASE_ID already expired`](https://github.com/coreos/etcd/pull/9047).
-  - <=3.2 prints `lease LEASE_ID granted with TTL(0s), remaining(-1s)`.
-
-### Added(`etcd`)
-
-- Add [`--experimental-enable-v2v3`](https://github.com/coreos/etcd/pull/8407) flag to [emulate v2 API with v3](https://github.com/coreos/etcd/issues/6925).
-- Add [`--experimental-corrupt-check-time`](https://github.com/coreos/etcd/pull/8420) flag to [raise corrupt alarm monitoring](https://github.com/coreos/etcd/issues/7125).
-- Add [`--experimental-initial-corrupt-check`](https://github.com/coreos/etcd/pull/8554) flag to [check database hash before serving client/peer traffic](https://github.com/coreos/etcd/issues/8313).
-- Add [`--max-txn-ops`](https://github.com/coreos/etcd/pull/7976) flag to [configure maximum number operations in transaction](https://github.com/coreos/etcd/issues/7826).
-- Add [`--max-request-bytes`](https://github.com/coreos/etcd/pull/7968) flag to [configure maximum client request size](https://github.com/coreos/etcd/issues/7923).
-  - If not configured, it defaults to 1.5 MiB.
-- Add [`--client-crl-file`, `--peer-crl-file`](https://github.com/coreos/etcd/pull/8124) flags for [Certificate revocation list](https://github.com/coreos/etcd/issues/4034).
-- Add [`--peer-require-cn`](https://github.com/coreos/etcd/pull/8616) flag to support [CN-based auth for inter-peer connection](https://github.com/coreos/etcd/issues/8262).
-- Add [`--listen-metrics-urls`](https://github.com/coreos/etcd/pull/8242) flag for additional `/metrics` endpoints.
-  - Support [additional (non) TLS `/metrics` endpoints for a TLS-enabled cluster](https://github.com/coreos/etcd/pull/8282).
-  - e.g. `--listen-metrics-urls=https://localhost:2378,http://localhost:9379` to serve `/metrics` in secure port 2378 and insecure port 9379.
-  - Useful for [bypassing critical APIs when monitoring etcd](https://github.com/coreos/etcd/issues/8060).
-- Add [`--auto-compaction-mode`](https://github.com/coreos/etcd/pull/8123) flag to [support revision-based compaction](https://github.com/coreos/etcd/issues/8098).
-- Change `--auto-compaction-retention` flag to [accept string values](https://github.com/coreos/etcd/pull/8563) with [finer granularity](https://github.com/coreos/etcd/issues/8503).
-- Add [`--grpc-keepalive-min-time`, `--grpc-keepalive-interval`, `--grpc-keepalive-timeout`](https://github.com/coreos/etcd/pull/8535) flags to configure server-side keepalive policies.
-- Serve [`/health` endpoint as unhealthy](https://github.com/coreos/etcd/pull/8272) when [alarm is raised](https://github.com/coreos/etcd/issues/8207).
-- Provide [error information in `/health`](https://github.com/coreos/etcd/pull/8312).
-  - e.g. `{"health":false,"errors":["NOSPACE"]}`.
-- Move [logging setup to embed package](https://github.com/coreos/etcd/pull/8810)
-  - Disable gRPC server log by default.
-- Use [monotonic time in Go 1.9](https://github.com/coreos/etcd/pull/8507) for `lease` package.
-- Warn on [empty hosts in advertise URLs](https://github.com/coreos/etcd/pull/8384).
-  - Address [advertise client URLs accepts empty hosts](https://github.com/coreos/etcd/issues/8379).
-  - etcd `v3.4` will exit on this error.
-    - e.g. `--advertise-client-urls=http://:2379`.
-- Warn on [shadowed environment variables](https://github.com/coreos/etcd/pull/8385).
-  - Address [error on shadowed environment variables](https://github.com/coreos/etcd/issues/8380).
-  - etcd `v3.4` will exit on this error.
-
-### Added(API)
-
-- Support [ranges in transaction comparisons](https://github.com/coreos/etcd/pull/8025) for [disconnected linearized reads](https://github.com/coreos/etcd/issues/7924).
-- Add [nested transactions](https://github.com/coreos/etcd/pull/8102) to extend [proxy use cases](https://github.com/coreos/etcd/issues/7857).
-- Add [lease comparison target in transaction](https://github.com/coreos/etcd/pull/8324).
-- Add [lease list](https://github.com/coreos/etcd/pull/8358).
-- Add [hash by revision](https://github.com/coreos/etcd/pull/8263) for [better corruption checking against boltdb](https://github.com/coreos/etcd/issues/8016).
-
-### Added(`etcd/clientv3`)
-
-- Add [health balancer](https://github.com/coreos/etcd/pull/8545) to fix [watch API hangs](https://github.com/coreos/etcd/issues/7247), improve [endpoint switch under network faults](https://github.com/coreos/etcd/issues/7941).
-- [Refactor balancer](https://github.com/coreos/etcd/pull/8840) and add [client-side keepalive pings](https://github.com/coreos/etcd/pull/8199) to handle [network partitions](https://github.com/coreos/etcd/issues/8711).
-- Add [`MaxCallSendMsgSize` and `MaxCallRecvMsgSize`](https://github.com/coreos/etcd/pull/9047) fields to [`clientv3.Config`](https://godoc.org/github.com/coreos/etcd/clientv3#Config).
-  - Fix [exceeded response size limit error in client-side](https://github.com/coreos/etcd/issues/9043).
-  - Address [kubernetes#51099](https://github.com/kubernetes/kubernetes/issues/51099).
-  - `MaxCallSendMsgSize` default value is 2 MiB, if not configured.
-  - `MaxCallRecvMsgSize` default value is `math.MaxInt32`, if not configured.
-- Accept [`Compare_LEASE`](https://github.com/coreos/etcd/pull/8324) in [`clientv3.Compare`](https://godoc.org/github.com/coreos/etcd/clientv3#Compare).
-- Add [`LeaseValue` helper](https://github.com/coreos/etcd/pull/8488) to `Cmp` `LeaseID` values in `Txn`.
-- Add [`MoveLeader`](https://github.com/coreos/etcd/pull/8153) to `Maintenance`.
-- Add [`HashKV`](https://github.com/coreos/etcd/pull/8351) to `Maintenance`.
-- Add [`Leases`](https://github.com/coreos/etcd/pull/8358) to `Lease`.
-- Add [`clientv3/ordering`](https://github.com/coreos/etcd/pull/8092) for enforce [ordering in serialized requests](https://github.com/coreos/etcd/issues/7623).
-
-### Added(v2 `etcdctl`)
-
-- Add [`backup --with-v3`](https://github.com/coreos/etcd/pull/8479) flag.
-
-### Added(v3 `etcdctl`)
-
-- Add [`--discovery-srv`](https://github.com/coreos/etcd/pull/8462) flag.
-- Add [`--keepalive-time`, `--keepalive-timeout`](https://github.com/coreos/etcd/pull/8663) flags.
-- Add [`lease list`](https://github.com/coreos/etcd/pull/8358) command.
-- Add [`lease keep-alive --once`](https://github.com/coreos/etcd/pull/8775) flag.
-- Make [`lease timetolive LEASE_ID`](https://github.com/coreos/etcd/issues/9028) on expired lease print [`lease LEASE_ID already expired`](https://github.com/coreos/etcd/pull/9047).
-  - <=3.2 prints `lease LEASE_ID granted with TTL(0s), remaining(-1s)`.
-- Add [`defrag --data-dir`](https://github.com/coreos/etcd/pull/8367) flag.
-- Add [`move-leader`](https://github.com/coreos/etcd/pull/8153) command.
-- Add [`endpoint hashkv`](https://github.com/coreos/etcd/pull/8351) command.
-- Add [`endpoint --cluster`](https://github.com/coreos/etcd/pull/8143) flag, equivalent to [v2 `etcdctl cluster-health`](https://github.com/coreos/etcd/issues/8117).
-- Make `endpoint health` command terminate with [non-zero exit code on unhealthy status](https://github.com/coreos/etcd/pull/8342).
-- Add [`lock --ttl`](https://github.com/coreos/etcd/pull/8370) flag.
-- Support [`watch [key] [range_end] -- [exec-command…]`](https://github.com/coreos/etcd/pull/8919), equivalent to [v2 `etcdctl exec-watch`](https://github.com/coreos/etcd/issues/8814).
-- Enable [`clientv3.WithRequireLeader(context.Context)` for `watch`](https://github.com/coreos/etcd/pull/8672) command.
-- Print [`"del"` instead of `"delete"`](https://github.com/coreos/etcd/pull/8297) in `txn` interactive mode.
-- Print [`ETCD_INITIAL_ADVERTISE_PEER_URLS` in `member add`](https://github.com/coreos/etcd/pull/8332).
-
-### Added(metrics)
-
-- Add [`etcd --listen-metrics-urls`](https://github.com/coreos/etcd/pull/8242) flag for additional `/metrics` endpoints.
-  - Useful for [bypassing critical APIs when monitoring etcd](https://github.com/coreos/etcd/issues/8060).
-- Add [`etcd_server_version`](https://github.com/coreos/etcd/pull/8960) Prometheus metric.
-  - To replace [Kubernetes `etcd-version-monitor`](https://github.com/coreos/etcd/issues/8948).
-- Add [`etcd_debugging_mvcc_db_compaction_keys_total`](https://github.com/coreos/etcd/pull/8280) Prometheus metric.
-- Add [`etcd_debugging_server_lease_expired_total`](https://github.com/coreos/etcd/pull/8064) Prometheus metric.
-  - To improve [lease revoke monitoring](https://github.com/coreos/etcd/issues/8050).
-- Document [Prometheus 2.0 rules](https://github.com/coreos/etcd/pull/8879).
-- Initialize gRPC server [metrics with zero values](https://github.com/coreos/etcd/pull/8878).
-
-### Added(`grpc-proxy`)
-
-- Add [`grpc-proxy start --experimental-leasing-prefix`](https://github.com/coreos/etcd/pull/8341) flag:
-  - For disconnected linearized reads.
-  - Based on [V system leasing](https://github.com/coreos/etcd/issues/6065).
-  - See ["Disconnected consistent reads with etcd" blog post](https://coreos.com/blog/coreos-labs-disconnected-consistent-reads-with-etcd).
-- Add [`grpc-proxy start --experimental-serializable-ordering`](https://github.com/coreos/etcd/pull/8315) flag.
-  - To ensure serializable reads have monotonically increasing store revisions across endpoints.
-- Add [`grpc-proxy start --metrics-addr`](https://github.com/coreos/etcd/pull/8242) flag for an additional `/metrics` endpoint.
-  - Set `--metrics-addr=http://[HOST]:9379` to serve `/metrics` in insecure port 9379.
-- Serve [`/health` endpoint in grpc-proxy](https://github.com/coreos/etcd/pull/8322).
-- Add [`grpc-proxy start --debug`](https://github.com/coreos/etcd/pull/8994) flag.
-
-### Added(gRPC gateway)
-
-- Replace [gRPC gateway](https://github.com/grpc-ecosystem/grpc-gateway) endpoint with [`/v3beta`](https://github.com/coreos/etcd/pull/8880).
-  - To deprecate [`/v3alpha`](https://github.com/coreos/etcd/issues/8125) in `v3.4`.
-- Support ["authorization" token](https://github.com/coreos/etcd/pull/7999).
-- Support [websocket for bi-directional streams](https://github.com/coreos/etcd/pull/8257).
-  - Fix [`Watch` API with gRPC gateway](https://github.com/coreos/etcd/issues/8237).
-- Upgrade gRPC gateway to [v1.3.0](https://github.com/coreos/etcd/issues/8838).
-
-### Added(`etcd/raft`)
-
-- Add [non-voting member](https://github.com/coreos/etcd/pull/8751).
-  - To implement [Raft thesis 4.2.1 Catching up new servers](https://github.com/coreos/etcd/issues/8568).
-  - `Learner` node does not vote or promote itself.
-
-### Added/Fixed(Security/Auth)
-
-- Add [CRL based connection rejection](https://github.com/coreos/etcd/pull/8124) to manage [revoked certs](https://github.com/coreos/etcd/issues/4034).
-- Document [TLS authentication changes](https://github.com/coreos/etcd/pull/8895):
-  - [Server accepts connections if IP matches, without checking DNS entries](https://github.com/coreos/etcd/pull/8223). For instance, if peer cert contains IP addresses and DNS names in Subject Alternative Name (SAN) field, and the remote IP address matches one of those IP addresses, server just accepts connection without further checking the DNS names.
-  - [Server supports reverse-lookup on wildcard DNS `SAN`](https://github.com/coreos/etcd/pull/8281). For instance, if peer cert contains only DNS names (no IP addresses) in Subject Alternative Name (SAN) field, server first reverse-lookups the remote IP address to get a list of names mapping to that address (e.g. `nslookup IPADDR`). Then accepts the connection if those names have a matching name with peer cert's DNS names (either by exact or wildcard match). If none is matched, server forward-lookups each DNS entry in peer cert (e.g. look up `example.default.svc` when the entry is `*.example.default.svc`), and accepts connection only when the host's resolved addresses have the matching IP address with the peer's remote IP address.
-- Add [`etcd --peer-require-cn`](https://github.com/coreos/etcd/pull/8616) flag.
-  - To support [CommonName(CN) based auth](https://github.com/coreos/etcd/issues/8262) for inter peer connection.
-- [Swap priority](https://github.com/coreos/etcd/pull/8594) of cert CommonName(CN) and username + password.
-  - To address ["username and password specified in the request should take priority over CN in the cert"](https://github.com/coreos/etcd/issues/8584).
-- Protect [lease revoke with auth](https://github.com/coreos/etcd/pull/8031).
-- Provide user's role on [auth permission error](https://github.com/coreos/etcd/pull/8164).
-- Fix [auth store panic with disabled token](https://github.com/coreos/etcd/pull/8695).
-- Update `golang.org/x/crypto/bcrypt` (see [golang/crypto@6c586e1](https://github.com/golang/crypto/commit/6c586e17d90a7d08bbbc4069984180dce3b04117)).
-
-### Fixed(v2)
-
-- [Fail-over v2 client](https://github.com/coreos/etcd/pull/8519) to next endpoint on [oneshot failure](https://github.com/coreos/etcd/issues/8515).
-- [Put back `/v2/machines`](https://github.com/coreos/etcd/pull/8062) endpoint for python-etcd wrapper.
-
-### Fixed(v3)
-
-- Fix [range/put/delete operation metrics](https://github.com/coreos/etcd/pull/8054) with transaction:
-  - `etcd_debugging_mvcc_range_total`
-  - `etcd_debugging_mvcc_put_total`
-  - `etcd_debugging_mvcc_delete_total`
-  - `etcd_debugging_mvcc_txn_total`
-- Fix [`etcd_debugging_mvcc_keys_total`](https://github.com/coreos/etcd/pull/8390) on restore.
-- Fix [`etcd_debugging_mvcc_db_total_size_in_bytes`](https://github.com/coreos/etcd/pull/8120) on restore.
-  - Also change to [`prometheus.NewGaugeFunc`](https://github.com/coreos/etcd/pull/8150).
-- Fix [backend database in-memory index corruption](https://github.com/coreos/etcd/pull/8127) issue on restore (only 3.2.0 is affected).
-- Fix [watch restore from snapshot](https://github.com/coreos/etcd/pull/8427).
-- Fix ["put at-most-once" in `clientv3`](https://github.com/coreos/etcd/pull/8335).
-- Handle [empty key permission](https://github.com/coreos/etcd/pull/8514) in `etcdctl`.
-- [Fix server crash](https://github.com/coreos/etcd/pull/8010) on [invalid transaction request from gRPC gateway](https://github.com/coreos/etcd/issues/7889).
-- Fix [`clientv3.WatchResponse.Canceled`](https://github.com/coreos/etcd/pull/8283) on [compacted watch request](https://github.com/coreos/etcd/issues/8231).
-- Handle [WAL renaming failure on Windows](https://github.com/coreos/etcd/pull/8286).
-- Make [peer dial timeout longer](https://github.com/coreos/etcd/pull/8599).
-  - See [coreos/etcd-operator#1300](https://github.com/coreos/etcd-operator/issues/1300) for more detail.
-- Make server [wait up to request time-out](https://github.com/coreos/etcd/pull/8267) with [pending RPCs](https://github.com/coreos/etcd/issues/8224).
-- Fix [`grpc.Server` panic on `GracefulStop`](https://github.com/coreos/etcd/pull/8987) with [TLS-enabled server](https://github.com/coreos/etcd/issues/8916).
-- Fix ["multiple peer URLs cannot start" issue](https://github.com/coreos/etcd/issues/8383).
-- Fix server-side auth so [concurrent auth operations do not return old revision error](https://github.com/coreos/etcd/pull/8442).
-- Fix [`concurrency/stm` `Put` with serializable snapshot](https://github.com/coreos/etcd/pull/8439).
-  - Use store revision from first fetch to resolve write conflicts instead of modified revision.
-- Fix [`grpc-proxy` Snapshot API error handling](https://github.com/coreos/etcd/commit/dbd16d52fbf81e5fd806d21ff5e9148d5bf203ab).
-- Fix [`grpc-proxy` KV API `PrevKv` flag handling](https://github.com/coreos/etcd/pull/8366).
-- Fix [`grpc-proxy` KV API `KeysOnly` flag handling](https://github.com/coreos/etcd/pull/8552).
-- Upgrade [`coreos/go-systemd`](https://github.com/coreos/go-systemd/releases) to `v15` (see https://github.com/coreos/go-systemd/releases/tag/v15).
-
-### Other
-
-- Support previous two minor versions (see our [new release policy](https://github.com/coreos/etcd/pull/8805)).
-- `v3.3.x` is the last release cycle that supports `ACI`:
-  - AppC was [officially suspended](https://github.com/appc/spec#-disclaimer-), as of late 2016.
-  - [`acbuild`](https://github.com/containers/build#this-project-is-currently-unmaintained) is not maintained anymore.
-  - `*.aci` files won't be available from etcd `v3.4` release.
-- Add container registry [`gcr.io/etcd-development/etcd`](https://gcr.io/etcd-development/etcd).
-  - [quay.io/coreos/etcd](https://quay.io/coreos/etcd) is still supported as secondary.
-
-
-## [v3.2.12](https://github.com/coreos/etcd/releases/tag/v3.2.12) (2017-12-20)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.2.11...v3.2.12) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes.
-
-### Fixed
-
-- Fix [error message of `Revision` compactor](https://github.com/coreos/etcd/pull/8999) in server-side.
-
-### Added(`etcd/clientv3`,`etcdctl/v3`)
-
-- Add [`MaxCallSendMsgSize` and `MaxCallRecvMsgSize`](https://github.com/coreos/etcd/pull/9047) fields to [`clientv3.Config`](https://godoc.org/github.com/coreos/etcd/clientv3#Config).
-  - Fix [exceeded response size limit error in client-side](https://github.com/coreos/etcd/issues/9043).
-  - Address [kubernetes#51099](https://github.com/kubernetes/kubernetes/issues/51099).
-  - `MaxCallSendMsgSize` default value is 2 MiB, if not configured.
-  - `MaxCallRecvMsgSize` default value is `math.MaxInt32`, if not configured.
-
-### Other
-
-- Pin [grpc v1.7.5](https://github.com/grpc/grpc-go/releases/tag/v1.7.5), [grpc-gateway v1.3.0](https://github.com/grpc-ecosystem/grpc-gateway/releases/tag/v1.3.0).
-  - No code change, just to be explicit about recommended versions.
-
-
-## [v3.2.11](https://github.com/coreos/etcd/releases/tag/v3.2.11) (2017-12-05)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.2.10...v3.2.11) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes.
-
-### Fixed
-
-- Fix racey grpc-go's server handler transport `WriteStatus` call to prevent [TLS-enabled etcd server crash](https://github.com/coreos/etcd/issues/8904):
-  - Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) `v1.7.3` to `v1.7.4`.
-  - Add [gRPC RPC failure warnings](https://github.com/coreos/etcd/pull/8939) to help debug such issues in the future.
-- Remove `--listen-metrics-urls` flag in monitoring document (non-released in `v3.2.x`, planned for `v3.3.x`).
-
-### Added
-
-- Provide [more cert details](https://github.com/coreos/etcd/pull/8952/files) on TLS handshake failures.
-
-
-## [v3.1.11](https://github.com/coreos/etcd/releases/tag/v3.1.11) (2017-11-28)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.1.10...v3.1.11) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes.
-
-### Fixed
-
-- [#8411](https://github.com/coreos/etcd/issues/8411),[#8806](https://github.com/coreos/etcd/pull/8806) mvcc: fix watch restore from snapshot
-- [#8009](https://github.com/coreos/etcd/issues/8009),[#8902](https://github.com/coreos/etcd/pull/8902) backport coreos/bbolt v1.3.1-coreos.5
-
-
-## [v3.2.10](https://github.com/coreos/etcd/releases/tag/v3.2.10) (2017-11-16)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.2.9...v3.2.10) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes.
-
-### Fixed
-
-- Replace backend key-value database `boltdb/bolt` with [`coreos/bbolt`](https://github.com/coreos/bbolt/releases) to address [backend database size issue](https://github.com/coreos/etcd/issues/8009).
-- Fix `clientv3` balancer to handle [network partitions](https://github.com/coreos/etcd/issues/8711):
-  - Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) `v1.2.1` to `v1.7.3`.
-  - Upgrade [`github.com/grpc-ecosystem/grpc-gateway`](https://github.com/grpc-ecosystem/grpc-gateway/releases) `v1.2` to `v1.3`.
-- Revert [discovery SRV auth `ServerName` with `*.{ROOT_DOMAIN}`](https://github.com/coreos/etcd/pull/8651) to support non-wildcard subject alternative names in the certs (see [issue #8445](https://github.com/coreos/etcd/issues/8445) for more contexts).
-  - For instance, `etcd --discovery-srv=etcd.local` will only authenticate peers/clients when the provided certs have root domain `etcd.local` (**not `*.etcd.local`**) as an entry in Subject Alternative Name (SAN) field.
-
-
-## [v3.2.9](https://github.com/coreos/etcd/releases/tag/v3.2.9) (2017-10-06)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.2.8...v3.2.9) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes.
-
-### Fixed(Security)
-
-- Compile with [Go 1.8.4](https://groups.google.com/d/msg/golang-nuts/sHfMg4gZNps/a-HDgDDDAAAJ).
-- Update `golang.org/x/crypto/bcrypt` (see [golang/crypto@6c586e1](https://github.com/golang/crypto/commit/6c586e17d90a7d08bbbc4069984180dce3b04117)).
-- Fix discovery SRV bootstrapping to [authenticate `ServerName` with `*.{ROOT_DOMAIN}`](https://github.com/coreos/etcd/pull/8651), in order to support sub-domain wildcard matching (see [issue #8445](https://github.com/coreos/etcd/issues/8445) for more contexts).
-  - For instance, `etcd --discovery-srv=etcd.local` will only authenticate peers/clients when the provided certs have root domain `*.etcd.local` as an entry in Subject Alternative Name (SAN) field.
-
-
-## [v3.2.8](https://github.com/coreos/etcd/releases/tag/v3.2.8) (2017-09-29)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.2.7...v3.2.8) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes.
-
-### Fixed
-
-- Fix v2 client failover to next endpoint on mutable operation.
-- Fix grpc-proxy to respect `KeysOnly` flag.
-
-
-## [v3.2.7](https://github.com/coreos/etcd/releases/tag/v3.2.7) (2017-09-01)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.2.6...v3.2.7) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes.
-
-### Fixed
-
-- Fix server-side auth so concurrent auth operations do not return old revision error.
-- Fix concurrency/stm Put with serializable snapshot
-  - Use store revision from first fetch to resolve write conflicts instead of modified revision.
-
-
-## [v3.2.6](https://github.com/coreos/etcd/releases/tag/v3.2.6) (2017-08-21)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.2.5...v3.2.6).
-
-### Fixed
-
-- Fix watch restore from snapshot.
-- Fix `etcd_debugging_mvcc_keys_total` inconsistency.
-- Fix multiple URLs for `--listen-peer-urls` flag.
-- Add `--enable-pprof` flag to etcd configuration file format.
-
-
-## [v3.2.5](https://github.com/coreos/etcd/releases/tag/v3.2.5) (2017-08-04)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.2.4...v3.2.5) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes.
-
-### Changed
-
-- Use reverse lookup to match wildcard DNS SAN.
-- Return non-zero exit code on unhealthy `endpoint health`.
-
-### Fixed
-
-- Fix unreachable /metrics endpoint when `--enable-v2=false`.
-- Fix grpc-proxy to respect `PrevKv` flag.
-
-### Added
-
-- Add container registry `gcr.io/etcd-development/etcd`.
-
-
-## [v3.2.4](https://github.com/coreos/etcd/releases/tag/v3.2.4) (2017-07-19)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.2.3...v3.2.4) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes.
-
-### Fixed
-
-- Do not block on active client stream when stopping server
-- Fix gRPC proxy Snapshot RPC error handling
-
-
-## [v3.2.3](https://github.com/coreos/etcd/releases/tag/v3.2.3) (2017-07-14)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.2.2...v3.2.3) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes.
-
-### Fixed
-
-- Let clients establish unlimited streams
-
-### Added
-
-- Tag docker images with minor versions
-  - e.g. `docker pull quay.io/coreos/etcd:v3.2` to fetch latest v3.2 versions
-
-
-## [v3.1.10](https://github.com/coreos/etcd/releases/tag/v3.1.10) (2017-07-14)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.1.9...v3.1.10) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes.
-
-### Changed
-
-- Compile with Go 1.8.3 to fix panic on `net/http.CloseNotify`
-
-### Added
-
-- Tag docker images with minor versions.
-  - e.g. `docker pull quay.io/coreos/etcd:v3.1` to fetch latest v3.1 versions.
-
-
-## [v3.2.2](https://github.com/coreos/etcd/releases/tag/v3.2.2) (2017-07-07)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.2.1...v3.2.2) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes.
-
-### Improved
-
-- Rate-limit lease revoke on expiration.
-- Extend leases on promote to avoid queueing effect on lease expiration.
-
-### Fixed
-
-- Use user-provided listen address to connect to gRPC gateway:
-  - `net.Listener` rewrites IPv4 0.0.0.0 to IPv6 [::], breaking IPv6 disabled hosts.
-  - Only v3.2.0, v3.2.1 are affected.
-- Accept connection with matched IP SAN but no DNS match.
-  - Don't check DNS entries in certs if there's a matching IP.
-- Fix 'tools/benchmark' watch command.
-
-
-## [v3.2.1](https://github.com/coreos/etcd/releases/tag/v3.2.1) (2017-06-23)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.2.0...v3.2.1) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes.
-
-### Fixed
-
-- Fix backend database in-memory index corruption issue on restore (only 3.2.0 is affected).
-- Fix gRPC gateway Txn marshaling issue.
-- Fix backend database size debugging metrics.
-
-
-## [v3.2.0](https://github.com/coreos/etcd/releases/tag/v3.2.0) (2017-06-09)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.1.0...v3.2.0) and [v3.2 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_2.md) for any breaking changes.
-
-### Improved
-
-- Improve backend read concurrency.
-
-### Added
-
-- Embedded etcd
-  - `Etcd.Peers` field is now `[]*peerListener`.
-- RPCs
-  - Add Election, Lock service.
-- Native client etcdserver/api/v3client
-  - client "embedded" in the server.
-- gRPC proxy
-  - Proxy endpoint discovery.
-  - Namespaces.
-  - Coalesce lease requests.
-- v3 client
-  - STM prefetching.
-  - Add namespace feature.
-  - Add `ErrOldCluster` with server version checking.
-  - Translate `WithPrefix()` into `WithFromKey()` for empty key.
-- v3 etcdctl
-  - Add `check perf` command.
-  - Add `--from-key` flag to role grant-permission command.
-  - `lock` command takes an optional command to execute.
-- etcd flags
-  - Add `--enable-v2` flag to configure v2 backend (enabled by default).
-  - Add `--auth-token` flag.
-- `etcd gateway`
-  - Support DNS SRV priority.
-- Auth
-  - Support Watch API.
-  - JWT tokens.
-- Logging, monitoring
-  - Server warns large snapshot operations.
-  - Add `etcd_debugging_server_lease_expired_total` metrics.
-- Security
-  - Deny incoming peer certs with wrong IP SAN.
-  - Resolve TLS `DNSNames` when SAN checking.
-  - Reload TLS certificates on every client connection.
-- Release
-  - Annotate acbuild with supports-systemd-notify.
-  - Add `nsswitch.conf` to Docker container image.
-  - Add ppc64le, arm64(experimental) builds.
-  - Compile with `Go 1.8.3`.
-
-### Changed
-
-- v3 client
-  - `LeaseTimeToLive` returns TTL=-1 resp on lease not found.
-  - `clientv3.NewFromConfigFile` is moved to `clientv3/yaml.NewConfig`.
-  - concurrency package's elections updated to match RPC interfaces.
-  - let client dial endpoints not in the balancer.
-- Dependencies
-  - Update [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) to `v1.2.1`.
-  - Update [`github.com/grpc-ecosystem/grpc-gateway`](https://github.com/grpc-ecosystem/grpc-gateway/releases) to `v1.2.0`.
-
-### Fixed
-
-- Allow v2 snapshot over 512MB.
-
-
-## [v3.1.9](https://github.com/coreos/etcd/releases/tag/v3.1.9) (2017-06-09)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.1.8...v3.1.9) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes.
-
-### Fixed
-
-- Allow v2 snapshot over 512MB.
-
-
-## [v3.1.8](https://github.com/coreos/etcd/releases/tag/v3.1.8) (2017-05-19)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.1.7...v3.1.8) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes.
-
-
-## [v3.1.7](https://github.com/coreos/etcd/releases/tag/v3.1.7) (2017-04-28)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.1.6...v3.1.7) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes.
-
-
-## [v3.1.6](https://github.com/coreos/etcd/releases/tag/v3.1.6) (2017-04-19)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.1.5...v3.1.6) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes.
-
-### Changed
-
-- Remove auth check in Status API.
-
-### Fixed
-
-- Fill in Auth API response header.
-
-
-## [v3.1.5](https://github.com/coreos/etcd/releases/tag/v3.1.5) (2017-03-27)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.1.4...v3.1.5) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes.
-
-### Added
-
-- Add `/etc/nsswitch.conf` file to alpine-based Docker image.
-
-### Fixed
-
-- Fix raft memory leak issue.
-- Fix Windows file path issues.
-
-
-## [v3.1.4](https://github.com/coreos/etcd/releases/tag/v3.1.4) (2017-03-22)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.1.3...v3.1.4) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes.
-
-
-## [v3.1.3](https://github.com/coreos/etcd/releases/tag/v3.1.3) (2017-03-10)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.1.2...v3.1.3) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes.
-
-### Changed
-
-- Use machine default host when advertise URLs are default values(`localhost:2379,2380`) AND if listen URL is `0.0.0.0`.
-
-### Fixed
-
-- Fix `etcd gateway` schema handling in DNS discovery.
-- Fix sd_notify behaviors in `gateway`, `grpc-proxy`.
-
-
-## [v3.1.2](https://github.com/coreos/etcd/releases/tag/v3.1.2) (2017-02-24)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.1.1...v3.1.2) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes.
-
-### Changed
-
-- Use IPv4 default host, by default (when IPv4 and IPv6 are available).
-
-### Fixed
-
-- Fix `etcd gateway` with multiple endpoints.
-
-
-## [v3.1.1](https://github.com/coreos/etcd/releases/tag/v3.1.1) (2017-02-17)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.1.0...v3.1.1) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes.
-
-### Changed
-
-- Compile with `Go 1.7.5`.
-
-
-## [v2.3.8](https://github.com/coreos/etcd/releases/tag/v2.3.8) (2017-02-17)
-
-See [code changes](https://github.com/coreos/etcd/compare/v2.3.7...v2.3.8).
-
-### Changed
-
-- Compile with `Go 1.7.5`.
-
-
-## [v3.1.0](https://github.com/coreos/etcd/releases/tag/v3.1.0) (2017-01-20)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.0.0...v3.1.0) and [v3.1 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_1.md) for any breaking changes.
-
-### Improved
-
-- Faster linearizable reads (implements Raft read-index).
-- v3 authentication API is now stable.
-
-### Added
-
-- Automatic leadership transfer when leader steps down.
-- etcd flags
-  - `--strict-reconfig-check` flag is set by default.
-  - Add `--log-output` flag.
-  - Add `--metrics` flag.
-- v3 client
-  - Add `SetEndpoints` method; update endpoints at runtime.
-  - Add `Sync` method; auto-update endpoints at runtime.
-  - Add `Lease TimeToLive` API; fetch lease information.
-  - replace Config.Logger field with global logger.
-  - Get API responses are sorted in ascending order by default.
-- v3 etcdctl
-  - Add `lease timetolive` command.
-  - Add `--print-value-only` flag to get command.
-  - Add `--dest-prefix` flag to make-mirror command.
-  - `get` command responses are sorted in ascending order by default.
-- `recipes` now conform to sessions defined in `clientv3/concurrency`.
-- ACI has symlinks to `/usr/local/bin/etcd*`.
-- Experimental gRPC proxy feature.
-
-### Changed
-
-- Deprecated following gRPC metrics in favor of [go-grpc-prometheus](https://github.com/grpc-ecosystem/go-grpc-prometheus):
-  - `etcd_grpc_requests_total`
-  - `etcd_grpc_requests_failed_total`
-  - `etcd_grpc_active_streams`
-  - `etcd_grpc_unary_requests_duration_seconds`
-- etcd uses default route IP if advertise URL is not given.
-- Cluster rejects removing members if quorum will be lost.
-- SRV records (e.g., infra1.example.com) must match the discovery domain (i.e., example.com) if no custom certificate authority is given.
-  - `TLSConfig.ServerName` is ignored with user-provided certificates for backwards compatibility; to be deprecated.
-  - For example, `etcd --discovery-srv=example.com` will only authenticate peers/clients when the provided certs have root domain `example.com` as an entry in Subject Alternative Name (SAN) field.
-- Discovery now has upper limit for waiting on retries.
-- Warn on binding listeners through domain names; to be deprecated.
-
-
-## [v3.0.16](https://github.com/coreos/etcd/releases/tag/v3.0.16) (2016-11-13)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.0.15...v3.0.16) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes.
-
-
-## [v3.0.15](https://github.com/coreos/etcd/releases/tag/v3.0.15) (2016-11-11)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.0.14...v3.0.15) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes.
-
-### Fixed
-
-- Fix cancel watch request with wrong range end.
-
-
-## [v3.0.14](https://github.com/coreos/etcd/releases/tag/v3.0.14) (2016-11-04)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.0.13...v3.0.14) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes.
-
-### Added
-
-- v3 `etcdctl migrate` command now supports `--no-ttl` flag to discard keys on transform.
-
-
-## [v3.0.13](https://github.com/coreos/etcd/releases/tag/v3.0.13) (2016-10-24)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.0.12...v3.0.13) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes.
-
-
-## [v3.0.12](https://github.com/coreos/etcd/releases/tag/v3.0.12) (2016-10-07)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.0.11...v3.0.12) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes.
-
-
-## [v3.0.11](https://github.com/coreos/etcd/releases/tag/v3.0.11) (2016-10-07)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.0.10...v3.0.11) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes.
-
-### Added
-
-- Server returns previous key-value (optional)
-  - `clientv3.WithPrevKV` option
-  - v3 etcdctl `put,watch,del --prev-kv` flag
-
-
-## [v3.0.10](https://github.com/coreos/etcd/releases/tag/v3.0.10) (2016-09-23)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.0.9...v3.0.10) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes.
-
-
-## [v3.0.9](https://github.com/coreos/etcd/releases/tag/v3.0.9) (2016-09-15)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.0.8...v3.0.9) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes.
-
-### Added
-
-- Warn on domain names on listen URLs (v3.2 will reject domain names).
-
-
-## [v3.0.8](https://github.com/coreos/etcd/releases/tag/v3.0.8) (2016-09-09)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.0.7...v3.0.8) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes.
-
-### Changed
-
-- Allow only IP addresses in listen URLs (domain names are rejected).
-
-
-## [v3.0.7](https://github.com/coreos/etcd/releases/tag/v3.0.7) (2016-08-31)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.0.6...v3.0.7) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes.
-
-### Changed
-
-- SRV records only allow A records (RFC 2052).
-
-
-## [v3.0.6](https://github.com/coreos/etcd/releases/tag/v3.0.6) (2016-08-19)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.0.5...v3.0.6) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes.
-
-
-## [v3.0.5](https://github.com/coreos/etcd/releases/tag/v3.0.5) (2016-08-19)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.0.4...v3.0.5) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes.
-
-### Changed
-
-- SRV records (e.g., infra1.example.com) must match the discovery domain (i.e., example.com) if no custom certificate authority is given.
-
-
-## [v3.0.4](https://github.com/coreos/etcd/releases/tag/v3.0.4) (2016-07-27)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.0.3...v3.0.4) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes.
-
-### Changed
-
-- v2 auth can now use common name from TLS certificate when `--client-cert-auth` is enabled.
-
-### Added
-
-- v2 `etcdctl ls` command now supports `--output=json`.
-- Add /var/lib/etcd directory to etcd official Docker image.
-
-
-## [v3.0.3](https://github.com/coreos/etcd/releases/tag/v3.0.3) (2016-07-15)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.0.2...v3.0.3) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes.
-
-### Changed
-
-- Revert Dockerfile to use `CMD`, instead of `ENTRYPOINT`, to support `etcdctl` run.
-  - Docker commands for v3.0.2 won't work without specifying executable binary paths.
-- v3 etcdctl default endpoints are now `127.0.0.1:2379`.
-
-
-## [v3.0.2](https://github.com/coreos/etcd/releases/tag/v3.0.2) (2016-07-08)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.0.1...v3.0.2) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes.
-
-### Changed
-
-- Dockerfile uses `ENTRYPOINT`, instead of `CMD`, to run etcd without binary path specified.
-
-
-## [v3.0.1](https://github.com/coreos/etcd/releases/tag/v3.0.1) (2016-07-01)
-
-See [code changes](https://github.com/coreos/etcd/compare/v3.0.0...v3.0.1) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes.
-
-
-## [v3.0.0](https://github.com/coreos/etcd/releases/tag/v3.0.0) (2016-06-30)
-
-See [code changes](https://github.com/coreos/etcd/compare/v2.3.0...v3.0.0) and [v3.0 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_0.md) for any breaking changes.

Dockerfile

@@ -1,6 +0,0 @@
-FROM golang
-ADD . /go/src/github.com/coreos/etcd
-ADD cmd/vendor /go/src/github.com/coreos/etcd/vendor
-RUN go install github.com/coreos/etcd
-EXPOSE 2379 2380
-ENTRYPOINT ["etcd"]

Dockerfile-release.ppc64le

@@ -1,4 +1,4 @@
-FROM ppc64le/ubuntu:16.04
+FROM ppc64le/alpine:latest
 
 ADD etcd /usr/local/bin/
 ADD etcdctl /usr/local/bin/

Documentation/benchmarks/etcd-2-2-0-benchmarks.md

@@ -26,7 +26,7 @@ Go OS/Arch: linux/amd64
 
 Bootstrap another machine, outside of the etcd cluster, and run the [`hey` HTTP benchmark tool](https://github.com/rakyll/hey) with a connection reuse patch to send requests to each etcd cluster member. See the [benchmark instructions](../../hack/benchmark/) for the patch and the steps to reproduce our procedures.
 
-The performance is calulated through results of 100 benchmark rounds.
+The performance is calculated through results of 100 benchmark rounds.
 
 ## Performance
 

Documentation/dev-guide/api_concurrency_reference_v3.md

@@ -20,7 +20,7 @@ The lock service exposes client-side locking facilities as a gRPC interface.
 | Field | Description | Type |
 | ----- | ----------- | ---- |
 | name | name is the identifier for the distributed shared lock to be acquired. | bytes |
-| lease | lease is the ID of the lease that will be attached to ownership of the lock. If the lease expires or is revoked and currently holds the lock, the lock is automatically released. Calls to Lock with the same lease will be treated as a single acquistion; locking twice with the same lease is a no-op. | int64 |
+| lease | lease is the ID of the lease that will be attached to ownership of the lock. If the lease expires or is revoked and currently holds the lock, the lock is automatically released. Calls to Lock with the same lease will be treated as a single acquisition; locking twice with the same lease is a no-op. | int64 |
 
 
 

Documentation/dev-guide/api_grpc_gateway.md

@@ -1,16 +1,26 @@
 
-## Why grpc-gateway
+## Why gRPC gateway
 
-etcd v3 uses [gRPC][grpc] for its messaging protocol. The etcd project includes a gRPC-based [Go client][go-client] and a command line utility, [etcdctl][etcdctl], for communicating with an etcd cluster through gRPC. For languages with no gRPC support, etcd provides a JSON [grpc-gateway][grpc-gateway]. This gateway serves a RESTful proxy that translates HTTP/JSON requests into gRPC messages.
+etcd v3 uses [gRPC][grpc] for its messaging protocol. The etcd project includes a gRPC-based [Go client][go-client] and a command line utility, [etcdctl][etcdctl], for communicating with an etcd cluster through gRPC. For languages with no gRPC support, etcd provides a JSON [gRPC gateway][grpc-gateway]. This gateway serves a RESTful proxy that translates HTTP/JSON requests into gRPC messages.
 
-
-## Using grpc-gateway
+## Using gRPC gateway
 
 The gateway accepts a [JSON mapping][json-mapping] for etcd's [protocol buffer][api-ref] message definitions. Note that `key` and `value` fields are defined as byte arrays and therefore must be base64 encoded in JSON. The following examples use `curl`, but any HTTP/JSON client should work all the same.
 
+### Notes
+
+gRPC gateway endpoint has changed since etcd v3.3:
+
+- etcd v3.2 or before uses only `[CLIENT-URL]/v3alpha/*`.
+- etcd v3.3 uses `[CLIENT-URL]/v3beta/*` while keeping `[CLIENT-URL]/v3alpha/*`.
+- etcd v3.4 uses `[CLIENT-URL]/v3/*` while keeping `[CLIENT-URL]/v3beta/*`.
+  - **`[CLIENT-URL]/v3alpha/*` is deprecated**.
+- etcd v3.5 or later uses only `[CLIENT-URL]/v3/*`.
+  - **`[CLIENT-URL]/v3beta/*` is deprecated**.
+
 ### Put and get keys
 
-Use the `/v3beta/kv/range` and `/v3beta/kv/put` services to read and write keys:
+Use the `/v3/kv/range` and `/v3/kv/put` services to read and write keys:
 
 ```bash
 <<COMMENT
@@ -19,85 +29,85 @@ foo is 'Zm9v' in Base64
 bar is 'YmFy'
 COMMENT
 
-curl -L http://localhost:2379/v3beta/kv/put \
-	-X POST -d '{"key": "Zm9v", "value": "YmFy"}'
+curl -L http://localhost:2379/v3/kv/put \
+  -X POST -d '{"key": "Zm9v", "value": "YmFy"}'
 # {"header":{"cluster_id":"12585971608760269493","member_id":"13847567121247652255","revision":"2","raft_term":"3"}}
 
-curl -L http://localhost:2379/v3beta/kv/range \
-	-X POST -d '{"key": "Zm9v"}'
+curl -L http://localhost:2379/v3/kv/range \
+  -X POST -d '{"key": "Zm9v"}'
 # {"header":{"cluster_id":"12585971608760269493","member_id":"13847567121247652255","revision":"2","raft_term":"3"},"kvs":[{"key":"Zm9v","create_revision":"2","mod_revision":"2","version":"1","value":"YmFy"}],"count":"1"}
 
 # get all keys prefixed with "foo"
-curl -L http://localhost:2379/v3beta/kv/range \
-	-X POST -d '{"key": "Zm9v", "range_end": "Zm9w"}'
+curl -L http://localhost:2379/v3/kv/range \
+  -X POST -d '{"key": "Zm9v", "range_end": "Zm9w"}'
 # {"header":{"cluster_id":"12585971608760269493","member_id":"13847567121247652255","revision":"2","raft_term":"3"},"kvs":[{"key":"Zm9v","create_revision":"2","mod_revision":"2","version":"1","value":"YmFy"}],"count":"1"}
 ```
 
 ### Watch keys
 
-Use the `/v3beta/watch` service to watch keys:
+Use the `/v3/watch` service to watch keys:
 
 ```bash
-curl http://localhost:2379/v3beta/watch \
-        -X POST -d '{"create_request": {"key":"Zm9v"} }' &
+curl http://localhost:2379/v3/watch \
+  -X POST -d '{"create_request": {"key":"Zm9v"} }' &
 # {"result":{"header":{"cluster_id":"12585971608760269493","member_id":"13847567121247652255","revision":"1","raft_term":"2"},"created":true}}
 
-curl -L http://localhost:2379/v3beta/kv/put \
-	-X POST -d '{"key": "Zm9v", "value": "YmFy"}' >/dev/null 2>&1
+curl -L http://localhost:2379/v3/kv/put \
+  -X POST -d '{"key": "Zm9v", "value": "YmFy"}' >/dev/null 2>&1
 # {"result":{"header":{"cluster_id":"12585971608760269493","member_id":"13847567121247652255","revision":"2","raft_term":"2"},"events":[{"kv":{"key":"Zm9v","create_revision":"2","mod_revision":"2","version":"1","value":"YmFy"}}]}}
 ```
 
 ### Transactions
 
-Issue a transaction with `/v3beta/kv/txn`:
+Issue a transaction with `/v3/kv/txn`:
 
 ```bash
-curl -L http://localhost:2379/v3beta/kv/txn \
-	-X POST \
-	-d '{"compare":[{"target":"CREATE","key":"Zm9v","createRevision":"2"}],"success":[{"requestPut":{"key":"Zm9v","value":"YmFy"}}]}'
+curl -L http://localhost:2379/v3/kv/txn \
+  -X POST \
+  -d '{"compare":[{"target":"CREATE","key":"Zm9v","createRevision":"2"}],"success":[{"requestPut":{"key":"Zm9v","value":"YmFy"}}]}'
 # {"header":{"cluster_id":"12585971608760269493","member_id":"13847567121247652255","revision":"3","raft_term":"2"},"succeeded":true,"responses":[{"response_put":{"header":{"revision":"3"}}}]}
 ```
 
 ### Authentication
 
-Set up authentication with the `/v3beta/auth` service:
+Set up authentication with the `/v3/auth` service:
 
 ```bash
 # create root user
-curl -L http://localhost:2379/v3beta/auth/user/add \
-	-X POST -d '{"name": "root", "password": "pass"}'
+curl -L http://localhost:2379/v3/auth/user/add \
+  -X POST -d '{"name": "root", "password": "pass"}'
 # {"header":{"cluster_id":"14841639068965178418","member_id":"10276657743932975437","revision":"1","raft_term":"2"}}
 
 # create root role
-curl -L http://localhost:2379/v3beta/auth/role/add \
-	-X POST -d '{"name": "root"}'
+curl -L http://localhost:2379/v3/auth/role/add \
+  -X POST -d '{"name": "root"}'
 # {"header":{"cluster_id":"14841639068965178418","member_id":"10276657743932975437","revision":"1","raft_term":"2"}}
 
 # grant root role
-curl -L http://localhost:2379/v3beta/auth/user/grant \
-	-X POST -d '{"user": "root", "role": "root"}'
+curl -L http://localhost:2379/v3/auth/user/grant \
+  -X POST -d '{"user": "root", "role": "root"}'
 # {"header":{"cluster_id":"14841639068965178418","member_id":"10276657743932975437","revision":"1","raft_term":"2"}}
 
 # enable auth
-curl -L http://localhost:2379/v3beta/auth/enable -X POST -d '{}'
+curl -L http://localhost:2379/v3/auth/enable -X POST -d '{}'
 # {"header":{"cluster_id":"14841639068965178418","member_id":"10276657743932975437","revision":"1","raft_term":"2"}}
 ```
 
-Authenticate with etcd for an authentication token using `/v3beta/auth/authenticate`:
+Authenticate with etcd for an authentication token using `/v3/auth/authenticate`:
 
 ```bash
 # get the auth token for the root user
-curl -L http://localhost:2379/v3beta/auth/authenticate \
-	-X POST -d '{"name": "root", "password": "pass"}'
+curl -L http://localhost:2379/v3/auth/authenticate \
+  -X POST -d '{"name": "root", "password": "pass"}'
 # {"header":{"cluster_id":"14841639068965178418","member_id":"10276657743932975437","revision":"1","raft_term":"2"},"token":"sssvIpwfnLAcWAQH.9"}
 ```
 
 Set the `Authorization` header to the authentication token to fetch a key using authentication credentials:
 
 ```bash
-curl -L http://localhost:2379/v3beta/kv/put \
-	-H 'Authorization : sssvIpwfnLAcWAQH.9' \
-	-X POST -d '{"key": "Zm9v", "value": "YmFy"}'
+curl -L http://localhost:2379/v3/kv/put \
+  -H 'Authorization : sssvIpwfnLAcWAQH.9' \
+  -X POST -d '{"key": "Zm9v", "value": "YmFy"}'
 # {"header":{"cluster_id":"14841639068965178418","member_id":"10276657743932975437","revision":"2","raft_term":"2"}}
 ```
 
@@ -113,4 +123,3 @@ Generated [Swagger][swagger] API definitions can be found at [rpc.swagger.json][
 [json-mapping]: https://developers.google.com/protocol-buffers/docs/proto3#json
 [swagger]: http://swagger.io/
 [swagger-doc]: apispec/swagger/rpc.swagger.json
-

Documentation/dev-guide/api_reference_v3.md

@@ -69,8 +69,8 @@ This is a generated documentation. Please read the proto files for more.
 | Alarm | AlarmRequest | AlarmResponse | Alarm activates, deactivates, and queries alarms regarding cluster health. |
 | Status | StatusRequest | StatusResponse | Status gets the status of the member. |
 | Defragment | DefragmentRequest | DefragmentResponse | Defragment defragments a member's backend database to recover storage space. |
-| Hash | HashRequest | HashResponse | Hash computes the hash of the KV's backend. This is designed for testing; do not use this in production when there are ongoing transactions. |
-| HashKV | HashKVRequest | HashKVResponse | HashKV computes the hash of all MVCC keys up to a given revision. |
+| Hash | HashRequest | HashResponse | Hash computes the hash of whole backend keyspace, including key, lease, and other buckets in storage. This is designed for testing ONLY! Do not rely on this in production with ongoing transactions, since Hash operation does not hold MVCC locks. Use "HashKV" API instead for "key" bucket consistency checks. |
+| HashKV | HashKVRequest | HashKVResponse | HashKV computes the hash of all MVCC keys up to a given revision. It only iterates "key" bucket in backend storage. |
 | Snapshot | SnapshotRequest | SnapshotResponse | Snapshot sends a snapshot of the entire backend from a member over a stream to a client. |
 | MoveLeader | MoveLeaderRequest | MoveLeaderResponse | MoveLeader requests current leader node to transfer its leadership to transferee. |
 
@@ -226,8 +226,8 @@ Empty field.
 | Field | Description | Type |
 | ----- | ----------- | ---- |
 | role |  | string |
-| key |  | string |
-| range_end |  | string |
+| key |  | bytes |
+| range_end |  | bytes |
 
 
 
@@ -476,6 +476,31 @@ Empty field.
 
 
 
+##### message `LeaseCheckpoint` (etcdserver/etcdserverpb/rpc.proto)
+
+| Field | Description | Type |
+| ----- | ----------- | ---- |
+| ID | ID is the lease ID to checkpoint. | int64 |
+| remaining_TTL | Remaining_TTL is the remaining time until expiry of the lease. | int64 |
+
+
+
+##### message `LeaseCheckpointRequest` (etcdserver/etcdserverpb/rpc.proto)
+
+| Field | Description | Type |
+| ----- | ----------- | ---- |
+| checkpoints |  | (slice of) LeaseCheckpoint |
+
+
+
+##### message `LeaseCheckpointResponse` (etcdserver/etcdserverpb/rpc.proto)
+
+| Field | Description | Type |
+| ----- | ----------- | ---- |
+| header |  | ResponseHeader |
+
+
+
 ##### message `LeaseGrantRequest` (etcdserver/etcdserverpb/rpc.proto)
 
 | Field | Description | Type |
@@ -706,7 +731,7 @@ Empty field.
 | count_only | count_only when set returns only the count of the keys in the range. | bool |
 | min_mod_revision | min_mod_revision is the lower bound for returned key mod revisions; all keys with lesser mod revisions will be filtered away. | int64 |
 | max_mod_revision | max_mod_revision is the upper bound for returned key mod revisions; all keys with greater mod revisions will be filtered away. | int64 |
-| min_create_revision | min_create_revision is the lower bound for returned key create revisions; all keys with lesser create trevisions will be filtered away. | int64 |
+| min_create_revision | min_create_revision is the lower bound for returned key create revisions; all keys with lesser create revisions will be filtered away. | int64 |
 | max_create_revision | max_create_revision is the upper bound for returned key create revisions; all keys with greater create revisions will be filtered away. | int64 |
 
 
@@ -740,7 +765,7 @@ Empty field.
 | ----- | ----------- | ---- |
 | cluster_id | cluster_id is the ID of the cluster which sent the response. | uint64 |
 | member_id | member_id is the ID of the member which sent the response. | uint64 |
-| revision | revision is the key-value store revision when the request was applied. | int64 |
+| revision | revision is the key-value store revision when the request was applied. For watch progress responses, the header.revision indicates progress. All future events recieved in this stream are guaranteed to have a higher revision number than the header.revision number. | int64 |
 | raft_term | raft_term is the raft term when the request was applied. | uint64 |
 
 
@@ -785,10 +810,13 @@ Empty field.
 | ----- | ----------- | ---- |
 | header |  | ResponseHeader |
 | version | version is the cluster protocol version used by the responding member. | string |
-| dbSize | dbSize is the size of the backend database, in bytes, of the responding member. | int64 |
+| dbSize | dbSize is the size of the backend database physically allocated, in bytes, of the responding member. | int64 |
 | leader | leader is the member ID which the responding member believes is the current leader. | uint64 |
-| raftIndex | raftIndex is the current raft index of the responding member. | uint64 |
+| raftIndex | raftIndex is the current raft committed index of the responding member. | uint64 |
 | raftTerm | raftTerm is the current raft term of the responding member. | uint64 |
+| raftAppliedIndex | raftAppliedIndex is the current raft applied index of the responding member. | uint64 |
+| errors | errors contains alarm/health information and status. | (slice of) string |
+| dbSizeInUse | dbSizeInUse is the size of the backend database logically in use, in bytes, of the responding member. | int64 |
 
 
 
@@ -832,6 +860,16 @@ From google paxosdb paper: Our implementation hinges around a powerful primitive
 | progress_notify | progress_notify is set so that the etcd server will periodically send a WatchResponse with no events to the new watcher if there are no recent events. It is useful when clients wish to recover a disconnected watcher starting from a recent known revision. The etcd server may decide how often it will send notifications based on current load. | bool |
 | filters | filters filter the events at server side before it sends back to the watcher. | (slice of) FilterType |
 | prev_kv | If prev_kv is set, created watcher gets the previous KV before the event happens. If the previous KV is already compacted, nothing will be returned. | bool |
+| watch_id | If watch_id is provided and non-zero, it will be assigned to this watcher. Since creating a watcher in etcd is not a synchronous operation, this can be used ensure that ordering is correct when creating multiple watchers on the same stream. Creating a watcher with an ID already in use on the stream will cause an error to be returned. | int64 |
+| fragment | fragment enables splitting large revisions into multiple watch responses. | bool |
+
+
+
+##### message `WatchProgressRequest` (etcdserver/etcdserverpb/rpc.proto)
+
+Requests the a watch stream progress status be sent in the watch response stream as soon as possible.
+
+Empty field.
 
 
 
@@ -842,6 +880,7 @@ From google paxosdb paper: Our implementation hinges around a powerful primitive
 | request_union | request_union is a request to either create a new watcher or cancel an existing watcher. | oneof |
 | create_request |  | WatchCreateRequest |
 | cancel_request |  | WatchCancelRequest |
+| progress_request |  | WatchProgressRequest |
 
 
 
@@ -855,6 +894,7 @@ From google paxosdb paper: Our implementation hinges around a powerful primitive
 | canceled | canceled is set to true if the response is for a cancel watch request. No further events will be sent to the canceled watcher. | bool |
 | compact_revision | compact_revision is set to the minimum index if a watcher tries to watch at a compacted index.  This happens when creating a watcher at a compacted revision or the watcher cannot catch up with the progress of the key-value store.  The client should treat the watcher as canceled and should not try to create any watcher with the same start_revision again. | int64 |
 | cancel_reason | cancel_reason indicates the reason for canceling the watcher. | string |
+| fragment | framgment is true if large watch response was split over multiple responses. | bool |
 | events |  | (slice of) mvccpb.Event |
 
 
@@ -888,6 +928,7 @@ From google paxosdb paper: Our implementation hinges around a powerful primitive
 | ----- | ----------- | ---- |
 | ID |  | int64 |
 | TTL |  | int64 |
+| RemainingTTL |  | int64 |
 
 
 

Documentation/dev-guide/apispec/swagger/rpc.swagger.json

@@ -15,7 +15,7 @@
     "version": "version not set"
   },
   "paths": {
-    "/v3beta/auth/authenticate": {
+    "/v3/auth/authenticate": {
       "post": {
         "tags": [
           "Auth"
@@ -42,7 +42,7 @@
         }
       }
     },
-    "/v3beta/auth/disable": {
+    "/v3/auth/disable": {
       "post": {
         "tags": [
           "Auth"
@@ -69,7 +69,7 @@
         }
       }
     },
-    "/v3beta/auth/enable": {
+    "/v3/auth/enable": {
       "post": {
         "tags": [
           "Auth"
@@ -96,7 +96,7 @@
         }
       }
     },
-    "/v3beta/auth/role/add": {
+    "/v3/auth/role/add": {
       "post": {
         "tags": [
           "Auth"
@@ -123,7 +123,7 @@
         }
       }
     },
-    "/v3beta/auth/role/delete": {
+    "/v3/auth/role/delete": {
       "post": {
         "tags": [
           "Auth"
@@ -150,7 +150,7 @@
         }
       }
     },
-    "/v3beta/auth/role/get": {
+    "/v3/auth/role/get": {
       "post": {
         "tags": [
           "Auth"
@@ -177,7 +177,7 @@
         }
       }
     },
-    "/v3beta/auth/role/grant": {
+    "/v3/auth/role/grant": {
       "post": {
         "tags": [
           "Auth"
@@ -204,7 +204,7 @@
         }
       }
     },
-    "/v3beta/auth/role/list": {
+    "/v3/auth/role/list": {
       "post": {
         "tags": [
           "Auth"
@@ -231,7 +231,7 @@
         }
       }
     },
-    "/v3beta/auth/role/revoke": {
+    "/v3/auth/role/revoke": {
       "post": {
         "tags": [
           "Auth"
@@ -258,7 +258,7 @@
         }
       }
     },
-    "/v3beta/auth/user/add": {
+    "/v3/auth/user/add": {
       "post": {
         "tags": [
           "Auth"
@@ -285,7 +285,7 @@
         }
       }
     },
-    "/v3beta/auth/user/changepw": {
+    "/v3/auth/user/changepw": {
       "post": {
         "tags": [
           "Auth"
@@ -312,7 +312,7 @@
         }
       }
     },
-    "/v3beta/auth/user/delete": {
+    "/v3/auth/user/delete": {
       "post": {
         "tags": [
           "Auth"
@@ -339,7 +339,7 @@
         }
       }
     },
-    "/v3beta/auth/user/get": {
+    "/v3/auth/user/get": {
       "post": {
         "tags": [
           "Auth"
@@ -366,7 +366,7 @@
         }
       }
     },
-    "/v3beta/auth/user/grant": {
+    "/v3/auth/user/grant": {
       "post": {
         "tags": [
           "Auth"
@@ -393,7 +393,7 @@
         }
       }
     },
-    "/v3beta/auth/user/list": {
+    "/v3/auth/user/list": {
       "post": {
         "tags": [
           "Auth"
@@ -420,7 +420,7 @@
         }
       }
     },
-    "/v3beta/auth/user/revoke": {
+    "/v3/auth/user/revoke": {
       "post": {
         "tags": [
           "Auth"
@@ -447,7 +447,7 @@
         }
       }
     },
-    "/v3beta/cluster/member/add": {
+    "/v3/cluster/member/add": {
       "post": {
         "tags": [
           "Cluster"
@@ -474,7 +474,7 @@
         }
       }
     },
-    "/v3beta/cluster/member/list": {
+    "/v3/cluster/member/list": {
       "post": {
         "tags": [
           "Cluster"
@@ -501,7 +501,7 @@
         }
       }
     },
-    "/v3beta/cluster/member/remove": {
+    "/v3/cluster/member/remove": {
       "post": {
         "tags": [
           "Cluster"
@@ -528,7 +528,7 @@
         }
       }
     },
-    "/v3beta/cluster/member/update": {
+    "/v3/cluster/member/update": {
       "post": {
         "tags": [
           "Cluster"
@@ -555,7 +555,7 @@
         }
       }
     },
-    "/v3beta/kv/compaction": {
+    "/v3/kv/compaction": {
       "post": {
         "tags": [
           "KV"
@@ -582,7 +582,7 @@
         }
       }
     },
-    "/v3beta/kv/deleterange": {
+    "/v3/kv/deleterange": {
       "post": {
         "tags": [
           "KV"
@@ -609,13 +609,13 @@
         }
       }
     },
-    "/v3beta/kv/lease/leases": {
+    "/v3/kv/lease/leases": {
       "post": {
         "tags": [
           "Lease"
         ],
         "summary": "LeaseLeases lists all existing leases.",
-        "operationId": "LeaseLeases",
+        "operationId": "LeaseLeases2",
         "parameters": [
           {
             "name": "body",
@@ -636,13 +636,13 @@
         }
       }
     },
-    "/v3beta/kv/lease/revoke": {
+    "/v3/kv/lease/revoke": {
       "post": {
         "tags": [
           "Lease"
         ],
         "summary": "LeaseRevoke revokes a lease. All keys attached to the lease will expire and be deleted.",
-        "operationId": "LeaseRevoke",
+        "operationId": "LeaseRevoke2",
         "parameters": [
           {
             "name": "body",
@@ -663,13 +663,13 @@
         }
       }
     },
-    "/v3beta/kv/lease/timetolive": {
+    "/v3/kv/lease/timetolive": {
       "post": {
         "tags": [
           "Lease"
         ],
         "summary": "LeaseTimeToLive retrieves lease information.",
-        "operationId": "LeaseTimeToLive",
+        "operationId": "LeaseTimeToLive2",
         "parameters": [
           {
             "name": "body",
@@ -690,7 +690,7 @@
         }
       }
     },
-    "/v3beta/kv/put": {
+    "/v3/kv/put": {
       "post": {
         "tags": [
           "KV"
@@ -717,7 +717,7 @@
         }
       }
     },
-    "/v3beta/kv/range": {
+    "/v3/kv/range": {
       "post": {
         "tags": [
           "KV"
@@ -744,7 +744,7 @@
         }
       }
     },
-    "/v3beta/kv/txn": {
+    "/v3/kv/txn": {
       "post": {
         "tags": [
           "KV"
@@ -771,7 +771,7 @@
         }
       }
     },
-    "/v3beta/lease/grant": {
+    "/v3/lease/grant": {
       "post": {
         "tags": [
           "Lease"
@@ -798,7 +798,7 @@
         }
       }
     },
-    "/v3beta/lease/keepalive": {
+    "/v3/lease/keepalive": {
       "post": {
         "tags": [
           "Lease"
@@ -826,7 +826,88 @@
         }
       }
     },
-    "/v3beta/maintenance/alarm": {
+    "/v3/lease/leases": {
+      "post": {
+        "tags": [
+          "Lease"
+        ],
+        "summary": "LeaseLeases lists all existing leases.",
+        "operationId": "LeaseLeases",
+        "parameters": [
+          {
+            "name": "body",
+            "in": "body",
+            "required": true,
+            "schema": {
+              "$ref": "#/definitions/etcdserverpbLeaseLeasesRequest"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "(empty)",
+            "schema": {
+              "$ref": "#/definitions/etcdserverpbLeaseLeasesResponse"
+            }
+          }
+        }
+      }
+    },
+    "/v3/lease/revoke": {
+      "post": {
+        "tags": [
+          "Lease"
+        ],
+        "summary": "LeaseRevoke revokes a lease. All keys attached to the lease will expire and be deleted.",
+        "operationId": "LeaseRevoke",
+        "parameters": [
+          {
+            "name": "body",
+            "in": "body",
+            "required": true,
+            "schema": {
+              "$ref": "#/definitions/etcdserverpbLeaseRevokeRequest"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "(empty)",
+            "schema": {
+              "$ref": "#/definitions/etcdserverpbLeaseRevokeResponse"
+            }
+          }
+        }
+      }
+    },
+    "/v3/lease/timetolive": {
+      "post": {
+        "tags": [
+          "Lease"
+        ],
+        "summary": "LeaseTimeToLive retrieves lease information.",
+        "operationId": "LeaseTimeToLive",
+        "parameters": [
+          {
+            "name": "body",
+            "in": "body",
+            "required": true,
+            "schema": {
+              "$ref": "#/definitions/etcdserverpbLeaseTimeToLiveRequest"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "(empty)",
+            "schema": {
+              "$ref": "#/definitions/etcdserverpbLeaseTimeToLiveResponse"
+            }
+          }
+        }
+      }
+    },
+    "/v3/maintenance/alarm": {
       "post": {
         "tags": [
           "Maintenance"
@@ -853,7 +934,7 @@
         }
       }
     },
-    "/v3beta/maintenance/defragment": {
+    "/v3/maintenance/defragment": {
       "post": {
         "tags": [
           "Maintenance"
@@ -880,12 +961,12 @@
         }
       }
     },
-    "/v3beta/maintenance/hash": {
+    "/v3/maintenance/hash": {
       "post": {
         "tags": [
           "Maintenance"
         ],
-        "summary": "HashKV computes the hash of all MVCC keys up to a given revision.",
+        "summary": "HashKV computes the hash of all MVCC keys up to a given revision.\nIt only iterates \"key\" bucket in backend storage.",
         "operationId": "HashKV",
         "parameters": [
           {
@@ -907,7 +988,7 @@
         }
       }
     },
-    "/v3beta/maintenance/snapshot": {
+    "/v3/maintenance/snapshot": {
       "post": {
         "tags": [
           "Maintenance"
@@ -934,7 +1015,7 @@
         }
       }
     },
-    "/v3beta/maintenance/status": {
+    "/v3/maintenance/status": {
       "post": {
         "tags": [
           "Maintenance"
@@ -961,7 +1042,7 @@
         }
       }
     },
-    "/v3beta/maintenance/transfer-leadership": {
+    "/v3/maintenance/transfer-leadership": {
       "post": {
         "tags": [
           "Maintenance"
@@ -988,7 +1069,7 @@
         }
       }
     },
-    "/v3beta/watch": {
+    "/v3/watch": {
       "post": {
         "tags": [
           "Watch"
@@ -1286,10 +1367,12 @@
       "type": "object",
       "properties": {
         "key": {
-          "type": "string"
+          "type": "string",
+          "format": "byte"
         },
         "range_end": {
-          "type": "string"
+          "type": "string",
+          "format": "byte"
         },
         "role": {
           "type": "string"
@@ -2017,7 +2100,7 @@
           "format": "int64"
         },
         "min_create_revision": {
-          "description": "min_create_revision is the lower bound for returned key create revisions; all keys with\nlesser create trevisions will be filtered away.",
+          "description": "min_create_revision is the lower bound for returned key create revisions; all keys with\nlesser create revisions will be filtered away.",
           "type": "string",
           "format": "int64"
         },
@@ -2112,7 +2195,7 @@
           "format": "uint64"
         },
         "revision": {
-          "description": "revision is the key-value store revision when the request was applied.",
+          "description": "revision is the key-value store revision when the request was applied.\nFor watch progress responses, the header.revision indicates progress. All future events\nrecieved in this stream are guaranteed to have a higher revision number than the\nheader.revision number.",
           "type": "string",
           "format": "int64"
         }
@@ -2164,10 +2247,22 @@
       "type": "object",
       "properties": {
         "dbSize": {
-          "description": "dbSize is the size of the backend database, in bytes, of the responding member.",
+          "description": "dbSize is the size of the backend database physically allocated, in bytes, of the responding member.",
           "type": "string",
           "format": "int64"
         },
+        "dbSizeInUse": {
+          "description": "dbSizeInUse is the size of the backend database logically in use, in bytes, of the responding member.",
+          "type": "string",
+          "format": "int64"
+        },
+        "errors": {
+          "description": "errors contains alarm/health information and status.",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
         "header": {
           "$ref": "#/definitions/etcdserverpbResponseHeader"
         },
@@ -2176,8 +2271,13 @@
           "type": "string",
           "format": "uint64"
         },
+        "raftAppliedIndex": {
+          "description": "raftAppliedIndex is the current raft applied index of the responding member.",
+          "type": "string",
+          "format": "uint64"
+        },
         "raftIndex": {
-          "description": "raftIndex is the current raft index of the responding member.",
+          "description": "raftIndex is the current raft committed index of the responding member.",
           "type": "string",
           "format": "uint64"
         },
@@ -2259,6 +2359,11 @@
             "$ref": "#/definitions/WatchCreateRequestFilterType"
           }
         },
+        "fragment": {
+          "description": "fragment enables splitting large revisions into multiple watch responses.",
+          "type": "boolean",
+          "format": "boolean"
+        },
         "key": {
           "description": "key is the key to register for watching.",
           "type": "string",
@@ -2283,9 +2388,18 @@
           "description": "start_revision is an optional revision to watch from (inclusive). No start_revision is \"now\".",
           "type": "string",
           "format": "int64"
+        },
+        "watch_id": {
+          "description": "If watch_id is provided and non-zero, it will be assigned to this watcher.\nSince creating a watcher in etcd is not a synchronous operation,\nthis can be used ensure that ordering is correct when creating multiple\nwatchers on the same stream. Creating a watcher with an ID already in\nuse on the stream will cause an error to be returned.",
+          "type": "string",
+          "format": "int64"
         }
       }
     },
+    "etcdserverpbWatchProgressRequest": {
+      "description": "Requests the a watch stream progress status be sent in the watch response stream as soon as\npossible.",
+      "type": "object"
+    },
     "etcdserverpbWatchRequest": {
       "type": "object",
       "properties": {
@@ -2294,6 +2408,9 @@
         },
         "create_request": {
           "$ref": "#/definitions/etcdserverpbWatchCreateRequest"
+        },
+        "progress_request": {
+          "$ref": "#/definitions/etcdserverpbWatchProgressRequest"
         }
       }
     },
@@ -2325,6 +2442,11 @@
             "$ref": "#/definitions/mvccpbEvent"
           }
         },
+        "fragment": {
+          "description": "framgment is true if large watch response was split over multiple responses.",
+          "type": "boolean",
+          "format": "boolean"
+        },
         "header": {
           "$ref": "#/definitions/etcdserverpbResponseHeader"
         },

Documentation/dev-guide/apispec/swagger/v3election.swagger.json

@@ -15,7 +15,7 @@
     "application/json"
   ],
   "paths": {
-    "/v3beta/election/campaign": {
+    "/v3/election/campaign": {
       "post": {
         "summary": "Campaign waits to acquire leadership in an election, returning a LeaderKey\nrepresenting the leadership if successful. The LeaderKey can then be used\nto issue new values on the election, transactionally guard API requests on\nleadership still being held, and resign from the election.",
         "operationId": "Campaign",
@@ -42,7 +42,7 @@
         ]
       }
     },
-    "/v3beta/election/leader": {
+    "/v3/election/leader": {
       "post": {
         "summary": "Leader returns the current election proclamation, if any.",
         "operationId": "Leader",
@@ -69,7 +69,7 @@
         ]
       }
     },
-    "/v3beta/election/observe": {
+    "/v3/election/observe": {
       "post": {
         "summary": "Observe streams election proclamations in-order as made by the election's\nelected leaders.",
         "operationId": "Observe",
@@ -96,7 +96,7 @@
         ]
       }
     },
-    "/v3beta/election/proclaim": {
+    "/v3/election/proclaim": {
       "post": {
         "summary": "Proclaim updates the leader's posted value with a new value.",
         "operationId": "Proclaim",
@@ -123,7 +123,7 @@
         ]
       }
     },
-    "/v3beta/election/resign": {
+    "/v3/election/resign": {
       "post": {
         "summary": "Resign releases election leadership so other campaigners may acquire\nleadership on the election.",
         "operationId": "Resign",
@@ -168,7 +168,7 @@
         "revision": {
           "type": "string",
           "format": "int64",
-          "description": "revision is the key-value store revision when the request was applied."
+          "description": "revision is the key-value store revision when the request was applied.\nFor watch progress responses, the header.revision indicates progress. All future events\nrecieved in this stream are guaranteed to have a higher revision number than the\nheader.revision number."
         },
         "raft_term": {
           "type": "string",

Documentation/dev-guide/apispec/swagger/v3lock.swagger.json

@@ -15,7 +15,7 @@
     "application/json"
   ],
   "paths": {
-    "/v3beta/lock/lock": {
+    "/v3/lock/lock": {
       "post": {
         "summary": "Lock acquires a distributed shared lock on a given named lock.\nOn success, it will return a unique key that exists so long as the\nlock is held by the caller. This key can be used in conjunction with\ntransactions to safely ensure updates to etcd only occur while holding\nlock ownership. The lock is held until Unlock is called on the key or the\nlease associate with the owner expires.",
         "operationId": "Lock",
@@ -42,7 +42,7 @@
         ]
       }
     },
-    "/v3beta/lock/unlock": {
+    "/v3/lock/unlock": {
       "post": {
         "summary": "Unlock takes a key returned by Lock and releases the hold on lock. The\nnext Lock caller waiting for the lock will then be woken up and given\nownership of the lock.",
         "operationId": "Unlock",
@@ -87,7 +87,7 @@
         "revision": {
           "type": "string",
           "format": "int64",
-          "description": "revision is the key-value store revision when the request was applied."
+          "description": "revision is the key-value store revision when the request was applied.\nFor watch progress responses, the header.revision indicates progress. All future events\nrecieved in this stream are guaranteed to have a higher revision number than the\nheader.revision number."
         },
         "raft_term": {
           "type": "string",
@@ -107,7 +107,7 @@
         "lease": {
           "type": "string",
           "format": "int64",
-          "description": "lease is the ID of the lease that will be attached to ownership of the\nlock. If the lease expires or is revoked and currently holds the lock,\nthe lock is automatically released. Calls to Lock with the same lease will\nbe treated as a single acquistion; locking twice with the same lease is a\nno-op."
+          "description": "lease is the ID of the lease that will be attached to ownership of the\nlock. If the lease expires or is revoked and currently holds the lock,\nthe lock is automatically released. Calls to Lock with the same lease will\nbe treated as a single acquisition; locking twice with the same lease is a\nno-op."
         }
       }
     },

Documentation/dev-guide/grpc_naming.md

@@ -19,7 +19,7 @@ import (
 cli, cerr := clientv3.NewFromURL("http://localhost:2379")
 r := &etcdnaming.GRPCResolver{Client: cli}
 b := grpc.RoundRobin(r)
-conn, gerr := grpc.Dial("my-service", grpc.WithBalancer(b))
+conn, gerr := grpc.Dial("my-service", grpc.WithBalancer(b), grpc.WithBlock(), ...)
 ```
 
 ## Managing service endpoints

Documentation/dev-guide/interacting_v3.md

@@ -355,6 +355,26 @@ foo         # key
 bar_latest  # value of foo key after modification
 ```
 
+## Watch progress
+
+Applications may want to check the progress of a watch to determine how up-to-date the watch stream is. For example, if a watch is used to update a cache, it can be useful to know if the cache is stale compared to the revision from a quorum read. 
+
+Progress requests can be issued using the "progress" command in interactive watch session to ask the etcd server to send a progress notify update in the watch stream:
+
+```bash
+$ etcdctl watch -i
+$ watch a
+$ progress
+progress notify: 1
+# in another terminal: etcdctl put x 0
+# in another terminal: etcdctl put y 1
+$ progress
+progress notify: 3
+```
+
+Note: The revision number in the progress notify response is the revision from the local etcd server node that the watch stream is connected to. If this node is partitioned and not part of quorum, this progress notify revision might be lower than 
+than the revision returned by a quorum read against a non-partitioned etcd server node.
+
 ## Compacted revisions
 
 As we mentioned, etcd keeps revisions so that applications can read past versions of keys. However, to avoid accumulating an unbounded amount of history, it is important to compact past revisions. After compacting, etcd removes historical revisions, releasing resources for future use. All superseded data with revisions before the compacted revision will be unavailable.

Documentation/dev-guide/limit.md

@@ -2,9 +2,8 @@
 
 ## Request size limit
 
-etcd is designed to handle small key value pairs typical for metadata. Larger requests will work, but may increase the latency of other requests. For the time being, etcd guarantees to support RPC requests with up to 1MB of data. In the future, the size limit may be loosened or made configurable.
+etcd is designed to handle small key value pairs typical for metadata. Larger requests will work, but may increase the latency of other requests. By default, the maximum size of any request is 1.5 MiB. This limit is configurable through `--max-request-bytes` flag for etcd server.
 
 ## Storage size limit
 
-The default storage size limit is 2GB, configurable with `--quota-backend-bytes` flag; supports up to 8GB.
-
+The default storage size limit is 2GB, configurable with `--quota-backend-bytes` flag. 8GB is a suggested maximum size for normal environments and etcd warns at startup if the configured value exceeds it.

Documentation/dev-internal/release.md

@@ -12,8 +12,8 @@ release and for ensuring the stability of the release branch.
 
 | Releases | Manager |
 | -------- | ------- |
-| 3.1 patch (post 3.1.0) | Joe Betz [@jpbetz](https://github.com/jpbetz) |
-| 3.2 patch (post 3.2.0) | Gyuho Lee [@gyuho](https://github.com/gyuho) |
+| 3.2 patch (post 3.2.0) | Joe Betz [@jpbetz](https://github.com/jpbetz) |
+| 3.3 patch (post 3.3.0) | Gyuho Lee [@gyuho](https://github.com/gyuho) |
 
 ## Prepare release
 
@@ -79,8 +79,8 @@ The following commands are used for public release sign:
 
 ```
 cd release
-for i in etcd-*{.zip,.tar.gz,.aci}; do gpg2 --default-key $SUBKEYID --armor --output ${i}.asc --detach-sign ${i}; done
-for i in etcd-*{.zip,.tar.gz,.aci}; do gpg2 --verify ${i}.asc ${i}; done
+for i in etcd-*{.zip,.tar.gz}; do gpg2 --default-key $SUBKEYID --armor --output ${i}.asc --detach-sign ${i}; done
+for i in etcd-*{.zip,.tar.gz}; do gpg2 --verify ${i}.asc ${i}; done
 
 # sign zipped source code files
 wget https://github.com/coreos/etcd/archive/${VERSION}.zip
@@ -99,7 +99,7 @@ The public key for GPG signing can be found at [CoreOS Application Signing Key](
 
 - Set release title as the version name.
 - Follow the format of previous release pages.
-- Attach the generated binaries, aci image and signatures.
+- Attach the generated binaries and signatures.
 - Select whether it is a pre-release.
 - Publish the release!
 

Documentation/dl_build.md

@@ -26,16 +26,7 @@ To build a vendored `etcd` from the `master` branch via `go get`:
 # GOPATH should be set
 $ echo $GOPATH
 /Users/example/go
-$ go get github.com/coreos/etcd/cmd/etcd
-```
-
-To build `etcd` from the `master` branch without vendoring (may not build due to upstream conflicts):
-
-```sh
-# GOPATH should be set
-$ echo $GOPATH
-/Users/example/go
-$ go get github.com/coreos/etcd
+$ go get -v github.com/coreos/etcd
 ```
 
 ## Test the installation

Documentation/etcd-mixin/README.md

@@ -0,0 +1,11 @@
+# Prometheus Monitoring Mixin for etcd
+
+> NOTE: This project is *alpha* stage. Flags, configuration, behaviour and design may change significantly in following releases.
+
+A set of customisable Prometheus alerts for etcd.
+
+Instructions for use are the same as the [kubernetes-mixin](https://github.com/kubernetes-monitoring/kubernetes-mixin).
+
+## Background
+
+* For more information about monitoring mixins, see this [design doc](https://docs.google.com/document/d/1A9xvzwqnFVSOZ5fD3blKODXfsat5fg6ZhnKu9LK3lB4/edit#).

Documentation/faq.md

@@ -22,7 +22,7 @@ A member's advertised peer URLs come from `--initial-advertise-peer-urls` on ini
 
 ### System requirements
 
-Since etcd writes data to disk, SSD is highly recommended. To prevent performance degradation or unintentionally overloading the key-value store, etcd enforces a 2GB default storage size quota, configurable up to 8GB. To avoid swapping or running out of memory, the machine should have at least as much RAM to cover the quota. At CoreOS, an etcd cluster is usually deployed on dedicated CoreOS Container Linux machines with dual-core processors, 2GB of RAM, and 80GB of SSD *at the very least*. **Note that performance is intrinsically workload dependent; please test before production deployment**. See [hardware][hardware-setup] for more recommendations.
+Since etcd writes data to disk, SSD is highly recommended. To prevent performance degradation or unintentionally overloading the key-value store, etcd enforces a configurable storage size quota set to 2GB by default. To avoid swapping or running out of memory, the machine should have at least as much RAM to cover the quota. 8GB is a suggested maximum size for normal environments and etcd warns at startup if the configured value exceeds it. At CoreOS, an etcd cluster is usually deployed on dedicated CoreOS Container Linux machines with dual-core processors, 2GB of RAM, and 80GB of SSD *at the very least*. **Note that performance is intrinsically workload dependent; please test before production deployment**. See [hardware][hardware-setup] for more recommendations.
 
 Most stable production environment is Linux operating system with amd64 architecture; see [supported platform][supported-platform] for more.
 
@@ -102,6 +102,12 @@ To recover from the low space quota alarm:
 2. [Defragment][maintenance-defragment] every etcd endpoint.
 3. [Disarm][maintenance-disarm] the alarm.
 
+### What does the etcd warning "etcdserver/api/v3rpc: transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:2379->127.0.0.1:43020: read: connection reset by peer" mean?
+
+This is gRPC-side warning when a server receives a TCP RST flag with client-side streams being prematurely closed. For example, a client closes its connection, while gRPC server has not yet processed all HTTP/2 frames in the TCP queue. Some data may have been lost in server side, but it is ok so long as client connection has already been closed.
+
+Only [old versions of gRPC](https://github.com/grpc/grpc-go/issues/1362) log this. etcd [>=v3.2.13 by default log this with DEBUG level](https://github.com/coreos/etcd/pull/9080), thus only visible with `--debug` flag enabled.
+
 ## Performance
 
 ### How should I benchmark etcd?

Documentation/integrations.md

@@ -15,6 +15,7 @@
 - [etcd-rest](https://github.com/mickep76/etcd-rest) - Create generic REST API in Go using etcd as a backend with validation using JSON schema
 - [etcdsh](https://github.com/kamilhark/etcdsh) - A command line client with support of command history and tab completion. Supports v2
 - [etcdloadtest](https://github.com/sinsharat/etcdloadtest) - A command line load test client for etcd version 3.0 and above.
+- [lucas](https://github.com/ringtail/lucas) - A web-based key-value viewer for kubernetes etcd3.0+ cluster.
 
 **Go libraries**
 
@@ -94,11 +95,13 @@
 
 - [wangjia184/etcdnet](https://github.com/wangjia184/etcdnet) - Supports v2
 - [drusellers/etcetera](https://github.com/drusellers/etcetera)
+- [shubhamranjan/dotnet-etcd](https://github.com/shubhamranjan/dotnet-etcd) - Supports v3+ (GRPC only)
 
 **PHP Libraries**
 
 - [linkorb/etcd-php](https://github.com/linkorb/etcd-php)
 - [activecollab/etcd](https://github.com/activecollab/etcd)
+- [ouqiang/etcd-php](https://github.com/ouqiang/etcd-php) - Client for v3 gRPC gateway
 
 **Haskell libraries**
 
@@ -152,7 +155,6 @@
 - [mattn/etcdenv](https://github.com/mattn/etcdenv) - "env" shebang with etcd integration
 - [kelseyhightower/confd](https://github.com/kelseyhightower/confd) - Manage local app config files using templates and data from etcd
 - [configdb](https://git.autistici.org/ai/configdb/tree/master) - A REST relational abstraction on top of arbitrary database backends, aimed at storing configs and inventories.
-- [fleet](https://github.com/coreos/fleet) - Distributed init system
 - [kubernetes/kubernetes](https://github.com/kubernetes/kubernetes) - Container cluster manager introduced by Google.
 - [mailgun/vulcand](https://github.com/mailgun/vulcand) - HTTP proxy that uses etcd as a configuration backend.
 - [duedil-ltd/discodns](https://github.com/duedil-ltd/discodns) - Simple DNS nameserver using etcd as a database for names and records.

Documentation/learning/api_guarantees.md

@@ -51,7 +51,7 @@ Linearizability (also known as Atomic Consistency or External Consistency) is a
 
 For linearizability, suppose each operation receives a timestamp from a loosely synchronized global clock. Operations are linearized if and only if they always complete as though they were executed in a sequential order and each operation appears to complete in the order specified by the program. Likewise, if an operation’s timestamp precedes another, that operation must also precede the other operation in the sequence.
 
-For example, consider a client completing a write at time point 1 (*t1*). A client issuing a read at *t2* (for *t2* > *t1*) should receive a value at least as recent as the previous write, completed at *t1*. However, the read might actually complete only by *t3*, and the returned value, current at *t2* when the read began, might be "stale" by *t3*.
+For example, consider a client completing a write at time point 1 (*t1*). A client issuing a read at *t2* (for *t2* > *t1*) should receive a value at least as recent as the previous write, completed at *t1*. However, the read might actually complete only by *t3*. Linearizability guarantees the read returns the most current value. Without linearizability guarantee, the returned value, current at *t2* when the read began, might be "stale" by *t3* because a concurrent write might happen between *t2* and *t3*.
 
 etcd does not ensure linearizability for watch operations. Users are expected to verify the revision of watch responses to ensure correct ordering.
 

Documentation/learning/auth_design.md

@@ -38,7 +38,7 @@ Therefore, the permission checking logic should be added to the state machine of
 
 ### Authentication
 
-At first, a client must create a gRPC connection only to authenticate its user ID and password. An etcd server will respond with an authentication reply. The reponse will be an authentication token on success or an error on failure. The client can use its authentication token to present its credentials to etcd when making API requests.
+At first, a client must create a gRPC connection only to authenticate its user ID and password. An etcd server will respond with an authentication reply. The response will be an authentication token on success or an error on failure. The client can use its authentication token to present its credentials to etcd when making API requests.
 
 The client connection used to request the authentication token is typically thrown away; it cannot carry the new token's credentials. This is because gRPC doesn't provide a way for adding per RPC credential after creation of the connection (calling `grpc.Dial()`). Therefore, a client cannot assign a token to its connection that is obtained through the connection. The client needs a new connection for using the token.
 

Documentation/learning/data_model.md

@@ -8,9 +8,9 @@ etcd stores data in a multiversion [persistent][persistent-ds] key-value store.
 
 The store’s logical view is a flat binary key space. The key space has a lexically sorted index on byte string keys so range queries are inexpensive.
 
-The key space maintains multiple revisions. Each atomic mutative operation (e.g., a transaction operation may contain multiple operations) creates a new revision on the key space. All data held by previous revisions remains unchanged. Old versions of key can still be accessed through previous revisions. Likewise, revisions are indexed as well; ranging over revisions with watchers is efficient. If the store is compacted to save space, revisions before the compact revision will be removed.
+The key space maintains multiple **revisions**. Each atomic mutative operation (e.g., a transaction operation may contain multiple operations) creates a new revision on the key space. All data held by previous revisions remains unchanged. Old versions of key can still be accessed through previous revisions. Likewise, revisions are indexed as well; ranging over revisions with watchers is efficient. If the store is compacted to save space, revisions before the compact revision will be removed. Revisions are monotonically increasing over the lifetime of a cluster.
 
-A key’s lifetime spans a generation, denoted by its version. Each key may have one or multiple generations. Creating a key increments the version of that key, starting at 1 if the key never existed. Deleting a key generates a key tombstone, concluding the key’s current generation by resetting its version. Each modification of a key increments its version. Once a compaction happens, any version ended before the given revision will be removed and values set before the compaction revision except the latest one will be removed.
+A key's life spans a generation, from creation to deletion. Each key may have one or multiple generations. Creating a key increments the **version** of that key, starting at 1 if the key does not exist at the current revision. Deleting a key generates a key tombstone, concluding the key’s current generation by resetting its version to 0. Each modification of a key increments its version; so, versions are monotonically increasing within a key's generation. Once a compaction happens, any generation ended before the compaction revision will be removed, and values set before the compaction revision except the latest one will be removed.
 
 ### Physical view
 

Documentation/learning/why.md

@@ -1,6 +1,6 @@
 # etcd versus other key-value stores
 
-The name "etcd" originated from two ideas, the unix "/etc" folder and "d"istibuted systems. The "/etc" folder is a place to store configuration data for a single system whereas etcd stores configuration information for large scale distributed systems. Hence, a "d"istributed "/etc" is "etcd".
+The name "etcd" originated from two ideas, the unix "/etc" folder and "d"istributed systems. The "/etc" folder is a place to store configuration data for a single system whereas etcd stores configuration information for large scale distributed systems. Hence, a "d"istributed "/etc" is "etcd".
 
 etcd is designed as a general substrate for large scale distributed systems. These are systems that will never tolerate split-brain operation and are willing to sacrifice availability to achieve this end. etcd stores metadata in a consistent and fault-tolerant way. An etcd cluster is meant to provide key-value storage with best of class stability, reliability, scalability and performance.
 
@@ -47,7 +47,7 @@ When considering features, support, and stability, new applications planning to
 
 ### Consul
 
-Consul bills itself as an end-to-end service discovery framework. To wit, it includes services such as health checking, failure detection, and DNS. Incidentally, Consul also exposes a key value store with mediocre performance and an intricate API. As it stands in Consul 0.7, the storage system does not scales well; systems requiring millions of keys will suffer from high latencies and memory pressure. The key value API is missing, most notably, multi-version keys, conditional transactions, and reliable streaming watches.
+Consul is an end-to-end service discovery framework. It provides built-in health checking, failure detection, and DNS services. In addition, Consul exposes a key value store with RESTful HTTP APIs. [As it stands in Consul 1.0][dbtester-comparison-results], the storage system does not scale as well as other systems like etcd or Zookeeper in key-value operations; systems requiring millions of keys will suffer from high latencies and memory pressure. The key value API is missing, most notably, multi-version keys, conditional transactions, and reliable streaming watches.
 
 etcd and Consul solve different problems. If looking for a distributed consistent key value store, etcd is a better choice over Consul. If looking for end-to-end cluster service discovery, etcd will not have enough features; choose Kubernetes, Consul, or SmartStack.
 
@@ -113,3 +113,4 @@ For distributed coordination, choosing etcd can help prevent operational headach
 [container-linux]: https://coreos.com/why
 [locksmith]: https://github.com/coreos/locksmith
 [kubernetes]: http://kubernetes.io/docs/whatisk8s
+[dbtester-comparison-results]: https://github.com/coreos/dbtester/tree/master/test-results/2018Q1-02-etcd-zookeeper-consul

Documentation/op-guide/authentication.md

@@ -32,7 +32,7 @@ Creating a user is as easy as
 $ etcdctl user add myusername
 ```
 
-Creating a new user will prompt for a new password. The password can be supplied from standard input when an option `--interactive=false` is given.
+Creating a new user will prompt for a new password. The password can be supplied from standard input when an option `--interactive=false` is given. `--new-user-password` can also be used for supplying the password.
 
 Roles can be granted and revoked for a user with:
 
@@ -157,8 +157,17 @@ The password can be taken from a prompt:
 $ etcdctl --user user get foo
 ```
 
+The password can also be taken from a command line flag `--password`:
+
+```
+$ etcdctl --user user --password password get foo
+```
+
+
 Otherwise, all `etcdctl` commands remain the same. Users and roles can still be created and modified, but require authentication by a user with the root role.
 
 ## Using TLS Common Name
+As of version v3.2 if an etcd server is launched with the option `--client-cert-auth=true`, the field of Common Name (CN) in the client's TLS cert will be used as an etcd user. In this case, the common name authenticates the user and the client does not need a password. Note that if both of 1. `--client-cert-auth=true` is passed and CN is provided by the client, and 2. username and password are provided by the client, the username and password based authentication is prioritized.
 
-If an etcd server is launched with the option `--client-cert-auth=true`, the field of Common Name (CN) in the client's TLS cert will be used as an etcd user. In this case, the common name authenticates the user and the client does not need a password. Note that if both of 1. `--client-cert-auth=true` is passed and CN is provided by the client, and 2. username and password are provided by the client, the username and password based authentication is prioritized.
+As of version v3.3 if an etcd server is launched with the option `--peer-cert-allowed-cn` filtering of CN inter-peer connections is enabled.  Nodes can only join the etcd cluster if their CN match the allowed one.
+See [etcd security page](https://github.com/coreos/etcd/blob/master/Documentation/op-guide/security.md) for more details.

Documentation/op-guide/clustering.md

@@ -342,8 +342,8 @@ etcdserver: discovery token ignored since a cluster has already been initialized
 ### DNS discovery
 
 DNS [SRV records][rfc-srv] can be used as a discovery mechanism.
-The `-discovery-srv` flag can be used to set the DNS domain name where the discovery SRV records can be found.
-The following DNS SRV records are looked up in the listed order:
+The `--discovery-srv` flag can be used to set the DNS domain name where the discovery SRV records can be found.
+Setting `--discovery-srv example.com` causes DNS SRV records to be looked up in the listed order:
 
 * _etcd-server-ssl._tcp.example.com
 * _etcd-server._tcp.example.com
@@ -357,8 +357,21 @@ To help clients discover the etcd cluster, the following DNS SRV records are loo
 
 If `_etcd-client-ssl._tcp.example.com` is found, clients will attempt to communicate with the etcd cluster over SSL/TLS.
 
+If etcd is using TLS, the discovery SRV record (e.g. `example.com`) must be included in the SSL certificate DNS SAN along with the hostname, or clustering will fail with log messages like the following:
+
+```
+[...] rejected connection from "10.0.1.11:53162" (error "remote error: tls: bad certificate", ServerName "example.com")
+```
+
 If etcd is using TLS without a custom certificate authority, the discovery domain (e.g., example.com) must match the SRV record domain (e.g., infra1.example.com). This is to mitigate attacks that forge SRV records to point to a different domain; the domain would have a valid certificate under PKI but be controlled by an unknown third party.
 
+The `-discovery-srv-name` flag additionally configures a suffix to the SRV name that is queried during discovery.
+Use this flag to differentiate between multiple etcd clusters under the same domain.
+For example, if `discovery-srv=example.com` and `-discovery-srv-name=foo` are set, the following DNS SRV queries are made:
+
+* _etcd-server-ssl-foo._tcp.example.com
+* _etcd-server-foo._tcp.example.com
+
 #### Create DNS SRV records
 
 ```
@@ -384,7 +397,8 @@ infra2.example.com.  300  IN  A  10.0.1.12
 
 #### Bootstrap the etcd cluster using DNS
 
-etcd cluster members can listen on domain names or IP address, the bootstrap process will resolve DNS A records.
+etcd cluster members can advertise domain names or IP address, the bootstrap process will resolve DNS A records.
+Since 3.2 (3.1 prints warnings) `--listen-peer-urls` and `--listen-client-urls` will reject domain name for the network interface binding.
 
 The resolved address in `--initial-advertise-peer-urls` *must match* one of the resolved addresses in the SRV targets. The etcd member reads the resolved address to find out if it belongs to the cluster defined in the SRV records.
 
@@ -395,8 +409,8 @@ $ etcd --name infra0 \
 --initial-cluster-token etcd-cluster-1 \
 --initial-cluster-state new \
 --advertise-client-urls http://infra0.example.com:2379 \
---listen-client-urls http://infra0.example.com:2379 \
---listen-peer-urls http://infra0.example.com:2380
+--listen-client-urls http://0.0.0.0:2379 \
+--listen-peer-urls http://0.0.0.0:2380
 ```
 
 ```
@@ -406,8 +420,8 @@ $ etcd --name infra1 \
 --initial-cluster-token etcd-cluster-1 \
 --initial-cluster-state new \
 --advertise-client-urls http://infra1.example.com:2379 \
---listen-client-urls http://infra1.example.com:2379 \
---listen-peer-urls http://infra1.example.com:2380
+--listen-client-urls http://0.0.0.0:2379 \
+--listen-peer-urls http://0.0.0.0:2380
 ```
 
 ```
@@ -417,8 +431,8 @@ $ etcd --name infra2 \
 --initial-cluster-token etcd-cluster-1 \
 --initial-cluster-state new \
 --advertise-client-urls http://infra2.example.com:2379 \
---listen-client-urls http://infra2.example.com:2379 \
---listen-peer-urls http://infra2.example.com:2380
+--listen-client-urls http://0.0.0.0:2379 \
+--listen-peer-urls http://0.0.0.0:2380
 ```
 
 The cluster can also bootstrap using IP addresses instead of domain names:

Documentation/op-guide/configuration.md

@@ -1,6 +1,11 @@
 # Configuration flags
 
-etcd is configurable through command-line flags and environment variables. Options set on the command line take precedence over those from the environment.
+etcd is configurable through a configuration file, various command-line flags, and environment variables.
+
+A reusable configuration file is a YAML file made with name and value of one or more command-line flags described below. In order to use this file, specify the file path as a value to the `--config-file` flag. The [sample configuration file][sample-config-file] can be used as a starting point to create a new configuration file as needed.
+
+Options set on the command line take precedence over those from the environment. If a configuration file is provided, other command line flags and environment variables will be ignored.
+For example, `etcd --config-file etcd.conf.yml.sample --data-dir /tmp` will ignore the `--data-dir` flag.
 
 The format of environment variable for flag `--my-flag` is `ETCD_MY_FLAG`. It applies to all flags.
 
@@ -104,7 +109,7 @@ To start etcd automatically using custom settings at startup in Linux, using a [
 
 ## Clustering flags
 
-`--initial` prefix flags are used in bootstrapping ([static bootstrap][build-cluster], [discovery-service bootstrap][discovery] or [runtime reconfiguration][reconfig]) a new member, and ignored when restarting an existing member.
+`--initial-advertise-peer-urls`, `--initial-cluster`, `--initial-cluster-state`, and `--initial-cluster-token` flags are used in bootstrapping ([static bootstrap][build-cluster], [discovery-service bootstrap][discovery] or [runtime reconfiguration][reconfig]) a new member, and ignored when restarting an existing member.
 
 `--discovery` prefix flags need to be set when using [discovery service][discovery].
 
@@ -150,6 +155,11 @@ To start etcd automatically using custom settings at startup in Linux, using a [
 + default: ""
 + env variable: ETCD_DISCOVERY_SRV
 
+### --discovery-srv-name
++ Suffix to the DNS srv name queried when bootstrapping using DNS.
++ default: ""
++ env variable: ETCD_DISCOVERY_SRV_NAME
+
 ### --discovery-fallback
 + Expected behavior ("exit" or "proxy") when discovery services fails. "proxy" supports v2 API only.
 + default: "proxy"
@@ -171,7 +181,7 @@ To start etcd automatically using custom settings at startup in Linux, using a [
 + env variable: ETCD_AUTO_COMPACTION_RETENTION
 
 ### --auto-compaction-mode
-+ Interpret 'auto-compaction-retention' one of: periodic|revision. 'periodic' for duration based retention, defaulting to hours if no time unit is provided (e.g. '5m'). 'revision' for revision number based retention.
++ Interpret 'auto-compaction-retention' one of: 'periodic', 'revision'. 'periodic' for duration based retention, defaulting to hours if no time unit is provided (e.g. '5m'). 'revision' for revision number based retention.
 + default: periodic
 + env variable: ETCD_AUTO_COMPACTION_MODE
 
@@ -266,12 +276,12 @@ The security flags help to [build a secure etcd cluster][security].
 + env variable: ETCD_PEER_CA_FILE
 
 ### --peer-cert-file
-+ Path to the peer server TLS cert file.
++ Path to the peer server TLS cert file. This is the cert for peer-to-peer traffic, used both for server and client.
 + default: ""
 + env variable: ETCD_PEER_CERT_FILE
 
 ### --peer-key-file
-+ Path to the peer server TLS key file.
++ Path to the peer server TLS key file. This is the key for peer-to-peer traffic, used both for server and client.
 + default: ""
 + env variable: ETCD_PEER_KEY_FILE
 
@@ -302,6 +312,16 @@ The security flags help to [build a secure etcd cluster][security].
 
 ## Logging flags
 
+### --logger
++ Specify 'zap' for structured logging or 'capnslog'.
++ default: capnslog
++ env variable: ETCD_LOGGER
+
+### --log-outputs
++ Specify 'stdout' or 'stderr' to skip journald logging even when running under systemd, or list of comma separated output targets.
++ default: default
++ env variable: ETCD_LOG_OUTPUT
+
 ### --debug
 + Drop the default log level to DEBUG for all subpackages.
 + default: false (INFO for all packages)
@@ -332,6 +352,7 @@ Follow the instructions when using these flags.
 ### --config-file
 + Load server configuration from a file.
 + default: ""
++ example: [sample configuration file][sample-config-file]
 
 ## Profiling flags
 
@@ -344,16 +365,21 @@ Follow the instructions when using these flags.
 + default: basic
 
 ### --listen-metrics-urls
-+ List of URLs to listen on for metrics.
++ List of additional URLs to listen on that will respond to both the `/metrics` and `/health` endpoints
 + default: ""
 
 ## Auth flags
 
 ### --auth-token
-+ Specify a token type and token specific options, especially for JWT. Its format is "type,var1=val1,var2=val2,...". Possible type is 'simple' or 'jwt'. Possible variables are 'sign-method' for specifying a sign method of jwt (its possible values are 'ES256', 'ES384', 'ES512', 'HS256', 'HS384', 'HS512', 'RS256', 'RS384', 'RS512', 'PS256', 'PS384', or 'PS512'), 'pub-key' for specifying a path to a public key for verifying jwt, and 'priv-key' for specifying a path to a private key for signing jwt.
-+ Example option of JWT: '--auth-token jwt,pub-key=app.rsa.pub,priv-key=app.rsa,sign-method=RS512'
++ Specify a token type and token specific options, especially for JWT. Its format is "type,var1=val1,var2=val2,...". Possible type is 'simple' or 'jwt'. Possible variables are 'sign-method' for specifying a sign method of jwt (its possible values are 'ES256', 'ES384', 'ES512', 'HS256', 'HS384', 'HS512', 'RS256', 'RS384', 'RS512', 'PS256', 'PS384', or 'PS512'), 'pub-key' for specifying a path to a public key for verifying jwt, 'priv-key' for specifying a path to a private key for signing jwt, and 'ttl' for specifying TTL of jwt tokens.
++ For asymmetric algorithms ('RS', 'PS', 'ES'), the public key is optional, as the private key contains enough information to both sign and verify tokens.
++ Example option of JWT: '--auth-token jwt,pub-key=app.rsa.pub,priv-key=app.rsa,sign-method=RS512,ttl=10m'
 + default: "simple"
 
+### --bcrypt-cost
++ Specify the cost / strength of the bcrypt algorithm for hashing auth passwords. Valid values are between 4 and 31.
++ default: 10
+
 ## Experimental flags
 
 ### --experimental-corrupt-check-time
@@ -369,3 +395,4 @@ Follow the instructions when using these flags.
 [security]: security.md
 [systemd-intro]: http://freedesktop.org/wiki/Software/systemd/
 [tuning]: ../tuning.md#time-parameters
+[sample-config-file]: ../../etcd.conf.yml.sample

Documentation/op-guide/container.md

@@ -17,14 +17,14 @@ export NODE1=192.168.1.21
 Trust the CoreOS [App Signing Key](https://coreos.com/security/app-signing-key/).
 
 ```
-sudo rkt trust --prefix coreos.com/etcd
+sudo rkt trust --prefix quay.io/coreos/etcd
 # gpg key fingerprint is: 18AD 5014 C99E F7E3 BA5F  6CE9 50BD D3E0 FC8A 365E
 ```
 
-Run the `v3.1.2` version of etcd or specify another release version.
+Run the `v3.2` version of etcd or specify another release version.
 
 ```
-sudo rkt run --net=default:IP=${NODE1} coreos.com/etcd:v3.1.2 -- -name=node1 -advertise-client-urls=http://${NODE1}:2379 -initial-advertise-peer-urls=http://${NODE1}:2380 -listen-client-urls=http://0.0.0.0:2379 -listen-peer-urls=http://${NODE1}:2380 -initial-cluster=node1=http://${NODE1}:2380
+sudo rkt run --net=default:IP=${NODE1} quay.io/coreos/etcd:v3.2 -- -name=node1 -advertise-client-urls=http://${NODE1}:2379 -initial-advertise-peer-urls=http://${NODE1}:2380 -listen-client-urls=http://0.0.0.0:2379 -listen-peer-urls=http://${NODE1}:2380 -initial-cluster=node1=http://${NODE1}:2380
 ```
 
 List the cluster member.
@@ -45,13 +45,13 @@ export NODE3=172.16.28.23
 
 ```
 # node 1
-sudo rkt run --net=default:IP=${NODE1} coreos.com/etcd:v3.1.2 -- -name=node1 -advertise-client-urls=http://${NODE1}:2379 -initial-advertise-peer-urls=http://${NODE1}:2380 -listen-client-urls=http://0.0.0.0:2379 -listen-peer-urls=http://${NODE1}:2380 -initial-cluster=node1=http://${NODE1}:2380,node2=http://${NODE2}:2380,node3=http://${NODE3}:2380
+sudo rkt run --net=default:IP=${NODE1} quay.io/coreos/etcd:v3.2 -- -name=node1 -advertise-client-urls=http://${NODE1}:2379 -initial-advertise-peer-urls=http://${NODE1}:2380 -listen-client-urls=http://0.0.0.0:2379 -listen-peer-urls=http://${NODE1}:2380 -initial-cluster=node1=http://${NODE1}:2380,node2=http://${NODE2}:2380,node3=http://${NODE3}:2380
 
 # node 2
-sudo rkt run --net=default:IP=${NODE2} coreos.com/etcd:v3.1.2 -- -name=node2 -advertise-client-urls=http://${NODE2}:2379 -initial-advertise-peer-urls=http://${NODE2}:2380 -listen-client-urls=http://0.0.0.0:2379 -listen-peer-urls=http://${NODE2}:2380 -initial-cluster=node1=http://${NODE1}:2380,node2=http://${NODE2}:2380,node3=http://${NODE3}:2380
+sudo rkt run --net=default:IP=${NODE2} quay.io/coreos/etcd:v3.2 -- -name=node2 -advertise-client-urls=http://${NODE2}:2379 -initial-advertise-peer-urls=http://${NODE2}:2380 -listen-client-urls=http://0.0.0.0:2379 -listen-peer-urls=http://${NODE2}:2380 -initial-cluster=node1=http://${NODE1}:2380,node2=http://${NODE2}:2380,node3=http://${NODE3}:2380
 
 # node 3
-sudo rkt run --net=default:IP=${NODE3} coreos.com/etcd:v3.1.2 -- -name=node3 -advertise-client-urls=http://${NODE3}:2379 -initial-advertise-peer-urls=http://${NODE3}:2380 -listen-client-urls=http://0.0.0.0:2379 -listen-peer-urls=http://${NODE3}:2380 -initial-cluster=node1=http://${NODE1}:2380,node2=http://${NODE2}:2380,node3=http://${NODE3}:2380
+sudo rkt run --net=default:IP=${NODE3} quay.io/coreos/etcd:v3.2 -- -name=node3 -advertise-client-urls=http://${NODE3}:2379 -initial-advertise-peer-urls=http://${NODE3}:2380 -listen-client-urls=http://0.0.0.0:2379 -listen-peer-urls=http://${NODE3}:2380 -initial-cluster=node1=http://${NODE1}:2380,node2=http://${NODE2}:2380,node3=http://${NODE3}:2380
 ```
 
 Verify the cluster is healthy and can be reached.

Documentation/op-guide/etcd3_alert.rules

@@ -43,8 +43,8 @@ ANNOTATIONS {
 
 # alert if more than 1% of gRPC method calls have failed within the last 5 minutes
 ALERT HighNumberOfFailedGRPCRequests
-IF sum by(grpc_method) (rate(etcd_grpc_requests_failed_total{job="etcd"}[5m]))
-  / sum by(grpc_method) (rate(etcd_grpc_total{job="etcd"}[5m])) > 0.01
+IF 100 * (sum by(grpc_method) (rate(etcd_grpc_requests_failed_total{job="etcd"}[5m]))
+  / sum by(grpc_method) (rate(etcd_grpc_total{job="etcd"}[5m]))) > 1
 FOR 10m
 LABELS {
   severity = "warning"
@@ -56,8 +56,8 @@ ANNOTATIONS {
 
 # alert if more than 5% of gRPC method calls have failed within the last 5 minutes
 ALERT HighNumberOfFailedGRPCRequests
-IF sum by(grpc_method) (rate(etcd_grpc_requests_failed_total{job="etcd"}[5m]))
-  / sum by(grpc_method) (rate(etcd_grpc_total{job="etcd"}[5m])) > 0.05
+IF 100 * (sum by(grpc_method) (rate(etcd_grpc_requests_failed_total{job="etcd"}[5m]))
+  / sum by(grpc_method) (rate(etcd_grpc_total{job="etcd"}[5m]))) > 5
 FOR 5m
 LABELS {
   severity = "critical"
@@ -79,47 +79,6 @@ ANNOTATIONS {
   description = "on etcd instance {{ $labels.instance }} gRPC requests to {{ $labels.grpc_method }} are slow",
 }
 
-# HTTP requests alerts
-# ====================
-
-# alert if more than 1% of requests to an HTTP endpoint have failed within the last 5 minutes
-ALERT HighNumberOfFailedHTTPRequests
-IF sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
-  / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method) > 0.01
-FOR 10m
-LABELS {
-  severity = "warning"
-}
-ANNOTATIONS {
-  summary = "a high number of HTTP requests are failing",
-  description = "{{ $value }}% of requests for {{ $labels.method }} failed on etcd instance {{ $labels.instance }}",
-}
-
-# alert if more than 5% of requests to an HTTP endpoint have failed within the last 5 minutes
-ALERT HighNumberOfFailedHTTPRequests
-IF sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
-  / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method) > 0.05
-FOR 5m
-LABELS {
-  severity = "critical"
-}
-ANNOTATIONS {
-  summary = "a high number of HTTP requests are failing",
-  description = "{{ $value }}% of requests for {{ $labels.method }} failed on etcd instance {{ $labels.instance }}",
-}
-
-# alert if the 99th percentile of HTTP requests take more than 150ms
-ALERT HTTPRequestsSlow
-IF histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m])) > 0.15
-FOR 10m
-LABELS {
-  severity = "warning"
-}
-ANNOTATIONS {
-  summary = "slow HTTP requests",
-  description = "on etcd instance {{ $labels.instance }} HTTP requests to {{ $labels.method }} are slow",
-}
-
 # file descriptor alerts
 # ======================
 

Documentation/op-guide/etcd3_alert.rules.yml

@@ -26,8 +26,8 @@ groups:
         changes within the last hour
       summary: a high number of leader changes within the etcd cluster are happening
   - alert: HighNumberOfFailedGRPCRequests
-    expr: sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
-      / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method) > 0.01
+    expr: 100 * (sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
+      / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method)) > 1
     for: 10m
     labels:
       severity: warning
@@ -36,8 +36,8 @@ groups:
         on etcd instance {{ $labels.instance }}'
       summary: a high number of gRPC requests are failing
   - alert: HighNumberOfFailedGRPCRequests
-    expr: sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
-      / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method) > 0.05
+    expr: 100 * (sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
+      / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method)) > 5
     for: 5m
     labels:
       severity: critical
@@ -55,36 +55,6 @@ groups:
       description: on etcd instance {{ $labels.instance }} gRPC requests to {{ $labels.grpc_method
         }} are slow
       summary: slow gRPC requests
-  - alert: HighNumberOfFailedHTTPRequests
-    expr: sum(rate(etcd_http_failed_total{job="etcd"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="etcd"}[5m]))
-      BY (method) > 0.01
-    for: 10m
-    labels:
-      severity: warning
-    annotations:
-      description: '{{ $value }}% of requests for {{ $labels.method }} failed on etcd
-        instance {{ $labels.instance }}'
-      summary: a high number of HTTP requests are failing
-  - alert: HighNumberOfFailedHTTPRequests
-    expr: sum(rate(etcd_http_failed_total{job="etcd"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="etcd"}[5m]))
-      BY (method) > 0.05
-    for: 5m
-    labels:
-      severity: critical
-    annotations:
-      description: '{{ $value }}% of requests for {{ $labels.method }} failed on etcd
-        instance {{ $labels.instance }}'
-      summary: a high number of HTTP requests are failing
-  - alert: HTTPRequestsSlow
-    expr: histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m]))
-      > 0.15
-    for: 10m
-    labels:
-      severity: warning
-    annotations:
-      description: on etcd instance {{ $labels.instance }} HTTP requests to {{ $labels.method
-        }} are slow
-      summary: slow HTTP requests
   - record: instance:fd_utilization
     expr: process_open_fds / process_max_fds
   - alert: FdExhaustionClose

Documentation/op-guide/grpc_proxy.md

@@ -85,7 +85,7 @@ Start the etcd gRPC proxy to use these static endpoints with the command:
 $ etcd grpc-proxy start --endpoints=infra0.example.com,infra1.example.com,infra2.example.com --listen-addr=127.0.0.1:2379
 ```
 
-The etcd gRPC proxy starts and listens on port 8080. It forwards client requests to one of the three endpoints provided above.
+The etcd gRPC proxy starts and listens on port 2379. It forwards client requests to one of the three endpoints provided above.
 
 Sending requests through the proxy:
 

Documentation/op-guide/hardware.md

@@ -48,7 +48,7 @@ Example application workload: A 50-node Kubernetes cluster
 | Provider | Type | vCPUs | Memory (GB) | Max concurrent IOPS | Disk bandwidth (MB/s) |
 |----------|------|-------|--------|------|----------------|
 | AWS | m4.large | 2 | 8 | 3600 | 56.25 |
-| GCE | n1-standard-1 + 50GB PD SSD | 2 | 7.5 | 1500 | 25 |
+| GCE | n1-standard-2 + 50GB PD SSD | 2 | 7.5 | 1500 | 25 |
 
 
 ### Medium cluster

Documentation/op-guide/maintenance.md

@@ -6,25 +6,27 @@ An etcd cluster needs periodic maintenance to remain reliable. Depending on an e
 
 All etcd maintenance manages storage resources consumed by the etcd keyspace. Failure to adequately control the keyspace size is guarded by storage space quotas; if an etcd member runs low on space, a quota will trigger cluster-wide alarms which will put the system into a limited-operation maintenance mode. To avoid running out of space for writes to the keyspace, the etcd keyspace history must be compacted. Storage space itself may be reclaimed by defragmenting etcd members. Finally, periodic snapshot backups of etcd member state makes it possible to recover any unintended logical data loss or corruption caused by operational error.
 
-## History compaction
+## Raft log retention
+
+`etcd --snapshot-count` configures the number of applied Raft entries to hold in-memory before compaction. When `--snapshot-count` reaches, server first persists snapshot data onto disk, and then truncates old entries. When a slow follower requests logs before a compacted index, leader sends the snapshot forcing the follower to overwrite its state.
+
+Higher `--snapshot-count` holds more Raft entries in memory until snapshot, thus causing [recurrent higher memory usage](https://github.com/kubernetes/kubernetes/issues/60589#issuecomment-371977156). Since leader retains latest Raft entries for longer, a slow follower has more time to catch up before leader snapshot. `--snapshot-count` is a tradeoff between higher memory usage and better availabilities of slow followers.
+
+Since v3.2, the default value of `--snapshot-count` has [changed from from 10,000 to 100,000](https://github.com/coreos/etcd/pull/7160).
+
+In performance-wise, `--snapshot-count` greater than 100,000 may impact the write throughput. Higher number of in-memory objects can slow down [Go GC mark phase `runtime.scanobject`](https://golang.org/src/runtime/mgc.go), and infrequent memory reclamation makes allocation slow. Performance varies depending on the workloads and system environments. However, in general, too frequent compaction affects cluster availabilities and write throughputs. Too infrequent compaction is also harmful placing too much pressure on Go garbage collector. See https://www.slideshare.net/mitakeh/understanding-performance-aspects-of-etcd-and-raft for more research results.
+
+## History compaction: v3 API Key-Value Database
 
 Since etcd keeps an exact history of its keyspace, this history should be periodically compacted to avoid performance degradation and eventual storage space exhaustion. Compacting the keyspace history drops all information about keys superseded prior to a given keyspace revision. The space used by these keys then becomes available for additional writes to the keyspace.
 
 The keyspace can be compacted automatically with `etcd`'s time windowed history retention policy, or manually with `etcdctl`. The `etcdctl` method provides fine-grained control over the compacting process whereas automatic compacting fits applications that only need key history for some length of time.
 
-`etcd` can be set to automatically compact the keyspace with the `--auto-compaction` option with a period of hours:
-
-```sh
-# keep one hour of history
-$ etcd --auto-compaction-retention=1
-```
-
 An `etcdctl` initiated compaction works as follows:
 
 ```sh
 # compact up to revision 3
 $ etcdctl compact 3
-
 ```
 
 Revisions prior to the compaction revision become inaccessible:
@@ -34,11 +36,43 @@ $ etcdctl get --rev=2 somekey
 Error:  rpc error: code = 11 desc = etcdserver: mvcc: required revision has been compacted
 ```
 
+### Auto Compaction
+
+`etcd` can be set to automatically compact the keyspace with the `--auto-compaction-*` option with a period of hours:
+
+```sh
+# keep one hour of history
+$ etcd --auto-compaction-retention=1
+```
+
+[v3.0.0](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.0.md) and [v3.1.0](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.1.md) with `--auto-compaction-retention=10` run periodic compaction on v3 key-value store for every 10-hour. Compactor only supports periodic compaction. Compactor records latest revisions every 5-minute, until it reaches the first compaction period (e.g. 10-hour). In order to retain key-value history of last compaction period, it uses the last revision that was fetched before compaction period, from the revision records that were collected every 5-minute. When `--auto-compaction-retention=10`, compactor uses revision 100 for compact revision where revision 100 is the latest revision fetched from 10 hours ago. If compaction succeeds or requested revision has already been compacted, it resets period timer and starts over with new historical revision records (e.g. restart revision collect and compact for the next 10-hour period). If compaction fails, it retries in 5 minutes.
+
+[v3.2.0](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.2.md) compactor runs [every hour](https://github.com/coreos/etcd/pull/7875). Compactor only supports periodic compaction. Compactor continues to record latest revisions every 5-minute. For every hour, it uses the last revision that was fetched before compaction period, from the revision records that were collected every 5-minute. That is, for every hour, compactor discards historical data created before compaction period. The retention window of compaction period moves to next hour. For instance, when hourly writes are 100 and `--auto-compaction-retention=10`, v3.1 compacts revision 1000, 2000, and 3000 for every 10-hour, while v3.2.x, v3.3.0, v3.3.1, and v3.3.2 compact revision 1000, 1100, and 1200 for every 1-hour. If compaction succeeds or requested revision has already been compacted, it resets period timer and removes used compacted revision from historical revision records (e.g. start next revision collect and compaction from previously collected revisions). If compaction fails, it retries in 5 minutes.
+
+In [v3.3.0](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.3.md), [v3.3.1](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.3.md), and [v3.3.2](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.3.md), `--auto-compaction-mode=revision --auto-compaction-retention=1000` automatically `Compact` on `"latest revision" - 1000` every 5-minute (when latest revision is 30000, compact on revision 29000). For instance, `--auto-compaction-mode=periodic --auto-compaction-retention=72h` automatically `Compact` with 72-hour retention windown, for every 7.2-hour. For instance, `--auto-compaction-mode=periodic --auto-compaction-retention=30m` automatically `Compact` with 30-minute retention windown, for every 3-minute. Periodic compactor continues to record latest revisions for every 1/10 of given compaction period (e.g. 1-hour when `--auto-compaction-mode=periodic --auto-compaction-retention=10h`). For every 1/10 of given compaction period, compactor uses the last revision that was fetched before compaction period, to discard historical data. The retention window of compaction period moves for every 1/10 of given compaction period. For instance, when hourly writes are 100 and `--auto-compaction-retention=10`, v3.1 compacts revision 1000, 2000, and 3000 for every 10-hour, while v3.2.x, v3.3.0, v3.3.1, and v3.3.2 compact revision 1000, 1100, and 1200 for every 1-hour. Futhermore, when writes per minute are 1000, v3.3.0, v3.3.1, and v3.3.2 with `--auto-compaction-mode=periodic --auto-compaction-retention=30m` compact revision 30000, 33000, and 36000, for every 3-minute with more finer granularity.
+
+When `--auto-compaction-retention=10h`, etcd first waits 10-hour for the first compaction, and then does compaction every hour (1/10 of 10-hour) afterwards like this:
+
+```
+0Hr  (rev = 1)
+1hr  (rev = 10)
+...
+8hr  (rev = 80)
+9hr  (rev = 90)
+10hr (rev = 100, Compact(1))
+11hr (rev = 110, Compact(10))
+...
+```
+
+Whether compaction succeeds or not, this process repeats for every 1/10 of given compaction period. If compaction succeeds, it just removes compacted revision from historical revision records.
+
+In [v3.3.3](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.3.md), `--auto-compaction-mode=revision --auto-compaction-retention=1000` automatically `Compact` on `"latest revision" - 1000` every 5-minute (when latest revision is 30000, compact on revision 29000). Previously, `--auto-compaction-mode=periodic --auto-compaction-retention=72h` automatically `Compact` with 72-hour retention windown for every 7.2-hour.  **Now, `Compact` happens, for every 1-hour but still with 72-hour retention window.** Previously, `--auto-compaction-mode=periodic --auto-compaction-retention=30m` automatically `Compact` with 30-minute retention windown for every 3-minute. **Now, `Compact` happens, for every 30-minute but still with 30-minute retention window.** Periodic compactor keeps recording latest revisions for every compaction period when given period is less than 1-hour, or for every 1-hour when given compaction period is greater than 1-hour (e.g. 1-hour when `--auto-compaction-mode=periodic --auto-compaction-retention=24h`). For every compaction period or 1-hour, compactor uses the last revision that was fetched before compaction period, to discard historical data. The retention window of compaction period moves for every given compaction period or hour. For instance, when hourly writes are 100 and `--auto-compaction-mode=periodic --auto-compaction-retention=24h`, `v3.2.x`, `v3.3.0`, `v3.3.1`, and `v3.3.2` compact revision 2400, 2640, and 2880 for every 2.4-hour, while `v3.3.3` *or later* compacts revision 2400, 2500, 2600 for every 1-hour. Futhermore, when `--auto-compaction-mode=periodic --auto-compaction-retention=30m` and writes per minute are about 1000, `v3.3.0`, `v3.3.1`, and `v3.3.2` compact revision 30000, 33000, and 36000, for every 3-minute, while `v3.3.3` *or later* compacts revision 30000, 60000, and 90000, for every 30-minute.
+
 ## Defragmentation
 
-After compacting the keyspace, the backend database may exhibit internal fragmentation. Any internal fragmentation is space that is free to use by the backend but still consumes storage space. The process of defragmentation releases this storage space back to the file system. Defragmentation is issued on a per-member so that cluster-wide latency spikes may be avoided.
+After compacting the keyspace, the backend database may exhibit internal fragmentation. Any internal fragmentation is space that is free to use by the backend but still consumes storage space. Compacting old revisions internally fragments `etcd` by leaving gaps in backend database. Fragmented space is available for use by `etcd` but unavailable to the host filesystem. In other words, deleting application data does not reclaim the space on disk.
 
-Compacting old revisions internally fragments `etcd` by leaving gaps in backend database. Fragmented space is available for use by `etcd` but unavailable to the host filesystem.
+The process of defragmentation releases this storage space back to the file system. Defragmentation is issued on a per-member so that cluster-wide latency spikes may be avoided.
 
 To defragment an etcd member, use the `etcdctl defrag` command:
 
@@ -47,6 +81,19 @@ $ etcdctl defrag
 Finished defragmenting etcd member[127.0.0.1:2379]
 ```
 
+**Note that defragmentation to a live member blocks the system from reading and writing data while rebuilding its states**.
+
+**Note that defragmentation request does not get replicated over cluster. That is, the request is only applied to the local node. Specify all members in `--endpoints` flag or `--cluster` flag to automatically find all cluster members.**
+
+Run defragment operations for all endpoints in the cluster associated with the default endpoint:
+
+```bash
+$ etcdctl defrag --cluster
+Finished defragmenting etcd member[http://127.0.0.1:2379]
+Finished defragmenting etcd member[http://127.0.0.1:22379]
+Finished defragmenting etcd member[http://127.0.0.1:32379]
+```
+
 To defragment an etcd data directory directly, while etcd is not running, use the command:
 
 ``` sh
@@ -80,14 +127,14 @@ $ ETCDCTL_API=3 etcdctl --write-out=table endpoint status
 +----------------+------------------+-----------+---------+-----------+-----------+------------+
 # confirm alarm is raised
 $ ETCDCTL_API=3 etcdctl alarm list
-memberID:13803658152347727308 alarm:NOSPACE 
+memberID:13803658152347727308 alarm:NOSPACE
 ```
 
 Removing excessive keyspace data and defragmenting the backend database will put the cluster back within the quota limits:
 
 ```sh
 # get current revision
-$ rev=$(ETCDCTL_API=3 etcdctl --endpoints=:2379 endpoint status --write-out="json" | egrep -o '"revision":[0-9]*' | egrep -o '[0-9]*')
+$ rev=$(ETCDCTL_API=3 etcdctl --endpoints=:2379 endpoint status --write-out="json" | egrep -o '"revision":[0-9]*' | egrep -o '[0-9].*')
 # compact away all old revisions
 $ ETCDCTL_API=3 etcdctl compact $rev
 compacted revision 1516
@@ -96,12 +143,16 @@ $ ETCDCTL_API=3 etcdctl defrag
 Finished defragmenting etcd member[127.0.0.1:2379]
 # disarm alarm
 $ ETCDCTL_API=3 etcdctl alarm disarm
-memberID:13803658152347727308 alarm:NOSPACE 
+memberID:13803658152347727308 alarm:NOSPACE
 # test puts are allowed again
 $ ETCDCTL_API=3 etcdctl put newkey 123
 OK
 ```
 
+The metric `etcd_mvcc_db_total_size_in_use_in_bytes` indicates the actual database usage after a history compaction, while `etcd_debugging_mvcc_db_total_size_in_bytes` shows the database size including free space waiting for defragmentation. The latter increases only when the former is close to it, meaning when both of these metrics are close to the quota, a history compaction is required to avoid triggering the space quota.
+
+`etcd_debugging_mvcc_db_total_size_in_bytes` is renamed to `etcd_mvcc_db_total_size_in_bytes` from v3.4.
+
 ## Snapshot backup
 
 Snapshotting the `etcd` cluster on a regular basis serves as a durable backup for an etcd keyspace. By taking periodic snapshots of an etcd member's backend database, an `etcd` cluster can be recovered to a point in time with a known good state.
@@ -116,5 +167,4 @@ $ etcdctl --write-out=table snapshot status backup.db
 +----------+----------+------------+------------+
 | fe01cf57 |       10 |          7 | 2.1 MB     |
 +----------+----------+------------+------------+
-
 ```

Documentation/op-guide/monitoring.md

@@ -20,14 +20,14 @@ Showing top 10 nodes out of 157 (cum >= 10ms)
     flat  flat%   sum%        cum   cum%
    130ms 27.08% 27.08%      130ms 27.08%  runtime.futex
     70ms 14.58% 41.67%       70ms 14.58%  syscall.Syscall
-    20ms  4.17% 45.83%       20ms  4.17%  github.com/coreos/etcd/cmd/vendor/golang.org/x/net/http2/hpack.huffmanDecode
+    20ms  4.17% 45.83%       20ms  4.17%  github.com/coreos/etcd/vendor/golang.org/x/net/http2/hpack.huffmanDecode
     20ms  4.17% 50.00%       30ms  6.25%  runtime.pcvalue
     20ms  4.17% 54.17%       50ms 10.42%  runtime.schedule
-    10ms  2.08% 56.25%       10ms  2.08%  github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/etcdserver.(*EtcdServer).AuthInfoFromCtx
-    10ms  2.08% 58.33%       10ms  2.08%  github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/etcdserver.(*EtcdServer).Lead
-    10ms  2.08% 60.42%       10ms  2.08%  github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/pkg/wait.(*timeList).Trigger
-    10ms  2.08% 62.50%       10ms  2.08%  github.com/coreos/etcd/cmd/vendor/github.com/prometheus/client_golang/prometheus.(*MetricVec).hashLabelValues
-    10ms  2.08% 64.58%       10ms  2.08%  github.com/coreos/etcd/cmd/vendor/golang.org/x/net/http2.(*Framer).WriteHeaders
+    10ms  2.08% 56.25%       10ms  2.08%  github.com/coreos/etcd/vendor/github.com/coreos/etcd/etcdserver.(*EtcdServer).AuthInfoFromCtx
+    10ms  2.08% 58.33%       10ms  2.08%  github.com/coreos/etcd/vendor/github.com/coreos/etcd/etcdserver.(*EtcdServer).Lead
+    10ms  2.08% 60.42%       10ms  2.08%  github.com/coreos/etcd/vendor/github.com/coreos/etcd/pkg/wait.(*timeList).Trigger
+    10ms  2.08% 62.50%       10ms  2.08%  github.com/coreos/etcd/vendor/github.com/prometheus/client_golang/prometheus.(*MetricVec).hashLabelValues
+    10ms  2.08% 64.58%       10ms  2.08%  github.com/coreos/etcd/vendor/golang.org/x/net/http2.(*Framer).WriteHeaders
 ```
 
 The `/debug/requests` endpoint gives gRPC traces and performance statistics through a web browser. For example, here is a `Range` request for the key `abc`:
@@ -43,7 +43,7 @@ When	Elapsed (s)
 
 ## Metrics endpoint
 
-Each etcd server exports metrics under the `/metrics` path on its client port and optionally on interfaces given by `--listen-metrics-urls`.
+Each etcd server exports metrics under the `/metrics` path on its client port and optionally on locations given by `--listen-metrics-urls`.
 
 The metrics can be fetched with `curl`:
 
@@ -59,6 +59,10 @@ etcd_disk_backend_commit_duration_seconds_bucket{le="0.016"} 406464
 ...
 ```
 
+## Health Check
+
+Since v3.3.0, in addition to responding to the `/metrics` endpoint, any locations specified by `--listen-metrics-urls` will also respond to the `/health` endpoint. This can be useful if the standard endpoint is configured with mutual (client) TLS authentication, but a load balancer or monitoring service still needs access to the health check.
+
 ## Prometheus
 
 Running a [Prometheus][prometheus] monitoring service is the easiest way to ingest and record etcd's metrics.

Documentation/op-guide/performance.md

@@ -4,7 +4,7 @@
 
 etcd provides stable, sustained high performance. Two factors define performance: latency and throughput. Latency is the time taken to complete an operation. Throughput is the total operations completed within some time period. Usually average latency increases as the overall throughput increases when etcd accepts concurrent client requests. In common cloud environments, like a standard `n-4` on Google Compute Engine (GCE) or a comparable machine type on AWS, a three member etcd cluster finishes a request in less than one millisecond under light load, and can complete more than 30,000 requests per second under heavy load.
 
-etcd uses the Raft consensus algorithm to replicate requests among members and reach agreement. Consensus performance, especially commit latency, is limited by two physical constraints: network IO latency and disk IO latency. The minimum time to finish an etcd request is the network Round Trip Time (RTT) between members, plus the time `fdatasync` requires to commit the data to permanant storage. The RTT within a datacenter may be as long as several hundred microseconds. A typical RTT within the United States is around 50ms, and can be as slow as 400ms between continents. The typical fdatasync latency for a spinning disk is about 10ms. For SSDs, the latency is often lower than 1ms. To increase throughput, etcd batches multiple requests together and submits them to Raft. This batching policy lets etcd attain high throughput despite heavy load.
+etcd uses the Raft consensus algorithm to replicate requests among members and reach agreement. Consensus performance, especially commit latency, is limited by two physical constraints: network IO latency and disk IO latency. The minimum time to finish an etcd request is the network Round Trip Time (RTT) between members, plus the time `fdatasync` requires to commit the data to permanent storage. The RTT within a datacenter may be as long as several hundred microseconds. A typical RTT within the United States is around 50ms, and can be as slow as 400ms between continents. The typical fdatasync latency for a spinning disk is about 10ms. For SSDs, the latency is often lower than 1ms. To increase throughput, etcd batches multiple requests together and submits them to Raft. This batching policy lets etcd attain high throughput despite heavy load.
 
 There are other sub-systems which impact the overall performance of etcd. Each serialized etcd request must run through etcd’s boltdb-backed MVCC storage engine, which usually takes tens of microseconds to finish. Periodically etcd incrementally snapshots its recently applied requests, merging them back with the previous on-disk snapshot. This process may lead to a latency spike. Although this is usually not a problem on SSDs, it may double the observed latency on HDD. Likewise, inflight compactions can impact etcd’s performance. Fortunately, the impact is often insignificant since the compaction is staggered so it does not compete for resources with regular requests. The RPC system, gRPC, gives etcd a well-defined, extensible API, but it also introduces additional latency, especially for local reads.
 

Documentation/op-guide/recovery.md

@@ -61,3 +61,9 @@ $ etcd \
 ```
 
 Now the restored etcd cluster should be available and serving the keyspace given by the snapshot.
+
+## Restoring a cluster from membership mis-reconfiguration with wrong URLs
+
+Previously, etcd panics on [membership mis-reconfiguration with wrong URLs](https://github.com/coreos/etcd/issues/9173) (v3.2.15 or later returns [error early in client-side](https://github.com/coreos/etcd/pull/9174) before etcd server panic).
+
+Recommended way is restore from [snapshot](#snapshotting-the-keyspace). `--force-new-cluster` can be used to overwrite cluster membership while keeping existing application data, but is strongly discouraged because it will panic if other members from previous cluster are still alive. Make sure to save snapshot periodically.

Documentation/op-guide/runtime-reconf-design.md

@@ -6,11 +6,11 @@ Read on to learn about the design of etcd's runtime reconfiguration commands and
 
 ## Two phase config changes keep the cluster safe
 
-In etcd, every runtime reconfiguration has to go through [two phases][add-member] for safety reasons. For example, to add a member, first inform cluster of new configuration and then start the new member.
+In etcd, every runtime reconfiguration has to go through [two phases][add-member] for safety reasons. For example, to add a member, first inform the cluster of the new configuration and then start the new member.
 
 Phase 1 - Inform cluster of new configuration
 
-To add a member into etcd cluster, make an API call to request a new member to be added to the cluster. This is the only way to add a new member into an existing cluster. The API call returns when the cluster agrees on the configuration change.
+To add a member into an etcd cluster, make an API call to request a new member to be added to the cluster. This is the only way to add a new member into an existing cluster. The API call returns when the cluster agrees on the configuration change.
 
 Phase 2 - Start new member
 
@@ -28,19 +28,19 @@ If a cluster permanently loses a majority of its members, a new cluster will nee
 
 It is entirely possible to force removing the failed members from the existing cluster to recover. However, we decided not to support this method since it bypasses the normal consensus committing phase, which is unsafe. If the member to remove is not actually dead or force removed through different members in the same cluster, etcd will end up with a diverged cluster with same clusterID. This is very dangerous and hard to debug/fix afterwards.
 
-With a correct deployment, the possibility of permanent majority lose is very low. But it is a severe enough problem that worth special care. We strongly suggest reading the [disaster recovery documentation][disaster-recovery] and preparing for permanent majority lose before putting etcd into production.
+With a correct deployment, the possibility of permanent majority loss is very low. But it is a severe enough problem that is worth special care. We strongly suggest reading the [disaster recovery documentation][disaster-recovery] and preparing for permanent majority loss before putting etcd into production.
 
 ## Do not use public discovery service for runtime reconfiguration
 
-The public discovery service should only be used for bootstrapping a cluster. To join member into an existing cluster, use runtime reconfiguration API.
+The public discovery service should only be used for bootstrapping a cluster. To join member into an existing cluster, use the runtime reconfiguration API.
 
-Discovery service is designed for bootstrapping an etcd cluster in the cloud environment, when the IP addresses of all the members are not known beforehand. After successfully bootstrapping a cluster, the IP addresses of all the members are known. Technically, the discovery service should no longer be needed.
+The discovery service is designed for bootstrapping an etcd cluster in a cloud environment, when the IP addresses of all the members are not known beforehand. After successfully bootstrapping a cluster, the IP addresses of all the members are known. Technically, the discovery service should no longer be needed.
 
 It seems that using public discovery service is a convenient way to do runtime reconfiguration, after all discovery service already has all the cluster configuration information. However relying on public discovery service brings troubles:
 
 1. it introduces external dependencies for the entire life-cycle of the cluster, not just bootstrap time. If there is a network issue between the cluster and public discovery service, the cluster will suffer from it.
 
-2. public discovery service must reflect correct runtime configuration of the cluster during it life-cycle. It has to provide security mechanism to avoid bad actions, and it is hard.
+2. public discovery service must reflect correct runtime configuration of the cluster during its life-cycle. It has to provide security mechanisms to avoid bad actions, and it is hard.
 
 3. public discovery service has to keep tens of thousands of cluster configurations. Our public discovery service backend is not ready for that workload.
 

Documentation/op-guide/security.md

@@ -38,6 +38,8 @@ The peer options work the same way as the client-to-server options:
 
 If either a client-to-server or peer certificate is supplied the key must also be set. All of these configuration options are also available through the environment variables, `ETCD_CA_FILE`, `ETCD_PEER_CA_FILE` and so on.
 
+`--cipher-suites`: Comma-separated list of supported TLS cipher suites between server/client and peers (empty will be auto-populated by Go). Available from v3.2.22+, v3.3.7+, and v3.4+.
+
 ## Example 1: Client-to-server transport security with HTTPS
 
 For this, have a CA certificate (`ca.crt`) and signed key pair (`server.crt`, `server.key`) ready.
@@ -122,6 +124,49 @@ And also the response from the server:
 }
 ```
 
+Specify cipher suites to block [weak TLS cipher suites](https://github.com/coreos/etcd/issues/8320).
+
+TLS handshake would fail when client hello is requested with invalid cipher suites.
+
+For instance:
+
+```bash
+$ etcd \
+  --cert-file ./server.crt \
+  --key-file ./server.key \
+  --trusted-ca-file ./ca.crt \
+  --cipher-suites TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+```
+
+Then, client requests must specify one of the cipher suites specified in the server:
+
+```bash
+# valid cipher suite
+$ curl \
+  --cacert ./ca.crt \
+  --cert ./server.crt \
+  --key ./server.key \
+  -L [CLIENT-URL]/metrics \
+  --ciphers ECDHE-RSA-AES128-GCM-SHA256
+
+# request succeeds
+etcd_server_version{server_version="3.2.22"} 1
+...
+```
+
+```bash
+# invalid cipher suite
+$ curl \
+  --cacert ./ca.crt \
+  --cert ./server.crt \
+  --key ./server.key \
+  -L [CLIENT-URL]/metrics \
+  --ciphers ECDHE-RSA-DES-CBC3-SHA
+
+# request fails with
+(35) error:14094410:SSL routines:ssl3_read_bytes:sslv3 alert handshake failure
+```
+
 ## Example 3: Transport security & client certificates in a cluster
 
 etcd supports the same model as above for **peer communication**, that means the communication between etcd members in a cluster.
@@ -195,9 +240,9 @@ When client authentication is enabled for an etcd member, the administrator must
 
 ## Notes for TLS authentication
 
-Since [v3.2.0](https://github.com/coreos/etcd/blob/master/CHANGELOG.md#v320-2017-06-09), [TLS certificates get reloaded on every client connection](https://github.com/coreos/etcd/pull/7829). This is useful when replacing expiry certs without stopping etcd servers; it can be done by overwriting old certs with new ones. Refreshing certs for every connection should not have too much overhead, but can be improved in the future, with caching layer. Example tests can be found [here](https://github.com/coreos/etcd/blob/b041ce5d514a4b4aaeefbffb008f0c7570a18986/integration/v3_grpc_test.go#L1601-L1757).
+Since [v3.2.0](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.2.md#v320-2017-06-09), [TLS certificates get reloaded on every client connection](https://github.com/coreos/etcd/pull/7829). This is useful when replacing expiry certs without stopping etcd servers; it can be done by overwriting old certs with new ones. Refreshing certs for every connection should not have too much overhead, but can be improved in the future, with caching layer. Example tests can be found [here](https://github.com/coreos/etcd/blob/b041ce5d514a4b4aaeefbffb008f0c7570a18986/integration/v3_grpc_test.go#L1601-L1757).
 
-Since [v3.2.0](https://github.com/coreos/etcd/blob/master/CHANGELOG.md#v320-2017-06-09), [server denies incoming peer certs with wrong IP `SAN`](https://github.com/coreos/etcd/pull/7687). For instance, if peer cert contains IP addresses in Subject Alternative Name (SAN) field, server authenticates a peer only when the remote IP address matches one of those IP addresses. This is to prevent unauthorized endpoints from joining the cluster. For example, peer B's CSR (with `cfssl`) is:
+Since [v3.2.0](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.2.md#v320-2017-06-09), [server denies incoming peer certs with wrong IP `SAN`](https://github.com/coreos/etcd/pull/7687). For instance, if peer cert contains any IP addresses in Subject Alternative Name (SAN) field, server authenticates a peer only when the remote IP address matches one of those IP addresses. This is to prevent unauthorized endpoints from joining the cluster. For example, peer B's CSR (with `cfssl`) is:
 
 ```json
 {
@@ -223,50 +268,133 @@ Since [v3.2.0](https://github.com/coreos/etcd/blob/master/CHANGELOG.md#v320-2017
 
 when peer B's actual IP address is `10.138.0.2`, not `10.138.0.27`. When peer B tries to join the cluster, peer A will reject B with the error `x509: certificate is valid for 10.138.0.27, not 10.138.0.2`, because B's remote IP address does not match the one in Subject Alternative Name (SAN) field.
 
-Since [v3.2.0](https://github.com/coreos/etcd/blob/master/CHANGELOG.md#v320-2017-06-09), [server resolves TLS `DNSNames` when checking `SAN`](https://github.com/coreos/etcd/pull/7767). For instance, if peer cert contains any DNS names in Subject Alternative Name (SAN) field, server authenticates a peer only when forward-lookups on those DNS names have matching IP with the remote IP address. For example, peer B's CSR (with `cfssl`) is:
+Since [v3.2.0](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.2.md#v320-2017-06-09), [server resolves TLS `DNSNames` when checking `SAN`](https://github.com/coreos/etcd/pull/7767). For instance, if peer cert contains only DNS names (no IP addresses) in Subject Alternative Name (SAN) field, server authenticates a peer only when forward-lookups (`dig b.com`) on those DNS names have matching IP with the remote IP address. For example, peer B's CSR (with `cfssl`) is:
 
 ```json
 {
-  ...
+  "CN": "etcd peer",
   "hosts": [
     "b.com"
   ],
-  ...
-}
 ```
 
 when peer B's remote IP address is `10.138.0.2`. When peer B tries to join the cluster, peer A looks up the incoming host `b.com` to get the list of IP addresses (e.g. `dig b.com`). And rejects B if the list does not contain the IP `10.138.0.2`, with the error `tls: 10.138.0.2 does not match any of DNSNames ["b.com"]`.
 
-Since [v3.2.2](https://github.com/coreos/etcd/blob/master/CHANGELOG.md#v322-2017-07-07), [server accepts connections if IP matches, without checking DNS entries](https://github.com/coreos/etcd/pull/8223). For instance, if peer cert contains IP addresses and DNS names in Subject Alternative Name (SAN) field, and the remote IP address matches one of those IP addresses, server just accepts connection without further checking the DNS names. For example, peer B's CSR (with `cfssl`) is:
+Since [v3.2.2](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.2.md#v322-2017-07-07), [server accepts connections if IP matches, without checking DNS entries](https://github.com/coreos/etcd/pull/8223). For instance, if peer cert contains IP addresses and DNS names in Subject Alternative Name (SAN) field, and the remote IP address matches one of those IP addresses, server just accepts connection without further checking the DNS names. For example, peer B's CSR (with `cfssl`) is:
 
 ```json
 {
-  ...
+  "CN": "etcd peer",
   "hosts": [
     "invalid.domain",
     "10.138.0.2"
   ],
-  ...
-}
 ```
 
 when peer B's remote IP address is `10.138.0.2` and `invalid.domain` is a invalid host. When peer B tries to join the cluster, peer A successfully authenticates B, since Subject Alternative Name (SAN) field has a valid matching IP address. See [issue#8206](https://github.com/coreos/etcd/issues/8206) for more detail.
 
-Since [v3.2.5](https://github.com/coreos/etcd/blob/master/CHANGELOG.md#v325-2017-08-04), [server supports reverse-lookup on wildcard DNS `SAN`](https://github.com/coreos/etcd/pull/8281). For instance, if peer cert contains only DNS names (no IP addresses) in Subject Alternative Name (SAN) field, server first reverse-lookups the remote IP address to get a list of names mapping to that address (e.g. `nslookup IPADDR`). Then accepts the connection if those names have a matching name with peer cert's DNS names (either by exact or wildcard match). If none is matched, server forward-lookups each DNS entry in peer cert (e.g. look up `example.default.svc` when the entry is `*.example.default.svc`), and accepts connection only when the host's resolved addresses have the matching IP address with the peer's remote IP address. For example, peer B's CSR (with `cfssl`) is:
+Since [v3.2.5](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.2.md#v325-2017-08-04), [server supports reverse-lookup on wildcard DNS `SAN`](https://github.com/coreos/etcd/pull/8281). For instance, if peer cert contains only DNS names (no IP addresses) in Subject Alternative Name (SAN) field, server first reverse-lookups the remote IP address to get a list of names mapping to that address (e.g. `nslookup IPADDR`). Then accepts the connection if those names have a matching name with peer cert's DNS names (either by exact or wildcard match). If none is matched, server forward-lookups each DNS entry in peer cert (e.g. look up `example.default.svc` when the entry is `*.example.default.svc`), and accepts connection only when the host's resolved addresses have the matching IP address with the peer's remote IP address. For example, peer B's CSR (with `cfssl`) is:
 
 ```json
 {
-  ...
+  "CN": "etcd peer",
   "hosts": [
     "*.example.default.svc",
     "*.example.default.svc.cluster.local"
   ],
-  ...
-}
 ```
 
 when peer B's remote IP address is `10.138.0.2`. When peer B tries to join the cluster, peer A reverse-lookup the IP `10.138.0.2` to get the list of host names. And either exact or wildcard match the host names with peer B's cert DNS names in Subject Alternative Name (SAN) field. If none of reverse/forward lookups worked, it returns an error `"tls: "10.138.0.2" does not match any of DNSNames ["*.example.default.svc","*.example.default.svc.cluster.local"]`. See [issue#8268](https://github.com/coreos/etcd/issues/8268) for more detail.
 
+[v3.3.0](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.3.md) adds [`etcd --peer-cert-allowed-cn`](https://github.com/coreos/etcd/pull/8616) flag to support [CN(Common Name)-based auth for inter-peer connections](https://github.com/coreos/etcd/issues/8262). Kubernetes TLS bootstrapping involves generating dynamic certificates for etcd members and other system components (e.g. API server, kubelet, etc.). Maintaining different CAs for each component provides tighter access control to etcd cluster but often tedious. When `--peer-cert-allowed-cn` flag is specified, node can only join with matching common name even with shared CAs. For example, each member in 3-node cluster is set up with CSRs (with `cfssl`) as below:
+
+```json
+{
+  "CN": "etcd.local",
+  "hosts": [
+    "m1.etcd.local",
+    "127.0.0.1",
+    "localhost"
+  ],
+```
+
+```json
+{
+  "CN": "etcd.local",
+  "hosts": [
+    "m2.etcd.local",
+    "127.0.0.1",
+    "localhost"
+  ],
+```
+
+```json
+{
+  "CN": "etcd.local",
+  "hosts": [
+    "m3.etcd.local",
+    "127.0.0.1",
+    "localhost"
+  ],
+```
+
+Then only peers with matching common names will be authenticated if `--peer-cert-allowed-cn etcd.local` is given. And nodes with different CNs in CSRs or different `--peer-cert-allowed-cn` will be rejected:
+
+```bash
+$ etcd --peer-cert-allowed-cn m1.etcd.local
+
+I | embed: rejected connection from "127.0.0.1:48044" (error "CommonName authentication failed", ServerName "m1.etcd.local")
+I | embed: rejected connection from "127.0.0.1:55702" (error "remote error: tls: bad certificate", ServerName "m3.etcd.local")
+```
+
+Each process should be started with:
+
+```bash
+etcd --peer-cert-allowed-cn etcd.local
+
+I | pkg/netutil: resolving m3.etcd.local:32380 to 127.0.0.1:32380
+I | pkg/netutil: resolving m2.etcd.local:22380 to 127.0.0.1:22380
+I | pkg/netutil: resolving m1.etcd.local:2380 to 127.0.0.1:2380
+I | etcdserver: published {Name:m3 ClientURLs:[https://m3.etcd.local:32379]} to cluster 9db03f09b20de32b
+I | embed: ready to serve client requests
+I | etcdserver: published {Name:m1 ClientURLs:[https://m1.etcd.local:2379]} to cluster 9db03f09b20de32b
+I | embed: ready to serve client requests
+I | etcdserver: published {Name:m2 ClientURLs:[https://m2.etcd.local:22379]} to cluster 9db03f09b20de32b
+I | embed: ready to serve client requests
+I | embed: serving client requests on 127.0.0.1:32379
+I | embed: serving client requests on 127.0.0.1:22379
+I | embed: serving client requests on 127.0.0.1:2379
+```
+
+[v3.2.19](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.2.md) and [v3.3.4](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.3.md) fixes TLS reload when [certificate SAN field only includes IP addresses but no domain names](https://github.com/coreos/etcd/issues/9541). For example, a member is set up with CSRs (with `cfssl`) as below:
+
+```json
+{
+  "CN": "etcd.local",
+  "hosts": [
+    "127.0.0.1"
+  ],
+```
+
+In Go, server calls `(*tls.Config).GetCertificate` for TLS reload if and only if server's `(*tls.Config).Certificates` field is not empty, or `(*tls.ClientHelloInfo).ServerName` is not empty with a valid SNI from the client. Previously, etcd always populates `(*tls.Config).Certificates` on the initial client TLS handshake, as non-empty. Thus, client was always expected to supply a matching SNI in order to pass the TLS verification and to trigger `(*tls.Config).GetCertificate` to reload TLS assets.
+
+However, a certificate whose SAN field does [not include any domain names but only IP addresses](https://github.com/coreos/etcd/issues/9541) would request `*tls.ClientHelloInfo` with an empty `ServerName` field, thus failing to trigger the TLS reload on initial TLS handshake; this becomes a problem when expired certificates need to be replaced online.
+
+Now, `(*tls.Config).Certificates` is created empty on initial TLS client handshake, first to trigger `(*tls.Config).GetCertificate`, and then to populate rest of the certificates on every new TLS connection, even when client SNI is empty (e.g. cert only includes IPs).
+
+## Notes for Host Whitelist
+
+`etcd --host-whitelist` flag specifies acceptable hostnames from HTTP client requests. Client origin policy protects against ["DNS Rebinding"](https://en.wikipedia.org/wiki/DNS_rebinding) attacks to insecure etcd servers. That is, any website can simply create an authorized DNS name, and direct DNS to `"localhost"` (or any other address). Then, all HTTP endpoints of etcd server listening on `"localhost"` becomes accessible, thus vulnerable to DNS rebinding attacks. See [CVE-2018-5702](https://bugs.chromium.org/p/project-zero/issues/detail?id=1447#c2) for more detail.
+
+Client origin policy works as follows:
+
+1. If client connection is secure via HTTPS, allow any hostnames.
+2. If client connection is not secure and `"HostWhitelist"` is not empty, only allow HTTP requests whose Host field is listed in whitelist.
+
+Note that the client origin policy is enforced whether authentication is enabled or not, for tighter controls.
+
+By default, `etcd --host-whitelist` and `embed.Config.HostWhitelist` are set *empty* to allow all hostnames. Note that when specifying hostnames, loopback addresses are not added automatically. To allow loopback interfaces, add them to whitelist manually (e.g. `"localhost"`, `"127.0.0.1"`, etc.).
+
 ## Frequently asked questions
 
 ### I'm seeing a SSLv3 alert handshake failure when using TLS client authentication?

Documentation/tuning.md

@@ -75,8 +75,8 @@ These errors may be resolved by prioritizing etcd's peer traffic over its client
 tc qdisc add dev eth0 root handle 1: prio bands 3
 tc filter add dev eth0 parent 1: protocol ip prio 1 u32 match ip sport 2380 0xffff flowid 1:1
 tc filter add dev eth0 parent 1: protocol ip prio 1 u32 match ip dport 2380 0xffff flowid 1:1
-tc filter add dev eth0 parent 1: protocol ip prio 2 u32 match ip sport 2739 0xffff flowid 1:1
-tc filter add dev eth0 parent 1: protocol ip prio 2 u32 match ip dport 2739 0xffff flowid 1:1
+tc filter add dev eth0 parent 1: protocol ip prio 2 u32 match ip sport 2379 0xffff flowid 1:1
+tc filter add dev eth0 parent 1: protocol ip prio 2 u32 match ip dport 2379 0xffff flowid 1:1
 ```
 
 [ping]: https://en.wikipedia.org/wiki/Ping_(networking_utility)

Documentation/upgrades/upgrade_3_0.md

@@ -8,6 +8,8 @@ Before [starting an upgrade](#upgrade-procedure), read through the rest of this
 
 ### Upgrade checklists
 
+**NOTE:** When [migrating from v2 with no v3 data](https://github.com/coreos/etcd/issues/9480), etcd server v3.2+ panics when etcd restores from existing snapshots but no v3 `ETCD_DATA_DIR/member/snap/db` file. This happens when the server had migrated from v2 with no previous v3 data. This also prevents accidental v3 data loss (e.g. `db` file might have been moved). etcd requires that post v3 migration can only happen with v3 data. Do not upgrade to newer v3 versions until v3.0 server contains v3 data.
+
 #### Upgrade requirements
 
 To upgrade an existing etcd deployment to 3.0, the running cluster must be 2.3 or greater. If it's before 2.3, please upgrade to [2.3](https://github.com/coreos/etcd/releases/tag/v2.3.8) before upgrading to 3.0.

Documentation/upgrades/upgrade_3_1.md

@@ -8,6 +8,8 @@ Before [starting an upgrade](#upgrade-procedure), read through the rest of this
 
 ### Upgrade checklists
 
+**NOTE:** When [migrating from v2 with no v3 data](https://github.com/coreos/etcd/issues/9480), etcd server v3.2+ panics when etcd restores from existing snapshots but no v3 `ETCD_DATA_DIR/member/snap/db` file. This happens when the server had migrated from v2 with no previous v3 data. This also prevents accidental v3 data loss (e.g. `db` file might have been moved). etcd requires that post v3 migration can only happen with v3 data. Do not upgrade to newer v3 versions until v3.0 server contains v3 data.
+
 #### Monitoring
 
 Following metrics from v3.0.x have been deprecated in favor of [go-grpc-prometheus](https://github.com/grpc-ecosystem/go-grpc-prometheus):

Documentation/upgrades/upgrade_3_2.md

@@ -8,13 +8,21 @@ Before [starting an upgrade](#upgrade-procedure), read through the rest of this
 
 ### Upgrade checklists
 
+**NOTE:** When [migrating from v2 with no v3 data](https://github.com/coreos/etcd/issues/9480), etcd server v3.2+ panics when etcd restores from existing snapshots but no v3 `ETCD_DATA_DIR/member/snap/db` file. This happens when the server had migrated from v2 with no previous v3 data. This also prevents accidental v3 data loss (e.g. `db` file might have been moved). etcd requires that post v3 migration can only happen with v3 data. Do not upgrade to newer v3 versions until v3.0 server contains v3 data.
+
 Highlighted breaking changes in 3.2.
 
-#### Change in gRPC dependency (>=3.2.10)
+#### Changed default `snapshot-count` value
+
+Higher `--snapshot-count` holds more Raft entries in memory until snapshot, thus causing [recurrent higher memory usage](https://github.com/kubernetes/kubernetes/issues/60589#issuecomment-371977156). Since leader retains latest Raft entries for longer, a slow follower has more time to catch up before leader snapshot. `--snapshot-count` is a tradeoff between higher memory usage and better availabilities of slow followers.
+
+Since v3.2, the default value of `--snapshot-count` has [changed from from 10,000 to 100,000](https://github.com/coreos/etcd/pull/7160).
+
+#### Changed gRPC dependency (>=3.2.10)
 
 3.2.10 or later now requires [grpc/grpc-go](https://github.com/grpc/grpc-go/releases) `v1.7.5` (<=3.2.9 requires `v1.2.1`).
 
-##### Deprecate `grpclog.Logger`
+##### Deprecated `grpclog.Logger`
 
 `grpclog.Logger` has been deprecated in favor of [`grpclog.LoggerV2`](https://github.com/grpc/grpc-go/blob/master/grpclog/loggerv2.go). `clientv3.Logger` is now `grpclog.LoggerV2`.
 
@@ -35,7 +43,7 @@ clientv3.SetLogger(grpclog.NewLoggerV2(os.Stderr, os.Stderr, os.Stderr))
 // log.New above cannot be used (not implement grpclog.LoggerV2 interface)
 ```
 
-##### Deprecate `grpc.ErrClientConnTimeout`
+##### Deprecated `grpc.ErrClientConnTimeout`
 
 Previously, `grpc.ErrClientConnTimeout` error is returned on client dial time-outs. 3.2 instead returns `context.DeadlineExceeded` (see [#8504](https://github.com/coreos/etcd/issues/8504)).
 
@@ -64,9 +72,9 @@ if err == context.DeadlineExceeded {
 }
 ```
 
-#### Change in maximum request size limits (>=3.2.10)
+#### Changed maximum request size limits (>=3.2.10)
 
-3.2.10 and 3.2.11 allow custom request size limits in server side. >=3.2.12 allows custom request size limits for both server and **client side**.
+3.2.10 and 3.2.11 allow custom request size limits in server side. >=3.2.12 allows custom request size limits for both server and **client side**. In previous versions(v3.2.10, v3.2.11), client response size was limited to only 4 MiB.
 
 Server-side request limits can be configured with `--max-request-bytes` flag:
 
@@ -137,7 +145,7 @@ err.Error() == "rpc error: code = ResourceExhausted desc = grpc: received messag
 
 **If not specified, client-side send limit defaults to 2 MiB (1.5 MiB + gRPC overhead bytes) and receive limit to `math.MaxInt32`**. Please see [clientv3 godoc](https://godoc.org/github.com/coreos/etcd/clientv3#Config) for more detail.
 
-#### Change in raw gRPC client wrappers
+#### Changed raw gRPC client wrappers
 
 3.2.12 or later changes the function signatures of `clientv3` gRPC client wrapper. This change was needed to support [custom `grpc.CallOption` on message size limits](https://github.com/coreos/etcd/pull/9047).
 
@@ -160,13 +168,7 @@ Before and after
 +func NewWatchFromWatchClient(wc pb.WatchClient, c *Client) Watcher {
 ```
 
-#### Change in `--listen-peer-urls` and `--listen-client-urls`
-
-3.2 now rejects domains names for `--listen-peer-urls` and `--listen-client-urls` (3.1 only prints out warnings), since domain name is invalid for network interface binding. Make sure that those URLs are properly formated as `scheme://IP:port`.
-
-See [issue #6336](https://github.com/coreos/etcd/issues/6336) for more contexts.
-
-#### Change in `clientv3.Lease.TimeToLive` API
+#### Changed `clientv3.Lease.TimeToLive` API
 
 Previously, `clientv3.Lease.TimeToLive` API returned `lease.ErrLeaseNotFound` on non-existent lease ID. 3.2 instead returns TTL=-1 in its response and no error (see [#7305](https://github.com/coreos/etcd/pull/7305)).
 
@@ -188,7 +190,7 @@ resp.TTL == -1
 err == nil
 ```
 
-#### Change in `clientv3.NewFromConfigFile`
+#### Moved `clientv3.NewFromConfigFile` to `clientv3.yaml.NewConfig`
 
 `clientv3.NewFromConfigFile` is moved to `yaml.NewConfig`.
 
@@ -206,6 +208,12 @@ import clientv3yaml "github.com/coreos/etcd/clientv3/yaml"
 clientv3yaml.NewConfig
 ```
 
+#### Change in `--listen-peer-urls` and `--listen-client-urls`
+
+3.2 now rejects domains names for `--listen-peer-urls` and `--listen-client-urls` (3.1 only prints out warnings), since domain name is invalid for network interface binding. Make sure that those URLs are properly formated as `scheme://IP:port`.
+
+See [issue #6336](https://github.com/coreos/etcd/issues/6336) for more contexts.
+
 ### Server upgrade checklists
 
 #### Upgrade requirements

Documentation/upgrades/upgrade_3_3.md

@@ -8,9 +8,24 @@ Before [starting an upgrade](#upgrade-procedure), read through the rest of this
 
 ### Upgrade checklists
 
+**NOTE:** When [migrating from v2 with no v3 data](https://github.com/coreos/etcd/issues/9480), etcd server v3.2+ panics when etcd restores from existing snapshots but no v3 `ETCD_DATA_DIR/member/snap/db` file. This happens when the server had migrated from v2 with no previous v3 data. This also prevents accidental v3 data loss (e.g. `db` file might have been moved). etcd requires that post v3 migration can only happen with v3 data. Do not upgrade to newer v3 versions until v3.0 server contains v3 data.
+
 Highlighted breaking changes in 3.3.
 
-#### Change in `etcdserver.EtcdServer` struct
+#### Changed value type of `etcd --auto-compaction-retention` flag to `string`
+
+Changed `--auto-compaction-retention` flag to [accept string values](https://github.com/coreos/etcd/pull/8563) with [finer granularity](https://github.com/coreos/etcd/issues/8503). Now that `--auto-compaction-retention` accepts string values, etcd configuration YAML file `auto-compaction-retention` field must be changed to `string` type. Previously, `--config-file etcd.config.yaml` can have `auto-compaction-retention: 24` field, now must be `auto-compaction-retention: "24"` or `auto-compaction-retention: "24h"`. If configured as `--auto-compaction-mode periodic --auto-compaction-retention "24h"`, the time duration value for `--auto-compaction-retention` flag must be valid for [`time.ParseDuration`](https://golang.org/pkg/time/#ParseDuration) function in Go.
+
+```diff
+# etcd.config.yaml
++auto-compaction-mode: periodic
+-auto-compaction-retention: 24
++auto-compaction-retention: "24"
++# Or
++auto-compaction-retention: "24h"
+```
+
+#### Changed `etcdserver.EtcdServer.ServerConfig` to `*etcdserver.EtcdServer.ServerConfig`
 
 `etcdserver.EtcdServer` has changed the type of its member field `*etcdserver.ServerConfig` to `etcdserver.ServerConfig`. And `etcdserver.NewServer` now takes `etcdserver.ServerConfig`, instead of `*etcdserver.ServerConfig`.
 
@@ -40,7 +55,9 @@ func (e *EtcdServer) Start() error {
     ...
 ```
 
-#### Change in `embed.EtcdServer` struct
+#### Added `embed.Config.LogOutput` struct
+
+**Note that this field has been renamed to `embed.Config.LogOutputs` in `[]string` type in v3.4. Please see [v3.4 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_4.md) for more details.**
 
 Field `LogOutput` is added to `embed.Config`:
 
@@ -63,6 +80,8 @@ WARNING: 2017/11/02 11:35:51 grpc: addrConn.resetTransport failed to create clie
 
 From v3.3, gRPC server logs are disabled by default.
 
+**Note that `embed.Config.SetupLogging` method has been deprecated in v3.4. Please see [v3.4 upgrade guide](https://github.com/coreos/etcd/blob/master/Documentation/upgrades/upgrade_3_4.md) for more details.**
+
 ```go
 import "github.com/coreos/etcd/embed"
 
@@ -72,48 +91,38 @@ cfg.SetupLogging()
 
 Set `embed.Config.Debug` field to `true` to enable gRPC server logs.
 
-#### Change in `/health` endpoint response value
+#### Changed `/health` endpoint response
 
-Previously, `[endpoint]:[client-port]/health` returned manually marshaled JSON value. 3.3 instead defines [`etcdhttp.Health`](https://godoc.org/github.com/coreos/etcd/etcdserver/api/etcdhttp#Health) struct and returns properly encoded JSON value with errors, if any.
+Previously, `[endpoint]:[client-port]/health` returned manually marshaled JSON value. 3.3 now defines [`etcdhttp.Health`](https://godoc.org/github.com/coreos/etcd/etcdserver/api/etcdhttp#Health) struct.
 
-Before
+Note that in v3.3.0-rc.0, v3.3.0-rc.1, and v3.3.0-rc.2, `etcdhttp.Health` has boolean type `"health"` and `"errors"` fields. For backward compatibilities, we reverted `"health"` field to `string` type and removed `"errors"` field. Further health information will be provided in separate APIs.
 
 ```bash
 $ curl http://localhost:2379/health
-{"health": "true"}
+{"health":"true"}
 ```
 
-After
-
-```bash
-$ curl http://localhost:2379/health
-{"health":true}
-
-# Or
-{"health":false,"errors":["NOSPACE"]}
-```
-
-#### Change in gRPC gateway HTTP endpoints (replaced `/v3alpha` with `/v3beta`)
+#### Changed gRPC gateway HTTP endpoints (replaced `/v3alpha` with `/v3beta`)
 
 Before
 
 ```bash
 curl -L http://localhost:2379/v3alpha/kv/put \
-	-X POST -d '{"key": "Zm9v", "value": "YmFy"}'
+  -X POST -d '{"key": "Zm9v", "value": "YmFy"}'
 ```
 
 After
 
 ```bash
 curl -L http://localhost:2379/v3beta/kv/put \
-	-X POST -d '{"key": "Zm9v", "value": "YmFy"}'
+  -X POST -d '{"key": "Zm9v", "value": "YmFy"}'
 ```
 
 Requests to `/v3alpha` endpoints will redirect to `/v3beta`, and `/v3alpha` will be removed in 3.4 release.
 
-#### Change in maximum request size limits
+#### Changed maximum request size limits
 
-3.3 now allows custom request size limits for both server and **client side**.
+3.3 now allows custom request size limits for both server and **client side**. In previous versions(v3.2.10, v3.2.11), client response size was limited to only 4 MiB.
 
 Server-side request limits can be configured with `--max-request-bytes` flag:
 
@@ -184,7 +193,7 @@ err.Error() == "rpc error: code = ResourceExhausted desc = grpc: received messag
 
 **If not specified, client-side send limit defaults to 2 MiB (1.5 MiB + gRPC overhead bytes) and receive limit to `math.MaxInt32`**. Please see [clientv3 godoc](https://godoc.org/github.com/coreos/etcd/clientv3#Config) for more detail.
 
-#### Change in raw gRPC client wrappers
+#### Changed raw gRPC client wrapper function signatures
 
 3.3 changes the function signatures of `clientv3` gRPC client wrapper. This change was needed to support [custom `grpc.CallOption` on message size limits](https://github.com/coreos/etcd/pull/9047).
 
@@ -207,7 +216,7 @@ Before and after
 +func NewWatchFromWatchClient(wc pb.WatchClient, c *Client) Watcher {
 ```
 
-#### Change in clientv3 `Snapshot` API error type
+#### Changed clientv3 `Snapshot` API error type
 
 Previously, clientv3 `Snapshot` API returned raw [`grpc/*status.statusError`] type error. v3.3 now translates those errors to corresponding public error types, to be consistent with other APIs.
 
@@ -253,7 +262,7 @@ _, err = io.Copy(f, rc)
 err == context.DeadlineExceeded
 ```
 
-#### Change in `etcdctl lease timetolive` command output
+#### Changed `etcdctl lease timetolive` command output
 
 Previously, `lease timetolive LEASE_ID` command on expired lease prints `-1s` for remaining seconds. 3.3 now outputs clearer messages.
 
@@ -270,7 +279,7 @@ After
 lease 2d8257079fa1bc0c already expired
 ```
 
-#### Change in `golang.org/x/net/context` imports
+#### Changed `golang.org/x/net/context` imports
 
 `clientv3` has deprecated `golang.org/x/net/context`. If a project vendors `golang.org/x/net/context` in other code (e.g. etcd generated protocol buffer code) and imports `github.com/coreos/etcd/clientv3`, it requires Go 1.9+ to compile.
 
@@ -288,11 +297,11 @@ import "context"
 cli.Put(context.Background(), "f", "v")
 ```
 
-#### Change in gRPC dependency
+#### Changed gRPC dependency
 
 3.3 now requires [grpc/grpc-go](https://github.com/grpc/grpc-go/releases) `v1.7.5`.
 
-##### Deprecate `grpclog.Logger`
+##### Deprecated `grpclog.Logger`
 
 `grpclog.Logger` has been deprecated in favor of [`grpclog.LoggerV2`](https://github.com/grpc/grpc-go/blob/master/grpclog/loggerv2.go). `clientv3.Logger` is now `grpclog.LoggerV2`.
 
@@ -313,7 +322,7 @@ clientv3.SetLogger(grpclog.NewLoggerV2(os.Stderr, os.Stderr, os.Stderr))
 // log.New above cannot be used (not implement grpclog.LoggerV2 interface)
 ```
 
-##### Deprecate `grpc.ErrClientConnTimeout`
+##### Deprecated `grpc.ErrClientConnTimeout`
 
 Previously, `grpc.ErrClientConnTimeout` error is returned on client dial time-outs. 3.3 instead returns `context.DeadlineExceeded` (see [#8504](https://github.com/coreos/etcd/issues/8504)).
 
@@ -342,7 +351,7 @@ if err == context.DeadlineExceeded {
 }
 ```
 
-#### Change in official container registry
+#### Changed official container registry
 
 etcd now uses [`gcr.io/etcd-development/etcd`](https://gcr.io/etcd-development/etcd) as a primary container registry, and [`quay.io/coreos/etcd`](https://quay.io/coreos/etcd) as secondary.
 

Documentation/upgrades/upgrade_3_4.md

@@ -0,0 +1,460 @@
+## Upgrade etcd from 3.3 to 3.4
+
+In the general case, upgrading from etcd 3.3 to 3.4 can be a zero-downtime, rolling upgrade:
+ - one by one, stop the etcd v3.3 processes and replace them with etcd v3.4 processes
+ - after running all v3.4 processes, new features in v3.4 are available to the cluster
+
+Before [starting an upgrade](#upgrade-procedure), read through the rest of this guide to prepare.
+
+
+
+### Upgrade checklists
+
+**NOTE:** When [migrating from v2 with no v3 data](https://github.com/coreos/etcd/issues/9480), etcd server v3.2+ panics when etcd restores from existing snapshots but no v3 `ETCD_DATA_DIR/member/snap/db` file. This happens when the server had migrated from v2 with no previous v3 data. This also prevents accidental v3 data loss (e.g. `db` file might have been moved). etcd requires that post v3 migration can only happen with v3 data. Do not upgrade to newer v3 versions until v3.0 server contains v3 data.
+
+Highlighted breaking changes in 3.4.
+
+#### Make `ETCDCTL_API=3 etcdctl` default
+
+`ETCDCTL_API=3` is now the default.
+
+```diff
+etcdctl set foo bar
+Error: unknown command "set" for "etcdctl"
+
+-etcdctl set foo bar
++ETCDCTL_API=2 etcdctl set foo bar
+bar
+
+ETCDCTL_API=3 etcdctl put foo bar
+OK
+
+-ETCDCTL_API=3 etcdctl put foo bar
++etcdctl put foo bar
+```
+
+#### Deprecated `etcd --ca-file` and `etcd --peer-ca-file` flags
+
+`--ca-file` and `--peer-ca-file` flags are deprecated; they have been deprecated since v2.1.
+
+```diff
+-etcd --ca-file ca-client.crt
++etcd --trusted-ca-file ca-client.crt
+```
+
+```diff
+-etcd --peer-ca-file ca-peer.crt
++etcd --peer-trusted-ca-file ca-peer.crt
+```
+
+#### Promote `etcd_debugging_mvcc_db_total_size_in_bytes` Prometheus metrics
+
+v3.4 promotes `etcd_debugging_mvcc_db_total_size_in_bytes` Prometheus metrics to `etcd_mvcc_db_total_size_in_bytes`, in order to encourage etcd storage monitoring.
+
+`etcd_debugging_mvcc_db_total_size_in_bytes` is still served in v3.4 for backward compatibilities. It will be completely deprecated in v3.5.
+
+```diff
+-etcd_debugging_mvcc_db_total_size_in_bytes
++etcd_mvcc_db_total_size_in_bytes
+```
+
+Note that `etcd_debugging_*` namespace metrics have been marked as experimental. As we improve monitoring guide, we will promote more metrics.
+
+#### Deprecating `etcd --log-output` flag (now `--log-outputs`)
+
+Rename [`etcd --log-output` to `--log-outputs`](https://github.com/coreos/etcd/pull/9624) to support multiple log outputs. **`etcd --logger=capnslog` does not support multiple log outputs.**
+
+**`etcd --log-output`** will be deprecated in v3.5. **`etcd --logger=capnslog` will be deprecated in v3.5**.
+
+```diff
+-etcd --log-output=stderr
++etcd --log-outputs=stderr
+
++# to write logs to stderr and a.log file at the same time
++# only "--logger=zap" supports multiple writers
++etcd --logger=zap --log-outputs=stderr,a.log
+```
+
+v3.4 adds `etcd --logger=zap --log-outputs=stderr` support for structured logging and multiple log outputs. Main motivation is to promote automated etcd monitoring, rather than looking back server logs when it starts breaking. Future development will make etcd log as few as possible, and make etcd easier to monitor with metrics and alerts. **`etcd --logger=capnslog` will be deprecated in v3.5**.
+
+#### Changed `log-outputs` field type in `etcd --config-file` to `[]string`
+
+Now that `log-outputs` (old field name `log-output`) accepts multiple writers, etcd configuration YAML file `log-outputs` field must be changed to `[]string` type as below:
+
+```diff
+ # Specify 'stdout' or 'stderr' to skip journald logging even when running under systemd.
+-log-output: default
++log-outputs: [default]
+```
+
+#### Renamed `embed.Config.LogOutput` to `embed.Config.LogOutputs`
+
+Renamed [**`embed.Config.LogOutput`** to **`embed.Config.LogOutputs`**](https://github.com/coreos/etcd/pull/9624) to support multiple log outputs. And changed [`embed.Config.LogOutput` type from `string` to `[]string`](https://github.com/coreos/etcd/pull/9579) to support multiple log outputs.
+
+```diff
+import "github.com/coreos/etcd/embed"
+
+cfg := &embed.Config{Debug: false}
+-cfg.LogOutput = "stderr"
++cfg.LogOutputs = []string{"stderr"}
+```
+
+#### v3.5 deprecates `capnslog`
+
+**v3.5 will deprecate `etcd --log-package-levels` flag for `capnslog`**; `etcd --logger=zap --log-outputs=stderr` will the default. **v3.5 will deprecate `[CLIENT-URL]/config/local/log` endpoint.**
+
+#### Deprecated `pkg/transport.TLSInfo.CAFile` field
+
+Deprecated `pkg/transport.TLSInfo.CAFile` field.
+
+```diff
+import "github.com/coreos/etcd/pkg/transport"
+
+tlsInfo := transport.TLSInfo{
+    CertFile: "/tmp/test-certs/test.pem",
+    KeyFile: "/tmp/test-certs/test-key.pem",
+-   CAFile: "/tmp/test-certs/trusted-ca.pem",
++   TrustedCAFile: "/tmp/test-certs/trusted-ca.pem",
+}
+tlsConfig, err := tlsInfo.ClientConfig()
+if err != nil {
+    panic(err)
+}
+```
+
+#### Changed `embed.Config.SnapCount` to `embed.Config.SnapshotCount`
+
+To be consistent with the flag name `etcd --snapshot-count`, `embed.Config.SnapCount` field has been renamed to `embed.Config.SnapshotCount`:
+
+```diff
+import "github.com/coreos/etcd/embed"
+
+cfg := embed.NewConfig()
+-cfg.SnapCount = 100000
++cfg.SnapshotCount = 100000
+```
+
+#### Changed `etcdserver.ServerConfig.SnapCount` to `etcdserver.ServerConfig.SnapshotCount`
+
+To be consistent with the flag name `etcd --snapshot-count`, `etcdserver.ServerConfig.SnapCount` field has been renamed to `etcdserver.ServerConfig.SnapshotCount`:
+
+```diff
+import "github.com/coreos/etcd/etcdserver"
+
+srvcfg := etcdserver.ServerConfig{
+-  SnapCount: 100000,
++  SnapshotCount: 100000,
+```
+
+#### Changed function signature in package `wal`
+
+Changed `wal` function signatures to support structured logger.
+
+```diff
+import "github.com/coreos/etcd/wal"
++import "go.uber.org/zap"
+
++lg, _ = zap.NewProduction()
+
+-wal.Open(dirpath, snap)
++wal.Open(lg, dirpath, snap)
+
+-wal.OpenForRead(dirpath, snap)
++wal.OpenForRead(lg, dirpath, snap)
+
+-wal.Repair(dirpath)
++wal.Repair(lg, dirpath)
+
+-wal.Create(dirpath, metadata)
++wal.Create(lg, dirpath, metadata)
+```
+
+#### Deprecated `embed.Config.SetupLogging`
+
+`embed.Config.SetupLogging` has been removed in order to prevent wrong logging configuration, and now set up automatically.
+
+```diff
+import "github.com/coreos/etcd/embed"
+
+cfg := &embed.Config{Debug: false}
+-cfg.SetupLogging()
+```
+
+#### Changed gRPC gateway HTTP endpoints (replaced `/v3beta` with `/v3`)
+
+Before
+
+```bash
+curl -L http://localhost:2379/v3beta/kv/put \
+  -X POST -d '{"key": "Zm9v", "value": "YmFy"}'
+```
+
+After
+
+```bash
+curl -L http://localhost:2379/v3/kv/put \
+  -X POST -d '{"key": "Zm9v", "value": "YmFy"}'
+```
+
+Requests to `/v3beta` endpoints will redirect to `/v3`, and `/v3beta` will be removed in 3.5 release.
+
+#### Deprecated container image tags
+
+`latest` and minor version images tags are deprecated:
+
+```diff
+-docker pull gcr.io/etcd-development/etcd:latest
++docker pull gcr.io/etcd-development/etcd:v3.4.0
+
+-docker pull gcr.io/etcd-development/etcd:v3.4
++docker pull gcr.io/etcd-development/etcd:v3.4.0
+
+-docker pull gcr.io/etcd-development/etcd:v3.4
++docker pull gcr.io/etcd-development/etcd:v3.4.1
+
+-docker pull gcr.io/etcd-development/etcd:v3.4
++docker pull gcr.io/etcd-development/etcd:v3.4.2
+```
+
+### Server upgrade checklists
+
+#### Upgrade requirements
+
+To upgrade an existing etcd deployment to 3.4, the running cluster must be 3.3 or greater. If it's before 3.3, please [upgrade to 3.3](upgrade_3_3.md) before upgrading to 3.4.
+
+Also, to ensure a smooth rolling upgrade, the running cluster must be healthy. Check the health of the cluster by using the `etcdctl endpoint health` command before proceeding.
+
+#### Preparation
+
+Before upgrading etcd, always test the services relying on etcd in a staging environment before deploying the upgrade to the production environment.
+
+Before beginning, [download the snapshot backup](../op-guide/maintenance.md#snapshot-backup). Should something go wrong with the upgrade, it is possible to use this backup to [downgrade](#downgrade) back to existing etcd version. Please note that the `snapshot` command only backs up the v3 data. For v2 data, see [backing up v2 datastore](../v2/admin_guide.md#backing-up-the-datastore).
+
+#### Mixed versions
+
+While upgrading, an etcd cluster supports mixed versions of etcd members, and operates with the protocol of the lowest common version. The cluster is only considered upgraded once all of its members are upgraded to version 3.4. Internally, etcd members negotiate with each other to determine the overall cluster version, which controls the reported version and the supported features.
+
+#### Limitations
+
+Note: If the cluster only has v3 data and no v2 data, it is not subject to this limitation.
+
+If the cluster is serving a v2 data set larger than 50MB, each newly upgraded member may take up to two minutes to catch up with the existing cluster. Check the size of a recent snapshot to estimate the total data size. In other words, it is safest to wait for 2 minutes between upgrading each member.
+
+For a much larger total data size, 100MB or more , this one-time process might take even more time. Administrators of very large etcd clusters of this magnitude can feel free to contact the [etcd team][etcd-contact] before upgrading, and we'll be happy to provide advice on the procedure.
+
+#### Downgrade
+
+If all members have been upgraded to v3.4, the cluster will be upgraded to v3.4, and downgrade from this completed state is **not possible**. If any single member is still v3.3, however, the cluster and its operations remains "v3.3", and it is possible from this mixed cluster state to return to using a v3.3 etcd binary on all members.
+
+Please [download the snapshot backup](../op-guide/maintenance.md#snapshot-backup) to make downgrading the cluster possible even after it has been completely upgraded.
+
+### Upgrade procedure
+
+This example shows how to upgrade a 3-member v3.3 ectd cluster running on a local machine.
+
+#### Step 1: check upgrade requirements
+
+Is the cluster healthy and running v3.3.x?
+
+```bash
+etcdctl --endpoints=localhost:2379,localhost:22379,localhost:32379 endpoint health
+<<COMMENT
+localhost:2379 is healthy: successfully committed proposal: took = 2.118638ms
+localhost:22379 is healthy: successfully committed proposal: took = 3.631388ms
+localhost:32379 is healthy: successfully committed proposal: took = 2.157051ms
+COMMENT
+
+curl http://localhost:2379/version
+<<COMMENT
+{"etcdserver":"3.3.5","etcdcluster":"3.3.0"}
+COMMENT
+
+curl http://localhost:22379/version
+<<COMMENT
+{"etcdserver":"3.3.5","etcdcluster":"3.3.0"}
+COMMENT
+
+curl http://localhost:32379/version
+<<COMMENT
+{"etcdserver":"3.3.5","etcdcluster":"3.3.0"}
+COMMENT
+```
+
+#### Step 2: download snapshot backup from leader
+
+[Download the snapshot backup](../op-guide/maintenance.md#snapshot-backup) to provide a downgrade path should any problems occur.
+
+etcd leader is guaranteed to have the latest application data, thus fetch snapshot from leader:
+
+```bash
+curl -sL http://localhost:2379/metrics | grep etcd_server_is_leader
+<<COMMENT
+# HELP etcd_server_is_leader Whether or not this member is a leader. 1 if is, 0 otherwise.
+# TYPE etcd_server_is_leader gauge
+etcd_server_is_leader 1
+COMMENT
+
+curl -sL http://localhost:22379/metrics | grep etcd_server_is_leader
+<<COMMENT
+etcd_server_is_leader 0
+COMMENT
+
+curl -sL http://localhost:32379/metrics | grep etcd_server_is_leader
+<<COMMENT
+etcd_server_is_leader 0
+COMMENT
+
+etcdctl --endpoints=localhost:2379 snapshot save backup.db
+<<COMMENT
+{"level":"info","ts":1526585787.148433,"caller":"snapshot/v3_snapshot.go:109","msg":"created temporary db file","path":"backup.db.part"}
+{"level":"info","ts":1526585787.1485257,"caller":"snapshot/v3_snapshot.go:120","msg":"fetching snapshot","endpoint":"localhost:2379"}
+{"level":"info","ts":1526585787.1519694,"caller":"snapshot/v3_snapshot.go:133","msg":"fetched snapshot","endpoint":"localhost:2379","took":0.003502721}
+{"level":"info","ts":1526585787.1520295,"caller":"snapshot/v3_snapshot.go:142","msg":"saved","path":"backup.db"}
+Snapshot saved at backup.db
+COMMENT
+```
+
+#### Step 3: stop one existing etcd server
+
+When each etcd process is stopped, expected errors will be logged by other cluster members. This is normal since a cluster member connection has been (temporarily) broken:
+
+```bash
+10.237579 I | etcdserver: updating the cluster version from 3.0 to 3.3
+10.238315 N | etcdserver/membership: updated the cluster version from 3.0 to 3.3
+10.238451 I | etcdserver/api: enabled capabilities for version 3.3
+
+
+^C21.192174 N | pkg/osutil: received interrupt signal, shutting down...
+21.192459 I | etcdserver: 7339c4e5e833c029 starts leadership transfer from 7339c4e5e833c029 to 729934363faa4a24
+21.192569 I | raft: 7339c4e5e833c029 [term 8] starts to transfer leadership to 729934363faa4a24
+21.192619 I | raft: 7339c4e5e833c029 sends MsgTimeoutNow to 729934363faa4a24 immediately as 729934363faa4a24 already has up-to-date log
+WARNING: 2018/05/17 12:45:21 grpc: addrConn.resetTransport failed to create client transport: connection error: desc = "transport: Error while dialing dial tcp: operation was canceled"; Reconnecting to {localhost:2379 0  <nil>}
+WARNING: 2018/05/17 12:45:21 grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing
+21.193589 I | raft: 7339c4e5e833c029 [term: 8] received a MsgVote message with higher term from 729934363faa4a24 [term: 9]
+21.193626 I | raft: 7339c4e5e833c029 became follower at term 9
+21.193651 I | raft: 7339c4e5e833c029 [logterm: 8, index: 9, vote: 0] cast MsgVote for 729934363faa4a24 [logterm: 8, index: 9] at term 9
+21.193675 I | raft: raft.node: 7339c4e5e833c029 lost leader 7339c4e5e833c029 at term 9
+21.194424 I | raft: raft.node: 7339c4e5e833c029 elected leader 729934363faa4a24 at term 9
+21.292898 I | etcdserver: 7339c4e5e833c029 finished leadership transfer from 7339c4e5e833c029 to 729934363faa4a24 (took 100.436391ms)
+21.292975 I | rafthttp: stopping peer 729934363faa4a24...
+21.293206 I | rafthttp: closed the TCP streaming connection with peer 729934363faa4a24 (stream MsgApp v2 writer)
+21.293225 I | rafthttp: stopped streaming with peer 729934363faa4a24 (writer)
+21.293437 I | rafthttp: closed the TCP streaming connection with peer 729934363faa4a24 (stream Message writer)
+21.293459 I | rafthttp: stopped streaming with peer 729934363faa4a24 (writer)
+21.293514 I | rafthttp: stopped HTTP pipelining with peer 729934363faa4a24
+21.293590 W | rafthttp: lost the TCP streaming connection with peer 729934363faa4a24 (stream MsgApp v2 reader)
+21.293610 I | rafthttp: stopped streaming with peer 729934363faa4a24 (stream MsgApp v2 reader)
+21.293680 W | rafthttp: lost the TCP streaming connection with peer 729934363faa4a24 (stream Message reader)
+21.293700 I | rafthttp: stopped streaming with peer 729934363faa4a24 (stream Message reader)
+21.293711 I | rafthttp: stopped peer 729934363faa4a24
+21.293720 I | rafthttp: stopping peer b548c2511513015...
+21.293987 I | rafthttp: closed the TCP streaming connection with peer b548c2511513015 (stream MsgApp v2 writer)
+21.294063 I | rafthttp: stopped streaming with peer b548c2511513015 (writer)
+21.294467 I | rafthttp: closed the TCP streaming connection with peer b548c2511513015 (stream Message writer)
+21.294561 I | rafthttp: stopped streaming with peer b548c2511513015 (writer)
+21.294742 I | rafthttp: stopped HTTP pipelining with peer b548c2511513015
+21.294867 W | rafthttp: lost the TCP streaming connection with peer b548c2511513015 (stream MsgApp v2 reader)
+21.294892 I | rafthttp: stopped streaming with peer b548c2511513015 (stream MsgApp v2 reader)
+21.294990 W | rafthttp: lost the TCP streaming connection with peer b548c2511513015 (stream Message reader)
+21.295004 E | rafthttp: failed to read b548c2511513015 on stream Message (context canceled)
+21.295013 I | rafthttp: peer b548c2511513015 became inactive
+21.295024 I | rafthttp: stopped streaming with peer b548c2511513015 (stream Message reader)
+21.295035 I | rafthttp: stopped peer b548c2511513015
+```
+
+#### Step 4: restart the etcd server with same configuration
+
+Restart the etcd server with same configuration but with the new etcd binary.
+
+```diff
+-etcd-old --name s1 \
++etcd-new --name s1 \
+  --data-dir /tmp/etcd/s1 \
+  --listen-client-urls http://localhost:2379 \
+  --advertise-client-urls http://localhost:2379 \
+  --listen-peer-urls http://localhost:2380 \
+  --initial-advertise-peer-urls http://localhost:2380 \
+  --initial-cluster s1=http://localhost:2380,s2=http://localhost:22380,s3=http://localhost:32380 \
+  --initial-cluster-token tkn \
++ --initial-cluster-state new \
++ --logger zap \
++ --log-outputs stderr
+```
+
+The new v3.4 etcd will publish its information to the cluster. At this point, cluster still operates as v3.3 protocol, which is the lowest common version.
+
+> `{"level":"info","ts":1526586617.1647713,"caller":"membership/cluster.go:485","msg":"set initial cluster version","cluster-id":"7dee9ba76d59ed53","local-member-id":"7339c4e5e833c029","cluster-version":"3.0"}`
+
+> `{"level":"info","ts":1526586617.1648536,"caller":"api/capability.go:76","msg":"enabled capabilities for version","cluster-version":"3.0"}`
+
+> `{"level":"info","ts":1526586617.1649303,"caller":"membership/cluster.go:473","msg":"updated cluster version","cluster-id":"7dee9ba76d59ed53","local-member-id":"7339c4e5e833c029","from":"3.0","from":"3.3"}`
+
+> `{"level":"info","ts":1526586617.1649797,"caller":"api/capability.go:76","msg":"enabled capabilities for version","cluster-version":"3.3"}`
+
+> `{"level":"info","ts":1526586617.2107732,"caller":"etcdserver/server.go:1770","msg":"published local member to cluster through raft","local-member-id":"7339c4e5e833c029","local-member-attributes":"{Name:s1 ClientURLs:[http://localhost:2379]}","request-path":"/0/members/7339c4e5e833c029/attributes","cluster-id":"7dee9ba76d59ed53","publish-timeout":7}`
+
+Verify that each member, and then the entire cluster, becomes healthy with the new v3.4 etcd binary:
+
+```bash
+etcdctl endpoint health --endpoints=localhost:2379,localhost:22379,localhost:32379
+<<COMMENT
+localhost:32379 is healthy: successfully committed proposal: took = 2.337471ms
+localhost:22379 is healthy: successfully committed proposal: took = 1.130717ms
+localhost:2379 is healthy: successfully committed proposal: took = 2.124843ms
+COMMENT
+```
+
+Un-upgraded members will log warnings like the following until the entire cluster is upgraded.
+
+This is expected and will cease after all etcd cluster members are upgraded to v3.4:
+
+```
+:41.942121 W | etcdserver: member 7339c4e5e833c029 has a higher version 3.4.0
+:45.945154 W | etcdserver: the local etcd version 3.3.5 is not up-to-date
+```
+
+#### Step 5: repeat *step 3* and *step 4* for rest of the members
+
+When all members are upgraded, the cluster will report upgrading to 3.4 successfully:
+
+Member 1:
+
+> `{"level":"info","ts":1526586949.0920913,"caller":"api/capability.go:76","msg":"enabled capabilities for version","cluster-version":"3.4"}`
+> `{"level":"info","ts":1526586949.0921566,"caller":"etcdserver/server.go:2272","msg":"cluster version is updated","cluster-version":"3.4"}`
+
+Member 2:
+
+> `{"level":"info","ts":1526586949.092117,"caller":"membership/cluster.go:473","msg":"updated cluster version","cluster-id":"7dee9ba76d59ed53","local-member-id":"729934363faa4a24","from":"3.3","from":"3.4"}`
+> `{"level":"info","ts":1526586949.0923078,"caller":"api/capability.go:76","msg":"enabled capabilities for version","cluster-version":"3.4"}`
+
+Member 3:
+
+> `{"level":"info","ts":1526586949.0921423,"caller":"membership/cluster.go:473","msg":"updated cluster version","cluster-id":"7dee9ba76d59ed53","local-member-id":"b548c2511513015","from":"3.3","from":"3.4"}`
+> `{"level":"info","ts":1526586949.0922918,"caller":"api/capability.go:76","msg":"enabled capabilities for version","cluster-version":"3.4"}`
+
+
+```bash
+endpoint health --endpoints=localhost:2379,localhost:22379,localhost:32379
+<<COMMENT
+localhost:2379 is healthy: successfully committed proposal: took = 492.834µs
+localhost:22379 is healthy: successfully committed proposal: took = 1.015025ms
+localhost:32379 is healthy: successfully committed proposal: took = 1.853077ms
+COMMENT
+
+curl http://localhost:2379/version
+<<COMMENT
+{"etcdserver":"3.4.0","etcdcluster":"3.4.0"}
+COMMENT
+
+curl http://localhost:22379/version
+<<COMMENT
+{"etcdserver":"3.4.0","etcdcluster":"3.4.0"}
+COMMENT
+
+curl http://localhost:32379/version
+<<COMMENT
+{"etcdserver":"3.4.0","etcdcluster":"3.4.0"}
+COMMENT
+```
+
+[etcd-contact]: https://groups.google.com/forum/#!forum/etcd-dev

Documentation/upgrades/upgrade_3_5.md

@@ -0,0 +1,341 @@
+## Upgrade etcd from 3.4 to 3.5
+
+In the general case, upgrading from etcd 3.4 to 3.5 can be a zero-downtime, rolling upgrade:
+ - one by one, stop the etcd v3.4 processes and replace them with etcd v3.5 processes
+ - after running all v3.5 processes, new features in v3.5 are available to the cluster
+
+Before [starting an upgrade](#upgrade-procedure), read through the rest of this guide to prepare.
+
+
+
+### Upgrade checklists
+
+**NOTE:** When [migrating from v2 with no v3 data](https://github.com/coreos/etcd/issues/9480), etcd server v3.2+ panics when etcd restores from existing snapshots but no v3 `ETCD_DATA_DIR/member/snap/db` file. This happens when the server had migrated from v2 with no previous v3 data. This also prevents accidental v3 data loss (e.g. `db` file might have been moved). etcd requires that post v3 migration can only happen with v3 data. Do not upgrade to newer v3 versions until v3.0 server contains v3 data.
+
+Highlighted breaking changes in 3.5.
+
+#### Deprecate `etcd_debugging_mvcc_db_total_size_in_bytes` Prometheus metrics
+
+v3.4 promoted `etcd_debugging_mvcc_db_total_size_in_bytes` Prometheus metrics to `etcd_mvcc_db_total_size_in_bytes`, in order to encourage etcd storage monitoring. And v3.5 completely deprcates `etcd_debugging_mvcc_db_total_size_in_bytes`.
+
+```diff
+-etcd_debugging_mvcc_db_total_size_in_bytes
++etcd_mvcc_db_total_size_in_bytes
+```
+
+Note that `etcd_debugging_*` namespace metrics have been marked as experimental. As we improve monitoring guide, we will promote more metrics.
+
+#### Deprecated in `etcd --logger capnslog`
+
+v3.4 defaults to `--logger=zap` in order to support multiple log outputs and structured logging.
+
+**`etcd --logger=capnslog` has been deprecated in v3.5**, and now `--logger=zap` is the default.
+
+```diff
+-etcd --logger=capnslog
++etcd --logger=zap --log-outputs=stderr
+
++# to write logs to stderr and a.log file at the same time
++etcd --logger=zap --log-outputs=stderr,a.log
+```
+
+TODO(add more monitoring guides); v3.4 adds `etcd --logger=zap` support for structured logging and multiple log outputs. Main motivation is to promote automated etcd monitoring, rather than looking back server logs when it starts breaking. Future development will make etcd log as few as possible, and make etcd easier to monitor with metrics and alerts. **`etcd --logger=capnslog` will be deprecated in v3.5.**
+
+#### Deprecated in `etcd --log-output`
+
+v3.4 renamed [`etcd --log-output` to `--log-outputs`](https://github.com/coreos/etcd/pull/9624) to support multiple log outputs.
+
+**`etcd --log-output` has been deprecated in v3.5.**
+
+```diff
+-etcd --log-output=stderr
++etcd --log-outputs=stderr
+```
+
+#### Deprecated `etcd --log-package-levels`
+
+**`etcd --log-package-levels` flag for `capnslog` has been deprecated.**
+
+Now, **`etcd --logger=zap`** is the default.
+
+```diff
+-etcd --log-package-levels 'etcdmain=CRITICAL,etcdserver=DEBUG'
++etcd --logger=zap --log-outputs=stderr
+```
+
+#### Deprecated `[CLIENT-URL]/config/local/log`
+
+**`/config/local/log` endpoint is being deprecated in v3.5, as is `etcd --log-package-levels` flag.**
+
+```diff
+-$ curl http://127.0.0.1:2379/config/local/log -XPUT -d '{"Level":"DEBUG"}'
+-# debug logging enabled
+```
+
+#### Changed gRPC gateway HTTP endpoints (deprecated `/v3beta`)
+
+Before
+
+```bash
+curl -L http://localhost:2379/v3beta/kv/put \
+  -X POST -d '{"key": "Zm9v", "value": "YmFy"}'
+```
+
+After
+
+```bash
+curl -L http://localhost:2379/v3/kv/put \
+  -X POST -d '{"key": "Zm9v", "value": "YmFy"}'
+```
+
+`/v3beta` has been removed in 3.5 release.
+
+
+
+### Server upgrade checklists
+
+#### Upgrade requirements
+
+To upgrade an existing etcd deployment to 3.5, the running cluster must be 3.4 or greater. If it's before 3.4, please [upgrade to 3.4](upgrade_3_3.md) before upgrading to 3.5.
+
+Also, to ensure a smooth rolling upgrade, the running cluster must be healthy. Check the health of the cluster by using the `etcdctl endpoint health` command before proceeding.
+
+#### Preparation
+
+Before upgrading etcd, always test the services relying on etcd in a staging environment before deploying the upgrade to the production environment.
+
+Before beginning, [download the snapshot backup](../op-guide/maintenance.md#snapshot-backup). Should something go wrong with the upgrade, it is possible to use this backup to [downgrade](#downgrade) back to existing etcd version. Please note that the `snapshot` command only backs up the v3 data. For v2 data, see [backing up v2 datastore](../v2/admin_guide.md#backing-up-the-datastore).
+
+#### Mixed versions
+
+While upgrading, an etcd cluster supports mixed versions of etcd members, and operates with the protocol of the lowest common version. The cluster is only considered upgraded once all of its members are upgraded to version 3.5. Internally, etcd members negotiate with each other to determine the overall cluster version, which controls the reported version and the supported features.
+
+#### Limitations
+
+Note: If the cluster only has v3 data and no v2 data, it is not subject to this limitation.
+
+If the cluster is serving a v2 data set larger than 50MB, each newly upgraded member may take up to two minutes to catch up with the existing cluster. Check the size of a recent snapshot to estimate the total data size. In other words, it is safest to wait for 2 minutes between upgrading each member.
+
+For a much larger total data size, 100MB or more , this one-time process might take even more time. Administrators of very large etcd clusters of this magnitude can feel free to contact the [etcd team][etcd-contact] before upgrading, and we'll be happy to provide advice on the procedure.
+
+#### Downgrade
+
+If all members have been upgraded to v3.5, the cluster will be upgraded to v3.5, and downgrade from this completed state is **not possible**. If any single member is still v3.4, however, the cluster and its operations remains "v3.4", and it is possible from this mixed cluster state to return to using a v3.4 etcd binary on all members.
+
+Please [download the snapshot backup](../op-guide/maintenance.md#snapshot-backup) to make downgrading the cluster possible even after it has been completely upgraded.
+
+### Upgrade procedure
+
+This example shows how to upgrade a 3-member v3.4 ectd cluster running on a local machine.
+
+#### Step 1: check upgrade requirements
+
+Is the cluster healthy and running v3.4.x?
+
+```bash
+etcdctl --endpoints=localhost:2379,localhost:22379,localhost:32379 endpoint health
+<<COMMENT
+localhost:2379 is healthy: successfully committed proposal: took = 2.118638ms
+localhost:22379 is healthy: successfully committed proposal: took = 3.631388ms
+localhost:32379 is healthy: successfully committed proposal: took = 2.157051ms
+COMMENT
+
+curl http://localhost:2379/version
+<<COMMENT
+{"etcdserver":"3.4.0","etcdcluster":"3.4.0"}
+COMMENT
+
+curl http://localhost:22379/version
+<<COMMENT
+{"etcdserver":"3.4.0","etcdcluster":"3.4.0"}
+COMMENT
+
+curl http://localhost:32379/version
+<<COMMENT
+{"etcdserver":"3.4.0","etcdcluster":"3.4.0"}
+COMMENT
+```
+
+#### Step 2: download snapshot backup from leader
+
+[Download the snapshot backup](../op-guide/maintenance.md#snapshot-backup) to provide a downgrade path should any problems occur.
+
+etcd leader is guaranteed to have the latest application data, thus fetch snapshot from leader:
+
+```bash
+curl -sL http://localhost:2379/metrics | grep etcd_server_is_leader
+<<COMMENT
+# HELP etcd_server_is_leader Whether or not this member is a leader. 1 if is, 0 otherwise.
+# TYPE etcd_server_is_leader gauge
+etcd_server_is_leader 1
+COMMENT
+
+curl -sL http://localhost:22379/metrics | grep etcd_server_is_leader
+<<COMMENT
+etcd_server_is_leader 0
+COMMENT
+
+curl -sL http://localhost:32379/metrics | grep etcd_server_is_leader
+<<COMMENT
+etcd_server_is_leader 0
+COMMENT
+
+etcdctl --endpoints=localhost:2379 snapshot save backup.db
+<<COMMENT
+{"level":"info","ts":1526585787.148433,"caller":"snapshot/v3_snapshot.go:109","msg":"created temporary db file","path":"backup.db.part"}
+{"level":"info","ts":1526585787.1485257,"caller":"snapshot/v3_snapshot.go:120","msg":"fetching snapshot","endpoint":"localhost:2379"}
+{"level":"info","ts":1526585787.1519694,"caller":"snapshot/v3_snapshot.go:133","msg":"fetched snapshot","endpoint":"localhost:2379","took":0.003502721}
+{"level":"info","ts":1526585787.1520295,"caller":"snapshot/v3_snapshot.go:142","msg":"saved","path":"backup.db"}
+Snapshot saved at backup.db
+COMMENT
+```
+
+#### Step 3: stop one existing etcd server
+
+When each etcd process is stopped, expected errors will be logged by other cluster members. This is normal since a cluster member connection has been (temporarily) broken:
+
+```bash
+{"level":"info","ts":1526587281.2001143,"caller":"etcdserver/server.go:2249","msg":"updating cluster version","from":"3.0","to":"3.4"}
+{"level":"info","ts":1526587281.2010646,"caller":"membership/cluster.go:473","msg":"updated cluster version","cluster-id":"7dee9ba76d59ed53","local-member-id":"7339c4e5e833c029","from":"3.0","from":"3.4"}
+{"level":"info","ts":1526587281.2012327,"caller":"api/capability.go:76","msg":"enabled capabilities for version","cluster-version":"3.4"}
+{"level":"info","ts":1526587281.2013083,"caller":"etcdserver/server.go:2272","msg":"cluster version is updated","cluster-version":"3.4"}
+
+
+
+^C{"level":"info","ts":1526587299.0717514,"caller":"osutil/interrupt_unix.go:63","msg":"received signal; shutting down","signal":"interrupt"}
+{"level":"info","ts":1526587299.0718873,"caller":"embed/etcd.go:285","msg":"closing etcd server","name":"s1","data-dir":"/tmp/etcd/s1","advertise-peer-urls":["http://localhost:2380"],"advertise-client-urls":["http://localhost:2379"]}
+{"level":"info","ts":1526587299.0722554,"caller":"etcdserver/server.go:1341","msg":"leadership transfer starting","local-member-id":"7339c4e5e833c029","current-leader-member-id":"7339c4e5e833c029","transferee-member-id":"729934363faa4a24"}
+{"level":"info","ts":1526587299.0723994,"caller":"raft/raft.go:1107","msg":"7339c4e5e833c029 [term 3] starts to transfer leadership to 729934363faa4a24"}
+{"level":"info","ts":1526587299.0724802,"caller":"raft/raft.go:1113","msg":"7339c4e5e833c029 sends MsgTimeoutNow to 729934363faa4a24 immediately as 729934363faa4a24 already has up-to-date log"}
+{"level":"info","ts":1526587299.0737045,"caller":"raft/raft.go:797","msg":"7339c4e5e833c029 [term: 3] received a MsgVote message with higher term from 729934363faa4a24 [term: 4]"}
+{"level":"info","ts":1526587299.0737681,"caller":"raft/raft.go:656","msg":"7339c4e5e833c029 became follower at term 4"}
+{"level":"info","ts":1526587299.073831,"caller":"raft/raft.go:882","msg":"7339c4e5e833c029 [logterm: 3, index: 9, vote: 0] cast MsgVote for 729934363faa4a24 [logterm: 3, index: 9] at term 4"}
+{"level":"info","ts":1526587299.0738947,"caller":"raft/node.go:312","msg":"raft.node: 7339c4e5e833c029 lost leader 7339c4e5e833c029 at term 4"}
+{"level":"info","ts":1526587299.0748374,"caller":"raft/node.go:306","msg":"raft.node: 7339c4e5e833c029 elected leader 729934363faa4a24 at term 4"}
+{"level":"info","ts":1526587299.1726425,"caller":"etcdserver/server.go:1362","msg":"leadership transfer finished","local-member-id":"7339c4e5e833c029","old-leader-member-id":"7339c4e5e833c029","new-leader-member-id":"729934363faa4a24","took":0.100389359}
+{"level":"info","ts":1526587299.1728148,"caller":"rafthttp/peer.go:333","msg":"stopping remote peer","remote-peer-id":"b548c2511513015"}
+{"level":"warn","ts":1526587299.1751974,"caller":"rafthttp/stream.go:291","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","remote-peer-id":"b548c2511513015"}
+{"level":"warn","ts":1526587299.1752589,"caller":"rafthttp/stream.go:301","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","remote-peer-id":"b548c2511513015"}
+{"level":"warn","ts":1526587299.177348,"caller":"rafthttp/stream.go:291","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream Message","remote-peer-id":"b548c2511513015"}
+{"level":"warn","ts":1526587299.1774004,"caller":"rafthttp/stream.go:301","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream Message","remote-peer-id":"b548c2511513015"}
+{"level":"info","ts":1526587299.177515,"caller":"rafthttp/pipeline.go:86","msg":"stopped HTTP pipelining with remote peer","local-member-id":"7339c4e5e833c029","remote-peer-id":"b548c2511513015"}
+{"level":"warn","ts":1526587299.1777067,"caller":"rafthttp/stream.go:436","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"7339c4e5e833c029","remote-peer-id":"b548c2511513015","error":"read tcp 127.0.0.1:34636->127.0.0.1:32380: use of closed network connection"}
+{"level":"info","ts":1526587299.1778402,"caller":"rafthttp/stream.go:459","msg":"stopped stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"7339c4e5e833c029","remote-peer-id":"b548c2511513015"}
+{"level":"warn","ts":1526587299.1780295,"caller":"rafthttp/stream.go:436","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"7339c4e5e833c029","remote-peer-id":"b548c2511513015","error":"read tcp 127.0.0.1:34634->127.0.0.1:32380: use of closed network connection"}
+{"level":"info","ts":1526587299.1780987,"caller":"rafthttp/stream.go:459","msg":"stopped stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"7339c4e5e833c029","remote-peer-id":"b548c2511513015"}
+{"level":"info","ts":1526587299.1781602,"caller":"rafthttp/peer.go:340","msg":"stopped remote peer","remote-peer-id":"b548c2511513015"}
+{"level":"info","ts":1526587299.1781986,"caller":"rafthttp/peer.go:333","msg":"stopping remote peer","remote-peer-id":"729934363faa4a24"}
+{"level":"warn","ts":1526587299.1802843,"caller":"rafthttp/stream.go:291","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","remote-peer-id":"729934363faa4a24"}
+{"level":"warn","ts":1526587299.1803446,"caller":"rafthttp/stream.go:301","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","remote-peer-id":"729934363faa4a24"}
+{"level":"warn","ts":1526587299.1824749,"caller":"rafthttp/stream.go:291","msg":"closed TCP streaming connection with remote peer","stream-writer-type":"stream Message","remote-peer-id":"729934363faa4a24"}
+{"level":"warn","ts":1526587299.18255,"caller":"rafthttp/stream.go:301","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream Message","remote-peer-id":"729934363faa4a24"}
+{"level":"info","ts":1526587299.18261,"caller":"rafthttp/pipeline.go:86","msg":"stopped HTTP pipelining with remote peer","local-member-id":"7339c4e5e833c029","remote-peer-id":"729934363faa4a24"}
+{"level":"warn","ts":1526587299.1827736,"caller":"rafthttp/stream.go:436","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"7339c4e5e833c029","remote-peer-id":"729934363faa4a24","error":"read tcp 127.0.0.1:51482->127.0.0.1:22380: use of closed network connection"}
+{"level":"info","ts":1526587299.182845,"caller":"rafthttp/stream.go:459","msg":"stopped stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"7339c4e5e833c029","remote-peer-id":"729934363faa4a24"}
+{"level":"warn","ts":1526587299.1830168,"caller":"rafthttp/stream.go:436","msg":"lost TCP streaming connection with remote peer","stream-reader-type":"stream Message","local-member-id":"7339c4e5e833c029","remote-peer-id":"729934363faa4a24","error":"context canceled"}
+{"level":"warn","ts":1526587299.1831107,"caller":"rafthttp/peer_status.go:65","msg":"peer became inactive","peer-id":"729934363faa4a24","error":"failed to read 729934363faa4a24 on stream Message (context canceled)"}
+{"level":"info","ts":1526587299.1831737,"caller":"rafthttp/stream.go:459","msg":"stopped stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"7339c4e5e833c029","remote-peer-id":"729934363faa4a24"}
+{"level":"info","ts":1526587299.1832306,"caller":"rafthttp/peer.go:340","msg":"stopped remote peer","remote-peer-id":"729934363faa4a24"}
+{"level":"warn","ts":1526587299.1837125,"caller":"rafthttp/http.go:424","msg":"failed to find remote peer in cluster","local-member-id":"7339c4e5e833c029","remote-peer-id-stream-handler":"7339c4e5e833c029","remote-peer-id-from":"b548c2511513015","cluster-id":"7dee9ba76d59ed53"}
+{"level":"warn","ts":1526587299.1840093,"caller":"rafthttp/http.go:424","msg":"failed to find remote peer in cluster","local-member-id":"7339c4e5e833c029","remote-peer-id-stream-handler":"7339c4e5e833c029","remote-peer-id-from":"b548c2511513015","cluster-id":"7dee9ba76d59ed53"}
+{"level":"warn","ts":1526587299.1842315,"caller":"rafthttp/http.go:424","msg":"failed to find remote peer in cluster","local-member-id":"7339c4e5e833c029","remote-peer-id-stream-handler":"7339c4e5e833c029","remote-peer-id-from":"729934363faa4a24","cluster-id":"7dee9ba76d59ed53"}
+{"level":"warn","ts":1526587299.1844475,"caller":"rafthttp/http.go:424","msg":"failed to find remote peer in cluster","local-member-id":"7339c4e5e833c029","remote-peer-id-stream-handler":"7339c4e5e833c029","remote-peer-id-from":"729934363faa4a24","cluster-id":"7dee9ba76d59ed53"}
+{"level":"info","ts":1526587299.2056687,"caller":"embed/etcd.go:473","msg":"stopping serving peer traffic","address":"127.0.0.1:2380"}
+{"level":"info","ts":1526587299.205819,"caller":"embed/etcd.go:480","msg":"stopped serving peer traffic","address":"127.0.0.1:2380"}
+{"level":"info","ts":1526587299.2058413,"caller":"embed/etcd.go:289","msg":"closed etcd server","name":"s1","data-dir":"/tmp/etcd/s1","advertise-peer-urls":["http://localhost:2380"],"advertise-client-urls":["http://localhost:2379"]}
+```
+
+#### Step 4: restart the etcd server with same configuration
+
+Restart the etcd server with same configuration but with the new etcd binary.
+
+```diff
+-etcd-old --name s1 \
++etcd-new --name s1 \
+  --data-dir /tmp/etcd/s1 \
+  --listen-client-urls http://localhost:2379 \
+  --advertise-client-urls http://localhost:2379 \
+  --listen-peer-urls http://localhost:2380 \
+  --initial-advertise-peer-urls http://localhost:2380 \
+  --initial-cluster s1=http://localhost:2380,s2=http://localhost:22380,s3=http://localhost:32380 \
+  --initial-cluster-token tkn \
+  --initial-cluster-state new
+```
+
+The new v3.5 etcd will publish its information to the cluster. At this point, cluster still operates as v3.4 protocol, which is the lowest common version.
+
+> `{"level":"info","ts":1526586617.1647713,"caller":"membership/cluster.go:485","msg":"set initial cluster version","cluster-id":"7dee9ba76d59ed53","local-member-id":"7339c4e5e833c029","cluster-version":"3.0"}`
+
+> `{"level":"info","ts":1526586617.1648536,"caller":"api/capability.go:76","msg":"enabled capabilities for version","cluster-version":"3.0"}`
+
+> `{"level":"info","ts":1526586617.1649303,"caller":"membership/cluster.go:473","msg":"updated cluster version","cluster-id":"7dee9ba76d59ed53","local-member-id":"7339c4e5e833c029","from":"3.0","from":"3.4"}`
+
+> `{"level":"info","ts":1526586617.1649797,"caller":"api/capability.go:76","msg":"enabled capabilities for version","cluster-version":"3.4"}`
+
+> `{"level":"info","ts":1526586617.2107732,"caller":"etcdserver/server.go:1770","msg":"published local member to cluster through raft","local-member-id":"7339c4e5e833c029","local-member-attributes":"{Name:s1 ClientURLs:[http://localhost:2379]}","request-path":"/0/members/7339c4e5e833c029/attributes","cluster-id":"7dee9ba76d59ed53","publish-timeout":7}`
+
+Verify that each member, and then the entire cluster, becomes healthy with the new v3.5 etcd binary:
+
+```bash
+etcdctl endpoint health --endpoints=localhost:2379,localhost:22379,localhost:32379
+<<COMMENT
+localhost:32379 is healthy: successfully committed proposal: took = 2.337471ms
+localhost:22379 is healthy: successfully committed proposal: took = 1.130717ms
+localhost:2379 is healthy: successfully committed proposal: took = 2.124843ms
+COMMENT
+```
+
+Un-upgraded members will log warnings like the following until the entire cluster is upgraded.
+
+This is expected and will cease after all etcd cluster members are upgraded to v3.5:
+
+```
+:41.942121 W | etcdserver: member 7339c4e5e833c029 has a higher version 3.5.0
+:45.945154 W | etcdserver: the local etcd version 3.4.0 is not up-to-date
+```
+
+#### Step 5: repeat *step 3* and *step 4* for rest of the members
+
+When all members are upgraded, the cluster will report upgrading to 3.5 successfully:
+
+Member 1:
+
+> `{"level":"info","ts":1526586949.0920913,"caller":"api/capability.go:76","msg":"enabled capabilities for version","cluster-version":"3.5"}`
+> `{"level":"info","ts":1526586949.0921566,"caller":"etcdserver/server.go:2272","msg":"cluster version is updated","cluster-version":"3.5"}`
+
+Member 2:
+
+> `{"level":"info","ts":1526586949.092117,"caller":"membership/cluster.go:473","msg":"updated cluster version","cluster-id":"7dee9ba76d59ed53","local-member-id":"729934363faa4a24","from":"3.4","from":"3.5"}`
+> `{"level":"info","ts":1526586949.0923078,"caller":"api/capability.go:76","msg":"enabled capabilities for version","cluster-version":"3.5"}`
+
+Member 3:
+
+> `{"level":"info","ts":1526586949.0921423,"caller":"membership/cluster.go:473","msg":"updated cluster version","cluster-id":"7dee9ba76d59ed53","local-member-id":"b548c2511513015","from":"3.4","from":"3.5"}`
+> `{"level":"info","ts":1526586949.0922918,"caller":"api/capability.go:76","msg":"enabled capabilities for version","cluster-version":"3.5"}`
+
+
+```bash
+endpoint health --endpoints=localhost:2379,localhost:22379,localhost:32379
+<<COMMENT
+localhost:2379 is healthy: successfully committed proposal: took = 492.834µs
+localhost:22379 is healthy: successfully committed proposal: took = 1.015025ms
+localhost:32379 is healthy: successfully committed proposal: took = 1.853077ms
+COMMENT
+
+curl http://localhost:2379/version
+<<COMMENT
+{"etcdserver":"3.5.0","etcdcluster":"3.5.0"}
+COMMENT
+
+curl http://localhost:22379/version
+<<COMMENT
+{"etcdserver":"3.5.0","etcdcluster":"3.5.0"}
+COMMENT
+
+curl http://localhost:32379/version
+<<COMMENT
+{"etcdserver":"3.5.0","etcdcluster":"3.5.0"}
+COMMENT
+```
+
+[etcd-contact]: https://groups.google.com/forum/#!forum/etcd-dev

Documentation/upgrades/upgrading-etcd.md

@@ -8,6 +8,8 @@ This section contains documents specific to upgrading etcd clusters and applicat
 ## Upgrading an etcd v3.x cluster
 * [Upgrade etcd from 3.0 to 3.1][upgrade-3-1]
 * [Upgrade etcd from 3.1 to 3.2][upgrade-3-2]
+* [Upgrade etcd from 3.2 to 3.3][upgrade-3-3]
+* [Upgrade etcd from 3.3 to 3.4][upgrade-3-4]
 
 ## Upgrading from etcd v2.3
 * [Upgrade a v2.3 cluster to v3.0][upgrade-cluster]
@@ -17,3 +19,5 @@ This section contains documents specific to upgrading etcd clusters and applicat
 [upgrade-cluster]: upgrade_3_0.md
 [upgrade-3-1]: upgrade_3_1.md
 [upgrade-3-2]: upgrade_3_2.md
+[upgrade-3-3]: upgrade_3_3.md
+[upgrade-3-4]: upgrade_3_4.md

Documentation/v2/admin_guide.md

@@ -45,12 +45,12 @@ It is important to monitor your production etcd cluster for healthy information
 
 #### Health Monitoring
 
-At lowest level, etcd exposes health information via HTTP at `/health` in JSON format. If it returns `{"health":true}`, then the cluster is healthy.
+At lowest level, etcd exposes health information via HTTP at `/health` in JSON format. If it returns `{"health":"true"}`, then the cluster is healthy.
 
 ```
 $ curl -L http://127.0.0.1:2379/health
 
-{"health":true}
+{"health":"true"}
 ```
 
 You can also use etcdctl to check the cluster-wide health information. It will contact all the members of the cluster and collect the health information for you.
@@ -77,12 +77,14 @@ easier.
 When you want to debug etcd without stopping it, you can enable debug logging at runtime.
 etcd exposes logging configuration at `/config/local/log`.
 
+**`/config/local/log` endpoint is being deprecated in v3.5.**
+
 ```
 $ curl http://127.0.0.1:2379/config/local/log -XPUT -d '{"Level":"DEBUG"}'
-$ # debug logging enabled
-$
+# debug logging enabled
+
 $ curl http://127.0.0.1:2379/config/local/log -XPUT -d '{"Level":"INFO"}'
-$ # debug logging disabled
+# debug logging disabled
 ```
 
 #### Debugging Variables
@@ -237,13 +239,13 @@ This command will rewrite some of the metadata contained in the backup (specific
 
 #### Restoring a backup
 
-To restore a backup using the procedure created above, start etcd with the `-force-new-cluster` option and pointing to the backup directory. This will initialize a new, single-member cluster with the default advertised peer URLs, but preserve the entire contents of the etcd data store. Continuing from the previous example:
+To restore a backup using the procedure created above, start etcd with the `--force-new-cluster` option and pointing to the backup directory. This will initialize a new, single-member cluster with the default advertised peer URLs, but preserve the entire contents of the etcd data store. Continuing from the previous example:
 
 ```sh
     etcd \
       -data-dir=%backup_data_dir% \
       [-wal-dir=%backup_wal_dir%] \
-      -force-new-cluster \
+      --force-new-cluster \
       ...
 ```
 

Documentation/v2/benchmarks/etcd-2-2-0-benchmarks.md

@@ -31,7 +31,7 @@ Go OS/Arch: linux/amd64
 
 Bootstrap another machine, outside of the etcd cluster, and run the [`boom` HTTP benchmark tool][boom] with a connection reuse patch to send requests to each etcd cluster member. See the [benchmark instructions][hack] for the patch and the steps to reproduce our procedures.
 
-The performance is calulated through results of 100 benchmark rounds.
+The performance is calculated through results of 100 benchmark rounds.
 
 ## Performance
 

Documentation/v2/clustering.md

@@ -292,6 +292,13 @@ To help clients discover the etcd cluster, the following DNS SRV records are loo
 
 If `_etcd-client-ssl._tcp.example.com` is found, clients will attempt to communicate with the etcd cluster over SSL.
 
+The `-discovery-srv-name` flag additionally configures a suffix to the SRV name that is queried during discovery.
+Use this flag to differentiate between multiple etcd clusters under the same domain.
+For example, if `discovery-srv=example.com` and `-discovery-srv-name=foo` are set, the following DNS SRV queries are made:
+
+* _etcd-server-ssl-foo._tcp.example.com
+* _etcd-server-foo._tcp.example.com
+
 #### Create DNS SRV records
 
 ```

Documentation/v2/configuration.md

@@ -127,6 +127,11 @@ To start etcd automatically using custom settings at startup in Linux, using a [
 + default: none
 + env variable: ETCD_DISCOVERY_SRV
 
+### --discovery-srv-name
++ Suffix to the DNS srv name queried when bootstrapping using DNS.
++ default: none
++ env variable: ETCD_DISCOVERY_SRV_NAME
+
 ### --discovery-fallback
 + Expected behavior ("exit" or "proxy") when discovery services fails.
 + default: "proxy"

Documentation/v2/libraries-and-tools.md

@@ -120,7 +120,6 @@
 - [mattn/etcdenv](https://github.com/mattn/etcdenv) - "env" shebang with etcd integration
 - [kelseyhightower/confd](https://github.com/kelseyhightower/confd) - Manage local app config files using templates and data from etcd
 - [configdb](https://git.autistici.org/ai/configdb/tree/master) - A REST relational abstraction on top of arbitrary database backends, aimed at storing configs and inventories.
-- [fleet](https://github.com/coreos/fleet) - Distributed init system
 - [kubernetes/kubernetes](https://github.com/kubernetes/kubernetes) - Container cluster manager introduced by Google.
 - [mailgun/vulcand](https://github.com/mailgun/vulcand) - HTTP proxy that uses etcd as a configuration backend.
 - [duedil-ltd/discodns](https://github.com/duedil-ltd/discodns) - Simple DNS nameserver using etcd as a database for names and records.

Documentation/v2/other_apis.md

@@ -29,5 +29,5 @@ curl http://10.0.0.10:2379/health
 ```
 
 ```json
-{"health":true}
+{"health":"true"}
 ```

Gopkg.lock

@@ -0,0 +1,549 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+  branch = "master"
+  digest = "1:d6afaeed1502aa28e80a4ed0981d570ad91b2579193404256ce672ed0a609e0d"
+  name = "github.com/beorn7/perks"
+  packages = ["quantile"]
+  pruneopts = "UT"
+  revision = "3a771d992973f24aa725d07868b467d1ddfceafb"
+
+[[projects]]
+  digest = "1:1343a2963481a305ca4d051e84bc2abd16b601ee22ed324f8d605de1adb291b0"
+  name = "github.com/bgentry/speakeasy"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "4aabc24848ce5fd31929f7d1e4ea74d3709c14cd"
+  version = "v0.1.0"
+
+[[projects]]
+  digest = "1:6bda4e232607261eeaee45b0a0e8a2ef097bd44e8a44d9b7ca1badb36e523891"
+  name = "github.com/coreos/bbolt"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "48ea1b39c25fc1bab3506fbc712ecbaa842c4d2d"
+  version = "v1.3.1-coreos.6"
+
+[[projects]]
+  digest = "1:0ef770954bca104ee99b3b6b7f9b240605ac03517d9f98cbc1893daa03f3c038"
+  name = "github.com/coreos/go-semver"
+  packages = ["semver"]
+  pruneopts = "UT"
+  revision = "8ab6407b697782a06568d4b7f1db25550ec2e4c6"
+  version = "v0.2.0"
+
+[[projects]]
+  digest = "1:caf7d199019565155810a525f332ad40d691095560e4bdf8acd02fffb0608f7e"
+  name = "github.com/coreos/go-systemd"
+  packages = [
+    "daemon",
+    "journal",
+    "util",
+  ]
+  pruneopts = "UT"
+  revision = "39ca1b05acc7ad1220e09f133283b8859a8b71ab"
+  version = "v17"
+
+[[projects]]
+  digest = "1:f5e7a28fc94786fd824a85ba27445eea2676ea5019c7c9ebc0c350b0895673a4"
+  name = "github.com/coreos/pkg"
+  packages = [
+    "capnslog",
+    "dlopen",
+  ]
+  pruneopts = "UT"
+  revision = "3ac0863d7acf3bc44daf49afef8919af12f704ef"
+  version = "v3"
+
+[[projects]]
+  digest = "1:76dc72490af7174349349838f2fe118996381b31ea83243812a97e5a0fd5ed55"
+  name = "github.com/dgrijalva/jwt-go"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e"
+  version = "v3.2.0"
+
+[[projects]]
+  digest = "1:ee7fe6a68ba9f21bc748293d4e1e66ad327f64a89b0f5ba2d021430babd4b8c9"
+  name = "github.com/dustin/go-humanize"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "bb3d318650d48840a39aa21a027c6630e198e626"
+
+[[projects]]
+  digest = "1:2cd7915ab26ede7d95b8749e6b1f933f1c6d5398030684e6505940a10f31cfda"
+  name = "github.com/ghodss/yaml"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7"
+  version = "v1.0.0"
+
+[[projects]]
+  digest = "1:8f13f978d35181fada0ea713c14a224f2b329fa91c13438b2811331ecd8b16e1"
+  name = "github.com/gogo/protobuf"
+  packages = [
+    "gogoproto",
+    "proto",
+    "protoc-gen-gogo/descriptor",
+  ]
+  pruneopts = "UT"
+  revision = "1adfc126b41513cc696b209667c8656ea7aac67c"
+  version = "v1.0.0"
+
+[[projects]]
+  digest = "1:7672c206322f45b33fac1ae2cb899263533ce0adcc6481d207725560208ec84e"
+  name = "github.com/golang/groupcache"
+  packages = ["lru"]
+  pruneopts = "UT"
+  revision = "02826c3e79038b59d737d3b1c0a1d937f71a4433"
+
+[[projects]]
+  digest = "1:69d5e4ea47a10826ceeacc88f99b1f3e509b07690cee667d70e13b5dd505bdcc"
+  name = "github.com/golang/protobuf"
+  packages = [
+    "jsonpb",
+    "proto",
+    "ptypes",
+    "ptypes/any",
+    "ptypes/duration",
+    "ptypes/struct",
+    "ptypes/timestamp",
+  ]
+  pruneopts = "UT"
+  revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
+  version = "v1.1.0"
+
+[[projects]]
+  digest = "1:9887333bbef17574b1db5f9893ea137ac44107235d624408a3ac9e0b98fbb2cb"
+  name = "github.com/google/btree"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "e89373fe6b4a7413d7acd6da1725b83ef713e6e4"
+
+[[projects]]
+  digest = "1:269d9336f94f48817bc0958dae75c285cb0d107cbfe3e1bc6b69d9105d427748"
+  name = "github.com/gorilla/websocket"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "4201258b820c74ac8e6922fc9e6b52f71fe46f8d"
+
+[[projects]]
+  digest = "1:d4fb49314d7f792a14c72f094f8fcb9ecfdf7a4c3e2186efb1f3d3a88806a844"
+  name = "github.com/grpc-ecosystem/go-grpc-middleware"
+  packages = [
+    ".",
+    "util/backoffutils",
+  ]
+  pruneopts = "UT"
+  revision = "c250d6563d4d4c20252cd865923440e829844f4e"
+  version = "v1.0.0"
+
+[[projects]]
+  digest = "1:9b7a07ac7577787a8ecc1334cb9f34df1c76ed82a917d556c5713d3ab84fbc43"
+  name = "github.com/grpc-ecosystem/go-grpc-prometheus"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "c225b8c3b01faf2899099b768856a9e916e5087b"
+  version = "v1.2.0"
+
+[[projects]]
+  digest = "1:6a06217fc753e65aea71446b777393009ef6797dadc0086c4f75bd16590be04d"
+  name = "github.com/grpc-ecosystem/grpc-gateway"
+  packages = [
+    "runtime",
+    "runtime/internal",
+    "utilities",
+  ]
+  pruneopts = "UT"
+  revision = "92583770e3f01b09a0d3e9bdf64321d8bebd48f2"
+  version = "v1.4.1"
+
+[[projects]]
+  digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
+  name = "github.com/inconshreveable/mousetrap"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
+  version = "v1.0"
+
+[[projects]]
+  digest = "1:75ab90ae3f5d876167e60f493beadfe66f0ed861a710f283fb06c86437a09538"
+  name = "github.com/jonboulle/clockwork"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "2eee05ed794112d45db504eb05aa693efd2b8b09"
+  version = "v0.1.0"
+
+[[projects]]
+  digest = "1:075fe2b22b9fcec90cecd98546c2096d73a3ab9458261f5647512fba3e0420eb"
+  name = "github.com/kr/pty"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "2c10821df3c3cf905230d078702dfbe9404c9b23"
+  version = "v1.0.0"
+
+[[projects]]
+  digest = "1:e2d1d410fb367567c2b53ed9e2d719d3c1f0891397bb2fa49afd747cfbf1e8e4"
+  name = "github.com/mattn/go-runewidth"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "9e777a8366cce605130a531d2cd6363d07ad7317"
+  version = "v0.0.2"
+
+[[projects]]
+  digest = "1:5985ef4caf91ece5d54817c11ea25f182697534f8ae6521eadcd628c142ac4b6"
+  name = "github.com/matttproud/golang_protobuf_extensions"
+  packages = ["pbutil"]
+  pruneopts = "UT"
+  revision = "3247c84500bff8d9fb6d579d800f20b3e091582c"
+  version = "v1.0.0"
+
+[[projects]]
+  digest = "1:51c406be3d166964b459a34db5faaafe8b4cc28e123e644f5274db97e558c791"
+  name = "github.com/olekukonko/tablewriter"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "a0225b3f23b5ce0cbec6d7a66a968f8a59eca9c4"
+
+[[projects]]
+  branch = "master"
+  digest = "1:3bf17a6e6eaa6ad24152148a631d18662f7212e21637c2699bff3369b7f00fa2"
+  name = "github.com/petar/GoLLRB"
+  packages = ["llrb"]
+  pruneopts = "UT"
+  revision = "53be0d36a84c2a886ca057d34b6aa4468df9ccb4"
+
+[[projects]]
+  digest = "1:d14a5f4bfecf017cb780bdde1b6483e5deb87e12c332544d2c430eda58734bcb"
+  name = "github.com/prometheus/client_golang"
+  packages = [
+    "prometheus",
+    "prometheus/promhttp",
+  ]
+  pruneopts = "UT"
+  revision = "c5b7fccd204277076155f10851dad72b76a49317"
+  version = "v0.8.0"
+
+[[projects]]
+  digest = "1:32d10bdfa8f09ecf13598324dba86ab891f11db3c538b6a34d1c3b5b99d7c36b"
+  name = "github.com/prometheus/client_model"
+  packages = ["go"]
+  pruneopts = "UT"
+  revision = "6f3806018612930941127f2a7c6c453ba2c527d2"
+
+[[projects]]
+  branch = "master"
+  digest = "1:e469cd65badf7694aeb44874518606d93c1d59e7735d3754ad442782437d3cc3"
+  name = "github.com/prometheus/common"
+  packages = [
+    "expfmt",
+    "internal/bitbucket.org/ww/goautoneg",
+    "model",
+  ]
+  pruneopts = "UT"
+  revision = "7600349dcfe1abd18d72d3a1770870d9800a7801"
+
+[[projects]]
+  branch = "master"
+  digest = "1:5b52592ee5c75444338e0949f8e80fca7d616c1bcc209b9336a1861b526400fb"
+  name = "github.com/prometheus/procfs"
+  packages = [
+    ".",
+    "internal/util",
+    "nfs",
+    "xfs",
+  ]
+  pruneopts = "UT"
+  revision = "7d6f385de8bea29190f15ba9931442a0eaef9af7"
+
+[[projects]]
+  digest = "1:9e9193aa51197513b3abcb108970d831fbcf40ef96aa845c4f03276e1fa316d2"
+  name = "github.com/sirupsen/logrus"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "c155da19408a8799da419ed3eeb0cb5db0ad5dbc"
+  version = "v1.0.5"
+
+[[projects]]
+  digest = "1:30e06e6d62a1d694e3cdbff29d8a9a96022e05a487d0c6eaf0ef898965ef28fb"
+  name = "github.com/soheilhy/cmux"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "e09e9389d85d8492d313d73d1469c029e710623f"
+  version = "v0.1.4"
+
+[[projects]]
+  digest = "1:645cabccbb4fa8aab25a956cbcbdf6a6845ca736b2c64e197ca7cbb9d210b939"
+  name = "github.com/spf13/cobra"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385"
+  version = "v0.0.3"
+
+[[projects]]
+  digest = "1:9424f440bba8f7508b69414634aef3b2b3a877e522d8a4624692412805407bb7"
+  name = "github.com/spf13/pflag"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "583c0c0531f06d5278b7d917446061adc344b5cd"
+  version = "v1.0.1"
+
+[[projects]]
+  digest = "1:28aee88434fcefe93f513c55f793bc36362b88c0ae40e63033899fb78ddfd402"
+  name = "github.com/tmc/grpc-websocket-proxy"
+  packages = ["wsproxy"]
+  pruneopts = "UT"
+  revision = "89b8d40f7ca833297db804fcb3be53a76d01c238"
+
+[[projects]]
+  digest = "1:03aa6e485e528acb119fb32901cf99582c380225fc7d5a02758e08b180cb56c3"
+  name = "github.com/ugorji/go"
+  packages = ["codec"]
+  pruneopts = "UT"
+  revision = "b4c50a2b199d93b13dc15e78929cfb23bfdf21ab"
+  version = "v1.1.1"
+
+[[projects]]
+  digest = "1:46ed28d956228a4113be4e08ba8fae6d78b154bbf78cb7c2108ab157faaac8dc"
+  name = "github.com/urfave/cli"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "1efa31f08b9333f1bd4882d61f9d668a70cd902e"
+  version = "v1.18.0"
+
+[[projects]]
+  digest = "1:6dff6d02950c110d7d61da0c200eaff9da9f312101291b2d8c07235954eaa19d"
+  name = "github.com/xiang90/probing"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "07dd2e8dfe18522e9c447ba95f2fe95262f63bb2"
+  version = "0.0.1"
+
+[[projects]]
+  digest = "1:3c1a69cdae3501bf75e76d0d86dc6f2b0a7421bc205c0cb7b96b19eed464a34d"
+  name = "go.uber.org/atomic"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "1ea20fb1cbb1cc08cbd0d913a96dead89aa18289"
+  version = "v1.3.2"
+
+[[projects]]
+  digest = "1:60bf2a5e347af463c42ed31a493d817f8a72f102543060ed992754e689805d1a"
+  name = "go.uber.org/multierr"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "3c4937480c32f4c13a875a1829af76c98ca3d40a"
+  version = "v1.1.0"
+
+[[projects]]
+  digest = "1:9580b1b079114140ade8cec957685344d14f00119e0241f6b369633cb346eeb3"
+  name = "go.uber.org/zap"
+  packages = [
+    ".",
+    "buffer",
+    "internal/bufferpool",
+    "internal/color",
+    "internal/exit",
+    "zapcore",
+  ]
+  pruneopts = "UT"
+  revision = "eeedf312bc6c57391d84767a4cd413f02a917974"
+  version = "v1.8.0"
+
+[[projects]]
+  branch = "master"
+  digest = "1:001a4e7a40e50ff2ef32e2556bca50c4f77daa457db3ac6afc8bea9bb2122cfb"
+  name = "golang.org/x/crypto"
+  packages = [
+    "bcrypt",
+    "blowfish",
+    "ssh/terminal",
+  ]
+  pruneopts = "UT"
+  revision = "8ac0e0d97ce45cd83d1d7243c060cb8461dda5e9"
+
+[[projects]]
+  branch = "master"
+  digest = "1:fc34fe79cf436681e18fdfad303dbe6a9caa69e8058006e8819ba9660f8e5fb5"
+  name = "golang.org/x/net"
+  packages = [
+    "context",
+    "http/httpguts",
+    "http2",
+    "http2/hpack",
+    "idna",
+    "internal/timeseries",
+    "trace",
+  ]
+  pruneopts = "UT"
+  revision = "db08ff08e8622530d9ed3a0e8ac279f6d4c02196"
+
+[[projects]]
+  branch = "master"
+  digest = "1:7e6d7e1a44eeefc8cd35d6445042f246702656a6597987db850c30a11f2d65d5"
+  name = "golang.org/x/sys"
+  packages = [
+    "unix",
+    "windows",
+  ]
+  pruneopts = "UT"
+  revision = "56ede360ec1c541828fb88741b3f1049406d28f5"
+
+[[projects]]
+  digest = "1:a2ab62866c75542dd18d2b069fec854577a20211d7c0ea6ae746072a1dccdd18"
+  name = "golang.org/x/text"
+  packages = [
+    "collate",
+    "collate/build",
+    "internal/colltab",
+    "internal/gen",
+    "internal/tag",
+    "internal/triegen",
+    "internal/ucd",
+    "language",
+    "secure/bidirule",
+    "transform",
+    "unicode/bidi",
+    "unicode/cldr",
+    "unicode/norm",
+    "unicode/rangetable",
+  ]
+  pruneopts = "UT"
+  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
+  version = "v0.3.0"
+
+[[projects]]
+  branch = "master"
+  digest = "1:c9e7a4b4d47c0ed205d257648b0e5b0440880cb728506e318f8ac7cd36270bc4"
+  name = "golang.org/x/time"
+  packages = ["rate"]
+  pruneopts = "UT"
+  revision = "fbb02b2291d28baffd63558aa44b4b56f178d650"
+
+[[projects]]
+  branch = "master"
+  digest = "1:601e63e7d4577f907118bec825902505291918859d223bce015539e79f1160e3"
+  name = "google.golang.org/genproto"
+  packages = ["googleapis/rpc/status"]
+  pruneopts = "UT"
+  revision = "32ee49c4dd805befd833990acba36cb75042378c"
+
+[[projects]]
+  digest = "1:047efbc3c9a51f3002b0002f92543857d372654a676fb6b01931982cd80467dd"
+  name = "google.golang.org/grpc"
+  packages = [
+    ".",
+    "balancer",
+    "balancer/base",
+    "balancer/roundrobin",
+    "codes",
+    "connectivity",
+    "credentials",
+    "encoding",
+    "encoding/proto",
+    "grpclog",
+    "health",
+    "health/grpc_health_v1",
+    "internal",
+    "internal/backoff",
+    "internal/channelz",
+    "internal/envconfig",
+    "internal/grpcrand",
+    "internal/transport",
+    "keepalive",
+    "metadata",
+    "naming",
+    "peer",
+    "resolver",
+    "resolver/dns",
+    "resolver/passthrough",
+    "stats",
+    "status",
+    "tap",
+  ]
+  pruneopts = "UT"
+  revision = "32fb0ac620c32ba40a4626ddf94d90d12cce3455"
+  version = "v1.14.0"
+
+[[projects]]
+  digest = "1:d8cd4f14785b5ae65100524a29ebba8b9dfc5401020fe7504f80b438bb8e8e0d"
+  name = "gopkg.in/cheggaaa/pb.v1"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "2af8bbdea9e99e83b3ac400d8f6b6d1b8cbbf338"
+  version = "v1.0.25"
+
+[[projects]]
+  digest = "1:342378ac4dcb378a5448dd723f0784ae519383532f5e70ade24132c4c8693202"
+  name = "gopkg.in/yaml.v2"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
+  version = "v2.2.1"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  input-imports = [
+    "github.com/bgentry/speakeasy",
+    "github.com/coreos/bbolt",
+    "github.com/coreos/go-semver/semver",
+    "github.com/coreos/go-systemd/daemon",
+    "github.com/coreos/go-systemd/journal",
+    "github.com/coreos/go-systemd/util",
+    "github.com/coreos/pkg/capnslog",
+    "github.com/dgrijalva/jwt-go",
+    "github.com/dustin/go-humanize",
+    "github.com/ghodss/yaml",
+    "github.com/gogo/protobuf/gogoproto",
+    "github.com/gogo/protobuf/proto",
+    "github.com/golang/groupcache/lru",
+    "github.com/golang/protobuf/proto",
+    "github.com/google/btree",
+    "github.com/grpc-ecosystem/go-grpc-middleware",
+    "github.com/grpc-ecosystem/go-grpc-middleware/util/backoffutils",
+    "github.com/grpc-ecosystem/go-grpc-prometheus",
+    "github.com/grpc-ecosystem/grpc-gateway/runtime",
+    "github.com/grpc-ecosystem/grpc-gateway/utilities",
+    "github.com/jonboulle/clockwork",
+    "github.com/kr/pty",
+    "github.com/olekukonko/tablewriter",
+    "github.com/prometheus/client_golang/prometheus",
+    "github.com/prometheus/client_golang/prometheus/promhttp",
+    "github.com/prometheus/client_model/go",
+    "github.com/soheilhy/cmux",
+    "github.com/spf13/cobra",
+    "github.com/spf13/pflag",
+    "github.com/tmc/grpc-websocket-proxy/wsproxy",
+    "github.com/ugorji/go/codec",
+    "github.com/urfave/cli",
+    "github.com/xiang90/probing",
+    "go.uber.org/zap",
+    "go.uber.org/zap/zapcore",
+    "golang.org/x/crypto/bcrypt",
+    "golang.org/x/net/context",
+    "golang.org/x/net/http2",
+    "golang.org/x/net/trace",
+    "golang.org/x/time/rate",
+    "google.golang.org/grpc",
+    "google.golang.org/grpc/balancer",
+    "google.golang.org/grpc/codes",
+    "google.golang.org/grpc/connectivity",
+    "google.golang.org/grpc/credentials",
+    "google.golang.org/grpc/grpclog",
+    "google.golang.org/grpc/health",
+    "google.golang.org/grpc/health/grpc_health_v1",
+    "google.golang.org/grpc/keepalive",
+    "google.golang.org/grpc/metadata",
+    "google.golang.org/grpc/naming",
+    "google.golang.org/grpc/peer",
+    "google.golang.org/grpc/resolver",
+    "google.golang.org/grpc/resolver/dns",
+    "google.golang.org/grpc/resolver/passthrough",
+    "google.golang.org/grpc/status",
+    "gopkg.in/cheggaaa/pb.v1",
+    "gopkg.in/yaml.v2",
+  ]
+  solver-name = "gps-cdcl"
+  solver-version = 1

Gopkg.toml

@@ -0,0 +1,3 @@
+[prune]
+  go-tests = true
+  unused-packages = true

MAINTAINERS

@@ -1,8 +1,17 @@
-Anthony Romano <anthony.romano@coreos.com> (@heyitsanthony) pkg:*
-Brandon Philips <brandon.philips@coreos.com> (@philips) pkg:*
-Fanmin Shi <fanmin.shi@coreos.com> (@fanminshi) pkg:*
-Gyu-Ho Lee <gyu_ho.lee@coreos.com> (@gyuho) pkg:*
-Xiang Li <xiang.li@coreos.com> (@xiang90) pkg:*
+# This is the official list of etcd maintainers.
+#
+# Names should be added to this file like so:
+#     Individual's name <submission email address> (@GITHUB_HANDLE) pkg:*
+#     Individual's name <submission email address> <email2> <emailN> (@GITHUB_HANDLE) pkg:*
+#
+# Please keep the list sorted.
+
+Anthony Romano <romanoanthony061@gmail.com> (@heyitsanthony) pkg:*
+Brandon Philips <bphilips@redhat.com> (@philips) pkg:*
+Fanmin Shi <fashi@redhat.com> (@fanminshi) pkg:*
+Gyuho Lee <gyuhox@gmail.com> <leegyuho@amazon.com> (@gyuho) pkg:*
+Hitoshi Mitake <h.mitake@gmail.com> (@mitake) pkg:*
+Joe Betz <jpbetz@google.com> (@jpbetz) pkg:*
+Xiang Li <xiangli.cs@gmail.com> (@xiang90) pkg:*
 
 Ben Darnell <ben@cockroachlabs.com> (@bdarnell) pkg:github.com/coreos/etcd/raft
-Hitoshi Mitake <mitake.hitoshi@lab.ntt.co.jp> (@mitake) pkg:github.com/coreos/etcd/auth

MAINTAINERS_RULES.md

@@ -0,0 +1,16 @@
+
+This document describes basic expectations for maintainers. To become a maintainer, start taking on these responsibilities. Consistent contributors then discuss with existing maintainers to become the official [MAINTAINERS](./MAINTAINERS).
+
+### Top-level maintainer
+
+- Running the etcd release processes
+- Ownership of test and debug infrastructure
+- Resolve or redirect issues to keep the issue count low (goal: under 100)
+- Regularly review pull requests across all pkgs
+- Providing cross pkg design review
+
+### Package/feature maintainer
+
+- Ownership of test and debug failures in a pkg/feature
+- Resolution of bugs triaged to a package/feature
+- Regularly review pull requests to the pkg subsystem

Makefile

@@ -0,0 +1,518 @@
+# run from repository root
+
+
+
+# Example:
+#   make build
+#   make clean
+#   make docker-clean
+#   make docker-start
+#   make docker-kill
+#   make docker-remove
+
+.PHONY: build
+build:
+	GO_BUILD_FLAGS="-v" ./build
+	./bin/etcd --version
+	ETCDCTL_API=3 ./bin/etcdctl version
+
+clean:
+	rm -f ./codecov
+	rm -rf ./agent-*
+	rm -rf ./covdir
+	rm -f ./*.coverprofile
+	rm -f ./*.log
+	rm -f ./bin/Dockerfile-release
+	rm -rf ./bin/*.etcd
+	rm -rf ./default.etcd
+	rm -rf ./tests/e2e/default.etcd
+	rm -rf ./gopath
+	rm -rf ./gopath.proto
+	rm -rf ./release
+	rm -f ./snapshot/localhost:*
+	rm -f ./tools/etcd-dump-metrics/localhost:*
+	rm -f ./integration/127.0.0.1:* ./integration/localhost:*
+	rm -f ./clientv3/integration/127.0.0.1:* ./clientv3/integration/localhost:*
+	rm -f ./clientv3/ordering/127.0.0.1:* ./clientv3/ordering/localhost:*
+
+docker-clean:
+	docker images
+	docker image prune --force
+
+docker-start:
+	service docker restart
+
+docker-kill:
+	docker kill `docker ps -q` || true
+
+docker-remove:
+	docker rm --force `docker ps -a -q` || true
+	docker rmi --force `docker images -q` || true
+
+
+
+GO_VERSION ?= 1.10.3
+ETCD_VERSION ?= $(shell git rev-parse --short HEAD || echo "GitNotFound")
+
+TEST_SUFFIX = $(shell date +%s | base64 | head -c 15)
+TEST_OPTS ?= PASSES='unit'
+
+TMP_DIR_MOUNT_FLAG = --mount type=tmpfs,destination=/tmp
+ifdef HOST_TMP_DIR
+	TMP_DIR_MOUNT_FLAG = --mount type=bind,source=$(HOST_TMP_DIR),destination=/tmp
+endif
+
+
+
+# Example:
+#   GO_VERSION=1.8.7 make build-docker-test
+#   make build-docker-test
+#
+#   gcloud docker -- login -u _json_key -p "$(cat /etc/gcp-key-etcd-development.json)" https://gcr.io
+#   GO_VERSION=1.8.7 make push-docker-test
+#   make push-docker-test
+#
+#   gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
+#   make pull-docker-test
+
+build-docker-test:
+	$(info GO_VERSION: $(GO_VERSION))
+	@sed -i.bak 's|REPLACE_ME_GO_VERSION|$(GO_VERSION)|g' ./tests/Dockerfile
+	docker build \
+	  --tag gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \
+	  --file ./tests/Dockerfile .
+	@mv ./tests/Dockerfile.bak ./tests/Dockerfile
+
+push-docker-test:
+	$(info GO_VERSION: $(GO_VERSION))
+	gcloud docker -- push gcr.io/etcd-development/etcd-test:go$(GO_VERSION)
+
+pull-docker-test:
+	$(info GO_VERSION: $(GO_VERSION))
+	docker pull gcr.io/etcd-development/etcd-test:go$(GO_VERSION)
+
+
+
+# Example:
+#   make build-docker-test
+#   make compile-with-docker-test
+#   make compile-setup-gopath-with-docker-test
+
+compile-with-docker-test:
+	$(info GO_VERSION: $(GO_VERSION))
+	docker run \
+	  --rm \
+	  --mount type=bind,source=`pwd`,destination=/go/src/github.com/coreos/etcd \
+	  gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \
+	  /bin/bash -c "GO_BUILD_FLAGS=-v ./build && ./bin/etcd --version"
+
+compile-setup-gopath-with-docker-test:
+	$(info GO_VERSION: $(GO_VERSION))
+	docker run \
+	  --rm \
+	  --mount type=bind,source=`pwd`,destination=/etcd \
+	  gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \
+	  /bin/bash -c "cd /etcd && ETCD_SETUP_GOPATH=1 GO_BUILD_FLAGS=-v ./build && ./bin/etcd --version && rm -rf ./gopath"
+
+
+
+# Example:
+#
+# Local machine:
+#   TEST_OPTS="PASSES='fmt'" make test
+#   TEST_OPTS="PASSES='fmt bom dep build unit'" make test
+#   TEST_OPTS="PASSES='build unit release integration_e2e functional'" make test
+#   TEST_OPTS="PASSES='build grpcproxy'" make test
+#
+# Example (test with docker):
+#   make pull-docker-test
+#   TEST_OPTS="PASSES='fmt'" make docker-test
+#   TEST_OPTS="VERBOSE=2 PASSES='unit'" make docker-test
+#
+# Travis CI (test with docker):
+#   TEST_OPTS="PASSES='fmt bom dep build unit'" make docker-test
+#
+# Semaphore CI (test with docker):
+#   TEST_OPTS="PASSES='build unit release integration_e2e functional'" make docker-test
+#   HOST_TMP_DIR=/tmp TEST_OPTS="PASSES='build unit release integration_e2e functional'" make docker-test
+#   TEST_OPTS="GOARCH=386 PASSES='build unit integration_e2e'" make docker-test
+#
+# grpc-proxy tests (test with docker):
+#   TEST_OPTS="PASSES='build grpcproxy'" make docker-test
+#   HOST_TMP_DIR=/tmp TEST_OPTS="PASSES='build grpcproxy'" make docker-test
+
+.PHONY: test
+test:
+	$(info TEST_OPTS: $(TEST_OPTS))
+	$(info log-file: test-$(TEST_SUFFIX).log)
+	$(TEST_OPTS) ./test 2>&1 | tee test-$(TEST_SUFFIX).log
+	! egrep "(--- FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test-$(TEST_SUFFIX).log
+
+docker-test:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info ETCD_VERSION: $(ETCD_VERSION))
+	$(info TEST_OPTS: $(TEST_OPTS))
+	$(info log-file: test-$(TEST_SUFFIX).log)
+	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
+	docker run \
+	  --rm \
+	  $(TMP_DIR_MOUNT_FLAG) \
+	  --mount type=bind,source=`pwd`,destination=/go/src/github.com/coreos/etcd \
+	  gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \
+	  /bin/bash -c "$(TEST_OPTS) ./test 2>&1 | tee test-$(TEST_SUFFIX).log"
+	! egrep "(--- FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test-$(TEST_SUFFIX).log
+
+docker-test-coverage:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info ETCD_VERSION: $(ETCD_VERSION))
+	$(info log-file: docker-test-coverage-$(TEST_SUFFIX).log)
+	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
+	docker run \
+	  --rm \
+	  $(TMP_DIR_MOUNT_FLAG) \
+	  --mount type=bind,source=`pwd`,destination=/go/src/github.com/coreos/etcd \
+	  gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \
+	  /bin/bash -c "COVERDIR=covdir PASSES='build build_cov cov' ./test 2>&1 | tee docker-test-coverage-$(TEST_SUFFIX).log && /codecov -t 6040de41-c073-4d6f-bbf8-d89256ef31e1"
+	! egrep "(--- FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 docker-test-coverage-$(TEST_SUFFIX).log
+
+
+
+# Example:
+#   make compile-with-docker-test
+#   ETCD_VERSION=v3-test make build-docker-release-master
+#   ETCD_VERSION=v3-test make push-docker-release-master
+#   gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
+
+build-docker-release-master:
+	$(info ETCD_VERSION: $(ETCD_VERSION))
+	cp ./Dockerfile-release ./bin/Dockerfile-release
+	docker build \
+	  --tag gcr.io/etcd-development/etcd:$(ETCD_VERSION) \
+	  --file ./bin/Dockerfile-release \
+	  ./bin
+	rm -f ./bin/Dockerfile-release
+
+	docker run \
+	  --rm \
+	  gcr.io/etcd-development/etcd:$(ETCD_VERSION) \
+	  /bin/sh -c "/usr/local/bin/etcd --version && ETCDCTL_API=3 /usr/local/bin/etcdctl version"
+
+push-docker-release-master:
+	$(info ETCD_VERSION: $(ETCD_VERSION))
+	gcloud docker -- push gcr.io/etcd-development/etcd:$(ETCD_VERSION)
+
+
+
+# Example:
+#   make build-docker-test
+#   make compile-with-docker-test
+#   make build-docker-static-ip-test
+#
+#   gcloud docker -- login -u _json_key -p "$(cat /etc/gcp-key-etcd-development.json)" https://gcr.io
+#   make push-docker-static-ip-test
+#
+#   gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
+#   make pull-docker-static-ip-test
+#
+#   make docker-static-ip-test-certs-run
+#   make docker-static-ip-test-certs-metrics-proxy-run
+
+build-docker-static-ip-test:
+	$(info GO_VERSION: $(GO_VERSION))
+	@sed -i.bak 's|REPLACE_ME_GO_VERSION|$(GO_VERSION)|g' ./tests/docker-static-ip/Dockerfile
+	docker build \
+	  --tag gcr.io/etcd-development/etcd-static-ip-test:go$(GO_VERSION) \
+	  --file ./tests/docker-static-ip/Dockerfile \
+	  ./tests/docker-static-ip
+	@mv ./tests/docker-static-ip/Dockerfile.bak ./tests/docker-static-ip/Dockerfile
+
+push-docker-static-ip-test:
+	$(info GO_VERSION: $(GO_VERSION))
+	gcloud docker -- push gcr.io/etcd-development/etcd-static-ip-test:go$(GO_VERSION)
+
+pull-docker-static-ip-test:
+	$(info GO_VERSION: $(GO_VERSION))
+	docker pull gcr.io/etcd-development/etcd-static-ip-test:go$(GO_VERSION)
+
+docker-static-ip-test-certs-run:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
+	docker run \
+	  --rm \
+	  --tty \
+	  $(TMP_DIR_MOUNT_FLAG) \
+	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
+	  --mount type=bind,source=`pwd`/tests/docker-static-ip/certs,destination=/certs \
+	  gcr.io/etcd-development/etcd-static-ip-test:go$(GO_VERSION) \
+	  /bin/bash -c "cd /etcd && /certs/run.sh && rm -rf m*.etcd"
+
+docker-static-ip-test-certs-metrics-proxy-run:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
+	docker run \
+	  --rm \
+	  --tty \
+	  $(TMP_DIR_MOUNT_FLAG) \
+	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
+	  --mount type=bind,source=`pwd`/tests/docker-static-ip/certs-metrics-proxy,destination=/certs-metrics-proxy \
+	  gcr.io/etcd-development/etcd-static-ip-test:go$(GO_VERSION) \
+	  /bin/bash -c "cd /etcd && /certs-metrics-proxy/run.sh && rm -rf m*.etcd"
+
+
+
+# Example:
+#   make build-docker-test
+#   make compile-with-docker-test
+#   make build-docker-dns-test
+#
+#   gcloud docker -- login -u _json_key -p "$(cat /etc/gcp-key-etcd-development.json)" https://gcr.io
+#   make push-docker-dns-test
+#
+#   gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
+#   make pull-docker-dns-test
+#
+#   make docker-dns-test-insecure-run
+#   make docker-dns-test-certs-run
+#   make docker-dns-test-certs-gateway-run
+#   make docker-dns-test-certs-wildcard-run
+#   make docker-dns-test-certs-common-name-auth-run
+#   make docker-dns-test-certs-common-name-multi-run
+
+build-docker-dns-test:
+	$(info GO_VERSION: $(GO_VERSION))
+	@sed -i.bak 's|REPLACE_ME_GO_VERSION|$(GO_VERSION)|g' ./tests/docker-dns/Dockerfile
+	docker build \
+	  --tag gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
+	  --file ./tests/docker-dns/Dockerfile \
+	  ./tests/docker-dns
+	@mv ./tests/docker-dns/Dockerfile.bak ./tests/docker-dns/Dockerfile
+
+	docker run \
+	  --rm \
+	  --dns 127.0.0.1 \
+	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
+	  /bin/bash -c "/etc/init.d/bind9 start && cat /dev/null >/etc/hosts && dig etcd.local"
+
+push-docker-dns-test:
+	$(info GO_VERSION: $(GO_VERSION))
+	gcloud docker -- push gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION)
+
+pull-docker-dns-test:
+	$(info GO_VERSION: $(GO_VERSION))
+	docker pull gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION)
+
+docker-dns-test-insecure-run:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
+	docker run \
+	  --rm \
+	  --tty \
+	  --dns 127.0.0.1 \
+	  $(TMP_DIR_MOUNT_FLAG) \
+	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
+	  --mount type=bind,source=`pwd`/tests/docker-dns/insecure,destination=/insecure \
+	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
+	  /bin/bash -c "cd /etcd && /insecure/run.sh && rm -rf m*.etcd"
+
+docker-dns-test-certs-run:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
+	docker run \
+	  --rm \
+	  --tty \
+	  --dns 127.0.0.1 \
+	  $(TMP_DIR_MOUNT_FLAG) \
+	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
+	  --mount type=bind,source=`pwd`/tests/docker-dns/certs,destination=/certs \
+	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
+	  /bin/bash -c "cd /etcd && /certs/run.sh && rm -rf m*.etcd"
+
+docker-dns-test-certs-gateway-run:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
+	docker run \
+	  --rm \
+	  --tty \
+	  --dns 127.0.0.1 \
+	  $(TMP_DIR_MOUNT_FLAG) \
+	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
+	  --mount type=bind,source=`pwd`/tests/docker-dns/certs-gateway,destination=/certs-gateway \
+	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
+	  /bin/bash -c "cd /etcd && /certs-gateway/run.sh && rm -rf m*.etcd"
+
+docker-dns-test-certs-wildcard-run:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
+	docker run \
+	  --rm \
+	  --tty \
+	  --dns 127.0.0.1 \
+	  $(TMP_DIR_MOUNT_FLAG) \
+	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
+	  --mount type=bind,source=`pwd`/tests/docker-dns/certs-wildcard,destination=/certs-wildcard \
+	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
+	  /bin/bash -c "cd /etcd && /certs-wildcard/run.sh && rm -rf m*.etcd"
+
+docker-dns-test-certs-common-name-auth-run:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
+	docker run \
+	  --rm \
+	  --tty \
+	  --dns 127.0.0.1 \
+	  $(TMP_DIR_MOUNT_FLAG) \
+	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
+	  --mount type=bind,source=`pwd`/tests/docker-dns/certs-common-name-auth,destination=/certs-common-name-auth \
+	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
+	  /bin/bash -c "cd /etcd && /certs-common-name-auth/run.sh && rm -rf m*.etcd"
+
+docker-dns-test-certs-common-name-multi-run:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
+	docker run \
+	  --rm \
+	  --tty \
+	  --dns 127.0.0.1 \
+	  $(TMP_DIR_MOUNT_FLAG) \
+	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
+	  --mount type=bind,source=`pwd`/tests/docker-dns/certs-common-name-multi,destination=/certs-common-name-multi \
+	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
+	  /bin/bash -c "cd /etcd && /certs-common-name-multi/run.sh && rm -rf m*.etcd"
+
+
+
+# Example:
+#   make build-docker-test
+#   make compile-with-docker-test
+#   make build-docker-dns-srv-test
+#   gcloud docker -- login -u _json_key -p "$(cat /etc/gcp-key-etcd-development.json)" https://gcr.io
+#   make push-docker-dns-srv-test
+#   gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
+#   make pull-docker-dns-srv-test
+#   make docker-dns-srv-test-certs-run
+#   make docker-dns-srv-test-certs-gateway-run
+#   make docker-dns-srv-test-certs-wildcard-run
+
+build-docker-dns-srv-test:
+	$(info GO_VERSION: $(GO_VERSION))
+	@sed -i.bak 's|REPLACE_ME_GO_VERSION|$(GO_VERSION)|g' ./tests/docker-dns-srv/Dockerfile
+	docker build \
+	  --tag gcr.io/etcd-development/etcd-dns-srv-test:go$(GO_VERSION) \
+	  --file ./tests/docker-dns-srv/Dockerfile \
+	  ./tests/docker-dns-srv
+	@mv ./tests/docker-dns-srv/Dockerfile.bak ./tests/docker-dns-srv/Dockerfile
+
+	docker run \
+	  --rm \
+	  --dns 127.0.0.1 \
+	  gcr.io/etcd-development/etcd-dns-srv-test:go$(GO_VERSION) \
+	  /bin/bash -c "/etc/init.d/bind9 start && cat /dev/null >/etc/hosts && dig +noall +answer SRV _etcd-client-ssl._tcp.etcd.local && dig +noall +answer SRV _etcd-server-ssl._tcp.etcd.local && dig +noall +answer m1.etcd.local m2.etcd.local m3.etcd.local"
+
+push-docker-dns-srv-test:
+	$(info GO_VERSION: $(GO_VERSION))
+	gcloud docker -- push gcr.io/etcd-development/etcd-dns-srv-test:go$(GO_VERSION)
+
+pull-docker-dns-srv-test:
+	$(info GO_VERSION: $(GO_VERSION))
+	docker pull gcr.io/etcd-development/etcd-dns-srv-test:go$(GO_VERSION)
+
+docker-dns-srv-test-certs-run:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
+	docker run \
+	  --rm \
+	  --tty \
+	  --dns 127.0.0.1 \
+	  $(TMP_DIR_MOUNT_FLAG) \
+	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
+	  --mount type=bind,source=`pwd`/tests/docker-dns-srv/certs,destination=/certs \
+	  gcr.io/etcd-development/etcd-dns-srv-test:go$(GO_VERSION) \
+	  /bin/bash -c "cd /etcd && /certs/run.sh && rm -rf m*.etcd"
+
+docker-dns-srv-test-certs-gateway-run:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
+	docker run \
+	  --rm \
+	  --tty \
+	  --dns 127.0.0.1 \
+	  $(TMP_DIR_MOUNT_FLAG) \
+	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
+	  --mount type=bind,source=`pwd`/tests/docker-dns-srv/certs-gateway,destination=/certs-gateway \
+	  gcr.io/etcd-development/etcd-dns-srv-test:go$(GO_VERSION) \
+	  /bin/bash -c "cd /etcd && /certs-gateway/run.sh && rm -rf m*.etcd"
+
+docker-dns-srv-test-certs-wildcard-run:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
+	docker run \
+	  --rm \
+	  --tty \
+	  --dns 127.0.0.1 \
+	  $(TMP_DIR_MOUNT_FLAG) \
+	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
+	  --mount type=bind,source=`pwd`/tests/docker-dns-srv/certs-wildcard,destination=/certs-wildcard \
+	  gcr.io/etcd-development/etcd-dns-srv-test:go$(GO_VERSION) \
+	  /bin/bash -c "cd /etcd && /certs-wildcard/run.sh && rm -rf m*.etcd"
+
+
+
+# Example:
+#   make build-functional
+#   make build-docker-functional
+#   make push-docker-functional
+#   make pull-docker-functional
+
+build-functional:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info ETCD_VERSION: $(ETCD_VERSION))
+	./functional/build
+	./bin/etcd-agent -help || true && \
+	  ./bin/etcd-proxy -help || true && \
+	  ./bin/etcd-runner --help || true && \
+	  ./bin/etcd-tester -help || true
+
+build-docker-functional:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info ETCD_VERSION: $(ETCD_VERSION))
+	@sed -i.bak 's|REPLACE_ME_GO_VERSION|$(GO_VERSION)|g' ./functional/Dockerfile
+	docker build \
+	  --tag gcr.io/etcd-development/etcd-functional:go$(GO_VERSION) \
+	  --file ./functional/Dockerfile \
+	  .
+	@mv ./functional/Dockerfile.bak ./functional/Dockerfile
+
+	docker run \
+	  --rm \
+	  gcr.io/etcd-development/etcd-functional:go$(GO_VERSION) \
+	  /bin/bash -c "./bin/etcd --version && \
+	   ./bin/etcd-failpoints --version && \
+	   ETCDCTL_API=3 ./bin/etcdctl version && \
+	   ./bin/etcd-agent -help || true && \
+	   ./bin/etcd-proxy -help || true && \
+	   ./bin/etcd-runner --help || true && \
+	   ./bin/etcd-tester -help || true && \
+	   ./bin/benchmark --help || true"
+
+push-docker-functional:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info ETCD_VERSION: $(ETCD_VERSION))
+	gcloud docker -- push gcr.io/etcd-development/etcd-functional:go$(GO_VERSION)
+
+pull-docker-functional:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info ETCD_VERSION: $(ETCD_VERSION))
+	docker pull gcr.io/etcd-development/etcd-functional:go$(GO_VERSION)

Procfile

@@ -1,5 +1,5 @@
 # Use goreman to run `go get github.com/mattn/goreman`
-etcd1: bin/etcd --name infra1 --listen-client-urls http://127.0.0.1:2379 --advertise-client-urls http://127.0.0.1:2379 --listen-peer-urls http://127.0.0.1:12380 --initial-advertise-peer-urls http://127.0.0.1:12380 --initial-cluster-token etcd-cluster-1 --initial-cluster 'infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380' --initial-cluster-state new --enable-pprof
-etcd2: bin/etcd --name infra2 --listen-client-urls http://127.0.0.1:22379 --advertise-client-urls http://127.0.0.1:22379 --listen-peer-urls http://127.0.0.1:22380 --initial-advertise-peer-urls http://127.0.0.1:22380 --initial-cluster-token etcd-cluster-1 --initial-cluster 'infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380' --initial-cluster-state new --enable-pprof
-etcd3: bin/etcd --name infra3 --listen-client-urls http://127.0.0.1:32379 --advertise-client-urls http://127.0.0.1:32379 --listen-peer-urls http://127.0.0.1:32380 --initial-advertise-peer-urls http://127.0.0.1:32380 --initial-cluster-token etcd-cluster-1 --initial-cluster 'infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380' --initial-cluster-state new --enable-pprof
+etcd1: bin/etcd --name infra1 --listen-client-urls http://127.0.0.1:2379 --advertise-client-urls http://127.0.0.1:2379 --listen-peer-urls http://127.0.0.1:12380 --initial-advertise-peer-urls http://127.0.0.1:12380 --initial-cluster-token etcd-cluster-1 --initial-cluster 'infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380' --initial-cluster-state new --enable-pprof --logger=zap --log-outputs=stderr
+etcd2: bin/etcd --name infra2 --listen-client-urls http://127.0.0.1:22379 --advertise-client-urls http://127.0.0.1:22379 --listen-peer-urls http://127.0.0.1:22380 --initial-advertise-peer-urls http://127.0.0.1:22380 --initial-cluster-token etcd-cluster-1 --initial-cluster 'infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380' --initial-cluster-state new --enable-pprof --logger=zap --log-outputs=stderr
+etcd3: bin/etcd --name infra3 --listen-client-urls http://127.0.0.1:32379 --advertise-client-urls http://127.0.0.1:32379 --listen-peer-urls http://127.0.0.1:32380 --initial-advertise-peer-urls http://127.0.0.1:32380 --initial-cluster-token etcd-cluster-1 --initial-cluster 'infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380' --initial-cluster-state new --enable-pprof --logger=zap --log-outputs=stderr
 #proxy: bin/etcd grpc-proxy start --endpoints=127.0.0.1:2379,127.0.0.1:22379,127.0.0.1:32379 --listen-addr=127.0.0.1:23790 --advertise-client-url=127.0.0.1:23790 --enable-pprof

README.md

@@ -1,9 +1,15 @@
+
+
+**WARNING**: etcd will be transferred from https://github.com/coreos to https://github.com/etcd-io, on 5PM PST, Mon, August 27, 2018. See https://github.com/coreos/etcd/issues/9965 for detailed timeline.
+
+
 # etcd
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/coreos/etcd?style=flat-square)](https://goreportcard.com/report/github.com/coreos/etcd)
 [![Coverage](https://codecov.io/gh/coreos/etcd/branch/master/graph/badge.svg)](https://codecov.io/gh/coreos/etcd)
 [![Build Status Travis](https://img.shields.io/travis/coreos/etcdlabs.svg?style=flat-square&&branch=master)](https://travis-ci.org/coreos/etcd)
 [![Build Status Semaphore](https://semaphoreci.com/api/v1/coreos/etcd/branches/master/shields_badge.svg)](https://semaphoreci.com/coreos/etcd)
+[![Docs](https://readthedocs.org/projects/etcd/badge/?version=latest&style=flat-square)](https://etcd.readthedocs.io/en/latest/?badge=latest)
 [![Godoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godoc.org/github.com/coreos/etcd)
 [![Releases](https://img.shields.io/github/release/coreos/etcd/all.svg?style=flat-square)](https://github.com/coreos/etcd/releases)
 [![LICENSE](https://img.shields.io/github/license/coreos/etcd.svg?style=flat-square)](https://github.com/coreos/etcd/blob/master/LICENSE)
@@ -12,7 +18,7 @@
 
 *the etcd v2 [documentation](Documentation/v2/README.md) has moved*
 
-![etcd Logo](logos/etcd-horizontal-color.png)
+![etcd Logo](logos/etcd-horizontal-color.svg)
 
 etcd is a distributed reliable key-value store for the most critical data of a distributed system, with a focus on being:
 
@@ -23,37 +29,35 @@ etcd is a distributed reliable key-value store for the most critical data of a d
 
 etcd is written in Go and uses the [Raft][raft] consensus algorithm to manage a highly-available replicated log.
 
-etcd is used [in production by many companies](./Documentation/production-users.md), and the development team stands behind it in critical deployment scenarios, where etcd is frequently teamed with applications such as [Kubernetes][k8s], [fleet][fleet], [locksmith][locksmith], [vulcand][vulcand], [Doorman][doorman], and many others. Reliability is further ensured by rigorous [testing][etcd-tests].
+etcd is used [in production by many companies](./Documentation/production-users.md), and the development team stands behind it in critical deployment scenarios, where etcd is frequently teamed with applications such as [Kubernetes][k8s], [locksmith][locksmith], [vulcand][vulcand], [Doorman][doorman], and many others. Reliability is further ensured by [**rigorous testing**](https://github.com/coreos/etcd/tree/master/functional).
 
 See [etcdctl][etcdctl] for a simple command line client.
 
 [raft]: https://raft.github.io/
 [k8s]: http://kubernetes.io/
 [doorman]: https://github.com/youtube/doorman
-[fleet]: https://github.com/coreos/fleet
 [locksmith]: https://github.com/coreos/locksmith
 [vulcand]: https://github.com/vulcand/vulcand
 [etcdctl]: https://github.com/coreos/etcd/tree/master/etcdctl
-[etcd-tests]: http://dash.etcd.io
 
 ## Community meetings
 
-etcd contributors and maintainers have bi-weekly meetings at 11:00 AM (USA Pacific) on Tuesdays. There is an [iCalendar][rfc5545] format for the meetings [here](meeting.ics). Anyone is welcome to join via [Zoom][zoom] or audio-only: +1 669 900 6833. An initial agenda will be posted to the [shared Google docs][shared-meeting-notes] a day before each meeting, and everyone is welcome to suggest additional topics or other agendas.
+etcd contributors and maintainers have bi-weekly meetings at 11:00 AM (USA Pacific) on Tuesdays. Anyone is welcome to join via [BlueJeans][bluejeans]. An initial agenda will be posted to the [shared Google docs][shared-meeting-notes] a day before each meeting, and everyone is welcome to suggest additional topics or other agendas.
 
-[rfc5545]: https://tools.ietf.org/html/rfc5545
-[zoom]: https://coreos.zoom.us/j/854793406
-[shared-meeting-notes]: https://docs.google.com/document/d/1DbVXOHvd9scFsSmL2oNg4YGOHJdXqtx583DmeVWrB_M/edit#
+[bluejeans]: https://bluejeans.com/2732309610
+[shared-meeting-notes]: https://docs.google.com/document/d/16XEGyPBisZvmmoIHSZzv__LoyOeluC5a4x353CX0SIM/edit
 
 ## Getting started
 
 ### Getting etcd
 
-The easiest way to get etcd is to use one of the pre-built release binaries which are available for OSX, Linux, Windows, [rkt][rkt], and Docker. Instructions for using these binaries are on the [GitHub releases page][github-release].
+The easiest way to get etcd is to use one of the pre-built release binaries which are available for OSX, Linux, Windows, and Docker on the [release page][github-release].
 
-For those wanting to try the very latest version, [build the latest version of etcd][dl-build] from the `master` branch. This first needs [*Go*](https://golang.org/) installed (version 1.9+ is required). All development occurs on `master`, including new features and bug fixes. Bug fixes are first targeted at `master` and subsequently ported to release branches, as described in the [branch management][branch-management] guide.
+For more installation guides, please check out [play.etcd.io](http://play.etcd.io) and [operating etcd](https://github.com/coreos/etcd/tree/master/Documentation#operating-etcd-clusters).
 
-[rkt]: https://github.com/rkt/rkt/releases/
-[github-release]: https://github.com/coreos/etcd/releases/
+For those wanting to try the very latest version, [build the latest version of etcd][dl-build] from the `master` branch. This first needs [*Go*](https://golang.org/) installed (version 1.10+ is required). All development occurs on `master`, including new features and bug fixes. Bug fixes are first targeted at `master` and subsequently ported to release branches, as described in the [branch management][branch-management] guide.
+
+[github-release]: https://github.com/coreos/etcd/releases
 [branch-management]: ./Documentation/branch_management.md
 [dl-build]: ./Documentation/dl_build.md#build-the-latest-version
 
@@ -66,15 +70,15 @@ If etcd is installed using the [pre-built release binaries][github-release], run
 ```sh
 /tmp/etcd-download-test/etcd
 ```
+
 The etcd command can be simply run as such if it is moved to the system path as below:
 
 ```sh
-mv /tmp/etcd-download-test/etcd /usr/locale/bin/
-
+mv /tmp/etcd-download-test/etcd /usr/local/bin/
 etcd
 ```
 
-If etcd is [build from the master branch][dl-build], run it as below:
+If etcd is [built from the master branch][dl-build], run it as below:
 
 ```sh
 ./bin/etcd
@@ -89,13 +93,10 @@ ETCDCTL_API=3 etcdctl put mykey "this is awesome"
 ETCDCTL_API=3 etcdctl get mykey
 ```
 
-That's it! etcd is now running and serving client requests. For more
+etcd is now running and serving client requests. For more, please check out:
 
-- [Animated quick demo][demo-gif]
-- [Interactive etcd playground][etcd-play]
-
-[demo-gif]: ./Documentation/demo.md
-[etcd-play]: http://play.etcd.io/
+- [Interactive etcd playground](http://play.etcd.io)
+- [Animated quick demo](./Documentation/demo.md)
 
 ### etcd TCP ports
 
@@ -156,6 +157,12 @@ See [CONTRIBUTING](CONTRIBUTING.md) for details on submitting patches and the co
 
 See [reporting bugs](Documentation/reporting_bugs.md) for details about reporting any issues.
 
+## Reporting a security vulnerability
+
+Due to their public nature, GitHub and mailing lists are NOT appropriate places for reporting vulnerabilities. Please refer to CoreOS's [security disclosure][disclosure] process when reporting issues that may be security related.
+
+[disclosure]: https://coreos.com/security/disclosure/
+
 ### License
 
 etcd is under the Apache 2.0 license. See the [LICENSE](LICENSE) file for details.

ROADMAP.md

@@ -6,18 +6,17 @@ This document defines a high level roadmap for etcd development.
 
 The dates below should not be considered authoritative, but rather indicative of the projected timeline of the project. The [milestones defined in GitHub](https://github.com/coreos/etcd/milestones) represent the most up-to-date and issue-for-issue plans.
 
-etcd 3.2 is our current stable branch. The roadmap below outlines new features that will be added to etcd, and while subject to change, define what future stable will look like.
+etcd 3.3 is our current stable branch. The roadmap below outlines new features that will be added to etcd, and while subject to change, define what future stable will look like.
 
-### etcd 3.2 (2017-May)
-- Stable scalable proxy
-- Proxy-as-client interface passthrough
-- Lock service
-- Namespacing proxy
-- TLS Command Name and JWT token based authentication
-- Read-modify-write V3 Put
-- Improved watch performance
-- Support non-blocking concurrent read
-
-### etcd 3.3 (?)
-- TBD
+### etcd 3.4 (Q2, 2018)
+- Stabilization of 3.3 experimental features
+- Support/document downgrade
+- Snapshot restore as Go library
+- Improved client balancer with new gRPC balancer interface
+- Improve single-client put performance
+- Improve large response handling
+- Improve test coverage
+- Decrease test runtime
+- Migrate to `golang/dep` for dependency management
 
+Please join our [bi-weekly etcd meeting](https://docs.google.com/document/d/1DbVXOHvd9scFsSmL2oNg4YGOHJdXqtx583DmeVWrB_M/edit) for latest discussions on roadmap. Most up-to-date roadmap can be found [here](https://docs.google.com/spreadsheets/d/1kT4xY_y1p3R8Xqp1wRDoImT7MjiepePnC14E-IACEfM/edit#gid=0).

V2Procfile

@@ -1,5 +0,0 @@
-# Use goreman to run `go get github.com/mattn/goreman`
-etcd1: bin/etcd --name infra1 --listen-client-urls http://127.0.0.1:12379 --advertise-client-urls http://127.0.0.1:12379 --listen-peer-urls http://127.0.0.1:12380 --initial-advertise-peer-urls http://127.0.0.1:12380 --initial-cluster-token etcd-cluster-1 --initial-cluster 'infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380' --initial-cluster-state new --enable-pprof
-etcd2: bin/etcd --name infra2 --listen-client-urls http://127.0.0.1:22379 --advertise-client-urls http://127.0.0.1:22379 --listen-peer-urls http://127.0.0.1:22380 --initial-advertise-peer-urls http://127.0.0.1:22380 --initial-cluster-token etcd-cluster-1 --initial-cluster 'infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380' --initial-cluster-state new --enable-pprof
-etcd3: bin/etcd --name infra3 --listen-client-urls http://127.0.0.1:32379 --advertise-client-urls http://127.0.0.1:32379 --listen-peer-urls http://127.0.0.1:32380 --initial-advertise-peer-urls http://127.0.0.1:32380 --initial-cluster-token etcd-cluster-1 --initial-cluster 'infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380' --initial-cluster-state new --enable-pprof
-proxy: bin/etcd --name infra-proxy1 --proxy=on --listen-client-urls http://127.0.0.1:2379 --initial-cluster 'infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380' --enable-pprof

auth/jwt.go

@@ -16,16 +16,21 @@ package auth
 
 import (
 	"context"
+	"crypto/ecdsa"
 	"crypto/rsa"
-	"io/ioutil"
+	"errors"
+	"time"
 
 	jwt "github.com/dgrijalva/jwt-go"
+	"go.uber.org/zap"
 )
 
 type tokenJWT struct {
-	signMethod string
-	signKey    *rsa.PrivateKey
-	verifyKey  *rsa.PublicKey
+	lg         *zap.Logger
+	signMethod jwt.SigningMethod
+	key        interface{}
+	ttl        time.Duration
+	verifyOnly bool
 }
 
 func (t *tokenJWT) enable()                         {}
@@ -41,98 +46,138 @@ func (t *tokenJWT) info(ctx context.Context, token string, rev uint64) (*AuthInf
 	)
 
 	parsed, err := jwt.Parse(token, func(token *jwt.Token) (interface{}, error) {
-		return t.verifyKey, nil
+		if token.Method.Alg() != t.signMethod.Alg() {
+			return nil, errors.New("invalid signing method")
+		}
+		switch k := t.key.(type) {
+		case *rsa.PrivateKey:
+			return &k.PublicKey, nil
+		case *ecdsa.PrivateKey:
+			return &k.PublicKey, nil
+		default:
+			return t.key, nil
+		}
 	})
 
-	switch err.(type) {
-	case nil:
-		if !parsed.Valid {
-			plog.Warningf("invalid jwt token: %s", token)
-			return nil, false
+	if err != nil {
+		if t.lg != nil {
+			t.lg.Warn(
+				"failed to parse a JWT token",
+				zap.String("token", token),
+				zap.Error(err),
+			)
+		} else {
+			plog.Warningf("failed to parse jwt token: %s", err)
 		}
-
-		claims := parsed.Claims.(jwt.MapClaims)
-
-		username = claims["username"].(string)
-		revision = uint64(claims["revision"].(float64))
-	default:
-		plog.Warningf("failed to parse jwt token: %s", err)
 		return nil, false
 	}
 
+	claims, ok := parsed.Claims.(jwt.MapClaims)
+	if !parsed.Valid || !ok {
+		if t.lg != nil {
+			t.lg.Warn("invalid JWT token", zap.String("token", token))
+		} else {
+			plog.Warningf("invalid jwt token: %s", token)
+		}
+		return nil, false
+	}
+
+	username = claims["username"].(string)
+	revision = uint64(claims["revision"].(float64))
+
 	return &AuthInfo{Username: username, Revision: revision}, true
 }
 
 func (t *tokenJWT) assign(ctx context.Context, username string, revision uint64) (string, error) {
+	if t.verifyOnly {
+		return "", ErrVerifyOnly
+	}
+
 	// Future work: let a jwt token include permission information would be useful for
 	// permission checking in proxy side.
-	tk := jwt.NewWithClaims(jwt.GetSigningMethod(t.signMethod),
+	tk := jwt.NewWithClaims(t.signMethod,
 		jwt.MapClaims{
 			"username": username,
 			"revision": revision,
+			"exp":      time.Now().Add(t.ttl).Unix(),
 		})
 
-	token, err := tk.SignedString(t.signKey)
+	token, err := tk.SignedString(t.key)
 	if err != nil {
-		plog.Debugf("failed to sign jwt token: %s", err)
+		if t.lg != nil {
+			t.lg.Warn(
+				"failed to sign a JWT token",
+				zap.String("user-name", username),
+				zap.Uint64("revision", revision),
+				zap.Error(err),
+			)
+		} else {
+			plog.Debugf("failed to sign jwt token: %s", err)
+		}
 		return "", err
 	}
 
-	plog.Debugf("jwt token: %s", token)
-
+	if t.lg != nil {
+		t.lg.Info(
+			"created/assigned a new JWT token",
+			zap.String("user-name", username),
+			zap.Uint64("revision", revision),
+			zap.String("token", token),
+		)
+	} else {
+		plog.Debugf("jwt token: %s", token)
+	}
 	return token, err
 }
 
-func prepareOpts(opts map[string]string) (jwtSignMethod, jwtPubKeyPath, jwtPrivKeyPath string, err error) {
-	for k, v := range opts {
-		switch k {
-		case "sign-method":
-			jwtSignMethod = v
-		case "pub-key":
-			jwtPubKeyPath = v
-		case "priv-key":
-			jwtPrivKeyPath = v
-		default:
-			plog.Errorf("unknown token specific option: %s", k)
-			return "", "", "", ErrInvalidAuthOpts
-		}
-	}
-	if len(jwtSignMethod) == 0 {
-		return "", "", "", ErrInvalidAuthOpts
-	}
-	return jwtSignMethod, jwtPubKeyPath, jwtPrivKeyPath, nil
-}
-
-func newTokenProviderJWT(opts map[string]string) (*tokenJWT, error) {
-	jwtSignMethod, jwtPubKeyPath, jwtPrivKeyPath, err := prepareOpts(opts)
+func newTokenProviderJWT(lg *zap.Logger, optMap map[string]string) (*tokenJWT, error) {
+	var err error
+	var opts jwtOptions
+	err = opts.ParseWithDefaults(optMap)
 	if err != nil {
+		if lg != nil {
+			lg.Warn("problem loading JWT options", zap.Error(err))
+		} else {
+			plog.Errorf("problem loading JWT options: %s", err)
+		}
 		return nil, ErrInvalidAuthOpts
 	}
 
-	t := &tokenJWT{}
-
-	t.signMethod = jwtSignMethod
-
-	verifyBytes, err := ioutil.ReadFile(jwtPubKeyPath)
-	if err != nil {
-		plog.Errorf("failed to read public key (%s) for jwt: %s", jwtPubKeyPath, err)
-		return nil, err
+	var keys = make([]string, 0, len(optMap))
+	for k := range optMap {
+		if !knownOptions[k] {
+			keys = append(keys, k)
+		}
 	}
-	t.verifyKey, err = jwt.ParseRSAPublicKeyFromPEM(verifyBytes)
+	if len(keys) > 0 {
+		if lg != nil {
+			lg.Warn("unknown JWT options", zap.Strings("keys", keys))
+		} else {
+			plog.Warningf("unknown JWT options: %v", keys)
+		}
+	}
+
+	key, err := opts.Key()
 	if err != nil {
-		plog.Errorf("failed to parse public key (%s): %s", jwtPubKeyPath, err)
 		return nil, err
 	}
 
-	signBytes, err := ioutil.ReadFile(jwtPrivKeyPath)
-	if err != nil {
-		plog.Errorf("failed to read private key (%s) for jwt: %s", jwtPrivKeyPath, err)
-		return nil, err
+	t := &tokenJWT{
+		lg:         lg,
+		ttl:        opts.TTL,
+		signMethod: opts.SignMethod,
+		key:        key,
 	}
-	t.signKey, err = jwt.ParseRSAPrivateKeyFromPEM(signBytes)
-	if err != nil {
-		plog.Errorf("failed to parse private key (%s): %s", jwtPrivKeyPath, err)
-		return nil, err
+
+	switch t.signMethod.(type) {
+	case *jwt.SigningMethodECDSA:
+		if _, ok := t.key.(*ecdsa.PublicKey); ok {
+			t.verifyOnly = true
+		}
+	case *jwt.SigningMethodRSA, *jwt.SigningMethodRSAPSS:
+		if _, ok := t.key.(*rsa.PublicKey); ok {
+			t.verifyOnly = true
+		}
 	}
 
 	return t, nil

auth/jwt_test.go

@@ -16,79 +16,189 @@ package auth
 
 import (
 	"context"
+	"fmt"
 	"testing"
+
+	"go.uber.org/zap"
 )
 
 const (
-	jwtPubKey  = "../integration/fixtures/server.crt"
-	jwtPrivKey = "../integration/fixtures/server.key.insecure"
+	jwtRSAPubKey  = "../integration/fixtures/server.crt"
+	jwtRSAPrivKey = "../integration/fixtures/server.key.insecure"
+
+	jwtECPubKey  = "../integration/fixtures/server-ecdsa.crt"
+	jwtECPrivKey = "../integration/fixtures/server-ecdsa.key.insecure"
 )
 
 func TestJWTInfo(t *testing.T) {
-	opts := map[string]string{
-		"pub-key":     jwtPubKey,
-		"priv-key":    jwtPrivKey,
-		"sign-method": "RS256",
+	optsMap := map[string]map[string]string{
+		"RSA-priv": {
+			"priv-key":    jwtRSAPrivKey,
+			"sign-method": "RS256",
+			"ttl":         "1h",
+		},
+		"RSA": {
+			"pub-key":     jwtRSAPubKey,
+			"priv-key":    jwtRSAPrivKey,
+			"sign-method": "RS256",
+		},
+		"RSAPSS-priv": {
+			"priv-key":    jwtRSAPrivKey,
+			"sign-method": "PS256",
+		},
+		"RSAPSS": {
+			"pub-key":     jwtRSAPubKey,
+			"priv-key":    jwtRSAPrivKey,
+			"sign-method": "PS256",
+		},
+		"ECDSA-priv": {
+			"priv-key":    jwtECPrivKey,
+			"sign-method": "ES256",
+		},
+		"ECDSA": {
+			"pub-key":     jwtECPubKey,
+			"priv-key":    jwtECPrivKey,
+			"sign-method": "ES256",
+		},
+		"HMAC": {
+			"priv-key":    jwtECPrivKey, // any file, raw bytes used as shared secret
+			"sign-method": "HS256",
+		},
 	}
-	jwt, err := newTokenProviderJWT(opts)
+
+	for k, opts := range optsMap {
+		t.Run(k, func(tt *testing.T) {
+			testJWTInfo(tt, opts)
+		})
+	}
+}
+
+func testJWTInfo(t *testing.T, opts map[string]string) {
+	lg := zap.NewNop()
+	jwt, err := newTokenProviderJWT(lg, opts)
 	if err != nil {
 		t.Fatal(err)
 	}
-	token, aerr := jwt.assign(context.TODO(), "abc", 123)
+
+	ctx := context.TODO()
+
+	token, aerr := jwt.assign(ctx, "abc", 123)
 	if aerr != nil {
-		t.Fatal(err)
+		t.Fatalf("%#v", aerr)
 	}
-	ai, ok := jwt.info(context.TODO(), token, 123)
+	ai, ok := jwt.info(ctx, token, 123)
 	if !ok {
 		t.Fatalf("failed to authenticate with token %s", token)
 	}
 	if ai.Revision != 123 {
 		t.Fatalf("expected revision 123, got %d", ai.Revision)
 	}
-	ai, ok = jwt.info(context.TODO(), "aaa", 120)
+	ai, ok = jwt.info(ctx, "aaa", 120)
 	if ok || ai != nil {
 		t.Fatalf("expected aaa to fail to authenticate, got %+v", ai)
 	}
+
+	// test verify-only provider
+	if opts["pub-key"] != "" && opts["priv-key"] != "" {
+		t.Run("verify-only", func(t *testing.T) {
+			newOpts := make(map[string]string, len(opts))
+			for k, v := range opts {
+				newOpts[k] = v
+			}
+			delete(newOpts, "priv-key")
+			verify, err := newTokenProviderJWT(lg, newOpts)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			ai, ok := verify.info(ctx, token, 123)
+			if !ok {
+				t.Fatalf("failed to authenticate with token %s", token)
+			}
+			if ai.Revision != 123 {
+				t.Fatalf("expected revision 123, got %d", ai.Revision)
+			}
+			ai, ok = verify.info(ctx, "aaa", 120)
+			if ok || ai != nil {
+				t.Fatalf("expected aaa to fail to authenticate, got %+v", ai)
+			}
+
+			_, aerr := verify.assign(ctx, "abc", 123)
+			if aerr != ErrVerifyOnly {
+				t.Fatalf("unexpected error when attempting to sign with public key: %v", aerr)
+			}
+
+		})
+	}
 }
 
 func TestJWTBad(t *testing.T) {
-	opts := map[string]string{
-		"pub-key":     jwtPubKey,
-		"priv-key":    jwtPrivKey,
-		"sign-method": "RS256",
-	}
-	// private key instead of public key
-	opts["pub-key"] = jwtPrivKey
-	if _, err := newTokenProviderJWT(opts); err == nil {
-		t.Fatalf("expected failure on missing public key")
-	}
-	opts["pub-key"] = jwtPubKey
 
-	// public key instead of private key
-	opts["priv-key"] = jwtPubKey
-	if _, err := newTokenProviderJWT(opts); err == nil {
-		t.Fatalf("expected failure on missing public key")
+	var badCases = map[string]map[string]string{
+		"no options": {},
+		"invalid method": {
+			"sign-method": "invalid",
+		},
+		"rsa no key": {
+			"sign-method": "RS256",
+		},
+		"invalid ttl": {
+			"sign-method": "RS256",
+			"ttl":         "forever",
+		},
+		"rsa invalid public key": {
+			"sign-method": "RS256",
+			"pub-key":     jwtRSAPrivKey,
+			"priv-key":    jwtRSAPrivKey,
+		},
+		"rsa invalid private key": {
+			"sign-method": "RS256",
+			"pub-key":     jwtRSAPubKey,
+			"priv-key":    jwtRSAPubKey,
+		},
+		"hmac no key": {
+			"sign-method": "HS256",
+		},
+		"hmac pub key": {
+			"sign-method": "HS256",
+			"pub-key":     jwtRSAPubKey,
+		},
+		"missing public key file": {
+			"sign-method": "HS256",
+			"pub-key":     "missing-file",
+		},
+		"missing private key file": {
+			"sign-method": "HS256",
+			"priv-key":    "missing-file",
+		},
+		"ecdsa no key": {
+			"sign-method": "ES256",
+		},
+		"ecdsa invalid public key": {
+			"sign-method": "ES256",
+			"pub-key":     jwtECPrivKey,
+			"priv-key":    jwtECPrivKey,
+		},
+		"ecdsa invalid private key": {
+			"sign-method": "ES256",
+			"pub-key":     jwtECPubKey,
+			"priv-key":    jwtECPubKey,
+		},
 	}
-	opts["priv-key"] = jwtPrivKey
 
-	// missing signing option
-	delete(opts, "sign-method")
-	if _, err := newTokenProviderJWT(opts); err == nil {
-		t.Fatal("expected error on missing option")
-	}
-	opts["sign-method"] = "RS256"
+	lg := zap.NewNop()
 
-	// bad file for pubkey
-	opts["pub-key"] = "whatever"
-	if _, err := newTokenProviderJWT(opts); err == nil {
-		t.Fatalf("expected failure on missing public key")
+	for k, v := range badCases {
+		t.Run(k, func(t *testing.T) {
+			_, err := newTokenProviderJWT(lg, v)
+			if err == nil {
+				t.Errorf("expected error for options %v", v)
+			}
+		})
 	}
-	opts["pub-key"] = jwtPubKey
-
-	// bad file for private key
-	opts["priv-key"] = "whatever"
-	if _, err := newTokenProviderJWT(opts); err == nil {
-		t.Fatalf("expeceted failure on missing private key")
-	}
-	opts["priv-key"] = jwtPrivKey
+}
+
+// testJWTOpts is useful for passing to NewTokenProvider which requires a string.
+func testJWTOpts() string {
+	return fmt.Sprintf("%s,pub-key=%s,priv-key=%s,sign-method=RS256", tokenTypeJWT, jwtRSAPubKey, jwtRSAPrivKey)
 }

auth/nop.go

@@ -0,0 +1,35 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package auth
+
+import (
+	"context"
+)
+
+type tokenNop struct{}
+
+func (t *tokenNop) enable()                         {}
+func (t *tokenNop) disable()                        {}
+func (t *tokenNop) invalidateUser(string)           {}
+func (t *tokenNop) genTokenPrefix() (string, error) { return "", nil }
+func (t *tokenNop) info(ctx context.Context, token string, rev uint64) (*AuthInfo, bool) {
+	return nil, false
+}
+func (t *tokenNop) assign(ctx context.Context, username string, revision uint64) (string, error) {
+	return "", ErrAuthFailed
+}
+func newTokenProviderNop() (*tokenNop, error) {
+	return &tokenNop{}, nil
+}

auth/options.go

@@ -0,0 +1,192 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package auth
+
+import (
+	"crypto/ecdsa"
+	"crypto/rsa"
+	"fmt"
+	"io/ioutil"
+	"time"
+
+	jwt "github.com/dgrijalva/jwt-go"
+)
+
+const (
+	optSignMethod = "sign-method"
+	optPublicKey  = "pub-key"
+	optPrivateKey = "priv-key"
+	optTTL        = "ttl"
+)
+
+var knownOptions = map[string]bool{
+	optSignMethod: true,
+	optPublicKey:  true,
+	optPrivateKey: true,
+	optTTL:        true,
+}
+
+var (
+	// DefaultTTL will be used when a 'ttl' is not specified
+	DefaultTTL = 5 * time.Minute
+)
+
+type jwtOptions struct {
+	SignMethod jwt.SigningMethod
+	PublicKey  []byte
+	PrivateKey []byte
+	TTL        time.Duration
+}
+
+// ParseWithDefaults will load options from the specified map or set defaults where appropriate
+func (opts *jwtOptions) ParseWithDefaults(optMap map[string]string) error {
+	if opts.TTL == 0 && optMap[optTTL] == "" {
+		opts.TTL = DefaultTTL
+	}
+
+	return opts.Parse(optMap)
+}
+
+// Parse will load options from the specified map
+func (opts *jwtOptions) Parse(optMap map[string]string) error {
+	var err error
+	if ttl := optMap[optTTL]; ttl != "" {
+		opts.TTL, err = time.ParseDuration(ttl)
+		if err != nil {
+			return err
+		}
+	}
+
+	if file := optMap[optPublicKey]; file != "" {
+		opts.PublicKey, err = ioutil.ReadFile(file)
+		if err != nil {
+			return err
+		}
+	}
+
+	if file := optMap[optPrivateKey]; file != "" {
+		opts.PrivateKey, err = ioutil.ReadFile(file)
+		if err != nil {
+			return err
+		}
+	}
+
+	// signing method is a required field
+	method := optMap[optSignMethod]
+	opts.SignMethod = jwt.GetSigningMethod(method)
+	if opts.SignMethod == nil {
+		return ErrInvalidAuthMethod
+	}
+
+	return nil
+}
+
+// Key will parse and return the appropriately typed key for the selected signature method
+func (opts *jwtOptions) Key() (interface{}, error) {
+	switch opts.SignMethod.(type) {
+	case *jwt.SigningMethodRSA, *jwt.SigningMethodRSAPSS:
+		return opts.rsaKey()
+	case *jwt.SigningMethodECDSA:
+		return opts.ecKey()
+	case *jwt.SigningMethodHMAC:
+		return opts.hmacKey()
+	default:
+		return nil, fmt.Errorf("unsupported signing method: %T", opts.SignMethod)
+	}
+}
+
+func (opts *jwtOptions) hmacKey() (interface{}, error) {
+	if len(opts.PrivateKey) == 0 {
+		return nil, ErrMissingKey
+	}
+	return opts.PrivateKey, nil
+}
+
+func (opts *jwtOptions) rsaKey() (interface{}, error) {
+	var (
+		priv *rsa.PrivateKey
+		pub  *rsa.PublicKey
+		err  error
+	)
+
+	if len(opts.PrivateKey) > 0 {
+		priv, err = jwt.ParseRSAPrivateKeyFromPEM(opts.PrivateKey)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if len(opts.PublicKey) > 0 {
+		pub, err = jwt.ParseRSAPublicKeyFromPEM(opts.PublicKey)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if priv == nil {
+		if pub == nil {
+			// Neither key given
+			return nil, ErrMissingKey
+		}
+		// Public key only, can verify tokens
+		return pub, nil
+	}
+
+	// both keys provided, make sure they match
+	if pub != nil && pub.E != priv.E && pub.N.Cmp(priv.N) != 0 {
+		return nil, ErrKeyMismatch
+	}
+
+	return priv, nil
+}
+
+func (opts *jwtOptions) ecKey() (interface{}, error) {
+	var (
+		priv *ecdsa.PrivateKey
+		pub  *ecdsa.PublicKey
+		err  error
+	)
+
+	if len(opts.PrivateKey) > 0 {
+		priv, err = jwt.ParseECPrivateKeyFromPEM(opts.PrivateKey)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if len(opts.PublicKey) > 0 {
+		pub, err = jwt.ParseECPublicKeyFromPEM(opts.PublicKey)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if priv == nil {
+		if pub == nil {
+			// Neither key given
+			return nil, ErrMissingKey
+		}
+		// Public key only, can verify tokens
+		return pub, nil
+	}
+
+	// both keys provided, make sure they match
+	if pub != nil && pub.Curve != priv.Curve &&
+		pub.X.Cmp(priv.X) != 0 && pub.Y.Cmp(priv.Y) != 0 {
+		return nil, ErrKeyMismatch
+	}
+
+	return priv, nil
+}

auth/range_perm_cache.go

@@ -18,12 +18,13 @@ import (
 	"github.com/coreos/etcd/auth/authpb"
 	"github.com/coreos/etcd/mvcc/backend"
 	"github.com/coreos/etcd/pkg/adt"
+
+	"go.uber.org/zap"
 )
 
-func getMergedPerms(tx backend.BatchTx, userName string) *unifiedRangePermissions {
-	user := getUser(tx, userName)
+func getMergedPerms(lg *zap.Logger, tx backend.BatchTx, userName string) *unifiedRangePermissions {
+	user := getUser(lg, tx, userName)
 	if user == nil {
-		plog.Errorf("invalid user name %s", userName)
 		return nil
 	}
 
@@ -70,7 +71,11 @@ func getMergedPerms(tx backend.BatchTx, userName string) *unifiedRangePermission
 	}
 }
 
-func checkKeyInterval(cachedPerms *unifiedRangePermissions, key, rangeEnd []byte, permtyp authpb.Permission_Type) bool {
+func checkKeyInterval(
+	lg *zap.Logger,
+	cachedPerms *unifiedRangePermissions,
+	key, rangeEnd []byte,
+	permtyp authpb.Permission_Type) bool {
 	if len(rangeEnd) == 1 && rangeEnd[0] == 0 {
 		rangeEnd = nil
 	}
@@ -82,12 +87,16 @@ func checkKeyInterval(cachedPerms *unifiedRangePermissions, key, rangeEnd []byte
 	case authpb.WRITE:
 		return cachedPerms.writePerms.Contains(ivl)
 	default:
-		plog.Panicf("unknown auth type: %v", permtyp)
+		if lg != nil {
+			lg.Panic("unknown auth type", zap.String("auth-type", permtyp.String()))
+		} else {
+			plog.Panicf("unknown auth type: %v", permtyp)
+		}
 	}
 	return false
 }
 
-func checkKeyPoint(cachedPerms *unifiedRangePermissions, key []byte, permtyp authpb.Permission_Type) bool {
+func checkKeyPoint(lg *zap.Logger, cachedPerms *unifiedRangePermissions, key []byte, permtyp authpb.Permission_Type) bool {
 	pt := adt.NewBytesAffinePoint(key)
 	switch permtyp {
 	case authpb.READ:
@@ -95,7 +104,11 @@ func checkKeyPoint(cachedPerms *unifiedRangePermissions, key []byte, permtyp aut
 	case authpb.WRITE:
 		return cachedPerms.writePerms.Intersects(pt)
 	default:
-		plog.Panicf("unknown auth type: %v", permtyp)
+		if lg != nil {
+			lg.Panic("unknown auth type", zap.String("auth-type", permtyp.String()))
+		} else {
+			plog.Panicf("unknown auth type: %v", permtyp)
+		}
 	}
 	return false
 }
@@ -104,19 +117,26 @@ func (as *authStore) isRangeOpPermitted(tx backend.BatchTx, userName string, key
 	// assumption: tx is Lock()ed
 	_, ok := as.rangePermCache[userName]
 	if !ok {
-		perms := getMergedPerms(tx, userName)
+		perms := getMergedPerms(as.lg, tx, userName)
 		if perms == nil {
-			plog.Errorf("failed to create a unified permission of user %s", userName)
+			if as.lg != nil {
+				as.lg.Warn(
+					"failed to create a merged permission",
+					zap.String("user-name", userName),
+				)
+			} else {
+				plog.Errorf("failed to create a unified permission of user %s", userName)
+			}
 			return false
 		}
 		as.rangePermCache[userName] = perms
 	}
 
 	if len(rangeEnd) == 0 {
-		return checkKeyPoint(as.rangePermCache[userName], key, permtyp)
+		return checkKeyPoint(as.lg, as.rangePermCache[userName], key, permtyp)
 	}
 
-	return checkKeyInterval(as.rangePermCache[userName], key, rangeEnd, permtyp)
+	return checkKeyInterval(as.lg, as.rangePermCache[userName], key, rangeEnd, permtyp)
 }
 
 func (as *authStore) clearCachedPerm() {

auth/range_perm_cache_test.go

@@ -19,6 +19,8 @@ import (
 
 	"github.com/coreos/etcd/auth/authpb"
 	"github.com/coreos/etcd/pkg/adt"
+
+	"go.uber.org/zap"
 )
 
 func TestRangePermission(t *testing.T) {
@@ -51,7 +53,7 @@ func TestRangePermission(t *testing.T) {
 			readPerms.Insert(p, struct{}{})
 		}
 
-		result := checkKeyInterval(&unifiedRangePermissions{readPerms: readPerms}, tt.begin, tt.end, authpb.READ)
+		result := checkKeyInterval(zap.NewExample(), &unifiedRangePermissions{readPerms: readPerms}, tt.begin, tt.end, authpb.READ)
 		if result != tt.want {
 			t.Errorf("#%d: result=%t, want=%t", i, result, tt.want)
 		}

auth/simple_token.go

@@ -26,6 +26,8 @@ import (
 	"strings"
 	"sync"
 	"time"
+
+	"go.uber.org/zap"
 )
 
 const (
@@ -94,6 +96,7 @@ func (tm *simpleTokenTTLKeeper) run() {
 }
 
 type tokenSimple struct {
+	lg                *zap.Logger
 	indexWaiter       func(uint64) <-chan struct{}
 	simpleTokenKeeper *simpleTokenTTLKeeper
 	simpleTokensMu    sync.Mutex
@@ -124,7 +127,15 @@ func (t *tokenSimple) assignSimpleTokenToUser(username, token string) {
 
 	_, ok := t.simpleTokens[token]
 	if ok {
-		plog.Panicf("token %s is alredy used", token)
+		if t.lg != nil {
+			t.lg.Panic(
+				"failed to assign already-used simple token to a user",
+				zap.String("user-name", username),
+				zap.String("token", token),
+			)
+		} else {
+			plog.Panicf("token %s is alredy used", token)
+		}
 	}
 
 	t.simpleTokens[token] = username
@@ -137,7 +148,7 @@ func (t *tokenSimple) invalidateUser(username string) {
 	}
 	t.simpleTokensMu.Lock()
 	for token, name := range t.simpleTokens {
-		if strings.Compare(name, username) == 0 {
+		if name == username {
 			delete(t.simpleTokens, token)
 			t.simpleTokenKeeper.deleteSimpleToken(token)
 		}
@@ -148,7 +159,15 @@ func (t *tokenSimple) invalidateUser(username string) {
 func (t *tokenSimple) enable() {
 	delf := func(tk string) {
 		if username, ok := t.simpleTokens[tk]; ok {
-			plog.Infof("deleting token %s for user %s", tk, username)
+			if t.lg != nil {
+				t.lg.Info(
+					"deleted a simple token",
+					zap.String("user-name", username),
+					zap.String("token", tk),
+				)
+			} else {
+				plog.Infof("deleting token %s for user %s", tk, username)
+			}
 			delete(t.simpleTokens, tk)
 		}
 	}
@@ -215,8 +234,9 @@ func (t *tokenSimple) isValidSimpleToken(ctx context.Context, token string) bool
 	return false
 }
 
-func newTokenProviderSimple(indexWaiter func(uint64) <-chan struct{}) *tokenSimple {
+func newTokenProviderSimple(lg *zap.Logger, indexWaiter func(uint64) <-chan struct{}) *tokenSimple {
 	return &tokenSimple{
+		lg:           lg,
 		simpleTokens: make(map[string]string),
 		indexWaiter:  indexWaiter,
 	}

auth/simple_token_test.go

@@ -17,14 +17,16 @@ package auth
 import (
 	"context"
 	"testing"
+
+	"go.uber.org/zap"
 )
 
 // TestSimpleTokenDisabled ensures that TokenProviderSimple behaves correctly when
 // disabled.
 func TestSimpleTokenDisabled(t *testing.T) {
-	initialState := newTokenProviderSimple(dummyIndexWaiter)
+	initialState := newTokenProviderSimple(zap.NewExample(), dummyIndexWaiter)
 
-	explicitlyDisabled := newTokenProviderSimple(dummyIndexWaiter)
+	explicitlyDisabled := newTokenProviderSimple(zap.NewExample(), dummyIndexWaiter)
 	explicitlyDisabled.enable()
 	explicitlyDisabled.disable()
 
@@ -46,7 +48,7 @@ func TestSimpleTokenDisabled(t *testing.T) {
 // TestSimpleTokenAssign ensures that TokenProviderSimple can correctly assign a
 // token, look it up with info, and invalidate it by user.
 func TestSimpleTokenAssign(t *testing.T) {
-	tp := newTokenProviderSimple(dummyIndexWaiter)
+	tp := newTokenProviderSimple(zap.NewExample(), dummyIndexWaiter)
 	tp.enable()
 	ctx := context.WithValue(context.WithValue(context.TODO(), AuthenticateParamIndex{}, uint64(1)), AuthenticateParamSimpleTokenPrefix{}, "dummy")
 	token, err := tp.assign(ctx, "user1", 0)

auth/store_test.go

@@ -25,15 +25,15 @@ import (
 	"time"
 
 	"github.com/coreos/etcd/auth/authpb"
+	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
 	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
 	"github.com/coreos/etcd/mvcc/backend"
 
+	"go.uber.org/zap"
 	"golang.org/x/crypto/bcrypt"
 	"google.golang.org/grpc/metadata"
 )
 
-func init() { BcryptCost = bcrypt.MinCost }
-
 func dummyIndexWaiter(index uint64) <-chan struct{} {
 	ch := make(chan struct{})
 	go func() {
@@ -48,39 +48,61 @@ func TestNewAuthStoreRevision(t *testing.T) {
 	b, tPath := backend.NewDefaultTmpBackend()
 	defer os.Remove(tPath)
 
-	tp, err := NewTokenProvider("simple", dummyIndexWaiter)
+	tp, err := NewTokenProvider(zap.NewExample(), tokenTypeSimple, dummyIndexWaiter)
 	if err != nil {
 		t.Fatal(err)
 	}
-	as := NewAuthStore(b, tp)
+	as := NewAuthStore(zap.NewExample(), b, tp, bcrypt.MinCost)
 	err = enableAuthAndCreateRoot(as)
 	if err != nil {
 		t.Fatal(err)
 	}
 	old := as.Revision()
-	b.Close()
 	as.Close()
+	b.Close()
 
 	// no changes to commit
 	b2 := backend.NewDefaultBackend(tPath)
-	as = NewAuthStore(b2, tp)
+	as = NewAuthStore(zap.NewExample(), b2, tp, bcrypt.MinCost)
 	new := as.Revision()
-	b2.Close()
 	as.Close()
+	b2.Close()
 
 	if old != new {
 		t.Fatalf("expected revision %d, got %d", old, new)
 	}
 }
 
-func setupAuthStore(t *testing.T) (store *authStore, teardownfunc func(t *testing.T)) {
+// TestNewAuthStoreBryptCost ensures that NewAuthStore uses default when given bcrypt-cost is invalid
+func TestNewAuthStoreBcryptCost(t *testing.T) {
 	b, tPath := backend.NewDefaultTmpBackend()
+	defer os.Remove(tPath)
 
-	tp, err := NewTokenProvider("simple", dummyIndexWaiter)
+	tp, err := NewTokenProvider(zap.NewExample(), tokenTypeSimple, dummyIndexWaiter)
 	if err != nil {
 		t.Fatal(err)
 	}
-	as := NewAuthStore(b, tp)
+
+	invalidCosts := [2]int{bcrypt.MinCost - 1, bcrypt.MaxCost + 1}
+	for _, invalidCost := range invalidCosts {
+		as := NewAuthStore(zap.NewExample(), b, tp, invalidCost)
+		if as.BcryptCost() != bcrypt.DefaultCost {
+			t.Fatalf("expected DefaultCost when bcryptcost is invalid")
+		}
+		as.Close()
+	}
+
+	b.Close()
+}
+
+func setupAuthStore(t *testing.T) (store *authStore, teardownfunc func(t *testing.T)) {
+	b, tPath := backend.NewDefaultTmpBackend()
+
+	tp, err := NewTokenProvider(zap.NewExample(), tokenTypeSimple, dummyIndexWaiter)
+	if err != nil {
+		t.Fatal(err)
+	}
+	as := NewAuthStore(zap.NewExample(), b, tp, bcrypt.MinCost)
 	err = enableAuthAndCreateRoot(as)
 	if err != nil {
 		t.Fatal(err)
@@ -364,8 +386,8 @@ func TestRoleRevokePermission(t *testing.T) {
 
 	_, err = as.RoleRevokePermission(&pb.AuthRoleRevokePermissionRequest{
 		Role:     "role-test-1",
-		Key:      "Keys",
-		RangeEnd: "RangeEnd",
+		Key:      []byte("Keys"),
+		RangeEnd: []byte("RangeEnd"),
 	})
 	if err != nil {
 		t.Fatal(err)
@@ -467,19 +489,19 @@ func TestAuthInfoFromCtx(t *testing.T) {
 		t.Error(err)
 	}
 
-	ctx = metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{"token": "Invalid Token"}))
+	ctx = metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{rpctypes.TokenFieldNameGRPC: "Invalid Token"}))
 	_, err = as.AuthInfoFromCtx(ctx)
 	if err != ErrInvalidAuthToken {
 		t.Errorf("expected %v, got %v", ErrInvalidAuthToken, err)
 	}
 
-	ctx = metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{"token": "Invalid.Token"}))
+	ctx = metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{rpctypes.TokenFieldNameGRPC: "Invalid.Token"}))
 	_, err = as.AuthInfoFromCtx(ctx)
 	if err != ErrInvalidAuthToken {
 		t.Errorf("expected %v, got %v", ErrInvalidAuthToken, err)
 	}
 
-	ctx = metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{"token": resp.Token}))
+	ctx = metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{rpctypes.TokenFieldNameGRPC: resp.Token}))
 	ai, err = as.AuthInfoFromCtx(ctx)
 	if err != nil {
 		t.Error(err)
@@ -513,17 +535,17 @@ func TestAuthInfoFromCtxRace(t *testing.T) {
 	b, tPath := backend.NewDefaultTmpBackend()
 	defer os.Remove(tPath)
 
-	tp, err := NewTokenProvider("simple", dummyIndexWaiter)
+	tp, err := NewTokenProvider(zap.NewExample(), tokenTypeSimple, dummyIndexWaiter)
 	if err != nil {
 		t.Fatal(err)
 	}
-	as := NewAuthStore(b, tp)
+	as := NewAuthStore(zap.NewExample(), b, tp, bcrypt.MinCost)
 	defer as.Close()
 
 	donec := make(chan struct{})
 	go func() {
 		defer close(donec)
-		ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{"token": "test"}))
+		ctx := metadata.NewIncomingContext(context.Background(), metadata.New(map[string]string{rpctypes.TokenFieldNameGRPC: "test"}))
 		as.AuthInfoFromCtx(ctx)
 	}()
 	as.UserAdd(&pb.AuthUserAddRequest{Name: "test"})
@@ -579,16 +601,16 @@ func TestRecoverFromSnapshot(t *testing.T) {
 
 	as.Close()
 
-	tp, err := NewTokenProvider("simple", dummyIndexWaiter)
+	tp, err := NewTokenProvider(zap.NewExample(), tokenTypeSimple, dummyIndexWaiter)
 	if err != nil {
 		t.Fatal(err)
 	}
-	as2 := NewAuthStore(as.be, tp)
+	as2 := NewAuthStore(zap.NewExample(), as.be, tp, bcrypt.MinCost)
 	defer func(a *authStore) {
 		a.Close()
 	}(as2)
 
-	if !as2.isAuthEnabled() {
+	if !as2.IsAuthEnabled() {
 		t.Fatal("recovering authStore from existing backend failed")
 	}
 
@@ -661,36 +683,36 @@ func TestRolesOrder(t *testing.T) {
 	b, tPath := backend.NewDefaultTmpBackend()
 	defer os.Remove(tPath)
 
-	tp, err := NewTokenProvider("simple", dummyIndexWaiter)
+	tp, err := NewTokenProvider(zap.NewExample(), tokenTypeSimple, dummyIndexWaiter)
 	if err != nil {
 		t.Fatal(err)
 	}
-	as := NewAuthStore(b, tp)
+	as := NewAuthStore(zap.NewExample(), b, tp, bcrypt.MinCost)
 	err = enableAuthAndCreateRoot(as)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	username := "user"
-	_, err = as.UserAdd(&pb.AuthUserAddRequest{username, "pass"})
+	_, err = as.UserAdd(&pb.AuthUserAddRequest{Name: username, Password: "pass"})
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	roles := []string{"role1", "role2", "abc", "xyz", "role3"}
 	for _, role := range roles {
-		_, err = as.RoleAdd(&pb.AuthRoleAddRequest{role})
+		_, err = as.RoleAdd(&pb.AuthRoleAddRequest{Name: role})
 		if err != nil {
 			t.Fatal(err)
 		}
 
-		_, err = as.UserGrantRole(&pb.AuthUserGrantRoleRequest{username, role})
+		_, err = as.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: username, Role: role})
 		if err != nil {
 			t.Fatal(err)
 		}
 	}
 
-	user, err := as.UserGet(&pb.AuthUserGetRequest{username})
+	user, err := as.UserGet(&pb.AuthUserGetRequest{Name: username})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -702,16 +724,25 @@ func TestRolesOrder(t *testing.T) {
 	}
 }
 
-// TestAuthInfoFromCtxWithRoot ensures "WithRoot" properly embeds token in the context.
-func TestAuthInfoFromCtxWithRoot(t *testing.T) {
+func TestAuthInfoFromCtxWithRootSimple(t *testing.T) {
+	testAuthInfoFromCtxWithRoot(t, tokenTypeSimple)
+}
+
+func TestAuthInfoFromCtxWithRootJWT(t *testing.T) {
+	opts := testJWTOpts()
+	testAuthInfoFromCtxWithRoot(t, opts)
+}
+
+// testAuthInfoFromCtxWithRoot ensures "WithRoot" properly embeds token in the context.
+func testAuthInfoFromCtxWithRoot(t *testing.T, opts string) {
 	b, tPath := backend.NewDefaultTmpBackend()
 	defer os.Remove(tPath)
 
-	tp, err := NewTokenProvider("simple", dummyIndexWaiter)
+	tp, err := NewTokenProvider(zap.NewExample(), opts, dummyIndexWaiter)
 	if err != nil {
 		t.Fatal(err)
 	}
-	as := NewAuthStore(b, tp)
+	as := NewAuthStore(zap.NewExample(), b, tp, bcrypt.MinCost)
 	defer as.Close()
 
 	if err = enableAuthAndCreateRoot(as); err != nil {

bill-of-materials.json

@@ -1,13 +1,4 @@
 [
-	{
-		"project": "bitbucket.org/ww/goautoneg",
-		"licenses": [
-			{
-				"type": "BSD 3-clause \"New\" or \"Revised\" License",
-				"confidence": 1
-			}
-		]
-	},
 	{
 		"project": "github.com/beorn7/perks/quantile",
 		"licenses": [
@@ -71,15 +62,6 @@
 			}
 		]
 	},
-	{
-		"project": "github.com/cpuguy83/go-md2man/md2man",
-		"licenses": [
-			{
-				"type": "MIT License",
-				"confidence": 1
-			}
-		]
-	},
 	{
 		"project": "github.com/dgrijalva/jwt-go",
 		"licenses": [
@@ -152,6 +134,15 @@
 			}
 		]
 	},
+	{
+		"project": "github.com/grpc-ecosystem/go-grpc-middleware",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
 	{
 		"project": "github.com/grpc-ecosystem/go-grpc-prometheus",
 		"licenses": [
@@ -173,10 +164,6 @@
 	{
 		"project": "github.com/inconshreveable/mousetrap",
 		"licenses": [
-			{
-				"type": "MIT License and BSD 3-clause \"New\" or \"Revised\" License",
-				"confidence": 1
-			},
 			{
 				"type": "Apache License 2.0",
 				"confidence": 1
@@ -255,15 +242,6 @@
 			}
 		]
 	},
-	{
-		"project": "github.com/russross/blackfriday",
-		"licenses": [
-			{
-				"type": "BSD 2-clause \"Simplified\" License",
-				"confidence": 0.9626168224299065
-			}
-		]
-	},
 	{
 		"project": "github.com/sirupsen/logrus",
 		"licenses": [
@@ -336,6 +314,33 @@
 			}
 		]
 	},
+	{
+		"project": "go.uber.org/atomic",
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 0.9891304347826086
+			}
+		]
+	},
+	{
+		"project": "go.uber.org/multierr",
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 0.9891304347826086
+			}
+		]
+	},
+	{
+		"project": "go.uber.org/zap",
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 0.9891304347826086
+			}
+		]
+	},
 	{
 		"project": "golang.org/x/crypto",
 		"licenses": [
@@ -412,8 +417,8 @@
 		"project": "gopkg.in/yaml.v2",
 		"licenses": [
 			{
-				"type": "The Unlicense",
-				"confidence": 0.35294117647058826
+				"type": "Apache License 2.0",
+				"confidence": 1
 			},
 			{
 				"type": "MIT License",

bill-of-materials.override.json

@@ -1,12 +1,4 @@
 [
-	{
-		"project": "bitbucket.org/ww/goautoneg",
-		"licenses": [
-			{
-				"type": "BSD 3-clause \"New\" or \"Revised\" License"
-			}
-		]
-	},
 	{
 		"project": "github.com/ghodss/yaml",
 		"licenses": [

build

@@ -1,66 +1,98 @@
-#!/bin/sh -e
+#!/usr/bin/env bash
 
 # set some environment variables
 ORG_PATH="github.com/coreos"
 REPO_PATH="${ORG_PATH}/etcd"
 
 GIT_SHA=$(git rev-parse --short HEAD || echo "GitNotFound")
-if [ ! -z "$FAILPOINTS" ]; then
+if [[ ! -z "$FAILPOINTS" ]]; then
 	GIT_SHA="$GIT_SHA"-FAILPOINTS
 fi
 
 # Set GO_LDFLAGS="-s" for building without symbols for debugging.
-GO_LDFLAGS="$GO_LDFLAGS -X ${REPO_PATH}/cmd/vendor/${REPO_PATH}/version.GitSHA=${GIT_SHA}"
+GO_LDFLAGS="$GO_LDFLAGS -X ${REPO_PATH}/version.GitSHA=${GIT_SHA}"
 
 # enable/disable failpoints
 toggle_failpoints() {
 	mode="$1"
 	if which gofail >/dev/null 2>&1; then
 		gofail "$mode" etcdserver/ mvcc/backend/
-	elif [ "$mode" != "disable" ]; then
+	elif [[ "$mode" != "disable" ]]; then
 		echo "FAILPOINTS set but gofail not found"
 		exit 1
 	fi
 }
 
+etcd_setup_gopath() {
+	echo "Setting GOPATH from vendor directory at 'gopath'"
+	d=$(dirname "$0")
+	CDIR=$(cd "$d" || return && pwd)
+	cd "$CDIR" || return
+	etcdGOPATH="${CDIR}/gopath"
+	# preserve old gopath to support building with unvendored tooling deps (e.g., gofail)
+	if [[ -n "$GOPATH" ]]; then
+		GOPATH=":$GOPATH"
+	fi
+	rm -rf "${etcdGOPATH:?}/"
+	mkdir -p "${etcdGOPATH}/vendor" "${etcdGOPATH}/etcd_src/src/github.com/coreos"
+	export GOPATH=${etcdGOPATH}/vendor:${etcdGOPATH}/etcd_src${GOPATH}
+	ln -s "${CDIR}/vendor" "${etcdGOPATH}/vendor/src"
+	ln -s "${CDIR}" "${etcdGOPATH}/etcd_src/src/github.com/coreos/etcd"
+}
+
 toggle_failpoints_default() {
 	mode="disable"
-	if [ ! -z "$FAILPOINTS" ]; then mode="enable"; fi
+	if [[ ! -z "$FAILPOINTS" ]]; then mode="enable"; fi
 	toggle_failpoints "$mode"
 }
 
 etcd_build() {
 	out="bin"
-	if [ -n "${BINDIR}" ]; then out="${BINDIR}"; fi
+	if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi
 	toggle_failpoints_default
-	# Static compilation is useful when etcd is run in a container. $GO_BUILD_FLAGS is OK
 
+	# Static compilation is useful when etcd is run in a container. $GO_BUILD_FLAGS is OK
 	# shellcheck disable=SC2086
-	CGO_ENABLED=0 go build $GO_BUILD_FLAGS -installsuffix cgo -ldflags "$GO_LDFLAGS" -o "${out}/etcd" ${REPO_PATH}/cmd/etcd || return
+	CGO_ENABLED=0 go build $GO_BUILD_FLAGS \
+		-installsuffix cgo \
+		-ldflags "$GO_LDFLAGS" \
+		-o "${out}/etcd" ${REPO_PATH} || return
 	# shellcheck disable=SC2086
-	CGO_ENABLED=0 go build $GO_BUILD_FLAGS -installsuffix cgo -ldflags "$GO_LDFLAGS" -o "${out}/etcdctl" ${REPO_PATH}/cmd/etcdctl || return
+	CGO_ENABLED=0 go build $GO_BUILD_FLAGS \
+		-installsuffix cgo \
+		-ldflags "$GO_LDFLAGS" \
+		-o "${out}/etcdctl" ${REPO_PATH}/etcdctl || return
 }
 
-etcd_setup_gopath() {
-	d=$(dirname "$0")
-	CDIR=$(cd "$d" && pwd)
-	cd "$CDIR"
-	etcdGOPATH="${CDIR}/gopath"
-	# preserve old gopath to support building with unvendored tooling deps (e.g., gofail)
-	if [ -n "$GOPATH" ]; then
-		GOPATH=":$GOPATH"
-	fi
-	export GOPATH=${etcdGOPATH}$GOPATH
-	rm -rf "${etcdGOPATH}/src"
-	mkdir -p "${etcdGOPATH}"
-	ln -s "${CDIR}/cmd/vendor" "${etcdGOPATH}/src"
+tools_build() {
+	out="bin"
+	if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi
+	tools_path="tools/benchmark
+	tools/etcd-dump-db
+	tools/etcd-dump-logs
+	tools/local-tester/bridge
+	functional/cmd/etcd-agent
+	functional/cmd/etcd-proxy
+	functional/cmd/etcd-runner
+	functional/cmd/etcd-tester"
+	for tool in ${tools_path}
+	do
+		echo "Building" "'${tool}'"...
+		# shellcheck disable=SC2086
+		CGO_ENABLED=0 go build ${GO_BUILD_FLAGS} \
+		  -installsuffix cgo \
+		  -ldflags "${GO_LDFLAGS}" \
+		  -o "${out}/${tool}" "${REPO_PATH}/${tool}" || return
+	done
 }
 
 toggle_failpoints_default
 
+if [[ "${ETCD_SETUP_GOPATH}" == "1" ]]; then
+	etcd_setup_gopath
+fi
+
 # only build when called directly, not sourced
 if echo "$0" | grep "build$" >/dev/null; then
-	# force new gopath so builds outside of gopath work
-	etcd_setup_gopath
 	etcd_build
 fi

build.ps1

@@ -10,7 +10,7 @@ if ($FSYS.StartsWith("FAT","CurrentCultureIgnoreCase")) {
 }
 
 # Set $Env:GO_LDFLAGS="-s" for building without symbols.
-$GO_LDFLAGS="$Env:GO_LDFLAGS -X $REPO_PATH/cmd/vendor/$REPO_PATH/version.GitSHA=$GIT_SHA"
+$GO_LDFLAGS="$Env:GO_LDFLAGS -X $REPO_PATH/version.GitSHA=$GIT_SHA"
 
 # rebuild symlinks
 git ls-files -s cmd | select-string -pattern 120000 | ForEach {
@@ -77,5 +77,5 @@ if (-not $env:GOPATH) {
 $env:CGO_ENABLED = 0
 $env:GO15VENDOREXPERIMENT = 1
 $GIT_SHA="$(git rev-parse --short HEAD)"
-go build -a -installsuffix cgo -ldflags $GO_LDFLAGS -o bin\etcd.exe "$REPO_PATH\cmd\etcd"
-go build -a -installsuffix cgo -ldflags $GO_LDFLAGS -o bin\etcdctl.exe "$REPO_PATH\cmd\etcdctl"
+go build -a -installsuffix cgo -ldflags $GO_LDFLAGS -o bin\etcd.exe "$REPO_PATH"
+go build -a -installsuffix cgo -ldflags $GO_LDFLAGS -o bin\etcdctl.exe "$REPO_PATH\etcdctl"

client/README.md

@@ -4,12 +4,7 @@ etcd/client is the Go client library for etcd.
 
 [![GoDoc](https://godoc.org/github.com/coreos/etcd/client?status.png)](https://godoc.org/github.com/coreos/etcd/client)
 
-etcd uses `cmd/vendor` directory to store external dependencies, which are
-to be compiled into etcd release binaries. `client` can be imported without
-vendoring. For full compatibility, it is recommended to vendor builds using
-etcd's vendored packages, using tools like godep, as in
-[vendor directories](https://golang.org/cmd/go/#hdr-Vendor_Directories).
-For more detail, please read [Go vendor design](https://golang.org/s/go15vendor).
+For full compatibility, it is recommended to vendor builds using etcd's vendored packages, using tools like `golang/dep`, as in [vendor directories](https://golang.org/cmd/go/#hdr-Vendor_Directories).
 
 ## Install
 

client/client_test.go

@@ -472,7 +472,7 @@ func TestHTTPClusterClientDoDeadlineExceedContext(t *testing.T) {
 
 type fakeCancelContext struct{}
 
-var fakeCancelContextError = errors.New("fake context canceled")
+var errFakeCancelContext = errors.New("fake context canceled")
 
 func (f fakeCancelContext) Deadline() (time.Time, bool) { return time.Time{}, false }
 func (f fakeCancelContext) Done() <-chan struct{} {
@@ -480,11 +480,17 @@ func (f fakeCancelContext) Done() <-chan struct{} {
 	d <- struct{}{}
 	return d
 }
-func (f fakeCancelContext) Err() error                        { return fakeCancelContextError }
+func (f fakeCancelContext) Err() error                        { return errFakeCancelContext }
 func (f fakeCancelContext) Value(key interface{}) interface{} { return 1 }
 
-func withTimeout(parent context.Context, timeout time.Duration) (context.Context, context.CancelFunc) {
-	return parent, func() { parent = nil }
+func withTimeout(parent context.Context, timeout time.Duration) (
+	ctx context.Context,
+	cancel context.CancelFunc) {
+	ctx = parent
+	cancel = func() {
+		ctx = nil
+	}
+	return ctx, cancel
 }
 
 func TestHTTPClusterClientDoCanceledContext(t *testing.T) {
@@ -506,8 +512,8 @@ func TestHTTPClusterClientDoCanceledContext(t *testing.T) {
 
 	select {
 	case err := <-errc:
-		if err != fakeCancelContextError {
-			t.Errorf("err = %+v, want %+v", err, fakeCancelContextError)
+		if err != errFakeCancelContext {
+			t.Errorf("err = %+v, want %+v", err, errFakeCancelContext)
 		}
 	case <-time.After(time.Second):
 		t.Fatalf("unexpected timeout when waiting for request to fake context canceled")

client/integration/client_test.go

@@ -46,7 +46,7 @@ func TestV2NoRetryEOF(t *testing.T) {
 			conn.Close()
 		}
 	}()
-	eofURL := integration.UrlScheme + "://" + lEOF.Addr().String()
+	eofURL := integration.URLScheme + "://" + lEOF.Addr().String()
 	cli := integration.MustNewHTTPClient(t, []string{eofURL, eofURL}, nil)
 	kapi := client.NewKeysAPI(cli)
 	for i, f := range noRetryList(kapi) {
@@ -64,16 +64,16 @@ func TestV2NoRetryEOF(t *testing.T) {
 // TestV2NoRetryNoLeader tests destructive api calls won't retry if given an error code.
 func TestV2NoRetryNoLeader(t *testing.T) {
 	defer testutil.AfterTest(t)
-	lHttp := integration.NewListenerWithAddr(t, fmt.Sprintf("127.0.0.1:%05d", os.Getpid()))
+	lHTTP := integration.NewListenerWithAddr(t, fmt.Sprintf("127.0.0.1:%05d", os.Getpid()))
 	eh := &errHandler{errCode: http.StatusServiceUnavailable}
 	srv := httptest.NewUnstartedServer(eh)
-	defer lHttp.Close()
+	defer lHTTP.Close()
 	defer srv.Close()
-	srv.Listener = lHttp
+	srv.Listener = lHTTP
 	go srv.Start()
-	lHttpURL := integration.UrlScheme + "://" + lHttp.Addr().String()
+	lHTTPURL := integration.URLScheme + "://" + lHTTP.Addr().String()
 
-	cli := integration.MustNewHTTPClient(t, []string{lHttpURL, lHttpURL}, nil)
+	cli := integration.MustNewHTTPClient(t, []string{lHTTPURL, lHTTPURL}, nil)
 	kapi := client.NewKeysAPI(cli)
 	// test error code
 	for i, f := range noRetryList(kapi) {
@@ -94,7 +94,7 @@ func TestV2RetryRefuse(t *testing.T) {
 	cl.Launch(t)
 	defer cl.Terminate(t)
 	// test connection refused; expect no error failover
-	cli := integration.MustNewHTTPClient(t, []string{integration.UrlScheme + "://refuseconn:123", cl.URL(0)}, nil)
+	cli := integration.MustNewHTTPClient(t, []string{integration.URLScheme + "://refuseconn:123", cl.URL(0)}, nil)
 	kapi := client.NewKeysAPI(cli)
 	if _, err := kapi.Set(context.Background(), "/delkey", "def", nil); err != nil {
 		t.Fatal(err)

clientv3/README.md

@@ -1,9 +1,12 @@
 # etcd/clientv3
 
+[![Docs](https://readthedocs.org/projects/etcd/badge/?version=latest&style=flat-square)](https://etcd.readthedocs.io/en/latest/?badge=latest)
 [![Godoc](https://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godoc.org/github.com/coreos/etcd/clientv3)
 
 `etcd/clientv3` is the official Go etcd client for v3.
 
+See https://etcd.readthedocs.io/en/latest for latest client architecture.
+
 ## Install
 
 ```bash
@@ -26,7 +29,7 @@ defer cli.Close()
 ```
 
 etcd v3 uses [`gRPC`](http://www.grpc.io) for remote procedure calls. And `clientv3` uses
-[`grpc-go`](https://github.com/grpc/grpc-go) to connect to etcd. Make sure to close the client after using it. 
+[`grpc-go`](https://github.com/grpc/grpc-go) to connect to etcd. Make sure to close the client after using it.
 If the client is not closed, the connection will have leaky goroutines. To specify client request timeout,
 pass `context.WithTimeout` to APIs:
 
@@ -40,12 +43,7 @@ if err != nil {
 // use the response
 ```
 
-etcd uses `cmd/vendor` directory to store external dependencies, which are
-to be compiled into etcd release binaries. `client` can be imported without
-vendoring. For full compatibility, it is recommended to vendor builds using
-etcd's vendored packages, using tools like godep, as in
-[vendor directories](https://golang.org/cmd/go/#hdr-Vendor_Directories).
-For more detail, please read [Go vendor design](https://golang.org/s/go15vendor).
+For full compatibility, it is recommended to vendor builds using etcd's vendored packages, using tools like `golang/dep`, as in [vendor directories](https://golang.org/cmd/go/#hdr-Vendor_Directories).
 
 ## Error Handling
 
@@ -80,6 +78,10 @@ The etcd client optionally exposes RPC metrics through [go-grpc-prometheus](http
 
 The [namespace](https://godoc.org/github.com/coreos/etcd/clientv3/namespace) package provides `clientv3` interface wrappers to transparently isolate client requests to a user-defined prefix.
 
+## Request size limit
+
+Client request size limit is configurable via `clientv3.Config.MaxCallSendMsgSize` and `MaxCallRecvMsgSize` in bytes. If none given, client request send limit defaults to 2 MiB including gRPC overhead bytes. And receive limit defaults to `math.MaxInt32`.
+
 ## Examples
 
 More code examples can be found at [GoDoc](https://godoc.org/github.com/coreos/etcd/clientv3).

clientv3/auth.go

@@ -21,7 +21,6 @@ import (
 
 	"github.com/coreos/etcd/auth/authpb"
 	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
-
 	"google.golang.org/grpc"
 )
 
@@ -100,70 +99,70 @@ type Auth interface {
 	RoleDelete(ctx context.Context, role string) (*AuthRoleDeleteResponse, error)
 }
 
-type auth struct {
+type authClient struct {
 	remote   pb.AuthClient
 	callOpts []grpc.CallOption
 }
 
 func NewAuth(c *Client) Auth {
-	api := &auth{remote: RetryAuthClient(c)}
+	api := &authClient{remote: RetryAuthClient(c)}
 	if c != nil {
 		api.callOpts = c.callOpts
 	}
 	return api
 }
 
-func (auth *auth) AuthEnable(ctx context.Context) (*AuthEnableResponse, error) {
+func (auth *authClient) AuthEnable(ctx context.Context) (*AuthEnableResponse, error) {
 	resp, err := auth.remote.AuthEnable(ctx, &pb.AuthEnableRequest{}, auth.callOpts...)
 	return (*AuthEnableResponse)(resp), toErr(ctx, err)
 }
 
-func (auth *auth) AuthDisable(ctx context.Context) (*AuthDisableResponse, error) {
+func (auth *authClient) AuthDisable(ctx context.Context) (*AuthDisableResponse, error) {
 	resp, err := auth.remote.AuthDisable(ctx, &pb.AuthDisableRequest{}, auth.callOpts...)
 	return (*AuthDisableResponse)(resp), toErr(ctx, err)
 }
 
-func (auth *auth) UserAdd(ctx context.Context, name string, password string) (*AuthUserAddResponse, error) {
+func (auth *authClient) UserAdd(ctx context.Context, name string, password string) (*AuthUserAddResponse, error) {
 	resp, err := auth.remote.UserAdd(ctx, &pb.AuthUserAddRequest{Name: name, Password: password}, auth.callOpts...)
 	return (*AuthUserAddResponse)(resp), toErr(ctx, err)
 }
 
-func (auth *auth) UserDelete(ctx context.Context, name string) (*AuthUserDeleteResponse, error) {
+func (auth *authClient) UserDelete(ctx context.Context, name string) (*AuthUserDeleteResponse, error) {
 	resp, err := auth.remote.UserDelete(ctx, &pb.AuthUserDeleteRequest{Name: name}, auth.callOpts...)
 	return (*AuthUserDeleteResponse)(resp), toErr(ctx, err)
 }
 
-func (auth *auth) UserChangePassword(ctx context.Context, name string, password string) (*AuthUserChangePasswordResponse, error) {
+func (auth *authClient) UserChangePassword(ctx context.Context, name string, password string) (*AuthUserChangePasswordResponse, error) {
 	resp, err := auth.remote.UserChangePassword(ctx, &pb.AuthUserChangePasswordRequest{Name: name, Password: password}, auth.callOpts...)
 	return (*AuthUserChangePasswordResponse)(resp), toErr(ctx, err)
 }
 
-func (auth *auth) UserGrantRole(ctx context.Context, user string, role string) (*AuthUserGrantRoleResponse, error) {
+func (auth *authClient) UserGrantRole(ctx context.Context, user string, role string) (*AuthUserGrantRoleResponse, error) {
 	resp, err := auth.remote.UserGrantRole(ctx, &pb.AuthUserGrantRoleRequest{User: user, Role: role}, auth.callOpts...)
 	return (*AuthUserGrantRoleResponse)(resp), toErr(ctx, err)
 }
 
-func (auth *auth) UserGet(ctx context.Context, name string) (*AuthUserGetResponse, error) {
+func (auth *authClient) UserGet(ctx context.Context, name string) (*AuthUserGetResponse, error) {
 	resp, err := auth.remote.UserGet(ctx, &pb.AuthUserGetRequest{Name: name}, auth.callOpts...)
 	return (*AuthUserGetResponse)(resp), toErr(ctx, err)
 }
 
-func (auth *auth) UserList(ctx context.Context) (*AuthUserListResponse, error) {
+func (auth *authClient) UserList(ctx context.Context) (*AuthUserListResponse, error) {
 	resp, err := auth.remote.UserList(ctx, &pb.AuthUserListRequest{}, auth.callOpts...)
 	return (*AuthUserListResponse)(resp), toErr(ctx, err)
 }
 
-func (auth *auth) UserRevokeRole(ctx context.Context, name string, role string) (*AuthUserRevokeRoleResponse, error) {
+func (auth *authClient) UserRevokeRole(ctx context.Context, name string, role string) (*AuthUserRevokeRoleResponse, error) {
 	resp, err := auth.remote.UserRevokeRole(ctx, &pb.AuthUserRevokeRoleRequest{Name: name, Role: role}, auth.callOpts...)
 	return (*AuthUserRevokeRoleResponse)(resp), toErr(ctx, err)
 }
 
-func (auth *auth) RoleAdd(ctx context.Context, name string) (*AuthRoleAddResponse, error) {
+func (auth *authClient) RoleAdd(ctx context.Context, name string) (*AuthRoleAddResponse, error) {
 	resp, err := auth.remote.RoleAdd(ctx, &pb.AuthRoleAddRequest{Name: name}, auth.callOpts...)
 	return (*AuthRoleAddResponse)(resp), toErr(ctx, err)
 }
 
-func (auth *auth) RoleGrantPermission(ctx context.Context, name string, key, rangeEnd string, permType PermissionType) (*AuthRoleGrantPermissionResponse, error) {
+func (auth *authClient) RoleGrantPermission(ctx context.Context, name string, key, rangeEnd string, permType PermissionType) (*AuthRoleGrantPermissionResponse, error) {
 	perm := &authpb.Permission{
 		Key:      []byte(key),
 		RangeEnd: []byte(rangeEnd),
@@ -173,22 +172,22 @@ func (auth *auth) RoleGrantPermission(ctx context.Context, name string, key, ran
 	return (*AuthRoleGrantPermissionResponse)(resp), toErr(ctx, err)
 }
 
-func (auth *auth) RoleGet(ctx context.Context, role string) (*AuthRoleGetResponse, error) {
+func (auth *authClient) RoleGet(ctx context.Context, role string) (*AuthRoleGetResponse, error) {
 	resp, err := auth.remote.RoleGet(ctx, &pb.AuthRoleGetRequest{Role: role}, auth.callOpts...)
 	return (*AuthRoleGetResponse)(resp), toErr(ctx, err)
 }
 
-func (auth *auth) RoleList(ctx context.Context) (*AuthRoleListResponse, error) {
+func (auth *authClient) RoleList(ctx context.Context) (*AuthRoleListResponse, error) {
 	resp, err := auth.remote.RoleList(ctx, &pb.AuthRoleListRequest{}, auth.callOpts...)
 	return (*AuthRoleListResponse)(resp), toErr(ctx, err)
 }
 
-func (auth *auth) RoleRevokePermission(ctx context.Context, role string, key, rangeEnd string) (*AuthRoleRevokePermissionResponse, error) {
-	resp, err := auth.remote.RoleRevokePermission(ctx, &pb.AuthRoleRevokePermissionRequest{Role: role, Key: key, RangeEnd: rangeEnd}, auth.callOpts...)
+func (auth *authClient) RoleRevokePermission(ctx context.Context, role string, key, rangeEnd string) (*AuthRoleRevokePermissionResponse, error) {
+	resp, err := auth.remote.RoleRevokePermission(ctx, &pb.AuthRoleRevokePermissionRequest{Role: role, Key: []byte(key), RangeEnd: []byte(rangeEnd)}, auth.callOpts...)
 	return (*AuthRoleRevokePermissionResponse)(resp), toErr(ctx, err)
 }
 
-func (auth *auth) RoleDelete(ctx context.Context, role string) (*AuthRoleDeleteResponse, error) {
+func (auth *authClient) RoleDelete(ctx context.Context, role string) (*AuthRoleDeleteResponse, error) {
 	resp, err := auth.remote.RoleDelete(ctx, &pb.AuthRoleDeleteRequest{Role: role}, auth.callOpts...)
 	return (*AuthRoleDeleteResponse)(resp), toErr(ctx, err)
 }
@@ -216,8 +215,8 @@ func (auth *authenticator) close() {
 	auth.conn.Close()
 }
 
-func newAuthenticator(endpoint string, opts []grpc.DialOption, c *Client) (*authenticator, error) {
-	conn, err := grpc.Dial(endpoint, opts...)
+func newAuthenticator(ctx context.Context, target string, opts []grpc.DialOption, c *Client) (*authenticator, error) {
+	conn, err := grpc.DialContext(ctx, target, opts...)
 	if err != nil {
 		return nil, err
 	}

clientv3/balancer/balancer.go

@@ -0,0 +1,275 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package balancer
+
+import (
+	"fmt"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/coreos/etcd/clientv3/balancer/picker"
+
+	"go.uber.org/zap"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/resolver"
+	_ "google.golang.org/grpc/resolver/dns"         // register DNS resolver
+	_ "google.golang.org/grpc/resolver/passthrough" // register passthrough resolver
+)
+
+// RegisterBuilder creates and registers a builder. Since this function calls balancer.Register, it
+// must be invoked at initialization time.
+func RegisterBuilder(cfg Config) {
+	bb := &builder{cfg}
+	balancer.Register(bb)
+
+	bb.cfg.Logger.Info(
+		"registered balancer",
+		zap.String("policy", bb.cfg.Policy.String()),
+		zap.String("name", bb.cfg.Name),
+	)
+}
+
+type builder struct {
+	cfg Config
+}
+
+// Build is called initially when creating "ccBalancerWrapper".
+// "grpc.Dial" is called to this client connection.
+// Then, resolved addresses will be handled via "HandleResolvedAddrs".
+func (b *builder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balancer.Balancer {
+	bb := &baseBalancer{
+		id:     strconv.FormatInt(time.Now().UnixNano(), 36),
+		policy: b.cfg.Policy,
+		name:   b.cfg.Policy.String(),
+		lg:     b.cfg.Logger,
+
+		addrToSc: make(map[resolver.Address]balancer.SubConn),
+		scToAddr: make(map[balancer.SubConn]resolver.Address),
+		scToSt:   make(map[balancer.SubConn]connectivity.State),
+
+		currentConn: nil,
+		csEvltr:     &connectivityStateEvaluator{},
+
+		// initialize picker always returns "ErrNoSubConnAvailable"
+		Picker: picker.NewErr(balancer.ErrNoSubConnAvailable),
+	}
+	if b.cfg.Name != "" {
+		bb.name = b.cfg.Name
+	}
+	if bb.lg == nil {
+		bb.lg = zap.NewNop()
+	}
+
+	// TODO: support multiple connections
+	bb.mu.Lock()
+	bb.currentConn = cc
+	bb.mu.Unlock()
+
+	bb.lg.Info(
+		"built balancer",
+		zap.String("balancer-id", bb.id),
+		zap.String("policy", bb.policy.String()),
+		zap.String("resolver-target", cc.Target()),
+	)
+	return bb
+}
+
+// Name implements "grpc/balancer.Builder" interface.
+func (b *builder) Name() string { return b.cfg.Name }
+
+// Balancer defines client balancer interface.
+type Balancer interface {
+	// Balancer is called on specified client connection. Client initiates gRPC
+	// connection with "grpc.Dial(addr, grpc.WithBalancerName)", and then those resolved
+	// addresses are passed to "grpc/balancer.Balancer.HandleResolvedAddrs".
+	// For each resolved address, balancer calls "balancer.ClientConn.NewSubConn".
+	// "grpc/balancer.Balancer.HandleSubConnStateChange" is called when connectivity state
+	// changes, thus requires failover logic in this method.
+	balancer.Balancer
+
+	// Picker calls "Pick" for every client request.
+	picker.Picker
+}
+
+type baseBalancer struct {
+	id     string
+	policy picker.Policy
+	name   string
+	lg     *zap.Logger
+
+	mu sync.RWMutex
+
+	addrToSc map[resolver.Address]balancer.SubConn
+	scToAddr map[balancer.SubConn]resolver.Address
+	scToSt   map[balancer.SubConn]connectivity.State
+
+	currentConn  balancer.ClientConn
+	currentState connectivity.State
+	csEvltr      *connectivityStateEvaluator
+
+	picker.Picker
+}
+
+// HandleResolvedAddrs implements "grpc/balancer.Balancer" interface.
+// gRPC sends initial or updated resolved addresses from "Build".
+func (bb *baseBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) {
+	if err != nil {
+		bb.lg.Warn("HandleResolvedAddrs called with error", zap.String("balancer-id", bb.id), zap.Error(err))
+		return
+	}
+	bb.lg.Info("resolved", zap.String("balancer-id", bb.id), zap.Strings("addresses", addrsToStrings(addrs)))
+
+	bb.mu.Lock()
+	defer bb.mu.Unlock()
+
+	resolved := make(map[resolver.Address]struct{})
+	for _, addr := range addrs {
+		resolved[addr] = struct{}{}
+		if _, ok := bb.addrToSc[addr]; !ok {
+			sc, err := bb.currentConn.NewSubConn([]resolver.Address{addr}, balancer.NewSubConnOptions{})
+			if err != nil {
+				bb.lg.Warn("NewSubConn failed", zap.String("balancer-id", bb.id), zap.Error(err), zap.String("address", addr.Addr))
+				continue
+			}
+			bb.addrToSc[addr] = sc
+			bb.scToAddr[sc] = addr
+			bb.scToSt[sc] = connectivity.Idle
+			sc.Connect()
+		}
+	}
+
+	for addr, sc := range bb.addrToSc {
+		if _, ok := resolved[addr]; !ok {
+			// was removed by resolver or failed to create subconn
+			bb.currentConn.RemoveSubConn(sc)
+			delete(bb.addrToSc, addr)
+
+			bb.lg.Info(
+				"removed subconn",
+				zap.String("balancer-id", bb.id),
+				zap.String("address", addr.Addr),
+				zap.String("subconn", scToString(sc)),
+			)
+
+			// Keep the state of this sc in bb.scToSt until sc's state becomes Shutdown.
+			// The entry will be deleted in HandleSubConnStateChange.
+			// (DO NOT) delete(bb.scToAddr, sc)
+			// (DO NOT) delete(bb.scToSt, sc)
+		}
+	}
+}
+
+// HandleSubConnStateChange implements "grpc/balancer.Balancer" interface.
+func (bb *baseBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) {
+	bb.mu.Lock()
+	defer bb.mu.Unlock()
+
+	old, ok := bb.scToSt[sc]
+	if !ok {
+		bb.lg.Warn(
+			"state change for an unknown subconn",
+			zap.String("balancer-id", bb.id),
+			zap.String("subconn", scToString(sc)),
+			zap.String("state", s.String()),
+		)
+		return
+	}
+
+	bb.lg.Info(
+		"state changed",
+		zap.String("balancer-id", bb.id),
+		zap.Bool("connected", s == connectivity.Ready),
+		zap.String("subconn", scToString(sc)),
+		zap.String("address", bb.scToAddr[sc].Addr),
+		zap.String("old-state", old.String()),
+		zap.String("new-state", s.String()),
+	)
+
+	bb.scToSt[sc] = s
+	switch s {
+	case connectivity.Idle:
+		sc.Connect()
+	case connectivity.Shutdown:
+		// When an address was removed by resolver, b called RemoveSubConn but
+		// kept the sc's state in scToSt. Remove state for this sc here.
+		delete(bb.scToAddr, sc)
+		delete(bb.scToSt, sc)
+	}
+
+	oldAggrState := bb.currentState
+	bb.currentState = bb.csEvltr.recordTransition(old, s)
+
+	// Regenerate picker when one of the following happens:
+	//  - this sc became ready from not-ready
+	//  - this sc became not-ready from ready
+	//  - the aggregated state of balancer became TransientFailure from non-TransientFailure
+	//  - the aggregated state of balancer became non-TransientFailure from TransientFailure
+	if (s == connectivity.Ready) != (old == connectivity.Ready) ||
+		(bb.currentState == connectivity.TransientFailure) != (oldAggrState == connectivity.TransientFailure) {
+		bb.regeneratePicker()
+	}
+
+	bb.currentConn.UpdateBalancerState(bb.currentState, bb.Picker)
+	return
+}
+
+func (bb *baseBalancer) regeneratePicker() {
+	if bb.currentState == connectivity.TransientFailure {
+		bb.lg.Info(
+			"generated transient error picker",
+			zap.String("balancer-id", bb.id),
+			zap.String("policy", bb.policy.String()),
+		)
+		bb.Picker = picker.NewErr(balancer.ErrTransientFailure)
+		return
+	}
+
+	// only pass ready subconns to picker
+	scs := make([]balancer.SubConn, 0)
+	addrToSc := make(map[resolver.Address]balancer.SubConn)
+	scToAddr := make(map[balancer.SubConn]resolver.Address)
+	for addr, sc := range bb.addrToSc {
+		if st, ok := bb.scToSt[sc]; ok && st == connectivity.Ready {
+			scs = append(scs, sc)
+			addrToSc[addr] = sc
+			scToAddr[sc] = addr
+		}
+	}
+
+	switch bb.policy {
+	case picker.RoundrobinBalanced:
+		bb.Picker = picker.NewRoundrobinBalanced(bb.lg, scs, addrToSc, scToAddr)
+
+	default:
+		panic(fmt.Errorf("invalid balancer picker policy (%d)", bb.policy))
+	}
+
+	bb.lg.Info(
+		"generated picker",
+		zap.String("balancer-id", bb.id),
+		zap.String("policy", bb.policy.String()),
+		zap.Strings("subconn-ready", scsToStrings(addrToSc)),
+		zap.Int("subconn-size", len(addrToSc)),
+	)
+}
+
+// Close implements "grpc/balancer.Balancer" interface.
+// Close is a nop because base balancer doesn't have internal state to clean up,
+// and it doesn't need to call RemoveSubConn for the SubConns.
+func (bb *baseBalancer) Close() {
+	// TODO
+}

clientv3/balancer/balancer_test.go

@@ -0,0 +1,310 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package balancer
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/coreos/etcd/clientv3/balancer/picker"
+	"github.com/coreos/etcd/clientv3/balancer/resolver/endpoint"
+	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
+	"github.com/coreos/etcd/pkg/mock/mockserver"
+
+	"go.uber.org/zap"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/status"
+)
+
+// TestRoundRobinBalancedResolvableNoFailover ensures that
+// requests to a resolvable endpoint can be balanced between
+// multiple, if any, nodes. And there needs be no failover.
+func TestRoundRobinBalancedResolvableNoFailover(t *testing.T) {
+	testCases := []struct {
+		name        string
+		serverCount int
+		reqN        int
+		network     string
+	}{
+		{name: "rrBalanced_1", serverCount: 1, reqN: 5, network: "tcp"},
+		{name: "rrBalanced_1_unix_sockets", serverCount: 1, reqN: 5, network: "unix"},
+		{name: "rrBalanced_3", serverCount: 3, reqN: 7, network: "tcp"},
+		{name: "rrBalanced_5", serverCount: 5, reqN: 10, network: "tcp"},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			ms, err := mockserver.StartMockServersOnNetwork(tc.serverCount, tc.network)
+			if err != nil {
+				t.Fatalf("failed to start mock servers: %v", err)
+			}
+			defer ms.Stop()
+
+			var eps []string
+			for _, svr := range ms.Servers {
+				eps = append(eps, svr.ResolverAddress().Addr)
+			}
+
+			rsv, err := endpoint.NewResolverGroup("nofailover")
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer rsv.Close()
+			rsv.SetEndpoints(eps)
+
+			name := genName()
+			cfg := Config{
+				Policy: picker.RoundrobinBalanced,
+				Name:   name,
+				Logger: zap.NewExample(),
+			}
+			RegisterBuilder(cfg)
+			conn, err := grpc.Dial(fmt.Sprintf("endpoint://nofailover/*"), grpc.WithInsecure(), grpc.WithBalancerName(name))
+			if err != nil {
+				t.Fatalf("failed to dial mock server: %v", err)
+			}
+			defer conn.Close()
+			cli := pb.NewKVClient(conn)
+
+			reqFunc := func(ctx context.Context) (picked string, err error) {
+				var p peer.Peer
+				_, err = cli.Range(ctx, &pb.RangeRequest{Key: []byte("/x")}, grpc.Peer(&p))
+				if p.Addr != nil {
+					picked = p.Addr.String()
+				}
+				return picked, err
+			}
+
+			prev, switches := "", 0
+			for i := 0; i < tc.reqN; i++ {
+				picked, err := reqFunc(context.Background())
+				if err != nil {
+					t.Fatalf("#%d: unexpected failure %v", i, err)
+				}
+				if prev == "" {
+					prev = picked
+					continue
+				}
+				if prev != picked {
+					switches++
+				}
+				prev = picked
+			}
+			if tc.serverCount > 1 && switches < tc.reqN-3 { // -3 for initial resolutions
+				// TODO: FIX ME
+				t.Skipf("expected balanced loads for %d requests, got switches %d", tc.reqN, switches)
+			}
+		})
+	}
+}
+
+// TestRoundRobinBalancedResolvableFailoverFromServerFail ensures that
+// loads be rebalanced while one server goes down and comes back.
+func TestRoundRobinBalancedResolvableFailoverFromServerFail(t *testing.T) {
+	serverCount := 5
+	ms, err := mockserver.StartMockServers(serverCount)
+	if err != nil {
+		t.Fatalf("failed to start mock servers: %s", err)
+	}
+	defer ms.Stop()
+	var eps []string
+	for _, svr := range ms.Servers {
+		eps = append(eps, svr.ResolverAddress().Addr)
+	}
+
+	rsv, err := endpoint.NewResolverGroup("serverfail")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer rsv.Close()
+	rsv.SetEndpoints(eps)
+
+	name := genName()
+	cfg := Config{
+		Policy: picker.RoundrobinBalanced,
+		Name:   name,
+		Logger: zap.NewExample(),
+	}
+	RegisterBuilder(cfg)
+	conn, err := grpc.Dial(fmt.Sprintf("endpoint://serverfail/mock.server"), grpc.WithInsecure(), grpc.WithBalancerName(name))
+	if err != nil {
+		t.Fatalf("failed to dial mock server: %s", err)
+	}
+	defer conn.Close()
+	cli := pb.NewKVClient(conn)
+
+	reqFunc := func(ctx context.Context) (picked string, err error) {
+		var p peer.Peer
+		_, err = cli.Range(ctx, &pb.RangeRequest{Key: []byte("/x")}, grpc.Peer(&p))
+		if p.Addr != nil {
+			picked = p.Addr.String()
+		}
+		return picked, err
+	}
+
+	// stop first server, loads should be redistributed
+	// stopped server should never be picked
+	ms.StopAt(0)
+	available := make(map[string]struct{})
+	for i := 1; i < serverCount; i++ {
+		available[eps[i]] = struct{}{}
+	}
+
+	reqN := 10
+	prev, switches := "", 0
+	for i := 0; i < reqN; i++ {
+		picked, err := reqFunc(context.Background())
+		if err != nil && strings.Contains(err.Error(), "transport is closing") {
+			continue
+		}
+		if prev == "" { // first failover
+			if eps[0] == picked {
+				t.Fatalf("expected failover from %q, picked %q", eps[0], picked)
+			}
+			prev = picked
+			continue
+		}
+		if _, ok := available[picked]; !ok {
+			t.Fatalf("picked unavailable address %q (available %v)", picked, available)
+		}
+		if prev != picked {
+			switches++
+		}
+		prev = picked
+	}
+	if switches < reqN-3 { // -3 for initial resolutions + failover
+		// TODO: FIX ME!
+		t.Skipf("expected balanced loads for %d requests, got switches %d", reqN, switches)
+	}
+
+	// now failed server comes back
+	ms.StartAt(0)
+
+	// enough time for reconnecting to recovered server
+	time.Sleep(time.Second)
+
+	prev, switches = "", 0
+	recoveredAddr, recovered := eps[0], 0
+	available[recoveredAddr] = struct{}{}
+
+	for i := 0; i < 2*reqN; i++ {
+		picked, err := reqFunc(context.Background())
+		if err != nil {
+			t.Fatalf("#%d: unexpected failure %v", i, err)
+		}
+		if prev == "" {
+			prev = picked
+			continue
+		}
+		if _, ok := available[picked]; !ok {
+			t.Fatalf("#%d: picked unavailable address %q (available %v)", i, picked, available)
+		}
+		if prev != picked {
+			switches++
+		}
+		if picked == recoveredAddr {
+			recovered++
+		}
+		prev = picked
+	}
+	if switches < reqN-3 { // -3 for initial resolutions
+		t.Fatalf("expected balanced loads for %d requests, got switches %d", reqN, switches)
+	}
+	if recovered < reqN/serverCount {
+		t.Fatalf("recovered server %q got only %d requests", recoveredAddr, recovered)
+	}
+}
+
+// TestRoundRobinBalancedResolvableFailoverFromRequestFail ensures that
+// loads be rebalanced while some requests are failed.
+func TestRoundRobinBalancedResolvableFailoverFromRequestFail(t *testing.T) {
+	serverCount := 5
+	ms, err := mockserver.StartMockServers(serverCount)
+	if err != nil {
+		t.Fatalf("failed to start mock servers: %s", err)
+	}
+	defer ms.Stop()
+	var eps []string
+	available := make(map[string]struct{})
+	for _, svr := range ms.Servers {
+		eps = append(eps, svr.ResolverAddress().Addr)
+		available[svr.Address] = struct{}{}
+	}
+	rsv, err := endpoint.NewResolverGroup("requestfail")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer rsv.Close()
+	rsv.SetEndpoints(eps)
+
+	name := genName()
+	cfg := Config{
+		Policy: picker.RoundrobinBalanced,
+		Name:   name,
+		Logger: zap.NewExample(),
+	}
+	RegisterBuilder(cfg)
+	conn, err := grpc.Dial(fmt.Sprintf("endpoint://requestfail/mock.server"), grpc.WithInsecure(), grpc.WithBalancerName(name))
+	if err != nil {
+		t.Fatalf("failed to dial mock server: %s", err)
+	}
+	defer conn.Close()
+	cli := pb.NewKVClient(conn)
+
+	reqFunc := func(ctx context.Context) (picked string, err error) {
+		var p peer.Peer
+		_, err = cli.Range(ctx, &pb.RangeRequest{Key: []byte("/x")}, grpc.Peer(&p))
+		if p.Addr != nil {
+			picked = p.Addr.String()
+		}
+		return picked, err
+	}
+
+	reqN := 20
+	prev, switches := "", 0
+	for i := 0; i < reqN; i++ {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		if i%2 == 0 {
+			cancel()
+		}
+		picked, err := reqFunc(ctx)
+		if i%2 == 0 {
+			if s, ok := status.FromError(err); ok && s.Code() != codes.Canceled || picked != "" {
+				t.Fatalf("#%d: expected %v, got %v", i, context.Canceled, err)
+			}
+			continue
+		}
+		if prev == "" && picked != "" {
+			prev = picked
+			continue
+		}
+		if _, ok := available[picked]; !ok {
+			t.Fatalf("#%d: picked unavailable address %q (available %v)", i, picked, available)
+		}
+		if prev != picked {
+			switches++
+		}
+		prev = picked
+	}
+	if switches < reqN/2-3 { // -3 for initial resolutions + failover
+		t.Fatalf("expected balanced loads for %d requests, got switches %d", reqN, switches)
+	}
+}

clientv3/balancer/config.go

@@ -0,0 +1,36 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package balancer
+
+import (
+	"github.com/coreos/etcd/clientv3/balancer/picker"
+
+	"go.uber.org/zap"
+)
+
+// Config defines balancer configurations.
+type Config struct {
+	// Policy configures balancer policy.
+	Policy picker.Policy
+
+	// Name defines an additional name for balancer.
+	// Useful for balancer testing to avoid register conflicts.
+	// If empty, defaults to policy name.
+	Name string
+
+	// Logger configures balancer logging.
+	// If nil, logs are discarded.
+	Logger *zap.Logger
+}