version: bump up to 3.2.4

Signed-off-by: Gyu-Ho Lee <gyuhox@gmail.com>

Documentation/dev-guide/api_grpc_gateway.md

@@ -24,6 +24,11 @@ curl -L http://localhost:2379/v3alpha/kv/put \
 curl -L http://localhost:2379/v3alpha/kv/range \
 	-X POST -d '{"key": "Zm9v"}'
 # {"header":{"cluster_id":"12585971608760269493","member_id":"13847567121247652255","revision":"2","raft_term":"3"},"kvs":[{"key":"Zm9v","create_revision":"2","mod_revision":"2","version":"1","value":"YmFy"}],"count":"1"}
+
+# get all keys prefixed with "foo"
+curl -L http://localhost:2379/v3alpha/kv/range \
+	-X POST -d '{"key": "Zm9v", "range_end": "Zm9w"}'
+# {"header":{"cluster_id":"12585971608760269493","member_id":"13847567121247652255","revision":"2","raft_term":"3"},"kvs":[{"key":"Zm9v","create_revision":"2","mod_revision":"2","version":"1","value":"YmFy"}],"count":"1"}
 ```
 
 Use `curl` to watch a key:

Documentation/dev-guide/experimental_apis.md

@@ -4,8 +4,4 @@ For the most part, the etcd project is stable, but we are still moving fast! We
 
 ## The current experimental API/features are:
 
-- [gateway][gateway]: beta, to be stable in 3.2 release
-- [gRPC proxy][grpc-proxy]: alpha, to be stable in 3.2 release
-
-[gateway]: ../op-guide/gateway.md
-[grpc-proxy]: ../op-guide/grpc_proxy.md
+(none currently)

Documentation/faq.md

@@ -8,11 +8,11 @@
 
 ### Configuration
 
-#### What is the difference between advertise-urls and listen-urls?
+#### What is the difference between listen-<client,peer>-urls, advertise-client-urls or initial-advertise-peer-urls?
 
-`listen-urls` specifies the local addresses etcd server binds to for accepting incoming connections. To listen on a port for all interfaces, specify `0.0.0.0` as the listen IP address.
+`listen-client-urls` and `listen-peer-urls` specify the local addresses etcd server binds to for accepting incoming connections. To listen on a port for all interfaces, specify `0.0.0.0` as the listen IP address.
 
-`advertise-urls` specifies the addresses etcd clients or other etcd members should use to contact the etcd server. The advertise addresses must be reachable from the remote machines. Do not advertise addresses like `localhost` or `0.0.0.0` for a production setup since these addresses are unreachable from remote machines.
+`advertise-client-urls` and `initial-advertise-peer-urls` specify the addresses etcd clients or other etcd members should use to contact the etcd server. The advertise addresses must be reachable from the remote machines. Do not advertise addresses like `localhost` or `0.0.0.0` for a production setup since these addresses are unreachable from remote machines.
 
 ### Deployment
 
@@ -107,7 +107,7 @@ Try the [benchmark] tool. Current [benchmark results][benchmark-result] are avai
 #### What does the etcd warning "apply entries took too long" mean?
 
 After a majority of etcd members agree to commit a request, each etcd server applies the request to its data store and persists the result to disk. Even with a slow mechanical disk or a virtualized network disk, such as Amazon’s EBS or Google’s PD, applying a request should normally take fewer than 50 milliseconds. If the average apply duration exceeds 100 milliseconds, etcd will warn that entries are taking too long to apply.
- 
+
 Usually this issue is caused by a slow disk. The disk could be experiencing contention among etcd and other applications, or the disk is too simply slow (e.g., a shared virtualized disk). To rule out a slow disk from causing this warning, monitor  [backend_commit_duration_seconds][backend_commit_metrics] (p99 duration should be less than 25ms) to confirm the disk is reasonably fast. If the disk is too slow, assigning a dedicated disk to etcd or using faster disk will typically solve the problem.
 
 The second most common cause is CPU starvation. If monitoring of the machine’s CPU usage shows heavy utilization, there may not be enough compute capacity for etcd. Moving etcd to dedicated machine, increasing process resource isolation  cgroups, or renicing the etcd server process into a higher priority can usually solve the problem.

Documentation/learning/api.md

@@ -449,7 +449,7 @@ message LeaseRevokeRequest {
 
 ### Keep alives
 
-Leases are refreshed using a bi-directional stream created with the `LeaseKeepAlive` API call. When the client wishes to refresh a lease, it sends a `LeaseGrantRequest` over the stream:
+Leases are refreshed using a bi-directional stream created with the `LeaseKeepAlive` API call. When the client wishes to refresh a lease, it sends a `LeaseKeepAliveRequest` over the stream:
 
 ```protobuf
 message LeaseKeepAliveRequest {

Documentation/v2/04_to_2_snapshot_migration.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Snapshot Migration
 
 You can migrate a snapshot of your data from a v0.4.9+ cluster into a new etcd 2.2 cluster using a snapshot migration. After snapshot migration, the etcd indexes of your data will change. Many etcd applications rely on these indexes to behave correctly. This operation should only be done while all etcd applications are stopped.

Documentation/v2/README.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # etcd2
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/coreos/etcd)](https://goreportcard.com/report/github.com/coreos/etcd)

Documentation/v2/admin_guide.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Administration
 
 ## Data Directory
@@ -8,7 +13,7 @@ When first started, etcd stores its configuration into a data directory specifie
 Configuration is stored in the write ahead log and includes: the local member ID, cluster ID, and initial cluster configuration.
 The write ahead log and snapshot files are used during member operation and to recover after a restart.
 
-Having a dedicated disk to store wal files can improve the throughput and stabilize the cluster. 
+Having a dedicated disk to store wal files can improve the throughput and stabilize the cluster.
 It is highly recommended to dedicate a wal disk and set `--wal-dir` to point to a directory on that device for a production cluster deployment.
 
 If a member’s data directory is ever lost or corrupted then the user should [remove][remove-a-member] the etcd member from the cluster using `etcdctl` tool.
@@ -51,7 +56,7 @@ $ curl -L http://127.0.0.1:2379/health
 You can also use etcdctl to check the cluster-wide health information. It will contact all the members of the cluster and collect the health information for you.
 
 ```
-$./etcdctl cluster-health 
+$./etcdctl cluster-health
 member 8211f1d0f64f3269 is healthy: got healthy result from http://127.0.0.1:12379
 member 91bc3c398fb3c146 is healthy: got healthy result from http://127.0.0.1:22379
 member fd422379fda50e48 is healthy: got healthy result from http://127.0.0.1:32379

Documentation/v2/api.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # etcd API
 
 ## Running a Single Machine Cluster
@@ -318,7 +323,7 @@ The first terminal should get the notification and return with the same response
 
 However, the watch command can do more than this.
 Using the index, we can watch for commands that have happened in the past.
-This is useful for ensuring you don't miss events between watch commands. 
+This is useful for ensuring you don't miss events between watch commands.
 Typically, we watch again from the `modifiedIndex` + 1 of the node we got.
 
 Let's try to watch for the set command of index 7 again:
@@ -338,13 +343,13 @@ curl 'http://127.0.0.1:2379/v2/keys/foo?wait=true&waitIndex=8'
 Then even if etcd is on index 9 or 800, the first event to occur to the `/foo`
 key between 8 and the current index will be returned.
 
-**Note**: etcd only keeps the responses of the most recent 1000 events across all etcd keys. 
+**Note**: etcd only keeps the responses of the most recent 1000 events across all etcd keys.
 It is recommended to send the response to another thread to process immediately
-instead of blocking the watch while processing the result. 
+instead of blocking the watch while processing the result.
 
 #### Watch from cleared event index
 
-If we miss all the 1000 events, we need to recover the current state of the 
+If we miss all the 1000 events, we need to recover the current state of the
 watching key space through a get and then start to watch from the
 `X-Etcd-Index` + 1.
 
@@ -366,7 +371,7 @@ To start watch, first we need to fetch the current state of key `/foo`:
 curl 'http://127.0.0.1:2379/v2/keys/foo' -vv
 ```
 
-``` 
+```
 < HTTP/1.1 200 OK
 < Content-Type: application/json
 < X-Etcd-Cluster-Id: 7e27652122e8b2ae
@@ -375,7 +380,7 @@ curl 'http://127.0.0.1:2379/v2/keys/foo' -vv
 < X-Raft-Term: 2
 < Date: Mon, 05 Jan 2015 18:54:43 GMT
 < Transfer-Encoding: chunked
-< 
+<
 {"action":"get","node":{"key":"/foo","value":"bar","modifiedIndex":7,"createdIndex":7}}
 ```
 

Documentation/v2/api_v3.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # etcd3 API
 
 TODO: API doc

Documentation/v2/auth_api.md

@@ -1,13 +1,18 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # v2 Auth and Security
 
-## etcd Resources 
+## etcd Resources
 There are three types of resources in etcd
 
 1. permission resources: users and roles in the user store
 2. key-value resources: key-value pairs in the key-value store
 3. settings resources: security settings, auth settings, and dynamic etcd cluster settings (election/heartbeat)
 
-### Permission Resources 
+### Permission Resources
 
 #### Users
 A user is an identity to be authenticated. Each user can have multiple roles. The user has a capability (such as reading or writing) on the resource if one of the roles has that capability.
@@ -15,7 +20,7 @@ A user is an identity to be authenticated. Each user can have multiple roles. Th
 A user named `root` is required before authentication can be enabled, and it always has the ROOT role. The ROOT role can be granted to multiple users, but `root` is required for recovery purposes.
 
 #### Roles
-Each role has exact one associated Permission List. An permission list exists for each permission on key-value resources. 
+Each role has exact one associated Permission List. An permission list exists for each permission on key-value resources.
 
 The special static ROOT (named `root`) role has a full permissions on all key-value resources, the permission to manage user resources and settings resources. Only the ROOT role has the permission to manage user resources and modify settings resources. The ROOT role is built-in and does not need to be created.
 
@@ -30,8 +35,8 @@ A Permission List is a list of allowed patterns for that particular permission (
 ### Key-Value Resources
 A key-value resource is a key-value pairs in the store. Given a list of matching patterns, permission for any given key in a request is granted if any of the patterns in the list match.
 
-Only prefixes or exact keys are supported. A prefix permission string ends in `*`. 
-A permission on `/foo` is for that exact key or directory, not its children or recursively. `/foo*` is a prefix that matches `/foo` recursively, and all keys thereunder, and keys with that prefix (eg. `/foobar`. Contrast to the prefix `/foo/*`). `*` alone is permission on the full keyspace. 
+Only prefixes or exact keys are supported. A prefix permission string ends in `*`.
+A permission on `/foo` is for that exact key or directory, not its children or recursively. `/foo*` is a prefix that matches `/foo` recursively, and all keys thereunder, and keys with that prefix (eg. `/foobar`. Contrast to the prefix `/foo/*`). `*` alone is permission on the full keyspace.
 
 ### Settings Resources
 
@@ -66,7 +71,7 @@ An Error JSON corresponds to:
 }
 
 #### Enable and Disable Authentication
-        
+
 **Get auth status**
 
 GET  /v2/auth/enable
@@ -215,8 +220,8 @@ PUT  /v2/auth/users/charlie
     Sent Headers:
         Authorization: Basic <BasicAuthString>
     Put Body:
-        JSON struct, above, matching the appropriate name 
-          * Starting password and roles when creating. 
+        JSON struct, above, matching the appropriate name
+          * Starting password and roles when creating.
           * Grant/Revoke/Password filled in when updating (to grant roles, revoke roles, or change the password).
     Possible Status Codes:
         200 OK
@@ -345,7 +350,7 @@ PUT  /v2/auth/roles/rkt
         401 Unauthorized
         404 Not Found (update non-existent roles)
         409 Conflict (when granting duplicated permission or revoking non-existent permission)
-    200 Body: 
+    200 Body:
         JSON state of the role
 
 **Remove A Role**

Documentation/v2/authentication.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Authentication Guide
 
 ## Overview
@@ -14,7 +19,7 @@ There is one special user, `root`, and there are two special roles, `root` and `
 
 ### User `root`
 
-User `root` must be created before security can be activated. It has the `root` role and allows for the changing of anything inside etcd. The idea behind the `root` user is for recovery purposes -- a password is generated and stored somewhere -- and the root role is granted to the administrator accounts on the system. In the future, for troubleshooting and recovery, we will need to assume some access to the system, and future documentation will assume this root user (though anyone with the role will suffice). 
+User `root` must be created before security can be activated. It has the `root` role and allows for the changing of anything inside etcd. The idea behind the `root` user is for recovery purposes -- a password is generated and stored somewhere -- and the root role is granted to the administrator accounts on the system. In the future, for troubleshooting and recovery, we will need to assume some access to the system, and future documentation will assume this root user (though anyone with the role will suffice).
 
 ### Role `root`
 
@@ -104,7 +109,7 @@ $ etcdctl role grant myrolename -path '/foo/bar' -write
 $ etcdctl role grant myrolename -path '/pub/*' -readwrite
 ```
 
-Beware that 
+Beware that
 
 ```
 # Give full access to keys under /pub??
@@ -133,12 +138,12 @@ $ etcdctl role remove myrolename
 
 ## Enabling authentication
 
-The minimal steps to enabling auth are as follows. The administrator can set up users and roles before or after enabling authentication, as a matter of preference. 
+The minimal steps to enabling auth are as follows. The administrator can set up users and roles before or after enabling authentication, as a matter of preference.
 
 Make sure the root user is created:
 
 ```
-$ etcdctl user add root 
+$ etcdctl user add root
 New password:
 ```
 

Documentation/v2/backward_compatibility.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Backward Compatibility
 
 The main goal of etcd 2.0 release is to improve cluster safety around bootstrapping and dynamic reconfiguration. To do this, we deprecated the old error-prone APIs and provide a new set of APIs.

Documentation/v2/benchmarks/README.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../../docs.md#documentation
+
+
 # Benchmarks
 
 etcd benchmarks will be published regularly and tracked for each release below:

Documentation/v2/benchmarks/etcd-2-1-0-alpha-benchmarks.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../../docs.md#documentation
+
+
 ## Physical machines
 
 GCE n1-highcpu-2 machine type

Documentation/v2/benchmarks/etcd-2-2-0-benchmarks.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../../docs.md#documentation
+
+
 # Benchmarking etcd v2.2.0
 
 ## Physical Machines

Documentation/v2/benchmarks/etcd-2-2-0-rc-benchmarks.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../../docs.md#documentation
+
+
 ## Physical machines
 
 GCE n1-highcpu-2 machine type

Documentation/v2/benchmarks/etcd-2-2-0-rc-memory-benchmarks.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../../docs.md#documentation
+
+
 ## Physical machine
 
 GCE n1-standard-2 machine type

Documentation/v2/benchmarks/etcd-3-demo-benchmarks.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../../docs.md#documentation
+
+
 ## Physical machines
 
 GCE n1-highcpu-2 machine type

Documentation/v2/benchmarks/etcd-3-watch-memory-benchmark.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../../docs.md#documentation
+
+
 # Watch Memory Usage Benchmark
 
 *NOTE*: The watch features are under active development, and their memory usage may change as that development progresses. We do not expect it to significantly increase beyond the figures stated below.
@@ -5,10 +10,10 @@
 A primary goal of etcd is supporting a very large number of watchers doing a massively large amount of watching. etcd aims to support O(10k) clients, O(100K) watch streams (O(10) streams per client) and O(10M) total watchings (O(100) watching per stream). The memory consumed by each individual watching accounts for the largest portion of etcd's overall usage, and is therefore the focus of current and future optimizations.
 
 
-Three related components of etcd watch consume physical memory: each `grpc.Conn`, each watch stream, and each instance of the watching activity. `grpc.Conn` maintains the actual TCP connection and other gRPC connection state. Each `grpc.Conn` consumes O(10kb) of memory, and might have multiple watch streams attached. 
+Three related components of etcd watch consume physical memory: each `grpc.Conn`, each watch stream, and each instance of the watching activity. `grpc.Conn` maintains the actual TCP connection and other gRPC connection state. Each `grpc.Conn` consumes O(10kb) of memory, and might have multiple watch streams attached.
 
-Each watch stream is an independent HTTP2 connection which consumes another O(10kb) of memory. 
-Multiple watchings might share one watch stream. 
+Each watch stream is an independent HTTP2 connection which consumes another O(10kb) of memory.
+Multiple watchings might share one watch stream.
 
 Watching is the actual struct that tracks the changes on the key-value store. Each watching should only consume < O(1kb).
 

Documentation/v2/benchmarks/etcd-storage-memory-benchmark.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../../docs.md#documentation
+
+
 # Storage Memory Usage Benchmark
 
 <!---todo: link storage to storage design doc-->
@@ -60,7 +65,7 @@ GCE n1-standard-2 machine type
 
 In this test, we only benchmark the memory usage of the in-memory index. The goal is to find `c1` and `c2` mentioned above and to understand the hard limit of memory consumption of the storage.
 
-We calculate the memory usage consumption via the Go runtime.ReadMemStats. We calculate the total allocated bytes difference before creating the index and after creating the index. It cannot perfectly reflect the memory usage of the in-memory index itself but can show the rough consumption pattern. 
+We calculate the memory usage consumption via the Go runtime.ReadMemStats. We calculate the total allocated bytes difference before creating the index and after creating the index. It cannot perfectly reflect the memory usage of the in-memory index itself but can show the rough consumption pattern.
 
 | N    | versions | key size | memory usage |
 |------|----------|----------|--------------|

Documentation/v2/branch_management.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Branch Management
 
 ## Guide

Documentation/v2/clustering.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Clustering Guide
 
 ## Overview

Documentation/v2/configuration.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Configuration Flags
 
 etcd is configurable through command-line flags and environment variables. Options set on the command line take precedence over those from the environment.

Documentation/v2/dev/release.md

@@ -1,8 +1,13 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../../docs.md#documentation
+
+
 # etcd release guide
 
 The guide talks about how to release a new version of etcd.
 
-The procedure includes some manual steps for sanity checking but it can probably be further scripted. Please keep this document up-to-date if you want to make changes to the release process. 
+The procedure includes some manual steps for sanity checking but it can probably be further scripted. Please keep this document up-to-date if you want to make changes to the release process.
 
 ## Prepare Release
 

Documentation/v2/discovery_protocol.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Discovery Service Protocol
 
 Discovery service protocol helps new etcd member to discover all other members in cluster bootstrap phase using a shared discovery URL.

Documentation/v2/docker_guide.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Running etcd under Docker
 
 The following guide will show you how to run etcd under Docker using the [static bootstrap process](clustering.md#static).

Documentation/v2/errorcode.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Error Code
 ======
 

Documentation/v2/faq.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # FAQ
 
 ## 1) Why can an etcd client read an old version of data when a majority of the etcd cluster members are down?

Documentation/v2/glossary.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Glossary
 
 This document defines the various terms used in etcd documentation, command line and source code.

Documentation/v2/implementation-faq.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # FAQ
 
 ## Initial Bootstrapping UX

Documentation/v2/internal-protocol-versioning.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Versioning
 
 Goal: We want to be able to upgrade an individual peer in an etcd cluster to a newer version of etcd.

Documentation/v2/libraries-and-tools.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Libraries and Tools
 
 **Tools**

Documentation/v2/members_api.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Members API
 
 * [List members](#list-members)

Documentation/v2/metrics.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Metrics
 
 etcd uses [Prometheus][prometheus] for metrics reporting. The metrics can be used for real-time monitoring and debugging. etcd does not persist its metrics; if a member restarts, the metrics will be reset.
@@ -14,9 +19,9 @@ The metrics under the `etcd` prefix are for monitoring and alerting. They are st
 
 ### http requests
 
-These metrics describe the serving of requests (non-watch events) served by etcd members in non-proxy mode: total 
+These metrics describe the serving of requests (non-watch events) served by etcd members in non-proxy mode: total
 incoming requests, request failures and processing latency (inc. raft rounds for storage). They are useful for tracking
- user-generated traffic hitting the etcd cluster . 
+ user-generated traffic hitting the etcd cluster .
 
 All these metrics are prefixed with `etcd_http_`
 
@@ -28,20 +33,20 @@ All these metrics are prefixed with `etcd_http_`
 
 
 Example Prometheus queries that may be useful from these metrics (across all etcd members):
- 
- * `sum(rate(etcd_http_failed_total{job="etcd"}[1m]) by (method) / sum(rate(etcd_http_events_received_total{job="etcd"})[1m]) by (method)` 
-    
+
+ * `sum(rate(etcd_http_failed_total{job="etcd"}[1m]) by (method) / sum(rate(etcd_http_events_received_total{job="etcd"})[1m]) by (method)`
+
     Shows the fraction of events that failed by HTTP method across all members, across a time window of `1m`.
- 
+
  * `sum(rate(etcd_http_received_total{job="etcd",method="GET})[1m]) by (method)`
    `sum(rate(etcd_http_received_total{job="etcd",method~="GET})[1m]) by (method)`
-    
+
     Shows the rate of successful readonly/write queries across all servers, across a time window of `1m`.
-    
+
  * `histogram_quantile(0.9, sum(rate(etcd_http_successful_duration_seconds{job="etcd",method="GET"}[5m]) ) by (le))`
    `histogram_quantile(0.9, sum(rate(etcd_http_successful_duration_seconds{job="etcd",method!="GET"}[5m]) ) by (le))`
-    
-    Show the 0.90-tile latency (in seconds) of read/write (respectively) event handling across all members, with a window of `5m`.      
+
+    Show the 0.90-tile latency (in seconds) of read/write (respectively) event handling across all members, with a window of `5m`.
 
 ### proxy
 
@@ -56,21 +61,21 @@ All these metrics are prefixed with `etcd_proxy_`
 | requests_total            | Total number of requests by this proxy instance.                                | Counter(method)        |
 | handled_total             | Total number of fully handled requests, with responses from etcd members.           | Counter(method)        |
 | dropped_total             | Total number of dropped requests due to forwarding errors to etcd members.          | Counter(method,error)  |
-| handling_duration_seconds | Bucketed handling times by HTTP method, including round trip to member instances.   | Histogram(method)      |  
+| handling_duration_seconds | Bucketed handling times by HTTP method, including round trip to member instances.   | Histogram(method)      |
 
 Example Prometheus queries that may be useful from these metrics (across all etcd servers):
 
  *  `sum(rate(etcd_proxy_handled_total{job="etcd"}[1m])) by (method)`
-    
-    Rate of requests (by HTTP method) handled by all proxies, across a window of `1m`. 
+
+    Rate of requests (by HTTP method) handled by all proxies, across a window of `1m`.
 
  * `histogram_quantile(0.9, sum(rate(handling_duration_seconds{job="etcd",method="GET"}[5m])) by (le))`
    `histogram_quantile(0.9, sum(rate(handling_duration_seconds{job="etcd",method!="GET"}[5m])) by (le))`
-    
-    Show the 0.90-tile latency (in seconds) of handling of user requests across all proxy machines, with a window of `5m`.  
-    
+
+    Show the 0.90-tile latency (in seconds) of handling of user requests across all proxy machines, with a window of `5m`.
+
  * `sum(rate(etcd_proxy_dropped_total{job="etcd"}[1m])) by (proxying_error)`
-    
+
     Number of failed request on the proxy. This should be 0, spikes here indicate connectivity issues to the etcd cluster.
 
 ## etcd_debugging namespace metrics

Documentation/v2/other_apis.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Miscellaneous APIs
 
 * [Getting the etcd version](#getting-the-etcd-version)

Documentation/v2/platforms/freebsd.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../../docs.md#documentation
+
+
 # FreeBSD
 
 Starting with version 0.1.2 both etcd and etcdctl have been ported to FreeBSD and can

Documentation/v2/production-users.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Production Users
 
 This document tracks people and use cases for etcd in production. By creating a list of production use cases we hope to build a community of advisors that we can reach out to with experience using various etcd applications, operation environments, and cluster sizes. The etcd development team may reach out periodically to check-in on your experience and update this list.

Documentation/v2/proxy.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Proxy
 
 etcd can run as a transparent proxy. Doing so allows for easy discovery of etcd within your infrastructure, since it can run on each machine as a local service. In this mode, etcd acts as a reverse proxy and forwards client requests to an active etcd cluster. The etcd proxy does not participate in the consensus replication of the etcd cluster, thus it neither increases the resilience nor decreases the write performance of the etcd cluster.

Documentation/v2/reporting_bugs.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Reporting Bugs
 
 If you find bugs or documentation mistakes in the etcd project, please let us know by [opening an issue][etcd-issue]. We treat bugs and mistakes very seriously and believe no issue is too small. Before creating a bug report, please check that an issue reporting the same problem does not already exist.

Documentation/v2/rfc/v3api.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../../docs.md#documentation
+
+
 # Overview
 
 The etcd v3 API is designed to give users a more efficient and cleaner abstraction compared to etcd v2. There are a number of semantic and protocol changes in this new API. For an overview [see Xiang Li's video](https://youtu.be/J5AioGtEPeQ?t=211).

Documentation/v2/runtime-configuration.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Runtime Reconfiguration
 
 etcd comes with support for incremental runtime reconfiguration, which allows users to update the membership of the cluster at run time.
@@ -61,9 +66,9 @@ A wrongly updated client URL will not affect the health of the etcd cluster.
 
 #### Update advertise peer URLs
 
-If you would like to update the advertise peer URLs of a member, you have to first update 
+If you would like to update the advertise peer URLs of a member, you have to first update
 it explicitly via member command and then restart the member. The additional action is required
-since updating peer URLs changes the cluster wide configuration and can affect the health of the etcd cluster. 
+since updating peer URLs changes the cluster wide configuration and can affect the health of the etcd cluster.
 
 To update the peer URLs, first, we need to find the target member's ID. You can list all members with `etcdctl`:
 

Documentation/v2/runtime-reconf-design.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Design of Runtime Reconfiguration
 
 Runtime reconfiguration is one of the hardest and most error prone features in a distributed system, especially in a consensus based system like etcd.
@@ -26,21 +31,21 @@ We think runtime reconfiguration should be a low frequent operation. We made the
 
 If a cluster permanently loses a majority of its members, a new cluster will need to be started from an old data directory to recover the previous state.
 
-It is entirely possible to force removing the failed members from the existing cluster to recover. However, we decided not to support this method since it bypasses the normal consensus committing phase, which is unsafe. If the member to remove is not actually dead or you force to remove different members through different members in the same cluster, you will end up with diverged cluster with same clusterID. This is very dangerous and hard to debug/fix afterwards. 
+It is entirely possible to force removing the failed members from the existing cluster to recover. However, we decided not to support this method since it bypasses the normal consensus committing phase, which is unsafe. If the member to remove is not actually dead or you force to remove different members through different members in the same cluster, you will end up with diverged cluster with same clusterID. This is very dangerous and hard to debug/fix afterwards.
 
 If you have a correct deployment, the possibility of permanent majority lose is very low. But it is a severe enough problem that worth special care. We strongly suggest you to read the [disaster recovery documentation][disaster-recovery] and prepare for permanent majority lose before you put etcd into production.
 
 ## Do Not Use Public Discovery Service For Runtime Reconfiguration
 
-The public discovery service should only be used for bootstrapping a cluster. To join member into an existing cluster, you should use runtime reconfiguration API. 
+The public discovery service should only be used for bootstrapping a cluster. To join member into an existing cluster, you should use runtime reconfiguration API.
 
 Discovery service is designed for bootstrapping an etcd cluster in the cloud environment, when you do not know the IP addresses of all the members beforehand. After you successfully bootstrap a cluster, the IP addresses of all the members are known. Technically, you should not need the discovery service any more.
 
-It seems that using public discovery service is a convenient way to do runtime reconfiguration, after all discovery service already has all the cluster configuration information. However relying on public discovery service brings troubles: 
+It seems that using public discovery service is a convenient way to do runtime reconfiguration, after all discovery service already has all the cluster configuration information. However relying on public discovery service brings troubles:
 
 1. it introduces external dependencies for the entire life-cycle of your cluster, not just bootstrap time. If there is a network issue between your cluster and public discovery service, your cluster will suffer from it.
- 
-2. public discovery service must reflect correct runtime configuration of your cluster during it life-cycle. It has to provide security mechanism to avoid bad actions, and it is hard. 
+
+2. public discovery service must reflect correct runtime configuration of your cluster during it life-cycle. It has to provide security mechanism to avoid bad actions, and it is hard.
 
 3. public discovery service has to keep tens of thousands of cluster configurations. Our public discovery service backend is not ready for that workload.
 

Documentation/v2/security.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Security Model
 
 etcd supports SSL/TLS as well as authentication through client certificates, both for clients to server as well as peer (server to server / cluster) communication.

Documentation/v2/tuning.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Tuning
 
 The default settings in etcd should work well for installations on a local network where the average network latency is low.

Documentation/v2/upgrade_2_1.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Upgrade etcd to 2.1
 
 In the general case, upgrading from etcd 2.0 to 2.1 can be a zero-downtime, rolling upgrade:
@@ -12,11 +17,11 @@ Before [starting an upgrade](#upgrade-procedure), read through the rest of this
 
 To upgrade an existing etcd deployment to 2.1, you must be running 2.0. If you’re running a version of etcd before 2.0, you must upgrade to [2.0][v2.0] before upgrading to 2.1.
 
-Also, to ensure a smooth rolling upgrade, your running cluster must be healthy. You can check the health of the cluster by using `etcdctl cluster-health` command. 
+Also, to ensure a smooth rolling upgrade, your running cluster must be healthy. You can check the health of the cluster by using `etcdctl cluster-health` command.
 
-### Preparedness 
+### Preparedness
 
-Before upgrading etcd, always test the services relying on etcd in a staging environment before deploying the upgrade to the production environment. 
+Before upgrading etcd, always test the services relying on etcd in a staging environment before deploying the upgrade to the production environment.
 
 You might also want to [backup your data directory][backup-datastore] for a potential [downgrade](#downgrade).
 
@@ -38,7 +43,7 @@ If you have even more data, this might take more time. If you have a data size l
 
 ### Downgrade
 
-If all members have been upgraded to v2.1, the cluster will be upgraded to v2.1, and downgrade is **not possible**. If any member is still v2.0, the cluster will remain in v2.0, and you can go back to use v2.0 binary. 
+If all members have been upgraded to v2.1, the cluster will be upgraded to v2.1, and downgrade is **not possible**. If any member is still v2.0, the cluster will remain in v2.0, and you can go back to use v2.0 binary.
 
 Please [backup your data directory][backup-datastore] of all etcd members if you want to downgrade the cluster, even if it is upgraded.
 
@@ -96,7 +101,7 @@ member 924e2e83e93f2560 is healthy
 member a8266ecf031671f3 is healthy
 ```
 
-#### 4. Repeat step 2 to step 3 for all other members 
+#### 4. Repeat step 2 to step 3 for all other members
 
 #### 5. Finish
 

Documentation/v2/upgrade_2_2.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Upgrade etcd from 2.1 to 2.2
 
 In the general case, upgrading from etcd 2.1 to 2.2 can be a zero-downtime, rolling upgrade:
@@ -13,11 +18,11 @@ Before [starting an upgrade](#upgrade-procedure), read through the rest of this
 
 To upgrade an existing etcd deployment to 2.2, you must be running 2.1. If you’re running a version of etcd before 2.1, you must upgrade to [2.1][v2.1] before upgrading to 2.2.
 
-Also, to ensure a smooth rolling upgrade, your running cluster must be healthy. You can check the health of the cluster by using `etcdctl cluster-health` command. 
+Also, to ensure a smooth rolling upgrade, your running cluster must be healthy. You can check the health of the cluster by using `etcdctl cluster-health` command.
 
-### Preparedness 
+### Preparedness
 
-Before upgrading etcd, always test the services relying on etcd in a staging environment before deploying the upgrade to the production environment. 
+Before upgrading etcd, always test the services relying on etcd in a staging environment before deploying the upgrade to the production environment.
 
 You might also want to [backup the data directory][backup-datastore] for a potential [downgrade].
 
@@ -31,11 +36,11 @@ Internally, etcd members negotiate with each other to determine the overall etcd
 
 If you have a data size larger than 100MB you should contact us before upgrading, so we can make sure the upgrades work smoothly.
 
-Every etcd 2.2 member will do health checking across the cluster periodically. etcd 2.1 member does not support health checking. During the upgrade, etcd 2.2 member will log warning about the unhealthy state of etcd 2.1 member. You can ignore the warning. 
+Every etcd 2.2 member will do health checking across the cluster periodically. etcd 2.1 member does not support health checking. During the upgrade, etcd 2.2 member will log warning about the unhealthy state of etcd 2.1 member. You can ignore the warning.
 
 ### Downgrade
 
-If all members have been upgraded to v2.2, the cluster will be upgraded to v2.2, and downgrade is **not possible**. If any member is still v2.1, the cluster will remain in v2.1, and you can go back to use v2.1 binary. 
+If all members have been upgraded to v2.2, the cluster will be upgraded to v2.2, and downgrade is **not possible**. If any member is still v2.1, the cluster will remain in v2.1, and you can go back to use v2.1 binary.
 
 Please [backup the data directory][backup-datastore] of all etcd members if you want to downgrade the cluster, even if it is upgraded.
 
@@ -112,7 +117,7 @@ member a8266ecf031671f3 is healthy: got healthy result from http://localhost:123
 cluster is healthy
 ```
 
-#### 4. Repeat step 2 to step 3 for all other members 
+#### 4. Repeat step 2 to step 3 for all other members
 
 #### 5. Finish
 

Documentation/v2/upgrade_2_3.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 ## Upgrade etcd from 2.2 to 2.3
 
 In the general case, upgrading from etcd 2.2 to 2.3 can be a zero-downtime, rolling upgrade:

embed/etcd.go

@@ -186,11 +186,29 @@ func (e *Etcd) Config() Config {
 func (e *Etcd) Close() {
 	e.closeOnce.Do(func() { close(e.stopc) })
 
-	// (gRPC server) stops accepting new connections,
-	// RPCs, and blocks until all pending RPCs are finished
+	timeout := 2 * time.Second
+	if e.Server != nil {
+		timeout = e.Server.Cfg.ReqTimeout()
+	}
 	for _, sctx := range e.sctxs {
 		for gs := range sctx.grpcServerC {
-			gs.GracefulStop()
+			ch := make(chan struct{})
+			go func() {
+				defer close(ch)
+				// close listeners to stop accepting new connections,
+				// will block on any existing transports
+				gs.GracefulStop()
+			}()
+			// wait until all pending RPCs are finished
+			select {
+			case <-ch:
+			case <-time.After(timeout):
+				// took too long, manually close open transports
+				// e.g. watch streams
+				gs.Stop()
+				// concurrent GracefulStop should be interrupted
+				<-ch
+			}
 		}
 	}
 
@@ -326,6 +344,9 @@ func startClientListeners(cfg *Config) (sctxs map[string]*serveCtx, err error) {
 		if sctx.l, err = net.Listen(proto, addr); err != nil {
 			return nil, err
 		}
+		// net.Listener will rewrite ipv4 0.0.0.0 to ipv6 [::], breaking
+		// hosts that disable ipv6. So, use the address given by the user.
+		sctx.addr = addr
 
 		if fdLimit, fderr := runtimeutil.FDLimit(); fderr == nil {
 			if fdLimit <= reservedInternalFDNum {

embed/serve.go

@@ -44,6 +44,7 @@ import (
 
 type serveCtx struct {
 	l        net.Listener
+	addr     string
 	secure   bool
 	insecure bool
 
@@ -160,28 +161,38 @@ func grpcHandlerFunc(grpcServer *grpc.Server, otherHandler http.Handler) http.Ha
 	})
 }
 
-type registerHandlerFunc func(context.Context, *gw.ServeMux, string, []grpc.DialOption) error
+type registerHandlerFunc func(context.Context, *gw.ServeMux, *grpc.ClientConn) error
 
 func (sctx *serveCtx) registerGateway(opts []grpc.DialOption) (*gw.ServeMux, error) {
 	ctx := sctx.ctx
-	addr := sctx.l.Addr().String()
+	conn, err := grpc.DialContext(ctx, sctx.addr, opts...)
+	if err != nil {
+		return nil, err
+	}
 	gwmux := gw.NewServeMux()
 
 	handlers := []registerHandlerFunc{
-		etcdservergw.RegisterKVHandlerFromEndpoint,
-		etcdservergw.RegisterWatchHandlerFromEndpoint,
-		etcdservergw.RegisterLeaseHandlerFromEndpoint,
-		etcdservergw.RegisterClusterHandlerFromEndpoint,
-		etcdservergw.RegisterMaintenanceHandlerFromEndpoint,
-		etcdservergw.RegisterAuthHandlerFromEndpoint,
-		v3lockgw.RegisterLockHandlerFromEndpoint,
-		v3electiongw.RegisterElectionHandlerFromEndpoint,
+		etcdservergw.RegisterKVHandler,
+		etcdservergw.RegisterWatchHandler,
+		etcdservergw.RegisterLeaseHandler,
+		etcdservergw.RegisterClusterHandler,
+		etcdservergw.RegisterMaintenanceHandler,
+		etcdservergw.RegisterAuthHandler,
+		v3lockgw.RegisterLockHandler,
+		v3electiongw.RegisterElectionHandler,
 	}
 	for _, h := range handlers {
-		if err := h(ctx, gwmux, addr, opts); err != nil {
+		if err := h(ctx, gwmux, conn); err != nil {
 			return nil, err
 		}
 	}
+	go func() {
+		<-ctx.Done()
+		if cerr := conn.Close(); cerr != nil {
+			plog.Warningf("failed to close conn to %s: %v", sctx.l.Addr().String(), cerr)
+		}
+	}()
+
 	return gwmux, nil
 }
 

etcdserver/api/v3rpc/grpc.go

@@ -16,6 +16,7 @@ package v3rpc
 
 import (
 	"crypto/tls"
+	"math"
 
 	"github.com/coreos/etcd/etcdserver"
 	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
@@ -24,6 +25,8 @@ import (
 	"google.golang.org/grpc/grpclog"
 )
 
+const maxStreams = math.MaxUint32
+
 func init() {
 	grpclog.SetLogger(plog)
 }
@@ -36,8 +39,9 @@ func Server(s *etcdserver.EtcdServer, tls *tls.Config) *grpc.Server {
 	}
 	opts = append(opts, grpc.UnaryInterceptor(newUnaryInterceptor(s)))
 	opts = append(opts, grpc.StreamInterceptor(newStreamInterceptor(s)))
-
+	opts = append(opts, grpc.MaxConcurrentStreams(maxStreams))
 	grpcServer := grpc.NewServer(opts...)
+
 	pb.RegisterKVServer(grpcServer, NewQuotaKVServer(s))
 	pb.RegisterWatchServer(grpcServer, NewWatchServer(s))
 	pb.RegisterLeaseServer(grpcServer, NewQuotaLeaseServer(s))

integration/embed_test.go

@@ -15,13 +15,16 @@
 package integration
 
 import (
+	"context"
 	"fmt"
 	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
+	"github.com/coreos/etcd/clientv3"
 	"github.com/coreos/etcd/embed"
 )
 
@@ -102,6 +105,47 @@ func TestEmbedEtcd(t *testing.T) {
 	}
 }
 
+// TestEmbedEtcdGracefulStop ensures embedded server stops
+// cutting existing transports.
+func TestEmbedEtcdGracefulStop(t *testing.T) {
+	cfg := embed.NewConfig()
+
+	urls := newEmbedURLs(2)
+	setupEmbedCfg(cfg, []url.URL{urls[0]}, []url.URL{urls[1]})
+
+	cfg.Dir = filepath.Join(os.TempDir(), fmt.Sprintf("embed-etcd"))
+	os.RemoveAll(cfg.Dir)
+	defer os.RemoveAll(cfg.Dir)
+
+	e, err := embed.StartEtcd(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	<-e.Server.ReadyNotify() // wait for e.Server to join the cluster
+
+	cli, err := clientv3.New(clientv3.Config{Endpoints: []string{urls[0].String()}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cli.Close()
+
+	// open watch connection
+	cli.Watch(context.Background(), "foo")
+
+	donec := make(chan struct{})
+	go func() {
+		e.Close()
+		close(donec)
+	}()
+	select {
+	case err := <-e.Err():
+		t.Fatal(err)
+	case <-donec:
+	case <-time.After(2*time.Second + e.Server.Cfg.ReqTimeout()):
+		t.Fatalf("took too long to close server")
+	}
+}
+
 func newEmbedURLs(n int) (urls []url.URL) {
 	for i := 0; i < n; i++ {
 		u, _ := url.Parse(fmt.Sprintf("unix://localhost:%d%06d", os.Getpid(), i))

integration/fixtures/ca-csr.json

@@ -0,0 +1,19 @@
+{
+  "key": {
+    "algo": "rsa",
+    "size": 4096
+  },
+  "names": [
+    {
+      "O": "etcd",
+      "OU": "etcd Security",
+      "L": "San Francisco",
+      "ST": "California",
+      "C": "USA"
+    }
+  ],
+  "CN": "ca",
+  "ca": {
+    "expiry": "87600h"
+  }
+}

integration/fixtures/ca.crt

@@ -1,23 +1,33 @@
 -----BEGIN CERTIFICATE-----
-MIID2zCCAsOgAwIBAgIUZXdXtcOe421Geq9VjM35+SRJUS8wDQYJKoZIhvcNAQEL
-BQAwdTEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
-Ew1TYW4gRnJhbmNpc2NvMRAwDgYDVQQKEwdldGNkLWNhMQswCQYDVQQLEwJDQTEZ
-MBcGA1UEAxMQQXV0b2dlbmVyYXRlZCBDQTAeFw0xNjA3MDUxOTQ1MDBaFw0yMTA3
-MDQxOTQ1MDBaMHUxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEW
-MBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEQMA4GA1UEChMHZXRjZC1jYTELMAkGA1UE
-CxMCQ0ExGTAXBgNVBAMTEEF1dG9nZW5lcmF0ZWQgQ0EwggEiMA0GCSqGSIb3DQEB
-AQUAA4IBDwAwggEKAoIBAQDBMoRjH0ULs+0cRZWZ8BGJ7Fmf152J9uUE3/NgYV3M
-4Ntu6l3IYALXT5QSHQZIz5425HP6827mwAOZ/bk6E3yzq6XR/vHzxPFLzBMzFuq/
-elQA4nb7eYHICriEFUdJo2EUg3lSD3m6Deof/NjPMgUHtuvhn1OJMezaALZiMZ0K
-9B9/1ktW4Roi6FMVFfJM5rKr9EIz6P2mFUpVHI7KSGbeuHiTPq0FLVv7wFPxRFX5
-Ygd/nF6bbSsE2LAx/JdY1j0LQi0WUcA/HaWYVOpFSKohO6FmshP5bX0o//wWSkg2
-8CSbtqvSxRF/Ril7raZlX713AAZVn8+B83tpjFqOLH+7AgMBAAGjYzBhMA4GA1Ud
-DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSlyMYprKNDkzyP
-gGA5cYnEEe9Y8DAfBgNVHSMEGDAWgBSlyMYprKNDkzyPgGA5cYnEEe9Y8DANBgkq
-hkiG9w0BAQsFAAOCAQEAjjZkuoBl6meveg1frQuUhWtgtN/g9JqIjhEQ7tr4H46/
-cHz3ngCuJh/GKSt7MTqafP99kqtm1GBs7BcoFKwsNFxNOo/a2MV2oYe2T5ol5U6/
-RnmPv7yXzV1WlSC2IxFdtKEIfM859TFrWFN+NyH7yyYzjx+CzFdu6SHMwrQkETKr
-R/PJrb0pV+gbeFpe/VfVyT7tFSxRTkSqwvMFNjQmbSLSiIFDNdZmPBmnWk418zoP
-lkUESi3OQc4Eh/yQuldDXKl7L8+Ar8DddAu4nsni9EAJWi1u5wPPaLd+3s5USr1f
-zFC3tb8o+WfNf+VSxWWPWyZXlcnB2glT+TWW40Ng1w==
+MIIFrjCCA5agAwIBAgIUCwleGnPMSwoODcFBty/IC/L6CUIwDQYJKoZIhvcNAQEN
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xNzA2MTYyMDMzMDBaFw0yNzA2MTQyMDMz
+MDBaMG8xDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTELMAkGA1UEAxMCY2EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK
+AoICAQDhp9t3WUGpaRtbM52hudffXT0V9dbl1ac4DD37MdIit2yDFsut1IxSgZ40
+9FliVStAWzDhZL6nX4rpInXOEI1WV1xKXu+T8i2LcxnW4QjvKTLMpBdF6q0KzsiZ
+CV5uNTQvIuR/hQN4ij03j75nnj/ds5TUCQfz/Mh6T/xwbHp1XUimcVnh38+q+ZE2
+eCmEvcdAEQ9DXj7WTDD4dN0xaJz8rvZSVWVBwuP7dtN54FJmJyRXcCuus5pUd/Lm
+n4mEEZ3DLceUM13AK/gwAS3SNHOwuH4pl6IKJ10qSUdzrB+Lt0rx2iqyodN/EMnh
+kYJRWG8mv5spN/s695A3MLKk0hZ/bkys91n0hycaPFg8TwxmdXP8P/AOFQXyK4x9
+YhvtF6mGhD/RHqdaujF/tCH34DpMVY9ObTu59R/6qG4Zr3KfqpDp5iM1LjggT4QU
+2JBn9zc5rAd/j3clcgfJfW5CZ8ek31HLIKPm5pa8q5l4qL7qWu0FjZTpSgUps29O
+ekRhtSCFI3R8TZkWOAV5DM+FkXJACsOJT/Ds4/BFgia05dglNEkFTuSDAT6BfQjy
+bghuxYkFP3bPj8rflM9AhXsfHM5qEcSkZcSdjHqn4u2uvRnpc1/T8MVADqkpMukf
+IUabqJ0Iy5SHXmqouO2ZkPG8C4ytkUuQW3WKrLNBSXRJVQ3pAQIDAQABo0IwQDAO
+BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUiZ/XuFgs
+FCGDhz0eMvNuB/aMvSgwDQYJKoZIhvcNAQENBQADggIBAHHsADO+SiUi51IibgF0
+gdKMurtJq2cdC8YNjkkDeI8jgIljrEi7HgYs9l3IbfRmBd5/5DRdVn8NLkjEVXSL
+fcKfGHqJSsA7qLylfXoBUAwcwObdo0fTMBn+NEfK3zb5BndClTaQRs2XiHmEwntR
+HUcSruOsWOJs9dxYHe89odMLIZv8rhbEH1vUIKC2vTnxF8vysJfx/ob3kpWiGClO
+pwpt5sc/BkWM+zo8gVnypqZzhWkYMJj5xrz0/1Wk9I8NwJnsjCcyFB+GMwX6b0ei
+TUU2MgS3krmG8A43JwUzPs8DVkQeWvsZejZzRCqDwlTwXM9pP8zGJFV0MYpyszc6
+Fx+qM2Xso5Gyja8RgHDlgJKAtnZe/vu6ocgnRXeLzLsWYVN3on2PLwL3dXxjciL0
+y4uCuLBb9ckbG3jJd4uvc6OdKVV47xsL6qgm4knHijclhkG4DXojAmdY2g0S0ptX
+ingwbLw5YHARLrOeXCgRp23SzXdvtnzbfgbI+9YQrxet8vFWg2Y+7NP2iF2/JufU
+HcPpuVGjsLkZBj4j9tOhUMDFk8esy6dBVpJ9+4d9slY0Eg5s5+XmnnVb6+QOCEii
+Gcq4nDgM8VEJxYFX9pxpjtiwiy3KVOP5QU+H0fjYfKIAi3IUdW03vzIu/H0vPk5h
+zceob2+4yKU2W+OQNeVChUzc
 -----END CERTIFICATE-----

integration/fixtures/gencert.json

@@ -0,0 +1,13 @@
+{
+  "signing": {
+    "default": {
+        "usages": [
+          "signing",
+          "key encipherment",
+          "server auth",
+          "client auth"
+        ],
+        "expiry": "87600h"
+    }
+  }
+}

integration/fixtures/gencerts.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+if ! [[ "$0" =~ "./gencerts.sh" ]]; then
+	echo "must be run from 'fixtures'"
+	exit 255
+fi
+
+if ! which cfssl; then
+	echo "cfssl is not installed"
+	exit 255
+fi
+
+cfssl gencert --initca=true ./ca-csr.json | cfssljson --bare ./ca
+mv ca.pem ca.crt
+
+cfssl gencert \
+    --ca ./ca.crt \
+    --ca-key ./ca-key.pem \
+    --config ./gencert.json \
+    ./server-ca-csr.json | cfssljson --bare ./server
+mv server.pem server.crt
+mv server-key.pem server.key.insecure
+
+cfssl gencert --ca ./ca.crt \
+    --ca-key ./ca-key.pem \
+    --config ./gencert.json \
+    ./server-ca-csr.json 2>revoked.stderr | cfssljson --bare ./server-revoked
+mv server-revoked.pem server-revoked.crt
+mv server-revoked-key.pem server-revoked.key.insecure
+
+grep serial revoked.stderr | awk ' { print $9 } ' >revoke.txt
+cfssl gencrl revoke.txt ca.crt ca-key.pem | base64 -d >revoke.crl
+
+rm -f *.csr *.pem *.stderr *.txt

integration/fixtures/server-ca-csr.json

@@ -0,0 +1,20 @@
+{
+  "key": {
+    "algo": "rsa",
+    "size": 4096
+  },
+  "names": [
+    {
+      "O": "etcd",
+      "OU": "etcd Security",
+      "L": "San Francisco",
+      "ST": "California",
+      "C": "USA"
+    }
+  ],
+  "CN": "example.com",
+  "hosts": [
+    "127.0.0.1",
+    "localhost"
+  ]
+}

integration/fixtures/server-revoked.crt

@@ -0,0 +1,35 @@
+-----BEGIN CERTIFICATE-----
+MIIGEjCCA/qgAwIBAgIUAyLIF+/vIdTKKf1wxsU+CfQkuvAwDQYJKoZIhvcNAQEN
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xNzA2MTYyMDMzMDBaFw0yNzA2MTQyMDMz
+MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggIiMA0GCSqGSIb3DQEBAQUA
+A4ICDwAwggIKAoICAQCzZzCUS5co1BFjkyPDhtxTSfJ0bdaVjkgvM9wmf5X8pBLc
+sb3iZO2oh1Dz24CNtpHDbfiN4oVW+BF5BX/rkcr6KYl/znjrP44kodUNN3uM8doP
+cfJ/ZFujmfdjtFXCgq9j3BkGW5+6ZGF/MBOtiDLXjT6JiS/F4jljxyepfdcRhnL3
+qxOiOOy5b9h+CSwxp48ubLVEzSz5qZb7ZGI+xp2tvLuoR/ZwL1Iiq4yrR4n42Crw
+oG7HOjlLBcwtxGedSLGz0LgUTPwliWA1dSd2sL3NnLUURilihSUfTZB57RMj1Uo5
+aQXAxXPXxyQx46zQXXhO/7YgCGK7vzgCP4Lr48cn6RQ4znIoLmUejWUxN+4CCVJc
+Py0Vn+j1PynPb4YhdWlOFjHMsVFMKpNbInSe/QG78+n8yJlYpVH09xvK6i+UQLex
+RfTYtNWtBQ7B22+ebgn6IWRiEWRpgzl02qeQnT/ndkSdfpn0soAH1tV1iATP8h+3
+Fznie+vpfUzeqKVA1W2akINs3LKVeW3yV1HSsqZQApF0i6cclevUL3K5uTevlhBy
+o+xvNUTG+bOtfegGrWVysbeaOyAglFGSv2K5Z3/flOXKSqg8dKc51RKA4sRK1zCQ
+kn5aNhMXjZUFWd8k0p8BvJCVTBofMlSwik2u8rkIOZh+ompe95YGnk+iFl3X/wID
+AQABo4GcMIGZMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYI
+KwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU97hg/c0lnnI30HFa3elg
+4ahPZ1AwHwYDVR0jBBgwFoAUiZ/XuFgsFCGDhz0eMvNuB/aMvSgwGgYDVR0RBBMw
+EYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEBDQUAA4ICAQAZwuZcAxWhOb28
+pFztpMAjOyW1zjFqxjECYLbMnJpf8Yf6SxvbD6J/U691jPuaR0PRAG/dL/Lcmqgg
+AAat4YnhnDYC5zG4ty0xaYsUk1AuK7iJXnAHT4klUzXmvajTrMT1uW9Yf4wVuIH/
+6fS7PvIT1oWe8ZFN72uAsNzv5I9wIFxlS6St1blFmA9HYvBpNIBJ7RaidGTs9nsP
+I8HawmD/iKhzbXZUWfYKiQ/JVsK/l5T2WYoRWgGEo605CuqBDah890up6dN4KaUx
+1Qi6WZ+MN6uaU5AA/Lvb7sS7viPdqZfraoJFNEBU/jNEmT0WL/EK6HzjredLlfE8
+Hzvy78/EZx1WbRsuDX3MG2/vYnZiWSL6DMdi3XxbJyC30FF9bc+0H7D73nSnZ22p
+9vluEdX6jsYkOglq/l5uuwK8BqWwB4tdgXJWMCWy+aQi38wz0UY7HLdS8cj7LNZQ
+9KI05vwZ8L5W30fhzWbO4jnYXbEhFNNW0yCKI174nAJM0m+vlw8w6np77l70AsCw
+MI4m3uvOGqIDjCPsuwJ4kjcpycMDeQS5+YCrkelixa0RWwgJAXJbHDSWeoQuVXW4
+UZkpdA2j9nSe3EbUMtAfCxLthxlSs6AiYcnYm3K9FKlmj1hIDxafMPxPrYDbZ9YE
+mdixLrkAUlyB50yoiYjbdTvFzvw40A==
+-----END CERTIFICATE-----

integration/fixtures/server-revoked.key.insecure

@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKAIBAAKCAgEAs2cwlEuXKNQRY5Mjw4bcU0nydG3WlY5ILzPcJn+V/KQS3LG9
+4mTtqIdQ89uAjbaRw234jeKFVvgReQV/65HK+imJf8546z+OJKHVDTd7jPHaD3Hy
+f2Rbo5n3Y7RVwoKvY9wZBlufumRhfzATrYgy140+iYkvxeI5Y8cnqX3XEYZy96sT
+ojjsuW/YfgksMaePLmy1RM0s+amW+2RiPsadrby7qEf2cC9SIquMq0eJ+Ngq8KBu
+xzo5SwXMLcRnnUixs9C4FEz8JYlgNXUndrC9zZy1FEYpYoUlH02Qee0TI9VKOWkF
+wMVz18ckMeOs0F14Tv+2IAhiu784Aj+C6+PHJ+kUOM5yKC5lHo1lMTfuAglSXD8t
+FZ/o9T8pz2+GIXVpThYxzLFRTCqTWyJ0nv0Bu/Pp/MiZWKVR9PcbyuovlEC3sUX0
+2LTVrQUOwdtvnm4J+iFkYhFkaYM5dNqnkJ0/53ZEnX6Z9LKAB9bVdYgEz/Iftxc5
+4nvr6X1M3qilQNVtmpCDbNyylXlt8ldR0rKmUAKRdIunHJXr1C9yubk3r5YQcqPs
+bzVExvmzrX3oBq1lcrG3mjsgIJRRkr9iuWd/35TlykqoPHSnOdUSgOLEStcwkJJ+
+WjYTF42VBVnfJNKfAbyQlUwaHzJUsIpNrvK5CDmYfqJqXveWBp5PohZd1/8CAwEA
+AQKCAgAmHEujlRM9Zx9yibVVOfbf8puAxDyLdLg83sVroDraenhPTarKxyn9XRGD
+XCPI9vmsDFZ6vZ4ZxYTgspxkDIqT7fL5pYDmaI/nlEFQF3M1k8MA+PHMwiL9dB2r
+nomBUoWzrvPZ9+jMjbpwbGQhvwcd5zFbwjrVzKLoYUw5ozPm0yrlFgCLu4/+LJZO
+39/1hGTUNd+kB/n/51jdeousTkD8wVUUAIWHe2X1W3/8eqwCotksWMhvphy5pTek
+mU5xiRnG2xXfqiL+TlqTwJlri0wmu51z+xubhDFInw+L9yLTv+GOpJLGSqu7MOCt
+gHIbqV9/WK71yxI+U/av85H6Tl7l7h0k7IcYvziFRCF2OHtj/4epUIGaC4c+JUHf
+cumDb0xQMNsXPeFBqzzS1pVocHb3A35YGuy5UNqEbvA0Pa1akxtMtDjOB+asoOu7
+b2ebxZpVI0RuzJwXGm7RXopTQ34Prvb45ZYfgx50izpTXGSHNsUjcPVE9JkToECr
+s2BP8+l/1iIyaRdFrKbD0dnDs78A77x/LsdQFMwFj8yIjpMywDf7oOcKQQTitKrb
+o1a3YEFDVmp4pKjoijsmqWkgSyYoZ++rYRrSg1XN7J3fDTtkmVnAVpW0vGKp8tDc
+iBhYXMiA3xhK5MLbW1jgG+IUmbW9BqaYEUusyoW3vjBWNUOIEQKCAQEA6sxOc1Y7
+rXxoWRjz1R/vwC4nMM2KBVShiIFbbxwmkL2ZEnk5BcdQao5Cy46n6prDMpjhO2Nu
+UHuAgwYJIVL7LAU5BWLkQbgsCIT/DpNw9MqsZpz9scExLNvWCGudKsWv4nG6YtBK
+0CyDYd+iFIM8FyCBm5NKpRAmIrcHrTlX2oSasLALzfGli0Tvanx/lJ3LG1XXx0aS
+RLI7ETrIpBXW1LAEw/b8z7h1H8OLoLZqzQVD5RAF7eLEOfSEPGeSkArysltNi0r4
+KcAAjPNX5jStFw/gvnKPIOzl/Z0l2BXQNN8nY3IhMI/hzlQ5kLrazSIfYONjsscZ
+DhFjhcDh7Jsi0wKCAQEAw5pZUOvxruSVdh0yR9F7mJ8PuZ+sUGC8XqblQh2W2MKE
+jwGTd+VYFhqubz1dF30Co7lD+3NmpKU5eWfkwtZ9DH+TJP82cz/X3A7ZENceHZlI
+HDUO0AvkCVIc8GV5nE30HFy6M/fCy6ww+v51JLu6aPJxkuFc/+Z1/SGXNgrxeKiS
+b0oesIB0Siqe9KyfGzBDBfOWYZ+h9mK1G63ged34Ufe61NcMYMh0PtLQ4QMS3eZj
+GezYr1SU0goSqDPdwxMI8YtYgjObMTDTqTcItqQajkCms5vI/BQqxLDL4zpDIG/C
+q+r3htufpzmDy78sC82VGYOqJSbuRUQEJn6Pz0JCpQKCAQEAhk3NBf4qbqa9zVEP
+kDYPXn9H3YuESl5Jc5qCoYCZsqsw1rdOXbbPkdPD80mrVO+nfdai5uSOIi0yqj+W
+k2Ay2dA2+JTDebAOR+VDO6QstWkEykE8gCPArCcnO3PK179yRvXEpmb8lC0SDbbs
+sA0NHzFx0Xqj9NzS44KMOpKVZSH6ldxtNCToC0yoIeIK3AiSOBWXMp50ZiUI0mcs
+g2cWllwErc8mdu7M2BlzUb9mQdWkK29J5POUSI5L4X1hAVGx10GQjn+YidEQv48m
+FUDGTW3AoU9H/Y5kU/hCwGJh0QxgLGsI/w0eHXZh4x5ur224QyRuT8HA3CIoKkbV
+NBDdDwKCAQAYXc3qirncs5T61cBBuhLPfGEVGpL4oFRW5iuBBPaZogGpXuTj9qn2
+dvRr3xUNrAD8LEQPutx6fiqeuRaCPv3s5KeL4E8EJFvbie34bxPNVB7rhKCHwjNy
+BcydFccEdaGHNvXrK14UySqA2dn17XzXPfxv86sGJctk8R5JGSvjf/xd6LnR+u6B
+nrWWfTqExIpU33dISf5/FuTAwfRIuiII8/dONEvScqYofVKP9TYQdna58As+LCct
+0xtn4K4rct+WV2l7LrqSciar2bM4LVExj1Q0458x58E8J1n+wG9ERyfVCrwzDIDi
+8AM/Cl2OStncB87WOfdMoaByuylnBDYRAoIBAAnU2EB5rtyXtYBHikNMY7AltEBw
+DlLFtTtnzQIkDFTJzc/P41agnB9IhxSiZi8AIAx5Z1GczoOevZwg1whA8fpVhAtg
+CD7yb3fsalaKMns9+X7vrMOqs5VpMg17yeJL5gmLhoWhQcpuxJpHLjCjt6Y+p1sU
++QdVMXcPELmZ7+Oo+Wvy6tYZwwYK78N4WF6c8Bxg8CgPp+66f4H73vEM+Kp0UNiU
+Ddcu4W+ywo6zQbh/u4EgMDihiijtuGLLCX5hvkz1P+gyJTv1XlZZuLdjUlHNqLMN
+XB2TrwVfYEQ1MDa6PEqmQs6vvcOhnrMs7u2nvONMpVFYIOl2Djh7TgTEfCs=
+-----END RSA PRIVATE KEY-----

integration/fixtures/server.crt

@@ -1,24 +1,35 @@
 -----BEGIN CERTIFICATE-----
-MIID9TCCAt2gAwIBAgIUXtrXPwZLfKUJiGr6ClP3lqhOuKUwDQYJKoZIhvcNAQEL
-BQAwdTEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
-Ew1TYW4gRnJhbmNpc2NvMRAwDgYDVQQKEwdldGNkLWNhMQswCQYDVQQLEwJDQTEZ
-MBcGA1UEAxMQQXV0b2dlbmVyYXRlZCBDQTAeFw0xNjA3MDUxOTQ1MDBaFw0xNzA3
-MDUxOTQ1MDBaMFUxFTATBgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0
-b2dlbmVyYXRlZDEVMBMGA1UECxMMZXRjZCBjbHVzdGVyMQ0wCwYDVQQDEwRldGNk
-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArGvOLPmy5i+1j4JitG4V
-g99w125ncs2ImhqucmJe6YtSnKruaLdOx93X7lzC3k0umnvLgh643R4eYS5PvrDk
-vw1dSYB7BHhveFPmmWd7m7n7bXtgbcdkCmUeTbSeqvptPgyMJOQfXzfOGbEHfu7U
-0raulR6KtqAatofKpRZhZgzZQpVkhdd0UTsOwqCWdX3Qe0D1MS922kX99c4UlGyD
-OTVL6tulvDBBYgHbGErFmhxdgwm4e6dFfdkPUeHczzUWnKo2sIGBvo4R/NwPIp6G
-PnebrO0VWvcQfdSqjYk3BmILl8BVL5W1/EBRLtz9mZuQgc/VC62LvsgXusC9pwXC
-3QIDAQABo4GcMIGZMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcD
-AQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQULLrktzdBK6iLINQ7
-hGRjQbMYXKowHwYDVR0jBBgwFoAUpcjGKayjQ5M8j4BgOXGJxBHvWPAwGgYDVR0R
-BBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEBCwUAA4IBAQCI2Tp4pMYk
-LFLzGy4e/5pwpA4x/C2zl01Sv/eC79RA5Zz1NtSF/7LCfL+KPNpNkxzPyTxWOaX5
-YMuAbD49ZBQYeEyNUxKcwWEpaVlmlIUj3b21fBXQ7Nw25Uea45bNhdZcdMUOTums
-J1/BrA2eoEB0guTlh3E8iadbVmSf6elA9TbYLd7QTTgcb3XclYCwhV3eKdm3IEiX
-g4q50iM6/LRz1E5C3LlQ0aNqpGroBv/9ahLVfLr06ziSRcecLJ4485MtJOxP4guA
-1tc6qPyw2MLmAlLZfOCHKLbK3KboZI8IANmrpNyL590D9bDl9nLnHmJuitBpIVp1
-Hw0I8e4ZYhab
+MIIGEjCCA/qgAwIBAgIUPXKyWW706lJ+NA6yXBEj5LotfsUwDQYJKoZIhvcNAQEN
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xNzA2MTYyMDMzMDBaFw0yNzA2MTQyMDMz
+MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggIiMA0GCSqGSIb3DQEBAQUA
+A4ICDwAwggIKAoICAQDAVrpxGgA7iNs5Av00t0jnndty0Fy5/bc8/YxO3N0LrVdD
+jSicA/1KvStKpx38ecJwuEw68xsHQc4Q0QKZCJ1sLEGNyCEOzpR1zveRlDwUbC/O
+0RTIaH5xWo+7ZbNNJjvt84/uxmSdYi3ezBFwilPQkH3vsc5r3plfEEG1MnW6rL6s
+C19R+6Vg3WlExq5qaPlmnkNso08d7gExOgeTQZ3oBHzCphWajKBhuoJAOQFzGe95
+kgg0ltXRPQ3LQDrswzVgev3K1jYKfbWBrQHnaL8SQNXHEfTdciwDlQdXk4KfjDE7
+ldwXPvx28xC1ELg4Kp/vFPF8YK9t/OFL/L1OpT/YmCl8tFwcPvesBVqnOZOSwVoj
+hRUOCfpk2TcYuRRJzKITKWMZWwzchufxdqYLSKg4aB+OD45vA5FER0hxxkmhxIVE
+6b+eqcZU43c8k5V31oC6uAiLSOCmuqAkFiaYIBDBUwEcne8FLaLIpL0ZDOySn6O9
+wiS7+20JRECk68z0VhQpmaA9yMu1rFHoukKJT9eHnw12Yx4RQNh10mM1fRdEpGbg
+PrQCdawNULofc1Kd4YiR2D6u8r58ZYHlUM5cUPWsr8mjQe7Clk++QSNF3UO+T7OR
+u1j/h4cg187kAo8bp8+DjOzryOgn4zRT4GpkWfW4A7ruXJ0lH1/vID/m+A+uIQID
+AQABo4GcMIGZMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYI
+KwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUBT0f8B8klRcSGL9rAtGm
+nuhDSCswHwYDVR0jBBgwFoAUiZ/XuFgsFCGDhz0eMvNuB/aMvSgwGgYDVR0RBBMw
+EYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEBDQUAA4ICAQAZiECIs74lFqKo
+x1HZI1txYfPmDau3kYhVLQTjcxr7959ATx7SGyDCaVDw9/n0yxfFw8JSyYgAmUTp
+WwBaROX+1zzq07QlB4xFkfBT0HBphFvWBCifRRX0Sv8VD6Zs31fFsEvVdAYpUlZd
+H4dzYXOzk6rlyGqQRx8CQWfJGaTNupkIS/aByPRpkDpNJrkObbMv2daqovV4dEjW
+vYNUT58cyQLxFtZoKEnRLJZrDB4nJVY3M81mtDeGHf/tOZV/MOz4W74VOy1xMCkk
+nXDpv6o9V0C2kiv3UIEAW7yoCL14q3Ou8z2XA9smsVrBoJCjzP185YmmnBZztpE5
+2Z6XTc77mJYIBdmx2pQz5sRw4dc5wWALgMF15PUa1/oNt7F+BiD0RCl7Eb+/n+1U
+qBh64ey3dv+SfsljbH52ywvFV1+kCiq6g9XT70DN85faGTsuRclyqR4ekttR83Sk
+HqJIVCkuthgh6BTkbyCzlF1QxNXDMWIlr+lGdwtIW1Vd85fPMrN9t4NwOYQiEpSF
+V46pIvEejokKECcajDjU1qmzmZwK5wl4hn6hMpYaYEr7LQTJjhW02+9yfGb/YU9O
+00h90p5EjHZMMxdLl1wxGM2DuV7dOPkmrNddhAXN45n9LLplieqE7u/avw3axLBI
++kjdtdYqFRJ8ON3Vu6xjnC7fNOLsyA==
 -----END CERTIFICATE-----

integration/fixtures/server.key.insecure

@@ -1,27 +1,51 @@
 -----BEGIN RSA PRIVATE KEY-----
-MIIEowIBAAKCAQEArGvOLPmy5i+1j4JitG4Vg99w125ncs2ImhqucmJe6YtSnKru
-aLdOx93X7lzC3k0umnvLgh643R4eYS5PvrDkvw1dSYB7BHhveFPmmWd7m7n7bXtg
-bcdkCmUeTbSeqvptPgyMJOQfXzfOGbEHfu7U0raulR6KtqAatofKpRZhZgzZQpVk
-hdd0UTsOwqCWdX3Qe0D1MS922kX99c4UlGyDOTVL6tulvDBBYgHbGErFmhxdgwm4
-e6dFfdkPUeHczzUWnKo2sIGBvo4R/NwPIp6GPnebrO0VWvcQfdSqjYk3BmILl8BV
-L5W1/EBRLtz9mZuQgc/VC62LvsgXusC9pwXC3QIDAQABAoIBAQCNseKz71hn9tk8
-YKiBIt6nix6OzHpTTDlwe3DVK6ZYQ1jWy1o10D773YIwryatzbv41Ld/7YN6o9/P
-eWGrkm/J2k/Jsb5nBBqHRlwBwZtBdOv9IyEx1mSObl8i+MZUOI1CKsmZH6fwdkn3
-rxY76EYaDGsYvQq93oFVc+7DEMtmMtr03xm2bleEvsUH0VVqLhiAof/PCgOzja/L
-mPxhK0FqOmhk94JFo2l0XNMn/b2lpUhrx+xny5RD6/W/k2C1DuzBiFiNZkbPW1r1
-n5QccJHpe/S3Y4WZ75yKyQdrcIz6AKSeHNNGw2mYERAOmejpVV+8OIvKY6pzyXi9
-EM/BsLaBAoGBAN+XiqHHGilsrvjLKGak2KIaPRxA7EgFKKWBv8DojpXLqgkoloDL
-1wS6uG4XE0FeJCiKZk/DpVgPSiKYkQJEFLgU8N3q8OO2cGYW8kfH/TuejWRebtgJ
-GC7o5CqAHjFqRbTPJBLLNlSUZP08HVIRhob3t0zkvVRdDjA1rZIM/FlxAoGBAMVp
-jTcimGEOhFbOvfLwFeMCFLglTzbxjSnxCLCKF5TbxcBN7iUE2wYRfexBLoP/3+rk
-RheyRnMr4PeZ/JPQLHs80TUm9HGg8Phy+jAsIW/rF8BJ4aAExt2T4uLNsj4TXw1y
-ckDMBLmZi0OFy4vDtwg4T2wVo55eN/oQfVNFFaotAoGAGLQ8q/08pcENYA3KS/UA
-voBZqip+MMLpJ8g7MIxBXMmg4twqLNbYzfv3bqp8BSfqpNQN09hRB3bBASuMMgzl
-oSUnK83OicpZht4YLNgq4ZB2HNXWN2Zh1qUCuLNpIpqUUxLj8HOlcBjpQ5WFw9CN
-5ZGvHf7T8GNLswXrRIzMwPECgYAC5Q5WDaLQYYcdQsDUTCL2BjTJknp74sTgJZGs
-DQpVe3eF316rmkuf5ifDjB0jgGAHMLu6YznXPIB7AP4MKNROJlEnB2A0PljqO71h
-cXQ4EOlzP2IYl5lW7HE6RCvl7yDIsLHuM0+qbQ72uYKHlSIc875uZk7U5qrJdu5v
-hybPLQKBgAmswE0nM9Fnj4ue9QaDgOvp1p7peZuzywBI4+TTJ/3++5vtUrgRl9Ak
-UVzSVvltxhFpFtNfVrZxckDwb6louumRtBrLVWJDlAakvc5eG5tky+SA2u/bdXSr
-8tq8c24K19Pg+OLkdZpiJqmKyyV0dVn6NNmiBmiLe2tClNsUHI47
+MIIJKQIBAAKCAgEAwFa6cRoAO4jbOQL9NLdI553bctBcuf23PP2MTtzdC61XQ40o
+nAP9Sr0rSqcd/HnCcLhMOvMbB0HOENECmQidbCxBjcghDs6Udc73kZQ8FGwvztEU
+yGh+cVqPu2WzTSY77fOP7sZknWIt3swRcIpT0JB977HOa96ZXxBBtTJ1uqy+rAtf
+UfulYN1pRMauamj5Zp5DbKNPHe4BMToHk0Gd6AR8wqYVmoygYbqCQDkBcxnveZII
+NJbV0T0Ny0A67MM1YHr9ytY2Cn21ga0B52i/EkDVxxH03XIsA5UHV5OCn4wxO5Xc
+Fz78dvMQtRC4OCqf7xTxfGCvbfzhS/y9TqU/2JgpfLRcHD73rAVapzmTksFaI4UV
+Dgn6ZNk3GLkUScyiEyljGVsM3Ibn8XamC0ioOGgfjg+ObwORREdIccZJocSFROm/
+nqnGVON3PJOVd9aAurgIi0jgprqgJBYmmCAQwVMBHJ3vBS2iyKS9GQzskp+jvcIk
+u/ttCURApOvM9FYUKZmgPcjLtaxR6LpCiU/Xh58NdmMeEUDYddJjNX0XRKRm4D60
+AnWsDVC6H3NSneGIkdg+rvK+fGWB5VDOXFD1rK/Jo0HuwpZPvkEjRd1Dvk+zkbtY
+/4eHINfO5AKPG6fPg4zs68joJ+M0U+BqZFn1uAO67lydJR9f7yA/5vgPriECAwEA
+AQKCAgB4Nzz86CVxEI8EyUt9oXld2xqIXpc2YNAgvNDvkbhPTh6WyCmzqgKsriPa
+2y0w2uGfFnH+/mfMV2L2u8yF3g6Wx+qJNacD3DaKk7vFMAOFOEGBYMk+oaE2NEZV
++LDi7ZzTk0JJGZNVk0HcWWwlDTBp0YYFRPsiDNWLx1tqZ4mSDdp3Kfx//2tUac8b
+/5qQQ+BuNUkrI7+Vk2cHX5/QeFi75eIcvzYNjQvJSYuTnI29ZV56e73JMOWfjCuM
++C5cQX6hOP5bwxuv3vaNgJcKDjwuqntzm1OS/YNpEcKvCDHCixtShM41cVbW+gFd
+Sn/6Cxo/krWE6eF8O2yN1AB2oW6PWDi5IYTmOFLD6f4gyWAuKd1Wdgv4qSZospic
+3YrGOkPqzV50xy1jtmMeJej+tMtZUG8cxLpxtnZOwPtyR2CbbMBX8LPuLkayBvnE
+4/6R7/V6rC41fViL2GA+f/rCi1qZ6scWM4YW0hFOWIi5bR0JKwM+s4NuMpcrrf94
+5O62PY7l1M0aAT4kupTMwPgrZD8Dk6Qq6YGxUmFFRzvBCKc+H6cG+k6yKeOpfyVP
+FvJPQjd64GOxyrtUVwXTQWcbcJ2vkfIm+TwUrLh/lyQe/WMJKaJXM3k/s4erEegP
+H1v+zf/d+mbihycWujZHQXuYpby5qzvaA19dTWmwtku+/EylKQKCAQEA/ppr32pR
+n8NA6S4FkYDYZ4vIk6cW7EmxXrOHXQOfJiP6os1mXVaBuPa82gfoqp7ALzC/bI0K
+4VeFrtXen88o4P9cp3pt4tl2qYdqkZZZtKTD0qsiY+OB1rauyol7uI/DwOAY/cuI
+63woihMjHIzsWVbfwa0h89RpdrxtFzbWrakMmypWijrxBwsE+9tUrIiA5T+rs+Ok
+Z1IF6cqiisG/m7aWRtHncKvGMyMLm0c5FRxmPksUkueY9Xce2XXKIxPqs9XXN4ZR
+5PNJksgVwQ7vAWMt+e0WtFeitbGOu+BL9wT5eAVv/0le7B4G0IBr3ko5mwF+Fwpq
+hFk+Ua9QIC8c+wKCAQEAwWTb/tE4PxQLIo+vK+Xu2RP2WhJdh2p3zNoKvYAGbUgt
+5EKDy7BP2zyC0GVCV5PcuAYi0jVCx5elE4C3WxuYyoz9drskSfNtqKuP4ABn00qu
+GgTw7ZnHbTRmKD68vurdaey16buzZ5xaxWLqFy5s1vDMaRLiJqk/Kaefj3E1168Z
+68fYIWdLnAfCZZbWqjfkGTA1Xmp3uUXkdlUmj9ZO+yHRJA7gGzN6qHQNgVCjmsub
+2/UQd0gATx4fYRHeg8VIcULGYdwPZH69WtMYAH63Glg769EPR9vX0ayIPWBWzxxJ
+GHx1PNeO2O6rNYTv8AsK9whtSg1Ey0FMrKN69PP+kwKCAQEAqhJ5MJqSv8fsGld1
+vMZ2yGRlrcwV0e1AIGSIbae3rgCmsg3G7V6V9PpNGM/jqeW1t1A27soglHqkgC+5
+MMNvKS0rtWxg7wIY9BHTLEFk1vihHp7WsCcPWmBZzk2caaMPPk30I8TMR5F+ew1h
+jPbOAre3bGa8oWYotPoVXUZyjTq5gwT7HgdVbkD8KUlP/JiKkZjAl7/e2G3AKXYE
+se/OkLOdGt2oXmlAUnR8klM3XFTGL6JzsdYnJQFXiCfYySnKxCp6Q0rkda3WXchu
+T7IsEZ8w3Rx06C8KL5jWIcYouG+IPZZURBG86Lj+EFVYuPqYFGUg1z9L+/CpYmNb
+gMQ39QKCAQEAgwF60awSyRAf1uK0jC0vVlfievOT7gaebuOJgQM311Gc3NJ+Cz49
+akscZmUhwsfELdjiP7maqjA8v/BixNH198f4bzSoiXvgXfER/nDn5Ebjl3afaqTQ
+ZlanmT2eiEn7gSS7ukDPcDGHf8zYvKlTS2tFXSxQjFX8FpCZUwcirR/NlF4FN+OM
+YX8UHdNHSXmhB7sTfAjtX1FMSSi3fZp0WER5M07sWyDNR02OVHo2ycaBp0fkRpk8
+tFu4b/412NeMkkTktCfpOW9tgBgdW0B7ctowhTdSfaAcG8ofZExdoeEMVw0AMh3n
+g2ZyDkgmbVpdrChGOHKylml+sjfwM/0RBwKCAQA6mGbaRsIZpsRM//aA++p+FRA4
+U9K4S+xdAQhcAcZRr+gL16Bpk2LDB3o3L7giUldi7wgFhJFBbZV3U7kj1qx7TWf3
+63db8piTdujUbp/PnIEAVVBx+taBZxlK9mRylL/mptYVsoD6hWaNboQiYR1HA1Tg
+dEN2cTIjhcCbUgwgXc/BGxeIjyuw/StsW0m8NsGGNNnkebzTdRBznvV7+vXr7Vnb
+8ybaOR4bp1RFwBlCmSVGbcvao5A2N9iexosxrWeOt+T9E5XKPrFG/eFfRJlbmFDZ
+A3K/rqdSsdSRz9XEKvaHzblJfn0cqSVNeliWy4YdaNYv8CtDAa9HAv1EUbKL
 -----END RSA PRIVATE KEY-----

lease/lessor.go

@@ -31,12 +31,15 @@ import (
 const (
 	// NoLease is a special LeaseID representing the absence of a lease.
 	NoLease = LeaseID(0)
+
+	forever = monotime.Time(math.MaxInt64)
 )
 
 var (
 	leaseBucketName = []byte("lease")
 
-	forever = monotime.Time(math.MaxInt64)
+	// maximum number of leases to revoke per second; configurable for tests
+	leaseRevokeRate = 1000
 
 	ErrNotPrimary    = errors.New("not a primary lessor")
 	ErrLeaseNotFound = errors.New("lease not found")
@@ -324,8 +327,53 @@ func (le *lessor) Promote(extend time.Duration) {
 	for _, l := range le.leaseMap {
 		l.refresh(extend)
 	}
+
+	if len(le.leaseMap) < leaseRevokeRate {
+		// no possibility of lease pile-up
+		return
+	}
+
+	// adjust expiries in case of overlap
+	leases := make([]*Lease, 0, len(le.leaseMap))
+	for _, l := range le.leaseMap {
+		leases = append(leases, l)
+	}
+	sort.Sort(leasesByExpiry(leases))
+
+	baseWindow := leases[0].Remaining()
+	nextWindow := baseWindow + time.Second
+	expires := 0
+	// have fewer expires than the total revoke rate so piled up leases
+	// don't consume the entire revoke limit
+	targetExpiresPerSecond := (3 * leaseRevokeRate) / 4
+	for _, l := range leases {
+		remaining := l.Remaining()
+		if remaining > nextWindow {
+			baseWindow = remaining
+			nextWindow = baseWindow + time.Second
+			expires = 1
+			continue
+		}
+		expires++
+		if expires <= targetExpiresPerSecond {
+			continue
+		}
+		rateDelay := float64(time.Second) * (float64(expires) / float64(targetExpiresPerSecond))
+		// If leases are extended by n seconds, leases n seconds ahead of the
+		// base window should be extended by only one second.
+		rateDelay -= float64(remaining - baseWindow)
+		delay := time.Duration(rateDelay)
+		nextWindow = baseWindow + delay
+		l.refresh(delay + extend)
+	}
 }
 
+type leasesByExpiry []*Lease
+
+func (le leasesByExpiry) Len() int           { return len(le) }
+func (le leasesByExpiry) Less(i, j int) bool { return le[i].Remaining() < le[j].Remaining() }
+func (le leasesByExpiry) Swap(i, j int)      { le[i], le[j] = le[j], le[i] }
+
 func (le *lessor) Demote() {
 	le.mu.Lock()
 	defer le.mu.Unlock()
@@ -422,6 +470,10 @@ func (le *lessor) runLoop() {
 		le.mu.Unlock()
 
 		if len(ls) != 0 {
+			// rate limit
+			if len(ls) > leaseRevokeRate/2 {
+				ls = ls[:leaseRevokeRate/2]
+			}
 			select {
 			case <-le.stopC:
 				return

lease/lessor_test.go

@@ -42,6 +42,7 @@ func TestLessorGrant(t *testing.T) {
 	defer be.Close()
 
 	le := newLessor(be, minLeaseTTL)
+	defer le.Stop()
 	le.Promote(0)
 
 	l, err := le.Grant(1, 1)
@@ -87,6 +88,7 @@ func TestLeaseConcurrentKeys(t *testing.T) {
 	defer be.Close()
 
 	le := newLessor(be, minLeaseTTL)
+	defer le.Stop()
 	le.SetRangeDeleter(func() TxnDelete { return newFakeDeleter(be) })
 
 	// grant a lease with long term (100 seconds) to
@@ -134,6 +136,7 @@ func TestLessorRevoke(t *testing.T) {
 	defer be.Close()
 
 	le := newLessor(be, minLeaseTTL)
+	defer le.Stop()
 	var fd *fakeDeleter
 	le.SetRangeDeleter(func() TxnDelete {
 		fd = newFakeDeleter(be)
@@ -185,6 +188,7 @@ func TestLessorRenew(t *testing.T) {
 	defer os.RemoveAll(dir)
 
 	le := newLessor(be, minLeaseTTL)
+	defer le.Stop()
 	le.Promote(0)
 
 	l, err := le.Grant(1, minLeaseTTL)
@@ -210,12 +214,66 @@ func TestLessorRenew(t *testing.T) {
 	}
 }
 
+// TestLessorRenewExtendPileup ensures Lessor extends leases on promotion if too many
+// expire at the same time.
+func TestLessorRenewExtendPileup(t *testing.T) {
+	oldRevokeRate := leaseRevokeRate
+	defer func() { leaseRevokeRate = oldRevokeRate }()
+	leaseRevokeRate = 10
+
+	dir, be := NewTestBackend(t)
+	defer os.RemoveAll(dir)
+
+	le := newLessor(be, minLeaseTTL)
+	ttl := int64(10)
+	for i := 1; i <= leaseRevokeRate*10; i++ {
+		if _, err := le.Grant(LeaseID(2*i), ttl); err != nil {
+			t.Fatal(err)
+		}
+		// ttls that overlap spillover for ttl=10
+		if _, err := le.Grant(LeaseID(2*i+1), ttl+1); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// simulate stop and recovery
+	le.Stop()
+	be.Close()
+	bcfg := backend.DefaultBackendConfig()
+	bcfg.Path = filepath.Join(dir, "be")
+	be = backend.New(bcfg)
+	defer be.Close()
+	le = newLessor(be, minLeaseTTL)
+	defer le.Stop()
+
+	// extend after recovery should extend expiration on lease pile-up
+	le.Promote(0)
+
+	windowCounts := make(map[int64]int)
+	for _, l := range le.leaseMap {
+		// round up slightly for baseline ttl
+		s := int64(l.Remaining().Seconds() + 0.1)
+		windowCounts[s]++
+	}
+
+	for i := ttl; i < ttl+20; i++ {
+		c := windowCounts[i]
+		if c > leaseRevokeRate {
+			t.Errorf("expected at most %d expiring at %ds, got %d", leaseRevokeRate, i, c)
+		}
+		if c < leaseRevokeRate/2 {
+			t.Errorf("expected at least %d expiring at %ds, got %d", leaseRevokeRate/2, i, c)
+		}
+	}
+}
+
 func TestLessorDetach(t *testing.T) {
 	dir, be := NewTestBackend(t)
 	defer os.RemoveAll(dir)
 	defer be.Close()
 
 	le := newLessor(be, minLeaseTTL)
+	defer le.Stop()
 	le.SetRangeDeleter(func() TxnDelete { return newFakeDeleter(be) })
 
 	// grant a lease with long term (100 seconds) to
@@ -255,6 +313,7 @@ func TestLessorRecover(t *testing.T) {
 	defer be.Close()
 
 	le := newLessor(be, minLeaseTTL)
+	defer le.Stop()
 	l1, err1 := le.Grant(1, 10)
 	l2, err2 := le.Grant(2, 20)
 	if err1 != nil || err2 != nil {
@@ -263,6 +322,7 @@ func TestLessorRecover(t *testing.T) {
 
 	// Create a new lessor with the same backend
 	nle := newLessor(be, minLeaseTTL)
+	defer nle.Stop()
 	nl1 := nle.Lookup(l1.ID)
 	if nl1 == nil || nl1.ttl != l1.ttl {
 		t.Errorf("nl1 = %v, want nl1.ttl= %d", nl1.ttl, l1.ttl)

pkg/testutil/leak.go

@@ -118,6 +118,7 @@ func interestingGoroutines() (gs []string) {
 		}
 		stack := strings.TrimSpace(sl[1])
 		if stack == "" ||
+			strings.Contains(stack, "sync.(*WaitGroup).Done") ||
 			strings.Contains(stack, "created by os/signal.init") ||
 			strings.Contains(stack, "runtime/panic.go") ||
 			strings.Contains(stack, "created by testing.RunTests") ||

pkg/transport/listener_tls.go

@@ -142,7 +142,11 @@ func checkCert(ctx context.Context, cert *x509.Certificate, remoteAddr string) e
 		return herr
 	}
 	if len(cert.IPAddresses) > 0 {
-		if cerr := cert.VerifyHostname(h); cerr != nil && len(cert.DNSNames) == 0 {
+		cerr := cert.VerifyHostname(h)
+		if cerr == nil {
+			return nil
+		}
+		if len(cert.DNSNames) == 0 {
 			return cerr
 		}
 	}

proxy/grpcproxy/maintenance.go

@@ -15,6 +15,8 @@
 package grpcproxy
 
 import (
+	"io"
+
 	"golang.org/x/net/context"
 
 	"github.com/coreos/etcd/clientv3"
@@ -49,6 +51,9 @@ func (mp *maintenanceProxy) Snapshot(sr *pb.SnapshotRequest, stream pb.Maintenan
 	for {
 		rr, err := sc.Recv()
 		if err != nil {
+			if err == io.EOF {
+				return nil
+			}
 			return err
 		}
 		err = stream.Send(rr)

test

@@ -100,13 +100,21 @@ function functional_pass {
 		agent_pids="${agent_pids} $pid"
 	done
 
+	for a in 1 2 3; do
+		echo "Waiting for 'etcd-agent' on ${a}9027..."
+		while ! nc -z localhost ${a}9027; do
+			sleep 1
+		done
+	done
+
+	echo "Starting 'etcd-tester'"
 	./bin/etcd-tester \
 		-agent-endpoints "127.0.0.1:19027,127.0.0.1:29027,127.0.0.1:39027" \
 		-client-ports 12379,22379,32379 \
 		-peer-ports 12380,22380,32380 \
 		-limit 1 \
 		-schedule-cases "0 1 2 3 4 5" \
-		-exit-on-failure
+		-exit-on-failure && echo "'etcd-tester' succeeded"
 	ETCD_TESTER_EXIT_CODE=$?
 	echo "ETCD_TESTER_EXIT_CODE:" ${ETCD_TESTER_EXIT_CODE}
 

tools/benchmark/cmd/watch.go

@@ -15,6 +15,7 @@
 package cmd
 
 import (
+	"context"
 	"encoding/binary"
 	"fmt"
 	"math/rand"
@@ -22,11 +23,11 @@ import (
 	"sync/atomic"
 	"time"
 
-	v3 "github.com/coreos/etcd/clientv3"
+	"github.com/coreos/etcd/clientv3"
 	"github.com/coreos/etcd/pkg/report"
 
 	"github.com/spf13/cobra"
-	"golang.org/x/net/context"
+	"golang.org/x/time/rate"
 	"gopkg.in/cheggaaa/pb.v1"
 )
 
@@ -50,9 +51,9 @@ Each key is watched by (--total/--watched-key-total) watchers.
 }
 
 var (
-	watchTotalStreams int
-	watchTotal        int
-	watchedKeyTotal   int
+	watchStreams          int
+	watchWatchesPerStream int
+	watchedKeyTotal       int
 
 	watchPutRate  int
 	watchPutTotal int
@@ -60,23 +61,27 @@ var (
 	watchKeySize      int
 	watchKeySpaceSize int
 	watchSeqKeys      bool
-
-	eventsTotal int
-
-	nrWatchCompleted       int32
-	nrRecvCompleted        int32
-	watchCompletedNotifier chan struct{}
-	recvCompletedNotifier  chan struct{}
 )
 
+type watchedKeys struct {
+	watched     []string
+	numWatchers map[string]int
+
+	watches []clientv3.WatchChan
+
+	// ctx to control all watches
+	ctx    context.Context
+	cancel context.CancelFunc
+}
+
 func init() {
 	RootCmd.AddCommand(watchCmd)
-	watchCmd.Flags().IntVar(&watchTotalStreams, "watchers", 10000, "Total number of watchers")
-	watchCmd.Flags().IntVar(&watchTotal, "total", 100000, "Total number of watch requests")
-	watchCmd.Flags().IntVar(&watchedKeyTotal, "watched-key-total", 10000, "Total number of keys to be watched")
+	watchCmd.Flags().IntVar(&watchStreams, "streams", 10, "Total watch streams")
+	watchCmd.Flags().IntVar(&watchWatchesPerStream, "watch-per-stream", 100, "Total watchers per stream")
+	watchCmd.Flags().IntVar(&watchedKeyTotal, "watched-key-total", 1, "Total number of keys to be watched")
 
-	watchCmd.Flags().IntVar(&watchPutRate, "put-rate", 100, "Number of keys to put per second")
-	watchCmd.Flags().IntVar(&watchPutTotal, "put-total", 10000, "Number of put requests")
+	watchCmd.Flags().IntVar(&watchPutRate, "put-rate", 0, "Number of keys to put per second")
+	watchCmd.Flags().IntVar(&watchPutTotal, "put-total", 1000, "Number of put requests")
 
 	watchCmd.Flags().IntVar(&watchKeySize, "key-size", 32, "Key size of watch request")
 	watchCmd.Flags().IntVar(&watchKeySpaceSize, "key-space-size", 1, "Maximum possible keys")
@@ -88,9 +93,76 @@ func watchFunc(cmd *cobra.Command, args []string) {
 		fmt.Fprintf(os.Stderr, "expected positive --key-space-size, got (%v)", watchKeySpaceSize)
 		os.Exit(1)
 	}
+	grpcConns := int(totalClients)
+	if totalClients > totalConns {
+		grpcConns = int(totalConns)
+	}
+	wantedConns := 1 + (watchStreams / 100)
+	if grpcConns < wantedConns {
+		fmt.Fprintf(os.Stderr, "warning: grpc limits 100 streams per client connection, have %d but need %d\n", grpcConns, wantedConns)
+	}
+	clients := mustCreateClients(totalClients, totalConns)
+	wk := newWatchedKeys()
+	benchMakeWatches(clients, wk)
+	benchPutWatches(clients, wk)
+}
 
+func benchMakeWatches(clients []*clientv3.Client, wk *watchedKeys) {
+	streams := make([]clientv3.Watcher, watchStreams)
+	for i := range streams {
+		streams[i] = clientv3.NewWatcher(clients[i%len(clients)])
+	}
+
+	keyc := make(chan string, watchStreams)
+	bar = pb.New(watchStreams * watchWatchesPerStream)
+	bar.Format("Bom !")
+	bar.Start()
+
+	r := newReport()
+	rch := r.Results()
+
+	wg.Add(len(streams) + 1)
+	wc := make(chan []clientv3.WatchChan, len(streams))
+	for _, s := range streams {
+		go func(s clientv3.Watcher) {
+			defer wg.Done()
+			var ws []clientv3.WatchChan
+			for i := 0; i < watchWatchesPerStream; i++ {
+				k := <-keyc
+				st := time.Now()
+				wch := s.Watch(wk.ctx, k)
+				rch <- report.Result{Start: st, End: time.Now()}
+				ws = append(ws, wch)
+				bar.Increment()
+			}
+			wc <- ws
+		}(s)
+	}
+	go func() {
+		defer func() {
+			close(keyc)
+			wg.Done()
+		}()
+		for i := 0; i < watchStreams*watchWatchesPerStream; i++ {
+			key := wk.watched[i%len(wk.watched)]
+			keyc <- key
+			wk.numWatchers[key]++
+		}
+	}()
+
+	rc := r.Run()
+	wg.Wait()
+	bar.Finish()
+	close(r.Results())
+	fmt.Printf("Watch creation summary:\n%s", <-rc)
+
+	for i := 0; i < len(streams); i++ {
+		wk.watches = append(wk.watches, (<-wc)...)
+	}
+}
+
+func newWatchedKeys() *watchedKeys {
 	watched := make([]string, watchedKeyTotal)
-	numWatchers := make(map[string]int)
 	for i := range watched {
 		k := make([]byte, watchKeySize)
 		if watchSeqKeys {
@@ -100,112 +172,76 @@ func watchFunc(cmd *cobra.Command, args []string) {
 		}
 		watched[i] = string(k)
 	}
-
-	requests := make(chan string, totalClients)
-
-	clients := mustCreateClients(totalClients, totalConns)
-
-	streams := make([]v3.Watcher, watchTotalStreams)
-	for i := range streams {
-		streams[i] = v3.NewWatcher(clients[i%len(clients)])
+	ctx, cancel := context.WithCancel(context.TODO())
+	return &watchedKeys{
+		watched:     watched,
+		numWatchers: make(map[string]int),
+		ctx:         ctx,
+		cancel:      cancel,
 	}
+}
 
-	// watching phase
-	bar = pb.New(watchTotal)
-	bar.Format("Bom !")
-	bar.Start()
-
-	atomic.StoreInt32(&nrWatchCompleted, int32(0))
-	watchCompletedNotifier = make(chan struct{})
-
-	r := report.NewReportRate("%4.4f")
-	for i := range streams {
-		go doWatch(streams[i], requests, r.Results())
-	}
-
-	go func() {
-		for i := 0; i < watchTotal; i++ {
-			key := watched[i%len(watched)]
-			requests <- key
-			numWatchers[key]++
-		}
-		close(requests)
-	}()
-
-	rc := r.Run()
-	<-watchCompletedNotifier
-	bar.Finish()
-	close(r.Results())
-	fmt.Printf("Watch creation summary:\n%s", <-rc)
-
-	// put phase
-	eventsTotal = 0
+func benchPutWatches(clients []*clientv3.Client, wk *watchedKeys) {
+	eventsTotal := 0
 	for i := 0; i < watchPutTotal; i++ {
-		eventsTotal += numWatchers[watched[i%len(watched)]]
+		eventsTotal += wk.numWatchers[wk.watched[i%len(wk.watched)]]
 	}
 
 	bar = pb.New(eventsTotal)
 	bar.Format("Bom !")
 	bar.Start()
 
-	atomic.StoreInt32(&nrRecvCompleted, 0)
-	recvCompletedNotifier = make(chan struct{})
-	putreqc := make(chan v3.Op)
+	r := newReport()
 
-	r = report.NewReportRate("%4.4f")
-	for i := 0; i < watchPutTotal; i++ {
-		go func(c *v3.Client) {
-			for op := range putreqc {
-				if _, err := c.Do(context.TODO(), op); err != nil {
-					fmt.Fprintf(os.Stderr, "failed to Put for watch benchmark: %v\n", err)
-					os.Exit(1)
-				}
-			}
-		}(clients[i%len(clients)])
+	wg.Add(len(wk.watches))
+	nrRxed := int32(eventsTotal)
+	for _, w := range wk.watches {
+		go func(wc clientv3.WatchChan) {
+			defer wg.Done()
+			recvWatchChan(wc, r.Results(), &nrRxed)
+			wk.cancel()
+		}(w)
 	}
 
+	putreqc := make(chan clientv3.Op, len(clients))
 	go func() {
+		defer close(putreqc)
 		for i := 0; i < watchPutTotal; i++ {
-			putreqc <- v3.OpPut(watched[i%(len(watched))], "data")
-			// TODO: use a real rate-limiter instead of sleep.
-			time.Sleep(time.Second / time.Duration(watchPutRate))
+			putreqc <- clientv3.OpPut(wk.watched[i%(len(wk.watched))], "data")
 		}
-		close(putreqc)
 	}()
 
-	rc = r.Run()
-	<-recvCompletedNotifier
+	limit := rate.NewLimiter(rate.Limit(watchPutRate), 1)
+	for _, cc := range clients {
+		go func(c *clientv3.Client) {
+			for op := range putreqc {
+				if err := limit.Wait(context.TODO()); err != nil {
+					panic(err)
+				}
+				if _, err := c.Do(context.TODO(), op); err != nil {
+					panic(err)
+				}
+			}
+		}(cc)
+	}
+
+	rc := r.Run()
+	wg.Wait()
 	bar.Finish()
 	close(r.Results())
 	fmt.Printf("Watch events received summary:\n%s", <-rc)
+
 }
 
-func doWatch(stream v3.Watcher, requests <-chan string, results chan<- report.Result) {
-	for r := range requests {
-		st := time.Now()
-		wch := stream.Watch(context.TODO(), r)
-		results <- report.Result{Start: st, End: time.Now()}
-		bar.Increment()
-		go recvWatchChan(wch, results)
-	}
-	atomic.AddInt32(&nrWatchCompleted, 1)
-	if atomic.LoadInt32(&nrWatchCompleted) == int32(watchTotalStreams) {
-		watchCompletedNotifier <- struct{}{}
-	}
-}
-
-func recvWatchChan(wch v3.WatchChan, results chan<- report.Result) {
+func recvWatchChan(wch clientv3.WatchChan, results chan<- report.Result, nrRxed *int32) {
 	for r := range wch {
 		st := time.Now()
 		for range r.Events {
 			results <- report.Result{Start: st, End: time.Now()}
 			bar.Increment()
-			atomic.AddInt32(&nrRecvCompleted, 1)
-		}
-
-		if atomic.LoadInt32(&nrRecvCompleted) == int32(eventsTotal) {
-			recvCompletedNotifier <- struct{}{}
-			break
+			if atomic.AddInt32(nrRxed, -1) <= 0 {
+				return
+			}
 		}
 	}
 }

version/version.go

@@ -26,7 +26,7 @@ import (
 var (
 	// MinClusterVersion is the min cluster version this etcd binary is compatible with.
 	MinClusterVersion = "3.0.0"
-	Version           = "3.2.1"
+	Version           = "3.2.4"
 	APIVersion        = "unknown"
 
 	// Git SHA Value will be set during build