version: bump up to 3.2.2

Signed-off-by: Gyu-Ho Lee <gyuhox@gmail.com>

Documentation/dev-guide/api_grpc_gateway.md

@@ -38,6 +38,15 @@ curl -L http://localhost:2379/v3alpha/kv/put \
 # {"result":{"header":{"cluster_id":"12585971608760269493","member_id":"13847567121247652255","revision":"2","raft_term":"2"},"events":[{"kv":{"key":"Zm9v","create_revision":"2","mod_revision":"2","version":"1","value":"YmFy"}}]}}
 ```
 
+Use `curl` to issue a transaction:
+
+```bash
+curl -L http://localhost:2379/v3alpha/kv/txn \
+	-X POST \
+	-d '{"compare":[{"target":"CREATE","key":"Zm9v","createRevision":"2"}],"success":[{"requestPut":{"key":"Zm9v","value":"YmFy"}}]}'
+# {"header":{"cluster_id":"12585971608760269493","member_id":"13847567121247652255","revision":"3","raft_term":"2"},"succeeded":true,"responses":[{"response_put":{"header":{"revision":"3"}}}]}
+```
+
 ## Swagger
 
 Generated [Swagger][swagger] API definitions can be found at [rpc.swagger.json][swagger-doc].

Documentation/dev-guide/experimental_apis.md

@@ -4,8 +4,4 @@ For the most part, the etcd project is stable, but we are still moving fast! We
 
 ## The current experimental API/features are:
 
-- [gateway][gateway]: beta, to be stable in 3.2 release
+(none currently)
-- [gRPC proxy][grpc-proxy]: alpha, to be stable in 3.2 release
-
-[gateway]: ../op-guide/gateway.md
-[grpc-proxy]: ../op-guide/grpc_proxy.md

Documentation/faq.md

@@ -8,11 +8,11 @@
 
 ### Configuration
 
-#### What is the difference between advertise-urls and listen-urls?
+#### What is the difference between listen-<client,peer>-urls, advertise-client-urls or initial-advertise-peer-urls?
 
-`listen-urls` specifies the local addresses etcd server binds to for accepting incoming connections. To listen on a port for all interfaces, specify `0.0.0.0` as the listen IP address.
+`listen-client-urls` and `listen-peer-urls` specify the local addresses etcd server binds to for accepting incoming connections. To listen on a port for all interfaces, specify `0.0.0.0` as the listen IP address.
 
-`advertise-urls` specifies the addresses etcd clients or other etcd members should use to contact the etcd server. The advertise addresses must be reachable from the remote machines. Do not advertise addresses like `localhost` or `0.0.0.0` for a production setup since these addresses are unreachable from remote machines.
+`advertise-client-urls` and `initial-advertise-peer-urls` specify the addresses etcd clients or other etcd members should use to contact the etcd server. The advertise addresses must be reachable from the remote machines. Do not advertise addresses like `localhost` or `0.0.0.0` for a production setup since these addresses are unreachable from remote machines.
 
 ### Deployment
 

Documentation/op-guide/grafana.json

@@ -114,18 +114,21 @@
                     "span": 5,
                     "stack": false,
                     "steppedLine": false,
-                    "targets": [{
+                    "targets": [
-                            "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"} [1m]))",
+                        {
+                            "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[5m]))",
+                            "format": "time_series",
                             "intervalFactor": 2,
-                            "legendFormat": "{{instance}} RPC Rate",
+                            "legendFormat": "RPC Rate",
                             "metric": "grpc_server_started_total",
                             "refId": "A",
                             "step": 2
                         },
                         {
-                            "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"} [1m])) - sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"} [1m]))",
+                            "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[5m]))",
+                            "format": "time_series",
                             "intervalFactor": 2,
-                            "legendFormat": "{{instance}} RPC Failed Rate",
+                            "legendFormat": "RPC Failed Rate",
                             "metric": "grpc_server_handled_total",
                             "refId": "B",
                             "step": 2
@@ -522,7 +525,7 @@
                     "stack": true,
                     "steppedLine": false,
                     "targets": [{
-                        "expr": "rate(etcd_network_client_grpc_received_bytes_total [1m])",
+                        "expr": "rate(etcd_network_client_grpc_received_bytes_total[5m])",
                         "intervalFactor": 2,
                         "legendFormat": "{{instance}} Client Traffic In",
                         "metric": "etcd_network_client_grpc_received_bytes_total",
@@ -595,7 +598,7 @@
                     "stack": true,
                     "steppedLine": false,
                     "targets": [{
-                        "expr": "rate(etcd_network_client_grpc_sent_bytes_total [1m])",
+                        "expr": "rate(etcd_network_client_grpc_sent_bytes_total[5m])",
                         "intervalFactor": 2,
                         "legendFormat": "{{instance}} Client Traffic Out",
                         "metric": "etcd_network_client_grpc_sent_bytes_total",
@@ -668,7 +671,7 @@
                     "stack": false,
                     "steppedLine": false,
                     "targets": [{
-                        "expr": "sum(rate(etcd_network_peer_received_bytes_total [1m])) by (instance)",
+                        "expr": "sum(rate(etcd_network_peer_received_bytes_total[5m])) by (instance)",
                         "intervalFactor": 2,
                         "legendFormat": "{{instance}} Peer Traffic In",
                         "metric": "etcd_network_peer_received_bytes_total",
@@ -742,7 +745,7 @@
                     "stack": false,
                     "steppedLine": false,
                     "targets": [{
-                        "expr": "sum(rate(etcd_network_peer_sent_bytes_total [1m])) by (instance)",
+                        "expr": "sum(rate(etcd_network_peer_sent_bytes_total[5m])) by (instance)",
                         "hide": false,
                         "interval": "",
                         "intervalFactor": 2,
@@ -822,7 +825,7 @@
                     "stack": false,
                     "steppedLine": false,
                     "targets": [{
-                            "expr": "sum(rate(etcd_server_proposals_failed_total [1m]))",
+                            "expr": "sum(rate(etcd_server_proposals_failed_total[5m]))",
                             "intervalFactor": 2,
                             "legendFormat": "Proposal Failure Rate",
                             "metric": "etcd_server_proposals_failed_total",
@@ -838,7 +841,7 @@
                             "step": 2
                         },
                         {
-                            "expr": "sum(rate(etcd_server_proposals_committed_total [1m]))",
+                            "expr": "sum(rate(etcd_server_proposals_committed_total[5m]))",
                             "intervalFactor": 2,
                             "legendFormat": "Proposal Commit Rate",
                             "metric": "etcd_server_proposals_committed_total",
@@ -846,7 +849,7 @@
                             "step": 2
                         },
                         {
-                            "expr": "sum(rate(etcd_server_proposals_applied_total [1m]))",
+                            "expr": "sum(rate(etcd_server_proposals_applied_total[5m]))",
                             "intervalFactor": 2,
                             "legendFormat": "Proposal Apply Rate",
                             "refId": "D",
@@ -922,9 +925,9 @@
                     "stack": false,
                     "steppedLine": false,
                     "targets": [{
-                        "expr": "etcd_server_leader_changes_seen_total",
+                        "expr": "changes(etcd_server_leader_changes_seen_total[1d])",
                         "intervalFactor": 2,
-                        "legendFormat": "{{instance}} Leader Change Seen",
+                        "legendFormat": "{{instance}} Total Leader Elections Per Day",
                         "metric": "etcd_server_leader_changes_seen_total",
                         "refId": "A",
                         "step": 2
@@ -932,7 +935,7 @@
                     "thresholds": [],
                     "timeFrom": null,
                     "timeShift": null,
-                    "title": "Rate Leader Elections",
+                    "title": "Total Leader Elections Per Day",
                     "tooltip": {
                         "msResolution": false,
                         "shared": true,

Documentation/upgrades/upgrade_3_0.md

@@ -10,7 +10,7 @@ Before [starting an upgrade](#upgrade-procedure), read through the rest of this
 
 #### Upgrade requirements
 
-To upgrade an existing etcd deployment to 3.0, the running cluster must be 2.3 or greater. If it's before 2.3, please upgrade to [2.3](https://github.com/coreos/etcd/releases/tag/v2.3.0) before upgrading to 3.0.
+To upgrade an existing etcd deployment to 3.0, the running cluster must be 2.3 or greater. If it's before 2.3, please upgrade to [2.3](https://github.com/coreos/etcd/releases/tag/v2.3.8) before upgrading to 3.0.
 
 Also, to ensure a smooth rolling upgrade, the running cluster must be healthy. Check the health of the cluster by using the `etcdctl cluster-health` command before proceeding.
 
@@ -52,7 +52,7 @@ member 8211f1d0f64f3269 is healthy: got healthy result from http://localhost:123
 cluster is healthy
 
 $ curl http://localhost:2379/version
-{"etcdserver":"2.3.x","etcdcluster":"2.3.0"}
+{"etcdserver":"2.3.x","etcdcluster":"2.3.8"}
 ```
 
 #### 2. Stop the existing etcd process

Documentation/upgrades/upgrade_3_1.md

@@ -10,7 +10,7 @@ Before [starting an upgrade](#upgrade-procedure), read through the rest of this
 
 #### Upgrade requirements
 
-To upgrade an existing etcd deployment to 3.1, the running cluster must be 3.0 or greater. If it's before 3.0, please upgrade to [3.0](https://github.com/coreos/etcd/releases/tag/v3.0.16) before upgrading to 3.1.
+To upgrade an existing etcd deployment to 3.1, the running cluster must be 3.0 or greater. If it's before 3.0, please [upgrade to 3.0](upgrade_3_0.md) before upgrading to 3.1.
 
 Also, to ensure a smooth rolling upgrade, the running cluster must be healthy. Check the health of the cluster by using the `etcdctl endpoint health` command before proceeding.
 

Documentation/upgrades/upgrade_3_2.md

@@ -50,7 +50,7 @@ clientv3yaml.NewConfig
 
 #### Upgrade requirements
 
-To upgrade an existing etcd deployment to 3.2, the running cluster must be 3.1 or greater. If it's before 3.1, please upgrade to [3.1](https://github.com/coreos/etcd/releases/tag/v3.1.7) before upgrading to 3.2.
+To upgrade an existing etcd deployment to 3.2, the running cluster must be 3.1 or greater. If it's before 3.1, please [upgrade to 3.1](upgrade_3_1.md) before upgrading to 3.2.
 
 Also, to ensure a smooth rolling upgrade, the running cluster must be healthy. Check the health of the cluster by using the `etcdctl endpoint health` command before proceeding.
 

Documentation/v2/04_to_2_snapshot_migration.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Snapshot Migration
 
 You can migrate a snapshot of your data from a v0.4.9+ cluster into a new etcd 2.2 cluster using a snapshot migration. After snapshot migration, the etcd indexes of your data will change. Many etcd applications rely on these indexes to behave correctly. This operation should only be done while all etcd applications are stopped.

Documentation/v2/README.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # etcd2
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/coreos/etcd)](https://goreportcard.com/report/github.com/coreos/etcd)

Documentation/v2/admin_guide.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Administration
 
 ## Data Directory

Documentation/v2/api.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # etcd API
 
 ## Running a Single Machine Cluster

Documentation/v2/api_v3.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # etcd3 API
 
 TODO: API doc

Documentation/v2/auth_api.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # v2 Auth and Security
 
 ## etcd Resources

Documentation/v2/authentication.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Authentication Guide
 
 ## Overview

Documentation/v2/backward_compatibility.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Backward Compatibility
 
 The main goal of etcd 2.0 release is to improve cluster safety around bootstrapping and dynamic reconfiguration. To do this, we deprecated the old error-prone APIs and provide a new set of APIs.

Documentation/v2/benchmarks/README.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../../docs.md#documentation
+
+
 # Benchmarks
 
 etcd benchmarks will be published regularly and tracked for each release below:

Documentation/v2/benchmarks/etcd-2-1-0-alpha-benchmarks.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../../docs.md#documentation
+
+
 ## Physical machines
 
 GCE n1-highcpu-2 machine type

Documentation/v2/benchmarks/etcd-2-2-0-benchmarks.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../../docs.md#documentation
+
+
 # Benchmarking etcd v2.2.0
 
 ## Physical Machines

Documentation/v2/benchmarks/etcd-2-2-0-rc-benchmarks.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../../docs.md#documentation
+
+
 ## Physical machines
 
 GCE n1-highcpu-2 machine type

Documentation/v2/benchmarks/etcd-2-2-0-rc-memory-benchmarks.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../../docs.md#documentation
+
+
 ## Physical machine
 
 GCE n1-standard-2 machine type

Documentation/v2/benchmarks/etcd-3-demo-benchmarks.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../../docs.md#documentation
+
+
 ## Physical machines
 
 GCE n1-highcpu-2 machine type

Documentation/v2/benchmarks/etcd-3-watch-memory-benchmark.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../../docs.md#documentation
+
+
 # Watch Memory Usage Benchmark
 
 *NOTE*: The watch features are under active development, and their memory usage may change as that development progresses. We do not expect it to significantly increase beyond the figures stated below.

Documentation/v2/benchmarks/etcd-storage-memory-benchmark.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../../docs.md#documentation
+
+
 # Storage Memory Usage Benchmark
 
 <!---todo: link storage to storage design doc-->

Documentation/v2/branch_management.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Branch Management
 
 ## Guide

Documentation/v2/clustering.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Clustering Guide
 
 ## Overview

Documentation/v2/configuration.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Configuration Flags
 
 etcd is configurable through command-line flags and environment variables. Options set on the command line take precedence over those from the environment.

Documentation/v2/dev/release.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../../docs.md#documentation
+
+
 # etcd release guide
 
 The guide talks about how to release a new version of etcd.

Documentation/v2/discovery_protocol.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Discovery Service Protocol
 
 Discovery service protocol helps new etcd member to discover all other members in cluster bootstrap phase using a shared discovery URL.

Documentation/v2/docker_guide.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Running etcd under Docker
 
 The following guide will show you how to run etcd under Docker using the [static bootstrap process](clustering.md#static).

Documentation/v2/errorcode.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Error Code
 ======
 

Documentation/v2/faq.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # FAQ
 
 ## 1) Why can an etcd client read an old version of data when a majority of the etcd cluster members are down?

Documentation/v2/glossary.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Glossary
 
 This document defines the various terms used in etcd documentation, command line and source code.

Documentation/v2/implementation-faq.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # FAQ
 
 ## Initial Bootstrapping UX

Documentation/v2/internal-protocol-versioning.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Versioning
 
 Goal: We want to be able to upgrade an individual peer in an etcd cluster to a newer version of etcd.

Documentation/v2/libraries-and-tools.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Libraries and Tools
 
 **Tools**

Documentation/v2/members_api.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Members API
 
 * [List members](#list-members)

Documentation/v2/metrics.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Metrics
 
 etcd uses [Prometheus][prometheus] for metrics reporting. The metrics can be used for real-time monitoring and debugging. etcd does not persist its metrics; if a member restarts, the metrics will be reset.

Documentation/v2/other_apis.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Miscellaneous APIs
 
 * [Getting the etcd version](#getting-the-etcd-version)

Documentation/v2/platforms/freebsd.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../../docs.md#documentation
+
+
 # FreeBSD
 
 Starting with version 0.1.2 both etcd and etcdctl have been ported to FreeBSD and can

Documentation/v2/production-users.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Production Users
 
 This document tracks people and use cases for etcd in production. By creating a list of production use cases we hope to build a community of advisors that we can reach out to with experience using various etcd applications, operation environments, and cluster sizes. The etcd development team may reach out periodically to check-in on your experience and update this list.

Documentation/v2/proxy.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Proxy
 
 etcd can run as a transparent proxy. Doing so allows for easy discovery of etcd within your infrastructure, since it can run on each machine as a local service. In this mode, etcd acts as a reverse proxy and forwards client requests to an active etcd cluster. The etcd proxy does not participate in the consensus replication of the etcd cluster, thus it neither increases the resilience nor decreases the write performance of the etcd cluster.

Documentation/v2/reporting_bugs.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Reporting Bugs
 
 If you find bugs or documentation mistakes in the etcd project, please let us know by [opening an issue][etcd-issue]. We treat bugs and mistakes very seriously and believe no issue is too small. Before creating a bug report, please check that an issue reporting the same problem does not already exist.

Documentation/v2/rfc/v3api.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../../docs.md#documentation
+
+
 # Overview
 
 The etcd v3 API is designed to give users a more efficient and cleaner abstraction compared to etcd v2. There are a number of semantic and protocol changes in this new API. For an overview [see Xiang Li's video](https://youtu.be/J5AioGtEPeQ?t=211).

Documentation/v2/runtime-configuration.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Runtime Reconfiguration
 
 etcd comes with support for incremental runtime reconfiguration, which allows users to update the membership of the cluster at run time.

Documentation/v2/runtime-reconf-design.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Design of Runtime Reconfiguration
 
 Runtime reconfiguration is one of the hardest and most error prone features in a distributed system, especially in a consensus based system like etcd.

Documentation/v2/security.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Security Model
 
 etcd supports SSL/TLS as well as authentication through client certificates, both for clients to server as well as peer (server to server / cluster) communication.

Documentation/v2/tuning.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Tuning
 
 The default settings in etcd should work well for installations on a local network where the average network latency is low.

Documentation/v2/upgrade_2_1.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Upgrade etcd to 2.1
 
 In the general case, upgrading from etcd 2.0 to 2.1 can be a zero-downtime, rolling upgrade:

Documentation/v2/upgrade_2_2.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 # Upgrade etcd from 2.1 to 2.2
 
 In the general case, upgrading from etcd 2.1 to 2.2 can be a zero-downtime, rolling upgrade:

Documentation/v2/upgrade_2_3.md

@@ -1,3 +1,8 @@
+**This is the documentation for etcd2 releases. Read [etcd3 doc][v3-docs] for etcd3 releases.**
+
+[v3-docs]: ../docs.md#documentation
+
+
 ## Upgrade etcd from 2.2 to 2.3
 
 In the general case, upgrading from etcd 2.2 to 2.3 can be a zero-downtime, rolling upgrade:

bill-of-materials.json

@@ -1,207 +1,379 @@
 [
 	{
 		"project": "bitbucket.org/ww/goautoneg",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License",
+		"licenses": [
-		"confidence": 1
-	},
 			{
-		"project": "github.com/beorn7/perks/quantile",
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
-		"license": "MIT License",
-		"confidence": 0.989
-	},
-	{
-		"project": "github.com/bgentry/speakeasy",
-		"license": "MIT License",
-		"confidence": 0.944
-	},
-	{
-		"project": "github.com/boltdb/bolt",
-		"license": "MIT License",
-		"confidence": 1
-	},
-	{
-		"project": "github.com/cockroachdb/cmux",
-		"license": "Apache License 2.0",
-		"confidence": 1
-	},
-	{
-		"project": "github.com/coreos/etcd",
-		"license": "Apache License 2.0",
-		"confidence": 1
-	},
-	{
-		"project": "github.com/coreos/go-semver/semver",
-		"license": "Apache License 2.0",
-		"confidence": 1
-	},
-	{
-		"project": "github.com/coreos/go-systemd",
-		"license": "Apache License 2.0",
-		"confidence": 0.997
-	},
-	{
-		"project": "github.com/coreos/pkg",
-		"license": "Apache License 2.0",
-		"confidence": 1
-	},
-	{
-		"project": "github.com/cpuguy83/go-md2man/md2man",
-		"license": "MIT License",
-		"confidence": 1
-	},
-	{
-		"project": "github.com/dgrijalva/jwt-go",
-		"license": "MIT License",
-		"confidence": 0.989
-	},
-	{
-		"project": "github.com/dustin/go-humanize",
-		"license": "MIT License",
-		"confidence": 0.969
-	},
-	{
-		"project": "github.com/ghodss/yaml",
-		"license": "MIT License and BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 1
-	},
-	{
-		"project": "github.com/gogo/protobuf/proto",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 0.909
-	},
-	{
-		"project": "github.com/golang/groupcache/lru",
-		"license": "Apache License 2.0",
-		"confidence": 0.997
-	},
-	{
-		"project": "github.com/golang/protobuf",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 0.92
-	},
-	{
-		"project": "github.com/google/btree",
-		"license": "Apache License 2.0",
-		"confidence": 1
-	},
-	{
-		"project": "github.com/grpc-ecosystem/go-grpc-prometheus",
-		"license": "Apache License 2.0",
-		"confidence": 1
-	},
-	{
-		"project": "github.com/grpc-ecosystem/grpc-gateway",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 0.979
-	},
-	{
-		"project": "github.com/inconshreveable/mousetrap",
-		"license": "Apache License 2.0",
-		"confidence": 1
-	},
-	{
-		"project": "github.com/jonboulle/clockwork",
-		"license": "Apache License 2.0",
-		"confidence": 1
-	},
-	{
-		"project": "github.com/mattn/go-runewidth",
-		"license": "MIT License",
-		"confidence": 1
-	},
-	{
-		"project": "github.com/matttproud/golang_protobuf_extensions/pbutil",
-		"license": "Apache License 2.0",
-		"confidence": 1
-	},
-	{
-		"project": "github.com/olekukonko/tablewriter",
-		"license": "MIT License",
-		"confidence": 0.989
-	},
-	{
-		"project": "github.com/prometheus/client_golang/prometheus",
-		"license": "Apache License 2.0",
-		"confidence": 1
-	},
-	{
-		"project": "github.com/prometheus/client_model/go",
-		"license": "Apache License 2.0",
-		"confidence": 1
-	},
-	{
-		"project": "github.com/prometheus/common",
-		"license": "Apache License 2.0",
-		"confidence": 1
-	},
-	{
-		"project": "github.com/prometheus/procfs",
-		"license": "Apache License 2.0",
-		"confidence": 1
-	},
-	{
-		"project": "github.com/russross/blackfriday",
-		"license": "BSD 2-clause \"Simplified\" License",
-		"confidence": 0.963
-	},
-	{
-		"project": "github.com/spf13/cobra",
-		"license": "Apache License 2.0",
-		"confidence": 0.957
-	},
-	{
-		"project": "github.com/spf13/pflag",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 0.966
-	},
-	{
-		"project": "github.com/ugorji/go/codec",
-		"license": "MIT License",
-		"confidence": 0.995
-	},
-	{
-		"project": "github.com/urfave/cli",
-		"license": "MIT License",
-		"confidence": 1
-	},
-	{
-		"project": "github.com/xiang90/probing",
-		"license": "MIT License",
-		"confidence": 1
-	},
-	{
-		"project": "golang.org/x/crypto",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 0.966
-	},
-	{
-		"project": "golang.org/x/net",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 0.966
-	},
-	{
-		"project": "golang.org/x/text",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 0.966
-	},
-	{
-		"project": "golang.org/x/time/rate",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 0.966
-	},
-	{
-		"project": "google.golang.org/grpc",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 0.979
-	},
-	{
-		"project": "gopkg.in/cheggaaa/pb.v1",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 0.992
-	},
-	{
-		"project": "gopkg.in/yaml.v2",
-		"license": "Apache License 2.0 and MIT License",
 				"confidence": 1
 			}
 		]
+	},
+	{
+		"project": "github.com/beorn7/perks/quantile",
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 0.9891304347826086
+			}
+		]
+	},
+	{
+		"project": "github.com/bgentry/speakeasy",
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 0.9441624365482234
+			}
+		]
+	},
+	{
+		"project": "github.com/boltdb/bolt",
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 1
+			}
+		]
+	},
+	{
+		"project": "github.com/cockroachdb/cmux",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
+	{
+		"project": "github.com/coreos/etcd",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
+	{
+		"project": "github.com/coreos/go-semver/semver",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
+	{
+		"project": "github.com/coreos/go-systemd",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 0.9966703662597114
+			}
+		]
+	},
+	{
+		"project": "github.com/coreos/pkg",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
+	{
+		"project": "github.com/cpuguy83/go-md2man/md2man",
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 1
+			}
+		]
+	},
+	{
+		"project": "github.com/dgrijalva/jwt-go",
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 0.9891304347826086
+			}
+		]
+	},
+	{
+		"project": "github.com/dustin/go-humanize",
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 0.96875
+			}
+		]
+	},
+	{
+		"project": "github.com/ghodss/yaml",
+		"licenses": [
+			{
+				"type": "MIT License and BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 1
+			}
+		]
+	},
+	{
+		"project": "github.com/gogo/protobuf/proto",
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 0.9090909090909091
+			}
+		]
+	},
+	{
+		"project": "github.com/golang/groupcache/lru",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 0.9966703662597114
+			}
+		]
+	},
+	{
+		"project": "github.com/golang/protobuf",
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 0.92
+			}
+		]
+	},
+	{
+		"project": "github.com/google/btree",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
+	{
+		"project": "github.com/grpc-ecosystem/go-grpc-prometheus",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
+	{
+		"project": "github.com/grpc-ecosystem/grpc-gateway",
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 0.979253112033195
+			}
+		]
+	},
+	{
+		"project": "github.com/inconshreveable/mousetrap",
+		"licenses": [
+			{
+				"type": "MIT License and BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 1
+			},
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
+	{
+		"project": "github.com/jonboulle/clockwork",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
+	{
+		"project": "github.com/mattn/go-runewidth",
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 1
+			}
+		]
+	},
+	{
+		"project": "github.com/matttproud/golang_protobuf_extensions/pbutil",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
+	{
+		"project": "github.com/olekukonko/tablewriter",
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 0.9891304347826086
+			}
+		]
+	},
+	{
+		"project": "github.com/prometheus/client_golang/prometheus",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
+	{
+		"project": "github.com/prometheus/client_model/go",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
+	{
+		"project": "github.com/prometheus/common",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
+	{
+		"project": "github.com/prometheus/procfs",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
+	{
+		"project": "github.com/russross/blackfriday",
+		"licenses": [
+			{
+				"type": "BSD 2-clause \"Simplified\" License",
+				"confidence": 0.9626168224299065
+			}
+		]
+	},
+	{
+		"project": "github.com/spf13/cobra",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 0.9573241061130334
+			}
+		]
+	},
+	{
+		"project": "github.com/spf13/pflag",
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 0.9663865546218487
+			}
+		]
+	},
+	{
+		"project": "github.com/ugorji/go/codec",
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 0.9946524064171123
+			}
+		]
+	},
+	{
+		"project": "github.com/urfave/cli",
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 1
+			}
+		]
+	},
+	{
+		"project": "github.com/xiang90/probing",
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 1
+			}
+		]
+	},
+	{
+		"project": "golang.org/x/crypto",
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 0.9663865546218487
+			}
+		]
+	},
+	{
+		"project": "golang.org/x/net",
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 0.9663865546218487
+			}
+		]
+	},
+	{
+		"project": "golang.org/x/text",
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 0.9663865546218487
+			}
+		]
+	},
+	{
+		"project": "golang.org/x/time/rate",
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 0.9663865546218487
+			}
+		]
+	},
+	{
+		"project": "google.golang.org/grpc",
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 0.979253112033195
+			}
+		]
+	},
+	{
+		"project": "gopkg.in/cheggaaa/pb.v1",
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 0.9916666666666667
+			}
+		]
+	},
+	{
+		"project": "gopkg.in/yaml.v2",
+		"licenses": [
+			{
+				"type": "The Unlicense",
+				"confidence": 0.35294117647058826
+			},
+			{
+				"type": "MIT License",
+				"confidence": 0.8975609756097561
+			}
+		]
+	}
+]

bill-of-materials.override.json

@@ -1,18 +1,26 @@
 [
 	{
 		"project": "bitbucket.org/ww/goautoneg",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License"
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License"
+			}
+		]
 	},
 	{
 		"project": "github.com/ghodss/yaml",
-		"license": "MIT License and BSD 3-clause \"New\" or \"Revised\" License"
+		"licenses": [
+			{
+				"type": "MIT License and BSD 3-clause \"New\" or \"Revised\" License"
+			}
+		]
 	},
 	{
 		"project": "github.com/inconshreveable/mousetrap",
-		"license": "Apache License 2.0"
+		"licenses": [
-	},
 			{
-		"project": "gopkg.in/yaml.v2",
+				"type": "Apache License 2.0"
-		"license": "Apache License 2.0 and MIT License"
+			}
+		]
 	}
 ]

e2e/v3_curl_test.go

@@ -20,6 +20,8 @@ import (
 
 	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
 	"github.com/coreos/etcd/pkg/testutil"
+
+	"github.com/grpc-ecosystem/grpc-gateway/runtime"
 )
 
 func TestV3CurlPutGetNoTLS(t *testing.T)     { testCurlPutGetGRPCGateway(t, &configNoTLS) }
@@ -111,3 +113,52 @@ func TestV3CurlWatch(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestV3CurlTxn(t *testing.T) {
+	defer testutil.AfterTest(t)
+	epc, err := newEtcdProcessCluster(&configNoTLS)
+	if err != nil {
+		t.Fatalf("could not start etcd process cluster (%v)", err)
+	}
+	defer func() {
+		if cerr := epc.Close(); err != nil {
+			t.Fatalf("error closing etcd processes (%v)", cerr)
+		}
+	}()
+
+	txn := &pb.TxnRequest{
+		Compare: []*pb.Compare{
+			{
+				Key:         []byte("foo"),
+				Result:      pb.Compare_EQUAL,
+				Target:      pb.Compare_CREATE,
+				TargetUnion: &pb.Compare_CreateRevision{0},
+			},
+		},
+		Success: []*pb.RequestOp{
+			{
+				Request: &pb.RequestOp_RequestPut{
+					RequestPut: &pb.PutRequest{
+						Key:   []byte("foo"),
+						Value: []byte("bar"),
+					},
+				},
+			},
+		},
+	}
+	m := &runtime.JSONPb{}
+	jsonDat, jerr := m.Marshal(txn)
+	if jerr != nil {
+		t.Fatal(jerr)
+	}
+	expected := `"succeeded":true,"responses":[{"response_put":{"header":{"revision":"2"}}}]`
+	if err = cURLPost(epc, cURLReq{endpoint: "/v3alpha/kv/txn", value: string(jsonDat), expected: expected}); err != nil {
+		t.Fatalf("failed txn with curl (%v)", err)
+	}
+
+	// was crashing etcd server
+	malformed := `{"compare":[{"result":0,"target":1,"key":"Zm9v","TargetUnion":null}],"success":[{"Request":{"RequestPut":{"key":"Zm9v","value":"YmFy"}}}]}`
+	if err = cURLPost(epc, cURLReq{endpoint: "/v3alpha/kv/txn", value: malformed, expected: "error"}); err != nil {
+		t.Fatalf("failed put with curl (%v)", err)
+	}
+}

embed/etcd.go

@@ -326,6 +326,9 @@ func startClientListeners(cfg *Config) (sctxs map[string]*serveCtx, err error) {
 		if sctx.l, err = net.Listen(proto, addr); err != nil {
 			return nil, err
 		}
+		// net.Listener will rewrite ipv4 0.0.0.0 to ipv6 [::], breaking
+		// hosts that disable ipv6. So, use the address given by the user.
+		sctx.addr = addr
 
 		if fdLimit, fderr := runtimeutil.FDLimit(); fderr == nil {
 			if fdLimit <= reservedInternalFDNum {

embed/serve.go

@@ -44,6 +44,7 @@ import (
 
 type serveCtx struct {
 	l        net.Listener
+	addr     string
 	secure   bool
 	insecure bool
 
@@ -160,28 +161,38 @@ func grpcHandlerFunc(grpcServer *grpc.Server, otherHandler http.Handler) http.Ha
 	})
 }
 
-type registerHandlerFunc func(context.Context, *gw.ServeMux, string, []grpc.DialOption) error
+type registerHandlerFunc func(context.Context, *gw.ServeMux, *grpc.ClientConn) error
 
 func (sctx *serveCtx) registerGateway(opts []grpc.DialOption) (*gw.ServeMux, error) {
 	ctx := sctx.ctx
-	addr := sctx.l.Addr().String()
+	conn, err := grpc.DialContext(ctx, sctx.addr, opts...)
+	if err != nil {
+		return nil, err
+	}
 	gwmux := gw.NewServeMux()
 
 	handlers := []registerHandlerFunc{
-		etcdservergw.RegisterKVHandlerFromEndpoint,
+		etcdservergw.RegisterKVHandler,
-		etcdservergw.RegisterWatchHandlerFromEndpoint,
+		etcdservergw.RegisterWatchHandler,
-		etcdservergw.RegisterLeaseHandlerFromEndpoint,
+		etcdservergw.RegisterLeaseHandler,
-		etcdservergw.RegisterClusterHandlerFromEndpoint,
+		etcdservergw.RegisterClusterHandler,
-		etcdservergw.RegisterMaintenanceHandlerFromEndpoint,
+		etcdservergw.RegisterMaintenanceHandler,
-		etcdservergw.RegisterAuthHandlerFromEndpoint,
+		etcdservergw.RegisterAuthHandler,
-		v3lockgw.RegisterLockHandlerFromEndpoint,
+		v3lockgw.RegisterLockHandler,
-		v3electiongw.RegisterElectionHandlerFromEndpoint,
+		v3electiongw.RegisterElectionHandler,
 	}
 	for _, h := range handlers {
-		if err := h(ctx, gwmux, addr, opts); err != nil {
+		if err := h(ctx, gwmux, conn); err != nil {
 			return nil, err
 		}
 	}
+	go func() {
+		<-ctx.Done()
+		if cerr := conn.Close(); cerr != nil {
+			plog.Warningf("failed to close conn to %s: %v", sctx.l.Addr().String(), cerr)
+		}
+	}()
+
 	return gwmux, nil
 }
 

etcdserver/api/v3rpc/key.go

@@ -252,8 +252,8 @@ func checkRequestOp(u *pb.RequestOp) error {
 			return checkDeleteRequest(uv.RequestDeleteRange)
 		}
 	default:
-		// empty op
+		// empty op / nil entry
-		return nil
+		return rpctypes.ErrGRPCKeyNotFound
 	}
 	return nil
 }

integration/fixtures/ca-csr.json

@@ -0,0 +1,19 @@
+{
+  "key": {
+    "algo": "rsa",
+    "size": 4096
+  },
+  "names": [
+    {
+      "O": "etcd",
+      "OU": "etcd Security",
+      "L": "San Francisco",
+      "ST": "California",
+      "C": "USA"
+    }
+  ],
+  "CN": "ca",
+  "ca": {
+    "expiry": "87600h"
+  }
+}

integration/fixtures/ca.crt

@@ -1,23 +1,33 @@
 -----BEGIN CERTIFICATE-----
-MIID2zCCAsOgAwIBAgIUZXdXtcOe421Geq9VjM35+SRJUS8wDQYJKoZIhvcNAQEL
+MIIFrjCCA5agAwIBAgIUCwleGnPMSwoODcFBty/IC/L6CUIwDQYJKoZIhvcNAQEN
-BQAwdTEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
-Ew1TYW4gRnJhbmNpc2NvMRAwDgYDVQQKEwdldGNkLWNhMQswCQYDVQQLEwJDQTEZ
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
-MBcGA1UEAxMQQXV0b2dlbmVyYXRlZCBDQTAeFw0xNjA3MDUxOTQ1MDBaFw0yMTA3
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xNzA2MTYyMDMzMDBaFw0yNzA2MTQyMDMz
-MDQxOTQ1MDBaMHUxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEW
+MDBaMG8xDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
-MBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEQMA4GA1UEChMHZXRjZC1jYTELMAkGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
-CxMCQ0ExGTAXBgNVBAMTEEF1dG9nZW5lcmF0ZWQgQ0EwggEiMA0GCSqGSIb3DQEB
+ZWN1cml0eTELMAkGA1UEAxMCY2EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK
-AQUAA4IBDwAwggEKAoIBAQDBMoRjH0ULs+0cRZWZ8BGJ7Fmf152J9uUE3/NgYV3M
+AoICAQDhp9t3WUGpaRtbM52hudffXT0V9dbl1ac4DD37MdIit2yDFsut1IxSgZ40
-4Ntu6l3IYALXT5QSHQZIz5425HP6827mwAOZ/bk6E3yzq6XR/vHzxPFLzBMzFuq/
+9FliVStAWzDhZL6nX4rpInXOEI1WV1xKXu+T8i2LcxnW4QjvKTLMpBdF6q0KzsiZ
-elQA4nb7eYHICriEFUdJo2EUg3lSD3m6Deof/NjPMgUHtuvhn1OJMezaALZiMZ0K
+CV5uNTQvIuR/hQN4ij03j75nnj/ds5TUCQfz/Mh6T/xwbHp1XUimcVnh38+q+ZE2
-9B9/1ktW4Roi6FMVFfJM5rKr9EIz6P2mFUpVHI7KSGbeuHiTPq0FLVv7wFPxRFX5
+eCmEvcdAEQ9DXj7WTDD4dN0xaJz8rvZSVWVBwuP7dtN54FJmJyRXcCuus5pUd/Lm
-Ygd/nF6bbSsE2LAx/JdY1j0LQi0WUcA/HaWYVOpFSKohO6FmshP5bX0o//wWSkg2
+n4mEEZ3DLceUM13AK/gwAS3SNHOwuH4pl6IKJ10qSUdzrB+Lt0rx2iqyodN/EMnh
-8CSbtqvSxRF/Ril7raZlX713AAZVn8+B83tpjFqOLH+7AgMBAAGjYzBhMA4GA1Ud
+kYJRWG8mv5spN/s695A3MLKk0hZ/bkys91n0hycaPFg8TwxmdXP8P/AOFQXyK4x9
-DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSlyMYprKNDkzyP
+YhvtF6mGhD/RHqdaujF/tCH34DpMVY9ObTu59R/6qG4Zr3KfqpDp5iM1LjggT4QU
-gGA5cYnEEe9Y8DAfBgNVHSMEGDAWgBSlyMYprKNDkzyPgGA5cYnEEe9Y8DANBgkq
+2JBn9zc5rAd/j3clcgfJfW5CZ8ek31HLIKPm5pa8q5l4qL7qWu0FjZTpSgUps29O
-hkiG9w0BAQsFAAOCAQEAjjZkuoBl6meveg1frQuUhWtgtN/g9JqIjhEQ7tr4H46/
+ekRhtSCFI3R8TZkWOAV5DM+FkXJACsOJT/Ds4/BFgia05dglNEkFTuSDAT6BfQjy
-cHz3ngCuJh/GKSt7MTqafP99kqtm1GBs7BcoFKwsNFxNOo/a2MV2oYe2T5ol5U6/
+bghuxYkFP3bPj8rflM9AhXsfHM5qEcSkZcSdjHqn4u2uvRnpc1/T8MVADqkpMukf
-RnmPv7yXzV1WlSC2IxFdtKEIfM859TFrWFN+NyH7yyYzjx+CzFdu6SHMwrQkETKr
+IUabqJ0Iy5SHXmqouO2ZkPG8C4ytkUuQW3WKrLNBSXRJVQ3pAQIDAQABo0IwQDAO
-R/PJrb0pV+gbeFpe/VfVyT7tFSxRTkSqwvMFNjQmbSLSiIFDNdZmPBmnWk418zoP
+BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUiZ/XuFgs
-lkUESi3OQc4Eh/yQuldDXKl7L8+Ar8DddAu4nsni9EAJWi1u5wPPaLd+3s5USr1f
+FCGDhz0eMvNuB/aMvSgwDQYJKoZIhvcNAQENBQADggIBAHHsADO+SiUi51IibgF0
-zFC3tb8o+WfNf+VSxWWPWyZXlcnB2glT+TWW40Ng1w==
+gdKMurtJq2cdC8YNjkkDeI8jgIljrEi7HgYs9l3IbfRmBd5/5DRdVn8NLkjEVXSL
+fcKfGHqJSsA7qLylfXoBUAwcwObdo0fTMBn+NEfK3zb5BndClTaQRs2XiHmEwntR
+HUcSruOsWOJs9dxYHe89odMLIZv8rhbEH1vUIKC2vTnxF8vysJfx/ob3kpWiGClO
+pwpt5sc/BkWM+zo8gVnypqZzhWkYMJj5xrz0/1Wk9I8NwJnsjCcyFB+GMwX6b0ei
+TUU2MgS3krmG8A43JwUzPs8DVkQeWvsZejZzRCqDwlTwXM9pP8zGJFV0MYpyszc6
+Fx+qM2Xso5Gyja8RgHDlgJKAtnZe/vu6ocgnRXeLzLsWYVN3on2PLwL3dXxjciL0
+y4uCuLBb9ckbG3jJd4uvc6OdKVV47xsL6qgm4knHijclhkG4DXojAmdY2g0S0ptX
+ingwbLw5YHARLrOeXCgRp23SzXdvtnzbfgbI+9YQrxet8vFWg2Y+7NP2iF2/JufU
+HcPpuVGjsLkZBj4j9tOhUMDFk8esy6dBVpJ9+4d9slY0Eg5s5+XmnnVb6+QOCEii
+Gcq4nDgM8VEJxYFX9pxpjtiwiy3KVOP5QU+H0fjYfKIAi3IUdW03vzIu/H0vPk5h
+zceob2+4yKU2W+OQNeVChUzc
 -----END CERTIFICATE-----

integration/fixtures/gencert.json

@@ -0,0 +1,13 @@
+{
+  "signing": {
+    "default": {
+        "usages": [
+          "signing",
+          "key encipherment",
+          "server auth",
+          "client auth"
+        ],
+        "expiry": "87600h"
+    }
+  }
+}

integration/fixtures/gencerts.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+if ! [[ "$0" =~ "./gencerts.sh" ]]; then
+	echo "must be run from 'fixtures'"
+	exit 255
+fi
+
+if ! which cfssl; then
+	echo "cfssl is not installed"
+	exit 255
+fi
+
+cfssl gencert --initca=true ./ca-csr.json | cfssljson --bare ./ca
+mv ca.pem ca.crt
+
+cfssl gencert \
+    --ca ./ca.crt \
+    --ca-key ./ca-key.pem \
+    --config ./gencert.json \
+    ./server-ca-csr.json | cfssljson --bare ./server
+mv server.pem server.crt
+mv server-key.pem server.key.insecure
+
+cfssl gencert --ca ./ca.crt \
+    --ca-key ./ca-key.pem \
+    --config ./gencert.json \
+    ./server-ca-csr.json 2>revoked.stderr | cfssljson --bare ./server-revoked
+mv server-revoked.pem server-revoked.crt
+mv server-revoked-key.pem server-revoked.key.insecure
+
+grep serial revoked.stderr | awk ' { print $9 } ' >revoke.txt
+cfssl gencrl revoke.txt ca.crt ca-key.pem | base64 -d >revoke.crl
+
+rm -f *.csr *.pem *.stderr *.txt

integration/fixtures/server-ca-csr.json

@@ -0,0 +1,20 @@
+{
+  "key": {
+    "algo": "rsa",
+    "size": 4096
+  },
+  "names": [
+    {
+      "O": "etcd",
+      "OU": "etcd Security",
+      "L": "San Francisco",
+      "ST": "California",
+      "C": "USA"
+    }
+  ],
+  "CN": "example.com",
+  "hosts": [
+    "127.0.0.1",
+    "localhost"
+  ]
+}

integration/fixtures/server-revoked.crt

@@ -0,0 +1,35 @@
+-----BEGIN CERTIFICATE-----
+MIIGEjCCA/qgAwIBAgIUAyLIF+/vIdTKKf1wxsU+CfQkuvAwDQYJKoZIhvcNAQEN
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xNzA2MTYyMDMzMDBaFw0yNzA2MTQyMDMz
+MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggIiMA0GCSqGSIb3DQEBAQUA
+A4ICDwAwggIKAoICAQCzZzCUS5co1BFjkyPDhtxTSfJ0bdaVjkgvM9wmf5X8pBLc
+sb3iZO2oh1Dz24CNtpHDbfiN4oVW+BF5BX/rkcr6KYl/znjrP44kodUNN3uM8doP
+cfJ/ZFujmfdjtFXCgq9j3BkGW5+6ZGF/MBOtiDLXjT6JiS/F4jljxyepfdcRhnL3
+qxOiOOy5b9h+CSwxp48ubLVEzSz5qZb7ZGI+xp2tvLuoR/ZwL1Iiq4yrR4n42Crw
+oG7HOjlLBcwtxGedSLGz0LgUTPwliWA1dSd2sL3NnLUURilihSUfTZB57RMj1Uo5
+aQXAxXPXxyQx46zQXXhO/7YgCGK7vzgCP4Lr48cn6RQ4znIoLmUejWUxN+4CCVJc
+Py0Vn+j1PynPb4YhdWlOFjHMsVFMKpNbInSe/QG78+n8yJlYpVH09xvK6i+UQLex
+RfTYtNWtBQ7B22+ebgn6IWRiEWRpgzl02qeQnT/ndkSdfpn0soAH1tV1iATP8h+3
+Fznie+vpfUzeqKVA1W2akINs3LKVeW3yV1HSsqZQApF0i6cclevUL3K5uTevlhBy
+o+xvNUTG+bOtfegGrWVysbeaOyAglFGSv2K5Z3/flOXKSqg8dKc51RKA4sRK1zCQ
+kn5aNhMXjZUFWd8k0p8BvJCVTBofMlSwik2u8rkIOZh+ompe95YGnk+iFl3X/wID
+AQABo4GcMIGZMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYI
+KwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU97hg/c0lnnI30HFa3elg
+4ahPZ1AwHwYDVR0jBBgwFoAUiZ/XuFgsFCGDhz0eMvNuB/aMvSgwGgYDVR0RBBMw
+EYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEBDQUAA4ICAQAZwuZcAxWhOb28
+pFztpMAjOyW1zjFqxjECYLbMnJpf8Yf6SxvbD6J/U691jPuaR0PRAG/dL/Lcmqgg
+AAat4YnhnDYC5zG4ty0xaYsUk1AuK7iJXnAHT4klUzXmvajTrMT1uW9Yf4wVuIH/
+6fS7PvIT1oWe8ZFN72uAsNzv5I9wIFxlS6St1blFmA9HYvBpNIBJ7RaidGTs9nsP
+I8HawmD/iKhzbXZUWfYKiQ/JVsK/l5T2WYoRWgGEo605CuqBDah890up6dN4KaUx
+1Qi6WZ+MN6uaU5AA/Lvb7sS7viPdqZfraoJFNEBU/jNEmT0WL/EK6HzjredLlfE8
+Hzvy78/EZx1WbRsuDX3MG2/vYnZiWSL6DMdi3XxbJyC30FF9bc+0H7D73nSnZ22p
+9vluEdX6jsYkOglq/l5uuwK8BqWwB4tdgXJWMCWy+aQi38wz0UY7HLdS8cj7LNZQ
+9KI05vwZ8L5W30fhzWbO4jnYXbEhFNNW0yCKI174nAJM0m+vlw8w6np77l70AsCw
+MI4m3uvOGqIDjCPsuwJ4kjcpycMDeQS5+YCrkelixa0RWwgJAXJbHDSWeoQuVXW4
+UZkpdA2j9nSe3EbUMtAfCxLthxlSs6AiYcnYm3K9FKlmj1hIDxafMPxPrYDbZ9YE
+mdixLrkAUlyB50yoiYjbdTvFzvw40A==
+-----END CERTIFICATE-----

integration/fixtures/server-revoked.key.insecure

@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKAIBAAKCAgEAs2cwlEuXKNQRY5Mjw4bcU0nydG3WlY5ILzPcJn+V/KQS3LG9
+4mTtqIdQ89uAjbaRw234jeKFVvgReQV/65HK+imJf8546z+OJKHVDTd7jPHaD3Hy
+f2Rbo5n3Y7RVwoKvY9wZBlufumRhfzATrYgy140+iYkvxeI5Y8cnqX3XEYZy96sT
+ojjsuW/YfgksMaePLmy1RM0s+amW+2RiPsadrby7qEf2cC9SIquMq0eJ+Ngq8KBu
+xzo5SwXMLcRnnUixs9C4FEz8JYlgNXUndrC9zZy1FEYpYoUlH02Qee0TI9VKOWkF
+wMVz18ckMeOs0F14Tv+2IAhiu784Aj+C6+PHJ+kUOM5yKC5lHo1lMTfuAglSXD8t
+FZ/o9T8pz2+GIXVpThYxzLFRTCqTWyJ0nv0Bu/Pp/MiZWKVR9PcbyuovlEC3sUX0
+2LTVrQUOwdtvnm4J+iFkYhFkaYM5dNqnkJ0/53ZEnX6Z9LKAB9bVdYgEz/Iftxc5
+4nvr6X1M3qilQNVtmpCDbNyylXlt8ldR0rKmUAKRdIunHJXr1C9yubk3r5YQcqPs
+bzVExvmzrX3oBq1lcrG3mjsgIJRRkr9iuWd/35TlykqoPHSnOdUSgOLEStcwkJJ+
+WjYTF42VBVnfJNKfAbyQlUwaHzJUsIpNrvK5CDmYfqJqXveWBp5PohZd1/8CAwEA
+AQKCAgAmHEujlRM9Zx9yibVVOfbf8puAxDyLdLg83sVroDraenhPTarKxyn9XRGD
+XCPI9vmsDFZ6vZ4ZxYTgspxkDIqT7fL5pYDmaI/nlEFQF3M1k8MA+PHMwiL9dB2r
+nomBUoWzrvPZ9+jMjbpwbGQhvwcd5zFbwjrVzKLoYUw5ozPm0yrlFgCLu4/+LJZO
+39/1hGTUNd+kB/n/51jdeousTkD8wVUUAIWHe2X1W3/8eqwCotksWMhvphy5pTek
+mU5xiRnG2xXfqiL+TlqTwJlri0wmu51z+xubhDFInw+L9yLTv+GOpJLGSqu7MOCt
+gHIbqV9/WK71yxI+U/av85H6Tl7l7h0k7IcYvziFRCF2OHtj/4epUIGaC4c+JUHf
+cumDb0xQMNsXPeFBqzzS1pVocHb3A35YGuy5UNqEbvA0Pa1akxtMtDjOB+asoOu7
+b2ebxZpVI0RuzJwXGm7RXopTQ34Prvb45ZYfgx50izpTXGSHNsUjcPVE9JkToECr
+s2BP8+l/1iIyaRdFrKbD0dnDs78A77x/LsdQFMwFj8yIjpMywDf7oOcKQQTitKrb
+o1a3YEFDVmp4pKjoijsmqWkgSyYoZ++rYRrSg1XN7J3fDTtkmVnAVpW0vGKp8tDc
+iBhYXMiA3xhK5MLbW1jgG+IUmbW9BqaYEUusyoW3vjBWNUOIEQKCAQEA6sxOc1Y7
+rXxoWRjz1R/vwC4nMM2KBVShiIFbbxwmkL2ZEnk5BcdQao5Cy46n6prDMpjhO2Nu
+UHuAgwYJIVL7LAU5BWLkQbgsCIT/DpNw9MqsZpz9scExLNvWCGudKsWv4nG6YtBK
+0CyDYd+iFIM8FyCBm5NKpRAmIrcHrTlX2oSasLALzfGli0Tvanx/lJ3LG1XXx0aS
+RLI7ETrIpBXW1LAEw/b8z7h1H8OLoLZqzQVD5RAF7eLEOfSEPGeSkArysltNi0r4
+KcAAjPNX5jStFw/gvnKPIOzl/Z0l2BXQNN8nY3IhMI/hzlQ5kLrazSIfYONjsscZ
+DhFjhcDh7Jsi0wKCAQEAw5pZUOvxruSVdh0yR9F7mJ8PuZ+sUGC8XqblQh2W2MKE
+jwGTd+VYFhqubz1dF30Co7lD+3NmpKU5eWfkwtZ9DH+TJP82cz/X3A7ZENceHZlI
+HDUO0AvkCVIc8GV5nE30HFy6M/fCy6ww+v51JLu6aPJxkuFc/+Z1/SGXNgrxeKiS
+b0oesIB0Siqe9KyfGzBDBfOWYZ+h9mK1G63ged34Ufe61NcMYMh0PtLQ4QMS3eZj
+GezYr1SU0goSqDPdwxMI8YtYgjObMTDTqTcItqQajkCms5vI/BQqxLDL4zpDIG/C
+q+r3htufpzmDy78sC82VGYOqJSbuRUQEJn6Pz0JCpQKCAQEAhk3NBf4qbqa9zVEP
+kDYPXn9H3YuESl5Jc5qCoYCZsqsw1rdOXbbPkdPD80mrVO+nfdai5uSOIi0yqj+W
+k2Ay2dA2+JTDebAOR+VDO6QstWkEykE8gCPArCcnO3PK179yRvXEpmb8lC0SDbbs
+sA0NHzFx0Xqj9NzS44KMOpKVZSH6ldxtNCToC0yoIeIK3AiSOBWXMp50ZiUI0mcs
+g2cWllwErc8mdu7M2BlzUb9mQdWkK29J5POUSI5L4X1hAVGx10GQjn+YidEQv48m
+FUDGTW3AoU9H/Y5kU/hCwGJh0QxgLGsI/w0eHXZh4x5ur224QyRuT8HA3CIoKkbV
+NBDdDwKCAQAYXc3qirncs5T61cBBuhLPfGEVGpL4oFRW5iuBBPaZogGpXuTj9qn2
+dvRr3xUNrAD8LEQPutx6fiqeuRaCPv3s5KeL4E8EJFvbie34bxPNVB7rhKCHwjNy
+BcydFccEdaGHNvXrK14UySqA2dn17XzXPfxv86sGJctk8R5JGSvjf/xd6LnR+u6B
+nrWWfTqExIpU33dISf5/FuTAwfRIuiII8/dONEvScqYofVKP9TYQdna58As+LCct
+0xtn4K4rct+WV2l7LrqSciar2bM4LVExj1Q0458x58E8J1n+wG9ERyfVCrwzDIDi
+8AM/Cl2OStncB87WOfdMoaByuylnBDYRAoIBAAnU2EB5rtyXtYBHikNMY7AltEBw
+DlLFtTtnzQIkDFTJzc/P41agnB9IhxSiZi8AIAx5Z1GczoOevZwg1whA8fpVhAtg
+CD7yb3fsalaKMns9+X7vrMOqs5VpMg17yeJL5gmLhoWhQcpuxJpHLjCjt6Y+p1sU
++QdVMXcPELmZ7+Oo+Wvy6tYZwwYK78N4WF6c8Bxg8CgPp+66f4H73vEM+Kp0UNiU
+Ddcu4W+ywo6zQbh/u4EgMDihiijtuGLLCX5hvkz1P+gyJTv1XlZZuLdjUlHNqLMN
+XB2TrwVfYEQ1MDa6PEqmQs6vvcOhnrMs7u2nvONMpVFYIOl2Djh7TgTEfCs=
+-----END RSA PRIVATE KEY-----

integration/fixtures/server.crt

@@ -1,24 +1,35 @@
 -----BEGIN CERTIFICATE-----
-MIID9TCCAt2gAwIBAgIUXtrXPwZLfKUJiGr6ClP3lqhOuKUwDQYJKoZIhvcNAQEL
+MIIGEjCCA/qgAwIBAgIUPXKyWW706lJ+NA6yXBEj5LotfsUwDQYJKoZIhvcNAQEN
-BQAwdTEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
-Ew1TYW4gRnJhbmNpc2NvMRAwDgYDVQQKEwdldGNkLWNhMQswCQYDVQQLEwJDQTEZ
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
-MBcGA1UEAxMQQXV0b2dlbmVyYXRlZCBDQTAeFw0xNjA3MDUxOTQ1MDBaFw0xNzA3
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xNzA2MTYyMDMzMDBaFw0yNzA2MTQyMDMz
-MDUxOTQ1MDBaMFUxFTATBgNVBAcTDHRoZSBpbnRlcm5ldDEWMBQGA1UEChMNYXV0
+MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
-b2dlbmVyYXRlZDEVMBMGA1UECxMMZXRjZCBjbHVzdGVyMQ0wCwYDVQQDEwRldGNk
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArGvOLPmy5i+1j4JitG4V
+ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggIiMA0GCSqGSIb3DQEBAQUA
-g99w125ncs2ImhqucmJe6YtSnKruaLdOx93X7lzC3k0umnvLgh643R4eYS5PvrDk
+A4ICDwAwggIKAoICAQDAVrpxGgA7iNs5Av00t0jnndty0Fy5/bc8/YxO3N0LrVdD
-vw1dSYB7BHhveFPmmWd7m7n7bXtgbcdkCmUeTbSeqvptPgyMJOQfXzfOGbEHfu7U
+jSicA/1KvStKpx38ecJwuEw68xsHQc4Q0QKZCJ1sLEGNyCEOzpR1zveRlDwUbC/O
-0raulR6KtqAatofKpRZhZgzZQpVkhdd0UTsOwqCWdX3Qe0D1MS922kX99c4UlGyD
+0RTIaH5xWo+7ZbNNJjvt84/uxmSdYi3ezBFwilPQkH3vsc5r3plfEEG1MnW6rL6s
-OTVL6tulvDBBYgHbGErFmhxdgwm4e6dFfdkPUeHczzUWnKo2sIGBvo4R/NwPIp6G
+C19R+6Vg3WlExq5qaPlmnkNso08d7gExOgeTQZ3oBHzCphWajKBhuoJAOQFzGe95
-PnebrO0VWvcQfdSqjYk3BmILl8BVL5W1/EBRLtz9mZuQgc/VC62LvsgXusC9pwXC
+kgg0ltXRPQ3LQDrswzVgev3K1jYKfbWBrQHnaL8SQNXHEfTdciwDlQdXk4KfjDE7
-3QIDAQABo4GcMIGZMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcD
+ldwXPvx28xC1ELg4Kp/vFPF8YK9t/OFL/L1OpT/YmCl8tFwcPvesBVqnOZOSwVoj
-AQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQULLrktzdBK6iLINQ7
+hRUOCfpk2TcYuRRJzKITKWMZWwzchufxdqYLSKg4aB+OD45vA5FER0hxxkmhxIVE
-hGRjQbMYXKowHwYDVR0jBBgwFoAUpcjGKayjQ5M8j4BgOXGJxBHvWPAwGgYDVR0R
+6b+eqcZU43c8k5V31oC6uAiLSOCmuqAkFiaYIBDBUwEcne8FLaLIpL0ZDOySn6O9
-BBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEBCwUAA4IBAQCI2Tp4pMYk
+wiS7+20JRECk68z0VhQpmaA9yMu1rFHoukKJT9eHnw12Yx4RQNh10mM1fRdEpGbg
-LFLzGy4e/5pwpA4x/C2zl01Sv/eC79RA5Zz1NtSF/7LCfL+KPNpNkxzPyTxWOaX5
+PrQCdawNULofc1Kd4YiR2D6u8r58ZYHlUM5cUPWsr8mjQe7Clk++QSNF3UO+T7OR
-YMuAbD49ZBQYeEyNUxKcwWEpaVlmlIUj3b21fBXQ7Nw25Uea45bNhdZcdMUOTums
+u1j/h4cg187kAo8bp8+DjOzryOgn4zRT4GpkWfW4A7ruXJ0lH1/vID/m+A+uIQID
-J1/BrA2eoEB0guTlh3E8iadbVmSf6elA9TbYLd7QTTgcb3XclYCwhV3eKdm3IEiX
+AQABo4GcMIGZMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYI
-g4q50iM6/LRz1E5C3LlQ0aNqpGroBv/9ahLVfLr06ziSRcecLJ4485MtJOxP4guA
+KwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUBT0f8B8klRcSGL9rAtGm
-1tc6qPyw2MLmAlLZfOCHKLbK3KboZI8IANmrpNyL590D9bDl9nLnHmJuitBpIVp1
+nuhDSCswHwYDVR0jBBgwFoAUiZ/XuFgsFCGDhz0eMvNuB/aMvSgwGgYDVR0RBBMw
-Hw0I8e4ZYhab
+EYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEBDQUAA4ICAQAZiECIs74lFqKo
+x1HZI1txYfPmDau3kYhVLQTjcxr7959ATx7SGyDCaVDw9/n0yxfFw8JSyYgAmUTp
+WwBaROX+1zzq07QlB4xFkfBT0HBphFvWBCifRRX0Sv8VD6Zs31fFsEvVdAYpUlZd
+H4dzYXOzk6rlyGqQRx8CQWfJGaTNupkIS/aByPRpkDpNJrkObbMv2daqovV4dEjW
+vYNUT58cyQLxFtZoKEnRLJZrDB4nJVY3M81mtDeGHf/tOZV/MOz4W74VOy1xMCkk
+nXDpv6o9V0C2kiv3UIEAW7yoCL14q3Ou8z2XA9smsVrBoJCjzP185YmmnBZztpE5
+2Z6XTc77mJYIBdmx2pQz5sRw4dc5wWALgMF15PUa1/oNt7F+BiD0RCl7Eb+/n+1U
+qBh64ey3dv+SfsljbH52ywvFV1+kCiq6g9XT70DN85faGTsuRclyqR4ekttR83Sk
+HqJIVCkuthgh6BTkbyCzlF1QxNXDMWIlr+lGdwtIW1Vd85fPMrN9t4NwOYQiEpSF
+V46pIvEejokKECcajDjU1qmzmZwK5wl4hn6hMpYaYEr7LQTJjhW02+9yfGb/YU9O
+00h90p5EjHZMMxdLl1wxGM2DuV7dOPkmrNddhAXN45n9LLplieqE7u/avw3axLBI
++kjdtdYqFRJ8ON3Vu6xjnC7fNOLsyA==
 -----END CERTIFICATE-----

integration/fixtures/server.key.insecure

@@ -1,27 +1,51 @@
 -----BEGIN RSA PRIVATE KEY-----
-MIIEowIBAAKCAQEArGvOLPmy5i+1j4JitG4Vg99w125ncs2ImhqucmJe6YtSnKru
+MIIJKQIBAAKCAgEAwFa6cRoAO4jbOQL9NLdI553bctBcuf23PP2MTtzdC61XQ40o
-aLdOx93X7lzC3k0umnvLgh643R4eYS5PvrDkvw1dSYB7BHhveFPmmWd7m7n7bXtg
+nAP9Sr0rSqcd/HnCcLhMOvMbB0HOENECmQidbCxBjcghDs6Udc73kZQ8FGwvztEU
-bcdkCmUeTbSeqvptPgyMJOQfXzfOGbEHfu7U0raulR6KtqAatofKpRZhZgzZQpVk
+yGh+cVqPu2WzTSY77fOP7sZknWIt3swRcIpT0JB977HOa96ZXxBBtTJ1uqy+rAtf
-hdd0UTsOwqCWdX3Qe0D1MS922kX99c4UlGyDOTVL6tulvDBBYgHbGErFmhxdgwm4
+UfulYN1pRMauamj5Zp5DbKNPHe4BMToHk0Gd6AR8wqYVmoygYbqCQDkBcxnveZII
-e6dFfdkPUeHczzUWnKo2sIGBvo4R/NwPIp6GPnebrO0VWvcQfdSqjYk3BmILl8BV
+NJbV0T0Ny0A67MM1YHr9ytY2Cn21ga0B52i/EkDVxxH03XIsA5UHV5OCn4wxO5Xc
-L5W1/EBRLtz9mZuQgc/VC62LvsgXusC9pwXC3QIDAQABAoIBAQCNseKz71hn9tk8
+Fz78dvMQtRC4OCqf7xTxfGCvbfzhS/y9TqU/2JgpfLRcHD73rAVapzmTksFaI4UV
-YKiBIt6nix6OzHpTTDlwe3DVK6ZYQ1jWy1o10D773YIwryatzbv41Ld/7YN6o9/P
+Dgn6ZNk3GLkUScyiEyljGVsM3Ibn8XamC0ioOGgfjg+ObwORREdIccZJocSFROm/
-eWGrkm/J2k/Jsb5nBBqHRlwBwZtBdOv9IyEx1mSObl8i+MZUOI1CKsmZH6fwdkn3
+nqnGVON3PJOVd9aAurgIi0jgprqgJBYmmCAQwVMBHJ3vBS2iyKS9GQzskp+jvcIk
-rxY76EYaDGsYvQq93oFVc+7DEMtmMtr03xm2bleEvsUH0VVqLhiAof/PCgOzja/L
+u/ttCURApOvM9FYUKZmgPcjLtaxR6LpCiU/Xh58NdmMeEUDYddJjNX0XRKRm4D60
-mPxhK0FqOmhk94JFo2l0XNMn/b2lpUhrx+xny5RD6/W/k2C1DuzBiFiNZkbPW1r1
+AnWsDVC6H3NSneGIkdg+rvK+fGWB5VDOXFD1rK/Jo0HuwpZPvkEjRd1Dvk+zkbtY
-n5QccJHpe/S3Y4WZ75yKyQdrcIz6AKSeHNNGw2mYERAOmejpVV+8OIvKY6pzyXi9
+/4eHINfO5AKPG6fPg4zs68joJ+M0U+BqZFn1uAO67lydJR9f7yA/5vgPriECAwEA
-EM/BsLaBAoGBAN+XiqHHGilsrvjLKGak2KIaPRxA7EgFKKWBv8DojpXLqgkoloDL
+AQKCAgB4Nzz86CVxEI8EyUt9oXld2xqIXpc2YNAgvNDvkbhPTh6WyCmzqgKsriPa
-1wS6uG4XE0FeJCiKZk/DpVgPSiKYkQJEFLgU8N3q8OO2cGYW8kfH/TuejWRebtgJ
+2y0w2uGfFnH+/mfMV2L2u8yF3g6Wx+qJNacD3DaKk7vFMAOFOEGBYMk+oaE2NEZV
-GC7o5CqAHjFqRbTPJBLLNlSUZP08HVIRhob3t0zkvVRdDjA1rZIM/FlxAoGBAMVp
++LDi7ZzTk0JJGZNVk0HcWWwlDTBp0YYFRPsiDNWLx1tqZ4mSDdp3Kfx//2tUac8b
-jTcimGEOhFbOvfLwFeMCFLglTzbxjSnxCLCKF5TbxcBN7iUE2wYRfexBLoP/3+rk
+/5qQQ+BuNUkrI7+Vk2cHX5/QeFi75eIcvzYNjQvJSYuTnI29ZV56e73JMOWfjCuM
-RheyRnMr4PeZ/JPQLHs80TUm9HGg8Phy+jAsIW/rF8BJ4aAExt2T4uLNsj4TXw1y
++C5cQX6hOP5bwxuv3vaNgJcKDjwuqntzm1OS/YNpEcKvCDHCixtShM41cVbW+gFd
-ckDMBLmZi0OFy4vDtwg4T2wVo55eN/oQfVNFFaotAoGAGLQ8q/08pcENYA3KS/UA
+Sn/6Cxo/krWE6eF8O2yN1AB2oW6PWDi5IYTmOFLD6f4gyWAuKd1Wdgv4qSZospic
-voBZqip+MMLpJ8g7MIxBXMmg4twqLNbYzfv3bqp8BSfqpNQN09hRB3bBASuMMgzl
+3YrGOkPqzV50xy1jtmMeJej+tMtZUG8cxLpxtnZOwPtyR2CbbMBX8LPuLkayBvnE
-oSUnK83OicpZht4YLNgq4ZB2HNXWN2Zh1qUCuLNpIpqUUxLj8HOlcBjpQ5WFw9CN
+4/6R7/V6rC41fViL2GA+f/rCi1qZ6scWM4YW0hFOWIi5bR0JKwM+s4NuMpcrrf94
-5ZGvHf7T8GNLswXrRIzMwPECgYAC5Q5WDaLQYYcdQsDUTCL2BjTJknp74sTgJZGs
+5O62PY7l1M0aAT4kupTMwPgrZD8Dk6Qq6YGxUmFFRzvBCKc+H6cG+k6yKeOpfyVP
-DQpVe3eF316rmkuf5ifDjB0jgGAHMLu6YznXPIB7AP4MKNROJlEnB2A0PljqO71h
+FvJPQjd64GOxyrtUVwXTQWcbcJ2vkfIm+TwUrLh/lyQe/WMJKaJXM3k/s4erEegP
-cXQ4EOlzP2IYl5lW7HE6RCvl7yDIsLHuM0+qbQ72uYKHlSIc875uZk7U5qrJdu5v
+H1v+zf/d+mbihycWujZHQXuYpby5qzvaA19dTWmwtku+/EylKQKCAQEA/ppr32pR
-hybPLQKBgAmswE0nM9Fnj4ue9QaDgOvp1p7peZuzywBI4+TTJ/3++5vtUrgRl9Ak
+n8NA6S4FkYDYZ4vIk6cW7EmxXrOHXQOfJiP6os1mXVaBuPa82gfoqp7ALzC/bI0K
-UVzSVvltxhFpFtNfVrZxckDwb6louumRtBrLVWJDlAakvc5eG5tky+SA2u/bdXSr
+4VeFrtXen88o4P9cp3pt4tl2qYdqkZZZtKTD0qsiY+OB1rauyol7uI/DwOAY/cuI
-8tq8c24K19Pg+OLkdZpiJqmKyyV0dVn6NNmiBmiLe2tClNsUHI47
+63woihMjHIzsWVbfwa0h89RpdrxtFzbWrakMmypWijrxBwsE+9tUrIiA5T+rs+Ok
+Z1IF6cqiisG/m7aWRtHncKvGMyMLm0c5FRxmPksUkueY9Xce2XXKIxPqs9XXN4ZR
+5PNJksgVwQ7vAWMt+e0WtFeitbGOu+BL9wT5eAVv/0le7B4G0IBr3ko5mwF+Fwpq
+hFk+Ua9QIC8c+wKCAQEAwWTb/tE4PxQLIo+vK+Xu2RP2WhJdh2p3zNoKvYAGbUgt
+5EKDy7BP2zyC0GVCV5PcuAYi0jVCx5elE4C3WxuYyoz9drskSfNtqKuP4ABn00qu
+GgTw7ZnHbTRmKD68vurdaey16buzZ5xaxWLqFy5s1vDMaRLiJqk/Kaefj3E1168Z
+68fYIWdLnAfCZZbWqjfkGTA1Xmp3uUXkdlUmj9ZO+yHRJA7gGzN6qHQNgVCjmsub
+2/UQd0gATx4fYRHeg8VIcULGYdwPZH69WtMYAH63Glg769EPR9vX0ayIPWBWzxxJ
+GHx1PNeO2O6rNYTv8AsK9whtSg1Ey0FMrKN69PP+kwKCAQEAqhJ5MJqSv8fsGld1
+vMZ2yGRlrcwV0e1AIGSIbae3rgCmsg3G7V6V9PpNGM/jqeW1t1A27soglHqkgC+5
+MMNvKS0rtWxg7wIY9BHTLEFk1vihHp7WsCcPWmBZzk2caaMPPk30I8TMR5F+ew1h
+jPbOAre3bGa8oWYotPoVXUZyjTq5gwT7HgdVbkD8KUlP/JiKkZjAl7/e2G3AKXYE
+se/OkLOdGt2oXmlAUnR8klM3XFTGL6JzsdYnJQFXiCfYySnKxCp6Q0rkda3WXchu
+T7IsEZ8w3Rx06C8KL5jWIcYouG+IPZZURBG86Lj+EFVYuPqYFGUg1z9L+/CpYmNb
+gMQ39QKCAQEAgwF60awSyRAf1uK0jC0vVlfievOT7gaebuOJgQM311Gc3NJ+Cz49
+akscZmUhwsfELdjiP7maqjA8v/BixNH198f4bzSoiXvgXfER/nDn5Ebjl3afaqTQ
+ZlanmT2eiEn7gSS7ukDPcDGHf8zYvKlTS2tFXSxQjFX8FpCZUwcirR/NlF4FN+OM
+YX8UHdNHSXmhB7sTfAjtX1FMSSi3fZp0WER5M07sWyDNR02OVHo2ycaBp0fkRpk8
+tFu4b/412NeMkkTktCfpOW9tgBgdW0B7ctowhTdSfaAcG8ofZExdoeEMVw0AMh3n
+g2ZyDkgmbVpdrChGOHKylml+sjfwM/0RBwKCAQA6mGbaRsIZpsRM//aA++p+FRA4
+U9K4S+xdAQhcAcZRr+gL16Bpk2LDB3o3L7giUldi7wgFhJFBbZV3U7kj1qx7TWf3
+63db8piTdujUbp/PnIEAVVBx+taBZxlK9mRylL/mptYVsoD6hWaNboQiYR1HA1Tg
+dEN2cTIjhcCbUgwgXc/BGxeIjyuw/StsW0m8NsGGNNnkebzTdRBznvV7+vXr7Vnb
+8ybaOR4bp1RFwBlCmSVGbcvao5A2N9iexosxrWeOt+T9E5XKPrFG/eFfRJlbmFDZ
+A3K/rqdSsdSRz9XEKvaHzblJfn0cqSVNeliWy4YdaNYv8CtDAa9HAv1EUbKL
 -----END RSA PRIVATE KEY-----

integration/metrics_test.go

@@ -0,0 +1,97 @@
+// Copyright 2017 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package integration
+
+import (
+	"context"
+	"strconv"
+	"testing"
+	"time"
+
+	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
+	"github.com/coreos/etcd/pkg/testutil"
+)
+
+// TestMetricDbSizeBoot checks that the db size metric is set on boot.
+func TestMetricDbSizeBoot(t *testing.T) {
+	defer testutil.AfterTest(t)
+	clus := NewClusterV3(t, &ClusterConfig{Size: 1})
+	defer clus.Terminate(t)
+
+	v, err := clus.Members[0].Metric("etcd_debugging_mvcc_db_total_size_in_bytes")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if v == "0" {
+		t.Fatalf("expected non-zero, got %q", v)
+	}
+}
+
+// TestMetricDbSizeDefrag checks that the db size metric is set after defrag.
+func TestMetricDbSizeDefrag(t *testing.T) {
+	defer testutil.AfterTest(t)
+	clus := NewClusterV3(t, &ClusterConfig{Size: 1})
+	defer clus.Terminate(t)
+
+	kvc := toGRPC(clus.Client(0)).KV
+	mc := toGRPC(clus.Client(0)).Maintenance
+
+	// expand the db size
+	numPuts := 10
+	putreq := &pb.PutRequest{Key: []byte("k"), Value: make([]byte, 4096)}
+	for i := 0; i < numPuts; i++ {
+		if _, err := kvc.Put(context.TODO(), putreq); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// wait for backend txn sync
+	time.Sleep(500 * time.Millisecond)
+
+	beforeDefrag, err := clus.Members[0].Metric("etcd_debugging_mvcc_db_total_size_in_bytes")
+	if err != nil {
+		t.Fatal(err)
+	}
+	bv, err := strconv.Atoi(beforeDefrag)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if expected := numPuts * len(putreq.Value); bv < expected {
+		t.Fatalf("expected db size greater than %d, got %d", expected, bv)
+	}
+
+	// clear out historical keys
+	creq := &pb.CompactionRequest{Revision: int64(numPuts), Physical: true}
+	if _, err := kvc.Compact(context.TODO(), creq); err != nil {
+		t.Fatal(err)
+	}
+
+	// defrag should give freed space back to fs
+	mc.Defragment(context.TODO(), &pb.DefragmentRequest{})
+	afterDefrag, err := clus.Members[0].Metric("etcd_debugging_mvcc_db_total_size_in_bytes")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	av, err := strconv.Atoi(afterDefrag)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if bv <= av {
+		t.Fatalf("expected less than %d, got %d after defrag", bv, av)
+	}
+}

lease/lessor.go

@@ -31,12 +31,15 @@ import (
 const (
 	// NoLease is a special LeaseID representing the absence of a lease.
 	NoLease = LeaseID(0)
+
+	forever = monotime.Time(math.MaxInt64)
 )
 
 var (
 	leaseBucketName = []byte("lease")
 
-	forever = monotime.Time(math.MaxInt64)
+	// maximum number of leases to revoke per second; configurable for tests
+	leaseRevokeRate = 1000
 
 	ErrNotPrimary    = errors.New("not a primary lessor")
 	ErrLeaseNotFound = errors.New("lease not found")
@@ -324,8 +327,53 @@ func (le *lessor) Promote(extend time.Duration) {
 	for _, l := range le.leaseMap {
 		l.refresh(extend)
 	}
+
+	if len(le.leaseMap) < leaseRevokeRate {
+		// no possibility of lease pile-up
+		return
 	}
 
+	// adjust expiries in case of overlap
+	leases := make([]*Lease, 0, len(le.leaseMap))
+	for _, l := range le.leaseMap {
+		leases = append(leases, l)
+	}
+	sort.Sort(leasesByExpiry(leases))
+
+	baseWindow := leases[0].Remaining()
+	nextWindow := baseWindow + time.Second
+	expires := 0
+	// have fewer expires than the total revoke rate so piled up leases
+	// don't consume the entire revoke limit
+	targetExpiresPerSecond := (3 * leaseRevokeRate) / 4
+	for _, l := range leases {
+		remaining := l.Remaining()
+		if remaining > nextWindow {
+			baseWindow = remaining
+			nextWindow = baseWindow + time.Second
+			expires = 1
+			continue
+		}
+		expires++
+		if expires <= targetExpiresPerSecond {
+			continue
+		}
+		rateDelay := float64(time.Second) * (float64(expires) / float64(targetExpiresPerSecond))
+		// If leases are extended by n seconds, leases n seconds ahead of the
+		// base window should be extended by only one second.
+		rateDelay -= float64(remaining - baseWindow)
+		delay := time.Duration(rateDelay)
+		nextWindow = baseWindow + delay
+		l.refresh(delay + extend)
+	}
+}
+
+type leasesByExpiry []*Lease
+
+func (le leasesByExpiry) Len() int           { return len(le) }
+func (le leasesByExpiry) Less(i, j int) bool { return le[i].Remaining() < le[j].Remaining() }
+func (le leasesByExpiry) Swap(i, j int)      { le[i], le[j] = le[j], le[i] }
+
 func (le *lessor) Demote() {
 	le.mu.Lock()
 	defer le.mu.Unlock()
@@ -422,6 +470,10 @@ func (le *lessor) runLoop() {
 		le.mu.Unlock()
 
 		if len(ls) != 0 {
+			// rate limit
+			if len(ls) > leaseRevokeRate/2 {
+				ls = ls[:leaseRevokeRate/2]
+			}
 			select {
 			case <-le.stopC:
 				return

lease/lessor_test.go

@@ -42,6 +42,7 @@ func TestLessorGrant(t *testing.T) {
 	defer be.Close()
 
 	le := newLessor(be, minLeaseTTL)
+	defer le.Stop()
 	le.Promote(0)
 
 	l, err := le.Grant(1, 1)
@@ -87,6 +88,7 @@ func TestLeaseConcurrentKeys(t *testing.T) {
 	defer be.Close()
 
 	le := newLessor(be, minLeaseTTL)
+	defer le.Stop()
 	le.SetRangeDeleter(func() TxnDelete { return newFakeDeleter(be) })
 
 	// grant a lease with long term (100 seconds) to
@@ -134,6 +136,7 @@ func TestLessorRevoke(t *testing.T) {
 	defer be.Close()
 
 	le := newLessor(be, minLeaseTTL)
+	defer le.Stop()
 	var fd *fakeDeleter
 	le.SetRangeDeleter(func() TxnDelete {
 		fd = newFakeDeleter(be)
@@ -185,6 +188,7 @@ func TestLessorRenew(t *testing.T) {
 	defer os.RemoveAll(dir)
 
 	le := newLessor(be, minLeaseTTL)
+	defer le.Stop()
 	le.Promote(0)
 
 	l, err := le.Grant(1, minLeaseTTL)
@@ -210,12 +214,66 @@ func TestLessorRenew(t *testing.T) {
 	}
 }
 
+// TestLessorRenewExtendPileup ensures Lessor extends leases on promotion if too many
+// expire at the same time.
+func TestLessorRenewExtendPileup(t *testing.T) {
+	oldRevokeRate := leaseRevokeRate
+	defer func() { leaseRevokeRate = oldRevokeRate }()
+	leaseRevokeRate = 10
+
+	dir, be := NewTestBackend(t)
+	defer os.RemoveAll(dir)
+
+	le := newLessor(be, minLeaseTTL)
+	ttl := int64(10)
+	for i := 1; i <= leaseRevokeRate*10; i++ {
+		if _, err := le.Grant(LeaseID(2*i), ttl); err != nil {
+			t.Fatal(err)
+		}
+		// ttls that overlap spillover for ttl=10
+		if _, err := le.Grant(LeaseID(2*i+1), ttl+1); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// simulate stop and recovery
+	le.Stop()
+	be.Close()
+	bcfg := backend.DefaultBackendConfig()
+	bcfg.Path = filepath.Join(dir, "be")
+	be = backend.New(bcfg)
+	defer be.Close()
+	le = newLessor(be, minLeaseTTL)
+	defer le.Stop()
+
+	// extend after recovery should extend expiration on lease pile-up
+	le.Promote(0)
+
+	windowCounts := make(map[int64]int)
+	for _, l := range le.leaseMap {
+		// round up slightly for baseline ttl
+		s := int64(l.Remaining().Seconds() + 0.1)
+		windowCounts[s]++
+	}
+
+	for i := ttl; i < ttl+20; i++ {
+		c := windowCounts[i]
+		if c > leaseRevokeRate {
+			t.Errorf("expected at most %d expiring at %ds, got %d", leaseRevokeRate, i, c)
+		}
+		if c < leaseRevokeRate/2 {
+			t.Errorf("expected at least %d expiring at %ds, got %d", leaseRevokeRate/2, i, c)
+		}
+	}
+}
+
 func TestLessorDetach(t *testing.T) {
 	dir, be := NewTestBackend(t)
 	defer os.RemoveAll(dir)
 	defer be.Close()
 
 	le := newLessor(be, minLeaseTTL)
+	defer le.Stop()
 	le.SetRangeDeleter(func() TxnDelete { return newFakeDeleter(be) })
 
 	// grant a lease with long term (100 seconds) to
@@ -255,6 +313,7 @@ func TestLessorRecover(t *testing.T) {
 	defer be.Close()
 
 	le := newLessor(be, minLeaseTTL)
+	defer le.Stop()
 	l1, err1 := le.Grant(1, 10)
 	l2, err2 := le.Grant(2, 20)
 	if err1 != nil || err2 != nil {
@@ -263,6 +322,7 @@ func TestLessorRecover(t *testing.T) {
 
 	// Create a new lessor with the same backend
 	nle := newLessor(be, minLeaseTTL)
+	defer nle.Stop()
 	nl1 := nle.Lookup(l1.ID)
 	if nl1 == nil || nl1.ttl != l1.ttl {
 		t.Errorf("nl1 = %v, want nl1.ttl= %d", nl1.ttl, l1.ttl)

mvcc/index.go

@@ -29,7 +29,9 @@ type index interface {
 	RangeSince(key, end []byte, rev int64) []revision
 	Compact(rev int64) map[revision]struct{}
 	Equal(b index) bool
+
 	Insert(ki *keyIndex)
+	KeyIndex(ki *keyIndex) *keyIndex
 }
 
 type treeIndex struct {
@@ -60,18 +62,27 @@ func (ti *treeIndex) Put(key []byte, rev revision) {
 
 func (ti *treeIndex) Get(key []byte, atRev int64) (modified, created revision, ver int64, err error) {
 	keyi := &keyIndex{key: key}
-
 	ti.RLock()
 	defer ti.RUnlock()
-	item := ti.tree.Get(keyi)
+	if keyi = ti.keyIndex(keyi); keyi == nil {
-	if item == nil {
 		return revision{}, revision{}, 0, ErrRevisionNotFound
 	}
-
-	keyi = item.(*keyIndex)
 	return keyi.get(atRev)
 }
 
+func (ti *treeIndex) KeyIndex(keyi *keyIndex) *keyIndex {
+	ti.RLock()
+	defer ti.RUnlock()
+	return ti.keyIndex(keyi)
+}
+
+func (ti *treeIndex) keyIndex(keyi *keyIndex) *keyIndex {
+	if item := ti.tree.Get(keyi); item != nil {
+		return item.(*keyIndex)
+	}
+	return nil
+}
+
 func (ti *treeIndex) Range(key, end []byte, atRev int64) (keys [][]byte, revs []revision) {
 	if end == nil {
 		rev, _, _, err := ti.Get(key, atRev)

mvcc/kvstore.go

@@ -52,10 +52,10 @@ const (
 	markedRevBytesLen      = revBytesLen + 1
 	markBytePosition       = markedRevBytesLen - 1
 	markTombstone     byte = 't'
-
-	restoreChunkKeys = 10000
 )
 
+var restoreChunkKeys = 10000 // non-const for testing
+
 // ConsistentIndexGetter is an interface that wraps the Get method.
 // Consistent index is the offset of an entry in a consistent replicated log.
 type ConsistentIndexGetter interface {
@@ -245,6 +245,11 @@ func (s *store) Restore(b backend.Backend) error {
 }
 
 func (s *store) restore() error {
+	reportDbTotalSizeInBytesMu.Lock()
+	b := s.b
+	reportDbTotalSizeInBytes = func() float64 { return float64(b.Size()) }
+	reportDbTotalSizeInBytesMu.Unlock()
+
 	min, max := newRevBytes(), newRevBytes()
 	revToBytes(revision{main: 1}, min)
 	revToBytes(revision{main: math.MaxInt64, sub: math.MaxInt64}, max)
@@ -254,6 +259,7 @@ func (s *store) restore() error {
 	// restore index
 	tx := s.b.BatchTx()
 	tx.Lock()
+
 	_, finishedCompactBytes := tx.UnsafeRange(metaBucketName, finishedCompactKeyName, nil, 0)
 	if len(finishedCompactBytes) != 0 {
 		s.compactMainRev = bytesToRev(finishedCompactBytes[0]).main
@@ -266,23 +272,15 @@ func (s *store) restore() error {
 	}
 
 	// index keys concurrently as they're loaded in from tx
-	unorderedc, donec := make(chan map[string]*keyIndex), make(chan struct{})
+	rkvc, revc := restoreIntoIndex(s.kvindex)
-	go func() {
-		defer close(donec)
-		for unordered := range unorderedc {
-			// restore the tree index from the unordered index.
-			for _, v := range unordered {
-				s.kvindex.Insert(v)
-			}
-		}
-	}()
 	for {
-		keys, vals := tx.UnsafeRange(keyBucketName, min, max, restoreChunkKeys)
+		keys, vals := tx.UnsafeRange(keyBucketName, min, max, int64(restoreChunkKeys))
 		if len(keys) == 0 {
 			break
 		}
-		// unbuffered so keys don't pile up in memory
+		// rkvc blocks if the total pending keys exceeds the restore
-		unorderedc <- s.restoreChunk(keys, vals, keyToLease)
+		// chunk size to keep keys from consuming too much memory.
+		restoreChunk(rkvc, keys, vals, keyToLease)
 		if len(keys) < restoreChunkKeys {
 			// partial set implies final set
 			break
@@ -292,8 +290,8 @@ func (s *store) restore() error {
 		newMin.sub++
 		revToBytes(newMin, min)
 	}
-	close(unorderedc)
+	close(rkvc)
-	<-donec
+	s.currentRev = <-revc
 
 	// keys in the range [compacted revision -N, compaction] might all be deleted due to compaction.
 	// the correct revision should be set to compaction revision in the case, not the largest revision
@@ -325,39 +323,74 @@ func (s *store) restore() error {
 	return nil
 }
 
-func (s *store) restoreChunk(keys, vals [][]byte, keyToLease map[string]lease.LeaseID) map[string]*keyIndex {
+type revKeyValue struct {
-	// assume half of keys are overwrites
+	key  []byte
-	unordered := make(map[string]*keyIndex, len(keys)/2)
+	kv   mvccpb.KeyValue
-	for i, key := range keys {
+	kstr string
-		var kv mvccpb.KeyValue
-		if err := kv.Unmarshal(vals[i]); err != nil {
-			plog.Fatalf("cannot unmarshal event: %v", err)
 }
-		rev := bytesToRev(key[:revBytesLen])
+
-		s.currentRev = rev.main
+func restoreIntoIndex(idx index) (chan<- revKeyValue, <-chan int64) {
-		kstr := string(kv.Key)
+	rkvc, revc := make(chan revKeyValue, restoreChunkKeys), make(chan int64, 1)
-		if isTombstone(key) {
+	go func() {
-			if ki, ok := unordered[kstr]; ok {
+		currentRev := int64(1)
+		defer func() { revc <- currentRev }()
+		// restore the tree index from streaming the unordered index.
+		kiCache := make(map[string]*keyIndex, restoreChunkKeys)
+		for rkv := range rkvc {
+			ki, ok := kiCache[rkv.kstr]
+			// purge kiCache if many keys but still missing in the cache
+			if !ok && len(kiCache) >= restoreChunkKeys {
+				i := 10
+				for k := range kiCache {
+					delete(kiCache, k)
+					if i--; i == 0 {
+						break
+					}
+				}
+			}
+			// cache miss, fetch from tree index if there
+			if !ok {
+				ki = &keyIndex{key: rkv.kv.Key}
+				if idxKey := idx.KeyIndex(ki); idxKey != nil {
+					kiCache[rkv.kstr], ki = idxKey, idxKey
+					ok = true
+				}
+			}
+			rev := bytesToRev(rkv.key)
+			currentRev = rev.main
+			if ok {
+				if isTombstone(rkv.key) {
 					ki.tombstone(rev.main, rev.sub)
-			}
-			delete(keyToLease, kstr)
 					continue
 				}
-		if ki, ok := unordered[kstr]; ok {
 				ki.put(rev.main, rev.sub)
+			} else if !isTombstone(rkv.key) {
+				ki.restore(revision{rkv.kv.CreateRevision, 0}, rev, rkv.kv.Version)
+				idx.Insert(ki)
+				kiCache[rkv.kstr] = ki
+			}
+		}
+	}()
+	return rkvc, revc
+}
+
+func restoreChunk(kvc chan<- revKeyValue, keys, vals [][]byte, keyToLease map[string]lease.LeaseID) {
+	for i, key := range keys {
+		rkv := revKeyValue{key: key}
+		if err := rkv.kv.Unmarshal(vals[i]); err != nil {
+			plog.Fatalf("cannot unmarshal event: %v", err)
+		}
+		rkv.kstr = string(rkv.kv.Key)
+		if isTombstone(key) {
+			delete(keyToLease, rkv.kstr)
+		} else if lid := lease.LeaseID(rkv.kv.Lease); lid != lease.NoLease {
+			keyToLease[rkv.kstr] = lid
 		} else {
-			ki = &keyIndex{key: kv.Key}
+			delete(keyToLease, rkv.kstr)
-			ki.restore(revision{kv.CreateRevision, 0}, rev, kv.Version)
-			unordered[kstr] = ki
 		}
-		if lid := lease.LeaseID(kv.Lease); lid != lease.NoLease {
+		kvc <- rkv
-			keyToLease[kstr] = lid
-		} else {
-			delete(keyToLease, kstr)
 	}
 }
-	return unordered
-}
 
 func (s *store) Close() error {
 	close(s.stopc)

mvcc/kvstore_test.go

@@ -17,7 +17,9 @@ package mvcc
 import (
 	"crypto/rand"
 	"encoding/binary"
+	"fmt"
 	"math"
+	mrand "math/rand"
 	"os"
 	"reflect"
 	"testing"
@@ -401,6 +403,7 @@ func TestStoreRestore(t *testing.T) {
 	}
 	ki := &keyIndex{key: []byte("foo"), modified: revision{5, 0}, generations: gens}
 	wact = []testutil.Action{
+		{"keyIndex", []interface{}{ki}},
 		{"insert", []interface{}{ki}},
 	}
 	if g := fi.Action(); !reflect.DeepEqual(g, wact) {
@@ -408,6 +411,56 @@ func TestStoreRestore(t *testing.T) {
 	}
 }
 
+func TestRestoreDelete(t *testing.T) {
+	oldChunk := restoreChunkKeys
+	restoreChunkKeys = mrand.Intn(3) + 2
+	defer func() { restoreChunkKeys = oldChunk }()
+
+	b, tmpPath := backend.NewDefaultTmpBackend()
+	s := NewStore(b, &lease.FakeLessor{}, nil)
+	defer os.Remove(tmpPath)
+
+	keys := make(map[string]struct{})
+	for i := 0; i < 20; i++ {
+		ks := fmt.Sprintf("foo-%d", i)
+		k := []byte(ks)
+		s.Put(k, []byte("bar"), lease.NoLease)
+		keys[ks] = struct{}{}
+		switch mrand.Intn(3) {
+		case 0:
+			// put random key from past via random range on map
+			ks = fmt.Sprintf("foo-%d", mrand.Intn(i+1))
+			s.Put([]byte(ks), []byte("baz"), lease.NoLease)
+			keys[ks] = struct{}{}
+		case 1:
+			// delete random key via random range on map
+			for k := range keys {
+				s.DeleteRange([]byte(k), nil)
+				delete(keys, k)
+				break
+			}
+		}
+	}
+	s.Close()
+
+	s = NewStore(b, &lease.FakeLessor{}, nil)
+	defer s.Close()
+	for i := 0; i < 20; i++ {
+		ks := fmt.Sprintf("foo-%d", i)
+		r, err := s.Range([]byte(ks), nil, RangeOptions{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if _, ok := keys[ks]; ok {
+			if len(r.KVs) == 0 {
+				t.Errorf("#%d: expected %q, got deleted", i, ks)
+			}
+		} else if len(r.KVs) != 0 {
+			t.Errorf("#%d: expected deleted, got %q", i, ks)
+		}
+	}
+}
+
 func TestRestoreContinueUnfinishedCompaction(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
 	s0 := NewStore(b, &lease.FakeLessor{}, nil)
@@ -646,6 +699,11 @@ func (i *fakeIndex) Insert(ki *keyIndex) {
 	i.Recorder.Record(testutil.Action{Name: "insert", Params: []interface{}{ki}})
 }
 
+func (i *fakeIndex) KeyIndex(ki *keyIndex) *keyIndex {
+	i.Recorder.Record(testutil.Action{Name: "keyIndex", Params: []interface{}{ki}})
+	return nil
+}
+
 func createBytesSlice(bytesN, sliceN int) [][]byte {
 	rs := [][]byte{}
 	for len(rs) != sliceN {

mvcc/kvstore_txn.go

@@ -105,7 +105,6 @@ func (tw *storeTxnWrite) End() {
 	if len(tw.changes) != 0 {
 		tw.s.revMu.Unlock()
 	}
-	dbTotalSize.Set(float64(tw.s.b.Size()))
 	tw.s.mu.RUnlock()
 }
 

mvcc/metrics.go

@@ -15,6 +15,8 @@
 package mvcc
 
 import (
+	"sync"
+
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -129,12 +131,21 @@ var (
 			Buckets: prometheus.ExponentialBuckets(100, 2, 14),
 		})
 
-	dbTotalSize = prometheus.NewGauge(prometheus.GaugeOpts{
+	dbTotalSize = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
 		Namespace: "etcd_debugging",
 		Subsystem: "mvcc",
 		Name:      "db_total_size_in_bytes",
 		Help:      "Total size of the underlying database in bytes.",
-	})
+	},
+		func() float64 {
+			reportDbTotalSizeInBytesMu.RLock()
+			defer reportDbTotalSizeInBytesMu.RUnlock()
+			return reportDbTotalSizeInBytes()
+		},
+	)
+	// overridden by mvcc initialization
+	reportDbTotalSizeInBytesMu sync.RWMutex
+	reportDbTotalSizeInBytes   func() float64 = func() float64 { return 0 }
 )
 
 func init() {

pkg/transport/listener_tls.go

@@ -142,7 +142,11 @@ func checkCert(ctx context.Context, cert *x509.Certificate, remoteAddr string) e
 		return herr
 	}
 	if len(cert.IPAddresses) > 0 {
-		if cerr := cert.VerifyHostname(h); cerr != nil && len(cert.DNSNames) == 0 {
+		cerr := cert.VerifyHostname(h)
+		if cerr == nil {
+			return nil
+		}
+		if len(cert.DNSNames) == 0 {
 			return cerr
 		}
 	}

scripts/build-aci

@@ -36,6 +36,7 @@ fi
 acbuild --debug begin
 
 TMPHOSTS="$(mktemp)"
+ACI_ARCH="$(go2aci ${GOARCH})"
 
 acbuildEnd() {
     rm "$TMPHOSTS"
@@ -71,7 +72,7 @@ acbuild --debug port add peer tcp 2380
 
 acbuild --debug copy "$TMPHOSTS" /etc/hosts
 
-acbuild --debug label add arch "$(go2aci ${GOARCH})"
+acbuild --debug label add arch "${ACI_ARCH}"
 
 # mkdir default data-dir
 mkdir -p .acbuild/currentaci/rootfs/var/lib/etcd
@@ -80,4 +81,4 @@ mkdir -p .acbuild/currentaci/rootfs/var/lib/etcd
 ln -s ./usr/local/bin/etcd .acbuild/currentaci/rootfs/etcd
 ln -s ./usr/local/bin/etcdctl .acbuild/currentaci/rootfs/etcdctl
 
-acbuild --debug write --overwrite $BUILDDIR/etcd-${1}-linux-${GOARCH}.aci
+acbuild --debug write --overwrite $BUILDDIR/etcd-${1}-linux-${ACI_ARCH}.aci

test

@@ -16,6 +16,11 @@ set -e
 
 source ./build
 
+# build before setting up test GOPATH
+if [[ "${PASSES}" == *"functional"* ]]; then
+	./tools/functional-tester/build
+fi
+
 # build tests with vendored dependencies
 etcd_setup_gopath
 
@@ -87,6 +92,35 @@ function integration_pass {
 	go test -timeout 1m -v ${RACE} -cpu 1,2,4 -run=Example $@ ${TEST}
 }
 
+function functional_pass {
+	for a in 1 2 3; do
+		mkdir -p ./agent-$a
+		./bin/etcd-agent -etcd-path ./bin/etcd -etcd-log-dir "./agent-$a" -port ":${a}9027" -use-root=false &
+		pid="$!"
+		agent_pids="${agent_pids} $pid"
+	done
+
+	./bin/etcd-tester \
+		-agent-endpoints "127.0.0.1:19027,127.0.0.1:29027,127.0.0.1:39027" \
+		-client-ports 12379,22379,32379 \
+		-peer-ports 12380,22380,32380 \
+		-limit 1 \
+		-schedule-cases "0 1 2 3 4 5" \
+		-exit-on-failure
+	ETCD_TESTER_EXIT_CODE=$?
+	echo "ETCD_TESTER_EXIT_CODE:" ${ETCD_TESTER_EXIT_CODE}
+
+	echo "Waiting for processes to exit"
+	kill -s TERM ${agent_pids}
+	for a in ${agent_pids}; do wait $a || true; done
+	rm -rf ./agent-*
+
+	if [[ "${ETCD_TESTER_EXIT_CODE}" -ne "0" ]]; then
+		echo "FAIL with exit code" ${ETCD_TESTER_EXIT_CODE}
+		exit ${ETCD_TESTER_EXIT_CODE}
+	fi
+}
+
 function cov_pass {
 	echo "Running code coverage..."
 	# install gocovmerge before running code coverage from github.com/wadey/gocovmerge
@@ -175,6 +209,10 @@ function release_pass {
 		# in case, we need to test against different version
 		UPGRADE_VER=$MANUAL_VER
 	fi
+	if [[ -z ${UPGRADE_VER} ]]; then
+		UPGRADE_VER="v3.2.0"
+		echo "fallback to" ${UPGRADE_VER}
+	fi
 
 	local file="etcd-$UPGRADE_VER-linux-$GOARCH.tar.gz"
 	echo "Downloading $file"
@@ -185,9 +223,8 @@ function release_pass {
 	set -e
 	case $result in
 		0)	;;
-		22)	return 0
+		*)	echo "FAIL with" ${result}
-			;;
+			exit $result
-		*)	exit $result
 			;;
 	esac
 

tools/benchmark/cmd/watch.go

@@ -15,6 +15,7 @@
 package cmd
 
 import (
+	"context"
 	"encoding/binary"
 	"fmt"
 	"math/rand"
@@ -22,11 +23,11 @@ import (
 	"sync/atomic"
 	"time"
 
-	v3 "github.com/coreos/etcd/clientv3"
+	"github.com/coreos/etcd/clientv3"
 	"github.com/coreos/etcd/pkg/report"
 
 	"github.com/spf13/cobra"
-	"golang.org/x/net/context"
+	"golang.org/x/time/rate"
 	"gopkg.in/cheggaaa/pb.v1"
 )
 
@@ -50,8 +51,8 @@ Each key is watched by (--total/--watched-key-total) watchers.
 }
 
 var (
-	watchTotalStreams int
+	watchStreams          int
-	watchTotal        int
+	watchWatchesPerStream int
 	watchedKeyTotal       int
 
 	watchPutRate  int
@@ -60,23 +61,27 @@ var (
 	watchKeySize      int
 	watchKeySpaceSize int
 	watchSeqKeys      bool
-
-	eventsTotal int
-
-	nrWatchCompleted       int32
-	nrRecvCompleted        int32
-	watchCompletedNotifier chan struct{}
-	recvCompletedNotifier  chan struct{}
 )
 
+type watchedKeys struct {
+	watched     []string
+	numWatchers map[string]int
+
+	watches []clientv3.WatchChan
+
+	// ctx to control all watches
+	ctx    context.Context
+	cancel context.CancelFunc
+}
+
 func init() {
 	RootCmd.AddCommand(watchCmd)
-	watchCmd.Flags().IntVar(&watchTotalStreams, "watchers", 10000, "Total number of watchers")
+	watchCmd.Flags().IntVar(&watchStreams, "streams", 10, "Total watch streams")
-	watchCmd.Flags().IntVar(&watchTotal, "total", 100000, "Total number of watch requests")
+	watchCmd.Flags().IntVar(&watchWatchesPerStream, "watch-per-stream", 100, "Total watchers per stream")
-	watchCmd.Flags().IntVar(&watchedKeyTotal, "watched-key-total", 10000, "Total number of keys to be watched")
+	watchCmd.Flags().IntVar(&watchedKeyTotal, "watched-key-total", 1, "Total number of keys to be watched")
 
-	watchCmd.Flags().IntVar(&watchPutRate, "put-rate", 100, "Number of keys to put per second")
+	watchCmd.Flags().IntVar(&watchPutRate, "put-rate", 0, "Number of keys to put per second")
-	watchCmd.Flags().IntVar(&watchPutTotal, "put-total", 10000, "Number of put requests")
+	watchCmd.Flags().IntVar(&watchPutTotal, "put-total", 1000, "Number of put requests")
 
 	watchCmd.Flags().IntVar(&watchKeySize, "key-size", 32, "Key size of watch request")
 	watchCmd.Flags().IntVar(&watchKeySpaceSize, "key-space-size", 1, "Maximum possible keys")
@@ -88,9 +93,76 @@ func watchFunc(cmd *cobra.Command, args []string) {
 		fmt.Fprintf(os.Stderr, "expected positive --key-space-size, got (%v)", watchKeySpaceSize)
 		os.Exit(1)
 	}
+	grpcConns := int(totalClients)
+	if totalClients > totalConns {
+		grpcConns = int(totalConns)
+	}
+	wantedConns := 1 + (watchStreams / 100)
+	if grpcConns < wantedConns {
+		fmt.Fprintf(os.Stderr, "warning: grpc limits 100 streams per client connection, have %d but need %d\n", grpcConns, wantedConns)
+	}
+	clients := mustCreateClients(totalClients, totalConns)
+	wk := newWatchedKeys()
+	benchMakeWatches(clients, wk)
+	benchPutWatches(clients, wk)
+}
 
+func benchMakeWatches(clients []*clientv3.Client, wk *watchedKeys) {
+	streams := make([]clientv3.Watcher, watchStreams)
+	for i := range streams {
+		streams[i] = clientv3.NewWatcher(clients[i%len(clients)])
+	}
+
+	keyc := make(chan string, watchStreams)
+	bar = pb.New(watchStreams * watchWatchesPerStream)
+	bar.Format("Bom !")
+	bar.Start()
+
+	r := newReport()
+	rch := r.Results()
+
+	wg.Add(len(streams) + 1)
+	wc := make(chan []clientv3.WatchChan, len(streams))
+	for _, s := range streams {
+		go func(s clientv3.Watcher) {
+			defer wg.Done()
+			var ws []clientv3.WatchChan
+			for i := 0; i < watchWatchesPerStream; i++ {
+				k := <-keyc
+				st := time.Now()
+				wch := s.Watch(wk.ctx, k)
+				rch <- report.Result{Start: st, End: time.Now()}
+				ws = append(ws, wch)
+				bar.Increment()
+			}
+			wc <- ws
+		}(s)
+	}
+	go func() {
+		defer func() {
+			close(keyc)
+			wg.Done()
+		}()
+		for i := 0; i < watchStreams*watchWatchesPerStream; i++ {
+			key := wk.watched[i%len(wk.watched)]
+			keyc <- key
+			wk.numWatchers[key]++
+		}
+	}()
+
+	rc := r.Run()
+	wg.Wait()
+	bar.Finish()
+	close(r.Results())
+	fmt.Printf("Watch creation summary:\n%s", <-rc)
+
+	for i := 0; i < len(streams); i++ {
+		wk.watches = append(wk.watches, (<-wc)...)
+	}
+}
+
+func newWatchedKeys() *watchedKeys {
 	watched := make([]string, watchedKeyTotal)
-	numWatchers := make(map[string]int)
 	for i := range watched {
 		k := make([]byte, watchKeySize)
 		if watchSeqKeys {
@@ -100,112 +172,76 @@ func watchFunc(cmd *cobra.Command, args []string) {
 		}
 		watched[i] = string(k)
 	}
-
+	ctx, cancel := context.WithCancel(context.TODO())
-	requests := make(chan string, totalClients)
+	return &watchedKeys{
-
+		watched:     watched,
-	clients := mustCreateClients(totalClients, totalConns)
+		numWatchers: make(map[string]int),
-
+		ctx:         ctx,
-	streams := make([]v3.Watcher, watchTotalStreams)
+		cancel:      cancel,
-	for i := range streams {
+	}
-		streams[i] = v3.NewWatcher(clients[i%len(clients)])
 }
 
-	// watching phase
+func benchPutWatches(clients []*clientv3.Client, wk *watchedKeys) {
-	bar = pb.New(watchTotal)
+	eventsTotal := 0
-	bar.Format("Bom !")
-	bar.Start()
-
-	atomic.StoreInt32(&nrWatchCompleted, int32(0))
-	watchCompletedNotifier = make(chan struct{})
-
-	r := report.NewReportRate("%4.4f")
-	for i := range streams {
-		go doWatch(streams[i], requests, r.Results())
-	}
-
-	go func() {
-		for i := 0; i < watchTotal; i++ {
-			key := watched[i%len(watched)]
-			requests <- key
-			numWatchers[key]++
-		}
-		close(requests)
-	}()
-
-	rc := r.Run()
-	<-watchCompletedNotifier
-	bar.Finish()
-	close(r.Results())
-	fmt.Printf("Watch creation summary:\n%s", <-rc)
-
-	// put phase
-	eventsTotal = 0
 	for i := 0; i < watchPutTotal; i++ {
-		eventsTotal += numWatchers[watched[i%len(watched)]]
+		eventsTotal += wk.numWatchers[wk.watched[i%len(wk.watched)]]
 	}
 
 	bar = pb.New(eventsTotal)
 	bar.Format("Bom !")
 	bar.Start()
 
-	atomic.StoreInt32(&nrRecvCompleted, 0)
+	r := newReport()
-	recvCompletedNotifier = make(chan struct{})
-	putreqc := make(chan v3.Op)
 
-	r = report.NewReportRate("%4.4f")
+	wg.Add(len(wk.watches))
-	for i := 0; i < watchPutTotal; i++ {
+	nrRxed := int32(eventsTotal)
-		go func(c *v3.Client) {
+	for _, w := range wk.watches {
-			for op := range putreqc {
+		go func(wc clientv3.WatchChan) {
-				if _, err := c.Do(context.TODO(), op); err != nil {
+			defer wg.Done()
-					fmt.Fprintf(os.Stderr, "failed to Put for watch benchmark: %v\n", err)
+			recvWatchChan(wc, r.Results(), &nrRxed)
-					os.Exit(1)
+			wk.cancel()
-				}
+		}(w)
-			}
-		}(clients[i%len(clients)])
 	}
 
+	putreqc := make(chan clientv3.Op, len(clients))
 	go func() {
+		defer close(putreqc)
 		for i := 0; i < watchPutTotal; i++ {
-			putreqc <- v3.OpPut(watched[i%(len(watched))], "data")
+			putreqc <- clientv3.OpPut(wk.watched[i%(len(wk.watched))], "data")
-			// TODO: use a real rate-limiter instead of sleep.
-			time.Sleep(time.Second / time.Duration(watchPutRate))
 		}
-		close(putreqc)
 	}()
 
-	rc = r.Run()
+	limit := rate.NewLimiter(rate.Limit(watchPutRate), 1)
-	<-recvCompletedNotifier
+	for _, cc := range clients {
+		go func(c *clientv3.Client) {
+			for op := range putreqc {
+				if err := limit.Wait(context.TODO()); err != nil {
+					panic(err)
+				}
+				if _, err := c.Do(context.TODO(), op); err != nil {
+					panic(err)
+				}
+			}
+		}(cc)
+	}
+
+	rc := r.Run()
+	wg.Wait()
 	bar.Finish()
 	close(r.Results())
 	fmt.Printf("Watch events received summary:\n%s", <-rc)
+
 }
 
-func doWatch(stream v3.Watcher, requests <-chan string, results chan<- report.Result) {
+func recvWatchChan(wch clientv3.WatchChan, results chan<- report.Result, nrRxed *int32) {
-	for r := range requests {
-		st := time.Now()
-		wch := stream.Watch(context.TODO(), r)
-		results <- report.Result{Start: st, End: time.Now()}
-		bar.Increment()
-		go recvWatchChan(wch, results)
-	}
-	atomic.AddInt32(&nrWatchCompleted, 1)
-	if atomic.LoadInt32(&nrWatchCompleted) == int32(watchTotalStreams) {
-		watchCompletedNotifier <- struct{}{}
-	}
-}
-
-func recvWatchChan(wch v3.WatchChan, results chan<- report.Result) {
 	for r := range wch {
 		st := time.Now()
 		for range r.Events {
 			results <- report.Result{Start: st, End: time.Now()}
 			bar.Increment()
-			atomic.AddInt32(&nrRecvCompleted, 1)
+			if atomic.AddInt32(nrRxed, -1) <= 0 {
-		}
+				return
-
+			}
-		if atomic.LoadInt32(&nrRecvCompleted) == int32(eventsTotal) {
-			recvCompletedNotifier <- struct{}{}
-			break
 		}
 	}
 }

tools/functional-tester/etcd-tester/main.go

@@ -46,6 +46,7 @@ func main() {
 	stressKeySize := flag.Uint("stress-key-size", 100, "the size of each small key written into etcd.")
 	stressKeySuffixRange := flag.Uint("stress-key-count", 250000, "the count of key range written into etcd.")
 	limit := flag.Int("limit", -1, "the limit of rounds to run failure set (-1 to run without limits).")
+	exitOnFailure := flag.Bool("exit-on-failure", false, "exit tester on first failure")
 	stressQPS := flag.Int("stress-qps", 10000, "maximum number of stresser requests per second.")
 	schedCases := flag.String("schedule-cases", "", "test case schedule")
 	consistencyCheck := flag.Bool("consistency-check", true, "true to check consistency (revision, hash)")
@@ -128,6 +129,7 @@ func main() {
 		failures:      schedule,
 		cluster:       c,
 		limit:         *limit,
+		exitOnFailure: *exitOnFailure,
 
 		scfg:         scfg,
 		stresserType: *stresserType,

tools/functional-tester/etcd-tester/tester.go

@@ -16,12 +16,14 @@ package main
 
 import (
 	"fmt"
+	"os"
 	"time"
 )
 
 type tester struct {
 	cluster       *cluster
 	limit         int
+	exitOnFailure bool
 
 	failures        []failure
 	status          Status
@@ -49,6 +51,7 @@ func (tt *tester) runLoop() {
 
 	if err := tt.resetStressCheck(); err != nil {
 		plog.Errorf("%s failed to start stresser (%v)", tt.logPrefix(), err)
+		tt.failed()
 		return
 	}
 
@@ -87,6 +90,7 @@ func (tt *tester) runLoop() {
 		if round > 0 && round%500 == 0 { // every 500 rounds
 			if err := tt.defrag(); err != nil {
 				plog.Warningf("%s functional-tester returning with error (%v)", tt.logPrefix(), err)
+				tt.failed()
 				return
 			}
 		}
@@ -209,7 +213,18 @@ func (tt *tester) logPrefix() string {
 	return prefix
 }
 
+func (tt *tester) failed() {
+	if !tt.exitOnFailure {
+		return
+	}
+	plog.Warningf("%s exiting on failure", tt.logPrefix())
+	tt.cluster.Terminate()
+	os.Exit(2)
+}
+
 func (tt *tester) cleanup() error {
+	defer tt.failed()
+
 	roundFailedTotalCounter.Inc()
 	desc := "compact/defrag"
 	if tt.status.Case != -1 {

version/version.go

@@ -26,7 +26,7 @@ import (
 var (
 	// MinClusterVersion is the min cluster version this etcd binary is compatible with.
 	MinClusterVersion = "3.0.0"
-	Version           = "3.2.0"
+	Version           = "3.2.2"
 	APIVersion        = "unknown"
 
 	// Git SHA Value will be set during build