version: bump up to 3.2.1

Signed-off-by: Gyu-Ho Lee <gyuhox@gmail.com>

.travis.yml

@@ -41,10 +41,13 @@ matrix:
 
 addons:
   apt:
+    sources:
+    - debian-sid
     packages:
     - libpcap-dev
     - libaspell-dev
     - libhunspell-dev
+    - shellcheck
 
 before_install:
  - go get -v -u github.com/chzchzchz/goword

Documentation/dev-guide/api_grpc_gateway.md

@@ -38,6 +38,15 @@ curl -L http://localhost:2379/v3alpha/kv/put \
 # {"result":{"header":{"cluster_id":"12585971608760269493","member_id":"13847567121247652255","revision":"2","raft_term":"2"},"events":[{"kv":{"key":"Zm9v","create_revision":"2","mod_revision":"2","version":"1","value":"YmFy"}}]}}
 ```
 
+Use `curl` to issue a transaction:
+
+```bash
+curl -L http://localhost:2379/v3alpha/kv/txn \
+	-X POST \
+	-d '{"compare":[{"target":"CREATE","key":"Zm9v","createRevision":"2"}],"success":[{"requestPut":{"key":"Zm9v","value":"YmFy"}}]}'
+# {"header":{"cluster_id":"12585971608760269493","member_id":"13847567121247652255","revision":"3","raft_term":"2"},"succeeded":true,"responses":[{"response_put":{"header":{"revision":"3"}}}]}
+```
+
 ## Swagger
 
 Generated [Swagger][swagger] API definitions can be found at [rpc.swagger.json][swagger-doc].

Documentation/dev-guide/api_reference_v3.md

@@ -790,6 +790,7 @@ From google paxosdb paper: Our implementation hinges around a powerful primitive
 | created | created is set to true if the response is for a create watch request. The client should record the watch_id and expect to receive events for the created watcher from the same stream. All events sent to the created watcher will attach with the same watch_id. | bool |
 | canceled | canceled is set to true if the response is for a cancel watch request. No further events will be sent to the canceled watcher. | bool |
 | compact_revision | compact_revision is set to the minimum index if a watcher tries to watch at a compacted index.  This happens when creating a watcher at a compacted revision or the watcher cannot catch up with the progress of the key-value store.  The client should treat the watcher as canceled and should not try to create any watcher with the same start_revision again. | int64 |
+| cancel_reason | cancel_reason indicates the reason for canceling the watcher. | string |
 | events |  | (slice of) mvccpb.Event |
 
 

Documentation/dev-guide/apispec/swagger/rpc.swagger.json

@@ -2179,6 +2179,10 @@
           "format": "int64",
           "description": "compact_revision is set to the minimum index if a watcher tries to watch\nat a compacted index.\n\nThis happens when creating a watcher at a compacted revision or the watcher cannot\ncatch up with the progress of the key-value store. \n\nThe client should treat the watcher as canceled and should not try to create any\nwatcher with the same start_revision again."
         },
+        "cancel_reason": {
+          "type": "string",
+          "description": "cancel_reason indicates the reason for canceling the watcher."
+        },
         "events": {
           "type": "array",
           "items": {

Documentation/faq.md

@@ -78,10 +78,26 @@ On the other hand, if the downed member is removed from cluster membership first
 
 etcd sets `strict-reconfig-check` in order to reject reconfiguration requests that would cause quorum loss. Abandoning quorum is really risky (especially when the cluster is already unhealthy). Although it may be tempting to disable quorum checking if there's quorum loss to add a new member, this could lead to full fledged cluster inconsistency. For many applications, this will make the problem even worse ("disk geometry corruption" being a candidate for most terrifying).
 
-### Why does etcd lose its leader from disk latency spikes?
+#### Why does etcd lose its leader from disk latency spikes?
 
 This is intentional; disk latency is part of leader liveness. Suppose the cluster leader takes a minute to fsync a raft log update to disk, but the etcd cluster has a one second election timeout. Even though the leader can process network messages within the election interval (e.g., send heartbeats), it's effectively unavailable because it can't commit any new proposals; it's waiting on the slow disk. If the cluster frequently loses its leader due to disk latencies, try [tuning][tuning] the disk settings or etcd time parameters.
 
+#### What does the etcd warning "request ignored (cluster ID mismatch)" mean?
+
+Every new etcd cluster generates a new cluster ID based on the initial cluster configuration and a user-provided unique `initial-cluster-token` value. By having unique cluster ID's, etcd is protected from cross-cluster interaction which could corrupt the cluster.
+
+Usually this warning happens after tearing down an old cluster, then reusing some of the peer addresses for the new cluster. If any etcd process from the old cluster is still running it will try to contact the new cluster. The new cluster will recognize a cluster ID mismatch, then ignore the request and emit this warning. This warning is often cleared by ensuring peer addresses among distinct clusters are disjoint.
+
+#### What does "mvcc: database space exceeded" mean and how do I fix it?
+
+The [multi-version concurrency control][api-mvcc] data model in etcd keeps an exact history of the keyspace. Without periodically compacting this history (e.g., by setting `--auto-compaction`), etcd will eventually exhaust its storage space. If etcd runs low on storage space, it raises a space quota alarm to protect the cluster from further writes. So long as the alarm is raised, etcd responds to write requests with the error `mvcc: database space exceeded`.
+
+To recover from the low space quota alarm:
+
+1. [Compact][maintenance-compact] etcd's history.
+2. [Defragment][maintenance-defragment] every etcd endpoint.
+3. [Disarm][maintenance-disarm] the alarm.
+
 ### Performance
 
 #### How should I benchmark etcd?
@@ -112,12 +128,6 @@ A slow network can also cause this issue. If network metrics among the etcd mach
 
 If none of the above suggestions clear the warnings, please [open an issue][new_issue] with detailed logging, monitoring, metrics and optionally workload information.
 
-#### What does the etcd warning "request ignored (cluster ID mismatch)" mean?
-
-Every new etcd cluster generates a new cluster ID based on the initial cluster configuration and a user-provided unique `initial-cluster-token` value. By having unique cluster ID's, etcd is protected from cross-cluster interaction which could corrupt the cluster.
-
-Usually this warning happens after tearing down an old cluster, then reusing some of the peer addresses for the new cluster. If any etcd process from the old cluster is still running it will try to contact the new cluster. The new cluster will recognize a cluster ID mismatch, then ignore the request and emit this warning. This warning is often cleared by ensuring peer addresses among distinct clusters are disjoint.
-
 #### What does the etcd warning "snapshotting is taking more than x seconds to finish ..." mean?
 
 etcd sends a snapshot of its complete key-value store to refresh slow followers and for [backups][backup]. Slow snapshot transfer times increase MTTR; if the cluster is ingesting data with high throughput, slow followers may livelock by needing a new snapshot before finishing receiving a snapshot. To catch slow snapshot performance, etcd warns when sending a snapshot takes more than thirty seconds and exceeds the expected transfer time for a 1Gbps connection.
@@ -135,3 +145,7 @@ etcd sends a snapshot of its complete key-value store to refresh slow followers
 [runtime reconfiguration]: https://github.com/coreos/etcd/blob/master/Documentation/op-guide/runtime-configuration.md
 [benchmark]: https://github.com/coreos/etcd/tree/master/tools/benchmark
 [benchmark-result]: https://github.com/coreos/etcd/blob/master/Documentation/op-guide/performance.md
+[api-mvcc]: learning/api.md#revisions
+[maintenance-compact]:  op-guide/maintenance.md#history-compaction
+[maintenance-defragment]: op-guide/maintenance.md#defragmentation
+[maintenance-disarm]: ../etcdctl/README.md#alarm-disarm

Documentation/op-guide/container.md

@@ -79,14 +79,16 @@ export NODE1=192.168.1.21
 Run the latest version of etcd:
 
 ```
-docker run --net=host \
-    --volume=${DATA_DIR}:/etcd-data \
-    --name etcd quay.io/coreos/etcd:latest \
-	/usr/local/bin/etcd \
-    --data-dir=/etcd-data --name node1 \
-    --initial-advertise-peer-urls http://${NODE1}:2380 --listen-peer-urls http://${NODE1}:2380 \
-    --advertise-client-urls http://${NODE1}:2379 --listen-client-urls http://${NODE1}:2379 \
-    --initial-cluster node1=http://${NODE1}:2380
+docker run \
+  -p 2379:2379 \
+  -p 2380:2380 \
+  --volume=${DATA_DIR}:/etcd-data \
+  --name etcd quay.io/coreos/etcd:latest \
+  /usr/local/bin/etcd \
+  --data-dir=/etcd-data --name node1 \
+  --initial-advertise-peer-urls http://${NODE1}:2380 --listen-peer-urls http://${NODE1}:2380 \
+  --advertise-client-urls http://${NODE1}:2379 --listen-client-urls http://${NODE1}:2379 \
+  --initial-cluster node1=http://${NODE1}:2380
 ```
 
 List the cluster member:
@@ -114,41 +116,47 @@ DATA_DIR=/var/lib/etcd
 # For node 1
 THIS_NAME=${NAME_1}
 THIS_IP=${HOST_1}
-docker run --net=host \
-    --volume=${DATA_DIR}:/etcd-data \
-    --name etcd quay.io/coreos/etcd:${ETCD_VERSION} \
-	/usr/local/bin/etcd \
-    --data-dir=/etcd-data --name ${THIS_NAME} \
-    --initial-advertise-peer-urls http://${THIS_IP}:2380 --listen-peer-urls http://${THIS_IP}:2380 \
-    --advertise-client-urls http://${THIS_IP}:2379 --listen-client-urls http://${THIS_IP}:2379 \
-    --initial-cluster ${CLUSTER} \
-    --initial-cluster-state ${CLUSTER_STATE} --initial-cluster-token ${TOKEN}
+docker run \
+  -p 2379:2379 \
+  -p 2380:2380 \
+  --volume=${DATA_DIR}:/etcd-data \
+  --name etcd quay.io/coreos/etcd:${ETCD_VERSION} \
+  /usr/local/bin/etcd \
+  --data-dir=/etcd-data --name ${THIS_NAME} \
+  --initial-advertise-peer-urls http://${THIS_IP}:2380 --listen-peer-urls http://${THIS_IP}:2380 \
+  --advertise-client-urls http://${THIS_IP}:2379 --listen-client-urls http://${THIS_IP}:2379 \
+  --initial-cluster ${CLUSTER} \
+  --initial-cluster-state ${CLUSTER_STATE} --initial-cluster-token ${TOKEN}
 
 # For node 2
 THIS_NAME=${NAME_2}
 THIS_IP=${HOST_2}
-docker run --net=host \
-    --volume=${DATA_DIR}:/etcd-data \
-    --name etcd quay.io/coreos/etcd:${ETCD_VERSION} \
-	/usr/local/bin/etcd \
-    --data-dir=/etcd-data --name ${THIS_NAME} \
-    --initial-advertise-peer-urls http://${THIS_IP}:2380 --listen-peer-urls http://${THIS_IP}:2380 \
-    --advertise-client-urls http://${THIS_IP}:2379 --listen-client-urls http://${THIS_IP}:2379 \
-    --initial-cluster ${CLUSTER} \
-    --initial-cluster-state ${CLUSTER_STATE} --initial-cluster-token ${TOKEN}
+docker run \
+  -p 2379:2379 \
+  -p 2380:2380 \
+  --volume=${DATA_DIR}:/etcd-data \
+  --name etcd quay.io/coreos/etcd:${ETCD_VERSION} \
+  /usr/local/bin/etcd \
+  --data-dir=/etcd-data --name ${THIS_NAME} \
+  --initial-advertise-peer-urls http://${THIS_IP}:2380 --listen-peer-urls http://${THIS_IP}:2380 \
+  --advertise-client-urls http://${THIS_IP}:2379 --listen-client-urls http://${THIS_IP}:2379 \
+  --initial-cluster ${CLUSTER} \
+  --initial-cluster-state ${CLUSTER_STATE} --initial-cluster-token ${TOKEN}
 
 # For node 3
 THIS_NAME=${NAME_3}
 THIS_IP=${HOST_3}
-docker run --net=host \
-    --volume=${DATA_DIR}:/etcd-data \
-    --name etcd quay.io/coreos/etcd:${ETCD_VERSION} \
-	/usr/local/bin/etcd \
-    --data-dir=/etcd-data --name ${THIS_NAME} \
-    --initial-advertise-peer-urls http://${THIS_IP}:2380 --listen-peer-urls http://${THIS_IP}:2380 \
-    --advertise-client-urls http://${THIS_IP}:2379 --listen-client-urls http://${THIS_IP}:2379 \
-    --initial-cluster ${CLUSTER} \
-    --initial-cluster-state ${CLUSTER_STATE} --initial-cluster-token ${TOKEN}
+docker run \
+  -p 2379:2379 \
+  -p 2380:2380 \
+  --volume=${DATA_DIR}:/etcd-data \
+  --name etcd quay.io/coreos/etcd:${ETCD_VERSION} \
+  /usr/local/bin/etcd \
+  --data-dir=/etcd-data --name ${THIS_NAME} \
+  --initial-advertise-peer-urls http://${THIS_IP}:2380 --listen-peer-urls http://${THIS_IP}:2380 \
+  --advertise-client-urls http://${THIS_IP}:2379 --listen-client-urls http://${THIS_IP}:2379 \
+  --initial-cluster ${CLUSTER} \
+  --initial-cluster-state ${CLUSTER_STATE} --initial-cluster-token ${TOKEN}
 ```
 
 To run `etcdctl` using API version 3:
@@ -170,17 +178,19 @@ rkt run \
   --volume etcd-ssl-certs-bundle,kind=host,source=/etc/ssl/certs/ca-certificates.crt \
   --mount volume=etcd-ssl-certs-bundle,target=/etc/ssl/certs/ca-certificates.crt \
   quay.io/coreos/etcd:latest -- --name my-name \
-	--initial-advertise-peer-urls http://localhost:2380 --listen-peer-urls http://localhost:2380 \
-	--advertise-client-urls http://localhost:2379 --listen-client-urls http://localhost:2379 \
-	--discovery https://discovery.etcd.io/c11fbcdc16972e45253491a24fcf45e1
+  --initial-advertise-peer-urls http://localhost:2380 --listen-peer-urls http://localhost:2380 \
+  --advertise-client-urls http://localhost:2379 --listen-client-urls http://localhost:2379 \
+  --discovery https://discovery.etcd.io/c11fbcdc16972e45253491a24fcf45e1
 ```
 
 ```
 docker run \
-    --volume=/etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt \
-    quay.io/coreos/etcd:latest \
-    /usr/local/bin/etcd --name my-name \
-	--initial-advertise-peer-urls http://localhost:2380 --listen-peer-urls http://localhost:2380 \
-	--advertise-client-urls http://localhost:2379 --listen-client-urls http://localhost:2379 \
-	--discovery https://discovery.etcd.io/86a9ff6c8cb8b4c4544c1a2f88f8b801
+  -p 2379:2379 \
+  -p 2380:2380 \
+  --volume=/etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt \
+  quay.io/coreos/etcd:latest \
+  /usr/local/bin/etcd --name my-name \
+  --initial-advertise-peer-urls http://localhost:2380 --listen-peer-urls http://localhost:2380 \
+  --advertise-client-urls http://localhost:2379 --listen-client-urls http://localhost:2379 \
+  --discovery https://discovery.etcd.io/86a9ff6c8cb8b4c4544c1a2f88f8b801
 ```

Documentation/op-guide/gateway.md

@@ -10,8 +10,7 @@ The gateway supports multiple etcd server endpoints and works on a simple round-
 
 Every application that accesses etcd must first have the address of an etcd cluster client endpoint. If multiple applications on the same server access the same etcd cluster, every application still needs to know the advertised client endpoints of the etcd cluster. If the etcd cluster is reconfigured to have different endpoints, every application may also need to update its endpoint list. This wide-scale reconfiguration is both tedious and error prone.
 
-etcd gateway solves this problem by serving as a stable local endpoint. A typical etcd gateway configuration has 
-each machine running a gateway listening on a local address and every etcd application connecting to its local gateway. The upshot is only the gateway needs to update its endpoints instead of updating each and every application.
+etcd gateway solves this problem by serving as a stable local endpoint. A typical etcd gateway configuration has each machine running a gateway listening on a local address and every etcd application connecting to its local gateway. The upshot is only the gateway needs to update its endpoints instead of updating each and every application.
 
 In summary, to automatically propagate cluster endpoint changes, the etcd gateway runs on every machine serving multiple applications accessing the same etcd cluster.
 
@@ -64,3 +63,43 @@ Start the etcd gateway to fetch the endpoints from the DNS SRV entries with the
 $ etcd gateway --discovery-srv=example.com
 2016-08-16 11:21:18.867350 I | tcpproxy: ready to proxy client requests to [...]
 ```
+
+## Configuration flags
+
+### etcd cluster
+
+#### --endpoints
+
+ * Comma-separated list of etcd server targets for forwarding client connections.
+ * Default: `127.0.0.1:2379`
+ * Invalid example: `https://127.0.0.1:2379` (gateway does not terminate TLS)
+
+#### --discovery-srv
+
+ * DNS domain used to bootstrap cluster endpoints through SRV recrods.
+ * Default: (not set)
+
+### Network
+
+#### --listen-addr
+
+ * Interface and port to bind for accepting client requests.
+ * Default: `127.0.0.1:23790`
+
+#### --retry-delay
+
+ * Duration of delay before retrying to connect to failed endpoints.
+ * Default: 1m0s
+ * Invalid example: "123" (expects time unit in format)
+
+### Security
+
+#### --insecure-discovery
+
+ * Accept SRV records that are insecure or susceptible to man-in-the-middle attacks.
+ * Default: `false`
+
+#### --trusted-ca-file
+
+ * Path to the client TLS CA file for the etcd cluster. Used to authenticate endpoints.
+ * Default: (not set)

Documentation/op-guide/grafana.json

@@ -114,18 +114,21 @@
                     "span": 5,
                     "stack": false,
                     "steppedLine": false,
-                    "targets": [{
-                            "expr": "sum(rate({grpc_type=\"unary\",grpc_code!=\"OK\"} [1m]))",
+                    "targets": [
+                        {
+                            "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[5m]))",
+                            "format": "time_series",
                             "intervalFactor": 2,
-                            "legendFormat": "{{instance}} RPC Rate",
+                            "legendFormat": "RPC Rate",
                             "metric": "grpc_server_started_total",
                             "refId": "A",
                             "step": 2
                         },
                         {
-                            "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\",grpc_code!=\"OK\"} [1m])) - sum(rate(grpc_server_handled_total{grpc_type=\"unary\"} [1m]))",
+                            "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[5m]))",
+                            "format": "time_series",
                             "intervalFactor": 2,
-                            "legendFormat": "{{instance}} RPC Failed Rate",
+                            "legendFormat": "RPC Failed Rate",
                             "metric": "grpc_server_handled_total",
                             "refId": "B",
                             "step": 2
@@ -197,7 +200,7 @@
                     "stack": true,
                     "steppedLine": false,
                     "targets": [{
-                            "expr": "sum(grpc_server_started_total {grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\",grpc_code!=\"OK\"}) - sum(grpc_server_handled_total {grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})",
+                            "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})",
                             "intervalFactor": 2,
                             "legendFormat": "Watch Streams",
                             "metric": "grpc_server_handled_total",
@@ -205,7 +208,7 @@
                             "step": 4
                         },
                         {
-                            "expr": "sum(grpc_server_started_total {grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total {grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})",
+                            "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})",
                             "intervalFactor": 2,
                             "legendFormat": "Lease Streams",
                             "metric": "grpc_server_handled_total",
@@ -361,7 +364,7 @@
                     "stack": false,
                     "steppedLine": true,
                     "targets": [{
-                            "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket [5m])) by (instance, le))",
+                            "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) by (instance, le))",
                             "hide": false,
                             "intervalFactor": 2,
                             "legendFormat": "{{instance}} WAL fsync",
@@ -370,7 +373,7 @@
                             "step": 4
                         },
                         {
-                            "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket [5m])) by (instance, le))",
+                            "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) by (instance, le))",
                             "intervalFactor": 2,
                             "legendFormat": "{{instance}} DB fsync",
                             "metric": "etcd_disk_backend_commit_duration_seconds_bucket",
@@ -522,7 +525,7 @@
                     "stack": true,
                     "steppedLine": false,
                     "targets": [{
-                        "expr": "rate(etcd_network_client_grpc_received_bytes_total [1m])",
+                        "expr": "rate(etcd_network_client_grpc_received_bytes_total[5m])",
                         "intervalFactor": 2,
                         "legendFormat": "{{instance}} Client Traffic In",
                         "metric": "etcd_network_client_grpc_received_bytes_total",
@@ -595,7 +598,7 @@
                     "stack": true,
                     "steppedLine": false,
                     "targets": [{
-                        "expr": "rate(etcd_network_client_grpc_sent_bytes_total [1m])",
+                        "expr": "rate(etcd_network_client_grpc_sent_bytes_total[5m])",
                         "intervalFactor": 2,
                         "legendFormat": "{{instance}} Client Traffic Out",
                         "metric": "etcd_network_client_grpc_sent_bytes_total",
@@ -668,7 +671,7 @@
                     "stack": false,
                     "steppedLine": false,
                     "targets": [{
-                        "expr": "sum(rate(etcd_network_peer_received_bytes_total [1m])) by (instance)",
+                        "expr": "sum(rate(etcd_network_peer_received_bytes_total[5m])) by (instance)",
                         "intervalFactor": 2,
                         "legendFormat": "{{instance}} Peer Traffic In",
                         "metric": "etcd_network_peer_received_bytes_total",
@@ -742,7 +745,7 @@
                     "stack": false,
                     "steppedLine": false,
                     "targets": [{
-                        "expr": "sum(rate(etcd_network_peer_sent_bytes_total [1m])) by (instance)",
+                        "expr": "sum(rate(etcd_network_peer_sent_bytes_total[5m])) by (instance)",
                         "hide": false,
                         "interval": "",
                         "intervalFactor": 2,
@@ -822,7 +825,7 @@
                     "stack": false,
                     "steppedLine": false,
                     "targets": [{
-                            "expr": "sum(rate(etcd_server_proposals_failed_total [1m]))",
+                            "expr": "sum(rate(etcd_server_proposals_failed_total[5m]))",
                             "intervalFactor": 2,
                             "legendFormat": "Proposal Failure Rate",
                             "metric": "etcd_server_proposals_failed_total",
@@ -838,7 +841,7 @@
                             "step": 2
                         },
                         {
-                            "expr": "sum(rate(etcd_server_proposals_committed_total [1m]))",
+                            "expr": "sum(rate(etcd_server_proposals_committed_total[5m]))",
                             "intervalFactor": 2,
                             "legendFormat": "Proposal Commit Rate",
                             "metric": "etcd_server_proposals_committed_total",
@@ -846,7 +849,7 @@
                             "step": 2
                         },
                         {
-                            "expr": "sum(rate(etcd_server_proposals_applied_total [1m]))",
+                            "expr": "sum(rate(etcd_server_proposals_applied_total[5m]))",
                             "intervalFactor": 2,
                             "legendFormat": "Proposal Apply Rate",
                             "refId": "D",
@@ -922,9 +925,9 @@
                     "stack": false,
                     "steppedLine": false,
                     "targets": [{
-                        "expr": "etcd_server_leader_changes_seen_total",
+                        "expr": "changes(etcd_server_leader_changes_seen_total[1d])",
                         "intervalFactor": 2,
-                        "legendFormat": "{{instance}} Leader Change Seen",
+                        "legendFormat": "{{instance}} Total Leader Elections Per Day",
                         "metric": "etcd_server_leader_changes_seen_total",
                         "refId": "A",
                         "step": 2
@@ -932,7 +935,7 @@
                     "thresholds": [],
                     "timeFrom": null,
                     "timeShift": null,
-                    "title": "Rate Leader Elections",
+                    "title": "Total Leader Elections Per Day",
                     "tooltip": {
                         "msResolution": false,
                         "shared": true,
@@ -1009,4 +1012,4 @@
     "version": 215,
     "links": [],
     "gnetId": null
-}
+}

Documentation/op-guide/performance.md

@@ -17,58 +17,54 @@ For some baseline performance numbers, we consider a three member etcd cluster w
 - Google Cloud Compute Engine
 - 3 machines of 8 vCPUs + 16GB Memory + 50GB SSD
 - 1 machine(client) of 16 vCPUs + 30GB Memory + 50GB SSD
-- Ubuntu 15.10
-- etcd v3 master branch (commit SHA d8f325d), Go 1.6.2
+- Ubuntu 17.04
+- etcd 3.2.0, go 1.8.3
 
 With this configuration, etcd can approximately write:
 
-| Number of keys | Key size in bytes | Value size in bytes | Number of connections | Number of clients | Target etcd server | Average write QPS | Average latency per request | Memory |
-|----------------|-------------------|---------------------|-----------------------|-------------------|--------------------|-------------------|-----------------------------|--------|
-| 10,000 | 8 | 256 | 1 | 1 | leader only | 525 | 2ms | 35 MB |
-| 100,000 | 8 | 256 | 100 | 1000 | leader only | 25,000 | 30ms | 35 MB |
-| 100,000 | 8 | 256 | 100 | 1000 | all members | 33,000 | 25ms | 35 MB |
+| Number of keys | Key size in bytes | Value size in bytes | Number of connections | Number of clients | Target etcd server | Average write QPS | Average latency per request | Average server RSS |
+|---------------:|------------------:|--------------------:|----------------------:|------------------:|--------------------|------------------:|----------------------------:|-------------------:|
+| 10,000 | 8 | 256 | 1 | 1 | leader only | 583 | 1.6ms | 48 MB |
+| 100,000 | 8 | 256 | 100 | 1000 | leader only | 44,341 | 22ms |  124MB |
+| 100,000 | 8 | 256 | 100 | 1000 | all members |  50,104 | 20ms |  126MB |
 
 Sample commands are:
 
-```
-# assuming IP_1 is leader, write requests to the leader
-benchmark --endpoints={IP_1} --conns=1 --clients=1 \
+```sh
+# write to leader
+benchmark --endpoints=${HOST_1} --target-leader --conns=1 --clients=1 \
     put --key-size=8 --sequential-keys --total=10000 --val-size=256
-benchmark --endpoints={IP_1} --conns=100 --clients=1000 \
+benchmark --endpoints=${HOST_1} --target-leader  --conns=100 --clients=1000 \
     put --key-size=8 --sequential-keys --total=100000 --val-size=256
 
 # write to all members
-benchmark --endpoints={IP_1},{IP_2},{IP_3} --conns=100 --clients=1000 \
+benchmark --endpoints=${HOST_1},${HOST_2},${HOST_3} --conns=100 --clients=1000 \
     put --key-size=8 --sequential-keys --total=100000 --val-size=256
 ```
 
 Linearizable read requests go through a quorum of cluster members for consensus to fetch the most recent data. Serializable read requests are cheaper than linearizable reads since they are served by any single etcd member, instead of a quorum of members, in exchange for possibly serving stale data. etcd can read: 
 
-| Number of requests | Key size in bytes | Value size in bytes | Number of connections | Number of clients | Consistency | Average latency per request | Average read QPS |
-|--------------------|-------------------|---------------------|-----------------------|-------------------|-------------|-----------------------------|------------------|
-| 10,000 | 8 | 256 | 1 | 1 | Linearizable | 2ms | 560 |
-| 10,000 | 8 | 256 | 1 | 1 | Serializable | 0.4ms | 7,500 |
-| 100,000 | 8 | 256 | 100 | 1000 | Linearizable | 15ms | 43,000 |
-| 100,000 | 8 | 256 | 100 | 1000 | Serializable | 9ms | 93,000 |
+| Number of requests | Key size in bytes | Value size in bytes | Number of connections | Number of clients | Consistency | Average read QPS | Average latency per request |
+|-------------------:|------------------:|--------------------:|----------------------:|------------------:|-------------|-----------------:|----------------------------:|
+| 10,000 | 8 | 256 | 1 | 1 | Linearizable | 1,353 | 0.7ms |
+| 10,000 | 8 | 256 | 1 | 1 | Serializable | 2,909 | 0.3ms |
+| 100,000 | 8 | 256 | 100 | 1000 | Linearizable | 141,578 | 5.5ms |
+| 100,000 | 8 | 256 | 100 | 1000 | Serializable | 185,758 | 2.2ms |
 
 Sample commands are:
 
-```
-# Linearizable read requests
-benchmark --endpoints={IP_1},{IP_2},{IP_3} --conns=1 --clients=1 \
+```sh
+# Single connection read requests
+benchmark --endpoints=${HOST_1},${HOST_2},${HOST_3} --conns=1 --clients=1 \
     range YOUR_KEY --consistency=l --total=10000
-benchmark --endpoints={IP_1},{IP_2},{IP_3} --conns=100 --clients=1000 \
-    range YOUR_KEY --consistency=l --total=100000
+benchmark --endpoints=${HOST_1},${HOST_2},${HOST_3} --conns=1 --clients=1 \
+    range YOUR_KEY --consistency=s --total=10000
 
-# Serializable read requests for each member and sum up the numbers
-for endpoint in {IP_1} {IP_2} {IP_3}; do
-    benchmark --endpoints=$endpoint --conns=1 --clients=1 \
-        range YOUR_KEY --consistency=s --total=10000
-done
-for endpoint in {IP_1} {IP_2} {IP_3}; do
-    benchmark --endpoints=$endpoint --conns=100 --clients=1000 \
-        range YOUR_KEY --consistency=s --total=100000
-done
+# Many concurrent read requests
+benchmark --endpoints=${HOST_1},${HOST_2},${HOST_3} --conns=100 --clients=1000 \
+    range YOUR_KEY --consistency=l --total=100000
+benchmark --endpoints=${HOST_1},${HOST_2},${HOST_3} --conns=100 --clients=1000 \
+    range YOUR_KEY --consistency=s --total=100000
 ```
 
-We encourage running the benchmark test when setting up an etcd cluster for the first time in a new environment to ensure the cluster achieves adequate performance; cluster latency and throughput can be sensitive to minor environment differences.
+We encourage running the benchmark test when setting up an etcd cluster for the first time in a new environment to ensure the cluster achieves adequate performance; cluster latency and throughput can be sensitive to minor environment differences.

Documentation/op-guide/security.md

@@ -16,7 +16,7 @@ etcd takes several certificate related configuration options, either through com
 
 `--key-file=<path>`: Key for the certificate. Must be unencrypted.
 
-`--client-cert-auth`: When this is set etcd will check all incoming HTTPS requests for a client certificate signed by the trusted CA, requests that don't supply a valid client certificate will fail.
+`--client-cert-auth`: When this is set etcd will check all incoming HTTPS requests for a client certificate signed by the trusted CA, requests that don't supply a valid client certificate will fail. If [authentication][auth] is enabled, the certificate provides credentials for the user name given by the Common Name field.
 
 `--trusted-ca-file=<path>`: Trusted certificate authority.
 
@@ -222,3 +222,4 @@ The certificate needs to be signed for the member's FQDN in its Subject Name, us
 [tls-setup]: ../../hack/tls-setup
 [tls-guide]: https://github.com/coreos/docs/blob/master/os/generate-self-signed-certificates.md
 [alt-name]: http://wiki.cacert.org/FAQ/subjectAltName
+[auth]: authentication.md

Documentation/upgrades/upgrade_3_0.md

@@ -10,7 +10,7 @@ Before [starting an upgrade](#upgrade-procedure), read through the rest of this
 
 #### Upgrade requirements
 
-To upgrade an existing etcd deployment to 3.0, the running cluster must be 2.3 or greater. If it's before 2.3, please upgrade to [2.3](https://github.com/coreos/etcd/releases/tag/v2.3.0) before upgrading to 3.0.
+To upgrade an existing etcd deployment to 3.0, the running cluster must be 2.3 or greater. If it's before 2.3, please upgrade to [2.3](https://github.com/coreos/etcd/releases/tag/v2.3.8) before upgrading to 3.0.
 
 Also, to ensure a smooth rolling upgrade, the running cluster must be healthy. Check the health of the cluster by using the `etcdctl cluster-health` command before proceeding.
 
@@ -52,7 +52,7 @@ member 8211f1d0f64f3269 is healthy: got healthy result from http://localhost:123
 cluster is healthy
 
 $ curl http://localhost:2379/version
-{"etcdserver":"2.3.x","etcdcluster":"2.3.0"}
+{"etcdserver":"2.3.x","etcdcluster":"2.3.8"}
 ```
 
 #### 2. Stop the existing etcd process

Documentation/upgrades/upgrade_3_1.md

@@ -10,7 +10,7 @@ Before [starting an upgrade](#upgrade-procedure), read through the rest of this
 
 #### Upgrade requirements
 
-To upgrade an existing etcd deployment to 3.1, the running cluster must be 3.0 or greater. If it's before 3.0, please upgrade to [3.0](https://github.com/coreos/etcd/releases/tag/v3.0.16) before upgrading to 3.1.
+To upgrade an existing etcd deployment to 3.1, the running cluster must be 3.0 or greater. If it's before 3.0, please [upgrade to 3.0](upgrade_3_0.md) before upgrading to 3.1.
 
 Also, to ensure a smooth rolling upgrade, the running cluster must be healthy. Check the health of the cluster by using the `etcdctl endpoint health` command before proceeding.
 

Documentation/upgrades/upgrade_3_2.md

@@ -30,11 +30,27 @@ resp.TTL == -1
 err == nil
 ```
 
+`clientv3.NewFromConfigFile` is moved to `yaml.NewConfig`.
+
+Before
+
+```go
+import "github.com/coreos/etcd/clientv3"
+clientv3.NewFromConfigFile
+```
+
+After
+
+```go
+import clientv3yaml "github.com/coreos/etcd/clientv3/yaml"
+clientv3yaml.NewConfig
+```
+
 ### Server upgrade checklists
 
 #### Upgrade requirements
 
-To upgrade an existing etcd deployment to 3.2, the running cluster must be 3.1 or greater. If it's before 3.1, please upgrade to [3.1](https://github.com/coreos/etcd/releases/tag/v3.1.7) before upgrading to 3.2.
+To upgrade an existing etcd deployment to 3.2, the running cluster must be 3.1 or greater. If it's before 3.1, please [upgrade to 3.1](upgrade_3_1.md) before upgrading to 3.2.
 
 Also, to ensure a smooth rolling upgrade, the running cluster must be healthy. Check the health of the cluster by using the `etcdctl endpoint health` command before proceeding.
 

Documentation/v2/security.md

@@ -16,7 +16,7 @@ etcd takes several certificate related configuration options, either through com
 
 `--key-file=<path>`: Key for the certificate. Must be unencrypted.
 
-`--client-cert-auth`: When this is set etcd will check all incoming HTTPS requests for a client certificate signed by the trusted CA, requests that don't supply a valid client certificate will fail.
+`--client-cert-auth`: When this is set etcd will check all incoming HTTPS requests for a client certificate signed by the trusted CA, requests that don't supply a valid client certificate will fail. If [authentication][auth] is enabled, the certificate provides credentials for the user name given by the Common Name field.
 
 `--trusted-ca-file=<path>`: Trusted certificate authority.
 
@@ -191,3 +191,4 @@ If you need your certificate to be signed for your member's FQDN in its Subject
 [tls-setup]: ../../hack/tls-setup
 [tls-guide]: https://github.com/coreos/docs/blob/master/os/generate-self-signed-certificates.md
 [alt-name]: http://wiki.cacert.org/FAQ/subjectAltName
+[auth]: authentication.md

bill-of-materials.json

@@ -1,212 +1,379 @@
 [
 	{
 		"project": "bitbucket.org/ww/goautoneg",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 1
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 1
+			}
+		]
 	},
 	{
 		"project": "github.com/beorn7/perks/quantile",
-		"license": "MIT License",
-		"confidence": 0.989
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 0.9891304347826086
+			}
+		]
 	},
 	{
 		"project": "github.com/bgentry/speakeasy",
-		"license": "MIT License",
-		"confidence": 0.944
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 0.9441624365482234
+			}
+		]
 	},
 	{
 		"project": "github.com/boltdb/bolt",
-		"license": "MIT License",
-		"confidence": 1
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 1
+			}
+		]
 	},
 	{
 		"project": "github.com/cockroachdb/cmux",
-		"license": "Apache License 2.0",
-		"confidence": 1
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
 	},
 	{
 		"project": "github.com/coreos/etcd",
-		"license": "Apache License 2.0",
-		"confidence": 1
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
 	},
 	{
 		"project": "github.com/coreos/go-semver/semver",
-		"license": "Apache License 2.0",
-		"confidence": 1
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
 	},
 	{
 		"project": "github.com/coreos/go-systemd",
-		"license": "Apache License 2.0",
-		"confidence": 0.997
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 0.9966703662597114
+			}
+		]
 	},
 	{
 		"project": "github.com/coreos/pkg",
-		"license": "Apache License 2.0",
-		"confidence": 1
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
 	},
 	{
 		"project": "github.com/cpuguy83/go-md2man/md2man",
-		"license": "MIT License",
-		"confidence": 1
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 1
+			}
+		]
 	},
 	{
 		"project": "github.com/dgrijalva/jwt-go",
-		"license": "MIT License",
-		"confidence": 0.989
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 0.9891304347826086
+			}
+		]
 	},
 	{
 		"project": "github.com/dustin/go-humanize",
-		"license": "MIT License",
-		"confidence": 0.969
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 0.96875
+			}
+		]
 	},
 	{
 		"project": "github.com/ghodss/yaml",
-		"license": "MIT License and BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 1
+		"licenses": [
+			{
+				"type": "MIT License and BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 1
+			}
+		]
 	},
 	{
 		"project": "github.com/gogo/protobuf/proto",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 0.909
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 0.9090909090909091
+			}
+		]
 	},
 	{
 		"project": "github.com/golang/groupcache/lru",
-		"license": "Apache License 2.0",
-		"confidence": 0.997
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 0.9966703662597114
+			}
+		]
 	},
 	{
 		"project": "github.com/golang/protobuf",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 0.92
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 0.92
+			}
+		]
 	},
 	{
 		"project": "github.com/google/btree",
-		"license": "Apache License 2.0",
-		"confidence": 1
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
 	},
 	{
 		"project": "github.com/grpc-ecosystem/go-grpc-prometheus",
-		"license": "Apache License 2.0",
-		"confidence": 1
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
 	},
 	{
 		"project": "github.com/grpc-ecosystem/grpc-gateway",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 0.979
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 0.979253112033195
+			}
+		]
 	},
 	{
 		"project": "github.com/inconshreveable/mousetrap",
-		"license": "Apache License 2.0",
-		"confidence": 1
+		"licenses": [
+			{
+				"type": "MIT License and BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 1
+			},
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
 	},
 	{
 		"project": "github.com/jonboulle/clockwork",
-		"license": "Apache License 2.0",
-		"confidence": 1
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
 	},
 	{
 		"project": "github.com/mattn/go-runewidth",
-		"license": "MIT License",
-		"confidence": 1
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 1
+			}
+		]
 	},
 	{
 		"project": "github.com/matttproud/golang_protobuf_extensions/pbutil",
-		"license": "Apache License 2.0",
-		"confidence": 1
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
 	},
 	{
 		"project": "github.com/olekukonko/tablewriter",
-		"license": "MIT License",
-		"confidence": 0.989
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 0.9891304347826086
+			}
+		]
 	},
 	{
 		"project": "github.com/prometheus/client_golang/prometheus",
-		"license": "Apache License 2.0",
-		"confidence": 1
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
 	},
 	{
 		"project": "github.com/prometheus/client_model/go",
-		"license": "Apache License 2.0",
-		"confidence": 1
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
 	},
 	{
 		"project": "github.com/prometheus/common",
-		"license": "Apache License 2.0",
-		"confidence": 1
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
 	},
 	{
 		"project": "github.com/prometheus/procfs",
-		"license": "Apache License 2.0",
-		"confidence": 1
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
 	},
 	{
 		"project": "github.com/russross/blackfriday",
-		"license": "BSD 2-clause \"Simplified\" License",
-		"confidence": 0.963
-	},
-	{
-		"project": "github.com/shurcooL/sanitized_anchor_name",
-		"license": "MIT License",
-		"confidence": 1
+		"licenses": [
+			{
+				"type": "BSD 2-clause \"Simplified\" License",
+				"confidence": 0.9626168224299065
+			}
+		]
 	},
 	{
 		"project": "github.com/spf13/cobra",
-		"license": "Apache License 2.0",
-		"confidence": 0.957
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 0.9573241061130334
+			}
+		]
 	},
 	{
 		"project": "github.com/spf13/pflag",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 0.966
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 0.9663865546218487
+			}
+		]
 	},
 	{
 		"project": "github.com/ugorji/go/codec",
-		"license": "MIT License",
-		"confidence": 0.995
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 0.9946524064171123
+			}
+		]
 	},
 	{
 		"project": "github.com/urfave/cli",
-		"license": "MIT License",
-		"confidence": 1
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 1
+			}
+		]
 	},
 	{
 		"project": "github.com/xiang90/probing",
-		"license": "MIT License",
-		"confidence": 1
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 1
+			}
+		]
 	},
 	{
 		"project": "golang.org/x/crypto",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 0.966
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 0.9663865546218487
+			}
+		]
 	},
 	{
 		"project": "golang.org/x/net",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 0.966
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 0.9663865546218487
+			}
+		]
 	},
 	{
 		"project": "golang.org/x/text",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 0.966
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 0.9663865546218487
+			}
+		]
 	},
 	{
 		"project": "golang.org/x/time/rate",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 0.966
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 0.9663865546218487
+			}
+		]
 	},
 	{
 		"project": "google.golang.org/grpc",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 0.979
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 0.979253112033195
+			}
+		]
 	},
 	{
 		"project": "gopkg.in/cheggaaa/pb.v1",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License",
-		"confidence": 0.992
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 0.9916666666666667
+			}
+		]
 	},
 	{
 		"project": "gopkg.in/yaml.v2",
-		"license": "Apache License 2.0 and MIT License",
-		"confidence": 1
+		"licenses": [
+			{
+				"type": "The Unlicense",
+				"confidence": 0.35294117647058826
+			},
+			{
+				"type": "MIT License",
+				"confidence": 0.8975609756097561
+			}
+		]
 	}
 ]

bill-of-materials.override.json

@@ -1,18 +1,26 @@
 [
 	{
 		"project": "bitbucket.org/ww/goautoneg",
-		"license": "BSD 3-clause \"New\" or \"Revised\" License"
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License"
+			}
+		]
 	},
 	{
 		"project": "github.com/ghodss/yaml",
-		"license": "MIT License and BSD 3-clause \"New\" or \"Revised\" License"
+		"licenses": [
+			{
+				"type": "MIT License and BSD 3-clause \"New\" or \"Revised\" License"
+			}
+		]
 	},
 	{
 		"project": "github.com/inconshreveable/mousetrap",
-		"license": "Apache License 2.0"
-	},
-	{
-		"project": "gopkg.in/yaml.v2",
-		"license": "Apache License 2.0 and MIT License"
+		"licenses": [
+			{
+				"type": "Apache License 2.0"
+			}
+		]
 	}
 ]

build

@@ -3,9 +3,7 @@
 # set some environment variables
 ORG_PATH="github.com/coreos"
 REPO_PATH="${ORG_PATH}/etcd"
-export GO15VENDOREXPERIMENT="1"
 
-eval $(go env)
 GIT_SHA=`git rev-parse --short HEAD || echo "GitNotFound"`
 if [ ! -z "$FAILPOINTS" ]; then
 	GIT_SHA="$GIT_SHA"-FAILPOINTS
@@ -17,11 +15,7 @@ GO_LDFLAGS="$GO_LDFLAGS -X ${REPO_PATH}/cmd/vendor/${REPO_PATH}/version.GitSHA=$
 # enable/disable failpoints
 toggle_failpoints() {
 	FAILPKGS="etcdserver/ mvcc/backend/"
-
-	mode="disable"
-	if [ ! -z "$FAILPOINTS" ]; then mode="enable"; fi
-	if [ ! -z "$1" ]; then mode="$1"; fi
-
+	mode="$1"
 	if which gofail >/dev/null 2>&1; then
 		gofail "$mode" $FAILPKGS
 	elif [ "$mode" != "disable" ]; then
@@ -30,19 +24,26 @@ toggle_failpoints() {
 	fi
 }
 
+toggle_failpoints_default() {
+	mode="disable"
+	if [ ! -z "$FAILPOINTS" ]; then mode="enable"; fi
+	toggle_failpoints "$mode"
+}
+
 etcd_build() {
 	out="bin"
 	if [ -n "${BINDIR}" ]; then out="${BINDIR}"; fi
-	toggle_failpoints
+	toggle_failpoints_default
 	# Static compilation is useful when etcd is run in a container
 	CGO_ENABLED=0 go build $GO_BUILD_FLAGS -installsuffix cgo -ldflags "$GO_LDFLAGS" -o ${out}/etcd ${REPO_PATH}/cmd/etcd || return
 	CGO_ENABLED=0 go build $GO_BUILD_FLAGS -installsuffix cgo -ldflags "$GO_LDFLAGS" -o ${out}/etcdctl ${REPO_PATH}/cmd/etcdctl || return
 }
 
 etcd_setup_gopath() {
-	CDIR=$(cd `dirname "$0"` && pwd)
+	d=$(dirname "$0")
+	CDIR=$(cd "$d" && pwd)
 	cd "$CDIR"
-	etcdGOPATH=${CDIR}/gopath
+	etcdGOPATH="${CDIR}/gopath"
 	# preserve old gopath to support building with unvendored tooling deps (e.g., gofail)
 	if [ -n "$GOPATH" ]; then
 		GOPATH=":$GOPATH"
@@ -53,7 +54,7 @@ etcd_setup_gopath() {
 	ln -s ${CDIR}/cmd/vendor ${etcdGOPATH}/src
 }
 
-toggle_failpoints
+toggle_failpoints_default
 
 # only build when called directly, not sourced
 if echo "$0" | grep "build$" >/dev/null; then

clientv3/client.go

@@ -182,7 +182,7 @@ func parseEndpoint(endpoint string) (proto string, host string, scheme string) {
 	host = url.Host
 	switch url.Scheme {
 	case "http", "https":
-	case "unix":
+	case "unix", "unixs":
 		proto = "unix"
 		host = url.Host + url.Path
 	default:
@@ -197,7 +197,7 @@ func (c *Client) processCreds(scheme string) (creds *credentials.TransportCreden
 	case "unix":
 	case "http":
 		creds = nil
-	case "https":
+	case "https", "unixs":
 		if creds != nil {
 			break
 		}
@@ -322,7 +322,7 @@ func (c *Client) dial(endpoint string, dopts ...grpc.DialOption) (*grpc.ClientCo
 
 	opts = append(opts, c.cfg.DialOptions...)
 
-	conn, err := grpc.Dial(host, opts...)
+	conn, err := grpc.DialContext(c.ctx, host, opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -367,7 +367,9 @@ func newClient(cfg *Config) (*Client, error) {
 	}
 
 	client.balancer = newSimpleBalancer(cfg.Endpoints)
-	conn, err := client.dial("", grpc.WithBalancer(client.balancer))
+	// use Endpoints[0] so that for https:// without any tls config given, then
+	// grpc will assume the ServerName is in the endpoint.
+	conn, err := client.dial(cfg.Endpoints[0], grpc.WithBalancer(client.balancer))
 	if err != nil {
 		client.cancel()
 		client.balancer.Close()

clientv3/example_metrics_test.go

@@ -30,7 +30,7 @@ import (
 	"google.golang.org/grpc"
 )
 
-func ExampleMetrics_range() {
+func ExampleClient_metrics() {
 	cli, err := clientv3.New(clientv3.Config{
 		Endpoints: endpoints,
 		DialOptions: []grpc.DialOption{

clientv3/integration/dial_test.go

@@ -66,6 +66,22 @@ func TestDialTLSExpired(t *testing.T) {
 	}
 }
 
+// TestDialTLSNoConfig ensures the client fails to dial / times out
+// when TLS endpoints (https, unixs) are given but no tls config.
+func TestDialTLSNoConfig(t *testing.T) {
+	defer testutil.AfterTest(t)
+	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1, ClientTLS: &testTLSInfo})
+	defer clus.Terminate(t)
+	// expect 'signed by unknown authority'
+	_, err := clientv3.New(clientv3.Config{
+		Endpoints:   []string{clus.Members[0].GRPCAddr()},
+		DialTimeout: time.Second,
+	})
+	if err != grpc.ErrClientConnTimeout {
+		t.Fatalf("expected %v, got %v", grpc.ErrClientConnTimeout, err)
+	}
+}
+
 // TestDialSetEndpoints ensures SetEndpoints can replace unavailable endpoints with available ones.
 func TestDialSetEndpointsBeforeFail(t *testing.T) {
 	testDialSetEndpoints(t, true)

cmd/vendor/github.com/coreos/go-semver/semver/semver.go

@@ -1,3 +1,18 @@
+// Copyright 2013-2015 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Semantic Versions http://semver.org
 package semver
 
 import (
@@ -29,35 +44,21 @@ func splitOff(input *string, delim string) (val string) {
 	return val
 }
 
+func New(version string) *Version {
+	return Must(NewVersion(version))
+}
+
 func NewVersion(version string) (*Version, error) {
 	v := Version{}
 
-	dotParts := strings.SplitN(version, ".", 3)
-
-	if len(dotParts) != 3 {
-		return nil, errors.New(fmt.Sprintf("%s is not in dotted-tri format", version))
+	if err := v.Set(version); err != nil {
+		return nil, err
 	}
 
-	v.Metadata = splitOff(&dotParts[2], "+")
-	v.PreRelease = PreRelease(splitOff(&dotParts[2], "-"))
-
-	parsed := make([]int64, 3, 3)
-
-	for i, v := range dotParts[:3] {
-		val, err := strconv.ParseInt(v, 10, 64)
-		parsed[i] = val
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	v.Major = parsed[0]
-	v.Minor = parsed[1]
-	v.Patch = parsed[2]
-
 	return &v, nil
 }
 
+// Must is a helper for wrapping NewVersion and will panic if err is not nil.
 func Must(v *Version, err error) *Version {
 	if err != nil {
 		panic(err)
@@ -65,45 +66,99 @@ func Must(v *Version, err error) *Version {
 	return v
 }
 
-func (v *Version) String() string {
+// Set parses and updates v from the given version string. Implements flag.Value
+func (v *Version) Set(version string) error {
+	metadata := splitOff(&version, "+")
+	preRelease := PreRelease(splitOff(&version, "-"))
+	dotParts := strings.SplitN(version, ".", 3)
+
+	if len(dotParts) != 3 {
+		return fmt.Errorf("%s is not in dotted-tri format", version)
+	}
+
+	parsed := make([]int64, 3, 3)
+
+	for i, v := range dotParts[:3] {
+		val, err := strconv.ParseInt(v, 10, 64)
+		parsed[i] = val
+		if err != nil {
+			return err
+		}
+	}
+
+	v.Metadata = metadata
+	v.PreRelease = preRelease
+	v.Major = parsed[0]
+	v.Minor = parsed[1]
+	v.Patch = parsed[2]
+	return nil
+}
+
+func (v Version) String() string {
 	var buffer bytes.Buffer
 
-	base := fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch)
-	buffer.WriteString(base)
+	fmt.Fprintf(&buffer, "%d.%d.%d", v.Major, v.Minor, v.Patch)
 
 	if v.PreRelease != "" {
-		buffer.WriteString(fmt.Sprintf("-%s", v.PreRelease))
+		fmt.Fprintf(&buffer, "-%s", v.PreRelease)
 	}
 
 	if v.Metadata != "" {
-		buffer.WriteString(fmt.Sprintf("+%s", v.Metadata))
+		fmt.Fprintf(&buffer, "+%s", v.Metadata)
 	}
 
 	return buffer.String()
 }
 
-func (v *Version) LessThan(versionB Version) bool {
-	versionA := *v
-	cmp := recursiveCompare(versionA.Slice(), versionB.Slice())
-
-	if cmp == 0 {
-		cmp = preReleaseCompare(versionA, versionB)
+func (v *Version) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var data string
+	if err := unmarshal(&data); err != nil {
+		return err
 	}
-
-	if cmp == -1 {
-		return true
-	}
-
-	return false
+	return v.Set(data)
 }
 
-/* Slice converts the comparable parts of the semver into a slice of strings */
-func (v *Version) Slice() []int64 {
+func (v Version) MarshalJSON() ([]byte, error) {
+	return []byte(`"` + v.String() + `"`), nil
+}
+
+func (v *Version) UnmarshalJSON(data []byte) error {
+	l := len(data)
+	if l == 0 || string(data) == `""` {
+		return nil
+	}
+	if l < 2 || data[0] != '"' || data[l-1] != '"' {
+		return errors.New("invalid semver string")
+	}
+	return v.Set(string(data[1 : l-1]))
+}
+
+// Compare tests if v is less than, equal to, or greater than versionB,
+// returning -1, 0, or +1 respectively.
+func (v Version) Compare(versionB Version) int {
+	if cmp := recursiveCompare(v.Slice(), versionB.Slice()); cmp != 0 {
+		return cmp
+	}
+	return preReleaseCompare(v, versionB)
+}
+
+// Equal tests if v is equal to versionB.
+func (v Version) Equal(versionB Version) bool {
+	return v.Compare(versionB) == 0
+}
+
+// LessThan tests if v is less than versionB.
+func (v Version) LessThan(versionB Version) bool {
+	return v.Compare(versionB) < 0
+}
+
+// Slice converts the comparable parts of the semver into a slice of integers.
+func (v Version) Slice() []int64 {
 	return []int64{v.Major, v.Minor, v.Patch}
 }
 
-func (p *PreRelease) Slice() []string {
-	preRelease := string(*p)
+func (p PreRelease) Slice() []string {
+	preRelease := string(p)
 	return strings.Split(preRelease, ".")
 }
 
@@ -119,7 +174,7 @@ func preReleaseCompare(versionA Version, versionB Version) int {
 		return -1
 	}
 
-	// If there is a prelease, check and compare each part.
+	// If there is a prerelease, check and compare each part.
 	return recursivePreReleaseCompare(a.Slice(), b.Slice())
 }
 
@@ -141,9 +196,12 @@ func recursiveCompare(versionA []int64, versionB []int64) int {
 }
 
 func recursivePreReleaseCompare(versionA []string, versionB []string) int {
-	// Handle slice length disparity.
+	// A larger set of pre-release fields has a higher precedence than a smaller set,
+	// if all of the preceding identifiers are equal.
 	if len(versionA) == 0 {
-		// Nothing to compare too, so we return 0
+		if len(versionB) > 0 {
+			return -1
+		}
 		return 0
 	} else if len(versionB) == 0 {
 		// We're longer than versionB so return 1.
@@ -153,7 +211,8 @@ func recursivePreReleaseCompare(versionA []string, versionB []string) int {
 	a := versionA[0]
 	b := versionB[0]
 
-	aInt := false; bInt := false
+	aInt := false
+	bInt := false
 
 	aI, err := strconv.Atoi(versionA[0])
 	if err == nil {

cmd/vendor/github.com/coreos/go-semver/semver/sort.go

@@ -1,3 +1,17 @@
+// Copyright 2013-2015 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package semver
 
 import (

cmd/vendor/github.com/ghodss/yaml/fields.go

@@ -45,7 +45,11 @@ func indirect(v reflect.Value, decodingNull bool) (json.Unmarshaler, encoding.Te
 			break
 		}
 		if v.IsNil() {
-			v.Set(reflect.New(v.Type().Elem()))
+			if v.CanSet() {
+				v.Set(reflect.New(v.Type().Elem()))
+			} else {
+				v = reflect.New(v.Type().Elem())
+			}
 		}
 		if v.Type().NumMethod() > 0 {
 			if u, ok := v.Interface().(json.Unmarshaler); ok {

cmd/vendor/github.com/ghodss/yaml/yaml.go

@@ -15,12 +15,12 @@ import (
 func Marshal(o interface{}) ([]byte, error) {
 	j, err := json.Marshal(o)
 	if err != nil {
-		return nil, fmt.Errorf("error marshaling into JSON: ", err)
+		return nil, fmt.Errorf("error marshaling into JSON: %v", err)
 	}
 
 	y, err := JSONToYAML(j)
 	if err != nil {
-		return nil, fmt.Errorf("error converting JSON to YAML: ", err)
+		return nil, fmt.Errorf("error converting JSON to YAML: %v", err)
 	}
 
 	return y, nil
@@ -48,7 +48,7 @@ func JSONToYAML(j []byte) ([]byte, error) {
 	var jsonObj interface{}
 	// We are using yaml.Unmarshal here (instead of json.Unmarshal) because the
 	// Go JSON library doesn't try to pick the right number type (int, float,
-	// etc.) when unmarshling to interface{}, it just picks float64
+	// etc.) when unmarshalling to interface{}, it just picks float64
 	// universally. go-yaml does go through the effort of picking the right
 	// number type, so we can preserve number type throughout this process.
 	err := yaml.Unmarshal(j, &jsonObj)

cmd/vendor/github.com/kr/pty/ioctl.go

@@ -1,3 +1,5 @@
+// +build !windows
+
 package pty
 
 import "syscall"

cmd/vendor/github.com/kr/pty/pty_dragonfly.go

@@ -0,0 +1,76 @@
+package pty
+
+import (
+	"errors"
+	"os"
+	"strings"
+	"syscall"
+	"unsafe"
+)
+
+// same code as pty_darwin.go
+func open() (pty, tty *os.File, err error) {
+	p, err := os.OpenFile("/dev/ptmx", os.O_RDWR, 0)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	sname, err := ptsname(p)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	err = grantpt(p)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	err = unlockpt(p)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	t, err := os.OpenFile(sname, os.O_RDWR, 0)
+	if err != nil {
+		return nil, nil, err
+	}
+	return p, t, nil
+}
+
+func grantpt(f *os.File) error {
+	_, err := isptmaster(f.Fd())
+	return err
+}
+
+func unlockpt(f *os.File) error {
+	_, err := isptmaster(f.Fd())
+	return err
+}
+
+func isptmaster(fd uintptr) (bool, error) {
+	err := ioctl(fd, syscall.TIOCISPTMASTER, 0)
+	return err == nil, err
+}
+
+var (
+	emptyFiodgnameArg fiodgnameArg
+	ioctl_FIODNAME    = _IOW('f', 120, unsafe.Sizeof(emptyFiodgnameArg))
+)
+
+func ptsname(f *os.File) (string, error) {
+	name := make([]byte, _C_SPECNAMELEN)
+	fa := fiodgnameArg{Name: (*byte)(unsafe.Pointer(&name[0])), Len: _C_SPECNAMELEN, Pad_cgo_0: [4]byte{0, 0, 0, 0}}
+
+	err := ioctl(f.Fd(), ioctl_FIODNAME, uintptr(unsafe.Pointer(&fa)))
+	if err != nil {
+		return "", err
+	}
+
+	for i, c := range name {
+		if c == 0 {
+			s := "/dev/" + string(name[:i])
+			return strings.Replace(s, "ptm", "pts", -1), nil
+		}
+	}
+	return "", errors.New("TIOCPTYGNAME string not NUL-terminated")
+}

cmd/vendor/github.com/kr/pty/pty_unsupported.go

@@ -1,4 +1,4 @@
-// +build !linux,!darwin,!freebsd
+// +build !linux,!darwin,!freebsd,!dragonfly
 
 package pty
 

cmd/vendor/github.com/kr/pty/run.go

@@ -1,3 +1,5 @@
+// +build !windows
+
 package pty
 
 import (

cmd/vendor/github.com/kr/pty/types_dragonfly.go

@@ -0,0 +1,17 @@
+// +build ignore
+
+package pty
+
+/*
+#define _KERNEL
+#include <sys/conf.h>
+#include <sys/param.h>
+#include <sys/filio.h>
+*/
+import "C"
+
+const (
+	_C_SPECNAMELEN = C.SPECNAMELEN /* max length of devicename */
+)
+
+type fiodgnameArg C.struct_fiodname_args

cmd/vendor/github.com/kr/pty/util.go

@@ -1,3 +1,5 @@
+// +build !windows
+
 package pty
 
 import (

cmd/vendor/github.com/kr/pty/ztypes_dragonfly_amd64.go

@@ -0,0 +1,14 @@
+// Created by cgo -godefs - DO NOT EDIT
+// cgo -godefs types_dragonfly.go
+
+package pty
+
+const (
+	_C_SPECNAMELEN = 0x3f
+)
+
+type fiodgnameArg struct {
+	Name      *byte
+	Len       uint32
+	Pad_cgo_0 [4]byte
+}

cmd/vendor/github.com/kr/pty/ztypes_mipsx.go

@@ -0,0 +1,12 @@
+// Created by cgo -godefs - DO NOT EDIT
+// cgo -godefs types.go
+
+// +build linux
+// +build mips mipsle mips64 mips64le
+
+package pty
+
+type (
+	_C_int  int32
+	_C_uint uint32
+)

cmd/vendor/github.com/russross/blackfriday/block.go

@@ -15,8 +15,7 @@ package blackfriday
 
 import (
 	"bytes"
-
-	"github.com/shurcooL/sanitized_anchor_name"
+	"unicode"
 )
 
 // Parse block-level data.
@@ -243,7 +242,7 @@ func (p *parser) prefixHeader(out *bytes.Buffer, data []byte) int {
 	}
 	if end > i {
 		if id == "" && p.flags&EXTENSION_AUTO_HEADER_IDS != 0 {
-			id = sanitized_anchor_name.Create(string(data[i:end]))
+			id = SanitizedAnchorName(string(data[i:end]))
 		}
 		work := func() bool {
 			p.inline(out, data[i:end])
@@ -1364,7 +1363,7 @@ func (p *parser) paragraph(out *bytes.Buffer, data []byte) int {
 
 				id := ""
 				if p.flags&EXTENSION_AUTO_HEADER_IDS != 0 {
-					id = sanitized_anchor_name.Create(string(data[prev:eol]))
+					id = SanitizedAnchorName(string(data[prev:eol]))
 				}
 
 				p.r.Header(out, work, level, id)
@@ -1428,3 +1427,24 @@ func (p *parser) paragraph(out *bytes.Buffer, data []byte) int {
 	p.renderParagraph(out, data[:i])
 	return i
 }
+
+// SanitizedAnchorName returns a sanitized anchor name for the given text.
+//
+// It implements the algorithm specified in the package comment.
+func SanitizedAnchorName(text string) string {
+	var anchorName []rune
+	futureDash := false
+	for _, r := range text {
+		switch {
+		case unicode.IsLetter(r) || unicode.IsNumber(r):
+			if futureDash && len(anchorName) > 0 {
+				anchorName = append(anchorName, '-')
+			}
+			futureDash = false
+			anchorName = append(anchorName, unicode.ToLower(r))
+		default:
+			futureDash = true
+		}
+	}
+	return string(anchorName)
+}

cmd/vendor/github.com/russross/blackfriday/doc.go

@@ -0,0 +1,32 @@
+// Package blackfriday is a Markdown processor.
+//
+// It translates plain text with simple formatting rules into HTML or LaTeX.
+//
+// Sanitized Anchor Names
+//
+// Blackfriday includes an algorithm for creating sanitized anchor names
+// corresponding to a given input text. This algorithm is used to create
+// anchors for headings when EXTENSION_AUTO_HEADER_IDS is enabled. The
+// algorithm is specified below, so that other packages can create
+// compatible anchor names and links to those anchors.
+//
+// The algorithm iterates over the input text, interpreted as UTF-8,
+// one Unicode code point (rune) at a time. All runes that are letters (category L)
+// or numbers (category N) are considered valid characters. They are mapped to
+// lower case, and included in the output. All other runes are considered
+// invalid characters. Invalid characters that preceed the first valid character,
+// as well as invalid character that follow the last valid character
+// are dropped completely. All other sequences of invalid characters
+// between two valid characters are replaced with a single dash character '-'.
+//
+// SanitizedAnchorName exposes this functionality, and can be used to
+// create compatible links to the anchor names generated by blackfriday.
+// This algorithm is also implemented in a small standalone package at
+// github.com/shurcooL/sanitized_anchor_name. It can be useful for clients
+// that want a small package and don't need full functionality of blackfriday.
+package blackfriday
+
+// NOTE: Keep Sanitized Anchor Name algorithm in sync with package
+//       github.com/shurcooL/sanitized_anchor_name.
+//       Otherwise, users of sanitized_anchor_name will get anchor names
+//       that are incompatible with those generated by blackfriday.

cmd/vendor/github.com/russross/blackfriday/markdown.go

@@ -13,9 +13,6 @@
 //
 //
 
-// Blackfriday markdown processor.
-//
-// Translates plain text with simple formatting rules into HTML or LaTeX.
 package blackfriday
 
 import (

cmd/vendor/github.com/shurcooL/sanitized_anchor_name/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2015 Dmitri Shuralyov
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

cmd/vendor/github.com/shurcooL/sanitized_anchor_name/main.go

@@ -1,29 +0,0 @@
-// Package sanitized_anchor_name provides a func to create sanitized anchor names.
-//
-// Its logic can be reused by multiple packages to create interoperable anchor names
-// and links to those anchors.
-//
-// At this time, it does not try to ensure that generated anchor names
-// are unique, that responsibility falls on the caller.
-package sanitized_anchor_name // import "github.com/shurcooL/sanitized_anchor_name"
-
-import "unicode"
-
-// Create returns a sanitized anchor name for the given text.
-func Create(text string) string {
-	var anchorName []rune
-	var futureDash = false
-	for _, r := range []rune(text) {
-		switch {
-		case unicode.IsLetter(r) || unicode.IsNumber(r):
-			if futureDash && len(anchorName) > 0 {
-				anchorName = append(anchorName, '-')
-			}
-			futureDash = false
-			anchorName = append(anchorName, unicode.ToLower(r))
-		default:
-			futureDash = true
-		}
-	}
-	return string(anchorName)
-}

cmd/vendor/golang.org/x/text/unicode/norm/input.go

@@ -90,16 +90,20 @@ func (in *input) charinfoNFKC(p int) (uint16, int) {
 }
 
 func (in *input) hangul(p int) (r rune) {
+	var size int
 	if in.bytes == nil {
 		if !isHangulString(in.str[p:]) {
 			return 0
 		}
-		r, _ = utf8.DecodeRuneInString(in.str[p:])
+		r, size = utf8.DecodeRuneInString(in.str[p:])
 	} else {
 		if !isHangul(in.bytes[p:]) {
 			return 0
 		}
-		r, _ = utf8.DecodeRune(in.bytes[p:])
+		r, size = utf8.DecodeRune(in.bytes[p:])
+	}
+	if size != hangulUTF8Size {
+		return 0
 	}
 	return r
 }

contrib/raftexample/raft.go

@@ -288,14 +288,11 @@ func (rc *raftNode) startRaft() {
 		rc.node = raft.StartNode(c, startPeers)
 	}
 
-	ss := &stats.ServerStats{}
-	ss.Initialize()
-
 	rc.transport = &rafthttp.Transport{
 		ID:          types.ID(rc.id),
 		ClusterID:   0x1000,
 		Raft:        rc,
-		ServerStats: ss,
+		ServerStats: stats.NewServerStats("", ""),
 		LeaderStats: stats.NewLeaderStats(strconv.Itoa(rc.id)),
 		ErrorC:      make(chan error),
 	}

e2e/ctl_v3_elect_test.go

@@ -80,7 +80,7 @@ func testElect(cx ctlCtx) {
 	if err = blocked.Signal(os.Interrupt); err != nil {
 		cx.t.Fatal(err)
 	}
-	if err := closeWithTimeout(blocked, time.Second); err != nil {
+	if err = closeWithTimeout(blocked, time.Second); err != nil {
 		cx.t.Fatal(err)
 	}
 

e2e/v3_curl_test.go

@@ -20,6 +20,8 @@ import (
 
 	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
 	"github.com/coreos/etcd/pkg/testutil"
+
+	"github.com/grpc-ecosystem/grpc-gateway/runtime"
 )
 
 func TestV3CurlPutGetNoTLS(t *testing.T)     { testCurlPutGetGRPCGateway(t, &configNoTLS) }
@@ -111,3 +113,52 @@ func TestV3CurlWatch(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestV3CurlTxn(t *testing.T) {
+	defer testutil.AfterTest(t)
+	epc, err := newEtcdProcessCluster(&configNoTLS)
+	if err != nil {
+		t.Fatalf("could not start etcd process cluster (%v)", err)
+	}
+	defer func() {
+		if cerr := epc.Close(); err != nil {
+			t.Fatalf("error closing etcd processes (%v)", cerr)
+		}
+	}()
+
+	txn := &pb.TxnRequest{
+		Compare: []*pb.Compare{
+			{
+				Key:         []byte("foo"),
+				Result:      pb.Compare_EQUAL,
+				Target:      pb.Compare_CREATE,
+				TargetUnion: &pb.Compare_CreateRevision{0},
+			},
+		},
+		Success: []*pb.RequestOp{
+			{
+				Request: &pb.RequestOp_RequestPut{
+					RequestPut: &pb.PutRequest{
+						Key:   []byte("foo"),
+						Value: []byte("bar"),
+					},
+				},
+			},
+		},
+	}
+	m := &runtime.JSONPb{}
+	jsonDat, jerr := m.Marshal(txn)
+	if jerr != nil {
+		t.Fatal(jerr)
+	}
+	expected := `"succeeded":true,"responses":[{"response_put":{"header":{"revision":"2"}}}]`
+	if err = cURLPost(epc, cURLReq{endpoint: "/v3alpha/kv/txn", value: string(jsonDat), expected: expected}); err != nil {
+		t.Fatalf("failed txn with curl (%v)", err)
+	}
+
+	// was crashing etcd server
+	malformed := `{"compare":[{"result":0,"target":1,"key":"Zm9v","TargetUnion":null}],"success":[{"Request":{"RequestPut":{"key":"Zm9v","value":"YmFy"}}}]}`
+	if err = cURLPost(epc, cURLReq{endpoint: "/v3alpha/kv/txn", value: malformed, expected: "error"}); err != nil {
+		t.Fatalf("failed put with curl (%v)", err)
+	}
+}

etcdctl/ctlv3/command/check.go

@@ -150,8 +150,8 @@ func newCheckPerfCommand(cmd *cobra.Command, args []string) {
 	}
 
 	go func() {
-		cctx, _ := context.WithTimeout(context.Background(), time.Duration(cfg.duration)*time.Second)
-
+		cctx, ccancel := context.WithTimeout(context.Background(), time.Duration(cfg.duration)*time.Second)
+		defer ccancel()
 		for limit.Wait(cctx) == nil {
 			binary.PutVarint(k, int64(rand.Int63n(math.MaxInt64)))
 			requests <- v3.OpPut(checkPerfPrefix+string(k), v)

etcdserver/api/v2http/client.go

@@ -46,15 +46,16 @@ import (
 )
 
 const (
-	authPrefix    = "/v2/auth"
-	keysPrefix    = "/v2/keys"
-	membersPrefix = "/v2/members"
-	statsPrefix   = "/v2/stats"
-	varsPath      = "/debug/vars"
-	metricsPath   = "/metrics"
-	healthPath    = "/health"
-	versionPath   = "/version"
-	configPath    = "/config"
+	authPrefix     = "/v2/auth"
+	keysPrefix     = "/v2/keys"
+	machinesPrefix = "/v2/machines"
+	membersPrefix  = "/v2/members"
+	statsPrefix    = "/v2/stats"
+	varsPath       = "/debug/vars"
+	metricsPath    = "/metrics"
+	healthPath     = "/health"
+	versionPath    = "/version"
+	configPath     = "/config"
 )
 
 // NewClientHandler generates a muxed http.Handler with the given parameters to serve etcd client requests.
@@ -83,6 +84,8 @@ func NewClientHandler(server *etcdserver.EtcdServer, timeout time.Duration) http
 		clientCertAuthEnabled: server.Cfg.ClientCertAuthEnabled,
 	}
 
+	mah := &machinesHandler{cluster: server.Cluster()}
+
 	sech := &authHandler{
 		sec:                   sec,
 		cluster:               server.Cluster(),
@@ -103,6 +106,7 @@ func NewClientHandler(server *etcdserver.EtcdServer, timeout time.Duration) http
 	mux.Handle(metricsPath, prometheus.Handler())
 	mux.Handle(membersPrefix, mh)
 	mux.Handle(membersPrefix+"/", mh)
+	mux.Handle(machinesPrefix, mah)
 	handleAuth(mux, sech)
 
 	return requestLogger(mux)
@@ -164,6 +168,18 @@ func (h *keysHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+type machinesHandler struct {
+	cluster api.Cluster
+}
+
+func (h *machinesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if !allowMethod(w, r.Method, "GET", "HEAD") {
+		return
+	}
+	endpoints := h.cluster.ClientURLs()
+	w.Write([]byte(strings.Join(endpoints, ", ")))
+}
+
 type membersHandler struct {
 	sec                   auth.Store
 	server                etcdserver.Server

etcdserver/api/v2http/client_test.go

@@ -1220,6 +1220,56 @@ func TestWriteEvent(t *testing.T) {
 	}
 }
 
+func TestV2DMachinesEndpoint(t *testing.T) {
+	tests := []struct {
+		method string
+		wcode  int
+	}{
+		{"GET", http.StatusOK},
+		{"HEAD", http.StatusOK},
+		{"POST", http.StatusMethodNotAllowed},
+	}
+
+	m := &machinesHandler{cluster: &fakeCluster{}}
+	s := httptest.NewServer(m)
+	defer s.Close()
+
+	for _, tt := range tests {
+		req, err := http.NewRequest(tt.method, s.URL+machinesPrefix, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if resp.StatusCode != tt.wcode {
+			t.Errorf("StatusCode = %d, expected %d", resp.StatusCode, tt.wcode)
+		}
+	}
+}
+
+func TestServeMachines(t *testing.T) {
+	cluster := &fakeCluster{
+		clientURLs: []string{"http://localhost:8080", "http://localhost:8081", "http://localhost:8082"},
+	}
+	writer := httptest.NewRecorder()
+	req, err := http.NewRequest("GET", "", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	h := &machinesHandler{cluster: cluster}
+	h.ServeHTTP(writer, req)
+	w := "http://localhost:8080, http://localhost:8081, http://localhost:8082"
+	if g := writer.Body.String(); g != w {
+		t.Errorf("body = %s, want %s", g, w)
+	}
+	if writer.Code != http.StatusOK {
+		t.Errorf("code = %d, want %d", writer.Code, http.StatusOK)
+	}
+}
+
 func TestGetID(t *testing.T) {
 	tests := []struct {
 		path string

etcdserver/api/v3rpc/key.go

@@ -252,8 +252,8 @@ func checkRequestOp(u *pb.RequestOp) error {
 			return checkDeleteRequest(uv.RequestDeleteRange)
 		}
 	default:
-		// empty op
-		return nil
+		// empty op / nil entry
+		return rpctypes.ErrGRPCKeyNotFound
 	}
 	return nil
 }

etcdserver/apply.go

@@ -318,33 +318,36 @@ func (a *applierV3backend) Range(txn mvcc.TxnRead, r *pb.RangeRequest) (*pb.Rang
 }
 
 func (a *applierV3backend) Txn(rt *pb.TxnRequest) (*pb.TxnResponse, error) {
-	ok := true
-	for _, c := range rt.Compare {
-		if _, ok = a.applyCompare(c); !ok {
-			break
+	isWrite := !isTxnReadonly(rt)
+	txn := mvcc.NewReadOnlyTxnWrite(a.s.KV().Read())
+
+	reqs, ok := a.compareToOps(txn, rt)
+	if isWrite {
+		if err := a.checkRequestPut(txn, reqs); err != nil {
+			txn.End()
+			return nil, err
 		}
 	}
-
-	var reqs []*pb.RequestOp
-	if ok {
-		reqs = rt.Success
-	} else {
-		reqs = rt.Failure
-	}
-
-	if err := a.checkRequestPut(reqs); err != nil {
-		return nil, err
-	}
-	if err := a.checkRequestRange(reqs); err != nil {
+	if err := checkRequestRange(txn, reqs); err != nil {
+		txn.End()
 		return nil, err
 	}
 
 	resps := make([]*pb.ResponseOp, len(reqs))
+	txnResp := &pb.TxnResponse{
+		Responses: resps,
+		Succeeded: ok,
+		Header:    &pb.ResponseHeader{},
+	}
 
-	// When executing the operations of txn, etcd must hold the txn lock so
-	// readers do not see any intermediate results.
-	// TODO: use Read txn if only Ranges
-	txn := a.s.KV().Write()
+	// When executing mutable txn ops, etcd must hold the txn lock so
+	// readers do not see any intermediate results. Since writes are
+	// serialized on the raft loop, the revision in the read view will
+	// be the revision of the write txn.
+	if isWrite {
+		txn.End()
+		txn = a.s.KV().Write()
+	}
 	for i := range reqs {
 		resps[i] = a.applyUnion(txn, reqs[i])
 	}
@@ -354,23 +357,25 @@ func (a *applierV3backend) Txn(rt *pb.TxnRequest) (*pb.TxnResponse, error) {
 	}
 	txn.End()
 
-	txnResp := &pb.TxnResponse{}
-	txnResp.Header = &pb.ResponseHeader{}
 	txnResp.Header.Revision = rev
-	txnResp.Responses = resps
-	txnResp.Succeeded = ok
 	return txnResp, nil
 }
 
-// applyCompare applies the compare request.
-// It returns the revision at which the comparison happens. If the comparison
-// succeeds, the it returns true. Otherwise it returns false.
-func (a *applierV3backend) applyCompare(c *pb.Compare) (int64, bool) {
-	rr, err := a.s.KV().Range(c.Key, nil, mvcc.RangeOptions{})
-	rev := rr.Rev
+func (a *applierV3backend) compareToOps(rv mvcc.ReadView, rt *pb.TxnRequest) ([]*pb.RequestOp, bool) {
+	for _, c := range rt.Compare {
+		if !applyCompare(rv, c) {
+			return rt.Failure, false
+		}
+	}
+	return rt.Success, true
+}
 
+// applyCompare applies the compare request.
+// If the comparison succeeds, it returns true. Otherwise, returns false.
+func applyCompare(rv mvcc.ReadView, c *pb.Compare) bool {
+	rr, err := rv.Range(c.Key, nil, mvcc.RangeOptions{})
 	if err != nil {
-		return rev, false
+		return false
 	}
 	var ckv mvccpb.KeyValue
 	if len(rr.KVs) != 0 {
@@ -382,7 +387,7 @@ func (a *applierV3backend) applyCompare(c *pb.Compare) (int64, bool) {
 			// We can treat non-existence as the empty set explicitly, such that
 			// even a key with a value of length 0 bytes is still a real key
 			// that was written that way
-			return rev, false
+			return false
 		}
 	}
 
@@ -414,23 +419,15 @@ func (a *applierV3backend) applyCompare(c *pb.Compare) (int64, bool) {
 
 	switch c.Result {
 	case pb.Compare_EQUAL:
-		if result != 0 {
-			return rev, false
-		}
+		return result == 0
 	case pb.Compare_NOT_EQUAL:
-		if result == 0 {
-			return rev, false
-		}
+		return result != 0
 	case pb.Compare_GREATER:
-		if result != 1 {
-			return rev, false
-		}
+		return result > 0
 	case pb.Compare_LESS:
-		if result != -1 {
-			return rev, false
-		}
+		return result < 0
 	}
-	return rev, true
+	return true
 }
 
 func (a *applierV3backend) applyUnion(txn mvcc.TxnWrite, union *pb.RequestOp) *pb.ResponseOp {
@@ -770,7 +767,7 @@ func (s *kvSortByValue) Less(i, j int) bool {
 	return bytes.Compare(s.kvs[i].Value, s.kvs[j].Value) < 0
 }
 
-func (a *applierV3backend) checkRequestPut(reqs []*pb.RequestOp) error {
+func (a *applierV3backend) checkRequestPut(rv mvcc.ReadView, reqs []*pb.RequestOp) error {
 	for _, requ := range reqs {
 		tv, ok := requ.Request.(*pb.RequestOp_RequestPut)
 		if !ok {
@@ -782,7 +779,7 @@ func (a *applierV3backend) checkRequestPut(reqs []*pb.RequestOp) error {
 		}
 		if preq.IgnoreValue || preq.IgnoreLease {
 			// expects previous key-value, error if not exist
-			rr, err := a.s.KV().Range(preq.Key, nil, mvcc.RangeOptions{})
+			rr, err := rv.Range(preq.Key, nil, mvcc.RangeOptions{})
 			if err != nil {
 				return err
 			}
@@ -800,7 +797,7 @@ func (a *applierV3backend) checkRequestPut(reqs []*pb.RequestOp) error {
 	return nil
 }
 
-func (a *applierV3backend) checkRequestRange(reqs []*pb.RequestOp) error {
+func checkRequestRange(rv mvcc.ReadView, reqs []*pb.RequestOp) error {
 	for _, requ := range reqs {
 		tv, ok := requ.Request.(*pb.RequestOp_RequestRange)
 		if !ok {
@@ -811,10 +808,10 @@ func (a *applierV3backend) checkRequestRange(reqs []*pb.RequestOp) error {
 			continue
 		}
 
-		if greq.Revision > a.s.KV().Rev() {
+		if greq.Revision > rv.Rev() {
 			return mvcc.ErrFutureRev
 		}
-		if greq.Revision < a.s.KV().FirstRev() {
+		if greq.Revision < rv.FirstRev() {
 			return mvcc.ErrCompacted
 		}
 	}

etcdserver/metrics.go

@@ -58,6 +58,12 @@ var (
 		Name:      "proposals_failed_total",
 		Help:      "The total number of failed proposals seen.",
 	})
+	leaseExpired = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "etcd_debugging",
+		Subsystem: "server",
+		Name:      "lease_expired_total",
+		Help:      "The total number of expired leases.",
+	})
 )
 
 func init() {
@@ -67,6 +73,7 @@ func init() {
 	prometheus.MustRegister(proposalsApplied)
 	prometheus.MustRegister(proposalsPending)
 	prometheus.MustRegister(proposalsFailed)
+	prometheus.MustRegister(leaseExpired)
 }
 
 func monitorFileDescriptor(done <-chan struct{}) {

etcdserver/server.go

@@ -395,11 +395,7 @@ func NewServer(cfg *ServerConfig) (srv *EtcdServer, err error) {
 		return nil, fmt.Errorf("cannot access member directory: %v", terr)
 	}
 
-	sstats := &stats.ServerStats{
-		Name: cfg.Name,
-		ID:   id.String(),
-	}
-	sstats.Initialize()
+	sstats := stats.NewServerStats(cfg.Name, id.String())
 	lstats := stats.NewLeaderStats(id.String())
 
 	heartbeat := time.Duration(cfg.TickMs) * time.Millisecond
@@ -743,6 +739,7 @@ func (s *EtcdServer) run() {
 					lid := lease.ID
 					s.goAttach(func() {
 						s.LeaseRevoke(s.ctx, &pb.LeaseRevokeRequest{ID: int64(lid)})
+						leaseExpired.Inc()
 						<-c
 					})
 				}

etcdserver/stats/leader.go

@@ -24,25 +24,30 @@ import (
 // LeaderStats is used by the leader in an etcd cluster, and encapsulates
 // statistics about communication with its followers
 type LeaderStats struct {
+	leaderStats
+	sync.Mutex
+}
+
+type leaderStats struct {
 	// Leader is the ID of the leader in the etcd cluster.
 	// TODO(jonboulle): clarify that these are IDs, not names
 	Leader    string                    `json:"leader"`
 	Followers map[string]*FollowerStats `json:"followers"`
-
-	sync.Mutex
 }
 
 // NewLeaderStats generates a new LeaderStats with the given id as leader
 func NewLeaderStats(id string) *LeaderStats {
 	return &LeaderStats{
-		Leader:    id,
-		Followers: make(map[string]*FollowerStats),
+		leaderStats: leaderStats{
+			Leader:    id,
+			Followers: make(map[string]*FollowerStats),
+		},
 	}
 }
 
 func (ls *LeaderStats) JSON() []byte {
 	ls.Lock()
-	stats := *ls
+	stats := ls.leaderStats
 	ls.Unlock()
 	b, err := json.Marshal(stats)
 	// TODO(jonboulle): appropriate error handling?

etcdserver/stats/server.go

@@ -26,6 +26,26 @@ import (
 // ServerStats encapsulates various statistics about an EtcdServer and its
 // communication with other members of the cluster
 type ServerStats struct {
+	serverStats
+	sync.Mutex
+}
+
+func NewServerStats(name, id string) *ServerStats {
+	ss := &ServerStats{
+		serverStats: serverStats{
+			Name: name,
+			ID:   id,
+		},
+	}
+	now := time.Now()
+	ss.StartTime = now
+	ss.LeaderInfo.StartTime = now
+	ss.sendRateQueue = &statsQueue{back: -1}
+	ss.recvRateQueue = &statsQueue{back: -1}
+	return ss
+}
+
+type serverStats struct {
 	Name string `json:"name"`
 	// ID is the raft ID of the node.
 	// TODO(jonboulle): use ID instead of name?
@@ -49,17 +69,15 @@ type ServerStats struct {
 
 	sendRateQueue *statsQueue
 	recvRateQueue *statsQueue
-
-	sync.Mutex
 }
 
 func (ss *ServerStats) JSON() []byte {
 	ss.Lock()
-	stats := *ss
+	stats := ss.serverStats
 	ss.Unlock()
 	stats.LeaderInfo.Uptime = time.Since(stats.LeaderInfo.StartTime).String()
-	stats.SendingPkgRate, stats.SendingBandwidthRate = stats.SendRates()
-	stats.RecvingPkgRate, stats.RecvingBandwidthRate = stats.RecvRates()
+	stats.SendingPkgRate, stats.SendingBandwidthRate = stats.sendRateQueue.Rate()
+	stats.RecvingPkgRate, stats.RecvingBandwidthRate = stats.recvRateQueue.Rate()
 	b, err := json.Marshal(stats)
 	// TODO(jonboulle): appropriate error handling?
 	if err != nil {
@@ -68,32 +86,6 @@ func (ss *ServerStats) JSON() []byte {
 	return b
 }
 
-// Initialize clears the statistics of ServerStats and resets its start time
-func (ss *ServerStats) Initialize() {
-	if ss == nil {
-		return
-	}
-	now := time.Now()
-	ss.StartTime = now
-	ss.LeaderInfo.StartTime = now
-	ss.sendRateQueue = &statsQueue{
-		back: -1,
-	}
-	ss.recvRateQueue = &statsQueue{
-		back: -1,
-	}
-}
-
-// RecvRates calculates and returns the rate of received append requests
-func (ss *ServerStats) RecvRates() (float64, float64) {
-	return ss.recvRateQueue.Rate()
-}
-
-// SendRates calculates and returns the rate of sent append requests
-func (ss *ServerStats) SendRates() (float64, float64) {
-	return ss.sendRateQueue.Rate()
-}
-
 // RecvAppendReq updates the ServerStats in response to an AppendRequest
 // from the given leader being received
 func (ss *ServerStats) RecvAppendReq(leader string, reqSize int) {

glide.lock

@@ -1,5 +1,5 @@
-hash: 4248f4a610b399df10cab942b0b3ef8a6d7db9c942bafd115f25d05293571658
-updated: 2017-04-24T16:15:17.066493631-07:00
+hash: cee1f2629857e9c2384ad89ff6014db09498c9af53771e5144ad3a4b510ff00e
+updated: 2017-05-30T10:29:08.22609283-07:00
 imports:
 - name: github.com/beorn7/perks
   version: 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9
@@ -12,7 +12,7 @@ imports:
 - name: github.com/cockroachdb/cmux
   version: 112f0506e7743d64a6eb8fedbcff13d9979bbf92
 - name: github.com/coreos/go-semver
-  version: 568e959cd89871e61434c1143528d9162da89ef2
+  version: 8ab6407b697782a06568d4b7f1db25550ec2e4c6
   subpackages:
   - semver
 - name: github.com/coreos/go-systemd
@@ -27,7 +27,7 @@ imports:
   - capnslog
   - dlopen
 - name: github.com/cpuguy83/go-md2man
-  version: a65d4d2de4d5f7c74868dfa9b202a3c8be315aaa
+  version: bcc0a711c5e6bbe72c7cb13d81c7109b45267fd2
   subpackages:
   - md2man
 - name: github.com/dgrijalva/jwt-go
@@ -35,7 +35,7 @@ imports:
 - name: github.com/dustin/go-humanize
   version: 8929fe90cee4b2cb9deb468b51fb34eba64d1bf0
 - name: github.com/ghodss/yaml
-  version: 73d445a93680fa1a78ae23a5839bad48f32ba1ee
+  version: 0ca9ea5df5451ffdf184b4428c902747c2c11cd7
 - name: github.com/gogo/protobuf
   version: 909568be09de550ed094403c2bf8a261b5bb730a
   subpackages:
@@ -64,7 +64,7 @@ imports:
 - name: github.com/jonboulle/clockwork
   version: 2eee05ed794112d45db504eb05aa693efd2b8b09
 - name: github.com/kr/pty
-  version: f7ee69f31298ecbe5d2b349c711e2547a617d398
+  version: 2c10821df3c3cf905230d078702dfbe9404c9b23
 - name: github.com/mattn/go-runewidth
   version: 9e777a8366cce605130a531d2cd6363d07ad7317
   subpackages:
@@ -94,9 +94,7 @@ imports:
   subpackages:
   - xfs
 - name: github.com/russross/blackfriday
-  version: b253417e1cb644d645a0a3bb1fa5034c8030127c
-- name: github.com/shurcooL/sanitized_anchor_name
-  version: 79c90efaf01eddc01945af5bc1797859189b830b
+  version: 0ba0f2b6ed7c475a92e4df8641825cb7a11d1fa3
 - name: github.com/spf13/cobra
   version: 1c44ec8d3f1552cac48999f9306da23c4d8a288b
 - name: github.com/spf13/pflag
@@ -129,7 +127,7 @@ imports:
   subpackages:
   - unix
 - name: golang.org/x/text
-  version: a9a820217f98f7c8a207ec1e45a874e1fe12c478
+  version: 4ee4af566555f5fbe026368b75596286a312663a
   subpackages:
   - secure/bidirule
   - transform

glide.yaml

@@ -7,7 +7,7 @@ import:
 - package: github.com/cockroachdb/cmux
   version: 112f0506e7743d64a6eb8fedbcff13d9979bbf92
 - package: github.com/coreos/go-semver
-  version: 568e959cd89871e61434c1143528d9162da89ef2
+  version: v0.2.0
   subpackages:
   - semver
 - package: github.com/coreos/go-systemd
@@ -23,7 +23,7 @@ import:
 - package: github.com/dustin/go-humanize
   version: 8929fe90cee4b2cb9deb468b51fb34eba64d1bf0
 - package: github.com/ghodss/yaml
-  version: 73d445a93680fa1a78ae23a5839bad48f32ba1ee
+  version: v1.0.0
 - package: github.com/gogo/protobuf
   version: v0.3
   subpackages:
@@ -48,7 +48,7 @@ import:
 - package: github.com/jonboulle/clockwork
   version: v0.1.0
 - package: github.com/kr/pty
-  version: f7ee69f31298ecbe5d2b349c711e2547a617d398
+  version: v1.0.0
 - package: github.com/olekukonko/tablewriter
   version: a0225b3f23b5ce0cbec6d7a66a968f8a59eca9c4
 - package: github.com/mattn/go-runewidth

integration/cluster.go

@@ -554,7 +554,7 @@ func (m *member) listenGRPC() error {
 		l.Close()
 		return err
 	}
-	m.grpcAddr = m.grpcBridge.URL()
+	m.grpcAddr = schemeFromTLSInfo(m.ClientTLSInfo) + "://" + m.grpcBridge.inaddr
 	m.grpcListener = l
 	return nil
 }

integration/metrics_test.go

@@ -0,0 +1,97 @@
+// Copyright 2017 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package integration
+
+import (
+	"context"
+	"strconv"
+	"testing"
+	"time"
+
+	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
+	"github.com/coreos/etcd/pkg/testutil"
+)
+
+// TestMetricDbSizeBoot checks that the db size metric is set on boot.
+func TestMetricDbSizeBoot(t *testing.T) {
+	defer testutil.AfterTest(t)
+	clus := NewClusterV3(t, &ClusterConfig{Size: 1})
+	defer clus.Terminate(t)
+
+	v, err := clus.Members[0].Metric("etcd_debugging_mvcc_db_total_size_in_bytes")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if v == "0" {
+		t.Fatalf("expected non-zero, got %q", v)
+	}
+}
+
+// TestMetricDbSizeDefrag checks that the db size metric is set after defrag.
+func TestMetricDbSizeDefrag(t *testing.T) {
+	defer testutil.AfterTest(t)
+	clus := NewClusterV3(t, &ClusterConfig{Size: 1})
+	defer clus.Terminate(t)
+
+	kvc := toGRPC(clus.Client(0)).KV
+	mc := toGRPC(clus.Client(0)).Maintenance
+
+	// expand the db size
+	numPuts := 10
+	putreq := &pb.PutRequest{Key: []byte("k"), Value: make([]byte, 4096)}
+	for i := 0; i < numPuts; i++ {
+		if _, err := kvc.Put(context.TODO(), putreq); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// wait for backend txn sync
+	time.Sleep(500 * time.Millisecond)
+
+	beforeDefrag, err := clus.Members[0].Metric("etcd_debugging_mvcc_db_total_size_in_bytes")
+	if err != nil {
+		t.Fatal(err)
+	}
+	bv, err := strconv.Atoi(beforeDefrag)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if expected := numPuts * len(putreq.Value); bv < expected {
+		t.Fatalf("expected db size greater than %d, got %d", expected, bv)
+	}
+
+	// clear out historical keys
+	creq := &pb.CompactionRequest{Revision: int64(numPuts), Physical: true}
+	if _, err := kvc.Compact(context.TODO(), creq); err != nil {
+		t.Fatal(err)
+	}
+
+	// defrag should give freed space back to fs
+	mc.Defragment(context.TODO(), &pb.DefragmentRequest{})
+	afterDefrag, err := clus.Members[0].Metric("etcd_debugging_mvcc_db_total_size_in_bytes")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	av, err := strconv.Atoi(afterDefrag)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if bv <= av {
+		t.Fatalf("expected less than %d, got %d after defrag", bv, av)
+	}
+}

integration/v3_grpc_test.go

@@ -328,6 +328,58 @@ func TestV3TxnRevision(t *testing.T) {
 	}
 }
 
+// Testv3TxnCmpHeaderRev tests that the txn header revision is set as expected
+// when compared to the Succeeded field in the txn response.
+func TestV3TxnCmpHeaderRev(t *testing.T) {
+	defer testutil.AfterTest(t)
+	clus := NewClusterV3(t, &ClusterConfig{Size: 1})
+	defer clus.Terminate(t)
+
+	kvc := toGRPC(clus.RandClient()).KV
+
+	for i := 0; i < 10; i++ {
+		// Concurrently put a key with a txn comparing on it.
+		revc := make(chan int64, 1)
+		go func() {
+			defer close(revc)
+			pr := &pb.PutRequest{Key: []byte("k"), Value: []byte("v")}
+			presp, err := kvc.Put(context.TODO(), pr)
+			if err != nil {
+				t.Fatal(err)
+			}
+			revc <- presp.Header.Revision
+		}()
+
+		// The read-only txn uses the optimized readindex server path.
+		txnget := &pb.RequestOp{Request: &pb.RequestOp_RequestRange{
+			RequestRange: &pb.RangeRequest{Key: []byte("k")}}}
+		txn := &pb.TxnRequest{Success: []*pb.RequestOp{txnget}}
+		// i = 0 /\ Succeeded => put followed txn
+		cmp := &pb.Compare{
+			Result:      pb.Compare_EQUAL,
+			Target:      pb.Compare_VERSION,
+			Key:         []byte("k"),
+			TargetUnion: &pb.Compare_Version{Version: int64(i)},
+		}
+		txn.Compare = append(txn.Compare, cmp)
+
+		tresp, err := kvc.Txn(context.TODO(), txn)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		prev := <-revc
+		// put followed txn; should eval to false
+		if prev > tresp.Header.Revision && !tresp.Succeeded {
+			t.Errorf("#%d: got else but put rev %d followed txn rev (%+v)", i, prev, tresp)
+		}
+		// txn follows put; should eval to true
+		if tresp.Header.Revision >= prev && tresp.Succeeded {
+			t.Errorf("#%d: got then but put rev %d preceded txn (%+v)", i, prev, tresp)
+		}
+	}
+}
+
 // TestV3PutIgnoreValue ensures that writes with ignore_value overwrites with previous key-value pair.
 func TestV3PutIgnoreValue(t *testing.T) {
 	defer testutil.AfterTest(t)
@@ -1402,9 +1454,9 @@ func TestTLSReloadAtomicReplace(t *testing.T) {
 	defer os.RemoveAll(certsDirExp)
 
 	cloneFunc := func() transport.TLSInfo {
-		tlsInfo, err := copyTLSFiles(testTLSInfo, certsDir)
-		if err != nil {
-			t.Fatal(err)
+		tlsInfo, terr := copyTLSFiles(testTLSInfo, certsDir)
+		if terr != nil {
+			t.Fatal(terr)
 		}
 		if _, err = copyTLSFiles(testTLSInfoExpired, certsDirExp); err != nil {
 			t.Fatal(err)
@@ -1448,9 +1500,9 @@ func TestTLSReloadCopy(t *testing.T) {
 	defer os.RemoveAll(certsDir)
 
 	cloneFunc := func() transport.TLSInfo {
-		tlsInfo, err := copyTLSFiles(testTLSInfo, certsDir)
-		if err != nil {
-			t.Fatal(err)
+		tlsInfo, terr := copyTLSFiles(testTLSInfo, certsDir)
+		if terr != nil {
+			t.Fatal(terr)
 		}
 		return tlsInfo
 	}

mvcc/index.go

@@ -29,7 +29,9 @@ type index interface {
 	RangeSince(key, end []byte, rev int64) []revision
 	Compact(rev int64) map[revision]struct{}
 	Equal(b index) bool
+
 	Insert(ki *keyIndex)
+	KeyIndex(ki *keyIndex) *keyIndex
 }
 
 type treeIndex struct {
@@ -60,18 +62,27 @@ func (ti *treeIndex) Put(key []byte, rev revision) {
 
 func (ti *treeIndex) Get(key []byte, atRev int64) (modified, created revision, ver int64, err error) {
 	keyi := &keyIndex{key: key}
-
 	ti.RLock()
 	defer ti.RUnlock()
-	item := ti.tree.Get(keyi)
-	if item == nil {
+	if keyi = ti.keyIndex(keyi); keyi == nil {
 		return revision{}, revision{}, 0, ErrRevisionNotFound
 	}
-
-	keyi = item.(*keyIndex)
 	return keyi.get(atRev)
 }
 
+func (ti *treeIndex) KeyIndex(keyi *keyIndex) *keyIndex {
+	ti.RLock()
+	defer ti.RUnlock()
+	return ti.keyIndex(keyi)
+}
+
+func (ti *treeIndex) keyIndex(keyi *keyIndex) *keyIndex {
+	if item := ti.tree.Get(keyi); item != nil {
+		return item.(*keyIndex)
+	}
+	return nil
+}
+
 func (ti *treeIndex) Range(key, end []byte, atRev int64) (keys [][]byte, revs []revision) {
 	if end == nil {
 		rev, _, _, err := ti.Get(key, atRev)

mvcc/kv.go

@@ -93,7 +93,9 @@ func (trw *txnReadWrite) DeleteRange(key, end []byte) (n, rev int64) { panic("un
 func (trw *txnReadWrite) Put(key, value []byte, lease lease.LeaseID) (rev int64) {
 	panic("unexpected Put")
 }
-func (trw *txnReadWrite) Changes() []mvccpb.KeyValue { panic("unexpected Changes") }
+func (trw *txnReadWrite) Changes() []mvccpb.KeyValue { return nil }
+
+func NewReadOnlyTxnWrite(txn TxnRead) TxnWrite { return &txnReadWrite{txn} }
 
 type KV interface {
 	ReadView

mvcc/kvstore.go

@@ -33,13 +33,6 @@ var (
 	keyBucketName  = []byte("key")
 	metaBucketName = []byte("meta")
 
-	// markedRevBytesLen is the byte length of marked revision.
-	// The first `revBytesLen` bytes represents a normal revision. The last
-	// one byte is the mark.
-	markedRevBytesLen      = revBytesLen + 1
-	markBytePosition       = markedRevBytesLen - 1
-	markTombstone     byte = 't'
-
 	consistentIndexKeyName  = []byte("consistent_index")
 	scheduledCompactKeyName = []byte("scheduledCompactRev")
 	finishedCompactKeyName  = []byte("finishedCompactRev")
@@ -52,6 +45,17 @@ var (
 	plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "mvcc")
 )
 
+const (
+	// markedRevBytesLen is the byte length of marked revision.
+	// The first `revBytesLen` bytes represents a normal revision. The last
+	// one byte is the mark.
+	markedRevBytesLen      = revBytesLen + 1
+	markBytePosition       = markedRevBytesLen - 1
+	markTombstone     byte = 't'
+)
+
+var restoreChunkKeys = 10000 // non-const for testing
+
 // ConsistentIndexGetter is an interface that wraps the Get method.
 // Consistent index is the offset of an entry in a consistent replicated log.
 type ConsistentIndexGetter interface {
@@ -241,67 +245,53 @@ func (s *store) Restore(b backend.Backend) error {
 }
 
 func (s *store) restore() error {
+	reportDbTotalSizeInBytesMu.Lock()
+	b := s.b
+	reportDbTotalSizeInBytes = func() float64 { return float64(b.Size()) }
+	reportDbTotalSizeInBytesMu.Unlock()
+
 	min, max := newRevBytes(), newRevBytes()
 	revToBytes(revision{main: 1}, min)
 	revToBytes(revision{main: math.MaxInt64, sub: math.MaxInt64}, max)
 
 	keyToLease := make(map[string]lease.LeaseID)
 
-	// use an unordered map to hold the temp index data to speed up
-	// the initial key index recovery.
-	// we will convert this unordered map into the tree index later.
-	unordered := make(map[string]*keyIndex, 100000)
-
 	// restore index
 	tx := s.b.BatchTx()
 	tx.Lock()
+
 	_, finishedCompactBytes := tx.UnsafeRange(metaBucketName, finishedCompactKeyName, nil, 0)
 	if len(finishedCompactBytes) != 0 {
 		s.compactMainRev = bytesToRev(finishedCompactBytes[0]).main
 		plog.Printf("restore compact to %d", s.compactMainRev)
 	}
-
-	// TODO: limit N to reduce max memory usage
-	keys, vals := tx.UnsafeRange(keyBucketName, min, max, 0)
-	for i, key := range keys {
-		var kv mvccpb.KeyValue
-		if err := kv.Unmarshal(vals[i]); err != nil {
-			plog.Fatalf("cannot unmarshal event: %v", err)
-		}
-
-		rev := bytesToRev(key[:revBytesLen])
-		s.currentRev = rev.main
-
-		// restore index
-		switch {
-		case isTombstone(key):
-			if ki, ok := unordered[string(kv.Key)]; ok {
-				ki.tombstone(rev.main, rev.sub)
-			}
-			delete(keyToLease, string(kv.Key))
-
-		default:
-			ki, ok := unordered[string(kv.Key)]
-			if ok {
-				ki.put(rev.main, rev.sub)
-			} else {
-				ki = &keyIndex{key: kv.Key}
-				ki.restore(revision{kv.CreateRevision, 0}, rev, kv.Version)
-				unordered[string(kv.Key)] = ki
-			}
-
-			if lid := lease.LeaseID(kv.Lease); lid != lease.NoLease {
-				keyToLease[string(kv.Key)] = lid
-			} else {
-				delete(keyToLease, string(kv.Key))
-			}
-		}
+	_, scheduledCompactBytes := tx.UnsafeRange(metaBucketName, scheduledCompactKeyName, nil, 0)
+	scheduledCompact := int64(0)
+	if len(scheduledCompactBytes) != 0 {
+		scheduledCompact = bytesToRev(scheduledCompactBytes[0]).main
 	}
 
-	// restore the tree index from the unordered index.
-	for _, v := range unordered {
-		s.kvindex.Insert(v)
+	// index keys concurrently as they're loaded in from tx
+	rkvc, revc := restoreIntoIndex(s.kvindex)
+	for {
+		keys, vals := tx.UnsafeRange(keyBucketName, min, max, int64(restoreChunkKeys))
+		if len(keys) == 0 {
+			break
+		}
+		// rkvc blocks if the total pending keys exceeds the restore
+		// chunk size to keep keys from consuming too much memory.
+		restoreChunk(rkvc, keys, vals, keyToLease)
+		if len(keys) < restoreChunkKeys {
+			// partial set implies final set
+			break
+		}
+		// next set begins after where this one ended
+		newMin := bytesToRev(keys[len(keys)-1][:revBytesLen])
+		newMin.sub++
+		revToBytes(newMin, min)
 	}
+	close(rkvc)
+	s.currentRev = <-revc
 
 	// keys in the range [compacted revision -N, compaction] might all be deleted due to compaction.
 	// the correct revision should be set to compaction revision in the case, not the largest revision
@@ -309,6 +299,9 @@ func (s *store) restore() error {
 	if s.currentRev < s.compactMainRev {
 		s.currentRev = s.compactMainRev
 	}
+	if scheduledCompact <= s.compactMainRev {
+		scheduledCompact = 0
+	}
 
 	for key, lid := range keyToLease {
 		if s.le == nil {
@@ -320,15 +313,6 @@ func (s *store) restore() error {
 		}
 	}
 
-	_, scheduledCompactBytes := tx.UnsafeRange(metaBucketName, scheduledCompactKeyName, nil, 0)
-	scheduledCompact := int64(0)
-	if len(scheduledCompactBytes) != 0 {
-		scheduledCompact = bytesToRev(scheduledCompactBytes[0]).main
-		if scheduledCompact <= s.compactMainRev {
-			scheduledCompact = 0
-		}
-	}
-
 	tx.Unlock()
 
 	if scheduledCompact != 0 {
@@ -339,6 +323,75 @@ func (s *store) restore() error {
 	return nil
 }
 
+type revKeyValue struct {
+	key  []byte
+	kv   mvccpb.KeyValue
+	kstr string
+}
+
+func restoreIntoIndex(idx index) (chan<- revKeyValue, <-chan int64) {
+	rkvc, revc := make(chan revKeyValue, restoreChunkKeys), make(chan int64, 1)
+	go func() {
+		currentRev := int64(1)
+		defer func() { revc <- currentRev }()
+		// restore the tree index from streaming the unordered index.
+		kiCache := make(map[string]*keyIndex, restoreChunkKeys)
+		for rkv := range rkvc {
+			ki, ok := kiCache[rkv.kstr]
+			// purge kiCache if many keys but still missing in the cache
+			if !ok && len(kiCache) >= restoreChunkKeys {
+				i := 10
+				for k := range kiCache {
+					delete(kiCache, k)
+					if i--; i == 0 {
+						break
+					}
+				}
+			}
+			// cache miss, fetch from tree index if there
+			if !ok {
+				ki = &keyIndex{key: rkv.kv.Key}
+				if idxKey := idx.KeyIndex(ki); idxKey != nil {
+					kiCache[rkv.kstr], ki = idxKey, idxKey
+					ok = true
+				}
+			}
+			rev := bytesToRev(rkv.key)
+			currentRev = rev.main
+			if ok {
+				if isTombstone(rkv.key) {
+					ki.tombstone(rev.main, rev.sub)
+					continue
+				}
+				ki.put(rev.main, rev.sub)
+			} else if !isTombstone(rkv.key) {
+				ki.restore(revision{rkv.kv.CreateRevision, 0}, rev, rkv.kv.Version)
+				idx.Insert(ki)
+				kiCache[rkv.kstr] = ki
+			}
+		}
+	}()
+	return rkvc, revc
+}
+
+func restoreChunk(kvc chan<- revKeyValue, keys, vals [][]byte, keyToLease map[string]lease.LeaseID) {
+	for i, key := range keys {
+		rkv := revKeyValue{key: key}
+		if err := rkv.kv.Unmarshal(vals[i]); err != nil {
+			plog.Fatalf("cannot unmarshal event: %v", err)
+		}
+		rkv.kstr = string(rkv.kv.Key)
+		if isTombstone(key) {
+			delete(keyToLease, rkv.kstr)
+		} else if lid := lease.LeaseID(rkv.kv.Lease); lid != lease.NoLease {
+			keyToLease[rkv.kstr] = lid
+		} else {
+			delete(keyToLease, rkv.kstr)
+		}
+		kvc <- rkv
+	}
+}
+
 func (s *store) Close() error {
 	close(s.stopc)
 	s.fifoSched.Stop()

mvcc/kvstore_bench_test.go

@@ -89,7 +89,8 @@ func benchmarkStoreRestore(revsPerKey int, b *testing.B) {
 	var i fakeConsistentIndex
 	be, tmpPath := backend.NewDefaultTmpBackend()
 	s := NewStore(be, &lease.FakeLessor{}, &i)
-	defer cleanup(s, be, tmpPath)
+	// use closure to capture 's' to pick up the reassignment
+	defer func() { cleanup(s, be, tmpPath) }()
 
 	// arbitrary number of bytes
 	bytesN := 64
@@ -103,7 +104,11 @@ func benchmarkStoreRestore(revsPerKey int, b *testing.B) {
 			txn.End()
 		}
 	}
+	s.Close()
+
+	b.ReportAllocs()
 	b.ResetTimer()
+	s = NewStore(be, &lease.FakeLessor{}, &i)
 }
 
 func BenchmarkStoreRestoreRevs1(b *testing.B) {

mvcc/kvstore_test.go

@@ -17,7 +17,9 @@ package mvcc
 import (
 	"crypto/rand"
 	"encoding/binary"
+	"fmt"
 	"math"
+	mrand "math/rand"
 	"os"
 	"reflect"
 	"testing"
@@ -373,9 +375,11 @@ func TestStoreRestore(t *testing.T) {
 		t.Fatal(err)
 	}
 	b.tx.rangeRespc <- rangeResp{[][]byte{finishedCompactKeyName}, [][]byte{newTestRevBytes(revision{3, 0})}}
-	b.tx.rangeRespc <- rangeResp{[][]byte{putkey, delkey}, [][]byte{putkvb, delkvb}}
 	b.tx.rangeRespc <- rangeResp{[][]byte{scheduledCompactKeyName}, [][]byte{newTestRevBytes(revision{3, 0})}}
 
+	b.tx.rangeRespc <- rangeResp{[][]byte{putkey, delkey}, [][]byte{putkvb, delkvb}}
+	b.tx.rangeRespc <- rangeResp{nil, nil}
+
 	s.restore()
 
 	if s.compactMainRev != 3 {
@@ -386,8 +390,8 @@ func TestStoreRestore(t *testing.T) {
 	}
 	wact := []testutil.Action{
 		{"range", []interface{}{metaBucketName, finishedCompactKeyName, []byte(nil), int64(0)}},
-		{"range", []interface{}{keyBucketName, newTestRevBytes(revision{1, 0}), newTestRevBytes(revision{math.MaxInt64, math.MaxInt64}), int64(0)}},
 		{"range", []interface{}{metaBucketName, scheduledCompactKeyName, []byte(nil), int64(0)}},
+		{"range", []interface{}{keyBucketName, newTestRevBytes(revision{1, 0}), newTestRevBytes(revision{math.MaxInt64, math.MaxInt64}), int64(restoreChunkKeys)}},
 	}
 	if g := b.tx.Action(); !reflect.DeepEqual(g, wact) {
 		t.Errorf("tx actions = %+v, want %+v", g, wact)
@@ -399,6 +403,7 @@ func TestStoreRestore(t *testing.T) {
 	}
 	ki := &keyIndex{key: []byte("foo"), modified: revision{5, 0}, generations: gens}
 	wact = []testutil.Action{
+		{"keyIndex", []interface{}{ki}},
 		{"insert", []interface{}{ki}},
 	}
 	if g := fi.Action(); !reflect.DeepEqual(g, wact) {
@@ -406,6 +411,56 @@ func TestStoreRestore(t *testing.T) {
 	}
 }
 
+func TestRestoreDelete(t *testing.T) {
+	oldChunk := restoreChunkKeys
+	restoreChunkKeys = mrand.Intn(3) + 2
+	defer func() { restoreChunkKeys = oldChunk }()
+
+	b, tmpPath := backend.NewDefaultTmpBackend()
+	s := NewStore(b, &lease.FakeLessor{}, nil)
+	defer os.Remove(tmpPath)
+
+	keys := make(map[string]struct{})
+	for i := 0; i < 20; i++ {
+		ks := fmt.Sprintf("foo-%d", i)
+		k := []byte(ks)
+		s.Put(k, []byte("bar"), lease.NoLease)
+		keys[ks] = struct{}{}
+		switch mrand.Intn(3) {
+		case 0:
+			// put random key from past via random range on map
+			ks = fmt.Sprintf("foo-%d", mrand.Intn(i+1))
+			s.Put([]byte(ks), []byte("baz"), lease.NoLease)
+			keys[ks] = struct{}{}
+		case 1:
+			// delete random key via random range on map
+			for k := range keys {
+				s.DeleteRange([]byte(k), nil)
+				delete(keys, k)
+				break
+			}
+		}
+	}
+	s.Close()
+
+	s = NewStore(b, &lease.FakeLessor{}, nil)
+	defer s.Close()
+	for i := 0; i < 20; i++ {
+		ks := fmt.Sprintf("foo-%d", i)
+		r, err := s.Range([]byte(ks), nil, RangeOptions{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if _, ok := keys[ks]; ok {
+			if len(r.KVs) == 0 {
+				t.Errorf("#%d: expected %q, got deleted", i, ks)
+			}
+		} else if len(r.KVs) != 0 {
+			t.Errorf("#%d: expected deleted, got %q", i, ks)
+		}
+	}
+}
+
 func TestRestoreContinueUnfinishedCompaction(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
 	s0 := NewStore(b, &lease.FakeLessor{}, nil)
@@ -644,6 +699,11 @@ func (i *fakeIndex) Insert(ki *keyIndex) {
 	i.Recorder.Record(testutil.Action{Name: "insert", Params: []interface{}{ki}})
 }
 
+func (i *fakeIndex) KeyIndex(ki *keyIndex) *keyIndex {
+	i.Recorder.Record(testutil.Action{Name: "keyIndex", Params: []interface{}{ki}})
+	return nil
+}
+
 func createBytesSlice(bytesN, sliceN int) [][]byte {
 	rs := [][]byte{}
 	for len(rs) != sliceN {

mvcc/kvstore_txn.go

@@ -105,7 +105,6 @@ func (tw *storeTxnWrite) End() {
 	if len(tw.changes) != 0 {
 		tw.s.revMu.Unlock()
 	}
-	dbTotalSize.Set(float64(tw.s.b.Size()))
 	tw.s.mu.RUnlock()
 }
 

mvcc/metrics.go

@@ -15,6 +15,8 @@
 package mvcc
 
 import (
+	"sync"
+
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -129,12 +131,21 @@ var (
 			Buckets: prometheus.ExponentialBuckets(100, 2, 14),
 		})
 
-	dbTotalSize = prometheus.NewGauge(prometheus.GaugeOpts{
+	dbTotalSize = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
 		Namespace: "etcd_debugging",
 		Subsystem: "mvcc",
 		Name:      "db_total_size_in_bytes",
 		Help:      "Total size of the underlying database in bytes.",
-	})
+	},
+		func() float64 {
+			reportDbTotalSizeInBytesMu.RLock()
+			defer reportDbTotalSizeInBytesMu.RUnlock()
+			return reportDbTotalSizeInBytes()
+		},
+	)
+	// overridden by mvcc initialization
+	reportDbTotalSizeInBytesMu sync.RWMutex
+	reportDbTotalSizeInBytes   func() float64 = func() float64 { return 0 }
 )
 
 func init() {

pkg/fileutil/fileutil_test.go

@@ -127,6 +127,11 @@ func TestZeroToEnd(t *testing.T) {
 	}
 	defer f.Close()
 
+	// Ensure 0 size is a nop so zero-to-end on an empty file won't give EINVAL.
+	if err = ZeroToEnd(f); err != nil {
+		t.Fatal(err)
+	}
+
 	b := make([]byte, 1024)
 	for i := range b {
 		b[i] = 12

pkg/fileutil/preallocate.go

@@ -25,6 +25,10 @@ import (
 // If the operation is unsupported, no error will be returned.
 // Otherwise, the error encountered will be returned.
 func Preallocate(f *os.File, sizeInBytes int64, extendFile bool) error {
+	if sizeInBytes == 0 {
+		// fallocate will return EINVAL if length is 0; skip
+		return nil
+	}
 	if extendFile {
 		return preallocExtend(f, sizeInBytes)
 	}

pkg/osutil/signal.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// +build !linux
+// +build !linux cov
 
 package osutil
 

pkg/osutil/signal_linux.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// +build linux
+// +build linux,!cov
 
 package osutil
 

rafthttp/functional_test.go

@@ -140,9 +140,7 @@ func TestSendMessageWhenStreamIsBroken(t *testing.T) {
 }
 
 func newServerStats() *stats.ServerStats {
-	ss := &stats.ServerStats{}
-	ss.Initialize()
-	return ss
+	return stats.NewServerStats("", "")
 }
 
 func waitStreamWorking(p *peer) bool {

rafthttp/http.go

@@ -183,7 +183,8 @@ func (h *snapshotHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 
 	dec := &messageDecoder{r: r.Body}
-	m, err := dec.decode()
+	// let snapshots be very large since they can exceed 512MB for large installations
+	m, err := dec.decodeLimit(uint64(1 << 63))
 	if err != nil {
 		msg := fmt.Sprintf("failed to decode raft message (%v)", err)
 		plog.Errorf(msg)

rafthttp/msg_codec.go

@@ -48,12 +48,16 @@ var (
 )
 
 func (dec *messageDecoder) decode() (raftpb.Message, error) {
+	return dec.decodeLimit(readBytesLimit)
+}
+
+func (dec *messageDecoder) decodeLimit(numBytes uint64) (raftpb.Message, error) {
 	var m raftpb.Message
 	var l uint64
 	if err := binary.Read(dec.r, binary.BigEndian, &l); err != nil {
 		return m, err
 	}
-	if l > readBytesLimit {
+	if l > numBytes {
 		return m, ErrExceedSizeLimit
 	}
 	buf := make([]byte, int(l))

rafthttp/transport_test.go

@@ -30,12 +30,10 @@ import (
 // TestTransportSend tests that transport can send messages using correct
 // underlying peer, and drop local or unknown-target messages.
 func TestTransportSend(t *testing.T) {
-	ss := &stats.ServerStats{}
-	ss.Initialize()
 	peer1 := newFakePeer()
 	peer2 := newFakePeer()
 	tr := &Transport{
-		ServerStats: ss,
+		ServerStats: stats.NewServerStats("", ""),
 		peers:       map[types.ID]Peer{types.ID(1): peer1, types.ID(2): peer2},
 	}
 	wmsgsIgnored := []raftpb.Message{
@@ -67,12 +65,10 @@ func TestTransportSend(t *testing.T) {
 }
 
 func TestTransportCutMend(t *testing.T) {
-	ss := &stats.ServerStats{}
-	ss.Initialize()
 	peer1 := newFakePeer()
 	peer2 := newFakePeer()
 	tr := &Transport{
-		ServerStats: ss,
+		ServerStats: stats.NewServerStats("", ""),
 		peers:       map[types.ID]Peer{types.ID(1): peer1, types.ID(2): peer2},
 	}
 

scripts/build-aci

@@ -36,6 +36,7 @@ fi
 acbuild --debug begin
 
 TMPHOSTS="$(mktemp)"
+ACI_ARCH="$(go2aci ${GOARCH})"
 
 acbuildEnd() {
     rm "$TMPHOSTS"
@@ -71,7 +72,7 @@ acbuild --debug port add peer tcp 2380
 
 acbuild --debug copy "$TMPHOSTS" /etc/hosts
 
-acbuild --debug label add arch $(go2aci ${GOARCH})
+acbuild --debug label add arch "${ACI_ARCH}"
 
 # mkdir default data-dir
 mkdir -p .acbuild/currentaci/rootfs/var/lib/etcd
@@ -80,4 +81,4 @@ mkdir -p .acbuild/currentaci/rootfs/var/lib/etcd
 ln -s ./usr/local/bin/etcd .acbuild/currentaci/rootfs/etcd
 ln -s ./usr/local/bin/etcdctl .acbuild/currentaci/rootfs/etcdctl
 
-acbuild --debug write --overwrite $BUILDDIR/etcd-${1}-linux-${GOARCH}.aci
+acbuild --debug write --overwrite $BUILDDIR/etcd-${1}-linux-${ACI_ARCH}.aci

scripts/build-docker

@@ -17,8 +17,7 @@ if [ -z ${BINARYDIR} ]; then
 	BINARYDIR="${RELEASE}"
 	TARFILE="${RELEASE}.tar.gz"
 	TARURL="https://github.com/coreos/etcd/releases/download/${1}/${TARFILE}"
-	curl -f -L -o ${TARFILE} ${TARURL}
-	if [ $? != 0 ]; then
+	if ! curl -f -L -o ${TARFILE} ${TARURL} ; then
 		echo "Failed to download ${TARURL}."
 		exit 1
 	fi

scripts/genproto.sh

@@ -57,7 +57,7 @@ popd
 
 for dir in ${DIRS}; do
 	pushd ${dir}
-		protoc --gofast_out=plugins=grpc,import_prefix=github.com/coreos/:. -I=.:"${GOGOPROTO_PATH}":"${COREOS_ROOT}":"${GRPC_GATEWAY_ROOT}/third_party/googleapis" *.proto
+		protoc --gofast_out=plugins=grpc,import_prefix=github.com/coreos/:. -I=".:${GOGOPROTO_PATH}:${COREOS_ROOT}:${GRPC_GATEWAY_ROOT}/third_party/googleapis" *.proto
 		sed -i.bak -E "s/github\.com\/coreos\/(gogoproto|github\.com|golang\.org|google\.golang\.org)/\1/g" *.pb.go
 		sed -i.bak -E 's/github\.com\/coreos\/(errors|fmt|io)/\1/g' *.pb.go
 		sed -i.bak -E 's/import _ \"gogoproto\"//g' *.pb.go
@@ -68,6 +68,8 @@ for dir in ${DIRS}; do
 	popd
 done
 
+# remove old swagger files so it's obvious whether the files fail to generate
+rm -rf Documentation/dev-guide/apispec/swagger/*json
 for pb in etcdserverpb/rpc api/v3lock/v3lockpb/v3lock api/v3election/v3electionpb/v3election; do
 	protobase="etcdserver/${pb}"
 	protoc -I. \
@@ -93,8 +95,9 @@ for pb in etcdserverpb/rpc api/v3lock/v3lockpb/v3lock api/v3election/v3electionp
 	go fmt ${gwfile}
 	mv ${gwfile} ${pkgpath}/gw/
 	rm -f ./etcdserver/${pb}*.bak
+	swaggerName=`basename ${pb}`
 	mv	Documentation/dev-guide/apispec/swagger/etcdserver/${pb}.swagger.json \
-		Documentation/dev-guide/apispec/swagger/${name}.swagger.json
+		Documentation/dev-guide/apispec/swagger/${swaggerName}.swagger.json
 done
 rm -rf Documentation/dev-guide/apispec/swagger/etcdserver/
 
@@ -107,7 +110,7 @@ rm -rf Documentation/dev-guide/apispec/swagger/etcdserver/
 if [ "$1" = "-g" ]; then
 	echo "protodoc is auto-generating grpc API reference documentation..."
 	go get -v -u github.com/coreos/protodoc
-	SHA_PROTODOC="e901a6d4fa64c815acf117864521a8dadb11390f"
+	SHA_PROTODOC="4372ee725035a208404e2d5465ba921469decc32"
 	PROTODOC_PATH="${GOPATH}/src/github.com/coreos/protodoc"
 	pushd "${PROTODOC_PATH}"
 		git reset --hard "${SHA_PROTODOC}"

scripts/release.sh

@@ -21,7 +21,7 @@ if ! command -v docker >/dev/null; then
     exit 1
 fi
 
-ETCD_ROOT=$(dirname "${BASH_SOURCE}")/..
+ETCD_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 
 pushd ${ETCD_ROOT} >/dev/null
 	echo Building etcd binary...

scripts/updatedep.sh

@@ -43,7 +43,7 @@ pushd "${GLIDE_VC_ROOT}"
 popd
 
 if [ -n "$1" ]; then
-	echo "glide get on $(echo $1)"
+	echo "glide get on $1"
 	matches=`grep "name: $1" glide.lock`
 	if [ ! -z "$matches" ]; then
 		echo "glide update on $1"

test

@@ -16,6 +16,11 @@ set -e
 
 source ./build
 
+# build before setting up test GOPATH
+if [[ "${PASSES}" == *"functional"* ]]; then
+	./tools/functional-tester/build
+fi
+
 # build tests with vendored dependencies
 etcd_setup_gopath
 
@@ -23,6 +28,8 @@ if [ -z "$PASSES" ]; then
 	PASSES="fmt bom dep compile build unit"
 fi
 
+USERPKG=${PKG:-}
+
 # Invoke ./cover for HTML output
 COVER=${COVER:-"-cover"}
 
@@ -31,14 +38,31 @@ IGNORE_PKGS="(cmd/|etcdserverpb|rafttest|gopath.proto|v3lockpb|v3electionpb)"
 INTEGRATION_PKGS="(integration|e2e|contrib|functional-tester)"
 
 # all github.com/coreos/etcd/whatever pkgs that are not auto-generated / tools
-PKGS=`find . -name \*.go | while read a; do dirname $a; done | sort | uniq | egrep -v "$IGNORE_PKGS" | egrep -v "(tools/|contrib/|e2e|pb)" | sed "s|\.|${REPO_PATH}|g"`
+PKGS=`find . -name \*.go | while read a; do dirname $a; done | sort | uniq | egrep -v "$IGNORE_PKGS" | egrep -v "(tools/|contrib/|e2e|pb)" | sed "s|\.|${REPO_PATH}|g" | xargs echo`
 # pkg1,pkg2,pkg3
-PKGS_COMMA=`echo ${PKGS} | sed 's/ /,/g'`
+PKGS_COMMA=${PKGS// /,}
 
 TEST_PKGS=`find . -name \*_test.go | while read a; do dirname $a; done | sort | uniq | egrep -v "$IGNORE_PKGS" | sed "s|\./||g"`
-FORMATTABLE=`find . -name \*.go | while read a; do echo $(dirname $a)/"*.go"; done | sort | uniq | egrep -v "$IGNORE_PKGS" | sed "s|\./||g"`
+FORMATTABLE=`find . -name \*.go | while read a; do echo "$(dirname $a)/*.go"; done | sort | uniq | egrep -v "$IGNORE_PKGS" | sed "s|\./||g"`
 TESTABLE_AND_FORMATTABLE=`echo "$TEST_PKGS" | egrep -v "$INTEGRATION_PKGS"`
 
+# check if user provided PKG override
+if [ -z "${USERPKG}" ]; then
+	TEST=$TESTABLE_AND_FORMATTABLE
+	FMT=$FORMATTABLE
+else
+	# strip out leading dotslashes and trailing slashes from PKG=./foo/
+	TEST=${USERPKG/#./}
+	TEST=${TEST/#\//}
+	TEST=${TEST/%\//}
+	# only run gofmt on packages provided by user
+	FMT="$TEST"
+fi
+
+# split TEST into an array and prepend REPO_PATH to each local package
+split=(${TEST// / })
+TEST=${split/#/${REPO_PATH}/}
+
 # TODO: 'client' pkg fails with gosimple from generated files
 # TODO: 'rafttest' is failing with unused
 STATIC_ANALYSIS_PATHS=`find . -name \*.go | while read a; do dirname $a; done | sort | uniq | egrep -v "$IGNORE_PKGS" | grep -v 'client'`
@@ -47,25 +71,6 @@ if [ -z "$GOARCH" ]; then
 	GOARCH=$(go env GOARCH);
 fi
 
-# user has not provided PKG override
-if [ -z "$PKG" ]; then
-	TEST=$TESTABLE_AND_FORMATTABLE
-	FMT=$FORMATTABLE
-
-# user has provided PKG override
-else
-	# strip out leading dotslashes and trailing slashes from PKG=./foo/
-	TEST=${PKG/#./}
-	TEST=${TEST/#\//}
-	TEST=${TEST/%\//}
-
-	# only run gofmt on packages provided by user
-	FMT="$TEST"
-fi
-
-# split TEST into an array and prepend REPO_PATH to each local package
-split=(${TEST// / })
-TEST=${split[@]/#/${REPO_PATH}/}
 
 # determine whether target supports race detection
 if [ "$GOARCH" == "amd64" ]; then
@@ -87,6 +92,35 @@ function integration_pass {
 	go test -timeout 1m -v ${RACE} -cpu 1,2,4 -run=Example $@ ${TEST}
 }
 
+function functional_pass {
+	for a in 1 2 3; do
+		mkdir -p ./agent-$a
+		./bin/etcd-agent -etcd-path ./bin/etcd -etcd-log-dir "./agent-$a" -port ":${a}9027" -use-root=false &
+		pid="$!"
+		agent_pids="${agent_pids} $pid"
+	done
+
+	./bin/etcd-tester \
+		-agent-endpoints "127.0.0.1:19027,127.0.0.1:29027,127.0.0.1:39027" \
+		-client-ports 12379,22379,32379 \
+		-peer-ports 12380,22380,32380 \
+		-limit 1 \
+		-schedule-cases "0 1 2 3 4 5" \
+		-exit-on-failure
+	ETCD_TESTER_EXIT_CODE=$?
+	echo "ETCD_TESTER_EXIT_CODE:" ${ETCD_TESTER_EXIT_CODE}
+
+	echo "Waiting for processes to exit"
+	kill -s TERM ${agent_pids}
+	for a in ${agent_pids}; do wait $a || true; done
+	rm -rf ./agent-*
+
+	if [[ "${ETCD_TESTER_EXIT_CODE}" -ne "0" ]]; then
+		echo "FAIL with exit code" ${ETCD_TESTER_EXIT_CODE}
+		exit ${ETCD_TESTER_EXIT_CODE}
+	fi
+}
+
 function cov_pass {
 	echo "Running code coverage..."
 	# install gocovmerge before running code coverage from github.com/wadey/gocovmerge
@@ -175,6 +209,10 @@ function release_pass {
 		# in case, we need to test against different version
 		UPGRADE_VER=$MANUAL_VER
 	fi
+	if [[ -z ${UPGRADE_VER} ]]; then
+		UPGRADE_VER="v3.2.0"
+		echo "fallback to" ${UPGRADE_VER}
+	fi
 
 	local file="etcd-$UPGRADE_VER-linux-$GOARCH.tar.gz"
 	echo "Downloading $file"
@@ -185,9 +223,8 @@ function release_pass {
 	set -e
 	case $result in
 		0)	;;
-		22)	return 0
-			;;
-		*)	exit $result
+		*)	echo "FAIL with" ${result}
+			exit $result
 			;;
 	esac
 
@@ -213,21 +250,33 @@ function fmt_pass {
 		exit 255
 	fi
 
-	echo "Checking 'go tool vet -shadow'..."
-	for path in $FMT; do
-		if [ "${path##*.}" != "go" ]; then
-			path="${path}/*.go"
+	echo "Checking 'go tool vet -all -shadow'..."
+	fmtpkgs=$(echo $FMT | xargs dirname | sort | uniq | sed '/\./d')
+	vetRes=$(go tool vet -all -shadow ${fmtpkgs} 2>&1 | grep -v '/gw/' || true)
+	if [ -n "${vetRes}" ]; then
+		echo -e "govet -all -shadow checking failed:\n${vetRes}"
+		exit 255
+	fi
+
+	if which shellcheck >/dev/null; then
+		echo "Checking shellcheck..."
+		shellcheckResult=$(shellcheck -fgcc build test scripts/* 2>&1 || true)
+		if [ -n "${shellcheckResult}" ]; then
+			# mask the most common ones; fix later
+			SHELLCHECK_MASK="SC(2086|2006|2068|2196|2035|2162|2076)"
+			errs=$(echo "${shellcheckResult}" | egrep -v "${SHELLCHECK_MASK}" || true)
+			if [ -n "${errs}" ]; then
+				echo -e "shellcheck checking failed:\n${shellcheckResult}\n===\nFailed:\n${errs}"
+				exit 255
+			fi
+			suppressed=$(echo "${shellcheckResult}" | cut -f4- -d':' | sort | uniq -c | sort -n)
+			echo -e "shellcheck suppressed warnings:\n${suppressed}"
 		fi
-		vetRes=$(go tool vet -shadow ${path})
-		if [ -n "${vetRes}" ]; then
-			echo -e "govet -shadow checking ${path} failed:\n${vetRes}"
-			exit 255
-		fi
-	done
+	fi
 
 	echo "Checking documentation style..."
 	# eschew you
-	yous=`find . -name \*.md | xargs egrep --color "[Yy]ou[r]?[ '.,;]" | grep -v /v2/ || true`
+	yous=`find . -name \*.md -exec egrep --color "[Yy]ou[r]?[ '.,;]" {} + | grep -v /v2/ || true`
 	if [ ! -z "$yous" ]; then
 		echo -e "found 'you' in documentation:\n${yous}"
 		exit 255
@@ -268,7 +317,7 @@ function fmt_pass {
 			# TODO: resolve these after go1.8 migration
 			SIMPLE_CHECK_MASK="S(1024)"
 			if echo "${gosimpleResult}" | egrep -v "$SIMPLE_CHECK_MASK"; then
-				echo -e "gosimple checking ${path} failed:\n${gosimpleResult}"
+				echo -e "gosimple checking failed:\n${gosimpleResult}"
 				exit 255
 			else
 				echo -e "gosimple warning:\n${gosimpleResult}"
@@ -297,7 +346,7 @@ function fmt_pass {
 			# See https://github.com/dominikh/go-tools/tree/master/cmd/staticcheck
 			STATIC_CHECK_MASK="SA(1019|2002)"
 			if echo "${staticcheckResult}" | egrep -v "$STATIC_CHECK_MASK"; then
-				echo -e "staticcheck checking ${path} failed:\n${staticcheckResult}"
+				echo -e "staticcheck checking failed:\n${staticcheckResult}"
 				exit 255
 			else
 				suppressed=`echo "${staticcheckResult}" | sed 's/ /\n/g' | grep "(SA" | sort | uniq -c`
@@ -309,16 +358,20 @@ function fmt_pass {
 	fi
 
 	echo "Checking for license header..."
-	licRes=$(for file in $(find . -type f -iname '*.go' ! -path './cmd/*' ! -path './gopath.proto/*'); do
-			head -n3 "${file}" | grep -Eq "(Copyright|generated|GENERATED)" || echo -e "  ${file}"
-		done;)
+	licRes=""
+	files=$(find . -type f -iname '*.go' ! -path './cmd/*' ! -path './gopath.proto/*')
+	for file in $files; do
+		if ! head -n3 "${file}" | grep -Eq "(Copyright|generated|GENERATED)" ; then
+			licRes="${licRes}"$(echo -e "  ${file}")
+		fi
+	done
 	if [ -n "${licRes}" ]; then
 		echo -e "license header checking failed:\n${licRes}"
 		exit 255
 	fi
 
 	echo "Checking commit titles..."
-	git log --oneline `git merge-base HEAD master`...HEAD | while read l; do
+	git log --oneline "$(git merge-base HEAD master)"...HEAD | while read l; do
 		commitMsg=`echo "$l" | cut -f2- -d' '`
 		if [[ "$commitMsg" == Merge* ]]; then
 			# ignore "Merge pull" commits
@@ -350,7 +403,7 @@ function bom_pass {
 		--override-file bill-of-materials.override.json \
 		github.com/coreos/etcd github.com/coreos/etcd/etcdctl >bom-now.json || true
 	if ! diff bill-of-materials.json bom-now.json; then
-		echo vendored licenses do not match given bill of materials
+		echo "vendored licenses do not match given bill of materials"
 		exit 255
 	fi
 	rm bom-now.json
@@ -372,7 +425,7 @@ function dep_pass {
 function build_cov_pass {
 	out="bin"
 	if [ -n "${BINDIR}" ]; then out="${BINDIR}"; fi
-	go test -c -covermode=set -coverpkg=$PKGS_COMMA -o ${out}/etcd_test
+	go test -tags cov -c -covermode=set -coverpkg=$PKGS_COMMA -o ${out}/etcd_test
 	go test -tags cov -c -covermode=set -coverpkg=$PKGS_COMMA -o ${out}/etcdctl_test ${REPO_PATH}/etcdctl
 }
 

tools/benchmark/cmd/util.go

@@ -44,9 +44,8 @@ func mustFindLeaderEndpoints(c *clientv3.Client) {
 
 	leaderId := uint64(0)
 	for _, ep := range c.Endpoints() {
-		resp, serr := c.Status(context.TODO(), ep)
-		if serr == nil {
-			leaderId = resp.Leader
+		if sresp, serr := c.Status(context.TODO(), ep); serr == nil {
+			leaderId = sresp.Leader
 			break
 		}
 	}

tools/functional-tester/etcd-runner/command/election_command.go

@@ -49,7 +49,6 @@ func runElectionFunc(cmd *cobra.Command, args []string) {
 	// nextc closes when election is ready for next round.
 	nextc := make(chan struct{})
 	eps := endpointsFromFlag(cmd)
-	dialTimeout := dialTimeoutFromCmd(cmd)
 
 	for i := range rcs {
 		v := fmt.Sprintf("%d", i)

tools/functional-tester/etcd-runner/command/global.go

@@ -112,11 +112,3 @@ func endpointsFromFlag(cmd *cobra.Command) []string {
 	}
 	return endpoints
 }
-
-func dialTimeoutFromCmd(cmd *cobra.Command) time.Duration {
-	dialTimeout, err := cmd.Flags().GetDuration("dial-timeout")
-	if err != nil {
-		ExitWithError(ExitError, err)
-	}
-	return dialTimeout
-}

tools/functional-tester/etcd-runner/command/lease_renewer_command.go

@@ -49,7 +49,6 @@ func runLeaseRenewerFunc(cmd *cobra.Command, args []string) {
 	}
 
 	eps := endpointsFromFlag(cmd)
-	dialTimeout := dialTimeoutFromCmd(cmd)
 	c := newClient(eps, dialTimeout)
 	ctx := context.Background()
 

tools/functional-tester/etcd-runner/command/lock_racer_command.go

@@ -53,7 +53,6 @@ func runRacerFunc(cmd *cobra.Command, args []string) {
 	cnt := 0
 
 	eps := endpointsFromFlag(cmd)
-	dialTimeout := dialTimeoutFromCmd(cmd)
 
 	for i := range rcs {
 		var (

tools/functional-tester/etcd-runner/command/watch_command.go

@@ -73,7 +73,6 @@ func performWatchOnPrefixes(ctx context.Context, cmd *cobra.Command, round int)
 	roundPrefix := fmt.Sprintf("%16x", round)
 
 	eps := endpointsFromFlag(cmd)
-	dialTimeout := dialTimeoutFromCmd(cmd)
 
 	var (
 		revision int64

tools/functional-tester/etcd-tester/main.go

@@ -46,6 +46,7 @@ func main() {
 	stressKeySize := flag.Uint("stress-key-size", 100, "the size of each small key written into etcd.")
 	stressKeySuffixRange := flag.Uint("stress-key-count", 250000, "the count of key range written into etcd.")
 	limit := flag.Int("limit", -1, "the limit of rounds to run failure set (-1 to run without limits).")
+	exitOnFailure := flag.Bool("exit-on-failure", false, "exit tester on first failure")
 	stressQPS := flag.Int("stress-qps", 10000, "maximum number of stresser requests per second.")
 	schedCases := flag.String("schedule-cases", "", "test case schedule")
 	consistencyCheck := flag.Bool("consistency-check", true, "true to check consistency (revision, hash)")
@@ -125,9 +126,10 @@ func main() {
 	}
 
 	t := &tester{
-		failures: schedule,
-		cluster:  c,
-		limit:    *limit,
+		failures:      schedule,
+		cluster:       c,
+		limit:         *limit,
+		exitOnFailure: *exitOnFailure,
 
 		scfg:         scfg,
 		stresserType: *stresserType,

tools/functional-tester/etcd-tester/tester.go

@@ -16,12 +16,14 @@ package main
 
 import (
 	"fmt"
+	"os"
 	"time"
 )
 
 type tester struct {
-	cluster *cluster
-	limit   int
+	cluster       *cluster
+	limit         int
+	exitOnFailure bool
 
 	failures        []failure
 	status          Status
@@ -49,6 +51,7 @@ func (tt *tester) runLoop() {
 
 	if err := tt.resetStressCheck(); err != nil {
 		plog.Errorf("%s failed to start stresser (%v)", tt.logPrefix(), err)
+		tt.failed()
 		return
 	}
 
@@ -87,6 +90,7 @@ func (tt *tester) runLoop() {
 		if round > 0 && round%500 == 0 { // every 500 rounds
 			if err := tt.defrag(); err != nil {
 				plog.Warningf("%s functional-tester returning with error (%v)", tt.logPrefix(), err)
+				tt.failed()
 				return
 			}
 		}
@@ -209,7 +213,18 @@ func (tt *tester) logPrefix() string {
 	return prefix
 }
 
+func (tt *tester) failed() {
+	if !tt.exitOnFailure {
+		return
+	}
+	plog.Warningf("%s exiting on failure", tt.logPrefix())
+	tt.cluster.Terminate()
+	os.Exit(2)
+}
+
 func (tt *tester) cleanup() error {
+	defer tt.failed()
+
 	roundFailedTotalCounter.Inc()
 	desc := "compact/defrag"
 	if tt.status.Case != -1 {

version/version.go

@@ -26,7 +26,7 @@ import (
 var (
 	// MinClusterVersion is the min cluster version this etcd binary is compatible with.
 	MinClusterVersion = "3.0.0"
-	Version           = "3.2.0-rc.1+git"
+	Version           = "3.2.1"
 	APIVersion        = "unknown"
 
 	// Git SHA Value will be set during build