Merge pull request #248 from benbjohnson/build-fail

Update build script to fail on error.
2013-10-17 09:27:22 -07:00 · 2013-10-17 10:02:59 -06:00 · 2013-10-16 18:37:39 -07:00 · 2013-10-16 19:36:38 -06:00 · 2013-10-16 17:33:35 -07:00 · 2013-10-16 17:29:53 -07:00
235 changed files with 12630 additions and 6678 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,4 +1,5 @@
 src/
 pkg/
-./etcd
+/etcd
-release_version.go
+/server/release_version.go
 /machine*
--- a/.travis.yml
+++ b/.travis.yml
@ -5,4 +5,4 @@ install:
 - echo "Skip install"
 script:
- - ./test
+ - ./test.sh
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -0,0 +1,75 @@
 # How to contribute
 etcd is open source, Apache 2.0 licensed and accepts contributions via Github pull requests.
 This document outlines some of the conventions on commit message formatting, contact points for developers and other resources to make getting your contribution into etcd easier.
 # Email and chat
 For simplicity etcd discussions happen on coreos-dev and in #coreos-dev.
 As the community grows we will move to a dedicated mailing list and IRC channel.
 - Email: [coreos-dev](https://groups.google.com/forum/#!forum/coreos-dev) 
 - IRC: #[coreos](irc://irc.freenode.org:6667/#coreos) IRC channel on freenode.org
 ## Getting Started
 - Fork the repository on GitHub
 - Read the README.md for build instructions
 ## Contribution flow
 This is a rough outline of what a contributor's workflow looks like:
 - Create a topic branch from where you want to base your work. This is usually master.
 - Make commits of logical units.
 - Make sure your commit messages are in the proper format, see below
 - Push your changes to a topic branch in your fork of the repository.
 - Submit a pull request to coreos/etcd
 Thanks for you contributions!
 ### Format of the commit message
 etcd follow a rough convention for commit messages borrowed from Angularjs.
 This is an example of a commit:
 ```
    feat(scripts/test-cluster): add a cluster test command
    this uses tmux to setup a test cluster that you can easily kill and
    start for debugging.
 ```
 To make it more formal it looks something like this:
 ```
 <type>(<scope>): <subject>
 <BLANK LINE>
 <body>
 <BLANK LINE>
 <footer>
 ```
 The first line is the subject and should not be longer than 70 characters, the second line is always blank and other lines should be wrapped at 80 characters.
 This allows the message to be easier to read on github as well as in various git tools.
 ### Subject line
 The subject line contains succinct description of the change.
 ### Allowed <type>
 feat (feature)
 fix (bug fix)
 docs (documentation)
 style (formatting, missing semi colons, …)
 refactor
 test (when adding missing tests)
 chore (maintain)
 ### Allowed <scope>
 Scopes could be anything specifying place of the commit change. For example store, api, etc.
 ### More details on commits
 For more details see the [angularjs commit style guide](https://docs.google.com/a/coreos.com/document/d/1QrDFcIiPjSLDn3EL15IJygNPiHORgU1_OOAqWjiDU5Y/edit#).
--- a/10
+++ b/10
@ -0,0 +1,10 @@
 FROM ubuntu:12.04
 RUN apt-get update
 RUN apt-get install -y python-software-properties git
 RUN add-apt-repository -y ppa:duh/golang
 RUN apt-get update
 RUN apt-get install -y golang
 ADD . /opt/etcd
 RUN cd /opt/etcd && ./build
 EXPOSE 4001 7001
 ENTRYPOINT ["/opt/etcd/etcd", "-c", "0.0.0.0:4001", "-s", "0.0.0.0:7001"]
--- a/Documentation/errorcode.md
+++ b/Documentation/errorcode.md
@ -0,0 +1,58 @@
 Error Code
 ======
 This document describes the error code in **Etcd** project.
 It's categorized into four groups:
 - Command Related Error
 - Post Form Related Error
 - Raft Related Error
 - Etcd Related Error
 Error code corresponding strerror
 ------
    const (
        EcodeKeyNotFound    = 100
        EcodeTestFailed     = 101
        EcodeNotFile        = 102
        EcodeNoMoreMachine  = 103
        EcodeNotDir         = 104
        EcodeNodeExist      = 105
        EcodeKeyIsPreserved = 106
        EcodeValueRequired     = 200
        EcodePrevValueRequired = 201
        EcodeTTLNaN            = 202
        EcodeIndexNaN          = 203
        EcodeRaftInternal = 300
        EcodeLeaderElect  = 301
        EcodeWatcherCleared = 400
        EcodeEventIndexCleared = 401
    )
    // command related errors
    errors[100] = "Key Not Found"
    errors[101] = "Test Failed" //test and set
    errors[102] = "Not A File"
    errors[103] = "Reached the max number of machines in the cluster"
    errors[104] = "Not A Directory"
    errors[105] = "Already exists" // create
    errors[106] = "The prefix of given key is a keyword in etcd"
    // Post form related errors
    errors[200] = "Value is Required in POST form"
    errors[201] = "PrevValue is Required in POST form"
    errors[202] = "The given TTL in POST form is not a number"
    errors[203] = "The given index in POST form is not a number"
    // raft related errors
    errors[300] = "Raft Internal Error"
    errors[301] = "During Leader Election"
    // etcd related errors
    errors[400] = "watcher is cleared due to etcd recovery"
    errors[401] = "The event in requested index is outdated and cleared"
--- a/Documentation/etcd-file-system.md
+++ b/Documentation/etcd-file-system.md
@ -0,0 +1,101 @@
 #Etcd File System
 ## Structure
 [TODO]
 ![alt text](./img/etcd_fs_structure.jpg "etcd file system structure")
 ## Node
 In **Etcd**, the **Node** is the rudimentary element constructing the whole.
 Currently **Etcd** file system is comprised in a Unix-like way of files and directories, and they are two kinds of nodes different in:
 - **File Node** has data associated with it.
 - **Directory Node** has children nodes associated with it.
 Besides the file and directory difference, all nodes have common attributes and operations as follows:
 ### Attributes:
 - **Expiration Time** [optional]
  The node will be deleted when it expires.
 - **ACL**
  The path of access control list of the node.
 ### Operation:
 - **Get** (path, recursive, sorted)
  Get the content of the node
    - If the node is a file, the data of the file will be returned.
    - If the node is a directory, the child nodes of the directory will be returned.
    - If recursive is true, it will recursively get the nodes of the directory.
    - If sorted is true, the result will be sorted based on the path.
 - **Create** (path, value[optional], ttl [optional])
  Create a file. Create operation will help to create intermediate directories with no expiration time.
    - If the file already exists, create will fail.
    - If the value is given, set will create a file.
    - If the value is not given, set will crate a directory.
    - If ttl is given, the node will be deleted when it expires.
 - **Update** (path, value[optional], ttl [optional])
  Update the content of the node.
    - If the value is given, the value of the key will be updated.
    - If ttl is given, the expiration time of the node will be updated.
 - **Delete** (path, recursive)
  Delete the node of given path.
    - If the node is a directory:
    - If recursive is true, the operation will delete all nodes under the directory.
    - If recursive is false, error will be returned.
 - **TestAndSet** (path, prevValue [prevIndex], value, ttl)
  Atomic *test and set* value to a file. If test succeeds, this operation will change the previous value of the file to the given value.
    - If the prevValue is given, it will test against previous value of 
    the node.
    - If the prevValue is empty, it will test if the node is not existing.
    - If the prevValue is not empty, it will test if the prevValue is equal to the current value of the file.
    - If the prevIndex is given, it will test if the create/last modified index of the node is equal to prevIndex.
 - **Renew** (path, ttl)
  Set the node's expiration time to (current time + ttl)
 ## ACL
 ### Theory
 Etcd exports a Unix-like file system interface consisting of files and directories, collectively called nodes.
 Each node has various meta-data, including three names of access control lists used to control reading, writing and changing (change ACL names for the node).
 We are storing the ACL names for nodes under a special *ACL* directory.
 Each node has ACL name corresponding to one file within *ACL* dir.
 Unless overridden, a node naturally inherits the ACL names of its parent directory on creation.
 For each ACL name, it has three children: *R (Reading)*, *W (Writing)*, *C (Changing)*
 Each permission is also a node. Under the node it contains the users who have this permission for the file refering to this ACL name.
 ### Example
 [TODO]
 ### Diagram
 [TODO]
 ### Interface
 Testing permissions:
 - (node *Node) get_perm()
 - (node *Node) has_perm(perm string, user string)
 Setting/Changing permissions:
 - (node *Node) set_perm(perm string)
 - (node *Node) change_ACLname(aclname string)
 ## User Group
 [TODO]
--- a/Documentation/img/etcd_fs_structure.jpg
+++ b/Documentation/img/etcd_fs_structure.jpg
--- a/Documentation/internal-protocol-versioning.md
+++ b/Documentation/internal-protocol-versioning.md
@ -0,0 +1,61 @@
 # Versioning
 Goal: We want to be able to upgrade an individual machine in an etcd cluster to a newer version of etcd.
 The process will take the form of individual followers upgrading to the latest version until the entire cluster is on the new version.
 Immediate need: etcd is moving too fast to version the internal API right now.
 But, we need to keep mixed version clusters from being started by a rollowing upgrade process (e.g. the CoreOS developer alpha).
 Longer term need: Having a mixed version cluster where all machines are not be running the exact same version of etcd itself but are able to speak one version of the internal protocol.
 Solution: The internal protocol needs to be versioned just as the client protocol is.
 Initially during the 0.\*.\* series of etcd releases we won't allow mixed versions at all.
 ## Join Control
 We will add a version field to the join command.
 But, who decides whether a newly upgraded follower should be able to join a cluster?
 ### Leader Controlled
 If the leader controls the version of followers joining the cluster then it compares its version to the version number presented by the follower in the JoinCommand and rejects the join if the number is less than the leader's version number.
 Advantages
 - Leader controls all cluster decisions still
 Disadvantages
 - Follower knows better what versions of the interal protocol it can talk than the leader
 ### Follower Controlled
 A newly upgraded follower should be able to figure out the leaders internal version from a defined internal backwards compatible API endpoint and figure out if it can join the cluster.
 If it cannot join the cluster then it simply exits.
 Advantages
 - The follower is running newer code and knows better if it can talk older protocols
 Disadvantages
 - This cluster decision isn't made by the leader
 ## Recommendation
 To solve the immediate need and to plan for the future lets do the following:
 - Add Version field to JoinCommand
 - Have a joining follower read the Version field of the leader and if its own version doesn't match the leader then sleep for some random interval and retry later to see if the leader has upgraded.
 # Research
 ## Zookeeper versioning
 Zookeeper very recently added versioning into the protocol and it doesn't seem to have seen any use yet.
 https://issues.apache.org/jira/browse/ZOOKEEPER-1633
 ## doozerd
 doozerd stores the version number of the machine in the datastore for other clients to check, no decisions are made off of this number currently.
--- a/README.md
+++ b/README.md
@ -1,4 +1,5 @@
 # etcd
 README version 0.1.0
 [![Build Status](https://travis-ci.org/coreos/etcd.png)](https://travis-ci.org/coreos/etcd)
@ -9,12 +10,12 @@ A highly-available key value store for shared configuration and service discover
 * Fast: benchmarked 1000s of writes/s per instance
 * Reliable: Properly distributed using Raft
-Etcd is written in Go and uses the [raft][raft] consensus algorithm to manage a highly availably replicated log.
+Etcd is written in Go and uses the [raft][raft] consensus algorithm to manage a highly-available replicated log.
-See [go-etcd][go-etcd] for a native Go client. Or feel free to just use curl, as in the examples below.
+See [etcdctl][etcdctl] for a simple command line client. Or feel free to just use curl, as in the examples below.
 [raft]: https://github.com/coreos/go-raft
-[go-etcd]: https://github.com/coreos/go-etcd
+[etcdctl]: http://coreos.com/docs/etcdctl/
 ## Getting Started
@ -24,22 +25,35 @@ The latest release is available as a binary at [Github][github-release].
 [github-release]: https://github.com/coreos/etcd/releases/
-You can also buildi etcd from source:
+### Building
 You can build etcd from source:
 ```sh
 git clone https://github.com/coreos/etcd
 cd etcd
 ./build
 ```
 This will generate a binary in the base directory called `./etcd`.
 _NOTE_: you need go 1.1+. Please check your installation with
 ```
 go version
 ```
 ### Running a single node
 These examples will use a single node cluster to show you the basics of the etcd REST API. Lets start etcd:
 ```sh
-./etcd -d node0
+./etcd -d node0 -n node0
 ```
-This will bring up an etcd node listening on port 4001 for client communication and on port 7001 for server-to-server communication. The `-d node0` argument tells etcd to write node configuration, logs and snapshots to the `./node0/` directory.
+This will bring up an etcd node listening on port 4001 for client communication and on port 7001 for server-to-server communication.
 The `-d node0` argument tells etcd to write node configuration, logs and snapshots to the `./node0/` directory.
 The `-n node0` tells the rest of the cluster that this node is named node0.
 ## Usage
@ -129,7 +143,7 @@ Now you can try to get the key by sending:
 curl -L http://127.0.0.1:4001/v1/keys/foo
 ```
-If the TTL has expired, the key will be deleted, and you will be returned a 404.
+If the TTL has expired, the key will be deleted, and you will be returned a 100.
 ```json
 {"errorCode":100,"message":"Key Not Found","cause":"/foo"}
@ -173,17 +187,17 @@ The watch command returns immediately with the same response as previous.
 Etcd can be used as a centralized coordination service in a cluster and `TestAndSet` is the most basic operation to build distributed lock service. This command will set the value only if the client provided `prevValue` is equal the current key value.
-Here is a simple example. Let's create a key-value pair first: `testAndSet=one`.
+Here is a simple example. Let's create a key-value pair first: `foo=one`.
 ```sh
-curl -L http://127.0.0.1:4001/v1/keys/testAndSet -d value=one
+curl -L http://127.0.0.1:4001/v1/keys/foo -d value=one
 ```
-Let's try an invaild `TestAndSet` command.
+Let's try an invalid `TestAndSet` command.
 We can give another parameter prevValue to set command to make it a TestAndSet command.
 ```sh
-curl -L http://127.0.0.1:4001/v1/keys/testAndSet -d prevValue=two -d value=three
+curl -L http://127.0.0.1:4001/v1/keys/foo -d prevValue=two -d value=three
 ```
 This will try to test if the previous of the key is two, it is change it to three.
@ -194,16 +208,16 @@ This will try to test if the previous of the key is two, it is change it to thre
 which means `testAndSet` failed.
-Let us try a vaild one.
+Let us try a valid one.
 ```sh
-curl -L http://127.0.0.1:4001/v1/keys/testAndSet -d prevValue=one -d value=two
+curl -L http://127.0.0.1:4001/v1/keys/foo -d prevValue=one -d value=two
 ```
 The response should be
 ```json
-{"action":"SET","key":"/testAndSet","prevValue":"one","value":"two","index":10}
+{"action":"SET","key":"/foo","prevValue":"one","value":"two","index":10}
 ```
 We successfully changed the value from “one” to “two”, since we give the correct previous value.
@ -243,10 +257,7 @@ which meas `foo=barbar` is a key-value pair under `/foo` and `foo_dir` is a dire
 Etcd supports SSL/TLS and client cert authentication for clients to server, as well as server to server communication
 First, you need to have a CA cert `clientCA.crt` and signed key pair `client.crt`, `client.key`. This site has a good reference for how to generate self-signed key pairs:
 ```url
 http://www.g-loaded.eu/2005/11/10/be-your-own-ca/
 ```
 For testing you can use the certificates in the `fixtures/ca` directory.
@ -262,7 +273,7 @@ Next, lets configure etcd to use this keypair:
 You can now test the configuration using https:
 ```sh
-curl --cacert fixtures/ca/ca.crt https://127.0.0.1:4001/v1/keys/foo -F value=bar
+curl --cacert fixtures/ca/ca.crt https://127.0.0.1:4001/v1/keys/foo -d value=bar -v
 ```
 You should be able to see the handshake succeed.
@ -292,7 +303,7 @@ We can also do authentication using CA certs. The clients will provide their cer
 Try the same request to this server:
 ```sh
-curl --cacert fixtures/ca/ca.crt https://127.0.0.1:4001/v1/keys/foo -F value=bar
+curl --cacert fixtures/ca/ca.crt https://127.0.0.1:4001/v1/keys/foo -d value=bar -v
 ```
 The request should be rejected by the server.
@ -334,26 +345,29 @@ Let start by creating 3 new etcd instances.
 We use -s to specify server port and -c to specify client port and -d to specify the directory to store the log and info of the node in the cluster
 ```sh
-./etcd -s 7001 -c 4001 -d nodes/node1
+./etcd -s 127.0.0.1:7001 -c 127.0.0.1:4001 -d nodes/node1 -n node1
 ```
 **Note:** If you want to run etcd on external IP address and still have access locally you need to add `-cl 0.0.0.0` so that it will listen on both external and localhost addresses.
 A similar argument `-sl` is used to setup the listening address for the server port.
 Let the join two more nodes to this cluster using the -C argument:
 ```sh
-./etcd -c 4002 -s 7002 -C 127.0.0.1:7001 -d nodes/node2
+./etcd -c 127.0.0.1:4002 -s 127.0.0.1:7002 -C 127.0.0.1:7001 -d nodes/node2 -n node2
-./etcd -c 4003 -s 7003 -C 127.0.0.1:7001 -d nodes/node3
+./etcd -c 127.0.0.1:4003 -s 127.0.0.1:7003 -C 127.0.0.1:7001 -d nodes/node3 -n node3
 ```
 Get the machines in the cluster:
 ```sh
-curl -L http://127.0.0.1:4001/machines
+curl -L http://127.0.0.1:4001/v1/machines
 ```
 We should see there are three nodes in the cluster
 ```
-0.0.0.0:4001,0.0.0.0:4002,0.0.0.0:4003
+http://127.0.0.1:4001, http://127.0.0.1:4002, http://127.0.0.1:4003
 ```
 The machine list is also available via this API:
@ -363,7 +377,7 @@ curl -L http://127.0.0.1:4001/v1/keys/_etcd/machines
 ```
 ```json
-[{"action":"GET","key":"/machines/node1","value":"0.0.0.0,7001,4001","index":4},{"action":"GET","key":"/machines/node3","value":"0.0.0.0,7002,4002","index":4},{"action":"GET","key":"/machines/node4","value":"0.0.0.0,7003,4003","index":4}]
+[{"action":"GET","key":"/_etcd/machines/node1","value":"raft=http://127.0.0.1:7001&etcd=http://127.0.0.1:4001","index":4},{"action":"GET","key":"/_etcd/machines/node2","value":"raft=http://127.0.0.1:7002&etcd=http://127.0.0.1:4002","index":4},{"action":"GET","key":"/_etcd/machines/node3","value":"raft=http://127.0.0.1:7003&etcd=http://127.0.0.1:4003","index":4}]
 ```
 The key of the machine is based on the ```commit index``` when it was added. The value of the machine is ```hostname```, ```raft port``` and ```client port```.
@ -371,12 +385,12 @@ The key of the machine is based on the ```commit index``` when it was added. The
 Also try to get the current leader in the cluster
 ```
-curl -L http://127.0.0.1:4001/leader
+curl -L http://127.0.0.1:4001/v1/leader
 ```
-The first server we set up should be the leader, if it has not dead during these commands.
+The first server we set up should be the leader, if it has not died during these commands.
 ```
-0.0.0.0:7001
+http://127.0.0.1:7001
 ```
 Now we can do normal SET and GET operations on keys as we explored earlier.
@ -400,11 +414,17 @@ curl -L http://127.0.0.1:4002/v1/keys/foo
 A new leader should have been elected.
 ```
-curl -L http://127.0.0.1:4001/leader
+curl -L http://127.0.0.1:4001/v1/leader
 ```
 ```
-0.0.0.0:7002 or 0.0.0.0:7003
+http://127.0.0.1:7002
 ```
 or
 ```
 http://127.0.0.1:7003
 ```
 You should be able to see this:
@ -435,6 +455,10 @@ In the previous example we showed how to use SSL client certs for client to serv
 If you are using SSL for server to server communication, you must use it on all instances of etcd.
 ## Contributing
 See [CONTRIBUTING](https://github.com/coreos/etcd/blob/master/CONTRIBUTING.md) for details on submitting patches and contacting developers via IRC and mailing lists.
 ## Libraries and Tools
 **Tools**
@ -445,16 +469,63 @@ If you are using SSL for server to server communication, you must use it on all
 - [go-etcd](https://github.com/coreos/go-etcd)
 **Java libraries**
 - [justinsb/jetcd](https://github.com/justinsb/jetcd)
 - [diwakergupta/jetcd](https://github.com/diwakergupta/jetcd)
 **Python libraries**
 - [transitorykris/etcd-py](https://github.com/transitorykris/etcd-py)
 - [jplana/python-etcd](https://github.com/jplana/python-etcd)
 - [russellhaering/txetcd](https://github.com/russellhaering/txetcd) - a Twisted Python library
 **Node libraries**
 - [stianeikeland/node-etcd](https://github.com/stianeikeland/node-etcd)
 **Ruby libraries**
 - [iconara/etcd-rb](https://github.com/iconara/etcd-rb)
 - [jpfuentes2/etcd-ruby](https://github.com/jpfuentes2/etcd-ruby)
 - [ranjib/etcd-ruby](https://github.com/ranjib/etcd-ruby)
 **C libraries**
 - [jdarcy/etcd-api](https://github.com/jdarcy/etcd-api)
 **Chef Integration**
 - [coderanger/etcd-chef](https://github.com/coderanger/etcd-chef)
 **Chef Cookbook**
 - [spheromak/etcd-cookbook](https://github.com/spheromak/etcd-cookbook)
 **Projects using etcd**
 - [binocarlos/yoda](https://github.com/binocarlos/yoda) - etcd + ZeroMQ
 - [calavera/active-proxy](https://github.com/calavera/active-proxy) - HTTP Proxy configured with etcd
 - [derekchiang/etcdplus](https://github.com/derekchiang/etcdplus) - A set of distributed synchronization primitives built upon etcd
 - [gleicon/goreman](https://github.com/gleicon/goreman/tree/etcd) - Branch of the Go Foreman clone with etcd support
 - [garethr/hiera-etcd](https://github.com/garethr/hiera-etcd) - Puppet hiera backend using etcd
 - [mattn/etcd-vim](https://github.com/mattn/etcd-vim) - SET and GET keys from inside vim
 - [mattn/etcdenv](https://github.com/mattn/etcdenv) - "env" shebang with etcd integration
 ## FAQ
 ### What size cluster should I use?
 Every command the client sends to the master is broadcast to all of the followers.
 But, the command is not committed until the majority of the cluster machines receive that command.
 Because of this majority voting property the ideal cluster should be kept small to keep speed up and be made up of an odd number of machines.
 Odd numbers are good because if you have 8 machines the majority will be 5 and if you have 9 machines the majority with be 5.
 The result is that an 8 machine cluster can tolerate 3 machine failures and a 9 machine cluster can tolerate 4 nodes failures.
 And in the best case when all 9 machines are responding the cluster will perform at the speed of the fastest 5 nodes.
 ## Project Details
 ### Versioning
@ -463,10 +534,10 @@ etcd uses [semantic versioning][semver].
 When we release v1.0.0 of etcd we will promise not to break the "v1" REST API.
 New minor versions may add additional features to the API however.
-You can get the version of etcd by requesting the root path of etcd:
+You can get the version of etcd by issuing a request to /version:
 ```sh
-curl -L http://127.0.0.1:4001
+curl -L http://127.0.0.1:4001/version
 ```
 During the v0 series of releases we may break the API as we fix bugs and get feedback.
--- a/27
+++ b/27
@ -1,25 +1,26 @@
-#!/bin/bash
+#!/bin/sh
 set -e
 ETCD_PACKAGE=github.com/coreos/etcd
-export GOPATH=${PWD}
+export GOPATH="${PWD}"
-SRC_DIR=$GOPATH/src
+SRC_DIR="$GOPATH/src"
-ETCD_DIR=$SRC_DIR/$ETCD_PACKAGE
+ETCD_DIR="$SRC_DIR/$ETCD_PACKAGE"
-ETCD_BASE=$(dirname ${ETCD_DIR})
+ETCD_BASE=$(dirname "${ETCD_DIR}")
-if [ ! -d ${ETCD_BASE} ]; then
+if [ ! -d "${ETCD_BASE}" ]; then
-	mkdir -p ${ETCD_BASE}
+	mkdir -p "${ETCD_BASE}"
 fi
-if [ ! -h ${ETCD_DIR} ]; then
+if [ ! -h "${ETCD_DIR}" ]; then
-	ln -s ../../../ ${ETCD_DIR}
+	ln -s ../../../ "${ETCD_DIR}"
 fi
 for i in third_party/*; do
-	if [ $i = "third_party/src" ]; then
+	if [ "$i" = "third_party/src" ]; then
 		continue
 	fi
-	cp -R $i src/
+	cp -R "$i" src/
 done
-./scripts/release-version > release_version.go
+./scripts/release-version > server/release_version.go
-go build ${ETCD_PACKAGE}
+go build "${ETCD_PACKAGE}"
--- a/build.ps1
+++ b/build.ps1
@ -0,0 +1,24 @@
 $ETCD_PACKAGE="github.com/coreos/etcd"
 $env:GOPATH=$pwd.Path
 $SRC_DIR="$env:GOPATH/src"
 $ETCD_DIR="$SRC_DIR/$ETCD_PACKAGE"
 $env:ETCD_DIR="$SRC_DIR/$ETCD_PACKAGE"
 $ETCD_BASE=(Split-Path $ETCD_DIR -Parent)
 if(-not(test-path $ETCD_DIR)){
 	mkdir -force "$ETCD_BASE" > $null
 }
 if(-not(test-path $ETCD_DIR )){
 	cmd /c 'mklink /D "%ETCD_DIR%" ..\..\..\'
 }
 foreach($i in (ls third_party/*)){
 	if("$i" -eq "third_party/src") {continue}
 	cp -Recurse -force "$i" src/
 }
 ./scripts/release-version.ps1 | Out-File -Encoding UTF8 server/release_version.go
 go build -v "${ETCD_PACKAGE}"
--- a/command.go
+++ b/command.go
@ -1,154 +0,0 @@
 package main
 import (
 	"encoding/json"
 	"fmt"
 	"github.com/coreos/etcd/store"
 	"github.com/coreos/go-raft"
 	"path"
 	"time"
 )
 // A command represents an action to be taken on the replicated state machine.
 type Command interface {
 	CommandName() string
 	Apply(server *raft.Server) (interface{}, error)
 }
 // Set command
 type SetCommand struct {
 	Key        string    `json:"key"`
 	Value      string    `json:"value"`
 	ExpireTime time.Time `json:"expireTime"`
 }
 // The name of the set command in the log
 func (c *SetCommand) CommandName() string {
 	return "etcd:set"
 }
 // Set the key-value pair
 func (c *SetCommand) Apply(server *raft.Server) (interface{}, error) {
 	return etcdStore.Set(c.Key, c.Value, c.ExpireTime, server.CommitIndex())
 }
 // TestAndSet command
 type TestAndSetCommand struct {
 	Key        string    `json:"key"`
 	Value      string    `json:"value"`
 	PrevValue  string    `json: prevValue`
 	ExpireTime time.Time `json:"expireTime"`
 }
 // The name of the testAndSet command in the log
 func (c *TestAndSetCommand) CommandName() string {
 	return "testAndSet"
 }
 // Set the key-value pair if the current value of the key equals to the given prevValue
 func (c *TestAndSetCommand) Apply(server *raft.Server) (interface{}, error) {
 	return etcdStore.TestAndSet(c.Key, c.PrevValue, c.Value, c.ExpireTime, server.CommitIndex())
 }
 // Get command
 type GetCommand struct {
 	Key string `json:"key"`
 }
 // The name of the get command in the log
 func (c *GetCommand) CommandName() string {
 	return "etcd:get"
 }
 // Get the value of key
 func (c *GetCommand) Apply(server *raft.Server) (interface{}, error) {
 	return etcdStore.Get(c.Key)
 }
 // Delete command
 type DeleteCommand struct {
 	Key string `json:"key"`
 }
 // The name of the delete command in the log
 func (c *DeleteCommand) CommandName() string {
 	return "etcd:delete"
 }
 // Delete the key
 func (c *DeleteCommand) Apply(server *raft.Server) (interface{}, error) {
 	return etcdStore.Delete(c.Key, server.CommitIndex())
 }
 // Watch command
 type WatchCommand struct {
 	Key        string `json:"key"`
 	SinceIndex uint64 `json:"sinceIndex"`
 }
 // The name of the watch command in the log
 func (c *WatchCommand) CommandName() string {
 	return "etcd:watch"
 }
 func (c *WatchCommand) Apply(server *raft.Server) (interface{}, error) {
 	// create a new watcher
 	watcher := store.NewWatcher()
 	// add to the watchers list
 	etcdStore.AddWatcher(c.Key, watcher, c.SinceIndex)
 	// wait for the notification for any changing
 	res := <-watcher.C
 	if res == nil {
 		return nil, fmt.Errorf("Clearing watch")
 	}
 	return json.Marshal(res)
 }
 // JoinCommand
 type JoinCommand struct {
 	Name    string `json:"name"`
 	RaftURL string `json:"raftURL"`
 	EtcdURL string `json:"etcdURL"`
 }
 // The name of the join command in the log
 func (c *JoinCommand) CommandName() string {
 	return "etcd:join"
 }
 // Join a server to the cluster
 func (c *JoinCommand) Apply(raftServer *raft.Server) (interface{}, error) {
 	// check if the join command is from a previous machine, who lost all its previous log.
 	response, _ := etcdStore.RawGet(path.Join("_etcd/machines", c.Name))
 	if response != nil {
 		return []byte("join success"), nil
 	}
 	// check machine number in the cluster
 	num := machineNum()
 	if num == maxClusterSize {
 		return []byte("join fail"), fmt.Errorf(errors[103])
 	}
 	addNameToURL(c.Name, c.RaftURL, c.EtcdURL)
 	// add peer in raft
 	err := raftServer.AddPeer(c.Name)
 	// add machine in etcd storage
 	key := path.Join("_etcd/machines", c.Name)
 	value := fmt.Sprintf("raft=%s&etcd=%s", c.RaftURL, c.EtcdURL)
 	etcdStore.Set(key, value, time.Unix(0, 0), raftServer.CommitIndex())
 	return []byte("join success"), err
 }
 func (c *JoinCommand) NodeName() string {
 	return c.Name
 }
--- a/config.go
+++ b/config.go
@ -0,0 +1,143 @@
 package main
 import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/json"
 	"encoding/pem"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"github.com/coreos/etcd/log"
 	"github.com/coreos/etcd/server"
 )
 //--------------------------------------
 // Config
 //--------------------------------------
 // Get the server info from previous conf file
 // or from the user
 func getInfo(path string) *Info {
 	infoPath := filepath.Join(path, "info")
 	if force {
 		// Delete the old configuration if exist
 		logPath := filepath.Join(path, "log")
 		confPath := filepath.Join(path, "conf")
 		snapshotPath := filepath.Join(path, "snapshot")
 		os.Remove(infoPath)
 		os.Remove(logPath)
 		os.Remove(confPath)
 		os.RemoveAll(snapshotPath)
 	} else if info := readInfo(infoPath); info != nil {
 		log.Infof("Found node configuration in '%s'. Ignoring flags", infoPath)
 		return info
 	}
 	// Read info from command line
 	info := &argInfo
 	// Write to file.
 	content, _ := json.MarshalIndent(info, "", " ")
 	content = []byte(string(content) + "\n")
 	if err := ioutil.WriteFile(infoPath, content, 0644); err != nil {
 		log.Fatalf("Unable to write info to file: %v", err)
 	}
 	log.Infof("Wrote node configuration to '%s'", infoPath)
 	return info
 }
 // readInfo reads from info file and decode to Info struct
 func readInfo(path string) *Info {
 	file, err := os.Open(path)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil
 		}
 		log.Fatal(err)
 	}
 	defer file.Close()
 	info := &Info{}
 	content, err := ioutil.ReadAll(file)
 	if err != nil {
 		log.Fatalf("Unable to read info: %v", err)
 		return nil
 	}
 	if err = json.Unmarshal(content, &info); err != nil {
 		log.Fatalf("Unable to parse info: %v", err)
 		return nil
 	}
 	return info
 }
 func tlsConfigFromInfo(info server.TLSInfo) (t server.TLSConfig, ok bool) {
 	var keyFile, certFile, CAFile string
 	var tlsCert tls.Certificate
 	var err error
 	t.Scheme = "http"
 	keyFile = info.KeyFile
 	certFile = info.CertFile
 	CAFile = info.CAFile
 	// If the user do not specify key file, cert file and
 	// CA file, the type will be HTTP
 	if keyFile == "" && certFile == "" && CAFile == "" {
 		return t, true
 	}
 	// both the key and cert must be present
 	if keyFile == "" || certFile == "" {
 		return t, false
 	}
 	tlsCert, err = tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
 		log.Fatal(err)
 	}
 	t.Scheme = "https"
 	t.Server.ClientAuth, t.Server.ClientCAs = newCertPool(CAFile)
 	// The client should trust the RootCA that the Server uses since
 	// everyone is a peer in the network.
 	t.Client.Certificates = []tls.Certificate{tlsCert}
 	t.Client.RootCAs = t.Server.ClientCAs
 	return t, true
 }
 // newCertPool creates x509 certPool and corresponding Auth Type.
 // If the given CAfile is valid, add the cert into the pool and verify the clients'
 // certs against the cert in the pool.
 // If the given CAfile is empty, do not verify the clients' cert.
 // If the given CAfile is not valid, fatal.
 func newCertPool(CAFile string) (tls.ClientAuthType, *x509.CertPool) {
 	if CAFile == "" {
 		return tls.NoClientCert, nil
 	}
 	pemByte, err := ioutil.ReadFile(CAFile)
 	check(err)
 	block, pemByte := pem.Decode(pemByte)
 	cert, err := x509.ParseCertificate(block.Bytes)
 	check(err)
 	certPool := x509.NewCertPool()
 	certPool.AddCert(cert)
 	return tls.RequireAndVerifyClientCert, certPool
 }
--- a/error.go
+++ b/error.go
@ -1,49 +0,0 @@
 package main
 import (
 	"encoding/json"
 )
 var errors map[int]string
 func init() {
 	errors = make(map[int]string)
 	// command related errors
 	errors[100] = "Key Not Found"
 	errors[101] = "The given PrevValue is not equal to the value of the key"
 	errors[102] = "Not A File"
 	errors[103] = "Reached the max number of machines in the cluster"
 	// Post form related errors
 	errors[200] = "Value is Required in POST form"
 	errors[201] = "PrevValue is Required in POST form"
 	errors[202] = "The given TTL in POST form is not a number"
 	errors[203] = "The given index in POST form is not a number"
 	// raft related errors
 	errors[300] = "Raft Internal Error"
 	errors[301] = "During Leader Election"
 	// keyword
 	errors[400] = "The prefix of the given key is a keyword in etcd"
 	// etcd related errors
 	errors[500] = "watcher is cleared due to etcd recovery"
 }
 type jsonError struct {
 	ErrorCode int    `json:"errorCode"`
 	Message   string `json:"message"`
 	Cause     string `json:"cause,omitempty"`
 }
 func newJsonError(errorCode int, cause string) []byte {
 	b, _ := json.Marshal(jsonError{
 		ErrorCode: errorCode,
 		Message:   errors[errorCode],
 		Cause:     cause,
 	})
 	return b
 }
--- a/error/error.go
+++ b/error/error.go
@ -0,0 +1,103 @@
 package error
 import (
 	"encoding/json"
 	"fmt"
 	"net/http"
 )
 var errors map[int]string
 const (
 	EcodeKeyNotFound    = 100
 	EcodeTestFailed     = 101
 	EcodeNotFile        = 102
 	EcodeNoMoreMachine  = 103
 	EcodeNotDir         = 104
 	EcodeNodeExist      = 105
 	EcodeKeyIsPreserved = 106
 	EcodeValueRequired      = 200
 	EcodePrevValueRequired  = 201
 	EcodeTTLNaN             = 202
 	EcodeIndexNaN           = 203
 	EcodeValueOrTTLRequired = 204
 	EcodeRaftInternal = 300
 	EcodeLeaderElect  = 301
 	EcodeWatcherCleared    = 400
 	EcodeEventIndexCleared = 401
 )
 func init() {
 	errors = make(map[int]string)
 	// command related errors
 	errors[EcodeKeyNotFound] = "Key Not Found"
 	errors[EcodeTestFailed] = "Test Failed" //test and set
 	errors[EcodeNotFile] = "Not A File"
 	errors[EcodeNoMoreMachine] = "Reached the max number of machines in the cluster"
 	errors[EcodeNotDir] = "Not A Directory"
 	errors[EcodeNodeExist] = "Already exists" // create
 	errors[EcodeKeyIsPreserved] = "The prefix of given key is a keyword in etcd"
 	// Post form related errors
 	errors[EcodeValueRequired] = "Value is Required in POST form"
 	errors[EcodePrevValueRequired] = "PrevValue is Required in POST form"
 	errors[EcodeTTLNaN] = "The given TTL in POST form is not a number"
 	errors[EcodeIndexNaN] = "The given index in POST form is not a number"
 	errors[EcodeValueOrTTLRequired] = "Value or TTL is required in POST form"
 	// raft related errors
 	errors[EcodeRaftInternal] = "Raft Internal Error"
 	errors[EcodeLeaderElect] = "During Leader Election"
 	// etcd related errors
 	errors[EcodeWatcherCleared] = "watcher is cleared due to etcd recovery"
 	errors[EcodeEventIndexCleared] = "The event in requested index is outdated and cleared"
 }
 type Error struct {
 	ErrorCode int    `json:"errorCode"`
 	Message   string `json:"message"`
 	Cause     string `json:"cause,omitempty"`
 	Index     uint64 `json:"index"`
 	Term      uint64 `json:"term"`
 }
 func NewError(errorCode int, cause string, index uint64, term uint64) *Error {
 	return &Error{
 		ErrorCode: errorCode,
 		Message:   errors[errorCode],
 		Cause:     cause,
 		Index:     index,
 		Term:      term,
 	}
 }
 func Message(code int) string {
 	return errors[code]
 }
 // Only for error interface
 func (e Error) Error() string {
 	return e.Message
 }
 func (e Error) toJsonString() string {
 	b, _ := json.Marshal(e)
 	return string(b)
 }
 func (e Error) Write(w http.ResponseWriter) {
 	w.Header().Add("X-Etcd-Index", fmt.Sprint(e.Index))
 	w.Header().Add("X-Etcd-Term", fmt.Sprint(e.Term))
 	// 3xx is reft internal error
 	if e.ErrorCode/100 == 3 {
 		http.Error(w, e.toJsonString(), http.StatusInternalServerError)
 	} else {
 		http.Error(w, e.toJsonString(), http.StatusBadRequest)
 	}
 }
--- a/etcd.go
+++ b/etcd.go
@ -1,25 +1,16 @@
 package main
 import (
 	"bytes"
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/json"
 	"encoding/pem"
 	"flag"
 	"fmt"
 	"github.com/coreos/etcd/store"
 	"github.com/coreos/etcd/web"
 	"github.com/coreos/go-raft"
 	"io/ioutil"
 	"net"
 	"net/http"
 	"net/url"
 	"os"
 	"os/signal"
 	"runtime/pprof"
 	"strings"
-	"time"
+
 	"github.com/coreos/etcd/log"
 	"github.com/coreos/etcd/server"
 	"github.com/coreos/etcd/store"
 	"github.com/coreos/go-raft"
 )
 //------------------------------------------------------------------------------
@ -28,39 +19,48 @@ import (
 //
 //------------------------------------------------------------------------------
-var verbose bool
+var (
-var veryVerbose bool
+	veryVerbose bool
-var machines string
+	machines     string
-var machinesFile string
+	machinesFile string
-var cluster []string
+	cluster []string
-var argInfo Info
+	argInfo Info
-var dirPath string
+	dirPath string
-var force bool
+	force bool
-var maxSize int
+	printVersion bool
-var snapshot bool
+	maxSize int
-var retryTimes int
+	snapshot bool
-var maxClusterSize int
+	retryTimes int
-var cpuprofile string
+	maxClusterSize int
 	cpuprofile string
 	cors string
 )
 func init() {
-	flag.BoolVar(&verbose, "v", false, "verbose logging")
+	flag.BoolVar(&printVersion, "version", false, "print the version and exit")
 	flag.BoolVar(&log.Verbose, "v", false, "verbose logging")
 	flag.BoolVar(&veryVerbose, "vv", false, "very verbose logging")
 	flag.StringVar(&machines, "C", "", "the ip address and port of a existing machines in the cluster, sepearate by comma")
 	flag.StringVar(&machinesFile, "CF", "", "the file contains a list of existing machines in the cluster, seperate by comma")
 	flag.StringVar(&argInfo.Name, "n", "default-name", "the node name (required)")
-	flag.StringVar(&argInfo.EtcdURL, "c", "127.0.0.1:4001", "the hostname:port for etcd client communication")
+	flag.StringVar(&argInfo.EtcdURL, "c", "127.0.0.1:4001", "the advertised public hostname:port for etcd client communication")
-	flag.StringVar(&argInfo.RaftURL, "s", "127.0.0.1:7001", "the hostname:port for raft server communication")
+	flag.StringVar(&argInfo.RaftURL, "s", "127.0.0.1:7001", "the advertised public hostname:port for raft server communication")
 	flag.StringVar(&argInfo.EtcdListenHost, "cl", "", "the listening hostname for etcd client communication (defaults to advertised ip)")
 	flag.StringVar(&argInfo.RaftListenHost, "sl", "", "the listening hostname for raft server communication (defaults to advertised ip)")
 	flag.StringVar(&argInfo.WebURL, "w", "", "the hostname:port of web interface")
 	flag.StringVar(&argInfo.RaftTLS.CAFile, "serverCAFile", "", "the path of the CAFile")
@ -84,31 +84,16 @@ func init() {
 	flag.IntVar(&maxClusterSize, "maxsize", 9, "the max size of the cluster")
 	flag.StringVar(&cpuprofile, "cpuprofile", "", "write cpu profile to file")
 	flag.StringVar(&cors, "cors", "", "whitelist origins for cross-origin resource sharing (e.g. '*' or 'http://localhost:8001,etc')")
 }
 const (
 	ELECTIONTIMEOUT  = 200 * time.Millisecond
 	HEARTBEATTIMEOUT = 50 * time.Millisecond
 	// Timeout for internal raft http connection
 	// The original timeout for http is 45 seconds
 	// which is too long for our usage.
 	HTTPTIMEOUT   = 10 * time.Second
 	RETRYINTERVAL = 10
 )
 //------------------------------------------------------------------------------
 //
 // Typedefs
 //
 //------------------------------------------------------------------------------
 type TLSInfo struct {
 	CertFile string `json:"CertFile"`
 	KeyFile  string `json:"KeyFile"`
 	CAFile   string `json:"CAFile"`
 }
 type Info struct {
 	Name string `json:"name"`
@ -116,51 +101,19 @@ type Info struct {
 	EtcdURL string `json:"etcdURL"`
 	WebURL  string `json:"webURL"`
-	RaftTLS TLSInfo `json:"raftTLS"`
+	RaftListenHost string `json:"raftListenHost"`
-	EtcdTLS TLSInfo `json:"etcdTLS"`
+	EtcdListenHost string `json:"etcdListenHost"`
 	RaftTLS server.TLSInfo `json:"raftTLS"`
 	EtcdTLS server.TLSInfo `json:"etcdTLS"`
 }
 //------------------------------------------------------------------------------
 //
 // Variables
 //
 //------------------------------------------------------------------------------
 var raftServer *raft.Server
 var raftTransporter transporter
 var etcdStore *store.Store
 var info *Info
 //------------------------------------------------------------------------------
 //
 // Functions
 //
 //------------------------------------------------------------------------------
 // sanitizeURL will cleanup a host string in the format hostname:port and
 // attach a schema.
 func sanitizeURL(host string, defaultScheme string) string {
 	// Blank URLs are fine input, just return it
 	if len(host) == 0 {
 		return host
 	}
 	p, err := url.Parse(host)
 	if err != nil {
 		fatal(err)
 	}
 	// Make sure the host is in Host:Port format
 	_, _, err = net.SplitHostPort(host)
 	if err != nil {
 		fatal(err)
 	}
 	p = &url.URL{Host: host, Scheme: defaultScheme}
 	return p.String()
 }
 //--------------------------------------
 // Main
 //--------------------------------------
@ -168,28 +121,17 @@ func sanitizeURL(host string, defaultScheme string) string {
 func main() {
 	flag.Parse()
 	if printVersion {
 		fmt.Println(server.ReleaseVersion)
 		os.Exit(0)
 	}
 	if cpuprofile != "" {
-		f, err := os.Create(cpuprofile)
+		runCPUProfile()
 		if err != nil {
 			fatal(err)
 		}
 		pprof.StartCPUProfile(f)
 		defer pprof.StopCPUProfile()
 		c := make(chan os.Signal, 1)
 		signal.Notify(c, os.Interrupt)
 		go func() {
 			for sig := range c {
 				fmt.Printf("captured %v, stopping profiler and exiting..", sig)
 				pprof.StopCPUProfile()
 				os.Exit(1)
 			}
 		}()
 	}
 	if veryVerbose {
-		verbose = true
+		log.Verbose = true
 		raft.SetLogLevel(raft.Debug)
 	}
@ -198,442 +140,60 @@ func main() {
 	} else if machinesFile != "" {
 		b, err := ioutil.ReadFile(machinesFile)
 		if err != nil {
-			fatalf("Unable to read the given machines file: %s", err)
+			log.Fatalf("Unable to read the given machines file: %s", err)
 		}
 		cluster = strings.Split(string(b), ",")
 	}
 	// Check TLS arguments
 	raftTLSConfig, ok := tlsConfigFromInfo(argInfo.RaftTLS)
 	if !ok {
-		fatal("Please specify cert and key file or cert and key file and CAFile or none of the three")
+		log.Fatal("Please specify cert and key file or cert and key file and CAFile or none of the three")
 	}
 	etcdTLSConfig, ok := tlsConfigFromInfo(argInfo.EtcdTLS)
 	if !ok {
-		fatal("Please specify cert and key file or cert and key file and CAFile or none of the three")
+		log.Fatal("Please specify cert and key file or cert and key file and CAFile or none of the three")
 	}
 	argInfo.Name = strings.TrimSpace(argInfo.Name)
 	if argInfo.Name == "" {
-		fatal("ERROR: server name required. e.g. '-n=server_name'")
+		log.Fatal("ERROR: server name required. e.g. '-n=server_name'")
 	}
 	// Check host name arguments
 	argInfo.RaftURL = sanitizeURL(argInfo.RaftURL, raftTLSConfig.Scheme)
 	argInfo.EtcdURL = sanitizeURL(argInfo.EtcdURL, etcdTLSConfig.Scheme)
 	argInfo.WebURL = sanitizeURL(argInfo.WebURL, "http")
-	// Setup commands.
+	argInfo.RaftListenHost = sanitizeListenHost(argInfo.RaftListenHost, argInfo.RaftURL)
-	registerCommands()
+	argInfo.EtcdListenHost = sanitizeListenHost(argInfo.EtcdListenHost, argInfo.EtcdURL)
 	// Read server info from file or grab it from user.
 	if err := os.MkdirAll(dirPath, 0744); err != nil {
-		fatalf("Unable to create path: %s", err)
+		log.Fatalf("Unable to create path: %s", err)
 	}
-	info = getInfo(dirPath)
+	info := getInfo(dirPath)
 	// Create etcd key-value store
-	etcdStore = store.CreateStore(maxSize)
+	store := store.New()
-	startRaft(raftTLSConfig)
+	// Create a shared node registry.
 	registry := server.NewRegistry(store)
-	if argInfo.WebURL != "" {
+	// Create peer server.
-		// start web
+	ps := server.NewPeerServer(info.Name, dirPath, info.RaftURL, info.RaftListenHost, &raftTLSConfig, &info.RaftTLS, registry, store)
-		argInfo.WebURL = sanitizeURL(argInfo.WebURL, "http")
+	ps.MaxClusterSize = maxClusterSize
-		go webHelper()
+	ps.RetryTimes = retryTimes
-		go web.Start(raftServer, argInfo.WebURL)
+
 	s := server.New(info.Name, info.EtcdURL, info.EtcdListenHost, &etcdTLSConfig, &info.EtcdTLS, ps, registry, store)
 	if err := s.AllowOrigins(cors); err != nil {
 		panic(err)
 	}
-	startEtcdTransport(*info, etcdTLSConfig.Scheme, etcdTLSConfig.Server)
+	ps.SetServer(s)
-}
+	ps.ListenAndServe(snapshot, cluster)
-
+	s.ListenAndServe()
 // Start the raft server
 func startRaft(tlsConfig TLSConfig) {
 	var err error
 	raftName := info.Name
 	// Create transporter for raft
 	raftTransporter = newTransporter(tlsConfig.Scheme, tlsConfig.Client)
 	// Create raft server
 	raftServer, err = raft.NewServer(raftName, dirPath, raftTransporter, etcdStore, nil)
 	if err != nil {
 		fatal(err)
 	}
 	// LoadSnapshot
 	if snapshot {
 		err = raftServer.LoadSnapshot()
 		if err == nil {
 			debugf("%s finished load snapshot", raftServer.Name())
 		} else {
 			debug(err)
 		}
 	}
 	raftServer.SetElectionTimeout(ELECTIONTIMEOUT)
 	raftServer.SetHeartbeatTimeout(HEARTBEATTIMEOUT)
 	raftServer.Start()
 	if raftServer.IsLogEmpty() {
 		// start as a leader in a new cluster
 		if len(cluster) == 0 {
 			time.Sleep(time.Millisecond * 20)
 			// leader need to join self as a peer
 			for {
 				command := &JoinCommand{
 					Name:    raftServer.Name(),
 					RaftURL: argInfo.RaftURL,
 					EtcdURL: argInfo.EtcdURL,
 				}
 				_, err := raftServer.Do(command)
 				if err == nil {
 					break
 				}
 			}
 			debugf("%s start as a leader", raftServer.Name())
 			// start as a follower in a existing cluster
 		} else {
 			time.Sleep(time.Millisecond * 20)
 			for i := 0; i < retryTimes; i++ {
 				success := false
 				for _, machine := range cluster {
 					if len(machine) == 0 {
 						continue
 					}
 					err = joinCluster(raftServer, machine)
 					if err != nil {
 						if err.Error() == errors[103] {
 							fmt.Println(err)
 							os.Exit(1)
 						}
 						debugf("cannot join to cluster via machine %s %s", machine, err)
 					} else {
 						success = true
 						break
 					}
 				}
 				if success {
 					break
 				}
 				warnf("cannot join to cluster via given machines, retry in %d seconds", RETRYINTERVAL)
 				time.Sleep(time.Second * RETRYINTERVAL)
 			}
 			if err != nil {
 				fatalf("Cannot join the cluster via given machines after %x retries", retryTimes)
 			}
 			debugf("%s success join to the cluster", raftServer.Name())
 		}
 	} else {
 		// rejoin the previous cluster
 		debugf("%s restart as a follower", raftServer.Name())
 	}
 	// open the snapshot
 	if snapshot {
 		go raftServer.Snapshot()
 	}
 	// start to response to raft requests
 	go startRaftTransport(*info, tlsConfig.Scheme, tlsConfig.Server)
 }
 // Create transporter using by raft server
 // Create http or https transporter based on
 // whether the user give the server cert and key
 func newTransporter(scheme string, tlsConf tls.Config) transporter {
 	t := transporter{}
 	t.scheme = scheme
 	tr := &http.Transport{
 		Dial:               dialTimeout,
 	}
 	if scheme == "https" {
 		tr.TLSClientConfig = &tlsConf
 		tr.DisableCompression = true
 	}
 	t.client = &http.Client{Transport: tr}
 	return t
 }
 // Dial with timeout
 func dialTimeout(network, addr string) (net.Conn, error) {
 	return net.DialTimeout(network, addr, HTTPTIMEOUT)
 }
 // Start to listen and response raft command
 func startRaftTransport(info Info, scheme string, tlsConf tls.Config) {
 	u, _ := url.Parse(info.RaftURL)
 	fmt.Printf("raft server [%s] listening on %s\n", info.Name, u)
 	raftMux := http.NewServeMux()
 	server := &http.Server{
 		Handler:   raftMux,
 		TLSConfig: &tlsConf,
 		Addr:      u.Host,
 	}
 	// internal commands
 	raftMux.HandleFunc("/name", NameHttpHandler)
 	raftMux.HandleFunc("/join", JoinHttpHandler)
 	raftMux.HandleFunc("/vote", VoteHttpHandler)
 	raftMux.HandleFunc("/log", GetLogHttpHandler)
 	raftMux.HandleFunc("/log/append", AppendEntriesHttpHandler)
 	raftMux.HandleFunc("/snapshot", SnapshotHttpHandler)
 	raftMux.HandleFunc("/snapshotRecovery", SnapshotRecoveryHttpHandler)
 	raftMux.HandleFunc("/etcdURL", EtcdURLHttpHandler)
 	if scheme == "http" {
 		fatal(server.ListenAndServe())
 	} else {
 		fatal(server.ListenAndServeTLS(info.RaftTLS.CertFile, info.RaftTLS.KeyFile))
 	}
 }
 // Start to listen and response client command
 func startEtcdTransport(info Info, scheme string, tlsConf tls.Config) {
 	u, _ := url.Parse(info.EtcdURL)
 	fmt.Printf("etcd server [%s] listening on %s\n", info.Name, u)
 	etcdMux := http.NewServeMux()
 	server := &http.Server{
 		Handler:   etcdMux,
 		TLSConfig: &tlsConf,
 		Addr:      u.Host,
 	}
 	// external commands
 	etcdMux.HandleFunc("/"+version+"/keys/", Multiplexer)
 	etcdMux.HandleFunc("/"+version+"/watch/", WatchHttpHandler)
 	etcdMux.HandleFunc("/leader", LeaderHttpHandler)
 	etcdMux.HandleFunc("/machines", MachinesHttpHandler)
 	etcdMux.HandleFunc("/", VersionHttpHandler)
 	etcdMux.HandleFunc("/stats", StatsHttpHandler)
 	etcdMux.HandleFunc("/test/", TestHttpHandler)
 	if scheme == "http" {
 		fatal(server.ListenAndServe())
 	} else {
 		fatal(server.ListenAndServeTLS(info.EtcdTLS.CertFile, info.EtcdTLS.KeyFile))
 	}
 }
 //--------------------------------------
 // Config
 //--------------------------------------
 type TLSConfig struct {
 	Scheme string
 	Server tls.Config
 	Client tls.Config
 }
 func tlsConfigFromInfo(info TLSInfo) (t TLSConfig, ok bool) {
 	var keyFile, certFile, CAFile string
 	var tlsCert tls.Certificate
 	var err error
 	t.Scheme = "http"
 	keyFile = info.KeyFile
 	certFile = info.CertFile
 	CAFile = info.CAFile
 	// If the user do not specify key file, cert file and
 	// CA file, the type will be HTTP
 	if keyFile == "" && certFile == "" && CAFile == "" {
 		return t, true
 	}
 	// both the key and cert must be present
 	if keyFile == "" || certFile == "" {
 		return t, false
 	}
 	tlsCert, err = tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
 		fatal(err)
 	}
 	t.Scheme = "https"
 	t.Server.ClientAuth, t.Server.ClientCAs = newCertPool(CAFile)
 	// The client should trust the RootCA that the Server uses since
 	// everyone is a peer in the network.
 	t.Client.Certificates = []tls.Certificate{tlsCert}
 	t.Client.RootCAs = t.Server.ClientCAs
 	return t, true
 }
 func parseInfo(path string) *Info {
 	file, err := os.Open(path)
 	if err != nil {
 		return nil
 	}
 	info := &Info{}
 	defer file.Close()
 	content, err := ioutil.ReadAll(file)
 	if err != nil {
 		fatalf("Unable to read info: %v", err)
 		return nil
 	}
 	if err = json.Unmarshal(content, &info); err != nil {
 		fatalf("Unable to parse info: %v", err)
 		return nil
 	}
 	return info
 }
 // Get the server info from previous conf file
 // or from the user
 func getInfo(path string) *Info {
 	// Read in the server info if available.
 	infoPath := fmt.Sprintf("%s/info", path)
 	// Delete the old configuration if exist
 	if force {
 		logPath := fmt.Sprintf("%s/log", path)
 		confPath := fmt.Sprintf("%s/conf", path)
 		snapshotPath := fmt.Sprintf("%s/snapshot", path)
 		os.Remove(infoPath)
 		os.Remove(logPath)
 		os.Remove(confPath)
 		os.RemoveAll(snapshotPath)
 	}
 	info := parseInfo(infoPath)
 	if info != nil {
 		fmt.Printf("Found node configuration in '%s'. Ignoring flags.\n", infoPath)
 		return info
 	}
 	info = &argInfo
 	// Write to file.
 	content, _ := json.MarshalIndent(info, "", " ")
 	content = []byte(string(content) + "\n")
 	if err := ioutil.WriteFile(infoPath, content, 0644); err != nil {
 		fatalf("Unable to write info to file: %v", err)
 	}
 	fmt.Printf("Wrote node configuration to '%s'.\n", infoPath)
 	return info
 }
 // Create client auth certpool
 func newCertPool(CAFile string) (tls.ClientAuthType, *x509.CertPool) {
 	if CAFile == "" {
 		return tls.NoClientCert, nil
 	}
 	pemByte, _ := ioutil.ReadFile(CAFile)
 	block, pemByte := pem.Decode(pemByte)
 	cert, err := x509.ParseCertificate(block.Bytes)
 	if err != nil {
 		fatal(err)
 	}
 	certPool := x509.NewCertPool()
 	certPool.AddCert(cert)
 	return tls.RequireAndVerifyClientCert, certPool
 }
 // Send join requests to the leader.
 func joinCluster(s *raft.Server, raftURL string) error {
 	var b bytes.Buffer
 	command := &JoinCommand{
 		Name:    s.Name(),
 		RaftURL: info.RaftURL,
 		EtcdURL: info.EtcdURL,
 	}
 	json.NewEncoder(&b).Encode(command)
 	// t must be ok
 	t, ok := raftServer.Transporter().(transporter)
 	if !ok {
 		panic("wrong type")
 	}
 	joinURL := url.URL{Host: raftURL, Scheme: raftTransporter.scheme, Path: "/join"}
 	debugf("Send Join Request to %s", raftURL)
 	resp, err := t.Post(joinURL.String(), &b)
 	for {
 		if err != nil {
 			return fmt.Errorf("Unable to join: %v", err)
 		}
 		if resp != nil {
 			defer resp.Body.Close()
 			if resp.StatusCode == http.StatusOK {
 				return nil
 			}
 			if resp.StatusCode == http.StatusTemporaryRedirect {
 				address := resp.Header.Get("Location")
 				debugf("Send Join Request to %s", address)
 				json.NewEncoder(&b).Encode(command)
 				resp, err = t.Post(address, &b)
 			} else if resp.StatusCode == http.StatusBadRequest {
 				debug("Reach max number machines in the cluster")
 				return fmt.Errorf(errors[103])
 			} else {
 				return fmt.Errorf("Unable to join")
 			}
 		}
 	}
 	return fmt.Errorf("Unable to join: %v", err)
 }
 // Register commands to raft server
 func registerCommands() {
 	raft.RegisterCommand(&JoinCommand{})
 	raft.RegisterCommand(&SetCommand{})
 	raft.RegisterCommand(&GetCommand{})
 	raft.RegisterCommand(&DeleteCommand{})
 	raft.RegisterCommand(&WatchCommand{})
 	raft.RegisterCommand(&TestAndSetCommand{})
 }
--- a/etcd_handlers.go
+++ b/etcd_handlers.go
@ -1,355 +0,0 @@
 package main
 import (
 	"fmt"
 	"github.com/coreos/etcd/store"
 	"net/http"
 	"strconv"
 	"time"
 )
 //-------------------------------------------------------------------
 // Handlers to handle etcd-store related request via etcd url
 //-------------------------------------------------------------------
 // Multiplex GET/POST/DELETE request to corresponding handlers
 func Multiplexer(w http.ResponseWriter, req *http.Request) {
 	switch req.Method {
 	case "GET":
 		GetHttpHandler(&w, req)
 	case "POST":
 		SetHttpHandler(&w, req)
 	case "DELETE":
 		DeleteHttpHandler(&w, req)
 	default:
 		w.WriteHeader(http.StatusMethodNotAllowed)
 		return
 	}
 }
 //--------------------------------------
 // State sensitive handlers
 // Set/Delete will dispatch to leader
 //--------------------------------------
 // Set Command Handler
 func SetHttpHandler(w *http.ResponseWriter, req *http.Request) {
 	key := req.URL.Path[len("/v1/keys/"):]
 	if store.CheckKeyword(key) {
 		(*w).WriteHeader(http.StatusBadRequest)
 		(*w).Write(newJsonError(400, "Set"))
 		return
 	}
 	debugf("[recv] POST %v/v1/keys/%s", raftServer.Name(), key)
 	value := req.FormValue("value")
 	if len(value) == 0 {
 		(*w).WriteHeader(http.StatusBadRequest)
 		(*w).Write(newJsonError(200, "Set"))
 		return
 	}
 	prevValue := req.FormValue("prevValue")
 	strDuration := req.FormValue("ttl")
 	expireTime, err := durationToExpireTime(strDuration)
 	if err != nil {
 		(*w).WriteHeader(http.StatusBadRequest)
 		(*w).Write(newJsonError(202, "Set"))
 		return
 	}
 	if len(prevValue) != 0 {
 		command := &TestAndSetCommand{
 			Key:        key,
 			Value:      value,
 			PrevValue:  prevValue,
 			ExpireTime: expireTime,
 		}
 		dispatch(command, w, req, true)
 	} else {
 		command := &SetCommand{
 			Key:        key,
 			Value:      value,
 			ExpireTime: expireTime,
 		}
 		dispatch(command, w, req, true)
 	}
 }
 // Delete Handler
 func DeleteHttpHandler(w *http.ResponseWriter, req *http.Request) {
 	key := req.URL.Path[len("/v1/keys/"):]
 	debugf("[recv] DELETE %v/v1/keys/%s", raftServer.Name(), key)
 	command := &DeleteCommand{
 		Key: key,
 	}
 	dispatch(command, w, req, true)
 }
 // Dispatch the command to leader
 func dispatch(c Command, w *http.ResponseWriter, req *http.Request, etcd bool) {
 	if raftServer.State() == "leader" {
 		if body, err := raftServer.Do(c); err != nil {
 			if _, ok := err.(store.NotFoundError); ok {
 				(*w).WriteHeader(http.StatusNotFound)
 				(*w).Write(newJsonError(100, err.Error()))
 				return
 			}
 			if _, ok := err.(store.TestFail); ok {
 				(*w).WriteHeader(http.StatusBadRequest)
 				(*w).Write(newJsonError(101, err.Error()))
 				return
 			}
 			if _, ok := err.(store.NotFile); ok {
 				(*w).WriteHeader(http.StatusBadRequest)
 				(*w).Write(newJsonError(102, err.Error()))
 				return
 			}
 			if err.Error() == errors[103] {
 				(*w).WriteHeader(http.StatusBadRequest)
 				(*w).Write(newJsonError(103, ""))
 				return
 			}
 			(*w).WriteHeader(http.StatusInternalServerError)
 			(*w).Write(newJsonError(300, err.Error()))
 			return
 		} else {
 			if body == nil {
 				(*w).WriteHeader(http.StatusNotFound)
 				(*w).Write(newJsonError(300, "Empty result from raft"))
 			} else {
 				body, ok := body.([]byte)
 				// this should not happen
 				if !ok {
 					panic("wrong type")
 				}
 				(*w).WriteHeader(http.StatusOK)
 				(*w).Write(body)
 			}
 			return
 		}
 	} else {
 		// current no leader
 		if raftServer.Leader() == "" {
 			(*w).WriteHeader(http.StatusInternalServerError)
 			(*w).Write(newJsonError(300, ""))
 			return
 		}
 		// tell the client where is the leader
 		path := req.URL.Path
 		var scheme string
 		if scheme = req.URL.Scheme; scheme == "" {
 			scheme = "http://"
 		}
 		var url string
 		if etcd {
 			etcdAddr, _ := nameToEtcdURL(raftServer.Leader())
 			url = etcdAddr + path
 		} else {
 			raftAddr, _ := nameToRaftURL(raftServer.Leader())
 			url = raftAddr + path
 		}
 		debugf("Redirect to %s", url)
 		http.Redirect(*w, req, url, http.StatusTemporaryRedirect)
 		return
 	}
 	(*w).WriteHeader(http.StatusInternalServerError)
 	(*w).Write(newJsonError(300, ""))
 	return
 }
 //--------------------------------------
 // State non-sensitive handlers
 // will not dispatch to leader
 // TODO: add sensitive version for these
 // command?
 //--------------------------------------
 // Handler to return the current leader's raft address
 func LeaderHttpHandler(w http.ResponseWriter, req *http.Request) {
 	leader := raftServer.Leader()
 	if leader != "" {
 		w.WriteHeader(http.StatusOK)
 		raftURL, _ := nameToRaftURL(leader)
 		w.Write([]byte(raftURL))
 	} else {
 		// not likely, but it may happen
 		w.WriteHeader(http.StatusInternalServerError)
 		w.Write(newJsonError(301, ""))
 	}
 }
 // Handler to return all the known machines in the current cluster
 func MachinesHttpHandler(w http.ResponseWriter, req *http.Request) {
 	peers := raftServer.Peers()
 	// Add itself to the machine list first
 	// Since peer map does not contain the server itself
 	machines, _ := getEtcdURL(raftServer.Name())
 	// Add all peers to the list and separate by comma
 	// We do not use json here since we accept machines list
 	// in the command line separate by comma.
 	for peerName, _ := range peers {
 		if addr, ok := getEtcdURL(peerName); ok {
 			machines = machines + "," + addr
 		}
 	}
 	w.WriteHeader(http.StatusOK)
 	w.Write([]byte(machines))
 }
 // Handler to return the current version of etcd
 func VersionHttpHandler(w http.ResponseWriter, req *http.Request) {
 	w.WriteHeader(http.StatusOK)
 	w.Write([]byte(fmt.Sprintf("etcd %s", releaseVersion)))
 }
 // Handler to return the basic stats of etcd
 func StatsHttpHandler(w http.ResponseWriter, req *http.Request) {
 	w.WriteHeader(http.StatusOK)
 	w.Write(etcdStore.Stats())
 }
 // Get Handler
 func GetHttpHandler(w *http.ResponseWriter, req *http.Request) {
 	key := req.URL.Path[len("/v1/keys/"):]
 	debugf("[recv] GET http://%v/v1/keys/%s", raftServer.Name(), key)
 	command := &GetCommand{
 		Key: key,
 	}
 	if body, err := command.Apply(raftServer); err != nil {
 		if _, ok := err.(store.NotFoundError); ok {
 			(*w).WriteHeader(http.StatusNotFound)
 			(*w).Write(newJsonError(100, err.Error()))
 			return
 		}
 		(*w).WriteHeader(http.StatusInternalServerError)
 		(*w).Write(newJsonError(300, ""))
 	} else {
 		body, ok := body.([]byte)
 		if !ok {
 			panic("wrong type")
 		}
 		(*w).WriteHeader(http.StatusOK)
 		(*w).Write(body)
 	}
 }
 // Watch handler
 func WatchHttpHandler(w http.ResponseWriter, req *http.Request) {
 	key := req.URL.Path[len("/v1/watch/"):]
 	command := &WatchCommand{
 		Key: key,
 	}
 	if req.Method == "GET" {
 		debugf("[recv] GET http://%v/watch/%s", raftServer.Name(), key)
 		command.SinceIndex = 0
 	} else if req.Method == "POST" {
 		// watch from a specific index
 		debugf("[recv] POST http://%v/watch/%s", raftServer.Name(), key)
 		content := req.FormValue("index")
 		sinceIndex, err := strconv.ParseUint(string(content), 10, 64)
 		if err != nil {
 			w.WriteHeader(http.StatusBadRequest)
 			w.Write(newJsonError(203, "Watch From Index"))
 		}
 		command.SinceIndex = sinceIndex
 	} else {
 		w.WriteHeader(http.StatusMethodNotAllowed)
 		return
 	}
 	if body, err := command.Apply(raftServer); err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		w.Write(newJsonError(500, key))
 	} else {
 		w.WriteHeader(http.StatusOK)
 		body, ok := body.([]byte)
 		if !ok {
 			panic("wrong type")
 		}
 		w.Write(body)
 	}
 }
 // TestHandler
 func TestHttpHandler(w http.ResponseWriter, req *http.Request) {
 	testType := req.URL.Path[len("/test/"):]
 	if testType == "speed" {
 		directSet()
 		w.WriteHeader(http.StatusOK)
 		w.Write([]byte("speed test success"))
 		return
 	}
 	w.WriteHeader(http.StatusBadRequest)
 }
 // Convert string duration to time format
 func durationToExpireTime(strDuration string) (time.Time, error) {
 	if strDuration != "" {
 		duration, err := strconv.Atoi(strDuration)
 		if err != nil {
 			return time.Unix(0, 0), err
 		}
 		return time.Now().Add(time.Second * (time.Duration)(duration)), nil
 	} else {
 		return time.Unix(0, 0), nil
 	}
 }
--- a/etcd_long_test.go
+++ b/etcd_long_test.go
@ -1,150 +0,0 @@
 package main
 import (
 	"fmt"
 	"math/rand"
 	"net/http"
 	"os"
 	"strconv"
 	"strings"
 	"testing"
 	"time"
 )
 // This test will kill the current leader and wait for the etcd cluster to elect a new leader for 200 times.
 // It will print out the election time and the average election time.
 func TestKillLeader(t *testing.T) {
 	procAttr := new(os.ProcAttr)
 	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
 	clusterSize := 5
 	argGroup, etcds, err := createCluster(clusterSize, procAttr, false)
 	if err != nil {
 		t.Fatal("cannot create cluster")
 	}
 	defer destroyCluster(etcds)
 	leaderChan := make(chan string, 1)
 	time.Sleep(time.Second)
 	go leaderMonitor(clusterSize, 1, leaderChan)
 	var totalTime time.Duration
 	leader := "http://127.0.0.1:7001"
 	for i := 0; i < clusterSize; i++ {
 		fmt.Println("leader is ", leader)
 		port, _ := strconv.Atoi(strings.Split(leader, ":")[2])
 		num := port - 7001
 		fmt.Println("kill server ", num)
 		etcds[num].Kill()
 		etcds[num].Release()
 		start := time.Now()
 		for {
 			newLeader := <-leaderChan
 			if newLeader != leader {
 				leader = newLeader
 				break
 			}
 		}
 		take := time.Now().Sub(start)
 		totalTime += take
 		avgTime := totalTime / (time.Duration)(i+1)
 		fmt.Println("Leader election time is ", take, "with election timeout", ELECTIONTIMEOUT)
 		fmt.Println("Leader election time average is", avgTime, "with election timeout", ELECTIONTIMEOUT)
 		etcds[num], err = os.StartProcess("etcd", argGroup[num], procAttr)
 	}
 }
 // TestKillRandom kills random machines in the cluster and
 // restart them after all other machines agree on the same leader
 func TestKillRandom(t *testing.T) {
 	procAttr := new(os.ProcAttr)
 	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
 	clusterSize := 9
 	argGroup, etcds, err := createCluster(clusterSize, procAttr, false)
 	if err != nil {
 		t.Fatal("cannot create cluster")
 	}
 	defer destroyCluster(etcds)
 	leaderChan := make(chan string, 1)
 	time.Sleep(3 * time.Second)
 	go leaderMonitor(clusterSize, 4, leaderChan)
 	toKill := make(map[int]bool)
 	for i := 0; i < 20; i++ {
 		fmt.Printf("TestKillRandom Round[%d/20]\n", i)
 		j := 0
 		for {
 			r := rand.Int31n(9)
 			if _, ok := toKill[int(r)]; !ok {
 				j++
 				toKill[int(r)] = true
 			}
 			if j > 3 {
 				break
 			}
 		}
 		for num, _ := range toKill {
 			etcds[num].Kill()
 			etcds[num].Release()
 		}
 		<-leaderChan
 		for num, _ := range toKill {
 			etcds[num], err = os.StartProcess("etcd", argGroup[num], procAttr)
 		}
 		toKill = make(map[int]bool)
 	}
 	<-leaderChan
 }
 func templateBenchmarkEtcdDirectCall(b *testing.B, tls bool) {
 	procAttr := new(os.ProcAttr)
 	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
 	clusterSize := 3
 	_, etcds, _ := createCluster(clusterSize, procAttr, tls)
 	defer destroyCluster(etcds)
 	time.Sleep(time.Second)
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		resp, _ := http.Get("http://127.0.0.1:4001/test/speed")
 		resp.Body.Close()
 	}
 }
 func BenchmarkEtcdDirectCall(b *testing.B) {
 	templateBenchmarkEtcdDirectCall(b, false)
 }
 func BenchmarkEtcdDirectCallTls(b *testing.B) {
 	templateBenchmarkEtcdDirectCall(b, true)
 }
--- a/etcd_test.go
+++ b/etcd_test.go
@ -1,209 +0,0 @@
 package main
 import (
 	"fmt"
 	"github.com/coreos/go-etcd/etcd"
 	"math/rand"
 	"os"
 	//"strconv"
 	"testing"
 	"time"
 )
 // Create a single node and try to set value
 func TestSingleNode(t *testing.T) {
 	procAttr := new(os.ProcAttr)
 	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
 	args := []string{"etcd", "-n=node1", "-f", "-d=/tmp/node1"}
 	process, err := os.StartProcess("etcd", args, procAttr)
 	if err != nil {
 		t.Fatal("start process failed:" + err.Error())
 		return
 	}
 	defer process.Kill()
 	time.Sleep(time.Second)
 	c := etcd.NewClient()
 	c.SyncCluster()
 	// Test Set
 	result, err := c.Set("foo", "bar", 100)
 	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.TTL != 99 {
 		if err != nil {
 			t.Fatal(err)
 		}
 		t.Fatalf("Set 1 failed with %s %s %v", result.Key, result.Value, result.TTL)
 	}
 	time.Sleep(time.Second)
 	result, err = c.Set("foo", "bar", 100)
 	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.PrevValue != "bar" || result.TTL != 99 {
 		if err != nil {
 			t.Fatal(err)
 		}
 		t.Fatalf("Set 2 failed with %s %s %v", result.Key, result.Value, result.TTL)
 	}
 }
 // This test creates a single node and then set a value to it.
 // Then this test kills the node and restart it and tries to get the value again.
 func TestSingleNodeRecovery(t *testing.T) {
 	procAttr := new(os.ProcAttr)
 	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
 	args := []string{"etcd", "-n=node1", "-d=/tmp/node1"}
 	process, err := os.StartProcess("etcd", append(args, "-f"), procAttr)
 	if err != nil {
 		t.Fatal("start process failed:" + err.Error())
 		return
 	}
 	time.Sleep(time.Second)
 	c := etcd.NewClient()
 	c.SyncCluster()
 	// Test Set
 	result, err := c.Set("foo", "bar", 100)
 	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.TTL != 99 {
 		if err != nil {
 			t.Fatal(err)
 		}
 		t.Fatalf("Set 1 failed with %s %s %v", result.Key, result.Value, result.TTL)
 	}
 	time.Sleep(time.Second)
 	process.Kill()
 	process, err = os.StartProcess("etcd", args, procAttr)
 	defer process.Kill()
 	if err != nil {
 		t.Fatal("start process failed:" + err.Error())
 		return
 	}
 	time.Sleep(time.Second)
 	results, err := c.Get("foo")
 	if err != nil {
 		t.Fatal("get fail: " + err.Error())
 		return
 	}
 	result = results[0]
 	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.TTL > 99 {
 		if err != nil {
 			t.Fatal(err)
 		}
 		t.Fatalf("Recovery Get failed with %s %s %v", result.Key, result.Value, result.TTL)
 	}
 }
 // Create a three nodes and try to set value
 func templateTestSimpleMultiNode(t *testing.T, tls bool) {
 	procAttr := new(os.ProcAttr)
 	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
 	clusterSize := 3
 	_, etcds, err := createCluster(clusterSize, procAttr, tls)
 	if err != nil {
 		t.Fatal("cannot create cluster")
 	}
 	defer destroyCluster(etcds)
 	time.Sleep(time.Second)
 	c := etcd.NewClient()
 	c.SyncCluster()
 	// Test Set
 	result, err := c.Set("foo", "bar", 100)
 	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.TTL != 99 {
 		if err != nil {
 			t.Fatal(err)
 		}
 		t.Fatalf("Set 1 failed with %s %s %v", result.Key, result.Value, result.TTL)
 	}
 	time.Sleep(time.Second)
 	result, err = c.Set("foo", "bar", 100)
 	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.PrevValue != "bar" || result.TTL != 99 {
 		if err != nil {
 			t.Fatal(err)
 		}
 		t.Fatalf("Set 2 failed with %s %s %v", result.Key, result.Value, result.TTL)
 	}
 }
 func TestSimpleMultiNode(t *testing.T) {
 	templateTestSimpleMultiNode(t, false)
 }
 func TestSimpleMultiNodeTls(t *testing.T) {
 	templateTestSimpleMultiNode(t, true)
 }
 // Create a five nodes
 // Randomly kill one of the node and keep on sending set command to the cluster
 func TestMultiNodeRecovery(t *testing.T) {
 	procAttr := new(os.ProcAttr)
 	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
 	clusterSize := 5
 	argGroup, etcds, err := createCluster(clusterSize, procAttr, false)
 	if err != nil {
 		t.Fatal("cannot create cluster")
 	}
 	defer destroyCluster(etcds)
 	time.Sleep(2 * time.Second)
 	c := etcd.NewClient()
 	c.SyncCluster()
 	stop := make(chan bool)
 	// Test Set
 	go set(stop)
 	for i := 0; i < 10; i++ {
 		num := rand.Int() % clusterSize
 		fmt.Println("kill node", num+1)
 		// kill
 		etcds[num].Kill()
 		etcds[num].Release()
 		time.Sleep(time.Second)
 		// restart
 		etcds[num], err = os.StartProcess("etcd", argGroup[num], procAttr)
 		if err != nil {
 			panic(err)
 		}
 		time.Sleep(time.Second)
 	}
 	fmt.Println("stop")
 	stop <- true
 	<-stop
 }
--- a/go_version.go
+++ b/go_version.go
@ -0,0 +1,3 @@
 // +build !go1.1
 "etcd requires go 1.1 or greater to build"
--- a/log/log.go
+++ b/log/log.go
@ -0,0 +1,44 @@
 package log
 import (
 	golog "github.com/coreos/go-log/log"
 	"os"
 )
 // The Verbose flag turns on verbose logging.
 var Verbose bool = false
 var logger *golog.Logger = golog.New("etcd", false,
 	golog.CombinedSink(os.Stdout, "[%s] %s %-9s | %s\n", []string{"prefix", "time", "priority", "message"}))
 func Infof(format string, v ...interface{}) {
 	logger.Infof(format, v...)
 }
 func Debugf(format string, v ...interface{}) {
 	if Verbose {
 		logger.Debugf(format, v...)
 	}
 }
 func Debug(v ...interface{}) {
 	if Verbose {
 		logger.Debug(v...)
 	}
 }
 func Warnf(format string, v ...interface{}) {
 	logger.Warningf(format, v...)
 }
 func Warn(v ...interface{}) {
 	logger.Warning(v...)
 }
 func Fatalf(format string, v ...interface{}) {
 	logger.Fatalf(format, v...)
 }
 func Fatal(v ...interface{}) {
 	logger.Fatalln(v...)
 }
--- a/machines.go
+++ b/machines.go
@ -1,27 +0,0 @@
 package main
 import (
 	"net/url"
 	"path"
 )
 func getEtcdURL(name string) (string, bool) {
 	resps, _ := etcdStore.RawGet(path.Join("_etcd/machines", name))
 	m, err := url.ParseQuery(resps[0].Value)
 	if err != nil {
 		panic("Failed to parse machines entry")
 	}
 	addr := m["etcd"][0]
 	return addr, true
 }
 // machineNum returns the number of machines in the cluster
 func machineNum() int {
 	response, _ := etcdStore.RawGet("_etcd/machines")
 	return len(response)
 }
--- a/name_url_map.go
+++ b/name_url_map.go
@ -1,68 +0,0 @@
 package main
 import (
 	"net/url"
 	"path"
 )
 // we map node name to url
 type nodeInfo struct {
 	raftURL string
 	etcdURL string
 }
 var namesMap = make(map[string]*nodeInfo)
 // nameToEtcdURL maps node name to its etcd http address
 func nameToEtcdURL(name string) (string, bool) {
 	if info, ok := namesMap[name]; ok {
 		// first try to read from the map
 		return info.etcdURL, true
 	}
 	// if fails, try to recover from etcd storage
 	return readURL(name, "etcd")
 }
 // nameToRaftURL maps node name to its raft http address
 func nameToRaftURL(name string) (string, bool) {
 	if info, ok := namesMap[name]; ok {
 		// first try to read from the map
 		return info.raftURL, true
 	}
 	// if fails, try to recover from etcd storage
 	return readURL(name, "raft")
 }
 // addNameToURL add a name that maps to raftURL and etcdURL
 func addNameToURL(name string, raftURL string, etcdURL string) {
 	namesMap[name] = &nodeInfo{
 		raftURL: raftURL,
 		etcdURL: etcdURL,
 	}
 }
 func readURL(nodeName string, urlName string) (string, bool) {
 	// if fails, try to recover from etcd storage
 	key := path.Join("/_etcd/machines", nodeName)
 	resps, err := etcdStore.RawGet(key)
 	if err != nil {
 		return "", false
 	}
 	m, err := url.ParseQuery(resps[0].Value)
 	if err != nil {
 		panic("Failed to parse machines entry")
 	}
 	url := m[urlName][0]
 	return url, true
 }
--- a/raft_handlers.go
+++ b/raft_handlers.go
@ -1,115 +0,0 @@
 package main
 import (
 	"encoding/json"
 	"github.com/coreos/go-raft"
 	"net/http"
 )
 //-------------------------------------------------------------
 // Handlers to handle raft related request via raft server port
 //-------------------------------------------------------------
 // Get all the current logs
 func GetLogHttpHandler(w http.ResponseWriter, req *http.Request) {
 	debugf("[recv] GET %s/log", raftTransporter.scheme+raftServer.Name())
 	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(raftServer.LogEntries())
 }
 // Response to vote request
 func VoteHttpHandler(w http.ResponseWriter, req *http.Request) {
 	rvreq := &raft.RequestVoteRequest{}
 	err := decodeJsonRequest(req, rvreq)
 	if err == nil {
 		debugf("[recv] POST %s/vote [%s]", raftTransporter.scheme+raftServer.Name(), rvreq.CandidateName)
 		if resp := raftServer.RequestVote(rvreq); resp != nil {
 			w.WriteHeader(http.StatusOK)
 			json.NewEncoder(w).Encode(resp)
 			return
 		}
 	}
 	warnf("[vote] ERROR: %v", err)
 	w.WriteHeader(http.StatusInternalServerError)
 }
 // Response to append entries request
 func AppendEntriesHttpHandler(w http.ResponseWriter, req *http.Request) {
 	aereq := &raft.AppendEntriesRequest{}
 	err := decodeJsonRequest(req, aereq)
 	if err == nil {
 		debugf("[recv] POST %s/log/append [%d]", raftTransporter.scheme+raftServer.Name(), len(aereq.Entries))
 		if resp := raftServer.AppendEntries(aereq); resp != nil {
 			w.WriteHeader(http.StatusOK)
 			json.NewEncoder(w).Encode(resp)
 			if !resp.Success {
 				debugf("[Append Entry] Step back")
 			}
 			return
 		}
 	}
 	warnf("[Append Entry] ERROR: %v", err)
 	w.WriteHeader(http.StatusInternalServerError)
 }
 // Response to recover from snapshot request
 func SnapshotHttpHandler(w http.ResponseWriter, req *http.Request) {
 	aereq := &raft.SnapshotRequest{}
 	err := decodeJsonRequest(req, aereq)
 	if err == nil {
 		debugf("[recv] POST %s/snapshot/ ", raftTransporter.scheme+raftServer.Name())
 		if resp := raftServer.RequestSnapshot(aereq); resp != nil {
 			w.WriteHeader(http.StatusOK)
 			json.NewEncoder(w).Encode(resp)
 			return
 		}
 	}
 	warnf("[Snapshot] ERROR: %v", err)
 	w.WriteHeader(http.StatusInternalServerError)
 }
 // Response to recover from snapshot request
 func SnapshotRecoveryHttpHandler(w http.ResponseWriter, req *http.Request) {
 	aereq := &raft.SnapshotRecoveryRequest{}
 	err := decodeJsonRequest(req, aereq)
 	if err == nil {
 		debugf("[recv] POST %s/snapshotRecovery/ ", raftTransporter.scheme+raftServer.Name())
 		if resp := raftServer.SnapshotRecoveryRequest(aereq); resp != nil {
 			w.WriteHeader(http.StatusOK)
 			json.NewEncoder(w).Encode(resp)
 			return
 		}
 	}
 	warnf("[Snapshot] ERROR: %v", err)
 	w.WriteHeader(http.StatusInternalServerError)
 }
 // Get the port that listening for etcd connecting of the server
 func EtcdURLHttpHandler(w http.ResponseWriter, req *http.Request) {
 	debugf("[recv] Get %s/etcdURL/ ", raftTransporter.scheme+raftServer.Name())
 	w.WriteHeader(http.StatusOK)
 	w.Write([]byte(argInfo.EtcdURL))
 }
 // Response to the join request
 func JoinHttpHandler(w http.ResponseWriter, req *http.Request) {
 	command := &JoinCommand{}
 	if err := decodeJsonRequest(req, command); err == nil {
 		debugf("Receive Join Request from %s", command.Name)
 		dispatch(command, &w, req, false)
 	} else {
 		w.WriteHeader(http.StatusInternalServerError)
 		return
 	}
 }
 // Response to the name request
 func NameHttpHandler(w http.ResponseWriter, req *http.Request) {
 	debugf("[recv] Get %s/name/ ", raftTransporter.scheme+raftServer.Name())
 	w.WriteHeader(http.StatusOK)
 	w.Write([]byte(raftServer.Name()))
 }
--- a/scripts/release-version
+++ b/scripts/release-version
@ -3,6 +3,6 @@
 VER=$(git describe --tags HEAD)
 cat <<EOF
-package main
+package server
-const releaseVersion = "$VER"
+const ReleaseVersion = "$VER"
 EOF
--- a/scripts/release-version.ps1
+++ b/scripts/release-version.ps1
@ -0,0 +1,7 @@
 $VER=(git describe --tags HEAD)
@"
 package main
 const releaseVersion = "$VER"
 "@
--- a/scripts/test-cluster
+++ b/scripts/test-cluster
@ -0,0 +1,19 @@
 #!/bin/bash
 SESSION=etcd-cluster
 tmux new-session -d -s $SESSION
 # Setup a window for tailing log files
 tmux new-window -t $SESSION:1 -n 'machines'
 tmux split-window -h
 tmux select-pane -t 0
 tmux send-keys "./etcd -s 127.0.0.1:7001 -c 127.0.0.1:4001 -d machine1 -n machine1" C-m
 for i in 2 3; do
 	tmux select-pane -t 0
 	tmux split-window -v
 	tmux send-keys "./etcd -cors='*' -s 127.0.0.1:700${i} -c 127.0.0.1:400${i} -C 127.0.0.1:7001 -d machine${i} -n machine${i}" C-m
 done
 # Attach to session
 tmux attach-session -t $SESSION
--- a/server/join_command.go
+++ b/server/join_command.go
@ -0,0 +1,75 @@
 package server
 import (
 	"encoding/binary"
 	etcdErr "github.com/coreos/etcd/error"
 	"github.com/coreos/etcd/log"
 	"github.com/coreos/go-raft"
 )
 func init() {
 	raft.RegisterCommand(&JoinCommand{})
 }
 // The JoinCommand adds a node to the cluster.
 type JoinCommand struct {
 	RaftVersion string `json:"raftVersion"`
 	Name        string `json:"name"`
 	RaftURL     string `json:"raftURL"`
 	EtcdURL     string `json:"etcdURL"`
 }
 func NewJoinCommand(version, name, raftUrl, etcdUrl string) *JoinCommand {
 	return &JoinCommand{
 		RaftVersion: version,
 		Name:        name,
 		RaftURL:     raftUrl,
 		EtcdURL:     etcdUrl,
 	}
 }
 // The name of the join command in the log
 func (c *JoinCommand) CommandName() string {
 	return "etcd:join"
 }
 // Join a server to the cluster
 func (c *JoinCommand) Apply(server raft.Server) (interface{}, error) {
 	ps, _ := server.Context().(*PeerServer)
 	b := make([]byte, 8)
 	binary.PutUvarint(b, server.CommitIndex())
 	// Make sure we're not getting a cached value from the registry.
 	ps.registry.Invalidate(c.Name)
 	// Check if the join command is from a previous machine, who lost all its previous log.
 	if _, ok := ps.registry.ClientURL(c.Name); ok {
 		return b, nil
 	}
 	// Check machine number in the cluster
 	if ps.registry.Count() == ps.MaxClusterSize {
 		log.Debug("Reject join request from ", c.Name)
 		return []byte{0}, etcdErr.NewError(etcdErr.EcodeNoMoreMachine, "", server.CommitIndex(), server.Term())
 	}
 	// Add to shared machine registry.
 	ps.registry.Register(c.Name, c.RaftVersion, c.RaftURL, c.EtcdURL, server.CommitIndex(), server.Term())
 	// Add peer in raft
 	err := server.AddPeer(c.Name, "")
 	// Add peer stats
 	if c.Name != ps.RaftServer().Name() {
 		ps.followersStats.Followers[c.Name] = &raftFollowerStats{}
 		ps.followersStats.Followers[c.Name].Latency.Minimum = 1 << 63
 	}
 	return b, err
 }
 func (c *JoinCommand) NodeName() string {
 	return c.Name
 }
--- a/server/package_stats.go
+++ b/server/package_stats.go
@ -0,0 +1,25 @@
 package server
 import (
 	"time"
 )
 // packageStats represent the stats we need for a package.
 // It has sending time and the size of the package.
 type packageStats struct {
 	sendingTime time.Time
 	size        int
 }
 // NewPackageStats creates a pacakgeStats and return the pointer to it.
 func NewPackageStats(now time.Time, size int) *packageStats {
 	return &packageStats{
 		sendingTime: now,
 		size:        size,
 	}
 }
 // Time return the sending time of the package.
 func (ps *packageStats) Time() time.Time {
 	return ps.sendingTime
 }
--- a/server/peer_server.go
+++ b/server/peer_server.go
@ -0,0 +1,396 @@
 package server
 import (
 	"bytes"
 	"crypto/tls"
 	"encoding/binary"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strings"
 	"time"
 	etcdErr "github.com/coreos/etcd/error"
 	"github.com/coreos/etcd/log"
 	"github.com/coreos/etcd/store"
 	"github.com/coreos/go-raft"
 )
 type PeerServer struct {
 	raftServer     raft.Server
 	server         *Server
 	joinIndex      uint64
 	name           string
 	url            string
 	listenHost     string
 	tlsConf        *TLSConfig
 	tlsInfo        *TLSInfo
 	followersStats *raftFollowersStats
 	serverStats    *raftServerStats
 	registry       *Registry
 	store          store.Store
 	snapConf       *snapshotConf
 	MaxClusterSize int
 	RetryTimes     int
 }
 // TODO: find a good policy to do snapshot
 type snapshotConf struct {
 	// Etcd will check if snapshot is need every checkingInterval
 	checkingInterval time.Duration
 	// The number of writes when the last snapshot happened
 	lastWrites uint64
 	// If the incremental number of writes since the last snapshot
 	// exceeds the write Threshold, etcd will do a snapshot
 	writesThr uint64
 }
 func NewPeerServer(name string, path string, url string, listenHost string, tlsConf *TLSConfig, tlsInfo *TLSInfo, registry *Registry, store store.Store) *PeerServer {
 	s := &PeerServer{
 		name:       name,
 		url:        url,
 		listenHost: listenHost,
 		tlsConf:    tlsConf,
 		tlsInfo:    tlsInfo,
 		registry:   registry,
 		store:      store,
 		snapConf:   &snapshotConf{time.Second * 3, 0, 20 * 1000},
 		followersStats: &raftFollowersStats{
 			Leader:    name,
 			Followers: make(map[string]*raftFollowerStats),
 		},
 		serverStats: &raftServerStats{
 			StartTime: time.Now(),
 			sendRateQueue: &statsQueue{
 				back: -1,
 			},
 			recvRateQueue: &statsQueue{
 				back: -1,
 			},
 		},
 	}
 	// Create transporter for raft
 	raftTransporter := newTransporter(tlsConf.Scheme, tlsConf.Client, s)
 	// Create raft server
 	raftServer, err := raft.NewServer(name, path, raftTransporter, s.store, s, "")
 	if err != nil {
 		log.Fatal(err)
 	}
 	s.raftServer = raftServer
 	return s
 }
 // Start the raft server
 func (s *PeerServer) ListenAndServe(snapshot bool, cluster []string) {
 	// LoadSnapshot
 	if snapshot {
 		err := s.raftServer.LoadSnapshot()
 		if err == nil {
 			log.Debugf("%s finished load snapshot", s.name)
 		} else {
 			log.Debug(err)
 		}
 	}
 	s.raftServer.SetElectionTimeout(ElectionTimeout)
 	s.raftServer.SetHeartbeatTimeout(HeartbeatTimeout)
 	s.raftServer.Start()
 	if s.raftServer.IsLogEmpty() {
 		// start as a leader in a new cluster
 		if len(cluster) == 0 {
 			s.startAsLeader()
 		} else {
 			s.startAsFollower(cluster)
 		}
 	} else {
 		// Rejoin the previous cluster
 		cluster = s.registry.PeerURLs(s.raftServer.Leader(), s.name)
 		for i := 0; i < len(cluster); i++ {
 			u, err := url.Parse(cluster[i])
 			if err != nil {
 				log.Debug("rejoin cannot parse url: ", err)
 			}
 			cluster[i] = u.Host
 		}
 		ok := s.joinCluster(cluster)
 		if !ok {
 			log.Warn("the entire cluster is down! this machine will restart the cluster.")
 		}
 		log.Debugf("%s restart as a follower", s.name)
 	}
 	// open the snapshot
 	if snapshot {
 		go s.monitorSnapshot()
 	}
 	// start to response to raft requests
 	go s.startTransport(s.tlsConf.Scheme, s.tlsConf.Server)
 }
 // Retrieves the underlying Raft server.
 func (s *PeerServer) RaftServer() raft.Server {
 	return s.raftServer
 }
 // Associates the client server with the peer server.
 func (s *PeerServer) SetServer(server *Server) {
 	s.server = server
 }
 func (s *PeerServer) startAsLeader() {
 	// leader need to join self as a peer
 	for {
 		_, err := s.raftServer.Do(NewJoinCommand(PeerVersion, s.raftServer.Name(), s.url, s.server.URL()))
 		if err == nil {
 			break
 		}
 	}
 	log.Debugf("%s start as a leader", s.name)
 }
 func (s *PeerServer) startAsFollower(cluster []string) {
 	// start as a follower in a existing cluster
 	for i := 0; i < s.RetryTimes; i++ {
 		ok := s.joinCluster(cluster)
 		if ok {
 			return
 		}
 		log.Warnf("cannot join to cluster via given machines, retry in %d seconds", RetryInterval)
 		time.Sleep(time.Second * RetryInterval)
 	}
 	log.Fatalf("Cannot join the cluster via given machines after %x retries", s.RetryTimes)
 }
 // Start to listen and response raft command
 func (s *PeerServer) startTransport(scheme string, tlsConf tls.Config) {
 	log.Infof("raft server [name %s, listen on %s, advertised url %s]", s.name, s.listenHost, s.url)
 	raftMux := http.NewServeMux()
 	server := &http.Server{
 		Handler:   raftMux,
 		TLSConfig: &tlsConf,
 		Addr:      s.listenHost,
 	}
 	// internal commands
 	raftMux.HandleFunc("/name", s.NameHttpHandler)
 	raftMux.HandleFunc("/version", s.RaftVersionHttpHandler)
 	raftMux.HandleFunc("/join", s.JoinHttpHandler)
 	raftMux.HandleFunc("/remove/", s.RemoveHttpHandler)
 	raftMux.HandleFunc("/vote", s.VoteHttpHandler)
 	raftMux.HandleFunc("/log", s.GetLogHttpHandler)
 	raftMux.HandleFunc("/log/append", s.AppendEntriesHttpHandler)
 	raftMux.HandleFunc("/snapshot", s.SnapshotHttpHandler)
 	raftMux.HandleFunc("/snapshotRecovery", s.SnapshotRecoveryHttpHandler)
 	raftMux.HandleFunc("/etcdURL", s.EtcdURLHttpHandler)
 	if scheme == "http" {
 		log.Fatal(server.ListenAndServe())
 	} else {
 		log.Fatal(server.ListenAndServeTLS(s.tlsInfo.CertFile, s.tlsInfo.KeyFile))
 	}
 }
 // getVersion fetches the raft version of a peer. This works for now but we
 // will need to do something more sophisticated later when we allow mixed
 // version clusters.
 func getVersion(t *transporter, versionURL url.URL) (string, error) {
 	resp, req, err := t.Get(versionURL.String())
 	if err != nil {
 		return "", err
 	}
 	defer resp.Body.Close()
 	t.CancelWhenTimeout(req)
 	body, err := ioutil.ReadAll(resp.Body)
 	return string(body), nil
 }
 func (s *PeerServer) joinCluster(cluster []string) bool {
 	for _, machine := range cluster {
 		if len(machine) == 0 {
 			continue
 		}
 		err := s.joinByMachine(s.raftServer, machine, s.tlsConf.Scheme)
 		if err == nil {
 			log.Debugf("%s success join to the cluster via machine %s", s.name, machine)
 			return true
 		} else {
 			if _, ok := err.(etcdErr.Error); ok {
 				log.Fatal(err)
 			}
 			log.Debugf("cannot join to cluster via machine %s %s", machine, err)
 		}
 	}
 	return false
 }
 // Send join requests to machine.
 func (s *PeerServer) joinByMachine(server raft.Server, machine string, scheme string) error {
 	var b bytes.Buffer
 	// t must be ok
 	t, _ := server.Transporter().(*transporter)
 	// Our version must match the leaders version
 	versionURL := url.URL{Host: machine, Scheme: scheme, Path: "/version"}
 	version, err := getVersion(t, versionURL)
 	if err != nil {
 		return fmt.Errorf("Error during join version check: %v", err)
 	}
 	// TODO: versioning of the internal protocol. See:
 	// Documentation/internatl-protocol-versioning.md
 	if version != PeerVersion {
 		return fmt.Errorf("Unable to join: internal version mismatch, entire cluster must be running identical versions of etcd")
 	}
 	json.NewEncoder(&b).Encode(NewJoinCommand(PeerVersion, server.Name(), s.url, s.server.URL()))
 	joinURL := url.URL{Host: machine, Scheme: scheme, Path: "/join"}
 	log.Debugf("Send Join Request to %s", joinURL.String())
 	resp, req, err := t.Post(joinURL.String(), &b)
 	for {
 		if err != nil {
 			return fmt.Errorf("Unable to join: %v", err)
 		}
 		if resp != nil {
 			defer resp.Body.Close()
 			t.CancelWhenTimeout(req)
 			if resp.StatusCode == http.StatusOK {
 				b, _ := ioutil.ReadAll(resp.Body)
 				s.joinIndex, _ = binary.Uvarint(b)
 				return nil
 			}
 			if resp.StatusCode == http.StatusTemporaryRedirect {
 				address := resp.Header.Get("Location")
 				log.Debugf("Send Join Request to %s", address)
 				json.NewEncoder(&b).Encode(NewJoinCommand(PeerVersion, server.Name(), s.url, s.server.URL()))
 				resp, req, err = t.Post(address, &b)
 			} else if resp.StatusCode == http.StatusBadRequest {
 				log.Debug("Reach max number machines in the cluster")
 				decoder := json.NewDecoder(resp.Body)
 				err := &etcdErr.Error{}
 				decoder.Decode(err)
 				return *err
 			} else {
 				return fmt.Errorf("Unable to join")
 			}
 		}
 	}
 }
 func (s *PeerServer) Stats() []byte {
 	s.serverStats.LeaderInfo.Uptime = time.Now().Sub(s.serverStats.LeaderInfo.startTime).String()
 	queue := s.serverStats.sendRateQueue
 	s.serverStats.SendingPkgRate, s.serverStats.SendingBandwidthRate = queue.Rate()
 	queue = s.serverStats.recvRateQueue
 	s.serverStats.RecvingPkgRate, s.serverStats.RecvingBandwidthRate = queue.Rate()
 	b, _ := json.Marshal(s.serverStats)
 	return b
 }
 func (s *PeerServer) PeerStats() []byte {
 	if s.raftServer.State() == raft.Leader {
 		b, _ := json.Marshal(s.followersStats)
 		return b
 	}
 	return nil
 }
 func (s *PeerServer) monitorSnapshot() {
 	for {
 		time.Sleep(s.snapConf.checkingInterval)
 		currentWrites := 0
 		if uint64(currentWrites) > s.snapConf.writesThr {
 			s.raftServer.TakeSnapshot()
 			s.snapConf.lastWrites = 0
 		}
 	}
 }
 func (s *PeerServer) dispatch(c raft.Command, w http.ResponseWriter, req *http.Request) error {
 	if s.raftServer.State() == raft.Leader {
 		result, err := s.raftServer.Do(c)
 		if err != nil {
 			return err
 		}
 		if result == nil {
 			return etcdErr.NewError(300, "Empty result from raft", store.UndefIndex, store.UndefTerm)
 		}
 		// response for raft related commands[join/remove]
 		if b, ok := result.([]byte); ok {
 			w.WriteHeader(http.StatusOK)
 			w.Write(b)
 			return nil
 		}
 		var b []byte
 		if strings.HasPrefix(req.URL.Path, "/v1") {
 			b, _ = json.Marshal(result.(*store.Event).Response())
 		} else {
 			b, _ = json.Marshal(result.(*store.Event))
 		}
 		w.WriteHeader(http.StatusOK)
 		w.Write(b)
 		return nil
 	} else {
 		leader := s.raftServer.Leader()
 		// No leader available.
 		if leader == "" {
 			return etcdErr.NewError(300, "", store.UndefIndex, store.UndefTerm)
 		}
 		var url string
 		switch c.(type) {
 		case *JoinCommand, *RemoveCommand:
 			url, _ = s.registry.PeerURL(leader)
 		default:
 			url, _ = s.registry.ClientURL(leader)
 		}
 		redirect(url, w, req)
 		return nil
 	}
 }
--- a/server/peer_server_handlers.go
+++ b/server/peer_server_handlers.go
@ -0,0 +1,158 @@
 package server
 import (
 	"encoding/json"
 	"net/http"
 	etcdErr "github.com/coreos/etcd/error"
 	"github.com/coreos/etcd/log"
 	"github.com/coreos/go-raft"
 )
 // Get all the current logs
 func (s *PeerServer) GetLogHttpHandler(w http.ResponseWriter, req *http.Request) {
 	log.Debugf("[recv] GET %s/log", s.url)
 	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(s.raftServer.LogEntries())
 }
 // Response to vote request
 func (s *PeerServer) VoteHttpHandler(w http.ResponseWriter, req *http.Request) {
 	rvreq := &raft.RequestVoteRequest{}
 	err := decodeJsonRequest(req, rvreq)
 	if err == nil {
 		log.Debugf("[recv] POST %s/vote [%s]", s.url, rvreq.CandidateName)
 		if resp := s.raftServer.RequestVote(rvreq); resp != nil {
 			w.WriteHeader(http.StatusOK)
 			json.NewEncoder(w).Encode(resp)
 			return
 		}
 	}
 	log.Warnf("[vote] ERROR: %v", err)
 	w.WriteHeader(http.StatusInternalServerError)
 }
 // Response to append entries request
 func (s *PeerServer) AppendEntriesHttpHandler(w http.ResponseWriter, req *http.Request) {
 	aereq := &raft.AppendEntriesRequest{}
 	err := decodeJsonRequest(req, aereq)
 	if err == nil {
 		log.Debugf("[recv] POST %s/log/append [%d]", s.url, len(aereq.Entries))
 		s.serverStats.RecvAppendReq(aereq.LeaderName, int(req.ContentLength))
 		if resp := s.raftServer.AppendEntries(aereq); resp != nil {
 			w.WriteHeader(http.StatusOK)
 			json.NewEncoder(w).Encode(resp)
 			if !resp.Success {
 				log.Debugf("[Append Entry] Step back")
 			}
 			return
 		}
 	}
 	log.Warnf("[Append Entry] ERROR: %v", err)
 	w.WriteHeader(http.StatusInternalServerError)
 }
 // Response to recover from snapshot request
 func (s *PeerServer) SnapshotHttpHandler(w http.ResponseWriter, req *http.Request) {
 	aereq := &raft.SnapshotRequest{}
 	err := decodeJsonRequest(req, aereq)
 	if err == nil {
 		log.Debugf("[recv] POST %s/snapshot/ ", s.url)
 		if resp := s.raftServer.RequestSnapshot(aereq); resp != nil {
 			w.WriteHeader(http.StatusOK)
 			json.NewEncoder(w).Encode(resp)
 			return
 		}
 	}
 	log.Warnf("[Snapshot] ERROR: %v", err)
 	w.WriteHeader(http.StatusInternalServerError)
 }
 // Response to recover from snapshot request
 func (s *PeerServer) SnapshotRecoveryHttpHandler(w http.ResponseWriter, req *http.Request) {
 	aereq := &raft.SnapshotRecoveryRequest{}
 	err := decodeJsonRequest(req, aereq)
 	if err == nil {
 		log.Debugf("[recv] POST %s/snapshotRecovery/ ", s.url)
 		if resp := s.raftServer.SnapshotRecoveryRequest(aereq); resp != nil {
 			w.WriteHeader(http.StatusOK)
 			json.NewEncoder(w).Encode(resp)
 			return
 		}
 	}
 	log.Warnf("[Snapshot] ERROR: %v", err)
 	w.WriteHeader(http.StatusInternalServerError)
 }
 // Get the port that listening for etcd connecting of the server
 func (s *PeerServer) EtcdURLHttpHandler(w http.ResponseWriter, req *http.Request) {
 	log.Debugf("[recv] Get %s/etcdURL/ ", s.url)
 	w.WriteHeader(http.StatusOK)
 	w.Write([]byte(s.server.URL()))
 }
 // Response to the join request
 func (s *PeerServer) JoinHttpHandler(w http.ResponseWriter, req *http.Request) {
 	command := &JoinCommand{}
 	// Write CORS header.
 	if s.server.OriginAllowed("*") {
 		w.Header().Add("Access-Control-Allow-Origin", "*")
 	} else if s.server.OriginAllowed(req.Header.Get("Origin")) {
 		w.Header().Add("Access-Control-Allow-Origin", req.Header.Get("Origin"))
 	}
 	err := decodeJsonRequest(req, command)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		return
 	}
 	log.Debugf("Receive Join Request from %s", command.Name)
 	err = s.dispatch(command, w, req)
 	// Return status.
 	if err != nil {
 		if etcdErr, ok := err.(*etcdErr.Error); ok {
 			log.Debug("Return error: ", (*etcdErr).Error())
 			etcdErr.Write(w)
 		} else {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 		}
 	}
 }
 // Response to remove request
 func (s *PeerServer) RemoveHttpHandler(w http.ResponseWriter, req *http.Request) {
 	if req.Method != "DELETE" {
 		w.WriteHeader(http.StatusMethodNotAllowed)
 		return
 	}
 	nodeName := req.URL.Path[len("/remove/"):]
 	command := &RemoveCommand{
 		Name: nodeName,
 	}
 	log.Debugf("[recv] Remove Request [%s]", command.Name)
 	s.dispatch(command, w, req)
 }
 // Response to the name request
 func (s *PeerServer) NameHttpHandler(w http.ResponseWriter, req *http.Request) {
 	log.Debugf("[recv] Get %s/name/ ", s.url)
 	w.WriteHeader(http.StatusOK)
 	w.Write([]byte(s.name))
 }
 // Response to the name request
 func (s *PeerServer) RaftVersionHttpHandler(w http.ResponseWriter, req *http.Request) {
 	log.Debugf("[recv] Get %s/version/ ", s.url)
 	w.WriteHeader(http.StatusOK)
 	w.Write([]byte(PeerVersion))
 }
--- a/server/raft_follower_stats.go
+++ b/server/raft_follower_stats.go
@ -0,0 +1,56 @@
 package server
 import (
 	"math"
 	"time"
 )
 type raftFollowersStats struct {
 	Leader    string                        `json:"leader"`
 	Followers map[string]*raftFollowerStats `json:"followers"`
 }
 type raftFollowerStats struct {
 	Latency struct {
 		Current           float64 `json:"current"`
 		Average           float64 `json:"average"`
 		averageSquare     float64
 		StandardDeviation float64 `json:"standardDeviation"`
 		Minimum           float64 `json:"minimum"`
 		Maximum           float64 `json:"maximum"`
 	} `json:"latency"`
 	Counts struct {
 		Fail    uint64 `json:"fail"`
 		Success uint64 `json:"success"`
 	} `json:"counts"`
 }
 // Succ function update the raftFollowerStats with a successful send
 func (ps *raftFollowerStats) Succ(d time.Duration) {
 	total := float64(ps.Counts.Success) * ps.Latency.Average
 	totalSquare := float64(ps.Counts.Success) * ps.Latency.averageSquare
 	ps.Counts.Success++
 	ps.Latency.Current = float64(d) / (1000000.0)
 	if ps.Latency.Current > ps.Latency.Maximum {
 		ps.Latency.Maximum = ps.Latency.Current
 	}
 	if ps.Latency.Current < ps.Latency.Minimum {
 		ps.Latency.Minimum = ps.Latency.Current
 	}
 	ps.Latency.Average = (total + ps.Latency.Current) / float64(ps.Counts.Success)
 	ps.Latency.averageSquare = (totalSquare + ps.Latency.Current*ps.Latency.Current) / float64(ps.Counts.Success)
 	// sdv = sqrt(avg(x^2) - avg(x)^2)
 	ps.Latency.StandardDeviation = math.Sqrt(ps.Latency.averageSquare - ps.Latency.Average*ps.Latency.Average)
 }
 // Fail function update the raftFollowerStats with a unsuccessful send
 func (ps *raftFollowerStats) Fail() {
 	ps.Counts.Fail++
 }
--- a/server/raft_server_stats.go
+++ b/server/raft_server_stats.go
@ -0,0 +1,55 @@
 package server
 import (
 	"time"
 	"github.com/coreos/go-raft"
 )
 type raftServerStats struct {
 	Name      string    `json:"name"`
 	State     string    `json:"state"`
 	StartTime time.Time `json:"startTime"`
 	LeaderInfo struct {
 		Name      string `json:"leader"`
 		Uptime    string `json:"uptime"`
 		startTime time.Time
 	} `json:"leaderInfo"`
 	RecvAppendRequestCnt uint64  `json:"recvAppendRequestCnt,"`
 	RecvingPkgRate       float64 `json:"recvPkgRate,omitempty"`
 	RecvingBandwidthRate float64 `json:"recvBandwidthRate,omitempty"`
 	SendAppendRequestCnt uint64  `json:"sendAppendRequestCnt"`
 	SendingPkgRate       float64 `json:"sendPkgRate,omitempty"`
 	SendingBandwidthRate float64 `json:"sendBandwidthRate,omitempty"`
 	sendRateQueue *statsQueue
 	recvRateQueue *statsQueue
 }
 func (ss *raftServerStats) RecvAppendReq(leaderName string, pkgSize int) {
 	ss.State = raft.Follower
 	if leaderName != ss.LeaderInfo.Name {
 		ss.LeaderInfo.Name = leaderName
 		ss.LeaderInfo.startTime = time.Now()
 	}
 	ss.recvRateQueue.Insert(NewPackageStats(time.Now(), pkgSize))
 	ss.RecvAppendRequestCnt++
 }
 func (ss *raftServerStats) SendAppendReq(pkgSize int) {
 	now := time.Now()
 	if ss.State != raft.Leader {
 		ss.State = raft.Leader
 		ss.LeaderInfo.Name = ss.Name
 		ss.LeaderInfo.startTime = now
 	}
 	ss.sendRateQueue.Insert(NewPackageStats(now, pkgSize))
 	ss.SendAppendRequestCnt++
 }
--- a/server/registry.go
+++ b/server/registry.go
@ -0,0 +1,180 @@
 package server
 import (
 	"fmt"
 	"net/url"
 	"path"
 	"path/filepath"
 	"strings"
 	"sync"
 	"github.com/coreos/etcd/log"
 	"github.com/coreos/etcd/store"
 )
 // The location of the machine URL data.
 const RegistryKey = "/_etcd/machines"
 // The Registry stores URL information for nodes.
 type Registry struct {
 	sync.Mutex
 	store store.Store
 	nodes map[string]*node
 }
 // The internal storage format of the registry.
 type node struct {
 	peerVersion string
 	peerURL     string
 	url         string
 }
 // Creates a new Registry.
 func NewRegistry(s store.Store) *Registry {
 	return &Registry{
 		store: s,
 		nodes: make(map[string]*node),
 	}
 }
 // Adds a node to the registry.
 func (r *Registry) Register(name string, peerVersion string, peerURL string, url string, commitIndex uint64, term uint64) error {
 	r.Lock()
 	defer r.Unlock()
 	// Write data to store.
 	key := path.Join(RegistryKey, name)
 	value := fmt.Sprintf("raft=%s&etcd=%s&raftVersion=%s", peerURL, url, peerVersion)
 	_, err := r.store.Create(key, value, false, store.Permanent, commitIndex, term)
 	log.Debugf("Register: %s (%v)", name, err)
 	return err
 }
 // Removes a node from the registry.
 func (r *Registry) Unregister(name string, commitIndex uint64, term uint64) error {
 	r.Lock()
 	defer r.Unlock()
 	// Remove from cache.
 	// delete(r.nodes, name)
 	// Remove the key from the store.
 	_, err := r.store.Delete(path.Join(RegistryKey, name), false, commitIndex, term)
 	log.Debugf("Unregister: %s (%v)", name, err)
 	return err
 }
 // Returns the number of nodes in the cluster.
 func (r *Registry) Count() int {
 	e, err := r.store.Get(RegistryKey, false, false, 0, 0)
 	if err != nil {
 		return 0
 	}
 	return len(e.KVPairs)
 }
 // Retrieves the client URL for a given node by name.
 func (r *Registry) ClientURL(name string) (string, bool) {
 	r.Lock()
 	defer r.Unlock()
 	return r.clientURL(name)
 }
 func (r *Registry) clientURL(name string) (string, bool) {
 	if r.nodes[name] == nil {
 		r.load(name)
 	}
 	if node := r.nodes[name]; node != nil {
 		return node.url, true
 	}
 	return "", false
 }
 // Retrieves the peer URL for a given node by name.
 func (r *Registry) PeerURL(name string) (string, bool) {
 	r.Lock()
 	defer r.Unlock()
 	return r.peerURL(name)
 }
 func (r *Registry) peerURL(name string) (string, bool) {
 	if r.nodes[name] == nil {
 		r.load(name)
 	}
 	if node := r.nodes[name]; node != nil {
 		return node.peerURL, true
 	}
 	return "", false
 }
 // Retrieves the Client URLs for all nodes.
 func (r *Registry) ClientURLs(leaderName, selfName string) []string {
 	return r.urls(leaderName, selfName, r.clientURL)
 }
 // Retrieves the Peer URLs for all nodes.
 func (r *Registry) PeerURLs(leaderName, selfName string) []string {
 	return r.urls(leaderName, selfName, r.peerURL)
 }
 // Retrieves the URLs for all nodes using url function.
 func (r *Registry) urls(leaderName, selfName string, url func(name string) (string, bool)) []string {
 	r.Lock()
 	defer r.Unlock()
 	// Build list including the leader and self.
 	urls := make([]string, 0)
 	if url, _ := url(leaderName); len(url) > 0 {
 		urls = append(urls, url)
 	}
 	// Retrieve a list of all nodes.
 	if e, _ := r.store.Get(RegistryKey, false, false, 0, 0); e != nil {
 		// Lookup the URL for each one.
 		for _, pair := range e.KVPairs {
 			_, name := filepath.Split(pair.Key)
 			if url, _ := url(name); len(url) > 0 && name != leaderName {
 				urls = append(urls, url)
 			}
 		}
 	}
 	log.Infof("URLs: %s / %s (%s)", leaderName, selfName, strings.Join(urls, ","))
 	return urls
 }
 // Removes a node from the cache.
 func (r *Registry) Invalidate(name string) {
 	delete(r.nodes, name)
 }
 // Loads the given node by name from the store into the cache.
 func (r *Registry) load(name string) {
 	if name == "" {
 		return
 	}
 	// Retrieve from store.
 	e, err := r.store.Get(path.Join(RegistryKey, name), false, false, 0, 0)
 	if err != nil {
 		return
 	}
 	// Parse as a query string.
 	m, err := url.ParseQuery(e.Value)
 	if err != nil {
 		panic(fmt.Sprintf("Failed to parse machines entry: %s", name))
 	}
 	// Create node.
 	r.nodes[name] = &node{
 		url:         m["etcd"][0],
 		peerURL:     m["raft"][0],
 		peerVersion: m["raftVersion"][0],
 	}
 }
--- a/server/remove_command.go
+++ b/server/remove_command.go
@ -0,0 +1,67 @@
 package server
 import (
 	"encoding/binary"
 	"os"
 	"github.com/coreos/etcd/log"
 	"github.com/coreos/go-raft"
 )
 func init() {
 	raft.RegisterCommand(&RemoveCommand{})
 }
 // The RemoveCommand removes a server from the cluster.
 type RemoveCommand struct {
 	Name string `json:"name"`
 }
 // The name of the remove command in the log
 func (c *RemoveCommand) CommandName() string {
 	return "etcd:remove"
 }
 // Remove a server from the cluster
 func (c *RemoveCommand) Apply(server raft.Server) (interface{}, error) {
 	ps, _ := server.Context().(*PeerServer)
 	// Remove node from the shared registry.
 	err := ps.registry.Unregister(c.Name, server.CommitIndex(), server.Term())
 	// Delete from stats
 	delete(ps.followersStats.Followers, c.Name)
 	if err != nil {
 		log.Debugf("Error while unregistering: %s (%v)", c.Name, err)
 		return []byte{0}, err
 	}
 	// Remove peer in raft
 	err = server.RemovePeer(c.Name)
 	if err != nil {
 		log.Debugf("Unable to remove peer: %s (%v)", c.Name, err)
 		return []byte{0}, err
 	}
 	if c.Name == server.Name() {
 		// the removed node is this node
 		// if the node is not replaying the previous logs
 		// and the node has sent out a join request in this
 		// start. It is sure that this node received a new remove
 		// command and need to be removed
 		if server.CommitIndex() > ps.joinIndex && ps.joinIndex != 0 {
 			log.Debugf("server [%s] is removed", server.Name())
 			os.Exit(0)
 		} else {
 			// else ignore remove
 			log.Debugf("ignore previous remove command.")
 		}
 	}
 	b := make([]byte, 8)
 	binary.PutUvarint(b, server.CommitIndex())
 	return b, err
 }
--- a/server/server.go
+++ b/server/server.go
@ -0,0 +1,275 @@
 package server
 import (
 	"fmt"
 	"net/http"
 	"net/url"
 	"strings"
 	"time"
 	etcdErr "github.com/coreos/etcd/error"
 	"github.com/coreos/etcd/log"
 	"github.com/coreos/etcd/server/v1"
 	"github.com/coreos/etcd/server/v2"
 	"github.com/coreos/etcd/store"
 	"github.com/coreos/go-raft"
 	"github.com/gorilla/mux"
 )
 // This is the default implementation of the Server interface.
 type Server struct {
 	http.Server
 	peerServer  *PeerServer
 	registry    *Registry
 	store       store.Store
 	name        string
 	url         string
 	tlsConf     *TLSConfig
 	tlsInfo     *TLSInfo
 	corsOrigins map[string]bool
 }
 // Creates a new Server.
 func New(name string, urlStr string, listenHost string, tlsConf *TLSConfig, tlsInfo *TLSInfo, peerServer *PeerServer, registry *Registry, store store.Store) *Server {
 	s := &Server{
 		Server: http.Server{
 			Handler:   mux.NewRouter(),
 			TLSConfig: &tlsConf.Server,
 			Addr:      listenHost,
 		},
 		name:       name,
 		store:      store,
 		registry:   registry,
 		url:        urlStr,
 		tlsConf:    tlsConf,
 		tlsInfo:    tlsInfo,
 		peerServer: peerServer,
 	}
 	// Install the routes.
 	s.handleFunc("/version", s.GetVersionHandler).Methods("GET")
 	s.installV1()
 	s.installV2()
 	return s
 }
 // The current state of the server in the cluster.
 func (s *Server) State() string {
 	return s.peerServer.RaftServer().State()
 }
 // The node name of the leader in the cluster.
 func (s *Server) Leader() string {
 	return s.peerServer.RaftServer().Leader()
 }
 // The current Raft committed index.
 func (s *Server) CommitIndex() uint64 {
 	return s.peerServer.RaftServer().CommitIndex()
 }
 // The current Raft term.
 func (s *Server) Term() uint64 {
 	return s.peerServer.RaftServer().Term()
 }
 // The server URL.
 func (s *Server) URL() string {
 	return s.url
 }
 // Retrives the Peer URL for a given node name.
 func (s *Server) PeerURL(name string) (string, bool) {
 	return s.registry.PeerURL(name)
 }
 // Returns a reference to the Store.
 func (s *Server) Store() store.Store {
 	return s.store
 }
 func (s *Server) installV1() {
 	s.handleFuncV1("/v1/keys/{key:.*}", v1.GetKeyHandler).Methods("GET")
 	s.handleFuncV1("/v1/keys/{key:.*}", v1.SetKeyHandler).Methods("POST", "PUT")
 	s.handleFuncV1("/v1/keys/{key:.*}", v1.DeleteKeyHandler).Methods("DELETE")
 	s.handleFuncV1("/v1/watch/{key:.*}", v1.WatchKeyHandler).Methods("GET", "POST")
 	s.handleFunc("/v1/leader", s.GetLeaderHandler).Methods("GET")
 	s.handleFunc("/v1/machines", s.GetMachinesHandler).Methods("GET")
 	s.handleFunc("/v1/stats/self", s.GetStatsHandler).Methods("GET")
 	s.handleFunc("/v1/stats/leader", s.GetLeaderStatsHandler).Methods("GET")
 	s.handleFunc("/v1/stats/store", s.GetStoreStatsHandler).Methods("GET")
 }
 func (s *Server) installV2() {
 	s.handleFuncV2("/v2/keys/{key:.*}", v2.GetHandler).Methods("GET")
 	s.handleFuncV2("/v2/keys/{key:.*}", v2.PostHandler).Methods("POST")
 	s.handleFuncV2("/v2/keys/{key:.*}", v2.PutHandler).Methods("PUT")
 	s.handleFuncV2("/v2/keys/{key:.*}", v2.DeleteHandler).Methods("DELETE")
 	s.handleFunc("/v2/leader", s.GetLeaderHandler).Methods("GET")
 	s.handleFunc("/v2/machines", s.GetMachinesHandler).Methods("GET")
 	s.handleFunc("/v2/stats/self", s.GetStatsHandler).Methods("GET")
 	s.handleFunc("/v2/stats/leader", s.GetLeaderStatsHandler).Methods("GET")
 	s.handleFunc("/v2/stats/store", s.GetStoreStatsHandler).Methods("GET")
 }
 // Adds a v1 server handler to the router.
 func (s *Server) handleFuncV1(path string, f func(http.ResponseWriter, *http.Request, v1.Server) error) *mux.Route {
 	return s.handleFunc(path, func(w http.ResponseWriter, req *http.Request) error {
 		return f(w, req, s)
 	})
 }
 // Adds a v2 server handler to the router.
 func (s *Server) handleFuncV2(path string, f func(http.ResponseWriter, *http.Request, v2.Server) error) *mux.Route {
 	return s.handleFunc(path, func(w http.ResponseWriter, req *http.Request) error {
 		return f(w, req, s)
 	})
 }
 // Adds a server handler to the router.
 func (s *Server) handleFunc(path string, f func(http.ResponseWriter, *http.Request) error) *mux.Route {
 	r := s.Handler.(*mux.Router)
 	// Wrap the standard HandleFunc interface to pass in the server reference.
 	return r.HandleFunc(path, func(w http.ResponseWriter, req *http.Request) {
 		// Log request.
 		log.Debugf("[recv] %s %s %s [%s]", req.Method, s.url, req.URL.Path, req.RemoteAddr)
 		// Write CORS header.
 		if s.OriginAllowed("*") {
 			w.Header().Add("Access-Control-Allow-Origin", "*")
 		} else if origin := req.Header.Get("Origin"); s.OriginAllowed(origin) {
 			w.Header().Add("Access-Control-Allow-Origin", origin)
 		}
 		// Execute handler function and return error if necessary.
 		if err := f(w, req); err != nil {
 			if etcdErr, ok := err.(*etcdErr.Error); ok {
 				log.Debug("Return error: ", (*etcdErr).Error())
 				etcdErr.Write(w)
 			} else {
 				http.Error(w, err.Error(), http.StatusInternalServerError)
 			}
 		}
 	})
 }
 // Start to listen and response etcd client command
 func (s *Server) ListenAndServe() {
 	log.Infof("etcd server [name %s, listen on %s, advertised url %s]", s.name, s.Server.Addr, s.url)
 	if s.tlsConf.Scheme == "http" {
 		log.Fatal(s.Server.ListenAndServe())
 	} else {
 		log.Fatal(s.Server.ListenAndServeTLS(s.tlsInfo.CertFile, s.tlsInfo.KeyFile))
 	}
 }
 func (s *Server) Dispatch(c raft.Command, w http.ResponseWriter, req *http.Request) error {
 	return s.peerServer.dispatch(c, w, req)
 }
 // Sets a comma-delimited list of origins that are allowed.
 func (s *Server) AllowOrigins(origins string) error {
 	// Construct a lookup of all origins.
 	m := make(map[string]bool)
 	for _, v := range strings.Split(origins, ",") {
 		if v != "*" {
 			if _, err := url.Parse(v); err != nil {
 				return fmt.Errorf("Invalid CORS origin: %s", err)
 			}
 		}
 		m[v] = true
 	}
 	s.corsOrigins = m
 	return nil
 }
 // Determines whether the server will allow a given CORS origin.
 func (s *Server) OriginAllowed(origin string) bool {
 	return s.corsOrigins["*"] || s.corsOrigins[origin]
 }
 // Handler to return the current version of etcd.
 func (s *Server) GetVersionHandler(w http.ResponseWriter, req *http.Request) error {
 	w.WriteHeader(http.StatusOK)
 	fmt.Fprintf(w, "etcd %s", ReleaseVersion)
 	return nil
 }
 // Handler to return the current leader's raft address
 func (s *Server) GetLeaderHandler(w http.ResponseWriter, req *http.Request) error {
 	leader := s.peerServer.RaftServer().Leader()
 	if leader == "" {
 		return etcdErr.NewError(etcdErr.EcodeLeaderElect, "", store.UndefIndex, store.UndefTerm)
 	}
 	w.WriteHeader(http.StatusOK)
 	url, _ := s.registry.PeerURL(leader)
 	w.Write([]byte(url))
 	return nil
 }
 // Handler to return all the known machines in the current cluster.
 func (s *Server) GetMachinesHandler(w http.ResponseWriter, req *http.Request) error {
 	machines := s.registry.ClientURLs(s.peerServer.RaftServer().Leader(), s.name)
 	w.WriteHeader(http.StatusOK)
 	w.Write([]byte(strings.Join(machines, ", ")))
 	return nil
 }
 // Retrieves stats on the Raft server.
 func (s *Server) GetStatsHandler(w http.ResponseWriter, req *http.Request) error {
 	w.Write(s.peerServer.Stats())
 	return nil
 }
 // Retrieves stats on the leader.
 func (s *Server) GetLeaderStatsHandler(w http.ResponseWriter, req *http.Request) error {
 	if s.peerServer.RaftServer().State() == raft.Leader {
 		w.Write(s.peerServer.PeerStats())
 		return nil
 	}
 	leader := s.peerServer.RaftServer().Leader()
 	if leader == "" {
 		return etcdErr.NewError(300, "", store.UndefIndex, store.UndefTerm)
 	}
 	hostname, _ := s.registry.ClientURL(leader)
 	redirect(hostname, w, req)
 	return nil
 }
 // Retrieves stats on the leader.
 func (s *Server) GetStoreStatsHandler(w http.ResponseWriter, req *http.Request) error {
 	w.Write(s.store.JsonStats())
 	return nil
 }
 // Executes a speed test to evaluate the performance of update replication.
 func (s *Server) SpeedTestHandler(w http.ResponseWriter, req *http.Request) error {
 	count := 1000
 	c := make(chan bool, count)
 	for i := 0; i < count; i++ {
 		go func() {
 			for j := 0; j < 10; j++ {
 				c := &store.SetCommand{
 					Key:        "foo",
 					Value:      "bar",
 					ExpireTime: time.Unix(0, 0),
 				}
 				s.peerServer.RaftServer().Do(c)
 			}
 			c <- true
 		}()
 	}
 	for i := 0; i < count; i++ {
 		<-c
 	}
 	w.WriteHeader(http.StatusOK)
 	w.Write([]byte("speed test success"))
 	return nil
 }
--- a/server/stats_queue.go
+++ b/server/stats_queue.go
@ -0,0 +1,89 @@
 package server
 import (
 	"sync"
 	"time"
 )
 const (
 	queueCapacity = 200
 )
 type statsQueue struct {
 	items        [queueCapacity]*packageStats
 	size         int
 	front        int
 	back         int
 	totalPkgSize int
 	rwl          sync.RWMutex
 }
 func (q *statsQueue) Len() int {
 	return q.size
 }
 func (q *statsQueue) PkgSize() int {
 	return q.totalPkgSize
 }
 // FrontAndBack gets the front and back elements in the queue
 // We must grab front and back together with the protection of the lock
 func (q *statsQueue) frontAndBack() (*packageStats, *packageStats) {
 	q.rwl.RLock()
 	defer q.rwl.RUnlock()
 	if q.size != 0 {
 		return q.items[q.front], q.items[q.back]
 	}
 	return nil, nil
 }
 // Insert function insert a packageStats into the queue and update the records
 func (q *statsQueue) Insert(p *packageStats) {
 	q.rwl.Lock()
 	defer q.rwl.Unlock()
 	q.back = (q.back + 1) % queueCapacity
 	if q.size == queueCapacity { //dequeue
 		q.totalPkgSize -= q.items[q.front].size
 		q.front = (q.back + 1) % queueCapacity
 	} else {
 		q.size++
 	}
 	q.items[q.back] = p
 	q.totalPkgSize += q.items[q.back].size
 }
 // Rate function returns the package rate and byte rate
 func (q *statsQueue) Rate() (float64, float64) {
 	front, back := q.frontAndBack()
 	if front == nil || back == nil {
 		return 0, 0
 	}
 	if time.Now().Sub(back.Time()) > time.Second {
 		q.Clear()
 		return 0, 0
 	}
 	sampleDuration := back.Time().Sub(front.Time())
 	pr := float64(q.Len()) / float64(sampleDuration) * float64(time.Second)
 	br := float64(q.PkgSize()) / float64(sampleDuration) * float64(time.Second)
 	return pr, br
 }
 // Clear function clear up the statsQueue
 func (q *statsQueue) Clear() {
 	q.rwl.Lock()
 	defer q.rwl.Unlock()
 	q.back = -1
 	q.front = 0
 	q.size = 0
 	q.totalPkgSize = 0
 }
--- a/server/timeout.go
+++ b/server/timeout.go
@ -0,0 +1,15 @@
 package server
 import (
 	"time"
 )
 const (
 	// The amount of time to elapse without a heartbeat before becoming a candidate.
 	ElectionTimeout = 200 * time.Millisecond
 	// The frequency by which heartbeats are sent to followers.
 	HeartbeatTimeout = 50 * time.Millisecond
 	RetryInterval = 10
 )
--- a/server/tls_config.go
+++ b/server/tls_config.go
@ -0,0 +1,11 @@
 package server
 import (
 	"crypto/tls"
 )
 type TLSConfig struct {
 	Scheme string
 	Server tls.Config
 	Client tls.Config
 }
--- a/server/tls_info.go
+++ b/server/tls_info.go
@ -0,0 +1,7 @@
 package server
 type TLSInfo struct {
 	CertFile string `json:"CertFile"`
 	KeyFile  string `json:"KeyFile"`
 	CAFile   string `json:"CAFile"`
 }
--- a/server/transporter.go
+++ b/server/transporter.go
@ -0,0 +1,227 @@
 package server
 import (
 	"bytes"
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net"
 	"net/http"
 	"time"
 	"github.com/coreos/etcd/log"
 	"github.com/coreos/go-raft"
 )
 // Timeout for setup internal raft http connection
 // This should not exceed 3 * RTT
 var dailTimeout = 3 * HeartbeatTimeout
 // Timeout for setup internal raft http connection + receive response header
 // This should not exceed 3 * RTT + RTT
 var responseHeaderTimeout = 4 * HeartbeatTimeout
 // Timeout for receiving the response body from the server
 // This should not exceed election timeout
 var tranTimeout = ElectionTimeout
 // Transporter layer for communication between raft nodes
 type transporter struct {
 	client     *http.Client
 	transport  *http.Transport
 	peerServer *PeerServer
 }
 // Create transporter using by raft server
 // Create http or https transporter based on
 // whether the user give the server cert and key
 func newTransporter(scheme string, tlsConf tls.Config, peerServer *PeerServer) *transporter {
 	t := transporter{}
 	tr := &http.Transport{
 		Dial: dialWithTimeout,
 		ResponseHeaderTimeout: responseHeaderTimeout,
 	}
 	if scheme == "https" {
 		tr.TLSClientConfig = &tlsConf
 		tr.DisableCompression = true
 	}
 	t.client = &http.Client{Transport: tr}
 	t.transport = tr
 	t.peerServer = peerServer
 	return &t
 }
 // Dial with timeout
 func dialWithTimeout(network, addr string) (net.Conn, error) {
 	return net.DialTimeout(network, addr, dailTimeout)
 }
 // Sends AppendEntries RPCs to a peer when the server is the leader.
 func (t *transporter) SendAppendEntriesRequest(server raft.Server, peer *raft.Peer, req *raft.AppendEntriesRequest) *raft.AppendEntriesResponse {
 	var aersp *raft.AppendEntriesResponse
 	var b bytes.Buffer
 	json.NewEncoder(&b).Encode(req)
 	size := b.Len()
 	t.peerServer.serverStats.SendAppendReq(size)
 	u, _ := t.peerServer.registry.PeerURL(peer.Name)
 	log.Debugf("Send LogEntries to %s ", u)
 	thisFollowerStats, ok := t.peerServer.followersStats.Followers[peer.Name]
 	if !ok { //this is the first time this follower has been seen
 		thisFollowerStats = &raftFollowerStats{}
 		thisFollowerStats.Latency.Minimum = 1 << 63
 		t.peerServer.followersStats.Followers[peer.Name] = thisFollowerStats
 	}
 	start := time.Now()
 	resp, httpRequest, err := t.Post(fmt.Sprintf("%s/log/append", u), &b)
 	end := time.Now()
 	if err != nil {
 		log.Debugf("Cannot send AppendEntriesRequest to %s: %s", u, err)
 		if ok {
 			thisFollowerStats.Fail()
 		}
 	} else {
 		if ok {
 			thisFollowerStats.Succ(end.Sub(start))
 		}
 	}
 	if resp != nil {
 		defer resp.Body.Close()
 		t.CancelWhenTimeout(httpRequest)
 		aersp = &raft.AppendEntriesResponse{}
 		if err := json.NewDecoder(resp.Body).Decode(&aersp); err == nil || err == io.EOF {
 			return aersp
 		}
 	}
 	return aersp
 }
 // Sends RequestVote RPCs to a peer when the server is the candidate.
 func (t *transporter) SendVoteRequest(server raft.Server, peer *raft.Peer, req *raft.RequestVoteRequest) *raft.RequestVoteResponse {
 	var rvrsp *raft.RequestVoteResponse
 	var b bytes.Buffer
 	json.NewEncoder(&b).Encode(req)
 	u, _ := t.peerServer.registry.PeerURL(peer.Name)
 	log.Debugf("Send Vote from %s to %s", server.Name(), u)
 	resp, httpRequest, err := t.Post(fmt.Sprintf("%s/vote", u), &b)
 	if err != nil {
 		log.Debugf("Cannot send VoteRequest to %s : %s", u, err)
 	}
 	if resp != nil {
 		defer resp.Body.Close()
 		t.CancelWhenTimeout(httpRequest)
 		rvrsp := &raft.RequestVoteResponse{}
 		if err := json.NewDecoder(resp.Body).Decode(&rvrsp); err == nil || err == io.EOF {
 			return rvrsp
 		}
 	}
 	return rvrsp
 }
 // Sends SnapshotRequest RPCs to a peer when the server is the candidate.
 func (t *transporter) SendSnapshotRequest(server raft.Server, peer *raft.Peer, req *raft.SnapshotRequest) *raft.SnapshotResponse {
 	var aersp *raft.SnapshotResponse
 	var b bytes.Buffer
 	json.NewEncoder(&b).Encode(req)
 	u, _ := t.peerServer.registry.PeerURL(peer.Name)
 	log.Debugf("Send Snapshot to %s [Last Term: %d, LastIndex %d]", u,
 		req.LastTerm, req.LastIndex)
 	resp, httpRequest, err := t.Post(fmt.Sprintf("%s/snapshot", u), &b)
 	if err != nil {
 		log.Debugf("Cannot send SendSnapshotRequest to %s : %s", u, err)
 	}
 	if resp != nil {
 		defer resp.Body.Close()
 		t.CancelWhenTimeout(httpRequest)
 		aersp = &raft.SnapshotResponse{}
 		if err = json.NewDecoder(resp.Body).Decode(&aersp); err == nil || err == io.EOF {
 			return aersp
 		}
 	}
 	return aersp
 }
 // Sends SnapshotRecoveryRequest RPCs to a peer when the server is the candidate.
 func (t *transporter) SendSnapshotRecoveryRequest(server raft.Server, peer *raft.Peer, req *raft.SnapshotRecoveryRequest) *raft.SnapshotRecoveryResponse {
 	var aersp *raft.SnapshotRecoveryResponse
 	var b bytes.Buffer
 	json.NewEncoder(&b).Encode(req)
 	u, _ := t.peerServer.registry.PeerURL(peer.Name)
 	log.Debugf("Send SnapshotRecovery to %s [Last Term: %d, LastIndex %d]", u,
 		req.LastTerm, req.LastIndex)
 	resp, _, err := t.Post(fmt.Sprintf("%s/snapshotRecovery", u), &b)
 	if err != nil {
 		log.Debugf("Cannot send SendSnapshotRecoveryRequest to %s : %s", u, err)
 	}
 	if resp != nil {
 		defer resp.Body.Close()
 		aersp = &raft.SnapshotRecoveryResponse{}
 		if err = json.NewDecoder(resp.Body).Decode(&aersp); err == nil || err == io.EOF {
 			return aersp
 		}
 	}
 	return aersp
 }
 // Send server side POST request
 func (t *transporter) Post(urlStr string, body io.Reader) (*http.Response, *http.Request, error) {
 	req, _ := http.NewRequest("POST", urlStr, body)
 	resp, err := t.client.Do(req)
 	return resp, req, err
 }
 // Send server side GET request
 func (t *transporter) Get(urlStr string) (*http.Response, *http.Request, error) {
 	req, _ := http.NewRequest("GET", urlStr, nil)
 	resp, err := t.client.Do(req)
 	return resp, req, err
 }
 // Cancel the on fly HTTP transaction when timeout happens.
 func (t *transporter) CancelWhenTimeout(req *http.Request) {
 	go func() {
 		time.Sleep(ElectionTimeout)
 		t.transport.CancelRequest(req)
 	}()
 }
--- a/server/transporter_test.go
+++ b/server/transporter_test.go
@ -0,0 +1,61 @@
 package server
 import (
 	"crypto/tls"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"testing"
 	"time"
 )
 func TestTransporterTimeout(t *testing.T) {
 	http.HandleFunc("/timeout", func(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprintf(w, "timeout")
 		w.(http.Flusher).Flush() // send headers and some body
 		time.Sleep(time.Second * 100)
 	})
 	go http.ListenAndServe(":8080", nil)
 	conf := tls.Config{}
 	ts := newTransporter("http", conf, nil)
 	ts.Get("http://google.com")
 	_, _, err := ts.Get("http://google.com:9999")
 	if err == nil {
 		t.Fatal("timeout error")
 	}
 	res, req, err := ts.Get("http://localhost:8080/timeout")
 	if err != nil {
 		t.Fatal("should not timeout")
 	}
 	ts.CancelWhenTimeout(req)
 	body, err := ioutil.ReadAll(res.Body)
 	if err == nil {
 		fmt.Println(string(body))
 		t.Fatal("expected an error reading the body")
 	}
 	_, _, err = ts.Post("http://google.com:9999", nil)
 	if err == nil {
 		t.Fatal("timeout error")
 	}
 	_, _, err = ts.Get("http://www.google.com")
 	if err != nil {
 		t.Fatal("get error: ", err.Error())
 	}
 	_, _, err = ts.Post("http://www.google.com", nil)
 	if err != nil {
 		t.Fatal("post error")
 	}
 }
--- a/server/util.go
+++ b/server/util.go
@ -0,0 +1,26 @@
 package server
 import (
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"github.com/coreos/etcd/log"
 )
 func decodeJsonRequest(req *http.Request, data interface{}) error {
 	decoder := json.NewDecoder(req.Body)
 	if err := decoder.Decode(&data); err != nil && err != io.EOF {
 		log.Warnf("Malformed json request: %v", err)
 		return fmt.Errorf("Malformed json request: %v", err)
 	}
 	return nil
 }
 func redirect(hostname string, w http.ResponseWriter, req *http.Request) {
 	path := req.URL.Path
 	url := hostname + path
 	log.Debugf("Redirect to %s", url)
 	http.Redirect(w, req, url, http.StatusTemporaryRedirect)
 }
--- a/server/v1/delete_key_handler.go
+++ b/server/v1/delete_key_handler.go
@ -0,0 +1,15 @@
 package v1
 import (
 	"github.com/coreos/etcd/store"
 	"github.com/gorilla/mux"
 	"net/http"
 )
 // Removes a key from the store.
 func DeleteKeyHandler(w http.ResponseWriter, req *http.Request, s Server) error {
 	vars := mux.Vars(req)
 	key := "/" + vars["key"]
 	c := &store.DeleteCommand{Key: key}
 	return s.Dispatch(c, w, req)
 }
--- a/server/v1/get_key_handler.go
+++ b/server/v1/get_key_handler.go
@ -0,0 +1,27 @@
 package v1
 import (
 	"encoding/json"
 	"net/http"
 	"github.com/gorilla/mux"
 )
 // Retrieves the value for a given key.
 func GetKeyHandler(w http.ResponseWriter, req *http.Request, s Server) error {
 	vars := mux.Vars(req)
 	key := "/" + vars["key"]
 	// Retrieve the key from the store.
 	event, err := s.Store().Get(key, false, false, s.CommitIndex(), s.Term())
 	if err != nil {
 		return err
 	}
 	// Convert event to a response and write to client.
 	b, _ := json.Marshal(event.Response())
 	w.WriteHeader(http.StatusOK)
 	w.Write(b)
 	return nil
 }
--- a/server/v1/set_key_handler.go
+++ b/server/v1/set_key_handler.go
@ -0,0 +1,50 @@
 package v1
 import (
 	"net/http"
 	etcdErr "github.com/coreos/etcd/error"
 	"github.com/coreos/etcd/store"
 	"github.com/coreos/go-raft"
 	"github.com/gorilla/mux"
 )
 // Sets the value for a given key.
 func SetKeyHandler(w http.ResponseWriter, req *http.Request, s Server) error {
 	vars := mux.Vars(req)
 	key := "/" + vars["key"]
 	req.ParseForm()
 	// Parse non-blank value.
 	value := req.Form.Get("value")
 	if len(value) == 0 {
 		return etcdErr.NewError(200, "Set", store.UndefIndex, store.UndefTerm)
 	}
 	// Convert time-to-live to an expiration time.
 	expireTime, err := store.TTL(req.Form.Get("ttl"))
 	if err != nil {
 		return etcdErr.NewError(202, "Set", store.UndefIndex, store.UndefTerm)
 	}
 	// If the "prevValue" is specified then test-and-set. Otherwise create a new key.
 	var c raft.Command
 	if prevValueArr, ok := req.Form["prevValue"]; ok && len(prevValueArr) > 0 {
 		c = &store.CompareAndSwapCommand{
 			Key:        key,
 			Value:      value,
 			PrevValue:  prevValueArr[0],
 			ExpireTime: expireTime,
 		}
 	} else {
 		c = &store.SetCommand{
 			Key:        key,
 			Value:      value,
 			ExpireTime: expireTime,
 		}
 	}
 	return s.Dispatch(c, w, req)
 }
--- a/server/v1/v1.go
+++ b/server/v1/v1.go
@ -0,0 +1,15 @@
 package v1
 import (
 	"github.com/coreos/etcd/store"
 	"github.com/coreos/go-raft"
 	"net/http"
 )
 // The Server interface provides all the methods required for the v1 API.
 type Server interface {
 	CommitIndex() uint64
 	Term() uint64
 	Store() store.Store
 	Dispatch(raft.Command, http.ResponseWriter, *http.Request) error
 }
--- a/server/v1/watch_key_handler.go
+++ b/server/v1/watch_key_handler.go
@ -0,0 +1,40 @@
 package v1
 import (
 	"encoding/json"
 	"net/http"
 	"strconv"
 	etcdErr "github.com/coreos/etcd/error"
 	"github.com/coreos/etcd/store"
 	"github.com/gorilla/mux"
 )
 // Watches a given key prefix for changes.
 func WatchKeyHandler(w http.ResponseWriter, req *http.Request, s Server) error {
 	var err error
 	vars := mux.Vars(req)
 	key := "/" + vars["key"]
 	// Create a command to watch from a given index (default 0).
 	var sinceIndex uint64 = 0
 	if req.Method == "POST" {
 		sinceIndex, err = strconv.ParseUint(string(req.FormValue("index")), 10, 64)
 		if err != nil {
 			return etcdErr.NewError(203, "Watch From Index", store.UndefIndex, store.UndefTerm)
 		}
 	}
 	// Start the watcher on the store.
 	c, err := s.Store().Watch(key, false, sinceIndex, s.CommitIndex(), s.Term())
 	if err != nil {
 		return etcdErr.NewError(500, key, store.UndefIndex, store.UndefTerm)
 	}
 	event := <-c
 	b, _ := json.Marshal(event.Response())
 	w.WriteHeader(http.StatusOK)
 	w.Write(b)
 	return nil
 }
--- a/server/v2/delete_handler.go
+++ b/server/v2/delete_handler.go
@ -0,0 +1,20 @@
 package v2
 import (
 	"net/http"
 	"github.com/coreos/etcd/store"
 	"github.com/gorilla/mux"
 )
 func DeleteHandler(w http.ResponseWriter, req *http.Request, s Server) error {
 	vars := mux.Vars(req)
 	key := "/" + vars["key"]
 	c := &store.DeleteCommand{
 		Key:       key,
 		Recursive: (req.FormValue("recursive") == "true"),
 	}
 	return s.Dispatch(c, w, req)
 }
--- a/server/v2/get_handler.go
+++ b/server/v2/get_handler.go
@ -0,0 +1,71 @@
 package v2
 import (
 	"encoding/json"
 	"fmt"
 	"net/http"
 	"strconv"
 	etcdErr "github.com/coreos/etcd/error"
 	"github.com/coreos/etcd/log"
 	"github.com/coreos/etcd/store"
 	"github.com/coreos/go-raft"
 	"github.com/gorilla/mux"
 )
 func GetHandler(w http.ResponseWriter, req *http.Request, s Server) error {
 	var err error
 	var event *store.Event
 	vars := mux.Vars(req)
 	key := "/" + vars["key"]
 	// Help client to redirect the request to the current leader
 	if req.FormValue("consistent") == "true" && s.State() != raft.Leader {
 		leader := s.Leader()
 		hostname, _ := s.PeerURL(leader)
 		url := hostname + req.URL.Path
 		log.Debugf("Redirect consistent get to %s", url)
 		http.Redirect(w, req, url, http.StatusTemporaryRedirect)
 		return nil
 	}
 	recursive := (req.FormValue("recursive") == "true")
 	sorted := (req.FormValue("sorted") == "true")
 	if req.FormValue("wait") == "true" { // watch
 		// Create a command to watch from a given index (default 0).
 		var sinceIndex uint64 = 0
 		waitIndex := req.FormValue("waitIndex")
 		if waitIndex != "" {
 			sinceIndex, err = strconv.ParseUint(string(req.FormValue("waitIndex")), 10, 64)
 			if err != nil {
 				return etcdErr.NewError(etcdErr.EcodeIndexNaN, "Watch From Index", store.UndefIndex, store.UndefTerm)
 			}
 		}
 		// Start the watcher on the store.
 		c, err := s.Store().Watch(key, recursive, sinceIndex, s.CommitIndex(), s.Term())
 		if err != nil {
 			return etcdErr.NewError(500, key, store.UndefIndex, store.UndefTerm)
 		}
 		event = <-c
 	} else { //get
 		// Retrieve the key from the store.
 		event, err = s.Store().Get(key, recursive, sorted, s.CommitIndex(), s.Term())
 		if err != nil {
 			return err
 		}
 	}
 	w.Header().Add("X-Etcd-Index", fmt.Sprint(event.Index))
 	w.Header().Add("X-Etcd-Term", fmt.Sprint(event.Term))
 	w.WriteHeader(http.StatusOK)
 	b, _ := json.Marshal(event)
 	w.Write(b)
 	return nil
 }
--- a/server/v2/post_handler.go
+++ b/server/v2/post_handler.go
@ -0,0 +1,29 @@
 package v2
 import (
 	"net/http"
 	etcdErr "github.com/coreos/etcd/error"
 	"github.com/coreos/etcd/store"
 	"github.com/gorilla/mux"
 )
 func PostHandler(w http.ResponseWriter, req *http.Request, s Server) error {
 	vars := mux.Vars(req)
 	key := "/" + vars["key"]
 	value := req.FormValue("value")
 	expireTime, err := store.TTL(req.FormValue("ttl"))
 	if err != nil {
 		return etcdErr.NewError(etcdErr.EcodeTTLNaN, "Create", store.UndefIndex, store.UndefTerm)
 	}
 	c := &store.CreateCommand{
 		Key:        key,
 		Value:      value,
 		ExpireTime: expireTime,
 		Unique:     true,
 	}
 	return s.Dispatch(c, w, req)
 }
--- a/server/v2/put_handler.go
+++ b/server/v2/put_handler.go
@ -0,0 +1,109 @@
 package v2
 import (
 	"net/http"
 	"strconv"
 	"time"
 	etcdErr "github.com/coreos/etcd/error"
 	"github.com/coreos/etcd/store"
 	"github.com/coreos/go-raft"
 	"github.com/gorilla/mux"
 )
 func PutHandler(w http.ResponseWriter, req *http.Request, s Server) error {
 	vars := mux.Vars(req)
 	key := "/" + vars["key"]
 	req.ParseForm()
 	value := req.Form.Get("value")
 	expireTime, err := store.TTL(req.Form.Get("ttl"))
 	if err != nil {
 		return etcdErr.NewError(etcdErr.EcodeTTLNaN, "Update", store.UndefIndex, store.UndefTerm)
 	}
 	prevValue, valueOk := req.Form["prevValue"]
 	prevIndexStr, indexOk := req.Form["prevIndex"]
 	prevExist, existOk := req.Form["prevExist"]
 	var c raft.Command
 	// Set handler: create a new node or replace the old one.
 	if !valueOk && !indexOk && !existOk {
 		return SetHandler(w, req, s, key, value, expireTime)
 	}
 	// update with test
 	if existOk {
 		if prevExist[0] == "false" {
 			// Create command: create a new node. Fail, if a node already exists
 			// Ignore prevIndex and prevValue
 			return CreateHandler(w, req, s, key, value, expireTime)
 		}
 		if prevExist[0] == "true" && !indexOk && !valueOk {
 			return UpdateHandler(w, req, s, key, value, expireTime)
 		}
 	}
 	var prevIndex uint64
 	if indexOk {
 		prevIndex, err = strconv.ParseUint(prevIndexStr[0], 10, 64)
 		// bad previous index
 		if err != nil {
 			return etcdErr.NewError(etcdErr.EcodeIndexNaN, "CompareAndSwap", store.UndefIndex, store.UndefTerm)
 		}
 	} else {
 		prevIndex = 0
 	}
 	if valueOk {
 		if prevValue[0] == "" {
 			return etcdErr.NewError(etcdErr.EcodePrevValueRequired, "CompareAndSwap", store.UndefIndex, store.UndefTerm)
 		}
 	}
 	c = &store.CompareAndSwapCommand{
 		Key:       key,
 		Value:     value,
 		PrevValue: prevValue[0],
 		PrevIndex: prevIndex,
 	}
 	return s.Dispatch(c, w, req)
 }
 func SetHandler(w http.ResponseWriter, req *http.Request, s Server, key, value string, expireTime time.Time) error {
 	c := &store.SetCommand{
 		Key:        key,
 		Value:      value,
 		ExpireTime: expireTime,
 	}
 	return s.Dispatch(c, w, req)
 }
 func CreateHandler(w http.ResponseWriter, req *http.Request, s Server, key, value string, expireTime time.Time) error {
 	c := &store.CreateCommand{
 		Key:        key,
 		Value:      value,
 		ExpireTime: expireTime,
 	}
 	return s.Dispatch(c, w, req)
 }
 func UpdateHandler(w http.ResponseWriter, req *http.Request, s Server, key, value string, expireTime time.Time) error {
 	// Update should give at least one option
 	if value == "" && expireTime.Sub(store.Permanent) == 0 {
 		return etcdErr.NewError(etcdErr.EcodeValueOrTTLRequired, "Update", store.UndefIndex, store.UndefTerm)
 	}
 	c := &store.UpdateCommand{
 		Key:        key,
 		Value:      value,
 		ExpireTime: expireTime,
 	}
 	return s.Dispatch(c, w, req)
 }
--- a/server/v2/v2.go
+++ b/server/v2/v2.go
@ -0,0 +1,18 @@
 package v2
 import (
 	"github.com/coreos/etcd/store"
 	"github.com/coreos/go-raft"
 	"net/http"
 )
 // The Server interface provides all the methods required for the v2 API.
 type Server interface {
 	State() string
 	Leader() string
 	CommitIndex() uint64
 	Term() uint64
 	PeerURL(string) (string, bool)
 	Store() store.Store
 	Dispatch(raft.Command, http.ResponseWriter, *http.Request) error
 }
--- a/server/version.go
+++ b/server/version.go
@ -0,0 +1,8 @@
 package server
 const Version = "v2"
 // TODO: The release version (generated from the git tag) will be the raft
 // protocol version for now. When things settle down we will fix it like the
 // client API above.
 const PeerVersion = ReleaseVersion
--- a/store/compare_and_swap_command.go
+++ b/store/compare_and_swap_command.go
@ -0,0 +1,41 @@
 package store
 import (
 	"time"
 	"github.com/coreos/etcd/log"
 	"github.com/coreos/go-raft"
 )
 func init() {
 	raft.RegisterCommand(&CompareAndSwapCommand{})
 }
 // The CompareAndSwap performs a conditional update on a key in the store.
 type CompareAndSwapCommand struct {
 	Key        string    `json:"key"`
 	Value      string    `json:"value"`
 	ExpireTime time.Time `json:"expireTime"`
 	PrevValue  string    `json: prevValue`
 	PrevIndex  uint64    `json: prevIndex`
 }
 // The name of the testAndSet command in the log
 func (c *CompareAndSwapCommand) CommandName() string {
 	return "etcd:compareAndSwap"
 }
 // Set the key-value pair if the current value of the key equals to the given prevValue
 func (c *CompareAndSwapCommand) Apply(server raft.Server) (interface{}, error) {
 	s, _ := server.StateMachine().(Store)
 	e, err := s.CompareAndSwap(c.Key, c.PrevValue, c.PrevIndex,
 		c.Value, c.ExpireTime, server.CommitIndex(), server.Term())
 	if err != nil {
 		log.Debug(err)
 		return nil, err
 	}
 	return e, nil
 }
--- a/store/create_command.go
+++ b/store/create_command.go
@ -0,0 +1,38 @@
 package store
 import (
 	"github.com/coreos/etcd/log"
 	"github.com/coreos/go-raft"
 	"time"
 )
 func init() {
 	raft.RegisterCommand(&CreateCommand{})
 }
 // Create command
 type CreateCommand struct {
 	Key        string    `json:"key"`
 	Value      string    `json:"value"`
 	ExpireTime time.Time `json:"expireTime"`
 	Unique     bool      `json:"unique"`
 }
 // The name of the create command in the log
 func (c *CreateCommand) CommandName() string {
 	return "etcd:create"
 }
 // Create node
 func (c *CreateCommand) Apply(server raft.Server) (interface{}, error) {
 	s, _ := server.StateMachine().(Store)
 	e, err := s.Create(c.Key, c.Value, c.Unique, c.ExpireTime, server.CommitIndex(), server.Term())
 	if err != nil {
 		log.Debug(err)
 		return nil, err
 	}
 	return e, nil
 }
--- a/store/delete_command.go
+++ b/store/delete_command.go
@ -0,0 +1,35 @@
 package store
 import (
 	"github.com/coreos/etcd/log"
 	"github.com/coreos/go-raft"
 )
 func init() {
 	raft.RegisterCommand(&DeleteCommand{})
 }
 // The DeleteCommand removes a key from the Store.
 type DeleteCommand struct {
 	Key       string `json:"key"`
 	Recursive bool   `json:"recursive"`
 }
 // The name of the delete command in the log
 func (c *DeleteCommand) CommandName() string {
 	return "etcd:delete"
 }
 // Delete the key
 func (c *DeleteCommand) Apply(server raft.Server) (interface{}, error) {
 	s, _ := server.StateMachine().(Store)
 	e, err := s.Delete(c.Key, c.Recursive, server.CommitIndex(), server.Term())
 	if err != nil {
 		log.Debug(err)
 		return nil, err
 	}
 	return e, nil
 }
--- a/store/error.go
+++ b/store/error.go
@ -1,25 +0,0 @@
 package store
 type NotFoundError string
 func (e NotFoundError) Error() string {
 	return string(e)
 }
 type NotFile string
 func (e NotFile) Error() string {
 	return string(e)
 }
 type TestFail string
 func (e TestFail) Error() string {
 	return string(e)
 }
 type Keyword string
 func (e Keyword) Error() string {
 	return string(e)
 }
--- a/store/event.go
+++ b/store/event.go
@ -0,0 +1,80 @@
 package store
 import (
 	"time"
 )
 const (
 	Get            = "get"
 	Create         = "create"
 	Set            = "set"
 	Update         = "update"
 	Delete         = "delete"
 	CompareAndSwap = "compareAndSwap"
 	Expire         = "expire"
 )
 const (
 	UndefIndex = 0
 	UndefTerm  = 0
 )
 type Event struct {
 	Action     string     `json:"action"`
 	Key        string     `json:"key, omitempty"`
 	Dir        bool       `json:"dir,omitempty"`
 	PrevValue  string     `json:"prevValue,omitempty"`
 	Value      string     `json:"value,omitempty"`
 	KVPairs    kvPairs    `json:"kvs,omitempty"`
 	Expiration *time.Time `json:"expiration,omitempty"`
 	TTL        int64      `json:"ttl,omitempty"` // Time to live in second
 	// The command index of the raft machine when the command is executed
 	Index uint64 `json:"index"`
 	Term  uint64 `json:"term"`
 }
 func newEvent(action string, key string, index uint64, term uint64) *Event {
 	return &Event{
 		Action: action,
 		Key:    key,
 		Index:  index,
 		Term:   term,
 	}
 }
 // Converts an event object into a response object.
 func (event *Event) Response() interface{} {
 	if !event.Dir {
 		response := &Response{
 			Action:     event.Action,
 			Key:        event.Key,
 			Value:      event.Value,
 			PrevValue:  event.PrevValue,
 			Index:      event.Index,
 			TTL:        event.TTL,
 			Expiration: event.Expiration,
 		}
 		if response.Action == Create || response.Action == Set {
 			response.Action = "set"
 			if response.PrevValue == "" {
 				response.NewKey = true
 			}
 		}
 		return response
 	} else {
 		responses := make([]*Response, len(event.KVPairs))
 		for i, kv := range event.KVPairs {
 			responses[i] = &Response{
 				Action: event.Action,
 				Key:    kv.Key,
 				Value:  kv.Value,
 				Dir:    kv.Dir,
 				Index:  event.Index,
 			}
 		}
 		return responses
 	}
 }
--- a/store/event_history.go
+++ b/store/event_history.go
@ -0,0 +1,112 @@
 package store
 import (
 	"fmt"
 	"strings"
 	"sync"
 	etcdErr "github.com/coreos/etcd/error"
 )
 type EventHistory struct {
 	Queue      eventQueue
 	StartIndex uint64
 	LastIndex  uint64
 	LastTerm   uint64
 	DupCnt     uint64 // help to compute the watching point with duplicated indexes in the queue
 	rwl        sync.RWMutex
 }
 func newEventHistory(capacity int) *EventHistory {
 	return &EventHistory{
 		Queue: eventQueue{
 			Capacity: capacity,
 			Events:   make([]*Event, capacity),
 		},
 	}
 }
 // addEvent function adds event into the eventHistory
 func (eh *EventHistory) addEvent(e *Event) *Event {
 	eh.rwl.Lock()
 	defer eh.rwl.Unlock()
 	var duped uint64
 	if e.Index == UndefIndex {
 		e.Index = eh.LastIndex
 		e.Term = eh.LastTerm
 		duped = 1
 	}
 	eh.Queue.insert(e)
 	eh.LastIndex = e.Index
 	eh.LastTerm = e.Term
 	eh.DupCnt += duped
 	eh.StartIndex = eh.Queue.Events[eh.Queue.Front].Index
 	return e
 }
 // scan function is enumerating events from the index in history and
 // stops till the first point where the key has identified prefix
 func (eh *EventHistory) scan(prefix string, index uint64) (*Event, *etcdErr.Error) {
 	eh.rwl.RLock()
 	defer eh.rwl.RUnlock()
 	start := index - eh.StartIndex
 	// the index should locate after the event history's StartIndex
 	if start < 0 {
 		return nil,
 			etcdErr.NewError(etcdErr.EcodeEventIndexCleared,
 				fmt.Sprintf("the requested history has been cleared [%v/%v]",
 					eh.StartIndex, index), UndefIndex, UndefTerm)
 	}
 	// the index should locate before the size of the queue minus the duplicate count
 	if start >= (uint64(eh.Queue.Size) - eh.DupCnt) { // future index
 		return nil, nil
 	}
 	i := int((start + uint64(eh.Queue.Front)) % uint64(eh.Queue.Capacity))
 	for {
 		e := eh.Queue.Events[i]
 		if strings.HasPrefix(e.Key, prefix) && index <= e.Index { // make sure we bypass the smaller one
 			return e, nil
 		}
 		i = (i + 1) % eh.Queue.Capacity
 		if i == eh.Queue.back() { // find nothing, return and watch from current index
 			return nil, nil
 		}
 	}
 }
 // clone will be protected by a stop-world lock
 // do not need to obtain internal lock
 func (eh *EventHistory) clone() *EventHistory {
 	clonedQueue := eventQueue{
 		Capacity: eh.Queue.Capacity,
 		Events:   make([]*Event, eh.Queue.Capacity),
 		Size:     eh.Queue.Size,
 		Front:    eh.Queue.Front,
 	}
 	for i, e := range eh.Queue.Events {
 		clonedQueue.Events[i] = e
 	}
 	return &EventHistory{
 		StartIndex: eh.StartIndex,
 		Queue:      clonedQueue,
 		LastIndex:  eh.LastIndex,
 		LastTerm:   eh.LastTerm,
 		DupCnt:     eh.DupCnt,
 	}
 }
--- a/store/event_queue.go
+++ b/store/event_queue.go
@ -0,0 +1,25 @@
 package store
 type eventQueue struct {
 	Events   []*Event
 	Size     int
 	Front    int
 	Capacity int
 }
 func (eq *eventQueue) back() int {
 	return (eq.Front + eq.Size - 1 + eq.Capacity) % eq.Capacity
 }
 func (eq *eventQueue) insert(e *Event) {
 	index := (eq.back() + 1) % eq.Capacity
 	eq.Events[index] = e
 	if eq.Size == eq.Capacity { //dequeue
 		eq.Front = (index + 1) % eq.Capacity
 	} else {
 		eq.Size++
 	}
 }
--- a/store/event_test.go
+++ b/store/event_test.go
@ -0,0 +1,66 @@
 package store
 import (
 	"testing"
 )
 // TestEventQueue tests a queue with capacity = 100
 // Add 200 events into that queue, and test if the
 // previous 100 events have been swapped out.
 func TestEventQueue(t *testing.T) {
 	eh := newEventHistory(100)
 	// Add
 	for i := 0; i < 200; i++ {
 		e := newEvent(Create, "/foo", uint64(i), 1)
 		eh.addEvent(e)
 	}
 	// Test
 	j := 100
 	i := eh.Queue.Front
 	n := eh.Queue.Size
 	for ; n > 0; n-- {
 		e := eh.Queue.Events[i]
 		if e.Index != uint64(j) {
 			t.Fatalf("queue error!")
 		}
 		j++
 		i = (i + 1) % eh.Queue.Capacity
 	}
 }
 func TestScanHistory(t *testing.T) {
 	eh := newEventHistory(100)
 	// Add
 	eh.addEvent(newEvent(Create, "/foo", 1, 1))
 	eh.addEvent(newEvent(Create, "/foo/bar", 2, 1))
 	eh.addEvent(newEvent(Create, "/foo/foo", 3, 1))
 	eh.addEvent(newEvent(Create, "/foo/bar/bar", 4, 1))
 	eh.addEvent(newEvent(Create, "/foo/foo/foo", 5, 1))
 	e, err := eh.scan("/foo", 1)
 	if err != nil || e.Index != 1 {
 		t.Fatalf("scan error [/foo] [1] %v", e.Index)
 	}
 	e, err = eh.scan("/foo/bar", 1)
 	if err != nil || e.Index != 2 {
 		t.Fatalf("scan error [/foo/bar] [2] %v", e.Index)
 	}
 	e, err = eh.scan("/foo/bar", 3)
 	if err != nil || e.Index != 4 {
 		t.Fatalf("scan error [/foo/bar/bar] [4] %v", e.Index)
 	}
 	e, err = eh.scan("/foo/bar", 6)
 	if e != nil {
 		t.Fatalf("bad index shoud reuturn nil")
 	}
 }
--- a/store/keyword_test.go
+++ b/store/keyword_test.go
@ -1,37 +0,0 @@
 package store
 import (
 	"testing"
 )
 func TestKeywords(t *testing.T) {
 	keyword := CheckKeyword("_etcd")
 	if !keyword {
 		t.Fatal("_etcd should be keyword")
 	}
 	keyword = CheckKeyword("/_etcd")
 	if !keyword {
 		t.Fatal("/_etcd should be keyword")
 	}
 	keyword = CheckKeyword("/_etcd/")
 	if !keyword {
 		t.Fatal("/_etcd/ contains keyword prefix")
 	}
 	keyword = CheckKeyword("/_etcd/node1")
 	if !keyword {
 		t.Fatal("/_etcd/* contains keyword prefix")
 	}
 	keyword = CheckKeyword("/nokeyword/_etcd/node1")
 	if keyword {
 		t.Fatal("this does not contain keyword prefix")
 	}
 }
--- a/store/keywords.go
+++ b/store/keywords.go
@ -1,33 +0,0 @@
 package store
 import (
 	"path"
 	"strings"
 )
 // keywords for internal useage
 // Key for string keyword; Value for only checking prefix
 var keywords = map[string]bool{
 	"/_etcd":          true,
 	"/ephemeralNodes": true,
 }
 // CheckKeyword will check if the key contains the keyword.
 // For now, we only check for prefix.
 func CheckKeyword(key string) bool {
 	key = path.Clean("/" + key)
 	// find the second "/"
 	i := strings.Index(key[1:], "/")
 	var prefix string
 	if i == -1 {
 		prefix = key
 	} else {
 		prefix = key[:i+1]
 	}
 	_, ok := keywords[prefix]
 	return ok
 }
--- a/store/kv_pairs.go
+++ b/store/kv_pairs.go
@ -0,0 +1,24 @@
 package store
 // When user list a directory, we add all the node into key-value pair slice
 type KeyValuePair struct {
 	Key     string  `json:"key, omitempty"`
 	Value   string  `json:"value,omitempty"`
 	Dir     bool    `json:"dir,omitempty"`
 	KVPairs kvPairs `json:"kvs,omitempty"`
 }
 type kvPairs []KeyValuePair
 // interfaces for sorting
 func (kvs kvPairs) Len() int {
 	return len(kvs)
 }
 func (kvs kvPairs) Less(i, j int) bool {
 	return kvs[i].Key < kvs[j].Key
 }
 func (kvs kvPairs) Swap(i, j int) {
 	kvs[i], kvs[j] = kvs[j], kvs[i]
 }
--- a/store/node.go
+++ b/store/node.go
@ -0,0 +1,417 @@
 package store
 import (
 	"path"
 	"sort"
 	"sync"
 	"time"
 	etcdErr "github.com/coreos/etcd/error"
 )
 var (
 	Permanent time.Time
 )
 const (
 	normal = iota
 	removed
 )
 // Node is the basic element in the store system.
 // A key-value pair will have a string value
 // A directory will have a children map
 type Node struct {
 	Path string
 	CreateIndex   uint64
 	CreateTerm    uint64
 	ModifiedIndex uint64
 	ModifiedTerm  uint64
 	Parent *Node `json:"-"` // should not encode this field! avoid cyclical dependency.
 	ExpireTime time.Time
 	ACL        string
 	Value      string           // for key-value pair
 	Children   map[string]*Node // for directory
 	// A reference to the store this node is attached to.
 	store *store
 	// a ttl node will have an expire routine associated with it.
 	// we need a channel to stop that routine when the expiration changes.
 	stopExpire chan bool
 	// ensure we only delete the node once
 	// expire and remove may try to delete a node twice
 	once sync.Once
 }
 // newKV creates a Key-Value pair
 func newKV(store *store, nodePath string, value string, createIndex uint64,
 	createTerm uint64, parent *Node, ACL string, expireTime time.Time) *Node {
 	return &Node{
 		Path:          nodePath,
 		CreateIndex:   createIndex,
 		CreateTerm:    createTerm,
 		ModifiedIndex: createIndex,
 		ModifiedTerm:  createTerm,
 		Parent:        parent,
 		ACL:           ACL,
 		store:         store,
 		stopExpire:    make(chan bool, 1),
 		ExpireTime:    expireTime,
 		Value:         value,
 	}
 }
 // newDir creates a directory
 func newDir(store *store, nodePath string, createIndex uint64, createTerm uint64,
 	parent *Node, ACL string, expireTime time.Time) *Node {
 	return &Node{
 		Path:        nodePath,
 		CreateIndex: createIndex,
 		CreateTerm:  createTerm,
 		Parent:      parent,
 		ACL:         ACL,
 		stopExpire:  make(chan bool, 1),
 		ExpireTime:  expireTime,
 		Children:    make(map[string]*Node),
 		store:       store,
 	}
 }
 // IsHidden function checks if the node is a hidden node. A hidden node
 // will begin with '_'
 // A hidden node will not be shown via get command under a directory
 // For example if we have /foo/_hidden and /foo/notHidden, get "/foo"
 // will only return /foo/notHidden
 func (n *Node) IsHidden() bool {
 	_, name := path.Split(n.Path)
 	return name[0] == '_'
 }
 // IsPermanent function checks if the node is a permanent one.
 func (n *Node) IsPermanent() bool {
 	return n.ExpireTime.Sub(Permanent) == 0
 }
 // IsExpired function checks if the node has been expired.
 func (n *Node) IsExpired() (bool, time.Duration) {
 	if n.IsPermanent() {
 		return false, 0
 	}
 	duration := n.ExpireTime.Sub(time.Now())
 	if duration <= 0 {
 		return true, 0
 	}
 	return false, duration
 }
 // IsDir function checks whether the node is a directory.
 // If the node is a directory, the function will return true.
 // Otherwise the function will return false.
 func (n *Node) IsDir() bool {
 	return !(n.Children == nil)
 }
 // Read function gets the value of the node.
 // If the receiver node is not a key-value pair, a "Not A File" error will be returned.
 func (n *Node) Read() (string, *etcdErr.Error) {
 	if n.IsDir() {
 		return "", etcdErr.NewError(etcdErr.EcodeNotFile, "", UndefIndex, UndefTerm)
 	}
 	return n.Value, nil
 }
 // Write function set the value of the node to the given value.
 // If the receiver node is a directory, a "Not A File" error will be returned.
 func (n *Node) Write(value string, index uint64, term uint64) *etcdErr.Error {
 	if n.IsDir() {
 		return etcdErr.NewError(etcdErr.EcodeNotFile, "", UndefIndex, UndefTerm)
 	}
 	n.Value = value
 	n.ModifiedIndex = index
 	n.ModifiedTerm = term
 	return nil
 }
 func (n *Node) ExpirationAndTTL() (*time.Time, int64) {
 	if n.ExpireTime.Sub(Permanent) != 0 {
 		return &n.ExpireTime, int64(n.ExpireTime.Sub(time.Now())/time.Second) + 1
 	}
 	return nil, 0
 }
 // List function return a slice of nodes under the receiver node.
 // If the receiver node is not a directory, a "Not A Directory" error will be returned.
 func (n *Node) List() ([]*Node, *etcdErr.Error) {
 	if !n.IsDir() {
 		return nil, etcdErr.NewError(etcdErr.EcodeNotDir, "", UndefIndex, UndefTerm)
 	}
 	nodes := make([]*Node, len(n.Children))
 	i := 0
 	for _, node := range n.Children {
 		nodes[i] = node
 		i++
 	}
 	return nodes, nil
 }
 // GetChild function returns the child node under the directory node.
 // On success, it returns the file node
 func (n *Node) GetChild(name string) (*Node, *etcdErr.Error) {
 	if !n.IsDir() {
 		return nil, etcdErr.NewError(etcdErr.EcodeNotDir, n.Path, UndefIndex, UndefTerm)
 	}
 	child, ok := n.Children[name]
 	if ok {
 		return child, nil
 	}
 	return nil, nil
 }
 // Add function adds a node to the receiver node.
 // If the receiver is not a directory, a "Not A Directory" error will be returned.
 // If there is a existing node with the same name under the directory, a "Already Exist"
 // error will be returned
 func (n *Node) Add(child *Node) *etcdErr.Error {
 	if !n.IsDir() {
 		return etcdErr.NewError(etcdErr.EcodeNotDir, "", UndefIndex, UndefTerm)
 	}
 	_, name := path.Split(child.Path)
 	_, ok := n.Children[name]
 	if ok {
 		return etcdErr.NewError(etcdErr.EcodeNodeExist, "", UndefIndex, UndefTerm)
 	}
 	n.Children[name] = child
 	return nil
 }
 // Remove function remove the node.
 func (n *Node) Remove(recursive bool, callback func(path string)) *etcdErr.Error {
 	if n.IsDir() && !recursive {
 		// cannot delete a directory without set recursive to true
 		return etcdErr.NewError(etcdErr.EcodeNotFile, "", UndefIndex, UndefTerm)
 	}
 	onceBody := func() {
 		n.internalRemove(recursive, callback)
 	}
 	// this function might be entered multiple times by expire and delete
 	// every node will only be deleted once.
 	n.once.Do(onceBody)
 	return nil
 }
 // internalRemove function will be called by remove()
 func (n *Node) internalRemove(recursive bool, callback func(path string)) {
 	if !n.IsDir() { // key-value pair
 		_, name := path.Split(n.Path)
 		// find its parent and remove the node from the map
 		if n.Parent != nil && n.Parent.Children[name] == n {
 			delete(n.Parent.Children, name)
 		}
 		if callback != nil {
 			callback(n.Path)
 		}
 		// the stop channel has a buffer. just send to it!
 		n.stopExpire <- true
 		return
 	}
 	for _, child := range n.Children { // delete all children
 		child.Remove(true, callback)
 	}
 	// delete self
 	_, name := path.Split(n.Path)
 	if n.Parent != nil && n.Parent.Children[name] == n {
 		delete(n.Parent.Children, name)
 		if callback != nil {
 			callback(n.Path)
 		}
 		n.stopExpire <- true
 	}
 }
 // Expire function will test if the node is expired.
 // if the node is already expired, delete the node and return.
 // if the node is permanent (this shouldn't happen), return at once.
 // else wait for a period time, then remove the node. and notify the watchhub.
 func (n *Node) Expire() {
 	expired, duration := n.IsExpired()
 	if expired { // has been expired
 		// since the parent function of Expire() runs serially,
 		// there is no need for lock here
 		e := newEvent(Expire, n.Path, UndefIndex, UndefTerm)
 		n.store.WatcherHub.notify(e)
 		n.Remove(true, nil)
 		n.store.Stats.Inc(ExpireCount)
 		return
 	}
 	if duration == 0 { // Permanent Node
 		return
 	}
 	go func() { // do monitoring
 		select {
 		// if timeout, delete the node
 		case <-time.After(duration):
 			// before expire get the lock, the expiration time
 			// of the node may be updated.
 			// we have to check again when get the lock
 			n.store.worldLock.Lock()
 			defer n.store.worldLock.Unlock()
 			expired, _ := n.IsExpired()
 			if expired {
 				e := newEvent(Expire, n.Path, UndefIndex, UndefTerm)
 				n.store.WatcherHub.notify(e)
 				n.Remove(true, nil)
 				n.store.Stats.Inc(ExpireCount)
 			}
 			return
 		// if stopped, return
 		case <-n.stopExpire:
 			return
 		}
 	}()
 }
 func (n *Node) Pair(recurisive, sorted bool) KeyValuePair {
 	if n.IsDir() {
 		pair := KeyValuePair{
 			Key: n.Path,
 			Dir: true,
 		}
 		if !recurisive {
 			return pair
 		}
 		children, _ := n.List()
 		pair.KVPairs = make([]KeyValuePair, len(children))
 		// we do not use the index in the children slice directly
 		// we need to skip the hidden one
 		i := 0
 		for _, child := range children {
 			if child.IsHidden() { // get will not list hidden node
 				continue
 			}
 			pair.KVPairs[i] = child.Pair(recurisive, sorted)
 			i++
 		}
 		// eliminate hidden nodes
 		pair.KVPairs = pair.KVPairs[:i]
 		if sorted {
 			sort.Sort(pair.KVPairs)
 		}
 		return pair
 	}
 	return KeyValuePair{
 		Key:   n.Path,
 		Value: n.Value,
 	}
 }
 func (n *Node) UpdateTTL(expireTime time.Time) {
 	if !n.IsPermanent() {
 		// check if the node has been expired
 		// if the node is not expired, we need to stop the go routine associated with
 		// that node.
 		expired, _ := n.IsExpired()
 		if !expired {
 			n.stopExpire <- true // suspend it to modify the expiration
 		}
 	}
 	if expireTime.Sub(Permanent) != 0 {
 		n.ExpireTime = expireTime
 		n.Expire()
 	}
 }
 // Clone function clone the node recursively and return the new node.
 // If the node is a directory, it will clone all the content under this directory.
 // If the node is a key-value pair, it will clone the pair.
 func (n *Node) Clone() *Node {
 	if !n.IsDir() {
 		return newKV(n.store, n.Path, n.Value, n.CreateIndex, n.CreateTerm, n.Parent, n.ACL, n.ExpireTime)
 	}
 	clone := newDir(n.store, n.Path, n.CreateIndex, n.CreateTerm, n.Parent, n.ACL, n.ExpireTime)
 	for key, child := range n.Children {
 		clone.Children[key] = child.Clone()
 	}
 	return clone
 }
 // recoverAndclean function help to do recovery.
 // Two things need to be done: 1. recovery structure; 2. delete expired nodes
 // If the node is a directory, it will help recover children's parent pointer and recursively
 // call this function on its children.
 // We check the expire last since we need to recover the whole structure first and add all the
 // notifications into the event history.
 func (n *Node) recoverAndclean() {
 	if n.IsDir() {
 		for _, child := range n.Children {
 			child.Parent = n
 			child.store = n.store
 			child.recoverAndclean()
 		}
 	}
 	n.stopExpire = make(chan bool, 1)
 	n.Expire()
 }
--- a/store/response_v1.go
+++ b/store/response_v1.go
@ -0,0 +1,26 @@
 package store
 import (
 	"time"
 )
 // The response from the store to the user who issue a command
 type Response struct {
 	Action    string `json:"action"`
 	Key       string `json:"key"`
 	Dir       bool   `json:"dir,omitempty"`
 	PrevValue string `json:"prevValue,omitempty"`
 	Value     string `json:"value,omitempty"`
 	// If the key did not exist before the action,
 	// this field should be set to true
 	NewKey bool `json:"newKey,omitempty"`
 	Expiration *time.Time `json:"expiration,omitempty"`
 	// Time to live in second
 	TTL int64 `json:"ttl,omitempty"`
 	// The command index of the raft machine when the command is executed
 	Index uint64 `json:"index"`
 }
--- a/store/set_command.go
+++ b/store/set_command.go
@ -0,0 +1,38 @@
 package store
 import (
 	"github.com/coreos/etcd/log"
 	"github.com/coreos/go-raft"
 	"time"
 )
 func init() {
 	raft.RegisterCommand(&SetCommand{})
 }
 // Create command
 type SetCommand struct {
 	Key        string    `json:"key"`
 	Value      string    `json:"value"`
 	ExpireTime time.Time `json:"expireTime"`
 }
 // The name of the create command in the log
 func (c *SetCommand) CommandName() string {
 	return "etcd:set"
 }
 // Create node
 func (c *SetCommand) Apply(server raft.Server) (interface{}, error) {
 	s, _ := server.StateMachine().(Store)
 	// create a new node or replace the old node.
 	e, err := s.Set(c.Key, c.Value, c.ExpireTime, server.CommitIndex(), server.Term())
 	if err != nil {
 		log.Debug(err)
 		return nil, err
 	}
 	return e, nil
 }
--- a/store/stats.go
+++ b/store/stats.go
@ -2,24 +2,111 @@ package store
 import (
 	"encoding/json"
 	"sync/atomic"
 )
-type EtcdStats struct {
+const (
 	SetSuccess = iota
 	SetFail
 	DeleteSuccess
 	DeleteFail
 	CreateSuccess
 	CreateFail
 	UpdateSuccess
 	UpdateFail
 	CompareAndSwapSuccess
 	CompareAndSwapFail
 	GetSuccess
 	GetFail
 	ExpireCount
 )
 type Stats struct {
 	// Number of get requests
-	Gets uint64 `json:"gets"`
+	GetSuccess uint64 `json:"getsSuccess"`
 	GetFail    uint64 `json:"getsFail"`
 	// Number of sets requests
-	Sets uint64 `json:"sets"`
+	SetSuccess uint64 `json:"setsSuccess"`
 	SetFail    uint64 `json:"setsFail"`
 	// Number of delete requests
-	Deletes uint64 `json:"deletes"`
+	DeleteSuccess uint64 `json:"deleteSuccess"`
 	DeleteFail    uint64 `json:"deleteFail"`
 	// Number of update requests
 	UpdateSuccess uint64 `json:"updateSuccess"`
 	UpdateFail    uint64 `json:"updateFail"`
 	// Number of create requests
 	CreateSuccess uint64 `json:"createSuccess"`
 	CreateFail    uint64 `json:createFail`
 	// Number of testAndSet requests
-	TestAndSets uint64 `json:"testAndSets"`
+	CompareAndSwapSuccess uint64 `json:"compareAndSwapSuccess"`
 	CompareAndSwapFail    uint64 `json:"compareAndSwapFail"`
 	ExpireCount uint64 `json:"expireCount"`
 	Watchers uint64 `json:"watchers"`
 }
-// Stats returns the basic statistics information of etcd storage
+func newStats() *Stats {
-func (s *Store) Stats() []byte {
+	s := new(Stats)
-	b, _ := json.Marshal(s.BasicStats)
+	return s
 }
 func (s *Stats) clone() *Stats {
 	return &Stats{s.GetSuccess, s.GetFail, s.SetSuccess, s.SetFail,
 		s.DeleteSuccess, s.DeleteFail, s.UpdateSuccess, s.UpdateFail, s.CreateSuccess,
 		s.CreateFail, s.CompareAndSwapSuccess, s.CompareAndSwapFail, s.Watchers, s.ExpireCount}
 }
 // Status() return the statistics info of etcd storage its recent start
 func (s *Stats) toJson() []byte {
 	b, _ := json.Marshal(s)
 	return b
 }
 func (s *Stats) TotalReads() uint64 {
 	return s.GetSuccess + s.GetFail
 }
 func (s *Stats) TotalWrites() uint64 {
 	return s.SetSuccess + s.SetFail +
 		s.DeleteSuccess + s.DeleteFail +
 		s.CompareAndSwapSuccess + s.CompareAndSwapFail +
 		s.UpdateSuccess + s.UpdateFail
 }
 func (s *Stats) Inc(field int) {
 	switch field {
 	case SetSuccess:
 		atomic.AddUint64(&s.SetSuccess, 1)
 	case SetFail:
 		atomic.AddUint64(&s.SetFail, 1)
 	case CreateSuccess:
 		atomic.AddUint64(&s.CreateSuccess, 1)
 	case CreateFail:
 		atomic.AddUint64(&s.CreateFail, 1)
 	case DeleteSuccess:
 		atomic.AddUint64(&s.DeleteSuccess, 1)
 	case DeleteFail:
 		atomic.AddUint64(&s.DeleteFail, 1)
 	case GetSuccess:
 		atomic.AddUint64(&s.GetSuccess, 1)
 	case GetFail:
 		atomic.AddUint64(&s.GetFail, 1)
 	case UpdateSuccess:
 		atomic.AddUint64(&s.UpdateSuccess, 1)
 	case UpdateFail:
 		atomic.AddUint64(&s.UpdateFail, 1)
 	case CompareAndSwapSuccess:
 		atomic.AddUint64(&s.CompareAndSwapSuccess, 1)
 	case CompareAndSwapFail:
 		atomic.AddUint64(&s.CompareAndSwapFail, 1)
 	case ExpireCount:
 		atomic.AddUint64(&s.ExpireCount, 1)
 	}
 }
--- a/store/stats_test.go
+++ b/store/stats_test.go
@ -0,0 +1,165 @@
 package store
 import (
 	"math/rand"
 	"testing"
 	"time"
 )
 func TestBasicStats(t *testing.T) {
 	s := newStore()
 	keys := GenKeys(rand.Intn(100), 5)
 	var i uint64
 	var GetSuccess, GetFail, CreateSuccess, CreateFail, DeleteSuccess, DeleteFail uint64
 	var UpdateSuccess, UpdateFail, CompareAndSwapSuccess, CompareAndSwapFail, watcher_number uint64
 	for _, k := range keys {
 		i++
 		_, err := s.Create(k, "bar", false, time.Now().Add(time.Second*time.Duration(rand.Intn(6))), i, 1)
 		if err != nil {
 			CreateFail++
 		} else {
 			CreateSuccess++
 		}
 	}
 	time.Sleep(time.Second * 3)
 	for _, k := range keys {
 		_, err := s.Get(k, false, false, i, 1)
 		if err != nil {
 			GetFail++
 		} else {
 			GetSuccess++
 		}
 	}
 	for _, k := range keys {
 		i++
 		_, err := s.Update(k, "foo", time.Now().Add(time.Second*time.Duration(rand.Intn(6))), i, 1)
 		if err != nil {
 			UpdateFail++
 		} else {
 			UpdateSuccess++
 		}
 	}
 	time.Sleep(time.Second * 3)
 	for _, k := range keys {
 		_, err := s.Get(k, false, false, i, 1)
 		if err != nil {
 			GetFail++
 		} else {
 			GetSuccess++
 		}
 	}
 	for _, k := range keys {
 		i++
 		_, err := s.CompareAndSwap(k, "foo", 0, "bar", Permanent, i, 1)
 		if err != nil {
 			CompareAndSwapFail++
 		} else {
 			CompareAndSwapSuccess++
 		}
 	}
 	for _, k := range keys {
 		s.Watch(k, false, 0, i, 1)
 		watcher_number++
 	}
 	for _, k := range keys {
 		_, err := s.Get(k, false, false, i, 1)
 		if err != nil {
 			GetFail++
 		} else {
 			GetSuccess++
 		}
 	}
 	for _, k := range keys {
 		i++
 		_, err := s.Delete(k, false, i, 1)
 		if err != nil {
 			DeleteFail++
 		} else {
 			watcher_number--
 			DeleteSuccess++
 		}
 	}
 	for _, k := range keys {
 		_, err := s.Get(k, false, false, i, 1)
 		if err != nil {
 			GetFail++
 		} else {
 			GetSuccess++
 		}
 	}
 	if GetSuccess != s.Stats.GetSuccess {
 		t.Fatalf("GetSuccess [%d] != Stats.GetSuccess [%d]", GetSuccess, s.Stats.GetSuccess)
 	}
 	if GetFail != s.Stats.GetFail {
 		t.Fatalf("GetFail [%d] != Stats.GetFail [%d]", GetFail, s.Stats.GetFail)
 	}
 	if CreateSuccess != s.Stats.CreateSuccess {
 		t.Fatalf("CreateSuccess [%d] != Stats.CreateSuccess [%d]", CreateSuccess, s.Stats.CreateSuccess)
 	}
 	if CreateFail != s.Stats.CreateFail {
 		t.Fatalf("CreateFail [%d] != Stats.CreateFail [%d]", CreateFail, s.Stats.CreateFail)
 	}
 	if DeleteSuccess != s.Stats.DeleteSuccess {
 		t.Fatalf("DeleteSuccess [%d] != Stats.DeleteSuccess [%d]", DeleteSuccess, s.Stats.DeleteSuccess)
 	}
 	if DeleteFail != s.Stats.DeleteFail {
 		t.Fatalf("DeleteFail [%d] != Stats.DeleteFail [%d]", DeleteFail, s.Stats.DeleteFail)
 	}
 	if UpdateSuccess != s.Stats.UpdateSuccess {
 		t.Fatalf("UpdateSuccess [%d] != Stats.UpdateSuccess [%d]", UpdateSuccess, s.Stats.UpdateSuccess)
 	}
 	if UpdateFail != s.Stats.UpdateFail {
 		t.Fatalf("UpdateFail [%d] != Stats.UpdateFail [%d]", UpdateFail, s.Stats.UpdateFail)
 	}
 	if CompareAndSwapSuccess != s.Stats.CompareAndSwapSuccess {
 		t.Fatalf("TestAndSetSuccess [%d] != Stats.CompareAndSwapSuccess [%d]", CompareAndSwapSuccess, s.Stats.CompareAndSwapSuccess)
 	}
 	if CompareAndSwapFail != s.Stats.CompareAndSwapFail {
 		t.Fatalf("TestAndSetFail [%d] != Stats.TestAndSetFail [%d]", CompareAndSwapFail, s.Stats.CompareAndSwapFail)
 	}
 	s = newStore()
 	CreateSuccess = 0
 	CreateFail = 0
 	for _, k := range keys {
 		i++
 		_, err := s.Create(k, "bar", false, time.Now().Add(time.Second*3), i, 1)
 		if err != nil {
 			CreateFail++
 		} else {
 			CreateSuccess++
 		}
 	}
 	time.Sleep(6 * time.Second)
 	ExpireCount := CreateSuccess
 	if ExpireCount != s.Stats.ExpireCount {
 		t.Fatalf("ExpireCount [%d] != Stats.ExpireCount [%d]", ExpireCount, s.Stats.ExpireCount)
 	}
 }
--- a/store/store.go
+++ b/store/store.go
--- a/store/store_test.go
+++ b/store/store_test.go
@ -1,222 +1,565 @@
 package store
 import (
-	"encoding/json"
+	"math/rand"
 	"strconv"
 	"testing"
 	"time"
 )
-func TestStoreGetDelete(t *testing.T) {
+func TestCreateAndGet(t *testing.T) {
 	s := newStore()
-	s := CreateStore(100)
+	s.Create("/foobar", "bar", false, Permanent, 1, 1)
 	s.Set("foo", "bar", time.Unix(0, 0), 1)
 	res, err := s.Get("foo")
-	if err != nil {
+	// already exist, create should fail
-		t.Fatalf("Unknown error")
+	_, err := s.Create("/foobar", "bar", false, Permanent, 1, 1)
 	}
 	var result Response
 	json.Unmarshal(res, &result)
 	if result.Value != "bar" {
 		t.Fatalf("Cannot get stored value")
 	}
 	s.Delete("foo", 2)
 	_, err = s.Get("foo")
 	if err == nil {
-		t.Fatalf("Got deleted value")
+		t.Fatal("Create should fail")
 	}
 	s.Delete("/foobar", true, 1, 1)
 	s.Create("/foobar/foo", "bar", false, Permanent, 1, 1)
 	// already exist, create should fail
 	_, err = s.Create("/foobar", "bar", false, Permanent, 1, 1)
 	if err == nil {
 		t.Fatal("Create should fail")
 	}
 	s.Delete("/foobar", true, 1, 1)
 	// this should create successfully
 	createAndGet(s, "/foobar", t)
 	createAndGet(s, "/foo/bar", t)
 	createAndGet(s, "/foo/foo/bar", t)
 	// meet file, create should fail
 	_, err = s.Create("/foo/bar/bar", "bar", false, Permanent, 2, 1)
 	if err == nil {
 		t.Fatal("Create should fail")
 	}
 	// create a directory
 	_, err = s.Create("/fooDir", "", false, Permanent, 3, 1)
 	if err != nil {
 		t.Fatal("Cannot create /fooDir")
 	}
 	e, err := s.Get("/fooDir", false, false, 3, 1)
 	if err != nil || e.Dir != true {
 		t.Fatal("Cannot create /fooDir ")
 	}
 	// create a file under directory
 	_, err = s.Create("/fooDir/bar", "bar", false, Permanent, 4, 1)
 	if err != nil {
 		t.Fatal("Cannot create /fooDir/bar = bar")
 	}
 }
-func TestSaveAndRecovery(t *testing.T) {
+func TestUpdateFile(t *testing.T) {
 	s := newStore()
-	s := CreateStore(100)
+	_, err := s.Create("/foo/bar", "bar", false, Permanent, 1, 1)
 	s.Set("foo", "bar", time.Unix(0, 0), 1)
 	s.Set("foo2", "bar2", time.Now().Add(time.Second*5), 2)
 	state, err := s.Save()
 	if err != nil {
-		t.Fatalf("Cannot Save %s", err)
+		t.Fatalf("cannot create %s=bar [%s]", "/foo/bar", err.Error())
 	}
-	newStore := CreateStore(100)
+	_, err = s.Update("/foo/bar", "barbar", Permanent, 2, 1)
-	// wait for foo2 expires
+	if err != nil {
-	time.Sleep(time.Second * 6)
+		t.Fatalf("cannot update %s=barbar [%s]", "/foo/bar", err.Error())
 	newStore.Recovery(state)
 	res, err := newStore.Get("foo")
 	var result Response
 	json.Unmarshal(res, &result)
 	if result.Value != "bar" {
 		t.Fatalf("Recovery Fail")
 	}
-	res, err = newStore.Get("foo2")
+	e, err := s.Get("/foo/bar", false, false, 2, 1)
 	if err != nil {
 		t.Fatalf("cannot get %s [%s]", "/foo/bar", err.Error())
 	}
 	if e.Value != "barbar" {
 		t.Fatalf("expect value of %s is barbar [%s]", "/foo/bar", e.Value)
 	}
 	// create a directory, update its ttl, to see if it will be deleted
 	_, err = s.Create("/foo/foo", "", false, Permanent, 3, 1)
 	if err != nil {
 		t.Fatalf("cannot create dir [%s] [%s]", "/foo/foo", err.Error())
 	}
 	_, err = s.Create("/foo/foo/foo1", "bar1", false, Permanent, 4, 1)
 	if err != nil {
 		t.Fatal("cannot create [%s]", err.Error())
 	}
 	_, err = s.Create("/foo/foo/foo2", "", false, Permanent, 5, 1)
 	if err != nil {
 		t.Fatal("cannot create [%s]", err.Error())
 	}
 	_, err = s.Create("/foo/foo/foo2/boo", "boo1", false, Permanent, 6, 1)
 	if err != nil {
 		t.Fatal("cannot create [%s]", err.Error())
 	}
 	expire := time.Now().Add(time.Second * 2)
 	_, err = s.Update("/foo/foo", "", expire, 7, 1)
 	if err != nil {
 		t.Fatalf("cannot update dir [%s] [%s]", "/foo/foo", err.Error())
 	}
 	// sleep 50ms, it should still reach the node
 	time.Sleep(time.Microsecond * 50)
 	e, err = s.Get("/foo/foo", true, false, 7, 1)
 	if err != nil || e.Key != "/foo/foo" {
 		t.Fatalf("cannot get dir before expiration [%s]", err.Error())
 	}
 	if e.KVPairs[0].Key != "/foo/foo/foo1" || e.KVPairs[0].Value != "bar1" {
 		t.Fatalf("cannot get sub node before expiration [%s]", err.Error())
 	}
 	if e.KVPairs[1].Key != "/foo/foo/foo2" || e.KVPairs[1].Dir != true {
 		t.Fatalf("cannot get sub dir before expiration [%s]", err.Error())
 	}
 	// wait for expiration
 	time.Sleep(time.Second * 3)
 	e, err = s.Get("/foo/foo", true, false, 7, 1)
 	if err == nil {
-		t.Fatalf("Get expired value")
+		t.Fatal("still can get dir after expiration [%s]")
 	}
-	s.Delete("foo", 3)
+	_, err = s.Get("/foo/foo/foo1", true, false, 7, 1)
 	if err == nil {
 		t.Fatal("still can get sub node after expiration [%s]")
 	}
 	_, err = s.Get("/foo/foo/foo2", true, false, 7, 1)
 	if err == nil {
 		t.Fatal("still can get sub dir after expiration [%s]")
 	}
 	_, err = s.Get("/foo/foo/foo2/boo", true, false, 7, 1)
 	if err == nil {
 		t.Fatalf("still can get sub node of sub dir after expiration [%s]", err.Error())
 	}
 }
 func TestListDirectory(t *testing.T) {
 	s := newStore()
 	// create dir /foo
 	// set key-value /foo/foo=bar
 	s.Create("/foo/foo", "bar", false, Permanent, 1, 1)
 	// create dir /foo/fooDir
 	// set key-value /foo/fooDir/foo=bar
 	s.Create("/foo/fooDir/foo", "bar", false, Permanent, 2, 1)
 	e, err := s.Get("/foo", true, false, 2, 1)
 	if err != nil {
 		t.Fatalf("%v", err)
 	}
 	if len(e.KVPairs) != 2 {
 		t.Fatalf("wrong number of kv pairs [%d/2]", len(e.KVPairs))
 	}
 	if e.KVPairs[0].Key != "/foo/foo" || e.KVPairs[0].Value != "bar" {
 		t.Fatalf("wrong kv [/foo/foo/ / %s] -> [bar / %s]", e.KVPairs[0].Key, e.KVPairs[0].Value)
 	}
 	if e.KVPairs[1].Key != "/foo/fooDir" || e.KVPairs[1].Dir != true {
 		t.Fatalf("wrong kv [/foo/fooDir/ / %s] -> [true / %v]", e.KVPairs[1].Key, e.KVPairs[1].Dir)
 	}
 	if e.KVPairs[1].KVPairs[0].Key != "/foo/fooDir/foo" || e.KVPairs[1].KVPairs[0].Value != "bar" {
 		t.Fatalf("wrong kv [/foo/fooDir/foo / %s] -> [bar / %v]", e.KVPairs[1].KVPairs[0].Key, e.KVPairs[1].KVPairs[0].Value)
 	}
 	// test hidden node
 	// create dir /foo/_hidden
 	// set key-value /foo/_hidden/foo -> bar
 	s.Create("/foo/_hidden/foo", "bar", false, Permanent, 3, 1)
 	e, _ = s.Get("/foo", false, false, 2, 1)
 	if len(e.KVPairs) != 2 {
 		t.Fatalf("hidden node is not hidden! %s", e.KVPairs[2].Key)
 	}
 }
 func TestRemove(t *testing.T) {
 	s := newStore()
 	s.Create("/foo", "bar", false, Permanent, 1, 1)
 	_, err := s.Delete("/foo", false, 1, 1)
 	if err != nil {
 		t.Fatalf("cannot delete %s [%s]", "/foo", err.Error())
 	}
 	_, err = s.Get("/foo", false, false, 1, 1)
 	if err == nil || err.Error() != "Key Not Found" {
 		t.Fatalf("can get the node after deletion")
 	}
 	s.Create("/foo/bar", "bar", false, Permanent, 1, 1)
 	s.Create("/foo/car", "car", false, Permanent, 1, 1)
 	s.Create("/foo/dar/dar", "dar", false, Permanent, 1, 1)
 	_, err = s.Delete("/foo", false, 1, 1)
 	if err == nil {
 		t.Fatalf("should not be able to delete a directory without recursive")
 	}
 	_, err = s.Delete("/foo", true, 1, 1)
 	if err != nil {
 		t.Fatalf("cannot delete %s [%s]", "/foo", err.Error())
 	}
 	_, err = s.Get("/foo", false, false, 1, 1)
 	if err == nil || err.Error() != "Key Not Found" {
 		t.Fatalf("can get the node after deletion ")
 	}
 }
 func TestExpire(t *testing.T) {
-	// test expire
+	s := newStore()
 	s := CreateStore(100)
 	s.Set("foo", "bar", time.Now().Add(time.Second*1), 0)
 	time.Sleep(2 * time.Second)
-	_, err := s.Get("foo")
+	expire := time.Now().Add(time.Second)
-	if err == nil {
+	s.Create("/foo", "bar", false, expire, 1, 1)
 		t.Fatalf("Got expired value")
 	}
-	//test change expire time
+	_, err := s.Get("/foo", false, false, 1, 1)
 	s.Set("foo", "bar", time.Now().Add(time.Second*10), 1)
 	_, err = s.Get("foo")
 	if err != nil {
-		t.Fatalf("Cannot get Value")
+		t.Fatalf("can not get the node")
 	}
-	s.Set("foo", "barbar", time.Now().Add(time.Second*1), 2)
+	time.Sleep(time.Second * 2)
-	time.Sleep(2 * time.Second)
+	_, err = s.Get("/foo", false, false, 1, 1)
 	_, err = s.Get("foo")
 	if err == nil {
-		t.Fatalf("Got expired value")
+		t.Fatalf("can get the node after expiration time")
 	}
-	// test change expire to stable
+	// test if we can reach the node before expiration
-	s.Set("foo", "bar", time.Now().Add(time.Second*1), 3)
+	expire = time.Now().Add(time.Second)
 	s.Create("/foo", "bar", false, expire, 1, 1)
-	s.Set("foo", "bar", time.Unix(0, 0), 4)
+	time.Sleep(time.Millisecond * 50)
-
+	_, err = s.Get("/foo", false, false, 1, 1)
 	time.Sleep(2 * time.Second)
 	_, err = s.Get("foo")
 	if err != nil {
-		t.Fatalf("Cannot get Value")
+		t.Fatalf("cannot get the node before expiration", err.Error())
 	}
-	// test stable to expire
+	expire = time.Now().Add(time.Second)
 	s.Set("foo", "bar", time.Now().Add(time.Second*1), 5)
 	time.Sleep(2 * time.Second)
 	_, err = s.Get("foo")
-	if err == nil {
+	s.Create("/foo", "bar", false, expire, 1, 1)
-		t.Fatalf("Got expired value")
+	_, err = s.Delete("/foo", false, 1, 1)
 	if err != nil {
 		t.Fatalf("cannot delete the node before expiration", err.Error())
 	}
 	// test set older node
 	s.Set("foo", "bar", time.Now().Add(-time.Second*1), 6)
 	_, err = s.Get("foo")
 	if err == nil {
 		t.Fatalf("Got expired value")
 	}
 }
-func BenchmarkStoreSet(b *testing.B) {
+func TestCompareAndSwap(t *testing.T) { // TODO prevValue == nil ?
-	s := CreateStore(100)
+	s := newStore()
 	s.Create("/foo", "bar", false, Permanent, 1, 1)
-	keys := GenKeys(10000, 5)
+	// test on wrong previous value
 	_, err := s.CompareAndSwap("/foo", "barbar", 0, "car", Permanent, 2, 1)
 	if err == nil {
 		t.Fatal("test and set should fail barbar != bar")
 	}
-	b.ResetTimer()
+	// test on value
-	for i := 0; i < b.N; i++ {
+	e, err := s.CompareAndSwap("/foo", "bar", 0, "car", Permanent, 3, 1)
-		for i, key := range keys {
+	if err != nil {
-			s.Set(key, "barbarbarbarbar", time.Unix(0, 0), uint64(i))
+		t.Fatal("test and set should succeed bar == bar")
 	}
 	if e.PrevValue != "bar" || e.Value != "car" {
 		t.Fatalf("[%v/%v] [%v/%v]", e.PrevValue, "bar", e.Value, "car")
 	}
 	// test on index
 	e, err = s.CompareAndSwap("/foo", "", 3, "bar", Permanent, 4, 1)
 	if err != nil {
 		t.Fatal("test and set should succeed index 3 == 3")
 	}
 	if e.PrevValue != "car" || e.Value != "bar" {
 		t.Fatalf("[%v/%v] [%v/%v]", e.PrevValue, "car", e.Value, "bar")
 	}
 }
 func TestWatch(t *testing.T) {
 	s := newStore()
 	// watch at a deeper path
 	c, _ := s.Watch("/foo/foo/foo", false, 0, 0, 1)
 	s.Create("/foo/foo/foo", "bar", false, Permanent, 1, 1)
 	e := nonblockingRetrive(c)
 	if e.Key != "/foo/foo/foo" || e.Action != Create {
 		t.Fatal("watch for Create node fails ", e)
 	}
 	c, _ = s.Watch("/foo/foo/foo", false, 0, 1, 1)
 	s.Update("/foo/foo/foo", "car", Permanent, 2, 1)
 	e = nonblockingRetrive(c)
 	if e.Key != "/foo/foo/foo" || e.Action != Update {
 		t.Fatal("watch for Update node fails ", e)
 	}
 	c, _ = s.Watch("/foo/foo/foo", false, 0, 2, 1)
 	s.CompareAndSwap("/foo/foo/foo", "car", 0, "bar", Permanent, 3, 1)
 	e = nonblockingRetrive(c)
 	if e.Key != "/foo/foo/foo" || e.Action != CompareAndSwap {
 		t.Fatal("watch for CompareAndSwap node fails")
 	}
 	c, _ = s.Watch("/foo/foo/foo", false, 0, 3, 1)
 	s.Delete("/foo", true, 4, 1) //recursively delete
 	e = nonblockingRetrive(c)
 	if e.Key != "/foo" || e.Action != Delete {
 		t.Fatal("watch for Delete node fails ", e)
 	}
 	// watch at a prefix
 	c, _ = s.Watch("/foo", true, 0, 4, 1)
 	s.Create("/foo/foo/boo", "bar", false, Permanent, 5, 1)
 	e = nonblockingRetrive(c)
 	if e.Key != "/foo/foo/boo" || e.Action != Create {
 		t.Fatal("watch for Create subdirectory fails")
 	}
 	c, _ = s.Watch("/foo", true, 0, 5, 1)
 	s.Update("/foo/foo/boo", "foo", Permanent, 6, 1)
 	e = nonblockingRetrive(c)
 	if e.Key != "/foo/foo/boo" || e.Action != Update {
 		t.Fatal("watch for Update subdirectory fails")
 	}
 	c, _ = s.Watch("/foo", true, 0, 6, 1)
 	s.CompareAndSwap("/foo/foo/boo", "foo", 0, "bar", Permanent, 7, 1)
 	e = nonblockingRetrive(c)
 	if e.Key != "/foo/foo/boo" || e.Action != CompareAndSwap {
 		t.Fatal("watch for CompareAndSwap subdirectory fails")
 	}
 	c, _ = s.Watch("/foo", true, 0, 7, 1)
 	s.Delete("/foo/foo/boo", false, 8, 1)
 	e = nonblockingRetrive(c)
 	if e == nil || e.Key != "/foo/foo/boo" || e.Action != Delete {
 		t.Fatal("watch for Delete subdirectory fails")
 	}
 	// watch expire
 	s.Create("/foo/foo/boo", "foo", false, time.Now().Add(time.Second*1), 9, 1)
 	c, _ = s.Watch("/foo", true, 0, 9, 1)
 	time.Sleep(time.Second * 2)
 	e = nonblockingRetrive(c)
 	if e.Key != "/foo/foo/boo" || e.Action != Expire || e.Index != 9 {
 		t.Fatal("watch for Expiration of Create() subdirectory fails ", e)
 	}
 	s.Create("/foo/foo/boo", "foo", false, Permanent, 10, 1)
 	s.Update("/foo/foo/boo", "bar", time.Now().Add(time.Second*1), 11, 1)
 	c, _ = s.Watch("/foo", true, 0, 11, 1)
 	time.Sleep(time.Second * 2)
 	e = nonblockingRetrive(c)
 	if e.Key != "/foo/foo/boo" || e.Action != Expire || e.Index != 11 {
 		t.Fatal("watch for Expiration of Update() subdirectory fails ", e)
 	}
 	s.Create("/foo/foo/boo", "foo", false, Permanent, 12, 1)
 	s.CompareAndSwap("/foo/foo/boo", "foo", 0, "bar", time.Now().Add(time.Second*1), 13, 1)
 	c, _ = s.Watch("/foo", true, 0, 13, 1)
 	time.Sleep(time.Second * 2)
 	e = nonblockingRetrive(c)
 	if e.Key != "/foo/foo/boo" || e.Action != Expire || e.Index != 13 {
 		t.Fatal("watch for Expiration of CompareAndSwap() subdirectory fails ", e)
 	}
 }
 func TestSort(t *testing.T) {
 	s := newStore()
 	// simulating random creation
 	keys := GenKeys(80, 4)
 	i := uint64(1)
 	for _, k := range keys {
 		_, err := s.Create(k, "bar", false, Permanent, i, 1)
 		if err != nil {
 			panic(err)
 		} else {
 			i++
 		}
 	}
 	e, err := s.Get("/foo", true, true, i, 1)
 	if err != nil {
 		t.Fatalf("get dir nodes failed [%s]", err.Error())
 	}
 	for i, k := range e.KVPairs[:len(e.KVPairs)-1] {
 		if k.Key >= e.KVPairs[i+1].Key {
 			t.Fatalf("sort failed, [%s] should be placed after [%s]", k.Key, e.KVPairs[i+1].Key)
 		}
-		s = CreateStore(100)
+		if k.Dir {
-	}
+			recursiveTestSort(k, t)
 }
 func BenchmarkStoreGet(b *testing.B) {
 	s := CreateStore(100)
 	keys := GenKeys(10000, 5)
 	for i, key := range keys {
 		s.Set(key, "barbarbarbarbar", time.Unix(0, 0), uint64(i))
 	}
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		for _, key := range keys {
 			s.Get(key)
 		}
 	}
 	if k := e.KVPairs[len(e.KVPairs)-1]; k.Dir {
 		recursiveTestSort(k, t)
 	}
 }
-func BenchmarkStoreSnapshotCopy(b *testing.B) {
+func TestSaveAndRecover(t *testing.T) {
-	s := CreateStore(100)
+	s := newStore()
-	keys := GenKeys(10000, 5)
+	// simulating random creation
 	keys := GenKeys(8, 4)
-	for i, key := range keys {
+	i := uint64(1)
-		s.Set(key, "barbarbarbarbar", time.Unix(0, 0), uint64(i))
+	for _, k := range keys {
 		_, err := s.Create(k, "bar", false, Permanent, i, 1)
 		if err != nil {
 			panic(err)
 		} else {
 			i++
 		}
 	}
-	var state []byte
+	// create a node with expiration
 	// test if we can reach the node before expiration
-	b.ResetTimer()
+	expire := time.Now().Add(time.Second)
-	for i := 0; i < b.N; i++ {
+	s.Create("/foo/foo", "bar", false, expire, 1, 1)
-		s.clone()
+	b, err := s.Save()
 	cloneFs := newStore()
 	time.Sleep(2 * time.Second)
 	cloneFs.Recovery(b)
 	for i, k := range keys {
 		_, err := cloneFs.Get(k, false, false, uint64(i), 1)
 		if err != nil {
 			panic(err)
 		}
 	}
 	// lock to avoid racing with Expire()
 	s.worldLock.RLock()
 	defer s.worldLock.RUnlock()
 	if s.WatcherHub.EventHistory.StartIndex != cloneFs.WatcherHub.EventHistory.StartIndex {
 		t.Fatalf("Error recovered event history start index[%v/%v]",
 			s.WatcherHub.EventHistory.StartIndex, cloneFs.WatcherHub.EventHistory.StartIndex)
 	}
 	for i = 0; int(i) < cloneFs.WatcherHub.EventHistory.Queue.Size; i++ {
 		if s.WatcherHub.EventHistory.Queue.Events[i].Key !=
 			cloneFs.WatcherHub.EventHistory.Queue.Events[i].Key {
 			t.Fatal("Error recovered event history")
 		}
 	}
 	_, err = s.Get("/foo/foo", false, false, 1, 1)
 	if err == nil || err.Error() != "Key Not Found" {
 		t.Fatalf("can get the node after deletion ")
 	}
 	b.SetBytes(int64(len(state)))
 }
-func BenchmarkSnapshotSaveJson(b *testing.B) {
+// GenKeys randomly generate num of keys with max depth
-	s := CreateStore(100)
+func GenKeys(num int, depth int) []string {
 	rand.Seed(time.Now().UnixNano())
 	keys := make([]string, num)
 	for i := 0; i < num; i++ {
-	keys := GenKeys(10000, 5)
+		keys[i] = "/foo"
 		depth := rand.Intn(depth) + 1
-	for i, key := range keys {
+		for j := 0; j < depth; j++ {
-		s.Set(key, "barbarbarbarbar", time.Unix(0, 0), uint64(i))
+			keys[i] += "/" + strconv.Itoa(rand.Int())
 		}
 	}
-	var state []byte
+	return keys
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		state, _ = s.Save()
 	}
 	b.SetBytes(int64(len(state)))
 }
-func BenchmarkSnapshotRecovery(b *testing.B) {
+func createAndGet(s *store, path string, t *testing.T) {
-	s := CreateStore(100)
+	_, err := s.Create(path, "bar", false, Permanent, 1, 1)
-	keys := GenKeys(10000, 5)
+	if err != nil {
-
+		t.Fatalf("cannot create %s=bar [%s]", path, err.Error())
 	for i, key := range keys {
 		s.Set(key, "barbarbarbarbar", time.Unix(0, 0), uint64(i))
 	}
-	state, _ := s.Save()
+	e, err := s.Get(path, false, false, 1, 1)
-	b.ResetTimer()
+	if err != nil {
-	for i := 0; i < b.N; i++ {
+		t.Fatalf("cannot get %s [%s]", path, err.Error())
-		newStore := CreateStore(100)
+	}
-		newStore.Recovery(state)
+
 	if e.Value != "bar" {
 		t.Fatalf("expect value of %s is bar [%s]", path, e.Value)
 	}
 }
 func recursiveTestSort(k KeyValuePair, t *testing.T) {
 	for i, v := range k.KVPairs[:len(k.KVPairs)-1] {
 		if v.Key >= k.KVPairs[i+1].Key {
 			t.Fatalf("sort failed, [%s] should be placed after [%s]", v.Key, k.KVPairs[i+1].Key)
 		}
 		if v.Dir {
 			recursiveTestSort(v, t)
 		}
 	}
 	if v := k.KVPairs[len(k.KVPairs)-1]; v.Dir {
 		recursiveTestSort(v, t)
 	}
 }
 func nonblockingRetrive(c <-chan *Event) *Event {
 	select {
 	case e := <-c:
 		return e
 	default:
 		return nil
 	}
 	b.SetBytes(int64(len(state)))
 }
--- a/store/test.go
+++ b/store/test.go
@ -1,21 +0,0 @@
 package store
 import (
 	"math/rand"
 	"strconv"
 )
 // GenKeys randomly generate num of keys with max depth
 func GenKeys(num int, depth int) []string {
 	keys := make([]string, num)
 	for i := 0; i < num; i++ {
 		keys[i] = "/foo/"
 		depth := rand.Intn(depth) + 1
 		for j := 0; j < depth; j++ {
 			keys[i] += "/" + strconv.Itoa(rand.Int())
 		}
 	}
 	return keys
 }
--- a/store/tree.go
+++ b/store/tree.go
@ -1,318 +0,0 @@
 package store
 import (
 	"path"
 	"sort"
 	"strings"
 	"time"
 )
 //------------------------------------------------------------------------------
 //
 // Typedefs
 //
 //------------------------------------------------------------------------------
 // A file system like tree structure. Each non-leaf node of the tree has a hashmap to
 // store its children nodes. Leaf nodes has no hashmap (a nil pointer)
 type tree struct {
 	Root *treeNode
 }
 // A treeNode wraps a Node. It has a hashmap to keep records of its children treeNodes.
 type treeNode struct {
 	InternalNode Node
 	Dir          bool
 	NodeMap      map[string]*treeNode
 }
 // TreeNode with its key. We use it when we need to sort the treeNodes.
 type tnWithKey struct {
 	key string
 	tn  *treeNode
 }
 // Define type and functions to match sort interface
 type tnWithKeySlice []tnWithKey
 func (s tnWithKeySlice) Len() int           { return len(s) }
 func (s tnWithKeySlice) Less(i, j int) bool { return s[i].key < s[j].key }
 func (s tnWithKeySlice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
 // CONSTANT VARIABLE
 // Represent an empty node
 var emptyNode = Node{"", PERMANENT, nil}
 //------------------------------------------------------------------------------
 //
 // Methods
 //
 //------------------------------------------------------------------------------
 // Set the key to the given value, return true if success
 // If any intermidate path of the key is not a directory type, it will fail
 // For example if the /foo = Node(bar) exists, set /foo/foo = Node(barbar)
 // will fail.
 func (t *tree) set(key string, value Node) bool {
 	nodesName := split(key)
 	// avoid set value to "/"
 	if len(nodesName) == 1 && len(nodesName[0]) == 0 {
 		return false
 	}
 	nodeMap := t.Root.NodeMap
 	i := 0
 	newDir := false
 	// go through all the path
 	for i = 0; i < len(nodesName)-1; i++ {
 		// if we meet a new directory, all the directory after it must be new
 		if newDir {
 			tn := &treeNode{emptyNode, true, make(map[string]*treeNode)}
 			nodeMap[nodesName[i]] = tn
 			nodeMap = tn.NodeMap
 			continue
 		}
 		// get the node from the nodeMap of the current level
 		tn, ok := nodeMap[nodesName[i]]
 		if !ok {
 			// add a new directory and set newDir to true
 			newDir = true
 			tn := &treeNode{emptyNode, true, make(map[string]*treeNode)}
 			nodeMap[nodesName[i]] = tn
 			nodeMap = tn.NodeMap
 		} else if ok && !tn.Dir {
 			// if we meet a non-directory node, we cannot set the key
 			return false
 		} else {
 			// update the nodeMap to next level
 			nodeMap = tn.NodeMap
 		}
 	}
 	// Add the last node
 	tn, ok := nodeMap[nodesName[i]]
 	if !ok {
 		// we add a new treeNode
 		tn := &treeNode{value, false, nil}
 		nodeMap[nodesName[i]] = tn
 	} else {
 		if tn.Dir {
 			return false
 		}
 		// we change the value of a old Treenode
 		tn.InternalNode = value
 	}
 	return true
 }
 // Get the tree node of the key
 func (t *tree) internalGet(key string) (*treeNode, bool) {
 	nodesName := split(key)
 	// should be able to get root
 	if len(nodesName) == 1 && nodesName[0] == "" {
 		return t.Root, true
 	}
 	nodeMap := t.Root.NodeMap
 	var i int
 	for i = 0; i < len(nodesName)-1; i++ {
 		node, ok := nodeMap[nodesName[i]]
 		if !ok || !node.Dir {
 			return nil, false
 		}
 		nodeMap = node.NodeMap
 	}
 	tn, ok := nodeMap[nodesName[i]]
 	if ok {
 		return tn, ok
 	} else {
 		return nil, ok
 	}
 }
 // get the internalNode of the key
 func (t *tree) get(key string) (Node, bool) {
 	tn, ok := t.internalGet(key)
 	if ok {
 		if tn.Dir {
 			return emptyNode, false
 		}
 		return tn.InternalNode, ok
 	} else {
 		return emptyNode, ok
 	}
 }
 // get the internalNode of the key
 func (t *tree) list(directory string) (interface{}, []string, bool) {
 	treeNode, ok := t.internalGet(directory)
 	if !ok {
 		return nil, nil, ok
 	} else {
 		if !treeNode.Dir {
 			return &treeNode.InternalNode, nil, ok
 		}
 		length := len(treeNode.NodeMap)
 		nodes := make([]*Node, length)
 		keys := make([]string, length)
 		i := 0
 		for key, node := range treeNode.NodeMap {
 			nodes[i] = &node.InternalNode
 			keys[i] = key
 			i++
 		}
 		return nodes, keys, ok
 	}
 }
 // delete the key, return true if success
 func (t *tree) delete(key string) bool {
 	nodesName := split(key)
 	nodeMap := t.Root.NodeMap
 	var i int
 	for i = 0; i < len(nodesName)-1; i++ {
 		node, ok := nodeMap[nodesName[i]]
 		if !ok || !node.Dir {
 			return false
 		}
 		nodeMap = node.NodeMap
 	}
 	node, ok := nodeMap[nodesName[i]]
 	if ok && !node.Dir {
 		delete(nodeMap, nodesName[i])
 		return true
 	}
 	return false
 }
 // traverse wrapper
 func (t *tree) traverse(f func(string, *Node), sort bool) {
 	if sort {
 		sortDfs("", t.Root, f)
 	} else {
 		dfs("", t.Root, f)
 	}
 }
 // clone() will return a deep cloned tree
 func (t *tree) clone() *tree {
 	newTree := new(tree)
 	newTree.Root = &treeNode{
 		Node{
 			"/",
 			time.Unix(0, 0),
 			nil,
 		},
 		true,
 		make(map[string]*treeNode),
 	}
 	recursiveClone(t.Root, newTree.Root)
 	return newTree
 }
 // recursiveClone is a helper function for clone()
 func recursiveClone(tnSrc *treeNode, tnDes *treeNode) {
 	if !tnSrc.Dir {
 		tnDes.InternalNode = tnSrc.InternalNode
 		return
 	} else {
 		tnDes.InternalNode = tnSrc.InternalNode
 		tnDes.Dir = true
 		tnDes.NodeMap = make(map[string]*treeNode)
 		for key, tn := range tnSrc.NodeMap {
 			newTn := new(treeNode)
 			recursiveClone(tn, newTn)
 			tnDes.NodeMap[key] = newTn
 		}
 	}
 }
 // deep first search to traverse the tree
 // apply the func f to each internal node
 func dfs(key string, t *treeNode, f func(string, *Node)) {
 	// base case
 	if len(t.NodeMap) == 0 {
 		f(key, &t.InternalNode)
 		// recursion
 	} else {
 		for tnKey, tn := range t.NodeMap {
 			tnKey := key + "/" + tnKey
 			dfs(tnKey, tn, f)
 		}
 	}
 }
 // sort deep first search to traverse the tree
 // apply the func f to each internal node
 func sortDfs(key string, t *treeNode, f func(string, *Node)) {
 	// base case
 	if len(t.NodeMap) == 0 {
 		f(key, &t.InternalNode)
 		// recursion
 	} else {
 		s := make(tnWithKeySlice, len(t.NodeMap))
 		i := 0
 		// copy
 		for tnKey, tn := range t.NodeMap {
 			tnKey := key + "/" + tnKey
 			s[i] = tnWithKey{tnKey, tn}
 			i++
 		}
 		// sort
 		sort.Sort(s)
 		// traverse
 		for i = 0; i < len(t.NodeMap); i++ {
 			sortDfs(s[i].key, s[i].tn, f)
 		}
 	}
 }
 // split the key by '/', get the intermediate node name
 func split(key string) []string {
 	key = "/" + key
 	key = path.Clean(key)
 	// get the intermidate nodes name
 	nodesName := strings.Split(key, "/")
 	// we do not need the root node, since we start with it
 	nodesName = nodesName[1:]
 	return nodesName
 }
--- a/store/tree_store_test.go
+++ b/store/tree_store_test.go
@ -1,247 +0,0 @@
 package store
 import (
 	"fmt"
 	"math/rand"
 	"strconv"
 	"testing"
 	"time"
 )
 func TestStoreGet(t *testing.T) {
 	ts := &tree{
 		&treeNode{
 			NewTestNode("/"),
 			true,
 			make(map[string]*treeNode),
 		},
 	}
 	// create key
 	ts.set("/foo", NewTestNode("bar"))
 	// change value
 	ts.set("/foo", NewTestNode("barbar"))
 	// create key
 	ts.set("/hello/foo", NewTestNode("barbarbar"))
 	treeNode, ok := ts.get("/foo")
 	if !ok {
 		t.Fatalf("Expect to get node, but not")
 	}
 	if treeNode.Value != "barbar" {
 		t.Fatalf("Expect value barbar, but got %s", treeNode.Value)
 	}
 	// create key
 	treeNode, ok = ts.get("/hello/foo")
 	if !ok {
 		t.Fatalf("Expect to get node, but not")
 	}
 	if treeNode.Value != "barbarbar" {
 		t.Fatalf("Expect value barbarbar, but got %s", treeNode.Value)
 	}
 	// create a key under other key
 	ok = ts.set("/foo/foo", NewTestNode("bar"))
 	if ok {
 		t.Fatalf("shoud not add key under a exisiting key")
 	}
 	// delete a key
 	ok = ts.delete("/foo")
 	if !ok {
 		t.Fatalf("cannot delete key")
 	}
 	// delete a directory
 	ok = ts.delete("/hello")
 	if ok {
 		t.Fatalf("Expect cannot delet /hello, but deleted! ")
 	}
 	// test list
 	ts.set("/hello/fooo", NewTestNode("barbarbar"))
 	ts.set("/hello/foooo/foo", NewTestNode("barbarbar"))
 	nodes, keys, ok := ts.list("/hello")
 	if !ok {
 		t.Fatalf("cannot list!")
 	} else {
 		nodes, _ := nodes.([]*Node)
 		length := len(nodes)
 		for i := 0; i < length; i++ {
 			fmt.Println(keys[i], "=", nodes[i].Value)
 		}
 	}
 	keys = GenKeys(100, 10)
 	for i := 0; i < 100; i++ {
 		value := strconv.Itoa(rand.Int())
 		ts.set(keys[i], NewTestNode(value))
 		treeNode, ok := ts.get(keys[i])
 		if !ok {
 			continue
 		}
 		if treeNode.Value != value {
 			t.Fatalf("Expect value %s, but got %s", value, treeNode.Value)
 		}
 	}
 	ts.traverse(f, true)
 }
 func TestTreeClone(t *testing.T) {
 	keys := GenKeys(10000, 10)
 	ts := &tree{
 		&treeNode{
 			NewTestNode("/"),
 			true,
 			make(map[string]*treeNode),
 		},
 	}
 	backTs := &tree{
 		&treeNode{
 			NewTestNode("/"),
 			true,
 			make(map[string]*treeNode),
 		},
 	}
 	// generate the first tree
 	for _, key := range keys {
 		value := strconv.Itoa(rand.Int())
 		ts.set(key, NewTestNode(value))
 		backTs.set(key, NewTestNode(value))
 	}
 	copyTs := ts.clone()
 	// test if they are identical
 	copyTs.traverse(ts.contain, false)
 	// remove all the keys from first tree
 	for _, key := range keys {
 		ts.delete(key)
 	}
 	// test if they are identical
 	// make sure changes in the first tree will affect the copy one
 	copyTs.traverse(backTs.contain, false)
 }
 func BenchmarkTreeStoreSet(b *testing.B) {
 	keys := GenKeys(10000, 10)
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		ts := &tree{
 			&treeNode{
 				NewTestNode("/"),
 				true,
 				make(map[string]*treeNode),
 			},
 		}
 		for _, key := range keys {
 			value := strconv.Itoa(rand.Int())
 			ts.set(key, NewTestNode(value))
 		}
 	}
 }
 func BenchmarkTreeStoreGet(b *testing.B) {
 	keys := GenKeys(10000, 10)
 	ts := &tree{
 		&treeNode{
 			NewTestNode("/"),
 			true,
 			make(map[string]*treeNode),
 		},
 	}
 	for _, key := range keys {
 		value := strconv.Itoa(rand.Int())
 		ts.set(key, NewTestNode(value))
 	}
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		for _, key := range keys {
 			ts.get(key)
 		}
 	}
 }
 func BenchmarkTreeStoreCopy(b *testing.B) {
 	keys := GenKeys(10000, 10)
 	ts := &tree{
 		&treeNode{
 			NewTestNode("/"),
 			true,
 			make(map[string]*treeNode),
 		},
 	}
 	for _, key := range keys {
 		value := strconv.Itoa(rand.Int())
 		ts.set(key, NewTestNode(value))
 	}
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		ts.clone()
 	}
 }
 func BenchmarkTreeStoreList(b *testing.B) {
 	keys := GenKeys(10000, 10)
 	ts := &tree{
 		&treeNode{
 			NewTestNode("/"),
 			true,
 			make(map[string]*treeNode),
 		},
 	}
 	for _, key := range keys {
 		value := strconv.Itoa(rand.Int())
 		ts.set(key, NewTestNode(value))
 	}
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		for _, key := range keys {
 			ts.list(key)
 		}
 	}
 }
 func (t *tree) contain(key string, node *Node) {
 	_, ok := t.get(key)
 	if !ok {
 		panic("tree do not contain the given key")
 	}
 }
 func f(key string, n *Node) {
 	return
 }
 func NewTestNode(value string) Node {
 	return Node{value, time.Unix(0, 0), nil}
 }
--- a/store/ttl.go
+++ b/store/ttl.go
@ -0,0 +1,20 @@
 package store
 import (
 	"strconv"
 	"time"
 )
 // Convert string duration to time format
 func TTL(duration string) (time.Time, error) {
 	if duration != "" {
 		duration, err := strconv.Atoi(duration)
 		if err != nil {
 			return Permanent, err
 		}
 		return time.Now().Add(time.Second * (time.Duration)(duration)), nil
 	} else {
 		return Permanent, nil
 	}
 }
--- a/store/update_command.go
+++ b/store/update_command.go
@ -0,0 +1,37 @@
 package store
 import (
 	"github.com/coreos/etcd/log"
 	"github.com/coreos/go-raft"
 	"time"
 )
 func init() {
 	raft.RegisterCommand(&UpdateCommand{})
 }
 // Update command
 type UpdateCommand struct {
 	Key        string    `json:"key"`
 	Value      string    `json:"value"`
 	ExpireTime time.Time `json:"expireTime"`
 }
 // The name of the update command in the log
 func (c *UpdateCommand) CommandName() string {
 	return "etcd:update"
 }
 // Create node
 func (c *UpdateCommand) Apply(server raft.Server) (interface{}, error) {
 	s, _ := server.StateMachine().(Store)
 	e, err := s.Update(c.Key, c.Value, c.ExpireTime, server.CommitIndex(), server.Term())
 	if err != nil {
 		log.Debug(err)
 		return nil, err
 	}
 	return e, nil
 }
--- a/store/watcher.go
+++ b/store/watcher.go
@ -1,129 +1,33 @@
 package store
-import (
+type watcher struct {
-	"path"
+	eventChan  chan *Event
-	"strconv"
+	recursive  bool
-	"strings"
+	sinceIndex uint64
 )
 //------------------------------------------------------------------------------
 //
 // Typedefs
 //
 //------------------------------------------------------------------------------
 // WatcherHub is where the client register its watcher
 type WatcherHub struct {
 	watchers map[string][]*Watcher
 }
-// Currently watcher only contains a response channel
+// notify function notifies the watcher. If the watcher interests in the given path,
-type Watcher struct {
+// the function will return true.
-	C chan *Response
+func (w *watcher) notify(e *Event, originalPath bool, deleted bool) bool {
-}
+	// watcher is interested the path in three cases and under one condition
 	// the condition is that the event happens after the watcher's sinceIndex
-// Create a new watcherHub
+	// 1. the path at which the event happens is the path the watcher is watching at.
-func newWatcherHub() *WatcherHub {
+	// For example if the watcher is watching at "/foo" and the event happens at "/foo",
-	w := new(WatcherHub)
+	// the watcher must be interested in that event.
 	w.watchers = make(map[string][]*Watcher)
 	return w
 }
-// Create a new watcher
+	// 2. the watcher is a recursive watcher, it interests in the event happens after
-func NewWatcher() *Watcher {
+	// its watching path. For example if watcher A watches at "/foo" and it is a recursive
-	return &Watcher{C: make(chan *Response, 1)}
+	// one, it will interest in the event happens at "/foo/bar".
 }
-// Add a watcher to the watcherHub
+	// 3. when we delete a directory, we need to force notify all the watchers who watches
-func (w *WatcherHub) addWatcher(prefix string, watcher *Watcher, sinceIndex uint64,
+	// at the file we need to delete.
-	responseStartIndex uint64, currentIndex uint64, resMap map[string]*Response) error {
+	// For example a watcher is watching at "/foo/bar". And we deletes "/foo". The watcher
 	// should get notified even if "/foo" is not the path it is watching.
-	prefix = path.Clean("/" + prefix)
+	if (w.recursive || originalPath || deleted) && e.Index >= w.sinceIndex {
-
+		w.eventChan <- e
-	if sinceIndex != 0 && sinceIndex >= responseStartIndex {
+		return true
 		for i := sinceIndex; i <= currentIndex; i++ {
 			if checkResponse(prefix, i, resMap) {
 				watcher.C <- resMap[strconv.FormatUint(i, 10)]
 				return nil
 			}
 		}
 	}
 	_, ok := w.watchers[prefix]
 	if !ok {
 		w.watchers[prefix] = make([]*Watcher, 0)
 	}
 	w.watchers[prefix] = append(w.watchers[prefix], watcher)
 	return nil
 }
 // Check if the response has what we are watching
 func checkResponse(prefix string, index uint64, resMap map[string]*Response) bool {
 	resp, ok := resMap[strconv.FormatUint(index, 10)]
 	if !ok {
 		// not storage system command
 		return false
 	} else {
 		path := resp.Key
 		if strings.HasPrefix(path, prefix) {
 			prefixLen := len(prefix)
 			if len(path) == prefixLen || path[prefixLen] == '/' {
 				return true
 			}
 		}
 	}
 	return false
 }
 // Notify the watcher a action happened
 func (w *WatcherHub) notify(resp Response) error {
 	resp.Key = path.Clean(resp.Key)
 	segments := strings.Split(resp.Key, "/")
 	currPath := "/"
 	// walk through all the pathes
 	for _, segment := range segments {
 		currPath = path.Join(currPath, segment)
 		watchers, ok := w.watchers[currPath]
 		if ok {
 			newWatchers := make([]*Watcher, 0)
 			// notify all the watchers
 			for _, watcher := range watchers {
 				watcher.C <- &resp
 			}
 			if len(newWatchers) == 0 {
 				// we have notified all the watchers at this path
 				// delete the map
 				delete(w.watchers, currPath)
 			} else {
 				w.watchers[currPath] = newWatchers
 			}
 		}
 	}
 	return nil
 }
 // stopWatchers stops all the watchers
 // This function is used when the etcd recovery from a snapshot at runtime
 func (w *WatcherHub) stopWatchers() {
 	for _, subWatchers := range w.watchers {
 		for _, watcher := range subWatchers {
 			watcher.C <- nil
 		}
 	}
 	w.watchers = nil
 }
--- a/store/watcher_hub.go
+++ b/store/watcher_hub.go
@ -0,0 +1,142 @@
 package store
 import (
 	"container/list"
 	"path"
 	"strings"
 	"sync/atomic"
 	etcdErr "github.com/coreos/etcd/error"
 )
 // A watcherHub contains all subscribed watchers
 // watchers is a map with watched path as key and watcher as value
 // EventHistory keeps the old events for watcherHub. It is used to help
 // watcher to get a continuous event history. Or a watcher might miss the
 // event happens between the end of the first watch command and the start
 // of the second command.
 type watcherHub struct {
 	watchers     map[string]*list.List
 	count        int64 // current number of watchers.
 	EventHistory *EventHistory
 }
 // newWatchHub creates a watchHub. The capacity determines how many events we will
 // keep in the eventHistory.
 // Typically, we only need to keep a small size of history[smaller than 20K].
 // Ideally, it should smaller than 20K/s[max throughput] * 2 * 50ms[RTT] = 2000
 func newWatchHub(capacity int) *watcherHub {
 	return &watcherHub{
 		watchers:     make(map[string]*list.List),
 		EventHistory: newEventHistory(capacity),
 	}
 }
 // watch function returns an Event channel.
 // If recursive is true, the first change after index under prefix will be sent to the event channel.
 // If recursive is false, the first change after index at prefix will be sent to the event channel.
 // If index is zero, watch will start from the current index + 1.
 func (wh *watcherHub) watch(prefix string, recursive bool, index uint64) (<-chan *Event, *etcdErr.Error) {
 	eventChan := make(chan *Event, 1)
 	e, err := wh.EventHistory.scan(prefix, index)
 	if err != nil {
 		return nil, err
 	}
 	if e != nil {
 		eventChan <- e
 		return eventChan, nil
 	}
 	w := &watcher{
 		eventChan:  eventChan,
 		recursive:  recursive,
 		sinceIndex: index - 1, // to catch Expire()
 	}
 	l, ok := wh.watchers[prefix]
 	if ok { // add the new watcher to the back of the list
 		l.PushBack(w)
 	} else { // create a new list and add the new watcher
 		l := list.New()
 		l.PushBack(w)
 		wh.watchers[prefix] = l
 	}
 	atomic.AddInt64(&wh.count, 1)
 	return eventChan, nil
 }
 // notify function accepts an event and notify to the watchers.
 func (wh *watcherHub) notify(e *Event) {
 	e = wh.EventHistory.addEvent(e) // add event into the eventHistory
 	segments := strings.Split(e.Key, "/")
 	currPath := "/"
 	// walk through all the segments of the path and notify the watchers
 	// if the path is "/foo/bar", it will notify watchers with path "/",
 	// "/foo" and "/foo/bar"
 	for _, segment := range segments {
 		currPath = path.Join(currPath, segment)
 		// notify the watchers who interests in the changes of current path
 		wh.notifyWatchers(e, currPath, false)
 	}
 }
 func (wh *watcherHub) notifyWatchers(e *Event, path string, deleted bool) {
 	l, ok := wh.watchers[path]
 	if ok {
 		curr := l.Front()
 		notifiedAll := true
 		for {
 			if curr == nil { // we have reached the end of the list
 				if notifiedAll {
 					// if we have notified all watcher in the list
 					// we can delete the list
 					delete(wh.watchers, path)
 				}
 				break
 			}
 			next := curr.Next() // save reference to the next one in the list
 			w, _ := curr.Value.(*watcher)
 			if w.notify(e, e.Key == path, deleted) {
 				// if we successfully notify a watcher
 				// we need to remove the watcher from the list
 				// and decrease the counter
 				l.Remove(curr)
 				atomic.AddInt64(&wh.count, -1)
 			} else {
 				// once there is a watcher in the list is not interested
 				// in the event, we should keep the list in the map
 				notifiedAll = false
 			}
 			curr = next // update current to the next
 		}
 	}
 }
 // clone function clones the watcherHub and return the cloned one.
 // only clone the static content. do not clone the current watchers.
 func (wh *watcherHub) clone() *watcherHub {
 	clonedHistory := wh.EventHistory.clone()
 	return &watcherHub{
 		EventHistory: clonedHistory,
 	}
 }
--- a/store/watcher_test.go
+++ b/store/watcher_test.go
@ -2,83 +2,54 @@ package store
 import (
 	"testing"
 	"time"
 )
-func TestWatch(t *testing.T) {
+func TestWatcher(t *testing.T) {
-
+	s := newStore()
-	s := CreateStore(100)
+	wh := s.WatcherHub
-
+	c, err := wh.watch("/foo", true, 1)
-	watchers := make([]*Watcher, 10)
+	if err != nil {
-
+		t.Fatal("%v", err)
 	for i, _ := range watchers {
 		// create a new watcher
 		watchers[i] = NewWatcher()
 		// add to the watchers list
 		s.AddWatcher("foo", watchers[i], 0)
 	}
-	s.Set("/foo/foo", "bar", time.Unix(0, 0), 1)
+	select {
-
+	case <-c:
-	for _, watcher := range watchers {
+		t.Fatal("should not receive from channel before send the event")
-
+	default:
-		// wait for the notification for any changing
+		// do nothing
 		res := <-watcher.C
 		if res == nil {
 			t.Fatal("watcher is cleared")
 		}
 	}
-	for i, _ := range watchers {
+	e := newEvent(Create, "/foo/bar", 1, 1)
-		// create a new watcher
+	wh.notify(e)
 		watchers[i] = NewWatcher()
 		// add to the watchers list
 		s.AddWatcher("foo/foo/foo", watchers[i], 0)
 	re := <-c
 	if e != re {
 		t.Fatal("recv != send")
 	}
-	s.watcher.stopWatchers()
+	c, _ = wh.watch("/foo", false, 2)
-	for _, watcher := range watchers {
+	e = newEvent(Create, "/foo/bar", 2, 1)
-		// wait for the notification for any changing
+	wh.notify(e)
 		res := <-watcher.C
-		if res != nil {
+	select {
-			t.Fatal("watcher is cleared")
+	case re = <-c:
-		}
+		t.Fatal("should not receive from channel if not recursive ", re)
-	}
+	default:
-}
+		// do nothing
 // BenchmarkWatch creates 10K watchers watch at /foo/[path] each time.
 // Path is randomly chosen with max depth 10.
 // It should take less than 15ms to wake up 10K watchers.
 func BenchmarkWatch(b *testing.B) {
 	s := CreateStore(100)
 	keys := GenKeys(10000, 10)
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		watchers := make([]*Watcher, 10000)
 		for i := 0; i < 10000; i++ {
 			// create a new watcher
 			watchers[i] = NewWatcher()
 			// add to the watchers list
 			s.AddWatcher(keys[i], watchers[i], 0)
 		}
 		s.watcher.stopWatchers()
 		for _, watcher := range watchers {
 			// wait for the notification for any changing
 			<-watcher.C
 		}
 		s.watcher = newWatcherHub()
 	}
 	e = newEvent(Create, "/foo", 3, 1)
 	wh.notify(e)
 	re = <-c
 	if e != re {
 		t.Fatal("recv != send")
 	}
 }
--- a/8
+++ b/8
@ -1,8 +0,0 @@
 #!/bin/sh
 # Get GOPATH, etc from build
 . ./build
 # Run the tests!
 go test -i
 go test -v
--- a/test.sh
+++ b/test.sh
@ -0,0 +1,11 @@
 #!/bin/sh
 set -e
 # Get GOPATH, etc from build
 . ./build
 # Unit tests
 go test -v ./store
 # Functional tests
 ETCD_BIN_PATH=$(pwd)/etcd go test -v ./tests/functional
--- a/tests/functional/etcd_direct_call.go
+++ b/tests/functional/etcd_direct_call.go
@ -0,0 +1,34 @@
 package test
 import (
 	"net/http"
 	"os"
 	"testing"
 	"time"
 )
 func BenchmarkEtcdDirectCall(b *testing.B) {
 	templateBenchmarkEtcdDirectCall(b, false)
 }
 func BenchmarkEtcdDirectCallTls(b *testing.B) {
 	templateBenchmarkEtcdDirectCall(b, true)
 }
 func templateBenchmarkEtcdDirectCall(b *testing.B, tls bool) {
 	procAttr := new(os.ProcAttr)
 	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
 	clusterSize := 3
 	_, etcds, _ := CreateCluster(clusterSize, procAttr, tls)
 	defer DestroyCluster(etcds)
 	time.Sleep(time.Second)
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		resp, _ := http.Get("http://127.0.0.1:4001/test/speed")
 		resp.Body.Close()
 	}
 }
--- a/tests/functional/init.go
+++ b/tests/functional/init.go
@ -0,0 +1,17 @@
 package test
 import (
 	"go/build"
 	"os"
 	"path/filepath"
 )
 var EtcdBinPath string
 func init() {
 	// Initialize the 'etcd' binary path or default it to the etcd diretory.
 	EtcdBinPath = os.Getenv("ETCD_BIN_PATH")
 	if EtcdBinPath == "" {
 		EtcdBinPath = filepath.Join(build.Default.GOPATH, "src", "github.com", "coreos", "etcd", "etcd")
 	}
 }
--- a/tests/functional/internal_version_test.go
+++ b/tests/functional/internal_version_test.go
@ -0,0 +1,57 @@
 package test
 import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"os"
 	"testing"
 	"time"
 )
 // Ensure that etcd does not come up if the internal raft versions do not match.
 func TestInternalVersion(t *testing.T) {
 	checkedVersion := false
 	testMux := http.NewServeMux()
 	testMux.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprintln(w, "This is not a version number")
 		checkedVersion = true
 	})
 	testMux.HandleFunc("/join", func(w http.ResponseWriter, r *http.Request) {
 		t.Fatal("should not attempt to join!")
 	})
 	ts := httptest.NewServer(testMux)
 	defer ts.Close()
 	fakeURL, _ := url.Parse(ts.URL)
 	procAttr := new(os.ProcAttr)
 	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
 	args := []string{"etcd", "-n=node1", "-f", "-d=/tmp/node1", "-C=" + fakeURL.Host}
 	process, err := os.StartProcess(EtcdBinPath, args, procAttr)
 	if err != nil {
 		t.Fatal("start process failed:" + err.Error())
 		return
 	}
 	defer process.Kill()
 	time.Sleep(time.Second)
 	_, err = http.Get("http://127.0.0.1:4001")
 	if err == nil {
 		t.Fatal("etcd node should not be up")
 		return
 	}
 	if checkedVersion == false {
 		t.Fatal("etcd did not check the version")
 		return
 	}
 }
--- a/tests/functional/kill_leader_test.go
+++ b/tests/functional/kill_leader_test.go
@ -0,0 +1,63 @@
 package test
 import (
 	"fmt"
 	"os"
 	"strconv"
 	"strings"
 	"testing"
 	"time"
 )
 // This test will kill the current leader and wait for the etcd cluster to elect a new leader for 200 times.
 // It will print out the election time and the average election time.
 func TestKillLeader(t *testing.T) {
 	procAttr := new(os.ProcAttr)
 	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
 	clusterSize := 5
 	argGroup, etcds, err := CreateCluster(clusterSize, procAttr, false)
 	if err != nil {
 		t.Fatal("cannot create cluster")
 	}
 	defer DestroyCluster(etcds)
 	stop := make(chan bool)
 	leaderChan := make(chan string, 1)
 	all := make(chan bool, 1)
 	time.Sleep(time.Second)
 	go Monitor(clusterSize, 1, leaderChan, all, stop)
 	var totalTime time.Duration
 	leader := "http://127.0.0.1:7001"
 	for i := 0; i < clusterSize; i++ {
 		fmt.Println("leader is ", leader)
 		port, _ := strconv.Atoi(strings.Split(leader, ":")[2])
 		num := port - 7001
 		fmt.Println("kill server ", num)
 		etcds[num].Kill()
 		etcds[num].Release()
 		start := time.Now()
 		for {
 			newLeader := <-leaderChan
 			if newLeader != leader {
 				leader = newLeader
 				break
 			}
 		}
 		take := time.Now().Sub(start)
 		totalTime += take
 		avgTime := totalTime / (time.Duration)(i+1)
 		fmt.Println("Total time:", totalTime, "; Avg time:", avgTime)
 		etcds[num], err = os.StartProcess(EtcdBinPath, argGroup[num], procAttr)
 	}
 	stop <- true
 }
--- a/tests/functional/kill_random_test.go
+++ b/tests/functional/kill_random_test.go
@ -0,0 +1,76 @@
 package test
 import (
 	"fmt"
 	"math/rand"
 	"os"
 	"testing"
 	"time"
 )
 // TestKillRandom kills random machines in the cluster and
 // restart them after all other machines agree on the same leader
 func TestKillRandom(t *testing.T) {
 	procAttr := new(os.ProcAttr)
 	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
 	clusterSize := 9
 	argGroup, etcds, err := CreateCluster(clusterSize, procAttr, false)
 	if err != nil {
 		t.Fatal("cannot create cluster")
 	}
 	defer DestroyCluster(etcds)
 	stop := make(chan bool)
 	leaderChan := make(chan string, 1)
 	all := make(chan bool, 1)
 	time.Sleep(3 * time.Second)
 	go Monitor(clusterSize, 4, leaderChan, all, stop)
 	toKill := make(map[int]bool)
 	for i := 0; i < 20; i++ {
 		fmt.Printf("TestKillRandom Round[%d/20]\n", i)
 		j := 0
 		for {
 			r := rand.Int31n(9)
 			if _, ok := toKill[int(r)]; !ok {
 				j++
 				toKill[int(r)] = true
 			}
 			if j > 3 {
 				break
 			}
 		}
 		for num, _ := range toKill {
 			err := etcds[num].Kill()
 			if err != nil {
 				panic(err)
 			}
 			etcds[num].Wait()
 		}
 		time.Sleep(1 * time.Second)
 		<-leaderChan
 		for num, _ := range toKill {
 			etcds[num], err = os.StartProcess(EtcdBinPath, argGroup[num], procAttr)
 		}
 		toKill = make(map[int]bool)
 		<-all
 	}
 	stop <- true
 }
--- a/tests/functional/multi_node_kill_all_and_recovery_test.go
+++ b/tests/functional/multi_node_kill_all_and_recovery_test.go
@ -0,0 +1,72 @@
 package test
 import (
 	"os"
 	"testing"
 	"time"
 	"github.com/coreos/go-etcd/etcd"
 )
 // Create a five nodes
 // Kill all the nodes and restart
 func TestMultiNodeKillAllAndRecovery(t *testing.T) {
 	procAttr := new(os.ProcAttr)
 	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
 	clusterSize := 5
 	argGroup, etcds, err := CreateCluster(clusterSize, procAttr, false)
 	defer DestroyCluster(etcds)
 	if err != nil {
 		t.Fatal("cannot create cluster")
 	}
 	c := etcd.NewClient(nil)
 	c.SyncCluster()
 	time.Sleep(time.Second)
 	// send 10 commands
 	for i := 0; i < 10; i++ {
 		// Test Set
 		_, err := c.Set("foo", "bar", 0)
 		if err != nil {
 			panic(err)
 		}
 	}
 	time.Sleep(time.Second)
 	// kill all
 	DestroyCluster(etcds)
 	time.Sleep(time.Second)
 	stop := make(chan bool)
 	leaderChan := make(chan string, 1)
 	all := make(chan bool, 1)
 	time.Sleep(time.Second)
 	for i := 0; i < clusterSize; i++ {
 		etcds[i], err = os.StartProcess(EtcdBinPath, argGroup[i], procAttr)
 	}
 	go Monitor(clusterSize, 1, leaderChan, all, stop)
 	<-all
 	<-leaderChan
 	result, err := c.Set("foo", "bar", 0)
 	if err != nil {
 		t.Fatalf("Recovery error: %s", err)
 	}
 	if result.Index != 18 {
 		t.Fatalf("recovery failed! [%d/18]", result.Index)
 	}
 }
--- a/tests/functional/multi_node_kill_one_test.go
+++ b/tests/functional/multi_node_kill_one_test.go
@ -0,0 +1,58 @@
 package test
 import (
 	"fmt"
 	"math/rand"
 	"os"
 	"testing"
 	"time"
 	"github.com/coreos/go-etcd/etcd"
 )
 // Create a five nodes
 // Randomly kill one of the node and keep on sending set command to the cluster
 func TestMultiNodeKillOne(t *testing.T) {
 	procAttr := new(os.ProcAttr)
 	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
 	clusterSize := 5
 	argGroup, etcds, err := CreateCluster(clusterSize, procAttr, false)
 	if err != nil {
 		t.Fatal("cannot create cluster")
 	}
 	defer DestroyCluster(etcds)
 	time.Sleep(2 * time.Second)
 	c := etcd.NewClient(nil)
 	c.SyncCluster()
 	stop := make(chan bool)
 	// Test Set
 	go Set(stop)
 	for i := 0; i < 10; i++ {
 		num := rand.Int() % clusterSize
 		fmt.Println("kill node", num+1)
 		// kill
 		etcds[num].Kill()
 		etcds[num].Release()
 		time.Sleep(time.Second)
 		// restart
 		etcds[num], err = os.StartProcess(EtcdBinPath, argGroup[num], procAttr)
 		if err != nil {
 			panic(err)
 		}
 		time.Sleep(time.Second)
 	}
 	fmt.Println("stop")
 	stop <- true
 	<-stop
 }
--- a/tests/functional/remove_node_test.go
+++ b/tests/functional/remove_node_test.go
@ -0,0 +1,113 @@
 package test
 import (
 	"net/http"
 	"os"
 	"testing"
 	"time"
 	"github.com/coreos/go-etcd/etcd"
 )
 // remove the node and node rejoin with previous log
 func TestRemoveNode(t *testing.T) {
 	procAttr := new(os.ProcAttr)
 	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
 	clusterSize := 3
 	argGroup, etcds, _ := CreateCluster(clusterSize, procAttr, false)
 	defer DestroyCluster(etcds)
 	time.Sleep(time.Second)
 	c := etcd.NewClient(nil)
 	c.SyncCluster()
 	rmReq, _ := http.NewRequest("DELETE", "http://127.0.0.1:7001/remove/node3", nil)
 	client := &http.Client{}
 	for i := 0; i < 2; i++ {
 		for i := 0; i < 2; i++ {
 			client.Do(rmReq)
 			etcds[2].Wait()
 			resp, err := c.Get("_etcd/machines")
 			if err != nil {
 				panic(err)
 			}
 			if len(resp) != 2 {
 				t.Fatal("cannot remove machine")
 			}
 			if i == 1 {
 				// rejoin with log
 				etcds[2], err = os.StartProcess(EtcdBinPath, argGroup[2], procAttr)
 			} else {
 				// rejoin without log
 				etcds[2], err = os.StartProcess(EtcdBinPath, append(argGroup[2], "-f"), procAttr)
 			}
 			if err != nil {
 				panic(err)
 			}
 			time.Sleep(time.Second)
 			resp, err = c.Get("_etcd/machines")
 			if err != nil {
 				panic(err)
 			}
 			if len(resp) != 3 {
 				t.Fatalf("add machine fails #1 (%d != 3)", len(resp))
 			}
 		}
 		// first kill the node, then remove it, then add it back
 		for i := 0; i < 2; i++ {
 			etcds[2].Kill()
 			etcds[2].Wait()
 			client.Do(rmReq)
 			resp, err := c.Get("_etcd/machines")
 			if err != nil {
 				panic(err)
 			}
 			if len(resp) != 2 {
 				t.Fatal("cannot remove machine")
 			}
 			if i == 1 {
 				// rejoin with log
 				etcds[2], err = os.StartProcess(EtcdBinPath, append(argGroup[2]), procAttr)
 			} else {
 				// rejoin without log
 				etcds[2], err = os.StartProcess(EtcdBinPath, append(argGroup[2], "-f"), procAttr)
 			}
 			if err != nil {
 				panic(err)
 			}
 			time.Sleep(time.Second)
 			resp, err = c.Get("_etcd/machines")
 			if err != nil {
 				panic(err)
 			}
 			if len(resp) != 3 {
 				t.Fatalf("add machine fails #2 (%d != 3)", len(resp))
 			}
 		}
 	}
 }
--- a/tests/functional/simple_multi_node_test.go
+++ b/tests/functional/simple_multi_node_test.go
@ -0,0 +1,62 @@
 package test
 import (
 	"os"
 	"testing"
 	"time"
 	"github.com/coreos/go-etcd/etcd"
 )
 func TestSimpleMultiNode(t *testing.T) {
 	templateTestSimpleMultiNode(t, false)
 }
 func TestSimpleMultiNodeTls(t *testing.T) {
 	templateTestSimpleMultiNode(t, true)
 }
 // Create a three nodes and try to set value
 func templateTestSimpleMultiNode(t *testing.T, tls bool) {
 	procAttr := new(os.ProcAttr)
 	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
 	clusterSize := 3
 	_, etcds, err := CreateCluster(clusterSize, procAttr, tls)
 	if err != nil {
 		t.Fatal("cannot create cluster")
 	}
 	defer DestroyCluster(etcds)
 	time.Sleep(time.Second)
 	c := etcd.NewClient(nil)
 	c.SyncCluster()
 	// Test Set
 	result, err := c.Set("foo", "bar", 100)
 	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.TTL < 95 {
 		if err != nil {
 			t.Fatal(err)
 		}
 		t.Fatalf("Set 1 failed with %s %s %v", result.Key, result.Value, result.TTL)
 	}
 	time.Sleep(time.Second)
 	result, err = c.Set("foo", "bar", 100)
 	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.PrevValue != "bar" || result.TTL != 100 {
 		if err != nil {
 			t.Fatal(err)
 		}
 		t.Fatalf("Set 2 failed with %s %s %v", result.Key, result.Value, result.TTL)
 	}
 }
--- a/tests/functional/single_node_recovery_test.go
+++ b/tests/functional/single_node_recovery_test.go
@ -0,0 +1,68 @@
 package test
 import (
 	"os"
 	"testing"
 	"time"
 	"github.com/coreos/go-etcd/etcd"
 )
 // This test creates a single node and then set a value to it.
 // Then this test kills the node and restart it and tries to get the value again.
 func TestSingleNodeRecovery(t *testing.T) {
 	procAttr := new(os.ProcAttr)
 	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
 	args := []string{"etcd", "-n=node1", "-d=/tmp/node1"}
 	process, err := os.StartProcess(EtcdBinPath, append(args, "-f"), procAttr)
 	if err != nil {
 		t.Fatal("start process failed:" + err.Error())
 		return
 	}
 	time.Sleep(time.Second)
 	c := etcd.NewClient(nil)
 	c.SyncCluster()
 	// Test Set
 	result, err := c.Set("foo", "bar", 100)
 	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.TTL < 95 {
 		if err != nil {
 			t.Fatal(err)
 		}
 		t.Fatalf("Set 1 failed with %s %s %v", result.Key, result.Value, result.TTL)
 	}
 	time.Sleep(time.Second)
 	process.Kill()
 	process, err = os.StartProcess(EtcdBinPath, args, procAttr)
 	defer process.Kill()
 	if err != nil {
 		t.Fatal("start process failed:" + err.Error())
 		return
 	}
 	time.Sleep(time.Second)
 	results, err := c.Get("foo")
 	if err != nil {
 		t.Fatal("get fail: " + err.Error())
 		return
 	}
 	result = results[0]
 	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.TTL > 99 {
 		if err != nil {
 			t.Fatal(err)
 		}
 		t.Fatalf("Recovery Get failed with %s %s %v", result.Key, result.Value, result.TTL)
 	}
 }
--- a/tests/functional/single_node_test.go
+++ b/tests/functional/single_node_test.go
@ -0,0 +1,77 @@
 package test
 import (
 	"os"
 	"testing"
 	"time"
 	"github.com/coreos/go-etcd/etcd"
 )
 // Create a single node and try to set value
 func TestSingleNode(t *testing.T) {
 	procAttr := new(os.ProcAttr)
 	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
 	args := []string{"etcd", "-n=node1", "-f", "-d=/tmp/node1"}
 	process, err := os.StartProcess(EtcdBinPath, args, procAttr)
 	if err != nil {
 		t.Fatal("start process failed:" + err.Error())
 		return
 	}
 	defer process.Kill()
 	time.Sleep(time.Second)
 	c := etcd.NewClient(nil)
 	c.SyncCluster()
 	// Test Set
 	result, err := c.Set("foo", "bar", 100)
 	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.TTL < 95 {
 		if err != nil {
 			t.Fatal("Set 1: ", err)
 		}
 		t.Fatalf("Set 1 failed with %s %s %v", result.Key, result.Value, result.TTL)
 	}
 	time.Sleep(time.Second)
 	result, err = c.Set("foo", "bar", 100)
 	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.PrevValue != "bar" || result.TTL != 100 {
 		if err != nil {
 			t.Fatal("Set 2: ", err)
 		}
 		t.Fatalf("Set 2 failed with %s %s %v", result.Key, result.Value, result.TTL)
 	}
 	// Add a test-and-set test
 	// First, we'll test we can change the value if we get it write
 	result, match, err := c.TestAndSet("foo", "bar", "foobar", 100)
 	if err != nil || result.Key != "/foo" || result.Value != "foobar" || result.PrevValue != "bar" || result.TTL != 100 || !match {
 		if err != nil {
 			t.Fatal(err)
 		}
 		t.Fatalf("Set 3 failed with %s %s %v", result.Key, result.Value, result.TTL)
 	}
 	// Next, we'll make sure we can't set it without the correct prior value
 	_, _, err = c.TestAndSet("foo", "bar", "foofoo", 100)
 	if err == nil {
 		t.Fatalf("Set 4 expecting error when setting key with incorrect previous value")
 	}
 	// Finally, we'll make sure a blank previous value still counts as a test-and-set and still has to match
 	_, _, err = c.TestAndSet("foo", "", "barbar", 100)
 	if err == nil {
 		t.Fatalf("Set 5 expecting error when setting key with blank (incorrect) previous value")
 	}
 }
--- a/tests/functional/util.go
+++ b/tests/functional/util.go
@ -1,4 +1,4 @@
-package main
+package test
 import (
 	"fmt"
@ -18,11 +18,11 @@ var client = http.Client{
 }
 // Sending set commands
-func set(stop chan bool) {
+func Set(stop chan bool) {
 	stopSet := false
 	i := 0
-	c := etcd.NewClient()
+	c := etcd.NewClient(nil)
 	for {
 		key := fmt.Sprintf("%s_%v", "foo", i)
@ -50,12 +50,11 @@ func set(stop chan bool) {
 		i++
 	}
 	fmt.Println("set stop")
 	stop <- true
 }
 // Create a cluster of etcd nodes
-func createCluster(size int, procAttr *os.ProcAttr, ssl bool) ([][]string, []*os.Process, error) {
+func CreateCluster(size int, procAttr *os.ProcAttr, ssl bool) ([][]string, []*os.Process, error) {
 	argGroup := make([][]string, size)
 	sslServer1 := []string{"-serverCAFile=./fixtures/ca/ca.crt",
@ -70,7 +69,7 @@ func createCluster(size int, procAttr *os.ProcAttr, ssl bool) ([][]string, []*os
 	for i := 0; i < size; i++ {
 		if i == 0 {
-			argGroup[i] = []string{"etcd", "-d=/tmp/node1", "-n=node1", "-vv"}
+			argGroup[i] = []string{"etcd", "-d=/tmp/node1", "-n=node1"}
 			if ssl {
 				argGroup[i] = append(argGroup[i], sslServer1...)
 			}
@ -87,7 +86,7 @@ func createCluster(size int, procAttr *os.ProcAttr, ssl bool) ([][]string, []*os
 	for i, _ := range etcds {
 		var err error
-		etcds[i], err = os.StartProcess("etcd", append(argGroup[i], "-f"), procAttr)
+		etcds[i], err = os.StartProcess(EtcdBinPath, append(argGroup[i], "-f"), procAttr)
 		if err != nil {
 			return nil, nil, err
 		}
@ -97,7 +96,7 @@ func createCluster(size int, procAttr *os.ProcAttr, ssl bool) ([][]string, []*os
 		// have to retry. This retry can take upwards of 15 seconds
 		// which slows tests way down and some of them fail.
 		if i == 0 {
-			time.Sleep(time.Second)
+			time.Sleep(time.Second * 2)
 		}
 	}
@ -105,10 +104,9 @@ func createCluster(size int, procAttr *os.ProcAttr, ssl bool) ([][]string, []*os
 }
 // Destroy all the nodes in the cluster
-func destroyCluster(etcds []*os.Process) error {
+func DestroyCluster(etcds []*os.Process) error {
-	for i, etcd := range etcds {
+	for _, etcd := range etcds {
 		err := etcd.Kill()
 		fmt.Println("kill ", i)
 		if err != nil {
 			panic(err.Error())
 		}
@ -118,7 +116,7 @@ func destroyCluster(etcds []*os.Process) error {
 }
 //
-func leaderMonitor(size int, allowDeadNum int, leaderChan chan string) {
+func Monitor(size int, allowDeadNum int, leaderChan chan string, all chan bool, stop chan bool) {
 	leaderMap := make(map[int]string)
 	baseAddrFormat := "http://0.0.0.0:400%d"
@ -153,6 +151,8 @@ func leaderMonitor(size int, allowDeadNum int, leaderChan chan string) {
 		if i == size {
 			select {
 			case <-stop:
 				return
 			case <-leaderChan:
 				leaderChan <- knownLeader
 			default:
@ -160,6 +160,14 @@ func leaderMonitor(size int, allowDeadNum int, leaderChan chan string) {
 			}
 		}
 		if dead == 0 {
 			select {
 			case <-all:
 				all <- true
 			default:
 				all <- true
 			}
 		}
 		time.Sleep(time.Millisecond * 10)
 	}
@ -168,7 +176,7 @@ func leaderMonitor(size int, allowDeadNum int, leaderChan chan string) {
 func getLeader(addr string) (string, error) {
-	resp, err := client.Get(addr + "/leader")
+	resp, err := client.Get(addr + "/v1/leader")
 	if err != nil {
 		return "", err
@ -191,28 +199,6 @@ func getLeader(addr string) (string, error) {
 }
 func directSet() {
 	c := make(chan bool, 1000)
 	for i := 0; i < 1000; i++ {
 		go send(c)
 	}
 	for i := 0; i < 1000; i++ {
 		<-c
 	}
 }
 func send(c chan bool) {
 	for i := 0; i < 10; i++ {
 		command := &SetCommand{}
 		command.Key = "foo"
 		command.Value = "bar"
 		command.ExpireTime = time.Unix(0, 0)
 		raftServer.Do(command)
 	}
 	c <- true
 }
 // Dial with timeout
 func dialTimeoutFast(network, addr string) (net.Conn, error) {
 	return net.DialTimeout(network, addr, time.Millisecond*10)
--- a/third_party/bitbucket.org/kardianos/osext/osext_sysctl.go
+++ b/third_party/bitbucket.org/kardianos/osext/osext_sysctl.go
@ -8,6 +8,7 @@ package osext
 import (
 	"os"
 	"path/filepath"
 	"runtime"
 	"syscall"
 	"unsafe"
@ -47,18 +48,35 @@ func executable() (string, error) {
 			break
 		}
 	}
 	var strpath string
 	if buf[0] != '/' {
-		if getwdError != nil {
+		var e error
-			return string(buf), getwdError
+		if strpath, e = getAbs(buf); e != nil {
-		} else {
+			return strpath, e
-			if buf[0] == '.' {
+		}
-				buf = buf[1:]
+	} else {
-			}
+		strpath = string(buf)
-			if startUpcwd[len(startUpcwd)-1] != '/' {
+	}
-				return startUpcwd + "/" + string(buf), nil
+	// darwin KERN_PROCARGS may return the path to a symlink rather than the
-			}
+	// actual executable
-			return startUpcwd + string(buf), nil
+	if runtime.GOOS == "darwin" {
 		if strpath, err := filepath.EvalSymlinks(strpath); err != nil {
 			return strpath, err
 		}
 	}
-	return string(buf), nil
+	return strpath, nil
 }
 func getAbs(buf []byte) (string, error) {
 	if getwdError != nil {
 		return string(buf), getwdError
 	} else {
 		if buf[0] == '.' {
 			buf = buf[1:]
 		}
 		if startUpcwd[len(startUpcwd)-1] != '/' && buf[0] != '/' {
 			return startUpcwd + "/" + string(buf), nil
 		}
 		return startUpcwd + string(buf), nil
 	}
 }
--- a/third_party/code.google.com/p/go.net/ipv4/gen.go
+++ b/third_party/code.google.com/p/go.net/ipv4/gen.go
@ -97,20 +97,16 @@ func parseICMPv4Parameters(w io.Writer, r io.Reader) error {
 }
 type icmpv4Parameters struct {
-	XMLName    xml.Name              `xml:"registry"`
+	XMLName    xml.Name `xml:"registry"`
-	Title      string                `xml:"title"`
+	Title      string   `xml:"title"`
-	Updated    string                `xml:"updated"`
+	Updated    string   `xml:"updated"`
-	Registries []icmpv4ParamRegistry `xml:"registry"`
+	Registries []struct {
-}
+		Title   string `xml:"title"`
-
+		Records []struct {
-type icmpv4ParamRegistry struct {
+			Value string `xml:"value"`
-	Title   string              `xml:"title"`
+			Descr string `xml:"description"`
-	Records []icmpv4ParamRecord `xml:"record"`
+		} `xml:"record"`
-}
+	} `xml:"registry"`
 type icmpv4ParamRecord struct {
 	Value string `xml:"value"`
 	Descr string `xml:"description"`
 }
 type canonICMPv4ParamRecord struct {
@ -193,18 +189,16 @@ func parseProtocolNumbers(w io.Writer, r io.Reader) error {
 }
 type protocolNumbers struct {
-	XMLName  xml.Name         `xml:"registry"`
+	XMLName  xml.Name `xml:"registry"`
-	Title    string           `xml:"title"`
+	Title    string   `xml:"title"`
-	Updated  string           `xml:"updated"`
+	Updated  string   `xml:"updated"`
-	RegTitle string           `xml:"registry>title"`
+	RegTitle string   `xml:"registry>title"`
-	Note     string           `xml:"registry>note"`
+	Note     string   `xml:"registry>note"`
-	Records  []protocolRecord `xml:"registry>record"`
+	Records  []struct {
-}
+		Value string `xml:"value"`
-
+		Name  string `xml:"name"`
-type protocolRecord struct {
+		Descr string `xml:"description"`
-	Value string `xml:"value"`
+	} `xml:"registry>record"`
 	Name  string `xml:"name"`
 	Descr string `xml:"description"`
 }
 type canonProtocolRecord struct {
--- a/third_party/code.google.com/p/go.net/ipv4/gentest.go
+++ b/third_party/code.google.com/p/go.net/ipv4/gentest.go
@ -39,7 +39,7 @@ var registries = []struct {
 func main() {
 	var bb bytes.Buffer
-	fmt.Fprintf(&bb, "// go run gentv.go\n")
+	fmt.Fprintf(&bb, "// go run gentest.go\n")
 	fmt.Fprintf(&bb, "// GENERATED BY THE COMMAND ABOVE; DO NOT EDIT\n\n")
 	fmt.Fprintf(&bb, "package ipv4_test\n\n")
 	for _, r := range registries {
@ -85,18 +85,19 @@ func parseDSCPRegistry(w io.Writer, r io.Reader) error {
 }
 type dscpRegistry struct {
-	XMLName     xml.Name     `xml:"registry"`
+	XMLName     xml.Name `xml:"registry"`
-	Title       string       `xml:"title"`
+	Title       string   `xml:"title"`
-	Updated     string       `xml:"updated"`
+	Updated     string   `xml:"updated"`
-	Note        string       `xml:"note"`
+	Note        string   `xml:"note"`
-	RegTitle    string       `xml:"registry>title"`
+	RegTitle    string   `xml:"registry>title"`
-	PoolRecords []dscpRecord `xml:"registry>record"`
+	PoolRecords []struct {
-	Records     []dscpRecord `xml:"registry>registry>record"`
+		Name  string `xml:"name"`
-}
+		Space string `xml:"space"`
-
+	} `xml:"registry>record"`
-type dscpRecord struct {
+	Records []struct {
-	Name  string `xml:"name"`
+		Name  string `xml:"name"`
-	Space string `xml:"space"`
+		Space string `xml:"space"`
 	} `xml:"registry>registry>record"`
 }
 type canonDSCPRecord struct {
@ -145,17 +146,15 @@ func parseTOSTCByte(w io.Writer, r io.Reader) error {
 }
 type tosTCByte struct {
-	XMLName  xml.Name          `xml:"registry"`
+	XMLName  xml.Name `xml:"registry"`
-	Title    string            `xml:"title"`
+	Title    string   `xml:"title"`
-	Updated  string            `xml:"updated"`
+	Updated  string   `xml:"updated"`
-	Note     string            `xml:"note"`
+	Note     string   `xml:"note"`
-	RegTitle string            `xml:"registry>title"`
+	RegTitle string   `xml:"registry>title"`
-	Records  []tosTCByteRecord `xml:"registry>record"`
+	Records  []struct {
-}
+		Binary  string `xml:"binary"`
-
+		Keyword string `xml:"keyword"`
-type tosTCByteRecord struct {
+	} `xml:"registry>record"`
 	Binary  string `xml:"binary"`
 	Keyword string `xml:"keyword"`
 }
 type canonTOSTCByteRecord struct {
--- a/third_party/code.google.com/p/go.net/ipv4/header.go
+++ b/third_party/code.google.com/p/go.net/ipv4/header.go
@ -36,41 +36,47 @@ const (
 	maxHeaderLen = 60 // sensible default, revisit if later RFCs define new usage of version and header length fields
 )
-type headerField int
+const (
 	posTOS      = 1  // type-of-service
 	posTotalLen = 2  // packet total length
 	posID       = 4  // identification
 	posFragOff  = 6  // fragment offset
 	posTTL      = 8  // time-to-live
 	posProtocol = 9  // next protocol
 	posChecksum = 10 // checksum
 	posSrc      = 12 // source address
 	posDst      = 16 // destination address
 )
 type HeaderFlags int
 const (
-	posTOS      headerField = 1  // type-of-service
+	MoreFragments HeaderFlags = 1 << iota // more fragments flag
-	posTotalLen             = 2  // packet total length
+	DontFragment                          // don't fragment flag
 	posID                   = 4  // identification
 	posFragOff              = 6  // fragment offset
 	posTTL                  = 8  // time-to-live
 	posProtocol             = 9  // next protocol
 	posChecksum             = 10 // checksum
 	posSrc                  = 12 // source address
 	posDst                  = 16 // destination address
 )
 // A Header represents an IPv4 header.
 type Header struct {
-	Version  int    // protocol version
+	Version  int         // protocol version
-	Len      int    // header length
+	Len      int         // header length
-	TOS      int    // type-of-service
+	TOS      int         // type-of-service
-	TotalLen int    // packet total length
+	TotalLen int         // packet total length
-	ID       int    // identification
+	ID       int         // identification
-	FragOff  int    // fragment offset
+	Flags    HeaderFlags // flags
-	TTL      int    // time-to-live
+	FragOff  int         // fragment offset
-	Protocol int    // next protocol
+	TTL      int         // time-to-live
-	Checksum int    // checksum
+	Protocol int         // next protocol
-	Src      net.IP // source address
+	Checksum int         // checksum
-	Dst      net.IP // destination address
+	Src      net.IP      // source address
-	Options  []byte // options, extension headers
+	Dst      net.IP      // destination address
 	Options  []byte      // options, extension headers
 }
 func (h *Header) String() string {
 	if h == nil {
 		return "<nil>"
 	}
-	return fmt.Sprintf("ver: %v, hdrlen: %v, tos: %#x, totallen: %v, id: %#x, fragoff: %#x, ttl: %v, proto: %v, cksum: %#x, src: %v, dst: %v", h.Version, h.Len, h.TOS, h.TotalLen, h.ID, h.FragOff, h.TTL, h.Protocol, h.Checksum, h.Src, h.Dst)
+	return fmt.Sprintf("ver: %v, hdrlen: %v, tos: %#x, totallen: %v, id: %#x, flags: %#x, fragoff: %#x, ttl: %v, proto: %v, cksum: %#x, src: %v, dst: %v", h.Version, h.Len, h.TOS, h.TotalLen, h.ID, h.Flags, h.FragOff, h.TTL, h.Protocol, h.Checksum, h.Src, h.Dst)
 }
 // Please refer to the online manual; IP(4) on Darwin, FreeBSD and
@ -89,12 +95,13 @@ func (h *Header) Marshal() ([]byte, error) {
 	b := make([]byte, hdrlen)
 	b[0] = byte(Version<<4 | (hdrlen >> 2 & 0x0f))
 	b[posTOS] = byte(h.TOS)
 	flagsAndFragOff := (h.FragOff & 0x1fff) | int(h.Flags<<13)
 	if supportsNewIPInput {
 		b[posTotalLen], b[posTotalLen+1] = byte(h.TotalLen>>8), byte(h.TotalLen)
-		b[posFragOff], b[posFragOff+1] = byte(h.FragOff>>8), byte(h.FragOff)
+		b[posFragOff], b[posFragOff+1] = byte(flagsAndFragOff>>8), byte(flagsAndFragOff)
 	} else {
 		*(*uint16)(unsafe.Pointer(&b[posTotalLen : posTotalLen+1][0])) = uint16(h.TotalLen)
-		*(*uint16)(unsafe.Pointer(&b[posFragOff : posFragOff+1][0])) = uint16(h.FragOff)
+		*(*uint16)(unsafe.Pointer(&b[posFragOff : posFragOff+1][0])) = uint16(flagsAndFragOff)
 	}
 	b[posID], b[posID+1] = byte(h.ID>>8), byte(h.ID)
 	b[posTTL] = byte(h.TTL)
@ -135,6 +142,8 @@ func ParseHeader(b []byte) (*Header, error) {
 		h.TotalLen += hdrlen
 		h.FragOff = int(*(*uint16)(unsafe.Pointer(&b[posFragOff : posFragOff+1][0])))
 	}
 	h.Flags = HeaderFlags(h.FragOff&0xe000) >> 13
 	h.FragOff = h.FragOff & 0x1fff
 	h.ID = int(b[posID])<<8 | int(b[posID+1])
 	h.TTL = int(b[posTTL])
 	h.Protocol = int(b[posProtocol])
--- a/third_party/code.google.com/p/go.net/ipv4/header_test.go
+++ b/third_party/code.google.com/p/go.net/ipv4/header_test.go
@ -16,28 +16,28 @@ import (
 var (
 	wireHeaderFromKernel = [ipv4.HeaderLen]byte{
 		0x45, 0x01, 0xbe, 0xef,
-		0xca, 0xfe, 0x05, 0xdc,
+		0xca, 0xfe, 0x45, 0xdc,
 		0xff, 0x01, 0xde, 0xad,
 		172, 16, 254, 254,
 		192, 168, 0, 1,
 	}
 	wireHeaderToKernel = [ipv4.HeaderLen]byte{
 		0x45, 0x01, 0xbe, 0xef,
-		0xca, 0xfe, 0x05, 0xdc,
+		0xca, 0xfe, 0x45, 0xdc,
 		0xff, 0x01, 0xde, 0xad,
 		172, 16, 254, 254,
 		192, 168, 0, 1,
 	}
 	wireHeaderFromTradBSDKernel = [ipv4.HeaderLen]byte{
 		0x45, 0x01, 0xdb, 0xbe,
-		0xca, 0xfe, 0xdc, 0x05,
+		0xca, 0xfe, 0xdc, 0x45,
 		0xff, 0x01, 0xde, 0xad,
 		172, 16, 254, 254,
 		192, 168, 0, 1,
 	}
 	wireHeaderToTradBSDKernel = [ipv4.HeaderLen]byte{
 		0x45, 0x01, 0xef, 0xbe,
-		0xca, 0xfe, 0xdc, 0x05,
+		0xca, 0xfe, 0xdc, 0x45,
 		0xff, 0x01, 0xde, 0xad,
 		172, 16, 254, 254,
 		192, 168, 0, 1,
@ -51,6 +51,7 @@ var (
 		TOS:      1,
 		TotalLen: 0xbeef,
 		ID:       0xcafe,
 		Flags:    ipv4.DontFragment,
 		FragOff:  1500,
 		TTL:      255,
 		Protocol: 1,
--- a/third_party/code.google.com/p/go.net/ipv4/iana_test.go
+++ b/third_party/code.google.com/p/go.net/ipv4/iana_test.go
@ -1,9 +1,9 @@
-// go run gentv.go
+// go run gentest.go
 // GENERATED BY THE COMMAND ABOVE; DO NOT EDIT
 package ipv4_test
-// Differentiated Services Field Codepoints, Updated: 2010-05-11
+// Differentiated Services Field Codepoints (DSCP), Updated: 2013-06-25
 const (
 	DiffServCS0        = 0x0  // CS0
 	DiffServCS1        = 0x20 // CS1
--- a/Show More
+++ b/Show More
		`@ -0,0 +1,3 @@`
							`// +build !go1.1`

							`"etcd requires go 1.1 or greater to build"`