Merge pull request #248 from benbjohnson/build-fail

Update build script to fail on error.
2013-10-17 09:27:22 -07:00 · 2013-10-17 10:02:59 -06:00 · 2013-10-16 18:37:39 -07:00 · 2013-10-16 19:36:38 -06:00 · 2013-10-16 17:33:35 -07:00 · 2013-10-16 17:29:53 -07:00
235 changed files with 12630 additions and 6678 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,4 +1,5 @@
 src/
 pkg/
-./etcd
-release_version.go
+/etcd
+/server/release_version.go
+/machine*
--- a/.travis.yml
+++ b/.travis.yml
@ -5,4 +5,4 @@ install:
 - echo "Skip install"

 script:
- - ./test
+ - ./test.sh
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -0,0 +1,75 @@
+# How to contribute
+
+etcd is open source, Apache 2.0 licensed and accepts contributions via Github pull requests.
+This document outlines some of the conventions on commit message formatting, contact points for developers and other resources to make getting your contribution into etcd easier.
+
+# Email and chat
+
+For simplicity etcd discussions happen on coreos-dev and in #coreos-dev.
+As the community grows we will move to a dedicated mailing list and IRC channel.
+
+- Email: [coreos-dev](https://groups.google.com/forum/#!forum/coreos-dev) 
+- IRC: #[coreos](irc://irc.freenode.org:6667/#coreos) IRC channel on freenode.org
+
+## Getting Started
+
+- Fork the repository on GitHub
+- Read the README.md for build instructions
+
+## Contribution flow
+
+This is a rough outline of what a contributor's workflow looks like:
+
+- Create a topic branch from where you want to base your work. This is usually master.
+- Make commits of logical units.
+- Make sure your commit messages are in the proper format, see below
+- Push your changes to a topic branch in your fork of the repository.
+- Submit a pull request to coreos/etcd
+
+Thanks for you contributions!
+
+### Format of the commit message
+
+etcd follow a rough convention for commit messages borrowed from Angularjs.
+This is an example of a commit:
+
+```
+    feat(scripts/test-cluster): add a cluster test command
+
+    this uses tmux to setup a test cluster that you can easily kill and
+    start for debugging.
+```
+
+To make it more formal it looks something like this:
+
+```
+<type>(<scope>): <subject>
+<BLANK LINE>
+<body>
+<BLANK LINE>
+<footer>
+```
+
+The first line is the subject and should not be longer than 70 characters, the second line is always blank and other lines should be wrapped at 80 characters.
+This allows the message to be easier to read on github as well as in various git tools.
+
+### Subject line
+
+The subject line contains succinct description of the change.
+
+### Allowed <type>
+feat (feature)
+fix (bug fix)
+docs (documentation)
+style (formatting, missing semi colons, …)
+refactor
+test (when adding missing tests)
+chore (maintain)
+
+### Allowed <scope>
+
+Scopes could be anything specifying place of the commit change. For example store, api, etc.
+
+### More details on commits
+
+For more details see the [angularjs commit style guide](https://docs.google.com/a/coreos.com/document/d/1QrDFcIiPjSLDn3EL15IJygNPiHORgU1_OOAqWjiDU5Y/edit#).
--- a/10
+++ b/10
@ -0,0 +1,10 @@
+FROM ubuntu:12.04
+RUN apt-get update
+RUN apt-get install -y python-software-properties git
+RUN add-apt-repository -y ppa:duh/golang
+RUN apt-get update
+RUN apt-get install -y golang
+ADD . /opt/etcd
+RUN cd /opt/etcd && ./build
+EXPOSE 4001 7001
+ENTRYPOINT ["/opt/etcd/etcd", "-c", "0.0.0.0:4001", "-s", "0.0.0.0:7001"]
--- a/Documentation/errorcode.md
+++ b/Documentation/errorcode.md
@ -0,0 +1,58 @@
+Error Code
+======
+
+This document describes the error code in **Etcd** project.
+
+It's categorized into four groups:
+
+- Command Related Error
+- Post Form Related Error
+- Raft Related Error
+- Etcd Related Error
+
+Error code corresponding strerror
+------
+
+    const (
+        EcodeKeyNotFound    = 100
+        EcodeTestFailed     = 101
+        EcodeNotFile        = 102
+        EcodeNoMoreMachine  = 103
+        EcodeNotDir         = 104
+        EcodeNodeExist      = 105
+        EcodeKeyIsPreserved = 106
+
+        EcodeValueRequired     = 200
+        EcodePrevValueRequired = 201
+        EcodeTTLNaN            = 202
+        EcodeIndexNaN          = 203
+
+        EcodeRaftInternal = 300
+        EcodeLeaderElect  = 301
+
+        EcodeWatcherCleared = 400
+        EcodeEventIndexCleared = 401
+    )
+
+    // command related errors
+    errors[100] = "Key Not Found"
+    errors[101] = "Test Failed" //test and set
+    errors[102] = "Not A File"
+    errors[103] = "Reached the max number of machines in the cluster"
+    errors[104] = "Not A Directory"
+    errors[105] = "Already exists" // create
+    errors[106] = "The prefix of given key is a keyword in etcd"
+
+    // Post form related errors
+    errors[200] = "Value is Required in POST form"
+    errors[201] = "PrevValue is Required in POST form"
+    errors[202] = "The given TTL in POST form is not a number"
+    errors[203] = "The given index in POST form is not a number"
+
+    // raft related errors
+    errors[300] = "Raft Internal Error"
+    errors[301] = "During Leader Election"
+
+    // etcd related errors
+    errors[400] = "watcher is cleared due to etcd recovery"
+    errors[401] = "The event in requested index is outdated and cleared"
--- a/Documentation/etcd-file-system.md
+++ b/Documentation/etcd-file-system.md
@ -0,0 +1,101 @@
+#Etcd File System
+
+## Structure
+[TODO]
+![alt text](./img/etcd_fs_structure.jpg "etcd file system structure")
+
+## Node
+In **Etcd**, the **Node** is the rudimentary element constructing the whole.
+Currently **Etcd** file system is comprised in a Unix-like way of files and directories, and they are two kinds of nodes different in:
+
+- **File Node** has data associated with it.
+- **Directory Node** has children nodes associated with it.
+
+Besides the file and directory difference, all nodes have common attributes and operations as follows:
+
+### Attributes:
+- **Expiration Time** [optional]
+
+  The node will be deleted when it expires.
+
+- **ACL**
+
+  The path of access control list of the node.
+
+### Operation:
+- **Get** (path, recursive, sorted)
+
+  Get the content of the node
+    - If the node is a file, the data of the file will be returned.
+    - If the node is a directory, the child nodes of the directory will be returned.
+    - If recursive is true, it will recursively get the nodes of the directory.
+    - If sorted is true, the result will be sorted based on the path.
+
+- **Create** (path, value[optional], ttl [optional])
+
+  Create a file. Create operation will help to create intermediate directories with no expiration time.
+    - If the file already exists, create will fail.
+    - If the value is given, set will create a file.
+    - If the value is not given, set will crate a directory.
+    - If ttl is given, the node will be deleted when it expires.
+
+- **Update** (path, value[optional], ttl [optional])
+
+  Update the content of the node.
+    - If the value is given, the value of the key will be updated.
+    - If ttl is given, the expiration time of the node will be updated.
+
+- **Delete** (path, recursive)
+
+  Delete the node of given path.
+    - If the node is a directory:
+    - If recursive is true, the operation will delete all nodes under the directory.
+    - If recursive is false, error will be returned.
+
+- **TestAndSet** (path, prevValue [prevIndex], value, ttl)
+
+  Atomic *test and set* value to a file. If test succeeds, this operation will change the previous value of the file to the given value.
+    - If the prevValue is given, it will test against previous value of 
+    the node.
+    - If the prevValue is empty, it will test if the node is not existing.
+    - If the prevValue is not empty, it will test if the prevValue is equal to the current value of the file.
+    - If the prevIndex is given, it will test if the create/last modified index of the node is equal to prevIndex.
+
+- **Renew** (path, ttl)
+
+  Set the node's expiration time to (current time + ttl)
+
+## ACL
+
+### Theory
+Etcd exports a Unix-like file system interface consisting of files and directories, collectively called nodes.
+Each node has various meta-data, including three names of access control lists used to control reading, writing and changing (change ACL names for the node).
+
+We are storing the ACL names for nodes under a special *ACL* directory.
+Each node has ACL name corresponding to one file within *ACL* dir.
+Unless overridden, a node naturally inherits the ACL names of its parent directory on creation.
+
+For each ACL name, it has three children: *R (Reading)*, *W (Writing)*, *C (Changing)*
+
+Each permission is also a node. Under the node it contains the users who have this permission for the file refering to this ACL name.
+
+### Example
+[TODO]
+### Diagram
+[TODO]
+
+### Interface
+
+Testing permissions:
+
+- (node *Node) get_perm()
+- (node *Node) has_perm(perm string, user string)
+
+Setting/Changing permissions:
+
+- (node *Node) set_perm(perm string)
+- (node *Node) change_ACLname(aclname string)
+
+
+## User Group
+[TODO]
--- a/Documentation/img/etcd_fs_structure.jpg
+++ b/Documentation/img/etcd_fs_structure.jpg
--- a/Documentation/internal-protocol-versioning.md
+++ b/Documentation/internal-protocol-versioning.md
@ -0,0 +1,61 @@
+# Versioning
+
+Goal: We want to be able to upgrade an individual machine in an etcd cluster to a newer version of etcd.
+The process will take the form of individual followers upgrading to the latest version until the entire cluster is on the new version.
+
+Immediate need: etcd is moving too fast to version the internal API right now.
+But, we need to keep mixed version clusters from being started by a rollowing upgrade process (e.g. the CoreOS developer alpha).
+
+Longer term need: Having a mixed version cluster where all machines are not be running the exact same version of etcd itself but are able to speak one version of the internal protocol.
+
+Solution: The internal protocol needs to be versioned just as the client protocol is.
+Initially during the 0.\*.\* series of etcd releases we won't allow mixed versions at all.
+
+## Join Control
+
+We will add a version field to the join command.
+But, who decides whether a newly upgraded follower should be able to join a cluster?
+
+### Leader Controlled
+
+If the leader controls the version of followers joining the cluster then it compares its version to the version number presented by the follower in the JoinCommand and rejects the join if the number is less than the leader's version number.
+
+Advantages
+
+- Leader controls all cluster decisions still
+
+Disadvantages
+
+- Follower knows better what versions of the interal protocol it can talk than the leader
+
+
+### Follower Controlled
+
+A newly upgraded follower should be able to figure out the leaders internal version from a defined internal backwards compatible API endpoint and figure out if it can join the cluster.
+If it cannot join the cluster then it simply exits.
+
+Advantages
+
+- The follower is running newer code and knows better if it can talk older protocols
+
+Disadvantages
+
+- This cluster decision isn't made by the leader
+
+## Recommendation
+
+To solve the immediate need and to plan for the future lets do the following:
+
+- Add Version field to JoinCommand
+- Have a joining follower read the Version field of the leader and if its own version doesn't match the leader then sleep for some random interval and retry later to see if the leader has upgraded.
+
+# Research
+
+## Zookeeper versioning
+
+Zookeeper very recently added versioning into the protocol and it doesn't seem to have seen any use yet.
+https://issues.apache.org/jira/browse/ZOOKEEPER-1633
+
+## doozerd
+
+doozerd stores the version number of the machine in the datastore for other clients to check, no decisions are made off of this number currently.
--- a/README.md
+++ b/README.md
@ -1,4 +1,5 @@
 # etcd
+README version 0.1.0

 [![Build Status](https://travis-ci.org/coreos/etcd.png)](https://travis-ci.org/coreos/etcd)

@ -9,12 +10,12 @@ A highly-available key value store for shared configuration and service discover
 * Fast: benchmarked 1000s of writes/s per instance
 * Reliable: Properly distributed using Raft

-Etcd is written in Go and uses the [raft][raft] consensus algorithm to manage a highly availably replicated log.
+Etcd is written in Go and uses the [raft][raft] consensus algorithm to manage a highly-available replicated log.

-See [go-etcd][go-etcd] for a native Go client. Or feel free to just use curl, as in the examples below.
+See [etcdctl][etcdctl] for a simple command line client. Or feel free to just use curl, as in the examples below.

 [raft]: https://github.com/coreos/go-raft
-[go-etcd]: https://github.com/coreos/go-etcd
+[etcdctl]: http://coreos.com/docs/etcdctl/

 ## Getting Started

@ -24,22 +25,35 @@ The latest release is available as a binary at [Github][github-release].

 [github-release]: https://github.com/coreos/etcd/releases/

-You can also buildi etcd from source:
+### Building
+
+You can build etcd from source:

 ```sh
 git clone https://github.com/coreos/etcd
+cd etcd
 ./build
 ```

+This will generate a binary in the base directory called `./etcd`.
+
+_NOTE_: you need go 1.1+. Please check your installation with
+
+```
+go version
+```
+
 ### Running a single node

 These examples will use a single node cluster to show you the basics of the etcd REST API. Lets start etcd:

 ```sh
-./etcd -d node0
+./etcd -d node0 -n node0
 ```

-This will bring up an etcd node listening on port 4001 for client communication and on port 7001 for server-to-server communication. The `-d node0` argument tells etcd to write node configuration, logs and snapshots to the `./node0/` directory.
+This will bring up an etcd node listening on port 4001 for client communication and on port 7001 for server-to-server communication.
+The `-d node0` argument tells etcd to write node configuration, logs and snapshots to the `./node0/` directory.
+The `-n node0` tells the rest of the cluster that this node is named node0.

 ## Usage

@ -129,7 +143,7 @@ Now you can try to get the key by sending:
 curl -L http://127.0.0.1:4001/v1/keys/foo
 ```

-If the TTL has expired, the key will be deleted, and you will be returned a 404.
+If the TTL has expired, the key will be deleted, and you will be returned a 100.

 ```json
 {"errorCode":100,"message":"Key Not Found","cause":"/foo"}
@ -173,17 +187,17 @@ The watch command returns immediately with the same response as previous.

 Etcd can be used as a centralized coordination service in a cluster and `TestAndSet` is the most basic operation to build distributed lock service. This command will set the value only if the client provided `prevValue` is equal the current key value.

-Here is a simple example. Let's create a key-value pair first: `testAndSet=one`.
+Here is a simple example. Let's create a key-value pair first: `foo=one`.

 ```sh
-curl -L http://127.0.0.1:4001/v1/keys/testAndSet -d value=one
+curl -L http://127.0.0.1:4001/v1/keys/foo -d value=one
 ```

-Let's try an invaild `TestAndSet` command.
+Let's try an invalid `TestAndSet` command.
 We can give another parameter prevValue to set command to make it a TestAndSet command.

 ```sh
-curl -L http://127.0.0.1:4001/v1/keys/testAndSet -d prevValue=two -d value=three
+curl -L http://127.0.0.1:4001/v1/keys/foo -d prevValue=two -d value=three
 ```

 This will try to test if the previous of the key is two, it is change it to three.
@ -194,16 +208,16 @@ This will try to test if the previous of the key is two, it is change it to thre

 which means `testAndSet` failed.

-Let us try a vaild one.
+Let us try a valid one.

 ```sh
-curl -L http://127.0.0.1:4001/v1/keys/testAndSet -d prevValue=one -d value=two
+curl -L http://127.0.0.1:4001/v1/keys/foo -d prevValue=one -d value=two
 ```

 The response should be

 ```json
-{"action":"SET","key":"/testAndSet","prevValue":"one","value":"two","index":10}
+{"action":"SET","key":"/foo","prevValue":"one","value":"two","index":10}
 ```

 We successfully changed the value from “one” to “two”, since we give the correct previous value.
@ -243,10 +257,7 @@ which meas `foo=barbar` is a key-value pair under `/foo` and `foo_dir` is a dire
 Etcd supports SSL/TLS and client cert authentication for clients to server, as well as server to server communication

 First, you need to have a CA cert `clientCA.crt` and signed key pair `client.crt`, `client.key`. This site has a good reference for how to generate self-signed key pairs:
-
-```url
 http://www.g-loaded.eu/2005/11/10/be-your-own-ca/
-```

 For testing you can use the certificates in the `fixtures/ca` directory.

@ -262,7 +273,7 @@ Next, lets configure etcd to use this keypair:
 You can now test the configuration using https:

 ```sh
-curl --cacert fixtures/ca/ca.crt https://127.0.0.1:4001/v1/keys/foo -F value=bar
+curl --cacert fixtures/ca/ca.crt https://127.0.0.1:4001/v1/keys/foo -d value=bar -v
 ```

 You should be able to see the handshake succeed.
@ -292,7 +303,7 @@ We can also do authentication using CA certs. The clients will provide their cer
 Try the same request to this server:

 ```sh
-curl --cacert fixtures/ca/ca.crt https://127.0.0.1:4001/v1/keys/foo -F value=bar
+curl --cacert fixtures/ca/ca.crt https://127.0.0.1:4001/v1/keys/foo -d value=bar -v
 ```

 The request should be rejected by the server.
@ -334,26 +345,29 @@ Let start by creating 3 new etcd instances.
 We use -s to specify server port and -c to specify client port and -d to specify the directory to store the log and info of the node in the cluster

 ```sh
-./etcd -s 7001 -c 4001 -d nodes/node1
+./etcd -s 127.0.0.1:7001 -c 127.0.0.1:4001 -d nodes/node1 -n node1
 ```

+**Note:** If you want to run etcd on external IP address and still have access locally you need to add `-cl 0.0.0.0` so that it will listen on both external and localhost addresses.
+A similar argument `-sl` is used to setup the listening address for the server port.
+
 Let the join two more nodes to this cluster using the -C argument:

 ```sh
-./etcd -c 4002 -s 7002 -C 127.0.0.1:7001 -d nodes/node2
-./etcd -c 4003 -s 7003 -C 127.0.0.1:7001 -d nodes/node3
+./etcd -c 127.0.0.1:4002 -s 127.0.0.1:7002 -C 127.0.0.1:7001 -d nodes/node2 -n node2
+./etcd -c 127.0.0.1:4003 -s 127.0.0.1:7003 -C 127.0.0.1:7001 -d nodes/node3 -n node3
 ```

 Get the machines in the cluster:

 ```sh
-curl -L http://127.0.0.1:4001/machines
+curl -L http://127.0.0.1:4001/v1/machines
 ```

 We should see there are three nodes in the cluster

 ```
-0.0.0.0:4001,0.0.0.0:4002,0.0.0.0:4003
+http://127.0.0.1:4001, http://127.0.0.1:4002, http://127.0.0.1:4003
 ```

 The machine list is also available via this API:
@ -363,7 +377,7 @@ curl -L http://127.0.0.1:4001/v1/keys/_etcd/machines
 ```

 ```json
-[{"action":"GET","key":"/machines/node1","value":"0.0.0.0,7001,4001","index":4},{"action":"GET","key":"/machines/node3","value":"0.0.0.0,7002,4002","index":4},{"action":"GET","key":"/machines/node4","value":"0.0.0.0,7003,4003","index":4}]
+[{"action":"GET","key":"/_etcd/machines/node1","value":"raft=http://127.0.0.1:7001&etcd=http://127.0.0.1:4001","index":4},{"action":"GET","key":"/_etcd/machines/node2","value":"raft=http://127.0.0.1:7002&etcd=http://127.0.0.1:4002","index":4},{"action":"GET","key":"/_etcd/machines/node3","value":"raft=http://127.0.0.1:7003&etcd=http://127.0.0.1:4003","index":4}]
 ```

 The key of the machine is based on the ```commit index``` when it was added. The value of the machine is ```hostname```, ```raft port``` and ```client port```.
@ -371,12 +385,12 @@ The key of the machine is based on the ```commit index``` when it was added. The
 Also try to get the current leader in the cluster

 ```
-curl -L http://127.0.0.1:4001/leader
+curl -L http://127.0.0.1:4001/v1/leader
 ```
-The first server we set up should be the leader, if it has not dead during these commands.
+The first server we set up should be the leader, if it has not died during these commands.

 ```
-0.0.0.0:7001
+http://127.0.0.1:7001
 ```

 Now we can do normal SET and GET operations on keys as we explored earlier.
@ -400,11 +414,17 @@ curl -L http://127.0.0.1:4002/v1/keys/foo
 A new leader should have been elected.

 ```
-curl -L http://127.0.0.1:4001/leader
+curl -L http://127.0.0.1:4001/v1/leader
 ```

 ```
-0.0.0.0:7002 or 0.0.0.0:7003
+http://127.0.0.1:7002
+```
+
+or
+
+```
+http://127.0.0.1:7003
 ```

 You should be able to see this:
@ -435,6 +455,10 @@ In the previous example we showed how to use SSL client certs for client to serv

 If you are using SSL for server to server communication, you must use it on all instances of etcd.

+## Contributing
+
+See [CONTRIBUTING](https://github.com/coreos/etcd/blob/master/CONTRIBUTING.md) for details on submitting patches and contacting developers via IRC and mailing lists.
+
 ## Libraries and Tools

 **Tools**
@ -445,16 +469,63 @@ If you are using SSL for server to server communication, you must use it on all

 - [go-etcd](https://github.com/coreos/go-etcd)

+**Java libraries**
+
+- [justinsb/jetcd](https://github.com/justinsb/jetcd)
+- [diwakergupta/jetcd](https://github.com/diwakergupta/jetcd)
+
+
+**Python libraries**
+
+- [transitorykris/etcd-py](https://github.com/transitorykris/etcd-py)
+- [jplana/python-etcd](https://github.com/jplana/python-etcd)
+- [russellhaering/txetcd](https://github.com/russellhaering/txetcd) - a Twisted Python library
+
+**Node libraries**
+
+- [stianeikeland/node-etcd](https://github.com/stianeikeland/node-etcd)
+
 **Ruby libraries**

 - [iconara/etcd-rb](https://github.com/iconara/etcd-rb)
 - [jpfuentes2/etcd-ruby](https://github.com/jpfuentes2/etcd-ruby)
 - [ranjib/etcd-ruby](https://github.com/ranjib/etcd-ruby)

+**C libraries**
+
+- [jdarcy/etcd-api](https://github.com/jdarcy/etcd-api)
+
+**Chef Integration**
+
+- [coderanger/etcd-chef](https://github.com/coderanger/etcd-chef)
+
 **Chef Cookbook**

 - [spheromak/etcd-cookbook](https://github.com/spheromak/etcd-cookbook)

+**Projects using etcd**
+
+- [binocarlos/yoda](https://github.com/binocarlos/yoda) - etcd + ZeroMQ
+- [calavera/active-proxy](https://github.com/calavera/active-proxy) - HTTP Proxy configured with etcd
+- [derekchiang/etcdplus](https://github.com/derekchiang/etcdplus) - A set of distributed synchronization primitives built upon etcd
+- [gleicon/goreman](https://github.com/gleicon/goreman/tree/etcd) - Branch of the Go Foreman clone with etcd support
+- [garethr/hiera-etcd](https://github.com/garethr/hiera-etcd) - Puppet hiera backend using etcd
+- [mattn/etcd-vim](https://github.com/mattn/etcd-vim) - SET and GET keys from inside vim
+- [mattn/etcdenv](https://github.com/mattn/etcdenv) - "env" shebang with etcd integration
+
+## FAQ
+
+### What size cluster should I use?
+
+Every command the client sends to the master is broadcast to all of the followers.
+But, the command is not committed until the majority of the cluster machines receive that command.
+
+Because of this majority voting property the ideal cluster should be kept small to keep speed up and be made up of an odd number of machines.
+
+Odd numbers are good because if you have 8 machines the majority will be 5 and if you have 9 machines the majority with be 5.
+The result is that an 8 machine cluster can tolerate 3 machine failures and a 9 machine cluster can tolerate 4 nodes failures.
+And in the best case when all 9 machines are responding the cluster will perform at the speed of the fastest 5 nodes.
+
 ## Project Details

 ### Versioning
@ -463,10 +534,10 @@ etcd uses [semantic versioning][semver].
 When we release v1.0.0 of etcd we will promise not to break the "v1" REST API.
 New minor versions may add additional features to the API however.

-You can get the version of etcd by requesting the root path of etcd:
+You can get the version of etcd by issuing a request to /version:

 ```sh
-curl -L http://127.0.0.1:4001
+curl -L http://127.0.0.1:4001/version
 ```

 During the v0 series of releases we may break the API as we fix bugs and get feedback.
--- a/27
+++ b/27
@ -1,25 +1,26 @@
-#!/bin/bash
+#!/bin/sh
+set -e

 ETCD_PACKAGE=github.com/coreos/etcd
-export GOPATH=${PWD}
-SRC_DIR=$GOPATH/src
-ETCD_DIR=$SRC_DIR/$ETCD_PACKAGE
+export GOPATH="${PWD}"
+SRC_DIR="$GOPATH/src"
+ETCD_DIR="$SRC_DIR/$ETCD_PACKAGE"

-ETCD_BASE=$(dirname ${ETCD_DIR})
-if [ ! -d ${ETCD_BASE} ]; then
-	mkdir -p ${ETCD_BASE}
+ETCD_BASE=$(dirname "${ETCD_DIR}")
+if [ ! -d "${ETCD_BASE}" ]; then
+	mkdir -p "${ETCD_BASE}"
 fi

-if [ ! -h ${ETCD_DIR} ]; then
-	ln -s ../../../ ${ETCD_DIR}
+if [ ! -h "${ETCD_DIR}" ]; then
+	ln -s ../../../ "${ETCD_DIR}"
 fi

 for i in third_party/*; do
-	if [ $i = "third_party/src" ]; then
+	if [ "$i" = "third_party/src" ]; then
 		continue
 	fi
-	cp -R $i src/
+	cp -R "$i" src/
 done

-./scripts/release-version > release_version.go
-go build ${ETCD_PACKAGE}
+./scripts/release-version > server/release_version.go
+go build "${ETCD_PACKAGE}"
--- a/build.ps1
+++ b/build.ps1
@ -0,0 +1,24 @@
+
+$ETCD_PACKAGE="github.com/coreos/etcd"
+$env:GOPATH=$pwd.Path
+$SRC_DIR="$env:GOPATH/src"
+$ETCD_DIR="$SRC_DIR/$ETCD_PACKAGE"
+$env:ETCD_DIR="$SRC_DIR/$ETCD_PACKAGE"
+
+$ETCD_BASE=(Split-Path $ETCD_DIR -Parent)
+if(-not(test-path $ETCD_DIR)){
+	mkdir -force "$ETCD_BASE" > $null
+}
+
+if(-not(test-path $ETCD_DIR )){
+	cmd /c 'mklink /D "%ETCD_DIR%" ..\..\..\'
+}
+
+foreach($i in (ls third_party/*)){
+	if("$i" -eq "third_party/src") {continue}
+	
+	cp -Recurse -force "$i" src/
+}
+
+./scripts/release-version.ps1 | Out-File -Encoding UTF8 server/release_version.go
+go build -v "${ETCD_PACKAGE}"
--- a/command.go
+++ b/command.go
@ -1,154 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"github.com/coreos/etcd/store"
-	"github.com/coreos/go-raft"
-	"path"
-	"time"
-)
-
-// A command represents an action to be taken on the replicated state machine.
-type Command interface {
-	CommandName() string
-	Apply(server *raft.Server) (interface{}, error)
-}
-
-// Set command
-type SetCommand struct {
-	Key        string    `json:"key"`
-	Value      string    `json:"value"`
-	ExpireTime time.Time `json:"expireTime"`
-}
-
-// The name of the set command in the log
-func (c *SetCommand) CommandName() string {
-	return "etcd:set"
-}
-
-// Set the key-value pair
-func (c *SetCommand) Apply(server *raft.Server) (interface{}, error) {
-	return etcdStore.Set(c.Key, c.Value, c.ExpireTime, server.CommitIndex())
-}
-
-// TestAndSet command
-type TestAndSetCommand struct {
-	Key        string    `json:"key"`
-	Value      string    `json:"value"`
-	PrevValue  string    `json: prevValue`
-	ExpireTime time.Time `json:"expireTime"`
-}
-
-// The name of the testAndSet command in the log
-func (c *TestAndSetCommand) CommandName() string {
-	return "testAndSet"
-}
-
-// Set the key-value pair if the current value of the key equals to the given prevValue
-func (c *TestAndSetCommand) Apply(server *raft.Server) (interface{}, error) {
-	return etcdStore.TestAndSet(c.Key, c.PrevValue, c.Value, c.ExpireTime, server.CommitIndex())
-}
-
-// Get command
-type GetCommand struct {
-	Key string `json:"key"`
-}
-
-// The name of the get command in the log
-func (c *GetCommand) CommandName() string {
-	return "etcd:get"
-}
-
-// Get the value of key
-func (c *GetCommand) Apply(server *raft.Server) (interface{}, error) {
-	return etcdStore.Get(c.Key)
-}
-
-// Delete command
-type DeleteCommand struct {
-	Key string `json:"key"`
-}
-
-// The name of the delete command in the log
-func (c *DeleteCommand) CommandName() string {
-	return "etcd:delete"
-}
-
-// Delete the key
-func (c *DeleteCommand) Apply(server *raft.Server) (interface{}, error) {
-	return etcdStore.Delete(c.Key, server.CommitIndex())
-}
-
-// Watch command
-type WatchCommand struct {
-	Key        string `json:"key"`
-	SinceIndex uint64 `json:"sinceIndex"`
-}
-
-// The name of the watch command in the log
-func (c *WatchCommand) CommandName() string {
-	return "etcd:watch"
-}
-
-func (c *WatchCommand) Apply(server *raft.Server) (interface{}, error) {
-	// create a new watcher
-	watcher := store.NewWatcher()
-
-	// add to the watchers list
-	etcdStore.AddWatcher(c.Key, watcher, c.SinceIndex)
-
-	// wait for the notification for any changing
-	res := <-watcher.C
-
-	if res == nil {
-		return nil, fmt.Errorf("Clearing watch")
-	}
-
-	return json.Marshal(res)
-}
-
-// JoinCommand
-type JoinCommand struct {
-	Name    string `json:"name"`
-	RaftURL string `json:"raftURL"`
-	EtcdURL string `json:"etcdURL"`
-}
-
-// The name of the join command in the log
-func (c *JoinCommand) CommandName() string {
-	return "etcd:join"
-}
-
-// Join a server to the cluster
-func (c *JoinCommand) Apply(raftServer *raft.Server) (interface{}, error) {
-
-	// check if the join command is from a previous machine, who lost all its previous log.
-	response, _ := etcdStore.RawGet(path.Join("_etcd/machines", c.Name))
-
-	if response != nil {
-		return []byte("join success"), nil
-	}
-
-	// check machine number in the cluster
-	num := machineNum()
-	if num == maxClusterSize {
-		return []byte("join fail"), fmt.Errorf(errors[103])
-	}
-
-	addNameToURL(c.Name, c.RaftURL, c.EtcdURL)
-
-	// add peer in raft
-	err := raftServer.AddPeer(c.Name)
-
-	// add machine in etcd storage
-	key := path.Join("_etcd/machines", c.Name)
-	value := fmt.Sprintf("raft=%s&etcd=%s", c.RaftURL, c.EtcdURL)
-	etcdStore.Set(key, value, time.Unix(0, 0), raftServer.CommitIndex())
-
-	return []byte("join success"), err
-}
-
-func (c *JoinCommand) NodeName() string {
-	return c.Name
-}
--- a/config.go
+++ b/config.go
@ -0,0 +1,143 @@
+package main
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/coreos/etcd/log"
+	"github.com/coreos/etcd/server"
+)
+
+//--------------------------------------
+// Config
+//--------------------------------------
+
+// Get the server info from previous conf file
+// or from the user
+func getInfo(path string) *Info {
+
+	infoPath := filepath.Join(path, "info")
+
+	if force {
+		// Delete the old configuration if exist
+		logPath := filepath.Join(path, "log")
+		confPath := filepath.Join(path, "conf")
+		snapshotPath := filepath.Join(path, "snapshot")
+		os.Remove(infoPath)
+		os.Remove(logPath)
+		os.Remove(confPath)
+		os.RemoveAll(snapshotPath)
+	} else if info := readInfo(infoPath); info != nil {
+		log.Infof("Found node configuration in '%s'. Ignoring flags", infoPath)
+		return info
+	}
+
+	// Read info from command line
+	info := &argInfo
+
+	// Write to file.
+	content, _ := json.MarshalIndent(info, "", " ")
+	content = []byte(string(content) + "\n")
+	if err := ioutil.WriteFile(infoPath, content, 0644); err != nil {
+		log.Fatalf("Unable to write info to file: %v", err)
+	}
+
+	log.Infof("Wrote node configuration to '%s'", infoPath)
+
+	return info
+}
+
+// readInfo reads from info file and decode to Info struct
+func readInfo(path string) *Info {
+	file, err := os.Open(path)
+
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		log.Fatal(err)
+	}
+	defer file.Close()
+
+	info := &Info{}
+
+	content, err := ioutil.ReadAll(file)
+	if err != nil {
+		log.Fatalf("Unable to read info: %v", err)
+		return nil
+	}
+
+	if err = json.Unmarshal(content, &info); err != nil {
+		log.Fatalf("Unable to parse info: %v", err)
+		return nil
+	}
+
+	return info
+}
+
+func tlsConfigFromInfo(info server.TLSInfo) (t server.TLSConfig, ok bool) {
+	var keyFile, certFile, CAFile string
+	var tlsCert tls.Certificate
+	var err error
+
+	t.Scheme = "http"
+
+	keyFile = info.KeyFile
+	certFile = info.CertFile
+	CAFile = info.CAFile
+
+	// If the user do not specify key file, cert file and
+	// CA file, the type will be HTTP
+	if keyFile == "" && certFile == "" && CAFile == "" {
+		return t, true
+	}
+
+	// both the key and cert must be present
+	if keyFile == "" || certFile == "" {
+		return t, false
+	}
+
+	tlsCert, err = tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	t.Scheme = "https"
+	t.Server.ClientAuth, t.Server.ClientCAs = newCertPool(CAFile)
+
+	// The client should trust the RootCA that the Server uses since
+	// everyone is a peer in the network.
+	t.Client.Certificates = []tls.Certificate{tlsCert}
+	t.Client.RootCAs = t.Server.ClientCAs
+
+	return t, true
+}
+
+// newCertPool creates x509 certPool and corresponding Auth Type.
+// If the given CAfile is valid, add the cert into the pool and verify the clients'
+// certs against the cert in the pool.
+// If the given CAfile is empty, do not verify the clients' cert.
+// If the given CAfile is not valid, fatal.
+func newCertPool(CAFile string) (tls.ClientAuthType, *x509.CertPool) {
+	if CAFile == "" {
+		return tls.NoClientCert, nil
+	}
+	pemByte, err := ioutil.ReadFile(CAFile)
+	check(err)
+
+	block, pemByte := pem.Decode(pemByte)
+
+	cert, err := x509.ParseCertificate(block.Bytes)
+	check(err)
+
+	certPool := x509.NewCertPool()
+
+	certPool.AddCert(cert)
+
+	return tls.RequireAndVerifyClientCert, certPool
+}
--- a/error.go
+++ b/error.go
@ -1,49 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-)
-
-var errors map[int]string
-
-func init() {
-	errors = make(map[int]string)
-
-	// command related errors
-	errors[100] = "Key Not Found"
-	errors[101] = "The given PrevValue is not equal to the value of the key"
-	errors[102] = "Not A File"
-	errors[103] = "Reached the max number of machines in the cluster"
-
-	// Post form related errors
-	errors[200] = "Value is Required in POST form"
-	errors[201] = "PrevValue is Required in POST form"
-	errors[202] = "The given TTL in POST form is not a number"
-	errors[203] = "The given index in POST form is not a number"
-
-	// raft related errors
-	errors[300] = "Raft Internal Error"
-	errors[301] = "During Leader Election"
-
-	// keyword
-	errors[400] = "The prefix of the given key is a keyword in etcd"
-
-	// etcd related errors
-	errors[500] = "watcher is cleared due to etcd recovery"
-
-}
-
-type jsonError struct {
-	ErrorCode int    `json:"errorCode"`
-	Message   string `json:"message"`
-	Cause     string `json:"cause,omitempty"`
-}
-
-func newJsonError(errorCode int, cause string) []byte {
-	b, _ := json.Marshal(jsonError{
-		ErrorCode: errorCode,
-		Message:   errors[errorCode],
-		Cause:     cause,
-	})
-	return b
-}
--- a/error/error.go
+++ b/error/error.go
@ -0,0 +1,103 @@
+package error
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+)
+
+var errors map[int]string
+
+const (
+	EcodeKeyNotFound    = 100
+	EcodeTestFailed     = 101
+	EcodeNotFile        = 102
+	EcodeNoMoreMachine  = 103
+	EcodeNotDir         = 104
+	EcodeNodeExist      = 105
+	EcodeKeyIsPreserved = 106
+
+	EcodeValueRequired      = 200
+	EcodePrevValueRequired  = 201
+	EcodeTTLNaN             = 202
+	EcodeIndexNaN           = 203
+	EcodeValueOrTTLRequired = 204
+
+	EcodeRaftInternal = 300
+	EcodeLeaderElect  = 301
+
+	EcodeWatcherCleared    = 400
+	EcodeEventIndexCleared = 401
+)
+
+func init() {
+	errors = make(map[int]string)
+
+	// command related errors
+	errors[EcodeKeyNotFound] = "Key Not Found"
+	errors[EcodeTestFailed] = "Test Failed" //test and set
+	errors[EcodeNotFile] = "Not A File"
+	errors[EcodeNoMoreMachine] = "Reached the max number of machines in the cluster"
+	errors[EcodeNotDir] = "Not A Directory"
+	errors[EcodeNodeExist] = "Already exists" // create
+	errors[EcodeKeyIsPreserved] = "The prefix of given key is a keyword in etcd"
+
+	// Post form related errors
+	errors[EcodeValueRequired] = "Value is Required in POST form"
+	errors[EcodePrevValueRequired] = "PrevValue is Required in POST form"
+	errors[EcodeTTLNaN] = "The given TTL in POST form is not a number"
+	errors[EcodeIndexNaN] = "The given index in POST form is not a number"
+	errors[EcodeValueOrTTLRequired] = "Value or TTL is required in POST form"
+
+	// raft related errors
+	errors[EcodeRaftInternal] = "Raft Internal Error"
+	errors[EcodeLeaderElect] = "During Leader Election"
+
+	// etcd related errors
+	errors[EcodeWatcherCleared] = "watcher is cleared due to etcd recovery"
+	errors[EcodeEventIndexCleared] = "The event in requested index is outdated and cleared"
+
+}
+
+type Error struct {
+	ErrorCode int    `json:"errorCode"`
+	Message   string `json:"message"`
+	Cause     string `json:"cause,omitempty"`
+	Index     uint64 `json:"index"`
+	Term      uint64 `json:"term"`
+}
+
+func NewError(errorCode int, cause string, index uint64, term uint64) *Error {
+	return &Error{
+		ErrorCode: errorCode,
+		Message:   errors[errorCode],
+		Cause:     cause,
+		Index:     index,
+		Term:      term,
+	}
+}
+
+func Message(code int) string {
+	return errors[code]
+}
+
+// Only for error interface
+func (e Error) Error() string {
+	return e.Message
+}
+
+func (e Error) toJsonString() string {
+	b, _ := json.Marshal(e)
+	return string(b)
+}
+
+func (e Error) Write(w http.ResponseWriter) {
+	w.Header().Add("X-Etcd-Index", fmt.Sprint(e.Index))
+	w.Header().Add("X-Etcd-Term", fmt.Sprint(e.Term))
+	// 3xx is reft internal error
+	if e.ErrorCode/100 == 3 {
+		http.Error(w, e.toJsonString(), http.StatusInternalServerError)
+	} else {
+		http.Error(w, e.toJsonString(), http.StatusBadRequest)
+	}
+}
--- a/etcd.go
+++ b/etcd.go
@ -1,25 +1,16 @@
 package main

 import (
-	"bytes"
-	"crypto/tls"
-	"crypto/x509"
-	"encoding/json"
-	"encoding/pem"
 	"flag"
 	"fmt"
-	"github.com/coreos/etcd/store"
-	"github.com/coreos/etcd/web"
-	"github.com/coreos/go-raft"
 	"io/ioutil"
-	"net"
-	"net/http"
-	"net/url"
 	"os"
-	"os/signal"
-	"runtime/pprof"
 	"strings"
-	"time"
+
+	"github.com/coreos/etcd/log"
+	"github.com/coreos/etcd/server"
+	"github.com/coreos/etcd/store"
+	"github.com/coreos/go-raft"
 )

 //------------------------------------------------------------------------------
@ -28,39 +19,48 @@ import (
 //
 //------------------------------------------------------------------------------

-var verbose bool
-var veryVerbose bool
+var (
+	veryVerbose bool

-var machines string
-var machinesFile string
+	machines     string
+	machinesFile string

-var cluster []string
+	cluster []string

-var argInfo Info
-var dirPath string
+	argInfo Info
+	dirPath string

-var force bool
+	force bool

-var maxSize int
+	printVersion bool

-var snapshot bool
+	maxSize int

-var retryTimes int
+	snapshot bool

-var maxClusterSize int
+	retryTimes int

-var cpuprofile string
+	maxClusterSize int
+
+	cpuprofile string
+
+	cors string
+)

 func init() {
-	flag.BoolVar(&verbose, "v", false, "verbose logging")
+	flag.BoolVar(&printVersion, "version", false, "print the version and exit")
+
+	flag.BoolVar(&log.Verbose, "v", false, "verbose logging")
 	flag.BoolVar(&veryVerbose, "vv", false, "very verbose logging")

 	flag.StringVar(&machines, "C", "", "the ip address and port of a existing machines in the cluster, sepearate by comma")
 	flag.StringVar(&machinesFile, "CF", "", "the file contains a list of existing machines in the cluster, seperate by comma")

 	flag.StringVar(&argInfo.Name, "n", "default-name", "the node name (required)")
-	flag.StringVar(&argInfo.EtcdURL, "c", "127.0.0.1:4001", "the hostname:port for etcd client communication")
-	flag.StringVar(&argInfo.RaftURL, "s", "127.0.0.1:7001", "the hostname:port for raft server communication")
+	flag.StringVar(&argInfo.EtcdURL, "c", "127.0.0.1:4001", "the advertised public hostname:port for etcd client communication")
+	flag.StringVar(&argInfo.RaftURL, "s", "127.0.0.1:7001", "the advertised public hostname:port for raft server communication")
+	flag.StringVar(&argInfo.EtcdListenHost, "cl", "", "the listening hostname for etcd client communication (defaults to advertised ip)")
+	flag.StringVar(&argInfo.RaftListenHost, "sl", "", "the listening hostname for raft server communication (defaults to advertised ip)")
 	flag.StringVar(&argInfo.WebURL, "w", "", "the hostname:port of web interface")

 	flag.StringVar(&argInfo.RaftTLS.CAFile, "serverCAFile", "", "the path of the CAFile")
@ -84,31 +84,16 @@ func init() {
 	flag.IntVar(&maxClusterSize, "maxsize", 9, "the max size of the cluster")

 	flag.StringVar(&cpuprofile, "cpuprofile", "", "write cpu profile to file")
+
+	flag.StringVar(&cors, "cors", "", "whitelist origins for cross-origin resource sharing (e.g. '*' or 'http://localhost:8001,etc')")
 }

-const (
-	ELECTIONTIMEOUT  = 200 * time.Millisecond
-	HEARTBEATTIMEOUT = 50 * time.Millisecond
-
-	// Timeout for internal raft http connection
-	// The original timeout for http is 45 seconds
-	// which is too long for our usage.
-	HTTPTIMEOUT   = 10 * time.Second
-	RETRYINTERVAL = 10
-)
-
 //------------------------------------------------------------------------------
 //
 // Typedefs
 //
 //------------------------------------------------------------------------------

-type TLSInfo struct {
-	CertFile string `json:"CertFile"`
-	KeyFile  string `json:"KeyFile"`
-	CAFile   string `json:"CAFile"`
-}
-
 type Info struct {
 	Name string `json:"name"`

@ -116,51 +101,19 @@ type Info struct {
 	EtcdURL string `json:"etcdURL"`
 	WebURL  string `json:"webURL"`

-	RaftTLS TLSInfo `json:"raftTLS"`
-	EtcdTLS TLSInfo `json:"etcdTLS"`
+	RaftListenHost string `json:"raftListenHost"`
+	EtcdListenHost string `json:"etcdListenHost"`
+
+	RaftTLS server.TLSInfo `json:"raftTLS"`
+	EtcdTLS server.TLSInfo `json:"etcdTLS"`
 }

-//------------------------------------------------------------------------------
-//
-// Variables
-//
-//------------------------------------------------------------------------------
-
-var raftServer *raft.Server
-var raftTransporter transporter
-var etcdStore *store.Store
-var info *Info
-
 //------------------------------------------------------------------------------
 //
 // Functions
 //
 //------------------------------------------------------------------------------

-// sanitizeURL will cleanup a host string in the format hostname:port and
-// attach a schema.
-func sanitizeURL(host string, defaultScheme string) string {
-	// Blank URLs are fine input, just return it
-	if len(host) == 0 {
-		return host
-	}
-
-	p, err := url.Parse(host)
-	if err != nil {
-		fatal(err)
-	}
-
-	// Make sure the host is in Host:Port format
-	_, _, err = net.SplitHostPort(host)
-	if err != nil {
-		fatal(err)
-	}
-
-	p = &url.URL{Host: host, Scheme: defaultScheme}
-
-	return p.String()
-}
-
 //--------------------------------------
 // Main
 //--------------------------------------
@ -168,28 +121,17 @@ func sanitizeURL(host string, defaultScheme string) string {
 func main() {
 	flag.Parse()

+	if printVersion {
+		fmt.Println(server.ReleaseVersion)
+		os.Exit(0)
+	}
+
 	if cpuprofile != "" {
-		f, err := os.Create(cpuprofile)
-		if err != nil {
-			fatal(err)
-		}
-		pprof.StartCPUProfile(f)
-		defer pprof.StopCPUProfile()
-
-		c := make(chan os.Signal, 1)
-		signal.Notify(c, os.Interrupt)
-		go func() {
-			for sig := range c {
-				fmt.Printf("captured %v, stopping profiler and exiting..", sig)
-				pprof.StopCPUProfile()
-				os.Exit(1)
-			}
-		}()
-
+		runCPUProfile()
 	}

 	if veryVerbose {
-		verbose = true
+		log.Verbose = true
 		raft.SetLogLevel(raft.Debug)
 	}

@ -198,442 +140,60 @@ func main() {
 	} else if machinesFile != "" {
 		b, err := ioutil.ReadFile(machinesFile)
 		if err != nil {
-			fatalf("Unable to read the given machines file: %s", err)
+			log.Fatalf("Unable to read the given machines file: %s", err)
 		}
 		cluster = strings.Split(string(b), ",")
 	}

+	// Check TLS arguments
 	raftTLSConfig, ok := tlsConfigFromInfo(argInfo.RaftTLS)
 	if !ok {
-		fatal("Please specify cert and key file or cert and key file and CAFile or none of the three")
+		log.Fatal("Please specify cert and key file or cert and key file and CAFile or none of the three")
 	}

 	etcdTLSConfig, ok := tlsConfigFromInfo(argInfo.EtcdTLS)
 	if !ok {
-		fatal("Please specify cert and key file or cert and key file and CAFile or none of the three")
+		log.Fatal("Please specify cert and key file or cert and key file and CAFile or none of the three")
 	}

 	argInfo.Name = strings.TrimSpace(argInfo.Name)
 	if argInfo.Name == "" {
-		fatal("ERROR: server name required. e.g. '-n=server_name'")
+		log.Fatal("ERROR: server name required. e.g. '-n=server_name'")
 	}

+	// Check host name arguments
 	argInfo.RaftURL = sanitizeURL(argInfo.RaftURL, raftTLSConfig.Scheme)
 	argInfo.EtcdURL = sanitizeURL(argInfo.EtcdURL, etcdTLSConfig.Scheme)
 	argInfo.WebURL = sanitizeURL(argInfo.WebURL, "http")

-	// Setup commands.
-	registerCommands()
+	argInfo.RaftListenHost = sanitizeListenHost(argInfo.RaftListenHost, argInfo.RaftURL)
+	argInfo.EtcdListenHost = sanitizeListenHost(argInfo.EtcdListenHost, argInfo.EtcdURL)

 	// Read server info from file or grab it from user.
 	if err := os.MkdirAll(dirPath, 0744); err != nil {
-		fatalf("Unable to create path: %s", err)
+		log.Fatalf("Unable to create path: %s", err)
 	}

-	info = getInfo(dirPath)
+	info := getInfo(dirPath)

 	// Create etcd key-value store
-	etcdStore = store.CreateStore(maxSize)
+	store := store.New()

-	startRaft(raftTLSConfig)
+	// Create a shared node registry.
+	registry := server.NewRegistry(store)

-	if argInfo.WebURL != "" {
-		// start web
-		argInfo.WebURL = sanitizeURL(argInfo.WebURL, "http")
-		go webHelper()
-		go web.Start(raftServer, argInfo.WebURL)
+	// Create peer server.
+	ps := server.NewPeerServer(info.Name, dirPath, info.RaftURL, info.RaftListenHost, &raftTLSConfig, &info.RaftTLS, registry, store)
+	ps.MaxClusterSize = maxClusterSize
+	ps.RetryTimes = retryTimes
+
+	s := server.New(info.Name, info.EtcdURL, info.EtcdListenHost, &etcdTLSConfig, &info.EtcdTLS, ps, registry, store)
+	if err := s.AllowOrigins(cors); err != nil {
+		panic(err)
 	}

-	startEtcdTransport(*info, etcdTLSConfig.Scheme, etcdTLSConfig.Server)
+	ps.SetServer(s)

-}
-
-// Start the raft server
-func startRaft(tlsConfig TLSConfig) {
-	var err error
-
-	raftName := info.Name
-
-	// Create transporter for raft
-	raftTransporter = newTransporter(tlsConfig.Scheme, tlsConfig.Client)
-
-	// Create raft server
-	raftServer, err = raft.NewServer(raftName, dirPath, raftTransporter, etcdStore, nil)
-
-	if err != nil {
-		fatal(err)
-	}
-
-	// LoadSnapshot
-	if snapshot {
-		err = raftServer.LoadSnapshot()
-
-		if err == nil {
-			debugf("%s finished load snapshot", raftServer.Name())
-		} else {
-			debug(err)
-		}
-	}
-
-	raftServer.SetElectionTimeout(ELECTIONTIMEOUT)
-	raftServer.SetHeartbeatTimeout(HEARTBEATTIMEOUT)
-
-	raftServer.Start()
-
-	if raftServer.IsLogEmpty() {
-
-		// start as a leader in a new cluster
-		if len(cluster) == 0 {
-
-			time.Sleep(time.Millisecond * 20)
-
-			// leader need to join self as a peer
-			for {
-				command := &JoinCommand{
-					Name:    raftServer.Name(),
-					RaftURL: argInfo.RaftURL,
-					EtcdURL: argInfo.EtcdURL,
-				}
-				_, err := raftServer.Do(command)
-				if err == nil {
-					break
-				}
-			}
-			debugf("%s start as a leader", raftServer.Name())
-
-			// start as a follower in a existing cluster
-		} else {
-
-			time.Sleep(time.Millisecond * 20)
-
-			for i := 0; i < retryTimes; i++ {
-
-				success := false
-				for _, machine := range cluster {
-					if len(machine) == 0 {
-						continue
-					}
-					err = joinCluster(raftServer, machine)
-					if err != nil {
-						if err.Error() == errors[103] {
-							fmt.Println(err)
-							os.Exit(1)
-						}
-						debugf("cannot join to cluster via machine %s %s", machine, err)
-					} else {
-						success = true
-						break
-					}
-				}
-
-				if success {
-					break
-				}
-
-				warnf("cannot join to cluster via given machines, retry in %d seconds", RETRYINTERVAL)
-				time.Sleep(time.Second * RETRYINTERVAL)
-			}
-			if err != nil {
-				fatalf("Cannot join the cluster via given machines after %x retries", retryTimes)
-			}
-			debugf("%s success join to the cluster", raftServer.Name())
-		}
-
-	} else {
-		// rejoin the previous cluster
-		debugf("%s restart as a follower", raftServer.Name())
-	}
-
-	// open the snapshot
-	if snapshot {
-		go raftServer.Snapshot()
-	}
-
-	// start to response to raft requests
-	go startRaftTransport(*info, tlsConfig.Scheme, tlsConfig.Server)
-
-}
-
-// Create transporter using by raft server
-// Create http or https transporter based on
-// whether the user give the server cert and key
-func newTransporter(scheme string, tlsConf tls.Config) transporter {
-	t := transporter{}
-
-	t.scheme = scheme
-
-	tr := &http.Transport{
-		Dial:               dialTimeout,
-	}
-
-	if scheme == "https" {
-		tr.TLSClientConfig = &tlsConf
-		tr.DisableCompression = true
-	}
-
-	t.client = &http.Client{Transport: tr}
-
-	return t
-}
-
-// Dial with timeout
-func dialTimeout(network, addr string) (net.Conn, error) {
-	return net.DialTimeout(network, addr, HTTPTIMEOUT)
-}
-
-// Start to listen and response raft command
-func startRaftTransport(info Info, scheme string, tlsConf tls.Config) {
-	u, _ := url.Parse(info.RaftURL)
-	fmt.Printf("raft server [%s] listening on %s\n", info.Name, u)
-
-	raftMux := http.NewServeMux()
-
-	server := &http.Server{
-		Handler:   raftMux,
-		TLSConfig: &tlsConf,
-		Addr:      u.Host,
-	}
-
-	// internal commands
-	raftMux.HandleFunc("/name", NameHttpHandler)
-	raftMux.HandleFunc("/join", JoinHttpHandler)
-	raftMux.HandleFunc("/vote", VoteHttpHandler)
-	raftMux.HandleFunc("/log", GetLogHttpHandler)
-	raftMux.HandleFunc("/log/append", AppendEntriesHttpHandler)
-	raftMux.HandleFunc("/snapshot", SnapshotHttpHandler)
-	raftMux.HandleFunc("/snapshotRecovery", SnapshotRecoveryHttpHandler)
-	raftMux.HandleFunc("/etcdURL", EtcdURLHttpHandler)
-
-	if scheme == "http" {
-		fatal(server.ListenAndServe())
-	} else {
-		fatal(server.ListenAndServeTLS(info.RaftTLS.CertFile, info.RaftTLS.KeyFile))
-	}
-
-}
-
-// Start to listen and response client command
-func startEtcdTransport(info Info, scheme string, tlsConf tls.Config) {
-	u, _ := url.Parse(info.EtcdURL)
-	fmt.Printf("etcd server [%s] listening on %s\n", info.Name, u)
-
-	etcdMux := http.NewServeMux()
-
-	server := &http.Server{
-		Handler:   etcdMux,
-		TLSConfig: &tlsConf,
-		Addr:      u.Host,
-	}
-
-	// external commands
-	etcdMux.HandleFunc("/"+version+"/keys/", Multiplexer)
-	etcdMux.HandleFunc("/"+version+"/watch/", WatchHttpHandler)
-	etcdMux.HandleFunc("/leader", LeaderHttpHandler)
-	etcdMux.HandleFunc("/machines", MachinesHttpHandler)
-	etcdMux.HandleFunc("/", VersionHttpHandler)
-	etcdMux.HandleFunc("/stats", StatsHttpHandler)
-	etcdMux.HandleFunc("/test/", TestHttpHandler)
-
-	if scheme == "http" {
-		fatal(server.ListenAndServe())
-	} else {
-		fatal(server.ListenAndServeTLS(info.EtcdTLS.CertFile, info.EtcdTLS.KeyFile))
-	}
-}
-
-//--------------------------------------
-// Config
-//--------------------------------------
-
-type TLSConfig struct {
-	Scheme string
-	Server tls.Config
-	Client tls.Config
-}
-
-func tlsConfigFromInfo(info TLSInfo) (t TLSConfig, ok bool) {
-	var keyFile, certFile, CAFile string
-	var tlsCert tls.Certificate
-	var err error
-
-	t.Scheme = "http"
-
-	keyFile = info.KeyFile
-	certFile = info.CertFile
-	CAFile = info.CAFile
-
-	// If the user do not specify key file, cert file and
-	// CA file, the type will be HTTP
-	if keyFile == "" && certFile == "" && CAFile == "" {
-		return t, true
-	}
-
-	// both the key and cert must be present
-	if keyFile == "" || certFile == "" {
-		return t, false
-	}
-
-	tlsCert, err = tls.LoadX509KeyPair(certFile, keyFile)
-	if err != nil {
-		fatal(err)
-	}
-
-	t.Scheme = "https"
-	t.Server.ClientAuth, t.Server.ClientCAs = newCertPool(CAFile)
-
-	// The client should trust the RootCA that the Server uses since
-	// everyone is a peer in the network.
-	t.Client.Certificates = []tls.Certificate{tlsCert}
-	t.Client.RootCAs = t.Server.ClientCAs
-
-	return t, true
-}
-
-func parseInfo(path string) *Info {
-	file, err := os.Open(path)
-
-	if err != nil {
-		return nil
-	}
-
-	info := &Info{}
-	defer file.Close()
-
-	content, err := ioutil.ReadAll(file)
-	if err != nil {
-		fatalf("Unable to read info: %v", err)
-		return nil
-	}
-
-	if err = json.Unmarshal(content, &info); err != nil {
-		fatalf("Unable to parse info: %v", err)
-		return nil
-	}
-
-	return info
-}
-
-// Get the server info from previous conf file
-// or from the user
-func getInfo(path string) *Info {
-
-	// Read in the server info if available.
-	infoPath := fmt.Sprintf("%s/info", path)
-
-	// Delete the old configuration if exist
-	if force {
-		logPath := fmt.Sprintf("%s/log", path)
-		confPath := fmt.Sprintf("%s/conf", path)
-		snapshotPath := fmt.Sprintf("%s/snapshot", path)
-		os.Remove(infoPath)
-		os.Remove(logPath)
-		os.Remove(confPath)
-		os.RemoveAll(snapshotPath)
-	}
-
-	info := parseInfo(infoPath)
-	if info != nil {
-		fmt.Printf("Found node configuration in '%s'. Ignoring flags.\n", infoPath)
-		return info
-	}
-
-	info = &argInfo
-
-	// Write to file.
-	content, _ := json.MarshalIndent(info, "", " ")
-	content = []byte(string(content) + "\n")
-	if err := ioutil.WriteFile(infoPath, content, 0644); err != nil {
-		fatalf("Unable to write info to file: %v", err)
-	}
-
-	fmt.Printf("Wrote node configuration to '%s'.\n", infoPath)
-
-	return info
-}
-
-// Create client auth certpool
-func newCertPool(CAFile string) (tls.ClientAuthType, *x509.CertPool) {
-	if CAFile == "" {
-		return tls.NoClientCert, nil
-	}
-	pemByte, _ := ioutil.ReadFile(CAFile)
-
-	block, pemByte := pem.Decode(pemByte)
-
-	cert, err := x509.ParseCertificate(block.Bytes)
-
-	if err != nil {
-		fatal(err)
-	}
-
-	certPool := x509.NewCertPool()
-
-	certPool.AddCert(cert)
-
-	return tls.RequireAndVerifyClientCert, certPool
-}
-
-// Send join requests to the leader.
-func joinCluster(s *raft.Server, raftURL string) error {
-	var b bytes.Buffer
-
-	command := &JoinCommand{
-		Name:    s.Name(),
-		RaftURL: info.RaftURL,
-		EtcdURL: info.EtcdURL,
-	}
-
-	json.NewEncoder(&b).Encode(command)
-
-	// t must be ok
-	t, ok := raftServer.Transporter().(transporter)
-
-	if !ok {
-		panic("wrong type")
-	}
-
-	joinURL := url.URL{Host: raftURL, Scheme: raftTransporter.scheme, Path: "/join"}
-
-	debugf("Send Join Request to %s", raftURL)
-	
-	resp, err := t.Post(joinURL.String(), &b)
-
-	for {
-		if err != nil {
-			return fmt.Errorf("Unable to join: %v", err)
-		}
-		if resp != nil {
-			defer resp.Body.Close()
-			if resp.StatusCode == http.StatusOK {
-				return nil
-			}
-			if resp.StatusCode == http.StatusTemporaryRedirect {
-
-				address := resp.Header.Get("Location")
-				debugf("Send Join Request to %s", address)
-
-				json.NewEncoder(&b).Encode(command)
-
-				resp, err = t.Post(address, &b)
-
-			} else if resp.StatusCode == http.StatusBadRequest {
-				debug("Reach max number machines in the cluster")
-				return fmt.Errorf(errors[103])
-			} else {
-				return fmt.Errorf("Unable to join")
-			}
-		}
-
-	}
-	return fmt.Errorf("Unable to join: %v", err)
-}
-
-// Register commands to raft server
-func registerCommands() {
-	raft.RegisterCommand(&JoinCommand{})
-	raft.RegisterCommand(&SetCommand{})
-	raft.RegisterCommand(&GetCommand{})
-	raft.RegisterCommand(&DeleteCommand{})
-	raft.RegisterCommand(&WatchCommand{})
-	raft.RegisterCommand(&TestAndSetCommand{})
+	ps.ListenAndServe(snapshot, cluster)
+	s.ListenAndServe()
 }
--- a/etcd_handlers.go
+++ b/etcd_handlers.go
@ -1,355 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"github.com/coreos/etcd/store"
-	"net/http"
-	"strconv"
-	"time"
-)
-
-//-------------------------------------------------------------------
-// Handlers to handle etcd-store related request via etcd url
-//-------------------------------------------------------------------
-
-// Multiplex GET/POST/DELETE request to corresponding handlers
-func Multiplexer(w http.ResponseWriter, req *http.Request) {
-
-	switch req.Method {
-	case "GET":
-		GetHttpHandler(&w, req)
-	case "POST":
-		SetHttpHandler(&w, req)
-	case "DELETE":
-		DeleteHttpHandler(&w, req)
-	default:
-		w.WriteHeader(http.StatusMethodNotAllowed)
-		return
-	}
-}
-
-//--------------------------------------
-// State sensitive handlers
-// Set/Delete will dispatch to leader
-//--------------------------------------
-
-// Set Command Handler
-func SetHttpHandler(w *http.ResponseWriter, req *http.Request) {
-	key := req.URL.Path[len("/v1/keys/"):]
-
-	if store.CheckKeyword(key) {
-
-		(*w).WriteHeader(http.StatusBadRequest)
-
-		(*w).Write(newJsonError(400, "Set"))
-		return
-	}
-
-	debugf("[recv] POST %v/v1/keys/%s", raftServer.Name(), key)
-
-	value := req.FormValue("value")
-
-	if len(value) == 0 {
-		(*w).WriteHeader(http.StatusBadRequest)
-
-		(*w).Write(newJsonError(200, "Set"))
-		return
-	}
-
-	prevValue := req.FormValue("prevValue")
-
-	strDuration := req.FormValue("ttl")
-
-	expireTime, err := durationToExpireTime(strDuration)
-
-	if err != nil {
-
-		(*w).WriteHeader(http.StatusBadRequest)
-
-		(*w).Write(newJsonError(202, "Set"))
-		return
-	}
-
-	if len(prevValue) != 0 {
-		command := &TestAndSetCommand{
-			Key:        key,
-			Value:      value,
-			PrevValue:  prevValue,
-			ExpireTime: expireTime,
-		}
-
-		dispatch(command, w, req, true)
-
-	} else {
-		command := &SetCommand{
-			Key:        key,
-			Value:      value,
-			ExpireTime: expireTime,
-		}
-
-		dispatch(command, w, req, true)
-	}
-
-}
-
-// Delete Handler
-func DeleteHttpHandler(w *http.ResponseWriter, req *http.Request) {
-	key := req.URL.Path[len("/v1/keys/"):]
-
-	debugf("[recv] DELETE %v/v1/keys/%s", raftServer.Name(), key)
-
-	command := &DeleteCommand{
-		Key: key,
-	}
-
-	dispatch(command, w, req, true)
-}
-
-// Dispatch the command to leader
-func dispatch(c Command, w *http.ResponseWriter, req *http.Request, etcd bool) {
-	if raftServer.State() == "leader" {
-		if body, err := raftServer.Do(c); err != nil {
-			if _, ok := err.(store.NotFoundError); ok {
-				(*w).WriteHeader(http.StatusNotFound)
-				(*w).Write(newJsonError(100, err.Error()))
-				return
-			}
-
-			if _, ok := err.(store.TestFail); ok {
-				(*w).WriteHeader(http.StatusBadRequest)
-				(*w).Write(newJsonError(101, err.Error()))
-				return
-			}
-
-			if _, ok := err.(store.NotFile); ok {
-				(*w).WriteHeader(http.StatusBadRequest)
-				(*w).Write(newJsonError(102, err.Error()))
-				return
-			}
-			if err.Error() == errors[103] {
-				(*w).WriteHeader(http.StatusBadRequest)
-				(*w).Write(newJsonError(103, ""))
-				return
-			}
-			(*w).WriteHeader(http.StatusInternalServerError)
-			(*w).Write(newJsonError(300, err.Error()))
-			return
-		} else {
-
-			if body == nil {
-				(*w).WriteHeader(http.StatusNotFound)
-				(*w).Write(newJsonError(300, "Empty result from raft"))
-			} else {
-				body, ok := body.([]byte)
-				// this should not happen
-				if !ok {
-					panic("wrong type")
-				}
-				(*w).WriteHeader(http.StatusOK)
-				(*w).Write(body)
-			}
-			return
-		}
-	} else {
-		// current no leader
-		if raftServer.Leader() == "" {
-			(*w).WriteHeader(http.StatusInternalServerError)
-			(*w).Write(newJsonError(300, ""))
-			return
-		}
-
-		// tell the client where is the leader
-
-		path := req.URL.Path
-
-		var scheme string
-
-		if scheme = req.URL.Scheme; scheme == "" {
-			scheme = "http://"
-		}
-
-		var url string
-
-		if etcd {
-			etcdAddr, _ := nameToEtcdURL(raftServer.Leader())
-			url = etcdAddr + path
-		} else {
-			raftAddr, _ := nameToRaftURL(raftServer.Leader())
-			url = raftAddr + path
-		}
-
-		debugf("Redirect to %s", url)
-
-		http.Redirect(*w, req, url, http.StatusTemporaryRedirect)
-		return
-	}
-	(*w).WriteHeader(http.StatusInternalServerError)
-	(*w).Write(newJsonError(300, ""))
-	return
-}
-
-//--------------------------------------
-// State non-sensitive handlers
-// will not dispatch to leader
-// TODO: add sensitive version for these
-// command?
-//--------------------------------------
-
-// Handler to return the current leader's raft address
-func LeaderHttpHandler(w http.ResponseWriter, req *http.Request) {
-	leader := raftServer.Leader()
-
-	if leader != "" {
-		w.WriteHeader(http.StatusOK)
-		raftURL, _ := nameToRaftURL(leader)
-		w.Write([]byte(raftURL))
-	} else {
-
-		// not likely, but it may happen
-		w.WriteHeader(http.StatusInternalServerError)
-		w.Write(newJsonError(301, ""))
-	}
-}
-
-// Handler to return all the known machines in the current cluster
-func MachinesHttpHandler(w http.ResponseWriter, req *http.Request) {
-	peers := raftServer.Peers()
-
-	// Add itself to the machine list first
-	// Since peer map does not contain the server itself
-	machines, _ := getEtcdURL(raftServer.Name())
-
-	// Add all peers to the list and separate by comma
-	// We do not use json here since we accept machines list
-	// in the command line separate by comma.
-
-	for peerName, _ := range peers {
-		if addr, ok := getEtcdURL(peerName); ok {
-			machines = machines + "," + addr
-		}
-	}
-
-	w.WriteHeader(http.StatusOK)
-	w.Write([]byte(machines))
-
-}
-
-// Handler to return the current version of etcd
-func VersionHttpHandler(w http.ResponseWriter, req *http.Request) {
-	w.WriteHeader(http.StatusOK)
-	w.Write([]byte(fmt.Sprintf("etcd %s", releaseVersion)))
-}
-
-// Handler to return the basic stats of etcd
-func StatsHttpHandler(w http.ResponseWriter, req *http.Request) {
-	w.WriteHeader(http.StatusOK)
-	w.Write(etcdStore.Stats())
-}
-
-// Get Handler
-func GetHttpHandler(w *http.ResponseWriter, req *http.Request) {
-	key := req.URL.Path[len("/v1/keys/"):]
-
-	debugf("[recv] GET http://%v/v1/keys/%s", raftServer.Name(), key)
-
-	command := &GetCommand{
-		Key: key,
-	}
-
-	if body, err := command.Apply(raftServer); err != nil {
-
-		if _, ok := err.(store.NotFoundError); ok {
-			(*w).WriteHeader(http.StatusNotFound)
-			(*w).Write(newJsonError(100, err.Error()))
-			return
-		}
-
-		(*w).WriteHeader(http.StatusInternalServerError)
-		(*w).Write(newJsonError(300, ""))
-
-	} else {
-		body, ok := body.([]byte)
-		if !ok {
-			panic("wrong type")
-		}
-
-		(*w).WriteHeader(http.StatusOK)
-		(*w).Write(body)
-
-	}
-
-}
-
-// Watch handler
-func WatchHttpHandler(w http.ResponseWriter, req *http.Request) {
-	key := req.URL.Path[len("/v1/watch/"):]
-
-	command := &WatchCommand{
-		Key: key,
-	}
-
-	if req.Method == "GET" {
-		debugf("[recv] GET http://%v/watch/%s", raftServer.Name(), key)
-		command.SinceIndex = 0
-
-	} else if req.Method == "POST" {
-		// watch from a specific index
-
-		debugf("[recv] POST http://%v/watch/%s", raftServer.Name(), key)
-		content := req.FormValue("index")
-
-		sinceIndex, err := strconv.ParseUint(string(content), 10, 64)
-		if err != nil {
-			w.WriteHeader(http.StatusBadRequest)
-			w.Write(newJsonError(203, "Watch From Index"))
-		}
-		command.SinceIndex = sinceIndex
-
-	} else {
-		w.WriteHeader(http.StatusMethodNotAllowed)
-		return
-	}
-
-	if body, err := command.Apply(raftServer); err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-		w.Write(newJsonError(500, key))
-	} else {
-		w.WriteHeader(http.StatusOK)
-
-		body, ok := body.([]byte)
-		if !ok {
-			panic("wrong type")
-		}
-
-		w.Write(body)
-	}
-
-}
-
-// TestHandler
-func TestHttpHandler(w http.ResponseWriter, req *http.Request) {
-	testType := req.URL.Path[len("/test/"):]
-
-	if testType == "speed" {
-		directSet()
-		w.WriteHeader(http.StatusOK)
-		w.Write([]byte("speed test success"))
-		return
-	}
-
-	w.WriteHeader(http.StatusBadRequest)
-}
-
-// Convert string duration to time format
-func durationToExpireTime(strDuration string) (time.Time, error) {
-	if strDuration != "" {
-		duration, err := strconv.Atoi(strDuration)
-
-		if err != nil {
-			return time.Unix(0, 0), err
-		}
-		return time.Now().Add(time.Second * (time.Duration)(duration)), nil
-	} else {
-		return time.Unix(0, 0), nil
-	}
-}
--- a/etcd_long_test.go
+++ b/etcd_long_test.go
@ -1,150 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"math/rand"
-	"net/http"
-	"os"
-	"strconv"
-	"strings"
-	"testing"
-	"time"
-)
-
-// This test will kill the current leader and wait for the etcd cluster to elect a new leader for 200 times.
-// It will print out the election time and the average election time.
-func TestKillLeader(t *testing.T) {
-	procAttr := new(os.ProcAttr)
-	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
-
-	clusterSize := 5
-	argGroup, etcds, err := createCluster(clusterSize, procAttr, false)
-
-	if err != nil {
-		t.Fatal("cannot create cluster")
-	}
-
-	defer destroyCluster(etcds)
-
-	leaderChan := make(chan string, 1)
-
-	time.Sleep(time.Second)
-
-	go leaderMonitor(clusterSize, 1, leaderChan)
-
-	var totalTime time.Duration
-
-	leader := "http://127.0.0.1:7001"
-
-	for i := 0; i < clusterSize; i++ {
-		fmt.Println("leader is ", leader)
-		port, _ := strconv.Atoi(strings.Split(leader, ":")[2])
-		num := port - 7001
-		fmt.Println("kill server ", num)
-		etcds[num].Kill()
-		etcds[num].Release()
-
-		start := time.Now()
-		for {
-			newLeader := <-leaderChan
-			if newLeader != leader {
-				leader = newLeader
-				break
-			}
-		}
-		take := time.Now().Sub(start)
-
-		totalTime += take
-		avgTime := totalTime / (time.Duration)(i+1)
-
-		fmt.Println("Leader election time is ", take, "with election timeout", ELECTIONTIMEOUT)
-		fmt.Println("Leader election time average is", avgTime, "with election timeout", ELECTIONTIMEOUT)
-		etcds[num], err = os.StartProcess("etcd", argGroup[num], procAttr)
-	}
-}
-
-// TestKillRandom kills random machines in the cluster and
-// restart them after all other machines agree on the same leader
-func TestKillRandom(t *testing.T) {
-	procAttr := new(os.ProcAttr)
-	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
-
-	clusterSize := 9
-	argGroup, etcds, err := createCluster(clusterSize, procAttr, false)
-
-	if err != nil {
-		t.Fatal("cannot create cluster")
-	}
-
-	defer destroyCluster(etcds)
-
-	leaderChan := make(chan string, 1)
-
-	time.Sleep(3 * time.Second)
-
-	go leaderMonitor(clusterSize, 4, leaderChan)
-
-	toKill := make(map[int]bool)
-
-	for i := 0; i < 20; i++ {
-		fmt.Printf("TestKillRandom Round[%d/20]\n", i)
-
-		j := 0
-		for {
-
-			r := rand.Int31n(9)
-			if _, ok := toKill[int(r)]; !ok {
-				j++
-				toKill[int(r)] = true
-			}
-
-			if j > 3 {
-				break
-			}
-
-		}
-
-		for num, _ := range toKill {
-			etcds[num].Kill()
-			etcds[num].Release()
-		}
-
-		<-leaderChan
-
-		for num, _ := range toKill {
-			etcds[num], err = os.StartProcess("etcd", argGroup[num], procAttr)
-		}
-
-		toKill = make(map[int]bool)
-	}
-
-	<-leaderChan
-
-}
-
-func templateBenchmarkEtcdDirectCall(b *testing.B, tls bool) {
-	procAttr := new(os.ProcAttr)
-	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
-
-	clusterSize := 3
-	_, etcds, _ := createCluster(clusterSize, procAttr, tls)
-
-	defer destroyCluster(etcds)
-
-	time.Sleep(time.Second)
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		resp, _ := http.Get("http://127.0.0.1:4001/test/speed")
-		resp.Body.Close()
-	}
-
-}
-
-func BenchmarkEtcdDirectCall(b *testing.B) {
-	templateBenchmarkEtcdDirectCall(b, false)
-}
-
-func BenchmarkEtcdDirectCallTls(b *testing.B) {
-	templateBenchmarkEtcdDirectCall(b, true)
-}
--- a/etcd_test.go
+++ b/etcd_test.go
@ -1,209 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"github.com/coreos/go-etcd/etcd"
-	"math/rand"
-	"os"
-	//"strconv"
-	"testing"
-	"time"
-)
-
-// Create a single node and try to set value
-func TestSingleNode(t *testing.T) {
-	procAttr := new(os.ProcAttr)
-	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
-	args := []string{"etcd", "-n=node1", "-f", "-d=/tmp/node1"}
-
-	process, err := os.StartProcess("etcd", args, procAttr)
-	if err != nil {
-		t.Fatal("start process failed:" + err.Error())
-		return
-	}
-	defer process.Kill()
-
-	time.Sleep(time.Second)
-
-	c := etcd.NewClient()
-
-	c.SyncCluster()
-	// Test Set
-	result, err := c.Set("foo", "bar", 100)
-
-	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.TTL != 99 {
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		t.Fatalf("Set 1 failed with %s %s %v", result.Key, result.Value, result.TTL)
-	}
-
-	time.Sleep(time.Second)
-
-	result, err = c.Set("foo", "bar", 100)
-
-	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.PrevValue != "bar" || result.TTL != 99 {
-		if err != nil {
-			t.Fatal(err)
-		}
-		t.Fatalf("Set 2 failed with %s %s %v", result.Key, result.Value, result.TTL)
-	}
-}
-
-// This test creates a single node and then set a value to it.
-// Then this test kills the node and restart it and tries to get the value again.
-func TestSingleNodeRecovery(t *testing.T) {
-	procAttr := new(os.ProcAttr)
-	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
-	args := []string{"etcd", "-n=node1", "-d=/tmp/node1"}
-
-	process, err := os.StartProcess("etcd", append(args, "-f"), procAttr)
-	if err != nil {
-		t.Fatal("start process failed:" + err.Error())
-		return
-	}
-
-	time.Sleep(time.Second)
-
-	c := etcd.NewClient()
-
-	c.SyncCluster()
-	// Test Set
-	result, err := c.Set("foo", "bar", 100)
-
-	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.TTL != 99 {
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		t.Fatalf("Set 1 failed with %s %s %v", result.Key, result.Value, result.TTL)
-	}
-
-	time.Sleep(time.Second)
-
-	process.Kill()
-
-	process, err = os.StartProcess("etcd", args, procAttr)
-	defer process.Kill()
-	if err != nil {
-		t.Fatal("start process failed:" + err.Error())
-		return
-	}
-
-	time.Sleep(time.Second)
-
-	results, err := c.Get("foo")
-	if err != nil {
-		t.Fatal("get fail: " + err.Error())
-		return
-	}
-
-	result = results[0]
-
-	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.TTL > 99 {
-		if err != nil {
-			t.Fatal(err)
-		}
-		t.Fatalf("Recovery Get failed with %s %s %v", result.Key, result.Value, result.TTL)
-	}
-}
-
-// Create a three nodes and try to set value
-func templateTestSimpleMultiNode(t *testing.T, tls bool) {
-	procAttr := new(os.ProcAttr)
-	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
-
-	clusterSize := 3
-
-	_, etcds, err := createCluster(clusterSize, procAttr, tls)
-
-	if err != nil {
-		t.Fatal("cannot create cluster")
-	}
-
-	defer destroyCluster(etcds)
-
-	time.Sleep(time.Second)
-
-	c := etcd.NewClient()
-
-	c.SyncCluster()
-
-	// Test Set
-	result, err := c.Set("foo", "bar", 100)
-
-	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.TTL != 99 {
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		t.Fatalf("Set 1 failed with %s %s %v", result.Key, result.Value, result.TTL)
-	}
-
-	time.Sleep(time.Second)
-
-	result, err = c.Set("foo", "bar", 100)
-
-	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.PrevValue != "bar" || result.TTL != 99 {
-		if err != nil {
-			t.Fatal(err)
-		}
-		t.Fatalf("Set 2 failed with %s %s %v", result.Key, result.Value, result.TTL)
-	}
-
-}
-
-func TestSimpleMultiNode(t *testing.T) {
-	templateTestSimpleMultiNode(t, false)
-}
-
-func TestSimpleMultiNodeTls(t *testing.T) {
-	templateTestSimpleMultiNode(t, true)
-}
-
-// Create a five nodes
-// Randomly kill one of the node and keep on sending set command to the cluster
-func TestMultiNodeRecovery(t *testing.T) {
-	procAttr := new(os.ProcAttr)
-	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
-
-	clusterSize := 5
-	argGroup, etcds, err := createCluster(clusterSize, procAttr, false)
-
-	if err != nil {
-		t.Fatal("cannot create cluster")
-	}
-
-	defer destroyCluster(etcds)
-
-	time.Sleep(2 * time.Second)
-
-	c := etcd.NewClient()
-
-	c.SyncCluster()
-
-	stop := make(chan bool)
-	// Test Set
-	go set(stop)
-
-	for i := 0; i < 10; i++ {
-		num := rand.Int() % clusterSize
-		fmt.Println("kill node", num+1)
-
-		// kill
-		etcds[num].Kill()
-		etcds[num].Release()
-		time.Sleep(time.Second)
-
-		// restart
-		etcds[num], err = os.StartProcess("etcd", argGroup[num], procAttr)
-		if err != nil {
-			panic(err)
-		}
-		time.Sleep(time.Second)
-	}
-	fmt.Println("stop")
-	stop <- true
-	<-stop
-}
--- a/go_version.go
+++ b/go_version.go
@ -0,0 +1,3 @@
+// +build !go1.1
+
+"etcd requires go 1.1 or greater to build"
--- a/log/log.go
+++ b/log/log.go
@ -0,0 +1,44 @@
+package log
+
+import (
+	golog "github.com/coreos/go-log/log"
+	"os"
+)
+
+// The Verbose flag turns on verbose logging.
+var Verbose bool = false
+
+var logger *golog.Logger = golog.New("etcd", false,
+	golog.CombinedSink(os.Stdout, "[%s] %s %-9s | %s\n", []string{"prefix", "time", "priority", "message"}))
+
+func Infof(format string, v ...interface{}) {
+	logger.Infof(format, v...)
+}
+
+func Debugf(format string, v ...interface{}) {
+	if Verbose {
+		logger.Debugf(format, v...)
+	}
+}
+
+func Debug(v ...interface{}) {
+	if Verbose {
+		logger.Debug(v...)
+	}
+}
+
+func Warnf(format string, v ...interface{}) {
+	logger.Warningf(format, v...)
+}
+
+func Warn(v ...interface{}) {
+	logger.Warning(v...)
+}
+
+func Fatalf(format string, v ...interface{}) {
+	logger.Fatalf(format, v...)
+}
+
+func Fatal(v ...interface{}) {
+	logger.Fatalln(v...)
+}
--- a/machines.go
+++ b/machines.go
@ -1,27 +0,0 @@
-package main
-
-import (
-	"net/url"
-	"path"
-)
-
-func getEtcdURL(name string) (string, bool) {
-	resps, _ := etcdStore.RawGet(path.Join("_etcd/machines", name))
-
-	m, err := url.ParseQuery(resps[0].Value)
-
-	if err != nil {
-		panic("Failed to parse machines entry")
-	}
-
-	addr := m["etcd"][0]
-
-	return addr, true
-}
-
-// machineNum returns the number of machines in the cluster
-func machineNum() int {
-	response, _ := etcdStore.RawGet("_etcd/machines")
-
-	return len(response)
-}
--- a/name_url_map.go
+++ b/name_url_map.go
@ -1,68 +0,0 @@
-package main
-
-import (
-	"net/url"
-	"path"
-)
-
-// we map node name to url
-type nodeInfo struct {
-	raftURL string
-	etcdURL string
-}
-
-var namesMap = make(map[string]*nodeInfo)
-
-// nameToEtcdURL maps node name to its etcd http address
-func nameToEtcdURL(name string) (string, bool) {
-
-	if info, ok := namesMap[name]; ok {
-		// first try to read from the map
-		return info.etcdURL, true
-	}
-
-	// if fails, try to recover from etcd storage
-	return readURL(name, "etcd")
-
-}
-
-// nameToRaftURL maps node name to its raft http address
-func nameToRaftURL(name string) (string, bool) {
-	if info, ok := namesMap[name]; ok {
-		// first try to read from the map
-		return info.raftURL, true
-
-	}
-
-	// if fails, try to recover from etcd storage
-	return readURL(name, "raft")
-}
-
-// addNameToURL add a name that maps to raftURL and etcdURL
-func addNameToURL(name string, raftURL string, etcdURL string) {
-	namesMap[name] = &nodeInfo{
-		raftURL: raftURL,
-		etcdURL: etcdURL,
-	}
-}
-
-func readURL(nodeName string, urlName string) (string, bool) {
-	// if fails, try to recover from etcd storage
-	key := path.Join("/_etcd/machines", nodeName)
-
-	resps, err := etcdStore.RawGet(key)
-
-	if err != nil {
-		return "", false
-	}
-
-	m, err := url.ParseQuery(resps[0].Value)
-
-	if err != nil {
-		panic("Failed to parse machines entry")
-	}
-
-	url := m[urlName][0]
-
-	return url, true
-}
--- a/raft_handlers.go
+++ b/raft_handlers.go
@ -1,115 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"github.com/coreos/go-raft"
-	"net/http"
-)
-
-//-------------------------------------------------------------
-// Handlers to handle raft related request via raft server port
-//-------------------------------------------------------------
-
-// Get all the current logs
-func GetLogHttpHandler(w http.ResponseWriter, req *http.Request) {
-	debugf("[recv] GET %s/log", raftTransporter.scheme+raftServer.Name())
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(http.StatusOK)
-	json.NewEncoder(w).Encode(raftServer.LogEntries())
-}
-
-// Response to vote request
-func VoteHttpHandler(w http.ResponseWriter, req *http.Request) {
-	rvreq := &raft.RequestVoteRequest{}
-	err := decodeJsonRequest(req, rvreq)
-	if err == nil {
-		debugf("[recv] POST %s/vote [%s]", raftTransporter.scheme+raftServer.Name(), rvreq.CandidateName)
-		if resp := raftServer.RequestVote(rvreq); resp != nil {
-			w.WriteHeader(http.StatusOK)
-			json.NewEncoder(w).Encode(resp)
-			return
-		}
-	}
-	warnf("[vote] ERROR: %v", err)
-	w.WriteHeader(http.StatusInternalServerError)
-}
-
-// Response to append entries request
-func AppendEntriesHttpHandler(w http.ResponseWriter, req *http.Request) {
-	aereq := &raft.AppendEntriesRequest{}
-	err := decodeJsonRequest(req, aereq)
-
-	if err == nil {
-		debugf("[recv] POST %s/log/append [%d]", raftTransporter.scheme+raftServer.Name(), len(aereq.Entries))
-		if resp := raftServer.AppendEntries(aereq); resp != nil {
-			w.WriteHeader(http.StatusOK)
-			json.NewEncoder(w).Encode(resp)
-			if !resp.Success {
-				debugf("[Append Entry] Step back")
-			}
-			return
-		}
-	}
-	warnf("[Append Entry] ERROR: %v", err)
-	w.WriteHeader(http.StatusInternalServerError)
-}
-
-// Response to recover from snapshot request
-func SnapshotHttpHandler(w http.ResponseWriter, req *http.Request) {
-	aereq := &raft.SnapshotRequest{}
-	err := decodeJsonRequest(req, aereq)
-	if err == nil {
-		debugf("[recv] POST %s/snapshot/ ", raftTransporter.scheme+raftServer.Name())
-		if resp := raftServer.RequestSnapshot(aereq); resp != nil {
-			w.WriteHeader(http.StatusOK)
-			json.NewEncoder(w).Encode(resp)
-			return
-		}
-	}
-	warnf("[Snapshot] ERROR: %v", err)
-	w.WriteHeader(http.StatusInternalServerError)
-}
-
-// Response to recover from snapshot request
-func SnapshotRecoveryHttpHandler(w http.ResponseWriter, req *http.Request) {
-	aereq := &raft.SnapshotRecoveryRequest{}
-	err := decodeJsonRequest(req, aereq)
-	if err == nil {
-		debugf("[recv] POST %s/snapshotRecovery/ ", raftTransporter.scheme+raftServer.Name())
-		if resp := raftServer.SnapshotRecoveryRequest(aereq); resp != nil {
-			w.WriteHeader(http.StatusOK)
-			json.NewEncoder(w).Encode(resp)
-			return
-		}
-	}
-	warnf("[Snapshot] ERROR: %v", err)
-	w.WriteHeader(http.StatusInternalServerError)
-}
-
-// Get the port that listening for etcd connecting of the server
-func EtcdURLHttpHandler(w http.ResponseWriter, req *http.Request) {
-	debugf("[recv] Get %s/etcdURL/ ", raftTransporter.scheme+raftServer.Name())
-	w.WriteHeader(http.StatusOK)
-	w.Write([]byte(argInfo.EtcdURL))
-}
-
-// Response to the join request
-func JoinHttpHandler(w http.ResponseWriter, req *http.Request) {
-
-	command := &JoinCommand{}
-
-	if err := decodeJsonRequest(req, command); err == nil {
-		debugf("Receive Join Request from %s", command.Name)
-		dispatch(command, &w, req, false)
-	} else {
-		w.WriteHeader(http.StatusInternalServerError)
-		return
-	}
-}
-
-// Response to the name request
-func NameHttpHandler(w http.ResponseWriter, req *http.Request) {
-	debugf("[recv] Get %s/name/ ", raftTransporter.scheme+raftServer.Name())
-	w.WriteHeader(http.StatusOK)
-	w.Write([]byte(raftServer.Name()))
-}
--- a/scripts/release-version
+++ b/scripts/release-version
@ -3,6 +3,6 @@
 VER=$(git describe --tags HEAD)

 cat <<EOF
-package main
-const releaseVersion = "$VER"
+package server
+const ReleaseVersion = "$VER"
 EOF
--- a/scripts/release-version.ps1
+++ b/scripts/release-version.ps1
@ -0,0 +1,7 @@
+
+$VER=(git describe --tags HEAD)
+
+@"
+package main
+const releaseVersion = "$VER"
+"@
--- a/scripts/test-cluster
+++ b/scripts/test-cluster
@ -0,0 +1,19 @@
+#!/bin/bash
+SESSION=etcd-cluster
+
+tmux new-session -d -s $SESSION
+
+# Setup a window for tailing log files
+tmux new-window -t $SESSION:1 -n 'machines'
+tmux split-window -h
+tmux select-pane -t 0
+tmux send-keys "./etcd -s 127.0.0.1:7001 -c 127.0.0.1:4001 -d machine1 -n machine1" C-m
+
+for i in 2 3; do
+	tmux select-pane -t 0
+	tmux split-window -v
+	tmux send-keys "./etcd -cors='*' -s 127.0.0.1:700${i} -c 127.0.0.1:400${i} -C 127.0.0.1:7001 -d machine${i} -n machine${i}" C-m
+done
+
+# Attach to session
+tmux attach-session -t $SESSION
--- a/server/join_command.go
+++ b/server/join_command.go
@ -0,0 +1,75 @@
+package server
+
+import (
+	"encoding/binary"
+
+	etcdErr "github.com/coreos/etcd/error"
+	"github.com/coreos/etcd/log"
+	"github.com/coreos/go-raft"
+)
+
+func init() {
+	raft.RegisterCommand(&JoinCommand{})
+}
+
+// The JoinCommand adds a node to the cluster.
+type JoinCommand struct {
+	RaftVersion string `json:"raftVersion"`
+	Name        string `json:"name"`
+	RaftURL     string `json:"raftURL"`
+	EtcdURL     string `json:"etcdURL"`
+}
+
+func NewJoinCommand(version, name, raftUrl, etcdUrl string) *JoinCommand {
+	return &JoinCommand{
+		RaftVersion: version,
+		Name:        name,
+		RaftURL:     raftUrl,
+		EtcdURL:     etcdUrl,
+	}
+}
+
+// The name of the join command in the log
+func (c *JoinCommand) CommandName() string {
+	return "etcd:join"
+}
+
+// Join a server to the cluster
+func (c *JoinCommand) Apply(server raft.Server) (interface{}, error) {
+	ps, _ := server.Context().(*PeerServer)
+
+	b := make([]byte, 8)
+	binary.PutUvarint(b, server.CommitIndex())
+
+	// Make sure we're not getting a cached value from the registry.
+	ps.registry.Invalidate(c.Name)
+
+	// Check if the join command is from a previous machine, who lost all its previous log.
+	if _, ok := ps.registry.ClientURL(c.Name); ok {
+		return b, nil
+	}
+
+	// Check machine number in the cluster
+	if ps.registry.Count() == ps.MaxClusterSize {
+		log.Debug("Reject join request from ", c.Name)
+		return []byte{0}, etcdErr.NewError(etcdErr.EcodeNoMoreMachine, "", server.CommitIndex(), server.Term())
+	}
+
+	// Add to shared machine registry.
+	ps.registry.Register(c.Name, c.RaftVersion, c.RaftURL, c.EtcdURL, server.CommitIndex(), server.Term())
+
+	// Add peer in raft
+	err := server.AddPeer(c.Name, "")
+
+	// Add peer stats
+	if c.Name != ps.RaftServer().Name() {
+		ps.followersStats.Followers[c.Name] = &raftFollowerStats{}
+		ps.followersStats.Followers[c.Name].Latency.Minimum = 1 << 63
+	}
+
+	return b, err
+}
+
+func (c *JoinCommand) NodeName() string {
+	return c.Name
+}
--- a/server/package_stats.go
+++ b/server/package_stats.go
@ -0,0 +1,25 @@
+package server
+
+import (
+	"time"
+)
+
+// packageStats represent the stats we need for a package.
+// It has sending time and the size of the package.
+type packageStats struct {
+	sendingTime time.Time
+	size        int
+}
+
+// NewPackageStats creates a pacakgeStats and return the pointer to it.
+func NewPackageStats(now time.Time, size int) *packageStats {
+	return &packageStats{
+		sendingTime: now,
+		size:        size,
+	}
+}
+
+// Time return the sending time of the package.
+func (ps *packageStats) Time() time.Time {
+	return ps.sendingTime
+}
--- a/server/peer_server.go
+++ b/server/peer_server.go
@ -0,0 +1,396 @@
+package server
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	etcdErr "github.com/coreos/etcd/error"
+	"github.com/coreos/etcd/log"
+	"github.com/coreos/etcd/store"
+	"github.com/coreos/go-raft"
+)
+
+type PeerServer struct {
+	raftServer     raft.Server
+	server         *Server
+	joinIndex      uint64
+	name           string
+	url            string
+	listenHost     string
+	tlsConf        *TLSConfig
+	tlsInfo        *TLSInfo
+	followersStats *raftFollowersStats
+	serverStats    *raftServerStats
+	registry       *Registry
+	store          store.Store
+	snapConf       *snapshotConf
+	MaxClusterSize int
+	RetryTimes     int
+}
+
+// TODO: find a good policy to do snapshot
+type snapshotConf struct {
+	// Etcd will check if snapshot is need every checkingInterval
+	checkingInterval time.Duration
+
+	// The number of writes when the last snapshot happened
+	lastWrites uint64
+
+	// If the incremental number of writes since the last snapshot
+	// exceeds the write Threshold, etcd will do a snapshot
+	writesThr uint64
+}
+
+func NewPeerServer(name string, path string, url string, listenHost string, tlsConf *TLSConfig, tlsInfo *TLSInfo, registry *Registry, store store.Store) *PeerServer {
+	s := &PeerServer{
+		name:       name,
+		url:        url,
+		listenHost: listenHost,
+		tlsConf:    tlsConf,
+		tlsInfo:    tlsInfo,
+		registry:   registry,
+		store:      store,
+		snapConf:   &snapshotConf{time.Second * 3, 0, 20 * 1000},
+		followersStats: &raftFollowersStats{
+			Leader:    name,
+			Followers: make(map[string]*raftFollowerStats),
+		},
+		serverStats: &raftServerStats{
+			StartTime: time.Now(),
+			sendRateQueue: &statsQueue{
+				back: -1,
+			},
+			recvRateQueue: &statsQueue{
+				back: -1,
+			},
+		},
+	}
+
+	// Create transporter for raft
+	raftTransporter := newTransporter(tlsConf.Scheme, tlsConf.Client, s)
+
+	// Create raft server
+	raftServer, err := raft.NewServer(name, path, raftTransporter, s.store, s, "")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	s.raftServer = raftServer
+
+	return s
+}
+
+// Start the raft server
+func (s *PeerServer) ListenAndServe(snapshot bool, cluster []string) {
+	// LoadSnapshot
+	if snapshot {
+		err := s.raftServer.LoadSnapshot()
+
+		if err == nil {
+			log.Debugf("%s finished load snapshot", s.name)
+		} else {
+			log.Debug(err)
+		}
+	}
+
+	s.raftServer.SetElectionTimeout(ElectionTimeout)
+	s.raftServer.SetHeartbeatTimeout(HeartbeatTimeout)
+
+	s.raftServer.Start()
+
+	if s.raftServer.IsLogEmpty() {
+		// start as a leader in a new cluster
+		if len(cluster) == 0 {
+			s.startAsLeader()
+		} else {
+			s.startAsFollower(cluster)
+		}
+
+	} else {
+		// Rejoin the previous cluster
+		cluster = s.registry.PeerURLs(s.raftServer.Leader(), s.name)
+		for i := 0; i < len(cluster); i++ {
+			u, err := url.Parse(cluster[i])
+			if err != nil {
+				log.Debug("rejoin cannot parse url: ", err)
+			}
+			cluster[i] = u.Host
+		}
+		ok := s.joinCluster(cluster)
+		if !ok {
+			log.Warn("the entire cluster is down! this machine will restart the cluster.")
+		}
+
+		log.Debugf("%s restart as a follower", s.name)
+	}
+
+	// open the snapshot
+	if snapshot {
+		go s.monitorSnapshot()
+	}
+
+	// start to response to raft requests
+	go s.startTransport(s.tlsConf.Scheme, s.tlsConf.Server)
+
+}
+
+// Retrieves the underlying Raft server.
+func (s *PeerServer) RaftServer() raft.Server {
+	return s.raftServer
+}
+
+// Associates the client server with the peer server.
+func (s *PeerServer) SetServer(server *Server) {
+	s.server = server
+}
+
+func (s *PeerServer) startAsLeader() {
+	// leader need to join self as a peer
+	for {
+		_, err := s.raftServer.Do(NewJoinCommand(PeerVersion, s.raftServer.Name(), s.url, s.server.URL()))
+		if err == nil {
+			break
+		}
+	}
+	log.Debugf("%s start as a leader", s.name)
+}
+
+func (s *PeerServer) startAsFollower(cluster []string) {
+	// start as a follower in a existing cluster
+	for i := 0; i < s.RetryTimes; i++ {
+		ok := s.joinCluster(cluster)
+		if ok {
+			return
+		}
+		log.Warnf("cannot join to cluster via given machines, retry in %d seconds", RetryInterval)
+		time.Sleep(time.Second * RetryInterval)
+	}
+
+	log.Fatalf("Cannot join the cluster via given machines after %x retries", s.RetryTimes)
+}
+
+// Start to listen and response raft command
+func (s *PeerServer) startTransport(scheme string, tlsConf tls.Config) {
+	log.Infof("raft server [name %s, listen on %s, advertised url %s]", s.name, s.listenHost, s.url)
+
+	raftMux := http.NewServeMux()
+
+	server := &http.Server{
+		Handler:   raftMux,
+		TLSConfig: &tlsConf,
+		Addr:      s.listenHost,
+	}
+
+	// internal commands
+	raftMux.HandleFunc("/name", s.NameHttpHandler)
+	raftMux.HandleFunc("/version", s.RaftVersionHttpHandler)
+	raftMux.HandleFunc("/join", s.JoinHttpHandler)
+	raftMux.HandleFunc("/remove/", s.RemoveHttpHandler)
+	raftMux.HandleFunc("/vote", s.VoteHttpHandler)
+	raftMux.HandleFunc("/log", s.GetLogHttpHandler)
+	raftMux.HandleFunc("/log/append", s.AppendEntriesHttpHandler)
+	raftMux.HandleFunc("/snapshot", s.SnapshotHttpHandler)
+	raftMux.HandleFunc("/snapshotRecovery", s.SnapshotRecoveryHttpHandler)
+	raftMux.HandleFunc("/etcdURL", s.EtcdURLHttpHandler)
+
+	if scheme == "http" {
+		log.Fatal(server.ListenAndServe())
+	} else {
+		log.Fatal(server.ListenAndServeTLS(s.tlsInfo.CertFile, s.tlsInfo.KeyFile))
+	}
+
+}
+
+// getVersion fetches the raft version of a peer. This works for now but we
+// will need to do something more sophisticated later when we allow mixed
+// version clusters.
+func getVersion(t *transporter, versionURL url.URL) (string, error) {
+	resp, req, err := t.Get(versionURL.String())
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	t.CancelWhenTimeout(req)
+
+	body, err := ioutil.ReadAll(resp.Body)
+
+	return string(body), nil
+}
+
+func (s *PeerServer) joinCluster(cluster []string) bool {
+	for _, machine := range cluster {
+		if len(machine) == 0 {
+			continue
+		}
+
+		err := s.joinByMachine(s.raftServer, machine, s.tlsConf.Scheme)
+		if err == nil {
+			log.Debugf("%s success join to the cluster via machine %s", s.name, machine)
+			return true
+
+		} else {
+			if _, ok := err.(etcdErr.Error); ok {
+				log.Fatal(err)
+			}
+
+			log.Debugf("cannot join to cluster via machine %s %s", machine, err)
+		}
+	}
+	return false
+}
+
+// Send join requests to machine.
+func (s *PeerServer) joinByMachine(server raft.Server, machine string, scheme string) error {
+	var b bytes.Buffer
+
+	// t must be ok
+	t, _ := server.Transporter().(*transporter)
+
+	// Our version must match the leaders version
+	versionURL := url.URL{Host: machine, Scheme: scheme, Path: "/version"}
+	version, err := getVersion(t, versionURL)
+	if err != nil {
+		return fmt.Errorf("Error during join version check: %v", err)
+	}
+
+	// TODO: versioning of the internal protocol. See:
+	// Documentation/internatl-protocol-versioning.md
+	if version != PeerVersion {
+		return fmt.Errorf("Unable to join: internal version mismatch, entire cluster must be running identical versions of etcd")
+	}
+
+	json.NewEncoder(&b).Encode(NewJoinCommand(PeerVersion, server.Name(), s.url, s.server.URL()))
+
+	joinURL := url.URL{Host: machine, Scheme: scheme, Path: "/join"}
+
+	log.Debugf("Send Join Request to %s", joinURL.String())
+
+	resp, req, err := t.Post(joinURL.String(), &b)
+
+	for {
+		if err != nil {
+			return fmt.Errorf("Unable to join: %v", err)
+		}
+		if resp != nil {
+			defer resp.Body.Close()
+
+			t.CancelWhenTimeout(req)
+
+			if resp.StatusCode == http.StatusOK {
+				b, _ := ioutil.ReadAll(resp.Body)
+				s.joinIndex, _ = binary.Uvarint(b)
+				return nil
+			}
+			if resp.StatusCode == http.StatusTemporaryRedirect {
+				address := resp.Header.Get("Location")
+				log.Debugf("Send Join Request to %s", address)
+				json.NewEncoder(&b).Encode(NewJoinCommand(PeerVersion, server.Name(), s.url, s.server.URL()))
+				resp, req, err = t.Post(address, &b)
+
+			} else if resp.StatusCode == http.StatusBadRequest {
+				log.Debug("Reach max number machines in the cluster")
+				decoder := json.NewDecoder(resp.Body)
+				err := &etcdErr.Error{}
+				decoder.Decode(err)
+				return *err
+			} else {
+				return fmt.Errorf("Unable to join")
+			}
+		}
+
+	}
+}
+
+func (s *PeerServer) Stats() []byte {
+	s.serverStats.LeaderInfo.Uptime = time.Now().Sub(s.serverStats.LeaderInfo.startTime).String()
+
+	queue := s.serverStats.sendRateQueue
+
+	s.serverStats.SendingPkgRate, s.serverStats.SendingBandwidthRate = queue.Rate()
+
+	queue = s.serverStats.recvRateQueue
+
+	s.serverStats.RecvingPkgRate, s.serverStats.RecvingBandwidthRate = queue.Rate()
+
+	b, _ := json.Marshal(s.serverStats)
+
+	return b
+}
+
+func (s *PeerServer) PeerStats() []byte {
+	if s.raftServer.State() == raft.Leader {
+		b, _ := json.Marshal(s.followersStats)
+		return b
+	}
+	return nil
+}
+
+func (s *PeerServer) monitorSnapshot() {
+	for {
+		time.Sleep(s.snapConf.checkingInterval)
+		currentWrites := 0
+		if uint64(currentWrites) > s.snapConf.writesThr {
+			s.raftServer.TakeSnapshot()
+			s.snapConf.lastWrites = 0
+		}
+	}
+}
+
+func (s *PeerServer) dispatch(c raft.Command, w http.ResponseWriter, req *http.Request) error {
+	if s.raftServer.State() == raft.Leader {
+		result, err := s.raftServer.Do(c)
+		if err != nil {
+			return err
+		}
+
+		if result == nil {
+			return etcdErr.NewError(300, "Empty result from raft", store.UndefIndex, store.UndefTerm)
+		}
+
+		// response for raft related commands[join/remove]
+		if b, ok := result.([]byte); ok {
+			w.WriteHeader(http.StatusOK)
+			w.Write(b)
+			return nil
+		}
+
+		var b []byte
+		if strings.HasPrefix(req.URL.Path, "/v1") {
+			b, _ = json.Marshal(result.(*store.Event).Response())
+		} else {
+			b, _ = json.Marshal(result.(*store.Event))
+		}
+		w.WriteHeader(http.StatusOK)
+		w.Write(b)
+
+		return nil
+
+	} else {
+		leader := s.raftServer.Leader()
+
+		// No leader available.
+		if leader == "" {
+			return etcdErr.NewError(300, "", store.UndefIndex, store.UndefTerm)
+		}
+
+		var url string
+		switch c.(type) {
+		case *JoinCommand, *RemoveCommand:
+			url, _ = s.registry.PeerURL(leader)
+		default:
+			url, _ = s.registry.ClientURL(leader)
+		}
+		redirect(url, w, req)
+
+		return nil
+	}
+}
--- a/server/peer_server_handlers.go
+++ b/server/peer_server_handlers.go
@ -0,0 +1,158 @@
+package server
+
+import (
+	"encoding/json"
+	"net/http"
+
+	etcdErr "github.com/coreos/etcd/error"
+	"github.com/coreos/etcd/log"
+	"github.com/coreos/go-raft"
+)
+
+// Get all the current logs
+func (s *PeerServer) GetLogHttpHandler(w http.ResponseWriter, req *http.Request) {
+	log.Debugf("[recv] GET %s/log", s.url)
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+	json.NewEncoder(w).Encode(s.raftServer.LogEntries())
+}
+
+// Response to vote request
+func (s *PeerServer) VoteHttpHandler(w http.ResponseWriter, req *http.Request) {
+	rvreq := &raft.RequestVoteRequest{}
+	err := decodeJsonRequest(req, rvreq)
+	if err == nil {
+		log.Debugf("[recv] POST %s/vote [%s]", s.url, rvreq.CandidateName)
+		if resp := s.raftServer.RequestVote(rvreq); resp != nil {
+			w.WriteHeader(http.StatusOK)
+			json.NewEncoder(w).Encode(resp)
+			return
+		}
+	}
+	log.Warnf("[vote] ERROR: %v", err)
+	w.WriteHeader(http.StatusInternalServerError)
+}
+
+// Response to append entries request
+func (s *PeerServer) AppendEntriesHttpHandler(w http.ResponseWriter, req *http.Request) {
+	aereq := &raft.AppendEntriesRequest{}
+	err := decodeJsonRequest(req, aereq)
+
+	if err == nil {
+		log.Debugf("[recv] POST %s/log/append [%d]", s.url, len(aereq.Entries))
+
+		s.serverStats.RecvAppendReq(aereq.LeaderName, int(req.ContentLength))
+
+		if resp := s.raftServer.AppendEntries(aereq); resp != nil {
+			w.WriteHeader(http.StatusOK)
+			json.NewEncoder(w).Encode(resp)
+			if !resp.Success {
+				log.Debugf("[Append Entry] Step back")
+			}
+			return
+		}
+	}
+	log.Warnf("[Append Entry] ERROR: %v", err)
+	w.WriteHeader(http.StatusInternalServerError)
+}
+
+// Response to recover from snapshot request
+func (s *PeerServer) SnapshotHttpHandler(w http.ResponseWriter, req *http.Request) {
+	aereq := &raft.SnapshotRequest{}
+	err := decodeJsonRequest(req, aereq)
+	if err == nil {
+		log.Debugf("[recv] POST %s/snapshot/ ", s.url)
+		if resp := s.raftServer.RequestSnapshot(aereq); resp != nil {
+			w.WriteHeader(http.StatusOK)
+			json.NewEncoder(w).Encode(resp)
+			return
+		}
+	}
+	log.Warnf("[Snapshot] ERROR: %v", err)
+	w.WriteHeader(http.StatusInternalServerError)
+}
+
+// Response to recover from snapshot request
+func (s *PeerServer) SnapshotRecoveryHttpHandler(w http.ResponseWriter, req *http.Request) {
+	aereq := &raft.SnapshotRecoveryRequest{}
+	err := decodeJsonRequest(req, aereq)
+	if err == nil {
+		log.Debugf("[recv] POST %s/snapshotRecovery/ ", s.url)
+		if resp := s.raftServer.SnapshotRecoveryRequest(aereq); resp != nil {
+			w.WriteHeader(http.StatusOK)
+			json.NewEncoder(w).Encode(resp)
+			return
+		}
+	}
+	log.Warnf("[Snapshot] ERROR: %v", err)
+	w.WriteHeader(http.StatusInternalServerError)
+}
+
+// Get the port that listening for etcd connecting of the server
+func (s *PeerServer) EtcdURLHttpHandler(w http.ResponseWriter, req *http.Request) {
+	log.Debugf("[recv] Get %s/etcdURL/ ", s.url)
+	w.WriteHeader(http.StatusOK)
+	w.Write([]byte(s.server.URL()))
+}
+
+// Response to the join request
+func (s *PeerServer) JoinHttpHandler(w http.ResponseWriter, req *http.Request) {
+	command := &JoinCommand{}
+
+	// Write CORS header.
+	if s.server.OriginAllowed("*") {
+		w.Header().Add("Access-Control-Allow-Origin", "*")
+	} else if s.server.OriginAllowed(req.Header.Get("Origin")) {
+		w.Header().Add("Access-Control-Allow-Origin", req.Header.Get("Origin"))
+	}
+
+	err := decodeJsonRequest(req, command)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	log.Debugf("Receive Join Request from %s", command.Name)
+	err = s.dispatch(command, w, req)
+
+	// Return status.
+	if err != nil {
+		if etcdErr, ok := err.(*etcdErr.Error); ok {
+			log.Debug("Return error: ", (*etcdErr).Error())
+			etcdErr.Write(w)
+		} else {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+		}
+	}
+}
+
+// Response to remove request
+func (s *PeerServer) RemoveHttpHandler(w http.ResponseWriter, req *http.Request) {
+	if req.Method != "DELETE" {
+		w.WriteHeader(http.StatusMethodNotAllowed)
+		return
+	}
+
+	nodeName := req.URL.Path[len("/remove/"):]
+	command := &RemoveCommand{
+		Name: nodeName,
+	}
+
+	log.Debugf("[recv] Remove Request [%s]", command.Name)
+
+	s.dispatch(command, w, req)
+}
+
+// Response to the name request
+func (s *PeerServer) NameHttpHandler(w http.ResponseWriter, req *http.Request) {
+	log.Debugf("[recv] Get %s/name/ ", s.url)
+	w.WriteHeader(http.StatusOK)
+	w.Write([]byte(s.name))
+}
+
+// Response to the name request
+func (s *PeerServer) RaftVersionHttpHandler(w http.ResponseWriter, req *http.Request) {
+	log.Debugf("[recv] Get %s/version/ ", s.url)
+	w.WriteHeader(http.StatusOK)
+	w.Write([]byte(PeerVersion))
+}
--- a/server/raft_follower_stats.go
+++ b/server/raft_follower_stats.go
@ -0,0 +1,56 @@
+package server
+
+import (
+	"math"
+	"time"
+)
+
+type raftFollowersStats struct {
+	Leader    string                        `json:"leader"`
+	Followers map[string]*raftFollowerStats `json:"followers"`
+}
+
+type raftFollowerStats struct {
+	Latency struct {
+		Current           float64 `json:"current"`
+		Average           float64 `json:"average"`
+		averageSquare     float64
+		StandardDeviation float64 `json:"standardDeviation"`
+		Minimum           float64 `json:"minimum"`
+		Maximum           float64 `json:"maximum"`
+	} `json:"latency"`
+
+	Counts struct {
+		Fail    uint64 `json:"fail"`
+		Success uint64 `json:"success"`
+	} `json:"counts"`
+}
+
+// Succ function update the raftFollowerStats with a successful send
+func (ps *raftFollowerStats) Succ(d time.Duration) {
+	total := float64(ps.Counts.Success) * ps.Latency.Average
+	totalSquare := float64(ps.Counts.Success) * ps.Latency.averageSquare
+
+	ps.Counts.Success++
+
+	ps.Latency.Current = float64(d) / (1000000.0)
+
+	if ps.Latency.Current > ps.Latency.Maximum {
+		ps.Latency.Maximum = ps.Latency.Current
+	}
+
+	if ps.Latency.Current < ps.Latency.Minimum {
+		ps.Latency.Minimum = ps.Latency.Current
+	}
+
+	ps.Latency.Average = (total + ps.Latency.Current) / float64(ps.Counts.Success)
+	ps.Latency.averageSquare = (totalSquare + ps.Latency.Current*ps.Latency.Current) / float64(ps.Counts.Success)
+
+	// sdv = sqrt(avg(x^2) - avg(x)^2)
+	ps.Latency.StandardDeviation = math.Sqrt(ps.Latency.averageSquare - ps.Latency.Average*ps.Latency.Average)
+}
+
+// Fail function update the raftFollowerStats with a unsuccessful send
+func (ps *raftFollowerStats) Fail() {
+	ps.Counts.Fail++
+}
--- a/server/raft_server_stats.go
+++ b/server/raft_server_stats.go
@ -0,0 +1,55 @@
+package server
+
+import (
+	"time"
+
+	"github.com/coreos/go-raft"
+)
+
+type raftServerStats struct {
+	Name      string    `json:"name"`
+	State     string    `json:"state"`
+	StartTime time.Time `json:"startTime"`
+
+	LeaderInfo struct {
+		Name      string `json:"leader"`
+		Uptime    string `json:"uptime"`
+		startTime time.Time
+	} `json:"leaderInfo"`
+
+	RecvAppendRequestCnt uint64  `json:"recvAppendRequestCnt,"`
+	RecvingPkgRate       float64 `json:"recvPkgRate,omitempty"`
+	RecvingBandwidthRate float64 `json:"recvBandwidthRate,omitempty"`
+
+	SendAppendRequestCnt uint64  `json:"sendAppendRequestCnt"`
+	SendingPkgRate       float64 `json:"sendPkgRate,omitempty"`
+	SendingBandwidthRate float64 `json:"sendBandwidthRate,omitempty"`
+
+	sendRateQueue *statsQueue
+	recvRateQueue *statsQueue
+}
+
+func (ss *raftServerStats) RecvAppendReq(leaderName string, pkgSize int) {
+	ss.State = raft.Follower
+	if leaderName != ss.LeaderInfo.Name {
+		ss.LeaderInfo.Name = leaderName
+		ss.LeaderInfo.startTime = time.Now()
+	}
+
+	ss.recvRateQueue.Insert(NewPackageStats(time.Now(), pkgSize))
+	ss.RecvAppendRequestCnt++
+}
+
+func (ss *raftServerStats) SendAppendReq(pkgSize int) {
+	now := time.Now()
+
+	if ss.State != raft.Leader {
+		ss.State = raft.Leader
+		ss.LeaderInfo.Name = ss.Name
+		ss.LeaderInfo.startTime = now
+	}
+
+	ss.sendRateQueue.Insert(NewPackageStats(now, pkgSize))
+
+	ss.SendAppendRequestCnt++
+}
--- a/server/registry.go
+++ b/server/registry.go
@ -0,0 +1,180 @@
+package server
+
+import (
+	"fmt"
+	"net/url"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/coreos/etcd/log"
+	"github.com/coreos/etcd/store"
+)
+
+// The location of the machine URL data.
+const RegistryKey = "/_etcd/machines"
+
+// The Registry stores URL information for nodes.
+type Registry struct {
+	sync.Mutex
+	store store.Store
+	nodes map[string]*node
+}
+
+// The internal storage format of the registry.
+type node struct {
+	peerVersion string
+	peerURL     string
+	url         string
+}
+
+// Creates a new Registry.
+func NewRegistry(s store.Store) *Registry {
+	return &Registry{
+		store: s,
+		nodes: make(map[string]*node),
+	}
+}
+
+// Adds a node to the registry.
+func (r *Registry) Register(name string, peerVersion string, peerURL string, url string, commitIndex uint64, term uint64) error {
+	r.Lock()
+	defer r.Unlock()
+
+	// Write data to store.
+	key := path.Join(RegistryKey, name)
+	value := fmt.Sprintf("raft=%s&etcd=%s&raftVersion=%s", peerURL, url, peerVersion)
+	_, err := r.store.Create(key, value, false, store.Permanent, commitIndex, term)
+	log.Debugf("Register: %s (%v)", name, err)
+	return err
+}
+
+// Removes a node from the registry.
+func (r *Registry) Unregister(name string, commitIndex uint64, term uint64) error {
+	r.Lock()
+	defer r.Unlock()
+
+	// Remove from cache.
+	// delete(r.nodes, name)
+
+	// Remove the key from the store.
+	_, err := r.store.Delete(path.Join(RegistryKey, name), false, commitIndex, term)
+	log.Debugf("Unregister: %s (%v)", name, err)
+	return err
+}
+
+// Returns the number of nodes in the cluster.
+func (r *Registry) Count() int {
+	e, err := r.store.Get(RegistryKey, false, false, 0, 0)
+	if err != nil {
+		return 0
+	}
+	return len(e.KVPairs)
+}
+
+// Retrieves the client URL for a given node by name.
+func (r *Registry) ClientURL(name string) (string, bool) {
+	r.Lock()
+	defer r.Unlock()
+	return r.clientURL(name)
+}
+
+func (r *Registry) clientURL(name string) (string, bool) {
+	if r.nodes[name] == nil {
+		r.load(name)
+	}
+
+	if node := r.nodes[name]; node != nil {
+		return node.url, true
+	}
+
+	return "", false
+}
+
+// Retrieves the peer URL for a given node by name.
+func (r *Registry) PeerURL(name string) (string, bool) {
+	r.Lock()
+	defer r.Unlock()
+	return r.peerURL(name)
+}
+
+func (r *Registry) peerURL(name string) (string, bool) {
+	if r.nodes[name] == nil {
+		r.load(name)
+	}
+
+	if node := r.nodes[name]; node != nil {
+		return node.peerURL, true
+	}
+
+	return "", false
+}
+
+// Retrieves the Client URLs for all nodes.
+func (r *Registry) ClientURLs(leaderName, selfName string) []string {
+	return r.urls(leaderName, selfName, r.clientURL)
+}
+
+// Retrieves the Peer URLs for all nodes.
+func (r *Registry) PeerURLs(leaderName, selfName string) []string {
+	return r.urls(leaderName, selfName, r.peerURL)
+}
+
+// Retrieves the URLs for all nodes using url function.
+func (r *Registry) urls(leaderName, selfName string, url func(name string) (string, bool)) []string {
+	r.Lock()
+	defer r.Unlock()
+
+	// Build list including the leader and self.
+	urls := make([]string, 0)
+	if url, _ := url(leaderName); len(url) > 0 {
+		urls = append(urls, url)
+	}
+
+	// Retrieve a list of all nodes.
+	if e, _ := r.store.Get(RegistryKey, false, false, 0, 0); e != nil {
+		// Lookup the URL for each one.
+		for _, pair := range e.KVPairs {
+			_, name := filepath.Split(pair.Key)
+			if url, _ := url(name); len(url) > 0 && name != leaderName {
+				urls = append(urls, url)
+			}
+		}
+	}
+
+	log.Infof("URLs: %s / %s (%s)", leaderName, selfName, strings.Join(urls, ","))
+
+	return urls
+}
+
+// Removes a node from the cache.
+func (r *Registry) Invalidate(name string) {
+	delete(r.nodes, name)
+}
+
+// Loads the given node by name from the store into the cache.
+func (r *Registry) load(name string) {
+	if name == "" {
+		return
+	}
+
+	// Retrieve from store.
+	e, err := r.store.Get(path.Join(RegistryKey, name), false, false, 0, 0)
+	if err != nil {
+		return
+	}
+
+	// Parse as a query string.
+	m, err := url.ParseQuery(e.Value)
+	if err != nil {
+		panic(fmt.Sprintf("Failed to parse machines entry: %s", name))
+	}
+
+	// Create node.
+	r.nodes[name] = &node{
+		url:         m["etcd"][0],
+		peerURL:     m["raft"][0],
+		peerVersion: m["raftVersion"][0],
+	}
+}
--- a/server/remove_command.go
+++ b/server/remove_command.go
@ -0,0 +1,67 @@
+package server
+
+import (
+	"encoding/binary"
+	"os"
+
+	"github.com/coreos/etcd/log"
+	"github.com/coreos/go-raft"
+)
+
+func init() {
+	raft.RegisterCommand(&RemoveCommand{})
+}
+
+// The RemoveCommand removes a server from the cluster.
+type RemoveCommand struct {
+	Name string `json:"name"`
+}
+
+// The name of the remove command in the log
+func (c *RemoveCommand) CommandName() string {
+	return "etcd:remove"
+}
+
+// Remove a server from the cluster
+func (c *RemoveCommand) Apply(server raft.Server) (interface{}, error) {
+	ps, _ := server.Context().(*PeerServer)
+
+	// Remove node from the shared registry.
+	err := ps.registry.Unregister(c.Name, server.CommitIndex(), server.Term())
+
+	// Delete from stats
+	delete(ps.followersStats.Followers, c.Name)
+
+	if err != nil {
+		log.Debugf("Error while unregistering: %s (%v)", c.Name, err)
+		return []byte{0}, err
+	}
+
+	// Remove peer in raft
+	err = server.RemovePeer(c.Name)
+	if err != nil {
+		log.Debugf("Unable to remove peer: %s (%v)", c.Name, err)
+		return []byte{0}, err
+	}
+
+	if c.Name == server.Name() {
+		// the removed node is this node
+
+		// if the node is not replaying the previous logs
+		// and the node has sent out a join request in this
+		// start. It is sure that this node received a new remove
+		// command and need to be removed
+		if server.CommitIndex() > ps.joinIndex && ps.joinIndex != 0 {
+			log.Debugf("server [%s] is removed", server.Name())
+			os.Exit(0)
+		} else {
+			// else ignore remove
+			log.Debugf("ignore previous remove command.")
+		}
+	}
+
+	b := make([]byte, 8)
+	binary.PutUvarint(b, server.CommitIndex())
+
+	return b, err
+}
--- a/server/server.go
+++ b/server/server.go
@ -0,0 +1,275 @@
+package server
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	etcdErr "github.com/coreos/etcd/error"
+	"github.com/coreos/etcd/log"
+	"github.com/coreos/etcd/server/v1"
+	"github.com/coreos/etcd/server/v2"
+	"github.com/coreos/etcd/store"
+	"github.com/coreos/go-raft"
+	"github.com/gorilla/mux"
+)
+
+// This is the default implementation of the Server interface.
+type Server struct {
+	http.Server
+	peerServer  *PeerServer
+	registry    *Registry
+	store       store.Store
+	name        string
+	url         string
+	tlsConf     *TLSConfig
+	tlsInfo     *TLSInfo
+	corsOrigins map[string]bool
+}
+
+// Creates a new Server.
+func New(name string, urlStr string, listenHost string, tlsConf *TLSConfig, tlsInfo *TLSInfo, peerServer *PeerServer, registry *Registry, store store.Store) *Server {
+	s := &Server{
+		Server: http.Server{
+			Handler:   mux.NewRouter(),
+			TLSConfig: &tlsConf.Server,
+			Addr:      listenHost,
+		},
+		name:       name,
+		store:      store,
+		registry:   registry,
+		url:        urlStr,
+		tlsConf:    tlsConf,
+		tlsInfo:    tlsInfo,
+		peerServer: peerServer,
+	}
+
+	// Install the routes.
+	s.handleFunc("/version", s.GetVersionHandler).Methods("GET")
+	s.installV1()
+	s.installV2()
+
+	return s
+}
+
+// The current state of the server in the cluster.
+func (s *Server) State() string {
+	return s.peerServer.RaftServer().State()
+}
+
+// The node name of the leader in the cluster.
+func (s *Server) Leader() string {
+	return s.peerServer.RaftServer().Leader()
+}
+
+// The current Raft committed index.
+func (s *Server) CommitIndex() uint64 {
+	return s.peerServer.RaftServer().CommitIndex()
+}
+
+// The current Raft term.
+func (s *Server) Term() uint64 {
+	return s.peerServer.RaftServer().Term()
+}
+
+// The server URL.
+func (s *Server) URL() string {
+	return s.url
+}
+
+// Retrives the Peer URL for a given node name.
+func (s *Server) PeerURL(name string) (string, bool) {
+	return s.registry.PeerURL(name)
+}
+
+// Returns a reference to the Store.
+func (s *Server) Store() store.Store {
+	return s.store
+}
+
+func (s *Server) installV1() {
+	s.handleFuncV1("/v1/keys/{key:.*}", v1.GetKeyHandler).Methods("GET")
+	s.handleFuncV1("/v1/keys/{key:.*}", v1.SetKeyHandler).Methods("POST", "PUT")
+	s.handleFuncV1("/v1/keys/{key:.*}", v1.DeleteKeyHandler).Methods("DELETE")
+	s.handleFuncV1("/v1/watch/{key:.*}", v1.WatchKeyHandler).Methods("GET", "POST")
+	s.handleFunc("/v1/leader", s.GetLeaderHandler).Methods("GET")
+	s.handleFunc("/v1/machines", s.GetMachinesHandler).Methods("GET")
+	s.handleFunc("/v1/stats/self", s.GetStatsHandler).Methods("GET")
+	s.handleFunc("/v1/stats/leader", s.GetLeaderStatsHandler).Methods("GET")
+	s.handleFunc("/v1/stats/store", s.GetStoreStatsHandler).Methods("GET")
+}
+
+func (s *Server) installV2() {
+	s.handleFuncV2("/v2/keys/{key:.*}", v2.GetHandler).Methods("GET")
+	s.handleFuncV2("/v2/keys/{key:.*}", v2.PostHandler).Methods("POST")
+	s.handleFuncV2("/v2/keys/{key:.*}", v2.PutHandler).Methods("PUT")
+	s.handleFuncV2("/v2/keys/{key:.*}", v2.DeleteHandler).Methods("DELETE")
+	s.handleFunc("/v2/leader", s.GetLeaderHandler).Methods("GET")
+	s.handleFunc("/v2/machines", s.GetMachinesHandler).Methods("GET")
+	s.handleFunc("/v2/stats/self", s.GetStatsHandler).Methods("GET")
+	s.handleFunc("/v2/stats/leader", s.GetLeaderStatsHandler).Methods("GET")
+	s.handleFunc("/v2/stats/store", s.GetStoreStatsHandler).Methods("GET")
+}
+
+// Adds a v1 server handler to the router.
+func (s *Server) handleFuncV1(path string, f func(http.ResponseWriter, *http.Request, v1.Server) error) *mux.Route {
+	return s.handleFunc(path, func(w http.ResponseWriter, req *http.Request) error {
+		return f(w, req, s)
+	})
+}
+
+// Adds a v2 server handler to the router.
+func (s *Server) handleFuncV2(path string, f func(http.ResponseWriter, *http.Request, v2.Server) error) *mux.Route {
+	return s.handleFunc(path, func(w http.ResponseWriter, req *http.Request) error {
+		return f(w, req, s)
+	})
+}
+
+// Adds a server handler to the router.
+func (s *Server) handleFunc(path string, f func(http.ResponseWriter, *http.Request) error) *mux.Route {
+	r := s.Handler.(*mux.Router)
+
+	// Wrap the standard HandleFunc interface to pass in the server reference.
+	return r.HandleFunc(path, func(w http.ResponseWriter, req *http.Request) {
+		// Log request.
+		log.Debugf("[recv] %s %s %s [%s]", req.Method, s.url, req.URL.Path, req.RemoteAddr)
+
+		// Write CORS header.
+		if s.OriginAllowed("*") {
+			w.Header().Add("Access-Control-Allow-Origin", "*")
+		} else if origin := req.Header.Get("Origin"); s.OriginAllowed(origin) {
+			w.Header().Add("Access-Control-Allow-Origin", origin)
+		}
+
+		// Execute handler function and return error if necessary.
+		if err := f(w, req); err != nil {
+			if etcdErr, ok := err.(*etcdErr.Error); ok {
+				log.Debug("Return error: ", (*etcdErr).Error())
+				etcdErr.Write(w)
+			} else {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+			}
+		}
+	})
+}
+
+// Start to listen and response etcd client command
+func (s *Server) ListenAndServe() {
+	log.Infof("etcd server [name %s, listen on %s, advertised url %s]", s.name, s.Server.Addr, s.url)
+
+	if s.tlsConf.Scheme == "http" {
+		log.Fatal(s.Server.ListenAndServe())
+	} else {
+		log.Fatal(s.Server.ListenAndServeTLS(s.tlsInfo.CertFile, s.tlsInfo.KeyFile))
+	}
+}
+
+func (s *Server) Dispatch(c raft.Command, w http.ResponseWriter, req *http.Request) error {
+	return s.peerServer.dispatch(c, w, req)
+}
+
+// Sets a comma-delimited list of origins that are allowed.
+func (s *Server) AllowOrigins(origins string) error {
+	// Construct a lookup of all origins.
+	m := make(map[string]bool)
+	for _, v := range strings.Split(origins, ",") {
+		if v != "*" {
+			if _, err := url.Parse(v); err != nil {
+				return fmt.Errorf("Invalid CORS origin: %s", err)
+			}
+		}
+		m[v] = true
+	}
+	s.corsOrigins = m
+
+	return nil
+}
+
+// Determines whether the server will allow a given CORS origin.
+func (s *Server) OriginAllowed(origin string) bool {
+	return s.corsOrigins["*"] || s.corsOrigins[origin]
+}
+
+// Handler to return the current version of etcd.
+func (s *Server) GetVersionHandler(w http.ResponseWriter, req *http.Request) error {
+	w.WriteHeader(http.StatusOK)
+	fmt.Fprintf(w, "etcd %s", ReleaseVersion)
+	return nil
+}
+
+// Handler to return the current leader's raft address
+func (s *Server) GetLeaderHandler(w http.ResponseWriter, req *http.Request) error {
+	leader := s.peerServer.RaftServer().Leader()
+	if leader == "" {
+		return etcdErr.NewError(etcdErr.EcodeLeaderElect, "", store.UndefIndex, store.UndefTerm)
+	}
+	w.WriteHeader(http.StatusOK)
+	url, _ := s.registry.PeerURL(leader)
+	w.Write([]byte(url))
+	return nil
+}
+
+// Handler to return all the known machines in the current cluster.
+func (s *Server) GetMachinesHandler(w http.ResponseWriter, req *http.Request) error {
+	machines := s.registry.ClientURLs(s.peerServer.RaftServer().Leader(), s.name)
+	w.WriteHeader(http.StatusOK)
+	w.Write([]byte(strings.Join(machines, ", ")))
+	return nil
+}
+
+// Retrieves stats on the Raft server.
+func (s *Server) GetStatsHandler(w http.ResponseWriter, req *http.Request) error {
+	w.Write(s.peerServer.Stats())
+	return nil
+}
+
+// Retrieves stats on the leader.
+func (s *Server) GetLeaderStatsHandler(w http.ResponseWriter, req *http.Request) error {
+	if s.peerServer.RaftServer().State() == raft.Leader {
+		w.Write(s.peerServer.PeerStats())
+		return nil
+	}
+
+	leader := s.peerServer.RaftServer().Leader()
+	if leader == "" {
+		return etcdErr.NewError(300, "", store.UndefIndex, store.UndefTerm)
+	}
+	hostname, _ := s.registry.ClientURL(leader)
+	redirect(hostname, w, req)
+	return nil
+}
+
+// Retrieves stats on the leader.
+func (s *Server) GetStoreStatsHandler(w http.ResponseWriter, req *http.Request) error {
+	w.Write(s.store.JsonStats())
+	return nil
+}
+
+// Executes a speed test to evaluate the performance of update replication.
+func (s *Server) SpeedTestHandler(w http.ResponseWriter, req *http.Request) error {
+	count := 1000
+	c := make(chan bool, count)
+	for i := 0; i < count; i++ {
+		go func() {
+			for j := 0; j < 10; j++ {
+				c := &store.SetCommand{
+					Key:        "foo",
+					Value:      "bar",
+					ExpireTime: time.Unix(0, 0),
+				}
+				s.peerServer.RaftServer().Do(c)
+			}
+			c <- true
+		}()
+	}
+
+	for i := 0; i < count; i++ {
+		<-c
+	}
+
+	w.WriteHeader(http.StatusOK)
+	w.Write([]byte("speed test success"))
+	return nil
+}
--- a/server/stats_queue.go
+++ b/server/stats_queue.go
@ -0,0 +1,89 @@
+package server
+
+import (
+	"sync"
+	"time"
+)
+
+const (
+	queueCapacity = 200
+)
+
+type statsQueue struct {
+	items        [queueCapacity]*packageStats
+	size         int
+	front        int
+	back         int
+	totalPkgSize int
+	rwl          sync.RWMutex
+}
+
+func (q *statsQueue) Len() int {
+	return q.size
+}
+
+func (q *statsQueue) PkgSize() int {
+	return q.totalPkgSize
+}
+
+// FrontAndBack gets the front and back elements in the queue
+// We must grab front and back together with the protection of the lock
+func (q *statsQueue) frontAndBack() (*packageStats, *packageStats) {
+	q.rwl.RLock()
+	defer q.rwl.RUnlock()
+	if q.size != 0 {
+		return q.items[q.front], q.items[q.back]
+	}
+	return nil, nil
+}
+
+// Insert function insert a packageStats into the queue and update the records
+func (q *statsQueue) Insert(p *packageStats) {
+	q.rwl.Lock()
+	defer q.rwl.Unlock()
+
+	q.back = (q.back + 1) % queueCapacity
+
+	if q.size == queueCapacity { //dequeue
+		q.totalPkgSize -= q.items[q.front].size
+		q.front = (q.back + 1) % queueCapacity
+	} else {
+		q.size++
+	}
+
+	q.items[q.back] = p
+	q.totalPkgSize += q.items[q.back].size
+
+}
+
+// Rate function returns the package rate and byte rate
+func (q *statsQueue) Rate() (float64, float64) {
+	front, back := q.frontAndBack()
+
+	if front == nil || back == nil {
+		return 0, 0
+	}
+
+	if time.Now().Sub(back.Time()) > time.Second {
+		q.Clear()
+		return 0, 0
+	}
+
+	sampleDuration := back.Time().Sub(front.Time())
+
+	pr := float64(q.Len()) / float64(sampleDuration) * float64(time.Second)
+
+	br := float64(q.PkgSize()) / float64(sampleDuration) * float64(time.Second)
+
+	return pr, br
+}
+
+// Clear function clear up the statsQueue
+func (q *statsQueue) Clear() {
+	q.rwl.Lock()
+	defer q.rwl.Unlock()
+	q.back = -1
+	q.front = 0
+	q.size = 0
+	q.totalPkgSize = 0
+}
--- a/server/timeout.go
+++ b/server/timeout.go
@ -0,0 +1,15 @@
+package server
+
+import (
+	"time"
+)
+
+const (
+	// The amount of time to elapse without a heartbeat before becoming a candidate.
+	ElectionTimeout = 200 * time.Millisecond
+
+	// The frequency by which heartbeats are sent to followers.
+	HeartbeatTimeout = 50 * time.Millisecond
+
+	RetryInterval = 10
+)
--- a/server/tls_config.go
+++ b/server/tls_config.go
@ -0,0 +1,11 @@
+package server
+
+import (
+	"crypto/tls"
+)
+
+type TLSConfig struct {
+	Scheme string
+	Server tls.Config
+	Client tls.Config
+}
--- a/server/tls_info.go
+++ b/server/tls_info.go
@ -0,0 +1,7 @@
+package server
+
+type TLSInfo struct {
+	CertFile string `json:"CertFile"`
+	KeyFile  string `json:"KeyFile"`
+	CAFile   string `json:"CAFile"`
+}
--- a/server/transporter.go
+++ b/server/transporter.go
@ -0,0 +1,227 @@
+package server
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/coreos/etcd/log"
+	"github.com/coreos/go-raft"
+)
+
+// Timeout for setup internal raft http connection
+// This should not exceed 3 * RTT
+var dailTimeout = 3 * HeartbeatTimeout
+
+// Timeout for setup internal raft http connection + receive response header
+// This should not exceed 3 * RTT + RTT
+var responseHeaderTimeout = 4 * HeartbeatTimeout
+
+// Timeout for receiving the response body from the server
+// This should not exceed election timeout
+var tranTimeout = ElectionTimeout
+
+// Transporter layer for communication between raft nodes
+type transporter struct {
+	client     *http.Client
+	transport  *http.Transport
+	peerServer *PeerServer
+}
+
+// Create transporter using by raft server
+// Create http or https transporter based on
+// whether the user give the server cert and key
+func newTransporter(scheme string, tlsConf tls.Config, peerServer *PeerServer) *transporter {
+	t := transporter{}
+
+	tr := &http.Transport{
+		Dial: dialWithTimeout,
+		ResponseHeaderTimeout: responseHeaderTimeout,
+	}
+
+	if scheme == "https" {
+		tr.TLSClientConfig = &tlsConf
+		tr.DisableCompression = true
+	}
+
+	t.client = &http.Client{Transport: tr}
+	t.transport = tr
+	t.peerServer = peerServer
+
+	return &t
+}
+
+// Dial with timeout
+func dialWithTimeout(network, addr string) (net.Conn, error) {
+	return net.DialTimeout(network, addr, dailTimeout)
+}
+
+// Sends AppendEntries RPCs to a peer when the server is the leader.
+func (t *transporter) SendAppendEntriesRequest(server raft.Server, peer *raft.Peer, req *raft.AppendEntriesRequest) *raft.AppendEntriesResponse {
+	var aersp *raft.AppendEntriesResponse
+	var b bytes.Buffer
+
+	json.NewEncoder(&b).Encode(req)
+
+	size := b.Len()
+
+	t.peerServer.serverStats.SendAppendReq(size)
+
+	u, _ := t.peerServer.registry.PeerURL(peer.Name)
+
+	log.Debugf("Send LogEntries to %s ", u)
+
+	thisFollowerStats, ok := t.peerServer.followersStats.Followers[peer.Name]
+
+	if !ok { //this is the first time this follower has been seen
+		thisFollowerStats = &raftFollowerStats{}
+		thisFollowerStats.Latency.Minimum = 1 << 63
+		t.peerServer.followersStats.Followers[peer.Name] = thisFollowerStats
+	}
+
+	start := time.Now()
+
+	resp, httpRequest, err := t.Post(fmt.Sprintf("%s/log/append", u), &b)
+
+	end := time.Now()
+
+	if err != nil {
+		log.Debugf("Cannot send AppendEntriesRequest to %s: %s", u, err)
+		if ok {
+			thisFollowerStats.Fail()
+		}
+	} else {
+		if ok {
+			thisFollowerStats.Succ(end.Sub(start))
+		}
+	}
+
+	if resp != nil {
+		defer resp.Body.Close()
+
+		t.CancelWhenTimeout(httpRequest)
+
+		aersp = &raft.AppendEntriesResponse{}
+		if err := json.NewDecoder(resp.Body).Decode(&aersp); err == nil || err == io.EOF {
+			return aersp
+		}
+
+	}
+
+	return aersp
+}
+
+// Sends RequestVote RPCs to a peer when the server is the candidate.
+func (t *transporter) SendVoteRequest(server raft.Server, peer *raft.Peer, req *raft.RequestVoteRequest) *raft.RequestVoteResponse {
+	var rvrsp *raft.RequestVoteResponse
+	var b bytes.Buffer
+	json.NewEncoder(&b).Encode(req)
+
+	u, _ := t.peerServer.registry.PeerURL(peer.Name)
+	log.Debugf("Send Vote from %s to %s", server.Name(), u)
+
+	resp, httpRequest, err := t.Post(fmt.Sprintf("%s/vote", u), &b)
+
+	if err != nil {
+		log.Debugf("Cannot send VoteRequest to %s : %s", u, err)
+	}
+
+	if resp != nil {
+		defer resp.Body.Close()
+
+		t.CancelWhenTimeout(httpRequest)
+
+		rvrsp := &raft.RequestVoteResponse{}
+		if err := json.NewDecoder(resp.Body).Decode(&rvrsp); err == nil || err == io.EOF {
+			return rvrsp
+		}
+
+	}
+	return rvrsp
+}
+
+// Sends SnapshotRequest RPCs to a peer when the server is the candidate.
+func (t *transporter) SendSnapshotRequest(server raft.Server, peer *raft.Peer, req *raft.SnapshotRequest) *raft.SnapshotResponse {
+	var aersp *raft.SnapshotResponse
+	var b bytes.Buffer
+	json.NewEncoder(&b).Encode(req)
+
+	u, _ := t.peerServer.registry.PeerURL(peer.Name)
+	log.Debugf("Send Snapshot to %s [Last Term: %d, LastIndex %d]", u,
+		req.LastTerm, req.LastIndex)
+
+	resp, httpRequest, err := t.Post(fmt.Sprintf("%s/snapshot", u), &b)
+
+	if err != nil {
+		log.Debugf("Cannot send SendSnapshotRequest to %s : %s", u, err)
+	}
+
+	if resp != nil {
+		defer resp.Body.Close()
+
+		t.CancelWhenTimeout(httpRequest)
+
+		aersp = &raft.SnapshotResponse{}
+		if err = json.NewDecoder(resp.Body).Decode(&aersp); err == nil || err == io.EOF {
+
+			return aersp
+		}
+	}
+
+	return aersp
+}
+
+// Sends SnapshotRecoveryRequest RPCs to a peer when the server is the candidate.
+func (t *transporter) SendSnapshotRecoveryRequest(server raft.Server, peer *raft.Peer, req *raft.SnapshotRecoveryRequest) *raft.SnapshotRecoveryResponse {
+	var aersp *raft.SnapshotRecoveryResponse
+	var b bytes.Buffer
+	json.NewEncoder(&b).Encode(req)
+
+	u, _ := t.peerServer.registry.PeerURL(peer.Name)
+	log.Debugf("Send SnapshotRecovery to %s [Last Term: %d, LastIndex %d]", u,
+		req.LastTerm, req.LastIndex)
+
+	resp, _, err := t.Post(fmt.Sprintf("%s/snapshotRecovery", u), &b)
+
+	if err != nil {
+		log.Debugf("Cannot send SendSnapshotRecoveryRequest to %s : %s", u, err)
+	}
+
+	if resp != nil {
+		defer resp.Body.Close()
+		aersp = &raft.SnapshotRecoveryResponse{}
+
+		if err = json.NewDecoder(resp.Body).Decode(&aersp); err == nil || err == io.EOF {
+			return aersp
+		}
+	}
+
+	return aersp
+}
+
+// Send server side POST request
+func (t *transporter) Post(urlStr string, body io.Reader) (*http.Response, *http.Request, error) {
+	req, _ := http.NewRequest("POST", urlStr, body)
+	resp, err := t.client.Do(req)
+	return resp, req, err
+}
+
+// Send server side GET request
+func (t *transporter) Get(urlStr string) (*http.Response, *http.Request, error) {
+	req, _ := http.NewRequest("GET", urlStr, nil)
+	resp, err := t.client.Do(req)
+	return resp, req, err
+}
+
+// Cancel the on fly HTTP transaction when timeout happens.
+func (t *transporter) CancelWhenTimeout(req *http.Request) {
+	go func() {
+		time.Sleep(ElectionTimeout)
+		t.transport.CancelRequest(req)
+	}()
+}
--- a/server/transporter_test.go
+++ b/server/transporter_test.go
@ -0,0 +1,61 @@
+package server
+
+import (
+	"crypto/tls"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"testing"
+	"time"
+)
+
+func TestTransporterTimeout(t *testing.T) {
+
+	http.HandleFunc("/timeout", func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "timeout")
+		w.(http.Flusher).Flush() // send headers and some body
+		time.Sleep(time.Second * 100)
+	})
+
+	go http.ListenAndServe(":8080", nil)
+
+	conf := tls.Config{}
+
+	ts := newTransporter("http", conf, nil)
+
+	ts.Get("http://google.com")
+	_, _, err := ts.Get("http://google.com:9999")
+	if err == nil {
+		t.Fatal("timeout error")
+	}
+
+	res, req, err := ts.Get("http://localhost:8080/timeout")
+
+	if err != nil {
+		t.Fatal("should not timeout")
+	}
+
+	ts.CancelWhenTimeout(req)
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err == nil {
+		fmt.Println(string(body))
+		t.Fatal("expected an error reading the body")
+	}
+
+	_, _, err = ts.Post("http://google.com:9999", nil)
+	if err == nil {
+		t.Fatal("timeout error")
+	}
+
+	_, _, err = ts.Get("http://www.google.com")
+	if err != nil {
+		t.Fatal("get error: ", err.Error())
+	}
+
+	_, _, err = ts.Post("http://www.google.com", nil)
+	if err != nil {
+		t.Fatal("post error")
+	}
+
+}
--- a/server/util.go
+++ b/server/util.go
@ -0,0 +1,26 @@
+package server
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/coreos/etcd/log"
+)
+
+func decodeJsonRequest(req *http.Request, data interface{}) error {
+	decoder := json.NewDecoder(req.Body)
+	if err := decoder.Decode(&data); err != nil && err != io.EOF {
+		log.Warnf("Malformed json request: %v", err)
+		return fmt.Errorf("Malformed json request: %v", err)
+	}
+	return nil
+}
+
+func redirect(hostname string, w http.ResponseWriter, req *http.Request) {
+	path := req.URL.Path
+	url := hostname + path
+	log.Debugf("Redirect to %s", url)
+	http.Redirect(w, req, url, http.StatusTemporaryRedirect)
+}
--- a/server/v1/delete_key_handler.go
+++ b/server/v1/delete_key_handler.go
@ -0,0 +1,15 @@
+package v1
+
+import (
+	"github.com/coreos/etcd/store"
+	"github.com/gorilla/mux"
+	"net/http"
+)
+
+// Removes a key from the store.
+func DeleteKeyHandler(w http.ResponseWriter, req *http.Request, s Server) error {
+	vars := mux.Vars(req)
+	key := "/" + vars["key"]
+	c := &store.DeleteCommand{Key: key}
+	return s.Dispatch(c, w, req)
+}
--- a/server/v1/get_key_handler.go
+++ b/server/v1/get_key_handler.go
@ -0,0 +1,27 @@
+package v1
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/gorilla/mux"
+)
+
+// Retrieves the value for a given key.
+func GetKeyHandler(w http.ResponseWriter, req *http.Request, s Server) error {
+	vars := mux.Vars(req)
+	key := "/" + vars["key"]
+
+	// Retrieve the key from the store.
+	event, err := s.Store().Get(key, false, false, s.CommitIndex(), s.Term())
+	if err != nil {
+		return err
+	}
+
+	// Convert event to a response and write to client.
+	b, _ := json.Marshal(event.Response())
+	w.WriteHeader(http.StatusOK)
+	w.Write(b)
+
+	return nil
+}
--- a/server/v1/set_key_handler.go
+++ b/server/v1/set_key_handler.go
@ -0,0 +1,50 @@
+package v1
+
+import (
+	"net/http"
+
+	etcdErr "github.com/coreos/etcd/error"
+	"github.com/coreos/etcd/store"
+	"github.com/coreos/go-raft"
+	"github.com/gorilla/mux"
+)
+
+// Sets the value for a given key.
+func SetKeyHandler(w http.ResponseWriter, req *http.Request, s Server) error {
+	vars := mux.Vars(req)
+	key := "/" + vars["key"]
+
+	req.ParseForm()
+
+	// Parse non-blank value.
+	value := req.Form.Get("value")
+	if len(value) == 0 {
+		return etcdErr.NewError(200, "Set", store.UndefIndex, store.UndefTerm)
+	}
+
+	// Convert time-to-live to an expiration time.
+	expireTime, err := store.TTL(req.Form.Get("ttl"))
+	if err != nil {
+		return etcdErr.NewError(202, "Set", store.UndefIndex, store.UndefTerm)
+	}
+
+	// If the "prevValue" is specified then test-and-set. Otherwise create a new key.
+	var c raft.Command
+	if prevValueArr, ok := req.Form["prevValue"]; ok && len(prevValueArr) > 0 {
+		c = &store.CompareAndSwapCommand{
+			Key:        key,
+			Value:      value,
+			PrevValue:  prevValueArr[0],
+			ExpireTime: expireTime,
+		}
+
+	} else {
+		c = &store.SetCommand{
+			Key:        key,
+			Value:      value,
+			ExpireTime: expireTime,
+		}
+	}
+
+	return s.Dispatch(c, w, req)
+}
--- a/server/v1/v1.go
+++ b/server/v1/v1.go
@ -0,0 +1,15 @@
+package v1
+
+import (
+	"github.com/coreos/etcd/store"
+	"github.com/coreos/go-raft"
+	"net/http"
+)
+
+// The Server interface provides all the methods required for the v1 API.
+type Server interface {
+	CommitIndex() uint64
+	Term() uint64
+	Store() store.Store
+	Dispatch(raft.Command, http.ResponseWriter, *http.Request) error
+}
--- a/server/v1/watch_key_handler.go
+++ b/server/v1/watch_key_handler.go
@ -0,0 +1,40 @@
+package v1
+
+import (
+	"encoding/json"
+	"net/http"
+	"strconv"
+
+	etcdErr "github.com/coreos/etcd/error"
+	"github.com/coreos/etcd/store"
+	"github.com/gorilla/mux"
+)
+
+// Watches a given key prefix for changes.
+func WatchKeyHandler(w http.ResponseWriter, req *http.Request, s Server) error {
+	var err error
+	vars := mux.Vars(req)
+	key := "/" + vars["key"]
+
+	// Create a command to watch from a given index (default 0).
+	var sinceIndex uint64 = 0
+	if req.Method == "POST" {
+		sinceIndex, err = strconv.ParseUint(string(req.FormValue("index")), 10, 64)
+		if err != nil {
+			return etcdErr.NewError(203, "Watch From Index", store.UndefIndex, store.UndefTerm)
+		}
+	}
+
+	// Start the watcher on the store.
+	c, err := s.Store().Watch(key, false, sinceIndex, s.CommitIndex(), s.Term())
+	if err != nil {
+		return etcdErr.NewError(500, key, store.UndefIndex, store.UndefTerm)
+	}
+	event := <-c
+
+	b, _ := json.Marshal(event.Response())
+	w.WriteHeader(http.StatusOK)
+	w.Write(b)
+
+	return nil
+}
--- a/server/v2/delete_handler.go
+++ b/server/v2/delete_handler.go
@ -0,0 +1,20 @@
+package v2
+
+import (
+	"net/http"
+
+	"github.com/coreos/etcd/store"
+	"github.com/gorilla/mux"
+)
+
+func DeleteHandler(w http.ResponseWriter, req *http.Request, s Server) error {
+	vars := mux.Vars(req)
+	key := "/" + vars["key"]
+
+	c := &store.DeleteCommand{
+		Key:       key,
+		Recursive: (req.FormValue("recursive") == "true"),
+	}
+
+	return s.Dispatch(c, w, req)
+}
--- a/server/v2/get_handler.go
+++ b/server/v2/get_handler.go
@ -0,0 +1,71 @@
+package v2
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strconv"
+
+	etcdErr "github.com/coreos/etcd/error"
+	"github.com/coreos/etcd/log"
+	"github.com/coreos/etcd/store"
+	"github.com/coreos/go-raft"
+	"github.com/gorilla/mux"
+)
+
+func GetHandler(w http.ResponseWriter, req *http.Request, s Server) error {
+	var err error
+	var event *store.Event
+
+	vars := mux.Vars(req)
+	key := "/" + vars["key"]
+
+	// Help client to redirect the request to the current leader
+	if req.FormValue("consistent") == "true" && s.State() != raft.Leader {
+		leader := s.Leader()
+		hostname, _ := s.PeerURL(leader)
+		url := hostname + req.URL.Path
+		log.Debugf("Redirect consistent get to %s", url)
+		http.Redirect(w, req, url, http.StatusTemporaryRedirect)
+		return nil
+	}
+
+	recursive := (req.FormValue("recursive") == "true")
+	sorted := (req.FormValue("sorted") == "true")
+
+	if req.FormValue("wait") == "true" { // watch
+		// Create a command to watch from a given index (default 0).
+		var sinceIndex uint64 = 0
+
+		waitIndex := req.FormValue("waitIndex")
+		if waitIndex != "" {
+			sinceIndex, err = strconv.ParseUint(string(req.FormValue("waitIndex")), 10, 64)
+			if err != nil {
+				return etcdErr.NewError(etcdErr.EcodeIndexNaN, "Watch From Index", store.UndefIndex, store.UndefTerm)
+			}
+		}
+
+		// Start the watcher on the store.
+		c, err := s.Store().Watch(key, recursive, sinceIndex, s.CommitIndex(), s.Term())
+		if err != nil {
+			return etcdErr.NewError(500, key, store.UndefIndex, store.UndefTerm)
+		}
+		event = <-c
+
+	} else { //get
+		// Retrieve the key from the store.
+		event, err = s.Store().Get(key, recursive, sorted, s.CommitIndex(), s.Term())
+		if err != nil {
+			return err
+		}
+	}
+
+	w.Header().Add("X-Etcd-Index", fmt.Sprint(event.Index))
+	w.Header().Add("X-Etcd-Term", fmt.Sprint(event.Term))
+	w.WriteHeader(http.StatusOK)
+
+	b, _ := json.Marshal(event)
+	w.Write(b)
+
+	return nil
+}
--- a/server/v2/post_handler.go
+++ b/server/v2/post_handler.go
@ -0,0 +1,29 @@
+package v2
+
+import (
+	"net/http"
+
+	etcdErr "github.com/coreos/etcd/error"
+	"github.com/coreos/etcd/store"
+	"github.com/gorilla/mux"
+)
+
+func PostHandler(w http.ResponseWriter, req *http.Request, s Server) error {
+	vars := mux.Vars(req)
+	key := "/" + vars["key"]
+
+	value := req.FormValue("value")
+	expireTime, err := store.TTL(req.FormValue("ttl"))
+	if err != nil {
+		return etcdErr.NewError(etcdErr.EcodeTTLNaN, "Create", store.UndefIndex, store.UndefTerm)
+	}
+
+	c := &store.CreateCommand{
+		Key:        key,
+		Value:      value,
+		ExpireTime: expireTime,
+		Unique:     true,
+	}
+
+	return s.Dispatch(c, w, req)
+}
--- a/server/v2/put_handler.go
+++ b/server/v2/put_handler.go
@ -0,0 +1,109 @@
+package v2
+
+import (
+	"net/http"
+	"strconv"
+	"time"
+
+	etcdErr "github.com/coreos/etcd/error"
+	"github.com/coreos/etcd/store"
+	"github.com/coreos/go-raft"
+	"github.com/gorilla/mux"
+)
+
+func PutHandler(w http.ResponseWriter, req *http.Request, s Server) error {
+	vars := mux.Vars(req)
+	key := "/" + vars["key"]
+
+	req.ParseForm()
+
+	value := req.Form.Get("value")
+	expireTime, err := store.TTL(req.Form.Get("ttl"))
+	if err != nil {
+		return etcdErr.NewError(etcdErr.EcodeTTLNaN, "Update", store.UndefIndex, store.UndefTerm)
+	}
+
+	prevValue, valueOk := req.Form["prevValue"]
+	prevIndexStr, indexOk := req.Form["prevIndex"]
+	prevExist, existOk := req.Form["prevExist"]
+
+	var c raft.Command
+
+	// Set handler: create a new node or replace the old one.
+	if !valueOk && !indexOk && !existOk {
+		return SetHandler(w, req, s, key, value, expireTime)
+	}
+
+	// update with test
+	if existOk {
+		if prevExist[0] == "false" {
+			// Create command: create a new node. Fail, if a node already exists
+			// Ignore prevIndex and prevValue
+			return CreateHandler(w, req, s, key, value, expireTime)
+		}
+
+		if prevExist[0] == "true" && !indexOk && !valueOk {
+			return UpdateHandler(w, req, s, key, value, expireTime)
+		}
+	}
+
+	var prevIndex uint64
+
+	if indexOk {
+		prevIndex, err = strconv.ParseUint(prevIndexStr[0], 10, 64)
+
+		// bad previous index
+		if err != nil {
+			return etcdErr.NewError(etcdErr.EcodeIndexNaN, "CompareAndSwap", store.UndefIndex, store.UndefTerm)
+		}
+	} else {
+		prevIndex = 0
+	}
+
+	if valueOk {
+		if prevValue[0] == "" {
+			return etcdErr.NewError(etcdErr.EcodePrevValueRequired, "CompareAndSwap", store.UndefIndex, store.UndefTerm)
+		}
+	}
+
+	c = &store.CompareAndSwapCommand{
+		Key:       key,
+		Value:     value,
+		PrevValue: prevValue[0],
+		PrevIndex: prevIndex,
+	}
+
+	return s.Dispatch(c, w, req)
+}
+
+func SetHandler(w http.ResponseWriter, req *http.Request, s Server, key, value string, expireTime time.Time) error {
+	c := &store.SetCommand{
+		Key:        key,
+		Value:      value,
+		ExpireTime: expireTime,
+	}
+	return s.Dispatch(c, w, req)
+}
+
+func CreateHandler(w http.ResponseWriter, req *http.Request, s Server, key, value string, expireTime time.Time) error {
+	c := &store.CreateCommand{
+		Key:        key,
+		Value:      value,
+		ExpireTime: expireTime,
+	}
+	return s.Dispatch(c, w, req)
+}
+
+func UpdateHandler(w http.ResponseWriter, req *http.Request, s Server, key, value string, expireTime time.Time) error {
+	// Update should give at least one option
+	if value == "" && expireTime.Sub(store.Permanent) == 0 {
+		return etcdErr.NewError(etcdErr.EcodeValueOrTTLRequired, "Update", store.UndefIndex, store.UndefTerm)
+	}
+
+	c := &store.UpdateCommand{
+		Key:        key,
+		Value:      value,
+		ExpireTime: expireTime,
+	}
+	return s.Dispatch(c, w, req)
+}
--- a/server/v2/v2.go
+++ b/server/v2/v2.go
@ -0,0 +1,18 @@
+package v2
+
+import (
+	"github.com/coreos/etcd/store"
+	"github.com/coreos/go-raft"
+	"net/http"
+)
+
+// The Server interface provides all the methods required for the v2 API.
+type Server interface {
+	State() string
+	Leader() string
+	CommitIndex() uint64
+	Term() uint64
+	PeerURL(string) (string, bool)
+	Store() store.Store
+	Dispatch(raft.Command, http.ResponseWriter, *http.Request) error
+}
--- a/server/version.go
+++ b/server/version.go
@ -0,0 +1,8 @@
+package server
+
+const Version = "v2"
+
+// TODO: The release version (generated from the git tag) will be the raft
+// protocol version for now. When things settle down we will fix it like the
+// client API above.
+const PeerVersion = ReleaseVersion
--- a/store/compare_and_swap_command.go
+++ b/store/compare_and_swap_command.go
@ -0,0 +1,41 @@
+package store
+
+import (
+	"time"
+
+	"github.com/coreos/etcd/log"
+	"github.com/coreos/go-raft"
+)
+
+func init() {
+	raft.RegisterCommand(&CompareAndSwapCommand{})
+}
+
+// The CompareAndSwap performs a conditional update on a key in the store.
+type CompareAndSwapCommand struct {
+	Key        string    `json:"key"`
+	Value      string    `json:"value"`
+	ExpireTime time.Time `json:"expireTime"`
+	PrevValue  string    `json: prevValue`
+	PrevIndex  uint64    `json: prevIndex`
+}
+
+// The name of the testAndSet command in the log
+func (c *CompareAndSwapCommand) CommandName() string {
+	return "etcd:compareAndSwap"
+}
+
+// Set the key-value pair if the current value of the key equals to the given prevValue
+func (c *CompareAndSwapCommand) Apply(server raft.Server) (interface{}, error) {
+	s, _ := server.StateMachine().(Store)
+
+	e, err := s.CompareAndSwap(c.Key, c.PrevValue, c.PrevIndex,
+		c.Value, c.ExpireTime, server.CommitIndex(), server.Term())
+
+	if err != nil {
+		log.Debug(err)
+		return nil, err
+	}
+
+	return e, nil
+}
--- a/store/create_command.go
+++ b/store/create_command.go
@ -0,0 +1,38 @@
+package store
+
+import (
+	"github.com/coreos/etcd/log"
+	"github.com/coreos/go-raft"
+	"time"
+)
+
+func init() {
+	raft.RegisterCommand(&CreateCommand{})
+}
+
+// Create command
+type CreateCommand struct {
+	Key        string    `json:"key"`
+	Value      string    `json:"value"`
+	ExpireTime time.Time `json:"expireTime"`
+	Unique     bool      `json:"unique"`
+}
+
+// The name of the create command in the log
+func (c *CreateCommand) CommandName() string {
+	return "etcd:create"
+}
+
+// Create node
+func (c *CreateCommand) Apply(server raft.Server) (interface{}, error) {
+	s, _ := server.StateMachine().(Store)
+
+	e, err := s.Create(c.Key, c.Value, c.Unique, c.ExpireTime, server.CommitIndex(), server.Term())
+
+	if err != nil {
+		log.Debug(err)
+		return nil, err
+	}
+
+	return e, nil
+}
--- a/store/delete_command.go
+++ b/store/delete_command.go
@ -0,0 +1,35 @@
+package store
+
+import (
+	"github.com/coreos/etcd/log"
+	"github.com/coreos/go-raft"
+)
+
+func init() {
+	raft.RegisterCommand(&DeleteCommand{})
+}
+
+// The DeleteCommand removes a key from the Store.
+type DeleteCommand struct {
+	Key       string `json:"key"`
+	Recursive bool   `json:"recursive"`
+}
+
+// The name of the delete command in the log
+func (c *DeleteCommand) CommandName() string {
+	return "etcd:delete"
+}
+
+// Delete the key
+func (c *DeleteCommand) Apply(server raft.Server) (interface{}, error) {
+	s, _ := server.StateMachine().(Store)
+
+	e, err := s.Delete(c.Key, c.Recursive, server.CommitIndex(), server.Term())
+
+	if err != nil {
+		log.Debug(err)
+		return nil, err
+	}
+
+	return e, nil
+}
--- a/store/error.go
+++ b/store/error.go
@ -1,25 +0,0 @@
-package store
-
-type NotFoundError string
-
-func (e NotFoundError) Error() string {
-	return string(e)
-}
-
-type NotFile string
-
-func (e NotFile) Error() string {
-	return string(e)
-}
-
-type TestFail string
-
-func (e TestFail) Error() string {
-	return string(e)
-}
-
-type Keyword string
-
-func (e Keyword) Error() string {
-	return string(e)
-}
--- a/store/event.go
+++ b/store/event.go
@ -0,0 +1,80 @@
+package store
+
+import (
+	"time"
+)
+
+const (
+	Get            = "get"
+	Create         = "create"
+	Set            = "set"
+	Update         = "update"
+	Delete         = "delete"
+	CompareAndSwap = "compareAndSwap"
+	Expire         = "expire"
+)
+
+const (
+	UndefIndex = 0
+	UndefTerm  = 0
+)
+
+type Event struct {
+	Action     string     `json:"action"`
+	Key        string     `json:"key, omitempty"`
+	Dir        bool       `json:"dir,omitempty"`
+	PrevValue  string     `json:"prevValue,omitempty"`
+	Value      string     `json:"value,omitempty"`
+	KVPairs    kvPairs    `json:"kvs,omitempty"`
+	Expiration *time.Time `json:"expiration,omitempty"`
+	TTL        int64      `json:"ttl,omitempty"` // Time to live in second
+	// The command index of the raft machine when the command is executed
+	Index uint64 `json:"index"`
+	Term  uint64 `json:"term"`
+}
+
+func newEvent(action string, key string, index uint64, term uint64) *Event {
+	return &Event{
+		Action: action,
+		Key:    key,
+		Index:  index,
+		Term:   term,
+	}
+}
+
+// Converts an event object into a response object.
+func (event *Event) Response() interface{} {
+	if !event.Dir {
+		response := &Response{
+			Action:     event.Action,
+			Key:        event.Key,
+			Value:      event.Value,
+			PrevValue:  event.PrevValue,
+			Index:      event.Index,
+			TTL:        event.TTL,
+			Expiration: event.Expiration,
+		}
+
+		if response.Action == Create || response.Action == Set {
+			response.Action = "set"
+			if response.PrevValue == "" {
+				response.NewKey = true
+			}
+		}
+
+		return response
+	} else {
+		responses := make([]*Response, len(event.KVPairs))
+
+		for i, kv := range event.KVPairs {
+			responses[i] = &Response{
+				Action: event.Action,
+				Key:    kv.Key,
+				Value:  kv.Value,
+				Dir:    kv.Dir,
+				Index:  event.Index,
+			}
+		}
+		return responses
+	}
+}
--- a/store/event_history.go
+++ b/store/event_history.go
@ -0,0 +1,112 @@
+package store
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	etcdErr "github.com/coreos/etcd/error"
+)
+
+type EventHistory struct {
+	Queue      eventQueue
+	StartIndex uint64
+	LastIndex  uint64
+	LastTerm   uint64
+	DupCnt     uint64 // help to compute the watching point with duplicated indexes in the queue
+	rwl        sync.RWMutex
+}
+
+func newEventHistory(capacity int) *EventHistory {
+	return &EventHistory{
+		Queue: eventQueue{
+			Capacity: capacity,
+			Events:   make([]*Event, capacity),
+		},
+	}
+}
+
+// addEvent function adds event into the eventHistory
+func (eh *EventHistory) addEvent(e *Event) *Event {
+	eh.rwl.Lock()
+	defer eh.rwl.Unlock()
+
+	var duped uint64
+
+	if e.Index == UndefIndex {
+		e.Index = eh.LastIndex
+		e.Term = eh.LastTerm
+		duped = 1
+	}
+
+	eh.Queue.insert(e)
+
+	eh.LastIndex = e.Index
+	eh.LastTerm = e.Term
+	eh.DupCnt += duped
+
+	eh.StartIndex = eh.Queue.Events[eh.Queue.Front].Index
+
+	return e
+}
+
+// scan function is enumerating events from the index in history and
+// stops till the first point where the key has identified prefix
+func (eh *EventHistory) scan(prefix string, index uint64) (*Event, *etcdErr.Error) {
+	eh.rwl.RLock()
+	defer eh.rwl.RUnlock()
+
+	start := index - eh.StartIndex
+
+	// the index should locate after the event history's StartIndex
+	if start < 0 {
+		return nil,
+			etcdErr.NewError(etcdErr.EcodeEventIndexCleared,
+				fmt.Sprintf("the requested history has been cleared [%v/%v]",
+					eh.StartIndex, index), UndefIndex, UndefTerm)
+	}
+
+	// the index should locate before the size of the queue minus the duplicate count
+	if start >= (uint64(eh.Queue.Size) - eh.DupCnt) { // future index
+		return nil, nil
+	}
+
+	i := int((start + uint64(eh.Queue.Front)) % uint64(eh.Queue.Capacity))
+
+	for {
+		e := eh.Queue.Events[i]
+		if strings.HasPrefix(e.Key, prefix) && index <= e.Index { // make sure we bypass the smaller one
+			return e, nil
+		}
+
+		i = (i + 1) % eh.Queue.Capacity
+
+		if i == eh.Queue.back() { // find nothing, return and watch from current index
+			return nil, nil
+		}
+	}
+}
+
+// clone will be protected by a stop-world lock
+// do not need to obtain internal lock
+func (eh *EventHistory) clone() *EventHistory {
+	clonedQueue := eventQueue{
+		Capacity: eh.Queue.Capacity,
+		Events:   make([]*Event, eh.Queue.Capacity),
+		Size:     eh.Queue.Size,
+		Front:    eh.Queue.Front,
+	}
+
+	for i, e := range eh.Queue.Events {
+		clonedQueue.Events[i] = e
+	}
+
+	return &EventHistory{
+		StartIndex: eh.StartIndex,
+		Queue:      clonedQueue,
+		LastIndex:  eh.LastIndex,
+		LastTerm:   eh.LastTerm,
+		DupCnt:     eh.DupCnt,
+	}
+
+}
--- a/store/event_queue.go
+++ b/store/event_queue.go
@ -0,0 +1,25 @@
+package store
+
+type eventQueue struct {
+	Events   []*Event
+	Size     int
+	Front    int
+	Capacity int
+}
+
+func (eq *eventQueue) back() int {
+	return (eq.Front + eq.Size - 1 + eq.Capacity) % eq.Capacity
+}
+
+func (eq *eventQueue) insert(e *Event) {
+	index := (eq.back() + 1) % eq.Capacity
+
+	eq.Events[index] = e
+
+	if eq.Size == eq.Capacity { //dequeue
+		eq.Front = (index + 1) % eq.Capacity
+	} else {
+		eq.Size++
+	}
+
+}
--- a/store/event_test.go
+++ b/store/event_test.go
@ -0,0 +1,66 @@
+package store
+
+import (
+	"testing"
+)
+
+// TestEventQueue tests a queue with capacity = 100
+// Add 200 events into that queue, and test if the
+// previous 100 events have been swapped out.
+func TestEventQueue(t *testing.T) {
+
+	eh := newEventHistory(100)
+
+	// Add
+	for i := 0; i < 200; i++ {
+		e := newEvent(Create, "/foo", uint64(i), 1)
+		eh.addEvent(e)
+	}
+
+	// Test
+	j := 100
+	i := eh.Queue.Front
+	n := eh.Queue.Size
+	for ; n > 0; n-- {
+		e := eh.Queue.Events[i]
+		if e.Index != uint64(j) {
+			t.Fatalf("queue error!")
+		}
+		j++
+		i = (i + 1) % eh.Queue.Capacity
+	}
+}
+
+func TestScanHistory(t *testing.T) {
+	eh := newEventHistory(100)
+
+	// Add
+	eh.addEvent(newEvent(Create, "/foo", 1, 1))
+	eh.addEvent(newEvent(Create, "/foo/bar", 2, 1))
+	eh.addEvent(newEvent(Create, "/foo/foo", 3, 1))
+	eh.addEvent(newEvent(Create, "/foo/bar/bar", 4, 1))
+	eh.addEvent(newEvent(Create, "/foo/foo/foo", 5, 1))
+
+	e, err := eh.scan("/foo", 1)
+	if err != nil || e.Index != 1 {
+		t.Fatalf("scan error [/foo] [1] %v", e.Index)
+	}
+
+	e, err = eh.scan("/foo/bar", 1)
+
+	if err != nil || e.Index != 2 {
+		t.Fatalf("scan error [/foo/bar] [2] %v", e.Index)
+	}
+
+	e, err = eh.scan("/foo/bar", 3)
+
+	if err != nil || e.Index != 4 {
+		t.Fatalf("scan error [/foo/bar/bar] [4] %v", e.Index)
+	}
+
+	e, err = eh.scan("/foo/bar", 6)
+
+	if e != nil {
+		t.Fatalf("bad index shoud reuturn nil")
+	}
+}
--- a/store/keyword_test.go
+++ b/store/keyword_test.go
@ -1,37 +0,0 @@
-package store
-
-import (
-	"testing"
-)
-
-func TestKeywords(t *testing.T) {
-	keyword := CheckKeyword("_etcd")
-	if !keyword {
-		t.Fatal("_etcd should be keyword")
-	}
-
-	keyword = CheckKeyword("/_etcd")
-
-	if !keyword {
-		t.Fatal("/_etcd should be keyword")
-	}
-
-	keyword = CheckKeyword("/_etcd/")
-
-	if !keyword {
-		t.Fatal("/_etcd/ contains keyword prefix")
-	}
-
-	keyword = CheckKeyword("/_etcd/node1")
-
-	if !keyword {
-		t.Fatal("/_etcd/* contains keyword prefix")
-	}
-
-	keyword = CheckKeyword("/nokeyword/_etcd/node1")
-
-	if keyword {
-		t.Fatal("this does not contain keyword prefix")
-	}
-
-}
--- a/store/keywords.go
+++ b/store/keywords.go
@ -1,33 +0,0 @@
-package store
-
-import (
-	"path"
-	"strings"
-)
-
-// keywords for internal useage
-// Key for string keyword; Value for only checking prefix
-var keywords = map[string]bool{
-	"/_etcd":          true,
-	"/ephemeralNodes": true,
-}
-
-// CheckKeyword will check if the key contains the keyword.
-// For now, we only check for prefix.
-func CheckKeyword(key string) bool {
-	key = path.Clean("/" + key)
-
-	// find the second "/"
-	i := strings.Index(key[1:], "/")
-
-	var prefix string
-
-	if i == -1 {
-		prefix = key
-	} else {
-		prefix = key[:i+1]
-	}
-	_, ok := keywords[prefix]
-
-	return ok
-}
--- a/store/kv_pairs.go
+++ b/store/kv_pairs.go
@ -0,0 +1,24 @@
+package store
+
+// When user list a directory, we add all the node into key-value pair slice
+type KeyValuePair struct {
+	Key     string  `json:"key, omitempty"`
+	Value   string  `json:"value,omitempty"`
+	Dir     bool    `json:"dir,omitempty"`
+	KVPairs kvPairs `json:"kvs,omitempty"`
+}
+
+type kvPairs []KeyValuePair
+
+// interfaces for sorting
+func (kvs kvPairs) Len() int {
+	return len(kvs)
+}
+
+func (kvs kvPairs) Less(i, j int) bool {
+	return kvs[i].Key < kvs[j].Key
+}
+
+func (kvs kvPairs) Swap(i, j int) {
+	kvs[i], kvs[j] = kvs[j], kvs[i]
+}
--- a/store/node.go
+++ b/store/node.go
@ -0,0 +1,417 @@
+package store
+
+import (
+	"path"
+	"sort"
+	"sync"
+	"time"
+
+	etcdErr "github.com/coreos/etcd/error"
+)
+
+var (
+	Permanent time.Time
+)
+
+const (
+	normal = iota
+	removed
+)
+
+// Node is the basic element in the store system.
+// A key-value pair will have a string value
+// A directory will have a children map
+type Node struct {
+	Path string
+
+	CreateIndex   uint64
+	CreateTerm    uint64
+	ModifiedIndex uint64
+	ModifiedTerm  uint64
+
+	Parent *Node `json:"-"` // should not encode this field! avoid cyclical dependency.
+
+	ExpireTime time.Time
+	ACL        string
+	Value      string           // for key-value pair
+	Children   map[string]*Node // for directory
+
+	// A reference to the store this node is attached to.
+	store *store
+
+	// a ttl node will have an expire routine associated with it.
+	// we need a channel to stop that routine when the expiration changes.
+	stopExpire chan bool
+
+	// ensure we only delete the node once
+	// expire and remove may try to delete a node twice
+	once sync.Once
+}
+
+// newKV creates a Key-Value pair
+func newKV(store *store, nodePath string, value string, createIndex uint64,
+	createTerm uint64, parent *Node, ACL string, expireTime time.Time) *Node {
+
+	return &Node{
+		Path:          nodePath,
+		CreateIndex:   createIndex,
+		CreateTerm:    createTerm,
+		ModifiedIndex: createIndex,
+		ModifiedTerm:  createTerm,
+		Parent:        parent,
+		ACL:           ACL,
+		store:         store,
+		stopExpire:    make(chan bool, 1),
+		ExpireTime:    expireTime,
+		Value:         value,
+	}
+}
+
+// newDir creates a directory
+func newDir(store *store, nodePath string, createIndex uint64, createTerm uint64,
+	parent *Node, ACL string, expireTime time.Time) *Node {
+
+	return &Node{
+		Path:        nodePath,
+		CreateIndex: createIndex,
+		CreateTerm:  createTerm,
+		Parent:      parent,
+		ACL:         ACL,
+		stopExpire:  make(chan bool, 1),
+		ExpireTime:  expireTime,
+		Children:    make(map[string]*Node),
+		store:       store,
+	}
+}
+
+// IsHidden function checks if the node is a hidden node. A hidden node
+// will begin with '_'
+// A hidden node will not be shown via get command under a directory
+// For example if we have /foo/_hidden and /foo/notHidden, get "/foo"
+// will only return /foo/notHidden
+func (n *Node) IsHidden() bool {
+	_, name := path.Split(n.Path)
+
+	return name[0] == '_'
+}
+
+// IsPermanent function checks if the node is a permanent one.
+func (n *Node) IsPermanent() bool {
+	return n.ExpireTime.Sub(Permanent) == 0
+}
+
+// IsExpired function checks if the node has been expired.
+func (n *Node) IsExpired() (bool, time.Duration) {
+	if n.IsPermanent() {
+		return false, 0
+	}
+
+	duration := n.ExpireTime.Sub(time.Now())
+	if duration <= 0 {
+		return true, 0
+	}
+
+	return false, duration
+}
+
+// IsDir function checks whether the node is a directory.
+// If the node is a directory, the function will return true.
+// Otherwise the function will return false.
+func (n *Node) IsDir() bool {
+	return !(n.Children == nil)
+}
+
+// Read function gets the value of the node.
+// If the receiver node is not a key-value pair, a "Not A File" error will be returned.
+func (n *Node) Read() (string, *etcdErr.Error) {
+	if n.IsDir() {
+		return "", etcdErr.NewError(etcdErr.EcodeNotFile, "", UndefIndex, UndefTerm)
+	}
+
+	return n.Value, nil
+}
+
+// Write function set the value of the node to the given value.
+// If the receiver node is a directory, a "Not A File" error will be returned.
+func (n *Node) Write(value string, index uint64, term uint64) *etcdErr.Error {
+	if n.IsDir() {
+		return etcdErr.NewError(etcdErr.EcodeNotFile, "", UndefIndex, UndefTerm)
+	}
+
+	n.Value = value
+	n.ModifiedIndex = index
+	n.ModifiedTerm = term
+
+	return nil
+}
+
+func (n *Node) ExpirationAndTTL() (*time.Time, int64) {
+	if n.ExpireTime.Sub(Permanent) != 0 {
+		return &n.ExpireTime, int64(n.ExpireTime.Sub(time.Now())/time.Second) + 1
+	}
+	return nil, 0
+}
+
+// List function return a slice of nodes under the receiver node.
+// If the receiver node is not a directory, a "Not A Directory" error will be returned.
+func (n *Node) List() ([]*Node, *etcdErr.Error) {
+	if !n.IsDir() {
+		return nil, etcdErr.NewError(etcdErr.EcodeNotDir, "", UndefIndex, UndefTerm)
+	}
+
+	nodes := make([]*Node, len(n.Children))
+
+	i := 0
+	for _, node := range n.Children {
+		nodes[i] = node
+		i++
+	}
+
+	return nodes, nil
+}
+
+// GetChild function returns the child node under the directory node.
+// On success, it returns the file node
+func (n *Node) GetChild(name string) (*Node, *etcdErr.Error) {
+	if !n.IsDir() {
+		return nil, etcdErr.NewError(etcdErr.EcodeNotDir, n.Path, UndefIndex, UndefTerm)
+	}
+
+	child, ok := n.Children[name]
+
+	if ok {
+		return child, nil
+	}
+
+	return nil, nil
+}
+
+// Add function adds a node to the receiver node.
+// If the receiver is not a directory, a "Not A Directory" error will be returned.
+// If there is a existing node with the same name under the directory, a "Already Exist"
+// error will be returned
+func (n *Node) Add(child *Node) *etcdErr.Error {
+	if !n.IsDir() {
+		return etcdErr.NewError(etcdErr.EcodeNotDir, "", UndefIndex, UndefTerm)
+	}
+
+	_, name := path.Split(child.Path)
+
+	_, ok := n.Children[name]
+
+	if ok {
+		return etcdErr.NewError(etcdErr.EcodeNodeExist, "", UndefIndex, UndefTerm)
+	}
+
+	n.Children[name] = child
+
+	return nil
+}
+
+// Remove function remove the node.
+func (n *Node) Remove(recursive bool, callback func(path string)) *etcdErr.Error {
+
+	if n.IsDir() && !recursive {
+		// cannot delete a directory without set recursive to true
+		return etcdErr.NewError(etcdErr.EcodeNotFile, "", UndefIndex, UndefTerm)
+	}
+
+	onceBody := func() {
+		n.internalRemove(recursive, callback)
+	}
+
+	// this function might be entered multiple times by expire and delete
+	// every node will only be deleted once.
+	n.once.Do(onceBody)
+
+	return nil
+}
+
+// internalRemove function will be called by remove()
+func (n *Node) internalRemove(recursive bool, callback func(path string)) {
+	if !n.IsDir() { // key-value pair
+		_, name := path.Split(n.Path)
+
+		// find its parent and remove the node from the map
+		if n.Parent != nil && n.Parent.Children[name] == n {
+			delete(n.Parent.Children, name)
+		}
+
+		if callback != nil {
+			callback(n.Path)
+		}
+
+		// the stop channel has a buffer. just send to it!
+		n.stopExpire <- true
+		return
+	}
+
+	for _, child := range n.Children { // delete all children
+		child.Remove(true, callback)
+	}
+
+	// delete self
+	_, name := path.Split(n.Path)
+	if n.Parent != nil && n.Parent.Children[name] == n {
+		delete(n.Parent.Children, name)
+
+		if callback != nil {
+			callback(n.Path)
+		}
+
+		n.stopExpire <- true
+	}
+}
+
+// Expire function will test if the node is expired.
+// if the node is already expired, delete the node and return.
+// if the node is permanent (this shouldn't happen), return at once.
+// else wait for a period time, then remove the node. and notify the watchhub.
+func (n *Node) Expire() {
+	expired, duration := n.IsExpired()
+
+	if expired { // has been expired
+		// since the parent function of Expire() runs serially,
+		// there is no need for lock here
+		e := newEvent(Expire, n.Path, UndefIndex, UndefTerm)
+		n.store.WatcherHub.notify(e)
+
+		n.Remove(true, nil)
+		n.store.Stats.Inc(ExpireCount)
+
+		return
+	}
+
+	if duration == 0 { // Permanent Node
+		return
+	}
+
+	go func() { // do monitoring
+		select {
+		// if timeout, delete the node
+		case <-time.After(duration):
+
+			// before expire get the lock, the expiration time
+			// of the node may be updated.
+			// we have to check again when get the lock
+			n.store.worldLock.Lock()
+			defer n.store.worldLock.Unlock()
+
+			expired, _ := n.IsExpired()
+
+			if expired {
+				e := newEvent(Expire, n.Path, UndefIndex, UndefTerm)
+				n.store.WatcherHub.notify(e)
+
+				n.Remove(true, nil)
+				n.store.Stats.Inc(ExpireCount)
+			}
+
+			return
+
+		// if stopped, return
+		case <-n.stopExpire:
+			return
+		}
+	}()
+}
+
+func (n *Node) Pair(recurisive, sorted bool) KeyValuePair {
+	if n.IsDir() {
+		pair := KeyValuePair{
+			Key: n.Path,
+			Dir: true,
+		}
+
+		if !recurisive {
+			return pair
+		}
+
+		children, _ := n.List()
+		pair.KVPairs = make([]KeyValuePair, len(children))
+
+		// we do not use the index in the children slice directly
+		// we need to skip the hidden one
+		i := 0
+
+		for _, child := range children {
+
+			if child.IsHidden() { // get will not list hidden node
+				continue
+			}
+
+			pair.KVPairs[i] = child.Pair(recurisive, sorted)
+
+			i++
+		}
+
+		// eliminate hidden nodes
+		pair.KVPairs = pair.KVPairs[:i]
+		if sorted {
+			sort.Sort(pair.KVPairs)
+		}
+
+		return pair
+	}
+
+	return KeyValuePair{
+		Key:   n.Path,
+		Value: n.Value,
+	}
+}
+
+func (n *Node) UpdateTTL(expireTime time.Time) {
+	if !n.IsPermanent() {
+		// check if the node has been expired
+		// if the node is not expired, we need to stop the go routine associated with
+		// that node.
+		expired, _ := n.IsExpired()
+
+		if !expired {
+			n.stopExpire <- true // suspend it to modify the expiration
+		}
+	}
+
+	if expireTime.Sub(Permanent) != 0 {
+		n.ExpireTime = expireTime
+		n.Expire()
+	}
+}
+
+// Clone function clone the node recursively and return the new node.
+// If the node is a directory, it will clone all the content under this directory.
+// If the node is a key-value pair, it will clone the pair.
+func (n *Node) Clone() *Node {
+	if !n.IsDir() {
+		return newKV(n.store, n.Path, n.Value, n.CreateIndex, n.CreateTerm, n.Parent, n.ACL, n.ExpireTime)
+	}
+
+	clone := newDir(n.store, n.Path, n.CreateIndex, n.CreateTerm, n.Parent, n.ACL, n.ExpireTime)
+
+	for key, child := range n.Children {
+		clone.Children[key] = child.Clone()
+	}
+
+	return clone
+}
+
+// recoverAndclean function help to do recovery.
+// Two things need to be done: 1. recovery structure; 2. delete expired nodes
+
+// If the node is a directory, it will help recover children's parent pointer and recursively
+// call this function on its children.
+// We check the expire last since we need to recover the whole structure first and add all the
+// notifications into the event history.
+func (n *Node) recoverAndclean() {
+	if n.IsDir() {
+		for _, child := range n.Children {
+			child.Parent = n
+			child.store = n.store
+			child.recoverAndclean()
+		}
+	}
+
+	n.stopExpire = make(chan bool, 1)
+
+	n.Expire()
+}
--- a/store/response_v1.go
+++ b/store/response_v1.go
@ -0,0 +1,26 @@
+package store
+
+import (
+	"time"
+)
+
+// The response from the store to the user who issue a command
+type Response struct {
+	Action    string `json:"action"`
+	Key       string `json:"key"`
+	Dir       bool   `json:"dir,omitempty"`
+	PrevValue string `json:"prevValue,omitempty"`
+	Value     string `json:"value,omitempty"`
+
+	// If the key did not exist before the action,
+	// this field should be set to true
+	NewKey bool `json:"newKey,omitempty"`
+
+	Expiration *time.Time `json:"expiration,omitempty"`
+
+	// Time to live in second
+	TTL int64 `json:"ttl,omitempty"`
+
+	// The command index of the raft machine when the command is executed
+	Index uint64 `json:"index"`
+}
--- a/store/set_command.go
+++ b/store/set_command.go
@ -0,0 +1,38 @@
+package store
+
+import (
+	"github.com/coreos/etcd/log"
+	"github.com/coreos/go-raft"
+	"time"
+)
+
+func init() {
+	raft.RegisterCommand(&SetCommand{})
+}
+
+// Create command
+type SetCommand struct {
+	Key        string    `json:"key"`
+	Value      string    `json:"value"`
+	ExpireTime time.Time `json:"expireTime"`
+}
+
+// The name of the create command in the log
+func (c *SetCommand) CommandName() string {
+	return "etcd:set"
+}
+
+// Create node
+func (c *SetCommand) Apply(server raft.Server) (interface{}, error) {
+	s, _ := server.StateMachine().(Store)
+
+	// create a new node or replace the old node.
+	e, err := s.Set(c.Key, c.Value, c.ExpireTime, server.CommitIndex(), server.Term())
+
+	if err != nil {
+		log.Debug(err)
+		return nil, err
+	}
+
+	return e, nil
+}
--- a/store/stats.go
+++ b/store/stats.go
@ -2,24 +2,111 @@ package store

 import (
 	"encoding/json"
+	"sync/atomic"
 )

-type EtcdStats struct {
+const (
+	SetSuccess = iota
+	SetFail
+	DeleteSuccess
+	DeleteFail
+	CreateSuccess
+	CreateFail
+	UpdateSuccess
+	UpdateFail
+	CompareAndSwapSuccess
+	CompareAndSwapFail
+	GetSuccess
+	GetFail
+	ExpireCount
+)
+
+type Stats struct {
+
 	// Number of get requests
-	Gets uint64 `json:"gets"`
+	GetSuccess uint64 `json:"getsSuccess"`
+	GetFail    uint64 `json:"getsFail"`

 	// Number of sets requests
-	Sets uint64 `json:"sets"`
+	SetSuccess uint64 `json:"setsSuccess"`
+	SetFail    uint64 `json:"setsFail"`

 	// Number of delete requests
-	Deletes uint64 `json:"deletes"`
+	DeleteSuccess uint64 `json:"deleteSuccess"`
+	DeleteFail    uint64 `json:"deleteFail"`
+
+	// Number of update requests
+	UpdateSuccess uint64 `json:"updateSuccess"`
+	UpdateFail    uint64 `json:"updateFail"`
+
+	// Number of create requests
+	CreateSuccess uint64 `json:"createSuccess"`
+	CreateFail    uint64 `json:createFail`

 	// Number of testAndSet requests
-	TestAndSets uint64 `json:"testAndSets"`
+	CompareAndSwapSuccess uint64 `json:"compareAndSwapSuccess"`
+	CompareAndSwapFail    uint64 `json:"compareAndSwapFail"`
+
+	ExpireCount uint64 `json:"expireCount"`
+
+	Watchers uint64 `json:"watchers"`
 }

-// Stats returns the basic statistics information of etcd storage
-func (s *Store) Stats() []byte {
-	b, _ := json.Marshal(s.BasicStats)
+func newStats() *Stats {
+	s := new(Stats)
+	return s
+}
+
+func (s *Stats) clone() *Stats {
+	return &Stats{s.GetSuccess, s.GetFail, s.SetSuccess, s.SetFail,
+		s.DeleteSuccess, s.DeleteFail, s.UpdateSuccess, s.UpdateFail, s.CreateSuccess,
+		s.CreateFail, s.CompareAndSwapSuccess, s.CompareAndSwapFail, s.Watchers, s.ExpireCount}
+}
+
+// Status() return the statistics info of etcd storage its recent start
+func (s *Stats) toJson() []byte {
+	b, _ := json.Marshal(s)
 	return b
 }
+
+func (s *Stats) TotalReads() uint64 {
+	return s.GetSuccess + s.GetFail
+}
+
+func (s *Stats) TotalWrites() uint64 {
+	return s.SetSuccess + s.SetFail +
+		s.DeleteSuccess + s.DeleteFail +
+		s.CompareAndSwapSuccess + s.CompareAndSwapFail +
+		s.UpdateSuccess + s.UpdateFail
+}
+
+func (s *Stats) Inc(field int) {
+	switch field {
+	case SetSuccess:
+		atomic.AddUint64(&s.SetSuccess, 1)
+	case SetFail:
+		atomic.AddUint64(&s.SetFail, 1)
+	case CreateSuccess:
+		atomic.AddUint64(&s.CreateSuccess, 1)
+	case CreateFail:
+		atomic.AddUint64(&s.CreateFail, 1)
+	case DeleteSuccess:
+		atomic.AddUint64(&s.DeleteSuccess, 1)
+	case DeleteFail:
+		atomic.AddUint64(&s.DeleteFail, 1)
+	case GetSuccess:
+		atomic.AddUint64(&s.GetSuccess, 1)
+	case GetFail:
+		atomic.AddUint64(&s.GetFail, 1)
+	case UpdateSuccess:
+		atomic.AddUint64(&s.UpdateSuccess, 1)
+	case UpdateFail:
+		atomic.AddUint64(&s.UpdateFail, 1)
+	case CompareAndSwapSuccess:
+		atomic.AddUint64(&s.CompareAndSwapSuccess, 1)
+	case CompareAndSwapFail:
+		atomic.AddUint64(&s.CompareAndSwapFail, 1)
+	case ExpireCount:
+		atomic.AddUint64(&s.ExpireCount, 1)
+	}
+}
--- a/store/stats_test.go
+++ b/store/stats_test.go
@ -0,0 +1,165 @@
+package store
+
+import (
+	"math/rand"
+	"testing"
+	"time"
+)
+
+func TestBasicStats(t *testing.T) {
+	s := newStore()
+	keys := GenKeys(rand.Intn(100), 5)
+
+	var i uint64
+	var GetSuccess, GetFail, CreateSuccess, CreateFail, DeleteSuccess, DeleteFail uint64
+	var UpdateSuccess, UpdateFail, CompareAndSwapSuccess, CompareAndSwapFail, watcher_number uint64
+
+	for _, k := range keys {
+		i++
+		_, err := s.Create(k, "bar", false, time.Now().Add(time.Second*time.Duration(rand.Intn(6))), i, 1)
+		if err != nil {
+			CreateFail++
+		} else {
+			CreateSuccess++
+		}
+	}
+
+	time.Sleep(time.Second * 3)
+
+	for _, k := range keys {
+		_, err := s.Get(k, false, false, i, 1)
+		if err != nil {
+			GetFail++
+		} else {
+			GetSuccess++
+		}
+	}
+
+	for _, k := range keys {
+		i++
+		_, err := s.Update(k, "foo", time.Now().Add(time.Second*time.Duration(rand.Intn(6))), i, 1)
+		if err != nil {
+			UpdateFail++
+		} else {
+			UpdateSuccess++
+		}
+	}
+
+	time.Sleep(time.Second * 3)
+
+	for _, k := range keys {
+		_, err := s.Get(k, false, false, i, 1)
+		if err != nil {
+			GetFail++
+		} else {
+			GetSuccess++
+		}
+	}
+
+	for _, k := range keys {
+		i++
+		_, err := s.CompareAndSwap(k, "foo", 0, "bar", Permanent, i, 1)
+		if err != nil {
+			CompareAndSwapFail++
+		} else {
+			CompareAndSwapSuccess++
+		}
+	}
+
+	for _, k := range keys {
+		s.Watch(k, false, 0, i, 1)
+		watcher_number++
+	}
+
+	for _, k := range keys {
+		_, err := s.Get(k, false, false, i, 1)
+		if err != nil {
+			GetFail++
+		} else {
+			GetSuccess++
+		}
+	}
+
+	for _, k := range keys {
+		i++
+		_, err := s.Delete(k, false, i, 1)
+		if err != nil {
+			DeleteFail++
+		} else {
+			watcher_number--
+			DeleteSuccess++
+		}
+	}
+
+	for _, k := range keys {
+		_, err := s.Get(k, false, false, i, 1)
+		if err != nil {
+			GetFail++
+		} else {
+			GetSuccess++
+		}
+	}
+
+	if GetSuccess != s.Stats.GetSuccess {
+		t.Fatalf("GetSuccess [%d] != Stats.GetSuccess [%d]", GetSuccess, s.Stats.GetSuccess)
+	}
+
+	if GetFail != s.Stats.GetFail {
+		t.Fatalf("GetFail [%d] != Stats.GetFail [%d]", GetFail, s.Stats.GetFail)
+	}
+
+	if CreateSuccess != s.Stats.CreateSuccess {
+		t.Fatalf("CreateSuccess [%d] != Stats.CreateSuccess [%d]", CreateSuccess, s.Stats.CreateSuccess)
+	}
+
+	if CreateFail != s.Stats.CreateFail {
+		t.Fatalf("CreateFail [%d] != Stats.CreateFail [%d]", CreateFail, s.Stats.CreateFail)
+	}
+
+	if DeleteSuccess != s.Stats.DeleteSuccess {
+		t.Fatalf("DeleteSuccess [%d] != Stats.DeleteSuccess [%d]", DeleteSuccess, s.Stats.DeleteSuccess)
+	}
+
+	if DeleteFail != s.Stats.DeleteFail {
+		t.Fatalf("DeleteFail [%d] != Stats.DeleteFail [%d]", DeleteFail, s.Stats.DeleteFail)
+	}
+
+	if UpdateSuccess != s.Stats.UpdateSuccess {
+		t.Fatalf("UpdateSuccess [%d] != Stats.UpdateSuccess [%d]", UpdateSuccess, s.Stats.UpdateSuccess)
+	}
+
+	if UpdateFail != s.Stats.UpdateFail {
+		t.Fatalf("UpdateFail [%d] != Stats.UpdateFail [%d]", UpdateFail, s.Stats.UpdateFail)
+	}
+
+	if CompareAndSwapSuccess != s.Stats.CompareAndSwapSuccess {
+		t.Fatalf("TestAndSetSuccess [%d] != Stats.CompareAndSwapSuccess [%d]", CompareAndSwapSuccess, s.Stats.CompareAndSwapSuccess)
+	}
+
+	if CompareAndSwapFail != s.Stats.CompareAndSwapFail {
+		t.Fatalf("TestAndSetFail [%d] != Stats.TestAndSetFail [%d]", CompareAndSwapFail, s.Stats.CompareAndSwapFail)
+	}
+
+	s = newStore()
+	CreateSuccess = 0
+	CreateFail = 0
+
+	for _, k := range keys {
+		i++
+		_, err := s.Create(k, "bar", false, time.Now().Add(time.Second*3), i, 1)
+		if err != nil {
+			CreateFail++
+		} else {
+			CreateSuccess++
+		}
+	}
+
+	time.Sleep(6 * time.Second)
+
+	ExpireCount := CreateSuccess
+
+	if ExpireCount != s.Stats.ExpireCount {
+		t.Fatalf("ExpireCount [%d] != Stats.ExpireCount [%d]", ExpireCount, s.Stats.ExpireCount)
+	}
+
+}
--- a/store/store.go
+++ b/store/store.go
--- a/store/store_test.go
+++ b/store/store_test.go
@ -1,222 +1,565 @@
 package store

 import (
-	"encoding/json"
+	"math/rand"
+	"strconv"
 	"testing"
 	"time"
 )

-func TestStoreGetDelete(t *testing.T) {
+func TestCreateAndGet(t *testing.T) {
+	s := newStore()

-	s := CreateStore(100)
-	s.Set("foo", "bar", time.Unix(0, 0), 1)
-	res, err := s.Get("foo")
+	s.Create("/foobar", "bar", false, Permanent, 1, 1)

-	if err != nil {
-		t.Fatalf("Unknown error")
-	}
-
-	var result Response
-	json.Unmarshal(res, &result)
-
-	if result.Value != "bar" {
-		t.Fatalf("Cannot get stored value")
-	}
-
-	s.Delete("foo", 2)
-	_, err = s.Get("foo")
+	// already exist, create should fail
+	_, err := s.Create("/foobar", "bar", false, Permanent, 1, 1)

 	if err == nil {
-		t.Fatalf("Got deleted value")
+		t.Fatal("Create should fail")
+	}
+
+	s.Delete("/foobar", true, 1, 1)
+
+	s.Create("/foobar/foo", "bar", false, Permanent, 1, 1)
+
+	// already exist, create should fail
+	_, err = s.Create("/foobar", "bar", false, Permanent, 1, 1)
+
+	if err == nil {
+		t.Fatal("Create should fail")
+	}
+
+	s.Delete("/foobar", true, 1, 1)
+
+	// this should create successfully
+	createAndGet(s, "/foobar", t)
+	createAndGet(s, "/foo/bar", t)
+	createAndGet(s, "/foo/foo/bar", t)
+
+	// meet file, create should fail
+	_, err = s.Create("/foo/bar/bar", "bar", false, Permanent, 2, 1)
+
+	if err == nil {
+		t.Fatal("Create should fail")
+	}
+
+	// create a directory
+	_, err = s.Create("/fooDir", "", false, Permanent, 3, 1)
+
+	if err != nil {
+		t.Fatal("Cannot create /fooDir")
+	}
+
+	e, err := s.Get("/fooDir", false, false, 3, 1)
+
+	if err != nil || e.Dir != true {
+		t.Fatal("Cannot create /fooDir ")
+	}
+
+	// create a file under directory
+	_, err = s.Create("/fooDir/bar", "bar", false, Permanent, 4, 1)
+
+	if err != nil {
+		t.Fatal("Cannot create /fooDir/bar = bar")
 	}
 }

-func TestSaveAndRecovery(t *testing.T) {
+func TestUpdateFile(t *testing.T) {
+	s := newStore()

-	s := CreateStore(100)
-	s.Set("foo", "bar", time.Unix(0, 0), 1)
-	s.Set("foo2", "bar2", time.Now().Add(time.Second*5), 2)
-	state, err := s.Save()
+	_, err := s.Create("/foo/bar", "bar", false, Permanent, 1, 1)

 	if err != nil {
-		t.Fatalf("Cannot Save %s", err)
+		t.Fatalf("cannot create %s=bar [%s]", "/foo/bar", err.Error())
 	}

-	newStore := CreateStore(100)
+	_, err = s.Update("/foo/bar", "barbar", Permanent, 2, 1)

-	// wait for foo2 expires
-	time.Sleep(time.Second * 6)
-
-	newStore.Recovery(state)
-
-	res, err := newStore.Get("foo")
-
-	var result Response
-	json.Unmarshal(res, &result)
-
-	if result.Value != "bar" {
-		t.Fatalf("Recovery Fail")
+	if err != nil {
+		t.Fatalf("cannot update %s=barbar [%s]", "/foo/bar", err.Error())
 	}

-	res, err = newStore.Get("foo2")
+	e, err := s.Get("/foo/bar", false, false, 2, 1)
+
+	if err != nil {
+		t.Fatalf("cannot get %s [%s]", "/foo/bar", err.Error())
+	}
+
+	if e.Value != "barbar" {
+		t.Fatalf("expect value of %s is barbar [%s]", "/foo/bar", e.Value)
+	}
+
+	// create a directory, update its ttl, to see if it will be deleted
+	_, err = s.Create("/foo/foo", "", false, Permanent, 3, 1)
+
+	if err != nil {
+		t.Fatalf("cannot create dir [%s] [%s]", "/foo/foo", err.Error())
+	}
+
+	_, err = s.Create("/foo/foo/foo1", "bar1", false, Permanent, 4, 1)
+
+	if err != nil {
+		t.Fatal("cannot create [%s]", err.Error())
+	}
+
+	_, err = s.Create("/foo/foo/foo2", "", false, Permanent, 5, 1)
+	if err != nil {
+		t.Fatal("cannot create [%s]", err.Error())
+	}
+
+	_, err = s.Create("/foo/foo/foo2/boo", "boo1", false, Permanent, 6, 1)
+	if err != nil {
+		t.Fatal("cannot create [%s]", err.Error())
+	}
+
+	expire := time.Now().Add(time.Second * 2)
+	_, err = s.Update("/foo/foo", "", expire, 7, 1)
+	if err != nil {
+		t.Fatalf("cannot update dir [%s] [%s]", "/foo/foo", err.Error())
+	}
+
+	// sleep 50ms, it should still reach the node
+	time.Sleep(time.Microsecond * 50)
+	e, err = s.Get("/foo/foo", true, false, 7, 1)
+
+	if err != nil || e.Key != "/foo/foo" {
+		t.Fatalf("cannot get dir before expiration [%s]", err.Error())
+	}
+
+	if e.KVPairs[0].Key != "/foo/foo/foo1" || e.KVPairs[0].Value != "bar1" {
+		t.Fatalf("cannot get sub node before expiration [%s]", err.Error())
+	}
+
+	if e.KVPairs[1].Key != "/foo/foo/foo2" || e.KVPairs[1].Dir != true {
+		t.Fatalf("cannot get sub dir before expiration [%s]", err.Error())
+	}
+
+	// wait for expiration
+	time.Sleep(time.Second * 3)
+	e, err = s.Get("/foo/foo", true, false, 7, 1)

 	if err == nil {
-		t.Fatalf("Get expired value")
+		t.Fatal("still can get dir after expiration [%s]")
 	}

-	s.Delete("foo", 3)
+	_, err = s.Get("/foo/foo/foo1", true, false, 7, 1)
+	if err == nil {
+		t.Fatal("still can get sub node after expiration [%s]")
+	}

+	_, err = s.Get("/foo/foo/foo2", true, false, 7, 1)
+	if err == nil {
+		t.Fatal("still can get sub dir after expiration [%s]")
+	}
+
+	_, err = s.Get("/foo/foo/foo2/boo", true, false, 7, 1)
+	if err == nil {
+		t.Fatalf("still can get sub node of sub dir after expiration [%s]", err.Error())
+	}
+
+}
+
+func TestListDirectory(t *testing.T) {
+	s := newStore()
+
+	// create dir /foo
+	// set key-value /foo/foo=bar
+	s.Create("/foo/foo", "bar", false, Permanent, 1, 1)
+
+	// create dir /foo/fooDir
+	// set key-value /foo/fooDir/foo=bar
+	s.Create("/foo/fooDir/foo", "bar", false, Permanent, 2, 1)
+
+	e, err := s.Get("/foo", true, false, 2, 1)
+
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	if len(e.KVPairs) != 2 {
+		t.Fatalf("wrong number of kv pairs [%d/2]", len(e.KVPairs))
+	}
+
+	if e.KVPairs[0].Key != "/foo/foo" || e.KVPairs[0].Value != "bar" {
+		t.Fatalf("wrong kv [/foo/foo/ / %s] -> [bar / %s]", e.KVPairs[0].Key, e.KVPairs[0].Value)
+	}
+
+	if e.KVPairs[1].Key != "/foo/fooDir" || e.KVPairs[1].Dir != true {
+		t.Fatalf("wrong kv [/foo/fooDir/ / %s] -> [true / %v]", e.KVPairs[1].Key, e.KVPairs[1].Dir)
+	}
+
+	if e.KVPairs[1].KVPairs[0].Key != "/foo/fooDir/foo" || e.KVPairs[1].KVPairs[0].Value != "bar" {
+		t.Fatalf("wrong kv [/foo/fooDir/foo / %s] -> [bar / %v]", e.KVPairs[1].KVPairs[0].Key, e.KVPairs[1].KVPairs[0].Value)
+	}
+	// test hidden node
+
+	// create dir /foo/_hidden
+	// set key-value /foo/_hidden/foo -> bar
+	s.Create("/foo/_hidden/foo", "bar", false, Permanent, 3, 1)
+
+	e, _ = s.Get("/foo", false, false, 2, 1)
+
+	if len(e.KVPairs) != 2 {
+		t.Fatalf("hidden node is not hidden! %s", e.KVPairs[2].Key)
+	}
+}
+
+func TestRemove(t *testing.T) {
+	s := newStore()
+
+	s.Create("/foo", "bar", false, Permanent, 1, 1)
+	_, err := s.Delete("/foo", false, 1, 1)
+
+	if err != nil {
+		t.Fatalf("cannot delete %s [%s]", "/foo", err.Error())
+	}
+
+	_, err = s.Get("/foo", false, false, 1, 1)
+
+	if err == nil || err.Error() != "Key Not Found" {
+		t.Fatalf("can get the node after deletion")
+	}
+
+	s.Create("/foo/bar", "bar", false, Permanent, 1, 1)
+	s.Create("/foo/car", "car", false, Permanent, 1, 1)
+	s.Create("/foo/dar/dar", "dar", false, Permanent, 1, 1)
+
+	_, err = s.Delete("/foo", false, 1, 1)
+
+	if err == nil {
+		t.Fatalf("should not be able to delete a directory without recursive")
+	}
+
+	_, err = s.Delete("/foo", true, 1, 1)
+
+	if err != nil {
+		t.Fatalf("cannot delete %s [%s]", "/foo", err.Error())
+	}
+
+	_, err = s.Get("/foo", false, false, 1, 1)
+
+	if err == nil || err.Error() != "Key Not Found" {
+		t.Fatalf("can get the node after deletion ")
+	}
 }

 func TestExpire(t *testing.T) {
-	// test expire
-	s := CreateStore(100)
-	s.Set("foo", "bar", time.Now().Add(time.Second*1), 0)
-	time.Sleep(2 * time.Second)
+	s := newStore()

-	_, err := s.Get("foo")
+	expire := time.Now().Add(time.Second)

-	if err == nil {
-		t.Fatalf("Got expired value")
-	}
+	s.Create("/foo", "bar", false, expire, 1, 1)

-	//test change expire time
-	s.Set("foo", "bar", time.Now().Add(time.Second*10), 1)
-
-	_, err = s.Get("foo")
+	_, err := s.Get("/foo", false, false, 1, 1)

 	if err != nil {
-		t.Fatalf("Cannot get Value")
+		t.Fatalf("can not get the node")
 	}

-	s.Set("foo", "barbar", time.Now().Add(time.Second*1), 2)
+	time.Sleep(time.Second * 2)

-	time.Sleep(2 * time.Second)
-
-	_, err = s.Get("foo")
+	_, err = s.Get("/foo", false, false, 1, 1)

 	if err == nil {
-		t.Fatalf("Got expired value")
+		t.Fatalf("can get the node after expiration time")
 	}

-	// test change expire to stable
-	s.Set("foo", "bar", time.Now().Add(time.Second*1), 3)
+	// test if we can reach the node before expiration
+	expire = time.Now().Add(time.Second)
+	s.Create("/foo", "bar", false, expire, 1, 1)

-	s.Set("foo", "bar", time.Unix(0, 0), 4)
-
-	time.Sleep(2 * time.Second)
-
-	_, err = s.Get("foo")
+	time.Sleep(time.Millisecond * 50)
+	_, err = s.Get("/foo", false, false, 1, 1)

 	if err != nil {
-		t.Fatalf("Cannot get Value")
+		t.Fatalf("cannot get the node before expiration", err.Error())
 	}

-	// test stable to expire
-	s.Set("foo", "bar", time.Now().Add(time.Second*1), 5)
-	time.Sleep(2 * time.Second)
-	_, err = s.Get("foo")
+	expire = time.Now().Add(time.Second)

-	if err == nil {
-		t.Fatalf("Got expired value")
+	s.Create("/foo", "bar", false, expire, 1, 1)
+	_, err = s.Delete("/foo", false, 1, 1)
+
+	if err != nil {
+		t.Fatalf("cannot delete the node before expiration", err.Error())
 	}
-
-	// test set older node
-	s.Set("foo", "bar", time.Now().Add(-time.Second*1), 6)
-	_, err = s.Get("foo")
-
-	if err == nil {
-		t.Fatalf("Got expired value")
-	}
-
 }

-func BenchmarkStoreSet(b *testing.B) {
-	s := CreateStore(100)
+func TestCompareAndSwap(t *testing.T) { // TODO prevValue == nil ?
+	s := newStore()
+	s.Create("/foo", "bar", false, Permanent, 1, 1)

-	keys := GenKeys(10000, 5)
+	// test on wrong previous value
+	_, err := s.CompareAndSwap("/foo", "barbar", 0, "car", Permanent, 2, 1)
+	if err == nil {
+		t.Fatal("test and set should fail barbar != bar")
+	}

-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
+	// test on value
+	e, err := s.CompareAndSwap("/foo", "bar", 0, "car", Permanent, 3, 1)

-		for i, key := range keys {
-			s.Set(key, "barbarbarbarbar", time.Unix(0, 0), uint64(i))
+	if err != nil {
+		t.Fatal("test and set should succeed bar == bar")
+	}
+
+	if e.PrevValue != "bar" || e.Value != "car" {
+		t.Fatalf("[%v/%v] [%v/%v]", e.PrevValue, "bar", e.Value, "car")
+	}
+
+	// test on index
+	e, err = s.CompareAndSwap("/foo", "", 3, "bar", Permanent, 4, 1)
+
+	if err != nil {
+		t.Fatal("test and set should succeed index 3 == 3")
+	}
+
+	if e.PrevValue != "car" || e.Value != "bar" {
+		t.Fatalf("[%v/%v] [%v/%v]", e.PrevValue, "car", e.Value, "bar")
+	}
+}
+
+func TestWatch(t *testing.T) {
+	s := newStore()
+	// watch at a deeper path
+	c, _ := s.Watch("/foo/foo/foo", false, 0, 0, 1)
+	s.Create("/foo/foo/foo", "bar", false, Permanent, 1, 1)
+
+	e := nonblockingRetrive(c)
+	if e.Key != "/foo/foo/foo" || e.Action != Create {
+		t.Fatal("watch for Create node fails ", e)
+	}
+
+	c, _ = s.Watch("/foo/foo/foo", false, 0, 1, 1)
+	s.Update("/foo/foo/foo", "car", Permanent, 2, 1)
+	e = nonblockingRetrive(c)
+	if e.Key != "/foo/foo/foo" || e.Action != Update {
+		t.Fatal("watch for Update node fails ", e)
+	}
+
+	c, _ = s.Watch("/foo/foo/foo", false, 0, 2, 1)
+	s.CompareAndSwap("/foo/foo/foo", "car", 0, "bar", Permanent, 3, 1)
+	e = nonblockingRetrive(c)
+	if e.Key != "/foo/foo/foo" || e.Action != CompareAndSwap {
+		t.Fatal("watch for CompareAndSwap node fails")
+	}
+
+	c, _ = s.Watch("/foo/foo/foo", false, 0, 3, 1)
+	s.Delete("/foo", true, 4, 1) //recursively delete
+	e = nonblockingRetrive(c)
+	if e.Key != "/foo" || e.Action != Delete {
+		t.Fatal("watch for Delete node fails ", e)
+	}
+
+	// watch at a prefix
+	c, _ = s.Watch("/foo", true, 0, 4, 1)
+	s.Create("/foo/foo/boo", "bar", false, Permanent, 5, 1)
+	e = nonblockingRetrive(c)
+	if e.Key != "/foo/foo/boo" || e.Action != Create {
+		t.Fatal("watch for Create subdirectory fails")
+	}
+
+	c, _ = s.Watch("/foo", true, 0, 5, 1)
+	s.Update("/foo/foo/boo", "foo", Permanent, 6, 1)
+	e = nonblockingRetrive(c)
+	if e.Key != "/foo/foo/boo" || e.Action != Update {
+		t.Fatal("watch for Update subdirectory fails")
+	}
+
+	c, _ = s.Watch("/foo", true, 0, 6, 1)
+	s.CompareAndSwap("/foo/foo/boo", "foo", 0, "bar", Permanent, 7, 1)
+	e = nonblockingRetrive(c)
+	if e.Key != "/foo/foo/boo" || e.Action != CompareAndSwap {
+		t.Fatal("watch for CompareAndSwap subdirectory fails")
+	}
+
+	c, _ = s.Watch("/foo", true, 0, 7, 1)
+	s.Delete("/foo/foo/boo", false, 8, 1)
+	e = nonblockingRetrive(c)
+	if e == nil || e.Key != "/foo/foo/boo" || e.Action != Delete {
+		t.Fatal("watch for Delete subdirectory fails")
+	}
+
+	// watch expire
+	s.Create("/foo/foo/boo", "foo", false, time.Now().Add(time.Second*1), 9, 1)
+	c, _ = s.Watch("/foo", true, 0, 9, 1)
+	time.Sleep(time.Second * 2)
+	e = nonblockingRetrive(c)
+	if e.Key != "/foo/foo/boo" || e.Action != Expire || e.Index != 9 {
+		t.Fatal("watch for Expiration of Create() subdirectory fails ", e)
+	}
+
+	s.Create("/foo/foo/boo", "foo", false, Permanent, 10, 1)
+	s.Update("/foo/foo/boo", "bar", time.Now().Add(time.Second*1), 11, 1)
+	c, _ = s.Watch("/foo", true, 0, 11, 1)
+	time.Sleep(time.Second * 2)
+	e = nonblockingRetrive(c)
+	if e.Key != "/foo/foo/boo" || e.Action != Expire || e.Index != 11 {
+		t.Fatal("watch for Expiration of Update() subdirectory fails ", e)
+	}
+
+	s.Create("/foo/foo/boo", "foo", false, Permanent, 12, 1)
+	s.CompareAndSwap("/foo/foo/boo", "foo", 0, "bar", time.Now().Add(time.Second*1), 13, 1)
+	c, _ = s.Watch("/foo", true, 0, 13, 1)
+	time.Sleep(time.Second * 2)
+	e = nonblockingRetrive(c)
+	if e.Key != "/foo/foo/boo" || e.Action != Expire || e.Index != 13 {
+		t.Fatal("watch for Expiration of CompareAndSwap() subdirectory fails ", e)
+	}
+}
+
+func TestSort(t *testing.T) {
+	s := newStore()
+
+	// simulating random creation
+	keys := GenKeys(80, 4)
+
+	i := uint64(1)
+	for _, k := range keys {
+		_, err := s.Create(k, "bar", false, Permanent, i, 1)
+		if err != nil {
+			panic(err)
+		} else {
+			i++
+		}
+	}
+
+	e, err := s.Get("/foo", true, true, i, 1)
+	if err != nil {
+		t.Fatalf("get dir nodes failed [%s]", err.Error())
+	}
+
+	for i, k := range e.KVPairs[:len(e.KVPairs)-1] {
+
+		if k.Key >= e.KVPairs[i+1].Key {
+			t.Fatalf("sort failed, [%s] should be placed after [%s]", k.Key, e.KVPairs[i+1].Key)
 		}

-		s = CreateStore(100)
-	}
-}
-
-func BenchmarkStoreGet(b *testing.B) {
-	s := CreateStore(100)
-
-	keys := GenKeys(10000, 5)
-
-	for i, key := range keys {
-		s.Set(key, "barbarbarbarbar", time.Unix(0, 0), uint64(i))
-	}
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-
-		for _, key := range keys {
-			s.Get(key)
+		if k.Dir {
+			recursiveTestSort(k, t)
 		}

 	}
+
+	if k := e.KVPairs[len(e.KVPairs)-1]; k.Dir {
+		recursiveTestSort(k, t)
+	}
 }

-func BenchmarkStoreSnapshotCopy(b *testing.B) {
-	s := CreateStore(100)
+func TestSaveAndRecover(t *testing.T) {
+	s := newStore()

-	keys := GenKeys(10000, 5)
+	// simulating random creation
+	keys := GenKeys(8, 4)

-	for i, key := range keys {
-		s.Set(key, "barbarbarbarbar", time.Unix(0, 0), uint64(i))
+	i := uint64(1)
+	for _, k := range keys {
+		_, err := s.Create(k, "bar", false, Permanent, i, 1)
+		if err != nil {
+			panic(err)
+		} else {
+			i++
+		}
 	}

-	var state []byte
+	// create a node with expiration
+	// test if we can reach the node before expiration

-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		s.clone()
+	expire := time.Now().Add(time.Second)
+	s.Create("/foo/foo", "bar", false, expire, 1, 1)
+	b, err := s.Save()
+
+	cloneFs := newStore()
+	time.Sleep(2 * time.Second)
+
+	cloneFs.Recovery(b)
+
+	for i, k := range keys {
+		_, err := cloneFs.Get(k, false, false, uint64(i), 1)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	// lock to avoid racing with Expire()
+	s.worldLock.RLock()
+	defer s.worldLock.RUnlock()
+
+	if s.WatcherHub.EventHistory.StartIndex != cloneFs.WatcherHub.EventHistory.StartIndex {
+		t.Fatalf("Error recovered event history start index[%v/%v]",
+			s.WatcherHub.EventHistory.StartIndex, cloneFs.WatcherHub.EventHistory.StartIndex)
+	}
+
+	for i = 0; int(i) < cloneFs.WatcherHub.EventHistory.Queue.Size; i++ {
+		if s.WatcherHub.EventHistory.Queue.Events[i].Key !=
+			cloneFs.WatcherHub.EventHistory.Queue.Events[i].Key {
+			t.Fatal("Error recovered event history")
+		}
+	}
+
+	_, err = s.Get("/foo/foo", false, false, 1, 1)
+
+	if err == nil || err.Error() != "Key Not Found" {
+		t.Fatalf("can get the node after deletion ")
 	}
-	b.SetBytes(int64(len(state)))
 }

-func BenchmarkSnapshotSaveJson(b *testing.B) {
-	s := CreateStore(100)
+// GenKeys randomly generate num of keys with max depth
+func GenKeys(num int, depth int) []string {
+	rand.Seed(time.Now().UnixNano())
+	keys := make([]string, num)
+	for i := 0; i < num; i++ {

-	keys := GenKeys(10000, 5)
+		keys[i] = "/foo"
+		depth := rand.Intn(depth) + 1

-	for i, key := range keys {
-		s.Set(key, "barbarbarbarbar", time.Unix(0, 0), uint64(i))
+		for j := 0; j < depth; j++ {
+			keys[i] += "/" + strconv.Itoa(rand.Int())
+		}
 	}

-	var state []byte
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		state, _ = s.Save()
-	}
-	b.SetBytes(int64(len(state)))
+	return keys
 }

-func BenchmarkSnapshotRecovery(b *testing.B) {
-	s := CreateStore(100)
+func createAndGet(s *store, path string, t *testing.T) {
+	_, err := s.Create(path, "bar", false, Permanent, 1, 1)

-	keys := GenKeys(10000, 5)
-
-	for i, key := range keys {
-		s.Set(key, "barbarbarbarbar", time.Unix(0, 0), uint64(i))
+	if err != nil {
+		t.Fatalf("cannot create %s=bar [%s]", path, err.Error())
 	}

-	state, _ := s.Save()
+	e, err := s.Get(path, false, false, 1, 1)

-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		newStore := CreateStore(100)
-		newStore.Recovery(state)
+	if err != nil {
+		t.Fatalf("cannot get %s [%s]", path, err.Error())
+	}
+
+	if e.Value != "bar" {
+		t.Fatalf("expect value of %s is bar [%s]", path, e.Value)
+	}
+}
+
+func recursiveTestSort(k KeyValuePair, t *testing.T) {
+	for i, v := range k.KVPairs[:len(k.KVPairs)-1] {
+		if v.Key >= k.KVPairs[i+1].Key {
+			t.Fatalf("sort failed, [%s] should be placed after [%s]", v.Key, k.KVPairs[i+1].Key)
+		}
+
+		if v.Dir {
+			recursiveTestSort(v, t)
+		}
+	}
+
+	if v := k.KVPairs[len(k.KVPairs)-1]; v.Dir {
+		recursiveTestSort(v, t)
+	}
+}
+
+func nonblockingRetrive(c <-chan *Event) *Event {
+	select {
+	case e := <-c:
+		return e
+	default:
+		return nil
 	}
-	b.SetBytes(int64(len(state)))
 }
--- a/store/test.go
+++ b/store/test.go
@ -1,21 +0,0 @@
-package store
-
-import (
-	"math/rand"
-	"strconv"
-)
-
-// GenKeys randomly generate num of keys with max depth
-func GenKeys(num int, depth int) []string {
-	keys := make([]string, num)
-	for i := 0; i < num; i++ {
-
-		keys[i] = "/foo/"
-		depth := rand.Intn(depth) + 1
-
-		for j := 0; j < depth; j++ {
-			keys[i] += "/" + strconv.Itoa(rand.Int())
-		}
-	}
-	return keys
-}
--- a/store/tree.go
+++ b/store/tree.go
@ -1,318 +0,0 @@
-package store
-
-import (
-	"path"
-	"sort"
-	"strings"
-	"time"
-)
-
-//------------------------------------------------------------------------------
-//
-// Typedefs
-//
-//------------------------------------------------------------------------------
-
-// A file system like tree structure. Each non-leaf node of the tree has a hashmap to
-// store its children nodes. Leaf nodes has no hashmap (a nil pointer)
-type tree struct {
-	Root *treeNode
-}
-
-// A treeNode wraps a Node. It has a hashmap to keep records of its children treeNodes.
-type treeNode struct {
-	InternalNode Node
-	Dir          bool
-	NodeMap      map[string]*treeNode
-}
-
-// TreeNode with its key. We use it when we need to sort the treeNodes.
-type tnWithKey struct {
-	key string
-	tn  *treeNode
-}
-
-// Define type and functions to match sort interface
-type tnWithKeySlice []tnWithKey
-
-func (s tnWithKeySlice) Len() int           { return len(s) }
-func (s tnWithKeySlice) Less(i, j int) bool { return s[i].key < s[j].key }
-func (s tnWithKeySlice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
-
-// CONSTANT VARIABLE
-
-// Represent an empty node
-var emptyNode = Node{"", PERMANENT, nil}
-
-//------------------------------------------------------------------------------
-//
-// Methods
-//
-//------------------------------------------------------------------------------
-
-// Set the key to the given value, return true if success
-// If any intermidate path of the key is not a directory type, it will fail
-// For example if the /foo = Node(bar) exists, set /foo/foo = Node(barbar)
-// will fail.
-func (t *tree) set(key string, value Node) bool {
-
-	nodesName := split(key)
-
-	// avoid set value to "/"
-	if len(nodesName) == 1 && len(nodesName[0]) == 0 {
-		return false
-	}
-
-	nodeMap := t.Root.NodeMap
-
-	i := 0
-	newDir := false
-
-	// go through all the path
-	for i = 0; i < len(nodesName)-1; i++ {
-
-		// if we meet a new directory, all the directory after it must be new
-		if newDir {
-			tn := &treeNode{emptyNode, true, make(map[string]*treeNode)}
-			nodeMap[nodesName[i]] = tn
-			nodeMap = tn.NodeMap
-			continue
-		}
-
-		// get the node from the nodeMap of the current level
-		tn, ok := nodeMap[nodesName[i]]
-
-		if !ok {
-			// add a new directory and set newDir to true
-			newDir = true
-			tn := &treeNode{emptyNode, true, make(map[string]*treeNode)}
-			nodeMap[nodesName[i]] = tn
-			nodeMap = tn.NodeMap
-
-		} else if ok && !tn.Dir {
-
-			// if we meet a non-directory node, we cannot set the key
-			return false
-		} else {
-
-			// update the nodeMap to next level
-			nodeMap = tn.NodeMap
-		}
-
-	}
-
-	// Add the last node
-	tn, ok := nodeMap[nodesName[i]]
-
-	if !ok {
-		// we add a new treeNode
-		tn := &treeNode{value, false, nil}
-		nodeMap[nodesName[i]] = tn
-
-	} else {
-		if tn.Dir {
-			return false
-		}
-		// we change the value of a old Treenode
-		tn.InternalNode = value
-	}
-	return true
-
-}
-
-// Get the tree node of the key
-func (t *tree) internalGet(key string) (*treeNode, bool) {
-	nodesName := split(key)
-
-	// should be able to get root
-	if len(nodesName) == 1 && nodesName[0] == "" {
-		return t.Root, true
-	}
-
-	nodeMap := t.Root.NodeMap
-
-	var i int
-
-	for i = 0; i < len(nodesName)-1; i++ {
-		node, ok := nodeMap[nodesName[i]]
-		if !ok || !node.Dir {
-			return nil, false
-		}
-		nodeMap = node.NodeMap
-	}
-
-	tn, ok := nodeMap[nodesName[i]]
-	if ok {
-		return tn, ok
-	} else {
-		return nil, ok
-	}
-}
-
-// get the internalNode of the key
-func (t *tree) get(key string) (Node, bool) {
-	tn, ok := t.internalGet(key)
-
-	if ok {
-		if tn.Dir {
-			return emptyNode, false
-		}
-		return tn.InternalNode, ok
-	} else {
-		return emptyNode, ok
-	}
-}
-
-// get the internalNode of the key
-func (t *tree) list(directory string) (interface{}, []string, bool) {
-	treeNode, ok := t.internalGet(directory)
-
-	if !ok {
-		return nil, nil, ok
-
-	} else {
-		if !treeNode.Dir {
-			return &treeNode.InternalNode, nil, ok
-		}
-		length := len(treeNode.NodeMap)
-		nodes := make([]*Node, length)
-		keys := make([]string, length)
-
-		i := 0
-		for key, node := range treeNode.NodeMap {
-			nodes[i] = &node.InternalNode
-			keys[i] = key
-			i++
-		}
-
-		return nodes, keys, ok
-	}
-}
-
-// delete the key, return true if success
-func (t *tree) delete(key string) bool {
-	nodesName := split(key)
-
-	nodeMap := t.Root.NodeMap
-
-	var i int
-
-	for i = 0; i < len(nodesName)-1; i++ {
-		node, ok := nodeMap[nodesName[i]]
-		if !ok || !node.Dir {
-			return false
-		}
-		nodeMap = node.NodeMap
-	}
-
-	node, ok := nodeMap[nodesName[i]]
-	if ok && !node.Dir {
-		delete(nodeMap, nodesName[i])
-		return true
-	}
-	return false
-}
-
-// traverse wrapper
-func (t *tree) traverse(f func(string, *Node), sort bool) {
-	if sort {
-		sortDfs("", t.Root, f)
-	} else {
-		dfs("", t.Root, f)
-	}
-}
-
-// clone() will return a deep cloned tree
-func (t *tree) clone() *tree {
-	newTree := new(tree)
-	newTree.Root = &treeNode{
-		Node{
-			"/",
-			time.Unix(0, 0),
-			nil,
-		},
-		true,
-		make(map[string]*treeNode),
-	}
-	recursiveClone(t.Root, newTree.Root)
-	return newTree
-}
-
-// recursiveClone is a helper function for clone()
-func recursiveClone(tnSrc *treeNode, tnDes *treeNode) {
-	if !tnSrc.Dir {
-		tnDes.InternalNode = tnSrc.InternalNode
-		return
-
-	} else {
-		tnDes.InternalNode = tnSrc.InternalNode
-		tnDes.Dir = true
-		tnDes.NodeMap = make(map[string]*treeNode)
-
-		for key, tn := range tnSrc.NodeMap {
-			newTn := new(treeNode)
-			recursiveClone(tn, newTn)
-			tnDes.NodeMap[key] = newTn
-		}
-
-	}
-}
-
-// deep first search to traverse the tree
-// apply the func f to each internal node
-func dfs(key string, t *treeNode, f func(string, *Node)) {
-
-	// base case
-	if len(t.NodeMap) == 0 {
-		f(key, &t.InternalNode)
-
-		// recursion
-	} else {
-		for tnKey, tn := range t.NodeMap {
-			tnKey := key + "/" + tnKey
-			dfs(tnKey, tn, f)
-		}
-	}
-}
-
-// sort deep first search to traverse the tree
-// apply the func f to each internal node
-func sortDfs(key string, t *treeNode, f func(string, *Node)) {
-	// base case
-	if len(t.NodeMap) == 0 {
-		f(key, &t.InternalNode)
-
-		// recursion
-	} else {
-
-		s := make(tnWithKeySlice, len(t.NodeMap))
-		i := 0
-
-		// copy
-		for tnKey, tn := range t.NodeMap {
-			tnKey := key + "/" + tnKey
-			s[i] = tnWithKey{tnKey, tn}
-			i++
-		}
-
-		// sort
-		sort.Sort(s)
-
-		// traverse
-		for i = 0; i < len(t.NodeMap); i++ {
-			sortDfs(s[i].key, s[i].tn, f)
-		}
-	}
-}
-
-// split the key by '/', get the intermediate node name
-func split(key string) []string {
-	key = "/" + key
-	key = path.Clean(key)
-
-	// get the intermidate nodes name
-	nodesName := strings.Split(key, "/")
-	// we do not need the root node, since we start with it
-	nodesName = nodesName[1:]
-	return nodesName
-}
--- a/store/tree_store_test.go
+++ b/store/tree_store_test.go
@ -1,247 +0,0 @@
-package store
-
-import (
-	"fmt"
-	"math/rand"
-	"strconv"
-	"testing"
-	"time"
-)
-
-func TestStoreGet(t *testing.T) {
-
-	ts := &tree{
-		&treeNode{
-			NewTestNode("/"),
-			true,
-			make(map[string]*treeNode),
-		},
-	}
-
-	// create key
-	ts.set("/foo", NewTestNode("bar"))
-	// change value
-	ts.set("/foo", NewTestNode("barbar"))
-	// create key
-	ts.set("/hello/foo", NewTestNode("barbarbar"))
-	treeNode, ok := ts.get("/foo")
-
-	if !ok {
-		t.Fatalf("Expect to get node, but not")
-	}
-	if treeNode.Value != "barbar" {
-		t.Fatalf("Expect value barbar, but got %s", treeNode.Value)
-	}
-
-	// create key
-	treeNode, ok = ts.get("/hello/foo")
-	if !ok {
-		t.Fatalf("Expect to get node, but not")
-	}
-	if treeNode.Value != "barbarbar" {
-		t.Fatalf("Expect value barbarbar, but got %s", treeNode.Value)
-	}
-
-	// create a key under other key
-	ok = ts.set("/foo/foo", NewTestNode("bar"))
-	if ok {
-		t.Fatalf("shoud not add key under a exisiting key")
-	}
-
-	// delete a key
-	ok = ts.delete("/foo")
-	if !ok {
-		t.Fatalf("cannot delete key")
-	}
-
-	// delete a directory
-	ok = ts.delete("/hello")
-	if ok {
-		t.Fatalf("Expect cannot delet /hello, but deleted! ")
-	}
-
-	// test list
-	ts.set("/hello/fooo", NewTestNode("barbarbar"))
-	ts.set("/hello/foooo/foo", NewTestNode("barbarbar"))
-
-	nodes, keys, ok := ts.list("/hello")
-
-	if !ok {
-		t.Fatalf("cannot list!")
-	} else {
-		nodes, _ := nodes.([]*Node)
-		length := len(nodes)
-
-		for i := 0; i < length; i++ {
-			fmt.Println(keys[i], "=", nodes[i].Value)
-		}
-	}
-
-	keys = GenKeys(100, 10)
-
-	for i := 0; i < 100; i++ {
-		value := strconv.Itoa(rand.Int())
-		ts.set(keys[i], NewTestNode(value))
-		treeNode, ok := ts.get(keys[i])
-
-		if !ok {
-			continue
-		}
-		if treeNode.Value != value {
-			t.Fatalf("Expect value %s, but got %s", value, treeNode.Value)
-		}
-
-	}
-	ts.traverse(f, true)
-}
-
-func TestTreeClone(t *testing.T) {
-	keys := GenKeys(10000, 10)
-
-	ts := &tree{
-		&treeNode{
-			NewTestNode("/"),
-			true,
-			make(map[string]*treeNode),
-		},
-	}
-
-	backTs := &tree{
-		&treeNode{
-			NewTestNode("/"),
-			true,
-			make(map[string]*treeNode),
-		},
-	}
-
-	// generate the first tree
-	for _, key := range keys {
-		value := strconv.Itoa(rand.Int())
-		ts.set(key, NewTestNode(value))
-		backTs.set(key, NewTestNode(value))
-	}
-
-	copyTs := ts.clone()
-
-	// test if they are identical
-	copyTs.traverse(ts.contain, false)
-
-	// remove all the keys from first tree
-	for _, key := range keys {
-		ts.delete(key)
-	}
-
-	// test if they are identical
-	// make sure changes in the first tree will affect the copy one
-	copyTs.traverse(backTs.contain, false)
-
-}
-
-func BenchmarkTreeStoreSet(b *testing.B) {
-
-	keys := GenKeys(10000, 10)
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-
-		ts := &tree{
-			&treeNode{
-				NewTestNode("/"),
-				true,
-				make(map[string]*treeNode),
-			},
-		}
-
-		for _, key := range keys {
-			value := strconv.Itoa(rand.Int())
-			ts.set(key, NewTestNode(value))
-		}
-	}
-}
-
-func BenchmarkTreeStoreGet(b *testing.B) {
-
-	keys := GenKeys(10000, 10)
-
-	ts := &tree{
-		&treeNode{
-			NewTestNode("/"),
-			true,
-			make(map[string]*treeNode),
-		},
-	}
-
-	for _, key := range keys {
-		value := strconv.Itoa(rand.Int())
-		ts.set(key, NewTestNode(value))
-	}
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		for _, key := range keys {
-			ts.get(key)
-		}
-	}
-}
-
-func BenchmarkTreeStoreCopy(b *testing.B) {
-	keys := GenKeys(10000, 10)
-
-	ts := &tree{
-		&treeNode{
-			NewTestNode("/"),
-			true,
-			make(map[string]*treeNode),
-		},
-	}
-
-	for _, key := range keys {
-		value := strconv.Itoa(rand.Int())
-		ts.set(key, NewTestNode(value))
-	}
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		ts.clone()
-	}
-}
-
-func BenchmarkTreeStoreList(b *testing.B) {
-
-	keys := GenKeys(10000, 10)
-
-	ts := &tree{
-		&treeNode{
-			NewTestNode("/"),
-			true,
-			make(map[string]*treeNode),
-		},
-	}
-
-	for _, key := range keys {
-		value := strconv.Itoa(rand.Int())
-		ts.set(key, NewTestNode(value))
-	}
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		for _, key := range keys {
-			ts.list(key)
-		}
-	}
-}
-
-func (t *tree) contain(key string, node *Node) {
-	_, ok := t.get(key)
-	if !ok {
-		panic("tree do not contain the given key")
-	}
-}
-
-func f(key string, n *Node) {
-	return
-}
-
-func NewTestNode(value string) Node {
-	return Node{value, time.Unix(0, 0), nil}
-}
--- a/store/ttl.go
+++ b/store/ttl.go
@ -0,0 +1,20 @@
+package store
+
+import (
+	"strconv"
+	"time"
+)
+
+// Convert string duration to time format
+func TTL(duration string) (time.Time, error) {
+	if duration != "" {
+		duration, err := strconv.Atoi(duration)
+		if err != nil {
+			return Permanent, err
+		}
+		return time.Now().Add(time.Second * (time.Duration)(duration)), nil
+
+	} else {
+		return Permanent, nil
+	}
+}
--- a/store/update_command.go
+++ b/store/update_command.go
@ -0,0 +1,37 @@
+package store
+
+import (
+	"github.com/coreos/etcd/log"
+	"github.com/coreos/go-raft"
+	"time"
+)
+
+func init() {
+	raft.RegisterCommand(&UpdateCommand{})
+}
+
+// Update command
+type UpdateCommand struct {
+	Key        string    `json:"key"`
+	Value      string    `json:"value"`
+	ExpireTime time.Time `json:"expireTime"`
+}
+
+// The name of the update command in the log
+func (c *UpdateCommand) CommandName() string {
+	return "etcd:update"
+}
+
+// Create node
+func (c *UpdateCommand) Apply(server raft.Server) (interface{}, error) {
+	s, _ := server.StateMachine().(Store)
+
+	e, err := s.Update(c.Key, c.Value, c.ExpireTime, server.CommitIndex(), server.Term())
+
+	if err != nil {
+		log.Debug(err)
+		return nil, err
+	}
+
+	return e, nil
+}
--- a/store/watcher.go
+++ b/store/watcher.go
@ -1,129 +1,33 @@
 package store

-import (
-	"path"
-	"strconv"
-	"strings"
-)
-
-//------------------------------------------------------------------------------
-//
-// Typedefs
-//
-//------------------------------------------------------------------------------
-
-// WatcherHub is where the client register its watcher
-type WatcherHub struct {
-	watchers map[string][]*Watcher
+type watcher struct {
+	eventChan  chan *Event
+	recursive  bool
+	sinceIndex uint64
 }

-// Currently watcher only contains a response channel
-type Watcher struct {
-	C chan *Response
-}
+// notify function notifies the watcher. If the watcher interests in the given path,
+// the function will return true.
+func (w *watcher) notify(e *Event, originalPath bool, deleted bool) bool {
+	// watcher is interested the path in three cases and under one condition
+	// the condition is that the event happens after the watcher's sinceIndex

-// Create a new watcherHub
-func newWatcherHub() *WatcherHub {
-	w := new(WatcherHub)
-	w.watchers = make(map[string][]*Watcher)
-	return w
-}
+	// 1. the path at which the event happens is the path the watcher is watching at.
+	// For example if the watcher is watching at "/foo" and the event happens at "/foo",
+	// the watcher must be interested in that event.

-// Create a new watcher
-func NewWatcher() *Watcher {
-	return &Watcher{C: make(chan *Response, 1)}
-}
+	// 2. the watcher is a recursive watcher, it interests in the event happens after
+	// its watching path. For example if watcher A watches at "/foo" and it is a recursive
+	// one, it will interest in the event happens at "/foo/bar".

-// Add a watcher to the watcherHub
-func (w *WatcherHub) addWatcher(prefix string, watcher *Watcher, sinceIndex uint64,
-	responseStartIndex uint64, currentIndex uint64, resMap map[string]*Response) error {
+	// 3. when we delete a directory, we need to force notify all the watchers who watches
+	// at the file we need to delete.
+	// For example a watcher is watching at "/foo/bar". And we deletes "/foo". The watcher
+	// should get notified even if "/foo" is not the path it is watching.

-	prefix = path.Clean("/" + prefix)
-
-	if sinceIndex != 0 && sinceIndex >= responseStartIndex {
-		for i := sinceIndex; i <= currentIndex; i++ {
-			if checkResponse(prefix, i, resMap) {
-				watcher.C <- resMap[strconv.FormatUint(i, 10)]
-				return nil
-			}
-		}
+	if (w.recursive || originalPath || deleted) && e.Index >= w.sinceIndex {
+		w.eventChan <- e
+		return true
 	}
-
-	_, ok := w.watchers[prefix]
-
-	if !ok {
-		w.watchers[prefix] = make([]*Watcher, 0)
-	}
-
-	w.watchers[prefix] = append(w.watchers[prefix], watcher)
-
-	return nil
-}
-
-// Check if the response has what we are watching
-func checkResponse(prefix string, index uint64, resMap map[string]*Response) bool {
-
-	resp, ok := resMap[strconv.FormatUint(index, 10)]
-
-	if !ok {
-		// not storage system command
-		return false
-	} else {
-		path := resp.Key
-		if strings.HasPrefix(path, prefix) {
-			prefixLen := len(prefix)
-			if len(path) == prefixLen || path[prefixLen] == '/' {
-				return true
-			}
-
-		}
-	}
-
 	return false
 }
-
-// Notify the watcher a action happened
-func (w *WatcherHub) notify(resp Response) error {
-	resp.Key = path.Clean(resp.Key)
-
-	segments := strings.Split(resp.Key, "/")
-	currPath := "/"
-
-	// walk through all the pathes
-	for _, segment := range segments {
-		currPath = path.Join(currPath, segment)
-
-		watchers, ok := w.watchers[currPath]
-
-		if ok {
-
-			newWatchers := make([]*Watcher, 0)
-			// notify all the watchers
-			for _, watcher := range watchers {
-				watcher.C <- &resp
-			}
-
-			if len(newWatchers) == 0 {
-				// we have notified all the watchers at this path
-				// delete the map
-				delete(w.watchers, currPath)
-			} else {
-				w.watchers[currPath] = newWatchers
-			}
-		}
-
-	}
-
-	return nil
-}
-
-// stopWatchers stops all the watchers
-// This function is used when the etcd recovery from a snapshot at runtime
-func (w *WatcherHub) stopWatchers() {
-	for _, subWatchers := range w.watchers {
-		for _, watcher := range subWatchers {
-			watcher.C <- nil
-		}
-	}
-	w.watchers = nil
-}
--- a/store/watcher_hub.go
+++ b/store/watcher_hub.go
@ -0,0 +1,142 @@
+package store
+
+import (
+	"container/list"
+	"path"
+	"strings"
+	"sync/atomic"
+
+	etcdErr "github.com/coreos/etcd/error"
+)
+
+// A watcherHub contains all subscribed watchers
+// watchers is a map with watched path as key and watcher as value
+// EventHistory keeps the old events for watcherHub. It is used to help
+// watcher to get a continuous event history. Or a watcher might miss the
+// event happens between the end of the first watch command and the start
+// of the second command.
+type watcherHub struct {
+	watchers     map[string]*list.List
+	count        int64 // current number of watchers.
+	EventHistory *EventHistory
+}
+
+// newWatchHub creates a watchHub. The capacity determines how many events we will
+// keep in the eventHistory.
+// Typically, we only need to keep a small size of history[smaller than 20K].
+// Ideally, it should smaller than 20K/s[max throughput] * 2 * 50ms[RTT] = 2000
+func newWatchHub(capacity int) *watcherHub {
+	return &watcherHub{
+		watchers:     make(map[string]*list.List),
+		EventHistory: newEventHistory(capacity),
+	}
+}
+
+// watch function returns an Event channel.
+// If recursive is true, the first change after index under prefix will be sent to the event channel.
+// If recursive is false, the first change after index at prefix will be sent to the event channel.
+// If index is zero, watch will start from the current index + 1.
+func (wh *watcherHub) watch(prefix string, recursive bool, index uint64) (<-chan *Event, *etcdErr.Error) {
+	eventChan := make(chan *Event, 1)
+
+	e, err := wh.EventHistory.scan(prefix, index)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if e != nil {
+		eventChan <- e
+		return eventChan, nil
+	}
+
+	w := &watcher{
+		eventChan:  eventChan,
+		recursive:  recursive,
+		sinceIndex: index - 1, // to catch Expire()
+	}
+
+	l, ok := wh.watchers[prefix]
+
+	if ok { // add the new watcher to the back of the list
+		l.PushBack(w)
+
+	} else { // create a new list and add the new watcher
+		l := list.New()
+		l.PushBack(w)
+		wh.watchers[prefix] = l
+	}
+
+	atomic.AddInt64(&wh.count, 1)
+
+	return eventChan, nil
+}
+
+// notify function accepts an event and notify to the watchers.
+func (wh *watcherHub) notify(e *Event) {
+	e = wh.EventHistory.addEvent(e) // add event into the eventHistory
+
+	segments := strings.Split(e.Key, "/")
+
+	currPath := "/"
+
+	// walk through all the segments of the path and notify the watchers
+	// if the path is "/foo/bar", it will notify watchers with path "/",
+	// "/foo" and "/foo/bar"
+
+	for _, segment := range segments {
+		currPath = path.Join(currPath, segment)
+		// notify the watchers who interests in the changes of current path
+		wh.notifyWatchers(e, currPath, false)
+	}
+}
+
+func (wh *watcherHub) notifyWatchers(e *Event, path string, deleted bool) {
+	l, ok := wh.watchers[path]
+
+	if ok {
+		curr := l.Front()
+		notifiedAll := true
+
+		for {
+			if curr == nil { // we have reached the end of the list
+				if notifiedAll {
+					// if we have notified all watcher in the list
+					// we can delete the list
+					delete(wh.watchers, path)
+				}
+
+				break
+			}
+
+			next := curr.Next() // save reference to the next one in the list
+
+			w, _ := curr.Value.(*watcher)
+
+			if w.notify(e, e.Key == path, deleted) {
+				// if we successfully notify a watcher
+				// we need to remove the watcher from the list
+				// and decrease the counter
+
+				l.Remove(curr)
+				atomic.AddInt64(&wh.count, -1)
+			} else {
+				// once there is a watcher in the list is not interested
+				// in the event, we should keep the list in the map
+				notifiedAll = false
+			}
+
+			curr = next // update current to the next
+		}
+	}
+}
+
+// clone function clones the watcherHub and return the cloned one.
+// only clone the static content. do not clone the current watchers.
+func (wh *watcherHub) clone() *watcherHub {
+	clonedHistory := wh.EventHistory.clone()
+
+	return &watcherHub{
+		EventHistory: clonedHistory,
+	}
+}
--- a/store/watcher_test.go
+++ b/store/watcher_test.go
@ -2,83 +2,54 @@ package store

 import (
 	"testing"
-	"time"
 )

-func TestWatch(t *testing.T) {
-
-	s := CreateStore(100)
-
-	watchers := make([]*Watcher, 10)
-
-	for i, _ := range watchers {
-
-		// create a new watcher
-		watchers[i] = NewWatcher()
-		// add to the watchers list
-		s.AddWatcher("foo", watchers[i], 0)
-
+func TestWatcher(t *testing.T) {
+	s := newStore()
+	wh := s.WatcherHub
+	c, err := wh.watch("/foo", true, 1)
+	if err != nil {
+		t.Fatal("%v", err)
 	}

-	s.Set("/foo/foo", "bar", time.Unix(0, 0), 1)
-
-	for _, watcher := range watchers {
-
-		// wait for the notification for any changing
-		res := <-watcher.C
-
-		if res == nil {
-			t.Fatal("watcher is cleared")
-		}
+	select {
+	case <-c:
+		t.Fatal("should not receive from channel before send the event")
+	default:
+		// do nothing
 	}

-	for i, _ := range watchers {
+	e := newEvent(Create, "/foo/bar", 1, 1)

-		// create a new watcher
-		watchers[i] = NewWatcher()
-		// add to the watchers list
-		s.AddWatcher("foo/foo/foo", watchers[i], 0)
+	wh.notify(e)

+	re := <-c
+
+	if e != re {
+		t.Fatal("recv != send")
 	}

-	s.watcher.stopWatchers()
+	c, _ = wh.watch("/foo", false, 2)

-	for _, watcher := range watchers {
+	e = newEvent(Create, "/foo/bar", 2, 1)

-		// wait for the notification for any changing
-		res := <-watcher.C
+	wh.notify(e)

-		if res != nil {
-			t.Fatal("watcher is cleared")
-		}
-	}
-}
-
-// BenchmarkWatch creates 10K watchers watch at /foo/[path] each time.
-// Path is randomly chosen with max depth 10.
-// It should take less than 15ms to wake up 10K watchers.
-func BenchmarkWatch(b *testing.B) {
-	s := CreateStore(100)
-
-	keys := GenKeys(10000, 10)
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		watchers := make([]*Watcher, 10000)
-		for i := 0; i < 10000; i++ {
-			// create a new watcher
-			watchers[i] = NewWatcher()
-			// add to the watchers list
-			s.AddWatcher(keys[i], watchers[i], 0)
-		}
-
-		s.watcher.stopWatchers()
-
-		for _, watcher := range watchers {
-			// wait for the notification for any changing
-			<-watcher.C
-		}
-
-		s.watcher = newWatcherHub()
+	select {
+	case re = <-c:
+		t.Fatal("should not receive from channel if not recursive ", re)
+	default:
+		// do nothing
 	}
+
+	e = newEvent(Create, "/foo", 3, 1)
+
+	wh.notify(e)
+
+	re = <-c
+
+	if e != re {
+		t.Fatal("recv != send")
+	}
+
 }
--- a/8
+++ b/8
@ -1,8 +0,0 @@
-#!/bin/sh
-
-# Get GOPATH, etc from build
-. ./build
-
-# Run the tests!
-go test -i
-go test -v
--- a/test.sh
+++ b/test.sh
@ -0,0 +1,11 @@
+#!/bin/sh
+set -e
+
+# Get GOPATH, etc from build
+. ./build
+
+# Unit tests
+go test -v ./store
+
+# Functional tests
+ETCD_BIN_PATH=$(pwd)/etcd go test -v ./tests/functional
--- a/tests/functional/etcd_direct_call.go
+++ b/tests/functional/etcd_direct_call.go
@ -0,0 +1,34 @@
+package test
+
+import (
+	"net/http"
+	"os"
+	"testing"
+	"time"
+)
+
+func BenchmarkEtcdDirectCall(b *testing.B) {
+	templateBenchmarkEtcdDirectCall(b, false)
+}
+
+func BenchmarkEtcdDirectCallTls(b *testing.B) {
+	templateBenchmarkEtcdDirectCall(b, true)
+}
+
+func templateBenchmarkEtcdDirectCall(b *testing.B, tls bool) {
+	procAttr := new(os.ProcAttr)
+	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
+
+	clusterSize := 3
+	_, etcds, _ := CreateCluster(clusterSize, procAttr, tls)
+
+	defer DestroyCluster(etcds)
+
+	time.Sleep(time.Second)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		resp, _ := http.Get("http://127.0.0.1:4001/test/speed")
+		resp.Body.Close()
+	}
+}
--- a/tests/functional/init.go
+++ b/tests/functional/init.go
@ -0,0 +1,17 @@
+package test
+
+import (
+	"go/build"
+	"os"
+	"path/filepath"
+)
+
+var EtcdBinPath string
+
+func init() {
+	// Initialize the 'etcd' binary path or default it to the etcd diretory.
+	EtcdBinPath = os.Getenv("ETCD_BIN_PATH")
+	if EtcdBinPath == "" {
+		EtcdBinPath = filepath.Join(build.Default.GOPATH, "src", "github.com", "coreos", "etcd", "etcd")
+	}
+}
--- a/tests/functional/internal_version_test.go
+++ b/tests/functional/internal_version_test.go
@ -0,0 +1,57 @@
+package test
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"testing"
+	"time"
+)
+
+// Ensure that etcd does not come up if the internal raft versions do not match.
+func TestInternalVersion(t *testing.T) {
+	checkedVersion := false
+	testMux := http.NewServeMux()
+
+	testMux.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, "This is not a version number")
+		checkedVersion = true
+	})
+
+	testMux.HandleFunc("/join", func(w http.ResponseWriter, r *http.Request) {
+		t.Fatal("should not attempt to join!")
+	})
+
+	ts := httptest.NewServer(testMux)
+	defer ts.Close()
+
+	fakeURL, _ := url.Parse(ts.URL)
+
+	procAttr := new(os.ProcAttr)
+	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
+	args := []string{"etcd", "-n=node1", "-f", "-d=/tmp/node1", "-C=" + fakeURL.Host}
+
+	process, err := os.StartProcess(EtcdBinPath, args, procAttr)
+	if err != nil {
+		t.Fatal("start process failed:" + err.Error())
+		return
+	}
+	defer process.Kill()
+
+	time.Sleep(time.Second)
+
+	_, err = http.Get("http://127.0.0.1:4001")
+
+	if err == nil {
+		t.Fatal("etcd node should not be up")
+		return
+	}
+
+	if checkedVersion == false {
+		t.Fatal("etcd did not check the version")
+		return
+	}
+}
+
--- a/tests/functional/kill_leader_test.go
+++ b/tests/functional/kill_leader_test.go
@ -0,0 +1,63 @@
+package test
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+)
+
+// This test will kill the current leader and wait for the etcd cluster to elect a new leader for 200 times.
+// It will print out the election time and the average election time.
+func TestKillLeader(t *testing.T) {
+	procAttr := new(os.ProcAttr)
+	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
+
+	clusterSize := 5
+	argGroup, etcds, err := CreateCluster(clusterSize, procAttr, false)
+	if err != nil {
+		t.Fatal("cannot create cluster")
+	}
+	defer DestroyCluster(etcds)
+
+	stop := make(chan bool)
+	leaderChan := make(chan string, 1)
+	all := make(chan bool, 1)
+
+	time.Sleep(time.Second)
+
+	go Monitor(clusterSize, 1, leaderChan, all, stop)
+
+	var totalTime time.Duration
+
+	leader := "http://127.0.0.1:7001"
+
+	for i := 0; i < clusterSize; i++ {
+		fmt.Println("leader is ", leader)
+		port, _ := strconv.Atoi(strings.Split(leader, ":")[2])
+		num := port - 7001
+		fmt.Println("kill server ", num)
+		etcds[num].Kill()
+		etcds[num].Release()
+
+		start := time.Now()
+		for {
+			newLeader := <-leaderChan
+			if newLeader != leader {
+				leader = newLeader
+				break
+			}
+		}
+		take := time.Now().Sub(start)
+
+		totalTime += take
+		avgTime := totalTime / (time.Duration)(i+1)
+		fmt.Println("Total time:", totalTime, "; Avg time:", avgTime)
+		
+		etcds[num], err = os.StartProcess(EtcdBinPath, argGroup[num], procAttr)
+	}
+	stop <- true
+}
+
--- a/tests/functional/kill_random_test.go
+++ b/tests/functional/kill_random_test.go
@ -0,0 +1,76 @@
+package test
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"testing"
+	"time"
+)
+
+// TestKillRandom kills random machines in the cluster and
+// restart them after all other machines agree on the same leader
+func TestKillRandom(t *testing.T) {
+	procAttr := new(os.ProcAttr)
+	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
+
+	clusterSize := 9
+	argGroup, etcds, err := CreateCluster(clusterSize, procAttr, false)
+
+	if err != nil {
+		t.Fatal("cannot create cluster")
+	}
+
+	defer DestroyCluster(etcds)
+
+	stop := make(chan bool)
+	leaderChan := make(chan string, 1)
+	all := make(chan bool, 1)
+
+	time.Sleep(3 * time.Second)
+
+	go Monitor(clusterSize, 4, leaderChan, all, stop)
+
+	toKill := make(map[int]bool)
+
+	for i := 0; i < 20; i++ {
+		fmt.Printf("TestKillRandom Round[%d/20]\n", i)
+
+		j := 0
+		for {
+
+			r := rand.Int31n(9)
+			if _, ok := toKill[int(r)]; !ok {
+				j++
+				toKill[int(r)] = true
+			}
+
+			if j > 3 {
+				break
+			}
+
+		}
+
+		for num, _ := range toKill {
+			err := etcds[num].Kill()
+			if err != nil {
+				panic(err)
+			}
+			etcds[num].Wait()
+		}
+
+		time.Sleep(1 * time.Second)
+
+		<-leaderChan
+
+		for num, _ := range toKill {
+			etcds[num], err = os.StartProcess(EtcdBinPath, argGroup[num], procAttr)
+		}
+
+		toKill = make(map[int]bool)
+		<-all
+	}
+
+	stop <- true
+}
+
--- a/tests/functional/multi_node_kill_all_and_recovery_test.go
+++ b/tests/functional/multi_node_kill_all_and_recovery_test.go
@ -0,0 +1,72 @@
+package test
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/coreos/go-etcd/etcd"
+)
+
+// Create a five nodes
+// Kill all the nodes and restart
+func TestMultiNodeKillAllAndRecovery(t *testing.T) {
+	procAttr := new(os.ProcAttr)
+	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
+
+	clusterSize := 5
+	argGroup, etcds, err := CreateCluster(clusterSize, procAttr, false)
+	defer DestroyCluster(etcds)
+
+	if err != nil {
+		t.Fatal("cannot create cluster")
+	}
+
+	c := etcd.NewClient(nil)
+
+	c.SyncCluster()
+
+	time.Sleep(time.Second)
+
+	// send 10 commands
+	for i := 0; i < 10; i++ {
+		// Test Set
+		_, err := c.Set("foo", "bar", 0)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	time.Sleep(time.Second)
+
+	// kill all
+	DestroyCluster(etcds)
+
+	time.Sleep(time.Second)
+
+	stop := make(chan bool)
+	leaderChan := make(chan string, 1)
+	all := make(chan bool, 1)
+
+	time.Sleep(time.Second)
+
+	for i := 0; i < clusterSize; i++ {
+		etcds[i], err = os.StartProcess(EtcdBinPath, argGroup[i], procAttr)
+	}
+
+	go Monitor(clusterSize, 1, leaderChan, all, stop)
+
+	<-all
+	<-leaderChan
+
+	result, err := c.Set("foo", "bar", 0)
+
+	if err != nil {
+		t.Fatalf("Recovery error: %s", err)
+	}
+
+	if result.Index != 18 {
+		t.Fatalf("recovery failed! [%d/18]", result.Index)
+	}
+}
+
--- a/tests/functional/multi_node_kill_one_test.go
+++ b/tests/functional/multi_node_kill_one_test.go
@ -0,0 +1,58 @@
+package test
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/coreos/go-etcd/etcd"
+)
+
+// Create a five nodes
+// Randomly kill one of the node and keep on sending set command to the cluster
+func TestMultiNodeKillOne(t *testing.T) {
+	procAttr := new(os.ProcAttr)
+	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
+
+	clusterSize := 5
+	argGroup, etcds, err := CreateCluster(clusterSize, procAttr, false)
+
+	if err != nil {
+		t.Fatal("cannot create cluster")
+	}
+
+	defer DestroyCluster(etcds)
+
+	time.Sleep(2 * time.Second)
+
+	c := etcd.NewClient(nil)
+
+	c.SyncCluster()
+
+	stop := make(chan bool)
+	// Test Set
+	go Set(stop)
+
+	for i := 0; i < 10; i++ {
+		num := rand.Int() % clusterSize
+		fmt.Println("kill node", num+1)
+
+		// kill
+		etcds[num].Kill()
+		etcds[num].Release()
+		time.Sleep(time.Second)
+
+		// restart
+		etcds[num], err = os.StartProcess(EtcdBinPath, argGroup[num], procAttr)
+		if err != nil {
+			panic(err)
+		}
+		time.Sleep(time.Second)
+	}
+	fmt.Println("stop")
+	stop <- true
+	<-stop
+}
+
--- a/tests/functional/remove_node_test.go
+++ b/tests/functional/remove_node_test.go
@ -0,0 +1,113 @@
+package test
+
+import (
+	"net/http"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/coreos/go-etcd/etcd"
+)
+
+// remove the node and node rejoin with previous log
+func TestRemoveNode(t *testing.T) {
+	procAttr := new(os.ProcAttr)
+	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
+
+	clusterSize := 3
+	argGroup, etcds, _ := CreateCluster(clusterSize, procAttr, false)
+	defer DestroyCluster(etcds)
+
+	time.Sleep(time.Second)
+
+	c := etcd.NewClient(nil)
+
+	c.SyncCluster()
+
+	rmReq, _ := http.NewRequest("DELETE", "http://127.0.0.1:7001/remove/node3", nil)
+
+	client := &http.Client{}
+	for i := 0; i < 2; i++ {
+		for i := 0; i < 2; i++ {
+			client.Do(rmReq)
+
+			etcds[2].Wait()
+
+			resp, err := c.Get("_etcd/machines")
+
+			if err != nil {
+				panic(err)
+			}
+
+			if len(resp) != 2 {
+				t.Fatal("cannot remove machine")
+			}
+
+			if i == 1 {
+				// rejoin with log
+				etcds[2], err = os.StartProcess(EtcdBinPath, argGroup[2], procAttr)
+			} else {
+				// rejoin without log
+				etcds[2], err = os.StartProcess(EtcdBinPath, append(argGroup[2], "-f"), procAttr)
+			}
+
+			if err != nil {
+				panic(err)
+			}
+
+			time.Sleep(time.Second)
+
+			resp, err = c.Get("_etcd/machines")
+
+			if err != nil {
+				panic(err)
+			}
+
+			if len(resp) != 3 {
+				t.Fatalf("add machine fails #1 (%d != 3)", len(resp))
+			}
+		}
+
+		// first kill the node, then remove it, then add it back
+		for i := 0; i < 2; i++ {
+			etcds[2].Kill()
+			etcds[2].Wait()
+
+			client.Do(rmReq)
+
+			resp, err := c.Get("_etcd/machines")
+
+			if err != nil {
+				panic(err)
+			}
+
+			if len(resp) != 2 {
+				t.Fatal("cannot remove machine")
+			}
+
+			if i == 1 {
+				// rejoin with log
+				etcds[2], err = os.StartProcess(EtcdBinPath, append(argGroup[2]), procAttr)
+			} else {
+				// rejoin without log
+				etcds[2], err = os.StartProcess(EtcdBinPath, append(argGroup[2], "-f"), procAttr)
+			}
+
+			if err != nil {
+				panic(err)
+			}
+
+			time.Sleep(time.Second)
+
+			resp, err = c.Get("_etcd/machines")
+
+			if err != nil {
+				panic(err)
+			}
+
+			if len(resp) != 3 {
+				t.Fatalf("add machine fails #2 (%d != 3)", len(resp))
+			}
+		}
+	}
+}
--- a/tests/functional/simple_multi_node_test.go
+++ b/tests/functional/simple_multi_node_test.go
@ -0,0 +1,62 @@
+package test
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/coreos/go-etcd/etcd"
+)
+
+func TestSimpleMultiNode(t *testing.T) {
+	templateTestSimpleMultiNode(t, false)
+}
+
+func TestSimpleMultiNodeTls(t *testing.T) {
+	templateTestSimpleMultiNode(t, true)
+}
+
+// Create a three nodes and try to set value
+func templateTestSimpleMultiNode(t *testing.T, tls bool) {
+	procAttr := new(os.ProcAttr)
+	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
+
+	clusterSize := 3
+
+	_, etcds, err := CreateCluster(clusterSize, procAttr, tls)
+
+	if err != nil {
+		t.Fatal("cannot create cluster")
+	}
+
+	defer DestroyCluster(etcds)
+
+	time.Sleep(time.Second)
+
+	c := etcd.NewClient(nil)
+
+	c.SyncCluster()
+
+	// Test Set
+	result, err := c.Set("foo", "bar", 100)
+
+	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.TTL < 95 {
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		t.Fatalf("Set 1 failed with %s %s %v", result.Key, result.Value, result.TTL)
+	}
+
+	time.Sleep(time.Second)
+
+	result, err = c.Set("foo", "bar", 100)
+
+	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.PrevValue != "bar" || result.TTL != 100 {
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Fatalf("Set 2 failed with %s %s %v", result.Key, result.Value, result.TTL)
+	}
+
+}
--- a/tests/functional/single_node_recovery_test.go
+++ b/tests/functional/single_node_recovery_test.go
@ -0,0 +1,68 @@
+package test
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/coreos/go-etcd/etcd"
+)
+
+// This test creates a single node and then set a value to it.
+// Then this test kills the node and restart it and tries to get the value again.
+func TestSingleNodeRecovery(t *testing.T) {
+	procAttr := new(os.ProcAttr)
+	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
+	args := []string{"etcd", "-n=node1", "-d=/tmp/node1"}
+
+	process, err := os.StartProcess(EtcdBinPath, append(args, "-f"), procAttr)
+	if err != nil {
+		t.Fatal("start process failed:" + err.Error())
+		return
+	}
+
+	time.Sleep(time.Second)
+
+	c := etcd.NewClient(nil)
+
+	c.SyncCluster()
+	// Test Set
+	result, err := c.Set("foo", "bar", 100)
+
+	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.TTL < 95 {
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		t.Fatalf("Set 1 failed with %s %s %v", result.Key, result.Value, result.TTL)
+	}
+
+	time.Sleep(time.Second)
+
+	process.Kill()
+
+	process, err = os.StartProcess(EtcdBinPath, args, procAttr)
+	defer process.Kill()
+	if err != nil {
+		t.Fatal("start process failed:" + err.Error())
+		return
+	}
+
+	time.Sleep(time.Second)
+
+	results, err := c.Get("foo")
+	if err != nil {
+		t.Fatal("get fail: " + err.Error())
+		return
+	}
+
+	result = results[0]
+
+	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.TTL > 99 {
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Fatalf("Recovery Get failed with %s %s %v", result.Key, result.Value, result.TTL)
+	}
+}
+
--- a/tests/functional/single_node_test.go
+++ b/tests/functional/single_node_test.go
@ -0,0 +1,77 @@
+package test
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/coreos/go-etcd/etcd"
+)
+
+// Create a single node and try to set value
+func TestSingleNode(t *testing.T) {
+	procAttr := new(os.ProcAttr)
+	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
+	args := []string{"etcd", "-n=node1", "-f", "-d=/tmp/node1"}
+
+	process, err := os.StartProcess(EtcdBinPath, args, procAttr)
+	if err != nil {
+		t.Fatal("start process failed:" + err.Error())
+		return
+	}
+	defer process.Kill()
+
+	time.Sleep(time.Second)
+
+	c := etcd.NewClient(nil)
+
+	c.SyncCluster()
+	// Test Set
+	result, err := c.Set("foo", "bar", 100)
+
+	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.TTL < 95 {
+		if err != nil {
+			t.Fatal("Set 1: ", err)
+		}
+
+		t.Fatalf("Set 1 failed with %s %s %v", result.Key, result.Value, result.TTL)
+	}
+
+	time.Sleep(time.Second)
+
+	result, err = c.Set("foo", "bar", 100)
+
+	if err != nil || result.Key != "/foo" || result.Value != "bar" || result.PrevValue != "bar" || result.TTL != 100 {
+		if err != nil {
+			t.Fatal("Set 2: ", err)
+		}
+		t.Fatalf("Set 2 failed with %s %s %v", result.Key, result.Value, result.TTL)
+	}
+
+	// Add a test-and-set test
+
+	// First, we'll test we can change the value if we get it write
+	result, match, err := c.TestAndSet("foo", "bar", "foobar", 100)
+
+	if err != nil || result.Key != "/foo" || result.Value != "foobar" || result.PrevValue != "bar" || result.TTL != 100 || !match {
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Fatalf("Set 3 failed with %s %s %v", result.Key, result.Value, result.TTL)
+	}
+
+	// Next, we'll make sure we can't set it without the correct prior value
+	_, _, err = c.TestAndSet("foo", "bar", "foofoo", 100)
+
+	if err == nil {
+		t.Fatalf("Set 4 expecting error when setting key with incorrect previous value")
+	}
+
+	// Finally, we'll make sure a blank previous value still counts as a test-and-set and still has to match
+	_, _, err = c.TestAndSet("foo", "", "barbar", 100)
+
+	if err == nil {
+		t.Fatalf("Set 5 expecting error when setting key with blank (incorrect) previous value")
+	}
+}
+
--- a/tests/functional/util.go
+++ b/tests/functional/util.go
@ -1,4 +1,4 @@
-package main
+package test

 import (
 	"fmt"
@ -18,11 +18,11 @@ var client = http.Client{
 }

 // Sending set commands
-func set(stop chan bool) {
+func Set(stop chan bool) {

 	stopSet := false
 	i := 0
-	c := etcd.NewClient()
+	c := etcd.NewClient(nil)
 	for {
 		key := fmt.Sprintf("%s_%v", "foo", i)

@ -50,12 +50,11 @@ func set(stop chan bool) {

 		i++
 	}
-	fmt.Println("set stop")
 	stop <- true
 }

 // Create a cluster of etcd nodes
-func createCluster(size int, procAttr *os.ProcAttr, ssl bool) ([][]string, []*os.Process, error) {
+func CreateCluster(size int, procAttr *os.ProcAttr, ssl bool) ([][]string, []*os.Process, error) {
 	argGroup := make([][]string, size)

 	sslServer1 := []string{"-serverCAFile=./fixtures/ca/ca.crt",
@ -70,7 +69,7 @@ func createCluster(size int, procAttr *os.ProcAttr, ssl bool) ([][]string, []*os

 	for i := 0; i < size; i++ {
 		if i == 0 {
-			argGroup[i] = []string{"etcd", "-d=/tmp/node1", "-n=node1", "-vv"}
+			argGroup[i] = []string{"etcd", "-d=/tmp/node1", "-n=node1"}
 			if ssl {
 				argGroup[i] = append(argGroup[i], sslServer1...)
 			}
@ -87,7 +86,7 @@ func createCluster(size int, procAttr *os.ProcAttr, ssl bool) ([][]string, []*os

 	for i, _ := range etcds {
 		var err error
-		etcds[i], err = os.StartProcess("etcd", append(argGroup[i], "-f"), procAttr)
+		etcds[i], err = os.StartProcess(EtcdBinPath, append(argGroup[i], "-f"), procAttr)
 		if err != nil {
 			return nil, nil, err
 		}
@ -97,7 +96,7 @@ func createCluster(size int, procAttr *os.ProcAttr, ssl bool) ([][]string, []*os
 		// have to retry. This retry can take upwards of 15 seconds
 		// which slows tests way down and some of them fail.
 		if i == 0 {
-			time.Sleep(time.Second)
+			time.Sleep(time.Second * 2)
 		}
 	}

@ -105,10 +104,9 @@ func createCluster(size int, procAttr *os.ProcAttr, ssl bool) ([][]string, []*os
 }

 // Destroy all the nodes in the cluster
-func destroyCluster(etcds []*os.Process) error {
-	for i, etcd := range etcds {
+func DestroyCluster(etcds []*os.Process) error {
+	for _, etcd := range etcds {
 		err := etcd.Kill()
-		fmt.Println("kill ", i)
 		if err != nil {
 			panic(err.Error())
 		}
@ -118,7 +116,7 @@ func destroyCluster(etcds []*os.Process) error {
 }

 //
-func leaderMonitor(size int, allowDeadNum int, leaderChan chan string) {
+func Monitor(size int, allowDeadNum int, leaderChan chan string, all chan bool, stop chan bool) {
 	leaderMap := make(map[int]string)
 	baseAddrFormat := "http://0.0.0.0:400%d"

@ -153,6 +151,8 @@ func leaderMonitor(size int, allowDeadNum int, leaderChan chan string) {

 		if i == size {
 			select {
+			case <-stop:
+				return
 			case <-leaderChan:
 				leaderChan <- knownLeader
 			default:
@ -160,6 +160,14 @@ func leaderMonitor(size int, allowDeadNum int, leaderChan chan string) {
 			}

 		}
+		if dead == 0 {
+			select {
+			case <-all:
+				all <- true
+			default:
+				all <- true
+			}
+		}

 		time.Sleep(time.Millisecond * 10)
 	}
@ -168,7 +176,7 @@ func leaderMonitor(size int, allowDeadNum int, leaderChan chan string) {

 func getLeader(addr string) (string, error) {

-	resp, err := client.Get(addr + "/leader")
+	resp, err := client.Get(addr + "/v1/leader")

 	if err != nil {
 		return "", err
@ -191,28 +199,6 @@ func getLeader(addr string) (string, error) {

 }

-func directSet() {
-	c := make(chan bool, 1000)
-	for i := 0; i < 1000; i++ {
-		go send(c)
-	}
-
-	for i := 0; i < 1000; i++ {
-		<-c
-	}
-}
-
-func send(c chan bool) {
-	for i := 0; i < 10; i++ {
-		command := &SetCommand{}
-		command.Key = "foo"
-		command.Value = "bar"
-		command.ExpireTime = time.Unix(0, 0)
-		raftServer.Do(command)
-	}
-	c <- true
-}
-
 // Dial with timeout
 func dialTimeoutFast(network, addr string) (net.Conn, error) {
 	return net.DialTimeout(network, addr, time.Millisecond*10)
--- a/third_party/bitbucket.org/kardianos/osext/osext_sysctl.go
+++ b/third_party/bitbucket.org/kardianos/osext/osext_sysctl.go
@ -8,6 +8,7 @@ package osext

 import (
 	"os"
+	"path/filepath"
 	"runtime"
 	"syscall"
 	"unsafe"
@ -47,18 +48,35 @@ func executable() (string, error) {
 			break
 		}
 	}
+	var strpath string
 	if buf[0] != '/' {
-		if getwdError != nil {
-			return string(buf), getwdError
-		} else {
-			if buf[0] == '.' {
-				buf = buf[1:]
-			}
-			if startUpcwd[len(startUpcwd)-1] != '/' {
-				return startUpcwd + "/" + string(buf), nil
-			}
-			return startUpcwd + string(buf), nil
+		var e error
+		if strpath, e = getAbs(buf); e != nil {
+			return strpath, e
+		}
+	} else {
+		strpath = string(buf)
+	}
+	// darwin KERN_PROCARGS may return the path to a symlink rather than the
+	// actual executable
+	if runtime.GOOS == "darwin" {
+		if strpath, err := filepath.EvalSymlinks(strpath); err != nil {
+			return strpath, err
 		}
 	}
-	return string(buf), nil
+	return strpath, nil
+}
+
+func getAbs(buf []byte) (string, error) {
+	if getwdError != nil {
+		return string(buf), getwdError
+	} else {
+		if buf[0] == '.' {
+			buf = buf[1:]
+		}
+		if startUpcwd[len(startUpcwd)-1] != '/' && buf[0] != '/' {
+			return startUpcwd + "/" + string(buf), nil
+		}
+		return startUpcwd + string(buf), nil
+	}
 }
--- a/third_party/code.google.com/p/go.net/ipv4/gen.go
+++ b/third_party/code.google.com/p/go.net/ipv4/gen.go
@ -97,20 +97,16 @@ func parseICMPv4Parameters(w io.Writer, r io.Reader) error {
 }

 type icmpv4Parameters struct {
-	XMLName    xml.Name              `xml:"registry"`
-	Title      string                `xml:"title"`
-	Updated    string                `xml:"updated"`
-	Registries []icmpv4ParamRegistry `xml:"registry"`
-}
-
-type icmpv4ParamRegistry struct {
-	Title   string              `xml:"title"`
-	Records []icmpv4ParamRecord `xml:"record"`
-}
-
-type icmpv4ParamRecord struct {
-	Value string `xml:"value"`
-	Descr string `xml:"description"`
+	XMLName    xml.Name `xml:"registry"`
+	Title      string   `xml:"title"`
+	Updated    string   `xml:"updated"`
+	Registries []struct {
+		Title   string `xml:"title"`
+		Records []struct {
+			Value string `xml:"value"`
+			Descr string `xml:"description"`
+		} `xml:"record"`
+	} `xml:"registry"`
 }

 type canonICMPv4ParamRecord struct {
@ -193,18 +189,16 @@ func parseProtocolNumbers(w io.Writer, r io.Reader) error {
 }

 type protocolNumbers struct {
-	XMLName  xml.Name         `xml:"registry"`
-	Title    string           `xml:"title"`
-	Updated  string           `xml:"updated"`
-	RegTitle string           `xml:"registry>title"`
-	Note     string           `xml:"registry>note"`
-	Records  []protocolRecord `xml:"registry>record"`
-}
-
-type protocolRecord struct {
-	Value string `xml:"value"`
-	Name  string `xml:"name"`
-	Descr string `xml:"description"`
+	XMLName  xml.Name `xml:"registry"`
+	Title    string   `xml:"title"`
+	Updated  string   `xml:"updated"`
+	RegTitle string   `xml:"registry>title"`
+	Note     string   `xml:"registry>note"`
+	Records  []struct {
+		Value string `xml:"value"`
+		Name  string `xml:"name"`
+		Descr string `xml:"description"`
+	} `xml:"registry>record"`
 }

 type canonProtocolRecord struct {
--- a/third_party/code.google.com/p/go.net/ipv4/gentest.go
+++ b/third_party/code.google.com/p/go.net/ipv4/gentest.go
@ -39,7 +39,7 @@ var registries = []struct {

 func main() {
 	var bb bytes.Buffer
-	fmt.Fprintf(&bb, "// go run gentv.go\n")
+	fmt.Fprintf(&bb, "// go run gentest.go\n")
 	fmt.Fprintf(&bb, "// GENERATED BY THE COMMAND ABOVE; DO NOT EDIT\n\n")
 	fmt.Fprintf(&bb, "package ipv4_test\n\n")
 	for _, r := range registries {
@ -85,18 +85,19 @@ func parseDSCPRegistry(w io.Writer, r io.Reader) error {
 }

 type dscpRegistry struct {
-	XMLName     xml.Name     `xml:"registry"`
-	Title       string       `xml:"title"`
-	Updated     string       `xml:"updated"`
-	Note        string       `xml:"note"`
-	RegTitle    string       `xml:"registry>title"`
-	PoolRecords []dscpRecord `xml:"registry>record"`
-	Records     []dscpRecord `xml:"registry>registry>record"`
-}
-
-type dscpRecord struct {
-	Name  string `xml:"name"`
-	Space string `xml:"space"`
+	XMLName     xml.Name `xml:"registry"`
+	Title       string   `xml:"title"`
+	Updated     string   `xml:"updated"`
+	Note        string   `xml:"note"`
+	RegTitle    string   `xml:"registry>title"`
+	PoolRecords []struct {
+		Name  string `xml:"name"`
+		Space string `xml:"space"`
+	} `xml:"registry>record"`
+	Records []struct {
+		Name  string `xml:"name"`
+		Space string `xml:"space"`
+	} `xml:"registry>registry>record"`
 }

 type canonDSCPRecord struct {
@ -145,17 +146,15 @@ func parseTOSTCByte(w io.Writer, r io.Reader) error {
 }

 type tosTCByte struct {
-	XMLName  xml.Name          `xml:"registry"`
-	Title    string            `xml:"title"`
-	Updated  string            `xml:"updated"`
-	Note     string            `xml:"note"`
-	RegTitle string            `xml:"registry>title"`
-	Records  []tosTCByteRecord `xml:"registry>record"`
-}
-
-type tosTCByteRecord struct {
-	Binary  string `xml:"binary"`
-	Keyword string `xml:"keyword"`
+	XMLName  xml.Name `xml:"registry"`
+	Title    string   `xml:"title"`
+	Updated  string   `xml:"updated"`
+	Note     string   `xml:"note"`
+	RegTitle string   `xml:"registry>title"`
+	Records  []struct {
+		Binary  string `xml:"binary"`
+		Keyword string `xml:"keyword"`
+	} `xml:"registry>record"`
 }

 type canonTOSTCByteRecord struct {
--- a/third_party/code.google.com/p/go.net/ipv4/header.go
+++ b/third_party/code.google.com/p/go.net/ipv4/header.go
@ -36,41 +36,47 @@ const (
 	maxHeaderLen = 60 // sensible default, revisit if later RFCs define new usage of version and header length fields
 )

-type headerField int
+const (
+	posTOS      = 1  // type-of-service
+	posTotalLen = 2  // packet total length
+	posID       = 4  // identification
+	posFragOff  = 6  // fragment offset
+	posTTL      = 8  // time-to-live
+	posProtocol = 9  // next protocol
+	posChecksum = 10 // checksum
+	posSrc      = 12 // source address
+	posDst      = 16 // destination address
+)
+
+type HeaderFlags int

 const (
-	posTOS      headerField = 1  // type-of-service
-	posTotalLen             = 2  // packet total length
-	posID                   = 4  // identification
-	posFragOff              = 6  // fragment offset
-	posTTL                  = 8  // time-to-live
-	posProtocol             = 9  // next protocol
-	posChecksum             = 10 // checksum
-	posSrc                  = 12 // source address
-	posDst                  = 16 // destination address
+	MoreFragments HeaderFlags = 1 << iota // more fragments flag
+	DontFragment                          // don't fragment flag
 )

 // A Header represents an IPv4 header.
 type Header struct {
-	Version  int    // protocol version
-	Len      int    // header length
-	TOS      int    // type-of-service
-	TotalLen int    // packet total length
-	ID       int    // identification
-	FragOff  int    // fragment offset
-	TTL      int    // time-to-live
-	Protocol int    // next protocol
-	Checksum int    // checksum
-	Src      net.IP // source address
-	Dst      net.IP // destination address
-	Options  []byte // options, extension headers
+	Version  int         // protocol version
+	Len      int         // header length
+	TOS      int         // type-of-service
+	TotalLen int         // packet total length
+	ID       int         // identification
+	Flags    HeaderFlags // flags
+	FragOff  int         // fragment offset
+	TTL      int         // time-to-live
+	Protocol int         // next protocol
+	Checksum int         // checksum
+	Src      net.IP      // source address
+	Dst      net.IP      // destination address
+	Options  []byte      // options, extension headers
 }

 func (h *Header) String() string {
 	if h == nil {
 		return "<nil>"
 	}
-	return fmt.Sprintf("ver: %v, hdrlen: %v, tos: %#x, totallen: %v, id: %#x, fragoff: %#x, ttl: %v, proto: %v, cksum: %#x, src: %v, dst: %v", h.Version, h.Len, h.TOS, h.TotalLen, h.ID, h.FragOff, h.TTL, h.Protocol, h.Checksum, h.Src, h.Dst)
+	return fmt.Sprintf("ver: %v, hdrlen: %v, tos: %#x, totallen: %v, id: %#x, flags: %#x, fragoff: %#x, ttl: %v, proto: %v, cksum: %#x, src: %v, dst: %v", h.Version, h.Len, h.TOS, h.TotalLen, h.ID, h.Flags, h.FragOff, h.TTL, h.Protocol, h.Checksum, h.Src, h.Dst)
 }

 // Please refer to the online manual; IP(4) on Darwin, FreeBSD and
@ -89,12 +95,13 @@ func (h *Header) Marshal() ([]byte, error) {
 	b := make([]byte, hdrlen)
 	b[0] = byte(Version<<4 | (hdrlen >> 2 & 0x0f))
 	b[posTOS] = byte(h.TOS)
+	flagsAndFragOff := (h.FragOff & 0x1fff) | int(h.Flags<<13)
 	if supportsNewIPInput {
 		b[posTotalLen], b[posTotalLen+1] = byte(h.TotalLen>>8), byte(h.TotalLen)
-		b[posFragOff], b[posFragOff+1] = byte(h.FragOff>>8), byte(h.FragOff)
+		b[posFragOff], b[posFragOff+1] = byte(flagsAndFragOff>>8), byte(flagsAndFragOff)
 	} else {
 		*(*uint16)(unsafe.Pointer(&b[posTotalLen : posTotalLen+1][0])) = uint16(h.TotalLen)
-		*(*uint16)(unsafe.Pointer(&b[posFragOff : posFragOff+1][0])) = uint16(h.FragOff)
+		*(*uint16)(unsafe.Pointer(&b[posFragOff : posFragOff+1][0])) = uint16(flagsAndFragOff)
 	}
 	b[posID], b[posID+1] = byte(h.ID>>8), byte(h.ID)
 	b[posTTL] = byte(h.TTL)
@ -135,6 +142,8 @@ func ParseHeader(b []byte) (*Header, error) {
 		h.TotalLen += hdrlen
 		h.FragOff = int(*(*uint16)(unsafe.Pointer(&b[posFragOff : posFragOff+1][0])))
 	}
+	h.Flags = HeaderFlags(h.FragOff&0xe000) >> 13
+	h.FragOff = h.FragOff & 0x1fff
 	h.ID = int(b[posID])<<8 | int(b[posID+1])
 	h.TTL = int(b[posTTL])
 	h.Protocol = int(b[posProtocol])
--- a/third_party/code.google.com/p/go.net/ipv4/header_test.go
+++ b/third_party/code.google.com/p/go.net/ipv4/header_test.go
@ -16,28 +16,28 @@ import (
 var (
 	wireHeaderFromKernel = [ipv4.HeaderLen]byte{
 		0x45, 0x01, 0xbe, 0xef,
-		0xca, 0xfe, 0x05, 0xdc,
+		0xca, 0xfe, 0x45, 0xdc,
 		0xff, 0x01, 0xde, 0xad,
 		172, 16, 254, 254,
 		192, 168, 0, 1,
 	}
 	wireHeaderToKernel = [ipv4.HeaderLen]byte{
 		0x45, 0x01, 0xbe, 0xef,
-		0xca, 0xfe, 0x05, 0xdc,
+		0xca, 0xfe, 0x45, 0xdc,
 		0xff, 0x01, 0xde, 0xad,
 		172, 16, 254, 254,
 		192, 168, 0, 1,
 	}
 	wireHeaderFromTradBSDKernel = [ipv4.HeaderLen]byte{
 		0x45, 0x01, 0xdb, 0xbe,
-		0xca, 0xfe, 0xdc, 0x05,
+		0xca, 0xfe, 0xdc, 0x45,
 		0xff, 0x01, 0xde, 0xad,
 		172, 16, 254, 254,
 		192, 168, 0, 1,
 	}
 	wireHeaderToTradBSDKernel = [ipv4.HeaderLen]byte{
 		0x45, 0x01, 0xef, 0xbe,
-		0xca, 0xfe, 0xdc, 0x05,
+		0xca, 0xfe, 0xdc, 0x45,
 		0xff, 0x01, 0xde, 0xad,
 		172, 16, 254, 254,
 		192, 168, 0, 1,
@ -51,6 +51,7 @@ var (
 		TOS:      1,
 		TotalLen: 0xbeef,
 		ID:       0xcafe,
+		Flags:    ipv4.DontFragment,
 		FragOff:  1500,
 		TTL:      255,
 		Protocol: 1,
--- a/third_party/code.google.com/p/go.net/ipv4/iana_test.go
+++ b/third_party/code.google.com/p/go.net/ipv4/iana_test.go
@ -1,9 +1,9 @@
-// go run gentv.go
+// go run gentest.go
 // GENERATED BY THE COMMAND ABOVE; DO NOT EDIT

 package ipv4_test

-// Differentiated Services Field Codepoints, Updated: 2010-05-11
+// Differentiated Services Field Codepoints (DSCP), Updated: 2013-06-25
 const (
 	DiffServCS0        = 0x0  // CS0
 	DiffServCS1        = 0x20 // CS1
--- a/Show More
+++ b/Show More