Compare commits

...

1652 Commits

Author SHA1 Message Date
0b493ac8d4 version: bump to alpha.2 2014-11-07 16:44:02 -08:00
c73d41d98b Merge pull request #1658 from jonboulle/doc_etcdctl_backup
Add doc for backup and force-new-cluster
2014-11-07 16:39:45 -08:00
3d2f65fc0d docs: clarify rewriting 2014-11-07 16:35:33 -08:00
6b283f6ea1 docs: reword failure descriptions 2014-11-07 16:34:19 -08:00
4367c9a1db docs: no need to stop etcd while doing backup 2014-11-07 16:25:38 -08:00
c9894687fc Merge pull request #1662 from yichengq/211
etcdserver: fix data race in cluster
2014-11-07 16:20:59 -08:00
a56fa60fb4 doc: add backup/restore guide 2014-11-07 16:14:45 -08:00
014ef0f52d etcdserver: fix data race in cluster
The data race happens when etcd updates member attributes and fetches
member info in http handler at the same time.
2014-11-07 16:13:07 -08:00
2fc47034ee Merge pull request #1660 from yichengq/209
etcdserver: not add sender when it has existed
2014-11-07 16:10:50 -08:00
46cbfbc630 etcdserver: not add sender when it has existed 2014-11-07 14:05:00 -08:00
d3fd10798b Merge pull request #1656 from jonboulle/1656_sender_garbage
sender logs garbage
2014-11-07 13:44:18 -08:00
a6ba4d357c Merge pull request #1474 from jonboulle/1474_print_peers
print out remote peers' information and config change in the cluster
2014-11-07 13:39:32 -08:00
e707af7c3a Merge pull request #1654 from yichengq/208
integration: use client to do requests
2014-11-07 13:37:14 -08:00
ca06fd0060 etcdserver: log cluster when adding/removing node 2014-11-07 13:36:41 -08:00
958ade86a5 etcdserver: log message after loading peers from snapshot 2014-11-07 13:34:43 -08:00
85a4477f71 integration: use client to do requests 2014-11-07 13:34:30 -08:00
38ec84693f etcdserver: clean up sender error message 2014-11-07 13:32:44 -08:00
78865aa7f7 Merge pull request #1657 from xiangli-cmu/backup
*: add ctl backup support
2014-11-07 13:30:54 -08:00
0d541e6338 *: add ctl backup support 2014-11-07 13:27:44 -08:00
5f6e536be8 Merge pull request #1639 from bcwaldon/etcdctl-tls
Wire up TLS flags for etcdctl
2014-11-07 13:19:36 -08:00
4f85a68c25 Merge pull request #1650 from jonboulle/build_release
scripts: clean build-release script a bit
2014-11-07 12:51:30 -08:00
c3aae88b0c Merge pull request #1653 from jonboulle/server_order
etcdserver: re-order ServerConfig fields
2014-11-07 12:28:29 -08:00
32a82bb423 Merge pull request #1651 from jonboulle/discard
etcdserver: discard log output in tests
2014-11-07 12:05:40 -08:00
285cd404e3 etcdserver: print peerURLs when adding member 2014-11-07 12:00:41 -08:00
a607e097c6 etcdserver: re-order ServerConfig fields 2014-11-07 11:45:59 -08:00
55b4ff0cdf etcdserver: discard log output in tests 2014-11-07 11:45:46 -08:00
82094f05e0 scripts: clean build-release script a bit 2014-11-07 11:45:40 -08:00
c4f273478d Merge pull request #1652 from jonboulle/fix_tests
etcdserver: sort IDs and s/getIDset/getIDs/
2014-11-07 11:43:51 -08:00
14e1442d2d etcdserver: sort IDs and s/getIDset/getIDs/ 2014-11-07 10:57:42 -08:00
810a5146dd Merge pull request #1635 from jonboulle/doc
etcdserver: add docstrings for confchanges
2014-11-07 10:20:05 -08:00
5055863e09 etcdserver: add docstrings for confchanges 2014-11-07 10:19:55 -08:00
bf47fe7cac Merge pull request #1647 from xiangli-cmu/force_cluster
etcdserver: force new cluster
2014-11-07 10:15:53 -08:00
9d19429993 Merge pull request #1609 from yichengq/202
etcdserver: refactor sender
2014-11-07 10:12:02 -08:00
142dfc7d88 etcdserver: add doc for getIDset 2014-11-07 09:00:58 -08:00
0a9c6164af etcdserver: add support for force cluster 2014-11-07 08:49:01 -08:00
376268391b Merge pull request #1646 from jonboulle/1536_disco_proxy
discovery: add command line flag for discovery-proxy
2014-11-07 08:32:23 -08:00
d6f37ec9ad Merge pull request #1648 from jonboulle/delete_member_404
etcdhttp: return 404 when removing nonexistent member
2014-11-06 17:24:17 -08:00
ca1b30db10 etcdhttp: return 404 when removing nonexistent member 2014-11-06 16:59:40 -08:00
9454d30854 etcdserver: add sendHub tests 2014-11-06 16:49:13 -08:00
f75e56932a Merge pull request #1643 from jonboulle/fix_flags
pkg: fix SetFlagsFromEnv behaviour
2014-11-06 16:43:21 -08:00
5604b4c57c flag: split out SetFlagsFromEnvBad test; declare return error 2014-11-06 16:40:13 -08:00
8f1885a398 discovery: add command line flag for discovery-proxy 2014-11-06 16:35:24 -08:00
ccded6644a Merge pull request #1505 from yichengq/193
etcdserver: refactor non-blocking check for sync tests
2014-11-06 15:48:18 -08:00
321d65c4ac pkg: fix SetFlagsFromEnv behaviour
This function was fundamentally buggy, as a panic could be trivially
triggered by setting the wrong environment variable (e.g.
ETCD_BIND_ADDR=foo). Instead, let's propagate the error and present it
to the user in a cleaner way.
2014-11-06 14:39:30 -08:00
c5e6053fcd Merge pull request #1638 from xiangli-cmu/better_logging
etcdserver: better logging for clusterFromPeerURLs
2014-11-06 14:33:53 -08:00
eb0d80767e etcdserver: better logging for clusterFromPeerURLs 2014-11-06 14:28:07 -08:00
6fa031fa69 Merge pull request #1641 from bdarnell/remove-raftlog-reset
raft: remove raftLog.resetUnstable and resetNextEnts
2014-11-06 14:26:08 -08:00
21987c8701 raft: remove raftLog.resetUnstable and resetNextEnts
These methods are no longer used outside of tests and are redundant with
the new stableTo and appliedTo methods.
2014-11-06 17:18:00 -05:00
457b30e585 etcdserver: add/remove sender in sendhub explicitly 2014-11-06 14:04:14 -08:00
2138163c61 etcdserver: code clean on sender struct 2014-11-06 14:04:14 -08:00
211c5e3e29 etcdserver: fix data race in Cluster struct 2014-11-06 14:04:14 -08:00
c3b0de943c etcdserver: discard messages if sender reaches max serving
It is the correct thing to do to ensure that the communication is full
of out-of-date messages.

It results in that integration testing is very easy to throw MsgProp away,
and makes client wait until 5 min timeout. Sync interval and heartbeat are
increased to alleviate the traffic.
2014-11-06 14:04:14 -08:00
1e05cd75c7 etcdserver: refactor sender
1. restrict the number of inflight connections to remote member
2. support stop
2014-11-06 14:04:14 -08:00
2d942e970b etcdctl: add --ca-file, --cert-file, --key-file flags 2014-11-06 12:50:38 -08:00
087e0e8b62 Merge pull request #1636 from xiangli-cmu/client
client: add error handling for addmember
2014-11-06 12:46:15 -08:00
b65dd84e1a Merge pull request #1632 from jonboulle/cs_flag
etcdmain: use StringsFlag for initialclusterstate
2014-11-06 12:36:22 -08:00
66572561bf client: add error handling for addmember 2014-11-06 12:31:24 -08:00
902f06c5c4 pkg/transport: generate TLS client config w/ only CAFile 2014-11-06 12:13:36 -08:00
b53a98eb38 Merge pull request #1631 from xiangli-cmu/validate_doc
Validate doc
2014-11-06 11:45:00 -08:00
a1f5df22ad doc: document conflict case when adding a member 2014-11-06 11:16:49 -08:00
04f6208ace etcdmain: use StringsFlag for initialclusterstate 2014-11-06 11:13:24 -08:00
3cb885c6b2 etcdhttp: return 409 instead of 412 when there is a conflict when adding a member 2014-11-06 11:07:25 -08:00
f4ea274555 etcdctl: centralize getEndpoints logic 2014-11-06 10:54:59 -08:00
4b555dba99 client: add SyncableHTTPClient.Endpoints 2014-11-06 10:54:56 -08:00
9c8f9b3560 Merge pull request #1585 from coreos/clean-up-other-apis-docs
docs: clean up other apis
2014-11-06 10:48:02 -08:00
4ed60471fe Merge pull request #1627 from xiangli-cmu/validate_peer_url
etcdserver: validate peerurl when adding members
2014-11-06 10:43:22 -08:00
7d28d80e5a Merge pull request #1626 from jonboulle/proxy_stuff
discovery: simplify interface
2014-11-06 10:09:16 -08:00
45d7ef99c4 Merge pull request #1629 from asmundg/x-fix-typo
Fix typo in docs
2014-11-06 09:58:54 -08:00
0d8345e0c1 Fix typo in docs
Suggesting that users add two nodes with the same name is probably not a
good idea.
2014-11-06 10:49:40 +01:00
2760739ceb Merge pull request #1625 from yichengq/205
docs: describe the lifetime of discovery url
2014-11-06 00:32:49 -08:00
5d755bd54a docs: describe the lifetime of discovery url 2014-11-06 00:31:19 -08:00
bd2b18b6de etcdserver: validate peerurl when adding members 2014-11-05 23:12:48 -08:00
68bca981de discovery: simplify interface
There's no real need to expose a Discoverer interface/struct when the
only use of the interface (and indeed the module) is to invoke a single
function. This isn't Java, after all. So instead, simplify to Discovery
exposing just two functions: JoinCluster (i.e. what was formerly called
"discovery"), and GetCluster (hitherto "ProxyDiscovery")
2014-11-05 22:45:01 -08:00
6fdbb086f4 Merge pull request #1623 from xiangli-cmu/valid_configuration
Valid configuration
2014-11-05 18:13:04 -08:00
99b1af40c6 etcdserver: move config validation to cluster 2014-11-05 17:55:07 -08:00
99bb479a60 Merge pull request #1618 from yichengq/203
etcdserver: improve panic message in Cluster
2014-11-05 17:14:26 -08:00
98406af448 cluster: separate out membersFromStore from newClusterFromStore 2014-11-05 15:56:43 -08:00
6c9169b4f4 etcdserver: improve panic message in Cluster 2014-11-05 15:39:28 -08:00
3fc6f9c24f Merge pull request #1586 from xiangli-cmu/fix_node
*: add Advance interface to raft.Node
2014-11-05 15:09:51 -08:00
0d7c43d885 *: add a Advance interface to raft.Node
Node set the applied to committed right after it sends out Ready to application. This is not
correct since the application has not actually applied the entries at that point. We add a
Advance interface to Node. Application needs to call Advance to tell raft Node its progress.
Also this change can avoid unnecessary copying when application is still applying entires but
there are more entries to be applied.
2014-11-05 15:04:14 -08:00
c5140d5c18 Merge pull request #1614 from yichengq/194
*: handle panic and fatal organizedly
2014-11-05 14:08:35 -08:00
fdb82718e0 Merge pull request #1612 from jonboulle/proxy
proxy: add docstrings
2014-11-05 13:56:24 -08:00
791b2fd503 *: handle panic and fatal more consistently
1. etcd fatals if there is critical error in the system and operator should
do something for it
2. etcd panics if there happens something unexpected, and it should be
reported to us to debug.
2014-11-05 13:53:24 -08:00
3c3cae57c6 Merge pull request #1616 from jonboulle/philips-add-error-to-test
test: add error package
2014-11-05 13:40:58 -08:00
bdd2a0a018 test: add error package 2014-11-05 13:31:42 -08:00
c6104c1e2a Merge pull request #1613 from jonboulle/proxy_clean
etcdmain: simplify proxy start logic
2014-11-05 11:41:15 -08:00
b85496922f etcdmain: simplify proxy start logic 2014-11-05 11:41:03 -08:00
89eac70d09 proxy: add docstrings 2014-11-05 10:30:05 -08:00
58b171b3e5 Merge pull request #1610 from jonboulle/discovery_docs
discovery: add clarifying docstrings
2014-11-04 19:38:59 -08:00
bb84aaebaf discovery: add clarifying docstrings 2014-11-04 17:02:33 -08:00
ab00d23cd3 Merge pull request #1608 from jonboulle/flags
pkg: move to more generic StringsFlag
2014-11-04 16:53:48 -08:00
5de9d38cc6 pkg: move to more generic StringsFlag 2014-11-04 16:52:56 -08:00
d36f09d643 Merge pull request #1602 from jonboulle/bump_timeout
integration: bump timeout for good path
2014-11-04 16:52:44 -08:00
f71c247d87 Merge pull request #1604 from xiangli-cmu/fallback_proxy
*: support discovery fallback
2014-11-04 16:41:28 -08:00
71acd0c3d0 discovery: consolidate proxyDiscover and Discover interface 2014-11-04 16:38:05 -08:00
288624550e Merge pull request #1581 from jonboulle/log_changes
No logs when members added/removed from cluster
2014-11-04 15:13:12 -08:00
e4d0c25365 etcdserver: log adding and removing nodes 2014-11-04 15:05:15 -08:00
c628d7f412 Merge pull request #1601 from jonboulle/client
client: return ErrNoEndpoint when none available
2014-11-04 14:58:22 -08:00
5cb13fd071 *: support discovery fallback 2014-11-04 14:30:22 -08:00
9e001dee29 Merge pull request #1603 from jonboulle/typo
etcdhttp: fix typo in test comment
2014-11-04 13:18:12 -08:00
4d40816a90 etcdserver: refactor non-blocking check for sync tests
to make it much more reliable and avoid false errors.
2014-11-04 13:07:44 -08:00
0f7add9722 etcdhttp: fix typo in test comment 2014-11-04 12:57:59 -08:00
9f29545f66 integration: bump timeout for good path
When waiting for a watch result, we expect the good path to complete
quickly here so we don't need to time out so aggressively. (Failure
noted in #1600)
2014-11-04 12:55:40 -08:00
45b7c9a4ac client: return ErrNoEndpoint when none available
In certain cases (for example, if a cluster peer is accessible but it
has no members listed), the httpClusterClient could have an empty set of
endpoints as a result of the Sync. This means that its Do function could
potentially return a nil response and nil error, with catastrophic
consequences for callers.

To be safe (particularly about this latter behaviour), this change
errors in both Sync and Do if no endpoints are available.
2014-11-04 12:51:43 -08:00
34dabe281b Merge pull request #1591 from philips/application-json-errors
error: use application/json as the content-type
2014-11-04 12:31:06 -08:00
5fbef59dbc error: use application/json as the content-type
Fixes #1584
2014-11-04 12:08:18 -08:00
915f8f4822 Merge pull request #1531 from jonboulle/410_gone
return 410 Gone for member that has been removed in /v2/members -XDELETE
2014-11-04 11:54:01 -08:00
cedcc0d8df etchttp: return 410 gone for permanently removed members 2014-11-04 11:21:24 -08:00
ac49e1d50f Merge pull request #1594 from unihorn/201
etcdhttp/etcdserver: support HEAD on /v2/keys/ namespace
2014-11-04 00:11:47 -08:00
866ec5948c etcdhttp/etcdserver: support HEAD on /v2/keys/ namespace 2014-11-04 00:06:49 -08:00
aa5711bd0f Merge pull request #1595 from jonboulle/header
*: add copyright header to remaining files
2014-11-03 23:42:14 -08:00
f7434b55e5 *: add copyright header to remaining files 2014-11-03 23:29:15 -08:00
2235b47030 Merge pull request #1545 from unihorn/197
etcdhttp: always respond json-format error to client
2014-11-03 23:25:14 -08:00
5ead800ff5 Merge pull request #1572 from xiangli-cmu/raft_test
raft: add paper tests for section 5.4.1
2014-11-03 22:37:26 -08:00
e4b12a8e28 Merge pull request #1593 from unihorn/200
etcdserver: print out initial cluster members
2014-11-03 22:23:40 -08:00
9aefb91531 etcdhttp: always respond json-format error to client 2014-11-03 22:19:17 -08:00
5ed5d44652 etcdserver: print out initial cluster members
It is moved from etcdmain pkg because the line should only be printed out
when etcd bootstraps at the first time.
2014-11-03 19:34:24 -08:00
cc0ef16346 docs: clean up other apis
The docs for the other APIs use curl for example usage, which matches
the docs for the etcd APIs.

Other cleanup include fixing usage of peer ports and using 10.0.0.x IPs
throughout.
2014-11-03 17:14:08 -08:00
a272f5d7e3 Merge pull request #1592 from jonboulle/integration_tests
integration: add keys API integration tests
2014-11-03 16:31:59 -08:00
63cf0b9d90 integration: add keys API integration tests 2014-11-03 16:30:29 -08:00
ab69c2adbd etcdhttp: use EcodePrevValueRequired when appropriate 2014-11-03 16:12:50 -08:00
075ab6415f Merge pull request #1587 from xiangli-cmu/fix_wal
wal: sync before returning from create
2014-11-03 15:58:47 -08:00
dd09042632 etcdserver: try to listen on ports before initializing etcd server 2014-11-03 15:55:58 -08:00
165ac654e8 raft: add paper tests for section 5.4.1 2014-11-03 15:50:56 -08:00
dbdeceda7b raft: do not load empty state and ents 2014-11-03 15:16:41 -08:00
ff1f5a9d57 wal: sync before returning from create 2014-11-03 14:28:59 -08:00
d1ec13210f Merge pull request #1571 from bcwaldon/client-redirects
client: follow redirects
2014-11-03 14:26:20 -08:00
2ba02c04be Merge pull request #1576 from coreos/print-initial-cluster-members
etcd: print initial cluster members during startup
2014-11-03 14:24:33 -08:00
6dd4944e62 client: follow redirects 2014-11-03 12:15:16 -08:00
5da481213e Merge pull request #1478 from unihorn/190
etcdserver: panic on storage error
2014-11-03 11:07:55 -08:00
433b4138c5 etcdserver: panic on storage error
It is a critical error to etcd, and etcd is not able to recover it now.
2014-11-03 10:46:04 -08:00
729770f32a Merge pull request #1570 from bcwaldon/client-endpoints
client: use all endpoints
2014-11-03 10:44:09 -08:00
3ec4da6ac6 etcd: print initial cluster members during startup
etcd now prints the initial clusters members during startup.

```
2014/11/03 10:32:46 etcd: initial cluster members: etcd0=http://127.0.0.1:2380,etcd1=http://127.0.0.1:2390,etcd2=http://127.0.0.1:2400
```
2014-11-03 10:38:18 -08:00
9df06bfa94 Merge pull request #1579 from coreos/cleanup-clustering-doc
docs: clean up clustering doc
2014-11-03 10:25:04 -08:00
20df86e3c3 docs: clean up clustering doc 2014-11-03 09:51:50 -08:00
6433be5738 Merge pull request #1575 from coreos/improve-admin-docs
docs: fix usage of peers urls
2014-11-02 22:56:23 -08:00
3068340a83 docs: fix usage of peers urls 2014-11-02 22:00:41 -08:00
da6827f09e client: use all endpoints 2014-10-31 20:51:47 -07:00
75104c10d4 Merge pull request #1553 from bcwaldon/client-sync
Support syncing and `--no-sync` flag in `etcdctl member` commands
2014-10-31 20:51:01 -07:00
58af26736c client: further clarify external interfaces 2014-10-31 20:45:55 -07:00
17c6f21d68 client: elevate context to caller of KeysAPI 2014-10-31 17:27:43 -07:00
f0760d6246 client: elevate context to caller of MembersAPI 2014-10-31 17:27:42 -07:00
913d102a81 client: remove unused field 2014-10-31 17:25:05 -07:00
824049897d client: export necessary interfaces/methods 2014-10-31 17:25:05 -07:00
b47631b38f etcdctl: respect --no-sync in member subcommands 2014-10-31 17:25:05 -07:00
22b86684f0 etcdctl: sync before running member subcommands 2014-10-31 17:25:05 -07:00
5ed5d018be client: add httpClusterClient.Sync 2014-10-31 17:25:05 -07:00
f6e8b677cf client: pass httpActionDo into NewMembersAPI 2014-10-31 17:25:05 -07:00
0ef270c25c client: pass httpActionDo into New[Discovery]KeysAPI 2014-10-31 17:25:05 -07:00
1130273178 client: s/newHTTPClusterClient/NewHTTPClient/ 2014-10-31 17:25:05 -07:00
3eb126af4d client: use httpActionDo in httpClusterClient 2014-10-31 17:25:05 -07:00
c282664c23 client: s/transport/CancelableTransport/
CancelableTransport is implemented by callers of the
client pkg, so we should export it so it is
documented publicly.
2014-10-31 17:25:04 -07:00
d52d836761 client: return full http.Response in httpActionDo 2014-10-31 17:25:04 -07:00
5bdf6a4110 Merge pull request #1528 from unihorn/191
raft: add tests based on section 5.3 in raft paper
2014-10-31 16:35:36 -07:00
421d5fbe72 raft: add tests based on section 5.3 in raft paper 2014-10-31 16:32:34 -07:00
f35130a0ed etcdctl: clean up formatting of member add 2014-10-31 15:37:08 -07:00
500e9e2212 version: bump to v0.5.0-alpha.1 2014-10-31 15:22:13 -07:00
7c52a86325 Merge pull request #1569 from philips/dynamic-configuration-docs
Documentation: add the runtime configuration document
2014-10-31 15:19:42 -07:00
124dd7096a Documentation: add the runtime configuration document 2014-10-31 15:16:45 -07:00
388b4aeb71 Merge pull request #1568 from philips/dynamic-configuration
etcdctl: two member fixes
2014-10-31 15:08:40 -07:00
6b4485d1ae etcdctl: take a name and print out the initial cluster
To help the user lets print out the configuration that they will need to give
to their new member:

$ etcdctl member add infra4 http://localhost:7004
added member 9bf1b35fc7761a23 to cluster
ETCD_NAME="infra4"
ETCD_INITIAL_CLUSTER="node2=http://localhost:7002,node3=http://localhost:7003,infra4=http://localhost:7004,node1=http://localhost:7001"

This is a little weird because the API doesn't take a name so the user gives us
a name and we just pass it on through.
2014-10-31 15:05:23 -07:00
74886713db Merge pull request #1567 from unihorn/199
*: name node{1,2,3} -> infra{1,2,3}
2014-10-31 14:53:34 -07:00
8f3be206ed etcdctl: add help on the members subcommands 2014-10-31 14:50:23 -07:00
1db23109ad *: name node{1,2,3} -> infra{1,2,3}
Be consistent with the naming in documentations.
2014-10-31 14:46:39 -07:00
749097429f Merge pull request #1565 from jonboulle/int
integration: clean up licenses and docs
2014-10-31 14:34:05 -07:00
34b2fecd28 integration: clean up licenses and docs 2014-10-31 14:33:56 -07:00
faede90293 Merge pull request #1556 from philips/fixup-the-admin-guide
Documentation: fixup the admin_guide
2014-10-31 14:11:39 -07:00
b6cc34b52e Documentation: fixup the admin_guide
- Provide more concrete examples and explanation
- Cleanup the formatting to one sentence per line, this makes reviewing
  easier
- Point to existing docs on wal and snap instead of trying to duplicate
  it here again.
2014-10-31 14:11:27 -07:00
308b8796e4 Merge pull request #1557 from bcwaldon/id-logging
etcdserver: fix logging of IDs
2014-10-31 12:27:37 -07:00
6e038e02a6 etcdserver: fix logging of IDs 2014-10-31 12:26:53 -07:00
38250d3fac Merge pull request #1541 from bcwaldon/client-peers
Use `-peers` in `etcdctl members` commands
2014-10-31 12:26:15 -07:00
eab4692744 client: use v2MembersURL helper 2014-10-31 12:21:15 -07:00
f0c3385cfc etcdctl: wire up --peers for member commands 2014-10-31 12:21:15 -07:00
8b8b3efdaa client: accept slice of endpoints 2014-10-31 12:21:15 -07:00
8d519ffdb8 client: introduce httpClusterClient 2014-10-31 12:21:15 -07:00
323fb1ec85 client: introduce httpActionDo interface 2014-10-31 12:21:15 -07:00
9d07db4432 client: move timeout into caller of httpClient 2014-10-31 12:21:15 -07:00
7c1f4a9baf client: explicitly carry API prefix around 2014-10-31 12:21:15 -07:00
dee912f2fd etcdctl: break out mustNewMembersAPI 2014-10-31 12:21:08 -07:00
bc62b05c7f etcdctl: break out getPeersFlagValue 2014-10-31 12:21:00 -07:00
48ec876af9 godep: bump github.com/codegangsta/cli 2014-10-31 12:21:00 -07:00
a576dbca43 Merge pull request #1554 from xiangli-cmu/removed_logging
etcdserver: better logging when member is removed
2014-10-31 12:07:57 -07:00
eb472b7745 etcdserver: better logging when member is removed 2014-10-31 12:00:50 -07:00
a535161a84 Merge pull request #1552 from philips/fixup-wal-doc
wal: update the docs to show the optional metadata field
2014-10-31 11:32:26 -07:00
513c72ec8b wal: update the docs to show the optional metadata field 2014-10-31 11:32:17 -07:00
e02ef6b141 Merge pull request #1546 from unihorn/198
etcdserver: better logging for assign ids from upstream
2014-10-31 11:13:43 -07:00
2c5f062b7f etcdserver: better logging for assign ids from upstream 2014-10-31 11:06:31 -07:00
1bb07115f2 Merge pull request #1550 from jonboulle/bump_timeout
etcdhttp: bump default Server timeout to 5 mins
2014-10-31 11:05:44 -07:00
9726d3909c etcdhttp: bump default Server timeout to 5 mins 2014-10-31 10:52:46 -07:00
c53e58e97c Merge pull request #1309 from jonboulle/1309_standard_id
standardize ID serialization
2014-10-31 10:50:32 -07:00
55c92ad456 *: create ID type
This creates a simple ID type (wrapped around uint64) to provide for
standard serialization/deserialization to a string (i.e. base 16
encoded). This replaces strutil so now that package is removed.
2014-10-31 10:34:07 -07:00
781abc1db0 Merge pull request #1539 from unihorn/195
*: clean log.Print
2014-10-30 18:18:35 -07:00
aa50af1c69 *: clean log.Print
1. only log things by default that the operator of etcd may need to react to
2. put package name at the head of log lines
2014-10-30 18:15:53 -07:00
7f29045c0f Merge pull request #1543 from xiangli-cmu/fix_logging
etcdserver: fix sender logging
2014-10-30 18:13:16 -07:00
0f8b035253 etcdserver: fix sender logging 2014-10-30 18:00:00 -07:00
42a7c928d4 Merge pull request #1542 from xiangli-cmu/fix_logging
etcdhttp: fix logging in raft handler
2014-10-30 17:41:47 -07:00
02ff59514f etcdhttp: fix logging in raft handler 2014-10-30 17:39:01 -07:00
9a56001d63 Merge pull request #1537 from xiangli-cmu/cluster-token
Cluster token
2014-10-30 17:09:25 -07:00
8e633db5cb doc: add doc for initial-cluster-token 2014-10-30 17:08:15 -07:00
64a12e9341 Merge pull request #1511 from coreos/set-watch-consistency-to-strong
etcdctl: Set watch consistency to STRONG
2014-10-30 16:52:50 -07:00
ac71ad92af Merge pull request #1452 from unihorn/187
etcdserver: exit program when node is removed
2014-10-30 15:32:26 -07:00
ed30b6deca etcdserver: exit program when node is removed
Originally added in 400dd2d7bc,
and removed by mistake when refactor cluster.
2014-10-30 15:31:58 -07:00
76298ebcd8 Merge pull request #1534 from bcwaldon/client-tests
client: test assertStatusCode
2014-10-30 13:50:21 -07:00
d36a3e18d2 etcdctl: remove SetConsistency call
This call to SetConsistency is explicitly setting the default
value, so it's really unnecessary.
2014-10-30 13:45:59 -07:00
3dfb6723b2 *: rename initial-cluster-name to initial-cluster-token 2014-10-30 13:43:38 -07:00
6087e2b2f6 Merge pull request #1509 from bcwaldon/etcdctl-102
etcdctl: add --sort flag to ls command
2014-10-30 13:33:37 -07:00
6e8de1f426 Merge pull request #1508 from bcwaldon/etcdctl-96
etcdctl: add -p to ls command
2014-10-30 13:33:30 -07:00
052521eaf1 client: test assertStatusCode 2014-10-30 11:46:44 -07:00
549c643bfe Merge pull request #1530 from xiangli-cmu/discovery_doc
doc: add a Custom etcd discovery service section
2014-10-30 11:40:23 -07:00
af7d73717c doc: add a Custom etcd discovery service section 2014-10-30 11:34:46 -07:00
816c173edf Merge pull request #1526 from xiangli-cmu/leader_log
raft: better logging for leader transition
2014-10-30 10:13:58 -07:00
9359a57211 Merge pull request #1523 from jonboulle/raft_stuff
raft: minor cleanup in comments
2014-10-30 10:01:59 -07:00
b99633207c raft: minor cleanup in comments 2014-10-30 10:01:44 -07:00
4f6206bf65 Merge pull request #1522 from jonboulle/raft_tests
raft: add tests for progress.maybeDecr
2014-10-30 10:00:01 -07:00
bf44219766 Merge pull request #1492 from jonboulle/1492_nonexistent_member
Attempting to remove nonexistent member mishandled
2014-10-30 09:58:02 -07:00
19881b2f15 etcdhttp: return 404 when removing nonexistent member 2014-10-30 09:57:54 -07:00
46ebf69c02 raft: better logging for leader transition 2014-10-30 09:33:38 -07:00
0cf0cb3d02 raft: add tests for progress.maybeDecr 2014-10-29 22:57:25 -07:00
83ca16188c Merge pull request #1417 from jonboulle/master
RFC: move main logic to etcd subpackage
2014-10-29 18:47:42 -07:00
cf9dd31daa etcd: move main logic to etcdmain subpackage 2014-10-29 18:43:22 -07:00
38617f5c9b Merge pull request #1510 from xiangli-cmu/fix_discovery
discovery: fix discovery for not working on customized discovery service
2014-10-29 18:33:06 -07:00
027e944985 discovery: fix discovery for not working on customized discovery service 2014-10-29 18:30:59 -07:00
2be3f870cc etcdctl: Set watch consistency to STRONG 2014-10-29 18:11:05 -07:00
ba38847bdd Merge pull request #1507 from bcwaldon/doc-fix
doc: link directly to members API
2014-10-29 18:01:27 -07:00
97597eca03 etcdctl: add --sort flag to ls command
This is a port of coreos/etcdctl#102
2014-10-29 17:45:11 -07:00
243886edc8 etcdctl: add -p to ls command
This is a port of coreos/etcdctl#96
2014-10-29 17:42:21 -07:00
f61824ce01 Merge pull request #1500 from bcwaldon/client-tests
Add tests for client pkg
2014-10-29 17:26:26 -07:00
ac810b86bc doc: link directly to members API 2014-10-29 17:21:10 -07:00
e85ba2f384 Merge pull request #1504 from xiangli-cmu/admin_guide
doc: add a doc for data directory
2014-10-29 17:16:27 -07:00
f5c1da6967 doc: add a doc for data directory 2014-10-29 17:07:21 -07:00
0f51cbde6c Merge pull request #1502 from unihorn/192
store: copy Nodes correctly in NodeExtern.Clone
2014-10-29 16:57:04 -07:00
a910d8ba9f store: copy Nodes correctly in NodeExtern.Clone 2014-10-29 16:54:09 -07:00
d756dd2079 client: test membersAPIActionRemove 2014-10-29 16:42:16 -07:00
5264c05ddb client: clean up style of TestMembersAPIActionList 2014-10-29 16:42:16 -07:00
4e759b46ce client: use httptypes.MemberCreateRequest in member add 2014-10-29 16:42:14 -07:00
011a67c878 httptypes: add MemberCreateRequest.MarshalJSON 2014-10-29 16:37:07 -07:00
e457d52f5c client: log incorrect HTTP resp body as string 2014-10-29 16:37:07 -07:00
ccca32b138 Merge pull request #1497 from unihorn/189
raft: add tests based on section 5.1 in raft paper
2014-10-29 16:35:25 -07:00
dabb5c150d raft: add tests based on section 5.1 in raft paper 2014-10-29 16:22:17 -07:00
b7b3bf40e0 Merge pull request #1501 from bcwaldon/rename-members-tests
etcdhttp: s/TestServeAdminMembers*/TestServeMembers*/
2014-10-29 16:21:16 -07:00
2c0f6e4bf9 etcdhttp: s/TestServeAdminMembers*/TestServeMembers*/ 2014-10-29 16:18:03 -07:00
3f6e584702 Merge pull request #1499 from bcwaldon/client_clean
client: pass around statuscode instead of Response
2014-10-29 15:51:58 -07:00
97c23c4333 client: pass around statuscode instead of Response
There's no real need for do and doWithTimeout to return Responses when
the only field of interest is the status code.

This also removes the superfluous httpMembersAPIResponse struct.
2014-10-29 15:47:55 -07:00
95231c1278 Merge pull request #1493 from bcwaldon/etcdctl-members
etcdctl members [list|add|remove]
2014-10-29 15:44:30 -07:00
f810dda9b2 Merge pull request #1481 from bcwaldon/no-more-admin-api
Remove "admin" from /v2/admin/members
2014-10-29 15:43:37 -07:00
f6e242aa01 etcdctl: impl members commands 2014-10-29 15:09:23 -07:00
8b12e1aa37 client: fill in MembersAPI 2014-10-29 15:03:22 -07:00
b59961228b Merge pull request #1495 from xiangli-cmu/fix_raft_test
raft: fix a incorrect in testMaybeAppend
2014-10-29 15:03:13 -07:00
738da2b3fa raft: fix a incorrect in testMaybeAppend 2014-10-29 14:57:39 -07:00
de0cf2fb8e Merge pull request #1459 from bcwaldon/member-list
etcdhttp: encode MembersCollection properly
2014-10-29 14:49:22 -07:00
4b1431109e Merge pull request #1490 from jonboulle/1490
Logging member ID as int
2014-10-29 14:42:02 -07:00
6375bd7960 Merge pull request #1469 from xiangli-cmu/raft_log_test
raft: add tests for maybeappend
2014-10-29 14:36:07 -07:00
e99da41539 etcdserver: log member ID as hex string 2014-10-29 14:36:05 -07:00
14f4163e41 raft: add several test cases for testMaybeAppend 2014-10-29 14:35:13 -07:00
5bba81f5fc Merge pull request #1472 from jonboulle/clone_event
waitIndex is not working
2014-10-29 14:23:32 -07:00
a59e8cf1a6 Merge pull request #1487 from jonboulle/peerurls
etcdserver: make peer URLs log message more readable
2014-10-29 14:23:25 -07:00
9546df9a6c etcdserver: make peer URLs log message more readable 2014-10-29 14:18:51 -07:00
f7631be453 Merge pull request #1485 from jonboulle/raft_clean
raft: move test helper function into tests
2014-10-29 14:06:39 -07:00
81304b2b7e raft: move test helper function into tests 2014-10-29 14:04:45 -07:00
8298e06627 doc: remove trailing slashes 2014-10-29 12:16:02 -07:00
ab67fa4cc6 api: remove admin prefix from members API 2014-10-29 12:12:51 -07:00
bab19e3b0b doc: move admin_api.md to other_apis.md 2014-10-29 12:12:51 -07:00
d3bafd6aa4 etcdhttp: encode MembersCollection properly 2014-10-29 12:06:22 -07:00
84be7c1e9e etcdserver/store: clone Events before modifying 2014-10-29 11:54:35 -07:00
ad1718a3e5 raft: add a test case for testLogMaybeAppend 2014-10-29 11:44:18 -07:00
35bba87d2a Merge pull request #1471 from unihorn/189
raft: add tests based on section 5.2 in raft paper
2014-10-29 10:49:23 -07:00
bffe611fe6 raft: add tests based on section 5.2 in raft paper 2014-10-29 10:17:09 -07:00
6bcfa2b05d Merge pull request #1475 from bketelsen/patch-1
Add Skydns, crypt, viper
2014-10-29 08:55:09 -07:00
3857e92cad Add Skydns, crypt, viper
Added a plethora of tools we've written or contributed to that use etcd.
2014-10-29 09:09:31 -04:00
04e56a454e Merge pull request #1465 from bcwaldon/member-post
Clean up POST /v2/admin/members
2014-10-28 17:08:30 -07:00
658a84312b Merge pull request #1464 from bcwaldon/error-ctype
Set Content-Type before calling WriteHeader
2014-10-28 15:48:23 -07:00
ae7280dcf3 Merge pull request #1466 from jonboulle/errors
main: catch a few unhandled errors
2014-10-28 15:43:57 -07:00
c6873c1eab raft: add tests for maybeappend 2014-10-28 15:07:49 -07:00
a96f5ab146 main: catch a few unhandled errors
If any of this initialization fails, something very bad has happened,
and we should not continue as-is (this has previously manifested in
strange bugs)
2014-10-28 11:18:22 -07:00
6f851ac885 httptypes: set headers before call to WriteHeader 2014-10-28 11:08:10 -07:00
2b4201c53d httptypes: test HTTPError 2014-10-28 10:59:53 -07:00
57d447fef6 Merge pull request #1463 from bcwaldon/writeError-logging
etcdhttp: only log when error deserves it
2014-10-28 10:45:55 -07:00
c07b9ae32e etcdhttp: 415 when content-type not JSON 2014-10-28 10:38:09 -07:00
8fbf887e52 etcdhttp: only log when error deserves it 2014-10-28 10:30:05 -07:00
d1fb732e63 etcdhttp: properly serialize Member on POST 2014-10-28 10:21:11 -07:00
8b0eaa9e15 etcdhttp: separate member create deserialization 2014-10-28 10:09:05 -07:00
ad0664da9c doc: fix documentation of POST /v2/admin/members 2014-10-28 09:44:59 -07:00
b6b5081254 Merge pull request #1450 from bcwaldon/raft-logging
raft: stop logging IDs with 0x prefix
2014-10-28 08:17:11 -07:00
6796669484 raft: stop logging IDs with 0x prefix 2014-10-27 18:56:13 -07:00
87327a245d Merge pull request #1448 from bcwaldon/member-URLs
fix usage of copy in newMemberCollection
2014-10-27 18:43:33 -07:00
e08c2bbe3e Merge pull request #1449 from bcwaldon/id-to-hex
Move ID*Hex functions to pkg/strutil
2014-10-27 18:43:00 -07:00
8d052dd374 etcdhttp: copy Member URLs properly 2014-10-27 18:39:33 -07:00
480e92d340 strutil: move IDAsHex/IDFromHex to new pkg 2014-10-27 18:39:09 -07:00
dad7500d13 Merge pull request #1443 from bcwaldon/member-ID
Serialize member ID in API
2014-10-27 18:38:53 -07:00
d55546d62e Merge pull request #1446 from philips/add-quay-badge
README: add the quay badge to the official git builds
2014-10-27 17:40:50 -07:00
acd8eecd4e README: add the quay badge to the official git builds 2014-10-27 17:39:16 -07:00
2d31e5ab56 Merge pull request #1441 from philips/add-root-dockerfile
Dockerfile: initial commit
2014-10-27 17:27:57 -07:00
2472953939 etcdhttp: hex-encode member ID 2014-10-27 17:25:22 -07:00
80172c3d4a etcdserver: s/parseMemberID/mustParseMemberIDFromKey/ 2014-10-27 17:25:00 -07:00
b316c6b002 Merge pull request #1435 from xiangli-cmu/json
etcdhttp: make admin HTTP endpoint return json format error
2014-10-27 17:07:58 -07:00
6cb45236ac etcdhttp: make admin HTTP endpoint return json format error 2014-10-27 17:03:58 -07:00
04b5853261 Merge pull request #1439 from kelseyhightower/check-data-dir-permissions
etcd: ensure data dir is writable
2014-10-27 16:57:19 -07:00
b1731f0843 etcd: ensure data dir is writable
etcd checks that the data dir is writable by writing and removing an
empty file to the data dir during startup and exits non-zero if that
fails.

fixes #876
2014-10-27 16:52:05 -07:00
36cacb8bd8 Merge pull request #1429 from coreos/1392
Fix #1392
2014-10-27 16:50:46 -07:00
e849d8e157 etcdhttp: DELETE on members = MethodNotAllowed 2014-10-27 16:49:04 -07:00
387639e802 etcdserver/etcdhttp: treat /v2/admin/members and /v2/admin/members/ equally 2014-10-27 16:49:03 -07:00
3e234918ee Dockerfile: initial commit 2014-10-27 16:43:27 -07:00
0ce78d7a9c Merge pull request #1431 from philips/build-release-script
scripts: import script from 0.5 release
2014-10-27 16:42:30 -07:00
52350d1d2f Merge pull request #1433 from philips/add-build-docker-script
scripts: build-docker
2014-10-27 16:42:11 -07:00
7384ee39a6 Merge pull request #1436 from xiangli-cmu/writeTo
error: write->writeTo
2014-10-27 15:34:17 -07:00
d0604c7d5c error: write->writeTo 2014-10-27 15:32:36 -07:00
74c257f63d Merge pull request #1419 from xiangli-cmu/raft_log_test
raft: add test for findConflict
2014-10-27 14:30:36 -07:00
460d6490ba raft: address issues in comments 2014-10-27 14:20:42 -07:00
60cb18b6c2 Merge pull request #1432 from unihorn/187
raft: use raft-specific rand.Rand instead of global one
2014-10-27 14:15:58 -07:00
e8302c8413 scripts: build-docker
Build docker images from the release tarballs. The default CMD is
etcd.
2014-10-27 12:34:51 -07:00
b986a52579 raft: use raft-specific rand.Rand instead of global one 2014-10-27 12:32:11 -07:00
538ce935f0 scripts: import script from 0.5 release
It isn't pretty but this was the tool used to build the zip and tar
files for 0.5
2014-10-27 12:18:24 -07:00
94e4595af5 Merge pull request #1427 from bcwaldon/members-serialization
Centralize Members serialization
2014-10-27 11:33:39 -07:00
753bc5e166 httptypes: add doc.go 2014-10-27 11:22:47 -07:00
80ca168cbe client: simplify MembersAPI response parsing 2014-10-27 11:22:47 -07:00
14795d8ed9 httptypes: use MemberCollection for JSON (de)serialization 2014-10-27 11:22:47 -07:00
7545152318 httptypes: use []string for Member URLs 2014-10-27 11:22:47 -07:00
54a2d8ffc9 client: move Member models to new types pkg 2014-10-27 11:22:46 -07:00
ee27846d5b Merge pull request #1422 from unihorn/187
etcdserver: parse context error for better message
2014-10-27 11:00:01 -07:00
e77f8e311c etcdserver: parse context error for better message 2014-10-27 10:59:15 -07:00
585881a870 Merge pull request #1428 from jonboulle/subpackages
pkg: move everything into subpackages
2014-10-27 10:32:11 -07:00
9964bfa6b9 Merge pull request #1426 from xiangli-cmu/clusterid
etcdhttp: attach clusterID to key and adminMember endpoint
2014-10-27 10:16:16 -07:00
6e6d1897d8 pkg: move everything into subpackages 2014-10-27 09:57:28 -07:00
328d8f2d26 Merge pull request #1424 from bcwaldon/pkg-readme
pkg: add README.md
2014-10-27 09:13:26 -07:00
6f792354ca etcdhttp: attach clusterID to key and adminMember endpoint 2014-10-27 07:52:39 -07:00
40048d7300 Merge pull request #1420 from xiangli-cmu/clean_log
raft: remove unused code
2014-10-26 20:14:05 -07:00
000962d689 Merge pull request #1421 from xiangli-cmu/logging
*: better logging
2014-10-26 20:13:55 -07:00
444e6e952b pkg: add README.md 2014-10-26 16:28:48 -07:00
f9af07eb5b Merge pull request #1423 from bcwaldon/http-refactor-hdlrs
eradicate serverHandler
2014-10-26 14:44:30 -07:00
b06499d0c2 etcdserver/etcdhttp: break apart HTTP handlers 2014-10-26 13:20:53 -07:00
4b77082b6e Merge pull request #1415 from bcwaldon/http-refactor
etcdserver/etcdhttp: break apart http.go
2014-10-26 12:39:22 -07:00
009b737cef *: better logging 2014-10-26 08:13:03 -07:00
94f701cf95 raft: refactor isUpToDate and add a test 2014-10-25 20:34:14 -07:00
8cd95e916d raft: comments for isUpToDate 2014-10-25 20:12:54 -07:00
86c66cd802 raft: remove unused code 2014-10-25 19:56:13 -07:00
90f26e4a56 raft: add test for findConflict 2014-10-25 18:58:11 -07:00
73215447c1 Merge pull request #1414 from bcwaldon/client-members-API
Add MembersAPI w/ List method
2014-10-25 11:33:42 -07:00
cba19e348f client: MembersAPI.List 2014-10-25 11:30:15 -07:00
435611cf0d etcdserver/etcdhttp: break apart http.go 2014-10-25 11:28:52 -07:00
00dcbf8bf7 client: unexport HTTPKeysAPI 2014-10-25 08:58:25 -07:00
73e48068c2 client: add prefix to KeysAPI 2014-10-25 08:58:25 -07:00
2b9cabcbcd version: bump to v0.5.0-alpha.0 2014-10-24 17:25:49 -07:00
2baf3e0d79 Merge pull request #1407 from bcwaldon/old-data-dir
etcd: use old default data-dir format
2014-10-24 16:44:30 -07:00
677d9d1bea etcd: use old default data-dir format 2014-10-24 16:41:42 -07:00
91a4aa151a Merge pull request #1406 from bcwaldon/doc-clustering
doc: PUT instead of POST discovery config
2014-10-24 16:36:54 -07:00
b62fdac193 doc: PUT instead of POST discovery config 2014-10-24 16:30:48 -07:00
e630910e31 Merge pull request #1398 from jonboulle/proxy
proxy not working on master
2014-10-24 15:55:31 -07:00
338f59db74 Merge pull request #1401 from unihorn/186
etcdhttp: parse quorum field from http request
2014-10-24 15:54:40 -07:00
719c57a29d proxy: retrieve ClientURLs from cluster
This is a simple solution to having the proxy keep up to date with the
state of the cluster. Basically, it uses the cluster configuration
provided at start up (i.e. with `-initial-cluster-state`) to determine
where to reach peer(s) in the cluster, and then it will periodically hit
the `/members` endpoint of those peer(s) (using the same mechanism that
`-cluster-state=existing` does to initialise) to update the set of valid
client URLs to proxy to.

This does not address discovery (#1376), and it would probably be better
to update the set of proxyURLs dynamically whenever we fetch the new
state of the cluster; but it needs a bit more thinking to have this done
in a clean way with the proxy interface.

Example in Procfile works again.
2014-10-24 15:54:12 -07:00
0e1d1646fd etcdhttp: parse quorum field from http request 2014-10-24 15:44:39 -07:00
0fcb59e7d9 Merge pull request #1393 from philips/0.5-docs2
Documentation/0.5: add api.md
2014-10-24 15:18:59 -07:00
5456ef7049 Documentation/0.5: add api.md
- Update all of the ports to use the new IANA port numbers
- Update the stats section to talk about the id fields
- Remove mention of the modules
- Remove -L from all of the curl commands since it is no longer needed
- Point people at the clustering.md guide for the cluster APIs
2014-10-24 15:18:47 -07:00
cb59a46576 Merge pull request #1396 from unihorn/185
etcdhttp: trim StoreKeysPrefix from error in serveKeys
2014-10-24 14:48:31 -07:00
46528ee17b Merge pull request #1391 from jonboulle/version
/version is 404 now
2014-10-24 14:48:02 -07:00
9a465b9cf5 etcdhttp: add /version endpoint 2014-10-24 14:47:37 -07:00
dc46d40b8e Merge pull request #1397 from jonboulle/log_raft_msg
etcdserver/etcdhttp: remove log message for every received raft
2014-10-24 14:29:38 -07:00
bac13b5cb2 etcdserver/etcdhttp: remove log message for every received raft 2014-10-24 14:25:20 -07:00
34dcbb4679 etcdhttp: trim StoreKeysPrefix from error in serveKeys
It returns error messaage like this now:
'{"errorCode":100,"message":"Key not found","cause":"/1/pants","index":10}'

The commit trims '/1' prefix from cause field if exists.

This is a hack to make it display well. It is correct because all error causes
that contain Path puts Path at the head of the string.
2014-10-24 14:12:53 -07:00
030aed8205 Merge pull request #1394 from xiangli-cmu/initial-advertise-peer-urls
*: change advertise-peer-urls -> initial-advertise-peer-urls
2014-10-24 14:08:25 -07:00
2e84eb3c36 *: change advertise-peer-urls -> initial-advertise-peer-urls 2014-10-24 13:51:59 -07:00
9a38be297c Merge pull request #1380 from bcwaldon/client-keys-API
Break apart KeysAPI
2014-10-24 13:48:12 -07:00
f21d93ba60 client: define DefaultRequestTimeout 2014-10-24 13:28:04 -07:00
45d8fbdcda client: move discovery path logic into client pkg 2014-10-24 13:28:04 -07:00
ce4df96e69 client: break apart KeysAPI from httpClient 2014-10-24 13:27:59 -07:00
d7f9228133 Merge pull request #1381 from jonboulle/members
/v2/admin/members API should use JSON containers in response
2014-10-24 13:20:10 -07:00
49a68adcf1 Merge pull request #1386 from unihorn/184
etcdserver: update member attribute when apply request
2014-10-24 12:46:11 -07:00
ea0bff80c0 etcdserver: update member attribute when apply request 2014-10-24 12:43:53 -07:00
d1b57b448d Merge pull request #1383 from unihorn/183
etcdserver: support newly-join member bootstrap
2014-10-24 12:43:25 -07:00
08593bcdf6 etcdserver: support newly-join member bootstrap 2014-10-24 12:38:44 -07:00
9fb02eb6fa Merge pull request #1387 from xiangli-cmu/fix_raft
Fix raft
2014-10-24 12:10:54 -07:00
543e12074a etcdserver/member: change JSON fields to lowerCamelCase 2014-10-24 12:03:17 -07:00
14852662ef etcdhttp: rename Members -> members in JSON, update doc 2014-10-24 12:03:17 -07:00
7ef468b315 etcdhttp: remove /v2/admin/members/x serving 2014-10-24 12:03:17 -07:00
9b679de9dd etcdserver/etcdhttp: use container for admin/members endpoint 2014-10-24 12:03:17 -07:00
c8634428fa Merge pull request #1364 from bcwaldon/import-etcdctl
Import etcdctl
2014-10-24 12:02:41 -07:00
507300130b raft: add tests for ignoring heartbeat reply 2014-10-24 11:50:21 -07:00
e081ad7298 Merge pull request #1368 from xiangli-cmu/doc
doc: add admin api doc
2014-10-24 10:42:14 -07:00
85800fd8f6 Merge pull request #1385 from jonboulle/config
etcdserver: fix + expand config tests
2014-10-24 10:10:07 -07:00
0276089ed9 etcdserver: fix + expand config tests 2014-10-24 10:09:42 -07:00
3a41161e76 Merge pull request #1384 from jonboulle/test
etcdserver: add test for cluster.String
2014-10-24 10:03:37 -07:00
b9514ea265 etcdserver: add test for cluster.String 2014-10-24 10:03:28 -07:00
4c9d67aaa2 Merge pull request #1379 from xiangli-cmu/fix_member
etcdserver: fix member endpoint on peerurl
2014-10-23 17:25:40 -07:00
ed29259801 etcdserver: fix member endpoint on peerurl 2014-10-23 17:18:01 -07:00
500d21591f Merge pull request #1377 from bcwaldon/client-cleanup
Make httpClient.SetPrefix safer
2014-10-23 16:35:45 -07:00
992e7c76e0 client: copy v2KeysPrefix to httpClient
It's poor form to muck with a global variable. Make a copy when the
httpClient object is instantiated to make httpClient.SetPrefix safe.
2014-10-23 16:31:45 -07:00
a85a47c8f9 client: s/v2URL/v2KeysURL/ 2014-10-23 16:31:42 -07:00
ebe32689d4 client: s/v2Prefix/DefaultV2KeysPrefix/ 2014-10-23 16:30:56 -07:00
d1d12abfd7 Merge pull request #1372 from jonboulle/proxy
main: fix proxy initialization and setupCluster
2014-10-23 16:12:19 -07:00
d7301a5cf4 main: fix proxy initialization and setupCluster 2014-10-23 16:09:29 -07:00
d8258c38be Merge pull request #1374 from xiangli-cmu/cluster_on_peerurl
etcdserver: add member endpoint to peerurl
2014-10-23 16:08:45 -07:00
0eddf3db1f Merge pull request #1375 from jonboulle/sort
etcdserver: sort Members() in Cluster
2014-10-23 16:01:29 -07:00
af42f4a56b etcdserver: sort Members() in Cluster 2014-10-23 15:57:27 -07:00
02551c277d etcdserver: add member endpoint to peerurl 2014-10-23 15:55:00 -07:00
17bd5c3d21 Merge pull request #1373 from jonboulle/header
main: add header
2014-10-23 15:37:35 -07:00
0d36385bb4 main: add header 2014-10-23 14:46:05 -07:00
4089475c90 Merge pull request #1361 from unihorn/182
etcdserver: refactor cluster and clusterStore
2014-10-23 14:40:37 -07:00
4d80f01201 etcdserver: Cluster.IsIDremoved -> Cluster.IsIDRemoved 2014-10-23 14:29:58 -07:00
c25c50582e doc: add admin api doc 2014-10-23 14:26:33 -07:00
d47de988e4 etcdserver: panic when >1 members have the given name in MemberByName 2014-10-23 14:24:07 -07:00
5fbe6c7134 main: add genClusterString func and its test 2014-10-23 14:19:40 -07:00
8eee8c260e etcdserver: rebase on master and code clean 2014-10-23 13:58:55 -07:00
e21de51768 etcdserver: remove NewClusterFromMemberInfos 2014-10-23 13:27:55 -07:00
f8b8bdeb17 etcdserver: use path.Join for member key in cluster 2014-10-23 13:27:54 -07:00
3d243baacd etcdserver: generate id when new cluster 2014-10-23 13:27:54 -07:00
d2c4e981ed etcdserver: Member.Pick -> Member.PickPeerURL 2014-10-23 13:27:54 -07:00
67412e07f8 etcdserver: MemberFromName -> MemberByName 2014-10-23 13:27:54 -07:00
89572b5fd7 etcdserver: refactor cluster and clusterStore
Integrate clusterStore into cluster, and let cluster become the source of
cluster info.
2014-10-23 13:27:54 -07:00
09e9618b02 raft: change raftLog.maybeAppend to return the last new index
As per @unihorn's comment on #1366, we change raftLog.maybeAppend to
return the last new index of entries in maybeAppend.
2014-10-23 15:42:47 -04:00
233617bea2 raft: Make MsgAppRes ack only the last index in MsgApp
As explained in #1366, the leader will fail to transmit the missed
logs if the leader receives a hearbeat response from a follower
that is not yet matched in the leader. In other words, there are
append responses that do not explicitly reject an append but
implied a gap.

This commit is based on @xiangli-cmu's idea. We should only acknowledge
upto the index of logs in the append message. This way responses to
heartbeats would never interfer with the log synchronization because
their log index is always 0.

Fixes #1366
2014-10-23 14:56:17 -04:00
16c9970a03 Merge pull request #1365 from xiangli-cmu/admin_get
etcdserver: support GET on admin endpoint
2014-10-23 11:30:51 -07:00
86facb3f91 etcdhttp: sort member 2014-10-23 11:21:20 -07:00
8d6bb4a471 etcdserver: support GET on admin endpoint 2014-10-23 10:53:55 -07:00
051ad7585f Merge pull request #1363 from coreos/return_json
etcdserver: admin PUT returns the json representation of the newly creat...
2014-10-22 20:11:08 -07:00
2cb8efb9b5 Merge pull request #1362 from philips/consistent-hex-use
etcdserver: use hex for cluster and machine id
2014-10-22 18:43:21 -07:00
1539d5c49c etcdctl: update to meet go vet 2014-10-22 17:52:40 -07:00
69842b344d etcdctl: reflect build script location in README.md 2014-10-22 17:52:40 -07:00
52e08720b7 etcdctl: remove build indicator from README.md 2014-10-22 17:52:40 -07:00
29ef918808 etcdctl: import from external repo 2014-10-22 17:52:40 -07:00
9f2a42bf7d godep: add deps for etcdctl 2014-10-22 17:52:40 -07:00
9af5b74a8d gitignore: be more specific when ignoring /etcd 2014-10-22 17:52:40 -07:00
46be5d7d5f version: move version string into new package 2014-10-22 17:52:40 -07:00
63fa2a626a etcdserver: admin PUT returns the json representation of the newly created member 2014-10-22 17:19:28 -07:00
712a05be83 Merge pull request #1357 from coreos/accept_json
etcdserver: admin endpoint accepts json body
2014-10-22 16:30:22 -07:00
ab90369f9e etcdserver: use hex for cluster and machine id
Continue using hex everywhere. Including here.

TODO: cleanup the printing of the structs which currently have decimal
to/from:

`{Type:MsgAppResp To:9973738105406047488 From:17050684879817348455 T...`
2014-10-22 16:24:50 -07:00
7be0f4b618 etcdserver: admin endpoint accepts json body 2014-10-22 16:19:48 -07:00
ec7af4f519 Merge pull request #1359 from philips/fix-mismatch-error
etcdserver: fixup mismatch error line
2014-10-22 15:55:41 -07:00
0d0bc3a57e etcdserver: fixup mismatch error line
Both wcid and gcid are strings, don't try to print it out in hex.
2014-10-22 15:48:22 -07:00
2c21ae0f16 Merge pull request #1358 from philips/etcdserver-cleanup-cluster-id-msg
etcdserver: cleanup the cluster ID error message
2014-10-22 15:42:30 -07:00
1f84991b3a Merge pull request #1355 from barakmich/cluster_after_wal
etcdserver: Check the initial cluster settings after checking if the WAL...
2014-10-22 18:31:28 -04:00
400dd2d7bc etcdserver: cleanup the cluster ID error message
1) Don't panic since we know exactly where this is coming from and don't
need the user to see a full back trace

2) Add docs explaining this situation a bit further

3) Cleanup the error to look like other similiar errors
2014-10-22 15:26:35 -07:00
e42d65da12 etcdserver: Check the initial cluster settings after checking if the WAL exists 2014-10-22 18:16:43 -04:00
d27d308935 Merge pull request #1356 from coreos/fix_dis
main: use our own flagset in setupCluster
2014-10-22 14:55:40 -07:00
5444a366da main: use our own flagset in setupCluster 2014-10-22 14:51:51 -07:00
5780dfe690 Merge pull request #1347 from coreos/gen_nodeID
etcdserver: etcdserver generates the ID when adding new member.
2014-10-22 14:30:49 -07:00
6d9eb57555 etcdhttp: test bad url for POST on admin endopint 2014-10-22 14:27:08 -07:00
d6a5dc9e61 Merge pull request #1352 from coreos/fix_main
main: fix default case
2014-10-22 14:05:11 -07:00
ca4f12182a main: fix default case 2014-10-22 13:49:52 -07:00
d00152765a etcdserver: etcdserver generates the ID when adding new member.
When adding new member, the etcdserver generates the ID based on the current time
and the given peerurls. We include time to add the uniqueness, since the node with
same peerurls should be able to (add, then remove) several times.
2014-10-22 13:15:56 -07:00
13656eb4e7 Merge pull request #1340 from barakmich/better_ids2
etcdserver: Calculate IDs based on PeerURLs and --initial-cluster-name
2014-10-22 14:49:49 -04:00
829cec8ccf Merge stylistic comments 2014-10-22 14:26:05 -04:00
6b32395637 Merge pull request #1343 from unihorn/175
etcdserver: record removed member to check incoming message
2014-10-22 11:16:55 -07:00
5014558b00 etcdserver: clean code 2014-10-22 11:09:36 -07:00
3162ead7b1 etcdhttp: do message statistics only on valid ones 2014-10-22 11:09:36 -07:00
9ad4a8e33a etcdserver: add isKeyNotFound helper func 2014-10-22 11:09:36 -07:00
89b032cd69 etcdserver: Member.storeKey -> memberStoreKey 2014-10-22 11:09:36 -07:00
7498234e40 etcdserver: record removed member to check incoming message 2014-10-22 11:09:35 -07:00
b40d30a8d2 Merge pull request #1346 from unihorn/178
integration: fix master
2014-10-22 11:09:06 -07:00
c5d1fcd70a fix wrong name 2014-10-22 13:58:43 -04:00
cb5a638c44 style comments 2014-10-22 13:52:42 -04:00
52dedab7b4 Move setupCluster inside startEtcd 2014-10-22 13:52:42 -04:00
502a3c2460 Refactor Cluster to hold and add members. 2014-10-22 13:52:42 -04:00
1347e3952f docs and cluster ID change based on name 2014-10-22 13:52:42 -04:00
ad0b7b7dbb Add cluster name to etcd flags 2014-10-22 13:52:42 -04:00
1ca7c031ff first round of comments
Conflicts:
	etcdserver/config.go
	etcdserver/config_test.go
	etcdserver/server.go
	main.go
2014-10-22 13:49:54 -04:00
456d1ebcae etcdserver: Calculate IDs for nodes solely on PeerURLs
Removes the notion of name being anything more than advisory or
command-line grouping, and adds checks for bootstrapping the command
line. IDs are consistent if the URLs are consistent.
2014-10-22 13:49:54 -04:00
4f52d371c1 integration: skip TestGoroutinesRunning 2014-10-22 10:10:43 -07:00
e475388bc0 Merge pull request #1294 from kelseyhightower/issue_619_fix_listing_docs
doc: fix listing a directory
2014-10-22 09:53:48 -07:00
48992cced3 Merge pull request #1344 from unihorn/176
integration: add remaining goroutine check
2014-10-21 18:05:28 -07:00
f356648252 integration: add remaining goroutine check 2014-10-21 16:59:14 -07:00
f26bb6ad44 Merge pull request #1335 from unihorn/174
etcdserver/etcdhttp: store location adjustment
2014-10-21 16:24:30 -07:00
06b196e345 etcdhttp: writeEvent -> writeKeyEvent 2014-10-21 16:10:49 -07:00
ca73f25615 etcdhttp: parseRequest -> parseKeyRequest 2014-10-21 16:10:49 -07:00
e2b6a4fc4c etcdserver: const XXXDir -> StoreXXXPrefix
and code clean
2014-10-21 16:10:49 -07:00
2ff3cac653 etcdserver/etcdhttp: store location adjustment
Detailed adjustment:
/_etcd/machines/* -> /0/members/*
/* -> /1/*

And it keeps key path returned to user the same as before.
2014-10-21 16:10:19 -07:00
0398a31b16 Merge pull request #1339 from coreos/checkcid
etcdserver: checking clusterID
2014-10-21 16:00:49 -07:00
e69c37adf0 etcdserver: panic on clusterid mismatch 2014-10-21 15:22:16 -07:00
00dc61d169 Merge pull request #1341 from coreos/fix_bench
raft: fix node bench
2014-10-21 13:01:37 -07:00
48c4145f1b raft: fix node bench 2014-10-21 12:46:39 -07:00
a9984fda4f Merge pull request #1102 from coreos/node_bench
raft: add a one node bench
2014-10-21 11:44:46 -07:00
50d4abc676 raft: add a one node bench 2014-10-21 11:43:55 -07:00
c3f83f9275 Merge pull request #1337 from jonboulle/govet
tests: add `go vet` coverage
2014-10-21 11:40:26 -07:00
341c7190d3 test: add govet to tests 2014-10-21 11:32:38 -07:00
894e678ad6 etcdserver: checking clusterID 2014-10-21 11:05:24 -07:00
ae4403c945 test: add discovery to tests 2014-10-21 00:17:14 -07:00
a44849deec Merge pull request #1286 from coreos/clusterid
*: generate clusterid
2014-10-20 19:07:03 -07:00
99dd42026b Merge pull request #1336 from jonboulle/govet
*: fix various formatting issues
2014-10-20 18:23:26 -07:00
b3d5333cb3 *: fix various formatting issues 2014-10-20 17:55:18 -07:00
0fd28169c8 etcdserver: use id,cid 2014-10-20 16:35:41 -07:00
dc68dc9ebd etcdserver: add a todo for clusterid generation 2014-10-20 16:26:31 -07:00
d67a2855c1 Merge pull request #1334 from jonboulle/exit_help
main: exit 0 on -h/--help
2014-10-20 16:19:39 -07:00
92230cee63 Merge pull request #1333 from unihorn/172
etcdserver/raft: remove msgDenied, removedNodes, shouldStop
2014-10-20 16:04:09 -07:00
63b328ed11 main: exit 0 on -h/--help 2014-10-20 15:32:58 -07:00
e200d2a8e2 etcdserver/raft: remove msgDenied, removedNodes, shouldStop
The future plan is to do all these in etcdserver level.
2014-10-20 15:13:18 -07:00
ea6bcacfe4 *: generate clusterid 2014-10-20 15:00:54 -07:00
68da8084d0 Merge pull request #1318 from unihorn/164
main: add basic integration test
2014-10-20 14:45:22 -07:00
ef44ba10cf integration: add doc 2014-10-20 14:43:21 -07:00
80212aaf4d integration: retry on setKey to avoid timeout due to bootstrap 2014-10-20 14:43:21 -07:00
500a72962e integration: clean code
remove extra space, rename variables, remove unused function.
2014-10-20 14:43:21 -07:00
7af679333a integration: log microsecond time for integration tests 2014-10-20 14:43:21 -07:00
1b7947357f *: pkg functional -> integration 2014-10-20 14:43:21 -07:00
40279d324a functional: add TODO 2014-10-20 14:43:21 -07:00
f7a0d5387b etcdserver: stop server gracefully 2014-10-20 14:43:21 -07:00
058537f34a main: add basic functional test 2014-10-20 14:43:21 -07:00
8fa3834d69 Merge pull request #1327 from unihorn/168
docs: add naming.md
2014-10-20 14:23:36 -07:00
3184e1c66f docs: add glossary.md 2014-10-20 14:16:53 -07:00
dcaa7f0a37 Merge pull request #1328 from unihorn/169
skip initial-cluster check when reboot
2014-10-20 13:06:24 -07:00
17382ec905 etcdserver: skip initial-cluster check when reboot
If etcd is provided with data-dir that has data, it will not use
initial-cluster, and initial-cluster could be set to be empty.
2014-10-20 12:41:07 -07:00
da23327265 Merge pull request #1302 from jonboulle/license
add copyright info at the head of each file
2014-10-20 10:49:52 -07:00
7a4d42166b *: add license header to all source files 2014-10-17 15:41:22 -07:00
aa176610f3 main: remove duplicated self name check 2014-10-17 15:37:30 -07:00
f12583c163 Merge pull request #1021 from jonboulle/clock
Implement fake clock for testing
2014-10-17 13:56:30 -07:00
1456ae4453 store: restore minExpireTime check and advance FakeClock appropriately 2014-10-17 10:05:29 -07:00
e0801360d3 godep: update clockwork dependency 2014-10-17 10:05:29 -07:00
ec18e46641 etcdserver/etcdhttp: switch to using fake clock 2014-10-17 10:05:29 -07:00
3134658ded store: switch to fake clock 2014-10-17 10:05:29 -07:00
47c2421f7b godeps: add clockwork 2014-10-17 10:05:29 -07:00
de3bf58876 discovery: switch to fake clock 2014-10-17 10:05:29 -07:00
c5ba66e6aa Merge pull request #1319 from jonboulle/stats_race
etcdserver: fix data race in retrieving self stats
2014-10-17 09:04:17 -07:00
aed525edee Merge pull request #1325 from jonboulle/store_cleaner_still
store: remove unused code
2014-10-17 09:02:36 -07:00
766aa85320 Merge pull request #1326 from jurmous/patch-1
Update clients-matrix.md
2014-10-17 16:05:37 +02:00
6aa46d20d4 Update clients-matrix.md
Fix new boon etcd client entry so it complies to the rest of the entries with Y instead of F for HTTPS and Reconnect. Also fix the sorting.
2014-10-17 15:30:35 +02:00
da64e7509c etcdserver/stats: lock on leaderstats too 2014-10-17 00:11:25 -07:00
5c4edf65f9 store: remove another unused function 2014-10-17 00:07:45 -07:00
1fa763b47b store: remove unused function 2014-10-17 00:07:23 -07:00
70bbf8b470 store: remove more unused code 2014-10-17 00:05:56 -07:00
82023c591d etcdserver/stats: log any marshaling error 2014-10-16 23:05:48 -07:00
233e940410 etcdserver: copy stats instead of marshaling with lock 2014-10-16 23:05:48 -07:00
c28907ba95 etcdserver: fix race and improve stats interfaces 2014-10-16 23:05:48 -07:00
c30b82b596 etcdserver: fix data race in retrieving self stats 2014-10-16 23:05:48 -07:00
7311a2a67d Merge pull request #1321 from jonboulle/peers
main: correct peer-addr reference
2014-10-16 20:17:22 -07:00
bc7d372d5c main: correct peer-addr reference 2014-10-16 20:06:26 -07:00
67368ac7fa Merge pull request #1320 from jonboulle/proxy_doc
doc: add note about discovery proxy
2014-10-16 17:23:29 -07:00
2af0b2031f doc: add note about discovery proxy 2014-10-16 17:14:38 -07:00
b7c42b0d76 Merge pull request #1314 from jonboulle/raft_tests
raft: remove unused compactThreshold
2014-10-16 17:11:55 -07:00
fc42bdb904 raft: remove unused compactThreshold 2014-10-16 17:11:10 -07:00
f7988e6069 Merge pull request #909 from jonboulle/proxy
Support corporate proxy via env var
2014-10-16 16:50:31 -07:00
3a29db1e9d Merge pull request #1311 from jonboulle/remove_ttl
store: remove unused code
2014-10-16 16:35:01 -07:00
7ef375efbd Merge pull request #1317 from unihorn/169
etcdserver: fix data race in leaderStats.Followers
2014-10-16 16:24:26 -07:00
782d91f2d9 etcdserver: fix data race in leaderStats.Followers 2014-10-16 16:18:53 -07:00
074ddb5876 Merge pull request #1316 from unihorn/168
stats: fix data race when recording send result
2014-10-16 16:11:22 -07:00
eb72bdc3d2 stats: fix data race when recording send result 2014-10-16 15:39:31 -07:00
da2ee9a90c Merge pull request #1312 from unihorn/165
etcdhttp: set stats field in peer handler
2014-10-16 14:47:57 -07:00
8609acf573 etcdhttp: set stats field in peer handler 2014-10-16 14:23:09 -07:00
2cd6594485 store: remove unused code 2014-10-16 12:28:05 -07:00
7f8f371b0e discovery: add ability to proxy discovery requests 2014-10-16 11:15:04 -07:00
6a30d3ba04 Merge pull request #1285 from jonboulle/stats_leader_followers
etcdserver: add StoreStats, ServerStats and LeaderStats
2014-10-16 10:44:48 -07:00
9b9e72e2a3 etcdhttp: add tests for ServeStats/ServeLeader 2014-10-16 10:43:49 -07:00
97ae531eda etcdserver: split out storestats and serverstats 2014-10-16 10:43:49 -07:00
fedb67a71a etcdserver: fix TODO and change to base 16 2014-10-16 10:43:49 -07:00
8168fed825 etcdserver: add ServerStats and LeaderStats
This adds the remaining two stats endpoints: `/v2/stats/self`, for
various statistics on the EtcdServer, and `/v2/stats/leader`, for
statistics on a leader's followers.

By and large most of the stats code is copied across from 0.4.x, updated
where necessary to integrate with the new decoupling of raft from
transport.

This does not satisfactorily resolve the question of name vs ID. In the
old world, names were unique in the cluster and transmitted over the
wire, so they could be used safely in all statistics. In the new world,
a given EtcdServer only knows its own name, and it is instead IDs that
are communicated among the cluster members. Hence in most places here we
simply substitute a string-encoded ID in place of name, and only where
possible do we retain the actual given name of the EtcdServer.
2014-10-16 10:43:44 -07:00
0a8721a708 etcdserver: expose store statistics 2014-10-16 10:43:23 -07:00
79e9f2ab81 Merge pull request #1296 from unihorn/161
etcdserver: add checking when apply conf change
2014-10-16 09:58:51 -07:00
8cd6030a1d etcdserver: add checking when apply conf change 2014-10-16 09:49:26 -07:00
82476e04f0 Merge pull request #1303 from coreos/datadir
main: use node name as default dir
2014-10-16 21:31:18 +08:00
efba919a93 main: use node name as default dir
We use nodeID as the default dir previously. It works fine before we do dynamic nodeID
generation (introducing time). After the change the dynamic nodeID will change every
time we restart the etcd process. If the user does not provide the data dir, the default
dir will change every time. It is not the desired behavior.

In this commit, we change the default data dir to node name. If the user changes the node
name and does provide the data dir, etcd still cannot recover from previous state. But it
is much better than using nodeID. And it is actually a doucmentation issue.

Conflicts:
	main.go
2014-10-16 21:22:12 +08:00
c18acd7d6f Merge pull request #1306 from jonboulle/tls
pkg: set minimum TLS version to 1.0 (disable SSL3)
2014-10-15 14:02:05 -07:00
e334148a91 pkg: set minimum TLS version to 1.0 (disable SSL3)
SSLv3 is no longer considered secure, and is not supported by golang
clients. Set the minimum version of all TLSConfigs that etcd uses to
ensure that only TLS >=1.0 can be used.
2014-10-15 14:00:16 -07:00
eb2dd1892f raft: add RemovedNodes to SoftState 2014-10-15 10:53:07 -07:00
a8a1d4fd93 Merge pull request #1299 from unihorn/162
etcdhttp: add PUT and DELETE on /v2/admin/members/
2014-10-15 10:52:11 -07:00
f62d4908b0 Merge pull request #1304 from coreos/logging
etcdserver: better logging
2014-10-15 13:36:32 +08:00
828accf07b etcdserver: better logging 2014-10-15 13:07:34 +08:00
6d0658c8ca etcdhttp: check error returned by ParseForm 2014-10-14 14:50:11 -07:00
48c195fac7 Merge pull request #1298 from jonboulle/members
etcdhttp: remove members handling for now
2014-10-14 13:57:52 -07:00
7656069675 etcdhttp: remove members handling for now 2014-10-14 12:07:28 -07:00
99e35554c0 etcdhttp: add doc for member management http endpoint 2014-10-14 11:09:14 -07:00
b0fcd680f8 Merge pull request #1287 from coreos/refactor
*: proto refactoring
2014-10-15 01:51:44 +08:00
f98fbbfc14 *: proto refactoring 2014-10-14 21:07:23 +08:00
2b03d35ab9 Merge pull request #1300 from jonboulle/godep
*: move from third_party to Godep
2014-10-14 01:03:58 -07:00
4183b69e12 *: move from third_party to Godep 2014-10-14 00:37:52 -07:00
31264e7eb5 etcdhttp: add PUT and DELETE on /v2/admin/members/ 2014-10-13 22:36:06 -07:00
09f9884c6a Merge pull request #1288 from coreos/cleanup
etcdserver: clean NewServer
2014-10-14 11:32:13 +08:00
fbb874172c etcdserver: fix typo 2014-10-14 11:30:49 +08:00
6fc0b1977b etcdserver: Id -> ID 2014-10-14 11:30:49 +08:00
b53b74733a etcdserver: add s.apply 2014-10-14 11:30:49 +08:00
30c7a7f2dd etcdserver: add shouldDiscover 2014-10-14 11:29:00 +08:00
a85ec90d68 etcdserver: clean NewServer 2014-10-14 11:29:00 +08:00
57ae19b500 Merge pull request #1290 from coreos/member_endpoint
etcdserver: change machines endpoint to members
2014-10-14 11:04:08 +08:00
1177b07535 etcdserver: change machines endpoint to members 2014-10-14 11:02:17 +08:00
d7dfe07e5d Merge pull request #1293 from unihorn/160
raft: protobuf messageType
2014-10-14 09:16:38 +08:00
9722aac10d Merge pull request #1295 from jonboulle/typos
main: fix typos and initial-cluster references
2014-10-13 15:16:21 -07:00
a6dfde85e4 main: fix typos and initial-cluster references 2014-10-13 14:52:49 -07:00
85c2d852f3 Merge pull request #1259 from unihorn/156
etcdserver/raft: apply bootstrap addnode entries
2014-10-13 12:18:43 -07:00
f693c6ddf2 etcdserver: apply bootstrap conf change 2014-10-13 11:22:23 -07:00
0319b033ea etcdserver/raft: set context for bootstrap addnode entries 2014-10-13 11:22:23 -07:00
32c38820c1 raft: protobuf messageType 2014-10-13 11:13:43 -07:00
d06d55193b doc: fix listing a directory 2014-10-13 10:59:24 -07:00
b8b4852ec9 Merge pull request #972 from tazjin/etcd-security-docs
Documentation: Clarify etcd security documentation
2014-10-13 08:55:12 -07:00
6ffaa4db5d Merge pull request #1289 from coreos/uint
*: change all ID to uint64
2014-10-12 13:58:43 +08:00
3516cc3ee5 *: change all ID to uint64 2014-10-12 08:38:48 +08:00
f16a272898 Merge pull request #1234 from unihorn/152
etcdserver: save PeerURLs and Others into two different keys
2014-10-10 12:21:32 -07:00
f8b338d423 Merge pull request #1236 from unihorn/153
wal: record node id at the head of WAL file
2014-10-10 12:09:17 -07:00
447caf1afc etcdserver/wal: record info at the head of WAL file 2014-10-10 11:57:09 -07:00
181cbbdfe0 Merge pull request #1280 from RichardHightower/patch-2
Update clients-matrix.md
2014-10-10 10:05:26 -07:00
d241275518 Merge pull request #1275 from RichardHightower/patch-1
Update libraries-and-tools.md
2014-10-10 09:58:32 -07:00
2ec999ab3b Update clients-matrix.md 2014-10-10 02:03:35 -07:00
77271b0663 etcdserver: split Member into RaftAttributes and Attributes
The split helps to save them in different key paths, because they have
distinct life cycle on update.
2014-10-09 20:28:00 -07:00
74ab003e1f Merge pull request #1256 from barakmich/retry
Add logging and backoff and simple retry logic
2014-10-09 13:24:14 -04:00
5e7267a751 last comment 2014-10-09 13:16:45 -04:00
3e5073e9be Merge pull request #1261 from coreos/uint
raft: int64 -> uint64
2014-10-09 14:47:03 +08:00
1eb1020717 raft: fix raft test 2014-10-09 14:42:29 +08:00
8bbbaa88b2 *: raft related int64 -> uint64 2014-10-09 14:29:21 +08:00
af5b8c6c44 raft: int64 -> uint64 2014-10-09 14:26:43 +08:00
8c5aa16d03 Update libraries-and-tools.md 2014-10-08 21:50:22 -07:00
5fde52a403 add too many retries test case 2014-10-08 21:58:19 -04:00
9b35ca3a52 simplify and multiple retry 2014-10-08 21:53:15 -04:00
38af14b0f4 Merge pull request #1260 from coreos/snap_rm
raft: save removed nodes in snapshot
2014-10-09 08:00:33 +08:00
dbac2e8f15 test retry logic 2014-10-08 19:41:59 -04:00
ea99d3c002 Merge pull request #1269 from jonboulle/disco_warn
etcdserver: add discovery warning message
2014-10-08 16:24:51 -07:00
682008724d etcdserver: add discovery warning message 2014-10-08 16:09:12 -07:00
abe97e49d5 raft: more comment 2014-10-09 07:02:05 +08:00
73f2aaf98f raft: removedSlice -> removedNodes 2014-10-09 06:55:25 +08:00
c67fd14fe8 Merge pull request #1257 from bdarnell/cleanups
Raft: assorted cleanups (golint and go vet)
2014-10-09 05:55:21 +08:00
b4a7680bc4 Merge pull request #1268 from jonboulle/store_typo
store: fix typo "recurisive"
2014-10-08 14:22:07 -07:00
a07c51a9c9 store: fix typo "recurisive" 2014-10-08 14:13:38 -07:00
d2e858587f Raft: a few more improvements to test messages. 2014-10-08 15:07:11 -04:00
7b61565c0a raft: save removed nodes in snapshot 2014-10-08 15:33:55 +08:00
f3870598b9 fix retry to not retry on createSelf() 2014-10-07 21:20:45 -04:00
5f3fe7c61f Merge pull request #1254 from coreos/rand_etimeout
Rand etimeout
2014-10-08 08:10:26 +08:00
1cd3345e00 raft: address issues with election timeout 2014-10-08 07:41:17 +08:00
75f6643982 Merge pull request #1212 from unihorn/144
etcdserver: apply config change on cluster store
2014-10-07 16:32:59 -07:00
8e8719f6ac etcdserver: Create, Delete -> Add, Remove 2014-10-07 16:21:58 -07:00
1083ce8f73 raft: remove misleading labels in array definition
Since these are arrays instead of maps, the "keys" here are actually
(useless) goto labels. What really matters is that the ordering is
the same between the constant declarations and the array.
2014-10-07 18:44:06 -04:00
36558b1924 Raft: fix printf strings found by go vet. 2014-10-07 18:44:06 -04:00
3ad0df3722 Raft: fix problems reported by golint. 2014-10-07 18:44:06 -04:00
fe0e168b3b Add *.test to gitignore.
These files are left by some go commands.
2014-10-07 18:22:09 -04:00
ca1bbee737 add logging and backoff and simple retry logic 2014-10-07 16:58:58 -04:00
5b8c4f4e0d Merge pull request #1251 from philips/hack-insta-discovery
hack: introduce a insta-discovery script
2014-10-07 13:47:03 -07:00
cdea98d434 etcdserver: skip bootstrap addNode entry
This is a hack to make etcd work normally.
2014-10-07 11:11:31 -07:00
6a62621695 etcdserver: update panic info 2014-10-07 11:04:50 -07:00
3859297225 etcdserver: check id match 2014-10-07 11:04:50 -07:00
d051af4d3d etcdserver: apply config change on cluster store 2014-10-07 11:04:50 -07:00
ef0ed31210 hack: introduce a insta-discovery script
This sets up a localhost cluster via a new discovery token. Very useful
for testing.
2014-10-07 10:52:40 -07:00
f65d117462 raft: add a test for randElectionTimeout 2014-10-07 20:34:15 +08:00
d7d6f84f64 raft: rand election timeout 2014-10-07 20:12:49 +08:00
2e0fec7a84 Merge pull request #1253 from jonboulle/streaming_events
store: use a larger buffer for eventChan
2014-10-07 02:14:20 -07:00
26160b2154 store: use a larger buffer for eventChan
The event channel for watches can drop events too easily in the case of
streaming watches. This increases the buffer to a more useful level.
2014-10-07 01:57:05 -07:00
3f3b9866c6 Merge pull request #1241 from coreos/compact
raft: compact takes index and nodes parameters
2014-10-07 16:07:25 +08:00
45e4a8643a raft: add tests for raft.compact 2014-10-07 16:03:11 +08:00
7fe4385ef9 raft: add comment for Compact interface of Node 2014-10-07 16:03:11 +08:00
5587e0d73f raft: compact takes index and nodes parameters
Before this commit, compact always compact log at current appliedindex of raft.
This prevents us from doing non-blocking snapshot since we have to make snapshot
and compact atomically. To prepare for non-blocking snapshot, this commit make
compact supports index and nodes parameters. After completing snapshot, the applier
should call compact with the snapshot index and the nodes at snapshot index to do
a compaction at snapsohot index.
2014-10-07 16:03:11 +08:00
de024ec844 Merge pull request #1250 from philips/fix-single-node
main: use a new cluster by default
2014-10-06 19:10:58 -07:00
6cbc282be1 main: use a new cluster by default
This is a safe default for now. Use it to make `./etcd` work again.
2014-10-06 17:34:14 -07:00
39e0a0cd0a Merge pull request #1249 from barakmich/sanity_check
Split config into a separate file and add sanity check and test
2014-10-06 20:19:23 -04:00
d6aea2a795 add golint on the new box and fix appropriate lint 2014-10-06 20:16:25 -04:00
8a311e5b76 remove new cluster check 2014-10-06 20:07:51 -04:00
1a0195e07e tableize the test 2014-10-06 20:05:53 -04:00
3ca3c9ad4c Merge pull request #1245 from unihorn/155
main/raft: write addNode ConfChange entries in log when start raft
2014-10-06 17:00:28 -07:00
b9c8ac73be Merge pull request #1248 from jonboulle/disco
main: fix temporary discovery cluster
2014-10-06 16:54:06 -07:00
120b088723 Split config into a separate file and add sanity check and test 2014-10-06 19:42:13 -04:00
5098cb0d32 main: fix temporary discovery cluster 2014-10-06 16:10:29 -07:00
45ebfb4217 raft: refine initial entries logic in StartNode 2014-10-06 16:06:01 -07:00
d98fe2ce1a Merge pull request #1247 from jonboulle/doc
documentation: clustering guide fixes
2014-10-06 15:16:25 -07:00
c15c3eab4c etcdserver: move int64Slice into pkg/types/ 2014-10-06 15:12:02 -07:00
a10c62ae25 documentation: clustering guide fixes 2014-10-06 15:07:44 -07:00
3500b56e54 Merge pull request #1237 from philips/0.5-docs
use initial-cluster and initial-cluster-state flags
2014-10-06 15:04:04 -07:00
e2d8037ded main: use initial-cluster and initial-cluster-state flags
In preperation for adding the ability to join a machine to an existing
cluster force the user to specify whether they expect this to me a new
cluster or an active one.

The error for not specifying the initial-cluster-state is:
```
etcd: initial cluster state unset and no wal found
```
2014-10-06 14:59:25 -07:00
314d425718 main/raft: write addNode ConfChange entries in log when start raft 2014-10-06 14:33:12 -07:00
7081dabd12 Merge pull request #1244 from jonboulle/peers
main: fail on peers/peers-file flags
2014-10-06 14:25:39 -07:00
ec7bcbb50d main: fail on peers/peers-file flags 2014-10-06 14:25:11 -07:00
6bc160b4e3 Documentation: introduce the initial 0.5 clustering guide
Introduce the concepts of static initial cluster configuration and
document the flag migrations.
2014-10-06 12:25:12 -07:00
9e3d045b2b *:discovery hook up 2014-10-06 10:12:42 -07:00
824b7231b8 Merge pull request #1240 from coreos/fix_proposal
raft: fix send
2014-10-06 05:53:54 +08:00
dc9cb4b4ba raft: fix send
send should not attach current term to msgProp. Send should simply do proxy for msgProp without
changing its term. msgProp has a special term 0, which indicates that it is a local message.
2014-10-06 04:48:35 +08:00
7ce3bb180c Merge pull request #1238 from jonboulle/etcdserver_cluster
etcdserver: restore test coverage of cluster
2014-10-03 17:06:36 -07:00
d2df23183d etcdserver: restore test coverage of cluster 2014-10-03 17:06:23 -07:00
2606508d1c Merge pull request #1215 from jonboulle/1215
Flag `-timeout` not descriptive
2014-10-03 14:06:18 -07:00
f432b9d29b etcdhttp: remove configurable timeout
It's slightly unclear why we expose this timeout as being configurable,
and the `-timeout` flag does not exist in 0.4.x, so for now, remove the
flag until we have evidence that it is needed.
2014-10-03 13:47:34 -07:00
1ca5991c8c Merge pull request #1096 from jonboulle/1096_embedded
simplify initialization of EtcdServer to support embedded etcd
2014-10-03 13:39:25 -07:00
1308c3e809 etcdserver: unexport EtcdServer fields where possible 2014-10-03 13:34:56 -07:00
e5f5fcff48 etcdserver: collapse dir configuration to DataDir 2014-10-03 13:34:49 -07:00
af6b29f291 main: move server configuration to etcdserver package 2014-10-03 13:34:49 -07:00
1c4163faf8 Merge pull request #1187 from jonboulle/1187_watches_header
Watches do not return X-Etcd-Index header
2014-10-03 10:13:25 -07:00
01ecc60a88 Merge pull request #1203 from coreos/fix_raft
raft: commitIndex=min(leaderCommit, index of last new entry)
2014-10-03 22:24:07 +08:00
172bd7d096 raft: add test for maybeappend change 2014-10-03 22:21:35 +08:00
5b291b521b Merge pull request #1207 from coreos/check_stale_resp
raft: do not decrease nextIndex and send entries for stale reply
2014-10-03 13:45:58 +08:00
70bf464cd6 raft: add comment to decrTo 2014-10-03 13:42:34 +08:00
16ba77767e raft: do not decrease nextIndex and send entries for stale reply 2014-10-03 13:41:27 +08:00
1c11f6a144 *: expose etcd-index in watch requests
This adds a StartIndex field to the Watcher interface, which represents
the Etcd-Index at which the Watcher is created.

Also refactors the HTTP tests to use a table for most handleWatch tests
2014-10-02 18:10:11 -07:00
8490904f20 Merge pull request #1224 from unihorn/149
raft: msg.Denied -> msg.Reject
2014-10-02 12:24:09 -07:00
fff918c672 raft: msg.Denied -> msg.Reject
Change the field name because it has msgDenied already.
2014-10-02 12:22:12 -07:00
4a5bf2e1b7 Merge pull request #1151 from unihorn/138
raft: add removed
2014-10-01 15:01:30 -07:00
b64246720b etcdserver: fix test due to rebase 2014-10-01 14:57:39 -07:00
182c8316e1 raft: refine comment for doc and removed list tests 2014-10-01 14:57:39 -07:00
e4a6c9651a raft: add removed
The usage of removed:
1. tell removed node about its removal explicitly using msgDenied
2. prevent removed node disrupt cluster progress by launching leader election

It is set when apply node removal, or receive msgDenied.
2014-10-01 14:57:38 -07:00
b1fc0feb72 Merge pull request #1221 from philips/fixing-stuff
pkg/types: introduce a URLs type
2014-10-01 14:55:41 -07:00
83137f9eba pkg/types: introduce a URLs type
Cleanup the usage of URLs into its own type so we don't have to use a
FlagValue everywhere we have a list of URLs.
2014-10-01 14:41:01 -07:00
619c7f9fdb Merge pull request #1220 from bcwaldon/bkcompat
Backwards-compatibility with v0.4.6 addr-related flags
2014-10-01 12:53:50 -07:00
f84b5b1071 Merge pull request #1201 from unihorn/143
etcdserver: publish self info when start
2014-10-01 11:56:20 -07:00
b3c7711da8 Merge pull request #1219 from unihorn/148
flags/urls: reject url without port
2014-10-01 11:51:32 -07:00
073eb7677d etcdserver: move grep member logic into publish func 2014-10-01 11:50:47 -07:00
dd88a08f8e etcd: support v0.4.6 addr flags
The -addr, -bind-addr, -peer-addr, and peer-bind-addr flags are
superseded by -advertise-client-urls, -listen-client-urls,
-advertise-peer-urls, and -listen-peer-urls, respectively.
If any of the former flags are provided, however, they will still
work. If the new counterparts to the old flags are provided, etcd
will log an error and fail.
2014-10-01 11:49:01 -07:00
11582b0f5f pkg: add URLsFromFlags 2014-10-01 11:49:01 -07:00
add3906f6c pkg/flags: add NewURLs helper 2014-10-01 11:49:01 -07:00
c9cac5fee5 flags: replace Addrs with IPAddressPort 2014-10-01 11:49:00 -07:00
cbc84bc70e etcdserver: minimize input info 2014-10-01 11:47:17 -07:00
a40a270e19 etcdserver: publish self info when start 2014-10-01 11:47:17 -07:00
1356037fc6 flags/urls: reject url without port
For now, if etcd receives a url without port, it will listen on a random
port, which is useless.
2014-10-01 11:44:17 -07:00
c0c0b08ff2 Merge pull request #1218 from unihorn/147
flags/urls: assign urls instead of append
2014-10-01 11:34:24 -07:00
fd758c71b8 Merge pull request #1217 from unihorn/146
main: not use loop variable in goroutine in loop body
2014-10-01 11:25:53 -07:00
8944364884 flags/urls: assign urls instead of append 2014-10-01 11:23:20 -07:00
7a698be6a3 main: not use loop variable in goroutine in loop body
Because loop variable are reused in following loops.
2014-10-01 11:07:32 -07:00
6760345453 Merge pull request #1216 from philips/improve-cluster-procfile
etcdserver: stop worrying about scheme
2014-09-30 17:13:41 -07:00
04bd48fef3 etcdserver: stop worrying about scheme
Stop worrying about the scheme. This puts a TODO on adding validation to
the schemes if TLS is specified. But we can worry about that later.
2014-09-30 17:05:20 -07:00
7639752c82 Merge pull request #1213 from philips/improve-cluster
etcdserver: stop using addScheme
2014-09-30 16:43:26 -07:00
c2f96631d3 etcdserver: stop using addScheme
This standardizes the flags to use a list of URLs everywhere. The next
step is to enforce the scheme based on TLS settings and support compat
flags.
2014-09-30 16:40:32 -07:00
aa5b6cdc9e etcdserver: have newMember take an optional time field
This will be used by members joining an existing cluster or joining
using discovery.
2014-09-30 14:57:04 -07:00
ce70e63cc6 Merge pull request #1200 from coreos/raft_heartbeat
raft: heartbeat is only response for maintaining leader dominance
2014-09-29 17:00:26 -07:00
d7b4e44a66 raft: heartbeat is only response for maintaining leader dominance 2014-09-29 16:57:43 -07:00
b3c1bd5616 raft: commitIndex=min(leaderCommit, index of last new entry) 2014-09-29 14:38:17 -07:00
b6a73c9358 Merge pull request #1202 from unihorn/144
etcdserver: fix left Id -> ID
2014-09-29 14:21:42 -07:00
34547229a6 etcdserver: fix left Id -> ID 2014-09-29 14:14:45 -07:00
0e8345aa73 Merge pull request #1143 from unihorn/136
*: Id -> ID for protobuf types
2014-09-29 13:58:02 -07:00
768090754c Merge pull request #1199 from AdoHe/master
update the Documentation/libraries-and-tools.md to add a new Java client
2014-09-29 13:21:40 -07:00
08e5f39d8a Merge pull request #1192 from unihorn/140
etcdserver: add publish func
2014-09-29 10:20:39 -07:00
89077167c3 etcdserver: add publish func 2014-09-29 10:13:06 -07:00
1eb09acd8b Merge pull request #1197 from coreos/raft_t
Add raft msg denied
2014-09-28 23:53:13 -07:00
1c27fad2cf update the Documentation/libraries-and-tools.md 2014-09-29 13:42:12 +08:00
e26ff32fd8 raft: fix error msg 2014-09-28 21:17:51 -07:00
51529cc3f2 raft: remove index field in msg AppResp 2014-09-28 21:13:53 -07:00
adefd83855 raft: remove index field in msg voteResp 2014-09-28 21:13:43 -07:00
86473d8a27 raft: add msg denied field 2014-09-28 21:13:33 -07:00
e38fbfe9de Merge pull request #1196 from philips/add-flagtypes-package
flagtypes: introduce flagtypes package
2014-09-28 19:48:12 -07:00
d8bf9728d2 Merge pull request #1194 from unihorn/141
etcdserver: return client urls for /v2/machines
2014-09-28 17:29:52 -07:00
78a9bba276 pkg/types/flags: introduce flags package
I want to use the Addrs type in another experimental proxy that I am
implementing. Pull it out into a separate package.
2014-09-28 14:56:30 -07:00
5784693a39 etcdserver: Cluster.Endpoints() -> Cluster.PeerURLs() 2014-09-28 14:39:47 -07:00
e83f851995 etcdserver: move cluster test from etcdhttp to etcdserver 2014-09-28 14:35:48 -07:00
15798a73d9 etcdserver: return client urls for /v2/machines 2014-09-26 16:50:15 -07:00
2a0f3d85c8 Merge pull request #1190 from jonboulle/undeprecate_peers_file
main: undeprecate peers-file for now
2014-09-26 13:57:36 -07:00
1d5d2e3726 *: Id -> ID for protobuf types
We use ID instead of Id in this project based on golang conventions.
2014-09-26 11:49:30 -07:00
04f6993108 main: undeprecate peers-file for now 2014-09-26 09:54:47 -07:00
61dc89e7f3 Merge pull request #1147 from philips/add-name
introduce cluster store
2014-09-25 18:46:18 -07:00
5e3fd6ee3f etcdserver: introduce the cluster and member
This introduces two new concepts: the cluster and the member.

Members are logical etcd instances that have a name, raft ID, and a list
of peer and client addresses.

A cluster is made up of a list of members.
2014-09-25 18:04:10 -07:00
56c64ab2e8 Merge pull request #1153 from jonboulle/1153_etcd_index
X-Etcd-Index off by one
2014-09-25 17:00:22 -07:00
9c9437a9e7 Merge pull request #1152 from jonboulle/1152_raft_headers
Missing X-Raft-Term & X-Raft-Index headers
2014-09-25 17:00:02 -07:00
9b3478218e etcdserver: add RaftIndex and RaftTerm 2014-09-25 16:46:24 -07:00
5874387871 Merge pull request #1183 from coreos/corspeer
main: remove cors wrapper for raft handler
2014-09-25 16:10:11 -07:00
8a60257bff main: remove cors wrapper for raft handler 2014-09-25 11:58:11 -07:00
9e46d54483 Merge pull request #1182 from coreos/cors
pkg: move cors.go to /pkg/cors.go
2014-09-25 11:49:04 -07:00
720aa6aeae pkg: move cors.go to /pkg/cors.go 2014-09-25 11:46:08 -07:00
0662afc95f Merge pull request #1181 from coreos/functional
functional: remove functional
2014-09-25 11:39:35 -07:00
5feef73a17 Merge pull request #1180 from coreos/elog
elog: remove elog
2014-09-25 11:35:58 -07:00
f393b1459a functional: remove functional 2014-09-25 11:34:39 -07:00
aab41f06d0 elog: remove elog 2014-09-25 11:33:10 -07:00
f96f1041fd Merge pull request #1179 from coreos/scripts
scripts: remove broken script
2014-09-25 11:31:56 -07:00
05d8f7270f scripts: remove broken scripts 2014-09-25 11:29:30 -07:00
e20e286064 Merge pull request #1178 from coreos/crc
pkg: move /crc to /pkg/crc
2014-09-25 10:55:24 -07:00
20ac0ee80d Merge pull request #1177 from coreos/tran
pkg: move /transport to pkg/transport
2014-09-25 10:55:11 -07:00
9158d293fd Merge pull request #1176 from coreos/testutil
pkg: move testutil to pkg
2014-09-25 10:51:56 -07:00
2f6086de22 pkg: move /crc to /pkg/crc 2014-09-25 10:50:33 -07:00
a0f5625728 pkg: move /transport to pkg/transport 2014-09-25 10:47:14 -07:00
45f71af33e pkg: move testutil to pkg 2014-09-25 10:40:40 -07:00
2e4f725c2b Merge pull request #1172 from jonboulle/1172_ttl_now
TTL=0 should expire keys "now"
2014-09-24 17:41:56 -07:00
2da1010cf7 etcdhttp: make TTL= equivalent to unset, and TTL=0 expire 2014-09-24 17:41:44 -07:00
db3afb18df Merge pull request #1164 from jonboulle/1164_dirs
Unable to create directories
2014-09-24 17:16:04 -07:00
f196276ca6 Merge pull request #1170 from jonboulle/version
main: add version flag
2014-09-24 17:14:13 -07:00
fa762e6b25 etcdhttp: process dir parameter in request 2014-09-24 17:11:28 -07:00
c82309d2b4 main: add version flag 2014-09-24 17:08:42 -07:00
98561f6b5d Merge pull request #1158 from jonboulle/1158_unset_ttl
"unsetting" a TTL fails
2014-09-24 17:01:50 -07:00
9ba35bc95e Merge pull request #1163 from jonboulle/1163_doc
Messages in documentation need updating
2014-09-24 17:01:18 -07:00
20ac7d6732 Merge pull request #1157 from jonboulle/1019_etcdserver
etcdserver: extend storeRecorder to test parameters
2014-09-24 16:58:57 -07:00
f603d90775 Merge pull request #1167 from bcwaldon/move-flag-code
Move flag-related code into new pkg package
2014-09-24 16:57:48 -07:00
42957815d3 Merge pull request #1168 from bcwaldon/moar-flags
etcd: deprecate remaining v0.4.6 flags
2014-09-24 16:51:33 -07:00
48e8ea1569 etcd: sort list of deprecated flags 2014-09-24 16:49:11 -07:00
2b52384e7e etcdhttp: allow empty values for TTL parameter 2014-09-24 16:44:55 -07:00
a6b7f4e5ea etcd: deprecate remaining v0.4.6 flags 2014-09-24 16:44:32 -07:00
4a65813a66 test: alphabetize list of testable packages 2014-09-24 16:37:46 -07:00
e30c1eeefd pkg: move SetFlagsFromEnv to pkg package 2014-09-24 16:29:47 -07:00
b0617be7e3 etcd: rewrite SetFlagsFromEnv test to use custom flagset 2014-09-24 16:27:00 -07:00
f7c353a703 etcd: export SetFlagsFromEnv 2014-09-24 16:12:06 -07:00
18c300f80c etcd: pass flagset into setFlagsFromEnv 2014-09-24 16:11:47 -07:00
314c13a8f0 pkg: move DeprecatedFlag to new package 2014-09-24 16:09:37 -07:00
002ace2403 etcd: remove unnecessary flag desc 2014-09-24 16:03:53 -07:00
13c20b1b64 docs: update error codes in api document 2014-09-24 15:59:40 -07:00
c2ced7dc70 etcdserver: add tests for Set and Dir=true/false 2014-09-24 15:57:06 -07:00
ef0ba361df etcdhttp: add test for expiration time 2014-09-24 15:46:40 -07:00
5059062275 Merge pull request #1166 from bcwaldon/override-usage
etcd: correctly override flag.Usage
2014-09-24 15:42:28 -07:00
69fba03fc1 etcd: correctly override flag.Usage 2014-09-24 15:39:06 -07:00
510213b1c2 Merge pull request #1159 from jonboulle/1159_prevvalue_required
`prevValue` ignored when no value provided
2014-09-24 15:30:25 -07:00
2e2cd12407 etcdhttp: disallow empty prevValue fields 2014-09-24 15:21:18 -07:00
f27b4cbbce store: add etcd-index tests 2014-09-24 15:04:32 -07:00
a45d490598 Merge pull request #1146 from jonboulle/1146_protobuf
script protobuf generation
2014-09-24 14:34:49 -07:00
c28fd92d10 etcdserver: extend storeRecorder to test parameters 2014-09-24 14:33:51 -07:00
b15fefa8ea store: use nextIndex where necessary 2014-09-24 14:07:52 -07:00
bbbd5fd5ec Merge pull request #1150 from bcwaldon/deprecated-flags
Slurp up deprecated flags, logging a warning
2014-09-24 14:04:30 -07:00
bcedef83d3 etcd: override flag usage func
No need to communicate deprecated flags, so prevent them from getting
printed in the usage info.
2014-09-24 13:52:58 -07:00
c8c55aa378 scripts: consolidate and standardize protobuf generation 2014-09-24 13:45:00 -07:00
1f736263b2 Merge pull request #1149 from unihorn/137
raft: stop tickElection when the node is not in peer list
2014-09-24 13:20:20 -07:00
ec1df42d04 Merge pull request #1138 from jonboulle/1138_timeout
etcdserver: handle watch timeouts and streaming
2014-09-24 12:39:34 -07:00
784d7ac680 etcd: warn on deprecated flags 2014-09-24 12:14:11 -07:00
a9caa24f8a etcdhttp: add test for streaming watches 2014-09-24 11:27:36 -07:00
ddc30c0a33 etcdserver: use Context to communicate timeout, and add tests 2014-09-24 11:27:36 -07:00
172a32e5e3 etcdserver: correct timeout and streaming handling
This reintroduces the 'stream' parameter to support long-lived watch
sessions. These sessions respect a server timeout (set to 5 minutes by
default).
2014-09-24 11:27:36 -07:00
7aaaf49fee Merge pull request #1148 from ThomasAlxDmy/master
Add link for etcd-backup
2014-09-24 11:17:06 -07:00
1ca03d8e9d raft: move logic to separate func 2014-09-24 10:23:44 -07:00
b07be74a82 raft: stop tickElection when the node is not in peer list
This prevents the bug like this:
1. a node sends join to a cluster and succeeds
2. it starts with empty peers and waits for sync, but it have not
received anything
3. election timeout passes, and it promotes itself to leader
4. it commits some log entry
5. its log conflicts with the cluster's
2014-09-23 23:15:02 -07:00
af01e11a5b Added link for etcd-backup 2014-09-23 17:07:17 -07:00
f34e37f68f Merge pull request #1134 from jonboulle/1134_flaky_test
flaky test TestSyncTriggerDeleteExpriedKeys
2014-09-23 16:13:19 -07:00
e97134e767 etcdserver: eliminate race in sync test 2014-09-23 15:49:20 -07:00
b85ad9bbc2 Merge pull request #1082 from jonboulle/1082_x_etcd_index
X-Etcd-Index returns the node's modifiedIndex
2014-09-23 15:14:21 -07:00
290b3915c2 Merge pull request #1144 from bcwaldon/TLS
Simplify TLS testing in transport pkg
2014-09-23 14:55:42 -07:00
03152004d7 Merge pull request #1145 from coreos/fix_panic
raft: node ignores unexpected local messages receiving from network
2014-09-23 14:11:56 -07:00
25c2768b8f raft: node ignores unexpected local messages receiving from network 2014-09-23 13:50:43 -07:00
dcdc7913c0 Merge pull request #1100 from unihorn/129
raft: add Configure
2014-09-23 13:50:10 -07:00
a299e92dfa transport: fake out certificate parser
The work being done in the tests is completely wasted, as we do not
need to test the udnerlying x509 library. Faking out the parser function
allows the tests to run much faster without having to carry massive
fixtures, either.
2014-09-23 13:49:32 -07:00
4d68c933d1 Merge pull request #1142 from bcwaldon/TLS
TLS: peer server/transport and proxy transport
2014-09-23 13:19:13 -07:00
c6cb635e01 etcdserver: refine comments of config change tests 2014-09-23 13:03:30 -07:00
4649a28097 transport: exercise TLSInfo.ClientConfig & ServerConfig 2014-09-23 12:52:59 -07:00
6ac4aea2bf transport: test TLSInfo.Empty() 2014-09-23 12:19:32 -07:00
3c4b155395 transport: spot-check NewTransport 2014-09-23 12:14:50 -07:00
bc7b0108dc raft: ConfigChange -> ConfChange 2014-09-23 12:02:44 -07:00
73504dca41 proxy: determine scheme based on TLSClientConfig 2014-09-23 11:01:58 -07:00
1ea3197feb proxy: pass addrs and scheme into newDirector 2014-09-23 11:01:55 -07:00
99e9f561ee etcdserver: add Peers.Addrs 2014-09-23 10:56:41 -07:00
b94d0281d4 etcd: use TLS in proxy transport 2014-09-23 10:19:01 -07:00
10220335f7 etcdserver: determine scheme based on TLSClientConfig 2014-09-23 10:14:12 -07:00
e19b0442f8 etcdserver: pass scheme into send 2014-09-23 10:14:11 -07:00
fb7968d704 etcdserver: Peers.Pick returns just an addr 2014-09-23 10:09:34 -07:00
27813599a1 etcd: wire up peer TLS flags 2014-09-23 10:03:17 -07:00
342ea18239 transport: build TLS config in NewTransport 2014-09-23 10:03:17 -07:00
e880dd41f2 transport: add TLSInfo.ClientConfig 2014-09-23 10:03:16 -07:00
0c7351c309 etcd: manually construct HTTP client for peer communication 2014-09-23 10:03:14 -07:00
5470a6d3d6 etcdserver: pass http client into Sender 2014-09-23 10:02:23 -07:00
db12e5704b transport: no need to set RootCAs in TLSInfo.ServerConfig 2014-09-23 09:04:26 -07:00
b754406f10 Merge pull request #1140 from bcwaldon/TLS
client server TLS
2014-09-23 08:04:50 -07:00
e89f6efd20 Merge pull request #1122 from coreos/discovery
discovery
2014-09-22 23:48:05 -07:00
d92931853e raft: Config -> ConfigChange
Configure -> ProposeConfigChange
AddNode, RemoveNode -> ApplyConfigChange
2014-09-22 23:39:53 -07:00
87df94dbd4 Merge pull request #1130 from jonboulle/1130_wal_log
wal logs saveState too frequently
2014-09-22 23:20:26 -07:00
08ebb05335 discovery: add a overfull case for waitNodes 2014-09-22 19:51:31 -07:00
fdfaf07c46 discovery: address comments 2014-09-22 19:51:31 -07:00
a35df0ad7c discovery: remove redundant comment for createSelf 2014-09-22 19:51:31 -07:00
17068c5110 discovery: key config -> key _config 2014-09-22 19:51:31 -07:00
6081311db5 discovery: store size inside /[cluster]/config 2014-09-22 19:51:31 -07:00
00d1daaf1e discovery: add tests for sortableNodes 2014-09-22 19:51:31 -07:00
2fd5a9863b discovery: add d.selfKey() 2014-09-22 19:51:31 -07:00
b8eb21c027 discovery: add fast path 2014-09-22 19:51:31 -07:00
2b623cf0fa discovery: init commit 2014-09-22 19:51:31 -07:00
cf4af47f7e wal: remove logging messages
These are noisy and unhelpful by default. We can re-add if at some point
we reintroduce debugging/multiple log levels.
2014-09-22 19:08:47 -07:00
5441c6aa54 etcdserver: correct X-Etcd-Index header
This adds an EtcdIndex field to store.Event and uses that as the header
instead of the node's modifiedIndex. To facilitate this in a non-racy
way, we set the EtcdIndex while holding the lock.
2014-09-22 18:56:12 -07:00
f2d3d90b60 etcd: wire up client TLS flags 2014-09-22 17:58:31 -07:00
17459c7bfc transport: wrap net.Listener with TLSInfo 2014-09-22 17:58:30 -07:00
a782a1a7d1 etcd: manually construct listeners 2014-09-22 17:57:44 -07:00
a4cca35e9d Merge pull request #1137 from philips/gofmt-hack
test: stop running gofmt ./
2014-09-22 17:28:48 -07:00
fd48f3f2a4 test: stop running gofmt ./
gofmt ./ will recursively check the .git directory. So use *.go for
gofmt instead
2014-09-22 17:28:31 -07:00
ec8f493fde raft: refine comments for Configure 2014-09-22 15:44:47 -07:00
dc36ae7058 raft: use pb.Config instead of []byte for Configure 2014-09-22 15:44:47 -07:00
4203569da2 etcdserver: use ConfigType in protobuf 2014-09-22 15:44:46 -07:00
b82d70871f raft: use EntryType in protobuf 2014-09-22 15:44:46 -07:00
b801f1affe raft: refine comment for raft.pendingConf 2014-09-22 15:44:46 -07:00
abdb2cad15 etcdserver: Config.Id -> Config.ID 2014-09-22 15:44:46 -07:00
aaffb9eb78 etcdserver: add AddNode, RemoveNode
AddNode and RemoveNode is used to propose config change to the cluster.
If succeeds, it will add/remove node from the cluster.
2014-09-22 15:44:46 -07:00
ff6705b94b raft: add Configure, AddNode, RemoveNode
Configure is used to propose config change. AddNode and RemoveNode is
used to apply cluster change to raft state machine. They are the
basics for dynamic configuration.
2014-09-22 15:43:13 -07:00
ceab948831 Merge pull request #1131 from bcwaldon/1129
etcdserver: init time.Time only if Expiration > 0
2014-09-22 15:39:32 -07:00
7d1126fb35 etcdserver: init time.Time only if Expiration > 0
golang's concept of "zero" time does not align with the zero value of
a unix timestamp. Initializing time.Time with a unix timestamp of 0
makes time.Time.IsZero fail. Solve this by initializing time.Time only
if we care about the time.
2014-09-22 15:31:31 -07:00
9711e70980 Merge pull request #1132 from jonboulle/fix_store
store: fix recursive/dir arguments
2014-09-22 14:54:37 -07:00
e08df4c8d2 store: fix recursive/dir arguments 2014-09-22 14:44:01 -07:00
34380ab096 Merge pull request #1013 from jonboulle/1013_cors
Add CORS support
2014-09-22 11:50:03 -07:00
e27b80643d Merge pull request #1109 from coreos/client
min client for discovery
2014-09-21 10:31:52 -07:00
49cc76d33b client: min client for discovery 2014-09-21 10:29:52 -07:00
0335422e81 Merge pull request #1098 from unihorn/131
fix sync tests
2014-09-20 17:47:31 -07:00
8ba801ec06 etcdserver: fix sync tests
This is to fix possible testing failures caused by sync tests.

Changes:
1. Get rid of time sleep operations, which introduces uncertainty.
2. Use fake Store.
2014-09-20 17:41:30 -07:00
ddfcb67ce3 Merge pull request #1118 from unihorn/133
etcdserver: add TestRecvSnapshot
2014-09-20 17:10:01 -07:00
fac38aad33 etcdserver: add TestRecvSlowSnapshot 2014-09-19 15:09:04 -07:00
1a36b53f14 main: add CORS support 2014-09-19 14:58:35 -07:00
b8e59a3c6a etcdserver: not push ready to readyNode automatically 2014-09-19 14:40:11 -07:00
f2ebd64a1b *: add testutil pkg 2014-09-19 14:32:38 -07:00
4eb156a324 Merge pull request #1104 from unihorn/132
etcdserver: add TestDoBadLocalAction
2014-09-19 09:30:34 -07:00
d5988c3ec2 etcdserver: add TestDoBadLocalAction 2014-09-19 09:22:26 -07:00
44ab66d858 etcdserver: add TestRecvSnapshot 2014-09-18 23:19:00 -07:00
da9956df11 Merge pull request #1107 from bcwaldon/Procfile
Isolate ./build from environment
2014-09-18 22:12:28 -07:00
24aa4d9bd9 Merge pull request #1117 from bcwaldon/prevExist
*: s/prevExists/prevExist/
2014-09-18 21:45:37 -07:00
91003cb994 *: s/prevExists/prevExist/ 2014-09-18 21:25:08 -07:00
517e4271e1 Merge pull request #1113 from jonboulle/unix
main: add address validation
2014-09-18 16:38:42 -07:00
f0789e7349 main: add address validation for bind-addr flag 2014-09-18 16:38:14 -07:00
6449387ccb Merge pull request #1110 from bcwaldon/proxy-readonly
Readonly proxy mode
2014-09-18 15:08:46 -07:00
98221cf6c0 Merge pull request #1112 from bcwaldon/update-link
CONTRIBUTING: update link to style guide
2014-09-18 15:06:59 -07:00
6e782b0e63 etcd: add -proxy=readonly 2014-09-18 15:05:33 -07:00
fd1f46313a proxy: add readonly handler 2014-09-18 15:05:30 -07:00
5b4fe8a558 CONTRIBUTING: update link to style guide 2014-09-18 15:03:55 -07:00
ec0e9c6e6a Procfile: run etcd from bin/etcd 2014-09-18 14:36:29 -07:00
81585716dc build: ignore calling environment 2014-09-18 14:36:26 -07:00
9cc114df36 Merge pull request #1106 from bcwaldon/proxy-flag
Replace -proxy-mode=<bool> with -proxy=<enum>
2014-09-18 14:31:00 -07:00
afce2948d2 etcd: replace -proxy-mode with -proxy 2014-09-18 14:30:22 -07:00
fcf50e756d test: actually test main pkg 2014-09-18 11:01:00 -07:00
df70f653a4 Merge pull request #1099 from unihorn/128
wal: OpenFromIndex fails if it cannot find previous index
2014-09-17 16:00:51 -07:00
655efca308 Merge pull request #1092 from jonboulle/dumb_gen_id
GenID: use a fast prnd
2014-09-17 16:00:25 -07:00
29f6d8a9e6 wal: add ErrFileNotFound and ErrIndexNotFound 2014-09-17 15:58:06 -07:00
a42d52482c etcdserver: use simple PRNG for GenID 2014-09-17 15:29:14 -07:00
1c544667ff Merge pull request #1040 from jonboulle/1040_config_env_vars
etcd needs to be configurable through env vars
2014-09-17 15:28:02 -07:00
7160b5ae26 wal: OpenFromIndex fails if it cannot find previous index
Example:
We save entry 1, 2, 3 to WAL.
If we try to open 100, it should fail.
2014-09-17 15:07:07 -07:00
f2e92d9140 Merge pull request #1097 from unihorn/130
etcdserver: use Action to fix possible data race
2014-09-17 14:28:37 -07:00
6b03135527 Merge pull request #1093 from jonboulle/node_interface
Create Node Interface
2014-09-17 14:20:42 -07:00
b66a40495d raft: introduce Node interface 2014-09-17 14:18:56 -07:00
844897360c main: add test for setting flags 2014-09-17 14:12:04 -07:00
ba851b2eca Merge pull request #1095 from coreos/nsnap
WIP *: init for on disk snap support
2014-09-17 14:02:48 -07:00
ab61a8aa9a *: init for on disk snap support 2014-09-17 13:56:12 -07:00
0ad0e24a86 etcdserver: use Action to fix possible data race 2014-09-17 13:48:03 -07:00
40c19e525c main: retrieve config from env vars 2014-09-17 12:15:39 -07:00
f17391a72b Merge pull request #1090 from jonboulle/1090_multiple_sockets
listen on multiple sockets
2014-09-16 15:29:05 -07:00
f786de13d0 Merge pull request #1081 from unihorn/125
make TTL work
2014-09-16 15:14:44 -07:00
a91d745c46 etcdserver: refine printout 2014-09-16 14:54:22 -07:00
936ecd097a main: listen for client requests on multiple sockets 2014-09-16 14:46:51 -07:00
757f400f5d Merge pull request #1091 from jonboulle/993_default_port
main: switch back to default ports for now
2014-09-16 14:37:57 -07:00
6d4c79b157 main: switch back to default ports 2014-09-16 14:36:59 -07:00
47b9b5520f Merge pull request #1010 from jonboulle/1010_separate_ports
separate ports used for client/server communication
2014-09-16 14:36:24 -07:00
de21c39ca5 raft: isStateEqual -> isHardStateEqual, IsEmptyState -> IsEmptyHardState 2014-09-16 13:55:00 -07:00
023dc7cba2 etcdserver: add SYNC request 2014-09-16 13:42:03 -07:00
38c690b155 Procfile: update to new address flags 2014-09-16 13:19:16 -07:00
67e57ffca4 etcdserver: split out client and server listeners 2014-09-16 11:36:37 -07:00
7a63e52901 Merge pull request #1088 from jonboulle/peers_tests
etcdserver/etcdhttp: add simple test for peers send
2014-09-15 22:13:46 -07:00
30b70e18c5 etcdserver/etcdhttp: add simple test for peers send 2014-09-15 21:10:00 -07:00
6f17fa6c90 Merge pull request #1087 from jonboulle/tests
etcdserver/etcdhttp: add tests for serveKeys
2014-09-15 20:52:03 -07:00
a7ec09c877 etcdserver/etcdhttp: use empty node for test 2014-09-15 20:34:14 -07:00
f846c5286a etcdserver/etcdhttp: fix comment, simplify resServer 2014-09-15 19:01:40 -07:00
27cf7747ea etcdserver/etcdhttp: add tests for serveKeys 2014-09-15 18:33:10 -07:00
43acdef660 Merge pull request #1086 from jonboulle/serve_raft_test
etcdserver/etcdhttp: add test for serveRaft
2014-09-15 16:13:08 -07:00
7ac3b32de6 etcdserver: change raft endpoint to send StatusNoContent 2014-09-15 16:10:31 -07:00
3e2c160eed etcdserver/etcdhttp: add test coverage for serveRaft 2014-09-15 16:07:06 -07:00
699bc50365 Merge pull request #1071 from jonboulle/serve_mux
etcdserver/etcdhttp: switch to using http.ServeMux
2014-09-15 16:05:58 -07:00
e04c028d64 etcdserver/etcdhttp: switch to using http.ServeMux 2014-09-15 16:05:20 -07:00
763c276d27 Merge pull request #1061 from jonboulle/server_interface
etcdserver: introduce Server interface
2014-09-15 15:36:11 -07:00
8a5ab2ec06 etcdserver: introduce Server interface
This changes etcdserver.Server to an interface, with the former Server
(now "EtcdServer") becoming the canonical/production implementation.
This will facilitate better testing of the http server et al with mock
implementations of the interface.
It also more clearly defines the boundary for users of the Server.
2014-09-15 15:11:01 -07:00
96624b1129 Merge pull request #1085 from unihorn/127
raft: remove unused raftpb.LastIndex
2014-09-15 15:09:09 -07:00
35ae488120 Merge pull request #1069 from jonboulle/methods
etcdhttp: check method for every endpoint, add tests
2014-09-15 14:56:47 -07:00
d9cfc35bed etcdhttp: check method for every endpoint, add tests 2014-09-15 14:56:43 -07:00
cc8d8f2102 raft: remove unused raftpb.LastIndex 2014-09-15 14:34:23 -07:00
07648f1f25 Merge pull request #1084 from unihorn/126
raft: remove unused return
2014-09-15 13:49:49 -07:00
9607665323 raft: remove unused return 2014-09-15 13:22:21 -07:00
9bf2c2ed9d Merge pull request #1052 from unihorn/121
server: add unit tests
2014-09-15 13:20:50 -07:00
6cd4434ff3 server: add unit tests
Make test coverage >= 90%
2014-09-15 13:16:48 -07:00
bc5791af11 Merge pull request #1077 from coreos/wal_cut
wal: cut(i uint64) -> cut
2014-09-15 10:37:41 -07:00
69f2d5c590 wal: refactor 2014-09-15 10:35:36 -07:00
77fbd2610c wal: add comment 2014-09-15 10:17:21 -07:00
1d09c25f5f wal: add walName function; cleanup test 2014-09-15 10:17:21 -07:00
cec1956b8f wal: better comment 2014-09-15 10:14:18 -07:00
8e0ee1cc5e wal: remove wrong comment for cut 2014-09-15 10:14:18 -07:00
1164c4b83d wal: recover writting seq 2014-09-15 10:14:18 -07:00
21860bc017 wal: cut(i uint64) -> cut 2014-09-15 10:14:18 -07:00
f9c12e2053 Merge pull request #1075 from coreos/fix_heartbeat
raft: fix heartbeat
2014-09-15 10:04:12 -07:00
38c074cb05 Merge pull request #1067 from unihorn/122
raft: write entry 0 into log
2014-09-15 10:00:49 -07:00
21d116d3e1 raft: fix heartbeat 2014-09-15 09:58:22 -07:00
140fd6d6c4 raft: restart using last written entry also 2014-09-15 09:56:33 -07:00
a9af70c52b raft: write entry 0 into log 2014-09-15 09:55:52 -07:00
29f9372370 Merge pull request #1080 from coreos/check_id
main: check node id is not noneid
2014-09-14 23:28:39 -07:00
e7ea6a374a main: check node id is not noneid 2014-09-14 23:28:11 -07:00
ff7f340bba Merge pull request #1063 from coreos/node_run_test
raft: test node block proposal
2014-09-14 22:46:50 -07:00
ed57a7b561 Merge pull request #1079 from philips/fix-procfile-non-zero
Procfile: don't use a 0x0 peer
2014-09-14 22:40:16 -07:00
e6e1f2ff7d Procfile: don't use a 0x0 peer
0x0 peer id is no longer valid, don't use it.
2014-09-14 22:39:46 -07:00
e085cc4e06 Merge pull request #1042 from unihorn/118
raft: set none to be 0
2014-09-14 22:32:37 -07:00
2927cc6e3b raft: refine plus assignment 2014-09-14 22:07:32 -07:00
03f0ed657a Merge pull request #1076 from coreos/test_race
test: test with race by default
2014-09-14 21:42:45 -07:00
10c9f7389b test: turn off go 1.2 testing on travis 2014-09-14 21:40:09 -07:00
e9b790e27b Merge pull request #1078 from unihorn/124
raft: remove unused field
2014-09-14 17:24:51 -07:00
da575c46fa raft: remove unused field 2014-09-14 13:08:35 -07:00
79b8153eac test: test with race by default 2014-09-14 13:08:14 -07:00
1037e7ce55 Merge pull request #1055 from bcwaldon/proxy-monster
proxy mode, pt II
2014-09-12 23:01:24 -07:00
a155f0bda6 proxy: rewrite stdlib ReverseProxy
The ReverseProxy code from the standard library doesn't actually
give us the control that we want. Pull it down and rip out what
we don't need, adding tests in the process.

All available endpoints are attempted when proxying a request. If a
proxied request fails, the upstream will be considered unavailable
for 5s and no more requests will be proxied to it. After the 5s is
up, the endpoint will be put back to rotation.
2014-09-12 22:56:23 -07:00
54b9c55af3 etcdserver: remove useless sleep
etcdserver.Do will block until there exists leader
2014-09-12 13:01:33 -07:00
34db45a948 raft: set propc, readyc to be nil before run loop 2014-09-12 12:57:50 -07:00
ccee264b7d raft: move defer after run 2014-09-12 12:28:15 -07:00
45f56a5377 raft: forceGoSched -> forceGosched 2014-09-12 12:27:18 -07:00
df253a2b14 Merge pull request #1066 from philips/add-raft-doc
add-raft-doc
2014-09-12 12:21:53 -07:00
5c884c7797 raft: better comment and naming 2014-09-12 12:18:30 -07:00
0c09862494 raft: add isStateEqual test 2014-09-12 11:59:32 -07:00
f9ef453894 raft: test contain updates 2014-09-12 11:51:25 -07:00
b261a5edc1 raft: test node block proposal 2014-09-12 11:51:25 -07:00
30289dad5c raft: more descriptive panic info 2014-09-12 11:39:53 -07:00
072a21782e Merge pull request #1049 from unihorn/120
raftLog: enhance check in compact
2014-09-12 11:35:41 -07:00
d31443f5a3 raftLog: compact applied entries only
compact MUST happen on entries that have been applied, or
1. it may screw up the log by setting wrong commitIndex
2. discard unapplied entries
2014-09-12 11:34:08 -07:00
6edb471d58 Merge pull request #1045 from unihorn/119
wal: change wal filename format
2014-09-12 11:31:46 -07:00
2030ca202f wal: change wal filename format
Make raftIndex section to be expected raftIndex of next entry.

It makes filename more intuitive and straight-forward.

The commit also adds comments for filename format.
2014-09-12 11:24:28 -07:00
1a0ad54d3e Merge pull request #1059 from coreos/node_test
test node step
2014-09-12 10:23:53 -07:00
98f9ee3613 raft: add useful comments
These comments were things I learned about the units, use case or
meaning of various fields and functions in the raft package.
2014-09-12 09:58:04 -07:00
58b8610024 raft: add a doc.go file
Explain the package from a high level.
2014-09-12 09:57:48 -07:00
50c1a34f78 Merge pull request #1060 from jonboulle/peers
etcdserver/etcdhttp: split out peers and add tests
2014-09-11 23:53:54 -07:00
073411f23f raft: remove vote with default value 2014-09-11 23:17:13 -07:00
dc1357afa9 raft: use raft helper function in testing 2014-09-11 19:26:29 -07:00
0e0fc2bd24 raft: use [1,n] as address list for raft state machines 2014-09-11 19:23:05 -07:00
84c2bd0b7d raft: set From of local message to be local id 2014-09-11 19:18:47 -07:00
20776f1947 etcdserver: use int64slice directly 2014-09-11 19:17:07 -07:00
281afa74ee raft: comments for TestNodeStep 2014-09-11 19:00:40 -07:00
e18b8c12be etcdserver: switch to proper int64Slice for sorting 2014-09-11 17:28:08 -07:00
81b5967e0a Merge pull request #1054 from philips/add-wal-doc
wal: add a doc file
2014-09-11 17:18:53 -07:00
eb1dcb324c raft: test node step 2014-09-11 17:10:36 -07:00
5f66b35852 etcdserver/etcdhttp: split out peers and add tests 2014-09-11 16:53:55 -07:00
c03798f99b Merge pull request #1011 from jonboulle/clean_http
Validate HTTP API user input
2014-09-11 15:31:25 -07:00
786982d8e5 simplify 2014-09-11 15:13:56 -07:00
52ddd389ff etcdhttp: use Form values over query parameters 2014-09-11 15:12:31 -07:00
40341b488c Merge pull request #1058 from philips/grammar-fix
raft: log comment grammar fix
2014-09-11 14:06:33 -07:00
3bc4b2db12 raft: log comment grammar fix 2014-09-11 13:59:50 -07:00
f7f65ec464 Merge pull request #1056 from coreos/test
test: test wait pkg
2014-09-11 13:56:51 -07:00
884c702512 wal: wal.OpenFromIndex -> wal.OpenAtIndex
The first entry read from the wal should be the index provided. This
name makes that more clear.
2014-09-11 12:52:06 -07:00
413b6a59ff wal: add more explanation to wal filename format 2014-09-11 12:51:50 -07:00
9bd9d88a9d test: sort the pkgs 2014-09-11 12:48:18 -07:00
0feb153034 test: test wait pkg 2014-09-11 12:38:43 -07:00
7c03704b19 etcdserver: restructure to hopefully simplify
Obviate parseBool helper, define emptyReq locally to parseRequest,
have writeEvent return an error which gets logged
2014-09-11 08:50:41 -07:00
04abd5603f wal: add a doc file 2014-09-10 20:40:20 -07:00
71a1c1aa84 Merge pull request #1053 from philips/proxy-in-procfile
Procfile: add a proxy
2014-09-10 18:47:53 -07:00
1ee8392a8f Procfile: add a proxy
Woo! Now we have a cluster and a proxy and etcdctl works when running
the procfile by default!
2014-09-10 18:47:32 -07:00
338ca6050e Merge pull request #1048 from bcwaldon/proxy-monster
proxy mode
2014-09-10 18:09:50 -07:00
7415d53020 proxy: add proxy-mode functionality to etcd daemon 2014-09-10 18:06:56 -07:00
e5a482266f proxy: introduce director
The director class drives an httputil.ReverseProxy. This is used when
etcd is deployed in proxy mode.
2014-09-10 18:04:03 -07:00
a3334eed23 main: break out startEtcd func 2014-09-10 18:04:02 -07:00
bafe960dba etcdhttp: add Peers.Endpoints 2014-09-10 18:02:02 -07:00
2b39ee1bb3 Merge pull request #1051 from bcwaldon/Id
*: s/Id/ID/
2014-09-10 16:13:04 -07:00
c251304068 *: s/Id/ID/
golang convention dictates that the individual characters in an
abbreviation should all have the same case. Use ID instead of Id.

The protobuf generator still generates code that does not meet
this convention, but that's a fight for another day.
2014-09-10 16:09:08 -07:00
e2d01eff35 etcdserver: switch to using etcd.Error 2014-09-10 15:46:13 -07:00
21c214ac03 Merge pull request #1050 from jonboulle/ignore
gitignore: ignore vim swap files
2014-09-10 15:09:45 -07:00
2342402434 etcdserver: address style comments 2014-09-10 14:14:14 -07:00
255e62dcdd gitignore: ignore vim swap files 2014-09-10 12:06:28 -07:00
bed63cddf7 etcdhttp: add tests for ParseBool/ParseUint64 helpers 2014-09-10 12:05:36 -07:00
e736a11ac4 etcdhttp: perform validation of query parameters
Add basic input validation of all query parameters supported by
serveKeys. Also restructures etcdhttp a bit to better facilitate
testing.

Test coverage is slightly improved.
2014-09-10 12:01:58 -07:00
3d272c2686 Merge pull request #1044 from unihorn/119
http: check err return from http.NewRequest
2014-09-09 20:48:56 -07:00
51e4bbfeb0 http: check err return from http.NewRequest 2014-09-09 20:46:56 -07:00
4e31bb308d raft: use const firstId to track ids 2014-09-09 19:07:20 -07:00
8c58684fb7 etcdserver: fix testServer 2014-09-09 16:59:42 -07:00
bac88c047b Merge pull request #1039 from philips/Procfile-note
Procfile: add a comment on how to use this
2014-09-09 16:49:08 -07:00
a9c288aadc Procfile: add a comment on how to use this 2014-09-09 16:48:25 -07:00
b50f96e2e1 raft: set none to be 0 2014-09-09 16:47:21 -07:00
70443adc8d Merge pull request #1038 from philips/vegan-etcd
*: stop using 0xBEEF
2014-09-09 16:43:54 -07:00
24fd126822 *: stop using 0xBEEF
Using 0xBEEF is annoying in examples because it makes it makes it look
like the user can use ascii or something. In the Procfile use
0x0,0x1,0x2,etc and use 0xBAD0 in test.
2014-09-09 16:40:51 -07:00
d9b35470a1 Merge pull request #1023 from unihorn/117
etcdhttp: add /v2/admin/machines endpoint
2014-09-09 16:23:23 -07:00
01871e7c29 http: use sort.Strings 2014-09-09 16:20:02 -07:00
68aa114301 Merge pull request #1035 from coreos/no_copy
raft: do not need to copy committed entries
2014-09-09 16:07:20 -07:00
d519491545 http: add TODO for peers var 2014-09-09 15:55:42 -07:00
7ac3afc02b Merge pull request #1037 from jonboulle/travis
build: add travis configuration
2014-09-09 15:50:21 -07:00
961a61d708 http: add allow func
For further extendability
2014-09-09 15:45:34 -07:00
0241b8ba9a build: add travis configuration 2014-09-09 15:44:41 -07:00
4087fa5c7a http: allow GET, HEAD for /v2/machines 2014-09-09 15:22:52 -07:00
0c1d1b7aeb etcdhttp: add /v2/machines endpoint 2014-09-09 15:00:29 -07:00
b8f2db36dd Merge pull request #1036 from philips/gitignore-etcd-data
gitignore: add _etcd_data directory ignores
2014-09-09 14:30:38 -07:00
6b647fd481 gitignore: add _etcd_data directory ignores
After 9a57d1067d we want to ignore all of
these _etcd_data directories.
2014-09-09 14:30:09 -07:00
e5cc58c179 Merge pull request #1032 from jonboulle/fix_tests
scripts: use bash for build script
2014-09-09 14:21:51 -07:00
eaffaacf5e raft: do not need to copy committed entries 2014-09-09 14:09:30 -07:00
1a677164be Merge pull request #1026 from coreos/fix_node
Fix node
2014-09-09 13:41:38 -07:00
a5e72258d2 Merge pull request #1029 from jonboulle/docker
scripts: remove non-functional Dockerfile
2014-09-09 12:15:23 -07:00
f7baea7406 Merge pull request #1017 from jonboulle/api_tests
HTTP API test coverage, pt 1
2014-09-09 12:12:56 -07:00
676b5be972 raft: use IsEmptyState 2014-09-09 11:58:15 -07:00
800de8e3bf scripts: use bash for build script 2014-09-09 11:34:55 -07:00
44acd57ea4 etcdserver: remove superfluous context 2014-09-09 11:18:25 -07:00
07ce8bc4bc etcdserver: split out functional test to new package 2014-09-09 11:18:24 -07:00
8473f3bf52 api: address review comments 2014-09-09 11:17:53 -07:00
c78239a629 etcdserver: add test coverage for parseRequest 2014-09-09 11:17:53 -07:00
b33b85870d etcdserver: add waitForEvent and encodeResponse coverage 2014-09-09 11:17:53 -07:00
f7444ff300 store: convert Watch to interface 2014-09-09 11:17:53 -07:00
7c8b1a553f Merge pull request #1028 from jonboulle/build
scripts: add build, cover and update test
2014-09-09 11:12:35 -07:00
6f06923e96 wal: test save empty state 2014-09-09 10:41:15 -07:00
ac0443bc89 raft: remove extra code 2014-09-09 10:16:39 -07:00
df259e5878 raft: add a newReady helper function 2014-09-09 10:16:39 -07:00
90c0db3d42 wal: do not save empty state 2014-09-09 10:15:29 -07:00
c28fef5fc4 raft: set ready.st to empty state, if current state is equal to previous state 2014-09-09 10:15:29 -07:00
b0d865e845 raft: record previous state is enough 2014-09-09 10:15:29 -07:00
0060c0749a raft: prev should be set only when we sucessfully send out rd to the channel 2014-09-09 10:15:29 -07:00
57ea72d3c4 Merge pull request #1030 from jonboulle/assorted
*: fix a few small issues identified by go vet
2014-09-09 07:46:45 -07:00
9997c9488a *: fix a few small issues identified by go vet 2014-09-08 23:52:36 -07:00
55b4267c30 Merge pull request #1001 from coreos/sender
etcdhttp: non-blocking sender
2014-09-08 23:36:10 -07:00
e7ad45b064 Merge pull request #1027 from jonboulle/master
raft: remove unused alwaysreadyc
2014-09-08 23:35:58 -07:00
e1d5caa7e8 scripts: remove non-functional Dockerfile 2014-09-08 23:20:14 -07:00
491362f5db scripts: add build, cover and update test
This adds a build script that attempts to be as user friendly as
possible: if they have already set $GOPATH and/or $GOBIN, use those
environment variables. If not, create a gopath for them in this
directory. This should facilitate both `go get` and `git clone` usage.

The `test` script is updated, and the new `cover` script facilitates
easy coverage generation for the repo's constituent packages by setting
the PKG environment variable.
2014-09-08 23:09:49 -07:00
946e35c958 raft: remove unused alwaysreadyc 2014-09-08 22:21:03 -07:00
4f330a9ba2 Merge pull request #999 from coreos/fix_flag_set
etcdhttp: set peer should clean up previous values
2014-09-08 22:11:51 -07:00
908d326e22 etcdhttp: use peer.String() in test 2014-09-08 20:50:35 -07:00
378cadf073 etcdhttp: implment Peers.String() 2014-09-08 20:45:10 -07:00
f4a33dd6df etcdhttp: simple test for peer set cleanup 2014-09-08 17:04:36 -07:00
5c8839387d etcdhttp: set peer should clean up previous values 2014-09-08 16:38:01 -07:00
9215ebb6aa Merge pull request #992 from coreos/wal
wal
2014-09-08 16:24:28 -07:00
12c1300d48 Merge pull request #987 from philips/add-test-script
test: introduce test script
2014-09-08 16:23:15 -07:00
4a02a1a60c main: clarify fatal message for unexpect nodeid 2014-09-08 16:21:27 -07:00
91c52630b6 test: introduce test script
Why do this? `go test ./...` has a ton of annoying output:

```
?   	github.com/coreos/etcd	[no test files]
?   	github.com/coreos/etcd/crc	[no test files]
?   	github.com/coreos/etcd/elog	[no test files]
?   	github.com/coreos/etcd/error	[no test files]
ok  	github.com/coreos/etcd/etcdserver	0.267s
ok  	github.com/coreos/etcd/etcdserver/etcdhttp	0.022s
?   	github.com/coreos/etcd/etcdserver/etcdserverpb	[no test files]
ok  	github.com/coreos/etcd/raft	0.157s
?   	github.com/coreos/etcd/raft/raftpb	[no test files]
ok  	github.com/coreos/etcd/snap	0.018s
?   	github.com/coreos/etcd/snap/snappb	[no test files]
third_party/code.google.com/p/gogoprotobuf/proto/testdata/test.pb.go🔢
undefined: __emptyarchive__.Extension
ok  	github.com/coreos/etcd/store	4.247s
ok
github.com/coreos/etcd/third_party/code.google.com/p/go.net/context
2.724s
FAIL
github.com/coreos/etcd/third_party/code.google.com/p/gogoprotobuf/proto
[build failed]
ok
github.com/coreos/etcd/third_party/github.com/stretchr/testify/assert
0.013s
ok  	github.com/coreos/etcd/wait	0.010s
ok  	github.com/coreos/etcd/wal	0.024s
?   	github.com/coreos/etcd/wal/walpb	[no test files]
```

And we have no had to manually configure drone.io which I want to avoid:
https://drone.io/github.com/coreos/etcd/admin
2014-09-08 16:18:10 -07:00
f3348d6e13 raft: fix the comment in test 2014-09-08 16:16:58 -07:00
9a57d1067d main: make default data to have _data_etcd suffix 2014-09-08 16:15:18 -07:00
19235c8104 raft: refactor restart test 2014-09-08 16:10:13 -07:00
e4d2b2a06a Merge pull request #1022 from philips/add-ready-docs
raft: add doc for the Ready struct
2014-09-08 16:02:21 -07:00
145882244f raft: add doc for the Ready struct 2014-09-08 15:59:15 -07:00
54734b0903 main/wal: add a const for 0700 magic number 2014-09-08 15:45:58 -07:00
b094410066 wal: change entries->ents for consistency 2014-09-08 15:40:12 -07:00
adff0f3813 wal: named return values for ReadAll. 2014-09-08 15:36:25 -07:00
a3b6a646eb main: check the id read by from wal. 2014-09-08 15:31:11 -07:00
ffd198808e Merge pull request #1020 from jonboulle/master
docs: add note about version, remove old Vagrantfile
2014-09-08 15:09:27 -07:00
26a5aaec34 docs: add link to 32-bit issue 2014-09-08 15:08:07 -07:00
9180919a30 docs: remove build instructions from README 2014-09-08 14:57:16 -07:00
8cd1b3a4f2 docs: remove nonfunctional Vagrantfile and instructions 2014-09-08 14:49:35 -07:00
7caa33d819 docs: update README to warn about master branch 2014-09-08 14:45:38 -07:00
8eadc9b8a5 Merge pull request #1005 from philips/quorum-maintainers
MAINTAINERS: add Brandon Philips
2014-09-08 14:38:55 -07:00
0461c517e4 wal: clarify TODO 2014-09-08 13:57:35 -07:00
36730ca613 main: use -> using 2014-09-08 10:32:49 -07:00
138ac0b296 CONTRIBUTING: require two people to review on etcd 2014-09-07 19:46:06 -07:00
c7ff6d4410 MAINTAINERS: add Brandon Philips
Need to make sure we have a quorum :)
2014-09-07 19:45:34 -07:00
4f57330e29 Merge pull request #1003 from bmizerany/b-newemail
MAINTAINERS: change email
2014-09-07 19:26:29 -07:00
756b54b4c3 MAINTAINERS: change email 2014-09-07 17:53:52 -07:00
cbec48e8f6 etcdhttp: non-blocking sender 2014-09-05 13:54:40 -07:00
8c9d7e3e93 main: add a helper function for starting raft 2014-09-05 10:24:13 -07:00
0851a1fe7f main: better error msg 2014-09-05 10:02:46 -07:00
ee78890f22 main: use data-dir as the path to data directory flag 2014-09-05 09:56:25 -07:00
1a6e908971 *: add wal 2014-09-05 09:52:40 -07:00
769c043537 Merge pull request #995 from jonboulle/master
doc: update build instructions in README
2014-09-04 23:25:48 -07:00
4cc39e4517 doc: update build instructions in README 2014-09-04 23:21:21 -07:00
dd49b7a133 Merge pull request #988 from coreos/default_peers
etcdserver: add default peer
2014-09-04 22:33:18 -07:00
bca8f9e0ed etcdserver: change default id to 0x1 2014-09-04 22:26:42 -07:00
8e6c8e068d .: add MAINTAINERS file 2014-09-04 18:01:09 -07:00
8408acf995 .: explain the LGTM protocol 2014-09-04 17:08:21 -07:00
f41fac0719 Merge pull request #983 from marineam/doc
docs: fix documentation for HTTP read/write timeouts
2014-09-04 15:17:07 -07:00
5ddfe18cda Merge pull request #989 from coreos/fix_ignore_term
raft: fix ignore term
2014-09-04 14:36:56 -07:00
8dfa490e49 raft: fix ignore term 2014-09-04 14:33:51 -07:00
28102e536b Revert "build: add build script and update readme"
This reverts commit 2866a7488e.
2014-09-04 14:06:19 -07:00
08ad47fe1a Merge pull request #991 from jonboulle/master
build: add build script and update readme
2014-09-04 13:58:09 -07:00
2866a7488e build: add build script and update readme 2014-09-04 13:57:45 -07:00
25ee66c6c7 Merge pull request #990 from coreos/fix_node_test
Fix node test
2014-09-04 13:55:50 -07:00
e30505d33b raft: fix node test 2014-09-04 11:42:11 -07:00
d7a289ee41 raft: init prev: vote for none 2014-09-04 11:09:55 -07:00
c1e7a788cd etcdserver: add default peer 2014-09-04 10:11:32 -07:00
a7102d491b docs: fix documentation for HTTP read/write timeouts
The documentation for these options has been incorrect ever since their
addition via commit 084dcb55. This broke upgrading a CoreOS stable
cluster to 410.0.0 because the user was using the example TOML config
which contains:

    http_write_timeout = 10

This was silently ignored in the old stable version which predates the
addition of these options. After upgrading etcd began failing with:

    Type mismatch for 'config.Config.http_write_timeout': Expected float
    but found 'int64’.

Original issue: https://github.com/coreos/update_engine/issues/45
2014-09-03 23:36:36 -07:00
e393509879 Merge pull request #981 from coreos/v2prefix
etcdhttp: add v2prefix
2014-09-03 22:05:07 -07:00
e7eff1a975 Merge pull request #982 from coreos/fix_tick
Fix tick
2014-09-03 22:02:33 -07:00
8f4c615704 raft: leader should tick heartbeat 2014-09-03 22:01:31 -07:00
f1856abe60 etcdhttp: add v2prefix 2014-09-03 21:52:16 -07:00
efe2141c16 Merge pull request #980 from bmizerany/b-fixthings
fix vendoring and a test
2014-09-03 21:47:11 -07:00
d844377ca6 *: vendor context 2014-09-03 20:06:16 -07:00
172c1eae5d etcdhttp: fix test build 2014-09-03 20:01:54 -07:00
99c7371337 etcdserver: use vendored gogoproto 2014-09-03 19:47:05 -07:00
a4c8bfa7b4 Merge pull request #874 from etcd-team/raft
replace goraft
2014-09-03 17:07:54 -07:00
dae165eeaa Merge pull request #83 from etcd-team/snappb
snapshot: move pb to snappb; remove clusterId
2014-09-03 17:03:28 -07:00
16d337db70 Merge pull request #84 from etcd-team/walpb
wal: move pb files to walpb
2014-09-03 17:03:16 -07:00
b98cf17209 wal: move pb files to walpb 2014-09-03 16:46:42 -07:00
9e77d180c4 snapshot: move pb to snappb; remove clusterId 2014-09-03 16:33:41 -07:00
3ee83bc194 Merge pull request #82 from etcd-team/b-fiximports
coerce gogoproto files to import from third_party
2014-09-03 16:20:58 -07:00
5a62df3691 Merge pull request #81 from etcd-team/b-fixbrokenwaltests
wal: fix broken tests
2014-09-03 16:07:45 -07:00
3aa90b32f9 snap: coerce import gogoproto from third_party 2014-09-03 16:01:18 -07:00
10b73418cf wal: coerce import gogoproto from third_party 2014-09-03 16:00:42 -07:00
46a7a61b7d wal: fix broken tests 2014-09-03 15:56:28 -07:00
814558306e raft: coerce import gogoproto from third_party 2014-09-03 15:51:09 -07:00
215662188b Documentation: Clarify etcd security documentation
Signed-off-by: Vincent Ambo <vincent@kivra.com>
2014-09-04 00:40:22 +02:00
077f57d1c6 Merge pull request #80 from etcd-team/b-rmconf
raft: remove configuration
2014-09-03 15:26:35 -07:00
8d9b7b1680 raft: remove entry type 2014-09-03 15:24:47 -07:00
8463421448 raft: remove configuration 2014-09-03 15:23:05 -07:00
ade71208a2 Merge pull request #70 from etcd-team/wal_crc
wal: add basic crc
2014-09-03 15:12:08 -07:00
bdb954b2f5 wal: fix wal 2014-09-03 15:10:15 -07:00
70580de197 Merge pull request #79 from etcd-team/use_msg
raft: make tick send out messages.
2014-09-03 13:50:59 -07:00
3c77693881 raft: make tick send out messages. all state machine actions should be taken inside step function. 2014-09-03 10:17:09 -07:00
966cfd6e8e raft: fix elapsed reset 2014-09-03 09:20:20 -07:00
01be21985d third_party: remove go-etcd/go-systemd dependencies 2014-09-03 09:20:20 -07:00
2ba57ee75d *: remove unused pkgs 2014-09-03 09:20:20 -07:00
03174c9361 etcdhttp: log m.From 2014-09-03 09:20:20 -07:00
146e447cea etcdhttp: remove sending log 2014-09-03 09:20:20 -07:00
f2d200a826 raft: log only new/no leader 2014-09-03 09:20:20 -07:00
db8e4a2fc0 raft: remove debug logging 2014-09-03 09:20:20 -07:00
cf91035a36 main: log hex of id 2014-09-03 09:20:20 -07:00
576e26ea39 oops 2014-09-03 09:20:20 -07:00
d218034630 boom 2014-09-03 09:20:20 -07:00
fa11eef6d0 all campaign 2014-09-03 09:20:19 -07:00
bb185a858f raft: fix params 2014-09-03 09:20:19 -07:00
b796d227f1 add some logging 2014-09-03 09:20:19 -07:00
06dea4830d Procfile 2014-09-03 09:20:19 -07:00
dcd64f2a59 ... 2014-09-03 09:20:19 -07:00
728690fa03 hanging 2014-09-03 09:20:19 -07:00
8f3544ece8 server of one is alive! 2014-09-03 09:20:19 -07:00
cb7b321240 boom 2014-09-03 09:20:19 -07:00
c87f1d3924 fix printf 2014-09-03 09:20:19 -07:00
24afe8d22b remove prefix 2014-09-03 09:20:19 -07:00
9cdb7f073d ... 2014-09-03 09:20:19 -07:00
51d3dc7f6b typo 2014-09-03 09:20:19 -07:00
22c8ec0a80 ... 2014-09-03 09:20:18 -07:00
fc9fc3f888 ... 2014-09-03 09:20:18 -07:00
6d720fb33f raft: replace Context with Stop 2014-09-03 09:20:18 -07:00
a0922ebd0f etcdserver: light docs 2014-09-03 09:20:18 -07:00
6d7acc6b1e etcdhttp: never return 0 from genId 2014-09-03 09:20:18 -07:00
56c6fab53c etcdserver: refactor a bit 2014-09-03 09:20:18 -07:00
2b260a7ae5 etcdserver: QGET 2014-09-03 09:20:18 -07:00
2d9553cb18 etcdserver: cleanup 2014-09-03 09:20:18 -07:00
68968c2cb0 etcdserver: randomize Step 2014-09-03 09:20:18 -07:00
f4613dd466 store: remove Root 2014-09-03 09:20:18 -07:00
ba7d174349 etcdserver: add docs 2014-09-03 09:20:18 -07:00
2a212c9016 etcdserver: replace fmt with t.Logf 2014-09-03 09:20:18 -07:00
e9bb7c26fa etcdserver: set 10x the keys in test 2014-09-03 09:20:17 -07:00
97ee4dc847 etcdserver: working test 2014-09-03 09:20:17 -07:00
f8be54b416 ...deadlocked... 2014-09-03 09:20:17 -07:00
17e56a1c57 etcdserver/etcdhttp: basic test working 2014-09-03 09:20:17 -07:00
7469871d20 raft: remove proposal id / add test 2014-09-03 09:20:17 -07:00
c84a25e433 etcdhttp: parse boolpointer 2014-09-03 09:20:17 -07:00
c7cb8275c3 etcdserver: move protobuf to etcdserverpb 2014-09-03 09:20:17 -07:00
e8e588c67b raft: move protobufs into raftpb 2014-09-03 09:20:17 -07:00
59115b85f7 etcdserver/etcdhttp: remove unncessary else 2014-09-03 09:20:17 -07:00
163b27c759 etcdserver/etcdhttp: remove error return from parseRequest 2014-09-03 09:20:17 -07:00
05b2d76d54 etcdserver/etcdhttp: parseRequest 2014-09-03 09:20:17 -07:00
07058d3e0b etcdserver/etcdhttp: be specific about the keys path match 2014-09-03 09:20:17 -07:00
ddd219f297 many: marshal message 2014-09-03 09:20:16 -07:00
1eb2512961 raft: only allow one message to Step 2014-09-03 09:20:16 -07:00
38e8f3b764 etcdserver/etcdhttp: fix typo 2014-09-03 09:20:16 -07:00
c4defbc45c etcdserver/etcdhttp: break out wait
The assignment of ev was getting burried in the select. This makes it
easier to grok encodeResponse.
2014-09-03 09:20:16 -07:00
8c3450e200 etcdserver/etcdhttp: cancel wait on conn close 2014-09-03 09:20:16 -07:00
665af71888 etcdserver/etcdhttp: apply in single goroutine to avoid race 2014-09-03 09:20:16 -07:00
cd3f047ccd etcdserver/etcdhttp: remove closenotify watch 2014-09-03 09:20:16 -07:00
7638acdf37 etcdserver/etcdhttp: response with 504 2014-09-03 09:20:16 -07:00
735647e6a3 etcdserver/etcdhttp: cancel long requests on conn close 2014-09-03 09:20:16 -07:00
8cffd75e00 etcdserver/etcdhttp: encode response 2014-09-03 09:20:16 -07:00
b2e0836bb3 etcdserver/etcdhttp: wip 2014-09-03 09:20:16 -07:00
2d3cef2496 etcdserver: set -> existsSet 2014-09-03 09:20:16 -07:00
78d7b38a17 etcdserver: introduce Set 2014-09-03 09:20:15 -07:00
e79e9c4853 etcdserver: give update/create higher priority 2014-09-03 09:20:15 -07:00
5585984ed7 etcdserver: add CompareAndDelete 2014-09-03 09:20:15 -07:00
07c997f98c etcdserver: add create and delete 2014-09-03 09:20:15 -07:00
30fa119345 etcdserver: use context.TODO() 2014-09-03 09:20:15 -07:00
d2b99aa7c9 etcdserver: note that apply may need Context 2014-09-03 09:20:15 -07:00
f4d8c3fc66 etcdserver: more methods added 2014-09-03 09:20:15 -07:00
225e618b8f raft: add Id to Entry 2014-09-03 09:20:15 -07:00
3dd6280df5 etcdserver: need to add id to proposal :/ 2014-09-03 09:20:15 -07:00
c98f1cb501 etcdserver: remove panic and return default err 2014-09-03 09:20:15 -07:00
021e231476 raft: ReadState -> Ready, which returns a chan 2014-09-03 09:20:15 -07:00
432bc74a58 etcdserver: Send and Save 2014-09-03 09:20:15 -07:00
008337e150 etcdserver: wip 2014-09-03 09:20:14 -07:00
edac2e909b etcdserver: wip 2014-09-03 09:20:14 -07:00
91b62c0fbf raft: messages -> byMsgType 2014-09-03 09:20:14 -07:00
ad307c6965 raft: go fmt 2014-09-03 09:20:14 -07:00
7486d3d4c5 raft: mention error in doc 2014-09-03 09:20:14 -07:00
481e229ad4 raft: unblock progress by deproitizing proposals in Step 2014-09-03 09:20:14 -07:00
f87a6f3c1f raft: sift proposals from other message types in Step 2014-09-03 09:20:14 -07:00
4aa15294a8 raft: re-remove clusterId from raft 2014-09-03 09:20:14 -07:00
134a962222 raft: move raft2 to raft 2014-09-03 09:20:14 -07:00
15bb84d320 raft: move in entry.proto 2014-09-03 09:20:14 -07:00
5e91b3b716 raft: remove unused commentted-out code 2014-09-03 09:20:14 -07:00
9acb4cf2b0 raft: move raft.nextEnts to tests 2014-09-03 09:20:14 -07:00
c4b6896338 raft: add/deleteIns -> add/delProgress 2014-09-03 09:20:13 -07:00
5abdfda06a raft: pendingConf -> configuring 2014-09-03 09:20:13 -07:00
dd7aa95379 raft: remove atomicInt 2014-09-03 09:20:13 -07:00
75dce35a5d raft: correct hasLeader 2014-09-03 09:20:13 -07:00
72c65e74f5 raft: remove setX methods 2014-09-03 09:20:13 -07:00
5deaada0dc raft: remove EmptyState 2014-09-03 09:20:13 -07:00
9616162dbc raft: remove return bool 2014-09-03 09:20:13 -07:00
022a663e91 raft: newStateMachine -> newRaft 2014-09-03 09:20:13 -07:00
a8c123c6c2 raft: ins -> prs 2014-09-03 09:20:13 -07:00
79de1dc339 raft: sm -> r 2014-09-03 09:20:13 -07:00
3c483e19d7 raft: index{} -> progress{} 2014-09-03 09:20:13 -07:00
1b37f313a6 raft: remove unused IsMsgApp 2014-09-03 09:20:13 -07:00
0f50188652 raft: remove clearState 2014-09-03 09:20:12 -07:00
3823b74208 raft: remove unneeded fields 2014-09-03 09:20:12 -07:00
1d40440830 raft: remove old pointer 2014-09-03 09:20:12 -07:00
ea66d94273 raft: fix ReadState 2014-09-03 09:20:12 -07:00
61a413c219 raft: block ReadState until state updates 2014-09-03 09:20:12 -07:00
10eb997621 raft: whoops 2014-09-03 09:20:12 -07:00
c312d6efad raft: use Equal in example 2014-09-03 09:20:12 -07:00
92bdb1390d raft: update based on example 2014-09-03 09:20:12 -07:00
ff14d1de52 raft: example 2014-09-03 09:20:12 -07:00
6e6d81f094 raft: copy ents to avoid races 2014-09-03 09:20:12 -07:00
71ed92b6bd raft: implement ReadState 2014-09-03 09:20:12 -07:00
f1f8fc4228 raft: sm.vote -> State.Vote 2014-09-03 09:20:12 -07:00
40ed560e16 raft: index -> State.LastIndex 2014-09-03 09:20:12 -07:00
c7e17e2f9e raft: term -> State.Term 2014-09-03 09:20:11 -07:00
220a923a2d raft: remove info 2014-09-03 09:20:11 -07:00
0453d09af6 raft: moved into new raft 2014-09-03 09:20:11 -07:00
f03c3bce05 raft: seperate dequeuing from slicing 2014-09-03 09:20:11 -07:00
e17f79ee70 raft: start tick 2014-09-03 09:20:11 -07:00
8d37587e47 raft: remove sync
We don't need SyncNode. We only care about things that have made it
through the store. We can do this there.
2014-09-03 09:20:11 -07:00
ce7536e564 raft2: wip - to bring in ../raft soon 2014-09-03 09:20:11 -07:00
e2d8e1868b wait: Waiter -> WaitList 2014-09-03 09:20:11 -07:00
f9c4ba3ea3 wait: init 2014-09-03 09:20:11 -07:00
85103adfe0 wal: move record method to record.go 2014-09-03 09:20:11 -07:00
3699f2e5f9 etcdserver: rename someMachineMessage->machineMessage 2014-09-03 09:20:11 -07:00
4c116a5a01 etcd: add consistent get 2014-09-03 09:20:10 -07:00
f6a9599eb4 *: rename etcd to etcdserver 2014-09-03 09:20:10 -07:00
2ff75cda0e server: remove panic or fatal in participant.run 2014-09-03 09:20:10 -07:00
ebafba0043 server: don't print out errors that cannot be resolved 2014-09-03 09:20:10 -07:00
fab17f216d server: recover peerHub when restart from snapshot 2014-09-03 09:20:10 -07:00
27c9a0535c server: add tests for on-disk snapshot 2014-09-03 09:20:10 -07:00
5fdc124578 etcd: clean up snap 2014-09-03 09:20:10 -07:00
a28dc4559b raft/etcd: recover node 2014-09-03 09:20:10 -07:00
63489b9ef5 etcd: snap and wal init 2014-09-03 09:20:10 -07:00
442cae6844 wal: support multiple files 2014-09-03 09:20:10 -07:00
4296cd3fa4 *: remove old server 2014-09-03 09:20:10 -07:00
f891199ab0 snap: refactoring 2014-09-03 09:20:09 -07:00
f664c02a40 snap: handle snapshot file error robustly 2014-09-03 09:20:09 -07:00
338f120cbf snap: check mkdir error in tests 2014-09-03 09:20:09 -07:00
aff5b75487 snap: separate Load to Load and snapNames 2014-09-03 09:20:09 -07:00
71d12d3330 snap: fix typo 2014-09-03 09:20:09 -07:00
3b2a90e775 snap: separate different sections in the filename by adding dash 2014-09-03 09:20:09 -07:00
f7417cf693 server: add testCluster.Add
and fix possible testing error.
2014-09-03 09:20:09 -07:00
f7580cd3b6 snap: fix missing continue 2014-09-03 09:20:09 -07:00
12a9b84b2b snap: better naming 2014-09-03 09:20:09 -07:00
94521738a9 snap: init commit 2014-09-03 09:20:09 -07:00
c1da78601a server: use random generated node id 2014-09-03 09:20:09 -07:00
a5df254e53 raft: add clusterId to snapshot 2014-09-03 09:20:08 -07:00
c1c2aeffab etcd: attach default scheme to the peer-list if scheme is not given 2014-09-03 09:20:08 -07:00
cc1df691cc store: fix the bug caused by random iteration order over map 2014-09-03 09:20:08 -07:00
e9a22d0f34 etcd: make retry configurable 2014-09-03 09:20:08 -07:00
33465a7e42 server: fix testings
Make WaitMode panic to alert.
2014-09-03 09:20:08 -07:00
bb95187bc7 server: more better testing 2014-09-03 09:20:08 -07:00
cce88a8504 etcd: better testing... 2014-09-03 09:20:08 -07:00
6e9947599a server: add testServer and testCluster 2014-09-03 09:20:08 -07:00
14a6584a22 etcd: defer afterTest 2014-09-03 09:20:08 -07:00
0ba26afb4d etcd: remove skipped test 2014-09-03 09:20:08 -07:00
d71865e094 etcd: cleanup tests 2014-09-03 09:20:08 -07:00
8e2557697f etcd: rename initTestServer -> newUnstartedTestServer 2014-09-03 09:20:08 -07:00
3229c91dbb etcd: replace buildServer with runServer in tests 2014-09-03 09:20:07 -07:00
0881021e54 all config -> cfg 2014-09-03 09:20:07 -07:00
a5eec89113 cfg/ -> conf/ 2014-09-03 09:20:07 -07:00
08b370ebe4 main: remove query_params*.go 2014-09-03 09:20:07 -07:00
3653287c97 Revert "Documentation -> doc"
This reverts commit 9c179224aa04f0f25c52dd7b66576c3faf56cfc9.
2014-09-03 09:20:07 -07:00
9203f68894 server: add /v2/stats/self 2014-09-03 09:20:07 -07:00
5574b6e224 server: add /v2/stats/leader 2014-09-03 09:20:07 -07:00
41ccf13393 config -> cfg
It is extremely confusing to use variables named the same as a package.
Please no longer do this.
2014-09-03 09:20:07 -07:00
18c4be469f move ./http to main 2014-09-03 09:20:07 -07:00
6f3162af69 Documentation -> doc 2014-09-03 09:20:06 -07:00
b4fd9353cf remove fixtures 2014-09-03 09:20:06 -07:00
a2b9f9310c remove pkg/strings 2014-09-03 09:20:06 -07:00
1039f3dcea remove bench 2014-09-03 09:20:06 -07:00
d77773acb3 server: ignore server in build/tests 2014-09-03 09:20:06 -07:00
afc69b6a74 remove osext 2014-09-03 09:20:06 -07:00
ecc0f97e27 remove go-log 2014-09-03 09:20:06 -07:00
e2798405f6 remove test.sh 2014-09-03 09:20:06 -07:00
7b2474681b remove pkg/http 2014-09-03 09:20:06 -07:00
0ecbe891b5 remove pkg/btrfs 2014-09-03 09:20:05 -07:00
68b3644ac7 remove vestigial build script 2014-09-03 09:20:05 -07:00
7aa0da7c91 remove go_version 2014-09-03 09:20:05 -07:00
902f1864b1 remove contrib 2014-09-03 09:20:05 -07:00
45dc778fbb remove goprotobuf 2014-09-03 09:20:05 -07:00
0f1e18f8f0 remove toml dep 2014-09-03 09:20:05 -07:00
65f7833c22 remove go-httpclient 2014-09-03 09:20:05 -07:00
91daf1da86 remove gorilla and its users 2014-09-03 09:20:05 -07:00
7da180cfc5 metrics: remove old metrics 2014-09-03 09:20:05 -07:00
e9a45ae35d remove goraft 2014-09-03 09:20:05 -07:00
1805c15093 config: remove toml 2014-09-03 09:20:04 -07:00
622ee60d4f etcd: remove unused field seeds in participant struct 2014-09-03 09:20:04 -07:00
cc1b4b7ef0 etcd: no more mutex 2014-09-03 09:20:04 -07:00
d9ff56d7b7 etcd: cleanup locking in run 2014-09-03 09:20:04 -07:00
3af8c7da3c etcd: we should trust people 2014-09-03 09:20:04 -07:00
55c4a3307d server: cleanup participant stop 2014-09-03 09:20:04 -07:00
20c995c142 server: remove return value of func run 2014-09-03 09:20:04 -07:00
d8424a15dd etcd: remove unnecessary StopProposalWaiters function 2014-09-03 09:20:04 -07:00
b5b2031d5b wal: prune repeative logging 2014-09-03 09:20:04 -07:00
c19932c9ba raft: prune unnecessary logging - fixes #15 2014-09-03 09:20:04 -07:00
f911196e5f server: not add node that has been in the cluster 2014-09-03 09:20:04 -07:00
c9edb762b7 server: add TLSHandshakeTimeout for peer hub client 2014-09-03 09:20:03 -07:00
394893ff92 server: not set participant status for peer that points to itself 2014-09-03 09:20:03 -07:00
01fdaea8a6 server: not create new peer if exists 2014-09-03 09:20:03 -07:00
434bf8ca81 server: rewrite keepSetting in TestMultiNodeKillOne
It avoids blocking http request.
2014-09-03 09:20:03 -07:00
b1426e2635 server: wait for entries to be committed in TestClusterConfigReload 2014-09-03 09:20:03 -07:00
659eb5fd2a wal: make record a protobuf type 2014-09-03 09:20:03 -07:00
d0dc7427dd server: serve /v2/admin on raftPubAddr 2014-09-03 09:20:03 -07:00
8e06333d45 raft: make State a protobuf type 2014-09-03 09:20:03 -07:00
b9e7c20a4c etcd: make cmd a protobuf type
Conflicts:
	etcd/v2_apply.go
	etcd/v2_store.go
2014-09-03 09:20:03 -07:00
d6c3ebb1a0 raft: make Info a protobuf type 2014-09-03 09:20:03 -07:00
a191df10a3 server: p.Get -> p.QuorumGet 2014-09-03 09:20:03 -07:00
cd4b35c841 server: add QuorumGet 2014-09-03 09:20:02 -07:00
5ffc0adccc server: participant.Get -> participant.Store.Get 2014-09-03 09:20:02 -07:00
3529c381cf server: add TODO for /v2/stats 2014-09-03 09:20:02 -07:00
4b3c3203ff server: stop timer when exit 2014-09-03 09:20:02 -07:00
4f10917ce9 etcd: add a read/write timeout to server
The default is for connections to last forever[1]. This leads to fds
leaking. I set the timeout so high by default so that watches don't have
to keep retrying but perhaps we should set it slower.

Tested on a cluster with lots of clients and it seems to have relieved
the problem.

[1] https://groups.google.com/forum/#!msg/golang-nuts/JFhGwh1q9xU/heh4J8pul3QJ

Original Commit: 084dcb5596
From: philips <brandon@ifup.org>
2014-09-03 09:20:02 -07:00
e587402c26 server: Flush headers when using wait=true and stream=true
Many http clients will missbehave unless they get an initial http-
response, even when long-polling. It also saves the user/client from
having to handle headers on the first action of the watch, but rather
handle the response immediately.

Original commit: 2338481bb1
From: Christoffer Vikström
2014-09-03 09:20:02 -07:00
431ff3cce1 raft: minor change on code 2014-09-03 09:20:02 -07:00
7c3e202c94 etcd: add recover 2014-09-03 09:20:02 -07:00
15029381e1 raft: fix promotable 2014-09-03 09:20:02 -07:00
01c40fcf50 raft: not recover from outdated snapshot 2014-09-03 09:20:02 -07:00
ba63cf666d raft: add recover 2014-09-03 09:20:02 -07:00
6030261363 etcd/raft: add snap 2014-09-03 09:20:02 -07:00
943fede19c server: panic when fail to save in wal 2014-09-03 09:20:01 -07:00
2d870fa65b wal: add first level logging 2014-09-03 09:20:01 -07:00
969b529b08 wal: fix O_RDONLY attr when opening old file 2014-09-03 09:20:01 -07:00
38ec659cd6 raft: make Entry a protobuf type 2014-09-03 09:20:01 -07:00
193679e041 server: wait longer in afterTest
Because we sets dial timeout to be 200ms, afterTest should wait
> 200ms.
2014-09-03 09:20:01 -07:00
7bb6230588 peerHub: add response timeout for http client 2014-09-03 09:20:01 -07:00
069a288a59 peer: wait a little before closing idle connections
It seems that it needs some time to set connections that just used
as idle.
2014-09-03 09:20:01 -07:00
611d564865 raft: remove unused prevTerm field 2014-09-03 09:20:01 -07:00
43c9ca895b wal: fix append entry 2014-09-03 09:20:01 -07:00
54b4f52e48 raft: add index to entry 2014-09-03 09:20:01 -07:00
46eab903e9 etcd: do not generate id if next state is stop 2014-09-03 09:20:01 -07:00
0197ce4c66 server: move TestMultiNodeKillAllAndRecovery to etcd/ 2014-09-03 09:20:00 -07:00
215820dd40 server: move TestMultiNodeKillOne to etcd/ 2014-09-03 09:20:00 -07:00
a0fbc289ec server: move discovery tests to etcd/
TestDiscoveryDownNoBackupPeersWithDataDir -> TestRunByDataDir
remove TestDiscoveryRestart
2014-09-03 09:20:00 -07:00
5283002132 server: set short dial timeout for raft communication 2014-09-03 09:20:00 -07:00
28634fce47 wal: Flush -> Sync 2014-09-03 09:20:00 -07:00
2b0936271c server: fix 2nd watch timeout in TestSingleNodeRecovery
When recovering from data dir, the node needs election timeout to
elect itself to be the leader.
2014-09-03 09:20:00 -07:00
01322cd243 server: move TestClusterConfigReload to etcd/ 2014-09-03 09:20:00 -07:00
21fb7b4fbb server: improve TestKillLeader and TestKillRandom
Remove testing on 11-node cluster due to lack of fds
2014-09-03 09:20:00 -07:00
efaef49734 server: move TestSingleNodeRecovery under etcd/ 2014-09-03 09:20:00 -07:00
ffa5eb08c3 server: apply ents when loading log 2014-09-03 09:20:00 -07:00
a6132d459f server: save correct nodeId for participant 2014-09-03 09:20:00 -07:00
12dd380d26 etcd: use tmpDir as testing directory 2014-09-03 09:20:00 -07:00
653a63fa00 etcd: add destoryCluster for testing 2014-09-03 09:19:59 -07:00
88e5bce63d wip: wal glue 2014-09-03 09:19:59 -07:00
d70df4a15d wal: reorgnize wal file 2014-09-03 09:19:59 -07:00
a25cd45876 wal: add CoreOS copyright 2014-09-03 09:19:59 -07:00
b383cd5acf raft: refactor recover 2014-09-03 09:19:59 -07:00
9f5c7b310c wal: add newWal helper func 2014-09-03 09:19:59 -07:00
9c6f2ed5bb wal: use buffer 2014-09-03 09:19:59 -07:00
5baefcce26 wal: cleanup 2014-09-03 09:19:59 -07:00
363e952551 wal: add Read 2014-09-03 09:19:59 -07:00
3c7935a21d wal: check written data in tests 2014-09-03 09:19:59 -07:00
2bd74bc328 raft: add Load 2014-09-03 09:19:59 -07:00
d1cdc02afc wal: add tests for write 2014-09-03 09:19:59 -07:00
7b180d585e wal: make new work 2014-09-03 09:19:58 -07:00
e72e75876c wal: init 2014-09-03 09:19:58 -07:00
3fba10c8e6 etcd: add version handlers 2014-09-03 09:19:58 -07:00
e850c644da raft: return offset for unstableEnts 2014-09-03 09:19:58 -07:00
311db876b0 raft: add unstableState 2014-09-03 09:19:58 -07:00
609e13a240 raft: add node.Unstable
Be able to return all unstable log entries. Application must store unstable
log entries before send out any messages after calling step.
2014-09-03 09:19:58 -07:00
05e77ecf90 server: move id to the head of log line 2014-09-03 09:19:58 -07:00
e7bc7becf3 server: add first level logging 2014-09-03 09:19:58 -07:00
1288e1f39d raft: log->raftlog 2014-09-03 09:19:58 -07:00
c7d1beaaa5 raft: add first level logging
We log the message to step and the state of the statemachine before and after
stepping the message.
2014-09-03 09:19:58 -07:00
d7eef6a64e etcd: add a bootstrap test 2014-09-03 09:19:58 -07:00
b49cbc959b etcd: fix start test goroutine leak 2014-09-03 09:19:57 -07:00
b652a0d232 etcd: reorganize etce_start test 2014-09-03 09:19:57 -07:00
93104f114e all: discovery moved to etcd/ yet 2014-09-03 09:19:57 -07:00
6534525cf8 server: add bootstrap tests 2014-09-03 09:19:57 -07:00
c952e91c4f server: add discovery 2014-09-03 09:19:57 -07:00
2665cc1cc8 raft: heartbeat should not contain entries 2014-09-03 09:19:57 -07:00
7aa9838d8d server: move TestModeChange to functional tests 2014-09-03 09:19:57 -07:00
0c26c0bfab etcd: add a simple endtoend benchmark 2014-09-03 09:19:57 -07:00
1be8550672 etcd: refactor waitCluster 2014-09-03 09:19:57 -07:00
061fad12a9 main: do not pass id to etcd 2014-09-03 09:19:57 -07:00
30099d9135 etcd: generate a id for the new participant after a mode switch 2014-09-03 09:19:57 -07:00
8111930981 etcd: standby does not need to send join 2014-09-03 09:19:57 -07:00
155bd09902 etcd: add clusterid to participant 2014-09-03 09:19:56 -07:00
13ec81c87f raft: fix panic in putVarint; varint can take up to 10 bytes for a 64bit integer. 2014-09-03 09:19:56 -07:00
bb328d5aa5 etcd: fix standby test 2014-09-03 09:19:56 -07:00
468c345e74 etcd: standby should not be in a tight loop 2014-09-03 09:19:56 -07:00
060de128a7 raft: add clusterId 2014-09-03 09:19:56 -07:00
9793c518ab raft: forbid to use none as id 2014-09-03 09:19:56 -07:00
79689872af raft: use From field to distinguish local message 2014-09-03 09:19:56 -07:00
daa49023cf etcd: remove nodes in standby 2014-09-03 09:19:56 -07:00
b4cf146a52 server: notice 2013 -> 2014 2014-09-03 09:19:56 -07:00
0429fe04df server: add notice comment to all files 2014-09-03 09:19:56 -07:00
5bfaaa7964 etcd: fix serverHttp 2014-09-03 09:19:56 -07:00
0b77b42cad standby: fix leader var race 2014-09-03 09:19:55 -07:00
92778afd0b server: retry remove in TestBecomeStandby
To prevent from false error message.
2014-09-03 09:19:55 -07:00
2f51735e8a etcd: move raftprefix to raft_handler 2014-09-03 09:19:55 -07:00
7e7cfb1ce8 etcd: remove participant.seeds 2014-09-03 09:19:55 -07:00
9ffc0b9f2c etcd: remove unncessary code in participant 2014-09-03 09:19:55 -07:00
2e59635bea etcd: fix a race in peer.go 2014-09-03 09:19:55 -07:00
6dc49def25 server: move var noneId to standby.go 2014-09-03 09:19:55 -07:00
d120962959 server: remove func barrier
It could be replaced by func watch.
2014-09-03 09:19:55 -07:00
1c9928d721 participant: retry join more times 2014-09-03 09:19:55 -07:00
e709f1b572 server: fix possible race when switching mode 2014-09-03 09:19:55 -07:00
6059db1f4b participant: stop http serving when stopped 2014-09-03 09:19:55 -07:00
6d46fc39aa server: remove modeC var 2014-09-03 09:19:55 -07:00
5d498918bf etcd: abstract out mode change logic 2014-09-03 09:19:54 -07:00
4c324fe3a4 etcd: cleanup etcd.go 2014-09-03 09:19:54 -07:00
6d81aabd48 server: refactor server 2014-09-03 09:19:54 -07:00
447f6a16cc etcd: stop peerhub 2014-09-03 09:19:54 -07:00
d198173fd7 etcd: cleanup peerhub 2014-09-03 09:19:54 -07:00
92cc288f6e server: fix possible join back in TestRemove 2014-09-03 09:19:54 -07:00
e02baf33c9 etcd: unknowNode -> unknownPeer 2014-09-03 09:19:54 -07:00
e0e8495ace etcd: move fetch logic into peerhub 2014-09-03 09:19:54 -07:00
9756dba57a etcd: fix cluster sync 2014-09-03 09:19:53 -07:00
9746de91bf server: clean new func 2014-09-03 09:19:53 -07:00
46974ef473 server: add stop serving func to v2_client 2014-09-03 09:19:53 -07:00
7bf3e062bd server: fix standby waitgroup on doing requests 2014-09-03 09:19:53 -07:00
6f481af383 etcd: clean up sync 2014-09-03 09:19:53 -07:00
a922947bb0 etcd: move s.setClusterConfig to v2_admin.go 2014-09-03 09:19:53 -07:00
a50857d38a etcd: move s.ClusterConfig to v2_admin.go 2014-09-03 09:19:53 -07:00
da19964959 etcd: fix mode change race 2014-09-03 09:19:53 -07:00
8ea840c19a store: fix index data race 2014-09-03 09:19:53 -07:00
7adb765660 etcd: fix datarace in peer.go 2014-09-03 09:19:53 -07:00
b8b5734689 etcd: transporter->rafthandler 2014-09-03 09:19:53 -07:00
d2a553f6c4 etcd: refactor transporter 2014-09-03 09:19:53 -07:00
5f9a5e6a5d server: simplify mode transition 2014-09-03 09:19:52 -07:00
18001dd779 server/v2_client: add func CloseConnections 2014-09-03 09:19:52 -07:00
404dc96645 server: add TestBecomeStandby 2014-09-03 09:19:52 -07:00
a274e5b192 server: use status for transporter 2014-09-03 09:19:52 -07:00
429b9487f7 server: use transporter as raft HTTP handler 2014-09-03 09:19:52 -07:00
a2c5c844a0 server: add standby 2014-09-03 09:19:52 -07:00
54e39a30f7 server: refactor client
To be more readable and get rid of false error message.
2014-09-03 09:19:52 -07:00
f21842cd04 etcd: reportLead -> getLead 2014-09-03 09:19:52 -07:00
bea28933d3 raft: init lead to none 2014-09-03 09:19:52 -07:00
0ccd09532b etcd: add joinThroughFollower test 2014-09-03 09:19:52 -07:00
638f36956b etcd: fix transporter leak in test 2014-09-03 09:19:51 -07:00
16e9aa77e3 server: clear proposal wait in time 2014-09-03 09:19:51 -07:00
ee2d5d66af v2_client: read whole response body before close
Client have to read whole response bodies
if they want the advantage of reusing TCP connections.

https://code.google.com/p/go/source/detail?r=d4e1ec84876c0f5611ab86a03826da14b866efb2&name=release-branch.go1.1&path=/src/pkg/net/http/transport.go
2014-09-03 09:19:51 -07:00
984f309815 server: use buffer for proposal channel 2014-09-03 09:19:51 -07:00
02922fa7a5 etcd: clean testAdd 2014-09-03 09:19:51 -07:00
a0c0638744 tests: remove unnecessary test 2014-09-03 09:19:51 -07:00
60c8dbe0c9 etcd: rewrite kill_leader and kill_random test 2014-09-03 09:19:51 -07:00
83e1fe77c8 server: refactor add 2014-09-03 09:19:51 -07:00
3ea913e76a etcd: refactor remove 2014-09-03 09:19:51 -07:00
b951aaf925 store: check remove func before call it 2014-09-03 09:19:50 -07:00
2e86cf2dc8 raft: add more randomness 2014-09-03 09:19:50 -07:00
9c8aff66a1 raft: randomize election timeout 2014-09-03 09:19:50 -07:00
3921295b21 raft: update lead to none when receives vaild msgVote 2014-09-03 09:19:50 -07:00
605c4ce702 etcd: fix TestGetAdminMachinesEndPoint 2014-09-03 09:19:50 -07:00
a69e416604 etcd: group the prefix consts 2014-09-03 09:19:50 -07:00
7b11dc1c05 server: maintain cluster members in nodes var 2014-09-03 09:19:50 -07:00
5562c3b4ec server: v2 propose sends error back 2014-09-03 09:19:50 -07:00
5af8fe9a84 server: use /v2/admin/machines/ http endpoint to join 2014-09-03 09:19:50 -07:00
041524432d server: implement join and remove http endpoint 2014-09-03 09:19:50 -07:00
f34b77216f server: add add function 2014-09-03 09:19:50 -07:00
5fc5681cb4 server: make removal go through run loop
It should not send to raft endpoint directly.
2014-09-03 09:19:50 -07:00
98fdbaaae0 server: add remove function 2014-09-03 09:19:50 -07:00
f1853b4364 raft: add msgDenial to deny removed nodes 2014-09-03 09:19:49 -07:00
690edb2c56 raft: update lead for follower sm when receiving msgApp
Or follower may take `none` as its leader forever if it just launched
a failed election whose term is the same as the current leader.
2014-09-03 09:19:49 -07:00
edd8d7e534 etcd: add /v2/admin/machines/ endpoint 2014-09-03 09:19:49 -07:00
f95f53e446 etcd: add /v2/admin/config endpoint 2014-09-03 09:19:49 -07:00
447d7dc51b raft: fix log append; add tests 2014-09-03 09:19:49 -07:00
8ccb8b1f9f etcd: fake standby 2014-09-03 09:19:49 -07:00
44836d9099 etcd: move server/usage.go to etcd/v2_usage.go 2014-09-03 09:19:49 -07:00
b8d71dfe70 v2: remove old tests 2014-09-03 09:19:49 -07:00
02ced2c2d7 v1: deprecate v1 support
Etcd moves to 0.5 without the support of v1.
2014-09-03 09:19:49 -07:00
8d758be3e4 server: remove unused file 2014-09-03 09:05:15 -07:00
042a8e3d4f etcd: move util to test 2014-09-03 09:05:15 -07:00
00935c873f etcd: refactor tests 2014-09-03 09:05:14 -07:00
2ff3ce74c5 raft: add sm.Index 2014-09-03 09:05:14 -07:00
0886e0ddf4 raft: change term to atomicInt 2014-09-03 09:05:14 -07:00
30f4d9faea raft: change index and term to int64 2014-09-03 09:05:14 -07:00
e11c7f35b4 raft: add atomicInt 2014-09-03 09:05:14 -07:00
284e76f0da etcd: fix data race in transporter 2014-09-03 09:05:14 -07:00
3f435571d3 raft: atomic load id 2014-09-03 09:05:14 -07:00
18d95b336f main: generate 64bit id 2014-09-03 09:05:14 -07:00
c4e7432ef9 etcd: support cors 2014-09-03 09:05:14 -07:00
45c6bf80e1 etcd: support etcd server tls 2014-09-03 09:05:14 -07:00
4181f1b2e1 etcd: support raft tls 2014-09-03 09:05:14 -07:00
c3f8eabac3 etcd: separate raft and client port 2014-09-03 09:05:13 -07:00
10b2f88b83 etcd: support v2 store stats endpoint 2014-09-03 09:05:13 -07:00
9a59f16964 etcd: support v2 peers endpoint 2014-09-03 09:05:13 -07:00
f534d6c8f6 etcd: fix machines endpoint test 2014-09-03 09:05:13 -07:00
6fb2c7c883 etcd: support v2 leader endpoint 2014-09-03 09:05:13 -07:00
6234164f28 etcd: add a kv tests todo 2014-09-03 09:05:13 -07:00
ac44e56ea0 etcd: support v2/machines endpoint 2014-09-03 09:05:13 -07:00
5e486dd912 etcd: use v2 machines prefix 2014-09-03 09:05:13 -07:00
a88f077348 config: remove unused map 2014-09-03 09:05:13 -07:00
77233b26d0 etcd: support old flags 2014-09-03 09:05:13 -07:00
02c854717b config: make config a self-contained pkg 2014-09-03 09:05:13 -07:00
60b1f2f437 config: remove deprecated flags 2014-09-03 09:05:13 -07:00
fc35324ba7 etcd: pass v2 kv api tests 2014-09-03 09:05:13 -07:00
2af0ad505a raft: make id int64 2014-09-03 09:05:12 -07:00
5651272ec8 raft: handle snapshot message 2014-09-03 09:05:12 -07:00
2a11c1487c raft: sm.compact and sm.restore 2014-09-03 09:05:12 -07:00
62a90e77b3 raft: send Normal with nil Data when leader is elected out 2014-09-03 09:05:12 -07:00
064004b899 raft: add log compact 2014-09-03 09:05:12 -07:00
6a232dfc13 raft: add offset for log 2014-09-03 09:05:12 -07:00
7be945f59b etcd: glue 2014-09-03 09:05:12 -07:00
8eac28350d raft: avoid unexpected self-bootstrap state machine 2014-09-03 09:05:12 -07:00
c86f484712 raft: make judge clear 2014-09-03 09:05:12 -07:00
7dc02b947d raft: refine TestVote 2014-09-03 09:05:12 -07:00
095251f1fa raft: Network supports discontinuous ids 2014-09-03 09:05:12 -07:00
62bdcf6f49 raft: range over sm.ins 2014-09-03 09:05:12 -07:00
e9cb510ef5 raft: node.Add needs the pubAddr field 2014-09-03 09:05:12 -07:00
5778c49689 raft: set term in reset 2014-09-03 09:05:11 -07:00
67808f8db1 raft: remove vestigial canStep 2014-09-03 09:05:11 -07:00
6044b1a0d7 raft: break Step into pieces 2014-09-03 09:05:11 -07:00
f387e3e27d raft: add Entry.isConfig 2014-09-03 09:05:11 -07:00
3817661f82 raft: rename ConfigAdd/ConfigRemove -> AddNode/RemoveNode 2014-09-03 09:05:11 -07:00
9f315ffe10 raft: make entry type public 2014-09-03 09:05:11 -07:00
584186c7ff add HasLeader function 2014-09-03 09:05:11 -07:00
dd94d5d4e8 raft: move tick comments into cases 2014-09-03 09:05:11 -07:00
15a8b46359 raft: add/remove -> addNode/removeNode 2014-09-03 09:05:11 -07:00
e5b9e22518 raft: Step returns ok 2014-09-03 09:05:11 -07:00
a10461f60d raft: Add/Remove -> add/remove 2014-09-03 09:05:11 -07:00
8344303b1a raft: addr -> id 2014-09-03 09:05:11 -07:00
cb2095cddc raft: fix typo 2014-09-03 09:05:10 -07:00
cc3a8e26c8 raft: add leader change test for configuration 2014-09-03 09:05:10 -07:00
3a85d97fd9 raft: group Node fields 2014-09-03 09:05:10 -07:00
6e95448ad7 raft: correct comment for n.Next 2014-09-03 09:05:10 -07:00
7e27d588ff raft: return all the appliable entries 2014-09-03 09:05:10 -07:00
f8a3ac9338 raft: make Msgs one line 2014-09-03 09:05:10 -07:00
d12b2c39dd raft: blake is OCD 2014-09-03 09:05:10 -07:00
78bbb37018 raft: Config -> config 2014-09-03 09:05:10 -07:00
fb1ca245a7 raft: remove unused ClusterId 2014-09-03 09:05:10 -07:00
8c12d6d00f raft: refactor Node.Add/Remove 2014-09-03 09:05:10 -07:00
96059a496a raft: replace Node.id 2014-09-03 09:05:10 -07:00
abd2448931 raft: addr -> id 2014-09-03 09:05:10 -07:00
c24b6b4150 raft: init stateMachine in New 2014-09-03 09:05:10 -07:00
7cdd148e24 raft: StartCluster -> Dictate 2014-09-03 09:05:09 -07:00
1a75beb57c raft: add confAdd and confRemove entry type 2014-09-03 09:05:09 -07:00
b5f887f5d2 raft: add cluster test 2014-09-03 09:05:09 -07:00
5cad4e595c raft: add create cluster of itself 2014-09-03 09:05:09 -07:00
193756fa38 raft: add remove node 2014-09-03 09:05:09 -07:00
961518c893 raft: execute log entries in node.step 2014-09-03 09:05:09 -07:00
989f41477d raft: rename peer -> peers 2014-09-03 09:05:09 -07:00
8ce6b94e05 raft: add and remove 2014-09-03 09:05:09 -07:00
d293c4915c raft: fix pendingConf 2014-09-03 09:05:09 -07:00
c03fbf68d6 raft: add conf safety
To make configuration change safe without adding configuration protocol:

1. We only allow to add/remove one node at a time.

2. We only allow one uncommitted configuration entry in the log.

These two rules can make sure there is no disjoint quorums in both current cluster and the
future(after applied any number of committed entries or uncommitted entries in log) clusters.

We add a type field in Entry structure for two reasons:

1. Statemachine needs to know if there is a pending configuration change.

2. Configuration entry should be executed by raft package rather application who is using raft.
2014-09-03 09:05:09 -07:00
853a458a0d raft: change ins from array to map 2014-09-03 09:05:09 -07:00
f9c299da8b raft: return after handling msgBeat
It makes the workflow more clear.
2014-09-03 09:05:09 -07:00
c7e358922b raft: add elaspe reset test 2014-09-03 09:05:09 -07:00
5d710c0f7a raft: add recvMsgBeat test 2014-09-03 09:05:08 -07:00
12572e5412 raft: add state transition test 2014-09-03 09:05:08 -07:00
920b80c41f raft: add single node commit test 2014-09-03 09:05:08 -07:00
8111d4fbb8 raft: introduce network.cut 2014-09-03 09:05:08 -07:00
a6a63d116f raft: add a test for commit rule 2014-09-03 09:05:08 -07:00
28f87c2a43 raft: use sm.maybeCommit; should not call log.maybeCommit directly 2014-09-03 09:05:08 -07:00
4d22ff90d5 raft: refactor network simulator
Signed-off-by: Blake Mizerany <blake.mizerany@gmail.com>
2014-09-03 09:05:08 -07:00
a9c81088f8 raft: add new diff
Signed-off-by: Blake Mizerany <blake.mizerany@gmail.com>
2014-09-03 09:05:08 -07:00
c32d34166e raft: leader updates its own match; tries to commit after a prop 2014-09-03 09:05:08 -07:00
6fa74b0e33 raft: do bcast when receiving msgBeat 2014-09-03 09:05:08 -07:00
09d1575eeb raft: node.tick 2014-09-03 09:05:08 -07:00
eff3aadba1 raft: add full stop to every function comment 2014-09-03 09:05:08 -07:00
167ef7e8b0 raft: test leader replies to appResp 2014-09-03 09:05:08 -07:00
9bb7265d64 raft: leader only sends append to the unsynced peer when get a bad appResp 2014-09-03 09:05:07 -07:00
88674a623a raft: use if statement to handle voteResp 2014-09-03 09:05:07 -07:00
b3de2b3450 raft: testVote - breaks state machine init to multiple lines 2014-09-03 09:05:07 -07:00
59a720d8be raft: vote test for candidate and leader 2014-09-03 09:05:07 -07:00
828a8cf326 raft: leader denies vote with the same term 2014-09-03 09:05:07 -07:00
93b08502e4 raft: check voteFor 2014-09-03 09:05:07 -07:00
c1c45575be raft: ins: []*index -> []index
It could make raft faster, use less storage.
2014-09-03 09:05:07 -07:00
1170c21f89 raft: add a single candidate test 2014-09-03 09:05:07 -07:00
2066ff5acb raft: candidate should become leader directly if it is a one node cluster 2014-09-03 09:05:07 -07:00
0cdd1b58a4 raft: rename log.commit to log.committed 2014-09-03 09:05:07 -07:00
9cd3b2153f raft: comment log.nextEnts 2014-09-03 09:05:07 -07:00
a06729a96a raft: use log.lastIndex() 2014-09-03 09:05:07 -07:00
888ddacd3c raft: remove the init cap of log entries 2014-09-03 09:05:06 -07:00
2ef9498d6f raft: remove TLA comment 2014-09-03 09:05:06 -07:00
1f0e13e956 raft: remove sm.voteWorthy 2014-09-03 09:05:06 -07:00
bee9d8bea5 raft: add log.maybeAppend 2014-09-03 09:05:06 -07:00
b70be19653 raft: add log.maybeCommit 2014-09-03 09:05:06 -07:00
092461d7c8 raft: rename log.len to log.lastIndex 2014-09-03 09:05:06 -07:00
8f3d109c18 raft: rename log.isOk to log.matchTerm 2014-09-03 09:05:06 -07:00
4c609ec59c raft: new log struct 2014-09-03 09:05:06 -07:00
cb7b75c15f raft: remove unused function maybeAppend
Signed-off-by: Blake Mizerany <blake.mizerany@gmail.com>
2014-09-03 09:05:06 -07:00
74737b76cc raft: leader commit and test
Signed-off-by: Blake Mizerany <blake.mizerany@gmail.com>
2014-09-03 09:05:06 -07:00
c223eca938 raft: clean stateMachine 2014-09-03 09:05:06 -07:00
5b052e1e10 raft: prop msg type should be MsgProp 2014-09-03 09:05:06 -07:00
ab79550693 raft: use becomeFollower in cadidate state 2014-09-03 09:05:06 -07:00
73e3394d2d raft: reverse sort to figure out the ci 2014-09-03 09:05:05 -07:00
8ddcd9799d raft: go fmt 2014-09-03 09:05:05 -07:00
6e0a668455 raft: add Node.Next 2014-09-03 09:05:05 -07:00
9545662c6b raft: return *Node 2014-09-03 09:05:05 -07:00
50e0db4038 raft: attempt first version of Interface 2014-09-03 09:05:05 -07:00
8d7be33dd8 raft: move to raft pkg 2014-09-03 09:05:05 -07:00
f6f4329899 raft: test logs converge 2014-09-03 09:05:05 -07:00
935320289e raft: remove defunct empty var 2014-09-03 09:05:05 -07:00
8353340697 raft: remove defunct println 2014-09-03 09:05:05 -07:00
45500c5e7b raft: test candidate concede 2014-09-03 09:05:05 -07:00
8942415933 raft: test failed election 2014-09-03 09:05:05 -07:00
fcc7a42d6c raft: remove heal from network 2014-09-03 09:05:05 -07:00
13012ddd9a raft: test vote 2014-09-03 09:05:04 -07:00
706c6df2ce raft: add TODO 2014-09-03 09:05:04 -07:00
0ac49ba58d raft: fix logDiff false positives 2014-09-03 09:05:04 -07:00
b06c0cc3ec raft: ignore old messages 2014-09-03 09:05:04 -07:00
895d80d0e1 raft: test dualing proposers 2014-09-03 09:05:04 -07:00
0ca153e1e5 raft: make testing logs easier 2014-09-03 09:05:04 -07:00
9f8ede7b03 start new raft implementation 2014-09-03 09:05:04 -07:00
0a2384bf4d Merge pull request #969 from jurmous/patch-5
Add Etcd4j as a new Java client option
2014-08-29 09:15:16 -07:00
be0bb56525 Add Etcd4j as a new Java client option 2014-08-29 10:45:46 +02:00
6e70dfc33a Merge pull request #967 from jurmous/patch-3
Update clients-matrix.md
2014-08-28 14:38:58 -07:00
0e1f0a734b Swapped the clients with the features on the clients matrix so it is more comprehensible. The old version got too large to show on the screen. This new layout added a language column so you can quickly see the language of the client library.
Added alphabetical sorting to the clients list.

Also moved v1-only clients to the bottom since v2 is what most people are looking for today.
2014-08-28 23:20:23 +02:00
2b6e45d0ee Merge pull request #966 from justintime/patch-1
Add link to etcd-browser under Tools heading
2014-08-28 11:11:19 -07:00
07217b8d7c Add link to etcd-browser under Tools heading 2014-08-28 12:02:39 -05:00
6ea8da077e Merge pull request #1 from coreos/master
Pull coreos changes into this fork
2014-08-28 10:14:14 +02:00
ab4bcc1869 Merge pull request #962 from aterreno/master
Added etc-clojure to the list of clients supporting V.2 api
2014-08-27 17:31:06 -07:00
0e53287ea2 Added etc-clojure to the list of clients supporting V.2 api 2014-08-27 09:50:08 +01:00
3fc68f00f0 Merge pull request #956 from jurmous/patch-2
Remove an invalid comma from api.md example
2014-08-25 08:20:31 -07:00
4a04a89cb4 Merge pull request #955 from jurmous/patch-1
Adding missing comma to api.md
2014-08-25 08:18:21 -07:00
b3970c8db1 Merge pull request #942 from robszumski/master
docs: clarify tuning timeouts
2014-08-25 08:16:33 -07:00
36c57b7717 Merge pull request #939 from suzaku/ignore
Add .dockerignore to avoid including .git in docker build context
2014-08-25 08:15:03 -07:00
8f629aae4b Remove an invalid comma from api.md example 2014-08-25 11:09:15 +02:00
dead5e0eef Adding missing comma to api.md
Invalid json example
2014-08-25 10:55:56 +02:00
fa156aaf53 Merge pull request #954 from ddysher/master
Fix Typo in api.md.
2014-08-24 13:36:49 -07:00
e491c16afa Fix Typo in api.md. 2014-08-24 10:42:47 -04:00
95fe075a17 Merge pull request #950 from zxc122333/patch-1
doc:[readme.md] fix Planning/Roadmap 404
2014-08-22 15:58:40 -07:00
e698c378e7 doc:readme.md fix Planning/Roadmap 404 2014-08-21 16:08:03 +08:00
b083766608 Merge pull request #937 from andybons/patch-3
Update to Go 1.3.1
2014-08-20 12:55:33 -07:00
82db1e7919 Merge pull request #923 from unihorn/113
docs: set snapshot to be true in default configuration
2014-08-20 10:02:04 -07:00
7c7a70202b docs: clarify tuning timeouts 2014-08-18 16:15:59 -04:00
f734ba9974 Add .dockerignore to avoid including .git in docker build context
.git is currently the biggest directory
2014-08-15 16:38:29 +08:00
511ac3280f Merge pull request #927 from bmatsuo/topic/Documentation/fix-tuning-timeout-toml
Documentation: fix timeout tuning TOML example
2014-08-14 00:52:39 -07:00
a32cd31d3b Update to Go 1.3.1 2014-08-13 10:54:51 -07:00
186879cce0 Merge pull request #928 from grossws/patch-1
Typo fixes
2014-08-12 11:40:27 -07:00
dd9aa95f1f Merge pull request #918 from edwardcapriolo/patch-1
Update libraries-and-tools.md
2014-08-12 11:38:36 -07:00
616db55e08 Typo fix 2014-08-11 15:57:00 +04:00
b52d4cdbb2 Documentation: fix timeout tuning TOML example
this makes the TOML example for tuning the peer election timeout
consistent with the guide's advice and with the other examples.
2014-08-10 20:09:14 -07:00
d99567f698 Update clients-matrix.md 2014-08-10 16:48:59 -04:00
629c578cea docs: set snapshot to be true in default configuration
If snapshot is false, it will consume much memory and become rather
slow after several days. Make it on as default.
2014-08-09 09:35:29 -07:00
95d295da54 Documentation: remove "token" from discovery docs
It was a bit confusing how the term "token" was overriden in various ways in this doc. Clean it up by referring to it as a discovery URL.
2014-08-08 17:56:58 -07:00
97a1ca1ad3 Update libraries-and-tools.md
Added a new small c++ library
2014-08-06 10:50:28 -04:00
c9db87a302 server: bump to 0.4.6+git 2014-07-29 10:33:44 -07:00
787 changed files with 32945 additions and 78949 deletions

1
.dockerignore Normal file
View File

@ -0,0 +1 @@
.git

5
.gitignore vendored
View File

@ -1,6 +1,11 @@
/coverage
/gopath
/go-bindata
/machine*
/bin
.vagrant
*.etcd
/etcd
*.swp
/hack/insta-discovery/.env
*.test

10
.travis.yml Normal file
View File

@ -0,0 +1,10 @@
language: go
go:
- 1.3
install:
- go get code.google.com/p/go.tools/cmd/cover
- go get code.google.com/p/go.tools/cmd/vet
script:
- ./test

View File

@ -21,12 +21,13 @@ This is a rough outline of what a contributor's workflow looks like:
- Make sure your commit messages are in the proper format (see below).
- Push your changes to a topic branch in your fork of the repository.
- Submit a pull request to coreos/etcd.
- Your PR must receive a LGTM from two maintainers found in the MAINTAINERS file.
Thanks for your contributions!
### Code style
The coding style suggested by the Golang community is used in etcd. See [style doc](https://code.google.com/p/go-wiki/wiki/Style) for details.
The coding style suggested by the Golang community is used in etcd. See the [style doc](https://code.google.com/p/go-wiki/wiki/CodeReviewComments) for details.
Please follow this style to make etcd easy to review, maintain and develop.

View File

@ -1,12 +1,2 @@
FROM ubuntu:12.04
# Let's install go just like Docker (from source).
RUN apt-get update -q
RUN DEBIAN_FRONTEND=noninteractive apt-get install -qy build-essential curl git
RUN curl -s https://storage.googleapis.com/golang/go1.3.src.tar.gz | tar -v -C /usr/local -xz
RUN cd /usr/local/go/src && ./make.bash --no-clean 2>&1
ENV PATH /usr/local/go/bin:$PATH
ADD . /opt/etcd
RUN cd /opt/etcd && ./build
EXPOSE 4001 7001
ENTRYPOINT ["/opt/etcd/bin/etcd"]
FROM golang:onbuild
EXPOSE 4001 7001 2379 2380

View File

@ -0,0 +1,79 @@
## Administration
### Data Directory
#### Lifecycle
When first started, etcd stores its configuration into a data directory specified by the data-dir configuration parameter.
Configuration is stored in the write ahead log and includes: the local member ID, cluster ID, and initial cluster configuration.
The write ahead log and snapshot files are used during member operation and to recover after a restart.
If a members data directory is ever lost or corrupted then the user should remove the etcd member from the cluster via the [members API][members-api].
A user should avoid restarting an etcd member with a data directory from an out-of-date backup.
Using an out-of-date data directory can lead to inconsistency as the member had agreed to store information via raft then re-joins saying it needs that information again.
For maximum safety, if an etcd member suffers any sort of data corruption or loss, it must be removed from the cluster.
Once removed the member can be re-added with an empty data directory.
[members-api]: https://github.com/coreos/etcd/blob/master/Documentation/0.5/other_apis.md#members-api
#### Contents
The data directory has two sub-directories in it:
1. wal: write ahead log files are stored here. For details see the [wal package documentation][wal-pkg]
2. snap: log snapshots are stored here. For details see the [snap package documentation][snap-pkg]
[wal-pkg]: http://godoc.org/github.com/coreos/etcd/wal
[snap-pkg]: http://godoc.org/github.com/coreos/etcd/snap
### Cluster Lifecycle
If you are spinning up multiple clusters for testing it is recommended that you specify a unique initial-cluster-token for the different clusters.
This can protect you from cluster corruption in case of mis-configuration because two members started with different cluster tokens will refuse members from each other.
### Disaster Recovery
etcd is designed to be resilient to machine failures. An etcd cluster can automatically recover from any number of temporary failures (for example, machine reboots), and a cluster of N members can tolerate up to _(N/2)-1_ permanent failures (where a member can no longer access the cluster, due to hardware failure or disk corruption). However, in extreme circumstances, a cluster might permanently lose enough members such that quorum is irrevocably lost. For example, if a three-node cluster suffered two simultaneous and unrecoverable machine failures, it would be normally impossible for the cluster to restore quorum and continue functioning.
To recover from such scenarios, etcd provides functionality to backup and restore the datastore and recreate the cluster without data loss.
#### Backing up the datastore
The first step of the recovery is to backup the data directory on a functioning etcd node. To do this, use the `etcdctl backup` command, passing in the original data directory used by etcd. For example:
```sh
etcdctl backup \
--data-dir /var/lib/etcd \
--backup-dir /tmp/etcd_backup
```
This command will rewrite some of the metadata contained in the backup (specifically, the node ID and cluster ID), which means that the node will lose its former identity. In order to recreate a cluster from the backup, you will need to start a new, single-node cluster. The metadata is rewritten to prevent the new node from inadvertently being joined onto an existing cluster.
#### Restoring a backup
To restore a backup using the procedure created above, start etcd with the `-force-new-cluster` option and pointing to the backup directory. This will initialize a new, single-member cluster with the default advertised peer URLs, but preserve the entire contents of the etcd data store. Continuing from the previous example:
```sh
etcd \
-data-dir=/tmp/etcd_backup \
-force-new-cluster \
...
```
Now etcd should be available on this node and serving the original datastore.
Once you have verified that etcd has started successfully, shut it down and move the data back to the previous location (you may wish to make another copy as well to be safe):
```sh
pkill etcd
rm -fr /var/lib/etcd
mv /tmp/etcd_backup /var/lib/etcd
etcd \
-data-dir=/var/lib/etcd \
...
```
#### Restoring the cluster
Now that the node is running successfully, you can add more nodes to the cluster and restore resiliency. See the [runtime configuration](runtime-configuration.md) guide for more details.

1010
Documentation/0.5/api.md Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,205 @@
# Clustering Guide
This guide will walk you through configuring a three machine etcd cluster with the following details:
|Name |Address |
|-------|-----------|
|infra0 |10.0.1.10 |
|infra1 |10.0.1.11 |
|infra2 |10.0.1.12 |
## Static
As we know the cluster members, their addresses and the size of the cluster before starting we can use an offline bootstrap configuration. Each machine will get either the following command line or environment variables:
```
ETCD_INITIAL_CLUSTER="infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380"
ETCD_INITIAL_CLUSTER_STATE=new
```
```
-initial-cluster infra0=http://10.0.1.10:2380,http://10.0.1.11:2380,infra2=http://10.0.1.12:2380 \
-initial-cluster-state new
```
If you are spinning up multiple clusters (or creating and destroying a single cluster) with same configuration for testing purpose, it is highly recommended that you specify a unique `initial-cluster-token` for the different clusters. By doing this, etcd can generate unique cluster IDs and member IDs for the clusters even if they otherwise have the exact same configuration. This can protect you from cross-cluster-interaction, which might corrupt your clusters.
On each machine you would start etcd with these flags:
```
$ etcd -name infra0 -initial-advertise-peer-urls https://10.0.1.10:2380 \
-initial-cluster-token etcd-cluster-1 \
-initial-cluster infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380 \
-initial-cluster-state new
```
```
$ etcd -name infra1 -initial-advertise-peer-urls https://10.0.1.11:2380 \
-initial-cluster-token etcd-cluster-1 \
-initial-cluster infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380 \
-initial-cluster-state new
```
```
$ etcd -name infra2 -initial-advertise-peer-urls https://10.0.1.12:2380 \
-initial-cluster-token etcd-cluster-1 \
-initial-cluster infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380 \
-initial-cluster-state new
```
The command line parameters starting with `-initial-cluster` will be ignored on subsequent runs of etcd. You are free to remove the environment variables or command line flags after the initial bootstrap process. If you need to make changes to the configuration later see our guide on [runtime configuration](runtime-configuration.md).
### Error Cases
In the following case we have not included our new host in the list of enumerated nodes. If this is a new cluster, the node must be added to the list of initial cluster members.
```
$ etcd -name infra1 -initial-advertise-peer-urls http://10.0.1.11:2380 \
-initial-cluster infra0=http://10.0.1.10:2380 \
-initial-cluster-state new
etcd: infra1 not listed in the initial cluster config
exit 1
```
In this case we are attempting to map a node (infra0) on a different address (127.0.0.1:2380) than its enumerated address in the cluster list (10.0.1.10:2380). If this node is to listen on multiple addresses, all addresses must be reflected in the "initial-cluster" configuration directive.
```
$ etcd -name infra0 -initial-advertise-peer-urls http://127.0.0.1:2380 \
-initial-cluster infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380 \
-initial-cluster-state=new
etcd: infra0 has different advertised URLs in the cluster and advertised peer URLs list
exit 1
```
If you configure a peer with a different set of configuration and attempt to join this cluster you will get a cluster ID mismatch and etcd will exit.
```
$ etcd -name infra3 -initial-advertise-peer-urls http://10.0.1.13:2380 \
-initial-cluster infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra3=http://10.0.1.13:2380 \
-initial-cluster-state=new
etcd: conflicting cluster ID to the target cluster (c6ab534d07e8fcc4 != bc25ea2a74fb18b0). Exiting.
exit 1
```
## Discovery
In a number of cases you might not know the IPs of your cluster peers ahead of time. This is common when utilizing cloud providers or when your network uses DHCP. In these cases you can use an existing etcd cluster to bootstrap a new one. We call this process "discovery".
### Lifetime of a Discovery URL
A discovery URL identifies a unique etcd cluster. Instead of reusing a discovery URL, you should always create discovery URLs for new clusters.
Moreover, discovery URLs should ONLY be used for the initial bootstrapping of a cluster. To change cluster membership after the cluster is already running, see the [runtime reconfiguration][runtime] guide.
[runtime]: https://github.com/coreos/etcd/blob/master/Documentation/0.5/runtime-configuration.md
### Custom etcd discovery service
Discovery uses an existing cluster to bootstrap itself. If you are using your own etcd cluster you can create a URL like so:
```
$ curl -X PUT https://myetcd.local/v2/keys/discovery/6c007a14875d53d9bf0ef5a6fc0257c817f0fb83/_config/size -d value=3
```
By setting the size key to the URL, you create a discovery URL with expected-cluster-size of 3.
The URL you will use in this case will be `https://myetcd.local/v2/keys/discovery/6c007a14875d53d9bf0ef5a6fc0257c817f0fb83` and the etcd members will use the `https://myetcd.local/v2/keys/discovery/6c007a14875d53d9bf0ef5a6fc0257c817f0fb83` directory for registration as they start.
Now we start etcd with those relevant flags for each member:
```
$ etcd -name infra0 -initial-advertise-peer-urls http://10.0.1.10:2380 \
-discovery https://myetcd.local/v2/keys/discovery/6c007a14875d53d9bf0ef5a6fc0257c817f0fb83
```
```
$ etcd -name infra1 -initial-advertise-peer-urls http://10.0.1.11:2380 \
-discovery https://myetcd.local/v2/keys/discovery/6c007a14875d53d9bf0ef5a6fc0257c817f0fb83
```
```
$ etcd -name infra2 -initial-advertise-peer-urls http://10.0.1.12:2380 \
-discovery https://myetcd.local/v2/keys/discovery/6c007a14875d53d9bf0ef5a6fc0257c817f0fb83
```
This will cause each member to register itself with the custom etcd discovery service and begin the cluster once all machines have been registered.
### Public discovery service
If you do not have access to an existing cluster you can use the public discovery service hosted at discovery.etcd.io. You can create a private discovery URL using the "new" endpoint like so:
```
$ curl https://discovery.etcd.io/new?size=3
https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
```
This will create the cluster with an initial expected size of 3 members. If you do not specify a size a default of 3 will be used.
```
ETCD_DISCOVERY=https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
```
```
-discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
```
Now we start etcd with those relevant flags for each member:
```
$ etcd -name infra0 -initial-advertise-peer-urls http://10.0.1.10:2380 \
-discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
```
```
$ etcd -name infra1 -initial-advertise-peer-urls http://10.0.1.11:2380 \
-discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
```
```
$ etcd -name infra2 -initial-advertise-peer-urls http://10.0.1.12:2380 \
-discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
```
This will cause each member to register itself with the discovery service and begin the cluster once all members have been registered.
You can use the environment variable `ETCD_DISCOVERY_PROXY` to cause etcd to use an HTTP proxy to connect to the discovery service.
### Error and Warning Cases
#### Discovery Server Errors
```
$ etcd -name infra0 -initial-advertise-peer-urls http://10.0.1.10:2380 \
-discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
etcd: error: the cluster doesnt have a size configuration value in https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de/_config
exit 1
```
#### User Errors
```
$ etcd -name infra0 -initial-advertise-peer-urls http://10.0.1.10:2380 \
-discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
etcd: error: the cluster using discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de has already started with all 5 members
exit 1
```
#### Warnings
This is a harmless warning notifying you that the discovery URL will be
ignored on this machine.
```
$ etcd -name infra0 -initial-advertise-peer-urls http://10.0.1.10:2380 \
-discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
etcd: warn: ignoring discovery URL: etcd has already been initialized and has a valid log in /var/lib/etcd
```
# 0.4 to 0.5+ Migration Guide
In etcd 0.5 we introduced the ability to listen on more than one address and to advertise multiple addresses. This makes using etcd easier when you have complex networking, such as private and public networks on various cloud providers.
To make understanding this feature easier, we changed the naming of some flags, but we support the old flags to make the migration from the old to new version easier.
|Old Flag |New Flag |Migration Behavior |
|-----------------------|-----------------------|---------------------------------------------------------------------------------------|
|-peer-addr |-initial-advertise-peer-urls |If specified, peer-addr will be used as the only peer URL. Error if both flags specified.|
|-addr |-advertise-client-urls |If specified, addr will be used as the only client URL. Error if both flags specified.|
|-peer-bind-addr |-listen-peer-urls |If specified, peer-bind-addr will be used as the only peer bind URL. Error if both flags specified.|
|-bind-addr |-listen-client-urls |If specified, bind-addr will be used as the only client bind URL. Error if both flags specified.|
|-peers |none |Deprecated. The -initial-cluster flag provides a similar concept with different semantics. Please read this guide on cluster startup.|
|-peers-file |none |Deprecated. The -initial-cluster flag provides a similar concept with different semantics. Please read this guide on cluster startup.|

View File

@ -0,0 +1,31 @@
## Glossary
This document defines the various terms used in etcd documentation, command line and source code.
### Node
Node is an instance of raft state machine.
It has a unique identification, and records other nodes' progress internally when it is the leader.
### Member
Member is an instance of etcd. It hosts a node, and provides service to clients.
### Cluster
Cluster consists of several members.
The node in each member follows raft consensus protocol to replicate logs. Cluster receives proposals from members, commits them and apply to local store.
### Peer
Peer is another member of the same cluster.
### Client
Client is a caller of the cluster's HTTP API.
### Machine (deprecated)
The alternative of Member in etcd before 0.5

View File

@ -0,0 +1,96 @@
## Members API
* [List members](#list-members)
* [Add a member](#add-a-member)
* [Delete a member](#delete-a-member)
## List members
Return an HTTP 200 OK response code and a representation of all members in the etcd cluster.
### Request
```
GET /v2/members HTTP/1.1
```
### Example
```
curl http://10.0.0.10:2379/v2/members
```
```json
{
"members": [
{
"id": "272e204152",
"name": "infra1",
"peerURLs": [
"http://10.0.0.10:2380"
],
"clientURLs": [
"http://10.0.0.10:2379"
]
},
{
"id": "2225373f43",
"name": "infra2",
"peerURLs": [
"http://10.0.0.11:2380"
],
"clientURLs": [
"http://10.0.0.11:2379"
]
},
]
}
```
## Add a member
Returns an HTTP 201 response code and the representation of added member with a newly generated a memberID when successful. Returns a string describing the failure condition when unsuccessful.
If the POST body is malformed an HTTP 400 will be returned. If the member exists in the cluster or existed in the cluster at some point in the past an HTTP 409 will be returned. If any of the given peerURLs exists in the cluster an HTTP 409 will be returned. If the cluster fails to process the request within timeout an HTTP 500 will be returned, though the request may be processed later.
### Request
```
POST /v2/members HTTP/1.1
{"peerURLs": ["http://10.0.0.10:2379"]}
```
### Example
```
curl http://10.0.0.10:2379/v2/members -XPOST -H "Content-Type: application/json" -d '{"peerURLs":["http://10.0.0.10:2379"]}'
```
```json
{
"id": "3777296169",
"peerURLs": [
"http://10.0.0.10:2379"
]
}
```
## Delete a member
Remove a member from the cluster. The member ID must be a hex-encoded uint64.
Returns empty when successful. Returns a string describing the failure condition when unsuccessful.
If the member does not exist in the cluster an HTTP 500(TODO: fix this) will be returned. If the cluster fails to process the request within timeout an HTTP 500 will be returned, though the request may be processed later.
### Request
```
DELETE /v2/members/<id> HTTP/1.1
```
### Example
```
curl http://10.0.0.10:2379/v2/members/272e204152 -XDELETE
```

View File

@ -0,0 +1,143 @@
## Runtime Reconfiguration
etcd comes with support for incremental runtime reconfiguration, which allows users to update the membership of the cluster at run time.
## Reconfiguration Use Cases
Let us walk through the four use cases for re-configuring a cluster: replacing a member, increasing or decreasing cluster size, and restarting a cluster from a majority failure.
### Replace a Member
The most common use case of cluster reconfiguration is to replace a member because of a permanent failure of the existing member: for example, hardware failure, loss of network address, or data directory corruption.
It is important to replace failed members as soon as the failure is detected.
If etcd falls below a simple majority of members it can no longer accept writes: e.g. in a 3 member cluster the loss of two members will cause writes to fail and the cluster to stop operating.
### Increase Cluster Size
To make your cluster more resilient to machine failure you can increase the size of the cluster.
For example, if the cluster consists of three machines, it can tolerate one failure.
If we increase the cluster size to five, it can tolerate two machine failures.
Increasing the cluster size can also provide better read performance.
When a client accesses etcd, the normal read gets the data from the local copy of each member (members always shares the same view of the cluster at the same index, which is guaranteed by the sequential consistency of etcd).
Since clients can read from any member, increasing the number of members thus increases overall read throughput.
### Decrease Cluster Size
To improve the write performance of a cluster, you might want to trade off resilience by removing members.
etcd replicates the data to the majority of members of the cluster before committing the write.
Decreasing the cluster size means the etcd cluster has to do less work for each write, thus increasing the write performance.
### Restart Cluster from Majority Failure
If the majority of your cluster is lost, then you need to take manual action in order to recover safely.
The basic steps in the recovery process include creating a new cluster using the old data, forcing a single member to act as the leader, and finally using runtime configuration to add members to this new cluster.
TODO: https://github.com/coreos/etcd/issues/1242
## Cluster Reconfiguration Operations
Now that we have the use cases in mind, let us lay out the operations involved in each.
Before making any change, the simple majority (quorum) of etcd members must be available.
This is essentially the same requirement as for any other write to etcd.
All changes to the cluster are done one at a time:
To replace a single member you will make an add then a remove operation
To increase from 3 to 5 members you will make two add operations
To decrease from 5 to 3 you will make two remove operations
All of these examples will use the `etcdctl` command line tool that ships with etcd.
If you want to use the member API directly you can find the documentation [here](https://github.com/coreos/etcd/blob/master/Documentation/0.5/other_apis.md).
### Remove a Member
First, we need to find the target member:
```
$ etcdctl member list
6e3bd23ae5f1eae0: name=node2 peerURLs=http://localhost:7002 clientURLs=http://127.0.0.1:4002
924e2e83e93f2560: name=node3 peerURLs=http://localhost:7003 clientURLs=http://127.0.0.1:4003
a8266ecf031671f3: name=node1 peerURLs=http://localhost:7001 clientURLs=http://127.0.0.1:4001
```
Let us say the member ID we want to remove is a8266ecf031671f3.
We then use the `remove` command to perform the removal:
```
$ etcdctl member remove a8266ecf031671f3
Removed member a8266ecf031671f3 from cluster
```
The target member will stop itself at this point and print out the removal in the log:
```
etcd: this member has been permanently removed from the cluster. Exiting.
```
Removal of the leader is safe, but the cluster will be out of progress for a period of election timeout because it needs to elect the new leader.
### Add a Member
Adding a member is a two step process:
* Add the new member to the cluster via the [members API](https://github.com/coreos/etcd/blob/master/Documentation/0.5/other_apis.md#post-v2members) or the `etcdctl member add` command.
* Start the member with the correct configuration.
Using `etcdctl` let's add the new member to the cluster:
```
$ etcdctl member add infra3 http://10.0.1.13:2380
added member 9bf1b35fc7761a23 to cluster
ETCD_NAME="infra3"
ETCD_INITIAL_CLUSTER="infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380,infra3=http://10.0.1.13:2380"
ETCD_INITIAL_CLUSTER_STATE=existing
```
> Notice that infra3 was added to the cluster using its advertised peer URL.
Now start the new etcd process with the relevant flags for the new member:
```
$ export ETCD_NAME="infra3"
$ export ETCD_INITIAL_CLUSTER="infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380,infra3=http://10.0.1.13:2380"
$ export ETCD_INITIAL_CLUSTER_STATE=existing
$ etcd -listen-client-urls http://10.0.1.13:2379 -advertise-client-urls http://10.0.1.13:2379 -listen-peer-urls http://10.0.1.13:2380 -initial-advertise-peer-urls http://10.0.1.13:2380
```
The new member will run as a part of the cluster and immediately begin catching up with the rest of the cluster.
If you are adding multiple members the best practice is to configure the new member, then start the process, then configure the next, and so on.
A common case is increasing a cluster from 1 to 3: if you add one member to a 1-node cluster, the cluster cannot make progress before the new member starts because it needs two members as majority to agree on the consensus.
#### Error Cases
In the following case we have not included our new host in the list of enumerated nodes.
If this is a new cluster, the node must be added to the list of initial cluster members.
```
$ etcd -name infra3 \
-initial-cluster infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380 \
-initial-cluster-state existing
etcdserver: assign ids error: the member count is unequal
exit 1
```
In this case we give a different address (10.0.1.14:2380) to the one that we used to join the cluster (10.0.1.13:2380).
```
$ etcd -name infra4 \
-initial-cluster infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380,infra4=http://10.0.1.14:2380 \
-initial-cluster-state existing
etcdserver: assign ids error: unmatched member while checking PeerURLs
exit 1
```
When we start etcd using the data directory of a removed member, etcd will exit automatically if it connects to any alive member in the cluster:
```
$ etcd
etcd: this member has been permanently removed from the cluster. Exiting.
exit 1
```

View File

@ -126,10 +126,10 @@ curl -L http://127.0.0.1:4001/v2/keys/message -XPUT -d value="Hello etcd"
"value": "Hello etcd"
},
"prevNode": {
"createdIndex": 2
"createdIndex": 2,
"key": "/message",
"value": "Hello world",
"modifiedIndex": 2,
"modifiedIndex": 2
}
}
```
@ -206,7 +206,7 @@ If the TTL has expired, the key will have been deleted, and you will be returned
"cause": "/foo",
"errorCode": 100,
"index": 6,
"message": "Key Not Found"
"message": "Key not found"
}
```
@ -224,7 +224,7 @@ curl -L http://127.0.0.1:4001/v2/keys/foo -XPUT -d value=bar -d ttl= -d prevExis
"key": "/foo",
"modifiedIndex": 6,
"value": "bar"
}
},
"prevNode": {
"createdIndex": 5,
"expiration": "2013-12-04T12:01:21.874888581-08:00",
@ -383,7 +383,7 @@ curl -L http://127.0.0.1:4001/v2/keys/dir -XPUT -d ttl=30 -d dir=true
"createdIndex": 17,
"dir": true,
"expiration": "2013-12-11T10:37:33.689275857-08:00",
"key": "/newdir",
"key": "/dir",
"modifiedIndex": 17,
"ttl": 30
}
@ -417,7 +417,7 @@ curl 'http://127.0.0.1:4001/v2/keys/dir/asdf?consistent=true&wait=true'
"dir":true,
"modifiedIndex": 17,
"expiration": "2013-12-11T10:39:35.689275857-08:00"
},
}
}
```
@ -457,7 +457,7 @@ The error code explains the problem:
"cause": "/foo",
"errorCode": 105,
"index": 39776,
"message": "Already exists"
"message": "Key already exists"
}
```
@ -474,7 +474,7 @@ This will try to compare the previous value of the key and the previous value we
"cause": "[two != one]",
"errorCode": 101,
"index": 8,
"message": "Test Failed"
"message": "Compare failed"
}
```
@ -643,16 +643,22 @@ We should see the response as an array of items:
```json
{
"action": "get",
"node": {
"dir": true,
"key": "/",
"nodes": [
{
"createdIndex": 2,
"dir": true,
"key": "/foo_dir",
"modifiedIndex": 2
}
    "node": {
     "key": "/",
       "dir": true,
       "nodes": [
         {
             "key": "/foo_dir",
             "dir": true,
             "modifiedIndex": 2,
             "createdIndex": 2
          },
          {
             "key": "/foo",
             "value": "two",
             "modifiedIndex": 1,
             "createdIndex": 1
          }
]
}
}
@ -667,27 +673,33 @@ curl -L http://127.0.0.1:4001/v2/keys/?recursive=true
```json
{
"action": "get",
"node": {
"dir": true,
"key": "/",
"nodes": [
{
"createdIndex": 2,
"dir": true,
"key": "/foo_dir",
"modifiedIndex": 2,
"nodes": [
{
"createdIndex": 2,
"key": "/foo_dir/foo",
"modifiedIndex": 2,
"value": "bar"
}
]
}
]
}
    "action": "get",
    "node": {
        "key": "/",
        "dir": true,
        "nodes": [
            {
                "key": "/foo_dir",
                "dir": true,
                "nodes": [
                    {
                        "key": "/foo_dir/foo",
                        "value": "bar",
                        "modifiedIndex": 2,
                        "createdIndex": 2
                    }
                ],
                "modifiedIndex": 2,
                "createdIndex": 2
            },
            {
                "key": "/foo",
                "value": "two",
                "modifiedIndex": 1,
                "createdIndex": 1
            }
        ]
    }
}
```

View File

@ -1,46 +1,43 @@
# Client libraries support matrix for etcd
As etcd features support is really uneven between client libraries, a compatibility matrix can be important.
We will consider in detail only the features of clients supporting the v2 API. Clients still supporting the v1 API *only* are listed below.
## v1-only clients
Clients supporting only the API version 1
- [justinsb/jetcd](https://github.com/justinsb/jetcd) Java
- [transitorykris/etcd-py](https://github.com/transitorykris/etcd-py) Python
- [russellhaering/txetcd](https://github.com/russellhaering/txetcd) Python
- [iconara/etcd-rb](https://github.com/iconara/etcd-rb) Ruby
- [jpfuentes2/etcd-ruby](https://github.com/jpfuentes2/etcd-ruby) Ruby
- [aterreno/etcd-clojure](https://github.com/aterreno/etcd-clojure) Clojure
- [marshall-lee/etcd.erl](https://github.com/marshall-lee/etcd.erl) Erlang
## v2 clients
The v2 API has a lot of features, we will categorize them in a few categories:
- **Language**: The language in which the client library was written.
- **HTTPS Auth**: Support for SSL-certificate based authentication
- **Reconnect**: If the client is able to reconnect automatically to another server if one fails.
- **Mod/Lock**: Support for the locking module
- **Mod/Leader**: Support for the leader election module
- **GET,PUT,POST,DEL Features**: Support for all the modifiers when calling the etcd server with said HTTP method.
### Supported features matrix
| Client| [go-etcd](https://github.com/coreos/go-etcd) | [jetcd](https://github.com/diwakergupta/jetcd) | [python-etcd](https://github.com/jplana/python-etcd) | [python-etcd-client](https://github.com/dsoprea/PythonEtcdClient) | [node-etcd](https://github.com/stianeikeland/node-etcd) | [nodejs-etcd](https://github.com/lavagetto/nodejs-etcd) | [etcd-ruby](https://github.com/ranjib/etcd-ruby) | [etcd-api](https://github.com/jdarcy/etcd-api) | [cetcd](https://github.com/dwwoelfel/cetcd) | [clj-etcd](https://github.com/rthomas/clj-etcd) | [etcetera](https://github.com/drusellers/etcetera)| [Etcd.jl](https://github.com/forio/Etcd.jl) | [p5-etcd](https://metacpan.org/release/Etcd)
| --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |
| **HTTPS Auth** | Y | Y | Y | Y | Y | Y | - | - | - | - | - | - | - |
| **Reconnect** | Y | - | Y | Y | - | - | - | Y | - | - | - | - | - |
| **Mod/Lock** | - | - | Y | Y | - | - | - | - | - | - | - | Y | - |
| **Mod/Leader** | - | - | - | Y | - | - | - | - | - | - | - | Y | - |
| **GET Features** | F | B | F | F | F | F | F | B | F | G | F | F | F |
| **PUT Features** | F | B | F | F | F | F | F | G | F | G | F | F | F |
| **POST Features** | F | - | F | F | - | F | F | - | - | - | F | F | F |
| **DEL Features** | F | B | F | F | F | F | F | B | G | B | F | F | F |
**Legend**
**F**: Full support **G**: Good support **B**: Basic support
**Y**: Feature supported **-**: Feature not supported
Sorted alphabetically on language/name
|Client |**Language**|**HTTPS Auth**|**Re-connect**|**GET**|**PUT**|**POST**|**DEL**|**Mod Lock**|**Mod Leader**|
| --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |
|[etcd-api](https://github.com/jdarcy/etcd-api) |C |-|Y|B|G|-|B|-|-|
|[etcdcpp](https://github.com/edwardcapriolo/etcdcpp) |C++ |-|-|F|F|G|-|-|-|
|[cetcd](https://github.com/dwwoelfel/cetcd) |Clojure|-|-|F|F|-|G|-|-|
|[clj-etcd](https://github.com/rthomas/clj-etcd) |Clojure|-|-|G|G|-|B|-|-|
|[etcd-clojure](https://github.com/aterreno/etcd-clojure) |Clojure|-|-|F|F|F|F|-|-|
|[go-etcd](https://github.com/coreos/go-etcd) |go |Y|Y|F|F|F|F|-|-|
|[boon etcd client](https://github.com/boonproject/boon/blob/master/etcd/README.md) |java |Y|Y|F|F|F|F|-|F|
|[etcd4j](https://github.com/jurmous/etcd4j) |java |Y|Y|F|F|F|F|-|-|
|[jetcd](https://github.com/diwakergupta/jetcd) |java |Y|-|B|B|-|B|-|-|
|[jetcd](https://github.com/justinsb/jetcd) |java |-|-|B|B|-|B|-|-|
|[Etcd.jl](https://github.com/forio/Etcd.jl) |Julia |-|-|F|F|F|F|Y|Y|
|[etcetera](https://github.com/drusellers/etcetera) |.net |-|-|F|F|F|F|-|-|
|[node-etcd](https://github.com/stianeikeland/node-etcd) |nodejs |Y|-|F|F|-|F|-|-|
|[nodejs-etcd](https://github.com/lavagetto/nodejs-etcd) |nodejs |Y|-|F|F|F|F|-|-|
|[p5-etcd](https://metacpan.org/release/Etcd) |perl |-|-|F|F|F|F|-|-|
|[python-etcd](https://github.com/jplana/python-etcd) |python |Y|Y|F|F|F|F|Y|-|
|[python-etcd-client](https://github.com/dsoprea/PythonEtcdClient)|python |Y|Y|F|F|F|F|Y|Y|
|[txetcd](https://github.com/russellhaering/txetcd) |python |-|-|G|G|F|G|-|-|
|[etcd-ruby](https://github.com/ranjib/etcd-ruby) |ruby |-|-|F|F|F|F|-|-|

View File

@ -13,11 +13,11 @@ Please note - at least 3 nodes are required for [cluster availability][optimal-c
## Using discovery.etcd.io
### Create a Token
### Create a Discovery URL
To use the discovery API, you must first create a token for your etcd cluster. Visit [https://discovery.etcd.io/new](https://discovery.etcd.io/new) to create a new token.
To use the discovery API, you must first create a unique discovery URL for your etcd cluster. Visit [https://discovery.etcd.io/new](https://discovery.etcd.io/new) to create a new discovery URL.
You can inspect the list of peers by viewing `https://discovery.etcd.io/<token>`.
You can inspect the list of peers by viewing `https://discovery.etcd.io/<cluster id>`.
### Start etcd With the Discovery Flag
@ -26,10 +26,10 @@ Specify the `-discovery` flag when you start each etcd instance. The list of exi
Here's a full example:
```
TOKEN=$(curl https://discovery.etcd.io/new)
./etcd -name instance1 -peer-addr 10.1.2.3:7001 -addr 10.1.2.3:4001 -discovery $TOKEN
./etcd -name instance2 -peer-addr 10.1.2.4:7001 -addr 10.1.2.4:4001 -discovery $TOKEN
./etcd -name instance3 -peer-addr 10.1.2.5:7001 -addr 10.1.2.5:4001 -discovery $TOKEN
URL=$(curl https://discovery.etcd.io/new)
./etcd -name instance1 -peer-addr 10.1.2.3:7001 -addr 10.1.2.3:4001 -discovery $URL
./etcd -name instance2 -peer-addr 10.1.2.4:7001 -addr 10.1.2.4:4001 -discovery $URL
./etcd -name instance3 -peer-addr 10.1.2.5:7001 -addr 10.1.2.5:4001 -discovery $URL
```
## Running Your Own Discovery Endpoint
@ -37,10 +37,10 @@ TOKEN=$(curl https://discovery.etcd.io/new)
The discovery API communicates with a separate etcd cluster to store and retrieve the list of peers. CoreOS provides [https://discovery.etcd.io](https://discovery.etcd.io) as a free service, but you can easily run your own etcd cluster for this purpose. Here's an example using an etcd cluster located at `10.10.10.10:4001`:
```
TOKEN="testcluster"
./etcd -name instance1 -peer-addr 10.1.2.3:7001 -addr 10.1.2.3:4001 -discovery http://10.10.10.10:4001/v2/keys/$TOKEN
./etcd -name instance2 -peer-addr 10.1.2.4:7001 -addr 10.1.2.4:4001 -discovery http://10.10.10.10:4001/v2/keys/$TOKEN
./etcd -name instance3 -peer-addr 10.1.2.5:7001 -addr 10.1.2.5:4001 -discovery http://10.10.10.10:4001/v2/keys/$TOKEN
URL="http://10.10.10.10:4001/v2/keys/testcluster"
./etcd -name instance1 -peer-addr 10.1.2.3:7001 -addr 10.1.2.3:4001 -discovery $URL
./etcd -name instance2 -peer-addr 10.1.2.4:7001 -addr 10.1.2.4:4001 -discovery $URL
./etcd -name instance3 -peer-addr 10.1.2.5:7001 -addr 10.1.2.5:4001 -discovery $URL
```
If you're interested in how to discovery API works behind the scenes, read about the [Discovery Protocol](https://github.com/coreos/etcd/blob/master/Documentation/discovery-protocol.md).

View File

@ -167,3 +167,9 @@ Etcd can also do internal server-to-server communication using SSL client certs.
To do this just change the `-*-file` flags to `-peer-*-file`.
If you are using SSL for server-to-server communication, you must use it on all instances of etcd.
### Bootstrapping a new cluster by name
An etcd server is uniquely defined by the peer addresses it listens to. Suppose, however, that you wish to start over, while maintaining the data from the previous cluster -- that is, to pretend that this machine has never joined a cluster before.
You can use `--initial-cluster-name` to generate a new unique ID for each node, as a shared token that every node understands. Nodes also take this into account for bootstrapping the new cluster ID, so it also provides a way for a machine to listen on the same interfaces, disconnect from one cluster, and join a different cluster.

View File

@ -75,8 +75,8 @@ cors = []
cpu_profile_file = ""
data_dir = "."
discovery = "http://etcd.local:4001/v2/keys/_etcd/registry/examplecluster"
http_read_timeout = 10
http_write_timeout = 10
http_read_timeout = 10.0
http_write_timeout = 10.0
key_file = ""
peers = []
peers_file = ""
@ -84,7 +84,7 @@ max_cluster_size = 9
max_result_buffer = 1024
max_retry_attempts = 3
name = "default-name"
snapshot = false
snapshot = true
verbose = false
very_verbose = false
@ -112,8 +112,8 @@ sync_interval = 5.0
* `ETCD_CPU_PROFILE_FILE`
* `ETCD_DATA_DIR`
* `ETCD_DISCOVERY`
* `ETCD_CLUSTER_HTTP_READ_TIMEOUT`
* `ETCD_CLUSTER_HTTP_WRITE_TIMEOUT`
* `ETCD_HTTP_READ_TIMEOUT`
* `ETCD_HTTP_WRITE_TIMEOUT`
* `ETCD_KEY_FILE`
* `ETCD_PEERS`
* `ETCD_PEERS_FILE`

View File

@ -1,12 +0,0 @@
# Development tools
## Vagrant
For fast start you can use Vagrant. `vagrant up` will make etcd build and running on virtual machine. Required Vagrant version is 1.5.0.
Next lets set a single key and then retrieve it:
```
curl -L http://127.0.0.1:4001/v2/keys/mykey -XPUT -d value="this is awesome"
curl -L http://127.0.0.1:4001/v2/keys/mykey
```

View File

@ -3,8 +3,10 @@
**Tools**
- [etcdctl](https://github.com/coreos/etcdctl) - A command line client for etcd
- [etcd-backup](https://github.com/fanhattan/etcd-backup) - A powerful command line utility for dumping/restoring etcd - Supports v2
- [etcd-dump](https://npmjs.org/package/etcd-dump) - Command line utility for dumping/restoring etcd.
- [etcd-fs](https://github.com/xetorthio/etcd-fs) - FUSE filesystem for etcd
- [etcd-browser](https://github.com/henszey/etcd-browser) - A web-based key/value editor for etcd using AngularJS
**Go libraries**
@ -12,8 +14,12 @@
**Java libraries**
- [boonproject/etcd](https://github.com/boonproject/boon/blob/master/etcd/README.md) - Supports v2, Async/Sync and waits
- [justinsb/jetcd](https://github.com/justinsb/jetcd)
- [diwakergupta/jetcd](https://github.com/diwakergupta/jetcd) - Supports v2
- [jurmous/etcd4j](https://github.com/jurmous/etcd4j) - Supports v2
- [AdoHe/etcd4j](http://github.com/AdoHe/etcd4j) - Supports v2 (enhance for real production cluster)
**Python libraries**
@ -36,6 +42,9 @@
- [jdarcy/etcd-api](https://github.com/jdarcy/etcd-api) - Supports v2
**C++ libraries**
- [edwardcapriolo/etcdcpp](https://github.com/edwardcapriolo/etcdcpp) - Supports v2
**Clojure libraries**
- [aterreno/etcd-clojure](https://github.com/aterreno/etcd-clojure)
@ -92,3 +101,6 @@ A detailed recap of client functionalities can be found in the [clients compatib
- [GoogleCloudPlatform/kubernetes](https://github.com/GoogleCloudPlatform/kubernetes) - Container cluster manager.
- [mailgun/vulcand](https://github.com/mailgun/vulcand) - HTTP proxy that uses etcd as a configuration backend.
- [duedil-ltd/discodns](https://github.com/duedil-ltd/discodns) - Simple DNS nameserver using etcd as a database for names and records.
- [skynetservices/skydns](https://github.com/skynetservices/skydns) - RFC compliant DNS server
- [xordataexchange/crypt](https://github.com/xordataexchange/crypt) - Securely store values in etcd using GPG encryption
- [spf13/viper](https://github.com/spf13/viper) - Go configuration library, reads values from ENV, pflags, files, and etcd with optional encryption

View File

@ -1,34 +1,50 @@
# Reading and Writing over HTTPS
# Etcd security model
## Transport Security with HTTPS
Etcd supports SSL/TLS as well as authentication through client certificates, both for clients to server as well as peer (server to server / cluster) communication.
Etcd supports SSL/TLS and client cert authentication for clients to server, as well as server to server communication.
To get up and running you first need to have a CA certificate and a signed key pair for your node. It is recommended to create and sign a new key pair for every node in a cluster.
For convenience the [etcd-ca](https://github.com/coreos/etcd-ca) tool provides an easy interface to certificate generation, alternatively this site provides a good reference on how to generate self-signed key pairs:
First, you need to have a CA cert `clientCA.crt` and signed key pair `client.crt`, `client.key`.
This site has a good reference for how to generate self-signed key pairs:
http://www.g-loaded.eu/2005/11/10/be-your-own-ca/
Or you could use [etcd-ca](https://github.com/coreos/etcd-ca) to generate certs and keys.
For testing you can use the certificates in the `fixtures/ca` directory.
## Basic setup
Let's configure etcd to use this keypair:
Etcd takes several certificate related configuration options, either through command-line flags or environment variables:
**Client-to-server communication:**
`--cert-file=<path>`: Certificate used for SSL/TLS connections **to** etcd. When this option is set, you can reach etcd through HTTPS - for example at `https://127.0.0.1:4001`
`--key-file=<path>`: Key for the certificate. Must be unencrypted.
`--ca-file=<path>`: When this is set etcd will check all incoming HTTPS requests for a client certificate signed by the supplied CA, requests that don't supply a valid client certificate will fail.
**Peer (server-to-server / cluster) communication:**
The peer options work the same way as the client-to-server options:
`--peer-cert-file=<path>`: Certificate used for SSL/TLS connections between peers. This will be used both for listening on the peer address as well as sending requests to other peers.
`--peer-key-file=<path>`: Key for the certificate. Must be unencrypted.
`--peer-ca-file=<path>`: When set, etcd will check all incoming peer requests from the cluster for valid client certificates signed by the supplied CA.
If either a client-to-server or peer certificate is supplied the key must also be set. All of these configuration options are also available through the environment variables, `ETCD_CA_FILE`, `ETCD_PEER_CA_FILE` and so on.
## Example 1: Client-to-server transport security with HTTPS
For this you need your CA certificate (`ca.crt`) and signed key pair (`server.crt`, `server.key`) ready. If you just want to test the functionality, there are example certificates provided in the [etcd git repository](https://github.com/coreos/etcd/tree/master/fixtures/ca) (namely `server.crt` and `server.key.insecure`).
Assuming you have these files ready, let's configure etcd to use them to provide simple HTTPS transport security.
```sh
./etcd -f -name machine0 -data-dir machine0 -cert-file=./fixtures/ca/server.crt -key-file=./fixtures/ca/server.key.insecure
etcd -name machine0 -data-dir machine0 -cert-file=/path/to/server.crt -key-file=/path/to/server.key
```
There are a few new options we're using:
* `-f` - forces a new machine configuration, even if an existing configuration is found. (WARNING: data loss!)
* `-cert-file` and `-key-file` specify the location of the cert and key files to be used for for transport layer security between the client and server.
You can now test the configuration using HTTPS:
This should start up fine and you can now test the configuration by speaking HTTPS to etcd:
```sh
curl --cacert ./fixtures/ca/server-chain.pem https://127.0.0.1:4001/v2/keys/foo -XPUT -d value=bar -v
curl --cacert /path/to/ca.crt https://127.0.0.1:4001/v2/keys/foo -XPUT -d value=bar -v
```
You should be able to see the handshake succeed.
You should be able to see the handshake succeed. Because we use self-signed certificates with our own certificate authorities you need to provide the CA to curl using the `--cacert` option. Another possibility would be to add your CA certificate to the trusted certificates on your system (usually in `/etc/ssl/certs`).
**OSX 10.9+ Users**: curl 7.30.0 on OSX 10.9+ doesn't understand certificates passed in on the command line.
Instead you must import the dummy ca.crt directly into the keychain or add the `-k` flag to curl to ignore errors.
@ -36,42 +52,28 @@ If you want to test without the `-k` flag run `open ./fixtures/ca/ca.crt` and fo
Please remove this certificate after you are done testing!
If you know of a workaround let us know.
```
...
SSLv3, TLS handshake, Finished (20):
...
```
## Example 2: Client-to-server authentication with HTTPS client certificates
And also the response from the etcd server:
For now we've given the etcd client the ability to verify the server identity and provide transport security. We can however also use client certificates to prevent unauthorized access to etcd.
```json
{
"action": "set",
"key": "/foo",
"modifiedIndex": 3,
"value": "bar"
}
```
The clients will provide their certificates to the server and the server will check whether the cert is signed by the supplied CA and decide whether to serve the request.
## Authentication with HTTPS Client Certificates
We can also do authentication using CA certs.
The clients will provide their cert to the server and the server will check whether the cert is signed by the CA and decide whether to serve the request.
You need the same files mentioned in the first example for this, as well as a key pair for the client (`client.crt`, `client.key`) signed by the same certificate authority.
```sh
./etcd -f -name machine0 -data-dir machine0 -ca-file=./fixtures/ca/ca.crt -cert-file=./fixtures/ca/server.crt -key-file=./fixtures/ca/server.key.insecure
etcd -name machine0 -data-dir machine0 -ca-file=/path/to/ca.crt -cert-file=/path/to/server.crt -key-file=/path/to/server.key
```
```-ca-file``` is the path to the CA cert.
Notice that the addition of the `-ca-file` option automatically enables client certificate checking.
Try the same request to this server:
Now try the same request as above to this server:
```sh
curl --cacert ./fixtures/ca/server-chain.pem https://127.0.0.1:4001/v2/keys/foo -XPUT -d value=bar -v
curl --cacert /path/to/ca.crt https://127.0.0.1:4001/v2/keys/foo -XPUT -d value=bar -v
```
The request should be rejected by the server.
The request should be rejected by the server:
```
...
@ -79,10 +81,10 @@ routines:SSL3_READ_BYTES:sslv3 alert bad certificate
...
```
We need to give the CA signed cert to the server.
To make it succeed, we need to give the CA signed client certificate to the server:
```sh
curl --key ./fixtures/ca/server2.key.insecure --cert ./fixtures/ca/server2.crt --cacert ./fixtures/ca/server-chain.pem -L https://127.0.0.1:4001/v2/keys/foo -XPUT -d value=bar -v
curl --cacert /path/to/ca.crt --cert /path/to/client.crt --key /path/to/client.key -L https://127.0.0.1:4001/v2/keys/foo -XPUT -d value=bar -v
```
You should able to see:
@ -108,7 +110,28 @@ And also the response from the server:
}
```
### Why SSLv3 alert handshake failure when using SSL client auth?
## Example 3: Transport security & client certificates in a cluster
Etcd supports the same model as above for **peer communication**, that means the communication between etcd nodes in a cluster.
Assuming we have our `ca.crt` and two nodes with their own keypairs (`node1.crt` & `node1.key`, `node2.crt` & `node2.key`) signed by this CA, we launch etcd as follows:
```sh
DISCOVERY_URL=... # from https://discovery.etcd.io/new
# Node1
etcd -name node1 -data-dir node1 -ca-file=/path/to/ca.crt -cert-file=/path/to/node1.crt -key-file=/path/to/node1.key -peer-addr ${node1_public_ip}:7001 -discovery ${DISCOVERY_URL}
# Node2
etcd -name node2 -data-dir node2 -ca-file=/path/to/ca.crt -cert-file=/path/to/node2.crt -key-file=/path/to/node2.key -peer-addr ${node2_public_ip}:7001 -discovery ${DISCOVERY_URL}
```
The etcd nodes will form a cluster and all communication between nodes in the cluster will be encrypted and authenticated using the client certificates. You will see in the output of etcd that the addresses it connects to use HTTPS.
## Frequently Asked Questions
### I'm seeing a SSLv3 alert handshake failure when using SSL client authentication?
The `crypto/tls` package of `golang` checks the key usage of the certificate public key before using it.
To use the certificate public key to do client auth, we need to add `clientAuth` to `Extended Key Usage` when creating the certificate public key.
@ -129,3 +152,8 @@ When creating the cert be sure to reference it in the `-extensions` flag:
```
openssl ca -config openssl.cnf -policy policy_anything -extensions ssl_client -out certs/machine.crt -infiles machine.csr
```
### With peer certificate authentication I receive "certificate is valid for 127.0.0.1, not $MY_IP"
Make sure that you sign your certificates with a Subject Name your node's public IP address. The `etcd-ca` tool for example provides an `--ip=` option for its `new-cert` command.
If you need your certificate to be signed for your node's FQDN in its Subject Name then you could use Subject Alternative Names (short IP SNAs) to add your IP address. This is not [currently supported](https://github.com/coreos/etcd-ca/issues/29) by `etcd-ca` but can be done [with openssl](http://wiki.cacert.org/FAQ/subjectAltName).

View File

@ -3,6 +3,8 @@
The default settings in etcd should work well for installations on a local network where the average network latency is low.
However, when using etcd across multiple data centers or over networks with high latency you may need to tweak the heartbeat interval and election timeout settings.
The network isn't the only source of latency. Each request and response may be impacted by slow disks on both the leader and follower. Each of these timeouts represents the total time from request to successful response from the other machine.
### Time Parameters
The underlying distributed consensus protocol relies on two separate time parameters to ensure that nodes can handoff leadership if one stalls or goes offline.
@ -41,7 +43,7 @@ Or you can set the values within the configuration file:
```toml
[peer]
heartbeat_interval = 100
election_timeout = 100
election_timeout = 500
```
The values are specified in milliseconds.

36
Godeps/Godeps.json generated Normal file
View File

@ -0,0 +1,36 @@
{
"ImportPath": "github.com/coreos/etcd",
"GoVersion": "go1.3.1",
"Packages": [
"./..."
],
"Deps": [
{
"ImportPath": "code.google.com/p/go.net/context",
"Comment": "null-144",
"Rev": "ad01a6fcc8a19d3a4478c836895ffe883bd2ceab"
},
{
"ImportPath": "code.google.com/p/gogoprotobuf/proto",
"Rev": "7fd1620f09261338b6b1ca1289ace83aee0ec946"
},
{
"ImportPath": "github.com/codegangsta/cli",
"Comment": "1.2.0-26-gf7ebb76",
"Rev": "f7ebb761e83e21225d1d8954fde853bf8edd46c4"
},
{
"ImportPath": "github.com/coreos/go-etcd/etcd",
"Comment": "v0.2.0-rc1-127-g6fe04d5",
"Rev": "6fe04d580dfb71c9e34cbce2f4df9eefd1e1241e"
},
{
"ImportPath": "github.com/jonboulle/clockwork",
"Rev": "72f9bd7c4e0c2a40055ab3d0f09654f730cce982"
},
{
"ImportPath": "github.com/stretchr/testify/assert",
"Rev": "9cc77fa25329013ce07362c7742952ff887361f2"
}
]
}

5
Godeps/Readme generated Normal file
View File

@ -0,0 +1,5 @@
This directory tree is generated automatically by godep.
Please do not edit.
See https://github.com/tools/godep for more information.

2
Godeps/_workspace/.gitignore generated vendored Normal file
View File

@ -0,0 +1,2 @@
/pkg
/bin

View File

@ -0,0 +1,431 @@
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package context defines the Context type, which carries deadlines,
// cancelation signals, and other request-scoped values across API boundaries
// and between processes.
//
// Incoming requests to a server should create a Context, and outgoing calls to
// servers should accept a Context. The chain of function calls between must
// propagate the Context, optionally replacing it with a modified copy created
// using WithDeadline, WithTimeout, WithCancel, or WithValue.
//
// Programs that use Contexts should follow these rules to keep interfaces
// consistent across packages and enable static analysis tools to check context
// propagation:
//
// Do not store Contexts inside a struct type; instead, pass a Context
// explicitly to each function that needs it. The Context should be the first
// parameter, typically named ctx:
//
// func DoSomething(ctx context.Context, arg Arg) error {
// // ... use ctx ...
// }
//
// Do not pass a nil Context, even if a function permits it. Pass context.TODO
// if you are unsure about which Context to use.
//
// Use context Values only for request-scoped data that transits processes and
// APIs, not for passing optional parameters to functions.
//
// The same Context may be passed to functions running in different goroutines;
// Contexts are safe for simultaneous use by multiple goroutines.
//
// See http://blog.golang.org/context for example code for a server that uses
// Contexts.
package context
import (
"errors"
"fmt"
"sync"
"time"
)
// A Context carries a deadline, a cancelation signal, and other values across
// API boundaries.
//
// Context's methods may be called by multiple goroutines simultaneously.
type Context interface {
// Deadline returns the time when work done on behalf of this context
// should be canceled. Deadline returns ok==false when no deadline is
// set. Successive calls to Deadline return the same results.
Deadline() (deadline time.Time, ok bool)
// Done returns a channel that's closed when work done on behalf of this
// context should be canceled. Done may return nil if this context can
// never be canceled. Successive calls to Done return the same value.
//
// WithCancel arranges for Done to be closed when cancel is called;
// WithDeadline arranges for Done to be closed when the deadline
// expires; WithTimeout arranges for Done to be closed when the timeout
// elapses.
//
// Done is provided for use in select statements:
//
// // DoSomething calls DoSomethingSlow and returns as soon as
// // it returns or ctx.Done is closed.
// func DoSomething(ctx context.Context) (Result, error) {
// c := make(chan Result, 1)
// go func() { c <- DoSomethingSlow(ctx) }()
// select {
// case res := <-c:
// return res, nil
// case <-ctx.Done():
// return nil, ctx.Err()
// }
// }
//
// See http://blog.golang.org/pipelines for more examples of how to use
// a Done channel for cancelation.
Done() <-chan struct{}
// Err returns a non-nil error value after Done is closed. Err returns
// Canceled if the context was canceled or DeadlineExceeded if the
// context's deadline passed. No other values for Err are defined.
// After Done is closed, successive calls to Err return the same value.
Err() error
// Value returns the value associated with this context for key, or nil
// if no value is associated with key. Successive calls to Value with
// the same key returns the same result.
//
// Use context values only for request-scoped data that transits
// processes and API boundaries, not for passing optional parameters to
// functions.
//
// A key identifies a specific value in a Context. Functions that wish
// to store values in Context typically allocate a key in a global
// variable then use that key as the argument to context.WithValue and
// Context.Value. A key can be any type that supports equality;
// packages should define keys as an unexported type to avoid
// collisions.
//
// Packages that define a Context key should provide type-safe accessors
// for the values stores using that key:
//
// // Package user defines a User type that's stored in Contexts.
// package user
//
// import "code.google.com/p/go.net/context"
//
// // User is the type of value stored in the Contexts.
// type User struct {...}
//
// // key is an unexported type for keys defined in this package.
// // This prevents collisions with keys defined in other packages.
// type key int
//
// // userKey is the key for user.User values in Contexts. It is
// // unexported; clients use user.NewContext and user.FromContext
// // instead of using this key directly.
// var userKey key = 0
//
// // NewContext returns a new Context that carries value u.
// func NewContext(ctx context.Context, u *User) context.Context {
// return context.WithValue(userKey, u)
// }
//
// // FromContext returns the User value stored in ctx, if any.
// func FromContext(ctx context.Context) (*User, bool) {
// u, ok := ctx.Value(userKey).(*User)
// return u, ok
// }
Value(key interface{}) interface{}
}
// Canceled is the error returned by Context.Err when the context is canceled.
var Canceled = errors.New("context canceled")
// DeadlineExceeded is the error returned by Context.Err when the context's
// deadline passes.
var DeadlineExceeded = errors.New("context deadline exceeded")
// An emptyCtx is never canceled, has no values, and has no deadline.
type emptyCtx int
func (emptyCtx) Deadline() (deadline time.Time, ok bool) {
return
}
func (emptyCtx) Done() <-chan struct{} {
return nil
}
func (emptyCtx) Err() error {
return nil
}
func (emptyCtx) Value(key interface{}) interface{} {
return nil
}
func (n emptyCtx) String() string {
switch n {
case background:
return "context.Background"
case todo:
return "context.TODO"
}
return "unknown empty Context"
}
const (
background emptyCtx = 1
todo emptyCtx = 2
)
// Background returns a non-nil, empty Context. It is never canceled, has no
// values, and has no deadline. It is typically used by the main function,
// initialization, and tests, and as the top-level Context for incoming
// requests.
func Background() Context {
return background
}
// TODO returns a non-nil, empty Context. Code should use context.TODO when
// it's unclear which Context to use or it's is not yet available (because the
// surrounding function has not yet been extended to accept a Context
// parameter). TODO is recognized by static analysis tools that determine
// whether Contexts are propagated correctly in a program.
func TODO() Context {
return todo
}
// A CancelFunc tells an operation to abandon its work.
// A CancelFunc does not wait for the work to stop.
// After the first call, subsequent calls to a CancelFunc do nothing.
type CancelFunc func()
// WithCancel returns a copy of parent with a new Done channel. The returned
// context's Done channel is closed when the returned cancel function is called
// or when the parent context's Done channel is closed, whichever happens first.
func WithCancel(parent Context) (ctx Context, cancel CancelFunc) {
c := newCancelCtx(parent)
propagateCancel(parent, &c)
return &c, func() { c.cancel(true, Canceled) }
}
// newCancelCtx returns an initialized cancelCtx.
func newCancelCtx(parent Context) cancelCtx {
return cancelCtx{
Context: parent,
done: make(chan struct{}),
}
}
// propagateCancel arranges for child to be canceled when parent is.
func propagateCancel(parent Context, child canceler) {
if parent.Done() == nil {
return // parent is never canceled
}
if p, ok := parentCancelCtx(parent); ok {
p.mu.Lock()
if p.err != nil {
// parent has already been canceled
child.cancel(false, p.err)
} else {
if p.children == nil {
p.children = make(map[canceler]bool)
}
p.children[child] = true
}
p.mu.Unlock()
} else {
go func() {
select {
case <-parent.Done():
child.cancel(false, parent.Err())
case <-child.Done():
}
}()
}
}
// parentCancelCtx follows a chain of parent references until it finds a
// *cancelCtx. This function understands how each of the concrete types in this
// package represents its parent.
func parentCancelCtx(parent Context) (*cancelCtx, bool) {
for {
switch c := parent.(type) {
case *cancelCtx:
return c, true
case *timerCtx:
return &c.cancelCtx, true
case *valueCtx:
parent = c.Context
default:
return nil, false
}
}
}
// A canceler is a context type that can be canceled directly. The
// implementations are *cancelCtx and *timerCtx.
type canceler interface {
cancel(removeFromParent bool, err error)
Done() <-chan struct{}
}
// A cancelCtx can be canceled. When canceled, it also cancels any children
// that implement canceler.
type cancelCtx struct {
Context
done chan struct{} // closed by the first cancel call.
mu sync.Mutex
children map[canceler]bool // set to nil by the first cancel call
err error // set to non-nil by the first cancel call
}
func (c *cancelCtx) Done() <-chan struct{} {
return c.done
}
func (c *cancelCtx) Err() error {
c.mu.Lock()
defer c.mu.Unlock()
return c.err
}
func (c *cancelCtx) String() string {
return fmt.Sprintf("%v.WithCancel", c.Context)
}
// cancel closes c.done, cancels each of c's children, and, if
// removeFromParent is true, removes c from its parent's children.
func (c *cancelCtx) cancel(removeFromParent bool, err error) {
if err == nil {
panic("context: internal error: missing cancel error")
}
c.mu.Lock()
if c.err != nil {
c.mu.Unlock()
return // already canceled
}
c.err = err
close(c.done)
for child := range c.children {
// NOTE: acquiring the child's lock while holding parent's lock.
child.cancel(false, err)
}
c.children = nil
c.mu.Unlock()
if removeFromParent {
if p, ok := parentCancelCtx(c.Context); ok {
p.mu.Lock()
if p.children != nil {
delete(p.children, c)
}
p.mu.Unlock()
}
}
}
// WithDeadline returns a copy of the parent context with the deadline adjusted
// to be no later than d. If the parent's deadline is already earlier than d,
// WithDeadline(parent, d) is semantically equivalent to parent. The returned
// context's Done channel is closed when the deadline expires, when the returned
// cancel function is called, or when the parent context's Done channel is
// closed, whichever happens first.
//
// Canceling this context releases resources associated with the deadline
// timer, so code should call cancel as soon as the operations running in this
// Context complete.
func WithDeadline(parent Context, deadline time.Time) (Context, CancelFunc) {
if cur, ok := parent.Deadline(); ok && cur.Before(deadline) {
// The current deadline is already sooner than the new one.
return WithCancel(parent)
}
c := &timerCtx{
cancelCtx: newCancelCtx(parent),
deadline: deadline,
}
propagateCancel(parent, c)
d := deadline.Sub(time.Now())
if d <= 0 {
c.cancel(true, DeadlineExceeded) // deadline has already passed
return c, func() { c.cancel(true, Canceled) }
}
c.mu.Lock()
defer c.mu.Unlock()
if c.err == nil {
c.timer = time.AfterFunc(d, func() {
c.cancel(true, DeadlineExceeded)
})
}
return c, func() { c.cancel(true, Canceled) }
}
// A timerCtx carries a timer and a deadline. It embeds a cancelCtx to
// implement Done and Err. It implements cancel by stopping its timer then
// delegating to cancelCtx.cancel.
type timerCtx struct {
cancelCtx
timer *time.Timer // Under cancelCtx.mu.
deadline time.Time
}
func (c *timerCtx) Deadline() (deadline time.Time, ok bool) {
return c.deadline, true
}
func (c *timerCtx) String() string {
return fmt.Sprintf("%v.WithDeadline(%s [%s])", c.cancelCtx.Context, c.deadline, c.deadline.Sub(time.Now()))
}
func (c *timerCtx) cancel(removeFromParent bool, err error) {
c.cancelCtx.cancel(removeFromParent, err)
c.mu.Lock()
if c.timer != nil {
c.timer.Stop()
c.timer = nil
}
c.mu.Unlock()
}
// WithTimeout returns WithDeadline(parent, time.Now().Add(timeout)).
//
// Canceling this context releases resources associated with the deadline
// timer, so code should call cancel as soon as the operations running in this
// Context complete:
//
// func slowOperationWithTimeout(ctx context.Context) (Result, error) {
// ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond)
// defer cancel() // releases resources if slowOperation completes before timeout elapses
// return slowOperation(ctx)
// }
func WithTimeout(parent Context, timeout time.Duration) (Context, CancelFunc) {
return WithDeadline(parent, time.Now().Add(timeout))
}
// WithValue returns a copy of parent in which the value associated with key is
// val.
//
// Use context Values only for request-scoped data that transits processes and
// APIs, not for passing optional parameters to functions.
func WithValue(parent Context, key interface{}, val interface{}) Context {
return &valueCtx{parent, key, val}
}
// A valueCtx carries a key-value pair. It implements Value for that key and
// delegates all other calls to the embedded Context.
type valueCtx struct {
Context
key, val interface{}
}
func (c *valueCtx) String() string {
return fmt.Sprintf("%v.WithValue(%#v, %#v)", c.Context, c.key, c.val)
}
func (c *valueCtx) Value(key interface{}) interface{} {
if c.key == key {
return c.val
}
return c.Context.Value(key)
}

View File

@ -0,0 +1,553 @@
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package context
import (
"fmt"
"math/rand"
"runtime"
"strings"
"sync"
"testing"
"time"
)
// otherContext is a Context that's not one of the types defined in context.go.
// This lets us test code paths that differ based on the underlying type of the
// Context.
type otherContext struct {
Context
}
func TestBackground(t *testing.T) {
c := Background()
if c == nil {
t.Fatalf("Background returned nil")
}
select {
case x := <-c.Done():
t.Errorf("<-c.Done() == %v want nothing (it should block)", x)
default:
}
if got, want := fmt.Sprint(c), "context.Background"; got != want {
t.Errorf("Background().String() = %q want %q", got, want)
}
}
func TestTODO(t *testing.T) {
c := TODO()
if c == nil {
t.Fatalf("TODO returned nil")
}
select {
case x := <-c.Done():
t.Errorf("<-c.Done() == %v want nothing (it should block)", x)
default:
}
if got, want := fmt.Sprint(c), "context.TODO"; got != want {
t.Errorf("TODO().String() = %q want %q", got, want)
}
}
func TestWithCancel(t *testing.T) {
c1, cancel := WithCancel(Background())
if got, want := fmt.Sprint(c1), "context.Background.WithCancel"; got != want {
t.Errorf("c1.String() = %q want %q", got, want)
}
o := otherContext{c1}
c2, _ := WithCancel(o)
contexts := []Context{c1, o, c2}
for i, c := range contexts {
if d := c.Done(); d == nil {
t.Errorf("c[%d].Done() == %v want non-nil", i, d)
}
if e := c.Err(); e != nil {
t.Errorf("c[%d].Err() == %v want nil", i, e)
}
select {
case x := <-c.Done():
t.Errorf("<-c.Done() == %v want nothing (it should block)", x)
default:
}
}
cancel()
time.Sleep(100 * time.Millisecond) // let cancelation propagate
for i, c := range contexts {
select {
case <-c.Done():
default:
t.Errorf("<-c[%d].Done() blocked, but shouldn't have", i)
}
if e := c.Err(); e != Canceled {
t.Errorf("c[%d].Err() == %v want %v", i, e, Canceled)
}
}
}
func TestParentFinishesChild(t *testing.T) {
// Context tree:
// parent -> cancelChild
// parent -> valueChild -> timerChild
parent, cancel := WithCancel(Background())
cancelChild, stop := WithCancel(parent)
defer stop()
valueChild := WithValue(parent, "key", "value")
timerChild, stop := WithTimeout(valueChild, 10000*time.Hour)
defer stop()
select {
case x := <-parent.Done():
t.Errorf("<-parent.Done() == %v want nothing (it should block)", x)
case x := <-cancelChild.Done():
t.Errorf("<-cancelChild.Done() == %v want nothing (it should block)", x)
case x := <-timerChild.Done():
t.Errorf("<-timerChild.Done() == %v want nothing (it should block)", x)
case x := <-valueChild.Done():
t.Errorf("<-valueChild.Done() == %v want nothing (it should block)", x)
default:
}
// The parent's children should contain the two cancelable children.
pc := parent.(*cancelCtx)
cc := cancelChild.(*cancelCtx)
tc := timerChild.(*timerCtx)
pc.mu.Lock()
if len(pc.children) != 2 || !pc.children[cc] || !pc.children[tc] {
t.Errorf("bad linkage: pc.children = %v, want %v and %v",
pc.children, cc, tc)
}
pc.mu.Unlock()
if p, ok := parentCancelCtx(cc.Context); !ok || p != pc {
t.Errorf("bad linkage: parentCancelCtx(cancelChild.Context) = %v, %v want %v, true", p, ok, pc)
}
if p, ok := parentCancelCtx(tc.Context); !ok || p != pc {
t.Errorf("bad linkage: parentCancelCtx(timerChild.Context) = %v, %v want %v, true", p, ok, pc)
}
cancel()
pc.mu.Lock()
if len(pc.children) != 0 {
t.Errorf("pc.cancel didn't clear pc.children = %v", pc.children)
}
pc.mu.Unlock()
// parent and children should all be finished.
check := func(ctx Context, name string) {
select {
case <-ctx.Done():
default:
t.Errorf("<-%s.Done() blocked, but shouldn't have", name)
}
if e := ctx.Err(); e != Canceled {
t.Errorf("%s.Err() == %v want %v", name, e, Canceled)
}
}
check(parent, "parent")
check(cancelChild, "cancelChild")
check(valueChild, "valueChild")
check(timerChild, "timerChild")
// WithCancel should return a canceled context on a canceled parent.
precanceledChild := WithValue(parent, "key", "value")
select {
case <-precanceledChild.Done():
default:
t.Errorf("<-precanceledChild.Done() blocked, but shouldn't have")
}
if e := precanceledChild.Err(); e != Canceled {
t.Errorf("precanceledChild.Err() == %v want %v", e, Canceled)
}
}
func TestChildFinishesFirst(t *testing.T) {
cancelable, stop := WithCancel(Background())
defer stop()
for _, parent := range []Context{Background(), cancelable} {
child, cancel := WithCancel(parent)
select {
case x := <-parent.Done():
t.Errorf("<-parent.Done() == %v want nothing (it should block)", x)
case x := <-child.Done():
t.Errorf("<-child.Done() == %v want nothing (it should block)", x)
default:
}
cc := child.(*cancelCtx)
pc, pcok := parent.(*cancelCtx) // pcok == false when parent == Background()
if p, ok := parentCancelCtx(cc.Context); ok != pcok || (ok && pc != p) {
t.Errorf("bad linkage: parentCancelCtx(cc.Context) = %v, %v want %v, %v", p, ok, pc, pcok)
}
if pcok {
pc.mu.Lock()
if len(pc.children) != 1 || !pc.children[cc] {
t.Errorf("bad linkage: pc.children = %v, cc = %v", pc.children, cc)
}
pc.mu.Unlock()
}
cancel()
if pcok {
pc.mu.Lock()
if len(pc.children) != 0 {
t.Errorf("child's cancel didn't remove self from pc.children = %v", pc.children)
}
pc.mu.Unlock()
}
// child should be finished.
select {
case <-child.Done():
default:
t.Errorf("<-child.Done() blocked, but shouldn't have")
}
if e := child.Err(); e != Canceled {
t.Errorf("child.Err() == %v want %v", e, Canceled)
}
// parent should not be finished.
select {
case x := <-parent.Done():
t.Errorf("<-parent.Done() == %v want nothing (it should block)", x)
default:
}
if e := parent.Err(); e != nil {
t.Errorf("parent.Err() == %v want nil", e)
}
}
}
func testDeadline(c Context, wait time.Duration, t *testing.T) {
select {
case <-time.After(wait):
t.Fatalf("context should have timed out")
case <-c.Done():
}
if e := c.Err(); e != DeadlineExceeded {
t.Errorf("c.Err() == %v want %v", e, DeadlineExceeded)
}
}
func TestDeadline(t *testing.T) {
c, _ := WithDeadline(Background(), time.Now().Add(100*time.Millisecond))
if got, prefix := fmt.Sprint(c), "context.Background.WithDeadline("; !strings.HasPrefix(got, prefix) {
t.Errorf("c.String() = %q want prefix %q", got, prefix)
}
testDeadline(c, 200*time.Millisecond, t)
c, _ = WithDeadline(Background(), time.Now().Add(100*time.Millisecond))
o := otherContext{c}
testDeadline(o, 200*time.Millisecond, t)
c, _ = WithDeadline(Background(), time.Now().Add(100*time.Millisecond))
o = otherContext{c}
c, _ = WithDeadline(o, time.Now().Add(300*time.Millisecond))
testDeadline(c, 200*time.Millisecond, t)
}
func TestTimeout(t *testing.T) {
c, _ := WithTimeout(Background(), 100*time.Millisecond)
if got, prefix := fmt.Sprint(c), "context.Background.WithDeadline("; !strings.HasPrefix(got, prefix) {
t.Errorf("c.String() = %q want prefix %q", got, prefix)
}
testDeadline(c, 200*time.Millisecond, t)
c, _ = WithTimeout(Background(), 100*time.Millisecond)
o := otherContext{c}
testDeadline(o, 200*time.Millisecond, t)
c, _ = WithTimeout(Background(), 100*time.Millisecond)
o = otherContext{c}
c, _ = WithTimeout(o, 300*time.Millisecond)
testDeadline(c, 200*time.Millisecond, t)
}
func TestCanceledTimeout(t *testing.T) {
c, _ := WithTimeout(Background(), 200*time.Millisecond)
o := otherContext{c}
c, cancel := WithTimeout(o, 400*time.Millisecond)
cancel()
time.Sleep(100 * time.Millisecond) // let cancelation propagate
select {
case <-c.Done():
default:
t.Errorf("<-c.Done() blocked, but shouldn't have")
}
if e := c.Err(); e != Canceled {
t.Errorf("c.Err() == %v want %v", e, Canceled)
}
}
type key1 int
type key2 int
var k1 = key1(1)
var k2 = key2(1) // same int as k1, different type
var k3 = key2(3) // same type as k2, different int
func TestValues(t *testing.T) {
check := func(c Context, nm, v1, v2, v3 string) {
if v, ok := c.Value(k1).(string); ok == (len(v1) == 0) || v != v1 {
t.Errorf(`%s.Value(k1).(string) = %q, %t want %q, %t`, nm, v, ok, v1, len(v1) != 0)
}
if v, ok := c.Value(k2).(string); ok == (len(v2) == 0) || v != v2 {
t.Errorf(`%s.Value(k2).(string) = %q, %t want %q, %t`, nm, v, ok, v2, len(v2) != 0)
}
if v, ok := c.Value(k3).(string); ok == (len(v3) == 0) || v != v3 {
t.Errorf(`%s.Value(k3).(string) = %q, %t want %q, %t`, nm, v, ok, v3, len(v3) != 0)
}
}
c0 := Background()
check(c0, "c0", "", "", "")
c1 := WithValue(Background(), k1, "c1k1")
check(c1, "c1", "c1k1", "", "")
if got, want := fmt.Sprint(c1), `context.Background.WithValue(1, "c1k1")`; got != want {
t.Errorf("c.String() = %q want %q", got, want)
}
c2 := WithValue(c1, k2, "c2k2")
check(c2, "c2", "c1k1", "c2k2", "")
c3 := WithValue(c2, k3, "c3k3")
check(c3, "c2", "c1k1", "c2k2", "c3k3")
c4 := WithValue(c3, k1, nil)
check(c4, "c4", "", "c2k2", "c3k3")
o0 := otherContext{Background()}
check(o0, "o0", "", "", "")
o1 := otherContext{WithValue(Background(), k1, "c1k1")}
check(o1, "o1", "c1k1", "", "")
o2 := WithValue(o1, k2, "o2k2")
check(o2, "o2", "c1k1", "o2k2", "")
o3 := otherContext{c4}
check(o3, "o3", "", "c2k2", "c3k3")
o4 := WithValue(o3, k3, nil)
check(o4, "o4", "", "c2k2", "")
}
func TestAllocs(t *testing.T) {
bg := Background()
for _, test := range []struct {
desc string
f func()
limit float64
gccgoLimit float64
}{
{
desc: "Background()",
f: func() { Background() },
limit: 0,
gccgoLimit: 0,
},
{
desc: fmt.Sprintf("WithValue(bg, %v, nil)", k1),
f: func() {
c := WithValue(bg, k1, nil)
c.Value(k1)
},
limit: 1,
gccgoLimit: 3,
},
{
desc: "WithTimeout(bg, 15*time.Millisecond)",
f: func() {
c, _ := WithTimeout(bg, 15*time.Millisecond)
<-c.Done()
},
limit: 8,
gccgoLimit: 13,
},
{
desc: "WithCancel(bg)",
f: func() {
c, cancel := WithCancel(bg)
cancel()
<-c.Done()
},
limit: 5,
gccgoLimit: 8,
},
{
desc: "WithTimeout(bg, 100*time.Millisecond)",
f: func() {
c, cancel := WithTimeout(bg, 100*time.Millisecond)
cancel()
<-c.Done()
},
limit: 8,
gccgoLimit: 25,
},
} {
limit := test.limit
if runtime.Compiler == "gccgo" {
// gccgo does not yet do escape analysis.
// TOOD(iant): Remove this when gccgo does do escape analysis.
limit = test.gccgoLimit
}
if n := testing.AllocsPerRun(100, test.f); n > limit {
t.Errorf("%s allocs = %f want %d", test.desc, n, int(limit))
}
}
}
func TestSimultaneousCancels(t *testing.T) {
root, cancel := WithCancel(Background())
m := map[Context]CancelFunc{root: cancel}
q := []Context{root}
// Create a tree of contexts.
for len(q) != 0 && len(m) < 100 {
parent := q[0]
q = q[1:]
for i := 0; i < 4; i++ {
ctx, cancel := WithCancel(parent)
m[ctx] = cancel
q = append(q, ctx)
}
}
// Start all the cancels in a random order.
var wg sync.WaitGroup
wg.Add(len(m))
for _, cancel := range m {
go func(cancel CancelFunc) {
cancel()
wg.Done()
}(cancel)
}
// Wait on all the contexts in a random order.
for ctx := range m {
select {
case <-ctx.Done():
case <-time.After(1 * time.Second):
buf := make([]byte, 10<<10)
n := runtime.Stack(buf, true)
t.Fatalf("timed out waiting for <-ctx.Done(); stacks:\n%s", buf[:n])
}
}
// Wait for all the cancel functions to return.
done := make(chan struct{})
go func() {
wg.Wait()
close(done)
}()
select {
case <-done:
case <-time.After(1 * time.Second):
buf := make([]byte, 10<<10)
n := runtime.Stack(buf, true)
t.Fatalf("timed out waiting for cancel functions; stacks:\n%s", buf[:n])
}
}
func TestInterlockedCancels(t *testing.T) {
parent, cancelParent := WithCancel(Background())
child, cancelChild := WithCancel(parent)
go func() {
parent.Done()
cancelChild()
}()
cancelParent()
select {
case <-child.Done():
case <-time.After(1 * time.Second):
buf := make([]byte, 10<<10)
n := runtime.Stack(buf, true)
t.Fatalf("timed out waiting for child.Done(); stacks:\n%s", buf[:n])
}
}
func TestLayersCancel(t *testing.T) {
testLayers(t, time.Now().UnixNano(), false)
}
func TestLayersTimeout(t *testing.T) {
testLayers(t, time.Now().UnixNano(), true)
}
func testLayers(t *testing.T, seed int64, testTimeout bool) {
rand.Seed(seed)
errorf := func(format string, a ...interface{}) {
t.Errorf(fmt.Sprintf("seed=%d: %s", seed, format), a...)
}
const (
timeout = 200 * time.Millisecond
minLayers = 30
)
type value int
var (
vals []*value
cancels []CancelFunc
numTimers int
ctx = Background()
)
for i := 0; i < minLayers || numTimers == 0 || len(cancels) == 0 || len(vals) == 0; i++ {
switch rand.Intn(3) {
case 0:
v := new(value)
ctx = WithValue(ctx, v, v)
vals = append(vals, v)
case 1:
var cancel CancelFunc
ctx, cancel = WithCancel(ctx)
cancels = append(cancels, cancel)
case 2:
var cancel CancelFunc
ctx, cancel = WithTimeout(ctx, timeout)
cancels = append(cancels, cancel)
numTimers++
}
}
checkValues := func(when string) {
for _, key := range vals {
if val := ctx.Value(key).(*value); key != val {
errorf("%s: ctx.Value(%p) = %p want %p", when, key, val, key)
}
}
}
select {
case <-ctx.Done():
errorf("ctx should not be canceled yet")
default:
}
if s, prefix := fmt.Sprint(ctx), "context.Background."; !strings.HasPrefix(s, prefix) {
t.Errorf("ctx.String() = %q want prefix %q", s, prefix)
}
t.Log(ctx)
checkValues("before cancel")
if testTimeout {
select {
case <-ctx.Done():
case <-time.After(timeout + timeout/10):
errorf("ctx should have timed out")
}
checkValues("after timeout")
} else {
cancel := cancels[rand.Intn(len(cancels))]
cancel()
select {
case <-ctx.Done():
default:
errorf("ctx should be canceled")
}
checkValues("after cancel")
}
}

View File

@ -0,0 +1,26 @@
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package context_test
import (
"fmt"
"time"
"github.com/coreos/etcd/Godeps/_workspace/src/code.google.com/p/go.net/context"
)
func ExampleWithTimeout() {
// Pass a context with a timeout to tell a blocking function that it
// should abandon its work after the timeout elapses.
ctx, _ := context.WithTimeout(context.Background(), 100*time.Millisecond)
select {
case <-time.After(200 * time.Millisecond):
fmt.Println("overslept")
case <-ctx.Done():
fmt.Println(ctx.Err()) // prints "context deadline exceeded"
}
// Output:
// context deadline exceeded
}

View File

@ -44,7 +44,7 @@ import (
"time"
. "./testdata"
. "github.com/coreos/etcd/third_party/code.google.com/p/gogoprotobuf/proto"
. "github.com/coreos/etcd/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/proto"
)
var globalO *Buffer

View File

@ -34,7 +34,7 @@ package proto_test
import (
"testing"
"github.com/coreos/etcd/third_party/code.google.com/p/gogoprotobuf/proto"
"github.com/coreos/etcd/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/proto"
pb "./testdata"
)

View File

@ -35,7 +35,7 @@ import (
"testing"
pb "./testdata"
. "github.com/coreos/etcd/third_party/code.google.com/p/gogoprotobuf/proto"
. "github.com/coreos/etcd/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/proto"
)
// Four identical base messages.

View File

@ -89,7 +89,7 @@
package example
import "github.com/coreos/etcd/third_party/code.google.com/p/gogoprotobuf/proto"
import "code.google.com/p/gogoprotobuf/proto"
type FOO int32
const (
@ -168,7 +168,7 @@
import (
"log"
"github.com/coreos/etcd/third_party/code.google.com/p/gogoprotobuf/proto"
"code.google.com/p/gogoprotobuf/proto"
"./example.pb"
)

View File

@ -36,7 +36,7 @@ import (
"testing"
pb "./testdata"
. "github.com/coreos/etcd/third_party/code.google.com/p/gogoprotobuf/proto"
. "github.com/coreos/etcd/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/proto"
)
var messageWithExtension1 = &pb.MyMessage{Count: Int32(7)}

View File

@ -36,7 +36,7 @@ It has these top-level messages:
*/
package testdata
import proto "github.com/coreos/etcd/third_party/github.com/coreos/etcd/third_party/code.google.com/p/gogoprotobuf/proto"
import proto "github.com/coreos/etcd/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/proto"
import json "encoding/json"
import math "math"

View File

@ -4,7 +4,7 @@
package testdata
import proto "github.com/coreos/etcd/third_party/code.google.com/p/gogoprotobuf/proto"
import proto "code.google.com/p/gogoprotobuf/proto"
import json "encoding/json"
import math "math"

View File

@ -37,7 +37,7 @@ import (
"testing"
. "./testdata"
. "github.com/coreos/etcd/third_party/code.google.com/p/gogoprotobuf/proto"
. "github.com/coreos/etcd/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/proto"
)
type UnmarshalTextTest struct {

View File

@ -39,7 +39,7 @@ import (
"strings"
"testing"
"github.com/coreos/etcd/third_party/code.google.com/p/gogoprotobuf/proto"
"github.com/coreos/etcd/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/proto"
pb "./testdata"
)

View File

@ -0,0 +1,6 @@
language: go
go: 1.1
script:
- go vet ./...
- go test -v ./...

View File

@ -0,0 +1,21 @@
Copyright (C) 2013 Jeremy Saenz
All Rights Reserved.
MIT LICENSE
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

View File

@ -0,0 +1,287 @@
[![Build Status](https://travis-ci.org/codegangsta/cli.png?branch=master)](https://travis-ci.org/codegangsta/cli)
# cli.go
cli.go is simple, fast, and fun package for building command line apps in Go. The goal is to enable developers to write fast and distributable command line applications in an expressive way.
You can view the API docs here:
http://godoc.org/github.com/codegangsta/cli
## Overview
Command line apps are usually so tiny that there is absolutely no reason why your code should *not* be self-documenting. Things like generating help text and parsing command flags/options should not hinder productivity when writing a command line app.
**This is where cli.go comes into play.** cli.go makes command line programming fun, organized, and expressive!
## Installation
Make sure you have a working Go environment (go 1.1 is *required*). [See the install instructions](http://golang.org/doc/install.html).
To install `cli.go`, simply run:
```
$ go get github.com/codegangsta/cli
```
Make sure your `PATH` includes to the `$GOPATH/bin` directory so your commands can be easily used:
```
export PATH=$PATH:$GOPATH/bin
```
## Getting Started
One of the philosophies behind cli.go is that an API should be playful and full of discovery. So a cli.go app can be as little as one line of code in `main()`.
``` go
package main
import (
"os"
"github.com/codegangsta/cli"
)
func main() {
cli.NewApp().Run(os.Args)
}
```
This app will run and show help text, but is not very useful. Let's give an action to execute and some help documentation:
``` go
package main
import (
"os"
"github.com/codegangsta/cli"
)
func main() {
app := cli.NewApp()
app.Name = "boom"
app.Usage = "make an explosive entrance"
app.Action = func(c *cli.Context) {
println("boom! I say!")
}
app.Run(os.Args)
}
```
Running this already gives you a ton of functionality, plus support for things like subcommands and flags, which are covered below.
## Example
Being a programmer can be a lonely job. Thankfully by the power of automation that is not the case! Let's create a greeter app to fend off our demons of loneliness!
``` go
/* greet.go */
package main
import (
"os"
"github.com/codegangsta/cli"
)
func main() {
app := cli.NewApp()
app.Name = "greet"
app.Usage = "fight the loneliness!"
app.Action = func(c *cli.Context) {
println("Hello friend!")
}
app.Run(os.Args)
}
```
Install our command to the `$GOPATH/bin` directory:
```
$ go install
```
Finally run our new command:
```
$ greet
Hello friend!
```
cli.go also generates some bitchass help text:
```
$ greet help
NAME:
greet - fight the loneliness!
USAGE:
greet [global options] command [command options] [arguments...]
VERSION:
0.0.0
COMMANDS:
help, h Shows a list of commands or help for one command
GLOBAL OPTIONS
--version Shows version information
```
### Arguments
You can lookup arguments by calling the `Args` function on `cli.Context`.
``` go
...
app.Action = func(c *cli.Context) {
println("Hello", c.Args()[0])
}
...
```
### Flags
Setting and querying flags is simple.
``` go
...
app.Flags = []cli.Flag {
cli.StringFlag{
Name: "lang",
Value: "english",
Usage: "language for the greeting",
},
}
app.Action = func(c *cli.Context) {
name := "someone"
if len(c.Args()) > 0 {
name = c.Args()[0]
}
if c.String("lang") == "spanish" {
println("Hola", name)
} else {
println("Hello", name)
}
}
...
```
#### Alternate Names
You can set alternate (or short) names for flags by providing a comma-delimited list for the `Name`. e.g.
``` go
app.Flags = []cli.Flag {
cli.StringFlag{
Name: "lang, l",
Value: "english",
Usage: "language for the greeting",
},
}
```
#### Values from the Environment
You can also have the default value set from the environment via `EnvVar`. e.g.
``` go
app.Flags = []cli.Flag {
cli.StringFlag{
Name: "lang, l",
Value: "english",
Usage: "language for the greeting",
EnvVar: "APP_LANG",
},
}
```
That flag can then be set with `--lang spanish` or `-l spanish`. Note that giving two different forms of the same flag in the same command invocation is an error.
### Subcommands
Subcommands can be defined for a more git-like command line app.
```go
...
app.Commands = []cli.Command{
{
Name: "add",
ShortName: "a",
Usage: "add a task to the list",
Action: func(c *cli.Context) {
println("added task: ", c.Args().First())
},
},
{
Name: "complete",
ShortName: "c",
Usage: "complete a task on the list",
Action: func(c *cli.Context) {
println("completed task: ", c.Args().First())
},
},
{
Name: "template",
ShortName: "r",
Usage: "options for task templates",
Subcommands: []cli.Command{
{
Name: "add",
Usage: "add a new template",
Action: func(c *cli.Context) {
println("new task template: ", c.Args().First())
},
},
{
Name: "remove",
Usage: "remove an existing template",
Action: func(c *cli.Context) {
println("removed task template: ", c.Args().First())
},
},
},
},
}
...
```
### Bash Completion
You can enable completion commands by setting the `EnableBashCompletion`
flag on the `App` object. By default, this setting will only auto-complete to
show an app's subcommands, but you can write your own completion methods for
the App or its subcommands.
```go
...
var tasks = []string{"cook", "clean", "laundry", "eat", "sleep", "code"}
app := cli.NewApp()
app.EnableBashCompletion = true
app.Commands = []cli.Command{
{
Name: "complete",
ShortName: "c",
Usage: "complete a task on the list",
Action: func(c *cli.Context) {
println("completed task: ", c.Args().First())
},
BashComplete: func(c *cli.Context) {
// This will complete if no args are passed
if len(c.Args()) > 0 {
return
}
for _, t := range tasks {
fmt.Println(t)
}
},
}
}
...
```
#### To Enable
Source the `autocomplete/bash_autocomplete` file in your `.bashrc` file while
setting the `PROG` variable to the name of your program:
`PROG=myprogram source /.../cli/autocomplete/bash_autocomplete`
## Contribution Guidelines
Feel free to put up a pull request to fix a bug or maybe add a feature. I will give it a code review and make sure that it does not break backwards compatibility. If I or any other collaborators agree that it is in line with the vision of the project, we will work with you to get the code into a mergeable state and merge it into the master branch.
If you are have contributed something significant to the project, I will most likely add you as a collaborator. As a collaborator you are given the ability to merge others pull requests. It is very important that new code does not break existing code, so be careful about what code you do choose to merge. If you have any questions feel free to link @codegangsta to the issue in question and we can review it together.
If you feel like you have contributed to the project but have not yet been added as a collaborator, I probably forgot to add you. Hit @codegangsta up over email and we will get it figured out.
## About
cli.go is written by none other than the [Code Gangsta](http://codegangsta.io)

246
Godeps/_workspace/src/github.com/codegangsta/cli/app.go generated vendored Normal file
View File

@ -0,0 +1,246 @@
package cli
import (
"fmt"
"io/ioutil"
"os"
"time"
)
// App is the main structure of a cli application. It is recomended that
// and app be created with the cli.NewApp() function
type App struct {
// The name of the program. Defaults to os.Args[0]
Name string
// Description of the program.
Usage string
// Version of the program
Version string
// List of commands to execute
Commands []Command
// List of flags to parse
Flags []Flag
// Boolean to enable bash completion commands
EnableBashCompletion bool
// Boolean to hide built-in help command
HideHelp bool
// An action to execute when the bash-completion flag is set
BashComplete func(context *Context)
// An action to execute before any subcommands are run, but after the context is ready
// If a non-nil error is returned, no subcommands are run
Before func(context *Context) error
// The action to execute when no subcommands are specified
Action func(context *Context)
// Execute this function if the proper command cannot be found
CommandNotFound func(context *Context, command string)
// Compilation date
Compiled time.Time
// Author
Author string
// Author e-mail
Email string
}
// Tries to find out when this binary was compiled.
// Returns the current time if it fails to find it.
func compileTime() time.Time {
info, err := os.Stat(os.Args[0])
if err != nil {
return time.Now()
}
return info.ModTime()
}
// Creates a new cli Application with some reasonable defaults for Name, Usage, Version and Action.
func NewApp() *App {
return &App{
Name: os.Args[0],
Usage: "A new cli application",
Version: "0.0.0",
BashComplete: DefaultAppComplete,
Action: helpCommand.Action,
Compiled: compileTime(),
}
}
// Entry point to the cli app. Parses the arguments slice and routes to the proper flag/args combination
func (a *App) Run(arguments []string) error {
// append help to commands
if a.Command(helpCommand.Name) == nil && !a.HideHelp {
a.Commands = append(a.Commands, helpCommand)
a.appendFlag(HelpFlag)
}
//append version/help flags
if a.EnableBashCompletion {
a.appendFlag(BashCompletionFlag)
}
a.appendFlag(VersionFlag)
// parse flags
set := flagSet(a.Name, a.Flags)
set.SetOutput(ioutil.Discard)
err := set.Parse(arguments[1:])
nerr := normalizeFlags(a.Flags, set)
if nerr != nil {
fmt.Println(nerr)
context := NewContext(a, set, set)
ShowAppHelp(context)
fmt.Println("")
return nerr
}
context := NewContext(a, set, set)
if err != nil {
fmt.Printf("Incorrect Usage.\n\n")
ShowAppHelp(context)
fmt.Println("")
return err
}
if checkCompletions(context) {
return nil
}
if checkHelp(context) {
return nil
}
if checkVersion(context) {
return nil
}
if a.Before != nil {
err := a.Before(context)
if err != nil {
return err
}
}
args := context.Args()
if args.Present() {
name := args.First()
c := a.Command(name)
if c != nil {
return c.Run(context)
}
}
// Run default Action
a.Action(context)
return nil
}
// Another entry point to the cli app, takes care of passing arguments and error handling
func (a *App) RunAndExitOnError() {
if err := a.Run(os.Args); err != nil {
os.Stderr.WriteString(fmt.Sprintln(err))
os.Exit(1)
}
}
// Invokes the subcommand given the context, parses ctx.Args() to generate command-specific flags
func (a *App) RunAsSubcommand(ctx *Context) error {
// append help to commands
if len(a.Commands) > 0 {
if a.Command(helpCommand.Name) == nil && !a.HideHelp {
a.Commands = append(a.Commands, helpCommand)
a.appendFlag(HelpFlag)
}
}
// append flags
if a.EnableBashCompletion {
a.appendFlag(BashCompletionFlag)
}
// parse flags
set := flagSet(a.Name, a.Flags)
set.SetOutput(ioutil.Discard)
err := set.Parse(ctx.Args().Tail())
nerr := normalizeFlags(a.Flags, set)
context := NewContext(a, set, ctx.globalSet)
if nerr != nil {
fmt.Println(nerr)
if len(a.Commands) > 0 {
ShowSubcommandHelp(context)
} else {
ShowCommandHelp(ctx, context.Args().First())
}
fmt.Println("")
return nerr
}
if err != nil {
fmt.Printf("Incorrect Usage.\n\n")
ShowSubcommandHelp(context)
return err
}
if checkCompletions(context) {
return nil
}
if len(a.Commands) > 0 {
if checkSubcommandHelp(context) {
return nil
}
} else {
if checkCommandHelp(ctx, context.Args().First()) {
return nil
}
}
if a.Before != nil {
err := a.Before(context)
if err != nil {
return err
}
}
args := context.Args()
if args.Present() {
name := args.First()
c := a.Command(name)
if c != nil {
return c.Run(context)
}
}
// Run default Action
if len(a.Commands) > 0 {
a.Action(context)
} else {
a.Action(ctx)
}
return nil
}
// Returns the named command on App. Returns nil if the command does not exist
func (a *App) Command(name string) *Command {
for _, c := range a.Commands {
if c.HasName(name) {
return &c
}
}
return nil
}
func (a *App) hasFlag(flag Flag) bool {
for _, f := range a.Flags {
if flag == f {
return true
}
}
return false
}
func (a *App) appendFlag(flag Flag) {
if !a.hasFlag(flag) {
a.Flags = append(a.Flags, flag)
}
}

View File

@ -0,0 +1,423 @@
package cli_test
import (
"fmt"
"os"
"testing"
"github.com/codegangsta/cli"
)
func ExampleApp() {
// set args for examples sake
os.Args = []string{"greet", "--name", "Jeremy"}
app := cli.NewApp()
app.Name = "greet"
app.Flags = []cli.Flag{
cli.StringFlag{Name: "name", Value: "bob", Usage: "a name to say"},
}
app.Action = func(c *cli.Context) {
fmt.Printf("Hello %v\n", c.String("name"))
}
app.Run(os.Args)
// Output:
// Hello Jeremy
}
func ExampleAppSubcommand() {
// set args for examples sake
os.Args = []string{"say", "hi", "english", "--name", "Jeremy"}
app := cli.NewApp()
app.Name = "say"
app.Commands = []cli.Command{
{
Name: "hello",
ShortName: "hi",
Usage: "use it to see a description",
Description: "This is how we describe hello the function",
Subcommands: []cli.Command{
{
Name: "english",
ShortName: "en",
Usage: "sends a greeting in english",
Description: "greets someone in english",
Flags: []cli.Flag{
cli.StringFlag{
Name: "name",
Value: "Bob",
Usage: "Name of the person to greet",
},
},
Action: func(c *cli.Context) {
fmt.Println("Hello,", c.String("name"))
},
},
},
},
}
app.Run(os.Args)
// Output:
// Hello, Jeremy
}
func ExampleAppHelp() {
// set args for examples sake
os.Args = []string{"greet", "h", "describeit"}
app := cli.NewApp()
app.Name = "greet"
app.Flags = []cli.Flag{
cli.StringFlag{Name: "name", Value: "bob", Usage: "a name to say"},
}
app.Commands = []cli.Command{
{
Name: "describeit",
ShortName: "d",
Usage: "use it to see a description",
Description: "This is how we describe describeit the function",
Action: func(c *cli.Context) {
fmt.Printf("i like to describe things")
},
},
}
app.Run(os.Args)
// Output:
// NAME:
// describeit - use it to see a description
//
// USAGE:
// command describeit [arguments...]
//
// DESCRIPTION:
// This is how we describe describeit the function
}
func ExampleAppBashComplete() {
// set args for examples sake
os.Args = []string{"greet", "--generate-bash-completion"}
app := cli.NewApp()
app.Name = "greet"
app.EnableBashCompletion = true
app.Commands = []cli.Command{
{
Name: "describeit",
ShortName: "d",
Usage: "use it to see a description",
Description: "This is how we describe describeit the function",
Action: func(c *cli.Context) {
fmt.Printf("i like to describe things")
},
}, {
Name: "next",
Usage: "next example",
Description: "more stuff to see when generating bash completion",
Action: func(c *cli.Context) {
fmt.Printf("the next example")
},
},
}
app.Run(os.Args)
// Output:
// describeit
// d
// next
// help
// h
}
func TestApp_Run(t *testing.T) {
s := ""
app := cli.NewApp()
app.Action = func(c *cli.Context) {
s = s + c.Args().First()
}
err := app.Run([]string{"command", "foo"})
expect(t, err, nil)
err = app.Run([]string{"command", "bar"})
expect(t, err, nil)
expect(t, s, "foobar")
}
var commandAppTests = []struct {
name string
expected bool
}{
{"foobar", true},
{"batbaz", true},
{"b", true},
{"f", true},
{"bat", false},
{"nothing", false},
}
func TestApp_Command(t *testing.T) {
app := cli.NewApp()
fooCommand := cli.Command{Name: "foobar", ShortName: "f"}
batCommand := cli.Command{Name: "batbaz", ShortName: "b"}
app.Commands = []cli.Command{
fooCommand,
batCommand,
}
for _, test := range commandAppTests {
expect(t, app.Command(test.name) != nil, test.expected)
}
}
func TestApp_CommandWithArgBeforeFlags(t *testing.T) {
var parsedOption, firstArg string
app := cli.NewApp()
command := cli.Command{
Name: "cmd",
Flags: []cli.Flag{
cli.StringFlag{Name: "option", Value: "", Usage: "some option"},
},
Action: func(c *cli.Context) {
parsedOption = c.String("option")
firstArg = c.Args().First()
},
}
app.Commands = []cli.Command{command}
app.Run([]string{"", "cmd", "my-arg", "--option", "my-option"})
expect(t, parsedOption, "my-option")
expect(t, firstArg, "my-arg")
}
func TestApp_Float64Flag(t *testing.T) {
var meters float64
app := cli.NewApp()
app.Flags = []cli.Flag{
cli.Float64Flag{Name: "height", Value: 1.5, Usage: "Set the height, in meters"},
}
app.Action = func(c *cli.Context) {
meters = c.Float64("height")
}
app.Run([]string{"", "--height", "1.93"})
expect(t, meters, 1.93)
}
func TestApp_ParseSliceFlags(t *testing.T) {
var parsedOption, firstArg string
var parsedIntSlice []int
var parsedStringSlice []string
app := cli.NewApp()
command := cli.Command{
Name: "cmd",
Flags: []cli.Flag{
cli.IntSliceFlag{Name: "p", Value: &cli.IntSlice{}, Usage: "set one or more ip addr"},
cli.StringSliceFlag{Name: "ip", Value: &cli.StringSlice{}, Usage: "set one or more ports to open"},
},
Action: func(c *cli.Context) {
parsedIntSlice = c.IntSlice("p")
parsedStringSlice = c.StringSlice("ip")
parsedOption = c.String("option")
firstArg = c.Args().First()
},
}
app.Commands = []cli.Command{command}
app.Run([]string{"", "cmd", "my-arg", "-p", "22", "-p", "80", "-ip", "8.8.8.8", "-ip", "8.8.4.4"})
IntsEquals := func(a, b []int) bool {
if len(a) != len(b) {
return false
}
for i, v := range a {
if v != b[i] {
return false
}
}
return true
}
StrsEquals := func(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i, v := range a {
if v != b[i] {
return false
}
}
return true
}
var expectedIntSlice = []int{22, 80}
var expectedStringSlice = []string{"8.8.8.8", "8.8.4.4"}
if !IntsEquals(parsedIntSlice, expectedIntSlice) {
t.Errorf("%v does not match %v", parsedIntSlice, expectedIntSlice)
}
if !StrsEquals(parsedStringSlice, expectedStringSlice) {
t.Errorf("%v does not match %v", parsedStringSlice, expectedStringSlice)
}
}
func TestApp_BeforeFunc(t *testing.T) {
beforeRun, subcommandRun := false, false
beforeError := fmt.Errorf("fail")
var err error
app := cli.NewApp()
app.Before = func(c *cli.Context) error {
beforeRun = true
s := c.String("opt")
if s == "fail" {
return beforeError
}
return nil
}
app.Commands = []cli.Command{
cli.Command{
Name: "sub",
Action: func(c *cli.Context) {
subcommandRun = true
},
},
}
app.Flags = []cli.Flag{
cli.StringFlag{Name: "opt"},
}
// run with the Before() func succeeding
err = app.Run([]string{"command", "--opt", "succeed", "sub"})
if err != nil {
t.Fatalf("Run error: %s", err)
}
if beforeRun == false {
t.Errorf("Before() not executed when expected")
}
if subcommandRun == false {
t.Errorf("Subcommand not executed when expected")
}
// reset
beforeRun, subcommandRun = false, false
// run with the Before() func failing
err = app.Run([]string{"command", "--opt", "fail", "sub"})
// should be the same error produced by the Before func
if err != beforeError {
t.Errorf("Run error expected, but not received")
}
if beforeRun == false {
t.Errorf("Before() not executed when expected")
}
if subcommandRun == true {
t.Errorf("Subcommand executed when NOT expected")
}
}
func TestAppHelpPrinter(t *testing.T) {
oldPrinter := cli.HelpPrinter
defer func() {
cli.HelpPrinter = oldPrinter
}()
var wasCalled = false
cli.HelpPrinter = func(template string, data interface{}) {
wasCalled = true
}
app := cli.NewApp()
app.Run([]string{"-h"})
if wasCalled == false {
t.Errorf("Help printer expected to be called, but was not")
}
}
func TestAppVersionPrinter(t *testing.T) {
oldPrinter := cli.VersionPrinter
defer func() {
cli.VersionPrinter = oldPrinter
}()
var wasCalled = false
cli.VersionPrinter = func(c *cli.Context) {
wasCalled = true
}
app := cli.NewApp()
ctx := cli.NewContext(app, nil, nil)
cli.ShowVersion(ctx)
if wasCalled == false {
t.Errorf("Version printer expected to be called, but was not")
}
}
func TestAppCommandNotFound(t *testing.T) {
beforeRun, subcommandRun := false, false
app := cli.NewApp()
app.CommandNotFound = func(c *cli.Context, command string) {
beforeRun = true
}
app.Commands = []cli.Command{
cli.Command{
Name: "bar",
Action: func(c *cli.Context) {
subcommandRun = true
},
},
}
app.Run([]string{"command", "foo"})
expect(t, beforeRun, true)
expect(t, subcommandRun, false)
}
func TestGlobalFlagsInSubcommands(t *testing.T) {
subcommandRun := false
app := cli.NewApp()
app.Flags = []cli.Flag{
cli.BoolFlag{Name: "debug, d", Usage: "Enable debugging"},
}
app.Commands = []cli.Command{
cli.Command{
Name: "foo",
Subcommands: []cli.Command{
{
Name: "bar",
Action: func(c *cli.Context) {
if c.GlobalBool("debug") {
subcommandRun = true
}
},
},
},
},
}
app.Run([]string{"command", "-d", "foo", "bar"})
expect(t, subcommandRun, true)
}

View File

@ -0,0 +1,13 @@
#! /bin/bash
_cli_bash_autocomplete() {
local cur prev opts base
COMPREPLY=()
cur="${COMP_WORDS[COMP_CWORD]}"
prev="${COMP_WORDS[COMP_CWORD-1]}"
opts=$( ${COMP_WORDS[@]:0:$COMP_CWORD} --generate-bash-completion )
COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
return 0
}
complete -F _cli_bash_autocomplete $PROG

View File

@ -0,0 +1,5 @@
autoload -U compinit && compinit
autoload -U bashcompinit && bashcompinit
script_dir=$(dirname $0)
source ${script_dir}/bash_autocomplete

View File

@ -0,0 +1,19 @@
// Package cli provides a minimal framework for creating and organizing command line
// Go applications. cli is designed to be easy to understand and write, the most simple
// cli application can be written as follows:
// func main() {
// cli.NewApp().Run(os.Args)
// }
//
// Of course this application does not do much, so let's make this an actual application:
// func main() {
// app := cli.NewApp()
// app.Name = "greet"
// app.Usage = "say a greeting"
// app.Action = func(c *cli.Context) {
// println("Greetings")
// }
//
// app.Run(os.Args)
// }
package cli

View File

@ -0,0 +1,100 @@
package cli_test
import (
"os"
"github.com/codegangsta/cli"
)
func Example() {
app := cli.NewApp()
app.Name = "todo"
app.Usage = "task list on the command line"
app.Commands = []cli.Command{
{
Name: "add",
ShortName: "a",
Usage: "add a task to the list",
Action: func(c *cli.Context) {
println("added task: ", c.Args().First())
},
},
{
Name: "complete",
ShortName: "c",
Usage: "complete a task on the list",
Action: func(c *cli.Context) {
println("completed task: ", c.Args().First())
},
},
}
app.Run(os.Args)
}
func ExampleSubcommand() {
app := cli.NewApp()
app.Name = "say"
app.Commands = []cli.Command{
{
Name: "hello",
ShortName: "hi",
Usage: "use it to see a description",
Description: "This is how we describe hello the function",
Subcommands: []cli.Command{
{
Name: "english",
ShortName: "en",
Usage: "sends a greeting in english",
Description: "greets someone in english",
Flags: []cli.Flag{
cli.StringFlag{
Name: "name",
Value: "Bob",
Usage: "Name of the person to greet",
},
},
Action: func(c *cli.Context) {
println("Hello, ", c.String("name"))
},
}, {
Name: "spanish",
ShortName: "sp",
Usage: "sends a greeting in spanish",
Flags: []cli.Flag{
cli.StringFlag{
Name: "surname",
Value: "Jones",
Usage: "Surname of the person to greet",
},
},
Action: func(c *cli.Context) {
println("Hola, ", c.String("surname"))
},
}, {
Name: "french",
ShortName: "fr",
Usage: "sends a greeting in french",
Flags: []cli.Flag{
cli.StringFlag{
Name: "nickname",
Value: "Stevie",
Usage: "Nickname of the person to greet",
},
},
Action: func(c *cli.Context) {
println("Bonjour, ", c.String("nickname"))
},
},
},
}, {
Name: "bye",
Usage: "says goodbye",
Action: func(c *cli.Context) {
println("bye")
},
},
}
app.Run(os.Args)
}

View File

@ -0,0 +1,144 @@
package cli
import (
"fmt"
"io/ioutil"
"strings"
)
// Command is a subcommand for a cli.App.
type Command struct {
// The name of the command
Name string
// short name of the command. Typically one character
ShortName string
// A short description of the usage of this command
Usage string
// A longer explanation of how the command works
Description string
// The function to call when checking for bash command completions
BashComplete func(context *Context)
// An action to execute before any sub-subcommands are run, but after the context is ready
// If a non-nil error is returned, no sub-subcommands are run
Before func(context *Context) error
// The function to call when this command is invoked
Action func(context *Context)
// List of child commands
Subcommands []Command
// List of flags to parse
Flags []Flag
// Treat all flags as normal arguments if true
SkipFlagParsing bool
// Boolean to hide built-in help command
HideHelp bool
}
// Invokes the command given the context, parses ctx.Args() to generate command-specific flags
func (c Command) Run(ctx *Context) error {
if len(c.Subcommands) > 0 || c.Before != nil {
return c.startApp(ctx)
}
if !c.HideHelp {
// append help to flags
c.Flags = append(
c.Flags,
HelpFlag,
)
}
if ctx.App.EnableBashCompletion {
c.Flags = append(c.Flags, BashCompletionFlag)
}
set := flagSet(c.Name, c.Flags)
set.SetOutput(ioutil.Discard)
firstFlagIndex := -1
for index, arg := range ctx.Args() {
if strings.HasPrefix(arg, "-") {
firstFlagIndex = index
break
}
}
var err error
if firstFlagIndex > -1 && !c.SkipFlagParsing {
args := ctx.Args()
regularArgs := args[1:firstFlagIndex]
flagArgs := args[firstFlagIndex:]
err = set.Parse(append(flagArgs, regularArgs...))
} else {
err = set.Parse(ctx.Args().Tail())
}
if err != nil {
fmt.Printf("Incorrect Usage.\n\n")
ShowCommandHelp(ctx, c.Name)
fmt.Println("")
return err
}
nerr := normalizeFlags(c.Flags, set)
if nerr != nil {
fmt.Println(nerr)
fmt.Println("")
ShowCommandHelp(ctx, c.Name)
fmt.Println("")
return nerr
}
context := NewContext(ctx.App, set, ctx.globalSet)
if checkCommandCompletions(context, c.Name) {
return nil
}
if checkCommandHelp(context, c.Name) {
return nil
}
context.Command = c
c.Action(context)
return nil
}
// Returns true if Command.Name or Command.ShortName matches given name
func (c Command) HasName(name string) bool {
return c.Name == name || c.ShortName == name
}
func (c Command) startApp(ctx *Context) error {
app := NewApp()
// set the name and usage
app.Name = fmt.Sprintf("%s %s", ctx.App.Name, c.Name)
if c.Description != "" {
app.Usage = c.Description
} else {
app.Usage = c.Usage
}
// set CommandNotFound
app.CommandNotFound = ctx.App.CommandNotFound
// set the flags and commands
app.Commands = c.Subcommands
app.Flags = c.Flags
app.HideHelp = c.HideHelp
// bash completion
app.EnableBashCompletion = ctx.App.EnableBashCompletion
if c.BashComplete != nil {
app.BashComplete = c.BashComplete
}
// set the actions
app.Before = c.Before
if c.Action != nil {
app.Action = c.Action
} else {
app.Action = helpSubcommand.Action
}
return app.RunAsSubcommand(ctx)
}

View File

@ -0,0 +1,49 @@
package cli_test
import (
"flag"
"testing"
"github.com/codegangsta/cli"
)
func TestCommandDoNotIgnoreFlags(t *testing.T) {
app := cli.NewApp()
set := flag.NewFlagSet("test", 0)
test := []string{"blah", "blah", "-break"}
set.Parse(test)
c := cli.NewContext(app, set, set)
command := cli.Command{
Name: "test-cmd",
ShortName: "tc",
Usage: "this is for testing",
Description: "testing",
Action: func(_ *cli.Context) {},
}
err := command.Run(c)
expect(t, err.Error(), "flag provided but not defined: -break")
}
func TestCommandIgnoreFlags(t *testing.T) {
app := cli.NewApp()
set := flag.NewFlagSet("test", 0)
test := []string{"blah", "blah"}
set.Parse(test)
c := cli.NewContext(app, set, set)
command := cli.Command{
Name: "test-cmd",
ShortName: "tc",
Usage: "this is for testing",
Description: "testing",
Action: func(_ *cli.Context) {},
SkipFlagParsing: true,
}
err := command.Run(c)
expect(t, err, nil)
}

View File

@ -0,0 +1,315 @@
package cli
import (
"errors"
"flag"
"strconv"
"strings"
"time"
)
// Context is a type that is passed through to
// each Handler action in a cli application. Context
// can be used to retrieve context-specific Args and
// parsed command-line options.
type Context struct {
App *App
Command Command
flagSet *flag.FlagSet
globalSet *flag.FlagSet
setFlags map[string]bool
}
// Creates a new context. For use in when invoking an App or Command action.
func NewContext(app *App, set *flag.FlagSet, globalSet *flag.FlagSet) *Context {
return &Context{App: app, flagSet: set, globalSet: globalSet}
}
// Looks up the value of a local int flag, returns 0 if no int flag exists
func (c *Context) Int(name string) int {
return lookupInt(name, c.flagSet)
}
// Looks up the value of a local time.Duration flag, returns 0 if no time.Duration flag exists
func (c *Context) Duration(name string) time.Duration {
return lookupDuration(name, c.flagSet)
}
// Looks up the value of a local float64 flag, returns 0 if no float64 flag exists
func (c *Context) Float64(name string) float64 {
return lookupFloat64(name, c.flagSet)
}
// Looks up the value of a local bool flag, returns false if no bool flag exists
func (c *Context) Bool(name string) bool {
return lookupBool(name, c.flagSet)
}
// Looks up the value of a local boolT flag, returns false if no bool flag exists
func (c *Context) BoolT(name string) bool {
return lookupBoolT(name, c.flagSet)
}
// Looks up the value of a local string flag, returns "" if no string flag exists
func (c *Context) String(name string) string {
return lookupString(name, c.flagSet)
}
// Looks up the value of a local string slice flag, returns nil if no string slice flag exists
func (c *Context) StringSlice(name string) []string {
return lookupStringSlice(name, c.flagSet)
}
// Looks up the value of a local int slice flag, returns nil if no int slice flag exists
func (c *Context) IntSlice(name string) []int {
return lookupIntSlice(name, c.flagSet)
}
// Looks up the value of a local generic flag, returns nil if no generic flag exists
func (c *Context) Generic(name string) interface{} {
return lookupGeneric(name, c.flagSet)
}
// Looks up the value of a global int flag, returns 0 if no int flag exists
func (c *Context) GlobalInt(name string) int {
return lookupInt(name, c.globalSet)
}
// Looks up the value of a global time.Duration flag, returns 0 if no time.Duration flag exists
func (c *Context) GlobalDuration(name string) time.Duration {
return lookupDuration(name, c.globalSet)
}
// Looks up the value of a global bool flag, returns false if no bool flag exists
func (c *Context) GlobalBool(name string) bool {
return lookupBool(name, c.globalSet)
}
// Looks up the value of a global string flag, returns "" if no string flag exists
func (c *Context) GlobalString(name string) string {
return lookupString(name, c.globalSet)
}
// Looks up the value of a global string slice flag, returns nil if no string slice flag exists
func (c *Context) GlobalStringSlice(name string) []string {
return lookupStringSlice(name, c.globalSet)
}
// Looks up the value of a global int slice flag, returns nil if no int slice flag exists
func (c *Context) GlobalIntSlice(name string) []int {
return lookupIntSlice(name, c.globalSet)
}
// Looks up the value of a global generic flag, returns nil if no generic flag exists
func (c *Context) GlobalGeneric(name string) interface{} {
return lookupGeneric(name, c.globalSet)
}
// Determines if the flag was actually set exists
func (c *Context) IsSet(name string) bool {
if c.setFlags == nil {
c.setFlags = make(map[string]bool)
c.flagSet.Visit(func(f *flag.Flag) {
c.setFlags[f.Name] = true
})
}
return c.setFlags[name] == true
}
// Returns a slice of flag names used in this context.
func (c *Context) FlagNames() (names []string) {
for _, flag := range c.Command.Flags {
name := strings.Split(flag.getName(), ",")[0]
if name == "help" {
continue
}
names = append(names, name)
}
return
}
type Args []string
// Returns the command line arguments associated with the context.
func (c *Context) Args() Args {
args := Args(c.flagSet.Args())
return args
}
// Returns the nth argument, or else a blank string
func (a Args) Get(n int) string {
if len(a) > n {
return a[n]
}
return ""
}
// Returns the first argument, or else a blank string
func (a Args) First() string {
return a.Get(0)
}
// Return the rest of the arguments (not the first one)
// or else an empty string slice
func (a Args) Tail() []string {
if len(a) >= 2 {
return []string(a)[1:]
}
return []string{}
}
// Checks if there are any arguments present
func (a Args) Present() bool {
return len(a) != 0
}
// Swaps arguments at the given indexes
func (a Args) Swap(from, to int) error {
if from >= len(a) || to >= len(a) {
return errors.New("index out of range")
}
a[from], a[to] = a[to], a[from]
return nil
}
func lookupInt(name string, set *flag.FlagSet) int {
f := set.Lookup(name)
if f != nil {
val, err := strconv.Atoi(f.Value.String())
if err != nil {
return 0
}
return val
}
return 0
}
func lookupDuration(name string, set *flag.FlagSet) time.Duration {
f := set.Lookup(name)
if f != nil {
val, err := time.ParseDuration(f.Value.String())
if err == nil {
return val
}
}
return 0
}
func lookupFloat64(name string, set *flag.FlagSet) float64 {
f := set.Lookup(name)
if f != nil {
val, err := strconv.ParseFloat(f.Value.String(), 64)
if err != nil {
return 0
}
return val
}
return 0
}
func lookupString(name string, set *flag.FlagSet) string {
f := set.Lookup(name)
if f != nil {
return f.Value.String()
}
return ""
}
func lookupStringSlice(name string, set *flag.FlagSet) []string {
f := set.Lookup(name)
if f != nil {
return (f.Value.(*StringSlice)).Value()
}
return nil
}
func lookupIntSlice(name string, set *flag.FlagSet) []int {
f := set.Lookup(name)
if f != nil {
return (f.Value.(*IntSlice)).Value()
}
return nil
}
func lookupGeneric(name string, set *flag.FlagSet) interface{} {
f := set.Lookup(name)
if f != nil {
return f.Value
}
return nil
}
func lookupBool(name string, set *flag.FlagSet) bool {
f := set.Lookup(name)
if f != nil {
val, err := strconv.ParseBool(f.Value.String())
if err != nil {
return false
}
return val
}
return false
}
func lookupBoolT(name string, set *flag.FlagSet) bool {
f := set.Lookup(name)
if f != nil {
val, err := strconv.ParseBool(f.Value.String())
if err != nil {
return true
}
return val
}
return false
}
func copyFlag(name string, ff *flag.Flag, set *flag.FlagSet) {
switch ff.Value.(type) {
case *StringSlice:
default:
set.Set(name, ff.Value.String())
}
}
func normalizeFlags(flags []Flag, set *flag.FlagSet) error {
visited := make(map[string]bool)
set.Visit(func(f *flag.Flag) {
visited[f.Name] = true
})
for _, f := range flags {
parts := strings.Split(f.getName(), ",")
if len(parts) == 1 {
continue
}
var ff *flag.Flag
for _, name := range parts {
name = strings.Trim(name, " ")
if visited[name] {
if ff != nil {
return errors.New("Cannot use two forms of the same flag: " + name + " " + ff.Name)
}
ff = set.Lookup(name)
}
}
if ff == nil {
continue
}
for _, name := range parts {
name = strings.Trim(name, " ")
if !visited[name] {
copyFlag(name, ff, set)
}
}
}
return nil
}

View File

@ -0,0 +1,77 @@
package cli_test
import (
"flag"
"testing"
"time"
"github.com/codegangsta/cli"
)
func TestNewContext(t *testing.T) {
set := flag.NewFlagSet("test", 0)
set.Int("myflag", 12, "doc")
globalSet := flag.NewFlagSet("test", 0)
globalSet.Int("myflag", 42, "doc")
command := cli.Command{Name: "mycommand"}
c := cli.NewContext(nil, set, globalSet)
c.Command = command
expect(t, c.Int("myflag"), 12)
expect(t, c.GlobalInt("myflag"), 42)
expect(t, c.Command.Name, "mycommand")
}
func TestContext_Int(t *testing.T) {
set := flag.NewFlagSet("test", 0)
set.Int("myflag", 12, "doc")
c := cli.NewContext(nil, set, set)
expect(t, c.Int("myflag"), 12)
}
func TestContext_Duration(t *testing.T) {
set := flag.NewFlagSet("test", 0)
set.Duration("myflag", time.Duration(12*time.Second), "doc")
c := cli.NewContext(nil, set, set)
expect(t, c.Duration("myflag"), time.Duration(12*time.Second))
}
func TestContext_String(t *testing.T) {
set := flag.NewFlagSet("test", 0)
set.String("myflag", "hello world", "doc")
c := cli.NewContext(nil, set, set)
expect(t, c.String("myflag"), "hello world")
}
func TestContext_Bool(t *testing.T) {
set := flag.NewFlagSet("test", 0)
set.Bool("myflag", false, "doc")
c := cli.NewContext(nil, set, set)
expect(t, c.Bool("myflag"), false)
}
func TestContext_BoolT(t *testing.T) {
set := flag.NewFlagSet("test", 0)
set.Bool("myflag", true, "doc")
c := cli.NewContext(nil, set, set)
expect(t, c.BoolT("myflag"), true)
}
func TestContext_Args(t *testing.T) {
set := flag.NewFlagSet("test", 0)
set.Bool("myflag", false, "doc")
c := cli.NewContext(nil, set, set)
set.Parse([]string{"--myflag", "bat", "baz"})
expect(t, len(c.Args()), 2)
expect(t, c.Bool("myflag"), true)
}
func TestContext_IsSet(t *testing.T) {
set := flag.NewFlagSet("test", 0)
set.Bool("myflag", false, "doc")
set.String("otherflag", "hello world", "doc")
c := cli.NewContext(nil, set, set)
set.Parse([]string{"--myflag", "bat", "baz"})
expect(t, c.IsSet("myflag"), true)
expect(t, c.IsSet("otherflag"), false)
expect(t, c.IsSet("bogusflag"), false)
}

View File

@ -0,0 +1,410 @@
package cli
import (
"flag"
"fmt"
"os"
"strconv"
"strings"
"time"
)
// This flag enables bash-completion for all commands and subcommands
var BashCompletionFlag = BoolFlag{
Name: "generate-bash-completion",
}
// This flag prints the version for the application
var VersionFlag = BoolFlag{
Name: "version, v",
Usage: "print the version",
}
// This flag prints the help for all commands and subcommands
var HelpFlag = BoolFlag{
Name: "help, h",
Usage: "show help",
}
// Flag is a common interface related to parsing flags in cli.
// For more advanced flag parsing techniques, it is recomended that
// this interface be implemented.
type Flag interface {
fmt.Stringer
// Apply Flag settings to the given flag set
Apply(*flag.FlagSet)
getName() string
}
func flagSet(name string, flags []Flag) *flag.FlagSet {
set := flag.NewFlagSet(name, flag.ContinueOnError)
for _, f := range flags {
f.Apply(set)
}
return set
}
func eachName(longName string, fn func(string)) {
parts := strings.Split(longName, ",")
for _, name := range parts {
name = strings.Trim(name, " ")
fn(name)
}
}
// Generic is a generic parseable type identified by a specific flag
type Generic interface {
Set(value string) error
String() string
}
// GenericFlag is the flag type for types implementing Generic
type GenericFlag struct {
Name string
Value Generic
Usage string
EnvVar string
}
func (f GenericFlag) String() string {
return withEnvHint(f.EnvVar, fmt.Sprintf("%s%s %v\t`%v` %s", prefixFor(f.Name), f.Name, f.Value, "-"+f.Name+" option -"+f.Name+" option", f.Usage))
}
func (f GenericFlag) Apply(set *flag.FlagSet) {
val := f.Value
if f.EnvVar != "" {
if envVal := os.Getenv(f.EnvVar); envVal != "" {
val.Set(envVal)
}
}
eachName(f.Name, func(name string) {
set.Var(f.Value, name, f.Usage)
})
}
func (f GenericFlag) getName() string {
return f.Name
}
type StringSlice []string
func (f *StringSlice) Set(value string) error {
*f = append(*f, value)
return nil
}
func (f *StringSlice) String() string {
return fmt.Sprintf("%s", *f)
}
func (f *StringSlice) Value() []string {
return *f
}
type StringSliceFlag struct {
Name string
Value *StringSlice
Usage string
EnvVar string
}
func (f StringSliceFlag) String() string {
firstName := strings.Trim(strings.Split(f.Name, ",")[0], " ")
pref := prefixFor(firstName)
return withEnvHint(f.EnvVar, fmt.Sprintf("%s '%v'\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage))
}
func (f StringSliceFlag) Apply(set *flag.FlagSet) {
if f.EnvVar != "" {
if envVal := os.Getenv(f.EnvVar); envVal != "" {
newVal := &StringSlice{}
for _, s := range strings.Split(envVal, ",") {
newVal.Set(s)
}
f.Value = newVal
}
}
eachName(f.Name, func(name string) {
set.Var(f.Value, name, f.Usage)
})
}
func (f StringSliceFlag) getName() string {
return f.Name
}
type IntSlice []int
func (f *IntSlice) Set(value string) error {
tmp, err := strconv.Atoi(value)
if err != nil {
return err
} else {
*f = append(*f, tmp)
}
return nil
}
func (f *IntSlice) String() string {
return fmt.Sprintf("%d", *f)
}
func (f *IntSlice) Value() []int {
return *f
}
type IntSliceFlag struct {
Name string
Value *IntSlice
Usage string
EnvVar string
}
func (f IntSliceFlag) String() string {
firstName := strings.Trim(strings.Split(f.Name, ",")[0], " ")
pref := prefixFor(firstName)
return withEnvHint(f.EnvVar, fmt.Sprintf("%s '%v'\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage))
}
func (f IntSliceFlag) Apply(set *flag.FlagSet) {
if f.EnvVar != "" {
if envVal := os.Getenv(f.EnvVar); envVal != "" {
newVal := &IntSlice{}
for _, s := range strings.Split(envVal, ",") {
err := newVal.Set(s)
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
}
}
f.Value = newVal
}
}
eachName(f.Name, func(name string) {
set.Var(f.Value, name, f.Usage)
})
}
func (f IntSliceFlag) getName() string {
return f.Name
}
type BoolFlag struct {
Name string
Usage string
EnvVar string
}
func (f BoolFlag) String() string {
return withEnvHint(f.EnvVar, fmt.Sprintf("%s\t%v", prefixedNames(f.Name), f.Usage))
}
func (f BoolFlag) Apply(set *flag.FlagSet) {
val := false
if f.EnvVar != "" {
if envVal := os.Getenv(f.EnvVar); envVal != "" {
envValBool, err := strconv.ParseBool(envVal)
if err == nil {
val = envValBool
}
}
}
eachName(f.Name, func(name string) {
set.Bool(name, val, f.Usage)
})
}
func (f BoolFlag) getName() string {
return f.Name
}
type BoolTFlag struct {
Name string
Usage string
EnvVar string
}
func (f BoolTFlag) String() string {
return withEnvHint(f.EnvVar, fmt.Sprintf("%s\t%v", prefixedNames(f.Name), f.Usage))
}
func (f BoolTFlag) Apply(set *flag.FlagSet) {
val := true
if f.EnvVar != "" {
if envVal := os.Getenv(f.EnvVar); envVal != "" {
envValBool, err := strconv.ParseBool(envVal)
if err == nil {
val = envValBool
}
}
}
eachName(f.Name, func(name string) {
set.Bool(name, val, f.Usage)
})
}
func (f BoolTFlag) getName() string {
return f.Name
}
type StringFlag struct {
Name string
Value string
Usage string
EnvVar string
}
func (f StringFlag) String() string {
var fmtString string
fmtString = "%s %v\t%v"
if len(f.Value) > 0 {
fmtString = "%s '%v'\t%v"
} else {
fmtString = "%s %v\t%v"
}
return withEnvHint(f.EnvVar, fmt.Sprintf(fmtString, prefixedNames(f.Name), f.Value, f.Usage))
}
func (f StringFlag) Apply(set *flag.FlagSet) {
if f.EnvVar != "" {
if envVal := os.Getenv(f.EnvVar); envVal != "" {
f.Value = envVal
}
}
eachName(f.Name, func(name string) {
set.String(name, f.Value, f.Usage)
})
}
func (f StringFlag) getName() string {
return f.Name
}
type IntFlag struct {
Name string
Value int
Usage string
EnvVar string
}
func (f IntFlag) String() string {
return withEnvHint(f.EnvVar, fmt.Sprintf("%s '%v'\t%v", prefixedNames(f.Name), f.Value, f.Usage))
}
func (f IntFlag) Apply(set *flag.FlagSet) {
if f.EnvVar != "" {
if envVal := os.Getenv(f.EnvVar); envVal != "" {
envValInt, err := strconv.ParseUint(envVal, 10, 64)
if err == nil {
f.Value = int(envValInt)
}
}
}
eachName(f.Name, func(name string) {
set.Int(name, f.Value, f.Usage)
})
}
func (f IntFlag) getName() string {
return f.Name
}
type DurationFlag struct {
Name string
Value time.Duration
Usage string
EnvVar string
}
func (f DurationFlag) String() string {
return withEnvHint(f.EnvVar, fmt.Sprintf("%s '%v'\t%v", prefixedNames(f.Name), f.Value, f.Usage))
}
func (f DurationFlag) Apply(set *flag.FlagSet) {
if f.EnvVar != "" {
if envVal := os.Getenv(f.EnvVar); envVal != "" {
envValDuration, err := time.ParseDuration(envVal)
if err == nil {
f.Value = envValDuration
}
}
}
eachName(f.Name, func(name string) {
set.Duration(name, f.Value, f.Usage)
})
}
func (f DurationFlag) getName() string {
return f.Name
}
type Float64Flag struct {
Name string
Value float64
Usage string
EnvVar string
}
func (f Float64Flag) String() string {
return withEnvHint(f.EnvVar, fmt.Sprintf("%s '%v'\t%v", prefixedNames(f.Name), f.Value, f.Usage))
}
func (f Float64Flag) Apply(set *flag.FlagSet) {
if f.EnvVar != "" {
if envVal := os.Getenv(f.EnvVar); envVal != "" {
envValFloat, err := strconv.ParseFloat(envVal, 10)
if err == nil {
f.Value = float64(envValFloat)
}
}
}
eachName(f.Name, func(name string) {
set.Float64(name, f.Value, f.Usage)
})
}
func (f Float64Flag) getName() string {
return f.Name
}
func prefixFor(name string) (prefix string) {
if len(name) == 1 {
prefix = "-"
} else {
prefix = "--"
}
return
}
func prefixedNames(fullName string) (prefixed string) {
parts := strings.Split(fullName, ",")
for i, name := range parts {
name = strings.Trim(name, " ")
prefixed += prefixFor(name) + name
if i < len(parts)-1 {
prefixed += ", "
}
}
return
}
func withEnvHint(envVar, str string) string {
envText := ""
if envVar != "" {
envText = fmt.Sprintf(" [$%s]", envVar)
}
return str + envText
}

View File

@ -0,0 +1,587 @@
package cli_test
import (
"fmt"
"os"
"reflect"
"strings"
"testing"
"github.com/codegangsta/cli"
)
var boolFlagTests = []struct {
name string
expected string
}{
{"help", "--help\t"},
{"h", "-h\t"},
}
func TestBoolFlagHelpOutput(t *testing.T) {
for _, test := range boolFlagTests {
flag := cli.BoolFlag{Name: test.name}
output := flag.String()
if output != test.expected {
t.Errorf("%s does not match %s", output, test.expected)
}
}
}
var stringFlagTests = []struct {
name string
value string
expected string
}{
{"help", "", "--help \t"},
{"h", "", "-h \t"},
{"h", "", "-h \t"},
{"test", "Something", "--test 'Something'\t"},
}
func TestStringFlagHelpOutput(t *testing.T) {
for _, test := range stringFlagTests {
flag := cli.StringFlag{Name: test.name, Value: test.value}
output := flag.String()
if output != test.expected {
t.Errorf("%s does not match %s", output, test.expected)
}
}
}
func TestStringFlagWithEnvVarHelpOutput(t *testing.T) {
os.Setenv("APP_FOO", "derp")
for _, test := range stringFlagTests {
flag := cli.StringFlag{Name: test.name, Value: test.value, EnvVar: "APP_FOO"}
output := flag.String()
if !strings.HasSuffix(output, " [$APP_FOO]") {
t.Errorf("%s does not end with [$APP_FOO]", output)
}
}
}
var stringSliceFlagTests = []struct {
name string
value *cli.StringSlice
expected string
}{
{"help", func() *cli.StringSlice {
s := &cli.StringSlice{}
s.Set("")
return s
}(), "--help '--help option --help option'\t"},
{"h", func() *cli.StringSlice {
s := &cli.StringSlice{}
s.Set("")
return s
}(), "-h '-h option -h option'\t"},
{"h", func() *cli.StringSlice {
s := &cli.StringSlice{}
s.Set("")
return s
}(), "-h '-h option -h option'\t"},
{"test", func() *cli.StringSlice {
s := &cli.StringSlice{}
s.Set("Something")
return s
}(), "--test '--test option --test option'\t"},
}
func TestStringSliceFlagHelpOutput(t *testing.T) {
for _, test := range stringSliceFlagTests {
flag := cli.StringSliceFlag{Name: test.name, Value: test.value}
output := flag.String()
if output != test.expected {
t.Errorf("%q does not match %q", output, test.expected)
}
}
}
func TestStringSliceFlagWithEnvVarHelpOutput(t *testing.T) {
os.Setenv("APP_QWWX", "11,4")
for _, test := range stringSliceFlagTests {
flag := cli.StringSliceFlag{Name: test.name, Value: test.value, EnvVar: "APP_QWWX"}
output := flag.String()
if !strings.HasSuffix(output, " [$APP_QWWX]") {
t.Errorf("%q does not end with [$APP_QWWX]", output)
}
}
}
var intFlagTests = []struct {
name string
expected string
}{
{"help", "--help '0'\t"},
{"h", "-h '0'\t"},
}
func TestIntFlagHelpOutput(t *testing.T) {
for _, test := range intFlagTests {
flag := cli.IntFlag{Name: test.name}
output := flag.String()
if output != test.expected {
t.Errorf("%s does not match %s", output, test.expected)
}
}
}
func TestIntFlagWithEnvVarHelpOutput(t *testing.T) {
os.Setenv("APP_BAR", "2")
for _, test := range intFlagTests {
flag := cli.IntFlag{Name: test.name, EnvVar: "APP_BAR"}
output := flag.String()
if !strings.HasSuffix(output, " [$APP_BAR]") {
t.Errorf("%s does not end with [$APP_BAR]", output)
}
}
}
var durationFlagTests = []struct {
name string
expected string
}{
{"help", "--help '0'\t"},
{"h", "-h '0'\t"},
}
func TestDurationFlagHelpOutput(t *testing.T) {
for _, test := range durationFlagTests {
flag := cli.DurationFlag{Name: test.name}
output := flag.String()
if output != test.expected {
t.Errorf("%s does not match %s", output, test.expected)
}
}
}
func TestDurationFlagWithEnvVarHelpOutput(t *testing.T) {
os.Setenv("APP_BAR", "2h3m6s")
for _, test := range durationFlagTests {
flag := cli.DurationFlag{Name: test.name, EnvVar: "APP_BAR"}
output := flag.String()
if !strings.HasSuffix(output, " [$APP_BAR]") {
t.Errorf("%s does not end with [$APP_BAR]", output)
}
}
}
var intSliceFlagTests = []struct {
name string
value *cli.IntSlice
expected string
}{
{"help", &cli.IntSlice{}, "--help '--help option --help option'\t"},
{"h", &cli.IntSlice{}, "-h '-h option -h option'\t"},
{"h", &cli.IntSlice{}, "-h '-h option -h option'\t"},
{"test", func() *cli.IntSlice {
i := &cli.IntSlice{}
i.Set("9")
return i
}(), "--test '--test option --test option'\t"},
}
func TestIntSliceFlagHelpOutput(t *testing.T) {
for _, test := range intSliceFlagTests {
flag := cli.IntSliceFlag{Name: test.name, Value: test.value}
output := flag.String()
if output != test.expected {
t.Errorf("%q does not match %q", output, test.expected)
}
}
}
func TestIntSliceFlagWithEnvVarHelpOutput(t *testing.T) {
os.Setenv("APP_SMURF", "42,3")
for _, test := range intSliceFlagTests {
flag := cli.IntSliceFlag{Name: test.name, Value: test.value, EnvVar: "APP_SMURF"}
output := flag.String()
if !strings.HasSuffix(output, " [$APP_SMURF]") {
t.Errorf("%q does not end with [$APP_SMURF]", output)
}
}
}
var float64FlagTests = []struct {
name string
expected string
}{
{"help", "--help '0'\t"},
{"h", "-h '0'\t"},
}
func TestFloat64FlagHelpOutput(t *testing.T) {
for _, test := range float64FlagTests {
flag := cli.Float64Flag{Name: test.name}
output := flag.String()
if output != test.expected {
t.Errorf("%s does not match %s", output, test.expected)
}
}
}
func TestFloat64FlagWithEnvVarHelpOutput(t *testing.T) {
os.Setenv("APP_BAZ", "99.4")
for _, test := range float64FlagTests {
flag := cli.Float64Flag{Name: test.name, EnvVar: "APP_BAZ"}
output := flag.String()
if !strings.HasSuffix(output, " [$APP_BAZ]") {
t.Errorf("%s does not end with [$APP_BAZ]", output)
}
}
}
var genericFlagTests = []struct {
name string
value cli.Generic
expected string
}{
{"help", &Parser{}, "--help <nil>\t`-help option -help option` "},
{"h", &Parser{}, "-h <nil>\t`-h option -h option` "},
{"test", &Parser{}, "--test <nil>\t`-test option -test option` "},
}
func TestGenericFlagHelpOutput(t *testing.T) {
for _, test := range genericFlagTests {
flag := cli.GenericFlag{Name: test.name}
output := flag.String()
if output != test.expected {
t.Errorf("%q does not match %q", output, test.expected)
}
}
}
func TestGenericFlagWithEnvVarHelpOutput(t *testing.T) {
os.Setenv("APP_ZAP", "3")
for _, test := range genericFlagTests {
flag := cli.GenericFlag{Name: test.name, EnvVar: "APP_ZAP"}
output := flag.String()
if !strings.HasSuffix(output, " [$APP_ZAP]") {
t.Errorf("%s does not end with [$APP_ZAP]", output)
}
}
}
func TestParseMultiString(t *testing.T) {
(&cli.App{
Flags: []cli.Flag{
cli.StringFlag{Name: "serve, s"},
},
Action: func(ctx *cli.Context) {
if ctx.String("serve") != "10" {
t.Errorf("main name not set")
}
if ctx.String("s") != "10" {
t.Errorf("short name not set")
}
},
}).Run([]string{"run", "-s", "10"})
}
func TestParseMultiStringFromEnv(t *testing.T) {
os.Setenv("APP_COUNT", "20")
(&cli.App{
Flags: []cli.Flag{
cli.StringFlag{Name: "count, c", EnvVar: "APP_COUNT"},
},
Action: func(ctx *cli.Context) {
if ctx.String("count") != "20" {
t.Errorf("main name not set")
}
if ctx.String("c") != "20" {
t.Errorf("short name not set")
}
},
}).Run([]string{"run"})
}
func TestParseMultiStringSlice(t *testing.T) {
(&cli.App{
Flags: []cli.Flag{
cli.StringSliceFlag{Name: "serve, s", Value: &cli.StringSlice{}},
},
Action: func(ctx *cli.Context) {
if !reflect.DeepEqual(ctx.StringSlice("serve"), []string{"10", "20"}) {
t.Errorf("main name not set")
}
if !reflect.DeepEqual(ctx.StringSlice("s"), []string{"10", "20"}) {
t.Errorf("short name not set")
}
},
}).Run([]string{"run", "-s", "10", "-s", "20"})
}
func TestParseMultiStringSliceFromEnv(t *testing.T) {
os.Setenv("APP_INTERVALS", "20,30,40")
(&cli.App{
Flags: []cli.Flag{
cli.StringSliceFlag{Name: "intervals, i", Value: &cli.StringSlice{}, EnvVar: "APP_INTERVALS"},
},
Action: func(ctx *cli.Context) {
if !reflect.DeepEqual(ctx.StringSlice("intervals"), []string{"20", "30", "40"}) {
t.Errorf("main name not set from env")
}
if !reflect.DeepEqual(ctx.StringSlice("i"), []string{"20", "30", "40"}) {
t.Errorf("short name not set from env")
}
},
}).Run([]string{"run"})
}
func TestParseMultiInt(t *testing.T) {
a := cli.App{
Flags: []cli.Flag{
cli.IntFlag{Name: "serve, s"},
},
Action: func(ctx *cli.Context) {
if ctx.Int("serve") != 10 {
t.Errorf("main name not set")
}
if ctx.Int("s") != 10 {
t.Errorf("short name not set")
}
},
}
a.Run([]string{"run", "-s", "10"})
}
func TestParseMultiIntFromEnv(t *testing.T) {
os.Setenv("APP_TIMEOUT_SECONDS", "10")
a := cli.App{
Flags: []cli.Flag{
cli.IntFlag{Name: "timeout, t", EnvVar: "APP_TIMEOUT_SECONDS"},
},
Action: func(ctx *cli.Context) {
if ctx.Int("timeout") != 10 {
t.Errorf("main name not set")
}
if ctx.Int("t") != 10 {
t.Errorf("short name not set")
}
},
}
a.Run([]string{"run"})
}
func TestParseMultiIntSlice(t *testing.T) {
(&cli.App{
Flags: []cli.Flag{
cli.IntSliceFlag{Name: "serve, s", Value: &cli.IntSlice{}},
},
Action: func(ctx *cli.Context) {
if !reflect.DeepEqual(ctx.IntSlice("serve"), []int{10, 20}) {
t.Errorf("main name not set")
}
if !reflect.DeepEqual(ctx.IntSlice("s"), []int{10, 20}) {
t.Errorf("short name not set")
}
},
}).Run([]string{"run", "-s", "10", "-s", "20"})
}
func TestParseMultiIntSliceFromEnv(t *testing.T) {
os.Setenv("APP_INTERVALS", "20,30,40")
(&cli.App{
Flags: []cli.Flag{
cli.IntSliceFlag{Name: "intervals, i", Value: &cli.IntSlice{}, EnvVar: "APP_INTERVALS"},
},
Action: func(ctx *cli.Context) {
if !reflect.DeepEqual(ctx.IntSlice("intervals"), []int{20, 30, 40}) {
t.Errorf("main name not set from env")
}
if !reflect.DeepEqual(ctx.IntSlice("i"), []int{20, 30, 40}) {
t.Errorf("short name not set from env")
}
},
}).Run([]string{"run"})
}
func TestParseMultiFloat64(t *testing.T) {
a := cli.App{
Flags: []cli.Flag{
cli.Float64Flag{Name: "serve, s"},
},
Action: func(ctx *cli.Context) {
if ctx.Float64("serve") != 10.2 {
t.Errorf("main name not set")
}
if ctx.Float64("s") != 10.2 {
t.Errorf("short name not set")
}
},
}
a.Run([]string{"run", "-s", "10.2"})
}
func TestParseMultiFloat64FromEnv(t *testing.T) {
os.Setenv("APP_TIMEOUT_SECONDS", "15.5")
a := cli.App{
Flags: []cli.Flag{
cli.Float64Flag{Name: "timeout, t", EnvVar: "APP_TIMEOUT_SECONDS"},
},
Action: func(ctx *cli.Context) {
if ctx.Float64("timeout") != 15.5 {
t.Errorf("main name not set")
}
if ctx.Float64("t") != 15.5 {
t.Errorf("short name not set")
}
},
}
a.Run([]string{"run"})
}
func TestParseMultiBool(t *testing.T) {
a := cli.App{
Flags: []cli.Flag{
cli.BoolFlag{Name: "serve, s"},
},
Action: func(ctx *cli.Context) {
if ctx.Bool("serve") != true {
t.Errorf("main name not set")
}
if ctx.Bool("s") != true {
t.Errorf("short name not set")
}
},
}
a.Run([]string{"run", "--serve"})
}
func TestParseMultiBoolFromEnv(t *testing.T) {
os.Setenv("APP_DEBUG", "1")
a := cli.App{
Flags: []cli.Flag{
cli.BoolFlag{Name: "debug, d", EnvVar: "APP_DEBUG"},
},
Action: func(ctx *cli.Context) {
if ctx.Bool("debug") != true {
t.Errorf("main name not set from env")
}
if ctx.Bool("d") != true {
t.Errorf("short name not set from env")
}
},
}
a.Run([]string{"run"})
}
func TestParseMultiBoolT(t *testing.T) {
a := cli.App{
Flags: []cli.Flag{
cli.BoolTFlag{Name: "serve, s"},
},
Action: func(ctx *cli.Context) {
if ctx.BoolT("serve") != true {
t.Errorf("main name not set")
}
if ctx.BoolT("s") != true {
t.Errorf("short name not set")
}
},
}
a.Run([]string{"run", "--serve"})
}
func TestParseMultiBoolTFromEnv(t *testing.T) {
os.Setenv("APP_DEBUG", "0")
a := cli.App{
Flags: []cli.Flag{
cli.BoolTFlag{Name: "debug, d", EnvVar: "APP_DEBUG"},
},
Action: func(ctx *cli.Context) {
if ctx.BoolT("debug") != false {
t.Errorf("main name not set from env")
}
if ctx.BoolT("d") != false {
t.Errorf("short name not set from env")
}
},
}
a.Run([]string{"run"})
}
type Parser [2]string
func (p *Parser) Set(value string) error {
parts := strings.Split(value, ",")
if len(parts) != 2 {
return fmt.Errorf("invalid format")
}
(*p)[0] = parts[0]
(*p)[1] = parts[1]
return nil
}
func (p *Parser) String() string {
return fmt.Sprintf("%s,%s", p[0], p[1])
}
func TestParseGeneric(t *testing.T) {
a := cli.App{
Flags: []cli.Flag{
cli.GenericFlag{Name: "serve, s", Value: &Parser{}},
},
Action: func(ctx *cli.Context) {
if !reflect.DeepEqual(ctx.Generic("serve"), &Parser{"10", "20"}) {
t.Errorf("main name not set")
}
if !reflect.DeepEqual(ctx.Generic("s"), &Parser{"10", "20"}) {
t.Errorf("short name not set")
}
},
}
a.Run([]string{"run", "-s", "10,20"})
}
func TestParseGenericFromEnv(t *testing.T) {
os.Setenv("APP_SERVE", "20,30")
a := cli.App{
Flags: []cli.Flag{
cli.GenericFlag{Name: "serve, s", Value: &Parser{}, EnvVar: "APP_SERVE"},
},
Action: func(ctx *cli.Context) {
if !reflect.DeepEqual(ctx.Generic("serve"), &Parser{"20", "30"}) {
t.Errorf("main name not set from env")
}
if !reflect.DeepEqual(ctx.Generic("s"), &Parser{"20", "30"}) {
t.Errorf("short name not set from env")
}
},
}
a.Run([]string{"run"})
}

View File

@ -0,0 +1,224 @@
package cli
import (
"fmt"
"os"
"text/tabwriter"
"text/template"
)
// The text template for the Default help topic.
// cli.go uses text/template to render templates. You can
// render custom help text by setting this variable.
var AppHelpTemplate = `NAME:
{{.Name}} - {{.Usage}}
USAGE:
{{.Name}} {{if .Flags}}[global options] {{end}}command{{if .Flags}} [command options]{{end}} [arguments...]
VERSION:
{{.Version}}{{if or .Author .Email}}
AUTHOR:{{if .Author}}
{{.Author}}{{if .Email}} - <{{.Email}}>{{end}}{{else}}
{{.Email}}{{end}}{{end}}
COMMANDS:
{{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}}
{{end}}{{if .Flags}}
GLOBAL OPTIONS:
{{range .Flags}}{{.}}
{{end}}{{end}}
`
// The text template for the command help topic.
// cli.go uses text/template to render templates. You can
// render custom help text by setting this variable.
var CommandHelpTemplate = `NAME:
{{.Name}} - {{.Usage}}
USAGE:
command {{.Name}}{{if .Flags}} [command options]{{end}} [arguments...]{{if .Description}}
DESCRIPTION:
{{.Description}}{{end}}{{if .Flags}}
OPTIONS:
{{range .Flags}}{{.}}
{{end}}{{ end }}
`
// The text template for the subcommand help topic.
// cli.go uses text/template to render templates. You can
// render custom help text by setting this variable.
var SubcommandHelpTemplate = `NAME:
{{.Name}} - {{.Usage}}
USAGE:
{{.Name}} command{{if .Flags}} [command options]{{end}} [arguments...]
COMMANDS:
{{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}}
{{end}}{{if .Flags}}
OPTIONS:
{{range .Flags}}{{.}}
{{end}}{{end}}
`
var helpCommand = Command{
Name: "help",
ShortName: "h",
Usage: "Shows a list of commands or help for one command",
Action: func(c *Context) {
args := c.Args()
if args.Present() {
ShowCommandHelp(c, args.First())
} else {
ShowAppHelp(c)
}
},
}
var helpSubcommand = Command{
Name: "help",
ShortName: "h",
Usage: "Shows a list of commands or help for one command",
Action: func(c *Context) {
args := c.Args()
if args.Present() {
ShowCommandHelp(c, args.First())
} else {
ShowSubcommandHelp(c)
}
},
}
// Prints help for the App
var HelpPrinter = printHelp
// Prints version for the App
var VersionPrinter = printVersion
func ShowAppHelp(c *Context) {
HelpPrinter(AppHelpTemplate, c.App)
}
// Prints the list of subcommands as the default app completion method
func DefaultAppComplete(c *Context) {
for _, command := range c.App.Commands {
fmt.Println(command.Name)
if command.ShortName != "" {
fmt.Println(command.ShortName)
}
}
}
// Prints help for the given command
func ShowCommandHelp(c *Context, command string) {
for _, c := range c.App.Commands {
if c.HasName(command) {
HelpPrinter(CommandHelpTemplate, c)
return
}
}
if c.App.CommandNotFound != nil {
c.App.CommandNotFound(c, command)
} else {
fmt.Printf("No help topic for '%v'\n", command)
}
}
// Prints help for the given subcommand
func ShowSubcommandHelp(c *Context) {
HelpPrinter(SubcommandHelpTemplate, c.App)
}
// Prints the version number of the App
func ShowVersion(c *Context) {
VersionPrinter(c)
}
func printVersion(c *Context) {
fmt.Printf("%v version %v\n", c.App.Name, c.App.Version)
}
// Prints the lists of commands within a given context
func ShowCompletions(c *Context) {
a := c.App
if a != nil && a.BashComplete != nil {
a.BashComplete(c)
}
}
// Prints the custom completions for a given command
func ShowCommandCompletions(ctx *Context, command string) {
c := ctx.App.Command(command)
if c != nil && c.BashComplete != nil {
c.BashComplete(ctx)
}
}
func printHelp(templ string, data interface{}) {
w := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', 0)
t := template.Must(template.New("help").Parse(templ))
err := t.Execute(w, data)
if err != nil {
panic(err)
}
w.Flush()
}
func checkVersion(c *Context) bool {
if c.GlobalBool("version") {
ShowVersion(c)
return true
}
return false
}
func checkHelp(c *Context) bool {
if c.GlobalBool("h") || c.GlobalBool("help") {
ShowAppHelp(c)
return true
}
return false
}
func checkCommandHelp(c *Context, name string) bool {
if c.Bool("h") || c.Bool("help") {
ShowCommandHelp(c, name)
return true
}
return false
}
func checkSubcommandHelp(c *Context) bool {
if c.GlobalBool("h") || c.GlobalBool("help") {
ShowSubcommandHelp(c)
return true
}
return false
}
func checkCompletions(c *Context) bool {
if c.GlobalBool(BashCompletionFlag.Name) && c.App.EnableBashCompletion {
ShowCompletions(c)
return true
}
return false
}
func checkCommandCompletions(c *Context, name string) bool {
if c.Bool(BashCompletionFlag.Name) && c.App.EnableBashCompletion {
ShowCommandCompletions(c, name)
return true
}
return false
}

View File

@ -0,0 +1,19 @@
package cli_test
import (
"reflect"
"testing"
)
/* Test Helpers */
func expect(t *testing.T, a interface{}, b interface{}) {
if a != b {
t.Errorf("Expected %v (type %v) - Got %v (type %v)", b, reflect.TypeOf(b), a, reflect.TypeOf(a))
}
}
func refute(t *testing.T, a interface{}, b interface{}) {
if a == b {
t.Errorf("Did not expect %v (type %v) - Got %v (type %v)", b, reflect.TypeOf(b), a, reflect.TypeOf(a))
}
}

View File

@ -8,7 +8,7 @@ func (c *Client) AddChildDir(key string, ttl uint64) (*Response, error) {
return nil, err
}
return raw.toResponse()
return raw.Unmarshal()
}
// Add a new file with a random etcd-generated key under the given path.
@ -19,5 +19,5 @@ func (c *Client) AddChild(key string, value string, ttl uint64) (*Response, erro
return nil, err
}
return raw.toResponse()
return raw.Unmarshal()
}

View File

@ -32,8 +32,8 @@ type Config struct {
CertFile string `json:"certFile"`
KeyFile string `json:"keyFile"`
CaCertFile []string `json:"caCertFiles"`
Timeout time.Duration `json:"timeout"`
Consistency string `json: "consistency"`
DialTimeout time.Duration `json:"timeout"`
Consistency string `json:"consistency"`
}
type Client struct {
@ -42,7 +42,20 @@ type Client struct {
httpClient *http.Client
persistence io.Writer
cURLch chan string
keyPrefix string
// CheckRetry can be used to control the policy for failed requests
// and modify the cluster if needed.
// The client calls it before sending requests again, and
// stops retrying if CheckRetry returns some error. The cases that
// this function needs to handle include no response and unexpected
// http status code of response.
// If CheckRetry is nil, client will call the default one
// `DefaultCheckRetry`.
// Argument cluster is the etcd.Cluster object that these requests have been made on.
// Argument numReqs is the number of http.Requests that have been made so far.
// Argument lastResp is the http.Responses from the last request.
// Argument err is the reason of the failure.
CheckRetry func(cluster *Cluster, numReqs int,
lastResp http.Response, err error) error
}
// NewClient create a basic client that is configured to be used
@ -50,15 +63,14 @@ type Client struct {
func NewClient(machines []string) *Client {
config := Config{
// default timeout is one second
Timeout: time.Second,
DialTimeout: time.Second,
// default consistency level is STRONG
Consistency: STRONG_CONSISTENCY,
}
client := &Client{
cluster: NewCluster(machines),
config: config,
keyPrefix: path.Join(version, "keys"),
cluster: NewCluster(machines),
config: config,
}
client.initHTTPClient()
@ -76,7 +88,7 @@ func NewTLSClient(machines []string, cert, key, caCert string) (*Client, error)
config := Config{
// default timeout is one second
Timeout: time.Second,
DialTimeout: time.Second,
// default consistency level is STRONG
Consistency: STRONG_CONSISTENCY,
CertFile: cert,
@ -85,9 +97,8 @@ func NewTLSClient(machines []string, cert, key, caCert string) (*Client, error)
}
client := &Client{
cluster: NewCluster(machines),
config: config,
keyPrefix: path.Join(version, "keys"),
cluster: NewCluster(machines),
config: config,
}
err := client.initHTTPSClient(cert, key)
@ -157,16 +168,10 @@ func (c *Client) SetTransport(tr *http.Transport) {
c.httpClient.Transport = tr
}
// SetKeyPrefix changes the key prefix from the default `/v2/keys` to whatever
// is set.
func (c *Client) SetKeyPrefix(prefix string) {
c.keyPrefix = prefix
}
// initHTTPClient initializes a HTTP client for etcd client
func (c *Client) initHTTPClient() {
tr := &http.Transport{
Dial: dialTimeout,
Dial: c.dial,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
@ -192,7 +197,7 @@ func (c *Client) initHTTPSClient(cert, key string) error {
tr := &http.Transport{
TLSClientConfig: tlsConfig,
Dial: dialTimeout,
Dial: c.dial,
}
c.httpClient = &http.Client{Transport: tr}
@ -226,6 +231,11 @@ func (c *Client) SetConsistency(consistency string) error {
return nil
}
// Sets the DialTimeout value
func (c *Client) SetDialTimeout(d time.Duration) {
c.config.DialTimeout = d
}
// AddRootCA adds a root CA cert for the etcd client
func (c *Client) AddRootCA(caCert string) error {
if c.httpClient == nil {
@ -326,9 +336,29 @@ func (c *Client) createHttpPath(serverName string, _path string) string {
return u.String()
}
// Dial with timeout.
func dialTimeout(network, addr string) (net.Conn, error) {
return net.DialTimeout(network, addr, time.Second)
// dial attempts to open a TCP connection to the provided address, explicitly
// enabling keep-alives with a one-second interval.
func (c *Client) dial(network, addr string) (net.Conn, error) {
conn, err := net.DialTimeout(network, addr, c.config.DialTimeout)
if err != nil {
return nil, err
}
tcpConn, ok := conn.(*net.TCPConn)
if !ok {
return nil, errors.New("Failed type-assertion of net.Conn as *net.TCPConn")
}
// Keep TCP alive to check whether or not the remote machine is down
if err = tcpConn.SetKeepAlive(true); err != nil {
return nil, err
}
if err = tcpConn.SetKeepAlivePeriod(time.Second); err != nil {
return nil, err
}
return tcpConn, nil
}
func (c *Client) OpenCURL() {
@ -391,8 +421,8 @@ func (c *Client) MarshalJSON() ([]byte, error) {
// as defined by the standard JSON package.
func (c *Client) UnmarshalJSON(b []byte) error {
temp := struct {
Config Config `json: "config"`
Cluster *Cluster `json: "cluster"`
Config Config `json:"config"`
Cluster *Cluster `json:"cluster"`
}{}
err := json.Unmarshal(b, &temp)
if err != nil {

View File

@ -8,7 +8,7 @@ func (c *Client) CompareAndDelete(key string, prevValue string, prevIndex uint64
return nil, err
}
return raw.toResponse()
return raw.Unmarshal()
}
func (c *Client) RawCompareAndDelete(key string, prevValue string, prevIndex uint64) (*RawResponse, error) {
@ -16,7 +16,7 @@ func (c *Client) RawCompareAndDelete(key string, prevValue string, prevIndex uin
return nil, fmt.Errorf("You must give either prevValue or prevIndex.")
}
options := options{}
options := Options{}
if prevValue != "" {
options["prevValue"] = prevValue
}

View File

@ -9,7 +9,7 @@ func (c *Client) CompareAndSwap(key string, value string, ttl uint64,
return nil, err
}
return raw.toResponse()
return raw.Unmarshal()
}
func (c *Client) RawCompareAndSwap(key string, value string, ttl uint64,
@ -18,7 +18,7 @@ func (c *Client) RawCompareAndSwap(key string, value string, ttl uint64,
return nil, fmt.Errorf("You must give either prevValue or prevIndex.")
}
options := options{}
options := Options{}
if prevValue != "" {
options["prevValue"] = prevValue
}

View File

@ -0,0 +1,55 @@
package etcd
import (
"fmt"
"io/ioutil"
"log"
"strings"
)
var logger *etcdLogger
func SetLogger(l *log.Logger) {
logger = &etcdLogger{l}
}
func GetLogger() *log.Logger {
return logger.log
}
type etcdLogger struct {
log *log.Logger
}
func (p *etcdLogger) Debug(args ...interface{}) {
msg := "DEBUG: " + fmt.Sprint(args...)
p.log.Println(msg)
}
func (p *etcdLogger) Debugf(f string, args ...interface{}) {
msg := "DEBUG: " + fmt.Sprintf(f, args...)
// Append newline if necessary
if !strings.HasSuffix(msg, "\n") {
msg = msg + "\n"
}
p.log.Print(msg)
}
func (p *etcdLogger) Warning(args ...interface{}) {
msg := "WARNING: " + fmt.Sprint(args...)
p.log.Println(msg)
}
func (p *etcdLogger) Warningf(f string, args ...interface{}) {
msg := "WARNING: " + fmt.Sprintf(f, args...)
// Append newline if necessary
if !strings.HasSuffix(msg, "\n") {
msg = msg + "\n"
}
p.log.Print(msg)
}
func init() {
// Default logger uses the go default log.
SetLogger(log.New(ioutil.Discard, "go-etcd", log.LstdFlags))
}

View File

@ -0,0 +1,28 @@
package etcd
import (
"testing"
)
type Foo struct{}
type Bar struct {
one string
two int
}
// Tests that logs don't panic with arbitrary interfaces
func TestDebug(t *testing.T) {
f := &Foo{}
b := &Bar{"asfd", 3}
for _, test := range []interface{}{
1234,
"asdf",
f,
b,
} {
logger.Debug(test)
logger.Debugf("something, %s", test)
logger.Warning(test)
logger.Warningf("something, %s", test)
}
}

View File

@ -16,7 +16,7 @@ func (c *Client) Delete(key string, recursive bool) (*Response, error) {
return nil, err
}
return raw.toResponse()
return raw.Unmarshal()
}
// DeleteDir deletes an empty directory or a key value pair
@ -27,11 +27,11 @@ func (c *Client) DeleteDir(key string) (*Response, error) {
return nil, err
}
return raw.toResponse()
return raw.Unmarshal()
}
func (c *Client) RawDelete(key string, recursive bool, dir bool) (*RawResponse, error) {
ops := options{
ops := Options{
"recursive": recursive,
"dir": dir,
}

View File

@ -14,11 +14,11 @@ func (c *Client) Get(key string, sort, recursive bool) (*Response, error) {
return nil, err
}
return raw.toResponse()
return raw.Unmarshal()
}
func (c *Client) RawGet(key string, sort, recursive bool) (*RawResponse, error) {
ops := options{
ops := Options{
"recursive": recursive,
"sorted": sort,
}

View File

@ -18,9 +18,9 @@ func cleanResult(result *Response) {
// TODO(philips): make this recursive.
cleanNode(result.Node)
for i, _ := range result.Node.Nodes {
cleanNode(&result.Node.Nodes[i])
cleanNode(result.Node.Nodes[i])
for j, _ := range result.Node.Nodes[i].Nodes {
cleanNode(&result.Node.Nodes[i].Nodes[j])
cleanNode(result.Node.Nodes[i].Nodes[j])
}
}
}
@ -67,12 +67,12 @@ func TestGetAll(t *testing.T) {
}
expected := Nodes{
Node{
&Node{
Key: "/fooDir/k0",
Value: "v0",
TTL: 5,
},
Node{
&Node{
Key: "/fooDir/k1",
Value: "v1",
TTL: 5,
@ -99,11 +99,11 @@ func TestGetAll(t *testing.T) {
}
expected = Nodes{
Node{
&Node{
Key: "/fooDir/childDir",
Dir: true,
Nodes: Nodes{
Node{
&Node{
Key: "/fooDir/childDir/k2",
Value: "v2",
TTL: 5,
@ -111,12 +111,12 @@ func TestGetAll(t *testing.T) {
},
TTL: 5,
},
Node{
&Node{
Key: "/fooDir/k0",
Value: "v0",
TTL: 5,
},
Node{
&Node{
Key: "/fooDir/k1",
Value: "v1",
TTL: 5,

View File

@ -6,7 +6,7 @@ import (
"reflect"
)
type options map[string]interface{}
type Options map[string]interface{}
// An internally-used data structure that represents a mapping
// between valid options and their kinds
@ -42,7 +42,7 @@ var (
)
// Convert options to a string of HTML parameters
func (ops options) toParameters(validOps validOptions) (string, error) {
func (ops Options) toParameters(validOps validOptions) (string, error) {
p := "?"
values := url.Values{}

View File

@ -0,0 +1,396 @@
package etcd
import (
"errors"
"fmt"
"io"
"io/ioutil"
"math/rand"
"net/http"
"net/url"
"path"
"strings"
"sync"
"time"
)
// Errors introduced by handling requests
var (
ErrRequestCancelled = errors.New("sending request is cancelled")
)
type RawRequest struct {
Method string
RelativePath string
Values url.Values
Cancel <-chan bool
}
// NewRawRequest returns a new RawRequest
func NewRawRequest(method, relativePath string, values url.Values, cancel <-chan bool) *RawRequest {
return &RawRequest{
Method: method,
RelativePath: relativePath,
Values: values,
Cancel: cancel,
}
}
// getCancelable issues a cancelable GET request
func (c *Client) getCancelable(key string, options Options,
cancel <-chan bool) (*RawResponse, error) {
logger.Debugf("get %s [%s]", key, c.cluster.Leader)
p := keyToPath(key)
// If consistency level is set to STRONG, append
// the `consistent` query string.
if c.config.Consistency == STRONG_CONSISTENCY {
options["consistent"] = true
}
str, err := options.toParameters(VALID_GET_OPTIONS)
if err != nil {
return nil, err
}
p += str
req := NewRawRequest("GET", p, nil, cancel)
resp, err := c.SendRequest(req)
if err != nil {
return nil, err
}
return resp, nil
}
// get issues a GET request
func (c *Client) get(key string, options Options) (*RawResponse, error) {
return c.getCancelable(key, options, nil)
}
// put issues a PUT request
func (c *Client) put(key string, value string, ttl uint64,
options Options) (*RawResponse, error) {
logger.Debugf("put %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.Leader)
p := keyToPath(key)
str, err := options.toParameters(VALID_PUT_OPTIONS)
if err != nil {
return nil, err
}
p += str
req := NewRawRequest("PUT", p, buildValues(value, ttl), nil)
resp, err := c.SendRequest(req)
if err != nil {
return nil, err
}
return resp, nil
}
// post issues a POST request
func (c *Client) post(key string, value string, ttl uint64) (*RawResponse, error) {
logger.Debugf("post %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.Leader)
p := keyToPath(key)
req := NewRawRequest("POST", p, buildValues(value, ttl), nil)
resp, err := c.SendRequest(req)
if err != nil {
return nil, err
}
return resp, nil
}
// delete issues a DELETE request
func (c *Client) delete(key string, options Options) (*RawResponse, error) {
logger.Debugf("delete %s [%s]", key, c.cluster.Leader)
p := keyToPath(key)
str, err := options.toParameters(VALID_DELETE_OPTIONS)
if err != nil {
return nil, err
}
p += str
req := NewRawRequest("DELETE", p, nil, nil)
resp, err := c.SendRequest(req)
if err != nil {
return nil, err
}
return resp, nil
}
// SendRequest sends a HTTP request and returns a Response as defined by etcd
func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) {
var req *http.Request
var resp *http.Response
var httpPath string
var err error
var respBody []byte
var numReqs = 1
checkRetry := c.CheckRetry
if checkRetry == nil {
checkRetry = DefaultCheckRetry
}
cancelled := make(chan bool, 1)
reqLock := new(sync.Mutex)
if rr.Cancel != nil {
cancelRoutine := make(chan bool)
defer close(cancelRoutine)
go func() {
select {
case <-rr.Cancel:
cancelled <- true
logger.Debug("send.request is cancelled")
case <-cancelRoutine:
return
}
// Repeat canceling request until this thread is stopped
// because we have no idea about whether it succeeds.
for {
reqLock.Lock()
c.httpClient.Transport.(*http.Transport).CancelRequest(req)
reqLock.Unlock()
select {
case <-time.After(100 * time.Millisecond):
case <-cancelRoutine:
return
}
}
}()
}
// If we connect to a follower and consistency is required, retry until
// we connect to a leader
sleep := 25 * time.Millisecond
maxSleep := time.Second
for attempt := 0; ; attempt++ {
if attempt > 0 {
select {
case <-cancelled:
return nil, ErrRequestCancelled
case <-time.After(sleep):
sleep = sleep * 2
if sleep > maxSleep {
sleep = maxSleep
}
}
}
logger.Debug("Connecting to etcd: attempt ", attempt+1, " for ", rr.RelativePath)
if rr.Method == "GET" && c.config.Consistency == WEAK_CONSISTENCY {
// If it's a GET and consistency level is set to WEAK,
// then use a random machine.
httpPath = c.getHttpPath(true, rr.RelativePath)
} else {
// Else use the leader.
httpPath = c.getHttpPath(false, rr.RelativePath)
}
// Return a cURL command if curlChan is set
if c.cURLch != nil {
command := fmt.Sprintf("curl -X %s %s", rr.Method, httpPath)
for key, value := range rr.Values {
command += fmt.Sprintf(" -d %s=%s", key, value[0])
}
c.sendCURL(command)
}
logger.Debug("send.request.to ", httpPath, " | method ", rr.Method)
req, err := func() (*http.Request, error) {
reqLock.Lock()
defer reqLock.Unlock()
if rr.Values == nil {
if req, err = http.NewRequest(rr.Method, httpPath, nil); err != nil {
return nil, err
}
} else {
body := strings.NewReader(rr.Values.Encode())
if req, err = http.NewRequest(rr.Method, httpPath, body); err != nil {
return nil, err
}
req.Header.Set("Content-Type",
"application/x-www-form-urlencoded; param=value")
}
return req, nil
}()
if err != nil {
return nil, err
}
resp, err = c.httpClient.Do(req)
defer func() {
if resp != nil {
resp.Body.Close()
}
}()
// If the request was cancelled, return ErrRequestCancelled directly
select {
case <-cancelled:
return nil, ErrRequestCancelled
default:
}
numReqs++
// network error, change a machine!
if err != nil {
logger.Debug("network error: ", err.Error())
lastResp := http.Response{}
if checkErr := checkRetry(c.cluster, numReqs, lastResp, err); checkErr != nil {
return nil, checkErr
}
c.cluster.switchLeader(attempt % len(c.cluster.Machines))
continue
}
// if there is no error, it should receive response
logger.Debug("recv.response.from ", httpPath)
if validHttpStatusCode[resp.StatusCode] {
// try to read byte code and break the loop
respBody, err = ioutil.ReadAll(resp.Body)
if err == nil {
logger.Debug("recv.success ", httpPath)
break
}
// ReadAll error may be caused due to cancel request
select {
case <-cancelled:
return nil, ErrRequestCancelled
default:
}
if err == io.ErrUnexpectedEOF {
// underlying connection was closed prematurely, probably by timeout
// TODO: empty body or unexpectedEOF can cause http.Transport to get hosed;
// this allows the client to detect that and take evasive action. Need
// to revisit once code.google.com/p/go/issues/detail?id=8648 gets fixed.
respBody = []byte{}
break
}
}
// if resp is TemporaryRedirect, set the new leader and retry
if resp.StatusCode == http.StatusTemporaryRedirect {
u, err := resp.Location()
if err != nil {
logger.Warning(err)
} else {
// Update cluster leader based on redirect location
// because it should point to the leader address
c.cluster.updateLeaderFromURL(u)
logger.Debug("recv.response.relocate ", u.String())
}
resp.Body.Close()
continue
}
if checkErr := checkRetry(c.cluster, numReqs, *resp,
errors.New("Unexpected HTTP status code")); checkErr != nil {
return nil, checkErr
}
resp.Body.Close()
}
r := &RawResponse{
StatusCode: resp.StatusCode,
Body: respBody,
Header: resp.Header,
}
return r, nil
}
// DefaultCheckRetry defines the retrying behaviour for bad HTTP requests
// If we have retried 2 * machine number, stop retrying.
// If status code is InternalServerError, sleep for 200ms.
func DefaultCheckRetry(cluster *Cluster, numReqs int, lastResp http.Response,
err error) error {
if numReqs >= 2*len(cluster.Machines) {
return newError(ErrCodeEtcdNotReachable,
"Tried to connect to each peer twice and failed", 0)
}
code := lastResp.StatusCode
if code == http.StatusInternalServerError {
time.Sleep(time.Millisecond * 200)
}
logger.Warning("bad response status code", code)
return nil
}
func (c *Client) getHttpPath(random bool, s ...string) string {
var machine string
if random {
machine = c.cluster.Machines[rand.Intn(len(c.cluster.Machines))]
} else {
machine = c.cluster.Leader
}
fullPath := machine + "/" + version
for _, seg := range s {
fullPath = fullPath + "/" + seg
}
return fullPath
}
// buildValues builds a url.Values map according to the given value and ttl
func buildValues(value string, ttl uint64) url.Values {
v := url.Values{}
if value != "" {
v.Set("value", value)
}
if ttl > 0 {
v.Set("ttl", fmt.Sprintf("%v", ttl))
}
return v
}
// convert key string to http path exclude version
// for example: key[foo] -> path[keys/foo]
// key[/] -> path[keys/]
func keyToPath(key string) string {
p := path.Join("keys", key)
// corner case: if key is "/" or "//" ect
// path join will clear the tailing "/"
// we need to add it back
if p == "keys" {
p = "keys/"
}
return p
}

View File

@ -31,7 +31,8 @@ var (
}
)
func (rr *RawResponse) toResponse() (*Response, error) {
// Unmarshal parses RawResponse and stores the result in Response
func (rr *RawResponse) Unmarshal() (*Response, error) {
if rr.StatusCode != http.StatusOK && rr.StatusCode != http.StatusCreated {
return nil, handleError(rr.Body)
}
@ -72,7 +73,7 @@ type Node struct {
CreatedIndex uint64 `json:"createdIndex,omitempty"`
}
type Nodes []Node
type Nodes []*Node
// interfaces for sorting
func (ns Nodes) Len() int {

View File

@ -10,10 +10,10 @@ func (c *Client) Set(key string, value string, ttl uint64) (*Response, error) {
return nil, err
}
return raw.toResponse()
return raw.Unmarshal()
}
// Set sets the given key to a directory.
// SetDir sets the given key to a directory.
// It will create a new directory or replace the old key value pair by a directory.
// It will not replace a existing directory.
func (c *Client) SetDir(key string, ttl uint64) (*Response, error) {
@ -23,7 +23,7 @@ func (c *Client) SetDir(key string, ttl uint64) (*Response, error) {
return nil, err
}
return raw.toResponse()
return raw.Unmarshal()
}
// CreateDir creates a directory. It succeeds only if
@ -35,7 +35,7 @@ func (c *Client) CreateDir(key string, ttl uint64) (*Response, error) {
return nil, err
}
return raw.toResponse()
return raw.Unmarshal()
}
// UpdateDir updates the given directory. It succeeds only if the
@ -47,7 +47,7 @@ func (c *Client) UpdateDir(key string, ttl uint64) (*Response, error) {
return nil, err
}
return raw.toResponse()
return raw.Unmarshal()
}
// Create creates a file with the given value under the given key. It succeeds
@ -59,7 +59,19 @@ func (c *Client) Create(key string, value string, ttl uint64) (*Response, error)
return nil, err
}
return raw.toResponse()
return raw.Unmarshal()
}
// CreateInOrder creates a file with a key that's guaranteed to be higher than other
// keys in the given directory. It is useful for creating queues.
func (c *Client) CreateInOrder(dir string, value string, ttl uint64) (*Response, error) {
raw, err := c.RawCreateInOrder(dir, value, ttl)
if err != nil {
return nil, err
}
return raw.Unmarshal()
}
// Update updates the given key to the given value. It succeeds only if the
@ -71,11 +83,11 @@ func (c *Client) Update(key string, value string, ttl uint64) (*Response, error)
return nil, err
}
return raw.toResponse()
return raw.Unmarshal()
}
func (c *Client) RawUpdateDir(key string, ttl uint64) (*RawResponse, error) {
ops := options{
ops := Options{
"prevExist": true,
"dir": true,
}
@ -84,7 +96,7 @@ func (c *Client) RawUpdateDir(key string, ttl uint64) (*RawResponse, error) {
}
func (c *Client) RawCreateDir(key string, ttl uint64) (*RawResponse, error) {
ops := options{
ops := Options{
"prevExist": false,
"dir": true,
}
@ -97,7 +109,7 @@ func (c *Client) RawSet(key string, value string, ttl uint64) (*RawResponse, err
}
func (c *Client) RawSetDir(key string, ttl uint64) (*RawResponse, error) {
ops := options{
ops := Options{
"dir": true,
}
@ -105,7 +117,7 @@ func (c *Client) RawSetDir(key string, ttl uint64) (*RawResponse, error) {
}
func (c *Client) RawUpdate(key string, value string, ttl uint64) (*RawResponse, error) {
ops := options{
ops := Options{
"prevExist": true,
}
@ -113,9 +125,13 @@ func (c *Client) RawUpdate(key string, value string, ttl uint64) (*RawResponse,
}
func (c *Client) RawCreate(key string, value string, ttl uint64) (*RawResponse, error) {
ops := options{
ops := Options{
"prevExist": false,
}
return c.put(key, value, ttl, ops)
}
func (c *Client) RawCreateInOrder(dir string, value string, ttl uint64) (*RawResponse, error) {
return c.post(dir, value, ttl)
}

View File

@ -98,6 +98,43 @@ func TestCreate(t *testing.T) {
}
}
func TestCreateInOrder(t *testing.T) {
c := NewClient(nil)
dir := "/queue"
defer func() {
c.DeleteDir(dir)
}()
var firstKey, secondKey string
resp, err := c.CreateInOrder(dir, "1", 5)
if err != nil {
t.Fatal(err)
}
if !(resp.Action == "create" && resp.Node.Value == "1" && resp.Node.TTL == 5) {
t.Fatalf("Create 1 failed: %#v", resp)
}
firstKey = resp.Node.Key
resp, err = c.CreateInOrder(dir, "2", 5)
if err != nil {
t.Fatal(err)
}
if !(resp.Action == "create" && resp.Node.Value == "2" && resp.Node.TTL == 5) {
t.Fatalf("Create 2 failed: %#v", resp)
}
secondKey = resp.Node.Key
if firstKey >= secondKey {
t.Fatalf("Expected first key to be greater than second key, but %s is not greater than %s",
firstKey, secondKey)
}
}
func TestSetDir(t *testing.T) {
c := NewClient(nil)
defer func() {

View File

@ -31,8 +31,9 @@ func (c *Client) Watch(prefix string, waitIndex uint64, recursive bool,
return nil, err
}
return raw.toResponse()
return raw.Unmarshal()
}
defer close(receiver)
for {
raw, err := c.watchOnce(prefix, waitIndex, recursive, stop)
@ -41,7 +42,7 @@ func (c *Client) Watch(prefix string, waitIndex uint64, recursive bool,
return nil, err
}
resp, err := raw.toResponse()
resp, err := raw.Unmarshal()
if err != nil {
return nil, err
@ -50,8 +51,6 @@ func (c *Client) Watch(prefix string, waitIndex uint64, recursive bool,
waitIndex = resp.Node.ModifiedIndex + 1
receiver <- resp
}
return nil, nil
}
func (c *Client) RawWatch(prefix string, waitIndex uint64, recursive bool,
@ -69,7 +68,7 @@ func (c *Client) RawWatch(prefix string, waitIndex uint64, recursive bool,
return nil, err
}
resp, err := raw.toResponse()
resp, err := raw.Unmarshal()
if err != nil {
return nil, err
@ -78,44 +77,27 @@ func (c *Client) RawWatch(prefix string, waitIndex uint64, recursive bool,
waitIndex = resp.Node.ModifiedIndex + 1
receiver <- raw
}
return nil, nil
}
// helper func
// return when there is change under the given prefix
func (c *Client) watchOnce(key string, waitIndex uint64, recursive bool, stop chan bool) (*RawResponse, error) {
respChan := make(chan *RawResponse, 1)
errChan := make(chan error)
options := Options{
"wait": true,
}
if waitIndex > 0 {
options["waitIndex"] = waitIndex
}
if recursive {
options["recursive"] = true
}
go func() {
options := options{
"wait": true,
}
if waitIndex > 0 {
options["waitIndex"] = waitIndex
}
if recursive {
options["recursive"] = true
}
resp, err := c.getCancelable(key, options, stop)
resp, err := c.get(key, options)
if err != nil {
errChan <- err
return
}
respChan <- resp
}()
select {
case resp := <-respChan:
return resp, nil
case err := <-errChan:
return nil, err
case <-stop:
if err == ErrRequestCancelled {
return nil, ErrWatchStoppedByUser
}
return resp, err
}

Some files were not shown because too many files have changed in this diff Show More