diff --git a/tests/framework/integration/cluster.go b/tests/framework/integration/cluster.go index 19618aff9..b09d05470 100644 --- a/tests/framework/integration/cluster.go +++ b/tests/framework/integration/cluster.go @@ -398,12 +398,43 @@ func (c *Cluster) WaitMembersMatch(t testutil.TB, membs []*pb.Member) { } } -// WaitLeader returns index of the member in c.Members that is leader (or -1). -func (c *Cluster) WaitLeader(t testutil.TB) int { return c.WaitMembersForLeader(t, c.Members) } +// WaitLeader returns index of the member in c.Members that is leader +// or fails the test (if not established in 30min). +func (c *Cluster) WaitLeader(t testutil.TB) int { + return c.WaitMembersForLeader(t, c.Members) +} // WaitMembersForLeader waits until given members agree on the same leader, -// and returns its 'index' in the 'membs' list (or -1). +// and returns its 'index' in the 'membs' list func (c *Cluster) WaitMembersForLeader(t testutil.TB, membs []*Member) int { + t.Logf("WaitMembersForLeader") + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + l := 0 + for l = c.waitMembersForLeader(t, ctx, membs); l < 0; { + if ctx.Err() != nil { + t.Fatal("WaitLeader FAILED: %v", ctx.Err()) + } + } + t.Logf("WaitMembersForLeader succeeded. Cluster leader index: %v", l) + + // TODO: Consider second pass check as sometimes leadership is lost + // soon after election: + // + // We perform multiple attempts, as some-times just after successful WaitLLeader + // there is a race and leadership is quickly lost: + // - MsgAppResp message with higher term from 2acc3d3b521981 [term: 3] {"member": "m0"} + // - 9903a56eaf96afac became follower at term 3 {"member": "m0"} + // - 9903a56eaf96afac lost leader 9903a56eaf96afac at term 3 {"member": "m0"} + + + return l +} + +// WaitMembersForLeader waits until given members agree on the same leader, +// and returns its 'index' in the 'membs' list +func (c *Cluster) waitMembersForLeader(t testutil.TB, ctx context.Context, membs []*Member) int { + t.Logf("WaitMembersForLeader...") possibleLead := make(map[uint64]bool) var lead uint64 for _, m := range membs { @@ -415,7 +446,7 @@ func (c *Cluster) WaitMembersForLeader(t testutil.TB, membs []*Member) int { } // ensure leader is up via linearizable get for { - ctx, cancel := context.WithTimeout(context.Background(), 10*TickDuration+time.Second) + ctx, cancel := context.WithTimeout(ctx, 10*TickDuration+time.Second) _, err := cc.Get(ctx, "0") cancel() if err == nil || strings.Contains(err.Error(), "Key not found") { @@ -442,10 +473,12 @@ func (c *Cluster) WaitMembersForLeader(t testutil.TB, membs []*Member) int { for i, m := range membs { if uint64(m.Server.ID()) == lead { + t.Logf("WaitMembersForLeader found leader. Member: %v lead: %x", i, lead) return i } } + t.Logf("WaitMembersForLeader FAILED (-1)") return -1 } diff --git a/tests/integration/clientv3/maintenance_test.go b/tests/integration/clientv3/maintenance_test.go index ce1e629c0..75b134413 100644 --- a/tests/integration/clientv3/maintenance_test.go +++ b/tests/integration/clientv3/maintenance_test.go @@ -274,18 +274,22 @@ func TestMaintenanceStatus(t *testing.T) { clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 3}) defer clus.Terminate(t) + t.Logf("Waiting for leader...") clus.WaitLeader(t) + t.Logf("Leader established.") eps := make([]string, 3) for i := 0; i < 3; i++ { eps[i] = clus.Members[i].GRPCURL() } + t.Logf("Creating client...") cli, err := integration2.NewClient(t, clientv3.Config{Endpoints: eps, DialOptions: []grpc.DialOption{grpc.WithBlock()}}) if err != nil { t.Fatal(err) } defer cli.Close() + t.Logf("Creating client [DONE]") prevID, leaderFound := uint64(0), false for i := 0; i < 3; i++ { @@ -293,6 +297,7 @@ func TestMaintenanceStatus(t *testing.T) { if err != nil { t.Fatal(err) } + t.Logf("Response from %v: %v", i, resp) if prevID == 0 { prevID, leaderFound = resp.Header.MemberId, resp.Header.MemberId == resp.Leader continue diff --git a/tests/integration/clientv3/watch_test.go b/tests/integration/clientv3/watch_test.go index f464ec9bb..d52cb8a7c 100644 --- a/tests/integration/clientv3/watch_test.go +++ b/tests/integration/clientv3/watch_test.go @@ -415,15 +415,7 @@ func TestWatchResumeCompacted(t *testing.T) { } clus.Members[0].Stop(t) - ticker := time.After(time.Second * 10) - for clus.WaitLeader(t) <= 0 { - select { - case <-ticker: - t.Fatalf("failed to wait for new leader") - default: - time.Sleep(10 * time.Millisecond) - } - } + clus.WaitLeader(t) // put some data and compact away numPuts := 5