*: detect leaky goroutines, fix leaks

gexpect.Interact leaks. This adds ReadLine method to wait for the leaky
goroutine to accept an EOF.

Fixes https://github.com/coreos/etcd/issues/4258.

Reference: https://github.com/coreos/etcd/pull/4261#issuecomment-174198945.

integration/main_test.go

@@ -5,119 +5,16 @@
 package integration
 
 import (
-	"fmt"
-	"net/http"
 	"os"
-	"runtime"
-	"sort"
-	"strings"
 	"testing"
-	"time"
+
+	"github.com/coreos/etcd/pkg/testutil"
 )
 
-var atLeastGo15 bool = false
-
-func init() {
-	var major, minor int
-	var discard string
-	i, err := fmt.Sscanf(runtime.Version(), "go%d.%d%s", &major, &minor, &discard)
-	atLeastGo15 = (err == nil && i == 3 && (major > 1 || major == 1 && minor >= 5))
-}
-
-func interestingGoroutines() (gs []string) {
-	buf := make([]byte, 2<<20)
-	buf = buf[:runtime.Stack(buf, true)]
-	for _, g := range strings.Split(string(buf), "\n\n") {
-		sl := strings.SplitN(g, "\n", 2)
-		if len(sl) != 2 {
-			continue
-		}
-		stack := strings.TrimSpace(sl[1])
-		if stack == "" ||
-			strings.Contains(stack, "created by testing.RunTests") ||
-			strings.Contains(stack, "testing.Main(") ||
-			strings.Contains(stack, "runtime.goexit") ||
-			strings.Contains(stack, "github.com/coreos/etcd/integration.interestingGoroutines") ||
-			strings.Contains(stack, "github.com/coreos/etcd/pkg/logutil.(*MergeLogger).outputLoop") ||
-			strings.Contains(stack, "github.com/golang/glog.(*loggingT).flushDaemon") ||
-			strings.Contains(stack, "created by runtime.gc") ||
-			strings.Contains(stack, "runtime.MHeap_Scavenger") {
-			continue
-		}
-		gs = append(gs, stack)
-	}
-	sort.Strings(gs)
-	return
-}
-
-// Verify the other tests didn't leave any goroutines running.
 func TestMain(m *testing.M) {
 	v := m.Run()
-	if v == 0 && goroutineLeaked() {
+	if v == 0 && testutil.CheckLeakedGoroutine() {
 		os.Exit(1)
 	}
 	os.Exit(v)
 }
-
-func goroutineLeaked() bool {
-	if testing.Short() {
-		// not counting goroutines for leakage in -short mode
-		return false
-	}
-	gs := interestingGoroutines()
-
-	n := 0
-	stackCount := make(map[string]int)
-	for _, g := range gs {
-		stackCount[g]++
-		n++
-	}
-
-	if n == 0 {
-		return false
-	}
-	fmt.Fprintf(os.Stderr, "Too many goroutines running after integration test(s).\n")
-	for stack, count := range stackCount {
-		fmt.Fprintf(os.Stderr, "%d instances of:\n%s\n", count, stack)
-	}
-	return true
-}
-
-func afterTest(t *testing.T) {
-	http.DefaultTransport.(*http.Transport).CloseIdleConnections()
-	if testing.Short() {
-		return
-	}
-	var bad string
-	badSubstring := map[string]string{
-		").writeLoop(":                                 "a Transport",
-		"created by net/http/httptest.(*Server).Start": "an httptest.Server",
-		"timeoutHandler":                               "a TimeoutHandler",
-		"net.(*netFD).connect(":                        "a timing out dial",
-		").noteClientGone(":                            "a closenotifier sender",
-	}
-
-	// readLoop was buggy before go1.5:
-	// https://github.com/golang/go/issues/10457
-	if atLeastGo15 {
-		badSubstring[").readLoop("] = "a Transport"
-	}
-
-	var stacks string
-	for i := 0; i < 6; i++ {
-		bad = ""
-		stacks = strings.Join(interestingGoroutines(), "\n\n")
-		for substr, what := range badSubstring {
-			if strings.Contains(stacks, substr) {
-				bad = what
-			}
-		}
-		if bad == "" {
-			return
-		}
-		// Bad stuff found, but goroutines might just still be
-		// shutting down, so give it some time.
-		time.Sleep(50 * time.Millisecond)
-	}
-	t.Errorf("Test appears to have leaked %s:\n%s", bad, stacks)
-}