grpcproxy, etcdmain, integration: add close channel to kv proxy

ccache launches goroutines that need to be explicitly stopped.

Fixes #7158

integration/cluster_proxy.go

@@ -30,8 +30,9 @@ var (
 )
 
 type grpcClientProxy struct {
-	grpc   grpcAPI
-	wdonec <-chan struct{}
+	grpc    grpcAPI
+	wdonec  <-chan struct{}
+	kvdonec <-chan struct{}
 }
 
 func toGRPC(c *clientv3.Client) grpcAPI {
@@ -43,26 +44,30 @@ func toGRPC(c *clientv3.Client) grpcAPI {
 	}
 
 	wp, wpch := grpcproxy.NewWatchProxy(c)
+	kvp, kvpch := grpcproxy.NewKvProxy(c)
 	grpc := grpcAPI{
 		pb.NewClusterClient(c.ActiveConnection()),
-		grpcproxy.KvServerToKvClient(grpcproxy.NewKvProxy(c)),
+		grpcproxy.KvServerToKvClient(kvp),
 		pb.NewLeaseClient(c.ActiveConnection()),
 		grpcproxy.WatchServerToWatchClient(wp),
 		pb.NewMaintenanceClient(c.ActiveConnection()),
 		pb.NewAuthClient(c.ActiveConnection()),
 	}
-	proxies[c] = grpcClientProxy{grpc: grpc, wdonec: wpch}
+	proxies[c] = grpcClientProxy{grpc: grpc, wdonec: wpch, kvdonec: kvpch}
 	return grpc
 }
 
-type watchCloser struct {
+type proxyCloser struct {
 	clientv3.Watcher
-	wdonec <-chan struct{}
+	wdonec  <-chan struct{}
+	kvdonec <-chan struct{}
 }
 
-func (wc *watchCloser) Close() error {
-	err := wc.Watcher.Close()
-	<-wc.wdonec
+func (pc *proxyCloser) Close() error {
+	// client ctx is canceled before calling close, so kv will close out
+	<-pc.kvdonec
+	err := pc.Watcher.Close()
+	<-pc.wdonec
 	return err
 }
 
@@ -74,9 +79,10 @@ func newClientV3(cfg clientv3.Config) (*clientv3.Client, error) {
 	rpc := toGRPC(c)
 	c.KV = clientv3.NewKVFromKVClient(rpc.KV)
 	pmu.Lock()
-	c.Watcher = &watchCloser{
+	c.Watcher = &proxyCloser{
 		Watcher: clientv3.NewWatchFromWatchClient(rpc.Watch),
 		wdonec:  proxies[c].wdonec,
+		kvdonec: proxies[c].kvdonec,
 	}
 	pmu.Unlock()
 	return c, nil