Decouple StartHTTP{,AndTLS}Server from Listen() (#2791)

* Decouple StartHTTP{,AndTLS}Server from Listen()

This should help solve cosmos/cosmos-sdk#2715

* Fix small mistake

* Update StartGRPCServer

* s/rpc/rpcserver/

* Start grpccore.StartGRPCServer in a goroutine

* Reinstate l.Close()

* Fix rpc/lib/test/main.go

* Update code comment

* update changelog and comments

* fix tm-monitor. more comments

rpc/grpc/client_server.go

@@ -1,12 +1,9 @@
 package core_grpc
 
 import (
-	"fmt"
 	"net"
-	"strings"
 	"time"
 
-	"golang.org/x/net/netutil"
 	"google.golang.org/grpc"
 
 	cmn "github.com/tendermint/tendermint/libs/common"
@@ -17,28 +14,12 @@ type Config struct {
 	MaxOpenConnections int
 }
 
-// StartGRPCServer starts a new gRPC BroadcastAPIServer, listening on
-// protoAddr, in a goroutine. Returns a listener and an error, if it fails to
-// parse an address.
-func StartGRPCServer(protoAddr string, config Config) (net.Listener, error) {
-	parts := strings.SplitN(protoAddr, "://", 2)
-	if len(parts) != 2 {
-		return nil, fmt.Errorf("Invalid listen address for grpc server (did you forget a tcp:// prefix?) : %s", protoAddr)
-	}
-	proto, addr := parts[0], parts[1]
-	ln, err := net.Listen(proto, addr)
-	if err != nil {
-		return nil, err
-	}
-	if config.MaxOpenConnections > 0 {
-		ln = netutil.LimitListener(ln, config.MaxOpenConnections)
-	}
-
+// StartGRPCServer starts a new gRPC BroadcastAPIServer using the given net.Listener.
+// NOTE: This function blocks - you may want to call it in a go-routine.
+func StartGRPCServer(ln net.Listener) error {
 	grpcServer := grpc.NewServer()
 	RegisterBroadcastAPIServer(grpcServer, &broadcastAPI{})
-	go grpcServer.Serve(ln) // nolint: errcheck
-
-	return ln, nil
+	return grpcServer.Serve(ln)
 }
 
 // StartGRPCClient dials the gRPC server using protoAddr and returns a new

rpc/lib/doc.go

@@ -70,12 +70,9 @@
 //   wm := rpcserver.NewWebsocketManager(Routes)
 //   mux.HandleFunc("/websocket", wm.WebsocketHandler)
 //   logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout))
-//   go func() {
-//		_, err := rpcserver.StartHTTPServer("0.0.0.0:8008", mux, logger)
-//   		if err != nil {
-//			panic(err)
-//		}
-//   }()
+//   listener, err := rpc.Listen("0.0.0.0:8080", rpcserver.Config{})
+//   if err != nil { panic(err) }
+//   go rpcserver.StartHTTPServer(listener, mux, logger)
 //
 // Note that unix sockets are supported as well (eg. `/path/to/socket` instead of `0.0.0.0:8008`)
 // Now see all available endpoints by sending a GET request to `0.0.0.0:8008`.

rpc/lib/rpc_test.go

@@ -121,12 +121,11 @@ func setup() {
 	wm := server.NewWebsocketManager(Routes, RoutesCdc, server.ReadWait(5*time.Second), server.PingPeriod(1*time.Second))
 	wm.SetLogger(tcpLogger)
 	mux.HandleFunc(websocketEndpoint, wm.WebsocketHandler)
-	go func() {
-		_, err := server.StartHTTPServer(tcpAddr, mux, tcpLogger, server.Config{})
-		if err != nil {
-			panic(err)
-		}
-	}()
+	listener1, err := server.Listen(tcpAddr, server.Config{})
+	if err != nil {
+		panic(err)
+	}
+	go server.StartHTTPServer(listener1, mux, tcpLogger)
 
 	unixLogger := logger.With("socket", "unix")
 	mux2 := http.NewServeMux()
@@ -134,12 +133,8 @@ func setup() {
 	wm = server.NewWebsocketManager(Routes, RoutesCdc)
 	wm.SetLogger(unixLogger)
 	mux2.HandleFunc(websocketEndpoint, wm.WebsocketHandler)
-	go func() {
-		_, err := server.StartHTTPServer(unixAddr, mux2, unixLogger, server.Config{})
-		if err != nil {
-			panic(err)
-		}
-	}()
+	listener2, err := server.Listen(unixAddr, server.Config{})
+	go server.StartHTTPServer(listener2, mux2, unixLogger)
 
 	// wait for servers to start
 	time.Sleep(time.Second * 2)

rpc/lib/server/http_server.go

@@ -29,90 +29,46 @@ const (
 	maxBodyBytes = int64(1000000) // 1MB
 )
 
-// StartHTTPServer starts an HTTP server on listenAddr with the given handler.
+// StartHTTPServer takes a listener and starts an HTTP server with the given handler.
 // It wraps handler with RecoverAndLogHandler.
-func StartHTTPServer(
-	listenAddr string,
-	handler http.Handler,
-	logger log.Logger,
-	config Config,
-) (listener net.Listener, err error) {
-	var proto, addr string
-	parts := strings.SplitN(listenAddr, "://", 2)
-	if len(parts) != 2 {
-		return nil, errors.Errorf(
-			"Invalid listening address %s (use fully formed addresses, including the tcp:// or unix:// prefix)",
-			listenAddr,
-		)
-	}
-	proto, addr = parts[0], parts[1]
+// NOTE: This function blocks - you may want to call it in a go-routine.
+func StartHTTPServer(listener net.Listener, handler http.Handler, logger log.Logger) error {
+	err := http.Serve(
+		listener,
+		RecoverAndLogHandler(maxBytesHandler{h: handler, n: maxBodyBytes}, logger),
+	)
+	logger.Info("RPC HTTP server stopped", "err", err)
 
-	logger.Info(fmt.Sprintf("Starting RPC HTTP server on %s", listenAddr))
-	listener, err = net.Listen(proto, addr)
-	if err != nil {
-		return nil, errors.Errorf("Failed to listen on %v: %v", listenAddr, err)
-	}
-	if config.MaxOpenConnections > 0 {
-		listener = netutil.LimitListener(listener, config.MaxOpenConnections)
-	}
-
-	go func() {
-		err := http.Serve(
-			listener,
-			RecoverAndLogHandler(maxBytesHandler{h: handler, n: maxBodyBytes}, logger),
-		)
-		logger.Info("RPC HTTP server stopped", "err", err)
-	}()
-	return listener, nil
+	return err
 }
 
-// StartHTTPAndTLSServer starts an HTTPS server on listenAddr with the given
-// handler.
+// StartHTTPAndTLSServer takes a listener and starts an HTTPS server with the given handler.
 // It wraps handler with RecoverAndLogHandler.
+// NOTE: This function blocks - you may want to call it in a go-routine.
 func StartHTTPAndTLSServer(
-	listenAddr string,
+	listener net.Listener,
 	handler http.Handler,
 	certFile, keyFile string,
 	logger log.Logger,
-	config Config,
-) (listener net.Listener, err error) {
-	var proto, addr string
-	parts := strings.SplitN(listenAddr, "://", 2)
-	if len(parts) != 2 {
-		return nil, errors.Errorf(
-			"Invalid listening address %s (use fully formed addresses, including the tcp:// or unix:// prefix)",
-			listenAddr,
-		)
-	}
-	proto, addr = parts[0], parts[1]
-
+) error {
 	logger.Info(
 		fmt.Sprintf(
 			"Starting RPC HTTPS server on %s (cert: %q, key: %q)",
-			listenAddr,
+			listener.Addr(),
 			certFile,
 			keyFile,
 		),
 	)
-	listener, err = net.Listen(proto, addr)
-	if err != nil {
-		return nil, errors.Errorf("Failed to listen on %v: %v", listenAddr, err)
-	}
-	if config.MaxOpenConnections > 0 {
-		listener = netutil.LimitListener(listener, config.MaxOpenConnections)
-	}
-
-	err = http.ServeTLS(
+	if err := http.ServeTLS(
 		listener,
 		RecoverAndLogHandler(maxBytesHandler{h: handler, n: maxBodyBytes}, logger),
 		certFile,
 		keyFile,
-	)
-	if err != nil {
+	); err != nil {
 		logger.Error("RPC HTTPS server stopped", "err", err)
-		return nil, err
+		return err
 	}
-	return listener, nil
+	return nil
 }
 
 func WriteRPCResponseHTTPError(
@@ -213,3 +169,35 @@ func (h maxBytesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	r.Body = http.MaxBytesReader(w, r.Body, h.n)
 	h.h.ServeHTTP(w, r)
 }
+
+// MustListen starts a new net.Listener on the given address.
+// It panics in case of error.
+func MustListen(addr string, config Config) net.Listener {
+	l, err := Listen(addr, config)
+	if err != nil {
+		panic(fmt.Errorf("Listen() failed: %v", err))
+	}
+	return l
+}
+
+// Listen starts a new net.Listener on the given address.
+// It returns an error if the address is invalid or the call to Listen() fails.
+func Listen(addr string, config Config) (listener net.Listener, err error) {
+	parts := strings.SplitN(addr, "://", 2)
+	if len(parts) != 2 {
+		return nil, errors.Errorf(
+			"Invalid listening address %s (use fully formed addresses, including the tcp:// or unix:// prefix)",
+			addr,
+		)
+	}
+	proto, addr := parts[0], parts[1]
+	listener, err = net.Listen(proto, addr)
+	if err != nil {
+		return nil, errors.Errorf("Failed to listen on %v: %v", addr, err)
+	}
+	if config.MaxOpenConnections > 0 {
+		listener = netutil.LimitListener(listener, config.MaxOpenConnections)
+	}
+
+	return listener, nil
+}

rpc/lib/server/http_server_test.go

@@ -30,11 +30,10 @@ func TestMaxOpenConnections(t *testing.T) {
 		time.Sleep(10 * time.Millisecond)
 		fmt.Fprint(w, "some body")
 	})
-	l, err := StartHTTPServer("tcp://127.0.0.1:0", mux, log.TestingLogger(), Config{MaxOpenConnections: max})
-	if err != nil {
-		t.Fatal(err)
-	}
+	l, err := Listen("tcp://127.0.0.1:0", Config{MaxOpenConnections: max})
+	require.NoError(t, err)
 	defer l.Close()
+	go StartHTTPServer(l, mux, log.TestingLogger())
 
 	// Make N GET calls to the server.
 	attempts := max * 2
@@ -67,11 +66,14 @@ func TestMaxOpenConnections(t *testing.T) {
 func TestStartHTTPAndTLSServer(t *testing.T) {
 	// set up fixtures
 	listenerAddr := "tcp://0.0.0.0:0"
+	listener, err := Listen(listenerAddr, Config{MaxOpenConnections: 1})
+	require.NoError(t, err)
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {})
 
 	// test failure
-	gotListener, err := StartHTTPAndTLSServer(listenerAddr, mux, "", "", log.TestingLogger(), Config{MaxOpenConnections: 1})
-	require.Nil(t, gotListener)
+	err = StartHTTPAndTLSServer(listener, mux, "", "", log.TestingLogger())
 	require.IsType(t, (*os.PathError)(nil), err)
+
+	// TODO: test that starting the server can actually work
 }

rpc/lib/test/main.go

@@ -28,11 +28,11 @@ func main() {
 	cdc := amino.NewCodec()
 	logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout))
 	rpcserver.RegisterRPCFuncs(mux, routes, cdc, logger)
-	_, err := rpcserver.StartHTTPServer("0.0.0.0:8008", mux, logger, rpcserver.Config{})
+	listener, err := rpcserver.Listen("0.0.0.0:8008", rpcserver.Config{})
 	if err != nil {
 		cmn.Exit(err.Error())
 	}
-
+	go rpcserver.StartHTTPServer(listener, mux, logger)
 	// Wait forever
 	cmn.TrapSignal(func() {
 	})