privval: add grpc (#5725)

Co-authored-by: Anton Kaliaev <anton.kalyaev@gmail.com>
2026-01-06 13:26:23 +00:00 · 2021-01-06 10:49:30 -08:00
parent e986602649
commit 09cf0bcb01
25 changed files with 1233 additions and 45 deletions
--- a/CHANGELOG_PENDING.md
+++ b/CHANGELOG_PENDING.md
@@ -47,6 +47,7 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi

 - [crypto/ed25519] \#5632 Adopt zip215 `ed25519` verification. (@marbar3778)
 - [privval] \#5603 Add `--key` to `init`, `gen_validator`, `testnet` & `unsafe_reset_priv_validator` for use in generating `secp256k1` keys.
+- [privval] \#5725 add gRPC support to private validator. 
 - [abci/client] \#5673 `Async` requests return an error if queue is full (@melekes)
 - [mempool] \#5673 Cancel `CheckTx` requests if RPC client disconnects or times out (@melekes)
 - [abci] \#5706 Added `AbciVersion` to `RequestInfo` allowing applications to check ABCI version when connecting to Tendermint. (@marbar3778)
--- a/23
+++ b/23
@@ -63,7 +63,7 @@ include tools/Makefile
 include test/Makefile

 ###############################################################################
-###                                Build Tendermint                        ###
+###                                Build Tendermint                         ###
 ###############################################################################

 build: $(BUILDDIR)/
@@ -127,6 +127,27 @@ install_abci:
 	@go install -mod=readonly ./abci/cmd/...
 .PHONY: install_abci

+###############################################################################
+###                           	Privval Server                              ###
+###############################################################################
+
+build_privval_server:
+	@go build -mod=readonly -o $(BUILDDIR)/ -i ./cmd/priv_val_server/...
+.PHONY: build_privval_server
+
+generate_test_cert:
+	# generate self signing ceritificate authority
+	@certstrap init --common-name "root CA" --expires "20 years"
+	# generate server cerificate
+	@certstrap request-cert -cn server -ip 127.0.0.1
+	# self-sign server cerificate with rootCA
+	@certstrap sign server --CA "root CA" 
+	# generate client cerificate
+	@certstrap request-cert -cn client -ip 127.0.0.1
+	# self-sign client cerificate with rootCA
+	@certstrap sign client --CA "root CA" 
+.PHONY: generate_test_cert
+
 ###############################################################################
 ###                              Distribution                               ###
 ###############################################################################
--- a/cmd/priv_val_server/main.go
+++ b/cmd/priv_val_server/main.go
@@ -1,24 +1,50 @@
 package main

 import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"flag"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
 	"os"
 	"time"

-	"github.com/tendermint/tendermint/crypto/ed25519"
+	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+
 	"github.com/tendermint/tendermint/libs/log"
 	tmnet "github.com/tendermint/tendermint/libs/net"
 	tmos "github.com/tendermint/tendermint/libs/os"
-
 	"github.com/tendermint/tendermint/privval"
+	grpcprivval "github.com/tendermint/tendermint/privval/grpc"
+	privvalproto "github.com/tendermint/tendermint/proto/tendermint/privval"
+)
+
+var (
+	// Create a metrics registry.
+	reg = prometheus.NewRegistry()
+
+	// Create some standard server metrics.
+	grpcMetrics = grpc_prometheus.NewServerMetrics()
 )

 func main() {
 	var (
-		addr             = flag.String("addr", ":26659", "Address of client to connect to")
+		addr             = flag.String("addr", "127.0.0.1:26659", "Address to listen on (host:port)")
 		chainID          = flag.String("chain-id", "mychain", "chain id")
 		privValKeyPath   = flag.String("priv-key", "", "priv val key file path")
 		privValStatePath = flag.String("priv-state", "", "priv val state file path")
+		insecure         = flag.Bool("insecure", false, "allow server to run insecurely (no TLS)")
+		certFile         = flag.String("certfile", "", "absolute path to server certificate")
+		keyFile          = flag.String("keyfile", "", "absolute path to server key")
+		rootCA           = flag.String("rootcafile", "", "absolute path to root CA")
+		prometheusAddr   = flag.String("prometheus-addr", "", "address for prometheus endpoint (host:port)")

 		logger = log.NewTMLogger(
 			log.NewSyncWriter(os.Stdout),
@@ -32,39 +58,106 @@ func main() {
 		"chainID", *chainID,
 		"privKeyPath", *privValKeyPath,
 		"privStatePath", *privValStatePath,
+		"insecure", *insecure,
+		"certFile", *certFile,
+		"keyFile", *keyFile,
+		"rootCA", *rootCA,
 	)

 	pv := privval.LoadFilePV(*privValKeyPath, *privValStatePath)

-	var dialer privval.SocketDialer
+	opts := []grpc.ServerOption{}
+	if !*insecure {
+		certificate, err := tls.LoadX509KeyPair(*certFile, *keyFile)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed to load X509 key pair: %v", err)
+			os.Exit(1)
+		}
+
+		certPool := x509.NewCertPool()
+		bs, err := ioutil.ReadFile(*rootCA)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed to read client ca cert: %s", err)
+			os.Exit(1)
+		}
+
+		if ok := certPool.AppendCertsFromPEM(bs); !ok {
+			fmt.Fprintf(os.Stderr, "failed to append client certs")
+			os.Exit(1)
+		}
+
+		tlsConfig := &tls.Config{
+			ClientAuth:   tls.RequireAndVerifyClientCert,
+			Certificates: []tls.Certificate{certificate},
+			ClientCAs:    certPool,
+			MinVersion:   tls.VersionTLS13,
+		}
+
+		creds := grpc.Creds(credentials.NewTLS(tlsConfig))
+		opts = append(opts, creds)
+		logger.Info("SignerServer: Creating security credentials")
+	} else {
+		logger.Info("SignerServer: You are using an insecure gRPC connection!")
+	}
+
+	// add prometheus metrics for unary RPC calls
+	opts = append(opts, grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor))
+
+	ss := grpcprivval.NewSignerServer(*chainID, pv, logger)
+
 	protocol, address := tmnet.ProtocolAndAddress(*addr)
-	switch protocol {
-	case "unix":
-		dialer = privval.DialUnixFn(address)
-	case "tcp":
-		connTimeout := 3 * time.Second // TODO
-		dialer = privval.DialTCPFn(address, connTimeout, ed25519.GenPrivKey())
-	default:
-		logger.Error("Unknown protocol", "protocol", protocol)
+
+	lis, err := net.Listen(protocol, address)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "SignerServer: Failed to listen %v", err)
 		os.Exit(1)
 	}

-	sd := privval.NewSignerDialerEndpoint(logger, dialer)
-	ss := privval.NewSignerServer(sd, *chainID, pv)
+	s := grpc.NewServer(opts...)

-	err := ss.Start()
-	if err != nil {
-		panic(err)
+	privvalproto.RegisterPrivValidatorAPIServer(s, ss)
+
+	var httpSrv *http.Server
+	if *prometheusAddr != "" {
+		httpSrv = registerPrometheus(*prometheusAddr, s)
+	}
+
+	logger.Info("SignerServer: Starting grpc server")
+	if err := s.Serve(lis); err != nil {
+		fmt.Fprintf(os.Stderr, "Unable to listen on port %s: %v", *addr, err)
+		os.Exit(1)
 	}

 	// Stop upon receiving SIGTERM or CTRL-C.
 	tmos.TrapSignal(logger, func() {
-		err := ss.Stop()
-		if err != nil {
-			panic(err)
+		logger.Debug("SignerServer: calling Close")
+		if *prometheusAddr != "" {
+			ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+			defer cancel()
+			if err := httpSrv.Shutdown(ctx); err != nil {
+				fmt.Fprintf(os.Stderr, "Unable to stop http server: %v", err)
+				os.Exit(1)
+			}
 		}
+		s.GracefulStop()
 	})

 	// Run forever.
 	select {}
 }
+
+func registerPrometheus(addr string, s *grpc.Server) *http.Server {
+	// Initialize all metrics.
+	grpcMetrics.InitializeMetrics(s)
+	// create http server to serve prometheus
+	httpServer := &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: addr}
+
+	go func() {
+		if err := httpServer.ListenAndServe(); err != nil {
+			fmt.Fprintf(os.Stderr, "Unable to start a http server: %v", err)
+			os.Exit(1)
+		}
+	}()
+
+	return httpServer
+}
--- a/config/config.go
+++ b/config/config.go
@@ -204,6 +204,16 @@ type BaseConfig struct { //nolint: maligned
 	// connections from an external PrivValidator process
 	PrivValidatorListenAddr string `mapstructure:"priv-validator-laddr"`

+	// Client certificate generated while creating needed files for secure connection.
+	// If a remote validator address is provided but no certificate, the connection will be insecure
+	PrivValidatorClientCertificate string `mapstructure:"priv-validator-client-certificate-file"`
+
+	// Client key generated while creating certificates for secure connection
+	PrivValidatorClientKey string `mapstructure:"priv-validator-client-key-file"`
+
+	// Path Root Certificate Authority used to sign both client and server certificates
+	PrivValidatorRootCA string `mapstructure:"priv-validator-root-ca-file"`
+
 	// A JSON file containing the private key to use for p2p authenticated encryption
 	NodeKey string `mapstructure:"node-key-file"`

@@ -253,6 +263,21 @@ func (cfg BaseConfig) GenesisFile() string {
 	return rootify(cfg.Genesis, cfg.RootDir)
 }

+// PrivValidatorClientKeyFile returns the full path to the priv_validator_key.json file
+func (cfg BaseConfig) PrivValidatorClientKeyFile() string {
+	return rootify(cfg.PrivValidatorClientKey, cfg.RootDir)
+}
+
+// PrivValidatorClientCertificateFile returns the full path to the priv_validator_key.json file
+func (cfg BaseConfig) PrivValidatorClientCertificateFile() string {
+	return rootify(cfg.PrivValidatorClientCertificate, cfg.RootDir)
+}
+
+// PrivValidatorCertificateAuthorityFile returns the full path to the priv_validator_key.json file
+func (cfg BaseConfig) PrivValidatorRootCAFile() string {
+	return rootify(cfg.PrivValidatorRootCA, cfg.RootDir)
+}
+
 // PrivValidatorKeyFile returns the full path to the priv_validator_key.json file
 func (cfg BaseConfig) PrivValidatorKeyFile() string {
 	return rootify(cfg.PrivValidatorKey, cfg.RootDir)
@@ -273,6 +298,19 @@ func (cfg BaseConfig) DBDir() string {
 	return rootify(cfg.DBPath, cfg.RootDir)
 }

+func (cfg *BaseConfig) ArePrivValidatorClientSecurityOptionsPresent() bool {
+	switch {
+	case cfg.PrivValidatorRootCA == "":
+		return false
+	case cfg.PrivValidatorClientKey == "":
+		return false
+	case cfg.PrivValidatorClientCertificate == "":
+		return false
+	default:
+		return true
+	}
+}
+
 // ValidateBasic performs basic validation (checking param bounds, etc.) and
 // returns an error if any check fails.
 func (cfg BaseConfig) ValidateBasic() error {
--- a/config/toml.go
+++ b/config/toml.go
@@ -136,8 +136,19 @@ priv-validator-state-file = "{{ js .BaseConfig.PrivValidatorState }}"

 # TCP or UNIX socket address for Tendermint to listen on for
 # connections from an external PrivValidator process
+# when the listenAddr is prefixed with grpc instead of tcp it will use the gRPC Client
 priv-validator-laddr = "{{ .BaseConfig.PrivValidatorListenAddr }}"

+# Client certificate generated while creating needed files for secure connection.
+# If a remote validator address is provided but no certificate, the connection will be insecure
+priv-validator-client-certificate-file = "{{ js .BaseConfig.PrivValidatorClientCertificate }}"
+
+# Client key generated while creating certificates for secure connection
+priv-validator-client-key-file = "{{ js .BaseConfig.PrivValidatorClientKey }}"
+
+# Path Root Certificate Authority used to sign both client and server certificates
+priv-validator-certificate-authority = "{{ js .BaseConfig.PrivValidatorRootCA }}"
+
 # Path to the JSON file containing the private key to use for node authentication in the p2p protocol
 node-key-file = "{{ js .BaseConfig.NodeKey }}"

--- a/docs/nodes/remote_signer.md
+++ b/docs/nodes/remote_signer.md
@@ -0,0 +1,70 @@
+---
+order: 7
+---
+
+# Remote signer
+
+Tendermint provides a remote signer option for validators. A remote signer enables the operator to store the validator key on a different machine minimizing the attack surface if a server were to be compromised.
+
+The remote signer protocol implements a [client and server architecture](https://en.wikipedia.org/wiki/Client%E2%80%93server_model). When Tendermint requires the public key or signature for a proposal or vote it requests it from the remote signer.
+
+To run a secure validator and remote signer system it is recommended to use a VPC (virtual private cloud) or a private connection. 
+
+There are two different configurations that can be used: Raw or gRPC.
+
+## Raw
+
+While both options use tcp or unix sockets the raw option uses tcp or unix sockets without http. The raw protocol sets up Tendermint as the server and the remote signer as the client. This aids in not exposing the remote signer to public network. 
+
+> Warning: Raw will be deprecated in a future major release, we recommend implementing your key management server against the gRPC configuration.
+
+## gRPC
+
+[gRPC](https://grpc.io/) is an RPC framework built with [HTTP/2](https://en.wikipedia.org/wiki/HTTP/2), uses [Protocol Buffers](https://developers.google.com/protocol-buffers) to define services and has been standardized within the cloud infrastructure community. gRPC provides a language agnostic way to implement services. This aids developers in the writing key management servers in various different languages.
+
+GRPC utilizes [TLS](https://en.wikipedia.org/wiki/Transport_Layer_Security), another widely standardized protocol, to secure connections. There are two forms of TLS to secure a connection, one-way and two-way. One way is when the client identifies the server but the server allows anyone to connect to it. Two-way is when the client identifies the server and the server identifies the client, prohibiting connections from unknown parties.
+
+When using gRPC Tendermint is setup as the client. Tendermint will make calls to the remote signer. We recommend not exposing the remote signer to the public network with the use of virtual private cloud.
+
+Securing your remote signers connection is highly recommended, but we provide the option to run it with a insecure connection.
+
+### Generating Certificates
+
+To run a secure connection with gRPC we need to generate certificates and keys. We will walkthrough how to self sign certificates for two-way TLS.
+
+There are two ways to generate certificates, [openssl](https://www.openssl.org/) and [certstarp](https://github.com/square/certstrap). Both of these options can be used but we will be covering `certstrap` because it provides a simpler process then openssl.
+
+- Install `Certstrap`:
+
+```sh
+  go get github.com/square/certstrap@v1.2.0
+```
+
+- Create certificate authority for self signing.
+
+```sh
+ # generate self signing ceritificate authority
+ certstrap init --common-name "<name_CA>" --expires "20 years"
+```
+
+- Request a certificate for the server.
+  - For generalization purposes we set the ip to `127.0.0.1`, but for your node please use the servers IP.
+- Sign the servers certificate with your certificate authority
+
+```sh
+ # generate server cerificate
+ certstrap request-cert -cn server -ip 127.0.0.1
+ # self-sign server cerificate with rootCA
+ certstrap sign server --CA "<name_CA>" 127.0.0.1
+  ```
+
+- Request a certificate for the client.
+  - For generalization purposes we set the ip to `127.0.0.1`, but for your node please use the clients IP.
+- Sign the clients certificate with your certificate authority
+
+```sh
+# generate client cerificate
+ certstrap request-cert -cn client -ip 127.0.0.1
+# self-sign client cerificate with rootCA
+ certstrap sign client --CA "<name_CA>" 127.0.0.1
+```
--- a/go.mod
+++ b/go.mod
@@ -17,6 +17,8 @@ require (
 	github.com/golang/protobuf v1.4.3
 	github.com/google/orderedcode v0.0.1
 	github.com/gorilla/websocket v1.4.2
+	github.com/grpc-ecosystem/go-grpc-middleware v1.2.2
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/gtank/merlin v0.1.1
 	github.com/hdevalence/ed25519consensus v0.0.0-20201207055737-7fde80a9d5ff
 	github.com/libp2p/go-buffer-pool v0.0.2
--- a/go.sum
+++ b/go.sum
@@ -32,9 +32,11 @@ github.com/Workiva/go-datastructures v1.0.52/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
@@ -243,7 +245,9 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.2.1/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI=
+github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 h1:FlFbCRLd5Jr4iYXZufAvgWN6Ao0JrI5chLINnUXDDr0=
 github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
@@ -481,6 +485,7 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
@@ -812,6 +817,7 @@ google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEG
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
--- a/libs/net/net_test.go
+++ b/libs/net/net_test.go
@@ -18,6 +18,11 @@ func TestProtocolAndAddress(t *testing.T) {
 			"tcp",
 			"mydomain:80",
 		},
+		{
+			"grpc://mydomain:80",
+			"grpc",
+			"mydomain:80",
+		},
 		{
 			"mydomain:80",
 			"tcp",
--- a/node/node.go
+++ b/node/node.go
@@ -8,12 +8,13 @@ import (
 	"net"
 	"net/http"
 	_ "net/http/pprof" // nolint: gosec // securely exposed on separate, optional port
-	"strings"
 	"time"

+	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/cors"
+	"google.golang.org/grpc"

 	dbm "github.com/tendermint/tm-db"

@@ -26,6 +27,7 @@ import (
 	"github.com/tendermint/tendermint/evidence"
 	tmjson "github.com/tendermint/tendermint/libs/json"
 	"github.com/tendermint/tendermint/libs/log"
+	tmnet "github.com/tendermint/tendermint/libs/net"
 	tmpubsub "github.com/tendermint/tendermint/libs/pubsub"
 	"github.com/tendermint/tendermint/libs/service"
 	"github.com/tendermint/tendermint/light"
@@ -33,6 +35,7 @@ import (
 	"github.com/tendermint/tendermint/p2p"
 	"github.com/tendermint/tendermint/p2p/pex"
 	"github.com/tendermint/tendermint/privval"
+	tmgrpc "github.com/tendermint/tendermint/privval/grpc"
 	"github.com/tendermint/tendermint/proxy"
 	rpccore "github.com/tendermint/tendermint/rpc/core"
 	grpccore "github.com/tendermint/tendermint/rpc/grpc"
@@ -675,10 +678,19 @@ func NewNode(config *cfg.Config,
 	// If an address is provided, listen on the socket for a connection from an
 	// external signing process.
 	if config.PrivValidatorListenAddr != "" {
+		protocol, address := tmnet.ProtocolAndAddress(config.PrivValidatorListenAddr)
 		// FIXME: we should start services inside OnStart
-		privValidator, err = createAndStartPrivValidatorSocketClient(config.PrivValidatorListenAddr, genDoc.ChainID, logger)
-		if err != nil {
-			return nil, fmt.Errorf("error with private validator socket client: %w", err)
+		switch protocol {
+		case "grpc":
+			privValidator, err = createAndStartPrivValidatorGRPCClient(config, address, genDoc.ChainID, logger)
+			if err != nil {
+				return nil, fmt.Errorf("error with private validator grpc client: %w", err)
+			}
+		default:
+			privValidator, err = createAndStartPrivValidatorSocketClient(config.PrivValidatorListenAddr, genDoc.ChainID, logger)
+			if err != nil {
+				return nil, fmt.Errorf("error with private validator socket client: %w", err)
+			}
 		}
 	}

@@ -1395,6 +1407,7 @@ func createAndStartPrivValidatorSocketClient(
 	chainID string,
 	logger log.Logger,
 ) (types.PrivValidator, error) {
+
 	pve, err := privval.NewSignerListener(listenAddr, logger)
 	if err != nil {
 		return nil, fmt.Errorf("failed to start private validator: %w", err)
@@ -1420,23 +1433,44 @@ func createAndStartPrivValidatorSocketClient(
 	return pvscWithRetries, nil
 }

-// splitAndTrimEmpty slices s into all subslices separated by sep and returns a
-// slice of the string s with all leading and trailing Unicode code points
-// contained in cutset removed. If sep is empty, SplitAndTrim splits after each
-// UTF-8 sequence. First part is equivalent to strings.SplitN with a count of
-// -1.  also filter out empty strings, only return non-empty strings.
-func splitAndTrimEmpty(s, sep, cutset string) []string {
-	if s == "" {
-		return []string{}
+func createAndStartPrivValidatorGRPCClient(
+	config *cfg.Config,
+	address,
+	chainID string,
+	logger log.Logger,
+) (types.PrivValidator, error) {
+	var transportSecurity grpc.DialOption
+	if config.BaseConfig.ArePrivValidatorClientSecurityOptionsPresent() {
+		transportSecurity = tmgrpc.GenerateTLS(config.PrivValidatorClientCertificateFile(),
+			config.PrivValidatorClientKeyFile(), config.PrivValidatorRootCAFile(), logger)
+	} else {
+		transportSecurity = grpc.WithInsecure()
+		logger.Info("Using an insecure gRPC connection!")
+	}
+	dialOptions := tmgrpc.DefaultDialOptions()
+	if config.Instrumentation.Prometheus {
+		grpcMetrics := grpc_prometheus.DefaultClientMetrics
+		dialOptions = append(dialOptions, grpc.WithUnaryInterceptor(grpcMetrics.UnaryClientInterceptor()))
 	}

-	spl := strings.Split(s, sep)
-	nonEmptyStrings := make([]string, 0, len(spl))
-	for i := 0; i < len(spl); i++ {
-		element := strings.Trim(spl[i], cutset)
-		if element != "" {
-			nonEmptyStrings = append(nonEmptyStrings, element)
-		}
+	dialOptions = append(dialOptions, transportSecurity)
+
+	ctx := context.Background()
+
+	conn, err := grpc.DialContext(ctx, address, dialOptions...)
+	if err != nil {
+		logger.Error("unable to connect to server", "target", address, "err", err)
 	}
-	return nonEmptyStrings
+	pvsc, err := tmgrpc.NewSignerClient(conn, chainID, logger)
+	if err != nil {
+		return nil, fmt.Errorf("failed to start private validator: %w", err)
+	}
+
+	// try to get a pubkey from private validate first time
+	_, err = pvsc.GetPubKey()
+	if err != nil {
+		return nil, fmt.Errorf("can't get pubkey: %w", err)
+	}
+
+	return pvsc, nil
 }
--- a/node/utils.go
+++ b/node/utils.go
@@ -0,0 +1,26 @@
+package node
+
+import (
+	"strings"
+)
+
+// splitAndTrimEmpty slices s into all subslices separated by sep and returns a
+// slice of the string s with all leading and trailing Unicode code points
+// contained in cutset removed. If sep is empty, SplitAndTrim splits after each
+// UTF-8 sequence. First part is equivalent to strings.SplitN with a count of
+// -1.  also filter out empty strings, only return non-empty strings.
+func splitAndTrimEmpty(s, sep, cutset string) []string {
+	if s == "" {
+		return []string{}
+	}
+
+	spl := strings.Split(s, sep)
+	nonEmptyStrings := make([]string, 0, len(spl))
+	for i := 0; i < len(spl); i++ {
+		element := strings.Trim(spl[i], cutset)
+		if element != "" {
+			nonEmptyStrings = append(nonEmptyStrings, element)
+		}
+	}
+	return nonEmptyStrings
+}
--- a/privval/grpc/client.go
+++ b/privval/grpc/client.go
@@ -0,0 +1,108 @@
+package grpc
+
+import (
+	"context"
+	"time"
+
+	grpc "google.golang.org/grpc"
+	"google.golang.org/grpc/status"
+
+	"github.com/tendermint/tendermint/crypto"
+	cryptoenc "github.com/tendermint/tendermint/crypto/encoding"
+	"github.com/tendermint/tendermint/libs/log"
+	privvalproto "github.com/tendermint/tendermint/proto/tendermint/privval"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+// SignerClient implements PrivValidator.
+// Handles remote validator connections that provide signing services
+type SignerClient struct {
+	logger log.Logger
+
+	client  privvalproto.PrivValidatorAPIClient
+	conn    *grpc.ClientConn
+	chainID string
+}
+
+var _ types.PrivValidator = (*SignerClient)(nil)
+
+// NewSignerClient returns an instance of SignerClient.
+// it will start the endpoint (if not already started)
+func NewSignerClient(conn *grpc.ClientConn,
+	chainID string, log log.Logger) (*SignerClient, error) {
+
+	sc := &SignerClient{
+		logger:  log,
+		chainID: chainID,
+		client:  privvalproto.NewPrivValidatorAPIClient(conn), // Create the Private Validator Client
+	}
+
+	return sc, nil
+}
+
+// Close closes the underlying connection
+func (sc *SignerClient) Close() error {
+	sc.logger.Info("Stopping service")
+	if sc.conn != nil {
+		return sc.conn.Close()
+	}
+	return nil
+}
+
+//--------------------------------------------------------
+// Implement PrivValidator
+
+// GetPubKey retrieves a public key from a remote signer
+// returns an error if client is not able to provide the key
+func (sc *SignerClient) GetPubKey() (crypto.PubKey, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) // Todo: should this be configurable?
+	defer cancel()
+	resp, err := sc.client.GetPubKey(ctx, &privvalproto.PubKeyRequest{ChainId: sc.chainID})
+	if err != nil {
+		errStatus, _ := status.FromError(err)
+		sc.logger.Error("SignerClient::GetPubKey", "err", errStatus.Message())
+		return nil, errStatus.Err()
+	}
+
+	pk, err := cryptoenc.PubKeyFromProto(resp.PubKey)
+	if err != nil {
+		return nil, err
+	}
+
+	return pk, nil
+}
+
+// SignVote requests a remote signer to sign a vote
+func (sc *SignerClient) SignVote(chainID string, vote *tmproto.Vote) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	resp, err := sc.client.SignVote(ctx, &privvalproto.SignVoteRequest{ChainId: sc.chainID, Vote: vote})
+	if err != nil {
+		errStatus, _ := status.FromError(err)
+		sc.logger.Error("Client SignVote", "err", errStatus.Message())
+		return errStatus.Err()
+	}
+
+	*vote = resp.Vote
+
+	return nil
+}
+
+// SignProposal requests a remote signer to sign a proposal
+func (sc *SignerClient) SignProposal(chainID string, proposal *tmproto.Proposal) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	resp, err := sc.client.SignProposal(
+		ctx, &privvalproto.SignProposalRequest{ChainId: chainID, Proposal: proposal})
+
+	if err != nil {
+		errStatus, _ := status.FromError(err)
+		sc.logger.Error("SignerClient::SignProposal", "err", errStatus.Message())
+		return errStatus.Err()
+	}
+
+	*proposal = resp.Proposal
+
+	return nil
+}
--- a/privval/grpc/client_test.go
+++ b/privval/grpc/client_test.go
@@ -0,0 +1,168 @@
+package grpc_test
+
+import (
+	"context"
+	"net"
+	"testing"
+	"time"
+
+	grpc "google.golang.org/grpc"
+	"google.golang.org/grpc/test/bufconn"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/crypto/tmhash"
+	"github.com/tendermint/tendermint/libs/log"
+	tmrand "github.com/tendermint/tendermint/libs/rand"
+	tmgrpc "github.com/tendermint/tendermint/privval/grpc"
+	privvalproto "github.com/tendermint/tendermint/proto/tendermint/privval"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+const chainID = "chain-id"
+
+func dialer(pv types.PrivValidator, logger log.Logger) (*grpc.Server, func(context.Context, string) (net.Conn, error)) {
+	listener := bufconn.Listen(1024 * 1024)
+
+	server := grpc.NewServer()
+
+	s := tmgrpc.NewSignerServer(chainID, pv, logger)
+
+	privvalproto.RegisterPrivValidatorAPIServer(server, s)
+
+	go func() {
+		if err := server.Serve(listener); err != nil {
+			panic(err)
+		}
+	}()
+
+	return server, func(context.Context, string) (net.Conn, error) {
+		return listener.Dial()
+	}
+}
+
+func TestSignerClient_GetPubKey(t *testing.T) {
+
+	ctx := context.Background()
+	mockPV := types.NewMockPV()
+	logger := log.TestingLogger()
+	srv, dialer := dialer(mockPV, logger)
+	defer srv.Stop()
+
+	conn, err := grpc.DialContext(ctx, "", grpc.WithInsecure(), grpc.WithContextDialer(dialer))
+	if err != nil {
+		panic(err)
+	}
+	defer conn.Close()
+
+	client, err := tmgrpc.NewSignerClient(conn, chainID, logger)
+	require.NoError(t, err)
+
+	pk, err := client.GetPubKey()
+	require.NoError(t, err)
+	assert.Equal(t, mockPV.PrivKey.PubKey(), pk)
+}
+
+func TestSignerClient_SignVote(t *testing.T) {
+
+	ctx := context.Background()
+	mockPV := types.NewMockPV()
+	logger := log.TestingLogger()
+	srv, dialer := dialer(mockPV, logger)
+	defer srv.Stop()
+
+	conn, err := grpc.DialContext(ctx, "", grpc.WithInsecure(), grpc.WithContextDialer(dialer))
+	if err != nil {
+		panic(err)
+	}
+	defer conn.Close()
+
+	client, err := tmgrpc.NewSignerClient(conn, chainID, logger)
+	require.NoError(t, err)
+
+	ts := time.Now()
+	hash := tmrand.Bytes(tmhash.Size)
+	valAddr := tmrand.Bytes(crypto.AddressSize)
+
+	want := &types.Vote{
+		Type:             tmproto.PrecommitType,
+		Height:           1,
+		Round:            2,
+		BlockID:          types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}},
+		Timestamp:        ts,
+		ValidatorAddress: valAddr,
+		ValidatorIndex:   1,
+	}
+
+	have := &types.Vote{
+		Type:             tmproto.PrecommitType,
+		Height:           1,
+		Round:            2,
+		BlockID:          types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}},
+		Timestamp:        ts,
+		ValidatorAddress: valAddr,
+		ValidatorIndex:   1,
+	}
+
+	pbHave := have.ToProto()
+
+	err = client.SignVote(chainID, pbHave)
+	require.NoError(t, err)
+
+	pbWant := want.ToProto()
+
+	require.NoError(t, mockPV.SignVote(chainID, pbWant))
+
+	assert.Equal(t, pbWant.Signature, pbHave.Signature)
+}
+
+func TestSignerClient_SignProposal(t *testing.T) {
+
+	ctx := context.Background()
+	mockPV := types.NewMockPV()
+	logger := log.TestingLogger()
+	srv, dialer := dialer(mockPV, logger)
+	defer srv.Stop()
+
+	conn, err := grpc.DialContext(ctx, "", grpc.WithInsecure(), grpc.WithContextDialer(dialer))
+	if err != nil {
+		panic(err)
+	}
+	defer conn.Close()
+
+	client, err := tmgrpc.NewSignerClient(conn, chainID, logger)
+	require.NoError(t, err)
+
+	ts := time.Now()
+	hash := tmrand.Bytes(tmhash.Size)
+
+	have := &types.Proposal{
+		Type:      tmproto.ProposalType,
+		Height:    1,
+		Round:     2,
+		POLRound:  2,
+		BlockID:   types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}},
+		Timestamp: ts,
+	}
+	want := &types.Proposal{
+		Type:      tmproto.ProposalType,
+		Height:    1,
+		Round:     2,
+		POLRound:  2,
+		BlockID:   types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}},
+		Timestamp: ts,
+	}
+
+	pbHave := have.ToProto()
+
+	err = client.SignProposal(chainID, pbHave)
+	require.NoError(t, err)
+
+	pbWant := want.ToProto()
+
+	require.NoError(t, mockPV.SignProposal(chainID, pbWant))
+
+	assert.Equal(t, pbWant.Signature, pbHave.Signature)
+}
--- a/privval/grpc/server.go
+++ b/privval/grpc/server.go
@@ -0,0 +1,87 @@
+package grpc
+
+import (
+	context "context"
+
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
+	"github.com/tendermint/tendermint/crypto"
+	cryptoenc "github.com/tendermint/tendermint/crypto/encoding"
+	"github.com/tendermint/tendermint/libs/log"
+	privvalproto "github.com/tendermint/tendermint/proto/tendermint/privval"
+	"github.com/tendermint/tendermint/types"
+)
+
+// SignerServer implements PrivValidatorAPIServer 9generated via protobuf services)
+// Handles remote validator connections that provide signing services
+type SignerServer struct {
+	logger  log.Logger
+	chainID string
+	privVal types.PrivValidator
+}
+
+func NewSignerServer(chainID string,
+	privVal types.PrivValidator, log log.Logger) *SignerServer {
+
+	return &SignerServer{
+		logger:  log,
+		chainID: chainID,
+		privVal: privVal,
+	}
+}
+
+var _ privvalproto.PrivValidatorAPIServer = (*SignerServer)(nil)
+
+// PubKey receives a request for the pubkey
+// returns the pubkey on success and error on failure
+func (ss *SignerServer) GetPubKey(ctx context.Context, req *privvalproto.PubKeyRequest) (
+	*privvalproto.PubKeyResponse, error) {
+	var pubKey crypto.PubKey
+
+	pubKey, err := ss.privVal.GetPubKey()
+	if err != nil {
+		return nil, status.Errorf(codes.NotFound, "error getting pubkey: %v", err)
+	}
+
+	pk, err := cryptoenc.PubKeyToProto(pubKey)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "error transitioning pubkey to proto: %v", err)
+	}
+
+	ss.logger.Info("SignerServer: GetPubKey Success")
+
+	return &privvalproto.PubKeyResponse{PubKey: pk}, nil
+}
+
+// SignVote receives a vote sign requests, attempts to sign it
+// returns SignedVoteResponse on success and error on failure
+func (ss *SignerServer) SignVote(ctx context.Context, req *privvalproto.SignVoteRequest) (
+	*privvalproto.SignedVoteResponse, error) {
+	vote := req.Vote
+
+	err := ss.privVal.SignVote(req.ChainId, vote)
+	if err != nil {
+		return nil, status.Errorf(codes.InvalidArgument, "error signing vote: %v", err)
+	}
+
+	ss.logger.Info("SignerServer: SignVote Success")
+
+	return &privvalproto.SignedVoteResponse{Vote: *vote}, nil
+}
+
+// SignProposal receives a proposal sign requests, attempts to sign it
+// returns SignedProposalResponse on success and error on failure
+func (ss *SignerServer) SignProposal(ctx context.Context, req *privvalproto.SignProposalRequest) (
+	*privvalproto.SignedProposalResponse, error) {
+	proposal := req.Proposal
+
+	err := ss.privVal.SignProposal(req.ChainId, proposal)
+	if err != nil {
+		return nil, status.Errorf(codes.InvalidArgument, "error signing proposal: %v", err)
+	}
+
+	ss.logger.Info("SignerServer: SignProposal Success")
+
+	return &privvalproto.SignedProposalResponse{Proposal: *proposal}, nil
+}
--- a/privval/grpc/server_test.go
+++ b/privval/grpc/server_test.go
@@ -0,0 +1,187 @@
+package grpc_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/crypto/tmhash"
+	"github.com/tendermint/tendermint/libs/log"
+	tmrand "github.com/tendermint/tendermint/libs/rand"
+	tmgrpc "github.com/tendermint/tendermint/privval/grpc"
+	privvalproto "github.com/tendermint/tendermint/proto/tendermint/privval"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+const ChainID = "123"
+
+func TestGetPubKey(t *testing.T) {
+
+	testCases := []struct {
+		name string
+		pv   types.PrivValidator
+		err  bool
+	}{
+		{name: "valid", pv: types.NewMockPV(), err: false},
+		{name: "error on pubkey", pv: types.NewErroringMockPV(), err: true},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			s := tmgrpc.NewSignerServer(ChainID, tc.pv, log.TestingLogger())
+
+			req := &privvalproto.PubKeyRequest{ChainId: ChainID}
+			resp, err := s.GetPubKey(context.Background(), req)
+			if tc.err {
+				require.Error(t, err)
+			} else {
+				pk, err := tc.pv.GetPubKey()
+				require.NoError(t, err)
+				assert.Equal(t, resp.PubKey.GetEd25519(), pk.Bytes())
+			}
+		})
+	}
+
+}
+
+func TestSignVote(t *testing.T) {
+
+	ts := time.Now()
+	hash := tmrand.Bytes(tmhash.Size)
+	valAddr := tmrand.Bytes(crypto.AddressSize)
+
+	testCases := []struct {
+		name       string
+		pv         types.PrivValidator
+		have, want *types.Vote
+		err        bool
+	}{
+		{name: "valid", pv: types.NewMockPV(), have: &types.Vote{
+			Type:             tmproto.PrecommitType,
+			Height:           1,
+			Round:            2,
+			BlockID:          types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}},
+			Timestamp:        ts,
+			ValidatorAddress: valAddr,
+			ValidatorIndex:   1,
+		}, want: &types.Vote{
+			Type:             tmproto.PrecommitType,
+			Height:           1,
+			Round:            2,
+			BlockID:          types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}},
+			Timestamp:        ts,
+			ValidatorAddress: valAddr,
+			ValidatorIndex:   1,
+		},
+			err: false},
+		{name: "invalid vote", pv: types.NewErroringMockPV(), have: &types.Vote{
+			Type:             tmproto.PrecommitType,
+			Height:           1,
+			Round:            2,
+			BlockID:          types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}},
+			Timestamp:        ts,
+			ValidatorAddress: valAddr,
+			ValidatorIndex:   1,
+			Signature:        []byte("signed"),
+		}, want: &types.Vote{
+			Type:             tmproto.PrecommitType,
+			Height:           1,
+			Round:            2,
+			BlockID:          types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}},
+			Timestamp:        ts,
+			ValidatorAddress: valAddr,
+			ValidatorIndex:   1,
+			Signature:        []byte("signed"),
+		},
+			err: true},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			s := tmgrpc.NewSignerServer(ChainID, tc.pv, log.TestingLogger())
+
+			req := &privvalproto.SignVoteRequest{ChainId: ChainID, Vote: tc.have.ToProto()}
+			resp, err := s.SignVote(context.Background(), req)
+			if tc.err {
+				require.Error(t, err)
+			} else {
+				pbVote := tc.want.ToProto()
+
+				require.NoError(t, tc.pv.SignVote(ChainID, pbVote))
+				assert.Equal(t, pbVote.Signature, resp.Vote.Signature)
+			}
+		})
+	}
+}
+
+func TestSignProposal(t *testing.T) {
+
+	ts := time.Now()
+	hash := tmrand.Bytes(tmhash.Size)
+
+	testCases := []struct {
+		name       string
+		pv         types.PrivValidator
+		have, want *types.Proposal
+		err        bool
+	}{
+		{name: "valid", pv: types.NewMockPV(), have: &types.Proposal{
+			Type:      tmproto.ProposalType,
+			Height:    1,
+			Round:     2,
+			POLRound:  2,
+			BlockID:   types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}},
+			Timestamp: ts,
+		}, want: &types.Proposal{
+			Type:      tmproto.ProposalType,
+			Height:    1,
+			Round:     2,
+			POLRound:  2,
+			BlockID:   types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}},
+			Timestamp: ts,
+		},
+			err: false},
+		{name: "invalid proposal", pv: types.NewErroringMockPV(), have: &types.Proposal{
+			Type:      tmproto.ProposalType,
+			Height:    1,
+			Round:     2,
+			POLRound:  2,
+			BlockID:   types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}},
+			Timestamp: ts,
+			Signature: []byte("signed"),
+		}, want: &types.Proposal{
+			Type:      tmproto.ProposalType,
+			Height:    1,
+			Round:     2,
+			POLRound:  2,
+			BlockID:   types.BlockID{Hash: hash, PartSetHeader: types.PartSetHeader{Hash: hash, Total: 2}},
+			Timestamp: ts,
+			Signature: []byte("signed"),
+		},
+			err: true},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			s := tmgrpc.NewSignerServer(ChainID, tc.pv, log.TestingLogger())
+
+			req := &privvalproto.SignProposalRequest{ChainId: ChainID, Proposal: tc.have.ToProto()}
+			resp, err := s.SignProposal(context.Background(), req)
+			if tc.err {
+				require.Error(t, err)
+			} else {
+				pbProposal := tc.want.ToProto()
+				require.NoError(t, tc.pv.SignProposal(ChainID, pbProposal))
+				assert.Equal(t, pbProposal.Signature, resp.Proposal.Signature)
+			}
+		})
+	}
+}
--- a/privval/grpc/util.go
+++ b/privval/grpc/util.go
@@ -0,0 +1,82 @@
+package grpc
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"io/ioutil"
+	"os"
+	"time"
+
+	grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/retry"
+	"github.com/tendermint/tendermint/libs/log"
+	grpc "google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/keepalive"
+)
+
+// DefaultDialOptions constructs a list of grpc dial options
+func DefaultDialOptions(
+	extraOpts ...grpc.DialOption,
+) []grpc.DialOption {
+	const (
+		retries            = 50 // 50 * 100ms = 5s total
+		timeout            = 1 * time.Second
+		maxCallRecvMsgSize = 1 << 20 // Default 5Mb
+	)
+
+	var kacp = keepalive.ClientParameters{
+		Time:    10 * time.Second, // send pings every 10 seconds if there is no activity
+		Timeout: 2 * time.Second,  // wait 2 seconds for ping ack before considering the connection dead
+	}
+
+	opts := []grpc_retry.CallOption{
+		grpc_retry.WithBackoff(grpc_retry.BackoffExponential(timeout)),
+	}
+
+	dialOpts := []grpc.DialOption{
+		grpc.WithKeepaliveParams(kacp),
+		grpc.WithDefaultCallOptions(
+			grpc.MaxCallRecvMsgSize(maxCallRecvMsgSize),
+			grpc_retry.WithMax(retries),
+		),
+		grpc.WithUnaryInterceptor(
+			grpc_retry.UnaryClientInterceptor(opts...),
+		),
+	}
+
+	dialOpts = append(dialOpts, extraOpts...)
+
+	return dialOpts
+}
+
+func GenerateTLS(certPath, keyPath, ca string, log log.Logger) grpc.DialOption {
+	certificate, err := tls.LoadX509KeyPair(
+		certPath,
+		keyPath,
+	)
+	if err != nil {
+		log.Error("error", err)
+		os.Exit(1)
+	}
+
+	certPool := x509.NewCertPool()
+	bs, err := ioutil.ReadFile(ca)
+	if err != nil {
+		log.Error("failed to read ca cert:", "error", err)
+		os.Exit(1)
+	}
+
+	ok := certPool.AppendCertsFromPEM(bs)
+	if !ok {
+		log.Error("failed to append certs")
+		os.Exit(1)
+	}
+
+	transportCreds := credentials.NewTLS(&tls.Config{
+		Certificates: []tls.Certificate{certificate},
+		RootCAs:      certPool,
+		MinVersion:   tls.VersionTLS13,
+	})
+
+	return grpc.WithTransportCredentials(transportCreds)
+}
--- a/proto/tendermint/privval/service.pb.go
+++ b/proto/tendermint/privval/service.pb.go
@@ -0,0 +1,199 @@
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: tendermint/privval/service.proto
+
+package privval
+
+import (
+	context "context"
+	fmt "fmt"
+	proto "github.com/gogo/protobuf/proto"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+	math "math"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
+
+func init() { proto.RegisterFile("tendermint/privval/service.proto", fileDescriptor_7afe74f9f46d3dc9) }
+
+var fileDescriptor_7afe74f9f46d3dc9 = []byte{
+	// 251 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x28, 0x49, 0xcd, 0x4b,
+	0x49, 0x2d, 0xca, 0xcd, 0xcc, 0x2b, 0xd1, 0x2f, 0x28, 0xca, 0x2c, 0x2b, 0x4b, 0xcc, 0xd1, 0x2f,
+	0x4e, 0x2d, 0x2a, 0xcb, 0x4c, 0x4e, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x12, 0x42, 0xa8,
+	0xd0, 0x83, 0xaa, 0x90, 0x92, 0xc3, 0xa2, 0xab, 0xa4, 0xb2, 0x20, 0xb5, 0x18, 0xa2, 0xc7, 0x68,
+	0x09, 0x13, 0x97, 0x40, 0x40, 0x51, 0x66, 0x59, 0x58, 0x62, 0x4e, 0x66, 0x4a, 0x62, 0x49, 0x7e,
+	0x91, 0x63, 0x80, 0xa7, 0x50, 0x10, 0x17, 0xa7, 0x7b, 0x6a, 0x49, 0x40, 0x69, 0x92, 0x77, 0x6a,
+	0xa5, 0x90, 0xa2, 0x1e, 0xa6, 0xb1, 0x7a, 0x10, 0xb9, 0xa0, 0xd4, 0xc2, 0xd2, 0xd4, 0xe2, 0x12,
+	0x29, 0x25, 0x7c, 0x4a, 0x8a, 0x0b, 0xf2, 0xf3, 0x8a, 0x53, 0x85, 0xc2, 0xb9, 0x38, 0x82, 0x33,
+	0xd3, 0xf3, 0xc2, 0xf2, 0x4b, 0x52, 0x85, 0x94, 0xb1, 0xa9, 0x87, 0xc9, 0xc2, 0x0c, 0x55, 0xc3,
+	0xa5, 0x28, 0x35, 0x05, 0xa2, 0x0c, 0x6a, 0x70, 0x32, 0x17, 0x0f, 0x48, 0x34, 0xa0, 0x28, 0xbf,
+	0x20, 0xbf, 0x38, 0x31, 0x47, 0x48, 0x1d, 0x97, 0x3e, 0x98, 0x0a, 0x98, 0x05, 0x5a, 0xb8, 0x2d,
+	0x40, 0x28, 0x85, 0x58, 0xe2, 0x14, 0x7c, 0xe2, 0x91, 0x1c, 0xe3, 0x85, 0x47, 0x72, 0x8c, 0x0f,
+	0x1e, 0xc9, 0x31, 0x4e, 0x78, 0x2c, 0xc7, 0x70, 0xe1, 0xb1, 0x1c, 0xc3, 0x8d, 0xc7, 0x72, 0x0c,
+	0x51, 0x96, 0xe9, 0x99, 0x25, 0x19, 0xa5, 0x49, 0x7a, 0xc9, 0xf9, 0xb9, 0xfa, 0x48, 0x61, 0x8d,
+	0x12, 0xec, 0xf9, 0x25, 0xf9, 0xfa, 0x98, 0xf1, 0x90, 0xc4, 0x06, 0x96, 0x31, 0x06, 0x04, 0x00,
+	0x00, 0xff, 0xff, 0x42, 0x60, 0x24, 0x48, 0xda, 0x01, 0x00, 0x00,
+}
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConn
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion4
+
+// PrivValidatorAPIClient is the client API for PrivValidatorAPI service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
+type PrivValidatorAPIClient interface {
+	GetPubKey(ctx context.Context, in *PubKeyRequest, opts ...grpc.CallOption) (*PubKeyResponse, error)
+	SignVote(ctx context.Context, in *SignVoteRequest, opts ...grpc.CallOption) (*SignedVoteResponse, error)
+	SignProposal(ctx context.Context, in *SignProposalRequest, opts ...grpc.CallOption) (*SignedProposalResponse, error)
+}
+
+type privValidatorAPIClient struct {
+	cc *grpc.ClientConn
+}
+
+func NewPrivValidatorAPIClient(cc *grpc.ClientConn) PrivValidatorAPIClient {
+	return &privValidatorAPIClient{cc}
+}
+
+func (c *privValidatorAPIClient) GetPubKey(ctx context.Context, in *PubKeyRequest, opts ...grpc.CallOption) (*PubKeyResponse, error) {
+	out := new(PubKeyResponse)
+	err := c.cc.Invoke(ctx, "/tendermint.privval.PrivValidatorAPI/GetPubKey", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *privValidatorAPIClient) SignVote(ctx context.Context, in *SignVoteRequest, opts ...grpc.CallOption) (*SignedVoteResponse, error) {
+	out := new(SignedVoteResponse)
+	err := c.cc.Invoke(ctx, "/tendermint.privval.PrivValidatorAPI/SignVote", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *privValidatorAPIClient) SignProposal(ctx context.Context, in *SignProposalRequest, opts ...grpc.CallOption) (*SignedProposalResponse, error) {
+	out := new(SignedProposalResponse)
+	err := c.cc.Invoke(ctx, "/tendermint.privval.PrivValidatorAPI/SignProposal", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// PrivValidatorAPIServer is the server API for PrivValidatorAPI service.
+type PrivValidatorAPIServer interface {
+	GetPubKey(context.Context, *PubKeyRequest) (*PubKeyResponse, error)
+	SignVote(context.Context, *SignVoteRequest) (*SignedVoteResponse, error)
+	SignProposal(context.Context, *SignProposalRequest) (*SignedProposalResponse, error)
+}
+
+// UnimplementedPrivValidatorAPIServer can be embedded to have forward compatible implementations.
+type UnimplementedPrivValidatorAPIServer struct {
+}
+
+func (*UnimplementedPrivValidatorAPIServer) GetPubKey(ctx context.Context, req *PubKeyRequest) (*PubKeyResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method GetPubKey not implemented")
+}
+func (*UnimplementedPrivValidatorAPIServer) SignVote(ctx context.Context, req *SignVoteRequest) (*SignedVoteResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method SignVote not implemented")
+}
+func (*UnimplementedPrivValidatorAPIServer) SignProposal(ctx context.Context, req *SignProposalRequest) (*SignedProposalResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method SignProposal not implemented")
+}
+
+func RegisterPrivValidatorAPIServer(s *grpc.Server, srv PrivValidatorAPIServer) {
+	s.RegisterService(&_PrivValidatorAPI_serviceDesc, srv)
+}
+
+func _PrivValidatorAPI_GetPubKey_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(PubKeyRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(PrivValidatorAPIServer).GetPubKey(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/tendermint.privval.PrivValidatorAPI/GetPubKey",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(PrivValidatorAPIServer).GetPubKey(ctx, req.(*PubKeyRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _PrivValidatorAPI_SignVote_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(SignVoteRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(PrivValidatorAPIServer).SignVote(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/tendermint.privval.PrivValidatorAPI/SignVote",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(PrivValidatorAPIServer).SignVote(ctx, req.(*SignVoteRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _PrivValidatorAPI_SignProposal_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(SignProposalRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(PrivValidatorAPIServer).SignProposal(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/tendermint.privval.PrivValidatorAPI/SignProposal",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(PrivValidatorAPIServer).SignProposal(ctx, req.(*SignProposalRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _PrivValidatorAPI_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "tendermint.privval.PrivValidatorAPI",
+	HandlerType: (*PrivValidatorAPIServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "GetPubKey",
+			Handler:    _PrivValidatorAPI_GetPubKey_Handler,
+		},
+		{
+			MethodName: "SignVote",
+			Handler:    _PrivValidatorAPI_SignVote_Handler,
+		},
+		{
+			MethodName: "SignProposal",
+			Handler:    _PrivValidatorAPI_SignProposal_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "tendermint/privval/service.proto",
+}
--- a/proto/tendermint/privval/service.proto
+++ b/proto/tendermint/privval/service.proto
@@ -0,0 +1,14 @@
+syntax = "proto3";
+package tendermint.privval;
+option  go_package = "github.com/tendermint/tendermint/proto/tendermint/privval";
+
+import "tendermint/privval/types.proto";
+
+//----------------------------------------
+// Service Definition
+
+service PrivValidatorAPI {
+  rpc GetPubKey(PubKeyRequest) returns (PubKeyResponse);
+  rpc SignVote(SignVoteRequest) returns (SignedVoteResponse);
+  rpc SignProposal(SignProposalRequest) returns (SignedProposalResponse);
+}
--- a/test/e2e/app/main.go
+++ b/test/e2e/app/main.go
@@ -3,12 +3,14 @@ package main
 import (
 	"errors"
 	"fmt"
+	"net"
 	"os"
 	"path/filepath"
 	"strconv"
 	"time"

 	"github.com/spf13/viper"
+	"google.golang.org/grpc"

 	"github.com/tendermint/tendermint/abci/server"
 	"github.com/tendermint/tendermint/config"
@@ -19,6 +21,8 @@ import (
 	"github.com/tendermint/tendermint/node"
 	"github.com/tendermint/tendermint/p2p"
 	"github.com/tendermint/tendermint/privval"
+	grpcprivval "github.com/tendermint/tendermint/privval/grpc"
+	privvalproto "github.com/tendermint/tendermint/proto/tendermint/privval"
 	"github.com/tendermint/tendermint/proxy"
 	mcs "github.com/tendermint/tendermint/test/maverick/consensus"
 	maverick "github.com/tendermint/tendermint/test/maverick/node"
@@ -182,6 +186,24 @@ func startSigner(cfg *Config) error {
 		dialFn = privval.DialTCPFn(address, 3*time.Second, ed25519.GenPrivKey())
 	case "unix":
 		dialFn = privval.DialUnixFn(address)
+	case "grpc":
+		lis, err := net.Listen("tcp", address)
+		if err != nil {
+			return err
+		}
+		ss := grpcprivval.NewSignerServer(cfg.ChainID, filePV, logger)
+
+		s := grpc.NewServer()
+
+		privvalproto.RegisterPrivValidatorAPIServer(s, ss)
+
+		go func() { // no need to clean up since we remove docker containers
+			if err := s.Serve(lis); err != nil {
+				panic(err)
+			}
+		}()
+
+		return nil
 	default:
 		return fmt.Errorf("invalid privval protocol %q", protocol)
 	}
@@ -193,6 +215,7 @@ func startSigner(cfg *Config) error {
 	if err != nil {
 		return err
 	}
+
 	logger.Info(fmt.Sprintf("Remote signer connecting to %v", cfg.PrivValServer))
 	return nil
 }
--- a/test/e2e/generator/generate.go
+++ b/test/e2e/generator/generate.go
@@ -30,7 +30,7 @@ var (
 	nodeDatabases = uniformChoice{"goleveldb", "cleveldb", "rocksdb", "boltdb", "badgerdb"}
 	// FIXME: grpc disabled due to https://github.com/tendermint/tendermint/issues/5439
 	nodeABCIProtocols    = uniformChoice{"unix", "tcp", "builtin"} // "grpc"
-	nodePrivvalProtocols = uniformChoice{"file", "unix", "tcp"}
+	nodePrivvalProtocols = uniformChoice{"file", "unix", "tcp", "grpc"}
 	// FIXME: v2 disabled due to flake
 	nodeFastSyncs         = uniformChoice{"", "v0"} // "v2"
 	nodeStateSyncs        = uniformChoice{false, true}
--- a/test/e2e/networks/ci.toml
+++ b/test/e2e/networks/ci.toml
@@ -51,7 +51,7 @@ seeds = ["seed01"]
 database = "badgerdb"
 # FIXME: should be grpc, disabled due to https://github.com/tendermint/tendermint/issues/5439
 #abci_protocol = "grpc"
-privval_protocol = "unix"
+privval_protocol = "grpc"
 persist_interval = 3
 retain_blocks = 3
 perturb = ["kill"]
--- a/test/e2e/pkg/manifest.go
+++ b/test/e2e/pkg/manifest.go
@@ -78,8 +78,9 @@ type ManifestNode struct {
 	ABCIProtocol string `toml:"abci_protocol"`

 	// PrivvalProtocol specifies the protocol used to sign consensus messages:
-	// "file", "unix", or "tcp". Defaults to "file". For unix and tcp, the ABCI
+	// "file", "unix", "tcp", or "grpc". Defaults to "file". For tcp and unix, the ABCI
 	// application will launch a remote signer client in a separate goroutine.
+	// For grpc the ABCI application will launch a remote signer server.
 	// Only nodes with mode=validator will actually make use of this.
 	PrivvalProtocol string `toml:"privval_protocol"`

--- a/test/e2e/pkg/testnet.go
+++ b/test/e2e/pkg/testnet.go
@@ -322,7 +322,7 @@ func (n Node) Validate(testnet Testnet) error {
 		return fmt.Errorf("invalid ABCI protocol setting %q", n.ABCIProtocol)
 	}
 	switch n.PrivvalProtocol {
-	case ProtocolFile, ProtocolUNIX, ProtocolTCP:
+	case ProtocolFile, ProtocolTCP, ProtocolGRPC, ProtocolUNIX:
 	default:
 		return fmt.Errorf("invalid privval protocol setting %q", n.PrivvalProtocol)
 	}
--- a/test/e2e/runner/setup.go
+++ b/test/e2e/runner/setup.go
@@ -32,6 +32,7 @@ const (
 	AppAddressUNIX = "unix:///var/run/app.sock"

 	PrivvalAddressTCP     = "tcp://0.0.0.0:27559"
+	PrivvalAddressGRPC    = "grpc://0.0.0.0:27559"
 	PrivvalAddressUNIX    = "unix:///var/run/privval.sock"
 	PrivvalKeyFile        = "config/priv_validator_key.json"
 	PrivvalStateFile      = "data/priv_validator_state.json"
@@ -265,6 +266,8 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) {
 			cfg.PrivValidatorListenAddr = PrivvalAddressUNIX
 		case e2e.ProtocolTCP:
 			cfg.PrivValidatorListenAddr = PrivvalAddressTCP
+		case e2e.ProtocolGRPC:
+			cfg.PrivValidatorListenAddr = PrivvalAddressGRPC
 		default:
 			return nil, fmt.Errorf("invalid privval protocol setting %q", node.PrivvalProtocol)
 		}
@@ -351,6 +354,10 @@ func MakeAppConfig(node *e2e.Node) ([]byte, error) {
 			cfg["privval_server"] = PrivvalAddressUNIX
 			cfg["privval_key"] = PrivvalKeyFile
 			cfg["privval_state"] = PrivvalStateFile
+		case e2e.ProtocolGRPC:
+			cfg["privval_server"] = PrivvalAddressGRPC
+			cfg["privval_key"] = PrivvalKeyFile
+			cfg["privval_state"] = PrivvalStateFile
 		default:
 			return nil, fmt.Errorf("unexpected privval protocol setting %q", node.PrivvalProtocol)
 		}
--- a/types/priv_validator.go
+++ b/types/priv_validator.go
@@ -128,6 +128,11 @@ type ErroringMockPV struct {

 var ErroringMockPVErr = errors.New("erroringMockPV always returns an error")

+// Implements PrivValidator.
+func (pv *ErroringMockPV) GetPubKey() (crypto.PubKey, error) {
+	return nil, ErroringMockPVErr
+}
+
 // Implements PrivValidator.
 func (pv *ErroringMockPV) SignVote(chainID string, vote *tmproto.Vote) error {
 	return ErroringMockPVErr