seaweedfs/weed/util/http/client/http_client.go

package client

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"

	"google.golang.org/grpc/credentials/tls/certprovider"

	"github.com/seaweedfs/seaweedfs/weed/security/certreload"
	util "github.com/seaweedfs/seaweedfs/weed/util"
	"github.com/spf13/viper"
)

var (
	loadSecurityConfigOnce sync.Once
)

// Intra-cluster peers (volume/filer/master) answer headers within milliseconds.
// A peer that is TCP-reachable but not answering -- a volume server still loading
// after a restart, or a stale keep-alive to a container that returned on a new IP
// -- otherwise blocks a chunk read or a replicated write forever. The response
// timeout bounds that wait so the read fails over to another replica and the
// write fails fast to retry; the idle timeout evicts sockets to a departed server.
const (
	responseHeaderTimeout = 30 * time.Second
	idleConnTimeout       = 90 * time.Second
)

type HTTPClient struct {
	Client            *http.Client
	Transport         *http.Transport
	expectHttpsScheme bool
	// certProvider, when non-nil, owns a background refresh goroutine for
	// the client mTLS cert/key pair. Close() must be called to stop it.
	certProvider certprovider.Provider
}

// Close stops any background cert refresh goroutine. Safe to call on a
// client that was constructed without mTLS. Existing pooled connections
// are also closed via CloseIdleConnections.
func (httpClient *HTTPClient) Close() {
	if httpClient == nil {
		return
	}
	if httpClient.certProvider != nil {
		httpClient.certProvider.Close()
		httpClient.certProvider = nil
	}
	if httpClient.Client != nil {
		httpClient.Client.CloseIdleConnections()
	}
}

func (httpClient *HTTPClient) Do(req *http.Request) (*http.Response, error) {
	req.URL.Scheme = httpClient.GetHttpScheme()
	return httpClient.Client.Do(req)
}

func (httpClient *HTTPClient) Get(url string) (resp *http.Response, err error) {
	url, err = httpClient.NormalizeHttpScheme(url)
	if err != nil {
		return nil, err
	}
	return httpClient.Client.Get(url)
}

func (httpClient *HTTPClient) Post(url, contentType string, body io.Reader) (resp *http.Response, err error) {
	url, err = httpClient.NormalizeHttpScheme(url)
	if err != nil {
		return nil, err
	}
	return httpClient.Client.Post(url, contentType, body)
}

func (httpClient *HTTPClient) PostForm(url string, data url.Values) (resp *http.Response, err error) {
	url, err = httpClient.NormalizeHttpScheme(url)
	if err != nil {
		return nil, err
	}
	return httpClient.Client.PostForm(url, data)
}

func (httpClient *HTTPClient) Head(url string) (resp *http.Response, err error) {
	url, err = httpClient.NormalizeHttpScheme(url)
	if err != nil {
		return nil, err
	}
	return httpClient.Client.Head(url)
}
func (httpClient *HTTPClient) CloseIdleConnections() {
	httpClient.Client.CloseIdleConnections()
}

func (httpClient *HTTPClient) GetClientTransport() *http.Transport {
	return httpClient.Transport
}

func (httpClient *HTTPClient) GetHttpScheme() string {
	if httpClient.expectHttpsScheme {
		return "https"
	}
	return "http"
}

func (httpClient *HTTPClient) NormalizeHttpScheme(rawURL string) (string, error) {
	expectedScheme := httpClient.GetHttpScheme()

	if !(strings.HasPrefix(rawURL, "http://") || strings.HasPrefix(rawURL, "https://")) {
		return expectedScheme + "://" + rawURL, nil
	}

	parsedURL, err := url.Parse(rawURL)
	if err != nil {
		return "", err
	}

	if expectedScheme != parsedURL.Scheme {
		parsedURL.Scheme = expectedScheme
	}
	return parsedURL.String(), nil
}

func NewHttpClient(clientName ClientName, opts ...HttpClientOpt) (*HTTPClient, error) {
	httpClient := HTTPClient{}
	httpClient.expectHttpsScheme = checkIsHttpsClientEnabled(clientName)
	var tlsConfig *tls.Config = nil

	if httpClient.expectHttpsScheme {
		certFileName, keyFileName, hasClientCert, err := clientCertPaths(clientName)
		if err != nil {
			return nil, err
		}

		clientCaCert, clientCaCertName, err := getClientCaCert(clientName)
		if err != nil {
			return nil, err
		}

		if hasClientCert || len(clientCaCert) != 0 {
			caCertPool, err := createHTTPClientCertPool(clientCaCert, clientCaCertName)
			if err != nil {
				return nil, err
			}

			tlsConfig = &tls.Config{
				RootCAs:            caCertPool,
				InsecureSkipVerify: false,
			}

			if hasClientCert {
				getClientCert, provider, err := certreload.NewClientGetCertificate(certFileName, keyFileName)
				if err != nil {
					return nil, fmt.Errorf("error loading client certificate and key: %s", err)
				}
				tlsConfig.GetClientCertificate = getClientCert
				httpClient.certProvider = provider
			}
		}

		if getBoolOptionFromSecurityConfiguration(clientName, "insecure_skip_verify") {
			if tlsConfig == nil {
				tlsConfig = &tls.Config{}
			}
			tlsConfig.InsecureSkipVerify = true
		}
	}

	httpClient.Transport = &http.Transport{
		MaxIdleConns:        1024,
		MaxIdleConnsPerHost: 1024,
		TLSClientConfig:     tlsConfig,
		// Bind outbound HTTP to the -ip.bind source address.
		DialContext:           util.OutboundDialContext,
		ResponseHeaderTimeout: responseHeaderTimeout,
		IdleConnTimeout:       idleConnTimeout,
	}
	httpClient.Client = &http.Client{
		Transport: httpClient.Transport,
	}

	for _, opt := range opts {
		opt(&httpClient)
	}
	return &httpClient, nil
}

func getStringOptionFromSecurityConfiguration(clientName ClientName, stringOptionName string) string {
	util.LoadSecurityConfiguration()
	return viper.GetString(fmt.Sprintf("https.%s.%s", clientName.LowerCaseString(), stringOptionName))
}

func getBoolOptionFromSecurityConfiguration(clientName ClientName, boolOptionName string) bool {
	util.LoadSecurityConfiguration()
	return viper.GetBool(fmt.Sprintf("https.%s.%s", clientName.LowerCaseString(), boolOptionName))
}

func checkIsHttpsClientEnabled(clientName ClientName) bool {
	return getBoolOptionFromSecurityConfiguration(clientName, "enabled")
}

func getFileContentFromSecurityConfiguration(clientName ClientName, fileType string) ([]byte, string, error) {
	if fileName := getStringOptionFromSecurityConfiguration(clientName, fileType); fileName != "" {
		fileContent, err := os.ReadFile(fileName)
		if err != nil {
			return nil, fileName, err
		}
		return fileContent, fileName, err
	}
	return nil, "", nil
}

// clientCertPaths reads the https.<clientName>.{cert,key} paths from the
// security config, validates they're either both set or both empty, and
// returns them along with a hasClientCert flag. Loading is deferred to
// certreload so the cert/key pair is picked up from disk on rotation.
func clientCertPaths(clientName ClientName) (certFile, keyFile string, hasClientCert bool, err error) {
	certFile = getStringOptionFromSecurityConfiguration(clientName, "cert")
	keyFile = getStringOptionFromSecurityConfiguration(clientName, "key")
	if certFile == "" && keyFile == "" {
		return "", "", false, nil
	}
	if certFile == "" || keyFile == "" {
		return "", "", false, fmt.Errorf("https.%s: both cert and key must be set (got cert=%q key=%q)", clientName.LowerCaseString(), certFile, keyFile)
	}
	return certFile, keyFile, true, nil
}

func getClientCaCert(clientName ClientName) ([]byte, string, error) {
	return getFileContentFromSecurityConfiguration(clientName, "ca")
}

// NewHttpClientWithTLS creates an HTTPClient with explicit TLS certificate
// parameters instead of reading from the global security configuration.
// This is used by filer.sync to create per-cluster HTTP clients when clusters
// use different certificates.
func NewHttpClientWithTLS(certFile, keyFile, caFile string, insecureSkipVerify bool, opts ...HttpClientOpt) (*HTTPClient, error) {
	httpClient := HTTPClient{}
	httpClient.expectHttpsScheme = true
	var tlsConfig *tls.Config

	if (certFile == "") != (keyFile == "") {
		return nil, fmt.Errorf("both cert and key are required for mTLS, got cert=%q key=%q", certFile, keyFile)
	}

	var getClientCert func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
	if certFile != "" && keyFile != "" {
		cb, provider, err := certreload.NewClientGetCertificate(certFile, keyFile)
		if err != nil {
			return nil, fmt.Errorf("error loading client certificate and key: %s", err)
		}
		getClientCert = cb
		httpClient.certProvider = provider
	}
	// closeProviderOnError ensures the cert reloader's background refresh
	// goroutine is shut down if any subsequent step fails before we hand
	// the client back to the caller.
	closeProviderOnError := func() {
		if httpClient.certProvider != nil {
			httpClient.certProvider.Close()
			httpClient.certProvider = nil
		}
	}

	var caCertPool *x509.CertPool
	if caFile != "" {
		caCert, err := os.ReadFile(caFile)
		if err != nil {
			closeProviderOnError()
			return nil, fmt.Errorf("error reading CA cert %s: %s", caFile, err)
		}
		caCertPool, err = createHTTPClientCertPool(caCert, caFile)
		if err != nil {
			closeProviderOnError()
			return nil, err
		}
	}

	if getClientCert != nil || caCertPool != nil || insecureSkipVerify {
		tlsConfig = &tls.Config{
			GetClientCertificate: getClientCert,
			RootCAs:              caCertPool,
			InsecureSkipVerify:   insecureSkipVerify,
		}
	}

	httpClient.Transport = &http.Transport{
		MaxIdleConns:        1024,
		MaxIdleConnsPerHost: 1024,
		TLSClientConfig:     tlsConfig,
		// Bind outbound HTTP to the -ip.bind source address.
		DialContext:           util.OutboundDialContext,
		ResponseHeaderTimeout: responseHeaderTimeout,
		IdleConnTimeout:       idleConnTimeout,
	}
	httpClient.Client = &http.Client{
		Transport: httpClient.Transport,
	}

	for _, opt := range opts {
		opt(&httpClient)
	}
	return &httpClient, nil
}

func createHTTPClientCertPool(certContent []byte, fileName string) (*x509.CertPool, error) {
	certPool := x509.NewCertPool()
	if len(certContent) == 0 {
		return certPool, nil
	}

	ok := certPool.AppendCertsFromPEM(certContent)
	if !ok {
		return nil, fmt.Errorf("error processing certificate in %s", fileName)
	}
	return certPool, nil
}