From dc45a5010d02cb3bb9f24619fd59752b98c32d6e Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 26 Mar 2024 08:00:38 -0700 Subject: [PATCH] bring back minor DNS cache for k8s setups (#19341) k8s as it stands is flaky in DNS lookups, bring this change back such that we can cache DNS atleast for 30secs TTL. --- cmd/common-main.go | 21 +++++++++++---------- cmd/endpoint.go | 5 ----- cmd/grid.go | 6 +----- cmd/net.go | 7 +------ cmd/server-main.go | 23 +++++++++-------------- cmd/utils.go | 42 ++++++------------------------------------ 6 files changed, 28 insertions(+), 76 deletions(-) diff --git a/cmd/common-main.go b/cmd/common-main.go index cd493eae1..221ae64b9 100644 --- a/cmd/common-main.go +++ b/cmd/common-main.go @@ -66,9 +66,11 @@ import ( ) // serverDebugLog will enable debug printing -var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn - -var currentReleaseTime time.Time +var ( + serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn + currentReleaseTime time.Time + orchestrated = IsKubernetes() || IsDocker() +) func init() { if runtime.GOOS == "windows" { @@ -494,7 +496,11 @@ func runDNSCache(ctx *cli.Context) { dnsTTL := ctx.Duration("dns-cache-ttl") // Check if we have configured a custom DNS cache TTL. if dnsTTL <= 0 { - dnsTTL = 10 * time.Minute + if orchestrated { + dnsTTL = 30 * time.Second + } else { + dnsTTL = 10 * time.Minute + } } // Call to refresh will refresh names in cache. @@ -757,12 +763,7 @@ func serverHandleEnvVars() { for _, endpoint := range minioEndpoints { if net.ParseIP(endpoint) == nil { // Checking if the IP is a DNS entry. - lookupHost := globalDNSCache.LookupHost - if IsKubernetes() || IsDocker() { - lookupHost = net.DefaultResolver.LookupHost - } - - addrs, err := lookupHost(GlobalContext, endpoint) + addrs, err := globalDNSCache.LookupHost(GlobalContext, endpoint) if err != nil { logger.FatalIf(err, "Unable to initialize MinIO server with [%s] invalid entry found in MINIO_PUBLIC_IPS", endpoint) } diff --git a/cmd/endpoint.go b/cmd/endpoint.go index 7c69ba37d..a80605dd4 100644 --- a/cmd/endpoint.go +++ b/cmd/endpoint.go @@ -590,8 +590,6 @@ func hostResolveToLocalhost(endpoint Endpoint) bool { // UpdateIsLocal - resolves the host and discovers the local host. func (endpoints Endpoints) UpdateIsLocal() error { - orchestrated := IsDocker() || IsKubernetes() - var epsResolved int var foundLocal bool resolvedList := make([]bool, len(endpoints)) @@ -775,8 +773,6 @@ type PoolEndpointList []Endpoints // UpdateIsLocal - resolves all hosts and discovers which are local func (p PoolEndpointList) UpdateIsLocal() error { - orchestrated := IsDocker() || IsKubernetes() - var epsResolved int var epCount int @@ -1034,7 +1030,6 @@ func CreatePoolEndpoints(serverAddr string, poolsLayout ...poolDisksLayout) ([]E } } - orchestrated := IsKubernetes() || IsDocker() reverseProxy := (env.Get("_MINIO_REVERSE_PROXY", "") != "") && ((env.Get("MINIO_CI_CD", "") != "") || (env.Get("CI", "") != "")) // If not orchestrated // and not setup in reverse proxy diff --git a/cmd/grid.go b/cmd/grid.go index e347e994d..5e20797de 100644 --- a/cmd/grid.go +++ b/cmd/grid.go @@ -35,13 +35,9 @@ var globalGrid atomic.Pointer[grid.Manager] var globalGridStart = make(chan struct{}) func initGlobalGrid(ctx context.Context, eps EndpointServerPools) error { - lookupHost := globalDNSCache.LookupHost - if IsKubernetes() || IsDocker() { - lookupHost = nil - } hosts, local := eps.GridHosts() g, err := grid.NewManager(ctx, grid.ManagerOptions{ - Dialer: grid.ContextDialer(xhttp.DialContextWithLookupHost(lookupHost, xhttp.NewInternodeDialContext(rest.DefaultTimeout, globalTCPOptions))), + Dialer: grid.ContextDialer(xhttp.DialContextWithLookupHost(globalDNSCache.LookupHost, xhttp.NewInternodeDialContext(rest.DefaultTimeout, globalTCPOptions))), Local: local, Hosts: hosts, AddAuth: newCachedAuthToken(), diff --git a/cmd/net.go b/cmd/net.go index e468532d2..50b6b2280 100644 --- a/cmd/net.go +++ b/cmd/net.go @@ -98,12 +98,7 @@ func mustGetLocalIP6() (ipList set.StringSet) { // getHostIP returns IP address of given host. func getHostIP(host string) (ipList set.StringSet, err error) { - lookupHost := globalDNSCache.LookupHost - if IsKubernetes() || IsDocker() { - lookupHost = net.DefaultResolver.LookupHost - } - - addrs, err := lookupHost(GlobalContext, host) + addrs, err := globalDNSCache.LookupHost(GlobalContext, host) if err != nil { return ipList, err } diff --git a/cmd/server-main.go b/cmd/server-main.go index d4098e6af..42166b293 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -141,9 +141,14 @@ var ServerFlags = []cli.Flag{ }, cli.DurationFlag{ Name: "dns-cache-ttl", - Usage: "custom DNS cache TTL for baremetal setups", + Usage: "custom DNS cache TTL", Hidden: true, - Value: 10 * time.Minute, + Value: func() time.Duration { + if orchestrated { + return 30 * time.Second + } + return 10 * time.Minute + }(), EnvVar: "MINIO_DNS_CACHE_TTL", }, cli.IntFlag{ @@ -593,12 +598,7 @@ func setGlobalInternodeInterface(interfaceName string) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - lookupHost := globalDNSCache.LookupHost - if IsKubernetes() || IsDocker() { - lookupHost = net.DefaultResolver.LookupHost - } - - haddrs, err := lookupHost(ctx, host) + haddrs, err := globalDNSCache.LookupHost(ctx, host) if err == nil { ip = haddrs[0] } @@ -636,12 +636,7 @@ func getServerListenAddrs() []string { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - lookupHost := globalDNSCache.LookupHost - if IsKubernetes() || IsDocker() { - lookupHost = net.DefaultResolver.LookupHost - } - - haddrs, err := lookupHost(ctx, host) + haddrs, err := globalDNSCache.LookupHost(ctx, host) if err == nil { for _, addr := range haddrs { addrs.Add(net.JoinHostPort(addr, globalMinioPort)) diff --git a/cmd/utils.go b/cmd/utils.go index c50a27f97..248277009 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -572,13 +572,8 @@ func ToS3ETag(etag string) string { // GetDefaultConnSettings returns default HTTP connection settings. func GetDefaultConnSettings() xhttp.ConnSettings { - lookupHost := globalDNSCache.LookupHost - if IsKubernetes() || IsDocker() { - lookupHost = nil - } - return xhttp.ConnSettings{ - LookupHost: lookupHost, + LookupHost: globalDNSCache.LookupHost, DialTimeout: rest.DefaultTimeout, RootCAs: globalRootCAs, TCPOptions: globalTCPOptions, @@ -588,13 +583,8 @@ func GetDefaultConnSettings() xhttp.ConnSettings { // NewInternodeHTTPTransport returns a transport for internode MinIO // connections. func NewInternodeHTTPTransport(maxIdleConnsPerHost int) func() http.RoundTripper { - lookupHost := globalDNSCache.LookupHost - if IsKubernetes() || IsDocker() { - lookupHost = nil - } - return xhttp.ConnSettings{ - LookupHost: lookupHost, + LookupHost: globalDNSCache.LookupHost, DialTimeout: rest.DefaultTimeout, RootCAs: globalRootCAs, CipherSuites: fips.TLSCiphers(), @@ -607,13 +597,8 @@ func NewInternodeHTTPTransport(maxIdleConnsPerHost int) func() http.RoundTripper // NewCustomHTTPProxyTransport is used only for proxied requests, specifically // only supports HTTP/1.1 func NewCustomHTTPProxyTransport() func() *http.Transport { - lookupHost := globalDNSCache.LookupHost - if IsKubernetes() || IsDocker() { - lookupHost = nil - } - return xhttp.ConnSettings{ - LookupHost: lookupHost, + LookupHost: globalDNSCache.LookupHost, DialTimeout: rest.DefaultTimeout, RootCAs: globalRootCAs, CipherSuites: fips.TLSCiphers(), @@ -626,13 +611,8 @@ func NewCustomHTTPProxyTransport() func() *http.Transport { // NewHTTPTransportWithClientCerts returns a new http configuration // used while communicating with the cloud backends. func NewHTTPTransportWithClientCerts(clientCert, clientKey string) *http.Transport { - lookupHost := globalDNSCache.LookupHost - if IsKubernetes() || IsDocker() { - lookupHost = nil - } - s := xhttp.ConnSettings{ - LookupHost: lookupHost, + LookupHost: globalDNSCache.LookupHost, DialTimeout: defaultDialTimeout, RootCAs: globalRootCAs, TCPOptions: globalTCPOptions, @@ -663,14 +643,9 @@ const defaultDialTimeout = 5 * time.Second // NewHTTPTransportWithTimeout allows setting a timeout. func NewHTTPTransportWithTimeout(timeout time.Duration) *http.Transport { - lookupHost := globalDNSCache.LookupHost - if IsKubernetes() || IsDocker() { - lookupHost = nil - } - return xhttp.ConnSettings{ DialContext: newCustomDialContext(), - LookupHost: lookupHost, + LookupHost: globalDNSCache.LookupHost, DialTimeout: defaultDialTimeout, RootCAs: globalRootCAs, TCPOptions: globalTCPOptions, @@ -702,14 +677,9 @@ func newCustomDialContext() xhttp.DialContext { // NewRemoteTargetHTTPTransport returns a new http configuration // used while communicating with the remote replication targets. func NewRemoteTargetHTTPTransport(insecure bool) func() *http.Transport { - lookupHost := globalDNSCache.LookupHost - if IsKubernetes() || IsDocker() { - lookupHost = nil - } - return xhttp.ConnSettings{ DialContext: newCustomDialContext(), - LookupHost: lookupHost, + LookupHost: globalDNSCache.LookupHost, RootCAs: globalRootCAs, TCPOptions: globalTCPOptions, EnableHTTP2: false,