From 91b904d10fc9de52b76da9d87d8a708d97e9f492 Mon Sep 17 00:00:00 2001
From: Ben McClelland <ben.mcclelland@versity.com>
Date: Mon, 7 Jul 2025 21:29:17 -0700
Subject: [PATCH] fix: add retry for iam freeipa http requests

The IPA service connections have been seen to not always work
correctly on the first network connection attempt. Add retry
logic for errors that appear to be transient network issues.
---
 auth/iam_ipa.go | 76 ++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 69 insertions(+), 7 deletions(-)

diff --git a/auth/iam_ipa.go b/auth/iam_ipa.go
index 6b3735dc..6f23a67a 100644
--- a/auth/iam_ipa.go
+++ b/auth/iam_ipa.go
@@ -27,12 +27,15 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net"
 	"net/http"
 	"net/http/cookiejar"
 	"net/url"
 	"slices"
 	"strconv"
 	"strings"
+	"syscall"
+	"time"
 )
 
 const IpaVersion = "2.254"
@@ -221,6 +224,8 @@ func (ipa *IpaIAMService) Shutdown() error {
 
 // Implementation
 
+const requestRetries = 3
+
 func (ipa *IpaIAMService) login() error {
 	form := url.Values{}
 	form.Set("user", ipa.username)
@@ -237,17 +242,33 @@ func (ipa *IpaIAMService) login() error {
 	req.Header.Set("referer", fmt.Sprintf("%s/ipa", ipa.host))
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 
-	resp, err := ipa.client.Do(req)
-	if err != nil {
-		return err
+	var resp *http.Response
+	for i := range requestRetries {
+		resp, err = ipa.client.Do(req)
+		if err == nil {
+			break
+		}
+		// Check for transient network errors
+		if isRetryable(err) {
+			time.Sleep(time.Second * time.Duration(i+1))
+			continue
+		}
+		return fmt.Errorf("login POST to %s failed: %w", req.URL, err)
 	}
+	if err != nil {
+		return fmt.Errorf("login POST to %s failed after retries: %w",
+			req.URL, err)
+	}
+
+	defer resp.Body.Close()
 
 	if resp.StatusCode == 401 {
 		return errors.New("cannot login to FreeIPA: invalid credentials")
 	}
 
 	if resp.StatusCode != 200 {
-		return fmt.Errorf("cannot login to FreeIPA: status code %d", resp.StatusCode)
+		return fmt.Errorf("cannot login to FreeIPA: status code %d",
+			resp.StatusCode)
 	}
 
 	return nil
@@ -294,10 +315,27 @@ func (ipa *IpaIAMService) rpcInternal(req rpcRequest) (rpcResponse, error) {
 	httpReq.Header.Set("referer", fmt.Sprintf("%s/ipa", ipa.host))
 	httpReq.Header.Set("Content-Type", "application/json")
 
-	httpResp, err := ipa.client.Do(httpReq)
-	if err != nil {
-		return rpcResponse{}, err
+	var httpResp *http.Response
+	for i := range requestRetries {
+		httpResp, err = ipa.client.Do(httpReq)
+		if err == nil {
+			break
+		}
+		// Check for transient network errors
+		if isRetryable(err) {
+			time.Sleep(time.Second * time.Duration(i+1))
+			continue
+		}
+		return rpcResponse{}, fmt.Errorf("ipa request to %s failed: %w",
+			httpReq.URL, err)
 	}
+	if err != nil {
+		return rpcResponse{},
+			fmt.Errorf("ipa request to %s failed after retries: %w",
+				httpReq.URL, err)
+	}
+
+	defer httpResp.Body.Close()
 
 	bytes, err := io.ReadAll(httpResp.Body)
 	ipa.log(string(bytes))
@@ -333,6 +371,30 @@ func (ipa *IpaIAMService) rpcInternal(req rpcRequest) (rpcResponse, error) {
 	}, nil
 }
 
+func isRetryable(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	if errors.Is(err, io.EOF) {
+		return true
+	}
+
+	if err, ok := err.(net.Error); ok && err.Timeout() {
+		return true
+	}
+
+	if opErr, ok := err.(*net.OpError); ok {
+		if sysErr, ok := opErr.Err.(*syscall.Errno); ok {
+			if *sysErr == syscall.ECONNRESET {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
 func (ipa *IpaIAMService) newRequest(method string, args []string, dict map[string]any) (rpcRequest, error) {
 
 	id := ipa.id