feat(agent): randomize retry-after interval when header is undefined

This commit enhances the client’s retry logic by replacing a fixed
fallback wait time with a randomized interval whenever the server does
not supply a Retry-After header. Instead of always pausing for 5
seconds, the code now computes a random duration between 5 seconds and
65 seconds. This change helps distribute retry attempts more evenly
across clients, mitigating the risk of simultaneous reconnections (the
“thundering herd” problem) when a service becomes available again.

Author: henrybarreto

pkg/api/client/client.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"math"
+	"math/rand/v2"
 	"net"
 	"net/http"
 	"net/url"
@@ -61,8 +62,21 @@ func NewClient(address string, opts ...Opt) (Client, error) {
 
 	// DefaultMaxRetryWaitTime is the default value for wait time between retries.
 	const DefaultMaxRetryWaitTime time.Duration = 1 * time.Hour
-	// DefaultRetryAfterTime is the retry default time when the header [RetryAfterHeader] isn't defined on the response.
-	const DefaultRetryAfterTime time.Duration = 5 * time.Second
+
+	randomWaitTimeSecs := func() time.Duration {
+		const DefaultMinRetryAfterSecs int = 5
+		const DefaultMaxRetryAfterSecs int = 65
+
+		t := time.Duration(rand.IntN(DefaultMaxRetryAfterSecs-DefaultMinRetryAfterSecs)+DefaultMinRetryAfterSecs) * time.Second //nolint:gosec
+
+		log.WithFields(log.Fields{
+			"min":         DefaultMinRetryAfterSecs,
+			"max":         DefaultMaxRetryAfterSecs,
+			"retry_after": t,
+		}).Debug("retrying request after a random time period")
+
+		return t
+	}
 
 	client := new(client)
 	client.http = resty.New()
@@ -104,15 +118,15 @@ func NewClient(address string, opts ...Opt) (Client, error) {
 		case http.StatusTooManyRequests, http.StatusServiceUnavailable:
 			retryAfterHeader := r.Header().Get(RetryAfterHeader)
 			if retryAfterHeader == "" {
-				return DefaultRetryAfterTime, nil
+				return randomWaitTimeSecs(), nil
 			}
 
 			// NOTE: The `Retry-After` supports delay in seconds and and a date time, but currently we will support only
 			// one of them.
 			// https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Retry-After
 			retryAfterSeconds, err := strconv.Atoi(retryAfterHeader)
 			if err != nil {
-				return DefaultRetryAfterTime, err
+				return randomWaitTimeSecs(), err
 			}
 
 			log.WithFields(log.Fields{
@@ -123,7 +137,7 @@ func NewClient(address string, opts ...Opt) (Client, error) {
 
 			return time.Duration(retryAfterSeconds) * time.Second, nil
 		default:
-			return DefaultRetryAfterTime, nil
+			return randomWaitTimeSecs(), nil
 		}
 	})
 	client.http.SetRetryMaxWaitTime(DefaultMaxRetryWaitTime)