retry on gateway call (#94)

Author: shileiwill

cmd/secrets/common/gateway.go

@@ -5,18 +5,63 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"time"
+
+	"github.com/avast/retry-go/v4"
 )
 
 type GatewayClient interface {
 	Post(body []byte) (respBody []byte, status int, err error)
 }
 
 type HTTPClient struct {
-	URL    string
-	Client *http.Client
+	URL           string
+	Client        *http.Client
+	RetryAttempts uint
+	RetryDelay    time.Duration
 }
 
 func (g *HTTPClient) Post(body []byte) ([]byte, int, error) {
+	attempts := g.RetryAttempts
+	if attempts == 0 {
+		attempts = 5
+	}
+	delay := g.RetryDelay
+	if delay == 0 {
+		delay = 3 * time.Second
+	}
+
+	var respBody []byte
+	var status int
+
+	err := retry.Do(
+		func() error {
+			b, s, e := g.postOnce(body)
+			respBody, status = b, s
+			if e != nil {
+				return e // retry on any error
+			}
+			if s != http.StatusOK {
+				return fmt.Errorf("gateway returned non-200: %d", s) // retry on any non-200
+			}
+			return nil // success
+		},
+		retry.Attempts(attempts),
+		retry.Delay(delay),
+		retry.LastErrorOnly(true),
+		retry.OnRetry(func(n uint, err error) {
+			fmt.Printf("Waiting for block confirmation and retrying gateway POST (attempt %d/%d): %v", n+1, attempts, err)
+		}),
+	)
+
+	if err != nil {
+		// Return the last seen body/status to aid debugging.
+		return respBody, status, fmt.Errorf("gateway POST failed after %d attempts: %w", attempts, err)
+	}
+	return respBody, status, nil
+}
+
+func (g *HTTPClient) postOnce(body []byte) ([]byte, int, error) {
 	req, err := http.NewRequest("POST", g.URL, bytes.NewBuffer(body))
 	if err != nil {
 		return nil, 0, fmt.Errorf("create HTTP request: %w", err)

cmd/secrets/common/gateway_test.go

@@ -6,65 +6,206 @@ import (
 	"io"
 	"net/http"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 )
 
-type MockRoundTripper struct {
+// errReadCloser simulates a failure while reading the body.
+type errReadCloser struct{}
+
+func (e *errReadCloser) Read(p []byte) (int, error) { return 0, errors.New("read error") }
+func (e *errReadCloser) Close() error               { return nil }
+
+// RTResponse holds one RoundTrip outcome.
+type RTResponse struct {
 	Response *http.Response
 	Err      error
 }
 
-func (mrt *MockRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
-	return mrt.Response, mrt.Err
+// SeqRoundTripper returns a sequence of outcomes across calls.
+// If calls exceed length, it repeats the last element.
+type SeqRoundTripper struct {
+	Seq   []RTResponse
+	Calls int
 }
 
-type errReadCloser struct{}
+func (s *SeqRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	i := s.Calls
+	if i >= len(s.Seq) {
+		i = len(s.Seq) - 1
+	}
+	s.Calls++
+	entry := s.Seq[i]
+	return entry.Response, entry.Err
+}
 
-func (e *errReadCloser) Read(p []byte) (int, error) { return 0, errors.New("read error") }
-func (e *errReadCloser) Close() error               { return nil }
+// makeResp builds a tiny HTTP response with given status and body.
+func makeResp(code int, body string) *http.Response {
+	return &http.Response{
+		StatusCode: code,
+		Body:       io.NopCloser(bytes.NewBufferString(body)),
+		Header:     make(http.Header),
+	}
+}
 
 func TestPostToGateway(t *testing.T) {
 	h, _, _ := newMockHandler(t)
 
-	t.Run("success", func(t *testing.T) {
-		mockResponseBody := `{"jsonrpc":"2.0","id":"abc","result":{"ok":true}}`
-		mockResponse := &http.Response{
-			StatusCode: 200,
-			Body:       io.NopCloser(bytes.NewBufferString(mockResponseBody)),
-			Header:     make(http.Header),
-		}
+	t.Run("success (single attempt)", func(t *testing.T) {
+		body := `{"jsonrpc":"2.0","id":"abc","result":{"ok":true}}`
 
-		// Wire mock transport into Handler's HTTP client
-		mockedHttpClient := &http.Client{Transport: &MockRoundTripper{Response: mockResponse}}
-		h.Gw = &HTTPClient{URL: "https://unit-test.gw", Client: mockedHttpClient}
+		rt := &SeqRoundTripper{
+			Seq: []RTResponse{
+				{Response: makeResp(200, body)},
+			},
+		}
+		mockedHTTP := &http.Client{Transport: rt}
+		h.Gw = &HTTPClient{
+			URL:           "https://unit-test.gw",
+			Client:        mockedHTTP,
+			RetryAttempts: 3,
+			RetryDelay:    0, // fast tests
+		}
 
 		respBytes, status, err := h.Gw.Post([]byte(`{"x":1}`))
 		assert.NoError(t, err)
 		assert.Equal(t, 200, status)
-		assert.Equal(t, mockResponseBody, string(respBytes))
+		assert.Equal(t, body, string(respBytes))
+		assert.Equal(t, 1, rt.Calls)
 	})
 
-	t.Run("http error", func(t *testing.T) {
-		mockedHttpClient := &http.Client{Transport: &MockRoundTripper{Response: nil, Err: errors.New("network down")}}
-		h.Gw = &HTTPClient{URL: "https://unit-test.gw", Client: mockedHttpClient}
+	t.Run("http transport error -> retries then fail", func(t *testing.T) {
+		rt := &SeqRoundTripper{
+			Seq: []RTResponse{
+				{Err: errors.New("network down")},
+				{Err: errors.New("network still down")},
+			},
+		}
+		mockedHTTP := &http.Client{Transport: rt}
+		h.Gw = &HTTPClient{
+			URL:           "https://unit-test.gw",
+			Client:        mockedHTTP,
+			RetryAttempts: 2,
+			RetryDelay:    0,
+		}
 
 		_, _, err := h.Gw.Post([]byte(`{}`))
 		assert.Error(t, err)
-		assert.Contains(t, err.Error(), "network down")
+		assert.Contains(t, err.Error(), "network")
+		assert.Equal(t, 2, rt.Calls)
 	})
 
-	t.Run("read error", func(t *testing.T) {
-		mockResponse := &http.Response{
-			StatusCode: 200,
-			Body:       &errReadCloser{},
-			Header:     make(http.Header),
+	t.Run("read error -> retries then fail", func(t *testing.T) {
+		rt := &SeqRoundTripper{
+			Seq: []RTResponse{
+				{Response: &http.Response{StatusCode: 200, Body: &errReadCloser{}, Header: make(http.Header)}},
+				{Response: &http.Response{StatusCode: 200, Body: &errReadCloser{}, Header: make(http.Header)}},
+			},
+		}
+		mockedHTTP := &http.Client{Transport: rt}
+		h.Gw = &HTTPClient{
+			URL:           "https://unit-test.gw",
+			Client:        mockedHTTP,
+			RetryAttempts: 2,
+			RetryDelay:    0,
 		}
-		mockedHttpClient := &http.Client{Transport: &MockRoundTripper{Response: mockResponse}}
-		h.Gw = &HTTPClient{URL: "https://unit-test.gw", Client: mockedHttpClient}
 
 		_, _, err := h.Gw.Post([]byte(`{}`))
 		assert.Error(t, err)
 		assert.Contains(t, err.Error(), "read response body: read error")
+		assert.Equal(t, 2, rt.Calls)
+	})
+
+	t.Run("non-200 then 200 -> success after retry", func(t *testing.T) {
+		successBody := `{"ok":true}`
+		rt := &SeqRoundTripper{
+			Seq: []RTResponse{
+				{Response: makeResp(503, "temporary outage")},
+				{Response: makeResp(200, successBody)},
+			},
+		}
+		mockedHTTP := &http.Client{Transport: rt}
+		h.Gw = &HTTPClient{
+			URL:           "https://unit-test.gw",
+			Client:        mockedHTTP,
+			RetryAttempts: 3,
+			RetryDelay:    0,
+		}
+
+		respBytes, status, err := h.Gw.Post([]byte(`{}`))
+		assert.NoError(t, err)
+		assert.Equal(t, 200, status)
+		assert.Equal(t, successBody, string(respBytes))
+		assert.Equal(t, 2, rt.Calls)
+	})
+
+	t.Run("always non-200 -> fail after attempts", func(t *testing.T) {
+		rt := &SeqRoundTripper{
+			Seq: []RTResponse{
+				{Response: makeResp(500, "err1")},
+				{Response: makeResp(429, "err2")},
+				{Response: makeResp(400, "err3")}, // any non-200 should still retry/fail
+			},
+		}
+		mockedHTTP := &http.Client{Transport: rt}
+		h.Gw = &HTTPClient{
+			URL:           "https://unit-test.gw",
+			Client:        mockedHTTP,
+			RetryAttempts: 3,
+			RetryDelay:    0,
+		}
+
+		_, status, err := h.Gw.Post([]byte(`{}`))
+		assert.Error(t, err)
+		// status is from the last attempt
+		assert.Equal(t, 400, status)
+		assert.Contains(t, err.Error(), "gateway POST failed")
+		assert.Equal(t, 3, rt.Calls)
+	})
+
+	t.Run("error then success -> ok", func(t *testing.T) {
+		body := `{"ok":true}`
+		rt := &SeqRoundTripper{
+			Seq: []RTResponse{
+				{Err: errors.New("dial tcp: i/o timeout")},
+				{Response: makeResp(200, body)},
+			},
+		}
+		mockedHTTP := &http.Client{Transport: rt}
+		h.Gw = &HTTPClient{
+			URL:           "https://unit-test.gw",
+			Client:        mockedHTTP,
+			RetryAttempts: 5,
+			RetryDelay:    0,
+		}
+
+		respBytes, status, err := h.Gw.Post([]byte(`{}`))
+		assert.NoError(t, err)
+		assert.Equal(t, 200, status)
+		assert.Equal(t, body, string(respBytes))
+		assert.Equal(t, 2, rt.Calls)
+	})
+
+	// Optional: prove delays are honored if set
+	t.Run("honors small delay", func(t *testing.T) {
+		rt := &SeqRoundTripper{
+			Seq: []RTResponse{
+				{Response: makeResp(503, "nope")},
+				{Response: makeResp(200, `{"ok":true}`)},
+			},
+		}
+		mockedHTTP := &http.Client{Transport: rt}
+		h.Gw = &HTTPClient{
+			URL:           "https://unit-test.gw",
+			Client:        mockedHTTP,
+			RetryAttempts: 2,
+			RetryDelay:    5 * time.Millisecond,
+		}
+
+		_, status, err := h.Gw.Post([]byte(`{}`))
+		assert.NoError(t, err)
+		assert.Equal(t, 200, status)
+		assert.Equal(t, 2, rt.Calls)
 	})
 }