fix(dbng): bound database readiness polling

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## To Be Released
 
+* fix(dbng): bound database readiness polling
+
 ## 1.51.1
 
 * chore(go): update Go to 1.27.3

dbng/create.go

@@ -11,6 +11,12 @@ import (
 	"github.com/Scalingo/cli/utils"
 	"github.com/Scalingo/go-scalingo/v11"
 	"github.com/Scalingo/go-utils/errors/v3"
+	"github.com/Scalingo/go-utils/retry"
+)
+
+const (
+	databaseStatusPollInterval = 10 * time.Second
+	databaseStatusMaxWait      = 30 * time.Minute
 )
 
 func Create(ctx context.Context, params scalingo.DatabaseCreateParams, wait bool) error {
@@ -58,18 +64,44 @@ func Create(ctx context.Context, params scalingo.DatabaseCreateParams, wait bool
 	return nil
 }
 
-// waitForRunningDatabase is a blocking function waiting for the database to be running.
+// waitForRunningDatabase waits until the database is running, the context is
+// canceled, or the provisioning timeout is reached.
 func waitForRunningDatabase(ctx context.Context, client *scalingo.Client, appID string) error {
-	for range time.Tick(10 * time.Second) {
-		db, err := client.Preview().DatabaseShow(ctx, appID)
+	return waitForRunningDatabaseWithTimeout(ctx, client.Preview().DatabaseShow, appID, databaseStatusMaxWait)
+}
+
+func waitForRunningDatabaseWithTimeout(ctx context.Context, show databaseShowFunc, appID string, maxWait time.Duration) error {
+	pollCtx, cancel := context.WithTimeout(ctx, maxWait)
+	defer cancel()
+
+	retryer := retry.New(
+		retry.WithWaitDuration(databaseStatusPollInterval),
+		retry.WithMaxDuration(maxWait),
+		retry.WithoutMaxAttempts(),
+	)
+
+	return waitForRunningDatabaseWithRetry(pollCtx, show, retryer, appID)
+}
+
+type databaseShowFunc func(context.Context, string) (scalingo.DatabaseNG, error)
+
+func waitForRunningDatabaseWithRetry(ctx context.Context, show databaseShowFunc, retryer retry.Retry, appID string) error {
+	return retryer.Do(ctx, func(ctx context.Context) error {
+		db, err := show(ctx, appID)
 		if errors.Is(err, scalingo.ErrDatabaseNotFound) {
-			continue
-		} else if err != nil {
 			return errors.Wrapf(ctx, err, "get database %s status", appID)
 		}
-		if db.Database.Status == scalingo.DatabaseStatusRunning {
-			break
+		if err != nil {
+			return retry.NewRetryCancelError(errors.Wrapf(ctx, err, "get database %s status", appID))
 		}
-	}
-	return nil
+
+		switch db.Database.Status {
+		case scalingo.DatabaseStatusRunning:
+			return nil
+		case scalingo.DatabaseStatusStopped:
+			return retry.NewRetryCancelError(errors.Newf(ctx, "database %s is stopped", appID))
+		default:
+			return errors.Newf(ctx, "database %s status is %s", appID, db.Database.Status)
+		}
+	})
 }

dbng/create_test.go

@@ -0,0 +1,120 @@
+package dbng
+
+import (
+	"context"
+	stderrors "errors"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/Scalingo/go-scalingo/v11"
+	"github.com/Scalingo/go-utils/retry"
+)
+
+func TestWaitForRunningDatabaseWithRetry(t *testing.T) {
+	t.Run("retries until the database is running", func(t *testing.T) {
+		calls := 0
+		statuses := []scalingo.DatabaseStatus{
+			scalingo.DatabaseStatusCreating,
+			scalingo.DatabaseStatusMigrating,
+			scalingo.DatabaseStatusRunning,
+		}
+		show := func(_ context.Context, appID string) (scalingo.DatabaseNG, error) {
+			assert.Equal(t, "db-123", appID)
+			status := statuses[calls]
+			calls++
+			return databaseWithStatus(status), nil
+		}
+
+		err := waitForRunningDatabaseWithRetry(t.Context(), show, testRetryer(3), "db-123")
+
+		require.NoError(t, err)
+		assert.Equal(t, 3, calls)
+	})
+
+	t.Run("retries a database that is not visible yet", func(t *testing.T) {
+		calls := 0
+		show := func(_ context.Context, _ string) (scalingo.DatabaseNG, error) {
+			calls++
+			if calls == 1 {
+				return scalingo.DatabaseNG{}, scalingo.ErrDatabaseNotFound
+			}
+			return databaseWithStatus(scalingo.DatabaseStatusRunning), nil
+		}
+
+		err := waitForRunningDatabaseWithRetry(t.Context(), show, testRetryer(2), "db-123")
+
+		require.NoError(t, err)
+		assert.Equal(t, 2, calls)
+	})
+
+	t.Run("stops immediately when the database is stopped", func(t *testing.T) {
+		calls := 0
+		show := func(_ context.Context, _ string) (scalingo.DatabaseNG, error) {
+			calls++
+			return databaseWithStatus(scalingo.DatabaseStatusStopped), nil
+		}
+
+		err := waitForRunningDatabaseWithRetry(t.Context(), show, testRetryer(5), "db-123")
+
+		require.ErrorContains(t, err, "database db-123 is stopped")
+		assert.Equal(t, 1, calls)
+	})
+
+	t.Run("stops immediately on an API error", func(t *testing.T) {
+		apiErr := stderrors.New("API unavailable")
+		calls := 0
+		show := func(_ context.Context, _ string) (scalingo.DatabaseNG, error) {
+			calls++
+			return scalingo.DatabaseNG{}, apiErr
+		}
+
+		err := waitForRunningDatabaseWithRetry(t.Context(), show, testRetryer(5), "db-123")
+
+		require.ErrorIs(t, err, apiErr)
+		require.ErrorContains(t, err, "get database db-123 status")
+		assert.Equal(t, 1, calls)
+	})
+
+	t.Run("returns after the retry limit", func(t *testing.T) {
+		calls := 0
+		show := func(_ context.Context, _ string) (scalingo.DatabaseNG, error) {
+			calls++
+			return databaseWithStatus(scalingo.DatabaseStatusCreating), nil
+		}
+
+		err := waitForRunningDatabaseWithRetry(t.Context(), show, testRetryer(2), "db-123")
+
+		require.ErrorContains(t, err, "database db-123 status is creating")
+		assert.Equal(t, 2, calls)
+	})
+}
+
+func TestWaitForRunningDatabaseWithTimeoutCancelsActiveLookup(t *testing.T) {
+	show := func(ctx context.Context, _ string) (scalingo.DatabaseNG, error) {
+		deadline, ok := ctx.Deadline()
+		require.True(t, ok)
+		assert.WithinDuration(t, time.Now().Add(10*time.Millisecond), deadline, 10*time.Millisecond)
+
+		<-ctx.Done()
+		return scalingo.DatabaseNG{}, ctx.Err()
+	}
+
+	err := waitForRunningDatabaseWithTimeout(t.Context(), show, "db-123", 10*time.Millisecond)
+
+	require.ErrorIs(t, err, context.DeadlineExceeded)
+	require.ErrorContains(t, err, "get database db-123 status")
+}
+
+func databaseWithStatus(status scalingo.DatabaseStatus) scalingo.DatabaseNG {
+	return scalingo.DatabaseNG{Database: scalingo.Database{Status: status}}
+}
+
+func testRetryer(maxAttempts int) retry.Retry {
+	return retry.New(
+		retry.WithMaxAttempts(maxAttempts),
+		retry.WithWaitDuration(time.Nanosecond),
+	)
+}

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/Scalingo/go-utils/errors/v3 v3.2.1
 	github.com/Scalingo/go-utils/logger v1.13.0
 	github.com/Scalingo/go-utils/pagination v1.2.0
+	github.com/Scalingo/go-utils/retry v1.4.2
 	github.com/Scalingo/gopassword v1.1.1
 	github.com/briandowns/spinner v1.23.2
 	github.com/cheggaaa/pb/v3 v3.2.1

go.sum

@@ -23,6 +23,8 @@ github.com/Scalingo/go-utils/logger v1.13.0 h1:3dW2hDo50Oqvdoi1iEFXX0GRRKLvjJLeV
 github.com/Scalingo/go-utils/logger v1.13.0/go.mod h1:EYTGwK0n5mOtLYLus65qYxoZKkwcaVRhumErnI8mloU=
 github.com/Scalingo/go-utils/pagination v1.2.0 h1:dTvce+thcZXH/S13HF5dxGNPCMdGALNsW6p2EYe2Pno=
 github.com/Scalingo/go-utils/pagination v1.2.0/go.mod h1:MwT1RRecXktFgVNeib+B4dnM4r0cPc93JichOPs5tj0=
+github.com/Scalingo/go-utils/retry v1.4.2 h1:1Mg9aH7/9spI7ZYxB4HtCcv7O58IYm/8nK950uCVVek=
+github.com/Scalingo/go-utils/retry v1.4.2/go.mod h1:0ZCGJRmYuQCCreG28dUDOXONv3PMSuuWuc+B0VBIjeg=
 github.com/Scalingo/gopassword v1.1.1 h1:RMsYJPz/DyfvWicL8RgysncnP6/0CmP3DJx1q+Dg1hU=
 github.com/Scalingo/gopassword v1.1.1/go.mod h1:Fw7SFIydC5aPCLewCKjFVEQ1bPC83VR2yh6t5bofViQ=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=