v3.42.33: Add cenkalti/backoff for exponential backoff retry

- Exponential backoff retry for all cloud operations (S3, Azure, GCS)
- RetryConfig presets: Default (5x), Aggressive (10x), Quick (3x)
- Smart error classification: IsPermanentError, IsRetryableError
- Automatic file position reset on upload retry
- Retry logging with wait duration
- Multipart uploads use aggressive retry (more tolerance)

CHANGELOG.md

@@ -5,6 +5,25 @@ All notable changes to dbbackup will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.42.33] - 2026-01-14 "Exponential Backoff Retry"
+
+### Added - cenkalti/backoff for Cloud Operation Retry
+- **Exponential backoff retry** for all cloud operations (S3, Azure, GCS)
+- **Retry configurations**:
+  - `DefaultRetryConfig()` - 5 retries, 500ms→30s backoff, 5 min max
+  - `AggressiveRetryConfig()` - 10 retries, 1s→60s backoff, 15 min max
+  - `QuickRetryConfig()` - 3 retries, 100ms→5s backoff, 30s max
+- **Smart error classification**:
+  - `IsPermanentError()` - Auth/bucket errors (no retry)
+  - `IsRetryableError()` - Timeout/network errors (retry)
+- **Retry logging** - Each retry attempt is logged with wait duration
+
+### Changed
+- S3 simple upload, multipart upload, download now retry on transient failures
+- Azure simple upload, download now retry on transient failures
+- GCS upload, download now retry on transient failures
+- Large file multipart uploads use `AggressiveRetryConfig()` (more retries)
+
 ## [3.42.32] - 2026-01-14 "Cross-Platform Colors"
 
 ### Added - fatih/color for Cross-Platform Terminal Colors

bin/README.md

@@ -3,9 +3,9 @@
 This directory contains pre-compiled binaries for the DB Backup Tool across multiple platforms and architectures.
 
 ## Build Information
-- **Version**: 3.42.31
+- **Version**: 3.42.32
-- **Build Time**: 2026-01-14_15:07:12_UTC
+- **Build Time**: 2026-01-14_15:13:08_UTC
-- **Git Commit**: dc6dfd8
+- **Git Commit**: 6a24ee3
 
 ## Recent Updates (v1.1.0)
 - ✅ Fixed TUI progress display with line-by-line output

go.mod

@@ -57,6 +57,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sts v1.41.2 // indirect
 	github.com/aws/smithy-go v1.23.2 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect
 	github.com/charmbracelet/x/ansi v0.10.1 // indirect

go.sum

@@ -84,6 +84,8 @@ github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM=
 github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
+github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
+github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/charmbracelet/bubbles v0.21.0 h1:9TdC97SdRVg/1aaXNVWfFH3nnLAwOXr8Fn6u6mfQdFs=

internal/cloud/azure.go

@@ -151,8 +151,14 @@ func (a *AzureBackend) Upload(ctx context.Context, localPath, remotePath string,
 	return a.uploadSimple(ctx, file, blobName, fileSize, progress)
 }
 
-// uploadSimple uploads a file using simple upload (single request)
+// uploadSimple uploads a file using simple upload (single request) with retry
 func (a *AzureBackend) uploadSimple(ctx context.Context, file *os.File, blobName string, fileSize int64, progress ProgressCallback) error {
+	return RetryOperationWithNotify(ctx, DefaultRetryConfig(), func() error {
+		// Reset file position for retry
+		if _, err := file.Seek(0, 0); err != nil {
+			return fmt.Errorf("failed to reset file position: %w", err)
+		}
+
 		blockBlobClient := a.client.ServiceClient().NewContainerClient(a.containerName).NewBlockBlobClient(blobName)
 
 		// Wrap reader with progress tracking
@@ -182,6 +188,9 @@ func (a *AzureBackend) uploadSimple(ctx context.Context, file *os.File, blobName
 		}
 
 		return nil
+	}, func(err error, duration time.Duration) {
+		fmt.Printf("[Azure] Upload retry in %v: %v\n", duration, err)
+	})
 }
 
 // uploadBlocks uploads a file using block blob staging (for large files)
@@ -251,7 +260,7 @@ func (a *AzureBackend) uploadBlocks(ctx context.Context, file *os.File, blobName
 	return nil
 }
 
-// Download downloads a file from Azure Blob Storage
+// Download downloads a file from Azure Blob Storage with retry
 func (a *AzureBackend) Download(ctx context.Context, remotePath, localPath string, progress ProgressCallback) error {
 	blobName := strings.TrimPrefix(remotePath, "/")
 	blockBlobClient := a.client.ServiceClient().NewContainerClient(a.containerName).NewBlockBlobClient(blobName)
@@ -264,6 +273,7 @@ func (a *AzureBackend) Download(ctx context.Context, remotePath, localPath strin
 
 	fileSize := *props.ContentLength
 
+	return RetryOperationWithNotify(ctx, DefaultRetryConfig(), func() error {
 		// Download blob
 		resp, err := blockBlobClient.DownloadStream(ctx, nil)
 		if err != nil {
@@ -271,7 +281,7 @@ func (a *AzureBackend) Download(ctx context.Context, remotePath, localPath strin
 		}
 		defer resp.Body.Close()
 
-	// Create local file
+		// Create/truncate local file
 		file, err := os.Create(localPath)
 		if err != nil {
 			return fmt.Errorf("failed to create file: %w", err)
@@ -288,6 +298,9 @@ func (a *AzureBackend) Download(ctx context.Context, remotePath, localPath strin
 		}
 
 		return nil
+	}, func(err error, duration time.Duration) {
+		fmt.Printf("[Azure] Download retry in %v: %v\n", duration, err)
+	})
 }
 
 // Delete deletes a file from Azure Blob Storage

internal/cloud/gcs.go

@@ -89,7 +89,7 @@ func (g *GCSBackend) Name() string {
 	return "gcs"
 }
 
-// Upload uploads a file to Google Cloud Storage
+// Upload uploads a file to Google Cloud Storage with retry
 func (g *GCSBackend) Upload(ctx context.Context, localPath, remotePath string, progress ProgressCallback) error {
 	file, err := os.Open(localPath)
 	if err != nil {
@@ -106,6 +106,12 @@ func (g *GCSBackend) Upload(ctx context.Context, localPath, remotePath string, p
 	// Remove leading slash from remote path
 	objectName := strings.TrimPrefix(remotePath, "/")
 
+	return RetryOperationWithNotify(ctx, DefaultRetryConfig(), func() error {
+		// Reset file position for retry
+		if _, err := file.Seek(0, 0); err != nil {
+			return fmt.Errorf("failed to reset file position: %w", err)
+		}
+
 		bucket := g.client.Bucket(g.bucketName)
 		object := bucket.Object(objectName)
 
@@ -142,9 +148,12 @@ func (g *GCSBackend) Upload(ctx context.Context, localPath, remotePath string, p
 		}
 
 		return nil
+	}, func(err error, duration time.Duration) {
+		fmt.Printf("[GCS] Upload retry in %v: %v\n", duration, err)
+	})
 }
 
-// Download downloads a file from Google Cloud Storage
+// Download downloads a file from Google Cloud Storage with retry
 func (g *GCSBackend) Download(ctx context.Context, remotePath, localPath string, progress ProgressCallback) error {
 	objectName := strings.TrimPrefix(remotePath, "/")
 
@@ -159,6 +168,7 @@ func (g *GCSBackend) Download(ctx context.Context, remotePath, localPath string,
 
 	fileSize := attrs.Size
 
+	return RetryOperationWithNotify(ctx, DefaultRetryConfig(), func() error {
 		// Create reader
 		reader, err := object.NewReader(ctx)
 		if err != nil {
@@ -166,7 +176,7 @@ func (g *GCSBackend) Download(ctx context.Context, remotePath, localPath string,
 		}
 		defer reader.Close()
 
-	// Create local file
+		// Create/truncate local file
 		file, err := os.Create(localPath)
 		if err != nil {
 			return fmt.Errorf("failed to create file: %w", err)
@@ -183,6 +193,9 @@ func (g *GCSBackend) Download(ctx context.Context, remotePath, localPath string,
 		}
 
 		return nil
+	}, func(err error, duration time.Duration) {
+		fmt.Printf("[GCS] Download retry in %v: %v\n", duration, err)
+	})
 }
 
 // Delete deletes a file from Google Cloud Storage

internal/cloud/retry.go

@@ -0,0 +1,257 @@
+package cloud
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strings"
+	"time"
+
+	"github.com/cenkalti/backoff/v4"
+)
+
+// RetryConfig configures retry behavior
+type RetryConfig struct {
+	MaxRetries      int           // Maximum number of retries (0 = unlimited)
+	InitialInterval time.Duration // Initial backoff interval
+	MaxInterval     time.Duration // Maximum backoff interval
+	MaxElapsedTime  time.Duration // Maximum total time for retries
+	Multiplier      float64       // Backoff multiplier
+}
+
+// DefaultRetryConfig returns sensible defaults for cloud operations
+func DefaultRetryConfig() *RetryConfig {
+	return &RetryConfig{
+		MaxRetries:      5,
+		InitialInterval: 500 * time.Millisecond,
+		MaxInterval:     30 * time.Second,
+		MaxElapsedTime:  5 * time.Minute,
+		Multiplier:      2.0,
+	}
+}
+
+// AggressiveRetryConfig returns config for critical operations that need more retries
+func AggressiveRetryConfig() *RetryConfig {
+	return &RetryConfig{
+		MaxRetries:      10,
+		InitialInterval: 1 * time.Second,
+		MaxInterval:     60 * time.Second,
+		MaxElapsedTime:  15 * time.Minute,
+		Multiplier:      1.5,
+	}
+}
+
+// QuickRetryConfig returns config for operations that should fail fast
+func QuickRetryConfig() *RetryConfig {
+	return &RetryConfig{
+		MaxRetries:      3,
+		InitialInterval: 100 * time.Millisecond,
+		MaxInterval:     5 * time.Second,
+		MaxElapsedTime:  30 * time.Second,
+		Multiplier:      2.0,
+	}
+}
+
+// RetryOperation executes an operation with exponential backoff retry
+func RetryOperation(ctx context.Context, cfg *RetryConfig, operation func() error) error {
+	if cfg == nil {
+		cfg = DefaultRetryConfig()
+	}
+
+	// Create exponential backoff
+	expBackoff := backoff.NewExponentialBackOff()
+	expBackoff.InitialInterval = cfg.InitialInterval
+	expBackoff.MaxInterval = cfg.MaxInterval
+	expBackoff.MaxElapsedTime = cfg.MaxElapsedTime
+	expBackoff.Multiplier = cfg.Multiplier
+	expBackoff.Reset()
+
+	// Wrap with max retries if specified
+	var b backoff.BackOff = expBackoff
+	if cfg.MaxRetries > 0 {
+		b = backoff.WithMaxRetries(expBackoff, uint64(cfg.MaxRetries))
+	}
+
+	// Add context support
+	b = backoff.WithContext(b, ctx)
+
+	// Track attempts for logging
+	attempt := 0
+
+	// Wrap operation to handle permanent vs retryable errors
+	wrappedOp := func() error {
+		attempt++
+		err := operation()
+		if err == nil {
+			return nil
+		}
+
+		// Check if error is permanent (should not retry)
+		if IsPermanentError(err) {
+			return backoff.Permanent(err)
+		}
+
+		return err
+	}
+
+	return backoff.Retry(wrappedOp, b)
+}
+
+// RetryOperationWithNotify executes an operation with retry and calls notify on each retry
+func RetryOperationWithNotify(ctx context.Context, cfg *RetryConfig, operation func() error, notify func(err error, duration time.Duration)) error {
+	if cfg == nil {
+		cfg = DefaultRetryConfig()
+	}
+
+	// Create exponential backoff
+	expBackoff := backoff.NewExponentialBackOff()
+	expBackoff.InitialInterval = cfg.InitialInterval
+	expBackoff.MaxInterval = cfg.MaxInterval
+	expBackoff.MaxElapsedTime = cfg.MaxElapsedTime
+	expBackoff.Multiplier = cfg.Multiplier
+	expBackoff.Reset()
+
+	// Wrap with max retries if specified
+	var b backoff.BackOff = expBackoff
+	if cfg.MaxRetries > 0 {
+		b = backoff.WithMaxRetries(expBackoff, uint64(cfg.MaxRetries))
+	}
+
+	// Add context support
+	b = backoff.WithContext(b, ctx)
+
+	// Wrap operation to handle permanent vs retryable errors
+	wrappedOp := func() error {
+		err := operation()
+		if err == nil {
+			return nil
+		}
+
+		// Check if error is permanent (should not retry)
+		if IsPermanentError(err) {
+			return backoff.Permanent(err)
+		}
+
+		return err
+	}
+
+	return backoff.RetryNotify(wrappedOp, b, notify)
+}
+
+// IsPermanentError returns true if the error should not be retried
+func IsPermanentError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	errStr := strings.ToLower(err.Error())
+
+	// Authentication/authorization errors - don't retry
+	permanentPatterns := []string{
+		"access denied",
+		"forbidden",
+		"unauthorized",
+		"invalid credentials",
+		"invalid access key",
+		"invalid secret",
+		"no such bucket",
+		"bucket not found",
+		"container not found",
+		"nosuchbucket",
+		"nosuchkey",
+		"invalid argument",
+		"malformed",
+		"invalid request",
+		"permission denied",
+		"access control",
+		"policy",
+	}
+
+	for _, pattern := range permanentPatterns {
+		if strings.Contains(errStr, pattern) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// IsRetryableError returns true if the error is transient and should be retried
+func IsRetryableError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	// Network errors are typically retryable
+	var netErr net.Error
+	if ok := isNetError(err, &netErr); ok {
+		return netErr.Timeout() || netErr.Temporary()
+	}
+
+	errStr := strings.ToLower(err.Error())
+
+	// Transient errors - should retry
+	retryablePatterns := []string{
+		"timeout",
+		"connection reset",
+		"connection refused",
+		"connection closed",
+		"eof",
+		"broken pipe",
+		"temporary failure",
+		"service unavailable",
+		"internal server error",
+		"bad gateway",
+		"gateway timeout",
+		"too many requests",
+		"rate limit",
+		"throttl",
+		"slowdown",
+		"try again",
+		"retry",
+	}
+
+	for _, pattern := range retryablePatterns {
+		if strings.Contains(errStr, pattern) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// isNetError checks if err wraps a net.Error
+func isNetError(err error, target *net.Error) bool {
+	for err != nil {
+		if ne, ok := err.(net.Error); ok {
+			*target = ne
+			return true
+		}
+		// Try to unwrap
+		if unwrapper, ok := err.(interface{ Unwrap() error }); ok {
+			err = unwrapper.Unwrap()
+		} else {
+			break
+		}
+	}
+	return false
+}
+
+// WithRetry is a helper that wraps a function with default retry logic
+func WithRetry(ctx context.Context, operationName string, fn func() error) error {
+	notify := func(err error, duration time.Duration) {
+		// Log retry attempts (caller can provide their own logger if needed)
+		fmt.Printf("[RETRY] %s failed, retrying in %v: %v\n", operationName, duration, err)
+	}
+
+	return RetryOperationWithNotify(ctx, DefaultRetryConfig(), fn, notify)
+}
+
+// WithRetryConfig is a helper that wraps a function with custom retry config
+func WithRetryConfig(ctx context.Context, cfg *RetryConfig, operationName string, fn func() error) error {
+	notify := func(err error, duration time.Duration) {
+		fmt.Printf("[RETRY] %s failed, retrying in %v: %v\n", operationName, duration, err)
+	}
+
+	return RetryOperationWithNotify(ctx, cfg, fn, notify)
+}

internal/cloud/s3.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
@@ -123,8 +124,14 @@ func (s *S3Backend) Upload(ctx context.Context, localPath, remotePath string, pr
 	return s.uploadSimple(ctx, file, key, fileSize, progress)
 }
 
-// uploadSimple performs a simple single-part upload
+// uploadSimple performs a simple single-part upload with retry
 func (s *S3Backend) uploadSimple(ctx context.Context, file *os.File, key string, fileSize int64, progress ProgressCallback) error {
+	return RetryOperationWithNotify(ctx, DefaultRetryConfig(), func() error {
+		// Reset file position for retry
+		if _, err := file.Seek(0, 0); err != nil {
+			return fmt.Errorf("failed to reset file position: %w", err)
+		}
+
 		// Create progress reader
 		var reader io.Reader = file
 		if progress != nil {
@@ -143,10 +150,19 @@ func (s *S3Backend) uploadSimple(ctx context.Context, file *os.File, key string,
 		}
 
 		return nil
+	}, func(err error, duration time.Duration) {
+		fmt.Printf("[S3] Upload retry in %v: %v\n", duration, err)
+	})
 }
 
-// uploadMultipart performs a multipart upload for large files
+// uploadMultipart performs a multipart upload for large files with retry
 func (s *S3Backend) uploadMultipart(ctx context.Context, file *os.File, key string, fileSize int64, progress ProgressCallback) error {
+	return RetryOperationWithNotify(ctx, AggressiveRetryConfig(), func() error {
+		// Reset file position for retry
+		if _, err := file.Seek(0, 0); err != nil {
+			return fmt.Errorf("failed to reset file position: %w", err)
+		}
+
 		// Create uploader with custom options
 		uploader := manager.NewUploader(s.client, func(u *manager.Uploader) {
 			// Part size: 10MB
@@ -177,9 +193,12 @@ func (s *S3Backend) uploadMultipart(ctx context.Context, file *os.File, key stri
 		}
 
 		return nil
+	}, func(err error, duration time.Duration) {
+		fmt.Printf("[S3] Multipart upload retry in %v: %v\n", duration, err)
+	})
 }
 
-// Download downloads a file from S3
+// Download downloads a file from S3 with retry
 func (s *S3Backend) Download(ctx context.Context, remotePath, localPath string, progress ProgressCallback) error {
 	// Build S3 key
 	key := s.buildKey(remotePath)
@@ -190,6 +209,12 @@ func (s *S3Backend) Download(ctx context.Context, remotePath, localPath string,
 		return fmt.Errorf("failed to get object size: %w", err)
 	}
 
+	// Create directory for local file
+	if err := os.MkdirAll(filepath.Dir(localPath), 0755); err != nil {
+		return fmt.Errorf("failed to create directory: %w", err)
+	}
+
+	return RetryOperationWithNotify(ctx, DefaultRetryConfig(), func() error {
 		// Download from S3
 		result, err := s.client.GetObject(ctx, &s3.GetObjectInput{
 			Bucket: aws.String(s.bucket),
@@ -200,11 +225,7 @@ func (s *S3Backend) Download(ctx context.Context, remotePath, localPath string,
 		}
 		defer result.Body.Close()
 
-	// Create local file
+		// Create/truncate local file
-	if err := os.MkdirAll(filepath.Dir(localPath), 0755); err != nil {
-		return fmt.Errorf("failed to create directory: %w", err)
-	}
-
 		outFile, err := os.Create(localPath)
 		if err != nil {
 			return fmt.Errorf("failed to create local file: %w", err)
@@ -223,6 +244,9 @@ func (s *S3Backend) Download(ctx context.Context, remotePath, localPath string,
 		}
 
 		return nil
+	}, func(err error, duration time.Duration) {
+		fmt.Printf("[S3] Download retry in %v: %v\n", duration, err)
+	})
 }
 
 // List lists all backup files in S3

main.go

@@ -16,7 +16,7 @@ import (
 
 // Build information (set by ldflags)
 var (
-	version   = "3.42.32"
+	version   = "3.42.33"
 	buildTime = "unknown"
 	gitCommit = "unknown"
 )