v3.42.33: Add cenkalti/backoff for exponential backoff retry

- Exponential backoff retry for all cloud operations (S3, Azure, GCS) - RetryConfig presets: Default (5x), Aggressive (10x), Quick (3x) - Smart error classification: IsPermanentError, IsRetryableError - Automatic file position reset on upload retry - Retry logging with wait duration - Multipart uploads use aggressive retry (more tolerance)
v3.42.32: Add fatih/color for cross-platform terminal colors
2026-01-14 16:19:40 +01:00 · 2026-01-14 16:13:00 +01:00
12 changed files with 712 additions and 200 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,42 @@ All notable changes to dbbackup will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

+## [3.42.33] - 2026-01-14 "Exponential Backoff Retry"
+
+### Added - cenkalti/backoff for Cloud Operation Retry
+- **Exponential backoff retry** for all cloud operations (S3, Azure, GCS)
+- **Retry configurations**:
+  - `DefaultRetryConfig()` - 5 retries, 500ms→30s backoff, 5 min max
+  - `AggressiveRetryConfig()` - 10 retries, 1s→60s backoff, 15 min max
+  - `QuickRetryConfig()` - 3 retries, 100ms→5s backoff, 30s max
+- **Smart error classification**:
+  - `IsPermanentError()` - Auth/bucket errors (no retry)
+  - `IsRetryableError()` - Timeout/network errors (retry)
+- **Retry logging** - Each retry attempt is logged with wait duration
+
+### Changed
+- S3 simple upload, multipart upload, download now retry on transient failures
+- Azure simple upload, download now retry on transient failures
+- GCS upload, download now retry on transient failures
+- Large file multipart uploads use `AggressiveRetryConfig()` (more retries)
+
+## [3.42.32] - 2026-01-14 "Cross-Platform Colors"
+
+### Added - fatih/color for Cross-Platform Terminal Colors
+- **Windows-compatible colors** - Native Windows console API support
+- **Color helper functions** in `logger` package:
+  - `Success()`, `Error()`, `Warning()`, `Info()` - Status messages with icons
+  - `Header()`, `Dim()`, `Bold()` - Text styling
+  - `Green()`, `Red()`, `Yellow()`, `Cyan()` - Colored text
+  - `StatusLine()`, `TableRow()` - Formatted output
+  - `DisableColors()`, `EnableColors()` - Runtime control
+- **Consistent color scheme** across all log levels
+
+### Changed
+- Logger `CleanFormatter` now uses fatih/color instead of raw ANSI codes
+- All progress indicators use fatih/color for `[OK]`/`[FAIL]` status
+- Automatic color detection (disabled for non-TTY)
+
 ## [3.42.31] - 2026-01-14 "Visual Progress Bars"

 ### Added - schollz/progressbar for Enhanced Progress Display
--- a/bin/README.md
+++ b/bin/README.md
@@ -3,9 +3,9 @@
 This directory contains pre-compiled binaries for the DB Backup Tool across multiple platforms and architectures.

 ## Build Information
- **Version**: 3.42.30
- **Build Time**: 2026-01-14_14:59:20_UTC
- **Git Commit**: 7b4ab76
+- **Version**: 3.42.32
+- **Build Time**: 2026-01-14_15:13:08_UTC
+- **Git Commit**: 6a24ee3

 ## Recent Updates (v1.1.0)
 - ✅ Fixed TUI progress display with line-by-line output
--- a/go.mod
+++ b/go.mod
@@ -57,6 +57,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sts v1.41.2 // indirect
 	github.com/aws/smithy-go v1.23.2 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect
 	github.com/charmbracelet/x/ansi v0.10.1 // indirect
@@ -66,6 +67,7 @@ require (
 	github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
 	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
+	github.com/fatih/color v1.18.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-jose/go-jose/v4 v4.1.2 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
@@ -83,6 +85,7 @@ require (
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-localereader v0.0.1 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
--- a/go.sum
+++ b/go.sum
@@ -84,6 +84,8 @@ github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM=
 github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
+github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
+github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/charmbracelet/bubbles v0.21.0 h1:9TdC97SdRVg/1aaXNVWfFH3nnLAwOXr8Fn6u6mfQdFs=
@@ -119,6 +121,8 @@ github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfU
 github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU=
 github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
 github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
+github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
+github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/go-jose/go-jose/v4 v4.1.2 h1:TK/7NqRQZfgAh+Td8AlsrvtPoUyiHh0LqVvokh+1vHI=
@@ -169,6 +173,9 @@ github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69
 github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
@@ -259,6 +266,7 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
--- a/internal/cloud/azure.go
+++ b/internal/cloud/azure.go
@@ -151,37 +151,46 @@ func (a *AzureBackend) Upload(ctx context.Context, localPath, remotePath string,
 	return a.uploadSimple(ctx, file, blobName, fileSize, progress)
 }

-// uploadSimple uploads a file using simple upload (single request)
+// uploadSimple uploads a file using simple upload (single request) with retry
 func (a *AzureBackend) uploadSimple(ctx context.Context, file *os.File, blobName string, fileSize int64, progress ProgressCallback) error {
-	blockBlobClient := a.client.ServiceClient().NewContainerClient(a.containerName).NewBlockBlobClient(blobName)
+	return RetryOperationWithNotify(ctx, DefaultRetryConfig(), func() error {
+		// Reset file position for retry
+		if _, err := file.Seek(0, 0); err != nil {
+			return fmt.Errorf("failed to reset file position: %w", err)
+		}

-	// Wrap reader with progress tracking
-	reader := NewProgressReader(file, fileSize, progress)
+		blockBlobClient := a.client.ServiceClient().NewContainerClient(a.containerName).NewBlockBlobClient(blobName)

-	// Calculate MD5 hash for integrity
-	hash := sha256.New()
-	teeReader := io.TeeReader(reader, hash)
+		// Wrap reader with progress tracking
+		reader := NewProgressReader(file, fileSize, progress)

-	_, err := blockBlobClient.UploadStream(ctx, teeReader, &blockblob.UploadStreamOptions{
-		BlockSize: 4 * 1024 * 1024, // 4MB blocks
+		// Calculate MD5 hash for integrity
+		hash := sha256.New()
+		teeReader := io.TeeReader(reader, hash)
+
+		_, err := blockBlobClient.UploadStream(ctx, teeReader, &blockblob.UploadStreamOptions{
+			BlockSize: 4 * 1024 * 1024, // 4MB blocks
+		})
+		if err != nil {
+			return fmt.Errorf("failed to upload blob: %w", err)
+		}
+
+		// Store checksum as metadata
+		checksum := hex.EncodeToString(hash.Sum(nil))
+		metadata := map[string]*string{
+			"sha256": &checksum,
+		}
+
+		_, err = blockBlobClient.SetMetadata(ctx, metadata, nil)
+		if err != nil {
+			// Non-fatal: upload succeeded but metadata failed
+			fmt.Fprintf(os.Stderr, "Warning: failed to set blob metadata: %v\n", err)
+		}
+
+		return nil
+	}, func(err error, duration time.Duration) {
+		fmt.Printf("[Azure] Upload retry in %v: %v\n", duration, err)
 	})
-	if err != nil {
-		return fmt.Errorf("failed to upload blob: %w", err)
-	}
-
-	// Store checksum as metadata
-	checksum := hex.EncodeToString(hash.Sum(nil))
-	metadata := map[string]*string{
-		"sha256": &checksum,
-	}
-
-	_, err = blockBlobClient.SetMetadata(ctx, metadata, nil)
-	if err != nil {
-		// Non-fatal: upload succeeded but metadata failed
-		fmt.Fprintf(os.Stderr, "Warning: failed to set blob metadata: %v\n", err)
-	}
-
-	return nil
 }

 // uploadBlocks uploads a file using block blob staging (for large files)
@@ -251,7 +260,7 @@ func (a *AzureBackend) uploadBlocks(ctx context.Context, file *os.File, blobName
 	return nil
 }

-// Download downloads a file from Azure Blob Storage
+// Download downloads a file from Azure Blob Storage with retry
 func (a *AzureBackend) Download(ctx context.Context, remotePath, localPath string, progress ProgressCallback) error {
 	blobName := strings.TrimPrefix(remotePath, "/")
 	blockBlobClient := a.client.ServiceClient().NewContainerClient(a.containerName).NewBlockBlobClient(blobName)
@@ -264,30 +273,34 @@ func (a *AzureBackend) Download(ctx context.Context, remotePath, localPath strin

 	fileSize := *props.ContentLength

-	// Download blob
-	resp, err := blockBlobClient.DownloadStream(ctx, nil)
-	if err != nil {
-		return fmt.Errorf("failed to download blob: %w", err)
-	}
-	defer resp.Body.Close()
+	return RetryOperationWithNotify(ctx, DefaultRetryConfig(), func() error {
+		// Download blob
+		resp, err := blockBlobClient.DownloadStream(ctx, nil)
+		if err != nil {
+			return fmt.Errorf("failed to download blob: %w", err)
+		}
+		defer resp.Body.Close()

-	// Create local file
-	file, err := os.Create(localPath)
-	if err != nil {
-		return fmt.Errorf("failed to create file: %w", err)
-	}
-	defer file.Close()
+		// Create/truncate local file
+		file, err := os.Create(localPath)
+		if err != nil {
+			return fmt.Errorf("failed to create file: %w", err)
+		}
+		defer file.Close()

-	// Wrap reader with progress tracking
-	reader := NewProgressReader(resp.Body, fileSize, progress)
+		// Wrap reader with progress tracking
+		reader := NewProgressReader(resp.Body, fileSize, progress)

-	// Copy with progress
-	_, err = io.Copy(file, reader)
-	if err != nil {
-		return fmt.Errorf("failed to write file: %w", err)
-	}
+		// Copy with progress
+		_, err = io.Copy(file, reader)
+		if err != nil {
+			return fmt.Errorf("failed to write file: %w", err)
+		}

-	return nil
+		return nil
+	}, func(err error, duration time.Duration) {
+		fmt.Printf("[Azure] Download retry in %v: %v\n", duration, err)
+	})
 }

 // Delete deletes a file from Azure Blob Storage
--- a/internal/cloud/gcs.go
+++ b/internal/cloud/gcs.go
@@ -89,7 +89,7 @@ func (g *GCSBackend) Name() string {
 	return "gcs"
 }

-// Upload uploads a file to Google Cloud Storage
+// Upload uploads a file to Google Cloud Storage with retry
 func (g *GCSBackend) Upload(ctx context.Context, localPath, remotePath string, progress ProgressCallback) error {
 	file, err := os.Open(localPath)
 	if err != nil {
@@ -106,45 +106,54 @@ func (g *GCSBackend) Upload(ctx context.Context, localPath, remotePath string, p
 	// Remove leading slash from remote path
 	objectName := strings.TrimPrefix(remotePath, "/")

-	bucket := g.client.Bucket(g.bucketName)
-	object := bucket.Object(objectName)
+	return RetryOperationWithNotify(ctx, DefaultRetryConfig(), func() error {
+		// Reset file position for retry
+		if _, err := file.Seek(0, 0); err != nil {
+			return fmt.Errorf("failed to reset file position: %w", err)
+		}

-	// Create writer with automatic chunking for large files
-	writer := object.NewWriter(ctx)
-	writer.ChunkSize = 16 * 1024 * 1024 // 16MB chunks for streaming
+		bucket := g.client.Bucket(g.bucketName)
+		object := bucket.Object(objectName)

-	// Wrap reader with progress tracking and hash calculation
-	hash := sha256.New()
-	reader := NewProgressReader(io.TeeReader(file, hash), fileSize, progress)
+		// Create writer with automatic chunking for large files
+		writer := object.NewWriter(ctx)
+		writer.ChunkSize = 16 * 1024 * 1024 // 16MB chunks for streaming

-	// Upload with progress tracking
-	_, err = io.Copy(writer, reader)
-	if err != nil {
-		writer.Close()
-		return fmt.Errorf("failed to upload object: %w", err)
-	}
+		// Wrap reader with progress tracking and hash calculation
+		hash := sha256.New()
+		reader := NewProgressReader(io.TeeReader(file, hash), fileSize, progress)

-	// Close writer (finalizes upload)
-	if err := writer.Close(); err != nil {
-		return fmt.Errorf("failed to finalize upload: %w", err)
-	}
+		// Upload with progress tracking
+		_, err = io.Copy(writer, reader)
+		if err != nil {
+			writer.Close()
+			return fmt.Errorf("failed to upload object: %w", err)
+		}

-	// Store checksum as metadata
-	checksum := hex.EncodeToString(hash.Sum(nil))
-	_, err = object.Update(ctx, storage.ObjectAttrsToUpdate{
-		Metadata: map[string]string{
-			"sha256": checksum,
-		},
+		// Close writer (finalizes upload)
+		if err := writer.Close(); err != nil {
+			return fmt.Errorf("failed to finalize upload: %w", err)
+		}
+
+		// Store checksum as metadata
+		checksum := hex.EncodeToString(hash.Sum(nil))
+		_, err = object.Update(ctx, storage.ObjectAttrsToUpdate{
+			Metadata: map[string]string{
+				"sha256": checksum,
+			},
+		})
+		if err != nil {
+			// Non-fatal: upload succeeded but metadata failed
+			fmt.Fprintf(os.Stderr, "Warning: failed to set object metadata: %v\n", err)
+		}
+
+		return nil
+	}, func(err error, duration time.Duration) {
+		fmt.Printf("[GCS] Upload retry in %v: %v\n", duration, err)
 	})
-	if err != nil {
-		// Non-fatal: upload succeeded but metadata failed
-		fmt.Fprintf(os.Stderr, "Warning: failed to set object metadata: %v\n", err)
-	}
-
-	return nil
 }

-// Download downloads a file from Google Cloud Storage
+// Download downloads a file from Google Cloud Storage with retry
 func (g *GCSBackend) Download(ctx context.Context, remotePath, localPath string, progress ProgressCallback) error {
 	objectName := strings.TrimPrefix(remotePath, "/")

@@ -159,30 +168,34 @@ func (g *GCSBackend) Download(ctx context.Context, remotePath, localPath string,

 	fileSize := attrs.Size

-	// Create reader
-	reader, err := object.NewReader(ctx)
-	if err != nil {
-		return fmt.Errorf("failed to download object: %w", err)
-	}
-	defer reader.Close()
+	return RetryOperationWithNotify(ctx, DefaultRetryConfig(), func() error {
+		// Create reader
+		reader, err := object.NewReader(ctx)
+		if err != nil {
+			return fmt.Errorf("failed to download object: %w", err)
+		}
+		defer reader.Close()

-	// Create local file
-	file, err := os.Create(localPath)
-	if err != nil {
-		return fmt.Errorf("failed to create file: %w", err)
-	}
-	defer file.Close()
+		// Create/truncate local file
+		file, err := os.Create(localPath)
+		if err != nil {
+			return fmt.Errorf("failed to create file: %w", err)
+		}
+		defer file.Close()

-	// Wrap reader with progress tracking
-	progressReader := NewProgressReader(reader, fileSize, progress)
+		// Wrap reader with progress tracking
+		progressReader := NewProgressReader(reader, fileSize, progress)

-	// Copy with progress
-	_, err = io.Copy(file, progressReader)
-	if err != nil {
-		return fmt.Errorf("failed to write file: %w", err)
-	}
+		// Copy with progress
+		_, err = io.Copy(file, progressReader)
+		if err != nil {
+			return fmt.Errorf("failed to write file: %w", err)
+		}

-	return nil
+		return nil
+	}, func(err error, duration time.Duration) {
+		fmt.Printf("[GCS] Download retry in %v: %v\n", duration, err)
+	})
 }

 // Delete deletes a file from Google Cloud Storage
--- a/internal/cloud/retry.go
+++ b/internal/cloud/retry.go
@@ -0,0 +1,257 @@
+package cloud
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strings"
+	"time"
+
+	"github.com/cenkalti/backoff/v4"
+)
+
+// RetryConfig configures retry behavior
+type RetryConfig struct {
+	MaxRetries      int           // Maximum number of retries (0 = unlimited)
+	InitialInterval time.Duration // Initial backoff interval
+	MaxInterval     time.Duration // Maximum backoff interval
+	MaxElapsedTime  time.Duration // Maximum total time for retries
+	Multiplier      float64       // Backoff multiplier
+}
+
+// DefaultRetryConfig returns sensible defaults for cloud operations
+func DefaultRetryConfig() *RetryConfig {
+	return &RetryConfig{
+		MaxRetries:      5,
+		InitialInterval: 500 * time.Millisecond,
+		MaxInterval:     30 * time.Second,
+		MaxElapsedTime:  5 * time.Minute,
+		Multiplier:      2.0,
+	}
+}
+
+// AggressiveRetryConfig returns config for critical operations that need more retries
+func AggressiveRetryConfig() *RetryConfig {
+	return &RetryConfig{
+		MaxRetries:      10,
+		InitialInterval: 1 * time.Second,
+		MaxInterval:     60 * time.Second,
+		MaxElapsedTime:  15 * time.Minute,
+		Multiplier:      1.5,
+	}
+}
+
+// QuickRetryConfig returns config for operations that should fail fast
+func QuickRetryConfig() *RetryConfig {
+	return &RetryConfig{
+		MaxRetries:      3,
+		InitialInterval: 100 * time.Millisecond,
+		MaxInterval:     5 * time.Second,
+		MaxElapsedTime:  30 * time.Second,
+		Multiplier:      2.0,
+	}
+}
+
+// RetryOperation executes an operation with exponential backoff retry
+func RetryOperation(ctx context.Context, cfg *RetryConfig, operation func() error) error {
+	if cfg == nil {
+		cfg = DefaultRetryConfig()
+	}
+
+	// Create exponential backoff
+	expBackoff := backoff.NewExponentialBackOff()
+	expBackoff.InitialInterval = cfg.InitialInterval
+	expBackoff.MaxInterval = cfg.MaxInterval
+	expBackoff.MaxElapsedTime = cfg.MaxElapsedTime
+	expBackoff.Multiplier = cfg.Multiplier
+	expBackoff.Reset()
+
+	// Wrap with max retries if specified
+	var b backoff.BackOff = expBackoff
+	if cfg.MaxRetries > 0 {
+		b = backoff.WithMaxRetries(expBackoff, uint64(cfg.MaxRetries))
+	}
+
+	// Add context support
+	b = backoff.WithContext(b, ctx)
+
+	// Track attempts for logging
+	attempt := 0
+
+	// Wrap operation to handle permanent vs retryable errors
+	wrappedOp := func() error {
+		attempt++
+		err := operation()
+		if err == nil {
+			return nil
+		}
+
+		// Check if error is permanent (should not retry)
+		if IsPermanentError(err) {
+			return backoff.Permanent(err)
+		}
+
+		return err
+	}
+
+	return backoff.Retry(wrappedOp, b)
+}
+
+// RetryOperationWithNotify executes an operation with retry and calls notify on each retry
+func RetryOperationWithNotify(ctx context.Context, cfg *RetryConfig, operation func() error, notify func(err error, duration time.Duration)) error {
+	if cfg == nil {
+		cfg = DefaultRetryConfig()
+	}
+
+	// Create exponential backoff
+	expBackoff := backoff.NewExponentialBackOff()
+	expBackoff.InitialInterval = cfg.InitialInterval
+	expBackoff.MaxInterval = cfg.MaxInterval
+	expBackoff.MaxElapsedTime = cfg.MaxElapsedTime
+	expBackoff.Multiplier = cfg.Multiplier
+	expBackoff.Reset()
+
+	// Wrap with max retries if specified
+	var b backoff.BackOff = expBackoff
+	if cfg.MaxRetries > 0 {
+		b = backoff.WithMaxRetries(expBackoff, uint64(cfg.MaxRetries))
+	}
+
+	// Add context support
+	b = backoff.WithContext(b, ctx)
+
+	// Wrap operation to handle permanent vs retryable errors
+	wrappedOp := func() error {
+		err := operation()
+		if err == nil {
+			return nil
+		}
+
+		// Check if error is permanent (should not retry)
+		if IsPermanentError(err) {
+			return backoff.Permanent(err)
+		}
+
+		return err
+	}
+
+	return backoff.RetryNotify(wrappedOp, b, notify)
+}
+
+// IsPermanentError returns true if the error should not be retried
+func IsPermanentError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	errStr := strings.ToLower(err.Error())
+
+	// Authentication/authorization errors - don't retry
+	permanentPatterns := []string{
+		"access denied",
+		"forbidden",
+		"unauthorized",
+		"invalid credentials",
+		"invalid access key",
+		"invalid secret",
+		"no such bucket",
+		"bucket not found",
+		"container not found",
+		"nosuchbucket",
+		"nosuchkey",
+		"invalid argument",
+		"malformed",
+		"invalid request",
+		"permission denied",
+		"access control",
+		"policy",
+	}
+
+	for _, pattern := range permanentPatterns {
+		if strings.Contains(errStr, pattern) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// IsRetryableError returns true if the error is transient and should be retried
+func IsRetryableError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	// Network errors are typically retryable
+	var netErr net.Error
+	if ok := isNetError(err, &netErr); ok {
+		return netErr.Timeout() || netErr.Temporary()
+	}
+
+	errStr := strings.ToLower(err.Error())
+
+	// Transient errors - should retry
+	retryablePatterns := []string{
+		"timeout",
+		"connection reset",
+		"connection refused",
+		"connection closed",
+		"eof",
+		"broken pipe",
+		"temporary failure",
+		"service unavailable",
+		"internal server error",
+		"bad gateway",
+		"gateway timeout",
+		"too many requests",
+		"rate limit",
+		"throttl",
+		"slowdown",
+		"try again",
+		"retry",
+	}
+
+	for _, pattern := range retryablePatterns {
+		if strings.Contains(errStr, pattern) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// isNetError checks if err wraps a net.Error
+func isNetError(err error, target *net.Error) bool {
+	for err != nil {
+		if ne, ok := err.(net.Error); ok {
+			*target = ne
+			return true
+		}
+		// Try to unwrap
+		if unwrapper, ok := err.(interface{ Unwrap() error }); ok {
+			err = unwrapper.Unwrap()
+		} else {
+			break
+		}
+	}
+	return false
+}
+
+// WithRetry is a helper that wraps a function with default retry logic
+func WithRetry(ctx context.Context, operationName string, fn func() error) error {
+	notify := func(err error, duration time.Duration) {
+		// Log retry attempts (caller can provide their own logger if needed)
+		fmt.Printf("[RETRY] %s failed, retrying in %v: %v\n", operationName, duration, err)
+	}
+
+	return RetryOperationWithNotify(ctx, DefaultRetryConfig(), fn, notify)
+}
+
+// WithRetryConfig is a helper that wraps a function with custom retry config
+func WithRetryConfig(ctx context.Context, cfg *RetryConfig, operationName string, fn func() error) error {
+	notify := func(err error, duration time.Duration) {
+		fmt.Printf("[RETRY] %s failed, retrying in %v: %v\n", operationName, duration, err)
+	}
+
+	return RetryOperationWithNotify(ctx, cfg, fn, notify)
+}
--- a/internal/cloud/s3.go
+++ b/internal/cloud/s3.go
@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"

 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
@@ -123,63 +124,81 @@ func (s *S3Backend) Upload(ctx context.Context, localPath, remotePath string, pr
 	return s.uploadSimple(ctx, file, key, fileSize, progress)
 }

-// uploadSimple performs a simple single-part upload
+// uploadSimple performs a simple single-part upload with retry
 func (s *S3Backend) uploadSimple(ctx context.Context, file *os.File, key string, fileSize int64, progress ProgressCallback) error {
-	// Create progress reader
-	var reader io.Reader = file
-	if progress != nil {
-		reader = NewProgressReader(file, fileSize, progress)
-	}
+	return RetryOperationWithNotify(ctx, DefaultRetryConfig(), func() error {
+		// Reset file position for retry
+		if _, err := file.Seek(0, 0); err != nil {
+			return fmt.Errorf("failed to reset file position: %w", err)
+		}

-	// Upload to S3
-	_, err := s.client.PutObject(ctx, &s3.PutObjectInput{
-		Bucket: aws.String(s.bucket),
-		Key:    aws.String(key),
-		Body:   reader,
+		// Create progress reader
+		var reader io.Reader = file
+		if progress != nil {
+			reader = NewProgressReader(file, fileSize, progress)
+		}
+
+		// Upload to S3
+		_, err := s.client.PutObject(ctx, &s3.PutObjectInput{
+			Bucket: aws.String(s.bucket),
+			Key:    aws.String(key),
+			Body:   reader,
+		})
+
+		if err != nil {
+			return fmt.Errorf("failed to upload to S3: %w", err)
+		}
+
+		return nil
+	}, func(err error, duration time.Duration) {
+		fmt.Printf("[S3] Upload retry in %v: %v\n", duration, err)
 	})
-
-	if err != nil {
-		return fmt.Errorf("failed to upload to S3: %w", err)
-	}
-
-	return nil
 }

-// uploadMultipart performs a multipart upload for large files
+// uploadMultipart performs a multipart upload for large files with retry
 func (s *S3Backend) uploadMultipart(ctx context.Context, file *os.File, key string, fileSize int64, progress ProgressCallback) error {
-	// Create uploader with custom options
-	uploader := manager.NewUploader(s.client, func(u *manager.Uploader) {
-		// Part size: 10MB
-		u.PartSize = 10 * 1024 * 1024
+	return RetryOperationWithNotify(ctx, AggressiveRetryConfig(), func() error {
+		// Reset file position for retry
+		if _, err := file.Seek(0, 0); err != nil {
+			return fmt.Errorf("failed to reset file position: %w", err)
+		}

-		// Upload up to 10 parts concurrently
-		u.Concurrency = 10
+		// Create uploader with custom options
+		uploader := manager.NewUploader(s.client, func(u *manager.Uploader) {
+			// Part size: 10MB
+			u.PartSize = 10 * 1024 * 1024

-		// Leave parts on failure for debugging
-		u.LeavePartsOnError = false
+			// Upload up to 10 parts concurrently
+			u.Concurrency = 10
+
+			// Leave parts on failure for debugging
+			u.LeavePartsOnError = false
+		})
+
+		// Wrap file with progress reader
+		var reader io.Reader = file
+		if progress != nil {
+			reader = NewProgressReader(file, fileSize, progress)
+		}
+
+		// Upload with multipart
+		_, err := uploader.Upload(ctx, &s3.PutObjectInput{
+			Bucket: aws.String(s.bucket),
+			Key:    aws.String(key),
+			Body:   reader,
+		})
+
+		if err != nil {
+			return fmt.Errorf("multipart upload failed: %w", err)
+		}
+
+		return nil
+	}, func(err error, duration time.Duration) {
+		fmt.Printf("[S3] Multipart upload retry in %v: %v\n", duration, err)
 	})
-
-	// Wrap file with progress reader
-	var reader io.Reader = file
-	if progress != nil {
-		reader = NewProgressReader(file, fileSize, progress)
-	}
-
-	// Upload with multipart
-	_, err := uploader.Upload(ctx, &s3.PutObjectInput{
-		Bucket: aws.String(s.bucket),
-		Key:    aws.String(key),
-		Body:   reader,
-	})
-
-	if err != nil {
-		return fmt.Errorf("multipart upload failed: %w", err)
-	}
-
-	return nil
 }

-// Download downloads a file from S3
+// Download downloads a file from S3 with retry
 func (s *S3Backend) Download(ctx context.Context, remotePath, localPath string, progress ProgressCallback) error {
 	// Build S3 key
 	key := s.buildKey(remotePath)
@@ -190,39 +209,44 @@ func (s *S3Backend) Download(ctx context.Context, remotePath, localPath string,
 		return fmt.Errorf("failed to get object size: %w", err)
 	}

-	// Download from S3
-	result, err := s.client.GetObject(ctx, &s3.GetObjectInput{
-		Bucket: aws.String(s.bucket),
-		Key:    aws.String(key),
-	})
-	if err != nil {
-		return fmt.Errorf("failed to download from S3: %w", err)
-	}
-	defer result.Body.Close()
-
-	// Create local file
+	// Create directory for local file
 	if err := os.MkdirAll(filepath.Dir(localPath), 0755); err != nil {
 		return fmt.Errorf("failed to create directory: %w", err)
 	}

-	outFile, err := os.Create(localPath)
-	if err != nil {
-		return fmt.Errorf("failed to create local file: %w", err)
-	}
-	defer outFile.Close()
+	return RetryOperationWithNotify(ctx, DefaultRetryConfig(), func() error {
+		// Download from S3
+		result, err := s.client.GetObject(ctx, &s3.GetObjectInput{
+			Bucket: aws.String(s.bucket),
+			Key:    aws.String(key),
+		})
+		if err != nil {
+			return fmt.Errorf("failed to download from S3: %w", err)
+		}
+		defer result.Body.Close()

-	// Copy with progress tracking
-	var reader io.Reader = result.Body
-	if progress != nil {
-		reader = NewProgressReader(result.Body, size, progress)
-	}
+		// Create/truncate local file
+		outFile, err := os.Create(localPath)
+		if err != nil {
+			return fmt.Errorf("failed to create local file: %w", err)
+		}
+		defer outFile.Close()

-	_, err = io.Copy(outFile, reader)
-	if err != nil {
-		return fmt.Errorf("failed to write file: %w", err)
-	}
+		// Copy with progress tracking
+		var reader io.Reader = result.Body
+		if progress != nil {
+			reader = NewProgressReader(result.Body, size, progress)
+		}

-	return nil
+		_, err = io.Copy(outFile, reader)
+		if err != nil {
+			return fmt.Errorf("failed to write file: %w", err)
+		}
+
+		return nil
+	}, func(err error, duration time.Duration) {
+		fmt.Printf("[S3] Download retry in %v: %v\n", duration, err)
+	})
 }

 // List lists all backup files in S3
--- a/internal/logger/colors.go
+++ b/internal/logger/colors.go
@@ -0,0 +1,118 @@
+package logger
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/fatih/color"
+)
+
+// CLI output helpers using fatih/color for cross-platform support
+
+// Success prints a success message with green checkmark
+func Success(format string, args ...interface{}) {
+	msg := fmt.Sprintf(format, args...)
+	SuccessColor.Fprint(os.Stdout, "✓ ")
+	fmt.Println(msg)
+}
+
+// Error prints an error message with red X
+func Error(format string, args ...interface{}) {
+	msg := fmt.Sprintf(format, args...)
+	ErrorColor.Fprint(os.Stderr, "✗ ")
+	fmt.Fprintln(os.Stderr, msg)
+}
+
+// Warning prints a warning message with yellow exclamation
+func Warning(format string, args ...interface{}) {
+	msg := fmt.Sprintf(format, args...)
+	WarnColor.Fprint(os.Stdout, "⚠ ")
+	fmt.Println(msg)
+}
+
+// Info prints an info message with blue arrow
+func Info(format string, args ...interface{}) {
+	msg := fmt.Sprintf(format, args...)
+	InfoColor.Fprint(os.Stdout, "→ ")
+	fmt.Println(msg)
+}
+
+// Header prints a bold header
+func Header(format string, args ...interface{}) {
+	msg := fmt.Sprintf(format, args...)
+	HighlightColor.Println(msg)
+}
+
+// Dim prints dimmed/secondary text
+func Dim(format string, args ...interface{}) {
+	msg := fmt.Sprintf(format, args...)
+	DimColor.Println(msg)
+}
+
+// Bold returns bold text
+func Bold(text string) string {
+	return color.New(color.Bold).Sprint(text)
+}
+
+// Green returns green text
+func Green(text string) string {
+	return SuccessColor.Sprint(text)
+}
+
+// Red returns red text
+func Red(text string) string {
+	return ErrorColor.Sprint(text)
+}
+
+// Yellow returns yellow text
+func Yellow(text string) string {
+	return WarnColor.Sprint(text)
+}
+
+// Cyan returns cyan text
+func Cyan(text string) string {
+	return InfoColor.Sprint(text)
+}
+
+// StatusLine prints a key-value status line
+func StatusLine(key, value string) {
+	DimColor.Printf("  %s: ", key)
+	fmt.Println(value)
+}
+
+// ProgressStatus prints operation status with timing
+func ProgressStatus(operation string, status string, isSuccess bool) {
+	if isSuccess {
+		SuccessColor.Print("[OK] ")
+	} else {
+		ErrorColor.Print("[FAIL] ")
+	}
+	fmt.Printf("%s: %s\n", operation, status)
+}
+
+// Table prints a simple formatted table row
+func TableRow(cols ...string) {
+	for i, col := range cols {
+		if i == 0 {
+			InfoColor.Printf("%-20s", col)
+		} else {
+			fmt.Printf("%-15s", col)
+		}
+	}
+	fmt.Println()
+}
+
+// DisableColors disables all color output (for non-TTY or --no-color flag)
+func DisableColors() {
+	color.NoColor = true
+}
+
+// EnableColors enables color output
+func EnableColors() {
+	color.NoColor = false
+}
+
+// IsColorEnabled returns whether colors are enabled
+func IsColorEnabled() bool {
+	return !color.NoColor
+}
--- a/internal/logger/logger.go
+++ b/internal/logger/logger.go
@@ -7,9 +7,29 @@ import (
 	"strings"
 	"time"

+	"github.com/fatih/color"
 	"github.com/sirupsen/logrus"
 )

+// Color printers for consistent output across the application
+var (
+	// Status colors
+	SuccessColor = color.New(color.FgGreen, color.Bold)
+	ErrorColor   = color.New(color.FgRed, color.Bold)
+	WarnColor    = color.New(color.FgYellow, color.Bold)
+	InfoColor    = color.New(color.FgCyan)
+	DebugColor   = color.New(color.FgWhite)
+
+	// Highlight colors
+	HighlightColor = color.New(color.FgMagenta, color.Bold)
+	DimColor       = color.New(color.FgHiBlack)
+
+	// Data colors
+	NumberColor = color.New(color.FgYellow)
+	PathColor   = color.New(color.FgBlue, color.Underline)
+	TimeColor   = color.New(color.FgCyan)
+)
+
 // Logger defines the interface for logging
 type Logger interface {
 	Debug(msg string, args ...any)
@@ -226,34 +246,32 @@ type CleanFormatter struct{}
 func (f *CleanFormatter) Format(entry *logrus.Entry) ([]byte, error) {
 	timestamp := entry.Time.Format("2006-01-02T15:04:05")

-	// Color codes for different log levels
-	var levelColor, levelText string
+	// Get level color and text using fatih/color
+	var levelPrinter *color.Color
+	var levelText string
 	switch entry.Level {
 	case logrus.DebugLevel:
-		levelColor = "\033[36m" // Cyan
+		levelPrinter = DebugColor
 		levelText = "DEBUG"
 	case logrus.InfoLevel:
-		levelColor = "\033[32m" // Green
+		levelPrinter = SuccessColor
 		levelText = "INFO "
 	case logrus.WarnLevel:
-		levelColor = "\033[33m" // Yellow
+		levelPrinter = WarnColor
 		levelText = "WARN "
 	case logrus.ErrorLevel:
-		levelColor = "\033[31m" // Red
+		levelPrinter = ErrorColor
 		levelText = "ERROR"
 	default:
-		levelColor = "\033[0m" // Reset
+		levelPrinter = InfoColor
 		levelText = "INFO "
 	}
-	resetColor := "\033[0m"

 	// Build the message with perfectly aligned columns
 	var output strings.Builder

 	// Column 1: Level (with color, fixed width 5 chars)
-	output.WriteString(levelColor)
-	output.WriteString(levelText)
-	output.WriteString(resetColor)
+	output.WriteString(levelPrinter.Sprint(levelText))
 	output.WriteString(" ")

 	// Column 2: Timestamp (fixed format)
--- a/internal/progress/progress.go
+++ b/internal/progress/progress.go
@@ -7,9 +7,17 @@ import (
 	"strings"
 	"time"

+	"github.com/fatih/color"
 	"github.com/schollz/progressbar/v3"
 )

+// Color printers for progress indicators
+var (
+	okColor   = color.New(color.FgGreen, color.Bold)
+	failColor = color.New(color.FgRed, color.Bold)
+	warnColor = color.New(color.FgYellow, color.Bold)
+)
+
 // Indicator represents a progress indicator interface
 type Indicator interface {
 	Start(message string)
@@ -94,13 +102,15 @@ func (s *Spinner) Update(message string) {
 // Complete stops the spinner with a success message
 func (s *Spinner) Complete(message string) {
 	s.Stop()
-	fmt.Fprintf(s.writer, "\n[OK] %s\n", message)
+	okColor.Fprint(s.writer, "[OK] ")
+	fmt.Fprintln(s.writer, message)
 }

 // Fail stops the spinner with a failure message
 func (s *Spinner) Fail(message string) {
 	s.Stop()
-	fmt.Fprintf(s.writer, "\n[FAIL] %s\n", message)
+	failColor.Fprint(s.writer, "[FAIL] ")
+	fmt.Fprintln(s.writer, message)
 }

 // Stop stops the spinner
@@ -169,13 +179,15 @@ func (d *Dots) Update(message string) {
 // Complete stops the dots with a success message
 func (d *Dots) Complete(message string) {
 	d.Stop()
-	fmt.Fprintf(d.writer, " [OK] %s\n", message)
+	okColor.Fprint(d.writer, " [OK] ")
+	fmt.Fprintln(d.writer, message)
 }

 // Fail stops the dots with a failure message
 func (d *Dots) Fail(message string) {
 	d.Stop()
-	fmt.Fprintf(d.writer, " [FAIL] %s\n", message)
+	failColor.Fprint(d.writer, " [FAIL] ")
+	fmt.Fprintln(d.writer, message)
 }

 // Stop stops the dots indicator
@@ -241,14 +253,16 @@ func (p *ProgressBar) Complete(message string) {
 	p.current = p.total
 	p.message = message
 	p.render()
-	fmt.Fprintf(p.writer, " [OK] %s\n", message)
+	okColor.Fprint(p.writer, " [OK] ")
+	fmt.Fprintln(p.writer, message)
 	p.Stop()
 }

 // Fail stops the progress bar with failure
 func (p *ProgressBar) Fail(message string) {
 	p.render()
-	fmt.Fprintf(p.writer, " [FAIL] %s\n", message)
+	failColor.Fprint(p.writer, " [FAIL] ")
+	fmt.Fprintln(p.writer, message)
 	p.Stop()
 }

@@ -300,12 +314,14 @@ func (s *Static) Update(message string) {

 // Complete shows completion message
 func (s *Static) Complete(message string) {
-	fmt.Fprintf(s.writer, " [OK] %s\n", message)
+	okColor.Fprint(s.writer, " [OK] ")
+	fmt.Fprintln(s.writer, message)
 }

 // Fail shows failure message
 func (s *Static) Fail(message string) {
-	fmt.Fprintf(s.writer, " [FAIL] %s\n", message)
+	failColor.Fprint(s.writer, " [FAIL] ")
+	fmt.Fprintln(s.writer, message)
 }

 // Stop does nothing for static indicator
@@ -382,12 +398,14 @@ func (l *LineByLine) SetEstimator(estimator *ETAEstimator) {

 // Complete shows completion message
 func (l *LineByLine) Complete(message string) {
-	fmt.Fprintf(l.writer, "[OK] %s\n\n", message)
+	okColor.Fprint(l.writer, "[OK] ")
+	fmt.Fprintf(l.writer, "%s\n\n", message)
 }

 // Fail shows failure message
 func (l *LineByLine) Fail(message string) {
-	fmt.Fprintf(l.writer, "[FAIL] %s\n\n", message)
+	failColor.Fprint(l.writer, "[FAIL] ")
+	fmt.Fprintf(l.writer, "%s\n\n", message)
 }

 // Stop does nothing for line-by-line (no cleanup needed)
@@ -410,13 +428,15 @@ func (l *Light) Update(message string) {

 func (l *Light) Complete(message string) {
 	if !l.silent {
-		fmt.Fprintf(l.writer, "[OK] %s\n", message)
+		okColor.Fprint(l.writer, "[OK] ")
+		fmt.Fprintln(l.writer, message)
 	}
 }

 func (l *Light) Fail(message string) {
 	if !l.silent {
-		fmt.Fprintf(l.writer, "[FAIL] %s\n", message)
+		failColor.Fprint(l.writer, "[FAIL] ")
+		fmt.Fprintln(l.writer, message)
 	}
 }

@@ -594,13 +614,15 @@ func (s *SchollzBar) ChangeMax64(max int64) {
 // Complete finishes with success (Indicator interface)
 func (s *SchollzBar) Complete(message string) {
 	_ = s.bar.Finish()
-	fmt.Printf("\n[green][OK][reset] %s\n", message)
+	okColor.Print("[OK] ")
+	fmt.Println(message)
 }

 // Fail finishes with failure (Indicator interface)
 func (s *SchollzBar) Fail(message string) {
 	_ = s.bar.Clear()
-	fmt.Printf("\n[red][FAIL][reset] %s\n", message)
+	failColor.Print("[FAIL] ")
+	fmt.Println(message)
 }

 // Stop stops the progress bar (Indicator interface)
--- a/main.go
+++ b/main.go
@@ -16,7 +16,7 @@ import (

 // Build information (set by ldflags)
 var (
-	version   = "3.42.31"
+	version   = "3.42.33"
 	buildTime = "unknown"
 	gitCommit = "unknown"
 )