Compare commits

..

3 Commits

Author SHA1 Message Date
510175ff04 TUI: Enhance completion/result screens for backup and restore
All checks were successful
CI/CD / Test (push) Successful in 1m17s
CI/CD / Lint (push) Successful in 1m28s
CI/CD / Build & Release (push) Successful in 3m12s
- Add box-style headers for success/failure states
- Display comprehensive summary with archive info, type, database count
- Show timing section with total time, throughput, and average per-DB stats
- Use consistent styling and formatting across all result views
- Improve visual hierarchy with section separators
2026-01-16 13:37:58 +01:00
a85ad0c88c TUI: Add timing and ETA tracking to cluster restore progress
All checks were successful
CI/CD / Test (push) Successful in 1m17s
CI/CD / Lint (push) Successful in 1m28s
CI/CD / Build & Release (push) Successful in 3m21s
- Add DatabaseProgressWithTimingCallback for timing-aware progress reporting
- Track elapsed time and average duration per database during restore phase
- Display ETA based on completed database restore times
- Show restore phase elapsed time in progress bar
- Enhance cluster restore progress bar with [elapsed / ETA: remaining] format
2026-01-16 09:42:05 +01:00
4938dc1918 fix: max_locks_per_transaction requires PostgreSQL restart
All checks were successful
CI/CD / Test (push) Successful in 1m19s
CI/CD / Lint (push) Successful in 1m31s
CI/CD / Build & Release (push) Successful in 3m34s
- Fixed critical bug where ALTER SYSTEM + pg_reload_conf() was used
  but max_locks_per_transaction requires a full PostgreSQL restart
- Added automatic restart attempt (systemctl, service, pg_ctl)
- Added loud warnings if restart fails with manual fix instructions
- Updated preflight checks to warn about low max_locks_per_transaction
- This was causing 'out of shared memory' errors on BLOB-heavy restores
2026-01-15 18:50:10 +01:00
5 changed files with 367 additions and 40 deletions

View File

@@ -4,8 +4,8 @@ This directory contains pre-compiled binaries for the DB Backup Tool across mult
## Build Information ## Build Information
- **Version**: 3.42.34 - **Version**: 3.42.34
- **Build Time**: 2026-01-15_14:16:33_UTC - **Build Time**: 2026-01-16_08:42:47_UTC
- **Git Commit**: eeacbfa - **Git Commit**: a85ad0c
## Recent Updates (v1.1.0) ## Recent Updates (v1.1.0)
- ✅ Fixed TUI progress display with line-by-line output - ✅ Fixed TUI progress display with line-by-line output

View File

@@ -34,6 +34,10 @@ type ProgressCallback func(current, total int64, description string)
// DatabaseProgressCallback is called with database count progress during cluster restore // DatabaseProgressCallback is called with database count progress during cluster restore
type DatabaseProgressCallback func(done, total int, dbName string) type DatabaseProgressCallback func(done, total int, dbName string)
// DatabaseProgressWithTimingCallback is called with database progress including timing info
// Parameters: done count, total count, database name, elapsed time for current restore phase, avg duration per DB
type DatabaseProgressWithTimingCallback func(done, total int, dbName string, phaseElapsed, avgPerDB time.Duration)
// Engine handles database restore operations // Engine handles database restore operations
type Engine struct { type Engine struct {
cfg *config.Config cfg *config.Config
@@ -45,8 +49,9 @@ type Engine struct {
debugLogPath string // Path to save debug log on error debugLogPath string // Path to save debug log on error
// TUI progress callback for detailed progress reporting // TUI progress callback for detailed progress reporting
progressCallback ProgressCallback progressCallback ProgressCallback
dbProgressCallback DatabaseProgressCallback dbProgressCallback DatabaseProgressCallback
dbProgressTimingCallback DatabaseProgressWithTimingCallback
} }
// New creates a new restore engine // New creates a new restore engine
@@ -112,6 +117,11 @@ func (e *Engine) SetDatabaseProgressCallback(cb DatabaseProgressCallback) {
e.dbProgressCallback = cb e.dbProgressCallback = cb
} }
// SetDatabaseProgressWithTimingCallback sets a callback for database progress with timing info
func (e *Engine) SetDatabaseProgressWithTimingCallback(cb DatabaseProgressWithTimingCallback) {
e.dbProgressTimingCallback = cb
}
// reportProgress safely calls the progress callback if set // reportProgress safely calls the progress callback if set
func (e *Engine) reportProgress(current, total int64, description string) { func (e *Engine) reportProgress(current, total int64, description string) {
if e.progressCallback != nil { if e.progressCallback != nil {
@@ -126,6 +136,13 @@ func (e *Engine) reportDatabaseProgress(done, total int, dbName string) {
} }
} }
// reportDatabaseProgressWithTiming safely calls the timing-aware callback if set
func (e *Engine) reportDatabaseProgressWithTiming(done, total int, dbName string, phaseElapsed, avgPerDB time.Duration) {
if e.dbProgressTimingCallback != nil {
e.dbProgressTimingCallback(done, total, dbName, phaseElapsed, avgPerDB)
}
}
// loggerAdapter adapts our logger to the progress.Logger interface // loggerAdapter adapts our logger to the progress.Logger interface
type loggerAdapter struct { type loggerAdapter struct {
logger logger.Logger logger logger.Logger
@@ -1037,6 +1054,11 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
var successCount, failCount int32 var successCount, failCount int32
var mu sync.Mutex // Protect shared resources (progress, logger) var mu sync.Mutex // Protect shared resources (progress, logger)
// Timing tracking for restore phase progress
restorePhaseStart := time.Now()
var completedDBTimes []time.Duration // Track duration for each completed DB restore
var completedDBTimesMu sync.Mutex
// Create semaphore to limit concurrency // Create semaphore to limit concurrency
semaphore := make(chan struct{}, parallelism) semaphore := make(chan struct{}, parallelism)
var wg sync.WaitGroup var wg sync.WaitGroup
@@ -1062,6 +1084,9 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
} }
}() }()
// Track timing for this database restore
dbRestoreStart := time.Now()
// Update estimator progress (thread-safe) // Update estimator progress (thread-safe)
mu.Lock() mu.Lock()
estimator.UpdateProgress(idx) estimator.UpdateProgress(idx)
@@ -1074,12 +1099,26 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
dbProgress := 15 + int(float64(idx)/float64(totalDBs)*85.0) dbProgress := 15 + int(float64(idx)/float64(totalDBs)*85.0)
// Calculate average time per DB and report progress with timing
completedDBTimesMu.Lock()
var avgPerDB time.Duration
if len(completedDBTimes) > 0 {
var totalDuration time.Duration
for _, d := range completedDBTimes {
totalDuration += d
}
avgPerDB = totalDuration / time.Duration(len(completedDBTimes))
}
phaseElapsed := time.Since(restorePhaseStart)
completedDBTimesMu.Unlock()
mu.Lock() mu.Lock()
statusMsg := fmt.Sprintf("Restoring database %s (%d/%d)", dbName, idx+1, totalDBs) statusMsg := fmt.Sprintf("Restoring database %s (%d/%d)", dbName, idx+1, totalDBs)
e.progress.Update(statusMsg) e.progress.Update(statusMsg)
e.log.Info("Restoring database", "name", dbName, "file", dumpFile, "progress", dbProgress) e.log.Info("Restoring database", "name", dbName, "file", dumpFile, "progress", dbProgress)
// Report database progress for TUI // Report database progress for TUI (both callbacks)
e.reportDatabaseProgress(idx, totalDBs, dbName) e.reportDatabaseProgress(idx, totalDBs, dbName)
e.reportDatabaseProgressWithTiming(idx, totalDBs, dbName, phaseElapsed, avgPerDB)
mu.Unlock() mu.Unlock()
// STEP 1: Drop existing database completely (clean slate) // STEP 1: Drop existing database completely (clean slate)
@@ -1144,6 +1183,12 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
return return
} }
// Track completed database restore duration for ETA calculation
dbRestoreDuration := time.Since(dbRestoreStart)
completedDBTimesMu.Lock()
completedDBTimes = append(completedDBTimes, dbRestoreDuration)
completedDBTimesMu.Unlock()
atomic.AddInt32(&successCount, 1) atomic.AddInt32(&successCount, 1)
}(dbIndex, entry.Name()) }(dbIndex, entry.Name())
@@ -1937,6 +1982,8 @@ type OriginalSettings struct {
} }
// boostPostgreSQLSettings boosts multiple PostgreSQL settings for large restores // boostPostgreSQLSettings boosts multiple PostgreSQL settings for large restores
// NOTE: max_locks_per_transaction requires a PostgreSQL RESTART to take effect!
// maintenance_work_mem can be changed with pg_reload_conf().
func (e *Engine) boostPostgreSQLSettings(ctx context.Context, lockBoostValue int) (*OriginalSettings, error) { func (e *Engine) boostPostgreSQLSettings(ctx context.Context, lockBoostValue int) (*OriginalSettings, error) {
connStr := e.buildConnString() connStr := e.buildConnString()
db, err := sql.Open("pgx", connStr) db, err := sql.Open("pgx", connStr)
@@ -1956,30 +2003,113 @@ func (e *Engine) boostPostgreSQLSettings(ctx context.Context, lockBoostValue int
// Get current maintenance_work_mem // Get current maintenance_work_mem
db.QueryRowContext(ctx, "SHOW maintenance_work_mem").Scan(&original.MaintenanceWorkMem) db.QueryRowContext(ctx, "SHOW maintenance_work_mem").Scan(&original.MaintenanceWorkMem)
// Boost max_locks_per_transaction (if not already high enough) // CRITICAL: max_locks_per_transaction requires a PostgreSQL RESTART!
// pg_reload_conf() is NOT sufficient for this parameter.
needsRestart := false
if original.MaxLocks < lockBoostValue { if original.MaxLocks < lockBoostValue {
_, err = db.ExecContext(ctx, fmt.Sprintf("ALTER SYSTEM SET max_locks_per_transaction = %d", lockBoostValue)) _, err = db.ExecContext(ctx, fmt.Sprintf("ALTER SYSTEM SET max_locks_per_transaction = %d", lockBoostValue))
if err != nil { if err != nil {
e.log.Warn("Could not boost max_locks_per_transaction", "error", err) e.log.Warn("Could not set max_locks_per_transaction", "error", err)
} else {
needsRestart = true
e.log.Warn("max_locks_per_transaction requires PostgreSQL restart to take effect",
"current", original.MaxLocks,
"target", lockBoostValue)
} }
} }
// Boost maintenance_work_mem to 2GB for faster index creation // Boost maintenance_work_mem to 2GB for faster index creation
// (this one CAN be applied via pg_reload_conf)
_, err = db.ExecContext(ctx, "ALTER SYSTEM SET maintenance_work_mem = '2GB'") _, err = db.ExecContext(ctx, "ALTER SYSTEM SET maintenance_work_mem = '2GB'")
if err != nil { if err != nil {
e.log.Warn("Could not boost maintenance_work_mem", "error", err) e.log.Warn("Could not boost maintenance_work_mem", "error", err)
} }
// Reload config to apply changes (no restart needed for these settings) // Reload config to apply maintenance_work_mem
_, err = db.ExecContext(ctx, "SELECT pg_reload_conf()") _, err = db.ExecContext(ctx, "SELECT pg_reload_conf()")
if err != nil { if err != nil {
return original, fmt.Errorf("failed to reload config: %w", err) return original, fmt.Errorf("failed to reload config: %w", err)
} }
// If max_locks_per_transaction needs a restart, try to do it
if needsRestart {
if restarted := e.tryRestartPostgreSQL(ctx); restarted {
e.log.Info("PostgreSQL restarted successfully - max_locks_per_transaction now active")
// Wait for PostgreSQL to be ready
time.Sleep(3 * time.Second)
} else {
// Cannot restart - warn user loudly
e.log.Error("=" + strings.Repeat("=", 70))
e.log.Error("WARNING: max_locks_per_transaction change requires PostgreSQL restart!")
e.log.Error("Current value: " + strconv.Itoa(original.MaxLocks) + ", needed: " + strconv.Itoa(lockBoostValue))
e.log.Error("Restore may fail with 'out of shared memory' error on BLOB-heavy databases.")
e.log.Error("")
e.log.Error("To fix manually:")
e.log.Error(" 1. sudo systemctl restart postgresql")
e.log.Error(" 2. Or: sudo -u postgres pg_ctl restart -D $PGDATA")
e.log.Error(" 3. Then re-run the restore")
e.log.Error("=" + strings.Repeat("=", 70))
// Continue anyway - might work for small restores
}
}
return original, nil return original, nil
} }
// tryRestartPostgreSQL attempts to restart PostgreSQL using various methods
// Returns true if restart was successful
func (e *Engine) tryRestartPostgreSQL(ctx context.Context) bool {
e.progress.Update("Attempting PostgreSQL restart for lock settings...")
// Method 1: systemctl (most common on modern Linux)
cmd := exec.CommandContext(ctx, "sudo", "systemctl", "restart", "postgresql")
if err := cmd.Run(); err == nil {
return true
}
// Method 2: systemctl with version suffix (e.g., postgresql-15)
for _, ver := range []string{"17", "16", "15", "14", "13", "12"} {
cmd = exec.CommandContext(ctx, "sudo", "systemctl", "restart", "postgresql-"+ver)
if err := cmd.Run(); err == nil {
return true
}
}
// Method 3: service command (older systems)
cmd = exec.CommandContext(ctx, "sudo", "service", "postgresql", "restart")
if err := cmd.Run(); err == nil {
return true
}
// Method 4: pg_ctl as postgres user
cmd = exec.CommandContext(ctx, "sudo", "-u", "postgres", "pg_ctl", "restart", "-D", "/var/lib/postgresql/data", "-m", "fast")
if err := cmd.Run(); err == nil {
return true
}
// Method 5: Try common PGDATA paths
pgdataPaths := []string{
"/var/lib/pgsql/data",
"/var/lib/pgsql/17/data",
"/var/lib/pgsql/16/data",
"/var/lib/pgsql/15/data",
"/var/lib/postgresql/17/main",
"/var/lib/postgresql/16/main",
"/var/lib/postgresql/15/main",
}
for _, pgdata := range pgdataPaths {
cmd = exec.CommandContext(ctx, "sudo", "-u", "postgres", "pg_ctl", "restart", "-D", pgdata, "-m", "fast")
if err := cmd.Run(); err == nil {
return true
}
}
return false
}
// resetPostgreSQLSettings restores original PostgreSQL settings // resetPostgreSQLSettings restores original PostgreSQL settings
// NOTE: max_locks_per_transaction changes are written but require restart to take effect.
// We don't restart here since we're done with the restore.
func (e *Engine) resetPostgreSQLSettings(ctx context.Context, original *OriginalSettings) error { func (e *Engine) resetPostgreSQLSettings(ctx context.Context, original *OriginalSettings) error {
connStr := e.buildConnString() connStr := e.buildConnString()
db, err := sql.Open("pgx", connStr) db, err := sql.Open("pgx", connStr)
@@ -1988,25 +2118,28 @@ func (e *Engine) resetPostgreSQLSettings(ctx context.Context, original *Original
} }
defer db.Close() defer db.Close()
// Reset max_locks_per_transaction // Reset max_locks_per_transaction (will take effect on next restart)
if original.MaxLocks == 64 { // Default if original.MaxLocks == 64 { // Default
db.ExecContext(ctx, "ALTER SYSTEM RESET max_locks_per_transaction") db.ExecContext(ctx, "ALTER SYSTEM RESET max_locks_per_transaction")
} else if original.MaxLocks > 0 { } else if original.MaxLocks > 0 {
db.ExecContext(ctx, fmt.Sprintf("ALTER SYSTEM SET max_locks_per_transaction = %d", original.MaxLocks)) db.ExecContext(ctx, fmt.Sprintf("ALTER SYSTEM SET max_locks_per_transaction = %d", original.MaxLocks))
} }
// Reset maintenance_work_mem // Reset maintenance_work_mem (takes effect immediately with reload)
if original.MaintenanceWorkMem == "64MB" { // Default if original.MaintenanceWorkMem == "64MB" { // Default
db.ExecContext(ctx, "ALTER SYSTEM RESET maintenance_work_mem") db.ExecContext(ctx, "ALTER SYSTEM RESET maintenance_work_mem")
} else if original.MaintenanceWorkMem != "" { } else if original.MaintenanceWorkMem != "" {
db.ExecContext(ctx, fmt.Sprintf("ALTER SYSTEM SET maintenance_work_mem = '%s'", original.MaintenanceWorkMem)) db.ExecContext(ctx, fmt.Sprintf("ALTER SYSTEM SET maintenance_work_mem = '%s'", original.MaintenanceWorkMem))
} }
// Reload config // Reload config (only maintenance_work_mem will take effect immediately)
_, err = db.ExecContext(ctx, "SELECT pg_reload_conf()") _, err = db.ExecContext(ctx, "SELECT pg_reload_conf()")
if err != nil { if err != nil {
return fmt.Errorf("failed to reload config: %w", err) return fmt.Errorf("failed to reload config: %w", err)
} }
e.log.Info("PostgreSQL settings reset queued",
"note", "max_locks_per_transaction will revert on next PostgreSQL restart")
return nil return nil
} }

View File

@@ -201,10 +201,19 @@ func (e *Engine) checkPostgreSQL(ctx context.Context, result *PreflightResult) {
result.PostgreSQL.IsSuperuser = isSuperuser result.PostgreSQL.IsSuperuser = isSuperuser
} }
// Add info/warnings // CRITICAL: max_locks_per_transaction requires PostgreSQL RESTART to change!
// Warn users loudly about this - it's the #1 cause of "out of shared memory" errors
if result.PostgreSQL.MaxLocksPerTransaction < 256 { if result.PostgreSQL.MaxLocksPerTransaction < 256 {
e.log.Info("PostgreSQL max_locks_per_transaction is low - will auto-boost", e.log.Warn("PostgreSQL max_locks_per_transaction is LOW",
"current", result.PostgreSQL.MaxLocksPerTransaction) "current", result.PostgreSQL.MaxLocksPerTransaction,
"recommended", "256+",
"note", "REQUIRES PostgreSQL restart to change!")
result.Warnings = append(result.Warnings,
fmt.Sprintf("max_locks_per_transaction=%d is low (recommend 256+). "+
"This setting requires PostgreSQL RESTART to change. "+
"BLOB-heavy databases may fail with 'out of shared memory' error.",
result.PostgreSQL.MaxLocksPerTransaction))
} }
// Parse shared_buffers and warn if very low // Parse shared_buffers and warn if very low

View File

@@ -380,22 +380,83 @@ func (m BackupExecutionModel) View() string {
s.WriteString("\n [KEY] Press Ctrl+C or ESC to cancel\n") s.WriteString("\n [KEY] Press Ctrl+C or ESC to cancel\n")
} }
} else { } else {
s.WriteString(fmt.Sprintf(" %s\n\n", m.status)) // Show completion summary with detailed stats
if m.err != nil { if m.err != nil {
s.WriteString(fmt.Sprintf(" [FAIL] Error: %v\n", m.err)) s.WriteString("\n")
} else if m.result != "" { s.WriteString(errorStyle.Render(" ╔══════════════════════════════════════════════════════════╗"))
// Parse and display result cleanly s.WriteString("\n")
lines := strings.Split(m.result, "\n") s.WriteString(errorStyle.Render(" ║ [FAIL] BACKUP FAILED ║"))
for _, line := range lines { s.WriteString("\n")
line = strings.TrimSpace(line) s.WriteString(errorStyle.Render(" ╚══════════════════════════════════════════════════════════╝"))
if line != "" { s.WriteString("\n\n")
s.WriteString(" " + line + "\n") s.WriteString(errorStyle.Render(fmt.Sprintf(" Error: %v", m.err)))
s.WriteString("\n")
} else {
s.WriteString("\n")
s.WriteString(successStyle.Render(" ╔══════════════════════════════════════════════════════════╗"))
s.WriteString("\n")
s.WriteString(successStyle.Render(" ║ [OK] BACKUP COMPLETED SUCCESSFULLY ║"))
s.WriteString("\n")
s.WriteString(successStyle.Render(" ╚══════════════════════════════════════════════════════════╝"))
s.WriteString("\n\n")
// Summary section
s.WriteString(infoStyle.Render(" ─── Summary ─────────────────────────────────────────────"))
s.WriteString("\n\n")
// Backup type specific info
switch m.backupType {
case "cluster":
s.WriteString(" Type: Cluster Backup\n")
if m.dbTotal > 0 {
s.WriteString(fmt.Sprintf(" Databases: %d backed up\n", m.dbTotal))
} }
case "single":
s.WriteString(" Type: Single Database Backup\n")
s.WriteString(fmt.Sprintf(" Database: %s\n", m.databaseName))
case "sample":
s.WriteString(" Type: Sample Backup\n")
s.WriteString(fmt.Sprintf(" Database: %s\n", m.databaseName))
s.WriteString(fmt.Sprintf(" Sample Ratio: %d\n", m.ratio))
} }
s.WriteString("\n")
// Timing section
s.WriteString(infoStyle.Render(" ─── Timing ──────────────────────────────────────────────"))
s.WriteString("\n\n")
elapsed := time.Since(m.startTime)
s.WriteString(fmt.Sprintf(" Total Time: %s\n", formatBackupDuration(elapsed)))
if m.backupType == "cluster" && m.dbTotal > 0 {
avgPerDB := elapsed / time.Duration(m.dbTotal)
s.WriteString(fmt.Sprintf(" Avg per DB: %s\n", formatBackupDuration(avgPerDB)))
}
s.WriteString("\n")
s.WriteString(infoStyle.Render(" ─────────────────────────────────────────────────────────"))
s.WriteString("\n")
} }
s.WriteString("\n [KEY] Press Enter or ESC to return to menu\n")
s.WriteString("\n")
s.WriteString(" [KEY] Press Enter or ESC to return to menu\n")
} }
return s.String() return s.String()
} }
// formatBackupDuration formats duration in human readable format
func formatBackupDuration(d time.Duration) string {
if d < time.Minute {
return fmt.Sprintf("%.1fs", d.Seconds())
}
if d < time.Hour {
minutes := int(d.Minutes())
seconds := int(d.Seconds()) % 60
return fmt.Sprintf("%dm %ds", minutes, seconds)
}
hours := int(d.Hours())
minutes := int(d.Minutes()) % 60
return fmt.Sprintf("%dh %dm", hours, minutes)
}

View File

@@ -57,6 +57,10 @@ type RestoreExecutionModel struct {
dbTotal int dbTotal int
dbDone int dbDone int
// Timing info for database restore phase (ETA calculation)
dbPhaseElapsed time.Duration // Elapsed time since restore phase started
dbAvgPerDB time.Duration // Average time per database restore
// Results // Results
done bool done bool
cancelling bool // True when user has requested cancellation cancelling bool // True when user has requested cancellation
@@ -136,6 +140,10 @@ type sharedProgressState struct {
dbTotal int dbTotal int
dbDone int dbDone int
// Timing info for database restore phase
dbPhaseElapsed time.Duration // Elapsed time since restore phase started
dbAvgPerDB time.Duration // Average time per database restore
// Rolling window for speed calculation // Rolling window for speed calculation
speedSamples []restoreSpeedSample speedSamples []restoreSpeedSample
} }
@@ -163,12 +171,12 @@ func clearCurrentRestoreProgress() {
currentRestoreProgressState = nil currentRestoreProgressState = nil
} }
func getCurrentRestoreProgress() (bytesTotal, bytesDone int64, description string, hasUpdate bool, dbTotal, dbDone int, speed float64) { func getCurrentRestoreProgress() (bytesTotal, bytesDone int64, description string, hasUpdate bool, dbTotal, dbDone int, speed float64, dbPhaseElapsed, dbAvgPerDB time.Duration) {
currentRestoreProgressMu.Lock() currentRestoreProgressMu.Lock()
defer currentRestoreProgressMu.Unlock() defer currentRestoreProgressMu.Unlock()
if currentRestoreProgressState == nil { if currentRestoreProgressState == nil {
return 0, 0, "", false, 0, 0, 0 return 0, 0, "", false, 0, 0, 0, 0, 0
} }
currentRestoreProgressState.mu.Lock() currentRestoreProgressState.mu.Lock()
@@ -179,7 +187,8 @@ func getCurrentRestoreProgress() (bytesTotal, bytesDone int64, description strin
return currentRestoreProgressState.bytesTotal, currentRestoreProgressState.bytesDone, return currentRestoreProgressState.bytesTotal, currentRestoreProgressState.bytesDone,
currentRestoreProgressState.description, currentRestoreProgressState.hasUpdate, currentRestoreProgressState.description, currentRestoreProgressState.hasUpdate,
currentRestoreProgressState.dbTotal, currentRestoreProgressState.dbDone, speed currentRestoreProgressState.dbTotal, currentRestoreProgressState.dbDone, speed,
currentRestoreProgressState.dbPhaseElapsed, currentRestoreProgressState.dbAvgPerDB
} }
// calculateRollingSpeed calculates speed from recent samples (last 5 seconds) // calculateRollingSpeed calculates speed from recent samples (last 5 seconds)
@@ -304,6 +313,21 @@ func executeRestoreWithTUIProgress(parentCtx context.Context, cfg *config.Config
progressState.bytesDone = 0 progressState.bytesDone = 0
}) })
// Set up timing-aware database progress callback for cluster restore ETA
engine.SetDatabaseProgressWithTimingCallback(func(done, total int, dbName string, phaseElapsed, avgPerDB time.Duration) {
progressState.mu.Lock()
defer progressState.mu.Unlock()
progressState.dbDone = done
progressState.dbTotal = total
progressState.description = fmt.Sprintf("Restoring %s", dbName)
progressState.dbPhaseElapsed = phaseElapsed
progressState.dbAvgPerDB = avgPerDB
progressState.hasUpdate = true
// Clear byte progress when switching to db progress
progressState.bytesTotal = 0
progressState.bytesDone = 0
})
// Store progress state in a package-level variable for the ticker to access // Store progress state in a package-level variable for the ticker to access
// This is a workaround because tea messages can't be sent from callbacks // This is a workaround because tea messages can't be sent from callbacks
setCurrentRestoreProgress(progressState) setCurrentRestoreProgress(progressState)
@@ -357,7 +381,7 @@ func (m RestoreExecutionModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
m.elapsed = time.Since(m.startTime) m.elapsed = time.Since(m.startTime)
// Poll shared progress state for real-time updates // Poll shared progress state for real-time updates
bytesTotal, bytesDone, description, hasUpdate, dbTotal, dbDone, speed := getCurrentRestoreProgress() bytesTotal, bytesDone, description, hasUpdate, dbTotal, dbDone, speed, dbPhaseElapsed, dbAvgPerDB := getCurrentRestoreProgress()
if hasUpdate && bytesTotal > 0 { if hasUpdate && bytesTotal > 0 {
m.bytesTotal = bytesTotal m.bytesTotal = bytesTotal
m.bytesDone = bytesDone m.bytesDone = bytesDone
@@ -370,9 +394,11 @@ func (m RestoreExecutionModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
m.phase = "Extracting" m.phase = "Extracting"
m.progress = int((bytesDone * 100) / bytesTotal) m.progress = int((bytesDone * 100) / bytesTotal)
} else if hasUpdate && dbTotal > 0 { } else if hasUpdate && dbTotal > 0 {
// Database count progress for cluster restore // Database count progress for cluster restore with timing
m.dbTotal = dbTotal m.dbTotal = dbTotal
m.dbDone = dbDone m.dbDone = dbDone
m.dbPhaseElapsed = dbPhaseElapsed
m.dbAvgPerDB = dbAvgPerDB
m.showBytes = false m.showBytes = false
m.status = fmt.Sprintf("Restoring database %d of %d...", dbDone+1, dbTotal) m.status = fmt.Sprintf("Restoring database %d of %d...", dbDone+1, dbTotal)
m.phase = "Restore" m.phase = "Restore"
@@ -518,22 +544,71 @@ func (m RestoreExecutionModel) View() string {
s.WriteString("\n") s.WriteString("\n")
if m.done { if m.done {
// Show result // Show result with comprehensive summary
if m.err != nil { if m.err != nil {
s.WriteString(errorStyle.Render("[FAIL] Restore Failed")) s.WriteString(errorStyle.Render("╔══════════════════════════════════════════════════════════════╗"))
s.WriteString("\n")
s.WriteString(errorStyle.Render("║ [FAIL] RESTORE FAILED ║"))
s.WriteString("\n")
s.WriteString(errorStyle.Render("╚══════════════════════════════════════════════════════════════╝"))
s.WriteString("\n\n") s.WriteString("\n\n")
s.WriteString(errorStyle.Render(fmt.Sprintf("Error: %v", m.err))) s.WriteString(errorStyle.Render(fmt.Sprintf(" Error: %v", m.err)))
s.WriteString("\n") s.WriteString("\n")
} else { } else {
s.WriteString(successStyle.Render("[OK] Restore Completed Successfully")) s.WriteString(successStyle.Render("╔══════════════════════════════════════════════════════════════╗"))
s.WriteString("\n")
s.WriteString(successStyle.Render("║ [OK] RESTORE COMPLETED SUCCESSFULLY ║"))
s.WriteString("\n")
s.WriteString(successStyle.Render("╚══════════════════════════════════════════════════════════════╝"))
s.WriteString("\n\n") s.WriteString("\n\n")
s.WriteString(successStyle.Render(m.result))
// Summary section
s.WriteString(infoStyle.Render(" ─── Summary ───────────────────────────────────────────────"))
s.WriteString("\n\n")
// Archive info
s.WriteString(fmt.Sprintf(" Archive: %s\n", m.archive.Name))
if m.archive.Size > 0 {
s.WriteString(fmt.Sprintf(" Archive Size: %s\n", FormatBytes(m.archive.Size)))
}
// Restore type specific info
if m.restoreType == "restore-cluster" {
s.WriteString(fmt.Sprintf(" Type: Cluster Restore\n"))
if m.dbTotal > 0 {
s.WriteString(fmt.Sprintf(" Databases: %d restored\n", m.dbTotal))
}
if m.cleanClusterFirst && len(m.existingDBs) > 0 {
s.WriteString(fmt.Sprintf(" Cleaned: %d existing database(s) dropped\n", len(m.existingDBs)))
}
} else {
s.WriteString(fmt.Sprintf(" Type: Single Database Restore\n"))
s.WriteString(fmt.Sprintf(" Target DB: %s\n", m.targetDB))
}
s.WriteString("\n") s.WriteString("\n")
} }
s.WriteString(fmt.Sprintf("\nElapsed Time: %s\n", formatDuration(m.elapsed))) // Timing section
s.WriteString(infoStyle.Render(" ─── Timing ────────────────────────────────────────────────"))
s.WriteString("\n\n")
s.WriteString(fmt.Sprintf(" Total Time: %s\n", formatDuration(m.elapsed)))
// Calculate and show throughput if we have size info
if m.archive.Size > 0 && m.elapsed.Seconds() > 0 {
throughput := float64(m.archive.Size) / m.elapsed.Seconds()
s.WriteString(fmt.Sprintf(" Throughput: %s/s (average)\n", FormatBytes(int64(throughput))))
}
if m.dbTotal > 0 && m.err == nil {
avgPerDB := m.elapsed / time.Duration(m.dbTotal)
s.WriteString(fmt.Sprintf(" Avg per DB: %s\n", formatDuration(avgPerDB)))
}
s.WriteString("\n") s.WriteString("\n")
s.WriteString(infoStyle.Render("[KEYS] Press Enter to continue")) s.WriteString(infoStyle.Render(" ───────────────────────────────────────────────────────────"))
s.WriteString("\n\n")
s.WriteString(infoStyle.Render(" [KEYS] Press Enter to continue"))
} else { } else {
// Show progress // Show progress
s.WriteString(fmt.Sprintf("Phase: %s\n", m.phase)) s.WriteString(fmt.Sprintf("Phase: %s\n", m.phase))
@@ -549,13 +624,13 @@ func (m RestoreExecutionModel) View() string {
s.WriteString(renderDetailedProgressBarWithSpeed(m.bytesDone, m.bytesTotal, m.speed)) s.WriteString(renderDetailedProgressBarWithSpeed(m.bytesDone, m.bytesTotal, m.speed))
s.WriteString("\n\n") s.WriteString("\n\n")
} else if m.dbTotal > 0 { } else if m.dbTotal > 0 {
// Database count progress for cluster restore // Database count progress for cluster restore with timing
spinner := m.spinnerFrames[m.spinnerFrame] spinner := m.spinnerFrames[m.spinnerFrame]
s.WriteString(fmt.Sprintf("Status: %s %s\n", spinner, m.status)) s.WriteString(fmt.Sprintf("Status: %s %s\n", spinner, m.status))
s.WriteString("\n") s.WriteString("\n")
// Show database progress bar // Show database progress bar with timing and ETA
s.WriteString(renderDatabaseProgressBar(m.dbDone, m.dbTotal)) s.WriteString(renderDatabaseProgressBarWithTiming(m.dbDone, m.dbTotal, m.dbPhaseElapsed, m.dbAvgPerDB))
s.WriteString("\n\n") s.WriteString("\n\n")
} else { } else {
// Show status with rotating spinner (for phases without detailed progress) // Show status with rotating spinner (for phases without detailed progress)
@@ -678,6 +753,55 @@ func renderDatabaseProgressBar(done, total int) string {
return s.String() return s.String()
} }
// renderDatabaseProgressBarWithTiming renders a progress bar for database count with timing and ETA
func renderDatabaseProgressBarWithTiming(done, total int, phaseElapsed, avgPerDB time.Duration) string {
var s strings.Builder
// Calculate percentage
percent := 0
if total > 0 {
percent = (done * 100) / total
if percent > 100 {
percent = 100
}
}
// Render progress bar
width := 30
filled := (percent * width) / 100
barFilled := strings.Repeat("█", filled)
barEmpty := strings.Repeat("░", width-filled)
s.WriteString(successStyle.Render("["))
s.WriteString(successStyle.Render(barFilled))
s.WriteString(infoStyle.Render(barEmpty))
s.WriteString(successStyle.Render("]"))
// Count and percentage
s.WriteString(fmt.Sprintf(" %3d%% %d / %d databases", percent, done, total))
// Timing and ETA
if phaseElapsed > 0 {
s.WriteString(fmt.Sprintf(" [%s", FormatDurationShort(phaseElapsed)))
// Calculate ETA based on average time per database
if avgPerDB > 0 && done < total {
remainingDBs := total - done
eta := time.Duration(remainingDBs) * avgPerDB
s.WriteString(fmt.Sprintf(" / ETA: %s", FormatDurationShort(eta)))
} else if done > 0 && done < total {
// Fallback: estimate ETA from overall elapsed time
avgElapsed := phaseElapsed / time.Duration(done)
remainingDBs := total - done
eta := time.Duration(remainingDBs) * avgElapsed
s.WriteString(fmt.Sprintf(" / ETA: ~%s", FormatDurationShort(eta)))
}
s.WriteString("]")
}
return s.String()
}
// formatDuration formats duration in human readable format // formatDuration formats duration in human readable format
func formatDuration(d time.Duration) string { func formatDuration(d time.Duration) string {
if d < time.Minute { if d < time.Minute {