TUI: Add timing and ETA tracking to cluster restore progress

- Add DatabaseProgressWithTimingCallback for timing-aware progress reporting
- Track elapsed time and average duration per database during restore phase
- Display ETA based on completed database restore times
- Show restore phase elapsed time in progress bar
- Enhance cluster restore progress bar with [elapsed / ETA: remaining] format

bin/README.md

@@ -4,8 +4,8 @@ This directory contains pre-compiled binaries for the DB Backup Tool across mult
 
 ## Build Information
 - **Version**: 3.42.34
-- **Build Time**: 2026-01-15_14:16:33_UTC
+- **Build Time**: 2026-01-15_17:50:35_UTC
-- **Git Commit**: eeacbfa
+- **Git Commit**: 4938dc1
 
 ## Recent Updates (v1.1.0)
 - ✅ Fixed TUI progress display with line-by-line output

internal/restore/engine.go

@@ -34,6 +34,10 @@ type ProgressCallback func(current, total int64, description string)
 // DatabaseProgressCallback is called with database count progress during cluster restore
 type DatabaseProgressCallback func(done, total int, dbName string)
 
+// DatabaseProgressWithTimingCallback is called with database progress including timing info
+// Parameters: done count, total count, database name, elapsed time for current restore phase, avg duration per DB
+type DatabaseProgressWithTimingCallback func(done, total int, dbName string, phaseElapsed, avgPerDB time.Duration)
+
 // Engine handles database restore operations
 type Engine struct {
 	cfg              *config.Config
@@ -47,6 +51,7 @@ type Engine struct {
 	// TUI progress callback for detailed progress reporting
 	progressCallback          ProgressCallback
 	dbProgressCallback        DatabaseProgressCallback
+	dbProgressTimingCallback  DatabaseProgressWithTimingCallback
 }
 
 // New creates a new restore engine
@@ -112,6 +117,11 @@ func (e *Engine) SetDatabaseProgressCallback(cb DatabaseProgressCallback) {
 	e.dbProgressCallback = cb
 }
 
+// SetDatabaseProgressWithTimingCallback sets a callback for database progress with timing info
+func (e *Engine) SetDatabaseProgressWithTimingCallback(cb DatabaseProgressWithTimingCallback) {
+	e.dbProgressTimingCallback = cb
+}
+
 // reportProgress safely calls the progress callback if set
 func (e *Engine) reportProgress(current, total int64, description string) {
 	if e.progressCallback != nil {
@@ -126,6 +136,13 @@ func (e *Engine) reportDatabaseProgress(done, total int, dbName string) {
 	}
 }
 
+// reportDatabaseProgressWithTiming safely calls the timing-aware callback if set
+func (e *Engine) reportDatabaseProgressWithTiming(done, total int, dbName string, phaseElapsed, avgPerDB time.Duration) {
+	if e.dbProgressTimingCallback != nil {
+		e.dbProgressTimingCallback(done, total, dbName, phaseElapsed, avgPerDB)
+	}
+}
+
 // loggerAdapter adapts our logger to the progress.Logger interface
 type loggerAdapter struct {
 	logger logger.Logger
@@ -1037,6 +1054,11 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
 	var successCount, failCount int32
 	var mu sync.Mutex // Protect shared resources (progress, logger)
 
+	// Timing tracking for restore phase progress
+	restorePhaseStart := time.Now()
+	var completedDBTimes []time.Duration // Track duration for each completed DB restore
+	var completedDBTimesMu sync.Mutex
+
 	// Create semaphore to limit concurrency
 	semaphore := make(chan struct{}, parallelism)
 	var wg sync.WaitGroup
@@ -1062,6 +1084,9 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
 				}
 			}()
 
+			// Track timing for this database restore
+			dbRestoreStart := time.Now()
+
 			// Update estimator progress (thread-safe)
 			mu.Lock()
 			estimator.UpdateProgress(idx)
@@ -1074,12 +1099,26 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
 
 			dbProgress := 15 + int(float64(idx)/float64(totalDBs)*85.0)
 
+			// Calculate average time per DB and report progress with timing
+			completedDBTimesMu.Lock()
+			var avgPerDB time.Duration
+			if len(completedDBTimes) > 0 {
+				var totalDuration time.Duration
+				for _, d := range completedDBTimes {
+					totalDuration += d
+				}
+				avgPerDB = totalDuration / time.Duration(len(completedDBTimes))
+			}
+			phaseElapsed := time.Since(restorePhaseStart)
+			completedDBTimesMu.Unlock()
+
 			mu.Lock()
 			statusMsg := fmt.Sprintf("Restoring database %s (%d/%d)", dbName, idx+1, totalDBs)
 			e.progress.Update(statusMsg)
 			e.log.Info("Restoring database", "name", dbName, "file", dumpFile, "progress", dbProgress)
-			// Report database progress for TUI
+			// Report database progress for TUI (both callbacks)
 			e.reportDatabaseProgress(idx, totalDBs, dbName)
+			e.reportDatabaseProgressWithTiming(idx, totalDBs, dbName, phaseElapsed, avgPerDB)
 			mu.Unlock()
 
 			// STEP 1: Drop existing database completely (clean slate)
@@ -1144,6 +1183,12 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
 				return
 			}
 
+			// Track completed database restore duration for ETA calculation
+			dbRestoreDuration := time.Since(dbRestoreStart)
+			completedDBTimesMu.Lock()
+			completedDBTimes = append(completedDBTimes, dbRestoreDuration)
+			completedDBTimesMu.Unlock()
+
 			atomic.AddInt32(&successCount, 1)
 		}(dbIndex, entry.Name())
 
@@ -1937,6 +1982,8 @@ type OriginalSettings struct {
 }
 
 // boostPostgreSQLSettings boosts multiple PostgreSQL settings for large restores
+// NOTE: max_locks_per_transaction requires a PostgreSQL RESTART to take effect!
+// maintenance_work_mem can be changed with pg_reload_conf().
 func (e *Engine) boostPostgreSQLSettings(ctx context.Context, lockBoostValue int) (*OriginalSettings, error) {
 	connStr := e.buildConnString()
 	db, err := sql.Open("pgx", connStr)
@@ -1956,30 +2003,113 @@ func (e *Engine) boostPostgreSQLSettings(ctx context.Context, lockBoostValue int
 	// Get current maintenance_work_mem
 	db.QueryRowContext(ctx, "SHOW maintenance_work_mem").Scan(&original.MaintenanceWorkMem)
 
-	// Boost max_locks_per_transaction (if not already high enough)
+	// CRITICAL: max_locks_per_transaction requires a PostgreSQL RESTART!
+	// pg_reload_conf() is NOT sufficient for this parameter.
+	needsRestart := false
 	if original.MaxLocks < lockBoostValue {
 		_, err = db.ExecContext(ctx, fmt.Sprintf("ALTER SYSTEM SET max_locks_per_transaction = %d", lockBoostValue))
 		if err != nil {
-			e.log.Warn("Could not boost max_locks_per_transaction", "error", err)
+			e.log.Warn("Could not set max_locks_per_transaction", "error", err)
+		} else {
+			needsRestart = true
+			e.log.Warn("max_locks_per_transaction requires PostgreSQL restart to take effect",
+				"current", original.MaxLocks,
+				"target", lockBoostValue)
 		}
 	}
 
 	// Boost maintenance_work_mem to 2GB for faster index creation
+	// (this one CAN be applied via pg_reload_conf)
 	_, err = db.ExecContext(ctx, "ALTER SYSTEM SET maintenance_work_mem = '2GB'")
 	if err != nil {
 		e.log.Warn("Could not boost maintenance_work_mem", "error", err)
 	}
 
-	// Reload config to apply changes (no restart needed for these settings)
+	// Reload config to apply maintenance_work_mem
 	_, err = db.ExecContext(ctx, "SELECT pg_reload_conf()")
 	if err != nil {
 		return original, fmt.Errorf("failed to reload config: %w", err)
 	}
 
+	// If max_locks_per_transaction needs a restart, try to do it
+	if needsRestart {
+		if restarted := e.tryRestartPostgreSQL(ctx); restarted {
+			e.log.Info("PostgreSQL restarted successfully - max_locks_per_transaction now active")
+			// Wait for PostgreSQL to be ready
+			time.Sleep(3 * time.Second)
+		} else {
+			// Cannot restart - warn user loudly
+			e.log.Error("=" + strings.Repeat("=", 70))
+			e.log.Error("WARNING: max_locks_per_transaction change requires PostgreSQL restart!")
+			e.log.Error("Current value: " + strconv.Itoa(original.MaxLocks) + ", needed: " + strconv.Itoa(lockBoostValue))
+			e.log.Error("Restore may fail with 'out of shared memory' error on BLOB-heavy databases.")
+			e.log.Error("")
+			e.log.Error("To fix manually:")
+			e.log.Error("  1. sudo systemctl restart postgresql")
+			e.log.Error("  2. Or: sudo -u postgres pg_ctl restart -D $PGDATA")
+			e.log.Error("  3. Then re-run the restore")
+			e.log.Error("=" + strings.Repeat("=", 70))
+			// Continue anyway - might work for small restores
+		}
+	}
+
 	return original, nil
 }
 
+// tryRestartPostgreSQL attempts to restart PostgreSQL using various methods
+// Returns true if restart was successful
+func (e *Engine) tryRestartPostgreSQL(ctx context.Context) bool {
+	e.progress.Update("Attempting PostgreSQL restart for lock settings...")
+
+	// Method 1: systemctl (most common on modern Linux)
+	cmd := exec.CommandContext(ctx, "sudo", "systemctl", "restart", "postgresql")
+	if err := cmd.Run(); err == nil {
+		return true
+	}
+
+	// Method 2: systemctl with version suffix (e.g., postgresql-15)
+	for _, ver := range []string{"17", "16", "15", "14", "13", "12"} {
+		cmd = exec.CommandContext(ctx, "sudo", "systemctl", "restart", "postgresql-"+ver)
+		if err := cmd.Run(); err == nil {
+			return true
+		}
+	}
+
+	// Method 3: service command (older systems)
+	cmd = exec.CommandContext(ctx, "sudo", "service", "postgresql", "restart")
+	if err := cmd.Run(); err == nil {
+		return true
+	}
+
+	// Method 4: pg_ctl as postgres user
+	cmd = exec.CommandContext(ctx, "sudo", "-u", "postgres", "pg_ctl", "restart", "-D", "/var/lib/postgresql/data", "-m", "fast")
+	if err := cmd.Run(); err == nil {
+		return true
+	}
+
+	// Method 5: Try common PGDATA paths
+	pgdataPaths := []string{
+		"/var/lib/pgsql/data",
+		"/var/lib/pgsql/17/data",
+		"/var/lib/pgsql/16/data",
+		"/var/lib/pgsql/15/data",
+		"/var/lib/postgresql/17/main",
+		"/var/lib/postgresql/16/main",
+		"/var/lib/postgresql/15/main",
+	}
+	for _, pgdata := range pgdataPaths {
+		cmd = exec.CommandContext(ctx, "sudo", "-u", "postgres", "pg_ctl", "restart", "-D", pgdata, "-m", "fast")
+		if err := cmd.Run(); err == nil {
+			return true
+		}
+	}
+
+	return false
+}
+
 // resetPostgreSQLSettings restores original PostgreSQL settings
+// NOTE: max_locks_per_transaction changes are written but require restart to take effect.
+// We don't restart here since we're done with the restore.
 func (e *Engine) resetPostgreSQLSettings(ctx context.Context, original *OriginalSettings) error {
 	connStr := e.buildConnString()
 	db, err := sql.Open("pgx", connStr)
@@ -1988,25 +2118,28 @@ func (e *Engine) resetPostgreSQLSettings(ctx context.Context, original *Original
 	}
 	defer db.Close()
 
-	// Reset max_locks_per_transaction
+	// Reset max_locks_per_transaction (will take effect on next restart)
 	if original.MaxLocks == 64 { // Default
 		db.ExecContext(ctx, "ALTER SYSTEM RESET max_locks_per_transaction")
 	} else if original.MaxLocks > 0 {
 		db.ExecContext(ctx, fmt.Sprintf("ALTER SYSTEM SET max_locks_per_transaction = %d", original.MaxLocks))
 	}
 
-	// Reset maintenance_work_mem
+	// Reset maintenance_work_mem (takes effect immediately with reload)
 	if original.MaintenanceWorkMem == "64MB" { // Default
 		db.ExecContext(ctx, "ALTER SYSTEM RESET maintenance_work_mem")
 	} else if original.MaintenanceWorkMem != "" {
 		db.ExecContext(ctx, fmt.Sprintf("ALTER SYSTEM SET maintenance_work_mem = '%s'", original.MaintenanceWorkMem))
 	}
 
-	// Reload config
+	// Reload config (only maintenance_work_mem will take effect immediately)
 	_, err = db.ExecContext(ctx, "SELECT pg_reload_conf()")
 	if err != nil {
 		return fmt.Errorf("failed to reload config: %w", err)
 	}
 
+	e.log.Info("PostgreSQL settings reset queued",
+		"note", "max_locks_per_transaction will revert on next PostgreSQL restart")
+
 	return nil
 }

internal/restore/preflight.go

@@ -201,10 +201,19 @@ func (e *Engine) checkPostgreSQL(ctx context.Context, result *PreflightResult) {
 		result.PostgreSQL.IsSuperuser = isSuperuser
 	}
 
-	// Add info/warnings
+	// CRITICAL: max_locks_per_transaction requires PostgreSQL RESTART to change!
+	// Warn users loudly about this - it's the #1 cause of "out of shared memory" errors
 	if result.PostgreSQL.MaxLocksPerTransaction < 256 {
-		e.log.Info("PostgreSQL max_locks_per_transaction is low - will auto-boost",
+		e.log.Warn("PostgreSQL max_locks_per_transaction is LOW",
-			"current", result.PostgreSQL.MaxLocksPerTransaction)
+			"current", result.PostgreSQL.MaxLocksPerTransaction,
+			"recommended", "256+",
+			"note", "REQUIRES PostgreSQL restart to change!")
+
+		result.Warnings = append(result.Warnings,
+			fmt.Sprintf("max_locks_per_transaction=%d is low (recommend 256+). "+
+				"This setting requires PostgreSQL RESTART to change. "+
+				"BLOB-heavy databases may fail with 'out of shared memory' error.",
+				result.PostgreSQL.MaxLocksPerTransaction))
 	}
 
 	// Parse shared_buffers and warn if very low

internal/tui/restore_exec.go

@@ -57,6 +57,10 @@ type RestoreExecutionModel struct {
 	dbTotal int
 	dbDone  int
 
+	// Timing info for database restore phase (ETA calculation)
+	dbPhaseElapsed time.Duration // Elapsed time since restore phase started
+	dbAvgPerDB     time.Duration // Average time per database restore
+
 	// Results
 	done       bool
 	cancelling bool // True when user has requested cancellation
@@ -136,6 +140,10 @@ type sharedProgressState struct {
 	dbTotal int
 	dbDone  int
 
+	// Timing info for database restore phase
+	dbPhaseElapsed time.Duration // Elapsed time since restore phase started
+	dbAvgPerDB     time.Duration // Average time per database restore
+
 	// Rolling window for speed calculation
 	speedSamples []restoreSpeedSample
 }
@@ -163,12 +171,12 @@ func clearCurrentRestoreProgress() {
 	currentRestoreProgressState = nil
 }
 
-func getCurrentRestoreProgress() (bytesTotal, bytesDone int64, description string, hasUpdate bool, dbTotal, dbDone int, speed float64) {
+func getCurrentRestoreProgress() (bytesTotal, bytesDone int64, description string, hasUpdate bool, dbTotal, dbDone int, speed float64, dbPhaseElapsed, dbAvgPerDB time.Duration) {
 	currentRestoreProgressMu.Lock()
 	defer currentRestoreProgressMu.Unlock()
 
 	if currentRestoreProgressState == nil {
-		return 0, 0, "", false, 0, 0, 0
+		return 0, 0, "", false, 0, 0, 0, 0, 0
 	}
 
 	currentRestoreProgressState.mu.Lock()
@@ -179,7 +187,8 @@ func getCurrentRestoreProgress() (bytesTotal, bytesDone int64, description strin
 
 	return currentRestoreProgressState.bytesTotal, currentRestoreProgressState.bytesDone,
 		currentRestoreProgressState.description, currentRestoreProgressState.hasUpdate,
-		currentRestoreProgressState.dbTotal, currentRestoreProgressState.dbDone, speed
+		currentRestoreProgressState.dbTotal, currentRestoreProgressState.dbDone, speed,
+		currentRestoreProgressState.dbPhaseElapsed, currentRestoreProgressState.dbAvgPerDB
 }
 
 // calculateRollingSpeed calculates speed from recent samples (last 5 seconds)
@@ -304,6 +313,21 @@ func executeRestoreWithTUIProgress(parentCtx context.Context, cfg *config.Config
 			progressState.bytesDone = 0
 		})
 
+		// Set up timing-aware database progress callback for cluster restore ETA
+		engine.SetDatabaseProgressWithTimingCallback(func(done, total int, dbName string, phaseElapsed, avgPerDB time.Duration) {
+			progressState.mu.Lock()
+			defer progressState.mu.Unlock()
+			progressState.dbDone = done
+			progressState.dbTotal = total
+			progressState.description = fmt.Sprintf("Restoring %s", dbName)
+			progressState.dbPhaseElapsed = phaseElapsed
+			progressState.dbAvgPerDB = avgPerDB
+			progressState.hasUpdate = true
+			// Clear byte progress when switching to db progress
+			progressState.bytesTotal = 0
+			progressState.bytesDone = 0
+		})
+
 		// Store progress state in a package-level variable for the ticker to access
 		// This is a workaround because tea messages can't be sent from callbacks
 		setCurrentRestoreProgress(progressState)
@@ -357,7 +381,7 @@ func (m RestoreExecutionModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 			m.elapsed = time.Since(m.startTime)
 
 			// Poll shared progress state for real-time updates
-			bytesTotal, bytesDone, description, hasUpdate, dbTotal, dbDone, speed := getCurrentRestoreProgress()
+			bytesTotal, bytesDone, description, hasUpdate, dbTotal, dbDone, speed, dbPhaseElapsed, dbAvgPerDB := getCurrentRestoreProgress()
 			if hasUpdate && bytesTotal > 0 {
 				m.bytesTotal = bytesTotal
 				m.bytesDone = bytesDone
@@ -370,9 +394,11 @@ func (m RestoreExecutionModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 				m.phase = "Extracting"
 				m.progress = int((bytesDone * 100) / bytesTotal)
 			} else if hasUpdate && dbTotal > 0 {
-				// Database count progress for cluster restore
+				// Database count progress for cluster restore with timing
 				m.dbTotal = dbTotal
 				m.dbDone = dbDone
+				m.dbPhaseElapsed = dbPhaseElapsed
+				m.dbAvgPerDB = dbAvgPerDB
 				m.showBytes = false
 				m.status = fmt.Sprintf("Restoring database %d of %d...", dbDone+1, dbTotal)
 				m.phase = "Restore"
@@ -549,13 +575,13 @@ func (m RestoreExecutionModel) View() string {
 			s.WriteString(renderDetailedProgressBarWithSpeed(m.bytesDone, m.bytesTotal, m.speed))
 			s.WriteString("\n\n")
 		} else if m.dbTotal > 0 {
-			// Database count progress for cluster restore
+			// Database count progress for cluster restore with timing
 			spinner := m.spinnerFrames[m.spinnerFrame]
 			s.WriteString(fmt.Sprintf("Status: %s %s\n", spinner, m.status))
 			s.WriteString("\n")
 
-			// Show database progress bar
+			// Show database progress bar with timing and ETA
-			s.WriteString(renderDatabaseProgressBar(m.dbDone, m.dbTotal))
+			s.WriteString(renderDatabaseProgressBarWithTiming(m.dbDone, m.dbTotal, m.dbPhaseElapsed, m.dbAvgPerDB))
 			s.WriteString("\n\n")
 		} else {
 			// Show status with rotating spinner (for phases without detailed progress)
@@ -678,6 +704,55 @@ func renderDatabaseProgressBar(done, total int) string {
 	return s.String()
 }
 
+// renderDatabaseProgressBarWithTiming renders a progress bar for database count with timing and ETA
+func renderDatabaseProgressBarWithTiming(done, total int, phaseElapsed, avgPerDB time.Duration) string {
+	var s strings.Builder
+
+	// Calculate percentage
+	percent := 0
+	if total > 0 {
+		percent = (done * 100) / total
+		if percent > 100 {
+			percent = 100
+		}
+	}
+
+	// Render progress bar
+	width := 30
+	filled := (percent * width) / 100
+	barFilled := strings.Repeat("█", filled)
+	barEmpty := strings.Repeat("░", width-filled)
+
+	s.WriteString(successStyle.Render("["))
+	s.WriteString(successStyle.Render(barFilled))
+	s.WriteString(infoStyle.Render(barEmpty))
+	s.WriteString(successStyle.Render("]"))
+
+	// Count and percentage
+	s.WriteString(fmt.Sprintf("  %3d%%  %d / %d databases", percent, done, total))
+
+	// Timing and ETA
+	if phaseElapsed > 0 {
+		s.WriteString(fmt.Sprintf("  [%s", FormatDurationShort(phaseElapsed)))
+
+		// Calculate ETA based on average time per database
+		if avgPerDB > 0 && done < total {
+			remainingDBs := total - done
+			eta := time.Duration(remainingDBs) * avgPerDB
+			s.WriteString(fmt.Sprintf(" / ETA: %s", FormatDurationShort(eta)))
+		} else if done > 0 && done < total {
+			// Fallback: estimate ETA from overall elapsed time
+			avgElapsed := phaseElapsed / time.Duration(done)
+			remainingDBs := total - done
+			eta := time.Duration(remainingDBs) * avgElapsed
+			s.WriteString(fmt.Sprintf(" / ETA: ~%s", FormatDurationShort(eta)))
+		}
+		s.WriteString("]")
+	}
+
+	return s.String()
+}
+
 // formatDuration formats duration in human readable format
 func formatDuration(d time.Duration) string {
 	if d < time.Minute {