Compare commits

...

3 Commits

Author SHA1 Message Date
a85ad0c88c TUI: Add timing and ETA tracking to cluster restore progress
All checks were successful
CI/CD / Test (push) Successful in 1m17s
CI/CD / Lint (push) Successful in 1m28s
CI/CD / Build & Release (push) Successful in 3m21s
- Add DatabaseProgressWithTimingCallback for timing-aware progress reporting
- Track elapsed time and average duration per database during restore phase
- Display ETA based on completed database restore times
- Show restore phase elapsed time in progress bar
- Enhance cluster restore progress bar with [elapsed / ETA: remaining] format
2026-01-16 09:42:05 +01:00
4938dc1918 fix: max_locks_per_transaction requires PostgreSQL restart
All checks were successful
CI/CD / Test (push) Successful in 1m19s
CI/CD / Lint (push) Successful in 1m31s
CI/CD / Build & Release (push) Successful in 3m34s
- Fixed critical bug where ALTER SYSTEM + pg_reload_conf() was used
  but max_locks_per_transaction requires a full PostgreSQL restart
- Added automatic restart attempt (systemctl, service, pg_ctl)
- Added loud warnings if restart fails with manual fix instructions
- Updated preflight checks to warn about low max_locks_per_transaction
- This was causing 'out of shared memory' errors on BLOB-heavy restores
2026-01-15 18:50:10 +01:00
09a917766f TUI: Add database progress tracking to cluster backup
All checks were successful
CI/CD / Test (push) Successful in 1m27s
CI/CD / Lint (push) Successful in 1m33s
CI/CD / Build & Release (push) Successful in 3m28s
- Add ProgressCallback and DatabaseProgressCallback to backup engine
- Add SetProgressCallback() and SetDatabaseProgressCallback() methods
- Wire database progress callback in backup_exec.go
- Show progress bar (Database: [████░░░] X/Y) during cluster backups
- Hide spinner when progress bar is displayed
- Use shared progress state pattern for thread-safe TUI updates
2026-01-15 15:32:26 +01:00
6 changed files with 402 additions and 35 deletions

View File

@@ -4,8 +4,8 @@ This directory contains pre-compiled binaries for the DB Backup Tool across mult
## Build Information
- **Version**: 3.42.34
- **Build Time**: 2026-01-14_16:19:00_UTC
- **Git Commit**: 7711a20
- **Build Time**: 2026-01-15_17:50:35_UTC
- **Git Commit**: 4938dc1
## Recent Updates (v1.1.0)
- ✅ Fixed TUI progress display with line-by-line output

View File

@@ -28,14 +28,22 @@ import (
"dbbackup/internal/swap"
)
// ProgressCallback is called with byte-level progress updates during backup operations
type ProgressCallback func(current, total int64, description string)
// DatabaseProgressCallback is called with database count progress during cluster backup
type DatabaseProgressCallback func(done, total int, dbName string)
// Engine handles backup operations
type Engine struct {
cfg *config.Config
log logger.Logger
db database.Database
progress progress.Indicator
detailedReporter *progress.DetailedReporter
silent bool // Silent mode for TUI
cfg *config.Config
log logger.Logger
db database.Database
progress progress.Indicator
detailedReporter *progress.DetailedReporter
silent bool // Silent mode for TUI
progressCallback ProgressCallback
dbProgressCallback DatabaseProgressCallback
}
// New creates a new backup engine
@@ -86,6 +94,30 @@ func NewSilent(cfg *config.Config, log logger.Logger, db database.Database, prog
}
}
// SetProgressCallback sets a callback for detailed progress reporting (for TUI mode)
func (e *Engine) SetProgressCallback(cb ProgressCallback) {
e.progressCallback = cb
}
// SetDatabaseProgressCallback sets a callback for database count progress during cluster backup
func (e *Engine) SetDatabaseProgressCallback(cb DatabaseProgressCallback) {
e.dbProgressCallback = cb
}
// reportProgress reports progress to the callback if set
func (e *Engine) reportProgress(current, total int64, description string) {
if e.progressCallback != nil {
e.progressCallback(current, total, description)
}
}
// reportDatabaseProgress reports database count progress to the callback if set
func (e *Engine) reportDatabaseProgress(done, total int, dbName string) {
if e.dbProgressCallback != nil {
e.dbProgressCallback(done, total, dbName)
}
}
// loggerAdapter adapts our logger to the progress.Logger interface
type loggerAdapter struct {
logger logger.Logger
@@ -465,6 +497,8 @@ func (e *Engine) BackupCluster(ctx context.Context) error {
estimator.UpdateProgress(idx)
e.printf(" [%d/%d] Backing up database: %s\n", idx+1, len(databases), name)
quietProgress.Update(fmt.Sprintf("Backing up database %d/%d: %s", idx+1, len(databases), name))
// Report database progress to TUI callback
e.reportDatabaseProgress(idx+1, len(databases), name)
mu.Unlock()
// Check database size and warn if very large

View File

@@ -34,6 +34,10 @@ type ProgressCallback func(current, total int64, description string)
// DatabaseProgressCallback is called with database count progress during cluster restore
type DatabaseProgressCallback func(done, total int, dbName string)
// DatabaseProgressWithTimingCallback is called with database progress including timing info
// Parameters: done count, total count, database name, elapsed time for current restore phase, avg duration per DB
type DatabaseProgressWithTimingCallback func(done, total int, dbName string, phaseElapsed, avgPerDB time.Duration)
// Engine handles database restore operations
type Engine struct {
cfg *config.Config
@@ -45,8 +49,9 @@ type Engine struct {
debugLogPath string // Path to save debug log on error
// TUI progress callback for detailed progress reporting
progressCallback ProgressCallback
dbProgressCallback DatabaseProgressCallback
progressCallback ProgressCallback
dbProgressCallback DatabaseProgressCallback
dbProgressTimingCallback DatabaseProgressWithTimingCallback
}
// New creates a new restore engine
@@ -112,6 +117,11 @@ func (e *Engine) SetDatabaseProgressCallback(cb DatabaseProgressCallback) {
e.dbProgressCallback = cb
}
// SetDatabaseProgressWithTimingCallback sets a callback for database progress with timing info
func (e *Engine) SetDatabaseProgressWithTimingCallback(cb DatabaseProgressWithTimingCallback) {
e.dbProgressTimingCallback = cb
}
// reportProgress safely calls the progress callback if set
func (e *Engine) reportProgress(current, total int64, description string) {
if e.progressCallback != nil {
@@ -126,6 +136,13 @@ func (e *Engine) reportDatabaseProgress(done, total int, dbName string) {
}
}
// reportDatabaseProgressWithTiming safely calls the timing-aware callback if set
func (e *Engine) reportDatabaseProgressWithTiming(done, total int, dbName string, phaseElapsed, avgPerDB time.Duration) {
if e.dbProgressTimingCallback != nil {
e.dbProgressTimingCallback(done, total, dbName, phaseElapsed, avgPerDB)
}
}
// loggerAdapter adapts our logger to the progress.Logger interface
type loggerAdapter struct {
logger logger.Logger
@@ -1037,6 +1054,11 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
var successCount, failCount int32
var mu sync.Mutex // Protect shared resources (progress, logger)
// Timing tracking for restore phase progress
restorePhaseStart := time.Now()
var completedDBTimes []time.Duration // Track duration for each completed DB restore
var completedDBTimesMu sync.Mutex
// Create semaphore to limit concurrency
semaphore := make(chan struct{}, parallelism)
var wg sync.WaitGroup
@@ -1062,6 +1084,9 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
}
}()
// Track timing for this database restore
dbRestoreStart := time.Now()
// Update estimator progress (thread-safe)
mu.Lock()
estimator.UpdateProgress(idx)
@@ -1074,12 +1099,26 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
dbProgress := 15 + int(float64(idx)/float64(totalDBs)*85.0)
// Calculate average time per DB and report progress with timing
completedDBTimesMu.Lock()
var avgPerDB time.Duration
if len(completedDBTimes) > 0 {
var totalDuration time.Duration
for _, d := range completedDBTimes {
totalDuration += d
}
avgPerDB = totalDuration / time.Duration(len(completedDBTimes))
}
phaseElapsed := time.Since(restorePhaseStart)
completedDBTimesMu.Unlock()
mu.Lock()
statusMsg := fmt.Sprintf("Restoring database %s (%d/%d)", dbName, idx+1, totalDBs)
e.progress.Update(statusMsg)
e.log.Info("Restoring database", "name", dbName, "file", dumpFile, "progress", dbProgress)
// Report database progress for TUI
// Report database progress for TUI (both callbacks)
e.reportDatabaseProgress(idx, totalDBs, dbName)
e.reportDatabaseProgressWithTiming(idx, totalDBs, dbName, phaseElapsed, avgPerDB)
mu.Unlock()
// STEP 1: Drop existing database completely (clean slate)
@@ -1144,6 +1183,12 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
return
}
// Track completed database restore duration for ETA calculation
dbRestoreDuration := time.Since(dbRestoreStart)
completedDBTimesMu.Lock()
completedDBTimes = append(completedDBTimes, dbRestoreDuration)
completedDBTimesMu.Unlock()
atomic.AddInt32(&successCount, 1)
}(dbIndex, entry.Name())
@@ -1937,6 +1982,8 @@ type OriginalSettings struct {
}
// boostPostgreSQLSettings boosts multiple PostgreSQL settings for large restores
// NOTE: max_locks_per_transaction requires a PostgreSQL RESTART to take effect!
// maintenance_work_mem can be changed with pg_reload_conf().
func (e *Engine) boostPostgreSQLSettings(ctx context.Context, lockBoostValue int) (*OriginalSettings, error) {
connStr := e.buildConnString()
db, err := sql.Open("pgx", connStr)
@@ -1956,30 +2003,113 @@ func (e *Engine) boostPostgreSQLSettings(ctx context.Context, lockBoostValue int
// Get current maintenance_work_mem
db.QueryRowContext(ctx, "SHOW maintenance_work_mem").Scan(&original.MaintenanceWorkMem)
// Boost max_locks_per_transaction (if not already high enough)
// CRITICAL: max_locks_per_transaction requires a PostgreSQL RESTART!
// pg_reload_conf() is NOT sufficient for this parameter.
needsRestart := false
if original.MaxLocks < lockBoostValue {
_, err = db.ExecContext(ctx, fmt.Sprintf("ALTER SYSTEM SET max_locks_per_transaction = %d", lockBoostValue))
if err != nil {
e.log.Warn("Could not boost max_locks_per_transaction", "error", err)
e.log.Warn("Could not set max_locks_per_transaction", "error", err)
} else {
needsRestart = true
e.log.Warn("max_locks_per_transaction requires PostgreSQL restart to take effect",
"current", original.MaxLocks,
"target", lockBoostValue)
}
}
// Boost maintenance_work_mem to 2GB for faster index creation
// (this one CAN be applied via pg_reload_conf)
_, err = db.ExecContext(ctx, "ALTER SYSTEM SET maintenance_work_mem = '2GB'")
if err != nil {
e.log.Warn("Could not boost maintenance_work_mem", "error", err)
}
// Reload config to apply changes (no restart needed for these settings)
// Reload config to apply maintenance_work_mem
_, err = db.ExecContext(ctx, "SELECT pg_reload_conf()")
if err != nil {
return original, fmt.Errorf("failed to reload config: %w", err)
}
// If max_locks_per_transaction needs a restart, try to do it
if needsRestart {
if restarted := e.tryRestartPostgreSQL(ctx); restarted {
e.log.Info("PostgreSQL restarted successfully - max_locks_per_transaction now active")
// Wait for PostgreSQL to be ready
time.Sleep(3 * time.Second)
} else {
// Cannot restart - warn user loudly
e.log.Error("=" + strings.Repeat("=", 70))
e.log.Error("WARNING: max_locks_per_transaction change requires PostgreSQL restart!")
e.log.Error("Current value: " + strconv.Itoa(original.MaxLocks) + ", needed: " + strconv.Itoa(lockBoostValue))
e.log.Error("Restore may fail with 'out of shared memory' error on BLOB-heavy databases.")
e.log.Error("")
e.log.Error("To fix manually:")
e.log.Error(" 1. sudo systemctl restart postgresql")
e.log.Error(" 2. Or: sudo -u postgres pg_ctl restart -D $PGDATA")
e.log.Error(" 3. Then re-run the restore")
e.log.Error("=" + strings.Repeat("=", 70))
// Continue anyway - might work for small restores
}
}
return original, nil
}
// tryRestartPostgreSQL attempts to restart PostgreSQL using various methods
// Returns true if restart was successful
func (e *Engine) tryRestartPostgreSQL(ctx context.Context) bool {
e.progress.Update("Attempting PostgreSQL restart for lock settings...")
// Method 1: systemctl (most common on modern Linux)
cmd := exec.CommandContext(ctx, "sudo", "systemctl", "restart", "postgresql")
if err := cmd.Run(); err == nil {
return true
}
// Method 2: systemctl with version suffix (e.g., postgresql-15)
for _, ver := range []string{"17", "16", "15", "14", "13", "12"} {
cmd = exec.CommandContext(ctx, "sudo", "systemctl", "restart", "postgresql-"+ver)
if err := cmd.Run(); err == nil {
return true
}
}
// Method 3: service command (older systems)
cmd = exec.CommandContext(ctx, "sudo", "service", "postgresql", "restart")
if err := cmd.Run(); err == nil {
return true
}
// Method 4: pg_ctl as postgres user
cmd = exec.CommandContext(ctx, "sudo", "-u", "postgres", "pg_ctl", "restart", "-D", "/var/lib/postgresql/data", "-m", "fast")
if err := cmd.Run(); err == nil {
return true
}
// Method 5: Try common PGDATA paths
pgdataPaths := []string{
"/var/lib/pgsql/data",
"/var/lib/pgsql/17/data",
"/var/lib/pgsql/16/data",
"/var/lib/pgsql/15/data",
"/var/lib/postgresql/17/main",
"/var/lib/postgresql/16/main",
"/var/lib/postgresql/15/main",
}
for _, pgdata := range pgdataPaths {
cmd = exec.CommandContext(ctx, "sudo", "-u", "postgres", "pg_ctl", "restart", "-D", pgdata, "-m", "fast")
if err := cmd.Run(); err == nil {
return true
}
}
return false
}
// resetPostgreSQLSettings restores original PostgreSQL settings
// NOTE: max_locks_per_transaction changes are written but require restart to take effect.
// We don't restart here since we're done with the restore.
func (e *Engine) resetPostgreSQLSettings(ctx context.Context, original *OriginalSettings) error {
connStr := e.buildConnString()
db, err := sql.Open("pgx", connStr)
@@ -1988,25 +2118,28 @@ func (e *Engine) resetPostgreSQLSettings(ctx context.Context, original *Original
}
defer db.Close()
// Reset max_locks_per_transaction
// Reset max_locks_per_transaction (will take effect on next restart)
if original.MaxLocks == 64 { // Default
db.ExecContext(ctx, "ALTER SYSTEM RESET max_locks_per_transaction")
} else if original.MaxLocks > 0 {
db.ExecContext(ctx, fmt.Sprintf("ALTER SYSTEM SET max_locks_per_transaction = %d", original.MaxLocks))
}
// Reset maintenance_work_mem
// Reset maintenance_work_mem (takes effect immediately with reload)
if original.MaintenanceWorkMem == "64MB" { // Default
db.ExecContext(ctx, "ALTER SYSTEM RESET maintenance_work_mem")
} else if original.MaintenanceWorkMem != "" {
db.ExecContext(ctx, fmt.Sprintf("ALTER SYSTEM SET maintenance_work_mem = '%s'", original.MaintenanceWorkMem))
}
// Reload config
// Reload config (only maintenance_work_mem will take effect immediately)
_, err = db.ExecContext(ctx, "SELECT pg_reload_conf()")
if err != nil {
return fmt.Errorf("failed to reload config: %w", err)
}
e.log.Info("PostgreSQL settings reset queued",
"note", "max_locks_per_transaction will revert on next PostgreSQL restart")
return nil
}

View File

@@ -201,10 +201,19 @@ func (e *Engine) checkPostgreSQL(ctx context.Context, result *PreflightResult) {
result.PostgreSQL.IsSuperuser = isSuperuser
}
// Add info/warnings
// CRITICAL: max_locks_per_transaction requires PostgreSQL RESTART to change!
// Warn users loudly about this - it's the #1 cause of "out of shared memory" errors
if result.PostgreSQL.MaxLocksPerTransaction < 256 {
e.log.Info("PostgreSQL max_locks_per_transaction is low - will auto-boost",
"current", result.PostgreSQL.MaxLocksPerTransaction)
e.log.Warn("PostgreSQL max_locks_per_transaction is LOW",
"current", result.PostgreSQL.MaxLocksPerTransaction,
"recommended", "256+",
"note", "REQUIRES PostgreSQL restart to change!")
result.Warnings = append(result.Warnings,
fmt.Sprintf("max_locks_per_transaction=%d is low (recommend 256+). "+
"This setting requires PostgreSQL RESTART to change. "+
"BLOB-heavy databases may fail with 'out of shared memory' error.",
result.PostgreSQL.MaxLocksPerTransaction))
}
// Parse shared_buffers and warn if very low

130
internal/tui/backup_exec.go Executable file → Normal file
View File

@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"strings"
"sync"
"time"
tea "github.com/charmbracelet/bubbletea"
@@ -33,6 +34,56 @@ type BackupExecutionModel struct {
startTime time.Time
details []string
spinnerFrame int
// Database count progress (for cluster backup)
dbTotal int
dbDone int
dbName string // Current database being backed up
}
// sharedBackupProgressState holds progress state that can be safely accessed from callbacks
type sharedBackupProgressState struct {
mu sync.Mutex
dbTotal int
dbDone int
dbName string
hasUpdate bool
}
// Package-level shared progress state for backup operations
var (
currentBackupProgressMu sync.Mutex
currentBackupProgressState *sharedBackupProgressState
)
func setCurrentBackupProgress(state *sharedBackupProgressState) {
currentBackupProgressMu.Lock()
defer currentBackupProgressMu.Unlock()
currentBackupProgressState = state
}
func clearCurrentBackupProgress() {
currentBackupProgressMu.Lock()
defer currentBackupProgressMu.Unlock()
currentBackupProgressState = nil
}
func getCurrentBackupProgress() (dbTotal, dbDone int, dbName string, hasUpdate bool) {
currentBackupProgressMu.Lock()
defer currentBackupProgressMu.Unlock()
if currentBackupProgressState == nil {
return 0, 0, "", false
}
currentBackupProgressState.mu.Lock()
defer currentBackupProgressState.mu.Unlock()
hasUpdate = currentBackupProgressState.hasUpdate
currentBackupProgressState.hasUpdate = false
return currentBackupProgressState.dbTotal, currentBackupProgressState.dbDone,
currentBackupProgressState.dbName, hasUpdate
}
func NewBackupExecution(cfg *config.Config, log logger.Logger, parent tea.Model, ctx context.Context, backupType, dbName string, ratio int) BackupExecutionModel {
@@ -55,7 +106,6 @@ func NewBackupExecution(cfg *config.Config, log logger.Logger, parent tea.Model,
}
func (m BackupExecutionModel) Init() tea.Cmd {
// TUI handles all display through View() - no progress callbacks needed
return tea.Batch(
executeBackupWithTUIProgress(m.ctx, m.config, m.logger, m.backupType, m.databaseName, m.ratio),
backupTickCmd(),
@@ -91,6 +141,11 @@ func executeBackupWithTUIProgress(parentCtx context.Context, cfg *config.Config,
start := time.Now()
// Setup shared progress state for TUI polling
progressState := &sharedBackupProgressState{}
setCurrentBackupProgress(progressState)
defer clearCurrentBackupProgress()
dbClient, err := database.New(cfg, log)
if err != nil {
return backupCompleteMsg{
@@ -110,6 +165,16 @@ func executeBackupWithTUIProgress(parentCtx context.Context, cfg *config.Config,
// Pass nil as indicator - TUI itself handles all display, no stdout printing
engine := backup.NewSilent(cfg, log, dbClient, nil)
// Set database progress callback for cluster backups
engine.SetDatabaseProgressCallback(func(done, total int, currentDB string) {
progressState.mu.Lock()
progressState.dbDone = done
progressState.dbTotal = total
progressState.dbName = currentDB
progressState.hasUpdate = true
progressState.mu.Unlock()
})
var backupErr error
switch backupType {
case "single":
@@ -157,10 +222,21 @@ func (m BackupExecutionModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
// Increment spinner frame for smooth animation
m.spinnerFrame = (m.spinnerFrame + 1) % len(spinnerFrames)
// Update status based on elapsed time to show progress
// Poll for database progress updates from callbacks
dbTotal, dbDone, dbName, hasUpdate := getCurrentBackupProgress()
if hasUpdate {
m.dbTotal = dbTotal
m.dbDone = dbDone
m.dbName = dbName
}
// Update status based on progress and elapsed time
elapsedSec := int(time.Since(m.startTime).Seconds())
if elapsedSec < 2 {
if m.dbTotal > 0 && m.dbDone > 0 {
// We have real progress from cluster backup
m.status = fmt.Sprintf("Backing up database: %s", m.dbName)
} else if elapsedSec < 2 {
m.status = "Initializing backup..."
} else if elapsedSec < 5 {
if m.backupType == "cluster" {
@@ -234,6 +310,34 @@ func (m BackupExecutionModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
return m, nil
}
// renderDatabaseProgressBar renders a progress bar for database count progress
func renderBackupDatabaseProgressBar(done, total int, dbName string, width int) string {
if total == 0 {
return ""
}
// Calculate progress percentage
percent := float64(done) / float64(total)
if percent > 1.0 {
percent = 1.0
}
// Calculate filled width
barWidth := width - 20 // Leave room for label and percentage
if barWidth < 10 {
barWidth = 10
}
filled := int(float64(barWidth) * percent)
if filled > barWidth {
filled = barWidth
}
// Build progress bar
bar := strings.Repeat("█", filled) + strings.Repeat("░", barWidth-filled)
return fmt.Sprintf(" Database: [%s] %d/%d", bar, done, total)
}
func (m BackupExecutionModel) View() string {
var s strings.Builder
s.Grow(512) // Pre-allocate estimated capacity for better performance
@@ -255,12 +359,24 @@ func (m BackupExecutionModel) View() string {
s.WriteString(fmt.Sprintf(" %-10s %s\n", "Duration:", time.Since(m.startTime).Round(time.Second)))
s.WriteString("\n")
// Status with spinner
// Status display
if !m.done {
if m.cancelling {
s.WriteString(fmt.Sprintf(" %s %s\n", spinnerFrames[m.spinnerFrame], m.status))
// Show database progress bar if we have progress data (cluster backup)
if m.dbTotal > 0 && m.dbDone > 0 {
// Show progress bar instead of spinner when we have real progress
progressBar := renderBackupDatabaseProgressBar(m.dbDone, m.dbTotal, m.dbName, 50)
s.WriteString(progressBar + "\n")
s.WriteString(fmt.Sprintf(" %s\n", m.status))
} else {
s.WriteString(fmt.Sprintf(" %s %s\n", spinnerFrames[m.spinnerFrame], m.status))
// Show spinner during initial phases
if m.cancelling {
s.WriteString(fmt.Sprintf(" %s %s\n", spinnerFrames[m.spinnerFrame], m.status))
} else {
s.WriteString(fmt.Sprintf(" %s %s\n", spinnerFrames[m.spinnerFrame], m.status))
}
}
if !m.cancelling {
s.WriteString("\n [KEY] Press Ctrl+C or ESC to cancel\n")
}
} else {

View File

@@ -57,6 +57,10 @@ type RestoreExecutionModel struct {
dbTotal int
dbDone int
// Timing info for database restore phase (ETA calculation)
dbPhaseElapsed time.Duration // Elapsed time since restore phase started
dbAvgPerDB time.Duration // Average time per database restore
// Results
done bool
cancelling bool // True when user has requested cancellation
@@ -136,6 +140,10 @@ type sharedProgressState struct {
dbTotal int
dbDone int
// Timing info for database restore phase
dbPhaseElapsed time.Duration // Elapsed time since restore phase started
dbAvgPerDB time.Duration // Average time per database restore
// Rolling window for speed calculation
speedSamples []restoreSpeedSample
}
@@ -163,12 +171,12 @@ func clearCurrentRestoreProgress() {
currentRestoreProgressState = nil
}
func getCurrentRestoreProgress() (bytesTotal, bytesDone int64, description string, hasUpdate bool, dbTotal, dbDone int, speed float64) {
func getCurrentRestoreProgress() (bytesTotal, bytesDone int64, description string, hasUpdate bool, dbTotal, dbDone int, speed float64, dbPhaseElapsed, dbAvgPerDB time.Duration) {
currentRestoreProgressMu.Lock()
defer currentRestoreProgressMu.Unlock()
if currentRestoreProgressState == nil {
return 0, 0, "", false, 0, 0, 0
return 0, 0, "", false, 0, 0, 0, 0, 0
}
currentRestoreProgressState.mu.Lock()
@@ -179,7 +187,8 @@ func getCurrentRestoreProgress() (bytesTotal, bytesDone int64, description strin
return currentRestoreProgressState.bytesTotal, currentRestoreProgressState.bytesDone,
currentRestoreProgressState.description, currentRestoreProgressState.hasUpdate,
currentRestoreProgressState.dbTotal, currentRestoreProgressState.dbDone, speed
currentRestoreProgressState.dbTotal, currentRestoreProgressState.dbDone, speed,
currentRestoreProgressState.dbPhaseElapsed, currentRestoreProgressState.dbAvgPerDB
}
// calculateRollingSpeed calculates speed from recent samples (last 5 seconds)
@@ -304,6 +313,21 @@ func executeRestoreWithTUIProgress(parentCtx context.Context, cfg *config.Config
progressState.bytesDone = 0
})
// Set up timing-aware database progress callback for cluster restore ETA
engine.SetDatabaseProgressWithTimingCallback(func(done, total int, dbName string, phaseElapsed, avgPerDB time.Duration) {
progressState.mu.Lock()
defer progressState.mu.Unlock()
progressState.dbDone = done
progressState.dbTotal = total
progressState.description = fmt.Sprintf("Restoring %s", dbName)
progressState.dbPhaseElapsed = phaseElapsed
progressState.dbAvgPerDB = avgPerDB
progressState.hasUpdate = true
// Clear byte progress when switching to db progress
progressState.bytesTotal = 0
progressState.bytesDone = 0
})
// Store progress state in a package-level variable for the ticker to access
// This is a workaround because tea messages can't be sent from callbacks
setCurrentRestoreProgress(progressState)
@@ -357,7 +381,7 @@ func (m RestoreExecutionModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
m.elapsed = time.Since(m.startTime)
// Poll shared progress state for real-time updates
bytesTotal, bytesDone, description, hasUpdate, dbTotal, dbDone, speed := getCurrentRestoreProgress()
bytesTotal, bytesDone, description, hasUpdate, dbTotal, dbDone, speed, dbPhaseElapsed, dbAvgPerDB := getCurrentRestoreProgress()
if hasUpdate && bytesTotal > 0 {
m.bytesTotal = bytesTotal
m.bytesDone = bytesDone
@@ -370,9 +394,11 @@ func (m RestoreExecutionModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
m.phase = "Extracting"
m.progress = int((bytesDone * 100) / bytesTotal)
} else if hasUpdate && dbTotal > 0 {
// Database count progress for cluster restore
// Database count progress for cluster restore with timing
m.dbTotal = dbTotal
m.dbDone = dbDone
m.dbPhaseElapsed = dbPhaseElapsed
m.dbAvgPerDB = dbAvgPerDB
m.showBytes = false
m.status = fmt.Sprintf("Restoring database %d of %d...", dbDone+1, dbTotal)
m.phase = "Restore"
@@ -549,13 +575,13 @@ func (m RestoreExecutionModel) View() string {
s.WriteString(renderDetailedProgressBarWithSpeed(m.bytesDone, m.bytesTotal, m.speed))
s.WriteString("\n\n")
} else if m.dbTotal > 0 {
// Database count progress for cluster restore
// Database count progress for cluster restore with timing
spinner := m.spinnerFrames[m.spinnerFrame]
s.WriteString(fmt.Sprintf("Status: %s %s\n", spinner, m.status))
s.WriteString("\n")
// Show database progress bar
s.WriteString(renderDatabaseProgressBar(m.dbDone, m.dbTotal))
// Show database progress bar with timing and ETA
s.WriteString(renderDatabaseProgressBarWithTiming(m.dbDone, m.dbTotal, m.dbPhaseElapsed, m.dbAvgPerDB))
s.WriteString("\n\n")
} else {
// Show status with rotating spinner (for phases without detailed progress)
@@ -678,6 +704,55 @@ func renderDatabaseProgressBar(done, total int) string {
return s.String()
}
// renderDatabaseProgressBarWithTiming renders a progress bar for database count with timing and ETA
func renderDatabaseProgressBarWithTiming(done, total int, phaseElapsed, avgPerDB time.Duration) string {
var s strings.Builder
// Calculate percentage
percent := 0
if total > 0 {
percent = (done * 100) / total
if percent > 100 {
percent = 100
}
}
// Render progress bar
width := 30
filled := (percent * width) / 100
barFilled := strings.Repeat("█", filled)
barEmpty := strings.Repeat("░", width-filled)
s.WriteString(successStyle.Render("["))
s.WriteString(successStyle.Render(barFilled))
s.WriteString(infoStyle.Render(barEmpty))
s.WriteString(successStyle.Render("]"))
// Count and percentage
s.WriteString(fmt.Sprintf(" %3d%% %d / %d databases", percent, done, total))
// Timing and ETA
if phaseElapsed > 0 {
s.WriteString(fmt.Sprintf(" [%s", FormatDurationShort(phaseElapsed)))
// Calculate ETA based on average time per database
if avgPerDB > 0 && done < total {
remainingDBs := total - done
eta := time.Duration(remainingDBs) * avgPerDB
s.WriteString(fmt.Sprintf(" / ETA: %s", FormatDurationShort(eta)))
} else if done > 0 && done < total {
// Fallback: estimate ETA from overall elapsed time
avgElapsed := phaseElapsed / time.Duration(done)
remainingDBs := total - done
eta := time.Duration(remainingDBs) * avgElapsed
s.WriteString(fmt.Sprintf(" / ETA: ~%s", FormatDurationShort(eta)))
}
s.WriteString("]")
}
return s.String()
}
// formatDuration formats duration in human readable format
func formatDuration(d time.Duration) string {
if d < time.Minute {