TUI: Add timing and ETA tracking to cluster restore progress
- Add DatabaseProgressWithTimingCallback for timing-aware progress reporting - Track elapsed time and average duration per database during restore phase - Display ETA based on completed database restore times - Show restore phase elapsed time in progress bar - Enhance cluster restore progress bar with [elapsed / ETA: remaining] format
This commit is contained in:
@@ -4,8 +4,8 @@ This directory contains pre-compiled binaries for the DB Backup Tool across mult
|
|||||||
|
|
||||||
## Build Information
|
## Build Information
|
||||||
- **Version**: 3.42.34
|
- **Version**: 3.42.34
|
||||||
- **Build Time**: 2026-01-15_14:33:12_UTC
|
- **Build Time**: 2026-01-15_17:50:35_UTC
|
||||||
- **Git Commit**: 09a9177
|
- **Git Commit**: 4938dc1
|
||||||
|
|
||||||
## Recent Updates (v1.1.0)
|
## Recent Updates (v1.1.0)
|
||||||
- ✅ Fixed TUI progress display with line-by-line output
|
- ✅ Fixed TUI progress display with line-by-line output
|
||||||
|
|||||||
@@ -34,6 +34,10 @@ type ProgressCallback func(current, total int64, description string)
|
|||||||
// DatabaseProgressCallback is called with database count progress during cluster restore
|
// DatabaseProgressCallback is called with database count progress during cluster restore
|
||||||
type DatabaseProgressCallback func(done, total int, dbName string)
|
type DatabaseProgressCallback func(done, total int, dbName string)
|
||||||
|
|
||||||
|
// DatabaseProgressWithTimingCallback is called with database progress including timing info
|
||||||
|
// Parameters: done count, total count, database name, elapsed time for current restore phase, avg duration per DB
|
||||||
|
type DatabaseProgressWithTimingCallback func(done, total int, dbName string, phaseElapsed, avgPerDB time.Duration)
|
||||||
|
|
||||||
// Engine handles database restore operations
|
// Engine handles database restore operations
|
||||||
type Engine struct {
|
type Engine struct {
|
||||||
cfg *config.Config
|
cfg *config.Config
|
||||||
@@ -47,6 +51,7 @@ type Engine struct {
|
|||||||
// TUI progress callback for detailed progress reporting
|
// TUI progress callback for detailed progress reporting
|
||||||
progressCallback ProgressCallback
|
progressCallback ProgressCallback
|
||||||
dbProgressCallback DatabaseProgressCallback
|
dbProgressCallback DatabaseProgressCallback
|
||||||
|
dbProgressTimingCallback DatabaseProgressWithTimingCallback
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new restore engine
|
// New creates a new restore engine
|
||||||
@@ -112,6 +117,11 @@ func (e *Engine) SetDatabaseProgressCallback(cb DatabaseProgressCallback) {
|
|||||||
e.dbProgressCallback = cb
|
e.dbProgressCallback = cb
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetDatabaseProgressWithTimingCallback sets a callback for database progress with timing info
|
||||||
|
func (e *Engine) SetDatabaseProgressWithTimingCallback(cb DatabaseProgressWithTimingCallback) {
|
||||||
|
e.dbProgressTimingCallback = cb
|
||||||
|
}
|
||||||
|
|
||||||
// reportProgress safely calls the progress callback if set
|
// reportProgress safely calls the progress callback if set
|
||||||
func (e *Engine) reportProgress(current, total int64, description string) {
|
func (e *Engine) reportProgress(current, total int64, description string) {
|
||||||
if e.progressCallback != nil {
|
if e.progressCallback != nil {
|
||||||
@@ -126,6 +136,13 @@ func (e *Engine) reportDatabaseProgress(done, total int, dbName string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// reportDatabaseProgressWithTiming safely calls the timing-aware callback if set
|
||||||
|
func (e *Engine) reportDatabaseProgressWithTiming(done, total int, dbName string, phaseElapsed, avgPerDB time.Duration) {
|
||||||
|
if e.dbProgressTimingCallback != nil {
|
||||||
|
e.dbProgressTimingCallback(done, total, dbName, phaseElapsed, avgPerDB)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// loggerAdapter adapts our logger to the progress.Logger interface
|
// loggerAdapter adapts our logger to the progress.Logger interface
|
||||||
type loggerAdapter struct {
|
type loggerAdapter struct {
|
||||||
logger logger.Logger
|
logger logger.Logger
|
||||||
@@ -1037,6 +1054,11 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
|
|||||||
var successCount, failCount int32
|
var successCount, failCount int32
|
||||||
var mu sync.Mutex // Protect shared resources (progress, logger)
|
var mu sync.Mutex // Protect shared resources (progress, logger)
|
||||||
|
|
||||||
|
// Timing tracking for restore phase progress
|
||||||
|
restorePhaseStart := time.Now()
|
||||||
|
var completedDBTimes []time.Duration // Track duration for each completed DB restore
|
||||||
|
var completedDBTimesMu sync.Mutex
|
||||||
|
|
||||||
// Create semaphore to limit concurrency
|
// Create semaphore to limit concurrency
|
||||||
semaphore := make(chan struct{}, parallelism)
|
semaphore := make(chan struct{}, parallelism)
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
@@ -1062,6 +1084,9 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
// Track timing for this database restore
|
||||||
|
dbRestoreStart := time.Now()
|
||||||
|
|
||||||
// Update estimator progress (thread-safe)
|
// Update estimator progress (thread-safe)
|
||||||
mu.Lock()
|
mu.Lock()
|
||||||
estimator.UpdateProgress(idx)
|
estimator.UpdateProgress(idx)
|
||||||
@@ -1074,12 +1099,26 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
|
|||||||
|
|
||||||
dbProgress := 15 + int(float64(idx)/float64(totalDBs)*85.0)
|
dbProgress := 15 + int(float64(idx)/float64(totalDBs)*85.0)
|
||||||
|
|
||||||
|
// Calculate average time per DB and report progress with timing
|
||||||
|
completedDBTimesMu.Lock()
|
||||||
|
var avgPerDB time.Duration
|
||||||
|
if len(completedDBTimes) > 0 {
|
||||||
|
var totalDuration time.Duration
|
||||||
|
for _, d := range completedDBTimes {
|
||||||
|
totalDuration += d
|
||||||
|
}
|
||||||
|
avgPerDB = totalDuration / time.Duration(len(completedDBTimes))
|
||||||
|
}
|
||||||
|
phaseElapsed := time.Since(restorePhaseStart)
|
||||||
|
completedDBTimesMu.Unlock()
|
||||||
|
|
||||||
mu.Lock()
|
mu.Lock()
|
||||||
statusMsg := fmt.Sprintf("Restoring database %s (%d/%d)", dbName, idx+1, totalDBs)
|
statusMsg := fmt.Sprintf("Restoring database %s (%d/%d)", dbName, idx+1, totalDBs)
|
||||||
e.progress.Update(statusMsg)
|
e.progress.Update(statusMsg)
|
||||||
e.log.Info("Restoring database", "name", dbName, "file", dumpFile, "progress", dbProgress)
|
e.log.Info("Restoring database", "name", dbName, "file", dumpFile, "progress", dbProgress)
|
||||||
// Report database progress for TUI
|
// Report database progress for TUI (both callbacks)
|
||||||
e.reportDatabaseProgress(idx, totalDBs, dbName)
|
e.reportDatabaseProgress(idx, totalDBs, dbName)
|
||||||
|
e.reportDatabaseProgressWithTiming(idx, totalDBs, dbName, phaseElapsed, avgPerDB)
|
||||||
mu.Unlock()
|
mu.Unlock()
|
||||||
|
|
||||||
// STEP 1: Drop existing database completely (clean slate)
|
// STEP 1: Drop existing database completely (clean slate)
|
||||||
@@ -1144,6 +1183,12 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Track completed database restore duration for ETA calculation
|
||||||
|
dbRestoreDuration := time.Since(dbRestoreStart)
|
||||||
|
completedDBTimesMu.Lock()
|
||||||
|
completedDBTimes = append(completedDBTimes, dbRestoreDuration)
|
||||||
|
completedDBTimesMu.Unlock()
|
||||||
|
|
||||||
atomic.AddInt32(&successCount, 1)
|
atomic.AddInt32(&successCount, 1)
|
||||||
}(dbIndex, entry.Name())
|
}(dbIndex, entry.Name())
|
||||||
|
|
||||||
|
|||||||
@@ -57,6 +57,10 @@ type RestoreExecutionModel struct {
|
|||||||
dbTotal int
|
dbTotal int
|
||||||
dbDone int
|
dbDone int
|
||||||
|
|
||||||
|
// Timing info for database restore phase (ETA calculation)
|
||||||
|
dbPhaseElapsed time.Duration // Elapsed time since restore phase started
|
||||||
|
dbAvgPerDB time.Duration // Average time per database restore
|
||||||
|
|
||||||
// Results
|
// Results
|
||||||
done bool
|
done bool
|
||||||
cancelling bool // True when user has requested cancellation
|
cancelling bool // True when user has requested cancellation
|
||||||
@@ -136,6 +140,10 @@ type sharedProgressState struct {
|
|||||||
dbTotal int
|
dbTotal int
|
||||||
dbDone int
|
dbDone int
|
||||||
|
|
||||||
|
// Timing info for database restore phase
|
||||||
|
dbPhaseElapsed time.Duration // Elapsed time since restore phase started
|
||||||
|
dbAvgPerDB time.Duration // Average time per database restore
|
||||||
|
|
||||||
// Rolling window for speed calculation
|
// Rolling window for speed calculation
|
||||||
speedSamples []restoreSpeedSample
|
speedSamples []restoreSpeedSample
|
||||||
}
|
}
|
||||||
@@ -163,12 +171,12 @@ func clearCurrentRestoreProgress() {
|
|||||||
currentRestoreProgressState = nil
|
currentRestoreProgressState = nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func getCurrentRestoreProgress() (bytesTotal, bytesDone int64, description string, hasUpdate bool, dbTotal, dbDone int, speed float64) {
|
func getCurrentRestoreProgress() (bytesTotal, bytesDone int64, description string, hasUpdate bool, dbTotal, dbDone int, speed float64, dbPhaseElapsed, dbAvgPerDB time.Duration) {
|
||||||
currentRestoreProgressMu.Lock()
|
currentRestoreProgressMu.Lock()
|
||||||
defer currentRestoreProgressMu.Unlock()
|
defer currentRestoreProgressMu.Unlock()
|
||||||
|
|
||||||
if currentRestoreProgressState == nil {
|
if currentRestoreProgressState == nil {
|
||||||
return 0, 0, "", false, 0, 0, 0
|
return 0, 0, "", false, 0, 0, 0, 0, 0
|
||||||
}
|
}
|
||||||
|
|
||||||
currentRestoreProgressState.mu.Lock()
|
currentRestoreProgressState.mu.Lock()
|
||||||
@@ -179,7 +187,8 @@ func getCurrentRestoreProgress() (bytesTotal, bytesDone int64, description strin
|
|||||||
|
|
||||||
return currentRestoreProgressState.bytesTotal, currentRestoreProgressState.bytesDone,
|
return currentRestoreProgressState.bytesTotal, currentRestoreProgressState.bytesDone,
|
||||||
currentRestoreProgressState.description, currentRestoreProgressState.hasUpdate,
|
currentRestoreProgressState.description, currentRestoreProgressState.hasUpdate,
|
||||||
currentRestoreProgressState.dbTotal, currentRestoreProgressState.dbDone, speed
|
currentRestoreProgressState.dbTotal, currentRestoreProgressState.dbDone, speed,
|
||||||
|
currentRestoreProgressState.dbPhaseElapsed, currentRestoreProgressState.dbAvgPerDB
|
||||||
}
|
}
|
||||||
|
|
||||||
// calculateRollingSpeed calculates speed from recent samples (last 5 seconds)
|
// calculateRollingSpeed calculates speed from recent samples (last 5 seconds)
|
||||||
@@ -304,6 +313,21 @@ func executeRestoreWithTUIProgress(parentCtx context.Context, cfg *config.Config
|
|||||||
progressState.bytesDone = 0
|
progressState.bytesDone = 0
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Set up timing-aware database progress callback for cluster restore ETA
|
||||||
|
engine.SetDatabaseProgressWithTimingCallback(func(done, total int, dbName string, phaseElapsed, avgPerDB time.Duration) {
|
||||||
|
progressState.mu.Lock()
|
||||||
|
defer progressState.mu.Unlock()
|
||||||
|
progressState.dbDone = done
|
||||||
|
progressState.dbTotal = total
|
||||||
|
progressState.description = fmt.Sprintf("Restoring %s", dbName)
|
||||||
|
progressState.dbPhaseElapsed = phaseElapsed
|
||||||
|
progressState.dbAvgPerDB = avgPerDB
|
||||||
|
progressState.hasUpdate = true
|
||||||
|
// Clear byte progress when switching to db progress
|
||||||
|
progressState.bytesTotal = 0
|
||||||
|
progressState.bytesDone = 0
|
||||||
|
})
|
||||||
|
|
||||||
// Store progress state in a package-level variable for the ticker to access
|
// Store progress state in a package-level variable for the ticker to access
|
||||||
// This is a workaround because tea messages can't be sent from callbacks
|
// This is a workaround because tea messages can't be sent from callbacks
|
||||||
setCurrentRestoreProgress(progressState)
|
setCurrentRestoreProgress(progressState)
|
||||||
@@ -357,7 +381,7 @@ func (m RestoreExecutionModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||||||
m.elapsed = time.Since(m.startTime)
|
m.elapsed = time.Since(m.startTime)
|
||||||
|
|
||||||
// Poll shared progress state for real-time updates
|
// Poll shared progress state for real-time updates
|
||||||
bytesTotal, bytesDone, description, hasUpdate, dbTotal, dbDone, speed := getCurrentRestoreProgress()
|
bytesTotal, bytesDone, description, hasUpdate, dbTotal, dbDone, speed, dbPhaseElapsed, dbAvgPerDB := getCurrentRestoreProgress()
|
||||||
if hasUpdate && bytesTotal > 0 {
|
if hasUpdate && bytesTotal > 0 {
|
||||||
m.bytesTotal = bytesTotal
|
m.bytesTotal = bytesTotal
|
||||||
m.bytesDone = bytesDone
|
m.bytesDone = bytesDone
|
||||||
@@ -370,9 +394,11 @@ func (m RestoreExecutionModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||||||
m.phase = "Extracting"
|
m.phase = "Extracting"
|
||||||
m.progress = int((bytesDone * 100) / bytesTotal)
|
m.progress = int((bytesDone * 100) / bytesTotal)
|
||||||
} else if hasUpdate && dbTotal > 0 {
|
} else if hasUpdate && dbTotal > 0 {
|
||||||
// Database count progress for cluster restore
|
// Database count progress for cluster restore with timing
|
||||||
m.dbTotal = dbTotal
|
m.dbTotal = dbTotal
|
||||||
m.dbDone = dbDone
|
m.dbDone = dbDone
|
||||||
|
m.dbPhaseElapsed = dbPhaseElapsed
|
||||||
|
m.dbAvgPerDB = dbAvgPerDB
|
||||||
m.showBytes = false
|
m.showBytes = false
|
||||||
m.status = fmt.Sprintf("Restoring database %d of %d...", dbDone+1, dbTotal)
|
m.status = fmt.Sprintf("Restoring database %d of %d...", dbDone+1, dbTotal)
|
||||||
m.phase = "Restore"
|
m.phase = "Restore"
|
||||||
@@ -549,13 +575,13 @@ func (m RestoreExecutionModel) View() string {
|
|||||||
s.WriteString(renderDetailedProgressBarWithSpeed(m.bytesDone, m.bytesTotal, m.speed))
|
s.WriteString(renderDetailedProgressBarWithSpeed(m.bytesDone, m.bytesTotal, m.speed))
|
||||||
s.WriteString("\n\n")
|
s.WriteString("\n\n")
|
||||||
} else if m.dbTotal > 0 {
|
} else if m.dbTotal > 0 {
|
||||||
// Database count progress for cluster restore
|
// Database count progress for cluster restore with timing
|
||||||
spinner := m.spinnerFrames[m.spinnerFrame]
|
spinner := m.spinnerFrames[m.spinnerFrame]
|
||||||
s.WriteString(fmt.Sprintf("Status: %s %s\n", spinner, m.status))
|
s.WriteString(fmt.Sprintf("Status: %s %s\n", spinner, m.status))
|
||||||
s.WriteString("\n")
|
s.WriteString("\n")
|
||||||
|
|
||||||
// Show database progress bar
|
// Show database progress bar with timing and ETA
|
||||||
s.WriteString(renderDatabaseProgressBar(m.dbDone, m.dbTotal))
|
s.WriteString(renderDatabaseProgressBarWithTiming(m.dbDone, m.dbTotal, m.dbPhaseElapsed, m.dbAvgPerDB))
|
||||||
s.WriteString("\n\n")
|
s.WriteString("\n\n")
|
||||||
} else {
|
} else {
|
||||||
// Show status with rotating spinner (for phases without detailed progress)
|
// Show status with rotating spinner (for phases without detailed progress)
|
||||||
@@ -678,6 +704,55 @@ func renderDatabaseProgressBar(done, total int) string {
|
|||||||
return s.String()
|
return s.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// renderDatabaseProgressBarWithTiming renders a progress bar for database count with timing and ETA
|
||||||
|
func renderDatabaseProgressBarWithTiming(done, total int, phaseElapsed, avgPerDB time.Duration) string {
|
||||||
|
var s strings.Builder
|
||||||
|
|
||||||
|
// Calculate percentage
|
||||||
|
percent := 0
|
||||||
|
if total > 0 {
|
||||||
|
percent = (done * 100) / total
|
||||||
|
if percent > 100 {
|
||||||
|
percent = 100
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Render progress bar
|
||||||
|
width := 30
|
||||||
|
filled := (percent * width) / 100
|
||||||
|
barFilled := strings.Repeat("█", filled)
|
||||||
|
barEmpty := strings.Repeat("░", width-filled)
|
||||||
|
|
||||||
|
s.WriteString(successStyle.Render("["))
|
||||||
|
s.WriteString(successStyle.Render(barFilled))
|
||||||
|
s.WriteString(infoStyle.Render(barEmpty))
|
||||||
|
s.WriteString(successStyle.Render("]"))
|
||||||
|
|
||||||
|
// Count and percentage
|
||||||
|
s.WriteString(fmt.Sprintf(" %3d%% %d / %d databases", percent, done, total))
|
||||||
|
|
||||||
|
// Timing and ETA
|
||||||
|
if phaseElapsed > 0 {
|
||||||
|
s.WriteString(fmt.Sprintf(" [%s", FormatDurationShort(phaseElapsed)))
|
||||||
|
|
||||||
|
// Calculate ETA based on average time per database
|
||||||
|
if avgPerDB > 0 && done < total {
|
||||||
|
remainingDBs := total - done
|
||||||
|
eta := time.Duration(remainingDBs) * avgPerDB
|
||||||
|
s.WriteString(fmt.Sprintf(" / ETA: %s", FormatDurationShort(eta)))
|
||||||
|
} else if done > 0 && done < total {
|
||||||
|
// Fallback: estimate ETA from overall elapsed time
|
||||||
|
avgElapsed := phaseElapsed / time.Duration(done)
|
||||||
|
remainingDBs := total - done
|
||||||
|
eta := time.Duration(remainingDBs) * avgElapsed
|
||||||
|
s.WriteString(fmt.Sprintf(" / ETA: ~%s", FormatDurationShort(eta)))
|
||||||
|
}
|
||||||
|
s.WriteString("]")
|
||||||
|
}
|
||||||
|
|
||||||
|
return s.String()
|
||||||
|
}
|
||||||
|
|
||||||
// formatDuration formats duration in human readable format
|
// formatDuration formats duration in human readable format
|
||||||
func formatDuration(d time.Duration) string {
|
func formatDuration(d time.Duration) string {
|
||||||
if d < time.Minute {
|
if d < time.Minute {
|
||||||
|
|||||||
Reference in New Issue
Block a user