From 59a717abe7b9e7aeff868f5af963e7d42c1d82d4 Mon Sep 17 00:00:00 2001 From: Alexander Renz Date: Sun, 18 Jan 2026 12:39:21 +0100 Subject: [PATCH] refactor(profiles): replace large-db profile with composable LargeDBMode BREAKING CHANGE: Removed 'large-db' as standalone profile New Design: - Resource Profiles now purely represent VM capacity: conservative, balanced, performance, max-performance - LargeDBMode is a separate boolean toggle that modifies any profile: - Reduces ClusterParallelism and Jobs by 50% - Forces MaxLocksPerTxn = 8192 - Increases MaintenanceWorkMem TUI Changes: - 'l' key now toggles LargeDBMode ON/OFF instead of applying large-db profile - New 'Large DB Mode' setting in settings menu - Settings are persisted to .dbbackup.conf This allows any resource profile to be combined with large database optimization, giving users more flexibility on both small and large VMs. --- README.md | 8 ++-- bin/README.md | 4 +- internal/config/config.go | 19 +++++++-- internal/config/persist.go | 62 ++++++++++++++++++--------- internal/cpu/profiles.go | 74 +++++++++++++++++++++++---------- internal/tui/restore_exec.go | 6 +-- internal/tui/restore_preview.go | 4 ++ internal/tui/settings.go | 62 +++++++++++++++++++-------- 8 files changed, 169 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index 148119a..d491345 100644 --- a/README.md +++ b/README.md @@ -221,7 +221,7 @@ Configuration Settings Database User: postgres SSL Mode: prefer -[KEYS] ↑↓ navigate | Enter edit | 'l' large-db | 'c' conservative | 'p' recommend | 's' save | 'q' menu +[KEYS] ↑↓ navigate | Enter edit | 'l' toggle LargeDB | 'c' conservative | 'p' recommend | 's' save | 'q' menu ``` **Resource Profiles for Large Databases:** @@ -233,9 +233,11 @@ When restoring large databases on VMs with limited resources, use the resource p | conservative | 1 | 1 | Small VMs (<16GB RAM) | | balanced | 2 | 2-4 | Medium VMs (16-32GB RAM) | | performance | 4 | 4-8 | Large servers (32GB+ RAM) | -| large-db | 1 | 2 | Large databases on any hardware | +| max-performance | 8 | 8-16 | High-end servers (64GB+) | -**Quick shortcuts:** Press `l` to apply large-db profile, `c` for conservative, `p` to show recommendation. +**Large DB Mode:** Toggle with `l` key. Reduces parallelism by 50% and sets max_locks_per_transaction=8192 for complex databases with many tables/LOBs. + +**Quick shortcuts:** Press `l` to toggle Large DB Mode, `c` for conservative, `p` to show recommendation. **Database Status:** ``` diff --git a/bin/README.md b/bin/README.md index 779413a..d52074d 100644 --- a/bin/README.md +++ b/bin/README.md @@ -4,8 +4,8 @@ This directory contains pre-compiled binaries for the DB Backup Tool across mult ## Build Information - **Version**: 3.42.50 -- **Build Time**: 2026-01-18_11:06:11_UTC -- **Git Commit**: ea4337e +- **Build Time**: 2026-01-18_11:19:47_UTC +- **Git Commit**: 490a12f ## Recent Updates (v1.1.0) - ✅ Fixed TUI progress display with line-by-line output diff --git a/internal/config/config.go b/internal/config/config.go index b8e4bf8..79f106f 100755 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -37,7 +37,8 @@ type Config struct { CPUWorkloadType string // "cpu-intensive", "io-intensive", "balanced" // Resource profile for backup/restore operations - ResourceProfile string // "conservative", "balanced", "performance", "max-performance", "large-db" + ResourceProfile string // "conservative", "balanced", "performance", "max-performance" + LargeDBMode bool // Enable large database mode (reduces parallelism, increases max_locks) // CPU detection CPUDetector *cpu.Detector @@ -209,6 +210,7 @@ func New() *Config { AutoDetectCores: getEnvBool("AUTO_DETECT_CORES", true), CPUWorkloadType: getEnvString("CPU_WORKLOAD_TYPE", "balanced"), ResourceProfile: defaultProfile, + LargeDBMode: getEnvBool("LARGE_DB_MODE", false), // CPU and memory detection CPUDetector: cpuDetector, @@ -430,7 +432,7 @@ func (c *Config) ApplyResourceProfile(profileName string) error { return &ConfigError{ Field: "resource_profile", Value: profileName, - Message: "unknown profile. Valid profiles: conservative, balanced, performance, max-performance, large-db", + Message: "unknown profile. Valid profiles: conservative, balanced, performance, max-performance", } } @@ -457,8 +459,19 @@ func (c *Config) GetResourceProfileRecommendation(isLargeDB bool) (string, strin } // GetCurrentProfile returns the current resource profile details +// If LargeDBMode is enabled, returns a modified profile with reduced parallelism func (c *Config) GetCurrentProfile() *cpu.ResourceProfile { - return cpu.GetProfileByName(c.ResourceProfile) + profile := cpu.GetProfileByName(c.ResourceProfile) + if profile == nil { + return nil + } + + // Apply LargeDBMode modifier if enabled + if c.LargeDBMode { + return cpu.ApplyLargeDBMode(profile) + } + + return profile } // GetCPUInfo returns CPU information, detecting if necessary diff --git a/internal/config/persist.go b/internal/config/persist.go index 568f9f0..5fd7c5d 100755 --- a/internal/config/persist.go +++ b/internal/config/persist.go @@ -28,9 +28,11 @@ type LocalConfig struct { DumpJobs int // Performance settings - CPUWorkload string - MaxCores int - ClusterTimeout int // Cluster operation timeout in minutes (default: 1440 = 24 hours) + CPUWorkload string + MaxCores int + ClusterTimeout int // Cluster operation timeout in minutes (default: 1440 = 24 hours) + ResourceProfile string + LargeDBMode bool // Enable large database mode (reduces parallelism, increases locks) // Security settings RetentionDays int @@ -126,6 +128,10 @@ func LoadLocalConfig() (*LocalConfig, error) { if ct, err := strconv.Atoi(value); err == nil { cfg.ClusterTimeout = ct } + case "resource_profile": + cfg.ResourceProfile = value + case "large_db_mode": + cfg.LargeDBMode = value == "true" || value == "1" } case "security": switch key { @@ -207,6 +213,12 @@ func SaveLocalConfig(cfg *LocalConfig) error { if cfg.ClusterTimeout != 0 { sb.WriteString(fmt.Sprintf("cluster_timeout = %d\n", cfg.ClusterTimeout)) } + if cfg.ResourceProfile != "" { + sb.WriteString(fmt.Sprintf("resource_profile = %s\n", cfg.ResourceProfile)) + } + if cfg.LargeDBMode { + sb.WriteString("large_db_mode = true\n") + } sb.WriteString("\n") // Security section @@ -280,6 +292,14 @@ func ApplyLocalConfig(cfg *Config, local *LocalConfig) { if local.ClusterTimeout != 0 { cfg.ClusterTimeoutMinutes = local.ClusterTimeout } + // Apply resource profile settings + if local.ResourceProfile != "" { + cfg.ResourceProfile = local.ResourceProfile + } + // LargeDBMode is a boolean - apply if true in config + if local.LargeDBMode { + cfg.LargeDBMode = true + } if cfg.RetentionDays == 30 && local.RetentionDays != 0 { cfg.RetentionDays = local.RetentionDays } @@ -294,22 +314,24 @@ func ApplyLocalConfig(cfg *Config, local *LocalConfig) { // ConfigFromConfig creates a LocalConfig from a Config func ConfigFromConfig(cfg *Config) *LocalConfig { return &LocalConfig{ - DBType: cfg.DatabaseType, - Host: cfg.Host, - Port: cfg.Port, - User: cfg.User, - Database: cfg.Database, - SSLMode: cfg.SSLMode, - BackupDir: cfg.BackupDir, - WorkDir: cfg.WorkDir, - Compression: cfg.CompressionLevel, - Jobs: cfg.Jobs, - DumpJobs: cfg.DumpJobs, - CPUWorkload: cfg.CPUWorkloadType, - MaxCores: cfg.MaxCores, - ClusterTimeout: cfg.ClusterTimeoutMinutes, - RetentionDays: cfg.RetentionDays, - MinBackups: cfg.MinBackups, - MaxRetries: cfg.MaxRetries, + DBType: cfg.DatabaseType, + Host: cfg.Host, + Port: cfg.Port, + User: cfg.User, + Database: cfg.Database, + SSLMode: cfg.SSLMode, + BackupDir: cfg.BackupDir, + WorkDir: cfg.WorkDir, + Compression: cfg.CompressionLevel, + Jobs: cfg.Jobs, + DumpJobs: cfg.DumpJobs, + CPUWorkload: cfg.CPUWorkloadType, + MaxCores: cfg.MaxCores, + ClusterTimeout: cfg.ClusterTimeoutMinutes, + ResourceProfile: cfg.ResourceProfile, + LargeDBMode: cfg.LargeDBMode, + RetentionDays: cfg.RetentionDays, + MinBackups: cfg.MinBackups, + MaxRetries: cfg.MaxRetries, } } diff --git a/internal/cpu/profiles.go b/internal/cpu/profiles.go index db9f9d1..aa2fb36 100644 --- a/internal/cpu/profiles.go +++ b/internal/cpu/profiles.go @@ -90,32 +90,17 @@ var ( DumpJobs: 16, MaintenanceWorkMem: "2GB", MaxLocksPerTxn: 512, - RecommendedForLarge: false, // Large DBs should use conservative + RecommendedForLarge: false, // Large DBs should use LargeDBMode MinMemoryGB: 64, MinCores: 16, } - // ProfileLargeDB - Optimized specifically for large databases - ProfileLargeDB = ResourceProfile{ - Name: "large-db", - Description: "Optimized for large databases with many tables/BLOBs. Prevents 'out of shared memory' errors.", - ClusterParallelism: 1, - Jobs: 2, - DumpJobs: 2, - MaintenanceWorkMem: "1GB", - MaxLocksPerTxn: 8192, - RecommendedForLarge: true, - MinMemoryGB: 8, - MinCores: 2, - } - - // AllProfiles contains all available profiles + // AllProfiles contains all available profiles (VM resource-based) AllProfiles = []ResourceProfile{ ProfileConservative, ProfileBalanced, ProfilePerformance, ProfileMaxPerformance, - ProfileLargeDB, } ) @@ -129,6 +114,51 @@ func GetProfileByName(name string) *ResourceProfile { return nil } +// ApplyLargeDBMode modifies a profile for large database operations. +// This is a modifier that reduces parallelism and increases max_locks_per_transaction +// to prevent "out of shared memory" errors with large databases (many tables, LOBs, etc.). +// It returns a new profile with adjusted settings, leaving the original unchanged. +func ApplyLargeDBMode(profile *ResourceProfile) *ResourceProfile { + if profile == nil { + return nil + } + + // Create a copy with adjusted settings + modified := *profile + + // Add "(large-db)" suffix to indicate this is modified + modified.Name = profile.Name + " +large-db" + modified.Description = fmt.Sprintf("%s [LargeDBMode: reduced parallelism, high locks]", profile.Description) + + // Reduce parallelism to avoid lock exhaustion + // Rule: halve parallelism, minimum 1 + modified.ClusterParallelism = max(1, profile.ClusterParallelism/2) + modified.Jobs = max(1, profile.Jobs/2) + modified.DumpJobs = max(2, profile.DumpJobs/2) + + // Force high max_locks_per_transaction for large schemas + modified.MaxLocksPerTxn = 8192 + + // Increase maintenance_work_mem for complex operations + // Keep or boost maintenance work mem + modified.MaintenanceWorkMem = "1GB" + if profile.MinMemoryGB >= 32 { + modified.MaintenanceWorkMem = "2GB" + } + + modified.RecommendedForLarge = true + + return &modified +} + +// max returns the larger of two integers +func max(a, b int) int { + if a > b { + return a + } + return b +} + // DetectMemory detects system memory information func DetectMemory() (*MemoryInfo, error) { info := &MemoryInfo{ @@ -293,11 +323,11 @@ func RecommendProfile(cpuInfo *CPUInfo, memInfo *MemoryInfo, isLargeDB bool) *Re memGB = memInfo.TotalGB } - // Special case: large databases should always use conservative/large-db profile + // Special case: large databases should use conservative profile + // The caller should also enable LargeDBMode for increased MaxLocksPerTxn if isLargeDB { - if memGB >= 32 && cores >= 8 { - return &ProfileLargeDB // Still conservative but with more memory for maintenance - } + // For large DBs, recommend conservative regardless of resources + // LargeDBMode flag will handle the lock settings separately return &ProfileConservative } @@ -339,7 +369,7 @@ func RecommendProfileWithReason(cpuInfo *CPUInfo, memInfo *MemoryInfo, isLargeDB profile := RecommendProfile(cpuInfo, memInfo, isLargeDB) if isLargeDB { - reason.WriteString("Large database detected - using conservative settings to avoid 'out of shared memory' errors.") + reason.WriteString("Large database mode - using conservative settings. Enable LargeDBMode for higher max_locks.") } else if profile.Name == "conservative" { reason.WriteString("Limited resources detected - using conservative profile for stability.") } else if profile.Name == "max-performance" { diff --git a/internal/tui/restore_exec.go b/internal/tui/restore_exec.go index c87d939..342cf8a 100755 --- a/internal/tui/restore_exec.go +++ b/internal/tui/restore_exec.go @@ -1159,8 +1159,8 @@ func formatRestoreError(errStr string) string { s.WriteString(" If you reduced VM size or max_connections, you need higher\n") s.WriteString(" max_locks_per_transaction to compensate.\n\n") s.WriteString(successStyle.Render(" FIX OPTIONS:\n")) - s.WriteString(" 1. Use 'conservative' or 'large-db' profile in Settings\n") - s.WriteString(" (press 'l' for large-db, 'c' for conservative)\n\n") + s.WriteString(" 1. Enable 'Large DB Mode' in Settings\n") + s.WriteString(" (press 'l' to toggle, reduces parallelism, increases locks)\n\n") s.WriteString(" 2. Increase PostgreSQL locks:\n") s.WriteString(" ALTER SYSTEM SET max_locks_per_transaction = 4096;\n") s.WriteString(" Then RESTART PostgreSQL.\n\n") @@ -1186,7 +1186,7 @@ func formatRestoreError(errStr string) string { s.WriteString("\n\n") s.WriteString(" 1. Check the full error log for details\n") s.WriteString(" 2. Try restoring with 'conservative' profile (press 'c')\n") - s.WriteString(" 3. For large databases, use 'large-db' profile (press 'l')\n") + s.WriteString(" 3. For complex databases, enable 'Large DB Mode' (press 'l')\n") } s.WriteString("\n") diff --git a/internal/tui/restore_preview.go b/internal/tui/restore_preview.go index 379520e..11ff7dd 100755 --- a/internal/tui/restore_preview.go +++ b/internal/tui/restore_preview.go @@ -410,6 +410,10 @@ func (m RestorePreviewModel) View() string { } else { s.WriteString(fmt.Sprintf(" Resource Profile: %s\n", m.config.ResourceProfile)) } + // Show Large DB Mode status + if m.config.LargeDBMode { + s.WriteString(" Large DB Mode: ON (reduced parallelism, high locks)\n") + } s.WriteString(fmt.Sprintf(" CPU Workload: %s\n", m.config.CPUWorkloadType)) s.WriteString(fmt.Sprintf(" Cluster Parallelism: %d databases\n", m.config.ClusterParallelism)) diff --git a/internal/tui/settings.go b/internal/tui/settings.go index c44b905..c0e2264 100755 --- a/internal/tui/settings.go +++ b/internal/tui/settings.go @@ -113,7 +113,7 @@ func NewSettingsModel(cfg *config.Config, log logger.Logger, parent tea.Model) S return c.ResourceProfile }, Update: func(c *config.Config, v string) error { - profiles := []string{"conservative", "balanced", "performance", "max-performance", "large-db"} + profiles := []string{"conservative", "balanced", "performance", "max-performance"} currentIdx := 0 for i, p := range profiles { if c.ResourceProfile == p { @@ -125,7 +125,23 @@ func NewSettingsModel(cfg *config.Config, log logger.Logger, parent tea.Model) S return c.ApplyResourceProfile(profiles[nextIdx]) }, Type: "selector", - Description: "Resource profile for backup/restore. Use 'conservative' or 'large-db' for large databases on small VMs.", + Description: "Resource profile for VM capacity. Toggle 'l' for Large DB Mode on any profile.", + }, + { + Key: "large_db_mode", + DisplayName: "Large DB Mode", + Value: func(c *config.Config) string { + if c.LargeDBMode { + return "ON (↓parallelism, ↑locks)" + } + return "OFF" + }, + Update: func(c *config.Config, v string) error { + c.LargeDBMode = !c.LargeDBMode + return nil + }, + Type: "selector", + Description: "Enable for databases with many tables/LOBs. Reduces parallelism, increases max_locks_per_transaction.", }, { Key: "cluster_parallelism", @@ -574,8 +590,8 @@ func (m SettingsModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return m.saveSettings() case "l": - // Quick shortcut: Apply "large-db" profile for large databases - return m.applyLargeDBProfile() + // Quick shortcut: Toggle Large DB Mode + return m.toggleLargeDBMode() case "c": // Quick shortcut: Apply "conservative" profile for constrained VMs @@ -590,13 +606,20 @@ func (m SettingsModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return m, nil } -// applyLargeDBProfile applies the large-db profile optimized for large databases -func (m SettingsModel) applyLargeDBProfile() (tea.Model, tea.Cmd) { - if err := m.config.ApplyResourceProfile("large-db"); err != nil { - m.message = errorStyle.Render(fmt.Sprintf("[FAIL] %s", err.Error())) - return m, nil +// toggleLargeDBMode toggles the Large DB Mode flag +func (m SettingsModel) toggleLargeDBMode() (tea.Model, tea.Cmd) { + m.config.LargeDBMode = !m.config.LargeDBMode + if m.config.LargeDBMode { + profile := m.config.GetCurrentProfile() + m.message = successStyle.Render(fmt.Sprintf( + "[ON] Large DB Mode enabled: %s → Parallel=%d, Jobs=%d, MaxLocks=%d", + profile.Name, profile.ClusterParallelism, profile.Jobs, profile.MaxLocksPerTxn)) + } else { + profile := m.config.GetCurrentProfile() + m.message = successStyle.Render(fmt.Sprintf( + "[OFF] Large DB Mode disabled: %s → Parallel=%d, Jobs=%d", + profile.Name, profile.ClusterParallelism, profile.Jobs)) } - m.message = successStyle.Render("[OK] Applied 'large-db' profile: Cluster=1, Jobs=2. Optimized for large DBs to avoid 'out of shared memory' errors.") return m, nil } @@ -613,14 +636,19 @@ func (m SettingsModel) applyConservativeProfile() (tea.Model, tea.Cmd) { // showProfileRecommendation displays the recommended profile based on system resources func (m SettingsModel) showProfileRecommendation() (tea.Model, tea.Cmd) { profileName, reason := m.config.GetResourceProfileRecommendation(false) - largeDBProfile, largeDBReason := m.config.GetResourceProfileRecommendation(true) + + var largeDBHint string + if m.config.LargeDBMode { + largeDBHint = "Large DB Mode: ON" + } else { + largeDBHint = "Large DB Mode: OFF (press 'l' to enable)" + } m.message = infoStyle.Render(fmt.Sprintf( - "[RECOMMEND] Default: %s | For Large DBs: %s\n"+ + "[RECOMMEND] Profile: %s | %s\n"+ " → %s\n"+ - " → Large DB: %s\n"+ - " Press 'l' for large-db profile, 'c' for conservative", - profileName, largeDBProfile, reason, largeDBReason)) + " Press 'l' to toggle Large DB Mode, 'c' for conservative", + profileName, largeDBHint, reason)) return m, nil } @@ -907,9 +935,9 @@ func (m SettingsModel) View() string { } else { // Show different help based on current selection if m.cursor >= 0 && m.cursor < len(m.settings) && m.settings[m.cursor].Type == "path" { - footer = infoStyle.Render("\n[KEYS] ↑↓ navigate | Enter edit | Tab dirs | 'l' large-db | 'c' conservative | 'p' recommend | 's' save | 'q' menu") + footer = infoStyle.Render("\n[KEYS] ↑↓ navigate | Enter edit | Tab dirs | 'l' toggle LargeDB | 'c' conservative | 'p' recommend | 's' save | 'q' menu") } else { - footer = infoStyle.Render("\n[KEYS] ↑↓ navigate | Enter edit | 'l' large-db profile | 'c' conservative | 'p' recommend | 's' save | 'r' reset | 'q' menu") + footer = infoStyle.Render("\n[KEYS] ↑↓ navigate | Enter edit | 'l' toggle LargeDB mode | 'c' conservative | 'p' recommend | 's' save | 'r' reset | 'q' menu") } } }