diff --git a/bin/README.md b/bin/README.md index 9391d98..6a1c2ef 100644 --- a/bin/README.md +++ b/bin/README.md @@ -4,8 +4,8 @@ This directory contains pre-compiled binaries for the DB Backup Tool across mult ## Build Information - **Version**: 3.42.50 -- **Build Time**: 2026-01-16_18:37:32_UTC -- **Git Commit**: 9200024 +- **Build Time**: 2026-01-17_06:25:57_UTC +- **Git Commit**: 4ea3ec2 ## Recent Updates (v1.1.0) - ✅ Fixed TUI progress display with line-by-line output diff --git a/internal/checks/error_hints.go b/internal/checks/error_hints.go index 67d0c97..fc3deae 100755 --- a/internal/checks/error_hints.go +++ b/internal/checks/error_hints.go @@ -68,8 +68,8 @@ func ClassifyError(errorMsg string) *ErrorClassification { Type: "critical", Category: "locks", Message: errorMsg, - Hint: "Lock table exhausted - typically caused by large objects (BLOBs) during restore", - Action: "Option 1: Increase max_locks_per_transaction to 1024+ in postgresql.conf (requires restart). Option 2: Update dbbackup and retry - phased restore now auto-enabled for BLOB databases", + Hint: "Lock table exhausted. Total capacity = max_locks_per_transaction × (max_connections + max_prepared_transactions). If you reduced VM size or max_connections, you need higher max_locks_per_transaction to compensate.", + Action: "Fix: ALTER SYSTEM SET max_locks_per_transaction = 4096; then RESTART PostgreSQL. For smaller VMs with fewer connections, you need higher max_locks_per_transaction values.", Severity: 2, } case "permission_denied": @@ -142,8 +142,8 @@ func ClassifyError(errorMsg string) *ErrorClassification { Type: "critical", Category: "locks", Message: errorMsg, - Hint: "Lock table exhausted - typically caused by large objects (BLOBs) during restore", - Action: "Option 1: Increase max_locks_per_transaction to 1024+ in postgresql.conf (requires restart). Option 2: Update dbbackup and retry - phased restore now auto-enabled for BLOB databases", + Hint: "Lock table exhausted. Total capacity = max_locks_per_transaction × (max_connections + max_prepared_transactions). If you reduced VM size or max_connections, you need higher max_locks_per_transaction to compensate.", + Action: "Fix: ALTER SYSTEM SET max_locks_per_transaction = 4096; then RESTART PostgreSQL. For smaller VMs with fewer connections, you need higher max_locks_per_transaction values.", Severity: 2, } } diff --git a/internal/restore/engine.go b/internal/restore/engine.go index 45442d2..af977a7 100755 --- a/internal/restore/engine.go +++ b/internal/restore/engine.go @@ -2125,9 +2125,10 @@ func (e *Engine) quickValidateSQLDump(archivePath string, compressed bool) error return nil } -// boostLockCapacity temporarily increases max_locks_per_transaction to prevent OOM -// during large restores with many BLOBs. Returns the original value for later reset. -// Uses ALTER SYSTEM + pg_reload_conf() so no restart is needed. +// boostLockCapacity checks and reports on max_locks_per_transaction capacity. +// IMPORTANT: max_locks_per_transaction requires a PostgreSQL RESTART to change! +// This function now calculates total lock capacity based on max_connections and +// warns the user if capacity is insufficient for the restore. func (e *Engine) boostLockCapacity(ctx context.Context) (int, error) { // Connect to PostgreSQL to run system commands connStr := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=postgres sslmode=disable", @@ -2145,7 +2146,7 @@ func (e *Engine) boostLockCapacity(ctx context.Context) (int, error) { } defer db.Close() - // Get current value + // Get current max_locks_per_transaction var currentValue int err = db.QueryRowContext(ctx, "SHOW max_locks_per_transaction").Scan(¤tValue) if err != nil { @@ -2158,22 +2159,56 @@ func (e *Engine) boostLockCapacity(ctx context.Context) (int, error) { fmt.Sscanf(currentValueStr, "%d", ¤tValue) } - // Skip if already high enough - if currentValue >= 2048 { - e.log.Info("max_locks_per_transaction already sufficient", "value", currentValue) - return currentValue, nil + // Get max_connections to calculate total lock capacity + var maxConns int + if err := db.QueryRowContext(ctx, "SHOW max_connections").Scan(&maxConns); err != nil { + maxConns = 100 // default } - // Boost to 2048 (enough for most BLOB-heavy databases) - _, err = db.ExecContext(ctx, "ALTER SYSTEM SET max_locks_per_transaction = 2048") - if err != nil { - return currentValue, fmt.Errorf("failed to set max_locks_per_transaction: %w", err) + // Get max_prepared_transactions + var maxPreparedTxns int + if err := db.QueryRowContext(ctx, "SHOW max_prepared_transactions").Scan(&maxPreparedTxns); err != nil { + maxPreparedTxns = 0 } - // Reload config without restart - _, err = db.ExecContext(ctx, "SELECT pg_reload_conf()") - if err != nil { - return currentValue, fmt.Errorf("failed to reload config: %w", err) + // Calculate total lock table capacity: + // Total locks = max_locks_per_transaction × (max_connections + max_prepared_transactions) + totalLockCapacity := currentValue * (maxConns + maxPreparedTxns) + + e.log.Info("PostgreSQL lock table capacity", + "max_locks_per_transaction", currentValue, + "max_connections", maxConns, + "max_prepared_transactions", maxPreparedTxns, + "total_lock_capacity", totalLockCapacity) + + // Minimum recommended total capacity for BLOB-heavy restores: 200,000 locks + minRecommendedCapacity := 200000 + if totalLockCapacity < minRecommendedCapacity { + recommendedMaxLocks := minRecommendedCapacity / (maxConns + maxPreparedTxns) + if recommendedMaxLocks < 4096 { + recommendedMaxLocks = 4096 + } + + e.log.Warn("Lock table capacity may be insufficient for BLOB-heavy restores", + "current_total_capacity", totalLockCapacity, + "recommended_capacity", minRecommendedCapacity, + "current_max_locks", currentValue, + "recommended_max_locks", recommendedMaxLocks, + "note", "max_locks_per_transaction requires PostgreSQL RESTART to change") + + // Write suggested fix to ALTER SYSTEM but warn about restart + _, err = db.ExecContext(ctx, fmt.Sprintf("ALTER SYSTEM SET max_locks_per_transaction = %d", recommendedMaxLocks)) + if err != nil { + e.log.Warn("Could not set recommended max_locks_per_transaction (needs superuser)", "error", err) + } else { + e.log.Warn("Wrote recommended max_locks_per_transaction to postgresql.auto.conf", + "value", recommendedMaxLocks, + "action", "RESTART PostgreSQL to apply: sudo systemctl restart postgresql") + } + } else { + e.log.Info("Lock table capacity is sufficient", + "total_capacity", totalLockCapacity, + "max_locks_per_transaction", currentValue) } return currentValue, nil diff --git a/internal/restore/preflight.go b/internal/restore/preflight.go index 636403c..f680b5e 100644 --- a/internal/restore/preflight.go +++ b/internal/restore/preflight.go @@ -48,12 +48,14 @@ type LinuxChecks struct { // PostgreSQLChecks contains PostgreSQL configuration checks type PostgreSQLChecks struct { - MaxLocksPerTransaction int // Current setting - MaintenanceWorkMem string // Current setting - SharedBuffers string // Current setting (info only) - MaxConnections int // Current setting - Version string // PostgreSQL version - IsSuperuser bool // Can we modify settings? + MaxLocksPerTransaction int // Current setting + MaxPreparedTransactions int // Current setting (affects lock capacity) + TotalLockCapacity int // Calculated: max_locks × (max_connections + max_prepared) + MaintenanceWorkMem string // Current setting + SharedBuffers string // Current setting (info only) + MaxConnections int // Current setting + Version string // PostgreSQL version + IsSuperuser bool // Can we modify settings? } // ArchiveChecks contains analysis of the backup archive @@ -201,6 +203,29 @@ func (e *Engine) checkPostgreSQL(ctx context.Context, result *PreflightResult) { result.PostgreSQL.IsSuperuser = isSuperuser } + // Check max_prepared_transactions for lock capacity calculation + var maxPreparedTxns string + if err := db.QueryRowContext(ctx, "SHOW max_prepared_transactions").Scan(&maxPreparedTxns); err == nil { + result.PostgreSQL.MaxPreparedTransactions, _ = strconv.Atoi(maxPreparedTxns) + } + + // CRITICAL: Calculate TOTAL lock table capacity + // Formula: max_locks_per_transaction × (max_connections + max_prepared_transactions) + // This is THE key capacity metric for BLOB-heavy restores + maxConns := result.PostgreSQL.MaxConnections + if maxConns == 0 { + maxConns = 100 // default + } + maxPrepared := result.PostgreSQL.MaxPreparedTransactions + totalLockCapacity := result.PostgreSQL.MaxLocksPerTransaction * (maxConns + maxPrepared) + result.PostgreSQL.TotalLockCapacity = totalLockCapacity + + e.log.Info("PostgreSQL lock table capacity", + "max_locks_per_transaction", result.PostgreSQL.MaxLocksPerTransaction, + "max_connections", maxConns, + "max_prepared_transactions", maxPrepared, + "total_lock_capacity", totalLockCapacity) + // CRITICAL: max_locks_per_transaction requires PostgreSQL RESTART to change! // Warn users loudly about this - it's the #1 cause of "out of shared memory" errors if result.PostgreSQL.MaxLocksPerTransaction < 256 { @@ -217,6 +242,33 @@ func (e *Engine) checkPostgreSQL(ctx context.Context, result *PreflightResult) { result.PostgreSQL.MaxLocksPerTransaction)) } + // NEW: Check total lock capacity is sufficient for typical BLOB operations + // Minimum recommended: 200,000 for moderate BLOB databases + minRecommendedCapacity := 200000 + if totalLockCapacity < minRecommendedCapacity { + recommendedMaxLocks := minRecommendedCapacity / (maxConns + maxPrepared) + if recommendedMaxLocks < 4096 { + recommendedMaxLocks = 4096 + } + + e.log.Warn("Total lock table capacity is LOW for BLOB-heavy restores", + "current_capacity", totalLockCapacity, + "recommended", minRecommendedCapacity, + "current_max_locks", result.PostgreSQL.MaxLocksPerTransaction, + "current_max_connections", maxConns, + "recommended_max_locks", recommendedMaxLocks, + "note", "VMs with fewer connections need higher max_locks_per_transaction") + + result.Warnings = append(result.Warnings, + fmt.Sprintf("Total lock capacity=%d is low (recommend %d+). "+ + "Capacity = max_locks_per_transaction(%d) × max_connections(%d). "+ + "If you reduced VM size/connections, increase max_locks_per_transaction to %d. "+ + "Fix: ALTER SYSTEM SET max_locks_per_transaction = %d; then RESTART PostgreSQL.", + totalLockCapacity, minRecommendedCapacity, + result.PostgreSQL.MaxLocksPerTransaction, maxConns, + recommendedMaxLocks, recommendedMaxLocks)) + } + // Parse shared_buffers and warn if very low sharedBuffersMB := parseMemoryToMB(result.PostgreSQL.SharedBuffers) if sharedBuffersMB > 0 && sharedBuffersMB < 256 { @@ -409,6 +461,13 @@ func (e *Engine) printPreflightSummary(result *PreflightResult) { humanize.Comma(int64(result.PostgreSQL.MaxLocksPerTransaction)), humanize.Comma(int64(result.Archive.RecommendedLockBoost))), true) + printCheck("max_connections", humanize.Comma(int64(result.PostgreSQL.MaxConnections)), true) + // Show total lock capacity with warning if low + totalCapacityOK := result.PostgreSQL.TotalLockCapacity >= 200000 + printCheck("Total Lock Capacity", + fmt.Sprintf("%s (max_locks × max_conns)", + humanize.Comma(int64(result.PostgreSQL.TotalLockCapacity))), + totalCapacityOK) printCheck("maintenance_work_mem", fmt.Sprintf("%s → 2GB (auto-boost)", result.PostgreSQL.MaintenanceWorkMem), true) printInfo("shared_buffers", result.PostgreSQL.SharedBuffers)