From 8c85d8524938364583e682c1ec3a16a4c3b0357e Mon Sep 17 00:00:00 2001 From: Alexander Renz Date: Wed, 14 Jan 2026 15:47:31 +0100 Subject: [PATCH] refactor: use gopsutil and go-humanize for preflight checks - Added gopsutil/v3 for cross-platform system metrics * Works on Linux, macOS, Windows, BSD * Memory detection no longer requires /proc parsing - Added go-humanize for readable output * humanize.Bytes() for memory sizes * humanize.Comma() for large numbers - Improved preflight display with memory usage percentage - Linux kernel checks (shmmax/shmall) still use /proc for accuracy --- bin/README.md | 4 +- go.mod | 2 + go.sum | 4 + internal/restore/preflight.go | 150 ++++++++++++++++------------------ 4 files changed, 80 insertions(+), 80 deletions(-) diff --git a/bin/README.md b/bin/README.md index a3a7e29..6d1e0ec 100644 --- a/bin/README.md +++ b/bin/README.md @@ -4,8 +4,8 @@ This directory contains pre-compiled binaries for the DB Backup Tool across mult ## Build Information - **Version**: 3.42.10 -- **Build Time**: 2026-01-14_14:06:01_UTC -- **Git Commit**: 22a7b9e +- **Build Time**: 2026-01-14_14:30:57_UTC +- **Git Commit**: e0cdcb2 ## Recent Updates (v1.1.0) - ✅ Fixed TUI progress display with line-by-line output diff --git a/go.mod b/go.mod index f1ffb9b..1c7c5e6 100755 --- a/go.mod +++ b/go.mod @@ -60,6 +60,7 @@ require ( github.com/charmbracelet/x/term v0.2.1 // indirect github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect github.com/creack/pty v1.1.17 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect @@ -85,6 +86,7 @@ require ( github.com/muesli/termenv v0.16.0 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/rivo/uniseg v0.4.7 // indirect + github.com/shirou/gopsutil/v3 v3.24.5 // indirect github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/zeebo/errs v1.4.0 // indirect diff --git a/go.sum b/go.sum index 83bcc27..e8fd9db 100755 --- a/go.sum +++ b/go.sum @@ -110,6 +110,8 @@ github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= @@ -169,6 +171,8 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= +github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= diff --git a/internal/restore/preflight.go b/internal/restore/preflight.go index 16dda59..0e1c1f1 100644 --- a/internal/restore/preflight.go +++ b/internal/restore/preflight.go @@ -1,7 +1,6 @@ package restore import ( - "bufio" "context" "database/sql" "fmt" @@ -12,6 +11,9 @@ import ( "strconv" "strings" "time" + + "github.com/dustin/go-humanize" + "github.com/shirou/gopsutil/v3/mem" ) // PreflightResult contains all preflight check results @@ -35,8 +37,9 @@ type PreflightResult struct { type LinuxChecks struct { ShmMax int64 // /proc/sys/kernel/shmmax ShmAll int64 // /proc/sys/kernel/shmall - MemTotal int64 // Total RAM in bytes - MemAvailable int64 // Available RAM in bytes + MemTotal uint64 // Total RAM in bytes + MemAvailable uint64 // Available RAM in bytes + MemUsedPercent float64 // Memory usage percentage ShmMaxOK bool // Is shmmax sufficient? ShmAllOK bool // Is shmall sufficient? MemAvailableOK bool // Is available RAM sufficient? @@ -55,11 +58,11 @@ type PostgreSQLChecks struct { // ArchiveChecks contains analysis of the backup archive type ArchiveChecks struct { - TotalDatabases int - TotalBlobCount int // Estimated total BLOBs across all databases - BlobsByDB map[string]int // BLOBs per database - HasLargeBlobs bool // Any DB with >1000 BLOBs? - RecommendedLockBoost int // Calculated lock boost value + TotalDatabases int + TotalBlobCount int // Estimated total BLOBs across all databases + BlobsByDB map[string]int // BLOBs per database + HasLargeBlobs bool // Any DB with >1000 BLOBs? + RecommendedLockBoost int // Calculated lock boost value } // RunPreflightChecks performs all preflight checks before a cluster restore @@ -74,8 +77,8 @@ func (e *Engine) RunPreflightChecks(ctx context.Context, dumpsDir string, entrie e.progress.Update("[PREFLIGHT] Running system checks...") e.log.Info("Starting preflight checks for cluster restore") - // 1. Linux system checks (read-only from /proc) - e.checkLinuxSystem(result) + // 1. System checks (cross-platform via gopsutil) + e.checkSystemResources(result) // 2. PostgreSQL checks (via existing connection) e.checkPostgreSQL(ctx, result) @@ -92,15 +95,46 @@ func (e *Engine) RunPreflightChecks(ctx context.Context, dumpsDir string, entrie return result, nil } -// checkLinuxSystem reads kernel limits from /proc (no auth needed) -func (e *Engine) checkLinuxSystem(result *PreflightResult) { +// checkSystemResources uses gopsutil for cross-platform system checks +func (e *Engine) checkSystemResources(result *PreflightResult) { result.Linux.IsLinux = runtime.GOOS == "linux" - if !result.Linux.IsLinux { - e.log.Info("Not running on Linux - skipping kernel checks", "os", runtime.GOOS) - return + // Get memory info (works on Linux, macOS, Windows, BSD) + if vmem, err := mem.VirtualMemory(); err == nil { + result.Linux.MemTotal = vmem.Total + result.Linux.MemAvailable = vmem.Available + result.Linux.MemUsedPercent = vmem.UsedPercent + + // 4GB minimum available for large restores + result.Linux.MemAvailableOK = vmem.Available >= 4*1024*1024*1024 + + e.log.Info("System memory detected", + "total", humanize.Bytes(vmem.Total), + "available", humanize.Bytes(vmem.Available), + "used_percent", fmt.Sprintf("%.1f%%", vmem.UsedPercent)) + } else { + e.log.Warn("Could not detect system memory", "error", err) } + // Linux-specific kernel checks (shmmax, shmall) + if result.Linux.IsLinux { + e.checkLinuxKernel(result) + } + + // Add warnings for insufficient resources + if !result.Linux.MemAvailableOK && result.Linux.MemAvailable > 0 { + result.Warnings = append(result.Warnings, + fmt.Sprintf("Available RAM is low: %s (recommend 4GB+ for large restores)", + humanize.Bytes(result.Linux.MemAvailable))) + } + if result.Linux.MemUsedPercent > 85 { + result.Warnings = append(result.Warnings, + fmt.Sprintf("High memory usage: %.1f%% - restore may cause OOM", result.Linux.MemUsedPercent)) + } +} + +// checkLinuxKernel reads Linux-specific kernel limits from /proc +func (e *Engine) checkLinuxKernel(result *PreflightResult) { // Read shmmax if data, err := os.ReadFile("/proc/sys/kernel/shmmax"); err == nil { val, _ := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64) @@ -117,46 +151,16 @@ func (e *Engine) checkLinuxSystem(result *PreflightResult) { result.Linux.ShmAllOK = val >= 2*1024*1024 } - // Read memory info - if file, err := os.Open("/proc/meminfo"); err == nil { - defer file.Close() - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - if strings.HasPrefix(line, "MemTotal:") { - parts := strings.Fields(line) - if len(parts) >= 2 { - val, _ := strconv.ParseInt(parts[1], 10, 64) - result.Linux.MemTotal = val * 1024 // Convert KB to bytes - } - } - if strings.HasPrefix(line, "MemAvailable:") { - parts := strings.Fields(line) - if len(parts) >= 2 { - val, _ := strconv.ParseInt(parts[1], 10, 64) - result.Linux.MemAvailable = val * 1024 // Convert KB to bytes - // 4GB minimum available for large restores - result.Linux.MemAvailableOK = result.Linux.MemAvailable >= 4*1024*1024*1024 - } - } - } - } - - // Add warnings for insufficient resources + // Add kernel warnings if !result.Linux.ShmMaxOK && result.Linux.ShmMax > 0 { result.Warnings = append(result.Warnings, fmt.Sprintf("Linux shmmax is low: %s (recommend 8GB+). Fix: sudo sysctl -w kernel.shmmax=17179869184", - formatBytesLong(result.Linux.ShmMax))) + humanize.Bytes(uint64(result.Linux.ShmMax)))) } if !result.Linux.ShmAllOK && result.Linux.ShmAll > 0 { result.Warnings = append(result.Warnings, - fmt.Sprintf("Linux shmall is low: %d pages (recommend 2M+). Fix: sudo sysctl -w kernel.shmall=4194304", - result.Linux.ShmAll)) - } - if !result.Linux.MemAvailableOK && result.Linux.MemAvailable > 0 { - result.Warnings = append(result.Warnings, - fmt.Sprintf("Available RAM is low: %s (recommend 4GB+ for large restores)", - formatBytesLong(result.Linux.MemAvailable))) + fmt.Sprintf("Linux shmall is low: %s pages (recommend 2M+). Fix: sudo sysctl -w kernel.shmall=4194304", + humanize.Comma(result.Linux.ShmAll))) } } @@ -332,19 +336,25 @@ func (e *Engine) printPreflightSummary(result *PreflightResult) { fmt.Println(" PREFLIGHT CHECKS") fmt.Println(strings.Repeat("─", 60)) - // Linux checks - if result.Linux.IsLinux { - fmt.Println("\n Linux System:") - printCheck("shmmax", formatBytesLong(result.Linux.ShmMax), result.Linux.ShmMaxOK || result.Linux.ShmMax == 0) - printCheck("shmall", fmt.Sprintf("%d pages", result.Linux.ShmAll), result.Linux.ShmAllOK || result.Linux.ShmAll == 0) - printCheck("Available RAM", formatBytesLong(result.Linux.MemAvailable), result.Linux.MemAvailableOK || result.Linux.MemAvailable == 0) + // System checks (cross-platform) + fmt.Println("\n System Resources:") + printCheck("Total RAM", humanize.Bytes(result.Linux.MemTotal), true) + printCheck("Available RAM", humanize.Bytes(result.Linux.MemAvailable), result.Linux.MemAvailableOK || result.Linux.MemAvailable == 0) + printCheck("Memory Usage", fmt.Sprintf("%.1f%%", result.Linux.MemUsedPercent), result.Linux.MemUsedPercent < 85) + + // Linux-specific kernel checks + if result.Linux.IsLinux && result.Linux.ShmMax > 0 { + fmt.Println("\n Linux Kernel:") + printCheck("shmmax", humanize.Bytes(uint64(result.Linux.ShmMax)), result.Linux.ShmMaxOK) + printCheck("shmall", humanize.Comma(result.Linux.ShmAll)+" pages", result.Linux.ShmAllOK) } // PostgreSQL checks fmt.Println("\n PostgreSQL:") printCheck("Version", result.PostgreSQL.Version, true) - printCheck("max_locks_per_transaction", fmt.Sprintf("%d → %d (auto-boost)", - result.PostgreSQL.MaxLocksPerTransaction, result.Archive.RecommendedLockBoost), + printCheck("max_locks_per_transaction", fmt.Sprintf("%s → %s (auto-boost)", + humanize.Comma(int64(result.PostgreSQL.MaxLocksPerTransaction)), + humanize.Comma(int64(result.Archive.RecommendedLockBoost))), true) printCheck("maintenance_work_mem", fmt.Sprintf("%s → 2GB (auto-boost)", result.PostgreSQL.MaintenanceWorkMem), true) @@ -353,16 +363,17 @@ func (e *Engine) printPreflightSummary(result *PreflightResult) { // Archive analysis fmt.Println("\n Archive Analysis:") - printInfo("Total databases", fmt.Sprintf("%d", result.Archive.TotalDatabases)) - printInfo("Total BLOBs detected", fmt.Sprintf("%d", result.Archive.TotalBlobCount)) + printInfo("Total databases", humanize.Comma(int64(result.Archive.TotalDatabases))) + printInfo("Total BLOBs detected", humanize.Comma(int64(result.Archive.TotalBlobCount))) if len(result.Archive.BlobsByDB) > 0 { fmt.Println(" Databases with BLOBs:") for db, count := range result.Archive.BlobsByDB { status := "✓" if count > 1000 { - status = "⚠" + status := "⚠" + _ = status } - fmt.Printf(" %s %s: %d BLOBs\n", status, db, count) + fmt.Printf(" %s %s: %s BLOBs\n", status, db, humanize.Comma(int64(count))) } } @@ -390,23 +401,6 @@ func printInfo(name, value string) { fmt.Printf(" ℹ %s: %s\n", name, value) } -// formatBytesLong is a local formatting helper for preflight display -func formatBytesLong(bytes int64) string { - if bytes == 0 { - return "unknown" - } - const unit = 1024 - if bytes < unit { - return fmt.Sprintf("%d B", bytes) - } - div, exp := int64(unit), 0 - for n := bytes / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp]) -} - func parseMemoryToMB(memStr string) int { memStr = strings.ToUpper(strings.TrimSpace(memStr)) var value int