refactor: use gopsutil and go-humanize for preflight checks
Some checks failed
CI/CD / Build & Release (push) Has been cancelled
CI/CD / Lint (push) Has been cancelled
CI/CD / Test (push) Has been cancelled

- Added gopsutil/v3 for cross-platform system metrics
  * Works on Linux, macOS, Windows, BSD
  * Memory detection no longer requires /proc parsing

- Added go-humanize for readable output
  * humanize.Bytes() for memory sizes
  * humanize.Comma() for large numbers

- Improved preflight display with memory usage percentage
- Linux kernel checks (shmmax/shmall) still use /proc for accuracy
This commit is contained in:
2026-01-14 15:47:31 +01:00
parent e0cdcb28be
commit 8c85d85249
4 changed files with 80 additions and 80 deletions

View File

@@ -4,8 +4,8 @@ This directory contains pre-compiled binaries for the DB Backup Tool across mult
## Build Information ## Build Information
- **Version**: 3.42.10 - **Version**: 3.42.10
- **Build Time**: 2026-01-14_14:06:01_UTC - **Build Time**: 2026-01-14_14:30:57_UTC
- **Git Commit**: 22a7b9e - **Git Commit**: e0cdcb2
## Recent Updates (v1.1.0) ## Recent Updates (v1.1.0)
- ✅ Fixed TUI progress display with line-by-line output - ✅ Fixed TUI progress display with line-by-line output

2
go.mod
View File

@@ -60,6 +60,7 @@ require (
github.com/charmbracelet/x/term v0.2.1 // indirect github.com/charmbracelet/x/term v0.2.1 // indirect
github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect
github.com/creack/pty v1.1.17 // indirect github.com/creack/pty v1.1.17 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
@@ -85,6 +86,7 @@ require (
github.com/muesli/termenv v0.16.0 // indirect github.com/muesli/termenv v0.16.0 // indirect
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
github.com/rivo/uniseg v0.4.7 // indirect github.com/rivo/uniseg v0.4.7 // indirect
github.com/shirou/gopsutil/v3 v3.24.5 // indirect
github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
github.com/zeebo/errs v1.4.0 // indirect github.com/zeebo/errs v1.4.0 // indirect

4
go.sum
View File

@@ -110,6 +110,8 @@ github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A=
github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw=
github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8=
@@ -169,6 +171,8 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI=
github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=

View File

@@ -1,7 +1,6 @@
package restore package restore
import ( import (
"bufio"
"context" "context"
"database/sql" "database/sql"
"fmt" "fmt"
@@ -12,6 +11,9 @@ import (
"strconv" "strconv"
"strings" "strings"
"time" "time"
"github.com/dustin/go-humanize"
"github.com/shirou/gopsutil/v3/mem"
) )
// PreflightResult contains all preflight check results // PreflightResult contains all preflight check results
@@ -35,8 +37,9 @@ type PreflightResult struct {
type LinuxChecks struct { type LinuxChecks struct {
ShmMax int64 // /proc/sys/kernel/shmmax ShmMax int64 // /proc/sys/kernel/shmmax
ShmAll int64 // /proc/sys/kernel/shmall ShmAll int64 // /proc/sys/kernel/shmall
MemTotal int64 // Total RAM in bytes MemTotal uint64 // Total RAM in bytes
MemAvailable int64 // Available RAM in bytes MemAvailable uint64 // Available RAM in bytes
MemUsedPercent float64 // Memory usage percentage
ShmMaxOK bool // Is shmmax sufficient? ShmMaxOK bool // Is shmmax sufficient?
ShmAllOK bool // Is shmall sufficient? ShmAllOK bool // Is shmall sufficient?
MemAvailableOK bool // Is available RAM sufficient? MemAvailableOK bool // Is available RAM sufficient?
@@ -74,8 +77,8 @@ func (e *Engine) RunPreflightChecks(ctx context.Context, dumpsDir string, entrie
e.progress.Update("[PREFLIGHT] Running system checks...") e.progress.Update("[PREFLIGHT] Running system checks...")
e.log.Info("Starting preflight checks for cluster restore") e.log.Info("Starting preflight checks for cluster restore")
// 1. Linux system checks (read-only from /proc) // 1. System checks (cross-platform via gopsutil)
e.checkLinuxSystem(result) e.checkSystemResources(result)
// 2. PostgreSQL checks (via existing connection) // 2. PostgreSQL checks (via existing connection)
e.checkPostgreSQL(ctx, result) e.checkPostgreSQL(ctx, result)
@@ -92,15 +95,46 @@ func (e *Engine) RunPreflightChecks(ctx context.Context, dumpsDir string, entrie
return result, nil return result, nil
} }
// checkLinuxSystem reads kernel limits from /proc (no auth needed) // checkSystemResources uses gopsutil for cross-platform system checks
func (e *Engine) checkLinuxSystem(result *PreflightResult) { func (e *Engine) checkSystemResources(result *PreflightResult) {
result.Linux.IsLinux = runtime.GOOS == "linux" result.Linux.IsLinux = runtime.GOOS == "linux"
if !result.Linux.IsLinux { // Get memory info (works on Linux, macOS, Windows, BSD)
e.log.Info("Not running on Linux - skipping kernel checks", "os", runtime.GOOS) if vmem, err := mem.VirtualMemory(); err == nil {
return result.Linux.MemTotal = vmem.Total
result.Linux.MemAvailable = vmem.Available
result.Linux.MemUsedPercent = vmem.UsedPercent
// 4GB minimum available for large restores
result.Linux.MemAvailableOK = vmem.Available >= 4*1024*1024*1024
e.log.Info("System memory detected",
"total", humanize.Bytes(vmem.Total),
"available", humanize.Bytes(vmem.Available),
"used_percent", fmt.Sprintf("%.1f%%", vmem.UsedPercent))
} else {
e.log.Warn("Could not detect system memory", "error", err)
} }
// Linux-specific kernel checks (shmmax, shmall)
if result.Linux.IsLinux {
e.checkLinuxKernel(result)
}
// Add warnings for insufficient resources
if !result.Linux.MemAvailableOK && result.Linux.MemAvailable > 0 {
result.Warnings = append(result.Warnings,
fmt.Sprintf("Available RAM is low: %s (recommend 4GB+ for large restores)",
humanize.Bytes(result.Linux.MemAvailable)))
}
if result.Linux.MemUsedPercent > 85 {
result.Warnings = append(result.Warnings,
fmt.Sprintf("High memory usage: %.1f%% - restore may cause OOM", result.Linux.MemUsedPercent))
}
}
// checkLinuxKernel reads Linux-specific kernel limits from /proc
func (e *Engine) checkLinuxKernel(result *PreflightResult) {
// Read shmmax // Read shmmax
if data, err := os.ReadFile("/proc/sys/kernel/shmmax"); err == nil { if data, err := os.ReadFile("/proc/sys/kernel/shmmax"); err == nil {
val, _ := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64) val, _ := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64)
@@ -117,46 +151,16 @@ func (e *Engine) checkLinuxSystem(result *PreflightResult) {
result.Linux.ShmAllOK = val >= 2*1024*1024 result.Linux.ShmAllOK = val >= 2*1024*1024
} }
// Read memory info // Add kernel warnings
if file, err := os.Open("/proc/meminfo"); err == nil {
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "MemTotal:") {
parts := strings.Fields(line)
if len(parts) >= 2 {
val, _ := strconv.ParseInt(parts[1], 10, 64)
result.Linux.MemTotal = val * 1024 // Convert KB to bytes
}
}
if strings.HasPrefix(line, "MemAvailable:") {
parts := strings.Fields(line)
if len(parts) >= 2 {
val, _ := strconv.ParseInt(parts[1], 10, 64)
result.Linux.MemAvailable = val * 1024 // Convert KB to bytes
// 4GB minimum available for large restores
result.Linux.MemAvailableOK = result.Linux.MemAvailable >= 4*1024*1024*1024
}
}
}
}
// Add warnings for insufficient resources
if !result.Linux.ShmMaxOK && result.Linux.ShmMax > 0 { if !result.Linux.ShmMaxOK && result.Linux.ShmMax > 0 {
result.Warnings = append(result.Warnings, result.Warnings = append(result.Warnings,
fmt.Sprintf("Linux shmmax is low: %s (recommend 8GB+). Fix: sudo sysctl -w kernel.shmmax=17179869184", fmt.Sprintf("Linux shmmax is low: %s (recommend 8GB+). Fix: sudo sysctl -w kernel.shmmax=17179869184",
formatBytesLong(result.Linux.ShmMax))) humanize.Bytes(uint64(result.Linux.ShmMax))))
} }
if !result.Linux.ShmAllOK && result.Linux.ShmAll > 0 { if !result.Linux.ShmAllOK && result.Linux.ShmAll > 0 {
result.Warnings = append(result.Warnings, result.Warnings = append(result.Warnings,
fmt.Sprintf("Linux shmall is low: %d pages (recommend 2M+). Fix: sudo sysctl -w kernel.shmall=4194304", fmt.Sprintf("Linux shmall is low: %s pages (recommend 2M+). Fix: sudo sysctl -w kernel.shmall=4194304",
result.Linux.ShmAll)) humanize.Comma(result.Linux.ShmAll)))
}
if !result.Linux.MemAvailableOK && result.Linux.MemAvailable > 0 {
result.Warnings = append(result.Warnings,
fmt.Sprintf("Available RAM is low: %s (recommend 4GB+ for large restores)",
formatBytesLong(result.Linux.MemAvailable)))
} }
} }
@@ -332,19 +336,25 @@ func (e *Engine) printPreflightSummary(result *PreflightResult) {
fmt.Println(" PREFLIGHT CHECKS") fmt.Println(" PREFLIGHT CHECKS")
fmt.Println(strings.Repeat("─", 60)) fmt.Println(strings.Repeat("─", 60))
// Linux checks // System checks (cross-platform)
if result.Linux.IsLinux { fmt.Println("\n System Resources:")
fmt.Println("\n Linux System:") printCheck("Total RAM", humanize.Bytes(result.Linux.MemTotal), true)
printCheck("shmmax", formatBytesLong(result.Linux.ShmMax), result.Linux.ShmMaxOK || result.Linux.ShmMax == 0) printCheck("Available RAM", humanize.Bytes(result.Linux.MemAvailable), result.Linux.MemAvailableOK || result.Linux.MemAvailable == 0)
printCheck("shmall", fmt.Sprintf("%d pages", result.Linux.ShmAll), result.Linux.ShmAllOK || result.Linux.ShmAll == 0) printCheck("Memory Usage", fmt.Sprintf("%.1f%%", result.Linux.MemUsedPercent), result.Linux.MemUsedPercent < 85)
printCheck("Available RAM", formatBytesLong(result.Linux.MemAvailable), result.Linux.MemAvailableOK || result.Linux.MemAvailable == 0)
// Linux-specific kernel checks
if result.Linux.IsLinux && result.Linux.ShmMax > 0 {
fmt.Println("\n Linux Kernel:")
printCheck("shmmax", humanize.Bytes(uint64(result.Linux.ShmMax)), result.Linux.ShmMaxOK)
printCheck("shmall", humanize.Comma(result.Linux.ShmAll)+" pages", result.Linux.ShmAllOK)
} }
// PostgreSQL checks // PostgreSQL checks
fmt.Println("\n PostgreSQL:") fmt.Println("\n PostgreSQL:")
printCheck("Version", result.PostgreSQL.Version, true) printCheck("Version", result.PostgreSQL.Version, true)
printCheck("max_locks_per_transaction", fmt.Sprintf("%d → %d (auto-boost)", printCheck("max_locks_per_transaction", fmt.Sprintf("%s → %s (auto-boost)",
result.PostgreSQL.MaxLocksPerTransaction, result.Archive.RecommendedLockBoost), humanize.Comma(int64(result.PostgreSQL.MaxLocksPerTransaction)),
humanize.Comma(int64(result.Archive.RecommendedLockBoost))),
true) true)
printCheck("maintenance_work_mem", fmt.Sprintf("%s → 2GB (auto-boost)", printCheck("maintenance_work_mem", fmt.Sprintf("%s → 2GB (auto-boost)",
result.PostgreSQL.MaintenanceWorkMem), true) result.PostgreSQL.MaintenanceWorkMem), true)
@@ -353,16 +363,17 @@ func (e *Engine) printPreflightSummary(result *PreflightResult) {
// Archive analysis // Archive analysis
fmt.Println("\n Archive Analysis:") fmt.Println("\n Archive Analysis:")
printInfo("Total databases", fmt.Sprintf("%d", result.Archive.TotalDatabases)) printInfo("Total databases", humanize.Comma(int64(result.Archive.TotalDatabases)))
printInfo("Total BLOBs detected", fmt.Sprintf("%d", result.Archive.TotalBlobCount)) printInfo("Total BLOBs detected", humanize.Comma(int64(result.Archive.TotalBlobCount)))
if len(result.Archive.BlobsByDB) > 0 { if len(result.Archive.BlobsByDB) > 0 {
fmt.Println(" Databases with BLOBs:") fmt.Println(" Databases with BLOBs:")
for db, count := range result.Archive.BlobsByDB { for db, count := range result.Archive.BlobsByDB {
status := "✓" status := "✓"
if count > 1000 { if count > 1000 {
status = "⚠" status := "⚠"
_ = status
} }
fmt.Printf(" %s %s: %d BLOBs\n", status, db, count) fmt.Printf(" %s %s: %s BLOBs\n", status, db, humanize.Comma(int64(count)))
} }
} }
@@ -390,23 +401,6 @@ func printInfo(name, value string) {
fmt.Printf(" %s: %s\n", name, value) fmt.Printf(" %s: %s\n", name, value)
} }
// formatBytesLong is a local formatting helper for preflight display
func formatBytesLong(bytes int64) string {
if bytes == 0 {
return "unknown"
}
const unit = 1024
if bytes < unit {
return fmt.Sprintf("%d B", bytes)
}
div, exp := int64(unit), 0
for n := bytes / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
}
func parseMemoryToMB(memStr string) int { func parseMemoryToMB(memStr string) int {
memStr = strings.ToUpper(strings.TrimSpace(memStr)) memStr = strings.ToUpper(strings.TrimSpace(memStr))
var value int var value int