refactor: use gopsutil and go-humanize for preflight checks
- Added gopsutil/v3 for cross-platform system metrics * Works on Linux, macOS, Windows, BSD * Memory detection no longer requires /proc parsing - Added go-humanize for readable output * humanize.Bytes() for memory sizes * humanize.Comma() for large numbers - Improved preflight display with memory usage percentage - Linux kernel checks (shmmax/shmall) still use /proc for accuracy
This commit is contained in:
@@ -4,8 +4,8 @@ This directory contains pre-compiled binaries for the DB Backup Tool across mult
|
||||
|
||||
## Build Information
|
||||
- **Version**: 3.42.10
|
||||
- **Build Time**: 2026-01-14_14:06:01_UTC
|
||||
- **Git Commit**: 22a7b9e
|
||||
- **Build Time**: 2026-01-14_14:30:57_UTC
|
||||
- **Git Commit**: e0cdcb2
|
||||
|
||||
## Recent Updates (v1.1.0)
|
||||
- ✅ Fixed TUI progress display with line-by-line output
|
||||
|
||||
2
go.mod
2
go.mod
@@ -60,6 +60,7 @@ require (
|
||||
github.com/charmbracelet/x/term v0.2.1 // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect
|
||||
github.com/creack/pty v1.1.17 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
|
||||
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
|
||||
@@ -85,6 +86,7 @@ require (
|
||||
github.com/muesli/termenv v0.16.0 // indirect
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/shirou/gopsutil/v3 v3.24.5 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
github.com/zeebo/errs v1.4.0 // indirect
|
||||
|
||||
4
go.sum
4
go.sum
@@ -110,6 +110,8 @@ github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A=
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw=
|
||||
github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8=
|
||||
@@ -169,6 +171,8 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI=
|
||||
github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk=
|
||||
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
|
||||
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package restore
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
@@ -12,6 +11,9 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/shirou/gopsutil/v3/mem"
|
||||
)
|
||||
|
||||
// PreflightResult contains all preflight check results
|
||||
@@ -35,8 +37,9 @@ type PreflightResult struct {
|
||||
type LinuxChecks struct {
|
||||
ShmMax int64 // /proc/sys/kernel/shmmax
|
||||
ShmAll int64 // /proc/sys/kernel/shmall
|
||||
MemTotal int64 // Total RAM in bytes
|
||||
MemAvailable int64 // Available RAM in bytes
|
||||
MemTotal uint64 // Total RAM in bytes
|
||||
MemAvailable uint64 // Available RAM in bytes
|
||||
MemUsedPercent float64 // Memory usage percentage
|
||||
ShmMaxOK bool // Is shmmax sufficient?
|
||||
ShmAllOK bool // Is shmall sufficient?
|
||||
MemAvailableOK bool // Is available RAM sufficient?
|
||||
@@ -55,11 +58,11 @@ type PostgreSQLChecks struct {
|
||||
|
||||
// ArchiveChecks contains analysis of the backup archive
|
||||
type ArchiveChecks struct {
|
||||
TotalDatabases int
|
||||
TotalBlobCount int // Estimated total BLOBs across all databases
|
||||
BlobsByDB map[string]int // BLOBs per database
|
||||
HasLargeBlobs bool // Any DB with >1000 BLOBs?
|
||||
RecommendedLockBoost int // Calculated lock boost value
|
||||
TotalDatabases int
|
||||
TotalBlobCount int // Estimated total BLOBs across all databases
|
||||
BlobsByDB map[string]int // BLOBs per database
|
||||
HasLargeBlobs bool // Any DB with >1000 BLOBs?
|
||||
RecommendedLockBoost int // Calculated lock boost value
|
||||
}
|
||||
|
||||
// RunPreflightChecks performs all preflight checks before a cluster restore
|
||||
@@ -74,8 +77,8 @@ func (e *Engine) RunPreflightChecks(ctx context.Context, dumpsDir string, entrie
|
||||
e.progress.Update("[PREFLIGHT] Running system checks...")
|
||||
e.log.Info("Starting preflight checks for cluster restore")
|
||||
|
||||
// 1. Linux system checks (read-only from /proc)
|
||||
e.checkLinuxSystem(result)
|
||||
// 1. System checks (cross-platform via gopsutil)
|
||||
e.checkSystemResources(result)
|
||||
|
||||
// 2. PostgreSQL checks (via existing connection)
|
||||
e.checkPostgreSQL(ctx, result)
|
||||
@@ -92,15 +95,46 @@ func (e *Engine) RunPreflightChecks(ctx context.Context, dumpsDir string, entrie
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// checkLinuxSystem reads kernel limits from /proc (no auth needed)
|
||||
func (e *Engine) checkLinuxSystem(result *PreflightResult) {
|
||||
// checkSystemResources uses gopsutil for cross-platform system checks
|
||||
func (e *Engine) checkSystemResources(result *PreflightResult) {
|
||||
result.Linux.IsLinux = runtime.GOOS == "linux"
|
||||
|
||||
if !result.Linux.IsLinux {
|
||||
e.log.Info("Not running on Linux - skipping kernel checks", "os", runtime.GOOS)
|
||||
return
|
||||
// Get memory info (works on Linux, macOS, Windows, BSD)
|
||||
if vmem, err := mem.VirtualMemory(); err == nil {
|
||||
result.Linux.MemTotal = vmem.Total
|
||||
result.Linux.MemAvailable = vmem.Available
|
||||
result.Linux.MemUsedPercent = vmem.UsedPercent
|
||||
|
||||
// 4GB minimum available for large restores
|
||||
result.Linux.MemAvailableOK = vmem.Available >= 4*1024*1024*1024
|
||||
|
||||
e.log.Info("System memory detected",
|
||||
"total", humanize.Bytes(vmem.Total),
|
||||
"available", humanize.Bytes(vmem.Available),
|
||||
"used_percent", fmt.Sprintf("%.1f%%", vmem.UsedPercent))
|
||||
} else {
|
||||
e.log.Warn("Could not detect system memory", "error", err)
|
||||
}
|
||||
|
||||
// Linux-specific kernel checks (shmmax, shmall)
|
||||
if result.Linux.IsLinux {
|
||||
e.checkLinuxKernel(result)
|
||||
}
|
||||
|
||||
// Add warnings for insufficient resources
|
||||
if !result.Linux.MemAvailableOK && result.Linux.MemAvailable > 0 {
|
||||
result.Warnings = append(result.Warnings,
|
||||
fmt.Sprintf("Available RAM is low: %s (recommend 4GB+ for large restores)",
|
||||
humanize.Bytes(result.Linux.MemAvailable)))
|
||||
}
|
||||
if result.Linux.MemUsedPercent > 85 {
|
||||
result.Warnings = append(result.Warnings,
|
||||
fmt.Sprintf("High memory usage: %.1f%% - restore may cause OOM", result.Linux.MemUsedPercent))
|
||||
}
|
||||
}
|
||||
|
||||
// checkLinuxKernel reads Linux-specific kernel limits from /proc
|
||||
func (e *Engine) checkLinuxKernel(result *PreflightResult) {
|
||||
// Read shmmax
|
||||
if data, err := os.ReadFile("/proc/sys/kernel/shmmax"); err == nil {
|
||||
val, _ := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64)
|
||||
@@ -117,46 +151,16 @@ func (e *Engine) checkLinuxSystem(result *PreflightResult) {
|
||||
result.Linux.ShmAllOK = val >= 2*1024*1024
|
||||
}
|
||||
|
||||
// Read memory info
|
||||
if file, err := os.Open("/proc/meminfo"); err == nil {
|
||||
defer file.Close()
|
||||
scanner := bufio.NewScanner(file)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if strings.HasPrefix(line, "MemTotal:") {
|
||||
parts := strings.Fields(line)
|
||||
if len(parts) >= 2 {
|
||||
val, _ := strconv.ParseInt(parts[1], 10, 64)
|
||||
result.Linux.MemTotal = val * 1024 // Convert KB to bytes
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(line, "MemAvailable:") {
|
||||
parts := strings.Fields(line)
|
||||
if len(parts) >= 2 {
|
||||
val, _ := strconv.ParseInt(parts[1], 10, 64)
|
||||
result.Linux.MemAvailable = val * 1024 // Convert KB to bytes
|
||||
// 4GB minimum available for large restores
|
||||
result.Linux.MemAvailableOK = result.Linux.MemAvailable >= 4*1024*1024*1024
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add warnings for insufficient resources
|
||||
// Add kernel warnings
|
||||
if !result.Linux.ShmMaxOK && result.Linux.ShmMax > 0 {
|
||||
result.Warnings = append(result.Warnings,
|
||||
fmt.Sprintf("Linux shmmax is low: %s (recommend 8GB+). Fix: sudo sysctl -w kernel.shmmax=17179869184",
|
||||
formatBytesLong(result.Linux.ShmMax)))
|
||||
humanize.Bytes(uint64(result.Linux.ShmMax))))
|
||||
}
|
||||
if !result.Linux.ShmAllOK && result.Linux.ShmAll > 0 {
|
||||
result.Warnings = append(result.Warnings,
|
||||
fmt.Sprintf("Linux shmall is low: %d pages (recommend 2M+). Fix: sudo sysctl -w kernel.shmall=4194304",
|
||||
result.Linux.ShmAll))
|
||||
}
|
||||
if !result.Linux.MemAvailableOK && result.Linux.MemAvailable > 0 {
|
||||
result.Warnings = append(result.Warnings,
|
||||
fmt.Sprintf("Available RAM is low: %s (recommend 4GB+ for large restores)",
|
||||
formatBytesLong(result.Linux.MemAvailable)))
|
||||
fmt.Sprintf("Linux shmall is low: %s pages (recommend 2M+). Fix: sudo sysctl -w kernel.shmall=4194304",
|
||||
humanize.Comma(result.Linux.ShmAll)))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -332,19 +336,25 @@ func (e *Engine) printPreflightSummary(result *PreflightResult) {
|
||||
fmt.Println(" PREFLIGHT CHECKS")
|
||||
fmt.Println(strings.Repeat("─", 60))
|
||||
|
||||
// Linux checks
|
||||
if result.Linux.IsLinux {
|
||||
fmt.Println("\n Linux System:")
|
||||
printCheck("shmmax", formatBytesLong(result.Linux.ShmMax), result.Linux.ShmMaxOK || result.Linux.ShmMax == 0)
|
||||
printCheck("shmall", fmt.Sprintf("%d pages", result.Linux.ShmAll), result.Linux.ShmAllOK || result.Linux.ShmAll == 0)
|
||||
printCheck("Available RAM", formatBytesLong(result.Linux.MemAvailable), result.Linux.MemAvailableOK || result.Linux.MemAvailable == 0)
|
||||
// System checks (cross-platform)
|
||||
fmt.Println("\n System Resources:")
|
||||
printCheck("Total RAM", humanize.Bytes(result.Linux.MemTotal), true)
|
||||
printCheck("Available RAM", humanize.Bytes(result.Linux.MemAvailable), result.Linux.MemAvailableOK || result.Linux.MemAvailable == 0)
|
||||
printCheck("Memory Usage", fmt.Sprintf("%.1f%%", result.Linux.MemUsedPercent), result.Linux.MemUsedPercent < 85)
|
||||
|
||||
// Linux-specific kernel checks
|
||||
if result.Linux.IsLinux && result.Linux.ShmMax > 0 {
|
||||
fmt.Println("\n Linux Kernel:")
|
||||
printCheck("shmmax", humanize.Bytes(uint64(result.Linux.ShmMax)), result.Linux.ShmMaxOK)
|
||||
printCheck("shmall", humanize.Comma(result.Linux.ShmAll)+" pages", result.Linux.ShmAllOK)
|
||||
}
|
||||
|
||||
// PostgreSQL checks
|
||||
fmt.Println("\n PostgreSQL:")
|
||||
printCheck("Version", result.PostgreSQL.Version, true)
|
||||
printCheck("max_locks_per_transaction", fmt.Sprintf("%d → %d (auto-boost)",
|
||||
result.PostgreSQL.MaxLocksPerTransaction, result.Archive.RecommendedLockBoost),
|
||||
printCheck("max_locks_per_transaction", fmt.Sprintf("%s → %s (auto-boost)",
|
||||
humanize.Comma(int64(result.PostgreSQL.MaxLocksPerTransaction)),
|
||||
humanize.Comma(int64(result.Archive.RecommendedLockBoost))),
|
||||
true)
|
||||
printCheck("maintenance_work_mem", fmt.Sprintf("%s → 2GB (auto-boost)",
|
||||
result.PostgreSQL.MaintenanceWorkMem), true)
|
||||
@@ -353,16 +363,17 @@ func (e *Engine) printPreflightSummary(result *PreflightResult) {
|
||||
|
||||
// Archive analysis
|
||||
fmt.Println("\n Archive Analysis:")
|
||||
printInfo("Total databases", fmt.Sprintf("%d", result.Archive.TotalDatabases))
|
||||
printInfo("Total BLOBs detected", fmt.Sprintf("%d", result.Archive.TotalBlobCount))
|
||||
printInfo("Total databases", humanize.Comma(int64(result.Archive.TotalDatabases)))
|
||||
printInfo("Total BLOBs detected", humanize.Comma(int64(result.Archive.TotalBlobCount)))
|
||||
if len(result.Archive.BlobsByDB) > 0 {
|
||||
fmt.Println(" Databases with BLOBs:")
|
||||
for db, count := range result.Archive.BlobsByDB {
|
||||
status := "✓"
|
||||
if count > 1000 {
|
||||
status = "⚠"
|
||||
status := "⚠"
|
||||
_ = status
|
||||
}
|
||||
fmt.Printf(" %s %s: %d BLOBs\n", status, db, count)
|
||||
fmt.Printf(" %s %s: %s BLOBs\n", status, db, humanize.Comma(int64(count)))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -390,23 +401,6 @@ func printInfo(name, value string) {
|
||||
fmt.Printf(" ℹ %s: %s\n", name, value)
|
||||
}
|
||||
|
||||
// formatBytesLong is a local formatting helper for preflight display
|
||||
func formatBytesLong(bytes int64) string {
|
||||
if bytes == 0 {
|
||||
return "unknown"
|
||||
}
|
||||
const unit = 1024
|
||||
if bytes < unit {
|
||||
return fmt.Sprintf("%d B", bytes)
|
||||
}
|
||||
div, exp := int64(unit), 0
|
||||
for n := bytes / unit; n >= unit; n /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
|
||||
func parseMemoryToMB(memStr string) int {
|
||||
memStr = strings.ToUpper(strings.TrimSpace(memStr))
|
||||
var value int
|
||||
|
||||
Reference in New Issue
Block a user