|
|
|
@@ -442,16 +442,18 @@ func (e *Engine) restorePostgreSQLSQL(ctx context.Context, archivePath, targetDB
|
|
|
|
var cmd []string
|
|
|
|
var cmd []string
|
|
|
|
|
|
|
|
|
|
|
|
// For localhost, omit -h to use Unix socket (avoids Ident auth issues)
|
|
|
|
// For localhost, omit -h to use Unix socket (avoids Ident auth issues)
|
|
|
|
|
|
|
|
// But always include -p for port (in case of non-standard port)
|
|
|
|
hostArg := ""
|
|
|
|
hostArg := ""
|
|
|
|
|
|
|
|
portArg := fmt.Sprintf("-p %d", e.cfg.Port)
|
|
|
|
if e.cfg.Host != "localhost" && e.cfg.Host != "" {
|
|
|
|
if e.cfg.Host != "localhost" && e.cfg.Host != "" {
|
|
|
|
hostArg = fmt.Sprintf("-h %s -p %d", e.cfg.Host, e.cfg.Port)
|
|
|
|
hostArg = fmt.Sprintf("-h %s", e.cfg.Host)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if compressed {
|
|
|
|
if compressed {
|
|
|
|
// Use ON_ERROR_STOP=1 to fail fast on first error (prevents millions of errors on truncated dumps)
|
|
|
|
// Use ON_ERROR_STOP=1 to fail fast on first error (prevents millions of errors on truncated dumps)
|
|
|
|
psqlCmd := fmt.Sprintf("psql -U %s -d %s -v ON_ERROR_STOP=1", e.cfg.User, targetDB)
|
|
|
|
psqlCmd := fmt.Sprintf("psql %s -U %s -d %s -v ON_ERROR_STOP=1", portArg, e.cfg.User, targetDB)
|
|
|
|
if hostArg != "" {
|
|
|
|
if hostArg != "" {
|
|
|
|
psqlCmd = fmt.Sprintf("psql %s -U %s -d %s -v ON_ERROR_STOP=1", hostArg, e.cfg.User, targetDB)
|
|
|
|
psqlCmd = fmt.Sprintf("psql %s %s -U %s -d %s -v ON_ERROR_STOP=1", hostArg, portArg, e.cfg.User, targetDB)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Set PGPASSWORD in the bash command for password-less auth
|
|
|
|
// Set PGPASSWORD in the bash command for password-less auth
|
|
|
|
cmd = []string{
|
|
|
|
cmd = []string{
|
|
|
|
@@ -472,6 +474,7 @@ func (e *Engine) restorePostgreSQLSQL(ctx context.Context, archivePath, targetDB
|
|
|
|
} else {
|
|
|
|
} else {
|
|
|
|
cmd = []string{
|
|
|
|
cmd = []string{
|
|
|
|
"psql",
|
|
|
|
"psql",
|
|
|
|
|
|
|
|
"-p", fmt.Sprintf("%d", e.cfg.Port),
|
|
|
|
"-U", e.cfg.User,
|
|
|
|
"-U", e.cfg.User,
|
|
|
|
"-d", targetDB,
|
|
|
|
"-d", targetDB,
|
|
|
|
"-v", "ON_ERROR_STOP=1",
|
|
|
|
"-v", "ON_ERROR_STOP=1",
|
|
|
|
@@ -1084,6 +1087,16 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
}()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Check for context cancellation before starting
|
|
|
|
|
|
|
|
if ctx.Err() != nil {
|
|
|
|
|
|
|
|
e.log.Warn("Context cancelled - skipping database restore", "file", filename)
|
|
|
|
|
|
|
|
atomic.AddInt32(&failCount, 1)
|
|
|
|
|
|
|
|
restoreErrorsMu.Lock()
|
|
|
|
|
|
|
|
restoreErrors = multierror.Append(restoreErrors, fmt.Errorf("%s: restore skipped (context cancelled)", strings.TrimSuffix(strings.TrimSuffix(filename, ".dump"), ".sql.gz")))
|
|
|
|
|
|
|
|
restoreErrorsMu.Unlock()
|
|
|
|
|
|
|
|
return
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Track timing for this database restore
|
|
|
|
// Track timing for this database restore
|
|
|
|
dbRestoreStart := time.Now()
|
|
|
|
dbRestoreStart := time.Now()
|
|
|
|
|
|
|
|
|
|
|
|
@@ -1201,6 +1214,35 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
|
|
|
|
successCountFinal := int(atomic.LoadInt32(&successCount))
|
|
|
|
successCountFinal := int(atomic.LoadInt32(&successCount))
|
|
|
|
failCountFinal := int(atomic.LoadInt32(&failCount))
|
|
|
|
failCountFinal := int(atomic.LoadInt32(&failCount))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// SANITY CHECK: Verify all databases were accounted for
|
|
|
|
|
|
|
|
// This catches any goroutine that exited without updating counters
|
|
|
|
|
|
|
|
accountedFor := successCountFinal + failCountFinal
|
|
|
|
|
|
|
|
if accountedFor != totalDBs {
|
|
|
|
|
|
|
|
missingCount := totalDBs - accountedFor
|
|
|
|
|
|
|
|
e.log.Error("INTERNAL ERROR: Some database restore goroutines did not report status",
|
|
|
|
|
|
|
|
"expected", totalDBs,
|
|
|
|
|
|
|
|
"success", successCountFinal,
|
|
|
|
|
|
|
|
"failed", failCountFinal,
|
|
|
|
|
|
|
|
"unaccounted", missingCount)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Treat unaccounted databases as failures
|
|
|
|
|
|
|
|
failCountFinal += missingCount
|
|
|
|
|
|
|
|
restoreErrorsMu.Lock()
|
|
|
|
|
|
|
|
restoreErrors = multierror.Append(restoreErrors, fmt.Errorf("%d database(s) did not complete (possible goroutine crash or deadlock)", missingCount))
|
|
|
|
|
|
|
|
restoreErrorsMu.Unlock()
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// CRITICAL: Check if no databases were restored at all
|
|
|
|
|
|
|
|
if successCountFinal == 0 {
|
|
|
|
|
|
|
|
e.progress.Fail(fmt.Sprintf("Cluster restore FAILED: 0 of %d databases restored", totalDBs))
|
|
|
|
|
|
|
|
operation.Fail("No databases were restored")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if failCountFinal > 0 && restoreErrors != nil {
|
|
|
|
|
|
|
|
return fmt.Errorf("cluster restore failed: all %d database(s) failed:\n%s", failCountFinal, restoreErrors.Error())
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
return fmt.Errorf("cluster restore failed: no databases were restored (0 of %d total). Check PostgreSQL logs for details", totalDBs)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if failCountFinal > 0 {
|
|
|
|
if failCountFinal > 0 {
|
|
|
|
// Format multi-error with detailed output
|
|
|
|
// Format multi-error with detailed output
|
|
|
|
restoreErrors.ErrorFormat = func(errs []error) string {
|
|
|
|
restoreErrors.ErrorFormat = func(errs []error) string {
|
|
|
|
@@ -2038,56 +2080,100 @@ func (e *Engine) boostPostgreSQLSettings(ctx context.Context, lockBoostValue int
|
|
|
|
// Wait for PostgreSQL to be ready
|
|
|
|
// Wait for PostgreSQL to be ready
|
|
|
|
time.Sleep(3 * time.Second)
|
|
|
|
time.Sleep(3 * time.Second)
|
|
|
|
} else {
|
|
|
|
} else {
|
|
|
|
// Cannot restart - warn user loudly
|
|
|
|
// Cannot restart - warn user but continue
|
|
|
|
e.log.Error("=" + strings.Repeat("=", 70))
|
|
|
|
// The setting is written to postgresql.auto.conf and will take effect on next restart
|
|
|
|
e.log.Error("WARNING: max_locks_per_transaction change requires PostgreSQL restart!")
|
|
|
|
e.log.Warn("=" + strings.Repeat("=", 70))
|
|
|
|
e.log.Error("Current value: " + strconv.Itoa(original.MaxLocks) + ", needed: " + strconv.Itoa(lockBoostValue))
|
|
|
|
e.log.Warn("NOTE: max_locks_per_transaction change requires PostgreSQL restart")
|
|
|
|
e.log.Error("Restore may fail with 'out of shared memory' error on BLOB-heavy databases.")
|
|
|
|
e.log.Warn("Current value: " + strconv.Itoa(original.MaxLocks) + ", target: " + strconv.Itoa(lockBoostValue))
|
|
|
|
e.log.Error("")
|
|
|
|
e.log.Warn("")
|
|
|
|
e.log.Error("To fix manually:")
|
|
|
|
e.log.Warn("The setting has been saved to postgresql.auto.conf and will take")
|
|
|
|
e.log.Error(" 1. sudo systemctl restart postgresql")
|
|
|
|
e.log.Warn("effect on the next PostgreSQL restart. If restore fails with")
|
|
|
|
e.log.Error(" 2. Or: sudo -u postgres pg_ctl restart -D $PGDATA")
|
|
|
|
e.log.Warn("'out of shared memory' errors, ask your DBA to restart PostgreSQL.")
|
|
|
|
e.log.Error(" 3. Then re-run the restore")
|
|
|
|
e.log.Warn("")
|
|
|
|
e.log.Error("=" + strings.Repeat("=", 70))
|
|
|
|
e.log.Warn("Continuing with restore - this may succeed if your databases")
|
|
|
|
// Continue anyway - might work for small restores
|
|
|
|
e.log.Warn("don't have many large objects (BLOBs).")
|
|
|
|
|
|
|
|
e.log.Warn("=" + strings.Repeat("=", 70))
|
|
|
|
|
|
|
|
// Continue anyway - might work for small restores or DBs without BLOBs
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
return original, nil
|
|
|
|
return original, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// canRestartPostgreSQL checks if we have the ability to restart PostgreSQL
|
|
|
|
|
|
|
|
// Returns false if running in a restricted environment (e.g., su postgres on enterprise systems)
|
|
|
|
|
|
|
|
func (e *Engine) canRestartPostgreSQL() bool {
|
|
|
|
|
|
|
|
// Check if we're running as postgres user - if so, we likely can't restart
|
|
|
|
|
|
|
|
// because PostgreSQL is managed by init/systemd, not directly by pg_ctl
|
|
|
|
|
|
|
|
currentUser := os.Getenv("USER")
|
|
|
|
|
|
|
|
if currentUser == "" {
|
|
|
|
|
|
|
|
currentUser = os.Getenv("LOGNAME")
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// If we're the postgres user, check if we have sudo access
|
|
|
|
|
|
|
|
if currentUser == "postgres" {
|
|
|
|
|
|
|
|
// Try a quick sudo check - if this fails, we can't restart
|
|
|
|
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
|
|
|
|
|
|
defer cancel()
|
|
|
|
|
|
|
|
cmd := exec.CommandContext(ctx, "sudo", "-n", "true")
|
|
|
|
|
|
|
|
cmd.Stdin = nil
|
|
|
|
|
|
|
|
if err := cmd.Run(); err != nil {
|
|
|
|
|
|
|
|
e.log.Info("Running as postgres user without sudo access - cannot restart PostgreSQL",
|
|
|
|
|
|
|
|
"user", currentUser,
|
|
|
|
|
|
|
|
"hint", "Ask system administrator to restart PostgreSQL if needed")
|
|
|
|
|
|
|
|
return false
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return true
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// tryRestartPostgreSQL attempts to restart PostgreSQL using various methods
|
|
|
|
// tryRestartPostgreSQL attempts to restart PostgreSQL using various methods
|
|
|
|
// Returns true if restart was successful
|
|
|
|
// Returns true if restart was successful
|
|
|
|
|
|
|
|
// IMPORTANT: Uses short timeouts and non-interactive sudo to avoid blocking on password prompts
|
|
|
|
|
|
|
|
// NOTE: This function will return false immediately if running as postgres without sudo
|
|
|
|
func (e *Engine) tryRestartPostgreSQL(ctx context.Context) bool {
|
|
|
|
func (e *Engine) tryRestartPostgreSQL(ctx context.Context) bool {
|
|
|
|
|
|
|
|
// First check if we can even attempt a restart
|
|
|
|
|
|
|
|
if !e.canRestartPostgreSQL() {
|
|
|
|
|
|
|
|
e.log.Info("Skipping PostgreSQL restart attempt (no privileges)")
|
|
|
|
|
|
|
|
return false
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
e.progress.Update("Attempting PostgreSQL restart for lock settings...")
|
|
|
|
e.progress.Update("Attempting PostgreSQL restart for lock settings...")
|
|
|
|
|
|
|
|
|
|
|
|
// Method 1: systemctl (most common on modern Linux)
|
|
|
|
// Use short timeout for each restart attempt (don't block on sudo password prompts)
|
|
|
|
cmd := exec.CommandContext(ctx, "sudo", "systemctl", "restart", "postgresql")
|
|
|
|
runWithTimeout := func(args ...string) bool {
|
|
|
|
if err := cmd.Run(); err == nil {
|
|
|
|
cmdCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
|
|
|
|
|
|
defer cancel()
|
|
|
|
|
|
|
|
cmd := exec.CommandContext(cmdCtx, args[0], args[1:]...)
|
|
|
|
|
|
|
|
// Set stdin to /dev/null to prevent sudo from waiting for password
|
|
|
|
|
|
|
|
cmd.Stdin = nil
|
|
|
|
|
|
|
|
return cmd.Run() == nil
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Method 1: systemctl (most common on modern Linux) - use sudo -n for non-interactive
|
|
|
|
|
|
|
|
if runWithTimeout("sudo", "-n", "systemctl", "restart", "postgresql") {
|
|
|
|
return true
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Method 2: systemctl with version suffix (e.g., postgresql-15)
|
|
|
|
// Method 2: systemctl with version suffix (e.g., postgresql-15)
|
|
|
|
for _, ver := range []string{"17", "16", "15", "14", "13", "12"} {
|
|
|
|
for _, ver := range []string{"17", "16", "15", "14", "13", "12"} {
|
|
|
|
cmd = exec.CommandContext(ctx, "sudo", "systemctl", "restart", "postgresql-"+ver)
|
|
|
|
if runWithTimeout("sudo", "-n", "systemctl", "restart", "postgresql-"+ver) {
|
|
|
|
if err := cmd.Run(); err == nil {
|
|
|
|
|
|
|
|
return true
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Method 3: service command (older systems)
|
|
|
|
// Method 3: service command (older systems)
|
|
|
|
cmd = exec.CommandContext(ctx, "sudo", "service", "postgresql", "restart")
|
|
|
|
if runWithTimeout("sudo", "-n", "service", "postgresql", "restart") {
|
|
|
|
if err := cmd.Run(); err == nil {
|
|
|
|
|
|
|
|
return true
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Method 4: pg_ctl as postgres user
|
|
|
|
// Method 4: pg_ctl as postgres user (if we ARE postgres user, no sudo needed)
|
|
|
|
cmd = exec.CommandContext(ctx, "sudo", "-u", "postgres", "pg_ctl", "restart", "-D", "/var/lib/postgresql/data", "-m", "fast")
|
|
|
|
if runWithTimeout("pg_ctl", "restart", "-D", "/var/lib/postgresql/data", "-m", "fast") {
|
|
|
|
if err := cmd.Run(); err == nil {
|
|
|
|
|
|
|
|
return true
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Method 5: Try common PGDATA paths
|
|
|
|
// Method 5: Try common PGDATA paths with pg_ctl directly (for postgres user)
|
|
|
|
pgdataPaths := []string{
|
|
|
|
pgdataPaths := []string{
|
|
|
|
"/var/lib/pgsql/data",
|
|
|
|
"/var/lib/pgsql/data",
|
|
|
|
"/var/lib/pgsql/17/data",
|
|
|
|
"/var/lib/pgsql/17/data",
|
|
|
|
@@ -2098,8 +2184,7 @@ func (e *Engine) tryRestartPostgreSQL(ctx context.Context) bool {
|
|
|
|
"/var/lib/postgresql/15/main",
|
|
|
|
"/var/lib/postgresql/15/main",
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for _, pgdata := range pgdataPaths {
|
|
|
|
for _, pgdata := range pgdataPaths {
|
|
|
|
cmd = exec.CommandContext(ctx, "sudo", "-u", "postgres", "pg_ctl", "restart", "-D", pgdata, "-m", "fast")
|
|
|
|
if runWithTimeout("pg_ctl", "restart", "-D", pgdata, "-m", "fast") {
|
|
|
|
if err := cmd.Run(); err == nil {
|
|
|
|
|
|
|
|
return true
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|