From f801c7a54912aff11153d73255f62270e23f580f Mon Sep 17 00:00:00 2001 From: Renz Date: Fri, 14 Nov 2025 09:42:52 +0000 Subject: [PATCH] add: version check psql db --- create_d7030_test.sh | 272 ++++++++++++++++++++++++++++++ internal/restore/engine.go | 23 +++ internal/restore/version_check.go | 231 +++++++++++++++++++++++++ 3 files changed, 526 insertions(+) create mode 100755 create_d7030_test.sh create mode 100644 internal/restore/version_check.go diff --git a/create_d7030_test.sh b/create_d7030_test.sh new file mode 100755 index 0000000..da45439 --- /dev/null +++ b/create_d7030_test.sh @@ -0,0 +1,272 @@ +#!/usr/bin/env bash +# create_d7030_test.sh +# Create a realistic d7030 database with tables, data, and many BLOBs to test large object restore + +set -euo pipefail + +DB_NAME="d7030" +NUM_DOCUMENTS=3000 # Number of documents with BLOBs (increased to stress test locks) +NUM_IMAGES=2000 # Number of image records (increased to stress test locks) + +echo "Creating database: $DB_NAME" + +# Drop if exists +sudo -u postgres psql -c "DROP DATABASE IF EXISTS $DB_NAME;" 2>/dev/null || true + +# Create database +sudo -u postgres psql -c "CREATE DATABASE $DB_NAME;" + +echo "Creating schema and tables..." + +# Enable pgcrypto extension for gen_random_bytes +sudo -u postgres psql -d "$DB_NAME" -c "CREATE EXTENSION IF NOT EXISTS pgcrypto;" + +# Create schema with realistic business tables +sudo -u postgres psql -d "$DB_NAME" <<'EOF' +-- Create tables for a document management system +CREATE TABLE departments ( + dept_id SERIAL PRIMARY KEY, + dept_name VARCHAR(100) NOT NULL, + created_at TIMESTAMP DEFAULT NOW() +); + +CREATE TABLE employees ( + emp_id SERIAL PRIMARY KEY, + dept_id INTEGER REFERENCES departments(dept_id), + first_name VARCHAR(50) NOT NULL, + last_name VARCHAR(50) NOT NULL, + email VARCHAR(100) UNIQUE, + hire_date DATE DEFAULT CURRENT_DATE +); + +CREATE TABLE document_types ( + type_id SERIAL PRIMARY KEY, + type_name VARCHAR(50) NOT NULL, + description TEXT +); + +-- Table with large objects (BLOBs) +CREATE TABLE documents ( + doc_id SERIAL PRIMARY KEY, + emp_id INTEGER REFERENCES employees(emp_id), + type_id INTEGER REFERENCES document_types(type_id), + title VARCHAR(255) NOT NULL, + description TEXT, + file_data OID, -- Large object reference + file_size INTEGER, + mime_type VARCHAR(100), + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() +); + +CREATE TABLE images ( + image_id SERIAL PRIMARY KEY, + doc_id INTEGER REFERENCES documents(doc_id), + image_name VARCHAR(255), + image_data OID, -- Large object reference + thumbnail_data OID, -- Another large object + width INTEGER, + height INTEGER, + created_at TIMESTAMP DEFAULT NOW() +); + +CREATE TABLE audit_log ( + log_id SERIAL PRIMARY KEY, + table_name VARCHAR(50), + record_id INTEGER, + action VARCHAR(20), + changed_by INTEGER, + changed_at TIMESTAMP DEFAULT NOW(), + details JSONB +); + +-- Create indexes +CREATE INDEX idx_documents_emp ON documents(emp_id); +CREATE INDEX idx_documents_type ON documents(type_id); +CREATE INDEX idx_images_doc ON images(doc_id); +CREATE INDEX idx_audit_table ON audit_log(table_name, record_id); + +-- Insert reference data +INSERT INTO departments (dept_name) VALUES + ('Engineering'), ('Sales'), ('Marketing'), ('HR'), ('Finance'); + +INSERT INTO document_types (type_name, description) VALUES + ('Contract', 'Legal contracts and agreements'), + ('Invoice', 'Financial invoices and receipts'), + ('Report', 'Business reports and analysis'), + ('Manual', 'Technical manuals and guides'), + ('Presentation', 'Presentation slides and materials'); + +-- Insert employees +INSERT INTO employees (dept_id, first_name, last_name, email) +SELECT + (random() * 4 + 1)::INTEGER, + 'Employee_' || generate_series, + 'LastName_' || generate_series, + 'employee' || generate_series || '@d7030.com' +FROM generate_series(1, 50); + +EOF + +echo "Inserting documents with large objects (BLOBs)..." + +# Create a temporary file with random data for importing in postgres home +TEMP_FILE="/var/lib/pgsql/test_blob_data.bin" +sudo dd if=/dev/urandom of="$TEMP_FILE" bs=1024 count=50 2>/dev/null +sudo chown postgres:postgres "$TEMP_FILE" + +# Create documents with actual large objects using lo_import +sudo -u postgres psql -d "$DB_NAME" </dev/null +sudo dd if=/dev/urandom of="$TEMP_THUMB" bs=1024 count=10 2>/dev/null +sudo chown postgres:postgres "$TEMP_IMAGE" "$TEMP_THUMB" + +# Create images with multiple large objects per record +sudo -u postgres psql -d "$DB_NAME" < Major: 17, Minor: 7 +func ParsePostgreSQLVersion(versionStr string) (*VersionInfo, error) { + // Match patterns like "PostgreSQL 17.7", "PostgreSQL 13.11", "PostgreSQL 10.23" + re := regexp.MustCompile(`PostgreSQL\s+(\d+)\.(\d+)`) + matches := re.FindStringSubmatch(versionStr) + + if len(matches) < 3 { + return nil, fmt.Errorf("could not parse PostgreSQL version from: %s", versionStr) + } + + major, err := strconv.Atoi(matches[1]) + if err != nil { + return nil, fmt.Errorf("invalid major version: %s", matches[1]) + } + + minor, err := strconv.Atoi(matches[2]) + if err != nil { + return nil, fmt.Errorf("invalid minor version: %s", matches[2]) + } + + return &VersionInfo{ + Major: major, + Minor: minor, + Full: versionStr, + }, nil +} + +// GetDumpFileVersion extracts the PostgreSQL version from a dump file +// Uses pg_restore -l to read the dump metadata +func GetDumpFileVersion(dumpPath string) (*VersionInfo, error) { + cmd := exec.Command("pg_restore", "-l", dumpPath) + output, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("failed to read dump file metadata: %w (output: %s)", err, string(output)) + } + + // Look for "Dumped from database version: X.Y.Z" in output + re := regexp.MustCompile(`Dumped from database version:\s+(\d+)\.(\d+)`) + matches := re.FindStringSubmatch(string(output)) + + if len(matches) < 3 { + // Try alternate format in some dumps + re = regexp.MustCompile(`PostgreSQL database dump.*(\d+)\.(\d+)`) + matches = re.FindStringSubmatch(string(output)) + } + + if len(matches) < 3 { + return nil, fmt.Errorf("could not find version information in dump file") + } + + major, _ := strconv.Atoi(matches[1]) + minor, _ := strconv.Atoi(matches[2]) + + return &VersionInfo{ + Major: major, + Minor: minor, + Full: fmt.Sprintf("PostgreSQL %d.%d", major, minor), + }, nil +} + +// CheckVersionCompatibility checks if restoring from source version to target version is safe +func CheckVersionCompatibility(sourceVer, targetVer *VersionInfo) *VersionCompatibilityResult { + result := &VersionCompatibilityResult{ + Compatible: true, + SourceVersion: sourceVer, + TargetVersion: targetVer, + } + + // Same major version - always compatible + if sourceVer.Major == targetVer.Major { + result.Level = CompatibilityLevelSafe + result.Message = "Same major version - fully compatible" + return result + } + + // Downgrade - not supported + if sourceVer.Major > targetVer.Major { + result.Compatible = false + result.Level = CompatibilityLevelUnsupported + result.Message = fmt.Sprintf("Downgrade from PostgreSQL %d to %d is not supported", sourceVer.Major, targetVer.Major) + result.Warnings = append(result.Warnings, "Database downgrades require pg_dump from the target version") + return result + } + + // Upgrade - check how many major versions + versionDiff := targetVer.Major - sourceVer.Major + + if versionDiff == 1 { + // One major version upgrade - generally safe + result.Level = CompatibilityLevelSafe + result.Message = fmt.Sprintf("Upgrading from PostgreSQL %d to %d - officially supported", sourceVer.Major, targetVer.Major) + } else if versionDiff <= 3 { + // 2-3 major versions - should work but review release notes + result.Level = CompatibilityLevelWarning + result.Message = fmt.Sprintf("Upgrading from PostgreSQL %d to %d - supported but review release notes", sourceVer.Major, targetVer.Major) + result.Warnings = append(result.Warnings, + fmt.Sprintf("You are jumping %d major versions - some features may have changed", versionDiff)) + result.Warnings = append(result.Warnings, + "Review release notes for deprecated features or behavior changes") + } else { + // 4+ major versions - high risk + result.Level = CompatibilityLevelRisky + result.Message = fmt.Sprintf("Upgrading from PostgreSQL %d to %d - large version jump", sourceVer.Major, targetVer.Major) + result.Warnings = append(result.Warnings, + fmt.Sprintf("WARNING: Jumping %d major versions may encounter compatibility issues", versionDiff)) + result.Warnings = append(result.Warnings, + "Deprecated features from PostgreSQL "+strconv.Itoa(sourceVer.Major)+" may not exist in "+strconv.Itoa(targetVer.Major)) + result.Warnings = append(result.Warnings, + "Extensions may need updates or may be incompatible") + result.Warnings = append(result.Warnings, + "Test thoroughly in a non-production environment first") + result.Recommendations = append(result.Recommendations, + "Consider using --schema-only first to validate schema compatibility") + result.Recommendations = append(result.Recommendations, + "Review PostgreSQL release notes for versions "+strconv.Itoa(sourceVer.Major)+" through "+strconv.Itoa(targetVer.Major)) + } + + // Add general upgrade advice + if versionDiff > 0 { + result.Recommendations = append(result.Recommendations, + "Run ANALYZE on all tables after restore for optimal query performance") + } + + return result +} + +// CompatibilityLevel indicates the risk level of version compatibility +type CompatibilityLevel int + +const ( + CompatibilityLevelSafe CompatibilityLevel = iota + CompatibilityLevelWarning + CompatibilityLevelRisky + CompatibilityLevelUnsupported +) + +func (c CompatibilityLevel) String() string { + switch c { + case CompatibilityLevelSafe: + return "SAFE" + case CompatibilityLevelWarning: + return "WARNING" + case CompatibilityLevelRisky: + return "RISKY" + case CompatibilityLevelUnsupported: + return "UNSUPPORTED" + default: + return "UNKNOWN" + } +} + +// VersionCompatibilityResult contains the result of version compatibility check +type VersionCompatibilityResult struct { + Compatible bool + Level CompatibilityLevel + SourceVersion *VersionInfo + TargetVersion *VersionInfo + Message string + Warnings []string + Recommendations []string +} + +// CheckRestoreVersionCompatibility performs version check for a restore operation +func (e *Engine) CheckRestoreVersionCompatibility(ctx context.Context, dumpPath string) (*VersionCompatibilityResult, error) { + // Get dump file version + dumpVer, err := GetDumpFileVersion(dumpPath) + if err != nil { + // Not critical if we can't read version - continue with warning + e.log.Warn("Could not determine dump file version", "error", err) + return nil, nil + } + + // Get target database version + targetVerStr, err := e.db.GetVersion(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get target database version: %w", err) + } + + targetVer, err := ParsePostgreSQLVersion(targetVerStr) + if err != nil { + return nil, fmt.Errorf("failed to parse target version: %w", err) + } + + // Check compatibility + result := CheckVersionCompatibility(dumpVer, targetVer) + + // Log the results + e.log.Info("Version compatibility check", + "source", dumpVer.Full, + "target", targetVer.Full, + "level", result.Level.String()) + + if len(result.Warnings) > 0 { + for _, warning := range result.Warnings { + e.log.Warn(warning) + } + } + + return result, nil +} + +// ValidatePostgreSQLDatabase ensures we're working with a PostgreSQL database +func ValidatePostgreSQLDatabase(db database.Database) error { + // Type assertion to check if it's PostgreSQL + switch db.(type) { + case *database.PostgreSQL: + return nil + default: + return fmt.Errorf("version compatibility checks only supported for PostgreSQL") + } +}