add: version check psql db

2025-11-14 09:42:52 +00:00
parent 98cb879ee1
commit f801c7a549
3 changed files with 526 additions and 0 deletions
--- a/create_d7030_test.sh
+++ b/create_d7030_test.sh
@@ -0,0 +1,272 @@
+#!/usr/bin/env bash
+# create_d7030_test.sh
+# Create a realistic d7030 database with tables, data, and many BLOBs to test large object restore
+
+set -euo pipefail
+
+DB_NAME="d7030"
+NUM_DOCUMENTS=3000  # Number of documents with BLOBs (increased to stress test locks)
+NUM_IMAGES=2000     # Number of image records (increased to stress test locks)
+
+echo "Creating database: $DB_NAME"
+
+# Drop if exists
+sudo -u postgres psql -c "DROP DATABASE IF EXISTS $DB_NAME;" 2>/dev/null || true
+
+# Create database
+sudo -u postgres psql -c "CREATE DATABASE $DB_NAME;"
+
+echo "Creating schema and tables..."
+
+# Enable pgcrypto extension for gen_random_bytes
+sudo -u postgres psql -d "$DB_NAME" -c "CREATE EXTENSION IF NOT EXISTS pgcrypto;"
+
+# Create schema with realistic business tables
+sudo -u postgres psql -d "$DB_NAME" <<'EOF'
+-- Create tables for a document management system
+CREATE TABLE departments (
+    dept_id SERIAL PRIMARY KEY,
+    dept_name VARCHAR(100) NOT NULL,
+    created_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE TABLE employees (
+    emp_id SERIAL PRIMARY KEY,
+    dept_id INTEGER REFERENCES departments(dept_id),
+    first_name VARCHAR(50) NOT NULL,
+    last_name VARCHAR(50) NOT NULL,
+    email VARCHAR(100) UNIQUE,
+    hire_date DATE DEFAULT CURRENT_DATE
+);
+
+CREATE TABLE document_types (
+    type_id SERIAL PRIMARY KEY,
+    type_name VARCHAR(50) NOT NULL,
+    description TEXT
+);
+
+-- Table with large objects (BLOBs)
+CREATE TABLE documents (
+    doc_id SERIAL PRIMARY KEY,
+    emp_id INTEGER REFERENCES employees(emp_id),
+    type_id INTEGER REFERENCES document_types(type_id),
+    title VARCHAR(255) NOT NULL,
+    description TEXT,
+    file_data OID,  -- Large object reference
+    file_size INTEGER,
+    mime_type VARCHAR(100),
+    created_at TIMESTAMP DEFAULT NOW(),
+    updated_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE TABLE images (
+    image_id SERIAL PRIMARY KEY,
+    doc_id INTEGER REFERENCES documents(doc_id),
+    image_name VARCHAR(255),
+    image_data OID,  -- Large object reference
+    thumbnail_data OID,  -- Another large object
+    width INTEGER,
+    height INTEGER,
+    created_at TIMESTAMP DEFAULT NOW()
+);
+
+CREATE TABLE audit_log (
+    log_id SERIAL PRIMARY KEY,
+    table_name VARCHAR(50),
+    record_id INTEGER,
+    action VARCHAR(20),
+    changed_by INTEGER,
+    changed_at TIMESTAMP DEFAULT NOW(),
+    details JSONB
+);
+
+-- Create indexes
+CREATE INDEX idx_documents_emp ON documents(emp_id);
+CREATE INDEX idx_documents_type ON documents(type_id);
+CREATE INDEX idx_images_doc ON images(doc_id);
+CREATE INDEX idx_audit_table ON audit_log(table_name, record_id);
+
+-- Insert reference data
+INSERT INTO departments (dept_name) VALUES 
+    ('Engineering'), ('Sales'), ('Marketing'), ('HR'), ('Finance');
+
+INSERT INTO document_types (type_name, description) VALUES
+    ('Contract', 'Legal contracts and agreements'),
+    ('Invoice', 'Financial invoices and receipts'),
+    ('Report', 'Business reports and analysis'),
+    ('Manual', 'Technical manuals and guides'),
+    ('Presentation', 'Presentation slides and materials');
+
+-- Insert employees
+INSERT INTO employees (dept_id, first_name, last_name, email)
+SELECT 
+    (random() * 4 + 1)::INTEGER,
+    'Employee_' || generate_series,
+    'LastName_' || generate_series,
+    'employee' || generate_series || '@d7030.com'
+FROM generate_series(1, 50);
+
+EOF
+
+echo "Inserting documents with large objects (BLOBs)..."
+
+# Create a temporary file with random data for importing in postgres home
+TEMP_FILE="/var/lib/pgsql/test_blob_data.bin"
+sudo dd if=/dev/urandom of="$TEMP_FILE" bs=1024 count=50 2>/dev/null
+sudo chown postgres:postgres "$TEMP_FILE"
+
+# Create documents with actual large objects using lo_import
+sudo -u postgres psql -d "$DB_NAME" <<EOF
+DO \$\$
+DECLARE
+    v_emp_id INTEGER;
+    v_type_id INTEGER;
+    v_loid OID;
+BEGIN
+    FOR i IN 1..$NUM_DOCUMENTS LOOP
+        -- Random employee and document type
+        v_emp_id := (random() * 49 + 1)::INTEGER;
+        v_type_id := (random() * 4 + 1)::INTEGER;
+        
+        -- Import file as large object (creates a unique BLOB for each)
+        v_loid := lo_import('$TEMP_FILE');
+        
+        -- Insert document record
+        INSERT INTO documents (emp_id, type_id, title, description, file_data, file_size, mime_type)
+        VALUES (
+            v_emp_id,
+            v_type_id,
+            'Document_' || i || '_' || (CASE v_type_id 
+                WHEN 1 THEN 'Contract'
+                WHEN 2 THEN 'Invoice'
+                WHEN 3 THEN 'Report'
+                WHEN 4 THEN 'Manual'
+                ELSE 'Presentation'
+            END),
+            'This is a test document with large object data. Document number ' || i,
+            v_loid,
+            51200,
+            (CASE v_type_id 
+                WHEN 1 THEN 'application/pdf'
+                WHEN 2 THEN 'application/pdf'
+                WHEN 3 THEN 'application/vnd.ms-excel'
+                WHEN 4 THEN 'application/pdf'
+                ELSE 'application/vnd.ms-powerpoint'
+            END)
+        );
+        
+        -- Progress indicator
+        IF i % 50 = 0 THEN
+            RAISE NOTICE 'Created % documents with BLOBs...', i;
+        END IF;
+    END LOOP;
+END \$\$;
+EOF
+
+rm -f "$TEMP_FILE"
+
+echo "Inserting images with large objects..."
+
+# Create temp files for image and thumbnail in postgres home
+TEMP_IMAGE="/var/lib/pgsql/test_image_data.bin"
+TEMP_THUMB="/var/lib/pgsql/test_thumb_data.bin"
+sudo dd if=/dev/urandom of="$TEMP_IMAGE" bs=1024 count=80 2>/dev/null
+sudo dd if=/dev/urandom of="$TEMP_THUMB" bs=1024 count=10 2>/dev/null
+sudo chown postgres:postgres "$TEMP_IMAGE" "$TEMP_THUMB"
+
+# Create images with multiple large objects per record
+sudo -u postgres psql -d "$DB_NAME" <<EOF
+DO \$\$
+DECLARE
+    v_doc_id INTEGER;
+    v_image_oid OID;
+    v_thumb_oid OID;
+BEGIN
+    FOR i IN 1..$NUM_IMAGES LOOP
+        -- Random document (only from successfully created documents)
+        SELECT doc_id INTO v_doc_id FROM documents ORDER BY random() LIMIT 1;
+        
+        IF v_doc_id IS NULL THEN
+            EXIT; -- No documents exist, skip images
+        END IF;
+        
+        -- Import full-size image as large object
+        v_image_oid := lo_import('$TEMP_IMAGE');
+        
+        -- Import thumbnail as large object
+        v_thumb_oid := lo_import('$TEMP_THUMB');
+        
+        -- Insert image record
+        INSERT INTO images (doc_id, image_name, image_data, thumbnail_data, width, height)
+        VALUES (
+            v_doc_id,
+            'Image_' || i || '.jpg',
+            v_image_oid,
+            v_thumb_oid,
+            (random() * 2000 + 800)::INTEGER,
+            (random() * 1500 + 600)::INTEGER
+        );
+        
+        IF i % 50 = 0 THEN
+            RAISE NOTICE 'Created % images with BLOBs...', i;
+        END IF;
+    END LOOP;
+END \$\$;
+EOF
+
+rm -f "$TEMP_IMAGE" "$TEMP_THUMB"
+
+echo "Inserting audit log data..."
+
+# Create audit log entries
+sudo -u postgres psql -d "$DB_NAME" <<EOF
+INSERT INTO audit_log (table_name, record_id, action, changed_by, details)
+SELECT 
+    'documents',
+    doc_id,
+    (ARRAY['INSERT', 'UPDATE', 'VIEW'])[(random() * 2 + 1)::INTEGER],
+    (random() * 49 + 1)::INTEGER,
+    jsonb_build_object(
+        'timestamp', NOW() - (random() * INTERVAL '90 days'),
+        'ip_address', '192.168.' || (random() * 255)::INTEGER || '.' || (random() * 255)::INTEGER,
+        'user_agent', 'Mozilla/5.0'
+    )
+FROM documents
+CROSS JOIN generate_series(1, 3);
+EOF
+
+echo ""
+echo "Database statistics:"
+sudo -u postgres psql -d "$DB_NAME" <<'EOF'
+SELECT 
+    'Departments' as table_name, 
+    COUNT(*) as row_count 
+FROM departments
+UNION ALL
+SELECT 'Employees', COUNT(*) FROM employees
+UNION ALL
+SELECT 'Document Types', COUNT(*) FROM document_types
+UNION ALL
+SELECT 'Documents (with BLOBs)', COUNT(*) FROM documents
+UNION ALL
+SELECT 'Images (with BLOBs)', COUNT(*) FROM images
+UNION ALL
+SELECT 'Audit Log', COUNT(*) FROM audit_log;
+
+-- Count large objects
+SELECT COUNT(*) as total_large_objects FROM pg_largeobject_metadata;
+
+-- Total size of large objects
+SELECT pg_size_pretty(SUM(pg_column_size(data))) as total_blob_size 
+FROM pg_largeobject;
+EOF
+
+echo ""
+echo "✅ Database $DB_NAME created successfully with realistic data and BLOBs!"
+echo ""
+echo "Large objects created:"
+echo "  - $NUM_DOCUMENTS documents (each with 1 BLOB)"
+echo "  - $NUM_IMAGES images (each with 2 BLOBs: full image + thumbnail)"
+echo "  - Total: ~$((NUM_DOCUMENTS + NUM_IMAGES * 2)) large objects"
+echo ""
+echo "You can now backup this database and test restore with large object locks."
--- a/internal/restore/engine.go
+++ b/internal/restore/engine.go
@@ -110,6 +110,29 @@ func (e *Engine) RestoreSingle(ctx context.Context, archivePath, targetDB string
 	format := DetectArchiveFormat(archivePath)
 	e.log.Info("Detected archive format", "format", format, "path", archivePath)

+	// Check version compatibility for PostgreSQL dumps
+	if format == FormatPostgreSQLDump || format == FormatPostgreSQLDumpGz {
+		if compatResult, err := e.CheckRestoreVersionCompatibility(ctx, archivePath); err == nil && compatResult != nil {
+			e.log.Info(compatResult.Message,
+				"source_version", compatResult.SourceVersion.Full,
+				"target_version", compatResult.TargetVersion.Full,
+				"compatibility", compatResult.Level.String())
+
+			// Block unsupported downgrades
+			if !compatResult.Compatible {
+				operation.Fail(compatResult.Message)
+				return fmt.Errorf("version compatibility error: %s", compatResult.Message)
+			}
+
+			// Show warnings for risky upgrades
+			if compatResult.Level == CompatibilityLevelRisky || compatResult.Level == CompatibilityLevelWarning {
+				for _, warning := range compatResult.Warnings {
+					e.log.Warn(warning)
+				}
+			}
+		}
+	}
+
 	if e.dryRun {
 		e.log.Info("DRY RUN: Would restore single database", "archive", archivePath, "target", targetDB)
 		return e.previewRestore(archivePath, targetDB, format)
--- a/internal/restore/version_check.go
+++ b/internal/restore/version_check.go
@@ -0,0 +1,231 @@
+package restore
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"regexp"
+	"strconv"
+
+	"dbbackup/internal/database"
+)
+
+// VersionInfo holds PostgreSQL version information
+type VersionInfo struct {
+	Major int
+	Minor int
+	Full  string
+}
+
+// ParsePostgreSQLVersion extracts major and minor version from version string
+// Example: "PostgreSQL 17.7 on x86_64-redhat-linux-gnu..." -> Major: 17, Minor: 7
+func ParsePostgreSQLVersion(versionStr string) (*VersionInfo, error) {
+	// Match patterns like "PostgreSQL 17.7", "PostgreSQL 13.11", "PostgreSQL 10.23"
+	re := regexp.MustCompile(`PostgreSQL\s+(\d+)\.(\d+)`)
+	matches := re.FindStringSubmatch(versionStr)
+	
+	if len(matches) < 3 {
+		return nil, fmt.Errorf("could not parse PostgreSQL version from: %s", versionStr)
+	}
+	
+	major, err := strconv.Atoi(matches[1])
+	if err != nil {
+		return nil, fmt.Errorf("invalid major version: %s", matches[1])
+	}
+	
+	minor, err := strconv.Atoi(matches[2])
+	if err != nil {
+		return nil, fmt.Errorf("invalid minor version: %s", matches[2])
+	}
+	
+	return &VersionInfo{
+		Major: major,
+		Minor: minor,
+		Full:  versionStr,
+	}, nil
+}
+
+// GetDumpFileVersion extracts the PostgreSQL version from a dump file
+// Uses pg_restore -l to read the dump metadata
+func GetDumpFileVersion(dumpPath string) (*VersionInfo, error) {
+	cmd := exec.Command("pg_restore", "-l", dumpPath)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return nil, fmt.Errorf("failed to read dump file metadata: %w (output: %s)", err, string(output))
+	}
+	
+	// Look for "Dumped from database version: X.Y.Z" in output
+	re := regexp.MustCompile(`Dumped from database version:\s+(\d+)\.(\d+)`)
+	matches := re.FindStringSubmatch(string(output))
+	
+	if len(matches) < 3 {
+		// Try alternate format in some dumps
+		re = regexp.MustCompile(`PostgreSQL database dump.*(\d+)\.(\d+)`)
+		matches = re.FindStringSubmatch(string(output))
+	}
+	
+	if len(matches) < 3 {
+		return nil, fmt.Errorf("could not find version information in dump file")
+	}
+	
+	major, _ := strconv.Atoi(matches[1])
+	minor, _ := strconv.Atoi(matches[2])
+	
+	return &VersionInfo{
+		Major: major,
+		Minor: minor,
+		Full:  fmt.Sprintf("PostgreSQL %d.%d", major, minor),
+	}, nil
+}
+
+// CheckVersionCompatibility checks if restoring from source version to target version is safe
+func CheckVersionCompatibility(sourceVer, targetVer *VersionInfo) *VersionCompatibilityResult {
+	result := &VersionCompatibilityResult{
+		Compatible: true,
+		SourceVersion: sourceVer,
+		TargetVersion: targetVer,
+	}
+	
+	// Same major version - always compatible
+	if sourceVer.Major == targetVer.Major {
+		result.Level = CompatibilityLevelSafe
+		result.Message = "Same major version - fully compatible"
+		return result
+	}
+	
+	// Downgrade - not supported
+	if sourceVer.Major > targetVer.Major {
+		result.Compatible = false
+		result.Level = CompatibilityLevelUnsupported
+		result.Message = fmt.Sprintf("Downgrade from PostgreSQL %d to %d is not supported", sourceVer.Major, targetVer.Major)
+		result.Warnings = append(result.Warnings, "Database downgrades require pg_dump from the target version")
+		return result
+	}
+	
+	// Upgrade - check how many major versions
+	versionDiff := targetVer.Major - sourceVer.Major
+	
+	if versionDiff == 1 {
+		// One major version upgrade - generally safe
+		result.Level = CompatibilityLevelSafe
+		result.Message = fmt.Sprintf("Upgrading from PostgreSQL %d to %d - officially supported", sourceVer.Major, targetVer.Major)
+	} else if versionDiff <= 3 {
+		// 2-3 major versions - should work but review release notes
+		result.Level = CompatibilityLevelWarning
+		result.Message = fmt.Sprintf("Upgrading from PostgreSQL %d to %d - supported but review release notes", sourceVer.Major, targetVer.Major)
+		result.Warnings = append(result.Warnings, 
+			fmt.Sprintf("You are jumping %d major versions - some features may have changed", versionDiff))
+		result.Warnings = append(result.Warnings,
+			"Review release notes for deprecated features or behavior changes")
+	} else {
+		// 4+ major versions - high risk
+		result.Level = CompatibilityLevelRisky
+		result.Message = fmt.Sprintf("Upgrading from PostgreSQL %d to %d - large version jump", sourceVer.Major, targetVer.Major)
+		result.Warnings = append(result.Warnings,
+			fmt.Sprintf("WARNING: Jumping %d major versions may encounter compatibility issues", versionDiff))
+		result.Warnings = append(result.Warnings,
+			"Deprecated features from PostgreSQL "+strconv.Itoa(sourceVer.Major)+" may not exist in "+strconv.Itoa(targetVer.Major))
+		result.Warnings = append(result.Warnings,
+			"Extensions may need updates or may be incompatible")
+		result.Warnings = append(result.Warnings,
+			"Test thoroughly in a non-production environment first")
+		result.Recommendations = append(result.Recommendations,
+			"Consider using --schema-only first to validate schema compatibility")
+		result.Recommendations = append(result.Recommendations,
+			"Review PostgreSQL release notes for versions "+strconv.Itoa(sourceVer.Major)+" through "+strconv.Itoa(targetVer.Major))
+	}
+	
+	// Add general upgrade advice
+	if versionDiff > 0 {
+		result.Recommendations = append(result.Recommendations,
+			"Run ANALYZE on all tables after restore for optimal query performance")
+	}
+	
+	return result
+}
+
+// CompatibilityLevel indicates the risk level of version compatibility
+type CompatibilityLevel int
+
+const (
+	CompatibilityLevelSafe CompatibilityLevel = iota
+	CompatibilityLevelWarning
+	CompatibilityLevelRisky
+	CompatibilityLevelUnsupported
+)
+
+func (c CompatibilityLevel) String() string {
+	switch c {
+	case CompatibilityLevelSafe:
+		return "SAFE"
+	case CompatibilityLevelWarning:
+		return "WARNING"
+	case CompatibilityLevelRisky:
+		return "RISKY"
+	case CompatibilityLevelUnsupported:
+		return "UNSUPPORTED"
+	default:
+		return "UNKNOWN"
+	}
+}
+
+// VersionCompatibilityResult contains the result of version compatibility check
+type VersionCompatibilityResult struct {
+	Compatible      bool
+	Level           CompatibilityLevel
+	SourceVersion   *VersionInfo
+	TargetVersion   *VersionInfo
+	Message         string
+	Warnings        []string
+	Recommendations []string
+}
+
+// CheckRestoreVersionCompatibility performs version check for a restore operation
+func (e *Engine) CheckRestoreVersionCompatibility(ctx context.Context, dumpPath string) (*VersionCompatibilityResult, error) {
+	// Get dump file version
+	dumpVer, err := GetDumpFileVersion(dumpPath)
+	if err != nil {
+		// Not critical if we can't read version - continue with warning
+		e.log.Warn("Could not determine dump file version", "error", err)
+		return nil, nil
+	}
+	
+	// Get target database version
+	targetVerStr, err := e.db.GetVersion(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get target database version: %w", err)
+	}
+	
+	targetVer, err := ParsePostgreSQLVersion(targetVerStr)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse target version: %w", err)
+	}
+	
+	// Check compatibility
+	result := CheckVersionCompatibility(dumpVer, targetVer)
+	
+	// Log the results
+	e.log.Info("Version compatibility check",
+		"source", dumpVer.Full,
+		"target", targetVer.Full,
+		"level", result.Level.String())
+	
+	if len(result.Warnings) > 0 {
+		for _, warning := range result.Warnings {
+			e.log.Warn(warning)
+		}
+	}
+	
+	return result, nil
+}
+
+// ValidatePostgreSQLDatabase ensures we're working with a PostgreSQL database
+func ValidatePostgreSQLDatabase(db database.Database) error {
+	// Type assertion to check if it's PostgreSQL
+	switch db.(type) {
+	case *database.PostgreSQL:
+		return nil
+	default:
+		return fmt.Errorf("version compatibility checks only supported for PostgreSQL")
+	}
+}