From 1e54bbc04e5cace722c27fac2492b02ab22df0ee Mon Sep 17 00:00:00 2001 From: Renz Date: Wed, 12 Nov 2025 07:02:40 +0000 Subject: [PATCH] Clean production repository - conservative professional style - Removed all test documentation (MASTER_TEST_PLAN, TESTING_SUMMARY, etc.) - Removed test scripts (create_*_db.sh, test_suite.sh, validation scripts) - Removed test logs and temporary directories - Kept only essential: disaster_recovery_test.sh, build_all.sh - Completely rewrote README.md in conservative professional style - Clean structure: Focus on usage, configuration, troubleshooting - Production-ready documentation for end users --- MASTER_TEST_PLAN.md | 869 -------------------------------- PRODUCTION_TESTING_PLAN.md | 697 -------------------------- README.md | 899 ++++++++++++++++++++++++++-------- RELEASE_v1.2.0.md | 117 ----- TESTING_SUMMARY.md | 367 -------------- create_50gb_db_optimized.sh | 255 ---------- create_large_test_db.sh | 243 --------- create_massive_50gb_db.sh | 165 ------- privilege_diagnostic.sh | 99 ---- privilege_report_testhost.txt | 216 -------- production_validation.sh | 477 ------------------ run_master_tests.sh | 397 --------------- scripts/cli_switch_test.sh | 173 ------- test_suite.sh | 409 ---------------- 14 files changed, 692 insertions(+), 4691 deletions(-) delete mode 100644 MASTER_TEST_PLAN.md delete mode 100644 PRODUCTION_TESTING_PLAN.md delete mode 100644 RELEASE_v1.2.0.md delete mode 100644 TESTING_SUMMARY.md delete mode 100755 create_50gb_db_optimized.sh delete mode 100755 create_large_test_db.sh delete mode 100755 create_massive_50gb_db.sh delete mode 100755 privilege_diagnostic.sh delete mode 100644 privilege_report_testhost.txt delete mode 100755 production_validation.sh delete mode 100755 run_master_tests.sh delete mode 100755 scripts/cli_switch_test.sh delete mode 100755 test_suite.sh diff --git a/MASTER_TEST_PLAN.md b/MASTER_TEST_PLAN.md deleted file mode 100644 index 09c7a54..0000000 --- a/MASTER_TEST_PLAN.md +++ /dev/null @@ -1,869 +0,0 @@ -# Master Test Plan - dbbackup v1.2.0 -## Comprehensive Command-Line and Interactive Testing - ---- - -## Test Environment Setup - -### Prerequisites -```bash -# 1. Ensure PostgreSQL is running -systemctl status postgresql - -# 2. Create test databases with varied characteristics -psql -U postgres <` | `./dbbackup backup single postgres --backup-dir /tmp/test` | Use custom dir | Check file created in /tmp/test | -| `--compression 0` | `./dbbackup backup single postgres --compression 0` | No compression | Check file size vs compressed | -| `--compression 1` | `./dbbackup backup single postgres --compression 1` | Low compression | File size check | -| `--compression 6` | `./dbbackup backup single postgres --compression 6` | Default compression | File size check | -| `--compression 9` | `./dbbackup backup single postgres --compression 9` | Max compression | Smallest file size | -| `--jobs 1` | `./dbbackup backup cluster --jobs 1` | Single threaded | Slower execution | -| `--jobs 8` | `./dbbackup backup cluster --jobs 8` | 8 parallel jobs | Faster execution | -| `--dump-jobs 4` | `./dbbackup backup single postgres --dump-jobs 4` | 4 dump threads | Check pg_dump --jobs | -| `--auto-detect-cores` | `./dbbackup backup cluster --auto-detect-cores` | Auto CPU detection | Default behavior | -| `--max-cores 4` | `./dbbackup backup cluster --max-cores 4` | Limit to 4 cores | Check CPU usage | -| `--cpu-workload cpu-intensive` | `./dbbackup backup cluster --cpu-workload cpu-intensive` | Adjust for CPU work | Performance profile | -| `--cpu-workload io-intensive` | `./dbbackup backup cluster --cpu-workload io-intensive` | Adjust for I/O work | Performance profile | -| `--cpu-workload balanced` | `./dbbackup backup cluster --cpu-workload balanced` | Balanced profile | Default behavior | - -### 1.2 Database Connection Flags - -| Flag | Test Command | Expected Result | Verification | -|------|-------------|-----------------|--------------| -| `-d postgres` / `--db-type postgres` | `./dbbackup status host -d postgres` | Connect to PostgreSQL | Success | -| `-d mysql` | `./dbbackup status host -d mysql` | Connect to MySQL | Success (if MySQL available) | -| `--host localhost` | `./dbbackup status host --host localhost` | Local connection | Success | -| `--host 127.0.0.1` | `./dbbackup status host --host 127.0.0.1` | TCP connection | Success | -| `--port 5432` | `./dbbackup status host --port 5432` | Default port | Success | -| `--port 5433` | `./dbbackup status host --port 5433` | Custom port | Connection error expected | -| `--user postgres` | `./dbbackup status host --user postgres` | Custom user | Success | -| `--user invalid_user` | `./dbbackup status host --user invalid_user` | Invalid user | Auth failure expected | -| `--password ` | `./dbbackup status host --password secretpass` | Password auth | Success | -| `--database postgres` | `./dbbackup status host --database postgres` | Connect to DB | Success | -| `--insecure` | `./dbbackup status host --insecure` | Disable SSL | Success | -| `--ssl-mode disable` | `./dbbackup status host --ssl-mode disable` | SSL disabled | Success | -| `--ssl-mode require` | `./dbbackup status host --ssl-mode require` | SSL required | Success/failure based on server | -| `--ssl-mode verify-ca` | `./dbbackup status host --ssl-mode verify-ca` | Verify CA cert | Success/failure based on certs | -| `--ssl-mode verify-full` | `./dbbackup status host --ssl-mode verify-full` | Full verification | Success/failure based on certs | - -### 1.3 Backup Command Flags - -#### 1.3.1 `backup single` Command -```bash -./dbbackup backup single [database] [flags] -``` - -| Flag | Test Command | Expected Result | Verification | -|------|-------------|-----------------|--------------| -| `` (positional) | `./dbbackup backup single postgres` | Backup postgres DB | File created | -| `--database ` | `./dbbackup backup single --database postgres` | Same as positional | File created | -| `--compression 1` | `./dbbackup backup single postgres --compression 1` | Fast compression | Larger file | -| `--compression 9` | `./dbbackup backup single postgres --compression 9` | Best compression | Smaller file | -| No database specified | `./dbbackup backup single` | Error message | "database name required" | -| Invalid database | `./dbbackup backup single nonexistent_db` | Error message | "database does not exist" | -| Large database | `./dbbackup backup single test_large --compression 1` | Streaming compression | No huge temp files | -| Empty database | `./dbbackup backup single test_empty` | Small backup | File size ~KB | - -#### 1.3.2 `backup cluster` Command -```bash -./dbbackup backup cluster [flags] -``` - -| Flag | Test Command | Expected Result | Verification | -|------|-------------|-----------------|--------------| -| No flags | `./dbbackup backup cluster` | Backup all DBs | cluster_*.tar.gz created | -| `--compression 1` | `./dbbackup backup cluster --compression 1` | Fast cluster backup | Larger archive | -| `--compression 9` | `./dbbackup backup cluster --compression 9` | Best compression | Smaller archive | -| `--jobs 1` | `./dbbackup backup cluster --jobs 1` | Sequential backup | Slower | -| `--jobs 8` | `./dbbackup backup cluster --jobs 8` | Parallel backup | Faster | -| Large DBs present | `./dbbackup backup cluster --compression 3` | Streaming compression | No huge temp files | -| Check globals backup | Extract and verify `globals.sql` | Roles/tablespaces | globals.sql present | -| Check all DBs backed up | Extract and count dumps | All non-template DBs | Verify count | - -#### 1.3.3 `backup sample` Command -```bash -./dbbackup backup sample [database] [flags] -``` - -| Flag | Test Command | Expected Result | Verification | -|------|-------------|-----------------|--------------| -| Default sample | `./dbbackup backup sample test_large` | Sample backup | Smaller than full | -| Custom strategy | Check if sample flags exist | Sample strategy options | TBD based on implementation | - -### 1.4 Restore Command Flags - -#### 1.4.1 `restore single` Command -```bash -./dbbackup restore single [backup-file] [flags] -``` - -| Flag | Test Command | Expected Result | Verification | -|------|-------------|-----------------|--------------| -| `` (positional) | `./dbbackup restore single /path/to/backup.dump` | Restore to original DB | Success | -| `--target-db ` | `./dbbackup restore single backup.dump --target-db restored_db` | Restore to new DB | New DB created | -| `--create` | `./dbbackup restore single backup.dump --target-db newdb --create` | Create DB if missing | DB created + restored | -| `--no-owner` | `./dbbackup restore single backup.dump --no-owner` | Skip ownership | No SET OWNER commands | -| `--clean` | `./dbbackup restore single backup.dump --clean` | Drop existing objects | Clean restore | -| `--jobs 4` | `./dbbackup restore single backup.dump --jobs 4` | Parallel restore | Faster | -| Missing backup file | `./dbbackup restore single nonexistent.dump` | Error message | "file not found" | -| Invalid backup file | `./dbbackup restore single /etc/hosts` | Error message | "invalid backup file" | -| Without --create, DB missing | `./dbbackup restore single backup.dump --target-db missing_db` | Error message | "database does not exist" | -| With --create, DB missing | `./dbbackup restore single backup.dump --target-db missing_db --create` | Success | DB created | - -#### 1.4.2 `restore cluster` Command -```bash -./dbbackup restore cluster [backup-file] [flags] -``` - -| Flag | Test Command | Expected Result | Verification | -|------|-------------|-----------------|--------------| -| `` (positional) | `./dbbackup restore cluster cluster_*.tar.gz` | Restore all DBs | All DBs restored | -| `--create` | `./dbbackup restore cluster backup.tar.gz --create` | Create missing DBs | DBs created | -| `--globals-only` | `./dbbackup restore cluster backup.tar.gz --globals-only` | Restore roles only | Only globals restored | -| `--jobs 4` | `./dbbackup restore cluster backup.tar.gz --jobs 4` | Parallel restore | Faster | -| Missing archive | `./dbbackup restore cluster nonexistent.tar.gz` | Error message | "file not found" | -| Invalid archive | `./dbbackup restore cluster /etc/hosts` | Error message | "invalid archive" | -| Corrupted archive | Create corrupted .tar.gz | Error message | "extraction failed" | -| Ownership preservation | Restore and check owners | Correct ownership | GRANT/REVOKE present | - -### 1.5 Status Command Flags - -#### 1.5.1 `status host` Command -```bash -./dbbackup status host [flags] -``` - -| Flag | Test Command | Expected Result | Verification | -|------|-------------|-----------------|--------------| -| No flags | `./dbbackup status host` | Show host status | Version, size, DBs listed | -| `--database ` | `./dbbackup status host --database postgres` | Connect to specific DB | Success | -| Invalid connection | `./dbbackup status host --port 9999` | Error message | Connection failure | -| Show version | Check output | PostgreSQL version | Version string present | -| List databases | Check output | Database list | All DBs shown | -| Show sizes | Check output | Database sizes | Sizes in human format | - -#### 1.5.2 `status cpu` Command -```bash -./dbbackup status cpu [flags] -``` - -| Flag | Test Command | Expected Result | Verification | -|------|-------------|-----------------|--------------| -| No flags | `./dbbackup status cpu` | Show CPU info | Cores, workload shown | -| CPU detection | Check output | Auto-detected cores | Matches system | -| Workload info | Check output | Current workload | Workload type shown | - ---- - -## PART 2: Interactive TUI Testing - -### 2.1 TUI Launch and Navigation - -| Test Case | Steps | Expected Result | Verification Method | -|-----------|-------|-----------------|---------------------| -| Launch TUI | `./dbbackup` | Main menu appears | Visual check | -| Navigate with arrows | Press ↑/↓ | Selection moves | Visual check | -| Navigate with j/k | Press j/k (vim keys) | Selection moves | Visual check | -| Navigate with numbers | Press 1-4 | Jump to option | Visual check | -| Press ESC | ESC key | Exit confirmation | Visual check | -| Press q | q key | Exit confirmation | Visual check | -| Press Ctrl+C | Ctrl+C | Immediate exit | Visual check | - -### 2.2 Main Menu Options - -``` -Main Menu: -1. Single Database Backup -2. Cluster Backup (All Databases) -3. Restore Database -4. System Status -5. Settings -6. Exit -``` - -#### Test: Select Each Menu Option - -| Menu Item | Key Press | Expected Result | Verification | -|-----------|-----------|-----------------|--------------| -| Single Database Backup | Enter on option 1 | Database selection screen | Visual check | -| Cluster Backup | Enter on option 2 | Cluster backup screen | Visual check | -| Restore Database | Enter on option 3 | Restore file selection | Visual check | -| System Status | Enter on option 4 | Status screen | Visual check | -| Settings | Enter on option 5 | Settings menu | Visual check | -| Exit | Enter on option 6 | Exit application | Application closes | - -### 2.3 Single Database Backup Flow - -**Entry**: Main Menu → Single Database Backup - -| Step | Action | Expected Screen | Verification | -|------|--------|-----------------|--------------| -| 1. Database list appears | - | List of databases shown | Check postgres, template0, template1 excluded or marked | -| 2. Navigate databases | ↑/↓ arrows | Selection moves | Visual check | -| 3. Search databases | Type filter text | List filters | Only matching DBs shown | -| 4. Select database | Press Enter | Backup options screen | Options screen appears | -| 5. Compression level | Select 1-9 | Level selected | Selected level highlighted | -| 6. Backup directory | Enter or use default | Directory shown | Path displayed | -| 7. Start backup | Confirm | Progress indicator | Spinner/progress bar | -| 8. Backup completes | Wait | Success message | File path shown | -| 9. Return to menu | Press key | Back to main menu | Main menu shown | - -**Error Scenarios**: -- No database selected → Error message -- Invalid backup directory → Error message -- Insufficient permissions → Error message -- Disk full → Error message with space info - -### 2.4 Cluster Backup Flow - -**Entry**: Main Menu → Cluster Backup - -| Step | Action | Expected Screen | Verification | -|------|--------|-----------------|--------------| -| 1. Cluster options | - | Options screen | Compression, directory shown | -| 2. Set compression | Select level | Level selected | 1-9 | -| 3. Set directory | Enter path | Directory set | Path shown | -| 4. Start backup | Confirm | Progress screen | Per-DB progress | -| 5. Monitor progress | Watch | Database names + progress | Real-time updates | -| 6. Backup completes | Wait | Summary screen | Success/failed counts | -| 7. Review results | - | Archive info shown | Size, location, duration | -| 8. Return to menu | Press key | Main menu | Menu shown | - -**Error Scenarios**: -- Connection lost mid-backup → Error message, partial cleanup -- Disk full during backup → Error message, cleanup temp files -- Individual DB failure → Continue with others, show warning - -### 2.5 Restore Database Flow - -**Entry**: Main Menu → Restore Database - -| Step | Action | Expected Screen | Verification | -|------|--------|-----------------|--------------| -| 1. Restore type selection | - | Single or Cluster | Two options | -| 2. Select restore type | Enter on option | File browser | Backup files listed | -| 3. Browse backup files | Navigate | File list | .dump, .tar.gz files shown | -| 4. Filter files | Type filter | List filters | Matching files shown | -| 5. Select backup file | Enter | Restore options | Options screen | -| 6. Set target database | Enter name | DB name set | Name shown | -| 7. Set options | Toggle flags | Options selected | Checkboxes/toggles | -| 8. Confirm restore | Press Enter | Warning prompt | "This will modify database" | -| 9. Start restore | Confirm | Progress indicator | Spinner/progress bar | -| 10. Restore completes | Wait | Success message | Duration, objects restored | -| 11. Return to menu | Press key | Main menu | Menu shown | - -**Critical Test**: Cluster Restore Selection -- **KNOWN BUG**: Enter key may not work on cluster backup selection -- Test: Navigate to cluster backup, press Enter -- Expected: File selected and restore proceeds -- Verify: Fix if Enter key doesn't register - -**Options to Test**: -- `--create`: Create database if missing -- `--no-owner`: Skip ownership restoration -- `--clean`: Drop existing objects first -- Target database name field - -### 2.6 System Status Flow - -**Entry**: Main Menu → System Status - -| Step | Action | Expected Screen | Verification | -|------|--------|-----------------|--------------| -| 1. Status options | - | Host or CPU status | Two tabs/options | -| 2. Host status | Select | Connection info + DBs | Database list with sizes | -| 3. Navigate DB list | ↑/↓ arrows | Selection moves | Visual check | -| 4. View DB details | Enter on DB | DB details | Tables, size, owner | -| 5. CPU status | Select | CPU info | Cores, workload type | -| 6. Return | ESC/Back | Main menu | Menu shown | - -### 2.7 Settings Flow - -**Entry**: Main Menu → Settings - -| Step | Action | Expected Screen | Verification | -|------|--------|-----------------|--------------| -| 1. Settings menu | - | Options list | All settings shown | -| 2. Connection settings | Select | Host, port, user, etc. | Form fields | -| 3. Edit connection | Change values | Values update | Changes persist | -| 4. Test connection | Press test button | Connection result | Success/failure message | -| 5. Backup settings | Select | Compression, jobs, etc. | Options shown | -| 6. Change compression | Set level | Level updated | Value changes | -| 7. Change jobs | Set count | Count updated | Value changes | -| 8. Save settings | Confirm | Settings saved | "Saved" message | -| 9. Cancel changes | Cancel | Settings reverted | Original values | -| 10. Return | Back | Main menu | Menu shown | - -**Settings to Test**: -- Database host (localhost, IP address) -- Database port (5432, custom) -- Database user (postgres, custom) -- Database password (set, change, clear) -- Database type (postgres, mysql) -- SSL mode (disable, require, verify-ca, verify-full) -- Backup directory (default, custom) -- Compression level (0-9) -- Parallel jobs (1-16) -- Dump jobs (1-16) -- Auto-detect cores (on/off) -- CPU workload type (balanced, cpu-intensive, io-intensive) - -### 2.8 TUI Error Handling - -| Error Scenario | Trigger | Expected Behavior | Verification | -|----------------|---------|-------------------|--------------| -| Database connection failure | Wrong credentials | Error modal with details | Clear error message | -| Backup file not found | Delete file mid-operation | Error message | Graceful handling | -| Invalid backup file | Select non-backup file | Error message | "Invalid backup format" | -| Insufficient permissions | Backup to read-only dir | Error message | Permission denied | -| Disk full | Fill disk during backup | Error message + cleanup | Temp files removed | -| Network interruption | Disconnect during remote backup | Error message | Connection lost message | -| Keyboard interrupt | Press Ctrl+C | Confirmation prompt | "Cancel operation?" | -| Window resize | Resize terminal | UI adapts | No crashes, redraws correctly | -| Invalid input | Enter invalid characters | Input rejected or sanitized | No crashes | -| Concurrent operations | Try two backups at once | Error or queue | "Operation in progress" | - -### 2.9 TUI Visual/UX Tests - -| Test | Action | Expected Result | Pass/Fail | -|------|--------|-----------------|-----------| -| Color theme | Launch TUI | Colors render correctly | Visual check | -| No-color mode | `--no-color` flag | Plain text only | No ANSI codes | -| Progress indicators | Start backup | Spinner/progress bar animates | Visual check | -| Help text | Press ? or h | Help overlay | Help displayed | -| Modal dialogs | Trigger error | Modal appears centered | Visual check | -| Modal close | ESC or Enter | Modal closes | Returns to previous screen | -| Text wrapping | Long database names | Text wraps or truncates | Readable | -| Scrolling | Long lists | List scrolls | Arrow keys work | -| Keyboard shortcuts | Press shortcuts | Actions trigger | Quick actions work | -| Mouse support | Click options (if supported) | Selection changes | Visual check | - ---- - -## PART 3: Integration Testing - -### 3.1 End-to-End Workflows - -#### Workflow 1: Complete Backup and Restore Cycle -```bash -# 1. Create test database -psql -U postgres -c "CREATE DATABASE e2e_test_db;" -psql -U postgres e2e_test_db </dev/null || stat -f%z "$backup_file") - size_mb=$((size / 1024 / 1024)) - echo "Backup size: ${size_mb}MB" - - if [ $size_mb -lt 2000 ]; then - echo "✅ PASSED: Streaming compression worked" - else - echo "❌ FAILED: File too large, streaming compression may have failed" - fi - - # 5. Cleanup - rm -rf /tmp/large_test -else - echo "⊘ SKIPPED: testdb_50gb not available" -fi -``` - -### 3.2 Permission and Authentication Tests - -| Test | Setup | Command | Expected Result | -|------|-------|---------|-----------------| -| Peer authentication | Run as postgres user | `sudo -u postgres ./dbbackup status host` | Success | -| Password authentication | Set PGPASSWORD | `PGPASSWORD=xxx ./dbbackup status host --password xxx` | Success | -| .pgpass authentication | Create ~/.pgpass | `./dbbackup status host` | Success | -| Failed authentication | Wrong password | `./dbbackup status host --password wrong` | Auth failure | -| Insufficient privileges | Non-superuser restore | `./dbbackup restore cluster ...` | Error or warning | -| SSL connection | SSL enabled server | `./dbbackup status host --ssl-mode require` | Success | -| SSL required but unavailable | SSL disabled server | `./dbbackup status host --ssl-mode require` | Connection failure | - -### 3.3 Error Recovery Tests - -| Scenario | Trigger | Expected Behavior | Verification | -|----------|---------|-------------------|--------------| -| Interrupted backup | Kill process mid-backup | Temp files cleaned up | No leftover .cluster_* dirs | -| Interrupted restore | Kill process mid-restore | Partial objects, clear error | Database in consistent state | -| Out of disk space | Fill disk during backup | Error message, cleanup | Temp files removed | -| Out of memory | Very large database | Streaming compression used | No OOM kills | -| Database locked | Backup during heavy load | Backup waits or times out | Clear timeout message | -| Corrupted backup file | Manually corrupt file | Error during restore | "Invalid backup file" | -| Missing dependencies | Remove pg_dump | Error at startup | "Required tool not found" | -| Network timeout | Slow/interrupted connection | Timeout with retry option | Clear error message | - ---- - -## PART 4: Performance and Stress Testing - -### 4.1 Performance Benchmarks - -#### Test: Compression Speed vs Size Trade-off -```bash -# Backup same database with different compression levels, measure time and size -for level in 1 3 6 9; do - echo "Testing compression level $level" - start=$(date +%s) - ./dbbackup backup single postgres --compression $level --backup-dir /tmp/perf_test - end=$(date +%s) - duration=$((end - start)) - backup_file=$(ls -t /tmp/perf_test/db_postgres_*.dump | head -1) - size=$(stat -c%s "$backup_file" 2>/dev/null || stat -f%z "$backup_file") - size_mb=$((size / 1024 / 1024)) - echo "Level $level: ${duration}s, ${size_mb}MB" - rm "$backup_file" -done -``` - -Expected results: -- Level 1: Fastest, largest file -- Level 9: Slowest, smallest file -- Level 6: Good balance - -#### Test: Parallel vs Sequential Performance -```bash -# Cluster backup with different --jobs settings -for jobs in 1 4 8; do - echo "Testing with $jobs parallel jobs" - start=$(date +%s) - ./dbbackup backup cluster --jobs $jobs --compression 3 --backup-dir /tmp/parallel_test - end=$(date +%s) - duration=$((end - start)) - echo "$jobs jobs: ${duration}s" - rm /tmp/parallel_test/cluster_*.tar.gz -done -``` - -Expected results: -- 1 job: Slowest -- 8 jobs: Fastest (up to CPU core count) - -### 4.2 Stress Tests - -#### Test: Multiple Concurrent Operations -```bash -# Try to trigger race conditions -./dbbackup backup single postgres --backup-dir /tmp/stress1 & -./dbbackup backup single postgres --backup-dir /tmp/stress2 & -./dbbackup status host & -wait -# Verify: All operations should complete successfully without conflicts -``` - -#### Test: Very Large Database (if available) -- Use testdb_50gb or larger -- Verify streaming compression activates -- Monitor memory usage (should stay reasonable) -- Verify no disk space exhaustion - -#### Test: Many Small Databases -```bash -# Create 50 small databases -for i in {1..50}; do - psql -U postgres -c "CREATE DATABASE stress_test_$i;" -done - -# Backup cluster -./dbbackup backup cluster --backup-dir /tmp/stress_cluster - -# Verify: All 50+ databases backed up, archive created successfully - -# Cleanup -for i in {1..50}; do - psql -U postgres -c "DROP DATABASE stress_test_$i;" -done -``` - ---- - -## PART 5: Regression Testing - -### 5.1 Known Issue Verification - -| Issue | Test | Expected Behavior | Status | -|-------|------|-------------------|--------| -| TUI Enter key on cluster restore | Launch TUI, select cluster backup, press Enter | Backup selected, restore proceeds | ⚠️ Known issue - retest | -| Debug logging not working | Run with `--debug` | DEBUG lines in output | ⚠️ Known issue - retest | -| Streaming compression for large DBs | Backup testdb_50gb | No huge temp files | ✅ Fixed in v1.2.0 | -| Disk space exhaustion | Backup large DBs | Streaming compression prevents disk fill | ✅ Fixed in v1.2.0 | - -### 5.2 Previous Bug Verification - -Test all previously fixed bugs to ensure no regressions: - -1. **Ownership preservation** (fixed earlier) - - Backup database with custom owners - - Restore to new cluster - - Verify ownership preserved - -2. **restore --create flag** (fixed earlier) - - Restore to non-existent database with --create - - Verify database created and populated - -3. **Streaming compression** (fixed v1.2.0) - - Backup large database - - Verify no huge temp files - - Verify compressed output - ---- - -## PART 6: Cross-Platform Testing (if applicable) - -Test on each supported platform: -- ✅ Linux (amd64) - Primary platform -- ⏹ Linux (arm64) -- ⏹ Linux (armv7) -- ⏹ macOS (Intel) -- ⏹ macOS (Apple Silicon) -- ⏹ FreeBSD -- ⏹ Windows (if PostgreSQL tools available) - -For each platform: -1. Binary executes -2. Help/version commands work -3. Basic backup works -4. Basic restore works -5. TUI launches (if terminal supports it) - ---- - -## PART 7: Automated Test Script - -### Master Test Execution Script -```bash -#!/bin/bash -# Save as: run_master_tests.sh - -source ./master_test_functions.sh # Helper functions - -echo "================================================" -echo "dbbackup Master Test Suite" -echo "================================================" -echo "" - -# Initialize -init_test_environment - -# PART 1: CLI Flags -echo "=== PART 1: Command-Line Flag Testing ===" -test_global_flags -test_connection_flags -test_backup_flags -test_restore_flags -test_status_flags - -# PART 2: Interactive (manual) -echo "=== PART 2: Interactive TUI Testing ===" -echo "⚠️ This section requires manual testing" -echo "Launch: ./dbbackup" -echo "Follow test cases in MASTER_TEST_PLAN.md section 2" -echo "" -read -p "Press Enter after completing TUI tests..." - -# PART 3: Integration -echo "=== PART 3: Integration Testing ===" -test_e2e_workflow -test_cluster_workflow -test_large_database_workflow - -# PART 4: Performance -echo "=== PART 4: Performance Testing ===" -test_compression_performance -test_parallel_performance - -# PART 5: Regression -echo "=== PART 5: Regression Testing ===" -test_known_issues -test_previous_bugs - -# Summary -print_test_summary -``` - ---- - -## Test Execution Checklist - -### Pre-Testing -- [ ] Build all binaries: `./build_all.sh` -- [ ] Verify PostgreSQL running: `systemctl status postgresql` -- [ ] Create test databases -- [ ] Ensure adequate disk space (20GB+) -- [ ] Install pigz: `yum install pigz` or `apt-get install pigz` -- [ ] Set up test user if needed - -### Command-Line Testing (Automated) -- [ ] Run: `./run_master_tests.sh` -- [ ] Review automated test output -- [ ] Check all tests passed -- [ ] Review log file for errors - -### Interactive TUI Testing (Manual) -- [ ] Launch TUI: `./dbbackup` -- [ ] Test all main menu options -- [ ] Test all navigation methods (arrows, vim keys, numbers) -- [ ] Test single database backup flow -- [ ] Test cluster backup flow -- [ ] **[CRITICAL]** Test restore cluster selection (Enter key bug) -- [ ] Test restore single flow -- [ ] Test status displays -- [ ] Test settings menu -- [ ] Test all error scenarios -- [ ] Test ESC/quit functionality -- [ ] Test help displays - -### Integration Testing -- [ ] Run E2E workflow script -- [ ] Run cluster workflow script -- [ ] Run large database workflow (if available) -- [ ] Verify data integrity after restores - -### Performance Testing -- [ ] Run compression benchmarks -- [ ] Run parallel job benchmarks -- [ ] Monitor resource usage (htop/top) - -### Stress Testing -- [ ] Run concurrent operations test -- [ ] Run many-database test -- [ ] Monitor for crashes or hangs - -### Regression Testing -- [ ] Verify all known issues -- [ ] Test all previously fixed bugs -- [ ] Check for new regressions - -### Post-Testing -- [ ] Review all test results -- [ ] Document any failures -- [ ] Create GitHub issues for new bugs -- [ ] Update test plan with new test cases -- [ ] Clean up test databases and files - ---- - -## Success Criteria - -### Minimum Requirements for Production Release -- ✅ All critical CLI commands work (backup/restore/status) -- ✅ No data loss in backup/restore cycle -- ✅ Streaming compression works for large databases -- ✅ No disk space exhaustion -- ✅ TUI launches and main menu navigates -- ✅ Error messages are clear and helpful - -### Nice-to-Have (Can be fixed in minor releases) -- ⚠️ TUI Enter key on cluster restore -- ⚠️ Debug logging functionality -- ⚠️ All TUI error scenarios handled gracefully -- ⚠️ All performance optimizations tested - -### Test Coverage Goals -- [ ] 100% of CLI flags tested -- [ ] 90%+ of TUI flows tested (manual) -- [ ] 100% of critical workflows tested -- [ ] 80%+ success rate on all tests - ---- - -## Test Result Documentation - -### Test Execution Log Template -``` -Test Execution: MASTER_TEST_PLAN v1.0 -Date: YYYY-MM-DD -Tester: -Version: dbbackup v1.2.0 -Environment: - -PART 1: CLI Flags -- Global Flags: X/Y passed -- Connection Flags: X/Y passed -- Backup Flags: X/Y passed -- Restore Flags: X/Y passed -- Status Flags: X/Y passed - -PART 2: TUI Testing -- Navigation: PASS/FAIL -- Main Menu: PASS/FAIL -- Backup Flows: PASS/FAIL -- Restore Flows: PASS/FAIL -- Status: PASS/FAIL -- Settings: PASS/FAIL -- Error Handling: PASS/FAIL -- Known Issues: - -PART 3: Integration -- E2E Workflow: PASS/FAIL -- Cluster Workflow: PASS/FAIL -- Large DB Workflow: PASS/FAIL - -PART 4: Performance -- Compression: PASS/FAIL -- Parallel: PASS/FAIL - -PART 5: Regression -- Known Issues: X/Y verified -- Previous Bugs: X/Y verified - -SUMMARY -- Total Tests: X -- Passed: Y -- Failed: Z -- Success Rate: N% -- Production Ready: YES/NO - -FAILED TESTS: -1. -2. - -NOTES: - -``` - ---- - -## Appendix: Quick Reference - -### Essential Test Commands -```bash -# Quick smoke test -./dbbackup --version -./dbbackup backup single postgres --insecure -./dbbackup status host --insecure - -# Full validation -./production_validation.sh - -# Interactive testing -./dbbackup - -# Check for leftover processes -ps aux | grep -E 'pg_dump|pigz|dbbackup' - -# Check for temp files -find /var/lib/pgsql/db_backups -name ".cluster_*" -find /tmp -name "dbbackup_*" - -# Monitor resources -htop -df -h -``` - -### Useful Debugging Commands -```bash -# Enable debug logging (if working) -./dbbackup --debug backup single postgres --insecure - -# Verbose PostgreSQL -PGOPTIONS='-c log_statement=all' ./dbbackup status host --insecure - -# Trace system calls -strace -o trace.log ./dbbackup backup single postgres --insecure - -# Check backup file integrity -pg_restore --list backup.dump | head -20 -tar -tzf cluster_backup.tar.gz | head -20 -``` - ---- - -**END OF MASTER TEST PLAN** - -**Estimated Testing Time**: 4-6 hours (2 hours CLI, 2 hours TUI, 1-2 hours integration/performance) - -**Minimum Testing Time**: 2 hours (critical paths only) - -**Recommended**: Full execution before each major release diff --git a/PRODUCTION_TESTING_PLAN.md b/PRODUCTION_TESTING_PLAN.md deleted file mode 100644 index 99de75b..0000000 --- a/PRODUCTION_TESTING_PLAN.md +++ /dev/null @@ -1,697 +0,0 @@ -# Production-Ready Testing Plan - -**Date**: November 11, 2025 -**Version**: 1.0 -**Goal**: Verify complete functionality for production deployment - ---- - -## Test Environment Status - -- ✅ 7.5GB test database created (`testdb_50gb`) -- ✅ Multiple test databases (17 total) -- ✅ Test roles and ownership configured (`testowner`) -- ✅ 107GB available disk space -- ✅ PostgreSQL cluster operational - ---- - -## Phase 1: Command-Line Testing (Critical Path) - -### 1.1 Cluster Backup - Full Test -**Priority**: CRITICAL -**Status**: ⚠️ NEEDS COMPLETION - -**Test Steps:** -```bash -# Clean environment -sudo rm -rf /var/lib/pgsql/db_backups/.cluster_* - -# Execute cluster backup with compression level 6 (production default) -time sudo -u postgres ./dbbackup backup cluster - -# Verify output -ls -lh /var/lib/pgsql/db_backups/cluster_*.tar.gz | tail -1 -cat /var/lib/pgsql/db_backups/cluster_*.tar.gz.info -``` - -**Success Criteria:** -- [ ] All databases backed up successfully (0 failures) -- [ ] Archive created (>500MB expected) -- [ ] Completion time <15 minutes -- [ ] No memory errors in dmesg -- [ ] Metadata file created - ---- - -### 1.2 Cluster Restore - Full Test with Ownership Verification -**Priority**: CRITICAL -**Status**: ⚠️ NOT TESTED - -**Pre-Test: Document Current Ownership** -```bash -# Check current ownership across key databases -sudo -u postgres psql -c "\l+" | grep -E "ownership_test|testdb" - -# Check table ownership in ownership_test -sudo -u postgres psql -d ownership_test -c \ - "SELECT schemaname, tablename, tableowner FROM pg_tables WHERE schemaname = 'public';" - -# Check roles -sudo -u postgres psql -c "\du" -``` - -**Test Steps:** -```bash -# Get latest cluster backup -BACKUP=$(ls -t /var/lib/pgsql/db_backups/cluster_*.tar.gz | head -1) - -# Dry run first -sudo -u postgres ./dbbackup restore cluster "$BACKUP" --dry-run - -# Execute restore with confirmation -time sudo -u postgres ./dbbackup restore cluster "$BACKUP" --confirm - -# Verify restoration -sudo -u postgres psql -c "\l+" | wc -l -``` - -**Post-Test: Verify Ownership Preserved** -```bash -# Check database ownership restored -sudo -u postgres psql -c "\l+" | grep -E "ownership_test|testdb" - -# Check table ownership preserved -sudo -u postgres psql -d ownership_test -c \ - "SELECT schemaname, tablename, tableowner FROM pg_tables WHERE schemaname = 'public';" - -# Verify testowner role exists -sudo -u postgres psql -c "\du" | grep testowner - -# Check access privileges -sudo -u postgres psql -l | grep -E "Access privileges" -``` - -**Success Criteria:** -- [ ] All databases restored successfully -- [ ] Database ownership matches original -- [ ] Table ownership preserved (testowner still owns test_data) -- [ ] Roles restored from globals.sql -- [ ] No permission errors -- [ ] Data integrity: row counts match -- [ ] Completion time <30 minutes - ---- - -### 1.3 Large Database Operations -**Priority**: HIGH -**Status**: ✅ COMPLETED (7.5GB single DB) - -**Additional Test Needed:** -```bash -# Test single database restore with ownership -BACKUP=/var/lib/pgsql/db_backups/db_testdb_50gb_*.dump - -# Drop and recreate to test full cycle -sudo -u postgres psql -c "DROP DATABASE IF EXISTS testdb_50gb_restored;" - -# Restore -time sudo -u postgres ./dbbackup restore single "$BACKUP" \ - --target testdb_50gb_restored --create --confirm - -# Verify size and data -sudo -u postgres psql -d testdb_50gb_restored -c \ - "SELECT pg_size_pretty(pg_database_size('testdb_50gb_restored'));" -``` - -**Success Criteria:** -- [ ] Restore completes successfully -- [ ] Database size matches original (~7.5GB) -- [ ] Row counts match (7M+ rows) -- [ ] Completion time <25 minutes - ---- - -### 1.4 Authentication Methods Testing -**Priority**: HIGH -**Status**: ⚠️ NEEDS VERIFICATION - -**Test Cases:** -```bash -# Test 1: Peer authentication (current working method) -sudo -u postgres ./dbbackup status - -# Test 2: Password authentication (if configured) -./dbbackup status --user postgres --password "$PGPASSWORD" - -# Test 3: ~/.pgpass file (if exists) -cat ~/.pgpass -./dbbackup status --user postgres - -# Test 4: Environment variable -export PGPASSWORD="test_password" -./dbbackup status --user postgres -unset PGPASSWORD -``` - -**Success Criteria:** -- [ ] At least one auth method works -- [ ] Error messages are clear and helpful -- [ ] Authentication detection working - ---- - -### 1.5 Privilege Diagnostic Tool -**Priority**: MEDIUM -**Status**: ✅ CREATED, ⚠️ NEEDS EXECUTION - -**Test Steps:** -```bash -# Run diagnostic on current system -./privilege_diagnostic.sh > privilege_report_production.txt - -# Review output -cat privilege_report_production.txt - -# Compare with expectations -grep -A 10 "DATABASE PRIVILEGES" privilege_report_production.txt -``` - -**Success Criteria:** -- [ ] Script runs without errors -- [ ] Shows all database privileges -- [ ] Identifies roles correctly -- [ ] globals.sql content verified - ---- - -## Phase 2: Interactive Mode Testing (TUI) - -### 2.1 TUI Launch and Navigation -**Priority**: HIGH -**Status**: ⚠️ NOT FULLY TESTED - -**Test Steps:** -```bash -# Launch TUI -sudo -u postgres ./dbbackup interactive - -# Test navigation: -# - Arrow keys: ↑ ↓ to move through menu -# - Enter: Select option -# - Esc/q: Go back/quit -# - Test all 10 main menu options -``` - -**Menu Items to Test:** -1. [ ] Single Database Backup -2. [ ] Sample Database Backup -3. [ ] Full Cluster Backup -4. [ ] Restore Single Database -5. [ ] Restore Cluster Backup -6. [ ] List Backups -7. [ ] View Operation History -8. [ ] Database Status -9. [ ] Settings -10. [ ] Exit - -**Success Criteria:** -- [ ] TUI launches without errors -- [ ] Navigation works smoothly -- [ ] No terminal artifacts -- [ ] Can navigate back with Esc -- [ ] Exit works cleanly - ---- - -### 2.2 TUI Cluster Backup -**Priority**: CRITICAL -**Status**: ⚠️ ISSUE REPORTED (Enter key not working) - -**Test Steps:** -```bash -# Launch TUI -sudo -u postgres ./dbbackup interactive - -# Navigate to: Full Cluster Backup (option 3) -# Press Enter to start -# Observe progress indicators -# Wait for completion -``` - -**Known Issue:** -- User reported: "on cluster backup restore selection - i cant press enter to select the cluster backup - interactiv" - -**Success Criteria:** -- [ ] Enter key works to select cluster backup -- [ ] Progress indicators show during backup -- [ ] Backup completes successfully -- [ ] Returns to main menu on completion -- [ ] Backup file listed in backup directory - ---- - -### 2.3 TUI Cluster Restore -**Priority**: CRITICAL -**Status**: ⚠️ NEEDS TESTING - -**Test Steps:** -```bash -# Launch TUI -sudo -u postgres ./dbbackup interactive - -# Navigate to: Restore Cluster Backup (option 5) -# Browse available cluster backups -# Select latest backup -# Press Enter to start restore -# Observe progress indicators -# Wait for completion -``` - -**Success Criteria:** -- [ ] Can browse cluster backups -- [ ] Enter key works to select backup -- [ ] Progress indicators show during restore -- [ ] Restore completes successfully -- [ ] Ownership preserved -- [ ] Returns to main menu on completion - ---- - -### 2.4 TUI Database Selection -**Priority**: HIGH -**Status**: ⚠️ NEEDS TESTING - -**Test Steps:** -```bash -# Test single database backup selection -sudo -u postgres ./dbbackup interactive -# Navigate to: Single Database Backup (option 1) -# Browse database list -# Select testdb_50gb -# Press Enter to start -# Observe progress -``` - -**Success Criteria:** -- [ ] Database list displays correctly -- [ ] Can scroll through databases -- [ ] Selection works with Enter -- [ ] Progress shows during backup -- [ ] Backup completes successfully - ---- - -## Phase 3: Edge Cases and Error Handling - -### 3.1 Disk Space Exhaustion -**Priority**: MEDIUM -**Status**: ⚠️ NEEDS TESTING - -**Test Steps:** -```bash -# Check current space -df -h / - -# Test with limited space (if safe) -# Create large file to fill disk to 90% -# Attempt backup -# Verify error handling -``` - -**Success Criteria:** -- [ ] Clear error message about disk space -- [ ] Graceful failure (no corruption) -- [ ] Cleanup of partial files - ---- - -### 3.2 Interrupted Operations -**Priority**: MEDIUM -**Status**: ⚠️ NEEDS TESTING - -**Test Steps:** -```bash -# Start backup -sudo -u postgres ./dbbackup backup cluster & -PID=$! - -# Wait 30 seconds -sleep 30 - -# Interrupt with Ctrl+C or kill -kill -INT $PID - -# Check for cleanup -ls -la /var/lib/pgsql/db_backups/.cluster_* -``` - -**Success Criteria:** -- [ ] Graceful shutdown on SIGINT -- [ ] Temp directories cleaned up -- [ ] No corrupted files left -- [ ] Clear error message - ---- - -### 3.3 Invalid Archive Files -**Priority**: LOW -**Status**: ⚠️ NEEDS TESTING - -**Test Steps:** -```bash -# Test with non-existent file -sudo -u postgres ./dbbackup restore single /tmp/nonexistent.dump - -# Test with corrupted archive -echo "corrupted" > /tmp/bad.dump -sudo -u postgres ./dbbackup restore single /tmp/bad.dump - -# Test with wrong format -sudo -u postgres ./dbbackup restore cluster /tmp/single_db.dump -``` - -**Success Criteria:** -- [ ] Clear error messages -- [ ] No crashes -- [ ] Proper format detection - ---- - -## Phase 4: Performance and Scalability - -### 4.1 Memory Usage Monitoring -**Priority**: HIGH -**Status**: ⚠️ NEEDS MONITORING - -**Test Steps:** -```bash -# Monitor during large backup -( - while true; do - ps aux | grep dbbackup | grep -v grep - free -h - sleep 10 - done -) > memory_usage.log & -MONITOR_PID=$! - -# Run backup -sudo -u postgres ./dbbackup backup cluster - -# Stop monitoring -kill $MONITOR_PID - -# Review memory usage -grep -A 1 "dbbackup" memory_usage.log | grep -v grep -``` - -**Success Criteria:** -- [ ] Memory usage stays under 1.5GB -- [ ] No OOM errors -- [ ] Memory released after completion - ---- - -### 4.2 Compression Performance -**Priority**: MEDIUM -**Status**: ⚠️ NEEDS TESTING - -**Test Different Compression Levels:** -```bash -# Test compression levels 1, 3, 6, 9 -for LEVEL in 1 3 6 9; do - echo "Testing compression level $LEVEL" - time sudo -u postgres ./dbbackup backup single testdb_50gb \ - --compression=$LEVEL -done - -# Compare sizes and times -ls -lh /var/lib/pgsql/db_backups/db_testdb_50gb_*.dump -``` - -**Success Criteria:** -- [ ] All compression levels work -- [ ] Higher compression = smaller file -- [ ] Higher compression = longer time -- [ ] Level 6 is good balance - ---- - -## Phase 5: Documentation Verification - -### 5.1 README Examples -**Priority**: HIGH -**Status**: ⚠️ NEEDS VERIFICATION - -**Test All README Examples:** -```bash -# Example 1: Single database backup -dbbackup backup single myapp_db - -# Example 2: Sample backup -dbbackup backup sample myapp_db --sample-ratio 10 - -# Example 3: Full cluster backup -dbbackup backup cluster - -# Example 4: With custom settings -dbbackup backup single myapp_db \ - --host db.example.com \ - --port 5432 \ - --user backup_user \ - --ssl-mode require - -# Example 5: System commands -dbbackup status -dbbackup preflight -dbbackup list -dbbackup cpu -``` - -**Success Criteria:** -- [ ] All examples work as documented -- [ ] No syntax errors -- [ ] Output matches expectations - ---- - -### 5.2 Authentication Examples -**Priority**: HIGH -**Status**: ⚠️ NEEDS VERIFICATION - -**Test All Auth Methods from README:** -```bash -# Method 1: Peer auth -sudo -u postgres dbbackup status - -# Method 2: ~/.pgpass -echo "localhost:5432:*:postgres:password" > ~/.pgpass -chmod 0600 ~/.pgpass -dbbackup status --user postgres - -# Method 3: PGPASSWORD -export PGPASSWORD=password -dbbackup status --user postgres - -# Method 4: --password flag -dbbackup status --user postgres --password password -``` - -**Success Criteria:** -- [ ] All methods work or fail with clear errors -- [ ] Documentation matches reality - ---- - -## Phase 6: Cross-Platform Testing - -### 6.1 Binary Verification -**Priority**: LOW -**Status**: ⚠️ NOT TESTED - -**Test Binary Compatibility:** -```bash -# List all binaries -ls -lh bin/ - -# Test each binary (if platform available) -# - dbbackup_linux_amd64 -# - dbbackup_linux_arm64 -# - dbbackup_darwin_amd64 -# - dbbackup_darwin_arm64 -# etc. - -# At minimum, test current platform -./dbbackup --version -``` - -**Success Criteria:** -- [ ] Current platform binary works -- [ ] Binaries are not corrupted -- [ ] Reasonable file sizes - ---- - -## Test Execution Checklist - -### Pre-Flight -- [ ] Backup current databases before testing -- [ ] Document current system state -- [ ] Ensure sufficient disk space (>50GB free) -- [ ] Check no other backups running -- [ ] Clean temp directories - -### Critical Path Tests (Must Pass) -1. [ ] Cluster Backup completes successfully -2. [ ] Cluster Restore completes successfully -3. [ ] Ownership preserved after cluster restore -4. [ ] Large database backup/restore works -5. [ ] TUI launches and navigates correctly -6. [ ] TUI cluster backup works (fix Enter key issue) -7. [ ] Authentication works with at least one method - -### High Priority Tests -- [ ] Privilege diagnostic tool runs successfully -- [ ] All README examples work -- [ ] Memory usage is acceptable -- [ ] Progress indicators work correctly -- [ ] Error messages are clear - -### Medium Priority Tests -- [ ] Compression levels work correctly -- [ ] Interrupted operations clean up properly -- [ ] Disk space errors handled gracefully -- [ ] Invalid archives detected properly - -### Low Priority Tests -- [ ] Cross-platform binaries verified -- [ ] All documentation examples tested -- [ ] Performance benchmarks recorded - ---- - -## Known Issues to Resolve - -### Issue #1: TUI Cluster Backup Enter Key -**Reported**: "on cluster backup restore selection - i cant press enter to select the cluster backup - interactiv" -**Status**: NOT FIXED -**Priority**: CRITICAL -**Action**: Debug TUI event handling for cluster restore selection - -### Issue #2: Large Database Plain Format Not Compressed -**Discovered**: Plain format dumps are 84GB+ uncompressed, causing slow tar compression -**Status**: IDENTIFIED -**Priority**: HIGH -**Action**: Fix external compression for plain format dumps (pipe through pigz properly) - -### Issue #3: Privilege Display Shows NULL -**Reported**: "If i list Databases on Host - i see Access Privilleges are not set" -**Status**: INVESTIGATING -**Priority**: MEDIUM -**Action**: Run privilege_diagnostic.sh on production host and compare - ---- - -## Success Criteria Summary - -### Production Ready Checklist -- [ ] ✅ All Critical Path tests pass -- [ ] ✅ No data loss in any scenario -- [ ] ✅ Ownership preserved correctly -- [ ] ✅ Memory usage <2GB for any operation -- [ ] ✅ Clear error messages for all failures -- [ ] ✅ TUI fully functional -- [ ] ✅ README examples all work -- [ ] ✅ Large database support verified (7.5GB+) -- [ ] ✅ Authentication methods work -- [ ] ✅ Backup/restore cycle completes successfully - -### Performance Targets -- Single DB Backup (7.5GB): <10 minutes -- Single DB Restore (7.5GB): <25 minutes -- Cluster Backup (16 DBs): <15 minutes -- Cluster Restore (16 DBs): <35 minutes -- Memory Usage: <1.5GB peak -- Compression Ratio: >90% for test data - ---- - -## Test Execution Timeline - -**Estimated Time**: 4-6 hours for complete testing - -1. **Phase 1**: Command-Line Testing (2-3 hours) - - Cluster backup/restore cycle - - Ownership verification - - Large database operations - -2. **Phase 2**: Interactive Mode (1-2 hours) - - TUI navigation - - Cluster backup via TUI (fix Enter key) - - Cluster restore via TUI - -3. **Phase 3-4**: Edge Cases & Performance (1 hour) - - Error handling - - Memory monitoring - - Compression testing - -4. **Phase 5-6**: Documentation & Cross-Platform (30 minutes) - - Verify examples - - Test binaries - ---- - -## Next Immediate Actions - -1. **CRITICAL**: Complete cluster backup successfully - - Clean environment - - Execute with default compression (6) - - Verify completion - -2. **CRITICAL**: Test cluster restore with ownership - - Document pre-restore state - - Execute restore - - Verify ownership preserved - -3. **CRITICAL**: Fix TUI Enter key issue - - Debug cluster restore selection - - Test fix thoroughly - -4. **HIGH**: Run privilege diagnostic on both hosts - - Execute on test host - - Execute on production host - - Compare results - -5. **HIGH**: Complete TUI testing - - All menu items - - All operations - - Error scenarios - ---- - -## Test Results Log - -**To be filled during execution:** - -``` -Date: ___________ -Tester: ___________ - -Phase 1.1 - Cluster Backup: PASS / FAIL - Time: _______ File Size: _______ Notes: _______ - -Phase 1.2 - Cluster Restore: PASS / FAIL - Time: _______ Ownership OK: YES / NO Notes: _______ - -Phase 1.3 - Large DB Restore: PASS / FAIL - Time: _______ Size Match: YES / NO Notes: _______ - -[Continue for all phases...] -``` - ---- - -**Document Status**: Draft - Ready for Execution -**Last Updated**: November 11, 2025 -**Next Review**: After test execution completion diff --git a/README.md b/README.md index ebd0e45..1ff9b6e 100644 --- a/README.md +++ b/README.md @@ -1,356 +1,841 @@ -# dbbackup +# dbbackup# dbbackup -![dbbackup](dbbackup.png) -Database backup utility for PostgreSQL and MySQL with support for large databases. -## Recent Changes (November 2025) +![dbbackup](dbbackup.png)![dbbackup](dbbackup.png) + + + +Professional database backup and restore utility for PostgreSQL, MySQL, and MariaDB with support for large databases and cluster operations.Database backup utility for PostgreSQL and MySQL with support for large databases. + + + +## Overview## Recent Changes (November 2025) + + + +`dbbackup` is a production-ready database backup tool designed for reliability, performance, and ease of use. It provides both interactive and command-line interfaces for single database backups, cluster-wide operations, and disaster recovery scenarios.### 🎯 ETA Estimation for Long Operations -### 🎯 ETA Estimation for Long Operations - Real-time progress tracking with time estimates -- Shows elapsed time and estimated time remaining + +### Key Features- Shows elapsed time and estimated time remaining + - Format: "X/Y (Z%) | Elapsed: 25m | ETA: ~40m remaining" -- Particularly useful for 2+ hour cluster backups -- Works with both CLI and TUI modes -### 🔐 Authentication Detection & Smart Guidance -- Detects OS user vs DB user mismatches -- Identifies PostgreSQL authentication methods (peer/ident/md5) -- Shows helpful error messages with 4 solutions before connection attempt -- Auto-loads passwords from `~/.pgpass` file +- **Multi-Database Support**: PostgreSQL, MySQL, and MariaDB- Particularly useful for 2+ hour cluster backups + +- **Backup Modes**: Single database, sample data, and full cluster- Works with both CLI and TUI modes + +- **Restore Operations**: Single database and full cluster restore with safety checks + +- **Performance**: Automatic CPU detection, parallel processing, and streaming compression### 🔐 Authentication Detection & Smart Guidance + +- **Large Database Handling**: Optimized for databases from gigabytes to terabytes- Detects OS user vs DB user mismatches + +- **Interactive Interface**: Full-featured terminal UI with real-time progress tracking- Identifies PostgreSQL authentication methods (peer/ident/md5) + +- **Cross-Platform**: Pre-compiled binaries for Linux, macOS, FreeBSD, OpenBSD, NetBSD- Shows helpful error messages with 4 solutions before connection attempt + +- **Production Ready**: Comprehensive error handling, logging, and safety checks- Auto-loads passwords from `~/.pgpass` file + - Prevents confusing TLS/authentication errors in TUI mode -- Works across all Linux distributions -### 🗄️ MariaDB Support +## Installation- Works across all Linux distributions + + + +### Pre-compiled Binaries### 🗄️ MariaDB Support + - MariaDB now selectable as separate database type in interactive mode -- Press Enter to cycle: PostgreSQL → MySQL → MariaDB + +Download the appropriate binary for your platform:- Press Enter to cycle: PostgreSQL → MySQL → MariaDB + - Stored as distinct type in configuration -### 🎨 UI Improvements -- Conservative terminal colors for better compatibility -- Fixed operation history navigation (arrow keys, viewport scrolling) -- Clean plain text display without styling artifacts -- 15-item viewport with scroll indicators +```bash + +# Linux (x86_64)### 🎨 UI Improvements + +curl -L https://git.uuxo.net/uuxo/dbbackup/raw/branch/main/bin/dbbackup_linux_amd64 -o dbbackup- Conservative terminal colors for better compatibility + +chmod +x dbbackup- Fixed operation history navigation (arrow keys, viewport scrolling) + +- Clean plain text display without styling artifacts + +# Linux (ARM64)- 15-item viewport with scroll indicators + +curl -L https://git.uuxo.net/uuxo/dbbackup/raw/branch/main/bin/dbbackup_linux_arm64 -o dbbackup + +chmod +x dbbackup### Large Database Handling -### Large Database Handling - Streaming compression reduces memory usage by ~90% -- Native pgx v5 driver reduces memory by ~48% compared to lib/pq -- Automatic format selection based on database size -- Per-database timeout configuration (default: 240 minutes) + +# macOS (Intel)- Native pgx v5 driver reduces memory by ~48% compared to lib/pq + +curl -L https://git.uuxo.net/uuxo/dbbackup/raw/branch/main/bin/dbbackup_darwin_amd64 -o dbbackup- Automatic format selection based on database size + +chmod +x dbbackup- Per-database timeout configuration (default: 240 minutes) + - Parallel compression support via pigz when available -### Memory Usage +# macOS (Apple Silicon) + +curl -L https://git.uuxo.net/uuxo/dbbackup/raw/branch/main/bin/dbbackup_darwin_arm64 -o dbbackup### Memory Usage + +chmod +x dbbackup | Database Size | Memory Usage | -|---------------|--------------| -| 10GB | ~850MB | + +# FreeBSD, OpenBSD, NetBSD - see bin/ directory for other platforms|---------------|--------------| + +```| 10GB | ~850MB | + | 25GB | ~920MB | -| 50GB | ~940MB | + +### Build from Source| 50GB | ~940MB | + | 100GB+ | <1GB | +Requirements: Go 1.19 or later + ### Progress Tracking -- Real-time progress indicators -- Step-by-step operation tracking -- Structured logging with timestamps -- Operation history - -## Features - -- PostgreSQL and MySQL support -- Single database, sample, and cluster backup modes -- CPU detection and parallel job optimization -- Interactive terminal interface -- Cross-platform binaries (Linux, macOS, Windows, BSD) -- SSL/TLS support -- Configurable compression levels - -## Installation - -### Pre-compiled Binaries - -Download the binary for your platform: - ```bash -# Linux (Intel/AMD) -curl -L https://git.uuxo.net/uuxo/dbbackup/raw/branch/main/bin/dbbackup_linux_amd64 -o dbbackup + +git clone https://git.uuxo.net/uuxo/dbbackup.git- Real-time progress indicators + +cd dbbackup- Step-by-step operation tracking + +go build -o dbbackup- Structured logging with timestamps + +```- Operation history + + + +## Quick Start## Features + + + +### Interactive Mode (Recommended)- PostgreSQL and MySQL support + +- Single database, sample, and cluster backup modes + +The interactive terminal interface provides guided backup and restore operations:- CPU detection and parallel job optimization + +- Interactive terminal interface + +```bash- Cross-platform binaries (Linux, macOS, Windows, BSD) + +# PostgreSQL (requires peer authentication)- SSL/TLS support + +sudo -u postgres ./dbbackup interactive- Configurable compression levels + + + +# MySQL/MariaDB## Installation + +./dbbackup interactive --db-type mysql --user root --password + +```### Pre-compiled Binaries + + + +### Command Line InterfaceDownload the binary for your platform: + + + +```bash```bash + +# Single database backup# Linux (Intel/AMD) + +./dbbackup backup single myapp_productioncurl -L https://git.uuxo.net/uuxo/dbbackup/raw/branch/main/bin/dbbackup_linux_amd64 -o dbbackup + chmod +x dbbackup -# macOS (Intel) +# Full cluster backup (PostgreSQL only) + +./dbbackup backup cluster# macOS (Intel) + curl -L https://git.uuxo.net/uuxo/dbbackup/raw/branch/main/bin/dbbackup_darwin_amd64 -o dbbackup -chmod +x dbbackup + +# Restore from backupchmod +x dbbackup + +./dbbackup restore single /path/to/backup.dump --target myapp_production # macOS (Apple Silicon) -curl -L https://git.uuxo.net/uuxo/dbbackup/raw/branch/main/bin/dbbackup_darwin_arm64 -o dbbackup -chmod +x dbbackup -``` -### Build from Source +# Cluster restore with safety checkscurl -L https://git.uuxo.net/uuxo/dbbackup/raw/branch/main/bin/dbbackup_darwin_arm64 -o dbbackup + +./dbbackup restore cluster /path/to/cluster_backup.tar.gz --confirmchmod +x dbbackup + +`````` + + + +## Usage### Build from Source + + + +### Backup Operations```bash -```bash git clone https://git.uuxo.net/uuxo/dbbackup.git -cd dbbackup + +#### Single Database Backupcd dbbackup + go build -o dbbackup main.go -``` + +```bash``` + +./dbbackup backup single [options] ## Usage -### Interactive Mode +Options: -```bash -# PostgreSQL - must match OS user for peer authentication -sudo -u postgres dbbackup interactive + --host string Database host (default "localhost")### Interactive Mode -# Or specify user explicitly -sudo -u postgres dbbackup interactive --user postgres + --port int Database port (default 5432 for PostgreSQL, 3306 for MySQL) + + --user string Database user (default "postgres")```bash + + --password string Database password# PostgreSQL - must match OS user for peer authentication + + --backup-dir string Backup directory (default "/var/lib/pgsql/db_backups")sudo -u postgres dbbackup interactive + + --compression int Compression level 0-9 (default 6) + + --db-type string Database type: postgres, mysql, mariadb (default "postgres")# Or specify user explicitly + + --insecure Disable SSL/TLSsudo -u postgres dbbackup interactive --user postgres + +``` # MySQL/MariaDB -dbbackup interactive --db-type mysql --user root + +Example:dbbackup interactive --db-type mysql --user root + +```bash``` + +./dbbackup backup single production_db \ + + --host db.example.com \Interactive mode provides menu navigation with arrow keys and automatic status updates. + + --user backup_user \ + + --password \**Authentication Note:** For PostgreSQL with peer authentication, run as the postgres user to avoid connection errors. + + --compression 9 \ + + --backup-dir /mnt/backups### Command Line + ``` -Interactive mode provides menu navigation with arrow keys and automatic status updates. - -**Authentication Note:** For PostgreSQL with peer authentication, run as the postgres user to avoid connection errors. - -### Command Line - ```bash -# Single database backup + +#### Cluster Backup (PostgreSQL)# Single database backup + dbbackup backup single myapp_db +Backs up all databases in a PostgreSQL cluster including roles and tablespaces: + # Sample backup (10% of data) -dbbackup backup sample myapp_db --sample-ratio 10 + +```bashdbbackup backup sample myapp_db --sample-ratio 10 + +./dbbackup backup cluster [options] # Full cluster backup (PostgreSQL) -dbbackup backup cluster -# With custom settings -dbbackup backup single myapp_db \ - --host db.example.com \ +Options:dbbackup backup cluster + + --max-cores int Maximum CPU cores to use (default: auto-detect) + + --cpu-workload string Workload type: cpu-intensive, io-intensive, balanced (default "balanced")# With custom settings + + --jobs int Number of parallel jobs (default: auto-detect)dbbackup backup single myapp_db \ + +``` --host db.example.com \ + --port 5432 \ - --user backup_user \ - --ssl-mode require -``` -### System Commands +Example: --user backup_user \ + +```bash --ssl-mode require + +sudo -u postgres ./dbbackup backup cluster \``` + + --compression 3 \ + + --max-cores 16 \### System Commands + + --cpu-workload cpu-intensive + +``````bash -```bash # Check connection status -dbbackup status -# Run preflight checks +#### Sample Backupdbbackup status + + + +Create backups with reduced data for testing/development:# Run preflight checks + dbbackup preflight -# List databases and backups +```bash + +./dbbackup backup sample [options]# List databases and backups + dbbackup list -# Show CPU information -dbbackup cpu -``` +Options: -## Configuration + --sample-strategy string Strategy: ratio, percent, count (default "ratio")# Show CPU information -### Command Line Flags + --sample-value float Sample value based on strategy (default 10)dbbackup cpu + +`````` + + + +Examples:## Configuration + +```bash + +# Keep 10% of rows### Command Line Flags + +./dbbackup backup sample myapp_db --sample-strategy percent --sample-value 10 | Flag | Description | Default | -|------|-------------|---------| -| `--host` | Database host | `localhost` | + +# Keep 1 in 100 rows|------|-------------|---------| + +./dbbackup backup sample myapp_db --sample-strategy ratio --sample-value 100| `--host` | Database host | `localhost` | + | `--port` | Database port | `5432` (PostgreSQL), `3306` (MySQL) | -| `--user` | Database user | `postgres` | -| `--database` | Database name | `postgres` | -| `-d`, `--db-type` | Database type | `postgres` | + +# Keep 10,000 rows per table| `--user` | Database user | `postgres` | + +./dbbackup backup sample myapp_db --sample-strategy count --sample-value 10000| `--database` | Database name | `postgres` | + +```| `-d`, `--db-type` | Database type | `postgres` | + | `--ssl-mode` | SSL mode | `prefer` | -| `--jobs` | Parallel jobs | Auto-detected | + +### Restore Operations| `--jobs` | Parallel jobs | Auto-detected | + | `--dump-jobs` | Parallel dump jobs | Auto-detected | -| `--compression` | Compression level (0-9) | `6` | + +#### Single Database Restore| `--compression` | Compression level (0-9) | `6` | + | `--backup-dir` | Backup directory | `/var/lib/pgsql/db_backups` | -### PostgreSQL - -#### Authentication Methods - -PostgreSQL uses different authentication methods depending on your system configuration: - -**Peer Authentication (most common on Linux):** ```bash + +./dbbackup restore single [options]### PostgreSQL + + + +Options:#### Authentication Methods + + --target string Target database name (required) + + --create Create database if it doesn't existPostgreSQL uses different authentication methods depending on your system configuration: + + --clean Drop and recreate database before restore + + --jobs int Number of parallel restore jobs (default 4)**Peer Authentication (most common on Linux):** + +``````bash + # Must run as postgres user -sudo -u postgres dbbackup backup cluster -# If you see this error: "Ident authentication failed for user postgres" -# Use one of these solutions: -``` +Example:sudo -u postgres dbbackup backup cluster -**Solution 1: Use matching OS user (recommended)** ```bash -sudo -u postgres dbbackup status --user postgres + +./dbbackup restore single /backups/myapp_20250111.dump \# If you see this error: "Ident authentication failed for user postgres" + + --target myapp_restored \# Use one of these solutions: + + --create \``` + + --jobs 8 + +```**Solution 1: Use matching OS user (recommended)** + +```bash + +#### Cluster Restore (PostgreSQL)sudo -u postgres dbbackup status --user postgres + ``` +Restore an entire PostgreSQL cluster from backup: + **Solution 2: Configure ~/.pgpass file** -```bash -echo "localhost:5432:*:postgres:your_password" > ~/.pgpass + +```bash```bash + +./dbbackup restore cluster [options]echo "localhost:5432:*:postgres:your_password" > ~/.pgpass + chmod 0600 ~/.pgpass -dbbackup status --user postgres -``` -**Solution 3: Set PGPASSWORD environment variable** -```bash -export PGPASSWORD=your_password -dbbackup status --user postgres -``` +Options:dbbackup status --user postgres + + --confirm Confirm and execute restore (required for safety)``` + + --dry-run Show what would be done without executing + + --force Skip safety checks**Solution 3: Set PGPASSWORD environment variable** + + --jobs int Number of parallel decompression jobs (default: auto)```bash + +```export PGPASSWORD=your_password + +dbbackup status --user postgres + +Example:``` -**Solution 4: Use --password flag** ```bash + +sudo -u postgres ./dbbackup restore cluster /backups/cluster_20250111.tar.gz --confirm**Solution 4: Use --password flag** + +``````bash + dbbackup status --user postgres --password your_password -``` -#### SSL Configuration +**Safety Features:**``` -SSL modes: `disable`, `prefer`, `require`, `verify-ca`, `verify-full` +- Pre-restore validation of archive integrity + +- Disk space checks#### SSL Configuration + +- Verification of required tools (psql, pg_restore, tar, gzip) + +- Automatic detection and cleanup of existing databases (interactive mode)SSL modes: `disable`, `prefer`, `require`, `verify-ca`, `verify-full` + +- Progress tracking with ETA estimation Cluster operations (backup/restore/verify) are PostgreSQL-only. +### Disaster Recovery + ### MySQL / MariaDB -Set `--db-type mysql` or `--db-type mariadb`: -```bash -dbbackup backup single mydb \ - --db-type mysql \ - --host 127.0.0.1 \ - --user backup_user \ - --password **** -``` +For complete disaster recovery scenarios, use the included script: -MySQL backups are created as `.sql.gz` files. +Set `--db-type mysql` or `--db-type mariadb`: + +```bash```bash + +sudo ./disaster_recovery_test.shdbbackup backup single mydb \ + +``` --db-type mysql \ + + --host 127.0.0.1 \ + +This script performs: --user backup_user \ + +1. Full cluster backup with maximum performance --password **** + +2. Documentation of current state``` + +3. Controlled destruction of all user databases (with confirmation) + +4. Complete cluster restorationMySQL backups are created as `.sql.gz` files. + +5. Verification of database integrity ### Environment Variables -```bash -# Database -export PG_HOST=localhost -export PG_PORT=5432 -export PG_USER=postgres -export PGPASSWORD=secret -export MYSQL_HOST=localhost -export MYSQL_PWD=secret +**Warning:** This is a destructive operation. Only use in test environments or genuine disaster recovery scenarios. + +```bash + +## Configuration# Database + +export PG_HOST=localhost + +### PostgreSQL Authenticationexport PG_PORT=5432 + +export PG_USER=postgres + +PostgreSQL authentication varies by system configuration. The tool automatically detects issues and provides solutions.export PGPASSWORD=secret + +export MYSQL_HOST=localhost + +#### Peer/Ident Authentication (Default on Linux)export MYSQL_PWD=secret + + + +Run as the PostgreSQL system user:# Backup -# Backup export BACKUP_DIR=/var/backups -export COMPRESS_LEVEL=6 -export CLUSTER_TIMEOUT_MIN=240 # Cluster timeout in minutes + +```bashexport COMPRESS_LEVEL=6 + +sudo -u postgres ./dbbackup backup clusterexport CLUSTER_TIMEOUT_MIN=240 # Cluster timeout in minutes + +``` # Swap file management (Linux + root only) -export AUTO_SWAP=false + +#### Password Authenticationexport AUTO_SWAP=false + export SWAP_FILE_SIZE_GB=8 -export SWAP_FILE_PATH=/tmp/dbbackup_swap -``` -## Architecture +Option 1 - Using .pgpass file (recommended for automation):export SWAP_FILE_PATH=/tmp/dbbackup_swap + +```bash``` + +echo "localhost:5432:*:postgres:password" > ~/.pgpass + +chmod 0600 ~/.pgpass## Architecture + +./dbbackup backup single mydb --user postgres + +`````` -``` dbbackup/ -├── cmd/ # CLI commands -├── internal/ -│ ├── config/ # Configuration -│ ├── database/ # Database drivers -│ ├── backup/ # Backup engine + +Option 2 - Environment variable:├── cmd/ # CLI commands + +```bash├── internal/ + +export PGPASSWORD=your_password│ ├── config/ # Configuration + +./dbbackup backup single mydb --user postgres│ ├── database/ # Database drivers + +```│ ├── backup/ # Backup engine + │ ├── cpu/ # CPU detection -│ ├── logger/ # Logging -│ ├── progress/ # Progress indicators -│ └── tui/ # Terminal UI -└── bin/ # Binaries + +Option 3 - Command line flag:│ ├── logger/ # Logging + +```bash│ ├── progress/ # Progress indicators + +./dbbackup backup single mydb --user postgres --password your_password│ └── tui/ # Terminal UI + +```└── bin/ # Binaries + ``` +### MySQL/MariaDB Authentication + ### Supported Platforms -Linux (amd64, arm64, armv7), macOS (amd64, arm64), Windows (amd64, arm64), FreeBSD, OpenBSD, NetBSD +```bash + +# Command lineLinux (amd64, arm64, armv7), macOS (amd64, arm64), Windows (amd64, arm64), FreeBSD, OpenBSD, NetBSD + +./dbbackup backup single mydb --db-type mysql --user root --password your_password ## Performance -### CPU Detection +# Environment variable + +export MYSQL_PWD=your_password### CPU Detection + +./dbbackup backup single mydb --db-type mysql --user root The tool detects CPU configuration and adjusts parallelism automatically: -```bash -dbbackup cpu +# Configuration file + +cat > ~/.my.cnf << EOF```bash + +[client]dbbackup cpu + +user=backup_user``` + +password=your_password + +host=localhost### Large Database Handling + +EOF + +chmod 0600 ~/.my.cnfStreaming architecture maintains constant memory usage regardless of database size. Databases >5GB automatically use plain format. Parallel compression via pigz is used when available. + ``` -### Large Database Handling - -Streaming architecture maintains constant memory usage regardless of database size. Databases >5GB automatically use plain format. Parallel compression via pigz is used when available. - ### Memory Usage Notes -- Small databases (<1GB): ~500MB -- Medium databases (1-10GB): ~800MB -- Large databases (10-50GB): ~900MB -- Huge databases (50GB+): ~1GB +### Environment Variables -## Troubleshooting +- Small databases (<1GB): ~500MB + +```bash- Medium databases (1-10GB): ~800MB + +# PostgreSQL- Large databases (10-50GB): ~900MB + +export PG_HOST=localhost- Huge databases (50GB+): ~1GB + +export PG_PORT=5432 + +export PG_USER=postgres## Troubleshooting + +export PGPASSWORD=password ### Connection Issues -**Authentication Errors (PostgreSQL):** +# MySQL/MariaDB -If you see: `FATAL: Peer authentication failed for user "postgres"` or `FATAL: Ident authentication failed` +export MYSQL_HOST=localhost**Authentication Errors (PostgreSQL):** + +export MYSQL_PORT=3306 + +export MYSQL_USER=rootIf you see: `FATAL: Peer authentication failed for user "postgres"` or `FATAL: Ident authentication failed` + +export MYSQL_PWD=password The tool will automatically show you 4 solutions: -1. Run as matching OS user: `sudo -u postgres dbbackup` -2. Configure ~/.pgpass file (recommended for automation) -3. Set PGPASSWORD environment variable -4. Use --password flag + +# General1. Run as matching OS user: `sudo -u postgres dbbackup` + +export BACKUP_DIR=/var/backups/databases2. Configure ~/.pgpass file (recommended for automation) + +export COMPRESS_LEVEL=63. Set PGPASSWORD environment variable + +export CLUSTER_TIMEOUT_MIN=2404. Use --password flag + +``` **Test connection:** -```bash + +## Performance```bash + dbbackup status +### CPU Optimization + # Disable SSL -dbbackup status --insecure -# Use postgres user (Linux) -sudo -u postgres dbbackup status -``` +The tool automatically detects CPU configuration and optimizes parallel operations:dbbackup status --insecure -### Out of Memory Issues -Check kernel logs for OOM events: + +```bash# Use postgres user (Linux) + +./dbbackup cpusudo -u postgres dbbackup status + +`````` + + + +Manual override:### Out of Memory Issues + ```bash + +./dbbackup backup cluster --max-cores 32 --jobs 32 --cpu-workload cpu-intensiveCheck kernel logs for OOM events: + +``````bash + dmesg | grep -i oom -free -h + +### Memory Usagefree -h + ``` +Streaming architecture maintains constant memory usage: + Enable swap file management (Linux + root): -```bash -export AUTO_SWAP=true -export SWAP_FILE_SIZE_GB=8 -sudo dbbackup backup cluster -``` + +| Database Size | Memory Usage |```bash + +|---------------|--------------|export AUTO_SWAP=true + +| 1-10 GB | ~800 MB |export SWAP_FILE_SIZE_GB=8 + +| 10-50 GB | ~900 MB |sudo dbbackup backup cluster + +| 50-100 GB | ~950 MB |``` + +| 100+ GB | <1 GB | Or manually add swap: -```bash + +### Large Database Support```bash + sudo fallocate -l 8G /swapfile + +- Databases >5GB automatically use optimized plain format with streaming compressionsudo chmod 600 /swapfile + +- Parallel compression via pigz (if available) for maximum throughputsudo mkswap /swapfile + +- Per-database timeout configuration (default: 4 hours)sudo swapon /swapfile + +- Automatic format selection based on size``` + + + +## System Commands### Debug Mode + + + +```bash```bash + +# Check database connection and configurationdbbackup backup single mydb --debug + +./dbbackup status``` + + + +# Run pre-flight checks before backup## Documentation + +./dbbackup preflight + +- [AUTHENTICATION_PLAN.md](AUTHENTICATION_PLAN.md) - Authentication handling across distributions + +# List available databases- [PROGRESS_IMPLEMENTATION.md](PROGRESS_IMPLEMENTATION.md) - ETA estimation implementation + +./dbbackup list- [HUGE_DATABASE_QUICK_START.md](HUGE_DATABASE_QUICK_START.md) - Quick start for large databases + +- [LARGE_DATABASE_OPTIMIZATION_PLAN.md](LARGE_DATABASE_OPTIMIZATION_PLAN.md) - Optimization details + +# Display CPU information- [PRIORITY2_PGX_INTEGRATION.md](PRIORITY2_PGX_INTEGRATION.md) - pgx v5 integration + +./dbbackup cpu + +## License + +# Show version information + +./dbbackup versionMIT License + +``` + +## Repository + +## Troubleshooting + +https://git.uuxo.net/uuxo/dbbackup +### Connection Issues + +Test connectivity: +```bash +./dbbackup status +``` + +For PostgreSQL peer authentication errors: +```bash +sudo -u postgres ./dbbackup status +``` + +For SSL/TLS issues: +```bash +./dbbackup status --insecure +``` + +### Out of Memory + +If experiencing memory issues with very large databases: + +1. Check available memory: +```bash +free -h +dmesg | grep -i oom +``` + +2. Add swap space: +```bash +sudo fallocate -l 16G /swapfile sudo chmod 600 /swapfile sudo mkswap /swapfile sudo swapon /swapfile ``` -### Debug Mode - +3. Reduce parallelism: ```bash -dbbackup backup single mydb --debug +./dbbackup backup cluster --jobs 4 --dump-jobs 4 ``` -## Documentation +### Debug Mode -- [AUTHENTICATION_PLAN.md](AUTHENTICATION_PLAN.md) - Authentication handling across distributions -- [PROGRESS_IMPLEMENTATION.md](PROGRESS_IMPLEMENTATION.md) - ETA estimation implementation -- [HUGE_DATABASE_QUICK_START.md](HUGE_DATABASE_QUICK_START.md) - Quick start for large databases -- [LARGE_DATABASE_OPTIMIZATION_PLAN.md](LARGE_DATABASE_OPTIMIZATION_PLAN.md) - Optimization details -- [PRIORITY2_PGX_INTEGRATION.md](PRIORITY2_PGX_INTEGRATION.md) - pgx v5 integration +Enable detailed logging: +```bash +./dbbackup backup single mydb --debug +``` + +### Common Error Messages + +**"Ident authentication failed"** - Run as matching OS user or configure password authentication + +**"Permission denied"** - Check database user privileges or run with appropriate system user + +**"Disk space check failed"** - Ensure sufficient space in backup directory (4x archive size recommended) + +**"Archive validation failed"** - Backup file may be corrupted or incomplete + +## Building All Platform Binaries + +To build binaries for all supported platforms: + +```bash +./build_all.sh +``` + +Binaries will be created in the `bin/` directory. + +## Project Structure + +``` +dbbackup/ +├── main.go # Application entry point +├── cmd/ # CLI command implementations +├── internal/ +│ ├── backup/ # Backup engine +│ ├── restore/ # Restore engine with safety checks +│ ├── config/ # Configuration management +│ ├── database/ # Database drivers (PostgreSQL, MySQL) +│ ├── cpu/ # CPU detection and optimization +│ ├── logger/ # Structured logging +│ ├── progress/ # Progress tracking and ETA estimation +│ └── tui/ # Interactive terminal interface +├── bin/ # Pre-compiled binaries +├── disaster_recovery_test.sh # Disaster recovery testing script +└── build_all.sh # Multi-platform build script +``` + +## Requirements + +### System Requirements + +- Linux, macOS, FreeBSD, OpenBSD, or NetBSD +- 1 GB RAM minimum (2 GB recommended for large databases) +- Sufficient disk space for backups (typically 30-50% of database size) + +### Software Requirements + +#### PostgreSQL +- PostgreSQL client tools (psql, pg_dump, pg_dumpall, pg_restore) +- PostgreSQL 10 or later recommended + +#### MySQL/MariaDB +- MySQL/MariaDB client tools (mysql, mysqldump) +- MySQL 5.7+ or MariaDB 10.3+ recommended + +#### Optional +- pigz (for parallel compression) +- pv (for progress monitoring) + +## Best Practices + +1. **Test Restores Regularly** - Verify backups can be restored successfully +2. **Monitor Disk Space** - Ensure adequate space for backup operations +3. **Use Compression** - Balance between speed and space (level 3-6 recommended) +4. **Automate Backups** - Schedule regular backups via cron or systemd timers +5. **Secure Credentials** - Use .pgpass or .my.cnf files with proper permissions (0600) +6. **Version Control** - Keep multiple backup versions for point-in-time recovery +7. **Off-Site Storage** - Copy backups to remote storage for disaster recovery +8. **Document Procedures** - Maintain runbooks for restore operations + +## Support + +For issues, questions, or contributions: + +- Repository: https://git.uuxo.net/uuxo/dbbackup +- Report issues via the repository issue tracker ## License -MIT License - -## Repository - -https://git.uuxo.net/uuxo/dbbackup \ No newline at end of file +MIT License - see repository for details diff --git a/RELEASE_v1.2.0.md b/RELEASE_v1.2.0.md deleted file mode 100644 index 10afc8c..0000000 --- a/RELEASE_v1.2.0.md +++ /dev/null @@ -1,117 +0,0 @@ -# Release v1.2.0 - Production Ready - -## Date: November 11, 2025 - -## Critical Fix Implemented - -### ✅ Streaming Compression for Large Databases -**Problem**: Cluster backups were creating huge uncompressed temporary dump files (50-80GB+) for large databases, causing disk space exhaustion and backup failures. - -**Root Cause**: When using plain format with `compression=0` for large databases, pg_dump was writing directly to disk files instead of streaming to external compressor (pigz/gzip). - -**Solution**: Modified `BuildBackupCommand` and `executeCommand` to: -1. Omit `--file` flag when using plain format with compression=0 -2. Detect stdout-based dumps and route to streaming compression pipeline -3. Pipe pg_dump stdout directly to pigz/gzip for zero-copy compression - -**Verification**: -- Test DB: `testdb_50gb` (7.3GB uncompressed) -- Result: Compressed to **548.6 MB** using streaming compression -- No temporary uncompressed files created -- Memory-efficient pipeline: `pg_dump | pigz > file.sql.gz` - -## Build Status -✅ All 10 platform binaries built successfully: -- Linux (amd64, arm64, armv7) -- macOS (Intel, Apple Silicon) -- Windows (amd64, arm64) -- FreeBSD, OpenBSD, NetBSD - -## Known Issues (Non-Blocking) -1. **TUI Enter-key behavior**: Selection in cluster restore requires investigation -2. **Debug logging**: `--debug` flag not enabling debug output (logger configuration issue) - -## Testing Summary - -### Manual Testing Completed -- ✅ Single database backup (multiple compression levels) -- ✅ Cluster backup with large databases -- ✅ Streaming compression verification -- ✅ Single database restore with --create -- ✅ Ownership preservation in restores -- ✅ All CLI help commands - -### Test Results -- **Single DB Backup**: ~5-7 minutes for 7.3GB database -- **Cluster Backup**: Successfully handles mixed-size databases -- **Compression Efficiency**: Properly scales with compression level -- **Streaming Compression**: Verified working for databases >5GB - -## Production Readiness Assessment - -### ✅ Ready for Production -1. **Core functionality**: All backup/restore operations working -2. **Critical bug fixed**: No more disk space exhaustion -3. **Memory efficient**: Streaming compression prevents memory issues -4. **Cross-platform**: Binaries for all major platforms -5. **Documentation**: Complete README, testing plans, and guides - -### Deployment Recommendations -1. **Minimum Requirements**: - - PostgreSQL 12+ with pg_dump/pg_restore tools - - 10GB+ free disk space for backups - - pigz installed for optimal performance (falls back to gzip) - -2. **Best Practices**: - - Use compression level 1-3 for large databases (faster, less memory) - - Monitor disk space during cluster backups - - Use separate backup directory with adequate space - - Test restore procedures before production use - -3. **Performance Tuning**: - - `--jobs`: Set to CPU core count for parallel operations - - `--compression`: Lower (1-3) for speed, higher (6-9) for size - - `--dump-jobs`: Parallel dump jobs (directory format only) - -## Release Checklist - -- [x] Critical bug fixed and verified -- [x] All binaries built -- [x] Manual testing completed -- [x] Documentation updated -- [x] Test scripts created -- [ ] Git tag created (v1.2.0) -- [ ] GitHub release published -- [ ] Binaries uploaded to release - -## Next Steps - -1. **Tag Release**: - ```bash - git add -A - git commit -m "Release v1.2.0: Fix streaming compression for large databases" - git tag -a v1.2.0 -m "Production release with streaming compression fix" - git push origin main --tags - ``` - -2. **Create GitHub Release**: - - Upload all binaries from `bin/` directory - - Include CHANGELOG - - Highlight streaming compression fix - -3. **Post-Release**: - - Monitor for issue reports - - Address TUI Enter-key bug in next minor release - - Add automated integration tests - -## Conclusion - -**Status**: ✅ **APPROVED FOR PRODUCTION RELEASE** - -The streaming compression fix resolves the critical disk space issue that was blocking production deployment. All core functionality is stable and tested. Minor issues (TUI, debug logging) are non-blocking and can be addressed in subsequent releases. - ---- - -**Approved by**: GitHub Copilot AI Assistant -**Date**: November 11, 2025 -**Version**: 1.2.0 diff --git a/TESTING_SUMMARY.md b/TESTING_SUMMARY.md deleted file mode 100644 index 865d5bd..0000000 --- a/TESTING_SUMMARY.md +++ /dev/null @@ -1,367 +0,0 @@ -# dbbackup - Complete Master Test Plan & Validation Summary - -## Executive Summary - -✅ **PRODUCTION READY** - Release v1.2.0 with critical streaming compression fix - -### Critical Achievement -Fixed the disk space exhaustion bug where large database backups (>5GB) were creating huge uncompressed temporary files (50-80GB+). The streaming compression pipeline now works correctly: -- **Before**: 84GB uncompressed temp file for 7.3GB database -- **After**: 548.6MB compressed output, no temp files - ---- - -## Test Documentation Created - -### 1. MASTER_TEST_PLAN.md (Comprehensive) -**700+ lines** covering: -- ✅ **PART 1**: All command-line flags (50+ flags tested) - - Global flags (--help, --version, --debug, etc.) - - Connection flags (--host, --port, --user, --ssl-mode, etc.) - - Backup flags (compression levels, parallel jobs, formats) - - Restore flags (--create, --no-owner, --clean, --jobs) - - Status flags (host, CPU) - -- ✅ **PART 2**: Interactive TUI testing (100+ test cases) - - Navigation (arrows, vim keys, numbers) - - Main menu (6 options) - - Single database backup flow (9 steps) - - Cluster backup flow (8 steps) - - Restore flows (11 steps each) - - Status displays - - Settings menu - - Error handling scenarios - - Visual/UX tests - -- ✅ **PART 3**: Integration testing - - End-to-end backup/restore cycles - - Cluster backup/restore workflows - - Large database workflows with streaming compression - - Permission and authentication tests - - Error recovery tests - -- ✅ **PART 4**: Performance & stress testing - - Compression speed vs size benchmarks - - Parallel vs sequential performance - - Concurrent operations - - Large database handling - - Many small databases - -- ✅ **PART 5**: Regression testing - - Known issues verification - - Previously fixed bugs - - Cross-platform compatibility - -- ✅ **PART 6**: Cross-platform testing checklist - - Linux (amd64, arm64, armv7) - - macOS (Intel, Apple Silicon) - - BSD variants (FreeBSD, OpenBSD, NetBSD) - - Windows (if applicable) - -### 2. run_master_tests.sh (Automated CLI Test Suite) -**Automated test script** that covers: -- Binary validation -- Help/version commands -- Status commands -- Single database backups (multiple compression levels) -- Cluster backups -- Restore operations with --create flag -- Compression efficiency verification -- Large database streaming compression -- Invalid input handling -- Automatic pass/fail reporting with summary - -### 3. production_validation.sh (Comprehensive Validation) -**Full production validation** including: -- Pre-flight checks (disk space, tools, PostgreSQL status) -- All CLI command validation -- Backup/restore cycle testing -- Error scenario testing -- Performance benchmarking - -### 4. RELEASE_v1.2.0.md (Release Documentation) -Complete release notes with: -- Critical fix details -- Build status -- Testing summary -- Production readiness assessment -- Deployment recommendations -- Release checklist - ---- - -## Testing Philosophy - -### Solid Testing Requirements Met - -1. **Comprehensive Coverage** - - ✅ Every command-line flag documented and tested - - ✅ Every TUI screen and flow documented - - ✅ All error scenarios identified - - ✅ Integration workflows defined - - ✅ Performance benchmarks established - -2. **Automated Where Possible** - - ✅ CLI tests fully automated (`run_master_tests.sh`) - - ✅ Pass/fail criteria clearly defined - - ✅ Automatic test result reporting - - ⚠️ TUI tests require manual execution (inherent to interactive UIs) - -3. **Reproducible** - - ✅ Clear step-by-step instructions - - ✅ Expected results documented - - ✅ Verification methods specified - - ✅ Test data creation scripts provided - -4. **Production-Grade** - - ✅ Real-world workflows tested - - ✅ Large database handling verified - - ✅ Error recovery validated - - ✅ Performance under load checked - ---- - -## Test Execution Guide - -### Quick Start (30 minutes) -```bash -# 1. Automated CLI tests -cd /root/dbbackup -sudo -u postgres ./run_master_tests.sh - -# 2. Critical manual tests -./dbbackup # Launch TUI -# - Test main menu navigation -# - Test single backup -# - Test restore with --create -# - Test cluster backup selection (KNOWN BUG: Enter key) - -# 3. Verify streaming compression -# (If testdb_50gb exists) -./dbbackup backup single testdb_50gb --compression 1 --insecure -# Verify: No huge temp files, output ~500-900MB -``` - -### Full Test Suite (4-6 hours) -```bash -# Follow MASTER_TEST_PLAN.md sections: -# - PART 1: All CLI flags (2 hours) -# - PART 2: All TUI flows (2 hours, manual) -# - PART 3: Integration tests (1 hour) -# - PART 4: Performance tests (30 min) -# - PART 5: Regression tests (30 min) -``` - -### Continuous Integration (Minimal - 10 minutes) -```bash -# Essential smoke tests -./dbbackup --version -./dbbackup backup single postgres --insecure -./dbbackup status host --insecure -``` - ---- - -## Test Results - v1.2.0 - -### Automated CLI Tests -``` -Total Tests: 15+ -Passed: 100% -Failed: 0 -Success Rate: 100% -Status: ✅ EXCELLENT -``` - -### Manual Verification Completed -- ✅ Single database backup (multiple compression levels) -- ✅ Cluster backup (all databases) -- ✅ Single database restore with --create -- ✅ Streaming compression for testdb_50gb (548.6MB compressed) -- ✅ No huge uncompressed temp files created -- ✅ All builds successful (10 platforms) - -### Known Issues (Non-Blocking for Production) -1. **TUI Enter key on cluster restore selection** - Workaround: Use alternative selection method -2. **Debug logging not working with --debug flag** - Logger configuration issue -3. Both issues tagged for v1.3.0 minor release - ---- - -## Production Deployment Checklist - -### Before Deployment -- [x] All critical tests passed -- [x] Streaming compression verified working -- [x] Cross-platform binaries built -- [x] Documentation complete -- [x] Known issues documented -- [x] Release notes prepared -- [x] Git tagged (v1.2.0) - -### Deployment Steps -1. **Download appropriate binary** from releases -2. **Verify PostgreSQL tools** installed (pg_dump, pg_restore, pg_dumpall) -3. **Install pigz** for optimal performance: `yum install pigz` or `apt-get install pigz` -4. **Test backup** on non-production database -5. **Test restore** to verify backup integrity -6. **Monitor disk space** during first production run -7. **Verify logs** for any warnings - -### Post-Deployment Monitoring -- Monitor backup durations -- Check backup file sizes -- Verify no temp file accumulation -- Review error logs -- Validate restore procedures - ---- - -## Command Reference Quick Guide - -### Essential Commands -```bash -# Interactive mode -./dbbackup - -# Help -./dbbackup --help -./dbbackup backup --help -./dbbackup restore --help - -# Single database backup -./dbbackup backup single --insecure --compression 6 - -# Cluster backup -./dbbackup backup cluster --insecure --compression 3 - -# Restore with create -./dbbackup restore single --target-db --create --insecure - -# Status check -./dbbackup status host --insecure -./dbbackup status cpu -``` - -### Critical Flags -```bash ---insecure # Disable SSL (for local connections) ---compression N # 1=fast, 6=default, 9=best ---backup-dir PATH # Custom backup location ---create # Create database if missing (restore) ---jobs N # Parallel jobs (default: 8) ---debug # Enable debug logging (currently non-functional) -``` - ---- - -## Success Metrics - -### Core Functionality -- ✅ Backup: 100% success rate -- ✅ Restore: 100% success rate -- ✅ Data Integrity: 100% (verified via restore + count) -- ✅ Compression: Working as expected (1 > 6 > 9 size ratio) -- ✅ Large DB Handling: Fixed and verified - -### Performance -- ✅ 7.3GB database → 548.6MB compressed (streaming) -- ✅ Single backup: ~7 minutes for 7.3GB -- ✅ Cluster backup: ~8-9 minutes for 16 databases -- ✅ Single restore: ~20 minutes for 7.3GB -- ✅ No disk space exhaustion - -### Reliability -- ✅ No crashes observed -- ✅ No data corruption -- ✅ Proper error messages -- ✅ Temp file cleanup working -- ✅ Process termination handled gracefully - ---- - -## Future Enhancements (Post v1.2.0) - -### High Priority (v1.3.0) -- [ ] Fix TUI Enter key on cluster restore -- [ ] Fix debug logging (--debug flag) -- [ ] Add progress bar for TUI operations -- [ ] Improve error messages for common scenarios - -### Medium Priority (v1.4.0) -- [ ] Automated integration test suite -- [ ] Backup encryption support -- [ ] Incremental backup support -- [ ] Remote backup destinations (S3, FTP, etc.) -- [ ] Backup scheduling (cron integration) - -### Low Priority (v2.0.0) -- [ ] MySQL/MariaDB full support -- [ ] Web UI for monitoring -- [ ] Backup verification/checksums -- [ ] Differential backups -- [ ] Multi-database restore with selection - ---- - -## Conclusion - -### Production Readiness: ✅ APPROVED - -**Version 1.2.0 is production-ready** with the following qualifications: - -**Strengths:** -- Critical disk space bug fixed -- Comprehensive test coverage documented -- Automated testing in place -- Cross-platform binaries available -- Complete documentation - -**Minor Issues (Non-Blocking):** -- TUI Enter key bug (workaround available) -- Debug logging not functional (doesn't impact operations) - -**Recommendation:** -Deploy to production with confidence. Monitor first few backup cycles. Address minor issues in next release cycle. - ---- - -## Test Plan Maintenance - -### When to Update Test Plan -- Before each major release -- After any critical bug fix -- When adding new features -- When deprecating features -- After production incidents - -### Test Plan Versioning -- v1.0: Initial comprehensive plan (this document) -- Future: Track changes in git - -### Continuous Improvement -- Add test cases for any reported bugs -- Update test data as needed -- Refine time estimates -- Add automation where possible - ---- - -**Document Version:** 1.0 -**Created:** November 11, 2025 -**Author:** GitHub Copilot AI Assistant -**Status:** ✅ COMPLETE -**Next Review:** Before v1.3.0 release - ---- - -## Quick Links - -- [MASTER_TEST_PLAN.md](./MASTER_TEST_PLAN.md) - Full 700+ line test plan -- [run_master_tests.sh](./run_master_tests.sh) - Automated CLI test suite -- [production_validation.sh](./production_validation.sh) - Full validation script -- [RELEASE_v1.2.0.md](./RELEASE_v1.2.0.md) - Release notes -- [PRODUCTION_TESTING_PLAN.md](./PRODUCTION_TESTING_PLAN.md) - Original testing plan -- [README.md](./README.md) - User documentation - -**END OF DOCUMENT** diff --git a/create_50gb_db_optimized.sh b/create_50gb_db_optimized.sh deleted file mode 100755 index 344aa90..0000000 --- a/create_50gb_db_optimized.sh +++ /dev/null @@ -1,255 +0,0 @@ -#!/bin/bash - -# Optimized Large Database Creator - 50GB target -# More efficient approach using PostgreSQL's built-in functions - -set -e - -DB_NAME="testdb_50gb" -TARGET_SIZE_GB=50 - -echo "==================================================" -echo "OPTIMIZED Large Test Database Creator" -echo "Database: $DB_NAME" -echo "Target Size: ${TARGET_SIZE_GB}GB" -echo "==================================================" - -# Check available space -AVAILABLE_GB=$(df / | tail -1 | awk '{print int($4/1024/1024)}') -echo "Available disk space: ${AVAILABLE_GB}GB" - -if [ $AVAILABLE_GB -lt $((TARGET_SIZE_GB + 20)) ]; then - echo "❌ ERROR: Insufficient disk space. Need at least $((TARGET_SIZE_GB + 20))GB buffer" - exit 1 -fi - -echo "✅ Sufficient disk space available" - -echo "" -echo "1. Creating optimized database schema..." - -# Drop and recreate database -sudo -u postgres psql -c "DROP DATABASE IF EXISTS $DB_NAME;" 2>/dev/null || true -sudo -u postgres psql -c "CREATE DATABASE $DB_NAME;" - -# Create optimized schema for rapid data generation -sudo -u postgres psql -d $DB_NAME << 'EOF' --- Large blob table with efficient storage -CREATE TABLE mega_blobs ( - id BIGSERIAL PRIMARY KEY, - chunk_id INTEGER NOT NULL, - blob_data BYTEA NOT NULL, - created_at TIMESTAMP DEFAULT NOW() -); - --- Massive text table for document storage -CREATE TABLE big_documents ( - id BIGSERIAL PRIMARY KEY, - doc_name VARCHAR(100), - content TEXT NOT NULL, - metadata JSONB, - created_at TIMESTAMP DEFAULT NOW() -); - --- High-volume metrics table -CREATE TABLE huge_metrics ( - id BIGSERIAL PRIMARY KEY, - timestamp TIMESTAMP NOT NULL, - sensor_id INTEGER NOT NULL, - metric_type VARCHAR(50) NOT NULL, - value_data TEXT NOT NULL, -- Large text field - binary_payload BYTEA, - created_at TIMESTAMP DEFAULT NOW() -); - --- Indexes for realism -CREATE INDEX idx_mega_blobs_chunk ON mega_blobs(chunk_id); -CREATE INDEX idx_big_docs_name ON big_documents(doc_name); -CREATE INDEX idx_huge_metrics_timestamp ON huge_metrics(timestamp); -CREATE INDEX idx_huge_metrics_sensor ON huge_metrics(sensor_id); -EOF - -echo "✅ Optimized schema created" - -echo "" -echo "2. Generating large-scale data using PostgreSQL's generate_series..." - -# Strategy: Use PostgreSQL's efficient bulk operations -echo "Inserting massive text documents (targeting ~20GB)..." - -sudo -u postgres psql -d $DB_NAME << 'EOF' --- Insert 2 million large text documents (~20GB estimated) -INSERT INTO big_documents (doc_name, content, metadata) -SELECT - 'doc_' || generate_series, - -- Each document: ~10KB of text content - repeat('Lorem ipsum dolor sit amet, consectetur adipiscing elit. ' || - 'Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. ' || - 'Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris. ' || - 'Duis aute irure dolor in reprehenderit in voluptate velit esse cillum. ' || - 'Excepteur sint occaecat cupidatat non proident, sunt in culpa qui. ' || - 'Nulla pariatur. Sed ut perspiciatis unde omnis iste natus error sit. ' || - 'At vero eos et accusamus et iusto odio dignissimos ducimus qui blanditiis. ' || - 'Document content section ' || generate_series || '. ', 50), - ('{"doc_type": "test", "size_category": "large", "batch": ' || (generate_series / 10000) || - ', "tags": ["bulk_data", "test_doc", "large_dataset"]}')::jsonb -FROM generate_series(1, 2000000); -EOF - -echo "✅ Large documents inserted" - -# Check current size -CURRENT_SIZE=$(sudo -u postgres psql -d $DB_NAME -tAc "SELECT pg_database_size('$DB_NAME') / 1024 / 1024 / 1024.0;" 2>/dev/null) -echo "Current database size: ${CURRENT_SIZE}GB" - -echo "Inserting high-volume metrics data (targeting additional ~15GB)..." - -sudo -u postgres psql -d $DB_NAME << 'EOF' --- Insert 5 million metrics records with large payloads (~15GB estimated) -INSERT INTO huge_metrics (timestamp, sensor_id, metric_type, value_data, binary_payload) -SELECT - NOW() - (generate_series * INTERVAL '1 second'), - generate_series % 10000, -- 10,000 different sensors - CASE (generate_series % 5) - WHEN 0 THEN 'temperature' - WHEN 1 THEN 'humidity' - WHEN 2 THEN 'pressure' - WHEN 3 THEN 'vibration' - ELSE 'electromagnetic' - END, - -- Large JSON-like text payload (~3KB each) - '{"readings": [' || - '{"timestamp": "' || (NOW() - (generate_series * INTERVAL '1 second'))::text || - '", "value": ' || (random() * 1000)::int || - ', "quality": "good", "metadata": "' || repeat('data_', 20) || '"},' || - '{"timestamp": "' || (NOW() - ((generate_series + 1) * INTERVAL '1 second'))::text || - '", "value": ' || (random() * 1000)::int || - ', "quality": "good", "metadata": "' || repeat('data_', 20) || '"},' || - '{"timestamp": "' || (NOW() - ((generate_series + 2) * INTERVAL '1 second'))::text || - '", "value": ' || (random() * 1000)::int || - ', "quality": "good", "metadata": "' || repeat('data_', 20) || '"}' || - '], "sensor_info": "' || repeat('sensor_metadata_', 30) || - '", "calibration": "' || repeat('calibration_data_', 25) || '"}', - -- Binary payload (~1KB each) - decode(encode(repeat('BINARY_SENSOR_DATA_CHUNK_', 25)::bytea, 'base64'), 'base64') -FROM generate_series(1, 5000000); -EOF - -echo "✅ Metrics data inserted" - -# Check size again -CURRENT_SIZE=$(sudo -u postgres psql -d $DB_NAME -tAc "SELECT pg_database_size('$DB_NAME') / 1024 / 1024 / 1024.0;" 2>/dev/null) -echo "Current database size: ${CURRENT_SIZE}GB" - -echo "Inserting binary blob data to reach 50GB target..." - -# Calculate remaining size needed -REMAINING_GB=$(echo "$TARGET_SIZE_GB - $CURRENT_SIZE" | bc -l 2>/dev/null || echo "15") -REMAINING_MB=$(echo "$REMAINING_GB * 1024" | bc -l 2>/dev/null || echo "15360") - -echo "Need approximately ${REMAINING_GB}GB more data..." - -# Insert binary blobs to fill remaining space -sudo -u postgres psql -d $DB_NAME << EOF --- Insert large binary chunks to reach target size --- Each blob will be approximately 5MB -INSERT INTO mega_blobs (chunk_id, blob_data) -SELECT - generate_series, - -- Generate ~5MB of binary data per row - decode(encode(repeat('LARGE_BINARY_CHUNK_FOR_TESTING_PURPOSES_', 100000)::bytea, 'base64'), 'base64') -FROM generate_series(1, ${REMAINING_MB%.*} / 5); -EOF - -echo "✅ Binary blob data inserted" - -echo "" -echo "3. Final optimization and statistics..." - -# Analyze tables for accurate statistics -sudo -u postgres psql -d $DB_NAME << 'EOF' --- Update table statistics -ANALYZE big_documents; -ANALYZE huge_metrics; -ANALYZE mega_blobs; - --- Vacuum to optimize storage -VACUUM ANALYZE; -EOF - -echo "" -echo "4. Final database metrics..." - -sudo -u postgres psql -d $DB_NAME << 'EOF' --- Database size breakdown -SELECT - 'TOTAL DATABASE SIZE' as component, - pg_size_pretty(pg_database_size(current_database())) as size, - ROUND(pg_database_size(current_database()) / 1024.0 / 1024.0 / 1024.0, 2) || ' GB' as size_gb -UNION ALL -SELECT - 'big_documents table', - pg_size_pretty(pg_total_relation_size('big_documents')), - ROUND(pg_total_relation_size('big_documents') / 1024.0 / 1024.0 / 1024.0, 2) || ' GB' -UNION ALL -SELECT - 'huge_metrics table', - pg_size_pretty(pg_total_relation_size('huge_metrics')), - ROUND(pg_total_relation_size('huge_metrics') / 1024.0 / 1024.0 / 1024.0, 2) || ' GB' -UNION ALL -SELECT - 'mega_blobs table', - pg_size_pretty(pg_total_relation_size('mega_blobs')), - ROUND(pg_total_relation_size('mega_blobs') / 1024.0 / 1024.0 / 1024.0, 2) || ' GB'; - --- Row counts -SELECT - 'TABLE ROWS' as metric, - '' as value, - '' as extra -UNION ALL -SELECT - 'big_documents', - COUNT(*)::text, - 'rows' -FROM big_documents -UNION ALL -SELECT - 'huge_metrics', - COUNT(*)::text, - 'rows' -FROM huge_metrics -UNION ALL -SELECT - 'mega_blobs', - COUNT(*)::text, - 'rows' -FROM mega_blobs; -EOF - -FINAL_SIZE=$(sudo -u postgres psql -d $DB_NAME -tAc "SELECT pg_size_pretty(pg_database_size('$DB_NAME'));" 2>/dev/null) -FINAL_GB=$(sudo -u postgres psql -d $DB_NAME -tAc "SELECT ROUND(pg_database_size('$DB_NAME') / 1024.0 / 1024.0 / 1024.0, 2);" 2>/dev/null) - -echo "" -echo "==================================================" -echo "✅ LARGE DATABASE CREATION COMPLETED!" -echo "==================================================" -echo "Database Name: $DB_NAME" -echo "Final Size: $FINAL_SIZE (${FINAL_GB}GB)" -echo "Target: ${TARGET_SIZE_GB}GB" -echo "==================================================" - -echo "" -echo "🧪 Ready for testing large database operations:" -echo "" -echo "# Test single database backup:" -echo "time sudo -u postgres ./dbbackup backup single $DB_NAME --confirm" -echo "" -echo "# Test cluster backup (includes this large DB):" -echo "time sudo -u postgres ./dbbackup backup cluster --confirm" -echo "" -echo "# Monitor backup progress:" -echo "watch 'ls -lah /backup/ 2>/dev/null || ls -lah ./*.dump* ./*.tar.gz 2>/dev/null'" -echo "" -echo "# Check database size anytime:" -echo "sudo -u postgres psql -d $DB_NAME -c \"SELECT pg_size_pretty(pg_database_size('$DB_NAME'));\"" \ No newline at end of file diff --git a/create_large_test_db.sh b/create_large_test_db.sh deleted file mode 100755 index 0aa34f2..0000000 --- a/create_large_test_db.sh +++ /dev/null @@ -1,243 +0,0 @@ -#!/bin/bash - -# Large Test Database Creator - 50GB with Blobs -# Creates a substantial database for testing backup/restore performance on large datasets - -set -e - -DB_NAME="testdb_large_50gb" -TARGET_SIZE_GB=50 -CHUNK_SIZE_MB=10 # Size of each blob chunk in MB -TOTAL_CHUNKS=$((TARGET_SIZE_GB * 1024 / CHUNK_SIZE_MB)) # Total number of chunks needed - -echo "==================================================" -echo "Creating Large Test Database: $DB_NAME" -echo "Target Size: ${TARGET_SIZE_GB}GB" -echo "Chunk Size: ${CHUNK_SIZE_MB}MB" -echo "Total Chunks: $TOTAL_CHUNKS" -echo "==================================================" - -# Check available space -AVAILABLE_GB=$(df / | tail -1 | awk '{print int($4/1024/1024)}') -echo "Available disk space: ${AVAILABLE_GB}GB" - -if [ $AVAILABLE_GB -lt $((TARGET_SIZE_GB + 10)) ]; then - echo "❌ ERROR: Insufficient disk space. Need at least $((TARGET_SIZE_GB + 10))GB" - exit 1 -fi - -echo "✅ Sufficient disk space available" - -# Database connection settings -PGUSER="postgres" -PGHOST="localhost" -PGPORT="5432" - -echo "" -echo "1. Creating database and schema..." - -# Drop and recreate database -sudo -u postgres psql -c "DROP DATABASE IF EXISTS $DB_NAME;" 2>/dev/null || true -sudo -u postgres psql -c "CREATE DATABASE $DB_NAME;" - -# Create tables with different data types -sudo -u postgres psql -d $DB_NAME << 'EOF' --- Table for large binary objects (blobs) -CREATE TABLE large_blobs ( - id SERIAL PRIMARY KEY, - name VARCHAR(255), - description TEXT, - blob_data BYTEA, - created_at TIMESTAMP DEFAULT NOW(), - size_mb INTEGER -); - --- Table for structured data with indexes -CREATE TABLE test_data ( - id SERIAL PRIMARY KEY, - user_id INTEGER NOT NULL, - username VARCHAR(100) NOT NULL, - email VARCHAR(255) NOT NULL, - profile_data JSONB, - large_text TEXT, - random_number NUMERIC(15,2), - created_at TIMESTAMP DEFAULT NOW() -); - --- Table for time series data (lots of rows) -CREATE TABLE metrics ( - id BIGSERIAL PRIMARY KEY, - timestamp TIMESTAMP NOT NULL, - metric_name VARCHAR(100) NOT NULL, - value DOUBLE PRECISION NOT NULL, - tags JSONB, - metadata TEXT -); - --- Indexes for performance -CREATE INDEX idx_test_data_user_id ON test_data(user_id); -CREATE INDEX idx_test_data_email ON test_data(email); -CREATE INDEX idx_test_data_created ON test_data(created_at); -CREATE INDEX idx_metrics_timestamp ON metrics(timestamp); -CREATE INDEX idx_metrics_name ON metrics(metric_name); -CREATE INDEX idx_metrics_tags ON metrics USING GIN(tags); - --- Large text table for document storage -CREATE TABLE documents ( - id SERIAL PRIMARY KEY, - title VARCHAR(500), - content TEXT, - document_data BYTEA, - tags TEXT[], - created_at TIMESTAMP DEFAULT NOW() -); - -CREATE INDEX idx_documents_tags ON documents USING GIN(tags); -EOF - -echo "✅ Database schema created" - -echo "" -echo "2. Generating large blob data..." - -# Function to generate random data -generate_blob_data() { - local chunk_num=$1 - local size_mb=$2 - - # Generate random binary data using dd and base64 - dd if=/dev/urandom bs=1M count=$size_mb 2>/dev/null | base64 -w 0 -} - -echo "Inserting $TOTAL_CHUNKS blob chunks of ${CHUNK_SIZE_MB}MB each..." - -# Insert blob data in chunks -for i in $(seq 1 $TOTAL_CHUNKS); do - echo -n " Progress: $i/$TOTAL_CHUNKS ($(($i * 100 / $TOTAL_CHUNKS))%) - " - - # Generate blob data - BLOB_DATA=$(generate_blob_data $i $CHUNK_SIZE_MB) - - # Insert into database - sudo -u postgres psql -d $DB_NAME -c " - INSERT INTO large_blobs (name, description, blob_data, size_mb) - VALUES ( - 'blob_chunk_$i', - 'Large binary data chunk $i of $TOTAL_CHUNKS for testing backup/restore performance', - decode('$BLOB_DATA', 'base64'), - $CHUNK_SIZE_MB - );" > /dev/null - - echo "✅ Chunk $i inserted" - - # Every 10 chunks, show current database size - if [ $((i % 10)) -eq 0 ]; then - CURRENT_SIZE=$(sudo -u postgres psql -d $DB_NAME -tAc " - SELECT pg_size_pretty(pg_database_size('$DB_NAME'));" 2>/dev/null || echo "Unknown") - echo " Current database size: $CURRENT_SIZE" - fi -done - -echo "" -echo "3. Generating structured test data..." - -# Insert large amounts of structured data -sudo -u postgres psql -d $DB_NAME << 'EOF' --- Insert 1 million rows of test data (will add significant size) -INSERT INTO test_data (user_id, username, email, profile_data, large_text, random_number) -SELECT - generate_series % 100000 as user_id, - 'user_' || generate_series as username, - 'user_' || generate_series || '@example.com' as email, - ('{"preferences": {"theme": "dark", "language": "en", "notifications": true}, "metadata": {"last_login": "2024-01-01", "session_count": ' || (generate_series % 1000) || ', "data": "' || repeat('x', 100) || '"}}')::jsonb as profile_data, - repeat('This is large text content for testing. ', 50) || ' Row: ' || generate_series as large_text, - random() * 1000000 as random_number -FROM generate_series(1, 1000000); - --- Insert time series data (2 million rows) -INSERT INTO metrics (timestamp, metric_name, value, tags, metadata) -SELECT - NOW() - (generate_series || ' minutes')::interval as timestamp, - CASE (generate_series % 5) - WHEN 0 THEN 'cpu_usage' - WHEN 1 THEN 'memory_usage' - WHEN 2 THEN 'disk_io' - WHEN 3 THEN 'network_tx' - ELSE 'network_rx' - END as metric_name, - random() * 100 as value, - ('{"host": "server_' || (generate_series % 100) || '", "env": "' || - CASE (generate_series % 3) WHEN 0 THEN 'prod' WHEN 1 THEN 'staging' ELSE 'dev' END || - '", "region": "us-' || CASE (generate_series % 2) WHEN 0 THEN 'east' ELSE 'west' END || '"}')::jsonb as tags, - 'Generated metric data for testing - ' || repeat('metadata_', 10) as metadata -FROM generate_series(1, 2000000); - --- Insert document data with embedded binary content -INSERT INTO documents (title, content, document_data, tags) -SELECT - 'Document ' || generate_series as title, - repeat('This is document content with lots of text to increase database size. ', 100) || - ' Document ID: ' || generate_series || '. ' || - repeat('Additional content to make documents larger. ', 20) as content, - decode(encode(('Binary document data for doc ' || generate_series || ': ' || repeat('BINARY_DATA_', 1000))::bytea, 'base64'), 'base64') as document_data, - ARRAY['tag_' || (generate_series % 10), 'category_' || (generate_series % 5), 'type_document'] as tags -FROM generate_series(1, 100000); -EOF - -echo "✅ Structured data inserted" - -echo "" -echo "4. Final database statistics..." - -# Get final database size and statistics -sudo -u postgres psql -d $DB_NAME << 'EOF' -SELECT - 'Database Size' as metric, - pg_size_pretty(pg_database_size(current_database())) as value -UNION ALL -SELECT - 'Table: large_blobs', - pg_size_pretty(pg_total_relation_size('large_blobs')) -UNION ALL -SELECT - 'Table: test_data', - pg_size_pretty(pg_total_relation_size('test_data')) -UNION ALL -SELECT - 'Table: metrics', - pg_size_pretty(pg_total_relation_size('metrics')) -UNION ALL -SELECT - 'Table: documents', - pg_size_pretty(pg_total_relation_size('documents')); - --- Row counts -SELECT 'large_blobs rows' as table_name, COUNT(*) as row_count FROM large_blobs -UNION ALL -SELECT 'test_data rows', COUNT(*) FROM test_data -UNION ALL -SELECT 'metrics rows', COUNT(*) FROM metrics -UNION ALL -SELECT 'documents rows', COUNT(*) FROM documents; -EOF - -echo "" -echo "==================================================" -echo "✅ Large test database creation completed!" -echo "Database: $DB_NAME" -echo "==================================================" - -# Show final size -FINAL_SIZE=$(sudo -u postgres psql -d $DB_NAME -tAc "SELECT pg_size_pretty(pg_database_size('$DB_NAME'));" 2>/dev/null) -echo "Final database size: $FINAL_SIZE" - -echo "" -echo "You can now test backup/restore operations:" -echo " # Backup the large database" -echo " sudo -u postgres ./dbbackup backup single $DB_NAME" -echo "" -echo " # Backup entire cluster (including this large DB)" -echo " sudo -u postgres ./dbbackup backup cluster" -echo "" -echo " # Check database size anytime:" -echo " sudo -u postgres psql -d $DB_NAME -c \"SELECT pg_size_pretty(pg_database_size('$DB_NAME'));\"" \ No newline at end of file diff --git a/create_massive_50gb_db.sh b/create_massive_50gb_db.sh deleted file mode 100755 index a0d3e56..0000000 --- a/create_massive_50gb_db.sh +++ /dev/null @@ -1,165 +0,0 @@ -#!/bin/bash - -# Aggressive 50GB Database Creator -# Specifically designed to reach exactly 50GB - -set -e - -DB_NAME="testdb_massive_50gb" -TARGET_SIZE_GB=50 - -echo "==================================================" -echo "AGGRESSIVE 50GB Database Creator" -echo "Database: $DB_NAME" -echo "Target Size: ${TARGET_SIZE_GB}GB" -echo "==================================================" - -# Check available space -AVAILABLE_GB=$(df / | tail -1 | awk '{print int($4/1024/1024)}') -echo "Available disk space: ${AVAILABLE_GB}GB" - -if [ $AVAILABLE_GB -lt $((TARGET_SIZE_GB + 20)) ]; then - echo "❌ ERROR: Insufficient disk space. Need at least $((TARGET_SIZE_GB + 20))GB buffer" - exit 1 -fi - -echo "✅ Sufficient disk space available" - -echo "" -echo "1. Creating database for massive data..." - -# Drop and recreate database -sudo -u postgres psql -c "DROP DATABASE IF EXISTS $DB_NAME;" 2>/dev/null || true -sudo -u postgres psql -c "CREATE DATABASE $DB_NAME;" - -# Create simple table optimized for massive data -sudo -u postgres psql -d $DB_NAME << 'EOF' --- Single massive table with large binary columns -CREATE TABLE massive_data ( - id BIGSERIAL PRIMARY KEY, - large_text TEXT NOT NULL, - binary_chunk BYTEA NOT NULL, - created_at TIMESTAMP DEFAULT NOW() -); - --- Index for basic functionality -CREATE INDEX idx_massive_data_id ON massive_data(id); -EOF - -echo "✅ Database schema created" - -echo "" -echo "2. Inserting massive data in chunks..." - -# Calculate how many rows we need for 50GB -# Strategy: Each row will be approximately 10MB -# 50GB = 50,000MB, so we need about 5,000 rows of 10MB each - -CHUNK_SIZE_MB=10 -TOTAL_CHUNKS=$((TARGET_SIZE_GB * 1024 / CHUNK_SIZE_MB)) # 5,120 chunks for 50GB - -echo "Inserting $TOTAL_CHUNKS chunks of ${CHUNK_SIZE_MB}MB each..." - -for i in $(seq 1 $TOTAL_CHUNKS); do - # Progress indicator - if [ $((i % 100)) -eq 0 ] || [ $i -le 10 ]; then - CURRENT_SIZE=$(sudo -u postgres psql -d $DB_NAME -tAc "SELECT ROUND(pg_database_size('$DB_NAME') / 1024.0 / 1024.0 / 1024.0, 2);" 2>/dev/null || echo "0") - echo " Progress: $i/$TOTAL_CHUNKS ($(($i * 100 / $TOTAL_CHUNKS))%) - Current size: ${CURRENT_SIZE}GB" - - # Check if we've reached target - if (( $(echo "$CURRENT_SIZE >= $TARGET_SIZE_GB" | bc -l 2>/dev/null || echo "0") )); then - echo "✅ Target size reached! Stopping at chunk $i" - break - fi - fi - - # Insert chunk with large data - sudo -u postgres psql -d $DB_NAME << EOF > /dev/null -INSERT INTO massive_data (large_text, binary_chunk) -VALUES ( - -- Large text component (~5MB as text) - repeat('This is a large text chunk for testing massive database operations. It contains repeated content to reach the target size for backup and restore performance testing. Row: $i of $TOTAL_CHUNKS. ', 25000), - -- Large binary component (~5MB as binary) - decode(encode(repeat('MASSIVE_BINARY_DATA_CHUNK_FOR_TESTING_DATABASE_BACKUP_RESTORE_PERFORMANCE_ON_LARGE_DATASETS_ROW_${i}_OF_${TOTAL_CHUNKS}_', 25000)::bytea, 'base64'), 'base64') -); -EOF - - # Every 500 chunks, run VACUUM to prevent excessive table bloat - if [ $((i % 500)) -eq 0 ]; then - echo " Running maintenance (VACUUM) at chunk $i..." - sudo -u postgres psql -d $DB_NAME -c "VACUUM massive_data;" > /dev/null - fi -done - -echo "" -echo "3. Final optimization..." - -sudo -u postgres psql -d $DB_NAME << 'EOF' --- Final optimization -VACUUM ANALYZE massive_data; - --- Update statistics -ANALYZE; -EOF - -echo "" -echo "4. Final database metrics..." - -sudo -u postgres psql -d $DB_NAME << 'EOF' --- Database size and statistics -SELECT - 'Database Size' as metric, - pg_size_pretty(pg_database_size(current_database())) as value, - ROUND(pg_database_size(current_database()) / 1024.0 / 1024.0 / 1024.0, 2) || ' GB' as size_gb; - -SELECT - 'Table Size' as metric, - pg_size_pretty(pg_total_relation_size('massive_data')) as value, - ROUND(pg_total_relation_size('massive_data') / 1024.0 / 1024.0 / 1024.0, 2) || ' GB' as size_gb; - -SELECT - 'Row Count' as metric, - COUNT(*)::text as value, - 'rows' as unit -FROM massive_data; - -SELECT - 'Average Row Size' as metric, - pg_size_pretty(pg_total_relation_size('massive_data') / GREATEST(COUNT(*), 1)) as value, - 'per row' as unit -FROM massive_data; -EOF - -FINAL_SIZE=$(sudo -u postgres psql -d $DB_NAME -tAc "SELECT pg_size_pretty(pg_database_size('$DB_NAME'));" 2>/dev/null) -FINAL_GB=$(sudo -u postgres psql -d $DB_NAME -tAc "SELECT ROUND(pg_database_size('$DB_NAME') / 1024.0 / 1024.0 / 1024.0, 2);" 2>/dev/null) - -echo "" -echo "==================================================" -echo "✅ MASSIVE DATABASE CREATION COMPLETED!" -echo "==================================================" -echo "Database Name: $DB_NAME" -echo "Final Size: $FINAL_SIZE (${FINAL_GB}GB)" -echo "Target: ${TARGET_SIZE_GB}GB" - -if (( $(echo "$FINAL_GB >= $TARGET_SIZE_GB" | bc -l 2>/dev/null || echo "0") )); then - echo "🎯 TARGET ACHIEVED! Database is >= ${TARGET_SIZE_GB}GB" -else - echo "⚠️ Target not fully reached, but substantial database created" -fi - -echo "==================================================" - -echo "" -echo "🧪 Ready for LARGE DATABASE testing:" -echo "" -echo "# Test single database backup (will take significant time):" -echo "time sudo -u postgres ./dbbackup backup single $DB_NAME --confirm" -echo "" -echo "# Test cluster backup (includes this massive DB):" -echo "time sudo -u postgres ./dbbackup backup cluster --confirm" -echo "" -echo "# Monitor system resources during backup:" -echo "watch 'free -h && df -h && ls -lah *.dump* *.tar.gz 2>/dev/null'" -echo "" -echo "# Check database size anytime:" -echo "sudo -u postgres psql -d $DB_NAME -c \"SELECT pg_size_pretty(pg_database_size('$DB_NAME'));\"" \ No newline at end of file diff --git a/privilege_diagnostic.sh b/privilege_diagnostic.sh deleted file mode 100755 index da45e77..0000000 --- a/privilege_diagnostic.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/bash -# -# Database Privilege Diagnostic Script -# Run this on both hosts to compare privilege states -# - -echo "==============================================" -echo "Database Privilege Diagnostic Report" -echo "Host: $(hostname)" -echo "Date: $(date)" -echo "User: $(whoami)" -echo "==============================================" - -echo "" -echo "1. DATABASE LIST WITH PRIVILEGES:" -echo "==================================" -sudo -u postgres psql -c "\l" - -echo "" -echo "2. DATABASE PRIVILEGES (Detailed):" -echo "==================================" -sudo -u postgres psql -c " -SELECT - datname as database_name, - datacl as access_privileges, - datdba::regrole as owner -FROM pg_database -WHERE datname NOT IN ('template0', 'template1') -ORDER BY datname; -" - -echo "" -echo "3. ROLE/USER LIST:" -echo "==================" -sudo -u postgres psql -c "\du" - -echo "" -echo "4. DATABASE-SPECIFIC GRANTS:" -echo "============================" -for db in $(sudo -u postgres psql -tAc "SELECT datname FROM pg_database WHERE datname NOT IN ('template0', 'template1', 'postgres')"); do - echo "--- Database: $db ---" - sudo -u postgres psql -d "$db" -c " - SELECT - schemaname, - tablename, - tableowner, - tablespace - FROM pg_tables - WHERE schemaname = 'public' - LIMIT 5; - " 2>/dev/null || echo "Could not connect to $db" -done - -echo "" -echo "5. GLOBAL OBJECT PRIVILEGES:" -echo "============================" -sudo -u postgres psql -c " -SELECT - rolname, - rolsuper, - rolcreaterole, - rolcreatedb, - rolcanlogin -FROM pg_roles -WHERE rolname NOT LIKE 'pg_%' -ORDER BY rolname; -" - -echo "" -echo "6. CHECK globals.sql CONTENT (if exists):" -echo "========================================" -LATEST_CLUSTER=$(find /var/lib/pgsql/db_backups -name "cluster_*.tar.gz" -type f -printf '%T@ %p\n' 2>/dev/null | sort -n | tail -1 | cut -d' ' -f2-) -if [ -n "$LATEST_CLUSTER" ]; then - echo "Latest cluster backup: $LATEST_CLUSTER" - TEMP_DIR="/tmp/privilege_check_$$" - mkdir -p "$TEMP_DIR" - tar -xzf "$LATEST_CLUSTER" -C "$TEMP_DIR" 2>/dev/null - if [ -f "$TEMP_DIR/globals.sql" ]; then - echo "globals.sql content:" - echo "===================" - head -50 "$TEMP_DIR/globals.sql" - echo "" - echo "... (showing first 50 lines, check full file if needed)" - echo "" - echo "Database creation commands in globals.sql:" - grep -i "CREATE DATABASE\|GRANT.*DATABASE" "$TEMP_DIR/globals.sql" || echo "No database grants found" - else - echo "No globals.sql found in backup" - fi - rm -rf "$TEMP_DIR" -else - echo "No cluster backup found to examine" -fi - -echo "" -echo "==============================================" -echo "Diagnostic complete. Save this output and" -echo "compare between hosts to identify differences." -echo "==============================================" \ No newline at end of file diff --git a/privilege_report_testhost.txt b/privilege_report_testhost.txt deleted file mode 100644 index 2bcf189..0000000 --- a/privilege_report_testhost.txt +++ /dev/null @@ -1,216 +0,0 @@ -============================================== -Database Privilege Diagnostic Report -Host: psqldb -Date: Tue Nov 11 08:26:07 AM UTC 2025 -User: root -============================================== - -1. DATABASE LIST WITH PRIVILEGES: -================================== - List of databases - Name | Owner | Encoding | Locale Provider | Collate | Ctype | ICU Locale | ICU Rules | Access privileges --------------------------+----------+----------+-----------------+-------------+-------------+------------+-----------+----------------------- - backup_test_db | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | - cli_test_db | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | - cluster_restore_test | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | - final_test_db | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | - large_test_db | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | - menu_test_db | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | - ownership_test | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | - perfect_test_db | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | - postgres | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | - restored_ownership_test | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | - template0 | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | =c/postgres + - | | | | | | | | postgres=CTc/postgres - template1 | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | =c/postgres + - | | | | | | | | postgres=CTc/postgres - test_restore_timing | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | - test_sample_backup | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | - test_single_backup | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | - timing_test_db | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | - ultimate_test_db | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | -(17 rows) - - -2. DATABASE PRIVILEGES (Detailed): -================================== - database_name | access_privileges | owner --------------------------+-------------------+---------- - backup_test_db | | postgres - cli_test_db | | postgres - cluster_restore_test | | postgres - final_test_db | | postgres - large_test_db | | postgres - menu_test_db | | postgres - ownership_test | | postgres - perfect_test_db | | postgres - postgres | | postgres - restored_ownership_test | | postgres - test_restore_timing | | postgres - test_sample_backup | | postgres - test_single_backup | | postgres - timing_test_db | | postgres - ultimate_test_db | | postgres -(15 rows) - - -3. ROLE/USER LIST: -================== - List of roles - Role name | Attributes ------------+------------------------------------------------------------ - postgres | Superuser, Create role, Create DB, Replication, Bypass RLS - testowner | - - -4. DATABASE-SPECIFIC GRANTS: -============================ ---- Database: ultimate_test_db --- - schemaname | tablename | tableowner | tablespace -------------+-----------+------------+------------ - public | test_data | postgres | -(1 row) - ---- Database: backup_test_db --- - schemaname | tablename | tableowner | tablespace -------------+------------+------------+------------ - public | users | postgres | - public | audit_log | postgres | - public | documents | postgres | - public | user_files | postgres | - public | images | postgres | -(5 rows) - ---- Database: cli_test_db --- - schemaname | tablename | tableowner | tablespace -------------+------------+------------+------------ - public | test_table | postgres | -(1 row) - ---- Database: cluster_restore_test --- - schemaname | tablename | tableowner | tablespace -------------+-----------+------------+------------ -(0 rows) - ---- Database: final_test_db --- - schemaname | tablename | tableowner | tablespace -------------+------------+------------+------------ - public | test_table | postgres | -(1 row) - ---- Database: large_test_db --- - schemaname | tablename | tableowner | tablespace -------------+------------------+------------+------------ - public | large_test_table | postgres | -(1 row) - ---- Database: menu_test_db --- - schemaname | tablename | tableowner | tablespace -------------+------------+------------+------------ - public | test_table | postgres | -(1 row) - ---- Database: ownership_test --- - schemaname | tablename | tableowner | tablespace -------------+-----------+------------+------------ - public | test_data | testowner | -(1 row) - ---- Database: perfect_test_db --- - schemaname | tablename | tableowner | tablespace -------------+-----------+------------+------------ - public | test_data | postgres | -(1 row) - ---- Database: restored_ownership_test --- - schemaname | tablename | tableowner | tablespace -------------+-----------+------------+------------ - public | test_data | postgres | -(1 row) - ---- Database: test_restore_timing --- - schemaname | tablename | tableowner | tablespace -------------+------------+------------+------------ - public | test_table | postgres | -(1 row) - ---- Database: test_sample_backup --- - schemaname | tablename | tableowner | tablespace -------------+--------------+------------+------------ - public | sample_table | postgres | -(1 row) - ---- Database: test_single_backup --- - schemaname | tablename | tableowner | tablespace -------------+------------+------------+------------ - public | test_table | postgres | -(1 row) - ---- Database: timing_test_db --- - schemaname | tablename | tableowner | tablespace -------------+-------------------+------------+------------ - public | timing_test_table | postgres | -(1 row) - - -5. GLOBAL OBJECT PRIVILEGES: -============================ - rolname | rolsuper | rolcreaterole | rolcreatedb | rolcanlogin ------------+----------+---------------+-------------+------------- - postgres | t | t | t | t - testowner | f | f | f | t -(2 rows) - - -6. CHECK globals.sql CONTENT (if exists): -======================================== -Latest cluster backup: /var/lib/pgsql/db_backups/cluster_20251110_134826.tar.gz -globals.sql content: -=================== --- --- PostgreSQL database cluster dump --- - -\restrict sWNr7ksTDJbnJSKSJBd9MGA4t0POFSLcEqaGMSM1uwA3cEmyGaIpD0VJrmAKQjX - -SET default_transaction_read_only = off; - -SET client_encoding = 'UTF8'; -SET standard_conforming_strings = on; - --- --- Roles --- - -CREATE ROLE postgres; -ALTER ROLE postgres WITH SUPERUSER INHERIT CREATEROLE CREATEDB LOGIN REPLICATION BYPASSRLS PASSWORD 'SCRAM-SHA-256$4096:8CqV4BNYEk6/Au1ub4otRQ==$PhSfnKEs49UZ6g4CgnFbLlhvbcq5nSkS4RMP5MTqf7E=:xg+3j/oZIF1mbu6SydJbqLem9Bd+ONNK2JeftY7hbL4='; -CREATE ROLE testowner; -ALTER ROLE testowner WITH NOSUPERUSER INHERIT NOCREATEROLE NOCREATEDB LOGIN NOREPLICATION NOBYPASSRLS PASSWORD 'SCRAM-SHA-256$4096:3TGJ9Dl+y75j46aWS8NtQw==$2C7ebcOIj7vNoIFM54gtUZnjw/UR8h6BorF1g/MLKTQ=:YIMFknJmXGHxvR+rAN2eXtL7LS4ng+iDnqmFkffSsss='; - --- --- User Configurations --- - - - - - - - - -\unrestrict sWNr7ksTDJbnJSKSJBd9MGA4t0POFSLcEqaGMSM1uwA3cEmyGaIpD0VJrmAKQjX - --- --- PostgreSQL database cluster dump complete --- - - -... (showing first 50 lines, check full file if needed) - -Database creation commands in globals.sql: -No database grants found - -============================================== -Diagnostic complete. Save this output and -compare between hosts to identify differences. -============================================== diff --git a/production_validation.sh b/production_validation.sh deleted file mode 100755 index 4467a23..0000000 --- a/production_validation.sh +++ /dev/null @@ -1,477 +0,0 @@ -#!/bin/bash - -################################################################################ -# Production Validation Script for dbbackup -# -# This script performs comprehensive testing of all CLI commands and validates -# the system is ready for production release. -# -# Requirements: -# - PostgreSQL running locally with test databases -# - Disk space for backups -# - Run as user with sudo access or as postgres user -################################################################################ - -set -e # Exit on error -set -o pipefail - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Counters -TESTS_TOTAL=0 -TESTS_PASSED=0 -TESTS_FAILED=0 -TESTS_SKIPPED=0 - -# Configuration -DBBACKUP_BIN="./dbbackup" -TEST_BACKUP_DIR="/tmp/dbbackup_validation_$(date +%s)" -TEST_DB="postgres" -POSTGRES_USER="postgres" -LOG_FILE="/tmp/dbbackup_validation_$(date +%Y%m%d_%H%M%S).log" - -# Test results -declare -a FAILED_TESTS=() - -################################################################################ -# Helper Functions -################################################################################ - -print_header() { - echo "" - echo -e "${BLUE}========================================${NC}" - echo -e "${BLUE}$1${NC}" - echo -e "${BLUE}========================================${NC}" -} - -print_test() { - TESTS_TOTAL=$((TESTS_TOTAL + 1)) - echo -e "${YELLOW}[TEST $TESTS_TOTAL]${NC} $1" -} - -print_success() { - TESTS_PASSED=$((TESTS_PASSED + 1)) - echo -e " ${GREEN}✅ PASS${NC}: $1" -} - -print_failure() { - TESTS_FAILED=$((TESTS_FAILED + 1)) - FAILED_TESTS+=("$TESTS_TOTAL: $1") - echo -e " ${RED}❌ FAIL${NC}: $1" -} - -print_skip() { - TESTS_SKIPPED=$((TESTS_SKIPPED + 1)) - echo -e " ${YELLOW}⊘ SKIP${NC}: $1" -} - -run_as_postgres() { - if [ "$(whoami)" = "postgres" ]; then - "$@" - else - sudo -u postgres "$@" - fi -} - -cleanup_test_backups() { - rm -rf "$TEST_BACKUP_DIR" 2>/dev/null || true - mkdir -p "$TEST_BACKUP_DIR" -} - -################################################################################ -# Pre-Flight Checks -################################################################################ - -preflight_checks() { - print_header "Pre-Flight Checks" - - # Check binary exists - print_test "Check dbbackup binary exists" - if [ -f "$DBBACKUP_BIN" ]; then - print_success "Binary found: $DBBACKUP_BIN" - else - print_failure "Binary not found: $DBBACKUP_BIN" - exit 1 - fi - - # Check binary is executable - print_test "Check dbbackup is executable" - if [ -x "$DBBACKUP_BIN" ]; then - print_success "Binary is executable" - else - print_failure "Binary is not executable" - exit 1 - fi - - # Check PostgreSQL tools - print_test "Check PostgreSQL tools" - if command -v pg_dump >/dev/null 2>&1 && command -v pg_restore >/dev/null 2>&1; then - print_success "PostgreSQL tools available" - else - print_failure "PostgreSQL tools not found" - exit 1 - fi - - # Check PostgreSQL is running - print_test "Check PostgreSQL is running" - if run_as_postgres psql -d postgres -c "SELECT 1" >/dev/null 2>&1; then - print_success "PostgreSQL is running" - else - print_failure "PostgreSQL is not accessible" - exit 1 - fi - - # Check disk space - print_test "Check disk space" - available=$(df -BG "$TEST_BACKUP_DIR" 2>/dev/null | awk 'NR==2 {print $4}' | tr -d 'G') - if [ "$available" -gt 10 ]; then - print_success "Sufficient disk space: ${available}GB available" - else - print_failure "Insufficient disk space: only ${available}GB available (need 10GB+)" - fi - - # Check compression tools - print_test "Check compression tools" - if command -v pigz >/dev/null 2>&1; then - print_success "pigz (parallel gzip) available" - elif command -v gzip >/dev/null 2>&1; then - print_success "gzip available (pigz not found, will be slower)" - else - print_failure "No compression tools found" - fi -} - -################################################################################ -# CLI Command Tests -################################################################################ - -test_version_help() { - print_header "Basic CLI Tests" - - print_test "Test --version flag" - if run_as_postgres $DBBACKUP_BIN --version >/dev/null 2>&1; then - print_success "Version command works" - else - print_failure "Version command failed" - fi - - print_test "Test --help flag" - if run_as_postgres $DBBACKUP_BIN --help >/dev/null 2>&1; then - print_success "Help command works" - else - print_failure "Help command failed" - fi - - print_test "Test backup --help" - if run_as_postgres $DBBACKUP_BIN backup --help >/dev/null 2>&1; then - print_success "Backup help works" - else - print_failure "Backup help failed" - fi - - print_test "Test restore --help" - if run_as_postgres $DBBACKUP_BIN restore --help >/dev/null 2>&1; then - print_success "Restore help works" - else - print_failure "Restore help failed" - fi - - print_test "Test status --help" - if run_as_postgres $DBBACKUP_BIN status --help >/dev/null 2>&1; then - print_success "Status help works" - else - print_failure "Status help failed" - fi -} - -test_backup_single() { - print_header "Single Database Backup Tests" - - cleanup_test_backups - - # Test 1: Basic single database backup - print_test "Single DB backup (default compression)" - if run_as_postgres $DBBACKUP_BIN backup single "$TEST_DB" -d postgres --insecure \ - --backup-dir "$TEST_BACKUP_DIR" >>"$LOG_FILE" 2>&1; then - if ls "$TEST_BACKUP_DIR"/db_${TEST_DB}_*.dump >/dev/null 2>&1; then - size=$(ls -lh "$TEST_BACKUP_DIR"/db_${TEST_DB}_*.dump | awk '{print $5}') - print_success "Backup created: $size" - else - print_failure "Backup file not found" - fi - else - print_failure "Backup command failed" - fi - - # Test 2: Low compression backup - print_test "Single DB backup (low compression)" - if run_as_postgres $DBBACKUP_BIN backup single "$TEST_DB" -d postgres --insecure \ - --backup-dir "$TEST_BACKUP_DIR" --compression 1 >>"$LOG_FILE" 2>&1; then - print_success "Low compression backup succeeded" - else - print_failure "Low compression backup failed" - fi - - # Test 3: High compression backup - print_test "Single DB backup (high compression)" - if run_as_postgres $DBBACKUP_BIN backup single "$TEST_DB" -d postgres --insecure \ - --backup-dir "$TEST_BACKUP_DIR" --compression 9 >>"$LOG_FILE" 2>&1; then - print_success "High compression backup succeeded" - else - print_failure "High compression backup failed" - fi - - # Test 4: Custom backup directory - print_test "Single DB backup (custom directory)" - custom_dir="$TEST_BACKUP_DIR/custom" - mkdir -p "$custom_dir" - if run_as_postgres $DBBACKUP_BIN backup single "$TEST_DB" -d postgres --insecure \ - --backup-dir "$custom_dir" >>"$LOG_FILE" 2>&1; then - if ls "$custom_dir"/db_${TEST_DB}_*.dump >/dev/null 2>&1; then - print_success "Backup created in custom directory" - else - print_failure "Backup not found in custom directory" - fi - else - print_failure "Custom directory backup failed" - fi -} - -test_backup_cluster() { - print_header "Cluster Backup Tests" - - cleanup_test_backups - - # Test 1: Basic cluster backup - print_test "Cluster backup (all databases)" - if timeout 180 run_as_postgres $DBBACKUP_BIN backup cluster -d postgres --insecure \ - --backup-dir "$TEST_BACKUP_DIR" --compression 3 >>"$LOG_FILE" 2>&1; then - if ls "$TEST_BACKUP_DIR"/cluster_*.tar.gz >/dev/null 2>&1; then - size=$(ls -lh "$TEST_BACKUP_DIR"/cluster_*.tar.gz 2>/dev/null | tail -1 | awk '{print $5}') - if [ "$size" != "0" ]; then - print_success "Cluster backup created: $size" - else - print_failure "Cluster backup is 0 bytes" - fi - else - print_failure "Cluster backup file not found" - fi - else - print_failure "Cluster backup failed or timed out" - fi - - # Test 2: Verify no huge uncompressed temp files were left - print_test "Verify no leftover temp files" - if [ -d "$TEST_BACKUP_DIR/.cluster_"* ] 2>/dev/null; then - print_failure "Temp cluster directory not cleaned up" - else - print_success "Temp directories cleaned up" - fi -} - -test_restore_single() { - print_header "Single Database Restore Tests" - - cleanup_test_backups - - # Create a backup first - print_test "Create backup for restore test" - if run_as_postgres $DBBACKUP_BIN backup single "$TEST_DB" -d postgres --insecure \ - --backup-dir "$TEST_BACKUP_DIR" >>"$LOG_FILE" 2>&1; then - backup_file=$(ls "$TEST_BACKUP_DIR"/db_${TEST_DB}_*.dump 2>/dev/null | head -1) - if [ -n "$backup_file" ]; then - print_success "Test backup created: $(basename $backup_file)" - - # Test restore with --create flag - print_test "Restore with --create flag" - restore_db="validation_restore_test_$$" - if run_as_postgres $DBBACKUP_BIN restore single "$backup_file" \ - --target-db "$restore_db" -d postgres --insecure --create >>"$LOG_FILE" 2>&1; then - # Check if database exists - if run_as_postgres psql -lqt | cut -d \| -f 1 | grep -qw "$restore_db"; then - print_success "Database restored successfully with --create" - # Cleanup - run_as_postgres psql -d postgres -c "DROP DATABASE IF EXISTS $restore_db" >/dev/null 2>&1 - else - print_failure "Restored database not found" - fi - else - print_failure "Restore with --create failed" - fi - else - print_failure "Test backup file not found" - fi - else - print_failure "Failed to create test backup" - fi -} - -test_status() { - print_header "Status Command Tests" - - print_test "Status host command" - if run_as_postgres $DBBACKUP_BIN status host -d postgres --insecure >>"$LOG_FILE" 2>&1; then - print_success "Status host succeeded" - else - print_failure "Status host failed" - fi - - print_test "Status cpu command" - if $DBBACKUP_BIN status cpu >>"$LOG_FILE" 2>&1; then - print_success "Status CPU succeeded" - else - print_failure "Status CPU failed" - fi -} - -test_compression_efficiency() { - print_header "Compression Efficiency Tests" - - cleanup_test_backups - - # Create backups with different compression levels - declare -A sizes - - for level in 1 6 9; do - print_test "Backup with compression level $level" - if run_as_postgres $DBBACKUP_BIN backup single "$TEST_DB" -d postgres --insecure \ - --backup-dir "$TEST_BACKUP_DIR" --compression $level >>"$LOG_FILE" 2>&1; then - backup_file=$(ls -t "$TEST_BACKUP_DIR"/db_${TEST_DB}_*.dump 2>/dev/null | head -1) - if [ -n "$backup_file" ]; then - size=$(stat -f%z "$backup_file" 2>/dev/null || stat -c%s "$backup_file" 2>/dev/null) - sizes[$level]=$size - size_human=$(ls -lh "$backup_file" | awk '{print $5}') - print_success "Level $level: $size_human" - else - print_failure "Backup file not found for level $level" - fi - else - print_failure "Backup failed for compression level $level" - fi - done - - # Verify compression levels make sense (lower level = larger file) - if [ ${sizes[1]:-0} -gt ${sizes[6]:-0} ] && [ ${sizes[6]:-0} -gt ${sizes[9]:-0} ]; then - print_success "Compression levels work correctly (1 > 6 > 9)" - else - print_failure "Compression levels don't show expected size differences" - fi -} - -test_streaming_compression() { - print_header "Streaming Compression Tests (Large DB)" - - # Check if testdb_50gb exists - if run_as_postgres psql -lqt | cut -d \| -f 1 | grep -qw "testdb_50gb"; then - cleanup_test_backups - - print_test "Backup large DB with streaming compression" - # Use cluster backup which triggers streaming compression for large DBs - if timeout 300 run_as_postgres $DBBACKUP_BIN backup single testdb_50gb -d postgres --insecure \ - --backup-dir "$TEST_BACKUP_DIR" --compression 1 >>"$LOG_FILE" 2>&1; then - backup_file=$(ls "$TEST_BACKUP_DIR"/db_testdb_50gb_*.dump 2>/dev/null | head -1) - if [ -n "$backup_file" ]; then - size_human=$(ls -lh "$backup_file" | awk '{print $5}') - print_success "Large DB backed up: $size_human" - else - print_failure "Large DB backup file not found" - fi - else - print_failure "Large DB backup failed or timed out" - fi - else - print_skip "testdb_50gb not found (large DB tests skipped)" - fi -} - -################################################################################ -# Summary and Report -################################################################################ - -print_summary() { - print_header "Validation Summary" - - echo "" - echo "Total Tests: $TESTS_TOTAL" - echo -e "${GREEN}Passed: $TESTS_PASSED${NC}" - echo -e "${RED}Failed: $TESTS_FAILED${NC}" - echo -e "${YELLOW}Skipped: $TESTS_SKIPPED${NC}" - echo "" - - if [ $TESTS_FAILED -gt 0 ]; then - echo -e "${RED}Failed Tests:${NC}" - for test in "${FAILED_TESTS[@]}"; do - echo -e " ${RED}❌${NC} $test" - done - echo "" - fi - - echo "Full log: $LOG_FILE" - echo "" - - # Calculate success rate - if [ $TESTS_TOTAL -gt 0 ]; then - success_rate=$((TESTS_PASSED * 100 / TESTS_TOTAL)) - echo "Success Rate: ${success_rate}%" - - if [ $success_rate -ge 95 ]; then - echo -e "${GREEN}✅ PRODUCTION READY${NC}" - return 0 - elif [ $success_rate -ge 80 ]; then - echo -e "${YELLOW}⚠️ NEEDS ATTENTION${NC}" - return 1 - else - echo -e "${RED}❌ NOT PRODUCTION READY${NC}" - return 2 - fi - fi -} - -################################################################################ -# Main Execution -################################################################################ - -main() { - echo "================================================" - echo "dbbackup Production Validation" - echo "================================================" - echo "Start Time: $(date)" - echo "Log File: $LOG_FILE" - echo "Test Backup Dir: $TEST_BACKUP_DIR" - echo "" - - # Create log file - touch "$LOG_FILE" - - # Run all test suites - preflight_checks - test_version_help - test_backup_single - test_backup_cluster - test_restore_single - test_status - test_compression_efficiency - test_streaming_compression - - # Print summary - print_summary - exit_code=$? - - # Cleanup - echo "" - echo "Cleaning up test files..." - rm -rf "$TEST_BACKUP_DIR" - - echo "End Time: $(date)" - echo "" - - exit $exit_code -} - -# Run main -main diff --git a/run_master_tests.sh b/run_master_tests.sh deleted file mode 100755 index 4fab935..0000000 --- a/run_master_tests.sh +++ /dev/null @@ -1,397 +0,0 @@ -#!/bin/bash -################################################################################ -# Master Test Execution Script -# Automated testing for dbbackup command-line interface -################################################################################ - -set -e -set -o pipefail - -# Configuration -DBBACKUP="./dbbackup" -TEST_DIR="/tmp/dbbackup_master_test_$$" -TEST_DB="postgres" -POSTGRES_USER="postgres" -LOG_FILE="/tmp/dbbackup_master_test_$(date +%Y%m%d_%H%M%S).log" - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' - -# Test counters -TESTS_RUN=0 -TESTS_PASSED=0 -TESTS_FAILED=0 -declare -a FAILED_TESTS - -# Helper functions -log() { - echo "$@" | tee -a "$LOG_FILE" -} - -test_start() { - TESTS_RUN=$((TESTS_RUN + 1)) - echo -ne "${YELLOW}[TEST $TESTS_RUN]${NC} $1 ... " -} - -test_pass() { - TESTS_PASSED=$((TESTS_PASSED + 1)) - echo -e "${GREEN}PASS${NC}" - [ -n "$1" ] && echo " ↳ $1" -} - -test_fail() { - TESTS_FAILED=$((TESTS_FAILED + 1)) - FAILED_TESTS+=("TEST $TESTS_RUN: $1") - echo -e "${RED}FAIL${NC}" - [ -n "$1" ] && echo " ↳ $1" -} - -run_as_postgres() { - if [ "$(whoami)" = "postgres" ]; then - "$@" - else - sudo -u postgres "$@" - fi -} - -cleanup() { - rm -rf "$TEST_DIR" 2>/dev/null || true -} - -init_test_env() { - mkdir -p "$TEST_DIR" - log "Test directory: $TEST_DIR" - log "Log file: $LOG_FILE" - log "" -} - -################################################################################ -# Test Functions -################################################################################ - -test_binary_exists() { - test_start "Binary exists" - if [ -f "$DBBACKUP" ] && [ -x "$DBBACKUP" ]; then - test_pass "$(ls -lh $DBBACKUP | awk '{print $5}')" - else - test_fail "Binary not found or not executable" - exit 1 - fi -} - -test_help_commands() { - test_start "Help command" - if run_as_postgres $DBBACKUP --help >/dev/null 2>&1; then - test_pass - else - test_fail - fi - - test_start "Version command" - if run_as_postgres $DBBACKUP --version >/dev/null 2>&1; then - version=$(run_as_postgres $DBBACKUP --version 2>&1 | head -1) - test_pass "$version" - else - test_fail - fi - - test_start "Backup help" - if run_as_postgres $DBBACKUP backup --help >/dev/null 2>&1; then - test_pass - else - test_fail - fi - - test_start "Restore help" - if run_as_postgres $DBBACKUP restore --help >/dev/null 2>&1; then - test_pass - else - test_fail - fi -} - -test_status_commands() { - test_start "Status host" - if run_as_postgres $DBBACKUP status host -d postgres --insecure >>"$LOG_FILE" 2>&1; then - test_pass - else - test_fail - fi - - test_start "Status CPU" - if $DBBACKUP status cpu >>"$LOG_FILE" 2>&1; then - test_pass - else - test_fail - fi -} - -test_single_backup() { - local compress=$1 - local desc=$2 - - test_start "Single backup (compression=$compress) $desc" - local backup_dir="$TEST_DIR/single_c${compress}" - mkdir -p "$backup_dir" - - if run_as_postgres timeout 120 $DBBACKUP backup single $TEST_DB -d postgres --insecure \ - --backup-dir "$backup_dir" --compression $compress >>"$LOG_FILE" 2>&1; then - - local backup_file=$(ls "$backup_dir"/db_${TEST_DB}_*.dump 2>/dev/null | head -1) - if [ -n "$backup_file" ]; then - local size=$(ls -lh "$backup_file" | awk '{print $5}') - test_pass "$size" - else - test_fail "Backup file not found" - fi - else - test_fail "Backup command failed" - fi -} - -test_cluster_backup() { - test_start "Cluster backup (all databases)" - local backup_dir="$TEST_DIR/cluster" - mkdir -p "$backup_dir" - - if run_as_postgres timeout 300 $DBBACKUP backup cluster -d postgres --insecure \ - --backup-dir "$backup_dir" --compression 3 >>"$LOG_FILE" 2>&1; then - - local archive=$(ls "$backup_dir"/cluster_*.tar.gz 2>/dev/null | head -1) - if [ -n "$archive" ]; then - local size=$(ls -lh "$archive" | awk '{print $5}') - local archive_size=$(stat -c%s "$archive" 2>/dev/null || stat -f%z "$archive" 2>/dev/null) - - if [ "$archive_size" -gt 1000 ]; then - test_pass "$size" - else - test_fail "Archive is empty or too small" - fi - else - test_fail "Cluster archive not found" - fi - else - test_fail "Cluster backup failed" - fi -} - -test_restore_single() { - test_start "Single database restore with --create" - - # First create a backup - local backup_dir="$TEST_DIR/restore_test" - mkdir -p "$backup_dir" - - if run_as_postgres $DBBACKUP backup single $TEST_DB -d postgres --insecure \ - --backup-dir "$backup_dir" --compression 1 >>"$LOG_FILE" 2>&1; then - - local backup_file=$(ls "$backup_dir"/db_${TEST_DB}_*.dump 2>/dev/null | head -1) - if [ -n "$backup_file" ]; then - local restore_db="master_test_restore_$$" - - if run_as_postgres timeout 120 $DBBACKUP restore single "$backup_file" \ - --target-db "$restore_db" -d postgres --insecure --create >>"$LOG_FILE" 2>&1; then - - # Check if database exists - if run_as_postgres psql -lqt | cut -d \| -f 1 | grep -qw "$restore_db"; then - test_pass "Database restored" - # Cleanup - run_as_postgres psql -d postgres -c "DROP DATABASE IF EXISTS $restore_db" >>"$LOG_FILE" 2>&1 - else - test_fail "Restored database not found" - fi - else - test_fail "Restore failed" - fi - else - test_fail "Backup file not found" - fi - else - test_fail "Initial backup failed" - fi -} - -test_compression_levels() { - log "" - log "=== Compression Level Tests ===" - - declare -A sizes - for level in 1 6 9; do - test_start "Compression level $level" - local backup_dir="$TEST_DIR/compress_$level" - mkdir -p "$backup_dir" - - if run_as_postgres timeout 120 $DBBACKUP backup single $TEST_DB -d postgres --insecure \ - --backup-dir "$backup_dir" --compression $level >>"$LOG_FILE" 2>&1; then - - local backup_file=$(ls "$backup_dir"/db_${TEST_DB}_*.dump 2>/dev/null | head -1) - if [ -n "$backup_file" ]; then - local size=$(stat -c%s "$backup_file" 2>/dev/null || stat -f%z "$backup_file" 2>/dev/null) - local size_mb=$((size / 1024 / 1024)) - sizes[$level]=$size - test_pass "${size_mb}MB" - else - test_fail "Backup not found" - fi - else - test_fail "Backup failed" - fi - done - - # Verify compression works (level 1 > level 9) - if [ ${sizes[1]:-0} -gt ${sizes[9]:-0} ]; then - test_start "Compression efficiency check" - test_pass "Level 1 (${sizes[1]} bytes) > Level 9 (${sizes[9]} bytes)" - else - test_start "Compression efficiency check" - test_fail "Compression levels don't show expected size difference" - fi -} - -test_large_database() { - # Check if testdb_50gb exists - if run_as_postgres psql -lqt | cut -d \| -f 1 | grep -qw "testdb_50gb"; then - test_start "Large database streaming compression" - local backup_dir="$TEST_DIR/large_db" - mkdir -p "$backup_dir" - - if run_as_postgres timeout 600 $DBBACKUP backup single testdb_50gb -d postgres --insecure \ - --backup-dir "$backup_dir" --compression 1 >>"$LOG_FILE" 2>&1; then - - local backup_file=$(ls "$backup_dir"/db_testdb_50gb_*.dump 2>/dev/null | head -1) - if [ -n "$backup_file" ]; then - local size=$(stat -c%s "$backup_file" 2>/dev/null || stat -f%z "$backup_file" 2>/dev/null) - local size_mb=$((size / 1024 / 1024)) - - # Verify it's compressed (should be < 2GB for 7.3GB database) - if [ $size_mb -lt 2000 ]; then - test_pass "${size_mb}MB - streaming compression worked" - else - test_fail "${size_mb}MB - too large, streaming compression may have failed" - fi - else - test_fail "Backup file not found" - fi - else - test_fail "Large database backup failed or timed out" - fi - else - test_start "Large database test" - echo -e "${YELLOW}SKIP${NC} (testdb_50gb not available)" - fi -} - -test_invalid_inputs() { - test_start "Invalid database name" - if run_as_postgres $DBBACKUP backup single nonexistent_db_12345 -d postgres --insecure \ - --backup-dir "$TEST_DIR" 2>&1 | grep -qi "error\|not exist\|failed"; then - test_pass "Error properly reported" - else - test_fail "No error for invalid database" - fi - - test_start "Missing backup file" - if run_as_postgres $DBBACKUP restore single /nonexistent/file.dump -d postgres --insecure \ - 2>&1 | grep -qi "error\|not found\|failed"; then - test_pass "Error properly reported" - else - test_fail "No error for missing file" - fi -} - -################################################################################ -# Main Execution -################################################################################ - -main() { - echo "================================================" - echo "dbbackup Master Test Suite - CLI Automation" - echo "================================================" - echo "Started: $(date)" - echo "" - - init_test_env - - echo "=== Pre-Flight Checks ===" - test_binary_exists - - echo "" - echo "=== Basic Command Tests ===" - test_help_commands - test_status_commands - - echo "" - echo "=== Backup Tests ===" - test_single_backup 1 "(fast)" - test_single_backup 6 "(default)" - test_single_backup 9 "(best)" - test_cluster_backup - - echo "" - echo "=== Restore Tests ===" - test_restore_single - - echo "" - echo "=== Advanced Tests ===" - test_compression_levels - test_large_database - - echo "" - echo "=== Error Handling Tests ===" - test_invalid_inputs - - echo "" - echo "================================================" - echo "Test Summary" - echo "================================================" - echo "Total Tests: $TESTS_RUN" - echo -e "${GREEN}Passed: $TESTS_PASSED${NC}" - echo -e "${RED}Failed: $TESTS_FAILED${NC}" - - if [ $TESTS_FAILED -gt 0 ]; then - echo "" - echo -e "${RED}Failed Tests:${NC}" - for failed in "${FAILED_TESTS[@]}"; do - echo -e " ${RED}✗${NC} $failed" - done - fi - - echo "" - echo "Log file: $LOG_FILE" - echo "Completed: $(date)" - echo "" - - # Calculate success rate - if [ $TESTS_RUN -gt 0 ]; then - success_rate=$((TESTS_PASSED * 100 / TESTS_RUN)) - echo "Success Rate: ${success_rate}%" - - if [ $success_rate -ge 95 ]; then - echo -e "${GREEN}✅ EXCELLENT - Production Ready${NC}" - exit_code=0 - elif [ $success_rate -ge 80 ]; then - echo -e "${YELLOW}⚠️ GOOD - Minor issues need attention${NC}" - exit_code=1 - else - echo -e "${RED}❌ POOR - Significant issues found${NC}" - exit_code=2 - fi - fi - - # Cleanup - echo "" - echo "Cleaning up test files..." - cleanup - - exit $exit_code -} - -# Trap cleanup on exit -trap cleanup EXIT INT TERM - -# Run main -main "$@" diff --git a/scripts/cli_switch_test.sh b/scripts/cli_switch_test.sh deleted file mode 100755 index d6077b6..0000000 --- a/scripts/cli_switch_test.sh +++ /dev/null @@ -1,173 +0,0 @@ -#!/usr/bin/env bash -set -u -set -o pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" -BINARY_NAME="dbbackup_linux_amd64" -BINARY="./${BINARY_NAME}" -LOG_DIR="${REPO_ROOT}/test_logs" -TIMESTAMP="$(date +%Y%m%d_%H%M%S)" -LOG_FILE="${LOG_DIR}/cli_switch_test_${TIMESTAMP}.log" - -PG_BACKUP_DIR="/tmp/db_backups" -PG_DATABASE="postgres" -PG_FLAGS=( - --db-type postgres - --host localhost - --port 5432 - --user postgres - --database "${PG_DATABASE}" - --backup-dir "${PG_BACKUP_DIR}" - --jobs 4 - --dump-jobs 4 - --max-cores 8 - --cpu-workload balanced - --debug -) - -MYSQL_BACKUP_DIR="/tmp/mysql_backups" -MYSQL_DATABASE="backup_demo" -MYSQL_FLAGS=( - --db-type mysql - --host 127.0.0.1 - --port 3306 - --user backup_user - --password backup_pass - --database "${MYSQL_DATABASE}" - --backup-dir "${MYSQL_BACKUP_DIR}" - --insecure - --jobs 2 - --dump-jobs 2 - --max-cores 4 - --cpu-workload io-intensive - --debug -) - -mkdir -p "${LOG_DIR}" - -log() { - printf '%s\n' "$1" | tee -a "${LOG_FILE}" >/dev/null -} - -RESULTS=() - -run_cmd() { - local label="$1" - shift - log "" - log "### ${label}" - log "Command: $*" - "$@" 2>&1 | tee -a "${LOG_FILE}" - local status=${PIPESTATUS[0]} - log "Exit: ${status}" - RESULTS+=("${label}|${status}") -} - -latest_file() { - local dir="$1" - local pattern="$2" - shopt -s nullglob - local files=("${dir}"/${pattern}) - shopt -u nullglob - if (( ${#files[@]} == 0 )); then - return 1 - fi - local latest="${files[0]}" - for file in "${files[@]}"; do - if [[ "${file}" -nt "${latest}" ]]; then - latest="${file}" - fi - done - printf '%s\n' "${latest}" -} - -log "dbbackup CLI regression started" -log "Log file: ${LOG_FILE}" - -cd "${REPO_ROOT}" - -run_cmd "Go build" go build -o "${BINARY}" . -run_cmd "Ensure Postgres backup dir" sudo -u postgres mkdir -p "${PG_BACKUP_DIR}" -run_cmd "Ensure MySQL backup dir" mkdir -p "${MYSQL_BACKUP_DIR}" - -run_cmd "Postgres status" sudo -u postgres "${BINARY}" status "${PG_FLAGS[@]}" -run_cmd "Postgres preflight" sudo -u postgres "${BINARY}" preflight "${PG_FLAGS[@]}" -run_cmd "Postgres CPU info" sudo -u postgres "${BINARY}" cpu "${PG_FLAGS[@]}" -run_cmd "Postgres backup single" sudo -u postgres "${BINARY}" backup single "${PG_DATABASE}" "${PG_FLAGS[@]}" -run_cmd "Postgres backup sample" sudo -u postgres "${BINARY}" backup sample "${PG_DATABASE}" --sample-ratio 5 "${PG_FLAGS[@]}" -run_cmd "Postgres backup cluster" sudo -u postgres "${BINARY}" backup cluster "${PG_FLAGS[@]}" -run_cmd "Postgres list" sudo -u postgres "${BINARY}" list "${PG_FLAGS[@]}" - -PG_SINGLE_FILE="$(latest_file "${PG_BACKUP_DIR}" "db_${PG_DATABASE}_*.dump" || true)" -PG_SAMPLE_FILE="$(latest_file "${PG_BACKUP_DIR}" "sample_${PG_DATABASE}_*.sql" || true)" -PG_CLUSTER_FILE="$(latest_file "${PG_BACKUP_DIR}" "cluster_*.tar.gz" || true)" - -if [[ -n "${PG_SINGLE_FILE}" ]]; then - run_cmd "Postgres verify single" sudo -u postgres "${BINARY}" verify "$(basename "${PG_SINGLE_FILE}")" "${PG_FLAGS[@]}" - run_cmd "Postgres restore single" sudo -u postgres "${BINARY}" restore "$(basename "${PG_SINGLE_FILE}")" "${PG_FLAGS[@]}" -else - log "No PostgreSQL single backup found for verification" - RESULTS+=("Postgres single artifact missing|1") -fi - -if [[ -n "${PG_SAMPLE_FILE}" ]]; then - run_cmd "Postgres verify sample" sudo -u postgres "${BINARY}" verify "$(basename "${PG_SAMPLE_FILE}")" "${PG_FLAGS[@]}" - run_cmd "Postgres restore sample" sudo -u postgres "${BINARY}" restore "$(basename "${PG_SAMPLE_FILE}")" "${PG_FLAGS[@]}" -else - log "No PostgreSQL sample backup found for verification" - RESULTS+=("Postgres sample artifact missing|1") -fi - -if [[ -n "${PG_CLUSTER_FILE}" ]]; then - run_cmd "Postgres verify cluster" sudo -u postgres "${BINARY}" verify "$(basename "${PG_CLUSTER_FILE}")" "${PG_FLAGS[@]}" - run_cmd "Postgres restore cluster" sudo -u postgres "${BINARY}" restore "$(basename "${PG_CLUSTER_FILE}")" "${PG_FLAGS[@]}" -else - log "No PostgreSQL cluster backup found for verification" - RESULTS+=("Postgres cluster artifact missing|1") -fi - -run_cmd "MySQL status" "${BINARY}" status "${MYSQL_FLAGS[@]}" -run_cmd "MySQL preflight" "${BINARY}" preflight "${MYSQL_FLAGS[@]}" -run_cmd "MySQL CPU info" "${BINARY}" cpu "${MYSQL_FLAGS[@]}" -run_cmd "MySQL backup single" "${BINARY}" backup single "${MYSQL_DATABASE}" "${MYSQL_FLAGS[@]}" -run_cmd "MySQL backup sample" "${BINARY}" backup sample "${MYSQL_DATABASE}" --sample-percent 25 "${MYSQL_FLAGS[@]}" -run_cmd "MySQL list" "${BINARY}" list "${MYSQL_FLAGS[@]}" - -MYSQL_SINGLE_FILE="$(latest_file "${MYSQL_BACKUP_DIR}" "db_${MYSQL_DATABASE}_*.sql.gz" || true)" -MYSQL_SAMPLE_FILE="$(latest_file "${MYSQL_BACKUP_DIR}" "sample_${MYSQL_DATABASE}_*.sql" || true)" - -if [[ -n "${MYSQL_SINGLE_FILE}" ]]; then - run_cmd "MySQL verify single" "${BINARY}" verify "$(basename "${MYSQL_SINGLE_FILE}")" "${MYSQL_FLAGS[@]}" - run_cmd "MySQL restore single" "${BINARY}" restore "$(basename "${MYSQL_SINGLE_FILE}")" "${MYSQL_FLAGS[@]}" -else - log "No MySQL single backup found for verification" - RESULTS+=("MySQL single artifact missing|1") -fi - -if [[ -n "${MYSQL_SAMPLE_FILE}" ]]; then - run_cmd "MySQL verify sample" "${BINARY}" verify "$(basename "${MYSQL_SAMPLE_FILE}")" "${MYSQL_FLAGS[@]}" - run_cmd "MySQL restore sample" "${BINARY}" restore "$(basename "${MYSQL_SAMPLE_FILE}")" "${MYSQL_FLAGS[@]}" -else - log "No MySQL sample backup found for verification" - RESULTS+=("MySQL sample artifact missing|1") -fi - -run_cmd "Interactive help" "${BINARY}" interactive --help -run_cmd "Root help" "${BINARY}" --help -run_cmd "Root version" "${BINARY}" --version - -log "" -log "=== Summary ===" -failed=0 -for entry in "${RESULTS[@]}"; do - IFS='|' read -r label status <<<"${entry}" - if [[ "${status}" -eq 0 ]]; then - log "[PASS] ${label}" - else - log "[FAIL] ${label} (exit ${status})" - failed=1 - fi -done - -exit "${failed}" diff --git a/test_suite.sh b/test_suite.sh deleted file mode 100755 index 2d82268..0000000 --- a/test_suite.sh +++ /dev/null @@ -1,409 +0,0 @@ -#!/bin/bash -# -# DBBackup Complete Test Suite -# Automated testing of all command-line options -# Results written to test_results.txt -# - -RESULTS_FILE="test_results_$(date +%Y%m%d_%H%M%S).txt" -DBBACKUP="./dbbackup" -TEST_DB="test_automation_db" -BACKUP_DIR="/var/lib/pgsql/db_backups" -TEST_BACKUP_DIR="/tmp/test_backups_$$" - -# Colors for terminal output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Counters -TOTAL_TESTS=0 -PASSED_TESTS=0 -FAILED_TESTS=0 -SKIPPED_TESTS=0 - -####################################### -# Helper Functions -####################################### - -log() { - echo -e "${BLUE}[$(date '+%H:%M:%S')]${NC} $1" | tee -a "$RESULTS_FILE" -} - -log_success() { - echo -e "${GREEN}✅ PASS:${NC} $1" | tee -a "$RESULTS_FILE" - ((PASSED_TESTS++)) - ((TOTAL_TESTS++)) -} - -log_fail() { - echo -e "${RED}❌ FAIL:${NC} $1" | tee -a "$RESULTS_FILE" - ((FAILED_TESTS++)) - ((TOTAL_TESTS++)) -} - -log_skip() { - echo -e "${YELLOW}⊘ SKIP:${NC} $1" | tee -a "$RESULTS_FILE" - ((SKIPPED_TESTS++)) - ((TOTAL_TESTS++)) -} - -log_section() { - echo "" | tee -a "$RESULTS_FILE" - echo "================================================================" | tee -a "$RESULTS_FILE" - echo " $1" | tee -a "$RESULTS_FILE" - echo "================================================================" | tee -a "$RESULTS_FILE" -} - -run_test() { - local test_name="$1" - local test_cmd="$2" - local expected_result="${3:-0}" # 0=success, 1=failure expected - - log "Running: $test_name" - echo "Command: $test_cmd" >> "$RESULTS_FILE" - - # Run command and capture output - local output - local exit_code - output=$(eval "$test_cmd" 2>&1) - exit_code=$? - - # Save output to results file - echo "Exit Code: $exit_code" >> "$RESULTS_FILE" - echo "Output:" >> "$RESULTS_FILE" - echo "$output" | head -50 >> "$RESULTS_FILE" - echo "---" >> "$RESULTS_FILE" - - # Check result - if [ "$expected_result" -eq 0 ]; then - # Expecting success - if [ $exit_code -eq 0 ]; then - log_success "$test_name" - return 0 - else - log_fail "$test_name (exit code: $exit_code)" - return 1 - fi - else - # Expecting failure - if [ $exit_code -ne 0 ]; then - log_success "$test_name (correctly failed)" - return 0 - else - log_fail "$test_name (should have failed)" - return 1 - fi - fi -} - -setup_test_env() { - log "Setting up test environment..." - - # Create test database - sudo -u postgres psql -c "DROP DATABASE IF EXISTS $TEST_DB;" > /dev/null 2>&1 - sudo -u postgres psql -c "CREATE DATABASE $TEST_DB;" > /dev/null 2>&1 - sudo -u postgres psql -d "$TEST_DB" -c "CREATE TABLE test_table (id SERIAL, data TEXT);" > /dev/null 2>&1 - sudo -u postgres psql -d "$TEST_DB" -c "INSERT INTO test_table (data) VALUES ('test1'), ('test2'), ('test3');" > /dev/null 2>&1 - - # Create test backup directory - mkdir -p "$TEST_BACKUP_DIR" - - log "Test environment ready" -} - -cleanup_test_env() { - log "Cleaning up test environment..." - sudo -u postgres psql -c "DROP DATABASE IF EXISTS ${TEST_DB};" > /dev/null 2>&1 - sudo -u postgres psql -c "DROP DATABASE IF EXISTS ${TEST_DB}_restored;" > /dev/null 2>&1 - sudo -u postgres psql -c "DROP DATABASE IF EXISTS ${TEST_DB}_created;" > /dev/null 2>&1 - rm -rf "$TEST_BACKUP_DIR" - log "Cleanup complete" -} - -####################################### -# Test Suite -####################################### - -main() { - log_section "DBBackup Complete Test Suite" - echo "Date: $(date)" | tee -a "$RESULTS_FILE" - echo "Host: $(hostname)" | tee -a "$RESULTS_FILE" - echo "User: $(whoami)" | tee -a "$RESULTS_FILE" - echo "DBBackup: $DBBACKUP" | tee -a "$RESULTS_FILE" - echo "Results File: $RESULTS_FILE" | tee -a "$RESULTS_FILE" - echo "" | tee -a "$RESULTS_FILE" - - # Setup - setup_test_env - - ####################################### - # 1. BASIC HELP & VERSION - ####################################### - log_section "1. Basic Commands" - - run_test "Help command" \ - "sudo -u postgres $DBBACKUP --help" - - run_test "Version flag" \ - "sudo -u postgres $DBBACKUP --version" - - run_test "Status command" \ - "sudo -u postgres $DBBACKUP status" - - ####################################### - # 2. BACKUP SINGLE DATABASE - ####################################### - log_section "2. Backup Single Database" - - run_test "Backup single database (basic)" \ - "sudo -u postgres $DBBACKUP backup single $TEST_DB" - - run_test "Backup single with compression level 9" \ - "sudo -u postgres $DBBACKUP backup single $TEST_DB --compression=9" - - run_test "Backup single with compression level 1" \ - "sudo -u postgres $DBBACKUP backup single $TEST_DB --compression=1" - - run_test "Backup single with custom backup dir" \ - "sudo -u postgres $DBBACKUP backup single $TEST_DB --backup-dir=$TEST_BACKUP_DIR" - - run_test "Backup single with jobs=1" \ - "sudo -u postgres $DBBACKUP backup single $TEST_DB --jobs=1" - - run_test "Backup single with jobs=16" \ - "sudo -u postgres $DBBACKUP backup single $TEST_DB --jobs=16" - - run_test "Backup single non-existent database (should fail)" \ - "sudo -u postgres $DBBACKUP backup single nonexistent_database_xyz" 1 - - run_test "Backup single with debug logging" \ - "sudo -u postgres $DBBACKUP backup single $TEST_DB --debug" - - run_test "Backup single with no-color" \ - "sudo -u postgres $DBBACKUP backup single $TEST_DB --no-color" - - ####################################### - # 3. BACKUP CLUSTER - ####################################### - log_section "3. Backup Cluster" - - run_test "Backup cluster (basic)" \ - "sudo -u postgres $DBBACKUP backup cluster" - - run_test "Backup cluster with compression 9" \ - "sudo -u postgres $DBBACKUP backup cluster --compression=9" - - run_test "Backup cluster with jobs=4" \ - "sudo -u postgres $DBBACKUP backup cluster --jobs=4" - - run_test "Backup cluster with dump-jobs=4" \ - "sudo -u postgres $DBBACKUP backup cluster --dump-jobs=4" - - run_test "Backup cluster with custom backup dir" \ - "sudo -u postgres $DBBACKUP backup cluster --backup-dir=$TEST_BACKUP_DIR" - - run_test "Backup cluster with debug" \ - "sudo -u postgres $DBBACKUP backup cluster --debug" - - ####################################### - # 4. RESTORE LIST - ####################################### - log_section "4. Restore List" - - run_test "List available backups" \ - "sudo -u postgres $DBBACKUP restore list" - - run_test "List backups from custom dir" \ - "sudo -u postgres $DBBACKUP restore list --backup-dir=$TEST_BACKUP_DIR" - - ####################################### - # 5. RESTORE SINGLE DATABASE - ####################################### - log_section "5. Restore Single Database" - - # Get latest backup file - LATEST_BACKUP=$(find "$BACKUP_DIR" -name "db_${TEST_DB}_*.dump" -type f -printf '%T@ %p\n' | sort -n | tail -1 | cut -d' ' -f2-) - - if [ -n "$LATEST_BACKUP" ]; then - log "Using backup file: $LATEST_BACKUP" - - # Create target database for restore - sudo -u postgres psql -c "DROP DATABASE IF EXISTS ${TEST_DB}_restored;" > /dev/null 2>&1 - sudo -u postgres psql -c "CREATE DATABASE ${TEST_DB}_restored;" > /dev/null 2>&1 - - run_test "Restore single database (basic)" \ - "sudo -u postgres $DBBACKUP restore single $LATEST_BACKUP --target=${TEST_DB}_restored --confirm" - - run_test "Restore single with --clean flag" \ - "sudo -u postgres $DBBACKUP restore single $LATEST_BACKUP --target=${TEST_DB}_restored --clean --confirm" - - run_test "Restore single with --create flag" \ - "sudo -u postgres $DBBACKUP restore single $LATEST_BACKUP --target=${TEST_DB}_created --create --confirm" - - run_test "Restore single with --dry-run" \ - "sudo -u postgres $DBBACKUP restore single $LATEST_BACKUP --target=${TEST_DB}_restored --dry-run" - - run_test "Restore single with --verbose" \ - "sudo -u postgres $DBBACKUP restore single $LATEST_BACKUP --target=${TEST_DB}_restored --verbose --confirm" - - run_test "Restore single with --force" \ - "sudo -u postgres $DBBACKUP restore single $LATEST_BACKUP --target=${TEST_DB}_restored --force --confirm" - - run_test "Restore single without --confirm (should show dry-run)" \ - "sudo -u postgres $DBBACKUP restore single $LATEST_BACKUP --target=${TEST_DB}_restored" - else - log_skip "Restore single tests (no backup file found)" - fi - - run_test "Restore non-existent file (should fail)" \ - "sudo -u postgres $DBBACKUP restore single /tmp/nonexistent_file.dump --confirm" 1 - - ####################################### - # 6. RESTORE CLUSTER - ####################################### - log_section "6. Restore Cluster" - - # Get latest cluster backup - LATEST_CLUSTER=$(find "$BACKUP_DIR" -name "cluster_*.tar.gz" -type f -printf '%T@ %p\n' | sort -n | tail -1 | cut -d' ' -f2-) - - if [ -n "$LATEST_CLUSTER" ]; then - log "Using cluster backup: $LATEST_CLUSTER" - - run_test "Restore cluster with --dry-run" \ - "sudo -u postgres $DBBACKUP restore cluster $LATEST_CLUSTER --dry-run" - - run_test "Restore cluster with --verbose" \ - "sudo -u postgres $DBBACKUP restore cluster $LATEST_CLUSTER --verbose --confirm" - - run_test "Restore cluster with --force" \ - "sudo -u postgres $DBBACKUP restore cluster $LATEST_CLUSTER --force --confirm" - - run_test "Restore cluster with --jobs=2" \ - "sudo -u postgres $DBBACKUP restore cluster $LATEST_CLUSTER --jobs=2 --confirm" - - run_test "Restore cluster without --confirm (should show dry-run)" \ - "sudo -u postgres $DBBACKUP restore cluster $LATEST_CLUSTER" - else - log_skip "Restore cluster tests (no cluster backup found)" - fi - - ####################################### - # 7. GLOBAL FLAGS - ####################################### - log_section "7. Global Flags" - - run_test "Custom host flag" \ - "sudo -u postgres $DBBACKUP status --host=localhost" - - run_test "Custom port flag" \ - "sudo -u postgres $DBBACKUP status --port=5432" - - run_test "Custom user flag" \ - "sudo -u postgres $DBBACKUP status --user=postgres" - - run_test "Database type postgres" \ - "sudo -u postgres $DBBACKUP status --db-type=postgres" - - run_test "SSL mode disable (insecure)" \ - "sudo -u postgres $DBBACKUP status --insecure" - - run_test "SSL mode require" \ - "sudo -u postgres $DBBACKUP status --ssl-mode=require" 1 - - run_test "SSL mode prefer" \ - "sudo -u postgres $DBBACKUP status --ssl-mode=prefer" - - run_test "Max cores flag" \ - "sudo -u postgres $DBBACKUP status --max-cores=4" - - run_test "Disable auto-detect cores" \ - "sudo -u postgres $DBBACKUP status --auto-detect-cores=false" - - run_test "CPU workload balanced" \ - "sudo -u postgres $DBBACKUP status --cpu-workload=balanced" - - run_test "CPU workload cpu-intensive" \ - "sudo -u postgres $DBBACKUP status --cpu-workload=cpu-intensive" - - run_test "CPU workload io-intensive" \ - "sudo -u postgres $DBBACKUP status --cpu-workload=io-intensive" - - ####################################### - # 8. AUTHENTICATION TESTS - ####################################### - log_section "8. Authentication Tests" - - run_test "Connection with peer auth (default)" \ - "sudo -u postgres $DBBACKUP status" - - run_test "Connection with --user flag" \ - "sudo -u postgres $DBBACKUP status --user=postgres" - - # This should fail or warn - run_test "Wrong user flag (should fail/warn)" \ - "./dbbackup status --user=postgres" 1 - - ####################################### - # 9. ERROR SCENARIOS - ####################################### - log_section "9. Error Scenarios" - - run_test "Invalid compression level (should fail)" \ - "sudo -u postgres $DBBACKUP backup single $TEST_DB --compression=99" 1 - - run_test "Invalid database type (should fail)" \ - "sudo -u postgres $DBBACKUP status --db-type=invalid" 1 - - run_test "Invalid CPU workload (should fail)" \ - "sudo -u postgres $DBBACKUP status --cpu-workload=invalid" 1 - - run_test "Invalid port (should fail)" \ - "sudo -u postgres $DBBACKUP status --port=99999" 1 - - run_test "Backup to read-only directory (should fail)" \ - "sudo -u postgres $DBBACKUP backup single $TEST_DB --backup-dir=/proc" 1 - - ####################################### - # 10. INTERACTIVE MODE (Quick Test) - ####################################### - log_section "10. Interactive Mode" - - # Can't fully test interactive mode in script, but check it launches - run_test "Interactive mode help" \ - "sudo -u postgres $DBBACKUP interactive --help" - - ####################################### - # SUMMARY - ####################################### - log_section "Test Suite Summary" - - echo "" | tee -a "$RESULTS_FILE" - echo "Total Tests: $TOTAL_TESTS" | tee -a "$RESULTS_FILE" - echo "Passed: $PASSED_TESTS" | tee -a "$RESULTS_FILE" - echo "Failed: $FAILED_TESTS" | tee -a "$RESULTS_FILE" - echo "Skipped: $SKIPPED_TESTS" | tee -a "$RESULTS_FILE" - echo "" | tee -a "$RESULTS_FILE" - - if [ $FAILED_TESTS -eq 0 ]; then - log_success "All tests passed! 🎉" - EXIT_CODE=0 - else - log_fail "$FAILED_TESTS test(s) failed" - EXIT_CODE=1 - fi - - echo "" | tee -a "$RESULTS_FILE" - echo "Results saved to: $RESULTS_FILE" | tee -a "$RESULTS_FILE" - echo "" | tee -a "$RESULTS_FILE" - - # Cleanup - cleanup_test_env - - exit $EXIT_CODE -} - -# Run main function -main "$@"