UUXO/dbbackup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 28 Wiki Activity

feat: v2.0 Sprint 3 - Cloud URI Support & Command Integration (Part 1)

Browse Source 
Sprint 3 Implementation - Cloud URI Support:

New Features:
✅ Cloud URI parser (s3://bucket/path)
✅ Backup command with --cloud URI flag
✅ Restore from cloud URIs
✅ Verify cloud backups
✅ Cleanup cloud storage with retention policy

New Files:
- internal/cloud/uri.go - Cloud URI parser
- internal/restore/ - Cloud download module
- internal/restore/cloud_download.go - Download & verify helper

Modified Commands:
- cmd/backup.go - Added --cloud s3://bucket/path flag
- cmd/restore.go - Auto-detect & download from cloud URIs
- cmd/verify.go - Verify backups from cloud storage
- cmd/cleanup.go - Apply retention policy to cloud storage

URI Support:
- s3://bucket/path/file.dump - AWS S3
- minio://bucket/path/file.dump - MinIO
- b2://bucket/path/file.dump - Backblaze B2
- gs://bucket/path/file.dump - Google Cloud Storage

Examples:
  # Backup with cloud URI
  dbbackup backup single mydb --cloud s3://my-bucket/backups/

  # Restore from cloud
  dbbackup restore single s3://my-bucket/backups/mydb.dump --confirm

  # Verify cloud backup
  dbbackup verify-backup s3://my-bucket/backups/mydb.dump

  # Cleanup old cloud backups
  dbbackup cleanup s3://my-bucket/backups/ --retention-days 30

Features:
- Automatic download to temp directory
- SHA-256 verification after download
- Automatic temp file cleanup
- Progress tracking for downloads
- Metadata synchronization
- Retention policy for cloud storage

Sprint 3 Part 1 COMPLETE!
This commit is contained in:
Renz
2025-11-25 20:30:28 +00:00
parent 20b7f1ec04
commit bdf9af0650
6 changed files with 798 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
cmd/backup.go
View File

@@ -3,6 +3,7 @@ package cmd
import (
"fmt"
"dbbackup/internal/cloud"
"github.com/spf13/cobra"
)
@@ -92,6 +93,7 @@ func init() {
// Cloud storage flags for all backup commands
for _, cmd := range []*cobra.Command{clusterCmd, singleCmd, sampleCmd} {
cmd.Flags().String("cloud", "", "Cloud storage URI (e.g., s3://bucket/path) - takes precedence over individual flags")
cmd.Flags().Bool("cloud-auto-upload", false, "Automatically upload backup to cloud after completion")
cmd.Flags().String("cloud-provider", "", "Cloud provider (s3, minio, b2)")
cmd.Flags().String("cloud-bucket", "", "Cloud bucket name")
@@ -109,32 +111,39 @@ func init() {
}
}
// Update cloud config from flags
if c.Flags().Changed("cloud-auto-upload") {
if autoUpload, _ := c.Flags().GetBool("cloud-auto-upload"); autoUpload {
cfg.CloudEnabled = true
cfg.CloudAutoUpload = true
// Check if --cloud URI flag is provided (takes precedence)
if c.Flags().Changed("cloud") {
if err := parseCloudURIFlag(c); err != nil {
return err
}
} else {
// Update cloud config from individual flags
if c.Flags().Changed("cloud-auto-upload") {
if autoUpload, _ := c.Flags().GetBool("cloud-auto-upload"); autoUpload {
cfg.CloudEnabled = true
cfg.CloudAutoUpload = true
}
}
if c.Flags().Changed("cloud-provider") {
cfg.CloudProvider, _ = c.Flags().GetString("cloud-provider")
}
if c.Flags().Changed("cloud-bucket") {
cfg.CloudBucket, _ = c.Flags().GetString("cloud-bucket")
}
if c.Flags().Changed("cloud-region") {
cfg.CloudRegion, _ = c.Flags().GetString("cloud-region")
}
if c.Flags().Changed("cloud-endpoint") {
cfg.CloudEndpoint, _ = c.Flags().GetString("cloud-endpoint")
}
if c.Flags().Changed("cloud-prefix") {
cfg.CloudPrefix, _ = c.Flags().GetString("cloud-prefix")
}
}
if c.Flags().Changed("cloud-provider") {
cfg.CloudProvider, _ = c.Flags().GetString("cloud-provider")
}
if c.Flags().Changed("cloud-bucket") {
cfg.CloudBucket, _ = c.Flags().GetString("cloud-bucket")
}
if c.Flags().Changed("cloud-region") {
cfg.CloudRegion, _ = c.Flags().GetString("cloud-region")
}
if c.Flags().Changed("cloud-endpoint") {
cfg.CloudEndpoint, _ = c.Flags().GetString("cloud-endpoint")
}
if c.Flags().Changed("cloud-prefix") {
cfg.CloudPrefix, _ = c.Flags().GetString("cloud-prefix")
}
return nil
@@ -177,4 +186,40 @@ func init() {
// Mark the strategy flags as mutually exclusive
sampleCmd.MarkFlagsMutuallyExclusive("sample-ratio", "sample-percent", "sample-count")
}
// parseCloudURIFlag parses the --cloud URI flag and updates config
func parseCloudURIFlag(cmd *cobra.Command) error {
cloudURI, _ := cmd.Flags().GetString("cloud")
if cloudURI == "" {
return nil
}
// Parse cloud URI
uri, err := cloud.ParseCloudURI(cloudURI)
if err != nil {
return fmt.Errorf("invalid cloud URI: %w", err)
}
// Enable cloud and auto-upload
cfg.CloudEnabled = true
cfg.CloudAutoUpload = true
// Update config from URI
cfg.CloudProvider = uri.Provider
cfg.CloudBucket = uri.Bucket
if uri.Region != "" {
cfg.CloudRegion = uri.Region
}
if uri.Endpoint != "" {
cfg.CloudEndpoint = uri.Endpoint
}
if uri.Path != "" {
cfg.CloudPrefix = uri.Dir()
}
return nil
}

184
cmd/cleanup.go
View File

@@ -1,11 +1,14 @@
package cmd
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"dbbackup/internal/cloud"
"dbbackup/internal/metadata"
"dbbackup/internal/retention"
"github.com/spf13/cobra"
@@ -53,7 +56,15 @@ func init() {
}
func runCleanup(cmd *cobra.Command, args []string) error {
backupDir := args[0]
backupPath := args[0]
// Check if this is a cloud URI
if isCloudURIPath(backupPath) {
return runCloudCleanup(cmd.Context(), backupPath)
}
// Local cleanup
backupDir := backupPath
// Validate directory exists
if !dirExists(backupDir) {
@@ -150,3 +161,174 @@ func dirExists(path string) bool {
}
return info.IsDir()
}
// isCloudURIPath checks if a path is a cloud URI
func isCloudURIPath(s string) bool {
return cloud.IsCloudURI(s)
}
// runCloudCleanup applies retention policy to cloud storage
func runCloudCleanup(ctx context.Context, uri string) error {
// Parse cloud URI
cloudURI, err := cloud.ParseCloudURI(uri)
if err != nil {
return fmt.Errorf("invalid cloud URI: %w", err)
}
fmt.Printf("☁️ Cloud Cleanup Policy:\n")
fmt.Printf(" URI: %s\n", uri)
fmt.Printf(" Provider: %s\n", cloudURI.Provider)
fmt.Printf(" Bucket: %s\n", cloudURI.Bucket)
if cloudURI.Path != "" {
fmt.Printf(" Prefix: %s\n", cloudURI.Path)
}
fmt.Printf(" Retention: %d days\n", retentionDays)
fmt.Printf(" Min backups: %d\n", minBackups)
if dryRun {
fmt.Printf(" Mode: DRY RUN (no files will be deleted)\n")
}
fmt.Println()
// Create cloud backend
cfg := cloudURI.ToConfig()
backend, err := cloud.NewBackend(cfg)
if err != nil {
return fmt.Errorf("failed to create cloud backend: %w", err)
}
// List all backups
backups, err := backend.List(ctx, cloudURI.Path)
if err != nil {
return fmt.Errorf("failed to list cloud backups: %w", err)
}
if len(backups) == 0 {
fmt.Println("No backups found in cloud storage")
return nil
}
fmt.Printf("Found %d backup(s) in cloud storage\n\n", len(backups))
// Filter backups based on pattern if specified
var filteredBackups []cloud.BackupInfo
if cleanupPattern != "" {
for _, backup := range backups {
matched, _ := filepath.Match(cleanupPattern, backup.Name)
if matched {
filteredBackups = append(filteredBackups, backup)
}
}
fmt.Printf("Pattern matched %d backup(s)\n\n", len(filteredBackups))
} else {
filteredBackups = backups
}
// Sort by modification time (oldest first)
// Already sorted by backend.List
// Calculate retention date
cutoffDate := time.Now().AddDate(0, 0, -retentionDays)
// Determine which backups to delete
var toDelete []cloud.BackupInfo
var toKeep []cloud.BackupInfo
for _, backup := range filteredBackups {
if backup.LastModified.Before(cutoffDate) {
toDelete = append(toDelete, backup)
} else {
toKeep = append(toKeep, backup)
}
}
// Ensure we keep minimum backups
totalBackups := len(filteredBackups)
if totalBackups-len(toDelete) < minBackups {
// Need to keep more backups
keepCount := minBackups - len(toKeep)
if keepCount > len(toDelete) {
keepCount = len(toDelete)
}
// Move oldest from toDelete to toKeep
for i := len(toDelete) - 1; i >= len(toDelete)-keepCount && i >= 0; i-- {
toKeep = append(toKeep, toDelete[i])
toDelete = toDelete[:i]
}
}
// Display results
fmt.Printf("📊 Results:\n")
fmt.Printf(" Total backups: %d\n", totalBackups)
fmt.Printf(" Eligible for deletion: %d\n", len(toDelete))
fmt.Printf(" Will keep: %d\n", len(toKeep))
fmt.Println()
if len(toDelete) > 0 {
if dryRun {
fmt.Printf("🔍 Would delete %d backup(s):\n", len(toDelete))
} else {
fmt.Printf("🗑️ Deleting %d backup(s):\n", len(toDelete))
}
var totalSize int64
var deletedCount int
for _, backup := range toDelete {
fmt.Printf(" - %s (%s, %s old)\n",
backup.Name,
cloud.FormatSize(backup.Size),
formatBackupAge(backup.LastModified))
totalSize += backup.Size
if !dryRun {
if err := backend.Delete(ctx, backup.Key); err != nil {
fmt.Printf(" ❌ Error: %v\n", err)
} else {
deletedCount++
// Also try to delete metadata
backend.Delete(ctx, backup.Key+".meta.json")
}
}
}
fmt.Printf("\n💾 Space %s: %s\n",
map[bool]string{true: "would be freed", false: "freed"}[dryRun],
cloud.FormatSize(totalSize))
if !dryRun && deletedCount > 0 {
fmt.Printf("✅ Successfully deleted %d backup(s)\n", deletedCount)
}
} else {
fmt.Println("No backups eligible for deletion")
}
return nil
}
// formatBackupAge returns a human-readable age string from a time.Time
func formatBackupAge(t time.Time) string {
d := time.Since(t)
days := int(d.Hours() / 24)
if days == 0 {
return "today"
} else if days == 1 {
return "1 day"
} else if days < 30 {
return fmt.Sprintf("%d days", days)
} else if days < 365 {
months := days / 30
if months == 1 {
return "1 month"
}
return fmt.Sprintf("%d months", months)
} else {
years := days / 365
if years == 1 {
return "1 year"
}
return fmt.Sprintf("%d years", years)
}
}

51
cmd/restore.go
View File

@@ -10,6 +10,7 @@ import (
"syscall"
"time"
"dbbackup/internal/cloud"
"dbbackup/internal/database"
"dbbackup/internal/restore"
"dbbackup/internal/security"
@@ -168,19 +169,49 @@ func init() {
// runRestoreSingle restores a single database
func runRestoreSingle(cmd *cobra.Command, args []string) error {
archivePath := args[0]
// Convert to absolute path
if !filepath.IsAbs(archivePath) {
absPath, err := filepath.Abs(archivePath)
// Check if this is a cloud URI
var cleanupFunc func() error
if cloud.IsCloudURI(archivePath) {
log.Info("Detected cloud URI, downloading backup...", "uri", archivePath)
// Download from cloud
result, err := restore.DownloadFromCloudURI(cmd.Context(), archivePath, restore.DownloadOptions{
VerifyChecksum: true,
KeepLocal: false, // Delete after restore
})
if err != nil {
return fmt.Errorf("invalid archive path: %w", err)
return fmt.Errorf("failed to download from cloud: %w", err)
}
archivePath = result.LocalPath
cleanupFunc = result.Cleanup
// Ensure cleanup happens on exit
defer func() {
if cleanupFunc != nil {
if err := cleanupFunc(); err != nil {
log.Warn("Failed to cleanup temp files", "error", err)
}
}
}()
log.Info("Download completed", "local_path", archivePath)
} else {
// Convert to absolute path for local files
if !filepath.IsAbs(archivePath) {
absPath, err := filepath.Abs(archivePath)
if err != nil {
return fmt.Errorf("invalid archive path: %w", err)
}
archivePath = absPath
}
archivePath = absPath
}
// Check if file exists
if _, err := os.Stat(archivePath); err != nil {
return fmt.Errorf("archive not found: %s", archivePath)
// Check if file exists
if _, err := os.Stat(archivePath); err != nil {
return fmt.Errorf("archive not found: %s", archivePath)
}
}
// Detect format

96
cmd/verify.go
View File

@@ -1,13 +1,16 @@
package cmd
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"dbbackup/internal/cloud"
"dbbackup/internal/metadata"
"dbbackup/internal/restore"
"dbbackup/internal/verification"
"github.com/spf13/cobra"
)
@@ -46,7 +49,21 @@ func init() {
}
func runVerifyBackup(cmd *cobra.Command, args []string) error {
// Expand glob patterns
// Check if any argument is a cloud URI
hasCloudURI := false
for _, arg := range args {
if isCloudURI(arg) {
hasCloudURI = true
break
}
}
// If cloud URIs detected, handle separately
if hasCloudURI {
return runVerifyCloudBackup(cmd, args)
}
// Expand glob patterns for local files
var backupFiles []string
for _, pattern := range args {
matches, err := filepath.Glob(pattern)
@@ -139,3 +156,80 @@ func runVerifyBackup(cmd *cobra.Command, args []string) error {
return nil
}
// isCloudURI checks if a string is a cloud URI
func isCloudURI(s string) bool {
return cloud.IsCloudURI(s)
}
// verifyCloudBackup downloads and verifies a backup from cloud storage
func verifyCloudBackup(ctx context.Context, uri string, quick, verbose bool) (*restore.DownloadResult, error) {
// Download from cloud with checksum verification
result, err := restore.DownloadFromCloudURI(ctx, uri, restore.DownloadOptions{
VerifyChecksum: !quick, // Skip checksum if quick mode
KeepLocal: false,
})
if err != nil {
return nil, err
}
// If not quick mode, also run full verification
if !quick {
_, err := verification.Verify(result.LocalPath)
if err != nil {
result.Cleanup()
return nil, err
}
}
return result, nil
}
// runVerifyCloudBackup verifies backups from cloud storage
func runVerifyCloudBackup(cmd *cobra.Command, args []string) error {
fmt.Printf("Verifying cloud backup(s)...\n\n")
successCount := 0
failureCount := 0
for _, uri := range args {
if !isCloudURI(uri) {
fmt.Printf("⚠️ Skipping non-cloud URI: %s\n", uri)
continue
}
fmt.Printf("☁️ %s\n", uri)
// Download and verify
result, err := verifyCloudBackup(cmd.Context(), uri, quickVerify, verboseVerify)
if err != nil {
fmt.Printf(" ❌ FAILED: %v\n\n", err)
failureCount++
continue
}
// Cleanup temp file
defer result.Cleanup()
fmt.Printf(" ✅ VALID\n")
if verboseVerify && result.MetadataPath != "" {
meta, _ := metadata.Load(result.MetadataPath)
if meta != nil {
fmt.Printf(" Size: %s\n", metadata.FormatSize(meta.SizeBytes))
fmt.Printf(" SHA-256: %s\n", meta.SHA256)
fmt.Printf(" Database: %s (%s)\n", meta.Database, meta.DatabaseType)
fmt.Printf(" Created: %s\n", meta.Timestamp.Format(time.RFC3339))
}
}
fmt.Println()
successCount++
}
fmt.Printf("\n✅ Summary: %d valid, %d failed\n", successCount, failureCount)
if failureCount > 0 {
os.Exit(1)
}
return nil
}