feat: v2.0 Sprint 2 - Auto-Upload to Cloud (Part 2)
- Add cloud configuration to Config struct
- Integrate automatic upload into backup flow
- Add --cloud-auto-upload flag to all backup commands
- Support environment variables for cloud credentials
- Upload both backup file and metadata to cloud
- Non-blocking: backup succeeds even if cloud upload fails
Usage:
dbbackup backup single mydb --cloud-auto-upload \
--cloud-bucket my-backups \
--cloud-provider s3
Or via environment:
export CLOUD_ENABLED=true
export CLOUD_AUTO_UPLOAD=true
export CLOUD_BUCKET=my-backups
export AWS_ACCESS_KEY_ID=...
export AWS_SECRET_ACCESS_KEY=...
dbbackup backup single mydb
Credentials from AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
This commit is contained in:
@@ -90,6 +90,57 @@ func init() {
|
|||||||
backupCmd.AddCommand(singleCmd)
|
backupCmd.AddCommand(singleCmd)
|
||||||
backupCmd.AddCommand(sampleCmd)
|
backupCmd.AddCommand(sampleCmd)
|
||||||
|
|
||||||
|
// Cloud storage flags for all backup commands
|
||||||
|
for _, cmd := range []*cobra.Command{clusterCmd, singleCmd, sampleCmd} {
|
||||||
|
cmd.Flags().Bool("cloud-auto-upload", false, "Automatically upload backup to cloud after completion")
|
||||||
|
cmd.Flags().String("cloud-provider", "", "Cloud provider (s3, minio, b2)")
|
||||||
|
cmd.Flags().String("cloud-bucket", "", "Cloud bucket name")
|
||||||
|
cmd.Flags().String("cloud-region", "us-east-1", "Cloud region")
|
||||||
|
cmd.Flags().String("cloud-endpoint", "", "Cloud endpoint (for MinIO/B2)")
|
||||||
|
cmd.Flags().String("cloud-prefix", "", "Cloud key prefix")
|
||||||
|
|
||||||
|
// Add PreRunE to update config from flags
|
||||||
|
originalPreRun := cmd.PreRunE
|
||||||
|
cmd.PreRunE = func(c *cobra.Command, args []string) error {
|
||||||
|
// Call original PreRunE if exists
|
||||||
|
if originalPreRun != nil {
|
||||||
|
if err := originalPreRun(c, args); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update cloud config from flags
|
||||||
|
if c.Flags().Changed("cloud-auto-upload") {
|
||||||
|
if autoUpload, _ := c.Flags().GetBool("cloud-auto-upload"); autoUpload {
|
||||||
|
cfg.CloudEnabled = true
|
||||||
|
cfg.CloudAutoUpload = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.Flags().Changed("cloud-provider") {
|
||||||
|
cfg.CloudProvider, _ = c.Flags().GetString("cloud-provider")
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.Flags().Changed("cloud-bucket") {
|
||||||
|
cfg.CloudBucket, _ = c.Flags().GetString("cloud-bucket")
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.Flags().Changed("cloud-region") {
|
||||||
|
cfg.CloudRegion, _ = c.Flags().GetString("cloud-region")
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.Flags().Changed("cloud-endpoint") {
|
||||||
|
cfg.CloudEndpoint, _ = c.Flags().GetString("cloud-endpoint")
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.Flags().Changed("cloud-prefix") {
|
||||||
|
cfg.CloudPrefix, _ = c.Flags().GetString("cloud-prefix")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Sample backup flags - use local variables to avoid cfg access during init
|
// Sample backup flags - use local variables to avoid cfg access during init
|
||||||
var sampleStrategy string
|
var sampleStrategy string
|
||||||
var sampleValue int
|
var sampleValue int
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"dbbackup/internal/checks"
|
"dbbackup/internal/checks"
|
||||||
|
"dbbackup/internal/cloud"
|
||||||
"dbbackup/internal/config"
|
"dbbackup/internal/config"
|
||||||
"dbbackup/internal/database"
|
"dbbackup/internal/database"
|
||||||
"dbbackup/internal/security"
|
"dbbackup/internal/security"
|
||||||
@@ -234,6 +235,14 @@ func (e *Engine) BackupSingle(ctx context.Context, databaseName string) error {
|
|||||||
metrics.GlobalMetrics.RecordOperation("backup_single", databaseName, time.Now().Add(-time.Minute), info.Size(), true, 0)
|
metrics.GlobalMetrics.RecordOperation("backup_single", databaseName, time.Now().Add(-time.Minute), info.Size(), true, 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Cloud upload if enabled
|
||||||
|
if e.cfg.CloudEnabled && e.cfg.CloudAutoUpload {
|
||||||
|
if err := e.uploadToCloud(ctx, outputFile, tracker); err != nil {
|
||||||
|
e.log.Warn("Cloud upload failed", "error", err)
|
||||||
|
// Don't fail the backup if cloud upload fails
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Complete operation
|
// Complete operation
|
||||||
tracker.UpdateProgress(100, "Backup operation completed successfully")
|
tracker.UpdateProgress(100, "Backup operation completed successfully")
|
||||||
tracker.Complete(fmt.Sprintf("Single database backup completed: %s", filepath.Base(outputFile)))
|
tracker.Complete(fmt.Sprintf("Single database backup completed: %s", filepath.Base(outputFile)))
|
||||||
@@ -1080,6 +1089,74 @@ func (e *Engine) createClusterMetadata(backupFile string, databases []string, su
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// uploadToCloud uploads a backup file to cloud storage
|
||||||
|
func (e *Engine) uploadToCloud(ctx context.Context, backupFile string, tracker *progress.OperationTracker) error {
|
||||||
|
uploadStep := tracker.AddStep("cloud_upload", "Uploading to cloud storage")
|
||||||
|
|
||||||
|
// Create cloud backend
|
||||||
|
cloudCfg := &cloud.Config{
|
||||||
|
Provider: e.cfg.CloudProvider,
|
||||||
|
Bucket: e.cfg.CloudBucket,
|
||||||
|
Region: e.cfg.CloudRegion,
|
||||||
|
Endpoint: e.cfg.CloudEndpoint,
|
||||||
|
AccessKey: e.cfg.CloudAccessKey,
|
||||||
|
SecretKey: e.cfg.CloudSecretKey,
|
||||||
|
Prefix: e.cfg.CloudPrefix,
|
||||||
|
UseSSL: true,
|
||||||
|
PathStyle: e.cfg.CloudProvider == "minio",
|
||||||
|
Timeout: 300,
|
||||||
|
MaxRetries: 3,
|
||||||
|
}
|
||||||
|
|
||||||
|
backend, err := cloud.NewBackend(cloudCfg)
|
||||||
|
if err != nil {
|
||||||
|
uploadStep.Fail(fmt.Errorf("failed to create cloud backend: %w", err))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get file info
|
||||||
|
info, err := os.Stat(backupFile)
|
||||||
|
if err != nil {
|
||||||
|
uploadStep.Fail(fmt.Errorf("failed to stat backup file: %w", err))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
filename := filepath.Base(backupFile)
|
||||||
|
e.log.Info("Uploading backup to cloud", "file", filename, "size", cloud.FormatSize(info.Size()))
|
||||||
|
|
||||||
|
// Progress callback
|
||||||
|
var lastPercent int
|
||||||
|
progressCallback := func(transferred, total int64) {
|
||||||
|
percent := int(float64(transferred) / float64(total) * 100)
|
||||||
|
if percent != lastPercent && percent%10 == 0 {
|
||||||
|
e.log.Debug("Upload progress", "percent", percent, "transferred", cloud.FormatSize(transferred), "total", cloud.FormatSize(total))
|
||||||
|
lastPercent = percent
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upload to cloud
|
||||||
|
err = backend.Upload(ctx, backupFile, filename, progressCallback)
|
||||||
|
if err != nil {
|
||||||
|
uploadStep.Fail(fmt.Errorf("cloud upload failed: %w", err))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also upload metadata file
|
||||||
|
metaFile := backupFile + ".meta.json"
|
||||||
|
if _, err := os.Stat(metaFile); err == nil {
|
||||||
|
metaFilename := filepath.Base(metaFile)
|
||||||
|
if err := backend.Upload(ctx, metaFile, metaFilename, nil); err != nil {
|
||||||
|
e.log.Warn("Failed to upload metadata file", "error", err)
|
||||||
|
// Don't fail if metadata upload fails
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
uploadStep.Complete(fmt.Sprintf("Uploaded to %s/%s/%s", backend.Name(), e.cfg.CloudBucket, filename))
|
||||||
|
e.log.Info("Backup uploaded to cloud", "provider", backend.Name(), "bucket", e.cfg.CloudBucket, "file", filename)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// executeCommand executes a backup command (optimized for huge databases)
|
// executeCommand executes a backup command (optimized for huge databases)
|
||||||
func (e *Engine) executeCommand(ctx context.Context, cmdArgs []string, outputFile string) error {
|
func (e *Engine) executeCommand(ctx context.Context, cmdArgs []string, outputFile string) error {
|
||||||
if len(cmdArgs) == 0 {
|
if len(cmdArgs) == 0 {
|
||||||
|
|||||||
@@ -85,6 +85,17 @@ type Config struct {
|
|||||||
TUIDryRun bool // TUI dry-run mode (simulate without execution)
|
TUIDryRun bool // TUI dry-run mode (simulate without execution)
|
||||||
TUIVerbose bool // Verbose TUI logging
|
TUIVerbose bool // Verbose TUI logging
|
||||||
TUILogFile string // TUI event log file path
|
TUILogFile string // TUI event log file path
|
||||||
|
|
||||||
|
// Cloud storage options (v2.0)
|
||||||
|
CloudEnabled bool // Enable cloud storage integration
|
||||||
|
CloudProvider string // "s3", "minio", "b2"
|
||||||
|
CloudBucket string // Bucket name
|
||||||
|
CloudRegion string // Region (for S3)
|
||||||
|
CloudEndpoint string // Custom endpoint (for MinIO, B2)
|
||||||
|
CloudAccessKey string // Access key
|
||||||
|
CloudSecretKey string // Secret key
|
||||||
|
CloudPrefix string // Key prefix
|
||||||
|
CloudAutoUpload bool // Automatically upload after backup
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new configuration with default values
|
// New creates a new configuration with default values
|
||||||
@@ -192,6 +203,17 @@ func New() *Config {
|
|||||||
TUIDryRun: getEnvBool("TUI_DRY_RUN", false), // Execute by default
|
TUIDryRun: getEnvBool("TUI_DRY_RUN", false), // Execute by default
|
||||||
TUIVerbose: getEnvBool("TUI_VERBOSE", false), // Quiet by default
|
TUIVerbose: getEnvBool("TUI_VERBOSE", false), // Quiet by default
|
||||||
TUILogFile: getEnvString("TUI_LOG_FILE", ""), // No log file by default
|
TUILogFile: getEnvString("TUI_LOG_FILE", ""), // No log file by default
|
||||||
|
|
||||||
|
// Cloud storage defaults (v2.0)
|
||||||
|
CloudEnabled: getEnvBool("CLOUD_ENABLED", false),
|
||||||
|
CloudProvider: getEnvString("CLOUD_PROVIDER", "s3"),
|
||||||
|
CloudBucket: getEnvString("CLOUD_BUCKET", ""),
|
||||||
|
CloudRegion: getEnvString("CLOUD_REGION", "us-east-1"),
|
||||||
|
CloudEndpoint: getEnvString("CLOUD_ENDPOINT", ""),
|
||||||
|
CloudAccessKey: getEnvString("CLOUD_ACCESS_KEY", getEnvString("AWS_ACCESS_KEY_ID", "")),
|
||||||
|
CloudSecretKey: getEnvString("CLOUD_SECRET_KEY", getEnvString("AWS_SECRET_ACCESS_KEY", "")),
|
||||||
|
CloudPrefix: getEnvString("CLOUD_PREFIX", ""),
|
||||||
|
CloudAutoUpload: getEnvBool("CLOUD_AUTO_UPLOAD", false),
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ensure canonical defaults are enforced
|
// Ensure canonical defaults are enforced
|
||||||
|
|||||||
Reference in New Issue
Block a user