Files
dbbackup/internal/cloud/interface.go
Renz 64f1458e9a feat: Sprint 4 - Azure Blob Storage and Google Cloud Storage support
Implemented full native support for Azure Blob Storage and Google Cloud Storage:

**Azure Blob Storage (internal/cloud/azure.go):**
- Native Azure SDK integration (github.com/Azure/azure-sdk-for-go)
- Block blob upload for large files (>256MB with 100MB blocks)
- Azurite emulator support for local testing
- Production Azure authentication (account name + key)
- SHA-256 integrity verification with metadata
- Streaming uploads with progress tracking

**Google Cloud Storage (internal/cloud/gcs.go):**
- Native GCS SDK integration (cloud.google.com/go/storage)
- Chunked upload for large files (16MB chunks)
- fake-gcs-server emulator support for local testing
- Application Default Credentials support
- Service account JSON key file support
- SHA-256 integrity verification with metadata
- Streaming uploads with progress tracking

**Backend Integration:**
- Updated NewBackend() factory to support azure/azblob and gs/gcs providers
- Added Name() methods to both backends
- Fixed ProgressReader usage across all backends
- Updated Config comments to document Azure/GCS support

**Testing Infrastructure:**
- docker-compose.azurite.yml: Azurite + PostgreSQL + MySQL test environment
- docker-compose.gcs.yml: fake-gcs-server + PostgreSQL + MySQL test environment
- scripts/test_azure_storage.sh: 8 comprehensive Azure integration tests
- scripts/test_gcs_storage.sh: 8 comprehensive GCS integration tests
- Both test scripts validate upload/download/verify/cleanup/restore operations

**Documentation:**
- AZURE.md: Complete guide (600+ lines) covering setup, authentication, usage
- GCS.md: Complete guide (600+ lines) covering setup, authentication, usage
- Updated CLOUD.md with Azure and GCS sections
- Updated internal/config/config.go with Azure/GCS field documentation

**Test Coverage:**
- Large file uploads (300MB for Azure, 200MB for GCS)
- Block/chunked upload verification
- Backup verification with SHA-256 checksums
- Restore from cloud URIs
- Cleanup and retention policies
- Emulator support for both providers

**Dependencies Added:**
- Azure: github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
- GCS: cloud.google.com/go/storage v1.57.2
- Plus transitive dependencies (~50+ packages)

**Build:**
- Compiles successfully: 68MB binary
- All imports resolved
- No compilation errors

Sprint 4 closes the multi-cloud gap identified in Sprint 3 evaluation.
Users can now use Azure and GCS URIs that were previously parsed but unsupported.
2025-11-25 21:31:21 +00:00

172 lines
4.9 KiB
Go

package cloud
import (
"context"
"fmt"
"io"
"time"
)
// Backend defines the interface for cloud storage providers
type Backend interface {
// Upload uploads a file to cloud storage
Upload(ctx context.Context, localPath, remotePath string, progress ProgressCallback) error
// Download downloads a file from cloud storage
Download(ctx context.Context, remotePath, localPath string, progress ProgressCallback) error
// List lists all backup files in cloud storage
List(ctx context.Context, prefix string) ([]BackupInfo, error)
// Delete deletes a file from cloud storage
Delete(ctx context.Context, remotePath string) error
// Exists checks if a file exists in cloud storage
Exists(ctx context.Context, remotePath string) (bool, error)
// GetSize returns the size of a remote file
GetSize(ctx context.Context, remotePath string) (int64, error)
// Name returns the backend name (e.g., "s3", "azure", "gcs")
Name() string
}
// BackupInfo contains information about a backup in cloud storage
type BackupInfo struct {
Key string // Full path/key in cloud storage
Name string // Base filename
Size int64 // Size in bytes
LastModified time.Time // Last modification time
ETag string // Entity tag (version identifier)
StorageClass string // Storage class (e.g., STANDARD, GLACIER)
}
// ProgressCallback is called during upload/download to report progress
type ProgressCallback func(bytesTransferred, totalBytes int64)
// Config contains common configuration for cloud backends
type Config struct {
Provider string // "s3", "minio", "azure", "gcs", "b2"
Bucket string // Bucket or container name
Region string // Region (for S3)
Endpoint string // Custom endpoint (for MinIO, S3-compatible)
AccessKey string // Access key or account ID
SecretKey string // Secret key or access token
UseSSL bool // Use SSL/TLS (default: true)
PathStyle bool // Use path-style addressing (for MinIO)
Prefix string // Prefix for all operations (e.g., "backups/")
Timeout int // Timeout in seconds (default: 300)
MaxRetries int // Maximum retry attempts (default: 3)
Concurrency int // Upload/download concurrency (default: 5)
}
// NewBackend creates a new cloud storage backend based on the provider
func NewBackend(cfg *Config) (Backend, error) {
switch cfg.Provider {
case "s3", "aws":
return NewS3Backend(cfg)
case "minio":
// MinIO uses S3 backend with custom endpoint
cfg.PathStyle = true
if cfg.Endpoint == "" {
return nil, fmt.Errorf("endpoint required for MinIO")
}
return NewS3Backend(cfg)
case "b2", "backblaze":
// Backblaze B2 uses S3-compatible API
cfg.PathStyle = false
if cfg.Endpoint == "" {
return nil, fmt.Errorf("endpoint required for Backblaze B2")
}
return NewS3Backend(cfg)
case "azure", "azblob":
return NewAzureBackend(cfg)
case "gs", "gcs", "google":
return NewGCSBackend(cfg)
default:
return nil, fmt.Errorf("unsupported cloud provider: %s (supported: s3, minio, b2, azure, gcs)", cfg.Provider)
}
}
// FormatSize returns human-readable size
func FormatSize(bytes int64) string {
const unit = 1024
if bytes < unit {
return fmt.Sprintf("%d B", bytes)
}
div, exp := int64(unit), 0
for n := bytes / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %ciB", float64(bytes)/float64(div), "KMGTPE"[exp])
}
// DefaultConfig returns a config with sensible defaults
func DefaultConfig() *Config {
return &Config{
Provider: "s3",
UseSSL: true,
PathStyle: false,
Timeout: 300,
MaxRetries: 3,
Concurrency: 5,
}
}
// Validate checks if the configuration is valid
func (c *Config) Validate() error {
if c.Provider == "" {
return fmt.Errorf("provider is required")
}
if c.Bucket == "" {
return fmt.Errorf("bucket name is required")
}
if c.Provider == "s3" || c.Provider == "aws" {
if c.Region == "" && c.Endpoint == "" {
return fmt.Errorf("region or endpoint is required for S3")
}
}
if c.Provider == "minio" || c.Provider == "b2" {
if c.Endpoint == "" {
return fmt.Errorf("endpoint is required for %s", c.Provider)
}
}
return nil
}
// ProgressReader wraps an io.Reader to track progress
type ProgressReader struct {
reader io.Reader
total int64
read int64
callback ProgressCallback
lastReport time.Time
}
// NewProgressReader creates a progress tracking reader
func NewProgressReader(r io.Reader, total int64, callback ProgressCallback) *ProgressReader {
return &ProgressReader{
reader: r,
total: total,
callback: callback,
lastReport: time.Now(),
}
}
func (pr *ProgressReader) Read(p []byte) (int, error) {
n, err := pr.reader.Read(p)
pr.read += int64(n)
// Report progress every 100ms or when complete
now := time.Now()
if now.Sub(pr.lastReport) > 100*time.Millisecond || err == io.EOF {
if pr.callback != nil {
pr.callback(pr.read, pr.total)
}
pr.lastReport = now
}
return n, err
}