feat: Add enterprise DBA features for production reliability

New features implemented:

1. Backup Catalog (internal/catalog/)
   - SQLite-based backup tracking
   - Gap detection and RPO monitoring
   - Search and statistics
   - Filesystem sync

2. DR Drill Testing (internal/drill/)
   - Automated restore testing in Docker containers
   - Database validation with custom queries
   - Catalog integration for drill-tested status

3. Smart Notifications (internal/notify/)
   - Event batching with configurable intervals
   - Time-based escalation policies
   - HTML/text/Slack templates

4. Compliance Reports (internal/report/)
   - SOC2, GDPR, HIPAA, PCI-DSS, ISO27001 frameworks
   - Evidence collection from catalog
   - JSON, Markdown, HTML output formats

5. RTO/RPO Calculator (internal/rto/)
   - Recovery objective analysis
   - RTO breakdown by phase
   - Recommendations for improvement

6. Replica-Aware Backup (internal/replica/)
   - Topology detection for PostgreSQL/MySQL
   - Automatic replica selection
   - Configurable selection strategies

7. Parallel Table Backup (internal/parallel/)
   - Concurrent table dumps
   - Worker pool with progress tracking
   - Large table optimization

8. MySQL/MariaDB PITR (internal/pitr/)
   - Binary log parsing and replay
   - Point-in-time recovery support
   - Transaction filtering

CLI commands added: catalog, drill, report, rto

All changes support the goal: reliable 3 AM database recovery.

internal/notify/batch.go

@@ -0,0 +1,261 @@
+// Package notify - Event batching for aggregated notifications
+package notify
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+)
+
+// BatchConfig configures notification batching
+type BatchConfig struct {
+	Enabled      bool          // Enable batching
+	Window       time.Duration // Batch window (e.g., 5 minutes)
+	MaxEvents    int           // Maximum events per batch before forced send
+	GroupBy      string        // Group by: "database", "type", "severity", "host"
+	DigestFormat string        // Format: "summary", "detailed", "compact"
+}
+
+// DefaultBatchConfig returns sensible batch defaults
+func DefaultBatchConfig() BatchConfig {
+	return BatchConfig{
+		Enabled:      false,
+		Window:       5 * time.Minute,
+		MaxEvents:    50,
+		GroupBy:      "database",
+		DigestFormat: "summary",
+	}
+}
+
+// Batcher collects events and sends them in batches
+type Batcher struct {
+	config    BatchConfig
+	manager   *Manager
+	events    []*Event
+	mu        sync.Mutex
+	timer     *time.Timer
+	ctx       context.Context
+	cancel    context.CancelFunc
+	startTime time.Time
+}
+
+// NewBatcher creates a new event batcher
+func NewBatcher(config BatchConfig, manager *Manager) *Batcher {
+	ctx, cancel := context.WithCancel(context.Background())
+	return &Batcher{
+		config:  config,
+		manager: manager,
+		events:  make([]*Event, 0),
+		ctx:     ctx,
+		cancel:  cancel,
+	}
+}
+
+// Add adds an event to the batch
+func (b *Batcher) Add(event *Event) {
+	if !b.config.Enabled {
+		// Batching disabled, send immediately
+		b.manager.Notify(event)
+		return
+	}
+
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	// Start timer on first event
+	if len(b.events) == 0 {
+		b.startTime = time.Now()
+		b.timer = time.AfterFunc(b.config.Window, func() {
+			b.Flush()
+		})
+	}
+
+	b.events = append(b.events, event)
+
+	// Check if we've hit max events
+	if len(b.events) >= b.config.MaxEvents {
+		b.flushLocked()
+	}
+}
+
+// Flush sends all batched events
+func (b *Batcher) Flush() {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.flushLocked()
+}
+
+// flushLocked sends batched events (must hold mutex)
+func (b *Batcher) flushLocked() {
+	if len(b.events) == 0 {
+		return
+	}
+
+	// Cancel pending timer
+	if b.timer != nil {
+		b.timer.Stop()
+		b.timer = nil
+	}
+
+	// Group events
+	groups := b.groupEvents()
+
+	// Create digest event for each group
+	for key, events := range groups {
+		digest := b.createDigest(key, events)
+		b.manager.Notify(digest)
+	}
+
+	// Clear events
+	b.events = make([]*Event, 0)
+}
+
+// groupEvents groups events by configured criteria
+func (b *Batcher) groupEvents() map[string][]*Event {
+	groups := make(map[string][]*Event)
+
+	for _, event := range b.events {
+		var key string
+		switch b.config.GroupBy {
+		case "database":
+			key = event.Database
+		case "type":
+			key = string(event.Type)
+		case "severity":
+			key = string(event.Severity)
+		case "host":
+			key = event.Hostname
+		default:
+			key = "all"
+		}
+		if key == "" {
+			key = "unknown"
+		}
+		groups[key] = append(groups[key], event)
+	}
+
+	return groups
+}
+
+// createDigest creates a digest event from multiple events
+func (b *Batcher) createDigest(groupKey string, events []*Event) *Event {
+	// Calculate summary stats
+	var (
+		successCount  int
+		failureCount  int
+		highestSev    = SeverityInfo
+		totalDuration time.Duration
+		databases     = make(map[string]bool)
+	)
+
+	for _, e := range events {
+		switch e.Type {
+		case EventBackupCompleted, EventRestoreCompleted, EventVerifyCompleted:
+			successCount++
+		case EventBackupFailed, EventRestoreFailed, EventVerifyFailed:
+			failureCount++
+		}
+
+		if severityOrder(e.Severity) > severityOrder(highestSev) {
+			highestSev = e.Severity
+		}
+
+		totalDuration += e.Duration
+		if e.Database != "" {
+			databases[e.Database] = true
+		}
+	}
+
+	// Create digest message
+	var message string
+	switch b.config.DigestFormat {
+	case "detailed":
+		message = b.formatDetailedDigest(events)
+	case "compact":
+		message = b.formatCompactDigest(events, successCount, failureCount)
+	default: // summary
+		message = b.formatSummaryDigest(events, successCount, failureCount, len(databases))
+	}
+
+	digest := NewEvent(EventType("digest"), highestSev, message)
+	digest.WithDetail("group", groupKey)
+	digest.WithDetail("event_count", fmt.Sprintf("%d", len(events)))
+	digest.WithDetail("success_count", fmt.Sprintf("%d", successCount))
+	digest.WithDetail("failure_count", fmt.Sprintf("%d", failureCount))
+	digest.WithDetail("batch_duration", fmt.Sprintf("%.0fs", time.Since(b.startTime).Seconds()))
+
+	if len(databases) == 1 {
+		for db := range databases {
+			digest.Database = db
+		}
+	}
+
+	return digest
+}
+
+func (b *Batcher) formatSummaryDigest(events []*Event, success, failure, dbCount int) string {
+	total := len(events)
+	return fmt.Sprintf("Batch Summary: %d events (%d success, %d failed) across %d database(s)",
+		total, success, failure, dbCount)
+}
+
+func (b *Batcher) formatCompactDigest(events []*Event, success, failure int) string {
+	if failure > 0 {
+		return fmt.Sprintf("⚠️ %d/%d operations failed", failure, len(events))
+	}
+	return fmt.Sprintf("✅ All %d operations successful", success)
+}
+
+func (b *Batcher) formatDetailedDigest(events []*Event) string {
+	var msg string
+	msg += fmt.Sprintf("=== Batch Digest (%d events) ===\n\n", len(events))
+
+	for _, e := range events {
+		icon := "•"
+		switch e.Severity {
+		case SeverityError, SeverityCritical:
+			icon = "❌"
+		case SeverityWarning:
+			icon = "⚠️"
+		}
+
+		msg += fmt.Sprintf("%s [%s] %s: %s\n",
+			icon,
+			e.Timestamp.Format("15:04:05"),
+			e.Type,
+			e.Message)
+	}
+
+	return msg
+}
+
+// Stop stops the batcher and flushes remaining events
+func (b *Batcher) Stop() {
+	b.cancel()
+	b.Flush()
+}
+
+// BatcherStats returns current batcher statistics
+type BatcherStats struct {
+	PendingEvents int           `json:"pending_events"`
+	BatchAge      time.Duration `json:"batch_age"`
+	Config        BatchConfig   `json:"config"`
+}
+
+// Stats returns current batcher statistics
+func (b *Batcher) Stats() BatcherStats {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	var age time.Duration
+	if len(b.events) > 0 {
+		age = time.Since(b.startTime)
+	}
+
+	return BatcherStats{
+		PendingEvents: len(b.events),
+		BatchAge:      age,
+		Config:        b.config,
+	}
+}

internal/notify/escalate.go

@@ -0,0 +1,363 @@
+// Package notify - Escalation for critical events
+package notify
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+)
+
+// EscalationConfig configures notification escalation
+type EscalationConfig struct {
+	Enabled         bool              // Enable escalation
+	Levels          []EscalationLevel // Escalation levels
+	AcknowledgeURL  string            // URL to acknowledge alerts
+	CooldownPeriod  time.Duration     // Cooldown between escalations
+	RepeatInterval  time.Duration     // Repeat unacknowledged alerts
+	MaxRepeats      int               // Maximum repeat attempts
+	TrackingEnabled bool              // Track escalation state
+}
+
+// EscalationLevel defines an escalation tier
+type EscalationLevel struct {
+	Name       string        // Level name (e.g., "primary", "secondary", "manager")
+	Delay      time.Duration // Delay before escalating to this level
+	Recipients []string      // Email recipients for this level
+	Webhook    string        // Webhook URL for this level
+	Severity   Severity      // Minimum severity to escalate
+	Message    string        // Custom message template
+}
+
+// DefaultEscalationConfig returns sensible defaults
+func DefaultEscalationConfig() EscalationConfig {
+	return EscalationConfig{
+		Enabled:        false,
+		CooldownPeriod: 15 * time.Minute,
+		RepeatInterval: 30 * time.Minute,
+		MaxRepeats:     3,
+		Levels: []EscalationLevel{
+			{
+				Name:     "primary",
+				Delay:    0,
+				Severity: SeverityError,
+			},
+			{
+				Name:     "secondary",
+				Delay:    15 * time.Minute,
+				Severity: SeverityError,
+			},
+			{
+				Name:     "critical",
+				Delay:    30 * time.Minute,
+				Severity: SeverityCritical,
+			},
+		},
+	}
+}
+
+// EscalationState tracks escalation for an alert
+type EscalationState struct {
+	AlertID        string     `json:"alert_id"`
+	Event          *Event     `json:"event"`
+	CurrentLevel   int        `json:"current_level"`
+	StartedAt      time.Time  `json:"started_at"`
+	LastEscalation time.Time  `json:"last_escalation"`
+	RepeatCount    int        `json:"repeat_count"`
+	Acknowledged   bool       `json:"acknowledged"`
+	AcknowledgedBy string     `json:"acknowledged_by,omitempty"`
+	AcknowledgedAt *time.Time `json:"acknowledged_at,omitempty"`
+	Resolved       bool       `json:"resolved"`
+}
+
+// Escalator manages alert escalation
+type Escalator struct {
+	config  EscalationConfig
+	manager *Manager
+	alerts  map[string]*EscalationState
+	mu      sync.RWMutex
+	ctx     context.Context
+	cancel  context.CancelFunc
+	ticker  *time.Ticker
+}
+
+// NewEscalator creates a new escalation manager
+func NewEscalator(config EscalationConfig, manager *Manager) *Escalator {
+	ctx, cancel := context.WithCancel(context.Background())
+	e := &Escalator{
+		config:  config,
+		manager: manager,
+		alerts:  make(map[string]*EscalationState),
+		ctx:     ctx,
+		cancel:  cancel,
+	}
+
+	if config.Enabled {
+		e.ticker = time.NewTicker(time.Minute)
+		go e.runEscalationLoop()
+	}
+
+	return e
+}
+
+// Handle processes an event for potential escalation
+func (e *Escalator) Handle(event *Event) {
+	if !e.config.Enabled {
+		return
+	}
+
+	// Only escalate errors and critical events
+	if severityOrder(event.Severity) < severityOrder(SeverityError) {
+		return
+	}
+
+	// Generate alert ID
+	alertID := e.generateAlertID(event)
+
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	// Check if alert already exists
+	if existing, ok := e.alerts[alertID]; ok {
+		if !existing.Acknowledged && !existing.Resolved {
+			// Alert already being escalated
+			return
+		}
+	}
+
+	// Create new escalation state
+	state := &EscalationState{
+		AlertID:        alertID,
+		Event:          event,
+		CurrentLevel:   0,
+		StartedAt:      time.Now(),
+		LastEscalation: time.Now(),
+	}
+
+	e.alerts[alertID] = state
+
+	// Send immediate notification to first level
+	e.notifyLevel(state, 0)
+}
+
+// generateAlertID creates a unique ID for an alert
+func (e *Escalator) generateAlertID(event *Event) string {
+	return fmt.Sprintf("%s_%s_%s",
+		event.Type,
+		event.Database,
+		event.Hostname)
+}
+
+// notifyLevel sends notification for a specific escalation level
+func (e *Escalator) notifyLevel(state *EscalationState, level int) {
+	if level >= len(e.config.Levels) {
+		return
+	}
+
+	lvl := e.config.Levels[level]
+
+	// Create escalated event
+	escalatedEvent := &Event{
+		Type:      state.Event.Type,
+		Severity:  state.Event.Severity,
+		Timestamp: time.Now(),
+		Database:  state.Event.Database,
+		Hostname:  state.Event.Hostname,
+		Message:   e.formatEscalationMessage(state, lvl),
+		Details:   make(map[string]string),
+	}
+
+	escalatedEvent.Details["escalation_level"] = lvl.Name
+	escalatedEvent.Details["alert_id"] = state.AlertID
+	escalatedEvent.Details["escalation_time"] = fmt.Sprintf("%d", int(time.Since(state.StartedAt).Minutes()))
+	escalatedEvent.Details["original_message"] = state.Event.Message
+
+	if state.Event.Error != "" {
+		escalatedEvent.Error = state.Event.Error
+	}
+
+	// Send via manager
+	e.manager.Notify(escalatedEvent)
+
+	state.CurrentLevel = level
+	state.LastEscalation = time.Now()
+}
+
+// formatEscalationMessage creates an escalation message
+func (e *Escalator) formatEscalationMessage(state *EscalationState, level EscalationLevel) string {
+	if level.Message != "" {
+		return level.Message
+	}
+
+	elapsed := time.Since(state.StartedAt)
+	return fmt.Sprintf("🚨 ESCALATION [%s] - Alert unacknowledged for %s\n\n%s",
+		level.Name,
+		formatDuration(elapsed),
+		state.Event.Message)
+}
+
+// runEscalationLoop checks for alerts that need escalation
+func (e *Escalator) runEscalationLoop() {
+	for {
+		select {
+		case <-e.ctx.Done():
+			return
+		case <-e.ticker.C:
+			e.checkEscalations()
+		}
+	}
+}
+
+// checkEscalations checks all alerts for needed escalation
+func (e *Escalator) checkEscalations() {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	now := time.Now()
+
+	for _, state := range e.alerts {
+		if state.Acknowledged || state.Resolved {
+			continue
+		}
+
+		// Check if we need to escalate to next level
+		nextLevel := state.CurrentLevel + 1
+		if nextLevel < len(e.config.Levels) {
+			lvl := e.config.Levels[nextLevel]
+			if now.Sub(state.StartedAt) >= lvl.Delay {
+				e.notifyLevel(state, nextLevel)
+			}
+		}
+
+		// Check if we need to repeat the alert
+		if state.RepeatCount < e.config.MaxRepeats {
+			if now.Sub(state.LastEscalation) >= e.config.RepeatInterval {
+				e.notifyLevel(state, state.CurrentLevel)
+				state.RepeatCount++
+			}
+		}
+	}
+}
+
+// Acknowledge acknowledges an alert
+func (e *Escalator) Acknowledge(alertID, user string) error {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	state, ok := e.alerts[alertID]
+	if !ok {
+		return fmt.Errorf("alert not found: %s", alertID)
+	}
+
+	now := time.Now()
+	state.Acknowledged = true
+	state.AcknowledgedBy = user
+	state.AcknowledgedAt = &now
+
+	return nil
+}
+
+// Resolve resolves an alert
+func (e *Escalator) Resolve(alertID string) error {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	state, ok := e.alerts[alertID]
+	if !ok {
+		return fmt.Errorf("alert not found: %s", alertID)
+	}
+
+	state.Resolved = true
+	return nil
+}
+
+// GetActiveAlerts returns all active (unacknowledged, unresolved) alerts
+func (e *Escalator) GetActiveAlerts() []*EscalationState {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+
+	var active []*EscalationState
+	for _, state := range e.alerts {
+		if !state.Acknowledged && !state.Resolved {
+			active = append(active, state)
+		}
+	}
+	return active
+}
+
+// GetAlert returns a specific alert
+func (e *Escalator) GetAlert(alertID string) (*EscalationState, bool) {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+
+	state, ok := e.alerts[alertID]
+	return state, ok
+}
+
+// CleanupOld removes old resolved/acknowledged alerts
+func (e *Escalator) CleanupOld(maxAge time.Duration) int {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	now := time.Now()
+	removed := 0
+
+	for id, state := range e.alerts {
+		if (state.Acknowledged || state.Resolved) && now.Sub(state.StartedAt) > maxAge {
+			delete(e.alerts, id)
+			removed++
+		}
+	}
+
+	return removed
+}
+
+// Stop stops the escalator
+func (e *Escalator) Stop() {
+	e.cancel()
+	if e.ticker != nil {
+		e.ticker.Stop()
+	}
+}
+
+// EscalatorStats returns escalator statistics
+type EscalatorStats struct {
+	ActiveAlerts       int  `json:"active_alerts"`
+	AcknowledgedAlerts int  `json:"acknowledged_alerts"`
+	ResolvedAlerts     int  `json:"resolved_alerts"`
+	EscalationEnabled  bool `json:"escalation_enabled"`
+	LevelCount         int  `json:"level_count"`
+}
+
+// Stats returns escalator statistics
+func (e *Escalator) Stats() EscalatorStats {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+
+	stats := EscalatorStats{
+		EscalationEnabled: e.config.Enabled,
+		LevelCount:        len(e.config.Levels),
+	}
+
+	for _, state := range e.alerts {
+		if state.Resolved {
+			stats.ResolvedAlerts++
+		} else if state.Acknowledged {
+			stats.AcknowledgedAlerts++
+		} else {
+			stats.ActiveAlerts++
+		}
+	}
+
+	return stats
+}
+
+func formatDuration(d time.Duration) string {
+	if d < time.Minute {
+		return fmt.Sprintf("%.0fs", d.Seconds())
+	}
+	if d < time.Hour {
+		return fmt.Sprintf("%.0fm", d.Minutes())
+	}
+	return fmt.Sprintf("%.0fh %.0fm", d.Hours(), d.Minutes()-d.Hours()*60)
+}

internal/notify/notify.go

@@ -11,41 +11,66 @@ import (
 type EventType string
 
 const (
-	EventBackupStarted   EventType = "backup_started"
-	EventBackupCompleted EventType = "backup_completed"
-	EventBackupFailed    EventType = "backup_failed"
-	EventRestoreStarted  EventType = "restore_started"
-	EventRestoreCompleted EventType = "restore_completed"
-	EventRestoreFailed   EventType = "restore_failed"
-	EventCleanupCompleted EventType = "cleanup_completed"
-	EventVerifyCompleted EventType = "verify_completed"
-	EventVerifyFailed    EventType = "verify_failed"
-	EventPITRRecovery    EventType = "pitr_recovery"
+	EventBackupStarted      EventType = "backup_started"
+	EventBackupCompleted    EventType = "backup_completed"
+	EventBackupFailed       EventType = "backup_failed"
+	EventRestoreStarted     EventType = "restore_started"
+	EventRestoreCompleted   EventType = "restore_completed"
+	EventRestoreFailed      EventType = "restore_failed"
+	EventCleanupCompleted   EventType = "cleanup_completed"
+	EventVerifyCompleted    EventType = "verify_completed"
+	EventVerifyFailed       EventType = "verify_failed"
+	EventPITRRecovery       EventType = "pitr_recovery"
+	EventVerificationPassed EventType = "verification_passed"
+	EventVerificationFailed EventType = "verification_failed"
+	EventDRDrillPassed      EventType = "dr_drill_passed"
+	EventDRDrillFailed      EventType = "dr_drill_failed"
+	EventGapDetected        EventType = "gap_detected"
+	EventRPOViolation       EventType = "rpo_violation"
 )
 
 // Severity represents the severity level of a notification
 type Severity string
 
 const (
-	SeverityInfo    Severity = "info"
-	SeverityWarning Severity = "warning"
-	SeverityError   Severity = "error"
+	SeverityInfo     Severity = "info"
+	SeveritySuccess  Severity = "success"
+	SeverityWarning  Severity = "warning"
+	SeverityError    Severity = "error"
 	SeverityCritical Severity = "critical"
 )
 
+// severityOrder returns numeric order for severity comparison
+func severityOrder(s Severity) int {
+	switch s {
+	case SeverityInfo:
+		return 0
+	case SeveritySuccess:
+		return 1
+	case SeverityWarning:
+		return 2
+	case SeverityError:
+		return 3
+	case SeverityCritical:
+		return 4
+	default:
+		return 0
+	}
+}
+
 // Event represents a notification event
 type Event struct {
-	Type        EventType         `json:"type"`
-	Severity    Severity          `json:"severity"`
-	Timestamp   time.Time         `json:"timestamp"`
-	Database    string            `json:"database,omitempty"`
-	Message     string            `json:"message"`
-	Details     map[string]string `json:"details,omitempty"`
-	Error       string            `json:"error,omitempty"`
-	Duration    time.Duration     `json:"duration,omitempty"`
-	BackupFile  string            `json:"backup_file,omitempty"`
-	BackupSize  int64             `json:"backup_size,omitempty"`
-	Hostname    string            `json:"hostname,omitempty"`
+	Type       EventType         `json:"type"`
+	Severity   Severity          `json:"severity"`
+	Timestamp  time.Time         `json:"timestamp"`
+	Database   string            `json:"database,omitempty"`
+	Message    string            `json:"message"`
+	Details    map[string]string `json:"details,omitempty"`
+	Error      string            `json:"error,omitempty"`
+	Duration   time.Duration     `json:"duration,omitempty"`
+	BackupFile string            `json:"backup_file,omitempty"`
+	BackupSize int64             `json:"backup_size,omitempty"`
+	Hostname   string            `json:"hostname,omitempty"`
 }
 
 // NewEvent creates a new notification event
@@ -132,27 +157,27 @@ type Config struct {
 	WebhookSecret  string // For signing payloads
 
 	// General settings
-	OnSuccess    bool // Send notifications on successful operations
-	OnFailure    bool // Send notifications on failed operations
-	OnWarning    bool // Send notifications on warnings
-	MinSeverity  Severity
-	Retries      int           // Number of retry attempts
-	RetryDelay   time.Duration // Delay between retries
+	OnSuccess   bool // Send notifications on successful operations
+	OnFailure   bool // Send notifications on failed operations
+	OnWarning   bool // Send notifications on warnings
+	MinSeverity Severity
+	Retries     int           // Number of retry attempts
+	RetryDelay  time.Duration // Delay between retries
 }
 
 // DefaultConfig returns a configuration with sensible defaults
 func DefaultConfig() Config {
 	return Config{
-		SMTPPort:     587,
-		SMTPTLS:      false,
-		SMTPStartTLS: true,
+		SMTPPort:      587,
+		SMTPTLS:       false,
+		SMTPStartTLS:  true,
 		WebhookMethod: "POST",
-		OnSuccess:    true,
-		OnFailure:    true,
-		OnWarning:    true,
-		MinSeverity:  SeverityInfo,
-		Retries:      3,
-		RetryDelay:   5 * time.Second,
+		OnSuccess:     true,
+		OnFailure:     true,
+		OnWarning:     true,
+		MinSeverity:   SeverityInfo,
+		Retries:       3,
+		RetryDelay:    5 * time.Second,
 	}
 }
 

internal/notify/templates.go

@@ -0,0 +1,497 @@
+// Package notify - Notification templates
+package notify
+
+import (
+	"bytes"
+	"fmt"
+	"html/template"
+	"strings"
+	"time"
+)
+
+// TemplateType represents the notification format type
+type TemplateType string
+
+const (
+	TemplateText     TemplateType = "text"
+	TemplateHTML     TemplateType = "html"
+	TemplateMarkdown TemplateType = "markdown"
+	TemplateSlack    TemplateType = "slack"
+)
+
+// Templates holds notification templates
+type Templates struct {
+	Subject  string
+	TextBody string
+	HTMLBody string
+}
+
+// DefaultTemplates returns default notification templates
+func DefaultTemplates() map[EventType]Templates {
+	return map[EventType]Templates{
+		EventBackupStarted: {
+			Subject:  "🔄 Backup Started: {{.Database}} on {{.Hostname}}",
+			TextBody: backupStartedText,
+			HTMLBody: backupStartedHTML,
+		},
+		EventBackupCompleted: {
+			Subject:  "✅ Backup Completed: {{.Database}} on {{.Hostname}}",
+			TextBody: backupCompletedText,
+			HTMLBody: backupCompletedHTML,
+		},
+		EventBackupFailed: {
+			Subject:  "❌ Backup FAILED: {{.Database}} on {{.Hostname}}",
+			TextBody: backupFailedText,
+			HTMLBody: backupFailedHTML,
+		},
+		EventRestoreStarted: {
+			Subject:  "🔄 Restore Started: {{.Database}} on {{.Hostname}}",
+			TextBody: restoreStartedText,
+			HTMLBody: restoreStartedHTML,
+		},
+		EventRestoreCompleted: {
+			Subject:  "✅ Restore Completed: {{.Database}} on {{.Hostname}}",
+			TextBody: restoreCompletedText,
+			HTMLBody: restoreCompletedHTML,
+		},
+		EventRestoreFailed: {
+			Subject:  "❌ Restore FAILED: {{.Database}} on {{.Hostname}}",
+			TextBody: restoreFailedText,
+			HTMLBody: restoreFailedHTML,
+		},
+		EventVerificationPassed: {
+			Subject:  "✅ Verification Passed: {{.Database}}",
+			TextBody: verificationPassedText,
+			HTMLBody: verificationPassedHTML,
+		},
+		EventVerificationFailed: {
+			Subject:  "❌ Verification FAILED: {{.Database}}",
+			TextBody: verificationFailedText,
+			HTMLBody: verificationFailedHTML,
+		},
+		EventDRDrillPassed: {
+			Subject:  "✅ DR Drill Passed: {{.Database}}",
+			TextBody: drDrillPassedText,
+			HTMLBody: drDrillPassedHTML,
+		},
+		EventDRDrillFailed: {
+			Subject:  "❌ DR Drill FAILED: {{.Database}}",
+			TextBody: drDrillFailedText,
+			HTMLBody: drDrillFailedHTML,
+		},
+	}
+}
+
+// Template strings
+const backupStartedText = `
+Backup Operation Started
+
+Database:   {{.Database}}
+Hostname:   {{.Hostname}}
+Started At: {{formatTime .Timestamp}}
+
+{{if .Message}}{{.Message}}{{end}}
+`
+
+const backupStartedHTML = `
+<div style="font-family: Arial, sans-serif; padding: 20px;">
+  <h2 style="color: #3498db;">🔄 Backup Started</h2>
+  <table style="border-collapse: collapse; width: 100%; max-width: 600px;">
+    <tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
+    <tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
+    <tr><td style="padding: 8px; font-weight: bold;">Started At:</td><td style="padding: 8px;">{{formatTime .Timestamp}}</td></tr>
+  </table>
+  {{if .Message}}<p style="margin-top: 20px;">{{.Message}}</p>{{end}}
+</div>
+`
+
+const backupCompletedText = `
+Backup Operation Completed Successfully
+
+Database:   {{.Database}}
+Hostname:   {{.Hostname}}
+Completed:  {{formatTime .Timestamp}}
+{{with .Details}}
+{{if .size}}Size:       {{.size}}{{end}}
+{{if .duration}}Duration:   {{.duration}}{{end}}
+{{if .path}}Path:       {{.path}}{{end}}
+{{end}}
+{{if .Message}}{{.Message}}{{end}}
+`
+
+const backupCompletedHTML = `
+<div style="font-family: Arial, sans-serif; padding: 20px;">
+  <h2 style="color: #27ae60;">✅ Backup Completed</h2>
+  <table style="border-collapse: collapse; width: 100%; max-width: 600px;">
+    <tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
+    <tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
+    <tr><td style="padding: 8px; font-weight: bold;">Completed:</td><td style="padding: 8px;">{{formatTime .Timestamp}}</td></tr>
+    {{with .Details}}
+    {{if .size}}<tr><td style="padding: 8px; font-weight: bold;">Size:</td><td style="padding: 8px;">{{.size}}</td></tr>{{end}}
+    {{if .duration}}<tr><td style="padding: 8px; font-weight: bold;">Duration:</td><td style="padding: 8px;">{{.duration}}</td></tr>{{end}}
+    {{if .path}}<tr><td style="padding: 8px; font-weight: bold;">Path:</td><td style="padding: 8px;">{{.path}}</td></tr>{{end}}
+    {{end}}
+  </table>
+  {{if .Message}}<p style="margin-top: 20px; color: #27ae60;">{{.Message}}</p>{{end}}
+</div>
+`
+
+const backupFailedText = `
+⚠️ BACKUP FAILED ⚠️
+
+Database:   {{.Database}}
+Hostname:   {{.Hostname}}
+Failed At:  {{formatTime .Timestamp}}
+{{if .Error}}
+Error:      {{.Error}}
+{{end}}
+{{if .Message}}{{.Message}}{{end}}
+
+Please investigate immediately.
+`
+
+const backupFailedHTML = `
+<div style="font-family: Arial, sans-serif; padding: 20px;">
+  <h2 style="color: #e74c3c;">❌ Backup FAILED</h2>
+  <table style="border-collapse: collapse; width: 100%; max-width: 600px;">
+    <tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
+    <tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
+    <tr><td style="padding: 8px; font-weight: bold;">Failed At:</td><td style="padding: 8px;">{{formatTime .Timestamp}}</td></tr>
+    {{if .Error}}<tr><td style="padding: 8px; font-weight: bold; color: #e74c3c;">Error:</td><td style="padding: 8px; color: #e74c3c;">{{.Error}}</td></tr>{{end}}
+  </table>
+  {{if .Message}}<p style="margin-top: 20px;">{{.Message}}</p>{{end}}
+  <p style="margin-top: 20px; color: #e74c3c; font-weight: bold;">Please investigate immediately.</p>
+</div>
+`
+
+const restoreStartedText = `
+Restore Operation Started
+
+Database:   {{.Database}}
+Hostname:   {{.Hostname}}
+Started At: {{formatTime .Timestamp}}
+
+{{if .Message}}{{.Message}}{{end}}
+`
+
+const restoreStartedHTML = `
+<div style="font-family: Arial, sans-serif; padding: 20px;">
+  <h2 style="color: #3498db;">🔄 Restore Started</h2>
+  <table style="border-collapse: collapse; width: 100%; max-width: 600px;">
+    <tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
+    <tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
+    <tr><td style="padding: 8px; font-weight: bold;">Started At:</td><td style="padding: 8px;">{{formatTime .Timestamp}}</td></tr>
+  </table>
+  {{if .Message}}<p style="margin-top: 20px;">{{.Message}}</p>{{end}}
+</div>
+`
+
+const restoreCompletedText = `
+Restore Operation Completed Successfully
+
+Database:   {{.Database}}
+Hostname:   {{.Hostname}}
+Completed:  {{formatTime .Timestamp}}
+{{with .Details}}
+{{if .duration}}Duration:   {{.duration}}{{end}}
+{{end}}
+{{if .Message}}{{.Message}}{{end}}
+`
+
+const restoreCompletedHTML = `
+<div style="font-family: Arial, sans-serif; padding: 20px;">
+  <h2 style="color: #27ae60;">✅ Restore Completed</h2>
+  <table style="border-collapse: collapse; width: 100%; max-width: 600px;">
+    <tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
+    <tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
+    <tr><td style="padding: 8px; font-weight: bold;">Completed:</td><td style="padding: 8px;">{{formatTime .Timestamp}}</td></tr>
+    {{with .Details}}
+    {{if .duration}}<tr><td style="padding: 8px; font-weight: bold;">Duration:</td><td style="padding: 8px;">{{.duration}}</td></tr>{{end}}
+    {{end}}
+  </table>
+  {{if .Message}}<p style="margin-top: 20px; color: #27ae60;">{{.Message}}</p>{{end}}
+</div>
+`
+
+const restoreFailedText = `
+⚠️ RESTORE FAILED ⚠️
+
+Database:   {{.Database}}
+Hostname:   {{.Hostname}}
+Failed At:  {{formatTime .Timestamp}}
+{{if .Error}}
+Error:      {{.Error}}
+{{end}}
+{{if .Message}}{{.Message}}{{end}}
+
+Please investigate immediately.
+`
+
+const restoreFailedHTML = `
+<div style="font-family: Arial, sans-serif; padding: 20px;">
+  <h2 style="color: #e74c3c;">❌ Restore FAILED</h2>
+  <table style="border-collapse: collapse; width: 100%; max-width: 600px;">
+    <tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
+    <tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
+    <tr><td style="padding: 8px; font-weight: bold;">Failed At:</td><td style="padding: 8px;">{{formatTime .Timestamp}}</td></tr>
+    {{if .Error}}<tr><td style="padding: 8px; font-weight: bold; color: #e74c3c;">Error:</td><td style="padding: 8px; color: #e74c3c;">{{.Error}}</td></tr>{{end}}
+  </table>
+  {{if .Message}}<p style="margin-top: 20px;">{{.Message}}</p>{{end}}
+  <p style="margin-top: 20px; color: #e74c3c; font-weight: bold;">Please investigate immediately.</p>
+</div>
+`
+
+const verificationPassedText = `
+Backup Verification Passed
+
+Database:   {{.Database}}
+Hostname:   {{.Hostname}}
+Verified:   {{formatTime .Timestamp}}
+{{with .Details}}
+{{if .checksum}}Checksum:   {{.checksum}}{{end}}
+{{end}}
+{{if .Message}}{{.Message}}{{end}}
+`
+
+const verificationPassedHTML = `
+<div style="font-family: Arial, sans-serif; padding: 20px;">
+  <h2 style="color: #27ae60;">✅ Verification Passed</h2>
+  <table style="border-collapse: collapse; width: 100%; max-width: 600px;">
+    <tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
+    <tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
+    <tr><td style="padding: 8px; font-weight: bold;">Verified:</td><td style="padding: 8px;">{{formatTime .Timestamp}}</td></tr>
+    {{with .Details}}
+    {{if .checksum}}<tr><td style="padding: 8px; font-weight: bold;">Checksum:</td><td style="padding: 8px; font-family: monospace;">{{.checksum}}</td></tr>{{end}}
+    {{end}}
+  </table>
+  {{if .Message}}<p style="margin-top: 20px; color: #27ae60;">{{.Message}}</p>{{end}}
+</div>
+`
+
+const verificationFailedText = `
+⚠️ VERIFICATION FAILED ⚠️
+
+Database:   {{.Database}}
+Hostname:   {{.Hostname}}
+Failed At:  {{formatTime .Timestamp}}
+{{if .Error}}
+Error:      {{.Error}}
+{{end}}
+{{if .Message}}{{.Message}}{{end}}
+
+Backup integrity may be compromised. Please investigate.
+`
+
+const verificationFailedHTML = `
+<div style="font-family: Arial, sans-serif; padding: 20px;">
+  <h2 style="color: #e74c3c;">❌ Verification FAILED</h2>
+  <table style="border-collapse: collapse; width: 100%; max-width: 600px;">
+    <tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
+    <tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
+    <tr><td style="padding: 8px; font-weight: bold;">Failed At:</td><td style="padding: 8px;">{{formatTime .Timestamp}}</td></tr>
+    {{if .Error}}<tr><td style="padding: 8px; font-weight: bold; color: #e74c3c;">Error:</td><td style="padding: 8px; color: #e74c3c;">{{.Error}}</td></tr>{{end}}
+  </table>
+  {{if .Message}}<p style="margin-top: 20px;">{{.Message}}</p>{{end}}
+  <p style="margin-top: 20px; color: #e74c3c; font-weight: bold;">Backup integrity may be compromised. Please investigate.</p>
+</div>
+`
+
+const drDrillPassedText = `
+DR Drill Test Passed
+
+Database:   {{.Database}}
+Hostname:   {{.Hostname}}
+Tested At:  {{formatTime .Timestamp}}
+{{with .Details}}
+{{if .tables_restored}}Tables:     {{.tables_restored}}{{end}}
+{{if .rows_validated}}Rows:       {{.rows_validated}}{{end}}
+{{if .duration}}Duration:   {{.duration}}{{end}}
+{{end}}
+{{if .Message}}{{.Message}}{{end}}
+
+Backup restore capability verified.
+`
+
+const drDrillPassedHTML = `
+<div style="font-family: Arial, sans-serif; padding: 20px;">
+  <h2 style="color: #27ae60;">✅ DR Drill Passed</h2>
+  <table style="border-collapse: collapse; width: 100%; max-width: 600px;">
+    <tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
+    <tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
+    <tr><td style="padding: 8px; font-weight: bold;">Tested At:</td><td style="padding: 8px;">{{formatTime .Timestamp}}</td></tr>
+    {{with .Details}}
+    {{if .tables_restored}}<tr><td style="padding: 8px; font-weight: bold;">Tables:</td><td style="padding: 8px;">{{.tables_restored}}</td></tr>{{end}}
+    {{if .rows_validated}}<tr><td style="padding: 8px; font-weight: bold;">Rows:</td><td style="padding: 8px;">{{.rows_validated}}</td></tr>{{end}}
+    {{if .duration}}<tr><td style="padding: 8px; font-weight: bold;">Duration:</td><td style="padding: 8px;">{{.duration}}</td></tr>{{end}}
+    {{end}}
+  </table>
+  {{if .Message}}<p style="margin-top: 20px; color: #27ae60;">{{.Message}}</p>{{end}}
+  <p style="margin-top: 20px; color: #27ae60;">✓ Backup restore capability verified</p>
+</div>
+`
+
+const drDrillFailedText = `
+⚠️ DR DRILL FAILED ⚠️
+
+Database:   {{.Database}}
+Hostname:   {{.Hostname}}
+Failed At:  {{formatTime .Timestamp}}
+{{if .Error}}
+Error:      {{.Error}}
+{{end}}
+{{if .Message}}{{.Message}}{{end}}
+
+Backup may not be restorable. Please investigate immediately.
+`
+
+const drDrillFailedHTML = `
+<div style="font-family: Arial, sans-serif; padding: 20px;">
+  <h2 style="color: #e74c3c;">❌ DR Drill FAILED</h2>
+  <table style="border-collapse: collapse; width: 100%; max-width: 600px;">
+    <tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
+    <tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
+    <tr><td style="padding: 8px; font-weight: bold;">Failed At:</td><td style="padding: 8px;">{{formatTime .Timestamp}}</td></tr>
+    {{if .Error}}<tr><td style="padding: 8px; font-weight: bold; color: #e74c3c;">Error:</td><td style="padding: 8px; color: #e74c3c;">{{.Error}}</td></tr>{{end}}
+  </table>
+  {{if .Message}}<p style="margin-top: 20px;">{{.Message}}</p>{{end}}
+  <p style="margin-top: 20px; color: #e74c3c; font-weight: bold;">Backup may not be restorable. Please investigate immediately.</p>
+</div>
+`
+
+// TemplateRenderer renders notification templates
+type TemplateRenderer struct {
+	templates map[EventType]Templates
+	funcMap   template.FuncMap
+}
+
+// NewTemplateRenderer creates a new template renderer
+func NewTemplateRenderer() *TemplateRenderer {
+	return &TemplateRenderer{
+		templates: DefaultTemplates(),
+		funcMap: template.FuncMap{
+			"formatTime": func(t time.Time) string {
+				return t.Format("2006-01-02 15:04:05 MST")
+			},
+			"upper": strings.ToUpper,
+			"lower": strings.ToLower,
+		},
+	}
+}
+
+// RenderSubject renders the subject template for an event
+func (r *TemplateRenderer) RenderSubject(event *Event) (string, error) {
+	tmpl, ok := r.templates[event.Type]
+	if !ok {
+		return fmt.Sprintf("[%s] %s: %s", event.Severity, event.Type, event.Database), nil
+	}
+
+	return r.render(tmpl.Subject, event)
+}
+
+// RenderText renders the text body template for an event
+func (r *TemplateRenderer) RenderText(event *Event) (string, error) {
+	tmpl, ok := r.templates[event.Type]
+	if !ok {
+		return event.Message, nil
+	}
+
+	return r.render(tmpl.TextBody, event)
+}
+
+// RenderHTML renders the HTML body template for an event
+func (r *TemplateRenderer) RenderHTML(event *Event) (string, error) {
+	tmpl, ok := r.templates[event.Type]
+	if !ok {
+		return fmt.Sprintf("<p>%s</p>", event.Message), nil
+	}
+
+	return r.render(tmpl.HTMLBody, event)
+}
+
+// render executes a template with the given event
+func (r *TemplateRenderer) render(templateStr string, event *Event) (string, error) {
+	tmpl, err := template.New("notification").Funcs(r.funcMap).Parse(templateStr)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse template: %w", err)
+	}
+
+	var buf bytes.Buffer
+	if err := tmpl.Execute(&buf, event); err != nil {
+		return "", fmt.Errorf("failed to execute template: %w", err)
+	}
+
+	return strings.TrimSpace(buf.String()), nil
+}
+
+// SetTemplate sets a custom template for an event type
+func (r *TemplateRenderer) SetTemplate(eventType EventType, templates Templates) {
+	r.templates[eventType] = templates
+}
+
+// RenderSlackMessage creates a Slack-formatted message
+func (r *TemplateRenderer) RenderSlackMessage(event *Event) map[string]interface{} {
+	color := "#3498db" // blue
+	switch event.Severity {
+	case SeveritySuccess:
+		color = "#27ae60" // green
+	case SeverityWarning:
+		color = "#f39c12" // orange
+	case SeverityError, SeverityCritical:
+		color = "#e74c3c" // red
+	}
+
+	fields := []map[string]interface{}{
+		{
+			"title": "Database",
+			"value": event.Database,
+			"short": true,
+		},
+		{
+			"title": "Hostname",
+			"value": event.Hostname,
+			"short": true,
+		},
+		{
+			"title": "Event",
+			"value": string(event.Type),
+			"short": true,
+		},
+		{
+			"title": "Severity",
+			"value": string(event.Severity),
+			"short": true,
+		},
+	}
+
+	if event.Error != "" {
+		fields = append(fields, map[string]interface{}{
+			"title": "Error",
+			"value": event.Error,
+			"short": false,
+		})
+	}
+
+	for key, value := range event.Details {
+		fields = append(fields, map[string]interface{}{
+			"title": key,
+			"value": value,
+			"short": true,
+		})
+	}
+
+	subject, _ := r.RenderSubject(event)
+
+	return map[string]interface{}{
+		"attachments": []map[string]interface{}{
+			{
+				"color":     color,
+				"title":     subject,
+				"text":      event.Message,
+				"fields":    fields,
+				"footer":    "dbbackup",
+				"ts":        event.Timestamp.Unix(),
+				"mrkdwn_in": []string{"text", "fields"},
+			},
+		},
+	}
+}