Build all platforms v3.42.22

2026-01-12 09:54:35 +01:00
parent b1f8c6d646
commit 59d2094241
6 changed files with 1481 additions and 24 deletions
--- a/internal/dedup/metrics.go
+++ b/internal/dedup/metrics.go
@@ -0,0 +1,235 @@
+package dedup
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+// DedupMetrics holds deduplication statistics for Prometheus
+type DedupMetrics struct {
+	// Global stats
+	TotalChunks     int64
+	TotalManifests  int64
+	TotalBackupSize int64   // Sum of all backup original sizes
+	TotalNewData    int64   // Sum of all new chunks stored
+	SpaceSaved      int64   // Bytes saved by deduplication
+	DedupRatio      float64 // Overall dedup ratio (0-1)
+	DiskUsage       int64   // Actual bytes on disk
+
+	// Per-database stats
+	ByDatabase map[string]*DatabaseDedupMetrics
+}
+
+// DatabaseDedupMetrics holds per-database dedup stats
+type DatabaseDedupMetrics struct {
+	Database       string
+	BackupCount    int
+	TotalSize      int64
+	StoredSize     int64
+	DedupRatio     float64
+	LastBackupTime time.Time
+	LastVerified   time.Time
+}
+
+// CollectMetrics gathers dedup statistics from the index and store
+func CollectMetrics(basePath string, indexPath string) (*DedupMetrics, error) {
+	var idx *ChunkIndex
+	var err error
+
+	if indexPath != "" {
+		idx, err = NewChunkIndexAt(indexPath)
+	} else {
+		idx, err = NewChunkIndex(basePath)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to open chunk index: %w", err)
+	}
+	defer idx.Close()
+
+	store, err := NewChunkStore(StoreConfig{BasePath: basePath})
+	if err != nil {
+		return nil, fmt.Errorf("failed to open chunk store: %w", err)
+	}
+
+	// Get index stats
+	stats, err := idx.Stats()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get index stats: %w", err)
+	}
+
+	// Get store stats
+	storeStats, err := store.Stats()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get store stats: %w", err)
+	}
+
+	metrics := &DedupMetrics{
+		TotalChunks:     stats.TotalChunks,
+		TotalManifests:  stats.TotalManifests,
+		TotalBackupSize: stats.TotalBackupSize,
+		TotalNewData:    stats.TotalNewData,
+		SpaceSaved:      stats.SpaceSaved,
+		DedupRatio:      stats.DedupRatio,
+		DiskUsage:       storeStats.TotalSize,
+		ByDatabase:      make(map[string]*DatabaseDedupMetrics),
+	}
+
+	// Collect per-database metrics from manifest store
+	manifestStore, err := NewManifestStore(basePath)
+	if err != nil {
+		return metrics, nil // Return partial metrics
+	}
+
+	manifests, err := manifestStore.ListAll()
+	if err != nil {
+		return metrics, nil // Return partial metrics
+	}
+
+	for _, m := range manifests {
+		dbKey := m.DatabaseName
+		if dbKey == "" {
+			dbKey = "_default"
+		}
+
+		dbMetrics, ok := metrics.ByDatabase[dbKey]
+		if !ok {
+			dbMetrics = &DatabaseDedupMetrics{
+				Database: dbKey,
+			}
+			metrics.ByDatabase[dbKey] = dbMetrics
+		}
+
+		dbMetrics.BackupCount++
+		dbMetrics.TotalSize += m.OriginalSize
+		dbMetrics.StoredSize += m.StoredSize
+
+		if m.CreatedAt.After(dbMetrics.LastBackupTime) {
+			dbMetrics.LastBackupTime = m.CreatedAt
+		}
+		if !m.VerifiedAt.IsZero() && m.VerifiedAt.After(dbMetrics.LastVerified) {
+			dbMetrics.LastVerified = m.VerifiedAt
+		}
+	}
+
+	// Calculate per-database dedup ratios
+	for _, dbMetrics := range metrics.ByDatabase {
+		if dbMetrics.TotalSize > 0 {
+			dbMetrics.DedupRatio = 1.0 - float64(dbMetrics.StoredSize)/float64(dbMetrics.TotalSize)
+		}
+	}
+
+	return metrics, nil
+}
+
+// WritePrometheusTextfile writes dedup metrics in Prometheus format
+func WritePrometheusTextfile(path string, instance string, basePath string, indexPath string) error {
+	metrics, err := CollectMetrics(basePath, indexPath)
+	if err != nil {
+		return err
+	}
+
+	output := FormatPrometheusMetrics(metrics, instance)
+
+	// Atomic write
+	dir := filepath.Dir(path)
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return fmt.Errorf("failed to create directory: %w", err)
+	}
+
+	tmpPath := path + ".tmp"
+	if err := os.WriteFile(tmpPath, []byte(output), 0644); err != nil {
+		return fmt.Errorf("failed to write temp file: %w", err)
+	}
+
+	if err := os.Rename(tmpPath, path); err != nil {
+		os.Remove(tmpPath)
+		return fmt.Errorf("failed to rename temp file: %w", err)
+	}
+
+	return nil
+}
+
+// FormatPrometheusMetrics formats dedup metrics in Prometheus exposition format
+func FormatPrometheusMetrics(m *DedupMetrics, instance string) string {
+	var b strings.Builder
+	now := time.Now().Unix()
+
+	b.WriteString("# DBBackup Deduplication Prometheus Metrics\n")
+	b.WriteString(fmt.Sprintf("# Generated at: %s\n", time.Now().Format(time.RFC3339)))
+	b.WriteString(fmt.Sprintf("# Instance: %s\n", instance))
+	b.WriteString("\n")
+
+	// Global dedup metrics
+	b.WriteString("# HELP dbbackup_dedup_chunks_total Total number of unique chunks stored\n")
+	b.WriteString("# TYPE dbbackup_dedup_chunks_total gauge\n")
+	b.WriteString(fmt.Sprintf("dbbackup_dedup_chunks_total{instance=%q} %d\n", instance, m.TotalChunks))
+	b.WriteString("\n")
+
+	b.WriteString("# HELP dbbackup_dedup_manifests_total Total number of deduplicated backups\n")
+	b.WriteString("# TYPE dbbackup_dedup_manifests_total gauge\n")
+	b.WriteString(fmt.Sprintf("dbbackup_dedup_manifests_total{instance=%q} %d\n", instance, m.TotalManifests))
+	b.WriteString("\n")
+
+	b.WriteString("# HELP dbbackup_dedup_backup_bytes_total Total logical size of all backups in bytes\n")
+	b.WriteString("# TYPE dbbackup_dedup_backup_bytes_total gauge\n")
+	b.WriteString(fmt.Sprintf("dbbackup_dedup_backup_bytes_total{instance=%q} %d\n", instance, m.TotalBackupSize))
+	b.WriteString("\n")
+
+	b.WriteString("# HELP dbbackup_dedup_stored_bytes_total Total unique data stored in bytes (after dedup)\n")
+	b.WriteString("# TYPE dbbackup_dedup_stored_bytes_total gauge\n")
+	b.WriteString(fmt.Sprintf("dbbackup_dedup_stored_bytes_total{instance=%q} %d\n", instance, m.TotalNewData))
+	b.WriteString("\n")
+
+	b.WriteString("# HELP dbbackup_dedup_space_saved_bytes Bytes saved by deduplication\n")
+	b.WriteString("# TYPE dbbackup_dedup_space_saved_bytes gauge\n")
+	b.WriteString(fmt.Sprintf("dbbackup_dedup_space_saved_bytes{instance=%q} %d\n", instance, m.SpaceSaved))
+	b.WriteString("\n")
+
+	b.WriteString("# HELP dbbackup_dedup_ratio Deduplication ratio (0-1, higher is better)\n")
+	b.WriteString("# TYPE dbbackup_dedup_ratio gauge\n")
+	b.WriteString(fmt.Sprintf("dbbackup_dedup_ratio{instance=%q} %.4f\n", instance, m.DedupRatio))
+	b.WriteString("\n")
+
+	b.WriteString("# HELP dbbackup_dedup_disk_usage_bytes Actual disk usage of chunk store\n")
+	b.WriteString("# TYPE dbbackup_dedup_disk_usage_bytes gauge\n")
+	b.WriteString(fmt.Sprintf("dbbackup_dedup_disk_usage_bytes{instance=%q} %d\n", instance, m.DiskUsage))
+	b.WriteString("\n")
+
+	// Per-database metrics
+	if len(m.ByDatabase) > 0 {
+		b.WriteString("# HELP dbbackup_dedup_database_backup_count Number of deduplicated backups per database\n")
+		b.WriteString("# TYPE dbbackup_dedup_database_backup_count gauge\n")
+		for _, db := range m.ByDatabase {
+			b.WriteString(fmt.Sprintf("dbbackup_dedup_database_backup_count{instance=%q,database=%q} %d\n",
+				instance, db.Database, db.BackupCount))
+		}
+		b.WriteString("\n")
+
+		b.WriteString("# HELP dbbackup_dedup_database_ratio Deduplication ratio per database (0-1)\n")
+		b.WriteString("# TYPE dbbackup_dedup_database_ratio gauge\n")
+		for _, db := range m.ByDatabase {
+			b.WriteString(fmt.Sprintf("dbbackup_dedup_database_ratio{instance=%q,database=%q} %.4f\n",
+				instance, db.Database, db.DedupRatio))
+		}
+		b.WriteString("\n")
+
+		b.WriteString("# HELP dbbackup_dedup_database_last_backup_timestamp Last backup timestamp per database\n")
+		b.WriteString("# TYPE dbbackup_dedup_database_last_backup_timestamp gauge\n")
+		for _, db := range m.ByDatabase {
+			if !db.LastBackupTime.IsZero() {
+				b.WriteString(fmt.Sprintf("dbbackup_dedup_database_last_backup_timestamp{instance=%q,database=%q} %d\n",
+					instance, db.Database, db.LastBackupTime.Unix()))
+			}
+		}
+		b.WriteString("\n")
+	}
+
+	b.WriteString("# HELP dbbackup_dedup_scrape_timestamp Unix timestamp when dedup metrics were collected\n")
+	b.WriteString("# TYPE dbbackup_dedup_scrape_timestamp gauge\n")
+	b.WriteString(fmt.Sprintf("dbbackup_dedup_scrape_timestamp{instance=%q} %d\n", instance, now))
+
+	return b.String()
+}