236 lines
7.7 KiB
Go
236 lines
7.7 KiB
Go
package dedup
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// DedupMetrics holds deduplication statistics for Prometheus
|
|
type DedupMetrics struct {
|
|
// Global stats
|
|
TotalChunks int64
|
|
TotalManifests int64
|
|
TotalBackupSize int64 // Sum of all backup original sizes
|
|
TotalNewData int64 // Sum of all new chunks stored
|
|
SpaceSaved int64 // Bytes saved by deduplication
|
|
DedupRatio float64 // Overall dedup ratio (0-1)
|
|
DiskUsage int64 // Actual bytes on disk
|
|
|
|
// Per-database stats
|
|
ByDatabase map[string]*DatabaseDedupMetrics
|
|
}
|
|
|
|
// DatabaseDedupMetrics holds per-database dedup stats
|
|
type DatabaseDedupMetrics struct {
|
|
Database string
|
|
BackupCount int
|
|
TotalSize int64
|
|
StoredSize int64
|
|
DedupRatio float64
|
|
LastBackupTime time.Time
|
|
LastVerified time.Time
|
|
}
|
|
|
|
// CollectMetrics gathers dedup statistics from the index and store
|
|
func CollectMetrics(basePath string, indexPath string) (*DedupMetrics, error) {
|
|
var idx *ChunkIndex
|
|
var err error
|
|
|
|
if indexPath != "" {
|
|
idx, err = NewChunkIndexAt(indexPath)
|
|
} else {
|
|
idx, err = NewChunkIndex(basePath)
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to open chunk index: %w", err)
|
|
}
|
|
defer idx.Close()
|
|
|
|
store, err := NewChunkStore(StoreConfig{BasePath: basePath})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to open chunk store: %w", err)
|
|
}
|
|
|
|
// Get index stats
|
|
stats, err := idx.Stats()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get index stats: %w", err)
|
|
}
|
|
|
|
// Get store stats
|
|
storeStats, err := store.Stats()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get store stats: %w", err)
|
|
}
|
|
|
|
metrics := &DedupMetrics{
|
|
TotalChunks: stats.TotalChunks,
|
|
TotalManifests: stats.TotalManifests,
|
|
TotalBackupSize: stats.TotalBackupSize,
|
|
TotalNewData: stats.TotalNewData,
|
|
SpaceSaved: stats.SpaceSaved,
|
|
DedupRatio: stats.DedupRatio,
|
|
DiskUsage: storeStats.TotalSize,
|
|
ByDatabase: make(map[string]*DatabaseDedupMetrics),
|
|
}
|
|
|
|
// Collect per-database metrics from manifest store
|
|
manifestStore, err := NewManifestStore(basePath)
|
|
if err != nil {
|
|
return metrics, nil // Return partial metrics
|
|
}
|
|
|
|
manifests, err := manifestStore.ListAll()
|
|
if err != nil {
|
|
return metrics, nil // Return partial metrics
|
|
}
|
|
|
|
for _, m := range manifests {
|
|
dbKey := m.DatabaseName
|
|
if dbKey == "" {
|
|
dbKey = "_default"
|
|
}
|
|
|
|
dbMetrics, ok := metrics.ByDatabase[dbKey]
|
|
if !ok {
|
|
dbMetrics = &DatabaseDedupMetrics{
|
|
Database: dbKey,
|
|
}
|
|
metrics.ByDatabase[dbKey] = dbMetrics
|
|
}
|
|
|
|
dbMetrics.BackupCount++
|
|
dbMetrics.TotalSize += m.OriginalSize
|
|
dbMetrics.StoredSize += m.StoredSize
|
|
|
|
if m.CreatedAt.After(dbMetrics.LastBackupTime) {
|
|
dbMetrics.LastBackupTime = m.CreatedAt
|
|
}
|
|
if !m.VerifiedAt.IsZero() && m.VerifiedAt.After(dbMetrics.LastVerified) {
|
|
dbMetrics.LastVerified = m.VerifiedAt
|
|
}
|
|
}
|
|
|
|
// Calculate per-database dedup ratios
|
|
for _, dbMetrics := range metrics.ByDatabase {
|
|
if dbMetrics.TotalSize > 0 {
|
|
dbMetrics.DedupRatio = 1.0 - float64(dbMetrics.StoredSize)/float64(dbMetrics.TotalSize)
|
|
}
|
|
}
|
|
|
|
return metrics, nil
|
|
}
|
|
|
|
// WritePrometheusTextfile writes dedup metrics in Prometheus format
|
|
func WritePrometheusTextfile(path string, instance string, basePath string, indexPath string) error {
|
|
metrics, err := CollectMetrics(basePath, indexPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
output := FormatPrometheusMetrics(metrics, instance)
|
|
|
|
// Atomic write
|
|
dir := filepath.Dir(path)
|
|
if err := os.MkdirAll(dir, 0755); err != nil {
|
|
return fmt.Errorf("failed to create directory: %w", err)
|
|
}
|
|
|
|
tmpPath := path + ".tmp"
|
|
if err := os.WriteFile(tmpPath, []byte(output), 0644); err != nil {
|
|
return fmt.Errorf("failed to write temp file: %w", err)
|
|
}
|
|
|
|
if err := os.Rename(tmpPath, path); err != nil {
|
|
os.Remove(tmpPath)
|
|
return fmt.Errorf("failed to rename temp file: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// FormatPrometheusMetrics formats dedup metrics in Prometheus exposition format
|
|
func FormatPrometheusMetrics(m *DedupMetrics, instance string) string {
|
|
var b strings.Builder
|
|
now := time.Now().Unix()
|
|
|
|
b.WriteString("# DBBackup Deduplication Prometheus Metrics\n")
|
|
b.WriteString(fmt.Sprintf("# Generated at: %s\n", time.Now().Format(time.RFC3339)))
|
|
b.WriteString(fmt.Sprintf("# Instance: %s\n", instance))
|
|
b.WriteString("\n")
|
|
|
|
// Global dedup metrics
|
|
b.WriteString("# HELP dbbackup_dedup_chunks_total Total number of unique chunks stored\n")
|
|
b.WriteString("# TYPE dbbackup_dedup_chunks_total gauge\n")
|
|
b.WriteString(fmt.Sprintf("dbbackup_dedup_chunks_total{instance=%q} %d\n", instance, m.TotalChunks))
|
|
b.WriteString("\n")
|
|
|
|
b.WriteString("# HELP dbbackup_dedup_manifests_total Total number of deduplicated backups\n")
|
|
b.WriteString("# TYPE dbbackup_dedup_manifests_total gauge\n")
|
|
b.WriteString(fmt.Sprintf("dbbackup_dedup_manifests_total{instance=%q} %d\n", instance, m.TotalManifests))
|
|
b.WriteString("\n")
|
|
|
|
b.WriteString("# HELP dbbackup_dedup_backup_bytes_total Total logical size of all backups in bytes\n")
|
|
b.WriteString("# TYPE dbbackup_dedup_backup_bytes_total gauge\n")
|
|
b.WriteString(fmt.Sprintf("dbbackup_dedup_backup_bytes_total{instance=%q} %d\n", instance, m.TotalBackupSize))
|
|
b.WriteString("\n")
|
|
|
|
b.WriteString("# HELP dbbackup_dedup_stored_bytes_total Total unique data stored in bytes (after dedup)\n")
|
|
b.WriteString("# TYPE dbbackup_dedup_stored_bytes_total gauge\n")
|
|
b.WriteString(fmt.Sprintf("dbbackup_dedup_stored_bytes_total{instance=%q} %d\n", instance, m.TotalNewData))
|
|
b.WriteString("\n")
|
|
|
|
b.WriteString("# HELP dbbackup_dedup_space_saved_bytes Bytes saved by deduplication\n")
|
|
b.WriteString("# TYPE dbbackup_dedup_space_saved_bytes gauge\n")
|
|
b.WriteString(fmt.Sprintf("dbbackup_dedup_space_saved_bytes{instance=%q} %d\n", instance, m.SpaceSaved))
|
|
b.WriteString("\n")
|
|
|
|
b.WriteString("# HELP dbbackup_dedup_ratio Deduplication ratio (0-1, higher is better)\n")
|
|
b.WriteString("# TYPE dbbackup_dedup_ratio gauge\n")
|
|
b.WriteString(fmt.Sprintf("dbbackup_dedup_ratio{instance=%q} %.4f\n", instance, m.DedupRatio))
|
|
b.WriteString("\n")
|
|
|
|
b.WriteString("# HELP dbbackup_dedup_disk_usage_bytes Actual disk usage of chunk store\n")
|
|
b.WriteString("# TYPE dbbackup_dedup_disk_usage_bytes gauge\n")
|
|
b.WriteString(fmt.Sprintf("dbbackup_dedup_disk_usage_bytes{instance=%q} %d\n", instance, m.DiskUsage))
|
|
b.WriteString("\n")
|
|
|
|
// Per-database metrics
|
|
if len(m.ByDatabase) > 0 {
|
|
b.WriteString("# HELP dbbackup_dedup_database_backup_count Number of deduplicated backups per database\n")
|
|
b.WriteString("# TYPE dbbackup_dedup_database_backup_count gauge\n")
|
|
for _, db := range m.ByDatabase {
|
|
b.WriteString(fmt.Sprintf("dbbackup_dedup_database_backup_count{instance=%q,database=%q} %d\n",
|
|
instance, db.Database, db.BackupCount))
|
|
}
|
|
b.WriteString("\n")
|
|
|
|
b.WriteString("# HELP dbbackup_dedup_database_ratio Deduplication ratio per database (0-1)\n")
|
|
b.WriteString("# TYPE dbbackup_dedup_database_ratio gauge\n")
|
|
for _, db := range m.ByDatabase {
|
|
b.WriteString(fmt.Sprintf("dbbackup_dedup_database_ratio{instance=%q,database=%q} %.4f\n",
|
|
instance, db.Database, db.DedupRatio))
|
|
}
|
|
b.WriteString("\n")
|
|
|
|
b.WriteString("# HELP dbbackup_dedup_database_last_backup_timestamp Last backup timestamp per database\n")
|
|
b.WriteString("# TYPE dbbackup_dedup_database_last_backup_timestamp gauge\n")
|
|
for _, db := range m.ByDatabase {
|
|
if !db.LastBackupTime.IsZero() {
|
|
b.WriteString(fmt.Sprintf("dbbackup_dedup_database_last_backup_timestamp{instance=%q,database=%q} %d\n",
|
|
instance, db.Database, db.LastBackupTime.Unix()))
|
|
}
|
|
}
|
|
b.WriteString("\n")
|
|
}
|
|
|
|
b.WriteString("# HELP dbbackup_dedup_scrape_timestamp Unix timestamp when dedup metrics were collected\n")
|
|
b.WriteString("# TYPE dbbackup_dedup_scrape_timestamp gauge\n")
|
|
b.WriteString(fmt.Sprintf("dbbackup_dedup_scrape_timestamp{instance=%q} %d\n", instance, now))
|
|
|
|
return b.String()
|
|
}
|