From 59d209424158028182cbc0c698c4c5e434db9bed Mon Sep 17 00:00:00 2001 From: Alexander Renz Date: Mon, 12 Jan 2026 09:54:35 +0100 Subject: [PATCH] Build all platforms v3.42.22 --- bin/README.md | 4 +- cmd/dedup.go | 725 +++++++++++++++++++++++++++++++- grafana/dbbackup-dashboard.json | 451 +++++++++++++++++++- internal/dedup/index.go | 85 +++- internal/dedup/manifest.go | 5 +- internal/dedup/metrics.go | 235 +++++++++++ 6 files changed, 1481 insertions(+), 24 deletions(-) create mode 100644 internal/dedup/metrics.go diff --git a/bin/README.md b/bin/README.md index cfea445..63494ba 100644 --- a/bin/README.md +++ b/bin/README.md @@ -4,8 +4,8 @@ This directory contains pre-compiled binaries for the DB Backup Tool across mult ## Build Information - **Version**: 3.42.10 -- **Build Time**: 2026-01-08_11:47:09_UTC -- **Git Commit**: b05c2be +- **Build Time**: 2026-01-12_08:50:35_UTC +- **Git Commit**: b1f8c6d ## Recent Updates (v1.1.0) - ✅ Fixed TUI progress display with line-by-line output diff --git a/cmd/dedup.go b/cmd/dedup.go index ece7d0e..c582add 100644 --- a/cmd/dedup.go +++ b/cmd/dedup.go @@ -1,11 +1,13 @@ package cmd import ( + "compress/gzip" "crypto/sha256" "encoding/hex" "fmt" "io" "os" + "os/exec" "path/filepath" "strings" "time" @@ -34,7 +36,24 @@ Storage Structure: chunks/ # Content-addressed chunk files ab/cdef... # Sharded by first 2 chars of hash manifests/ # JSON manifest per backup - chunks.db # SQLite index`, + chunks.db # SQLite index + +NFS/CIFS NOTICE: + SQLite may have locking issues on network storage. + Use --index-db to put the SQLite index on local storage while keeping + chunks on network storage: + + dbbackup dedup backup mydb.sql \ + --dedup-dir /mnt/nfs/backups/dedup \ + --index-db /var/lib/dbbackup/dedup-index.db + + This avoids "database is locked" errors while still storing chunks remotely. + +COMPRESSED INPUT NOTICE: + Pre-compressed files (.gz) have poor deduplication ratios (<10%). + Use --decompress-input to decompress before chunking for better results: + + dbbackup dedup backup mydb.sql.gz --decompress-input`, } var dedupBackupCmd = &cobra.Command{ @@ -89,9 +108,85 @@ var dedupDeleteCmd = &cobra.Command{ RunE: runDedupDelete, } +var dedupVerifyCmd = &cobra.Command{ + Use: "verify [manifest-id]", + Short: "Verify chunk integrity against manifests", + Long: `Verify that all chunks referenced by manifests exist and have correct hashes. + +Without arguments, verifies all backups. With a manifest ID, verifies only that backup. + +Examples: + dbbackup dedup verify # Verify all backups + dbbackup dedup verify 2026-01-07_mydb # Verify specific backup`, + RunE: runDedupVerify, +} + +var dedupPruneCmd = &cobra.Command{ + Use: "prune", + Short: "Apply retention policy to manifests", + Long: `Delete old manifests based on retention policy (like borg prune). + +Keeps a specified number of recent backups per database and deletes the rest. + +Examples: + dbbackup dedup prune --keep-last 7 # Keep 7 most recent + dbbackup dedup prune --keep-daily 7 --keep-weekly 4 # Keep 7 daily + 4 weekly`, + RunE: runDedupPrune, +} + +var dedupBackupDBCmd = &cobra.Command{ + Use: "backup-db", + Short: "Direct database dump with deduplication", + Long: `Dump a database directly into deduplicated chunks without temp files. + +Streams the database dump through the chunker for efficient deduplication. + +Examples: + dbbackup dedup backup-db --db-type postgres --db-name mydb + dbbackup dedup backup-db -d mariadb --database production_db --host db.local`, + RunE: runDedupBackupDB, +} + +// Prune flags +var ( + pruneKeepLast int + pruneKeepDaily int + pruneKeepWeekly int + pruneDryRun bool +) + +// backup-db flags +var ( + backupDBDatabase string + backupDBUser string + backupDBPassword string +) + +// metrics flags +var ( + dedupMetricsOutput string + dedupMetricsInstance string +) + +var dedupMetricsCmd = &cobra.Command{ + Use: "metrics", + Short: "Export dedup statistics as Prometheus metrics", + Long: `Export deduplication statistics in Prometheus format. + +Can write to a textfile for node_exporter's textfile collector, +or print to stdout for custom integrations. + +Examples: + dbbackup dedup metrics # Print to stdout + dbbackup dedup metrics --output /var/lib/node_exporter/textfile_collector/dedup.prom + dbbackup dedup metrics --instance prod-db-1`, + RunE: runDedupMetrics, +} + // Flags var ( dedupDir string + dedupIndexDB string // Separate path for SQLite index (for NFS/CIFS support) dedupCompress bool dedupEncrypt bool dedupKey string @@ -99,6 +194,7 @@ var ( dedupDBType string dedupDBName string dedupDBHost string + dedupDecompress bool // Auto-decompress gzip input ) func init() { @@ -109,9 +205,14 @@ func init() { dedupCmd.AddCommand(dedupStatsCmd) dedupCmd.AddCommand(dedupGCCmd) dedupCmd.AddCommand(dedupDeleteCmd) + dedupCmd.AddCommand(dedupVerifyCmd) + dedupCmd.AddCommand(dedupPruneCmd) + dedupCmd.AddCommand(dedupBackupDBCmd) + dedupCmd.AddCommand(dedupMetricsCmd) // Global dedup flags dedupCmd.PersistentFlags().StringVar(&dedupDir, "dedup-dir", "", "Dedup storage directory (default: $BACKUP_DIR/dedup)") + dedupCmd.PersistentFlags().StringVar(&dedupIndexDB, "index-db", "", "SQLite index path (local recommended for NFS/CIFS chunk dirs)") dedupCmd.PersistentFlags().BoolVar(&dedupCompress, "compress", true, "Compress chunks with gzip") dedupCmd.PersistentFlags().BoolVar(&dedupEncrypt, "encrypt", false, "Encrypt chunks with AES-256-GCM") dedupCmd.PersistentFlags().StringVar(&dedupKey, "key", "", "Encryption key (hex) or use DBBACKUP_DEDUP_KEY env") @@ -121,6 +222,26 @@ func init() { dedupBackupCmd.Flags().StringVar(&dedupDBType, "db-type", "", "Database type (postgres/mysql)") dedupBackupCmd.Flags().StringVar(&dedupDBName, "db-name", "", "Database name") dedupBackupCmd.Flags().StringVar(&dedupDBHost, "db-host", "", "Database host") + dedupBackupCmd.Flags().BoolVar(&dedupDecompress, "decompress-input", false, "Auto-decompress gzip input before chunking (improves dedup ratio)") + + // Prune flags + dedupPruneCmd.Flags().IntVar(&pruneKeepLast, "keep-last", 0, "Keep the last N backups") + dedupPruneCmd.Flags().IntVar(&pruneKeepDaily, "keep-daily", 0, "Keep N daily backups") + dedupPruneCmd.Flags().IntVar(&pruneKeepWeekly, "keep-weekly", 0, "Keep N weekly backups") + dedupPruneCmd.Flags().BoolVar(&pruneDryRun, "dry-run", false, "Show what would be deleted without actually deleting") + + // backup-db flags + dedupBackupDBCmd.Flags().StringVarP(&dedupDBType, "db-type", "d", "", "Database type (postgres/mariadb/mysql)") + dedupBackupDBCmd.Flags().StringVar(&backupDBDatabase, "database", "", "Database name to backup") + dedupBackupDBCmd.Flags().StringVar(&dedupDBHost, "host", "localhost", "Database host") + dedupBackupDBCmd.Flags().StringVarP(&backupDBUser, "user", "u", "", "Database user") + dedupBackupDBCmd.Flags().StringVarP(&backupDBPassword, "password", "p", "", "Database password (or use env)") + dedupBackupDBCmd.MarkFlagRequired("db-type") + dedupBackupDBCmd.MarkFlagRequired("database") + + // Metrics flags + dedupMetricsCmd.Flags().StringVarP(&dedupMetricsOutput, "output", "o", "", "Output file path (default: stdout)") + dedupMetricsCmd.Flags().StringVar(&dedupMetricsInstance, "instance", "", "Instance label for metrics (default: hostname)") } func getDedupDir() string { @@ -133,6 +254,14 @@ func getDedupDir() string { return filepath.Join(os.Getenv("HOME"), "db_backups", "dedup") } +func getIndexDBPath() string { + if dedupIndexDB != "" { + return dedupIndexDB + } + // Default: same directory as chunks (may have issues on NFS/CIFS) + return filepath.Join(getDedupDir(), "chunks.db") +} + func getEncryptionKey() string { if dedupKey != "" { return dedupKey @@ -155,6 +284,25 @@ func runDedupBackup(cmd *cobra.Command, args []string) error { return fmt.Errorf("failed to stat input file: %w", err) } + // Check for compressed input and warn/handle + var reader io.Reader = file + isGzipped := strings.HasSuffix(strings.ToLower(inputPath), ".gz") + if isGzipped && !dedupDecompress { + fmt.Printf("Warning: Input appears to be gzip compressed (.gz)\n") + fmt.Printf(" Compressed data typically has poor dedup ratios (<10%%).\n") + fmt.Printf(" Consider using --decompress-input for better deduplication.\n\n") + } + + if isGzipped && dedupDecompress { + fmt.Printf("Auto-decompressing gzip input for better dedup ratio...\n") + gzReader, err := gzip.NewReader(file) + if err != nil { + return fmt.Errorf("failed to decompress gzip input: %w", err) + } + defer gzReader.Close() + reader = gzReader + } + // Setup dedup storage basePath := getDedupDir() encKey := "" @@ -179,7 +327,7 @@ func runDedupBackup(cmd *cobra.Command, args []string) error { return fmt.Errorf("failed to open manifest store: %w", err) } - index, err := dedup.NewChunkIndex(basePath) + index, err := dedup.NewChunkIndexAt(getIndexDBPath()) if err != nil { return fmt.Errorf("failed to open chunk index: %w", err) } @@ -193,22 +341,43 @@ func runDedupBackup(cmd *cobra.Command, args []string) error { } else { base := filepath.Base(inputPath) ext := filepath.Ext(base) + // Remove .gz extension if decompressing + if isGzipped && dedupDecompress { + base = strings.TrimSuffix(base, ext) + ext = filepath.Ext(base) + } manifestID += "_" + strings.TrimSuffix(base, ext) } fmt.Printf("Creating deduplicated backup: %s\n", manifestID) fmt.Printf("Input: %s (%s)\n", inputPath, formatBytes(info.Size())) + if isGzipped && dedupDecompress { + fmt.Printf("Mode: Decompressing before chunking\n") + } fmt.Printf("Store: %s\n", basePath) + if dedupIndexDB != "" { + fmt.Printf("Index: %s\n", getIndexDBPath()) + } - // Hash the entire file for verification - file.Seek(0, 0) + // For decompressed input, we can't seek - use TeeReader to hash while chunking h := sha256.New() - io.Copy(h, file) - fileHash := hex.EncodeToString(h.Sum(nil)) - file.Seek(0, 0) + var chunkReader io.Reader + + if isGzipped && dedupDecompress { + // Can't seek on gzip stream - hash will be computed inline + chunkReader = io.TeeReader(reader, h) + } else { + // Regular file - hash first, then reset and chunk + file.Seek(0, 0) + io.Copy(h, file) + file.Seek(0, 0) + chunkReader = file + h = sha256.New() // Reset for inline hashing + chunkReader = io.TeeReader(file, h) + } // Chunk the file - chunker := dedup.NewChunker(file, dedup.DefaultChunkerConfig()) + chunker := dedup.NewChunker(chunkReader, dedup.DefaultChunkerConfig()) var chunks []dedup.ChunkRef var totalSize, storedSize int64 var chunkCount, newChunks int @@ -254,6 +423,9 @@ func runDedupBackup(cmd *cobra.Command, args []string) error { duration := time.Since(startTime) + // Get final hash (computed inline via TeeReader) + fileHash := hex.EncodeToString(h.Sum(nil)) + // Calculate dedup ratio dedupRatio := 0.0 if totalSize > 0 { @@ -277,6 +449,7 @@ func runDedupBackup(cmd *cobra.Command, args []string) error { Encrypted: dedupEncrypt, Compressed: dedupCompress, SHA256: fileHash, + Decompressed: isGzipped && dedupDecompress, // Track if we decompressed } if err := manifestStore.Save(manifest); err != nil { @@ -451,8 +624,12 @@ func runDedupStats(cmd *cobra.Command, args []string) error { fmt.Printf("Unique chunks: %d\n", stats.TotalChunks) fmt.Printf("Total raw size: %s\n", formatBytes(stats.TotalSizeRaw)) fmt.Printf("Stored size: %s\n", formatBytes(stats.TotalSizeStored)) - fmt.Printf("Dedup ratio: %.1f%%\n", stats.DedupRatio*100) - fmt.Printf("Space saved: %s\n", formatBytes(stats.TotalSizeRaw-stats.TotalSizeStored)) + fmt.Printf("\n") + fmt.Printf("Backup Statistics (accurate dedup calculation):\n") + fmt.Printf(" Total backed up: %s (across all backups)\n", formatBytes(stats.TotalBackupSize)) + fmt.Printf(" New data stored: %s\n", formatBytes(stats.TotalNewData)) + fmt.Printf(" Space saved: %s\n", formatBytes(stats.SpaceSaved)) + fmt.Printf(" Dedup ratio: %.1f%%\n", stats.DedupRatio*100) if storeStats != nil { fmt.Printf("Disk usage: %s\n", formatBytes(storeStats.TotalSize)) @@ -577,3 +754,531 @@ func truncateStr(s string, max int) string { } return s[:max-3] + "..." } + +func runDedupVerify(cmd *cobra.Command, args []string) error { + basePath := getDedupDir() + + store, err := dedup.NewChunkStore(dedup.StoreConfig{ + BasePath: basePath, + Compress: dedupCompress, + }) + if err != nil { + return fmt.Errorf("failed to open chunk store: %w", err) + } + + manifestStore, err := dedup.NewManifestStore(basePath) + if err != nil { + return fmt.Errorf("failed to open manifest store: %w", err) + } + + index, err := dedup.NewChunkIndexAt(getIndexDBPath()) + if err != nil { + return fmt.Errorf("failed to open chunk index: %w", err) + } + defer index.Close() + + var manifests []*dedup.Manifest + + if len(args) > 0 { + // Verify specific manifest + m, err := manifestStore.Load(args[0]) + if err != nil { + return fmt.Errorf("failed to load manifest: %w", err) + } + manifests = []*dedup.Manifest{m} + } else { + // Verify all manifests + manifests, err = manifestStore.ListAll() + if err != nil { + return fmt.Errorf("failed to list manifests: %w", err) + } + } + + if len(manifests) == 0 { + fmt.Println("No manifests to verify.") + return nil + } + + fmt.Printf("Verifying %d backup(s)...\n\n", len(manifests)) + + var totalChunks, missingChunks, corruptChunks int + var allOK = true + + for _, m := range manifests { + fmt.Printf("Verifying: %s (%d chunks)\n", m.ID, m.ChunkCount) + + var missing, corrupt int + seenHashes := make(map[string]bool) + + for i, ref := range m.Chunks { + if seenHashes[ref.Hash] { + continue // Already verified this chunk + } + seenHashes[ref.Hash] = true + totalChunks++ + + // Check if chunk exists + if !store.Has(ref.Hash) { + missing++ + missingChunks++ + if missing <= 5 { + fmt.Printf(" [MISSING] chunk %d: %s\n", i, ref.Hash[:16]) + } + continue + } + + // Verify chunk hash by reading it + chunk, err := store.Get(ref.Hash) + if err != nil { + corrupt++ + corruptChunks++ + if corrupt <= 5 { + fmt.Printf(" [CORRUPT] chunk %d: %s - %v\n", i, ref.Hash[:16], err) + } + continue + } + + // Verify size + if chunk.Length != ref.Length { + corrupt++ + corruptChunks++ + if corrupt <= 5 { + fmt.Printf(" [SIZE MISMATCH] chunk %d: expected %d, got %d\n", i, ref.Length, chunk.Length) + } + } + } + + if missing > 0 || corrupt > 0 { + allOK = false + fmt.Printf(" Result: FAILED (%d missing, %d corrupt)\n", missing, corrupt) + if missing > 5 || corrupt > 5 { + fmt.Printf(" ... and %d more errors\n", (missing+corrupt)-10) + } + } else { + fmt.Printf(" Result: OK (%d unique chunks verified)\n", len(seenHashes)) + // Update verified timestamp + m.VerifiedAt = time.Now() + manifestStore.Save(m) + index.UpdateManifestVerified(m.ID, m.VerifiedAt) + } + fmt.Println() + } + + fmt.Println("========================================") + if allOK { + fmt.Printf("All %d backup(s) verified successfully!\n", len(manifests)) + fmt.Printf("Total unique chunks checked: %d\n", totalChunks) + } else { + fmt.Printf("Verification FAILED!\n") + fmt.Printf("Missing chunks: %d\n", missingChunks) + fmt.Printf("Corrupt chunks: %d\n", corruptChunks) + return fmt.Errorf("verification failed: %d missing, %d corrupt chunks", missingChunks, corruptChunks) + } + + return nil +} + +func runDedupPrune(cmd *cobra.Command, args []string) error { + if pruneKeepLast == 0 && pruneKeepDaily == 0 && pruneKeepWeekly == 0 { + return fmt.Errorf("at least one of --keep-last, --keep-daily, or --keep-weekly must be specified") + } + + basePath := getDedupDir() + + manifestStore, err := dedup.NewManifestStore(basePath) + if err != nil { + return fmt.Errorf("failed to open manifest store: %w", err) + } + + index, err := dedup.NewChunkIndexAt(getIndexDBPath()) + if err != nil { + return fmt.Errorf("failed to open chunk index: %w", err) + } + defer index.Close() + + manifests, err := manifestStore.ListAll() + if err != nil { + return fmt.Errorf("failed to list manifests: %w", err) + } + + if len(manifests) == 0 { + fmt.Println("No backups to prune.") + return nil + } + + // Group by database name + byDatabase := make(map[string][]*dedup.Manifest) + for _, m := range manifests { + key := m.DatabaseName + if key == "" { + key = "_default" + } + byDatabase[key] = append(byDatabase[key], m) + } + + var toDelete []*dedup.Manifest + + for dbName, dbManifests := range byDatabase { + // Already sorted by time (newest first from ListAll) + kept := make(map[string]bool) + var keepReasons = make(map[string]string) + + // Keep last N + if pruneKeepLast > 0 { + for i := 0; i < pruneKeepLast && i < len(dbManifests); i++ { + kept[dbManifests[i].ID] = true + keepReasons[dbManifests[i].ID] = "keep-last" + } + } + + // Keep daily (one per day) + if pruneKeepDaily > 0 { + seenDays := make(map[string]bool) + count := 0 + for _, m := range dbManifests { + day := m.CreatedAt.Format("2006-01-02") + if !seenDays[day] { + seenDays[day] = true + if count < pruneKeepDaily { + kept[m.ID] = true + if keepReasons[m.ID] == "" { + keepReasons[m.ID] = "keep-daily" + } + count++ + } + } + } + } + + // Keep weekly (one per week) + if pruneKeepWeekly > 0 { + seenWeeks := make(map[string]bool) + count := 0 + for _, m := range dbManifests { + year, week := m.CreatedAt.ISOWeek() + weekKey := fmt.Sprintf("%d-W%02d", year, week) + if !seenWeeks[weekKey] { + seenWeeks[weekKey] = true + if count < pruneKeepWeekly { + kept[m.ID] = true + if keepReasons[m.ID] == "" { + keepReasons[m.ID] = "keep-weekly" + } + count++ + } + } + } + } + + if dbName != "_default" { + fmt.Printf("\nDatabase: %s\n", dbName) + } else { + fmt.Printf("\nUnnamed backups:\n") + } + + for _, m := range dbManifests { + if kept[m.ID] { + fmt.Printf(" [KEEP] %s (%s) - %s\n", m.ID, m.CreatedAt.Format("2006-01-02"), keepReasons[m.ID]) + } else { + fmt.Printf(" [DELETE] %s (%s)\n", m.ID, m.CreatedAt.Format("2006-01-02")) + toDelete = append(toDelete, m) + } + } + } + + if len(toDelete) == 0 { + fmt.Printf("\nNo backups to prune (all match retention policy).\n") + return nil + } + + fmt.Printf("\n%d backup(s) will be deleted.\n", len(toDelete)) + + if pruneDryRun { + fmt.Println("\n[DRY RUN] No changes made. Remove --dry-run to actually delete.") + return nil + } + + // Actually delete + for _, m := range toDelete { + // Decrement chunk references + for _, ref := range m.Chunks { + index.DecrementRef(ref.Hash) + } + + if err := manifestStore.Delete(m.ID); err != nil { + log.Warn("Failed to delete manifest", "id", m.ID, "error", err) + } + index.RemoveManifest(m.ID) + } + + fmt.Printf("\nDeleted %d backup(s).\n", len(toDelete)) + fmt.Println("Run 'dbbackup dedup gc' to reclaim space from unreferenced chunks.") + + return nil +} + +func runDedupBackupDB(cmd *cobra.Command, args []string) error { + dbType := strings.ToLower(dedupDBType) + dbName := backupDBDatabase + + // Validate db type + var dumpCmd string + var dumpArgs []string + + switch dbType { + case "postgres", "postgresql", "pg": + dbType = "postgres" + dumpCmd = "pg_dump" + dumpArgs = []string{"-Fc"} // Custom format for better compression + if dedupDBHost != "" && dedupDBHost != "localhost" { + dumpArgs = append(dumpArgs, "-h", dedupDBHost) + } + if backupDBUser != "" { + dumpArgs = append(dumpArgs, "-U", backupDBUser) + } + dumpArgs = append(dumpArgs, dbName) + + case "mysql": + dumpCmd = "mysqldump" + dumpArgs = []string{ + "--single-transaction", + "--routines", + "--triggers", + "--events", + } + if dedupDBHost != "" { + dumpArgs = append(dumpArgs, "-h", dedupDBHost) + } + if backupDBUser != "" { + dumpArgs = append(dumpArgs, "-u", backupDBUser) + } + if backupDBPassword != "" { + dumpArgs = append(dumpArgs, "-p"+backupDBPassword) + } + dumpArgs = append(dumpArgs, dbName) + + case "mariadb": + dumpCmd = "mariadb-dump" + // Fall back to mysqldump if mariadb-dump not available + if _, err := exec.LookPath(dumpCmd); err != nil { + dumpCmd = "mysqldump" + } + dumpArgs = []string{ + "--single-transaction", + "--routines", + "--triggers", + "--events", + } + if dedupDBHost != "" { + dumpArgs = append(dumpArgs, "-h", dedupDBHost) + } + if backupDBUser != "" { + dumpArgs = append(dumpArgs, "-u", backupDBUser) + } + if backupDBPassword != "" { + dumpArgs = append(dumpArgs, "-p"+backupDBPassword) + } + dumpArgs = append(dumpArgs, dbName) + + default: + return fmt.Errorf("unsupported database type: %s (use postgres, mysql, or mariadb)", dbType) + } + + // Verify dump command exists + if _, err := exec.LookPath(dumpCmd); err != nil { + return fmt.Errorf("%s not found in PATH: %w", dumpCmd, err) + } + + // Setup dedup storage + basePath := getDedupDir() + encKey := "" + if dedupEncrypt { + encKey = getEncryptionKey() + if encKey == "" { + return fmt.Errorf("encryption enabled but no key provided (use --key or DBBACKUP_DEDUP_KEY)") + } + } + + store, err := dedup.NewChunkStore(dedup.StoreConfig{ + BasePath: basePath, + Compress: dedupCompress, + EncryptionKey: encKey, + }) + if err != nil { + return fmt.Errorf("failed to open chunk store: %w", err) + } + + manifestStore, err := dedup.NewManifestStore(basePath) + if err != nil { + return fmt.Errorf("failed to open manifest store: %w", err) + } + + index, err := dedup.NewChunkIndexAt(getIndexDBPath()) + if err != nil { + return fmt.Errorf("failed to open chunk index: %w", err) + } + defer index.Close() + + // Generate manifest ID + now := time.Now() + manifestID := now.Format("2006-01-02_150405") + "_" + dbName + + fmt.Printf("Creating deduplicated database backup: %s\n", manifestID) + fmt.Printf("Database: %s (%s)\n", dbName, dbType) + fmt.Printf("Command: %s %s\n", dumpCmd, strings.Join(dumpArgs, " ")) + fmt.Printf("Store: %s\n", basePath) + + // Start the dump command + dumpExec := exec.Command(dumpCmd, dumpArgs...) + + // Set password via environment for postgres + if dbType == "postgres" && backupDBPassword != "" { + dumpExec.Env = append(os.Environ(), "PGPASSWORD="+backupDBPassword) + } + + stdout, err := dumpExec.StdoutPipe() + if err != nil { + return fmt.Errorf("failed to get stdout pipe: %w", err) + } + + stderr, err := dumpExec.StderrPipe() + if err != nil { + return fmt.Errorf("failed to get stderr pipe: %w", err) + } + + if err := dumpExec.Start(); err != nil { + return fmt.Errorf("failed to start %s: %w", dumpCmd, err) + } + + // Hash while chunking using TeeReader + h := sha256.New() + reader := io.TeeReader(stdout, h) + + // Chunk the stream directly + chunker := dedup.NewChunker(reader, dedup.DefaultChunkerConfig()) + var chunks []dedup.ChunkRef + var totalSize, storedSize int64 + var chunkCount, newChunks int + + startTime := time.Now() + + for { + chunk, err := chunker.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("chunking failed: %w", err) + } + + chunkCount++ + totalSize += int64(chunk.Length) + + // Store chunk (deduplication happens here) + isNew, err := store.Put(chunk) + if err != nil { + return fmt.Errorf("failed to store chunk: %w", err) + } + + if isNew { + newChunks++ + storedSize += int64(chunk.Length) + index.AddChunk(chunk.Hash, chunk.Length, chunk.Length) + } + + chunks = append(chunks, dedup.ChunkRef{ + Hash: chunk.Hash, + Offset: chunk.Offset, + Length: chunk.Length, + }) + + if chunkCount%1000 == 0 { + fmt.Printf("\r Processed %d chunks, %d new, %s...", chunkCount, newChunks, formatBytes(totalSize)) + } + } + + // Read any stderr + stderrBytes, _ := io.ReadAll(stderr) + + // Wait for command to complete + if err := dumpExec.Wait(); err != nil { + return fmt.Errorf("%s failed: %w\nstderr: %s", dumpCmd, err, string(stderrBytes)) + } + + duration := time.Since(startTime) + fileHash := hex.EncodeToString(h.Sum(nil)) + + // Calculate dedup ratio + dedupRatio := 0.0 + if totalSize > 0 { + dedupRatio = 1.0 - float64(storedSize)/float64(totalSize) + } + + // Create manifest + manifest := &dedup.Manifest{ + ID: manifestID, + Name: dedupName, + CreatedAt: now, + DatabaseType: dbType, + DatabaseName: dbName, + DatabaseHost: dedupDBHost, + Chunks: chunks, + OriginalSize: totalSize, + StoredSize: storedSize, + ChunkCount: chunkCount, + NewChunks: newChunks, + DedupRatio: dedupRatio, + Encrypted: dedupEncrypt, + Compressed: dedupCompress, + SHA256: fileHash, + } + + if err := manifestStore.Save(manifest); err != nil { + return fmt.Errorf("failed to save manifest: %w", err) + } + + if err := index.AddManifest(manifest); err != nil { + log.Warn("Failed to index manifest", "error", err) + } + + fmt.Printf("\r \r") + fmt.Printf("\nBackup complete!\n") + fmt.Printf(" Manifest: %s\n", manifestID) + fmt.Printf(" Chunks: %d total, %d new\n", chunkCount, newChunks) + fmt.Printf(" Dump size: %s\n", formatBytes(totalSize)) + fmt.Printf(" Stored: %s (new data)\n", formatBytes(storedSize)) + fmt.Printf(" Dedup ratio: %.1f%%\n", dedupRatio*100) + fmt.Printf(" Duration: %s\n", duration.Round(time.Millisecond)) + fmt.Printf(" Throughput: %s/s\n", formatBytes(int64(float64(totalSize)/duration.Seconds()))) + + return nil +} + +func runDedupMetrics(cmd *cobra.Command, args []string) error { + basePath := getDedupDir() + indexPath := getIndexDBPath() + + instance := dedupMetricsInstance + if instance == "" { + hostname, _ := os.Hostname() + instance = hostname + } + + metrics, err := dedup.CollectMetrics(basePath, indexPath) + if err != nil { + return fmt.Errorf("failed to collect metrics: %w", err) + } + + output := dedup.FormatPrometheusMetrics(metrics, instance) + + if dedupMetricsOutput != "" { + if err := dedup.WritePrometheusTextfile(dedupMetricsOutput, instance, basePath, indexPath); err != nil { + return fmt.Errorf("failed to write metrics: %w", err) + } + fmt.Printf("Wrote metrics to %s\n", dedupMetricsOutput) + } else { + fmt.Print(output) + } + + return nil +} diff --git a/grafana/dbbackup-dashboard.json b/grafana/dbbackup-dashboard.json index e501ee0..48665e3 100644 --- a/grafana/dbbackup-dashboard.json +++ b/grafana/dbbackup-dashboard.json @@ -794,6 +794,454 @@ } ], "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 100, + "panels": [], + "title": "Deduplication Statistics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 31 + }, + "id": 101, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "dbbackup_dedup_ratio{instance=~\"$instance\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Dedup Ratio", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 31 + }, + "id": 102, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "dbbackup_dedup_space_saved_bytes{instance=~\"$instance\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Space Saved", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 31 + }, + "id": 103, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "dbbackup_dedup_disk_usage_bytes{instance=~\"$instance\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Disk Usage", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 31 + }, + "id": 104, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "dbbackup_dedup_chunks_total{instance=~\"$instance\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total Chunks", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 105, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "dbbackup_dedup_database_ratio{instance=~\"$instance\"}", + "legendFormat": "{{database}}", + "range": true, + "refId": "A" + } + ], + "title": "Dedup Ratio by Database", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 106, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "dbbackup_dedup_space_saved_bytes{instance=~\"$instance\"}", + "legendFormat": "Space Saved", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "dbbackup_dedup_disk_usage_bytes{instance=~\"$instance\"}", + "legendFormat": "Disk Usage", + "range": true, + "refId": "B" + } + ], + "title": "Dedup Storage Over Time", + "type": "timeseries" } ], "refresh": "30s", @@ -801,7 +1249,8 @@ "tags": [ "dbbackup", "backup", - "database" + "database", + "dedup" ], "templating": { "list": [ diff --git a/internal/dedup/index.go b/internal/dedup/index.go index d810693..216a766 100644 --- a/internal/dedup/index.go +++ b/internal/dedup/index.go @@ -3,7 +3,9 @@ package dedup import ( "database/sql" "fmt" + "os" "path/filepath" + "strings" "time" _ "github.com/mattn/go-sqlite3" // SQLite driver @@ -11,27 +13,67 @@ import ( // ChunkIndex provides fast chunk lookups using SQLite type ChunkIndex struct { - db *sql.DB + db *sql.DB + dbPath string } -// NewChunkIndex opens or creates a chunk index database +// NewChunkIndex opens or creates a chunk index database at the default location func NewChunkIndex(basePath string) (*ChunkIndex, error) { dbPath := filepath.Join(basePath, "chunks.db") + return NewChunkIndexAt(dbPath) +} - db, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL&_synchronous=NORMAL") +// NewChunkIndexAt opens or creates a chunk index database at a specific path +// Use this to put the SQLite index on local storage when chunks are on NFS/CIFS +func NewChunkIndexAt(dbPath string) (*ChunkIndex, error) { + // Ensure parent directory exists + if err := os.MkdirAll(filepath.Dir(dbPath), 0700); err != nil { + return nil, fmt.Errorf("failed to create index directory: %w", err) + } + + // Add busy_timeout to handle lock contention gracefully + db, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=5000") if err != nil { return nil, fmt.Errorf("failed to open chunk index: %w", err) } - idx := &ChunkIndex{db: db} + // Test the connection and check for locking issues + if err := db.Ping(); err != nil { + db.Close() + if isNFSLockingError(err) { + return nil, fmt.Errorf("database locked (common on NFS/CIFS): %w\n\n"+ + "HINT: Use --index-db to put the SQLite index on local storage:\n"+ + " dbbackup dedup ... --index-db /var/lib/dbbackup/dedup-index.db", err) + } + return nil, fmt.Errorf("failed to connect to chunk index: %w", err) + } + + idx := &ChunkIndex{db: db, dbPath: dbPath} if err := idx.migrate(); err != nil { db.Close() + if isNFSLockingError(err) { + return nil, fmt.Errorf("database locked during migration (common on NFS/CIFS): %w\n\n"+ + "HINT: Use --index-db to put the SQLite index on local storage:\n"+ + " dbbackup dedup ... --index-db /var/lib/dbbackup/dedup-index.db", err) + } return nil, err } return idx, nil } +// isNFSLockingError checks if an error is likely due to NFS/CIFS locking issues +func isNFSLockingError(err error) bool { + if err == nil { + return false + } + errStr := err.Error() + return strings.Contains(errStr, "database is locked") || + strings.Contains(errStr, "SQLITE_BUSY") || + strings.Contains(errStr, "cannot lock") || + strings.Contains(errStr, "lock protocol") +} + // migrate creates the schema if needed func (idx *ChunkIndex) migrate() error { schema := ` @@ -166,15 +208,26 @@ func (idx *ChunkIndex) RemoveManifest(id string) error { return err } +// UpdateManifestVerified updates the verified timestamp for a manifest +func (idx *ChunkIndex) UpdateManifestVerified(id string, verifiedAt time.Time) error { + _, err := idx.db.Exec("UPDATE manifests SET verified_at = ? WHERE id = ?", verifiedAt, id) + return err +} + // IndexStats holds statistics about the dedup index type IndexStats struct { TotalChunks int64 TotalManifests int64 - TotalSizeRaw int64 // Uncompressed, undeduplicated - TotalSizeStored int64 // On-disk after dedup+compression - DedupRatio float64 + TotalSizeRaw int64 // Uncompressed, undeduplicated (per-chunk) + TotalSizeStored int64 // On-disk after dedup+compression (per-chunk) + DedupRatio float64 // Based on manifests (real dedup ratio) OldestChunk time.Time NewestChunk time.Time + + // Manifest-based stats (accurate dedup calculation) + TotalBackupSize int64 // Sum of all backup original sizes + TotalNewData int64 // Sum of all new chunks stored + SpaceSaved int64 // Difference = what dedup saved } // Stats returns statistics about the index @@ -206,8 +259,22 @@ func (idx *ChunkIndex) Stats() (*IndexStats, error) { idx.db.QueryRow("SELECT COUNT(*) FROM manifests").Scan(&stats.TotalManifests) - if stats.TotalSizeRaw > 0 { - stats.DedupRatio = 1.0 - float64(stats.TotalSizeStored)/float64(stats.TotalSizeRaw) + // Calculate accurate dedup ratio from manifests + // Sum all backup original sizes and all new data stored + err = idx.db.QueryRow(` + SELECT + COALESCE(SUM(original_size), 0), + COALESCE(SUM(stored_size), 0) + FROM manifests + `).Scan(&stats.TotalBackupSize, &stats.TotalNewData) + if err != nil { + return nil, err + } + + // Calculate real dedup ratio: how much data was deduplicated across all backups + if stats.TotalBackupSize > 0 { + stats.DedupRatio = 1.0 - float64(stats.TotalNewData)/float64(stats.TotalBackupSize) + stats.SpaceSaved = stats.TotalBackupSize - stats.TotalNewData } return stats, nil diff --git a/internal/dedup/manifest.go b/internal/dedup/manifest.go index b4a0f17..d7448a4 100644 --- a/internal/dedup/manifest.go +++ b/internal/dedup/manifest.go @@ -36,8 +36,9 @@ type Manifest struct { DedupRatio float64 `json:"dedup_ratio"` // 1.0 = no dedup, 0.0 = 100% dedup // Encryption and compression settings used - Encrypted bool `json:"encrypted"` - Compressed bool `json:"compressed"` + Encrypted bool `json:"encrypted"` + Compressed bool `json:"compressed"` + Decompressed bool `json:"decompressed,omitempty"` // Input was auto-decompressed before chunking // Verification SHA256 string `json:"sha256"` // Hash of reconstructed file diff --git a/internal/dedup/metrics.go b/internal/dedup/metrics.go new file mode 100644 index 0000000..9f19518 --- /dev/null +++ b/internal/dedup/metrics.go @@ -0,0 +1,235 @@ +package dedup + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +// DedupMetrics holds deduplication statistics for Prometheus +type DedupMetrics struct { + // Global stats + TotalChunks int64 + TotalManifests int64 + TotalBackupSize int64 // Sum of all backup original sizes + TotalNewData int64 // Sum of all new chunks stored + SpaceSaved int64 // Bytes saved by deduplication + DedupRatio float64 // Overall dedup ratio (0-1) + DiskUsage int64 // Actual bytes on disk + + // Per-database stats + ByDatabase map[string]*DatabaseDedupMetrics +} + +// DatabaseDedupMetrics holds per-database dedup stats +type DatabaseDedupMetrics struct { + Database string + BackupCount int + TotalSize int64 + StoredSize int64 + DedupRatio float64 + LastBackupTime time.Time + LastVerified time.Time +} + +// CollectMetrics gathers dedup statistics from the index and store +func CollectMetrics(basePath string, indexPath string) (*DedupMetrics, error) { + var idx *ChunkIndex + var err error + + if indexPath != "" { + idx, err = NewChunkIndexAt(indexPath) + } else { + idx, err = NewChunkIndex(basePath) + } + if err != nil { + return nil, fmt.Errorf("failed to open chunk index: %w", err) + } + defer idx.Close() + + store, err := NewChunkStore(StoreConfig{BasePath: basePath}) + if err != nil { + return nil, fmt.Errorf("failed to open chunk store: %w", err) + } + + // Get index stats + stats, err := idx.Stats() + if err != nil { + return nil, fmt.Errorf("failed to get index stats: %w", err) + } + + // Get store stats + storeStats, err := store.Stats() + if err != nil { + return nil, fmt.Errorf("failed to get store stats: %w", err) + } + + metrics := &DedupMetrics{ + TotalChunks: stats.TotalChunks, + TotalManifests: stats.TotalManifests, + TotalBackupSize: stats.TotalBackupSize, + TotalNewData: stats.TotalNewData, + SpaceSaved: stats.SpaceSaved, + DedupRatio: stats.DedupRatio, + DiskUsage: storeStats.TotalSize, + ByDatabase: make(map[string]*DatabaseDedupMetrics), + } + + // Collect per-database metrics from manifest store + manifestStore, err := NewManifestStore(basePath) + if err != nil { + return metrics, nil // Return partial metrics + } + + manifests, err := manifestStore.ListAll() + if err != nil { + return metrics, nil // Return partial metrics + } + + for _, m := range manifests { + dbKey := m.DatabaseName + if dbKey == "" { + dbKey = "_default" + } + + dbMetrics, ok := metrics.ByDatabase[dbKey] + if !ok { + dbMetrics = &DatabaseDedupMetrics{ + Database: dbKey, + } + metrics.ByDatabase[dbKey] = dbMetrics + } + + dbMetrics.BackupCount++ + dbMetrics.TotalSize += m.OriginalSize + dbMetrics.StoredSize += m.StoredSize + + if m.CreatedAt.After(dbMetrics.LastBackupTime) { + dbMetrics.LastBackupTime = m.CreatedAt + } + if !m.VerifiedAt.IsZero() && m.VerifiedAt.After(dbMetrics.LastVerified) { + dbMetrics.LastVerified = m.VerifiedAt + } + } + + // Calculate per-database dedup ratios + for _, dbMetrics := range metrics.ByDatabase { + if dbMetrics.TotalSize > 0 { + dbMetrics.DedupRatio = 1.0 - float64(dbMetrics.StoredSize)/float64(dbMetrics.TotalSize) + } + } + + return metrics, nil +} + +// WritePrometheusTextfile writes dedup metrics in Prometheus format +func WritePrometheusTextfile(path string, instance string, basePath string, indexPath string) error { + metrics, err := CollectMetrics(basePath, indexPath) + if err != nil { + return err + } + + output := FormatPrometheusMetrics(metrics, instance) + + // Atomic write + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("failed to create directory: %w", err) + } + + tmpPath := path + ".tmp" + if err := os.WriteFile(tmpPath, []byte(output), 0644); err != nil { + return fmt.Errorf("failed to write temp file: %w", err) + } + + if err := os.Rename(tmpPath, path); err != nil { + os.Remove(tmpPath) + return fmt.Errorf("failed to rename temp file: %w", err) + } + + return nil +} + +// FormatPrometheusMetrics formats dedup metrics in Prometheus exposition format +func FormatPrometheusMetrics(m *DedupMetrics, instance string) string { + var b strings.Builder + now := time.Now().Unix() + + b.WriteString("# DBBackup Deduplication Prometheus Metrics\n") + b.WriteString(fmt.Sprintf("# Generated at: %s\n", time.Now().Format(time.RFC3339))) + b.WriteString(fmt.Sprintf("# Instance: %s\n", instance)) + b.WriteString("\n") + + // Global dedup metrics + b.WriteString("# HELP dbbackup_dedup_chunks_total Total number of unique chunks stored\n") + b.WriteString("# TYPE dbbackup_dedup_chunks_total gauge\n") + b.WriteString(fmt.Sprintf("dbbackup_dedup_chunks_total{instance=%q} %d\n", instance, m.TotalChunks)) + b.WriteString("\n") + + b.WriteString("# HELP dbbackup_dedup_manifests_total Total number of deduplicated backups\n") + b.WriteString("# TYPE dbbackup_dedup_manifests_total gauge\n") + b.WriteString(fmt.Sprintf("dbbackup_dedup_manifests_total{instance=%q} %d\n", instance, m.TotalManifests)) + b.WriteString("\n") + + b.WriteString("# HELP dbbackup_dedup_backup_bytes_total Total logical size of all backups in bytes\n") + b.WriteString("# TYPE dbbackup_dedup_backup_bytes_total gauge\n") + b.WriteString(fmt.Sprintf("dbbackup_dedup_backup_bytes_total{instance=%q} %d\n", instance, m.TotalBackupSize)) + b.WriteString("\n") + + b.WriteString("# HELP dbbackup_dedup_stored_bytes_total Total unique data stored in bytes (after dedup)\n") + b.WriteString("# TYPE dbbackup_dedup_stored_bytes_total gauge\n") + b.WriteString(fmt.Sprintf("dbbackup_dedup_stored_bytes_total{instance=%q} %d\n", instance, m.TotalNewData)) + b.WriteString("\n") + + b.WriteString("# HELP dbbackup_dedup_space_saved_bytes Bytes saved by deduplication\n") + b.WriteString("# TYPE dbbackup_dedup_space_saved_bytes gauge\n") + b.WriteString(fmt.Sprintf("dbbackup_dedup_space_saved_bytes{instance=%q} %d\n", instance, m.SpaceSaved)) + b.WriteString("\n") + + b.WriteString("# HELP dbbackup_dedup_ratio Deduplication ratio (0-1, higher is better)\n") + b.WriteString("# TYPE dbbackup_dedup_ratio gauge\n") + b.WriteString(fmt.Sprintf("dbbackup_dedup_ratio{instance=%q} %.4f\n", instance, m.DedupRatio)) + b.WriteString("\n") + + b.WriteString("# HELP dbbackup_dedup_disk_usage_bytes Actual disk usage of chunk store\n") + b.WriteString("# TYPE dbbackup_dedup_disk_usage_bytes gauge\n") + b.WriteString(fmt.Sprintf("dbbackup_dedup_disk_usage_bytes{instance=%q} %d\n", instance, m.DiskUsage)) + b.WriteString("\n") + + // Per-database metrics + if len(m.ByDatabase) > 0 { + b.WriteString("# HELP dbbackup_dedup_database_backup_count Number of deduplicated backups per database\n") + b.WriteString("# TYPE dbbackup_dedup_database_backup_count gauge\n") + for _, db := range m.ByDatabase { + b.WriteString(fmt.Sprintf("dbbackup_dedup_database_backup_count{instance=%q,database=%q} %d\n", + instance, db.Database, db.BackupCount)) + } + b.WriteString("\n") + + b.WriteString("# HELP dbbackup_dedup_database_ratio Deduplication ratio per database (0-1)\n") + b.WriteString("# TYPE dbbackup_dedup_database_ratio gauge\n") + for _, db := range m.ByDatabase { + b.WriteString(fmt.Sprintf("dbbackup_dedup_database_ratio{instance=%q,database=%q} %.4f\n", + instance, db.Database, db.DedupRatio)) + } + b.WriteString("\n") + + b.WriteString("# HELP dbbackup_dedup_database_last_backup_timestamp Last backup timestamp per database\n") + b.WriteString("# TYPE dbbackup_dedup_database_last_backup_timestamp gauge\n") + for _, db := range m.ByDatabase { + if !db.LastBackupTime.IsZero() { + b.WriteString(fmt.Sprintf("dbbackup_dedup_database_last_backup_timestamp{instance=%q,database=%q} %d\n", + instance, db.Database, db.LastBackupTime.Unix())) + } + } + b.WriteString("\n") + } + + b.WriteString("# HELP dbbackup_dedup_scrape_timestamp Unix timestamp when dedup metrics were collected\n") + b.WriteString("# TYPE dbbackup_dedup_scrape_timestamp gauge\n") + b.WriteString(fmt.Sprintf("dbbackup_dedup_scrape_timestamp{instance=%q} %d\n", instance, now)) + + return b.String() +}