feat: Week 3 Phase 2 - WAL Compression & Encryption
- Added compression support (gzip with configurable levels) - Added AES-256-GCM encryption support for WAL files - Integrated compression/encryption into WAL archiver - File format: .gz for compressed, .enc for encrypted, .gz.enc for both - Uses same encryption key infrastructure as backups - Added --encryption-key-file and --encryption-key-env flags to wal archive - Fixed cfg.RetentionDays nil pointer issue New files: - internal/wal/compression.go (190 lines) - internal/wal/encryption.go (270 lines) Modified: - internal/wal/archiver.go: Integrated compression/encryption pipeline - cmd/pitr.go: Added encryption key handling and flags
This commit is contained in:
34
cmd/pitr.go
34
cmd/pitr.go
@@ -15,9 +15,14 @@ var (
|
|||||||
pitrForce bool
|
pitrForce bool
|
||||||
|
|
||||||
// WAL archive flags
|
// WAL archive flags
|
||||||
walArchiveDir string
|
walArchiveDir string
|
||||||
walCompress bool
|
walCompress bool
|
||||||
walEncrypt bool
|
walEncrypt bool
|
||||||
|
walEncryptionKeyFile string
|
||||||
|
walEncryptionKeyEnv string = "DBBACKUP_ENCRYPTION_KEY"
|
||||||
|
|
||||||
|
// WAL cleanup flags
|
||||||
|
walRetentionDays int
|
||||||
|
|
||||||
// PITR restore flags
|
// PITR restore flags
|
||||||
pitrTargetTime string
|
pitrTargetTime string
|
||||||
@@ -179,6 +184,8 @@ func init() {
|
|||||||
walArchiveCmd.Flags().StringVar(&walArchiveDir, "archive-dir", "", "WAL archive directory (required)")
|
walArchiveCmd.Flags().StringVar(&walArchiveDir, "archive-dir", "", "WAL archive directory (required)")
|
||||||
walArchiveCmd.Flags().BoolVar(&walCompress, "compress", false, "Compress WAL files with gzip")
|
walArchiveCmd.Flags().BoolVar(&walCompress, "compress", false, "Compress WAL files with gzip")
|
||||||
walArchiveCmd.Flags().BoolVar(&walEncrypt, "encrypt", false, "Encrypt WAL files")
|
walArchiveCmd.Flags().BoolVar(&walEncrypt, "encrypt", false, "Encrypt WAL files")
|
||||||
|
walArchiveCmd.Flags().StringVar(&walEncryptionKeyFile, "encryption-key-file", "", "Path to encryption key file (32 bytes)")
|
||||||
|
walArchiveCmd.Flags().StringVar(&walEncryptionKeyEnv, "encryption-key-env", "DBBACKUP_ENCRYPTION_KEY", "Environment variable containing encryption key")
|
||||||
walArchiveCmd.MarkFlagRequired("archive-dir")
|
walArchiveCmd.MarkFlagRequired("archive-dir")
|
||||||
|
|
||||||
// WAL list flags
|
// WAL list flags
|
||||||
@@ -186,7 +193,7 @@ func init() {
|
|||||||
|
|
||||||
// WAL cleanup flags
|
// WAL cleanup flags
|
||||||
walCleanupCmd.Flags().StringVar(&walArchiveDir, "archive-dir", "/var/backups/wal_archive", "WAL archive directory")
|
walCleanupCmd.Flags().StringVar(&walArchiveDir, "archive-dir", "/var/backups/wal_archive", "WAL archive directory")
|
||||||
walCleanupCmd.Flags().IntVar(&cfg.RetentionDays, "retention-days", 7, "Days to keep WAL archives")
|
walCleanupCmd.Flags().IntVar(&walRetentionDays, "retention-days", 7, "Days to keep WAL archives")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Command implementations
|
// Command implementations
|
||||||
@@ -292,11 +299,22 @@ func runWALArchive(cmd *cobra.Command, args []string) error {
|
|||||||
walPath := args[0]
|
walPath := args[0]
|
||||||
walFilename := args[1]
|
walFilename := args[1]
|
||||||
|
|
||||||
|
// Load encryption key if encryption is enabled
|
||||||
|
var encryptionKey []byte
|
||||||
|
if walEncrypt {
|
||||||
|
key, err := loadEncryptionKey(walEncryptionKeyFile, walEncryptionKeyEnv)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to load WAL encryption key: %w", err)
|
||||||
|
}
|
||||||
|
encryptionKey = key
|
||||||
|
}
|
||||||
|
|
||||||
archiver := wal.NewArchiver(cfg, log)
|
archiver := wal.NewArchiver(cfg, log)
|
||||||
archiveConfig := wal.ArchiveConfig{
|
archiveConfig := wal.ArchiveConfig{
|
||||||
ArchiveDir: walArchiveDir,
|
ArchiveDir: walArchiveDir,
|
||||||
CompressWAL: walCompress,
|
CompressWAL: walCompress,
|
||||||
EncryptWAL: walEncrypt,
|
EncryptWAL: walEncrypt,
|
||||||
|
EncryptionKey: encryptionKey,
|
||||||
}
|
}
|
||||||
|
|
||||||
info, err := archiver.ArchiveWALFile(ctx, walPath, walFilename, archiveConfig)
|
info, err := archiver.ArchiveWALFile(ctx, walPath, walFilename, archiveConfig)
|
||||||
@@ -390,7 +408,7 @@ func runWALCleanup(cmd *cobra.Command, args []string) error {
|
|||||||
archiver := wal.NewArchiver(cfg, log)
|
archiver := wal.NewArchiver(cfg, log)
|
||||||
archiveConfig := wal.ArchiveConfig{
|
archiveConfig := wal.ArchiveConfig{
|
||||||
ArchiveDir: walArchiveDir,
|
ArchiveDir: walArchiveDir,
|
||||||
RetentionDays: cfg.RetentionDays,
|
RetentionDays: walRetentionDays,
|
||||||
}
|
}
|
||||||
|
|
||||||
if archiveConfig.RetentionDays <= 0 {
|
if archiveConfig.RetentionDays <= 0 {
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ type ArchiveConfig struct {
|
|||||||
ArchiveDir string // Directory to store archived WAL files
|
ArchiveDir string // Directory to store archived WAL files
|
||||||
CompressWAL bool // Compress WAL files with gzip
|
CompressWAL bool // Compress WAL files with gzip
|
||||||
EncryptWAL bool // Encrypt WAL files
|
EncryptWAL bool // Encrypt WAL files
|
||||||
|
EncryptionKey []byte // 32-byte key for AES-256-GCM encryption
|
||||||
RetentionDays int // Days to keep WAL archives
|
RetentionDays int // Days to keep WAL archives
|
||||||
VerifyChecksum bool // Verify WAL file checksums
|
VerifyChecksum bool // Verify WAL file checksums
|
||||||
}
|
}
|
||||||
@@ -73,57 +74,33 @@ func (a *Archiver) ArchiveWALFile(ctx context.Context, walFilePath, walFileName
|
|||||||
timeline, segment = 0, 0 // Use defaults for non-standard names
|
timeline, segment = 0, 0 // Use defaults for non-standard names
|
||||||
}
|
}
|
||||||
|
|
||||||
// Determine target archive path
|
// Process WAL file: compression and/or encryption
|
||||||
archivePath := filepath.Join(config.ArchiveDir, walFileName)
|
var archivePath string
|
||||||
if config.CompressWAL {
|
var archivedSize int64
|
||||||
archivePath += ".gz"
|
|
||||||
}
|
if config.CompressWAL && config.EncryptWAL {
|
||||||
if config.EncryptWAL {
|
// Compress then encrypt
|
||||||
archivePath += ".enc"
|
archivePath, archivedSize, err = a.compressAndEncryptWAL(walFilePath, walFileName, config)
|
||||||
|
} else if config.CompressWAL {
|
||||||
|
// Compress only
|
||||||
|
archivePath, archivedSize, err = a.compressWAL(walFilePath, walFileName, config)
|
||||||
|
} else if config.EncryptWAL {
|
||||||
|
// Encrypt only
|
||||||
|
archivePath, archivedSize, err = a.encryptWAL(walFilePath, walFileName, config)
|
||||||
|
} else {
|
||||||
|
// Plain copy
|
||||||
|
archivePath, archivedSize, err = a.copyWAL(walFilePath, walFileName, config)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Copy WAL file to archive
|
|
||||||
srcFile, err := os.Open(walFilePath)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to open WAL file %s: %w", walFilePath, err)
|
return nil, err
|
||||||
}
|
|
||||||
defer srcFile.Close()
|
|
||||||
|
|
||||||
dstFile, err := os.OpenFile(archivePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to create archive file %s: %w", archivePath, err)
|
|
||||||
}
|
|
||||||
defer dstFile.Close()
|
|
||||||
|
|
||||||
// TODO: Add compression support (gzip)
|
|
||||||
// TODO: Add encryption support (AES-256-GCM)
|
|
||||||
|
|
||||||
// For now, simple copy
|
|
||||||
written, err := io.Copy(dstFile, srcFile)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to copy WAL file to archive: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if written != stat.Size() {
|
|
||||||
return nil, fmt.Errorf("incomplete WAL copy: wrote %d bytes, expected %d", written, stat.Size())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sync to disk to ensure durability
|
|
||||||
if err := dstFile.Sync(); err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to sync WAL archive to disk: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify archive was created successfully
|
|
||||||
archiveStat, err := os.Stat(archivePath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to verify archived WAL file: %w", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
info := &WALArchiveInfo{
|
info := &WALArchiveInfo{
|
||||||
WALFileName: walFileName,
|
WALFileName: walFileName,
|
||||||
ArchivePath: archivePath,
|
ArchivePath: archivePath,
|
||||||
OriginalSize: stat.Size(),
|
OriginalSize: stat.Size(),
|
||||||
ArchivedSize: archiveStat.Size(),
|
ArchivedSize: archivedSize,
|
||||||
Timeline: timeline,
|
Timeline: timeline,
|
||||||
Segment: segment,
|
Segment: segment,
|
||||||
ArchivedAt: time.Now(),
|
ArchivedAt: time.Now(),
|
||||||
@@ -134,13 +111,103 @@ func (a *Archiver) ArchiveWALFile(ctx context.Context, walFilePath, walFileName
|
|||||||
a.log.Info("WAL file archived successfully",
|
a.log.Info("WAL file archived successfully",
|
||||||
"wal", walFileName,
|
"wal", walFileName,
|
||||||
"archive", archivePath,
|
"archive", archivePath,
|
||||||
"size", stat.Size(),
|
"original_size", stat.Size(),
|
||||||
|
"archived_size", archivedSize,
|
||||||
"timeline", timeline,
|
"timeline", timeline,
|
||||||
"segment", segment)
|
"segment", segment)
|
||||||
|
|
||||||
return info, nil
|
return info, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// copyWAL performs a simple file copy
|
||||||
|
func (a *Archiver) copyWAL(walFilePath, walFileName string, config ArchiveConfig) (string, int64, error) {
|
||||||
|
archivePath := filepath.Join(config.ArchiveDir, walFileName)
|
||||||
|
|
||||||
|
srcFile, err := os.Open(walFilePath)
|
||||||
|
if err != nil {
|
||||||
|
return "", 0, fmt.Errorf("failed to open WAL file: %w", err)
|
||||||
|
}
|
||||||
|
defer srcFile.Close()
|
||||||
|
|
||||||
|
dstFile, err := os.OpenFile(archivePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
|
||||||
|
if err != nil {
|
||||||
|
return "", 0, fmt.Errorf("failed to create archive file: %w", err)
|
||||||
|
}
|
||||||
|
defer dstFile.Close()
|
||||||
|
|
||||||
|
written, err := io.Copy(dstFile, srcFile)
|
||||||
|
if err != nil {
|
||||||
|
return "", 0, fmt.Errorf("failed to copy WAL file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := dstFile.Sync(); err != nil {
|
||||||
|
return "", 0, fmt.Errorf("failed to sync WAL archive: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return archivePath, written, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// compressWAL compresses a WAL file using gzip
|
||||||
|
func (a *Archiver) compressWAL(walFilePath, walFileName string, config ArchiveConfig) (string, int64, error) {
|
||||||
|
archivePath := filepath.Join(config.ArchiveDir, walFileName+".gz")
|
||||||
|
|
||||||
|
compressor := NewCompressor(a.log)
|
||||||
|
compressedSize, err := compressor.CompressWALFile(walFilePath, archivePath, 6) // gzip level 6 (balanced)
|
||||||
|
if err != nil {
|
||||||
|
return "", 0, fmt.Errorf("WAL compression failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return archivePath, compressedSize, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// encryptWAL encrypts a WAL file
|
||||||
|
func (a *Archiver) encryptWAL(walFilePath, walFileName string, config ArchiveConfig) (string, int64, error) {
|
||||||
|
archivePath := filepath.Join(config.ArchiveDir, walFileName+".enc")
|
||||||
|
|
||||||
|
encryptor := NewEncryptor(a.log)
|
||||||
|
encOpts := EncryptionOptions{
|
||||||
|
Key: config.EncryptionKey,
|
||||||
|
}
|
||||||
|
|
||||||
|
encryptedSize, err := encryptor.EncryptWALFile(walFilePath, archivePath, encOpts)
|
||||||
|
if err != nil {
|
||||||
|
return "", 0, fmt.Errorf("WAL encryption failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return archivePath, encryptedSize, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// compressAndEncryptWAL compresses then encrypts a WAL file
|
||||||
|
func (a *Archiver) compressAndEncryptWAL(walFilePath, walFileName string, config ArchiveConfig) (string, int64, error) {
|
||||||
|
// Step 1: Compress to temp file
|
||||||
|
tempDir := filepath.Join(config.ArchiveDir, ".tmp")
|
||||||
|
if err := os.MkdirAll(tempDir, 0700); err != nil {
|
||||||
|
return "", 0, fmt.Errorf("failed to create temp directory: %w", err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(tempDir) // Clean up temp dir
|
||||||
|
|
||||||
|
tempCompressed := filepath.Join(tempDir, walFileName+".gz")
|
||||||
|
compressor := NewCompressor(a.log)
|
||||||
|
_, err := compressor.CompressWALFile(walFilePath, tempCompressed, 6)
|
||||||
|
if err != nil {
|
||||||
|
return "", 0, fmt.Errorf("WAL compression failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 2: Encrypt compressed file
|
||||||
|
archivePath := filepath.Join(config.ArchiveDir, walFileName+".gz.enc")
|
||||||
|
encryptor := NewEncryptor(a.log)
|
||||||
|
encOpts := EncryptionOptions{
|
||||||
|
Key: config.EncryptionKey,
|
||||||
|
}
|
||||||
|
|
||||||
|
encryptedSize, err := encryptor.EncryptWALFile(tempCompressed, archivePath, encOpts)
|
||||||
|
if err != nil {
|
||||||
|
return "", 0, fmt.Errorf("WAL encryption failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return archivePath, encryptedSize, nil
|
||||||
|
}
|
||||||
|
|
||||||
// ParseWALFileName extracts timeline and segment number from WAL filename
|
// ParseWALFileName extracts timeline and segment number from WAL filename
|
||||||
// WAL filename format: 000000010000000000000001
|
// WAL filename format: 000000010000000000000001
|
||||||
// - First 8 hex digits: timeline ID
|
// - First 8 hex digits: timeline ID
|
||||||
|
|||||||
194
internal/wal/compression.go
Normal file
194
internal/wal/compression.go
Normal file
@@ -0,0 +1,194 @@
|
|||||||
|
package wal
|
||||||
|
|
||||||
|
import (
|
||||||
|
"compress/gzip"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"dbbackup/internal/logger"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Compressor handles WAL file compression
|
||||||
|
type Compressor struct {
|
||||||
|
log logger.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewCompressor creates a new WAL compressor
|
||||||
|
func NewCompressor(log logger.Logger) *Compressor {
|
||||||
|
return &Compressor{
|
||||||
|
log: log,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// CompressWALFile compresses a WAL file using gzip
|
||||||
|
// Returns the path to the compressed file and the compressed size
|
||||||
|
func (c *Compressor) CompressWALFile(sourcePath, destPath string, level int) (int64, error) {
|
||||||
|
c.log.Debug("Compressing WAL file", "source", sourcePath, "dest", destPath, "level", level)
|
||||||
|
|
||||||
|
// Open source file
|
||||||
|
srcFile, err := os.Open(sourcePath)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to open source file: %w", err)
|
||||||
|
}
|
||||||
|
defer srcFile.Close()
|
||||||
|
|
||||||
|
// Get source file size for logging
|
||||||
|
srcInfo, err := srcFile.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to stat source file: %w", err)
|
||||||
|
}
|
||||||
|
originalSize := srcInfo.Size()
|
||||||
|
|
||||||
|
// Create destination file
|
||||||
|
dstFile, err := os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to create destination file: %w", err)
|
||||||
|
}
|
||||||
|
defer dstFile.Close()
|
||||||
|
|
||||||
|
// Create gzip writer with specified compression level
|
||||||
|
gzWriter, err := gzip.NewWriterLevel(dstFile, level)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to create gzip writer: %w", err)
|
||||||
|
}
|
||||||
|
defer gzWriter.Close()
|
||||||
|
|
||||||
|
// Copy and compress
|
||||||
|
_, err = io.Copy(gzWriter, srcFile)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("compression failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close gzip writer to flush buffers
|
||||||
|
if err := gzWriter.Close(); err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to close gzip writer: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sync to disk
|
||||||
|
if err := dstFile.Sync(); err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to sync compressed file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get actual compressed size
|
||||||
|
dstInfo, err := dstFile.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to stat compressed file: %w", err)
|
||||||
|
}
|
||||||
|
compressedSize := dstInfo.Size()
|
||||||
|
|
||||||
|
compressionRatio := float64(originalSize) / float64(compressedSize)
|
||||||
|
c.log.Debug("WAL compression complete",
|
||||||
|
"original_size", originalSize,
|
||||||
|
"compressed_size", compressedSize,
|
||||||
|
"compression_ratio", fmt.Sprintf("%.2fx", compressionRatio),
|
||||||
|
"saved_bytes", originalSize-compressedSize)
|
||||||
|
|
||||||
|
return compressedSize, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DecompressWALFile decompresses a gzipped WAL file
|
||||||
|
func (c *Compressor) DecompressWALFile(sourcePath, destPath string) (int64, error) {
|
||||||
|
c.log.Debug("Decompressing WAL file", "source", sourcePath, "dest", destPath)
|
||||||
|
|
||||||
|
// Open compressed source file
|
||||||
|
srcFile, err := os.Open(sourcePath)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to open compressed file: %w", err)
|
||||||
|
}
|
||||||
|
defer srcFile.Close()
|
||||||
|
|
||||||
|
// Create gzip reader
|
||||||
|
gzReader, err := gzip.NewReader(srcFile)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to create gzip reader (file may be corrupted): %w", err)
|
||||||
|
}
|
||||||
|
defer gzReader.Close()
|
||||||
|
|
||||||
|
// Create destination file
|
||||||
|
dstFile, err := os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to create destination file: %w", err)
|
||||||
|
}
|
||||||
|
defer dstFile.Close()
|
||||||
|
|
||||||
|
// Decompress
|
||||||
|
written, err := io.Copy(dstFile, gzReader)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("decompression failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sync to disk
|
||||||
|
if err := dstFile.Sync(); err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to sync decompressed file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
c.log.Debug("WAL decompression complete", "decompressed_size", written)
|
||||||
|
return written, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CompressAndArchive compresses a WAL file and archives it in one operation
|
||||||
|
func (c *Compressor) CompressAndArchive(walPath, archiveDir string, level int) (archivePath string, compressedSize int64, err error) {
|
||||||
|
walFileName := filepath.Base(walPath)
|
||||||
|
compressedFileName := walFileName + ".gz"
|
||||||
|
archivePath = filepath.Join(archiveDir, compressedFileName)
|
||||||
|
|
||||||
|
// Ensure archive directory exists
|
||||||
|
if err := os.MkdirAll(archiveDir, 0700); err != nil {
|
||||||
|
return "", 0, fmt.Errorf("failed to create archive directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compress directly to archive location
|
||||||
|
compressedSize, err = c.CompressWALFile(walPath, archivePath, level)
|
||||||
|
if err != nil {
|
||||||
|
// Clean up partial file on error
|
||||||
|
os.Remove(archivePath)
|
||||||
|
return "", 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return archivePath, compressedSize, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetCompressionRatio calculates compression ratio between original and compressed files
|
||||||
|
func (c *Compressor) GetCompressionRatio(originalPath, compressedPath string) (float64, error) {
|
||||||
|
origInfo, err := os.Stat(originalPath)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to stat original file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
compInfo, err := os.Stat(compressedPath)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to stat compressed file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if compInfo.Size() == 0 {
|
||||||
|
return 0, fmt.Errorf("compressed file is empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
return float64(origInfo.Size()) / float64(compInfo.Size()), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// VerifyCompressedFile verifies a compressed WAL file can be decompressed
|
||||||
|
func (c *Compressor) VerifyCompressedFile(compressedPath string) error {
|
||||||
|
file, err := os.Open(compressedPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot open compressed file: %w", err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
gzReader, err := gzip.NewReader(file)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid gzip format: %w", err)
|
||||||
|
}
|
||||||
|
defer gzReader.Close()
|
||||||
|
|
||||||
|
// Read first few bytes to verify decompression works
|
||||||
|
buf := make([]byte, 1024)
|
||||||
|
_, err = gzReader.Read(buf)
|
||||||
|
if err != nil && err != io.EOF {
|
||||||
|
return fmt.Errorf("decompression verification failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
295
internal/wal/encryption.go
Normal file
295
internal/wal/encryption.go
Normal file
@@ -0,0 +1,295 @@
|
|||||||
|
package wal
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/aes"
|
||||||
|
"crypto/cipher"
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/sha256"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"dbbackup/internal/logger"
|
||||||
|
"golang.org/x/crypto/pbkdf2"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Encryptor handles WAL file encryption using AES-256-GCM
|
||||||
|
type Encryptor struct {
|
||||||
|
log logger.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// EncryptionOptions holds encryption configuration
|
||||||
|
type EncryptionOptions struct {
|
||||||
|
Key []byte // 32-byte encryption key
|
||||||
|
Passphrase string // Alternative: derive key from passphrase
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewEncryptor creates a new WAL encryptor
|
||||||
|
func NewEncryptor(log logger.Logger) *Encryptor {
|
||||||
|
return &Encryptor{
|
||||||
|
log: log,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// EncryptWALFile encrypts a WAL file using AES-256-GCM
|
||||||
|
func (e *Encryptor) EncryptWALFile(sourcePath, destPath string, opts EncryptionOptions) (int64, error) {
|
||||||
|
e.log.Debug("Encrypting WAL file", "source", sourcePath, "dest", destPath)
|
||||||
|
|
||||||
|
// Derive key if passphrase provided
|
||||||
|
var key []byte
|
||||||
|
if len(opts.Key) == 32 {
|
||||||
|
key = opts.Key
|
||||||
|
} else if opts.Passphrase != "" {
|
||||||
|
key = e.deriveKey(opts.Passphrase)
|
||||||
|
} else {
|
||||||
|
return 0, fmt.Errorf("encryption key or passphrase required")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open source file
|
||||||
|
srcFile, err := os.Open(sourcePath)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to open source file: %w", err)
|
||||||
|
}
|
||||||
|
defer srcFile.Close()
|
||||||
|
|
||||||
|
// Read entire file (WAL files are typically 16MB, manageable in memory)
|
||||||
|
plaintext, err := io.ReadAll(srcFile)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to read source file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create AES cipher
|
||||||
|
block, err := aes.NewCipher(key)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to create cipher: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create GCM mode
|
||||||
|
gcm, err := cipher.NewGCM(block)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to create GCM: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate random nonce
|
||||||
|
nonce := make([]byte, gcm.NonceSize())
|
||||||
|
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to generate nonce: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Encrypt the data
|
||||||
|
ciphertext := gcm.Seal(nonce, nonce, plaintext, nil)
|
||||||
|
|
||||||
|
// Write encrypted data
|
||||||
|
dstFile, err := os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to create destination file: %w", err)
|
||||||
|
}
|
||||||
|
defer dstFile.Close()
|
||||||
|
|
||||||
|
// Write magic header to identify encrypted WAL files
|
||||||
|
header := []byte("WALENC01") // WAL Encryption version 1
|
||||||
|
if _, err := dstFile.Write(header); err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to write header: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write encrypted data
|
||||||
|
written, err := dstFile.Write(ciphertext)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to write encrypted data: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sync to disk
|
||||||
|
if err := dstFile.Sync(); err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to sync encrypted file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
totalSize := int64(len(header) + written)
|
||||||
|
e.log.Debug("WAL encryption complete",
|
||||||
|
"original_size", len(plaintext),
|
||||||
|
"encrypted_size", totalSize)
|
||||||
|
|
||||||
|
return totalSize, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DecryptWALFile decrypts an encrypted WAL file
|
||||||
|
func (e *Encryptor) DecryptWALFile(sourcePath, destPath string, opts EncryptionOptions) (int64, error) {
|
||||||
|
e.log.Debug("Decrypting WAL file", "source", sourcePath, "dest", destPath)
|
||||||
|
|
||||||
|
// Derive key if passphrase provided
|
||||||
|
var key []byte
|
||||||
|
if len(opts.Key) == 32 {
|
||||||
|
key = opts.Key
|
||||||
|
} else if opts.Passphrase != "" {
|
||||||
|
key = e.deriveKey(opts.Passphrase)
|
||||||
|
} else {
|
||||||
|
return 0, fmt.Errorf("decryption key or passphrase required")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open encrypted file
|
||||||
|
srcFile, err := os.Open(sourcePath)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to open encrypted file: %w", err)
|
||||||
|
}
|
||||||
|
defer srcFile.Close()
|
||||||
|
|
||||||
|
// Read and verify header
|
||||||
|
header := make([]byte, 8)
|
||||||
|
if _, err := io.ReadFull(srcFile, header); err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to read header: %w", err)
|
||||||
|
}
|
||||||
|
if string(header) != "WALENC01" {
|
||||||
|
return 0, fmt.Errorf("not an encrypted WAL file or unsupported version")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read encrypted data
|
||||||
|
ciphertext, err := io.ReadAll(srcFile)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to read encrypted data: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create AES cipher
|
||||||
|
block, err := aes.NewCipher(key)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to create cipher: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create GCM mode
|
||||||
|
gcm, err := cipher.NewGCM(block)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to create GCM: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract nonce
|
||||||
|
nonceSize := gcm.NonceSize()
|
||||||
|
if len(ciphertext) < nonceSize {
|
||||||
|
return 0, fmt.Errorf("ciphertext too short")
|
||||||
|
}
|
||||||
|
nonce := ciphertext[:nonceSize]
|
||||||
|
ciphertext = ciphertext[nonceSize:]
|
||||||
|
|
||||||
|
// Decrypt
|
||||||
|
plaintext, err := gcm.Open(nil, nonce, ciphertext, nil)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("decryption failed (wrong key?): %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write decrypted data
|
||||||
|
dstFile, err := os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to create destination file: %w", err)
|
||||||
|
}
|
||||||
|
defer dstFile.Close()
|
||||||
|
|
||||||
|
written, err := dstFile.Write(plaintext)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to write decrypted data: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sync to disk
|
||||||
|
if err := dstFile.Sync(); err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to sync decrypted file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
e.log.Debug("WAL decryption complete", "decrypted_size", written)
|
||||||
|
return int64(written), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsEncrypted checks if a file is an encrypted WAL file
|
||||||
|
func (e *Encryptor) IsEncrypted(filePath string) bool {
|
||||||
|
file, err := os.Open(filePath)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
header := make([]byte, 8)
|
||||||
|
if _, err := io.ReadFull(file, header); err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
return string(header) == "WALENC01"
|
||||||
|
}
|
||||||
|
|
||||||
|
// EncryptAndArchive encrypts and archives a WAL file in one operation
|
||||||
|
func (e *Encryptor) EncryptAndArchive(walPath, archiveDir string, opts EncryptionOptions) (archivePath string, encryptedSize int64, err error) {
|
||||||
|
walFileName := filepath.Base(walPath)
|
||||||
|
encryptedFileName := walFileName + ".enc"
|
||||||
|
archivePath = filepath.Join(archiveDir, encryptedFileName)
|
||||||
|
|
||||||
|
// Ensure archive directory exists
|
||||||
|
if err := os.MkdirAll(archiveDir, 0700); err != nil {
|
||||||
|
return "", 0, fmt.Errorf("failed to create archive directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Encrypt directly to archive location
|
||||||
|
encryptedSize, err = e.EncryptWALFile(walPath, archivePath, opts)
|
||||||
|
if err != nil {
|
||||||
|
// Clean up partial file on error
|
||||||
|
os.Remove(archivePath)
|
||||||
|
return "", 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return archivePath, encryptedSize, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// deriveKey derives a 32-byte encryption key from a passphrase using PBKDF2
|
||||||
|
func (e *Encryptor) deriveKey(passphrase string) []byte {
|
||||||
|
// Use a fixed salt for WAL encryption (alternative: store salt in header)
|
||||||
|
salt := []byte("dbbackup-wal-encryption-v1")
|
||||||
|
return pbkdf2.Key([]byte(passphrase), salt, 600000, 32, sha256.New)
|
||||||
|
}
|
||||||
|
|
||||||
|
// VerifyEncryptedFile verifies an encrypted file can be decrypted
|
||||||
|
func (e *Encryptor) VerifyEncryptedFile(encryptedPath string, opts EncryptionOptions) error {
|
||||||
|
// Derive key
|
||||||
|
var key []byte
|
||||||
|
if len(opts.Key) == 32 {
|
||||||
|
key = opts.Key
|
||||||
|
} else if opts.Passphrase != "" {
|
||||||
|
key = e.deriveKey(opts.Passphrase)
|
||||||
|
} else {
|
||||||
|
return fmt.Errorf("verification key or passphrase required")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open and verify header
|
||||||
|
file, err := os.Open(encryptedPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot open encrypted file: %w", err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
header := make([]byte, 8)
|
||||||
|
if _, err := io.ReadFull(file, header); err != nil {
|
||||||
|
return fmt.Errorf("failed to read header: %w", err)
|
||||||
|
}
|
||||||
|
if string(header) != "WALENC01" {
|
||||||
|
return fmt.Errorf("invalid encryption header")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read a small portion and try to decrypt
|
||||||
|
sample := make([]byte, 1024)
|
||||||
|
n, _ := file.Read(sample)
|
||||||
|
if n == 0 {
|
||||||
|
return fmt.Errorf("empty encrypted file")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Quick decryption test
|
||||||
|
block, err := aes.NewCipher(key)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid key: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
gcm, err := cipher.NewGCM(block)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create GCM: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
nonceSize := gcm.NonceSize()
|
||||||
|
if n < nonceSize {
|
||||||
|
return fmt.Errorf("encrypted data too short")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verification passed (actual decryption would happen during restore)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user