uuxo/hmac-file-server
1
1
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 4 Wiki Activity

2.2-stable

Browse Source
This commit is contained in:
Alexander Renz 2024-12-24 15:41:13 +01:00
parent 95cbd11ba3
commit e542a7e948
4 changed files with 359 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
README.MD
View File

@ -75,106 +75,11 @@ Set `thumbnail` to true in the `[server]` section of `config.toml` to enable ima
### Deduplication ### Deduplication
Set `enabled` to true in the `[deduplication]` section of `config.toml` to enable file deduplication. Specify the `storagepath` where deduplicated files will be stored.
---
## Deduplication
Set `enabled` to true in the `[deduplication]` section of `config.toml` to enable file deduplication. Specify the `directory` where deduplicated files will be stored. Set `enabled` to true in the `[deduplication]` section of `config.toml` to enable file deduplication. Specify the `directory` where deduplicated files will be stored.
### Example `config.toml`
```toml
[deduplication]
enabled = true
directory = "/mnt/hmac-storage/deduplication/"
```
## Thumbnails
Set `enabled` to true in the `[thumbnails]` section of `config.toml` to enable thumbnail creation. Specify the `directory` where thumbnails will be stored and the `size` of the thumbnails.
### Example `config.toml`
```toml
[thumbnails]
enabled = true
directory = "/mnt/hmac-storage/thumbnails/"
size = "200x200"
```
## Example `config.toml`
```toml
[server]
ListenPort = "8080"
UnixSocket = false
StoragePath = "./uploads"
LogLevel = "info"
LogFile = ""
MetricsEnabled = true
MetricsPort = "9090"
FileTTL = "1y"
DeduplicationEnabled = true
MinFreeBytes = "100MB"
AutoAdjustWorkers = true # Enable auto-adjustment for worker scaling
NetworkEvents = false # Disable logging and tracking of network-related events
PIDFilePath = "./hmac_file_server.pid" # Path to PID file
Precaching = true # Enable pre-caching of storage paths
ThumbnailEnabled = false # Whether to create thumbnails for uploaded images
[timeouts]
ReadTimeout = "480s"
WriteTimeout = "480s"
IdleTimeout = "65s" # nginx/apache2 keep-alive 60s
[security]
Secret = "changeme"
[versioning]
EnableVersioning = false
MaxVersions = 1
[uploads]
ResumableUploadsEnabled = true
ChunkedUploadsEnabled = true
ChunkSize = "64MB"
AllowedExtensions = [".txt", ".pdf", ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".svg", ".webp", ".wav", ".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".mpeg", ".mpg", ".m4v", ".3gp", ".3g2", ".mp3", ".ogg"]
[downloads]
ResumableDownloadsEnabled = true
ChunkedDownloadsEnabled = true
ChunkSize = "64MB"
[clamav]
ClamAVEnabled = false
ClamAVSocket = "/var/run/clamav/clamd.ctl"
NumScanWorkers = 2
ScanFileExtensions = [".exe", ".dll", ".bin", ".com", ".bat", ".sh", ".php", ".js"]
[redis]
RedisEnabled = false
RedisAddr = "localhost:6379"
RedisPassword = ""
RedisDBIndex = 0
RedisHealthCheckInterval = "120s"
[workers]
NumWorkers = 4
UploadQueueSize = 5000
[file]
FileRevision = 1 # Revision number for file handling
[deduplication]
enabled = true
storagepath = "/mnt/nfs_vol01/hmac-file-server/deduplication/"
```
--- ---
## HMAC File Server - Version 2.2 Stable ## Example `config.toml`
Below is an example configuration file (config.toml) you can use as a reference (with sensitive data removed): Below is an example configuration file (config.toml) you can use as a reference (with sensitive data removed):
@ -190,6 +95,7 @@ metricsport = "9090"
deduplicationenabled = true deduplicationenabled = true
minfreebytes = "5GB" minfreebytes = "5GB"
filettl = "2Y" filettl = "2Y"
filettlenabled = true # Enable or disable file TTL
autoadjustworkers = true autoadjustworkers = true
networkevents = false networkevents = false
pidfilepath = "./hmac-file-server.pid" pidfilepath = "./hmac-file-server.pid"
@ -216,7 +122,7 @@ writetimeout = "3600s"
idletimeout = "3600s" idletimeout = "3600s"
[security] [security]
secret = "stellar-wisdom-orbit-echo" secret = "changeme"
[versioning] [versioning]
enableversioning = false enableversioning = false
@ -328,6 +234,18 @@ Prometheus metrics include:
--- ---
### Overview of other Projects (xep0363)
| Feature/Project | HMAC FS | mod_http_upload_ext | xmpp-http-upload (horazont) | Prosody Filer | ngx_http_upload | xmpp-http-upload (nyovaya) |
|-----------------------------|---------|----------------------|-----------------------------|---------------|----------------|----------------------------|
| **Lang** | Go | PHP | Python | Go | C (Nginx) | Python |
| **Env** | Standalone | Prosody module | Standalone | Standalone | Nginx | Standalone |
| **XMPP** | No | Yes | Yes | Yes | No | Yes |
| **Ext. Storage** | Yes | No | Possible via plugins | No | No | Yes |
| **Auth / Security** | HMAC | Token-based | Token-based | None | Basic / None | Token-based |
---
## Build & Run ## Build & Run
1. Clone the repository. 1. Clone the repository.
2. Build the server: 2. Build the server:

204
RELEASE-NOTES.MD Normal file
View File

@ -0,0 +1,204 @@
# HMAC File Server
**HMAC File Server** is a secure, scalable, and feature-rich file server with advanced capabilities like HMAC authentication, resumable uploads, chunked uploads, file versioning, and optional ClamAV scanning for file integrity and security. This server is built with extensibility and operational monitoring in mind, including Prometheus metrics support and Redis integration.
> **Credits:** The **HMAC File Server** is based on the source code of [Thomas Leister's prosody-filer](https://github.com/ThomasLeister/prosody-filer). Many features and design elements have been inspired or derived from this project.
---
## Features
- **HMAC Authentication:** Secure file uploads and downloads with HMAC tokens.
- **File Versioning:** Enable versioning for uploaded files with configurable retention.
- **Chunked and Resumable Uploads:** Handle large files efficiently with support for resumable and chunked uploads.
- **ClamAV Scanning:** Optional virus scanning for uploaded files.
- **Prometheus Metrics:** Monitor system and application-level metrics.
- **Redis Integration:** Use Redis for caching or storing application states.
- **File Expiration:** Automatically delete files after a specified TTL.
- **Graceful Shutdown:** Handles signals and ensures proper cleanup.
- **Deduplication:** Remove duplicate files based on hashing for storage efficiency.
- **Auto-Adjust Worker Scaling:** Dynamically optimize HMAC and ClamAV workers based on system resources when enabled.
---
## Repository
- **Primary Repository**: [GitHub Repository](https://github.com/PlusOne/hmac-file-server)
- **Alternative Repository**: [uuxo.net Git Repository](https://git.uuxo.net/uuxo/hmac-file-server)
---
## Installation
### Prerequisites
- Go 1.20+
- Redis (optional, if Redis integration is enabled)
- ClamAV (optional, if file scanning is enabled)
### Clone and Build
```bash
# Clone from the primary repository
git clone https://github.com/PlusOne/hmac-file-server.git
# OR clone from the alternative repository
git clone https://git.uuxo.net/uuxo/hmac-file-server.git
cd hmac-file-server
go build -o hmac-file-server main.go
```
---
## Configuration
The server configuration is managed through a `config.toml` file. Below are the supported configuration options:
### Auto-Adjust Feature
When `AutoAdjustWorkers` is enabled, the number of workers for HMAC operations and ClamAV scans is dynamically determined based on system resources. This ensures efficient resource utilization.
If `AutoAdjustWorkers = true`, the values for `NumWorkers` and `NumScanWorkers` in the configuration file will be ignored, and the server will automatically adjust these values.
### Network Events Monitoring
Setting `NetworkEvents = false` in the server configuration disables the logging and tracking of network-related events within the application. This means that functionalities such as monitoring IP changes or recording network activity will be turned off.
### Precaching
The `precaching` feature allows the server to pre-cache storage paths for faster access. This can improve performance by reducing the time needed to access frequently used storage paths.
### Added thumbnail support
- New configuration option `thumbnail` in `[server]` to enable or disable generating image thumbnails
---
## New Features
### Deduplication Support
- **Description:** Added support for file deduplication to save storage space by storing a single copy of identical files.
- **Configuration:**
```toml
[deduplication]
enabled = true
directory = "/mnt/hmac-storage/deduplication/"
```
### Thumbnail Support
- **Description:** Added support for thumbnail creation to generate smaller versions of uploaded images.
- **Configuration:**
```toml
[thumbnails]
enabled = true
directory = "/mnt/hmac-storage/thumbnails/"
size = "200x200"
```
---
## Example `config.toml`
```toml
[server]
ListenPort = "8080"
UnixSocket = false
StoragePath = "./uploads"
LogLevel = "info"
LogFile = ""
MetricsEnabled = true
MetricsPort = "9090"
FileTTL = "1y"
FileTTLEnabled = true # Enable or disable file TTL
DeduplicationEnabled = true
MinFreeBytes = "100MB"
AutoAdjustWorkers = true # Enable auto-adjustment for worker scaling
NetworkEvents = false # Disable logging and tracking of network-related events
PIDFilePath = "./hmac_file_server.pid" # Path to PID file
Precaching = true # Enable pre-caching of storage paths
ThumbnailEnabled = false # Whether to create thumbnails for uploaded images
[timeouts]
ReadTimeout = "480s"
WriteTimeout = "480s"
IdleTimeout = "65s" # nginx/apache2 keep-alive 60s
[security]
Secret = "changeme"
[versioning]
EnableVersioning = false
MaxVersions = 1
[uploads]
ResumableUploadsEnabled = true
ChunkedUploadsEnabled = true
ChunkSize = "64MB"
AllowedExtensions = [".txt", ".pdf", ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".svg", ".webp", ".wav", ".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".mpeg", ".mpg", ".m4v", ".3gp", ".3g2", ".mp3", ".ogg"]
[downloads]
ResumableDownloadsEnabled = true
ChunkedDownloadsEnabled = true
ChunkSize = "64MB"
[clamav]
ClamAVEnabled = false
ClamAVSocket = "/var/run/clamav/clamd.ctl"
NumScanWorkers = 2
ScanFileExtensions = [".exe", ".dll", ".bin", ".com", ".bat", ".sh", ".php", ".js"]
[redis]
RedisEnabled = false
RedisAddr = "localhost:6379"
RedisPassword = ""
RedisDBIndex = 0
RedisHealthCheckInterval = "120s"
[workers]
NumWorkers = 4
UploadQueueSize = 5000
[file]
FileRevision = 1 # Revision number for file handling
```
---
## Running the Server
### Basic Usage
Run the server with a configuration file:
```bash
./hmac-file-server -config ./config.toml
```
---
### Metrics Server
If `MetricsEnabled` is set to `true`, the Prometheus metrics server will be available on the port specified in `MetricsPort` (default: `9090`).
---
## Testing
To run the server locally for development:
```bash
go run main.go -config ./config.toml
```
Use tools like **cURL** or **Postman** to test file uploads and downloads.
### Example File Upload with HMAC Token
```bash
curl -X PUT -H "Authorization: Bearer <HMAC-TOKEN>" -F "file=@example.txt" http://localhost:8080/uploads/example.txt
```
Replace `<HMAC-TOKEN>` with a valid HMAC signature

28
cmd/server/config.toml
View File

@ -1,26 +1,34 @@
[server] [server]
listenport = "8080" listenport = "8080"
unixsocket = false unixsocket = false
storagepath = "./uploads" storagepath = "/"
loglevel = "debug"Q loglevel = "debug"
logfile = "./hmac-file-server.log" logfile = "./tmp/hmac-file-server.log"
metricsenabled = true metricsenabled = true
metricsport = "8081" metricsport = "8081"
filettl = "180d" filettl = "180d"
minfreebytes = "2GB" minfreebytes = "2GB"
deduplicationenabled = true
autoadjustworkers = true autoadjustworkers = true
networkevents = false networkevents = false
temppath = "/tmp/hmac" temppath = "./tmp"
loggingjson = false loggingjson = false
pidfilepath = "./hmac_file_server.pid" pidfilepath = "hmac_file_server.pid"
cleanuponexit = true cleanuponexit = true
precaching = true precaching = false
[deduplication]
enabled = false
directory = "./deduplication"
[thumbnails]
enabled = false
directory = "./thumbnails"
size = "200x200"
[iso] [iso]
enabled = false enabled = false
size = "1TB" size = "1TB"
mountpoint = "/mnt/nfs_vol01/hmac-file-server/iso/" mountpoint = "./iso"
charset = "utf-8" charset = "utf-8"
[timeouts] [timeouts]
@ -42,8 +50,8 @@ chunksize = "64MB"
allowedextensions = [".txt", ".pdf", ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".svg", ".webp", ".wav", ".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".mpeg", ".mpg", ".m4v", ".3gp", ".3g2", ".mp3", ".ogg", ".zip", ".rar"] allowedextensions = [".txt", ".pdf", ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".svg", ".webp", ".wav", ".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".mpeg", ".mpg", ".m4v", ".3gp", ".3g2", ".mp3", ".ogg", ".zip", ".rar"]
[downloads] [downloads]
resumabledownloadsenabled = true resumabledownloadsenabled = false
chunkeddownloadsenabled = true chunkeddownloadsenabled = false
chunksize = "64MB" chunksize = "64MB"
[clamav] [clamav]

158
cmd/server/main.go
View File

@ -26,6 +26,7 @@ import (
"syscall" "syscall"
"time" "time"
"github.com/disintegration/imaging"
"github.com/dutchcoders/go-clamd" // ClamAV integration "github.com/dutchcoders/go-clamd" // ClamAV integration
"github.com/go-redis/redis/v8" // Redis integration "github.com/go-redis/redis/v8" // Redis integration
"github.com/patrickmn/go-cache" "github.com/patrickmn/go-cache"
@ -38,7 +39,6 @@ import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/spf13/viper" "github.com/spf13/viper"
"gopkg.in/natefinch/lumberjack.v2" "gopkg.in/natefinch/lumberjack.v2"
"github.com/disintegration/imaging"
) )
// parseSize converts a human-readable size string to bytes // parseSize converts a human-readable size string to bytes
@ -104,6 +104,7 @@ type ServerConfig struct {
LogFile string `mapstructure:"LogFile"` LogFile string `mapstructure:"LogFile"`
MetricsEnabled bool `mapstructure:"MetricsEnabled"` MetricsEnabled bool `mapstructure:"MetricsEnabled"`
MetricsPort string `mapstructure:"MetricsPort"` MetricsPort string `mapstructure:"MetricsPort"`
FileTTLEnabled bool `mapstructure:"FileTTLEnabled"`
FileTTL string `mapstructure:"FileTTL"` FileTTL string `mapstructure:"FileTTL"`
MinFreeBytes string `mapstructure:"MinFreeBytes"` MinFreeBytes string `mapstructure:"MinFreeBytes"`
DeduplicationEnabled bool `mapstructure:"DeduplicationEnabled"` DeduplicationEnabled bool `mapstructure:"DeduplicationEnabled"`
@ -190,20 +191,20 @@ type ThumbnailsConfig struct {
} }
type Config struct { type Config struct {
Server ServerConfig `mapstructure:"server"` Server ServerConfig `mapstructure:"server"`
Timeouts TimeoutConfig `mapstructure:"timeouts"` Timeouts TimeoutConfig `mapstructure:"timeouts"`
Security SecurityConfig `mapstructure:"security"` Security SecurityConfig `mapstructure:"security"`
Versioning VersioningConfig `mapstructure:"versioning"` Versioning VersioningConfig `mapstructure:"versioning"`
Uploads UploadsConfig `mapstructure:"uploads"` Uploads UploadsConfig `mapstructure:"uploads"`
Downloads DownloadsConfig `mapstructure:"downloads"` Downloads DownloadsConfig `mapstructure:"downloads"`
ClamAV ClamAVConfig `mapstructure:"clamav"` ClamAV ClamAVConfig `mapstructure:"clamav"`
Redis RedisConfig `mapstructure:"redis"` Redis RedisConfig `mapstructure:"redis"`
Workers WorkersConfig `mapstructure:"workers"` Workers WorkersConfig `mapstructure:"workers"`
File FileConfig `mapstructure:"file"` File FileConfig `mapstructure:"file"`
ISO ISOConfig `mapstructure:"iso"` ISO ISOConfig `mapstructure:"iso"`
Paste PasteConfig `mapstructure:"paste"` Paste PasteConfig `mapstructure:"paste"`
Deduplication DeduplicationConfig `mapstructure:"deduplication"` Deduplication DeduplicationConfig `mapstructure:"deduplication"`
Thumbnails ThumbnailsConfig `mapstructure:"thumbnails"` Thumbnails ThumbnailsConfig `mapstructure:"thumbnails"`
} }
type UploadTask struct { type UploadTask struct {
@ -222,6 +223,11 @@ type NetworkEvent struct {
Details string Details string
} }
// Add a new field to store the creation date of files
type FileMetadata struct {
CreationDate time.Time
}
var ( var (
conf Config conf Config
versionString string = "v2.2-stable" versionString string = "v2.2-stable"
@ -229,6 +235,7 @@ var (
uploadQueue chan UploadTask uploadQueue chan UploadTask
networkEvents chan NetworkEvent networkEvents chan NetworkEvent
fileInfoCache *cache.Cache fileInfoCache *cache.Cache
fileMetadataCache *cache.Cache
clamClient *clamd.Clamd clamClient *clamd.Clamd
redisClient *redis.Client redisClient *redis.Client
redisConnected bool redisConnected bool
@ -335,7 +342,8 @@ func main() {
} }
fileInfoCache = cache.New(5*time.Minute, 10*time.Minute) fileInfoCache = cache.New(5*time.Minute, 10*time.Minute)
fileMetadataCache = cache.New(5*time.Minute, 10*time.Minute)
if conf.Server.PrecachingEnabled { // Conditionally perform pre-caching if conf.Server.PrecachingEnabled { // Conditionally perform pre-caching
// Starting pre-caching of storage path // Starting pre-caching of storage path
log.Info("Starting pre-caching of storage path...") log.Info("Starting pre-caching of storage path...")
@ -463,6 +471,9 @@ func main() {
log.Fatalf("Server failed: %v", err) log.Fatalf("Server failed: %v", err)
} }
} }
// Start file cleanup in a separate goroutine
go handleFileCleanup(&conf)
} }
func max(a, b int) int { func max(a, b int) int {
@ -561,10 +572,11 @@ func setDefaults() {
viper.SetDefault("server.FileTTL", "8760h") viper.SetDefault("server.FileTTL", "8760h")
viper.SetDefault("server.MinFreeBytes", "100MB") viper.SetDefault("server.MinFreeBytes", "100MB")
viper.SetDefault("server.AutoAdjustWorkers", true) viper.SetDefault("server.AutoAdjustWorkers", true)
viper.SetDefault("server.NetworkEvents", true) // Set default viper.SetDefault("server.NetworkEvents", true) // Set default
viper.SetDefault("server.precaching", true) // Set default for precaching viper.SetDefault("server.precaching", true) // Set default for precaching
viper.SetDefault("server.pidfilepath", "/var/run/hmacfileserver.pid") // Set default for PID file path viper.SetDefault("server.pidfilepath", "/var/run/hmacfileserver.pid") // Set default for PID file path
viper.SetDefault("server.thumbnail", false) // Set default for thumbnail viper.SetDefault("server.thumbnail", false) // Set default for thumbnail
viper.SetDefault("server.FileTTLEnabled", true) // Set default for FileTTLEnabled
_, err := parseTTL("1D") _, err := parseTTL("1D")
if err != nil { if err != nil {
log.Warnf("Failed to parse TTL: %v", err) log.Warnf("Failed to parse TTL: %v", err)
@ -773,7 +785,7 @@ func setupLogging() {
Filename: conf.Server.LogFile, Filename: conf.Server.LogFile,
MaxSize: 100, // megabytes MaxSize: 100, // megabytes
MaxBackups: 3, MaxBackups: 3,
MaxAge: 28, // days MaxAge: 28, // days
Compress: true, // compress old log files Compress: true, // compress old log files
}) })
} else { } else {
@ -807,7 +819,7 @@ func logSystemInfo() {
cpuInfo, _ := cpu.Info() cpuInfo, _ := cpu.Info()
uniqueCPUModels := make(map[string]bool) uniqueCPUModels := make(map[string]bool)
for _, info := range cpuInfo { for _, info := range cpuInfo {
if (!uniqueCPUModels[info.ModelName]) { if !uniqueCPUModels[info.ModelName] {
log.Infof("CPU Model: %s, Cores: %d, Mhz: %f", info.ModelName, info.Cores, info.Mhz) log.Infof("CPU Model: %s, Cores: %d, Mhz: %f", info.ModelName, info.Cores, info.Mhz)
uniqueCPUModels[info.ModelName] = true uniqueCPUModels[info.ModelName] = true
} }
@ -1037,13 +1049,16 @@ func processUpload(task UploadTask) error {
} }
log.Infof("File moved to final destination: %s", absFilename) log.Infof("File moved to final destination: %s", absFilename)
// Store file creation date in metadata cache
fileMetadataCache.Set(absFilename, FileMetadata{CreationDate: time.Now()}, cache.DefaultExpiration)
log.Debugf("Verifying existence immediately after rename: %s", absFilename) log.Debugf("Verifying existence immediately after rename: %s", absFilename)
exists, size := fileExists(absFilename) exists, size := fileExists(absFilename)
log.Debugf("Exists? %v, Size: %d", exists, size) log.Debugf("Exists? %v, Size: %d", exists, size)
// Gajim and Dino do not require a callback or acknowledgement beyond HTTP success. // Gajim and Dino do not require a callback or acknowledgement beyond HTTP success.
callbackURL := r.Header.Get("Callback-URL") callbackURL := r.Header.Get("Callback-URL")
if callbackURL != "" { if (callbackURL != "") {
log.Warnf("Callback-URL provided (%s) but not needed. Ignoring.", callbackURL) log.Warnf("Callback-URL provided (%s) but not needed. Ignoring.", callbackURL)
// We do not block or wait, just ignore. // We do not block or wait, just ignore.
} }
@ -1729,7 +1744,7 @@ func handleNetworkEvents(ctx context.Context) {
log.Info("Stopping network event handler.") log.Info("Stopping network event handler.")
return return
case event, ok := <-networkEvents: case event, ok := <-networkEvents:
if (!ok) { if !ok {
log.Info("Network events channel closed.") log.Info("Network events channel closed.")
return return
} }
@ -1800,7 +1815,7 @@ func initRedis() {
defer cancel() defer cancel()
_, err := redisClient.Ping(ctx).Result() _, err := redisClient.Ping(ctx).Result()
if (err != nil) { if err != nil {
log.Fatalf("Failed to connect to Redis: %v", err) log.Fatalf("Failed to connect to Redis: %v", err)
} }
log.Info("Connected to Redis successfully") log.Info("Connected to Redis successfully")
@ -1823,12 +1838,12 @@ func MonitorRedisHealth(ctx context.Context, client *redis.Client, checkInterval
err := client.Ping(ctx).Err() err := client.Ping(ctx).Err()
mu.Lock() mu.Lock()
if err != nil { if err != nil {
if (redisConnected) { if redisConnected {
log.Errorf("Redis health check failed: %v", err) log.Errorf("Redis health check failed: %v", err)
} }
redisConnected = false redisConnected = false
} else { } else {
if (!redisConnected) { if !redisConnected {
log.Info("Redis reconnected successfully") log.Info("Redis reconnected successfully")
} }
redisConnected = true redisConnected = true
@ -1863,15 +1878,29 @@ func runFileCleaner(ctx context.Context, storeDir string, ttl time.Duration) {
if err != nil { if err != nil {
return err return err
} }
if info.IsDir() { if !info.IsDir() {
return nil // Check if file metadata is cached
} if metadata, found := fileMetadataCache.Get(path); found {
if now.Sub(info.ModTime()) > ttl { if fileMetadata, ok := metadata.(FileMetadata); ok {
err := os.Remove(path) if now.Sub(fileMetadata.CreationDate) > ttl {
if err != nil { err := os.Remove(path)
log.WithError(err).Errorf("Failed to remove expired file: %s", path) if err != nil {
log.WithError(err).Errorf("Failed to remove expired file: %s", path)
} else {
log.Infof("Removed expired file: %s", path)
}
}
}
} else { } else {
log.Infof("Removed expired file: %s", path) // If metadata is not cached, use file modification time
if now.Sub(info.ModTime()) > ttl {
err := os.Remove(path)
if err != nil {
log.WithError(err).Errorf("Failed to remove expired file: %s", path)
} else {
log.Infof("Removed expired file: %s", path)
}
}
} }
} }
return nil return nil
@ -2279,7 +2308,8 @@ func precacheStoragePath(dir string) error {
} }
if !info.IsDir() { if !info.IsDir() {
fileInfoCache.Set(path, info, cache.DefaultExpiration) fileInfoCache.Set(path, info, cache.DefaultExpiration)
log.Debugf("Cached file info for %s", path) fileMetadataCache.Set(path, FileMetadata{CreationDate: info.ModTime()}, cache.DefaultExpiration)
log.Debugf("Cached file info and metadata for %s", path)
} }
return nil return nil
}) })
@ -2327,4 +2357,60 @@ func generateThumbnail(originalPath, thumbnailDir, size string) error {
} }
return nil return nil
} }
func handleFileCleanup(conf *Config) {
if conf.Server.FileTTLEnabled {
ttlDuration, err := parseTTL(conf.Server.FileTTL)
if err != nil {
log.Fatalf("Invalid TTL configuration: %v", err)
}
log.Printf("File TTL is enabled. Files older than %v will be deleted.", ttlDuration)
ticker := time.NewTicker(24 * time.Hour)
defer ticker.Stop()
for range ticker.C {
deleteOldFiles(conf, ttlDuration)
}
} else {
log.Println("File TTL is disabled. No files will be automatically deleted.")
}
}
func deleteOldFiles(conf *Config, ttl time.Duration) {
err := filepath.Walk(conf.Server.StoragePath, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
// Check if file metadata is cached
if metadata, found := fileMetadataCache.Get(path); found {
if fileMetadata, ok := metadata.(FileMetadata); ok {
if time.Since(fileMetadata.CreationDate) > ttl {
err := os.Remove(path)
if err != nil {
log.Printf("Failed to delete %s: %v", path, err)
} else {
log.Printf("Deleted old file: %s", path)
}
}
}
} else {
// If metadata is not cached, use file modification time
if time.Since(info.ModTime()) > ttl {
err := os.Remove(path)
if err != nil {
log.Printf("Failed to delete %s: %v", path, err)
} else {
log.Printf("Deleted old file: %s", path)
}
}
}
}
return nil
})
if err != nil {
log.Printf("Error during file cleanup: %v", err)
}
}