uuxo/hmac-file-server
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

2.2-stable

Browse Source
This commit is contained in:
Alexander Renz 2024-12-24 15:41:13 +01:00
parent 95cbd11ba3
commit e542a7e948
4 changed files with 359 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
README.MD
View File

@ -75,106 +75,11 @@ Set `thumbnail` to true in the `[server]` section of `config.toml` to enable ima
### Deduplication
Set `enabled` to true in the `[deduplication]` section of `config.toml` to enable file deduplication. Specify the `storagepath` where deduplicated files will be stored.
---
## Deduplication
Set `enabled` to true in the `[deduplication]` section of `config.toml` to enable file deduplication. Specify the `directory` where deduplicated files will be stored.
### Example `config.toml`
```toml
[deduplication]
enabled = true
directory = "/mnt/hmac-storage/deduplication/"
```
## Thumbnails
Set `enabled` to true in the `[thumbnails]` section of `config.toml` to enable thumbnail creation. Specify the `directory` where thumbnails will be stored and the `size` of the thumbnails.
### Example `config.toml`
```toml
[thumbnails]
enabled = true
directory = "/mnt/hmac-storage/thumbnails/"
size = "200x200"
```
## Example `config.toml`
```toml
[server]
ListenPort = "8080"
UnixSocket = false
StoragePath = "./uploads"
LogLevel = "info"
LogFile = ""
MetricsEnabled = true
MetricsPort = "9090"
FileTTL = "1y"
DeduplicationEnabled = true
MinFreeBytes = "100MB"
AutoAdjustWorkers = true # Enable auto-adjustment for worker scaling
NetworkEvents = false # Disable logging and tracking of network-related events
PIDFilePath = "./hmac_file_server.pid" # Path to PID file
Precaching = true # Enable pre-caching of storage paths
ThumbnailEnabled = false # Whether to create thumbnails for uploaded images
[timeouts]
ReadTimeout = "480s"
WriteTimeout = "480s"
IdleTimeout = "65s" # nginx/apache2 keep-alive 60s
[security]
Secret = "changeme"
[versioning]
EnableVersioning = false
MaxVersions = 1
[uploads]
ResumableUploadsEnabled = true
ChunkedUploadsEnabled = true
ChunkSize = "64MB"
AllowedExtensions = [".txt", ".pdf", ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".svg", ".webp", ".wav", ".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".mpeg", ".mpg", ".m4v", ".3gp", ".3g2", ".mp3", ".ogg"]
[downloads]
ResumableDownloadsEnabled = true
ChunkedDownloadsEnabled = true
ChunkSize = "64MB"
[clamav]
ClamAVEnabled = false
ClamAVSocket = "/var/run/clamav/clamd.ctl"
NumScanWorkers = 2
ScanFileExtensions = [".exe", ".dll", ".bin", ".com", ".bat", ".sh", ".php", ".js"]
[redis]
RedisEnabled = false
RedisAddr = "localhost:6379"
RedisPassword = ""
RedisDBIndex = 0
RedisHealthCheckInterval = "120s"
[workers]
NumWorkers = 4
UploadQueueSize = 5000
[file]
FileRevision = 1 # Revision number for file handling
[deduplication]
enabled = true
storagepath = "/mnt/nfs_vol01/hmac-file-server/deduplication/"
```
---
## HMAC File Server - Version 2.2 Stable
## Example `config.toml`
Below is an example configuration file (config.toml) you can use as a reference (with sensitive data removed):
@ -190,6 +95,7 @@ metricsport = "9090"
deduplicationenabled = true
minfreebytes = "5GB"
filettl = "2Y"
filettlenabled = true # Enable or disable file TTL
autoadjustworkers = true
networkevents = false
pidfilepath = "./hmac-file-server.pid"
@ -216,7 +122,7 @@ writetimeout = "3600s"
idletimeout = "3600s"
[security]
secret = "stellar-wisdom-orbit-echo"
secret = "changeme"
[versioning]
enableversioning = false
@ -328,6 +234,18 @@ Prometheus metrics include:
---
### Overview of other Projects (xep0363)
| Feature/Project | HMAC FS | mod_http_upload_ext | xmpp-http-upload (horazont) | Prosody Filer | ngx_http_upload | xmpp-http-upload (nyovaya) |
|-----------------------------|---------|----------------------|-----------------------------|---------------|----------------|----------------------------|
| **Lang** | Go | PHP | Python | Go | C (Nginx) | Python |
| **Env** | Standalone | Prosody module | Standalone | Standalone | Nginx | Standalone |
| **XMPP** | No | Yes | Yes | Yes | No | Yes |
| **Ext. Storage** | Yes | No | Possible via plugins | No | No | Yes |
| **Auth / Security** | HMAC | Token-based | Token-based | None | Basic / None | Token-based |
---
## Build & Run
1. Clone the repository.
2. Build the server:

204
RELEASE-NOTES.MD Normal file
View File

@ -0,0 +1,204 @@
# HMAC File Server
**HMAC File Server** is a secure, scalable, and feature-rich file server with advanced capabilities like HMAC authentication, resumable uploads, chunked uploads, file versioning, and optional ClamAV scanning for file integrity and security. This server is built with extensibility and operational monitoring in mind, including Prometheus metrics support and Redis integration.
> **Credits:** The **HMAC File Server** is based on the source code of [Thomas Leister's prosody-filer](https://github.com/ThomasLeister/prosody-filer). Many features and design elements have been inspired or derived from this project.
---
## Features
- **HMAC Authentication:** Secure file uploads and downloads with HMAC tokens.
- **File Versioning:** Enable versioning for uploaded files with configurable retention.
- **Chunked and Resumable Uploads:** Handle large files efficiently with support for resumable and chunked uploads.
- **ClamAV Scanning:** Optional virus scanning for uploaded files.
- **Prometheus Metrics:** Monitor system and application-level metrics.
- **Redis Integration:** Use Redis for caching or storing application states.
- **File Expiration:** Automatically delete files after a specified TTL.
- **Graceful Shutdown:** Handles signals and ensures proper cleanup.
- **Deduplication:** Remove duplicate files based on hashing for storage efficiency.
- **Auto-Adjust Worker Scaling:** Dynamically optimize HMAC and ClamAV workers based on system resources when enabled.
---
## Repository
- **Primary Repository**: [GitHub Repository](https://github.com/PlusOne/hmac-file-server)
- **Alternative Repository**: [uuxo.net Git Repository](https://git.uuxo.net/uuxo/hmac-file-server)
---
## Installation
### Prerequisites
- Go 1.20+
- Redis (optional, if Redis integration is enabled)
- ClamAV (optional, if file scanning is enabled)
### Clone and Build
```bash
# Clone from the primary repository
git clone https://github.com/PlusOne/hmac-file-server.git
# OR clone from the alternative repository
git clone https://git.uuxo.net/uuxo/hmac-file-server.git
cd hmac-file-server
go build -o hmac-file-server main.go
```
---
## Configuration
The server configuration is managed through a `config.toml` file. Below are the supported configuration options:
### Auto-Adjust Feature
When `AutoAdjustWorkers` is enabled, the number of workers for HMAC operations and ClamAV scans is dynamically determined based on system resources. This ensures efficient resource utilization.
If `AutoAdjustWorkers = true`, the values for `NumWorkers` and `NumScanWorkers` in the configuration file will be ignored, and the server will automatically adjust these values.
### Network Events Monitoring
Setting `NetworkEvents = false` in the server configuration disables the logging and tracking of network-related events within the application. This means that functionalities such as monitoring IP changes or recording network activity will be turned off.
### Precaching
The `precaching` feature allows the server to pre-cache storage paths for faster access. This can improve performance by reducing the time needed to access frequently used storage paths.
### Added thumbnail support
- New configuration option `thumbnail` in `[server]` to enable or disable generating image thumbnails
---
## New Features
### Deduplication Support
- **Description:** Added support for file deduplication to save storage space by storing a single copy of identical files.
- **Configuration:**
```toml
[deduplication]
enabled = true
directory = "/mnt/hmac-storage/deduplication/"
```
### Thumbnail Support
- **Description:** Added support for thumbnail creation to generate smaller versions of uploaded images.
- **Configuration:**
```toml
[thumbnails]
enabled = true
directory = "/mnt/hmac-storage/thumbnails/"
size = "200x200"
```
---
## Example `config.toml`
```toml
[server]
ListenPort = "8080"
UnixSocket = false
StoragePath = "./uploads"
LogLevel = "info"
LogFile = ""
MetricsEnabled = true
MetricsPort = "9090"
FileTTL = "1y"
FileTTLEnabled = true # Enable or disable file TTL
DeduplicationEnabled = true
MinFreeBytes = "100MB"
AutoAdjustWorkers = true # Enable auto-adjustment for worker scaling
NetworkEvents = false # Disable logging and tracking of network-related events
PIDFilePath = "./hmac_file_server.pid" # Path to PID file
Precaching = true # Enable pre-caching of storage paths
ThumbnailEnabled = false # Whether to create thumbnails for uploaded images
[timeouts]
ReadTimeout = "480s"
WriteTimeout = "480s"
IdleTimeout = "65s" # nginx/apache2 keep-alive 60s
[security]
Secret = "changeme"
[versioning]
EnableVersioning = false
MaxVersions = 1
[uploads]
ResumableUploadsEnabled = true
ChunkedUploadsEnabled = true
ChunkSize = "64MB"
AllowedExtensions = [".txt", ".pdf", ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".svg", ".webp", ".wav", ".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".mpeg", ".mpg", ".m4v", ".3gp", ".3g2", ".mp3", ".ogg"]
[downloads]
ResumableDownloadsEnabled = true
ChunkedDownloadsEnabled = true
ChunkSize = "64MB"
[clamav]
ClamAVEnabled = false
ClamAVSocket = "/var/run/clamav/clamd.ctl"
NumScanWorkers = 2
ScanFileExtensions = [".exe", ".dll", ".bin", ".com", ".bat", ".sh", ".php", ".js"]
[redis]
RedisEnabled = false
RedisAddr = "localhost:6379"
RedisPassword = ""
RedisDBIndex = 0
RedisHealthCheckInterval = "120s"
[workers]
NumWorkers = 4
UploadQueueSize = 5000
[file]
FileRevision = 1 # Revision number for file handling
```
---
## Running the Server
### Basic Usage
Run the server with a configuration file:
```bash
./hmac-file-server -config ./config.toml
```
---
### Metrics Server
If `MetricsEnabled` is set to `true`, the Prometheus metrics server will be available on the port specified in `MetricsPort` (default: `9090`).
---
## Testing
To run the server locally for development:
```bash
go run main.go -config ./config.toml
```
Use tools like **cURL** or **Postman** to test file uploads and downloads.
### Example File Upload with HMAC Token
```bash
curl -X PUT -H "Authorization: Bearer <HMAC-TOKEN>" -F "file=@example.txt" http://localhost:8080/uploads/example.txt
```
Replace `<HMAC-TOKEN>` with a valid HMAC signature

28
cmd/server/config.toml
View File

@ -1,26 +1,34 @@
[server]
listenport = "8080"
unixsocket = false
storagepath = "./uploads"
loglevel = "debug"Q
logfile = "./hmac-file-server.log"
storagepath = "/"
loglevel = "debug"
logfile = "./tmp/hmac-file-server.log"
metricsenabled = true
metricsport = "8081"
filettl = "180d"
minfreebytes = "2GB"
deduplicationenabled = true
autoadjustworkers = true
networkevents = false
temppath = "/tmp/hmac"
temppath = "./tmp"
loggingjson = false
pidfilepath = "./hmac_file_server.pid"
pidfilepath = "hmac_file_server.pid"
cleanuponexit = true
precaching = true
precaching = false
[deduplication]
enabled = false
directory = "./deduplication"
[thumbnails]
enabled = false
directory = "./thumbnails"
size = "200x200"
[iso]
enabled = false
size = "1TB"
mountpoint = "/mnt/nfs_vol01/hmac-file-server/iso/"
mountpoint = "./iso"
charset = "utf-8"
[timeouts]
@ -42,8 +50,8 @@ chunksize = "64MB"
allowedextensions = [".txt", ".pdf", ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".svg", ".webp", ".wav", ".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".mpeg", ".mpg", ".m4v", ".3gp", ".3g2", ".mp3", ".ogg", ".zip", ".rar"]
[downloads]
resumabledownloadsenabled = true
chunkeddownloadsenabled = true
resumabledownloadsenabled = false
chunkeddownloadsenabled = false
chunksize = "64MB"
[clamav]

158
cmd/server/main.go
View File

@ -26,6 +26,7 @@ import (
"syscall"
"time"
"github.com/disintegration/imaging"
"github.com/dutchcoders/go-clamd" // ClamAV integration
"github.com/go-redis/redis/v8" // Redis integration
"github.com/patrickmn/go-cache"
@ -38,7 +39,6 @@ import (
"github.com/sirupsen/logrus"
"github.com/spf13/viper"
"gopkg.in/natefinch/lumberjack.v2"
"github.com/disintegration/imaging"
)
// parseSize converts a human-readable size string to bytes
@ -104,6 +104,7 @@ type ServerConfig struct {
LogFile string `mapstructure:"LogFile"`
MetricsEnabled bool `mapstructure:"MetricsEnabled"`
MetricsPort string `mapstructure:"MetricsPort"`
FileTTLEnabled bool `mapstructure:"FileTTLEnabled"`
FileTTL string `mapstructure:"FileTTL"`
MinFreeBytes string `mapstructure:"MinFreeBytes"`
DeduplicationEnabled bool `mapstructure:"DeduplicationEnabled"`
@ -190,20 +191,20 @@ type ThumbnailsConfig struct {
}
type Config struct {
Server ServerConfig `mapstructure:"server"`
Timeouts TimeoutConfig `mapstructure:"timeouts"`
Security SecurityConfig `mapstructure:"security"`
Versioning VersioningConfig `mapstructure:"versioning"`
Uploads UploadsConfig `mapstructure:"uploads"`
Downloads DownloadsConfig `mapstructure:"downloads"`
ClamAV ClamAVConfig `mapstructure:"clamav"`
Redis RedisConfig `mapstructure:"redis"`
Workers WorkersConfig `mapstructure:"workers"`
File FileConfig `mapstructure:"file"`
ISO ISOConfig `mapstructure:"iso"`
Paste PasteConfig `mapstructure:"paste"`
Deduplication DeduplicationConfig `mapstructure:"deduplication"`
Thumbnails ThumbnailsConfig `mapstructure:"thumbnails"`
Server ServerConfig `mapstructure:"server"`
Timeouts TimeoutConfig `mapstructure:"timeouts"`
Security SecurityConfig `mapstructure:"security"`
Versioning VersioningConfig `mapstructure:"versioning"`
Uploads UploadsConfig `mapstructure:"uploads"`
Downloads DownloadsConfig `mapstructure:"downloads"`
ClamAV ClamAVConfig `mapstructure:"clamav"`
Redis RedisConfig `mapstructure:"redis"`
Workers WorkersConfig `mapstructure:"workers"`
File FileConfig `mapstructure:"file"`
ISO ISOConfig `mapstructure:"iso"`
Paste PasteConfig `mapstructure:"paste"`
Deduplication DeduplicationConfig `mapstructure:"deduplication"`
Thumbnails ThumbnailsConfig `mapstructure:"thumbnails"`
}
type UploadTask struct {
@ -222,6 +223,11 @@ type NetworkEvent struct {
Details string
}
// Add a new field to store the creation date of files
type FileMetadata struct {
CreationDate time.Time
}
var (
conf Config
versionString string = "v2.2-stable"
@ -229,6 +235,7 @@ var (
uploadQueue chan UploadTask
networkEvents chan NetworkEvent
fileInfoCache *cache.Cache
fileMetadataCache *cache.Cache
clamClient *clamd.Clamd
redisClient *redis.Client
redisConnected bool
@ -335,7 +342,8 @@ func main() {
}
fileInfoCache = cache.New(5*time.Minute, 10*time.Minute)
fileMetadataCache = cache.New(5*time.Minute, 10*time.Minute)
if conf.Server.PrecachingEnabled { // Conditionally perform pre-caching
// Starting pre-caching of storage path
log.Info("Starting pre-caching of storage path...")
@ -463,6 +471,9 @@ func main() {
log.Fatalf("Server failed: %v", err)
}
}
// Start file cleanup in a separate goroutine
go handleFileCleanup(&conf)
}
func max(a, b int) int {
@ -561,10 +572,11 @@ func setDefaults() {
viper.SetDefault("server.FileTTL", "8760h")
viper.SetDefault("server.MinFreeBytes", "100MB")
viper.SetDefault("server.AutoAdjustWorkers", true)
viper.SetDefault("server.NetworkEvents", true) // Set default
viper.SetDefault("server.precaching", true) // Set default for precaching
viper.SetDefault("server.NetworkEvents", true) // Set default
viper.SetDefault("server.precaching", true) // Set default for precaching
viper.SetDefault("server.pidfilepath", "/var/run/hmacfileserver.pid") // Set default for PID file path
viper.SetDefault("server.thumbnail", false) // Set default for thumbnail
viper.SetDefault("server.thumbnail", false) // Set default for thumbnail
viper.SetDefault("server.FileTTLEnabled", true) // Set default for FileTTLEnabled
_, err := parseTTL("1D")
if err != nil {
log.Warnf("Failed to parse TTL: %v", err)
@ -773,7 +785,7 @@ func setupLogging() {
Filename: conf.Server.LogFile,
MaxSize: 100, // megabytes
MaxBackups: 3,
MaxAge: 28, // days
MaxAge: 28, // days
Compress: true, // compress old log files
})
} else {
@ -807,7 +819,7 @@ func logSystemInfo() {
cpuInfo, _ := cpu.Info()
uniqueCPUModels := make(map[string]bool)
for _, info := range cpuInfo {
if (!uniqueCPUModels[info.ModelName]) {
if !uniqueCPUModels[info.ModelName] {
log.Infof("CPU Model: %s, Cores: %d, Mhz: %f", info.ModelName, info.Cores, info.Mhz)
uniqueCPUModels[info.ModelName] = true
}
@ -1037,13 +1049,16 @@ func processUpload(task UploadTask) error {
}
log.Infof("File moved to final destination: %s", absFilename)
// Store file creation date in metadata cache
fileMetadataCache.Set(absFilename, FileMetadata{CreationDate: time.Now()}, cache.DefaultExpiration)
log.Debugf("Verifying existence immediately after rename: %s", absFilename)
exists, size := fileExists(absFilename)
log.Debugf("Exists? %v, Size: %d", exists, size)
// Gajim and Dino do not require a callback or acknowledgement beyond HTTP success.
callbackURL := r.Header.Get("Callback-URL")
if callbackURL != "" {
if (callbackURL != "") {
log.Warnf("Callback-URL provided (%s) but not needed. Ignoring.", callbackURL)
// We do not block or wait, just ignore.
}
@ -1729,7 +1744,7 @@ func handleNetworkEvents(ctx context.Context) {
log.Info("Stopping network event handler.")
return
case event, ok := <-networkEvents:
if (!ok) {
if !ok {
log.Info("Network events channel closed.")
return
}
@ -1800,7 +1815,7 @@ func initRedis() {
defer cancel()
_, err := redisClient.Ping(ctx).Result()
if (err != nil) {
if err != nil {
log.Fatalf("Failed to connect to Redis: %v", err)
}
log.Info("Connected to Redis successfully")
@ -1823,12 +1838,12 @@ func MonitorRedisHealth(ctx context.Context, client *redis.Client, checkInterval
err := client.Ping(ctx).Err()
mu.Lock()
if err != nil {
if (redisConnected) {
if redisConnected {
log.Errorf("Redis health check failed: %v", err)
}
redisConnected = false
} else {
if (!redisConnected) {
if !redisConnected {
log.Info("Redis reconnected successfully")
}
redisConnected = true
@ -1863,15 +1878,29 @@ func runFileCleaner(ctx context.Context, storeDir string, ttl time.Duration) {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
if now.Sub(info.ModTime()) > ttl {
err := os.Remove(path)
if err != nil {
log.WithError(err).Errorf("Failed to remove expired file: %s", path)
if !info.IsDir() {
// Check if file metadata is cached
if metadata, found := fileMetadataCache.Get(path); found {
if fileMetadata, ok := metadata.(FileMetadata); ok {
if now.Sub(fileMetadata.CreationDate) > ttl {
err := os.Remove(path)
if err != nil {
log.WithError(err).Errorf("Failed to remove expired file: %s", path)
} else {
log.Infof("Removed expired file: %s", path)
}
}
}
} else {
log.Infof("Removed expired file: %s", path)
// If metadata is not cached, use file modification time
if now.Sub(info.ModTime()) > ttl {
err := os.Remove(path)
if err != nil {
log.WithError(err).Errorf("Failed to remove expired file: %s", path)
} else {
log.Infof("Removed expired file: %s", path)
}
}
}
}
return nil
@ -2279,7 +2308,8 @@ func precacheStoragePath(dir string) error {
}
if !info.IsDir() {
fileInfoCache.Set(path, info, cache.DefaultExpiration)
log.Debugf("Cached file info for %s", path)
fileMetadataCache.Set(path, FileMetadata{CreationDate: info.ModTime()}, cache.DefaultExpiration)
log.Debugf("Cached file info and metadata for %s", path)
}
return nil
})
@ -2327,4 +2357,60 @@ func generateThumbnail(originalPath, thumbnailDir, size string) error {
}
return nil
}
}
func handleFileCleanup(conf *Config) {
if conf.Server.FileTTLEnabled {
ttlDuration, err := parseTTL(conf.Server.FileTTL)
if err != nil {
log.Fatalf("Invalid TTL configuration: %v", err)
}
log.Printf("File TTL is enabled. Files older than %v will be deleted.", ttlDuration)
ticker := time.NewTicker(24 * time.Hour)
defer ticker.Stop()
for range ticker.C {
deleteOldFiles(conf, ttlDuration)
}
} else {
log.Println("File TTL is disabled. No files will be automatically deleted.")
}
}
func deleteOldFiles(conf *Config, ttl time.Duration) {
err := filepath.Walk(conf.Server.StoragePath, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
// Check if file metadata is cached
if metadata, found := fileMetadataCache.Get(path); found {
if fileMetadata, ok := metadata.(FileMetadata); ok {
if time.Since(fileMetadata.CreationDate) > ttl {
err := os.Remove(path)
if err != nil {
log.Printf("Failed to delete %s: %v", path, err)
} else {
log.Printf("Deleted old file: %s", path)
}
}
}
} else {
// If metadata is not cached, use file modification time
if time.Since(info.ModTime()) > ttl {
err := os.Remove(path)
if err != nil {
log.Printf("Failed to delete %s: %v", path, err)
} else {
log.Printf("Deleted old file: %s", path)
}
}
}
}
return nil
})
if err != nil {
log.Printf("Error during file cleanup: %v", err)
}
}