uuxo/hmac-file-server
1
1
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 4 Wiki Activity

sync to github

Browse Source
This commit is contained in:
Alexander Renz 2024-12-09 14:32:33 +01:00
parent 8e5ef77165
commit 474c46668b
2 changed files with 125 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
README.MD
View File

@ -2,6 +2,8 @@
**HMAC File Server** is a secure, scalable, and feature-rich file server with advanced capabilities like HMAC authentication, resumable uploads, chunked uploads, file versioning, and optional ClamAV scanning for file integrity and security. This server is built with extensibility and operational monitoring in mind, including Prometheus metrics support and Redis integration. **HMAC File Server** is a secure, scalable, and feature-rich file server with advanced capabilities like HMAC authentication, resumable uploads, chunked uploads, file versioning, and optional ClamAV scanning for file integrity and security. This server is built with extensibility and operational monitoring in mind, including Prometheus metrics support and Redis integration.
> **Credits:** The **HMAC File Server** is based on the source code of [Thomas Leister's prosody-filer](https://github.com/ThomasLeister/prosody-filer). Many features and design elements have been inspired or derived from this project.
--- ---
## Features ## Features
@ -59,6 +61,10 @@ When `AutoAdjustWorkers` is enabled, the number of workers for HMAC operations a
If `AutoAdjustWorkers = true`, the values for `NumWorkers` and `NumScanWorkers` in the configuration file will be ignored, and the server will automatically adjust these values. If `AutoAdjustWorkers = true`, the values for `NumWorkers` and `NumScanWorkers` in the configuration file will be ignored, and the server will automatically adjust these values.
### Network Events Monitoring
Setting `NetworkEvents = false` in the server configuration disables the logging and tracking of network-related events within the application. This means that functionalities such as monitoring IP changes or recording network activity will be turned off.
--- ---
## Example `config.toml` ## Example `config.toml`
@ -76,6 +82,7 @@ FileTTL = "1y"
DeduplicationEnabled = true DeduplicationEnabled = true
MinFreeBytes = "100MB" MinFreeBytes = "100MB"
AutoAdjustWorkers = true # Enable auto-adjustment for worker scaling AutoAdjustWorkers = true # Enable auto-adjustment for worker scaling
NetworkEvents = false # Disable logging of network events
[timeouts] [timeouts]
ReadTimeout = "480s" ReadTimeout = "480s"
@ -176,4 +183,4 @@ Prometheus metrics include:
- **Versioning**: Store multiple versions of files and keep a maximum of `MaxVersions` versions. - **Versioning**: Store multiple versions of files and keep a maximum of `MaxVersions` versions.
- **ClamAV Integration**: Scan uploaded files for viruses using ClamAV. - **ClamAV Integration**: Scan uploaded files for viruses using ClamAV.
- **Redis Caching**: Utilize Redis for caching file metadata for faster access. - **Redis Caching**: Utilize Redis for caching file metadata for faster access.
- **Auto-Adjust Worker Scaling**: Optimize the number of workers dynamically based on system resources. - **Auto-Adjust Worker Scaling**: Optimize the number of workers dynamically based on system resources.

233
cmd/server/main.go
View File

@ -107,6 +107,7 @@ type ServerConfig struct {
DeduplicationEnabled bool `mapstructure:"DeduplicationEnabled"` DeduplicationEnabled bool `mapstructure:"DeduplicationEnabled"`
MinFreeByte string `mapstructure:"MinFreeByte"` MinFreeByte string `mapstructure:"MinFreeByte"`
AutoAdjustWorkers bool `mapstructure:"AutoAdjustWorkers"` AutoAdjustWorkers bool `mapstructure:"AutoAdjustWorkers"`
NetworkEvents bool `mapstructure:"NetworkEvents"` // Added field
} }
type TimeoutConfig struct { type TimeoutConfig struct {
@ -468,9 +469,9 @@ func setDefaults() {
viper.SetDefault("server.MetricsEnabled", true) viper.SetDefault("server.MetricsEnabled", true)
viper.SetDefault("server.MetricsPort", "9090") viper.SetDefault("server.MetricsPort", "9090")
viper.SetDefault("server.FileTTL", "8760h") viper.SetDefault("server.FileTTL", "8760h")
viper.SetDefault("server.MinFreeBytes", 100<<20) viper.SetDefault("server.MinFreeBytes", "100MB")
viper.SetDefault("server.AutoAdjustWorkers", true) viper.SetDefault("server.AutoAdjustWorkers", true)
viper.SetDefault("server.NetworkEvents", true) // Set default
_, err := parseTTL("1D") _, err := parseTTL("1D")
if err != nil { if err != nil {
log.Warnf("Failed to parse TTL: %v", err) log.Warnf("Failed to parse TTL: %v", err)
@ -889,34 +890,34 @@ func shouldScanFile(filename string) bool {
} }
func uploadWorker(ctx context.Context, workerID int) { func uploadWorker(ctx context.Context, workerID int) {
log.Infof("Upload worker %d started.", workerID) log.Infof("Upload worker %d started.", workerID)
defer log.Infof("Upload worker %d stopped.", workerID) defer log.Infof("Upload worker %d stopped.", workerID)
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
return return
case task, ok := <-uploadQueue: case task, ok := <-uploadQueue:
if !ok { if !ok {
return return
} }
log.Infof("Worker %d processing file: %s", workerID, task.AbsFilename) log.Infof("Worker %d processing file: %s", workerID, task.AbsFilename)
err := processUpload(task) err := processUpload(task)
if err != nil { if err != nil {
log.Errorf("Worker %d failed to process file %s: %v", workerID, task.AbsFilename, err) log.Errorf("Worker %d failed to process file %s: %v", workerID, task.AbsFilename, err)
} else { } else {
log.Infof("Worker %d successfully processed file: %s", workerID, task.AbsFilename) log.Infof("Worker %d successfully processed file: %s", workerID, task.AbsFilename)
} }
task.Result <- err task.Result <- err
} }
} }
} }
func initializeUploadWorkerPool(ctx context.Context, w *WorkersConfig) { func initializeUploadWorkerPool(ctx context.Context, w *WorkersConfig) {
for i := 0; i < w.NumWorkers; i++ { for i := 0; i < w.NumWorkers; i++ {
go uploadWorker(ctx, i) go uploadWorker(ctx, i)
log.Infof("Upload worker %d started.", i) log.Infof("Upload worker %d started.", i)
} }
log.Infof("Initialized %d upload workers", w.NumWorkers) log.Infof("Initialized %d upload workers", w.NumWorkers)
} }
func scanWorker(ctx context.Context, workerID int) { func scanWorker(ctx context.Context, workerID int) {
@ -1070,107 +1071,107 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
// handleUpload handles PUT requests for file uploads // handleUpload handles PUT requests for file uploads
func handleUpload(w http.ResponseWriter, r *http.Request, absFilename, fileStorePath string, a url.Values) { func handleUpload(w http.ResponseWriter, r *http.Request, absFilename, fileStorePath string, a url.Values) {
log.Infof("Using storage path: %s", conf.Server.StoragePath) log.Infof("Using storage path: %s", conf.Server.StoragePath)
// HMAC validation // HMAC validation
var protocolVersion string var protocolVersion string
if a.Get("v2") != "" { if a.Get("v2") != "" {
protocolVersion = "v2" protocolVersion = "v2"
} else if a.Get("token") != "" { } else if a.Get("token") != "" {
protocolVersion = "token" protocolVersion = "token"
} else if a.Get("v") != "" { } else if a.Get("v") != "" {
protocolVersion = "v" protocolVersion = "v"
} else { } else {
log.Warn("No HMAC attached to URL.") log.Warn("No HMAC attached to URL.")
http.Error(w, "No HMAC attached to URL. Expecting 'v', 'v2', or 'token' parameter as MAC", http.StatusForbidden) http.Error(w, "No HMAC attached to URL. Expecting 'v', 'v2', or 'token' parameter as MAC", http.StatusForbidden)
return return
} }
mac := hmac.New(sha256.New, []byte(conf.Security.Secret)) mac := hmac.New(sha256.New, []byte(conf.Security.Secret))
if protocolVersion == "v" { if protocolVersion == "v" {
mac.Write([]byte(fileStorePath + "\x20" + strconv.FormatInt(r.ContentLength, 10))) mac.Write([]byte(fileStorePath + "\x20" + strconv.FormatInt(r.ContentLength, 10)))
} else { } else {
contentType := mime.TypeByExtension(filepath.Ext(fileStorePath)) contentType := mime.TypeByExtension(filepath.Ext(fileStorePath))
if contentType == "" { if contentType == "" {
contentType = "application/octet-stream" contentType = "application/octet-stream"
} }
mac.Write([]byte(fileStorePath + "\x00" + strconv.FormatInt(r.ContentLength, 10) + "\x00" + contentType)) mac.Write([]byte(fileStorePath + "\x00" + strconv.FormatInt(r.ContentLength, 10) + "\x00" + contentType))
} }
calculatedMAC := mac.Sum(nil) calculatedMAC := mac.Sum(nil)
providedMACHex := a.Get(protocolVersion) providedMACHex := a.Get(protocolVersion)
providedMAC, err := hex.DecodeString(providedMACHex) providedMAC, err := hex.DecodeString(providedMACHex)
if err != nil { if err != nil {
log.Warn("Invalid MAC encoding") log.Warn("Invalid MAC encoding")
http.Error(w, "Invalid MAC encoding", http.StatusForbidden) http.Error(w, "Invalid MAC encoding", http.StatusForbidden)
return return
} }
if !hmac.Equal(calculatedMAC, providedMAC) { if !hmac.Equal(calculatedMAC, providedMAC) {
log.Warn("Invalid MAC") log.Warn("Invalid MAC")
http.Error(w, "Invalid MAC", http.StatusForbidden) http.Error(w, "Invalid MAC", http.StatusForbidden)
return return
} }
if !isExtensionAllowed(fileStorePath) { if !isExtensionAllowed(fileStorePath) {
log.Warn("Invalid file extension") log.Warn("Invalid file extension")
http.Error(w, "Invalid file extension", http.StatusBadRequest) http.Error(w, "Invalid file extension", http.StatusBadRequest)
uploadErrorsTotal.Inc() uploadErrorsTotal.Inc()
return return
} }
minFreeBytes, err := parseSize(conf.Server.MinFreeBytes) minFreeBytes, err := parseSize(conf.Server.MinFreeBytes)
if err != nil { if err != nil {
log.Fatalf("Invalid MinFreeBytes: %v", err) log.Fatalf("Invalid MinFreeBytes: %v", err)
} }
err = checkStorageSpace(conf.Server.StoragePath, minFreeBytes) err = checkStorageSpace(conf.Server.StoragePath, minFreeBytes)
if err != nil { if err != nil {
log.Warn("Not enough free space") log.Warn("Not enough free space")
http.Error(w, "Not enough free space", http.StatusInsufficientStorage) http.Error(w, "Not enough free space", http.StatusInsufficientStorage)
uploadErrorsTotal.Inc() uploadErrorsTotal.Inc()
return return
} }
// Check for Callback-URL header // Check for Callback-URL header
callbackURL := r.Header.Get("Callback-URL") callbackURL := r.Header.Get("Callback-URL")
if callbackURL != "" { if callbackURL != "" {
log.Warnf("Callback-URL provided (%s) but not needed. Ignoring.", callbackURL) log.Warnf("Callback-URL provided (%s) but not needed. Ignoring.", callbackURL)
// Do not perform any callback actions // Do not perform any callback actions
} }
// Enqueue the upload task // Enqueue the upload task
result := make(chan error) result := make(chan error)
task := UploadTask{ task := UploadTask{
AbsFilename: absFilename, AbsFilename: absFilename,
Request: r, Request: r,
Result: result, Result: result,
} }
log.Debug("Attempting to enqueue upload task") log.Debug("Attempting to enqueue upload task")
select { select {
case uploadQueue <- task: case uploadQueue <- task:
log.Debug("Upload task enqueued successfully") log.Debug("Upload task enqueued successfully")
default: default:
log.Warn("Upload queue is full.") log.Warn("Upload queue is full.")
http.Error(w, "Server busy. Try again later.", http.StatusServiceUnavailable) http.Error(w, "Server busy. Try again later.", http.StatusServiceUnavailable)
uploadErrorsTotal.Inc() uploadErrorsTotal.Inc()
return return
} }
log.Debug("Waiting for upload task to complete") log.Debug("Waiting for upload task to complete")
err = <-result err = <-result
if err != nil { if err != nil {
log.Errorf("Upload failed: %v", err) log.Errorf("Upload failed: %v", err)
http.Error(w, fmt.Sprintf("Upload failed: %v", err), http.StatusInternalServerError) http.Error(w, fmt.Sprintf("Upload failed: %v", err), http.StatusInternalServerError)
return return
} }
log.Debug("Upload task completed successfully") log.Debug("Upload task completed successfully")
// Respond with 201 Created on successful upload // Respond with 201 Created on successful upload
w.WriteHeader(http.StatusCreated) w.WriteHeader(http.StatusCreated)
log.Infof("Responded with 201 Created for file: %s", absFilename) log.Infof("Responded with 201 Created for file: %s", absFilename)
} }
func handleDownload(w http.ResponseWriter, r *http.Request, absFilename, fileStorePath string) { func handleDownload(w http.ResponseWriter, r *http.Request, absFilename, fileStorePath string) {
@ -1546,7 +1547,7 @@ func MonitorRedisHealth(ctx context.Context, client *redis.Client, checkInterval
} }
redisConnected = false redisConnected = false
} else { } else {
if (!redisConnected) { if !redisConnected {
log.Info("Redis reconnected successfully") log.Info("Redis reconnected successfully")
} }
redisConnected = true redisConnected = true