uuxo/hmac-file-server
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

fix: 2.6-stable

Browse Source
This commit is contained in:
Alexander Renz 2025-01-26 09:20:56 +01:00
parent 0cda54c97e
commit b14f046beb
5 changed files with 204 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
RELEASE-NOTES.MD
View File

@ -1,7 +1,7 @@
# Release Notes for HMAC File Server 2.5-Stable
# Release Notes for HMAC File Server 2.6-Stable
## Summary
Version 2.5-Stable focuses on improving the overall stability and performance of the HMAC File Server. Significant changes have been made to prioritize reliability and scalability for production environments.
Version 2.6-Stable focuses on improving the overall stability and performance of the HMAC File Server. Significant changes have been made to prioritize reliability and scalability for production environments.
## Key Changes
@ -12,6 +12,9 @@ Version 2.5-Stable focuses on improving the overall stability and performance of
- **ISO-Based Storage Support**: Introduced support for ISO-based storage to accommodate specialized use cases.
- **Enhanced ClamAV Integration**: Improved ClamAV scanning with concurrent workers, providing better performance for large-scale deployments.
- **Timeout Configuration**: Added granular timeout settings for read, write, and idle connections, improving connection management.
- **FileNaming Configuration**: Added support for a "None" option in the `FileNaming` configuration. When set to "None", the filename remains unchanged.
- **Example Configuration Generation**: If no configuration file is found, the server will output an example configuration for the user to copy and paste.
- **Prometheus Metrics**: Enhanced Prometheus metrics for better monitoring and performance tracking. New metrics include upload and download durations, error counts, memory and CPU usage, and more.
### Improvements
- **Worker Management**: Auto-scaling worker threads based on system load for optimal performance.
@ -25,6 +28,11 @@ Version 2.5-Stable focuses on improving the overall stability and performance of
1. **Thumbnail Settings**: Remove `[thumbnails]` configuration blocks from your `config.toml` file to avoid errors.
2. **Updated Configuration**: Review new timeout settings in `[timeouts]` and adjust as needed.
3. **ISO Integration**: Configure the new `[iso]` block for environments utilizing ISO-based storage.
4. **FileNaming Configuration**: Update the `FileNaming` setting in `[server]` to use the new "None" option if you want filenames to remain unchanged.
[server]
# FileNaming options: "HMAC", "None"
FileNaming = "HMAC"
## Recommendations
- **Security**: Ensure that the HMAC secret key in `config.toml` is updated to a strong, unique value.

2
cmd/monitor/monitor.go
View File

@ -52,7 +52,7 @@ func init() {
}
if err != nil {
log.Fatalf("Error loading config file: %v", err)
log.Fatalf("Error loading config file: %v\nPlease create a config.toml in one of the following locations:\n%v", err, configPaths)
}
// Metricsport auslesen

210
cmd/server/main.go
View File

@ -52,17 +52,16 @@ func parseSize(sizeStr string) (int64, error) {
valueStr := sizeStr[:len(sizeStr)-2]
value, err := strconv.Atoi(valueStr)
if err != nil {
log.WithError(err).Errorf("Failed to parse size from input: %s", sizeStr)
return 0, err
return 0, fmt.Errorf("invalid size value: %v", err)
}
switch unit {
case "KB":
return int64(value) * 1 << 10, nil
return int64(value) * 1024, nil
case "MB":
return int64(value) * 1 << 20, nil
return int64(value) * 1024 * 1024, nil
case "GB":
return int64(value) * 1 << 30, nil
return int64(value) * 1024 * 1024 * 1024, nil
default:
return 0, fmt.Errorf("unknown size unit: %s", unit)
}
@ -72,7 +71,7 @@ func parseSize(sizeStr string) (int64, error) {
func parseTTL(ttlStr string) (time.Duration, error) {
ttlStr = strings.ToLower(strings.TrimSpace(ttlStr))
if ttlStr == "" {
return 0, fmt.Errorf("empty TTL string")
return 0, fmt.Errorf("TTL string cannot be empty")
}
var valueStr string
var unit rune
@ -89,16 +88,20 @@ func parseTTL(ttlStr string) (time.Duration, error) {
return 0, fmt.Errorf("invalid TTL value: %v", err)
}
switch unit {
case 'h': // hours
case 's':
return time.Duration(val) * time.Second, nil
case 'm':
return time.Duration(val) * time.Minute, nil
case 'h':
return time.Duration(val) * time.Hour, nil
case 'd': // days
return time.Duration(val*24) * time.Hour, nil
case 'm': // months (approx. 30 days)
return time.Duration(val*24*30) * time.Hour, nil
case 'y': // years (approx. 365 days)
return time.Duration(val*24*365) * time.Hour, nil
default: // fallback to Go's standard parsing
return time.ParseDuration(ttlStr)
case 'd':
return time.Duration(val) * 24 * time.Hour, nil
case 'w':
return time.Duration(val) * 7 * 24 * time.Hour, nil
case 'y':
return time.Duration(val) * 365 * 24 * time.Hour, nil
default:
return 0, fmt.Errorf("unknown TTL unit: %c", unit)
}
}
@ -113,25 +116,26 @@ type LoggingConfig struct {
}
type ServerConfig struct {
BindIP string `mapstructure:"bind_ip"`
ListenPort string `mapstructure:"listenport"`
UnixSocket bool `mapstructure:"unixsocket"`
StoragePath string `mapstructure:"storagepath"`
LogFile string `mapstructure:"logfile"` // NEW field
MetricsEnabled bool `mapstructure:"metricsenabled"`
MetricsPort string `mapstructure:"metricsport"`
FileTTL string `mapstructure:"filettl"`
MinFreeBytes string `mapstructure:"minfreebytes"`
FileTTL string `mapstructure:"filettl"`
FileTTLEnabled bool `mapstructure:"filettlenabled"`
AutoAdjustWorkers bool `mapstructure:"autoadjustworkers"`
NetworkEvents bool `mapstructure:"networkevents"`
TempPath string `mapstructure:"temppath"`
LoggingJSON bool `mapstructure:"loggingjson"`
PIDFilePath string `mapstructure:"pidfilepath"`
CleanUponExit bool `mapstructure:"cleanuponexit"`
PreCaching bool `mapstructure:"precaching"`
FileTTLEnabled bool `mapstructure:"filettlenabled"`
DeduplicationEnabled bool `mapstructure:"deduplicationenabled"`
Logging LoggingConfig `mapstructure:"logging"`
GlobalExtensions []string `mapstructure:"globalextensions"`
BindIP string `mapstructure:"bind_ip"` // Hinzugefügt: bind_ip
FileNaming string `mapstructure:"filenaming"`
// Removed TempPath, LoggingJSON
}
type DeduplicationConfig struct {
@ -325,7 +329,8 @@ func writePIDFile(pidPath string) error {
pidStr := strconv.Itoa(pid)
err := os.WriteFile(pidPath, []byte(pidStr), 0644)
if err != nil {
return fmt.Errorf("failed to write PID file: %v", err)
log.Errorf("Failed to write PID file: %v", err) // Improved error logging
return err
}
log.Infof("PID %d written to %s", pid, pidPath)
return nil
@ -335,9 +340,9 @@ func writePIDFile(pidPath string) error {
func removePIDFile(pidPath string) {
err := os.Remove(pidPath)
if err != nil {
log.Warnf("failed to remove PID file %s: %v", pidPath, err)
log.Errorf("Failed to remove PID file: %v", err) // Improved error logging
} else {
log.Infof("PID file %s removed", pidPath)
log.Infof("PID file %s removed successfully", pidPath)
}
}
@ -403,7 +408,8 @@ func main() {
os.Exit(1)
}
} else {
fmt.Println("No configuration file found.")
fmt.Println("No configuration file found. Please create a config file with the following content:")
printExampleConfig()
os.Exit(1)
}
}
@ -411,7 +417,11 @@ func main() {
err := readConfig(configFile, &conf)
if err != nil {
log.Fatalf("Error reading config: %v", err)
log.Fatalf("Failed to load configuration: %v\nPlease ensure your config.toml is present at one of the following paths:\n%v", err, []string{
"/etc/hmac-file-server/config.toml",
"../config.toml",
"./config.toml",
})
}
log.Info("Configuration loaded successfully.")
@ -442,14 +452,13 @@ func main() {
log.Infof("Server MinFreeBytes: %s", conf.Server.MinFreeBytes)
log.Infof("Server AutoAdjustWorkers: %v", conf.Server.AutoAdjustWorkers)
log.Infof("Server NetworkEvents: %v", conf.Server.NetworkEvents)
log.Infof("Server TempPath: %s", conf.Server.TempPath)
log.Infof("Server LoggingJSON: %v", conf.Server.LoggingJSON)
log.Infof("Server PIDFilePath: %s", conf.Server.PIDFilePath)
log.Infof("Server CleanUponExit: %v", conf.Server.CleanUponExit)
log.Infof("Server PreCaching: %v", conf.Server.PreCaching)
log.Infof("Server FileTTLEnabled: %v", conf.Server.FileTTLEnabled)
log.Infof("Server DeduplicationEnabled: %v", conf.Server.DeduplicationEnabled)
log.Infof("Server BindIP: %s", conf.Server.BindIP) // Hinzugefügt: Logging für BindIP
log.Infof("Server FileNaming: %s", conf.Server.FileNaming) // Added: Logging for FileNaming
err = writePIDFile(conf.Server.PIDFilePath) // Write PID file after config is loaded
if err != nil {
@ -622,6 +631,97 @@ func main() {
go handleFileCleanup(&conf)
}
func printExampleConfig() {
fmt.Print(`
[server]
bind_ip = "0.0.0.0"
listenport = "8080"
unixsocket = false
storagepath = "./uploads"
logfile = "/var/log/hmac-file-server.log"
metricsenabled = true
metricsport = "9090"
minfreebytes = "100MB"
filettl = "8760h"
filettlenabled = true
autoadjustworkers = true
networkevents = true
pidfilepath = "/var/run/hmacfileserver.pid"
cleanuponexit = true
precaching = true
deduplicationenabled = true
globalextensions = [".txt", ".pdf", ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".svg", ".webp"]
# FileNaming options: "HMAC", "None"
filenaming = "HMAC"
[logging]
level = "info"
file = "/var/log/hmac-file-server.log"
max_size = 100
max_backups = 7
max_age = 30
compress = true
[deduplication]
enabled = true
directory = "./deduplication"
[iso]
enabled = true
size = "1GB"
mountpoint = "/mnt/iso"
charset = "utf-8"
containerfile = "/mnt/iso/container.iso"
[timeouts]
readtimeout = "4800s"
writetimeout = "4800s"
idletimeout = "4800s"
[security]
secret = "changeme"
[versioning]
enableversioning = false
maxversions = 1
[uploads]
resumableuploadsenabled = true
chunkeduploadsenabled = true
chunksize = "8192"
allowedextensions = [".txt", ".pdf", ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".svg", ".webp"]
[downloads]
resumabledownloadsenabled = true
chunkeddownloadsenabled = true
chunksize = "8192"
allowedextensions = [".txt", ".pdf", ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".svg", ".webp"]
[clamav]
clamavenabled = true
clamavsocket = "/var/run/clamav/clamd.ctl"
numscanworkers = 2
scanfileextensions = [".txt", ".pdf", ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".svg", ".webp"]
[redis]
redisenabled = true
redisdbindex = 0
redisaddr = "localhost:6379"
redispassword = ""
redishealthcheckinterval = "120s"
[workers]
numworkers = 4
uploadqueuesize = 50
[file]
# Add file-specific configurations here
[build]
version = "2.6-Stable"
`)
}
func max(a, b int) int {
if a > b {
return a
@ -1636,6 +1736,7 @@ func handleUpload(w http.ResponseWriter, r *http.Request, absFilename, fileStore
} else {
log.Warn("No HMAC attached to URL.")
http.Error(w, "No HMAC attached to URL. Expecting 'v', 'v2', or 'token' parameter as MAC", http.StatusForbidden)
uploadErrorsTotal.Inc()
return
}
@ -1658,12 +1759,14 @@ func handleUpload(w http.ResponseWriter, r *http.Request, absFilename, fileStore
if err != nil {
log.Warn("Invalid MAC encoding")
http.Error(w, "Invalid MAC encoding", http.StatusForbidden)
uploadErrorsTotal.Inc()
return
}
if !hmac.Equal(calculatedMAC, providedMAC) {
log.Warn("Invalid MAC")
http.Error(w, "Invalid MAC", http.StatusForbidden)
uploadErrorsTotal.Inc()
return
}
@ -1686,23 +1789,36 @@ func handleUpload(w http.ResponseWriter, r *http.Request, absFilename, fileStore
return
}
// Determine the final filename based on the FileNaming configuration
finalFilename := absFilename
switch conf.Server.FileNaming {
case "HMAC":
finalFilename = filepath.Join(filepath.Dir(absFilename), hex.EncodeToString(calculatedMAC)+filepath.Ext(absFilename))
case "None", "none":
// Do nothing: keep finalFilename as-is
default:
log.Warnf("Unrecognized filenaming config %q, skipping rename.", conf.Server.FileNaming)
}
// Create temp file and write the uploaded data
tempFilename := absFilename + ".tmp"
tempFilename := finalFilename + ".tmp"
err = createFile(tempFilename, r)
if err != nil {
log.WithFields(logrus.Fields{
"filename": absFilename,
"filename": finalFilename,
}).WithError(err).Error("Error creating temp file")
http.Error(w, "Error writing temp file", http.StatusInternalServerError)
uploadErrorsTotal.Inc()
return
}
// Move temp file to final destination
err = os.Rename(tempFilename, absFilename)
err = os.Rename(tempFilename, finalFilename)
if err != nil {
log.Errorf("Rename failed for %s: %v", absFilename, err)
log.Errorf("Rename failed for %s: %v", finalFilename, err)
os.Remove(tempFilename)
http.Error(w, "Error moving file to final destination", http.StatusInternalServerError)
uploadErrorsTotal.Inc()
return
}
@ -1711,55 +1827,55 @@ func handleUpload(w http.ResponseWriter, r *http.Request, absFilename, fileStore
if f, ok := w.(http.Flusher); ok {
f.Flush()
}
log.Infof("Responded with 201 Created for file: %s", absFilename)
log.Infof("Responded with 201 Created for file: %s", finalFilename)
// Asynchronous processing in the background
go func() {
var logMessages []string
// ClamAV scanning
if conf.ClamAV.ClamAVEnabled && shouldScanFile(absFilename) {
err := scanFileWithClamAV(absFilename)
if conf.ClamAV.ClamAVEnabled && shouldScanFile(finalFilename) {
err := scanFileWithClamAV(finalFilename)
if err != nil {
logMessages = append(logMessages, fmt.Sprintf("ClamAV failed for %s: %v", absFilename, err))
logMessages = append(logMessages, fmt.Sprintf("ClamAV failed for %s: %v", finalFilename, err))
for _, msg := range logMessages {
log.Info(msg)
}
return
} else {
logMessages = append(logMessages, fmt.Sprintf("ClamAV scan passed for file: %s", absFilename))
logMessages = append(logMessages, fmt.Sprintf("ClamAV scan passed for file: %s", finalFilename))
}
}
// Deduplication
if conf.Redis.RedisEnabled && conf.Server.DeduplicationEnabled {
err := handleDeduplication(context.Background(), absFilename)
err := handleDeduplication(context.Background(), finalFilename)
if err != nil {
log.Errorf("Deduplication failed for %s: %v", absFilename, err)
os.Remove(absFilename)
log.Errorf("Deduplication failed for %s: %v", finalFilename, err)
os.Remove(finalFilename)
uploadErrorsTotal.Inc()
return
} else {
logMessages = append(logMessages, fmt.Sprintf("Deduplication handled successfully for file: %s", absFilename))
logMessages = append(logMessages, fmt.Sprintf("Deduplication handled successfully for file: %s", finalFilename))
}
}
// Versioning
if conf.Versioning.EnableVersioning {
if exists, _ := fileExists(absFilename); exists {
err := versionFile(absFilename)
if exists, _ := fileExists(finalFilename); exists {
err := versionFile(finalFilename)
if err != nil {
log.Errorf("Versioning failed for %s: %v", absFilename, err)
os.Remove(absFilename)
log.Errorf("Versioning failed for %s: %v", finalFilename, err)
os.Remove(finalFilename)
uploadErrorsTotal.Inc()
return
} else {
logMessages = append(logMessages, fmt.Sprintf("File versioned successfully: %s", absFilename))
logMessages = append(logMessages, fmt.Sprintf("File versioned successfully: %s", finalFilename))
}
}
}
logMessages = append(logMessages, fmt.Sprintf("Processing completed successfully for %s", absFilename))
logMessages = append(logMessages, fmt.Sprintf("Processing completed successfully for %s", finalFilename))
uploadsTotal.Inc()
// Log all messages at once
@ -1788,6 +1904,7 @@ func handleDownload(w http.ResponseWriter, r *http.Request, absFilename, fileSto
}
}
http.NotFound(w, r)
downloadErrorsTotal.Inc()
return
}
@ -1796,6 +1913,7 @@ func handleDownload(w http.ResponseWriter, r *http.Request, absFilename, fileSto
contentType = "application/octet-stream"
}
w.Header().Set("Content-Type", contentType)
w.Header().Set("Content-Length", strconv.FormatInt(fileInfo.Size(), 10))
if conf.Uploads.ResumableUploadsEnabled {
handleResumableDownload(absFilename, w, r, fileInfo.Size())

45
config.example.toml
View File

@ -1,21 +1,23 @@
[server]
bind_ip = "0.0.0.0"
listenport = "8080"
unixsocket = false
storagepath = "./uploads"
logfile = "/var/log/hmac-file-server.log"
metricsenabled = true
metricsport = "9090"
filettl = "8760h"
minfreebytes = "100MB"
filettl = "8760h"
filettlenabled = true
autoadjustworkers = true
networkevents = true
temppath = "/tmp/hmac-file-server"
loggingjson = false
pidfilepath = "/var/run/hmacfileserver.pid"
cleanuponexit = true
precaching = true
filettlenabled = true
globalextensions = ["*"] # Allows all file types globally
bind_ip = "0.0.0.0" # Specify the IP address to bind to (IPv4 or IPv6)
deduplicationenabled = true
globalextensions = [".txt", ".pdf", ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".svg", ".webp"]
# FileNaming options: "HMAC", "None"
filenaming = "HMAC"
[logging]
level = "info"
@ -34,7 +36,7 @@ enabled = true
size = "1GB"
mountpoint = "/mnt/iso"
charset = "utf-8"
containerfile = "/path/to/iso/container.iso"
containerfile = "/mnt/iso/container.iso"
[timeouts]
readtimeout = "4800s"
@ -46,36 +48,39 @@ secret = "changeme"
[versioning]
enableversioning = false
maxversions = 5
maxversions = 1
[uploads]
resumableuploadsenabled = false
resumableuploadsenabled = true
chunkeduploadsenabled = true
chunksize = "64mb"
allowedextensions = ["*"] # Use ["*"] to allow all or specify extensions
chunksize = "8192"
allowedextensions = [".txt", ".pdf", ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".svg", ".webp"]
[downloads]
resumabledownloadsenabled = false
resumabledownloadsenabled = true
chunkeddownloadsenabled = true
chunksize = "64mb"
allowedextensions = [".jpg", ".png"] # Restricts downloads to specific types
chunksize = "8192"
allowedextensions = [".txt", ".pdf", ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".svg", ".webp"]
[clamav]
clamavenabled = false
clamavenabled = true
clamavsocket = "/var/run/clamav/clamd.ctl"
numscanworkers = 2
scanfileextensions = [".exe", ".dll", ".pdf"]
scanfileextensions = [".txt", ".pdf", ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".svg", ".webp"]
[redis]
redisenabled = false
redisenabled = true
redisdbindex = 0
redisaddr = "localhost:6379"
redispassword = ""
redisdbindex = 0
redishealthcheckinterval = "120s"
[workers]
numworkers = 4
uploadqueuesize = 5000
uploadqueuesize = 50
[file]
# Add file-specific configurations here
[build]
version = "v2.5"
version = "2.6-Stable"

84
config.toml
View File

@ -1,81 +1,5 @@
[server]
listenport = "8080"
unixsocket = false
storagepath = "./uploads"
metricsenabled = true
metricsport = "9090"
filettl = "8760h"
minfreebytes = "100MB"
autoadjustworkers = true
networkevents = true
temppath = "/tmp/hmac-file-server"
loggingjson = false
pidfilepath = "/var/run/hmacfileserver.pid"
cleanuponexit = true
precaching = true
filettlenabled = true
globalextensions = ["*"] # Allows all file types globally
bind_ip = "0.0.0.0" # Specify the IP address to bind to (IPv4 or IPv6)
[logging]
level = "info"
file = "/var/log/hmac-file-server.log"
max_size = 100
max_backups = 7
max_age = 30
compress = true
[deduplication]
enabled = true
directory = "./deduplication"
[iso]
enabled = true
size = "1GB"
mountpoint = "/mnt/iso"
charset = "utf-8"
containerfile = "/path/to/iso/container.iso"
[timeouts]
readtimeout = "4800s"
writetimeout = "4800s"
idletimeout = "4800s"
[security]
secret = "changeme"
[versioning]
enableversioning = false
maxversions = 5
[uploads]
resumableuploadsenabled = false
chunkeduploadsenabled = true
chunksize = "64mb"
allowedextensions = ["*"] # Use ["*"] to allow all or specify extensions
[downloads]
resumabledownloadsenabled = false
chunkeddownloadsenabled = true
chunksize = "64mb"
allowedextensions = [".jpg", ".png"] # Restricts downloads to specific types
[clamav]
clamavenabled = false
clamavsocket = "/var/run/clamav/clamd.ctl"
numscanworkers = 2
scanfileextensions = [".exe", ".dll", ".pdf"]
[redis]
redisenabled = false
redisaddr = "localhost:6379"
redispassword = ""
redisdbindex = 0
redishealthcheckinterval = "120s"
[workers]
numworkers = 4
uploadqueuesize = 5000
[build]
version = "v2.5"
// ...existing code...
# FileNaming options: "HMAC", "None"
FileNaming = "HMAC"
// ...existing code...