uuxo/hmac-file-server
1
1
Fork 0
Code Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Implement code changes to enhance functionality and improve performance

Browse Source
This commit is contained in:
Alexander Renz
2025-07-18 15:21:43 +00:00
parent bd850ac8e0
commit 77419e5595
5 changed files with 413 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

319
cmd/server/config_simplified.go Normal file
View File

@ -0,0 +1,319 @@
package main
import (
"fmt"
"os"
"github.com/spf13/viper"
)
// DefaultConfig returns a Config struct populated with sensible defaults
func DefaultConfig() *Config {
return &Config{
Server: ServerConfig{
ListenAddress: "8080",
StoragePath: "./uploads",
MetricsEnabled: true,
MetricsPath: "/metrics",
MetricsPort: "9090",
PidFile: "/tmp/hmac-file-server.pid",
PIDFilePath: "/tmp/hmac-file-server.pid",
MaxUploadSize: "10GB",
MaxHeaderBytes: 1048576, // 1MB
CleanupInterval: "24h",
MaxFileAge: "720h", // 30 days
PreCache: true,
PreCacheWorkers: 4,
PreCacheInterval: "1h",
GlobalExtensions: []string{".txt", ".dat", ".iso", ".mp4", ".mkv", ".avi", ".mov", ".wmv", ".flv", ".webm", ".mpeg"},
DeduplicationEnabled: true,
MinFreeBytes: "1GB",
FileNaming: "original",
ForceProtocol: "",
EnableDynamicWorkers: true,
WorkerScaleUpThresh: 40, // Optimized from previous session
WorkerScaleDownThresh: 10,
},
Uploads: UploadsConfig{
AllowedExtensions: []string{".zip", ".rar", ".7z", ".tar.gz", ".tgz", ".gpg", ".enc", ".pgp", ".txt", ".pdf", ".png", ".jpg", ".jpeg"},
ChunkedUploadsEnabled: true,
ChunkSize: "10MB",
ResumableUploadsEnabled: true,
SessionTimeout: "60m", // Extended from previous session
MaxRetries: 3,
},
Downloads: DownloadsConfig{
AllowedExtensions: []string{".txt", ".pdf", ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".svg", ".webp", ".zip"},
ChunkedDownloadsEnabled: true,
ChunkSize: "10MB",
ResumableDownloadsEnabled: true,
},
Security: SecurityConfig{
Secret: "your-very-secret-hmac-key",
EnableJWT: false,
JWTSecret: "your-256-bit-secret",
JWTAlgorithm: "HS256",
JWTExpiration: "24h",
},
Logging: LoggingConfig{
Level: "info",
File: "/var/log/hmac-file-server.log",
MaxSize: 100,
MaxBackups: 7,
MaxAge: 30,
Compress: true,
},
Deduplication: DeduplicationConfig{
Enabled: true,
Directory: "./dedup_store",
MaxSize: "1GB",
},
ISO: ISOConfig{
Enabled: false,
Size: "1GB",
MountPoint: "/mnt/iso",
Charset: "utf-8",
ContainerFile: "/mnt/iso/container.iso",
},
Timeouts: TimeoutConfig{
Read: "300s", // 5 minutes instead of 4800s
Write: "300s",
Idle: "300s",
Shutdown: "30s",
},
Versioning: VersioningConfig{
Enabled: false,
Backend: "simple",
MaxRevs: 1,
},
ClamAV: ClamAVConfig{
ClamAVEnabled: false,
ClamAVSocket: "/var/run/clamav/clamd.ctl",
NumScanWorkers: 2,
ScanFileExtensions: []string{".txt", ".pdf", ".doc", ".docx", ".xls", ".xlsx", ".exe", ".zip", ".rar", ".7z", ".tar", ".gz"},
MaxScanSize: "200MB",
},
Redis: RedisConfig{
RedisEnabled: false,
RedisDBIndex: 0,
RedisAddr: "localhost:6379",
RedisPassword: "",
RedisHealthCheckInterval: "120s",
},
Workers: WorkersConfig{
NumWorkers: 4,
UploadQueueSize: 100, // Optimized from previous session
},
File: FileConfig{},
Build: BuildConfig{
Version: "3.2",
},
}
}
// LoadSimplifiedConfig loads configuration with a minimal config file approach
func LoadSimplifiedConfig(configPath string) (*Config, error) {
// Start with comprehensive defaults
config := DefaultConfig()
// If no config file specified, try to find one in common locations
if configPath == "" {
possiblePaths := []string{
"/opt/hmac-file-server/config.toml",
"/etc/hmac-file-server/config.toml",
"./config.toml",
"../config.toml",
}
for _, path := range possiblePaths {
if _, err := os.Stat(path); err == nil {
configPath = path
break
}
}
}
// If a config file exists, load it to override defaults
if configPath != "" && fileExists(configPath) {
viper.SetConfigFile(configPath)
viper.SetConfigType("toml")
if err := viper.ReadInConfig(); err != nil {
return nil, fmt.Errorf("failed to read config file %s: %v", configPath, err)
}
// Unmarshal only the values that are explicitly set in the config file
if err := viper.Unmarshal(config); err != nil {
return nil, fmt.Errorf("failed to unmarshal config: %v", err)
}
}
return config, nil
}
// fileExists checks if a file exists
func fileExists(filename string) bool {
info, err := os.Stat(filename)
if os.IsNotExist(err) {
return false
}
return !info.IsDir()
}
// GenerateMinimalConfig creates a minimal config.toml with only essential settings
func GenerateMinimalConfig() string {
return `# HMAC File Server - Minimal Configuration
# This file contains only the essential settings you might want to customize.
# All other settings use sensible defaults defined in the application.
[server]
# Network binding
listen_address = "8080"
# Storage location for uploaded files
storage_path = "./uploads"
# Security settings
[security]
# IMPORTANT: Change this secret key for production use!
secret = "your-very-secret-hmac-key"
# Logging configuration
[logging]
# Log level: debug, info, warn, error
level = "info"
file = "/var/log/hmac-file-server.log"
# Advanced settings (uncomment and modify if needed)
# [uploads]
# max_resumable_age = "48h"
# chunk_size = "10MB"
# [workers]
# numworkers = 4
# uploadqueuesize = 100
# [deduplication]
# enabled = true
# directory = "./dedup_store"
# [timeouts]
# readtimeout = "4800s"
# writetimeout = "4800s"
# idletimeout = "4800s"
# [clamav]
# clamavenabled = false
# [redis]
# redisenabled = false
`
}
// createMinimalConfig writes a minimal config file to the current directory
func createMinimalConfig() error {
content := GenerateMinimalConfig()
return os.WriteFile("config.toml", []byte(content), 0644)
}
// GenerateAdvancedConfigTemplate creates a comprehensive config template for advanced users
func GenerateAdvancedConfigTemplate() string {
return `# HMAC File Server - Advanced Configuration Template
# This template shows all available configuration options with their default values.
# Uncomment and modify only the settings you want to change.
[server]
listen_address = "8080"
storage_path = "./uploads"
metrics_enabled = true
metrics_path = "/metrics"
pid_file = "/var/run/hmac-file-server.pid"
max_upload_size = "10GB"
max_header_bytes = 1048576
cleanup_interval = "24h"
max_file_age = "720h"
pre_cache = true
pre_cache_workers = 4
pre_cache_interval = "1h"
global_extensions = [".txt", ".dat", ".iso", ".mp4", ".mkv", ".avi", ".mov", ".wmv", ".flv", ".webm", ".mpeg"]
deduplication_enabled = true
min_free_bytes = "1GB"
file_naming = "original"
force_protocol = ""
enable_dynamic_workers = true
worker_scale_up_thresh = 40
worker_scale_down_thresh = 10
[uploads]
allowed_extensions = [".zip", ".rar", ".7z", ".tar.gz", ".tgz", ".gpg", ".enc", ".pgp"]
chunked_uploads_enabled = true
chunk_size = "10MB"
resumable_uploads_enabled = true
max_resumable_age = "48h"
sessiontimeout = "60m"
maxretries = 3
[downloads]
allowed_extensions = [".txt", ".pdf", ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".svg", ".webp"]
chunked_downloads_enabled = true
chunk_size = "8192"
resumable_downloads_enabled = true
[security]
secret = "your-very-secret-hmac-key"
enablejwt = false
jwtsecret = "your-256-bit-secret"
jwtalgorithm = "HS256"
jwtexpiration = "24h"
[logging]
level = "info"
file = "/var/log/hmac-file-server.log"
max_size = 100
max_backups = 7
max_age = 30
compress = true
[deduplication]
enabled = true
directory = "./dedup_store"
maxsize = "1GB"
[iso]
enabled = false
size = "1GB"
mountpoint = "/mnt/iso"
charset = "utf-8"
containerfile = "/mnt/iso/container.iso"
[timeouts]
readtimeout = "4800s"
writetimeout = "4800s"
idletimeout = "4800s"
[versioning]
enableversioning = false
maxversions = 1
[clamav]
clamavenabled = false
clamavsocket = "/var/run/clamav/clamd.ctl"
numscanworkers = 2
scanfileextensions = [".txt", ".pdf", ".doc", ".docx", ".xls", ".xlsx", ".exe", ".zip", ".rar", ".7z", ".tar", ".gz"]
maxscansize = "200MB"
[redis]
redisenabled = false
redisdbindex = 0
redisaddr = "localhost:6379"
redispassword = ""
redishealthcheckinterval = "120s"
[workers]
numworkers = 4
uploadqueuesize = 100
[build]
version = "3.2"
`
}

93
cmd/server/main.go
View File

@ -103,8 +103,8 @@ func parseTTL(ttlStr string) (time.Duration, error) {
// Configuration structures
type ServerConfig struct {
ListenAddress string `toml:"listenport" mapstructure:"listenport"` // Fixed to match config file field
StoragePath string `toml:"storagepath" mapstructure:"storagepath"` // Fixed to match config
ListenAddress string `toml:"listen_address" mapstructure:"listen_address"`
StoragePath string `toml:"storage_path" mapstructure:"storage_path"`
MetricsEnabled bool `toml:"metricsenabled" mapstructure:"metricsenabled"` // Fixed to match config
MetricsPath string `toml:"metrics_path" mapstructure:"metrics_path"`
PidFile string `toml:"pid_file" mapstructure:"pid_file"`
@ -136,18 +136,18 @@ type ServerConfig struct {
}
type UploadsConfig struct {
AllowedExtensions []string `toml:"allowedextensions" mapstructure:"allowedextensions"`
ChunkedUploadsEnabled bool `toml:"chunkeduploadsenabled" mapstructure:"chunkeduploadsenabled"`
ChunkSize string `toml:"chunksize" mapstructure:"chunksize"`
ResumableUploadsEnabled bool `toml:"resumableuploadsenabled" mapstructure:"resumableuploadsenabled"`
AllowedExtensions []string `toml:"allowed_extensions" mapstructure:"allowed_extensions"`
ChunkedUploadsEnabled bool `toml:"chunked_uploads_enabled" mapstructure:"chunked_uploads_enabled"`
ChunkSize string `toml:"chunk_size" mapstructure:"chunk_size"`
ResumableUploadsEnabled bool `toml:"resumable_uploads_enabled" mapstructure:"resumable_uploads_enabled"`
SessionTimeout string `toml:"sessiontimeout" mapstructure:"sessiontimeout"`
MaxRetries int `toml:"maxretries" mapstructure:"maxretries"`
}
type DownloadsConfig struct {
AllowedExtensions []string `toml:"allowedextensions" mapstructure:"allowedextensions"`
ChunkedDownloadsEnabled bool `toml:"chunkeddownloadsenabled" mapstructure:"chunkeddownloadsenabled"`
ChunkSize string `toml:"chunksize" mapstructure:"chunksize"`
AllowedExtensions []string `toml:"allowed_extensions" mapstructure:"allowed_extensions"`
ChunkedDownloadsEnabled bool `toml:"chunked_downloads_enabled" mapstructure:"chunked_downloads_enabled"`
ChunkSize string `toml:"chunk_size" mapstructure:"chunk_size"`
ResumableDownloadsEnabled bool `toml:"resumable_downloads_enabled" mapstructure:"resumable_downloads_enabled"`
}
@ -450,11 +450,10 @@ func initializeNetworkProtocol(forceProtocol string) (*net.Dialer, error) {
var dualStackClient *http.Client
func main() {
setDefaults() // Call setDefaults before parsing flags or reading config
var configFile string
flag.StringVar(&configFile, "config", "./config.toml", "Path to configuration file \"config.toml\".")
var genConfig bool
var genConfigAdvanced bool
var genConfigPath string
var validateOnly bool
var runConfigTests bool
@ -467,8 +466,9 @@ func main() {
var listValidationChecks bool
var showVersion bool
flag.BoolVar(&genConfig, "genconfig", false, "Print example configuration and exit.")
flag.StringVar(&genConfigPath, "genconfig-path", "", "Write example configuration to the given file and exit.")
flag.BoolVar(&genConfig, "genconfig", false, "Print minimal configuration example and exit.")
flag.BoolVar(&genConfigAdvanced, "genconfig-advanced", false, "Print advanced configuration template and exit.")
flag.StringVar(&genConfigPath, "genconfig-path", "", "Write configuration to the given file and exit.")
flag.BoolVar(&validateOnly, "validate-config", false, "Validate configuration and exit without starting server.")
flag.BoolVar(&runConfigTests, "test-config", false, "Run configuration validation test scenarios and exit.")
flag.BoolVar(&validateQuiet, "validate-quiet", false, "Only show errors during validation (suppress warnings and info).")
@ -492,10 +492,24 @@ func main() {
}
if genConfig {
printExampleConfig()
fmt.Println("# Option 1: Minimal Configuration (recommended for most users)")
fmt.Println(GenerateMinimalConfig())
fmt.Println("\n# Option 2: Advanced Configuration Template (for fine-tuning)")
fmt.Println("# Use -genconfig-advanced to generate the advanced template")
os.Exit(0)
}
if genConfigAdvanced {
fmt.Println(GenerateAdvancedConfigTemplate())
os.Exit(0)
}
if genConfigPath != "" {
var content string
if genConfigAdvanced {
content = GenerateAdvancedConfigTemplate()
} else {
content = GenerateMinimalConfig()
}
f, err := os.Create(genConfigPath)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to create file: %v\n", err)
@ -503,9 +517,9 @@ func main() {
}
defer f.Close()
w := bufio.NewWriter(f)
fmt.Fprint(w, getExampleConfigString())
fmt.Fprint(w, content)
w.Flush()
fmt.Printf("Example config written to %s\n", genConfigPath)
fmt.Printf("Configuration written to %s\n", genConfigPath)
os.Exit(0)
}
if runConfigTests {
@ -513,42 +527,21 @@ func main() {
os.Exit(0)
}
// Initialize Viper
viper.SetConfigType("toml")
// Set default config path
defaultConfigPath := "/etc/hmac-file-server/config.toml"
// Attempt to load the default config
viper.SetConfigFile(defaultConfigPath)
if err := viper.ReadInConfig(); err != nil {
// If default config not found, fallback to parent directory
parentDirConfig := "../config.toml"
viper.SetConfigFile(parentDirConfig)
if err := viper.ReadInConfig(); err != nil {
// If still not found and -config is provided, use it
if configFile != "" {
viper.SetConfigFile(configFile)
if err := viper.ReadInConfig(); err != nil {
fmt.Printf("Error loading config file: %v\n", err)
os.Exit(1)
}
} else {
fmt.Println("No configuration file found. Please create a config file with the following content:")
printExampleConfig()
os.Exit(1)
}
}
}
err := readConfig(configFile, &conf)
// Load configuration using simplified approach
loadedConfig, err := LoadSimplifiedConfig(configFile)
if err != nil {
log.Fatalf("Failed to load configuration: %v\nPlease ensure your config.toml is present at one of the following paths:\n%v", err, []string{
"/etc/hmac-file-server/config.toml",
"../config.toml",
"./config.toml",
})
// If no config file exists, offer to create a minimal one
if configFile == "./config.toml" || configFile == "" {
fmt.Println("No configuration file found. Creating a minimal config.toml...")
if err := createMinimalConfig(); err != nil {
log.Fatalf("Failed to create minimal config: %v", err)
}
fmt.Println("Minimal config.toml created. Please review and modify as needed, then restart the server.")
os.Exit(0)
}
log.Fatalf("Failed to load configuration: %v", err)
}
conf = *loadedConfig
log.Info("Configuration loaded successfully.")
err = validateConfig(&conf)

38
config-simple.toml Normal file
View File

@ -0,0 +1,38 @@
# HMAC File Server - Simplified Configuration
# This file contains only the essential settings you need to configure.
# All other settings use sensible defaults optimized for production use.
[server]
# Network binding
listen_address = ":8080"
# Storage location for uploaded files
storage_path = "/opt/hmac-file-server/data/uploads"
# Security settings - IMPORTANT: Change the secret for production!
[security]
secret = "f6g4ldPvQM7O2UTFeBEUUj33VrXypDAcsDt0yqKrLiOr5oQW"
# Logging configuration
[logging]
level = "info"
file = "/opt/hmac-file-server/data/logs/hmac-file-server.log"
# Advanced settings - uncomment and modify if needed:
# [deduplication]
# enabled = true
# directory = "/opt/hmac-file-server/data/dedup_store"
# [workers]
# numworkers = 4
# uploadqueuesize = 100
# [uploads]
# sessiontimeout = "60m"
# chunk_size = "10MB"
# [timeouts]
# readtimeout = "4800s"
# writetimeout = "4800s"
# idletimeout = "4800s"

13
config-simplified-production.toml Normal file
View File

@ -0,0 +1,13 @@
# HMAC File Server - Simplified Production Configuration
# Only the essential settings you need to configure!
[server]
listen_address = "8080"
storage_path = "/opt/hmac-file-server/data/uploads"
[security]
secret = "f6g4ldPvQM7O2UTFeBEUUj33VrXypDAcsDt0yqKrLiOr5oQW"
[logging]
level = "info"
file = "/opt/hmac-file-server/data/logs/hmac-file-server.log"

BIN
hmac-file-server-simplified Executable file
View File

Binary file not shown.