Ignore .vscode folder

README.MD

@@ -82,12 +82,12 @@ FileTTL = "1y"
 DeduplicationEnabled = true
 MinFreeBytes = "100MB"
 AutoAdjustWorkers = true # Enable auto-adjustment for worker scaling
-NetworkEvents = false # Disable logging of network events
+NetworkEvents = false # IP changes or recording network activity will be turned off.
 
 [timeouts]
 ReadTimeout = "480s"
 WriteTimeout = "480s"
-IdleTimeout = "480s"
+IdleTimeout = "65s" # nginx/apache2 keep-a-live 60s
 
 [security]
 Secret = "changeme"

cmd/monitor/monitor.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/gdamore/tcell/v2"
+	"github.com/pelletier/go-toml"
 	"github.com/prometheus/common/expfmt"
 	"github.com/rivo/tview"
 	"github.com/shirou/gopsutil/v3/cpu"
@@ -16,7 +17,31 @@ import (
 	"github.com/shirou/gopsutil/v3/process"
 )
 
-const prometheusURL = "http://localhost:9090/metrics"
+var prometheusURL string
+
+func init() {
+	configPaths := []string{
+		"/etc/hmac-file-server/config.toml",
+		"../config.toml",
+	}
+
+	var config *toml.Tree
+	var err error
+
+	for _, path := range configPaths {
+		config, err = toml.LoadFile(path)
+		if err == nil {
+			break
+		}
+	}
+
+	if err != nil {
+		log.Fatalf("Error loading config file: %v", err)
+	}
+
+	port := config.Get("server.metrics_port").(int64)
+	prometheusURL = fmt.Sprintf("http://localhost:%d/metrics", port)
+}
 
 // Thresholds for color coding
 const (
@@ -334,7 +359,7 @@ func updateProcessTable(processTable *tview.Table, processes []ProcessInfo) {
 	}
 }
 
-// Helper function to update hmac-file-server table
+// Helper function to update hmac-table
 func updateHmacTable(hmacTable *tview.Table, hmacInfo *ProcessInfo, metrics map[string]float64) {
 	hmacTable.Clear()
 	hmacTable.SetCell(0, 0, tview.NewTableCell("Property").SetAttributes(tcell.AttrBold))

cmd/server/main.go

@@ -46,16 +46,16 @@ func parseSize(sizeStr string) (int64, error) {
 		return 0, fmt.Errorf("invalid size: %s", sizeStr)
 	}
 
-	unit := sizeStr[len(sizeStr)-2:]
+	unit := strings.ToUpper(sizeStr[len(sizeStr)-2:])
 	valueStr := sizeStr[:len(sizeStr)-2]
 	value, err := strconv.Atoi(valueStr)
 	if err != nil {
 		return 0, fmt.Errorf("invalid size value: %s", valueStr)
 	}
 
-	switch strings.ToUpper(unit) {
+	switch unit {
 	case "KB":
-		return int64(value) * 4, nil
+		return int64(value) * 1024, nil
 	case "MB":
 		return int64(value) * 1024 * 1024, nil
 	case "GB":
@@ -194,7 +194,7 @@ type NetworkEvent struct {
 
 var (
 	conf           Config
-	versionString  string = "v2.0-dev"
+	versionString  string = "v2.0-stable"
 	log                   = logrus.New()
 	uploadQueue    chan UploadTask
 	networkEvents  chan NetworkEvent
@@ -228,7 +228,8 @@ const (
 
 var bufferPool = sync.Pool{
 	New: func() interface{} {
-		return make([]byte, 32*1024)
+		buf := make([]byte, 32*1024)
+		return &buf
 	},
 }
 
@@ -850,8 +851,6 @@ func processUpload(task UploadTask) error {
 }
 
 func createFile(tempFilename string, r *http.Request) error {
-	startTime := time.Now()
-
 	err := os.MkdirAll(filepath.Dir(tempFilename), 0755)
 	if err != nil {
 		return err
@@ -859,16 +858,14 @@ func createFile(tempFilename string, r *http.Request) error {
 
 	file, err := os.OpenFile(tempFilename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
 	if err != nil {
-		log.WithFields(logrus.Fields{"file": tempFilename, "error": err}).Error("Error creating file")
-		uploadDuration.Observe(time.Since(startTime).Seconds())
 		return err
 	}
 	defer file.Close()
 
 	bufWriter := bufio.NewWriter(file)
 	defer bufWriter.Flush()
-
+	bufPtr := bufferPool.Get().(*[]byte) // Correct type assertion
-	bufPtr := bufferPool.Get().(*[]byte)
+	defer bufferPool.Put(bufPtr)
 	defer bufferPool.Put(bufPtr)
 
 	_, err = io.CopyBuffer(bufWriter, r.Body, *bufPtr)
@@ -1134,44 +1131,71 @@ func handleUpload(w http.ResponseWriter, r *http.Request, absFilename, fileStore
 		return
 	}
 
-	// Check for Callback-URL header
+	// Create temp file and write the uploaded data
-	callbackURL := r.Header.Get("Callback-URL")
+	tempFilename := absFilename + ".tmp"
-	if callbackURL != "" {
+	err = createFile(tempFilename, r)
-		log.Warnf("Callback-URL provided (%s) but not needed. Ignoring.", callbackURL)
+	if err != nil {
-		// Do not perform any callback actions
+		log.WithFields(logrus.Fields{"file": tempFilename, "error": err}).Error("Error creating temp file")
+		http.Error(w, "Error writing temp file", http.StatusInternalServerError)
+		return
 	}
 
-	// Enqueue the upload task
+	// Move temp file to final destination
-	result := make(chan error)
+	err = os.Rename(tempFilename, absFilename)
-	task := UploadTask{
+	if err != nil {
-		AbsFilename: absFilename,
+		log.Errorf("Rename failed for %s: %v", absFilename, err)
-		Request:     r,
+		os.Remove(tempFilename)
-		Result:      result,
+		http.Error(w, "Error moving file to final destination", http.StatusInternalServerError)
+		return
 	}
 
-	log.Debug("Attempting to enqueue upload task")
+	// Respond with 201 Created immediately
-	select {
+	w.WriteHeader(http.StatusCreated)
-	case uploadQueue <- task:
+	if f, ok := w.(http.Flusher); ok {
-		log.Debug("Upload task enqueued successfully")
+		f.Flush()
-	default:
+	}
-		log.Warn("Upload queue is full.")
+	log.Infof("Responded with 201 Created for file: %s", absFilename)
-		http.Error(w, "Server busy. Try again later.", http.StatusServiceUnavailable)
+
+	// Asynchronous processing in the background
+	go func() {
+		// ClamAV scanning
+		if conf.ClamAV.ClamAVEnabled && shouldScanFile(absFilename) {
+			err := scanFileWithClamAV(absFilename)
+			if err != nil {
+				log.Errorf("ClamAV failed for %s: %v", absFilename, err)
+				os.Remove(absFilename)
 				uploadErrorsTotal.Inc()
 				return
 			}
+		}
 
-	log.Debug("Waiting for upload task to complete")
+		// Deduplication
-	err = <-result
+		if conf.Redis.RedisEnabled && conf.Server.DeduplicationEnabled {
+			err := handleDeduplication(context.Background(), absFilename)
 			if err != nil {
-		log.Errorf("Upload failed: %v", err)
+				log.Errorf("Deduplication failed for %s: %v", absFilename, err)
-		http.Error(w, fmt.Sprintf("Upload failed: %v", err), http.StatusInternalServerError)
+				os.Remove(absFilename)
+				uploadErrorsTotal.Inc()
 				return
 			}
-	log.Debug("Upload task completed successfully")
+		}
 
-	// Respond with 201 Created on successful upload
+		// Versioning
-	w.WriteHeader(http.StatusCreated)
+		if conf.Versioning.EnableVersioning {
-	log.Infof("Responded with 201 Created for file: %s", absFilename)
+			if exists, _ := fileExists(absFilename); exists {
+				err := versionFile(absFilename)
+				if err != nil {
+					log.Errorf("Versioning failed for %s: %v", absFilename, err)
+					os.Remove(absFilename)
+					uploadErrorsTotal.Inc()
+					return
+				}
+			}
+		}
+
+		log.Infof("Processing completed successfully for %s", absFilename)
+		uploadsTotal.Inc()
+	}()
 }
 
 func handleDownload(w http.ResponseWriter, r *http.Request, absFilename, fileStorePath string) {

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
+	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect

go.sum

@@ -58,6 +58,8 @@ github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
 github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
+github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=