Ignore .vscode folder
This commit is contained in:
parent
474c46668b
commit
1152fa28cc
@ -82,12 +82,12 @@ FileTTL = "1y"
|
||||
DeduplicationEnabled = true
|
||||
MinFreeBytes = "100MB"
|
||||
AutoAdjustWorkers = true # Enable auto-adjustment for worker scaling
|
||||
NetworkEvents = false # Disable logging of network events
|
||||
NetworkEvents = false # IP changes or recording network activity will be turned off.
|
||||
|
||||
[timeouts]
|
||||
ReadTimeout = "480s"
|
||||
WriteTimeout = "480s"
|
||||
IdleTimeout = "480s"
|
||||
IdleTimeout = "65s" # nginx/apache2 keep-a-live 60s
|
||||
|
||||
[security]
|
||||
Secret = "changeme"
|
||||
|
||||
@ -9,6 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gdamore/tcell/v2"
|
||||
"github.com/pelletier/go-toml"
|
||||
"github.com/prometheus/common/expfmt"
|
||||
"github.com/rivo/tview"
|
||||
"github.com/shirou/gopsutil/v3/cpu"
|
||||
@ -16,7 +17,31 @@ import (
|
||||
"github.com/shirou/gopsutil/v3/process"
|
||||
)
|
||||
|
||||
const prometheusURL = "http://localhost:9090/metrics"
|
||||
var prometheusURL string
|
||||
|
||||
func init() {
|
||||
configPaths := []string{
|
||||
"/etc/hmac-file-server/config.toml",
|
||||
"../config.toml",
|
||||
}
|
||||
|
||||
var config *toml.Tree
|
||||
var err error
|
||||
|
||||
for _, path := range configPaths {
|
||||
config, err = toml.LoadFile(path)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
log.Fatalf("Error loading config file: %v", err)
|
||||
}
|
||||
|
||||
port := config.Get("server.metrics_port").(int64)
|
||||
prometheusURL = fmt.Sprintf("http://localhost:%d/metrics", port)
|
||||
}
|
||||
|
||||
// Thresholds for color coding
|
||||
const (
|
||||
@ -334,7 +359,7 @@ func updateProcessTable(processTable *tview.Table, processes []ProcessInfo) {
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to update hmac-file-server table
|
||||
// Helper function to update hmac-table
|
||||
func updateHmacTable(hmacTable *tview.Table, hmacInfo *ProcessInfo, metrics map[string]float64) {
|
||||
hmacTable.Clear()
|
||||
hmacTable.SetCell(0, 0, tview.NewTableCell("Property").SetAttributes(tcell.AttrBold))
|
||||
|
||||
@ -46,16 +46,16 @@ func parseSize(sizeStr string) (int64, error) {
|
||||
return 0, fmt.Errorf("invalid size: %s", sizeStr)
|
||||
}
|
||||
|
||||
unit := sizeStr[len(sizeStr)-2:]
|
||||
unit := strings.ToUpper(sizeStr[len(sizeStr)-2:])
|
||||
valueStr := sizeStr[:len(sizeStr)-2]
|
||||
value, err := strconv.Atoi(valueStr)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid size value: %s", valueStr)
|
||||
}
|
||||
|
||||
switch strings.ToUpper(unit) {
|
||||
switch unit {
|
||||
case "KB":
|
||||
return int64(value) * 4, nil
|
||||
return int64(value) * 1024, nil
|
||||
case "MB":
|
||||
return int64(value) * 1024 * 1024, nil
|
||||
case "GB":
|
||||
@ -194,7 +194,7 @@ type NetworkEvent struct {
|
||||
|
||||
var (
|
||||
conf Config
|
||||
versionString string = "v2.0-dev"
|
||||
versionString string = "v2.0-stable"
|
||||
log = logrus.New()
|
||||
uploadQueue chan UploadTask
|
||||
networkEvents chan NetworkEvent
|
||||
@ -228,7 +228,8 @@ const (
|
||||
|
||||
var bufferPool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
return make([]byte, 32*1024)
|
||||
buf := make([]byte, 32*1024)
|
||||
return &buf
|
||||
},
|
||||
}
|
||||
|
||||
@ -850,8 +851,6 @@ func processUpload(task UploadTask) error {
|
||||
}
|
||||
|
||||
func createFile(tempFilename string, r *http.Request) error {
|
||||
startTime := time.Now()
|
||||
|
||||
err := os.MkdirAll(filepath.Dir(tempFilename), 0755)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -859,16 +858,14 @@ func createFile(tempFilename string, r *http.Request) error {
|
||||
|
||||
file, err := os.OpenFile(tempFilename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
log.WithFields(logrus.Fields{"file": tempFilename, "error": err}).Error("Error creating file")
|
||||
uploadDuration.Observe(time.Since(startTime).Seconds())
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
bufWriter := bufio.NewWriter(file)
|
||||
defer bufWriter.Flush()
|
||||
|
||||
bufPtr := bufferPool.Get().(*[]byte)
|
||||
bufPtr := bufferPool.Get().(*[]byte) // Correct type assertion
|
||||
defer bufferPool.Put(bufPtr)
|
||||
defer bufferPool.Put(bufPtr)
|
||||
|
||||
_, err = io.CopyBuffer(bufWriter, r.Body, *bufPtr)
|
||||
@ -1134,44 +1131,71 @@ func handleUpload(w http.ResponseWriter, r *http.Request, absFilename, fileStore
|
||||
return
|
||||
}
|
||||
|
||||
// Check for Callback-URL header
|
||||
callbackURL := r.Header.Get("Callback-URL")
|
||||
if callbackURL != "" {
|
||||
log.Warnf("Callback-URL provided (%s) but not needed. Ignoring.", callbackURL)
|
||||
// Do not perform any callback actions
|
||||
}
|
||||
|
||||
// Enqueue the upload task
|
||||
result := make(chan error)
|
||||
task := UploadTask{
|
||||
AbsFilename: absFilename,
|
||||
Request: r,
|
||||
Result: result,
|
||||
}
|
||||
|
||||
log.Debug("Attempting to enqueue upload task")
|
||||
select {
|
||||
case uploadQueue <- task:
|
||||
log.Debug("Upload task enqueued successfully")
|
||||
default:
|
||||
log.Warn("Upload queue is full.")
|
||||
http.Error(w, "Server busy. Try again later.", http.StatusServiceUnavailable)
|
||||
uploadErrorsTotal.Inc()
|
||||
return
|
||||
}
|
||||
|
||||
log.Debug("Waiting for upload task to complete")
|
||||
err = <-result
|
||||
// Create temp file and write the uploaded data
|
||||
tempFilename := absFilename + ".tmp"
|
||||
err = createFile(tempFilename, r)
|
||||
if err != nil {
|
||||
log.Errorf("Upload failed: %v", err)
|
||||
http.Error(w, fmt.Sprintf("Upload failed: %v", err), http.StatusInternalServerError)
|
||||
log.WithFields(logrus.Fields{"file": tempFilename, "error": err}).Error("Error creating temp file")
|
||||
http.Error(w, "Error writing temp file", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
log.Debug("Upload task completed successfully")
|
||||
|
||||
// Respond with 201 Created on successful upload
|
||||
// Move temp file to final destination
|
||||
err = os.Rename(tempFilename, absFilename)
|
||||
if err != nil {
|
||||
log.Errorf("Rename failed for %s: %v", absFilename, err)
|
||||
os.Remove(tempFilename)
|
||||
http.Error(w, "Error moving file to final destination", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// Respond with 201 Created immediately
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
if f, ok := w.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
log.Infof("Responded with 201 Created for file: %s", absFilename)
|
||||
|
||||
// Asynchronous processing in the background
|
||||
go func() {
|
||||
// ClamAV scanning
|
||||
if conf.ClamAV.ClamAVEnabled && shouldScanFile(absFilename) {
|
||||
err := scanFileWithClamAV(absFilename)
|
||||
if err != nil {
|
||||
log.Errorf("ClamAV failed for %s: %v", absFilename, err)
|
||||
os.Remove(absFilename)
|
||||
uploadErrorsTotal.Inc()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Deduplication
|
||||
if conf.Redis.RedisEnabled && conf.Server.DeduplicationEnabled {
|
||||
err := handleDeduplication(context.Background(), absFilename)
|
||||
if err != nil {
|
||||
log.Errorf("Deduplication failed for %s: %v", absFilename, err)
|
||||
os.Remove(absFilename)
|
||||
uploadErrorsTotal.Inc()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Versioning
|
||||
if conf.Versioning.EnableVersioning {
|
||||
if exists, _ := fileExists(absFilename); exists {
|
||||
err := versionFile(absFilename)
|
||||
if err != nil {
|
||||
log.Errorf("Versioning failed for %s: %v", absFilename, err)
|
||||
os.Remove(absFilename)
|
||||
uploadErrorsTotal.Inc()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.Infof("Processing completed successfully for %s", absFilename)
|
||||
uploadsTotal.Inc()
|
||||
}()
|
||||
}
|
||||
|
||||
func handleDownload(w http.ResponseWriter, r *http.Request, absFilename, fileStorePath string) {
|
||||
|
||||
1
go.mod
1
go.mod
@ -21,6 +21,7 @@ require (
|
||||
github.com/mattn/go-runewidth v0.0.15 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
|
||||
github.com/pelletier/go-toml v1.9.5 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
|
||||
2
go.sum
2
go.sum
@ -58,6 +58,8 @@ github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
|
||||
github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
|
||||
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
|
||||
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
|
||||
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
|
||||
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user