From 347f9b1ede5328b996d0c471fe5f3a2a82fe09bd Mon Sep 17 00:00:00 2001 From: Alexander Renz Date: Fri, 18 Jul 2025 14:46:50 +0000 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A5=20Tremora=20del=20Terra:=20ultimat?= =?UTF-8?q?e=20hmac-file-server=20fix=20=E2=80=93=20final=20push=20before?= =?UTF-8?q?=20the=20drop=20=F0=9F=92=BE=F0=9F=94=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CLAMAV_LARGE_FILE_FIX.md | 0 CLAMAV_SCANNING_FIX.md | 0 CLAMAV_SECURITY_CONFIG.md | 0 CLUSTER_RESTART_GUIDE.md | 0 DEDUPLICATION_1GB_OPTIMIZATION.md | 0 DOCUMENTATION_UPDATE_SUMMARY.md | 0 ENDLESS_ENCRYPTION_FIX.md | 0 FINAL_STATUS_REPORT.md | 0 INSTALL.MD | 0 LARGE_FILE_UPLOAD_FIX.md | 0 NETWORK_RESILIENCE_GUIDE.md | 0 PERFORMANCE_OPTIMIZATION.md | 0 QUEUE_RESILIENCE_GUIDE.md | 0 QUEUE_RESILIENCE_SUMMARY.md | 0 README.MD | 0 UNIVERSAL_LARGE_UPLOAD_FIX.md | 0 UPLOAD_COMPLETION_FIX.md | 0 UPLOAD_PERFORMANCE_VERIFICATION.md | 0 VIDEO_EXTENSION_FIX.md | 0 cmd/server/chunked_upload_handler.go | 28 +++++++++++ cmd/server/main.go | 69 ++++++++++++++++++++++++++++ cmd/server/queue_resilience.go | 1 - config-clean.toml | 0 tests/README.md | 0 verify_xmpp_upload.sh | 0 25 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 CLAMAV_LARGE_FILE_FIX.md create mode 100644 CLAMAV_SCANNING_FIX.md create mode 100644 CLAMAV_SECURITY_CONFIG.md create mode 100644 CLUSTER_RESTART_GUIDE.md create mode 100644 DEDUPLICATION_1GB_OPTIMIZATION.md create mode 100644 DOCUMENTATION_UPDATE_SUMMARY.md create mode 100644 ENDLESS_ENCRYPTION_FIX.md create mode 100644 FINAL_STATUS_REPORT.md create mode 100644 INSTALL.MD create mode 100644 LARGE_FILE_UPLOAD_FIX.md create mode 100644 NETWORK_RESILIENCE_GUIDE.md create mode 100644 PERFORMANCE_OPTIMIZATION.md create mode 100644 QUEUE_RESILIENCE_GUIDE.md create mode 100644 QUEUE_RESILIENCE_SUMMARY.md create mode 100644 README.MD create mode 100644 UNIVERSAL_LARGE_UPLOAD_FIX.md create mode 100644 UPLOAD_COMPLETION_FIX.md create mode 100644 UPLOAD_PERFORMANCE_VERIFICATION.md create mode 100644 VIDEO_EXTENSION_FIX.md create mode 100644 config-clean.toml create mode 100644 tests/README.md create mode 100644 verify_xmpp_upload.sh diff --git a/CLAMAV_LARGE_FILE_FIX.md b/CLAMAV_LARGE_FILE_FIX.md new file mode 100644 index 0000000..e69de29 diff --git a/CLAMAV_SCANNING_FIX.md b/CLAMAV_SCANNING_FIX.md new file mode 100644 index 0000000..e69de29 diff --git a/CLAMAV_SECURITY_CONFIG.md b/CLAMAV_SECURITY_CONFIG.md new file mode 100644 index 0000000..e69de29 diff --git a/CLUSTER_RESTART_GUIDE.md b/CLUSTER_RESTART_GUIDE.md new file mode 100644 index 0000000..e69de29 diff --git a/DEDUPLICATION_1GB_OPTIMIZATION.md b/DEDUPLICATION_1GB_OPTIMIZATION.md new file mode 100644 index 0000000..e69de29 diff --git a/DOCUMENTATION_UPDATE_SUMMARY.md b/DOCUMENTATION_UPDATE_SUMMARY.md new file mode 100644 index 0000000..e69de29 diff --git a/ENDLESS_ENCRYPTION_FIX.md b/ENDLESS_ENCRYPTION_FIX.md new file mode 100644 index 0000000..e69de29 diff --git a/FINAL_STATUS_REPORT.md b/FINAL_STATUS_REPORT.md new file mode 100644 index 0000000..e69de29 diff --git a/INSTALL.MD b/INSTALL.MD new file mode 100644 index 0000000..e69de29 diff --git a/LARGE_FILE_UPLOAD_FIX.md b/LARGE_FILE_UPLOAD_FIX.md new file mode 100644 index 0000000..e69de29 diff --git a/NETWORK_RESILIENCE_GUIDE.md b/NETWORK_RESILIENCE_GUIDE.md new file mode 100644 index 0000000..e69de29 diff --git a/PERFORMANCE_OPTIMIZATION.md b/PERFORMANCE_OPTIMIZATION.md new file mode 100644 index 0000000..e69de29 diff --git a/QUEUE_RESILIENCE_GUIDE.md b/QUEUE_RESILIENCE_GUIDE.md new file mode 100644 index 0000000..e69de29 diff --git a/QUEUE_RESILIENCE_SUMMARY.md b/QUEUE_RESILIENCE_SUMMARY.md new file mode 100644 index 0000000..e69de29 diff --git a/README.MD b/README.MD new file mode 100644 index 0000000..e69de29 diff --git a/UNIVERSAL_LARGE_UPLOAD_FIX.md b/UNIVERSAL_LARGE_UPLOAD_FIX.md new file mode 100644 index 0000000..e69de29 diff --git a/UPLOAD_COMPLETION_FIX.md b/UPLOAD_COMPLETION_FIX.md new file mode 100644 index 0000000..e69de29 diff --git a/UPLOAD_PERFORMANCE_VERIFICATION.md b/UPLOAD_PERFORMANCE_VERIFICATION.md new file mode 100644 index 0000000..e69de29 diff --git a/VIDEO_EXTENSION_FIX.md b/VIDEO_EXTENSION_FIX.md new file mode 100644 index 0000000..e69de29 diff --git a/cmd/server/chunked_upload_handler.go b/cmd/server/chunked_upload_handler.go index 9f0389b..dac2922 100644 --- a/cmd/server/chunked_upload_handler.go +++ b/cmd/server/chunked_upload_handler.go @@ -105,6 +105,34 @@ func handleChunkedUpload(w http.ResponseWriter, r *http.Request) { } } + // Pre-upload deduplication check for chunked uploads + if conf.Server.DeduplicationEnabled { + finalPath := filepath.Join(conf.Server.StoragePath, filename) + if existingFileInfo, err := os.Stat(finalPath); err == nil { + // File already exists - return success immediately for deduplication hit + duration := time.Since(startTime) + uploadDuration.Observe(duration.Seconds()) + uploadsTotal.Inc() + uploadSizeBytes.Observe(float64(existingFileInfo.Size())) + filesDeduplicatedTotal.Inc() + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + response := map[string]interface{}{ + "success": true, + "filename": filename, + "size": existingFileInfo.Size(), + "completed": true, + "message": "File already exists (deduplication hit)", + } + writeJSONResponse(w, response) + + log.Infof("Chunked upload deduplication hit: file %s already exists (%s), returning success immediately", + filename, formatBytes(existingFileInfo.Size())) + return + } + } + // Create new session clientIP := getClientIP(r) session := uploadSessionStore.CreateSession(filename, totalSize, clientIP) diff --git a/cmd/server/main.go b/cmd/server/main.go index 818d42f..4645bed 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -1515,6 +1515,32 @@ func handleUpload(w http.ResponseWriter, r *http.Request) { absFilename := filepath.Join(storagePath, filename) + // Pre-upload deduplication check: if file already exists and deduplication is enabled, return success immediately + if conf.Server.DeduplicationEnabled { + if existingFileInfo, err := os.Stat(absFilename); err == nil { + // File already exists - return success immediately for deduplication hit + duration := time.Since(startTime) + uploadDuration.Observe(duration.Seconds()) + uploadsTotal.Inc() + uploadSizeBytes.Observe(float64(existingFileInfo.Size())) + filesDeduplicatedTotal.Inc() + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + response := map[string]interface{}{ + "success": true, + "filename": filename, + "size": existingFileInfo.Size(), + "message": "File already exists (deduplication hit)", + } + json.NewEncoder(w).Encode(response) + + log.Infof("Deduplication hit: file %s already exists (%s), returning success immediately", + filename, formatBytes(existingFileInfo.Size())) + return + } + } + // Create the file dst, err := os.Create(absFilename) if err != nil { @@ -1752,6 +1778,32 @@ func handleV3Upload(w http.ResponseWriter, r *http.Request) { absFilename := filepath.Join(storagePath, filename) + // Pre-upload deduplication check: if file already exists and deduplication is enabled, return success immediately + if conf.Server.DeduplicationEnabled { + if existingFileInfo, err := os.Stat(absFilename); err == nil { + // File already exists - return success immediately for deduplication hit + duration := time.Since(startTime) + uploadDuration.Observe(duration.Seconds()) + uploadsTotal.Inc() + uploadSizeBytes.Observe(float64(existingFileInfo.Size())) + filesDeduplicatedTotal.Inc() + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + response := map[string]interface{}{ + "success": true, + "filename": filename, + "size": existingFileInfo.Size(), + "message": "File already exists (deduplication hit)", + } + json.NewEncoder(w).Encode(response) + + log.Infof("Deduplication hit: file %s already exists (%s), returning success immediately", + filename, formatBytes(existingFileInfo.Size())) + return + } + } + // Create the file dst, err := os.Create(absFilename) if err != nil { @@ -1907,6 +1959,23 @@ func handleLegacyUpload(w http.ResponseWriter, r *http.Request) { return } + // Pre-upload deduplication check: if file already exists and deduplication is enabled, return success immediately + if conf.Server.DeduplicationEnabled { + if existingFileInfo, err := os.Stat(absFilename); err == nil { + // File already exists - return success immediately for deduplication hit + duration := time.Since(startTime) + uploadDuration.Observe(duration.Seconds()) + uploadsTotal.Inc() + uploadSizeBytes.Observe(float64(existingFileInfo.Size())) + filesDeduplicatedTotal.Inc() + + w.WriteHeader(http.StatusCreated) // 201 Created for legacy compatibility + log.Infof("Deduplication hit: file %s already exists (%s), returning success immediately", + filename, formatBytes(existingFileInfo.Size())) + return + } + } + // Create the file dst, err := os.Create(absFilename) if err != nil { diff --git a/cmd/server/queue_resilience.go b/cmd/server/queue_resilience.go index e7105a5..8dab27c 100644 --- a/cmd/server/queue_resilience.go +++ b/cmd/server/queue_resilience.go @@ -5,7 +5,6 @@ package main import ( "context" "errors" - "fmt" "sync" "sync/atomic" "time" diff --git a/config-clean.toml b/config-clean.toml new file mode 100644 index 0000000..e69de29 diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..e69de29 diff --git a/verify_xmpp_upload.sh b/verify_xmpp_upload.sh new file mode 100644 index 0000000..e69de29