diff --git a/CLAMAV_LARGE_FILE_FIX.md b/CLAMAV_LARGE_FILE_FIX.md new file mode 100644 index 0000000..e69de29 diff --git a/CLAMAV_SCANNING_FIX.md b/CLAMAV_SCANNING_FIX.md new file mode 100644 index 0000000..e69de29 diff --git a/CLAMAV_SECURITY_CONFIG.md b/CLAMAV_SECURITY_CONFIG.md new file mode 100644 index 0000000..e69de29 diff --git a/CLUSTER_RESTART_GUIDE.md b/CLUSTER_RESTART_GUIDE.md new file mode 100644 index 0000000..e69de29 diff --git a/DEDUPLICATION_1GB_OPTIMIZATION.md b/DEDUPLICATION_1GB_OPTIMIZATION.md new file mode 100644 index 0000000..e69de29 diff --git a/DOCUMENTATION_UPDATE_SUMMARY.md b/DOCUMENTATION_UPDATE_SUMMARY.md new file mode 100644 index 0000000..e69de29 diff --git a/ENDLESS_ENCRYPTION_FIX.md b/ENDLESS_ENCRYPTION_FIX.md new file mode 100644 index 0000000..e69de29 diff --git a/FINAL_STATUS_REPORT.md b/FINAL_STATUS_REPORT.md new file mode 100644 index 0000000..e69de29 diff --git a/INSTALL.MD b/INSTALL.MD new file mode 100644 index 0000000..e69de29 diff --git a/LARGE_FILE_UPLOAD_FIX.md b/LARGE_FILE_UPLOAD_FIX.md new file mode 100644 index 0000000..e69de29 diff --git a/NETWORK_RESILIENCE_GUIDE.md b/NETWORK_RESILIENCE_GUIDE.md new file mode 100644 index 0000000..e69de29 diff --git a/PERFORMANCE_OPTIMIZATION.md b/PERFORMANCE_OPTIMIZATION.md new file mode 100644 index 0000000..e69de29 diff --git a/QUEUE_RESILIENCE_GUIDE.md b/QUEUE_RESILIENCE_GUIDE.md new file mode 100644 index 0000000..e69de29 diff --git a/QUEUE_RESILIENCE_SUMMARY.md b/QUEUE_RESILIENCE_SUMMARY.md new file mode 100644 index 0000000..e69de29 diff --git a/README.MD b/README.MD new file mode 100644 index 0000000..e69de29 diff --git a/UNIVERSAL_LARGE_UPLOAD_FIX.md b/UNIVERSAL_LARGE_UPLOAD_FIX.md new file mode 100644 index 0000000..e69de29 diff --git a/UPLOAD_COMPLETION_FIX.md b/UPLOAD_COMPLETION_FIX.md new file mode 100644 index 0000000..e69de29 diff --git a/UPLOAD_PERFORMANCE_VERIFICATION.md b/UPLOAD_PERFORMANCE_VERIFICATION.md new file mode 100644 index 0000000..e69de29 diff --git a/VIDEO_EXTENSION_FIX.md b/VIDEO_EXTENSION_FIX.md new file mode 100644 index 0000000..e69de29 diff --git a/cmd/server/chunked_upload_handler.go b/cmd/server/chunked_upload_handler.go index 9f0389b..dac2922 100644 --- a/cmd/server/chunked_upload_handler.go +++ b/cmd/server/chunked_upload_handler.go @@ -105,6 +105,34 @@ func handleChunkedUpload(w http.ResponseWriter, r *http.Request) { } } + // Pre-upload deduplication check for chunked uploads + if conf.Server.DeduplicationEnabled { + finalPath := filepath.Join(conf.Server.StoragePath, filename) + if existingFileInfo, err := os.Stat(finalPath); err == nil { + // File already exists - return success immediately for deduplication hit + duration := time.Since(startTime) + uploadDuration.Observe(duration.Seconds()) + uploadsTotal.Inc() + uploadSizeBytes.Observe(float64(existingFileInfo.Size())) + filesDeduplicatedTotal.Inc() + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + response := map[string]interface{}{ + "success": true, + "filename": filename, + "size": existingFileInfo.Size(), + "completed": true, + "message": "File already exists (deduplication hit)", + } + writeJSONResponse(w, response) + + log.Infof("Chunked upload deduplication hit: file %s already exists (%s), returning success immediately", + filename, formatBytes(existingFileInfo.Size())) + return + } + } + // Create new session clientIP := getClientIP(r) session := uploadSessionStore.CreateSession(filename, totalSize, clientIP) diff --git a/cmd/server/main.go b/cmd/server/main.go index 818d42f..4645bed 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -1515,6 +1515,32 @@ func handleUpload(w http.ResponseWriter, r *http.Request) { absFilename := filepath.Join(storagePath, filename) + // Pre-upload deduplication check: if file already exists and deduplication is enabled, return success immediately + if conf.Server.DeduplicationEnabled { + if existingFileInfo, err := os.Stat(absFilename); err == nil { + // File already exists - return success immediately for deduplication hit + duration := time.Since(startTime) + uploadDuration.Observe(duration.Seconds()) + uploadsTotal.Inc() + uploadSizeBytes.Observe(float64(existingFileInfo.Size())) + filesDeduplicatedTotal.Inc() + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + response := map[string]interface{}{ + "success": true, + "filename": filename, + "size": existingFileInfo.Size(), + "message": "File already exists (deduplication hit)", + } + json.NewEncoder(w).Encode(response) + + log.Infof("Deduplication hit: file %s already exists (%s), returning success immediately", + filename, formatBytes(existingFileInfo.Size())) + return + } + } + // Create the file dst, err := os.Create(absFilename) if err != nil { @@ -1752,6 +1778,32 @@ func handleV3Upload(w http.ResponseWriter, r *http.Request) { absFilename := filepath.Join(storagePath, filename) + // Pre-upload deduplication check: if file already exists and deduplication is enabled, return success immediately + if conf.Server.DeduplicationEnabled { + if existingFileInfo, err := os.Stat(absFilename); err == nil { + // File already exists - return success immediately for deduplication hit + duration := time.Since(startTime) + uploadDuration.Observe(duration.Seconds()) + uploadsTotal.Inc() + uploadSizeBytes.Observe(float64(existingFileInfo.Size())) + filesDeduplicatedTotal.Inc() + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + response := map[string]interface{}{ + "success": true, + "filename": filename, + "size": existingFileInfo.Size(), + "message": "File already exists (deduplication hit)", + } + json.NewEncoder(w).Encode(response) + + log.Infof("Deduplication hit: file %s already exists (%s), returning success immediately", + filename, formatBytes(existingFileInfo.Size())) + return + } + } + // Create the file dst, err := os.Create(absFilename) if err != nil { @@ -1907,6 +1959,23 @@ func handleLegacyUpload(w http.ResponseWriter, r *http.Request) { return } + // Pre-upload deduplication check: if file already exists and deduplication is enabled, return success immediately + if conf.Server.DeduplicationEnabled { + if existingFileInfo, err := os.Stat(absFilename); err == nil { + // File already exists - return success immediately for deduplication hit + duration := time.Since(startTime) + uploadDuration.Observe(duration.Seconds()) + uploadsTotal.Inc() + uploadSizeBytes.Observe(float64(existingFileInfo.Size())) + filesDeduplicatedTotal.Inc() + + w.WriteHeader(http.StatusCreated) // 201 Created for legacy compatibility + log.Infof("Deduplication hit: file %s already exists (%s), returning success immediately", + filename, formatBytes(existingFileInfo.Size())) + return + } + } + // Create the file dst, err := os.Create(absFilename) if err != nil { diff --git a/cmd/server/queue_resilience.go b/cmd/server/queue_resilience.go index e7105a5..8dab27c 100644 --- a/cmd/server/queue_resilience.go +++ b/cmd/server/queue_resilience.go @@ -5,7 +5,6 @@ package main import ( "context" "errors" - "fmt" "sync" "sync/atomic" "time" diff --git a/config-clean.toml b/config-clean.toml new file mode 100644 index 0000000..e69de29 diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..e69de29 diff --git a/verify_xmpp_upload.sh b/verify_xmpp_upload.sh new file mode 100644 index 0000000..e69de29