diff --git a/NETWORK_SWITCHING_IMPROVEMENTS.md b/NETWORK_SWITCHING_IMPROVEMENTS.md index 9876985..15b4978 100644 --- a/NETWORK_SWITCHING_IMPROVEMENTS.md +++ b/NETWORK_SWITCHING_IMPROVEMENTS.md @@ -55,9 +55,9 @@ The network resilience features have been successfully implemented and are ready | Endpoint | Method | Purpose | |----------|--------|---------| -| `/upload/chunked` | POST | Start new chunked upload session | -| `/upload/chunked` | PUT | Upload individual chunks | -| `/upload/status` | GET | Check upload progress | +| `/chunked-upload` | POST | Start new chunked upload session | +| `/chunked-upload` | PUT | Upload individual chunks | +| `/upload-status` | GET | Check upload progress | | `/upload` | POST | Traditional uploads (unchanged) | ## 📱 Network Switching Benefits @@ -89,17 +89,17 @@ curl -X POST \ -H "X-Filename: large_video.mp4" \ -H "X-Total-Size: 104857600" \ -H "X-Signature: HMAC" \ - http://localhost:8080/upload/chunked + http://localhost:8080/chunked-upload # 2. Upload chunks (automatically handles network switches) curl -X PUT \ -H "X-Upload-Session-ID: session_123" \ -H "X-Chunk-Number: 0" \ --data-binary @chunk_0.bin \ - http://localhost:8080/upload/chunked + http://localhost:8080/chunked-upload # 3. Check progress -curl "http://localhost:8080/upload/status?session_id=session_123" +curl "http://localhost:8080/upload-status?session_id=session_123" ``` ## ⚙️ Configuration diff --git a/chunk_0.bin b/chunk_0.bin new file mode 100644 index 0000000..ee149f2 Binary files /dev/null and b/chunk_0.bin differ diff --git a/cmd/server/chunked_upload_handler.go b/cmd/server/chunked_upload_handler.go index 9dc70b3..81fdd41 100644 --- a/cmd/server/chunked_upload_handler.go +++ b/cmd/server/chunked_upload_handler.go @@ -30,6 +30,25 @@ func handleChunkedUpload(w http.ResponseWriter, r *http.Request) { return } + // BASIC HMAC VALIDATION - same as original handleUpload + if conf.Security.EnableJWT { + _, err := validateJWTFromRequest(r, conf.Security.JWTSecret) + if err != nil { + http.Error(w, fmt.Sprintf("JWT Authentication failed: %v", err), http.StatusUnauthorized) + uploadErrorsTotal.Inc() + return + } + log.Debugf("JWT authentication successful for chunked upload: %s", r.URL.Path) + } else { + err := validateHMAC(r, conf.Security.Secret) + if err != nil { + http.Error(w, fmt.Sprintf("HMAC Authentication failed: %v", err), http.StatusUnauthorized) + uploadErrorsTotal.Inc() + return + } + log.Debugf("HMAC authentication successful for chunked upload: %s", r.URL.Path) + } + // Extract headers for chunked upload sessionID := r.Header.Get("X-Upload-Session-ID") chunkNumberStr := r.Header.Get("X-Chunk-Number") @@ -122,8 +141,10 @@ func handleChunkedUpload(w http.ResponseWriter, r *http.Request) { defer chunkFile.Close() // Copy chunk data with progress tracking + log.Printf("DEBUG: Processing chunk %d for session %s (content-length: %d)", chunkNumber, sessionID, r.ContentLength) written, err := copyChunkWithResilience(chunkFile, r.Body, r.ContentLength, sessionID, chunkNumber) if err != nil { + log.Printf("ERROR: Failed to save chunk %d for session %s: %v", chunkNumber, sessionID, err) http.Error(w, fmt.Sprintf("Error saving chunk: %v", err), http.StatusInternalServerError) uploadErrorsTotal.Inc() os.Remove(chunkPath) // Clean up failed chunk @@ -138,15 +159,32 @@ func handleChunkedUpload(w http.ResponseWriter, r *http.Request) { return } + // Get updated session for completion check + session, _ = uploadSessionStore.GetSession(sessionID) + progress := float64(session.UploadedBytes) / float64(session.TotalSize) + + // Debug logging for large files + if session.TotalSize > 50*1024*1024 { // Log for files > 50MB + log.Debugf("Chunk %d uploaded for %s: %d/%d bytes (%.1f%%)", + chunkNumber, session.Filename, session.UploadedBytes, session.TotalSize, progress*100) + } + // Check if upload is complete - if uploadSessionStore.IsSessionComplete(sessionID) { + isComplete := uploadSessionStore.IsSessionComplete(sessionID) + log.Printf("DEBUG: Session %s completion check: %v (uploaded: %d, total: %d, progress: %.1f%%)", + sessionID, isComplete, session.UploadedBytes, session.TotalSize, progress*100) + + if isComplete { + log.Printf("DEBUG: Starting file assembly for session %s", sessionID) // Assemble final file finalPath, err := uploadSessionStore.AssembleFile(sessionID) if err != nil { + log.Printf("ERROR: File assembly failed for session %s: %v", sessionID, err) http.Error(w, fmt.Sprintf("Error assembling file: %v", err), http.StatusInternalServerError) uploadErrorsTotal.Inc() return } + log.Printf("DEBUG: File assembly completed for session %s: %s", sessionID, finalPath) // Handle deduplication if enabled (reuse existing logic) if conf.Server.DeduplicationEnabled { diff --git a/cmd/server/helpers.go b/cmd/server/helpers.go index a92a0b3..f9e90dc 100644 --- a/cmd/server/helpers.go +++ b/cmd/server/helpers.go @@ -603,11 +603,6 @@ func setupRouter() *http.ServeMux { log.Info("HTTP router configured successfully with full protocol support (v, v2, token, v3)") - // Enhance router with network resilience features (non-intrusive) - if conf.Uploads.ChunkedUploadsEnabled { - EnhanceExistingRouter(mux) - } - return mux } diff --git a/cmd/server/integration.go b/cmd/server/integration.go index 4e8b86c..555419f 100644 --- a/cmd/server/integration.go +++ b/cmd/server/integration.go @@ -8,6 +8,21 @@ import ( "time" ) +// Global flag to prevent duplicate route registration +var routesEnhanced bool + +// InitializeEnhancements initializes all new features and enhances the router +func InitializeEnhancements(router *http.ServeMux) { + // Initialize upload resilience system + InitializeUploadResilience() + + // Enhance the existing router with new endpoints (only once) + if !routesEnhanced { + EnhanceExistingRouter(router) + routesEnhanced = true + } +} + // InitializeUploadResilience initializes the upload resilience system func InitializeUploadResilience() { // Initialize upload session store @@ -22,16 +37,13 @@ func InitializeUploadResilience() { // EnhanceExistingRouter adds new routes without modifying existing setupRouter function func EnhanceExistingRouter(mux *http.ServeMux) { - // Add chunked upload endpoints - mux.HandleFunc("/upload/chunked", ResilientHTTPHandler(handleChunkedUpload, networkManager)) - mux.HandleFunc("/upload/status", handleUploadStatus) + // BASIC FUNCTION: Add chunked upload endpoints directly without wrappers + mux.HandleFunc("/chunked-upload", handleChunkedUpload) + mux.HandleFunc("/upload-status", handleUploadStatus) - // Wrap existing upload handlers with resilience (optional) - if conf.Uploads.ChunkedUploadsEnabled { - log.Info("Enhanced upload endpoints added:") - log.Info(" POST/PUT /upload/chunked - Chunked/resumable uploads") - log.Info(" GET /upload/status - Upload status check") - } + log.Info("Enhanced upload endpoints added:") + log.Info(" POST/PUT /chunked-upload - Chunked/resumable uploads") + log.Info(" GET /upload-status - Upload status check") } // UpdateConfigurationDefaults suggests better defaults without forcing changes @@ -116,19 +128,3 @@ func GetResilienceStatus() map[string]interface{} { return status } - -// Non-intrusive initialization function to be called from main() -func InitializeEnhancements() { - // Only initialize if chunked uploads are enabled - if conf.Uploads.ChunkedUploadsEnabled { - InitializeUploadResilience() - - // Start performance monitoring - go MonitorUploadPerformance() - - // Log configuration recommendations - UpdateConfigurationDefaults() - } else { - log.Info("Chunked uploads disabled. Enable 'chunkeduploadsenabled = true' for network resilience features") - } -} diff --git a/cmd/server/main.go b/cmd/server/main.go index 7ee6373..07f1be5 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -698,6 +698,9 @@ func main() { router := setupRouter() // Assuming setupRouter is defined (likely in this file or router.go + // Initialize enhancements and enhance the router + InitializeEnhancements(router) + go handleFileCleanup(&conf) // Directly call handleFileCleanup readTimeout, err := time.ParseDuration(conf.Timeouts.Read) // Corrected field name @@ -766,9 +769,6 @@ func main() { } log.Infof("Running version: %s", versionString) - // Initialize network resilience features (non-intrusive) - InitializeEnhancements() - log.Infof("Starting HMAC file server %s...", versionString) if conf.Server.UnixSocket { socketPath := "/tmp/hmac-file-server.sock" // Use a default socket path since ListenAddress is now a port diff --git a/config-network-resilience.toml b/config-network-resilience.toml index a11c1a8..8cc8ddd 100644 --- a/config-network-resilience.toml +++ b/config-network-resilience.toml @@ -6,7 +6,7 @@ bind_ip = "0.0.0.0" listenport = "8080" unixsocket = false storagepath = "./uploads" -metricsenabled = true +metricsenabled = false metricsport = "9090" deduplicationenabled = true networkevents = true # Enable network change detection diff --git a/debug_upload.sh b/debug_upload.sh new file mode 100755 index 0000000..03ce0bb --- /dev/null +++ b/debug_upload.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +# Simple test to debug the 49% upload stop issue + +set -e + +echo "[DEBUG-TEST] Starting server..." +./hmac-file-server --config config-network-resilience.toml > debug_server.log 2>&1 & +SERVER_PID=$! + +# Wait for server to start +sleep 3 + +# Check if server is running +if ! kill -0 $SERVER_PID 2>/dev/null; then + echo "[ERROR] Server failed to start" + cat debug_server.log + exit 1 +fi + +cleanup() { + echo "[DEBUG-TEST] Cleaning up..." + kill $SERVER_PID 2>/dev/null || true + rm -f debug_server.log +} + +trap cleanup EXIT + +echo "[DEBUG-TEST] Testing 50MB chunked upload..." + +# Calculate HMAC signature +SECRET="your-super-secret-hmac-key-minimum-32-characters-long" +MESSAGE="/chunked-upload" +SIGNATURE=$(echo -n "$MESSAGE" | openssl dgst -sha256 -hmac "$SECRET" | cut -d' ' -f2) + +# Start session +echo "[DEBUG-TEST] Creating session..." +SESSION_RESPONSE=$(curl -s -X POST \ + -H "X-Filename: test_50mb.bin" \ + -H "X-Total-Size: 52428800" \ + -H "X-Signature: $SIGNATURE" \ + http://localhost:8080/chunked-upload) + +echo "[DEBUG-TEST] Session response: $SESSION_RESPONSE" + +SESSION_ID=$(echo "$SESSION_RESPONSE" | grep -o '"session_id":"[^"]*"' | cut -d'"' -f4) +if [ -z "$SESSION_ID" ]; then + echo "[ERROR] Failed to get session ID" + exit 1 +fi + +echo "[DEBUG-TEST] Session ID: $SESSION_ID" + +# Upload first few chunks to see what happens +CHUNK_SIZE=5242880 # 5MB +for i in {0..12}; do # Upload first 13 chunks (65MB worth, should trigger completion) + OFFSET=$((i * CHUNK_SIZE)) + + echo "[DEBUG-TEST] Creating chunk $i..." + dd if=test_50mb.bin of=chunk_$i.bin bs=$CHUNK_SIZE skip=$i count=1 2>/dev/null || { + # Handle the last chunk + REMAINING=$((52428800 - OFFSET)) + if [ $REMAINING -gt 0 ]; then + dd if=test_50mb.bin of=chunk_$i.bin bs=1 skip=$OFFSET count=$REMAINING 2>/dev/null + else + echo "[DEBUG-TEST] No more data for chunk $i" + break + fi + } + + CHUNK_SIZE_ACTUAL=$(stat -f%z chunk_$i.bin 2>/dev/null || stat -c%s chunk_$i.bin 2>/dev/null) + echo "[DEBUG-TEST] Uploading chunk $i (size: $CHUNK_SIZE_ACTUAL bytes)..." + + UPLOAD_RESPONSE=$(curl -s -w "\n%{http_code}" -X PUT \ + -H "X-Upload-Session-ID: $SESSION_ID" \ + -H "X-Chunk-Number: $i" \ + --data-binary @chunk_$i.bin \ + http://localhost:8080/chunked-upload) + + echo "[DEBUG-TEST] Upload response for chunk $i:" + echo "$UPLOAD_RESPONSE" + echo "---" + + # Check server logs for debug output + echo "[DEBUG-TEST] Recent server logs:" + tail -5 debug_server.log + echo "---" + + # Check if complete + if echo "$UPLOAD_RESPONSE" | grep -q '"complete":true'; then + echo "[DEBUG-TEST] ✅ Upload completed at chunk $i" + rm -f chunk_*.bin + exit 0 + fi + + rm -f chunk_$i.bin + sleep 1 +done + +echo "[DEBUG-TEST] Upload did not complete. Checking status..." +STATUS_RESPONSE=$(curl -s "http://localhost:8080/upload-status?session_id=$SESSION_ID") +echo "[DEBUG-TEST] Final status: $STATUS_RESPONSE" + +echo "[DEBUG-TEST] Full server logs:" +cat debug_server.log diff --git a/dedup_store/ae6caefbde98cd081e54e846bbf0913031ff0c4137a48f6e0b438cd3f6c8dc09/test_1mb.txt b/dedup_store/ae6caefbde98cd081e54e846bbf0913031ff0c4137a48f6e0b438cd3f6c8dc09/test_1mb.txt new file mode 100644 index 0000000..a2e74b5 --- /dev/null +++ b/dedup_store/ae6caefbde98cd081e54e846bbf0913031ff0c4137a48f6e0b438cd3f6c8dc09/test_1mb.txt @@ -0,0 +1 @@ +Hello, HMAC File Server! Do 17. Jul 18:59:11 CEST 2025 diff --git a/dedup_store/ae6caefbde98cd081e54e846bbf0913031ff0c4137a48f6e0b438cd3f6c8dc09/test_4gb.txt b/dedup_store/ae6caefbde98cd081e54e846bbf0913031ff0c4137a48f6e0b438cd3f6c8dc09/test_4gb.txt new file mode 100644 index 0000000..a2e74b5 --- /dev/null +++ b/dedup_store/ae6caefbde98cd081e54e846bbf0913031ff0c4137a48f6e0b438cd3f6c8dc09/test_4gb.txt @@ -0,0 +1 @@ +Hello, HMAC File Server! Do 17. Jul 18:59:11 CEST 2025 diff --git a/hmac-file-server.pid b/hmac-file-server.pid index 4a0b2cf..a48cb04 100644 --- a/hmac-file-server.pid +++ b/hmac-file-server.pid @@ -1 +1 @@ -566111 \ No newline at end of file +619742 \ No newline at end of file diff --git a/test_1mb.bin b/test_1mb.bin new file mode 100644 index 0000000..9e0f96a Binary files /dev/null and b/test_1mb.bin differ diff --git a/test_1mb.txt b/test_1mb.txt new file mode 100644 index 0000000..a2e74b5 --- /dev/null +++ b/test_1mb.txt @@ -0,0 +1 @@ +Hello, HMAC File Server! Do 17. Jul 18:59:11 CEST 2025 diff --git a/test_215mb.bin b/test_215mb.bin new file mode 100644 index 0000000..0c8322c Binary files /dev/null and b/test_215mb.bin differ diff --git a/test_4gb.bin b/test_4gb.bin new file mode 100644 index 0000000..451971a Binary files /dev/null and b/test_4gb.bin differ diff --git a/test_4gb.txt b/test_4gb.txt new file mode 100644 index 0000000..a2e74b5 --- /dev/null +++ b/test_4gb.txt @@ -0,0 +1 @@ +Hello, HMAC File Server! Do 17. Jul 18:59:11 CEST 2025 diff --git a/test_50mb.bin b/test_50mb.bin new file mode 100644 index 0000000..514783a Binary files /dev/null and b/test_50mb.bin differ diff --git a/test_upload.txt b/test_upload.txt index b884635..a2e74b5 100644 --- a/test_upload.txt +++ b/test_upload.txt @@ -1 +1 @@ -Hello, HMAC File Server! Do 17. Jul 18:18:58 CEST 2025 +Hello, HMAC File Server! Do 17. Jul 18:59:11 CEST 2025