feat: update chunked upload endpoints and enhance upload resilience with improved logging and HMAC validation

NETWORK_SWITCHING_IMPROVEMENTS.md

@@ -55,9 +55,9 @@ The network resilience features have been successfully implemented and are ready
 
 | Endpoint | Method | Purpose |
 |----------|--------|---------|
-| `/upload/chunked` | POST | Start new chunked upload session |
+| `/chunked-upload` | POST | Start new chunked upload session |
-| `/upload/chunked` | PUT | Upload individual chunks |
+| `/chunked-upload` | PUT | Upload individual chunks |
-| `/upload/status` | GET | Check upload progress |
+| `/upload-status` | GET | Check upload progress |
 | `/upload` | POST | Traditional uploads (unchanged) |
 
 ## 📱 Network Switching Benefits
@@ -89,17 +89,17 @@ curl -X POST \
   -H "X-Filename: large_video.mp4" \
   -H "X-Total-Size: 104857600" \
   -H "X-Signature: HMAC" \
-  http://localhost:8080/upload/chunked
+  http://localhost:8080/chunked-upload
 
 # 2. Upload chunks (automatically handles network switches)
 curl -X PUT \
   -H "X-Upload-Session-ID: session_123" \
   -H "X-Chunk-Number: 0" \
   --data-binary @chunk_0.bin \
-  http://localhost:8080/upload/chunked
+  http://localhost:8080/chunked-upload
 
 # 3. Check progress
-curl "http://localhost:8080/upload/status?session_id=session_123"
+curl "http://localhost:8080/upload-status?session_id=session_123"
 ```
 
 ## ⚙️ Configuration

cmd/server/chunked_upload_handler.go

@@ -30,6 +30,25 @@ func handleChunkedUpload(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// BASIC HMAC VALIDATION - same as original handleUpload
+	if conf.Security.EnableJWT {
+		_, err := validateJWTFromRequest(r, conf.Security.JWTSecret)
+		if err != nil {
+			http.Error(w, fmt.Sprintf("JWT Authentication failed: %v", err), http.StatusUnauthorized)
+			uploadErrorsTotal.Inc()
+			return
+		}
+		log.Debugf("JWT authentication successful for chunked upload: %s", r.URL.Path)
+	} else {
+		err := validateHMAC(r, conf.Security.Secret)
+		if err != nil {
+			http.Error(w, fmt.Sprintf("HMAC Authentication failed: %v", err), http.StatusUnauthorized)
+			uploadErrorsTotal.Inc()
+			return
+		}
+		log.Debugf("HMAC authentication successful for chunked upload: %s", r.URL.Path)
+	}
+
 	// Extract headers for chunked upload
 	sessionID := r.Header.Get("X-Upload-Session-ID")
 	chunkNumberStr := r.Header.Get("X-Chunk-Number")
@@ -122,8 +141,10 @@ func handleChunkedUpload(w http.ResponseWriter, r *http.Request) {
 	defer chunkFile.Close()
 
 	// Copy chunk data with progress tracking
+	log.Printf("DEBUG: Processing chunk %d for session %s (content-length: %d)", chunkNumber, sessionID, r.ContentLength)
 	written, err := copyChunkWithResilience(chunkFile, r.Body, r.ContentLength, sessionID, chunkNumber)
 	if err != nil {
+		log.Printf("ERROR: Failed to save chunk %d for session %s: %v", chunkNumber, sessionID, err)
 		http.Error(w, fmt.Sprintf("Error saving chunk: %v", err), http.StatusInternalServerError)
 		uploadErrorsTotal.Inc()
 		os.Remove(chunkPath) // Clean up failed chunk
@@ -138,15 +159,32 @@ func handleChunkedUpload(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Get updated session for completion check
+	session, _ = uploadSessionStore.GetSession(sessionID)
+	progress := float64(session.UploadedBytes) / float64(session.TotalSize)
+	
+	// Debug logging for large files
+	if session.TotalSize > 50*1024*1024 { // Log for files > 50MB
+		log.Debugf("Chunk %d uploaded for %s: %d/%d bytes (%.1f%%)", 
+			chunkNumber, session.Filename, session.UploadedBytes, session.TotalSize, progress*100)
+	}
+
 	// Check if upload is complete
-	if uploadSessionStore.IsSessionComplete(sessionID) {
+	isComplete := uploadSessionStore.IsSessionComplete(sessionID)
+	log.Printf("DEBUG: Session %s completion check: %v (uploaded: %d, total: %d, progress: %.1f%%)", 
+		sessionID, isComplete, session.UploadedBytes, session.TotalSize, progress*100)
+	
+	if isComplete {
+		log.Printf("DEBUG: Starting file assembly for session %s", sessionID)
 		// Assemble final file
 		finalPath, err := uploadSessionStore.AssembleFile(sessionID)
 		if err != nil {
+			log.Printf("ERROR: File assembly failed for session %s: %v", sessionID, err)
 			http.Error(w, fmt.Sprintf("Error assembling file: %v", err), http.StatusInternalServerError)
 			uploadErrorsTotal.Inc()
 			return
 		}
+		log.Printf("DEBUG: File assembly completed for session %s: %s", sessionID, finalPath)
 
 		// Handle deduplication if enabled (reuse existing logic)
 		if conf.Server.DeduplicationEnabled {

cmd/server/helpers.go

@@ -603,11 +603,6 @@ func setupRouter() *http.ServeMux {
 
 	log.Info("HTTP router configured successfully with full protocol support (v, v2, token, v3)")
 	
-	// Enhance router with network resilience features (non-intrusive)
-	if conf.Uploads.ChunkedUploadsEnabled {
-		EnhanceExistingRouter(mux)
-	}
-	
 	return mux
 }
 

cmd/server/integration.go

@@ -8,6 +8,21 @@ import (
 	"time"
 )
 
+// Global flag to prevent duplicate route registration
+var routesEnhanced bool
+
+// InitializeEnhancements initializes all new features and enhances the router
+func InitializeEnhancements(router *http.ServeMux) {
+	// Initialize upload resilience system
+	InitializeUploadResilience()
+	
+	// Enhance the existing router with new endpoints (only once)
+	if !routesEnhanced {
+		EnhanceExistingRouter(router)
+		routesEnhanced = true
+	}
+}
+
 // InitializeUploadResilience initializes the upload resilience system
 func InitializeUploadResilience() {
 	// Initialize upload session store
@@ -22,16 +37,13 @@ func InitializeUploadResilience() {
 
 // EnhanceExistingRouter adds new routes without modifying existing setupRouter function
 func EnhanceExistingRouter(mux *http.ServeMux) {
-	// Add chunked upload endpoints
+	// BASIC FUNCTION: Add chunked upload endpoints directly without wrappers
-	mux.HandleFunc("/upload/chunked", ResilientHTTPHandler(handleChunkedUpload, networkManager))
+	mux.HandleFunc("/chunked-upload", handleChunkedUpload)
-	mux.HandleFunc("/upload/status", handleUploadStatus)
+	mux.HandleFunc("/upload-status", handleUploadStatus)
 	
-	// Wrap existing upload handlers with resilience (optional)
+	log.Info("Enhanced upload endpoints added:")
-	if conf.Uploads.ChunkedUploadsEnabled {
+	log.Info("  POST/PUT /chunked-upload - Chunked/resumable uploads")
-		log.Info("Enhanced upload endpoints added:")
+	log.Info("  GET /upload-status - Upload status check")
-		log.Info("  POST/PUT /upload/chunked - Chunked/resumable uploads")
-		log.Info("  GET /upload/status - Upload status check")
-	}
 }
 
 // UpdateConfigurationDefaults suggests better defaults without forcing changes
@@ -116,19 +128,3 @@ func GetResilienceStatus() map[string]interface{} {
 	
 	return status
 }
-
-// Non-intrusive initialization function to be called from main()
-func InitializeEnhancements() {
-	// Only initialize if chunked uploads are enabled
-	if conf.Uploads.ChunkedUploadsEnabled {
-		InitializeUploadResilience()
-		
-		// Start performance monitoring
-		go MonitorUploadPerformance()
-		
-		// Log configuration recommendations
-		UpdateConfigurationDefaults()
-	} else {
-		log.Info("Chunked uploads disabled. Enable 'chunkeduploadsenabled = true' for network resilience features")
-	}
-}

cmd/server/main.go

@@ -698,6 +698,9 @@ func main() {
 
 	router := setupRouter() // Assuming setupRouter is defined (likely in this file or router.go
 
+	// Initialize enhancements and enhance the router
+	InitializeEnhancements(router)
+
 	go handleFileCleanup(&conf) // Directly call handleFileCleanup
 
 	readTimeout, err := time.ParseDuration(conf.Timeouts.Read) // Corrected field name
@@ -766,9 +769,6 @@ func main() {
 	}
 	log.Infof("Running version: %s", versionString)
 
-	// Initialize network resilience features (non-intrusive)
-	InitializeEnhancements()
-
 	log.Infof("Starting HMAC file server %s...", versionString)
 	if conf.Server.UnixSocket {
 		socketPath := "/tmp/hmac-file-server.sock" // Use a default socket path since ListenAddress is now a port

config-network-resilience.toml

@@ -6,7 +6,7 @@ bind_ip = "0.0.0.0"
 listenport = "8080"
 unixsocket = false
 storagepath = "./uploads"
-metricsenabled = true
+metricsenabled = false
 metricsport = "9090"
 deduplicationenabled = true
 networkevents = true              # Enable network change detection

debug_upload.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+
+# Simple test to debug the 49% upload stop issue
+
+set -e
+
+echo "[DEBUG-TEST] Starting server..."
+./hmac-file-server --config config-network-resilience.toml > debug_server.log 2>&1 &
+SERVER_PID=$!
+
+# Wait for server to start
+sleep 3
+
+# Check if server is running
+if ! kill -0 $SERVER_PID 2>/dev/null; then
+    echo "[ERROR] Server failed to start"
+    cat debug_server.log
+    exit 1
+fi
+
+cleanup() {
+    echo "[DEBUG-TEST] Cleaning up..."
+    kill $SERVER_PID 2>/dev/null || true
+    rm -f debug_server.log
+}
+
+trap cleanup EXIT
+
+echo "[DEBUG-TEST] Testing 50MB chunked upload..."
+
+# Calculate HMAC signature
+SECRET="your-super-secret-hmac-key-minimum-32-characters-long"
+MESSAGE="/chunked-upload"
+SIGNATURE=$(echo -n "$MESSAGE" | openssl dgst -sha256 -hmac "$SECRET" | cut -d' ' -f2)
+
+# Start session
+echo "[DEBUG-TEST] Creating session..."
+SESSION_RESPONSE=$(curl -s -X POST \
+    -H "X-Filename: test_50mb.bin" \
+    -H "X-Total-Size: 52428800" \
+    -H "X-Signature: $SIGNATURE" \
+    http://localhost:8080/chunked-upload)
+
+echo "[DEBUG-TEST] Session response: $SESSION_RESPONSE"
+
+SESSION_ID=$(echo "$SESSION_RESPONSE" | grep -o '"session_id":"[^"]*"' | cut -d'"' -f4)
+if [ -z "$SESSION_ID" ]; then
+    echo "[ERROR] Failed to get session ID"
+    exit 1
+fi
+
+echo "[DEBUG-TEST] Session ID: $SESSION_ID"
+
+# Upload first few chunks to see what happens
+CHUNK_SIZE=5242880  # 5MB
+for i in {0..12}; do  # Upload first 13 chunks (65MB worth, should trigger completion)
+    OFFSET=$((i * CHUNK_SIZE))
+    
+    echo "[DEBUG-TEST] Creating chunk $i..."
+    dd if=test_50mb.bin of=chunk_$i.bin bs=$CHUNK_SIZE skip=$i count=1 2>/dev/null || {
+        # Handle the last chunk
+        REMAINING=$((52428800 - OFFSET))
+        if [ $REMAINING -gt 0 ]; then
+            dd if=test_50mb.bin of=chunk_$i.bin bs=1 skip=$OFFSET count=$REMAINING 2>/dev/null
+        else
+            echo "[DEBUG-TEST] No more data for chunk $i"
+            break
+        fi
+    }
+    
+    CHUNK_SIZE_ACTUAL=$(stat -f%z chunk_$i.bin 2>/dev/null || stat -c%s chunk_$i.bin 2>/dev/null)
+    echo "[DEBUG-TEST] Uploading chunk $i (size: $CHUNK_SIZE_ACTUAL bytes)..."
+    
+    UPLOAD_RESPONSE=$(curl -s -w "\n%{http_code}" -X PUT \
+        -H "X-Upload-Session-ID: $SESSION_ID" \
+        -H "X-Chunk-Number: $i" \
+        --data-binary @chunk_$i.bin \
+        http://localhost:8080/chunked-upload)
+    
+    echo "[DEBUG-TEST] Upload response for chunk $i:"
+    echo "$UPLOAD_RESPONSE"
+    echo "---"
+    
+    # Check server logs for debug output
+    echo "[DEBUG-TEST] Recent server logs:"
+    tail -5 debug_server.log
+    echo "---"
+    
+    # Check if complete
+    if echo "$UPLOAD_RESPONSE" | grep -q '"complete":true'; then
+        echo "[DEBUG-TEST] ✅ Upload completed at chunk $i"
+        rm -f chunk_*.bin
+        exit 0
+    fi
+    
+    rm -f chunk_$i.bin
+    sleep 1
+done
+
+echo "[DEBUG-TEST] Upload did not complete. Checking status..."
+STATUS_RESPONSE=$(curl -s "http://localhost:8080/upload-status?session_id=$SESSION_ID")
+echo "[DEBUG-TEST] Final status: $STATUS_RESPONSE"
+
+echo "[DEBUG-TEST] Full server logs:"
+cat debug_server.log

dedup_store/ae6caefbde98cd081e54e846bbf0913031ff0c4137a48f6e0b438cd3f6c8dc09/test_1mb.txt

@@ -0,0 +1 @@
+Hello, HMAC File Server! Do 17. Jul 18:59:11 CEST 2025

dedup_store/ae6caefbde98cd081e54e846bbf0913031ff0c4137a48f6e0b438cd3f6c8dc09/test_4gb.txt

@@ -0,0 +1 @@
+Hello, HMAC File Server! Do 17. Jul 18:59:11 CEST 2025

hmac-file-server.pid

@@ -1 +1 @@
-566111
+619742

test_1mb.txt

@@ -0,0 +1 @@
+Hello, HMAC File Server! Do 17. Jul 18:59:11 CEST 2025

test_4gb.txt

@@ -0,0 +1 @@
+Hello, HMAC File Server! Do 17. Jul 18:59:11 CEST 2025

test_upload.txt

@@ -1 +1 @@
-Hello, HMAC File Server! Do 17. Jul 18:18:58 CEST 2025
+Hello, HMAC File Server! Do 17. Jul 18:59:11 CEST 2025