docs: update build guide and network switching improvements documentation for enhanced upload resilience
This commit is contained in:
@ -1,35 +1,27 @@
|
|||||||
# Build Guide - HMAC File Server with Network Resilience
|
# Build Guide - HMAC File Server with Network Resilience
|
||||||
|
|
||||||
## Quick Build
|
## ✅ Quick Build (Working)
|
||||||
|
|
||||||
### 1. Standard Build (existing functionality only)
|
### 1. Standard Build with Network Resilience
|
||||||
```bash
|
```bash
|
||||||
# Build with existing features
|
# Build with all features (including network resilience)
|
||||||
./buildgo.sh
|
./buildgo.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
### 2. Build with Network Resilience Features
|
**Output:**
|
||||||
```bash
|
```
|
||||||
# First ensure all network resilience files are present
|
[BUILD] Building HMAC File Server v3.2 with Network Resilience...
|
||||||
ls cmd/server/upload_session.go
|
[INFO] Found network resilience: upload_session.go
|
||||||
ls cmd/server/network_resilience.go
|
[INFO] Found network resilience: network_resilience.go
|
||||||
ls cmd/server/chunked_upload_handler.go
|
[INFO] Found network resilience: chunked_upload_handler.go
|
||||||
ls cmd/server/integration.go
|
[INFO] Found network resilience: integration.go
|
||||||
|
[BUILD] Build successful! Binary created: ./hmac-file-server
|
||||||
# Build with enhanced features
|
[INFO] Binary size: 16M
|
||||||
./buildgo.sh
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### 3. Manual Build (if you prefer)
|
### 2. Manual Build (Alternative)
|
||||||
```bash
|
```bash
|
||||||
# Core build (backward compatible)
|
# Build manually with all network resilience features
|
||||||
go build -o hmac-file-server \
|
|
||||||
cmd/server/main.go \
|
|
||||||
cmd/server/helpers.go \
|
|
||||||
cmd/server/config_validator.go \
|
|
||||||
cmd/server/config_test_scenarios.go
|
|
||||||
|
|
||||||
# With network resilience features
|
|
||||||
go build -o hmac-file-server \
|
go build -o hmac-file-server \
|
||||||
cmd/server/main.go \
|
cmd/server/main.go \
|
||||||
cmd/server/helpers.go \
|
cmd/server/helpers.go \
|
||||||
|
|||||||
@ -1,219 +1,162 @@
|
|||||||
# Network Switching Improvements for HMAC File Server
|
# Network Switching Improvements for HMAC File Server
|
||||||
|
|
||||||
## Issues Identified
|
## ✅ Implementation Complete
|
||||||
|
|
||||||
### 1. No Resumable Upload Support
|
The network resilience features have been successfully implemented and are ready to use!
|
||||||
- Current uploads fail completely on network interruption
|
|
||||||
- No chunked upload implementation despite configuration option
|
|
||||||
- File deletion on any upload error loses all progress
|
|
||||||
|
|
||||||
### 2. Aggressive Connection Timeouts
|
### 🚀 Quick Start
|
||||||
- ReadTimeout/WriteTimeout too short for large uploads over mobile networks
|
|
||||||
- IdleConnTimeout too aggressive for network switching scenarios
|
|
||||||
- No retry mechanisms for temporary network failures
|
|
||||||
|
|
||||||
### 3. No Connection State Management
|
1. **Build the enhanced server:**
|
||||||
- No detection of network changes
|
```bash
|
||||||
- No graceful handling of connection switches
|
./buildgo.sh
|
||||||
- No upload session persistence
|
```
|
||||||
|
|
||||||
## Recommended Improvements
|
2. **Use the network-resilient configuration:**
|
||||||
|
```bash
|
||||||
|
cp config-network-resilience.toml config.toml
|
||||||
|
# Edit config.toml with your settings
|
||||||
|
```
|
||||||
|
|
||||||
### 1. Implement Chunked/Resumable Uploads
|
3. **Start the server:**
|
||||||
|
```bash
|
||||||
|
./hmac-file-server --config config.toml
|
||||||
|
```
|
||||||
|
|
||||||
```go
|
4. **Test the features:**
|
||||||
// Add to upload configuration
|
```bash
|
||||||
type ChunkedUploadSession struct {
|
./test_network_resilience.sh
|
||||||
ID string
|
```
|
||||||
Filename string
|
|
||||||
TotalSize int64
|
|
||||||
ChunkSize int64
|
|
||||||
UploadedBytes int64
|
|
||||||
Chunks map[int]bool // Track completed chunks
|
|
||||||
LastActivity time.Time
|
|
||||||
ClientIP string
|
|
||||||
}
|
|
||||||
|
|
||||||
// New upload handler for chunked uploads
|
## 🔧 What's Been Fixed
|
||||||
func handleChunkedUpload(w http.ResponseWriter, r *http.Request) {
|
|
||||||
// Check for existing session
|
|
||||||
sessionID := r.Header.Get("X-Upload-Session-ID")
|
|
||||||
chunkNumber := r.Header.Get("X-Chunk-Number")
|
|
||||||
|
|
||||||
// Resume logic here
|
### ✅ Implementation Status
|
||||||
}
|
|
||||||
|
### ✅ Implementation Status
|
||||||
|
|
||||||
|
| Feature | Status | Description |
|
||||||
|
|---------|--------|-------------|
|
||||||
|
| **Chunked/Resumable Uploads** | ✅ **IMPLEMENTED** | 5MB chunks, survives network interruptions |
|
||||||
|
| **Network Change Detection** | ✅ **IMPLEMENTED** | Monitors interfaces, pauses/resumes uploads |
|
||||||
|
| **Session Persistence** | ✅ **IMPLEMENTED** | Redis or disk storage, 24h default timeout |
|
||||||
|
| **Enhanced Timeouts** | ✅ **IMPLEMENTED** | 5-minute read/write, 10-minute idle |
|
||||||
|
| **Retry Logic** | ✅ **IMPLEMENTED** | Exponential backoff with jitter |
|
||||||
|
| **Backward Compatibility** | ✅ **GUARANTEED** | Zero changes to existing upload handlers |
|
||||||
|
|
||||||
|
### 📂 New Files Added
|
||||||
|
|
||||||
|
- `cmd/server/upload_session.go` - Session management and persistence
|
||||||
|
- `cmd/server/network_resilience.go` - Network monitoring and pause/resume
|
||||||
|
- `cmd/server/chunked_upload_handler.go` - New chunked upload endpoint
|
||||||
|
- `cmd/server/integration.go` - Non-intrusive integration layer
|
||||||
|
- `config-network-resilience.toml` - Ready-to-use configuration
|
||||||
|
- `test_network_resilience.sh` - Automated testing script
|
||||||
|
|
||||||
|
### 🌐 New API Endpoints
|
||||||
|
|
||||||
|
| Endpoint | Method | Purpose |
|
||||||
|
|----------|--------|---------|
|
||||||
|
| `/upload/chunked` | POST | Start new chunked upload session |
|
||||||
|
| `/upload/chunked` | PUT | Upload individual chunks |
|
||||||
|
| `/upload/status` | GET | Check upload progress |
|
||||||
|
| `/upload` | POST | Traditional uploads (unchanged) |
|
||||||
|
|
||||||
|
## 📱 Network Switching Benefits
|
||||||
|
|
||||||
|
### Before (Problems Fixed)
|
||||||
|
- ❌ Upload fails completely on network interruption
|
||||||
|
- ❌ Progress lost when switching WiFi/WLAN
|
||||||
|
- ❌ Large files problematic on mobile networks
|
||||||
|
- ❌ No recovery from connection drops
|
||||||
|
|
||||||
|
### After (Solutions Implemented)
|
||||||
|
- ✅ **Seamless network switching** - uploads pause and resume automatically
|
||||||
|
- ✅ **Progress preservation** - no lost data during interruptions
|
||||||
|
- ✅ **Mobile optimized** - 5MB chunks perfect for cellular/WiFi
|
||||||
|
- ✅ **Intelligent retry** - exponential backoff handles temporary failures
|
||||||
|
- ✅ **Session persistence** - survives server restarts
|
||||||
|
|
||||||
|
## 📋 Usage Examples
|
||||||
|
|
||||||
|
### Traditional Upload (Unchanged)
|
||||||
|
```bash
|
||||||
|
curl -X POST -H "X-Signature: HMAC" -F 'file=@document.pdf' http://localhost:8080/upload
|
||||||
```
|
```
|
||||||
|
|
||||||
### 2. Enhanced Connection Management
|
### New Chunked Upload
|
||||||
|
```bash
|
||||||
|
# 1. Start session
|
||||||
|
curl -X POST \
|
||||||
|
-H "X-Filename: large_video.mp4" \
|
||||||
|
-H "X-Total-Size: 104857600" \
|
||||||
|
-H "X-Signature: HMAC" \
|
||||||
|
http://localhost:8080/upload/chunked
|
||||||
|
|
||||||
```go
|
# 2. Upload chunks (automatically handles network switches)
|
||||||
// Improved HTTP client configuration
|
curl -X PUT \
|
||||||
dualStackClient = &http.Client{
|
-H "X-Upload-Session-ID: session_123" \
|
||||||
Transport: &http.Transport{
|
-H "X-Chunk-Number: 0" \
|
||||||
DialContext: dialer.DialContext,
|
--data-binary @chunk_0.bin \
|
||||||
IdleConnTimeout: 300 * time.Second, // 5 minutes for mobile
|
http://localhost:8080/upload/chunked
|
||||||
MaxIdleConns: 50,
|
|
||||||
MaxIdleConnsPerHost: 20, // More connections per host
|
|
||||||
TLSHandshakeTimeout: 30 * time.Second, // Longer for mobile networks
|
|
||||||
ResponseHeaderTimeout: 60 * time.Second, // Account for network switches
|
|
||||||
DisableKeepAlives: false, // Enable keep-alives
|
|
||||||
MaxConnsPerHost: 30, // Allow more concurrent connections
|
|
||||||
},
|
|
||||||
Timeout: 0, // No overall timeout - let individual operations timeout
|
|
||||||
}
|
|
||||||
|
|
||||||
// Enhanced server timeouts
|
# 3. Check progress
|
||||||
server := &http.Server{
|
curl "http://localhost:8080/upload/status?session_id=session_123"
|
||||||
ReadTimeout: 5 * time.Minute, // Allow for slow mobile uploads
|
|
||||||
WriteTimeout: 5 * time.Minute, // Allow for slow responses
|
|
||||||
IdleTimeout: 10 * time.Minute, // Keep connections alive longer
|
|
||||||
}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### 3. Network Change Detection and Handling
|
## ⚙️ Configuration
|
||||||
|
|
||||||
```go
|
Essential settings for network resilience:
|
||||||
// Enhanced network monitoring
|
|
||||||
func monitorNetworkChanges(ctx context.Context) {
|
|
||||||
ticker := time.NewTicker(5 * time.Second) // More frequent checking
|
|
||||||
defer ticker.Stop()
|
|
||||||
|
|
||||||
var lastInterfaces []net.Interface
|
|
||||||
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-ticker.C:
|
|
||||||
currentInterfaces, err := net.Interfaces()
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Detect interface changes
|
|
||||||
if hasNetworkChanges(lastInterfaces, currentInterfaces) {
|
|
||||||
log.Info("Network change detected - pausing active uploads")
|
|
||||||
pauseActiveUploads()
|
|
||||||
|
|
||||||
// Wait for network stabilization
|
|
||||||
time.Sleep(2 * time.Second)
|
|
||||||
|
|
||||||
log.Info("Resuming uploads after network change")
|
|
||||||
resumeActiveUploads()
|
|
||||||
}
|
|
||||||
|
|
||||||
lastInterfaces = currentInterfaces
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. Upload Session Persistence
|
|
||||||
|
|
||||||
```go
|
|
||||||
// Store upload sessions in Redis or local cache
|
|
||||||
type UploadSessionStore struct {
|
|
||||||
sessions map[string]*ChunkedUploadSession
|
|
||||||
mutex sync.RWMutex
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *UploadSessionStore) SaveSession(session *ChunkedUploadSession) {
|
|
||||||
s.mutex.Lock()
|
|
||||||
defer s.mutex.Unlock()
|
|
||||||
|
|
||||||
// Store in Redis if available, otherwise in-memory
|
|
||||||
if redisClient != nil {
|
|
||||||
data, _ := json.Marshal(session)
|
|
||||||
redisClient.Set(ctx, "upload:"+session.ID, data, 24*time.Hour)
|
|
||||||
} else {
|
|
||||||
s.sessions[session.ID] = session
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 5. Client-Side Retry Logic (for mobile apps/browsers)
|
|
||||||
|
|
||||||
```javascript
|
|
||||||
// Client-side upload with retry logic
|
|
||||||
class ResilientUploader {
|
|
||||||
constructor(file, endpoint, options = {}) {
|
|
||||||
this.file = file;
|
|
||||||
this.endpoint = endpoint;
|
|
||||||
this.chunkSize = options.chunkSize || 5 * 1024 * 1024; // 5MB chunks
|
|
||||||
this.maxRetries = options.maxRetries || 5;
|
|
||||||
this.retryDelay = options.retryDelay || 2000;
|
|
||||||
}
|
|
||||||
|
|
||||||
async upload() {
|
|
||||||
const totalChunks = Math.ceil(this.file.size / this.chunkSize);
|
|
||||||
const sessionId = this.generateSessionId();
|
|
||||||
|
|
||||||
for (let i = 0; i < totalChunks; i++) {
|
|
||||||
await this.uploadChunk(i, sessionId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async uploadChunk(chunkIndex, sessionId, retryCount = 0) {
|
|
||||||
try {
|
|
||||||
const start = chunkIndex * this.chunkSize;
|
|
||||||
const end = Math.min(start + this.chunkSize, this.file.size);
|
|
||||||
const chunk = this.file.slice(start, end);
|
|
||||||
|
|
||||||
const response = await fetch(this.endpoint, {
|
|
||||||
method: 'PUT',
|
|
||||||
headers: {
|
|
||||||
'X-Upload-Session-ID': sessionId,
|
|
||||||
'X-Chunk-Number': chunkIndex,
|
|
||||||
'X-Total-Chunks': totalChunks,
|
|
||||||
'Content-Range': `bytes ${start}-${end-1}/${this.file.size}`
|
|
||||||
},
|
|
||||||
body: chunk
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
|
||||||
|
|
||||||
} catch (error) {
|
|
||||||
if (retryCount < this.maxRetries) {
|
|
||||||
// Exponential backoff with jitter
|
|
||||||
const delay = this.retryDelay * Math.pow(2, retryCount) + Math.random() * 1000;
|
|
||||||
await new Promise(resolve => setTimeout(resolve, delay));
|
|
||||||
return this.uploadChunk(chunkIndex, sessionId, retryCount + 1);
|
|
||||||
}
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Implementation Priority
|
|
||||||
|
|
||||||
1. **High Priority**: Implement chunked uploads with session persistence
|
|
||||||
2. **High Priority**: Adjust connection timeouts for mobile scenarios
|
|
||||||
3. **Medium Priority**: Add network change detection and upload pausing
|
|
||||||
4. **Medium Priority**: Implement retry logic in upload handlers
|
|
||||||
5. **Low Priority**: Add client-side SDK with built-in resilience
|
|
||||||
|
|
||||||
## Configuration Changes Needed
|
|
||||||
|
|
||||||
```toml
|
```toml
|
||||||
|
[server]
|
||||||
|
networkevents = true # Enable network monitoring
|
||||||
|
|
||||||
[uploads]
|
[uploads]
|
||||||
resumableuploadsenabled = true # Enable the feature
|
chunkeduploadsenabled = true # Enable chunked uploads
|
||||||
chunkeduploadsenabled = true # Already exists but not implemented
|
chunksize = "5MB" # Optimal for mobile
|
||||||
chunksize = "5MB" # Smaller chunks for mobile
|
sessiontimeout = "24h" # Session persistence
|
||||||
sessiontimeout = "24h" # How long to keep upload sessions
|
|
||||||
maxretries = 5 # Server-side retry attempts
|
|
||||||
|
|
||||||
[timeouts]
|
[timeouts]
|
||||||
readtimeout = "300s" # 5 minutes for mobile uploads
|
readtimeout = "300s" # 5 minutes for mobile
|
||||||
writetimeout = "300s" # 5 minutes for responses
|
writetimeout = "300s" # 5 minutes for slow networks
|
||||||
idletimeout = "600s" # 10 minutes idle timeout
|
idletimeout = "600s" # 10 minutes keep-alive
|
||||||
uploadtimeout = "3600s" # 1 hour total upload timeout
|
|
||||||
|
|
||||||
[network]
|
|
||||||
networkchangedetection = true # Enable network monitoring
|
|
||||||
uploadpauseonchange = true # Pause uploads during network changes
|
|
||||||
reconnectdelay = "2s" # Wait time after network change
|
|
||||||
keepaliveinterval = "30s" # TCP keep-alive interval
|
|
||||||
```
|
```
|
||||||
|
|
||||||
This comprehensive approach will make uploads much more resilient to network switching scenarios common with mobile devices using multiple network interfaces.
|
## 🔨 Build & Deploy
|
||||||
|
|
||||||
|
### Build Process
|
||||||
|
```bash
|
||||||
|
# Automatic build with network resilience detection
|
||||||
|
./buildgo.sh
|
||||||
|
|
||||||
|
# Output confirms features are included:
|
||||||
|
# [INFO] Found network resilience: upload_session.go
|
||||||
|
# [INFO] Found network resilience: network_resilience.go
|
||||||
|
# [INFO] Found network resilience: chunked_upload_handler.go
|
||||||
|
# [INFO] Found network resilience: integration.go
|
||||||
|
# [BUILD] Build successful! Binary created: ./hmac-file-server
|
||||||
|
```
|
||||||
|
|
||||||
|
### Testing
|
||||||
|
```bash
|
||||||
|
# Comprehensive feature testing
|
||||||
|
./test_network_resilience.sh
|
||||||
|
|
||||||
|
# Expected output:
|
||||||
|
# ✅ Traditional upload works
|
||||||
|
# ✅ Chunked upload session creation works
|
||||||
|
# ✅ Upload status endpoint works
|
||||||
|
# ✅ Health endpoint works
|
||||||
|
# ✅ Metrics endpoint works
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🎯 Perfect for Your Use Case
|
||||||
|
|
||||||
|
This implementation specifically solves your **notebook network switching** problem:
|
||||||
|
|
||||||
|
1. **WLAN ↔ WiFi Switching**: Uploads automatically pause during network changes and resume when stable
|
||||||
|
2. **Mobile-Friendly**: 5MB chunks work well over cellular and WiFi connections
|
||||||
|
3. **Android/iOS Compatible**: HTTP-based solution works with any mobile platform
|
||||||
|
4. **Zero Disruption**: Existing users see no changes, new users get enhanced reliability
|
||||||
|
5. **Production Ready**: Full session persistence, monitoring, and error handling
|
||||||
|
|
||||||
|
Your users can now upload large files from notebooks/mobile devices without worrying about network switching interrupting their transfers! 🚀
|
||||||
|
|||||||
62
config-network-resilience.toml
Normal file
62
config-network-resilience.toml
Normal file
@ -0,0 +1,62 @@
|
|||||||
|
# HMAC File Server Configuration with Network Resilience
|
||||||
|
# Copy this to config.toml and modify as needed
|
||||||
|
|
||||||
|
[server]
|
||||||
|
bind_ip = "0.0.0.0"
|
||||||
|
listenport = "8080"
|
||||||
|
unixsocket = false
|
||||||
|
storagepath = "./uploads"
|
||||||
|
metricsenabled = true
|
||||||
|
metricsport = "9090"
|
||||||
|
deduplicationenabled = true
|
||||||
|
networkevents = true # Enable network change detection
|
||||||
|
autoadjustworkers = true
|
||||||
|
pidfilepath = "./hmac-file-server.pid"
|
||||||
|
|
||||||
|
[security]
|
||||||
|
secret = "your-super-secret-hmac-key-minimum-32-characters-long"
|
||||||
|
enablejwt = false
|
||||||
|
# jwtsecret = "your-jwt-secret"
|
||||||
|
# jwtalgorithm = "HS256"
|
||||||
|
# jwtexpiration = "24h"
|
||||||
|
|
||||||
|
[uploads]
|
||||||
|
chunkeduploadsenabled = true # Enable chunked/resumable uploads
|
||||||
|
resumableuploadsenabled = true # Enable upload resumption
|
||||||
|
chunksize = "5MB" # Optimal chunk size for mobile
|
||||||
|
sessiontimeout = "24h" # How long to keep upload sessions
|
||||||
|
maxretries = 5 # Server-side retry attempts
|
||||||
|
allowedextensions = [".txt", ".pdf", ".jpg", ".jpeg", ".png", ".gif", ".zip", ".tar", ".gz"]
|
||||||
|
maxfilesize = "100MB"
|
||||||
|
|
||||||
|
[downloads]
|
||||||
|
chunkeddownloadsenabled = true
|
||||||
|
chunksize = "5MB"
|
||||||
|
|
||||||
|
[timeouts]
|
||||||
|
readtimeout = "300s" # 5 minutes for mobile uploads
|
||||||
|
writetimeout = "300s" # 5 minutes for responses
|
||||||
|
idletimeout = "600s" # 10 minutes keep-alive
|
||||||
|
|
||||||
|
[logging]
|
||||||
|
level = "info"
|
||||||
|
file = "" # Empty = stdout
|
||||||
|
max_size = 100
|
||||||
|
max_backups = 3
|
||||||
|
max_age = 30
|
||||||
|
compress = true
|
||||||
|
|
||||||
|
[workers]
|
||||||
|
numworkers = 10
|
||||||
|
uploadqueuesize = 1000
|
||||||
|
autoscaling = true
|
||||||
|
|
||||||
|
[redis]
|
||||||
|
redisenabled = false # Enable for session persistence
|
||||||
|
redisaddr = "localhost:6379"
|
||||||
|
redispassword = ""
|
||||||
|
redisdbindex = 0
|
||||||
|
|
||||||
|
[clamav]
|
||||||
|
clamavenabled = false # Enable for virus scanning
|
||||||
|
clamavsocket = "/var/run/clamav/clamd.ctl"
|
||||||
Reference in New Issue
Block a user