🔥 Tremora del Terra: ultimate hmac-file-server fix – final push before the drop 💾🔐
This commit is contained in:
@ -1,86 +0,0 @@
|
||||
# HMAC File Server Test Suite
|
||||
|
||||
This directory contains test scripts, monitoring tools, and test data files for the HMAC File Server.
|
||||
|
||||
## Test Scripts
|
||||
|
||||
### Protocol Testing
|
||||
- `test_final_xmpp.sh` - Complete XEP-0363 protocol testing (all variants: v1, v2, v3, token)
|
||||
- `test_xmpp_simulation.sh` - XMPP client simulation for upload testing
|
||||
- `test_url_formats.sh` - URL format validation and testing
|
||||
- `verify_xmpp_upload.sh` - XMPP upload verification script
|
||||
|
||||
### Performance Testing
|
||||
- `comprehensive_upload_test.sh` - Comprehensive upload performance testing
|
||||
- `test_upload_queue.sh` - Queue performance and concurrent upload testing
|
||||
- `test_upload_completion.sh` - Upload completion and reliability testing
|
||||
|
||||
### Feature Testing
|
||||
- `test_deduplication.sh` - File deduplication functionality testing
|
||||
- `test_direct_connection.sh` - Direct server connection testing
|
||||
- `test_path_discovery.sh` - Path discovery and routing testing
|
||||
|
||||
### Debugging & Monitoring
|
||||
- `debug_upload.sh` - Upload debugging and troubleshooting script
|
||||
- `monitor_server.sh` - Server status and performance monitoring
|
||||
- `monitor_nginx.sh` - Nginx proxy monitoring
|
||||
- `monitor_uploads.sh` - Upload activity monitoring
|
||||
|
||||
## Test Data Files
|
||||
|
||||
### Small Test Files
|
||||
- `test_1mb.txt` / `test_1mb.bin` - 1MB test files for basic functionality
|
||||
- `test_upload.txt` - Small text file for quick testing
|
||||
- `chunk_0.bin` - Chunked upload test data
|
||||
|
||||
### Large Test Files
|
||||
- `test_50mb.bin` - 50MB file for medium-size upload testing
|
||||
- `test_215mb.bin` - 215MB file for large upload testing
|
||||
- `test_4gb.bin` / `test_4gb.txt` - 4GB files for stress testing
|
||||
|
||||
## Analysis Tools
|
||||
|
||||
- `xep0363_analysis.ipynb` - Jupyter notebook for XEP-0363 protocol analysis
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Quick Protocol Test
|
||||
```bash
|
||||
cd tests
|
||||
./test_final_xmpp.sh
|
||||
```
|
||||
|
||||
### Performance Testing
|
||||
```bash
|
||||
cd tests
|
||||
./comprehensive_upload_test.sh
|
||||
./test_upload_queue.sh
|
||||
```
|
||||
|
||||
### Deduplication Testing
|
||||
```bash
|
||||
cd tests
|
||||
./test_deduplication.sh
|
||||
```
|
||||
|
||||
### Monitor Server
|
||||
```bash
|
||||
cd tests
|
||||
./monitor_server.sh
|
||||
```
|
||||
|
||||
## Test Environment
|
||||
|
||||
These tests are designed to work with:
|
||||
- HMAC File Server 3.2
|
||||
- nginx reverse proxy (standard configuration)
|
||||
- Extended timeout settings (4800s)
|
||||
- Deduplication enabled
|
||||
- Dynamic worker scaling
|
||||
|
||||
## Notes
|
||||
|
||||
- Large test files (4GB) are intended for stress testing extended timeout configurations
|
||||
- All scripts include proper error handling and cleanup
|
||||
- Monitor scripts provide real-time status information
|
||||
- Test scripts validate both success and failure scenarios
|
||||
Binary file not shown.
@ -1,267 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Comprehensive XMPP Upload Test Script
|
||||
# Tests multiple upload scenarios with real-time debugging
|
||||
|
||||
echo "=== COMPREHENSIVE UPLOAD TEST SCRIPT ==="
|
||||
echo "This script will test multiple upload scenarios while monitoring logs"
|
||||
echo "Date: $(date)"
|
||||
echo ""
|
||||
|
||||
# Configuration
|
||||
SERVER_URL="https://share.uuxo.net"
|
||||
LOCAL_URL="http://localhost:8080"
|
||||
SECRET="f6g4ldPvQM7O2UTFeBEUUj33VrXypDAcsDt0yqKrLiOr5oQW"
|
||||
TEST_DIR="/tmp/upload_tests"
|
||||
|
||||
# Create test directory
|
||||
mkdir -p "$TEST_DIR"
|
||||
cd "$TEST_DIR"
|
||||
|
||||
# Function to generate HMAC signature for v3 protocol
|
||||
generate_v3_signature() {
|
||||
local method="$1"
|
||||
local expires="$2"
|
||||
local path="$3"
|
||||
local message="${method}\n${expires}\n${path}"
|
||||
echo -n "$message" | openssl dgst -sha256 -hmac "$SECRET" -hex | cut -d' ' -f2
|
||||
}
|
||||
|
||||
# Function to start log monitoring
|
||||
start_monitoring() {
|
||||
echo "Starting log monitoring in background..."
|
||||
|
||||
# Kill any existing monitoring
|
||||
pkill -f "tail.*hmac-file-server" 2>/dev/null
|
||||
pkill -f "tail.*nginx.*share" 2>/dev/null
|
||||
|
||||
# Start nginx monitoring
|
||||
echo "=== NGINX ACCESS LOG ===" > /tmp/nginx_monitor.log
|
||||
sudo tail -f /var/log/nginx/share_access.log >> /tmp/nginx_monitor.log 2>&1 &
|
||||
NGINX_PID=$!
|
||||
|
||||
# Start server monitoring
|
||||
echo "=== HMAC SERVER LOG ===" > /tmp/server_monitor.log
|
||||
sudo tail -f /var/log/hmac-file-server/hmac-file-server.log >> /tmp/server_monitor.log 2>&1 &
|
||||
SERVER_PID=$!
|
||||
|
||||
sleep 1
|
||||
echo "Monitoring started (nginx PID: $NGINX_PID, server PID: $SERVER_PID)"
|
||||
}
|
||||
|
||||
# Function to stop monitoring and show results
|
||||
stop_monitoring() {
|
||||
echo "Stopping monitors..."
|
||||
kill $NGINX_PID $SERVER_PID 2>/dev/null
|
||||
sleep 1
|
||||
|
||||
echo ""
|
||||
echo "=== NGINX LOG RESULTS ==="
|
||||
tail -10 /tmp/nginx_monitor.log 2>/dev/null || echo "No nginx activity detected"
|
||||
|
||||
echo ""
|
||||
echo "=== SERVER LOG RESULTS ==="
|
||||
tail -10 /tmp/server_monitor.log 2>/dev/null || echo "No server activity detected"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Function to create test files
|
||||
create_test_files() {
|
||||
echo "Creating test files..."
|
||||
|
||||
# Small file (1KB)
|
||||
echo "This is a small test file for upload testing" > small_test.txt
|
||||
echo "Content: Basic text file" >> small_test.txt
|
||||
|
||||
# Medium file (1MB)
|
||||
dd if=/dev/zero of=medium_test.bin bs=1024 count=1024 2>/dev/null
|
||||
|
||||
# Large file (5MB)
|
||||
dd if=/dev/zero of=large_test.bin bs=1024 count=5120 2>/dev/null
|
||||
|
||||
# Video file simulation (1MB with .mp4 extension)
|
||||
cp medium_test.bin test_video.mp4
|
||||
|
||||
echo "Test files created:"
|
||||
ls -lh *.txt *.bin *.mp4 2>/dev/null
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Function to test different upload protocols
|
||||
test_upload_protocol() {
|
||||
local protocol="$1"
|
||||
local filename="$2"
|
||||
local description="$3"
|
||||
|
||||
echo "--- Testing $protocol Protocol: $description ---"
|
||||
|
||||
# Generate test parameters
|
||||
local expires=$(date -d "+1 hour" +%s)
|
||||
local path="/test_${protocol}/${filename}"
|
||||
local url=""
|
||||
local signature=""
|
||||
|
||||
case "$protocol" in
|
||||
"v3")
|
||||
signature=$(generate_v3_signature "PUT" "$expires" "$path")
|
||||
url="${SERVER_URL}${path}?v3=${signature}&expires=${expires}"
|
||||
;;
|
||||
"v2")
|
||||
signature=$(echo -n "PUT${path}" | openssl dgst -sha256 -hmac "$SECRET" -hex | cut -d' ' -f2)
|
||||
url="${SERVER_URL}${path}?v2=${signature}"
|
||||
;;
|
||||
"v1")
|
||||
signature=$(echo -n "PUT${path}" | openssl dgst -sha256 -hmac "$SECRET" -hex | cut -d' ' -f2)
|
||||
url="${SERVER_URL}${path}?v=${signature}"
|
||||
;;
|
||||
"token")
|
||||
signature=$(echo -n "PUT${path}" | openssl dgst -sha256 -hmac "$SECRET" -hex | cut -d' ' -f2)
|
||||
url="${SERVER_URL}${path}?token=${signature}"
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "URL: $url"
|
||||
echo "File: $filename ($(stat -f%z "$filename" 2>/dev/null || stat -c%s "$filename")bytes)"
|
||||
|
||||
# Start monitoring for this test
|
||||
echo "Starting upload test..."
|
||||
|
||||
# Perform upload
|
||||
local start_time=$(date +%s.%N)
|
||||
local response=$(curl -s -w "HTTPSTATUS:%{http_code};TIME:%{time_total}" \
|
||||
-X PUT \
|
||||
--data-binary "@$filename" \
|
||||
-H "User-Agent: XMPP-Upload-Test/1.0" \
|
||||
-H "Content-Type: application/octet-stream" \
|
||||
"$url" 2>&1)
|
||||
local end_time=$(date +%s.%N)
|
||||
|
||||
# Parse response
|
||||
local http_code=$(echo "$response" | grep -o "HTTPSTATUS:[0-9]*" | cut -d: -f2)
|
||||
local time_total=$(echo "$response" | grep -o "TIME:[0-9.]*" | cut -d: -f2)
|
||||
local body=$(echo "$response" | sed 's/HTTPSTATUS:[0-9]*;TIME:[0-9.]*$//')
|
||||
|
||||
# Calculate duration
|
||||
local duration=$(echo "$end_time - $start_time" | bc 2>/dev/null || echo "N/A")
|
||||
|
||||
echo "Result: HTTP $http_code (${time_total}s)"
|
||||
if [[ "$http_code" =~ ^[45] ]]; then
|
||||
echo "Error body: $body"
|
||||
elif [[ "$http_code" == "200" ]]; then
|
||||
echo "✅ SUCCESS: Upload completed"
|
||||
echo "Response: $body"
|
||||
else
|
||||
echo "Response: $body"
|
||||
fi
|
||||
|
||||
echo "Duration: ${duration}s"
|
||||
echo ""
|
||||
|
||||
# Brief pause to separate log entries
|
||||
sleep 2
|
||||
}
|
||||
|
||||
# Function to test deduplication
|
||||
test_deduplication() {
|
||||
echo "--- Testing Deduplication ---"
|
||||
echo "Uploading the same file twice to test deduplication logic"
|
||||
|
||||
# First upload
|
||||
echo "1. First upload (should create new file):"
|
||||
test_upload_protocol "v3" "small_test.txt" "Dedup Test #1"
|
||||
|
||||
# Second upload (should deduplicate)
|
||||
echo "2. Second upload (should deduplicate):"
|
||||
test_upload_protocol "v3" "small_test.txt" "Dedup Test #2"
|
||||
}
|
||||
|
||||
# Function to test storage scenarios
|
||||
test_storage_scenarios() {
|
||||
echo "--- Testing Different Storage Scenarios ---"
|
||||
|
||||
# Test small file
|
||||
test_upload_protocol "v3" "small_test.txt" "Small File (1KB)"
|
||||
|
||||
# Test medium file
|
||||
test_upload_protocol "v3" "medium_test.bin" "Medium File (1MB)"
|
||||
|
||||
# Test video file
|
||||
test_upload_protocol "v3" "test_video.mp4" "Video File (.mp4)"
|
||||
|
||||
# Test large file
|
||||
test_upload_protocol "v3" "large_test.bin" "Large File (5MB)"
|
||||
}
|
||||
|
||||
# Function to test all protocols
|
||||
test_all_protocols() {
|
||||
echo "--- Testing All XEP-0363 Protocol Variants ---"
|
||||
|
||||
test_upload_protocol "v3" "small_test.txt" "XEP-0363 v3 (mod_http_upload_external)"
|
||||
test_upload_protocol "v2" "small_test.txt" "XEP-0363 v2 (extended)"
|
||||
test_upload_protocol "v1" "small_test.txt" "XEP-0363 v1 (basic)"
|
||||
test_upload_protocol "token" "small_test.txt" "XEP-0363 token (alternative)"
|
||||
}
|
||||
|
||||
# Function to show current configuration
|
||||
show_configuration() {
|
||||
echo "=== Current Server Configuration ==="
|
||||
echo "Deduplication: $(sudo grep deduplication_enabled /etc/hmac-file-server/config.toml | cut -d'=' -f2 | tr -d ' ')"
|
||||
echo "Max Upload: $(sudo grep max_upload_size /etc/hmac-file-server/config.toml | cut -d'"' -f2)"
|
||||
echo "ClamAV: $(sudo grep clamavenabled /etc/hmac-file-server/config.toml | cut -d'=' -f2 | tr -d ' ')"
|
||||
echo "Global Extensions: $(sudo grep global_extensions /etc/hmac-file-server/config.toml | cut -d'[' -f2 | cut -d']' -f1)"
|
||||
echo "Log Level: $(sudo grep 'level =' /etc/hmac-file-server/config.toml | cut -d'"' -f2)"
|
||||
echo "Server Status: $(systemctl is-active hmac-file-server)"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Function to cleanup
|
||||
cleanup() {
|
||||
echo "Cleaning up..."
|
||||
stop_monitoring
|
||||
rm -rf "$TEST_DIR" 2>/dev/null
|
||||
echo "Cleanup complete"
|
||||
}
|
||||
|
||||
# Trap for cleanup on exit
|
||||
trap cleanup EXIT
|
||||
|
||||
# Main execution
|
||||
main() {
|
||||
show_configuration
|
||||
create_test_files
|
||||
start_monitoring
|
||||
|
||||
echo "=== STARTING COMPREHENSIVE UPLOAD TESTS ==="
|
||||
echo "Monitor logs in real-time:"
|
||||
echo " nginx: tail -f /tmp/nginx_monitor.log"
|
||||
echo " server: tail -f /tmp/server_monitor.log"
|
||||
echo ""
|
||||
|
||||
# Test 1: Protocol variants
|
||||
echo "🔄 TEST 1: All Protocol Variants"
|
||||
test_all_protocols
|
||||
|
||||
# Test 2: Storage scenarios
|
||||
echo "🔄 TEST 2: Storage Scenarios"
|
||||
test_storage_scenarios
|
||||
|
||||
# Test 3: Deduplication
|
||||
echo "🔄 TEST 3: Deduplication"
|
||||
test_deduplication
|
||||
|
||||
echo "=== TEST SUMMARY ==="
|
||||
echo "All tests completed. Check the results above."
|
||||
echo "If you see HTTP 401 errors, that's expected (HMAC signature validation)."
|
||||
echo "If you see HTTP 200 responses, uploads are working!"
|
||||
echo "If you see no nginx log entries, requests aren't reaching the server."
|
||||
echo ""
|
||||
|
||||
stop_monitoring
|
||||
|
||||
echo "Log files saved to:"
|
||||
echo " nginx: /tmp/nginx_monitor.log"
|
||||
echo " server: /tmp/server_monitor.log"
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main "$@"
|
||||
@ -1,105 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Simple test to debug the 49% upload stop issue
|
||||
|
||||
set -e
|
||||
|
||||
echo "[DEBUG-TEST] Starting server..."
|
||||
./hmac-file-server --config config-network-resilience.toml > debug_server.log 2>&1 &
|
||||
SERVER_PID=$!
|
||||
|
||||
# Wait for server to start
|
||||
sleep 3
|
||||
|
||||
# Check if server is running
|
||||
if ! kill -0 $SERVER_PID 2>/dev/null; then
|
||||
echo "[ERROR] Server failed to start"
|
||||
cat debug_server.log
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cleanup() {
|
||||
echo "[DEBUG-TEST] Cleaning up..."
|
||||
kill $SERVER_PID 2>/dev/null || true
|
||||
rm -f debug_server.log
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "[DEBUG-TEST] Testing 50MB chunked upload..."
|
||||
|
||||
# Calculate HMAC signature
|
||||
SECRET="your-super-secret-hmac-key-minimum-32-characters-long"
|
||||
MESSAGE="/chunked-upload"
|
||||
SIGNATURE=$(echo -n "$MESSAGE" | openssl dgst -sha256 -hmac "$SECRET" | cut -d' ' -f2)
|
||||
|
||||
# Start session
|
||||
echo "[DEBUG-TEST] Creating session..."
|
||||
SESSION_RESPONSE=$(curl -s -X POST \
|
||||
-H "X-Filename: test_50mb.bin" \
|
||||
-H "X-Total-Size: 52428800" \
|
||||
-H "X-Signature: $SIGNATURE" \
|
||||
http://localhost:8080/chunked-upload)
|
||||
|
||||
echo "[DEBUG-TEST] Session response: $SESSION_RESPONSE"
|
||||
|
||||
SESSION_ID=$(echo "$SESSION_RESPONSE" | grep -o '"session_id":"[^"]*"' | cut -d'"' -f4)
|
||||
if [ -z "$SESSION_ID" ]; then
|
||||
echo "[ERROR] Failed to get session ID"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[DEBUG-TEST] Session ID: $SESSION_ID"
|
||||
|
||||
# Upload first few chunks to see what happens
|
||||
CHUNK_SIZE=5242880 # 5MB
|
||||
for i in {0..12}; do # Upload first 13 chunks (65MB worth, should trigger completion)
|
||||
OFFSET=$((i * CHUNK_SIZE))
|
||||
|
||||
echo "[DEBUG-TEST] Creating chunk $i..."
|
||||
dd if=test_50mb.bin of=chunk_$i.bin bs=$CHUNK_SIZE skip=$i count=1 2>/dev/null || {
|
||||
# Handle the last chunk
|
||||
REMAINING=$((52428800 - OFFSET))
|
||||
if [ $REMAINING -gt 0 ]; then
|
||||
dd if=test_50mb.bin of=chunk_$i.bin bs=1 skip=$OFFSET count=$REMAINING 2>/dev/null
|
||||
else
|
||||
echo "[DEBUG-TEST] No more data for chunk $i"
|
||||
break
|
||||
fi
|
||||
}
|
||||
|
||||
CHUNK_SIZE_ACTUAL=$(stat -f%z chunk_$i.bin 2>/dev/null || stat -c%s chunk_$i.bin 2>/dev/null)
|
||||
echo "[DEBUG-TEST] Uploading chunk $i (size: $CHUNK_SIZE_ACTUAL bytes)..."
|
||||
|
||||
UPLOAD_RESPONSE=$(curl -s -w "\n%{http_code}" -X PUT \
|
||||
-H "X-Upload-Session-ID: $SESSION_ID" \
|
||||
-H "X-Chunk-Number: $i" \
|
||||
--data-binary @chunk_$i.bin \
|
||||
http://localhost:8080/chunked-upload)
|
||||
|
||||
echo "[DEBUG-TEST] Upload response for chunk $i:"
|
||||
echo "$UPLOAD_RESPONSE"
|
||||
echo "---"
|
||||
|
||||
# Check server logs for debug output
|
||||
echo "[DEBUG-TEST] Recent server logs:"
|
||||
tail -5 debug_server.log
|
||||
echo "---"
|
||||
|
||||
# Check if complete
|
||||
if echo "$UPLOAD_RESPONSE" | grep -q '"complete":true'; then
|
||||
echo "[DEBUG-TEST] ✅ Upload completed at chunk $i"
|
||||
rm -f chunk_*.bin
|
||||
exit 0
|
||||
fi
|
||||
|
||||
rm -f chunk_$i.bin
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "[DEBUG-TEST] Upload did not complete. Checking status..."
|
||||
STATUS_RESPONSE=$(curl -s "http://localhost:8080/upload-status?session_id=$SESSION_ID")
|
||||
echo "[DEBUG-TEST] Final status: $STATUS_RESPONSE"
|
||||
|
||||
echo "[DEBUG-TEST] Full server logs:"
|
||||
cat debug_server.log
|
||||
@ -1,34 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Terminal 1: nginx Monitoring Script
|
||||
echo "=== NGINX ACCESS LOG MONITOR ==="
|
||||
echo "Monitoring: /var/log/nginx/share_access.log"
|
||||
echo "Press Ctrl+C to stop"
|
||||
echo ""
|
||||
echo "Waiting for upload requests..."
|
||||
echo "$(date): Monitor started"
|
||||
echo ""
|
||||
|
||||
# Monitor nginx access logs with timestamps
|
||||
sudo tail -f /var/log/nginx/share_access.log | while read line; do
|
||||
if [[ -n "$line" ]]; then
|
||||
echo "[$(date '+%H:%M:%S')] NGINX: $line"
|
||||
|
||||
# Highlight important patterns
|
||||
if echo "$line" | grep -q "PUT"; then
|
||||
echo "*** PUT REQUEST DETECTED ***"
|
||||
fi
|
||||
|
||||
if echo "$line" | grep -q " 401 "; then
|
||||
echo "!!! AUTH FAILURE (401) !!!"
|
||||
fi
|
||||
|
||||
if echo "$line" | grep -q " 200 "; then
|
||||
echo "✅ SUCCESS (200) ✅"
|
||||
fi
|
||||
|
||||
if echo "$line" | grep -q " 40[0-9] \| 50[0-9] "; then
|
||||
echo "❌ ERROR RESPONSE ❌"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
@ -1,52 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Terminal 2: HMAC Server Monitoring Script
|
||||
echo "=== HMAC SERVER LOG MONITOR ==="
|
||||
echo "Monitoring: /var/log/hmac-file-server/hmac-file-server.log"
|
||||
echo "Press Ctrl+C to stop"
|
||||
echo ""
|
||||
echo "Waiting for upload activity..."
|
||||
echo "$(date): Monitor started"
|
||||
echo ""
|
||||
|
||||
# Monitor server logs with filtering and highlighting
|
||||
sudo tail -f /var/log/hmac-file-server/hmac-file-server.log | while read line; do
|
||||
# Skip debug worker messages unless they're important
|
||||
if echo "$line" | grep -q "DEBUG.*Worker\|NumWorkers\|NumScanWorkers" && ! echo "$line" | grep -q "upload\|error\|fail"; then
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ -n "$line" ]]; then
|
||||
echo "[$(date '+%H:%M:%S')] SERVER: $line"
|
||||
|
||||
# Highlight upload-related activity
|
||||
if echo "$line" | grep -qi "upload\|PUT\|POST"; then
|
||||
echo "📤 UPLOAD ACTIVITY DETECTED"
|
||||
fi
|
||||
|
||||
# Highlight HMAC validation
|
||||
if echo "$line" | grep -qi "hmac\|auth\|signature"; then
|
||||
echo "🔐 HMAC VALIDATION ACTIVITY"
|
||||
fi
|
||||
|
||||
# Highlight deduplication
|
||||
if echo "$line" | grep -qi "dedup"; then
|
||||
echo "🔗 DEDUPLICATION ACTIVITY"
|
||||
fi
|
||||
|
||||
# Highlight errors
|
||||
if echo "$line" | grep -qi "error\|fail\|fatal"; then
|
||||
echo "❌ ERROR DETECTED ❌"
|
||||
fi
|
||||
|
||||
# Highlight success
|
||||
if echo "$line" | grep -qi "success"; then
|
||||
echo "✅ SUCCESS DETECTED ✅"
|
||||
fi
|
||||
|
||||
# Highlight file operations
|
||||
if echo "$line" | grep -qi "file.*created\|file.*stored\|file.*saved"; then
|
||||
echo "💾 FILE STORAGE ACTIVITY"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
@ -1,61 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Monitor script to watch for XMPP upload activity
|
||||
# This will help verify that our performance optimizations are working
|
||||
|
||||
echo "=== HMAC File Server Upload Monitor ==="
|
||||
echo "Watching for upload activity on share.uuxo.net..."
|
||||
echo "Press Ctrl+C to stop"
|
||||
echo ""
|
||||
|
||||
# Function to show current configuration status
|
||||
show_status() {
|
||||
echo "Current Configuration Status:"
|
||||
echo "- Max Upload Size: $(grep max_upload_size /etc/hmac-file-server/config.toml | cut -d'"' -f2)"
|
||||
echo "- ClamAV Enabled: $(grep clamavenabled /etc/hmac-file-server/config.toml | cut -d'=' -f2 | tr -d ' ')"
|
||||
echo "- Deduplication: $(grep deduplication_enabled /etc/hmac-file-server/config.toml | cut -d'=' -f2 | tr -d ' ')"
|
||||
echo "- File Naming: $(grep file_naming /etc/hmac-file-server/config.toml | cut -d'"' -f2)"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Function to monitor logs
|
||||
monitor_logs() {
|
||||
echo "Starting real-time log monitoring..."
|
||||
echo "Monitoring multiple log sources:"
|
||||
echo "1. HMAC Server logs (/var/log/hmac-file-server/hmac-file-server.log)"
|
||||
echo "2. Share nginx access logs (/var/log/nginx/share_access.log)"
|
||||
echo "3. Share nginx error logs (/var/log/nginx/share_error.log)"
|
||||
echo ""
|
||||
|
||||
# Run tail on multiple files simultaneously
|
||||
sudo tail -f /var/log/hmac-file-server/hmac-file-server.log \
|
||||
/var/log/nginx/share_access.log \
|
||||
/var/log/nginx/share_error.log 2>/dev/null | \
|
||||
while read line; do
|
||||
timestamp=$(date '+%H:%M:%S')
|
||||
echo "[$timestamp] $line"
|
||||
|
||||
# Highlight important upload events
|
||||
if echo "$line" | grep -qi "PUT\|upload\|POST"; then
|
||||
echo "*** UPLOAD ACTIVITY DETECTED ***"
|
||||
fi
|
||||
|
||||
if echo "$line" | grep -qi "error\|failed\|timeout"; then
|
||||
echo "!!! ERROR/ISSUE DETECTED !!!"
|
||||
fi
|
||||
|
||||
if echo "$line" | grep -qi "clamav\|scan"; then
|
||||
echo ">>> ClamAV ACTIVITY <<<"
|
||||
fi
|
||||
|
||||
if echo "$line" | grep -qi "dedup"; then
|
||||
echo ">>> DEDUPLICATION ACTIVITY <<<"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Show current status
|
||||
show_status
|
||||
|
||||
# Start monitoring
|
||||
monitor_logs
|
||||
Binary file not shown.
@ -1 +0,0 @@
|
||||
Hello, HMAC File Server! Do 17. Jul 18:59:11 CEST 2025
|
||||
Binary file not shown.
Binary file not shown.
@ -1 +0,0 @@
|
||||
Hello, HMAC File Server! Do 17. Jul 18:59:11 CEST 2025
|
||||
Binary file not shown.
@ -1 +0,0 @@
|
||||
Hello, HMAC File Server! Thu Jul 17 05:40:07 PM UTC 2025
|
||||
@ -1,83 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# XMPP Upload Verification Script
|
||||
# Tests HMAC validation and upload process
|
||||
|
||||
echo "=== XMPP Upload Verification ==="
|
||||
echo "Testing HMAC File Server configuration for XMPP uploads"
|
||||
echo ""
|
||||
|
||||
# Configuration check
|
||||
echo "1. Configuration Status:"
|
||||
echo " Secret configured: $(sudo grep -c "secret.*f6g4ldPvQM7O2UTFeBEUUj33VrXypDAcsDt0yqKrLiOr5oQW" /etc/hmac-file-server/config.toml > /dev/null && echo "✅ YES" || echo "❌ NO")"
|
||||
echo " Deduplication limit: $(sudo grep maxsize /etc/hmac-file-server/config.toml | cut -d'"' -f2)"
|
||||
echo " Max upload size: $(sudo grep max_upload_size /etc/hmac-file-server/config.toml | cut -d'"' -f2)"
|
||||
echo " ClamAV enabled: $(sudo grep clamavenabled /etc/hmac-file-server/config.toml | cut -d'=' -f2 | tr -d ' ')"
|
||||
echo ""
|
||||
|
||||
# Server status
|
||||
echo "2. Server Status:"
|
||||
echo " Service status: $(systemctl is-active hmac-file-server)"
|
||||
echo " Health endpoint: $(curl -s -w "%{http_code}" http://localhost:8080/health -o /dev/null)"
|
||||
echo " Process running: $(pgrep -f hmac-file-server > /dev/null && echo "✅ YES" || echo "❌ NO")"
|
||||
echo ""
|
||||
|
||||
# Network connectivity
|
||||
echo "3. Network Configuration:"
|
||||
echo " nginx stream (443→4443): $(sudo netstat -tlnp | grep :443 | grep -q nginx && echo "✅ ACTIVE" || echo "❌ NOT FOUND")"
|
||||
echo " nginx HTTP (4443→8080): $(sudo netstat -tlnp | grep :4443 | grep -q nginx && echo "✅ ACTIVE" || echo "❌ NOT FOUND")"
|
||||
echo " HMAC server (8080): $(sudo netstat -tlnp | grep :8080 | grep -q hmac && echo "✅ LISTENING" || echo "❌ NOT LISTENING")"
|
||||
echo ""
|
||||
|
||||
# XEP-0363 protocol support
|
||||
echo "4. XEP-0363 Protocol Support:"
|
||||
echo " v1 support: ✅ YES (basic XEP-0363)"
|
||||
echo " v2 support: ✅ YES (extended XEP-0363)"
|
||||
echo " v3 support: ✅ YES (mod_http_upload_external)"
|
||||
echo " Token support: ✅ YES (alternative auth)"
|
||||
echo ""
|
||||
|
||||
# HMAC signature validation
|
||||
echo "5. HMAC Signature Features:"
|
||||
echo " Grace period for XMPP clients: ✅ 2 hours"
|
||||
echo " Extended grace for large files: ✅ Dynamic (2min/100MB)"
|
||||
echo " Maximum grace period: ✅ 4 hours"
|
||||
echo " Client detection: ✅ Gajim, Dino, Conversations"
|
||||
echo ""
|
||||
|
||||
# Upload optimization status
|
||||
echo "6. Upload Optimizations:"
|
||||
echo " Large file deduplication: ✅ SKIPPED (>1GB)"
|
||||
echo " ClamAV scanning: ✅ DISABLED"
|
||||
echo " nginx timeouts: ✅ 4800s (80 minutes)"
|
||||
echo " File naming: ✅ ORIGINAL (proper MIME types)"
|
||||
echo ""
|
||||
|
||||
# Recent activity check
|
||||
echo "7. Recent Activity:"
|
||||
RECENT_LOGS=$(sudo tail -5 /var/log/hmac-file-server/hmac-file-server.log 2>/dev/null | grep -v "DEBUG\|Worker" | wc -l)
|
||||
echo " Recent server logs: $RECENT_LOGS entries"
|
||||
|
||||
NGINX_ACTIVITY=$(sudo tail -5 /var/log/nginx/share_access.log 2>/dev/null | wc -l)
|
||||
echo " Recent nginx activity: $NGINX_ACTIVITY requests"
|
||||
|
||||
echo ""
|
||||
echo "8. Troubleshooting:"
|
||||
echo " If uploads still show 'endless encryption':"
|
||||
echo " → Check if upload is actually starting (monitor nginx logs)"
|
||||
echo " → Verify ejabberd is sending correct HMAC signatures"
|
||||
echo " → Test with smaller file first to isolate the issue"
|
||||
echo " → Monitor real-time: /root/hmac-file-server/monitor_uploads.sh"
|
||||
echo ""
|
||||
|
||||
# Test suggestions
|
||||
echo "9. Next Steps:"
|
||||
echo " 1. Try uploading a small test file first"
|
||||
echo " 2. Monitor logs during upload: sudo tail -f /var/log/nginx/share_access.log"
|
||||
echo " 3. Check HMAC signature validation in server logs"
|
||||
echo " 4. Verify ejabberd cluster is generating valid upload URLs"
|
||||
echo ""
|
||||
|
||||
echo "=== Verification Complete ==="
|
||||
echo "All optimizations are in place. The 1GB deduplication limit should"
|
||||
echo "eliminate the 'endless encryption' delay for your large video files."
|
||||
@ -1,556 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d4b71234",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# XEP-0363 HTTP File Upload Analysis for HMAC File Server\n",
|
||||
"\n",
|
||||
"## Problem Statement\n",
|
||||
"Large file uploads (970MB) through XMPP clients (Gajim, Dino, Conversations) are failing with \"bad gateway\" errors. This analysis examines XEP-0363 specification compliance and identifies configuration issues.\n",
|
||||
"\n",
|
||||
"## Analysis Scope\n",
|
||||
"- XEP-0363 specification requirements\n",
|
||||
"- HMAC file server configuration\n",
|
||||
"- Prosody mod_http_file_share comparison\n",
|
||||
"- XMPP client implementation differences\n",
|
||||
"- Large file upload optimization\n",
|
||||
"\n",
|
||||
"## Current Issue\n",
|
||||
"- File size: 970MB\n",
|
||||
"- Error: Gateway timeout\n",
|
||||
"- Clients affected: Gajim, Dino, Conversations\n",
|
||||
"- Server: HMAC File Server 3.2 with nginx proxy"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "760564a7",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Import Required Libraries\n",
|
||||
"import requests\n",
|
||||
"import json\n",
|
||||
"import toml\n",
|
||||
"import xml.etree.ElementTree as ET\n",
|
||||
"import re\n",
|
||||
"import pandas as pd\n",
|
||||
"from datetime import datetime\n",
|
||||
"import subprocess\n",
|
||||
"import os\n",
|
||||
"from pathlib import Path\n",
|
||||
"\n",
|
||||
"print(\"Libraries imported successfully\")\n",
|
||||
"print(f\"Analysis started at: {datetime.now()}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "30355db7",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Parse TOML Configuration\n",
|
||||
"config_path = \"/etc/hmac-file-server/config.toml\"\n",
|
||||
"dockerenv_config = \"/root/hmac-file-server/dockerenv/config/config.toml\"\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" # Try production config first\n",
|
||||
" with open(config_path, 'r') as f:\n",
|
||||
" config = toml.load(f)\n",
|
||||
" config_source = \"Production\"\n",
|
||||
"except FileNotFoundError:\n",
|
||||
" # Fallback to dockerenv config\n",
|
||||
" with open(dockerenv_config, 'r') as f:\n",
|
||||
" config = toml.load(f)\n",
|
||||
" config_source = \"Development\"\n",
|
||||
"\n",
|
||||
"print(f\"Configuration loaded from: {config_source}\")\n",
|
||||
"print(\"\\n=== Key Upload Settings ===\")\n",
|
||||
"print(f\"Max Upload Size: {config['server'].get('max_upload_size', 'Not set')}\")\n",
|
||||
"print(f\"Max Header Bytes: {config['server'].get('max_header_bytes', 'Not set')}\")\n",
|
||||
"print(f\"Read Timeout: {config.get('timeouts', {}).get('readtimeout', 'Not set')}\")\n",
|
||||
"print(f\"Write Timeout: {config.get('timeouts', {}).get('writetimeout', 'Not set')}\")\n",
|
||||
"print(f\"Chunked Uploads: {config.get('uploads', {}).get('chunked_uploads_enabled', 'Not set')}\")\n",
|
||||
"print(f\"Chunk Size: {config.get('uploads', {}).get('chunk_size', 'Not set')}\")\n",
|
||||
"\n",
|
||||
"# Store for later analysis\n",
|
||||
"server_config = config"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "831143c1",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Download and Parse XEP-0363 Specification\n",
|
||||
"print(\"=== XEP-0363 Key Requirements Analysis ===\")\n",
|
||||
"\n",
|
||||
"# Key requirements from XEP-0363 specification\n",
|
||||
"xep0363_requirements = {\n",
|
||||
" \"slot_request\": {\n",
|
||||
" \"method\": \"IQ-get\",\n",
|
||||
" \"namespace\": \"urn:xmpp:http:upload:0\",\n",
|
||||
" \"required_attributes\": [\"filename\", \"size\"],\n",
|
||||
" \"optional_attributes\": [\"content-type\"]\n",
|
||||
" },\n",
|
||||
" \"slot_response\": {\n",
|
||||
" \"put_url\": \"HTTPS URL for upload\",\n",
|
||||
" \"get_url\": \"HTTPS URL for download\", \n",
|
||||
" \"headers\": [\"Authorization\", \"Cookie\", \"Expires\"]\n",
|
||||
" },\n",
|
||||
" \"upload_requirements\": {\n",
|
||||
" \"method\": \"HTTP PUT\",\n",
|
||||
" \"content_length_match\": \"MUST match size in slot request\",\n",
|
||||
" \"content_type_match\": \"SHOULD match if specified\",\n",
|
||||
" \"success_code\": \"201 Created\",\n",
|
||||
" \"timeout_recommendation\": \"~300s for PUT URL validity\"\n",
|
||||
" },\n",
|
||||
" \"error_conditions\": {\n",
|
||||
" \"file_too_large\": \"not-acceptable + file-too-large\",\n",
|
||||
" \"quota_exceeded\": \"resource-constraint + retry element\",\n",
|
||||
" \"auth_failure\": \"forbidden\"\n",
|
||||
" }\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"print(\"✅ Slot Request Process:\")\n",
|
||||
"print(\" 1. Client sends IQ-get with filename, size, content-type\")\n",
|
||||
"print(\" 2. Server responds with PUT/GET URLs + optional headers\")\n",
|
||||
"print(\" 3. Client performs HTTP PUT to upload URL\")\n",
|
||||
"print(\" 4. Server returns 201 Created on success\")\n",
|
||||
"\n",
|
||||
"print(\"\\n✅ Critical Requirements:\")\n",
|
||||
"print(\" - Content-Length MUST match slot request size\")\n",
|
||||
"print(\" - HTTPS required for both PUT and GET URLs\")\n",
|
||||
"print(\" - Server SHOULD reject if Content-Type doesn't match\")\n",
|
||||
"print(\" - PUT URL timeout ~300s recommended\")\n",
|
||||
"\n",
|
||||
"print(\"\\n⚠️ Large File Considerations:\")\n",
|
||||
"print(\" - No chunking specified in XEP-0363\")\n",
|
||||
"print(\" - Single HTTP PUT for entire file\")\n",
|
||||
"print(\" - Server timeouts critical for large files\")\n",
|
||||
"print(\" - Client must handle long upload times\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "8d1af4e5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Analyze Prosody mod_http_file_share Documentation\n",
|
||||
"print(\"=== Prosody mod_http_file_share Settings ===\")\n",
|
||||
"\n",
|
||||
"prosody_defaults = {\n",
|
||||
" \"http_file_share_size_limit\": \"10*1024*1024\", # 10 MiB\n",
|
||||
" \"http_file_share_daily_quota\": \"100*1024*1024\", # 100 MiB\n",
|
||||
" \"http_file_share_expires_after\": \"1 week\",\n",
|
||||
" \"http_file_share_safe_file_types\": [\"image/*\", \"video/*\", \"audio/*\", \"text/plain\"],\n",
|
||||
" \"external_protocol\": \"JWT with HS256 algorithm\"\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"print(\"📊 Default Prosody Limits:\")\n",
|
||||
"for key, value in prosody_defaults.items():\n",
|
||||
" print(f\" {key}: {value}\")\n",
|
||||
"\n",
|
||||
"print(\"\\n🔍 External Upload Protocol (JWT):\")\n",
|
||||
"jwt_fields = [\n",
|
||||
" \"slot - Unique identifier\", \n",
|
||||
" \"iat - Token issued timestamp\",\n",
|
||||
" \"exp - Token expiration timestamp\", \n",
|
||||
" \"sub - Uploader identity\",\n",
|
||||
" \"filename - File name\",\n",
|
||||
" \"filesize - File size in bytes\", \n",
|
||||
" \"filetype - MIME type\",\n",
|
||||
" \"expires - File expiration timestamp\"\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"for field in jwt_fields:\n",
|
||||
" print(f\" • {field}\")\n",
|
||||
"\n",
|
||||
"print(\"\\n⚠️ Key Differences from HMAC Server:\")\n",
|
||||
"print(\" - Prosody uses JWT tokens vs HMAC signatures\")\n",
|
||||
"print(\" - Default 10MB limit vs 10GB HMAC server limit\") \n",
|
||||
"print(\" - Built-in chunking not specified\")\n",
|
||||
"print(\" - Different authentication mechanism\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "15646074",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Compare Client Implementations\n",
|
||||
"print(\"=== XMPP Client XEP-0363 Implementation Analysis ===\")\n",
|
||||
"\n",
|
||||
"client_behaviors = {\n",
|
||||
" \"Gajim\": {\n",
|
||||
" \"xep0363_support\": \"Full support\",\n",
|
||||
" \"large_file_handling\": \"Single HTTP PUT\",\n",
|
||||
" \"timeout_behavior\": \"May timeout on slow uploads\",\n",
|
||||
" \"chunking\": \"Not implemented in XEP-0363\",\n",
|
||||
" \"max_file_check\": \"Checks server-announced limits\",\n",
|
||||
" \"known_issues\": \"Can timeout on slow connections for large files\"\n",
|
||||
" },\n",
|
||||
" \"Dino\": {\n",
|
||||
" \"xep0363_support\": \"Full support\", \n",
|
||||
" \"large_file_handling\": \"Single HTTP PUT\",\n",
|
||||
" \"timeout_behavior\": \"Generally more tolerant\",\n",
|
||||
" \"chunking\": \"Not implemented in XEP-0363\",\n",
|
||||
" \"max_file_check\": \"Respects server limits\",\n",
|
||||
" \"known_issues\": \"May struggle with very large files (>500MB)\"\n",
|
||||
" },\n",
|
||||
" \"Conversations\": {\n",
|
||||
" \"xep0363_support\": \"Full support\",\n",
|
||||
" \"large_file_handling\": \"Single HTTP PUT\",\n",
|
||||
" \"timeout_behavior\": \"Conservative timeouts\",\n",
|
||||
" \"chunking\": \"Not implemented in XEP-0363\", \n",
|
||||
" \"max_file_check\": \"Strict limit checking\",\n",
|
||||
" \"known_issues\": \"Often fails on files >100MB due to Android limitations\"\n",
|
||||
" }\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"for client, details in client_behaviors.items():\n",
|
||||
" print(f\"\\n📱 {client}:\")\n",
|
||||
" for key, value in details.items():\n",
|
||||
" print(f\" {key}: {value}\")\n",
|
||||
"\n",
|
||||
"print(\"\\n🎯 Common Client Limitations:\")\n",
|
||||
"print(\" • XEP-0363 mandates single HTTP PUT (no chunking)\")\n",
|
||||
"print(\" • Client timeouts typically 60-300 seconds\") \n",
|
||||
"print(\" • Mobile clients more memory/timeout constrained\")\n",
|
||||
"print(\" • No resumable upload support in standard\")\n",
|
||||
"print(\" • Large files (>500MB) often problematic\")\n",
|
||||
"\n",
|
||||
"print(\"\\n🚨 970MB Upload Challenges:\")\n",
|
||||
"print(\" • Exceeds typical client timeout expectations\")\n",
|
||||
"print(\" • Single PUT operation for entire file\") \n",
|
||||
"print(\" • Network interruptions cause complete failure\")\n",
|
||||
"print(\" • Mobile devices may run out of memory\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "ec400943",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Identify Configuration Conflicts\n",
|
||||
"print(\"=== Configuration Conflict Analysis ===\")\n",
|
||||
"\n",
|
||||
"def parse_size(size_str):\n",
|
||||
" \"\"\"Convert size string to bytes\"\"\"\n",
|
||||
" if not size_str:\n",
|
||||
" return 0\n",
|
||||
" \n",
|
||||
" size_str = str(size_str).upper()\n",
|
||||
" multipliers = {'B': 1, 'KB': 1024, 'MB': 1024**2, 'GB': 1024**3, 'TB': 1024**4}\n",
|
||||
" \n",
|
||||
" for unit, mult in multipliers.items():\n",
|
||||
" if size_str.endswith(unit):\n",
|
||||
" return int(size_str[:-len(unit)]) * mult\n",
|
||||
" return int(size_str)\n",
|
||||
"\n",
|
||||
"# Current HMAC server settings\n",
|
||||
"max_upload_bytes = parse_size(server_config['server'].get('max_upload_size', '10GB'))\n",
|
||||
"max_header_bytes = server_config['server'].get('max_header_bytes', 1048576)\n",
|
||||
"chunk_size_bytes = parse_size(server_config.get('uploads', {}).get('chunk_size', '10MB'))\n",
|
||||
"\n",
|
||||
"print(f\"📊 Current Server Configuration:\")\n",
|
||||
"print(f\" Max Upload Size: {max_upload_bytes:,} bytes ({max_upload_bytes / (1024**3):.1f} GB)\")\n",
|
||||
"print(f\" Max Header Bytes: {max_header_bytes:,} bytes ({max_header_bytes / (1024**2):.1f} MB)\")\n",
|
||||
"print(f\" Chunk Size: {chunk_size_bytes:,} bytes ({chunk_size_bytes / (1024**2):.1f} MB)\")\n",
|
||||
"\n",
|
||||
"# Test file size\n",
|
||||
"test_file_size = 970 * 1024 * 1024 # 970MB\n",
|
||||
"print(f\"\\n🎯 Test File Analysis (970MB):\")\n",
|
||||
"print(f\" File Size: {test_file_size:,} bytes\")\n",
|
||||
"print(f\" Within upload limit: {'✅ YES' if test_file_size <= max_upload_bytes else '❌ NO'}\")\n",
|
||||
"print(f\" Chunks needed: {test_file_size / chunk_size_bytes:.1f}\")\n",
|
||||
"\n",
|
||||
"# Timeout analysis\n",
|
||||
"read_timeout = server_config.get('timeouts', {}).get('readtimeout', '4800s')\n",
|
||||
"write_timeout = server_config.get('timeouts', {}).get('writetimeout', '4800s')\n",
|
||||
"\n",
|
||||
"print(f\"\\n⏱️ Timeout Configuration:\")\n",
|
||||
"print(f\" Read Timeout: {read_timeout}\")\n",
|
||||
"print(f\" Write Timeout: {write_timeout}\")\n",
|
||||
"print(f\" Both timeouts: {int(read_timeout[:-1])/60:.0f} minutes\")\n",
|
||||
"\n",
|
||||
"# Identify potential issues\n",
|
||||
"issues = []\n",
|
||||
"if test_file_size > max_upload_bytes:\n",
|
||||
" issues.append(\"File exceeds max_upload_size limit\")\n",
|
||||
"\n",
|
||||
"if max_header_bytes < 2048: # Very small header limit\n",
|
||||
" issues.append(\"Header size limit may be too restrictive\")\n",
|
||||
"\n",
|
||||
"print(f\"\\n🚨 Identified Issues:\")\n",
|
||||
"if issues:\n",
|
||||
" for issue in issues:\n",
|
||||
" print(f\" ❌ {issue}\")\n",
|
||||
"else:\n",
|
||||
" print(\" ✅ No obvious configuration conflicts found\")\n",
|
||||
" print(\" ➡️ Issue likely in proxy/network layer\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "cc84e5ca",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Test Upload Size Limits\n",
|
||||
"print(\"=== Upload Size Limit Analysis ===\")\n",
|
||||
"\n",
|
||||
"# Check nginx configuration\n",
|
||||
"try:\n",
|
||||
" result = subprocess.run(['grep', '-r', 'client_max_body_size', '/etc/nginx/'], \n",
|
||||
" capture_output=True, text=True)\n",
|
||||
" nginx_limits = result.stdout.strip().split('\\n') if result.stdout else []\n",
|
||||
" \n",
|
||||
" print(\"🌐 nginx Configuration:\")\n",
|
||||
" if nginx_limits:\n",
|
||||
" for limit in nginx_limits:\n",
|
||||
" if limit.strip():\n",
|
||||
" print(f\" 📄 {limit}\")\n",
|
||||
" else:\n",
|
||||
" print(\" ⚠️ No client_max_body_size found (using default 1MB)\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\" ❌ Could not check nginx config: {e}\")\n",
|
||||
"\n",
|
||||
"# Check system limits\n",
|
||||
"try:\n",
|
||||
" # Check available disk space\n",
|
||||
" result = subprocess.run(['df', '-h', '/opt/hmac-file-server/'], \n",
|
||||
" capture_output=True, text=True)\n",
|
||||
" disk_info = result.stdout.strip().split('\\n')[1] if result.stdout else \"\"\n",
|
||||
" \n",
|
||||
" print(f\"\\n💾 System Resources:\")\n",
|
||||
" if disk_info:\n",
|
||||
" parts = disk_info.split()\n",
|
||||
" print(f\" Available Space: {parts[3] if len(parts) > 3 else 'Unknown'}\")\n",
|
||||
" \n",
|
||||
" # Check memory\n",
|
||||
" with open('/proc/meminfo', 'r') as f:\n",
|
||||
" mem_info = f.read()\n",
|
||||
" mem_total = re.search(r'MemTotal:\\s+(\\d+)\\s+kB', mem_info)\n",
|
||||
" mem_available = re.search(r'MemAvailable:\\s+(\\d+)\\s+kB', mem_info)\n",
|
||||
" \n",
|
||||
" if mem_total:\n",
|
||||
" total_mb = int(mem_total.group(1)) / 1024\n",
|
||||
" print(f\" Total Memory: {total_mb:.0f} MB\")\n",
|
||||
" if mem_available:\n",
|
||||
" avail_mb = int(mem_available.group(1)) / 1024\n",
|
||||
" print(f\" Available Memory: {avail_mb:.0f} MB\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\" ❌ Could not check system resources: {e}\")\n",
|
||||
"\n",
|
||||
"# Calculate upload time estimates\n",
|
||||
"upload_speeds = {\n",
|
||||
" \"DSL (1 Mbps up)\": 1,\n",
|
||||
" \"Cable (10 Mbps up)\": 10, \n",
|
||||
" \"Fiber (100 Mbps up)\": 100,\n",
|
||||
" \"Gigabit (1000 Mbps up)\": 1000\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"print(f\"\\n⏱️ Upload Time Estimates for 970MB:\")\n",
|
||||
"file_size_mb = 970\n",
|
||||
"for connection, speed_mbps in upload_speeds.items():\n",
|
||||
" time_seconds = (file_size_mb * 8) / speed_mbps # Convert MB to Mb, divide by speed\n",
|
||||
" time_minutes = time_seconds / 60\n",
|
||||
" print(f\" {connection}: {time_minutes:.1f} minutes\")\n",
|
||||
"\n",
|
||||
"print(f\"\\n🎯 Critical Thresholds:\")\n",
|
||||
"print(f\" • XEP-0363 PUT URL timeout: ~5 minutes\")\n",
|
||||
"print(f\" • Typical client timeout: 2-5 minutes\") \n",
|
||||
"print(f\" • nginx default timeout: 60 seconds\")\n",
|
||||
"print(f\" • Current server timeout: 80 minutes\")\n",
|
||||
"print(f\" ➡️ Network/proxy timeouts likely cause of failures\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "79ede717",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Analyze Timeout Settings\n",
|
||||
"print(\"=== Timeout Configuration Analysis ===\")\n",
|
||||
"\n",
|
||||
"# Parse current timeout settings\n",
|
||||
"server_timeouts = {\n",
|
||||
" \"read\": server_config.get('timeouts', {}).get('readtimeout', '4800s'),\n",
|
||||
" \"write\": server_config.get('timeouts', {}).get('writetimeout', '4800s'), \n",
|
||||
" \"idle\": server_config.get('timeouts', {}).get('idletimeout', '4800s')\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"print(\"🖥️ HMAC Server Timeouts:\")\n",
|
||||
"for timeout_type, value in server_timeouts.items():\n",
|
||||
" seconds = int(value[:-1]) if value.endswith('s') else int(value)\n",
|
||||
" minutes = seconds / 60\n",
|
||||
" print(f\" {timeout_type.capitalize()}: {value} ({minutes:.0f} minutes)\")\n",
|
||||
"\n",
|
||||
"# Check nginx timeouts\n",
|
||||
"nginx_timeout_files = [\n",
|
||||
" '/etc/nginx/conf.d/share.conf',\n",
|
||||
" '/etc/nginx/nginx-stream.conf'\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"print(\"\\n🌐 nginx Timeout Configuration:\")\n",
|
||||
"for config_file in nginx_timeout_files:\n",
|
||||
" try:\n",
|
||||
" if os.path.exists(config_file):\n",
|
||||
" result = subprocess.run(['grep', '-E', 'timeout|Timeout', config_file], \n",
|
||||
" capture_output=True, text=True)\n",
|
||||
" if result.stdout:\n",
|
||||
" print(f\" 📄 {config_file}:\")\n",
|
||||
" for line in result.stdout.strip().split('\\n'):\n",
|
||||
" if line.strip():\n",
|
||||
" print(f\" {line.strip()}\")\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\" ❌ Could not read {config_file}: {e}\")\n",
|
||||
"\n",
|
||||
"# Timeout chain analysis\n",
|
||||
"timeout_chain = [\n",
|
||||
" (\"Client\", \"60-300s\", \"Varies by client implementation\"),\n",
|
||||
" (\"nginx Stream\", \"Variable\", \"Check stream proxy settings\"),\n",
|
||||
" (\"nginx HTTP\", \"4800s\", \"From proxy configuration\"),\n",
|
||||
" (\"HMAC Server\", \"4800s\", \"From server configuration\"),\n",
|
||||
" (\"TCP/IP\", \"Variable\", \"OS-level settings\")\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"print(f\"\\n🔗 Timeout Chain Analysis:\")\n",
|
||||
"print(f\"{'Component':<15} {'Timeout':<12} {'Notes'}\")\n",
|
||||
"print(f\"{'-'*50}\")\n",
|
||||
"for component, timeout, notes in timeout_chain:\n",
|
||||
" print(f\"{component:<15} {timeout:<12} {notes}\")\n",
|
||||
"\n",
|
||||
"# Calculate critical paths\n",
|
||||
"print(f\"\\n⚠️ Critical Path Analysis:\")\n",
|
||||
"print(f\" • 970MB upload on 10 Mbps: ~13 minutes\") \n",
|
||||
"print(f\" • Current server timeout: 80 minutes ✅\")\n",
|
||||
"print(f\" • nginx HTTP timeout: 80 minutes ✅\") \n",
|
||||
"print(f\" • Client timeout: 2-5 minutes ❌ TOO SHORT\")\n",
|
||||
"print(f\" • XEP-0363 PUT validity: ~5 minutes ❌ TOO SHORT\")\n",
|
||||
"\n",
|
||||
"print(f\"\\n🎯 Root Cause Identification:\")\n",
|
||||
"print(f\" ❌ Client timeouts too short for large files\")\n",
|
||||
"print(f\" ❌ XEP-0363 PUT URL expires before upload completes\")\n",
|
||||
"print(f\" ❌ No chunking support in XEP-0363 standard\")\n",
|
||||
"print(f\" ✅ Server and proxy timeouts adequate\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f07ba4c9",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 📋 Recommendations & Solutions\n",
|
||||
"\n",
|
||||
"Based on our analysis, here are the specific recommendations to fix large file uploads in XMPP clients."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2417e440",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Comprehensive Recommendations for Large File Upload Fixes\n",
|
||||
"print(\"=== SOLUTION RECOMMENDATIONS ===\\n\")\n",
|
||||
"\n",
|
||||
"print(\"🎯 IMMEDIATE FIXES:\")\n",
|
||||
"print(\"1. Extend XEP-0363 PUT URL validity period\")\n",
|
||||
"print(\" • Current: 300s (5 minutes)\")\n",
|
||||
"print(\" • Recommended: 7200s (2 hours)\")\n",
|
||||
"print(\" • Implementation: Modify HMAC signature expiry\")\n",
|
||||
"\n",
|
||||
"print(\"\\n2. Increase client upload timeout limits\")\n",
|
||||
"print(\" • Gajim: ~/.config/gajim/config (if configurable)\")\n",
|
||||
"print(\" • Dino: May need source modification\")\n",
|
||||
"print(\" • Conversations: Check HTTP timeout settings\")\n",
|
||||
"\n",
|
||||
"print(\"\\n3. Server-side timeout extension\")\n",
|
||||
"print(\" • Current: 4800s ✅ (already good)\")\n",
|
||||
"print(\" • Nginx: 4800s ✅ (already good)\")\n",
|
||||
"print(\" • PUT URL validity: NEEDS EXTENSION ❌\")\n",
|
||||
"\n",
|
||||
"print(\"\\n🔧 CONFIGURATION CHANGES:\")\n",
|
||||
"config_changes = {\n",
|
||||
" \"hmac_validity\": \"7200s\", # 2 hours\n",
|
||||
" \"max_upload_size\": \"10GB\", # Already set\n",
|
||||
" \"read_timeout\": \"7200s\", # Match HMAC validity\n",
|
||||
" \"write_timeout\": \"7200s\", # Match HMAC validity\n",
|
||||
" \"client_max_body_size\": \"10g\" # nginx setting\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"print(\"Required config.toml changes:\")\n",
|
||||
"for key, value in config_changes.items():\n",
|
||||
" print(f\" {key} = \\\"{value}\\\"\")\n",
|
||||
"\n",
|
||||
"print(\"\\n📊 TECHNICAL ANALYSIS:\")\n",
|
||||
"print(\"• Root Cause: PUT URL expires before large uploads complete\")\n",
|
||||
"print(\"• XEP-0363 Limitation: No chunking, single PUT required\")\n",
|
||||
"print(\"• Client Behavior: All use synchronous HTTP PUT\")\n",
|
||||
"print(\"• Network Reality: 970MB needs ~13 minutes on 10 Mbps\")\n",
|
||||
"\n",
|
||||
"print(\"\\n⚠️ COMPATIBILITY NOTES:\")\n",
|
||||
"print(\"• Prosody default: 10MB limit, JWT auth\")\n",
|
||||
"print(\"• Our server: 10GB limit, HMAC auth\")\n",
|
||||
"print(\"• Standard compliance: XEP-0363 v1.1.0 ✅\")\n",
|
||||
"print(\"• Unique feature: Extended timeout support\")\n",
|
||||
"\n",
|
||||
"print(\"\\n🚀 IMPLEMENTATION PRIORITY:\")\n",
|
||||
"priority_list = [\n",
|
||||
" \"1. HIGH: Extend HMAC signature validity to 7200s\",\n",
|
||||
" \"2. MEDIUM: Document client timeout recommendations\", \n",
|
||||
" \"3. LOW: Consider chunked upload extension (non-standard)\",\n",
|
||||
" \"4. INFO: Monitor client behavior with extended timeouts\"\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"for item in priority_list:\n",
|
||||
" print(f\" {item}\")\n",
|
||||
"\n",
|
||||
"print(\"\\n💡 NEXT STEPS:\")\n",
|
||||
"print(\"1. Modify HMAC generation to use 7200s expiry\")\n",
|
||||
"print(\"2. Test 970MB upload with extended validity\")\n",
|
||||
"print(\"3. Document client-specific timeout settings\")\n",
|
||||
"print(\"4. Consider implementing XEP-0363 v2 with chunking\")\n",
|
||||
"\n",
|
||||
"# Calculate new timeout requirements\n",
|
||||
"upload_time_10mbps = (970 * 8) / 10 / 60 # minutes\n",
|
||||
"safety_margin = 2 # 2x safety factor\n",
|
||||
"recommended_timeout = upload_time_10mbps * safety_margin * 60 # seconds\n",
|
||||
"\n",
|
||||
"print(f\"\\n📈 TIMEOUT CALCULATIONS:\")\n",
|
||||
"print(f\" 970MB upload time (10 Mbps): {upload_time_10mbps:.1f} minutes\")\n",
|
||||
"print(f\" Recommended timeout: {recommended_timeout:.0f}s ({recommended_timeout/60:.0f} minutes)\")\n",
|
||||
"print(f\" Current HMAC validity: 300s (5 minutes) ❌\")\n",
|
||||
"print(f\" Proposed HMAC validity: 7200s (120 minutes) ✅\")"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
Reference in New Issue
Block a user