🔥 Tremora del Terra: ultimate hmac-file-server fix – final push before the drop 💾🔐

This commit is contained in:
2025-07-18 16:48:16 +00:00
parent 275ef6c031
commit 6d7042059b
8 changed files with 305 additions and 326 deletions

View File

@ -1,6 +1,6 @@
#!/bin/bash #!/bin/bash
# HMAC File Server v3.2 - Multi-Architecture Build Script # HMAC File Server v3.2 - Multi-Architecture Build Script
# Compiles binaries for AMD64, ARM64, and ARM32 architectures # Compiles binaries for AMD64, ARM64, ARM32, Windows, and macOS architectures
# Remove set -e to prevent early exit on errors # Remove set -e to prevent early exit on errors
@ -45,13 +45,57 @@ if [[ ! -d "$TEMP_DIR" ]]; then
print_info "Created temp directory: $TEMP_DIR" print_info "Created temp directory: $TEMP_DIR"
fi fi
# Source files to compile # Source directory to compile
SOURCE_FILES="cmd/server/main.go cmd/server/helpers.go cmd/server/config_validator.go cmd/server/config_test_scenarios.go" SOURCE_DIR="./cmd/server/"
print_status "Starting multi-architecture build for HMAC File Server v3.2" # Interactive menu function
print_info "Source files: $SOURCE_FILES" show_menu() {
print_info "Output directory: $TEMP_DIR" echo ""
echo "" echo "HMAC File Server Multi-Architecture Builder"
echo "=========================================="
echo "1) Build for current platform (auto-detect)"
echo "2) Build for Linux AMD64"
echo "3) Build for Linux ARM64"
echo "4) Build for Linux ARM32v7"
echo "5) Build for Windows AMD64"
echo "6) Build for macOS AMD64 (Intel)"
echo "7) Build for macOS ARM64 (Apple Silicon)"
echo "8) Build all supported architectures"
echo "9) Clean build artifacts"
echo "0) Exit"
echo ""
read -p "Choose an option [0-9]: " choice
}
# Clean function
clean_artifacts() {
print_info "Cleaning build artifacts..."
if [[ -d "$TEMP_DIR" ]]; then
rm -rf "$TEMP_DIR"/*
print_status "Build artifacts cleaned"
else
print_info "No artifacts to clean"
fi
}
# Detect current platform
detect_platform() {
local os=$(uname -s | tr '[:upper:]' '[:lower:]')
local arch=$(uname -m)
case "$arch" in
x86_64) arch="amd64" ;;
arm64|aarch64) arch="arm64" ;;
armv7l) arch="arm" ;;
*) arch="unknown" ;;
esac
case "$os" in
linux) echo "linux/$arch" ;;
darwin) echo "darwin/$arch" ;;
*) echo "unknown/unknown" ;;
esac
}
# Build function # Build function
build_for_arch() { build_for_arch() {
@ -68,7 +112,7 @@ build_for_arch() {
export CGO_ENABLED=0 export CGO_ENABLED=0
# Build the binary # Build the binary
if go build -ldflags="-w -s" -o "$TEMP_DIR/$output_name" $SOURCE_FILES 2>/dev/null; then if go build -ldflags="-w -s" -o "$TEMP_DIR/$output_name" $SOURCE_DIR 2>/dev/null; then
# Get file size # Get file size
if [[ "$OSTYPE" == "darwin"* ]]; then if [[ "$OSTYPE" == "darwin"* ]]; then
# macOS # macOS
@ -92,64 +136,171 @@ build_for_arch() {
return 0 return 0
else else
print_error "Build failed: $arch_description" print_error "Build failed: $arch_description"
if [[ "$goos" == "windows" ]]; then
print_warning " Windows builds may fail due to platform-specific code (syscalls)"
print_info " Consider using Linux subsystem or implementing Windows-specific storage checks"
fi
return 1 return 1
fi fi
} }
# Track build results # Build all architectures function
BUILDS_ATTEMPTED=0 build_all_architectures() {
BUILDS_SUCCESSFUL=0 print_status "Starting multi-architecture build for HMAC File Server v3.2"
FAILED_BUILDS=() print_info "Source directory: $SOURCE_DIR"
print_info "Output directory: $TEMP_DIR"
echo ""
echo "Starting builds..." # Track build results
echo "====================" BUILDS_ATTEMPTED=0
echo "" BUILDS_SUCCESSFUL=0
FAILED_BUILDS=()
# Build for AMD64 (x86_64) echo "Starting builds..."
print_arch "AMD64 (Intel/AMD 64-bit)" echo "===================="
((BUILDS_ATTEMPTED++)) echo ""
if build_for_arch "linux" "amd64" "hmac-file-server-linux-amd64" "AMD64 Linux"; then
# Build for AMD64 (x86_64)
print_arch "AMD64 (Intel/AMD 64-bit)"
((BUILDS_ATTEMPTED++))
if build_for_arch "linux" "amd64" "hmac-file-server-linux-amd64" "AMD64 Linux"; then
((BUILDS_SUCCESSFUL++)) ((BUILDS_SUCCESSFUL++))
else else
FAILED_BUILDS+=("AMD64") FAILED_BUILDS+=("AMD64")
fi fi
echo "" echo ""
# Build for ARM64 (AArch64) # Build for ARM64 (AArch64)
print_arch "ARM64 (AArch64)" print_arch "ARM64 (AArch64)"
((BUILDS_ATTEMPTED++)) ((BUILDS_ATTEMPTED++))
if build_for_arch "linux" "arm64" "hmac-file-server-linux-arm64" "ARM64 Linux"; then if build_for_arch "linux" "arm64" "hmac-file-server-linux-arm64" "ARM64 Linux"; then
((BUILDS_SUCCESSFUL++)) ((BUILDS_SUCCESSFUL++))
else else
FAILED_BUILDS+=("ARM64") FAILED_BUILDS+=("ARM64")
fi fi
echo "" echo ""
# Build for ARM32 (ARMv7) # Build for ARM32 (ARMv7)
print_arch "ARM32 (ARMv7)" print_arch "ARM32 (ARMv7)"
export GOARM=7 # ARMv7 with hardware floating point export GOARM=7 # ARMv7 with hardware floating point
((BUILDS_ATTEMPTED++)) ((BUILDS_ATTEMPTED++))
if build_for_arch "linux" "arm" "hmac-file-server-linux-arm32" "ARM32 Linux"; then if build_for_arch "linux" "arm" "hmac-file-server-linux-arm32v7" "ARM32 Linux"; then
((BUILDS_SUCCESSFUL++)) ((BUILDS_SUCCESSFUL++))
else else
FAILED_BUILDS+=("ARM32") FAILED_BUILDS+=("ARM32")
fi fi
echo "" echo ""
# Reset environment variables # Build for Windows AMD64
unset GOOS GOARCH CGO_ENABLED GOARM print_arch "Windows AMD64"
((BUILDS_ATTEMPTED++))
if build_for_arch "windows" "amd64" "hmac-file-server-windows-amd64.exe" "Windows AMD64"; then
((BUILDS_SUCCESSFUL++))
else
FAILED_BUILDS+=("Windows")
fi
echo ""
# Build summary # Build for macOS Intel
echo "Build Summary" print_arch "macOS Intel"
echo "================" ((BUILDS_ATTEMPTED++))
print_info "Builds attempted: $BUILDS_ATTEMPTED" if build_for_arch "darwin" "amd64" "hmac-file-server-darwin-amd64" "macOS Intel"; then
print_info "Builds successful: $BUILDS_SUCCESSFUL" ((BUILDS_SUCCESSFUL++))
else
FAILED_BUILDS+=("macOS Intel")
fi
echo ""
if [[ $BUILDS_SUCCESSFUL -eq $BUILDS_ATTEMPTED ]]; then # Build for macOS Apple Silicon
print_arch "macOS Apple Silicon"
((BUILDS_ATTEMPTED++))
if build_for_arch "darwin" "arm64" "hmac-file-server-darwin-arm64" "macOS Apple Silicon"; then
((BUILDS_SUCCESSFUL++))
else
FAILED_BUILDS+=("macOS ARM64")
fi
echo ""
# Reset environment variables
unset GOOS GOARCH CGO_ENABLED GOARM
show_build_summary
}
# Build single architecture function
build_single_arch() {
local platform_desc=$1
local goos=$2
local goarch=$3
local goarm=$4
local output_name=$5
print_status "Building for $platform_desc"
print_info "Source directory: $SOURCE_DIR"
print_info "Output directory: $TEMP_DIR"
echo ""
if [[ -n "$goarm" ]]; then
export GOARM=$goarm
fi
BUILDS_ATTEMPTED=1
BUILDS_SUCCESSFUL=0
FAILED_BUILDS=()
if build_for_arch "$goos" "$goarch" "$output_name" "$platform_desc"; then
BUILDS_SUCCESSFUL=1
else
FAILED_BUILDS+=("$platform_desc")
fi
unset GOOS GOARCH CGO_ENABLED GOARM
show_build_summary
}
# Build current platform function
build_current_platform() {
local platform=$(detect_platform)
local goos=$(echo "$platform" | cut -d'/' -f1)
local goarch=$(echo "$platform" | cut -d'/' -f2)
case "$platform" in
"linux/amd64")
build_single_arch "Current Platform (Linux AMD64)" "linux" "amd64" "" "hmac-file-server-linux-amd64"
;;
"linux/arm64")
build_single_arch "Current Platform (Linux ARM64)" "linux" "arm64" "" "hmac-file-server-linux-arm64"
;;
"linux/arm")
build_single_arch "Current Platform (Linux ARM32v7)" "linux" "arm" "7" "hmac-file-server-linux-arm32v7"
;;
"darwin/amd64")
build_single_arch "Current Platform (macOS Intel)" "darwin" "amd64" "" "hmac-file-server-darwin-amd64"
;;
"darwin/arm64")
build_single_arch "Current Platform (macOS Apple Silicon)" "darwin" "arm64" "" "hmac-file-server-darwin-arm64"
;;
*)
print_error "Unsupported platform: $platform"
print_info "Supported platforms: linux/amd64, linux/arm64, linux/arm, darwin/amd64, darwin/arm64"
exit 1
;;
esac
}
# Show build summary
show_build_summary() {
# Build summary
echo "Build Summary"
echo "================"
print_info "Builds attempted: $BUILDS_ATTEMPTED"
print_info "Builds successful: $BUILDS_SUCCESSFUL"
if [[ $BUILDS_SUCCESSFUL -eq $BUILDS_ATTEMPTED ]]; then
print_status "ALL BUILDS SUCCESSFUL!" print_status "ALL BUILDS SUCCESSFUL!"
echo "" echo ""
print_info "Generated binaries in $TEMP_DIR:" print_info "Generated binaries in $TEMP_DIR:"
ls -lh "$TEMP_DIR"/hmac-file-server-* | while read -r line; do ls -lh "$TEMP_DIR"/hmac-file-server-* 2>/dev/null | while read -r line; do
echo " $line" echo " $line"
done done
echo "" echo ""
@ -158,30 +309,32 @@ if [[ $BUILDS_SUCCESSFUL -eq $BUILDS_ATTEMPTED ]]; then
echo " - Deploy with installer: cp temp/hmac-file-server-linux-amd64 /opt/hmac-file-server/" echo " - Deploy with installer: cp temp/hmac-file-server-linux-amd64 /opt/hmac-file-server/"
echo " - Docker deployment: COPY temp/hmac-file-server-linux-amd64 /usr/local/bin/" echo " - Docker deployment: COPY temp/hmac-file-server-linux-amd64 /usr/local/bin/"
elif [[ $BUILDS_SUCCESSFUL -gt 0 ]]; then elif [[ $BUILDS_SUCCESSFUL -gt 0 ]]; then
print_warning "PARTIAL SUCCESS: $BUILDS_SUCCESSFUL/$BUILDS_ATTEMPTED builds completed" print_warning "PARTIAL SUCCESS: $BUILDS_SUCCESSFUL/$BUILDS_ATTEMPTED builds completed"
if [[ ${#FAILED_BUILDS[@]} -gt 0 ]]; then if [[ ${#FAILED_BUILDS[@]} -gt 0 ]]; then
print_error "Failed architectures: ${FAILED_BUILDS[*]}" print_error "Failed architectures: ${FAILED_BUILDS[*]}"
fi fi
else else
print_error "ALL BUILDS FAILED!" print_error "ALL BUILDS FAILED!"
exit 1 exit 1
fi fi
echo "" echo ""
print_info "Architecture compatibility:" print_info "Architecture compatibility:"
echo " - AMD64: Intel/AMD 64-bit servers, desktops, cloud instances" echo " - AMD64: Intel/AMD 64-bit servers, desktops, cloud instances"
echo " - ARM64: Apple Silicon, AWS Graviton, modern ARM servers" echo " - ARM64: Apple Silicon, AWS Graviton, modern ARM servers"
echo " - ARM32: Raspberry Pi, embedded systems, older ARM devices" echo " - ARM32: Raspberry Pi, embedded systems, older ARM devices"
echo " - Windows: Windows 10/11, Windows Server"
echo " - macOS: macOS 10.15+, Intel and Apple Silicon"
echo "" echo ""
print_status "Multi-architecture build completed!" print_status "Build completed!"
# Final verification # Final verification
echo "" echo ""
print_info "Final verification:" print_info "Final verification:"
for binary in "$TEMP_DIR"/hmac-file-server-*; do for binary in "$TEMP_DIR"/hmac-file-server-*; do
if [[ -f "$binary" ]]; then if [[ -f "$binary" ]]; then
filename=$(basename "$binary") filename=$(basename "$binary")
if file "$binary" >/dev/null 2>&1; then if file "$binary" >/dev/null 2>&1; then
@ -191,6 +344,62 @@ for binary in "$TEMP_DIR"/hmac-file-server-*; do
print_info " OK $filename: Binary file" print_info " OK $filename: Binary file"
fi fi
fi fi
done done
}
# Main execution
if [[ $# -eq 0 ]]; then
# Interactive mode
while true; do
show_menu
case $choice in
1)
build_current_platform
break
;;
2)
build_single_arch "Linux AMD64" "linux" "amd64" "" "hmac-file-server-linux-amd64"
break
;;
3)
build_single_arch "Linux ARM64" "linux" "arm64" "" "hmac-file-server-linux-arm64"
break
;;
4)
build_single_arch "Linux ARM32v7" "linux" "arm" "7" "hmac-file-server-linux-arm32v7"
break
;;
5)
build_single_arch "Windows AMD64" "windows" "amd64" "" "hmac-file-server-windows-amd64.exe"
break
;;
6)
build_single_arch "macOS Intel" "darwin" "amd64" "" "hmac-file-server-darwin-amd64"
break
;;
7)
build_single_arch "macOS Apple Silicon" "darwin" "arm64" "" "hmac-file-server-darwin-arm64"
break
;;
8)
build_all_architectures
break
;;
9)
clean_artifacts
;;
0)
print_info "Exiting build script"
exit 0
;;
*)
print_error "Invalid option. Please choose 0-9."
;;
esac
done
else
# Non-interactive mode - build all architectures
build_all_architectures
fi
exit 0 exit 0

View File

@ -1,111 +0,0 @@
[server]
listen_address = ":8080"
storage_path = "/srv/hmac-file-server/uploads"
metrics_enabled = true
metrics_path = "/metrics"
pid_file = "/var/run/hmac-file-server.pid"
max_upload_size = "10GB" # Supports B, KB, MB, GB, TB
max_header_bytes = 1048576 # 1MB
cleanup_interval = "24h"
max_file_age = "720h" # 30 days
pre_cache = true
pre_cache_workers = 4
pre_cache_interval = "1h"
global_extensions = [".txt", ".dat", ".iso", ".mp4", ".mkv", ".avi", ".mov", ".wmv", ".flv", ".webm", ".mpeg"] # If set, overrides upload/download extensions
deduplication_enabled = true
min_free_bytes = "1GB" # Minimum free space required for uploads
file_naming = "original" # Options: "original", "HMAC"
force_protocol = "" # Options: "http", "https" - if set, redirects to this protocol
enable_dynamic_workers = true # Enable dynamic worker scaling
worker_scale_up_thresh = 50 # Queue length to scale up workers
worker_scale_down_thresh = 10 # Queue length to scale down workers
# Cluster-aware settings for client restart resilience
graceful_shutdown_timeout = "300s" # Allow time for client reconnections
connection_drain_timeout = "120s" # Drain existing connections gracefully
max_idle_conns_per_host = 5 # Limit persistent connections per client
idle_conn_timeout = "90s" # Close idle connections regularly
disable_keep_alives = false # Keep HTTP keep-alives for performance
client_timeout = "300s" # Timeout for slow clients
restart_grace_period = "60s" # Grace period after restart for clients to reconnect
[uploads]
allowed_extensions = [".zip", ".rar", ".7z", ".tar.gz", ".tgz", ".gpg", ".enc", ".pgp"]
chunked_uploads_enabled = true
chunk_size = "10MB"
resumable_uploads_enabled = true
max_resumable_age = "48h"
# Cluster resilience for uploads
session_persistence = true # Persist upload sessions across restarts
session_recovery_timeout = "300s" # Time to wait for session recovery
client_reconnect_window = "120s" # Window for clients to reconnect after server restart
upload_slot_ttl = "3600s" # Upload slot validity time
retry_failed_uploads = true # Automatically retry failed uploads
max_upload_retries = 3 # Maximum retry attempts
[downloads]
resumable_downloads_enabled = true
chunked_downloads_enabled = true
chunk_size = "8192"
allowed_extensions = [".txt", ".pdf", ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".svg", ".webp"]
[security]
secret = "f6g4ldPvQM7O2UTFeBEUUj33VrXypDAcsDt0yqKrLiOr5oQW"
enablejwt = false
jwtsecret = "f6g4ldPvQM7O2UTFeBEUUj33VrXypDAcsDt0yqKrLiOr5oQW"
jwtalgorithm = "HS256"
jwtexpiration = "24h"
[logging]
level = "info"
file = "/var/log/hmac-file-server.log"
max_size = 100
max_backups = 7
max_age = 30
compress = true
[deduplication]
enabled = true
directory = "./deduplication"
maxsize = "1GB"
[iso]
enabled = true
size = "1GB"
mountpoint = "/mnt/iso"
charset = "utf-8"
containerfile = "/mnt/iso/container.iso"
[timeouts]
readtimeout = "4800s"
writetimeout = "4800s"
idletimeout = "4800s"
[versioning]
enableversioning = false
maxversions = 1
[clamav]
clamavenabled = true
clamavsocket = "/var/run/clamav/clamd.ctl"
numscanworkers = 2
# Only scan potentially dangerous file types, skip large media files
scanfileextensions = [".txt", ".pdf", ".doc", ".docx", ".xls", ".xlsx", ".exe", ".zip", ".rar", ".7z", ".tar", ".gz"]
# Skip scanning files larger than 200MB (ClamAV limit)
maxscansize = "200MB"
[redis]
redisenabled = true
redisdbindex = 0
redisaddr = "localhost:6379"
redispassword = ""
redishealthcheckinterval = "120s"
[workers]
numworkers = 4
uploadqueuesize = 50
[file]
# Add file-specific configurations here
[build]
version = "3.2"

View File

@ -5,7 +5,7 @@ services:
container_name: hmac-file-server container_name: hmac-file-server
image: hmac-file-server:latest image: hmac-file-server:latest
ports: ports:
- "8080:8080" - "8081:8080"
volumes: volumes:
- ./config:/etc/hmac-file-server - ./config:/etc/hmac-file-server
- ./data/uploads:/opt/hmac-file-server/data/uploads - ./data/uploads:/opt/hmac-file-server/data/uploads

View File

@ -6,7 +6,7 @@ RUN apk add --no-cache git
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN go mod download
COPY . . COPY . .
RUN CGO_ENABLED=0 go build -o hmac-file-server cmd/server/main.go cmd/server/helpers.go cmd/server/config_validator.go cmd/server/config_test_scenarios.go RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o hmac-file-server ./cmd/server/
# Stage 2: Runtime # Stage 2: Runtime
FROM alpine:latest FROM alpine:latest

Binary file not shown.

Before

Width:  |  Height:  |  Size: 60 KiB

View File

@ -1,80 +0,0 @@
package main
import (
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"fmt"
"mime"
"net/http"
"net/url"
"os"
"path/filepath" // Added this import for filepath usage
"strconv"
"testing"
)
const (
serverURL = "http://[::1]:8080" // Replace with your actual server URL
secret = "hmac-file-server-is-the-win" // Replace with your HMAC secret key
uploadPath = "hmac_icon.png" // Test file to upload
protocolType = "v2" // Use v2, v, or token as needed
)
// TestUpload performs a basic HMAC validation and upload test.
func TestUpload(t *testing.T) {
// File setup for testing
file, err := os.Open(uploadPath)
if err != nil {
t.Fatalf("Error opening file: %v", err)
}
defer file.Close()
fileInfo, _ := file.Stat()
fileStorePath := uploadPath
contentLength := fileInfo.Size()
// Generate HMAC based on protocol type
hmacValue := generateHMAC(fileStorePath, contentLength, protocolType)
// Formulate request URL with HMAC in query params
reqURL := fmt.Sprintf("%s/%s?%s=%s", serverURL, fileStorePath, protocolType, url.QueryEscape(hmacValue))
// Prepare HTTP PUT request with file data
req, err := http.NewRequest(http.MethodPut, reqURL, file)
if err != nil {
t.Fatalf("Error creating request: %v", err)
}
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Content-Length", strconv.FormatInt(contentLength, 10))
// Execute HTTP request
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("Error executing request: %v", err)
}
defer resp.Body.Close()
t.Logf("Response status: %s", resp.Status)
}
// Generates the HMAC based on your protocol version
func generateHMAC(filePath string, contentLength int64, protocol string) string {
mac := hmac.New(sha256.New, []byte(secret))
macString := ""
// Calculate HMAC according to protocol
if protocol == "v" {
mac.Write([]byte(filePath + "\x20" + strconv.FormatInt(contentLength, 10)))
macString = hex.EncodeToString(mac.Sum(nil))
} else if protocol == "v2" || protocol == "token" {
contentType := mime.TypeByExtension(filepath.Ext(filePath))
if contentType == "" {
contentType = "application/octet-stream"
}
mac.Write([]byte(filePath + "\x00" + strconv.FormatInt(contentLength, 10) + "\x00" + contentType))
macString = hex.EncodeToString(mac.Sum(nil))
}
return macString
}

View File

@ -1,39 +0,0 @@
package main
import (
"os"
"os/exec"
"strings"
"testing"
)
// TestGenConfigFlag runs the server with --genconfig and checks output for expected config keys
func TestGenConfigFlag(t *testing.T) {
cmd := exec.Command("go", "run", "../cmd/server/main.go", "--genconfig")
output, err := cmd.CombinedOutput()
if err != nil && !strings.Contains(string(output), "[server]") {
t.Fatalf("Failed to run with --genconfig: %v\nOutput: %s", err, output)
}
if !strings.Contains(string(output), "[server]") || !strings.Contains(string(output), "bind_ip") {
t.Errorf("Example config missing expected keys. Output: %s", output)
}
}
// TestIPv4IPv6Flag runs the server with forceprotocol=ipv4 and ipv6 and checks for startup errors
func TestIPv4IPv6Flag(t *testing.T) {
for _, proto := range []string{"ipv4", "ipv6", "auto"} {
cmd := exec.Command("go", "run", "../cmd/server/main.go", "--config", "../cmd/server/config.toml")
cmd.Env = append(os.Environ(), "FORCEPROTOCOL="+proto)
// Set Go module cache environment variables if not already set
if os.Getenv("GOMODCACHE") == "" {
cmd.Env = append(cmd.Env, "GOMODCACHE="+os.Getenv("HOME")+"/go/pkg/mod")
}
if os.Getenv("GOPATH") == "" {
cmd.Env = append(cmd.Env, "GOPATH="+os.Getenv("HOME")+"/go")
}
output, err := cmd.CombinedOutput()
if err != nil && !strings.Contains(string(output), "Configuration loaded successfully") {
t.Errorf("Server failed to start with forceprotocol=%s: %v\nOutput: %s", proto, err, output)
}
}
}

View File