🔥 Tremora del Terra: ultimate hmac-file-server fix – final push before the drop 💾🔐

This commit is contained in:
2025-07-18 16:48:16 +00:00
parent 275ef6c031
commit 6d7042059b
8 changed files with 305 additions and 326 deletions

View File

@ -1,6 +1,6 @@
#!/bin/bash
# HMAC File Server v3.2 - Multi-Architecture Build Script
# Compiles binaries for AMD64, ARM64, and ARM32 architectures
# Compiles binaries for AMD64, ARM64, ARM32, Windows, and macOS architectures
# Remove set -e to prevent early exit on errors
@ -45,13 +45,57 @@ if [[ ! -d "$TEMP_DIR" ]]; then
print_info "Created temp directory: $TEMP_DIR"
fi
# Source files to compile
SOURCE_FILES="cmd/server/main.go cmd/server/helpers.go cmd/server/config_validator.go cmd/server/config_test_scenarios.go"
# Source directory to compile
SOURCE_DIR="./cmd/server/"
print_status "Starting multi-architecture build for HMAC File Server v3.2"
print_info "Source files: $SOURCE_FILES"
print_info "Output directory: $TEMP_DIR"
echo ""
# Interactive menu function
show_menu() {
echo ""
echo "HMAC File Server Multi-Architecture Builder"
echo "=========================================="
echo "1) Build for current platform (auto-detect)"
echo "2) Build for Linux AMD64"
echo "3) Build for Linux ARM64"
echo "4) Build for Linux ARM32v7"
echo "5) Build for Windows AMD64"
echo "6) Build for macOS AMD64 (Intel)"
echo "7) Build for macOS ARM64 (Apple Silicon)"
echo "8) Build all supported architectures"
echo "9) Clean build artifacts"
echo "0) Exit"
echo ""
read -p "Choose an option [0-9]: " choice
}
# Clean function
clean_artifacts() {
print_info "Cleaning build artifacts..."
if [[ -d "$TEMP_DIR" ]]; then
rm -rf "$TEMP_DIR"/*
print_status "Build artifacts cleaned"
else
print_info "No artifacts to clean"
fi
}
# Detect current platform
detect_platform() {
local os=$(uname -s | tr '[:upper:]' '[:lower:]')
local arch=$(uname -m)
case "$arch" in
x86_64) arch="amd64" ;;
arm64|aarch64) arch="arm64" ;;
armv7l) arch="arm" ;;
*) arch="unknown" ;;
esac
case "$os" in
linux) echo "linux/$arch" ;;
darwin) echo "darwin/$arch" ;;
*) echo "unknown/unknown" ;;
esac
}
# Build function
build_for_arch() {
@ -68,7 +112,7 @@ build_for_arch() {
export CGO_ENABLED=0
# Build the binary
if go build -ldflags="-w -s" -o "$TEMP_DIR/$output_name" $SOURCE_FILES 2>/dev/null; then
if go build -ldflags="-w -s" -o "$TEMP_DIR/$output_name" $SOURCE_DIR 2>/dev/null; then
# Get file size
if [[ "$OSTYPE" == "darwin"* ]]; then
# macOS
@ -92,105 +136,270 @@ build_for_arch() {
return 0
else
print_error "Build failed: $arch_description"
if [[ "$goos" == "windows" ]]; then
print_warning " Windows builds may fail due to platform-specific code (syscalls)"
print_info " Consider using Linux subsystem or implementing Windows-specific storage checks"
fi
return 1
fi
}
# Track build results
BUILDS_ATTEMPTED=0
BUILDS_SUCCESSFUL=0
FAILED_BUILDS=()
echo "Starting builds..."
echo "===================="
echo ""
# Build for AMD64 (x86_64)
print_arch "AMD64 (Intel/AMD 64-bit)"
((BUILDS_ATTEMPTED++))
if build_for_arch "linux" "amd64" "hmac-file-server-linux-amd64" "AMD64 Linux"; then
((BUILDS_SUCCESSFUL++))
else
FAILED_BUILDS+=("AMD64")
fi
echo ""
# Build for ARM64 (AArch64)
print_arch "ARM64 (AArch64)"
((BUILDS_ATTEMPTED++))
if build_for_arch "linux" "arm64" "hmac-file-server-linux-arm64" "ARM64 Linux"; then
((BUILDS_SUCCESSFUL++))
else
FAILED_BUILDS+=("ARM64")
fi
echo ""
# Build for ARM32 (ARMv7)
print_arch "ARM32 (ARMv7)"
export GOARM=7 # ARMv7 with hardware floating point
((BUILDS_ATTEMPTED++))
if build_for_arch "linux" "arm" "hmac-file-server-linux-arm32" "ARM32 Linux"; then
((BUILDS_SUCCESSFUL++))
else
FAILED_BUILDS+=("ARM32")
fi
echo ""
# Reset environment variables
unset GOOS GOARCH CGO_ENABLED GOARM
# Build summary
echo "Build Summary"
echo "================"
print_info "Builds attempted: $BUILDS_ATTEMPTED"
print_info "Builds successful: $BUILDS_SUCCESSFUL"
if [[ $BUILDS_SUCCESSFUL -eq $BUILDS_ATTEMPTED ]]; then
print_status "ALL BUILDS SUCCESSFUL!"
# Build all architectures function
build_all_architectures() {
print_status "Starting multi-architecture build for HMAC File Server v3.2"
print_info "Source directory: $SOURCE_DIR"
print_info "Output directory: $TEMP_DIR"
echo ""
print_info "Generated binaries in $TEMP_DIR:"
ls -lh "$TEMP_DIR"/hmac-file-server-* | while read -r line; do
echo " $line"
done
# Track build results
BUILDS_ATTEMPTED=0
BUILDS_SUCCESSFUL=0
FAILED_BUILDS=()
echo "Starting builds..."
echo "===================="
echo ""
print_info "Usage examples:"
echo " - Copy to target system and run: ./hmac-file-server-linux-amd64 --version"
echo " - Deploy with installer: cp temp/hmac-file-server-linux-amd64 /opt/hmac-file-server/"
echo " - Docker deployment: COPY temp/hmac-file-server-linux-amd64 /usr/local/bin/"
# Build for AMD64 (x86_64)
print_arch "AMD64 (Intel/AMD 64-bit)"
((BUILDS_ATTEMPTED++))
if build_for_arch "linux" "amd64" "hmac-file-server-linux-amd64" "AMD64 Linux"; then
((BUILDS_SUCCESSFUL++))
else
FAILED_BUILDS+=("AMD64")
fi
echo ""
# Build for ARM64 (AArch64)
print_arch "ARM64 (AArch64)"
((BUILDS_ATTEMPTED++))
if build_for_arch "linux" "arm64" "hmac-file-server-linux-arm64" "ARM64 Linux"; then
((BUILDS_SUCCESSFUL++))
else
FAILED_BUILDS+=("ARM64")
fi
echo ""
# Build for ARM32 (ARMv7)
print_arch "ARM32 (ARMv7)"
export GOARM=7 # ARMv7 with hardware floating point
((BUILDS_ATTEMPTED++))
if build_for_arch "linux" "arm" "hmac-file-server-linux-arm32v7" "ARM32 Linux"; then
((BUILDS_SUCCESSFUL++))
else
FAILED_BUILDS+=("ARM32")
fi
echo ""
# Build for Windows AMD64
print_arch "Windows AMD64"
((BUILDS_ATTEMPTED++))
if build_for_arch "windows" "amd64" "hmac-file-server-windows-amd64.exe" "Windows AMD64"; then
((BUILDS_SUCCESSFUL++))
else
FAILED_BUILDS+=("Windows")
fi
echo ""
# Build for macOS Intel
print_arch "macOS Intel"
((BUILDS_ATTEMPTED++))
if build_for_arch "darwin" "amd64" "hmac-file-server-darwin-amd64" "macOS Intel"; then
((BUILDS_SUCCESSFUL++))
else
FAILED_BUILDS+=("macOS Intel")
fi
echo ""
# Build for macOS Apple Silicon
print_arch "macOS Apple Silicon"
((BUILDS_ATTEMPTED++))
if build_for_arch "darwin" "arm64" "hmac-file-server-darwin-arm64" "macOS Apple Silicon"; then
((BUILDS_SUCCESSFUL++))
else
FAILED_BUILDS+=("macOS ARM64")
fi
echo ""
# Reset environment variables
unset GOOS GOARCH CGO_ENABLED GOARM
show_build_summary
}
# Build single architecture function
build_single_arch() {
local platform_desc=$1
local goos=$2
local goarch=$3
local goarm=$4
local output_name=$5
elif [[ $BUILDS_SUCCESSFUL -gt 0 ]]; then
print_warning "PARTIAL SUCCESS: $BUILDS_SUCCESSFUL/$BUILDS_ATTEMPTED builds completed"
if [[ ${#FAILED_BUILDS[@]} -gt 0 ]]; then
print_error "Failed architectures: ${FAILED_BUILDS[*]}"
print_status "Building for $platform_desc"
print_info "Source directory: $SOURCE_DIR"
print_info "Output directory: $TEMP_DIR"
echo ""
if [[ -n "$goarm" ]]; then
export GOARM=$goarm
fi
else
print_error "ALL BUILDS FAILED!"
exit 1
fi
BUILDS_ATTEMPTED=1
BUILDS_SUCCESSFUL=0
FAILED_BUILDS=()
if build_for_arch "$goos" "$goarch" "$output_name" "$platform_desc"; then
BUILDS_SUCCESSFUL=1
else
FAILED_BUILDS+=("$platform_desc")
fi
unset GOOS GOARCH CGO_ENABLED GOARM
show_build_summary
}
echo ""
print_info "Architecture compatibility:"
echo " - AMD64: Intel/AMD 64-bit servers, desktops, cloud instances"
echo " - ARM64: Apple Silicon, AWS Graviton, modern ARM servers"
echo " - ARM32: Raspberry Pi, embedded systems, older ARM devices"
# Build current platform function
build_current_platform() {
local platform=$(detect_platform)
local goos=$(echo "$platform" | cut -d'/' -f1)
local goarch=$(echo "$platform" | cut -d'/' -f2)
case "$platform" in
"linux/amd64")
build_single_arch "Current Platform (Linux AMD64)" "linux" "amd64" "" "hmac-file-server-linux-amd64"
;;
"linux/arm64")
build_single_arch "Current Platform (Linux ARM64)" "linux" "arm64" "" "hmac-file-server-linux-arm64"
;;
"linux/arm")
build_single_arch "Current Platform (Linux ARM32v7)" "linux" "arm" "7" "hmac-file-server-linux-arm32v7"
;;
"darwin/amd64")
build_single_arch "Current Platform (macOS Intel)" "darwin" "amd64" "" "hmac-file-server-darwin-amd64"
;;
"darwin/arm64")
build_single_arch "Current Platform (macOS Apple Silicon)" "darwin" "arm64" "" "hmac-file-server-darwin-arm64"
;;
*)
print_error "Unsupported platform: $platform"
print_info "Supported platforms: linux/amd64, linux/arm64, linux/arm, darwin/amd64, darwin/arm64"
exit 1
;;
esac
}
echo ""
print_status "Multi-architecture build completed!"
# Show build summary
show_build_summary() {
# Build summary
echo "Build Summary"
echo "================"
print_info "Builds attempted: $BUILDS_ATTEMPTED"
print_info "Builds successful: $BUILDS_SUCCESSFUL"
# Final verification
echo ""
print_info "Final verification:"
for binary in "$TEMP_DIR"/hmac-file-server-*; do
if [[ -f "$binary" ]]; then
filename=$(basename "$binary")
if file "$binary" >/dev/null 2>&1; then
file_info=$(file "$binary" | cut -d: -f2- | sed 's/^ *//')
print_info " OK $filename: $file_info"
else
print_info " OK $filename: Binary file"
if [[ $BUILDS_SUCCESSFUL -eq $BUILDS_ATTEMPTED ]]; then
print_status "ALL BUILDS SUCCESSFUL!"
echo ""
print_info "Generated binaries in $TEMP_DIR:"
ls -lh "$TEMP_DIR"/hmac-file-server-* 2>/dev/null | while read -r line; do
echo " $line"
done
echo ""
print_info "Usage examples:"
echo " - Copy to target system and run: ./hmac-file-server-linux-amd64 --version"
echo " - Deploy with installer: cp temp/hmac-file-server-linux-amd64 /opt/hmac-file-server/"
echo " - Docker deployment: COPY temp/hmac-file-server-linux-amd64 /usr/local/bin/"
elif [[ $BUILDS_SUCCESSFUL -gt 0 ]]; then
print_warning "PARTIAL SUCCESS: $BUILDS_SUCCESSFUL/$BUILDS_ATTEMPTED builds completed"
if [[ ${#FAILED_BUILDS[@]} -gt 0 ]]; then
print_error "Failed architectures: ${FAILED_BUILDS[*]}"
fi
else
print_error "ALL BUILDS FAILED!"
exit 1
fi
done
echo ""
print_info "Architecture compatibility:"
echo " - AMD64: Intel/AMD 64-bit servers, desktops, cloud instances"
echo " - ARM64: Apple Silicon, AWS Graviton, modern ARM servers"
echo " - ARM32: Raspberry Pi, embedded systems, older ARM devices"
echo " - Windows: Windows 10/11, Windows Server"
echo " - macOS: macOS 10.15+, Intel and Apple Silicon"
echo ""
print_status "Build completed!"
# Final verification
echo ""
print_info "Final verification:"
for binary in "$TEMP_DIR"/hmac-file-server-*; do
if [[ -f "$binary" ]]; then
filename=$(basename "$binary")
if file "$binary" >/dev/null 2>&1; then
file_info=$(file "$binary" | cut -d: -f2- | sed 's/^ *//')
print_info " OK $filename: $file_info"
else
print_info " OK $filename: Binary file"
fi
fi
done
}
# Main execution
if [[ $# -eq 0 ]]; then
# Interactive mode
while true; do
show_menu
case $choice in
1)
build_current_platform
break
;;
2)
build_single_arch "Linux AMD64" "linux" "amd64" "" "hmac-file-server-linux-amd64"
break
;;
3)
build_single_arch "Linux ARM64" "linux" "arm64" "" "hmac-file-server-linux-arm64"
break
;;
4)
build_single_arch "Linux ARM32v7" "linux" "arm" "7" "hmac-file-server-linux-arm32v7"
break
;;
5)
build_single_arch "Windows AMD64" "windows" "amd64" "" "hmac-file-server-windows-amd64.exe"
break
;;
6)
build_single_arch "macOS Intel" "darwin" "amd64" "" "hmac-file-server-darwin-amd64"
break
;;
7)
build_single_arch "macOS Apple Silicon" "darwin" "arm64" "" "hmac-file-server-darwin-arm64"
break
;;
8)
build_all_architectures
break
;;
9)
clean_artifacts
;;
0)
print_info "Exiting build script"
exit 0
;;
*)
print_error "Invalid option. Please choose 0-9."
;;
esac
done
else
# Non-interactive mode - build all architectures
build_all_architectures
fi
exit 0

View File

@ -1,111 +0,0 @@
[server]
listen_address = ":8080"
storage_path = "/srv/hmac-file-server/uploads"
metrics_enabled = true
metrics_path = "/metrics"
pid_file = "/var/run/hmac-file-server.pid"
max_upload_size = "10GB" # Supports B, KB, MB, GB, TB
max_header_bytes = 1048576 # 1MB
cleanup_interval = "24h"
max_file_age = "720h" # 30 days
pre_cache = true
pre_cache_workers = 4
pre_cache_interval = "1h"
global_extensions = [".txt", ".dat", ".iso", ".mp4", ".mkv", ".avi", ".mov", ".wmv", ".flv", ".webm", ".mpeg"] # If set, overrides upload/download extensions
deduplication_enabled = true
min_free_bytes = "1GB" # Minimum free space required for uploads
file_naming = "original" # Options: "original", "HMAC"
force_protocol = "" # Options: "http", "https" - if set, redirects to this protocol
enable_dynamic_workers = true # Enable dynamic worker scaling
worker_scale_up_thresh = 50 # Queue length to scale up workers
worker_scale_down_thresh = 10 # Queue length to scale down workers
# Cluster-aware settings for client restart resilience
graceful_shutdown_timeout = "300s" # Allow time for client reconnections
connection_drain_timeout = "120s" # Drain existing connections gracefully
max_idle_conns_per_host = 5 # Limit persistent connections per client
idle_conn_timeout = "90s" # Close idle connections regularly
disable_keep_alives = false # Keep HTTP keep-alives for performance
client_timeout = "300s" # Timeout for slow clients
restart_grace_period = "60s" # Grace period after restart for clients to reconnect
[uploads]
allowed_extensions = [".zip", ".rar", ".7z", ".tar.gz", ".tgz", ".gpg", ".enc", ".pgp"]
chunked_uploads_enabled = true
chunk_size = "10MB"
resumable_uploads_enabled = true
max_resumable_age = "48h"
# Cluster resilience for uploads
session_persistence = true # Persist upload sessions across restarts
session_recovery_timeout = "300s" # Time to wait for session recovery
client_reconnect_window = "120s" # Window for clients to reconnect after server restart
upload_slot_ttl = "3600s" # Upload slot validity time
retry_failed_uploads = true # Automatically retry failed uploads
max_upload_retries = 3 # Maximum retry attempts
[downloads]
resumable_downloads_enabled = true
chunked_downloads_enabled = true
chunk_size = "8192"
allowed_extensions = [".txt", ".pdf", ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".svg", ".webp"]
[security]
secret = "f6g4ldPvQM7O2UTFeBEUUj33VrXypDAcsDt0yqKrLiOr5oQW"
enablejwt = false
jwtsecret = "f6g4ldPvQM7O2UTFeBEUUj33VrXypDAcsDt0yqKrLiOr5oQW"
jwtalgorithm = "HS256"
jwtexpiration = "24h"
[logging]
level = "info"
file = "/var/log/hmac-file-server.log"
max_size = 100
max_backups = 7
max_age = 30
compress = true
[deduplication]
enabled = true
directory = "./deduplication"
maxsize = "1GB"
[iso]
enabled = true
size = "1GB"
mountpoint = "/mnt/iso"
charset = "utf-8"
containerfile = "/mnt/iso/container.iso"
[timeouts]
readtimeout = "4800s"
writetimeout = "4800s"
idletimeout = "4800s"
[versioning]
enableversioning = false
maxversions = 1
[clamav]
clamavenabled = true
clamavsocket = "/var/run/clamav/clamd.ctl"
numscanworkers = 2
# Only scan potentially dangerous file types, skip large media files
scanfileextensions = [".txt", ".pdf", ".doc", ".docx", ".xls", ".xlsx", ".exe", ".zip", ".rar", ".7z", ".tar", ".gz"]
# Skip scanning files larger than 200MB (ClamAV limit)
maxscansize = "200MB"
[redis]
redisenabled = true
redisdbindex = 0
redisaddr = "localhost:6379"
redispassword = ""
redishealthcheckinterval = "120s"
[workers]
numworkers = 4
uploadqueuesize = 50
[file]
# Add file-specific configurations here
[build]
version = "3.2"

View File

@ -5,7 +5,7 @@ services:
container_name: hmac-file-server
image: hmac-file-server:latest
ports:
- "8080:8080"
- "8081:8080"
volumes:
- ./config:/etc/hmac-file-server
- ./data/uploads:/opt/hmac-file-server/data/uploads

View File

@ -6,7 +6,7 @@ RUN apk add --no-cache git
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=0 go build -o hmac-file-server cmd/server/main.go cmd/server/helpers.go cmd/server/config_validator.go cmd/server/config_test_scenarios.go
RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o hmac-file-server ./cmd/server/
# Stage 2: Runtime
FROM alpine:latest

Binary file not shown.

Before

Width:  |  Height:  |  Size: 60 KiB

View File

@ -1,80 +0,0 @@
package main
import (
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"fmt"
"mime"
"net/http"
"net/url"
"os"
"path/filepath" // Added this import for filepath usage
"strconv"
"testing"
)
const (
serverURL = "http://[::1]:8080" // Replace with your actual server URL
secret = "hmac-file-server-is-the-win" // Replace with your HMAC secret key
uploadPath = "hmac_icon.png" // Test file to upload
protocolType = "v2" // Use v2, v, or token as needed
)
// TestUpload performs a basic HMAC validation and upload test.
func TestUpload(t *testing.T) {
// File setup for testing
file, err := os.Open(uploadPath)
if err != nil {
t.Fatalf("Error opening file: %v", err)
}
defer file.Close()
fileInfo, _ := file.Stat()
fileStorePath := uploadPath
contentLength := fileInfo.Size()
// Generate HMAC based on protocol type
hmacValue := generateHMAC(fileStorePath, contentLength, protocolType)
// Formulate request URL with HMAC in query params
reqURL := fmt.Sprintf("%s/%s?%s=%s", serverURL, fileStorePath, protocolType, url.QueryEscape(hmacValue))
// Prepare HTTP PUT request with file data
req, err := http.NewRequest(http.MethodPut, reqURL, file)
if err != nil {
t.Fatalf("Error creating request: %v", err)
}
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Content-Length", strconv.FormatInt(contentLength, 10))
// Execute HTTP request
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("Error executing request: %v", err)
}
defer resp.Body.Close()
t.Logf("Response status: %s", resp.Status)
}
// Generates the HMAC based on your protocol version
func generateHMAC(filePath string, contentLength int64, protocol string) string {
mac := hmac.New(sha256.New, []byte(secret))
macString := ""
// Calculate HMAC according to protocol
if protocol == "v" {
mac.Write([]byte(filePath + "\x20" + strconv.FormatInt(contentLength, 10)))
macString = hex.EncodeToString(mac.Sum(nil))
} else if protocol == "v2" || protocol == "token" {
contentType := mime.TypeByExtension(filepath.Ext(filePath))
if contentType == "" {
contentType = "application/octet-stream"
}
mac.Write([]byte(filePath + "\x00" + strconv.FormatInt(contentLength, 10) + "\x00" + contentType))
macString = hex.EncodeToString(mac.Sum(nil))
}
return macString
}

View File

@ -1,39 +0,0 @@
package main
import (
"os"
"os/exec"
"strings"
"testing"
)
// TestGenConfigFlag runs the server with --genconfig and checks output for expected config keys
func TestGenConfigFlag(t *testing.T) {
cmd := exec.Command("go", "run", "../cmd/server/main.go", "--genconfig")
output, err := cmd.CombinedOutput()
if err != nil && !strings.Contains(string(output), "[server]") {
t.Fatalf("Failed to run with --genconfig: %v\nOutput: %s", err, output)
}
if !strings.Contains(string(output), "[server]") || !strings.Contains(string(output), "bind_ip") {
t.Errorf("Example config missing expected keys. Output: %s", output)
}
}
// TestIPv4IPv6Flag runs the server with forceprotocol=ipv4 and ipv6 and checks for startup errors
func TestIPv4IPv6Flag(t *testing.T) {
for _, proto := range []string{"ipv4", "ipv6", "auto"} {
cmd := exec.Command("go", "run", "../cmd/server/main.go", "--config", "../cmd/server/config.toml")
cmd.Env = append(os.Environ(), "FORCEPROTOCOL="+proto)
// Set Go module cache environment variables if not already set
if os.Getenv("GOMODCACHE") == "" {
cmd.Env = append(cmd.Env, "GOMODCACHE="+os.Getenv("HOME")+"/go/pkg/mod")
}
if os.Getenv("GOPATH") == "" {
cmd.Env = append(cmd.Env, "GOPATH="+os.Getenv("HOME")+"/go")
}
output, err := cmd.CombinedOutput()
if err != nil && !strings.Contains(string(output), "Configuration loaded successfully") {
t.Errorf("Server failed to start with forceprotocol=%s: %v\nOutput: %s", proto, err, output)
}
}
}

View File