Compare commits
57 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3e41d88445 | |||
| 5fb88b14ba | |||
| cccee4294f | |||
| 9688143176 | |||
| e821e131b4 | |||
| 15a60d2e71 | |||
| 9c65821250 | |||
| 627061cdbb | |||
| e1a7c57e0f | |||
| 22915102d4 | |||
| 3653ced6da | |||
| 9743d571ce | |||
| c519f08ef2 | |||
| b99b05fedb | |||
| c5f2c3322c | |||
| 56ad0824c7 | |||
| ec65df2976 | |||
| 23cc1e0e08 | |||
| 7770abab6f | |||
| f6a20f035b | |||
| 28e54d118f | |||
| ab0ff3f28d | |||
| b7dd325c51 | |||
| 2ed54141a3 | |||
| 495ee31247 | |||
| 78e10f5057 | |||
| f4a0e2d82c | |||
| f66d19acb0 | |||
| 16f377e9b5 | |||
| 7e32a0369d | |||
| 120ee33e3b | |||
| 9f375621d1 | |||
| 9ad925191e | |||
| 9d8a6e763e | |||
| 63b16eee8b | |||
| 91228552fb | |||
| 9ee55309bd | |||
| 0baf741c0b | |||
| faace7271c | |||
| c3ade7a693 | |||
| 52d475506c | |||
| 938ee61686 | |||
| 85b61048c0 | |||
| 30954cb7c2 | |||
| ddf46f190b | |||
| 4c6d44725e | |||
| be69c0e00f | |||
| ee1f58efdb | |||
| 5959d7313d | |||
| b856d8b3f8 | |||
| 886aa4810a | |||
| 14bd1f848c | |||
| 4c171c0e44 | |||
| e7f0a9f5eb | |||
| 2e942f04a4 | |||
| f29e6fe102 | |||
| 51fc570fc7 |
@@ -1,4 +1,6 @@
|
|||||||
# CI/CD Pipeline for dbbackup
|
# CI/CD Pipeline for dbbackup
|
||||||
|
# Main repo: Gitea (git.uuxo.net)
|
||||||
|
# Mirror: GitHub (github.com/PlusOne/dbbackup)
|
||||||
name: CI/CD
|
name: CI/CD
|
||||||
|
|
||||||
on:
|
on:
|
||||||
@@ -8,9 +10,6 @@ on:
|
|||||||
pull_request:
|
pull_request:
|
||||||
branches: [main, master]
|
branches: [main, master]
|
||||||
|
|
||||||
env:
|
|
||||||
GITEA_URL: https://git.uuxo.net
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
test:
|
test:
|
||||||
name: Test
|
name: Test
|
||||||
@@ -18,26 +17,25 @@ jobs:
|
|||||||
container:
|
container:
|
||||||
image: golang:1.24-bookworm
|
image: golang:1.24-bookworm
|
||||||
steps:
|
steps:
|
||||||
- name: Install git
|
|
||||||
run: apt-get update && apt-get install -y git ca-certificates
|
|
||||||
|
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
|
env:
|
||||||
|
TOKEN: ${{ github.token }}
|
||||||
run: |
|
run: |
|
||||||
|
apt-get update && apt-get install -y -qq git ca-certificates
|
||||||
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
||||||
git clone --depth 1 --branch ${GITHUB_REF_NAME} ${{ env.GITEA_URL }}/${GITHUB_REPOSITORY}.git .
|
git init
|
||||||
|
git remote add origin "https://${TOKEN}@git.uuxo.net/${GITHUB_REPOSITORY}.git"
|
||||||
|
git fetch --depth=1 origin "${GITHUB_SHA}"
|
||||||
|
git checkout FETCH_HEAD
|
||||||
|
|
||||||
- name: Download dependencies
|
- name: Download dependencies
|
||||||
run: go mod download
|
run: go mod download
|
||||||
|
|
||||||
- name: Run tests with race detection
|
- name: Run tests
|
||||||
env:
|
run: go test -race -coverprofile=coverage.out ./...
|
||||||
GOMAXPROCS: 8
|
|
||||||
run: go test -race -coverprofile=coverage.out -covermode=atomic ./...
|
|
||||||
|
|
||||||
- name: Generate coverage report
|
- name: Coverage summary
|
||||||
run: |
|
run: go tool cover -func=coverage.out | tail -1
|
||||||
go tool cover -func=coverage.out
|
|
||||||
go tool cover -html=coverage.out -o coverage.html
|
|
||||||
|
|
||||||
lint:
|
lint:
|
||||||
name: Lint
|
name: Lint
|
||||||
@@ -45,168 +43,119 @@ jobs:
|
|||||||
container:
|
container:
|
||||||
image: golang:1.24-bookworm
|
image: golang:1.24-bookworm
|
||||||
steps:
|
steps:
|
||||||
- name: Install git
|
|
||||||
run: apt-get update && apt-get install -y git ca-certificates
|
|
||||||
|
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
run: |
|
|
||||||
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
|
||||||
git clone --depth 1 --branch ${GITHUB_REF_NAME} ${{ env.GITEA_URL }}/${GITHUB_REPOSITORY}.git .
|
|
||||||
|
|
||||||
- name: Install golangci-lint
|
|
||||||
run: go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.62.2
|
|
||||||
|
|
||||||
- name: Run golangci-lint
|
|
||||||
env:
|
env:
|
||||||
GOMAXPROCS: 8
|
TOKEN: ${{ github.token }}
|
||||||
run: golangci-lint run --timeout=5m ./...
|
run: |
|
||||||
|
apt-get update && apt-get install -y -qq git ca-certificates
|
||||||
|
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
||||||
|
git init
|
||||||
|
git remote add origin "https://${TOKEN}@git.uuxo.net/${GITHUB_REPOSITORY}.git"
|
||||||
|
git fetch --depth=1 origin "${GITHUB_SHA}"
|
||||||
|
git checkout FETCH_HEAD
|
||||||
|
|
||||||
build:
|
- name: Install and run golangci-lint
|
||||||
name: Build (${{ matrix.goos }}-${{ matrix.goarch }})
|
run: |
|
||||||
|
go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.62.2
|
||||||
|
golangci-lint run --timeout=5m ./...
|
||||||
|
|
||||||
|
build-and-release:
|
||||||
|
name: Build & Release
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: [test, lint]
|
needs: [test, lint]
|
||||||
container:
|
|
||||||
image: golang:1.24-bookworm
|
|
||||||
strategy:
|
|
||||||
max-parallel: 8
|
|
||||||
matrix:
|
|
||||||
goos: [linux, darwin]
|
|
||||||
goarch: [amd64, arm64]
|
|
||||||
steps:
|
|
||||||
- name: Install git
|
|
||||||
run: apt-get update && apt-get install -y git ca-certificates
|
|
||||||
|
|
||||||
- name: Checkout code
|
|
||||||
run: |
|
|
||||||
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
|
||||||
git clone --depth 1 --branch ${GITHUB_REF_NAME} ${{ env.GITEA_URL }}/${GITHUB_REPOSITORY}.git .
|
|
||||||
|
|
||||||
- name: Build binary
|
|
||||||
env:
|
|
||||||
GOOS: ${{ matrix.goos }}
|
|
||||||
GOARCH: ${{ matrix.goarch }}
|
|
||||||
CGO_ENABLED: 0
|
|
||||||
GOMAXPROCS: 8
|
|
||||||
run: |
|
|
||||||
BINARY_NAME=dbbackup
|
|
||||||
go build -ldflags="-s -w" -o dist/${BINARY_NAME}-${{ matrix.goos }}-${{ matrix.goarch }} .
|
|
||||||
|
|
||||||
sbom:
|
|
||||||
name: Generate SBOM
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: [test]
|
|
||||||
container:
|
|
||||||
image: golang:1.24-bookworm
|
|
||||||
steps:
|
|
||||||
- name: Install git
|
|
||||||
run: apt-get update && apt-get install -y git ca-certificates
|
|
||||||
|
|
||||||
- name: Checkout code
|
|
||||||
run: |
|
|
||||||
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
|
||||||
git clone --depth 1 --branch ${GITHUB_REF_NAME} ${{ env.GITEA_URL }}/${GITHUB_REPOSITORY}.git .
|
|
||||||
|
|
||||||
- name: Install Syft
|
|
||||||
run: curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin
|
|
||||||
|
|
||||||
- name: Generate SBOM
|
|
||||||
run: |
|
|
||||||
syft . -o spdx-json=sbom-spdx.json
|
|
||||||
syft . -o cyclonedx-json=sbom-cyclonedx.json
|
|
||||||
|
|
||||||
release:
|
|
||||||
name: Release
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: [test, lint, build]
|
|
||||||
if: startsWith(github.ref, 'refs/tags/v')
|
if: startsWith(github.ref, 'refs/tags/v')
|
||||||
container:
|
container:
|
||||||
image: golang:1.24-bookworm
|
image: golang:1.24-bookworm
|
||||||
steps:
|
steps:
|
||||||
- name: Install tools
|
|
||||||
run: |
|
|
||||||
apt-get update && apt-get install -y git ca-certificates
|
|
||||||
curl -sSfL https://github.com/goreleaser/goreleaser/releases/download/v2.4.8/goreleaser_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin goreleaser
|
|
||||||
curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin
|
|
||||||
|
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
run: |
|
|
||||||
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
|
||||||
git clone --branch ${GITHUB_REF_NAME} ${{ env.GITEA_URL }}/${GITHUB_REPOSITORY}.git .
|
|
||||||
git fetch --tags
|
|
||||||
|
|
||||||
- name: Run goreleaser
|
|
||||||
env:
|
env:
|
||||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
TOKEN: ${{ github.token }}
|
||||||
run: goreleaser release --clean
|
|
||||||
|
|
||||||
docker:
|
|
||||||
name: Build & Push Docker Image
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: [test, lint]
|
|
||||||
if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
|
|
||||||
container:
|
|
||||||
image: docker:24-cli
|
|
||||||
options: --privileged
|
|
||||||
services:
|
|
||||||
docker:
|
|
||||||
image: docker:24-dind
|
|
||||||
options: --privileged
|
|
||||||
steps:
|
|
||||||
- name: Install dependencies
|
|
||||||
run: apk add --no-cache git curl
|
|
||||||
|
|
||||||
- name: Checkout code
|
|
||||||
run: |
|
run: |
|
||||||
|
apt-get update && apt-get install -y -qq git ca-certificates curl jq
|
||||||
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
||||||
git clone --depth 1 --branch ${GITHUB_REF_NAME} ${{ env.GITEA_URL }}/${GITHUB_REPOSITORY}.git .
|
git init
|
||||||
|
git remote add origin "https://${TOKEN}@git.uuxo.net/${GITHUB_REPOSITORY}.git"
|
||||||
|
git fetch --depth=1 origin "${GITHUB_SHA}"
|
||||||
|
git checkout FETCH_HEAD
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Build all platforms
|
||||||
run: |
|
run: |
|
||||||
docker buildx create --use --name builder --driver docker-container
|
mkdir -p release
|
||||||
docker buildx inspect --bootstrap
|
|
||||||
|
# Install cross-compilation tools for CGO
|
||||||
|
apt-get update && apt-get install -y -qq gcc-aarch64-linux-gnu
|
||||||
|
|
||||||
|
# Linux amd64 (with CGO for SQLite)
|
||||||
|
echo "Building linux/amd64 (CGO enabled)..."
|
||||||
|
CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o release/dbbackup-linux-amd64 .
|
||||||
|
|
||||||
|
# Linux arm64 (with CGO for SQLite)
|
||||||
|
echo "Building linux/arm64 (CGO enabled)..."
|
||||||
|
CC=aarch64-linux-gnu-gcc CGO_ENABLED=1 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w" -o release/dbbackup-linux-arm64 .
|
||||||
|
|
||||||
|
# Darwin amd64 (no CGO - cross-compile limitation)
|
||||||
|
echo "Building darwin/amd64 (CGO disabled)..."
|
||||||
|
CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -ldflags="-s -w" -o release/dbbackup-darwin-amd64 .
|
||||||
|
|
||||||
|
# Darwin arm64 (no CGO - cross-compile limitation)
|
||||||
|
echo "Building darwin/arm64 (CGO disabled)..."
|
||||||
|
CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -ldflags="-s -w" -o release/dbbackup-darwin-arm64 .
|
||||||
|
|
||||||
|
# FreeBSD amd64 (no CGO - cross-compile limitation)
|
||||||
|
echo "Building freebsd/amd64 (CGO disabled)..."
|
||||||
|
CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 go build -ldflags="-s -w" -o release/dbbackup-freebsd-amd64 .
|
||||||
|
|
||||||
|
echo "All builds complete:"
|
||||||
|
ls -lh release/
|
||||||
|
|
||||||
- name: Login to Gitea Registry
|
- name: Create Gitea Release
|
||||||
if: ${{ secrets.REGISTRY_USER != '' && secrets.REGISTRY_TOKEN != '' }}
|
env:
|
||||||
|
GITEA_TOKEN: ${{ github.token }}
|
||||||
run: |
|
run: |
|
||||||
echo "${{ secrets.REGISTRY_TOKEN }}" | docker login git.uuxo.net -u "${{ secrets.REGISTRY_USER }}" --password-stdin
|
TAG=${GITHUB_REF#refs/tags/}
|
||||||
|
|
||||||
- name: Build and push
|
echo "Creating Gitea release for ${TAG}..."
|
||||||
if: ${{ secrets.REGISTRY_USER != '' && secrets.REGISTRY_TOKEN != '' }}
|
echo "Debug: GITHUB_REPOSITORY=${GITHUB_REPOSITORY}"
|
||||||
run: |
|
echo "Debug: TAG=${TAG}"
|
||||||
# Determine tags
|
|
||||||
if [[ "${GITHUB_REF}" == refs/tags/* ]]; then
|
# Simple body without special characters
|
||||||
VERSION=${GITHUB_REF#refs/tags/}
|
BODY="Download binaries for your platform"
|
||||||
TAGS="-t git.uuxo.net/uuxo/dbbackup:${VERSION} -t git.uuxo.net/uuxo/dbbackup:latest"
|
|
||||||
else
|
# Create release via API with simple inline JSON
|
||||||
TAGS="-t git.uuxo.net/uuxo/dbbackup:${GITHUB_SHA::8} -t git.uuxo.net/uuxo/dbbackup:main"
|
RESPONSE=$(curl -s -w "\n%{http_code}" -X POST \
|
||||||
|
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{"tag_name":"'"${TAG}"'","name":"'"${TAG}"'","body":"'"${BODY}"'","draft":false,"prerelease":false}' \
|
||||||
|
"https://git.uuxo.net/api/v1/repos/${GITHUB_REPOSITORY}/releases")
|
||||||
|
|
||||||
|
HTTP_CODE=$(echo "$RESPONSE" | tail -1)
|
||||||
|
BODY_RESPONSE=$(echo "$RESPONSE" | sed '$d')
|
||||||
|
|
||||||
|
echo "HTTP Code: $HTTP_CODE"
|
||||||
|
echo "Response: $BODY_RESPONSE"
|
||||||
|
|
||||||
|
RELEASE_ID=$(echo "$BODY_RESPONSE" | jq -r '.id')
|
||||||
|
|
||||||
|
if [ "$RELEASE_ID" = "null" ] || [ -z "$RELEASE_ID" ]; then
|
||||||
|
echo "Failed to create release"
|
||||||
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
docker buildx build \
|
echo "Created release ID: $RELEASE_ID"
|
||||||
--platform linux/amd64,linux/arm64 \
|
|
||||||
--push \
|
# Upload each binary
|
||||||
${TAGS} \
|
echo "Files to upload:"
|
||||||
.
|
ls -la release/
|
||||||
# Test 1765481480
|
|
||||||
|
for file in release/dbbackup-*; do
|
||||||
mirror:
|
FILENAME=$(basename "$file")
|
||||||
name: Mirror to GitHub
|
echo "Uploading $FILENAME..."
|
||||||
runs-on: ubuntu-latest
|
UPLOAD_RESPONSE=$(curl -s -X POST \
|
||||||
needs: [test, lint]
|
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main' && vars.MIRROR_ENABLED != 'false'
|
-F "attachment=@${file}" \
|
||||||
container:
|
"https://git.uuxo.net/api/v1/repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID}/assets?name=${FILENAME}")
|
||||||
image: debian:bookworm-slim
|
echo "Upload response: $UPLOAD_RESPONSE"
|
||||||
volumes:
|
done
|
||||||
- /root/.ssh:/root/.ssh:ro
|
|
||||||
steps:
|
echo "Gitea release complete!"
|
||||||
- name: Install git
|
echo "GitHub mirror complete!"
|
||||||
run: apt-get update && apt-get install -y --no-install-recommends git openssh-client ca-certificates && rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
- name: Clone and mirror
|
|
||||||
env:
|
|
||||||
GIT_SSH_COMMAND: "ssh -i /root/.ssh/id_ed25519 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
|
|
||||||
run: |
|
|
||||||
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
|
||||||
git clone --mirror ${{ env.GITEA_URL }}/${GITHUB_REPOSITORY}.git repo.git
|
|
||||||
cd repo.git
|
|
||||||
git remote add github git@github.com:PlusOne/dbbackup.git
|
|
||||||
git push --mirror github || git push --force --all github && git push --force --tags github
|
|
||||||
8
.gitignore
vendored
8
.gitignore
vendored
@@ -9,10 +9,12 @@ logs/
|
|||||||
*.trace
|
*.trace
|
||||||
*.err
|
*.err
|
||||||
|
|
||||||
# Ignore built binaries in root (keep bin/ directory for releases)
|
# Ignore built binaries (built fresh via build_all.sh on release)
|
||||||
/dbbackup
|
/dbbackup
|
||||||
/dbbackup_*
|
/dbbackup_*
|
||||||
!dbbackup.png
|
!dbbackup.png
|
||||||
|
bin/dbbackup_*
|
||||||
|
bin/*.exe
|
||||||
|
|
||||||
# Ignore development artifacts
|
# Ignore development artifacts
|
||||||
*.swp
|
*.swp
|
||||||
@@ -32,3 +34,7 @@ coverage.html
|
|||||||
# Ignore temporary files
|
# Ignore temporary files
|
||||||
tmp/
|
tmp/
|
||||||
temp/
|
temp/
|
||||||
|
CRITICAL_BUGS_FIXED.md
|
||||||
|
LEGAL_DOCUMENTATION.md
|
||||||
|
LEGAL_*.md
|
||||||
|
legal/
|
||||||
|
|||||||
74
AZURE.md
74
AZURE.md
@@ -28,21 +28,16 @@ This guide covers using **Azure Blob Storage** with `dbbackup` for secure, scala
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Backup PostgreSQL to Azure
|
# Backup PostgreSQL to Azure
|
||||||
dbbackup backup postgres \
|
dbbackup backup single mydb \
|
||||||
--host localhost \
|
--cloud "azure://mycontainer/backups/?account=myaccount&key=ACCOUNT_KEY"
|
||||||
--database mydb \
|
|
||||||
--output backup.sql \
|
|
||||||
--cloud "azure://mycontainer/backups/db.sql?account=myaccount&key=ACCOUNT_KEY"
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### 3. Restore from Azure
|
### 3. Restore from Azure
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Restore from Azure backup
|
# Download backup from Azure and restore
|
||||||
dbbackup restore postgres \
|
dbbackup cloud download "azure://mycontainer/backups/mydb.dump.gz?account=myaccount&key=ACCOUNT_KEY" ./mydb.dump.gz
|
||||||
--source "azure://mycontainer/backups/db.sql?account=myaccount&key=ACCOUNT_KEY" \
|
dbbackup restore single ./mydb.dump.gz --target mydb_restored --confirm
|
||||||
--host localhost \
|
|
||||||
--database mydb_restored
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## URI Syntax
|
## URI Syntax
|
||||||
@@ -99,7 +94,7 @@ export AZURE_STORAGE_ACCOUNT="myaccount"
|
|||||||
export AZURE_STORAGE_KEY="YOUR_ACCOUNT_KEY"
|
export AZURE_STORAGE_KEY="YOUR_ACCOUNT_KEY"
|
||||||
|
|
||||||
# Use simplified URI (credentials from environment)
|
# Use simplified URI (credentials from environment)
|
||||||
dbbackup backup postgres --cloud "azure://container/path/backup.sql"
|
dbbackup backup single mydb --cloud "azure://container/path/"
|
||||||
```
|
```
|
||||||
|
|
||||||
### Method 3: Connection String
|
### Method 3: Connection String
|
||||||
@@ -109,7 +104,7 @@ Use Azure connection string:
|
|||||||
```bash
|
```bash
|
||||||
export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=YOUR_KEY;EndpointSuffix=core.windows.net"
|
export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=YOUR_KEY;EndpointSuffix=core.windows.net"
|
||||||
|
|
||||||
dbbackup backup postgres --cloud "azure://container/path/backup.sql"
|
dbbackup backup single mydb --cloud "azure://container/path/"
|
||||||
```
|
```
|
||||||
|
|
||||||
### Getting Your Account Key
|
### Getting Your Account Key
|
||||||
@@ -196,11 +191,8 @@ Configure automatic tier transitions:
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# PostgreSQL backup with automatic Azure upload
|
# PostgreSQL backup with automatic Azure upload
|
||||||
dbbackup backup postgres \
|
dbbackup backup single production_db \
|
||||||
--host localhost \
|
--cloud "azure://prod-backups/postgres/?account=myaccount&key=KEY" \
|
||||||
--database production_db \
|
|
||||||
--output /backups/db.sql \
|
|
||||||
--cloud "azure://prod-backups/postgres/$(date +%Y%m%d_%H%M%S).sql?account=myaccount&key=KEY" \
|
|
||||||
--compression 6
|
--compression 6
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -208,10 +200,7 @@ dbbackup backup postgres \
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Backup entire PostgreSQL cluster to Azure
|
# Backup entire PostgreSQL cluster to Azure
|
||||||
dbbackup backup postgres \
|
dbbackup backup cluster \
|
||||||
--host localhost \
|
|
||||||
--all-databases \
|
|
||||||
--output-dir /backups \
|
|
||||||
--cloud "azure://prod-backups/postgres/cluster/?account=myaccount&key=KEY"
|
--cloud "azure://prod-backups/postgres/cluster/?account=myaccount&key=KEY"
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -257,13 +246,9 @@ dbbackup cleanup "azure://prod-backups/postgres/?account=myaccount&key=KEY" --ke
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
# Azure backup script (run via cron)
|
# Azure backup script (run via cron)
|
||||||
|
|
||||||
DATE=$(date +%Y%m%d_%H%M%S)
|
AZURE_URI="azure://prod-backups/postgres/?account=myaccount&key=${AZURE_STORAGE_KEY}"
|
||||||
AZURE_URI="azure://prod-backups/postgres/${DATE}.sql?account=myaccount&key=${AZURE_STORAGE_KEY}"
|
|
||||||
|
|
||||||
dbbackup backup postgres \
|
dbbackup backup single production_db \
|
||||||
--host localhost \
|
|
||||||
--database production_db \
|
|
||||||
--output /tmp/backup.sql \
|
|
||||||
--cloud "${AZURE_URI}" \
|
--cloud "${AZURE_URI}" \
|
||||||
--compression 9
|
--compression 9
|
||||||
|
|
||||||
@@ -289,35 +274,25 @@ For large files (>256MB), dbbackup automatically uses Azure Block Blob staging:
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Large database backup (automatically uses block blob)
|
# Large database backup (automatically uses block blob)
|
||||||
dbbackup backup postgres \
|
dbbackup backup single huge_db \
|
||||||
--host localhost \
|
--cloud "azure://backups/?account=myaccount&key=KEY"
|
||||||
--database huge_db \
|
|
||||||
--output /backups/huge.sql \
|
|
||||||
--cloud "azure://backups/huge.sql?account=myaccount&key=KEY"
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Progress Tracking
|
### Progress Tracking
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Backup with progress display
|
# Backup with progress display
|
||||||
dbbackup backup postgres \
|
dbbackup backup single mydb \
|
||||||
--host localhost \
|
--cloud "azure://backups/?account=myaccount&key=KEY"
|
||||||
--database mydb \
|
|
||||||
--output backup.sql \
|
|
||||||
--cloud "azure://backups/backup.sql?account=myaccount&key=KEY" \
|
|
||||||
--progress
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Concurrent Operations
|
### Concurrent Operations
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Backup multiple databases in parallel
|
# Backup cluster with parallel jobs
|
||||||
dbbackup backup postgres \
|
dbbackup backup cluster \
|
||||||
--host localhost \
|
|
||||||
--all-databases \
|
|
||||||
--output-dir /backups \
|
|
||||||
--cloud "azure://backups/cluster/?account=myaccount&key=KEY" \
|
--cloud "azure://backups/cluster/?account=myaccount&key=KEY" \
|
||||||
--parallelism 4
|
--jobs 4
|
||||||
```
|
```
|
||||||
|
|
||||||
### Custom Metadata
|
### Custom Metadata
|
||||||
@@ -365,11 +340,8 @@ Endpoint: http://localhost:10000/devstoreaccount1
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Backup to Azurite
|
# Backup to Azurite
|
||||||
dbbackup backup postgres \
|
dbbackup backup single testdb \
|
||||||
--host localhost \
|
--cloud "azure://test-backups/?endpoint=http://localhost:10000&account=devstoreaccount1&key=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
|
||||||
--database testdb \
|
|
||||||
--output test.sql \
|
|
||||||
--cloud "azure://test-backups/test.sql?endpoint=http://localhost:10000&account=devstoreaccount1&key=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Run Integration Tests
|
### Run Integration Tests
|
||||||
@@ -492,8 +464,8 @@ Tests include:
|
|||||||
Enable debug mode:
|
Enable debug mode:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
dbbackup backup postgres \
|
dbbackup backup single mydb \
|
||||||
--cloud "azure://container/backup.sql?account=myaccount&key=KEY" \
|
--cloud "azure://container/?account=myaccount&key=KEY" \
|
||||||
--debug
|
--debug
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
390
CHANGELOG.md
390
CHANGELOG.md
@@ -5,6 +5,395 @@ All notable changes to dbbackup will be documented in this file.
|
|||||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
||||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||||
|
|
||||||
|
## [3.42.10] - 2026-01-08 "Code Quality"
|
||||||
|
|
||||||
|
### Fixed - Code Quality Issues
|
||||||
|
- Removed deprecated `io/ioutil` usage (replaced with `os`)
|
||||||
|
- Fixed `os.DirEntry.ModTime()` → `file.Info().ModTime()`
|
||||||
|
- Removed unused fields and variables
|
||||||
|
- Fixed ineffective assignments in TUI code
|
||||||
|
- Fixed error strings (no capitalization, no trailing punctuation)
|
||||||
|
|
||||||
|
## [3.42.9] - 2026-01-08 "Diagnose Timeout Fix"
|
||||||
|
|
||||||
|
### Fixed - diagnose.go Timeout Bugs
|
||||||
|
|
||||||
|
**More short timeouts that caused large archive failures:**
|
||||||
|
|
||||||
|
- `diagnoseClusterArchive()`: tar listing 60s → **5 minutes**
|
||||||
|
- `verifyWithPgRestore()`: pg_restore --list 60s → **5 minutes**
|
||||||
|
- `DiagnoseClusterDumps()`: archive listing 120s → **10 minutes**
|
||||||
|
|
||||||
|
**Impact:** These timeouts caused "context deadline exceeded" errors when
|
||||||
|
diagnosing multi-GB backup archives, preventing TUI restore from even starting.
|
||||||
|
|
||||||
|
## [3.42.8] - 2026-01-08 "TUI Timeout Fix"
|
||||||
|
|
||||||
|
### Fixed - TUI Timeout Bugs Causing Backup/Restore Failures
|
||||||
|
|
||||||
|
**ROOT CAUSE of 2-3 month TUI backup/restore failures identified and fixed:**
|
||||||
|
|
||||||
|
#### Critical Timeout Fixes:
|
||||||
|
- **restore_preview.go**: Safety check timeout increased from 60s → **10 minutes**
|
||||||
|
- Large archives (>1GB) take 2+ minutes to diagnose
|
||||||
|
- Users saw "context deadline exceeded" before backup even started
|
||||||
|
- **dbselector.go**: Database listing timeout increased from 15s → **60 seconds**
|
||||||
|
- Busy PostgreSQL servers need more time to respond
|
||||||
|
- **status.go**: Status check timeout increased from 10s → **30 seconds**
|
||||||
|
- SSL negotiation and slow networks caused failures
|
||||||
|
|
||||||
|
#### Stability Improvements:
|
||||||
|
- **Panic recovery** added to parallel goroutines in:
|
||||||
|
- `backup/engine.go:BackupCluster()` - cluster backup workers
|
||||||
|
- `restore/engine.go:RestoreCluster()` - cluster restore workers
|
||||||
|
- Prevents single database panic from crashing entire operation
|
||||||
|
|
||||||
|
#### Bug Fix:
|
||||||
|
- **restore/engine.go**: Fixed variable shadowing `err` → `cmdErr` for exit code detection
|
||||||
|
|
||||||
|
## [3.42.7] - 2026-01-08 "Context Killer Complete"
|
||||||
|
|
||||||
|
### Fixed - Additional Deadlock Bugs in Restore & Engine
|
||||||
|
|
||||||
|
**All remaining cmd.Wait() deadlock bugs fixed across the codebase:**
|
||||||
|
|
||||||
|
#### internal/restore/engine.go:
|
||||||
|
- `executeRestoreWithDecompression()` - gunzip/pigz pipeline restore
|
||||||
|
- `extractArchive()` - tar extraction for cluster restore
|
||||||
|
- `restoreGlobals()` - pg_dumpall globals restore
|
||||||
|
|
||||||
|
#### internal/backup/engine.go:
|
||||||
|
- `createArchive()` - tar/pigz archive creation pipeline
|
||||||
|
|
||||||
|
#### internal/engine/mysqldump.go:
|
||||||
|
- `Backup()` - mysqldump backup operation
|
||||||
|
- `BackupToWriter()` - streaming mysqldump to writer
|
||||||
|
|
||||||
|
**All 6 functions now use proper channel-based context handling with Process.Kill().**
|
||||||
|
|
||||||
|
## [3.42.6] - 2026-01-08 "Deadlock Killer"
|
||||||
|
|
||||||
|
### Fixed - Backup Command Context Handling
|
||||||
|
|
||||||
|
**Critical Bug: pg_dump/mysqldump could hang forever on context cancellation**
|
||||||
|
|
||||||
|
The `executeCommand`, `executeCommandWithProgress`, `executeMySQLWithProgressAndCompression`,
|
||||||
|
and `executeMySQLWithCompression` functions had a race condition where:
|
||||||
|
|
||||||
|
1. A goroutine was spawned to read stderr
|
||||||
|
2. `cmd.Wait()` was called directly
|
||||||
|
3. If context was cancelled, the process was NOT killed
|
||||||
|
4. The goroutine could hang forever waiting for stderr
|
||||||
|
|
||||||
|
**Fix**: All backup execution functions now use proper channel-based context handling:
|
||||||
|
```go
|
||||||
|
// Wait for command with context handling
|
||||||
|
cmdDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
cmdDone <- cmd.Wait()
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case cmdErr = <-cmdDone:
|
||||||
|
// Command completed
|
||||||
|
case <-ctx.Done():
|
||||||
|
// Context cancelled - kill process
|
||||||
|
cmd.Process.Kill()
|
||||||
|
<-cmdDone
|
||||||
|
cmdErr = ctx.Err()
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Affected Functions:**
|
||||||
|
- `executeCommand()` - pg_dump for cluster backup
|
||||||
|
- `executeCommandWithProgress()` - pg_dump for single backup with progress
|
||||||
|
- `executeMySQLWithProgressAndCompression()` - mysqldump pipeline
|
||||||
|
- `executeMySQLWithCompression()` - mysqldump pipeline
|
||||||
|
|
||||||
|
**This fixes:** Backup operations hanging indefinitely when cancelled or timing out.
|
||||||
|
|
||||||
|
## [3.42.5] - 2026-01-08 "False Positive Fix"
|
||||||
|
|
||||||
|
### Fixed - Encryption Detection Bug
|
||||||
|
|
||||||
|
**IsBackupEncrypted False Positive:**
|
||||||
|
- **BUG FIX**: `IsBackupEncrypted()` returned `true` for ALL files, blocking normal restores
|
||||||
|
- Root cause: Fallback logic checked if first 12 bytes (nonce size) could be read - always true
|
||||||
|
- Fix: Now properly detects known unencrypted formats by magic bytes:
|
||||||
|
- Gzip: `1f 8b`
|
||||||
|
- PostgreSQL custom: `PGDMP`
|
||||||
|
- Plain SQL: starts with `--`, `SET`, `CREATE`
|
||||||
|
- Returns `false` if no metadata present and format is recognized as unencrypted
|
||||||
|
- Affected file: `internal/backup/encryption.go`
|
||||||
|
|
||||||
|
## [3.42.4] - 2026-01-08 "The Long Haul"
|
||||||
|
|
||||||
|
### Fixed - Critical Restore Timeout Bug
|
||||||
|
|
||||||
|
**Removed Arbitrary Timeouts from Backup/Restore Operations:**
|
||||||
|
- **CRITICAL FIX**: Removed 4-hour timeout that was killing large database restores
|
||||||
|
- PostgreSQL cluster restores of 69GB+ databases no longer fail with "context deadline exceeded"
|
||||||
|
- All backup/restore operations now use `context.WithCancel` instead of `context.WithTimeout`
|
||||||
|
- Operations run until completion or manual cancellation (Ctrl+C)
|
||||||
|
|
||||||
|
**Affected Files:**
|
||||||
|
- `internal/tui/restore_exec.go`: Changed from 4-hour timeout to context.WithCancel
|
||||||
|
- `internal/tui/backup_exec.go`: Changed from 4-hour timeout to context.WithCancel
|
||||||
|
- `internal/backup/engine.go`: Removed per-database timeout in cluster backup
|
||||||
|
- `cmd/restore.go`: CLI restore commands use context.WithCancel
|
||||||
|
|
||||||
|
**exec.Command Context Audit:**
|
||||||
|
- Fixed `exec.Command` without Context in `internal/restore/engine.go:730`
|
||||||
|
- Added proper context handling to all external command calls
|
||||||
|
- Added timeouts only for quick diagnostic/version checks (not restore path):
|
||||||
|
- `restore/version_check.go`: 30s timeout for pg_restore --version check only
|
||||||
|
- `restore/error_report.go`: 10s timeout for tool version detection
|
||||||
|
- `restore/diagnose.go`: 60s timeout for diagnostic functions
|
||||||
|
- `pitr/binlog.go`: 10s timeout for mysqlbinlog --version check
|
||||||
|
- `cleanup/processes.go`: 5s timeout for process listing
|
||||||
|
- `auth/helper.go`: 30s timeout for auth helper commands
|
||||||
|
|
||||||
|
**Verification:**
|
||||||
|
- 54 total `exec.CommandContext` calls verified in backup/restore/pitr path
|
||||||
|
- 0 `exec.Command` without Context in critical restore path
|
||||||
|
- All 14 PostgreSQL exec calls use CommandContext (pg_dump, pg_restore, psql)
|
||||||
|
- All 15 MySQL/MariaDB exec calls use CommandContext (mysqldump, mysql, mysqlbinlog)
|
||||||
|
- All 14 test packages pass
|
||||||
|
|
||||||
|
### Technical Details
|
||||||
|
- Large Object (BLOB/BYTEA) restores are particularly affected by timeouts
|
||||||
|
- 69GB database with large objects can take 5+ hours to restore
|
||||||
|
- Previous 4-hour hard timeout was causing consistent failures
|
||||||
|
- Now: No timeout - runs until complete or user cancels
|
||||||
|
|
||||||
|
## [3.42.1] - 2026-01-07 "Resistance is Futile"
|
||||||
|
|
||||||
|
### Added - Content-Defined Chunking Deduplication
|
||||||
|
|
||||||
|
**Deduplication Engine:**
|
||||||
|
- New `dbbackup dedup` command family for space-efficient backups
|
||||||
|
- Gear hash content-defined chunking (CDC) with 92%+ overlap on shifted data
|
||||||
|
- SHA-256 content-addressed storage - chunks stored by hash
|
||||||
|
- AES-256-GCM per-chunk encryption (optional, via `--encrypt`)
|
||||||
|
- Gzip compression enabled by default
|
||||||
|
- SQLite index for fast chunk lookups
|
||||||
|
- JSON manifests track chunks per backup with full verification
|
||||||
|
|
||||||
|
**Dedup Commands:**
|
||||||
|
```bash
|
||||||
|
dbbackup dedup backup <file> # Create deduplicated backup
|
||||||
|
dbbackup dedup backup <file> --encrypt # With encryption
|
||||||
|
dbbackup dedup restore <id> <output> # Restore from manifest
|
||||||
|
dbbackup dedup list # List all backups
|
||||||
|
dbbackup dedup stats # Show deduplication statistics
|
||||||
|
dbbackup dedup delete <id> # Delete a backup manifest
|
||||||
|
dbbackup dedup gc # Garbage collect unreferenced chunks
|
||||||
|
```
|
||||||
|
|
||||||
|
**Storage Structure:**
|
||||||
|
```
|
||||||
|
<backup-dir>/dedup/
|
||||||
|
chunks/ # Content-addressed chunk files (sharded by hash prefix)
|
||||||
|
manifests/ # JSON manifest per backup
|
||||||
|
chunks.db # SQLite index for fast lookups
|
||||||
|
```
|
||||||
|
|
||||||
|
**Test Results:**
|
||||||
|
- First 5MB backup: 448 chunks, 5MB stored
|
||||||
|
- Modified 5MB file: 448 chunks, only 1 NEW chunk (1.6KB), 100% dedup ratio
|
||||||
|
- Restore with SHA-256 verification
|
||||||
|
|
||||||
|
### Added - Documentation Updates
|
||||||
|
- Prometheus alerting rules added to SYSTEMD.md
|
||||||
|
- Catalog sync instructions for existing backups
|
||||||
|
|
||||||
|
## [3.41.1] - 2026-01-07
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Enabled CGO for Linux builds (required for SQLite catalog)
|
||||||
|
|
||||||
|
## [3.41.0] - 2026-01-07 "The Operator"
|
||||||
|
|
||||||
|
### Added - Systemd Integration & Prometheus Metrics
|
||||||
|
|
||||||
|
**Embedded Systemd Installer:**
|
||||||
|
- New `dbbackup install` command installs as systemd service/timer
|
||||||
|
- Supports single-database (`--backup-type single`) and cluster (`--backup-type cluster`) modes
|
||||||
|
- Automatic `dbbackup` user/group creation with proper permissions
|
||||||
|
- Hardened service units with security features (NoNewPrivileges, ProtectSystem, CapabilityBoundingSet)
|
||||||
|
- Templated timer units with configurable schedules (daily, weekly, or custom OnCalendar)
|
||||||
|
- Built-in dry-run mode (`--dry-run`) to preview installation
|
||||||
|
- `dbbackup install --status` shows current installation state
|
||||||
|
- `dbbackup uninstall` cleanly removes all systemd units and optionally configuration
|
||||||
|
|
||||||
|
**Prometheus Metrics Support:**
|
||||||
|
- New `dbbackup metrics export` command writes textfile collector format
|
||||||
|
- New `dbbackup metrics serve` command runs HTTP exporter on port 9399
|
||||||
|
- Metrics: `dbbackup_last_success_timestamp`, `dbbackup_rpo_seconds`, `dbbackup_backup_total`, etc.
|
||||||
|
- Integration with node_exporter textfile collector
|
||||||
|
- Metrics automatically updated via ExecStopPost in service units
|
||||||
|
- `--with-metrics` flag during install sets up exporter as systemd service
|
||||||
|
|
||||||
|
**New Commands:**
|
||||||
|
```bash
|
||||||
|
# Install as systemd service
|
||||||
|
sudo dbbackup install --backup-type cluster --schedule daily
|
||||||
|
|
||||||
|
# Install with Prometheus metrics
|
||||||
|
sudo dbbackup install --with-metrics --metrics-port 9399
|
||||||
|
|
||||||
|
# Check installation status
|
||||||
|
dbbackup install --status
|
||||||
|
|
||||||
|
# Export metrics for node_exporter
|
||||||
|
dbbackup metrics export --output /var/lib/dbbackup/metrics/dbbackup.prom
|
||||||
|
|
||||||
|
# Run HTTP metrics server
|
||||||
|
dbbackup metrics serve --port 9399
|
||||||
|
```
|
||||||
|
|
||||||
|
### Technical Details
|
||||||
|
- Systemd templates embedded with `//go:embed` for self-contained binary
|
||||||
|
- Templates use ReadWritePaths for security isolation
|
||||||
|
- Service units include proper OOMScoreAdjust (-100) to protect backups
|
||||||
|
- Metrics exporter caches with 30-second TTL for performance
|
||||||
|
- Graceful shutdown on SIGTERM for metrics server
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [3.41.0] - 2026-01-07 "The Pre-Flight Check"
|
||||||
|
|
||||||
|
### Added - 🛡️ Pre-Restore Validation
|
||||||
|
|
||||||
|
**Automatic Dump Validation Before Restore:**
|
||||||
|
- SQL dump files are now validated BEFORE attempting restore
|
||||||
|
- Detects truncated COPY blocks that cause "syntax error" failures
|
||||||
|
- Catches corrupted backups in seconds instead of wasting 49+ minutes
|
||||||
|
- Cluster restore pre-validates ALL dumps upfront (fail-fast approach)
|
||||||
|
- Custom format `.dump` files now validated with `pg_restore --list`
|
||||||
|
|
||||||
|
**Improved Error Messages:**
|
||||||
|
- Clear indication when dump file is truncated
|
||||||
|
- Shows which table's COPY block was interrupted
|
||||||
|
- Displays sample orphaned data for diagnosis
|
||||||
|
- Provides actionable error messages with root cause
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- **P0: SQL Injection** - Added identifier validation for database names in CREATE/DROP DATABASE to prevent SQL injection attacks; uses safe quoting and regex validation (alphanumeric + underscore only)
|
||||||
|
- **P0: Data Race** - Fixed concurrent goroutines appending to shared error slice in notification manager; now uses mutex synchronization
|
||||||
|
- **P0: psql ON_ERROR_STOP** - Added `-v ON_ERROR_STOP=1` to psql commands to fail fast on first error instead of accumulating millions of errors
|
||||||
|
- **P1: Pipe deadlock** - Fixed streaming compression deadlock when pg_dump blocks on full pipe buffer; now uses goroutine with proper context timeout handling
|
||||||
|
- **P1: SIGPIPE handling** - Detect exit code 141 (broken pipe) and report compressor failure as root cause
|
||||||
|
- **P2: .dump validation** - Custom format dumps now validated with `pg_restore --list` before restore
|
||||||
|
- **P2: fsync durability** - Added `outFile.Sync()` after streaming compression to prevent truncation on power loss
|
||||||
|
- Truncated `.sql.gz` dumps no longer waste hours on doomed restores
|
||||||
|
- "syntax error at or near" errors now caught before restore begins
|
||||||
|
- Cluster restores abort immediately if any dump is corrupted
|
||||||
|
|
||||||
|
### Technical Details
|
||||||
|
- Integrated `Diagnoser` into restore pipeline for pre-validation
|
||||||
|
- Added `quickValidateSQLDump()` for fast integrity checks
|
||||||
|
- Pre-validation runs on all `.sql.gz` and `.dump` files in cluster archives
|
||||||
|
- Streaming compression uses channel-based wait with context cancellation
|
||||||
|
- Zero performance impact on valid backups (diagnosis is fast)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [3.40.0] - 2026-01-05 "The Diagnostician"
|
||||||
|
|
||||||
|
### Added - 🔍 Restore Diagnostics & Error Reporting
|
||||||
|
|
||||||
|
**Backup Diagnosis Command:**
|
||||||
|
- `restore diagnose <archive>` - Deep analysis of backup files before restore
|
||||||
|
- Detects truncated dumps, corrupted archives, incomplete COPY blocks
|
||||||
|
- PGDMP signature validation for PostgreSQL custom format
|
||||||
|
- Gzip integrity verification with decompression test
|
||||||
|
- `pg_restore --list` validation for custom format archives
|
||||||
|
- `--deep` flag for exhaustive line-by-line analysis
|
||||||
|
- `--json` flag for machine-readable output
|
||||||
|
- Cluster archive diagnosis scans all contained dumps
|
||||||
|
|
||||||
|
**Detailed Error Reporting:**
|
||||||
|
- Comprehensive error collector captures stderr during restore
|
||||||
|
- Ring buffer prevents OOM on high-error restores (2M+ errors)
|
||||||
|
- Error classification with actionable hints and recommendations
|
||||||
|
- `--save-debug-log <path>` saves JSON report on failure
|
||||||
|
- Reports include: exit codes, last errors, line context, tool versions
|
||||||
|
- Automatic recommendations based on error patterns
|
||||||
|
|
||||||
|
**TUI Restore Enhancements:**
|
||||||
|
- **Dump validity** safety check runs automatically before restore
|
||||||
|
- Detects truncated/corrupted backups in restore preview
|
||||||
|
- Press **`d`** to toggle debug log saving in Advanced Options
|
||||||
|
- Debug logs saved to `/tmp/dbbackup-restore-debug-*.json` on failure
|
||||||
|
- Press **`d`** in archive browser to run diagnosis on any backup
|
||||||
|
|
||||||
|
**New Commands:**
|
||||||
|
- `restore diagnose` - Analyze backup file integrity and structure
|
||||||
|
|
||||||
|
**New Flags:**
|
||||||
|
- `--save-debug-log <path>` - Save detailed JSON error report on failure
|
||||||
|
- `--diagnose` - Run deep diagnosis before cluster restore
|
||||||
|
- `--deep` - Enable exhaustive diagnosis (line-by-line analysis)
|
||||||
|
- `--json` - Output diagnosis in JSON format
|
||||||
|
- `--keep-temp` - Keep temporary files after diagnosis
|
||||||
|
- `--verbose` - Show detailed diagnosis progress
|
||||||
|
|
||||||
|
### Technical Details
|
||||||
|
- 1,200+ lines of new diagnostic code
|
||||||
|
- Error classification system with 15+ error patterns
|
||||||
|
- Ring buffer stderr capture (1MB max, 10K lines)
|
||||||
|
- Zero memory growth on high-error restores
|
||||||
|
- Full TUI integration for diagnostics
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [3.2.0] - 2025-12-13 "The Margin Eraser"
|
||||||
|
|
||||||
|
### Added - 🚀 Physical Backup Revolution
|
||||||
|
|
||||||
|
**MySQL Clone Plugin Integration:**
|
||||||
|
- Native physical backup using MySQL 8.0.17+ Clone Plugin
|
||||||
|
- No XtraBackup dependency - pure Go implementation
|
||||||
|
- Real-time progress monitoring via performance_schema
|
||||||
|
- Support for both local and remote clone operations
|
||||||
|
|
||||||
|
**Filesystem Snapshot Orchestration:**
|
||||||
|
- LVM snapshot support with automatic cleanup
|
||||||
|
- ZFS snapshot integration with send/receive
|
||||||
|
- Btrfs subvolume snapshot support
|
||||||
|
- Brief table lock (<100ms) for consistency
|
||||||
|
- Automatic snapshot backend detection
|
||||||
|
|
||||||
|
**Continuous Binlog Streaming:**
|
||||||
|
- Real-time binlog capture using MySQL replication protocol
|
||||||
|
- Multiple targets: file, compressed file, S3 direct streaming
|
||||||
|
- Sub-second RPO without impacting database server
|
||||||
|
- Automatic position tracking and checkpointing
|
||||||
|
|
||||||
|
**Parallel Cloud Streaming:**
|
||||||
|
- Direct database-to-S3 streaming (zero local storage)
|
||||||
|
- Configurable worker pool for parallel uploads
|
||||||
|
- S3 multipart upload with automatic retry
|
||||||
|
- Support for S3, GCS, and Azure Blob Storage
|
||||||
|
|
||||||
|
**Smart Engine Selection:**
|
||||||
|
- Automatic engine selection based on environment
|
||||||
|
- MySQL version detection and capability checking
|
||||||
|
- Filesystem type detection for optimal snapshot backend
|
||||||
|
- Database size-based recommendations
|
||||||
|
|
||||||
|
**New Commands:**
|
||||||
|
- `engine list` - List available backup engines
|
||||||
|
- `engine info <name>` - Show detailed engine information
|
||||||
|
- `backup --engine=<name>` - Use specific backup engine
|
||||||
|
|
||||||
|
### Technical Details
|
||||||
|
- 7,559 lines of new code
|
||||||
|
- Zero new external dependencies
|
||||||
|
- 10/10 platform builds successful
|
||||||
|
- Full test coverage for new engines
|
||||||
|
|
||||||
## [3.1.0] - 2025-11-26
|
## [3.1.0] - 2025-11-26
|
||||||
|
|
||||||
### Added - 🔄 Point-in-Time Recovery (PITR)
|
### Added - 🔄 Point-in-Time Recovery (PITR)
|
||||||
@@ -117,7 +506,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
### Documentation
|
### Documentation
|
||||||
- Added comprehensive PITR.md guide (complete PITR documentation)
|
- Added comprehensive PITR.md guide (complete PITR documentation)
|
||||||
- Updated README.md with PITR section (200+ lines)
|
- Updated README.md with PITR section (200+ lines)
|
||||||
- Added RELEASE_NOTES_v3.1.md (full feature list)
|
|
||||||
- Updated CHANGELOG.md with v3.1.0 details
|
- Updated CHANGELOG.md with v3.1.0 details
|
||||||
- Added NOTICE file for Apache License attribution
|
- Added NOTICE file for Apache License attribution
|
||||||
- Created comprehensive test suite (tests/pitr_complete_test.go - 700+ lines)
|
- Created comprehensive test suite (tests/pitr_complete_test.go - 700+ lines)
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ Be respectful, constructive, and professional in all interactions. We're buildin
|
|||||||
|
|
||||||
**Bug Report Template:**
|
**Bug Report Template:**
|
||||||
```
|
```
|
||||||
**Version:** dbbackup v3.1.0
|
**Version:** dbbackup v3.42.1
|
||||||
**OS:** Linux/macOS/BSD
|
**OS:** Linux/macOS/BSD
|
||||||
**Database:** PostgreSQL 14 / MySQL 8.0 / MariaDB 10.6
|
**Database:** PostgreSQL 14 / MySQL 8.0 / MariaDB 10.6
|
||||||
**Command:** The exact command that failed
|
**Command:** The exact command that failed
|
||||||
@@ -274,12 +274,11 @@ Fixes #56
|
|||||||
|
|
||||||
1. Update version in `main.go`
|
1. Update version in `main.go`
|
||||||
2. Update `CHANGELOG.md`
|
2. Update `CHANGELOG.md`
|
||||||
3. Create release notes (`RELEASE_NOTES_vX.Y.Z.md`)
|
3. Commit: `git commit -m "Release vX.Y.Z"`
|
||||||
4. Commit: `git commit -m "Release vX.Y.Z"`
|
4. Tag: `git tag -a vX.Y.Z -m "Release vX.Y.Z"`
|
||||||
5. Tag: `git tag -a vX.Y.Z -m "Release vX.Y.Z"`
|
5. Push: `git push origin main vX.Y.Z`
|
||||||
6. Push: `git push origin main vX.Y.Z`
|
6. Build binaries: `./build_all.sh`
|
||||||
7. Build binaries: `./build_all.sh`
|
7. Create GitHub Release with binaries
|
||||||
8. Create GitHub Release with binaries
|
|
||||||
|
|
||||||
## Questions?
|
## Questions?
|
||||||
|
|
||||||
|
|||||||
295
EMOTICON_REMOVAL_PLAN.md
Normal file
295
EMOTICON_REMOVAL_PLAN.md
Normal file
@@ -0,0 +1,295 @@
|
|||||||
|
# Emoticon Removal Plan for Python Code
|
||||||
|
|
||||||
|
## ⚠️ CRITICAL: Code Must Remain Functional After Removal
|
||||||
|
|
||||||
|
This document outlines a **safe, systematic approach** to removing emoticons from Python code without breaking functionality.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Identification Phase
|
||||||
|
|
||||||
|
### 1.1 Where Emoticons CAN Safely Exist (Safe to Remove)
|
||||||
|
| Location | Risk Level | Action |
|
||||||
|
|----------|------------|--------|
|
||||||
|
| Comments (`# 🎉 Success!`) | ✅ SAFE | Remove or replace with text |
|
||||||
|
| Docstrings (`"""📌 Note:..."""`) | ✅ SAFE | Remove or replace with text |
|
||||||
|
| Print statements for decoration (`print("✅ Done!")`) | ⚠️ LOW | Replace with ASCII or text |
|
||||||
|
| Logging messages (`logger.info("🔥 Starting...")`) | ⚠️ LOW | Replace with text equivalent |
|
||||||
|
|
||||||
|
### 1.2 Where Emoticons are DANGEROUS to Remove
|
||||||
|
| Location | Risk Level | Action |
|
||||||
|
|----------|------------|--------|
|
||||||
|
| String literals used in logic | 🚨 HIGH | **DO NOT REMOVE** without analysis |
|
||||||
|
| Dictionary keys (`{"🔑": value}`) | 🚨 CRITICAL | **NEVER REMOVE** - breaks code |
|
||||||
|
| Regex patterns | 🚨 CRITICAL | **NEVER REMOVE** - breaks matching |
|
||||||
|
| String comparisons (`if x == "✅"`) | 🚨 CRITICAL | Requires refactoring, not just removal |
|
||||||
|
| Database/API payloads | 🚨 CRITICAL | May break external systems |
|
||||||
|
| File content markers | 🚨 HIGH | May break parsing logic |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Pre-Removal Checklist
|
||||||
|
|
||||||
|
### 2.1 Before ANY Changes
|
||||||
|
- [ ] **Full backup** of the codebase
|
||||||
|
- [ ] **Run all tests** and record baseline results
|
||||||
|
- [ ] **Document all emoticon locations** with grep/search
|
||||||
|
- [ ] **Identify emoticon usage patterns** (decorative vs. functional)
|
||||||
|
|
||||||
|
### 2.2 Discovery Commands
|
||||||
|
```bash
|
||||||
|
# Find all files with emoticons (Unicode range for common emojis)
|
||||||
|
grep -rn --include="*.py" -P '[\x{1F300}-\x{1F9FF}]' .
|
||||||
|
|
||||||
|
# Find emoticons in strings
|
||||||
|
grep -rn --include="*.py" -E '["'"'"'][^"'"'"']*[\x{1F300}-\x{1F9FF}]' .
|
||||||
|
|
||||||
|
# List unique emoticons used
|
||||||
|
grep -oP '[\x{1F300}-\x{1F9FF}]' *.py | sort -u
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Replacement Strategy
|
||||||
|
|
||||||
|
### 3.1 Semantic Replacement Table
|
||||||
|
| Emoticon | Text Replacement | Context |
|
||||||
|
|----------|------------------|---------|
|
||||||
|
| ✅ | `[OK]` or `[SUCCESS]` | Status indicators |
|
||||||
|
| ❌ | `[FAIL]` or `[ERROR]` | Error indicators |
|
||||||
|
| ⚠️ | `[WARNING]` | Warning messages |
|
||||||
|
| 🔥 | `[HOT]` or `` (remove) | Decorative |
|
||||||
|
| 🎉 | `[DONE]` or `` (remove) | Celebration/completion |
|
||||||
|
| 📌 | `[NOTE]` | Notes/pinned items |
|
||||||
|
| 🚀 | `[START]` or `` (remove) | Launch/start indicators |
|
||||||
|
| 💾 | `[SAVE]` | Save operations |
|
||||||
|
| 🔑 | `[KEY]` | Key/authentication |
|
||||||
|
| 📁 | `[FILE]` | File operations |
|
||||||
|
| 🔍 | `[SEARCH]` | Search operations |
|
||||||
|
| ⏳ | `[WAIT]` or `[LOADING]` | Progress indicators |
|
||||||
|
| 🛑 | `[STOP]` | Stop/halt indicators |
|
||||||
|
| ℹ️ | `[INFO]` | Information |
|
||||||
|
| 🐛 | `[BUG]` or `[DEBUG]` | Debug messages |
|
||||||
|
|
||||||
|
### 3.2 Context-Aware Replacement Rules
|
||||||
|
|
||||||
|
```
|
||||||
|
RULE 1: Comments
|
||||||
|
- Remove emoticon entirely OR replace with text
|
||||||
|
- Example: `# 🎉 Feature complete` → `# Feature complete`
|
||||||
|
|
||||||
|
RULE 2: User-facing strings (print/logging)
|
||||||
|
- Replace with semantic text equivalent
|
||||||
|
- Example: `print("✅ Backup complete")` → `print("[OK] Backup complete")`
|
||||||
|
|
||||||
|
RULE 3: Functional strings (DANGER ZONE)
|
||||||
|
- DO NOT auto-replace
|
||||||
|
- Requires manual code refactoring
|
||||||
|
- Example: `status = "✅"` → Refactor to `status = "success"` AND update all comparisons
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Safe Removal Process
|
||||||
|
|
||||||
|
### Step 1: Audit
|
||||||
|
```python
|
||||||
|
# Python script to audit emoticon usage
|
||||||
|
import re
|
||||||
|
import ast
|
||||||
|
|
||||||
|
EMOJI_PATTERN = re.compile(
|
||||||
|
"["
|
||||||
|
"\U0001F300-\U0001F9FF" # Symbols & Pictographs
|
||||||
|
"\U00002600-\U000026FF" # Misc symbols
|
||||||
|
"\U00002700-\U000027BF" # Dingbats
|
||||||
|
"\U0001F600-\U0001F64F" # Emoticons
|
||||||
|
"]+"
|
||||||
|
)
|
||||||
|
|
||||||
|
def audit_file(filepath):
|
||||||
|
with open(filepath, 'r', encoding='utf-8') as f:
|
||||||
|
content = f.read()
|
||||||
|
|
||||||
|
# Parse AST to understand context
|
||||||
|
tree = ast.parse(content)
|
||||||
|
|
||||||
|
findings = []
|
||||||
|
for lineno, line in enumerate(content.split('\n'), 1):
|
||||||
|
matches = EMOJI_PATTERN.findall(line)
|
||||||
|
if matches:
|
||||||
|
# Determine context (comment, string, etc.)
|
||||||
|
context = classify_context(line, matches)
|
||||||
|
findings.append({
|
||||||
|
'line': lineno,
|
||||||
|
'content': line.strip(),
|
||||||
|
'emojis': matches,
|
||||||
|
'context': context,
|
||||||
|
'risk': assess_risk(context)
|
||||||
|
})
|
||||||
|
return findings
|
||||||
|
|
||||||
|
def classify_context(line, matches):
|
||||||
|
stripped = line.strip()
|
||||||
|
if stripped.startswith('#'):
|
||||||
|
return 'COMMENT'
|
||||||
|
if 'print(' in line or 'logging.' in line or 'logger.' in line:
|
||||||
|
return 'OUTPUT'
|
||||||
|
if '==' in line or '!=' in line:
|
||||||
|
return 'COMPARISON'
|
||||||
|
if re.search(r'["\'][^"\']*$', line.split('#')[0]):
|
||||||
|
return 'STRING_LITERAL'
|
||||||
|
return 'UNKNOWN'
|
||||||
|
|
||||||
|
def assess_risk(context):
|
||||||
|
risk_map = {
|
||||||
|
'COMMENT': 'LOW',
|
||||||
|
'OUTPUT': 'LOW',
|
||||||
|
'COMPARISON': 'CRITICAL',
|
||||||
|
'STRING_LITERAL': 'HIGH',
|
||||||
|
'UNKNOWN': 'HIGH'
|
||||||
|
}
|
||||||
|
return risk_map.get(context, 'HIGH')
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 2: Generate Change Plan
|
||||||
|
```python
|
||||||
|
def generate_change_plan(findings):
|
||||||
|
plan = {'safe': [], 'review_required': [], 'do_not_touch': []}
|
||||||
|
|
||||||
|
for finding in findings:
|
||||||
|
if finding['risk'] == 'LOW':
|
||||||
|
plan['safe'].append(finding)
|
||||||
|
elif finding['risk'] == 'HIGH':
|
||||||
|
plan['review_required'].append(finding)
|
||||||
|
else: # CRITICAL
|
||||||
|
plan['do_not_touch'].append(finding)
|
||||||
|
|
||||||
|
return plan
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 3: Apply Changes (SAFE items only)
|
||||||
|
```python
|
||||||
|
def apply_safe_replacements(filepath, replacements):
|
||||||
|
# Create backup first!
|
||||||
|
import shutil
|
||||||
|
shutil.copy(filepath, filepath + '.backup')
|
||||||
|
|
||||||
|
with open(filepath, 'r', encoding='utf-8') as f:
|
||||||
|
content = f.read()
|
||||||
|
|
||||||
|
for old, new in replacements:
|
||||||
|
content = content.replace(old, new)
|
||||||
|
|
||||||
|
with open(filepath, 'w', encoding='utf-8') as f:
|
||||||
|
f.write(content)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 4: Validate
|
||||||
|
```bash
|
||||||
|
# After each file change:
|
||||||
|
python -m py_compile <modified_file.py> # Syntax check
|
||||||
|
pytest <related_tests> # Run tests
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Validation Checklist
|
||||||
|
|
||||||
|
### After EACH File Modification
|
||||||
|
- [ ] File compiles without syntax errors (`python -m py_compile file.py`)
|
||||||
|
- [ ] All imports still work
|
||||||
|
- [ ] Related unit tests pass
|
||||||
|
- [ ] Integration tests pass
|
||||||
|
- [ ] Manual smoke test if applicable
|
||||||
|
|
||||||
|
### After ALL Modifications
|
||||||
|
- [ ] Full test suite passes
|
||||||
|
- [ ] Application starts correctly
|
||||||
|
- [ ] Key functionality verified manually
|
||||||
|
- [ ] No new warnings in logs
|
||||||
|
- [ ] Compare output with baseline
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Rollback Plan
|
||||||
|
|
||||||
|
### If Something Breaks
|
||||||
|
1. **Immediate**: Restore from `.backup` files
|
||||||
|
2. **Git**: `git checkout -- <file>` or `git stash pop`
|
||||||
|
3. **Full rollback**: Restore from pre-change backup
|
||||||
|
|
||||||
|
### Keep Until Verified
|
||||||
|
```bash
|
||||||
|
# Backup storage structure
|
||||||
|
backups/
|
||||||
|
├── pre_emoticon_removal/
|
||||||
|
│ ├── timestamp.tar.gz
|
||||||
|
│ └── git_commit_hash.txt
|
||||||
|
└── individual_files/
|
||||||
|
├── file1.py.backup
|
||||||
|
└── file2.py.backup
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Implementation Order
|
||||||
|
|
||||||
|
1. **Phase 1**: Comments only (LOWEST risk)
|
||||||
|
2. **Phase 2**: Docstrings (LOW risk)
|
||||||
|
3. **Phase 3**: Print/logging statements (LOW-MEDIUM risk)
|
||||||
|
4. **Phase 4**: Manual review items (HIGH risk) - one by one
|
||||||
|
5. **Phase 5**: NEVER touch CRITICAL items without full refactoring
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Example Workflow
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Create full backup
|
||||||
|
git stash && git checkout -b emoticon-removal
|
||||||
|
|
||||||
|
# 2. Run audit script
|
||||||
|
python emoticon_audit.py > audit_report.json
|
||||||
|
|
||||||
|
# 3. Review audit report
|
||||||
|
cat audit_report.json | jq '.do_not_touch' # Check critical items
|
||||||
|
|
||||||
|
# 4. Apply safe changes only
|
||||||
|
python apply_safe_changes.py --dry-run # Preview first!
|
||||||
|
python apply_safe_changes.py # Apply
|
||||||
|
|
||||||
|
# 5. Validate after each change
|
||||||
|
python -m pytest tests/
|
||||||
|
|
||||||
|
# 6. Commit incrementally
|
||||||
|
git add -p # Review each change
|
||||||
|
git commit -m "Remove emoticons from comments in module X"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. DO NOT DO
|
||||||
|
|
||||||
|
❌ **Never** use global find-replace on emoticons
|
||||||
|
❌ **Never** remove emoticons from string comparisons without refactoring
|
||||||
|
❌ **Never** change multiple files without testing between changes
|
||||||
|
❌ **Never** assume an emoticon is decorative - verify context
|
||||||
|
❌ **Never** proceed if tests fail after a change
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. Sign-Off Requirements
|
||||||
|
|
||||||
|
Before merging emoticon removal changes:
|
||||||
|
- [ ] All tests pass (100%)
|
||||||
|
- [ ] Code review by second developer
|
||||||
|
- [ ] Manual testing of affected features
|
||||||
|
- [ ] Documented all CRITICAL items left unchanged (with justification)
|
||||||
|
- [ ] Backup verified and accessible
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**Author**: Generated Plan
|
||||||
|
**Date**: 2026-01-07
|
||||||
|
**Status**: PLAN ONLY - No code changes made
|
||||||
80
GCS.md
80
GCS.md
@@ -28,21 +28,16 @@ This guide covers using **Google Cloud Storage (GCS)** with `dbbackup` for secur
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Backup PostgreSQL to GCS (using ADC)
|
# Backup PostgreSQL to GCS (using ADC)
|
||||||
dbbackup backup postgres \
|
dbbackup backup single mydb \
|
||||||
--host localhost \
|
--cloud "gs://mybucket/backups/"
|
||||||
--database mydb \
|
|
||||||
--output backup.sql \
|
|
||||||
--cloud "gs://mybucket/backups/db.sql"
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### 3. Restore from GCS
|
### 3. Restore from GCS
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Restore from GCS backup
|
# Download backup from GCS and restore
|
||||||
dbbackup restore postgres \
|
dbbackup cloud download "gs://mybucket/backups/mydb.dump.gz" ./mydb.dump.gz
|
||||||
--source "gs://mybucket/backups/db.sql" \
|
dbbackup restore single ./mydb.dump.gz --target mydb_restored --confirm
|
||||||
--host localhost \
|
|
||||||
--database mydb_restored
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## URI Syntax
|
## URI Syntax
|
||||||
@@ -107,7 +102,7 @@ gcloud auth application-default login
|
|||||||
gcloud auth activate-service-account --key-file=/path/to/key.json
|
gcloud auth activate-service-account --key-file=/path/to/key.json
|
||||||
|
|
||||||
# Use simplified URI (credentials from environment)
|
# Use simplified URI (credentials from environment)
|
||||||
dbbackup backup postgres --cloud "gs://mybucket/backups/backup.sql"
|
dbbackup backup single mydb --cloud "gs://mybucket/backups/"
|
||||||
```
|
```
|
||||||
|
|
||||||
### Method 2: Service Account JSON
|
### Method 2: Service Account JSON
|
||||||
@@ -121,14 +116,14 @@ Download service account key from GCP Console:
|
|||||||
|
|
||||||
**Use in URI:**
|
**Use in URI:**
|
||||||
```bash
|
```bash
|
||||||
dbbackup backup postgres \
|
dbbackup backup single mydb \
|
||||||
--cloud "gs://mybucket/backup.sql?credentials=/path/to/service-account.json"
|
--cloud "gs://mybucket/?credentials=/path/to/service-account.json"
|
||||||
```
|
```
|
||||||
|
|
||||||
**Or via environment:**
|
**Or via environment:**
|
||||||
```bash
|
```bash
|
||||||
export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account.json"
|
export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account.json"
|
||||||
dbbackup backup postgres --cloud "gs://mybucket/backup.sql"
|
dbbackup backup single mydb --cloud "gs://mybucket/"
|
||||||
```
|
```
|
||||||
|
|
||||||
### Method 3: Workload Identity (GKE)
|
### Method 3: Workload Identity (GKE)
|
||||||
@@ -147,7 +142,7 @@ metadata:
|
|||||||
Then use ADC in your pod:
|
Then use ADC in your pod:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
dbbackup backup postgres --cloud "gs://mybucket/backup.sql"
|
dbbackup backup single mydb --cloud "gs://mybucket/"
|
||||||
```
|
```
|
||||||
|
|
||||||
### Required IAM Permissions
|
### Required IAM Permissions
|
||||||
@@ -250,11 +245,8 @@ gsutil mb -l eu gs://mybucket/
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# PostgreSQL backup with automatic GCS upload
|
# PostgreSQL backup with automatic GCS upload
|
||||||
dbbackup backup postgres \
|
dbbackup backup single production_db \
|
||||||
--host localhost \
|
--cloud "gs://prod-backups/postgres/" \
|
||||||
--database production_db \
|
|
||||||
--output /backups/db.sql \
|
|
||||||
--cloud "gs://prod-backups/postgres/$(date +%Y%m%d_%H%M%S).sql" \
|
|
||||||
--compression 6
|
--compression 6
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -262,10 +254,7 @@ dbbackup backup postgres \
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Backup entire PostgreSQL cluster to GCS
|
# Backup entire PostgreSQL cluster to GCS
|
||||||
dbbackup backup postgres \
|
dbbackup backup cluster \
|
||||||
--host localhost \
|
|
||||||
--all-databases \
|
|
||||||
--output-dir /backups \
|
|
||||||
--cloud "gs://prod-backups/postgres/cluster/"
|
--cloud "gs://prod-backups/postgres/cluster/"
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -314,13 +303,9 @@ dbbackup cleanup "gs://prod-backups/postgres/" --keep 7
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
# GCS backup script (run via cron)
|
# GCS backup script (run via cron)
|
||||||
|
|
||||||
DATE=$(date +%Y%m%d_%H%M%S)
|
GCS_URI="gs://prod-backups/postgres/"
|
||||||
GCS_URI="gs://prod-backups/postgres/${DATE}.sql"
|
|
||||||
|
|
||||||
dbbackup backup postgres \
|
dbbackup backup single production_db \
|
||||||
--host localhost \
|
|
||||||
--database production_db \
|
|
||||||
--output /tmp/backup.sql \
|
|
||||||
--cloud "${GCS_URI}" \
|
--cloud "${GCS_URI}" \
|
||||||
--compression 9
|
--compression 9
|
||||||
|
|
||||||
@@ -360,35 +345,25 @@ For large files, dbbackup automatically uses GCS chunked upload:
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Large database backup (automatically uses chunked upload)
|
# Large database backup (automatically uses chunked upload)
|
||||||
dbbackup backup postgres \
|
dbbackup backup single huge_db \
|
||||||
--host localhost \
|
--cloud "gs://backups/"
|
||||||
--database huge_db \
|
|
||||||
--output /backups/huge.sql \
|
|
||||||
--cloud "gs://backups/huge.sql"
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Progress Tracking
|
### Progress Tracking
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Backup with progress display
|
# Backup with progress display
|
||||||
dbbackup backup postgres \
|
dbbackup backup single mydb \
|
||||||
--host localhost \
|
--cloud "gs://backups/"
|
||||||
--database mydb \
|
|
||||||
--output backup.sql \
|
|
||||||
--cloud "gs://backups/backup.sql" \
|
|
||||||
--progress
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Concurrent Operations
|
### Concurrent Operations
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Backup multiple databases in parallel
|
# Backup cluster with parallel jobs
|
||||||
dbbackup backup postgres \
|
dbbackup backup cluster \
|
||||||
--host localhost \
|
|
||||||
--all-databases \
|
|
||||||
--output-dir /backups \
|
|
||||||
--cloud "gs://backups/cluster/" \
|
--cloud "gs://backups/cluster/" \
|
||||||
--parallelism 4
|
--jobs 4
|
||||||
```
|
```
|
||||||
|
|
||||||
### Custom Metadata
|
### Custom Metadata
|
||||||
@@ -460,11 +435,8 @@ curl -X POST "http://localhost:4443/storage/v1/b?project=test-project" \
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Backup to fake-gcs-server
|
# Backup to fake-gcs-server
|
||||||
dbbackup backup postgres \
|
dbbackup backup single testdb \
|
||||||
--host localhost \
|
--cloud "gs://test-backups/?endpoint=http://localhost:4443/storage/v1"
|
||||||
--database testdb \
|
|
||||||
--output test.sql \
|
|
||||||
--cloud "gs://test-backups/test.sql?endpoint=http://localhost:4443/storage/v1"
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Run Integration Tests
|
### Run Integration Tests
|
||||||
@@ -593,8 +565,8 @@ Tests include:
|
|||||||
Enable debug mode:
|
Enable debug mode:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
dbbackup backup postgres \
|
dbbackup backup single mydb \
|
||||||
--cloud "gs://bucket/backup.sql" \
|
--cloud "gs://bucket/" \
|
||||||
--debug
|
--debug
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -110,10 +110,12 @@ dbbackup pitr mysql-enable --archive-dir /backups/binlog_archive
|
|||||||
### 3. Create a Base Backup
|
### 3. Create a Base Backup
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Create a PITR-capable backup
|
# Create a backup - binlog position is automatically recorded
|
||||||
dbbackup backup single mydb --pitr
|
dbbackup backup single mydb
|
||||||
```
|
```
|
||||||
|
|
||||||
|
> **Note:** All backups automatically capture the current binlog position when PITR is enabled at the MySQL level. This position is stored in the backup metadata and used as the starting point for binlog replay during recovery.
|
||||||
|
|
||||||
### 4. Start Binlog Archiving
|
### 4. Start Binlog Archiving
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
190
README.md
190
README.md
@@ -19,6 +19,8 @@ Database backup and restore utility for PostgreSQL, MySQL, and MariaDB.
|
|||||||
- Point-in-Time Recovery (PITR) for PostgreSQL and MySQL/MariaDB
|
- Point-in-Time Recovery (PITR) for PostgreSQL and MySQL/MariaDB
|
||||||
- **GFS retention policies**: Grandfather-Father-Son backup rotation
|
- **GFS retention policies**: Grandfather-Father-Son backup rotation
|
||||||
- **Notifications**: SMTP email and webhook alerts
|
- **Notifications**: SMTP email and webhook alerts
|
||||||
|
- **Systemd integration**: Install as service with scheduled timers
|
||||||
|
- **Prometheus metrics**: Textfile collector and HTTP exporter
|
||||||
- Interactive terminal UI
|
- Interactive terminal UI
|
||||||
- Cross-platform binaries
|
- Cross-platform binaries
|
||||||
|
|
||||||
@@ -54,7 +56,7 @@ Download from [releases](https://git.uuxo.net/UUXO/dbbackup/releases):
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Linux x86_64
|
# Linux x86_64
|
||||||
wget https://git.uuxo.net/UUXO/dbbackup/releases/download/v3.1.0/dbbackup-linux-amd64
|
wget https://git.uuxo.net/UUXO/dbbackup/releases/download/v3.42.1/dbbackup-linux-amd64
|
||||||
chmod +x dbbackup-linux-amd64
|
chmod +x dbbackup-linux-amd64
|
||||||
sudo mv dbbackup-linux-amd64 /usr/local/bin/dbbackup
|
sudo mv dbbackup-linux-amd64 /usr/local/bin/dbbackup
|
||||||
```
|
```
|
||||||
@@ -94,6 +96,7 @@ Database: postgres@localhost:5432 (PostgreSQL)
|
|||||||
────────────────────────────────
|
────────────────────────────────
|
||||||
Restore Single Database
|
Restore Single Database
|
||||||
Restore Cluster Backup
|
Restore Cluster Backup
|
||||||
|
Diagnose Backup File
|
||||||
List & Manage Backups
|
List & Manage Backups
|
||||||
────────────────────────────────
|
────────────────────────────────
|
||||||
View Active Operations
|
View Active Operations
|
||||||
@@ -140,7 +143,7 @@ Backup Execution
|
|||||||
|
|
||||||
Backup created: cluster_20251128_092928.tar.gz
|
Backup created: cluster_20251128_092928.tar.gz
|
||||||
Size: 22.5 GB (compressed)
|
Size: 22.5 GB (compressed)
|
||||||
Location: /u01/dba/dumps/
|
Location: /var/backups/postgres/
|
||||||
Databases: 7
|
Databases: 7
|
||||||
Checksum: SHA-256 verified
|
Checksum: SHA-256 verified
|
||||||
```
|
```
|
||||||
@@ -161,11 +164,15 @@ Cluster Restore Options
|
|||||||
|
|
||||||
Safety Checks
|
Safety Checks
|
||||||
[OK] Archive integrity verified
|
[OK] Archive integrity verified
|
||||||
|
[OK] Dump validity verified
|
||||||
[OK] Disk space: 140 GB available
|
[OK] Disk space: 140 GB available
|
||||||
[OK] Required tools found
|
[OK] Required tools found
|
||||||
[OK] Target database accessible
|
[OK] Target database accessible
|
||||||
|
|
||||||
c: Toggle cleanup | Enter: Proceed | Esc: Cancel
|
Advanced Options
|
||||||
|
✗ Debug Log: false (press 'd' to toggle)
|
||||||
|
|
||||||
|
c: Toggle cleanup | d: Debug log | Enter: Proceed | Esc: Cancel
|
||||||
```
|
```
|
||||||
|
|
||||||
**Backup Manager:**
|
**Backup Manager:**
|
||||||
@@ -180,7 +187,7 @@ FILENAME FORMAT SIZE MODIFIED
|
|||||||
[OK] myapp_prod_20250114.dump.gz PostgreSQL Custom 12.3 GB 2025-01-14
|
[OK] myapp_prod_20250114.dump.gz PostgreSQL Custom 12.3 GB 2025-01-14
|
||||||
[!!] users_db_20241220.dump.gz PostgreSQL Custom 850 MB 2024-12-20
|
[!!] users_db_20241220.dump.gz PostgreSQL Custom 850 MB 2024-12-20
|
||||||
|
|
||||||
r: Restore | v: Verify | i: Info | d: Delete | R: Refresh | Esc: Back
|
r: Restore | v: Verify | i: Info | d: Diagnose | D: Delete | R: Refresh | Esc: Back
|
||||||
```
|
```
|
||||||
|
|
||||||
**Configuration Settings:**
|
**Configuration Settings:**
|
||||||
@@ -190,6 +197,7 @@ Configuration Settings
|
|||||||
> Database Type: postgres
|
> Database Type: postgres
|
||||||
CPU Workload Type: balanced
|
CPU Workload Type: balanced
|
||||||
Backup Directory: /root/db_backups
|
Backup Directory: /root/db_backups
|
||||||
|
Work Directory: /tmp
|
||||||
Compression Level: 6
|
Compression Level: 6
|
||||||
Parallel Jobs: 16
|
Parallel Jobs: 16
|
||||||
Dump Jobs: 8
|
Dump Jobs: 8
|
||||||
@@ -240,6 +248,12 @@ dbbackup restore single backup.dump --target myapp_db --create --confirm
|
|||||||
# Restore cluster
|
# Restore cluster
|
||||||
dbbackup restore cluster cluster_backup.tar.gz --confirm
|
dbbackup restore cluster cluster_backup.tar.gz --confirm
|
||||||
|
|
||||||
|
# Restore with debug logging (saves detailed error report on failure)
|
||||||
|
dbbackup restore cluster backup.tar.gz --save-debug-log /tmp/restore-debug.json --confirm
|
||||||
|
|
||||||
|
# Diagnose backup before restore
|
||||||
|
dbbackup restore diagnose backup.dump.gz --deep
|
||||||
|
|
||||||
# Cloud backup
|
# Cloud backup
|
||||||
dbbackup backup single mydb --cloud s3://my-bucket/backups/
|
dbbackup backup single mydb --cloud s3://my-bucket/backups/
|
||||||
|
|
||||||
@@ -257,6 +271,7 @@ dbbackup backup single mydb --dry-run
|
|||||||
| `restore single` | Restore single database |
|
| `restore single` | Restore single database |
|
||||||
| `restore cluster` | Restore full cluster |
|
| `restore cluster` | Restore full cluster |
|
||||||
| `restore pitr` | Point-in-Time Recovery |
|
| `restore pitr` | Point-in-Time Recovery |
|
||||||
|
| `restore diagnose` | Diagnose backup file integrity |
|
||||||
| `verify-backup` | Verify backup integrity |
|
| `verify-backup` | Verify backup integrity |
|
||||||
| `cleanup` | Remove old backups |
|
| `cleanup` | Remove old backups |
|
||||||
| `status` | Check connection status |
|
| `status` | Check connection status |
|
||||||
@@ -271,6 +286,10 @@ dbbackup backup single mydb --dry-run
|
|||||||
| `drill` | DR drill testing |
|
| `drill` | DR drill testing |
|
||||||
| `report` | Compliance report generation |
|
| `report` | Compliance report generation |
|
||||||
| `rto` | RTO/RPO analysis |
|
| `rto` | RTO/RPO analysis |
|
||||||
|
| `install` | Install as systemd service |
|
||||||
|
| `uninstall` | Remove systemd service |
|
||||||
|
| `metrics export` | Export Prometheus metrics to textfile |
|
||||||
|
| `metrics serve` | Run Prometheus HTTP exporter |
|
||||||
|
|
||||||
## Global Flags
|
## Global Flags
|
||||||
|
|
||||||
@@ -287,8 +306,8 @@ dbbackup backup single mydb --dry-run
|
|||||||
| `--cloud` | Cloud storage URI | - |
|
| `--cloud` | Cloud storage URI | - |
|
||||||
| `--encrypt` | Enable encryption | false |
|
| `--encrypt` | Enable encryption | false |
|
||||||
| `--dry-run, -n` | Run preflight checks only | false |
|
| `--dry-run, -n` | Run preflight checks only | false |
|
||||||
| `--notify` | Enable notifications | false |
|
|
||||||
| `--debug` | Enable debug logging | false |
|
| `--debug` | Enable debug logging | false |
|
||||||
|
| `--save-debug-log` | Save error report to file on failure | - |
|
||||||
|
|
||||||
## Encryption
|
## Encryption
|
||||||
|
|
||||||
@@ -436,9 +455,64 @@ dbbackup backup cluster -n # Short flag
|
|||||||
Ready to backup. Remove --dry-run to execute.
|
Ready to backup. Remove --dry-run to execute.
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Backup Diagnosis
|
||||||
|
|
||||||
|
Diagnose backup files before restore to detect corruption or truncation:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Diagnose a backup file
|
||||||
|
dbbackup restore diagnose backup.dump.gz
|
||||||
|
|
||||||
|
# Deep analysis (line-by-line COPY block verification)
|
||||||
|
dbbackup restore diagnose backup.dump.gz --deep
|
||||||
|
|
||||||
|
# JSON output for automation
|
||||||
|
dbbackup restore diagnose backup.dump.gz --json
|
||||||
|
|
||||||
|
# Diagnose cluster archive (checks all contained dumps)
|
||||||
|
dbbackup restore diagnose cluster_backup.tar.gz --deep
|
||||||
|
```
|
||||||
|
|
||||||
|
**Checks performed:**
|
||||||
|
- PGDMP signature validation (PostgreSQL custom format)
|
||||||
|
- Gzip integrity verification
|
||||||
|
- COPY block termination (detects truncated dumps)
|
||||||
|
- `pg_restore --list` validation
|
||||||
|
- Archive structure analysis
|
||||||
|
|
||||||
|
**Example output:**
|
||||||
|
```
|
||||||
|
🔍 Backup Diagnosis Report
|
||||||
|
══════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
📁 File: mydb_20260105.dump.gz
|
||||||
|
Format: PostgreSQL Custom (gzip)
|
||||||
|
Size: 2.5 GB
|
||||||
|
|
||||||
|
🔬 Analysis Results:
|
||||||
|
✅ Gzip integrity: Valid
|
||||||
|
✅ PGDMP signature: Valid
|
||||||
|
✅ pg_restore --list: Success (245 objects)
|
||||||
|
❌ COPY block check: TRUNCATED
|
||||||
|
|
||||||
|
⚠️ Issues Found:
|
||||||
|
- COPY block for table 'orders' not terminated
|
||||||
|
- Dump appears truncated at line 1,234,567
|
||||||
|
|
||||||
|
💡 Recommendations:
|
||||||
|
- Re-run the backup for this database
|
||||||
|
- Check disk space on backup server
|
||||||
|
- Verify network stability during backup
|
||||||
|
```
|
||||||
|
|
||||||
|
**In Interactive Mode:**
|
||||||
|
- Press `d` in archive browser to diagnose any backup
|
||||||
|
- Automatic dump validity check in restore preview
|
||||||
|
- Toggle debug logging with `d` in restore options
|
||||||
|
|
||||||
## Notifications
|
## Notifications
|
||||||
|
|
||||||
Get alerted on backup events via email or webhooks.
|
Get alerted on backup events via email or webhooks. Configure via environment variables.
|
||||||
|
|
||||||
### SMTP Email
|
### SMTP Email
|
||||||
|
|
||||||
@@ -451,8 +525,8 @@ export NOTIFY_SMTP_PASSWORD="secret"
|
|||||||
export NOTIFY_SMTP_FROM="dbbackup@example.com"
|
export NOTIFY_SMTP_FROM="dbbackup@example.com"
|
||||||
export NOTIFY_SMTP_TO="admin@example.com,dba@example.com"
|
export NOTIFY_SMTP_TO="admin@example.com,dba@example.com"
|
||||||
|
|
||||||
# Enable notifications
|
# Run backup (notifications triggered when SMTP is configured)
|
||||||
dbbackup backup single mydb --notify
|
dbbackup backup single mydb
|
||||||
```
|
```
|
||||||
|
|
||||||
### Webhooks
|
### Webhooks
|
||||||
@@ -465,7 +539,8 @@ export NOTIFY_WEBHOOK_SECRET="signing-secret" # Optional HMAC signing
|
|||||||
# Slack webhook
|
# Slack webhook
|
||||||
export NOTIFY_WEBHOOK_URL="https://hooks.slack.com/services/T00/B00/XXX"
|
export NOTIFY_WEBHOOK_URL="https://hooks.slack.com/services/T00/B00/XXX"
|
||||||
|
|
||||||
dbbackup backup single mydb --notify
|
# Run backup (notifications triggered when webhook is configured)
|
||||||
|
dbbackup backup single mydb
|
||||||
```
|
```
|
||||||
|
|
||||||
**Webhook payload:**
|
**Webhook payload:**
|
||||||
@@ -604,6 +679,102 @@ dbbackup rto analyze mydb --target-rto 4h --target-rpo 1h
|
|||||||
- Compliance status
|
- Compliance status
|
||||||
- Recommendations for improvement
|
- Recommendations for improvement
|
||||||
|
|
||||||
|
## Systemd Integration
|
||||||
|
|
||||||
|
Install dbbackup as a systemd service for automated scheduled backups:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Install with Prometheus metrics exporter
|
||||||
|
sudo dbbackup install --backup-type cluster --with-metrics
|
||||||
|
|
||||||
|
# Preview what would be installed
|
||||||
|
dbbackup install --dry-run --backup-type cluster
|
||||||
|
|
||||||
|
# Check installation status
|
||||||
|
dbbackup install --status
|
||||||
|
|
||||||
|
# Uninstall
|
||||||
|
sudo dbbackup uninstall cluster --purge
|
||||||
|
```
|
||||||
|
|
||||||
|
**Schedule options:**
|
||||||
|
```bash
|
||||||
|
--schedule daily # Every day at midnight (default)
|
||||||
|
--schedule weekly # Every Monday at midnight
|
||||||
|
--schedule "*-*-* 02:00:00" # Every day at 2am
|
||||||
|
--schedule "Mon *-*-* 03:00" # Every Monday at 3am
|
||||||
|
```
|
||||||
|
|
||||||
|
**What gets installed:**
|
||||||
|
- Systemd service and timer units
|
||||||
|
- Dedicated `dbbackup` user with security hardening
|
||||||
|
- Directories: `/var/lib/dbbackup/`, `/etc/dbbackup/`
|
||||||
|
- Optional: Prometheus HTTP exporter on port 9399
|
||||||
|
|
||||||
|
📖 **Full documentation:** [SYSTEMD.md](SYSTEMD.md) - Manual setup, security hardening, multiple instances, troubleshooting
|
||||||
|
|
||||||
|
## Prometheus Metrics
|
||||||
|
|
||||||
|
Export backup metrics for monitoring with Prometheus:
|
||||||
|
|
||||||
|
### Textfile Collector
|
||||||
|
|
||||||
|
For integration with node_exporter:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Export metrics to textfile
|
||||||
|
dbbackup metrics export --output /var/lib/node_exporter/textfile_collector/dbbackup.prom
|
||||||
|
|
||||||
|
# Export for specific instance
|
||||||
|
dbbackup metrics export --instance production --output /var/lib/dbbackup/metrics/production.prom
|
||||||
|
```
|
||||||
|
|
||||||
|
Configure node_exporter:
|
||||||
|
```bash
|
||||||
|
node_exporter --collector.textfile.directory=/var/lib/node_exporter/textfile_collector/
|
||||||
|
```
|
||||||
|
|
||||||
|
### HTTP Exporter
|
||||||
|
|
||||||
|
Run a dedicated metrics HTTP server:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Start metrics server on default port 9399
|
||||||
|
dbbackup metrics serve
|
||||||
|
|
||||||
|
# Custom port
|
||||||
|
dbbackup metrics serve --port 9100
|
||||||
|
|
||||||
|
# Run as systemd service (installed via --with-metrics)
|
||||||
|
sudo systemctl start dbbackup-exporter
|
||||||
|
```
|
||||||
|
|
||||||
|
**Endpoints:**
|
||||||
|
- `/metrics` - Prometheus exposition format
|
||||||
|
- `/health` - Health check (returns 200 OK)
|
||||||
|
|
||||||
|
**Available metrics:**
|
||||||
|
| Metric | Type | Description |
|
||||||
|
|--------|------|-------------|
|
||||||
|
| `dbbackup_last_success_timestamp` | gauge | Unix timestamp of last successful backup |
|
||||||
|
| `dbbackup_last_backup_duration_seconds` | gauge | Duration of last backup |
|
||||||
|
| `dbbackup_last_backup_size_bytes` | gauge | Size of last backup |
|
||||||
|
| `dbbackup_backup_total` | counter | Total backups by status (success/failure) |
|
||||||
|
| `dbbackup_rpo_seconds` | gauge | Seconds since last successful backup |
|
||||||
|
| `dbbackup_backup_verified` | gauge | Whether last backup was verified (1/0) |
|
||||||
|
| `dbbackup_scrape_timestamp` | gauge | When metrics were collected |
|
||||||
|
|
||||||
|
**Labels:** `instance`, `database`, `engine`
|
||||||
|
|
||||||
|
**Example Prometheus query:**
|
||||||
|
```promql
|
||||||
|
# Alert if RPO exceeds 24 hours
|
||||||
|
dbbackup_rpo_seconds{instance="production"} > 86400
|
||||||
|
|
||||||
|
# Backup success rate
|
||||||
|
sum(rate(dbbackup_backup_total{status="success"}[24h])) / sum(rate(dbbackup_backup_total[24h]))
|
||||||
|
```
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
### PostgreSQL Authentication
|
### PostgreSQL Authentication
|
||||||
@@ -687,6 +858,7 @@ Workload types:
|
|||||||
|
|
||||||
## Documentation
|
## Documentation
|
||||||
|
|
||||||
|
- [SYSTEMD.md](SYSTEMD.md) - Systemd installation & scheduling
|
||||||
- [DOCKER.md](DOCKER.md) - Docker deployment
|
- [DOCKER.md](DOCKER.md) - Docker deployment
|
||||||
- [CLOUD.md](CLOUD.md) - Cloud storage configuration
|
- [CLOUD.md](CLOUD.md) - Cloud storage configuration
|
||||||
- [PITR.md](PITR.md) - Point-in-Time Recovery
|
- [PITR.md](PITR.md) - Point-in-Time Recovery
|
||||||
|
|||||||
108
RELEASE_NOTES.md
Normal file
108
RELEASE_NOTES.md
Normal file
@@ -0,0 +1,108 @@
|
|||||||
|
# v3.42.1 Release Notes
|
||||||
|
|
||||||
|
## What's New in v3.42.1
|
||||||
|
|
||||||
|
### Deduplication - Resistance is Futile
|
||||||
|
|
||||||
|
Content-defined chunking deduplication for space-efficient backups. Like restic/borgbackup but with **native database dump support**.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# First backup: 5MB stored
|
||||||
|
dbbackup dedup backup mydb.dump
|
||||||
|
|
||||||
|
# Second backup (modified): only 1.6KB new data stored!
|
||||||
|
# 100% deduplication ratio
|
||||||
|
dbbackup dedup backup mydb_modified.dump
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Features
|
||||||
|
- **Gear Hash CDC** - Content-defined chunking with 92%+ overlap on shifted data
|
||||||
|
- **SHA-256 Content-Addressed** - Chunks stored by hash, automatic deduplication
|
||||||
|
- **AES-256-GCM Encryption** - Optional per-chunk encryption
|
||||||
|
- **Gzip Compression** - Optional compression (enabled by default)
|
||||||
|
- **SQLite Index** - Fast chunk lookups and statistics
|
||||||
|
|
||||||
|
#### Commands
|
||||||
|
```bash
|
||||||
|
dbbackup dedup backup <file> # Create deduplicated backup
|
||||||
|
dbbackup dedup backup <file> --encrypt # With AES-256-GCM encryption
|
||||||
|
dbbackup dedup restore <id> <output> # Restore from manifest
|
||||||
|
dbbackup dedup list # List all backups
|
||||||
|
dbbackup dedup stats # Show deduplication statistics
|
||||||
|
dbbackup dedup delete <id> # Delete a backup
|
||||||
|
dbbackup dedup gc # Garbage collect unreferenced chunks
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Storage Structure
|
||||||
|
```
|
||||||
|
<backup-dir>/dedup/
|
||||||
|
chunks/ # Content-addressed chunk files
|
||||||
|
ab/cdef1234... # Sharded by first 2 chars of hash
|
||||||
|
manifests/ # JSON manifest per backup
|
||||||
|
chunks.db # SQLite index
|
||||||
|
```
|
||||||
|
|
||||||
|
### Also Included (from v3.41.x)
|
||||||
|
- **Systemd Integration** - One-command install with `dbbackup install`
|
||||||
|
- **Prometheus Metrics** - HTTP exporter on port 9399
|
||||||
|
- **Backup Catalog** - SQLite-based tracking of all backup operations
|
||||||
|
- **Prometheus Alerting Rules** - Added to SYSTEMD.md documentation
|
||||||
|
|
||||||
|
### Installation
|
||||||
|
|
||||||
|
#### Quick Install (Recommended)
|
||||||
|
```bash
|
||||||
|
# Download for your platform
|
||||||
|
curl -LO https://git.uuxo.net/UUXO/dbbackup/releases/download/v3.42.1/dbbackup-linux-amd64
|
||||||
|
|
||||||
|
# Install with systemd service
|
||||||
|
chmod +x dbbackup-linux-amd64
|
||||||
|
sudo ./dbbackup-linux-amd64 install --config /path/to/config.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Available Binaries
|
||||||
|
| Platform | Architecture | Binary |
|
||||||
|
|----------|--------------|--------|
|
||||||
|
| Linux | amd64 | `dbbackup-linux-amd64` |
|
||||||
|
| Linux | arm64 | `dbbackup-linux-arm64` |
|
||||||
|
| macOS | Intel | `dbbackup-darwin-amd64` |
|
||||||
|
| macOS | Apple Silicon | `dbbackup-darwin-arm64` |
|
||||||
|
| FreeBSD | amd64 | `dbbackup-freebsd-amd64` |
|
||||||
|
|
||||||
|
### Systemd Commands
|
||||||
|
```bash
|
||||||
|
dbbackup install --config config.yaml # Install service + timer
|
||||||
|
dbbackup install --status # Check service status
|
||||||
|
dbbackup install --uninstall # Remove services
|
||||||
|
```
|
||||||
|
|
||||||
|
### Prometheus Metrics
|
||||||
|
Available at `http://localhost:9399/metrics`:
|
||||||
|
|
||||||
|
| Metric | Description |
|
||||||
|
|--------|-------------|
|
||||||
|
| `dbbackup_last_backup_timestamp` | Unix timestamp of last backup |
|
||||||
|
| `dbbackup_last_backup_success` | 1 if successful, 0 if failed |
|
||||||
|
| `dbbackup_last_backup_duration_seconds` | Duration of last backup |
|
||||||
|
| `dbbackup_last_backup_size_bytes` | Size of last backup |
|
||||||
|
| `dbbackup_backup_total` | Total number of backups |
|
||||||
|
| `dbbackup_backup_errors_total` | Total number of failed backups |
|
||||||
|
|
||||||
|
### Security Features
|
||||||
|
- Hardened systemd service with `ProtectSystem=strict`
|
||||||
|
- `NoNewPrivileges=true` prevents privilege escalation
|
||||||
|
- Dedicated `dbbackup` system user (optional)
|
||||||
|
- Credential files with restricted permissions
|
||||||
|
|
||||||
|
### Documentation
|
||||||
|
- [SYSTEMD.md](SYSTEMD.md) - Complete systemd installation guide
|
||||||
|
- [README.md](README.md) - Full documentation
|
||||||
|
- [CHANGELOG.md](CHANGELOG.md) - Version history
|
||||||
|
|
||||||
|
### Bug Fixes
|
||||||
|
- Fixed SQLite time parsing in dedup stats
|
||||||
|
- Fixed function name collision in cmd package
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**Full Changelog**: https://git.uuxo.net/UUXO/dbbackup/compare/v3.41.1...v3.42.1
|
||||||
@@ -1,396 +0,0 @@
|
|||||||
# dbbackup v3.1.0 - Enterprise Backup Solution
|
|
||||||
|
|
||||||
**Released:** November 26, 2025
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🎉 Major Features
|
|
||||||
|
|
||||||
### Point-in-Time Recovery (PITR)
|
|
||||||
Complete PostgreSQL Point-in-Time Recovery implementation:
|
|
||||||
|
|
||||||
- **WAL Archiving**: Continuous archiving of Write-Ahead Log files
|
|
||||||
- **WAL Monitoring**: Real-time monitoring of archive status and statistics
|
|
||||||
- **Timeline Management**: Track and visualize PostgreSQL timeline branching
|
|
||||||
- **Recovery Targets**: Restore to any point in time:
|
|
||||||
- Specific timestamp (`--target-time "2024-11-26 12:00:00"`)
|
|
||||||
- Transaction ID (`--target-xid 1000000`)
|
|
||||||
- Log Sequence Number (`--target-lsn "0/3000000"`)
|
|
||||||
- Named restore point (`--target-name before_migration`)
|
|
||||||
- Earliest consistent point (`--target-immediate`)
|
|
||||||
- **Version Support**: Both PostgreSQL 12+ (modern) and legacy formats
|
|
||||||
- **Recovery Actions**: Promote to primary, pause for inspection, or shutdown
|
|
||||||
- **Comprehensive Testing**: 700+ lines of tests with 100% pass rate
|
|
||||||
|
|
||||||
**New Commands:**
|
|
||||||
- `pitr enable/disable/status` - PITR configuration management
|
|
||||||
- `wal archive/list/cleanup/timeline` - WAL archive operations
|
|
||||||
- `restore pitr` - Point-in-time recovery with multiple target types
|
|
||||||
|
|
||||||
### Cloud Storage Integration
|
|
||||||
Multi-cloud backend support with streaming efficiency:
|
|
||||||
|
|
||||||
- **Amazon S3 / MinIO**: Full S3-compatible storage support
|
|
||||||
- **Azure Blob Storage**: Native Azure integration
|
|
||||||
- **Google Cloud Storage**: GCS backend support
|
|
||||||
- **Streaming Operations**: Memory-efficient uploads/downloads
|
|
||||||
- **Cloud-Native**: Direct backup to cloud, no local disk required
|
|
||||||
|
|
||||||
**Features:**
|
|
||||||
- Automatic multipart uploads for large files
|
|
||||||
- Resumable downloads with retry logic
|
|
||||||
- Cloud-side encryption support
|
|
||||||
- Metadata preservation in cloud storage
|
|
||||||
|
|
||||||
### Incremental Backups
|
|
||||||
Space-efficient backup strategies:
|
|
||||||
|
|
||||||
- **PostgreSQL**: File-level incremental backups
|
|
||||||
- Track changed files since base backup
|
|
||||||
- Automatic base backup detection
|
|
||||||
- Efficient restore chain resolution
|
|
||||||
|
|
||||||
- **MySQL/MariaDB**: Binary log incremental backups
|
|
||||||
- Capture changes via binlog
|
|
||||||
- Automatic log rotation handling
|
|
||||||
- Point-in-time restore capability
|
|
||||||
|
|
||||||
**Benefits:**
|
|
||||||
- 70-90% reduction in backup size
|
|
||||||
- Faster backup completion times
|
|
||||||
- Automated backup chain management
|
|
||||||
- Intelligent dependency tracking
|
|
||||||
|
|
||||||
### AES-256-GCM Encryption
|
|
||||||
Military-grade encryption for data protection:
|
|
||||||
|
|
||||||
- **Algorithm**: AES-256-GCM authenticated encryption
|
|
||||||
- **Key Derivation**: PBKDF2-SHA256 with 600,000 iterations (OWASP 2023)
|
|
||||||
- **Streaming**: Memory-efficient for large backups
|
|
||||||
- **Key Sources**: File (raw/base64), environment variable, or passphrase
|
|
||||||
- **Auto-Detection**: Restore automatically detects encrypted backups
|
|
||||||
- **Tamper Protection**: Authenticated encryption prevents tampering
|
|
||||||
|
|
||||||
**Security:**
|
|
||||||
- Unique nonce per encryption (no key reuse)
|
|
||||||
- Cryptographically secure random generation
|
|
||||||
- 56-byte header with algorithm metadata
|
|
||||||
- ~1-2 GB/s encryption throughput
|
|
||||||
|
|
||||||
### Foundation Features
|
|
||||||
Production-ready backup operations:
|
|
||||||
|
|
||||||
- **SHA-256 Verification**: Cryptographic backup integrity checking
|
|
||||||
- **Intelligent Retention**: Day-based policies with minimum backup guarantees
|
|
||||||
- **Safe Cleanup**: Dry-run mode, safety checks, detailed reporting
|
|
||||||
- **Multi-Database**: PostgreSQL, MySQL, MariaDB support
|
|
||||||
- **Interactive TUI**: Beautiful terminal UI with progress tracking
|
|
||||||
- **CLI Mode**: Full command-line interface for automation
|
|
||||||
- **Cross-Platform**: Linux, macOS, FreeBSD, OpenBSD, NetBSD
|
|
||||||
- **Docker Support**: Official container images
|
|
||||||
- **100% Test Coverage**: Comprehensive test suite
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ✅ Production Validated
|
|
||||||
|
|
||||||
**Real-World Deployment:**
|
|
||||||
- ✅ 2 production hosts in production environment
|
|
||||||
- ✅ 8 databases backed up nightly
|
|
||||||
- ✅ 30-day retention with minimum 5 backups
|
|
||||||
- ✅ ~10MB/night backup volume
|
|
||||||
- ✅ Scheduled at 02:09 and 02:25 CET
|
|
||||||
- ✅ **Resolved 4-day backup failure immediately**
|
|
||||||
|
|
||||||
**User Feedback (Ansible Claude):**
|
|
||||||
> "cleanup command is SO gut, dass es alle verwenden sollten"
|
|
||||||
|
|
||||||
> "--dry-run feature: chef's kiss!" 💋
|
|
||||||
|
|
||||||
> "Modern tooling in place, pragmatic and maintainable"
|
|
||||||
|
|
||||||
> "CLI design: Professional & polished"
|
|
||||||
|
|
||||||
**Impact:**
|
|
||||||
- Fixed failing backup infrastructure on first deployment
|
|
||||||
- Stable operation in production environment
|
|
||||||
- Positive feedback from DevOps team
|
|
||||||
- Validation of feature set and UX design
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📦 Installation
|
|
||||||
|
|
||||||
### Download Pre-compiled Binary
|
|
||||||
|
|
||||||
**Linux (x86_64):**
|
|
||||||
```bash
|
|
||||||
wget https://git.uuxo.net/PlusOne/dbbackup/releases/download/v3.1.0/dbbackup-linux-amd64
|
|
||||||
chmod +x dbbackup-linux-amd64
|
|
||||||
sudo mv dbbackup-linux-amd64 /usr/local/bin/dbbackup
|
|
||||||
```
|
|
||||||
|
|
||||||
**Linux (ARM64):**
|
|
||||||
```bash
|
|
||||||
wget https://git.uuxo.net/PlusOne/dbbackup/releases/download/v3.1.0/dbbackup-linux-arm64
|
|
||||||
chmod +x dbbackup-linux-arm64
|
|
||||||
sudo mv dbbackup-linux-arm64 /usr/local/bin/dbbackup
|
|
||||||
```
|
|
||||||
|
|
||||||
**macOS (Intel):**
|
|
||||||
```bash
|
|
||||||
wget https://git.uuxo.net/PlusOne/dbbackup/releases/download/v3.1.0/dbbackup-darwin-amd64
|
|
||||||
chmod +x dbbackup-darwin-amd64
|
|
||||||
sudo mv dbbackup-darwin-amd64 /usr/local/bin/dbbackup
|
|
||||||
```
|
|
||||||
|
|
||||||
**macOS (Apple Silicon):**
|
|
||||||
```bash
|
|
||||||
wget https://git.uuxo.net/PlusOne/dbbackup/releases/download/v3.1.0/dbbackup-darwin-arm64
|
|
||||||
chmod +x dbbackup-darwin-arm64
|
|
||||||
sudo mv dbbackup-darwin-arm64 /usr/local/bin/dbbackup
|
|
||||||
```
|
|
||||||
|
|
||||||
### Build from Source
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git clone https://git.uuxo.net/PlusOne/dbbackup.git
|
|
||||||
cd dbbackup
|
|
||||||
go build -o dbbackup
|
|
||||||
sudo mv dbbackup /usr/local/bin/
|
|
||||||
```
|
|
||||||
|
|
||||||
### Docker
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker pull git.uuxo.net/PlusOne/dbbackup:v3.1.0
|
|
||||||
docker pull git.uuxo.net/PlusOne/dbbackup:latest
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🚀 Quick Start Examples
|
|
||||||
|
|
||||||
### Basic Backup
|
|
||||||
```bash
|
|
||||||
# Simple database backup
|
|
||||||
dbbackup backup single mydb
|
|
||||||
|
|
||||||
# Backup with verification
|
|
||||||
dbbackup backup single mydb
|
|
||||||
dbbackup verify mydb_backup.sql.gz
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cloud Backup
|
|
||||||
```bash
|
|
||||||
# Backup to S3
|
|
||||||
dbbackup backup single mydb --cloud s3://my-bucket/backups/
|
|
||||||
|
|
||||||
# Backup to Azure
|
|
||||||
dbbackup backup single mydb --cloud azure://container/backups/
|
|
||||||
|
|
||||||
# Backup to GCS
|
|
||||||
dbbackup backup single mydb --cloud gs://my-bucket/backups/
|
|
||||||
```
|
|
||||||
|
|
||||||
### Encrypted Backup
|
|
||||||
```bash
|
|
||||||
# Generate encryption key
|
|
||||||
head -c 32 /dev/urandom | base64 > encryption.key
|
|
||||||
|
|
||||||
# Encrypted backup
|
|
||||||
dbbackup backup single mydb --encrypt --encryption-key-file encryption.key
|
|
||||||
|
|
||||||
# Restore (automatic decryption)
|
|
||||||
dbbackup restore single mydb_backup.sql.gz --encryption-key-file encryption.key
|
|
||||||
```
|
|
||||||
|
|
||||||
### Incremental Backup
|
|
||||||
```bash
|
|
||||||
# Create base backup
|
|
||||||
dbbackup backup single mydb --backup-type full
|
|
||||||
|
|
||||||
# Create incremental backup
|
|
||||||
dbbackup backup single mydb --backup-type incremental \
|
|
||||||
--base-backup mydb_base_20241126_120000.tar.gz
|
|
||||||
|
|
||||||
# Restore (automatic chain resolution)
|
|
||||||
dbbackup restore single mydb_incr_20241126_150000.tar.gz
|
|
||||||
```
|
|
||||||
|
|
||||||
### Point-in-Time Recovery
|
|
||||||
```bash
|
|
||||||
# Enable PITR
|
|
||||||
dbbackup pitr enable --archive-dir /backups/wal_archive
|
|
||||||
|
|
||||||
# Take base backup
|
|
||||||
pg_basebackup -D /backups/base.tar.gz -Ft -z -P
|
|
||||||
|
|
||||||
# Perform PITR
|
|
||||||
dbbackup restore pitr \
|
|
||||||
--base-backup /backups/base.tar.gz \
|
|
||||||
--wal-archive /backups/wal_archive \
|
|
||||||
--target-time "2024-11-26 12:00:00" \
|
|
||||||
--target-dir /var/lib/postgresql/14/restored
|
|
||||||
|
|
||||||
# Monitor WAL archiving
|
|
||||||
dbbackup pitr status
|
|
||||||
dbbackup wal list
|
|
||||||
```
|
|
||||||
|
|
||||||
### Retention & Cleanup
|
|
||||||
```bash
|
|
||||||
# Cleanup old backups (dry-run first!)
|
|
||||||
dbbackup cleanup --retention-days 30 --min-backups 5 --dry-run
|
|
||||||
|
|
||||||
# Actually cleanup
|
|
||||||
dbbackup cleanup --retention-days 30 --min-backups 5
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cluster Operations
|
|
||||||
```bash
|
|
||||||
# Backup entire cluster
|
|
||||||
dbbackup backup cluster
|
|
||||||
|
|
||||||
# Restore entire cluster
|
|
||||||
dbbackup restore cluster --backups /path/to/backups/ --confirm
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🔮 What's Next (v3.2)
|
|
||||||
|
|
||||||
Based on production feedback from Ansible Claude:
|
|
||||||
|
|
||||||
### High Priority
|
|
||||||
1. **Config File Support** (2-3h)
|
|
||||||
- Persist flags like `--allow-root` in `.dbbackup.conf`
|
|
||||||
- Per-directory configuration management
|
|
||||||
- Better automation support
|
|
||||||
|
|
||||||
2. **Socket Auth Auto-Detection** (1-2h)
|
|
||||||
- Auto-detect Unix socket authentication
|
|
||||||
- Skip password prompts for socket connections
|
|
||||||
- Improved UX for root users
|
|
||||||
|
|
||||||
### Medium Priority
|
|
||||||
3. **Inline Backup Verification** (2-3h)
|
|
||||||
- Automatic verification after backup
|
|
||||||
- Immediate corruption detection
|
|
||||||
- Better workflow integration
|
|
||||||
|
|
||||||
4. **Progress Indicators** (4-6h)
|
|
||||||
- Progress bars for mysqldump operations
|
|
||||||
- Real-time backup size tracking
|
|
||||||
- ETA for large backups
|
|
||||||
|
|
||||||
### Additional Features
|
|
||||||
5. **Ansible Module** (4-6h)
|
|
||||||
- Native Ansible integration
|
|
||||||
- Declarative backup configuration
|
|
||||||
- DevOps automation support
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📊 Performance Metrics
|
|
||||||
|
|
||||||
**Backup Performance:**
|
|
||||||
- PostgreSQL: 50-150 MB/s (network dependent)
|
|
||||||
- MySQL: 30-100 MB/s (with compression)
|
|
||||||
- Encryption: ~1-2 GB/s (streaming)
|
|
||||||
- Compression: 70-80% size reduction (typical)
|
|
||||||
|
|
||||||
**PITR Performance:**
|
|
||||||
- WAL archiving: 100-200 MB/s
|
|
||||||
- WAL encryption: ~1-2 GB/s
|
|
||||||
- Recovery replay: 10-100 MB/s (disk I/O dependent)
|
|
||||||
|
|
||||||
**Resource Usage:**
|
|
||||||
- Memory: ~1GB constant (streaming architecture)
|
|
||||||
- CPU: 1-4 cores (configurable)
|
|
||||||
- Disk I/O: Streaming (no intermediate files)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🏗️ Architecture Highlights
|
|
||||||
|
|
||||||
**Split-Brain Development:**
|
|
||||||
- Human architects system design
|
|
||||||
- AI implements features and tests
|
|
||||||
- Micro-task decomposition (1-2h phases)
|
|
||||||
- Progressive enhancement approach
|
|
||||||
- **Result:** 52% faster development (5.75h vs 12h planned)
|
|
||||||
|
|
||||||
**Key Innovations:**
|
|
||||||
- Streaming architecture for constant memory usage
|
|
||||||
- Interface-first design for clean modularity
|
|
||||||
- Comprehensive test coverage (700+ test lines)
|
|
||||||
- Production validation in parallel with development
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📄 Documentation
|
|
||||||
|
|
||||||
**Core Documentation:**
|
|
||||||
- [README.md](README.md) - Complete feature overview and setup
|
|
||||||
- [PITR.md](PITR.md) - Comprehensive PITR guide
|
|
||||||
- [DOCKER.md](DOCKER.md) - Docker usage and deployment
|
|
||||||
- [CHANGELOG.md](CHANGELOG.md) - Detailed version history
|
|
||||||
|
|
||||||
**Getting Started:**
|
|
||||||
- [QUICKRUN.md](QUICKRUN.MD) - Quick start guide
|
|
||||||
- [PROGRESS_IMPLEMENTATION.md](PROGRESS_IMPLEMENTATION.md) - Progress tracking
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📜 License
|
|
||||||
|
|
||||||
Apache License 2.0
|
|
||||||
|
|
||||||
Copyright 2025 dbbackup Project
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0. See [LICENSE](LICENSE) for details.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🙏 Credits
|
|
||||||
|
|
||||||
**Development:**
|
|
||||||
- Built using Multi-Claude collaboration architecture
|
|
||||||
- Split-brain development pattern (human architecture + AI implementation)
|
|
||||||
- 5.75 hours intensive development (52% time savings)
|
|
||||||
|
|
||||||
**Production Validation:**
|
|
||||||
- Deployed in production environments
|
|
||||||
- Real-world testing and feedback
|
|
||||||
- DevOps validation and feature requests
|
|
||||||
|
|
||||||
**Technologies:**
|
|
||||||
- Go 1.21+
|
|
||||||
- PostgreSQL 9.5-17
|
|
||||||
- MySQL/MariaDB 5.7+
|
|
||||||
- AWS SDK, Azure SDK, Google Cloud SDK
|
|
||||||
- Cobra CLI framework
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🐛 Known Issues
|
|
||||||
|
|
||||||
None reported in production deployment.
|
|
||||||
|
|
||||||
If you encounter issues, please report them at:
|
|
||||||
https://git.uuxo.net/PlusOne/dbbackup/issues
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📞 Support
|
|
||||||
|
|
||||||
**Documentation:** See [README.md](README.md) and [PITR.md](PITR.md)
|
|
||||||
**Issues:** https://git.uuxo.net/PlusOne/dbbackup/issues
|
|
||||||
**Repository:** https://git.uuxo.net/PlusOne/dbbackup
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Thank you for using dbbackup!** 🎉
|
|
||||||
|
|
||||||
*Professional database backup and restore utility for PostgreSQL, MySQL, and MariaDB.*
|
|
||||||
@@ -85,7 +85,7 @@ We release security updates for the following versions:
|
|||||||
- ❌ Never store unencrypted backups on public storage
|
- ❌ Never store unencrypted backups on public storage
|
||||||
|
|
||||||
**Docker Usage:**
|
**Docker Usage:**
|
||||||
- ✅ Use specific version tags (`:v3.1.0` not `:latest`)
|
- ✅ Use specific version tags (`:v3.2.0` not `:latest`)
|
||||||
- ✅ Run as non-root user (default in our image)
|
- ✅ Run as non-root user (default in our image)
|
||||||
- ✅ Mount volumes read-only when possible
|
- ✅ Mount volumes read-only when possible
|
||||||
- ✅ Use Docker secrets for credentials
|
- ✅ Use Docker secrets for credentials
|
||||||
|
|||||||
616
SYSTEMD.md
Normal file
616
SYSTEMD.md
Normal file
@@ -0,0 +1,616 @@
|
|||||||
|
# Systemd Integration Guide
|
||||||
|
|
||||||
|
This guide covers installing dbbackup as a systemd service for automated scheduled backups.
|
||||||
|
|
||||||
|
## Quick Start (Installer)
|
||||||
|
|
||||||
|
The easiest way to set up systemd services is using the built-in installer:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Install as cluster backup service (daily at midnight)
|
||||||
|
sudo dbbackup install --backup-type cluster --schedule daily
|
||||||
|
|
||||||
|
# Check what would be installed (dry-run)
|
||||||
|
dbbackup install --dry-run --backup-type cluster
|
||||||
|
|
||||||
|
# Check installation status
|
||||||
|
dbbackup install --status
|
||||||
|
|
||||||
|
# Uninstall
|
||||||
|
sudo dbbackup uninstall cluster --purge
|
||||||
|
```
|
||||||
|
|
||||||
|
## Installer Options
|
||||||
|
|
||||||
|
| Flag | Description | Default |
|
||||||
|
|------|-------------|---------|
|
||||||
|
| `--instance NAME` | Instance name for named backups | - |
|
||||||
|
| `--backup-type TYPE` | Backup type: `cluster`, `single`, `sample` | `cluster` |
|
||||||
|
| `--schedule SPEC` | Timer schedule (see below) | `daily` |
|
||||||
|
| `--with-metrics` | Install Prometheus metrics exporter | false |
|
||||||
|
| `--metrics-port PORT` | HTTP port for metrics exporter | 9399 |
|
||||||
|
| `--dry-run` | Preview changes without applying | false |
|
||||||
|
|
||||||
|
### Schedule Format
|
||||||
|
|
||||||
|
The `--schedule` option accepts systemd OnCalendar format:
|
||||||
|
|
||||||
|
| Value | Description |
|
||||||
|
|-------|-------------|
|
||||||
|
| `daily` | Every day at midnight |
|
||||||
|
| `weekly` | Every Monday at midnight |
|
||||||
|
| `hourly` | Every hour |
|
||||||
|
| `*-*-* 02:00:00` | Every day at 2:00 AM |
|
||||||
|
| `*-*-* 00/6:00:00` | Every 6 hours |
|
||||||
|
| `Mon *-*-* 03:00` | Every Monday at 3:00 AM |
|
||||||
|
| `*-*-01 00:00:00` | First day of every month |
|
||||||
|
|
||||||
|
Test schedule with: `systemd-analyze calendar "Mon *-*-* 03:00"`
|
||||||
|
|
||||||
|
## What Gets Installed
|
||||||
|
|
||||||
|
### Directory Structure
|
||||||
|
|
||||||
|
```
|
||||||
|
/etc/dbbackup/
|
||||||
|
├── dbbackup.conf # Main configuration
|
||||||
|
└── env.d/
|
||||||
|
└── cluster.conf # Instance credentials (mode 0600)
|
||||||
|
|
||||||
|
/var/lib/dbbackup/
|
||||||
|
├── catalog/
|
||||||
|
│ └── backups.db # SQLite backup catalog
|
||||||
|
├── backups/ # Default backup storage
|
||||||
|
└── metrics/ # Prometheus textfile metrics
|
||||||
|
|
||||||
|
/var/log/dbbackup/ # Log files
|
||||||
|
|
||||||
|
/usr/local/bin/dbbackup # Binary copy
|
||||||
|
```
|
||||||
|
|
||||||
|
### Systemd Units
|
||||||
|
|
||||||
|
**For cluster backups:**
|
||||||
|
- `/etc/systemd/system/dbbackup-cluster.service` - Backup service
|
||||||
|
- `/etc/systemd/system/dbbackup-cluster.timer` - Backup scheduler
|
||||||
|
|
||||||
|
**For named instances:**
|
||||||
|
- `/etc/systemd/system/dbbackup@.service` - Template service
|
||||||
|
- `/etc/systemd/system/dbbackup@.timer` - Template timer
|
||||||
|
|
||||||
|
**Metrics exporter (optional):**
|
||||||
|
- `/etc/systemd/system/dbbackup-exporter.service`
|
||||||
|
|
||||||
|
### System User
|
||||||
|
|
||||||
|
A dedicated `dbbackup` user and group are created:
|
||||||
|
- Home: `/var/lib/dbbackup`
|
||||||
|
- Shell: `/usr/sbin/nologin`
|
||||||
|
- Purpose: Run backup services with minimal privileges
|
||||||
|
|
||||||
|
## Manual Installation
|
||||||
|
|
||||||
|
If you prefer to set up systemd services manually without the installer:
|
||||||
|
|
||||||
|
### Step 1: Create User and Directories
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Create system user
|
||||||
|
sudo useradd --system --home-dir /var/lib/dbbackup --shell /usr/sbin/nologin dbbackup
|
||||||
|
|
||||||
|
# Create directories
|
||||||
|
sudo mkdir -p /etc/dbbackup/env.d
|
||||||
|
sudo mkdir -p /var/lib/dbbackup/{catalog,backups,metrics}
|
||||||
|
sudo mkdir -p /var/log/dbbackup
|
||||||
|
|
||||||
|
# Set ownership
|
||||||
|
sudo chown -R dbbackup:dbbackup /var/lib/dbbackup /var/log/dbbackup
|
||||||
|
sudo chown root:dbbackup /etc/dbbackup
|
||||||
|
sudo chmod 750 /etc/dbbackup
|
||||||
|
|
||||||
|
# Copy binary
|
||||||
|
sudo cp dbbackup /usr/local/bin/
|
||||||
|
sudo chmod 755 /usr/local/bin/dbbackup
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 2: Create Configuration
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Main configuration
|
||||||
|
sudo tee /etc/dbbackup/dbbackup.conf << 'EOF'
|
||||||
|
# DBBackup Configuration
|
||||||
|
db-type=postgres
|
||||||
|
host=localhost
|
||||||
|
port=5432
|
||||||
|
user=postgres
|
||||||
|
backup-dir=/var/lib/dbbackup/backups
|
||||||
|
compression=6
|
||||||
|
retention-days=30
|
||||||
|
min-backups=7
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# Instance credentials (secure permissions)
|
||||||
|
sudo tee /etc/dbbackup/env.d/cluster.conf << 'EOF'
|
||||||
|
PGPASSWORD=your_secure_password
|
||||||
|
# Or for MySQL:
|
||||||
|
# MYSQL_PWD=your_secure_password
|
||||||
|
EOF
|
||||||
|
sudo chmod 600 /etc/dbbackup/env.d/cluster.conf
|
||||||
|
sudo chown dbbackup:dbbackup /etc/dbbackup/env.d/cluster.conf
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 3: Create Service Unit
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo tee /etc/systemd/system/dbbackup-cluster.service << 'EOF'
|
||||||
|
[Unit]
|
||||||
|
Description=DBBackup Cluster Backup
|
||||||
|
Documentation=https://github.com/PlusOne/dbbackup
|
||||||
|
After=network.target postgresql.service mysql.service
|
||||||
|
Wants=network.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
User=dbbackup
|
||||||
|
Group=dbbackup
|
||||||
|
|
||||||
|
# Load configuration
|
||||||
|
EnvironmentFile=-/etc/dbbackup/env.d/cluster.conf
|
||||||
|
|
||||||
|
# Working directory
|
||||||
|
WorkingDirectory=/var/lib/dbbackup
|
||||||
|
|
||||||
|
# Execute backup
|
||||||
|
ExecStart=/usr/local/bin/dbbackup backup cluster \
|
||||||
|
--config /etc/dbbackup/dbbackup.conf \
|
||||||
|
--backup-dir /var/lib/dbbackup/backups \
|
||||||
|
--allow-root
|
||||||
|
|
||||||
|
# Security hardening
|
||||||
|
NoNewPrivileges=yes
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=yes
|
||||||
|
PrivateTmp=yes
|
||||||
|
PrivateDevices=yes
|
||||||
|
ProtectKernelTunables=yes
|
||||||
|
ProtectKernelModules=yes
|
||||||
|
ProtectControlGroups=yes
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||||
|
RestrictNamespaces=yes
|
||||||
|
RestrictRealtime=yes
|
||||||
|
RestrictSUIDSGID=yes
|
||||||
|
MemoryDenyWriteExecute=yes
|
||||||
|
LockPersonality=yes
|
||||||
|
|
||||||
|
# Allow write to specific paths
|
||||||
|
ReadWritePaths=/var/lib/dbbackup /var/log/dbbackup
|
||||||
|
|
||||||
|
# Capability restrictions
|
||||||
|
CapabilityBoundingSet=CAP_DAC_READ_SEARCH CAP_NET_CONNECT
|
||||||
|
AmbientCapabilities=
|
||||||
|
|
||||||
|
# Resource limits
|
||||||
|
MemoryMax=4G
|
||||||
|
CPUQuota=80%
|
||||||
|
|
||||||
|
# Prevent OOM killer from terminating backups
|
||||||
|
OOMScoreAdjust=-100
|
||||||
|
|
||||||
|
# Logging
|
||||||
|
StandardOutput=journal
|
||||||
|
StandardError=journal
|
||||||
|
SyslogIdentifier=dbbackup
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
EOF
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 4: Create Timer Unit
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo tee /etc/systemd/system/dbbackup-cluster.timer << 'EOF'
|
||||||
|
[Unit]
|
||||||
|
Description=DBBackup Cluster Backup Timer
|
||||||
|
Documentation=https://github.com/PlusOne/dbbackup
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
# Run daily at midnight
|
||||||
|
OnCalendar=daily
|
||||||
|
|
||||||
|
# Randomize start time within 15 minutes to avoid thundering herd
|
||||||
|
RandomizedDelaySec=900
|
||||||
|
|
||||||
|
# Run immediately if we missed the last scheduled time
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
# Run even if system was sleeping
|
||||||
|
WakeSystem=false
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
|
EOF
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 5: Enable and Start
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Reload systemd
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
|
||||||
|
# Enable timer (auto-start on boot)
|
||||||
|
sudo systemctl enable dbbackup-cluster.timer
|
||||||
|
|
||||||
|
# Start timer
|
||||||
|
sudo systemctl start dbbackup-cluster.timer
|
||||||
|
|
||||||
|
# Verify timer is active
|
||||||
|
sudo systemctl status dbbackup-cluster.timer
|
||||||
|
|
||||||
|
# View next scheduled run
|
||||||
|
sudo systemctl list-timers dbbackup-cluster.timer
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 6: Test Backup
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Run backup manually
|
||||||
|
sudo systemctl start dbbackup-cluster.service
|
||||||
|
|
||||||
|
# Check status
|
||||||
|
sudo systemctl status dbbackup-cluster.service
|
||||||
|
|
||||||
|
# View logs
|
||||||
|
sudo journalctl -u dbbackup-cluster.service -f
|
||||||
|
```
|
||||||
|
|
||||||
|
## Prometheus Metrics Exporter (Manual)
|
||||||
|
|
||||||
|
### Service Unit
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo tee /etc/systemd/system/dbbackup-exporter.service << 'EOF'
|
||||||
|
[Unit]
|
||||||
|
Description=DBBackup Prometheus Metrics Exporter
|
||||||
|
Documentation=https://github.com/PlusOne/dbbackup
|
||||||
|
After=network.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=dbbackup
|
||||||
|
Group=dbbackup
|
||||||
|
|
||||||
|
# Working directory
|
||||||
|
WorkingDirectory=/var/lib/dbbackup
|
||||||
|
|
||||||
|
# Start HTTP metrics server
|
||||||
|
ExecStart=/usr/local/bin/dbbackup metrics serve --port 9399
|
||||||
|
|
||||||
|
# Restart on failure
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=10
|
||||||
|
|
||||||
|
# Security hardening
|
||||||
|
NoNewPrivileges=yes
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=yes
|
||||||
|
PrivateTmp=yes
|
||||||
|
PrivateDevices=yes
|
||||||
|
ProtectKernelTunables=yes
|
||||||
|
ProtectKernelModules=yes
|
||||||
|
ProtectControlGroups=yes
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||||
|
RestrictNamespaces=yes
|
||||||
|
RestrictRealtime=yes
|
||||||
|
RestrictSUIDSGID=yes
|
||||||
|
LockPersonality=yes
|
||||||
|
|
||||||
|
# Catalog access
|
||||||
|
ReadWritePaths=/var/lib/dbbackup
|
||||||
|
|
||||||
|
# Capability restrictions
|
||||||
|
CapabilityBoundingSet=CAP_NET_BIND_SERVICE
|
||||||
|
AmbientCapabilities=
|
||||||
|
|
||||||
|
# Logging
|
||||||
|
StandardOutput=journal
|
||||||
|
StandardError=journal
|
||||||
|
SyslogIdentifier=dbbackup-exporter
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
EOF
|
||||||
|
```
|
||||||
|
|
||||||
|
### Enable Exporter
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
sudo systemctl enable dbbackup-exporter
|
||||||
|
sudo systemctl start dbbackup-exporter
|
||||||
|
|
||||||
|
# Test
|
||||||
|
curl http://localhost:9399/health
|
||||||
|
curl http://localhost:9399/metrics
|
||||||
|
```
|
||||||
|
|
||||||
|
### Prometheus Configuration
|
||||||
|
|
||||||
|
Add to `prometheus.yml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: 'dbbackup'
|
||||||
|
static_configs:
|
||||||
|
- targets: ['localhost:9399']
|
||||||
|
scrape_interval: 60s
|
||||||
|
```
|
||||||
|
|
||||||
|
## Security Hardening
|
||||||
|
|
||||||
|
The systemd units include comprehensive security hardening:
|
||||||
|
|
||||||
|
| Setting | Purpose |
|
||||||
|
|---------|---------|
|
||||||
|
| `NoNewPrivileges=yes` | Prevent privilege escalation |
|
||||||
|
| `ProtectSystem=strict` | Read-only filesystem except allowed paths |
|
||||||
|
| `ProtectHome=yes` | Block access to /home, /root, /run/user |
|
||||||
|
| `PrivateTmp=yes` | Isolated /tmp namespace |
|
||||||
|
| `PrivateDevices=yes` | No access to physical devices |
|
||||||
|
| `RestrictAddressFamilies` | Only Unix and IP sockets |
|
||||||
|
| `MemoryDenyWriteExecute=yes` | Prevent code injection |
|
||||||
|
| `CapabilityBoundingSet` | Minimal Linux capabilities |
|
||||||
|
| `OOMScoreAdjust=-100` | Protect backup from OOM killer |
|
||||||
|
|
||||||
|
### Database Access
|
||||||
|
|
||||||
|
For PostgreSQL with peer authentication:
|
||||||
|
```bash
|
||||||
|
# Add dbbackup user to postgres group
|
||||||
|
sudo usermod -aG postgres dbbackup
|
||||||
|
|
||||||
|
# Or create a .pgpass file
|
||||||
|
sudo -u dbbackup tee /var/lib/dbbackup/.pgpass << EOF
|
||||||
|
localhost:5432:*:postgres:password
|
||||||
|
EOF
|
||||||
|
sudo chmod 600 /var/lib/dbbackup/.pgpass
|
||||||
|
```
|
||||||
|
|
||||||
|
For PostgreSQL with password authentication:
|
||||||
|
```bash
|
||||||
|
# Store password in environment file
|
||||||
|
echo "PGPASSWORD=your_password" | sudo tee /etc/dbbackup/env.d/cluster.conf
|
||||||
|
sudo chmod 600 /etc/dbbackup/env.d/cluster.conf
|
||||||
|
```
|
||||||
|
|
||||||
|
## Multiple Instances
|
||||||
|
|
||||||
|
Run different backup configurations as separate instances:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Install multiple instances
|
||||||
|
sudo dbbackup install --instance production --schedule "*-*-* 02:00:00"
|
||||||
|
sudo dbbackup install --instance staging --schedule "*-*-* 04:00:00"
|
||||||
|
sudo dbbackup install --instance analytics --schedule "weekly"
|
||||||
|
|
||||||
|
# Manage individually
|
||||||
|
sudo systemctl status dbbackup@production.timer
|
||||||
|
sudo systemctl start dbbackup@staging.service
|
||||||
|
```
|
||||||
|
|
||||||
|
Each instance has its own:
|
||||||
|
- Configuration: `/etc/dbbackup/env.d/<instance>.conf`
|
||||||
|
- Timer schedule
|
||||||
|
- Journal logs: `journalctl -u dbbackup@<instance>.service`
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### View Logs
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Real-time logs
|
||||||
|
sudo journalctl -u dbbackup-cluster.service -f
|
||||||
|
|
||||||
|
# Last backup run
|
||||||
|
sudo journalctl -u dbbackup-cluster.service -n 100
|
||||||
|
|
||||||
|
# All dbbackup logs
|
||||||
|
sudo journalctl -t dbbackup
|
||||||
|
|
||||||
|
# Exporter logs
|
||||||
|
sudo journalctl -u dbbackup-exporter -f
|
||||||
|
```
|
||||||
|
|
||||||
|
### Timer Not Running
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Check timer status
|
||||||
|
sudo systemctl status dbbackup-cluster.timer
|
||||||
|
|
||||||
|
# List all timers
|
||||||
|
sudo systemctl list-timers --all | grep dbbackup
|
||||||
|
|
||||||
|
# Check if timer is enabled
|
||||||
|
sudo systemctl is-enabled dbbackup-cluster.timer
|
||||||
|
```
|
||||||
|
|
||||||
|
### Service Fails to Start
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Check service status
|
||||||
|
sudo systemctl status dbbackup-cluster.service
|
||||||
|
|
||||||
|
# View detailed error
|
||||||
|
sudo journalctl -u dbbackup-cluster.service -n 50 --no-pager
|
||||||
|
|
||||||
|
# Test manually as dbbackup user
|
||||||
|
sudo -u dbbackup /usr/local/bin/dbbackup backup cluster --config /etc/dbbackup/dbbackup.conf
|
||||||
|
|
||||||
|
# Check permissions
|
||||||
|
ls -la /var/lib/dbbackup/
|
||||||
|
ls -la /etc/dbbackup/
|
||||||
|
```
|
||||||
|
|
||||||
|
### Permission Denied
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Fix ownership
|
||||||
|
sudo chown -R dbbackup:dbbackup /var/lib/dbbackup
|
||||||
|
|
||||||
|
# Check SELinux (if enabled)
|
||||||
|
sudo ausearch -m avc -ts recent
|
||||||
|
|
||||||
|
# Check AppArmor (if enabled)
|
||||||
|
sudo aa-status
|
||||||
|
```
|
||||||
|
|
||||||
|
### Exporter Not Accessible
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Check if running
|
||||||
|
sudo systemctl status dbbackup-exporter
|
||||||
|
|
||||||
|
# Check port binding
|
||||||
|
sudo ss -tlnp | grep 9399
|
||||||
|
|
||||||
|
# Test locally
|
||||||
|
curl -v http://localhost:9399/health
|
||||||
|
|
||||||
|
# Check firewall
|
||||||
|
sudo ufw status
|
||||||
|
sudo iptables -L -n | grep 9399
|
||||||
|
```
|
||||||
|
|
||||||
|
## Prometheus Alerting Rules
|
||||||
|
|
||||||
|
Add these alert rules to your Prometheus configuration for backup monitoring:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# /etc/prometheus/rules/dbbackup.yml
|
||||||
|
groups:
|
||||||
|
- name: dbbackup
|
||||||
|
rules:
|
||||||
|
# Alert if no successful backup in 24 hours
|
||||||
|
- alert: DBBackupMissing
|
||||||
|
expr: time() - dbbackup_last_success_timestamp > 86400
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "No backup in 24 hours on {{ $labels.instance }}"
|
||||||
|
description: "Database {{ $labels.database }} has not had a successful backup in over 24 hours."
|
||||||
|
|
||||||
|
# Alert if backup verification failed
|
||||||
|
- alert: DBBackupVerificationFailed
|
||||||
|
expr: dbbackup_backup_verified == 0
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "Backup verification failed on {{ $labels.instance }}"
|
||||||
|
description: "Last backup for {{ $labels.database }} failed verification check."
|
||||||
|
|
||||||
|
# Alert if RPO exceeded (48 hours)
|
||||||
|
- alert: DBBackupRPOExceeded
|
||||||
|
expr: dbbackup_rpo_seconds > 172800
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
annotations:
|
||||||
|
summary: "RPO exceeded on {{ $labels.instance }}"
|
||||||
|
description: "Recovery Point Objective exceeded 48 hours for {{ $labels.database }}."
|
||||||
|
|
||||||
|
# Alert if exporter is down
|
||||||
|
- alert: DBBackupExporterDown
|
||||||
|
expr: up{job="dbbackup"} == 0
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "DBBackup exporter down on {{ $labels.instance }}"
|
||||||
|
description: "Cannot scrape metrics from dbbackup-exporter."
|
||||||
|
|
||||||
|
# Alert if backup size dropped significantly (possible truncation)
|
||||||
|
- alert: DBBackupSizeAnomaly
|
||||||
|
expr: dbbackup_last_backup_size_bytes < (dbbackup_last_backup_size_bytes offset 1d) * 0.5
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: "Backup size anomaly on {{ $labels.instance }}"
|
||||||
|
description: "Backup size for {{ $labels.database }} dropped by more than 50%."
|
||||||
|
```
|
||||||
|
|
||||||
|
### Loading Alert Rules
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Test rules syntax
|
||||||
|
promtool check rules /etc/prometheus/rules/dbbackup.yml
|
||||||
|
|
||||||
|
# Reload Prometheus
|
||||||
|
sudo systemctl reload prometheus
|
||||||
|
# or via API:
|
||||||
|
curl -X POST http://localhost:9090/-/reload
|
||||||
|
```
|
||||||
|
|
||||||
|
## Catalog Sync for Existing Backups
|
||||||
|
|
||||||
|
If you have existing backups created before installing v3.41+, sync them to the catalog:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Sync existing backups to catalog
|
||||||
|
dbbackup catalog sync /path/to/backup/directory --allow-root
|
||||||
|
|
||||||
|
# Verify catalog contents
|
||||||
|
dbbackup catalog list --allow-root
|
||||||
|
|
||||||
|
# Show statistics
|
||||||
|
dbbackup catalog stats --allow-root
|
||||||
|
```
|
||||||
|
|
||||||
|
## Uninstallation
|
||||||
|
|
||||||
|
### Using Installer
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Remove cluster backup (keeps config)
|
||||||
|
sudo dbbackup uninstall cluster
|
||||||
|
|
||||||
|
# Remove and purge configuration
|
||||||
|
sudo dbbackup uninstall cluster --purge
|
||||||
|
|
||||||
|
# Remove named instance
|
||||||
|
sudo dbbackup uninstall production --purge
|
||||||
|
```
|
||||||
|
|
||||||
|
### Manual Removal
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Stop and disable services
|
||||||
|
sudo systemctl stop dbbackup-cluster.timer dbbackup-cluster.service dbbackup-exporter
|
||||||
|
sudo systemctl disable dbbackup-cluster.timer dbbackup-exporter
|
||||||
|
|
||||||
|
# Remove unit files
|
||||||
|
sudo rm /etc/systemd/system/dbbackup-cluster.service
|
||||||
|
sudo rm /etc/systemd/system/dbbackup-cluster.timer
|
||||||
|
sudo rm /etc/systemd/system/dbbackup-exporter.service
|
||||||
|
sudo rm /etc/systemd/system/dbbackup@.service
|
||||||
|
sudo rm /etc/systemd/system/dbbackup@.timer
|
||||||
|
|
||||||
|
# Reload systemd
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
|
||||||
|
# Optional: Remove user and directories
|
||||||
|
sudo userdel dbbackup
|
||||||
|
sudo rm -rf /var/lib/dbbackup
|
||||||
|
sudo rm -rf /etc/dbbackup
|
||||||
|
sudo rm -rf /var/log/dbbackup
|
||||||
|
sudo rm /usr/local/bin/dbbackup
|
||||||
|
```
|
||||||
|
|
||||||
|
## See Also
|
||||||
|
|
||||||
|
- [README.md](README.md) - Main documentation
|
||||||
|
- [DOCKER.md](DOCKER.md) - Docker deployment
|
||||||
|
- [CLOUD.md](CLOUD.md) - Cloud storage configuration
|
||||||
|
- [PITR.md](PITR.md) - Point-in-Time Recovery
|
||||||
@@ -43,7 +43,7 @@ What are you actually getting?
|
|||||||
```bash
|
```bash
|
||||||
# Physical backup at InnoDB page level
|
# Physical backup at InnoDB page level
|
||||||
# No XtraBackup. No external tools. Pure Go.
|
# No XtraBackup. No external tools. Pure Go.
|
||||||
dbbackup backup --engine=clone --output=s3://bucket/backup
|
dbbackup backup single mydb --db-type mysql --cloud s3://bucket/backups/
|
||||||
```
|
```
|
||||||
|
|
||||||
### Filesystem Snapshots
|
### Filesystem Snapshots
|
||||||
@@ -78,10 +78,10 @@ dbbackup backup --engine=streaming --parallel-workers=8
|
|||||||
**Day 1**: Run dbbackup alongside existing solution
|
**Day 1**: Run dbbackup alongside existing solution
|
||||||
```bash
|
```bash
|
||||||
# Test backup
|
# Test backup
|
||||||
dbbackup backup --database=mydb --output=s3://test-bucket/
|
dbbackup backup single mydb --cloud s3://test-bucket/
|
||||||
|
|
||||||
# Verify integrity
|
# Verify integrity
|
||||||
dbbackup verify s3://test-bucket/backup.sql.gz.enc
|
dbbackup verify s3://test-bucket/mydb_20260115.dump.gz
|
||||||
```
|
```
|
||||||
|
|
||||||
**Week 1**: Compare backup times, storage costs, recovery speed
|
**Week 1**: Compare backup times, storage costs, recovery speed
|
||||||
@@ -112,10 +112,9 @@ curl -LO https://github.com/UUXO/dbbackup/releases/latest/download/dbbackup_linu
|
|||||||
chmod +x dbbackup_linux_amd64
|
chmod +x dbbackup_linux_amd64
|
||||||
|
|
||||||
# Your first backup
|
# Your first backup
|
||||||
./dbbackup_linux_amd64 backup \
|
./dbbackup_linux_amd64 backup single production \
|
||||||
--database=production \
|
--db-type mysql \
|
||||||
--engine=auto \
|
--cloud s3://my-backups/
|
||||||
--output=s3://my-backups/$(date +%Y%m%d)/
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## The Bottom Line
|
## The Bottom Line
|
||||||
@@ -131,4 +130,4 @@ Every dollar you spend on backup licensing is a dollar not spent on:
|
|||||||
|
|
||||||
*Apache 2.0 Licensed. Free forever. No sales calls required.*
|
*Apache 2.0 Licensed. Free forever. No sales calls required.*
|
||||||
|
|
||||||
[GitHub](https://github.com/UUXO/dbbackup) | [Documentation](https://github.com/UUXO/dbbackup#readme) | [Release Notes](RELEASE_NOTES_v3.2.md)
|
[GitHub](https://github.com/UUXO/dbbackup) | [Documentation](https://github.com/UUXO/dbbackup#readme) | [Changelog](CHANGELOG.md)
|
||||||
|
|||||||
87
bin/README.md
Normal file
87
bin/README.md
Normal file
@@ -0,0 +1,87 @@
|
|||||||
|
# DB Backup Tool - Pre-compiled Binaries
|
||||||
|
|
||||||
|
This directory contains pre-compiled binaries for the DB Backup Tool across multiple platforms and architectures.
|
||||||
|
|
||||||
|
## Build Information
|
||||||
|
- **Version**: 3.42.1
|
||||||
|
- **Build Time**: 2026-01-08_05:03:53_UTC
|
||||||
|
- **Git Commit**: 9c65821
|
||||||
|
|
||||||
|
## Recent Updates (v1.1.0)
|
||||||
|
- ✅ Fixed TUI progress display with line-by-line output
|
||||||
|
- ✅ Added interactive configuration settings menu
|
||||||
|
- ✅ Improved menu navigation and responsiveness
|
||||||
|
- ✅ Enhanced completion status handling
|
||||||
|
- ✅ Better CPU detection and optimization
|
||||||
|
- ✅ Silent mode support for TUI operations
|
||||||
|
|
||||||
|
## Available Binaries
|
||||||
|
|
||||||
|
### Linux
|
||||||
|
- `dbbackup_linux_amd64` - Linux 64-bit (Intel/AMD)
|
||||||
|
- `dbbackup_linux_arm64` - Linux 64-bit (ARM)
|
||||||
|
- `dbbackup_linux_arm_armv7` - Linux 32-bit (ARMv7)
|
||||||
|
|
||||||
|
### macOS
|
||||||
|
- `dbbackup_darwin_amd64` - macOS 64-bit (Intel)
|
||||||
|
- `dbbackup_darwin_arm64` - macOS 64-bit (Apple Silicon)
|
||||||
|
|
||||||
|
### Windows
|
||||||
|
- `dbbackup_windows_amd64.exe` - Windows 64-bit (Intel/AMD)
|
||||||
|
- `dbbackup_windows_arm64.exe` - Windows 64-bit (ARM)
|
||||||
|
|
||||||
|
### BSD Systems
|
||||||
|
- `dbbackup_freebsd_amd64` - FreeBSD 64-bit
|
||||||
|
- `dbbackup_openbsd_amd64` - OpenBSD 64-bit
|
||||||
|
- `dbbackup_netbsd_amd64` - NetBSD 64-bit
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
1. Download the appropriate binary for your platform
|
||||||
|
2. Make it executable (Unix-like systems): `chmod +x dbbackup_*`
|
||||||
|
3. Run: `./dbbackup_* --help`
|
||||||
|
|
||||||
|
## Interactive Mode
|
||||||
|
|
||||||
|
Launch the interactive TUI menu for easy configuration and operation:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Interactive mode with TUI menu
|
||||||
|
./dbbackup_linux_amd64
|
||||||
|
|
||||||
|
# Features:
|
||||||
|
# - Interactive configuration settings
|
||||||
|
# - Real-time progress display
|
||||||
|
# - Operation history and status
|
||||||
|
# - CPU detection and optimization
|
||||||
|
```
|
||||||
|
|
||||||
|
## Command Line Mode
|
||||||
|
|
||||||
|
Direct command line usage with line-by-line progress:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Show CPU information and optimization settings
|
||||||
|
./dbbackup_linux_amd64 cpu
|
||||||
|
|
||||||
|
# Auto-optimize for your hardware
|
||||||
|
./dbbackup_linux_amd64 backup cluster --auto-detect-cores
|
||||||
|
|
||||||
|
# Manual CPU configuration
|
||||||
|
./dbbackup_linux_amd64 backup single mydb --jobs 8 --dump-jobs 4
|
||||||
|
|
||||||
|
# Line-by-line progress output
|
||||||
|
./dbbackup_linux_amd64 backup cluster --progress-type line
|
||||||
|
```
|
||||||
|
|
||||||
|
## CPU Detection
|
||||||
|
|
||||||
|
All binaries include advanced CPU detection capabilities:
|
||||||
|
- Automatic core detection for optimal parallelism
|
||||||
|
- Support for different workload types (CPU-intensive, I/O-intensive, balanced)
|
||||||
|
- Platform-specific optimizations for Linux, macOS, and Windows
|
||||||
|
- Interactive CPU configuration in TUI mode
|
||||||
|
|
||||||
|
## Support
|
||||||
|
|
||||||
|
For issues or questions, please refer to the main project documentation.
|
||||||
@@ -15,7 +15,7 @@ echo "🔧 Using Go version: $GO_VERSION"
|
|||||||
|
|
||||||
# Configuration
|
# Configuration
|
||||||
APP_NAME="dbbackup"
|
APP_NAME="dbbackup"
|
||||||
VERSION="3.1.0"
|
VERSION=$(grep 'version.*=' main.go | head -1 | sed 's/.*"\(.*\)".*/\1/')
|
||||||
BUILD_TIME=$(date -u '+%Y-%m-%d_%H:%M:%S_UTC')
|
BUILD_TIME=$(date -u '+%Y-%m-%d_%H:%M:%S_UTC')
|
||||||
GIT_COMMIT=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")
|
GIT_COMMIT=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")
|
||||||
BIN_DIR="bin"
|
BIN_DIR="bin"
|
||||||
@@ -83,7 +83,8 @@ for platform_config in "${PLATFORMS[@]}"; do
|
|||||||
echo -e "${YELLOW}[$current/$total_platforms]${NC} Building for ${BOLD}$description${NC} (${platform})"
|
echo -e "${YELLOW}[$current/$total_platforms]${NC} Building for ${BOLD}$description${NC} (${platform})"
|
||||||
|
|
||||||
# Set environment and build (using export for better compatibility)
|
# Set environment and build (using export for better compatibility)
|
||||||
export GOOS GOARCH
|
# CGO_ENABLED=0 creates static binaries without glibc dependency
|
||||||
|
export CGO_ENABLED=0 GOOS GOARCH
|
||||||
if go build -ldflags "$LDFLAGS" -o "${BIN_DIR}/${binary_name}" . 2>/dev/null; then
|
if go build -ldflags "$LDFLAGS" -o "${BIN_DIR}/${binary_name}" . 2>/dev/null; then
|
||||||
# Get file size
|
# Get file size
|
||||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||||
|
|||||||
116
cmd/catalog.go
116
cmd/catalog.go
@@ -252,8 +252,8 @@ func runCatalogSync(cmd *cobra.Command, args []string) error {
|
|||||||
}
|
}
|
||||||
defer cat.Close()
|
defer cat.Close()
|
||||||
|
|
||||||
fmt.Printf("📁 Syncing backups from: %s\n", absDir)
|
fmt.Printf("[DIR] Syncing backups from: %s\n", absDir)
|
||||||
fmt.Printf("📊 Catalog database: %s\n\n", catalogDBPath)
|
fmt.Printf("[STATS] Catalog database: %s\n\n", catalogDBPath)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
result, err := cat.SyncFromDirectory(ctx, absDir)
|
result, err := cat.SyncFromDirectory(ctx, absDir)
|
||||||
@@ -265,17 +265,17 @@ func runCatalogSync(cmd *cobra.Command, args []string) error {
|
|||||||
cat.SetLastSync(ctx)
|
cat.SetLastSync(ctx)
|
||||||
|
|
||||||
// Show results
|
// Show results
|
||||||
fmt.Printf("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n")
|
fmt.Printf("=====================================================\n")
|
||||||
fmt.Printf(" Sync Results\n")
|
fmt.Printf(" Sync Results\n")
|
||||||
fmt.Printf("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n")
|
fmt.Printf("=====================================================\n")
|
||||||
fmt.Printf(" ✅ Added: %d\n", result.Added)
|
fmt.Printf(" [OK] Added: %d\n", result.Added)
|
||||||
fmt.Printf(" 🔄 Updated: %d\n", result.Updated)
|
fmt.Printf(" [SYNC] Updated: %d\n", result.Updated)
|
||||||
fmt.Printf(" 🗑️ Removed: %d\n", result.Removed)
|
fmt.Printf(" [DEL] Removed: %d\n", result.Removed)
|
||||||
if result.Errors > 0 {
|
if result.Errors > 0 {
|
||||||
fmt.Printf(" ❌ Errors: %d\n", result.Errors)
|
fmt.Printf(" [FAIL] Errors: %d\n", result.Errors)
|
||||||
}
|
}
|
||||||
fmt.Printf(" ⏱️ Duration: %.2fs\n", result.Duration)
|
fmt.Printf(" [TIME] Duration: %.2fs\n", result.Duration)
|
||||||
fmt.Printf("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n")
|
fmt.Printf("=====================================================\n")
|
||||||
|
|
||||||
// Show details if verbose
|
// Show details if verbose
|
||||||
if catalogVerbose && len(result.Details) > 0 {
|
if catalogVerbose && len(result.Details) > 0 {
|
||||||
@@ -323,7 +323,7 @@ func runCatalogList(cmd *cobra.Command, args []string) error {
|
|||||||
// Table format
|
// Table format
|
||||||
fmt.Printf("%-30s %-12s %-10s %-20s %-10s %s\n",
|
fmt.Printf("%-30s %-12s %-10s %-20s %-10s %s\n",
|
||||||
"DATABASE", "TYPE", "SIZE", "CREATED", "STATUS", "PATH")
|
"DATABASE", "TYPE", "SIZE", "CREATED", "STATUS", "PATH")
|
||||||
fmt.Println(strings.Repeat("─", 120))
|
fmt.Println(strings.Repeat("-", 120))
|
||||||
|
|
||||||
for _, entry := range entries {
|
for _, entry := range entries {
|
||||||
dbName := truncateString(entry.Database, 28)
|
dbName := truncateString(entry.Database, 28)
|
||||||
@@ -331,10 +331,10 @@ func runCatalogList(cmd *cobra.Command, args []string) error {
|
|||||||
|
|
||||||
status := string(entry.Status)
|
status := string(entry.Status)
|
||||||
if entry.VerifyValid != nil && *entry.VerifyValid {
|
if entry.VerifyValid != nil && *entry.VerifyValid {
|
||||||
status = "✓ verified"
|
status = "[OK] verified"
|
||||||
}
|
}
|
||||||
if entry.DrillSuccess != nil && *entry.DrillSuccess {
|
if entry.DrillSuccess != nil && *entry.DrillSuccess {
|
||||||
status = "✓ tested"
|
status = "[OK] tested"
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("%-30s %-12s %-10s %-20s %-10s %s\n",
|
fmt.Printf("%-30s %-12s %-10s %-20s %-10s %s\n",
|
||||||
@@ -377,20 +377,20 @@ func runCatalogStats(cmd *cobra.Command, args []string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Table format
|
// Table format
|
||||||
fmt.Printf("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n")
|
fmt.Printf("=====================================================\n")
|
||||||
if catalogDatabase != "" {
|
if catalogDatabase != "" {
|
||||||
fmt.Printf(" Catalog Statistics: %s\n", catalogDatabase)
|
fmt.Printf(" Catalog Statistics: %s\n", catalogDatabase)
|
||||||
} else {
|
} else {
|
||||||
fmt.Printf(" Catalog Statistics\n")
|
fmt.Printf(" Catalog Statistics\n")
|
||||||
}
|
}
|
||||||
fmt.Printf("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n")
|
fmt.Printf("=====================================================\n\n")
|
||||||
|
|
||||||
fmt.Printf("📊 Total Backups: %d\n", stats.TotalBackups)
|
fmt.Printf("[STATS] Total Backups: %d\n", stats.TotalBackups)
|
||||||
fmt.Printf("💾 Total Size: %s\n", stats.TotalSizeHuman)
|
fmt.Printf("[SAVE] Total Size: %s\n", stats.TotalSizeHuman)
|
||||||
fmt.Printf("📏 Average Size: %s\n", catalog.FormatSize(stats.AvgSize))
|
fmt.Printf("[SIZE] Average Size: %s\n", catalog.FormatSize(stats.AvgSize))
|
||||||
fmt.Printf("⏱️ Average Duration: %.1fs\n", stats.AvgDuration)
|
fmt.Printf("[TIME] Average Duration: %.1fs\n", stats.AvgDuration)
|
||||||
fmt.Printf("✅ Verified: %d\n", stats.VerifiedCount)
|
fmt.Printf("[OK] Verified: %d\n", stats.VerifiedCount)
|
||||||
fmt.Printf("🧪 Drill Tested: %d\n", stats.DrillTestedCount)
|
fmt.Printf("[TEST] Drill Tested: %d\n", stats.DrillTestedCount)
|
||||||
|
|
||||||
if stats.OldestBackup != nil {
|
if stats.OldestBackup != nil {
|
||||||
fmt.Printf("📅 Oldest Backup: %s\n", stats.OldestBackup.Format("2006-01-02 15:04"))
|
fmt.Printf("📅 Oldest Backup: %s\n", stats.OldestBackup.Format("2006-01-02 15:04"))
|
||||||
@@ -400,27 +400,27 @@ func runCatalogStats(cmd *cobra.Command, args []string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if len(stats.ByDatabase) > 0 && catalogDatabase == "" {
|
if len(stats.ByDatabase) > 0 && catalogDatabase == "" {
|
||||||
fmt.Printf("\n📁 By Database:\n")
|
fmt.Printf("\n[DIR] By Database:\n")
|
||||||
for db, count := range stats.ByDatabase {
|
for db, count := range stats.ByDatabase {
|
||||||
fmt.Printf(" %-30s %d\n", db, count)
|
fmt.Printf(" %-30s %d\n", db, count)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(stats.ByType) > 0 {
|
if len(stats.ByType) > 0 {
|
||||||
fmt.Printf("\n📦 By Type:\n")
|
fmt.Printf("\n[PKG] By Type:\n")
|
||||||
for t, count := range stats.ByType {
|
for t, count := range stats.ByType {
|
||||||
fmt.Printf(" %-15s %d\n", t, count)
|
fmt.Printf(" %-15s %d\n", t, count)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(stats.ByStatus) > 0 {
|
if len(stats.ByStatus) > 0 {
|
||||||
fmt.Printf("\n📋 By Status:\n")
|
fmt.Printf("\n[LOG] By Status:\n")
|
||||||
for s, count := range stats.ByStatus {
|
for s, count := range stats.ByStatus {
|
||||||
fmt.Printf(" %-15s %d\n", s, count)
|
fmt.Printf(" %-15s %d\n", s, count)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n")
|
fmt.Printf("\n=====================================================\n")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -488,26 +488,26 @@ func runCatalogGaps(cmd *cobra.Command, args []string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if len(allGaps) == 0 {
|
if len(allGaps) == 0 {
|
||||||
fmt.Printf("✅ No backup gaps detected (expected interval: %s)\n", interval)
|
fmt.Printf("[OK] No backup gaps detected (expected interval: %s)\n", interval)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n")
|
fmt.Printf("=====================================================\n")
|
||||||
fmt.Printf(" Backup Gaps Detected (expected interval: %s)\n", interval)
|
fmt.Printf(" Backup Gaps Detected (expected interval: %s)\n", interval)
|
||||||
fmt.Printf("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n")
|
fmt.Printf("=====================================================\n\n")
|
||||||
|
|
||||||
totalGaps := 0
|
totalGaps := 0
|
||||||
criticalGaps := 0
|
criticalGaps := 0
|
||||||
|
|
||||||
for database, gaps := range allGaps {
|
for database, gaps := range allGaps {
|
||||||
fmt.Printf("📁 %s (%d gaps)\n", database, len(gaps))
|
fmt.Printf("[DIR] %s (%d gaps)\n", database, len(gaps))
|
||||||
|
|
||||||
for _, gap := range gaps {
|
for _, gap := range gaps {
|
||||||
totalGaps++
|
totalGaps++
|
||||||
icon := "ℹ️"
|
icon := "[INFO]"
|
||||||
switch gap.Severity {
|
switch gap.Severity {
|
||||||
case catalog.SeverityWarning:
|
case catalog.SeverityWarning:
|
||||||
icon = "⚠️"
|
icon = "[WARN]"
|
||||||
case catalog.SeverityCritical:
|
case catalog.SeverityCritical:
|
||||||
icon = "🚨"
|
icon = "🚨"
|
||||||
criticalGaps++
|
criticalGaps++
|
||||||
@@ -523,7 +523,7 @@ func runCatalogGaps(cmd *cobra.Command, args []string) error {
|
|||||||
fmt.Println()
|
fmt.Println()
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n")
|
fmt.Printf("=====================================================\n")
|
||||||
fmt.Printf("Total: %d gaps detected", totalGaps)
|
fmt.Printf("Total: %d gaps detected", totalGaps)
|
||||||
if criticalGaps > 0 {
|
if criticalGaps > 0 {
|
||||||
fmt.Printf(" (%d critical)", criticalGaps)
|
fmt.Printf(" (%d critical)", criticalGaps)
|
||||||
@@ -598,20 +598,20 @@ func runCatalogSearch(cmd *cobra.Command, args []string) error {
|
|||||||
fmt.Printf("Found %d matching backups:\n\n", len(entries))
|
fmt.Printf("Found %d matching backups:\n\n", len(entries))
|
||||||
|
|
||||||
for _, entry := range entries {
|
for _, entry := range entries {
|
||||||
fmt.Printf("📁 %s\n", entry.Database)
|
fmt.Printf("[DIR] %s\n", entry.Database)
|
||||||
fmt.Printf(" Path: %s\n", entry.BackupPath)
|
fmt.Printf(" Path: %s\n", entry.BackupPath)
|
||||||
fmt.Printf(" Type: %s | Size: %s | Created: %s\n",
|
fmt.Printf(" Type: %s | Size: %s | Created: %s\n",
|
||||||
entry.DatabaseType,
|
entry.DatabaseType,
|
||||||
catalog.FormatSize(entry.SizeBytes),
|
catalog.FormatSize(entry.SizeBytes),
|
||||||
entry.CreatedAt.Format("2006-01-02 15:04:05"))
|
entry.CreatedAt.Format("2006-01-02 15:04:05"))
|
||||||
if entry.Encrypted {
|
if entry.Encrypted {
|
||||||
fmt.Printf(" 🔒 Encrypted\n")
|
fmt.Printf(" [LOCK] Encrypted\n")
|
||||||
}
|
}
|
||||||
if entry.VerifyValid != nil && *entry.VerifyValid {
|
if entry.VerifyValid != nil && *entry.VerifyValid {
|
||||||
fmt.Printf(" ✅ Verified: %s\n", entry.VerifiedAt.Format("2006-01-02 15:04"))
|
fmt.Printf(" [OK] Verified: %s\n", entry.VerifiedAt.Format("2006-01-02 15:04"))
|
||||||
}
|
}
|
||||||
if entry.DrillSuccess != nil && *entry.DrillSuccess {
|
if entry.DrillSuccess != nil && *entry.DrillSuccess {
|
||||||
fmt.Printf(" 🧪 Drill Tested: %s\n", entry.DrillTestedAt.Format("2006-01-02 15:04"))
|
fmt.Printf(" [TEST] Drill Tested: %s\n", entry.DrillTestedAt.Format("2006-01-02 15:04"))
|
||||||
}
|
}
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
}
|
}
|
||||||
@@ -655,64 +655,64 @@ func runCatalogInfo(cmd *cobra.Command, args []string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n")
|
fmt.Printf("=====================================================\n")
|
||||||
fmt.Printf(" Backup Details\n")
|
fmt.Printf(" Backup Details\n")
|
||||||
fmt.Printf("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n")
|
fmt.Printf("=====================================================\n\n")
|
||||||
|
|
||||||
fmt.Printf("📁 Database: %s\n", entry.Database)
|
fmt.Printf("[DIR] Database: %s\n", entry.Database)
|
||||||
fmt.Printf("🔧 Type: %s\n", entry.DatabaseType)
|
fmt.Printf("🔧 Type: %s\n", entry.DatabaseType)
|
||||||
fmt.Printf("🖥️ Host: %s:%d\n", entry.Host, entry.Port)
|
fmt.Printf("[HOST] Host: %s:%d\n", entry.Host, entry.Port)
|
||||||
fmt.Printf("📂 Path: %s\n", entry.BackupPath)
|
fmt.Printf("📂 Path: %s\n", entry.BackupPath)
|
||||||
fmt.Printf("📦 Backup Type: %s\n", entry.BackupType)
|
fmt.Printf("[PKG] Backup Type: %s\n", entry.BackupType)
|
||||||
fmt.Printf("💾 Size: %s (%d bytes)\n", catalog.FormatSize(entry.SizeBytes), entry.SizeBytes)
|
fmt.Printf("[SAVE] Size: %s (%d bytes)\n", catalog.FormatSize(entry.SizeBytes), entry.SizeBytes)
|
||||||
fmt.Printf("🔐 SHA256: %s\n", entry.SHA256)
|
fmt.Printf("[HASH] SHA256: %s\n", entry.SHA256)
|
||||||
fmt.Printf("📅 Created: %s\n", entry.CreatedAt.Format("2006-01-02 15:04:05 MST"))
|
fmt.Printf("📅 Created: %s\n", entry.CreatedAt.Format("2006-01-02 15:04:05 MST"))
|
||||||
fmt.Printf("⏱️ Duration: %.2fs\n", entry.Duration)
|
fmt.Printf("[TIME] Duration: %.2fs\n", entry.Duration)
|
||||||
fmt.Printf("📋 Status: %s\n", entry.Status)
|
fmt.Printf("[LOG] Status: %s\n", entry.Status)
|
||||||
|
|
||||||
if entry.Compression != "" {
|
if entry.Compression != "" {
|
||||||
fmt.Printf("📦 Compression: %s\n", entry.Compression)
|
fmt.Printf("[PKG] Compression: %s\n", entry.Compression)
|
||||||
}
|
}
|
||||||
if entry.Encrypted {
|
if entry.Encrypted {
|
||||||
fmt.Printf("🔒 Encrypted: yes\n")
|
fmt.Printf("[LOCK] Encrypted: yes\n")
|
||||||
}
|
}
|
||||||
if entry.CloudLocation != "" {
|
if entry.CloudLocation != "" {
|
||||||
fmt.Printf("☁️ Cloud: %s\n", entry.CloudLocation)
|
fmt.Printf("[CLOUD] Cloud: %s\n", entry.CloudLocation)
|
||||||
}
|
}
|
||||||
if entry.RetentionPolicy != "" {
|
if entry.RetentionPolicy != "" {
|
||||||
fmt.Printf("📆 Retention: %s\n", entry.RetentionPolicy)
|
fmt.Printf("📆 Retention: %s\n", entry.RetentionPolicy)
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("\n📊 Verification:\n")
|
fmt.Printf("\n[STATS] Verification:\n")
|
||||||
if entry.VerifiedAt != nil {
|
if entry.VerifiedAt != nil {
|
||||||
status := "❌ Failed"
|
status := "[FAIL] Failed"
|
||||||
if entry.VerifyValid != nil && *entry.VerifyValid {
|
if entry.VerifyValid != nil && *entry.VerifyValid {
|
||||||
status = "✅ Valid"
|
status = "[OK] Valid"
|
||||||
}
|
}
|
||||||
fmt.Printf(" Status: %s (checked %s)\n", status, entry.VerifiedAt.Format("2006-01-02 15:04"))
|
fmt.Printf(" Status: %s (checked %s)\n", status, entry.VerifiedAt.Format("2006-01-02 15:04"))
|
||||||
} else {
|
} else {
|
||||||
fmt.Printf(" Status: ⏳ Not verified\n")
|
fmt.Printf(" Status: [WAIT] Not verified\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("\n🧪 DR Drill Test:\n")
|
fmt.Printf("\n[TEST] DR Drill Test:\n")
|
||||||
if entry.DrillTestedAt != nil {
|
if entry.DrillTestedAt != nil {
|
||||||
status := "❌ Failed"
|
status := "[FAIL] Failed"
|
||||||
if entry.DrillSuccess != nil && *entry.DrillSuccess {
|
if entry.DrillSuccess != nil && *entry.DrillSuccess {
|
||||||
status = "✅ Passed"
|
status = "[OK] Passed"
|
||||||
}
|
}
|
||||||
fmt.Printf(" Status: %s (tested %s)\n", status, entry.DrillTestedAt.Format("2006-01-02 15:04"))
|
fmt.Printf(" Status: %s (tested %s)\n", status, entry.DrillTestedAt.Format("2006-01-02 15:04"))
|
||||||
} else {
|
} else {
|
||||||
fmt.Printf(" Status: ⏳ Not tested\n")
|
fmt.Printf(" Status: [WAIT] Not tested\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(entry.Metadata) > 0 {
|
if len(entry.Metadata) > 0 {
|
||||||
fmt.Printf("\n📝 Additional Metadata:\n")
|
fmt.Printf("\n[NOTE] Additional Metadata:\n")
|
||||||
for k, v := range entry.Metadata {
|
for k, v := range entry.Metadata {
|
||||||
fmt.Printf(" %s: %s\n", k, v)
|
fmt.Printf(" %s: %s\n", k, v)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n")
|
fmt.Printf("\n=====================================================\n")
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -115,7 +115,7 @@ func runCleanup(cmd *cobra.Command, args []string) error {
|
|||||||
DryRun: dryRun,
|
DryRun: dryRun,
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("🗑️ Cleanup Policy:\n")
|
fmt.Printf("[CLEANUP] Cleanup Policy:\n")
|
||||||
fmt.Printf(" Directory: %s\n", backupDir)
|
fmt.Printf(" Directory: %s\n", backupDir)
|
||||||
fmt.Printf(" Retention: %d days\n", policy.RetentionDays)
|
fmt.Printf(" Retention: %d days\n", policy.RetentionDays)
|
||||||
fmt.Printf(" Min backups: %d\n", policy.MinBackups)
|
fmt.Printf(" Min backups: %d\n", policy.MinBackups)
|
||||||
@@ -142,16 +142,16 @@ func runCleanup(cmd *cobra.Command, args []string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Display results
|
// Display results
|
||||||
fmt.Printf("📊 Results:\n")
|
fmt.Printf("[RESULTS] Results:\n")
|
||||||
fmt.Printf(" Total backups: %d\n", result.TotalBackups)
|
fmt.Printf(" Total backups: %d\n", result.TotalBackups)
|
||||||
fmt.Printf(" Eligible for deletion: %d\n", result.EligibleForDeletion)
|
fmt.Printf(" Eligible for deletion: %d\n", result.EligibleForDeletion)
|
||||||
|
|
||||||
if len(result.Deleted) > 0 {
|
if len(result.Deleted) > 0 {
|
||||||
fmt.Printf("\n")
|
fmt.Printf("\n")
|
||||||
if dryRun {
|
if dryRun {
|
||||||
fmt.Printf("🔍 Would delete %d backup(s):\n", len(result.Deleted))
|
fmt.Printf("[DRY-RUN] Would delete %d backup(s):\n", len(result.Deleted))
|
||||||
} else {
|
} else {
|
||||||
fmt.Printf("✅ Deleted %d backup(s):\n", len(result.Deleted))
|
fmt.Printf("[OK] Deleted %d backup(s):\n", len(result.Deleted))
|
||||||
}
|
}
|
||||||
for _, file := range result.Deleted {
|
for _, file := range result.Deleted {
|
||||||
fmt.Printf(" - %s\n", filepath.Base(file))
|
fmt.Printf(" - %s\n", filepath.Base(file))
|
||||||
@@ -159,33 +159,33 @@ func runCleanup(cmd *cobra.Command, args []string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if len(result.Kept) > 0 && len(result.Kept) <= 10 {
|
if len(result.Kept) > 0 && len(result.Kept) <= 10 {
|
||||||
fmt.Printf("\n📦 Kept %d backup(s):\n", len(result.Kept))
|
fmt.Printf("\n[KEPT] Kept %d backup(s):\n", len(result.Kept))
|
||||||
for _, file := range result.Kept {
|
for _, file := range result.Kept {
|
||||||
fmt.Printf(" - %s\n", filepath.Base(file))
|
fmt.Printf(" - %s\n", filepath.Base(file))
|
||||||
}
|
}
|
||||||
} else if len(result.Kept) > 10 {
|
} else if len(result.Kept) > 10 {
|
||||||
fmt.Printf("\n📦 Kept %d backup(s)\n", len(result.Kept))
|
fmt.Printf("\n[KEPT] Kept %d backup(s)\n", len(result.Kept))
|
||||||
}
|
}
|
||||||
|
|
||||||
if !dryRun && result.SpaceFreed > 0 {
|
if !dryRun && result.SpaceFreed > 0 {
|
||||||
fmt.Printf("\n💾 Space freed: %s\n", metadata.FormatSize(result.SpaceFreed))
|
fmt.Printf("\n[FREED] Space freed: %s\n", metadata.FormatSize(result.SpaceFreed))
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(result.Errors) > 0 {
|
if len(result.Errors) > 0 {
|
||||||
fmt.Printf("\n⚠️ Errors:\n")
|
fmt.Printf("\n[WARN] Errors:\n")
|
||||||
for _, err := range result.Errors {
|
for _, err := range result.Errors {
|
||||||
fmt.Printf(" - %v\n", err)
|
fmt.Printf(" - %v\n", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println(strings.Repeat("─", 50))
|
fmt.Println(strings.Repeat("-", 50))
|
||||||
|
|
||||||
if dryRun {
|
if dryRun {
|
||||||
fmt.Println("✅ Dry run completed (no files were deleted)")
|
fmt.Println("[OK] Dry run completed (no files were deleted)")
|
||||||
} else if len(result.Deleted) > 0 {
|
} else if len(result.Deleted) > 0 {
|
||||||
fmt.Println("✅ Cleanup completed successfully")
|
fmt.Println("[OK] Cleanup completed successfully")
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("ℹ️ No backups eligible for deletion")
|
fmt.Println("[INFO] No backups eligible for deletion")
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -212,7 +212,7 @@ func runCloudCleanup(ctx context.Context, uri string) error {
|
|||||||
return fmt.Errorf("invalid cloud URI: %w", err)
|
return fmt.Errorf("invalid cloud URI: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("☁️ Cloud Cleanup Policy:\n")
|
fmt.Printf("[CLOUD] Cloud Cleanup Policy:\n")
|
||||||
fmt.Printf(" URI: %s\n", uri)
|
fmt.Printf(" URI: %s\n", uri)
|
||||||
fmt.Printf(" Provider: %s\n", cloudURI.Provider)
|
fmt.Printf(" Provider: %s\n", cloudURI.Provider)
|
||||||
fmt.Printf(" Bucket: %s\n", cloudURI.Bucket)
|
fmt.Printf(" Bucket: %s\n", cloudURI.Bucket)
|
||||||
@@ -295,7 +295,7 @@ func runCloudCleanup(ctx context.Context, uri string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Display results
|
// Display results
|
||||||
fmt.Printf("📊 Results:\n")
|
fmt.Printf("[RESULTS] Results:\n")
|
||||||
fmt.Printf(" Total backups: %d\n", totalBackups)
|
fmt.Printf(" Total backups: %d\n", totalBackups)
|
||||||
fmt.Printf(" Eligible for deletion: %d\n", len(toDelete))
|
fmt.Printf(" Eligible for deletion: %d\n", len(toDelete))
|
||||||
fmt.Printf(" Will keep: %d\n", len(toKeep))
|
fmt.Printf(" Will keep: %d\n", len(toKeep))
|
||||||
@@ -303,9 +303,9 @@ func runCloudCleanup(ctx context.Context, uri string) error {
|
|||||||
|
|
||||||
if len(toDelete) > 0 {
|
if len(toDelete) > 0 {
|
||||||
if dryRun {
|
if dryRun {
|
||||||
fmt.Printf("🔍 Would delete %d backup(s):\n", len(toDelete))
|
fmt.Printf("[DRY-RUN] Would delete %d backup(s):\n", len(toDelete))
|
||||||
} else {
|
} else {
|
||||||
fmt.Printf("🗑️ Deleting %d backup(s):\n", len(toDelete))
|
fmt.Printf("[DELETE] Deleting %d backup(s):\n", len(toDelete))
|
||||||
}
|
}
|
||||||
|
|
||||||
var totalSize int64
|
var totalSize int64
|
||||||
@@ -321,7 +321,7 @@ func runCloudCleanup(ctx context.Context, uri string) error {
|
|||||||
|
|
||||||
if !dryRun {
|
if !dryRun {
|
||||||
if err := backend.Delete(ctx, backup.Key); err != nil {
|
if err := backend.Delete(ctx, backup.Key); err != nil {
|
||||||
fmt.Printf(" ❌ Error: %v\n", err)
|
fmt.Printf(" [FAIL] Error: %v\n", err)
|
||||||
} else {
|
} else {
|
||||||
deletedCount++
|
deletedCount++
|
||||||
// Also try to delete metadata
|
// Also try to delete metadata
|
||||||
@@ -330,12 +330,12 @@ func runCloudCleanup(ctx context.Context, uri string) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("\n💾 Space %s: %s\n",
|
fmt.Printf("\n[FREED] Space %s: %s\n",
|
||||||
map[bool]string{true: "would be freed", false: "freed"}[dryRun],
|
map[bool]string{true: "would be freed", false: "freed"}[dryRun],
|
||||||
cloud.FormatSize(totalSize))
|
cloud.FormatSize(totalSize))
|
||||||
|
|
||||||
if !dryRun && deletedCount > 0 {
|
if !dryRun && deletedCount > 0 {
|
||||||
fmt.Printf("✅ Successfully deleted %d backup(s)\n", deletedCount)
|
fmt.Printf("[OK] Successfully deleted %d backup(s)\n", deletedCount)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("No backups eligible for deletion")
|
fmt.Println("No backups eligible for deletion")
|
||||||
@@ -405,7 +405,7 @@ func runGFSCleanup(backupDir string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Display tier breakdown
|
// Display tier breakdown
|
||||||
fmt.Printf("📊 Backup Classification:\n")
|
fmt.Printf("[STATS] Backup Classification:\n")
|
||||||
fmt.Printf(" Yearly: %d\n", result.YearlyKept)
|
fmt.Printf(" Yearly: %d\n", result.YearlyKept)
|
||||||
fmt.Printf(" Monthly: %d\n", result.MonthlyKept)
|
fmt.Printf(" Monthly: %d\n", result.MonthlyKept)
|
||||||
fmt.Printf(" Weekly: %d\n", result.WeeklyKept)
|
fmt.Printf(" Weekly: %d\n", result.WeeklyKept)
|
||||||
@@ -416,9 +416,9 @@ func runGFSCleanup(backupDir string) error {
|
|||||||
// Display deletions
|
// Display deletions
|
||||||
if len(result.Deleted) > 0 {
|
if len(result.Deleted) > 0 {
|
||||||
if dryRun {
|
if dryRun {
|
||||||
fmt.Printf("🔍 Would delete %d backup(s):\n", len(result.Deleted))
|
fmt.Printf("[SEARCH] Would delete %d backup(s):\n", len(result.Deleted))
|
||||||
} else {
|
} else {
|
||||||
fmt.Printf("✅ Deleted %d backup(s):\n", len(result.Deleted))
|
fmt.Printf("[OK] Deleted %d backup(s):\n", len(result.Deleted))
|
||||||
}
|
}
|
||||||
for _, file := range result.Deleted {
|
for _, file := range result.Deleted {
|
||||||
fmt.Printf(" - %s\n", filepath.Base(file))
|
fmt.Printf(" - %s\n", filepath.Base(file))
|
||||||
@@ -427,7 +427,7 @@ func runGFSCleanup(backupDir string) error {
|
|||||||
|
|
||||||
// Display kept backups (limited display)
|
// Display kept backups (limited display)
|
||||||
if len(result.Kept) > 0 && len(result.Kept) <= 15 {
|
if len(result.Kept) > 0 && len(result.Kept) <= 15 {
|
||||||
fmt.Printf("\n📦 Kept %d backup(s):\n", len(result.Kept))
|
fmt.Printf("\n[PKG] Kept %d backup(s):\n", len(result.Kept))
|
||||||
for _, file := range result.Kept {
|
for _, file := range result.Kept {
|
||||||
// Show tier classification
|
// Show tier classification
|
||||||
info, _ := os.Stat(file)
|
info, _ := os.Stat(file)
|
||||||
@@ -440,28 +440,28 @@ func runGFSCleanup(backupDir string) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else if len(result.Kept) > 15 {
|
} else if len(result.Kept) > 15 {
|
||||||
fmt.Printf("\n📦 Kept %d backup(s)\n", len(result.Kept))
|
fmt.Printf("\n[PKG] Kept %d backup(s)\n", len(result.Kept))
|
||||||
}
|
}
|
||||||
|
|
||||||
if !dryRun && result.SpaceFreed > 0 {
|
if !dryRun && result.SpaceFreed > 0 {
|
||||||
fmt.Printf("\n💾 Space freed: %s\n", metadata.FormatSize(result.SpaceFreed))
|
fmt.Printf("\n[SAVE] Space freed: %s\n", metadata.FormatSize(result.SpaceFreed))
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(result.Errors) > 0 {
|
if len(result.Errors) > 0 {
|
||||||
fmt.Printf("\n⚠️ Errors:\n")
|
fmt.Printf("\n[WARN] Errors:\n")
|
||||||
for _, err := range result.Errors {
|
for _, err := range result.Errors {
|
||||||
fmt.Printf(" - %v\n", err)
|
fmt.Printf(" - %v\n", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println(strings.Repeat("─", 50))
|
fmt.Println(strings.Repeat("-", 50))
|
||||||
|
|
||||||
if dryRun {
|
if dryRun {
|
||||||
fmt.Println("✅ GFS dry run completed (no files were deleted)")
|
fmt.Println("[OK] GFS dry run completed (no files were deleted)")
|
||||||
} else if len(result.Deleted) > 0 {
|
} else if len(result.Deleted) > 0 {
|
||||||
fmt.Println("✅ GFS cleanup completed successfully")
|
fmt.Println("[OK] GFS cleanup completed successfully")
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("ℹ️ No backups eligible for deletion under GFS policy")
|
fmt.Println("[INFO] No backups eligible for deletion under GFS policy")
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
34
cmd/cloud.go
34
cmd/cloud.go
@@ -189,12 +189,12 @@ func runCloudUpload(cmd *cobra.Command, args []string) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("☁️ Uploading %d file(s) to %s...\n\n", len(files), backend.Name())
|
fmt.Printf("[CLOUD] Uploading %d file(s) to %s...\n\n", len(files), backend.Name())
|
||||||
|
|
||||||
successCount := 0
|
successCount := 0
|
||||||
for _, localPath := range files {
|
for _, localPath := range files {
|
||||||
filename := filepath.Base(localPath)
|
filename := filepath.Base(localPath)
|
||||||
fmt.Printf("📤 %s\n", filename)
|
fmt.Printf("[UPLOAD] %s\n", filename)
|
||||||
|
|
||||||
// Progress callback
|
// Progress callback
|
||||||
var lastPercent int
|
var lastPercent int
|
||||||
@@ -214,21 +214,21 @@ func runCloudUpload(cmd *cobra.Command, args []string) error {
|
|||||||
|
|
||||||
err := backend.Upload(ctx, localPath, filename, progress)
|
err := backend.Upload(ctx, localPath, filename, progress)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf(" ❌ Failed: %v\n\n", err)
|
fmt.Printf(" [FAIL] Failed: %v\n\n", err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get file size
|
// Get file size
|
||||||
if info, err := os.Stat(localPath); err == nil {
|
if info, err := os.Stat(localPath); err == nil {
|
||||||
fmt.Printf(" ✅ Uploaded (%s)\n\n", cloud.FormatSize(info.Size()))
|
fmt.Printf(" [OK] Uploaded (%s)\n\n", cloud.FormatSize(info.Size()))
|
||||||
} else {
|
} else {
|
||||||
fmt.Printf(" ✅ Uploaded\n\n")
|
fmt.Printf(" [OK] Uploaded\n\n")
|
||||||
}
|
}
|
||||||
successCount++
|
successCount++
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println(strings.Repeat("─", 50))
|
fmt.Println(strings.Repeat("-", 50))
|
||||||
fmt.Printf("✅ Successfully uploaded %d/%d file(s)\n", successCount, len(files))
|
fmt.Printf("[OK] Successfully uploaded %d/%d file(s)\n", successCount, len(files))
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -248,8 +248,8 @@ func runCloudDownload(cmd *cobra.Command, args []string) error {
|
|||||||
localPath = filepath.Join(localPath, filepath.Base(remotePath))
|
localPath = filepath.Join(localPath, filepath.Base(remotePath))
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("☁️ Downloading from %s...\n\n", backend.Name())
|
fmt.Printf("[CLOUD] Downloading from %s...\n\n", backend.Name())
|
||||||
fmt.Printf("📥 %s → %s\n", remotePath, localPath)
|
fmt.Printf("[DOWNLOAD] %s -> %s\n", remotePath, localPath)
|
||||||
|
|
||||||
// Progress callback
|
// Progress callback
|
||||||
var lastPercent int
|
var lastPercent int
|
||||||
@@ -274,9 +274,9 @@ func runCloudDownload(cmd *cobra.Command, args []string) error {
|
|||||||
|
|
||||||
// Get file size
|
// Get file size
|
||||||
if info, err := os.Stat(localPath); err == nil {
|
if info, err := os.Stat(localPath); err == nil {
|
||||||
fmt.Printf(" ✅ Downloaded (%s)\n", cloud.FormatSize(info.Size()))
|
fmt.Printf(" [OK] Downloaded (%s)\n", cloud.FormatSize(info.Size()))
|
||||||
} else {
|
} else {
|
||||||
fmt.Printf(" ✅ Downloaded\n")
|
fmt.Printf(" [OK] Downloaded\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -294,7 +294,7 @@ func runCloudList(cmd *cobra.Command, args []string) error {
|
|||||||
prefix = args[0]
|
prefix = args[0]
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("☁️ Listing backups in %s/%s...\n\n", backend.Name(), cloudBucket)
|
fmt.Printf("[CLOUD] Listing backups in %s/%s...\n\n", backend.Name(), cloudBucket)
|
||||||
|
|
||||||
backups, err := backend.List(ctx, prefix)
|
backups, err := backend.List(ctx, prefix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -311,7 +311,7 @@ func runCloudList(cmd *cobra.Command, args []string) error {
|
|||||||
totalSize += backup.Size
|
totalSize += backup.Size
|
||||||
|
|
||||||
if cloudVerbose {
|
if cloudVerbose {
|
||||||
fmt.Printf("📦 %s\n", backup.Name)
|
fmt.Printf("[FILE] %s\n", backup.Name)
|
||||||
fmt.Printf(" Size: %s\n", cloud.FormatSize(backup.Size))
|
fmt.Printf(" Size: %s\n", cloud.FormatSize(backup.Size))
|
||||||
fmt.Printf(" Modified: %s\n", backup.LastModified.Format(time.RFC3339))
|
fmt.Printf(" Modified: %s\n", backup.LastModified.Format(time.RFC3339))
|
||||||
if backup.StorageClass != "" {
|
if backup.StorageClass != "" {
|
||||||
@@ -328,7 +328,7 @@ func runCloudList(cmd *cobra.Command, args []string) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println(strings.Repeat("─", 50))
|
fmt.Println(strings.Repeat("-", 50))
|
||||||
fmt.Printf("Total: %d backup(s), %s\n", len(backups), cloud.FormatSize(totalSize))
|
fmt.Printf("Total: %d backup(s), %s\n", len(backups), cloud.FormatSize(totalSize))
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -360,7 +360,7 @@ func runCloudDelete(cmd *cobra.Command, args []string) error {
|
|||||||
|
|
||||||
// Confirmation prompt
|
// Confirmation prompt
|
||||||
if !cloudConfirm {
|
if !cloudConfirm {
|
||||||
fmt.Printf("⚠️ Delete %s (%s) from cloud storage?\n", remotePath, cloud.FormatSize(size))
|
fmt.Printf("[WARN] Delete %s (%s) from cloud storage?\n", remotePath, cloud.FormatSize(size))
|
||||||
fmt.Print("Type 'yes' to confirm: ")
|
fmt.Print("Type 'yes' to confirm: ")
|
||||||
var response string
|
var response string
|
||||||
fmt.Scanln(&response)
|
fmt.Scanln(&response)
|
||||||
@@ -370,14 +370,14 @@ func runCloudDelete(cmd *cobra.Command, args []string) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("🗑️ Deleting %s...\n", remotePath)
|
fmt.Printf("[DELETE] Deleting %s...\n", remotePath)
|
||||||
|
|
||||||
err = backend.Delete(ctx, remotePath)
|
err = backend.Delete(ctx, remotePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("delete failed: %w", err)
|
return fmt.Errorf("delete failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("✅ Deleted %s (%s)\n", remotePath, cloud.FormatSize(size))
|
fmt.Printf("[OK] Deleted %s (%s)\n", remotePath, cloud.FormatSize(size))
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -61,10 +61,10 @@ func runCPUInfo(ctx context.Context) error {
|
|||||||
|
|
||||||
// Show current vs optimal
|
// Show current vs optimal
|
||||||
if cfg.AutoDetectCores {
|
if cfg.AutoDetectCores {
|
||||||
fmt.Println("\n✅ CPU optimization is enabled")
|
fmt.Println("\n[OK] CPU optimization is enabled")
|
||||||
fmt.Println("Job counts are automatically optimized based on detected hardware")
|
fmt.Println("Job counts are automatically optimized based on detected hardware")
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("\n⚠️ CPU optimization is disabled")
|
fmt.Println("\n[WARN] CPU optimization is disabled")
|
||||||
fmt.Println("Consider enabling --auto-detect-cores for better performance")
|
fmt.Println("Consider enabling --auto-detect-cores for better performance")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
579
cmd/dedup.go
Normal file
579
cmd/dedup.go
Normal file
@@ -0,0 +1,579 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"dbbackup/internal/dedup"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
)
|
||||||
|
|
||||||
|
var dedupCmd = &cobra.Command{
|
||||||
|
Use: "dedup",
|
||||||
|
Short: "Deduplicated backup operations",
|
||||||
|
Long: `Content-defined chunking deduplication for space-efficient backups.
|
||||||
|
|
||||||
|
Similar to restic/borgbackup but with native database dump support.
|
||||||
|
|
||||||
|
Features:
|
||||||
|
- Content-defined chunking (CDC) with Buzhash rolling hash
|
||||||
|
- SHA-256 content-addressed storage
|
||||||
|
- AES-256-GCM encryption (optional)
|
||||||
|
- Gzip compression (optional)
|
||||||
|
- SQLite index for fast lookups
|
||||||
|
|
||||||
|
Storage Structure:
|
||||||
|
<dedup-dir>/
|
||||||
|
chunks/ # Content-addressed chunk files
|
||||||
|
ab/cdef... # Sharded by first 2 chars of hash
|
||||||
|
manifests/ # JSON manifest per backup
|
||||||
|
chunks.db # SQLite index`,
|
||||||
|
}
|
||||||
|
|
||||||
|
var dedupBackupCmd = &cobra.Command{
|
||||||
|
Use: "backup <file>",
|
||||||
|
Short: "Create a deduplicated backup of a file",
|
||||||
|
Long: `Chunk a file using content-defined chunking and store deduplicated chunks.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
dbbackup dedup backup /path/to/database.dump
|
||||||
|
dbbackup dedup backup mydb.sql --compress --encrypt`,
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: runDedupBackup,
|
||||||
|
}
|
||||||
|
|
||||||
|
var dedupRestoreCmd = &cobra.Command{
|
||||||
|
Use: "restore <manifest-id> <output-file>",
|
||||||
|
Short: "Restore a backup from its manifest",
|
||||||
|
Long: `Reconstruct a file from its deduplicated chunks.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
dbbackup dedup restore 2026-01-07_120000_mydb /tmp/restored.dump
|
||||||
|
dbbackup dedup list # to see available manifests`,
|
||||||
|
Args: cobra.ExactArgs(2),
|
||||||
|
RunE: runDedupRestore,
|
||||||
|
}
|
||||||
|
|
||||||
|
var dedupListCmd = &cobra.Command{
|
||||||
|
Use: "list",
|
||||||
|
Short: "List all deduplicated backups",
|
||||||
|
RunE: runDedupList,
|
||||||
|
}
|
||||||
|
|
||||||
|
var dedupStatsCmd = &cobra.Command{
|
||||||
|
Use: "stats",
|
||||||
|
Short: "Show deduplication statistics",
|
||||||
|
RunE: runDedupStats,
|
||||||
|
}
|
||||||
|
|
||||||
|
var dedupGCCmd = &cobra.Command{
|
||||||
|
Use: "gc",
|
||||||
|
Short: "Garbage collect unreferenced chunks",
|
||||||
|
Long: `Remove chunks that are no longer referenced by any manifest.
|
||||||
|
|
||||||
|
Run after deleting old backups to reclaim space.`,
|
||||||
|
RunE: runDedupGC,
|
||||||
|
}
|
||||||
|
|
||||||
|
var dedupDeleteCmd = &cobra.Command{
|
||||||
|
Use: "delete <manifest-id>",
|
||||||
|
Short: "Delete a backup manifest (chunks cleaned by gc)",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: runDedupDelete,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flags
|
||||||
|
var (
|
||||||
|
dedupDir string
|
||||||
|
dedupCompress bool
|
||||||
|
dedupEncrypt bool
|
||||||
|
dedupKey string
|
||||||
|
dedupName string
|
||||||
|
dedupDBType string
|
||||||
|
dedupDBName string
|
||||||
|
dedupDBHost string
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
rootCmd.AddCommand(dedupCmd)
|
||||||
|
dedupCmd.AddCommand(dedupBackupCmd)
|
||||||
|
dedupCmd.AddCommand(dedupRestoreCmd)
|
||||||
|
dedupCmd.AddCommand(dedupListCmd)
|
||||||
|
dedupCmd.AddCommand(dedupStatsCmd)
|
||||||
|
dedupCmd.AddCommand(dedupGCCmd)
|
||||||
|
dedupCmd.AddCommand(dedupDeleteCmd)
|
||||||
|
|
||||||
|
// Global dedup flags
|
||||||
|
dedupCmd.PersistentFlags().StringVar(&dedupDir, "dedup-dir", "", "Dedup storage directory (default: $BACKUP_DIR/dedup)")
|
||||||
|
dedupCmd.PersistentFlags().BoolVar(&dedupCompress, "compress", true, "Compress chunks with gzip")
|
||||||
|
dedupCmd.PersistentFlags().BoolVar(&dedupEncrypt, "encrypt", false, "Encrypt chunks with AES-256-GCM")
|
||||||
|
dedupCmd.PersistentFlags().StringVar(&dedupKey, "key", "", "Encryption key (hex) or use DBBACKUP_DEDUP_KEY env")
|
||||||
|
|
||||||
|
// Backup-specific flags
|
||||||
|
dedupBackupCmd.Flags().StringVar(&dedupName, "name", "", "Optional backup name")
|
||||||
|
dedupBackupCmd.Flags().StringVar(&dedupDBType, "db-type", "", "Database type (postgres/mysql)")
|
||||||
|
dedupBackupCmd.Flags().StringVar(&dedupDBName, "db-name", "", "Database name")
|
||||||
|
dedupBackupCmd.Flags().StringVar(&dedupDBHost, "db-host", "", "Database host")
|
||||||
|
}
|
||||||
|
|
||||||
|
func getDedupDir() string {
|
||||||
|
if dedupDir != "" {
|
||||||
|
return dedupDir
|
||||||
|
}
|
||||||
|
if cfg != nil && cfg.BackupDir != "" {
|
||||||
|
return filepath.Join(cfg.BackupDir, "dedup")
|
||||||
|
}
|
||||||
|
return filepath.Join(os.Getenv("HOME"), "db_backups", "dedup")
|
||||||
|
}
|
||||||
|
|
||||||
|
func getEncryptionKey() string {
|
||||||
|
if dedupKey != "" {
|
||||||
|
return dedupKey
|
||||||
|
}
|
||||||
|
return os.Getenv("DBBACKUP_DEDUP_KEY")
|
||||||
|
}
|
||||||
|
|
||||||
|
func runDedupBackup(cmd *cobra.Command, args []string) error {
|
||||||
|
inputPath := args[0]
|
||||||
|
|
||||||
|
// Open input file
|
||||||
|
file, err := os.Open(inputPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open input file: %w", err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
info, err := file.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to stat input file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Setup dedup storage
|
||||||
|
basePath := getDedupDir()
|
||||||
|
encKey := ""
|
||||||
|
if dedupEncrypt {
|
||||||
|
encKey = getEncryptionKey()
|
||||||
|
if encKey == "" {
|
||||||
|
return fmt.Errorf("encryption enabled but no key provided (use --key or DBBACKUP_DEDUP_KEY)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
store, err := dedup.NewChunkStore(dedup.StoreConfig{
|
||||||
|
BasePath: basePath,
|
||||||
|
Compress: dedupCompress,
|
||||||
|
EncryptionKey: encKey,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open chunk store: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
manifestStore, err := dedup.NewManifestStore(basePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open manifest store: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
index, err := dedup.NewChunkIndex(basePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open chunk index: %w", err)
|
||||||
|
}
|
||||||
|
defer index.Close()
|
||||||
|
|
||||||
|
// Generate manifest ID
|
||||||
|
now := time.Now()
|
||||||
|
manifestID := now.Format("2006-01-02_150405")
|
||||||
|
if dedupDBName != "" {
|
||||||
|
manifestID += "_" + dedupDBName
|
||||||
|
} else {
|
||||||
|
base := filepath.Base(inputPath)
|
||||||
|
ext := filepath.Ext(base)
|
||||||
|
manifestID += "_" + strings.TrimSuffix(base, ext)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("Creating deduplicated backup: %s\n", manifestID)
|
||||||
|
fmt.Printf("Input: %s (%s)\n", inputPath, formatBytes(info.Size()))
|
||||||
|
fmt.Printf("Store: %s\n", basePath)
|
||||||
|
|
||||||
|
// Hash the entire file for verification
|
||||||
|
file.Seek(0, 0)
|
||||||
|
h := sha256.New()
|
||||||
|
io.Copy(h, file)
|
||||||
|
fileHash := hex.EncodeToString(h.Sum(nil))
|
||||||
|
file.Seek(0, 0)
|
||||||
|
|
||||||
|
// Chunk the file
|
||||||
|
chunker := dedup.NewChunker(file, dedup.DefaultChunkerConfig())
|
||||||
|
var chunks []dedup.ChunkRef
|
||||||
|
var totalSize, storedSize int64
|
||||||
|
var chunkCount, newChunks int
|
||||||
|
|
||||||
|
startTime := time.Now()
|
||||||
|
|
||||||
|
for {
|
||||||
|
chunk, err := chunker.Next()
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("chunking failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
chunkCount++
|
||||||
|
totalSize += int64(chunk.Length)
|
||||||
|
|
||||||
|
// Store chunk (deduplication happens here)
|
||||||
|
isNew, err := store.Put(chunk)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to store chunk: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if isNew {
|
||||||
|
newChunks++
|
||||||
|
storedSize += int64(chunk.Length)
|
||||||
|
// Record in index
|
||||||
|
index.AddChunk(chunk.Hash, chunk.Length, chunk.Length)
|
||||||
|
}
|
||||||
|
|
||||||
|
chunks = append(chunks, dedup.ChunkRef{
|
||||||
|
Hash: chunk.Hash,
|
||||||
|
Offset: chunk.Offset,
|
||||||
|
Length: chunk.Length,
|
||||||
|
})
|
||||||
|
|
||||||
|
// Progress
|
||||||
|
if chunkCount%1000 == 0 {
|
||||||
|
fmt.Printf("\r Processed %d chunks, %d new...", chunkCount, newChunks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
duration := time.Since(startTime)
|
||||||
|
|
||||||
|
// Calculate dedup ratio
|
||||||
|
dedupRatio := 0.0
|
||||||
|
if totalSize > 0 {
|
||||||
|
dedupRatio = 1.0 - float64(storedSize)/float64(totalSize)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create manifest
|
||||||
|
manifest := &dedup.Manifest{
|
||||||
|
ID: manifestID,
|
||||||
|
Name: dedupName,
|
||||||
|
CreatedAt: now,
|
||||||
|
DatabaseType: dedupDBType,
|
||||||
|
DatabaseName: dedupDBName,
|
||||||
|
DatabaseHost: dedupDBHost,
|
||||||
|
Chunks: chunks,
|
||||||
|
OriginalSize: totalSize,
|
||||||
|
StoredSize: storedSize,
|
||||||
|
ChunkCount: chunkCount,
|
||||||
|
NewChunks: newChunks,
|
||||||
|
DedupRatio: dedupRatio,
|
||||||
|
Encrypted: dedupEncrypt,
|
||||||
|
Compressed: dedupCompress,
|
||||||
|
SHA256: fileHash,
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := manifestStore.Save(manifest); err != nil {
|
||||||
|
return fmt.Errorf("failed to save manifest: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := index.AddManifest(manifest); err != nil {
|
||||||
|
log.Warn("Failed to index manifest", "error", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("\r \r")
|
||||||
|
fmt.Printf("\nBackup complete!\n")
|
||||||
|
fmt.Printf(" Manifest: %s\n", manifestID)
|
||||||
|
fmt.Printf(" Chunks: %d total, %d new\n", chunkCount, newChunks)
|
||||||
|
fmt.Printf(" Original: %s\n", formatBytes(totalSize))
|
||||||
|
fmt.Printf(" Stored: %s (new data)\n", formatBytes(storedSize))
|
||||||
|
fmt.Printf(" Dedup ratio: %.1f%%\n", dedupRatio*100)
|
||||||
|
fmt.Printf(" Duration: %s\n", duration.Round(time.Millisecond))
|
||||||
|
fmt.Printf(" Throughput: %s/s\n", formatBytes(int64(float64(totalSize)/duration.Seconds())))
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func runDedupRestore(cmd *cobra.Command, args []string) error {
|
||||||
|
manifestID := args[0]
|
||||||
|
outputPath := args[1]
|
||||||
|
|
||||||
|
basePath := getDedupDir()
|
||||||
|
encKey := ""
|
||||||
|
if dedupEncrypt {
|
||||||
|
encKey = getEncryptionKey()
|
||||||
|
}
|
||||||
|
|
||||||
|
store, err := dedup.NewChunkStore(dedup.StoreConfig{
|
||||||
|
BasePath: basePath,
|
||||||
|
Compress: dedupCompress,
|
||||||
|
EncryptionKey: encKey,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open chunk store: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
manifestStore, err := dedup.NewManifestStore(basePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open manifest store: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
manifest, err := manifestStore.Load(manifestID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to load manifest: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("Restoring backup: %s\n", manifestID)
|
||||||
|
fmt.Printf(" Created: %s\n", manifest.CreatedAt.Format(time.RFC3339))
|
||||||
|
fmt.Printf(" Size: %s\n", formatBytes(manifest.OriginalSize))
|
||||||
|
fmt.Printf(" Chunks: %d\n", manifest.ChunkCount)
|
||||||
|
|
||||||
|
// Create output file
|
||||||
|
outFile, err := os.Create(outputPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create output file: %w", err)
|
||||||
|
}
|
||||||
|
defer outFile.Close()
|
||||||
|
|
||||||
|
h := sha256.New()
|
||||||
|
writer := io.MultiWriter(outFile, h)
|
||||||
|
|
||||||
|
startTime := time.Now()
|
||||||
|
|
||||||
|
for i, ref := range manifest.Chunks {
|
||||||
|
chunk, err := store.Get(ref.Hash)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to read chunk %d (%s): %w", i, ref.Hash[:8], err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := writer.Write(chunk.Data); err != nil {
|
||||||
|
return fmt.Errorf("failed to write chunk %d: %w", i, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (i+1)%1000 == 0 {
|
||||||
|
fmt.Printf("\r Restored %d/%d chunks...", i+1, manifest.ChunkCount)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
duration := time.Since(startTime)
|
||||||
|
restoredHash := hex.EncodeToString(h.Sum(nil))
|
||||||
|
|
||||||
|
fmt.Printf("\r \r")
|
||||||
|
fmt.Printf("\nRestore complete!\n")
|
||||||
|
fmt.Printf(" Output: %s\n", outputPath)
|
||||||
|
fmt.Printf(" Duration: %s\n", duration.Round(time.Millisecond))
|
||||||
|
|
||||||
|
// Verify hash
|
||||||
|
if manifest.SHA256 != "" {
|
||||||
|
if restoredHash == manifest.SHA256 {
|
||||||
|
fmt.Printf(" Verification: [OK] SHA-256 matches\n")
|
||||||
|
} else {
|
||||||
|
fmt.Printf(" Verification: [FAIL] SHA-256 MISMATCH!\n")
|
||||||
|
fmt.Printf(" Expected: %s\n", manifest.SHA256)
|
||||||
|
fmt.Printf(" Got: %s\n", restoredHash)
|
||||||
|
return fmt.Errorf("integrity verification failed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func runDedupList(cmd *cobra.Command, args []string) error {
|
||||||
|
basePath := getDedupDir()
|
||||||
|
|
||||||
|
manifestStore, err := dedup.NewManifestStore(basePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open manifest store: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
manifests, err := manifestStore.ListAll()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to list manifests: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(manifests) == 0 {
|
||||||
|
fmt.Println("No deduplicated backups found.")
|
||||||
|
fmt.Printf("Store: %s\n", basePath)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("Deduplicated Backups (%s)\n\n", basePath)
|
||||||
|
fmt.Printf("%-30s %-12s %-10s %-10s %s\n", "ID", "SIZE", "DEDUP", "CHUNKS", "CREATED")
|
||||||
|
fmt.Println(strings.Repeat("-", 80))
|
||||||
|
|
||||||
|
for _, m := range manifests {
|
||||||
|
fmt.Printf("%-30s %-12s %-10.1f%% %-10d %s\n",
|
||||||
|
truncateStr(m.ID, 30),
|
||||||
|
formatBytes(m.OriginalSize),
|
||||||
|
m.DedupRatio*100,
|
||||||
|
m.ChunkCount,
|
||||||
|
m.CreatedAt.Format("2006-01-02 15:04"),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func runDedupStats(cmd *cobra.Command, args []string) error {
|
||||||
|
basePath := getDedupDir()
|
||||||
|
|
||||||
|
index, err := dedup.NewChunkIndex(basePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open chunk index: %w", err)
|
||||||
|
}
|
||||||
|
defer index.Close()
|
||||||
|
|
||||||
|
stats, err := index.Stats()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to get stats: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
store, err := dedup.NewChunkStore(dedup.StoreConfig{BasePath: basePath})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open chunk store: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
storeStats, err := store.Stats()
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("Failed to get store stats", "error", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("Deduplication Statistics\n")
|
||||||
|
fmt.Printf("========================\n\n")
|
||||||
|
fmt.Printf("Store: %s\n", basePath)
|
||||||
|
fmt.Printf("Manifests: %d\n", stats.TotalManifests)
|
||||||
|
fmt.Printf("Unique chunks: %d\n", stats.TotalChunks)
|
||||||
|
fmt.Printf("Total raw size: %s\n", formatBytes(stats.TotalSizeRaw))
|
||||||
|
fmt.Printf("Stored size: %s\n", formatBytes(stats.TotalSizeStored))
|
||||||
|
fmt.Printf("Dedup ratio: %.1f%%\n", stats.DedupRatio*100)
|
||||||
|
fmt.Printf("Space saved: %s\n", formatBytes(stats.TotalSizeRaw-stats.TotalSizeStored))
|
||||||
|
|
||||||
|
if storeStats != nil {
|
||||||
|
fmt.Printf("Disk usage: %s\n", formatBytes(storeStats.TotalSize))
|
||||||
|
fmt.Printf("Directories: %d\n", storeStats.Directories)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func runDedupGC(cmd *cobra.Command, args []string) error {
|
||||||
|
basePath := getDedupDir()
|
||||||
|
|
||||||
|
index, err := dedup.NewChunkIndex(basePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open chunk index: %w", err)
|
||||||
|
}
|
||||||
|
defer index.Close()
|
||||||
|
|
||||||
|
store, err := dedup.NewChunkStore(dedup.StoreConfig{
|
||||||
|
BasePath: basePath,
|
||||||
|
Compress: dedupCompress,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open chunk store: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find orphaned chunks
|
||||||
|
orphans, err := index.ListOrphanedChunks()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to find orphaned chunks: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(orphans) == 0 {
|
||||||
|
fmt.Println("No orphaned chunks to clean up.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("Found %d orphaned chunks\n", len(orphans))
|
||||||
|
|
||||||
|
var freed int64
|
||||||
|
for _, hash := range orphans {
|
||||||
|
if meta, _ := index.GetChunk(hash); meta != nil {
|
||||||
|
freed += meta.SizeStored
|
||||||
|
}
|
||||||
|
if err := store.Delete(hash); err != nil {
|
||||||
|
log.Warn("Failed to delete chunk", "hash", hash[:8], "error", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := index.RemoveChunk(hash); err != nil {
|
||||||
|
log.Warn("Failed to remove chunk from index", "hash", hash[:8], "error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("Deleted %d chunks, freed %s\n", len(orphans), formatBytes(freed))
|
||||||
|
|
||||||
|
// Vacuum the index
|
||||||
|
if err := index.Vacuum(); err != nil {
|
||||||
|
log.Warn("Failed to vacuum index", "error", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func runDedupDelete(cmd *cobra.Command, args []string) error {
|
||||||
|
manifestID := args[0]
|
||||||
|
basePath := getDedupDir()
|
||||||
|
|
||||||
|
manifestStore, err := dedup.NewManifestStore(basePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open manifest store: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
index, err := dedup.NewChunkIndex(basePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open chunk index: %w", err)
|
||||||
|
}
|
||||||
|
defer index.Close()
|
||||||
|
|
||||||
|
// Load manifest to decrement chunk refs
|
||||||
|
manifest, err := manifestStore.Load(manifestID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to load manifest: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decrement reference counts
|
||||||
|
for _, ref := range manifest.Chunks {
|
||||||
|
index.DecrementRef(ref.Hash)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete manifest
|
||||||
|
if err := manifestStore.Delete(manifestID); err != nil {
|
||||||
|
return fmt.Errorf("failed to delete manifest: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := index.RemoveManifest(manifestID); err != nil {
|
||||||
|
log.Warn("Failed to remove manifest from index", "error", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("Deleted backup: %s\n", manifestID)
|
||||||
|
fmt.Println("Run 'dbbackup dedup gc' to reclaim space from unreferenced chunks.")
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper functions
|
||||||
|
func formatBytes(b int64) string {
|
||||||
|
const unit = 1024
|
||||||
|
if b < unit {
|
||||||
|
return fmt.Sprintf("%d B", b)
|
||||||
|
}
|
||||||
|
div, exp := int64(unit), 0
|
||||||
|
for n := b / unit; n >= unit; n /= unit {
|
||||||
|
div *= unit
|
||||||
|
exp++
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
|
||||||
|
}
|
||||||
|
|
||||||
|
func truncateStr(s string, max int) string {
|
||||||
|
if len(s) <= max {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return s[:max-3] + "..."
|
||||||
|
}
|
||||||
58
cmd/drill.go
58
cmd/drill.go
@@ -318,7 +318,7 @@ func runDrillList(cmd *cobra.Command, args []string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("%-15s %-40s %-20s %s\n", "ID", "NAME", "IMAGE", "STATUS")
|
fmt.Printf("%-15s %-40s %-20s %s\n", "ID", "NAME", "IMAGE", "STATUS")
|
||||||
fmt.Println(strings.Repeat("─", 100))
|
fmt.Println(strings.Repeat("-", 100))
|
||||||
|
|
||||||
for _, c := range containers {
|
for _, c := range containers {
|
||||||
fmt.Printf("%-15s %-40s %-20s %s\n",
|
fmt.Printf("%-15s %-40s %-20s %s\n",
|
||||||
@@ -345,7 +345,7 @@ func runDrillCleanup(cmd *cobra.Command, args []string) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println("✅ Cleanup completed")
|
fmt.Println("[OK] Cleanup completed")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -369,32 +369,32 @@ func runDrillReport(cmd *cobra.Command, args []string) error {
|
|||||||
|
|
||||||
func printDrillResult(result *drill.DrillResult) {
|
func printDrillResult(result *drill.DrillResult) {
|
||||||
fmt.Printf("\n")
|
fmt.Printf("\n")
|
||||||
fmt.Printf("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n")
|
fmt.Printf("=====================================================\n")
|
||||||
fmt.Printf(" DR Drill Report: %s\n", result.DrillID)
|
fmt.Printf(" DR Drill Report: %s\n", result.DrillID)
|
||||||
fmt.Printf("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n")
|
fmt.Printf("=====================================================\n\n")
|
||||||
|
|
||||||
status := "✅ PASSED"
|
status := "[OK] PASSED"
|
||||||
if !result.Success {
|
if !result.Success {
|
||||||
status = "❌ FAILED"
|
status = "[FAIL] FAILED"
|
||||||
} else if result.Status == drill.StatusPartial {
|
} else if result.Status == drill.StatusPartial {
|
||||||
status = "⚠️ PARTIAL"
|
status = "[WARN] PARTIAL"
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("📋 Status: %s\n", status)
|
fmt.Printf("[LOG] Status: %s\n", status)
|
||||||
fmt.Printf("💾 Backup: %s\n", filepath.Base(result.BackupPath))
|
fmt.Printf("[SAVE] Backup: %s\n", filepath.Base(result.BackupPath))
|
||||||
fmt.Printf("🗄️ Database: %s (%s)\n", result.DatabaseName, result.DatabaseType)
|
fmt.Printf("[DB] Database: %s (%s)\n", result.DatabaseName, result.DatabaseType)
|
||||||
fmt.Printf("⏱️ Duration: %.2fs\n", result.Duration)
|
fmt.Printf("[TIME] Duration: %.2fs\n", result.Duration)
|
||||||
fmt.Printf("📅 Started: %s\n", result.StartTime.Format(time.RFC3339))
|
fmt.Printf("📅 Started: %s\n", result.StartTime.Format(time.RFC3339))
|
||||||
fmt.Printf("\n")
|
fmt.Printf("\n")
|
||||||
|
|
||||||
// Phases
|
// Phases
|
||||||
fmt.Printf("📊 Phases:\n")
|
fmt.Printf("[STATS] Phases:\n")
|
||||||
for _, phase := range result.Phases {
|
for _, phase := range result.Phases {
|
||||||
icon := "✅"
|
icon := "[OK]"
|
||||||
if phase.Status == "failed" {
|
if phase.Status == "failed" {
|
||||||
icon = "❌"
|
icon = "[FAIL]"
|
||||||
} else if phase.Status == "running" {
|
} else if phase.Status == "running" {
|
||||||
icon = "🔄"
|
icon = "[SYNC]"
|
||||||
}
|
}
|
||||||
fmt.Printf(" %s %-20s (%.2fs) %s\n", icon, phase.Name, phase.Duration, phase.Message)
|
fmt.Printf(" %s %-20s (%.2fs) %s\n", icon, phase.Name, phase.Duration, phase.Message)
|
||||||
}
|
}
|
||||||
@@ -412,10 +412,10 @@ func printDrillResult(result *drill.DrillResult) {
|
|||||||
fmt.Printf("\n")
|
fmt.Printf("\n")
|
||||||
|
|
||||||
// RTO
|
// RTO
|
||||||
fmt.Printf("⏱️ RTO Analysis:\n")
|
fmt.Printf("[TIME] RTO Analysis:\n")
|
||||||
rtoIcon := "✅"
|
rtoIcon := "[OK]"
|
||||||
if !result.RTOMet {
|
if !result.RTOMet {
|
||||||
rtoIcon = "❌"
|
rtoIcon = "[FAIL]"
|
||||||
}
|
}
|
||||||
fmt.Printf(" Actual RTO: %.2fs\n", result.ActualRTO)
|
fmt.Printf(" Actual RTO: %.2fs\n", result.ActualRTO)
|
||||||
fmt.Printf(" Target RTO: %.0fs\n", result.TargetRTO)
|
fmt.Printf(" Target RTO: %.0fs\n", result.TargetRTO)
|
||||||
@@ -424,11 +424,11 @@ func printDrillResult(result *drill.DrillResult) {
|
|||||||
|
|
||||||
// Validation results
|
// Validation results
|
||||||
if len(result.ValidationResults) > 0 {
|
if len(result.ValidationResults) > 0 {
|
||||||
fmt.Printf("🔍 Validation Queries:\n")
|
fmt.Printf("[SEARCH] Validation Queries:\n")
|
||||||
for _, vr := range result.ValidationResults {
|
for _, vr := range result.ValidationResults {
|
||||||
icon := "✅"
|
icon := "[OK]"
|
||||||
if !vr.Success {
|
if !vr.Success {
|
||||||
icon = "❌"
|
icon = "[FAIL]"
|
||||||
}
|
}
|
||||||
fmt.Printf(" %s %s: %s\n", icon, vr.Name, vr.Result)
|
fmt.Printf(" %s %s: %s\n", icon, vr.Name, vr.Result)
|
||||||
if vr.Error != "" {
|
if vr.Error != "" {
|
||||||
@@ -440,11 +440,11 @@ func printDrillResult(result *drill.DrillResult) {
|
|||||||
|
|
||||||
// Check results
|
// Check results
|
||||||
if len(result.CheckResults) > 0 {
|
if len(result.CheckResults) > 0 {
|
||||||
fmt.Printf("✓ Checks:\n")
|
fmt.Printf("[OK] Checks:\n")
|
||||||
for _, cr := range result.CheckResults {
|
for _, cr := range result.CheckResults {
|
||||||
icon := "✅"
|
icon := "[OK]"
|
||||||
if !cr.Success {
|
if !cr.Success {
|
||||||
icon = "❌"
|
icon = "[FAIL]"
|
||||||
}
|
}
|
||||||
fmt.Printf(" %s %s\n", icon, cr.Message)
|
fmt.Printf(" %s %s\n", icon, cr.Message)
|
||||||
}
|
}
|
||||||
@@ -453,7 +453,7 @@ func printDrillResult(result *drill.DrillResult) {
|
|||||||
|
|
||||||
// Errors and warnings
|
// Errors and warnings
|
||||||
if len(result.Errors) > 0 {
|
if len(result.Errors) > 0 {
|
||||||
fmt.Printf("❌ Errors:\n")
|
fmt.Printf("[FAIL] Errors:\n")
|
||||||
for _, e := range result.Errors {
|
for _, e := range result.Errors {
|
||||||
fmt.Printf(" • %s\n", e)
|
fmt.Printf(" • %s\n", e)
|
||||||
}
|
}
|
||||||
@@ -461,7 +461,7 @@ func printDrillResult(result *drill.DrillResult) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if len(result.Warnings) > 0 {
|
if len(result.Warnings) > 0 {
|
||||||
fmt.Printf("⚠️ Warnings:\n")
|
fmt.Printf("[WARN] Warnings:\n")
|
||||||
for _, w := range result.Warnings {
|
for _, w := range result.Warnings {
|
||||||
fmt.Printf(" • %s\n", w)
|
fmt.Printf(" • %s\n", w)
|
||||||
}
|
}
|
||||||
@@ -470,14 +470,14 @@ func printDrillResult(result *drill.DrillResult) {
|
|||||||
|
|
||||||
// Container info
|
// Container info
|
||||||
if result.ContainerKept {
|
if result.ContainerKept {
|
||||||
fmt.Printf("📦 Container kept: %s\n", result.ContainerID[:12])
|
fmt.Printf("[PKG] Container kept: %s\n", result.ContainerID[:12])
|
||||||
fmt.Printf(" Connect with: docker exec -it %s bash\n", result.ContainerID[:12])
|
fmt.Printf(" Connect with: docker exec -it %s bash\n", result.ContainerID[:12])
|
||||||
fmt.Printf("\n")
|
fmt.Printf("\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n")
|
fmt.Printf("=====================================================\n")
|
||||||
fmt.Printf(" %s\n", result.Message)
|
fmt.Printf(" %s\n", result.Message)
|
||||||
fmt.Printf("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n")
|
fmt.Printf("=====================================================\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
func updateCatalogWithDrillResult(ctx context.Context, backupPath string, result *drill.DrillResult) {
|
func updateCatalogWithDrillResult(ctx context.Context, backupPath string, result *drill.DrillResult) {
|
||||||
|
|||||||
@@ -63,9 +63,9 @@ func runEngineList(cmd *cobra.Command, args []string) error {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
status := "✓ Available"
|
status := "[Y] Available"
|
||||||
if !avail.Available {
|
if !avail.Available {
|
||||||
status = "✗ Not available"
|
status = "[N] Not available"
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("\n%s (%s)\n", info.Name, info.Description)
|
fmt.Printf("\n%s (%s)\n", info.Name, info.Description)
|
||||||
|
|||||||
239
cmd/install.go
Normal file
239
cmd/install.go
Normal file
@@ -0,0 +1,239 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"os/signal"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
"dbbackup/internal/installer"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
// Install flags
|
||||||
|
installInstance string
|
||||||
|
installSchedule string
|
||||||
|
installBackupType string
|
||||||
|
installUser string
|
||||||
|
installGroup string
|
||||||
|
installBackupDir string
|
||||||
|
installConfigPath string
|
||||||
|
installTimeout int
|
||||||
|
installWithMetrics bool
|
||||||
|
installMetricsPort int
|
||||||
|
installDryRun bool
|
||||||
|
installStatus bool
|
||||||
|
|
||||||
|
// Uninstall flags
|
||||||
|
uninstallPurge bool
|
||||||
|
)
|
||||||
|
|
||||||
|
// installCmd represents the install command
|
||||||
|
var installCmd = &cobra.Command{
|
||||||
|
Use: "install",
|
||||||
|
Short: "Install dbbackup as a systemd service",
|
||||||
|
Long: `Install dbbackup as a systemd service with automatic scheduling.
|
||||||
|
|
||||||
|
This command creates systemd service and timer units for automated database backups.
|
||||||
|
It supports both single database and cluster backup modes.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
# Interactive installation (will prompt for options)
|
||||||
|
sudo dbbackup install
|
||||||
|
|
||||||
|
# Install cluster backup running daily at 2am
|
||||||
|
sudo dbbackup install --backup-type cluster --schedule "daily"
|
||||||
|
|
||||||
|
# Install single database backup with custom schedule
|
||||||
|
sudo dbbackup install --instance production --backup-type single --schedule "*-*-* 03:00:00"
|
||||||
|
|
||||||
|
# Install with Prometheus metrics exporter
|
||||||
|
sudo dbbackup install --with-metrics --metrics-port 9399
|
||||||
|
|
||||||
|
# Check installation status
|
||||||
|
dbbackup install --status
|
||||||
|
|
||||||
|
# Dry-run to see what would be installed
|
||||||
|
sudo dbbackup install --dry-run
|
||||||
|
|
||||||
|
Schedule format (OnCalendar):
|
||||||
|
daily - Every day at midnight
|
||||||
|
weekly - Every Monday at midnight
|
||||||
|
*-*-* 02:00:00 - Every day at 2am
|
||||||
|
*-*-* 02,14:00 - Twice daily at 2am and 2pm
|
||||||
|
Mon *-*-* 03:00 - Every Monday at 3am
|
||||||
|
`,
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
// Handle --status flag
|
||||||
|
if installStatus {
|
||||||
|
return runInstallStatus(cmd.Context())
|
||||||
|
}
|
||||||
|
|
||||||
|
return runInstall(cmd.Context())
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// uninstallCmd represents the uninstall command
|
||||||
|
var uninstallCmd = &cobra.Command{
|
||||||
|
Use: "uninstall [instance]",
|
||||||
|
Short: "Uninstall dbbackup systemd service",
|
||||||
|
Long: `Uninstall dbbackup systemd service and timer.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
# Uninstall default instance
|
||||||
|
sudo dbbackup uninstall
|
||||||
|
|
||||||
|
# Uninstall specific instance
|
||||||
|
sudo dbbackup uninstall production
|
||||||
|
|
||||||
|
# Uninstall and remove all configuration
|
||||||
|
sudo dbbackup uninstall --purge
|
||||||
|
`,
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
instance := "cluster"
|
||||||
|
if len(args) > 0 {
|
||||||
|
instance = args[0]
|
||||||
|
}
|
||||||
|
return runUninstall(cmd.Context(), instance)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
rootCmd.AddCommand(installCmd)
|
||||||
|
rootCmd.AddCommand(uninstallCmd)
|
||||||
|
|
||||||
|
// Install flags
|
||||||
|
installCmd.Flags().StringVarP(&installInstance, "instance", "i", "", "Instance name (e.g., production, staging)")
|
||||||
|
installCmd.Flags().StringVarP(&installSchedule, "schedule", "s", "daily", "Backup schedule (OnCalendar format)")
|
||||||
|
installCmd.Flags().StringVarP(&installBackupType, "backup-type", "t", "cluster", "Backup type: single or cluster")
|
||||||
|
installCmd.Flags().StringVar(&installUser, "user", "dbbackup", "System user to run backups")
|
||||||
|
installCmd.Flags().StringVar(&installGroup, "group", "dbbackup", "System group for backup user")
|
||||||
|
installCmd.Flags().StringVar(&installBackupDir, "backup-dir", "/var/lib/dbbackup/backups", "Directory for backups")
|
||||||
|
installCmd.Flags().StringVar(&installConfigPath, "config-path", "/etc/dbbackup/dbbackup.conf", "Path to config file")
|
||||||
|
installCmd.Flags().IntVar(&installTimeout, "timeout", 3600, "Backup timeout in seconds")
|
||||||
|
installCmd.Flags().BoolVar(&installWithMetrics, "with-metrics", false, "Install Prometheus metrics exporter")
|
||||||
|
installCmd.Flags().IntVar(&installMetricsPort, "metrics-port", 9399, "Prometheus metrics port")
|
||||||
|
installCmd.Flags().BoolVar(&installDryRun, "dry-run", false, "Show what would be installed without making changes")
|
||||||
|
installCmd.Flags().BoolVar(&installStatus, "status", false, "Show installation status")
|
||||||
|
|
||||||
|
// Uninstall flags
|
||||||
|
uninstallCmd.Flags().BoolVar(&uninstallPurge, "purge", false, "Also remove configuration files")
|
||||||
|
}
|
||||||
|
|
||||||
|
func runInstall(ctx context.Context) error {
|
||||||
|
// Create context with signal handling
|
||||||
|
ctx, cancel := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
// Expand schedule shortcuts
|
||||||
|
schedule := expandSchedule(installSchedule)
|
||||||
|
|
||||||
|
// Create installer
|
||||||
|
inst := installer.NewInstaller(log, installDryRun)
|
||||||
|
|
||||||
|
// Set up options
|
||||||
|
opts := installer.InstallOptions{
|
||||||
|
Instance: installInstance,
|
||||||
|
BackupType: installBackupType,
|
||||||
|
Schedule: schedule,
|
||||||
|
User: installUser,
|
||||||
|
Group: installGroup,
|
||||||
|
BackupDir: installBackupDir,
|
||||||
|
ConfigPath: installConfigPath,
|
||||||
|
TimeoutSeconds: installTimeout,
|
||||||
|
WithMetrics: installWithMetrics,
|
||||||
|
MetricsPort: installMetricsPort,
|
||||||
|
}
|
||||||
|
|
||||||
|
// For cluster backup, override instance
|
||||||
|
if installBackupType == "cluster" {
|
||||||
|
opts.Instance = "cluster"
|
||||||
|
}
|
||||||
|
|
||||||
|
return inst.Install(ctx, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func runUninstall(ctx context.Context, instance string) error {
|
||||||
|
ctx, cancel := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
inst := installer.NewInstaller(log, false)
|
||||||
|
return inst.Uninstall(ctx, instance, uninstallPurge)
|
||||||
|
}
|
||||||
|
|
||||||
|
func runInstallStatus(ctx context.Context) error {
|
||||||
|
inst := installer.NewInstaller(log, false)
|
||||||
|
|
||||||
|
// Check cluster status
|
||||||
|
clusterStatus, err := inst.Status(ctx, "cluster")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println("[STATUS] DBBackup Installation Status")
|
||||||
|
fmt.Println(strings.Repeat("=", 50))
|
||||||
|
|
||||||
|
if clusterStatus.Installed {
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println(" * Cluster Backup:")
|
||||||
|
fmt.Printf(" Service: %s\n", formatStatus(clusterStatus.Installed, clusterStatus.Active))
|
||||||
|
fmt.Printf(" Timer: %s\n", formatStatus(clusterStatus.TimerEnabled, clusterStatus.TimerActive))
|
||||||
|
if clusterStatus.NextRun != "" {
|
||||||
|
fmt.Printf(" Next run: %s\n", clusterStatus.NextRun)
|
||||||
|
}
|
||||||
|
if clusterStatus.LastRun != "" {
|
||||||
|
fmt.Printf(" Last run: %s\n", clusterStatus.LastRun)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println("[NONE] No systemd services installed")
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println("Run 'sudo dbbackup install' to install as a systemd service")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for exporter
|
||||||
|
if _, err := os.Stat("/etc/systemd/system/dbbackup-exporter.service"); err == nil {
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println(" * Metrics Exporter:")
|
||||||
|
// Check if exporter is active using systemctl
|
||||||
|
cmd := exec.CommandContext(ctx, "systemctl", "is-active", "dbbackup-exporter")
|
||||||
|
if err := cmd.Run(); err == nil {
|
||||||
|
fmt.Printf(" Service: [OK] active\n")
|
||||||
|
} else {
|
||||||
|
fmt.Printf(" Service: [-] inactive\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func formatStatus(installed, active bool) string {
|
||||||
|
if !installed {
|
||||||
|
return "not installed"
|
||||||
|
}
|
||||||
|
if active {
|
||||||
|
return "[OK] active"
|
||||||
|
}
|
||||||
|
return "[-] inactive"
|
||||||
|
}
|
||||||
|
|
||||||
|
func expandSchedule(schedule string) string {
|
||||||
|
shortcuts := map[string]string{
|
||||||
|
"hourly": "*-*-* *:00:00",
|
||||||
|
"daily": "*-*-* 02:00:00",
|
||||||
|
"weekly": "Mon *-*-* 02:00:00",
|
||||||
|
"monthly": "*-*-01 02:00:00",
|
||||||
|
}
|
||||||
|
|
||||||
|
if expanded, ok := shortcuts[strings.ToLower(schedule)]; ok {
|
||||||
|
return expanded
|
||||||
|
}
|
||||||
|
return schedule
|
||||||
|
}
|
||||||
138
cmd/metrics.go
Normal file
138
cmd/metrics.go
Normal file
@@ -0,0 +1,138 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
"dbbackup/internal/prometheus"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
metricsInstance string
|
||||||
|
metricsOutput string
|
||||||
|
metricsPort int
|
||||||
|
)
|
||||||
|
|
||||||
|
// metricsCmd represents the metrics command
|
||||||
|
var metricsCmd = &cobra.Command{
|
||||||
|
Use: "metrics",
|
||||||
|
Short: "Prometheus metrics management",
|
||||||
|
Long: `Prometheus metrics management for dbbackup.
|
||||||
|
|
||||||
|
Export metrics to a textfile for node_exporter, or run an HTTP server
|
||||||
|
for direct Prometheus scraping.`,
|
||||||
|
}
|
||||||
|
|
||||||
|
// metricsExportCmd exports metrics to a textfile
|
||||||
|
var metricsExportCmd = &cobra.Command{
|
||||||
|
Use: "export",
|
||||||
|
Short: "Export metrics to textfile",
|
||||||
|
Long: `Export Prometheus metrics to a textfile for node_exporter.
|
||||||
|
|
||||||
|
The textfile collector in node_exporter can scrape metrics from files
|
||||||
|
in a designated directory (typically /var/lib/node_exporter/textfile_collector/).
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
# Export metrics to default location
|
||||||
|
dbbackup metrics export
|
||||||
|
|
||||||
|
# Export with custom output path
|
||||||
|
dbbackup metrics export --output /var/lib/dbbackup/metrics/dbbackup.prom
|
||||||
|
|
||||||
|
# Export for specific instance
|
||||||
|
dbbackup metrics export --instance production --output /var/lib/dbbackup/metrics/production.prom
|
||||||
|
|
||||||
|
After export, configure node_exporter with:
|
||||||
|
--collector.textfile.directory=/var/lib/dbbackup/metrics/
|
||||||
|
`,
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
return runMetricsExport(cmd.Context())
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// metricsServeCmd runs the HTTP metrics server
|
||||||
|
var metricsServeCmd = &cobra.Command{
|
||||||
|
Use: "serve",
|
||||||
|
Short: "Run Prometheus HTTP server",
|
||||||
|
Long: `Run an HTTP server exposing Prometheus metrics.
|
||||||
|
|
||||||
|
This starts a long-running daemon that serves metrics at /metrics.
|
||||||
|
Prometheus can scrape this endpoint directly.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
# Start server on default port 9399
|
||||||
|
dbbackup metrics serve
|
||||||
|
|
||||||
|
# Start server on custom port
|
||||||
|
dbbackup metrics serve --port 9100
|
||||||
|
|
||||||
|
# Run as systemd service (installed via 'dbbackup install --with-metrics')
|
||||||
|
sudo systemctl start dbbackup-exporter
|
||||||
|
|
||||||
|
Endpoints:
|
||||||
|
/metrics - Prometheus metrics
|
||||||
|
/health - Health check (returns 200 OK)
|
||||||
|
/ - Service info page
|
||||||
|
`,
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
return runMetricsServe(cmd.Context())
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
rootCmd.AddCommand(metricsCmd)
|
||||||
|
metricsCmd.AddCommand(metricsExportCmd)
|
||||||
|
metricsCmd.AddCommand(metricsServeCmd)
|
||||||
|
|
||||||
|
// Export flags
|
||||||
|
metricsExportCmd.Flags().StringVar(&metricsInstance, "instance", "default", "Instance name for metrics labels")
|
||||||
|
metricsExportCmd.Flags().StringVarP(&metricsOutput, "output", "o", "/var/lib/dbbackup/metrics/dbbackup.prom", "Output file path")
|
||||||
|
|
||||||
|
// Serve flags
|
||||||
|
metricsServeCmd.Flags().StringVar(&metricsInstance, "instance", "default", "Instance name for metrics labels")
|
||||||
|
metricsServeCmd.Flags().IntVarP(&metricsPort, "port", "p", 9399, "HTTP server port")
|
||||||
|
}
|
||||||
|
|
||||||
|
func runMetricsExport(ctx context.Context) error {
|
||||||
|
// Open catalog
|
||||||
|
cat, err := openCatalog()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open catalog: %w", err)
|
||||||
|
}
|
||||||
|
defer cat.Close()
|
||||||
|
|
||||||
|
// Create metrics writer
|
||||||
|
writer := prometheus.NewMetricsWriter(log, cat, metricsInstance)
|
||||||
|
|
||||||
|
// Write textfile
|
||||||
|
if err := writer.WriteTextfile(metricsOutput); err != nil {
|
||||||
|
return fmt.Errorf("failed to write metrics: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info("Exported metrics to textfile", "path", metricsOutput, "instance", metricsInstance)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func runMetricsServe(ctx context.Context) error {
|
||||||
|
// Setup signal handling
|
||||||
|
ctx, cancel := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
// Open catalog
|
||||||
|
cat, err := openCatalog()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open catalog: %w", err)
|
||||||
|
}
|
||||||
|
defer cat.Close()
|
||||||
|
|
||||||
|
// Create exporter
|
||||||
|
exporter := prometheus.NewExporter(log, cat, metricsInstance, metricsPort)
|
||||||
|
|
||||||
|
// Run server (blocks until context is cancelled)
|
||||||
|
return exporter.Serve(ctx)
|
||||||
|
}
|
||||||
@@ -203,9 +203,17 @@ func runMigrateCluster(cmd *cobra.Command, args []string) error {
|
|||||||
migrateTargetUser = migrateSourceUser
|
migrateTargetUser = migrateSourceUser
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Create source config first to get WorkDir
|
||||||
|
sourceCfg := config.New()
|
||||||
|
sourceCfg.Host = migrateSourceHost
|
||||||
|
sourceCfg.Port = migrateSourcePort
|
||||||
|
sourceCfg.User = migrateSourceUser
|
||||||
|
sourceCfg.Password = migrateSourcePassword
|
||||||
|
|
||||||
workdir := migrateWorkdir
|
workdir := migrateWorkdir
|
||||||
if workdir == "" {
|
if workdir == "" {
|
||||||
workdir = filepath.Join(os.TempDir(), "dbbackup-migrate")
|
// Use WorkDir from config if available
|
||||||
|
workdir = filepath.Join(sourceCfg.GetEffectiveWorkDir(), "dbbackup-migrate")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create working directory
|
// Create working directory
|
||||||
@@ -213,12 +221,7 @@ func runMigrateCluster(cmd *cobra.Command, args []string) error {
|
|||||||
return fmt.Errorf("failed to create working directory: %w", err)
|
return fmt.Errorf("failed to create working directory: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create source config
|
// Update source config with remaining settings
|
||||||
sourceCfg := config.New()
|
|
||||||
sourceCfg.Host = migrateSourceHost
|
|
||||||
sourceCfg.Port = migrateSourcePort
|
|
||||||
sourceCfg.User = migrateSourceUser
|
|
||||||
sourceCfg.Password = migrateSourcePassword
|
|
||||||
sourceCfg.SSLMode = migrateSourceSSLMode
|
sourceCfg.SSLMode = migrateSourceSSLMode
|
||||||
sourceCfg.Database = "postgres" // Default connection database
|
sourceCfg.Database = "postgres" // Default connection database
|
||||||
sourceCfg.DatabaseType = cfg.DatabaseType
|
sourceCfg.DatabaseType = cfg.DatabaseType
|
||||||
@@ -342,7 +345,8 @@ func runMigrateSingle(cmd *cobra.Command, args []string) error {
|
|||||||
|
|
||||||
workdir := migrateWorkdir
|
workdir := migrateWorkdir
|
||||||
if workdir == "" {
|
if workdir == "" {
|
||||||
workdir = filepath.Join(os.TempDir(), "dbbackup-migrate")
|
tempCfg := config.New()
|
||||||
|
workdir = filepath.Join(tempCfg.GetEffectiveWorkDir(), "dbbackup-migrate")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create working directory
|
// Create working directory
|
||||||
|
|||||||
76
cmd/pitr.go
76
cmd/pitr.go
@@ -436,7 +436,7 @@ func runPITREnable(cmd *cobra.Command, args []string) error {
|
|||||||
return fmt.Errorf("failed to enable PITR: %w", err)
|
return fmt.Errorf("failed to enable PITR: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("✅ PITR enabled successfully!")
|
log.Info("[OK] PITR enabled successfully!")
|
||||||
log.Info("")
|
log.Info("")
|
||||||
log.Info("Next steps:")
|
log.Info("Next steps:")
|
||||||
log.Info("1. Restart PostgreSQL: sudo systemctl restart postgresql")
|
log.Info("1. Restart PostgreSQL: sudo systemctl restart postgresql")
|
||||||
@@ -463,7 +463,7 @@ func runPITRDisable(cmd *cobra.Command, args []string) error {
|
|||||||
return fmt.Errorf("failed to disable PITR: %w", err)
|
return fmt.Errorf("failed to disable PITR: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("✅ PITR disabled successfully!")
|
log.Info("[OK] PITR disabled successfully!")
|
||||||
log.Info("PostgreSQL restart required: sudo systemctl restart postgresql")
|
log.Info("PostgreSQL restart required: sudo systemctl restart postgresql")
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -483,15 +483,15 @@ func runPITRStatus(cmd *cobra.Command, args []string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Display PITR configuration
|
// Display PITR configuration
|
||||||
fmt.Println("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
fmt.Println("======================================================")
|
||||||
fmt.Println(" Point-in-Time Recovery (PITR) Status")
|
fmt.Println(" Point-in-Time Recovery (PITR) Status")
|
||||||
fmt.Println("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
fmt.Println("======================================================")
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
|
|
||||||
if config.Enabled {
|
if config.Enabled {
|
||||||
fmt.Println("Status: ✅ ENABLED")
|
fmt.Println("Status: [OK] ENABLED")
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("Status: ❌ DISABLED")
|
fmt.Println("Status: [FAIL] DISABLED")
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("WAL Level: %s\n", config.WALLevel)
|
fmt.Printf("WAL Level: %s\n", config.WALLevel)
|
||||||
@@ -510,7 +510,7 @@ func runPITRStatus(cmd *cobra.Command, args []string) error {
|
|||||||
// Extract archive dir from command (simple parsing)
|
// Extract archive dir from command (simple parsing)
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
fmt.Println("WAL Archive Statistics:")
|
fmt.Println("WAL Archive Statistics:")
|
||||||
fmt.Println("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
fmt.Println("======================================================")
|
||||||
// TODO: Parse archive dir and show stats
|
// TODO: Parse archive dir and show stats
|
||||||
fmt.Println(" (Use 'dbbackup wal list --archive-dir <dir>' to view archives)")
|
fmt.Println(" (Use 'dbbackup wal list --archive-dir <dir>' to view archives)")
|
||||||
}
|
}
|
||||||
@@ -574,13 +574,13 @@ func runWALList(cmd *cobra.Command, args []string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Display archives
|
// Display archives
|
||||||
fmt.Println("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
fmt.Println("======================================================")
|
||||||
fmt.Printf(" WAL Archives (%d files)\n", len(archives))
|
fmt.Printf(" WAL Archives (%d files)\n", len(archives))
|
||||||
fmt.Println("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
fmt.Println("======================================================")
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
|
|
||||||
fmt.Printf("%-28s %10s %10s %8s %s\n", "WAL Filename", "Timeline", "Segment", "Size", "Archived At")
|
fmt.Printf("%-28s %10s %10s %8s %s\n", "WAL Filename", "Timeline", "Segment", "Size", "Archived At")
|
||||||
fmt.Println("────────────────────────────────────────────────────────────────────────────────")
|
fmt.Println("--------------------------------------------------------------------------------")
|
||||||
|
|
||||||
for _, archive := range archives {
|
for _, archive := range archives {
|
||||||
size := formatWALSize(archive.ArchivedSize)
|
size := formatWALSize(archive.ArchivedSize)
|
||||||
@@ -644,7 +644,7 @@ func runWALCleanup(cmd *cobra.Command, args []string) error {
|
|||||||
return fmt.Errorf("WAL cleanup failed: %w", err)
|
return fmt.Errorf("WAL cleanup failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("✅ WAL cleanup completed", "deleted", deleted, "retention_days", archiveConfig.RetentionDays)
|
log.Info("[OK] WAL cleanup completed", "deleted", deleted, "retention_days", archiveConfig.RetentionDays)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -671,7 +671,7 @@ func runWALTimeline(cmd *cobra.Command, args []string) error {
|
|||||||
// Display timeline details
|
// Display timeline details
|
||||||
if len(history.Timelines) > 0 {
|
if len(history.Timelines) > 0 {
|
||||||
fmt.Println("\nTimeline Details:")
|
fmt.Println("\nTimeline Details:")
|
||||||
fmt.Println("═════════════════")
|
fmt.Println("=================")
|
||||||
for _, tl := range history.Timelines {
|
for _, tl := range history.Timelines {
|
||||||
fmt.Printf("\nTimeline %d:\n", tl.TimelineID)
|
fmt.Printf("\nTimeline %d:\n", tl.TimelineID)
|
||||||
if tl.ParentTimeline > 0 {
|
if tl.ParentTimeline > 0 {
|
||||||
@@ -690,7 +690,7 @@ func runWALTimeline(cmd *cobra.Command, args []string) error {
|
|||||||
fmt.Printf(" Created: %s\n", tl.CreatedAt.Format("2006-01-02 15:04:05"))
|
fmt.Printf(" Created: %s\n", tl.CreatedAt.Format("2006-01-02 15:04:05"))
|
||||||
}
|
}
|
||||||
if tl.TimelineID == history.CurrentTimeline {
|
if tl.TimelineID == history.CurrentTimeline {
|
||||||
fmt.Printf(" Status: ⚡ CURRENT\n")
|
fmt.Printf(" Status: [CURR] CURRENT\n")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -759,15 +759,15 @@ func runBinlogList(cmd *cobra.Command, args []string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
fmt.Println("=============================================================")
|
||||||
fmt.Printf(" Binary Log Files (%s)\n", bm.ServerType())
|
fmt.Printf(" Binary Log Files (%s)\n", bm.ServerType())
|
||||||
fmt.Println("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
fmt.Println("=============================================================")
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
|
|
||||||
if len(binlogs) > 0 {
|
if len(binlogs) > 0 {
|
||||||
fmt.Println("Source Directory:")
|
fmt.Println("Source Directory:")
|
||||||
fmt.Printf("%-24s %10s %-19s %-19s %s\n", "Filename", "Size", "Start Time", "End Time", "Format")
|
fmt.Printf("%-24s %10s %-19s %-19s %s\n", "Filename", "Size", "Start Time", "End Time", "Format")
|
||||||
fmt.Println("────────────────────────────────────────────────────────────────────────────────")
|
fmt.Println("--------------------------------------------------------------------------------")
|
||||||
|
|
||||||
var totalSize int64
|
var totalSize int64
|
||||||
for _, b := range binlogs {
|
for _, b := range binlogs {
|
||||||
@@ -797,7 +797,7 @@ func runBinlogList(cmd *cobra.Command, args []string) error {
|
|||||||
fmt.Println()
|
fmt.Println()
|
||||||
fmt.Println("Archived Binlogs:")
|
fmt.Println("Archived Binlogs:")
|
||||||
fmt.Printf("%-24s %10s %-19s %s\n", "Original", "Size", "Archived At", "Flags")
|
fmt.Printf("%-24s %10s %-19s %s\n", "Original", "Size", "Archived At", "Flags")
|
||||||
fmt.Println("────────────────────────────────────────────────────────────────────────────────")
|
fmt.Println("--------------------------------------------------------------------------------")
|
||||||
|
|
||||||
var totalSize int64
|
var totalSize int64
|
||||||
for _, a := range archived {
|
for _, a := range archived {
|
||||||
@@ -914,7 +914,7 @@ func runBinlogArchive(cmd *cobra.Command, args []string) error {
|
|||||||
bm.SaveArchiveMetadata(allArchived)
|
bm.SaveArchiveMetadata(allArchived)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("✅ Binlog archiving completed", "archived", len(newArchives))
|
log.Info("[OK] Binlog archiving completed", "archived", len(newArchives))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1014,15 +1014,15 @@ func runBinlogValidate(cmd *cobra.Command, args []string) error {
|
|||||||
return fmt.Errorf("validating binlog chain: %w", err)
|
return fmt.Errorf("validating binlog chain: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
fmt.Println("=============================================================")
|
||||||
fmt.Println(" Binlog Chain Validation")
|
fmt.Println(" Binlog Chain Validation")
|
||||||
fmt.Println("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
fmt.Println("=============================================================")
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
|
|
||||||
if validation.Valid {
|
if validation.Valid {
|
||||||
fmt.Println("Status: ✅ VALID - Binlog chain is complete")
|
fmt.Println("Status: [OK] VALID - Binlog chain is complete")
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("Status: ❌ INVALID - Binlog chain has gaps")
|
fmt.Println("Status: [FAIL] INVALID - Binlog chain has gaps")
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("Files: %d binlog files\n", validation.LogCount)
|
fmt.Printf("Files: %d binlog files\n", validation.LogCount)
|
||||||
@@ -1055,7 +1055,7 @@ func runBinlogValidate(cmd *cobra.Command, args []string) error {
|
|||||||
fmt.Println()
|
fmt.Println()
|
||||||
fmt.Println("Errors:")
|
fmt.Println("Errors:")
|
||||||
for _, e := range validation.Errors {
|
for _, e := range validation.Errors {
|
||||||
fmt.Printf(" ✗ %s\n", e)
|
fmt.Printf(" [FAIL] %s\n", e)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1094,9 +1094,9 @@ func runBinlogPosition(cmd *cobra.Command, args []string) error {
|
|||||||
}
|
}
|
||||||
defer rows.Close()
|
defer rows.Close()
|
||||||
|
|
||||||
fmt.Println("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
fmt.Println("=============================================================")
|
||||||
fmt.Println(" Current Binary Log Position")
|
fmt.Println(" Current Binary Log Position")
|
||||||
fmt.Println("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
fmt.Println("=============================================================")
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
|
|
||||||
if rows.Next() {
|
if rows.Next() {
|
||||||
@@ -1178,24 +1178,24 @@ func runMySQLPITRStatus(cmd *cobra.Command, args []string) error {
|
|||||||
return fmt.Errorf("getting PITR status: %w", err)
|
return fmt.Errorf("getting PITR status: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
fmt.Println("=============================================================")
|
||||||
fmt.Printf(" MySQL/MariaDB PITR Status (%s)\n", status.DatabaseType)
|
fmt.Printf(" MySQL/MariaDB PITR Status (%s)\n", status.DatabaseType)
|
||||||
fmt.Println("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
fmt.Println("=============================================================")
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
|
|
||||||
if status.Enabled {
|
if status.Enabled {
|
||||||
fmt.Println("PITR Status: ✅ ENABLED")
|
fmt.Println("PITR Status: [OK] ENABLED")
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("PITR Status: ❌ NOT CONFIGURED")
|
fmt.Println("PITR Status: [FAIL] NOT CONFIGURED")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get binary logging status
|
// Get binary logging status
|
||||||
var logBin string
|
var logBin string
|
||||||
db.QueryRowContext(ctx, "SELECT @@log_bin").Scan(&logBin)
|
db.QueryRowContext(ctx, "SELECT @@log_bin").Scan(&logBin)
|
||||||
if logBin == "1" || logBin == "ON" {
|
if logBin == "1" || logBin == "ON" {
|
||||||
fmt.Println("Binary Logging: ✅ ENABLED")
|
fmt.Println("Binary Logging: [OK] ENABLED")
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("Binary Logging: ❌ DISABLED")
|
fmt.Println("Binary Logging: [FAIL] DISABLED")
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("Binlog Format: %s\n", status.LogLevel)
|
fmt.Printf("Binlog Format: %s\n", status.LogLevel)
|
||||||
@@ -1205,14 +1205,14 @@ func runMySQLPITRStatus(cmd *cobra.Command, args []string) error {
|
|||||||
if status.DatabaseType == pitr.DatabaseMariaDB {
|
if status.DatabaseType == pitr.DatabaseMariaDB {
|
||||||
db.QueryRowContext(ctx, "SELECT @@gtid_current_pos").Scan(>idMode)
|
db.QueryRowContext(ctx, "SELECT @@gtid_current_pos").Scan(>idMode)
|
||||||
if gtidMode != "" {
|
if gtidMode != "" {
|
||||||
fmt.Println("GTID Mode: ✅ ENABLED")
|
fmt.Println("GTID Mode: [OK] ENABLED")
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("GTID Mode: ❌ DISABLED")
|
fmt.Println("GTID Mode: [FAIL] DISABLED")
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
db.QueryRowContext(ctx, "SELECT @@gtid_mode").Scan(>idMode)
|
db.QueryRowContext(ctx, "SELECT @@gtid_mode").Scan(>idMode)
|
||||||
if gtidMode == "ON" {
|
if gtidMode == "ON" {
|
||||||
fmt.Println("GTID Mode: ✅ ENABLED")
|
fmt.Println("GTID Mode: [OK] ENABLED")
|
||||||
} else {
|
} else {
|
||||||
fmt.Printf("GTID Mode: %s\n", gtidMode)
|
fmt.Printf("GTID Mode: %s\n", gtidMode)
|
||||||
}
|
}
|
||||||
@@ -1237,12 +1237,12 @@ func runMySQLPITRStatus(cmd *cobra.Command, args []string) error {
|
|||||||
fmt.Println()
|
fmt.Println()
|
||||||
fmt.Println("PITR Requirements:")
|
fmt.Println("PITR Requirements:")
|
||||||
if logBin == "1" || logBin == "ON" {
|
if logBin == "1" || logBin == "ON" {
|
||||||
fmt.Println(" ✅ Binary logging enabled")
|
fmt.Println(" [OK] Binary logging enabled")
|
||||||
} else {
|
} else {
|
||||||
fmt.Println(" ❌ Binary logging must be enabled (log_bin = mysql-bin)")
|
fmt.Println(" [FAIL] Binary logging must be enabled (log_bin = mysql-bin)")
|
||||||
}
|
}
|
||||||
if status.LogLevel == "ROW" {
|
if status.LogLevel == "ROW" {
|
||||||
fmt.Println(" ✅ Row-based logging (recommended)")
|
fmt.Println(" [OK] Row-based logging (recommended)")
|
||||||
} else {
|
} else {
|
||||||
fmt.Printf(" ⚠ binlog_format = %s (ROW recommended for PITR)\n", status.LogLevel)
|
fmt.Printf(" ⚠ binlog_format = %s (ROW recommended for PITR)\n", status.LogLevel)
|
||||||
}
|
}
|
||||||
@@ -1299,7 +1299,7 @@ func runMySQLPITREnable(cmd *cobra.Command, args []string) error {
|
|||||||
return fmt.Errorf("enabling PITR: %w", err)
|
return fmt.Errorf("enabling PITR: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("✅ MySQL PITR enabled successfully!")
|
log.Info("[OK] MySQL PITR enabled successfully!")
|
||||||
log.Info("")
|
log.Info("")
|
||||||
log.Info("Next steps:")
|
log.Info("Next steps:")
|
||||||
log.Info("1. Start binlog archiving: dbbackup binlog watch --archive-dir " + mysqlArchiveDir)
|
log.Info("1. Start binlog archiving: dbbackup binlog watch --archive-dir " + mysqlArchiveDir)
|
||||||
|
|||||||
@@ -141,7 +141,7 @@ func runList(ctx context.Context) error {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("📦 %s\n", file.Name)
|
fmt.Printf("[FILE] %s\n", file.Name)
|
||||||
fmt.Printf(" Size: %s\n", formatFileSize(stat.Size()))
|
fmt.Printf(" Size: %s\n", formatFileSize(stat.Size()))
|
||||||
fmt.Printf(" Modified: %s\n", stat.ModTime().Format("2006-01-02 15:04:05"))
|
fmt.Printf(" Modified: %s\n", stat.ModTime().Format("2006-01-02 15:04:05"))
|
||||||
fmt.Printf(" Type: %s\n", getBackupType(file.Name))
|
fmt.Printf(" Type: %s\n", getBackupType(file.Name))
|
||||||
@@ -237,56 +237,56 @@ func runPreflight(ctx context.Context) error {
|
|||||||
totalChecks := 6
|
totalChecks := 6
|
||||||
|
|
||||||
// 1. Database connectivity check
|
// 1. Database connectivity check
|
||||||
fmt.Print("🔗 Database connectivity... ")
|
fmt.Print("[1] Database connectivity... ")
|
||||||
if err := testDatabaseConnection(); err != nil {
|
if err := testDatabaseConnection(); err != nil {
|
||||||
fmt.Printf("❌ FAILED: %v\n", err)
|
fmt.Printf("[FAIL] FAILED: %v\n", err)
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("✅ PASSED")
|
fmt.Println("[OK] PASSED")
|
||||||
checksPassed++
|
checksPassed++
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Required tools check
|
// 2. Required tools check
|
||||||
fmt.Print("🛠️ Required tools (pg_dump/pg_restore)... ")
|
fmt.Print("[2] Required tools (pg_dump/pg_restore)... ")
|
||||||
if err := checkRequiredTools(); err != nil {
|
if err := checkRequiredTools(); err != nil {
|
||||||
fmt.Printf("❌ FAILED: %v\n", err)
|
fmt.Printf("[FAIL] FAILED: %v\n", err)
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("✅ PASSED")
|
fmt.Println("[OK] PASSED")
|
||||||
checksPassed++
|
checksPassed++
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3. Backup directory check
|
// 3. Backup directory check
|
||||||
fmt.Print("📁 Backup directory access... ")
|
fmt.Print("[3] Backup directory access... ")
|
||||||
if err := checkBackupDirectory(); err != nil {
|
if err := checkBackupDirectory(); err != nil {
|
||||||
fmt.Printf("❌ FAILED: %v\n", err)
|
fmt.Printf("[FAIL] FAILED: %v\n", err)
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("✅ PASSED")
|
fmt.Println("[OK] PASSED")
|
||||||
checksPassed++
|
checksPassed++
|
||||||
}
|
}
|
||||||
|
|
||||||
// 4. Disk space check
|
// 4. Disk space check
|
||||||
fmt.Print("💾 Available disk space... ")
|
fmt.Print("[4] Available disk space... ")
|
||||||
if err := checkDiskSpace(); err != nil {
|
if err := checkDiskSpace(); err != nil {
|
||||||
fmt.Printf("❌ FAILED: %v\n", err)
|
fmt.Printf("[FAIL] FAILED: %v\n", err)
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("✅ PASSED")
|
fmt.Println("[OK] PASSED")
|
||||||
checksPassed++
|
checksPassed++
|
||||||
}
|
}
|
||||||
|
|
||||||
// 5. Permissions check
|
// 5. Permissions check
|
||||||
fmt.Print("🔐 File permissions... ")
|
fmt.Print("[5] File permissions... ")
|
||||||
if err := checkPermissions(); err != nil {
|
if err := checkPermissions(); err != nil {
|
||||||
fmt.Printf("❌ FAILED: %v\n", err)
|
fmt.Printf("[FAIL] FAILED: %v\n", err)
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("✅ PASSED")
|
fmt.Println("[OK] PASSED")
|
||||||
checksPassed++
|
checksPassed++
|
||||||
}
|
}
|
||||||
|
|
||||||
// 6. CPU/Memory resources check
|
// 6. CPU/Memory resources check
|
||||||
fmt.Print("🖥️ System resources... ")
|
fmt.Print("[6] System resources... ")
|
||||||
if err := checkSystemResources(); err != nil {
|
if err := checkSystemResources(); err != nil {
|
||||||
fmt.Printf("❌ FAILED: %v\n", err)
|
fmt.Printf("[FAIL] FAILED: %v\n", err)
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("✅ PASSED")
|
fmt.Println("[OK] PASSED")
|
||||||
checksPassed++
|
checksPassed++
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -294,10 +294,10 @@ func runPreflight(ctx context.Context) error {
|
|||||||
fmt.Printf("Results: %d/%d checks passed\n", checksPassed, totalChecks)
|
fmt.Printf("Results: %d/%d checks passed\n", checksPassed, totalChecks)
|
||||||
|
|
||||||
if checksPassed == totalChecks {
|
if checksPassed == totalChecks {
|
||||||
fmt.Println("🎉 All preflight checks passed! System is ready for backup operations.")
|
fmt.Println("[SUCCESS] All preflight checks passed! System is ready for backup operations.")
|
||||||
return nil
|
return nil
|
||||||
} else {
|
} else {
|
||||||
fmt.Printf("⚠️ %d check(s) failed. Please address the issues before running backups.\n", totalChecks-checksPassed)
|
fmt.Printf("[WARN] %d check(s) failed. Please address the issues before running backups.\n", totalChecks-checksPassed)
|
||||||
return fmt.Errorf("preflight checks failed: %d/%d passed", checksPassed, totalChecks)
|
return fmt.Errorf("preflight checks failed: %d/%d passed", checksPassed, totalChecks)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -414,44 +414,44 @@ func runRestore(ctx context.Context, archiveName string) error {
|
|||||||
fmt.Println()
|
fmt.Println()
|
||||||
|
|
||||||
// Show warning
|
// Show warning
|
||||||
fmt.Println("⚠️ WARNING: This will restore data to the target database.")
|
fmt.Println("[WARN] WARNING: This will restore data to the target database.")
|
||||||
fmt.Println(" Existing data may be overwritten or merged depending on the restore method.")
|
fmt.Println(" Existing data may be overwritten or merged depending on the restore method.")
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
|
|
||||||
// For safety, show what would be done without actually doing it
|
// For safety, show what would be done without actually doing it
|
||||||
switch archiveType {
|
switch archiveType {
|
||||||
case "Single Database (.dump)":
|
case "Single Database (.dump)":
|
||||||
fmt.Println("🔄 Would execute: pg_restore to restore single database")
|
fmt.Println("[EXEC] Would execute: pg_restore to restore single database")
|
||||||
fmt.Printf(" Command: pg_restore -h %s -p %d -U %s -d %s --verbose %s\n",
|
fmt.Printf(" Command: pg_restore -h %s -p %d -U %s -d %s --verbose %s\n",
|
||||||
cfg.Host, cfg.Port, cfg.User, cfg.Database, archivePath)
|
cfg.Host, cfg.Port, cfg.User, cfg.Database, archivePath)
|
||||||
case "Single Database (.dump.gz)":
|
case "Single Database (.dump.gz)":
|
||||||
fmt.Println("🔄 Would execute: gunzip and pg_restore to restore single database")
|
fmt.Println("[EXEC] Would execute: gunzip and pg_restore to restore single database")
|
||||||
fmt.Printf(" Command: gunzip -c %s | pg_restore -h %s -p %d -U %s -d %s --verbose\n",
|
fmt.Printf(" Command: gunzip -c %s | pg_restore -h %s -p %d -U %s -d %s --verbose\n",
|
||||||
archivePath, cfg.Host, cfg.Port, cfg.User, cfg.Database)
|
archivePath, cfg.Host, cfg.Port, cfg.User, cfg.Database)
|
||||||
case "SQL Script (.sql)":
|
case "SQL Script (.sql)":
|
||||||
if cfg.IsPostgreSQL() {
|
if cfg.IsPostgreSQL() {
|
||||||
fmt.Println("🔄 Would execute: psql to run SQL script")
|
fmt.Println("[EXEC] Would execute: psql to run SQL script")
|
||||||
fmt.Printf(" Command: psql -h %s -p %d -U %s -d %s -f %s\n",
|
fmt.Printf(" Command: psql -h %s -p %d -U %s -d %s -f %s\n",
|
||||||
cfg.Host, cfg.Port, cfg.User, cfg.Database, archivePath)
|
cfg.Host, cfg.Port, cfg.User, cfg.Database, archivePath)
|
||||||
} else if cfg.IsMySQL() {
|
} else if cfg.IsMySQL() {
|
||||||
fmt.Println("🔄 Would execute: mysql to run SQL script")
|
fmt.Println("[EXEC] Would execute: mysql to run SQL script")
|
||||||
fmt.Printf(" Command: %s\n", mysqlRestoreCommand(archivePath, false))
|
fmt.Printf(" Command: %s\n", mysqlRestoreCommand(archivePath, false))
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("🔄 Would execute: SQL client to run script (database type unknown)")
|
fmt.Println("[EXEC] Would execute: SQL client to run script (database type unknown)")
|
||||||
}
|
}
|
||||||
case "SQL Script (.sql.gz)":
|
case "SQL Script (.sql.gz)":
|
||||||
if cfg.IsPostgreSQL() {
|
if cfg.IsPostgreSQL() {
|
||||||
fmt.Println("🔄 Would execute: gunzip and psql to run SQL script")
|
fmt.Println("[EXEC] Would execute: gunzip and psql to run SQL script")
|
||||||
fmt.Printf(" Command: gunzip -c %s | psql -h %s -p %d -U %s -d %s\n",
|
fmt.Printf(" Command: gunzip -c %s | psql -h %s -p %d -U %s -d %s\n",
|
||||||
archivePath, cfg.Host, cfg.Port, cfg.User, cfg.Database)
|
archivePath, cfg.Host, cfg.Port, cfg.User, cfg.Database)
|
||||||
} else if cfg.IsMySQL() {
|
} else if cfg.IsMySQL() {
|
||||||
fmt.Println("🔄 Would execute: gunzip and mysql to run SQL script")
|
fmt.Println("[EXEC] Would execute: gunzip and mysql to run SQL script")
|
||||||
fmt.Printf(" Command: %s\n", mysqlRestoreCommand(archivePath, true))
|
fmt.Printf(" Command: %s\n", mysqlRestoreCommand(archivePath, true))
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("🔄 Would execute: gunzip and SQL client to run script (database type unknown)")
|
fmt.Println("[EXEC] Would execute: gunzip and SQL client to run script (database type unknown)")
|
||||||
}
|
}
|
||||||
case "Cluster Backup (.tar.gz)":
|
case "Cluster Backup (.tar.gz)":
|
||||||
fmt.Println("🔄 Would execute: Extract and restore cluster backup")
|
fmt.Println("[EXEC] Would execute: Extract and restore cluster backup")
|
||||||
fmt.Println(" Steps:")
|
fmt.Println(" Steps:")
|
||||||
fmt.Println(" 1. Extract tar.gz archive")
|
fmt.Println(" 1. Extract tar.gz archive")
|
||||||
fmt.Println(" 2. Restore global objects (roles, tablespaces)")
|
fmt.Println(" 2. Restore global objects (roles, tablespaces)")
|
||||||
@@ -461,7 +461,7 @@ func runRestore(ctx context.Context, archiveName string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
fmt.Println("🛡️ SAFETY MODE: Restore command is in preview mode.")
|
fmt.Println("[SAFETY] SAFETY MODE: Restore command is in preview mode.")
|
||||||
fmt.Println(" This shows what would be executed without making changes.")
|
fmt.Println(" This shows what would be executed without making changes.")
|
||||||
fmt.Println(" To enable actual restore, add --confirm flag (not yet implemented).")
|
fmt.Println(" To enable actual restore, add --confirm flag (not yet implemented).")
|
||||||
|
|
||||||
@@ -520,25 +520,25 @@ func runVerify(ctx context.Context, archiveName string) error {
|
|||||||
checksPassed := 0
|
checksPassed := 0
|
||||||
|
|
||||||
// Basic file existence and readability
|
// Basic file existence and readability
|
||||||
fmt.Print("📁 File accessibility... ")
|
fmt.Print("[CHK] File accessibility... ")
|
||||||
if file, err := os.Open(archivePath); err != nil {
|
if file, err := os.Open(archivePath); err != nil {
|
||||||
fmt.Printf("❌ FAILED: %v\n", err)
|
fmt.Printf("[FAIL] FAILED: %v\n", err)
|
||||||
} else {
|
} else {
|
||||||
file.Close()
|
file.Close()
|
||||||
fmt.Println("✅ PASSED")
|
fmt.Println("[OK] PASSED")
|
||||||
checksPassed++
|
checksPassed++
|
||||||
}
|
}
|
||||||
checksRun++
|
checksRun++
|
||||||
|
|
||||||
// File size sanity check
|
// File size sanity check
|
||||||
fmt.Print("📏 File size check... ")
|
fmt.Print("[CHK] File size check... ")
|
||||||
if stat.Size() == 0 {
|
if stat.Size() == 0 {
|
||||||
fmt.Println("❌ FAILED: File is empty")
|
fmt.Println("[FAIL] FAILED: File is empty")
|
||||||
} else if stat.Size() < 100 {
|
} else if stat.Size() < 100 {
|
||||||
fmt.Println("⚠️ WARNING: File is very small (< 100 bytes)")
|
fmt.Println("[WARN] WARNING: File is very small (< 100 bytes)")
|
||||||
checksPassed++
|
checksPassed++
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("✅ PASSED")
|
fmt.Println("[OK] PASSED")
|
||||||
checksPassed++
|
checksPassed++
|
||||||
}
|
}
|
||||||
checksRun++
|
checksRun++
|
||||||
@@ -546,51 +546,51 @@ func runVerify(ctx context.Context, archiveName string) error {
|
|||||||
// Type-specific verification
|
// Type-specific verification
|
||||||
switch archiveType {
|
switch archiveType {
|
||||||
case "Single Database (.dump)":
|
case "Single Database (.dump)":
|
||||||
fmt.Print("🔍 PostgreSQL dump format check... ")
|
fmt.Print("[CHK] PostgreSQL dump format check... ")
|
||||||
if err := verifyPgDump(archivePath); err != nil {
|
if err := verifyPgDump(archivePath); err != nil {
|
||||||
fmt.Printf("❌ FAILED: %v\n", err)
|
fmt.Printf("[FAIL] FAILED: %v\n", err)
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("✅ PASSED")
|
fmt.Println("[OK] PASSED")
|
||||||
checksPassed++
|
checksPassed++
|
||||||
}
|
}
|
||||||
checksRun++
|
checksRun++
|
||||||
|
|
||||||
case "Single Database (.dump.gz)":
|
case "Single Database (.dump.gz)":
|
||||||
fmt.Print("🔍 PostgreSQL dump format check (gzip)... ")
|
fmt.Print("[CHK] PostgreSQL dump format check (gzip)... ")
|
||||||
if err := verifyPgDumpGzip(archivePath); err != nil {
|
if err := verifyPgDumpGzip(archivePath); err != nil {
|
||||||
fmt.Printf("❌ FAILED: %v\n", err)
|
fmt.Printf("[FAIL] FAILED: %v\n", err)
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("✅ PASSED")
|
fmt.Println("[OK] PASSED")
|
||||||
checksPassed++
|
checksPassed++
|
||||||
}
|
}
|
||||||
checksRun++
|
checksRun++
|
||||||
|
|
||||||
case "SQL Script (.sql)":
|
case "SQL Script (.sql)":
|
||||||
fmt.Print("📜 SQL script validation... ")
|
fmt.Print("[CHK] SQL script validation... ")
|
||||||
if err := verifySqlScript(archivePath); err != nil {
|
if err := verifySqlScript(archivePath); err != nil {
|
||||||
fmt.Printf("❌ FAILED: %v\n", err)
|
fmt.Printf("[FAIL] FAILED: %v\n", err)
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("✅ PASSED")
|
fmt.Println("[OK] PASSED")
|
||||||
checksPassed++
|
checksPassed++
|
||||||
}
|
}
|
||||||
checksRun++
|
checksRun++
|
||||||
|
|
||||||
case "SQL Script (.sql.gz)":
|
case "SQL Script (.sql.gz)":
|
||||||
fmt.Print("📜 SQL script validation (gzip)... ")
|
fmt.Print("[CHK] SQL script validation (gzip)... ")
|
||||||
if err := verifyGzipSqlScript(archivePath); err != nil {
|
if err := verifyGzipSqlScript(archivePath); err != nil {
|
||||||
fmt.Printf("❌ FAILED: %v\n", err)
|
fmt.Printf("[FAIL] FAILED: %v\n", err)
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("✅ PASSED")
|
fmt.Println("[OK] PASSED")
|
||||||
checksPassed++
|
checksPassed++
|
||||||
}
|
}
|
||||||
checksRun++
|
checksRun++
|
||||||
|
|
||||||
case "Cluster Backup (.tar.gz)":
|
case "Cluster Backup (.tar.gz)":
|
||||||
fmt.Print("📦 Archive extraction test... ")
|
fmt.Print("[CHK] Archive extraction test... ")
|
||||||
if err := verifyTarGz(archivePath); err != nil {
|
if err := verifyTarGz(archivePath); err != nil {
|
||||||
fmt.Printf("❌ FAILED: %v\n", err)
|
fmt.Printf("[FAIL] FAILED: %v\n", err)
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("✅ PASSED")
|
fmt.Println("[OK] PASSED")
|
||||||
checksPassed++
|
checksPassed++
|
||||||
}
|
}
|
||||||
checksRun++
|
checksRun++
|
||||||
@@ -598,11 +598,11 @@ func runVerify(ctx context.Context, archiveName string) error {
|
|||||||
|
|
||||||
// Check for metadata file
|
// Check for metadata file
|
||||||
metadataPath := archivePath + ".info"
|
metadataPath := archivePath + ".info"
|
||||||
fmt.Print("📋 Metadata file check... ")
|
fmt.Print("[CHK] Metadata file check... ")
|
||||||
if _, err := os.Stat(metadataPath); os.IsNotExist(err) {
|
if _, err := os.Stat(metadataPath); os.IsNotExist(err) {
|
||||||
fmt.Println("⚠️ WARNING: No metadata file found")
|
fmt.Println("[WARN] WARNING: No metadata file found")
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("✅ PASSED")
|
fmt.Println("[OK] PASSED")
|
||||||
checksPassed++
|
checksPassed++
|
||||||
}
|
}
|
||||||
checksRun++
|
checksRun++
|
||||||
@@ -611,13 +611,13 @@ func runVerify(ctx context.Context, archiveName string) error {
|
|||||||
fmt.Printf("Verification Results: %d/%d checks passed\n", checksPassed, checksRun)
|
fmt.Printf("Verification Results: %d/%d checks passed\n", checksPassed, checksRun)
|
||||||
|
|
||||||
if checksPassed == checksRun {
|
if checksPassed == checksRun {
|
||||||
fmt.Println("🎉 Archive verification completed successfully!")
|
fmt.Println("[SUCCESS] Archive verification completed successfully!")
|
||||||
return nil
|
return nil
|
||||||
} else if float64(checksPassed)/float64(checksRun) >= 0.8 {
|
} else if float64(checksPassed)/float64(checksRun) >= 0.8 {
|
||||||
fmt.Println("⚠️ Archive verification completed with warnings.")
|
fmt.Println("[WARN] Archive verification completed with warnings.")
|
||||||
return nil
|
return nil
|
||||||
} else {
|
} else {
|
||||||
fmt.Println("❌ Archive verification failed. Archive may be corrupted.")
|
fmt.Println("[FAIL] Archive verification failed. Archive may be corrupted.")
|
||||||
return fmt.Errorf("verification failed: %d/%d checks passed", checksPassed, checksRun)
|
return fmt.Errorf("verification failed: %d/%d checks passed", checksPassed, checksRun)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
276
cmd/restore.go
276
cmd/restore.go
@@ -33,6 +33,13 @@ var (
|
|||||||
restoreNoProgress bool
|
restoreNoProgress bool
|
||||||
restoreWorkdir string
|
restoreWorkdir string
|
||||||
restoreCleanCluster bool
|
restoreCleanCluster bool
|
||||||
|
restoreDiagnose bool // Run diagnosis before restore
|
||||||
|
restoreSaveDebugLog string // Path to save debug log on failure
|
||||||
|
|
||||||
|
// Diagnose flags
|
||||||
|
diagnoseJSON bool
|
||||||
|
diagnoseDeep bool
|
||||||
|
diagnoseKeepTemp bool
|
||||||
|
|
||||||
// Encryption flags
|
// Encryption flags
|
||||||
restoreEncryptionKeyFile string
|
restoreEncryptionKeyFile string
|
||||||
@@ -214,12 +221,53 @@ Examples:
|
|||||||
RunE: runRestorePITR,
|
RunE: runRestorePITR,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// restoreDiagnoseCmd diagnoses backup files before restore
|
||||||
|
var restoreDiagnoseCmd = &cobra.Command{
|
||||||
|
Use: "diagnose [archive-file]",
|
||||||
|
Short: "Diagnose backup file integrity and format",
|
||||||
|
Long: `Perform deep analysis of backup files to detect issues before restore.
|
||||||
|
|
||||||
|
This command validates backup archives and provides detailed diagnostics
|
||||||
|
including truncation detection, format verification, and COPY block integrity.
|
||||||
|
|
||||||
|
Use this when:
|
||||||
|
- Restore fails with syntax errors
|
||||||
|
- You suspect backup corruption or truncation
|
||||||
|
- You want to verify backup integrity before restore
|
||||||
|
- Restore reports millions of errors
|
||||||
|
|
||||||
|
Checks performed:
|
||||||
|
- File format detection (custom dump vs SQL)
|
||||||
|
- PGDMP signature verification
|
||||||
|
- Gzip integrity validation
|
||||||
|
- COPY block termination check
|
||||||
|
- pg_restore --list verification
|
||||||
|
- Cluster archive structure validation
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
# Diagnose a single dump file
|
||||||
|
dbbackup restore diagnose mydb.dump.gz
|
||||||
|
|
||||||
|
# Diagnose with verbose output
|
||||||
|
dbbackup restore diagnose mydb.sql.gz --verbose
|
||||||
|
|
||||||
|
# Diagnose cluster archive and all contained dumps
|
||||||
|
dbbackup restore diagnose cluster_backup.tar.gz --deep
|
||||||
|
|
||||||
|
# Output as JSON for scripting
|
||||||
|
dbbackup restore diagnose mydb.dump --json
|
||||||
|
`,
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: runRestoreDiagnose,
|
||||||
|
}
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
rootCmd.AddCommand(restoreCmd)
|
rootCmd.AddCommand(restoreCmd)
|
||||||
restoreCmd.AddCommand(restoreSingleCmd)
|
restoreCmd.AddCommand(restoreSingleCmd)
|
||||||
restoreCmd.AddCommand(restoreClusterCmd)
|
restoreCmd.AddCommand(restoreClusterCmd)
|
||||||
restoreCmd.AddCommand(restoreListCmd)
|
restoreCmd.AddCommand(restoreListCmd)
|
||||||
restoreCmd.AddCommand(restorePITRCmd)
|
restoreCmd.AddCommand(restorePITRCmd)
|
||||||
|
restoreCmd.AddCommand(restoreDiagnoseCmd)
|
||||||
|
|
||||||
// Single restore flags
|
// Single restore flags
|
||||||
restoreSingleCmd.Flags().BoolVar(&restoreConfirm, "confirm", false, "Confirm and execute restore (required)")
|
restoreSingleCmd.Flags().BoolVar(&restoreConfirm, "confirm", false, "Confirm and execute restore (required)")
|
||||||
@@ -232,6 +280,8 @@ func init() {
|
|||||||
restoreSingleCmd.Flags().BoolVar(&restoreNoProgress, "no-progress", false, "Disable progress indicators")
|
restoreSingleCmd.Flags().BoolVar(&restoreNoProgress, "no-progress", false, "Disable progress indicators")
|
||||||
restoreSingleCmd.Flags().StringVar(&restoreEncryptionKeyFile, "encryption-key-file", "", "Path to encryption key file (required for encrypted backups)")
|
restoreSingleCmd.Flags().StringVar(&restoreEncryptionKeyFile, "encryption-key-file", "", "Path to encryption key file (required for encrypted backups)")
|
||||||
restoreSingleCmd.Flags().StringVar(&restoreEncryptionKeyEnv, "encryption-key-env", "DBBACKUP_ENCRYPTION_KEY", "Environment variable containing encryption key")
|
restoreSingleCmd.Flags().StringVar(&restoreEncryptionKeyEnv, "encryption-key-env", "DBBACKUP_ENCRYPTION_KEY", "Environment variable containing encryption key")
|
||||||
|
restoreSingleCmd.Flags().BoolVar(&restoreDiagnose, "diagnose", false, "Run deep diagnosis before restore to detect corruption/truncation")
|
||||||
|
restoreSingleCmd.Flags().StringVar(&restoreSaveDebugLog, "save-debug-log", "", "Save detailed error report to file on failure (e.g., /tmp/restore-debug.json)")
|
||||||
|
|
||||||
// Cluster restore flags
|
// Cluster restore flags
|
||||||
restoreClusterCmd.Flags().BoolVar(&restoreConfirm, "confirm", false, "Confirm and execute restore (required)")
|
restoreClusterCmd.Flags().BoolVar(&restoreConfirm, "confirm", false, "Confirm and execute restore (required)")
|
||||||
@@ -244,6 +294,8 @@ func init() {
|
|||||||
restoreClusterCmd.Flags().BoolVar(&restoreNoProgress, "no-progress", false, "Disable progress indicators")
|
restoreClusterCmd.Flags().BoolVar(&restoreNoProgress, "no-progress", false, "Disable progress indicators")
|
||||||
restoreClusterCmd.Flags().StringVar(&restoreEncryptionKeyFile, "encryption-key-file", "", "Path to encryption key file (required for encrypted backups)")
|
restoreClusterCmd.Flags().StringVar(&restoreEncryptionKeyFile, "encryption-key-file", "", "Path to encryption key file (required for encrypted backups)")
|
||||||
restoreClusterCmd.Flags().StringVar(&restoreEncryptionKeyEnv, "encryption-key-env", "DBBACKUP_ENCRYPTION_KEY", "Environment variable containing encryption key")
|
restoreClusterCmd.Flags().StringVar(&restoreEncryptionKeyEnv, "encryption-key-env", "DBBACKUP_ENCRYPTION_KEY", "Environment variable containing encryption key")
|
||||||
|
restoreClusterCmd.Flags().BoolVar(&restoreDiagnose, "diagnose", false, "Run deep diagnosis on all dumps before restore")
|
||||||
|
restoreClusterCmd.Flags().StringVar(&restoreSaveDebugLog, "save-debug-log", "", "Save detailed error report to file on failure (e.g., /tmp/restore-debug.json)")
|
||||||
|
|
||||||
// PITR restore flags
|
// PITR restore flags
|
||||||
restorePITRCmd.Flags().StringVar(&pitrBaseBackup, "base-backup", "", "Path to base backup file (.tar.gz) (required)")
|
restorePITRCmd.Flags().StringVar(&pitrBaseBackup, "base-backup", "", "Path to base backup file (.tar.gz) (required)")
|
||||||
@@ -264,6 +316,118 @@ func init() {
|
|||||||
restorePITRCmd.MarkFlagRequired("base-backup")
|
restorePITRCmd.MarkFlagRequired("base-backup")
|
||||||
restorePITRCmd.MarkFlagRequired("wal-archive")
|
restorePITRCmd.MarkFlagRequired("wal-archive")
|
||||||
restorePITRCmd.MarkFlagRequired("target-dir")
|
restorePITRCmd.MarkFlagRequired("target-dir")
|
||||||
|
|
||||||
|
// Diagnose flags
|
||||||
|
restoreDiagnoseCmd.Flags().BoolVar(&diagnoseJSON, "json", false, "Output diagnosis as JSON")
|
||||||
|
restoreDiagnoseCmd.Flags().BoolVar(&diagnoseDeep, "deep", false, "For cluster archives, extract and diagnose all contained dumps")
|
||||||
|
restoreDiagnoseCmd.Flags().BoolVar(&diagnoseKeepTemp, "keep-temp", false, "Keep temporary extraction directory (for debugging)")
|
||||||
|
restoreDiagnoseCmd.Flags().BoolVar(&restoreVerbose, "verbose", false, "Show detailed analysis progress")
|
||||||
|
}
|
||||||
|
|
||||||
|
// runRestoreDiagnose diagnoses backup files
|
||||||
|
func runRestoreDiagnose(cmd *cobra.Command, args []string) error {
|
||||||
|
archivePath := args[0]
|
||||||
|
|
||||||
|
// Convert to absolute path
|
||||||
|
if !filepath.IsAbs(archivePath) {
|
||||||
|
absPath, err := filepath.Abs(archivePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid archive path: %w", err)
|
||||||
|
}
|
||||||
|
archivePath = absPath
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if file exists
|
||||||
|
if _, err := os.Stat(archivePath); err != nil {
|
||||||
|
return fmt.Errorf("archive not found: %s", archivePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info("[DIAG] Diagnosing backup file", "path", archivePath)
|
||||||
|
|
||||||
|
diagnoser := restore.NewDiagnoser(log, restoreVerbose)
|
||||||
|
|
||||||
|
// Check if it's a cluster archive that needs deep analysis
|
||||||
|
format := restore.DetectArchiveFormat(archivePath)
|
||||||
|
|
||||||
|
if format.IsClusterBackup() && diagnoseDeep {
|
||||||
|
// Create temp directory for extraction in configured WorkDir
|
||||||
|
workDir := cfg.GetEffectiveWorkDir()
|
||||||
|
tempDir, err := os.MkdirTemp(workDir, "dbbackup-diagnose-*")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create temp directory in %s: %w", workDir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !diagnoseKeepTemp {
|
||||||
|
defer os.RemoveAll(tempDir)
|
||||||
|
} else {
|
||||||
|
log.Info("Temp directory preserved", "path", tempDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info("Extracting cluster archive for deep analysis...")
|
||||||
|
|
||||||
|
// Extract and diagnose all dumps
|
||||||
|
results, err := diagnoser.DiagnoseClusterDumps(archivePath, tempDir)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cluster diagnosis failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Output results
|
||||||
|
var hasErrors bool
|
||||||
|
for _, result := range results {
|
||||||
|
if diagnoseJSON {
|
||||||
|
diagnoser.PrintDiagnosisJSON(result)
|
||||||
|
} else {
|
||||||
|
diagnoser.PrintDiagnosis(result)
|
||||||
|
}
|
||||||
|
if !result.IsValid {
|
||||||
|
hasErrors = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Summary
|
||||||
|
if !diagnoseJSON {
|
||||||
|
fmt.Println("\n" + strings.Repeat("=", 70))
|
||||||
|
fmt.Printf("[SUMMARY] CLUSTER SUMMARY: %d databases analyzed\n", len(results))
|
||||||
|
|
||||||
|
validCount := 0
|
||||||
|
for _, r := range results {
|
||||||
|
if r.IsValid {
|
||||||
|
validCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if validCount == len(results) {
|
||||||
|
fmt.Println("[OK] All dumps are valid")
|
||||||
|
} else {
|
||||||
|
fmt.Printf("[FAIL] %d/%d dumps have issues\n", len(results)-validCount, len(results))
|
||||||
|
}
|
||||||
|
fmt.Println(strings.Repeat("=", 70))
|
||||||
|
}
|
||||||
|
|
||||||
|
if hasErrors {
|
||||||
|
return fmt.Errorf("one or more dumps have validation errors")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Single file diagnosis
|
||||||
|
result, err := diagnoser.DiagnoseFile(archivePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("diagnosis failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if diagnoseJSON {
|
||||||
|
diagnoser.PrintDiagnosisJSON(result)
|
||||||
|
} else {
|
||||||
|
diagnoser.PrintDiagnosis(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !result.IsValid {
|
||||||
|
return fmt.Errorf("backup file has validation errors")
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info("[OK] Backup file appears valid")
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// runRestoreSingle restores a single database
|
// runRestoreSingle restores a single database
|
||||||
@@ -381,7 +545,7 @@ func runRestoreSingle(cmd *cobra.Command, args []string) error {
|
|||||||
isDryRun := restoreDryRun || !restoreConfirm
|
isDryRun := restoreDryRun || !restoreConfirm
|
||||||
|
|
||||||
if isDryRun {
|
if isDryRun {
|
||||||
fmt.Println("\n🔍 DRY-RUN MODE - No changes will be made")
|
fmt.Println("\n[DRY-RUN] DRY-RUN MODE - No changes will be made")
|
||||||
fmt.Printf("\nWould restore:\n")
|
fmt.Printf("\nWould restore:\n")
|
||||||
fmt.Printf(" Archive: %s\n", archivePath)
|
fmt.Printf(" Archive: %s\n", archivePath)
|
||||||
fmt.Printf(" Format: %s\n", format.String())
|
fmt.Printf(" Format: %s\n", format.String())
|
||||||
@@ -401,6 +565,12 @@ func runRestoreSingle(cmd *cobra.Command, args []string) error {
|
|||||||
|
|
||||||
// Create restore engine
|
// Create restore engine
|
||||||
engine := restore.New(cfg, log, db)
|
engine := restore.New(cfg, log, db)
|
||||||
|
|
||||||
|
// Enable debug logging if requested
|
||||||
|
if restoreSaveDebugLog != "" {
|
||||||
|
engine.SetDebugLogPath(restoreSaveDebugLog)
|
||||||
|
log.Info("Debug logging enabled", "output", restoreSaveDebugLog)
|
||||||
|
}
|
||||||
|
|
||||||
// Setup signal handling
|
// Setup signal handling
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
@@ -416,6 +586,37 @@ func runRestoreSingle(cmd *cobra.Command, args []string) error {
|
|||||||
cancel()
|
cancel()
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
// Run pre-restore diagnosis if requested
|
||||||
|
if restoreDiagnose {
|
||||||
|
log.Info("[DIAG] Running pre-restore diagnosis...")
|
||||||
|
|
||||||
|
diagnoser := restore.NewDiagnoser(log, restoreVerbose)
|
||||||
|
result, err := diagnoser.DiagnoseFile(archivePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("diagnosis failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
diagnoser.PrintDiagnosis(result)
|
||||||
|
|
||||||
|
if !result.IsValid {
|
||||||
|
log.Error("[FAIL] Pre-restore diagnosis found issues")
|
||||||
|
if result.IsTruncated {
|
||||||
|
log.Error(" The backup file appears to be TRUNCATED")
|
||||||
|
}
|
||||||
|
if result.IsCorrupted {
|
||||||
|
log.Error(" The backup file appears to be CORRUPTED")
|
||||||
|
}
|
||||||
|
fmt.Println("\nUse --force to attempt restore anyway.")
|
||||||
|
|
||||||
|
if !restoreForce {
|
||||||
|
return fmt.Errorf("aborting restore due to backup file issues")
|
||||||
|
}
|
||||||
|
log.Warn("Continuing despite diagnosis errors (--force enabled)")
|
||||||
|
} else {
|
||||||
|
log.Info("[OK] Backup file passed diagnosis")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Execute restore
|
// Execute restore
|
||||||
log.Info("Starting restore...", "database", targetDB)
|
log.Info("Starting restore...", "database", targetDB)
|
||||||
|
|
||||||
@@ -432,7 +633,7 @@ func runRestoreSingle(cmd *cobra.Command, args []string) error {
|
|||||||
// Audit log: restore success
|
// Audit log: restore success
|
||||||
auditLogger.LogRestoreComplete(user, targetDB, time.Since(startTime))
|
auditLogger.LogRestoreComplete(user, targetDB, time.Since(startTime))
|
||||||
|
|
||||||
log.Info("✅ Restore completed successfully", "database", targetDB)
|
log.Info("[OK] Restore completed successfully", "database", targetDB)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -500,7 +701,7 @@ func runRestoreCluster(cmd *cobra.Command, args []string) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Warn("⚠️ Using alternative working directory for extraction")
|
log.Warn("[WARN] Using alternative working directory for extraction")
|
||||||
log.Warn(" This is recommended when system disk space is limited")
|
log.Warn(" This is recommended when system disk space is limited")
|
||||||
log.Warn(" Location: " + restoreWorkdir)
|
log.Warn(" Location: " + restoreWorkdir)
|
||||||
}
|
}
|
||||||
@@ -553,7 +754,7 @@ func runRestoreCluster(cmd *cobra.Command, args []string) error {
|
|||||||
isDryRun := restoreDryRun || !restoreConfirm
|
isDryRun := restoreDryRun || !restoreConfirm
|
||||||
|
|
||||||
if isDryRun {
|
if isDryRun {
|
||||||
fmt.Println("\n🔍 DRY-RUN MODE - No changes will be made")
|
fmt.Println("\n[DRY-RUN] DRY-RUN MODE - No changes will be made")
|
||||||
fmt.Printf("\nWould restore cluster:\n")
|
fmt.Printf("\nWould restore cluster:\n")
|
||||||
fmt.Printf(" Archive: %s\n", archivePath)
|
fmt.Printf(" Archive: %s\n", archivePath)
|
||||||
fmt.Printf(" Parallel Jobs: %d (0 = auto)\n", restoreJobs)
|
fmt.Printf(" Parallel Jobs: %d (0 = auto)\n", restoreJobs)
|
||||||
@@ -563,7 +764,7 @@ func runRestoreCluster(cmd *cobra.Command, args []string) error {
|
|||||||
if restoreCleanCluster {
|
if restoreCleanCluster {
|
||||||
fmt.Printf(" Clean Cluster: true (will drop %d existing database(s))\n", len(existingDBs))
|
fmt.Printf(" Clean Cluster: true (will drop %d existing database(s))\n", len(existingDBs))
|
||||||
if len(existingDBs) > 0 {
|
if len(existingDBs) > 0 {
|
||||||
fmt.Printf("\n⚠️ Databases to be dropped:\n")
|
fmt.Printf("\n[WARN] Databases to be dropped:\n")
|
||||||
for _, dbName := range existingDBs {
|
for _, dbName := range existingDBs {
|
||||||
fmt.Printf(" - %s\n", dbName)
|
fmt.Printf(" - %s\n", dbName)
|
||||||
}
|
}
|
||||||
@@ -575,7 +776,7 @@ func runRestoreCluster(cmd *cobra.Command, args []string) error {
|
|||||||
|
|
||||||
// Warning for clean-cluster
|
// Warning for clean-cluster
|
||||||
if restoreCleanCluster && len(existingDBs) > 0 {
|
if restoreCleanCluster && len(existingDBs) > 0 {
|
||||||
log.Warn("🔥 Clean cluster mode enabled")
|
log.Warn("[!!] Clean cluster mode enabled")
|
||||||
log.Warn(fmt.Sprintf(" %d existing database(s) will be DROPPED before restore!", len(existingDBs)))
|
log.Warn(fmt.Sprintf(" %d existing database(s) will be DROPPED before restore!", len(existingDBs)))
|
||||||
for _, dbName := range existingDBs {
|
for _, dbName := range existingDBs {
|
||||||
log.Warn(" - " + dbName)
|
log.Warn(" - " + dbName)
|
||||||
@@ -584,6 +785,12 @@ func runRestoreCluster(cmd *cobra.Command, args []string) error {
|
|||||||
|
|
||||||
// Create restore engine
|
// Create restore engine
|
||||||
engine := restore.New(cfg, log, db)
|
engine := restore.New(cfg, log, db)
|
||||||
|
|
||||||
|
// Enable debug logging if requested
|
||||||
|
if restoreSaveDebugLog != "" {
|
||||||
|
engine.SetDebugLogPath(restoreSaveDebugLog)
|
||||||
|
log.Info("Debug logging enabled", "output", restoreSaveDebugLog)
|
||||||
|
}
|
||||||
|
|
||||||
// Setup signal handling
|
// Setup signal handling
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
@@ -620,6 +827,53 @@ func runRestoreCluster(cmd *cobra.Command, args []string) error {
|
|||||||
log.Info("Database cleanup completed")
|
log.Info("Database cleanup completed")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Run pre-restore diagnosis if requested
|
||||||
|
if restoreDiagnose {
|
||||||
|
log.Info("[DIAG] Running pre-restore diagnosis...")
|
||||||
|
|
||||||
|
// Create temp directory for extraction in configured WorkDir
|
||||||
|
workDir := cfg.GetEffectiveWorkDir()
|
||||||
|
diagTempDir, err := os.MkdirTemp(workDir, "dbbackup-diagnose-*")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create temp directory for diagnosis in %s: %w", workDir, err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(diagTempDir)
|
||||||
|
|
||||||
|
diagnoser := restore.NewDiagnoser(log, restoreVerbose)
|
||||||
|
results, err := diagnoser.DiagnoseClusterDumps(archivePath, diagTempDir)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("diagnosis failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for any invalid dumps
|
||||||
|
var invalidDumps []string
|
||||||
|
for _, result := range results {
|
||||||
|
if !result.IsValid {
|
||||||
|
invalidDumps = append(invalidDumps, result.FileName)
|
||||||
|
diagnoser.PrintDiagnosis(result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(invalidDumps) > 0 {
|
||||||
|
log.Error("[FAIL] Pre-restore diagnosis found issues",
|
||||||
|
"invalid_dumps", len(invalidDumps),
|
||||||
|
"total_dumps", len(results))
|
||||||
|
fmt.Println("\n[WARN] The following dumps have issues and will likely fail during restore:")
|
||||||
|
for _, name := range invalidDumps {
|
||||||
|
fmt.Printf(" - %s\n", name)
|
||||||
|
}
|
||||||
|
fmt.Println("\nRun 'dbbackup restore diagnose <archive> --deep' for full details.")
|
||||||
|
fmt.Println("Use --force to attempt restore anyway.")
|
||||||
|
|
||||||
|
if !restoreForce {
|
||||||
|
return fmt.Errorf("aborting restore due to %d invalid dump(s)", len(invalidDumps))
|
||||||
|
}
|
||||||
|
log.Warn("Continuing despite diagnosis errors (--force enabled)")
|
||||||
|
} else {
|
||||||
|
log.Info("[OK] All dumps passed diagnosis", "count", len(results))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Execute cluster restore
|
// Execute cluster restore
|
||||||
log.Info("Starting cluster restore...")
|
log.Info("Starting cluster restore...")
|
||||||
|
|
||||||
@@ -636,7 +890,7 @@ func runRestoreCluster(cmd *cobra.Command, args []string) error {
|
|||||||
// Audit log: restore success
|
// Audit log: restore success
|
||||||
auditLogger.LogRestoreComplete(user, "all_databases", time.Since(startTime))
|
auditLogger.LogRestoreComplete(user, "all_databases", time.Since(startTime))
|
||||||
|
|
||||||
log.Info("✅ Cluster restore completed successfully")
|
log.Info("[OK] Cluster restore completed successfully")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -685,7 +939,7 @@ func runRestoreList(cmd *cobra.Command, args []string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Print header
|
// Print header
|
||||||
fmt.Printf("\n📦 Available backup archives in %s\n\n", backupDir)
|
fmt.Printf("\n[LIST] Available backup archives in %s\n\n", backupDir)
|
||||||
fmt.Printf("%-40s %-25s %-12s %-20s %s\n",
|
fmt.Printf("%-40s %-25s %-12s %-20s %s\n",
|
||||||
"FILENAME", "FORMAT", "SIZE", "MODIFIED", "DATABASE")
|
"FILENAME", "FORMAT", "SIZE", "MODIFIED", "DATABASE")
|
||||||
fmt.Println(strings.Repeat("-", 120))
|
fmt.Println(strings.Repeat("-", 120))
|
||||||
@@ -802,9 +1056,9 @@ func runRestorePITR(cmd *cobra.Command, args []string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Display recovery target info
|
// Display recovery target info
|
||||||
log.Info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
log.Info("=====================================================")
|
||||||
log.Info(" Point-in-Time Recovery (PITR)")
|
log.Info(" Point-in-Time Recovery (PITR)")
|
||||||
log.Info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
log.Info("=====================================================")
|
||||||
log.Info("")
|
log.Info("")
|
||||||
log.Info(target.String())
|
log.Info(target.String())
|
||||||
log.Info("")
|
log.Info("")
|
||||||
@@ -828,6 +1082,6 @@ func runRestorePITR(cmd *cobra.Command, args []string) error {
|
|||||||
return fmt.Errorf("PITR restore failed: %w", err)
|
return fmt.Errorf("PITR restore failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("✅ PITR restore completed successfully")
|
log.Info("[OK] PITR restore completed successfully")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
56
cmd/rto.go
56
cmd/rto.go
@@ -181,13 +181,13 @@ func runRTOStatus(cmd *cobra.Command, args []string) error {
|
|||||||
|
|
||||||
// Display status
|
// Display status
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
fmt.Println("╔═══════════════════════════════════════════════════════════╗")
|
fmt.Println("+-----------------------------------------------------------+")
|
||||||
fmt.Println("║ RTO/RPO STATUS SUMMARY ║")
|
fmt.Println("| RTO/RPO STATUS SUMMARY |")
|
||||||
fmt.Println("╠═══════════════════════════════════════════════════════════╣")
|
fmt.Println("+-----------------------------------------------------------+")
|
||||||
fmt.Printf("║ Target RTO: %-15s Target RPO: %-15s ║\n",
|
fmt.Printf("| Target RTO: %-15s Target RPO: %-15s |\n",
|
||||||
formatDuration(config.TargetRTO),
|
formatDuration(config.TargetRTO),
|
||||||
formatDuration(config.TargetRPO))
|
formatDuration(config.TargetRPO))
|
||||||
fmt.Println("╠═══════════════════════════════════════════════════════════╣")
|
fmt.Println("+-----------------------------------------------------------+")
|
||||||
|
|
||||||
// Compliance status
|
// Compliance status
|
||||||
rpoRate := 0.0
|
rpoRate := 0.0
|
||||||
@@ -199,31 +199,31 @@ func runRTOStatus(cmd *cobra.Command, args []string) error {
|
|||||||
fullRate = float64(summary.FullyCompliant) / float64(summary.TotalDatabases) * 100
|
fullRate = float64(summary.FullyCompliant) / float64(summary.TotalDatabases) * 100
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("║ Databases: %-5d ║\n", summary.TotalDatabases)
|
fmt.Printf("| Databases: %-5d |\n", summary.TotalDatabases)
|
||||||
fmt.Printf("║ RPO Compliant: %-5d (%.0f%%) ║\n", summary.RPOCompliant, rpoRate)
|
fmt.Printf("| RPO Compliant: %-5d (%.0f%%) |\n", summary.RPOCompliant, rpoRate)
|
||||||
fmt.Printf("║ RTO Compliant: %-5d (%.0f%%) ║\n", summary.RTOCompliant, rtoRate)
|
fmt.Printf("| RTO Compliant: %-5d (%.0f%%) |\n", summary.RTOCompliant, rtoRate)
|
||||||
fmt.Printf("║ Fully Compliant: %-3d (%.0f%%) ║\n", summary.FullyCompliant, fullRate)
|
fmt.Printf("| Fully Compliant: %-3d (%.0f%%) |\n", summary.FullyCompliant, fullRate)
|
||||||
|
|
||||||
if summary.CriticalIssues > 0 {
|
if summary.CriticalIssues > 0 {
|
||||||
fmt.Printf("║ ⚠️ Critical Issues: %-3d ║\n", summary.CriticalIssues)
|
fmt.Printf("| [WARN] Critical Issues: %-3d |\n", summary.CriticalIssues)
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println("╠═══════════════════════════════════════════════════════════╣")
|
fmt.Println("+-----------------------------------------------------------+")
|
||||||
fmt.Printf("║ Average RPO: %-15s Worst: %-15s ║\n",
|
fmt.Printf("| Average RPO: %-15s Worst: %-15s |\n",
|
||||||
formatDuration(summary.AverageRPO),
|
formatDuration(summary.AverageRPO),
|
||||||
formatDuration(summary.WorstRPO))
|
formatDuration(summary.WorstRPO))
|
||||||
fmt.Printf("║ Average RTO: %-15s Worst: %-15s ║\n",
|
fmt.Printf("| Average RTO: %-15s Worst: %-15s |\n",
|
||||||
formatDuration(summary.AverageRTO),
|
formatDuration(summary.AverageRTO),
|
||||||
formatDuration(summary.WorstRTO))
|
formatDuration(summary.WorstRTO))
|
||||||
|
|
||||||
if summary.WorstRPODatabase != "" {
|
if summary.WorstRPODatabase != "" {
|
||||||
fmt.Printf("║ Worst RPO Database: %-38s║\n", summary.WorstRPODatabase)
|
fmt.Printf("| Worst RPO Database: %-38s|\n", summary.WorstRPODatabase)
|
||||||
}
|
}
|
||||||
if summary.WorstRTODatabase != "" {
|
if summary.WorstRTODatabase != "" {
|
||||||
fmt.Printf("║ Worst RTO Database: %-38s║\n", summary.WorstRTODatabase)
|
fmt.Printf("| Worst RTO Database: %-38s|\n", summary.WorstRTODatabase)
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println("╚═══════════════════════════════════════════════════════════╝")
|
fmt.Println("+-----------------------------------------------------------+")
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
|
|
||||||
// Per-database status
|
// Per-database status
|
||||||
@@ -234,19 +234,19 @@ func runRTOStatus(cmd *cobra.Command, args []string) error {
|
|||||||
fmt.Println(strings.Repeat("-", 70))
|
fmt.Println(strings.Repeat("-", 70))
|
||||||
|
|
||||||
for _, a := range analyses {
|
for _, a := range analyses {
|
||||||
status := "✅"
|
status := "[OK]"
|
||||||
if !a.RPOCompliant || !a.RTOCompliant {
|
if !a.RPOCompliant || !a.RTOCompliant {
|
||||||
status = "❌"
|
status = "[FAIL]"
|
||||||
}
|
}
|
||||||
|
|
||||||
rpoStr := formatDuration(a.CurrentRPO)
|
rpoStr := formatDuration(a.CurrentRPO)
|
||||||
rtoStr := formatDuration(a.CurrentRTO)
|
rtoStr := formatDuration(a.CurrentRTO)
|
||||||
|
|
||||||
if !a.RPOCompliant {
|
if !a.RPOCompliant {
|
||||||
rpoStr = "⚠️ " + rpoStr
|
rpoStr = "[WARN] " + rpoStr
|
||||||
}
|
}
|
||||||
if !a.RTOCompliant {
|
if !a.RTOCompliant {
|
||||||
rtoStr = "⚠️ " + rtoStr
|
rtoStr = "[WARN] " + rtoStr
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("%-25s %-12s %-12s %s\n",
|
fmt.Printf("%-25s %-12s %-12s %s\n",
|
||||||
@@ -306,21 +306,21 @@ func runRTOCheck(cmd *cobra.Command, args []string) error {
|
|||||||
exitCode := 0
|
exitCode := 0
|
||||||
for _, a := range analyses {
|
for _, a := range analyses {
|
||||||
if !a.RPOCompliant {
|
if !a.RPOCompliant {
|
||||||
fmt.Printf("❌ %s: RPO violation - current %s exceeds target %s\n",
|
fmt.Printf("[FAIL] %s: RPO violation - current %s exceeds target %s\n",
|
||||||
a.Database,
|
a.Database,
|
||||||
formatDuration(a.CurrentRPO),
|
formatDuration(a.CurrentRPO),
|
||||||
formatDuration(config.TargetRPO))
|
formatDuration(config.TargetRPO))
|
||||||
exitCode = 1
|
exitCode = 1
|
||||||
}
|
}
|
||||||
if !a.RTOCompliant {
|
if !a.RTOCompliant {
|
||||||
fmt.Printf("❌ %s: RTO violation - estimated %s exceeds target %s\n",
|
fmt.Printf("[FAIL] %s: RTO violation - estimated %s exceeds target %s\n",
|
||||||
a.Database,
|
a.Database,
|
||||||
formatDuration(a.CurrentRTO),
|
formatDuration(a.CurrentRTO),
|
||||||
formatDuration(config.TargetRTO))
|
formatDuration(config.TargetRTO))
|
||||||
exitCode = 1
|
exitCode = 1
|
||||||
}
|
}
|
||||||
if a.RPOCompliant && a.RTOCompliant {
|
if a.RPOCompliant && a.RTOCompliant {
|
||||||
fmt.Printf("✅ %s: Compliant (RPO: %s, RTO: %s)\n",
|
fmt.Printf("[OK] %s: Compliant (RPO: %s, RTO: %s)\n",
|
||||||
a.Database,
|
a.Database,
|
||||||
formatDuration(a.CurrentRPO),
|
formatDuration(a.CurrentRPO),
|
||||||
formatDuration(a.CurrentRTO))
|
formatDuration(a.CurrentRTO))
|
||||||
@@ -371,13 +371,13 @@ func outputAnalysisText(analyses []*rto.Analysis) error {
|
|||||||
fmt.Println(strings.Repeat("=", 60))
|
fmt.Println(strings.Repeat("=", 60))
|
||||||
|
|
||||||
// Status
|
// Status
|
||||||
rpoStatus := "✅ Compliant"
|
rpoStatus := "[OK] Compliant"
|
||||||
if !a.RPOCompliant {
|
if !a.RPOCompliant {
|
||||||
rpoStatus = "❌ Violation"
|
rpoStatus = "[FAIL] Violation"
|
||||||
}
|
}
|
||||||
rtoStatus := "✅ Compliant"
|
rtoStatus := "[OK] Compliant"
|
||||||
if !a.RTOCompliant {
|
if !a.RTOCompliant {
|
||||||
rtoStatus = "❌ Violation"
|
rtoStatus = "[FAIL] Violation"
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
@@ -420,7 +420,7 @@ func outputAnalysisText(analyses []*rto.Analysis) error {
|
|||||||
fmt.Println(" Recommendations:")
|
fmt.Println(" Recommendations:")
|
||||||
fmt.Println(strings.Repeat("-", 50))
|
fmt.Println(strings.Repeat("-", 50))
|
||||||
for _, r := range a.Recommendations {
|
for _, r := range a.Recommendations {
|
||||||
icon := "💡"
|
icon := "[TIP]"
|
||||||
switch r.Priority {
|
switch r.Priority {
|
||||||
case rto.PriorityCritical:
|
case rto.PriorityCritical:
|
||||||
icon = "🔴"
|
icon = "🔴"
|
||||||
|
|||||||
@@ -141,7 +141,7 @@ func testConnection(ctx context.Context) error {
|
|||||||
|
|
||||||
// Display results
|
// Display results
|
||||||
fmt.Println("Connection Test Results:")
|
fmt.Println("Connection Test Results:")
|
||||||
fmt.Printf(" Status: Connected ✅\n")
|
fmt.Printf(" Status: Connected [OK]\n")
|
||||||
fmt.Printf(" Version: %s\n", version)
|
fmt.Printf(" Version: %s\n", version)
|
||||||
fmt.Printf(" Databases: %d found\n", len(databases))
|
fmt.Printf(" Databases: %d found\n", len(databases))
|
||||||
|
|
||||||
@@ -167,7 +167,7 @@ func testConnection(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
fmt.Println("✅ Status check completed successfully!")
|
fmt.Println("[OK] Status check completed successfully!")
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -96,17 +96,17 @@ func runVerifyBackup(cmd *cobra.Command, args []string) error {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("📁 %s\n", filepath.Base(backupFile))
|
fmt.Printf("[FILE] %s\n", filepath.Base(backupFile))
|
||||||
|
|
||||||
if quickVerify {
|
if quickVerify {
|
||||||
// Quick check: size only
|
// Quick check: size only
|
||||||
err := verification.QuickCheck(backupFile)
|
err := verification.QuickCheck(backupFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf(" ❌ FAILED: %v\n\n", err)
|
fmt.Printf(" [FAIL] FAILED: %v\n\n", err)
|
||||||
failureCount++
|
failureCount++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
fmt.Printf(" ✅ VALID (quick check)\n\n")
|
fmt.Printf(" [OK] VALID (quick check)\n\n")
|
||||||
successCount++
|
successCount++
|
||||||
} else {
|
} else {
|
||||||
// Full verification with SHA-256
|
// Full verification with SHA-256
|
||||||
@@ -116,7 +116,7 @@ func runVerifyBackup(cmd *cobra.Command, args []string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if result.Valid {
|
if result.Valid {
|
||||||
fmt.Printf(" ✅ VALID\n")
|
fmt.Printf(" [OK] VALID\n")
|
||||||
if verboseVerify {
|
if verboseVerify {
|
||||||
meta, _ := metadata.Load(backupFile)
|
meta, _ := metadata.Load(backupFile)
|
||||||
fmt.Printf(" Size: %s\n", metadata.FormatSize(meta.SizeBytes))
|
fmt.Printf(" Size: %s\n", metadata.FormatSize(meta.SizeBytes))
|
||||||
@@ -127,7 +127,7 @@ func runVerifyBackup(cmd *cobra.Command, args []string) error {
|
|||||||
fmt.Println()
|
fmt.Println()
|
||||||
successCount++
|
successCount++
|
||||||
} else {
|
} else {
|
||||||
fmt.Printf(" ❌ FAILED: %v\n", result.Error)
|
fmt.Printf(" [FAIL] FAILED: %v\n", result.Error)
|
||||||
if verboseVerify {
|
if verboseVerify {
|
||||||
if !result.FileExists {
|
if !result.FileExists {
|
||||||
fmt.Printf(" File does not exist\n")
|
fmt.Printf(" File does not exist\n")
|
||||||
@@ -147,11 +147,11 @@ func runVerifyBackup(cmd *cobra.Command, args []string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Summary
|
// Summary
|
||||||
fmt.Println(strings.Repeat("─", 50))
|
fmt.Println(strings.Repeat("-", 50))
|
||||||
fmt.Printf("Total: %d backups\n", len(backupFiles))
|
fmt.Printf("Total: %d backups\n", len(backupFiles))
|
||||||
fmt.Printf("✅ Valid: %d\n", successCount)
|
fmt.Printf("[OK] Valid: %d\n", successCount)
|
||||||
if failureCount > 0 {
|
if failureCount > 0 {
|
||||||
fmt.Printf("❌ Failed: %d\n", failureCount)
|
fmt.Printf("[FAIL] Failed: %d\n", failureCount)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -195,16 +195,16 @@ func runVerifyCloudBackup(cmd *cobra.Command, args []string) error {
|
|||||||
|
|
||||||
for _, uri := range args {
|
for _, uri := range args {
|
||||||
if !isCloudURI(uri) {
|
if !isCloudURI(uri) {
|
||||||
fmt.Printf("⚠️ Skipping non-cloud URI: %s\n", uri)
|
fmt.Printf("[WARN] Skipping non-cloud URI: %s\n", uri)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("☁️ %s\n", uri)
|
fmt.Printf("[CLOUD] %s\n", uri)
|
||||||
|
|
||||||
// Download and verify
|
// Download and verify
|
||||||
result, err := verifyCloudBackup(cmd.Context(), uri, quickVerify, verboseVerify)
|
result, err := verifyCloudBackup(cmd.Context(), uri, quickVerify, verboseVerify)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf(" ❌ FAILED: %v\n\n", err)
|
fmt.Printf(" [FAIL] FAILED: %v\n\n", err)
|
||||||
failureCount++
|
failureCount++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -212,7 +212,7 @@ func runVerifyCloudBackup(cmd *cobra.Command, args []string) error {
|
|||||||
// Cleanup temp file
|
// Cleanup temp file
|
||||||
defer result.Cleanup()
|
defer result.Cleanup()
|
||||||
|
|
||||||
fmt.Printf(" ✅ VALID\n")
|
fmt.Printf(" [OK] VALID\n")
|
||||||
if verboseVerify && result.MetadataPath != "" {
|
if verboseVerify && result.MetadataPath != "" {
|
||||||
meta, _ := metadata.Load(result.MetadataPath)
|
meta, _ := metadata.Load(result.MetadataPath)
|
||||||
if meta != nil {
|
if meta != nil {
|
||||||
@@ -226,7 +226,7 @@ func runVerifyCloudBackup(cmd *cobra.Command, args []string) error {
|
|||||||
successCount++
|
successCount++
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("\n✅ Summary: %d valid, %d failed\n", successCount, failureCount)
|
fmt.Printf("\n[OK] Summary: %d valid, %d failed\n", successCount, failureCount)
|
||||||
|
|
||||||
if failureCount > 0 {
|
if failureCount > 0 {
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
|
|||||||
@@ -2,12 +2,14 @@ package auth
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"dbbackup/internal/config"
|
"dbbackup/internal/config"
|
||||||
)
|
)
|
||||||
@@ -69,7 +71,10 @@ func checkPgHbaConf(user string) AuthMethod {
|
|||||||
|
|
||||||
// findHbaFileViaPostgres asks PostgreSQL for the hba_file location
|
// findHbaFileViaPostgres asks PostgreSQL for the hba_file location
|
||||||
func findHbaFileViaPostgres() string {
|
func findHbaFileViaPostgres() string {
|
||||||
cmd := exec.Command("psql", "-U", "postgres", "-t", "-c", "SHOW hba_file;")
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
cmd := exec.CommandContext(ctx, "psql", "-U", "postgres", "-t", "-c", "SHOW hba_file;")
|
||||||
output, err := cmd.Output()
|
output, err := cmd.Output()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ""
|
return ""
|
||||||
@@ -82,8 +87,11 @@ func parsePgHbaConf(path string, user string) AuthMethod {
|
|||||||
// Try with sudo if we can't read directly
|
// Try with sudo if we can't read directly
|
||||||
file, err := os.Open(path)
|
file, err := os.Open(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Try with sudo
|
// Try with sudo (with timeout)
|
||||||
cmd := exec.Command("sudo", "cat", path)
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
cmd := exec.CommandContext(ctx, "sudo", "cat", path)
|
||||||
output, err := cmd.Output()
|
output, err := cmd.Output()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return AuthUnknown
|
return AuthUnknown
|
||||||
@@ -196,13 +204,13 @@ func CheckAuthenticationMismatch(cfg *config.Config) (bool, string) {
|
|||||||
func buildAuthMismatchMessage(osUser, dbUser string, method AuthMethod) string {
|
func buildAuthMismatchMessage(osUser, dbUser string, method AuthMethod) string {
|
||||||
var msg strings.Builder
|
var msg strings.Builder
|
||||||
|
|
||||||
msg.WriteString("\n⚠️ Authentication Mismatch Detected\n")
|
msg.WriteString("\n[WARN] Authentication Mismatch Detected\n")
|
||||||
msg.WriteString(strings.Repeat("=", 60) + "\n\n")
|
msg.WriteString(strings.Repeat("=", 60) + "\n\n")
|
||||||
|
|
||||||
msg.WriteString(fmt.Sprintf(" PostgreSQL is using '%s' authentication\n", method))
|
msg.WriteString(fmt.Sprintf(" PostgreSQL is using '%s' authentication\n", method))
|
||||||
msg.WriteString(fmt.Sprintf(" OS user '%s' cannot authenticate as DB user '%s'\n\n", osUser, dbUser))
|
msg.WriteString(fmt.Sprintf(" OS user '%s' cannot authenticate as DB user '%s'\n\n", osUser, dbUser))
|
||||||
|
|
||||||
msg.WriteString("💡 Solutions (choose one):\n\n")
|
msg.WriteString("[TIP] Solutions (choose one):\n\n")
|
||||||
|
|
||||||
msg.WriteString(fmt.Sprintf(" 1. Run as matching user:\n"))
|
msg.WriteString(fmt.Sprintf(" 1. Run as matching user:\n"))
|
||||||
msg.WriteString(fmt.Sprintf(" sudo -u %s %s\n\n", dbUser, getCommandLine()))
|
msg.WriteString(fmt.Sprintf(" sudo -u %s %s\n\n", dbUser, getCommandLine()))
|
||||||
@@ -218,7 +226,7 @@ func buildAuthMismatchMessage(osUser, dbUser string, method AuthMethod) string {
|
|||||||
msg.WriteString(" 4. Provide password via flag:\n")
|
msg.WriteString(" 4. Provide password via flag:\n")
|
||||||
msg.WriteString(fmt.Sprintf(" %s --password your_password\n\n", getCommandLine()))
|
msg.WriteString(fmt.Sprintf(" %s --password your_password\n\n", getCommandLine()))
|
||||||
|
|
||||||
msg.WriteString("📝 Note: For production use, ~/.pgpass or PGPASSWORD are recommended\n")
|
msg.WriteString("[NOTE] Note: For production use, ~/.pgpass or PGPASSWORD are recommended\n")
|
||||||
msg.WriteString(" to avoid exposing passwords in command history.\n\n")
|
msg.WriteString(" to avoid exposing passwords in command history.\n\n")
|
||||||
|
|
||||||
msg.WriteString(strings.Repeat("=", 60) + "\n")
|
msg.WriteString(strings.Repeat("=", 60) + "\n")
|
||||||
|
|||||||
@@ -87,20 +87,46 @@ func IsBackupEncrypted(backupPath string) bool {
|
|||||||
return meta.Encrypted
|
return meta.Encrypted
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fallback: check if file starts with encryption nonce
|
// No metadata found - check file format to determine if encrypted
|
||||||
|
// Known unencrypted formats have specific magic bytes:
|
||||||
|
// - Gzip: 1f 8b
|
||||||
|
// - PGDMP (PostgreSQL custom): 50 47 44 4d 50 (PGDMP)
|
||||||
|
// - Plain SQL: starts with text (-- or SET or CREATE)
|
||||||
|
// - Tar: 75 73 74 61 72 (ustar) at offset 257
|
||||||
|
//
|
||||||
|
// If file doesn't match any known format, it MIGHT be encrypted,
|
||||||
|
// but we return false to avoid false positives. User must provide
|
||||||
|
// metadata file or use --encrypt flag explicitly.
|
||||||
file, err := os.Open(backupPath)
|
file, err := os.Open(backupPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
|
|
||||||
// Try to read nonce - if it succeeds, likely encrypted
|
header := make([]byte, 6)
|
||||||
nonce := make([]byte, crypto.NonceSize)
|
if n, err := file.Read(header); err != nil || n < 2 {
|
||||||
if n, err := file.Read(nonce); err != nil || n != crypto.NonceSize {
|
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
return true
|
// Check for known unencrypted formats
|
||||||
|
// Gzip magic: 1f 8b
|
||||||
|
if header[0] == 0x1f && header[1] == 0x8b {
|
||||||
|
return false // Gzip compressed - not encrypted
|
||||||
|
}
|
||||||
|
|
||||||
|
// PGDMP magic (PostgreSQL custom format)
|
||||||
|
if len(header) >= 5 && string(header[:5]) == "PGDMP" {
|
||||||
|
return false // PostgreSQL custom dump - not encrypted
|
||||||
|
}
|
||||||
|
|
||||||
|
// Plain text SQL (starts with --, SET, CREATE, etc.)
|
||||||
|
if header[0] == '-' || header[0] == 'S' || header[0] == 'C' || header[0] == '/' {
|
||||||
|
return false // Plain text SQL - not encrypted
|
||||||
|
}
|
||||||
|
|
||||||
|
// Without metadata, we cannot reliably determine encryption status
|
||||||
|
// Return false to avoid blocking restores with false positives
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// DecryptBackupFile decrypts an encrypted backup file
|
// DecryptBackupFile decrypts an encrypted backup file
|
||||||
|
|||||||
@@ -443,6 +443,14 @@ func (e *Engine) BackupCluster(ctx context.Context) error {
|
|||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
defer func() { <-semaphore }() // Release
|
defer func() { <-semaphore }() // Release
|
||||||
|
|
||||||
|
// Panic recovery - prevent one database failure from crashing entire cluster backup
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
e.log.Error("Panic in database backup goroutine", "database", name, "panic", r)
|
||||||
|
atomic.AddInt32(&failCount, 1)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
// Check for cancellation at start of goroutine
|
// Check for cancellation at start of goroutine
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
@@ -465,7 +473,7 @@ func (e *Engine) BackupCluster(ctx context.Context) error {
|
|||||||
mu.Lock()
|
mu.Lock()
|
||||||
e.printf(" Database size: %s\n", sizeStr)
|
e.printf(" Database size: %s\n", sizeStr)
|
||||||
if size > 10*1024*1024*1024 { // > 10GB
|
if size > 10*1024*1024*1024 { // > 10GB
|
||||||
e.printf(" ⚠️ Large database detected - this may take a while\n")
|
e.printf(" [WARN] Large database detected - this may take a while\n")
|
||||||
}
|
}
|
||||||
mu.Unlock()
|
mu.Unlock()
|
||||||
}
|
}
|
||||||
@@ -502,24 +510,24 @@ func (e *Engine) BackupCluster(ctx context.Context) error {
|
|||||||
|
|
||||||
cmd := e.db.BuildBackupCommand(name, dumpFile, options)
|
cmd := e.db.BuildBackupCommand(name, dumpFile, options)
|
||||||
|
|
||||||
dbCtx, cancel := context.WithTimeout(ctx, 2*time.Hour)
|
// NO TIMEOUT for individual database backups
|
||||||
defer cancel()
|
// Large databases with large objects can take many hours
|
||||||
err := e.executeCommand(dbCtx, cmd, dumpFile)
|
// The parent context handles cancellation if needed
|
||||||
cancel()
|
err := e.executeCommand(ctx, cmd, dumpFile)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
e.log.Warn("Failed to backup database", "database", name, "error", err)
|
e.log.Warn("Failed to backup database", "database", name, "error", err)
|
||||||
mu.Lock()
|
mu.Lock()
|
||||||
e.printf(" ⚠️ WARNING: Failed to backup %s: %v\n", name, err)
|
e.printf(" [WARN] WARNING: Failed to backup %s: %v\n", name, err)
|
||||||
mu.Unlock()
|
mu.Unlock()
|
||||||
atomic.AddInt32(&failCount, 1)
|
atomic.AddInt32(&failCount, 1)
|
||||||
} else {
|
} else {
|
||||||
compressedCandidate := strings.TrimSuffix(dumpFile, ".dump") + ".sql.gz"
|
compressedCandidate := strings.TrimSuffix(dumpFile, ".dump") + ".sql.gz"
|
||||||
mu.Lock()
|
mu.Lock()
|
||||||
if info, err := os.Stat(compressedCandidate); err == nil {
|
if info, err := os.Stat(compressedCandidate); err == nil {
|
||||||
e.printf(" ✅ Completed %s (%s)\n", name, formatBytes(info.Size()))
|
e.printf(" [OK] Completed %s (%s)\n", name, formatBytes(info.Size()))
|
||||||
} else if info, err := os.Stat(dumpFile); err == nil {
|
} else if info, err := os.Stat(dumpFile); err == nil {
|
||||||
e.printf(" ✅ Completed %s (%s)\n", name, formatBytes(info.Size()))
|
e.printf(" [OK] Completed %s (%s)\n", name, formatBytes(info.Size()))
|
||||||
}
|
}
|
||||||
mu.Unlock()
|
mu.Unlock()
|
||||||
atomic.AddInt32(&successCount, 1)
|
atomic.AddInt32(&successCount, 1)
|
||||||
@@ -598,12 +606,36 @@ func (e *Engine) executeCommandWithProgress(ctx context.Context, cmdArgs []strin
|
|||||||
return fmt.Errorf("failed to start command: %w", err)
|
return fmt.Errorf("failed to start command: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Monitor progress via stderr
|
// Monitor progress via stderr in goroutine
|
||||||
go e.monitorCommandProgress(stderr, tracker)
|
stderrDone := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
defer close(stderrDone)
|
||||||
|
e.monitorCommandProgress(stderr, tracker)
|
||||||
|
}()
|
||||||
|
|
||||||
// Wait for command to complete
|
// Wait for command to complete with proper context handling
|
||||||
if err := cmd.Wait(); err != nil {
|
cmdDone := make(chan error, 1)
|
||||||
return fmt.Errorf("backup command failed: %w", err)
|
go func() {
|
||||||
|
cmdDone <- cmd.Wait()
|
||||||
|
}()
|
||||||
|
|
||||||
|
var cmdErr error
|
||||||
|
select {
|
||||||
|
case cmdErr = <-cmdDone:
|
||||||
|
// Command completed (success or failure)
|
||||||
|
case <-ctx.Done():
|
||||||
|
// Context cancelled - kill process to unblock
|
||||||
|
e.log.Warn("Backup cancelled - killing process")
|
||||||
|
cmd.Process.Kill()
|
||||||
|
<-cmdDone // Wait for goroutine to finish
|
||||||
|
cmdErr = ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for stderr reader to finish
|
||||||
|
<-stderrDone
|
||||||
|
|
||||||
|
if cmdErr != nil {
|
||||||
|
return fmt.Errorf("backup command failed: %w", cmdErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -680,8 +712,12 @@ func (e *Engine) executeMySQLWithProgressAndCompression(ctx context.Context, cmd
|
|||||||
return fmt.Errorf("failed to get stderr pipe: %w", err)
|
return fmt.Errorf("failed to get stderr pipe: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start monitoring progress
|
// Start monitoring progress in goroutine
|
||||||
go e.monitorCommandProgress(stderr, tracker)
|
stderrDone := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
defer close(stderrDone)
|
||||||
|
e.monitorCommandProgress(stderr, tracker)
|
||||||
|
}()
|
||||||
|
|
||||||
// Start both commands
|
// Start both commands
|
||||||
if err := gzipCmd.Start(); err != nil {
|
if err := gzipCmd.Start(); err != nil {
|
||||||
@@ -689,20 +725,41 @@ func (e *Engine) executeMySQLWithProgressAndCompression(ctx context.Context, cmd
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err := dumpCmd.Start(); err != nil {
|
if err := dumpCmd.Start(); err != nil {
|
||||||
|
gzipCmd.Process.Kill()
|
||||||
return fmt.Errorf("failed to start mysqldump: %w", err)
|
return fmt.Errorf("failed to start mysqldump: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wait for mysqldump to complete
|
// Wait for mysqldump with context handling
|
||||||
if err := dumpCmd.Wait(); err != nil {
|
dumpDone := make(chan error, 1)
|
||||||
return fmt.Errorf("mysqldump failed: %w", err)
|
go func() {
|
||||||
|
dumpDone <- dumpCmd.Wait()
|
||||||
|
}()
|
||||||
|
|
||||||
|
var dumpErr error
|
||||||
|
select {
|
||||||
|
case dumpErr = <-dumpDone:
|
||||||
|
// mysqldump completed
|
||||||
|
case <-ctx.Done():
|
||||||
|
e.log.Warn("Backup cancelled - killing mysqldump")
|
||||||
|
dumpCmd.Process.Kill()
|
||||||
|
gzipCmd.Process.Kill()
|
||||||
|
<-dumpDone
|
||||||
|
return ctx.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Wait for stderr reader
|
||||||
|
<-stderrDone
|
||||||
|
|
||||||
// Close pipe and wait for gzip
|
// Close pipe and wait for gzip
|
||||||
pipe.Close()
|
pipe.Close()
|
||||||
if err := gzipCmd.Wait(); err != nil {
|
if err := gzipCmd.Wait(); err != nil {
|
||||||
return fmt.Errorf("gzip failed: %w", err)
|
return fmt.Errorf("gzip failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if dumpErr != nil {
|
||||||
|
return fmt.Errorf("mysqldump failed: %w", dumpErr)
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -733,19 +790,45 @@ func (e *Engine) executeMySQLWithCompression(ctx context.Context, cmdArgs []stri
|
|||||||
gzipCmd.Stdin = stdin
|
gzipCmd.Stdin = stdin
|
||||||
gzipCmd.Stdout = outFile
|
gzipCmd.Stdout = outFile
|
||||||
|
|
||||||
// Start both commands
|
// Start gzip first
|
||||||
if err := gzipCmd.Start(); err != nil {
|
if err := gzipCmd.Start(); err != nil {
|
||||||
return fmt.Errorf("failed to start gzip: %w", err)
|
return fmt.Errorf("failed to start gzip: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := dumpCmd.Run(); err != nil {
|
// Start mysqldump
|
||||||
return fmt.Errorf("mysqldump failed: %w", err)
|
if err := dumpCmd.Start(); err != nil {
|
||||||
|
gzipCmd.Process.Kill()
|
||||||
|
return fmt.Errorf("failed to start mysqldump: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Wait for mysqldump with context handling
|
||||||
|
dumpDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
dumpDone <- dumpCmd.Wait()
|
||||||
|
}()
|
||||||
|
|
||||||
|
var dumpErr error
|
||||||
|
select {
|
||||||
|
case dumpErr = <-dumpDone:
|
||||||
|
// mysqldump completed
|
||||||
|
case <-ctx.Done():
|
||||||
|
e.log.Warn("Backup cancelled - killing mysqldump")
|
||||||
|
dumpCmd.Process.Kill()
|
||||||
|
gzipCmd.Process.Kill()
|
||||||
|
<-dumpDone
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close pipe and wait for gzip
|
||||||
|
stdin.Close()
|
||||||
if err := gzipCmd.Wait(); err != nil {
|
if err := gzipCmd.Wait(); err != nil {
|
||||||
return fmt.Errorf("gzip failed: %w", err)
|
return fmt.Errorf("gzip failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if dumpErr != nil {
|
||||||
|
return fmt.Errorf("mysqldump failed: %w", dumpErr)
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -882,15 +965,46 @@ func (e *Engine) createArchive(ctx context.Context, sourceDir, outputFile string
|
|||||||
goto regularTar
|
goto regularTar
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wait for tar to finish
|
// Wait for tar with proper context handling
|
||||||
if err := cmd.Wait(); err != nil {
|
tarDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
tarDone <- cmd.Wait()
|
||||||
|
}()
|
||||||
|
|
||||||
|
var tarErr error
|
||||||
|
select {
|
||||||
|
case tarErr = <-tarDone:
|
||||||
|
// tar completed
|
||||||
|
case <-ctx.Done():
|
||||||
|
e.log.Warn("Archive creation cancelled - killing processes")
|
||||||
|
cmd.Process.Kill()
|
||||||
pigzCmd.Process.Kill()
|
pigzCmd.Process.Kill()
|
||||||
return fmt.Errorf("tar failed: %w", err)
|
<-tarDone
|
||||||
|
return ctx.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wait for pigz to finish
|
if tarErr != nil {
|
||||||
if err := pigzCmd.Wait(); err != nil {
|
pigzCmd.Process.Kill()
|
||||||
return fmt.Errorf("pigz compression failed: %w", err)
|
return fmt.Errorf("tar failed: %w", tarErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for pigz with proper context handling
|
||||||
|
pigzDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
pigzDone <- pigzCmd.Wait()
|
||||||
|
}()
|
||||||
|
|
||||||
|
var pigzErr error
|
||||||
|
select {
|
||||||
|
case pigzErr = <-pigzDone:
|
||||||
|
case <-ctx.Done():
|
||||||
|
pigzCmd.Process.Kill()
|
||||||
|
<-pigzDone
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
if pigzErr != nil {
|
||||||
|
return fmt.Errorf("pigz compression failed: %w", pigzErr)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -1235,8 +1349,10 @@ func (e *Engine) executeCommand(ctx context.Context, cmdArgs []string, outputFil
|
|||||||
return fmt.Errorf("failed to start backup command: %w", err)
|
return fmt.Errorf("failed to start backup command: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stream stderr output (don't buffer it all in memory)
|
// Stream stderr output in goroutine (don't buffer it all in memory)
|
||||||
|
stderrDone := make(chan struct{})
|
||||||
go func() {
|
go func() {
|
||||||
|
defer close(stderrDone)
|
||||||
scanner := bufio.NewScanner(stderr)
|
scanner := bufio.NewScanner(stderr)
|
||||||
scanner.Buffer(make([]byte, 64*1024), 1024*1024) // 1MB max line size
|
scanner.Buffer(make([]byte, 64*1024), 1024*1024) // 1MB max line size
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
@@ -1247,10 +1363,30 @@ func (e *Engine) executeCommand(ctx context.Context, cmdArgs []string, outputFil
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
// Wait for command to complete
|
// Wait for command to complete with proper context handling
|
||||||
if err := cmd.Wait(); err != nil {
|
cmdDone := make(chan error, 1)
|
||||||
e.log.Error("Backup command failed", "error", err, "database", filepath.Base(outputFile))
|
go func() {
|
||||||
return fmt.Errorf("backup command failed: %w", err)
|
cmdDone <- cmd.Wait()
|
||||||
|
}()
|
||||||
|
|
||||||
|
var cmdErr error
|
||||||
|
select {
|
||||||
|
case cmdErr = <-cmdDone:
|
||||||
|
// Command completed (success or failure)
|
||||||
|
case <-ctx.Done():
|
||||||
|
// Context cancelled - kill process to unblock
|
||||||
|
e.log.Warn("Backup cancelled - killing pg_dump process")
|
||||||
|
cmd.Process.Kill()
|
||||||
|
<-cmdDone // Wait for goroutine to finish
|
||||||
|
cmdErr = ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for stderr reader to finish
|
||||||
|
<-stderrDone
|
||||||
|
|
||||||
|
if cmdErr != nil {
|
||||||
|
e.log.Error("Backup command failed", "error", cmdErr, "database", filepath.Base(outputFile))
|
||||||
|
return fmt.Errorf("backup command failed: %w", cmdErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -1352,20 +1488,53 @@ func (e *Engine) executeWithStreamingCompression(ctx context.Context, cmdArgs []
|
|||||||
|
|
||||||
// Then start pg_dump
|
// Then start pg_dump
|
||||||
if err := dumpCmd.Start(); err != nil {
|
if err := dumpCmd.Start(); err != nil {
|
||||||
|
compressCmd.Process.Kill()
|
||||||
return fmt.Errorf("failed to start pg_dump: %w", err)
|
return fmt.Errorf("failed to start pg_dump: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wait for pg_dump to complete
|
// Wait for pg_dump in a goroutine to handle context timeout properly
|
||||||
if err := dumpCmd.Wait(); err != nil {
|
// This prevents deadlock if pipe buffer fills and pg_dump blocks
|
||||||
return fmt.Errorf("pg_dump failed: %w", err)
|
dumpDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
dumpDone <- dumpCmd.Wait()
|
||||||
|
}()
|
||||||
|
|
||||||
|
var dumpErr error
|
||||||
|
select {
|
||||||
|
case dumpErr = <-dumpDone:
|
||||||
|
// pg_dump completed (success or failure)
|
||||||
|
case <-ctx.Done():
|
||||||
|
// Context cancelled/timeout - kill pg_dump to unblock
|
||||||
|
e.log.Warn("Backup timeout - killing pg_dump process")
|
||||||
|
dumpCmd.Process.Kill()
|
||||||
|
<-dumpDone // Wait for goroutine to finish
|
||||||
|
dumpErr = ctx.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Close stdout pipe to signal compressor we're done
|
// Close stdout pipe to signal compressor we're done
|
||||||
|
// This MUST happen after pg_dump exits to avoid broken pipe
|
||||||
dumpStdout.Close()
|
dumpStdout.Close()
|
||||||
|
|
||||||
// Wait for compression to complete
|
// Wait for compression to complete
|
||||||
if err := compressCmd.Wait(); err != nil {
|
compressErr := compressCmd.Wait()
|
||||||
return fmt.Errorf("compression failed: %w", err)
|
|
||||||
|
// Check errors - compressor failure first (it's usually the root cause)
|
||||||
|
if compressErr != nil {
|
||||||
|
e.log.Error("Compressor failed", "error", compressErr)
|
||||||
|
return fmt.Errorf("compression failed (check disk space): %w", compressErr)
|
||||||
|
}
|
||||||
|
if dumpErr != nil {
|
||||||
|
// Check for SIGPIPE (exit code 141) - indicates compressor died first
|
||||||
|
if exitErr, ok := dumpErr.(*exec.ExitError); ok && exitErr.ExitCode() == 141 {
|
||||||
|
e.log.Error("pg_dump received SIGPIPE - compressor may have failed")
|
||||||
|
return fmt.Errorf("pg_dump broken pipe - check disk space and compressor")
|
||||||
|
}
|
||||||
|
return fmt.Errorf("pg_dump failed: %w", dumpErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sync file to disk to ensure durability (prevents truncation on power loss)
|
||||||
|
if err := outFile.Sync(); err != nil {
|
||||||
|
e.log.Warn("Failed to sync output file", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
e.log.Debug("Streaming compression completed", "output", compressedFile)
|
e.log.Debug("Streaming compression completed", "output", compressedFile)
|
||||||
|
|||||||
@@ -242,7 +242,7 @@ func TestIncrementalBackupRestore(t *testing.T) {
|
|||||||
t.Errorf("Unchanged file base/12345/1235 not found in restore: %v", err)
|
t.Errorf("Unchanged file base/12345/1235 not found in restore: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Log("✅ Incremental backup and restore test completed successfully")
|
t.Log("[OK] Incremental backup and restore test completed successfully")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestIncrementalBackupErrors tests error handling
|
// TestIncrementalBackupErrors tests error handling
|
||||||
|
|||||||
@@ -75,16 +75,16 @@ func FormatDiskSpaceMessage(check *DiskSpaceCheck) string {
|
|||||||
|
|
||||||
if check.Critical {
|
if check.Critical {
|
||||||
status = "CRITICAL"
|
status = "CRITICAL"
|
||||||
icon = "❌"
|
icon = "[X]"
|
||||||
} else if check.Warning {
|
} else if check.Warning {
|
||||||
status = "WARNING"
|
status = "WARNING"
|
||||||
icon = "⚠️ "
|
icon = "[!]"
|
||||||
} else {
|
} else {
|
||||||
status = "OK"
|
status = "OK"
|
||||||
icon = "✓"
|
icon = "[+]"
|
||||||
}
|
}
|
||||||
|
|
||||||
msg := fmt.Sprintf(`📊 Disk Space Check (%s):
|
msg := fmt.Sprintf(`[DISK] Disk Space Check (%s):
|
||||||
Path: %s
|
Path: %s
|
||||||
Total: %s
|
Total: %s
|
||||||
Available: %s (%.1f%% used)
|
Available: %s (%.1f%% used)
|
||||||
@@ -98,13 +98,13 @@ func FormatDiskSpaceMessage(check *DiskSpaceCheck) string {
|
|||||||
status)
|
status)
|
||||||
|
|
||||||
if check.Critical {
|
if check.Critical {
|
||||||
msg += "\n \n ⚠️ CRITICAL: Insufficient disk space!"
|
msg += "\n \n [!!] CRITICAL: Insufficient disk space!"
|
||||||
msg += "\n Operation blocked. Free up space before continuing."
|
msg += "\n Operation blocked. Free up space before continuing."
|
||||||
} else if check.Warning {
|
} else if check.Warning {
|
||||||
msg += "\n \n ⚠️ WARNING: Low disk space!"
|
msg += "\n \n [!] WARNING: Low disk space!"
|
||||||
msg += "\n Backup may fail if database is larger than estimated."
|
msg += "\n Backup may fail if database is larger than estimated."
|
||||||
} else {
|
} else {
|
||||||
msg += "\n \n ✓ Sufficient space available"
|
msg += "\n \n [+] Sufficient space available"
|
||||||
}
|
}
|
||||||
|
|
||||||
return msg
|
return msg
|
||||||
|
|||||||
@@ -75,16 +75,16 @@ func FormatDiskSpaceMessage(check *DiskSpaceCheck) string {
|
|||||||
|
|
||||||
if check.Critical {
|
if check.Critical {
|
||||||
status = "CRITICAL"
|
status = "CRITICAL"
|
||||||
icon = "❌"
|
icon = "[X]"
|
||||||
} else if check.Warning {
|
} else if check.Warning {
|
||||||
status = "WARNING"
|
status = "WARNING"
|
||||||
icon = "⚠️ "
|
icon = "[!]"
|
||||||
} else {
|
} else {
|
||||||
status = "OK"
|
status = "OK"
|
||||||
icon = "✓"
|
icon = "[+]"
|
||||||
}
|
}
|
||||||
|
|
||||||
msg := fmt.Sprintf(`📊 Disk Space Check (%s):
|
msg := fmt.Sprintf(`[DISK] Disk Space Check (%s):
|
||||||
Path: %s
|
Path: %s
|
||||||
Total: %s
|
Total: %s
|
||||||
Available: %s (%.1f%% used)
|
Available: %s (%.1f%% used)
|
||||||
@@ -98,13 +98,13 @@ func FormatDiskSpaceMessage(check *DiskSpaceCheck) string {
|
|||||||
status)
|
status)
|
||||||
|
|
||||||
if check.Critical {
|
if check.Critical {
|
||||||
msg += "\n \n ⚠️ CRITICAL: Insufficient disk space!"
|
msg += "\n \n [!!] CRITICAL: Insufficient disk space!"
|
||||||
msg += "\n Operation blocked. Free up space before continuing."
|
msg += "\n Operation blocked. Free up space before continuing."
|
||||||
} else if check.Warning {
|
} else if check.Warning {
|
||||||
msg += "\n \n ⚠️ WARNING: Low disk space!"
|
msg += "\n \n [!] WARNING: Low disk space!"
|
||||||
msg += "\n Backup may fail if database is larger than estimated."
|
msg += "\n Backup may fail if database is larger than estimated."
|
||||||
} else {
|
} else {
|
||||||
msg += "\n \n ✓ Sufficient space available"
|
msg += "\n \n [+] Sufficient space available"
|
||||||
}
|
}
|
||||||
|
|
||||||
return msg
|
return msg
|
||||||
|
|||||||
@@ -58,16 +58,16 @@ func FormatDiskSpaceMessage(check *DiskSpaceCheck) string {
|
|||||||
|
|
||||||
if check.Critical {
|
if check.Critical {
|
||||||
status = "CRITICAL"
|
status = "CRITICAL"
|
||||||
icon = "❌"
|
icon = "[X]"
|
||||||
} else if check.Warning {
|
} else if check.Warning {
|
||||||
status = "WARNING"
|
status = "WARNING"
|
||||||
icon = "⚠️ "
|
icon = "[!]"
|
||||||
} else {
|
} else {
|
||||||
status = "OK"
|
status = "OK"
|
||||||
icon = "✓"
|
icon = "[+]"
|
||||||
}
|
}
|
||||||
|
|
||||||
msg := fmt.Sprintf(`📊 Disk Space Check (%s):
|
msg := fmt.Sprintf(`[DISK] Disk Space Check (%s):
|
||||||
Path: %s
|
Path: %s
|
||||||
Total: %s
|
Total: %s
|
||||||
Available: %s (%.1f%% used)
|
Available: %s (%.1f%% used)
|
||||||
@@ -81,13 +81,13 @@ func FormatDiskSpaceMessage(check *DiskSpaceCheck) string {
|
|||||||
status)
|
status)
|
||||||
|
|
||||||
if check.Critical {
|
if check.Critical {
|
||||||
msg += "\n \n ⚠️ CRITICAL: Insufficient disk space!"
|
msg += "\n \n [!!] CRITICAL: Insufficient disk space!"
|
||||||
msg += "\n Operation blocked. Free up space before continuing."
|
msg += "\n Operation blocked. Free up space before continuing."
|
||||||
} else if check.Warning {
|
} else if check.Warning {
|
||||||
msg += "\n \n ⚠️ WARNING: Low disk space!"
|
msg += "\n \n [!] WARNING: Low disk space!"
|
||||||
msg += "\n Backup may fail if database is larger than estimated."
|
msg += "\n Backup may fail if database is larger than estimated."
|
||||||
} else {
|
} else {
|
||||||
msg += "\n \n ✓ Sufficient space available"
|
msg += "\n \n [+] Sufficient space available"
|
||||||
}
|
}
|
||||||
|
|
||||||
return msg
|
return msg
|
||||||
|
|||||||
@@ -94,16 +94,16 @@ func FormatDiskSpaceMessage(check *DiskSpaceCheck) string {
|
|||||||
|
|
||||||
if check.Critical {
|
if check.Critical {
|
||||||
status = "CRITICAL"
|
status = "CRITICAL"
|
||||||
icon = "❌"
|
icon = "[X]"
|
||||||
} else if check.Warning {
|
} else if check.Warning {
|
||||||
status = "WARNING"
|
status = "WARNING"
|
||||||
icon = "⚠️ "
|
icon = "[!]"
|
||||||
} else {
|
} else {
|
||||||
status = "OK"
|
status = "OK"
|
||||||
icon = "✓"
|
icon = "[+]"
|
||||||
}
|
}
|
||||||
|
|
||||||
msg := fmt.Sprintf(`📊 Disk Space Check (%s):
|
msg := fmt.Sprintf(`[DISK] Disk Space Check (%s):
|
||||||
Path: %s
|
Path: %s
|
||||||
Total: %s
|
Total: %s
|
||||||
Available: %s (%.1f%% used)
|
Available: %s (%.1f%% used)
|
||||||
@@ -117,13 +117,13 @@ func FormatDiskSpaceMessage(check *DiskSpaceCheck) string {
|
|||||||
status)
|
status)
|
||||||
|
|
||||||
if check.Critical {
|
if check.Critical {
|
||||||
msg += "\n \n ⚠️ CRITICAL: Insufficient disk space!"
|
msg += "\n \n [!!] CRITICAL: Insufficient disk space!"
|
||||||
msg += "\n Operation blocked. Free up space before continuing."
|
msg += "\n Operation blocked. Free up space before continuing."
|
||||||
} else if check.Warning {
|
} else if check.Warning {
|
||||||
msg += "\n \n ⚠️ WARNING: Low disk space!"
|
msg += "\n \n [!] WARNING: Low disk space!"
|
||||||
msg += "\n Backup may fail if database is larger than estimated."
|
msg += "\n Backup may fail if database is larger than estimated."
|
||||||
} else {
|
} else {
|
||||||
msg += "\n \n ✓ Sufficient space available"
|
msg += "\n \n [+] Sufficient space available"
|
||||||
}
|
}
|
||||||
|
|
||||||
return msg
|
return msg
|
||||||
|
|||||||
@@ -234,22 +234,22 @@ func FormatErrorWithHint(errorMsg string) string {
|
|||||||
var icon string
|
var icon string
|
||||||
switch classification.Type {
|
switch classification.Type {
|
||||||
case "ignorable":
|
case "ignorable":
|
||||||
icon = "ℹ️ "
|
icon = "[i]"
|
||||||
case "warning":
|
case "warning":
|
||||||
icon = "⚠️ "
|
icon = "[!]"
|
||||||
case "critical":
|
case "critical":
|
||||||
icon = "❌"
|
icon = "[X]"
|
||||||
case "fatal":
|
case "fatal":
|
||||||
icon = "🛑"
|
icon = "[!!]"
|
||||||
default:
|
default:
|
||||||
icon = "⚠️ "
|
icon = "[!]"
|
||||||
}
|
}
|
||||||
|
|
||||||
output := fmt.Sprintf("%s %s Error\n\n", icon, strings.ToUpper(classification.Type))
|
output := fmt.Sprintf("%s %s Error\n\n", icon, strings.ToUpper(classification.Type))
|
||||||
output += fmt.Sprintf("Category: %s\n", classification.Category)
|
output += fmt.Sprintf("Category: %s\n", classification.Category)
|
||||||
output += fmt.Sprintf("Message: %s\n\n", classification.Message)
|
output += fmt.Sprintf("Message: %s\n\n", classification.Message)
|
||||||
output += fmt.Sprintf("💡 Hint: %s\n\n", classification.Hint)
|
output += fmt.Sprintf("[HINT] Hint: %s\n\n", classification.Hint)
|
||||||
output += fmt.Sprintf("🔧 Action: %s\n", classification.Action)
|
output += fmt.Sprintf("[ACTION] Action: %s\n", classification.Action)
|
||||||
|
|
||||||
return output
|
return output
|
||||||
}
|
}
|
||||||
@@ -257,7 +257,7 @@ func FormatErrorWithHint(errorMsg string) string {
|
|||||||
// FormatMultipleErrors formats multiple errors with classification
|
// FormatMultipleErrors formats multiple errors with classification
|
||||||
func FormatMultipleErrors(errors []string) string {
|
func FormatMultipleErrors(errors []string) string {
|
||||||
if len(errors) == 0 {
|
if len(errors) == 0 {
|
||||||
return "✓ No errors"
|
return "[+] No errors"
|
||||||
}
|
}
|
||||||
|
|
||||||
ignorable := 0
|
ignorable := 0
|
||||||
@@ -285,22 +285,22 @@ func FormatMultipleErrors(errors []string) string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
output := "📊 Error Summary:\n\n"
|
output := "[SUMMARY] Error Summary:\n\n"
|
||||||
if ignorable > 0 {
|
if ignorable > 0 {
|
||||||
output += fmt.Sprintf(" ℹ️ %d ignorable (objects already exist)\n", ignorable)
|
output += fmt.Sprintf(" [i] %d ignorable (objects already exist)\n", ignorable)
|
||||||
}
|
}
|
||||||
if warnings > 0 {
|
if warnings > 0 {
|
||||||
output += fmt.Sprintf(" ⚠️ %d warnings\n", warnings)
|
output += fmt.Sprintf(" [!] %d warnings\n", warnings)
|
||||||
}
|
}
|
||||||
if critical > 0 {
|
if critical > 0 {
|
||||||
output += fmt.Sprintf(" ❌ %d critical errors\n", critical)
|
output += fmt.Sprintf(" [X] %d critical errors\n", critical)
|
||||||
}
|
}
|
||||||
if fatal > 0 {
|
if fatal > 0 {
|
||||||
output += fmt.Sprintf(" 🛑 %d fatal errors\n", fatal)
|
output += fmt.Sprintf(" [!!] %d fatal errors\n", fatal)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(criticalErrors) > 0 {
|
if len(criticalErrors) > 0 {
|
||||||
output += "\n📝 Critical Issues:\n\n"
|
output += "\n[CRITICAL] Critical Issues:\n\n"
|
||||||
for i, err := range criticalErrors {
|
for i, err := range criticalErrors {
|
||||||
class := ClassifyError(err)
|
class := ClassifyError(err)
|
||||||
output += fmt.Sprintf("%d. %s\n", i+1, class.Hint)
|
output += fmt.Sprintf("%d. %s\n", i+1, class.Hint)
|
||||||
|
|||||||
@@ -49,15 +49,15 @@ func (s CheckStatus) String() string {
|
|||||||
func (s CheckStatus) Icon() string {
|
func (s CheckStatus) Icon() string {
|
||||||
switch s {
|
switch s {
|
||||||
case StatusPassed:
|
case StatusPassed:
|
||||||
return "✓"
|
return "[+]"
|
||||||
case StatusWarning:
|
case StatusWarning:
|
||||||
return "⚠"
|
return "[!]"
|
||||||
case StatusFailed:
|
case StatusFailed:
|
||||||
return "✗"
|
return "[-]"
|
||||||
case StatusSkipped:
|
case StatusSkipped:
|
||||||
return "○"
|
return "[ ]"
|
||||||
default:
|
default:
|
||||||
return "?"
|
return "[?]"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -11,9 +11,9 @@ func FormatPreflightReport(result *PreflightResult, dbName string, verbose bool)
|
|||||||
var sb strings.Builder
|
var sb strings.Builder
|
||||||
|
|
||||||
sb.WriteString("\n")
|
sb.WriteString("\n")
|
||||||
sb.WriteString("╔══════════════════════════════════════════════════════════════╗\n")
|
sb.WriteString("+==============================================================+\n")
|
||||||
sb.WriteString("║ [DRY RUN] Preflight Check Results ║\n")
|
sb.WriteString("| [DRY RUN] Preflight Check Results |\n")
|
||||||
sb.WriteString("╚══════════════════════════════════════════════════════════════╝\n")
|
sb.WriteString("+==============================================================+\n")
|
||||||
sb.WriteString("\n")
|
sb.WriteString("\n")
|
||||||
|
|
||||||
// Database info
|
// Database info
|
||||||
@@ -29,7 +29,7 @@ func FormatPreflightReport(result *PreflightResult, dbName string, verbose bool)
|
|||||||
|
|
||||||
// Check results
|
// Check results
|
||||||
sb.WriteString(" Checks:\n")
|
sb.WriteString(" Checks:\n")
|
||||||
sb.WriteString(" ─────────────────────────────────────────────────────────────\n")
|
sb.WriteString(" --------------------------------------------------------------\n")
|
||||||
|
|
||||||
for _, check := range result.Checks {
|
for _, check := range result.Checks {
|
||||||
icon := check.Status.Icon()
|
icon := check.Status.Icon()
|
||||||
@@ -40,26 +40,26 @@ func FormatPreflightReport(result *PreflightResult, dbName string, verbose bool)
|
|||||||
color, icon, reset, check.Name+":", check.Message))
|
color, icon, reset, check.Name+":", check.Message))
|
||||||
|
|
||||||
if verbose && check.Details != "" {
|
if verbose && check.Details != "" {
|
||||||
sb.WriteString(fmt.Sprintf(" └─ %s\n", check.Details))
|
sb.WriteString(fmt.Sprintf(" +- %s\n", check.Details))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
sb.WriteString(" ─────────────────────────────────────────────────────────────\n")
|
sb.WriteString(" --------------------------------------------------------------\n")
|
||||||
sb.WriteString("\n")
|
sb.WriteString("\n")
|
||||||
|
|
||||||
// Summary
|
// Summary
|
||||||
if result.AllPassed {
|
if result.AllPassed {
|
||||||
if result.HasWarnings {
|
if result.HasWarnings {
|
||||||
sb.WriteString(" ⚠️ All checks passed with warnings\n")
|
sb.WriteString(" [!] All checks passed with warnings\n")
|
||||||
sb.WriteString("\n")
|
sb.WriteString("\n")
|
||||||
sb.WriteString(" Ready to backup. Remove --dry-run to execute.\n")
|
sb.WriteString(" Ready to backup. Remove --dry-run to execute.\n")
|
||||||
} else {
|
} else {
|
||||||
sb.WriteString(" ✅ All checks passed\n")
|
sb.WriteString(" [OK] All checks passed\n")
|
||||||
sb.WriteString("\n")
|
sb.WriteString("\n")
|
||||||
sb.WriteString(" Ready to backup. Remove --dry-run to execute.\n")
|
sb.WriteString(" Ready to backup. Remove --dry-run to execute.\n")
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
sb.WriteString(fmt.Sprintf(" ❌ %d check(s) failed\n", result.FailureCount))
|
sb.WriteString(fmt.Sprintf(" [FAIL] %d check(s) failed\n", result.FailureCount))
|
||||||
sb.WriteString("\n")
|
sb.WriteString("\n")
|
||||||
sb.WriteString(" Fix the issues above before running backup.\n")
|
sb.WriteString(" Fix the issues above before running backup.\n")
|
||||||
}
|
}
|
||||||
@@ -96,7 +96,7 @@ func FormatPreflightReportPlain(result *PreflightResult, dbName string) string {
|
|||||||
status := fmt.Sprintf("[%s]", check.Status.String())
|
status := fmt.Sprintf("[%s]", check.Status.String())
|
||||||
sb.WriteString(fmt.Sprintf(" %-10s %-25s %s\n", status, check.Name+":", check.Message))
|
sb.WriteString(fmt.Sprintf(" %-10s %-25s %s\n", status, check.Name+":", check.Message))
|
||||||
if check.Details != "" {
|
if check.Details != "" {
|
||||||
sb.WriteString(fmt.Sprintf(" └─ %s\n", check.Details))
|
sb.WriteString(fmt.Sprintf(" +- %s\n", check.Details))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"syscall"
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
"dbbackup/internal/logger"
|
"dbbackup/internal/logger"
|
||||||
)
|
)
|
||||||
@@ -116,8 +117,11 @@ func KillOrphanedProcesses(log logger.Logger) error {
|
|||||||
|
|
||||||
// findProcessesByName returns PIDs of processes matching the given name
|
// findProcessesByName returns PIDs of processes matching the given name
|
||||||
func findProcessesByName(name string, excludePID int) ([]int, error) {
|
func findProcessesByName(name string, excludePID int) ([]int, error) {
|
||||||
// Use pgrep for efficient process searching
|
// Use pgrep for efficient process searching with timeout
|
||||||
cmd := exec.Command("pgrep", "-x", name)
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
cmd := exec.CommandContext(ctx, "pgrep", "-x", name)
|
||||||
output, err := cmd.Output()
|
output, err := cmd.Output()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Exit code 1 means no processes found (not an error)
|
// Exit code 1 means no processes found (not an error)
|
||||||
|
|||||||
@@ -90,7 +90,7 @@ func NewAzureBackend(cfg *Config) (*AzureBackend, error) {
|
|||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Use default Azure credential (managed identity, environment variables, etc.)
|
// Use default Azure credential (managed identity, environment variables, etc.)
|
||||||
return nil, fmt.Errorf("Azure authentication requires account name and key, or use AZURE_STORAGE_CONNECTION_STRING environment variable")
|
return nil, fmt.Errorf("azure authentication requires account name and key, or use AZURE_STORAGE_CONNECTION_STRING environment variable")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -64,6 +64,9 @@ type Config struct {
|
|||||||
// Cluster parallelism
|
// Cluster parallelism
|
||||||
ClusterParallelism int // Number of concurrent databases during cluster operations (0 = sequential)
|
ClusterParallelism int // Number of concurrent databases during cluster operations (0 = sequential)
|
||||||
|
|
||||||
|
// Working directory for large operations (extraction, diagnosis)
|
||||||
|
WorkDir string // Alternative temp directory for large operations (default: system temp)
|
||||||
|
|
||||||
// Swap file management (for large backups)
|
// Swap file management (for large backups)
|
||||||
SwapFilePath string // Path to temporary swap file
|
SwapFilePath string // Path to temporary swap file
|
||||||
SwapFileSizeGB int // Size in GB (0 = disabled)
|
SwapFileSizeGB int // Size in GB (0 = disabled)
|
||||||
@@ -214,14 +217,17 @@ func New() *Config {
|
|||||||
SingleDBName: getEnvString("SINGLE_DB_NAME", ""),
|
SingleDBName: getEnvString("SINGLE_DB_NAME", ""),
|
||||||
RestoreDBName: getEnvString("RESTORE_DB_NAME", ""),
|
RestoreDBName: getEnvString("RESTORE_DB_NAME", ""),
|
||||||
|
|
||||||
// Timeouts
|
// Timeouts - default 24 hours (1440 min) to handle very large databases with large objects
|
||||||
ClusterTimeoutMinutes: getEnvInt("CLUSTER_TIMEOUT_MIN", 240),
|
ClusterTimeoutMinutes: getEnvInt("CLUSTER_TIMEOUT_MIN", 1440),
|
||||||
|
|
||||||
// Cluster parallelism (default: 2 concurrent operations for faster cluster backup/restore)
|
// Cluster parallelism (default: 2 concurrent operations for faster cluster backup/restore)
|
||||||
ClusterParallelism: getEnvInt("CLUSTER_PARALLELISM", 2),
|
ClusterParallelism: getEnvInt("CLUSTER_PARALLELISM", 2),
|
||||||
|
|
||||||
|
// Working directory for large operations (default: system temp)
|
||||||
|
WorkDir: getEnvString("WORK_DIR", ""),
|
||||||
|
|
||||||
// Swap file management
|
// Swap file management
|
||||||
SwapFilePath: getEnvString("SWAP_FILE_PATH", "/tmp/dbbackup_swap"),
|
SwapFilePath: "", // Will be set after WorkDir is initialized
|
||||||
SwapFileSizeGB: getEnvInt("SWAP_FILE_SIZE_GB", 0), // 0 = disabled by default
|
SwapFileSizeGB: getEnvInt("SWAP_FILE_SIZE_GB", 0), // 0 = disabled by default
|
||||||
AutoSwap: getEnvBool("AUTO_SWAP", false),
|
AutoSwap: getEnvBool("AUTO_SWAP", false),
|
||||||
|
|
||||||
@@ -261,6 +267,13 @@ func New() *Config {
|
|||||||
cfg.SSLMode = "prefer"
|
cfg.SSLMode = "prefer"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Set SwapFilePath using WorkDir if not explicitly set via env var
|
||||||
|
if envSwap := os.Getenv("SWAP_FILE_PATH"); envSwap != "" {
|
||||||
|
cfg.SwapFilePath = envSwap
|
||||||
|
} else {
|
||||||
|
cfg.SwapFilePath = filepath.Join(cfg.GetEffectiveWorkDir(), "dbbackup_swap")
|
||||||
|
}
|
||||||
|
|
||||||
return cfg
|
return cfg
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -496,6 +509,14 @@ func GetCurrentOSUser() string {
|
|||||||
return getCurrentUser()
|
return getCurrentUser()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetEffectiveWorkDir returns the configured WorkDir or system temp as fallback
|
||||||
|
func (c *Config) GetEffectiveWorkDir() string {
|
||||||
|
if c.WorkDir != "" {
|
||||||
|
return c.WorkDir
|
||||||
|
}
|
||||||
|
return os.TempDir()
|
||||||
|
}
|
||||||
|
|
||||||
func getDefaultBackupDir() string {
|
func getDefaultBackupDir() string {
|
||||||
// Try to create a sensible default backup directory
|
// Try to create a sensible default backup directory
|
||||||
homeDir, _ := os.UserHomeDir()
|
homeDir, _ := os.UserHomeDir()
|
||||||
@@ -513,7 +534,7 @@ func getDefaultBackupDir() string {
|
|||||||
return "/var/lib/pgsql/pg_backups"
|
return "/var/lib/pgsql/pg_backups"
|
||||||
}
|
}
|
||||||
|
|
||||||
return "/tmp/db_backups"
|
return filepath.Join(os.TempDir(), "db_backups")
|
||||||
}
|
}
|
||||||
|
|
||||||
// CPU-related helper functions
|
// CPU-related helper functions
|
||||||
|
|||||||
@@ -22,13 +22,15 @@ type LocalConfig struct {
|
|||||||
|
|
||||||
// Backup settings
|
// Backup settings
|
||||||
BackupDir string
|
BackupDir string
|
||||||
|
WorkDir string // Working directory for large operations
|
||||||
Compression int
|
Compression int
|
||||||
Jobs int
|
Jobs int
|
||||||
DumpJobs int
|
DumpJobs int
|
||||||
|
|
||||||
// Performance settings
|
// Performance settings
|
||||||
CPUWorkload string
|
CPUWorkload string
|
||||||
MaxCores int
|
MaxCores int
|
||||||
|
ClusterTimeout int // Cluster operation timeout in minutes (default: 1440 = 24 hours)
|
||||||
|
|
||||||
// Security settings
|
// Security settings
|
||||||
RetentionDays int
|
RetentionDays int
|
||||||
@@ -97,6 +99,8 @@ func LoadLocalConfig() (*LocalConfig, error) {
|
|||||||
switch key {
|
switch key {
|
||||||
case "backup_dir":
|
case "backup_dir":
|
||||||
cfg.BackupDir = value
|
cfg.BackupDir = value
|
||||||
|
case "work_dir":
|
||||||
|
cfg.WorkDir = value
|
||||||
case "compression":
|
case "compression":
|
||||||
if c, err := strconv.Atoi(value); err == nil {
|
if c, err := strconv.Atoi(value); err == nil {
|
||||||
cfg.Compression = c
|
cfg.Compression = c
|
||||||
@@ -118,6 +122,10 @@ func LoadLocalConfig() (*LocalConfig, error) {
|
|||||||
if mc, err := strconv.Atoi(value); err == nil {
|
if mc, err := strconv.Atoi(value); err == nil {
|
||||||
cfg.MaxCores = mc
|
cfg.MaxCores = mc
|
||||||
}
|
}
|
||||||
|
case "cluster_timeout":
|
||||||
|
if ct, err := strconv.Atoi(value); err == nil {
|
||||||
|
cfg.ClusterTimeout = ct
|
||||||
|
}
|
||||||
}
|
}
|
||||||
case "security":
|
case "security":
|
||||||
switch key {
|
switch key {
|
||||||
@@ -174,6 +182,9 @@ func SaveLocalConfig(cfg *LocalConfig) error {
|
|||||||
if cfg.BackupDir != "" {
|
if cfg.BackupDir != "" {
|
||||||
sb.WriteString(fmt.Sprintf("backup_dir = %s\n", cfg.BackupDir))
|
sb.WriteString(fmt.Sprintf("backup_dir = %s\n", cfg.BackupDir))
|
||||||
}
|
}
|
||||||
|
if cfg.WorkDir != "" {
|
||||||
|
sb.WriteString(fmt.Sprintf("work_dir = %s\n", cfg.WorkDir))
|
||||||
|
}
|
||||||
if cfg.Compression != 0 {
|
if cfg.Compression != 0 {
|
||||||
sb.WriteString(fmt.Sprintf("compression = %d\n", cfg.Compression))
|
sb.WriteString(fmt.Sprintf("compression = %d\n", cfg.Compression))
|
||||||
}
|
}
|
||||||
@@ -193,6 +204,9 @@ func SaveLocalConfig(cfg *LocalConfig) error {
|
|||||||
if cfg.MaxCores != 0 {
|
if cfg.MaxCores != 0 {
|
||||||
sb.WriteString(fmt.Sprintf("max_cores = %d\n", cfg.MaxCores))
|
sb.WriteString(fmt.Sprintf("max_cores = %d\n", cfg.MaxCores))
|
||||||
}
|
}
|
||||||
|
if cfg.ClusterTimeout != 0 {
|
||||||
|
sb.WriteString(fmt.Sprintf("cluster_timeout = %d\n", cfg.ClusterTimeout))
|
||||||
|
}
|
||||||
sb.WriteString("\n")
|
sb.WriteString("\n")
|
||||||
|
|
||||||
// Security section
|
// Security section
|
||||||
@@ -244,6 +258,9 @@ func ApplyLocalConfig(cfg *Config, local *LocalConfig) {
|
|||||||
if local.BackupDir != "" {
|
if local.BackupDir != "" {
|
||||||
cfg.BackupDir = local.BackupDir
|
cfg.BackupDir = local.BackupDir
|
||||||
}
|
}
|
||||||
|
if local.WorkDir != "" {
|
||||||
|
cfg.WorkDir = local.WorkDir
|
||||||
|
}
|
||||||
if cfg.CompressionLevel == 6 && local.Compression != 0 {
|
if cfg.CompressionLevel == 6 && local.Compression != 0 {
|
||||||
cfg.CompressionLevel = local.Compression
|
cfg.CompressionLevel = local.Compression
|
||||||
}
|
}
|
||||||
@@ -259,6 +276,10 @@ func ApplyLocalConfig(cfg *Config, local *LocalConfig) {
|
|||||||
if local.MaxCores != 0 {
|
if local.MaxCores != 0 {
|
||||||
cfg.MaxCores = local.MaxCores
|
cfg.MaxCores = local.MaxCores
|
||||||
}
|
}
|
||||||
|
// Apply cluster timeout from config file (overrides default)
|
||||||
|
if local.ClusterTimeout != 0 {
|
||||||
|
cfg.ClusterTimeoutMinutes = local.ClusterTimeout
|
||||||
|
}
|
||||||
if cfg.RetentionDays == 30 && local.RetentionDays != 0 {
|
if cfg.RetentionDays == 30 && local.RetentionDays != 0 {
|
||||||
cfg.RetentionDays = local.RetentionDays
|
cfg.RetentionDays = local.RetentionDays
|
||||||
}
|
}
|
||||||
@@ -273,20 +294,22 @@ func ApplyLocalConfig(cfg *Config, local *LocalConfig) {
|
|||||||
// ConfigFromConfig creates a LocalConfig from a Config
|
// ConfigFromConfig creates a LocalConfig from a Config
|
||||||
func ConfigFromConfig(cfg *Config) *LocalConfig {
|
func ConfigFromConfig(cfg *Config) *LocalConfig {
|
||||||
return &LocalConfig{
|
return &LocalConfig{
|
||||||
DBType: cfg.DatabaseType,
|
DBType: cfg.DatabaseType,
|
||||||
Host: cfg.Host,
|
Host: cfg.Host,
|
||||||
Port: cfg.Port,
|
Port: cfg.Port,
|
||||||
User: cfg.User,
|
User: cfg.User,
|
||||||
Database: cfg.Database,
|
Database: cfg.Database,
|
||||||
SSLMode: cfg.SSLMode,
|
SSLMode: cfg.SSLMode,
|
||||||
BackupDir: cfg.BackupDir,
|
BackupDir: cfg.BackupDir,
|
||||||
Compression: cfg.CompressionLevel,
|
WorkDir: cfg.WorkDir,
|
||||||
Jobs: cfg.Jobs,
|
Compression: cfg.CompressionLevel,
|
||||||
DumpJobs: cfg.DumpJobs,
|
Jobs: cfg.Jobs,
|
||||||
CPUWorkload: cfg.CPUWorkloadType,
|
DumpJobs: cfg.DumpJobs,
|
||||||
MaxCores: cfg.MaxCores,
|
CPUWorkload: cfg.CPUWorkloadType,
|
||||||
RetentionDays: cfg.RetentionDays,
|
MaxCores: cfg.MaxCores,
|
||||||
MinBackups: cfg.MinBackups,
|
ClusterTimeout: cfg.ClusterTimeoutMinutes,
|
||||||
MaxRetries: cfg.MaxRetries,
|
RetentionDays: cfg.RetentionDays,
|
||||||
|
MinBackups: cfg.MinBackups,
|
||||||
|
MaxRetries: cfg.MaxRetries,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -126,13 +126,46 @@ func (m *MySQL) ListTables(ctx context.Context, database string) ([]string, erro
|
|||||||
return tables, rows.Err()
|
return tables, rows.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// validateMySQLIdentifier checks if a database/table name is safe for use in SQL
|
||||||
|
// Prevents SQL injection by only allowing alphanumeric names with underscores
|
||||||
|
func validateMySQLIdentifier(name string) error {
|
||||||
|
if len(name) == 0 {
|
||||||
|
return fmt.Errorf("identifier cannot be empty")
|
||||||
|
}
|
||||||
|
if len(name) > 64 {
|
||||||
|
return fmt.Errorf("identifier too long (max 64 chars): %s", name)
|
||||||
|
}
|
||||||
|
// Only allow alphanumeric, underscores, and must start with letter or underscore
|
||||||
|
for i, c := range name {
|
||||||
|
if i == 0 && !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_') {
|
||||||
|
return fmt.Errorf("identifier must start with letter or underscore: %s", name)
|
||||||
|
}
|
||||||
|
if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_') {
|
||||||
|
return fmt.Errorf("identifier contains invalid character %q: %s", c, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// quoteMySQLIdentifier safely quotes a MySQL identifier
|
||||||
|
func quoteMySQLIdentifier(name string) string {
|
||||||
|
// Escape any backticks by doubling them and wrap in backticks
|
||||||
|
return "`" + strings.ReplaceAll(name, "`", "``") + "`"
|
||||||
|
}
|
||||||
|
|
||||||
// CreateDatabase creates a new database
|
// CreateDatabase creates a new database
|
||||||
func (m *MySQL) CreateDatabase(ctx context.Context, name string) error {
|
func (m *MySQL) CreateDatabase(ctx context.Context, name string) error {
|
||||||
if m.db == nil {
|
if m.db == nil {
|
||||||
return fmt.Errorf("not connected to database")
|
return fmt.Errorf("not connected to database")
|
||||||
}
|
}
|
||||||
|
|
||||||
query := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", name)
|
// Validate identifier to prevent SQL injection
|
||||||
|
if err := validateMySQLIdentifier(name); err != nil {
|
||||||
|
return fmt.Errorf("invalid database name: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Use safe quoting for identifier
|
||||||
|
query := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", quoteMySQLIdentifier(name))
|
||||||
_, err := m.db.ExecContext(ctx, query)
|
_, err := m.db.ExecContext(ctx, query)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to create database %s: %w", name, err)
|
return fmt.Errorf("failed to create database %s: %w", name, err)
|
||||||
@@ -148,7 +181,13 @@ func (m *MySQL) DropDatabase(ctx context.Context, name string) error {
|
|||||||
return fmt.Errorf("not connected to database")
|
return fmt.Errorf("not connected to database")
|
||||||
}
|
}
|
||||||
|
|
||||||
query := fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", name)
|
// Validate identifier to prevent SQL injection
|
||||||
|
if err := validateMySQLIdentifier(name); err != nil {
|
||||||
|
return fmt.Errorf("invalid database name: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Use safe quoting for identifier
|
||||||
|
query := fmt.Sprintf("DROP DATABASE IF EXISTS %s", quoteMySQLIdentifier(name))
|
||||||
_, err := m.db.ExecContext(ctx, query)
|
_, err := m.db.ExecContext(ctx, query)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to drop database %s: %w", name, err)
|
return fmt.Errorf("failed to drop database %s: %w", name, err)
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ import (
|
|||||||
|
|
||||||
"github.com/jackc/pgx/v5/pgxpool"
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
"github.com/jackc/pgx/v5/stdlib"
|
"github.com/jackc/pgx/v5/stdlib"
|
||||||
_ "github.com/jackc/pgx/v5/stdlib" // PostgreSQL driver (pgx)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// PostgreSQL implements Database interface for PostgreSQL
|
// PostgreSQL implements Database interface for PostgreSQL
|
||||||
@@ -163,14 +162,47 @@ func (p *PostgreSQL) ListTables(ctx context.Context, database string) ([]string,
|
|||||||
return tables, rows.Err()
|
return tables, rows.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// validateIdentifier checks if a database/table name is safe for use in SQL
|
||||||
|
// Prevents SQL injection by only allowing alphanumeric names with underscores
|
||||||
|
func validateIdentifier(name string) error {
|
||||||
|
if len(name) == 0 {
|
||||||
|
return fmt.Errorf("identifier cannot be empty")
|
||||||
|
}
|
||||||
|
if len(name) > 63 {
|
||||||
|
return fmt.Errorf("identifier too long (max 63 chars): %s", name)
|
||||||
|
}
|
||||||
|
// Only allow alphanumeric, underscores, and must start with letter or underscore
|
||||||
|
for i, c := range name {
|
||||||
|
if i == 0 && !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_') {
|
||||||
|
return fmt.Errorf("identifier must start with letter or underscore: %s", name)
|
||||||
|
}
|
||||||
|
if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_') {
|
||||||
|
return fmt.Errorf("identifier contains invalid character %q: %s", c, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// quoteIdentifier safely quotes a PostgreSQL identifier
|
||||||
|
func quoteIdentifier(name string) string {
|
||||||
|
// Double any existing double quotes and wrap in double quotes
|
||||||
|
return `"` + strings.ReplaceAll(name, `"`, `""`) + `"`
|
||||||
|
}
|
||||||
|
|
||||||
// CreateDatabase creates a new database
|
// CreateDatabase creates a new database
|
||||||
func (p *PostgreSQL) CreateDatabase(ctx context.Context, name string) error {
|
func (p *PostgreSQL) CreateDatabase(ctx context.Context, name string) error {
|
||||||
if p.db == nil {
|
if p.db == nil {
|
||||||
return fmt.Errorf("not connected to database")
|
return fmt.Errorf("not connected to database")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Validate identifier to prevent SQL injection
|
||||||
|
if err := validateIdentifier(name); err != nil {
|
||||||
|
return fmt.Errorf("invalid database name: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
// PostgreSQL doesn't support CREATE DATABASE in transactions or prepared statements
|
// PostgreSQL doesn't support CREATE DATABASE in transactions or prepared statements
|
||||||
query := fmt.Sprintf("CREATE DATABASE %s", name)
|
// Use quoted identifier for safety
|
||||||
|
query := fmt.Sprintf("CREATE DATABASE %s", quoteIdentifier(name))
|
||||||
_, err := p.db.ExecContext(ctx, query)
|
_, err := p.db.ExecContext(ctx, query)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to create database %s: %w", name, err)
|
return fmt.Errorf("failed to create database %s: %w", name, err)
|
||||||
@@ -186,8 +218,14 @@ func (p *PostgreSQL) DropDatabase(ctx context.Context, name string) error {
|
|||||||
return fmt.Errorf("not connected to database")
|
return fmt.Errorf("not connected to database")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Validate identifier to prevent SQL injection
|
||||||
|
if err := validateIdentifier(name); err != nil {
|
||||||
|
return fmt.Errorf("invalid database name: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
// Force drop connections and drop database
|
// Force drop connections and drop database
|
||||||
query := fmt.Sprintf("DROP DATABASE IF EXISTS %s", name)
|
// Use quoted identifier for safety
|
||||||
|
query := fmt.Sprintf("DROP DATABASE IF EXISTS %s", quoteIdentifier(name))
|
||||||
_, err := p.db.ExecContext(ctx, query)
|
_, err := p.db.ExecContext(ctx, query)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to drop database %s: %w", name, err)
|
return fmt.Errorf("failed to drop database %s: %w", name, err)
|
||||||
|
|||||||
228
internal/dedup/chunker.go
Normal file
228
internal/dedup/chunker.go
Normal file
@@ -0,0 +1,228 @@
|
|||||||
|
// Package dedup provides content-defined chunking and deduplication
|
||||||
|
// for database backups, similar to restic/borgbackup but with native
|
||||||
|
// database dump support.
|
||||||
|
package dedup
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"io"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Chunker constants for content-defined chunking
|
||||||
|
const (
|
||||||
|
// DefaultMinChunkSize is the minimum chunk size (4KB)
|
||||||
|
DefaultMinChunkSize = 4 * 1024
|
||||||
|
|
||||||
|
// DefaultAvgChunkSize is the target average chunk size (8KB)
|
||||||
|
DefaultAvgChunkSize = 8 * 1024
|
||||||
|
|
||||||
|
// DefaultMaxChunkSize is the maximum chunk size (32KB)
|
||||||
|
DefaultMaxChunkSize = 32 * 1024
|
||||||
|
|
||||||
|
// WindowSize for the rolling hash
|
||||||
|
WindowSize = 48
|
||||||
|
|
||||||
|
// ChunkMask determines average chunk size
|
||||||
|
// For 8KB average: we look for hash % 8192 == 0
|
||||||
|
ChunkMask = DefaultAvgChunkSize - 1
|
||||||
|
)
|
||||||
|
|
||||||
|
// Gear hash table - random values for each byte
|
||||||
|
// This is used for the Gear rolling hash which is simpler and faster than Buzhash
|
||||||
|
var gearTable = [256]uint64{
|
||||||
|
0x5c95c078, 0x22408989, 0x2d48a214, 0x12842087, 0x530f8afb, 0x474536b9, 0x2963b4f1, 0x44cb738b,
|
||||||
|
0x4ea7403d, 0x4d606b6e, 0x074ec5d3, 0x3f7e82f4, 0x4e3d26e7, 0x5cb4e82f, 0x7b0a1ef5, 0x3d4e7c92,
|
||||||
|
0x2a81ed69, 0x7f853df8, 0x452c8cf7, 0x0f4f3c9d, 0x3a5e81b7, 0x6cb2d819, 0x2e4c5f93, 0x7e8a1c57,
|
||||||
|
0x1f9d3e8c, 0x4b7c2a5d, 0x3c8f1d6e, 0x5d2a7b4f, 0x6e9c3f8a, 0x7a4d1e5c, 0x2b8c4f7d, 0x4f7d2c9e,
|
||||||
|
0x5a1e3d7c, 0x6b4f8a2d, 0x3e7c9d5a, 0x7d2a4f8b, 0x4c9e7d3a, 0x5b8a1c6e, 0x2d5f4a9c, 0x7a3c8d6b,
|
||||||
|
0x6e2a7b4d, 0x3f8c5d9a, 0x4a7d3e5b, 0x5c9a2d7e, 0x7b4e8f3c, 0x2a6d9c5b, 0x3e4a7d8c, 0x5d7b2e9a,
|
||||||
|
0x4c8a3d7b, 0x6e9d5c8a, 0x7a3e4d9c, 0x2b5c8a7d, 0x4d7e3a9c, 0x5a9c7d3e, 0x3c8b5a7d, 0x7d4e9c2a,
|
||||||
|
0x6a3d8c5b, 0x4e7a9d3c, 0x5c2a7b9e, 0x3a9d4e7c, 0x7b8c5a2d, 0x2d7e4a9c, 0x4a3c9d7b, 0x5e9a7c3d,
|
||||||
|
0x6c4d8a5b, 0x3b7e9c4a, 0x7a5c2d8b, 0x4d9a3e7c, 0x5b7c4a9e, 0x2e8a5d3c, 0x3c9e7a4d, 0x7d4a8c5b,
|
||||||
|
0x6b2d9a7c, 0x4a8c3e5d, 0x5d7a9c2e, 0x3e4c7b9a, 0x7c9d5a4b, 0x2a7e8c3d, 0x4c5a9d7e, 0x5a3e7c4b,
|
||||||
|
0x6d8a2c9e, 0x3c7b4a8d, 0x7e2d9c5a, 0x4b9a7e3c, 0x5c4d8a7b, 0x2d9e3c5a, 0x3a7c9d4e, 0x7b5a4c8d,
|
||||||
|
0x6a9c2e7b, 0x4d3e8a9c, 0x5e7b4d2a, 0x3b9a7c5d, 0x7c4e8a3b, 0x2e7d9c4a, 0x4a8b3e7d, 0x5d2c9a7e,
|
||||||
|
0x6c7a5d3e, 0x3e9c4a7b, 0x7a8d2c5e, 0x4c3e9a7d, 0x5b9c7e2a, 0x2a4d7c9e, 0x3d8a5c4b, 0x7e7b9a3c,
|
||||||
|
0x6b4a8d9e, 0x4e9c3b7a, 0x5a7d4e9c, 0x3c2a8b7d, 0x7d9e5c4a, 0x2b8a7d3e, 0x4d5c9a2b, 0x5e3a7c8d,
|
||||||
|
0x6a9d4b7c, 0x3b7a9c5e, 0x7c4b8a2d, 0x4a9e7c3b, 0x5d2b9a4e, 0x2e7c4d9a, 0x3a9b7e4c, 0x7e5a3c8b,
|
||||||
|
0x6c8a9d4e, 0x4b7c2a5e, 0x5a3e9c7d, 0x3d9a4b7c, 0x7a2d5e9c, 0x2c8b7a3d, 0x4e9c5a2b, 0x5b4d7e9a,
|
||||||
|
0x6d7a3c8b, 0x3e2b9a5d, 0x7c9d4a7e, 0x4a5e3c9b, 0x5e7a9d2c, 0x2b3c7e9a, 0x3a9e4b7d, 0x7d8a5c3e,
|
||||||
|
0x6b9c2d4a, 0x4c7e9a3b, 0x5a2c8b7e, 0x3b4d9a5c, 0x7e9b3a4d, 0x2d5a7c9e, 0x4b8d3e7a, 0x5c9a4b2d,
|
||||||
|
0x6a7c8d9e, 0x3c9e5a7b, 0x7b4a2c9d, 0x4d3b7e9a, 0x5e9c4a3b, 0x2a7b9d4e, 0x3e5c8a7b, 0x7a9d3e5c,
|
||||||
|
0x6c2a7b8d, 0x4e9a5c3b, 0x5b7d2a9e, 0x3a4e9c7b, 0x7d8b3a5c, 0x2c9e7a4b, 0x4a3d5e9c, 0x5d7b8a2e,
|
||||||
|
0x6b9a4c7d, 0x3d5a9e4b, 0x7e2c7b9a, 0x4b9d3a5e, 0x5c4e7a9d, 0x2e8a3c7b, 0x3b7c9e5a, 0x7a4d8b3e,
|
||||||
|
0x6d9c5a2b, 0x4a7e3d9c, 0x5e2a9b7d, 0x3c9a7e4b, 0x7b3e5c9a, 0x2a4b8d7e, 0x4d9c2a5b, 0x5a7d9e3c,
|
||||||
|
0x6c3b8a7d, 0x3e9d4a5c, 0x7d5c2b9e, 0x4c8a7d3b, 0x5b9e3c7a, 0x2d7a9c4e, 0x3a5e7b9d, 0x7e8b4a3c,
|
||||||
|
0x6a2d9e7b, 0x4b3e5a9d, 0x5d9c7b2a, 0x3b7d4e9c, 0x7c9a3b5e, 0x2e5c8a7d, 0x4a7b9d3e, 0x5c3a7e9b,
|
||||||
|
0x6d9e5c4a, 0x3c4a7b9e, 0x7a9d2e5c, 0x4e7c9a3d, 0x5a8b4e7c, 0x2b9a3d7e, 0x3d5b8a9c, 0x7b4e9a2d,
|
||||||
|
0x6c7d3a9e, 0x4a9c5e3b, 0x5e2b7d9a, 0x3a8d4c7b, 0x7d3e9a5c, 0x2c7a8b9e, 0x4b5d3a7c, 0x5c9a7e2b,
|
||||||
|
0x6a4b9d3e, 0x3e7c2a9d, 0x7c8a5b4e, 0x4d9e3c7a, 0x5b3a9e7c, 0x2e9c7b4a, 0x3b4e8a9d, 0x7a9c4e3b,
|
||||||
|
0x6d2a7c9e, 0x4c8b9a5d, 0x5a9e2b7c, 0x3c3d7a9e, 0x7e5a9c4b, 0x2a8d3e7c, 0x4e7a5c9b, 0x5d9b8a2e,
|
||||||
|
0x6b4c9e7a, 0x3a9d5b4e, 0x7b2e8a9c, 0x4a5c3e9b, 0x5c9a4d7e, 0x2d7e9a3c, 0x3e8b7c5a, 0x7c9e2a4d,
|
||||||
|
0x6a3b7d9c, 0x4d9a8b3e, 0x5e5c2a7b, 0x3b4a9d7c, 0x7a7c5e9b, 0x2c9b4a8d, 0x4b3e7c9a, 0x5a9d3b7e,
|
||||||
|
0x6c8a4e9d, 0x3d7b9c5a, 0x7e2a4b9c, 0x4c9e5d3a, 0x5b7a9c4e, 0x2e4d8a7b, 0x3a9c7e5d, 0x7b8d3a9e,
|
||||||
|
0x6d5c9a4b, 0x4a2e7b9d, 0x5d9b4c8a, 0x3c7a9e2b, 0x7d4b8c9e, 0x2b9a5c4d, 0x4e7d3a9c, 0x5c8a9e7b,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Chunk represents a single deduplicated chunk
|
||||||
|
type Chunk struct {
|
||||||
|
// Hash is the SHA-256 hash of the chunk data (content-addressed)
|
||||||
|
Hash string
|
||||||
|
|
||||||
|
// Data is the raw chunk bytes
|
||||||
|
Data []byte
|
||||||
|
|
||||||
|
// Offset is the byte offset in the original file
|
||||||
|
Offset int64
|
||||||
|
|
||||||
|
// Length is the size of this chunk
|
||||||
|
Length int
|
||||||
|
}
|
||||||
|
|
||||||
|
// ChunkerConfig holds configuration for the chunker
|
||||||
|
type ChunkerConfig struct {
|
||||||
|
MinSize int // Minimum chunk size
|
||||||
|
AvgSize int // Target average chunk size
|
||||||
|
MaxSize int // Maximum chunk size
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultChunkerConfig returns sensible defaults
|
||||||
|
func DefaultChunkerConfig() ChunkerConfig {
|
||||||
|
return ChunkerConfig{
|
||||||
|
MinSize: DefaultMinChunkSize,
|
||||||
|
AvgSize: DefaultAvgChunkSize,
|
||||||
|
MaxSize: DefaultMaxChunkSize,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Chunker performs content-defined chunking using Gear hash
|
||||||
|
type Chunker struct {
|
||||||
|
reader io.Reader
|
||||||
|
config ChunkerConfig
|
||||||
|
|
||||||
|
// Rolling hash state
|
||||||
|
hash uint64
|
||||||
|
|
||||||
|
// Current chunk state
|
||||||
|
buf []byte
|
||||||
|
offset int64
|
||||||
|
mask uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewChunker creates a new chunker for the given reader
|
||||||
|
func NewChunker(r io.Reader, config ChunkerConfig) *Chunker {
|
||||||
|
// Calculate mask for target average size
|
||||||
|
// We want: avg_size = 1 / P(boundary)
|
||||||
|
// With mask, P(boundary) = 1 / (mask + 1)
|
||||||
|
// So mask = avg_size - 1
|
||||||
|
mask := uint64(config.AvgSize - 1)
|
||||||
|
|
||||||
|
return &Chunker{
|
||||||
|
reader: r,
|
||||||
|
config: config,
|
||||||
|
buf: make([]byte, 0, config.MaxSize),
|
||||||
|
mask: mask,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Next returns the next chunk from the input stream
|
||||||
|
// Returns io.EOF when no more data is available
|
||||||
|
func (c *Chunker) Next() (*Chunk, error) {
|
||||||
|
c.buf = c.buf[:0]
|
||||||
|
c.hash = 0
|
||||||
|
|
||||||
|
// Read bytes until we find a chunk boundary or hit max size
|
||||||
|
singleByte := make([]byte, 1)
|
||||||
|
|
||||||
|
for {
|
||||||
|
n, err := c.reader.Read(singleByte)
|
||||||
|
if n == 0 {
|
||||||
|
if err == io.EOF {
|
||||||
|
// Return remaining data as final chunk
|
||||||
|
if len(c.buf) > 0 {
|
||||||
|
return c.makeChunk(), nil
|
||||||
|
}
|
||||||
|
return nil, io.EOF
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
b := singleByte[0]
|
||||||
|
c.buf = append(c.buf, b)
|
||||||
|
|
||||||
|
// Update Gear rolling hash
|
||||||
|
// Gear hash: hash = (hash << 1) + gear_table[byte]
|
||||||
|
c.hash = (c.hash << 1) + gearTable[b]
|
||||||
|
|
||||||
|
// Check for chunk boundary after minimum size
|
||||||
|
if len(c.buf) >= c.config.MinSize {
|
||||||
|
// Check if we hit a boundary (hash matches mask pattern)
|
||||||
|
if (c.hash & c.mask) == 0 {
|
||||||
|
return c.makeChunk(), nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Force boundary at max size
|
||||||
|
if len(c.buf) >= c.config.MaxSize {
|
||||||
|
return c.makeChunk(), nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// makeChunk creates a Chunk from the current buffer
|
||||||
|
func (c *Chunker) makeChunk() *Chunk {
|
||||||
|
// Compute SHA-256 hash
|
||||||
|
h := sha256.Sum256(c.buf)
|
||||||
|
hash := hex.EncodeToString(h[:])
|
||||||
|
|
||||||
|
// Copy data
|
||||||
|
data := make([]byte, len(c.buf))
|
||||||
|
copy(data, c.buf)
|
||||||
|
|
||||||
|
chunk := &Chunk{
|
||||||
|
Hash: hash,
|
||||||
|
Data: data,
|
||||||
|
Offset: c.offset,
|
||||||
|
Length: len(data),
|
||||||
|
}
|
||||||
|
|
||||||
|
c.offset += int64(len(data))
|
||||||
|
return chunk
|
||||||
|
}
|
||||||
|
|
||||||
|
// ChunkReader splits a reader into content-defined chunks
|
||||||
|
// and returns them via a channel for concurrent processing
|
||||||
|
func ChunkReader(r io.Reader, config ChunkerConfig) (<-chan *Chunk, <-chan error) {
|
||||||
|
chunks := make(chan *Chunk, 100)
|
||||||
|
errs := make(chan error, 1)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer close(chunks)
|
||||||
|
defer close(errs)
|
||||||
|
|
||||||
|
chunker := NewChunker(r, config)
|
||||||
|
for {
|
||||||
|
chunk, err := chunker.Next()
|
||||||
|
if err == io.EOF {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
errs <- err
|
||||||
|
return
|
||||||
|
}
|
||||||
|
chunks <- chunk
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
return chunks, errs
|
||||||
|
}
|
||||||
|
|
||||||
|
// HashData computes SHA-256 hash of data
|
||||||
|
func HashData(data []byte) string {
|
||||||
|
h := sha256.Sum256(data)
|
||||||
|
return hex.EncodeToString(h[:])
|
||||||
|
}
|
||||||
217
internal/dedup/chunker_test.go
Normal file
217
internal/dedup/chunker_test.go
Normal file
@@ -0,0 +1,217 @@
|
|||||||
|
package dedup
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"crypto/rand"
|
||||||
|
"io"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestChunker_Basic(t *testing.T) {
|
||||||
|
// Create test data
|
||||||
|
data := make([]byte, 100*1024) // 100KB
|
||||||
|
rand.Read(data)
|
||||||
|
|
||||||
|
chunker := NewChunker(bytes.NewReader(data), DefaultChunkerConfig())
|
||||||
|
|
||||||
|
var chunks []*Chunk
|
||||||
|
var totalBytes int
|
||||||
|
|
||||||
|
for {
|
||||||
|
chunk, err := chunker.Next()
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Chunker.Next() error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
chunks = append(chunks, chunk)
|
||||||
|
totalBytes += chunk.Length
|
||||||
|
|
||||||
|
// Verify chunk properties
|
||||||
|
if chunk.Length < DefaultMinChunkSize && len(chunks) < 10 {
|
||||||
|
// Only the last chunk can be smaller than min
|
||||||
|
// (unless file is smaller than min)
|
||||||
|
}
|
||||||
|
if chunk.Length > DefaultMaxChunkSize {
|
||||||
|
t.Errorf("Chunk %d exceeds max size: %d > %d", len(chunks), chunk.Length, DefaultMaxChunkSize)
|
||||||
|
}
|
||||||
|
if chunk.Hash == "" {
|
||||||
|
t.Errorf("Chunk %d has empty hash", len(chunks))
|
||||||
|
}
|
||||||
|
if len(chunk.Hash) != 64 { // SHA-256 hex length
|
||||||
|
t.Errorf("Chunk %d has invalid hash length: %d", len(chunks), len(chunk.Hash))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if totalBytes != len(data) {
|
||||||
|
t.Errorf("Total bytes mismatch: got %d, want %d", totalBytes, len(data))
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("Chunked %d bytes into %d chunks", totalBytes, len(chunks))
|
||||||
|
t.Logf("Average chunk size: %d bytes", totalBytes/len(chunks))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunker_Deterministic(t *testing.T) {
|
||||||
|
// Same data should produce same chunks
|
||||||
|
data := make([]byte, 50*1024)
|
||||||
|
rand.Read(data)
|
||||||
|
|
||||||
|
// First pass
|
||||||
|
chunker1 := NewChunker(bytes.NewReader(data), DefaultChunkerConfig())
|
||||||
|
var hashes1 []string
|
||||||
|
for {
|
||||||
|
chunk, err := chunker1.Next()
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hashes1 = append(hashes1, chunk.Hash)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Second pass
|
||||||
|
chunker2 := NewChunker(bytes.NewReader(data), DefaultChunkerConfig())
|
||||||
|
var hashes2 []string
|
||||||
|
for {
|
||||||
|
chunk, err := chunker2.Next()
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hashes2 = append(hashes2, chunk.Hash)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compare
|
||||||
|
if len(hashes1) != len(hashes2) {
|
||||||
|
t.Fatalf("Different chunk counts: %d vs %d", len(hashes1), len(hashes2))
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := range hashes1 {
|
||||||
|
if hashes1[i] != hashes2[i] {
|
||||||
|
t.Errorf("Hash mismatch at chunk %d: %s vs %s", i, hashes1[i], hashes2[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunker_ShiftedData(t *testing.T) {
|
||||||
|
// Test that shifted data still shares chunks (the key CDC benefit)
|
||||||
|
original := make([]byte, 100*1024)
|
||||||
|
rand.Read(original)
|
||||||
|
|
||||||
|
// Create shifted version (prepend some bytes)
|
||||||
|
prefix := make([]byte, 1000)
|
||||||
|
rand.Read(prefix)
|
||||||
|
shifted := append(prefix, original...)
|
||||||
|
|
||||||
|
// Chunk both
|
||||||
|
config := DefaultChunkerConfig()
|
||||||
|
|
||||||
|
chunker1 := NewChunker(bytes.NewReader(original), config)
|
||||||
|
hashes1 := make(map[string]bool)
|
||||||
|
for {
|
||||||
|
chunk, err := chunker1.Next()
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hashes1[chunk.Hash] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
chunker2 := NewChunker(bytes.NewReader(shifted), config)
|
||||||
|
var matched, total int
|
||||||
|
for {
|
||||||
|
chunk, err := chunker2.Next()
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
total++
|
||||||
|
if hashes1[chunk.Hash] {
|
||||||
|
matched++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Should have significant overlap despite the shift
|
||||||
|
overlapRatio := float64(matched) / float64(total)
|
||||||
|
t.Logf("Chunk overlap after %d-byte shift: %.1f%% (%d/%d chunks)",
|
||||||
|
len(prefix), overlapRatio*100, matched, total)
|
||||||
|
|
||||||
|
// We expect at least 50% overlap for content-defined chunking
|
||||||
|
if overlapRatio < 0.5 {
|
||||||
|
t.Errorf("Low chunk overlap: %.1f%% (expected >50%%)", overlapRatio*100)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunker_SmallFile(t *testing.T) {
|
||||||
|
// File smaller than min chunk size
|
||||||
|
data := []byte("hello world")
|
||||||
|
chunker := NewChunker(bytes.NewReader(data), DefaultChunkerConfig())
|
||||||
|
|
||||||
|
chunk, err := chunker.Next()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if chunk.Length != len(data) {
|
||||||
|
t.Errorf("Expected chunk length %d, got %d", len(data), chunk.Length)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Should be EOF after
|
||||||
|
_, err = chunker.Next()
|
||||||
|
if err != io.EOF {
|
||||||
|
t.Errorf("Expected EOF, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunker_EmptyFile(t *testing.T) {
|
||||||
|
chunker := NewChunker(bytes.NewReader(nil), DefaultChunkerConfig())
|
||||||
|
|
||||||
|
_, err := chunker.Next()
|
||||||
|
if err != io.EOF {
|
||||||
|
t.Errorf("Expected EOF for empty file, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHashData(t *testing.T) {
|
||||||
|
hash := HashData([]byte("test"))
|
||||||
|
if len(hash) != 64 {
|
||||||
|
t.Errorf("Expected 64-char hash, got %d", len(hash))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Known SHA-256 of "test"
|
||||||
|
expected := "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"
|
||||||
|
if hash != expected {
|
||||||
|
t.Errorf("Hash mismatch: got %s, want %s", hash, expected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkChunker(b *testing.B) {
|
||||||
|
// 1MB of random data
|
||||||
|
data := make([]byte, 1024*1024)
|
||||||
|
rand.Read(data)
|
||||||
|
|
||||||
|
b.ResetTimer()
|
||||||
|
b.SetBytes(int64(len(data)))
|
||||||
|
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
chunker := NewChunker(bytes.NewReader(data), DefaultChunkerConfig())
|
||||||
|
for {
|
||||||
|
_, err := chunker.Next()
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
239
internal/dedup/index.go
Normal file
239
internal/dedup/index.go
Normal file
@@ -0,0 +1,239 @@
|
|||||||
|
package dedup
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
_ "github.com/mattn/go-sqlite3" // SQLite driver
|
||||||
|
)
|
||||||
|
|
||||||
|
// ChunkIndex provides fast chunk lookups using SQLite
|
||||||
|
type ChunkIndex struct {
|
||||||
|
db *sql.DB
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewChunkIndex opens or creates a chunk index database
|
||||||
|
func NewChunkIndex(basePath string) (*ChunkIndex, error) {
|
||||||
|
dbPath := filepath.Join(basePath, "chunks.db")
|
||||||
|
|
||||||
|
db, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL&_synchronous=NORMAL")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to open chunk index: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
idx := &ChunkIndex{db: db}
|
||||||
|
if err := idx.migrate(); err != nil {
|
||||||
|
db.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return idx, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// migrate creates the schema if needed
|
||||||
|
func (idx *ChunkIndex) migrate() error {
|
||||||
|
schema := `
|
||||||
|
CREATE TABLE IF NOT EXISTS chunks (
|
||||||
|
hash TEXT PRIMARY KEY,
|
||||||
|
size_raw INTEGER NOT NULL,
|
||||||
|
size_stored INTEGER NOT NULL,
|
||||||
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
last_accessed DATETIME,
|
||||||
|
ref_count INTEGER DEFAULT 1
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS manifests (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
database_type TEXT,
|
||||||
|
database_name TEXT,
|
||||||
|
database_host TEXT,
|
||||||
|
created_at DATETIME,
|
||||||
|
original_size INTEGER,
|
||||||
|
stored_size INTEGER,
|
||||||
|
chunk_count INTEGER,
|
||||||
|
new_chunks INTEGER,
|
||||||
|
dedup_ratio REAL,
|
||||||
|
sha256 TEXT,
|
||||||
|
verified_at DATETIME
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_chunks_created ON chunks(created_at);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_chunks_accessed ON chunks(last_accessed);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_manifests_created ON manifests(created_at);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_manifests_database ON manifests(database_name);
|
||||||
|
`
|
||||||
|
|
||||||
|
_, err := idx.db.Exec(schema)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close closes the database
|
||||||
|
func (idx *ChunkIndex) Close() error {
|
||||||
|
return idx.db.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddChunk records a chunk in the index
|
||||||
|
func (idx *ChunkIndex) AddChunk(hash string, sizeRaw, sizeStored int) error {
|
||||||
|
_, err := idx.db.Exec(`
|
||||||
|
INSERT INTO chunks (hash, size_raw, size_stored, created_at, last_accessed, ref_count)
|
||||||
|
VALUES (?, ?, ?, ?, ?, 1)
|
||||||
|
ON CONFLICT(hash) DO UPDATE SET
|
||||||
|
ref_count = ref_count + 1,
|
||||||
|
last_accessed = ?
|
||||||
|
`, hash, sizeRaw, sizeStored, time.Now(), time.Now(), time.Now())
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// HasChunk checks if a chunk exists in the index
|
||||||
|
func (idx *ChunkIndex) HasChunk(hash string) (bool, error) {
|
||||||
|
var count int
|
||||||
|
err := idx.db.QueryRow("SELECT COUNT(*) FROM chunks WHERE hash = ?", hash).Scan(&count)
|
||||||
|
return count > 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetChunk retrieves chunk metadata
|
||||||
|
func (idx *ChunkIndex) GetChunk(hash string) (*ChunkMeta, error) {
|
||||||
|
var m ChunkMeta
|
||||||
|
err := idx.db.QueryRow(`
|
||||||
|
SELECT hash, size_raw, size_stored, created_at, ref_count
|
||||||
|
FROM chunks WHERE hash = ?
|
||||||
|
`, hash).Scan(&m.Hash, &m.SizeRaw, &m.SizeStored, &m.CreatedAt, &m.RefCount)
|
||||||
|
if err == sql.ErrNoRows {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ChunkMeta holds metadata about a chunk
|
||||||
|
type ChunkMeta struct {
|
||||||
|
Hash string
|
||||||
|
SizeRaw int64
|
||||||
|
SizeStored int64
|
||||||
|
CreatedAt time.Time
|
||||||
|
RefCount int
|
||||||
|
}
|
||||||
|
|
||||||
|
// DecrementRef decreases the reference count for a chunk
|
||||||
|
// Returns true if the chunk should be deleted (ref_count <= 0)
|
||||||
|
func (idx *ChunkIndex) DecrementRef(hash string) (shouldDelete bool, err error) {
|
||||||
|
result, err := idx.db.Exec(`
|
||||||
|
UPDATE chunks SET ref_count = ref_count - 1 WHERE hash = ?
|
||||||
|
`, hash)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
affected, _ := result.RowsAffected()
|
||||||
|
if affected == 0 {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var refCount int
|
||||||
|
err = idx.db.QueryRow("SELECT ref_count FROM chunks WHERE hash = ?", hash).Scan(&refCount)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return refCount <= 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// RemoveChunk removes a chunk from the index
|
||||||
|
func (idx *ChunkIndex) RemoveChunk(hash string) error {
|
||||||
|
_, err := idx.db.Exec("DELETE FROM chunks WHERE hash = ?", hash)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddManifest records a manifest in the index
|
||||||
|
func (idx *ChunkIndex) AddManifest(m *Manifest) error {
|
||||||
|
_, err := idx.db.Exec(`
|
||||||
|
INSERT OR REPLACE INTO manifests
|
||||||
|
(id, database_type, database_name, database_host, created_at,
|
||||||
|
original_size, stored_size, chunk_count, new_chunks, dedup_ratio, sha256)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
`, m.ID, m.DatabaseType, m.DatabaseName, m.DatabaseHost, m.CreatedAt,
|
||||||
|
m.OriginalSize, m.StoredSize, m.ChunkCount, m.NewChunks, m.DedupRatio, m.SHA256)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// RemoveManifest removes a manifest from the index
|
||||||
|
func (idx *ChunkIndex) RemoveManifest(id string) error {
|
||||||
|
_, err := idx.db.Exec("DELETE FROM manifests WHERE id = ?", id)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// IndexStats holds statistics about the dedup index
|
||||||
|
type IndexStats struct {
|
||||||
|
TotalChunks int64
|
||||||
|
TotalManifests int64
|
||||||
|
TotalSizeRaw int64 // Uncompressed, undeduplicated
|
||||||
|
TotalSizeStored int64 // On-disk after dedup+compression
|
||||||
|
DedupRatio float64
|
||||||
|
OldestChunk time.Time
|
||||||
|
NewestChunk time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stats returns statistics about the index
|
||||||
|
func (idx *ChunkIndex) Stats() (*IndexStats, error) {
|
||||||
|
stats := &IndexStats{}
|
||||||
|
|
||||||
|
var oldestStr, newestStr string
|
||||||
|
err := idx.db.QueryRow(`
|
||||||
|
SELECT
|
||||||
|
COUNT(*),
|
||||||
|
COALESCE(SUM(size_raw), 0),
|
||||||
|
COALESCE(SUM(size_stored), 0),
|
||||||
|
COALESCE(MIN(created_at), ''),
|
||||||
|
COALESCE(MAX(created_at), '')
|
||||||
|
FROM chunks
|
||||||
|
`).Scan(&stats.TotalChunks, &stats.TotalSizeRaw, &stats.TotalSizeStored,
|
||||||
|
&oldestStr, &newestStr)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse time strings
|
||||||
|
if oldestStr != "" {
|
||||||
|
stats.OldestChunk, _ = time.Parse("2006-01-02 15:04:05", oldestStr)
|
||||||
|
}
|
||||||
|
if newestStr != "" {
|
||||||
|
stats.NewestChunk, _ = time.Parse("2006-01-02 15:04:05", newestStr)
|
||||||
|
}
|
||||||
|
|
||||||
|
idx.db.QueryRow("SELECT COUNT(*) FROM manifests").Scan(&stats.TotalManifests)
|
||||||
|
|
||||||
|
if stats.TotalSizeRaw > 0 {
|
||||||
|
stats.DedupRatio = 1.0 - float64(stats.TotalSizeStored)/float64(stats.TotalSizeRaw)
|
||||||
|
}
|
||||||
|
|
||||||
|
return stats, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListOrphanedChunks returns chunks that have ref_count <= 0
|
||||||
|
func (idx *ChunkIndex) ListOrphanedChunks() ([]string, error) {
|
||||||
|
rows, err := idx.db.Query("SELECT hash FROM chunks WHERE ref_count <= 0")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
var hashes []string
|
||||||
|
for rows.Next() {
|
||||||
|
var hash string
|
||||||
|
if err := rows.Scan(&hash); err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
hashes = append(hashes, hash)
|
||||||
|
}
|
||||||
|
return hashes, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Vacuum cleans up the database
|
||||||
|
func (idx *ChunkIndex) Vacuum() error {
|
||||||
|
_, err := idx.db.Exec("VACUUM")
|
||||||
|
return err
|
||||||
|
}
|
||||||
188
internal/dedup/manifest.go
Normal file
188
internal/dedup/manifest.go
Normal file
@@ -0,0 +1,188 @@
|
|||||||
|
package dedup
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Manifest describes a single backup as a list of chunks
|
||||||
|
type Manifest struct {
|
||||||
|
// ID is the unique identifier (typically timestamp-based)
|
||||||
|
ID string `json:"id"`
|
||||||
|
|
||||||
|
// Name is an optional human-readable name
|
||||||
|
Name string `json:"name,omitempty"`
|
||||||
|
|
||||||
|
// CreatedAt is when this backup was created
|
||||||
|
CreatedAt time.Time `json:"created_at"`
|
||||||
|
|
||||||
|
// Database information
|
||||||
|
DatabaseType string `json:"database_type"` // postgres, mysql
|
||||||
|
DatabaseName string `json:"database_name"`
|
||||||
|
DatabaseHost string `json:"database_host"`
|
||||||
|
|
||||||
|
// Chunks is the ordered list of chunk hashes
|
||||||
|
// The file is reconstructed by concatenating chunks in order
|
||||||
|
Chunks []ChunkRef `json:"chunks"`
|
||||||
|
|
||||||
|
// Stats about the backup
|
||||||
|
OriginalSize int64 `json:"original_size"` // Size before deduplication
|
||||||
|
StoredSize int64 `json:"stored_size"` // Size after dedup (new chunks only)
|
||||||
|
ChunkCount int `json:"chunk_count"` // Total chunks
|
||||||
|
NewChunks int `json:"new_chunks"` // Chunks that weren't deduplicated
|
||||||
|
DedupRatio float64 `json:"dedup_ratio"` // 1.0 = no dedup, 0.0 = 100% dedup
|
||||||
|
|
||||||
|
// Encryption and compression settings used
|
||||||
|
Encrypted bool `json:"encrypted"`
|
||||||
|
Compressed bool `json:"compressed"`
|
||||||
|
|
||||||
|
// Verification
|
||||||
|
SHA256 string `json:"sha256"` // Hash of reconstructed file
|
||||||
|
VerifiedAt time.Time `json:"verified_at,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ChunkRef references a chunk in the manifest
|
||||||
|
type ChunkRef struct {
|
||||||
|
Hash string `json:"h"` // SHA-256 hash (64 chars)
|
||||||
|
Offset int64 `json:"o"` // Offset in original file
|
||||||
|
Length int `json:"l"` // Chunk length
|
||||||
|
}
|
||||||
|
|
||||||
|
// ManifestStore manages backup manifests
|
||||||
|
type ManifestStore struct {
|
||||||
|
basePath string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewManifestStore creates a new manifest store
|
||||||
|
func NewManifestStore(basePath string) (*ManifestStore, error) {
|
||||||
|
manifestDir := filepath.Join(basePath, "manifests")
|
||||||
|
if err := os.MkdirAll(manifestDir, 0700); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to create manifest directory: %w", err)
|
||||||
|
}
|
||||||
|
return &ManifestStore{basePath: basePath}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// manifestPath returns the path for a manifest ID
|
||||||
|
func (s *ManifestStore) manifestPath(id string) string {
|
||||||
|
return filepath.Join(s.basePath, "manifests", id+".manifest.json")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save writes a manifest to disk
|
||||||
|
func (s *ManifestStore) Save(m *Manifest) error {
|
||||||
|
path := s.manifestPath(m.ID)
|
||||||
|
|
||||||
|
data, err := json.MarshalIndent(m, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to marshal manifest: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Atomic write
|
||||||
|
tmpPath := path + ".tmp"
|
||||||
|
if err := os.WriteFile(tmpPath, data, 0600); err != nil {
|
||||||
|
return fmt.Errorf("failed to write manifest: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.Rename(tmpPath, path); err != nil {
|
||||||
|
os.Remove(tmpPath)
|
||||||
|
return fmt.Errorf("failed to commit manifest: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load reads a manifest from disk
|
||||||
|
func (s *ManifestStore) Load(id string) (*Manifest, error) {
|
||||||
|
path := s.manifestPath(id)
|
||||||
|
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read manifest %s: %w", id, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var m Manifest
|
||||||
|
if err := json.Unmarshal(data, &m); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to parse manifest %s: %w", id, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete removes a manifest
|
||||||
|
func (s *ManifestStore) Delete(id string) error {
|
||||||
|
path := s.manifestPath(id)
|
||||||
|
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||||
|
return fmt.Errorf("failed to delete manifest %s: %w", id, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// List returns all manifest IDs
|
||||||
|
func (s *ManifestStore) List() ([]string, error) {
|
||||||
|
manifestDir := filepath.Join(s.basePath, "manifests")
|
||||||
|
entries, err := os.ReadDir(manifestDir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to list manifests: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var ids []string
|
||||||
|
for _, e := range entries {
|
||||||
|
if e.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name := e.Name()
|
||||||
|
if len(name) > 14 && name[len(name)-14:] == ".manifest.json" {
|
||||||
|
ids = append(ids, name[:len(name)-14])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return ids, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListAll returns all manifests sorted by creation time (newest first)
|
||||||
|
func (s *ManifestStore) ListAll() ([]*Manifest, error) {
|
||||||
|
ids, err := s.List()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var manifests []*Manifest
|
||||||
|
for _, id := range ids {
|
||||||
|
m, err := s.Load(id)
|
||||||
|
if err != nil {
|
||||||
|
continue // Skip corrupted manifests
|
||||||
|
}
|
||||||
|
manifests = append(manifests, m)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort by creation time (newest first)
|
||||||
|
for i := 0; i < len(manifests)-1; i++ {
|
||||||
|
for j := i + 1; j < len(manifests); j++ {
|
||||||
|
if manifests[j].CreatedAt.After(manifests[i].CreatedAt) {
|
||||||
|
manifests[i], manifests[j] = manifests[j], manifests[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return manifests, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetChunkHashes returns all unique chunk hashes referenced by manifests
|
||||||
|
func (s *ManifestStore) GetChunkHashes() (map[string]int, error) {
|
||||||
|
manifests, err := s.ListAll()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Map hash -> reference count
|
||||||
|
refs := make(map[string]int)
|
||||||
|
for _, m := range manifests {
|
||||||
|
for _, c := range m.Chunks {
|
||||||
|
refs[c.Hash]++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return refs, nil
|
||||||
|
}
|
||||||
367
internal/dedup/store.go
Normal file
367
internal/dedup/store.go
Normal file
@@ -0,0 +1,367 @@
|
|||||||
|
package dedup
|
||||||
|
|
||||||
|
import (
|
||||||
|
"compress/gzip"
|
||||||
|
"crypto/aes"
|
||||||
|
"crypto/cipher"
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ChunkStore manages content-addressed chunk storage
|
||||||
|
// Chunks are stored as: <base>/<prefix>/<hash>.chunk[.gz][.enc]
|
||||||
|
type ChunkStore struct {
|
||||||
|
basePath string
|
||||||
|
compress bool
|
||||||
|
encryptionKey []byte // 32 bytes for AES-256
|
||||||
|
mu sync.RWMutex
|
||||||
|
existingChunks map[string]bool // Cache of known chunks
|
||||||
|
}
|
||||||
|
|
||||||
|
// StoreConfig holds configuration for the chunk store
|
||||||
|
type StoreConfig struct {
|
||||||
|
BasePath string
|
||||||
|
Compress bool // Enable gzip compression
|
||||||
|
EncryptionKey string // Optional: hex-encoded 32-byte key for AES-256-GCM
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewChunkStore creates a new chunk store
|
||||||
|
func NewChunkStore(config StoreConfig) (*ChunkStore, error) {
|
||||||
|
store := &ChunkStore{
|
||||||
|
basePath: config.BasePath,
|
||||||
|
compress: config.Compress,
|
||||||
|
existingChunks: make(map[string]bool),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse encryption key if provided
|
||||||
|
if config.EncryptionKey != "" {
|
||||||
|
key, err := hex.DecodeString(config.EncryptionKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("invalid encryption key: %w", err)
|
||||||
|
}
|
||||||
|
if len(key) != 32 {
|
||||||
|
return nil, fmt.Errorf("encryption key must be 32 bytes (got %d)", len(key))
|
||||||
|
}
|
||||||
|
store.encryptionKey = key
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create base directory structure
|
||||||
|
if err := os.MkdirAll(config.BasePath, 0700); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to create chunk store: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create chunks and manifests directories
|
||||||
|
for _, dir := range []string{"chunks", "manifests"} {
|
||||||
|
if err := os.MkdirAll(filepath.Join(config.BasePath, dir), 0700); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to create %s directory: %w", dir, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return store, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// chunkPath returns the filesystem path for a chunk hash
|
||||||
|
// Uses 2-character prefix for directory sharding (256 subdirs)
|
||||||
|
func (s *ChunkStore) chunkPath(hash string) string {
|
||||||
|
if len(hash) < 2 {
|
||||||
|
return filepath.Join(s.basePath, "chunks", "xx", hash+s.chunkExt())
|
||||||
|
}
|
||||||
|
prefix := hash[:2]
|
||||||
|
return filepath.Join(s.basePath, "chunks", prefix, hash+s.chunkExt())
|
||||||
|
}
|
||||||
|
|
||||||
|
// chunkExt returns the file extension based on compression/encryption settings
|
||||||
|
func (s *ChunkStore) chunkExt() string {
|
||||||
|
ext := ".chunk"
|
||||||
|
if s.compress {
|
||||||
|
ext += ".gz"
|
||||||
|
}
|
||||||
|
if s.encryptionKey != nil {
|
||||||
|
ext += ".enc"
|
||||||
|
}
|
||||||
|
return ext
|
||||||
|
}
|
||||||
|
|
||||||
|
// Has checks if a chunk exists in the store
|
||||||
|
func (s *ChunkStore) Has(hash string) bool {
|
||||||
|
s.mu.RLock()
|
||||||
|
if exists, ok := s.existingChunks[hash]; ok {
|
||||||
|
s.mu.RUnlock()
|
||||||
|
return exists
|
||||||
|
}
|
||||||
|
s.mu.RUnlock()
|
||||||
|
|
||||||
|
// Check filesystem
|
||||||
|
path := s.chunkPath(hash)
|
||||||
|
_, err := os.Stat(path)
|
||||||
|
exists := err == nil
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
s.existingChunks[hash] = exists
|
||||||
|
s.mu.Unlock()
|
||||||
|
|
||||||
|
return exists
|
||||||
|
}
|
||||||
|
|
||||||
|
// Put stores a chunk, returning true if it was new (not deduplicated)
|
||||||
|
func (s *ChunkStore) Put(chunk *Chunk) (isNew bool, err error) {
|
||||||
|
// Check if already exists (deduplication!)
|
||||||
|
if s.Has(chunk.Hash) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
path := s.chunkPath(chunk.Hash)
|
||||||
|
|
||||||
|
// Create prefix directory
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
|
||||||
|
return false, fmt.Errorf("failed to create chunk directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prepare data
|
||||||
|
data := chunk.Data
|
||||||
|
|
||||||
|
// Compress if enabled
|
||||||
|
if s.compress {
|
||||||
|
data, err = s.compressData(data)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("compression failed: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Encrypt if enabled
|
||||||
|
if s.encryptionKey != nil {
|
||||||
|
data, err = s.encryptData(data)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("encryption failed: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write atomically (write to temp, then rename)
|
||||||
|
tmpPath := path + ".tmp"
|
||||||
|
if err := os.WriteFile(tmpPath, data, 0600); err != nil {
|
||||||
|
return false, fmt.Errorf("failed to write chunk: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.Rename(tmpPath, path); err != nil {
|
||||||
|
os.Remove(tmpPath)
|
||||||
|
return false, fmt.Errorf("failed to commit chunk: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update cache
|
||||||
|
s.mu.Lock()
|
||||||
|
s.existingChunks[chunk.Hash] = true
|
||||||
|
s.mu.Unlock()
|
||||||
|
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get retrieves a chunk by hash
|
||||||
|
func (s *ChunkStore) Get(hash string) (*Chunk, error) {
|
||||||
|
path := s.chunkPath(hash)
|
||||||
|
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read chunk %s: %w", hash, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decrypt if encrypted
|
||||||
|
if s.encryptionKey != nil {
|
||||||
|
data, err = s.decryptData(data)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("decryption failed: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decompress if compressed
|
||||||
|
if s.compress {
|
||||||
|
data, err = s.decompressData(data)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("decompression failed: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify hash
|
||||||
|
h := sha256.Sum256(data)
|
||||||
|
actualHash := hex.EncodeToString(h[:])
|
||||||
|
if actualHash != hash {
|
||||||
|
return nil, fmt.Errorf("chunk hash mismatch: expected %s, got %s", hash, actualHash)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Chunk{
|
||||||
|
Hash: hash,
|
||||||
|
Data: data,
|
||||||
|
Length: len(data),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete removes a chunk from the store
|
||||||
|
func (s *ChunkStore) Delete(hash string) error {
|
||||||
|
path := s.chunkPath(hash)
|
||||||
|
|
||||||
|
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||||
|
return fmt.Errorf("failed to delete chunk %s: %w", hash, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
s.mu.Lock()
|
||||||
|
delete(s.existingChunks, hash)
|
||||||
|
s.mu.Unlock()
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stats returns storage statistics
|
||||||
|
type StoreStats struct {
|
||||||
|
TotalChunks int64
|
||||||
|
TotalSize int64 // Bytes on disk (after compression/encryption)
|
||||||
|
UniqueSize int64 // Bytes of unique data
|
||||||
|
Directories int
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stats returns statistics about the chunk store
|
||||||
|
func (s *ChunkStore) Stats() (*StoreStats, error) {
|
||||||
|
stats := &StoreStats{}
|
||||||
|
|
||||||
|
chunksDir := filepath.Join(s.basePath, "chunks")
|
||||||
|
err := filepath.Walk(chunksDir, func(path string, info os.FileInfo, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if info.IsDir() {
|
||||||
|
stats.Directories++
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
stats.TotalChunks++
|
||||||
|
stats.TotalSize += info.Size()
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
return stats, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoadIndex loads the existing chunk hashes into memory
|
||||||
|
func (s *ChunkStore) LoadIndex() error {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
|
s.existingChunks = make(map[string]bool)
|
||||||
|
|
||||||
|
chunksDir := filepath.Join(s.basePath, "chunks")
|
||||||
|
return filepath.Walk(chunksDir, func(path string, info os.FileInfo, err error) error {
|
||||||
|
if err != nil || info.IsDir() {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract hash from filename
|
||||||
|
base := filepath.Base(path)
|
||||||
|
hash := base
|
||||||
|
// Remove extensions
|
||||||
|
for _, ext := range []string{".enc", ".gz", ".chunk"} {
|
||||||
|
if len(hash) > len(ext) && hash[len(hash)-len(ext):] == ext {
|
||||||
|
hash = hash[:len(hash)-len(ext)]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(hash) == 64 { // SHA-256 hex length
|
||||||
|
s.existingChunks[hash] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// compressData compresses data using gzip
|
||||||
|
func (s *ChunkStore) compressData(data []byte) ([]byte, error) {
|
||||||
|
var buf []byte
|
||||||
|
w, err := gzip.NewWriterLevel((*bytesBuffer)(&buf), gzip.BestCompression)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if _, err := w.Write(data); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := w.Close(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return buf, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// bytesBuffer is a simple io.Writer that appends to a byte slice
|
||||||
|
type bytesBuffer []byte
|
||||||
|
|
||||||
|
func (b *bytesBuffer) Write(p []byte) (int, error) {
|
||||||
|
*b = append(*b, p...)
|
||||||
|
return len(p), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// decompressData decompresses gzip data
|
||||||
|
func (s *ChunkStore) decompressData(data []byte) ([]byte, error) {
|
||||||
|
r, err := gzip.NewReader(&bytesReader{data: data})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer r.Close()
|
||||||
|
return io.ReadAll(r)
|
||||||
|
}
|
||||||
|
|
||||||
|
// bytesReader is a simple io.Reader from a byte slice
|
||||||
|
type bytesReader struct {
|
||||||
|
data []byte
|
||||||
|
pos int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *bytesReader) Read(p []byte) (int, error) {
|
||||||
|
if r.pos >= len(r.data) {
|
||||||
|
return 0, io.EOF
|
||||||
|
}
|
||||||
|
n := copy(p, r.data[r.pos:])
|
||||||
|
r.pos += n
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// encryptData encrypts data using AES-256-GCM
|
||||||
|
func (s *ChunkStore) encryptData(plaintext []byte) ([]byte, error) {
|
||||||
|
block, err := aes.NewCipher(s.encryptionKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
gcm, err := cipher.NewGCM(block)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
nonce := make([]byte, gcm.NonceSize())
|
||||||
|
if _, err := rand.Read(nonce); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prepend nonce to ciphertext
|
||||||
|
return gcm.Seal(nonce, nonce, plaintext, nil), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// decryptData decrypts AES-256-GCM encrypted data
|
||||||
|
func (s *ChunkStore) decryptData(ciphertext []byte) ([]byte, error) {
|
||||||
|
block, err := aes.NewCipher(s.encryptionKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
gcm, err := cipher.NewGCM(block)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(ciphertext) < gcm.NonceSize() {
|
||||||
|
return nil, fmt.Errorf("ciphertext too short")
|
||||||
|
}
|
||||||
|
|
||||||
|
nonce := ciphertext[:gcm.NonceSize()]
|
||||||
|
ciphertext = ciphertext[gcm.NonceSize():]
|
||||||
|
|
||||||
|
return gcm.Open(nil, nonce, ciphertext, nil)
|
||||||
|
}
|
||||||
@@ -223,11 +223,11 @@ func (r *DrillResult) IsSuccess() bool {
|
|||||||
|
|
||||||
// Summary returns a human-readable summary of the drill
|
// Summary returns a human-readable summary of the drill
|
||||||
func (r *DrillResult) Summary() string {
|
func (r *DrillResult) Summary() string {
|
||||||
status := "✅ PASSED"
|
status := "[OK] PASSED"
|
||||||
if !r.Success {
|
if !r.Success {
|
||||||
status = "❌ FAILED"
|
status = "[FAIL] FAILED"
|
||||||
} else if r.Status == StatusPartial {
|
} else if r.Status == StatusPartial {
|
||||||
status = "⚠️ PARTIAL"
|
status = "[WARN] PARTIAL"
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Sprintf("%s - %s (%.2fs) - %d tables, %d rows",
|
return fmt.Sprintf("%s - %s (%.2fs) - %d tables, %d rows",
|
||||||
|
|||||||
@@ -41,20 +41,20 @@ func (e *Engine) Run(ctx context.Context, config *DrillConfig) (*DrillResult, er
|
|||||||
TargetRTO: float64(config.MaxRestoreSeconds),
|
TargetRTO: float64(config.MaxRestoreSeconds),
|
||||||
}
|
}
|
||||||
|
|
||||||
e.log.Info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
e.log.Info("=====================================================")
|
||||||
e.log.Info(" 🧪 DR Drill: " + result.DrillID)
|
e.log.Info(" [TEST] DR Drill: " + result.DrillID)
|
||||||
e.log.Info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
e.log.Info("=====================================================")
|
||||||
e.log.Info("")
|
e.log.Info("")
|
||||||
|
|
||||||
// Cleanup function for error cases
|
// Cleanup function for error cases
|
||||||
var containerID string
|
var containerID string
|
||||||
cleanup := func() {
|
cleanup := func() {
|
||||||
if containerID != "" && config.CleanupOnExit && (result.Success || !config.KeepOnFailure) {
|
if containerID != "" && config.CleanupOnExit && (result.Success || !config.KeepOnFailure) {
|
||||||
e.log.Info("🗑️ Cleaning up container...")
|
e.log.Info("[DEL] Cleaning up container...")
|
||||||
e.docker.RemoveContainer(context.Background(), containerID)
|
e.docker.RemoveContainer(context.Background(), containerID)
|
||||||
} else if containerID != "" {
|
} else if containerID != "" {
|
||||||
result.ContainerKept = true
|
result.ContainerKept = true
|
||||||
e.log.Info("📦 Container kept for debugging: " + containerID)
|
e.log.Info("[PKG] Container kept for debugging: " + containerID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
@@ -88,7 +88,7 @@ func (e *Engine) Run(ctx context.Context, config *DrillConfig) (*DrillResult, er
|
|||||||
}
|
}
|
||||||
containerID = container.ID
|
containerID = container.ID
|
||||||
result.ContainerID = containerID
|
result.ContainerID = containerID
|
||||||
e.log.Info("📦 Container started: " + containerID[:12])
|
e.log.Info("[PKG] Container started: " + containerID[:12])
|
||||||
|
|
||||||
// Wait for container to be healthy
|
// Wait for container to be healthy
|
||||||
if err := e.docker.WaitForHealth(ctx, containerID, config.DatabaseType, config.ContainerTimeout); err != nil {
|
if err := e.docker.WaitForHealth(ctx, containerID, config.DatabaseType, config.ContainerTimeout); err != nil {
|
||||||
@@ -118,7 +118,7 @@ func (e *Engine) Run(ctx context.Context, config *DrillConfig) (*DrillResult, er
|
|||||||
result.RestoreTime = time.Since(restoreStart).Seconds()
|
result.RestoreTime = time.Since(restoreStart).Seconds()
|
||||||
e.completePhase(&phase, fmt.Sprintf("Restored in %.2fs", result.RestoreTime))
|
e.completePhase(&phase, fmt.Sprintf("Restored in %.2fs", result.RestoreTime))
|
||||||
result.Phases = append(result.Phases, phase)
|
result.Phases = append(result.Phases, phase)
|
||||||
e.log.Info(fmt.Sprintf("✅ Backup restored in %.2fs", result.RestoreTime))
|
e.log.Info(fmt.Sprintf("[OK] Backup restored in %.2fs", result.RestoreTime))
|
||||||
|
|
||||||
// Phase 4: Validate
|
// Phase 4: Validate
|
||||||
phase = e.startPhase("Validate Database")
|
phase = e.startPhase("Validate Database")
|
||||||
@@ -182,24 +182,24 @@ func (e *Engine) preflightChecks(ctx context.Context, config *DrillConfig) error
|
|||||||
if err := e.docker.CheckDockerAvailable(ctx); err != nil {
|
if err := e.docker.CheckDockerAvailable(ctx); err != nil {
|
||||||
return fmt.Errorf("docker not available: %w", err)
|
return fmt.Errorf("docker not available: %w", err)
|
||||||
}
|
}
|
||||||
e.log.Info("✓ Docker is available")
|
e.log.Info("[OK] Docker is available")
|
||||||
|
|
||||||
// Check backup file exists
|
// Check backup file exists
|
||||||
if _, err := os.Stat(config.BackupPath); err != nil {
|
if _, err := os.Stat(config.BackupPath); err != nil {
|
||||||
return fmt.Errorf("backup file not found: %s", config.BackupPath)
|
return fmt.Errorf("backup file not found: %s", config.BackupPath)
|
||||||
}
|
}
|
||||||
e.log.Info("✓ Backup file exists: " + filepath.Base(config.BackupPath))
|
e.log.Info("[OK] Backup file exists: " + filepath.Base(config.BackupPath))
|
||||||
|
|
||||||
// Pull Docker image
|
// Pull Docker image
|
||||||
image := config.ContainerImage
|
image := config.ContainerImage
|
||||||
if image == "" {
|
if image == "" {
|
||||||
image = GetDefaultImage(config.DatabaseType, "")
|
image = GetDefaultImage(config.DatabaseType, "")
|
||||||
}
|
}
|
||||||
e.log.Info("⬇️ Pulling image: " + image)
|
e.log.Info("[DOWN] Pulling image: " + image)
|
||||||
if err := e.docker.PullImage(ctx, image); err != nil {
|
if err := e.docker.PullImage(ctx, image); err != nil {
|
||||||
return fmt.Errorf("failed to pull image: %w", err)
|
return fmt.Errorf("failed to pull image: %w", err)
|
||||||
}
|
}
|
||||||
e.log.Info("✓ Image ready: " + image)
|
e.log.Info("[OK] Image ready: " + image)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -243,7 +243,7 @@ func (e *Engine) restoreBackup(ctx context.Context, config *DrillConfig, contain
|
|||||||
backupName := filepath.Base(config.BackupPath)
|
backupName := filepath.Base(config.BackupPath)
|
||||||
containerBackupPath := "/tmp/" + backupName
|
containerBackupPath := "/tmp/" + backupName
|
||||||
|
|
||||||
e.log.Info("📁 Copying backup to container...")
|
e.log.Info("[DIR] Copying backup to container...")
|
||||||
if err := e.docker.CopyToContainer(ctx, containerID, config.BackupPath, containerBackupPath); err != nil {
|
if err := e.docker.CopyToContainer(ctx, containerID, config.BackupPath, containerBackupPath); err != nil {
|
||||||
return fmt.Errorf("failed to copy backup: %w", err)
|
return fmt.Errorf("failed to copy backup: %w", err)
|
||||||
}
|
}
|
||||||
@@ -256,7 +256,7 @@ func (e *Engine) restoreBackup(ctx context.Context, config *DrillConfig, contain
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Restore based on database type and format
|
// Restore based on database type and format
|
||||||
e.log.Info("🔄 Restoring backup...")
|
e.log.Info("[EXEC] Restoring backup...")
|
||||||
return e.executeRestore(ctx, config, containerID, containerBackupPath, containerConfig)
|
return e.executeRestore(ctx, config, containerID, containerBackupPath, containerConfig)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -366,13 +366,13 @@ func (e *Engine) validateDatabase(ctx context.Context, config *DrillConfig, resu
|
|||||||
tables, err := validator.GetTableList(ctx)
|
tables, err := validator.GetTableList(ctx)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
result.TableCount = len(tables)
|
result.TableCount = len(tables)
|
||||||
e.log.Info(fmt.Sprintf("📊 Tables found: %d", result.TableCount))
|
e.log.Info(fmt.Sprintf("[STATS] Tables found: %d", result.TableCount))
|
||||||
}
|
}
|
||||||
|
|
||||||
totalRows, err := validator.GetTotalRowCount(ctx)
|
totalRows, err := validator.GetTotalRowCount(ctx)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
result.TotalRows = totalRows
|
result.TotalRows = totalRows
|
||||||
e.log.Info(fmt.Sprintf("📊 Total rows: %d", result.TotalRows))
|
e.log.Info(fmt.Sprintf("[STATS] Total rows: %d", result.TotalRows))
|
||||||
}
|
}
|
||||||
|
|
||||||
dbSize, err := validator.GetDatabaseSize(ctx, config.DatabaseName)
|
dbSize, err := validator.GetDatabaseSize(ctx, config.DatabaseName)
|
||||||
@@ -387,9 +387,9 @@ func (e *Engine) validateDatabase(ctx context.Context, config *DrillConfig, resu
|
|||||||
result.CheckResults = append(result.CheckResults, tr)
|
result.CheckResults = append(result.CheckResults, tr)
|
||||||
if !tr.Success {
|
if !tr.Success {
|
||||||
errorCount++
|
errorCount++
|
||||||
e.log.Warn("❌ " + tr.Message)
|
e.log.Warn("[FAIL] " + tr.Message)
|
||||||
} else {
|
} else {
|
||||||
e.log.Info("✓ " + tr.Message)
|
e.log.Info("[OK] " + tr.Message)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -404,9 +404,9 @@ func (e *Engine) validateDatabase(ctx context.Context, config *DrillConfig, resu
|
|||||||
totalQueryTime += qr.Duration
|
totalQueryTime += qr.Duration
|
||||||
if !qr.Success {
|
if !qr.Success {
|
||||||
errorCount++
|
errorCount++
|
||||||
e.log.Warn(fmt.Sprintf("❌ %s: %s", qr.Name, qr.Error))
|
e.log.Warn(fmt.Sprintf("[FAIL] %s: %s", qr.Name, qr.Error))
|
||||||
} else {
|
} else {
|
||||||
e.log.Info(fmt.Sprintf("✓ %s: %s (%.0fms)", qr.Name, qr.Result, qr.Duration))
|
e.log.Info(fmt.Sprintf("[OK] %s: %s (%.0fms)", qr.Name, qr.Result, qr.Duration))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(queryResults) > 0 {
|
if len(queryResults) > 0 {
|
||||||
@@ -421,9 +421,9 @@ func (e *Engine) validateDatabase(ctx context.Context, config *DrillConfig, resu
|
|||||||
result.CheckResults = append(result.CheckResults, cr)
|
result.CheckResults = append(result.CheckResults, cr)
|
||||||
if !cr.Success {
|
if !cr.Success {
|
||||||
errorCount++
|
errorCount++
|
||||||
e.log.Warn("❌ " + cr.Message)
|
e.log.Warn("[FAIL] " + cr.Message)
|
||||||
} else {
|
} else {
|
||||||
e.log.Info("✓ " + cr.Message)
|
e.log.Info("[OK] " + cr.Message)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -433,7 +433,7 @@ func (e *Engine) validateDatabase(ctx context.Context, config *DrillConfig, resu
|
|||||||
errorCount++
|
errorCount++
|
||||||
msg := fmt.Sprintf("Total rows (%d) below minimum (%d)", result.TotalRows, config.MinRowCount)
|
msg := fmt.Sprintf("Total rows (%d) below minimum (%d)", result.TotalRows, config.MinRowCount)
|
||||||
result.Warnings = append(result.Warnings, msg)
|
result.Warnings = append(result.Warnings, msg)
|
||||||
e.log.Warn("⚠️ " + msg)
|
e.log.Warn("[WARN] " + msg)
|
||||||
}
|
}
|
||||||
|
|
||||||
return errorCount
|
return errorCount
|
||||||
@@ -441,7 +441,7 @@ func (e *Engine) validateDatabase(ctx context.Context, config *DrillConfig, resu
|
|||||||
|
|
||||||
// startPhase starts a new drill phase
|
// startPhase starts a new drill phase
|
||||||
func (e *Engine) startPhase(name string) DrillPhase {
|
func (e *Engine) startPhase(name string) DrillPhase {
|
||||||
e.log.Info("▶️ " + name)
|
e.log.Info("[RUN] " + name)
|
||||||
return DrillPhase{
|
return DrillPhase{
|
||||||
Name: name,
|
Name: name,
|
||||||
Status: "running",
|
Status: "running",
|
||||||
@@ -463,7 +463,7 @@ func (e *Engine) failPhase(phase *DrillPhase, message string) {
|
|||||||
phase.Duration = phase.EndTime.Sub(phase.StartTime).Seconds()
|
phase.Duration = phase.EndTime.Sub(phase.StartTime).Seconds()
|
||||||
phase.Status = "failed"
|
phase.Status = "failed"
|
||||||
phase.Message = message
|
phase.Message = message
|
||||||
e.log.Error("❌ Phase failed: " + message)
|
e.log.Error("[FAIL] Phase failed: " + message)
|
||||||
}
|
}
|
||||||
|
|
||||||
// finalize completes the drill result
|
// finalize completes the drill result
|
||||||
@@ -472,9 +472,9 @@ func (e *Engine) finalize(result *DrillResult) {
|
|||||||
result.Duration = result.EndTime.Sub(result.StartTime).Seconds()
|
result.Duration = result.EndTime.Sub(result.StartTime).Seconds()
|
||||||
|
|
||||||
e.log.Info("")
|
e.log.Info("")
|
||||||
e.log.Info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
e.log.Info("=====================================================")
|
||||||
e.log.Info(" " + result.Summary())
|
e.log.Info(" " + result.Summary())
|
||||||
e.log.Info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
e.log.Info("=====================================================")
|
||||||
|
|
||||||
if result.Success {
|
if result.Success {
|
||||||
e.log.Info(fmt.Sprintf(" RTO: %.2fs (target: %.0fs) %s",
|
e.log.Info(fmt.Sprintf(" RTO: %.2fs (target: %.0fs) %s",
|
||||||
@@ -484,9 +484,9 @@ func (e *Engine) finalize(result *DrillResult) {
|
|||||||
|
|
||||||
func boolIcon(b bool) string {
|
func boolIcon(b bool) string {
|
||||||
if b {
|
if b {
|
||||||
return "✅"
|
return "[OK]"
|
||||||
}
|
}
|
||||||
return "❌"
|
return "[FAIL]"
|
||||||
}
|
}
|
||||||
|
|
||||||
// Cleanup removes drill resources
|
// Cleanup removes drill resources
|
||||||
@@ -498,7 +498,7 @@ func (e *Engine) Cleanup(ctx context.Context, drillID string) error {
|
|||||||
|
|
||||||
for _, c := range containers {
|
for _, c := range containers {
|
||||||
if strings.Contains(c.Name, drillID) || (drillID == "" && strings.HasPrefix(c.Name, "drill_")) {
|
if strings.Contains(c.Name, drillID) || (drillID == "" && strings.HasPrefix(c.Name, "drill_")) {
|
||||||
e.log.Info("🗑️ Removing container: " + c.Name)
|
e.log.Info("[DEL] Removing container: " + c.Name)
|
||||||
if err := e.docker.RemoveContainer(ctx, c.ID); err != nil {
|
if err := e.docker.RemoveContainer(ctx, c.ID); err != nil {
|
||||||
e.log.Warn("Failed to remove container", "id", c.ID, "error", err)
|
e.log.Warn("Failed to remove container", "id", c.ID, "error", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import (
|
|||||||
|
|
||||||
func TestEncryptDecrypt(t *testing.T) {
|
func TestEncryptDecrypt(t *testing.T) {
|
||||||
// Test data
|
// Test data
|
||||||
original := []byte("This is a secret database backup that needs encryption! 🔒")
|
original := []byte("This is a secret database backup that needs encryption! [LOCK]")
|
||||||
|
|
||||||
// Test with passphrase
|
// Test with passphrase
|
||||||
t.Run("Passphrase", func(t *testing.T) {
|
t.Run("Passphrase", func(t *testing.T) {
|
||||||
@@ -57,7 +57,7 @@ func TestEncryptDecrypt(t *testing.T) {
|
|||||||
string(original), string(decrypted))
|
string(original), string(decrypted))
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Log("✅ Encryption/decryption successful")
|
t.Log("[OK] Encryption/decryption successful")
|
||||||
})
|
})
|
||||||
|
|
||||||
// Test with direct key
|
// Test with direct key
|
||||||
@@ -102,7 +102,7 @@ func TestEncryptDecrypt(t *testing.T) {
|
|||||||
t.Errorf("Decrypted data doesn't match original")
|
t.Errorf("Decrypted data doesn't match original")
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Log("✅ Direct key encryption/decryption successful")
|
t.Log("[OK] Direct key encryption/decryption successful")
|
||||||
})
|
})
|
||||||
|
|
||||||
// Test wrong password
|
// Test wrong password
|
||||||
@@ -133,7 +133,7 @@ func TestEncryptDecrypt(t *testing.T) {
|
|||||||
t.Error("Expected decryption to fail with wrong password, but it succeeded")
|
t.Error("Expected decryption to fail with wrong password, but it succeeded")
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Logf("✅ Wrong password correctly rejected: %v", err)
|
t.Logf("[OK] Wrong password correctly rejected: %v", err)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -183,7 +183,7 @@ func TestLargeData(t *testing.T) {
|
|||||||
t.Errorf("Large data decryption failed")
|
t.Errorf("Large data decryption failed")
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Log("✅ Large data encryption/decryption successful")
|
t.Log("[OK] Large data encryption/decryption successful")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestKeyGeneration(t *testing.T) {
|
func TestKeyGeneration(t *testing.T) {
|
||||||
@@ -207,7 +207,7 @@ func TestKeyGeneration(t *testing.T) {
|
|||||||
t.Error("Generated keys are identical - randomness broken!")
|
t.Error("Generated keys are identical - randomness broken!")
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Log("✅ Key generation successful")
|
t.Log("[OK] Key generation successful")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestKeyDerivation(t *testing.T) {
|
func TestKeyDerivation(t *testing.T) {
|
||||||
@@ -230,5 +230,5 @@ func TestKeyDerivation(t *testing.T) {
|
|||||||
t.Error("Different salts produced same key")
|
t.Error("Different salts produced same key")
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Log("✅ Key derivation successful")
|
t.Log("[OK] Key derivation successful")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -339,7 +339,7 @@ func (e *CloneEngine) Backup(ctx context.Context, opts *BackupOptions) (*BackupR
|
|||||||
|
|
||||||
// Save metadata
|
// Save metadata
|
||||||
meta := &metadata.BackupMetadata{
|
meta := &metadata.BackupMetadata{
|
||||||
Version: "3.1.0",
|
Version: "3.42.1",
|
||||||
Timestamp: startTime,
|
Timestamp: startTime,
|
||||||
Database: opts.Database,
|
Database: opts.Database,
|
||||||
DatabaseType: "mysql",
|
DatabaseType: "mysql",
|
||||||
|
|||||||
@@ -234,10 +234,26 @@ func (e *MySQLDumpEngine) Backup(ctx context.Context, opts *BackupOptions) (*Bac
|
|||||||
gzWriter.Close()
|
gzWriter.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wait for command
|
// Wait for command with proper context handling
|
||||||
if err := cmd.Wait(); err != nil {
|
cmdDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
cmdDone <- cmd.Wait()
|
||||||
|
}()
|
||||||
|
|
||||||
|
var cmdErr error
|
||||||
|
select {
|
||||||
|
case cmdErr = <-cmdDone:
|
||||||
|
// Command completed
|
||||||
|
case <-ctx.Done():
|
||||||
|
e.log.Warn("MySQL backup cancelled - killing process")
|
||||||
|
cmd.Process.Kill()
|
||||||
|
<-cmdDone
|
||||||
|
cmdErr = ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
if cmdErr != nil {
|
||||||
stderr := stderrBuf.String()
|
stderr := stderrBuf.String()
|
||||||
return nil, fmt.Errorf("mysqldump failed: %w\n%s", err, stderr)
|
return nil, fmt.Errorf("mysqldump failed: %w\n%s", cmdErr, stderr)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get file info
|
// Get file info
|
||||||
@@ -254,7 +270,7 @@ func (e *MySQLDumpEngine) Backup(ctx context.Context, opts *BackupOptions) (*Bac
|
|||||||
|
|
||||||
// Save metadata
|
// Save metadata
|
||||||
meta := &metadata.BackupMetadata{
|
meta := &metadata.BackupMetadata{
|
||||||
Version: "3.1.0",
|
Version: "3.42.1",
|
||||||
Timestamp: startTime,
|
Timestamp: startTime,
|
||||||
Database: opts.Database,
|
Database: opts.Database,
|
||||||
DatabaseType: "mysql",
|
DatabaseType: "mysql",
|
||||||
@@ -442,8 +458,25 @@ func (e *MySQLDumpEngine) BackupToWriter(ctx context.Context, w io.Writer, opts
|
|||||||
gzWriter.Close()
|
gzWriter.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := cmd.Wait(); err != nil {
|
// Wait for command with proper context handling
|
||||||
return nil, fmt.Errorf("mysqldump failed: %w\n%s", err, stderrBuf.String())
|
cmdDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
cmdDone <- cmd.Wait()
|
||||||
|
}()
|
||||||
|
|
||||||
|
var cmdErr error
|
||||||
|
select {
|
||||||
|
case cmdErr = <-cmdDone:
|
||||||
|
// Command completed
|
||||||
|
case <-ctx.Done():
|
||||||
|
e.log.Warn("MySQL streaming backup cancelled - killing process")
|
||||||
|
cmd.Process.Kill()
|
||||||
|
<-cmdDone
|
||||||
|
cmdErr = ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
if cmdErr != nil {
|
||||||
|
return nil, fmt.Errorf("mysqldump failed: %w\n%s", cmdErr, stderrBuf.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
return &BackupResult{
|
return &BackupResult{
|
||||||
|
|||||||
@@ -63,7 +63,7 @@ func (b *BtrfsBackend) Detect(dataDir string) (bool, error) {
|
|||||||
// CreateSnapshot creates a Btrfs snapshot
|
// CreateSnapshot creates a Btrfs snapshot
|
||||||
func (b *BtrfsBackend) CreateSnapshot(ctx context.Context, opts SnapshotOptions) (*Snapshot, error) {
|
func (b *BtrfsBackend) CreateSnapshot(ctx context.Context, opts SnapshotOptions) (*Snapshot, error) {
|
||||||
if b.config == nil || b.config.Subvolume == "" {
|
if b.config == nil || b.config.Subvolume == "" {
|
||||||
return nil, fmt.Errorf("Btrfs subvolume not configured")
|
return nil, fmt.Errorf("btrfs subvolume not configured")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Generate snapshot name
|
// Generate snapshot name
|
||||||
|
|||||||
@@ -188,6 +188,8 @@ func (e *SnapshotEngine) Backup(ctx context.Context, opts *BackupOptions) (*Back
|
|||||||
// Step 4: Mount snapshot
|
// Step 4: Mount snapshot
|
||||||
mountPoint := e.config.MountPoint
|
mountPoint := e.config.MountPoint
|
||||||
if mountPoint == "" {
|
if mountPoint == "" {
|
||||||
|
// Note: snapshot engine uses snapshot.Config which doesnt have GetEffectiveWorkDir()
|
||||||
|
// TODO: Refactor to use main config.Config for WorkDir support
|
||||||
mountPoint = filepath.Join(os.TempDir(), fmt.Sprintf("dbbackup_snap_%s", timestamp))
|
mountPoint = filepath.Join(os.TempDir(), fmt.Sprintf("dbbackup_snap_%s", timestamp))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -223,7 +225,7 @@ func (e *SnapshotEngine) Backup(ctx context.Context, opts *BackupOptions) (*Back
|
|||||||
|
|
||||||
// Save metadata
|
// Save metadata
|
||||||
meta := &metadata.BackupMetadata{
|
meta := &metadata.BackupMetadata{
|
||||||
Version: "3.1.0",
|
Version: "3.42.1",
|
||||||
Timestamp: startTime,
|
Timestamp: startTime,
|
||||||
Database: opts.Database,
|
Database: opts.Database,
|
||||||
DatabaseType: "mysql",
|
DatabaseType: "mysql",
|
||||||
|
|||||||
11
internal/installer/embed.go
Normal file
11
internal/installer/embed.go
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
// Package installer provides systemd service installation for dbbackup
|
||||||
|
package installer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"embed"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Templates contains embedded systemd unit files
|
||||||
|
//
|
||||||
|
//go:embed templates/*.service templates/*.timer
|
||||||
|
var Templates embed.FS
|
||||||
680
internal/installer/installer.go
Normal file
680
internal/installer/installer.go
Normal file
@@ -0,0 +1,680 @@
|
|||||||
|
// Package installer provides systemd service installation for dbbackup
|
||||||
|
package installer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"os/user"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"text/template"
|
||||||
|
|
||||||
|
"dbbackup/internal/logger"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Installer handles systemd service installation
|
||||||
|
type Installer struct {
|
||||||
|
log logger.Logger
|
||||||
|
unitDir string // /etc/systemd/system or custom
|
||||||
|
dryRun bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// InstallOptions configures the installation
|
||||||
|
type InstallOptions struct {
|
||||||
|
// Instance name (e.g., "production", "staging")
|
||||||
|
Instance string
|
||||||
|
|
||||||
|
// Binary path (auto-detected if empty)
|
||||||
|
BinaryPath string
|
||||||
|
|
||||||
|
// Backup configuration
|
||||||
|
BackupType string // "single" or "cluster"
|
||||||
|
Schedule string // OnCalendar format, e.g., "daily", "*-*-* 02:00:00"
|
||||||
|
|
||||||
|
// Service user/group
|
||||||
|
User string
|
||||||
|
Group string
|
||||||
|
|
||||||
|
// Paths
|
||||||
|
BackupDir string
|
||||||
|
ConfigPath string
|
||||||
|
|
||||||
|
// Timeout in seconds (default: 3600)
|
||||||
|
TimeoutSeconds int
|
||||||
|
|
||||||
|
// Metrics
|
||||||
|
WithMetrics bool
|
||||||
|
MetricsPort int
|
||||||
|
}
|
||||||
|
|
||||||
|
// ServiceStatus contains information about installed services
|
||||||
|
type ServiceStatus struct {
|
||||||
|
Installed bool
|
||||||
|
Enabled bool
|
||||||
|
Active bool
|
||||||
|
TimerEnabled bool
|
||||||
|
TimerActive bool
|
||||||
|
LastRun string
|
||||||
|
NextRun string
|
||||||
|
ServicePath string
|
||||||
|
TimerPath string
|
||||||
|
ExporterPath string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewInstaller creates a new Installer
|
||||||
|
func NewInstaller(log logger.Logger, dryRun bool) *Installer {
|
||||||
|
return &Installer{
|
||||||
|
log: log,
|
||||||
|
unitDir: "/etc/systemd/system",
|
||||||
|
dryRun: dryRun,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetUnitDir allows overriding the systemd unit directory (for testing)
|
||||||
|
func (i *Installer) SetUnitDir(dir string) {
|
||||||
|
i.unitDir = dir
|
||||||
|
}
|
||||||
|
|
||||||
|
// Install installs the systemd service and timer
|
||||||
|
func (i *Installer) Install(ctx context.Context, opts InstallOptions) error {
|
||||||
|
// Validate platform
|
||||||
|
if runtime.GOOS != "linux" {
|
||||||
|
return fmt.Errorf("systemd installation only supported on Linux (current: %s)", runtime.GOOS)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate prerequisites
|
||||||
|
if err := i.validatePrerequisites(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set defaults
|
||||||
|
if err := i.setDefaults(&opts); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create user if needed
|
||||||
|
if err := i.ensureUser(opts.User, opts.Group); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create directories
|
||||||
|
if err := i.createDirectories(opts); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Copy binary to /usr/local/bin (required for ProtectHome=yes)
|
||||||
|
if err := i.copyBinary(&opts); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write service and timer files
|
||||||
|
if err := i.writeUnitFiles(opts); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reload systemd
|
||||||
|
if err := i.systemctl(ctx, "daemon-reload"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enable timer
|
||||||
|
timerName := i.getTimerName(opts)
|
||||||
|
if err := i.systemctl(ctx, "enable", timerName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Install metrics exporter if requested
|
||||||
|
if opts.WithMetrics {
|
||||||
|
if err := i.installExporter(ctx, opts); err != nil {
|
||||||
|
i.log.Warn("Failed to install metrics exporter", "error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
i.log.Info("Installation complete",
|
||||||
|
"instance", opts.Instance,
|
||||||
|
"timer", timerName,
|
||||||
|
"schedule", opts.Schedule)
|
||||||
|
|
||||||
|
i.printNextSteps(opts)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Uninstall removes the systemd service and timer
|
||||||
|
func (i *Installer) Uninstall(ctx context.Context, instance string, purge bool) error {
|
||||||
|
if runtime.GOOS != "linux" {
|
||||||
|
return fmt.Errorf("systemd uninstallation only supported on Linux")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := i.validatePrerequisites(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Determine service names
|
||||||
|
var serviceName, timerName string
|
||||||
|
if instance == "cluster" || instance == "" {
|
||||||
|
serviceName = "dbbackup-cluster.service"
|
||||||
|
timerName = "dbbackup-cluster.timer"
|
||||||
|
} else {
|
||||||
|
serviceName = fmt.Sprintf("dbbackup@%s.service", instance)
|
||||||
|
timerName = fmt.Sprintf("dbbackup@%s.timer", instance)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop and disable timer
|
||||||
|
_ = i.systemctl(ctx, "stop", timerName)
|
||||||
|
_ = i.systemctl(ctx, "disable", timerName)
|
||||||
|
|
||||||
|
// Stop and disable service
|
||||||
|
_ = i.systemctl(ctx, "stop", serviceName)
|
||||||
|
_ = i.systemctl(ctx, "disable", serviceName)
|
||||||
|
|
||||||
|
// Remove unit files
|
||||||
|
servicePath := filepath.Join(i.unitDir, serviceName)
|
||||||
|
timerPath := filepath.Join(i.unitDir, timerName)
|
||||||
|
|
||||||
|
if !i.dryRun {
|
||||||
|
os.Remove(servicePath)
|
||||||
|
os.Remove(timerPath)
|
||||||
|
} else {
|
||||||
|
i.log.Info("Would remove", "service", servicePath)
|
||||||
|
i.log.Info("Would remove", "timer", timerPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also try to remove template units if they exist
|
||||||
|
if instance != "cluster" && instance != "" {
|
||||||
|
templateService := filepath.Join(i.unitDir, "dbbackup@.service")
|
||||||
|
templateTimer := filepath.Join(i.unitDir, "dbbackup@.timer")
|
||||||
|
|
||||||
|
// Only remove templates if no other instances are using them
|
||||||
|
if i.canRemoveTemplates() {
|
||||||
|
if !i.dryRun {
|
||||||
|
os.Remove(templateService)
|
||||||
|
os.Remove(templateTimer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove exporter
|
||||||
|
exporterPath := filepath.Join(i.unitDir, "dbbackup-exporter.service")
|
||||||
|
_ = i.systemctl(ctx, "stop", "dbbackup-exporter.service")
|
||||||
|
_ = i.systemctl(ctx, "disable", "dbbackup-exporter.service")
|
||||||
|
if !i.dryRun {
|
||||||
|
os.Remove(exporterPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reload systemd
|
||||||
|
_ = i.systemctl(ctx, "daemon-reload")
|
||||||
|
|
||||||
|
// Purge config files if requested
|
||||||
|
if purge {
|
||||||
|
configDirs := []string{
|
||||||
|
"/etc/dbbackup",
|
||||||
|
"/var/lib/dbbackup",
|
||||||
|
}
|
||||||
|
for _, dir := range configDirs {
|
||||||
|
if !i.dryRun {
|
||||||
|
if err := os.RemoveAll(dir); err != nil {
|
||||||
|
i.log.Warn("Failed to remove directory", "path", dir, "error", err)
|
||||||
|
} else {
|
||||||
|
i.log.Info("Removed directory", "path", dir)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
i.log.Info("Would remove directory", "path", dir)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
i.log.Info("Uninstallation complete", "instance", instance, "purge", purge)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Status returns the current installation status
|
||||||
|
func (i *Installer) Status(ctx context.Context, instance string) (*ServiceStatus, error) {
|
||||||
|
if runtime.GOOS != "linux" {
|
||||||
|
return nil, fmt.Errorf("systemd status only supported on Linux")
|
||||||
|
}
|
||||||
|
|
||||||
|
status := &ServiceStatus{}
|
||||||
|
|
||||||
|
// Determine service names
|
||||||
|
var serviceName, timerName string
|
||||||
|
if instance == "cluster" || instance == "" {
|
||||||
|
serviceName = "dbbackup-cluster.service"
|
||||||
|
timerName = "dbbackup-cluster.timer"
|
||||||
|
} else {
|
||||||
|
serviceName = fmt.Sprintf("dbbackup@%s.service", instance)
|
||||||
|
timerName = fmt.Sprintf("dbbackup@%s.timer", instance)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check service file exists
|
||||||
|
status.ServicePath = filepath.Join(i.unitDir, serviceName)
|
||||||
|
if _, err := os.Stat(status.ServicePath); err == nil {
|
||||||
|
status.Installed = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check timer file exists
|
||||||
|
status.TimerPath = filepath.Join(i.unitDir, timerName)
|
||||||
|
|
||||||
|
// Check exporter
|
||||||
|
status.ExporterPath = filepath.Join(i.unitDir, "dbbackup-exporter.service")
|
||||||
|
|
||||||
|
// Check enabled/active status
|
||||||
|
if status.Installed {
|
||||||
|
status.Enabled = i.isEnabled(ctx, serviceName)
|
||||||
|
status.Active = i.isActive(ctx, serviceName)
|
||||||
|
status.TimerEnabled = i.isEnabled(ctx, timerName)
|
||||||
|
status.TimerActive = i.isActive(ctx, timerName)
|
||||||
|
|
||||||
|
// Get timer info
|
||||||
|
status.NextRun = i.getTimerNext(ctx, timerName)
|
||||||
|
status.LastRun = i.getTimerLast(ctx, timerName)
|
||||||
|
}
|
||||||
|
|
||||||
|
return status, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// validatePrerequisites checks system requirements
|
||||||
|
func (i *Installer) validatePrerequisites() error {
|
||||||
|
// Check root (skip in dry-run mode)
|
||||||
|
if os.Getuid() != 0 && !i.dryRun {
|
||||||
|
return fmt.Errorf("installation requires root privileges (use sudo)")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check systemd
|
||||||
|
if _, err := exec.LookPath("systemctl"); err != nil {
|
||||||
|
return fmt.Errorf("systemctl not found - is this a systemd-based system?")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for container environment
|
||||||
|
if _, err := os.Stat("/.dockerenv"); err == nil {
|
||||||
|
i.log.Warn("Running inside Docker container - systemd may not work correctly")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// setDefaults fills in default values
|
||||||
|
func (i *Installer) setDefaults(opts *InstallOptions) error {
|
||||||
|
// Auto-detect binary path
|
||||||
|
if opts.BinaryPath == "" {
|
||||||
|
binPath, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to detect binary path: %w", err)
|
||||||
|
}
|
||||||
|
binPath, err = filepath.EvalSymlinks(binPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to resolve binary path: %w", err)
|
||||||
|
}
|
||||||
|
opts.BinaryPath = binPath
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default instance
|
||||||
|
if opts.Instance == "" {
|
||||||
|
opts.Instance = "default"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default backup type
|
||||||
|
if opts.BackupType == "" {
|
||||||
|
opts.BackupType = "single"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default schedule (daily at 2am)
|
||||||
|
if opts.Schedule == "" {
|
||||||
|
opts.Schedule = "*-*-* 02:00:00"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default user/group
|
||||||
|
if opts.User == "" {
|
||||||
|
opts.User = "dbbackup"
|
||||||
|
}
|
||||||
|
if opts.Group == "" {
|
||||||
|
opts.Group = "dbbackup"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default paths
|
||||||
|
if opts.BackupDir == "" {
|
||||||
|
opts.BackupDir = "/var/lib/dbbackup/backups"
|
||||||
|
}
|
||||||
|
if opts.ConfigPath == "" {
|
||||||
|
opts.ConfigPath = "/etc/dbbackup/dbbackup.conf"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default timeout (1 hour)
|
||||||
|
if opts.TimeoutSeconds == 0 {
|
||||||
|
opts.TimeoutSeconds = 3600
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default metrics port
|
||||||
|
if opts.MetricsPort == 0 {
|
||||||
|
opts.MetricsPort = 9399
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensureUser creates the service user if it doesn't exist
|
||||||
|
func (i *Installer) ensureUser(username, groupname string) error {
|
||||||
|
// Check if user exists
|
||||||
|
if _, err := user.Lookup(username); err == nil {
|
||||||
|
i.log.Debug("User already exists", "user", username)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if i.dryRun {
|
||||||
|
i.log.Info("Would create user", "user", username, "group", groupname)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create group first
|
||||||
|
groupCmd := exec.Command("groupadd", "--system", groupname)
|
||||||
|
if output, err := groupCmd.CombinedOutput(); err != nil {
|
||||||
|
// Ignore if group already exists
|
||||||
|
if !strings.Contains(string(output), "already exists") {
|
||||||
|
i.log.Debug("Group creation output", "output", string(output))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create user
|
||||||
|
userCmd := exec.Command("useradd",
|
||||||
|
"--system",
|
||||||
|
"--shell", "/usr/sbin/nologin",
|
||||||
|
"--home-dir", "/var/lib/dbbackup",
|
||||||
|
"--gid", groupname,
|
||||||
|
username)
|
||||||
|
|
||||||
|
if output, err := userCmd.CombinedOutput(); err != nil {
|
||||||
|
if !strings.Contains(string(output), "already exists") {
|
||||||
|
return fmt.Errorf("failed to create user %s: %w (%s)", username, err, output)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
i.log.Info("Created system user", "user", username, "group", groupname)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// createDirectories creates required directories
|
||||||
|
func (i *Installer) createDirectories(opts InstallOptions) error {
|
||||||
|
dirs := []struct {
|
||||||
|
path string
|
||||||
|
mode os.FileMode
|
||||||
|
}{
|
||||||
|
{"/etc/dbbackup", 0755},
|
||||||
|
{"/etc/dbbackup/env.d", 0700},
|
||||||
|
{"/var/lib/dbbackup", 0750},
|
||||||
|
{"/var/lib/dbbackup/backups", 0750},
|
||||||
|
{"/var/lib/dbbackup/metrics", 0755},
|
||||||
|
{"/var/log/dbbackup", 0750},
|
||||||
|
{opts.BackupDir, 0750},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, d := range dirs {
|
||||||
|
if i.dryRun {
|
||||||
|
i.log.Info("Would create directory", "path", d.path, "mode", d.mode)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.MkdirAll(d.path, d.mode); err != nil {
|
||||||
|
return fmt.Errorf("failed to create directory %s: %w", d.path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set ownership
|
||||||
|
u, err := user.Lookup(opts.User)
|
||||||
|
if err == nil {
|
||||||
|
var uid, gid int
|
||||||
|
fmt.Sscanf(u.Uid, "%d", &uid)
|
||||||
|
fmt.Sscanf(u.Gid, "%d", &gid)
|
||||||
|
os.Chown(d.path, uid, gid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// copyBinary copies the binary to /usr/local/bin for systemd access
|
||||||
|
// This is required because ProtectHome=yes blocks access to home directories
|
||||||
|
func (i *Installer) copyBinary(opts *InstallOptions) error {
|
||||||
|
const installPath = "/usr/local/bin/dbbackup"
|
||||||
|
|
||||||
|
// Check if binary is already in a system path
|
||||||
|
if opts.BinaryPath == installPath {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if i.dryRun {
|
||||||
|
i.log.Info("Would copy binary", "from", opts.BinaryPath, "to", installPath)
|
||||||
|
opts.BinaryPath = installPath
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read source binary
|
||||||
|
src, err := os.Open(opts.BinaryPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open source binary: %w", err)
|
||||||
|
}
|
||||||
|
defer src.Close()
|
||||||
|
|
||||||
|
// Create destination
|
||||||
|
dst, err := os.OpenFile(installPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create %s: %w", installPath, err)
|
||||||
|
}
|
||||||
|
defer dst.Close()
|
||||||
|
|
||||||
|
// Copy
|
||||||
|
if _, err := io.Copy(dst, src); err != nil {
|
||||||
|
return fmt.Errorf("failed to copy binary: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
i.log.Info("Copied binary", "from", opts.BinaryPath, "to", installPath)
|
||||||
|
opts.BinaryPath = installPath
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeUnitFiles renders and writes the systemd unit files
|
||||||
|
func (i *Installer) writeUnitFiles(opts InstallOptions) error {
|
||||||
|
// Prepare template data
|
||||||
|
data := map[string]interface{}{
|
||||||
|
"User": opts.User,
|
||||||
|
"Group": opts.Group,
|
||||||
|
"BinaryPath": opts.BinaryPath,
|
||||||
|
"BackupType": opts.BackupType,
|
||||||
|
"BackupDir": opts.BackupDir,
|
||||||
|
"ConfigPath": opts.ConfigPath,
|
||||||
|
"TimeoutSeconds": opts.TimeoutSeconds,
|
||||||
|
"Schedule": opts.Schedule,
|
||||||
|
"MetricsPort": opts.MetricsPort,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Determine which templates to use
|
||||||
|
var serviceTemplate, timerTemplate string
|
||||||
|
var serviceName, timerName string
|
||||||
|
|
||||||
|
if opts.BackupType == "cluster" {
|
||||||
|
serviceTemplate = "templates/dbbackup-cluster.service"
|
||||||
|
timerTemplate = "templates/dbbackup-cluster.timer"
|
||||||
|
serviceName = "dbbackup-cluster.service"
|
||||||
|
timerName = "dbbackup-cluster.timer"
|
||||||
|
} else {
|
||||||
|
serviceTemplate = "templates/dbbackup@.service"
|
||||||
|
timerTemplate = "templates/dbbackup@.timer"
|
||||||
|
serviceName = "dbbackup@.service"
|
||||||
|
timerName = "dbbackup@.timer"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write service file
|
||||||
|
if err := i.writeTemplateFile(serviceTemplate, serviceName, data); err != nil {
|
||||||
|
return fmt.Errorf("failed to write service file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write timer file
|
||||||
|
if err := i.writeTemplateFile(timerTemplate, timerName, data); err != nil {
|
||||||
|
return fmt.Errorf("failed to write timer file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeTemplateFile reads an embedded template and writes it to the unit directory
|
||||||
|
func (i *Installer) writeTemplateFile(templatePath, outputName string, data map[string]interface{}) error {
|
||||||
|
// Read template
|
||||||
|
content, err := Templates.ReadFile(templatePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to read template %s: %w", templatePath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse template
|
||||||
|
tmpl, err := template.New(outputName).Parse(string(content))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to parse template %s: %w", templatePath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Render template
|
||||||
|
var buf strings.Builder
|
||||||
|
if err := tmpl.Execute(&buf, data); err != nil {
|
||||||
|
return fmt.Errorf("failed to render template %s: %w", templatePath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write file
|
||||||
|
outputPath := filepath.Join(i.unitDir, outputName)
|
||||||
|
if i.dryRun {
|
||||||
|
i.log.Info("Would write unit file", "path", outputPath)
|
||||||
|
i.log.Debug("Unit file content", "content", buf.String())
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.WriteFile(outputPath, []byte(buf.String()), 0644); err != nil {
|
||||||
|
return fmt.Errorf("failed to write %s: %w", outputPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
i.log.Info("Created unit file", "path", outputPath)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// installExporter installs the metrics exporter service
|
||||||
|
func (i *Installer) installExporter(ctx context.Context, opts InstallOptions) error {
|
||||||
|
data := map[string]interface{}{
|
||||||
|
"User": opts.User,
|
||||||
|
"Group": opts.Group,
|
||||||
|
"BinaryPath": opts.BinaryPath,
|
||||||
|
"ConfigPath": opts.ConfigPath,
|
||||||
|
"MetricsPort": opts.MetricsPort,
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := i.writeTemplateFile("templates/dbbackup-exporter.service", "dbbackup-exporter.service", data); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := i.systemctl(ctx, "daemon-reload"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := i.systemctl(ctx, "enable", "dbbackup-exporter.service"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := i.systemctl(ctx, "start", "dbbackup-exporter.service"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
i.log.Info("Installed metrics exporter", "port", opts.MetricsPort)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// getTimerName returns the timer unit name for the given options
|
||||||
|
func (i *Installer) getTimerName(opts InstallOptions) string {
|
||||||
|
if opts.BackupType == "cluster" {
|
||||||
|
return "dbbackup-cluster.timer"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("dbbackup@%s.timer", opts.Instance)
|
||||||
|
}
|
||||||
|
|
||||||
|
// systemctl runs a systemctl command
|
||||||
|
func (i *Installer) systemctl(ctx context.Context, args ...string) error {
|
||||||
|
if i.dryRun {
|
||||||
|
i.log.Info("Would run: systemctl", "args", args)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd := exec.CommandContext(ctx, "systemctl", args...)
|
||||||
|
output, err := cmd.CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("systemctl %v failed: %w\n%s", args, err, string(output))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// isEnabled checks if a unit is enabled
|
||||||
|
func (i *Installer) isEnabled(ctx context.Context, unit string) bool {
|
||||||
|
cmd := exec.CommandContext(ctx, "systemctl", "is-enabled", unit)
|
||||||
|
return cmd.Run() == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// isActive checks if a unit is active
|
||||||
|
func (i *Installer) isActive(ctx context.Context, unit string) bool {
|
||||||
|
cmd := exec.CommandContext(ctx, "systemctl", "is-active", unit)
|
||||||
|
return cmd.Run() == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// getTimerNext gets the next run time for a timer
|
||||||
|
func (i *Installer) getTimerNext(ctx context.Context, timer string) string {
|
||||||
|
cmd := exec.CommandContext(ctx, "systemctl", "show", timer, "--property=NextElapseUSecRealtime", "--value")
|
||||||
|
output, err := cmd.Output()
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(string(output))
|
||||||
|
}
|
||||||
|
|
||||||
|
// getTimerLast gets the last run time for a timer
|
||||||
|
func (i *Installer) getTimerLast(ctx context.Context, timer string) string {
|
||||||
|
cmd := exec.CommandContext(ctx, "systemctl", "show", timer, "--property=LastTriggerUSec", "--value")
|
||||||
|
output, err := cmd.Output()
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(string(output))
|
||||||
|
}
|
||||||
|
|
||||||
|
// canRemoveTemplates checks if template units can be safely removed
|
||||||
|
func (i *Installer) canRemoveTemplates() bool {
|
||||||
|
// Check if any dbbackup@*.service instances exist
|
||||||
|
pattern := filepath.Join(i.unitDir, "dbbackup@*.service")
|
||||||
|
matches, _ := filepath.Glob(pattern)
|
||||||
|
|
||||||
|
// Also check for running instances
|
||||||
|
cmd := exec.Command("systemctl", "list-units", "--type=service", "--all", "dbbackup@*")
|
||||||
|
output, _ := cmd.Output()
|
||||||
|
|
||||||
|
return len(matches) == 0 && !strings.Contains(string(output), "dbbackup@")
|
||||||
|
}
|
||||||
|
|
||||||
|
// printNextSteps prints helpful next steps after installation
|
||||||
|
func (i *Installer) printNextSteps(opts InstallOptions) {
|
||||||
|
timerName := i.getTimerName(opts)
|
||||||
|
serviceName := strings.Replace(timerName, ".timer", ".service", 1)
|
||||||
|
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println("[OK] Installation successful!")
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println("[NEXT] Next steps:")
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Printf(" 1. Edit configuration: sudo nano %s\n", opts.ConfigPath)
|
||||||
|
fmt.Printf(" 2. Set credentials: sudo nano /etc/dbbackup/env.d/%s.conf\n", opts.Instance)
|
||||||
|
fmt.Printf(" 3. Start the timer: sudo systemctl start %s\n", timerName)
|
||||||
|
fmt.Printf(" 4. Verify timer status: sudo systemctl status %s\n", timerName)
|
||||||
|
fmt.Printf(" 5. Run backup manually: sudo systemctl start %s\n", serviceName)
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println("[LOGS] View backup logs:")
|
||||||
|
fmt.Printf(" journalctl -u %s -f\n", serviceName)
|
||||||
|
fmt.Println()
|
||||||
|
|
||||||
|
if opts.WithMetrics {
|
||||||
|
fmt.Println("[METRICS] Prometheus metrics:")
|
||||||
|
fmt.Printf(" curl http://localhost:%d/metrics\n", opts.MetricsPort)
|
||||||
|
fmt.Println()
|
||||||
|
}
|
||||||
|
}
|
||||||
47
internal/installer/templates/dbbackup-cluster.service
Normal file
47
internal/installer/templates/dbbackup-cluster.service
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Database Cluster Backup
|
||||||
|
Documentation=https://github.com/PlusOne/dbbackup
|
||||||
|
After=network-online.target postgresql.service mysql.service mariadb.service
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
User={{.User}}
|
||||||
|
Group={{.Group}}
|
||||||
|
|
||||||
|
# Security hardening
|
||||||
|
NoNewPrivileges=yes
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=read-only
|
||||||
|
PrivateTmp=yes
|
||||||
|
ProtectKernelTunables=yes
|
||||||
|
ProtectKernelModules=yes
|
||||||
|
ProtectControlGroups=yes
|
||||||
|
RestrictSUIDSGID=yes
|
||||||
|
RestrictRealtime=yes
|
||||||
|
LockPersonality=yes
|
||||||
|
RemoveIPC=yes
|
||||||
|
CapabilityBoundingSet=
|
||||||
|
AmbientCapabilities=
|
||||||
|
|
||||||
|
# Directories
|
||||||
|
ReadWritePaths={{.BackupDir}} /var/lib/dbbackup /var/log/dbbackup
|
||||||
|
|
||||||
|
# Network access for cloud uploads
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||||
|
|
||||||
|
# Environment
|
||||||
|
EnvironmentFile=-/etc/dbbackup/env.d/cluster.conf
|
||||||
|
|
||||||
|
# Execution - cluster backup (all databases)
|
||||||
|
ExecStart={{.BinaryPath}} backup cluster --config {{.ConfigPath}}
|
||||||
|
TimeoutStartSec={{.TimeoutSeconds}}
|
||||||
|
|
||||||
|
# Post-backup metrics export
|
||||||
|
ExecStopPost=-{{.BinaryPath}} metrics export --instance cluster --output /var/lib/dbbackup/metrics/cluster.prom
|
||||||
|
|
||||||
|
# OOM protection for large backups
|
||||||
|
OOMScoreAdjust=-500
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
11
internal/installer/templates/dbbackup-cluster.timer
Normal file
11
internal/installer/templates/dbbackup-cluster.timer
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Database Cluster Backup Timer
|
||||||
|
Documentation=https://github.com/PlusOne/dbbackup
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar={{.Schedule}}
|
||||||
|
Persistent=true
|
||||||
|
RandomizedDelaySec=1800
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
37
internal/installer/templates/dbbackup-exporter.service
Normal file
37
internal/installer/templates/dbbackup-exporter.service
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=DBBackup Prometheus Metrics Exporter
|
||||||
|
Documentation=https://github.com/PlusOne/dbbackup
|
||||||
|
After=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User={{.User}}
|
||||||
|
Group={{.Group}}
|
||||||
|
|
||||||
|
# Security hardening
|
||||||
|
NoNewPrivileges=yes
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=yes
|
||||||
|
PrivateTmp=yes
|
||||||
|
ProtectKernelTunables=yes
|
||||||
|
ProtectKernelModules=yes
|
||||||
|
ProtectControlGroups=yes
|
||||||
|
RestrictSUIDSGID=yes
|
||||||
|
RestrictRealtime=yes
|
||||||
|
LockPersonality=yes
|
||||||
|
RemoveIPC=yes
|
||||||
|
|
||||||
|
# Read-write access to catalog for metrics collection
|
||||||
|
ReadWritePaths=/var/lib/dbbackup
|
||||||
|
|
||||||
|
# Network for HTTP server
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||||
|
|
||||||
|
# Execution
|
||||||
|
ExecStart={{.BinaryPath}} metrics serve --port {{.MetricsPort}}
|
||||||
|
ExecReload=/bin/kill -HUP $MAINPID
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=5
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
47
internal/installer/templates/dbbackup@.service
Normal file
47
internal/installer/templates/dbbackup@.service
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Database Backup for %i
|
||||||
|
Documentation=https://github.com/PlusOne/dbbackup
|
||||||
|
After=network-online.target postgresql.service mysql.service mariadb.service
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
User={{.User}}
|
||||||
|
Group={{.Group}}
|
||||||
|
|
||||||
|
# Security hardening
|
||||||
|
NoNewPrivileges=yes
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=read-only
|
||||||
|
PrivateTmp=yes
|
||||||
|
ProtectKernelTunables=yes
|
||||||
|
ProtectKernelModules=yes
|
||||||
|
ProtectControlGroups=yes
|
||||||
|
RestrictSUIDSGID=yes
|
||||||
|
RestrictRealtime=yes
|
||||||
|
LockPersonality=yes
|
||||||
|
RemoveIPC=yes
|
||||||
|
CapabilityBoundingSet=
|
||||||
|
AmbientCapabilities=
|
||||||
|
|
||||||
|
# Directories
|
||||||
|
ReadWritePaths={{.BackupDir}} /var/lib/dbbackup /var/log/dbbackup
|
||||||
|
|
||||||
|
# Network access for cloud uploads
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||||
|
|
||||||
|
# Environment
|
||||||
|
EnvironmentFile=-/etc/dbbackup/env.d/%i.conf
|
||||||
|
|
||||||
|
# Execution
|
||||||
|
ExecStart={{.BinaryPath}} backup {{.BackupType}} %i --config {{.ConfigPath}}
|
||||||
|
TimeoutStartSec={{.TimeoutSeconds}}
|
||||||
|
|
||||||
|
# Post-backup metrics export
|
||||||
|
ExecStopPost=-{{.BinaryPath}} metrics export --instance %i --output /var/lib/dbbackup/metrics/%i.prom
|
||||||
|
|
||||||
|
# OOM protection for large backups
|
||||||
|
OOMScoreAdjust=-500
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
11
internal/installer/templates/dbbackup@.timer
Normal file
11
internal/installer/templates/dbbackup@.timer
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Database Backup Timer for %i
|
||||||
|
Documentation=https://github.com/PlusOne/dbbackup
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar={{.Schedule}}
|
||||||
|
Persistent=true
|
||||||
|
RandomizedDelaySec=1800
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
@@ -117,7 +117,7 @@ func NewEngine(sourceCfg, targetCfg *config.Config, log logger.Logger) (*Engine,
|
|||||||
targetDB: targetDB,
|
targetDB: targetDB,
|
||||||
log: log,
|
log: log,
|
||||||
progress: progress.NewSpinner(),
|
progress: progress.NewSpinner(),
|
||||||
workDir: os.TempDir(),
|
workDir: sourceCfg.GetEffectiveWorkDir(),
|
||||||
keepBackup: false,
|
keepBackup: false,
|
||||||
jobs: 4,
|
jobs: 4,
|
||||||
dryRun: false,
|
dryRun: false,
|
||||||
|
|||||||
@@ -202,9 +202,9 @@ func (b *Batcher) formatSummaryDigest(events []*Event, success, failure, dbCount
|
|||||||
|
|
||||||
func (b *Batcher) formatCompactDigest(events []*Event, success, failure int) string {
|
func (b *Batcher) formatCompactDigest(events []*Event, success, failure int) string {
|
||||||
if failure > 0 {
|
if failure > 0 {
|
||||||
return fmt.Sprintf("⚠️ %d/%d operations failed", failure, len(events))
|
return fmt.Sprintf("[WARN] %d/%d operations failed", failure, len(events))
|
||||||
}
|
}
|
||||||
return fmt.Sprintf("✅ All %d operations successful", success)
|
return fmt.Sprintf("[OK] All %d operations successful", success)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *Batcher) formatDetailedDigest(events []*Event) string {
|
func (b *Batcher) formatDetailedDigest(events []*Event) string {
|
||||||
@@ -215,9 +215,9 @@ func (b *Batcher) formatDetailedDigest(events []*Event) string {
|
|||||||
icon := "•"
|
icon := "•"
|
||||||
switch e.Severity {
|
switch e.Severity {
|
||||||
case SeverityError, SeverityCritical:
|
case SeverityError, SeverityCritical:
|
||||||
icon = "❌"
|
icon = "[FAIL]"
|
||||||
case SeverityWarning:
|
case SeverityWarning:
|
||||||
icon = "⚠️"
|
icon = "[WARN]"
|
||||||
}
|
}
|
||||||
|
|
||||||
msg += fmt.Sprintf("%s [%s] %s: %s\n",
|
msg += fmt.Sprintf("%s [%s] %s: %s\n",
|
||||||
|
|||||||
@@ -69,6 +69,7 @@ func (m *Manager) NotifySync(ctx context.Context, event *Event) error {
|
|||||||
m.mu.RUnlock()
|
m.mu.RUnlock()
|
||||||
|
|
||||||
var errors []error
|
var errors []error
|
||||||
|
var errMu sync.Mutex
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
for _, n := range notifiers {
|
for _, n := range notifiers {
|
||||||
@@ -80,7 +81,9 @@ func (m *Manager) NotifySync(ctx context.Context, event *Event) error {
|
|||||||
go func(notifier Notifier) {
|
go func(notifier Notifier) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
if err := notifier.Send(ctx, event); err != nil {
|
if err := notifier.Send(ctx, event); err != nil {
|
||||||
|
errMu.Lock()
|
||||||
errors = append(errors, fmt.Errorf("%s: %w", notifier.Name(), err))
|
errors = append(errors, fmt.Errorf("%s: %w", notifier.Name(), err))
|
||||||
|
errMu.Unlock()
|
||||||
}
|
}
|
||||||
}(n)
|
}(n)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -183,43 +183,43 @@ func DefaultConfig() Config {
|
|||||||
|
|
||||||
// FormatEventSubject generates a subject line for notifications
|
// FormatEventSubject generates a subject line for notifications
|
||||||
func FormatEventSubject(event *Event) string {
|
func FormatEventSubject(event *Event) string {
|
||||||
icon := "ℹ️"
|
icon := "[INFO]"
|
||||||
switch event.Severity {
|
switch event.Severity {
|
||||||
case SeverityWarning:
|
case SeverityWarning:
|
||||||
icon = "⚠️"
|
icon = "[WARN]"
|
||||||
case SeverityError, SeverityCritical:
|
case SeverityError, SeverityCritical:
|
||||||
icon = "❌"
|
icon = "[FAIL]"
|
||||||
}
|
}
|
||||||
|
|
||||||
verb := "Event"
|
verb := "Event"
|
||||||
switch event.Type {
|
switch event.Type {
|
||||||
case EventBackupStarted:
|
case EventBackupStarted:
|
||||||
verb = "Backup Started"
|
verb = "Backup Started"
|
||||||
icon = "🔄"
|
icon = "[EXEC]"
|
||||||
case EventBackupCompleted:
|
case EventBackupCompleted:
|
||||||
verb = "Backup Completed"
|
verb = "Backup Completed"
|
||||||
icon = "✅"
|
icon = "[OK]"
|
||||||
case EventBackupFailed:
|
case EventBackupFailed:
|
||||||
verb = "Backup Failed"
|
verb = "Backup Failed"
|
||||||
icon = "❌"
|
icon = "[FAIL]"
|
||||||
case EventRestoreStarted:
|
case EventRestoreStarted:
|
||||||
verb = "Restore Started"
|
verb = "Restore Started"
|
||||||
icon = "🔄"
|
icon = "[EXEC]"
|
||||||
case EventRestoreCompleted:
|
case EventRestoreCompleted:
|
||||||
verb = "Restore Completed"
|
verb = "Restore Completed"
|
||||||
icon = "✅"
|
icon = "[OK]"
|
||||||
case EventRestoreFailed:
|
case EventRestoreFailed:
|
||||||
verb = "Restore Failed"
|
verb = "Restore Failed"
|
||||||
icon = "❌"
|
icon = "[FAIL]"
|
||||||
case EventCleanupCompleted:
|
case EventCleanupCompleted:
|
||||||
verb = "Cleanup Completed"
|
verb = "Cleanup Completed"
|
||||||
icon = "🗑️"
|
icon = "[DEL]"
|
||||||
case EventVerifyCompleted:
|
case EventVerifyCompleted:
|
||||||
verb = "Verification Passed"
|
verb = "Verification Passed"
|
||||||
icon = "✅"
|
icon = "[OK]"
|
||||||
case EventVerifyFailed:
|
case EventVerifyFailed:
|
||||||
verb = "Verification Failed"
|
verb = "Verification Failed"
|
||||||
icon = "❌"
|
icon = "[FAIL]"
|
||||||
case EventPITRRecovery:
|
case EventPITRRecovery:
|
||||||
verb = "PITR Recovery"
|
verb = "PITR Recovery"
|
||||||
icon = "⏪"
|
icon = "⏪"
|
||||||
|
|||||||
@@ -30,52 +30,52 @@ type Templates struct {
|
|||||||
func DefaultTemplates() map[EventType]Templates {
|
func DefaultTemplates() map[EventType]Templates {
|
||||||
return map[EventType]Templates{
|
return map[EventType]Templates{
|
||||||
EventBackupStarted: {
|
EventBackupStarted: {
|
||||||
Subject: "🔄 Backup Started: {{.Database}} on {{.Hostname}}",
|
Subject: "[EXEC] Backup Started: {{.Database}} on {{.Hostname}}",
|
||||||
TextBody: backupStartedText,
|
TextBody: backupStartedText,
|
||||||
HTMLBody: backupStartedHTML,
|
HTMLBody: backupStartedHTML,
|
||||||
},
|
},
|
||||||
EventBackupCompleted: {
|
EventBackupCompleted: {
|
||||||
Subject: "✅ Backup Completed: {{.Database}} on {{.Hostname}}",
|
Subject: "[OK] Backup Completed: {{.Database}} on {{.Hostname}}",
|
||||||
TextBody: backupCompletedText,
|
TextBody: backupCompletedText,
|
||||||
HTMLBody: backupCompletedHTML,
|
HTMLBody: backupCompletedHTML,
|
||||||
},
|
},
|
||||||
EventBackupFailed: {
|
EventBackupFailed: {
|
||||||
Subject: "❌ Backup FAILED: {{.Database}} on {{.Hostname}}",
|
Subject: "[FAIL] Backup FAILED: {{.Database}} on {{.Hostname}}",
|
||||||
TextBody: backupFailedText,
|
TextBody: backupFailedText,
|
||||||
HTMLBody: backupFailedHTML,
|
HTMLBody: backupFailedHTML,
|
||||||
},
|
},
|
||||||
EventRestoreStarted: {
|
EventRestoreStarted: {
|
||||||
Subject: "🔄 Restore Started: {{.Database}} on {{.Hostname}}",
|
Subject: "[EXEC] Restore Started: {{.Database}} on {{.Hostname}}",
|
||||||
TextBody: restoreStartedText,
|
TextBody: restoreStartedText,
|
||||||
HTMLBody: restoreStartedHTML,
|
HTMLBody: restoreStartedHTML,
|
||||||
},
|
},
|
||||||
EventRestoreCompleted: {
|
EventRestoreCompleted: {
|
||||||
Subject: "✅ Restore Completed: {{.Database}} on {{.Hostname}}",
|
Subject: "[OK] Restore Completed: {{.Database}} on {{.Hostname}}",
|
||||||
TextBody: restoreCompletedText,
|
TextBody: restoreCompletedText,
|
||||||
HTMLBody: restoreCompletedHTML,
|
HTMLBody: restoreCompletedHTML,
|
||||||
},
|
},
|
||||||
EventRestoreFailed: {
|
EventRestoreFailed: {
|
||||||
Subject: "❌ Restore FAILED: {{.Database}} on {{.Hostname}}",
|
Subject: "[FAIL] Restore FAILED: {{.Database}} on {{.Hostname}}",
|
||||||
TextBody: restoreFailedText,
|
TextBody: restoreFailedText,
|
||||||
HTMLBody: restoreFailedHTML,
|
HTMLBody: restoreFailedHTML,
|
||||||
},
|
},
|
||||||
EventVerificationPassed: {
|
EventVerificationPassed: {
|
||||||
Subject: "✅ Verification Passed: {{.Database}}",
|
Subject: "[OK] Verification Passed: {{.Database}}",
|
||||||
TextBody: verificationPassedText,
|
TextBody: verificationPassedText,
|
||||||
HTMLBody: verificationPassedHTML,
|
HTMLBody: verificationPassedHTML,
|
||||||
},
|
},
|
||||||
EventVerificationFailed: {
|
EventVerificationFailed: {
|
||||||
Subject: "❌ Verification FAILED: {{.Database}}",
|
Subject: "[FAIL] Verification FAILED: {{.Database}}",
|
||||||
TextBody: verificationFailedText,
|
TextBody: verificationFailedText,
|
||||||
HTMLBody: verificationFailedHTML,
|
HTMLBody: verificationFailedHTML,
|
||||||
},
|
},
|
||||||
EventDRDrillPassed: {
|
EventDRDrillPassed: {
|
||||||
Subject: "✅ DR Drill Passed: {{.Database}}",
|
Subject: "[OK] DR Drill Passed: {{.Database}}",
|
||||||
TextBody: drDrillPassedText,
|
TextBody: drDrillPassedText,
|
||||||
HTMLBody: drDrillPassedHTML,
|
HTMLBody: drDrillPassedHTML,
|
||||||
},
|
},
|
||||||
EventDRDrillFailed: {
|
EventDRDrillFailed: {
|
||||||
Subject: "❌ DR Drill FAILED: {{.Database}}",
|
Subject: "[FAIL] DR Drill FAILED: {{.Database}}",
|
||||||
TextBody: drDrillFailedText,
|
TextBody: drDrillFailedText,
|
||||||
HTMLBody: drDrillFailedHTML,
|
HTMLBody: drDrillFailedHTML,
|
||||||
},
|
},
|
||||||
@@ -95,7 +95,7 @@ Started At: {{formatTime .Timestamp}}
|
|||||||
|
|
||||||
const backupStartedHTML = `
|
const backupStartedHTML = `
|
||||||
<div style="font-family: Arial, sans-serif; padding: 20px;">
|
<div style="font-family: Arial, sans-serif; padding: 20px;">
|
||||||
<h2 style="color: #3498db;">🔄 Backup Started</h2>
|
<h2 style="color: #3498db;">[EXEC] Backup Started</h2>
|
||||||
<table style="border-collapse: collapse; width: 100%; max-width: 600px;">
|
<table style="border-collapse: collapse; width: 100%; max-width: 600px;">
|
||||||
<tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
|
<tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
|
||||||
<tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
|
<tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
|
||||||
@@ -121,7 +121,7 @@ Completed: {{formatTime .Timestamp}}
|
|||||||
|
|
||||||
const backupCompletedHTML = `
|
const backupCompletedHTML = `
|
||||||
<div style="font-family: Arial, sans-serif; padding: 20px;">
|
<div style="font-family: Arial, sans-serif; padding: 20px;">
|
||||||
<h2 style="color: #27ae60;">✅ Backup Completed</h2>
|
<h2 style="color: #27ae60;">[OK] Backup Completed</h2>
|
||||||
<table style="border-collapse: collapse; width: 100%; max-width: 600px;">
|
<table style="border-collapse: collapse; width: 100%; max-width: 600px;">
|
||||||
<tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
|
<tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
|
||||||
<tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
|
<tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
|
||||||
@@ -137,7 +137,7 @@ const backupCompletedHTML = `
|
|||||||
`
|
`
|
||||||
|
|
||||||
const backupFailedText = `
|
const backupFailedText = `
|
||||||
⚠️ BACKUP FAILED ⚠️
|
[WARN] BACKUP FAILED [WARN]
|
||||||
|
|
||||||
Database: {{.Database}}
|
Database: {{.Database}}
|
||||||
Hostname: {{.Hostname}}
|
Hostname: {{.Hostname}}
|
||||||
@@ -152,7 +152,7 @@ Please investigate immediately.
|
|||||||
|
|
||||||
const backupFailedHTML = `
|
const backupFailedHTML = `
|
||||||
<div style="font-family: Arial, sans-serif; padding: 20px;">
|
<div style="font-family: Arial, sans-serif; padding: 20px;">
|
||||||
<h2 style="color: #e74c3c;">❌ Backup FAILED</h2>
|
<h2 style="color: #e74c3c;">[FAIL] Backup FAILED</h2>
|
||||||
<table style="border-collapse: collapse; width: 100%; max-width: 600px;">
|
<table style="border-collapse: collapse; width: 100%; max-width: 600px;">
|
||||||
<tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
|
<tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
|
||||||
<tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
|
<tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
|
||||||
@@ -176,7 +176,7 @@ Started At: {{formatTime .Timestamp}}
|
|||||||
|
|
||||||
const restoreStartedHTML = `
|
const restoreStartedHTML = `
|
||||||
<div style="font-family: Arial, sans-serif; padding: 20px;">
|
<div style="font-family: Arial, sans-serif; padding: 20px;">
|
||||||
<h2 style="color: #3498db;">🔄 Restore Started</h2>
|
<h2 style="color: #3498db;">[EXEC] Restore Started</h2>
|
||||||
<table style="border-collapse: collapse; width: 100%; max-width: 600px;">
|
<table style="border-collapse: collapse; width: 100%; max-width: 600px;">
|
||||||
<tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
|
<tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
|
||||||
<tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
|
<tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
|
||||||
@@ -200,7 +200,7 @@ Completed: {{formatTime .Timestamp}}
|
|||||||
|
|
||||||
const restoreCompletedHTML = `
|
const restoreCompletedHTML = `
|
||||||
<div style="font-family: Arial, sans-serif; padding: 20px;">
|
<div style="font-family: Arial, sans-serif; padding: 20px;">
|
||||||
<h2 style="color: #27ae60;">✅ Restore Completed</h2>
|
<h2 style="color: #27ae60;">[OK] Restore Completed</h2>
|
||||||
<table style="border-collapse: collapse; width: 100%; max-width: 600px;">
|
<table style="border-collapse: collapse; width: 100%; max-width: 600px;">
|
||||||
<tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
|
<tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
|
||||||
<tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
|
<tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
|
||||||
@@ -214,7 +214,7 @@ const restoreCompletedHTML = `
|
|||||||
`
|
`
|
||||||
|
|
||||||
const restoreFailedText = `
|
const restoreFailedText = `
|
||||||
⚠️ RESTORE FAILED ⚠️
|
[WARN] RESTORE FAILED [WARN]
|
||||||
|
|
||||||
Database: {{.Database}}
|
Database: {{.Database}}
|
||||||
Hostname: {{.Hostname}}
|
Hostname: {{.Hostname}}
|
||||||
@@ -229,7 +229,7 @@ Please investigate immediately.
|
|||||||
|
|
||||||
const restoreFailedHTML = `
|
const restoreFailedHTML = `
|
||||||
<div style="font-family: Arial, sans-serif; padding: 20px;">
|
<div style="font-family: Arial, sans-serif; padding: 20px;">
|
||||||
<h2 style="color: #e74c3c;">❌ Restore FAILED</h2>
|
<h2 style="color: #e74c3c;">[FAIL] Restore FAILED</h2>
|
||||||
<table style="border-collapse: collapse; width: 100%; max-width: 600px;">
|
<table style="border-collapse: collapse; width: 100%; max-width: 600px;">
|
||||||
<tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
|
<tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
|
||||||
<tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
|
<tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
|
||||||
@@ -255,7 +255,7 @@ Verified: {{formatTime .Timestamp}}
|
|||||||
|
|
||||||
const verificationPassedHTML = `
|
const verificationPassedHTML = `
|
||||||
<div style="font-family: Arial, sans-serif; padding: 20px;">
|
<div style="font-family: Arial, sans-serif; padding: 20px;">
|
||||||
<h2 style="color: #27ae60;">✅ Verification Passed</h2>
|
<h2 style="color: #27ae60;">[OK] Verification Passed</h2>
|
||||||
<table style="border-collapse: collapse; width: 100%; max-width: 600px;">
|
<table style="border-collapse: collapse; width: 100%; max-width: 600px;">
|
||||||
<tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
|
<tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
|
||||||
<tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
|
<tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
|
||||||
@@ -269,7 +269,7 @@ const verificationPassedHTML = `
|
|||||||
`
|
`
|
||||||
|
|
||||||
const verificationFailedText = `
|
const verificationFailedText = `
|
||||||
⚠️ VERIFICATION FAILED ⚠️
|
[WARN] VERIFICATION FAILED [WARN]
|
||||||
|
|
||||||
Database: {{.Database}}
|
Database: {{.Database}}
|
||||||
Hostname: {{.Hostname}}
|
Hostname: {{.Hostname}}
|
||||||
@@ -284,7 +284,7 @@ Backup integrity may be compromised. Please investigate.
|
|||||||
|
|
||||||
const verificationFailedHTML = `
|
const verificationFailedHTML = `
|
||||||
<div style="font-family: Arial, sans-serif; padding: 20px;">
|
<div style="font-family: Arial, sans-serif; padding: 20px;">
|
||||||
<h2 style="color: #e74c3c;">❌ Verification FAILED</h2>
|
<h2 style="color: #e74c3c;">[FAIL] Verification FAILED</h2>
|
||||||
<table style="border-collapse: collapse; width: 100%; max-width: 600px;">
|
<table style="border-collapse: collapse; width: 100%; max-width: 600px;">
|
||||||
<tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
|
<tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
|
||||||
<tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
|
<tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
|
||||||
@@ -314,7 +314,7 @@ Backup restore capability verified.
|
|||||||
|
|
||||||
const drDrillPassedHTML = `
|
const drDrillPassedHTML = `
|
||||||
<div style="font-family: Arial, sans-serif; padding: 20px;">
|
<div style="font-family: Arial, sans-serif; padding: 20px;">
|
||||||
<h2 style="color: #27ae60;">✅ DR Drill Passed</h2>
|
<h2 style="color: #27ae60;">[OK] DR Drill Passed</h2>
|
||||||
<table style="border-collapse: collapse; width: 100%; max-width: 600px;">
|
<table style="border-collapse: collapse; width: 100%; max-width: 600px;">
|
||||||
<tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
|
<tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
|
||||||
<tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
|
<tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
|
||||||
@@ -326,12 +326,12 @@ const drDrillPassedHTML = `
|
|||||||
{{end}}
|
{{end}}
|
||||||
</table>
|
</table>
|
||||||
{{if .Message}}<p style="margin-top: 20px; color: #27ae60;">{{.Message}}</p>{{end}}
|
{{if .Message}}<p style="margin-top: 20px; color: #27ae60;">{{.Message}}</p>{{end}}
|
||||||
<p style="margin-top: 20px; color: #27ae60;">✓ Backup restore capability verified</p>
|
<p style="margin-top: 20px; color: #27ae60;">[OK] Backup restore capability verified</p>
|
||||||
</div>
|
</div>
|
||||||
`
|
`
|
||||||
|
|
||||||
const drDrillFailedText = `
|
const drDrillFailedText = `
|
||||||
⚠️ DR DRILL FAILED ⚠️
|
[WARN] DR DRILL FAILED [WARN]
|
||||||
|
|
||||||
Database: {{.Database}}
|
Database: {{.Database}}
|
||||||
Hostname: {{.Hostname}}
|
Hostname: {{.Hostname}}
|
||||||
@@ -346,7 +346,7 @@ Backup may not be restorable. Please investigate immediately.
|
|||||||
|
|
||||||
const drDrillFailedHTML = `
|
const drDrillFailedHTML = `
|
||||||
<div style="font-family: Arial, sans-serif; padding: 20px;">
|
<div style="font-family: Arial, sans-serif; padding: 20px;">
|
||||||
<h2 style="color: #e74c3c;">❌ DR Drill FAILED</h2>
|
<h2 style="color: #e74c3c;">[FAIL] DR Drill FAILED</h2>
|
||||||
<table style="border-collapse: collapse; width: 100%; max-width: 600px;">
|
<table style="border-collapse: collapse; width: 100%; max-width: 600px;">
|
||||||
<tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
|
<tr><td style="padding: 8px; font-weight: bold;">Database:</td><td style="padding: 8px;">{{.Database}}</td></tr>
|
||||||
<tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
|
<tr><td style="padding: 8px; font-weight: bold;">Hostname:</td><td style="padding: 8px;">{{.Hostname}}</td></tr>
|
||||||
|
|||||||
@@ -212,7 +212,11 @@ func (m *BinlogManager) detectTools() error {
|
|||||||
|
|
||||||
// detectServerType determines if we're working with MySQL or MariaDB
|
// detectServerType determines if we're working with MySQL or MariaDB
|
||||||
func (m *BinlogManager) detectServerType() DatabaseType {
|
func (m *BinlogManager) detectServerType() DatabaseType {
|
||||||
cmd := exec.Command(m.mysqlbinlogPath, "--version")
|
// Use timeout to prevent blocking if command hangs
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
cmd := exec.CommandContext(ctx, m.mysqlbinlogPath, "--version")
|
||||||
output, err := cmd.Output()
|
output, err := cmd.Output()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return DatabaseMySQL // Default to MySQL
|
return DatabaseMySQL // Default to MySQL
|
||||||
|
|||||||
@@ -43,9 +43,9 @@ type RestoreOptions struct {
|
|||||||
|
|
||||||
// RestorePointInTime performs a Point-in-Time Recovery
|
// RestorePointInTime performs a Point-in-Time Recovery
|
||||||
func (ro *RestoreOrchestrator) RestorePointInTime(ctx context.Context, opts *RestoreOptions) error {
|
func (ro *RestoreOrchestrator) RestorePointInTime(ctx context.Context, opts *RestoreOptions) error {
|
||||||
ro.log.Info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
ro.log.Info("=====================================================")
|
||||||
ro.log.Info(" Point-in-Time Recovery (PITR)")
|
ro.log.Info(" Point-in-Time Recovery (PITR)")
|
||||||
ro.log.Info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
ro.log.Info("=====================================================")
|
||||||
ro.log.Info("")
|
ro.log.Info("")
|
||||||
ro.log.Info("Target:", "summary", opts.Target.Summary())
|
ro.log.Info("Target:", "summary", opts.Target.Summary())
|
||||||
ro.log.Info("Base Backup:", "path", opts.BaseBackupPath)
|
ro.log.Info("Base Backup:", "path", opts.BaseBackupPath)
|
||||||
@@ -91,11 +91,11 @@ func (ro *RestoreOrchestrator) RestorePointInTime(ctx context.Context, opts *Res
|
|||||||
return fmt.Errorf("failed to generate recovery configuration: %w", err)
|
return fmt.Errorf("failed to generate recovery configuration: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
ro.log.Info("✅ Recovery configuration generated successfully")
|
ro.log.Info("[OK] Recovery configuration generated successfully")
|
||||||
ro.log.Info("")
|
ro.log.Info("")
|
||||||
ro.log.Info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
ro.log.Info("=====================================================")
|
||||||
ro.log.Info(" Next Steps:")
|
ro.log.Info(" Next Steps:")
|
||||||
ro.log.Info("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
ro.log.Info("=====================================================")
|
||||||
ro.log.Info("")
|
ro.log.Info("")
|
||||||
ro.log.Info("1. Start PostgreSQL to begin recovery:")
|
ro.log.Info("1. Start PostgreSQL to begin recovery:")
|
||||||
ro.log.Info(fmt.Sprintf(" pg_ctl -D %s start", opts.TargetDataDir))
|
ro.log.Info(fmt.Sprintf(" pg_ctl -D %s start", opts.TargetDataDir))
|
||||||
@@ -192,7 +192,7 @@ func (ro *RestoreOrchestrator) validateInputs(opts *RestoreOptions) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ro.log.Info("✅ Validation passed")
|
ro.log.Info("[OK] Validation passed")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -238,7 +238,7 @@ func (ro *RestoreOrchestrator) extractTarGzBackup(ctx context.Context, source, d
|
|||||||
return fmt.Errorf("tar extraction failed: %w", err)
|
return fmt.Errorf("tar extraction failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
ro.log.Info("✅ Base backup extracted successfully")
|
ro.log.Info("[OK] Base backup extracted successfully")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -254,7 +254,7 @@ func (ro *RestoreOrchestrator) extractTarBackup(ctx context.Context, source, des
|
|||||||
return fmt.Errorf("tar extraction failed: %w", err)
|
return fmt.Errorf("tar extraction failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
ro.log.Info("✅ Base backup extracted successfully")
|
ro.log.Info("[OK] Base backup extracted successfully")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -270,7 +270,7 @@ func (ro *RestoreOrchestrator) copyDirectoryBackup(ctx context.Context, source,
|
|||||||
return fmt.Errorf("directory copy failed: %w", err)
|
return fmt.Errorf("directory copy failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
ro.log.Info("✅ Base backup copied successfully")
|
ro.log.Info("[OK] Base backup copied successfully")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -291,7 +291,7 @@ func (ro *RestoreOrchestrator) startPostgreSQL(ctx context.Context, opts *Restor
|
|||||||
return fmt.Errorf("pg_ctl start failed: %w", err)
|
return fmt.Errorf("pg_ctl start failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
ro.log.Info("✅ PostgreSQL started successfully")
|
ro.log.Info("[OK] PostgreSQL started successfully")
|
||||||
ro.log.Info("PostgreSQL is now performing recovery...")
|
ro.log.Info("PostgreSQL is now performing recovery...")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -320,7 +320,7 @@ func (ro *RestoreOrchestrator) monitorRecovery(ctx context.Context, opts *Restor
|
|||||||
// Check if recovery is complete by looking for postmaster.pid
|
// Check if recovery is complete by looking for postmaster.pid
|
||||||
pidFile := filepath.Join(opts.TargetDataDir, "postmaster.pid")
|
pidFile := filepath.Join(opts.TargetDataDir, "postmaster.pid")
|
||||||
if _, err := os.Stat(pidFile); err == nil {
|
if _, err := os.Stat(pidFile); err == nil {
|
||||||
ro.log.Info("✅ PostgreSQL is running")
|
ro.log.Info("[OK] PostgreSQL is running")
|
||||||
|
|
||||||
// Check if recovery files still exist
|
// Check if recovery files still exist
|
||||||
recoverySignal := filepath.Join(opts.TargetDataDir, "recovery.signal")
|
recoverySignal := filepath.Join(opts.TargetDataDir, "recovery.signal")
|
||||||
@@ -328,7 +328,7 @@ func (ro *RestoreOrchestrator) monitorRecovery(ctx context.Context, opts *Restor
|
|||||||
|
|
||||||
if _, err := os.Stat(recoverySignal); os.IsNotExist(err) {
|
if _, err := os.Stat(recoverySignal); os.IsNotExist(err) {
|
||||||
if _, err := os.Stat(recoveryConf); os.IsNotExist(err) {
|
if _, err := os.Stat(recoveryConf); os.IsNotExist(err) {
|
||||||
ro.log.Info("✅ Recovery completed - PostgreSQL promoted to primary")
|
ro.log.Info("[OK] Recovery completed - PostgreSQL promoted to primary")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -256,7 +256,7 @@ func (ot *OperationTracker) Complete(message string) {
|
|||||||
|
|
||||||
// Complete visual indicator
|
// Complete visual indicator
|
||||||
if ot.reporter.indicator != nil {
|
if ot.reporter.indicator != nil {
|
||||||
ot.reporter.indicator.Complete(fmt.Sprintf("✅ %s", message))
|
ot.reporter.indicator.Complete(fmt.Sprintf("[OK] %s", message))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Log completion with duration
|
// Log completion with duration
|
||||||
@@ -286,7 +286,7 @@ func (ot *OperationTracker) Fail(err error) {
|
|||||||
|
|
||||||
// Fail visual indicator
|
// Fail visual indicator
|
||||||
if ot.reporter.indicator != nil {
|
if ot.reporter.indicator != nil {
|
||||||
ot.reporter.indicator.Fail(fmt.Sprintf("❌ %s", err.Error()))
|
ot.reporter.indicator.Fail(fmt.Sprintf("[FAIL] %s", err.Error()))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Log failure
|
// Log failure
|
||||||
@@ -427,7 +427,7 @@ type OperationSummary struct {
|
|||||||
// FormatSummary returns a formatted string representation of the summary
|
// FormatSummary returns a formatted string representation of the summary
|
||||||
func (os *OperationSummary) FormatSummary() string {
|
func (os *OperationSummary) FormatSummary() string {
|
||||||
return fmt.Sprintf(
|
return fmt.Sprintf(
|
||||||
"📊 Operations Summary:\n"+
|
"[STATS] Operations Summary:\n"+
|
||||||
" Total: %d | Completed: %d | Failed: %d | Running: %d\n"+
|
" Total: %d | Completed: %d | Failed: %d | Running: %d\n"+
|
||||||
" Total Duration: %s",
|
" Total Duration: %s",
|
||||||
os.TotalOperations,
|
os.TotalOperations,
|
||||||
|
|||||||
@@ -92,13 +92,13 @@ func (s *Spinner) Update(message string) {
|
|||||||
// Complete stops the spinner with a success message
|
// Complete stops the spinner with a success message
|
||||||
func (s *Spinner) Complete(message string) {
|
func (s *Spinner) Complete(message string) {
|
||||||
s.Stop()
|
s.Stop()
|
||||||
fmt.Fprintf(s.writer, "\n✅ %s\n", message)
|
fmt.Fprintf(s.writer, "\n[OK] %s\n", message)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fail stops the spinner with a failure message
|
// Fail stops the spinner with a failure message
|
||||||
func (s *Spinner) Fail(message string) {
|
func (s *Spinner) Fail(message string) {
|
||||||
s.Stop()
|
s.Stop()
|
||||||
fmt.Fprintf(s.writer, "\n❌ %s\n", message)
|
fmt.Fprintf(s.writer, "\n[FAIL] %s\n", message)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop stops the spinner
|
// Stop stops the spinner
|
||||||
@@ -167,13 +167,13 @@ func (d *Dots) Update(message string) {
|
|||||||
// Complete stops the dots with a success message
|
// Complete stops the dots with a success message
|
||||||
func (d *Dots) Complete(message string) {
|
func (d *Dots) Complete(message string) {
|
||||||
d.Stop()
|
d.Stop()
|
||||||
fmt.Fprintf(d.writer, " ✅ %s\n", message)
|
fmt.Fprintf(d.writer, " [OK] %s\n", message)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fail stops the dots with a failure message
|
// Fail stops the dots with a failure message
|
||||||
func (d *Dots) Fail(message string) {
|
func (d *Dots) Fail(message string) {
|
||||||
d.Stop()
|
d.Stop()
|
||||||
fmt.Fprintf(d.writer, " ❌ %s\n", message)
|
fmt.Fprintf(d.writer, " [FAIL] %s\n", message)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop stops the dots indicator
|
// Stop stops the dots indicator
|
||||||
@@ -239,14 +239,14 @@ func (p *ProgressBar) Complete(message string) {
|
|||||||
p.current = p.total
|
p.current = p.total
|
||||||
p.message = message
|
p.message = message
|
||||||
p.render()
|
p.render()
|
||||||
fmt.Fprintf(p.writer, " ✅ %s\n", message)
|
fmt.Fprintf(p.writer, " [OK] %s\n", message)
|
||||||
p.Stop()
|
p.Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fail stops the progress bar with failure
|
// Fail stops the progress bar with failure
|
||||||
func (p *ProgressBar) Fail(message string) {
|
func (p *ProgressBar) Fail(message string) {
|
||||||
p.render()
|
p.render()
|
||||||
fmt.Fprintf(p.writer, " ❌ %s\n", message)
|
fmt.Fprintf(p.writer, " [FAIL] %s\n", message)
|
||||||
p.Stop()
|
p.Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -298,12 +298,12 @@ func (s *Static) Update(message string) {
|
|||||||
|
|
||||||
// Complete shows completion message
|
// Complete shows completion message
|
||||||
func (s *Static) Complete(message string) {
|
func (s *Static) Complete(message string) {
|
||||||
fmt.Fprintf(s.writer, " ✅ %s\n", message)
|
fmt.Fprintf(s.writer, " [OK] %s\n", message)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fail shows failure message
|
// Fail shows failure message
|
||||||
func (s *Static) Fail(message string) {
|
func (s *Static) Fail(message string) {
|
||||||
fmt.Fprintf(s.writer, " ❌ %s\n", message)
|
fmt.Fprintf(s.writer, " [FAIL] %s\n", message)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop does nothing for static indicator
|
// Stop does nothing for static indicator
|
||||||
@@ -359,7 +359,7 @@ func (l *LineByLine) Start(message string) {
|
|||||||
if l.estimator != nil {
|
if l.estimator != nil {
|
||||||
displayMsg = l.estimator.GetFullStatus(message)
|
displayMsg = l.estimator.GetFullStatus(message)
|
||||||
}
|
}
|
||||||
fmt.Fprintf(l.writer, "\n🔄 %s\n", displayMsg)
|
fmt.Fprintf(l.writer, "\n[SYNC] %s\n", displayMsg)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update shows an update message
|
// Update shows an update message
|
||||||
@@ -380,12 +380,12 @@ func (l *LineByLine) SetEstimator(estimator *ETAEstimator) {
|
|||||||
|
|
||||||
// Complete shows completion message
|
// Complete shows completion message
|
||||||
func (l *LineByLine) Complete(message string) {
|
func (l *LineByLine) Complete(message string) {
|
||||||
fmt.Fprintf(l.writer, "✅ %s\n\n", message)
|
fmt.Fprintf(l.writer, "[OK] %s\n\n", message)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fail shows failure message
|
// Fail shows failure message
|
||||||
func (l *LineByLine) Fail(message string) {
|
func (l *LineByLine) Fail(message string) {
|
||||||
fmt.Fprintf(l.writer, "❌ %s\n\n", message)
|
fmt.Fprintf(l.writer, "[FAIL] %s\n\n", message)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop does nothing for line-by-line (no cleanup needed)
|
// Stop does nothing for line-by-line (no cleanup needed)
|
||||||
@@ -396,7 +396,7 @@ func (l *LineByLine) Stop() {
|
|||||||
// Light indicator methods - minimal output
|
// Light indicator methods - minimal output
|
||||||
func (l *Light) Start(message string) {
|
func (l *Light) Start(message string) {
|
||||||
if !l.silent {
|
if !l.silent {
|
||||||
fmt.Fprintf(l.writer, "▶ %s\n", message)
|
fmt.Fprintf(l.writer, "> %s\n", message)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -408,13 +408,13 @@ func (l *Light) Update(message string) {
|
|||||||
|
|
||||||
func (l *Light) Complete(message string) {
|
func (l *Light) Complete(message string) {
|
||||||
if !l.silent {
|
if !l.silent {
|
||||||
fmt.Fprintf(l.writer, "✓ %s\n", message)
|
fmt.Fprintf(l.writer, "[OK] %s\n", message)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l *Light) Fail(message string) {
|
func (l *Light) Fail(message string) {
|
||||||
if !l.silent {
|
if !l.silent {
|
||||||
fmt.Fprintf(l.writer, "✗ %s\n", message)
|
fmt.Fprintf(l.writer, "[FAIL] %s\n", message)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
174
internal/prometheus/exporter.go
Normal file
174
internal/prometheus/exporter.go
Normal file
@@ -0,0 +1,174 @@
|
|||||||
|
// Package prometheus provides Prometheus metrics for dbbackup
|
||||||
|
package prometheus
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"dbbackup/internal/catalog"
|
||||||
|
"dbbackup/internal/logger"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Exporter provides an HTTP endpoint for Prometheus metrics
|
||||||
|
type Exporter struct {
|
||||||
|
log logger.Logger
|
||||||
|
catalog catalog.Catalog
|
||||||
|
instance string
|
||||||
|
port int
|
||||||
|
|
||||||
|
mu sync.RWMutex
|
||||||
|
cachedData string
|
||||||
|
lastRefresh time.Time
|
||||||
|
refreshTTL time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewExporter creates a new Prometheus exporter
|
||||||
|
func NewExporter(log logger.Logger, cat catalog.Catalog, instance string, port int) *Exporter {
|
||||||
|
return &Exporter{
|
||||||
|
log: log,
|
||||||
|
catalog: cat,
|
||||||
|
instance: instance,
|
||||||
|
port: port,
|
||||||
|
refreshTTL: 30 * time.Second,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Serve starts the HTTP server and blocks until context is cancelled
|
||||||
|
func (e *Exporter) Serve(ctx context.Context) error {
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
|
||||||
|
// /metrics endpoint
|
||||||
|
mux.HandleFunc("/metrics", e.handleMetrics)
|
||||||
|
|
||||||
|
// /health endpoint
|
||||||
|
mux.HandleFunc("/health", e.handleHealth)
|
||||||
|
|
||||||
|
// / root with info
|
||||||
|
mux.HandleFunc("/", e.handleRoot)
|
||||||
|
|
||||||
|
addr := fmt.Sprintf(":%d", e.port)
|
||||||
|
srv := &http.Server{
|
||||||
|
Addr: addr,
|
||||||
|
Handler: mux,
|
||||||
|
ReadTimeout: 10 * time.Second,
|
||||||
|
WriteTimeout: 30 * time.Second,
|
||||||
|
IdleTimeout: 60 * time.Second,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start refresh goroutine
|
||||||
|
go e.refreshLoop(ctx)
|
||||||
|
|
||||||
|
// Graceful shutdown
|
||||||
|
go func() {
|
||||||
|
<-ctx.Done()
|
||||||
|
e.log.Info("Shutting down metrics server...")
|
||||||
|
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := srv.Shutdown(shutdownCtx); err != nil {
|
||||||
|
e.log.Error("Server shutdown error", "error", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
e.log.Info("Starting Prometheus metrics server", "addr", addr)
|
||||||
|
if err := srv.ListenAndServe(); err != http.ErrServerClosed {
|
||||||
|
return fmt.Errorf("server error: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleMetrics handles /metrics endpoint
|
||||||
|
func (e *Exporter) handleMetrics(w http.ResponseWriter, r *http.Request) {
|
||||||
|
e.mu.RLock()
|
||||||
|
data := e.cachedData
|
||||||
|
e.mu.RUnlock()
|
||||||
|
|
||||||
|
if data == "" {
|
||||||
|
// Force refresh if cache is empty
|
||||||
|
if err := e.refresh(); err != nil {
|
||||||
|
http.Error(w, "Failed to collect metrics", http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
e.mu.RLock()
|
||||||
|
data = e.cachedData
|
||||||
|
e.mu.RUnlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
w.Write([]byte(data))
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleHealth handles /health endpoint
|
||||||
|
func (e *Exporter) handleHealth(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
w.Write([]byte(`{"status":"ok","service":"dbbackup-exporter"}`))
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleRoot handles / endpoint
|
||||||
|
func (e *Exporter) handleRoot(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/" {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", "text/html")
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
w.Write([]byte(`<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>DBBackup Exporter</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>DBBackup Prometheus Exporter</h1>
|
||||||
|
<p>This is a Prometheus metrics exporter for DBBackup.</p>
|
||||||
|
<ul>
|
||||||
|
<li><a href="/metrics">/metrics</a> - Prometheus metrics</li>
|
||||||
|
<li><a href="/health">/health</a> - Health check</li>
|
||||||
|
</ul>
|
||||||
|
</body>
|
||||||
|
</html>`))
|
||||||
|
}
|
||||||
|
|
||||||
|
// refreshLoop periodically refreshes the metrics cache
|
||||||
|
func (e *Exporter) refreshLoop(ctx context.Context) {
|
||||||
|
ticker := time.NewTicker(e.refreshTTL)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
// Initial refresh
|
||||||
|
if err := e.refresh(); err != nil {
|
||||||
|
e.log.Error("Initial metrics refresh failed", "error", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
if err := e.refresh(); err != nil {
|
||||||
|
e.log.Error("Metrics refresh failed", "error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// refresh updates the cached metrics
|
||||||
|
func (e *Exporter) refresh() error {
|
||||||
|
writer := NewMetricsWriter(e.log, e.catalog, e.instance)
|
||||||
|
data, err := writer.GenerateMetricsString()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
e.mu.Lock()
|
||||||
|
e.cachedData = data
|
||||||
|
e.lastRefresh = time.Now()
|
||||||
|
e.mu.Unlock()
|
||||||
|
|
||||||
|
e.log.Debug("Refreshed metrics cache")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
245
internal/prometheus/textfile.go
Normal file
245
internal/prometheus/textfile.go
Normal file
@@ -0,0 +1,245 @@
|
|||||||
|
// Package prometheus provides Prometheus metrics for dbbackup
|
||||||
|
package prometheus
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"dbbackup/internal/catalog"
|
||||||
|
"dbbackup/internal/logger"
|
||||||
|
)
|
||||||
|
|
||||||
|
// MetricsWriter writes metrics in Prometheus text format
|
||||||
|
type MetricsWriter struct {
|
||||||
|
log logger.Logger
|
||||||
|
catalog catalog.Catalog
|
||||||
|
instance string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewMetricsWriter creates a new MetricsWriter
|
||||||
|
func NewMetricsWriter(log logger.Logger, cat catalog.Catalog, instance string) *MetricsWriter {
|
||||||
|
return &MetricsWriter{
|
||||||
|
log: log,
|
||||||
|
catalog: cat,
|
||||||
|
instance: instance,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// BackupMetrics holds metrics for a single database
|
||||||
|
type BackupMetrics struct {
|
||||||
|
Database string
|
||||||
|
Engine string
|
||||||
|
LastSuccess time.Time
|
||||||
|
LastDuration time.Duration
|
||||||
|
LastSize int64
|
||||||
|
TotalBackups int
|
||||||
|
SuccessCount int
|
||||||
|
FailureCount int
|
||||||
|
Verified bool
|
||||||
|
RPOSeconds float64
|
||||||
|
}
|
||||||
|
|
||||||
|
// WriteTextfile writes metrics to a Prometheus textfile collector file
|
||||||
|
func (m *MetricsWriter) WriteTextfile(path string) error {
|
||||||
|
metrics, err := m.collectMetrics()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to collect metrics: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
output := m.formatMetrics(metrics)
|
||||||
|
|
||||||
|
// Atomic write: write to temp file, then rename
|
||||||
|
dir := filepath.Dir(path)
|
||||||
|
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||||
|
return fmt.Errorf("failed to create directory %s: %w", dir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
tmpPath := path + ".tmp"
|
||||||
|
if err := os.WriteFile(tmpPath, []byte(output), 0644); err != nil {
|
||||||
|
return fmt.Errorf("failed to write temp file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.Rename(tmpPath, path); err != nil {
|
||||||
|
os.Remove(tmpPath)
|
||||||
|
return fmt.Errorf("failed to rename temp file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
m.log.Debug("Wrote metrics to textfile", "path", path, "databases", len(metrics))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// collectMetrics gathers metrics from the catalog
|
||||||
|
func (m *MetricsWriter) collectMetrics() ([]BackupMetrics, error) {
|
||||||
|
if m.catalog == nil {
|
||||||
|
return nil, fmt.Errorf("catalog not available")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
// Get recent backups using Search with limit
|
||||||
|
query := &catalog.SearchQuery{
|
||||||
|
Limit: 1000,
|
||||||
|
}
|
||||||
|
entries, err := m.catalog.Search(ctx, query)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to search backups: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Group by database
|
||||||
|
byDB := make(map[string]*BackupMetrics)
|
||||||
|
|
||||||
|
for _, e := range entries {
|
||||||
|
key := e.Database
|
||||||
|
if key == "" {
|
||||||
|
key = "unknown"
|
||||||
|
}
|
||||||
|
|
||||||
|
metrics, ok := byDB[key]
|
||||||
|
if !ok {
|
||||||
|
metrics = &BackupMetrics{
|
||||||
|
Database: key,
|
||||||
|
Engine: e.DatabaseType,
|
||||||
|
}
|
||||||
|
byDB[key] = metrics
|
||||||
|
}
|
||||||
|
|
||||||
|
metrics.TotalBackups++
|
||||||
|
|
||||||
|
isSuccess := e.Status == catalog.StatusCompleted || e.Status == catalog.StatusVerified
|
||||||
|
if isSuccess {
|
||||||
|
metrics.SuccessCount++
|
||||||
|
// Track most recent success
|
||||||
|
if e.CreatedAt.After(metrics.LastSuccess) {
|
||||||
|
metrics.LastSuccess = e.CreatedAt
|
||||||
|
metrics.LastDuration = time.Duration(e.Duration * float64(time.Second))
|
||||||
|
metrics.LastSize = e.SizeBytes
|
||||||
|
metrics.Verified = e.VerifiedAt != nil && e.VerifyValid != nil && *e.VerifyValid
|
||||||
|
metrics.Engine = e.DatabaseType
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
metrics.FailureCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate RPO for each database
|
||||||
|
now := time.Now()
|
||||||
|
for _, metrics := range byDB {
|
||||||
|
if !metrics.LastSuccess.IsZero() {
|
||||||
|
metrics.RPOSeconds = now.Sub(metrics.LastSuccess).Seconds()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert to slice and sort
|
||||||
|
result := make([]BackupMetrics, 0, len(byDB))
|
||||||
|
for _, metrics := range byDB {
|
||||||
|
result = append(result, *metrics)
|
||||||
|
}
|
||||||
|
sort.Slice(result, func(i, j int) bool {
|
||||||
|
return result[i].Database < result[j].Database
|
||||||
|
})
|
||||||
|
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// formatMetrics formats metrics in Prometheus exposition format
|
||||||
|
func (m *MetricsWriter) formatMetrics(metrics []BackupMetrics) string {
|
||||||
|
var b strings.Builder
|
||||||
|
|
||||||
|
// Timestamp of metrics generation
|
||||||
|
now := time.Now().Unix()
|
||||||
|
|
||||||
|
// Header comment
|
||||||
|
b.WriteString("# DBBackup Prometheus Metrics\n")
|
||||||
|
b.WriteString(fmt.Sprintf("# Generated at: %s\n", time.Now().Format(time.RFC3339)))
|
||||||
|
b.WriteString(fmt.Sprintf("# Instance: %s\n", m.instance))
|
||||||
|
b.WriteString("\n")
|
||||||
|
|
||||||
|
// dbbackup_last_success_timestamp
|
||||||
|
b.WriteString("# HELP dbbackup_last_success_timestamp Unix timestamp of last successful backup\n")
|
||||||
|
b.WriteString("# TYPE dbbackup_last_success_timestamp gauge\n")
|
||||||
|
for _, met := range metrics {
|
||||||
|
if !met.LastSuccess.IsZero() {
|
||||||
|
b.WriteString(fmt.Sprintf("dbbackup_last_success_timestamp{instance=%q,database=%q,engine=%q} %d\n",
|
||||||
|
m.instance, met.Database, met.Engine, met.LastSuccess.Unix()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.WriteString("\n")
|
||||||
|
|
||||||
|
// dbbackup_last_backup_duration_seconds
|
||||||
|
b.WriteString("# HELP dbbackup_last_backup_duration_seconds Duration of last successful backup in seconds\n")
|
||||||
|
b.WriteString("# TYPE dbbackup_last_backup_duration_seconds gauge\n")
|
||||||
|
for _, met := range metrics {
|
||||||
|
if met.LastDuration > 0 {
|
||||||
|
b.WriteString(fmt.Sprintf("dbbackup_last_backup_duration_seconds{instance=%q,database=%q,engine=%q} %.2f\n",
|
||||||
|
m.instance, met.Database, met.Engine, met.LastDuration.Seconds()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.WriteString("\n")
|
||||||
|
|
||||||
|
// dbbackup_last_backup_size_bytes
|
||||||
|
b.WriteString("# HELP dbbackup_last_backup_size_bytes Size of last successful backup in bytes\n")
|
||||||
|
b.WriteString("# TYPE dbbackup_last_backup_size_bytes gauge\n")
|
||||||
|
for _, met := range metrics {
|
||||||
|
if met.LastSize > 0 {
|
||||||
|
b.WriteString(fmt.Sprintf("dbbackup_last_backup_size_bytes{instance=%q,database=%q,engine=%q} %d\n",
|
||||||
|
m.instance, met.Database, met.Engine, met.LastSize))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.WriteString("\n")
|
||||||
|
|
||||||
|
// dbbackup_backup_total (counter)
|
||||||
|
b.WriteString("# HELP dbbackup_backup_total Total number of backup attempts\n")
|
||||||
|
b.WriteString("# TYPE dbbackup_backup_total counter\n")
|
||||||
|
for _, met := range metrics {
|
||||||
|
b.WriteString(fmt.Sprintf("dbbackup_backup_total{instance=%q,database=%q,status=\"success\"} %d\n",
|
||||||
|
m.instance, met.Database, met.SuccessCount))
|
||||||
|
b.WriteString(fmt.Sprintf("dbbackup_backup_total{instance=%q,database=%q,status=\"failure\"} %d\n",
|
||||||
|
m.instance, met.Database, met.FailureCount))
|
||||||
|
}
|
||||||
|
b.WriteString("\n")
|
||||||
|
|
||||||
|
// dbbackup_rpo_seconds
|
||||||
|
b.WriteString("# HELP dbbackup_rpo_seconds Recovery Point Objective - seconds since last successful backup\n")
|
||||||
|
b.WriteString("# TYPE dbbackup_rpo_seconds gauge\n")
|
||||||
|
for _, met := range metrics {
|
||||||
|
if met.RPOSeconds > 0 {
|
||||||
|
b.WriteString(fmt.Sprintf("dbbackup_rpo_seconds{instance=%q,database=%q} %.0f\n",
|
||||||
|
m.instance, met.Database, met.RPOSeconds))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.WriteString("\n")
|
||||||
|
|
||||||
|
// dbbackup_backup_verified
|
||||||
|
b.WriteString("# HELP dbbackup_backup_verified Whether the last backup was verified (1=yes, 0=no)\n")
|
||||||
|
b.WriteString("# TYPE dbbackup_backup_verified gauge\n")
|
||||||
|
for _, met := range metrics {
|
||||||
|
verified := 0
|
||||||
|
if met.Verified {
|
||||||
|
verified = 1
|
||||||
|
}
|
||||||
|
b.WriteString(fmt.Sprintf("dbbackup_backup_verified{instance=%q,database=%q} %d\n",
|
||||||
|
m.instance, met.Database, verified))
|
||||||
|
}
|
||||||
|
b.WriteString("\n")
|
||||||
|
|
||||||
|
// dbbackup_scrape_timestamp
|
||||||
|
b.WriteString("# HELP dbbackup_scrape_timestamp Unix timestamp when metrics were collected\n")
|
||||||
|
b.WriteString("# TYPE dbbackup_scrape_timestamp gauge\n")
|
||||||
|
b.WriteString(fmt.Sprintf("dbbackup_scrape_timestamp{instance=%q} %d\n", m.instance, now))
|
||||||
|
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// GenerateMetricsString returns metrics as a string (for HTTP endpoint)
|
||||||
|
func (m *MetricsWriter) GenerateMetricsString() (string, error) {
|
||||||
|
metrics, err := m.collectMetrics()
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return m.formatMetrics(metrics), nil
|
||||||
|
}
|
||||||
@@ -296,11 +296,11 @@ func generateID() string {
|
|||||||
func StatusIcon(s ComplianceStatus) string {
|
func StatusIcon(s ComplianceStatus) string {
|
||||||
switch s {
|
switch s {
|
||||||
case StatusCompliant:
|
case StatusCompliant:
|
||||||
return "✅"
|
return "[OK]"
|
||||||
case StatusNonCompliant:
|
case StatusNonCompliant:
|
||||||
return "❌"
|
return "[FAIL]"
|
||||||
case StatusPartial:
|
case StatusPartial:
|
||||||
return "⚠️"
|
return "[WARN]"
|
||||||
case StatusNotApplicable:
|
case StatusNotApplicable:
|
||||||
return "➖"
|
return "➖"
|
||||||
default:
|
default:
|
||||||
|
|||||||
@@ -47,9 +47,10 @@ type DownloadResult struct {
|
|||||||
|
|
||||||
// Download downloads a backup from cloud storage
|
// Download downloads a backup from cloud storage
|
||||||
func (d *CloudDownloader) Download(ctx context.Context, remotePath string, opts DownloadOptions) (*DownloadResult, error) {
|
func (d *CloudDownloader) Download(ctx context.Context, remotePath string, opts DownloadOptions) (*DownloadResult, error) {
|
||||||
// Determine temp directory
|
// Determine temp directory (use from opts, or from config's WorkDir, or fallback to system temp)
|
||||||
tempDir := opts.TempDir
|
tempDir := opts.TempDir
|
||||||
if tempDir == "" {
|
if tempDir == "" {
|
||||||
|
// Try to get from config if available (passed via opts.TempDir)
|
||||||
tempDir = os.TempDir()
|
tempDir = os.TempDir()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
850
internal/restore/diagnose.go
Normal file
850
internal/restore/diagnose.go
Normal file
@@ -0,0 +1,850 @@
|
|||||||
|
package restore
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"bytes"
|
||||||
|
"compress/gzip"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"dbbackup/internal/logger"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DiagnoseResult contains the results of a dump file diagnosis
|
||||||
|
type DiagnoseResult struct {
|
||||||
|
FilePath string `json:"file_path"`
|
||||||
|
FileName string `json:"file_name"`
|
||||||
|
FileSize int64 `json:"file_size"`
|
||||||
|
Format ArchiveFormat `json:"format"`
|
||||||
|
DetectedFormat string `json:"detected_format"`
|
||||||
|
IsValid bool `json:"is_valid"`
|
||||||
|
IsTruncated bool `json:"is_truncated"`
|
||||||
|
IsCorrupted bool `json:"is_corrupted"`
|
||||||
|
Errors []string `json:"errors,omitempty"`
|
||||||
|
Warnings []string `json:"warnings,omitempty"`
|
||||||
|
Details *DiagnoseDetails `json:"details,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// DiagnoseDetails contains detailed analysis of the dump file
|
||||||
|
type DiagnoseDetails struct {
|
||||||
|
// Header info
|
||||||
|
HasPGDMPSignature bool `json:"has_pgdmp_signature,omitempty"`
|
||||||
|
HasSQLHeader bool `json:"has_sql_header,omitempty"`
|
||||||
|
FirstBytes string `json:"first_bytes,omitempty"`
|
||||||
|
LastBytes string `json:"last_bytes,omitempty"`
|
||||||
|
|
||||||
|
// COPY block analysis (for SQL dumps)
|
||||||
|
CopyBlockCount int `json:"copy_block_count,omitempty"`
|
||||||
|
UnterminatedCopy bool `json:"unterminated_copy,omitempty"`
|
||||||
|
LastCopyTable string `json:"last_copy_table,omitempty"`
|
||||||
|
LastCopyLineNumber int `json:"last_copy_line_number,omitempty"`
|
||||||
|
SampleCopyData []string `json:"sample_copy_data,omitempty"`
|
||||||
|
|
||||||
|
// Structure analysis
|
||||||
|
HasCreateStatements bool `json:"has_create_statements,omitempty"`
|
||||||
|
HasInsertStatements bool `json:"has_insert_statements,omitempty"`
|
||||||
|
HasCopyStatements bool `json:"has_copy_statements,omitempty"`
|
||||||
|
HasTransactionBlock bool `json:"has_transaction_block,omitempty"`
|
||||||
|
ProperlyTerminated bool `json:"properly_terminated,omitempty"`
|
||||||
|
|
||||||
|
// pg_restore analysis (for custom format)
|
||||||
|
PgRestoreListable bool `json:"pg_restore_listable,omitempty"`
|
||||||
|
PgRestoreError string `json:"pg_restore_error,omitempty"`
|
||||||
|
TableCount int `json:"table_count,omitempty"`
|
||||||
|
TableList []string `json:"table_list,omitempty"`
|
||||||
|
|
||||||
|
// Compression analysis
|
||||||
|
GzipValid bool `json:"gzip_valid,omitempty"`
|
||||||
|
GzipError string `json:"gzip_error,omitempty"`
|
||||||
|
ExpandedSize int64 `json:"expanded_size,omitempty"`
|
||||||
|
CompressionRatio float64 `json:"compression_ratio,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Diagnoser performs deep analysis of backup files
|
||||||
|
type Diagnoser struct {
|
||||||
|
log logger.Logger
|
||||||
|
verbose bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewDiagnoser creates a new diagnoser
|
||||||
|
func NewDiagnoser(log logger.Logger, verbose bool) *Diagnoser {
|
||||||
|
return &Diagnoser{
|
||||||
|
log: log,
|
||||||
|
verbose: verbose,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// DiagnoseFile performs comprehensive diagnosis of a backup file
|
||||||
|
func (d *Diagnoser) DiagnoseFile(filePath string) (*DiagnoseResult, error) {
|
||||||
|
result := &DiagnoseResult{
|
||||||
|
FilePath: filePath,
|
||||||
|
FileName: filepath.Base(filePath),
|
||||||
|
Details: &DiagnoseDetails{},
|
||||||
|
IsValid: true, // Assume valid until proven otherwise
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check file exists and get size
|
||||||
|
stat, err := os.Stat(filePath)
|
||||||
|
if err != nil {
|
||||||
|
result.IsValid = false
|
||||||
|
result.Errors = append(result.Errors, fmt.Sprintf("Cannot access file: %v", err))
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
result.FileSize = stat.Size()
|
||||||
|
|
||||||
|
if result.FileSize == 0 {
|
||||||
|
result.IsValid = false
|
||||||
|
result.IsTruncated = true
|
||||||
|
result.Errors = append(result.Errors, "File is empty (0 bytes)")
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Detect format
|
||||||
|
result.Format = DetectArchiveFormat(filePath)
|
||||||
|
result.DetectedFormat = result.Format.String()
|
||||||
|
|
||||||
|
// Analyze based on format
|
||||||
|
switch result.Format {
|
||||||
|
case FormatPostgreSQLDump:
|
||||||
|
d.diagnosePgDump(filePath, result)
|
||||||
|
case FormatPostgreSQLDumpGz:
|
||||||
|
d.diagnosePgDumpGz(filePath, result)
|
||||||
|
case FormatPostgreSQLSQL:
|
||||||
|
d.diagnoseSQLScript(filePath, false, result)
|
||||||
|
case FormatPostgreSQLSQLGz:
|
||||||
|
d.diagnoseSQLScript(filePath, true, result)
|
||||||
|
case FormatClusterTarGz:
|
||||||
|
d.diagnoseClusterArchive(filePath, result)
|
||||||
|
default:
|
||||||
|
result.Warnings = append(result.Warnings, "Unknown format - limited diagnosis available")
|
||||||
|
d.diagnoseUnknown(filePath, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// diagnosePgDump analyzes PostgreSQL custom format dump
|
||||||
|
func (d *Diagnoser) diagnosePgDump(filePath string, result *DiagnoseResult) {
|
||||||
|
file, err := os.Open(filePath)
|
||||||
|
if err != nil {
|
||||||
|
result.IsValid = false
|
||||||
|
result.Errors = append(result.Errors, fmt.Sprintf("Cannot open file: %v", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
// Read first 512 bytes
|
||||||
|
header := make([]byte, 512)
|
||||||
|
n, err := file.Read(header)
|
||||||
|
if err != nil && err != io.EOF {
|
||||||
|
result.IsValid = false
|
||||||
|
result.Errors = append(result.Errors, fmt.Sprintf("Cannot read header: %v", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check PGDMP signature
|
||||||
|
if n >= 5 && string(header[:5]) == "PGDMP" {
|
||||||
|
result.Details.HasPGDMPSignature = true
|
||||||
|
result.Details.FirstBytes = "PGDMP..."
|
||||||
|
} else {
|
||||||
|
result.IsValid = false
|
||||||
|
result.IsCorrupted = true
|
||||||
|
result.Details.HasPGDMPSignature = false
|
||||||
|
result.Details.FirstBytes = fmt.Sprintf("%q", header[:minInt(n, 20)])
|
||||||
|
result.Errors = append(result.Errors,
|
||||||
|
"Missing PGDMP signature - file is NOT PostgreSQL custom format",
|
||||||
|
"This file may be SQL format incorrectly named as .dump",
|
||||||
|
"Try: file "+filePath+" to check actual file type")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try pg_restore --list to verify dump integrity
|
||||||
|
d.verifyWithPgRestore(filePath, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// diagnosePgDumpGz analyzes compressed PostgreSQL custom format dump
|
||||||
|
func (d *Diagnoser) diagnosePgDumpGz(filePath string, result *DiagnoseResult) {
|
||||||
|
file, err := os.Open(filePath)
|
||||||
|
if err != nil {
|
||||||
|
result.IsValid = false
|
||||||
|
result.Errors = append(result.Errors, fmt.Sprintf("Cannot open file: %v", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
// Verify gzip integrity
|
||||||
|
gz, err := gzip.NewReader(file)
|
||||||
|
if err != nil {
|
||||||
|
result.IsValid = false
|
||||||
|
result.IsCorrupted = true
|
||||||
|
result.Details.GzipValid = false
|
||||||
|
result.Details.GzipError = err.Error()
|
||||||
|
result.Errors = append(result.Errors,
|
||||||
|
fmt.Sprintf("Invalid gzip format: %v", err),
|
||||||
|
"The file may be truncated or corrupted during transfer")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
result.Details.GzipValid = true
|
||||||
|
|
||||||
|
// Read and check header
|
||||||
|
header := make([]byte, 512)
|
||||||
|
n, err := gz.Read(header)
|
||||||
|
if err != nil && err != io.EOF {
|
||||||
|
result.IsValid = false
|
||||||
|
result.Errors = append(result.Errors, fmt.Sprintf("Cannot read decompressed header: %v", err))
|
||||||
|
gz.Close()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
gz.Close()
|
||||||
|
|
||||||
|
// Check PGDMP signature
|
||||||
|
if n >= 5 && string(header[:5]) == "PGDMP" {
|
||||||
|
result.Details.HasPGDMPSignature = true
|
||||||
|
result.Details.FirstBytes = "PGDMP..."
|
||||||
|
} else {
|
||||||
|
result.Details.HasPGDMPSignature = false
|
||||||
|
result.Details.FirstBytes = fmt.Sprintf("%q", header[:minInt(n, 20)])
|
||||||
|
|
||||||
|
// Check if it's actually SQL content
|
||||||
|
content := string(header[:n])
|
||||||
|
if strings.Contains(content, "PostgreSQL") || strings.Contains(content, "pg_dump") ||
|
||||||
|
strings.Contains(content, "SET ") || strings.Contains(content, "CREATE ") {
|
||||||
|
result.Details.HasSQLHeader = true
|
||||||
|
result.Warnings = append(result.Warnings,
|
||||||
|
"File contains SQL text but has .dump extension",
|
||||||
|
"This appears to be SQL format, not custom format",
|
||||||
|
"Restore should use psql, not pg_restore")
|
||||||
|
} else {
|
||||||
|
result.IsValid = false
|
||||||
|
result.IsCorrupted = true
|
||||||
|
result.Errors = append(result.Errors,
|
||||||
|
"Missing PGDMP signature in decompressed content",
|
||||||
|
"File is neither custom format nor valid SQL")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify full gzip stream integrity by reading to end
|
||||||
|
file.Seek(0, 0)
|
||||||
|
gz, _ = gzip.NewReader(file)
|
||||||
|
|
||||||
|
var totalRead int64
|
||||||
|
buf := make([]byte, 32*1024)
|
||||||
|
for {
|
||||||
|
n, err := gz.Read(buf)
|
||||||
|
totalRead += int64(n)
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
result.IsValid = false
|
||||||
|
result.IsTruncated = true
|
||||||
|
result.Details.ExpandedSize = totalRead
|
||||||
|
result.Errors = append(result.Errors,
|
||||||
|
fmt.Sprintf("Gzip stream truncated after %d bytes: %v", totalRead, err),
|
||||||
|
"The backup file appears to be incomplete",
|
||||||
|
"Check if backup process completed successfully")
|
||||||
|
gz.Close()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
gz.Close()
|
||||||
|
|
||||||
|
result.Details.ExpandedSize = totalRead
|
||||||
|
if result.FileSize > 0 {
|
||||||
|
result.Details.CompressionRatio = float64(totalRead) / float64(result.FileSize)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// diagnoseSQLScript analyzes SQL script format
|
||||||
|
func (d *Diagnoser) diagnoseSQLScript(filePath string, compressed bool, result *DiagnoseResult) {
|
||||||
|
var reader io.Reader
|
||||||
|
var file *os.File
|
||||||
|
var gz *gzip.Reader
|
||||||
|
var err error
|
||||||
|
|
||||||
|
file, err = os.Open(filePath)
|
||||||
|
if err != nil {
|
||||||
|
result.IsValid = false
|
||||||
|
result.Errors = append(result.Errors, fmt.Sprintf("Cannot open file: %v", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
if compressed {
|
||||||
|
gz, err = gzip.NewReader(file)
|
||||||
|
if err != nil {
|
||||||
|
result.IsValid = false
|
||||||
|
result.IsCorrupted = true
|
||||||
|
result.Details.GzipValid = false
|
||||||
|
result.Details.GzipError = err.Error()
|
||||||
|
result.Errors = append(result.Errors, fmt.Sprintf("Invalid gzip format: %v", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
result.Details.GzipValid = true
|
||||||
|
reader = gz
|
||||||
|
defer gz.Close()
|
||||||
|
} else {
|
||||||
|
reader = file
|
||||||
|
}
|
||||||
|
|
||||||
|
// Analyze SQL content
|
||||||
|
scanner := bufio.NewScanner(reader)
|
||||||
|
// Increase buffer size for large lines (COPY data can have long lines)
|
||||||
|
buf := make([]byte, 0, 1024*1024)
|
||||||
|
scanner.Buffer(buf, 10*1024*1024)
|
||||||
|
|
||||||
|
var lineNumber int
|
||||||
|
var inCopyBlock bool
|
||||||
|
var lastCopyTable string
|
||||||
|
var copyStartLine int
|
||||||
|
var copyDataSamples []string
|
||||||
|
|
||||||
|
copyBlockPattern := regexp.MustCompile(`^COPY\s+("?[\w\."]+)"?\s+\(`)
|
||||||
|
copyEndPattern := regexp.MustCompile(`^\\\.`)
|
||||||
|
|
||||||
|
for scanner.Scan() {
|
||||||
|
lineNumber++
|
||||||
|
line := scanner.Text()
|
||||||
|
|
||||||
|
// Check first few lines for header
|
||||||
|
if lineNumber <= 10 {
|
||||||
|
if strings.Contains(line, "PostgreSQL") || strings.Contains(line, "pg_dump") {
|
||||||
|
result.Details.HasSQLHeader = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Track structure
|
||||||
|
upperLine := strings.ToUpper(strings.TrimSpace(line))
|
||||||
|
if strings.HasPrefix(upperLine, "CREATE ") {
|
||||||
|
result.Details.HasCreateStatements = true
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(upperLine, "INSERT ") {
|
||||||
|
result.Details.HasInsertStatements = true
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(upperLine, "BEGIN") {
|
||||||
|
result.Details.HasTransactionBlock = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Track COPY blocks
|
||||||
|
if copyBlockPattern.MatchString(line) {
|
||||||
|
if inCopyBlock {
|
||||||
|
// Previous COPY block wasn't terminated!
|
||||||
|
result.Details.UnterminatedCopy = true
|
||||||
|
result.IsTruncated = true
|
||||||
|
result.IsValid = false
|
||||||
|
result.Errors = append(result.Errors,
|
||||||
|
fmt.Sprintf("COPY block for '%s' starting at line %d was never terminated",
|
||||||
|
lastCopyTable, copyStartLine))
|
||||||
|
}
|
||||||
|
|
||||||
|
inCopyBlock = true
|
||||||
|
result.Details.HasCopyStatements = true
|
||||||
|
result.Details.CopyBlockCount++
|
||||||
|
|
||||||
|
matches := copyBlockPattern.FindStringSubmatch(line)
|
||||||
|
if len(matches) > 1 {
|
||||||
|
lastCopyTable = matches[1]
|
||||||
|
}
|
||||||
|
copyStartLine = lineNumber
|
||||||
|
copyDataSamples = nil
|
||||||
|
|
||||||
|
} else if copyEndPattern.MatchString(line) {
|
||||||
|
inCopyBlock = false
|
||||||
|
|
||||||
|
} else if inCopyBlock {
|
||||||
|
// We're in COPY data
|
||||||
|
if len(copyDataSamples) < 3 {
|
||||||
|
copyDataSamples = append(copyDataSamples, truncateString(line, 100))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store last line for termination check
|
||||||
|
if lineNumber > 0 && (lineNumber%100000 == 0) && d.verbose {
|
||||||
|
d.log.Debug("Scanning SQL file", "lines_processed", lineNumber)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
result.IsValid = false
|
||||||
|
result.IsTruncated = true
|
||||||
|
result.Errors = append(result.Errors,
|
||||||
|
fmt.Sprintf("Error reading file at line %d: %v", lineNumber, err),
|
||||||
|
"File may be truncated or contain invalid data")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if we ended while still in a COPY block
|
||||||
|
if inCopyBlock {
|
||||||
|
result.Details.UnterminatedCopy = true
|
||||||
|
result.Details.LastCopyTable = lastCopyTable
|
||||||
|
result.Details.LastCopyLineNumber = copyStartLine
|
||||||
|
result.Details.SampleCopyData = copyDataSamples
|
||||||
|
result.IsTruncated = true
|
||||||
|
result.IsValid = false
|
||||||
|
result.Errors = append(result.Errors,
|
||||||
|
fmt.Sprintf("File ends inside COPY block for table '%s' (started at line %d)",
|
||||||
|
lastCopyTable, copyStartLine),
|
||||||
|
"The backup was truncated during data export",
|
||||||
|
"This explains the 'syntax error' during restore - COPY data is being interpreted as SQL")
|
||||||
|
|
||||||
|
if len(copyDataSamples) > 0 {
|
||||||
|
result.Errors = append(result.Errors,
|
||||||
|
fmt.Sprintf("Sample orphaned data: %s", copyDataSamples[0]))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
result.Details.ProperlyTerminated = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read last bytes for additional context
|
||||||
|
if !compressed {
|
||||||
|
file.Seek(-min(500, result.FileSize), 2)
|
||||||
|
lastBytes := make([]byte, 500)
|
||||||
|
n, _ := file.Read(lastBytes)
|
||||||
|
result.Details.LastBytes = strings.TrimSpace(string(lastBytes[:n]))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// diagnoseClusterArchive analyzes a cluster tar.gz archive
|
||||||
|
func (d *Diagnoser) diagnoseClusterArchive(filePath string, result *DiagnoseResult) {
|
||||||
|
// First verify tar.gz integrity with timeout
|
||||||
|
// 5 minutes for large archives (multi-GB archives need more time)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
cmd := exec.CommandContext(ctx, "tar", "-tzf", filePath)
|
||||||
|
output, err := cmd.Output()
|
||||||
|
if err != nil {
|
||||||
|
result.IsValid = false
|
||||||
|
result.IsCorrupted = true
|
||||||
|
result.Errors = append(result.Errors,
|
||||||
|
fmt.Sprintf("Tar archive is invalid or corrupted: %v", err),
|
||||||
|
"Run: tar -tzf "+filePath+" 2>&1 | tail -20")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse tar listing
|
||||||
|
files := strings.Split(strings.TrimSpace(string(output)), "\n")
|
||||||
|
var dumpFiles []string
|
||||||
|
hasGlobals := false
|
||||||
|
hasMetadata := false
|
||||||
|
|
||||||
|
for _, f := range files {
|
||||||
|
if strings.HasSuffix(f, ".dump") || strings.HasSuffix(f, ".sql.gz") {
|
||||||
|
dumpFiles = append(dumpFiles, f)
|
||||||
|
}
|
||||||
|
if strings.Contains(f, "globals.sql") {
|
||||||
|
hasGlobals = true
|
||||||
|
}
|
||||||
|
if strings.Contains(f, "manifest.json") || strings.Contains(f, "metadata.json") {
|
||||||
|
hasMetadata = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
result.Details.TableCount = len(dumpFiles)
|
||||||
|
result.Details.TableList = dumpFiles
|
||||||
|
|
||||||
|
if len(dumpFiles) == 0 {
|
||||||
|
result.Warnings = append(result.Warnings, "No database dump files found in archive")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !hasGlobals {
|
||||||
|
result.Warnings = append(result.Warnings, "No globals.sql found - roles/tablespaces won't be restored")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !hasMetadata {
|
||||||
|
result.Warnings = append(result.Warnings, "No manifest/metadata found - limited validation possible")
|
||||||
|
}
|
||||||
|
|
||||||
|
// For verbose mode, diagnose individual dumps inside the archive
|
||||||
|
if d.verbose && len(dumpFiles) > 0 {
|
||||||
|
d.log.Info("Cluster archive contains databases", "count", len(dumpFiles))
|
||||||
|
for _, df := range dumpFiles {
|
||||||
|
d.log.Info(" - " + df)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// diagnoseUnknown handles unknown format files
|
||||||
|
func (d *Diagnoser) diagnoseUnknown(filePath string, result *DiagnoseResult) {
|
||||||
|
file, err := os.Open(filePath)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
header := make([]byte, 512)
|
||||||
|
n, _ := file.Read(header)
|
||||||
|
result.Details.FirstBytes = fmt.Sprintf("%q", header[:minInt(n, 50)])
|
||||||
|
|
||||||
|
// Try to identify by content
|
||||||
|
content := string(header[:n])
|
||||||
|
if strings.Contains(content, "PGDMP") {
|
||||||
|
result.Warnings = append(result.Warnings, "File appears to be PostgreSQL custom format - rename to .dump")
|
||||||
|
} else if strings.Contains(content, "PostgreSQL") || strings.Contains(content, "pg_dump") {
|
||||||
|
result.Warnings = append(result.Warnings, "File appears to be PostgreSQL SQL - rename to .sql")
|
||||||
|
} else if bytes.HasPrefix(header, []byte{0x1f, 0x8b}) {
|
||||||
|
result.Warnings = append(result.Warnings, "File appears to be gzip compressed - add .gz extension")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// verifyWithPgRestore uses pg_restore --list to verify dump integrity
|
||||||
|
func (d *Diagnoser) verifyWithPgRestore(filePath string, result *DiagnoseResult) {
|
||||||
|
// Use timeout to prevent blocking on very large dump files
|
||||||
|
// 5 minutes for large dumps (multi-GB dumps with many tables)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
cmd := exec.CommandContext(ctx, "pg_restore", "--list", filePath)
|
||||||
|
output, err := cmd.CombinedOutput()
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
result.Details.PgRestoreListable = false
|
||||||
|
result.Details.PgRestoreError = string(output)
|
||||||
|
|
||||||
|
// Check for specific errors
|
||||||
|
errStr := string(output)
|
||||||
|
if strings.Contains(errStr, "unexpected end of file") ||
|
||||||
|
strings.Contains(errStr, "invalid large-object TOC entry") {
|
||||||
|
result.IsTruncated = true
|
||||||
|
result.IsValid = false
|
||||||
|
result.Errors = append(result.Errors,
|
||||||
|
"pg_restore reports truncated or incomplete dump file",
|
||||||
|
fmt.Sprintf("Error: %s", truncateString(errStr, 200)))
|
||||||
|
} else if strings.Contains(errStr, "not a valid archive") {
|
||||||
|
result.IsCorrupted = true
|
||||||
|
result.IsValid = false
|
||||||
|
result.Errors = append(result.Errors,
|
||||||
|
"pg_restore reports file is not a valid archive",
|
||||||
|
"File may be corrupted or wrong format")
|
||||||
|
} else {
|
||||||
|
result.Warnings = append(result.Warnings,
|
||||||
|
fmt.Sprintf("pg_restore --list warning: %s", truncateString(errStr, 200)))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
result.Details.PgRestoreListable = true
|
||||||
|
|
||||||
|
// Count tables in the TOC
|
||||||
|
lines := strings.Split(string(output), "\n")
|
||||||
|
tableCount := 0
|
||||||
|
var tables []string
|
||||||
|
for _, line := range lines {
|
||||||
|
if strings.Contains(line, " TABLE DATA ") {
|
||||||
|
tableCount++
|
||||||
|
if len(tables) < 20 {
|
||||||
|
parts := strings.Fields(line)
|
||||||
|
if len(parts) > 3 {
|
||||||
|
tables = append(tables, parts[len(parts)-1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result.Details.TableCount = tableCount
|
||||||
|
result.Details.TableList = tables
|
||||||
|
}
|
||||||
|
|
||||||
|
// DiagnoseClusterDumps extracts and diagnoses all dumps in a cluster archive
|
||||||
|
func (d *Diagnoser) DiagnoseClusterDumps(archivePath, tempDir string) ([]*DiagnoseResult, error) {
|
||||||
|
// First, try to list archive contents without extracting (fast check)
|
||||||
|
// 10 minutes for very large archives
|
||||||
|
listCtx, listCancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||||
|
defer listCancel()
|
||||||
|
|
||||||
|
listCmd := exec.CommandContext(listCtx, "tar", "-tzf", archivePath)
|
||||||
|
listOutput, listErr := listCmd.CombinedOutput()
|
||||||
|
if listErr != nil {
|
||||||
|
// Archive listing failed - likely corrupted
|
||||||
|
errResult := &DiagnoseResult{
|
||||||
|
FilePath: archivePath,
|
||||||
|
FileName: filepath.Base(archivePath),
|
||||||
|
Format: FormatClusterTarGz,
|
||||||
|
DetectedFormat: "Cluster Archive (tar.gz)",
|
||||||
|
IsValid: false,
|
||||||
|
IsCorrupted: true,
|
||||||
|
Details: &DiagnoseDetails{},
|
||||||
|
}
|
||||||
|
|
||||||
|
errOutput := string(listOutput)
|
||||||
|
if strings.Contains(errOutput, "unexpected end of file") ||
|
||||||
|
strings.Contains(errOutput, "Unexpected EOF") ||
|
||||||
|
strings.Contains(errOutput, "truncated") {
|
||||||
|
errResult.IsTruncated = true
|
||||||
|
errResult.Errors = append(errResult.Errors,
|
||||||
|
"Archive appears to be TRUNCATED - incomplete download or backup",
|
||||||
|
fmt.Sprintf("tar error: %s", truncateString(errOutput, 300)),
|
||||||
|
"Possible causes: disk full during backup, interrupted transfer, network timeout",
|
||||||
|
"Solution: Re-create the backup from source database")
|
||||||
|
} else {
|
||||||
|
errResult.Errors = append(errResult.Errors,
|
||||||
|
fmt.Sprintf("Cannot list archive contents: %v", listErr),
|
||||||
|
fmt.Sprintf("tar error: %s", truncateString(errOutput, 300)),
|
||||||
|
"Run manually: tar -tzf "+archivePath+" 2>&1 | tail -50")
|
||||||
|
}
|
||||||
|
|
||||||
|
return []*DiagnoseResult{errResult}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Archive is listable - now check disk space before extraction
|
||||||
|
files := strings.Split(strings.TrimSpace(string(listOutput)), "\n")
|
||||||
|
|
||||||
|
// Check if we have enough disk space (estimate 4x archive size needed)
|
||||||
|
archiveInfo, _ := os.Stat(archivePath)
|
||||||
|
requiredSpace := archiveInfo.Size() * 4
|
||||||
|
|
||||||
|
// Check temp directory space - try to extract metadata first
|
||||||
|
if stat, err := os.Stat(tempDir); err == nil && stat.IsDir() {
|
||||||
|
// Try extraction of a small test file first with timeout
|
||||||
|
testCtx, testCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
|
testCmd := exec.CommandContext(testCtx, "tar", "-xzf", archivePath, "-C", tempDir, "--wildcards", "*.json", "--wildcards", "globals.sql")
|
||||||
|
testCmd.Run() // Ignore error - just try to extract metadata
|
||||||
|
testCancel()
|
||||||
|
}
|
||||||
|
|
||||||
|
d.log.Info("Archive listing successful", "files", len(files))
|
||||||
|
|
||||||
|
// Try full extraction - NO TIMEOUT here as large archives can take a long time
|
||||||
|
// Use a generous timeout (30 minutes) for very large archives
|
||||||
|
extractCtx, extractCancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||||
|
defer extractCancel()
|
||||||
|
|
||||||
|
cmd := exec.CommandContext(extractCtx, "tar", "-xzf", archivePath, "-C", tempDir)
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
cmd.Stderr = &stderr
|
||||||
|
if err := cmd.Run(); err != nil {
|
||||||
|
// Extraction failed
|
||||||
|
errResult := &DiagnoseResult{
|
||||||
|
FilePath: archivePath,
|
||||||
|
FileName: filepath.Base(archivePath),
|
||||||
|
Format: FormatClusterTarGz,
|
||||||
|
DetectedFormat: "Cluster Archive (tar.gz)",
|
||||||
|
IsValid: false,
|
||||||
|
Details: &DiagnoseDetails{},
|
||||||
|
}
|
||||||
|
|
||||||
|
errOutput := stderr.String()
|
||||||
|
if strings.Contains(errOutput, "No space left") ||
|
||||||
|
strings.Contains(errOutput, "cannot write") ||
|
||||||
|
strings.Contains(errOutput, "Disk quota exceeded") {
|
||||||
|
errResult.Errors = append(errResult.Errors,
|
||||||
|
"INSUFFICIENT DISK SPACE to extract archive for diagnosis",
|
||||||
|
fmt.Sprintf("Archive size: %s (needs ~%s for extraction)",
|
||||||
|
formatBytes(archiveInfo.Size()), formatBytes(requiredSpace)),
|
||||||
|
"Use CLI diagnosis instead: dbbackup restore diagnose "+archivePath,
|
||||||
|
"Or use --workdir flag to specify a location with more space")
|
||||||
|
} else if strings.Contains(errOutput, "unexpected end of file") ||
|
||||||
|
strings.Contains(errOutput, "Unexpected EOF") {
|
||||||
|
errResult.IsTruncated = true
|
||||||
|
errResult.IsCorrupted = true
|
||||||
|
errResult.Errors = append(errResult.Errors,
|
||||||
|
"Archive is TRUNCATED - extraction failed mid-way",
|
||||||
|
fmt.Sprintf("Error: %s", truncateString(errOutput, 200)),
|
||||||
|
"The backup file is incomplete and cannot be restored",
|
||||||
|
"Solution: Re-create the backup from source database")
|
||||||
|
} else {
|
||||||
|
errResult.IsCorrupted = true
|
||||||
|
errResult.Errors = append(errResult.Errors,
|
||||||
|
fmt.Sprintf("Extraction failed: %v", err),
|
||||||
|
fmt.Sprintf("tar error: %s", truncateString(errOutput, 300)))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Still report what files we found in the listing
|
||||||
|
var dumpFiles []string
|
||||||
|
for _, f := range files {
|
||||||
|
if strings.HasSuffix(f, ".dump") || strings.HasSuffix(f, ".sql.gz") {
|
||||||
|
dumpFiles = append(dumpFiles, filepath.Base(f))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(dumpFiles) > 0 {
|
||||||
|
errResult.Details.TableList = dumpFiles
|
||||||
|
errResult.Details.TableCount = len(dumpFiles)
|
||||||
|
errResult.Warnings = append(errResult.Warnings,
|
||||||
|
fmt.Sprintf("Archive contains %d database dumps (listing only)", len(dumpFiles)))
|
||||||
|
}
|
||||||
|
|
||||||
|
return []*DiagnoseResult{errResult}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find dump files
|
||||||
|
dumpsDir := filepath.Join(tempDir, "dumps")
|
||||||
|
entries, err := os.ReadDir(dumpsDir)
|
||||||
|
if err != nil {
|
||||||
|
// Try without dumps subdirectory
|
||||||
|
entries, err = os.ReadDir(tempDir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot read extracted files: %w", err)
|
||||||
|
}
|
||||||
|
dumpsDir = tempDir
|
||||||
|
}
|
||||||
|
|
||||||
|
var results []*DiagnoseResult
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
name := entry.Name()
|
||||||
|
if !strings.HasSuffix(name, ".dump") && !strings.HasSuffix(name, ".sql.gz") &&
|
||||||
|
!strings.HasSuffix(name, ".sql") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
dumpPath := filepath.Join(dumpsDir, name)
|
||||||
|
d.log.Info("Diagnosing dump file", "file", name)
|
||||||
|
|
||||||
|
result, err := d.DiagnoseFile(dumpPath)
|
||||||
|
if err != nil {
|
||||||
|
d.log.Warn("Failed to diagnose file", "file", name, "error", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
results = append(results, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
return results, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// PrintDiagnosis outputs a human-readable diagnosis report
|
||||||
|
func (d *Diagnoser) PrintDiagnosis(result *DiagnoseResult) {
|
||||||
|
fmt.Println("\n" + strings.Repeat("=", 70))
|
||||||
|
fmt.Printf("[DIAG] DIAGNOSIS: %s\n", result.FileName)
|
||||||
|
fmt.Println(strings.Repeat("=", 70))
|
||||||
|
|
||||||
|
// Basic info
|
||||||
|
fmt.Printf("\nFile: %s\n", result.FilePath)
|
||||||
|
fmt.Printf("Size: %s\n", formatBytes(result.FileSize))
|
||||||
|
fmt.Printf("Format: %s\n", result.DetectedFormat)
|
||||||
|
|
||||||
|
// Status
|
||||||
|
if result.IsValid {
|
||||||
|
fmt.Println("\n[OK] STATUS: VALID")
|
||||||
|
} else {
|
||||||
|
fmt.Println("\n[FAIL] STATUS: INVALID")
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.IsTruncated {
|
||||||
|
fmt.Println("[WARN] TRUNCATED: Yes - file appears incomplete")
|
||||||
|
}
|
||||||
|
if result.IsCorrupted {
|
||||||
|
fmt.Println("[WARN] CORRUPTED: Yes - file structure is damaged")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Details
|
||||||
|
if result.Details != nil {
|
||||||
|
fmt.Println("\n[DETAILS]:")
|
||||||
|
|
||||||
|
if result.Details.HasPGDMPSignature {
|
||||||
|
fmt.Println(" [+] Has PGDMP signature (PostgreSQL custom format)")
|
||||||
|
}
|
||||||
|
if result.Details.HasSQLHeader {
|
||||||
|
fmt.Println(" [+] Has PostgreSQL SQL header")
|
||||||
|
}
|
||||||
|
if result.Details.GzipValid {
|
||||||
|
fmt.Println(" [+] Gzip compression valid")
|
||||||
|
}
|
||||||
|
if result.Details.PgRestoreListable {
|
||||||
|
fmt.Printf(" [+] pg_restore can list contents (%d tables)\n", result.Details.TableCount)
|
||||||
|
}
|
||||||
|
if result.Details.CopyBlockCount > 0 {
|
||||||
|
fmt.Printf(" [-] Contains %d COPY blocks\n", result.Details.CopyBlockCount)
|
||||||
|
}
|
||||||
|
if result.Details.UnterminatedCopy {
|
||||||
|
fmt.Printf(" [-] Unterminated COPY block: %s (line %d)\n",
|
||||||
|
result.Details.LastCopyTable, result.Details.LastCopyLineNumber)
|
||||||
|
}
|
||||||
|
if result.Details.ProperlyTerminated {
|
||||||
|
fmt.Println(" [+] All COPY blocks properly terminated")
|
||||||
|
}
|
||||||
|
if result.Details.ExpandedSize > 0 {
|
||||||
|
fmt.Printf(" [-] Expanded size: %s (ratio: %.1fx)\n",
|
||||||
|
formatBytes(result.Details.ExpandedSize), result.Details.CompressionRatio)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Errors
|
||||||
|
if len(result.Errors) > 0 {
|
||||||
|
fmt.Println("\n[ERRORS]:")
|
||||||
|
for _, e := range result.Errors {
|
||||||
|
fmt.Printf(" - %s\n", e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Warnings
|
||||||
|
if len(result.Warnings) > 0 {
|
||||||
|
fmt.Println("\n[WARNINGS]:")
|
||||||
|
for _, w := range result.Warnings {
|
||||||
|
fmt.Printf(" - %s\n", w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recommendations
|
||||||
|
if !result.IsValid {
|
||||||
|
fmt.Println("\n[HINT] RECOMMENDATIONS:")
|
||||||
|
if result.IsTruncated {
|
||||||
|
fmt.Println(" 1. Re-run the backup process for this database")
|
||||||
|
fmt.Println(" 2. Check disk space on backup server during backup")
|
||||||
|
fmt.Println(" 3. Verify network stability if backup was remote")
|
||||||
|
fmt.Println(" 4. Check backup logs for errors during the backup")
|
||||||
|
}
|
||||||
|
if result.IsCorrupted {
|
||||||
|
fmt.Println(" 1. Verify backup file was transferred completely")
|
||||||
|
fmt.Println(" 2. Check if backup file was modified after creation")
|
||||||
|
fmt.Println(" 3. Try restoring from a previous backup")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println(strings.Repeat("=", 70))
|
||||||
|
}
|
||||||
|
|
||||||
|
// PrintDiagnosisJSON outputs diagnosis as JSON
|
||||||
|
func (d *Diagnoser) PrintDiagnosisJSON(result *DiagnoseResult) error {
|
||||||
|
output, err := json.MarshalIndent(result, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(string(output))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper functions
|
||||||
|
|
||||||
|
func truncateString(s string, maxLen int) string {
|
||||||
|
if len(s) <= maxLen {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return s[:maxLen-3] + "..."
|
||||||
|
}
|
||||||
|
|
||||||
|
func formatBytes(bytes int64) string {
|
||||||
|
const unit = 1024
|
||||||
|
if bytes < unit {
|
||||||
|
return fmt.Sprintf("%d B", bytes)
|
||||||
|
}
|
||||||
|
div, exp := int64(unit), 0
|
||||||
|
for n := bytes / unit; n >= unit; n /= unit {
|
||||||
|
div *= unit
|
||||||
|
exp++
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
|
||||||
|
}
|
||||||
|
|
||||||
|
func min(a, b int64) int64 {
|
||||||
|
if a < b {
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
func minInt(a, b int) int {
|
||||||
|
if a < b {
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
@@ -27,6 +27,7 @@ type Engine struct {
|
|||||||
progress progress.Indicator
|
progress progress.Indicator
|
||||||
detailedReporter *progress.DetailedReporter
|
detailedReporter *progress.DetailedReporter
|
||||||
dryRun bool
|
dryRun bool
|
||||||
|
debugLogPath string // Path to save debug log on error
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new restore engine
|
// New creates a new restore engine
|
||||||
@@ -77,6 +78,11 @@ func NewWithProgress(cfg *config.Config, log logger.Logger, db database.Database
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetDebugLogPath enables saving detailed error reports on failure
|
||||||
|
func (e *Engine) SetDebugLogPath(path string) {
|
||||||
|
e.debugLogPath = path
|
||||||
|
}
|
||||||
|
|
||||||
// loggerAdapter adapts our logger to the progress.Logger interface
|
// loggerAdapter adapts our logger to the progress.Logger interface
|
||||||
type loggerAdapter struct {
|
type loggerAdapter struct {
|
||||||
logger logger.Logger
|
logger logger.Logger
|
||||||
@@ -121,7 +127,7 @@ func (e *Engine) RestoreSingle(ctx context.Context, archivePath, targetDB string
|
|||||||
e.log.Warn("Checksum verification failed", "error", checksumErr)
|
e.log.Warn("Checksum verification failed", "error", checksumErr)
|
||||||
e.log.Warn("Continuing restore without checksum verification (use with caution)")
|
e.log.Warn("Continuing restore without checksum verification (use with caution)")
|
||||||
} else {
|
} else {
|
||||||
e.log.Info("✓ Archive checksum verified successfully")
|
e.log.Info("[OK] Archive checksum verified successfully")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Detect archive format
|
// Detect archive format
|
||||||
@@ -245,6 +251,15 @@ func (e *Engine) restorePostgreSQLDumpWithOwnership(ctx context.Context, archive
|
|||||||
|
|
||||||
// restorePostgreSQLSQL restores from PostgreSQL SQL script
|
// restorePostgreSQLSQL restores from PostgreSQL SQL script
|
||||||
func (e *Engine) restorePostgreSQLSQL(ctx context.Context, archivePath, targetDB string, compressed bool) error {
|
func (e *Engine) restorePostgreSQLSQL(ctx context.Context, archivePath, targetDB string, compressed bool) error {
|
||||||
|
// Pre-validate SQL dump to detect truncation BEFORE attempting restore
|
||||||
|
// This saves time by catching corrupted files early (vs 49min failures)
|
||||||
|
if err := e.quickValidateSQLDump(archivePath, compressed); err != nil {
|
||||||
|
e.log.Error("Pre-restore validation failed - dump file appears corrupted",
|
||||||
|
"file", archivePath,
|
||||||
|
"error", err)
|
||||||
|
return fmt.Errorf("dump validation failed: %w - the backup file may be truncated or corrupted", err)
|
||||||
|
}
|
||||||
|
|
||||||
// Use psql for SQL scripts
|
// Use psql for SQL scripts
|
||||||
var cmd []string
|
var cmd []string
|
||||||
|
|
||||||
@@ -255,9 +270,10 @@ func (e *Engine) restorePostgreSQLSQL(ctx context.Context, archivePath, targetDB
|
|||||||
}
|
}
|
||||||
|
|
||||||
if compressed {
|
if compressed {
|
||||||
psqlCmd := fmt.Sprintf("psql -U %s -d %s", e.cfg.User, targetDB)
|
// Use ON_ERROR_STOP=1 to fail fast on first error (prevents millions of errors on truncated dumps)
|
||||||
|
psqlCmd := fmt.Sprintf("psql -U %s -d %s -v ON_ERROR_STOP=1", e.cfg.User, targetDB)
|
||||||
if hostArg != "" {
|
if hostArg != "" {
|
||||||
psqlCmd = fmt.Sprintf("psql %s -U %s -d %s", hostArg, e.cfg.User, targetDB)
|
psqlCmd = fmt.Sprintf("psql %s -U %s -d %s -v ON_ERROR_STOP=1", hostArg, e.cfg.User, targetDB)
|
||||||
}
|
}
|
||||||
// Set PGPASSWORD in the bash command for password-less auth
|
// Set PGPASSWORD in the bash command for password-less auth
|
||||||
cmd = []string{
|
cmd = []string{
|
||||||
@@ -272,6 +288,7 @@ func (e *Engine) restorePostgreSQLSQL(ctx context.Context, archivePath, targetDB
|
|||||||
"-p", fmt.Sprintf("%d", e.cfg.Port),
|
"-p", fmt.Sprintf("%d", e.cfg.Port),
|
||||||
"-U", e.cfg.User,
|
"-U", e.cfg.User,
|
||||||
"-d", targetDB,
|
"-d", targetDB,
|
||||||
|
"-v", "ON_ERROR_STOP=1",
|
||||||
"-f", archivePath,
|
"-f", archivePath,
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
@@ -279,6 +296,7 @@ func (e *Engine) restorePostgreSQLSQL(ctx context.Context, archivePath, targetDB
|
|||||||
"psql",
|
"psql",
|
||||||
"-U", e.cfg.User,
|
"-U", e.cfg.User,
|
||||||
"-d", targetDB,
|
"-d", targetDB,
|
||||||
|
"-v", "ON_ERROR_STOP=1",
|
||||||
"-f", archivePath,
|
"-f", archivePath,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -306,6 +324,11 @@ func (e *Engine) restoreMySQLSQL(ctx context.Context, archivePath, targetDB stri
|
|||||||
|
|
||||||
// executeRestoreCommand executes a restore command
|
// executeRestoreCommand executes a restore command
|
||||||
func (e *Engine) executeRestoreCommand(ctx context.Context, cmdArgs []string) error {
|
func (e *Engine) executeRestoreCommand(ctx context.Context, cmdArgs []string) error {
|
||||||
|
return e.executeRestoreCommandWithContext(ctx, cmdArgs, "", "", FormatUnknown)
|
||||||
|
}
|
||||||
|
|
||||||
|
// executeRestoreCommandWithContext executes a restore command with error collection context
|
||||||
|
func (e *Engine) executeRestoreCommandWithContext(ctx context.Context, cmdArgs []string, archivePath, targetDB string, format ArchiveFormat) error {
|
||||||
e.log.Info("Executing restore command", "command", strings.Join(cmdArgs, " "))
|
e.log.Info("Executing restore command", "command", strings.Join(cmdArgs, " "))
|
||||||
|
|
||||||
cmd := exec.CommandContext(ctx, cmdArgs[0], cmdArgs[1:]...)
|
cmd := exec.CommandContext(ctx, cmdArgs[0], cmdArgs[1:]...)
|
||||||
@@ -316,6 +339,12 @@ func (e *Engine) executeRestoreCommand(ctx context.Context, cmdArgs []string) er
|
|||||||
fmt.Sprintf("MYSQL_PWD=%s", e.cfg.Password),
|
fmt.Sprintf("MYSQL_PWD=%s", e.cfg.Password),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Create error collector if debug log path is set
|
||||||
|
var collector *ErrorCollector
|
||||||
|
if e.debugLogPath != "" {
|
||||||
|
collector = NewErrorCollector(e.cfg, e.log, archivePath, targetDB, format, true)
|
||||||
|
}
|
||||||
|
|
||||||
// Stream stderr to avoid memory issues with large output
|
// Stream stderr to avoid memory issues with large output
|
||||||
// Don't use CombinedOutput() as it loads everything into memory
|
// Don't use CombinedOutput() as it loads everything into memory
|
||||||
stderr, err := cmd.StderrPipe()
|
stderr, err := cmd.StderrPipe()
|
||||||
@@ -327,31 +356,68 @@ func (e *Engine) executeRestoreCommand(ctx context.Context, cmdArgs []string) er
|
|||||||
return fmt.Errorf("failed to start restore command: %w", err)
|
return fmt.Errorf("failed to start restore command: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read stderr in chunks to log errors without loading all into memory
|
// Read stderr in goroutine to avoid blocking
|
||||||
buf := make([]byte, 4096)
|
|
||||||
var lastError string
|
var lastError string
|
||||||
var errorCount int
|
var errorCount int
|
||||||
const maxErrors = 10 // Limit captured errors to prevent OOM
|
stderrDone := make(chan struct{})
|
||||||
for {
|
go func() {
|
||||||
n, err := stderr.Read(buf)
|
defer close(stderrDone)
|
||||||
if n > 0 {
|
buf := make([]byte, 4096)
|
||||||
chunk := string(buf[:n])
|
const maxErrors = 10 // Limit captured errors to prevent OOM
|
||||||
// Only capture REAL errors, not verbose output
|
for {
|
||||||
if strings.Contains(chunk, "ERROR:") || strings.Contains(chunk, "FATAL:") || strings.Contains(chunk, "error:") {
|
n, err := stderr.Read(buf)
|
||||||
lastError = strings.TrimSpace(chunk)
|
if n > 0 {
|
||||||
errorCount++
|
chunk := string(buf[:n])
|
||||||
if errorCount <= maxErrors {
|
|
||||||
e.log.Warn("Restore stderr", "output", chunk)
|
// Feed to error collector if enabled
|
||||||
|
if collector != nil {
|
||||||
|
collector.CaptureStderr(chunk)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Only capture REAL errors, not verbose output
|
||||||
|
if strings.Contains(chunk, "ERROR:") || strings.Contains(chunk, "FATAL:") || strings.Contains(chunk, "error:") {
|
||||||
|
lastError = strings.TrimSpace(chunk)
|
||||||
|
errorCount++
|
||||||
|
if errorCount <= maxErrors {
|
||||||
|
e.log.Warn("Restore stderr", "output", chunk)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Note: --verbose output is discarded to prevent OOM
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
break
|
||||||
}
|
}
|
||||||
// Note: --verbose output is discarded to prevent OOM
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
break
|
|
||||||
}
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Wait for command with proper context handling
|
||||||
|
cmdDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
cmdDone <- cmd.Wait()
|
||||||
|
}()
|
||||||
|
|
||||||
|
var cmdErr error
|
||||||
|
select {
|
||||||
|
case cmdErr = <-cmdDone:
|
||||||
|
// Command completed (success or failure)
|
||||||
|
case <-ctx.Done():
|
||||||
|
// Context cancelled - kill process
|
||||||
|
e.log.Warn("Restore cancelled - killing process")
|
||||||
|
cmd.Process.Kill()
|
||||||
|
<-cmdDone
|
||||||
|
cmdErr = ctx.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := cmd.Wait(); err != nil {
|
// Wait for stderr reader to finish
|
||||||
|
<-stderrDone
|
||||||
|
|
||||||
|
if cmdErr != nil {
|
||||||
|
// Get exit code
|
||||||
|
exitCode := 1
|
||||||
|
if exitErr, ok := cmdErr.(*exec.ExitError); ok {
|
||||||
|
exitCode = exitErr.ExitCode()
|
||||||
|
}
|
||||||
|
|
||||||
// PostgreSQL pg_restore returns exit code 1 even for ignorable errors
|
// PostgreSQL pg_restore returns exit code 1 even for ignorable errors
|
||||||
// Check if errors are ignorable (already exists, duplicate, etc.)
|
// Check if errors are ignorable (already exists, duplicate, etc.)
|
||||||
if lastError != "" && e.isIgnorableError(lastError) {
|
if lastError != "" && e.isIgnorableError(lastError) {
|
||||||
@@ -360,8 +426,12 @@ func (e *Engine) executeRestoreCommand(ctx context.Context, cmdArgs []string) er
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Classify error and provide helpful hints
|
// Classify error and provide helpful hints
|
||||||
|
var classification *checks.ErrorClassification
|
||||||
|
var errType, errHint string
|
||||||
if lastError != "" {
|
if lastError != "" {
|
||||||
classification := checks.ClassifyError(lastError)
|
classification = checks.ClassifyError(lastError)
|
||||||
|
errType = classification.Type
|
||||||
|
errHint = classification.Hint
|
||||||
e.log.Error("Restore command failed",
|
e.log.Error("Restore command failed",
|
||||||
"error", err,
|
"error", err,
|
||||||
"last_stderr", lastError,
|
"last_stderr", lastError,
|
||||||
@@ -369,11 +439,37 @@ func (e *Engine) executeRestoreCommand(ctx context.Context, cmdArgs []string) er
|
|||||||
"error_type", classification.Type,
|
"error_type", classification.Type,
|
||||||
"hint", classification.Hint,
|
"hint", classification.Hint,
|
||||||
"action", classification.Action)
|
"action", classification.Action)
|
||||||
return fmt.Errorf("restore failed: %w (last error: %s, total errors: %d) - %s",
|
} else {
|
||||||
err, lastError, errorCount, classification.Hint)
|
e.log.Error("Restore command failed", "error", err, "error_count", errorCount)
|
||||||
}
|
}
|
||||||
|
|
||||||
e.log.Error("Restore command failed", "error", err, "last_stderr", lastError, "error_count", errorCount)
|
// Generate and save error report if collector is enabled
|
||||||
|
if collector != nil {
|
||||||
|
collector.SetExitCode(exitCode)
|
||||||
|
report := collector.GenerateReport(
|
||||||
|
lastError,
|
||||||
|
errType,
|
||||||
|
errHint,
|
||||||
|
)
|
||||||
|
|
||||||
|
// Print report to console
|
||||||
|
collector.PrintReport(report)
|
||||||
|
|
||||||
|
// Save to file
|
||||||
|
if e.debugLogPath != "" {
|
||||||
|
if saveErr := collector.SaveReport(report, e.debugLogPath); saveErr != nil {
|
||||||
|
e.log.Warn("Failed to save debug log", "error", saveErr)
|
||||||
|
} else {
|
||||||
|
e.log.Info("Debug log saved", "path", e.debugLogPath)
|
||||||
|
fmt.Printf("\n[LOG] Detailed error report saved to: %s\n", e.debugLogPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if lastError != "" {
|
||||||
|
return fmt.Errorf("restore failed: %w (last error: %s, total errors: %d) - %s",
|
||||||
|
err, lastError, errorCount, errHint)
|
||||||
|
}
|
||||||
return fmt.Errorf("restore failed: %w", err)
|
return fmt.Errorf("restore failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -409,31 +505,56 @@ func (e *Engine) executeRestoreWithDecompression(ctx context.Context, archivePat
|
|||||||
return fmt.Errorf("failed to start restore command: %w", err)
|
return fmt.Errorf("failed to start restore command: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read stderr in chunks to log errors without loading all into memory
|
// Read stderr in goroutine to avoid blocking
|
||||||
buf := make([]byte, 4096)
|
|
||||||
var lastError string
|
var lastError string
|
||||||
var errorCount int
|
var errorCount int
|
||||||
const maxErrors = 10 // Limit captured errors to prevent OOM
|
stderrDone := make(chan struct{})
|
||||||
for {
|
go func() {
|
||||||
n, err := stderr.Read(buf)
|
defer close(stderrDone)
|
||||||
if n > 0 {
|
buf := make([]byte, 4096)
|
||||||
chunk := string(buf[:n])
|
const maxErrors = 10 // Limit captured errors to prevent OOM
|
||||||
// Only capture REAL errors, not verbose output
|
for {
|
||||||
if strings.Contains(chunk, "ERROR:") || strings.Contains(chunk, "FATAL:") || strings.Contains(chunk, "error:") {
|
n, err := stderr.Read(buf)
|
||||||
lastError = strings.TrimSpace(chunk)
|
if n > 0 {
|
||||||
errorCount++
|
chunk := string(buf[:n])
|
||||||
if errorCount <= maxErrors {
|
// Only capture REAL errors, not verbose output
|
||||||
e.log.Warn("Restore stderr", "output", chunk)
|
if strings.Contains(chunk, "ERROR:") || strings.Contains(chunk, "FATAL:") || strings.Contains(chunk, "error:") {
|
||||||
|
lastError = strings.TrimSpace(chunk)
|
||||||
|
errorCount++
|
||||||
|
if errorCount <= maxErrors {
|
||||||
|
e.log.Warn("Restore stderr", "output", chunk)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
// Note: --verbose output is discarded to prevent OOM
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
break
|
||||||
}
|
}
|
||||||
// Note: --verbose output is discarded to prevent OOM
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
break
|
|
||||||
}
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Wait for command with proper context handling
|
||||||
|
cmdDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
cmdDone <- cmd.Wait()
|
||||||
|
}()
|
||||||
|
|
||||||
|
var cmdErr error
|
||||||
|
select {
|
||||||
|
case cmdErr = <-cmdDone:
|
||||||
|
// Command completed (success or failure)
|
||||||
|
case <-ctx.Done():
|
||||||
|
// Context cancelled - kill process
|
||||||
|
e.log.Warn("Restore with decompression cancelled - killing process")
|
||||||
|
cmd.Process.Kill()
|
||||||
|
<-cmdDone
|
||||||
|
cmdErr = ctx.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := cmd.Wait(); err != nil {
|
// Wait for stderr reader to finish
|
||||||
|
<-stderrDone
|
||||||
|
|
||||||
|
if cmdErr != nil {
|
||||||
// PostgreSQL pg_restore returns exit code 1 even for ignorable errors
|
// PostgreSQL pg_restore returns exit code 1 even for ignorable errors
|
||||||
// Check if errors are ignorable (already exists, duplicate, etc.)
|
// Check if errors are ignorable (already exists, duplicate, etc.)
|
||||||
if lastError != "" && e.isIgnorableError(lastError) {
|
if lastError != "" && e.isIgnorableError(lastError) {
|
||||||
@@ -445,18 +566,18 @@ func (e *Engine) executeRestoreWithDecompression(ctx context.Context, archivePat
|
|||||||
if lastError != "" {
|
if lastError != "" {
|
||||||
classification := checks.ClassifyError(lastError)
|
classification := checks.ClassifyError(lastError)
|
||||||
e.log.Error("Restore with decompression failed",
|
e.log.Error("Restore with decompression failed",
|
||||||
"error", err,
|
"error", cmdErr,
|
||||||
"last_stderr", lastError,
|
"last_stderr", lastError,
|
||||||
"error_count", errorCount,
|
"error_count", errorCount,
|
||||||
"error_type", classification.Type,
|
"error_type", classification.Type,
|
||||||
"hint", classification.Hint,
|
"hint", classification.Hint,
|
||||||
"action", classification.Action)
|
"action", classification.Action)
|
||||||
return fmt.Errorf("restore failed: %w (last error: %s, total errors: %d) - %s",
|
return fmt.Errorf("restore failed: %w (last error: %s, total errors: %d) - %s",
|
||||||
err, lastError, errorCount, classification.Hint)
|
cmdErr, lastError, errorCount, classification.Hint)
|
||||||
}
|
}
|
||||||
|
|
||||||
e.log.Error("Restore with decompression failed", "error", err, "last_stderr", lastError, "error_count", errorCount)
|
e.log.Error("Restore with decompression failed", "error", cmdErr, "last_stderr", lastError, "error_count", errorCount)
|
||||||
return fmt.Errorf("restore failed: %w", err)
|
return fmt.Errorf("restore failed: %w", cmdErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -491,7 +612,7 @@ func (e *Engine) previewRestore(archivePath, targetDB string, format ArchiveForm
|
|||||||
fmt.Printf(" 1. Execute: mysql %s < %s\n", targetDB, archivePath)
|
fmt.Printf(" 1. Execute: mysql %s < %s\n", targetDB, archivePath)
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println("\n⚠️ WARNING: This will restore data to the target database.")
|
fmt.Println("\n[WARN] WARNING: This will restore data to the target database.")
|
||||||
fmt.Println(" Existing data may be overwritten or merged.")
|
fmt.Println(" Existing data may be overwritten or merged.")
|
||||||
fmt.Println("\nTo execute this restore, add the --confirm flag.")
|
fmt.Println("\nTo execute this restore, add the --confirm flag.")
|
||||||
fmt.Println(strings.Repeat("=", 60) + "\n")
|
fmt.Println(strings.Repeat("=", 60) + "\n")
|
||||||
@@ -522,7 +643,7 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
|
|||||||
e.log.Warn("Checksum verification failed", "error", checksumErr)
|
e.log.Warn("Checksum verification failed", "error", checksumErr)
|
||||||
e.log.Warn("Continuing restore without checksum verification (use with caution)")
|
e.log.Warn("Continuing restore without checksum verification (use with caution)")
|
||||||
} else {
|
} else {
|
||||||
e.log.Info("✓ Cluster archive checksum verified successfully")
|
e.log.Info("[OK] Cluster archive checksum verified successfully")
|
||||||
}
|
}
|
||||||
|
|
||||||
format := DetectArchiveFormat(archivePath)
|
format := DetectArchiveFormat(archivePath)
|
||||||
@@ -556,11 +677,12 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
|
|||||||
|
|
||||||
e.progress.Start(fmt.Sprintf("Restoring cluster from %s", filepath.Base(archivePath)))
|
e.progress.Start(fmt.Sprintf("Restoring cluster from %s", filepath.Base(archivePath)))
|
||||||
|
|
||||||
// Create temporary extraction directory
|
// Create temporary extraction directory in configured WorkDir
|
||||||
tempDir := filepath.Join(e.cfg.BackupDir, fmt.Sprintf(".restore_%d", time.Now().Unix()))
|
workDir := e.cfg.GetEffectiveWorkDir()
|
||||||
|
tempDir := filepath.Join(workDir, fmt.Sprintf(".restore_%d", time.Now().Unix()))
|
||||||
if err := os.MkdirAll(tempDir, 0755); err != nil {
|
if err := os.MkdirAll(tempDir, 0755); err != nil {
|
||||||
operation.Fail("Failed to create temporary directory")
|
operation.Fail("Failed to create temporary directory")
|
||||||
return fmt.Errorf("failed to create temp directory: %w", err)
|
return fmt.Errorf("failed to create temp directory in %s: %w", workDir, err)
|
||||||
}
|
}
|
||||||
defer os.RemoveAll(tempDir)
|
defer os.RemoveAll(tempDir)
|
||||||
|
|
||||||
@@ -581,7 +703,7 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
|
|||||||
|
|
||||||
if !isSuperuser {
|
if !isSuperuser {
|
||||||
e.log.Warn("Current user is not a superuser - database ownership may not be fully restored")
|
e.log.Warn("Current user is not a superuser - database ownership may not be fully restored")
|
||||||
e.progress.Update("⚠️ Warning: Non-superuser - ownership restoration limited")
|
e.progress.Update("[WARN] Warning: Non-superuser - ownership restoration limited")
|
||||||
time.Sleep(2 * time.Second) // Give user time to see warning
|
time.Sleep(2 * time.Second) // Give user time to see warning
|
||||||
} else {
|
} else {
|
||||||
e.log.Info("Superuser privileges confirmed - full ownership restoration enabled")
|
e.log.Info("Superuser privileges confirmed - full ownership restoration enabled")
|
||||||
@@ -622,6 +744,69 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
|
|||||||
return fmt.Errorf("failed to read dumps directory: %w", err)
|
return fmt.Errorf("failed to read dumps directory: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PRE-VALIDATE all SQL dumps BEFORE starting restore
|
||||||
|
// This catches truncated files early instead of failing after hours of work
|
||||||
|
e.log.Info("Pre-validating dump files before restore...")
|
||||||
|
e.progress.Update("Pre-validating dump files...")
|
||||||
|
var corruptedDumps []string
|
||||||
|
diagnoser := NewDiagnoser(e.log, false)
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
dumpFile := filepath.Join(dumpsDir, entry.Name())
|
||||||
|
if strings.HasSuffix(dumpFile, ".sql.gz") {
|
||||||
|
result, err := diagnoser.DiagnoseFile(dumpFile)
|
||||||
|
if err != nil {
|
||||||
|
e.log.Warn("Could not validate dump file", "file", entry.Name(), "error", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if result.IsTruncated || result.IsCorrupted || !result.IsValid {
|
||||||
|
dbName := strings.TrimSuffix(entry.Name(), ".sql.gz")
|
||||||
|
errDetail := "unknown issue"
|
||||||
|
if len(result.Errors) > 0 {
|
||||||
|
errDetail = result.Errors[0]
|
||||||
|
}
|
||||||
|
corruptedDumps = append(corruptedDumps, fmt.Sprintf("%s: %s", dbName, errDetail))
|
||||||
|
e.log.Error("CORRUPTED dump file detected",
|
||||||
|
"database", dbName,
|
||||||
|
"file", entry.Name(),
|
||||||
|
"truncated", result.IsTruncated,
|
||||||
|
"errors", result.Errors)
|
||||||
|
}
|
||||||
|
} else if strings.HasSuffix(dumpFile, ".dump") {
|
||||||
|
// Validate custom format dumps using pg_restore --list
|
||||||
|
cmd := exec.CommandContext(ctx, "pg_restore", "--list", dumpFile)
|
||||||
|
output, err := cmd.CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
dbName := strings.TrimSuffix(entry.Name(), ".dump")
|
||||||
|
errDetail := strings.TrimSpace(string(output))
|
||||||
|
if len(errDetail) > 100 {
|
||||||
|
errDetail = errDetail[:100] + "..."
|
||||||
|
}
|
||||||
|
// Check for truncation indicators
|
||||||
|
if strings.Contains(errDetail, "unexpected end") || strings.Contains(errDetail, "invalid") {
|
||||||
|
corruptedDumps = append(corruptedDumps, fmt.Sprintf("%s: %s", dbName, errDetail))
|
||||||
|
e.log.Error("CORRUPTED custom dump file detected",
|
||||||
|
"database", dbName,
|
||||||
|
"file", entry.Name(),
|
||||||
|
"error", errDetail)
|
||||||
|
} else {
|
||||||
|
e.log.Warn("pg_restore --list warning (may be recoverable)",
|
||||||
|
"file", entry.Name(),
|
||||||
|
"error", errDetail)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(corruptedDumps) > 0 {
|
||||||
|
operation.Fail("Corrupted dump files detected")
|
||||||
|
e.progress.Fail(fmt.Sprintf("Found %d corrupted dump files - restore aborted", len(corruptedDumps)))
|
||||||
|
return fmt.Errorf("pre-validation failed: %d corrupted dump files detected: %s - the backup archive appears to be damaged, restore from a different backup",
|
||||||
|
len(corruptedDumps), strings.Join(corruptedDumps, ", "))
|
||||||
|
}
|
||||||
|
e.log.Info("All dump files passed validation")
|
||||||
|
|
||||||
var failedDBs []string
|
var failedDBs []string
|
||||||
totalDBs := 0
|
totalDBs := 0
|
||||||
|
|
||||||
@@ -650,7 +835,7 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
|
|||||||
e.log.Warn("Large objects detected in dump files - reducing parallelism to avoid lock contention",
|
e.log.Warn("Large objects detected in dump files - reducing parallelism to avoid lock contention",
|
||||||
"original_parallelism", parallelism,
|
"original_parallelism", parallelism,
|
||||||
"adjusted_parallelism", 1)
|
"adjusted_parallelism", 1)
|
||||||
e.progress.Update("⚠️ Large objects detected - using sequential restore to avoid lock conflicts")
|
e.progress.Update("[WARN] Large objects detected - using sequential restore to avoid lock conflicts")
|
||||||
time.Sleep(2 * time.Second) // Give user time to see warning
|
time.Sleep(2 * time.Second) // Give user time to see warning
|
||||||
parallelism = 1
|
parallelism = 1
|
||||||
}
|
}
|
||||||
@@ -676,6 +861,14 @@ func (e *Engine) RestoreCluster(ctx context.Context, archivePath string) error {
|
|||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
defer func() { <-semaphore }() // Release
|
defer func() { <-semaphore }() // Release
|
||||||
|
|
||||||
|
// Panic recovery - prevent one database failure from crashing entire cluster restore
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
e.log.Error("Panic in database restore goroutine", "file", filename, "panic", r)
|
||||||
|
atomic.AddInt32(&failCount, 1)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
// Update estimator progress (thread-safe)
|
// Update estimator progress (thread-safe)
|
||||||
mu.Lock()
|
mu.Lock()
|
||||||
estimator.UpdateProgress(idx)
|
estimator.UpdateProgress(idx)
|
||||||
@@ -803,16 +996,39 @@ func (e *Engine) extractArchive(ctx context.Context, archivePath, destDir string
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Discard stderr output in chunks to prevent memory buildup
|
// Discard stderr output in chunks to prevent memory buildup
|
||||||
buf := make([]byte, 4096)
|
stderrDone := make(chan struct{})
|
||||||
for {
|
go func() {
|
||||||
_, err := stderr.Read(buf)
|
defer close(stderrDone)
|
||||||
if err != nil {
|
buf := make([]byte, 4096)
|
||||||
break
|
for {
|
||||||
|
_, err := stderr.Read(buf)
|
||||||
|
if err != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Wait for command with proper context handling
|
||||||
|
cmdDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
cmdDone <- cmd.Wait()
|
||||||
|
}()
|
||||||
|
|
||||||
|
var cmdErr error
|
||||||
|
select {
|
||||||
|
case cmdErr = <-cmdDone:
|
||||||
|
// Command completed
|
||||||
|
case <-ctx.Done():
|
||||||
|
e.log.Warn("Archive extraction cancelled - killing process")
|
||||||
|
cmd.Process.Kill()
|
||||||
|
<-cmdDone
|
||||||
|
cmdErr = ctx.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := cmd.Wait(); err != nil {
|
<-stderrDone
|
||||||
return fmt.Errorf("tar extraction failed: %w", err)
|
|
||||||
|
if cmdErr != nil {
|
||||||
|
return fmt.Errorf("tar extraction failed: %w", cmdErr)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -845,25 +1061,48 @@ func (e *Engine) restoreGlobals(ctx context.Context, globalsFile string) error {
|
|||||||
return fmt.Errorf("failed to start psql: %w", err)
|
return fmt.Errorf("failed to start psql: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read stderr in chunks
|
// Read stderr in chunks in goroutine
|
||||||
buf := make([]byte, 4096)
|
|
||||||
var lastError string
|
var lastError string
|
||||||
for {
|
stderrDone := make(chan struct{})
|
||||||
n, err := stderr.Read(buf)
|
go func() {
|
||||||
if n > 0 {
|
defer close(stderrDone)
|
||||||
chunk := string(buf[:n])
|
buf := make([]byte, 4096)
|
||||||
if strings.Contains(chunk, "ERROR") || strings.Contains(chunk, "FATAL") {
|
for {
|
||||||
lastError = chunk
|
n, err := stderr.Read(buf)
|
||||||
e.log.Warn("Globals restore stderr", "output", chunk)
|
if n > 0 {
|
||||||
|
chunk := string(buf[:n])
|
||||||
|
if strings.Contains(chunk, "ERROR") || strings.Contains(chunk, "FATAL") {
|
||||||
|
lastError = chunk
|
||||||
|
e.log.Warn("Globals restore stderr", "output", chunk)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err != nil {
|
}()
|
||||||
break
|
|
||||||
}
|
// Wait for command with proper context handling
|
||||||
|
cmdDone := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
cmdDone <- cmd.Wait()
|
||||||
|
}()
|
||||||
|
|
||||||
|
var cmdErr error
|
||||||
|
select {
|
||||||
|
case cmdErr = <-cmdDone:
|
||||||
|
// Command completed
|
||||||
|
case <-ctx.Done():
|
||||||
|
e.log.Warn("Globals restore cancelled - killing process")
|
||||||
|
cmd.Process.Kill()
|
||||||
|
<-cmdDone
|
||||||
|
cmdErr = ctx.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := cmd.Wait(); err != nil {
|
<-stderrDone
|
||||||
return fmt.Errorf("failed to restore globals: %w (last error: %s)", err, lastError)
|
|
||||||
|
if cmdErr != nil {
|
||||||
|
return fmt.Errorf("failed to restore globals: %w (last error: %s)", cmdErr, lastError)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -1100,7 +1339,7 @@ func (e *Engine) previewClusterRestore(archivePath string) error {
|
|||||||
fmt.Println(" 3. Restore all databases found in archive")
|
fmt.Println(" 3. Restore all databases found in archive")
|
||||||
fmt.Println(" 4. Cleanup temporary files")
|
fmt.Println(" 4. Cleanup temporary files")
|
||||||
|
|
||||||
fmt.Println("\n⚠️ WARNING: This will restore multiple databases.")
|
fmt.Println("\n[WARN] WARNING: This will restore multiple databases.")
|
||||||
fmt.Println(" Existing databases may be overwritten or merged.")
|
fmt.Println(" Existing databases may be overwritten or merged.")
|
||||||
fmt.Println("\nTo execute this restore, add the --confirm flag.")
|
fmt.Println("\nTo execute this restore, add the --confirm flag.")
|
||||||
fmt.Println(strings.Repeat("=", 60) + "\n")
|
fmt.Println(strings.Repeat("=", 60) + "\n")
|
||||||
@@ -1127,7 +1366,8 @@ func (e *Engine) detectLargeObjectsInDumps(dumpsDir string, entries []os.DirEntr
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Use pg_restore -l to list contents (fast, doesn't restore data)
|
// Use pg_restore -l to list contents (fast, doesn't restore data)
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
// 2 minutes for large dumps with many objects
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
cmd := exec.CommandContext(ctx, "pg_restore", "-l", dumpFile)
|
cmd := exec.CommandContext(ctx, "pg_restore", "-l", dumpFile)
|
||||||
@@ -1214,3 +1454,48 @@ func FormatBytes(bytes int64) string {
|
|||||||
}
|
}
|
||||||
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
|
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// quickValidateSQLDump performs a fast validation of SQL dump files
|
||||||
|
// by checking for truncated COPY blocks. This catches corrupted dumps
|
||||||
|
// BEFORE attempting a full restore (which could waste 49+ minutes).
|
||||||
|
func (e *Engine) quickValidateSQLDump(archivePath string, compressed bool) error {
|
||||||
|
e.log.Debug("Pre-validating SQL dump file", "path", archivePath, "compressed", compressed)
|
||||||
|
|
||||||
|
diagnoser := NewDiagnoser(e.log, false) // non-verbose for speed
|
||||||
|
result, err := diagnoser.DiagnoseFile(archivePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("diagnosis error: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for critical issues that would cause restore failure
|
||||||
|
if result.IsTruncated {
|
||||||
|
errMsg := "SQL dump file is TRUNCATED"
|
||||||
|
if result.Details != nil && result.Details.UnterminatedCopy {
|
||||||
|
errMsg = fmt.Sprintf("%s - unterminated COPY block for table '%s' at line %d",
|
||||||
|
errMsg, result.Details.LastCopyTable, result.Details.LastCopyLineNumber)
|
||||||
|
if len(result.Details.SampleCopyData) > 0 {
|
||||||
|
errMsg = fmt.Sprintf("%s (sample orphaned data: %s)", errMsg, result.Details.SampleCopyData[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%s", errMsg)
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.IsCorrupted {
|
||||||
|
return fmt.Errorf("SQL dump file is corrupted: %v", result.Errors)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !result.IsValid {
|
||||||
|
if len(result.Errors) > 0 {
|
||||||
|
return fmt.Errorf("dump validation failed: %s", result.Errors[0])
|
||||||
|
}
|
||||||
|
return fmt.Errorf("dump file is invalid (unknown reason)")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Log any warnings but don't fail
|
||||||
|
for _, warning := range result.Warnings {
|
||||||
|
e.log.Warn("Dump validation warning", "warning", warning)
|
||||||
|
}
|
||||||
|
|
||||||
|
e.log.Debug("SQL dump validation passed", "path", archivePath)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
574
internal/restore/error_report.go
Normal file
574
internal/restore/error_report.go
Normal file
@@ -0,0 +1,574 @@
|
|||||||
|
package restore
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"compress/gzip"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"dbbackup/internal/config"
|
||||||
|
"dbbackup/internal/logger"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RestoreErrorReport contains comprehensive information about a restore failure
|
||||||
|
type RestoreErrorReport struct {
|
||||||
|
// Metadata
|
||||||
|
Timestamp time.Time `json:"timestamp"`
|
||||||
|
Version string `json:"version"`
|
||||||
|
GoVersion string `json:"go_version"`
|
||||||
|
OS string `json:"os"`
|
||||||
|
Arch string `json:"arch"`
|
||||||
|
|
||||||
|
// Archive info
|
||||||
|
ArchivePath string `json:"archive_path"`
|
||||||
|
ArchiveSize int64 `json:"archive_size"`
|
||||||
|
ArchiveFormat string `json:"archive_format"`
|
||||||
|
|
||||||
|
// Database info
|
||||||
|
TargetDB string `json:"target_db"`
|
||||||
|
DatabaseType string `json:"database_type"`
|
||||||
|
|
||||||
|
// Error details
|
||||||
|
ExitCode int `json:"exit_code"`
|
||||||
|
ErrorMessage string `json:"error_message"`
|
||||||
|
ErrorType string `json:"error_type"`
|
||||||
|
ErrorHint string `json:"error_hint"`
|
||||||
|
TotalErrors int `json:"total_errors"`
|
||||||
|
|
||||||
|
// Captured output
|
||||||
|
LastStderr []string `json:"last_stderr"`
|
||||||
|
FirstErrors []string `json:"first_errors"`
|
||||||
|
|
||||||
|
// Context around failure
|
||||||
|
FailureContext *FailureContext `json:"failure_context,omitempty"`
|
||||||
|
|
||||||
|
// Diagnosis results
|
||||||
|
DiagnosisResult *DiagnoseResult `json:"diagnosis_result,omitempty"`
|
||||||
|
|
||||||
|
// Environment (sanitized)
|
||||||
|
PostgresVersion string `json:"postgres_version,omitempty"`
|
||||||
|
PgRestoreVersion string `json:"pg_restore_version,omitempty"`
|
||||||
|
PsqlVersion string `json:"psql_version,omitempty"`
|
||||||
|
|
||||||
|
// Recommendations
|
||||||
|
Recommendations []string `json:"recommendations"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// FailureContext captures context around where the failure occurred
|
||||||
|
type FailureContext struct {
|
||||||
|
// For SQL/COPY errors
|
||||||
|
FailedLine int `json:"failed_line,omitempty"`
|
||||||
|
FailedStatement string `json:"failed_statement,omitempty"`
|
||||||
|
SurroundingLines []string `json:"surrounding_lines,omitempty"`
|
||||||
|
|
||||||
|
// For COPY block errors
|
||||||
|
InCopyBlock bool `json:"in_copy_block,omitempty"`
|
||||||
|
CopyTableName string `json:"copy_table_name,omitempty"`
|
||||||
|
CopyStartLine int `json:"copy_start_line,omitempty"`
|
||||||
|
SampleCopyData []string `json:"sample_copy_data,omitempty"`
|
||||||
|
|
||||||
|
// File position info
|
||||||
|
BytePosition int64 `json:"byte_position,omitempty"`
|
||||||
|
PercentComplete float64 `json:"percent_complete,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrorCollector captures detailed error information during restore
|
||||||
|
type ErrorCollector struct {
|
||||||
|
log logger.Logger
|
||||||
|
cfg *config.Config
|
||||||
|
archivePath string
|
||||||
|
targetDB string
|
||||||
|
format ArchiveFormat
|
||||||
|
|
||||||
|
// Captured data
|
||||||
|
stderrLines []string
|
||||||
|
firstErrors []string
|
||||||
|
lastErrors []string
|
||||||
|
totalErrors int
|
||||||
|
exitCode int
|
||||||
|
|
||||||
|
// Limits
|
||||||
|
maxStderrLines int
|
||||||
|
maxErrorCapture int
|
||||||
|
|
||||||
|
// State
|
||||||
|
startTime time.Time
|
||||||
|
enabled bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewErrorCollector creates a new error collector
|
||||||
|
func NewErrorCollector(cfg *config.Config, log logger.Logger, archivePath, targetDB string, format ArchiveFormat, enabled bool) *ErrorCollector {
|
||||||
|
return &ErrorCollector{
|
||||||
|
log: log,
|
||||||
|
cfg: cfg,
|
||||||
|
archivePath: archivePath,
|
||||||
|
targetDB: targetDB,
|
||||||
|
format: format,
|
||||||
|
stderrLines: make([]string, 0, 100),
|
||||||
|
firstErrors: make([]string, 0, 10),
|
||||||
|
lastErrors: make([]string, 0, 10),
|
||||||
|
maxStderrLines: 100,
|
||||||
|
maxErrorCapture: 10,
|
||||||
|
startTime: time.Now(),
|
||||||
|
enabled: enabled,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// CaptureStderr processes and captures stderr output
|
||||||
|
func (ec *ErrorCollector) CaptureStderr(chunk string) {
|
||||||
|
if !ec.enabled {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
lines := strings.Split(chunk, "\n")
|
||||||
|
for _, line := range lines {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store last N lines of stderr
|
||||||
|
if len(ec.stderrLines) >= ec.maxStderrLines {
|
||||||
|
// Shift array, drop oldest
|
||||||
|
ec.stderrLines = ec.stderrLines[1:]
|
||||||
|
}
|
||||||
|
ec.stderrLines = append(ec.stderrLines, line)
|
||||||
|
|
||||||
|
// Check if this is an error line
|
||||||
|
if isErrorLine(line) {
|
||||||
|
ec.totalErrors++
|
||||||
|
|
||||||
|
// Capture first N errors
|
||||||
|
if len(ec.firstErrors) < ec.maxErrorCapture {
|
||||||
|
ec.firstErrors = append(ec.firstErrors, line)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keep last N errors (ring buffer style)
|
||||||
|
if len(ec.lastErrors) >= ec.maxErrorCapture {
|
||||||
|
ec.lastErrors = ec.lastErrors[1:]
|
||||||
|
}
|
||||||
|
ec.lastErrors = append(ec.lastErrors, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetExitCode records the exit code
|
||||||
|
func (ec *ErrorCollector) SetExitCode(code int) {
|
||||||
|
ec.exitCode = code
|
||||||
|
}
|
||||||
|
|
||||||
|
// GenerateReport creates a comprehensive error report
|
||||||
|
func (ec *ErrorCollector) GenerateReport(errMessage string, errType string, errHint string) *RestoreErrorReport {
|
||||||
|
report := &RestoreErrorReport{
|
||||||
|
Timestamp: time.Now(),
|
||||||
|
Version: "1.0.0", // TODO: inject actual version
|
||||||
|
GoVersion: runtime.Version(),
|
||||||
|
OS: runtime.GOOS,
|
||||||
|
Arch: runtime.GOARCH,
|
||||||
|
ArchivePath: ec.archivePath,
|
||||||
|
ArchiveFormat: ec.format.String(),
|
||||||
|
TargetDB: ec.targetDB,
|
||||||
|
DatabaseType: getDatabaseType(ec.format),
|
||||||
|
ExitCode: ec.exitCode,
|
||||||
|
ErrorMessage: errMessage,
|
||||||
|
ErrorType: errType,
|
||||||
|
ErrorHint: errHint,
|
||||||
|
TotalErrors: ec.totalErrors,
|
||||||
|
LastStderr: ec.stderrLines,
|
||||||
|
FirstErrors: ec.firstErrors,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get archive size
|
||||||
|
if stat, err := os.Stat(ec.archivePath); err == nil {
|
||||||
|
report.ArchiveSize = stat.Size()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get tool versions
|
||||||
|
report.PostgresVersion = getCommandVersion("postgres", "--version")
|
||||||
|
report.PgRestoreVersion = getCommandVersion("pg_restore", "--version")
|
||||||
|
report.PsqlVersion = getCommandVersion("psql", "--version")
|
||||||
|
|
||||||
|
// Analyze failure context
|
||||||
|
report.FailureContext = ec.analyzeFailureContext()
|
||||||
|
|
||||||
|
// Run diagnosis if not already done
|
||||||
|
diagnoser := NewDiagnoser(ec.log, false)
|
||||||
|
if diagResult, err := diagnoser.DiagnoseFile(ec.archivePath); err == nil {
|
||||||
|
report.DiagnosisResult = diagResult
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate recommendations
|
||||||
|
report.Recommendations = ec.generateRecommendations(report)
|
||||||
|
|
||||||
|
return report
|
||||||
|
}
|
||||||
|
|
||||||
|
// analyzeFailureContext extracts context around the failure
|
||||||
|
func (ec *ErrorCollector) analyzeFailureContext() *FailureContext {
|
||||||
|
ctx := &FailureContext{}
|
||||||
|
|
||||||
|
// Look for line number in errors
|
||||||
|
for _, errLine := range ec.lastErrors {
|
||||||
|
if lineNum := extractLineNumber(errLine); lineNum > 0 {
|
||||||
|
ctx.FailedLine = lineNum
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Look for COPY-related errors
|
||||||
|
for _, errLine := range ec.lastErrors {
|
||||||
|
if strings.Contains(errLine, "COPY") || strings.Contains(errLine, "syntax error") {
|
||||||
|
ctx.InCopyBlock = true
|
||||||
|
// Try to extract table name
|
||||||
|
if tableName := extractTableName(errLine); tableName != "" {
|
||||||
|
ctx.CopyTableName = tableName
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If we have a line number, try to get surrounding context from the dump
|
||||||
|
if ctx.FailedLine > 0 && ec.archivePath != "" {
|
||||||
|
ctx.SurroundingLines = ec.getSurroundingLines(ctx.FailedLine, 5)
|
||||||
|
}
|
||||||
|
|
||||||
|
return ctx
|
||||||
|
}
|
||||||
|
|
||||||
|
// getSurroundingLines reads lines around a specific line number from the dump
|
||||||
|
func (ec *ErrorCollector) getSurroundingLines(lineNum int, context int) []string {
|
||||||
|
var reader io.Reader
|
||||||
|
var lines []string
|
||||||
|
|
||||||
|
file, err := os.Open(ec.archivePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
// Handle compressed files
|
||||||
|
if strings.HasSuffix(ec.archivePath, ".gz") {
|
||||||
|
gz, err := gzip.NewReader(file)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
defer gz.Close()
|
||||||
|
reader = gz
|
||||||
|
} else {
|
||||||
|
reader = file
|
||||||
|
}
|
||||||
|
|
||||||
|
scanner := bufio.NewScanner(reader)
|
||||||
|
buf := make([]byte, 0, 1024*1024)
|
||||||
|
scanner.Buffer(buf, 10*1024*1024)
|
||||||
|
|
||||||
|
currentLine := 0
|
||||||
|
startLine := lineNum - context
|
||||||
|
endLine := lineNum + context
|
||||||
|
|
||||||
|
if startLine < 1 {
|
||||||
|
startLine = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
for scanner.Scan() {
|
||||||
|
currentLine++
|
||||||
|
if currentLine >= startLine && currentLine <= endLine {
|
||||||
|
prefix := " "
|
||||||
|
if currentLine == lineNum {
|
||||||
|
prefix = "> "
|
||||||
|
}
|
||||||
|
lines = append(lines, fmt.Sprintf("%s%d: %s", prefix, currentLine, truncateString(scanner.Text(), 100)))
|
||||||
|
}
|
||||||
|
if currentLine > endLine {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return lines
|
||||||
|
}
|
||||||
|
|
||||||
|
// generateRecommendations provides actionable recommendations based on the error
|
||||||
|
func (ec *ErrorCollector) generateRecommendations(report *RestoreErrorReport) []string {
|
||||||
|
var recs []string
|
||||||
|
|
||||||
|
// Check diagnosis results
|
||||||
|
if report.DiagnosisResult != nil {
|
||||||
|
if report.DiagnosisResult.IsTruncated {
|
||||||
|
recs = append(recs,
|
||||||
|
"CRITICAL: Backup file is truncated/incomplete",
|
||||||
|
"Action: Re-run the backup for the affected database",
|
||||||
|
"Check: Verify disk space was available during backup",
|
||||||
|
"Check: Verify network was stable during backup transfer",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if report.DiagnosisResult.IsCorrupted {
|
||||||
|
recs = append(recs,
|
||||||
|
"CRITICAL: Backup file appears corrupted",
|
||||||
|
"Action: Restore from a previous backup",
|
||||||
|
"Action: Verify backup file checksum if available",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if report.DiagnosisResult.Details != nil && report.DiagnosisResult.Details.UnterminatedCopy {
|
||||||
|
recs = append(recs,
|
||||||
|
fmt.Sprintf("ISSUE: COPY block for table '%s' was not terminated",
|
||||||
|
report.DiagnosisResult.Details.LastCopyTable),
|
||||||
|
"Cause: Backup was interrupted during data export",
|
||||||
|
"Action: Re-run backup ensuring it completes fully",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check error patterns
|
||||||
|
if report.TotalErrors > 1000000 {
|
||||||
|
recs = append(recs,
|
||||||
|
"ISSUE: Millions of errors indicate structural problem, not individual data issues",
|
||||||
|
"Cause: Likely wrong restore method or truncated dump",
|
||||||
|
"Check: Verify dump format matches restore command",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for common error types
|
||||||
|
errLower := strings.ToLower(report.ErrorMessage)
|
||||||
|
if strings.Contains(errLower, "syntax error") {
|
||||||
|
recs = append(recs,
|
||||||
|
"ISSUE: SQL syntax errors during restore",
|
||||||
|
"Cause: COPY data being interpreted as SQL commands",
|
||||||
|
"Check: Run 'dbbackup restore diagnose <archive>' for detailed analysis",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.Contains(errLower, "permission denied") {
|
||||||
|
recs = append(recs,
|
||||||
|
"ISSUE: Permission denied",
|
||||||
|
"Action: Check database user has sufficient privileges",
|
||||||
|
"Action: For ownership preservation, use a superuser account",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.Contains(errLower, "does not exist") {
|
||||||
|
recs = append(recs,
|
||||||
|
"ISSUE: Missing object reference",
|
||||||
|
"Action: Ensure globals.sql was restored first (for roles/tablespaces)",
|
||||||
|
"Action: Check if target database was created",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(recs) == 0 {
|
||||||
|
recs = append(recs,
|
||||||
|
"Run 'dbbackup restore diagnose <archive>' for detailed analysis",
|
||||||
|
"Check the stderr output above for specific error messages",
|
||||||
|
"Review the PostgreSQL/MySQL logs on the target server",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return recs
|
||||||
|
}
|
||||||
|
|
||||||
|
// SaveReport saves the error report to a file
|
||||||
|
func (ec *ErrorCollector) SaveReport(report *RestoreErrorReport, outputPath string) error {
|
||||||
|
// Create directory if needed
|
||||||
|
dir := filepath.Dir(outputPath)
|
||||||
|
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||||
|
return fmt.Errorf("failed to create directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Marshal to JSON with indentation
|
||||||
|
data, err := json.MarshalIndent(report, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to marshal report: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write file
|
||||||
|
if err := os.WriteFile(outputPath, data, 0644); err != nil {
|
||||||
|
return fmt.Errorf("failed to write report: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// PrintReport prints a human-readable summary of the error report
|
||||||
|
func (ec *ErrorCollector) PrintReport(report *RestoreErrorReport) {
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println(strings.Repeat("=", 70))
|
||||||
|
fmt.Println(" [ERROR] RESTORE ERROR REPORT")
|
||||||
|
fmt.Println(strings.Repeat("=", 70))
|
||||||
|
|
||||||
|
fmt.Printf("\n[TIME] Timestamp: %s\n", report.Timestamp.Format("2006-01-02 15:04:05"))
|
||||||
|
fmt.Printf("[FILE] Archive: %s\n", filepath.Base(report.ArchivePath))
|
||||||
|
fmt.Printf("[FMT] Format: %s\n", report.ArchiveFormat)
|
||||||
|
fmt.Printf("[TGT] Target DB: %s\n", report.TargetDB)
|
||||||
|
fmt.Printf("[CODE] Exit Code: %d\n", report.ExitCode)
|
||||||
|
fmt.Printf("[ERR] Total Errors: %d\n", report.TotalErrors)
|
||||||
|
|
||||||
|
fmt.Println("\n" + strings.Repeat("-", 70))
|
||||||
|
fmt.Println("ERROR DETAILS:")
|
||||||
|
fmt.Println(strings.Repeat("-", 70))
|
||||||
|
|
||||||
|
fmt.Printf("\nType: %s\n", report.ErrorType)
|
||||||
|
fmt.Printf("Message: %s\n", report.ErrorMessage)
|
||||||
|
if report.ErrorHint != "" {
|
||||||
|
fmt.Printf("Hint: %s\n", report.ErrorHint)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Show failure context
|
||||||
|
if report.FailureContext != nil && report.FailureContext.FailedLine > 0 {
|
||||||
|
fmt.Println("\n" + strings.Repeat("-", 70))
|
||||||
|
fmt.Println("FAILURE CONTEXT:")
|
||||||
|
fmt.Println(strings.Repeat("-", 70))
|
||||||
|
|
||||||
|
fmt.Printf("\nFailed at line: %d\n", report.FailureContext.FailedLine)
|
||||||
|
if report.FailureContext.InCopyBlock {
|
||||||
|
fmt.Printf("Inside COPY block for table: %s\n", report.FailureContext.CopyTableName)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(report.FailureContext.SurroundingLines) > 0 {
|
||||||
|
fmt.Println("\nSurrounding lines:")
|
||||||
|
for _, line := range report.FailureContext.SurroundingLines {
|
||||||
|
fmt.Println(line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Show first few errors
|
||||||
|
if len(report.FirstErrors) > 0 {
|
||||||
|
fmt.Println("\n" + strings.Repeat("-", 70))
|
||||||
|
fmt.Println("FIRST ERRORS:")
|
||||||
|
fmt.Println(strings.Repeat("-", 70))
|
||||||
|
|
||||||
|
for i, err := range report.FirstErrors {
|
||||||
|
if i >= 5 {
|
||||||
|
fmt.Printf("... and %d more\n", len(report.FirstErrors)-5)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
fmt.Printf(" %d. %s\n", i+1, truncateString(err, 100))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Show diagnosis summary
|
||||||
|
if report.DiagnosisResult != nil && !report.DiagnosisResult.IsValid {
|
||||||
|
fmt.Println("\n" + strings.Repeat("-", 70))
|
||||||
|
fmt.Println("DIAGNOSIS:")
|
||||||
|
fmt.Println(strings.Repeat("-", 70))
|
||||||
|
|
||||||
|
if report.DiagnosisResult.IsTruncated {
|
||||||
|
fmt.Println(" [FAIL] File is TRUNCATED")
|
||||||
|
}
|
||||||
|
if report.DiagnosisResult.IsCorrupted {
|
||||||
|
fmt.Println(" [FAIL] File is CORRUPTED")
|
||||||
|
}
|
||||||
|
for i, err := range report.DiagnosisResult.Errors {
|
||||||
|
if i >= 3 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
fmt.Printf(" • %s\n", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Show recommendations
|
||||||
|
fmt.Println("\n" + strings.Repeat("-", 70))
|
||||||
|
fmt.Println("[HINT] RECOMMENDATIONS:")
|
||||||
|
fmt.Println(strings.Repeat("-", 70))
|
||||||
|
|
||||||
|
for _, rec := range report.Recommendations {
|
||||||
|
fmt.Printf(" - %s\n", rec)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Show tool versions
|
||||||
|
fmt.Println("\n" + strings.Repeat("-", 70))
|
||||||
|
fmt.Println("ENVIRONMENT:")
|
||||||
|
fmt.Println(strings.Repeat("-", 70))
|
||||||
|
|
||||||
|
fmt.Printf(" OS: %s/%s\n", report.OS, report.Arch)
|
||||||
|
fmt.Printf(" Go: %s\n", report.GoVersion)
|
||||||
|
if report.PgRestoreVersion != "" {
|
||||||
|
fmt.Printf(" pg_restore: %s\n", report.PgRestoreVersion)
|
||||||
|
}
|
||||||
|
if report.PsqlVersion != "" {
|
||||||
|
fmt.Printf(" psql: %s\n", report.PsqlVersion)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println(strings.Repeat("=", 70))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper functions
|
||||||
|
|
||||||
|
func isErrorLine(line string) bool {
|
||||||
|
return strings.Contains(line, "ERROR:") ||
|
||||||
|
strings.Contains(line, "FATAL:") ||
|
||||||
|
strings.Contains(line, "error:") ||
|
||||||
|
strings.Contains(line, "PANIC:")
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractLineNumber(errLine string) int {
|
||||||
|
// Look for patterns like "LINE 1:" or "line 123"
|
||||||
|
patterns := []string{"LINE ", "line "}
|
||||||
|
for _, pattern := range patterns {
|
||||||
|
if idx := strings.Index(errLine, pattern); idx >= 0 {
|
||||||
|
numStart := idx + len(pattern)
|
||||||
|
numEnd := numStart
|
||||||
|
for numEnd < len(errLine) && errLine[numEnd] >= '0' && errLine[numEnd] <= '9' {
|
||||||
|
numEnd++
|
||||||
|
}
|
||||||
|
if numEnd > numStart {
|
||||||
|
var num int
|
||||||
|
fmt.Sscanf(errLine[numStart:numEnd], "%d", &num)
|
||||||
|
return num
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractTableName(errLine string) string {
|
||||||
|
// Look for patterns like 'COPY "tablename"' or 'table "tablename"'
|
||||||
|
patterns := []string{"COPY ", "table "}
|
||||||
|
for _, pattern := range patterns {
|
||||||
|
if idx := strings.Index(errLine, pattern); idx >= 0 {
|
||||||
|
start := idx + len(pattern)
|
||||||
|
// Skip optional quote
|
||||||
|
if start < len(errLine) && errLine[start] == '"' {
|
||||||
|
start++
|
||||||
|
}
|
||||||
|
end := start
|
||||||
|
for end < len(errLine) && errLine[end] != '"' && errLine[end] != ' ' && errLine[end] != '(' {
|
||||||
|
end++
|
||||||
|
}
|
||||||
|
if end > start {
|
||||||
|
return errLine[start:end]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func getDatabaseType(format ArchiveFormat) string {
|
||||||
|
if format.IsMySQL() {
|
||||||
|
return "mysql"
|
||||||
|
}
|
||||||
|
return "postgresql"
|
||||||
|
}
|
||||||
|
|
||||||
|
func getCommandVersion(cmd string, arg string) string {
|
||||||
|
// Use timeout to prevent blocking if command hangs
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
output, err := exec.CommandContext(ctx, cmd, arg).CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
// Return first line only
|
||||||
|
lines := strings.Split(string(output), "\n")
|
||||||
|
if len(lines) > 0 {
|
||||||
|
return strings.TrimSpace(lines[0])
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"os/exec"
|
"os/exec"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
"dbbackup/internal/database"
|
"dbbackup/internal/database"
|
||||||
)
|
)
|
||||||
@@ -47,8 +48,13 @@ func ParsePostgreSQLVersion(versionStr string) (*VersionInfo, error) {
|
|||||||
|
|
||||||
// GetDumpFileVersion extracts the PostgreSQL version from a dump file
|
// GetDumpFileVersion extracts the PostgreSQL version from a dump file
|
||||||
// Uses pg_restore -l to read the dump metadata
|
// Uses pg_restore -l to read the dump metadata
|
||||||
|
// Uses a 30-second timeout to avoid blocking on large files
|
||||||
func GetDumpFileVersion(dumpPath string) (*VersionInfo, error) {
|
func GetDumpFileVersion(dumpPath string) (*VersionInfo, error) {
|
||||||
cmd := exec.Command("pg_restore", "-l", dumpPath)
|
// Use a timeout context to prevent blocking on very large dump files
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
cmd := exec.CommandContext(ctx, "pg_restore", "-l", dumpPath)
|
||||||
output, err := cmd.CombinedOutput()
|
output, err := cmd.CombinedOutput()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to read dump file metadata: %w (output: %s)", err, string(output))
|
return nil, fmt.Errorf("failed to read dump file metadata: %w (output: %s)", err, string(output))
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ func (pc *PrivilegeChecker) CheckAndWarn(allowRoot bool) error {
|
|||||||
isRoot, user := pc.isRunningAsRoot()
|
isRoot, user := pc.isRunningAsRoot()
|
||||||
|
|
||||||
if isRoot {
|
if isRoot {
|
||||||
pc.log.Warn("⚠️ Running with elevated privileges (root/Administrator)")
|
pc.log.Warn("[WARN] Running with elevated privileges (root/Administrator)")
|
||||||
pc.log.Warn("Security recommendation: Create a dedicated backup user with minimal privileges")
|
pc.log.Warn("Security recommendation: Create a dedicated backup user with minimal privileges")
|
||||||
|
|
||||||
if !allowRoot {
|
if !allowRoot {
|
||||||
|
|||||||
@@ -64,7 +64,7 @@ func (rc *ResourceChecker) ValidateResourcesForBackup(estimatedSize int64) error
|
|||||||
|
|
||||||
if len(warnings) > 0 {
|
if len(warnings) > 0 {
|
||||||
for _, warning := range warnings {
|
for _, warning := range warnings {
|
||||||
rc.log.Warn("⚠️ Resource constraint: " + warning)
|
rc.log.Warn("[WARN] Resource constraint: " + warning)
|
||||||
}
|
}
|
||||||
rc.log.Info("Continuing backup operation (warnings are informational)")
|
rc.log.Info("Continuing backup operation (warnings are informational)")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ func (rc *ResourceChecker) checkPlatformLimits() (*ResourceLimits, error) {
|
|||||||
rc.log.Debug("Resource limit: max open files", "limit", rLimit.Cur, "max", rLimit.Max)
|
rc.log.Debug("Resource limit: max open files", "limit", rLimit.Cur, "max", rLimit.Max)
|
||||||
|
|
||||||
if rLimit.Cur < 1024 {
|
if rLimit.Cur < 1024 {
|
||||||
rc.log.Warn("⚠️ Low file descriptor limit detected",
|
rc.log.Warn("[WARN] Low file descriptor limit detected",
|
||||||
"current", rLimit.Cur,
|
"current", rLimit.Cur,
|
||||||
"recommended", 4096,
|
"recommended", 4096,
|
||||||
"hint", "Increase with: ulimit -n 4096")
|
"hint", "Increase with: ulimit -n 4096")
|
||||||
|
|||||||
@@ -201,14 +201,20 @@ func (m ArchiveBrowserModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||||||
if len(m.archives) > 0 && m.cursor < len(m.archives) {
|
if len(m.archives) > 0 && m.cursor < len(m.archives) {
|
||||||
selected := m.archives[m.cursor]
|
selected := m.archives[m.cursor]
|
||||||
|
|
||||||
|
// Handle diagnose mode - go directly to diagnosis view
|
||||||
|
if m.mode == "diagnose" {
|
||||||
|
diagnoseView := NewDiagnoseView(m.config, m.logger, m.parent, m.ctx, selected)
|
||||||
|
return diagnoseView, diagnoseView.Init()
|
||||||
|
}
|
||||||
|
|
||||||
// Validate selection based on mode
|
// Validate selection based on mode
|
||||||
if m.mode == "restore-cluster" && !selected.Format.IsClusterBackup() {
|
if m.mode == "restore-cluster" && !selected.Format.IsClusterBackup() {
|
||||||
m.message = errorStyle.Render("❌ Please select a cluster backup (.tar.gz)")
|
m.message = errorStyle.Render("[FAIL] Please select a cluster backup (.tar.gz)")
|
||||||
return m, nil
|
return m, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if m.mode == "restore-single" && selected.Format.IsClusterBackup() {
|
if m.mode == "restore-single" && selected.Format.IsClusterBackup() {
|
||||||
m.message = errorStyle.Render("❌ Please select a single database backup")
|
m.message = errorStyle.Render("[FAIL] Please select a single database backup")
|
||||||
return m, nil
|
return m, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -221,12 +227,20 @@ func (m ArchiveBrowserModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||||||
// Show detailed info
|
// Show detailed info
|
||||||
if len(m.archives) > 0 && m.cursor < len(m.archives) {
|
if len(m.archives) > 0 && m.cursor < len(m.archives) {
|
||||||
selected := m.archives[m.cursor]
|
selected := m.archives[m.cursor]
|
||||||
m.message = fmt.Sprintf("📦 %s | Format: %s | Size: %s | Modified: %s",
|
m.message = fmt.Sprintf("[PKG] %s | Format: %s | Size: %s | Modified: %s",
|
||||||
selected.Name,
|
selected.Name,
|
||||||
selected.Format.String(),
|
selected.Format.String(),
|
||||||
formatSize(selected.Size),
|
formatSize(selected.Size),
|
||||||
selected.Modified.Format("2006-01-02 15:04:05"))
|
selected.Modified.Format("2006-01-02 15:04:05"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case "d":
|
||||||
|
// Run diagnosis on selected archive
|
||||||
|
if len(m.archives) > 0 && m.cursor < len(m.archives) {
|
||||||
|
selected := m.archives[m.cursor]
|
||||||
|
diagnoseView := NewDiagnoseView(m.config, m.logger, m, m.ctx, selected)
|
||||||
|
return diagnoseView, diagnoseView.Init()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -237,11 +251,13 @@ func (m ArchiveBrowserModel) View() string {
|
|||||||
var s strings.Builder
|
var s strings.Builder
|
||||||
|
|
||||||
// Header
|
// Header
|
||||||
title := "📦 Backup Archives"
|
title := "[PKG] Backup Archives"
|
||||||
if m.mode == "restore-single" {
|
if m.mode == "restore-single" {
|
||||||
title = "📦 Select Archive to Restore (Single Database)"
|
title = "[PKG] Select Archive to Restore (Single Database)"
|
||||||
} else if m.mode == "restore-cluster" {
|
} else if m.mode == "restore-cluster" {
|
||||||
title = "📦 Select Archive to Restore (Cluster)"
|
title = "[PKG] Select Archive to Restore (Cluster)"
|
||||||
|
} else if m.mode == "diagnose" {
|
||||||
|
title = "[SEARCH] Select Archive to Diagnose"
|
||||||
}
|
}
|
||||||
|
|
||||||
s.WriteString(titleStyle.Render(title))
|
s.WriteString(titleStyle.Render(title))
|
||||||
@@ -253,7 +269,7 @@ func (m ArchiveBrowserModel) View() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if m.err != nil {
|
if m.err != nil {
|
||||||
s.WriteString(errorStyle.Render(fmt.Sprintf("❌ Error: %v", m.err)))
|
s.WriteString(errorStyle.Render(fmt.Sprintf("[FAIL] Error: %v", m.err)))
|
||||||
s.WriteString("\n\n")
|
s.WriteString("\n\n")
|
||||||
s.WriteString(infoStyle.Render("Press Esc to go back"))
|
s.WriteString(infoStyle.Render("Press Esc to go back"))
|
||||||
return s.String()
|
return s.String()
|
||||||
@@ -277,7 +293,7 @@ func (m ArchiveBrowserModel) View() string {
|
|||||||
s.WriteString(archiveHeaderStyle.Render(fmt.Sprintf("%-40s %-25s %-12s %-20s",
|
s.WriteString(archiveHeaderStyle.Render(fmt.Sprintf("%-40s %-25s %-12s %-20s",
|
||||||
"FILENAME", "FORMAT", "SIZE", "MODIFIED")))
|
"FILENAME", "FORMAT", "SIZE", "MODIFIED")))
|
||||||
s.WriteString("\n")
|
s.WriteString("\n")
|
||||||
s.WriteString(strings.Repeat("─", 100))
|
s.WriteString(strings.Repeat("-", 100))
|
||||||
s.WriteString("\n")
|
s.WriteString("\n")
|
||||||
|
|
||||||
// Show archives (limit to visible area)
|
// Show archives (limit to visible area)
|
||||||
@@ -301,13 +317,13 @@ func (m ArchiveBrowserModel) View() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Color code based on validity and age
|
// Color code based on validity and age
|
||||||
statusIcon := "✓"
|
statusIcon := "[+]"
|
||||||
if !archive.Valid {
|
if !archive.Valid {
|
||||||
statusIcon = "✗"
|
statusIcon = "[-]"
|
||||||
style = archiveInvalidStyle
|
style = archiveInvalidStyle
|
||||||
} else if time.Since(archive.Modified) > 30*24*time.Hour {
|
} else if time.Since(archive.Modified) > 30*24*time.Hour {
|
||||||
style = archiveOldStyle
|
style = archiveOldStyle
|
||||||
statusIcon = "⚠"
|
statusIcon = "[WARN]"
|
||||||
}
|
}
|
||||||
|
|
||||||
filename := truncate(archive.Name, 38)
|
filename := truncate(archive.Name, 38)
|
||||||
@@ -335,7 +351,7 @@ func (m ArchiveBrowserModel) View() string {
|
|||||||
s.WriteString(infoStyle.Render(fmt.Sprintf("Total: %d archive(s) | Selected: %d/%d",
|
s.WriteString(infoStyle.Render(fmt.Sprintf("Total: %d archive(s) | Selected: %d/%d",
|
||||||
len(m.archives), m.cursor+1, len(m.archives))))
|
len(m.archives), m.cursor+1, len(m.archives))))
|
||||||
s.WriteString("\n")
|
s.WriteString("\n")
|
||||||
s.WriteString(infoStyle.Render("⌨️ ↑/↓: Navigate | Enter: Select | f: Filter | i: Info | Esc: Back"))
|
s.WriteString(infoStyle.Render("[KEY] ↑/↓: Navigate | Enter: Select | d: Diagnose | f: Filter | i: Info | Esc: Back"))
|
||||||
|
|
||||||
return s.String()
|
return s.String()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,12 +20,14 @@ type BackupExecutionModel struct {
|
|||||||
logger logger.Logger
|
logger logger.Logger
|
||||||
parent tea.Model
|
parent tea.Model
|
||||||
ctx context.Context
|
ctx context.Context
|
||||||
|
cancel context.CancelFunc // Cancel function to stop the operation
|
||||||
backupType string
|
backupType string
|
||||||
databaseName string
|
databaseName string
|
||||||
ratio int
|
ratio int
|
||||||
status string
|
status string
|
||||||
progress int
|
progress int
|
||||||
done bool
|
done bool
|
||||||
|
cancelling bool // True when user has requested cancellation
|
||||||
err error
|
err error
|
||||||
result string
|
result string
|
||||||
startTime time.Time
|
startTime time.Time
|
||||||
@@ -34,11 +36,14 @@ type BackupExecutionModel struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewBackupExecution(cfg *config.Config, log logger.Logger, parent tea.Model, ctx context.Context, backupType, dbName string, ratio int) BackupExecutionModel {
|
func NewBackupExecution(cfg *config.Config, log logger.Logger, parent tea.Model, ctx context.Context, backupType, dbName string, ratio int) BackupExecutionModel {
|
||||||
|
// Create a cancellable context derived from parent
|
||||||
|
childCtx, cancel := context.WithCancel(ctx)
|
||||||
return BackupExecutionModel{
|
return BackupExecutionModel{
|
||||||
config: cfg,
|
config: cfg,
|
||||||
logger: log,
|
logger: log,
|
||||||
parent: parent,
|
parent: parent,
|
||||||
ctx: ctx,
|
ctx: childCtx,
|
||||||
|
cancel: cancel,
|
||||||
backupType: backupType,
|
backupType: backupType,
|
||||||
databaseName: dbName,
|
databaseName: dbName,
|
||||||
ratio: ratio,
|
ratio: ratio,
|
||||||
@@ -78,10 +83,10 @@ type backupCompleteMsg struct {
|
|||||||
|
|
||||||
func executeBackupWithTUIProgress(parentCtx context.Context, cfg *config.Config, log logger.Logger, backupType, dbName string, ratio int) tea.Cmd {
|
func executeBackupWithTUIProgress(parentCtx context.Context, cfg *config.Config, log logger.Logger, backupType, dbName string, ratio int) tea.Cmd {
|
||||||
return func() tea.Msg {
|
return func() tea.Msg {
|
||||||
// Use configurable cluster timeout (minutes) from config; default set in config.New()
|
// NO TIMEOUT for backup operations - a backup takes as long as it takes
|
||||||
// Use parent context to inherit cancellation from TUI
|
// Large databases can take many hours
|
||||||
clusterTimeout := time.Duration(cfg.ClusterTimeoutMinutes) * time.Minute
|
// Only manual cancellation (Ctrl+C) should stop the backup
|
||||||
ctx, cancel := context.WithTimeout(parentCtx, clusterTimeout)
|
ctx, cancel := context.WithCancel(parentCtx)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
@@ -131,11 +136,11 @@ func executeBackupWithTUIProgress(parentCtx context.Context, cfg *config.Config,
|
|||||||
var result string
|
var result string
|
||||||
switch backupType {
|
switch backupType {
|
||||||
case "single":
|
case "single":
|
||||||
result = fmt.Sprintf("✓ Single database backup of '%s' completed successfully in %v", dbName, elapsed)
|
result = fmt.Sprintf("[+] Single database backup of '%s' completed successfully in %v", dbName, elapsed)
|
||||||
case "sample":
|
case "sample":
|
||||||
result = fmt.Sprintf("✓ Sample backup of '%s' (ratio: %d) completed successfully in %v", dbName, ratio, elapsed)
|
result = fmt.Sprintf("[+] Sample backup of '%s' (ratio: %d) completed successfully in %v", dbName, ratio, elapsed)
|
||||||
case "cluster":
|
case "cluster":
|
||||||
result = fmt.Sprintf("✓ Cluster backup completed successfully in %v", elapsed)
|
result = fmt.Sprintf("[+] Cluster backup completed successfully in %v", elapsed)
|
||||||
}
|
}
|
||||||
|
|
||||||
return backupCompleteMsg{
|
return backupCompleteMsg{
|
||||||
@@ -195,9 +200,9 @@ func (m BackupExecutionModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||||||
m.err = msg.err
|
m.err = msg.err
|
||||||
m.result = msg.result
|
m.result = msg.result
|
||||||
if m.err == nil {
|
if m.err == nil {
|
||||||
m.status = "✅ Backup completed successfully!"
|
m.status = "[OK] Backup completed successfully!"
|
||||||
} else {
|
} else {
|
||||||
m.status = fmt.Sprintf("❌ Backup failed: %v", m.err)
|
m.status = fmt.Sprintf("[FAIL] Backup failed: %v", m.err)
|
||||||
}
|
}
|
||||||
// Auto-forward in debug/auto-confirm mode
|
// Auto-forward in debug/auto-confirm mode
|
||||||
if m.config.TUIAutoConfirm {
|
if m.config.TUIAutoConfirm {
|
||||||
@@ -206,9 +211,21 @@ func (m BackupExecutionModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||||||
return m, nil
|
return m, nil
|
||||||
|
|
||||||
case tea.KeyMsg:
|
case tea.KeyMsg:
|
||||||
if m.done {
|
switch msg.String() {
|
||||||
switch msg.String() {
|
case "ctrl+c", "esc":
|
||||||
case "enter", "esc", "q":
|
if !m.done && !m.cancelling {
|
||||||
|
// User requested cancellation - cancel the context
|
||||||
|
m.cancelling = true
|
||||||
|
m.status = "[STOP] Cancelling backup... (please wait)"
|
||||||
|
if m.cancel != nil {
|
||||||
|
m.cancel()
|
||||||
|
}
|
||||||
|
return m, nil
|
||||||
|
} else if m.done {
|
||||||
|
return m.parent, nil
|
||||||
|
}
|
||||||
|
case "enter", "q":
|
||||||
|
if m.done {
|
||||||
return m.parent, nil
|
return m.parent, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -223,7 +240,7 @@ func (m BackupExecutionModel) View() string {
|
|||||||
|
|
||||||
// Clear screen with newlines and render header
|
// Clear screen with newlines and render header
|
||||||
s.WriteString("\n\n")
|
s.WriteString("\n\n")
|
||||||
header := titleStyle.Render("🔄 Backup Execution")
|
header := titleStyle.Render("[EXEC] Backup Execution")
|
||||||
s.WriteString(header)
|
s.WriteString(header)
|
||||||
s.WriteString("\n\n")
|
s.WriteString("\n\n")
|
||||||
|
|
||||||
@@ -240,12 +257,17 @@ func (m BackupExecutionModel) View() string {
|
|||||||
|
|
||||||
// Status with spinner
|
// Status with spinner
|
||||||
if !m.done {
|
if !m.done {
|
||||||
s.WriteString(fmt.Sprintf(" %s %s\n", spinnerFrames[m.spinnerFrame], m.status))
|
if m.cancelling {
|
||||||
|
s.WriteString(fmt.Sprintf(" %s %s\n", spinnerFrames[m.spinnerFrame], m.status))
|
||||||
|
} else {
|
||||||
|
s.WriteString(fmt.Sprintf(" %s %s\n", spinnerFrames[m.spinnerFrame], m.status))
|
||||||
|
s.WriteString("\n [KEY] Press Ctrl+C or ESC to cancel\n")
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
s.WriteString(fmt.Sprintf(" %s\n\n", m.status))
|
s.WriteString(fmt.Sprintf(" %s\n\n", m.status))
|
||||||
|
|
||||||
if m.err != nil {
|
if m.err != nil {
|
||||||
s.WriteString(fmt.Sprintf(" ❌ Error: %v\n", m.err))
|
s.WriteString(fmt.Sprintf(" [FAIL] Error: %v\n", m.err))
|
||||||
} else if m.result != "" {
|
} else if m.result != "" {
|
||||||
// Parse and display result cleanly
|
// Parse and display result cleanly
|
||||||
lines := strings.Split(m.result, "\n")
|
lines := strings.Split(m.result, "\n")
|
||||||
@@ -256,7 +278,7 @@ func (m BackupExecutionModel) View() string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
s.WriteString("\n ⌨️ Press Enter or ESC to return to menu\n")
|
s.WriteString("\n [KEY] Press Enter or ESC to return to menu\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
return s.String()
|
return s.String()
|
||||||
|
|||||||
@@ -86,7 +86,7 @@ func (m BackupManagerModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||||||
// Verify archive
|
// Verify archive
|
||||||
if len(m.archives) > 0 && m.cursor < len(m.archives) {
|
if len(m.archives) > 0 && m.cursor < len(m.archives) {
|
||||||
selected := m.archives[m.cursor]
|
selected := m.archives[m.cursor]
|
||||||
m.message = fmt.Sprintf("🔍 Verifying %s...", selected.Name)
|
m.message = fmt.Sprintf("[SEARCH] Verifying %s...", selected.Name)
|
||||||
// In real implementation, would run verification
|
// In real implementation, would run verification
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -96,16 +96,16 @@ func (m BackupManagerModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||||||
selected := m.archives[m.cursor]
|
selected := m.archives[m.cursor]
|
||||||
archivePath := selected.Path
|
archivePath := selected.Path
|
||||||
confirm := NewConfirmationModelWithAction(m.config, m.logger, m,
|
confirm := NewConfirmationModelWithAction(m.config, m.logger, m,
|
||||||
"🗑️ Delete Archive",
|
"[DELETE] Delete Archive",
|
||||||
fmt.Sprintf("Delete archive '%s'? This cannot be undone.", selected.Name),
|
fmt.Sprintf("Delete archive '%s'? This cannot be undone.", selected.Name),
|
||||||
func() (tea.Model, tea.Cmd) {
|
func() (tea.Model, tea.Cmd) {
|
||||||
// Delete the archive
|
// Delete the archive
|
||||||
err := deleteArchive(archivePath)
|
err := deleteArchive(archivePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
m.err = fmt.Errorf("failed to delete archive: %v", err)
|
m.err = fmt.Errorf("failed to delete archive: %v", err)
|
||||||
m.message = fmt.Sprintf("❌ Failed to delete: %v", err)
|
m.message = fmt.Sprintf("[FAIL] Failed to delete: %v", err)
|
||||||
} else {
|
} else {
|
||||||
m.message = fmt.Sprintf("✅ Deleted: %s", selected.Name)
|
m.message = fmt.Sprintf("[OK] Deleted: %s", selected.Name)
|
||||||
}
|
}
|
||||||
// Refresh the archive list
|
// Refresh the archive list
|
||||||
m.loading = true
|
m.loading = true
|
||||||
@@ -118,7 +118,7 @@ func (m BackupManagerModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||||||
// Show info
|
// Show info
|
||||||
if len(m.archives) > 0 && m.cursor < len(m.archives) {
|
if len(m.archives) > 0 && m.cursor < len(m.archives) {
|
||||||
selected := m.archives[m.cursor]
|
selected := m.archives[m.cursor]
|
||||||
m.message = fmt.Sprintf("📦 %s | %s | %s | Modified: %s",
|
m.message = fmt.Sprintf("[PKG] %s | %s | %s | Modified: %s",
|
||||||
selected.Name,
|
selected.Name,
|
||||||
selected.Format.String(),
|
selected.Format.String(),
|
||||||
formatSize(selected.Size),
|
formatSize(selected.Size),
|
||||||
@@ -152,7 +152,7 @@ func (m BackupManagerModel) View() string {
|
|||||||
var s strings.Builder
|
var s strings.Builder
|
||||||
|
|
||||||
// Title
|
// Title
|
||||||
s.WriteString(titleStyle.Render("🗄️ Backup Archive Manager"))
|
s.WriteString(titleStyle.Render("[DB] Backup Archive Manager"))
|
||||||
s.WriteString("\n\n")
|
s.WriteString("\n\n")
|
||||||
|
|
||||||
if m.loading {
|
if m.loading {
|
||||||
@@ -161,7 +161,7 @@ func (m BackupManagerModel) View() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if m.err != nil {
|
if m.err != nil {
|
||||||
s.WriteString(errorStyle.Render(fmt.Sprintf("❌ Error: %v", m.err)))
|
s.WriteString(errorStyle.Render(fmt.Sprintf("[FAIL] Error: %v", m.err)))
|
||||||
s.WriteString("\n\n")
|
s.WriteString("\n\n")
|
||||||
s.WriteString(infoStyle.Render("Press Esc to go back"))
|
s.WriteString(infoStyle.Render("Press Esc to go back"))
|
||||||
return s.String()
|
return s.String()
|
||||||
@@ -184,7 +184,7 @@ func (m BackupManagerModel) View() string {
|
|||||||
s.WriteString(archiveHeaderStyle.Render(fmt.Sprintf("%-35s %-25s %-12s %-20s",
|
s.WriteString(archiveHeaderStyle.Render(fmt.Sprintf("%-35s %-25s %-12s %-20s",
|
||||||
"FILENAME", "FORMAT", "SIZE", "MODIFIED")))
|
"FILENAME", "FORMAT", "SIZE", "MODIFIED")))
|
||||||
s.WriteString("\n")
|
s.WriteString("\n")
|
||||||
s.WriteString(strings.Repeat("─", 95))
|
s.WriteString(strings.Repeat("-", 95))
|
||||||
s.WriteString("\n")
|
s.WriteString("\n")
|
||||||
|
|
||||||
// Show archives (limit to visible area)
|
// Show archives (limit to visible area)
|
||||||
@@ -208,12 +208,12 @@ func (m BackupManagerModel) View() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Status icon
|
// Status icon
|
||||||
statusIcon := "✓"
|
statusIcon := "[+]"
|
||||||
if !archive.Valid {
|
if !archive.Valid {
|
||||||
statusIcon = "✗"
|
statusIcon = "[-]"
|
||||||
style = archiveInvalidStyle
|
style = archiveInvalidStyle
|
||||||
} else if time.Since(archive.Modified) > 30*24*time.Hour {
|
} else if time.Since(archive.Modified) > 30*24*time.Hour {
|
||||||
statusIcon = "⚠"
|
statusIcon = "[WARN]"
|
||||||
}
|
}
|
||||||
|
|
||||||
filename := truncate(archive.Name, 33)
|
filename := truncate(archive.Name, 33)
|
||||||
@@ -240,7 +240,7 @@ func (m BackupManagerModel) View() string {
|
|||||||
|
|
||||||
s.WriteString(infoStyle.Render(fmt.Sprintf("Selected: %d/%d", m.cursor+1, len(m.archives))))
|
s.WriteString(infoStyle.Render(fmt.Sprintf("Selected: %d/%d", m.cursor+1, len(m.archives))))
|
||||||
s.WriteString("\n")
|
s.WriteString("\n")
|
||||||
s.WriteString(infoStyle.Render("⌨️ ↑/↓: Navigate | r: Restore | v: Verify | d: Delete | i: Info | R: Refresh | Esc: Back"))
|
s.WriteString(infoStyle.Render("[KEY] ↑/↓: Navigate | r: Restore | v: Verify | d: Delete | i: Info | R: Refresh | Esc: Back"))
|
||||||
|
|
||||||
return s.String()
|
return s.String()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -67,7 +67,6 @@ func (m ConfirmationModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||||||
switch msg := msg.(type) {
|
switch msg := msg.(type) {
|
||||||
case autoConfirmMsg:
|
case autoConfirmMsg:
|
||||||
// Auto-confirm triggered
|
// Auto-confirm triggered
|
||||||
m.confirmed = true
|
|
||||||
if m.onConfirm != nil {
|
if m.onConfirm != nil {
|
||||||
return m.onConfirm()
|
return m.onConfirm()
|
||||||
}
|
}
|
||||||
@@ -95,7 +94,6 @@ func (m ConfirmationModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||||||
|
|
||||||
case "enter", "y":
|
case "enter", "y":
|
||||||
if msg.String() == "y" || m.cursor == 0 {
|
if msg.String() == "y" || m.cursor == 0 {
|
||||||
m.confirmed = true
|
|
||||||
// Execute the onConfirm callback if provided
|
// Execute the onConfirm callback if provided
|
||||||
if m.onConfirm != nil {
|
if m.onConfirm != nil {
|
||||||
return m.onConfirm()
|
return m.onConfirm()
|
||||||
@@ -131,7 +129,7 @@ func (m ConfirmationModel) View() string {
|
|||||||
s.WriteString(" ")
|
s.WriteString(" ")
|
||||||
}
|
}
|
||||||
|
|
||||||
s.WriteString("\n\n⌨️ ←/→: Select • Enter/y: Confirm • n/ESC: Cancel\n")
|
s.WriteString("\n\n[KEYS] <-/->: Select | Enter/y: Confirm | n/ESC: Cancel\n")
|
||||||
|
|
||||||
return s.String()
|
return s.String()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -53,7 +53,8 @@ type databaseListMsg struct {
|
|||||||
|
|
||||||
func fetchDatabases(cfg *config.Config, log logger.Logger) tea.Cmd {
|
func fetchDatabases(cfg *config.Config, log logger.Logger) tea.Cmd {
|
||||||
return func() tea.Msg {
|
return func() tea.Msg {
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
// 60 seconds for database listing - busy servers may be slow
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
dbClient, err := database.New(cfg, log)
|
dbClient, err := database.New(cfg, log)
|
||||||
@@ -108,7 +109,7 @@ func (m DatabaseSelectorModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||||||
return executor, executor.Init()
|
return executor, executor.Init()
|
||||||
}
|
}
|
||||||
inputModel := NewInputModel(m.config, m.logger, m,
|
inputModel := NewInputModel(m.config, m.logger, m,
|
||||||
"📊 Sample Ratio",
|
"[STATS] Sample Ratio",
|
||||||
"Enter sample ratio (1-100):",
|
"Enter sample ratio (1-100):",
|
||||||
"10",
|
"10",
|
||||||
ValidateInt(1, 100))
|
ValidateInt(1, 100))
|
||||||
@@ -151,7 +152,7 @@ func (m DatabaseSelectorModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||||||
// If sample backup, ask for ratio first
|
// If sample backup, ask for ratio first
|
||||||
if m.backupType == "sample" {
|
if m.backupType == "sample" {
|
||||||
inputModel := NewInputModel(m.config, m.logger, m,
|
inputModel := NewInputModel(m.config, m.logger, m,
|
||||||
"📊 Sample Ratio",
|
"[STATS] Sample Ratio",
|
||||||
"Enter sample ratio (1-100):",
|
"Enter sample ratio (1-100):",
|
||||||
"10",
|
"10",
|
||||||
ValidateInt(1, 100))
|
ValidateInt(1, 100))
|
||||||
@@ -175,12 +176,12 @@ func (m DatabaseSelectorModel) View() string {
|
|||||||
s.WriteString(fmt.Sprintf("\n%s\n\n", header))
|
s.WriteString(fmt.Sprintf("\n%s\n\n", header))
|
||||||
|
|
||||||
if m.loading {
|
if m.loading {
|
||||||
s.WriteString("⏳ Loading databases...\n")
|
s.WriteString("[WAIT] Loading databases...\n")
|
||||||
return s.String()
|
return s.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
if m.err != nil {
|
if m.err != nil {
|
||||||
s.WriteString(fmt.Sprintf("❌ Error: %v\n", m.err))
|
s.WriteString(fmt.Sprintf("[FAIL] Error: %v\n", m.err))
|
||||||
s.WriteString("\nPress ESC to go back\n")
|
s.WriteString("\nPress ESC to go back\n")
|
||||||
return s.String()
|
return s.String()
|
||||||
}
|
}
|
||||||
@@ -202,7 +203,7 @@ func (m DatabaseSelectorModel) View() string {
|
|||||||
s.WriteString(fmt.Sprintf("\n%s\n", m.message))
|
s.WriteString(fmt.Sprintf("\n%s\n", m.message))
|
||||||
}
|
}
|
||||||
|
|
||||||
s.WriteString("\n⌨️ ↑/↓: Navigate • Enter: Select • ESC: Back • q: Quit\n")
|
s.WriteString("\n[KEYS] Up/Down: Navigate | Enter: Select | ESC: Back | q: Quit\n")
|
||||||
|
|
||||||
return s.String()
|
return s.String()
|
||||||
}
|
}
|
||||||
|
|||||||
463
internal/tui/diagnose_view.go
Normal file
463
internal/tui/diagnose_view.go
Normal file
@@ -0,0 +1,463 @@
|
|||||||
|
package tui
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
tea "github.com/charmbracelet/bubbletea"
|
||||||
|
"github.com/charmbracelet/lipgloss"
|
||||||
|
|
||||||
|
"dbbackup/internal/config"
|
||||||
|
"dbbackup/internal/logger"
|
||||||
|
"dbbackup/internal/restore"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
diagnoseBoxStyle = lipgloss.NewStyle().
|
||||||
|
Border(lipgloss.RoundedBorder()).
|
||||||
|
BorderForeground(lipgloss.Color("63")).
|
||||||
|
Padding(1, 2)
|
||||||
|
|
||||||
|
diagnosePassStyle = lipgloss.NewStyle().
|
||||||
|
Foreground(lipgloss.Color("2")).
|
||||||
|
Bold(true)
|
||||||
|
|
||||||
|
diagnoseFailStyle = lipgloss.NewStyle().
|
||||||
|
Foreground(lipgloss.Color("1")).
|
||||||
|
Bold(true)
|
||||||
|
|
||||||
|
diagnoseWarnStyle = lipgloss.NewStyle().
|
||||||
|
Foreground(lipgloss.Color("3"))
|
||||||
|
|
||||||
|
diagnoseInfoStyle = lipgloss.NewStyle().
|
||||||
|
Foreground(lipgloss.Color("244"))
|
||||||
|
|
||||||
|
diagnoseHeaderStyle = lipgloss.NewStyle().
|
||||||
|
Foreground(lipgloss.Color("63")).
|
||||||
|
Bold(true)
|
||||||
|
)
|
||||||
|
|
||||||
|
// DiagnoseViewModel shows backup file diagnosis results
|
||||||
|
type DiagnoseViewModel struct {
|
||||||
|
config *config.Config
|
||||||
|
logger logger.Logger
|
||||||
|
parent tea.Model
|
||||||
|
ctx context.Context
|
||||||
|
archive ArchiveInfo
|
||||||
|
result *restore.DiagnoseResult
|
||||||
|
results []*restore.DiagnoseResult // For cluster archives
|
||||||
|
running bool
|
||||||
|
completed bool
|
||||||
|
progress string
|
||||||
|
cursor int // For scrolling through cluster results
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewDiagnoseView creates a new diagnose view
|
||||||
|
func NewDiagnoseView(cfg *config.Config, log logger.Logger, parent tea.Model, ctx context.Context, archive ArchiveInfo) DiagnoseViewModel {
|
||||||
|
return DiagnoseViewModel{
|
||||||
|
config: cfg,
|
||||||
|
logger: log,
|
||||||
|
parent: parent,
|
||||||
|
ctx: ctx,
|
||||||
|
archive: archive,
|
||||||
|
running: true,
|
||||||
|
progress: "Starting diagnosis...",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m DiagnoseViewModel) Init() tea.Cmd {
|
||||||
|
return runDiagnosis(m.config, m.logger, m.archive)
|
||||||
|
}
|
||||||
|
|
||||||
|
type diagnoseCompleteMsg struct {
|
||||||
|
result *restore.DiagnoseResult
|
||||||
|
results []*restore.DiagnoseResult
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
type diagnoseProgressMsg struct {
|
||||||
|
message string
|
||||||
|
}
|
||||||
|
|
||||||
|
func runDiagnosis(cfg *config.Config, log logger.Logger, archive ArchiveInfo) tea.Cmd {
|
||||||
|
return func() tea.Msg {
|
||||||
|
diagnoser := restore.NewDiagnoser(log, true)
|
||||||
|
|
||||||
|
// For cluster archives, we can do deep analysis
|
||||||
|
if archive.Format.IsClusterBackup() {
|
||||||
|
// Create temp directory (use WorkDir if configured for large archives)
|
||||||
|
log.Info("Creating temp directory for diagnosis", "workDir", cfg.WorkDir)
|
||||||
|
tempDir, err := createTempDirIn(cfg.WorkDir, "dbbackup-diagnose-*")
|
||||||
|
if err != nil {
|
||||||
|
return diagnoseCompleteMsg{err: fmt.Errorf("failed to create temp dir (workDir=%s): %w", cfg.WorkDir, err)}
|
||||||
|
}
|
||||||
|
log.Info("Using temp directory", "path", tempDir)
|
||||||
|
defer removeTempDir(tempDir)
|
||||||
|
|
||||||
|
// Diagnose all dumps in the cluster
|
||||||
|
results, err := diagnoser.DiagnoseClusterDumps(archive.Path, tempDir)
|
||||||
|
if err != nil {
|
||||||
|
return diagnoseCompleteMsg{err: err}
|
||||||
|
}
|
||||||
|
|
||||||
|
return diagnoseCompleteMsg{results: results}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Single file diagnosis
|
||||||
|
result, err := diagnoser.DiagnoseFile(archive.Path)
|
||||||
|
if err != nil {
|
||||||
|
return diagnoseCompleteMsg{err: err}
|
||||||
|
}
|
||||||
|
|
||||||
|
return diagnoseCompleteMsg{result: result}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m DiagnoseViewModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||||
|
switch msg := msg.(type) {
|
||||||
|
case diagnoseCompleteMsg:
|
||||||
|
m.running = false
|
||||||
|
m.completed = true
|
||||||
|
if msg.err != nil {
|
||||||
|
m.err = msg.err
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
m.result = msg.result
|
||||||
|
m.results = msg.results
|
||||||
|
return m, nil
|
||||||
|
|
||||||
|
case diagnoseProgressMsg:
|
||||||
|
m.progress = msg.message
|
||||||
|
return m, nil
|
||||||
|
|
||||||
|
case tea.KeyMsg:
|
||||||
|
switch msg.String() {
|
||||||
|
case "ctrl+c", "q", "esc":
|
||||||
|
return m.parent, nil
|
||||||
|
|
||||||
|
case "up", "k":
|
||||||
|
if len(m.results) > 0 && m.cursor > 0 {
|
||||||
|
m.cursor--
|
||||||
|
}
|
||||||
|
|
||||||
|
case "down", "j":
|
||||||
|
if len(m.results) > 0 && m.cursor < len(m.results)-1 {
|
||||||
|
m.cursor++
|
||||||
|
}
|
||||||
|
|
||||||
|
case "enter", " ":
|
||||||
|
return m.parent, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m DiagnoseViewModel) View() string {
|
||||||
|
var s strings.Builder
|
||||||
|
|
||||||
|
// Header
|
||||||
|
s.WriteString(titleStyle.Render("[SEARCH] Backup Diagnosis"))
|
||||||
|
s.WriteString("\n\n")
|
||||||
|
|
||||||
|
// Archive info
|
||||||
|
s.WriteString(diagnoseHeaderStyle.Render("Archive: "))
|
||||||
|
s.WriteString(m.archive.Name)
|
||||||
|
s.WriteString("\n")
|
||||||
|
s.WriteString(diagnoseHeaderStyle.Render("Format: "))
|
||||||
|
s.WriteString(m.archive.Format.String())
|
||||||
|
s.WriteString("\n")
|
||||||
|
s.WriteString(diagnoseHeaderStyle.Render("Size: "))
|
||||||
|
s.WriteString(formatSize(m.archive.Size))
|
||||||
|
s.WriteString("\n\n")
|
||||||
|
|
||||||
|
if m.running {
|
||||||
|
s.WriteString(infoStyle.Render("[WAIT] " + m.progress))
|
||||||
|
s.WriteString("\n\n")
|
||||||
|
s.WriteString(diagnoseInfoStyle.Render("This may take a while for large archives..."))
|
||||||
|
return s.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
if m.err != nil {
|
||||||
|
s.WriteString(errorStyle.Render(fmt.Sprintf("[FAIL] Diagnosis failed: %v", m.err)))
|
||||||
|
s.WriteString("\n\n")
|
||||||
|
s.WriteString(infoStyle.Render("Press Enter or Esc to go back"))
|
||||||
|
return s.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// For cluster archives, show summary + details
|
||||||
|
if len(m.results) > 0 {
|
||||||
|
s.WriteString(m.renderClusterResults())
|
||||||
|
} else if m.result != nil {
|
||||||
|
s.WriteString(m.renderSingleResult(m.result))
|
||||||
|
}
|
||||||
|
|
||||||
|
s.WriteString("\n")
|
||||||
|
s.WriteString(infoStyle.Render("Press Enter or Esc to go back"))
|
||||||
|
|
||||||
|
return s.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m DiagnoseViewModel) renderSingleResult(result *restore.DiagnoseResult) string {
|
||||||
|
var s strings.Builder
|
||||||
|
|
||||||
|
// Status
|
||||||
|
s.WriteString(strings.Repeat("-", 60))
|
||||||
|
s.WriteString("\n")
|
||||||
|
|
||||||
|
if result.IsValid {
|
||||||
|
s.WriteString(diagnosePassStyle.Render("[OK] STATUS: VALID"))
|
||||||
|
} else {
|
||||||
|
s.WriteString(diagnoseFailStyle.Render("[FAIL] STATUS: INVALID"))
|
||||||
|
}
|
||||||
|
s.WriteString("\n")
|
||||||
|
|
||||||
|
if result.IsTruncated {
|
||||||
|
s.WriteString(diagnoseFailStyle.Render("[WARN] TRUNCATED: File appears incomplete"))
|
||||||
|
s.WriteString("\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.IsCorrupted {
|
||||||
|
s.WriteString(diagnoseFailStyle.Render("[WARN] CORRUPTED: File structure is damaged"))
|
||||||
|
s.WriteString("\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
s.WriteString(strings.Repeat("-", 60))
|
||||||
|
s.WriteString("\n\n")
|
||||||
|
|
||||||
|
// Details
|
||||||
|
if result.Details != nil {
|
||||||
|
s.WriteString(diagnoseHeaderStyle.Render("[STATS] DETAILS:"))
|
||||||
|
s.WriteString("\n")
|
||||||
|
|
||||||
|
if result.Details.HasPGDMPSignature {
|
||||||
|
s.WriteString(diagnosePassStyle.Render(" [+] "))
|
||||||
|
s.WriteString("Has PGDMP signature (custom format)\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.Details.HasSQLHeader {
|
||||||
|
s.WriteString(diagnosePassStyle.Render(" [+] "))
|
||||||
|
s.WriteString("Has PostgreSQL SQL header\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.Details.GzipValid {
|
||||||
|
s.WriteString(diagnosePassStyle.Render(" [+] "))
|
||||||
|
s.WriteString("Gzip compression valid\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.Details.PgRestoreListable {
|
||||||
|
s.WriteString(diagnosePassStyle.Render(" [+] "))
|
||||||
|
s.WriteString(fmt.Sprintf("pg_restore can list contents (%d tables)\n", result.Details.TableCount))
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.Details.CopyBlockCount > 0 {
|
||||||
|
s.WriteString(diagnoseInfoStyle.Render(" - "))
|
||||||
|
s.WriteString(fmt.Sprintf("Contains %d COPY blocks\n", result.Details.CopyBlockCount))
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.Details.UnterminatedCopy {
|
||||||
|
s.WriteString(diagnoseFailStyle.Render(" [-] "))
|
||||||
|
s.WriteString(fmt.Sprintf("Unterminated COPY block: %s (line %d)\n",
|
||||||
|
result.Details.LastCopyTable, result.Details.LastCopyLineNumber))
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.Details.ProperlyTerminated {
|
||||||
|
s.WriteString(diagnosePassStyle.Render(" [+] "))
|
||||||
|
s.WriteString("All COPY blocks properly terminated\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.Details.ExpandedSize > 0 {
|
||||||
|
s.WriteString(diagnoseInfoStyle.Render(" - "))
|
||||||
|
s.WriteString(fmt.Sprintf("Expanded size: %s (ratio: %.1fx)\n",
|
||||||
|
formatSize(result.Details.ExpandedSize), result.Details.CompressionRatio))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Errors
|
||||||
|
if len(result.Errors) > 0 {
|
||||||
|
s.WriteString("\n")
|
||||||
|
s.WriteString(diagnoseFailStyle.Render("[FAIL] ERRORS:"))
|
||||||
|
s.WriteString("\n")
|
||||||
|
for i, e := range result.Errors {
|
||||||
|
if i >= 5 {
|
||||||
|
s.WriteString(diagnoseInfoStyle.Render(fmt.Sprintf(" ... and %d more\n", len(result.Errors)-5)))
|
||||||
|
break
|
||||||
|
}
|
||||||
|
s.WriteString(diagnoseFailStyle.Render(" - "))
|
||||||
|
s.WriteString(truncate(e, 70))
|
||||||
|
s.WriteString("\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Warnings
|
||||||
|
if len(result.Warnings) > 0 {
|
||||||
|
s.WriteString("\n")
|
||||||
|
s.WriteString(diagnoseWarnStyle.Render("[WARN] WARNINGS:"))
|
||||||
|
s.WriteString("\n")
|
||||||
|
for i, w := range result.Warnings {
|
||||||
|
if i >= 3 {
|
||||||
|
s.WriteString(diagnoseInfoStyle.Render(fmt.Sprintf(" ... and %d more\n", len(result.Warnings)-3)))
|
||||||
|
break
|
||||||
|
}
|
||||||
|
s.WriteString(diagnoseWarnStyle.Render(" - "))
|
||||||
|
s.WriteString(truncate(w, 70))
|
||||||
|
s.WriteString("\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recommendations
|
||||||
|
if !result.IsValid {
|
||||||
|
s.WriteString("\n")
|
||||||
|
s.WriteString(diagnoseHeaderStyle.Render("[HINT] RECOMMENDATIONS:"))
|
||||||
|
s.WriteString("\n")
|
||||||
|
if result.IsTruncated {
|
||||||
|
s.WriteString(" 1. Re-run the backup process for this database\n")
|
||||||
|
s.WriteString(" 2. Check disk space on backup server\n")
|
||||||
|
s.WriteString(" 3. Verify network stability for remote backups\n")
|
||||||
|
}
|
||||||
|
if result.IsCorrupted {
|
||||||
|
s.WriteString(" 1. Verify backup was transferred completely\n")
|
||||||
|
s.WriteString(" 2. Try restoring from a previous backup\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return s.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m DiagnoseViewModel) renderClusterResults() string {
|
||||||
|
var s strings.Builder
|
||||||
|
|
||||||
|
// Summary
|
||||||
|
validCount := 0
|
||||||
|
invalidCount := 0
|
||||||
|
for _, r := range m.results {
|
||||||
|
if r.IsValid {
|
||||||
|
validCount++
|
||||||
|
} else {
|
||||||
|
invalidCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
s.WriteString(strings.Repeat("-", 60))
|
||||||
|
s.WriteString("\n")
|
||||||
|
s.WriteString(diagnoseHeaderStyle.Render(fmt.Sprintf("[STATS] CLUSTER SUMMARY: %d databases\n", len(m.results))))
|
||||||
|
s.WriteString(strings.Repeat("-", 60))
|
||||||
|
s.WriteString("\n\n")
|
||||||
|
|
||||||
|
if invalidCount == 0 {
|
||||||
|
s.WriteString(diagnosePassStyle.Render("[OK] All dumps are valid"))
|
||||||
|
s.WriteString("\n\n")
|
||||||
|
} else {
|
||||||
|
s.WriteString(diagnoseFailStyle.Render(fmt.Sprintf("[FAIL] %d/%d dumps have issues", invalidCount, len(m.results))))
|
||||||
|
s.WriteString("\n\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// List all dumps with status
|
||||||
|
s.WriteString(diagnoseHeaderStyle.Render("Database Dumps:"))
|
||||||
|
s.WriteString("\n")
|
||||||
|
|
||||||
|
// Show visible range based on cursor
|
||||||
|
start := m.cursor - 5
|
||||||
|
if start < 0 {
|
||||||
|
start = 0
|
||||||
|
}
|
||||||
|
end := start + 12
|
||||||
|
if end > len(m.results) {
|
||||||
|
end = len(m.results)
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := start; i < end; i++ {
|
||||||
|
r := m.results[i]
|
||||||
|
cursor := " "
|
||||||
|
if i == m.cursor {
|
||||||
|
cursor = ">"
|
||||||
|
}
|
||||||
|
|
||||||
|
var status string
|
||||||
|
if r.IsValid {
|
||||||
|
status = diagnosePassStyle.Render("[+]")
|
||||||
|
} else if r.IsTruncated {
|
||||||
|
status = diagnoseFailStyle.Render("[-] TRUNCATED")
|
||||||
|
} else if r.IsCorrupted {
|
||||||
|
status = diagnoseFailStyle.Render("[-] CORRUPTED")
|
||||||
|
} else {
|
||||||
|
status = diagnoseFailStyle.Render("[-] INVALID")
|
||||||
|
}
|
||||||
|
|
||||||
|
line := fmt.Sprintf("%s %s %-35s %s",
|
||||||
|
cursor,
|
||||||
|
status,
|
||||||
|
truncate(r.FileName, 35),
|
||||||
|
formatSize(r.FileSize))
|
||||||
|
|
||||||
|
if i == m.cursor {
|
||||||
|
s.WriteString(archiveSelectedStyle.Render(line))
|
||||||
|
} else {
|
||||||
|
s.WriteString(line)
|
||||||
|
}
|
||||||
|
s.WriteString("\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Show selected dump details
|
||||||
|
if m.cursor < len(m.results) {
|
||||||
|
selected := m.results[m.cursor]
|
||||||
|
s.WriteString("\n")
|
||||||
|
s.WriteString(strings.Repeat("-", 60))
|
||||||
|
s.WriteString("\n")
|
||||||
|
s.WriteString(diagnoseHeaderStyle.Render("Selected: " + selected.FileName))
|
||||||
|
s.WriteString("\n\n")
|
||||||
|
|
||||||
|
// Show condensed details for selected
|
||||||
|
if selected.Details != nil {
|
||||||
|
if selected.Details.UnterminatedCopy {
|
||||||
|
s.WriteString(diagnoseFailStyle.Render(" [-] Unterminated COPY: "))
|
||||||
|
s.WriteString(selected.Details.LastCopyTable)
|
||||||
|
s.WriteString(fmt.Sprintf(" (line %d)\n", selected.Details.LastCopyLineNumber))
|
||||||
|
}
|
||||||
|
if len(selected.Details.SampleCopyData) > 0 {
|
||||||
|
s.WriteString(diagnoseInfoStyle.Render(" Sample orphaned data: "))
|
||||||
|
s.WriteString(truncate(selected.Details.SampleCopyData[0], 50))
|
||||||
|
s.WriteString("\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(selected.Errors) > 0 {
|
||||||
|
for i, e := range selected.Errors {
|
||||||
|
if i >= 2 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
s.WriteString(diagnoseFailStyle.Render(" - "))
|
||||||
|
s.WriteString(truncate(e, 55))
|
||||||
|
s.WriteString("\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
s.WriteString("\n")
|
||||||
|
s.WriteString(infoStyle.Render("Use ↑/↓ to browse, Enter/Esc to go back"))
|
||||||
|
|
||||||
|
return s.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper functions for temp directory management
|
||||||
|
func createTempDir(pattern string) (string, error) {
|
||||||
|
return os.MkdirTemp("", pattern)
|
||||||
|
}
|
||||||
|
|
||||||
|
func createTempDirIn(baseDir, pattern string) (string, error) {
|
||||||
|
if baseDir == "" {
|
||||||
|
return os.MkdirTemp("", pattern)
|
||||||
|
}
|
||||||
|
// Ensure base directory exists
|
||||||
|
if err := os.MkdirAll(baseDir, 0755); err != nil {
|
||||||
|
return "", fmt.Errorf("cannot create work directory: %w", err)
|
||||||
|
}
|
||||||
|
return os.MkdirTemp(baseDir, pattern)
|
||||||
|
}
|
||||||
|
|
||||||
|
func removeTempDir(path string) error {
|
||||||
|
return os.RemoveAll(path)
|
||||||
|
}
|
||||||
@@ -208,7 +208,7 @@ func (dp *DirectoryPicker) View() string {
|
|||||||
if dp.allowFiles {
|
if dp.allowFiles {
|
||||||
pickerType = "File/Directory"
|
pickerType = "File/Directory"
|
||||||
}
|
}
|
||||||
header := fmt.Sprintf("📁 %s Picker - %s", pickerType, dp.currentPath)
|
header := fmt.Sprintf("[DIR] %s Picker - %s", pickerType, dp.currentPath)
|
||||||
content.WriteString(dp.styles.Header.Render(header))
|
content.WriteString(dp.styles.Header.Render(header))
|
||||||
content.WriteString("\n\n")
|
content.WriteString("\n\n")
|
||||||
|
|
||||||
@@ -216,13 +216,13 @@ func (dp *DirectoryPicker) View() string {
|
|||||||
for i, item := range dp.items {
|
for i, item := range dp.items {
|
||||||
var prefix string
|
var prefix string
|
||||||
if item.Name == ".." {
|
if item.Name == ".." {
|
||||||
prefix = "⬆️ "
|
prefix = "[UP] "
|
||||||
} else if item.Name == "Error reading directory" {
|
} else if item.Name == "Error reading directory" {
|
||||||
prefix = "❌ "
|
prefix = "[X] "
|
||||||
} else if item.IsDir {
|
} else if item.IsDir {
|
||||||
prefix = "📁 "
|
prefix = "[DIR] "
|
||||||
} else {
|
} else {
|
||||||
prefix = "📄 "
|
prefix = "[FILE] "
|
||||||
}
|
}
|
||||||
|
|
||||||
line := prefix + item.Name
|
line := prefix + item.Name
|
||||||
@@ -235,9 +235,9 @@ func (dp *DirectoryPicker) View() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Help text
|
// Help text
|
||||||
help := "\n↑/↓: Navigate • Enter: Open/Select File • s: Select Directory • q/Esc: Cancel"
|
help := "\nUp/Down: Navigate | Enter: Open/Select File | s: Select Directory | q/Esc: Cancel"
|
||||||
if !dp.allowFiles {
|
if !dp.allowFiles {
|
||||||
help = "\n↑/↓: Navigate • Enter: Open • s: Select Directory • q/Esc: Cancel"
|
help = "\nUp/Down: Navigate | Enter: Open | s: Select Directory | q/Esc: Cancel"
|
||||||
}
|
}
|
||||||
content.WriteString(dp.styles.Help.Render(help))
|
content.WriteString(dp.styles.Help.Render(help))
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ package tui
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/ioutil"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -59,7 +59,7 @@ func loadHistory(cfg *config.Config) []HistoryEntry {
|
|||||||
var entries []HistoryEntry
|
var entries []HistoryEntry
|
||||||
|
|
||||||
// Read backup files from backup directory
|
// Read backup files from backup directory
|
||||||
files, err := ioutil.ReadDir(cfg.BackupDir)
|
files, err := os.ReadDir(cfg.BackupDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return entries
|
return entries
|
||||||
}
|
}
|
||||||
@@ -74,6 +74,12 @@ func loadHistory(cfg *config.Config) []HistoryEntry {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Get file info for ModTime
|
||||||
|
info, err := file.Info()
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
var backupType string
|
var backupType string
|
||||||
var database string
|
var database string
|
||||||
|
|
||||||
@@ -97,8 +103,8 @@ func loadHistory(cfg *config.Config) []HistoryEntry {
|
|||||||
entries = append(entries, HistoryEntry{
|
entries = append(entries, HistoryEntry{
|
||||||
Type: backupType,
|
Type: backupType,
|
||||||
Database: database,
|
Database: database,
|
||||||
Timestamp: file.ModTime(),
|
Timestamp: info.ModTime(),
|
||||||
Status: "✅ Completed",
|
Status: "[OK] Completed",
|
||||||
Filename: name,
|
Filename: name,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -185,11 +191,11 @@ func (m HistoryViewModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||||||
func (m HistoryViewModel) View() string {
|
func (m HistoryViewModel) View() string {
|
||||||
var s strings.Builder
|
var s strings.Builder
|
||||||
|
|
||||||
header := titleStyle.Render("📜 Operation History")
|
header := titleStyle.Render("[HISTORY] Operation History")
|
||||||
s.WriteString(fmt.Sprintf("\n%s\n\n", header))
|
s.WriteString(fmt.Sprintf("\n%s\n\n", header))
|
||||||
|
|
||||||
if len(m.history) == 0 {
|
if len(m.history) == 0 {
|
||||||
s.WriteString("📭 No backup history found\n\n")
|
s.WriteString("[EMPTY] No backup history found\n\n")
|
||||||
} else {
|
} else {
|
||||||
maxVisible := 15 // Show max 15 items at once
|
maxVisible := 15 // Show max 15 items at once
|
||||||
|
|
||||||
@@ -205,7 +211,7 @@ func (m HistoryViewModel) View() string {
|
|||||||
|
|
||||||
// Show scroll indicators
|
// Show scroll indicators
|
||||||
if start > 0 {
|
if start > 0 {
|
||||||
s.WriteString(" ▲ More entries above...\n")
|
s.WriteString(" [^] More entries above...\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Display only visible entries
|
// Display only visible entries
|
||||||
@@ -227,13 +233,13 @@ func (m HistoryViewModel) View() string {
|
|||||||
|
|
||||||
// Show scroll indicator if more entries below
|
// Show scroll indicator if more entries below
|
||||||
if end < len(m.history) {
|
if end < len(m.history) {
|
||||||
s.WriteString(fmt.Sprintf(" ▼ %d more entries below...\n", len(m.history)-end))
|
s.WriteString(fmt.Sprintf(" [v] %d more entries below...\n", len(m.history)-end))
|
||||||
}
|
}
|
||||||
|
|
||||||
s.WriteString("\n")
|
s.WriteString("\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
s.WriteString("⌨️ ↑/↓: Navigate • PgUp/PgDn: Jump • Home/End: First/Last • ESC: Back • q: Quit\n")
|
s.WriteString("[KEYS] Up/Down: Navigate - PgUp/PgDn: Jump - Home/End: First/Last - ESC: Back - q: Quit\n")
|
||||||
|
|
||||||
return s.String()
|
return s.String()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -137,10 +137,10 @@ func (m InputModel) View() string {
|
|||||||
s.WriteString("\n\n")
|
s.WriteString("\n\n")
|
||||||
|
|
||||||
if m.err != nil {
|
if m.err != nil {
|
||||||
s.WriteString(errorStyle.Render(fmt.Sprintf("❌ Error: %v\n\n", m.err)))
|
s.WriteString(errorStyle.Render(fmt.Sprintf("[FAIL] Error: %v\n\n", m.err)))
|
||||||
}
|
}
|
||||||
|
|
||||||
s.WriteString("⌨️ Type value • Enter: Confirm • ESC: Cancel\n")
|
s.WriteString("[KEYS] Type value | Enter: Confirm | ESC: Cancel\n")
|
||||||
|
|
||||||
return s.String()
|
return s.String()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -89,11 +89,12 @@ func NewMenuModel(cfg *config.Config, log logger.Logger) *MenuModel {
|
|||||||
"Single Database Backup",
|
"Single Database Backup",
|
||||||
"Sample Database Backup (with ratio)",
|
"Sample Database Backup (with ratio)",
|
||||||
"Cluster Backup (all databases)",
|
"Cluster Backup (all databases)",
|
||||||
"────────────────────────────────",
|
"--------------------------------",
|
||||||
"Restore Single Database",
|
"Restore Single Database",
|
||||||
"Restore Cluster Backup",
|
"Restore Cluster Backup",
|
||||||
|
"Diagnose Backup File",
|
||||||
"List & Manage Backups",
|
"List & Manage Backups",
|
||||||
"────────────────────────────────",
|
"--------------------------------",
|
||||||
"View Active Operations",
|
"View Active Operations",
|
||||||
"Show Operation History",
|
"Show Operation History",
|
||||||
"Database Status & Health Check",
|
"Database Status & Health Check",
|
||||||
@@ -163,19 +164,21 @@ func (m *MenuModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||||||
return m.handleRestoreSingle()
|
return m.handleRestoreSingle()
|
||||||
case 5: // Restore Cluster Backup
|
case 5: // Restore Cluster Backup
|
||||||
return m.handleRestoreCluster()
|
return m.handleRestoreCluster()
|
||||||
case 6: // List & Manage Backups
|
case 6: // Diagnose Backup File
|
||||||
|
return m.handleDiagnoseBackup()
|
||||||
|
case 7: // List & Manage Backups
|
||||||
return m.handleBackupManager()
|
return m.handleBackupManager()
|
||||||
case 8: // View Active Operations
|
case 9: // View Active Operations
|
||||||
return m.handleViewOperations()
|
return m.handleViewOperations()
|
||||||
case 9: // Show Operation History
|
case 10: // Show Operation History
|
||||||
return m.handleOperationHistory()
|
return m.handleOperationHistory()
|
||||||
case 10: // Database Status
|
case 11: // Database Status
|
||||||
return m.handleStatus()
|
return m.handleStatus()
|
||||||
case 11: // Settings
|
case 12: // Settings
|
||||||
return m.handleSettings()
|
return m.handleSettings()
|
||||||
case 12: // Clear History
|
case 13: // Clear History
|
||||||
m.message = "🗑️ History cleared"
|
m.message = "[DEL] History cleared"
|
||||||
case 13: // Quit
|
case 14: // Quit
|
||||||
if m.cancel != nil {
|
if m.cancel != nil {
|
||||||
m.cancel()
|
m.cancel()
|
||||||
}
|
}
|
||||||
@@ -244,21 +247,23 @@ func (m *MenuModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||||||
return m.handleRestoreSingle()
|
return m.handleRestoreSingle()
|
||||||
case 5: // Restore Cluster Backup
|
case 5: // Restore Cluster Backup
|
||||||
return m.handleRestoreCluster()
|
return m.handleRestoreCluster()
|
||||||
case 6: // List & Manage Backups
|
case 6: // Diagnose Backup File
|
||||||
|
return m.handleDiagnoseBackup()
|
||||||
|
case 7: // List & Manage Backups
|
||||||
return m.handleBackupManager()
|
return m.handleBackupManager()
|
||||||
case 7: // Separator
|
case 8: // Separator
|
||||||
// Do nothing
|
// Do nothing
|
||||||
case 8: // View Active Operations
|
case 9: // View Active Operations
|
||||||
return m.handleViewOperations()
|
return m.handleViewOperations()
|
||||||
case 9: // Show Operation History
|
case 10: // Show Operation History
|
||||||
return m.handleOperationHistory()
|
return m.handleOperationHistory()
|
||||||
case 10: // Database Status
|
case 11: // Database Status
|
||||||
return m.handleStatus()
|
return m.handleStatus()
|
||||||
case 11: // Settings
|
case 12: // Settings
|
||||||
return m.handleSettings()
|
return m.handleSettings()
|
||||||
case 12: // Clear History
|
case 13: // Clear History
|
||||||
m.message = "🗑️ History cleared"
|
m.message = "[DEL] History cleared"
|
||||||
case 13: // Quit
|
case 14: // Quit
|
||||||
if m.cancel != nil {
|
if m.cancel != nil {
|
||||||
m.cancel()
|
m.cancel()
|
||||||
}
|
}
|
||||||
@@ -280,7 +285,7 @@ func (m *MenuModel) View() string {
|
|||||||
var s string
|
var s string
|
||||||
|
|
||||||
// Header
|
// Header
|
||||||
header := titleStyle.Render("🗄️ Database Backup Tool - Interactive Menu")
|
header := titleStyle.Render("[DB] Database Backup Tool - Interactive Menu")
|
||||||
s += fmt.Sprintf("\n%s\n\n", header)
|
s += fmt.Sprintf("\n%s\n\n", header)
|
||||||
|
|
||||||
if len(m.dbTypes) > 0 {
|
if len(m.dbTypes) > 0 {
|
||||||
@@ -294,7 +299,7 @@ func (m *MenuModel) View() string {
|
|||||||
}
|
}
|
||||||
selector := fmt.Sprintf("Target Engine: %s", strings.Join(options, menuStyle.Render(" | ")))
|
selector := fmt.Sprintf("Target Engine: %s", strings.Join(options, menuStyle.Render(" | ")))
|
||||||
s += dbSelectorLabelStyle.Render(selector) + "\n"
|
s += dbSelectorLabelStyle.Render(selector) + "\n"
|
||||||
hint := infoStyle.Render("Switch with ←/→ or t • Cluster backup requires PostgreSQL")
|
hint := infoStyle.Render("Switch with <-/-> or t | Cluster backup requires PostgreSQL")
|
||||||
s += hint + "\n"
|
s += hint + "\n"
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -321,7 +326,7 @@ func (m *MenuModel) View() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Footer
|
// Footer
|
||||||
footer := infoStyle.Render("\n⌨️ Press ↑/↓ to navigate • Enter to select • q to quit")
|
footer := infoStyle.Render("\n[KEYS] Press Up/Down to navigate | Enter to select | q to quit")
|
||||||
s += footer
|
s += footer
|
||||||
|
|
||||||
return s
|
return s
|
||||||
@@ -329,20 +334,20 @@ func (m *MenuModel) View() string {
|
|||||||
|
|
||||||
// handleSingleBackup opens database selector for single backup
|
// handleSingleBackup opens database selector for single backup
|
||||||
func (m *MenuModel) handleSingleBackup() (tea.Model, tea.Cmd) {
|
func (m *MenuModel) handleSingleBackup() (tea.Model, tea.Cmd) {
|
||||||
selector := NewDatabaseSelector(m.config, m.logger, m, m.ctx, "🗄️ Single Database Backup", "single")
|
selector := NewDatabaseSelector(m.config, m.logger, m, m.ctx, "[DB] Single Database Backup", "single")
|
||||||
return selector, selector.Init()
|
return selector, selector.Init()
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleSampleBackup opens database selector for sample backup
|
// handleSampleBackup opens database selector for sample backup
|
||||||
func (m *MenuModel) handleSampleBackup() (tea.Model, tea.Cmd) {
|
func (m *MenuModel) handleSampleBackup() (tea.Model, tea.Cmd) {
|
||||||
selector := NewDatabaseSelector(m.config, m.logger, m, m.ctx, "📊 Sample Database Backup", "sample")
|
selector := NewDatabaseSelector(m.config, m.logger, m, m.ctx, "[STATS] Sample Database Backup", "sample")
|
||||||
return selector, selector.Init()
|
return selector, selector.Init()
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleClusterBackup shows confirmation and executes cluster backup
|
// handleClusterBackup shows confirmation and executes cluster backup
|
||||||
func (m *MenuModel) handleClusterBackup() (tea.Model, tea.Cmd) {
|
func (m *MenuModel) handleClusterBackup() (tea.Model, tea.Cmd) {
|
||||||
if !m.config.IsPostgreSQL() {
|
if !m.config.IsPostgreSQL() {
|
||||||
m.message = errorStyle.Render("❌ Cluster backup is available only for PostgreSQL targets")
|
m.message = errorStyle.Render("[FAIL] Cluster backup is available only for PostgreSQL targets")
|
||||||
return m, nil
|
return m, nil
|
||||||
}
|
}
|
||||||
// Skip confirmation in auto-confirm mode
|
// Skip confirmation in auto-confirm mode
|
||||||
@@ -351,7 +356,7 @@ func (m *MenuModel) handleClusterBackup() (tea.Model, tea.Cmd) {
|
|||||||
return executor, executor.Init()
|
return executor, executor.Init()
|
||||||
}
|
}
|
||||||
confirm := NewConfirmationModelWithAction(m.config, m.logger, m,
|
confirm := NewConfirmationModelWithAction(m.config, m.logger, m,
|
||||||
"🗄️ Cluster Backup",
|
"[DB] Cluster Backup",
|
||||||
"This will backup ALL databases in the cluster. Continue?",
|
"This will backup ALL databases in the cluster. Continue?",
|
||||||
func() (tea.Model, tea.Cmd) {
|
func() (tea.Model, tea.Cmd) {
|
||||||
executor := NewBackupExecution(m.config, m.logger, m, m.ctx, "cluster", "", 0)
|
executor := NewBackupExecution(m.config, m.logger, m, m.ctx, "cluster", "", 0)
|
||||||
@@ -394,7 +399,7 @@ func (m *MenuModel) handleRestoreSingle() (tea.Model, tea.Cmd) {
|
|||||||
// handleRestoreCluster opens archive browser for cluster restore
|
// handleRestoreCluster opens archive browser for cluster restore
|
||||||
func (m *MenuModel) handleRestoreCluster() (tea.Model, tea.Cmd) {
|
func (m *MenuModel) handleRestoreCluster() (tea.Model, tea.Cmd) {
|
||||||
if !m.config.IsPostgreSQL() {
|
if !m.config.IsPostgreSQL() {
|
||||||
m.message = errorStyle.Render("❌ Cluster restore is available only for PostgreSQL")
|
m.message = errorStyle.Render("[FAIL] Cluster restore is available only for PostgreSQL")
|
||||||
return m, nil
|
return m, nil
|
||||||
}
|
}
|
||||||
browser := NewArchiveBrowser(m.config, m.logger, m, m.ctx, "restore-cluster")
|
browser := NewArchiveBrowser(m.config, m.logger, m, m.ctx, "restore-cluster")
|
||||||
@@ -407,6 +412,12 @@ func (m *MenuModel) handleBackupManager() (tea.Model, tea.Cmd) {
|
|||||||
return manager, manager.Init()
|
return manager, manager.Init()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// handleDiagnoseBackup opens archive browser for diagnosis
|
||||||
|
func (m *MenuModel) handleDiagnoseBackup() (tea.Model, tea.Cmd) {
|
||||||
|
browser := NewArchiveBrowser(m.config, m.logger, m, m.ctx, "diagnose")
|
||||||
|
return browser, browser.Init()
|
||||||
|
}
|
||||||
|
|
||||||
func (m *MenuModel) applyDatabaseSelection() {
|
func (m *MenuModel) applyDatabaseSelection() {
|
||||||
if m == nil || len(m.dbTypes) == 0 {
|
if m == nil || len(m.dbTypes) == 0 {
|
||||||
return
|
return
|
||||||
@@ -417,7 +428,7 @@ func (m *MenuModel) applyDatabaseSelection() {
|
|||||||
|
|
||||||
selection := m.dbTypes[m.dbTypeCursor]
|
selection := m.dbTypes[m.dbTypeCursor]
|
||||||
if err := m.config.SetDatabaseType(selection.value); err != nil {
|
if err := m.config.SetDatabaseType(selection.value); err != nil {
|
||||||
m.message = errorStyle.Render(fmt.Sprintf("❌ %v", err))
|
m.message = errorStyle.Render(fmt.Sprintf("[FAIL] %v", err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -426,7 +437,7 @@ func (m *MenuModel) applyDatabaseSelection() {
|
|||||||
m.config.Port = m.config.GetDefaultPort()
|
m.config.Port = m.config.GetDefaultPort()
|
||||||
}
|
}
|
||||||
|
|
||||||
m.message = successStyle.Render(fmt.Sprintf("🔀 Target database set to %s", m.config.DisplayDatabaseType()))
|
m.message = successStyle.Render(fmt.Sprintf("[SWITCH] Target database set to %s", m.config.DisplayDatabaseType()))
|
||||||
if m.logger != nil {
|
if m.logger != nil {
|
||||||
m.logger.Info("updated target database type", "type", m.config.DatabaseType, "port", m.config.Port)
|
m.logger.Info("updated target database type", "type", m.config.DatabaseType, "port", m.config.Port)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -49,14 +49,14 @@ func (m OperationsViewModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||||||
func (m OperationsViewModel) View() string {
|
func (m OperationsViewModel) View() string {
|
||||||
var s strings.Builder
|
var s strings.Builder
|
||||||
|
|
||||||
header := titleStyle.Render("📊 Active Operations")
|
header := titleStyle.Render("[STATS] Active Operations")
|
||||||
s.WriteString(fmt.Sprintf("\n%s\n\n", header))
|
s.WriteString(fmt.Sprintf("\n%s\n\n", header))
|
||||||
|
|
||||||
s.WriteString("Currently running operations:\n\n")
|
s.WriteString("Currently running operations:\n\n")
|
||||||
s.WriteString(infoStyle.Render("📭 No active operations"))
|
s.WriteString(infoStyle.Render("[NONE] No active operations"))
|
||||||
s.WriteString("\n\n")
|
s.WriteString("\n\n")
|
||||||
|
|
||||||
s.WriteString("⌨️ Press any key to return to menu\n")
|
s.WriteString("[KEYS] Press any key to return to menu\n")
|
||||||
|
|
||||||
return s.String()
|
return s.String()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -24,6 +25,7 @@ type RestoreExecutionModel struct {
|
|||||||
logger logger.Logger
|
logger logger.Logger
|
||||||
parent tea.Model
|
parent tea.Model
|
||||||
ctx context.Context
|
ctx context.Context
|
||||||
|
cancel context.CancelFunc // Cancel function to stop the operation
|
||||||
archive ArchiveInfo
|
archive ArchiveInfo
|
||||||
targetDB string
|
targetDB string
|
||||||
cleanFirst bool
|
cleanFirst bool
|
||||||
@@ -31,6 +33,8 @@ type RestoreExecutionModel struct {
|
|||||||
restoreType string
|
restoreType string
|
||||||
cleanClusterFirst bool // Drop all user databases before cluster restore
|
cleanClusterFirst bool // Drop all user databases before cluster restore
|
||||||
existingDBs []string // List of databases to drop
|
existingDBs []string // List of databases to drop
|
||||||
|
saveDebugLog bool // Save detailed error report on failure
|
||||||
|
workDir string // Custom work directory for extraction
|
||||||
|
|
||||||
// Progress tracking
|
// Progress tracking
|
||||||
status string
|
status string
|
||||||
@@ -42,19 +46,23 @@ type RestoreExecutionModel struct {
|
|||||||
spinnerFrames []string
|
spinnerFrames []string
|
||||||
|
|
||||||
// Results
|
// Results
|
||||||
done bool
|
done bool
|
||||||
err error
|
cancelling bool // True when user has requested cancellation
|
||||||
result string
|
err error
|
||||||
elapsed time.Duration
|
result string
|
||||||
|
elapsed time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewRestoreExecution creates a new restore execution model
|
// NewRestoreExecution creates a new restore execution model
|
||||||
func NewRestoreExecution(cfg *config.Config, log logger.Logger, parent tea.Model, ctx context.Context, archive ArchiveInfo, targetDB string, cleanFirst, createIfMissing bool, restoreType string, cleanClusterFirst bool, existingDBs []string) RestoreExecutionModel {
|
func NewRestoreExecution(cfg *config.Config, log logger.Logger, parent tea.Model, ctx context.Context, archive ArchiveInfo, targetDB string, cleanFirst, createIfMissing bool, restoreType string, cleanClusterFirst bool, existingDBs []string, saveDebugLog bool, workDir string) RestoreExecutionModel {
|
||||||
|
// Create a cancellable context derived from parent
|
||||||
|
childCtx, cancel := context.WithCancel(ctx)
|
||||||
return RestoreExecutionModel{
|
return RestoreExecutionModel{
|
||||||
config: cfg,
|
config: cfg,
|
||||||
logger: log,
|
logger: log,
|
||||||
parent: parent,
|
parent: parent,
|
||||||
ctx: ctx,
|
ctx: childCtx,
|
||||||
|
cancel: cancel,
|
||||||
archive: archive,
|
archive: archive,
|
||||||
targetDB: targetDB,
|
targetDB: targetDB,
|
||||||
cleanFirst: cleanFirst,
|
cleanFirst: cleanFirst,
|
||||||
@@ -62,6 +70,8 @@ func NewRestoreExecution(cfg *config.Config, log logger.Logger, parent tea.Model
|
|||||||
restoreType: restoreType,
|
restoreType: restoreType,
|
||||||
cleanClusterFirst: cleanClusterFirst,
|
cleanClusterFirst: cleanClusterFirst,
|
||||||
existingDBs: existingDBs,
|
existingDBs: existingDBs,
|
||||||
|
saveDebugLog: saveDebugLog,
|
||||||
|
workDir: workDir,
|
||||||
status: "Initializing...",
|
status: "Initializing...",
|
||||||
phase: "Starting",
|
phase: "Starting",
|
||||||
startTime: time.Now(),
|
startTime: time.Now(),
|
||||||
@@ -73,7 +83,7 @@ func NewRestoreExecution(cfg *config.Config, log logger.Logger, parent tea.Model
|
|||||||
|
|
||||||
func (m RestoreExecutionModel) Init() tea.Cmd {
|
func (m RestoreExecutionModel) Init() tea.Cmd {
|
||||||
return tea.Batch(
|
return tea.Batch(
|
||||||
executeRestoreWithTUIProgress(m.ctx, m.config, m.logger, m.archive, m.targetDB, m.cleanFirst, m.createIfMissing, m.restoreType, m.cleanClusterFirst, m.existingDBs),
|
executeRestoreWithTUIProgress(m.ctx, m.config, m.logger, m.archive, m.targetDB, m.cleanFirst, m.createIfMissing, m.restoreType, m.cleanClusterFirst, m.existingDBs, m.saveDebugLog),
|
||||||
restoreTickCmd(),
|
restoreTickCmd(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -99,12 +109,12 @@ type restoreCompleteMsg struct {
|
|||||||
elapsed time.Duration
|
elapsed time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
func executeRestoreWithTUIProgress(parentCtx context.Context, cfg *config.Config, log logger.Logger, archive ArchiveInfo, targetDB string, cleanFirst, createIfMissing bool, restoreType string, cleanClusterFirst bool, existingDBs []string) tea.Cmd {
|
func executeRestoreWithTUIProgress(parentCtx context.Context, cfg *config.Config, log logger.Logger, archive ArchiveInfo, targetDB string, cleanFirst, createIfMissing bool, restoreType string, cleanClusterFirst bool, existingDBs []string, saveDebugLog bool) tea.Cmd {
|
||||||
return func() tea.Msg {
|
return func() tea.Msg {
|
||||||
// Use configurable cluster timeout (minutes) from config; default set in config.New()
|
// NO TIMEOUT for restore operations - a restore takes as long as it takes
|
||||||
// Use parent context to inherit cancellation from TUI
|
// Large databases with large objects can take many hours
|
||||||
restoreTimeout := time.Duration(cfg.ClusterTimeoutMinutes) * time.Minute
|
// Only manual cancellation (Ctrl+C) should stop the restore
|
||||||
ctx, cancel := context.WithTimeout(parentCtx, restoreTimeout)
|
ctx, cancel := context.WithCancel(parentCtx)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
@@ -128,8 +138,8 @@ func executeRestoreWithTUIProgress(parentCtx context.Context, cfg *config.Config
|
|||||||
// This matches how cluster restore works - uses CLI tools, not database connections
|
// This matches how cluster restore works - uses CLI tools, not database connections
|
||||||
droppedCount := 0
|
droppedCount := 0
|
||||||
for _, dbName := range existingDBs {
|
for _, dbName := range existingDBs {
|
||||||
// Create timeout context for each database drop (30 seconds per DB)
|
// Create timeout context for each database drop (5 minutes per DB - large DBs take time)
|
||||||
dropCtx, dropCancel := context.WithTimeout(ctx, 30*time.Second)
|
dropCtx, dropCancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||||
if err := dropDatabaseCLI(dropCtx, cfg, dbName); err != nil {
|
if err := dropDatabaseCLI(dropCtx, cfg, dbName); err != nil {
|
||||||
log.Warn("Failed to drop database", "name", dbName, "error", err)
|
log.Warn("Failed to drop database", "name", dbName, "error", err)
|
||||||
// Continue with other databases
|
// Continue with other databases
|
||||||
@@ -146,6 +156,15 @@ func executeRestoreWithTUIProgress(parentCtx context.Context, cfg *config.Config
|
|||||||
// STEP 2: Create restore engine with silent progress (no stdout interference with TUI)
|
// STEP 2: Create restore engine with silent progress (no stdout interference with TUI)
|
||||||
engine := restore.NewSilent(cfg, log, dbClient)
|
engine := restore.NewSilent(cfg, log, dbClient)
|
||||||
|
|
||||||
|
// Enable debug logging if requested
|
||||||
|
if saveDebugLog {
|
||||||
|
// Generate debug log path using configured WorkDir
|
||||||
|
workDir := cfg.GetEffectiveWorkDir()
|
||||||
|
debugLogPath := filepath.Join(workDir, fmt.Sprintf("dbbackup-restore-debug-%s.json", time.Now().Format("20060102-150405")))
|
||||||
|
engine.SetDebugLogPath(debugLogPath)
|
||||||
|
log.Info("Debug logging enabled", "path", debugLogPath)
|
||||||
|
}
|
||||||
|
|
||||||
// Set up progress callback (but it won't work in goroutine - progress is already sent via logs)
|
// Set up progress callback (but it won't work in goroutine - progress is already sent via logs)
|
||||||
// The TUI will just use spinner animation to show activity
|
// The TUI will just use spinner animation to show activity
|
||||||
|
|
||||||
@@ -262,11 +281,32 @@ func (m RestoreExecutionModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||||||
|
|
||||||
case tea.KeyMsg:
|
case tea.KeyMsg:
|
||||||
switch msg.String() {
|
switch msg.String() {
|
||||||
case "ctrl+c", "q":
|
case "ctrl+c", "esc":
|
||||||
// Always allow quitting
|
if !m.done && !m.cancelling {
|
||||||
return m.parent, tea.Quit
|
// User requested cancellation - cancel the context
|
||||||
|
m.cancelling = true
|
||||||
case "enter", " ", "esc":
|
m.status = "[STOP] Cancelling restore... (please wait)"
|
||||||
|
m.phase = "Cancelling"
|
||||||
|
if m.cancel != nil {
|
||||||
|
m.cancel()
|
||||||
|
}
|
||||||
|
return m, nil
|
||||||
|
} else if m.done {
|
||||||
|
return m.parent, nil
|
||||||
|
}
|
||||||
|
case "q":
|
||||||
|
if !m.done && !m.cancelling {
|
||||||
|
m.cancelling = true
|
||||||
|
m.status = "[STOP] Cancelling restore... (please wait)"
|
||||||
|
m.phase = "Cancelling"
|
||||||
|
if m.cancel != nil {
|
||||||
|
m.cancel()
|
||||||
|
}
|
||||||
|
return m, nil
|
||||||
|
} else if m.done {
|
||||||
|
return m.parent, tea.Quit
|
||||||
|
}
|
||||||
|
case "enter", " ":
|
||||||
if m.done {
|
if m.done {
|
||||||
return m.parent, nil
|
return m.parent, nil
|
||||||
}
|
}
|
||||||
@@ -281,9 +321,9 @@ func (m RestoreExecutionModel) View() string {
|
|||||||
s.Grow(512) // Pre-allocate estimated capacity for better performance
|
s.Grow(512) // Pre-allocate estimated capacity for better performance
|
||||||
|
|
||||||
// Title
|
// Title
|
||||||
title := "💾 Restoring Database"
|
title := "[RESTORE] Restoring Database"
|
||||||
if m.restoreType == "restore-cluster" {
|
if m.restoreType == "restore-cluster" {
|
||||||
title = "💾 Restoring Cluster"
|
title = "[RESTORE] Restoring Cluster"
|
||||||
}
|
}
|
||||||
s.WriteString(titleStyle.Render(title))
|
s.WriteString(titleStyle.Render(title))
|
||||||
s.WriteString("\n\n")
|
s.WriteString("\n\n")
|
||||||
@@ -298,12 +338,12 @@ func (m RestoreExecutionModel) View() string {
|
|||||||
if m.done {
|
if m.done {
|
||||||
// Show result
|
// Show result
|
||||||
if m.err != nil {
|
if m.err != nil {
|
||||||
s.WriteString(errorStyle.Render("❌ Restore Failed"))
|
s.WriteString(errorStyle.Render("[FAIL] Restore Failed"))
|
||||||
s.WriteString("\n\n")
|
s.WriteString("\n\n")
|
||||||
s.WriteString(errorStyle.Render(fmt.Sprintf("Error: %v", m.err)))
|
s.WriteString(errorStyle.Render(fmt.Sprintf("Error: %v", m.err)))
|
||||||
s.WriteString("\n")
|
s.WriteString("\n")
|
||||||
} else {
|
} else {
|
||||||
s.WriteString(successStyle.Render("✅ Restore Completed Successfully"))
|
s.WriteString(successStyle.Render("[OK] Restore Completed Successfully"))
|
||||||
s.WriteString("\n\n")
|
s.WriteString("\n\n")
|
||||||
s.WriteString(successStyle.Render(m.result))
|
s.WriteString(successStyle.Render(m.result))
|
||||||
s.WriteString("\n")
|
s.WriteString("\n")
|
||||||
@@ -311,7 +351,7 @@ func (m RestoreExecutionModel) View() string {
|
|||||||
|
|
||||||
s.WriteString(fmt.Sprintf("\nElapsed Time: %s\n", formatDuration(m.elapsed)))
|
s.WriteString(fmt.Sprintf("\nElapsed Time: %s\n", formatDuration(m.elapsed)))
|
||||||
s.WriteString("\n")
|
s.WriteString("\n")
|
||||||
s.WriteString(infoStyle.Render("⌨️ Press Enter to continue"))
|
s.WriteString(infoStyle.Render("[KEYS] Press Enter to continue"))
|
||||||
} else {
|
} else {
|
||||||
// Show progress
|
// Show progress
|
||||||
s.WriteString(fmt.Sprintf("Phase: %s\n", m.phase))
|
s.WriteString(fmt.Sprintf("Phase: %s\n", m.phase))
|
||||||
@@ -333,7 +373,7 @@ func (m RestoreExecutionModel) View() string {
|
|||||||
// Elapsed time
|
// Elapsed time
|
||||||
s.WriteString(fmt.Sprintf("Elapsed: %s\n", formatDuration(m.elapsed)))
|
s.WriteString(fmt.Sprintf("Elapsed: %s\n", formatDuration(m.elapsed)))
|
||||||
s.WriteString("\n")
|
s.WriteString("\n")
|
||||||
s.WriteString(infoStyle.Render("⌨️ Press Ctrl+C to cancel"))
|
s.WriteString(infoStyle.Render("[KEYS] Press Ctrl+C to cancel"))
|
||||||
}
|
}
|
||||||
|
|
||||||
return s.String()
|
return s.String()
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user