This comprehensive guide provides a complete, secure, and fully automated backup solution for Proxmox VE using rclone with client-side encrypted iDrive E2 cloud storage. The system is designed for maximum reliability, performance, and security.
Key Features:
screen with live progress monitoringrclone_backup.sh – Core Backup Engine (FINAL VERSION)Purpose: Fully automated monthly backup with encryption, retention, and monitoring.
#!/usr/bin/env bash
# =============================================================================
# Proxmox Rclone Backup Script – Monthly, Encrypted, Max 5 Months Retention
# Features:
# - Lists all folders (newest first) with deletion preview
# - Deletes OLDEST folder if 5+ exist (BEFORE creating current)
# - Creates current month folder AFTER retention → 100% safe
# - High-performance: 8 transfers, 16 checkers, 64M chunks
# - Runs in screen with live progress
# - Full logging + detailed email report
# - Lockfile prevents double execution
# =============================================================================
set -euo pipefail
IFS=$'\n\t'
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
# === CONFIGURATION ===
logfile="/var/log/rclone_backup.log"
temp_log="/tmp/rclone_run.log"
upload_list="/tmp/rclone_upload_list.txt"
lockfile="/var/run/rclone_backup.lock"
email="[email protected]" # ← Your email address
current_date=$(date +%Y-%m) # e.g., 2025-11
month_tag=$(date +%Y_%m) # e.g., 2025_11
screen_name="rclone-upload"
backup_source="/mnt/USBBackup/dump/" # ← Source directory
backup_target="idrive-enc:" # ← Encrypted remote
retention_limit=5 # Max months to keep
max_expected_hours=24 # Warn if upload > 24h
# === Sanitize backup_target ===
backup_target="${backup_target%%$'\r'}"
backup_target="${backup_target%% }"
backup_target="${backup_target## }"
[[ -z "$backup_target" ]] && { echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: backup_target is empty!" >&2; exit 1; }
# === Logging Function ===
log() {
local msg="[$(date '+%Y-%m-%d %H:%M:%S')] $1"
echo "$msg" | tee -a "$logfile"
}
# === Lockfile (Prevent Concurrent Runs) ===
{
exec 9>"$lockfile"
flock -n 9 || { log "Another backup is already running. Exiting."; exit 1; }
log "Lockfile acquired: $lockfile"
} 9>"$lockfile"
# === Cleanup on Exit ===
cleanup() {
local exit_code=$?
rm -f "$temp_log" "$upload_list" "$lockfile"
log "Cleanup completed. Exit code: $exit_code"
}
trap cleanup EXIT
# === START ===
log "===== BACKUP START for $current_date ====="
# === RETENTION: CHECK & DELETE OLDEST BEFORE UPLOAD ===
log "Checking existing backup months BEFORE upload (keep max $((retention_limit - 1)))..."
mapfile -t months_array < <(
rclone lsf --dirs-only "$backup_target" 2>/dev/null | \
grep -E '^[0-9]{4}-[0-9]{2}/?$' | \
sed 's|/$||' | \
sort -r # newest first
)
months_count=${#months_array[@]}
log "Found $months_count month folders"
# Show current state
if (( months_count > 0 )); then
log "Current folders in bucket (newest first):"
for ((i=0; i= retention_limit && i == months_count - 1 )); then
marker=" → WILL BE DELETED (oldest)"
fi
log " [$((i+1))] ${months_array[$i]}/$marker"
done
fi
# Delete oldest if needed
if (( months_count >= retention_limit )); then
oldest="${months_array[months_count - 1]}"
log "RETENTION: Deleting oldest folder: $oldest/ (before upload)"
if rclone delete "$backup_target$oldest/" --rmdirs >> "$logfile" 2>&1; then
log "SUCCESS: $oldest/ deleted. Now: $((months_count - 1)) folders."
else
log "ERROR: Failed to delete $oldest/! Upload continues anyway."
fi
else
log "Retention: $months_count < $retention_limit → nothing to delete."
fi
# === CREATE TARGET DIRECTORY (AFTER RETENTION) ===
target_path="$backup_target$current_date"
log "Creating target directory: $target_path"
if rclone mkdir "$target_path" >> "$logfile" 2>&1; then
log "Target directory created successfully."
else
log "Target directory already exists or error (ignored)."
fi
log "Waiting 5 seconds for S3 consistency..."
sleep 5
# === FIND FILES TO UPLOAD ===
log "Searching for files matching *$month_tag*.zst..."
if ! cd "$backup_source" 2>/dev/null; then
log "ERROR: Source directory inaccessible: $backup_source"
exit 1
fi
find . -type f -name "*$month_tag*.zst" -printf '%P\n' > "$upload_list"
if [[ ! -s "$upload_list" ]]; then
log "No .zst files for month $month_tag found – skipping upload."
exit 0
fi
log "Found $(wc -l < "$upload_list") files to upload"
# === START SCREEN SESSION ===
if screen -list | grep -q "$screen_name"; then
log "Screen session '$screen_name' exists. Terminating previous..."
screen -S "$screen_name" -X quit || true
sleep 2
fi
log "Starting upload in screen session '$screen_name'..."
screen -dmS "$screen_name" bash -c "
cd \"$backup_source\" && \
rclone copy . \"$target_path\" \
--files-from-raw \"$upload_list\" \
--progress \
--log-file \"$temp_log\" \
--log-level INFO \
--transfers 8 \
--checkers 16 \
--tpslimit 100 \
--tpslimit-burst 50 \
--s3-chunk-size 64M \
--s3-upload-cutoff 5G \
--buffer-size 256M \
--retries 10 \
--low-level-retries 20 \
--timeout 5m \
--stats 10s
"
log "Upload running – monitor with: screen -r $screen_name"
# === WAIT FOR UPLOAD COMPLETION ===
max_expected_seconds=$((max_expected_hours * 3600))
elapsed=0
warned=false
while screen -list | grep -q "$screen_name"; do
sleep 60
elapsed=$((elapsed + 60))
if (( elapsed % 21600 == 0 )); then
hours=$((elapsed / 3600))
log "Upload has been running for $hours hours..."
fi
if (( elapsed >= max_expected_seconds && warned == false )); then
log "WARNING: Upload running longer than $max_expected_hours hours!"
warned=true
warn_subject="BACKUP TAKING TOO LONG ($(hostname))"
warn_body=$(cat </dev/null | tail -n 20 || true)
else
transferred="unknown"
speed="unknown"
elapsed_time="unknown"
errors=""
fi
log "Upload: $transferred, Time: $elapsed_time, Speed: $speed"
# === FINAL VERIFICATION ===
mapfile -t final_array < <(
rclone lsf --dirs-only "$backup_target" 2>/dev/null | \
grep -E '^[0-9]{4}-[0-9]{2}/?$' | \
sed 's|/$||' | \
sort -r
)
final_count=${#final_array[@]}
log "FINAL STATE: $final_count month folders in bucket (max $retention_limit allowed)."
# === SEND EMAIL REPORT ===
if [[ -z "$errors" ]] && [[ "$transferred" != *"unknown"* ]]; then
status="Upload successful"
emoji="Success"
else
status="Upload failed"
emoji="Error"
fi
mail_body=$(cat </dev/null || echo "Log not available")
EOF
)
[[ -n "$errors" ]] && mail_body="$mail_body
Errors:
$errors
"
subject="Rclone Backup – $emoji $(date '+%Y-%m-%d %H:%M')"
{
echo "To: $email"
echo "From: backup@$(hostname)"
echo "Subject: $subject"
echo "MIME-Version: 1.0"
echo "Content-Type: text/plain; charset=UTF-8"
echo "Content-Transfer-Encoding: 8bit"
echo ""
echo "$mail_body"
} | /usr/sbin/sendmail -t -i
log "Email report sent."
# === MERGE LOGS ===
[[ -f "$temp_log" ]] && cat "$temp_log" >> "$logfile"
# === END ===
log "===== BACKUP PROCESS COMPLETED ====="
exit 0
rclone_backup_dualwan.sh – Dedicated Upload Line (Optional)Use Case: Route backup traffic through a secondary WAN interface (e.g., 4G/5G) to avoid congesting primary internet.
Requirements:
192.168.123.2)vmbr0 or similar200 secondgw in /etc/iproute2/rt_tablesecho "200 secondgw" >> /etc/iproute2/rt_tables#!/bin/bash
# =============================================================================
# Dual-WAN Backup Router – Routes rclone traffic via secondary gateway
# =============================================================================
set -euo pipefail
# Root check
if [[ $EUID -ne 0 ]]; then
echo "This script must be run as root."
exit 1
fi
# === CONFIG ===
SECOND_GW="192.168.123.2" # ← Secondary gateway IP
TABLE_NAME="secondgw"
TABLE_ID="200"
RCLONE_BACKUP_SCRIPT="/usr/local/bin/rclone_backup.sh"
DEVICE="vmbr0" # ← Your network interface
logfile="/var/log/rclone_backup_dualwan.log"
log() {
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$logfile"
}
log "===== Starting Dual-WAN Backup via $SECOND_GW ====="
# Ensure routing table exists
if ! grep -q "^$TABLE_ID[[:space:]]\+$TABLE_NAME" /etc/iproute2/rt_tables; then
echo "$TABLE_ID $TABLE_NAME" >> /etc/iproute2/rt_tables
log "Routing table '$TABLE_NAME' (ID $TABLE_ID) created."
fi
# Setup routing
ip route flush table "$TABLE_NAME" || true
ip route add default via "$SECOND_GW" dev "$DEVICE" table "$TABLE_NAME"
ip rule add fwmark 0x1 table "$TABLE_NAME"
iptables -t mangle -A OUTPUT -p tcp --dport 443 -m owner --uid-owner root -j MARK --set-mark 1
log "Starting backup via secondary gateway..."
if bash "$RCLONE_BACKUP_SCRIPT"; then
log "Backup completed successfully."
else
log "Backup failed!"
fi
# Cleanup
log "Removing temporary routing rules..."
iptables -t mangle -D OUTPUT -p tcp --dport 443 -m owner --uid-owner root -j MARK --set-mark 1 || true
ip rule del fwmark 0x1 table "$TABLE_NAME" || true
ip route flush table "$TABLE_NAME" || true
log "Dual-WAN backup completed."
File: ~/.config/rclone/rclone.conf
[idrive-e2]
type = s3
provider = IDrive
access_key_id = GDFHGDBthbretu6
secret_access_key = egrtzHFDJRTzfjun6Mztumk
acl = private
endpoint = node.nl32.idrivee2-3.com
bucket_acl = private
[idrive-enc]
type = crypt
remote = idrive-e2:proxmox-backups
filename_encryption = off
directory_name_encryption = false
password = O_gfdgrtegDzutzjkzukkZUTOIYljuzutjrtzjjrKc
Security Notes:
rclone ls idrive-enc:2025-06rclone copy idrive-enc:2025-06/vzdump-qemu-100-2025_06_01.vma.zst /var/lib/vz/dump/qmrestore /var/lib/vz/dump/vzdump-qemu-100-2025_06_01.vma.zst 100zstd -t /var/lib/vz/dump/vzdump-qemu-100-2025_06_01.vma.zstRecommended: Run on the 1st of each month at 02:00 AM
0 2 1 * * /usr/local/bin/rclone_backup.sh >> /var/log/rclone_backup_cron.log 2>&1Alternative (with screen):
0 3 1 * * /usr/bin/screen -dmS monthly-backup /usr/local/bin/rclone_backup.sh| Feature | Implementation |
|---|---|
| Encryption | AES-256-CTR + HMAC-SHA256 |
| Password Strength | 64 chars → uncrackable |
| Retention | Max 5 months, safe deletion |
| Speed | 8 transfers, 64M chunks, 256M buffer |
| Reliability | 10 retries, 5m timeout, lockfile |
| Monitoring | Live screen + email + logs |
tail -f /var/log/rclone_backup.logscreen -r rclone-uploadrclone cat idrive-enc:2025-11/test.txtrclone lsf idrive-enc:vzdumpYour Proxmox data is now protected with enterprise-grade security and automation.