#!/bin/bash set -uo pipefail # Note: Removed -e to allow script to continue on errors # Errors are handled explicitly with ERROR_COUNT # Parse command line arguments ONLY_DB=false if [[ "${1:-}" == "-onlydb" ]]; then ONLY_DB=true echo "Mode: Database backup only" fi # Configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" CONFIG_FILE="$SCRIPT_DIR/backpm7.yaml" LOG_DIR="$SCRIPT_DIR/logs" mkdir -p "$LOG_DIR" LOG_FILE="$LOG_DIR/backpm7-$(date +%Y%m%d).log" ERROR_COUNT=0 EMAIL_TO="support@unikoffice.com" RECAP_FILE="/tmp/backup_recap_$$.txt" # Clean old log files (keep only last 10) find "$LOG_DIR" -maxdepth 1 -name "backpm7-*.log" -type f 2>/dev/null | sort -r | tail -n +11 | xargs -r rm -f || true # Check dependencies - COMMENTED OUT # for cmd in yq ssh tar openssl; do # if ! command -v "$cmd" &> /dev/null; then # echo "ERROR: $cmd is required but not installed" | tee -a "$LOG_FILE" # exit 1 # fi # done # Load config DIR_BACKUP=$(yq '.global.dir_backup' "$CONFIG_FILE" | tr -d '"') ENC_KEY_PATH=$(yq '.global.enc_key' "$CONFIG_FILE" | tr -d '"') # Load encryption key if [[ ! -f "$ENC_KEY_PATH" ]]; then echo "ERROR: Encryption key not found: $ENC_KEY_PATH" | tee -a "$LOG_FILE" exit 1 fi ENC_KEY=$(cat "$ENC_KEY_PATH") echo "=== Backup Started $(date) ===" | tee -a "$LOG_FILE" echo "Backup directory: $DIR_BACKUP" | tee -a "$LOG_FILE" # Check available disk space DISK_USAGE=$(df "$DIR_BACKUP" | tail -1 | awk '{print $5}' | sed 's/%//') DISK_FREE=$((100 - DISK_USAGE)) if [[ $DISK_FREE -lt 20 ]]; then echo "WARNING: Low disk space! Only ${DISK_FREE}% free on backup partition" | tee -a "$LOG_FILE" # Send warning email echo "Sending DISK SPACE WARNING email to $EMAIL_TO (${DISK_FREE}% free)" | tee -a "$LOG_FILE" if command -v msmtp &> /dev/null; then { echo "To: $EMAIL_TO" echo "Subject: BackupPM7 WARNING - Low disk space (${DISK_FREE}% free)" echo "" echo "WARNING: Low disk space on $(hostname)" echo "" echo "Backup directory: $DIR_BACKUP" echo "Disk usage: ${DISK_USAGE}%" echo "Free space: ${DISK_FREE}%" echo "" echo "The backup will continue but please free up some space soon." echo "" echo "Date: $(date '+%d.%m.%Y %H:%M')" } | msmtp "$EMAIL_TO" echo "DISK SPACE WARNING email sent successfully to $EMAIL_TO" | tee -a "$LOG_FILE" else echo "WARNING: msmtp not found - DISK WARNING email NOT sent" | tee -a "$LOG_FILE" fi else echo "Disk space OK: ${DISK_FREE}% free" | tee -a "$LOG_FILE" fi # Initialize recap file echo "BACKUP REPORT - $(hostname) - $(date '+%d.%m.%Y %H')h" > "$RECAP_FILE" echo "========================================" >> "$RECAP_FILE" echo "" >> "$RECAP_FILE" # Function to format size in MB with thousand separator format_size_mb() { local file="$1" if [[ -f "$file" ]]; then local size_kb=$(du -k "$file" | cut -f1) local size_mb=$((size_kb / 1024)) # Add thousand separator with printf and sed printf "%d" "$size_mb" | sed ':a;s/\B[0-9]\{3\}\>/\.&/;ta' else echo "0" fi } # Function to backup a single database (must be defined before use) backup_database() { local database="$1" local backup_file="$backup_dir/sql/${database}_$(date +%Y%m%d_%H).sql.gz.enc" echo " Backing up database: $database" | tee -a "$LOG_FILE" if [[ "$ssh_user" != "root" ]]; then CMD_PREFIX="sudo" else CMD_PREFIX="" fi # Execute backup with encryption # First test MySQL connection to get clear error messages (|| true to continue on error) MYSQL_TEST=$(ssh -i "$ssh_key" -p "$ssh_port" -o ConnectTimeout=20 "$ssh_user@$host_ip" \ "$CMD_PREFIX incus exec $container_name -- mariadb -h $db_host -u$db_user -p$db_pass -e 'SELECT 1' 2>&1" 2>/dev/null || true) if ssh -i "$ssh_key" -p "$ssh_port" -o ConnectTimeout=20 "$ssh_user@$host_ip" \ "$CMD_PREFIX incus exec $container_name -- bash -c 'mariadb-dump -h $db_host -u$db_user -p$db_pass --add-drop-table --create-options --databases $database 2>/dev/null | gzip'" | \ openssl enc -aes-256-cbc -salt -pass pass:"$ENC_KEY" -pbkdf2 > "$backup_file" 2>/dev/null; then # Validate backup file size (encrypted SQL should be > 100 bytes) if [[ -f "$backup_file" ]]; then file_size=$(stat -c%s "$backup_file" 2>/dev/null || echo 0) if [[ $file_size -lt 100 ]]; then # Analyze MySQL connection test results if [[ "$MYSQL_TEST" == *"Access denied"* ]]; then echo " ERROR: MySQL authentication failed for $database on $host_name/$container_name" | tee -a "$LOG_FILE" echo " User: $db_user@$db_host - Check password in configuration" | tee -a "$LOG_FILE" elif [[ "$MYSQL_TEST" == *"Unknown database"* ]]; then echo " ERROR: Database '$database' does not exist on $host_name/$container_name" | tee -a "$LOG_FILE" elif [[ "$MYSQL_TEST" == *"Can't connect"* ]]; then echo " ERROR: Cannot connect to MySQL server at $db_host in $container_name" | tee -a "$LOG_FILE" else echo " ERROR: Backup file too small (${file_size} bytes): $database on $host_name/$container_name" | tee -a "$LOG_FILE" fi ((ERROR_COUNT++)) rm -f "$backup_file" else size=$(du -h "$backup_file" | cut -f1) size_mb=$(format_size_mb "$backup_file") echo " ✓ Saved (encrypted): $(basename "$backup_file") ($size)" | tee -a "$LOG_FILE" echo " SQL: $(basename "$backup_file") - ${size_mb} Mo" >> "$RECAP_FILE" fi else echo " ERROR: Backup file not created: $database" | tee -a "$LOG_FILE" ((ERROR_COUNT++)) fi else # Analyze MySQL connection test for failed backup if [[ "$MYSQL_TEST" == *"Access denied"* ]]; then echo " ERROR: MySQL authentication failed for $database on $host_name/$container_name" | tee -a "$LOG_FILE" echo " User: $db_user@$db_host - Check password in configuration" | tee -a "$LOG_FILE" elif [[ "$MYSQL_TEST" == *"Unknown database"* ]]; then echo " ERROR: Database '$database' does not exist on $host_name/$container_name" | tee -a "$LOG_FILE" elif [[ "$MYSQL_TEST" == *"Can't connect"* ]]; then echo " ERROR: Cannot connect to MySQL server at $db_host in $container_name" | tee -a "$LOG_FILE" else echo " ERROR: Failed to backup database $database on $host_name/$container_name" | tee -a "$LOG_FILE" fi ((ERROR_COUNT++)) rm -f "$backup_file" fi } # Process each host host_count=$(yq '.hosts | length' "$CONFIG_FILE") for ((i=0; i<$host_count; i++)); do host_name=$(yq ".hosts[$i].name" "$CONFIG_FILE" | tr -d '"') host_ip=$(yq ".hosts[$i].ip" "$CONFIG_FILE" | tr -d '"') ssh_user=$(yq ".hosts[$i].user" "$CONFIG_FILE" | tr -d '"') ssh_key=$(yq ".hosts[$i].key" "$CONFIG_FILE" | tr -d '"') ssh_port=$(yq ".hosts[$i].port // 22" "$CONFIG_FILE" | tr -d '"') echo "Processing host: $host_name ($host_ip)" | tee -a "$LOG_FILE" echo "" >> "$RECAP_FILE" echo "HOST: $host_name ($host_ip)" >> "$RECAP_FILE" echo "----------------------------" >> "$RECAP_FILE" # Test SSH connection if ! ssh -i "$ssh_key" -p "$ssh_port" -o ConnectTimeout=20 -o StrictHostKeyChecking=no "$ssh_user@$host_ip" "true" 2>/dev/null; then echo " ERROR: Cannot connect to $host_name ($host_ip)" | tee -a "$LOG_FILE" ((ERROR_COUNT++)) continue fi # Process containers container_count=$(yq ".hosts[$i].containers | length" "$CONFIG_FILE" 2>/dev/null || echo "0") for ((c=0; c<$container_count; c++)); do container_name=$(yq ".hosts[$i].containers[$c].name" "$CONFIG_FILE" | tr -d '"') echo " Processing container: $container_name" | tee -a "$LOG_FILE" # Add container to recap echo " Container: $container_name" >> "$RECAP_FILE" # Create backup directories backup_dir="$DIR_BACKUP/$host_name/$container_name" mkdir -p "$backup_dir" mkdir -p "$backup_dir/sql" # Backup directories (skip if -onlydb mode) if [[ "$ONLY_DB" == "false" ]]; then dir_count=$(yq ".hosts[$i].containers[$c].dirs | length" "$CONFIG_FILE" 2>/dev/null || echo "0") for ((d=0; d<$dir_count; d++)); do dir_path=$(yq ".hosts[$i].containers[$c].dirs[$d]" "$CONFIG_FILE" | sed 's/^"\|"$//g') # Use sudo if not root if [[ "$ssh_user" != "root" ]]; then CMD_PREFIX="sudo" else CMD_PREFIX="" fi # Special handling for /var/www - backup each subdirectory separately if [[ "$dir_path" == "/var/www" ]]; then echo " Backing up subdirectories of $dir_path" | tee -a "$LOG_FILE" # Get list of subdirectories subdirs=$(ssh -i "$ssh_key" -p "$ssh_port" -o ConnectTimeout=20 "$ssh_user@$host_ip" \ "$CMD_PREFIX incus exec $container_name -- find /var/www -maxdepth 1 -type d ! -path /var/www" 2>/dev/null || echo "") for subdir in $subdirs; do subdir_name=$(basename "$subdir" | tr '/' '_') backup_file="$backup_dir/www_${subdir_name}_$(date +%Y%m%d_%H).tar.gz" echo " Backing up: $subdir" | tee -a "$LOG_FILE" if ssh -i "$ssh_key" -p "$ssh_port" -o ConnectTimeout=20 "$ssh_user@$host_ip" \ "$CMD_PREFIX incus exec $container_name -- tar czf - $subdir 2>/dev/null" > "$backup_file"; then # Validate backup file size (tar.gz should be > 1KB) if [[ -f "$backup_file" ]]; then file_size=$(stat -c%s "$backup_file" 2>/dev/null || echo 0) if [[ $file_size -lt 1024 ]]; then echo " WARNING: Backup file very small (${file_size} bytes): $subdir" | tee -a "$LOG_FILE" # Keep the file but note it's small size=$(du -h "$backup_file" | cut -f1) size_mb=$(format_size_mb "$backup_file") echo " ✓ Saved (small): $(basename "$backup_file") ($size)" | tee -a "$LOG_FILE" echo " DIR: $(basename "$backup_file") - ${size_mb} Mo (WARNING: small)" >> "$RECAP_FILE" else size=$(du -h "$backup_file" | cut -f1) size_mb=$(format_size_mb "$backup_file") echo " ✓ Saved: $(basename "$backup_file") ($size)" | tee -a "$LOG_FILE" echo " DIR: $(basename "$backup_file") - ${size_mb} Mo" >> "$RECAP_FILE" fi else echo " ERROR: Backup file not created: $subdir" | tee -a "$LOG_FILE" ((ERROR_COUNT++)) fi else echo " ERROR: Failed to backup $subdir" | tee -a "$LOG_FILE" ((ERROR_COUNT++)) rm -f "$backup_file" fi done else # Normal backup for other directories dir_name=$(basename "$dir_path" | tr '/' '_') backup_file="$backup_dir/${dir_name}_$(date +%Y%m%d_%H).tar.gz" echo " Backing up: $dir_path" | tee -a "$LOG_FILE" if ssh -i "$ssh_key" -p "$ssh_port" -o ConnectTimeout=20 "$ssh_user@$host_ip" \ "$CMD_PREFIX incus exec $container_name -- tar czf - $dir_path 2>/dev/null" > "$backup_file"; then # Validate backup file size (tar.gz should be > 1KB) if [[ -f "$backup_file" ]]; then file_size=$(stat -c%s "$backup_file" 2>/dev/null || echo 0) if [[ $file_size -lt 1024 ]]; then echo " WARNING: Backup file very small (${file_size} bytes): $dir_path" | tee -a "$LOG_FILE" # Keep the file but note it's small size=$(du -h "$backup_file" | cut -f1) size_mb=$(format_size_mb "$backup_file") echo " ✓ Saved (small): $(basename "$backup_file") ($size)" | tee -a "$LOG_FILE" echo " DIR: $(basename "$backup_file") - ${size_mb} Mo (WARNING: small)" >> "$RECAP_FILE" else size=$(du -h "$backup_file" | cut -f1) size_mb=$(format_size_mb "$backup_file") echo " ✓ Saved: $(basename "$backup_file") ($size)" | tee -a "$LOG_FILE" echo " DIR: $(basename "$backup_file") - ${size_mb} Mo" >> "$RECAP_FILE" fi else echo " ERROR: Backup file not created: $dir_path" | tee -a "$LOG_FILE" ((ERROR_COUNT++)) fi else echo " ERROR: Failed to backup $dir_path" | tee -a "$LOG_FILE" ((ERROR_COUNT++)) rm -f "$backup_file" fi fi done fi # End of directory backup section # Backup databases db_user=$(yq ".hosts[$i].containers[$c].db_user" "$CONFIG_FILE" 2>/dev/null | tr -d '"') db_pass=$(yq ".hosts[$i].containers[$c].db_pass" "$CONFIG_FILE" 2>/dev/null | tr -d '"') db_host=$(yq ".hosts[$i].containers[$c].db_host // \"localhost\"" "$CONFIG_FILE" 2>/dev/null | tr -d '"') # Check if we're in onlydb mode if [[ "$ONLY_DB" == "true" ]]; then # Use onlydb list if it exists onlydb_count=$(yq ".hosts[$i].containers[$c].onlydb | length" "$CONFIG_FILE" 2>/dev/null || echo "0") if [[ "$onlydb_count" != "0" ]] && [[ "$onlydb_count" != "null" ]]; then db_count="$onlydb_count" use_onlydb=true else # No onlydb list, skip this container in onlydb mode continue fi else # Normal mode - use databases list db_count=$(yq ".hosts[$i].containers[$c].databases | length" "$CONFIG_FILE" 2>/dev/null || echo "0") use_onlydb=false fi if [[ -n "$db_user" ]] && [[ -n "$db_pass" ]] && [[ "$db_count" != "0" ]]; then for ((db=0; db<$db_count; db++)); do if [[ "$use_onlydb" == "true" ]]; then db_name=$(yq ".hosts[$i].containers[$c].onlydb[$db]" "$CONFIG_FILE" | tr -d '"') else db_name=$(yq ".hosts[$i].containers[$c].databases[$db]" "$CONFIG_FILE" | tr -d '"') fi if [[ "$db_name" == "ALL" ]]; then echo " Fetching all databases..." | tee -a "$LOG_FILE" # Get database list if [[ "$ssh_user" != "root" ]]; then db_list=$(ssh -i "$ssh_key" -p "$ssh_port" -o ConnectTimeout=20 "$ssh_user@$host_ip" \ "sudo incus exec $container_name -- mariadb -h $db_host -u$db_user -p$db_pass -e 'SHOW DATABASES;' 2>/dev/null" | \ grep -Ev '^(Database|information_schema|performance_schema|mysql|sys)$' || echo "") else db_list=$(ssh -i "$ssh_key" -p "$ssh_port" -o ConnectTimeout=20 "$ssh_user@$host_ip" \ "incus exec $container_name -- mariadb -h $db_host -u$db_user -p$db_pass -e 'SHOW DATABASES;' 2>/dev/null" | \ grep -Ev '^(Database|information_schema|performance_schema|mysql|sys)$' || echo "") fi # Backup each database for single_db in $db_list; do backup_database "$single_db" done else backup_database "$db_name" fi done fi done done echo "=== Backup Completed $(date) ===" | tee -a "$LOG_FILE" # Show summary total_size=$(du -sh "$DIR_BACKUP" 2>/dev/null | cut -f1) echo "Total backup size: $total_size" | tee -a "$LOG_FILE" # Add summary to recap echo "" >> "$RECAP_FILE" echo "========================================" >> "$RECAP_FILE" # Add size details per host/container echo "BACKUP SIZES:" >> "$RECAP_FILE" for host_dir in "$DIR_BACKUP"/*; do if [[ -d "$host_dir" ]]; then host_name=$(basename "$host_dir") host_size=$(du -sh "$host_dir" 2>/dev/null | cut -f1) echo " $host_name: $host_size" >> "$RECAP_FILE" # Size per container for container_dir in "$host_dir"/*; do if [[ -d "$container_dir" ]]; then container_name=$(basename "$container_dir") container_size=$(du -sh "$container_dir" 2>/dev/null | cut -f1) echo " - $container_name: $container_size" >> "$RECAP_FILE" fi done fi done echo "" >> "$RECAP_FILE" echo "TOTAL SIZE: $total_size" >> "$RECAP_FILE" echo "COMPLETED: $(date '+%d.%m.%Y %H:%M')" >> "$RECAP_FILE" # Prepare email subject with date format DATE_SUBJECT=$(date '+%d.%m.%Y %H') # Send recap email if [[ $ERROR_COUNT -gt 0 ]]; then echo "Total errors: $ERROR_COUNT" | tee -a "$LOG_FILE" # Add errors to recap echo "" >> "$RECAP_FILE" echo "ERRORS DETECTED: $ERROR_COUNT" >> "$RECAP_FILE" echo "----------------------------" >> "$RECAP_FILE" grep -i "ERROR" "$LOG_FILE" >> "$RECAP_FILE" # Send email with ERROR in subject echo "Sending ERROR email to $EMAIL_TO (Errors found: $ERROR_COUNT)" | tee -a "$LOG_FILE" if command -v msmtp &> /dev/null; then { echo "To: $EMAIL_TO" echo "Subject: BackupPM7 ERROR $DATE_SUBJECT" echo "" cat "$RECAP_FILE" } | msmtp "$EMAIL_TO" echo "ERROR email sent successfully to $EMAIL_TO" | tee -a "$LOG_FILE" else echo "WARNING: msmtp not found - ERROR email NOT sent" | tee -a "$LOG_FILE" fi else echo "Backup completed successfully with no errors" | tee -a "$LOG_FILE" # Send success recap email echo "Sending SUCCESS recap email to $EMAIL_TO" | tee -a "$LOG_FILE" if command -v msmtp &> /dev/null; then { echo "To: $EMAIL_TO" echo "Subject: BackupPM7 $DATE_SUBJECT" echo "" cat "$RECAP_FILE" } | msmtp "$EMAIL_TO" echo "SUCCESS recap email sent successfully to $EMAIL_TO" | tee -a "$LOG_FILE" else echo "WARNING: msmtp not found - SUCCESS recap email NOT sent" | tee -a "$LOG_FILE" fi fi # Clean up recap file rm -f "$RECAP_FILE" # Exit with error code if there were errors if [[ $ERROR_COUNT -gt 0 ]]; then exit 1 fi