#!/bin/bash # Backup Verification Script # Verifies database backups are working correctly set -e # Colors GREEN='\033[0;32m' RED='\033[0;31m' YELLOW='\033[1;33m' NC='\033[0m' log_info() { echo -e "${GREEN}[INFO]${NC} $1" } log_error() { echo -e "${RED}[ERROR]${NC} $1" } log_warn() { echo -e "${YELLOW}[WARN]${NC} $1" } # Configuration BACKUP_DIR="${BACKUP_DIR:-/backups/postgres}" RETENTION_DAYS="${RETENTION_DAYS:-7}" DB_NAME="${DB_NAME:-sankofa}" # Check if kubectl is available if ! command -v kubectl &> /dev/null; then log_error "kubectl not found. This script requires kubectl access." exit 1 fi # Verify backup directory exists verify_backup_directory() { log_info "Verifying backup directory..." if [ -d "$BACKUP_DIR" ]; then test_pass "Backup directory exists" else test_fail "Backup directory does not exist: $BACKUP_DIR" return 1 fi } # Check recent backups check_recent_backups() { log_info "Checking for recent backups..." local recent_backups=$(find "$BACKUP_DIR" -name "*.sql" -o -name "*.dump" -mtime -1 2>/dev/null | wc -l) if [ $recent_backups -gt 0 ]; then log_info "Found $recent_backups backup(s) from last 24 hours" test_pass "Recent backups exist" else test_fail "No backups found in last 24 hours" return 1 fi } # Verify backup integrity verify_backup_integrity() { log_info "Verifying backup integrity..." local latest_backup=$(find "$BACKUP_DIR" -name "*.sql" -o -name "*.dump" -type f -printf '%T@ %p\n' | sort -n | tail -1 | cut -d' ' -f2-) if [ -z "$latest_backup" ]; then test_fail "No backup file found" return 1 fi log_info "Testing backup: $latest_backup" # Check if file is not empty if [ -s "$latest_backup" ]; then test_pass "Backup file is not empty" else test_fail "Backup file is empty" return 1 fi # Check if file is readable if [ -r "$latest_backup" ]; then test_pass "Backup file is readable" else test_fail "Backup file is not readable" return 1 fi # For SQL dumps, check for basic SQL structure if [[ "$latest_backup" == *.sql ]]; then if grep -q "CREATE TABLE\|INSERT INTO" "$latest_backup" 2>/dev/null; then test_pass "Backup file contains SQL data" else log_warn "Backup file may be incomplete (no SQL statements found)" fi fi } # Check backup retention check_backup_retention() { log_info "Checking backup retention policy..." local old_backups=$(find "$BACKUP_DIR" -name "*.sql" -o -name "*.dump" -mtime +$RETENTION_DAYS 2>/dev/null | wc -l) if [ $old_backups -gt 0 ]; then log_warn "Found $old_backups backup(s) older than $RETENTION_DAYS days" log_warn "Consider cleaning up old backups" else test_pass "Backup retention policy is being followed" fi } # Test backup restoration (dry run) test_backup_restore() { log_info "Testing backup restoration (dry run)..." local latest_backup=$(find "$BACKUP_DIR" -name "*.sql" -o -name "*.dump" -type f -printf '%T@ %p\n' | sort -n | tail -1 | cut -d' ' -f2-) if [ -z "$latest_backup" ]; then test_fail "No backup file found for restore test" return 1 fi # For SQL dumps, check if we can parse the file if [[ "$latest_backup" == *.sql ]]; then if head -100 "$latest_backup" | grep -q "PostgreSQL\|pg_dump" 2>/dev/null; then test_pass "Backup file appears to be a valid PostgreSQL dump" else log_warn "Backup file format may be unexpected" fi fi } # Check automated backup schedule check_backup_schedule() { log_info "Checking backup automation..." # Check for cron jobs or Kubernetes CronJobs if kubectl get cronjob -n api postgres-backup &> /dev/null; then local schedule=$(kubectl get cronjob -n api postgres-backup -o jsonpath='{.spec.schedule}') log_info "Found backup CronJob with schedule: $schedule" test_pass "Backup automation configured" else log_warn "No backup CronJob found. Backups may not be automated." log_warn "Consider creating a CronJob for automated backups." fi } # Verify database connectivity verify_db_connectivity() { log_info "Verifying database connectivity..." if kubectl exec -n api deployment/api -- \ psql "${DATABASE_URL}" -c "SELECT 1" > /dev/null 2>&1; then test_pass "Database is accessible" else test_fail "Cannot connect to database" return 1 fi } # Main execution main() { echo "==========================================" echo "Sankofa Phoenix Backup Verification" echo "==========================================" echo "" local passed=0 local failed=0 verify_backup_directory && ((passed++)) || ((failed++)) check_recent_backups && ((passed++)) || ((failed++)) verify_backup_integrity && ((passed++)) || ((failed++)) check_backup_retention && ((passed++)) || ((failed++)) test_backup_restore && ((passed++)) || ((failed++)) check_backup_schedule && ((passed++)) || ((failed++)) verify_db_connectivity && ((passed++)) || ((failed++)) echo "" echo "==========================================" echo "Verification Summary" echo "==========================================" echo "Passed: ${GREEN}${passed}${NC}" echo "Failed: ${RED}${failed}${NC}" echo "" if [ $failed -eq 0 ]; then log_info "All backup verifications passed!" exit 0 else log_error "Some verifications failed. Please review backup configuration." exit 1 fi } main "$@"