upgrade-manager.sh

#!/bin/bash

# ClickHouse Version Upgrade Manager
# Handles safe upgrades with data preservation

set -e

# Load environment variables safely
if [ -f .env-config ]; then
    set -a  # Automatically export all variables
    source .env-config
    set +a  # Turn off automatic export
fi

# Add Docker Desktop to PATH if it exists
if [ -d "/Applications/Docker.app/Contents/Resources/bin" ]; then
    export PATH="/Applications/Docker.app/Contents/Resources/bin:$PATH"
fi

# Auto-detect Docker and Docker Compose
if command -v docker &> /dev/null; then
    DOCKER_CMD="docker"
    if docker compose version &> /dev/null; then
        DOCKER_COMPOSE_CMD="docker compose"
    elif command -v docker-compose &> /dev/null; then
        DOCKER_COMPOSE_CMD="docker-compose"
    else
        echo "❌ docker compose or docker-compose not found"
        exit 1
    fi
else
    echo "❌ Docker not found"
    exit 1
fi

# Configuration from environment variables with fallbacks
CONTAINER_NAME="${CONTAINER_NAME:-clickhouse-server-hot-warm}"
BACKUP_DIR="${BACKUP_STORAGE_PATH:-/Volumes/Extreme-Pro/clickhouse-backup}/$(date +%Y%m%d_%H%M%S)"
HOT_STORAGE_PATH="${HOT_STORAGE_PATH:-$HOME/Downloads/clickhouse-hot}"
WARM_STORAGE_PATH="${WARM_STORAGE_PATH:-/Volumes/Extreme-Pro/clickhouse-warm}"

# Colors
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
BLUE='\033[0;34m'
NC='\033[0m'

print_status() { echo -e "${GREEN}✅ $1${NC}"; }
print_warning() { echo -e "${YELLOW}⚠️  $1${NC}"; }
print_error() { echo -e "${RED}❌ $1${NC}"; }
print_info() { echo -e "${BLUE}ℹ️  $1${NC}"; }

# Function to get current ClickHouse version
get_current_version() {
    if docker ps -q --filter name=$CONTAINER_NAME | grep -q .; then
        local version=$($DOCKER_CMD exec $CONTAINER_NAME clickhouse-client --query "SELECT version()" 2>/dev/null || echo "unknown")
        echo $version
    else
        echo "container_not_running"
    fi
}

# Function to backup data before upgrade
backup_data() {
    print_info "Creating backup before upgrade..."
    
    # Create backup directory
    mkdir -p "$BACKUP_DIR/hot"
    mkdir -p "$BACKUP_DIR/warm"
    mkdir -p "$BACKUP_DIR/metadata"
    
    # Stop ClickHouse gracefully
    print_info "Stopping ClickHouse gracefully..."
    $DOCKER_COMPOSE_CMD stop
    
    # Backup storage directories
    print_info "Backing up hot storage..."
    if [ -d "$HOT_STORAGE_PATH" ]; then
        rsync -av "$HOT_STORAGE_PATH/" "$BACKUP_DIR/hot/"
    fi
    
    print_info "Backing up warm storage..."
    if [ -d "$WARM_STORAGE_PATH" ]; then
        rsync -av "$WARM_STORAGE_PATH/" "$BACKUP_DIR/warm/"
    fi
    
    # Export metadata and schemas
    print_info "Starting temporary container for metadata export..."
    $DOCKER_COMPOSE_CMD up -d
    sleep 30  # Wait for startup
    
    # Export database schemas and metadata
    $DOCKER_CMD exec $CONTAINER_NAME clickhouse-client --query="SHOW DATABASES" > "$BACKUP_DIR/metadata/databases.sql"
    $DOCKER_CMD exec $CONTAINER_NAME clickhouse-client --query="SELECT create_table_query FROM system.tables WHERE database != 'system'" > "$BACKUP_DIR/metadata/tables.sql"
    $DOCKER_CMD exec $CONTAINER_NAME clickhouse-client --query="SELECT * FROM system.storage_policies FORMAT TabSeparated" > "$BACKUP_DIR/metadata/storage_policies.tsv"
    
    # Stop again for upgrade
    $DOCKER_COMPOSE_CMD stop
    
    print_status "Backup completed: $BACKUP_DIR"
}

# Function to update ClickHouse version
update_version() {
    local new_version="$1"
    
    if [ -z "$new_version" ]; then
        print_error "Please specify the new ClickHouse version"
        echo "Usage: $0 upgrade <version>"
        echo "Example: $0 upgrade 25.8.1"
        exit 1
    fi
    
    print_info "Updating ClickHouse to version $new_version..."
    
    # Update .env-config file with new version
    if [ -f ".env-config" ]; then
        sed -i.backup "s/CLICKHOUSE_VERSION=.*/CLICKHOUSE_VERSION=${new_version}/" .env-config
        print_status "Updated .env-config file to use version $new_version"
    else
        # Fallback: Update Dockerfile directly if no .env-config file
        if [ -f "Dockerfile" ]; then
            sed -i.backup "s/ARG CLICKHOUSE_VERSION=.*/ARG CLICKHOUSE_VERSION=${new_version}/" Dockerfile
            print_status "Updated Dockerfile to use version $new_version"
        fi
    fi
    
    # Rebuild and start with new version
    print_info "Building new image..."
    $DOCKER_COMPOSE_CMD build --no-cache
    
    print_info "Starting with new version..."
    $DOCKER_COMPOSE_CMD up -d
    
    # Wait for startup and verify
    print_info "Waiting for ClickHouse to start..."
    sleep 30
    
    # Check if upgrade was successful
    local current_version=$(get_current_version)
    if [[ "$current_version" == *"$new_version"* ]]; then
        print_status "Successfully upgraded to ClickHouse $current_version"
        verify_data_integrity
    else
        print_error "Upgrade may have failed. Current version: $current_version"
        print_warning "Check logs with: ./manage.sh logs"
    fi
}

# Function to verify data integrity after upgrade
verify_data_integrity() {
    print_info "Verifying data integrity..."
    
    # Wait for ClickHouse to be ready
    local retries=0
    while [ $retries -lt 30 ]; do
        if curl -s "http://localhost:8123/ping" | grep -q "Ok"; then
            break
        fi
        sleep 2
        retries=$((retries + 1))
    done
    
    if [ $retries -eq 30 ]; then
        print_error "ClickHouse is not responding after upgrade"
        return 1
    fi
    
    # Basic connectivity test
    local test_result=$(curl -s "http://localhost:8123/?query=SELECT 1" 2>/dev/null || echo "failed")
    if [ "$test_result" = "1" ]; then
        print_status "Basic connectivity: OK"
    else
        print_error "Basic connectivity: FAILED"
        return 1
    fi
    
    # Check storage policies
    local policy_count=$(curl -s "http://localhost:8123/?query=SELECT count() FROM system.storage_policies" 2>/dev/null || echo "0")
    if [ "$policy_count" -ge "3" ]; then
        print_status "Storage policies: OK ($policy_count policies)"
    else
        print_warning "Storage policies: Only $policy_count policies found"
    fi
    
    # Check database count
    local db_count=$(curl -s "http://localhost:8123/?query=SELECT count() FROM system.databases WHERE name NOT IN ('system', 'information_schema', 'INFORMATION_SCHEMA')" 2>/dev/null || echo "0")
    print_info "User databases: $db_count"
    
    print_status "Data integrity verification completed"
}

# Function to rollback to previous version
rollback() {
    print_warning "Rolling back to previous version..."
    
    # Stop current version
    $DOCKER_COMPOSE_CMD stop
    
    # Restore .env-config backup if it exists, otherwise restore Dockerfile backup
    if [ -f ".env-config.backup" ]; then
        mv .env-config.backup .env-config
        print_status "Restored previous .env-config file"
    elif [ -f "Dockerfile.backup" ]; then
        mv Dockerfile.backup Dockerfile
        print_status "Restored previous Dockerfile"
    fi
    
    # Find latest backup
    local latest_backup=$(ls -1 "${BACKUP_STORAGE_PATH:-/Volumes/Extreme-Pro/clickhouse-backup}/" 2>/dev/null | tail -1)
    if [ -n "$latest_backup" ]; then
        print_info "Restoring from backup: $latest_backup"
        
        # Restore data
        if [ -d "${BACKUP_STORAGE_PATH:-/Volumes/Extreme-Pro/clickhouse-backup}/$latest_backup/hot" ]; then
            rsync -av "${BACKUP_STORAGE_PATH:-/Volumes/Extreme-Pro/clickhouse-backup}/$latest_backup/hot/" "$HOT_STORAGE_PATH/"
        fi
        
        if [ -d "${BACKUP_STORAGE_PATH:-/Volumes/Extreme-Pro/clickhouse-backup}/$latest_backup/warm" ]; then
            rsync -av "${BACKUP_STORAGE_PATH:-/Volumes/Extreme-Pro/clickhouse-backup}/$latest_backup/warm/" "$WARM_STORAGE_PATH/"
        fi
        
        print_status "Data restored from backup"
    fi
    
    # Rebuild and start
    $DOCKER_COMPOSE_CMD build --no-cache
    $DOCKER_COMPOSE_CMD up -d
    
    print_status "Rollback completed"
}

# Function to list available backups
list_backups() {
    print_info "Available backups:"
    local backup_dir="${BACKUP_STORAGE_PATH:-/Volumes/Extreme-Pro/clickhouse-backup}"
    if [ -d "$backup_dir" ]; then
        ls -la "$backup_dir/" | grep "^d" | awk '{print $9, $6, $7, $8}'
    else
        print_warning "No backup directory found at $backup_dir"
    fi
}

# Function to show current version info
version_info() {
    local current_version=$(get_current_version)
    print_info "Current ClickHouse version: $current_version"
    
    # Show data statistics
    if [[ "$current_version" != "container_not_running" ]]; then
        local memory_usage=$(curl -s "http://localhost:8123/?query=SELECT formatReadableSize(value::UInt64) FROM system.server_settings WHERE name='max_server_memory_usage'" 2>/dev/null || echo "unknown")
        local db_count=$(curl -s "http://localhost:8123/?query=SELECT count() FROM system.databases" 2>/dev/null || echo "unknown")
        local table_count=$(curl -s "http://localhost:8123/?query=SELECT count() FROM system.tables WHERE database != 'system'" 2>/dev/null || echo "unknown")
        
        print_info "Memory limit: $memory_usage"
        print_info "Databases: $db_count"
        print_info "User tables: $table_count"
    fi
}

# Main script logic
case "${1:-help}" in
    upgrade)
        current_version=$(get_current_version)
        print_info "Current version: $current_version"
        backup_data
        update_version "$2"
        ;;
    rollback)
        rollback
        ;;
    backup)
        backup_data
        ;;
    verify)
        verify_data_integrity
        ;;
    list-backups)
        list_backups
        ;;
    version)
        version_info
        ;;
    *)
        echo "ClickHouse Upgrade Manager"
        echo ""
        echo "Usage: $0 {upgrade|rollback|backup|verify|list-backups|version}"
        echo ""
        echo "Commands:"
        echo "  upgrade <version>  - Upgrade ClickHouse to specified version with backup"
        echo "  rollback           - Rollback to previous version from backup"
        echo "  backup            - Create manual backup of current data"
        echo "  verify            - Verify data integrity and system health"
        echo "  list-backups      - List available backups"
        echo "  version           - Show current version and statistics"
        echo ""
        echo "Examples:"
        echo "  $0 upgrade 25.8.1  # Upgrade to ClickHouse 25.8.1"
        echo "  $0 backup          # Create manual backup"
        echo "  $0 rollback        # Rollback to previous version"
        ;;
esac