Skip to content

Rollback Production

Rollback Production #2

# =============================================================================
# Production Rollback Workflow (Enhanced)
# =============================================================================
#
# Usage:
# 1. Go to Actions → "Rollback Production"
# 2. Select service(s) to rollback
# 3. Choose rollback type (previous version or specific tag)
# 4. Type "ROLLBACK" to confirm
# 5. Wait for approval (if environment protection enabled)
# =============================================================================
name: Rollback Production
on:
workflow_dispatch:
inputs:
service:
description: 'Service to rollback'
required: true
type: choice
options:
- frontend
- user_service
- appointment_service
- service_management
- staff_management
- notification_service
- reports_analytics
- all
rollback_type:
description: 'Rollback type'
required: true
type: choice
options:
- previous # Go back to previous available tag
- specific # Use specific tag from input
default: 'previous'
specific_tag:
description: 'Specific tag (only if rollback_type=specific)'
required: false
type: string
confirm_rollback:
description: 'Type "ROLLBACK" to confirm'
required: true
type: string
env:
AWS_REGION: us-east-1
ECR_REGISTRY: '024955634588.dkr.ecr.us-east-1.amazonaws.com'
permissions:
contents: write
issues: write
jobs:
# ═══════════════════════════════════════════════════════════════════════════
# STEP 1: Validate the rollback request
# ═══════════════════════════════════════════════════════════════════════════
step1-validate:
name: Validate Request
runs-on: ubuntu-latest
outputs:
services: ${{ steps.parse.outputs.services }}
services_json: ${{ steps.parse.outputs.services_json }}
steps:
- name: Verify ROLLBACK confirmation
if: ${{ github.event.inputs.confirm_rollback != 'ROLLBACK' }}
run: |
echo "::error::❌ Rollback not confirmed. You must type 'ROLLBACK' exactly."
exit 1
- name: Parse services to rollback
id: parse
run: |
INPUT="${{ github.event.inputs.service }}"
if [ "$INPUT" == "all" ]; then
SERVICES="frontend user_service appointment_service service_management staff_management notification_service reports_analytics"
else
SERVICES="$INPUT"
fi
echo "✅ Services to rollback: $SERVICES"
echo "services=$SERVICES" >> $GITHUB_OUTPUT
# Create JSON array for matrix (if needed later)
JSON=$(echo "$SERVICES" | tr ' ' '\n' | jq -R . | jq -s .)
echo "services_json=$JSON" >> $GITHUB_OUTPUT
- name: Summary
run: |
echo "## Validation Passed" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "| Check | Status |" >> $GITHUB_STEP_SUMMARY
echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY
echo "| ROLLBACK confirmation | ✅ Confirmed |" >> $GITHUB_STEP_SUMMARY
echo "| Services | \`${{ github.event.inputs.service }}\` |" >> $GITHUB_STEP_SUMMARY
echo "| Rollback Type | \`${{ github.event.inputs.rollback_type }}\` |" >> $GITHUB_STEP_SUMMARY
echo "| Initiated By | @${{ github.actor }} |" >> $GITHUB_STEP_SUMMARY
# ═══════════════════════════════════════════════════════════════════════════
# STEP 2: Show current production versions
# ═══════════════════════════════════════════════════════════════════════════
step2-current-versions:
name: Check Current Versions
runs-on: ubuntu-latest
needs: step1-validate
outputs:
current_tags: ${{ steps.check.outputs.current_tags }}
steps:
- name: Checkout GitOps repo
uses: actions/checkout@v4
- name: Get current production tags
id: check
run: |
SERVICES="${{ needs.step1-validate.outputs.services }}"
echo "## Current Production Versions" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "| Service | Current Tag | Deployed Since |" >> $GITHUB_STEP_SUMMARY
echo "|---------|-------------|----------------|" >> $GITHUB_STEP_SUMMARY
CURRENT_TAGS=""
for SERVICE in $SERVICES; do
PROD_FILE="production/${SERVICE}/deployment.yaml"
if [ -f "$PROD_FILE" ]; then
CURRENT_TAG=$(grep "image:" "$PROD_FILE" | head -1 | awk -F: '{print $NF}' | tr -d ' ')
# Extract date from tag (format: abc12345-20251214185501)
TAG_DATE=$(echo "$CURRENT_TAG" | grep -oE '[0-9]{14}' | head -1)
if [ -n "$TAG_DATE" ]; then
FORMATTED_DATE="${TAG_DATE:0:4}-${TAG_DATE:4:2}-${TAG_DATE:6:2} ${TAG_DATE:8:2}:${TAG_DATE:10:2}"
else
FORMATTED_DATE="Unknown"
fi
echo "| $SERVICE | \`$CURRENT_TAG\` | $FORMATTED_DATE |" >> $GITHUB_STEP_SUMMARY
CURRENT_TAGS="$CURRENT_TAGS$SERVICE:$CURRENT_TAG,"
else
echo "| $SERVICE | ⚠️ No deployment file | - |" >> $GITHUB_STEP_SUMMARY
fi
done
echo "current_tags=$CURRENT_TAGS" >> $GITHUB_OUTPUT
# ═══════════════════════════════════════════════════════════════════════════
# STEP 3: List available images in ECR
# ═══════════════════════════════════════════════════════════════════════════
step3-available-images:
name: List ECR Images
runs-on: ubuntu-latest
needs: [step1-validate, step2-current-versions]
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ env.AWS_REGION }}
- name: List available images in ECR
run: |
SERVICES="${{ needs.step1-validate.outputs.services }}"
echo "## Available Images in ECR" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "_These are the images you can rollback to:_" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
for SERVICE in $SERVICES; do
echo "### $SERVICE" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "| # | Tag | Pushed Date |" >> $GITHUB_STEP_SUMMARY
echo "|---|-----|-------------|" >> $GITHUB_STEP_SUMMARY
# Get last 5 images with dates
aws ecr describe-images \
--repository-name "$SERVICE" \
--query 'sort_by(imageDetails,&imagePushedAt)[-5:].{Tag:imageTags[0],Pushed:imagePushedAt}' \
--output json 2>/dev/null | jq -r 'reverse | to_entries[] | "| \(.key + 1) | `\(.value.Tag // "untagged")` | \(.value.Pushed[:19]) |"' >> $GITHUB_STEP_SUMMARY || echo "| - | No images found | - |" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
done
# ═══════════════════════════════════════════════════════════════════════════
# STEP 4: Determine rollback target
# ═══════════════════════════════════════════════════════════════════════════
step4-determine-target:
name: Determine Rollback Target
runs-on: ubuntu-latest
needs: [step1-validate, step2-current-versions, step3-available-images]
outputs:
rollback_plan: ${{ steps.plan.outputs.rollback_plan }}
steps:
- name: Checkout GitOps repo
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ env.AWS_REGION }}
- name: Determine rollback targets with fallback
id: plan
run: |
SERVICES="${{ needs.step1-validate.outputs.services }}"
ROLLBACK_TYPE="${{ github.event.inputs.rollback_type }}"
SPECIFIC_TAG="${{ github.event.inputs.specific_tag }}"
echo "## Rollback Target Selection" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "| Service | Current | Target | Source | Status |" >> $GITHUB_STEP_SUMMARY
echo "|---------|---------|--------|--------|--------|" >> $GITHUB_STEP_SUMMARY
ROLLBACK_PLAN=""
for SERVICE in $SERVICES; do
echo " Analyzing: $SERVICE"
PROD_FILE="production/${SERVICE}/deployment.yaml"
if [ ! -f "$PROD_FILE" ]; then
echo "| $SERVICE | - | - | - | ⚠️ No file |" >> $GITHUB_STEP_SUMMARY
continue
fi
# Get current tag
CURRENT_TAG=$(grep "image:" "$PROD_FILE" | head -1 | awk -F: '{print $NF}' | tr -d ' ')
ROLLBACK_TAG=""
SOURCE=""
if [ "$ROLLBACK_TYPE" == "specific" ] && [ -n "$SPECIFIC_TAG" ]; then
# User specified exact tag
ROLLBACK_TAG="$SPECIFIC_TAG"
SOURCE="User specified"
else
# ════════════════════════════════════════════════════════════
# FALLBACK LOGIC
# ════════════════════════════════════════════════════════════
# Method 1: Try git history first
echo " → Trying git history..."
GIT_TAG=$(git log --oneline -n 2 --follow -- "$PROD_FILE" 2>/dev/null | tail -1 | awk '{print $1}' | xargs -I {} git show {}:"$PROD_FILE" 2>/dev/null | grep "image:" | head -1 | awk -F: '{print $NF}' | tr -d ' ')
if [ -n "$GIT_TAG" ] && [ "$GIT_TAG" != "$CURRENT_TAG" ]; then
# Verify it exists in ECR
if aws ecr describe-images --repository-name "$SERVICE" --image-ids imageTag="$GIT_TAG" --region ${{ env.AWS_REGION }} > /dev/null 2>&1; then
ROLLBACK_TAG="$GIT_TAG"
SOURCE="Git history"
echo " ✅ Found in git history: $ROLLBACK_TAG"
else
echo " ⚠️ Git tag $GIT_TAG not in ECR, trying fallback..."
fi
fi
# Method 2: Get available tags from ECR (sorted by push date, newest first)
if [ -z "$ROLLBACK_TAG" ]; then
echo " → Trying ECR image list (fallback)..."
# Get all production tags sorted by date (newest first)
# Exclude: 'latest', 'staging-*' prefixed tags
ECR_TAGS=$(aws ecr describe-images \
--repository-name "$SERVICE" \
--query 'sort_by(imageDetails,&imagePushedAt)[*].imageTags[0]' \
--output text 2>/dev/null | tr '\t' '\n' | grep -v "^None$" | grep -v "^latest$" | grep -v "^staging-" | tac)
# Find the FIRST tag that:
# 1. Is different from current
# 2. Actually exists in ECR
FOUND_CURRENT=false
for TAG in $ECR_TAGS; do
if [ -z "$TAG" ]; then continue; fi
# Skip if this IS the current tag (we want the one BEFORE it)
if [ "$TAG" == "$CURRENT_TAG" ]; then
FOUND_CURRENT=true
echo " Skipping current: $TAG"
continue
fi
# If we found current and this is different, this is our target
if [ "$FOUND_CURRENT" = true ] || [ "$TAG" != "$CURRENT_TAG" ]; then
# Double-verify it exists in ECR
if aws ecr describe-images --repository-name "$SERVICE" --image-ids imageTag="$TAG" --region ${{ env.AWS_REGION }} > /dev/null 2>&1; then
ROLLBACK_TAG="$TAG"
SOURCE="ECR auto-detect"
echo " ✅ Found in ECR: $ROLLBACK_TAG"
break
else
echo " Tag $TAG not in ECR, trying next..."
fi
fi
done
fi
fi
# ════════════════════════════════════════════════════════════
# Final validation and record result
# ════════════════════════════════════════════════════════════
if [ -z "$ROLLBACK_TAG" ]; then
echo "| $SERVICE | \`$CURRENT_TAG\` | - | - | ❌ No target found |" >> $GITHUB_STEP_SUMMARY
ROLLBACK_PLAN="$ROLLBACK_PLAN$SERVICE:SKIP:$CURRENT_TAG:NONE,"
elif [ "$ROLLBACK_TAG" == "$CURRENT_TAG" ]; then
echo "| $SERVICE | \`$CURRENT_TAG\` | Same | - | ⚠️ Already at target |" >> $GITHUB_STEP_SUMMARY
ROLLBACK_PLAN="$ROLLBACK_PLAN$SERVICE:SKIP:$CURRENT_TAG:$ROLLBACK_TAG,"
else
# Final ECR verification
if aws ecr describe-images --repository-name "$SERVICE" --image-ids imageTag="$ROLLBACK_TAG" --region ${{ env.AWS_REGION }} > /dev/null 2>&1; then
echo "| $SERVICE | \`$CURRENT_TAG\` | \`$ROLLBACK_TAG\` | $SOURCE | ✅ Ready |" >> $GITHUB_STEP_SUMMARY
ROLLBACK_PLAN="$ROLLBACK_PLAN$SERVICE:OK:$CURRENT_TAG:$ROLLBACK_TAG,"
else
echo "| $SERVICE | \`$CURRENT_TAG\` | \`$ROLLBACK_TAG\` | $SOURCE | ❌ Not in ECR |" >> $GITHUB_STEP_SUMMARY
ROLLBACK_PLAN="$ROLLBACK_PLAN$SERVICE:FAIL:$CURRENT_TAG:$ROLLBACK_TAG,"
fi
fi
done
echo "rollback_plan=$ROLLBACK_PLAN" >> $GITHUB_OUTPUT
echo "" >> $GITHUB_STEP_SUMMARY
echo "---" >> $GITHUB_STEP_SUMMARY
echo "### Fallback Logic" >> $GITHUB_STEP_SUMMARY
echo "1. **Git history** - Find previous tag from commit history" >> $GITHUB_STEP_SUMMARY
echo "2. **ECR auto-detect** - Find next available image before current" >> $GITHUB_STEP_SUMMARY
echo "3. **Skip staging** - Ignores \`staging-*\` and \`latest\` tags" >> $GITHUB_STEP_SUMMARY
# ═══════════════════════════════════════════════════════════════════════════
# STEP 5: Wait for manual approval (if configured)
# ═══════════════════════════════════════════════════════════════════════════
step5-approval:
name: Approval Gate
runs-on: ubuntu-latest
needs: [step1-validate, step4-determine-target]
environment: production # Requires approval if protection rules set
steps:
- name: Approval received
run: |
echo "## Approval Gate" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "✅ **Approved by:** Environment protection rules passed" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "_If no approval was required, environment protection is not configured._" >> $GITHUB_STEP_SUMMARY
# ═══════════════════════════════════════════════════════════════════════════
# STEP 6: Execute the rollback
# ═══════════════════════════════════════════════════════════════════════════
step6-execute:
name: Execute Rollback
runs-on: ubuntu-latest
needs: [step1-validate, step4-determine-target, step5-approval]
outputs:
rollback_executed: ${{ steps.execute.outputs.rollback_executed }}
steps:
- name: Checkout GitOps repo
uses: actions/checkout@v4
with:
token: ${{ secrets.PAT_TOKEN }}
fetch-depth: 0
- name: Execute rollback changes
id: execute
run: |
ROLLBACK_PLAN="${{ needs.step4-determine-target.outputs.rollback_plan }}"
echo "## Rollback Execution" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "| Service | Action | From | To | Result |" >> $GITHUB_STEP_SUMMARY
echo "|---------|--------|------|-----|--------|" >> $GITHUB_STEP_SUMMARY
CHANGES_MADE=false
# Parse rollback plan: SERVICE:STATUS:CURRENT:TARGET,
IFS=',' read -ra ENTRIES <<< "$ROLLBACK_PLAN"
for ENTRY in "${ENTRIES[@]}"; do
if [ -z "$ENTRY" ]; then continue; fi
SERVICE=$(echo "$ENTRY" | cut -d: -f1)
STATUS=$(echo "$ENTRY" | cut -d: -f2)
CURRENT=$(echo "$ENTRY" | cut -d: -f3)
TARGET=$(echo "$ENTRY" | cut -d: -f4)
if [ "$STATUS" != "OK" ]; then
echo "| $SERVICE | Skip | \`$CURRENT\` | - | $STATUS |" >> $GITHUB_STEP_SUMMARY
continue
fi
PROD_FILE="production/${SERVICE}/deployment.yaml"
# Update the deployment file
sed -i "s|image: ${{ env.ECR_REGISTRY }}/${SERVICE}:.*|image: ${{ env.ECR_REGISTRY }}/${SERVICE}:${TARGET}|g" "$PROD_FILE"
echo "| $SERVICE | Rollback | \`$CURRENT\` | \`$TARGET\` | ✅ Updated |" >> $GITHUB_STEP_SUMMARY
CHANGES_MADE=true
done
echo "rollback_executed=$CHANGES_MADE" >> $GITHUB_OUTPUT
- name: Commit and push changes
run: |
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
git add -A
if git diff --staged --quiet; then
echo "" >> $GITHUB_STEP_SUMMARY
echo "### No Changes" >> $GITHUB_STEP_SUMMARY
echo "No deployment files were modified." >> $GITHUB_STEP_SUMMARY
else
SERVICES="${{ needs.step1-validate.outputs.services }}"
git commit -m "[ROLLBACK] $SERVICES - initiated by ${{ github.actor }} - run ${{ github.run_id }}"
git remote set-url origin https://x-access-token:${{ secrets.PAT_TOKEN }}@github.com/${{ github.repository }}
git push
echo "" >> $GITHUB_STEP_SUMMARY
echo "### ✅ Changes Committed" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "**Commit:** \`[ROLLBACK] $SERVICES - initiated by ${{ github.actor }}\`" >> $GITHUB_STEP_SUMMARY
fi
# ═══════════════════════════════════════════════════════════════════════════
# STEP 7: Post-rollback actions (notify, create issue)
# ═══════════════════════════════════════════════════════════════════════════
step7-post-rollback:
name: Post-Rollback
runs-on: ubuntu-latest
needs: [step1-validate, step4-determine-target, step6-execute]
if: always()
steps:
- name: Create tracking issue
if: needs.step6-execute.outputs.rollback_executed == 'true'
uses: actions/github-script@v7
with:
script: |
const services = '${{ needs.step1-validate.outputs.services }}'.split(' ').join(', ');
const rollbackType = '${{ github.event.inputs.rollback_type }}';
const specificTag = '${{ github.event.inputs.specific_tag }}' || 'N/A';
await github.rest.issues.create({
owner: context.repo.owner,
repo: context.repo.repo,
title: `[ROLLBACK] Production rollback - ${services} - ${new Date().toISOString().split('T')[0]}`,
body: `## Production Rollback Executed
### Details
| Field | Value |
|-------|-------|
| **Services** | ${services} |
| **Initiated by** | @${{ github.actor }} |
| **Run ID** | ${{ github.run_id }} |
| **Rollback Type** | ${rollbackType} |
| **Specific Tag** | ${specificTag} |
| **Timestamp** | ${new Date().toISOString()} |
### Required Actions
- [ ] Investigate root cause of the issue that triggered rollback
- [ ] Document findings in this issue
- [ ] Create fix PR with proper testing
- [ ] Re-deploy after fix is validated in staging
- [ ] Close this issue after re-deployment is successful
### Links
- [Workflow Run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
- [ArgoCD Dashboard](https://argocd.aurora-glam.com)
- [Production Site](https://aurora-glam.com)
---
_This issue was auto-generated by the rollback workflow._
`,
labels: ['rollback', 'production', 'urgent', 'incident']
});
- name: Final summary
run: |
echo "## Rollback Complete" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Next Steps" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "1. **ArgoCD** will auto-sync within 3 minutes" >> $GITHUB_STEP_SUMMARY
echo "2. **Monitor** the rollout at [ArgoCD Dashboard](https://argocd.aurora-glam.com)" >> $GITHUB_STEP_SUMMARY
echo "3. **Verify** the application at [https://aurora-glam.com](https://aurora-glam.com)" >> $GITHUB_STEP_SUMMARY
echo "4. **Investigate** the root cause (see GitHub Issue created)" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Verification Commands" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY
echo "# Check pod status" >> $GITHUB_STEP_SUMMARY
echo "kubectl get pods -n production" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "# Check deployment image" >> $GITHUB_STEP_SUMMARY
echo "kubectl get deployment <service-name> -n production -o jsonpath='{.spec.template.spec.containers[0].image}'" >> $GITHUB_STEP_SUMMARY
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY