High Availability & Disaster Recovery

                           Global Load Balancer (Route53 / Cloud DNS)
                                         |
                     +-------------------+-------------------+
                     |                                       |
              Region: US-East                         Region: EU-West
                     |                                       |
           +--------+--------+                     +--------+--------+
           |                 |                     |                 |
      CDE Control       Kubernetes            CDE Control       Kubernetes
        Plane             Cluster               Plane             Cluster
      (3 replicas)      (Worker Nodes)        (3 replicas)      (Worker Nodes)
           |                 |                     |                 |
           +--------+--------+                     +--------+--------+
                    |                                       |
             PostgreSQL HA                           PostgreSQL HA
            (RDS Multi-AZ)                          (RDS Multi-AZ)
                    |                                       |
                    +------- Cross-Region Replication ------+
                                (Async, RPO: ~1 min)

   Developer Workspaces:          Developer Workspaces:
   - us-east-1a (AZ-1)           - eu-west-1a (AZ-1)
   - us-east-1b (AZ-2)           - eu-west-1b (AZ-2)
   - us-east-1c (AZ-3)           - eu-west-1c (AZ-3)

#!/bin/bash
# CDE Regional Failover Script

set -euo pipefail

PRIMARY_REGION="us-east-1"
DR_REGION="eu-west-1"
HEALTH_ENDPOINT="https://cde.company.com/api/v2/health"
SLACK_WEBHOOK="https://hooks.slack.com/services/xxx"

# Check primary health
check_health() {
    local status=$(curl -s -o /dev/null -w "%{http_code}" $HEALTH_ENDPOINT --max-time 10)
    [[ "$status" == "200" ]]
}

# Notify team
notify() {
    curl -X POST $SLACK_WEBHOOK \
        -H 'Content-Type: application/json' \
        -d "{\"text\": \"$1\"}"
}

# Failover database
failover_db() {
    echo "Promoting DR database to primary..."
    aws rds promote-read-replica \
        --db-instance-identifier cde-db-dr \
        --region $DR_REGION
}

# Update DNS
update_dns() {
    echo "Updating DNS to DR region..."
    aws route53 change-resource-record-sets \
        --hosted-zone-id $HOSTED_ZONE_ID \
        --change-batch file://failover-dns.json
}

# Main failover logic
main() {
    if ! check_health; then
        notify ":rotating_light: Primary CDE region unhealthy - initiating failover"

        failover_db
        update_dns

        notify ":white_check_mark: Failover complete - CDE now running in $DR_REGION"
    else
        echo "Primary region healthy - no action needed"
    fi
}

main "$@"